From b47bd60951d20a589c985ad43a30eca59f6a692f Mon Sep 17 00:00:00 2001 From: Matthew Thomas <25988124+Ma11hewThomas@users.noreply.github.com> Date: Sat, 10 Oct 2026 07:45:46 +0100 Subject: [PATCH 1/3] Add optional identity for the final test attempt --- CHANGELOG.md | 8 + examples/merged-attempts.json | 42 +++ examples/raw-final-attempt.json | 31 ++ examples/raw-initial-attempt.json | 31 ++ schema/ctrf.schema.json | 5 + spec/ctrf.md | 156 ++++++++- tests/ctrf.test.json | 15 + tests/normative/final-attempt-id.test.json | 365 +++++++++++++++++++++ 8 files changed, 648 insertions(+), 5 deletions(-) create mode 100644 examples/merged-attempts.json create mode 100644 examples/raw-final-attempt.json create mode 100644 examples/raw-initial-attempt.json create mode 100644 tests/normative/final-attempt-id.test.json diff --git a/CHANGELOG.md b/CHANGELOG.md index b8a1a68..2b7a623 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.2.0] - 2026-10-10 + +### Added + +- Optional Test-level `attemptId` for final-attempt correlation in specification + 0.2.0, with preservation across raw/merged/history representations. Existing + retry semantics and reports without attempt identity remain supported (#66). + ## [0.1.0] - 2026-10-03 ### Changed diff --git a/examples/merged-attempts.json b/examples/merged-attempts.json new file mode 100644 index 0000000..08137a5 --- /dev/null +++ b/examples/merged-attempts.json @@ -0,0 +1,42 @@ +{ + "reportFormat": "CTRF", + "specVersion": "0.2.0", + "runId": "run-login-1", + "results": { + "tool": { + "name": "coordinated-runner" + }, + "summary": { + "tests": 1, + "passed": 1, + "failed": 0, + "skipped": 0, + "pending": 0, + "other": 0, + "start": 1000, + "stop": 1250, + "flaky": 1 + }, + "tests": [ + { + "name": "login(alice)", + "testId": "login/alice", + "executionId": "login-alice-run-1", + "attemptId": "login-alice-attempt-2", + "status": "passed", + "duration": 50, + "retries": 1, + "flaky": true, + "retryAttempts": [ + { + "attempt": 1, + "attemptId": "login-alice-attempt-1", + "status": "failed", + "duration": 100 + } + ] + } + ] + }, + "reportId": "33333333-3333-4333-8333-333333333333" +} diff --git a/examples/raw-final-attempt.json b/examples/raw-final-attempt.json new file mode 100644 index 0000000..09d91f7 --- /dev/null +++ b/examples/raw-final-attempt.json @@ -0,0 +1,31 @@ +{ + "reportFormat": "CTRF", + "specVersion": "0.2.0", + "runId": "run-login-1", + "results": { + "tool": { + "name": "coordinated-runner" + }, + "summary": { + "tests": 1, + "passed": 1, + "failed": 0, + "skipped": 0, + "pending": 0, + "other": 0, + "start": 1200, + "stop": 1250 + }, + "tests": [ + { + "name": "login(alice)", + "testId": "login/alice", + "executionId": "login-alice-run-1", + "attemptId": "login-alice-attempt-2", + "status": "passed", + "duration": 50 + } + ] + }, + "reportId": "22222222-2222-4222-8222-222222222222" +} diff --git a/examples/raw-initial-attempt.json b/examples/raw-initial-attempt.json new file mode 100644 index 0000000..b7836dc --- /dev/null +++ b/examples/raw-initial-attempt.json @@ -0,0 +1,31 @@ +{ + "reportFormat": "CTRF", + "specVersion": "0.2.0", + "runId": "run-login-1", + "results": { + "tool": { + "name": "coordinated-runner" + }, + "summary": { + "tests": 1, + "passed": 0, + "failed": 1, + "skipped": 0, + "pending": 0, + "other": 0, + "start": 1000, + "stop": 1100 + }, + "tests": [ + { + "name": "login(alice)", + "testId": "login/alice", + "executionId": "login-alice-run-1", + "attemptId": "login-alice-attempt-1", + "status": "failed", + "duration": 100 + } + ] + }, + "reportId": "11111111-1111-4111-8111-111111111111" +} diff --git a/schema/ctrf.schema.json b/schema/ctrf.schema.json index 85bd935..a774029 100644 --- a/schema/ctrf.schema.json +++ b/schema/ctrf.schema.json @@ -153,6 +153,11 @@ "type": "string", "minLength": 1 }, + "attemptId": { + "description": "Unique identifier for the final attempt represented by this test. Preserve across raw and merged representations of the same attempt. UUID recommended", + "type": "string", + "minLength": 1 + }, "name": { "description": "Name or title of the test case", "type": "string", diff --git a/spec/ctrf.md b/spec/ctrf.md index 000d6d5..79ac9ad 100644 --- a/spec/ctrf.md +++ b/spec/ctrf.md @@ -2,10 +2,10 @@ ## Common Test Report Format -**Version:** 0.1.0 +**Version:** 0.2.0 (This version corresponds directly to the CTRF `specVersion` field.) -**Date:** 2026-10-03 +**Date:** 2026-10-08 **Status:** Working Draft @@ -441,7 +441,7 @@ Each layer addresses a distinct identification concern: 2. **Run**: `runId` identifies the logical test run. Multiple CTRF documents MAY share the same `runId` when they represent shards or partitions of a single coordinated execution. 3. **Test case**: `testId` identifies the logical test case within the producer's chosen scope. It SHOULD be deterministic and stable across runs within that scope, enabling cross-run analysis, trending, and flake detection. 4. **Execution**: `executionId` identifies the complete execution lifecycle of a test case within a run. An execution MAY include previous attempts represented in `retryAttempts` followed by the final attempt represented by the test object. -5. **Attempt**: `attemptId` identifies an individual attempt history entry within an execution. +5. **Attempt**: `attemptId` identifies an individual attempt within an execution, either the final attempt on the test object or an earlier attempt in `retryAttempts`. 6. **Attachment**: `attachmentId` identifies a specific attachment reference instance. 7. **Shard**: `shardId` labels the partition or shard that produced this document within a logical run. @@ -466,6 +466,8 @@ Each identity field has an intended uniqueness scope: Producers SHOULD NOT knowingly reuse identity values outside their intended scope. Producers SHOULD NOT emit duplicate identity values within a single CTRF document when those duplicates would make correlation ambiguous. +An attempt ID identifies the attempt, not its representation or containing document. Producers SHOULD preserve it when the same raw result is included as a merged final result or moved into `retryAttempts`. Distinct attempts within the same execution SHOULD have distinct attempt IDs, including the final attempt and all history entries. Retransmitting the same attempt or representing it in another document SHOULD retain its ID; this reuse does not identify a new attempt. Consumers SHOULD correlate using the enclosing execution identity and attempt ID together, and SHOULD NOT count repeated representations as additional attempts. An ID alone does not establish a retry relationship or resolve conflicting result payloads. + #### 4.9.2. Stability Identity fields differ in whether their values persist across runs or are unique to a single emission: @@ -923,6 +925,22 @@ If present, it MUST be a non-empty string. UUID is RECOMMENDED. `executionId` SHOULD be unique across executions and SHOULD NOT be reused across runs. +### 9.3.1. `attemptId` + +**Description:** +The identifier for the final attempt represented by this test object, including an initial attempt when no retry occurred. + +**Requirements:** +`attemptId` is OPTIONAL. +If present, it MUST be a non-empty string. +UUID is RECOMMENDED, consistently with history-entry `attemptId` (Section 11.2). +Consumers MUST treat the value as opaque. +The uniqueness and preservation guidance in Section 4.9.1 applies across final and history representations. + +This property is introduced in CTRF 0.2.0. Reports emitting it MUST declare a `specVersion` supporting it; the 0.1.0 schema rejects it. Until the new schema is released, producers MAY retain an attempt key under their own namespace in `extra`. Legacy inputs without attempt identity MAY omit `attemptId`; consumers MUST NOT infer it from `executionId`, which identifies the complete lifecycle rather than one attempt. + +Adding `attemptId` does not change retry semantics: the final attempt stays on the test object, earlier attempts stay in `retryAttempts`, and `retries` retains its existing meaning. There is no new all-attempts array. + ### 9.4. `name` **Description:** @@ -1445,7 +1463,7 @@ A unique identifier for this individual attempt. `attemptId` is OPTIONAL. If present, it MUST be a non-empty string. UUID is RECOMMENDED. -`attemptId` SHOULD be unique within the enclosing execution. +`attemptId` SHOULD be unique within the enclosing execution, including the final attempt (Section 9.3.1). The preservation and retransmission guidance in Section 4.9.1 also applies to history entries. ### 11.3. `status` @@ -1971,7 +1989,7 @@ It is the ONLY permitted extension point within the baseline object. CTRF follows **Semantic Versioning**. -This document defines CTRF version `0.1.0`. +This document defines CTRF version `0.2.0`. Before CTRF `1.0.0`: @@ -2232,6 +2250,11 @@ to this specification. "type": "string", "minLength": 1 }, + "attemptId": { + "description": "Unique identifier for the final attempt represented by this test. Preserve across raw and merged representations of the same attempt. UUID recommended", + "type": "string", + "minLength": 1 + }, "name": { "description": "Name or title of the test case", "type": "string", @@ -3520,3 +3543,126 @@ The `ctrf.` and `ctrf/` namespace prefixes are reserved for CTRF-defined extensi } } ``` + +### D.8. Correlating raw and merged attempts + +These three documents describe one coordinated lifecycle. Each raw document represents the final result available in that process; the orchestrator knows that the second process is a retry. The merged document preserves both attempt IDs, keeps the successful final attempt on Test, and records the earlier failed attempt as history. `executionId` and `testId` remain shared; each document has a distinct `reportId`. This example does not prescribe a merge algorithm or introduce the separate provenance proposal. + +**raw-initial-attempt.json** + +```json +{ + "reportFormat": "CTRF", + "specVersion": "0.2.0", + "runId": "run-login-1", + "results": { + "tool": { + "name": "coordinated-runner" + }, + "summary": { + "tests": 1, + "passed": 0, + "failed": 1, + "skipped": 0, + "pending": 0, + "other": 0, + "start": 1000, + "stop": 1100 + }, + "tests": [ + { + "name": "login(alice)", + "testId": "login/alice", + "executionId": "login-alice-run-1", + "attemptId": "login-alice-attempt-1", + "status": "failed", + "duration": 100 + } + ] + }, + "reportId": "11111111-1111-4111-8111-111111111111" +} +``` + +**raw-final-attempt.json** + +```json +{ + "reportFormat": "CTRF", + "specVersion": "0.2.0", + "runId": "run-login-1", + "results": { + "tool": { + "name": "coordinated-runner" + }, + "summary": { + "tests": 1, + "passed": 1, + "failed": 0, + "skipped": 0, + "pending": 0, + "other": 0, + "start": 1200, + "stop": 1250 + }, + "tests": [ + { + "name": "login(alice)", + "testId": "login/alice", + "executionId": "login-alice-run-1", + "attemptId": "login-alice-attempt-2", + "status": "passed", + "duration": 50 + } + ] + }, + "reportId": "22222222-2222-4222-8222-222222222222" +} +``` + +**merged-attempts.json** + +```json +{ + "reportFormat": "CTRF", + "specVersion": "0.2.0", + "runId": "run-login-1", + "results": { + "tool": { + "name": "coordinated-runner" + }, + "summary": { + "tests": 1, + "passed": 1, + "failed": 0, + "skipped": 0, + "pending": 0, + "other": 0, + "start": 1000, + "stop": 1250, + "flaky": 1 + }, + "tests": [ + { + "name": "login(alice)", + "testId": "login/alice", + "executionId": "login-alice-run-1", + "attemptId": "login-alice-attempt-2", + "status": "passed", + "duration": 50, + "retries": 1, + "flaky": true, + "retryAttempts": [ + { + "attempt": 1, + "attemptId": "login-alice-attempt-1", + "status": "failed", + "duration": 100 + } + ] + } + ] + }, + "reportId": "33333333-3333-4333-8333-333333333333" +} +``` diff --git a/tests/ctrf.test.json b/tests/ctrf.test.json index 295a9fb..95a6ab3 100644 --- a/tests/ctrf.test.json +++ b/tests/ctrf.test.json @@ -188,6 +188,21 @@ "tests": [{ "name": "test", "status": "passed", "duration": 100 }] } } + }, + { + "description": "raw-initial-attempt.json", + "valid": true, + "dataPath": "../examples/raw-initial-attempt.json" + }, + { + "description": "raw-final-attempt.json", + "valid": true, + "dataPath": "../examples/raw-final-attempt.json" + }, + { + "description": "merged-attempts.json", + "valid": true, + "dataPath": "../examples/merged-attempts.json" } ] } diff --git a/tests/normative/final-attempt-id.test.json b/tests/normative/final-attempt-id.test.json new file mode 100644 index 0000000..8942105 --- /dev/null +++ b/tests/normative/final-attempt-id.test.json @@ -0,0 +1,365 @@ +{ + "target": "../../schema/ctrf.schema.json", + "tests": [ + { + "description": "raw-initial-attempt.json", + "valid": true, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.2.0", + "runId": "run-login-1", + "results": { + "tool": { + "name": "coordinated-runner" + }, + "summary": { + "tests": 1, + "passed": 0, + "failed": 1, + "skipped": 0, + "pending": 0, + "other": 0, + "start": 1000, + "stop": 1100 + }, + "tests": [ + { + "name": "login(alice)", + "testId": "login/alice", + "executionId": "login-alice-run-1", + "attemptId": "login-alice-attempt-1", + "status": "failed", + "duration": 100 + } + ] + }, + "reportId": "11111111-1111-4111-8111-111111111111" + } + }, + { + "description": "raw-final-attempt.json", + "valid": true, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.2.0", + "runId": "run-login-1", + "results": { + "tool": { + "name": "coordinated-runner" + }, + "summary": { + "tests": 1, + "passed": 1, + "failed": 0, + "skipped": 0, + "pending": 0, + "other": 0, + "start": 1200, + "stop": 1250 + }, + "tests": [ + { + "name": "login(alice)", + "testId": "login/alice", + "executionId": "login-alice-run-1", + "attemptId": "login-alice-attempt-2", + "status": "passed", + "duration": 50 + } + ] + }, + "reportId": "22222222-2222-4222-8222-222222222222" + } + }, + { + "description": "merged-attempts.json", + "valid": true, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.2.0", + "runId": "run-login-1", + "results": { + "tool": { + "name": "coordinated-runner" + }, + "summary": { + "tests": 1, + "passed": 1, + "failed": 0, + "skipped": 0, + "pending": 0, + "other": 0, + "start": 1000, + "stop": 1250, + "flaky": 1 + }, + "tests": [ + { + "name": "login(alice)", + "testId": "login/alice", + "executionId": "login-alice-run-1", + "attemptId": "login-alice-attempt-2", + "status": "passed", + "duration": 50, + "retries": 1, + "flaky": true, + "retryAttempts": [ + { + "attempt": 1, + "attemptId": "login-alice-attempt-1", + "status": "failed", + "duration": 100 + } + ] + } + ] + }, + "reportId": "33333333-3333-4333-8333-333333333333" + } + }, + { + "description": "Reject final attemptId ''", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.2.0", + "runId": "run-login-1", + "results": { + "tool": { + "name": "coordinated-runner" + }, + "summary": { + "tests": 1, + "passed": 0, + "failed": 1, + "skipped": 0, + "pending": 0, + "other": 0, + "start": 1000, + "stop": 1100 + }, + "tests": [ + { + "name": "login(alice)", + "testId": "login/alice", + "executionId": "login-alice-run-1", + "attemptId": "", + "status": "failed", + "duration": 100 + } + ] + }, + "reportId": "11111111-1111-4111-8111-111111111111" + } + }, + { + "description": "Reject final attemptId None", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.2.0", + "runId": "run-login-1", + "results": { + "tool": { + "name": "coordinated-runner" + }, + "summary": { + "tests": 1, + "passed": 0, + "failed": 1, + "skipped": 0, + "pending": 0, + "other": 0, + "start": 1000, + "stop": 1100 + }, + "tests": [ + { + "name": "login(alice)", + "testId": "login/alice", + "executionId": "login-alice-run-1", + "attemptId": null, + "status": "failed", + "duration": 100 + } + ] + }, + "reportId": "11111111-1111-4111-8111-111111111111" + } + }, + { + "description": "Reject final attemptId 1", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.2.0", + "runId": "run-login-1", + "results": { + "tool": { + "name": "coordinated-runner" + }, + "summary": { + "tests": 1, + "passed": 0, + "failed": 1, + "skipped": 0, + "pending": 0, + "other": 0, + "start": 1000, + "stop": 1100 + }, + "tests": [ + { + "name": "login(alice)", + "testId": "login/alice", + "executionId": "login-alice-run-1", + "attemptId": 1, + "status": "failed", + "duration": 100 + } + ] + }, + "reportId": "11111111-1111-4111-8111-111111111111" + } + }, + { + "description": "Reject final attemptId True", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.2.0", + "runId": "run-login-1", + "results": { + "tool": { + "name": "coordinated-runner" + }, + "summary": { + "tests": 1, + "passed": 0, + "failed": 1, + "skipped": 0, + "pending": 0, + "other": 0, + "start": 1000, + "stop": 1100 + }, + "tests": [ + { + "name": "login(alice)", + "testId": "login/alice", + "executionId": "login-alice-run-1", + "attemptId": true, + "status": "failed", + "duration": 100 + } + ] + }, + "reportId": "11111111-1111-4111-8111-111111111111" + } + }, + { + "description": "Reject final attemptId []", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.2.0", + "runId": "run-login-1", + "results": { + "tool": { + "name": "coordinated-runner" + }, + "summary": { + "tests": 1, + "passed": 0, + "failed": 1, + "skipped": 0, + "pending": 0, + "other": 0, + "start": 1000, + "stop": 1100 + }, + "tests": [ + { + "name": "login(alice)", + "testId": "login/alice", + "executionId": "login-alice-run-1", + "attemptId": [], + "status": "failed", + "duration": 100 + } + ] + }, + "reportId": "11111111-1111-4111-8111-111111111111" + } + }, + { + "description": "Reject final attemptId {}", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.2.0", + "runId": "run-login-1", + "results": { + "tool": { + "name": "coordinated-runner" + }, + "summary": { + "tests": 1, + "passed": 0, + "failed": 1, + "skipped": 0, + "pending": 0, + "other": 0, + "start": 1000, + "stop": 1100 + }, + "tests": [ + { + "name": "login(alice)", + "testId": "login/alice", + "executionId": "login-alice-run-1", + "attemptId": {}, + "status": "failed", + "duration": 100 + } + ] + }, + "reportId": "11111111-1111-4111-8111-111111111111" + } + }, + { + "description": "Legacy result without attemptId remains valid", + "valid": true, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.2.0", + "runId": "run-login-1", + "results": { + "tool": { + "name": "coordinated-runner" + }, + "summary": { + "tests": 1, + "passed": 0, + "failed": 1, + "skipped": 0, + "pending": 0, + "other": 0, + "start": 1000, + "stop": 1100 + }, + "tests": [ + { + "name": "login(alice)", + "testId": "login/alice", + "executionId": "login-alice-run-1", + "status": "failed", + "duration": 100 + } + ] + }, + "reportId": "11111111-1111-4111-8111-111111111111" + } + } + ] +} From 33de7a925a819bf80f43d846e89746e33b67e406 Mon Sep 17 00:00:00 2001 From: Matthew Thomas <25988124+Ma11hewThomas@users.noreply.github.com> Date: Sat, 10 Oct 2026 07:58:30 +0100 Subject: [PATCH 2/3] Keep attempt identity change focused on the test object --- CHANGELOG.md | 6 +- examples/merged-attempts.json | 42 --- examples/raw-final-attempt.json | 31 --- examples/raw-initial-attempt.json | 31 --- examples/test-attempt-id.json | 27 ++ schema/ctrf.schema.json | 2 +- spec/ctrf.md | 126 ++------- tests/ctrf.test.json | 14 +- tests/normative/final-attempt-id.test.json | 289 +++++++-------------- 9 files changed, 137 insertions(+), 431 deletions(-) delete mode 100644 examples/merged-attempts.json delete mode 100644 examples/raw-final-attempt.json delete mode 100644 examples/raw-initial-attempt.json create mode 100644 examples/test-attempt-id.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 2b7a623..c0fb2d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,9 +11,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- Optional Test-level `attemptId` for final-attempt correlation in specification - 0.2.0, with preservation across raw/merged/history representations. Existing - retry semantics and reports without attempt identity remain supported (#66). +- Optional Test-level `attemptId` in specification 0.2.0, using the same + identifier contract as retry history. Reports without attempt identity remain + supported (#66). ## [0.1.0] - 2026-10-03 diff --git a/examples/merged-attempts.json b/examples/merged-attempts.json deleted file mode 100644 index 08137a5..0000000 --- a/examples/merged-attempts.json +++ /dev/null @@ -1,42 +0,0 @@ -{ - "reportFormat": "CTRF", - "specVersion": "0.2.0", - "runId": "run-login-1", - "results": { - "tool": { - "name": "coordinated-runner" - }, - "summary": { - "tests": 1, - "passed": 1, - "failed": 0, - "skipped": 0, - "pending": 0, - "other": 0, - "start": 1000, - "stop": 1250, - "flaky": 1 - }, - "tests": [ - { - "name": "login(alice)", - "testId": "login/alice", - "executionId": "login-alice-run-1", - "attemptId": "login-alice-attempt-2", - "status": "passed", - "duration": 50, - "retries": 1, - "flaky": true, - "retryAttempts": [ - { - "attempt": 1, - "attemptId": "login-alice-attempt-1", - "status": "failed", - "duration": 100 - } - ] - } - ] - }, - "reportId": "33333333-3333-4333-8333-333333333333" -} diff --git a/examples/raw-final-attempt.json b/examples/raw-final-attempt.json deleted file mode 100644 index 09d91f7..0000000 --- a/examples/raw-final-attempt.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "reportFormat": "CTRF", - "specVersion": "0.2.0", - "runId": "run-login-1", - "results": { - "tool": { - "name": "coordinated-runner" - }, - "summary": { - "tests": 1, - "passed": 1, - "failed": 0, - "skipped": 0, - "pending": 0, - "other": 0, - "start": 1200, - "stop": 1250 - }, - "tests": [ - { - "name": "login(alice)", - "testId": "login/alice", - "executionId": "login-alice-run-1", - "attemptId": "login-alice-attempt-2", - "status": "passed", - "duration": 50 - } - ] - }, - "reportId": "22222222-2222-4222-8222-222222222222" -} diff --git a/examples/raw-initial-attempt.json b/examples/raw-initial-attempt.json deleted file mode 100644 index b7836dc..0000000 --- a/examples/raw-initial-attempt.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "reportFormat": "CTRF", - "specVersion": "0.2.0", - "runId": "run-login-1", - "results": { - "tool": { - "name": "coordinated-runner" - }, - "summary": { - "tests": 1, - "passed": 0, - "failed": 1, - "skipped": 0, - "pending": 0, - "other": 0, - "start": 1000, - "stop": 1100 - }, - "tests": [ - { - "name": "login(alice)", - "testId": "login/alice", - "executionId": "login-alice-run-1", - "attemptId": "login-alice-attempt-1", - "status": "failed", - "duration": 100 - } - ] - }, - "reportId": "11111111-1111-4111-8111-111111111111" -} diff --git a/examples/test-attempt-id.json b/examples/test-attempt-id.json new file mode 100644 index 0000000..b66805b --- /dev/null +++ b/examples/test-attempt-id.json @@ -0,0 +1,27 @@ +{ + "reportFormat": "CTRF", + "specVersion": "0.2.0", + "results": { + "tool": { + "name": "example-runner" + }, + "summary": { + "tests": 1, + "passed": 1, + "failed": 0, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459201000 + }, + "tests": [ + { + "name": "should pass", + "status": "passed", + "duration": 100, + "attemptId": "a1b2c3d4-1111-4000-a000-000000000001" + } + ] + } +} diff --git a/schema/ctrf.schema.json b/schema/ctrf.schema.json index a774029..0135d5e 100644 --- a/schema/ctrf.schema.json +++ b/schema/ctrf.schema.json @@ -154,7 +154,7 @@ "minLength": 1 }, "attemptId": { - "description": "Unique identifier for the final attempt represented by this test. Preserve across raw and merged representations of the same attempt. UUID recommended", + "description": "Unique identifier for this individual attempt. UUID recommended", "type": "string", "minLength": 1 }, diff --git a/spec/ctrf.md b/spec/ctrf.md index 79ac9ad..f2cdad9 100644 --- a/spec/ctrf.md +++ b/spec/ctrf.md @@ -5,7 +5,7 @@ **Version:** 0.2.0 (This version corresponds directly to the CTRF `specVersion` field.) -**Date:** 2026-10-08 +**Date:** 2026-10-10 **Status:** Working Draft @@ -466,8 +466,6 @@ Each identity field has an intended uniqueness scope: Producers SHOULD NOT knowingly reuse identity values outside their intended scope. Producers SHOULD NOT emit duplicate identity values within a single CTRF document when those duplicates would make correlation ambiguous. -An attempt ID identifies the attempt, not its representation or containing document. Producers SHOULD preserve it when the same raw result is included as a merged final result or moved into `retryAttempts`. Distinct attempts within the same execution SHOULD have distinct attempt IDs, including the final attempt and all history entries. Retransmitting the same attempt or representing it in another document SHOULD retain its ID; this reuse does not identify a new attempt. Consumers SHOULD correlate using the enclosing execution identity and attempt ID together, and SHOULD NOT count repeated representations as additional attempts. An ID alone does not establish a retry relationship or resolve conflicting result payloads. - #### 4.9.2. Stability Identity fields differ in whether their values persist across runs or are unique to a single emission: @@ -928,18 +926,15 @@ UUID is RECOMMENDED. ### 9.3.1. `attemptId` **Description:** -The identifier for the final attempt represented by this test object, including an initial attempt when no retry occurred. +A unique identifier for the final attempt represented by this test object, including an initial attempt when no retry occurred. **Requirements:** `attemptId` is OPTIONAL. If present, it MUST be a non-empty string. -UUID is RECOMMENDED, consistently with history-entry `attemptId` (Section 11.2). -Consumers MUST treat the value as opaque. -The uniqueness and preservation guidance in Section 4.9.1 applies across final and history representations. - -This property is introduced in CTRF 0.2.0. Reports emitting it MUST declare a `specVersion` supporting it; the 0.1.0 schema rejects it. Until the new schema is released, producers MAY retain an attempt key under their own namespace in `extra`. Legacy inputs without attempt identity MAY omit `attemptId`; consumers MUST NOT infer it from `executionId`, which identifies the complete lifecycle rather than one attempt. +UUID is RECOMMENDED. +`attemptId` SHOULD be unique within the enclosing execution. -Adding `attemptId` does not change retry semantics: the final attempt stays on the test object, earlier attempts stay in `retryAttempts`, and `retries` retains its existing meaning. There is no new all-attempts array. +This property is introduced in CTRF 0.2.0 and uses the same identifier contract as `retryAttempts[].attemptId` (Section 11.2). ### 9.4. `name` @@ -1463,7 +1458,7 @@ A unique identifier for this individual attempt. `attemptId` is OPTIONAL. If present, it MUST be a non-empty string. UUID is RECOMMENDED. -`attemptId` SHOULD be unique within the enclosing execution, including the final attempt (Section 9.3.1). The preservation and retransmission guidance in Section 4.9.1 also applies to history entries. +`attemptId` SHOULD be unique within the enclosing execution. ### 11.3. `status` @@ -2251,7 +2246,7 @@ to this specification. "minLength": 1 }, "attemptId": { - "description": "Unique identifier for the final attempt represented by this test. Preserve across raw and merged representations of the same attempt. UUID recommended", + "description": "Unique identifier for this individual attempt. UUID recommended", "type": "string", "minLength": 1 }, @@ -3544,125 +3539,36 @@ The `ctrf.` and `ctrf/` namespace prefixes are reserved for CTRF-defined extensi } ``` -### D.8. Correlating raw and merged attempts - -These three documents describe one coordinated lifecycle. Each raw document represents the final result available in that process; the orchestrator knows that the second process is a retry. The merged document preserves both attempt IDs, keeps the successful final attempt on Test, and records the earlier failed attempt as history. `executionId` and `testId` remain shared; each document has a distinct `reportId`. This example does not prescribe a merge algorithm or introduce the separate provenance proposal. - -**raw-initial-attempt.json** - -```json -{ - "reportFormat": "CTRF", - "specVersion": "0.2.0", - "runId": "run-login-1", - "results": { - "tool": { - "name": "coordinated-runner" - }, - "summary": { - "tests": 1, - "passed": 0, - "failed": 1, - "skipped": 0, - "pending": 0, - "other": 0, - "start": 1000, - "stop": 1100 - }, - "tests": [ - { - "name": "login(alice)", - "testId": "login/alice", - "executionId": "login-alice-run-1", - "attemptId": "login-alice-attempt-1", - "status": "failed", - "duration": 100 - } - ] - }, - "reportId": "11111111-1111-4111-8111-111111111111" -} -``` +### D.8. Test Attempt Identifier -**raw-final-attempt.json** +This example gives the attempt represented by the test object an optional identifier. ```json { "reportFormat": "CTRF", "specVersion": "0.2.0", - "runId": "run-login-1", "results": { "tool": { - "name": "coordinated-runner" + "name": "example-runner" }, "summary": { "tests": 1, "passed": 1, "failed": 0, - "skipped": 0, "pending": 0, - "other": 0, - "start": 1200, - "stop": 1250 - }, - "tests": [ - { - "name": "login(alice)", - "testId": "login/alice", - "executionId": "login-alice-run-1", - "attemptId": "login-alice-attempt-2", - "status": "passed", - "duration": 50 - } - ] - }, - "reportId": "22222222-2222-4222-8222-222222222222" -} -``` - -**merged-attempts.json** - -```json -{ - "reportFormat": "CTRF", - "specVersion": "0.2.0", - "runId": "run-login-1", - "results": { - "tool": { - "name": "coordinated-runner" - }, - "summary": { - "tests": 1, - "passed": 1, - "failed": 0, "skipped": 0, - "pending": 0, "other": 0, - "start": 1000, - "stop": 1250, - "flaky": 1 + "start": 1609459200000, + "stop": 1609459201000 }, "tests": [ { - "name": "login(alice)", - "testId": "login/alice", - "executionId": "login-alice-run-1", - "attemptId": "login-alice-attempt-2", + "name": "should pass", "status": "passed", - "duration": 50, - "retries": 1, - "flaky": true, - "retryAttempts": [ - { - "attempt": 1, - "attemptId": "login-alice-attempt-1", - "status": "failed", - "duration": 100 - } - ] + "duration": 100, + "attemptId": "a1b2c3d4-1111-4000-a000-000000000001" } ] - }, - "reportId": "33333333-3333-4333-8333-333333333333" + } } ``` diff --git a/tests/ctrf.test.json b/tests/ctrf.test.json index 95a6ab3..e1c4269 100644 --- a/tests/ctrf.test.json +++ b/tests/ctrf.test.json @@ -190,19 +190,9 @@ } }, { - "description": "raw-initial-attempt.json", + "description": "test-attempt-id.json", "valid": true, - "dataPath": "../examples/raw-initial-attempt.json" - }, - { - "description": "raw-final-attempt.json", - "valid": true, - "dataPath": "../examples/raw-final-attempt.json" - }, - { - "description": "merged-attempts.json", - "valid": true, - "dataPath": "../examples/merged-attempts.json" + "dataPath": "../examples/test-attempt-id.json" } ] } diff --git a/tests/normative/final-attempt-id.test.json b/tests/normative/final-attempt-id.test.json index 8942105..9621e20 100644 --- a/tests/normative/final-attempt-id.test.json +++ b/tests/normative/final-attempt-id.test.json @@ -2,363 +2,250 @@ "target": "../../schema/ctrf.schema.json", "tests": [ { - "description": "raw-initial-attempt.json", + "description": "Accept non-empty Test.attemptId", "valid": true, "data": { "reportFormat": "CTRF", "specVersion": "0.2.0", - "runId": "run-login-1", "results": { "tool": { - "name": "coordinated-runner" - }, - "summary": { - "tests": 1, - "passed": 0, - "failed": 1, - "skipped": 0, - "pending": 0, - "other": 0, - "start": 1000, - "stop": 1100 - }, - "tests": [ - { - "name": "login(alice)", - "testId": "login/alice", - "executionId": "login-alice-run-1", - "attemptId": "login-alice-attempt-1", - "status": "failed", - "duration": 100 - } - ] - }, - "reportId": "11111111-1111-4111-8111-111111111111" - } - }, - { - "description": "raw-final-attempt.json", - "valid": true, - "data": { - "reportFormat": "CTRF", - "specVersion": "0.2.0", - "runId": "run-login-1", - "results": { - "tool": { - "name": "coordinated-runner" + "name": "example-runner" }, "summary": { "tests": 1, "passed": 1, "failed": 0, - "skipped": 0, "pending": 0, + "skipped": 0, "other": 0, - "start": 1200, - "stop": 1250 + "start": 1609459200000, + "stop": 1609459201000 }, "tests": [ { - "name": "login(alice)", - "testId": "login/alice", - "executionId": "login-alice-run-1", - "attemptId": "login-alice-attempt-2", + "name": "should pass", "status": "passed", - "duration": 50 + "duration": 100, + "attemptId": "native-attempt-1" } ] - }, - "reportId": "22222222-2222-4222-8222-222222222222" + } } }, { - "description": "merged-attempts.json", + "description": "Test.attemptId is optional", "valid": true, "data": { "reportFormat": "CTRF", "specVersion": "0.2.0", - "runId": "run-login-1", "results": { "tool": { - "name": "coordinated-runner" + "name": "example-runner" }, "summary": { "tests": 1, "passed": 1, "failed": 0, - "skipped": 0, "pending": 0, + "skipped": 0, "other": 0, - "start": 1000, - "stop": 1250, - "flaky": 1 + "start": 1609459200000, + "stop": 1609459201000 }, "tests": [ { - "name": "login(alice)", - "testId": "login/alice", - "executionId": "login-alice-run-1", - "attemptId": "login-alice-attempt-2", + "name": "should pass", "status": "passed", - "duration": 50, - "retries": 1, - "flaky": true, - "retryAttempts": [ - { - "attempt": 1, - "attemptId": "login-alice-attempt-1", - "status": "failed", - "duration": 100 - } - ] + "duration": 100 } ] - }, - "reportId": "33333333-3333-4333-8333-333333333333" + } } }, { - "description": "Reject final attemptId ''", + "description": "Reject Test.attemptId empty string", "valid": false, "data": { "reportFormat": "CTRF", "specVersion": "0.2.0", - "runId": "run-login-1", "results": { "tool": { - "name": "coordinated-runner" + "name": "example-runner" }, "summary": { "tests": 1, - "passed": 0, - "failed": 1, - "skipped": 0, + "passed": 1, + "failed": 0, "pending": 0, + "skipped": 0, "other": 0, - "start": 1000, - "stop": 1100 + "start": 1609459200000, + "stop": 1609459201000 }, "tests": [ { - "name": "login(alice)", - "testId": "login/alice", - "executionId": "login-alice-run-1", - "attemptId": "", - "status": "failed", - "duration": 100 + "name": "should pass", + "status": "passed", + "duration": 100, + "attemptId": "" } ] - }, - "reportId": "11111111-1111-4111-8111-111111111111" + } } }, { - "description": "Reject final attemptId None", + "description": "Reject Test.attemptId null", "valid": false, "data": { "reportFormat": "CTRF", "specVersion": "0.2.0", - "runId": "run-login-1", "results": { "tool": { - "name": "coordinated-runner" + "name": "example-runner" }, "summary": { "tests": 1, - "passed": 0, - "failed": 1, - "skipped": 0, + "passed": 1, + "failed": 0, "pending": 0, + "skipped": 0, "other": 0, - "start": 1000, - "stop": 1100 + "start": 1609459200000, + "stop": 1609459201000 }, "tests": [ { - "name": "login(alice)", - "testId": "login/alice", - "executionId": "login-alice-run-1", - "attemptId": null, - "status": "failed", - "duration": 100 + "name": "should pass", + "status": "passed", + "duration": 100, + "attemptId": null } ] - }, - "reportId": "11111111-1111-4111-8111-111111111111" + } } }, { - "description": "Reject final attemptId 1", + "description": "Reject Test.attemptId number", "valid": false, "data": { "reportFormat": "CTRF", "specVersion": "0.2.0", - "runId": "run-login-1", "results": { "tool": { - "name": "coordinated-runner" + "name": "example-runner" }, "summary": { "tests": 1, - "passed": 0, - "failed": 1, - "skipped": 0, + "passed": 1, + "failed": 0, "pending": 0, + "skipped": 0, "other": 0, - "start": 1000, - "stop": 1100 + "start": 1609459200000, + "stop": 1609459201000 }, "tests": [ { - "name": "login(alice)", - "testId": "login/alice", - "executionId": "login-alice-run-1", - "attemptId": 1, - "status": "failed", - "duration": 100 + "name": "should pass", + "status": "passed", + "duration": 100, + "attemptId": 1 } ] - }, - "reportId": "11111111-1111-4111-8111-111111111111" + } } }, { - "description": "Reject final attemptId True", + "description": "Reject Test.attemptId boolean", "valid": false, "data": { "reportFormat": "CTRF", "specVersion": "0.2.0", - "runId": "run-login-1", "results": { "tool": { - "name": "coordinated-runner" + "name": "example-runner" }, "summary": { "tests": 1, - "passed": 0, - "failed": 1, - "skipped": 0, + "passed": 1, + "failed": 0, "pending": 0, + "skipped": 0, "other": 0, - "start": 1000, - "stop": 1100 + "start": 1609459200000, + "stop": 1609459201000 }, "tests": [ { - "name": "login(alice)", - "testId": "login/alice", - "executionId": "login-alice-run-1", - "attemptId": true, - "status": "failed", - "duration": 100 + "name": "should pass", + "status": "passed", + "duration": 100, + "attemptId": true } ] - }, - "reportId": "11111111-1111-4111-8111-111111111111" + } } }, { - "description": "Reject final attemptId []", + "description": "Reject Test.attemptId array", "valid": false, "data": { "reportFormat": "CTRF", "specVersion": "0.2.0", - "runId": "run-login-1", "results": { "tool": { - "name": "coordinated-runner" + "name": "example-runner" }, "summary": { "tests": 1, - "passed": 0, - "failed": 1, - "skipped": 0, + "passed": 1, + "failed": 0, "pending": 0, + "skipped": 0, "other": 0, - "start": 1000, - "stop": 1100 + "start": 1609459200000, + "stop": 1609459201000 }, "tests": [ { - "name": "login(alice)", - "testId": "login/alice", - "executionId": "login-alice-run-1", - "attemptId": [], - "status": "failed", - "duration": 100 + "name": "should pass", + "status": "passed", + "duration": 100, + "attemptId": [] } ] - }, - "reportId": "11111111-1111-4111-8111-111111111111" + } } }, { - "description": "Reject final attemptId {}", + "description": "Reject Test.attemptId object", "valid": false, "data": { "reportFormat": "CTRF", "specVersion": "0.2.0", - "runId": "run-login-1", "results": { "tool": { - "name": "coordinated-runner" + "name": "example-runner" }, "summary": { "tests": 1, - "passed": 0, - "failed": 1, - "skipped": 0, + "passed": 1, + "failed": 0, "pending": 0, - "other": 0, - "start": 1000, - "stop": 1100 - }, - "tests": [ - { - "name": "login(alice)", - "testId": "login/alice", - "executionId": "login-alice-run-1", - "attemptId": {}, - "status": "failed", - "duration": 100 - } - ] - }, - "reportId": "11111111-1111-4111-8111-111111111111" - } - }, - { - "description": "Legacy result without attemptId remains valid", - "valid": true, - "data": { - "reportFormat": "CTRF", - "specVersion": "0.2.0", - "runId": "run-login-1", - "results": { - "tool": { - "name": "coordinated-runner" - }, - "summary": { - "tests": 1, - "passed": 0, - "failed": 1, "skipped": 0, - "pending": 0, "other": 0, - "start": 1000, - "stop": 1100 + "start": 1609459200000, + "stop": 1609459201000 }, "tests": [ { - "name": "login(alice)", - "testId": "login/alice", - "executionId": "login-alice-run-1", - "status": "failed", - "duration": 100 + "name": "should pass", + "status": "passed", + "duration": 100, + "attemptId": {} } ] - }, - "reportId": "11111111-1111-4111-8111-111111111111" + } } } ] From 3ce13bec24858335d99d8909afd88fa742a29008 Mon Sep 17 00:00:00 2001 From: Matthew Thomas <25988124+Ma11hewThomas@users.noreply.github.com> Date: Sat, 10 Oct 2026 08:22:26 +0100 Subject: [PATCH 3/3] Extend existing attempt identity examples and tests --- examples/comprehensive.json | 3 +- examples/test-attempt-id.json | 27 -- spec/ctrf.md | 39 +-- tests/ctrf.test.json | 5 - tests/normative/final-attempt-id.test.json | 252 ------------------- tests/normative/string-constraints.test.json | 31 +++ tests/normative/type-validation.test.json | 155 ++++++++++++ tests/normative/valid-documents.test.json | 4 +- 8 files changed, 192 insertions(+), 324 deletions(-) delete mode 100644 examples/test-attempt-id.json delete mode 100644 tests/normative/final-attempt-id.test.json diff --git a/examples/comprehensive.json b/examples/comprehensive.json index 70f0431..aec49ce 100644 --- a/examples/comprehensive.json +++ b/examples/comprehensive.json @@ -1,6 +1,6 @@ { "reportFormat": "CTRF", - "specVersion": "0.1.0", + "specVersion": "0.2.0", "reportId": "9d2c6a10-3f7a-4e22-9a8f-1a2b3c4d5e6f", "runId": "run-2025-11-24-pr-123", "timestamp": "2025-11-24T12:00:00Z", @@ -76,6 +76,7 @@ "id": "550e8400-e29b-41d4-a716-446655440001", "testId": "checkout-user-can-checkout", "executionId": "2cf7e0b4-ea0c-4ef4-a18a-b9d6f6f0e101", + "attemptId": "48c3f90b-95a9-4c13-a42f-0888bb0b1002", "name": "user can checkout", "status": "passed", "duration": 5200, diff --git a/examples/test-attempt-id.json b/examples/test-attempt-id.json deleted file mode 100644 index b66805b..0000000 --- a/examples/test-attempt-id.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "reportFormat": "CTRF", - "specVersion": "0.2.0", - "results": { - "tool": { - "name": "example-runner" - }, - "summary": { - "tests": 1, - "passed": 1, - "failed": 0, - "pending": 0, - "skipped": 0, - "other": 0, - "start": 1609459200000, - "stop": 1609459201000 - }, - "tests": [ - { - "name": "should pass", - "status": "passed", - "duration": 100, - "attemptId": "a1b2c3d4-1111-4000-a000-000000000001" - } - ] - } -} diff --git a/spec/ctrf.md b/spec/ctrf.md index f2cdad9..25e1399 100644 --- a/spec/ctrf.md +++ b/spec/ctrf.md @@ -934,8 +934,6 @@ If present, it MUST be a non-empty string. UUID is RECOMMENDED. `attemptId` SHOULD be unique within the enclosing execution. -This property is introduced in CTRF 0.2.0 and uses the same identifier contract as `retryAttempts[].attemptId` (Section 11.2). - ### 9.4. `name` **Description:** @@ -3273,7 +3271,7 @@ It includes: ```json title="Comprehensive CTRF document" { "reportFormat": "CTRF", - "specVersion": "0.1.0", + "specVersion": "0.2.0", "reportId": "9d2c6a10-3f7a-4e22-9a8f-1a2b3c4d5e6f", "runId": "run-20251124-e2e-staging", "timestamp": "2025-11-24T12:00:00Z", @@ -3317,6 +3315,7 @@ It includes: "id": "c6c9f8c0-8b5e-5f7a-9e2d-3c91f8a7d7c1", "testId": "c6c9f8c0-8b5e-5f7a-9e2d-3c91f8a7d7c1", "executionId": "a1b2c3d4-1111-4000-a000-000000000001", + "attemptId": "f1e2d3c4-2222-4000-b000-000000000002", "name": "user can log in", "suite": ["auth", "login"], "filePath": "tests/auth/login.test.js", @@ -3538,37 +3537,3 @@ The `ctrf.` and `ctrf/` namespace prefixes are reserved for CTRF-defined extensi } } ``` - -### D.8. Test Attempt Identifier - -This example gives the attempt represented by the test object an optional identifier. - -```json -{ - "reportFormat": "CTRF", - "specVersion": "0.2.0", - "results": { - "tool": { - "name": "example-runner" - }, - "summary": { - "tests": 1, - "passed": 1, - "failed": 0, - "pending": 0, - "skipped": 0, - "other": 0, - "start": 1609459200000, - "stop": 1609459201000 - }, - "tests": [ - { - "name": "should pass", - "status": "passed", - "duration": 100, - "attemptId": "a1b2c3d4-1111-4000-a000-000000000001" - } - ] - } -} -``` diff --git a/tests/ctrf.test.json b/tests/ctrf.test.json index e1c4269..295a9fb 100644 --- a/tests/ctrf.test.json +++ b/tests/ctrf.test.json @@ -188,11 +188,6 @@ "tests": [{ "name": "test", "status": "passed", "duration": 100 }] } } - }, - { - "description": "test-attempt-id.json", - "valid": true, - "dataPath": "../examples/test-attempt-id.json" } ] } diff --git a/tests/normative/final-attempt-id.test.json b/tests/normative/final-attempt-id.test.json deleted file mode 100644 index 9621e20..0000000 --- a/tests/normative/final-attempt-id.test.json +++ /dev/null @@ -1,252 +0,0 @@ -{ - "target": "../../schema/ctrf.schema.json", - "tests": [ - { - "description": "Accept non-empty Test.attemptId", - "valid": true, - "data": { - "reportFormat": "CTRF", - "specVersion": "0.2.0", - "results": { - "tool": { - "name": "example-runner" - }, - "summary": { - "tests": 1, - "passed": 1, - "failed": 0, - "pending": 0, - "skipped": 0, - "other": 0, - "start": 1609459200000, - "stop": 1609459201000 - }, - "tests": [ - { - "name": "should pass", - "status": "passed", - "duration": 100, - "attemptId": "native-attempt-1" - } - ] - } - } - }, - { - "description": "Test.attemptId is optional", - "valid": true, - "data": { - "reportFormat": "CTRF", - "specVersion": "0.2.0", - "results": { - "tool": { - "name": "example-runner" - }, - "summary": { - "tests": 1, - "passed": 1, - "failed": 0, - "pending": 0, - "skipped": 0, - "other": 0, - "start": 1609459200000, - "stop": 1609459201000 - }, - "tests": [ - { - "name": "should pass", - "status": "passed", - "duration": 100 - } - ] - } - } - }, - { - "description": "Reject Test.attemptId empty string", - "valid": false, - "data": { - "reportFormat": "CTRF", - "specVersion": "0.2.0", - "results": { - "tool": { - "name": "example-runner" - }, - "summary": { - "tests": 1, - "passed": 1, - "failed": 0, - "pending": 0, - "skipped": 0, - "other": 0, - "start": 1609459200000, - "stop": 1609459201000 - }, - "tests": [ - { - "name": "should pass", - "status": "passed", - "duration": 100, - "attemptId": "" - } - ] - } - } - }, - { - "description": "Reject Test.attemptId null", - "valid": false, - "data": { - "reportFormat": "CTRF", - "specVersion": "0.2.0", - "results": { - "tool": { - "name": "example-runner" - }, - "summary": { - "tests": 1, - "passed": 1, - "failed": 0, - "pending": 0, - "skipped": 0, - "other": 0, - "start": 1609459200000, - "stop": 1609459201000 - }, - "tests": [ - { - "name": "should pass", - "status": "passed", - "duration": 100, - "attemptId": null - } - ] - } - } - }, - { - "description": "Reject Test.attemptId number", - "valid": false, - "data": { - "reportFormat": "CTRF", - "specVersion": "0.2.0", - "results": { - "tool": { - "name": "example-runner" - }, - "summary": { - "tests": 1, - "passed": 1, - "failed": 0, - "pending": 0, - "skipped": 0, - "other": 0, - "start": 1609459200000, - "stop": 1609459201000 - }, - "tests": [ - { - "name": "should pass", - "status": "passed", - "duration": 100, - "attemptId": 1 - } - ] - } - } - }, - { - "description": "Reject Test.attemptId boolean", - "valid": false, - "data": { - "reportFormat": "CTRF", - "specVersion": "0.2.0", - "results": { - "tool": { - "name": "example-runner" - }, - "summary": { - "tests": 1, - "passed": 1, - "failed": 0, - "pending": 0, - "skipped": 0, - "other": 0, - "start": 1609459200000, - "stop": 1609459201000 - }, - "tests": [ - { - "name": "should pass", - "status": "passed", - "duration": 100, - "attemptId": true - } - ] - } - } - }, - { - "description": "Reject Test.attemptId array", - "valid": false, - "data": { - "reportFormat": "CTRF", - "specVersion": "0.2.0", - "results": { - "tool": { - "name": "example-runner" - }, - "summary": { - "tests": 1, - "passed": 1, - "failed": 0, - "pending": 0, - "skipped": 0, - "other": 0, - "start": 1609459200000, - "stop": 1609459201000 - }, - "tests": [ - { - "name": "should pass", - "status": "passed", - "duration": 100, - "attemptId": [] - } - ] - } - } - }, - { - "description": "Reject Test.attemptId object", - "valid": false, - "data": { - "reportFormat": "CTRF", - "specVersion": "0.2.0", - "results": { - "tool": { - "name": "example-runner" - }, - "summary": { - "tests": 1, - "passed": 1, - "failed": 0, - "pending": 0, - "skipped": 0, - "other": 0, - "start": 1609459200000, - "stop": 1609459201000 - }, - "tests": [ - { - "name": "should pass", - "status": "passed", - "duration": 100, - "attemptId": {} - } - ] - } - } - } - ] -} diff --git a/tests/normative/string-constraints.test.json b/tests/normative/string-constraints.test.json index c0c4b46..9c96387 100644 --- a/tests/normative/string-constraints.test.json +++ b/tests/normative/string-constraints.test.json @@ -200,6 +200,37 @@ }, "baseline": { "reportId": "" } } + }, + { + "description": "Test with empty attemptId violating minLength constraint", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.2.0", + "results": { + "tool": { + "name": "runner" + }, + "summary": { + "tests": 1, + "passed": 1, + "failed": 0, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459201000 + }, + "tests": [ + { + "name": "test", + "status": "passed", + "duration": 100, + "attemptId": "" + } + ] + } + } } ] } diff --git a/tests/normative/type-validation.test.json b/tests/normative/type-validation.test.json index d49a530..0419fad 100644 --- a/tests/normative/type-validation.test.json +++ b/tests/normative/type-validation.test.json @@ -1079,6 +1079,161 @@ }, "insights": { "passRate": { "current": 0.95, "baseline": 0.90, "change": "0.05" } } } + }, + { + "description": "Test with attemptId as number instead of string", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.2.0", + "results": { + "tool": { + "name": "runner" + }, + "summary": { + "tests": 1, + "passed": 1, + "failed": 0, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459201000 + }, + "tests": [ + { + "name": "test", + "status": "passed", + "duration": 100, + "attemptId": 12345 + } + ] + } + } + }, + { + "description": "Test with attemptId as null instead of string", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.2.0", + "results": { + "tool": { + "name": "runner" + }, + "summary": { + "tests": 1, + "passed": 1, + "failed": 0, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459201000 + }, + "tests": [ + { + "name": "test", + "status": "passed", + "duration": 100, + "attemptId": null + } + ] + } + } + }, + { + "description": "Test with attemptId as boolean instead of string", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.2.0", + "results": { + "tool": { + "name": "runner" + }, + "summary": { + "tests": 1, + "passed": 1, + "failed": 0, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459201000 + }, + "tests": [ + { + "name": "test", + "status": "passed", + "duration": 100, + "attemptId": true + } + ] + } + } + }, + { + "description": "Test with attemptId as array instead of string", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.2.0", + "results": { + "tool": { + "name": "runner" + }, + "summary": { + "tests": 1, + "passed": 1, + "failed": 0, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459201000 + }, + "tests": [ + { + "name": "test", + "status": "passed", + "duration": 100, + "attemptId": [] + } + ] + } + } + }, + { + "description": "Test with attemptId as object instead of string", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.2.0", + "results": { + "tool": { + "name": "runner" + }, + "summary": { + "tests": 1, + "passed": 1, + "failed": 0, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459201000 + }, + "tests": [ + { + "name": "test", + "status": "passed", + "duration": 100, + "attemptId": {} + } + ] + } + } } ] } diff --git a/tests/normative/valid-documents.test.json b/tests/normative/valid-documents.test.json index cc813d0..3d26cdf 100644 --- a/tests/normative/valid-documents.test.json +++ b/tests/normative/valid-documents.test.json @@ -32,14 +32,14 @@ "valid": true, "data": { "reportFormat": "CTRF", - "specVersion": "0.1.0", + "specVersion": "0.2.0", "runId": "run-1", "results": { "tool": { "name": "runner" }, "summary": { "tests": 2, "passed": 2, "failed": 0, "pending": 0, "skipped": 0, "other": 0, "start": 1609459200000, "stop": 1609459201000 }, "tests": [ { "name": "test", "testId": "tests/example.test.ts::suite::test", "executionId": "execution-1", "status": "passed", "duration": 100 }, - { "name": "test", "testId": "tests/example.test.ts::suite::test", "executionId": "execution-2", "status": "passed", "duration": 100 } + { "name": "test", "testId": "tests/example.test.ts::suite::test", "executionId": "execution-2", "attemptId": "native-attempt-1", "status": "passed", "duration": 100 } ] } }