diff --git a/CHANGELOG.md b/CHANGELOG.md index b8a1a68..c0fb2d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.2.0] - 2026-10-10 + +### Added + +- Optional Test-level `attemptId` in specification 0.2.0, using the same + identifier contract as retry history. Reports without attempt identity remain + supported (#66). + ## [0.1.0] - 2026-10-03 ### Changed diff --git a/examples/comprehensive.json b/examples/comprehensive.json index 70f0431..aec49ce 100644 --- a/examples/comprehensive.json +++ b/examples/comprehensive.json @@ -1,6 +1,6 @@ { "reportFormat": "CTRF", - "specVersion": "0.1.0", + "specVersion": "0.2.0", "reportId": "9d2c6a10-3f7a-4e22-9a8f-1a2b3c4d5e6f", "runId": "run-2025-11-24-pr-123", "timestamp": "2025-11-24T12:00:00Z", @@ -76,6 +76,7 @@ "id": "550e8400-e29b-41d4-a716-446655440001", "testId": "checkout-user-can-checkout", "executionId": "2cf7e0b4-ea0c-4ef4-a18a-b9d6f6f0e101", + "attemptId": "48c3f90b-95a9-4c13-a42f-0888bb0b1002", "name": "user can checkout", "status": "passed", "duration": 5200, diff --git a/schema/ctrf.schema.json b/schema/ctrf.schema.json index 85bd935..0135d5e 100644 --- a/schema/ctrf.schema.json +++ b/schema/ctrf.schema.json @@ -153,6 +153,11 @@ "type": "string", "minLength": 1 }, + "attemptId": { + "description": "Unique identifier for this individual attempt. UUID recommended", + "type": "string", + "minLength": 1 + }, "name": { "description": "Name or title of the test case", "type": "string", diff --git a/spec/ctrf.md b/spec/ctrf.md index 000d6d5..25e1399 100644 --- a/spec/ctrf.md +++ b/spec/ctrf.md @@ -2,10 +2,10 @@ ## Common Test Report Format -**Version:** 0.1.0 +**Version:** 0.2.0 (This version corresponds directly to the CTRF `specVersion` field.) -**Date:** 2026-10-03 +**Date:** 2026-10-10 **Status:** Working Draft @@ -441,7 +441,7 @@ Each layer addresses a distinct identification concern: 2. **Run**: `runId` identifies the logical test run. Multiple CTRF documents MAY share the same `runId` when they represent shards or partitions of a single coordinated execution. 3. **Test case**: `testId` identifies the logical test case within the producer's chosen scope. It SHOULD be deterministic and stable across runs within that scope, enabling cross-run analysis, trending, and flake detection. 4. **Execution**: `executionId` identifies the complete execution lifecycle of a test case within a run. An execution MAY include previous attempts represented in `retryAttempts` followed by the final attempt represented by the test object. -5. **Attempt**: `attemptId` identifies an individual attempt history entry within an execution. +5. **Attempt**: `attemptId` identifies an individual attempt within an execution, either the final attempt on the test object or an earlier attempt in `retryAttempts`. 6. **Attachment**: `attachmentId` identifies a specific attachment reference instance. 7. **Shard**: `shardId` labels the partition or shard that produced this document within a logical run. @@ -923,6 +923,17 @@ If present, it MUST be a non-empty string. UUID is RECOMMENDED. `executionId` SHOULD be unique across executions and SHOULD NOT be reused across runs. +### 9.3.1. `attemptId` + +**Description:** +A unique identifier for the final attempt represented by this test object, including an initial attempt when no retry occurred. + +**Requirements:** +`attemptId` is OPTIONAL. +If present, it MUST be a non-empty string. +UUID is RECOMMENDED. +`attemptId` SHOULD be unique within the enclosing execution. + ### 9.4. `name` **Description:** @@ -1971,7 +1982,7 @@ It is the ONLY permitted extension point within the baseline object. CTRF follows **Semantic Versioning**. -This document defines CTRF version `0.1.0`. +This document defines CTRF version `0.2.0`. Before CTRF `1.0.0`: @@ -2232,6 +2243,11 @@ to this specification. "type": "string", "minLength": 1 }, + "attemptId": { + "description": "Unique identifier for this individual attempt. UUID recommended", + "type": "string", + "minLength": 1 + }, "name": { "description": "Name or title of the test case", "type": "string", @@ -3255,7 +3271,7 @@ It includes: ```json title="Comprehensive CTRF document" { "reportFormat": "CTRF", - "specVersion": "0.1.0", + "specVersion": "0.2.0", "reportId": "9d2c6a10-3f7a-4e22-9a8f-1a2b3c4d5e6f", "runId": "run-20251124-e2e-staging", "timestamp": "2025-11-24T12:00:00Z", @@ -3299,6 +3315,7 @@ It includes: "id": "c6c9f8c0-8b5e-5f7a-9e2d-3c91f8a7d7c1", "testId": "c6c9f8c0-8b5e-5f7a-9e2d-3c91f8a7d7c1", "executionId": "a1b2c3d4-1111-4000-a000-000000000001", + "attemptId": "f1e2d3c4-2222-4000-b000-000000000002", "name": "user can log in", "suite": ["auth", "login"], "filePath": "tests/auth/login.test.js", diff --git a/tests/normative/string-constraints.test.json b/tests/normative/string-constraints.test.json index c0c4b46..9c96387 100644 --- a/tests/normative/string-constraints.test.json +++ b/tests/normative/string-constraints.test.json @@ -200,6 +200,37 @@ }, "baseline": { "reportId": "" } } + }, + { + "description": "Test with empty attemptId violating minLength constraint", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.2.0", + "results": { + "tool": { + "name": "runner" + }, + "summary": { + "tests": 1, + "passed": 1, + "failed": 0, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459201000 + }, + "tests": [ + { + "name": "test", + "status": "passed", + "duration": 100, + "attemptId": "" + } + ] + } + } } ] } diff --git a/tests/normative/type-validation.test.json b/tests/normative/type-validation.test.json index d49a530..0419fad 100644 --- a/tests/normative/type-validation.test.json +++ b/tests/normative/type-validation.test.json @@ -1079,6 +1079,161 @@ }, "insights": { "passRate": { "current": 0.95, "baseline": 0.90, "change": "0.05" } } } + }, + { + "description": "Test with attemptId as number instead of string", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.2.0", + "results": { + "tool": { + "name": "runner" + }, + "summary": { + "tests": 1, + "passed": 1, + "failed": 0, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459201000 + }, + "tests": [ + { + "name": "test", + "status": "passed", + "duration": 100, + "attemptId": 12345 + } + ] + } + } + }, + { + "description": "Test with attemptId as null instead of string", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.2.0", + "results": { + "tool": { + "name": "runner" + }, + "summary": { + "tests": 1, + "passed": 1, + "failed": 0, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459201000 + }, + "tests": [ + { + "name": "test", + "status": "passed", + "duration": 100, + "attemptId": null + } + ] + } + } + }, + { + "description": "Test with attemptId as boolean instead of string", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.2.0", + "results": { + "tool": { + "name": "runner" + }, + "summary": { + "tests": 1, + "passed": 1, + "failed": 0, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459201000 + }, + "tests": [ + { + "name": "test", + "status": "passed", + "duration": 100, + "attemptId": true + } + ] + } + } + }, + { + "description": "Test with attemptId as array instead of string", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.2.0", + "results": { + "tool": { + "name": "runner" + }, + "summary": { + "tests": 1, + "passed": 1, + "failed": 0, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459201000 + }, + "tests": [ + { + "name": "test", + "status": "passed", + "duration": 100, + "attemptId": [] + } + ] + } + } + }, + { + "description": "Test with attemptId as object instead of string", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.2.0", + "results": { + "tool": { + "name": "runner" + }, + "summary": { + "tests": 1, + "passed": 1, + "failed": 0, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459201000 + }, + "tests": [ + { + "name": "test", + "status": "passed", + "duration": 100, + "attemptId": {} + } + ] + } + } } ] } diff --git a/tests/normative/valid-documents.test.json b/tests/normative/valid-documents.test.json index cc813d0..3d26cdf 100644 --- a/tests/normative/valid-documents.test.json +++ b/tests/normative/valid-documents.test.json @@ -32,14 +32,14 @@ "valid": true, "data": { "reportFormat": "CTRF", - "specVersion": "0.1.0", + "specVersion": "0.2.0", "runId": "run-1", "results": { "tool": { "name": "runner" }, "summary": { "tests": 2, "passed": 2, "failed": 0, "pending": 0, "skipped": 0, "other": 0, "start": 1609459200000, "stop": 1609459201000 }, "tests": [ { "name": "test", "testId": "tests/example.test.ts::suite::test", "executionId": "execution-1", "status": "passed", "duration": 100 }, - { "name": "test", "testId": "tests/example.test.ts::suite::test", "executionId": "execution-2", "status": "passed", "duration": 100 } + { "name": "test", "testId": "tests/example.test.ts::suite::test", "executionId": "execution-2", "attemptId": "native-attempt-1", "status": "passed", "duration": 100 } ] } }