feat(evals): use OpenCode 1.18.31 for GPT-6 release runs

docs/release-qualification.md

@@ -104,7 +104,7 @@ Authorize dispatches using the [paid-run budget](../.agents/plans/05-release-sim
 Keep that ledger across retries. Budget-stopped campaigns cannot qualify.
 
 ```bash
-bun run eval -- --release --model openai/gpt-5.6-sol --model xai/grok-4.6
+bun run eval -- --release --model openai/gpt-6-sol --model xai/grok-4.6
 bun run eval:canary -- prepare --report <campaign-dir>/report.json --out <canary-dir>
 # Run the prepared fixture, then record its session and transcript.
 bun run eval:canary -- record <record-options>

evals/release-policy.ts

@@ -108,7 +108,7 @@ export const RELEASE_MAX_CAMPAIGN_AGE_MS = 7 * 24 * 60 * 60 * 1_000;
 export const RELEASE_ENVIRONMENT_RESERVES_PER_STRATUM = 1;
 
 export const RELEASE_HOST_POLICY = {
-	opencodeVersion: "1.18.6",
+	opencodeVersion: "1.18.31",
 	platform: "linux",
 	reviewerSteps: null,
 } as const;

package.json

@@ -81,7 +81,7 @@
 	},
 	"devDependencies": {
 		"@biomejs/biome": "2.5.14",
-		"@opencode-ai/plugin": "1.18.6",
+		"@opencode-ai/plugin": "1.18.31",
 		"@types/bun": "1.4.2",
 		"@types/node": "26.6.2",
 		"typescript": "7.0.2"

scripts/eval-canary.ts

@@ -23,6 +23,7 @@ import {
 	packedPackageManifest,
 	samePackedArtifact,
 } from "../evals/provenance.js";
+import { RELEASE_HOST_POLICY } from "../evals/release-policy.js";
 import type { ActorIdentity, ArtifactIdentity } from "../evals/report.js";
 import { reportArtifactForCanary } from "../evals/report-artifact.js";
 import { assuranceProjection } from "../src/application/delivery.js";
@@ -519,6 +520,7 @@ export function deriveCanaryResult(input: {
 		"loads-flow-tools":
 			host.preparedFixture &&
 			host.versions.length === 1 &&
+			host.versions[0] === RELEASE_HOST_POLICY.opencodeVersion &&
 			hasCompletedFlowCall &&
 			loadedPlugin,
 		"saves-plan":

tests/eval-canary.test.ts

@@ -11,6 +11,7 @@ import {
 } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
+import { RELEASE_HOST_POLICY } from "../evals/release-policy.js";
 import {
 	artifactIdentitySha256,
 	CANARY_CHECKLIST_SHA256,
@@ -146,7 +147,7 @@ function canaryTranscript(
 		},
 	}));
 	return {
-		info: { directory, version: "1.18.6" },
+		info: { directory, version: RELEASE_HOST_POLICY.opencodeVersion },
 		messages: [
 			{
 				info: {
@@ -377,6 +378,27 @@ describe("canary record boundary", () => {
 		]);
 	});
 
+	test("rejects a canary recorded on a different OpenCode release host", () => {
+		const value = prepared();
+		const transcript = canaryTranscript();
+		const oldHostTranscript = {
+			...transcript,
+			info: { ...transcript.info, version: "1.18.6" },
+		};
+		const derived = deriveCanaryResult({
+			packageVersion: value.artifact.packageVersion,
+			artifactSha256: value.artifactSha256,
+			tarballSha256: value.artifact.tarballSha256,
+			preparedSha256: value.sha256,
+			pluginEntrySha256: value.pluginEntrySha256,
+			installation: installation(value),
+			session: canarySession(),
+			transcript: oldHostTranscript,
+		});
+		expect(derived.checks["loads-flow-tools"]).toBe(false);
+		expect(derived.status).toBe("failed");
+	});
+
 	test("refuses empty validation and delivery assertion sets", () => {
 		const value = prepared();
 		const session = structuredClone(canarySession()) as unknown as {
@@ -727,8 +749,6 @@ describe("canary release verification", () => {
 				now: new Date(now),
 				...(freshness ? { freshness } : {}),
 			});
-		// Retained evidence is read exactly as it would have been read inside its
-		// window: the expiry stops applying, and nothing else does.
 		expect(await verify("2026-08-29T00:00:00.000Z", "retained")).toBe(
 			await verify("2026-08-25T01:00:00.000Z"),
 		);

tests/live-opencode-smoke.test.ts

@@ -333,8 +333,8 @@ function evidence(overrides: Partial<EvidenceInput> = {}) {
 }
 
 describe("OpenCode eval metadata probe", () => {
-	test("pins the Phase 0 host, endpoints, and reviewer bound", () => {
-		expect(HOST_METADATA_CONTRACT.hostVersion).toBe("1.18.6");
+	test("pins the current host, endpoints, and reviewer bound", () => {
+		expect(HOST_METADATA_CONTRACT.hostVersion).toBe("1.18.31");
 		expect(HOST_METADATA_CONTRACT.endpoints).toEqual({
 			agents: "GET /agent",
 			createSession: "POST /session",

tests/qualification-cli.test.ts

@@ -30,6 +30,7 @@ import {
 } from "../evals/qualification-bundle.js";
 import { regradeQualificationBundle } from "../evals/qualification-regrade.js";
 import {
+	RELEASE_HOST_POLICY,
 	releaseCatalog,
 	releaseGraderBundle,
 	releaseHostConfigSha256,
@@ -142,7 +143,10 @@ function canaryTranscript(input: {
 		state: { status: "completed", input: {}, output },
 	});
 	return {
-		info: { directory: input.fixture, version: "1.18.6" },
+		info: {
+			directory: input.fixture,
+			version: RELEASE_HOST_POLICY.opencodeVersion,
+		},
 		messages: [
 			{
 				info: {
@@ -225,7 +229,7 @@ test("qualifies and seals a complete exact-artifact campaign through the CLI", a
 			models,
 			scenarios,
 			sampling: { kind: "release" },
-			opencodeVersion: "1.18.6",
+			opencodeVersion: RELEASE_HOST_POLICY.opencodeVersion,
 		});
 		const evaluator = evaluatorIdentity({
 			sourceCommit: artifact.sourceCommit,
@@ -437,6 +441,48 @@ test("qualifies and seals a complete exact-artifact campaign through the CLI", a
 			recordedAt,
 		});
 		expect(canary.record.status).toBe("passed");
+		const currentHostTranscript = canaryTranscript({
+			fixture: join(preparedDirectory, "fixture"),
+			packageVersion: artifact.packageVersion,
+			pluginEntrySha256: prepared.pluginEntrySha256,
+			session,
+		});
+		const oldHostTranscript = {
+			...currentHostTranscript,
+			info: { ...currentHostTranscript.info, version: "1.18.6" },
+		};
+		const oldHostCanary = await recordCanary({
+			repositoryRoot: join(temporary, "old-host-canary-root"),
+			prepared,
+			preparedDirectory,
+			operator: "qualification-test",
+			session,
+			transcript: oldHostTranscript,
+			recordedAt,
+		});
+		expect(oldHostCanary.record.status).toBe("failed");
+		const rejected = Bun.spawn(
+			[
+				"bun",
+				"run",
+				"qualify",
+				"--",
+				"--campaign-dir",
+				campaignDirectory,
+				"--canary",
+				oldHostCanary.path,
+			],
+			{ cwd: repositoryRoot, stdout: "pipe", stderr: "pipe" },
+		);
+		const [rejectedStdout, rejectedStderr, rejectedExitCode] =
+			await Promise.all([
+				new Response(rejected.stdout).text(),
+				new Response(rejected.stderr).text(),
+				rejected.exited,
+			]);
+		expect(rejectedExitCode).not.toBe(0);
+		expect(rejectedStdout).not.toContain("VERIFIED:");
+		expect(rejectedStderr).toContain("Canary status is failed.");
 
 		const bundlesDirectory = join(temporary, "bundles");
 		const qualified = Bun.spawn(

tests/release-metadata.test.ts

@@ -99,7 +99,7 @@ const canarySession = (() => {
 	return SessionSchema.parse(session);
 })();
 const canaryTranscript = (packageVersion: string) => ({
-	info: { directory: "<flow-eval-workspace>", version: "1.18.6" },
+	info: { directory: "<flow-eval-workspace>", version: "1.18.31" },
 	messages: [
 		{
 			info: {

tests/release-qualification.test.ts

@@ -45,7 +45,7 @@ function releaseReport(stopped = false) {
 		models: MODELS,
 		scenarios,
 		sampling: { kind: "release" },
-		opencodeVersion: "1.18.6",
+		opencodeVersion: "1.18.31",
 	});
 	const evaluator = evaluatorIdentity({
 		sourceCommit: ARTIFACT.sourceCommit,