diff --git a/e2e-tests/archive-lifecycle.test.ts b/e2e-tests/archive-lifecycle.test.ts index 0072cb323..9ee1424a3 100644 --- a/e2e-tests/archive-lifecycle.test.ts +++ b/e2e-tests/archive-lifecycle.test.ts @@ -9,6 +9,7 @@ * - AWS credentials * - npm, git, uv installed */ +import { NOT_FOUND_STATUS, TERMINAL_STATUSES } from '../src/cli/operations/jobs/index.js'; import { parseJsonOutput, retry } from '../src/test-utils/index.js'; import { baseCanRun, @@ -23,9 +24,12 @@ import { existsSync } from 'node:fs'; import { mkdir, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; +import { setTimeout as delay } from 'node:timers/promises'; import { afterAll, beforeAll, describe, expect, it } from 'vitest'; const canRun = baseCanRun && hasAws; +const jobWaitTimeoutMs = 300000; +const jobPollIntervalMs = 10000; describe.sequential('e2e: archive command lifecycle', () => { let testDir: string; @@ -35,6 +39,9 @@ describe.sequential('e2e: archive command lifecycle', () => { // IDs captured from run steps and used in archive steps let batchEvaluationId: string; let recommendationId: string; + let batchEvaluationArchivable = false; + let batchEvaluationArchived = false; + let recommendationArchived = false; beforeAll(async () => { if (!canRun) return; @@ -75,6 +82,37 @@ describe.sequential('e2e: archive command lifecycle', () => { const run = (args: string[]) => runAgentCoreCLI(args, projectPath); + const waitForJob = async (type: 'batch-evaluation' | 'recommendation', id: string) => { + const deadline = Date.now() + jobWaitTimeoutMs; + let lastStatus = 'unknown'; + let lastError: string | undefined; + let resultsFetched: boolean | undefined; + + while (Date.now() < deadline) { + const result = await run(['view', type, id, '--json']); + if (result.exitCode !== 0) { + throw new Error(`view ${type} ${id} failed: ${result.stderr}\n${result.stdout}`); + } + + const json = parseJsonOutput(result.stdout) as Record; + lastStatus = typeof json.status === 'string' ? json.status : 'unknown'; + lastError = typeof json.error === 'string' ? json.error : undefined; + resultsFetched = typeof json.resultsFetched === 'boolean' ? json.resultsFetched : undefined; + if (lastError) { + throw new Error(`${type} ${id} refresh failed: ${lastError} (status: ${lastStatus})`); + } + if (TERMINAL_STATUSES[type].has(lastStatus)) { + return lastStatus; + } + + await delay(Math.min(jobPollIntervalMs, Math.max(0, deadline - Date.now()))); + } + + throw new Error( + `${type} ${id} did not finish within ${jobWaitTimeoutMs}ms; last status: ${lastStatus}, resultsFetched: ${resultsFetched ?? 'unknown'}, error: ${lastError ?? 'none'}` + ); + }; + // ════════════════════════════════════════════════════════════════════════ // Setup — deploy and generate traces // ════════════════════════════════════════════════════════════════════════ @@ -129,7 +167,6 @@ describe.sequential('e2e: archive command lifecycle', () => { 'Builtin.Faithfulness', '--lookback-days', '1', - '--wait', '--json', ]); expect(result.exitCode, `batch-evaluation failed (stdout: ${result.stdout}, stderr: ${result.stderr})`).toBe( @@ -138,20 +175,23 @@ describe.sequential('e2e: archive command lifecycle', () => { const json = parseJsonOutput(result.stdout) as Record; expect(json).toHaveProperty('success', true); expect(json.id).toBeTruthy(); - expect(['COMPLETED', 'COMPLETED_WITH_ERRORS', 'STOPPED']).toContain(json.status); batchEvaluationId = json.id as string; }, 6, 15000 ); + + const status = await waitForJob('batch-evaluation', batchEvaluationId); + batchEvaluationArchivable = status !== NOT_FOUND_STATUS; + expect(['COMPLETED', 'COMPLETED_WITH_ERRORS', 'STOPPED']).toContain(status); }, 600000 ); it.skipIf(!canRun)( 'local .cli/batch-eval-results contains the run record', - () => { - expect(batchEvaluationId, 'batchEvaluationId should have been captured').toBeTruthy(); + ctx => { + if (!batchEvaluationId) ctx.skip(); const filePath = join( projectPath, 'agentcore', @@ -225,8 +265,8 @@ describe.sequential('e2e: archive command lifecycle', () => { it.skipIf(!canRun)( 'archives the batch evaluation with --json flag', - async () => { - expect(batchEvaluationId, 'batchEvaluationId must have been captured').toBeTruthy(); + async ctx => { + if (!batchEvaluationArchivable) ctx.skip(); const result = await run(['archive', 'batch-evaluation', '--id', batchEvaluationId, '--json']); expect(result.exitCode, `archive batch-evaluation failed: ${result.stderr}\n${result.stdout}`).toBe(0); @@ -234,13 +274,15 @@ describe.sequential('e2e: archive command lifecycle', () => { const json = parseJsonOutput(result.stdout) as Record; expect(json).toHaveProperty('success', true); expect(json.id).toBe(batchEvaluationId); + batchEvaluationArchived = true; }, 120000 ); it.skipIf(!canRun)( 'local .cli/batch-eval-results no longer contains the archived record', - () => { + ctx => { + if (!batchEvaluationArchived) ctx.skip(); const filePath = join( projectPath, 'agentcore', @@ -256,7 +298,8 @@ describe.sequential('e2e: archive command lifecycle', () => { it.skipIf(!canRun)( 'evals history does not surface the archived batch evaluation ID', - async () => { + async ctx => { + if (!batchEvaluationArchived) ctx.skip(); // evals history lists on-demand (run eval) records — batch evals are stored separately. // Verify: the command succeeds and contains no entry matching our batch evaluation ID. const result = await run(['evals', 'history', '--json']); @@ -270,7 +313,8 @@ describe.sequential('e2e: archive command lifecycle', () => { it.skipIf(!canRun)( 'archiving the same batch evaluation again returns success false (already deleted)', - async () => { + async ctx => { + if (!batchEvaluationArchived) ctx.skip(); const result = await run(['archive', 'batch-evaluation', '--id', batchEvaluationId, '--json']); // Service should return an error (resource not found / already deleted) expect(result.exitCode).toBe(1); @@ -296,8 +340,10 @@ describe.sequential('e2e: archive command lifecycle', () => { it.skipIf(!canRun)( 'archives the recommendation with --json flag', - async () => { - expect(recommendationId, 'recommendationId must have been captured').toBeTruthy(); + async ctx => { + if (!recommendationId) ctx.skip(); + const status = await waitForJob('recommendation', recommendationId); + expect(['COMPLETED', 'FAILED']).toContain(status); const result = await run(['archive', 'recommendation', '--id', recommendationId, '--json']); expect(result.exitCode, `archive recommendation failed: ${result.stderr}\n${result.stdout}`).toBe(0); @@ -305,13 +351,15 @@ describe.sequential('e2e: archive command lifecycle', () => { const json = parseJsonOutput(result.stdout) as Record; expect(json).toHaveProperty('success', true); expect(json.id).toBe(recommendationId); + recommendationArchived = true; }, - 120000 + 600000 ); it.skipIf(!canRun)( 'local .cli/recommendations no longer contains the archived record', - () => { + ctx => { + if (!recommendationArchived) ctx.skip(); const filePath = join(projectPath, 'agentcore', '.cli', 'jobs', 'recommendations', `${recommendationId}.json`); expect(existsSync(filePath), `Local record should have been deleted from ${filePath}`).toBe(false); }, @@ -320,7 +368,8 @@ describe.sequential('e2e: archive command lifecycle', () => { it.skipIf(!canRun)( 'archiving the same recommendation again returns success false (already deleted)', - async () => { + async ctx => { + if (!recommendationArchived) ctx.skip(); const result = await run(['archive', 'recommendation', '--id', recommendationId, '--json']); expect(result.exitCode).toBe(1); const json = parseJsonOutput(result.stdout) as Record;