Skip to content

fix(ci): wait for jobs before lifecycle checks #2712

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
nborges-aws wants to merge 2 commits into from
Closed

fix(ci): wait for jobs before lifecycle checks #2712

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
2bd3264
fix(ci): wait for jobs before lifecycle checks
nborges-aws Oct 27, 2026
84a97f0
fix(e2e): bound polling length and handle terminal statuses
nborges-aws Oct 27, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
77 changes: 63 additions & 14 deletions e2e-tests/archive-lifecycle.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
* - AWS credentials
* - npm, git, uv installed
*/
import { NOT_FOUND_STATUS, TERMINAL_STATUSES } from '../src/cli/operations/jobs/index.js';
import { parseJsonOutput, retry } from '../src/test-utils/index.js';
import {
baseCanRun,
Expand All @@ -23,9 +24,12 @@ import { existsSync } from 'node:fs';
import { mkdir, rm } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { setTimeout as delay } from 'node:timers/promises';
import { afterAll, beforeAll, describe, expect, it } from 'vitest';

const canRun = baseCanRun && hasAws;
const jobWaitTimeoutMs = 300000;
const jobPollIntervalMs = 10000;

describe.sequential('e2e: archive command lifecycle', () => {
let testDir: string;
Expand All @@ -35,6 +39,9 @@ describe.sequential('e2e: archive command lifecycle', () => {
// IDs captured from run steps and used in archive steps
let batchEvaluationId: string;
let recommendationId: string;
let batchEvaluationArchivable = false;
let batchEvaluationArchived = false;
let recommendationArchived = false;

beforeAll(async () => {
if (!canRun) return;
Expand Down Expand Up @@ -75,6 +82,37 @@ describe.sequential('e2e: archive command lifecycle', () => {

const run = (args: string[]) => runAgentCoreCLI(args, projectPath);

const waitForJob = async (type: 'batch-evaluation' | 'recommendation', id: string) => {
const deadline = Date.now() + jobWaitTimeoutMs;
let lastStatus = 'unknown';
let lastError: string | undefined;
let resultsFetched: boolean | undefined;

while (Date.now() < deadline) {
const result = await run(['view', type, id, '--json']);
if (result.exitCode !== 0) {
throw new Error(`view ${type} ${id} failed: ${result.stderr}\n${result.stdout}`);
}

const json = parseJsonOutput(result.stdout) as Record<string, unknown>;
lastStatus = typeof json.status === 'string' ? json.status : 'unknown';
lastError = typeof json.error === 'string' ? json.error : undefined;
resultsFetched = typeof json.resultsFetched === 'boolean' ? json.resultsFetched : undefined;
if (lastError) {
throw new Error(`${type} ${id} refresh failed: ${lastError} (status: ${lastStatus})`);
}
if (TERMINAL_STATUSES[type].has(lastStatus)) {
return lastStatus;
}

await delay(Math.min(jobPollIntervalMs, Math.max(0, deadline - Date.now())));
}

throw new Error(
`${type} ${id} did not finish within ${jobWaitTimeoutMs}ms; last status: ${lastStatus}, resultsFetched: ${resultsFetched ?? 'unknown'}, error: ${lastError ?? 'none'}`
);
};

// ════════════════════════════════════════════════════════════════════════
// Setup — deploy and generate traces
// ════════════════════════════════════════════════════════════════════════
Expand Down Expand Up @@ -129,7 +167,6 @@ describe.sequential('e2e: archive command lifecycle', () => {
'Builtin.Faithfulness',
'--lookback-days',
'1',
'--wait',
'--json',
]);
expect(result.exitCode, `batch-evaluation failed (stdout: ${result.stdout}, stderr: ${result.stderr})`).toBe(
Expand All @@ -138,20 +175,23 @@ describe.sequential('e2e: archive command lifecycle', () => {
const json = parseJsonOutput(result.stdout) as Record<string, unknown>;
expect(json).toHaveProperty('success', true);
expect(json.id).toBeTruthy();
expect(['COMPLETED', 'COMPLETED_WITH_ERRORS', 'STOPPED']).toContain(json.status);
batchEvaluationId = json.id as string;
},
6,
15000
);

const status = await waitForJob('batch-evaluation', batchEvaluationId);
batchEvaluationArchivable = status !== NOT_FOUND_STATUS;
expect(['COMPLETED', 'COMPLETED_WITH_ERRORS', 'STOPPED']).toContain(status);
},
600000
);

it.skipIf(!canRun)(
'local .cli/batch-eval-results contains the run record',
() => {
expect(batchEvaluationId, 'batchEvaluationId should have been captured').toBeTruthy();
ctx => {
if (!batchEvaluationId) ctx.skip();
const filePath = join(
projectPath,
'agentcore',
Expand Down Expand Up @@ -225,22 +265,24 @@ describe.sequential('e2e: archive command lifecycle', () => {

it.skipIf(!canRun)(
'archives the batch evaluation with --json flag',
async () => {
expect(batchEvaluationId, 'batchEvaluationId must have been captured').toBeTruthy();
async ctx => {
if (!batchEvaluationArchivable) ctx.skip();

const result = await run(['archive', 'batch-evaluation', '--id', batchEvaluationId, '--json']);
expect(result.exitCode, `archive batch-evaluation failed: ${result.stderr}\n${result.stdout}`).toBe(0);

const json = parseJsonOutput(result.stdout) as Record<string, unknown>;
expect(json).toHaveProperty('success', true);
expect(json.id).toBe(batchEvaluationId);
batchEvaluationArchived = true;
},
120000
);

it.skipIf(!canRun)(
'local .cli/batch-eval-results no longer contains the archived record',
() => {
ctx => {
if (!batchEvaluationArchived) ctx.skip();
const filePath = join(
projectPath,
'agentcore',
Expand All @@ -256,7 +298,8 @@ describe.sequential('e2e: archive command lifecycle', () => {

it.skipIf(!canRun)(
'evals history does not surface the archived batch evaluation ID',
async () => {
async ctx => {
if (!batchEvaluationArchived) ctx.skip();
// evals history lists on-demand (run eval) records — batch evals are stored separately.
// Verify: the command succeeds and contains no entry matching our batch evaluation ID.
const result = await run(['evals', 'history', '--json']);
Expand All @@ -270,7 +313,8 @@ describe.sequential('e2e: archive command lifecycle', () => {

it.skipIf(!canRun)(
'archiving the same batch evaluation again returns success false (already deleted)',
async () => {
async ctx => {
if (!batchEvaluationArchived) ctx.skip();
const result = await run(['archive', 'batch-evaluation', '--id', batchEvaluationId, '--json']);
// Service should return an error (resource not found / already deleted)
expect(result.exitCode).toBe(1);
Expand All @@ -296,22 +340,26 @@ describe.sequential('e2e: archive command lifecycle', () => {

it.skipIf(!canRun)(
'archives the recommendation with --json flag',
async () => {
expect(recommendationId, 'recommendationId must have been captured').toBeTruthy();
async ctx => {
if (!recommendationId) ctx.skip();
const status = await waitForJob('recommendation', recommendationId);
expect(['COMPLETED', 'FAILED']).toContain(status);

const result = await run(['archive', 'recommendation', '--id', recommendationId, '--json']);
expect(result.exitCode, `archive recommendation failed: ${result.stderr}\n${result.stdout}`).toBe(0);

const json = parseJsonOutput(result.stdout) as Record<string, unknown>;
expect(json).toHaveProperty('success', true);
expect(json.id).toBe(recommendationId);
recommendationArchived = true;
},
120000
600000
);

it.skipIf(!canRun)(
'local .cli/recommendations no longer contains the archived record',
() => {
ctx => {
if (!recommendationArchived) ctx.skip();
const filePath = join(projectPath, 'agentcore', '.cli', 'jobs', 'recommendations', `${recommendationId}.json`);
expect(existsSync(filePath), `Local record should have been deleted from ${filePath}`).toBe(false);
},
Expand All @@ -320,7 +368,8 @@ describe.sequential('e2e: archive command lifecycle', () => {

it.skipIf(!canRun)(
'archiving the same recommendation again returns success false (already deleted)',
async () => {
async ctx => {
if (!recommendationArchived) ctx.skip();
const result = await run(['archive', 'recommendation', '--id', recommendationId, '--json']);
expect(result.exitCode).toBe(1);
const json = parseJsonOutput(result.stdout) as Record<string, unknown>;
Expand Down
Loading