diff --git a/docker/agent/Dockerfile b/docker/agent/Dockerfile index 99f7ec1..fc5b497 100644 --- a/docker/agent/Dockerfile +++ b/docker/agent/Dockerfile @@ -9,7 +9,7 @@ ARG CLAUDE_AMD64_LOCK=v2.1.306@sha256:9dc640d16d8228eff9620bde9e0f70d38385a2124f ARG CLAUDE_ARM64_LOCK=v2.1.306@sha256:871fd7f46210c13a95a38506e03ac90abfc5125fc25a1d4f840fbab551b5cd2e ARG CODEX_AMD64_LOCK=rust-v0.163.0@sha256:582646c6bf0561eb0de540f7fc85b9121b0091a826bc6e96830b7f1722122182 ARG CODEX_ARM64_LOCK=rust-v0.163.0@sha256:9d8ecf2d19d3e184877983902d6c85de48b3888dcb72c382a8a0c975845927fb -ARG HARNESS_VERSION=v0.1.18 +ARG HARNESS_VERSION=v0.1.19 FROM golang:1.27.3-trixie@sha256:1896654cfc5c2b5ad74c07f29cb36f5d2d1cbed42e3054070fbfe444e521e83c AS proxy ARG HARNESS_VERSION diff --git a/go.mod b/go.mod index a295f57..4cc137d 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/php/tamis go 1.27.3 require ( - github.com/alpha-omega-security/harness v0.1.18 + github.com/alpha-omega-security/harness v0.1.19 github.com/anthropics/anthropic-sdk-go v1.84.0 github.com/git-pkgs/cwe v0.1.1 github.com/openai/openai-go/v3 v3.85.1 diff --git a/go.sum b/go.sum index 38b050f..cff33b9 100644 --- a/go.sum +++ b/go.sum @@ -8,8 +8,8 @@ github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.2 h1:utpeoEeZjd+A8J41zvoLsOOr github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.2/go.mod h1:iptorS+VYKFL2N6PnebpS91dubG35eAOEERnT4PJbQU= github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 h1:fhqpLE3UEXi9lPaBRpQ6XuRW0nU7hgg4zlmZZa+a9q4= github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0/go.mod h1:7dCRMLwisfRH3dBupKeNCioWYUZ4SS09Z14H+7i8ZoY= -github.com/alpha-omega-security/harness v0.1.18 h1:99coqkQjuQ13jAX0ivZo7mu5xQyJW6n/QWj19USGMCI= -github.com/alpha-omega-security/harness v0.1.18/go.mod h1:4JI0CxVRwiWQ9zAtSTnM5p8l0A79bwREtps9BA9oLXY= +github.com/alpha-omega-security/harness v0.1.19 h1:X/EI0++ki34CFCROOCqfmiUSzodumqlJ0LLJks7N2Ws= +github.com/alpha-omega-security/harness v0.1.19/go.mod h1:4JI0CxVRwiWQ9zAtSTnM5p8l0A79bwREtps9BA9oLXY= github.com/anthropics/anthropic-sdk-go v1.84.0 h1:mnAhggc5H5A98JYmhQELT+j4J89PYBFKnnSvUTNwNd8= github.com/anthropics/anthropic-sdk-go v1.84.0/go.mod h1:GThfYqPJoaQ/6pmibCI98Cr4y5su2FXMUHn3NrSSnIc= github.com/aws/aws-sdk-go-v2 v1.47.1 h1:uOIZnp4PK3ZhKI0dNrJrhTEsLxbpXHTAJlwoS1pvAtw= diff --git a/internal/sandbox/runner.go b/internal/sandbox/runner.go index 7d76881..72a41ac 100644 --- a/internal/sandbox/runner.go +++ b/internal/sandbox/runner.go @@ -24,7 +24,23 @@ type agent struct { // provider's own API. func (*agent) EgressHosts() []string { return nil } -func (a *agent) ParseStream(r io.Reader, _ func(harness.Event)) { keepFailures(r, &a.failures) } +// ParseStream hands only the session event to emit, so that the run can be resumed. +func (a *agent) ParseStream(r io.Reader, emit func(harness.Event)) { + pr, pw := io.Pipe() + done := make(chan struct{}) + go func() { + a.Harness.ParseStream(pr, func(e harness.Event) { + if e.Kind == harness.KindSession { + emit(e) + } + }) + _, _ = io.Copy(io.Discard, pr) + close(done) + }() + keepFailures(io.TeeReader(r, pw), &a.failures) + _ = pw.Close() + <-done +} // runner keeps the agent on a network whose only way out is the egress proxy, allowing the model proxy and // hosts. harness refuses tunnels to its API port, so a model proxy served over TLS is reached as a separate diff --git a/internal/sandbox/runner_integration_test.go b/internal/sandbox/runner_integration_test.go index 4dda88f..680a219 100644 --- a/internal/sandbox/runner_integration_test.go +++ b/internal/sandbox/runner_integration_test.go @@ -15,6 +15,7 @@ import ( "os" "os/exec" "path/filepath" + "slices" "strconv" "strings" "sync/atomic" @@ -183,24 +184,42 @@ func TestReaderGetsOneCorrectionRound(t *testing.T) { t.Skip("TAMIS_TEST_IMAGE must end in :claude") } t.Run("corrected", func(t *testing.T) { - raw, seen, err := readWithCorrection(t, image, `{"ok":true}`) + raw, seen, resumed, err := readWithCorrection(t, image, `{"ok":true}`) require.NoError(t, err) require.JSONEq(t, `{"ok":true}`, string(raw)) require.Equal(t, []string{"not json", `{"ok":true}`}, seen) + require.True(t, resumed, "the correction did not resume the first conversation") }) t.Run("still invalid", func(t *testing.T) { - _, seen, err := readWithCorrection(t, image, "still not json") + _, seen, _, err := readWithCorrection(t, image, "still not json") require.ErrorIs(t, err, ErrNoReport) require.Equal(t, []string{"not json", "still not json"}, seen) }) } // readWithCorrection runs Claude Code against a model that writes invalid JSON, then corrected once the -// prompt carries the validation error, and returns what the validator saw. -func readWithCorrection(t *testing.T, image, corrected string) (json.RawMessage, []string, error) { +// prompt carries the validation error, and returns what the validator saw and whether the correction +// carried the first conversation. +func readWithCorrection(t *testing.T, image, corrected string) (json.RawMessage, []string, bool, error) { t.Helper() + var resumed bool + var calls atomic.Int32 _, port := testServerHandler(t, func(w http.ResponseWriter, r *http.Request) { body, _ := io.ReadAll(r.Body) + var request struct { + Messages []struct { + Role string `json:"role"` + Content json.RawMessage `json:"content"` + } `json:"messages"` + } + _ = json.Unmarshal(body, &request) + var called bool + for _, message := range slices.Backward(request.Messages) { + if message.Role == "assistant" { + called = bytes.Contains(message.Content, []byte(`"tool_use"`)) + break + } + } w.Header().Set("Content-Type", "text/event-stream") event := func(kind string, data any) { raw, _ := json.Marshal(data) @@ -208,17 +227,20 @@ func readWithCorrection(t *testing.T, image, corrected string) (json.RawMessage, } event("message_start", map[string]any{"type": "message_start", "message": map[string]any{"id": "msg_fixture", "type": "message", "role": "assistant", "model": "claude-opus-5-5", "content": []any{}, "stop_reason": nil, "usage": map[string]int{"input_tokens": 1, "output_tokens": 0}}}) stop := "end_turn" - if bytes.Contains(body, []byte(`"tool_result"`)) { + if called { event("content_block_start", map[string]any{"type": "content_block_start", "index": 0, "content_block": map[string]any{"type": "text", "text": ""}}) event("content_block_delta", map[string]any{"type": "content_block_delta", "index": 0, "delta": map[string]any{"type": "text_delta", "text": "Done."}}) } else { output := "not json" if bytes.Contains(body, []byte("did not produce valid output")) { output = corrected + resumed = bytes.Contains(body, []byte(`"tool_use_id"`)) } command, _ := json.Marshal(map[string]string{"command": "printf '%s' '" + output + "' > /work/" + outputFile}) stop = "tool_use" - event("content_block_start", map[string]any{"type": "content_block_start", "index": 0, "content_block": map[string]any{"type": "tool_use", "id": "tool_fixture", "name": "Bash", "input": map[string]any{}}}) + // A resumed conversation already holds the first run's tool call, so each call needs its own id. + id := fmt.Sprintf("tool_fixture_%d", calls.Add(1)) + event("content_block_start", map[string]any{"type": "content_block_start", "index": 0, "content_block": map[string]any{"type": "tool_use", "id": id, "name": "Bash", "input": map[string]any{}}}) event("content_block_delta", map[string]any{"type": "content_block_delta", "index": 0, "delta": map[string]any{"type": "input_json_delta", "partial_json": string(command)}}) } event("content_block_stop", map[string]any{"type": "content_block_stop", "index": 0}) @@ -237,7 +259,7 @@ func readWithCorrection(t *testing.T, image, corrected string) (json.RawMessage, } raw, err := reader.Read(t.Context(), Request{RepoPath: gitRepo(t), Prompt: "Answer", System: "System", Report: "Report", Schema: []byte(`{}`), Validate: validate}) - return raw, seen, err + return raw, seen, resumed, err } func testReader(t *testing.T, name, model, image, port string) *Reader { diff --git a/internal/sandbox/sandbox.go b/internal/sandbox/sandbox.go index 1ceec96..44b7b13 100644 --- a/internal/sandbox/sandbox.go +++ b/internal/sandbox/sandbox.go @@ -219,7 +219,7 @@ func (r *Reader) Read(ctx context.Context, req Request) (json.RawMessage, error) for attempt := 0; attempt < 2; attempt++ { a.failures = boundedWriter{} - err := scope.Run(ctx, job, nil) + session, err := harness.RunSession(ctx, scope.Run, job, nil) if err != nil { if failures := strings.TrimSpace(a.failures.String()); failures != "" { err = fmt.Errorf("%w: %s", err, failures) @@ -233,7 +233,10 @@ func (r *Reader) Read(ctx context.Context, req Request) (json.RawMessage, error) return raw, nil } } - job.Prompt = fmt.Sprintf("%s\n\nYour previous attempt did not produce valid output: %v. Write /work/%s again, fixed.", req.Prompt, err, outputFile) + // Prompt only serves when the agent announced no session or the resume cannot load it. + correction := fmt.Sprintf("Your previous attempt did not produce valid output: %v. Write /work/%s again, fixed.", err, outputFile) + job.Prompt = req.Prompt + "\n\n" + correction + job.ResumeSessionID, job.ResumePrompt = session.ID, correction } return nil, ErrNoReport diff --git a/internal/sandbox/sandbox_test.go b/internal/sandbox/sandbox_test.go index 425a72a..49742cc 100644 --- a/internal/sandbox/sandbox_test.go +++ b/internal/sandbox/sandbox_test.go @@ -165,14 +165,28 @@ func TestOnlyDockerCanRunTheAgent(t *testing.T) { require.Error(t, err, "an unsupported runtime must fail closed") } -func TestAgentKeepsOnlyTheFailuresOfItsOutput(t *testing.T) { +func TestAgentKeepsOnlyTheFailuresAndTheSessionOfItsOutput(t *testing.T) { h, err := harness.ByName("codex") require.NoError(t, err) a := &agent{Harness: h} + var events []harness.Event - a.ParseStream(strings.NewReader(`{"type":"item.completed","item":{"type":"agent_message","text":"report content"}}`+"\n"+`{"type":"turn.failed","error":{"message":"usage limit"}}`), nil) + a.ParseStream(strings.NewReader(`{"type":"thread.started","thread_id":"thread-1"}`+"\n"+`{"type":"item.completed","item":{"type":"agent_message","text":"report content"}}`+"\n"+`{"type":"turn.failed","error":{"message":"usage limit"}}`), func(e harness.Event) { events = append(events, e) }) require.Equal(t, `{"type":"turn.failed","error":{"message":"usage limit"}}`, a.failures.String()) + require.Equal(t, []harness.Event{{Kind: harness.KindSession, SessionID: "thread-1"}}, events) +} + +func TestAgentForwardsNoEventWithoutASession(t *testing.T) { + h, err := harness.ByName("claude") + require.NoError(t, err) + a := &agent{Harness: h} + var events []harness.Event + + a.ParseStream(strings.NewReader("No conversation found with session ID: dead\n"), func(e harness.Event) { events = append(events, e) }) + + require.Empty(t, events) + require.Empty(t, a.failures.String()) } func TestReadOutputIsBounded(t *testing.T) {