From 3592a77782aa74fb549ebe757c59e94cdd819696 Mon Sep 17 00:00:00 2001 From: Alexandre Daubois Date: Wed, 4 Nov 2026 14:56:10 +0100 Subject: [PATCH] Move to harness v0.1.20 and use what it now provides The model proxy is served by modelproxy.Serve and Codex is configured by modelproxy.CodexConfig and Provider.BaseURL, instead of a TLS listener, a config.toml and base paths written here. The correction round is harness.RunRepaired, and the reasons a run stopped come from the error events harness emits, so keepFailures, salvage and readOutput go. The integration tests stream their fixtures through modelproxytest, and Renovate takes the agent CLI managers from the harness preset. --- docker/agent/Dockerfile | 2 +- go.mod | 3 +- go.sum | 8 +- internal/sandbox/runner.go | 82 +------- internal/sandbox/runner_integration_test.go | 133 +++---------- internal/sandbox/sandbox.go | 163 ++++++---------- internal/sandbox/sandbox_test.go | 197 ++++++-------------- renovate.json | 42 +---- 8 files changed, 165 insertions(+), 465 deletions(-) diff --git a/docker/agent/Dockerfile b/docker/agent/Dockerfile index fc5b497..0ab18c6 100644 --- a/docker/agent/Dockerfile +++ b/docker/agent/Dockerfile @@ -9,7 +9,7 @@ ARG CLAUDE_AMD64_LOCK=v2.1.306@sha256:9dc640d16d8228eff9620bde9e0f70d38385a2124f ARG CLAUDE_ARM64_LOCK=v2.1.306@sha256:871fd7f46210c13a95a38506e03ac90abfc5125fc25a1d4f840fbab551b5cd2e ARG CODEX_AMD64_LOCK=rust-v0.163.0@sha256:582646c6bf0561eb0de540f7fc85b9121b0091a826bc6e96830b7f1722122182 ARG CODEX_ARM64_LOCK=rust-v0.163.0@sha256:9d8ecf2d19d3e184877983902d6c85de48b3888dcb72c382a8a0c975845927fb -ARG HARNESS_VERSION=v0.1.19 +ARG HARNESS_VERSION=v0.1.20 FROM golang:1.27.3-trixie@sha256:1896654cfc5c2b5ad74c07f29cb36f5d2d1cbed42e3054070fbfe444e521e83c AS proxy ARG HARNESS_VERSION diff --git a/go.mod b/go.mod index 2da55bc..12a35f4 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/php/tamis go 1.27.3 require ( - github.com/alpha-omega-security/harness v0.1.19 + github.com/alpha-omega-security/harness v0.1.20 github.com/anthropics/anthropic-sdk-go v1.84.0 github.com/git-pkgs/cwe v0.1.1 github.com/openai/openai-go/v3 v3.85.1 @@ -32,6 +32,7 @@ require ( github.com/mattn/go-isatty v0.0.24 // indirect github.com/pb33f/go-yaml v0.1.0 // indirect github.com/pb33f/ordered-map/v2 v2.3.2 // indirect + github.com/santhosh-tekuri/jsonschema/v6 v6.0.3 // indirect github.com/standard-webhooks/standard-webhooks/libraries v0.0.1 // indirect github.com/tidwall/gjson v1.20.0 // indirect github.com/tidwall/match v1.2.0 // indirect diff --git a/go.sum b/go.sum index 5b4807c..104a933 100644 --- a/go.sum +++ b/go.sum @@ -10,8 +10,8 @@ github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 h1:fhqpLE3UEXi9lPaBRpQ6Xu github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0/go.mod h1:7dCRMLwisfRH3dBupKeNCioWYUZ4SS09Z14H+7i8ZoY= github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c h1:pxW6RcqyfI9/kWtOwnv/G+AzdKuy2ZrqINhenH4HyNs= github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= -github.com/alpha-omega-security/harness v0.1.19 h1:X/EI0++ki34CFCROOCqfmiUSzodumqlJ0LLJks7N2Ws= -github.com/alpha-omega-security/harness v0.1.19/go.mod h1:4JI0CxVRwiWQ9zAtSTnM5p8l0A79bwREtps9BA9oLXY= +github.com/alpha-omega-security/harness v0.1.20 h1:BmWRfH13mycUnzzu51mkYH0HZtoKrvkeTr82dfTsbnc= +github.com/alpha-omega-security/harness v0.1.20/go.mod h1:A3YtZQo/uHk0t0BLNGLE8Ov9Lga7g5w0g2QYMu5z7oM= github.com/anthropics/anthropic-sdk-go v1.84.0 h1:mnAhggc5H5A98JYmhQELT+j4J89PYBFKnnSvUTNwNd8= github.com/anthropics/anthropic-sdk-go v1.84.0/go.mod h1:GThfYqPJoaQ/6pmibCI98Cr4y5su2FXMUHn3NrSSnIc= github.com/aws/aws-sdk-go-v2 v1.47.1 h1:uOIZnp4PK3ZhKI0dNrJrhTEsLxbpXHTAJlwoS1pvAtw= @@ -54,6 +54,8 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UF github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/coder/websocket v1.8.15 h1:6B2JPeOGlpff2Uz6vOEH1Vzpi0iUz20A+lPVhPHtNUA= github.com/coder/websocket v1.8.15/go.mod h1:NX3SzP+inril6yawo5CQXx8+fk145lPDC6pumgx0mVg= +github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI= +github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= github.com/dnephin/pflag v1.0.7 h1:oxONGlWxhmUct0YzKTgrpQv9AUA1wtPBn7zuSjJqptk= @@ -109,6 +111,8 @@ github.com/pb33f/ordered-map/v2 v2.3.2 h1:wDyaZ2Pv9QLh64X4utCeD5Zoi9nIv2aW3lwv17 github.com/pb33f/ordered-map/v2 v2.3.2/go.mod h1:1OhFrXu3OYw3kM+FXF+Ug3ugmmZ9LE8z0JfQMLvtV0w= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/santhosh-tekuri/jsonschema/v6 v6.0.3 h1:1EYB5IzjZawrrnELUi78f9fPu57HuXjmddZPjrls/28= +github.com/santhosh-tekuri/jsonschema/v6 v6.0.3/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= github.com/segmentio/asm v1.1.3 h1:WM03sfUOENvvKexOLp+pCqgb/WDjsi7EK8gIsICtzhc= github.com/segmentio/asm v1.1.3/go.mod h1:Ld3L4ZXGNcSLRg4JBsZ3//1+f/TjYl0Mzen/DQy1EJg= github.com/segmentio/encoding v0.5.4 h1:OW1VRern8Nw6ITAtwSZ7Idrl3MXCFwXHPgqESYfvNt0= diff --git a/internal/sandbox/runner.go b/internal/sandbox/runner.go index 799d484..2e4d300 100644 --- a/internal/sandbox/runner.go +++ b/internal/sandbox/runner.go @@ -1,53 +1,30 @@ package sandbox import ( - "bufio" - "bytes" - "encoding/json" - "errors" - "io" - "github.com/alpha-omega-security/harness" "github.com/alpha-omega-security/harness/container" ) -const maxAgentOutput = 1 << 20 - -// agent keeps the failure events of a run's output, which otherwise holds the report and the agent's -// reasoning about it. type agent struct { harness.Harness - failures boundedWriter } // EgressHosts is empty so the egress proxy allows the model proxy and the request's hosts only, not the // provider's own API. func (*agent) EgressHosts() []string { return nil } -// ParseStream hands every event to harness, which reads the session, a lost conversation and the account -// errors from them. Read emits none of them further. -func (a *agent) ParseStream(r io.Reader, emit func(harness.Event)) { - pr, pw := io.Pipe() - done := make(chan struct{}) - go func() { - a.Harness.ParseStream(pr, emit) - _, _ = io.Copy(io.Discard, pr) - close(done) - }() - keepFailures(io.TeeReader(r, pw), &a.failures) - _ = pw.Close() - <-done +// failures keeps the errors a run reports among its events. Codex exec --json reports why it stopped only there. +type failures []string + +func (f *failures) keep(e harness.Event) { + if e.Kind == harness.KindError { + *f = append(*f, e.Text) + } } // runner keeps the agent on a network whose only way out is the egress proxy, allowing the model proxy and -// hosts. harness refuses tunnels to its API port, so a model proxy served over TLS is reached as a separate -// host port. +// hosts. func (r *Reader) runner(state, source, token string, hosts, env []string) container.Runner { - sidecar := container.SidecarConfig{Allow: hosts, APIPort: r.modelPort} - if r.backend.accessToken { - sidecar.APIPort, sidecar.HostPorts = "", []string{r.modelPort} - } - return container.Runner{ Runtime: r.runtime, Image: r.image, @@ -59,7 +36,7 @@ func (r *Reader) runner(state, source, token string, hosts, env []string) contai ProcessEnv: []string{r.backend.keyEnv + "=" + token}, Hardened: true, IsolateHost: true, - Sidecar: sidecar, + Sidecar: container.SidecarConfig{Allow: hosts, APIPort: r.server.APIPort, HostPorts: r.server.HostPorts}, ReadOnlyWorkspace: true, Memory: "2g", CPUs: "2", @@ -67,44 +44,3 @@ func (r *Reader) runner(state, source, token string, hosts, env []string) contai FileSizeLimit: 64 << 20, } } - -// keepFailures copies the error events of a JSONL agent output and discards the rest, lines longer than -// maxAgentOutput included. Codex exec --json -// reports why it stopped only there, as "error" and "turn.failed" events or "error" items. -func keepFailures(r io.Reader, w io.Writer) { - br := bufio.NewReaderSize(r, maxAgentOutput) - for { - line, err := br.ReadSlice('\n') - for errors.Is(err, bufio.ErrBufferFull) { - line = nil - _, err = br.ReadSlice('\n') - } - var event struct { - Type string `json:"type"` - Item struct { - Type string `json:"type"` - } `json:"item"` - } - if json.Unmarshal(line, &event) == nil && (event.Type == "error" || event.Type == "turn.failed" || event.Item.Type == "error") { - _, _ = w.Write(line) - } - if err != nil { - _, _ = io.Copy(io.Discard, r) - return - } - } -} - -type boundedWriter struct { - b bytes.Buffer -} - -func (w *boundedWriter) Write(p []byte) (int, error) { - if remaining := maxAgentOutput - w.b.Len(); remaining > 0 { - w.b.Write(p[:min(remaining, len(p))]) - } - - return len(p), nil -} - -func (w *boundedWriter) String() string { return w.b.String() } diff --git a/internal/sandbox/runner_integration_test.go b/internal/sandbox/runner_integration_test.go index 680a219..1da101f 100644 --- a/internal/sandbox/runner_integration_test.go +++ b/internal/sandbox/runner_integration_test.go @@ -11,7 +11,6 @@ import ( "io" "net" "net/http" - "net/http/httptest" "os" "os/exec" "path/filepath" @@ -25,6 +24,8 @@ import ( "github.com/alpha-omega-security/harness" "github.com/alpha-omega-security/harness/container" "github.com/alpha-omega-security/harness/egress" + "github.com/alpha-omega-security/harness/modelproxy" + "github.com/alpha-omega-security/harness/modelproxy/modelproxytest" "github.com/stretchr/testify/require" ) @@ -44,7 +45,7 @@ func TestIsolatedNetworkReachesTheModelButNotOtherHostServices(t *testing.T) { require.NotEmpty(t, hostIP, "resolving the real host gateway") h, err := harness.ByName("claude") require.NoError(t, err) - reader := &Reader{backend: backends["claude"], harness: h, runtime: runtime, image: image, modelPort: modelPort} + reader := &Reader{backend: backends["claude"], harness: h, runtime: runtime, image: image, server: &modelproxy.Server{APIPort: modelPort}} env := []string{"MODEL_URL=http://" + egress.HostGatewayAlias + ":" + modelPort, "SECRET_URL=http://" + net.JoinHostPort(hostIP, secretPort), "PROXIED_SECRET_URL=http://" + egress.HostGatewayAlias + ":" + secretPort} scope, err := reader.runner(t.TempDir(), t.TempDir(), "proxy-token", allow, env).Open(ctx, &agent{Harness: h}) require.NoError(t, err) @@ -84,93 +85,23 @@ func testAgentCanReachTheFilteredModelProxy(t *testing.T, name, model, imageSuff var calls atomic.Int32 var first atomic.Value upstream, port := testServerHandler(t, func(w http.ResponseWriter, r *http.Request) { - switch { - case strings.HasSuffix(r.URL.Path, "/v1/user-auth-credential/whoami"): - _, _ = io.WriteString(w, `{"email":"private@example.com","chatgpt_user_id":"private-user","chatgpt_account_id":"private-account","chatgpt_plan_type":"enterprise","chatgpt_account_is_fedramp":false}`) - case strings.HasSuffix(r.URL.Path, "/wham/config/bundle"): - _, _ = io.WriteString(w, `{}`) - case strings.HasSuffix(r.URL.Path, "/codex/models"): - _, _ = io.WriteString(w, `{"models":[]}`) - default: - if calls.Add(1) == 1 { - body, _ := io.ReadAll(r.Body) - first.Store(string(body)) - w.Header().Set("Content-Type", "text/event-stream") - event := func(kind string, data any) { - raw, _ := json.Marshal(data) - _, _ = fmt.Fprintf(w, "event: %s\ndata: %s\n\n", kind, raw) - } - if name == "claude" { - event("message_start", map[string]any{"type": "message_start", "message": map[string]any{"id": "msg_fixture", "type": "message", "role": "assistant", "model": model, "content": []any{}, "stop_reason": nil, "usage": map[string]int{"input_tokens": 1, "output_tokens": 0}}}) - event("content_block_start", map[string]any{"type": "content_block_start", "index": 0, "content_block": map[string]any{"type": "tool_use", "id": "tool_fixture", "name": "Bash", "input": map[string]any{}}}) - event("content_block_delta", map[string]any{"type": "content_block_delta", "index": 0, "delta": map[string]any{"type": "input_json_delta", "partial_json": `{"command":"printf fixture"}`}}) - event("content_block_stop", map[string]any{"type": "content_block_stop", "index": 0}) - event("message_delta", map[string]any{"type": "message_delta", "delta": map[string]any{"stop_reason": "tool_use", "stop_sequence": nil}, "usage": map[string]int{"output_tokens": 1}}) - event("message_stop", map[string]any{"type": "message_stop"}) - } else { - tool := map[string]any{"type": "function_call", "id": "fc_fixture", "call_id": "call_fixture", "name": "exec_command", "arguments": `{"cmd":"printf fixture"}`} - event("response.created", map[string]any{"type": "response.created", "response": map[string]any{"id": "resp_fixture", "status": "in_progress", "output": []any{}}}) - event("response.output_item.done", map[string]any{"type": "response.output_item.done", "output_index": 0, "item": tool}) - event("response.completed", map[string]any{"type": "response.completed", "response": map[string]any{"id": "resp_fixture", "status": "completed", "model": model, "output": []any{tool}, "usage": map[string]int{"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}}}) - } - return - } + if modelproxytest.CodexStartup(w, r) { + return + } + if calls.Add(1) != 1 { http.Error(w, `{"type":"error","error":{"type":"invalid_request_error","message":"stop"}}`, http.StatusBadRequest) + return } - }) - t.Cleanup(func() { _ = upstream.Close() }) - reader := testReader(t, name, model, image, port) - reader.server.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { body, _ := io.ReadAll(r.Body) - r.Body = io.NopCloser(bytes.NewReader(body)) - recorder := httptest.NewRecorder() - reader.models.ServeHTTP(recorder, r) - if recorder.Code == http.StatusForbidden { - var value map[string]any - _ = json.Unmarshal(body, &value) - keys := func(m map[string]any) []string { - var keys []string - for key := range m { - keys = append(keys, key) - } - return keys - } - t.Logf("rejected fixture request fields: %v; context: %v; beta: %v; reason: %s", keys(value), value["context_management"], r.Header.Get("Anthropic-Beta"), recorder.Body.String()) - if blocks, ok := value["system"].([]any); ok { - for _, block := range blocks { - if block, ok := block.(map[string]any); ok { - t.Logf("system block %v: %v", block["type"], keys(block)) - } - } - } - if messages, ok := value["messages"].([]any); ok { - for _, message := range messages { - if message, ok := message.(map[string]any); ok { - t.Logf("message fields: %v", keys(message)) - if blocks, ok := message["content"].([]any); ok { - for _, block := range blocks { - if block, ok := block.(map[string]any); ok { - t.Logf("message block %v: %v", block["type"], keys(block)) - } - } - } - } - } - } - if ts, ok := value["tools"].([]any); ok { - for _, tool := range ts { - if tool, ok := tool.(map[string]any); ok { - t.Logf("tool type %v fields: %v", tool["type"], keys(tool)) - } - } - } - } - for name, values := range recorder.Header() { - w.Header()[name] = values + first.Store(string(body)) + if name == "claude" { + modelproxytest.AnthropicToolUse(w, model, "tool_fixture", "Bash", `{"command":"printf fixture"}`) + return } - w.WriteHeader(recorder.Code) - _, _ = w.Write(recorder.Body.Bytes()) + modelproxytest.ResponsesFunctionCall(w, model, "call_fixture", "exec_command", `{"cmd":"printf fixture"}`) }) + t.Cleanup(func() { _ = upstream.Close() }) + reader := testReader(t, name, model, image, port) _, readErr := reader.Read(t.Context(), Request{RepoPath: gitRepo(t), Prompt: "Answer", System: "System", Report: "Report", Schema: []byte(`{}`), Validate: func(json.RawMessage) error { return nil }}) require.GreaterOrEqual(t, calls.Load(), int32(2), "the legitimate %s request was blocked before the model provider: %v", name, errors.Unwrap(readErr)) if name != "claude" { @@ -220,32 +151,18 @@ func readWithCorrection(t *testing.T, image, corrected string) (json.RawMessage, break } } - w.Header().Set("Content-Type", "text/event-stream") - event := func(kind string, data any) { - raw, _ := json.Marshal(data) - _, _ = fmt.Fprintf(w, "event: %s\ndata: %s\n\n", kind, raw) - } - event("message_start", map[string]any{"type": "message_start", "message": map[string]any{"id": "msg_fixture", "type": "message", "role": "assistant", "model": "claude-opus-5-5", "content": []any{}, "stop_reason": nil, "usage": map[string]int{"input_tokens": 1, "output_tokens": 0}}}) - stop := "end_turn" if called { - event("content_block_start", map[string]any{"type": "content_block_start", "index": 0, "content_block": map[string]any{"type": "text", "text": ""}}) - event("content_block_delta", map[string]any{"type": "content_block_delta", "index": 0, "delta": map[string]any{"type": "text_delta", "text": "Done."}}) - } else { - output := "not json" - if bytes.Contains(body, []byte("did not produce valid output")) { - output = corrected - resumed = bytes.Contains(body, []byte(`"tool_use_id"`)) - } - command, _ := json.Marshal(map[string]string{"command": "printf '%s' '" + output + "' > /work/" + outputFile}) - stop = "tool_use" - // A resumed conversation already holds the first run's tool call, so each call needs its own id. - id := fmt.Sprintf("tool_fixture_%d", calls.Add(1)) - event("content_block_start", map[string]any{"type": "content_block_start", "index": 0, "content_block": map[string]any{"type": "tool_use", "id": id, "name": "Bash", "input": map[string]any{}}}) - event("content_block_delta", map[string]any{"type": "content_block_delta", "index": 0, "delta": map[string]any{"type": "input_json_delta", "partial_json": string(command)}}) + modelproxytest.AnthropicText(w, "claude-opus-5-5", "Done.") + return + } + output := "not json" + if bytes.Contains(body, []byte("failed validation")) { + output = corrected + resumed = bytes.Contains(body, []byte(`"tool_use_id"`)) } - event("content_block_stop", map[string]any{"type": "content_block_stop", "index": 0}) - event("message_delta", map[string]any{"type": "message_delta", "delta": map[string]any{"stop_reason": stop, "stop_sequence": nil}, "usage": map[string]int{"output_tokens": 1}}) - event("message_stop", map[string]any{"type": "message_stop"}) + command, _ := json.Marshal(map[string]string{"command": "printf '%s' '" + output + "' > /work/" + outputFile}) + // A resumed conversation already holds the first run's tool call, so each call needs its own id. + modelproxytest.AnthropicToolUse(w, "claude-opus-5-5", fmt.Sprintf("tool_fixture_%d", calls.Add(1)), "Bash", string(command)) }) reader := testReader(t, "claude", "claude-opus-5-5", image, port) diff --git a/internal/sandbox/sandbox.go b/internal/sandbox/sandbox.go index 89e592a..da57df1 100644 --- a/internal/sandbox/sandbox.go +++ b/internal/sandbox/sandbox.go @@ -5,24 +5,18 @@ package sandbox import ( "context" - "crypto/tls" _ "embed" "encoding/json" "errors" "fmt" "log/slog" - "net" - "net/http" "os" "os/exec" "path/filepath" - "strconv" "strings" - "time" "github.com/alpha-omega-security/harness" "github.com/alpha-omega-security/harness/container" - "github.com/alpha-omega-security/harness/egress" "github.com/alpha-omega-security/harness/modelproxy" "github.com/php/tamis/internal/budget" @@ -44,17 +38,15 @@ type backend struct { harness string provider modelproxy.Provider keyEnv string - // basePath completes the model proxy URL the way the CLI expects its API base URL. - basePath string // accessToken signs Codex in with a Codex access token, which it only sends over HTTPS. accessToken bool effort string } var backends = map[string]backend{ - "claude": {"claude", modelproxy.Anthropic, "ANTHROPIC_API_KEY", modelproxy.PathPrefix, false, ""}, - "codex": {"codex", modelproxy.OpenAI, "CODEX_API_KEY", modelproxy.PathPrefix + "/v1", false, "medium"}, - "codex-token": {"codex", modelproxy.ChatGPT, "CODEX_ACCESS_TOKEN", modelproxy.PathPrefix + "/backend-api/codex", true, "medium"}, + "claude": {"claude", modelproxy.Anthropic, "ANTHROPIC_API_KEY", false, ""}, + "codex": {"codex", modelproxy.OpenAI, "CODEX_API_KEY", false, "medium"}, + "codex-token": {"codex", modelproxy.ChatGPT, "CODEX_ACCESS_TOKEN", true, "medium"}, } // imagesJSON pins each agent image by digest, so a version of tamis always runs the images built for it. @@ -73,16 +65,13 @@ var Images = func() map[string]string { }() type Reader struct { - backend backend - harness harness.Harness - model string - image string - runtime container.Runtime - models *modelproxy.Proxy - modelPort string - server *http.Server - // ca is the PEM certificate the container trusts for the model proxy, with an access token only. - ca []byte + backend backend + harness harness.Harness + model string + image string + runtime container.Runtime + models *modelproxy.Proxy + server *modelproxy.Server // Daybreak turns on OpenAI's cyber access program in Codex. The API key's org must be approved for it. Daybreak bool @@ -126,33 +115,19 @@ func New(name, model, apiKey, image string) (*Reader, error) { if err != nil { return nil, err } - ln, err := net.Listen("tcp", ":0") + server, err := modelproxy.Serve(models, b.accessToken) if err != nil { return nil, err } - server := &http.Server{Handler: models, ReadHeaderTimeout: 10 * time.Second} - var ca []byte - if b.accessToken { - var cert tls.Certificate - if ca, cert, err = modelproxy.NewCertificate(egress.HostGatewayAlias); err != nil { - ln.Close() - return nil, err - } - ln = tls.NewListener(ln, &tls.Config{Certificates: []tls.Certificate{cert}, MinVersion: tls.VersionTLS12}) - } - go func() { _ = server.Serve(ln) }() - modelPort := strconv.Itoa(ln.Addr().(*net.TCPAddr).Port) return &Reader{ - backend: b, - harness: h, - model: model, - image: image, - runtime: rt, - models: models, - modelPort: modelPort, - server: server, - ca: ca, + backend: b, + harness: h, + model: model, + image: image, + runtime: rt, + models: models, + server: server, }, nil } @@ -192,17 +167,11 @@ func (r *Reader) Read(ctx context.Context, req Request) (json.RawMessage, error) if req.Offline { hosts = nil } - origin := "http://" + egress.HostGatewayAlias + ":" + r.modelPort - var env []string - if r.backend.accessToken { - origin = "https://" + egress.HostGatewayAlias + ":" + r.modelPort - env = []string{"CODEX_CA_CERTIFICATE=" + container.StateMount + "/" + caFile, "CODEX_AUTHAPI_BASE_URL=" + origin + modelproxy.PathPrefix + "/auth"} - } - if err := r.writeConfig(state, origin); err != nil { + env, err := r.writeConfig(state) + if err != nil { return nil, err } - a := &agent{Harness: r.harness} - scope, err := r.runner(state, source, token, hosts, env).Open(ctx, a) + scope, err := r.runner(state, source, token, hosts, env).Open(ctx, &agent{Harness: r.harness}) if err != nil { return nil, err } @@ -216,32 +185,32 @@ func (r *Reader) Read(ctx context.Context, req Request) (json.RawMessage, error) Effort: r.backend.effort, MaxTurns: maxTurns, OutputFile: outputFile, - BaseURL: origin + r.backend.basePath, + BaseURL: r.backend.provider.BaseURL(r.server.Origin), } - for attempt := 0; attempt < 2; attempt++ { - a.failures = boundedWriter{} - session, err := harness.RunSession(ctx, scope.Run, job, nil) - if err != nil { - if failures := strings.TrimSpace(a.failures.String()); failures != "" { - err = fmt.Errorf("%w: %s", err, failures) - } - return salvage(filepath.Join(workspace, outputFile), req.Validate, err) - } + var failed failures + raw, _, err := harness.RunRepaired(ctx, scope.Run, job, harness.Repair{Validate: func(raw []byte) error { return req.Validate(raw) }}, failed.keep) - raw, err := readOutput(filepath.Join(workspace, outputFile)) - if err == nil { - if err = req.Validate(raw); err == nil { - return raw, nil - } - } - // Prompt only serves when the agent announced no session or the resume cannot load it. - correction := fmt.Sprintf("Your previous attempt did not produce valid output: %v. Write /work/%s again, fixed.", err, outputFile) - job.Prompt = req.Prompt + "\n\n" + correction - job.ResumeSessionID, job.ResumePrompt = session.ID, correction + return outcome(raw, err, failed) +} + +// outcome keeps a valid report the agent wrote before failing: Codex has no turn limit to stop on before its budget. +func outcome(raw []byte, err error, failed failures) (json.RawMessage, error) { + if err == nil { + return raw, nil + } + // RunRepaired joins the validation error with the error of a failed run. + if _, invalid := err.(*harness.ValidationError); invalid { + return nil, ErrNoReport + } + if len(failed) > 0 { + err = fmt.Errorf("%w: %s", err, strings.Join(failed, "\n")) + } + if raw == nil { + return nil, &Error{Err: err} } - return nil, ErrNoReport + return raw, &Incomplete{Err: &Error{Err: err}} } func writeWorkspace(workspace string, req Request) error { @@ -265,47 +234,27 @@ func writeWorkspace(workspace string, req Request) error { return nil } -// salvage keeps a valid report the agent wrote before failing: Codex has no turn limit to stop on before its budget. -func salvage(path string, validate func(json.RawMessage) error, failure error) (json.RawMessage, error) { - raw, err := readOutput(path) - if err != nil || validate(raw) != nil { - return nil, &Error{Err: failure} - } - - return raw, &Incomplete{Err: &Error{Err: failure}} -} - -func readOutput(path string) ([]byte, error) { - data, err := os.ReadFile(path) - if err != nil { - return nil, err - } - if len(data) > 1<<20 { - return nil, errors.New("report is too large") - } - - return data, nil -} - -// writeConfig writes the agent CLI configuration into its state directory, which harness points CODEX_HOME at. -func (r *Reader) writeConfig(state, origin string) error { +// writeConfig writes the agent CLI configuration into its state directory, which harness points CODEX_HOME at, +// and returns the environment the CLI needs with it. +func (r *Reader) writeConfig(state string) ([]string, error) { if r.backend.harness != "codex" { if !r.Daybreak { - return nil + return nil, nil } - return errors.New("daybreak is only available with codex") + return nil, errors.New("daybreak is only available with codex") } - auth := "env_key = \"CODEX_API_KEY\"" - if r.backend.accessToken { - auth = "requires_openai_auth = true" - if err := os.WriteFile(filepath.Join(state, caFile), r.ca, 0o600); err != nil { - return err + if r.server.CA != nil { + if err := os.WriteFile(filepath.Join(state, caFile), r.server.CA, 0o600); err != nil { + return nil, err } } - // The ChatGPT features are off: they would call routes the model proxy refuses. - config := fmt.Sprintf("model_provider = \"tamis\"\ndaybreak = %t\nweb_search = \"disabled\"\nchatgpt_base_url = %q\n\n[features]\nremote_plugin = false\nplugin_sharing = false\ngoals = false\nimage_generation = false\nskill_mcp_dependency_install = false\n\n[model_providers.tamis]\nname = \"Tamis\"\nbase_url = %q\n%s\nwire_api = \"responses\"\nsupports_websockets = false\nsupports_standalone_web_search = false\nrequest_max_retries = 0\nstream_max_retries = 0\n", r.Daybreak, origin+modelproxy.PathPrefix+"/backend-api", origin+r.backend.basePath, auth) - return os.WriteFile(filepath.Join(state, "config.toml"), []byte(config), 0o600) + config, env, err := modelproxy.CodexConfig(r.backend.provider, r.server.Origin, container.StateMount+"/"+caFile) + if err != nil { + return nil, err + } + + return env, os.WriteFile(filepath.Join(state, "config.toml"), append(fmt.Appendf(nil, "daybreak = %t\n", r.Daybreak), config...), 0o600) } // SnapshotSource keeps the sandbox on tracked content while retaining the current commit's public history. diff --git a/internal/sandbox/sandbox_test.go b/internal/sandbox/sandbox_test.go index 0cc7080..2941e4d 100644 --- a/internal/sandbox/sandbox_test.go +++ b/internal/sandbox/sandbox_test.go @@ -2,8 +2,8 @@ package sandbox import ( "context" - "encoding/json" "errors" + "fmt" "os" "os/exec" "path/filepath" @@ -14,6 +14,7 @@ import ( "github.com/alpha-omega-security/harness" "github.com/alpha-omega-security/harness/container" + "github.com/alpha-omega-security/harness/modelproxy" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -99,28 +100,10 @@ func TestErrorHidesTheAgentOutput(t *testing.T) { require.ErrorContains(t, errors.Unwrap(err), "SECRET-PAYLOAD") } -func TestBoundedWriterRetainsOnlyTheOutputLimit(t *testing.T) { - var w boundedWriter - want := strings.Repeat("a", maxAgentOutput) - n, err := w.Write([]byte(want)) - require.NoError(t, err) - require.Equal(t, len(want), n) - require.Equal(t, want, w.String()) -} - -func TestBoundedWriterDiscardsOutputPastTheLimit(t *testing.T) { - var w boundedWriter - overflow := strings.Repeat("a", maxAgentOutput+1) - n, err := w.Write([]byte(overflow)) - require.NoError(t, err) - require.Equal(t, len(overflow), n) - require.Len(t, w.String(), maxAgentOutput) -} - func TestAgentRunsIsolatedWithLimitedResources(t *testing.T) { h, err := harness.ByName("claude") require.NoError(t, err) - reader := &Reader{backend: backends["claude"], harness: h, image: "agent", modelPort: "1234"} + reader := &Reader{backend: backends["claude"], harness: h, image: "agent", server: &modelproxy.Server{APIPort: "1234"}} runner := reader.runner("/host/state", "/host/snapshot", "proxy-token", allow, nil) @@ -135,7 +118,7 @@ func TestAgentRunsIsolatedWithLimitedResources(t *testing.T) { func TestOfflineAgentReachesOnlyTheModelProxy(t *testing.T) { h, err := harness.ByName("claude") require.NoError(t, err) - reader := &Reader{backend: backends["claude"], harness: h, modelPort: "1234"} + reader := &Reader{backend: backends["claude"], harness: h, server: &modelproxy.Server{APIPort: "1234"}} runner := reader.runner("/host/state", "/host/snapshot", "proxy-token", nil, nil) @@ -146,7 +129,7 @@ func TestOfflineAgentReachesOnlyTheModelProxy(t *testing.T) { func TestAccessTokenContainerGetsNoAmbientAPIKey(t *testing.T) { h, err := harness.ByName("codex") require.NoError(t, err) - reader := &Reader{backend: backends["codex-token"], harness: h, modelPort: "1234"} + reader := &Reader{backend: backends["codex-token"], harness: h, server: &modelproxy.Server{HostPorts: []string{"1234"}}} runner := reader.runner("/host/state", "/host/snapshot", "proxy-token", allow, nil) @@ -159,87 +142,23 @@ func TestAccessTokenContainerGetsNoAmbientAPIKey(t *testing.T) { func TestOnlyDockerCanRunTheAgent(t *testing.T) { h, err := harness.ByName("claude") require.NoError(t, err) - reader := &Reader{backend: backends["claude"], harness: h, runtime: container.Runtime{Bin: "podman"}, image: "agent", modelPort: "1234"} + reader := &Reader{backend: backends["claude"], harness: h, runtime: container.Runtime{Bin: "podman"}, image: "agent", server: &modelproxy.Server{APIPort: "1234"}} _, err = reader.runner(t.TempDir(), t.TempDir(), "proxy-token", allow, nil).Open(t.Context(), &agent{Harness: h}) require.Error(t, err, "an unsupported runtime must fail closed") } -func TestAgentKeepsOnlyTheFailuresOfItsOutput(t *testing.T) { - h, err := harness.ByName("codex") - require.NoError(t, err) - a := &agent{Harness: h} - var events []harness.Event - - a.ParseStream(strings.NewReader(`{"type":"thread.started","thread_id":"thread-1"}`+"\n"+`{"type":"item.completed","item":{"type":"agent_message","text":"report content"}}`+"\n"+`{"type":"turn.failed","error":{"message":"usage limit"}}`), func(e harness.Event) { events = append(events, e) }) - - require.Equal(t, `{"type":"turn.failed","error":{"message":"usage limit"}}`, a.failures.String()) - require.Len(t, events, 3) - require.Equal(t, harness.Event{Kind: harness.KindSession, SessionID: "thread-1"}, events[0]) -} - -func TestAgentLetsHarnessRestartALostConversation(t *testing.T) { - h, err := harness.ByName("claude") - require.NoError(t, err) - a := &agent{Harness: h} - var jobs []harness.Job - run := func(_ context.Context, j harness.Job, emit func(harness.Event)) error { - jobs = append(jobs, j) - if j.ResumeSessionID != "" { - a.ParseStream(strings.NewReader("No conversation found with session ID: dead\n"), emit) - return errors.New("exit status 1") - } - a.ParseStream(strings.NewReader(`{"type":"system","subtype":"init","session_id":"fresh"}`+"\n"), emit) - return nil - } - - session, err := harness.RunSession(t.Context(), run, harness.Job{Prompt: "Answer", ResumeSessionID: "dead", ResumePrompt: "Fix it"}, nil) - - require.NoError(t, err) - require.Len(t, jobs, 2, "a resume whose conversation is lost must restart fresh") - require.Empty(t, jobs[1].ResumeSessionID) - require.Equal(t, "fresh", session.ID) - require.Empty(t, a.failures.String()) -} - -func TestAgentLetsHarnessSeeTheAccountError(t *testing.T) { - h, err := harness.ByName("claude") - require.NoError(t, err) - a := &agent{Harness: h} - var seen []harness.Event - - a.ParseStream(strings.NewReader(`{"type":"error","error":"Credit balance is too low"}`+"\n"), func(e harness.Event) { seen = append(seen, e) }) - - require.True(t, slices.ContainsFunc(seen, func(e harness.Event) bool { return e.Kind == harness.KindError && h.AccountErrorText(e.Text) != "" }), "events = %+v", seen) -} - -func TestReadOutputIsBounded(t *testing.T) { - path := filepath.Join(t.TempDir(), "report.json") - for name, size := range map[string]int{"valid": 1 << 20, "too large": 1<<20 + 1} { - t.Run(name, func(t *testing.T) { - require.NoError(t, os.WriteFile(path, make([]byte, size), 0o600)) - _, err := readOutput(path) - if name == "too large" { - require.Error(t, err) - return - } - require.NoError(t, err) - }) - } -} - func TestBackends(t *testing.T) { - for name, want := range map[string]struct{ harness, keyEnv, basePath, effort string }{ - "claude": {"claude", "ANTHROPIC_API_KEY", "/model-proxy", ""}, - "codex": {"codex", "CODEX_API_KEY", "/model-proxy/v1", "medium"}, - "codex-token": {"codex", "CODEX_ACCESS_TOKEN", "/model-proxy/backend-api/codex", "medium"}, + for name, want := range map[string]struct{ harness, keyEnv, effort string }{ + "claude": {"claude", "ANTHROPIC_API_KEY", ""}, + "codex": {"codex", "CODEX_API_KEY", "medium"}, + "codex-token": {"codex", "CODEX_ACCESS_TOKEN", "medium"}, } { b, ok := backends[name] assert.True(t, ok, name) assert.Equal(t, want.harness, b.harness, name) assert.Equal(t, want.keyEnv, b.keyEnv, name) - assert.Equal(t, want.basePath, b.basePath, name) assert.Equal(t, want.effort, b.effort, name) assert.Equal(t, name == "codex-token", b.accessToken, name) } @@ -260,9 +179,13 @@ func TestWriteConfig(t *testing.T) { } { t.Run(name, func(t *testing.T) { state := t.TempDir() - r := &Reader{backend: backends[tc.backend], Daybreak: tc.daybreak, ca: []byte("CA")} + server := &modelproxy.Server{Origin: "http://proxy:1"} + if tc.backend == "codex-token" { + server = &modelproxy.Server{Origin: "https://proxy:1", CA: []byte("CA")} + } + r := &Reader{backend: backends[tc.backend], Daybreak: tc.daybreak, server: server} - err := r.writeConfig(state, "https://proxy:1") + env, err := r.writeConfig(state) if tc.wantErr { require.Error(t, err) @@ -270,20 +193,18 @@ func TestWriteConfig(t *testing.T) { require.NoError(t, err) } got, _ := os.ReadFile(filepath.Join(state, "config.toml")) - if tc.wantConfig { - require.NotEmpty(t, got) - } else { + ca, _ := os.ReadFile(filepath.Join(state, caFile)) + if !tc.wantConfig { require.Empty(t, got) + require.Empty(t, env) + return } - for _, want := range []string{"model_provider = \"tamis\"", "supports_websockets = false", "chatgpt_base_url = \"https://proxy:1/model-proxy/backend-api\"", "base_url = \"https://proxy:1" + backends[tc.backend].basePath + "\""} { - if tc.wantConfig { - require.Contains(t, string(got), want) - } - } - ca, _ := os.ReadFile(filepath.Join(state, caFile)) + require.True(t, strings.HasPrefix(string(got), fmt.Sprintf("daybreak = %t\n", tc.daybreak)), "config = %q", got) + require.Contains(t, string(got), "base_url = \""+backends[tc.backend].provider.BaseURL(server.Origin)+"\"") tokenAuth := strings.Contains(string(got), "requires_openai_auth = true") && !strings.Contains(string(got), "env_key") - require.Equal(t, tc.backend == "codex-token", tokenAuth, "access token config = %q, CA = %q", got, ca) - require.Equal(t, tokenAuth, string(ca) == "CA", "access token config = %q, CA = %q", got, ca) + require.Equal(t, tc.backend == "codex-token", tokenAuth, "config = %q", got) + require.Equal(t, tokenAuth, string(ca) == "CA", "CA = %q", ca) + require.Equal(t, tokenAuth, slices.Contains(env, "CODEX_CA_CERTIFICATE="+container.StateMount+"/"+caFile), "env = %v", env) }) } } @@ -333,70 +254,74 @@ func TestAgentImagePinsAgree(t *testing.T) { require.Equal(t, module[1], proxy[1], "harness-proxy in the agent image must match the harness module") } -func TestKeepFailuresKeepsOnlyCodexErrorEvents(t *testing.T) { - failed := `{"type":"turn.failed","error":{"message":"usage limit reached"}}` +func TestFailuresKeepTheErrorsOfACodexRun(t *testing.T) { + h, err := harness.ByName("codex") + require.NoError(t, err) stream := strings.Join([]string{ `{"type":"thread.started","thread_id":"t"}`, `{"type":"item.completed","item":{"type":"agent_message","text":"report content"}}`, `{"type":"item.completed","item":{"type":"error","message":"tool failed"}}`, - `not json`, `{"type":"error","message":"stream disconnected"}`, - failed, + `{"type":"turn.failed","error":{"message":"usage limit reached"}}`, }, "\n") - var kept strings.Builder + var failed failures - keepFailures(strings.NewReader(stream), &kept) + h.ParseStream(strings.NewReader(stream), failed.keep) - require.Equal(t, `{"type":"item.completed","item":{"type":"error","message":"tool failed"}}`+"\n"+`{"type":"error","message":"stream disconnected"}`+"\n"+failed, kept.String()) + require.Equal(t, failures{"tool failed", "stream disconnected", "usage limit reached"}, failed) } -func TestKeepFailuresSkipsALineLongerThanTheOutputLimit(t *testing.T) { - failed := `{"type":"turn.failed","error":{"message":"usage limit"}}` - stream := `{"type":"error","message":"` + strings.Repeat("a", 2*maxAgentOutput) + `"}` + "\n" + failed - var kept strings.Builder - - keepFailures(strings.NewReader(stream), &kept) - - require.Equal(t, failed, kept.String()) -} - -func TestKeepFailuresKeepsNothingFromASuccessfulRun(t *testing.T) { - var kept strings.Builder +func TestFailuresKeepNothingFromASuccessfulRun(t *testing.T) { + h, err := harness.ByName("codex") + require.NoError(t, err) + var failed failures - keepFailures(strings.NewReader(`{"type":"turn.completed","usage":{"input_tokens":1}}`+"\nnot json\n"), &kept) + h.ParseStream(strings.NewReader(`{"type":"item.completed","item":{"type":"agent_message","text":"report content"}}`+"\n"+`{"type":"turn.completed","usage":{"input_tokens":1}}`), failed.keep) - require.Empty(t, kept.String()) + require.Empty(t, failed) } -func TestSalvageKeepsAValidReportFromAFailedRun(t *testing.T) { - path := filepath.Join(t.TempDir(), "report.json") - require.NoError(t, os.WriteFile(path, []byte(`{"ok":true}`), 0o600)) +func TestOutcomeKeepsAValidReportFromAFailedRun(t *testing.T) { failure := errors.New("exit status 1") - raw, err := salvage(path, func(json.RawMessage) error { return nil }, failure) + raw, err := outcome([]byte(`{"ok":true}`), failure, failures{"usage limit reached"}) require.JSONEq(t, `{"ok":true}`, string(raw)) var incomplete *Incomplete require.ErrorAs(t, err, &incomplete) require.ErrorIs(t, err, failure, "the agent error was lost") + require.ErrorContains(t, errors.Unwrap(errors.Unwrap(err)), "usage limit reached") } -func TestSalvageFailsWithoutAValidReport(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "report.json") - require.NoError(t, os.WriteFile(path, []byte(`{"ok":false}`), 0o600)) - invalid := func(json.RawMessage) error { return errors.New("invalid") } +func TestOutcomeFailsAFailedRunWithoutAValidReport(t *testing.T) { + failure := errors.New("exit status 1") - for name, path := range map[string]string{"invalid": path, "missing": filepath.Join(dir, "missing.json")} { - raw, err := salvage(path, invalid, errors.New("exit status 1")) + for name, err := range map[string]error{"run": failure, "run and repair": errors.Join(&harness.ValidationError{Detail: "invalid"}, failure)} { + raw, err := outcome(nil, err, nil) require.Nil(t, raw, name) var agent *Error require.ErrorAs(t, err, &agent, name) + require.ErrorIs(t, err, failure, name) + require.NotErrorIs(t, err, ErrNoReport, name) var incomplete *Incomplete require.False(t, errors.As(err, &incomplete), name) } } +func TestOutcomeReportsAnInvalidReport(t *testing.T) { + raw, err := outcome(nil, &harness.ValidationError{Detail: "invalid"}, nil) + + require.Nil(t, raw) + require.ErrorIs(t, err, ErrNoReport) +} + +func TestOutcomeReturnsAValidReport(t *testing.T) { + raw, err := outcome([]byte(`{"ok":true}`), nil, nil) + + require.NoError(t, err) + require.JSONEq(t, `{"ok":true}`, string(raw)) +} + func TestWriteWorkspaceWritesTheRequestFiles(t *testing.T) { workspace := t.TempDir() diff --git a/renovate.json b/renovate.json index 0f902d7..6a7f42c 100644 --- a/renovate.json +++ b/renovate.json @@ -1,34 +1,14 @@ { "$schema": "https://docs.renovatebot.com/renovate-schema.json", - "extends": ["config:recommended", ":disableDependencyDashboard"], + "extends": [ + "config:recommended", + ":disableDependencyDashboard", + "github>alpha-omega-security/harness//renovate/agent-clis#v0.1.20" + ], "enabledManagers": ["gomod", "github-actions", "dockerfile", "custom.regex"], "automerge": false, "internalChecksFilter": "strict", "customManagers": [ - { - "customType": "regex", - "description": "Update Claude Code release assets and their SHA-256 pins", - "managerFilePatterns": ["/^docker/agent/Dockerfile$/"], - "matchStrings": [ - "ARG CLAUDE_(?:AMD64|ARM64)_LOCK=(?v\\d+\\.\\d+\\.\\d+)@sha256:(?[a-f0-9]{64})" - ], - "packageNameTemplate": "anthropics/claude-code", - "depNameTemplate": "anthropics/claude-code", - "datasourceTemplate": "github-release-attachments", - "versioningTemplate": "semver-coerced" - }, - { - "customType": "regex", - "description": "Update Codex CLI package assets and their SHA-256 pins", - "managerFilePatterns": ["/^docker/agent/Dockerfile$/"], - "matchStrings": [ - "ARG CODEX_(?:AMD64|ARM64)_LOCK=(?rust-v\\d+\\.\\d+\\.\\d+)@sha256:(?[a-f0-9]{64})" - ], - "packageNameTemplate": "openai/codex", - "depNameTemplate": "openai/codex", - "datasourceTemplate": "github-release-attachments", - "versioningTemplate": "regex:^rust-v(?\\d+)\\.(?\\d+)\\.(?\\d+)(?:-(?.+))?$" - }, { "customType": "regex", "description": "Update the harness-proxy version built into the agent image", @@ -75,18 +55,6 @@ "matchManagers": ["custom.regex"], "minimumReleaseAge": "7 days" }, - { - "description": "Keep both Claude Code assets on one release", - "matchPackageNames": ["anthropics/claude-code"], - "groupName": "Claude Code", - "minimumGroupSize": 2 - }, - { - "description": "Keep both Codex assets on one release", - "matchPackageNames": ["openai/codex"], - "groupName": "Codex CLI", - "minimumGroupSize": 2 - }, { "description": "harness-proxy in the agent image must match the harness module tamis is built with", "matchPackageNames": ["github.com/alpha-omega-security/harness"],