diff --git a/docker/agent/Dockerfile b/docker/agent/Dockerfile index fc5b497..0ab18c6 100644 --- a/docker/agent/Dockerfile +++ b/docker/agent/Dockerfile @@ -9,7 +9,7 @@ ARG CLAUDE_AMD64_LOCK=v2.1.306@sha256:9dc640d16d8228eff9620bde9e0f70d38385a2124f ARG CLAUDE_ARM64_LOCK=v2.1.306@sha256:871fd7f46210c13a95a38506e03ac90abfc5125fc25a1d4f840fbab551b5cd2e ARG CODEX_AMD64_LOCK=rust-v0.163.0@sha256:582646c6bf0561eb0de540f7fc85b9121b0091a826bc6e96830b7f1722122182 ARG CODEX_ARM64_LOCK=rust-v0.163.0@sha256:9d8ecf2d19d3e184877983902d6c85de48b3888dcb72c382a8a0c975845927fb -ARG HARNESS_VERSION=v0.1.19 +ARG HARNESS_VERSION=v0.1.20 FROM golang:1.27.3-trixie@sha256:1896654cfc5c2b5ad74c07f29cb36f5d2d1cbed42e3054070fbfe444e521e83c AS proxy ARG HARNESS_VERSION diff --git a/go.mod b/go.mod index 2da55bc..12a35f4 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/php/tamis go 1.27.3 require ( - github.com/alpha-omega-security/harness v0.1.19 + github.com/alpha-omega-security/harness v0.1.20 github.com/anthropics/anthropic-sdk-go v1.84.0 github.com/git-pkgs/cwe v0.1.1 github.com/openai/openai-go/v3 v3.85.1 @@ -32,6 +32,7 @@ require ( github.com/mattn/go-isatty v0.0.24 // indirect github.com/pb33f/go-yaml v0.1.0 // indirect github.com/pb33f/ordered-map/v2 v2.3.2 // indirect + github.com/santhosh-tekuri/jsonschema/v6 v6.0.3 // indirect github.com/standard-webhooks/standard-webhooks/libraries v0.0.1 // indirect github.com/tidwall/gjson v1.20.0 // indirect github.com/tidwall/match v1.2.0 // indirect diff --git a/go.sum b/go.sum index 5b4807c..104a933 100644 --- a/go.sum +++ b/go.sum @@ -10,8 +10,8 @@ github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 h1:fhqpLE3UEXi9lPaBRpQ6Xu github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0/go.mod h1:7dCRMLwisfRH3dBupKeNCioWYUZ4SS09Z14H+7i8ZoY= github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c h1:pxW6RcqyfI9/kWtOwnv/G+AzdKuy2ZrqINhenH4HyNs= github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= -github.com/alpha-omega-security/harness v0.1.19 h1:X/EI0++ki34CFCROOCqfmiUSzodumqlJ0LLJks7N2Ws= -github.com/alpha-omega-security/harness v0.1.19/go.mod h1:4JI0CxVRwiWQ9zAtSTnM5p8l0A79bwREtps9BA9oLXY= +github.com/alpha-omega-security/harness v0.1.20 h1:BmWRfH13mycUnzzu51mkYH0HZtoKrvkeTr82dfTsbnc= +github.com/alpha-omega-security/harness v0.1.20/go.mod h1:A3YtZQo/uHk0t0BLNGLE8Ov9Lga7g5w0g2QYMu5z7oM= github.com/anthropics/anthropic-sdk-go v1.84.0 h1:mnAhggc5H5A98JYmhQELT+j4J89PYBFKnnSvUTNwNd8= github.com/anthropics/anthropic-sdk-go v1.84.0/go.mod h1:GThfYqPJoaQ/6pmibCI98Cr4y5su2FXMUHn3NrSSnIc= github.com/aws/aws-sdk-go-v2 v1.47.1 h1:uOIZnp4PK3ZhKI0dNrJrhTEsLxbpXHTAJlwoS1pvAtw= @@ -54,6 +54,8 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UF github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/coder/websocket v1.8.15 h1:6B2JPeOGlpff2Uz6vOEH1Vzpi0iUz20A+lPVhPHtNUA= github.com/coder/websocket v1.8.15/go.mod h1:NX3SzP+inril6yawo5CQXx8+fk145lPDC6pumgx0mVg= +github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI= +github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= github.com/dnephin/pflag v1.0.7 h1:oxONGlWxhmUct0YzKTgrpQv9AUA1wtPBn7zuSjJqptk= @@ -109,6 +111,8 @@ github.com/pb33f/ordered-map/v2 v2.3.2 h1:wDyaZ2Pv9QLh64X4utCeD5Zoi9nIv2aW3lwv17 github.com/pb33f/ordered-map/v2 v2.3.2/go.mod h1:1OhFrXu3OYw3kM+FXF+Ug3ugmmZ9LE8z0JfQMLvtV0w= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/santhosh-tekuri/jsonschema/v6 v6.0.3 h1:1EYB5IzjZawrrnELUi78f9fPu57HuXjmddZPjrls/28= +github.com/santhosh-tekuri/jsonschema/v6 v6.0.3/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= github.com/segmentio/asm v1.1.3 h1:WM03sfUOENvvKexOLp+pCqgb/WDjsi7EK8gIsICtzhc= github.com/segmentio/asm v1.1.3/go.mod h1:Ld3L4ZXGNcSLRg4JBsZ3//1+f/TjYl0Mzen/DQy1EJg= github.com/segmentio/encoding v0.5.4 h1:OW1VRern8Nw6ITAtwSZ7Idrl3MXCFwXHPgqESYfvNt0= diff --git a/internal/sandbox/runner.go b/internal/sandbox/runner.go index 799d484..2e4d300 100644 --- a/internal/sandbox/runner.go +++ b/internal/sandbox/runner.go @@ -1,53 +1,30 @@ package sandbox import ( - "bufio" - "bytes" - "encoding/json" - "errors" - "io" - "github.com/alpha-omega-security/harness" "github.com/alpha-omega-security/harness/container" ) -const maxAgentOutput = 1 << 20 - -// agent keeps the failure events of a run's output, which otherwise holds the report and the agent's -// reasoning about it. type agent struct { harness.Harness - failures boundedWriter } // EgressHosts is empty so the egress proxy allows the model proxy and the request's hosts only, not the // provider's own API. func (*agent) EgressHosts() []string { return nil } -// ParseStream hands every event to harness, which reads the session, a lost conversation and the account -// errors from them. Read emits none of them further. -func (a *agent) ParseStream(r io.Reader, emit func(harness.Event)) { - pr, pw := io.Pipe() - done := make(chan struct{}) - go func() { - a.Harness.ParseStream(pr, emit) - _, _ = io.Copy(io.Discard, pr) - close(done) - }() - keepFailures(io.TeeReader(r, pw), &a.failures) - _ = pw.Close() - <-done +// failures keeps the errors a run reports among its events. Codex exec --json reports why it stopped only there. +type failures []string + +func (f *failures) keep(e harness.Event) { + if e.Kind == harness.KindError { + *f = append(*f, e.Text) + } } // runner keeps the agent on a network whose only way out is the egress proxy, allowing the model proxy and -// hosts. harness refuses tunnels to its API port, so a model proxy served over TLS is reached as a separate -// host port. +// hosts. func (r *Reader) runner(state, source, token string, hosts, env []string) container.Runner { - sidecar := container.SidecarConfig{Allow: hosts, APIPort: r.modelPort} - if r.backend.accessToken { - sidecar.APIPort, sidecar.HostPorts = "", []string{r.modelPort} - } - return container.Runner{ Runtime: r.runtime, Image: r.image, @@ -59,7 +36,7 @@ func (r *Reader) runner(state, source, token string, hosts, env []string) contai ProcessEnv: []string{r.backend.keyEnv + "=" + token}, Hardened: true, IsolateHost: true, - Sidecar: sidecar, + Sidecar: container.SidecarConfig{Allow: hosts, APIPort: r.server.APIPort, HostPorts: r.server.HostPorts}, ReadOnlyWorkspace: true, Memory: "2g", CPUs: "2", @@ -67,44 +44,3 @@ func (r *Reader) runner(state, source, token string, hosts, env []string) contai FileSizeLimit: 64 << 20, } } - -// keepFailures copies the error events of a JSONL agent output and discards the rest, lines longer than -// maxAgentOutput included. Codex exec --json -// reports why it stopped only there, as "error" and "turn.failed" events or "error" items. -func keepFailures(r io.Reader, w io.Writer) { - br := bufio.NewReaderSize(r, maxAgentOutput) - for { - line, err := br.ReadSlice('\n') - for errors.Is(err, bufio.ErrBufferFull) { - line = nil - _, err = br.ReadSlice('\n') - } - var event struct { - Type string `json:"type"` - Item struct { - Type string `json:"type"` - } `json:"item"` - } - if json.Unmarshal(line, &event) == nil && (event.Type == "error" || event.Type == "turn.failed" || event.Item.Type == "error") { - _, _ = w.Write(line) - } - if err != nil { - _, _ = io.Copy(io.Discard, r) - return - } - } -} - -type boundedWriter struct { - b bytes.Buffer -} - -func (w *boundedWriter) Write(p []byte) (int, error) { - if remaining := maxAgentOutput - w.b.Len(); remaining > 0 { - w.b.Write(p[:min(remaining, len(p))]) - } - - return len(p), nil -} - -func (w *boundedWriter) String() string { return w.b.String() } diff --git a/internal/sandbox/runner_integration_test.go b/internal/sandbox/runner_integration_test.go index 680a219..1da101f 100644 --- a/internal/sandbox/runner_integration_test.go +++ b/internal/sandbox/runner_integration_test.go @@ -11,7 +11,6 @@ import ( "io" "net" "net/http" - "net/http/httptest" "os" "os/exec" "path/filepath" @@ -25,6 +24,8 @@ import ( "github.com/alpha-omega-security/harness" "github.com/alpha-omega-security/harness/container" "github.com/alpha-omega-security/harness/egress" + "github.com/alpha-omega-security/harness/modelproxy" + "github.com/alpha-omega-security/harness/modelproxy/modelproxytest" "github.com/stretchr/testify/require" ) @@ -44,7 +45,7 @@ func TestIsolatedNetworkReachesTheModelButNotOtherHostServices(t *testing.T) { require.NotEmpty(t, hostIP, "resolving the real host gateway") h, err := harness.ByName("claude") require.NoError(t, err) - reader := &Reader{backend: backends["claude"], harness: h, runtime: runtime, image: image, modelPort: modelPort} + reader := &Reader{backend: backends["claude"], harness: h, runtime: runtime, image: image, server: &modelproxy.Server{APIPort: modelPort}} env := []string{"MODEL_URL=http://" + egress.HostGatewayAlias + ":" + modelPort, "SECRET_URL=http://" + net.JoinHostPort(hostIP, secretPort), "PROXIED_SECRET_URL=http://" + egress.HostGatewayAlias + ":" + secretPort} scope, err := reader.runner(t.TempDir(), t.TempDir(), "proxy-token", allow, env).Open(ctx, &agent{Harness: h}) require.NoError(t, err) @@ -84,93 +85,23 @@ func testAgentCanReachTheFilteredModelProxy(t *testing.T, name, model, imageSuff var calls atomic.Int32 var first atomic.Value upstream, port := testServerHandler(t, func(w http.ResponseWriter, r *http.Request) { - switch { - case strings.HasSuffix(r.URL.Path, "/v1/user-auth-credential/whoami"): - _, _ = io.WriteString(w, `{"email":"private@example.com","chatgpt_user_id":"private-user","chatgpt_account_id":"private-account","chatgpt_plan_type":"enterprise","chatgpt_account_is_fedramp":false}`) - case strings.HasSuffix(r.URL.Path, "/wham/config/bundle"): - _, _ = io.WriteString(w, `{}`) - case strings.HasSuffix(r.URL.Path, "/codex/models"): - _, _ = io.WriteString(w, `{"models":[]}`) - default: - if calls.Add(1) == 1 { - body, _ := io.ReadAll(r.Body) - first.Store(string(body)) - w.Header().Set("Content-Type", "text/event-stream") - event := func(kind string, data any) { - raw, _ := json.Marshal(data) - _, _ = fmt.Fprintf(w, "event: %s\ndata: %s\n\n", kind, raw) - } - if name == "claude" { - event("message_start", map[string]any{"type": "message_start", "message": map[string]any{"id": "msg_fixture", "type": "message", "role": "assistant", "model": model, "content": []any{}, "stop_reason": nil, "usage": map[string]int{"input_tokens": 1, "output_tokens": 0}}}) - event("content_block_start", map[string]any{"type": "content_block_start", "index": 0, "content_block": map[string]any{"type": "tool_use", "id": "tool_fixture", "name": "Bash", "input": map[string]any{}}}) - event("content_block_delta", map[string]any{"type": "content_block_delta", "index": 0, "delta": map[string]any{"type": "input_json_delta", "partial_json": `{"command":"printf fixture"}`}}) - event("content_block_stop", map[string]any{"type": "content_block_stop", "index": 0}) - event("message_delta", map[string]any{"type": "message_delta", "delta": map[string]any{"stop_reason": "tool_use", "stop_sequence": nil}, "usage": map[string]int{"output_tokens": 1}}) - event("message_stop", map[string]any{"type": "message_stop"}) - } else { - tool := map[string]any{"type": "function_call", "id": "fc_fixture", "call_id": "call_fixture", "name": "exec_command", "arguments": `{"cmd":"printf fixture"}`} - event("response.created", map[string]any{"type": "response.created", "response": map[string]any{"id": "resp_fixture", "status": "in_progress", "output": []any{}}}) - event("response.output_item.done", map[string]any{"type": "response.output_item.done", "output_index": 0, "item": tool}) - event("response.completed", map[string]any{"type": "response.completed", "response": map[string]any{"id": "resp_fixture", "status": "completed", "model": model, "output": []any{tool}, "usage": map[string]int{"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}}}) - } - return - } + if modelproxytest.CodexStartup(w, r) { + return + } + if calls.Add(1) != 1 { http.Error(w, `{"type":"error","error":{"type":"invalid_request_error","message":"stop"}}`, http.StatusBadRequest) + return } - }) - t.Cleanup(func() { _ = upstream.Close() }) - reader := testReader(t, name, model, image, port) - reader.server.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { body, _ := io.ReadAll(r.Body) - r.Body = io.NopCloser(bytes.NewReader(body)) - recorder := httptest.NewRecorder() - reader.models.ServeHTTP(recorder, r) - if recorder.Code == http.StatusForbidden { - var value map[string]any - _ = json.Unmarshal(body, &value) - keys := func(m map[string]any) []string { - var keys []string - for key := range m { - keys = append(keys, key) - } - return keys - } - t.Logf("rejected fixture request fields: %v; context: %v; beta: %v; reason: %s", keys(value), value["context_management"], r.Header.Get("Anthropic-Beta"), recorder.Body.String()) - if blocks, ok := value["system"].([]any); ok { - for _, block := range blocks { - if block, ok := block.(map[string]any); ok { - t.Logf("system block %v: %v", block["type"], keys(block)) - } - } - } - if messages, ok := value["messages"].([]any); ok { - for _, message := range messages { - if message, ok := message.(map[string]any); ok { - t.Logf("message fields: %v", keys(message)) - if blocks, ok := message["content"].([]any); ok { - for _, block := range blocks { - if block, ok := block.(map[string]any); ok { - t.Logf("message block %v: %v", block["type"], keys(block)) - } - } - } - } - } - } - if ts, ok := value["tools"].([]any); ok { - for _, tool := range ts { - if tool, ok := tool.(map[string]any); ok { - t.Logf("tool type %v fields: %v", tool["type"], keys(tool)) - } - } - } - } - for name, values := range recorder.Header() { - w.Header()[name] = values + first.Store(string(body)) + if name == "claude" { + modelproxytest.AnthropicToolUse(w, model, "tool_fixture", "Bash", `{"command":"printf fixture"}`) + return } - w.WriteHeader(recorder.Code) - _, _ = w.Write(recorder.Body.Bytes()) + modelproxytest.ResponsesFunctionCall(w, model, "call_fixture", "exec_command", `{"cmd":"printf fixture"}`) }) + t.Cleanup(func() { _ = upstream.Close() }) + reader := testReader(t, name, model, image, port) _, readErr := reader.Read(t.Context(), Request{RepoPath: gitRepo(t), Prompt: "Answer", System: "System", Report: "Report", Schema: []byte(`{}`), Validate: func(json.RawMessage) error { return nil }}) require.GreaterOrEqual(t, calls.Load(), int32(2), "the legitimate %s request was blocked before the model provider: %v", name, errors.Unwrap(readErr)) if name != "claude" { @@ -220,32 +151,18 @@ func readWithCorrection(t *testing.T, image, corrected string) (json.RawMessage, break } } - w.Header().Set("Content-Type", "text/event-stream") - event := func(kind string, data any) { - raw, _ := json.Marshal(data) - _, _ = fmt.Fprintf(w, "event: %s\ndata: %s\n\n", kind, raw) - } - event("message_start", map[string]any{"type": "message_start", "message": map[string]any{"id": "msg_fixture", "type": "message", "role": "assistant", "model": "claude-opus-5-5", "content": []any{}, "stop_reason": nil, "usage": map[string]int{"input_tokens": 1, "output_tokens": 0}}}) - stop := "end_turn" if called { - event("content_block_start", map[string]any{"type": "content_block_start", "index": 0, "content_block": map[string]any{"type": "text", "text": ""}}) - event("content_block_delta", map[string]any{"type": "content_block_delta", "index": 0, "delta": map[string]any{"type": "text_delta", "text": "Done."}}) - } else { - output := "not json" - if bytes.Contains(body, []byte("did not produce valid output")) { - output = corrected - resumed = bytes.Contains(body, []byte(`"tool_use_id"`)) - } - command, _ := json.Marshal(map[string]string{"command": "printf '%s' '" + output + "' > /work/" + outputFile}) - stop = "tool_use" - // A resumed conversation already holds the first run's tool call, so each call needs its own id. - id := fmt.Sprintf("tool_fixture_%d", calls.Add(1)) - event("content_block_start", map[string]any{"type": "content_block_start", "index": 0, "content_block": map[string]any{"type": "tool_use", "id": id, "name": "Bash", "input": map[string]any{}}}) - event("content_block_delta", map[string]any{"type": "content_block_delta", "index": 0, "delta": map[string]any{"type": "input_json_delta", "partial_json": string(command)}}) + modelproxytest.AnthropicText(w, "claude-opus-5-5", "Done.") + return + } + output := "not json" + if bytes.Contains(body, []byte("failed validation")) { + output = corrected + resumed = bytes.Contains(body, []byte(`"tool_use_id"`)) } - event("content_block_stop", map[string]any{"type": "content_block_stop", "index": 0}) - event("message_delta", map[string]any{"type": "message_delta", "delta": map[string]any{"stop_reason": stop, "stop_sequence": nil}, "usage": map[string]int{"output_tokens": 1}}) - event("message_stop", map[string]any{"type": "message_stop"}) + command, _ := json.Marshal(map[string]string{"command": "printf '%s' '" + output + "' > /work/" + outputFile}) + // A resumed conversation already holds the first run's tool call, so each call needs its own id. + modelproxytest.AnthropicToolUse(w, "claude-opus-5-5", fmt.Sprintf("tool_fixture_%d", calls.Add(1)), "Bash", string(command)) }) reader := testReader(t, "claude", "claude-opus-5-5", image, port) diff --git a/internal/sandbox/sandbox.go b/internal/sandbox/sandbox.go index 89e592a..da57df1 100644 --- a/internal/sandbox/sandbox.go +++ b/internal/sandbox/sandbox.go @@ -5,24 +5,18 @@ package sandbox import ( "context" - "crypto/tls" _ "embed" "encoding/json" "errors" "fmt" "log/slog" - "net" - "net/http" "os" "os/exec" "path/filepath" - "strconv" "strings" - "time" "github.com/alpha-omega-security/harness" "github.com/alpha-omega-security/harness/container" - "github.com/alpha-omega-security/harness/egress" "github.com/alpha-omega-security/harness/modelproxy" "github.com/php/tamis/internal/budget" @@ -44,17 +38,15 @@ type backend struct { harness string provider modelproxy.Provider keyEnv string - // basePath completes the model proxy URL the way the CLI expects its API base URL. - basePath string // accessToken signs Codex in with a Codex access token, which it only sends over HTTPS. accessToken bool effort string } var backends = map[string]backend{ - "claude": {"claude", modelproxy.Anthropic, "ANTHROPIC_API_KEY", modelproxy.PathPrefix, false, ""}, - "codex": {"codex", modelproxy.OpenAI, "CODEX_API_KEY", modelproxy.PathPrefix + "/v1", false, "medium"}, - "codex-token": {"codex", modelproxy.ChatGPT, "CODEX_ACCESS_TOKEN", modelproxy.PathPrefix + "/backend-api/codex", true, "medium"}, + "claude": {"claude", modelproxy.Anthropic, "ANTHROPIC_API_KEY", false, ""}, + "codex": {"codex", modelproxy.OpenAI, "CODEX_API_KEY", false, "medium"}, + "codex-token": {"codex", modelproxy.ChatGPT, "CODEX_ACCESS_TOKEN", true, "medium"}, } // imagesJSON pins each agent image by digest, so a version of tamis always runs the images built for it. @@ -73,16 +65,13 @@ var Images = func() map[string]string { }() type Reader struct { - backend backend - harness harness.Harness - model string - image string - runtime container.Runtime - models *modelproxy.Proxy - modelPort string - server *http.Server - // ca is the PEM certificate the container trusts for the model proxy, with an access token only. - ca []byte + backend backend + harness harness.Harness + model string + image string + runtime container.Runtime + models *modelproxy.Proxy + server *modelproxy.Server // Daybreak turns on OpenAI's cyber access program in Codex. The API key's org must be approved for it. Daybreak bool @@ -126,33 +115,19 @@ func New(name, model, apiKey, image string) (*Reader, error) { if err != nil { return nil, err } - ln, err := net.Listen("tcp", ":0") + server, err := modelproxy.Serve(models, b.accessToken) if err != nil { return nil, err } - server := &http.Server{Handler: models, ReadHeaderTimeout: 10 * time.Second} - var ca []byte - if b.accessToken { - var cert tls.Certificate - if ca, cert, err = modelproxy.NewCertificate(egress.HostGatewayAlias); err != nil { - ln.Close() - return nil, err - } - ln = tls.NewListener(ln, &tls.Config{Certificates: []tls.Certificate{cert}, MinVersion: tls.VersionTLS12}) - } - go func() { _ = server.Serve(ln) }() - modelPort := strconv.Itoa(ln.Addr().(*net.TCPAddr).Port) return &Reader{ - backend: b, - harness: h, - model: model, - image: image, - runtime: rt, - models: models, - modelPort: modelPort, - server: server, - ca: ca, + backend: b, + harness: h, + model: model, + image: image, + runtime: rt, + models: models, + server: server, }, nil } @@ -192,17 +167,11 @@ func (r *Reader) Read(ctx context.Context, req Request) (json.RawMessage, error) if req.Offline { hosts = nil } - origin := "http://" + egress.HostGatewayAlias + ":" + r.modelPort - var env []string - if r.backend.accessToken { - origin = "https://" + egress.HostGatewayAlias + ":" + r.modelPort - env = []string{"CODEX_CA_CERTIFICATE=" + container.StateMount + "/" + caFile, "CODEX_AUTHAPI_BASE_URL=" + origin + modelproxy.PathPrefix + "/auth"} - } - if err := r.writeConfig(state, origin); err != nil { + env, err := r.writeConfig(state) + if err != nil { return nil, err } - a := &agent{Harness: r.harness} - scope, err := r.runner(state, source, token, hosts, env).Open(ctx, a) + scope, err := r.runner(state, source, token, hosts, env).Open(ctx, &agent{Harness: r.harness}) if err != nil { return nil, err } @@ -216,32 +185,32 @@ func (r *Reader) Read(ctx context.Context, req Request) (json.RawMessage, error) Effort: r.backend.effort, MaxTurns: maxTurns, OutputFile: outputFile, - BaseURL: origin + r.backend.basePath, + BaseURL: r.backend.provider.BaseURL(r.server.Origin), } - for attempt := 0; attempt < 2; attempt++ { - a.failures = boundedWriter{} - session, err := harness.RunSession(ctx, scope.Run, job, nil) - if err != nil { - if failures := strings.TrimSpace(a.failures.String()); failures != "" { - err = fmt.Errorf("%w: %s", err, failures) - } - return salvage(filepath.Join(workspace, outputFile), req.Validate, err) - } + var failed failures + raw, _, err := harness.RunRepaired(ctx, scope.Run, job, harness.Repair{Validate: func(raw []byte) error { return req.Validate(raw) }}, failed.keep) - raw, err := readOutput(filepath.Join(workspace, outputFile)) - if err == nil { - if err = req.Validate(raw); err == nil { - return raw, nil - } - } - // Prompt only serves when the agent announced no session or the resume cannot load it. - correction := fmt.Sprintf("Your previous attempt did not produce valid output: %v. Write /work/%s again, fixed.", err, outputFile) - job.Prompt = req.Prompt + "\n\n" + correction - job.ResumeSessionID, job.ResumePrompt = session.ID, correction + return outcome(raw, err, failed) +} + +// outcome keeps a valid report the agent wrote before failing: Codex has no turn limit to stop on before its budget. +func outcome(raw []byte, err error, failed failures) (json.RawMessage, error) { + if err == nil { + return raw, nil + } + // RunRepaired joins the validation error with the error of a failed run. + if _, invalid := err.(*harness.ValidationError); invalid { + return nil, ErrNoReport + } + if len(failed) > 0 { + err = fmt.Errorf("%w: %s", err, strings.Join(failed, "\n")) + } + if raw == nil { + return nil, &Error{Err: err} } - return nil, ErrNoReport + return raw, &Incomplete{Err: &Error{Err: err}} } func writeWorkspace(workspace string, req Request) error { @@ -265,47 +234,27 @@ func writeWorkspace(workspace string, req Request) error { return nil } -// salvage keeps a valid report the agent wrote before failing: Codex has no turn limit to stop on before its budget. -func salvage(path string, validate func(json.RawMessage) error, failure error) (json.RawMessage, error) { - raw, err := readOutput(path) - if err != nil || validate(raw) != nil { - return nil, &Error{Err: failure} - } - - return raw, &Incomplete{Err: &Error{Err: failure}} -} - -func readOutput(path string) ([]byte, error) { - data, err := os.ReadFile(path) - if err != nil { - return nil, err - } - if len(data) > 1<<20 { - return nil, errors.New("report is too large") - } - - return data, nil -} - -// writeConfig writes the agent CLI configuration into its state directory, which harness points CODEX_HOME at. -func (r *Reader) writeConfig(state, origin string) error { +// writeConfig writes the agent CLI configuration into its state directory, which harness points CODEX_HOME at, +// and returns the environment the CLI needs with it. +func (r *Reader) writeConfig(state string) ([]string, error) { if r.backend.harness != "codex" { if !r.Daybreak { - return nil + return nil, nil } - return errors.New("daybreak is only available with codex") + return nil, errors.New("daybreak is only available with codex") } - auth := "env_key = \"CODEX_API_KEY\"" - if r.backend.accessToken { - auth = "requires_openai_auth = true" - if err := os.WriteFile(filepath.Join(state, caFile), r.ca, 0o600); err != nil { - return err + if r.server.CA != nil { + if err := os.WriteFile(filepath.Join(state, caFile), r.server.CA, 0o600); err != nil { + return nil, err } } - // The ChatGPT features are off: they would call routes the model proxy refuses. - config := fmt.Sprintf("model_provider = \"tamis\"\ndaybreak = %t\nweb_search = \"disabled\"\nchatgpt_base_url = %q\n\n[features]\nremote_plugin = false\nplugin_sharing = false\ngoals = false\nimage_generation = false\nskill_mcp_dependency_install = false\n\n[model_providers.tamis]\nname = \"Tamis\"\nbase_url = %q\n%s\nwire_api = \"responses\"\nsupports_websockets = false\nsupports_standalone_web_search = false\nrequest_max_retries = 0\nstream_max_retries = 0\n", r.Daybreak, origin+modelproxy.PathPrefix+"/backend-api", origin+r.backend.basePath, auth) - return os.WriteFile(filepath.Join(state, "config.toml"), []byte(config), 0o600) + config, env, err := modelproxy.CodexConfig(r.backend.provider, r.server.Origin, container.StateMount+"/"+caFile) + if err != nil { + return nil, err + } + + return env, os.WriteFile(filepath.Join(state, "config.toml"), append(fmt.Appendf(nil, "daybreak = %t\n", r.Daybreak), config...), 0o600) } // SnapshotSource keeps the sandbox on tracked content while retaining the current commit's public history. diff --git a/internal/sandbox/sandbox_test.go b/internal/sandbox/sandbox_test.go index 0cc7080..2941e4d 100644 --- a/internal/sandbox/sandbox_test.go +++ b/internal/sandbox/sandbox_test.go @@ -2,8 +2,8 @@ package sandbox import ( "context" - "encoding/json" "errors" + "fmt" "os" "os/exec" "path/filepath" @@ -14,6 +14,7 @@ import ( "github.com/alpha-omega-security/harness" "github.com/alpha-omega-security/harness/container" + "github.com/alpha-omega-security/harness/modelproxy" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -99,28 +100,10 @@ func TestErrorHidesTheAgentOutput(t *testing.T) { require.ErrorContains(t, errors.Unwrap(err), "SECRET-PAYLOAD") } -func TestBoundedWriterRetainsOnlyTheOutputLimit(t *testing.T) { - var w boundedWriter - want := strings.Repeat("a", maxAgentOutput) - n, err := w.Write([]byte(want)) - require.NoError(t, err) - require.Equal(t, len(want), n) - require.Equal(t, want, w.String()) -} - -func TestBoundedWriterDiscardsOutputPastTheLimit(t *testing.T) { - var w boundedWriter - overflow := strings.Repeat("a", maxAgentOutput+1) - n, err := w.Write([]byte(overflow)) - require.NoError(t, err) - require.Equal(t, len(overflow), n) - require.Len(t, w.String(), maxAgentOutput) -} - func TestAgentRunsIsolatedWithLimitedResources(t *testing.T) { h, err := harness.ByName("claude") require.NoError(t, err) - reader := &Reader{backend: backends["claude"], harness: h, image: "agent", modelPort: "1234"} + reader := &Reader{backend: backends["claude"], harness: h, image: "agent", server: &modelproxy.Server{APIPort: "1234"}} runner := reader.runner("/host/state", "/host/snapshot", "proxy-token", allow, nil) @@ -135,7 +118,7 @@ func TestAgentRunsIsolatedWithLimitedResources(t *testing.T) { func TestOfflineAgentReachesOnlyTheModelProxy(t *testing.T) { h, err := harness.ByName("claude") require.NoError(t, err) - reader := &Reader{backend: backends["claude"], harness: h, modelPort: "1234"} + reader := &Reader{backend: backends["claude"], harness: h, server: &modelproxy.Server{APIPort: "1234"}} runner := reader.runner("/host/state", "/host/snapshot", "proxy-token", nil, nil) @@ -146,7 +129,7 @@ func TestOfflineAgentReachesOnlyTheModelProxy(t *testing.T) { func TestAccessTokenContainerGetsNoAmbientAPIKey(t *testing.T) { h, err := harness.ByName("codex") require.NoError(t, err) - reader := &Reader{backend: backends["codex-token"], harness: h, modelPort: "1234"} + reader := &Reader{backend: backends["codex-token"], harness: h, server: &modelproxy.Server{HostPorts: []string{"1234"}}} runner := reader.runner("/host/state", "/host/snapshot", "proxy-token", allow, nil) @@ -159,87 +142,23 @@ func TestAccessTokenContainerGetsNoAmbientAPIKey(t *testing.T) { func TestOnlyDockerCanRunTheAgent(t *testing.T) { h, err := harness.ByName("claude") require.NoError(t, err) - reader := &Reader{backend: backends["claude"], harness: h, runtime: container.Runtime{Bin: "podman"}, image: "agent", modelPort: "1234"} + reader := &Reader{backend: backends["claude"], harness: h, runtime: container.Runtime{Bin: "podman"}, image: "agent", server: &modelproxy.Server{APIPort: "1234"}} _, err = reader.runner(t.TempDir(), t.TempDir(), "proxy-token", allow, nil).Open(t.Context(), &agent{Harness: h}) require.Error(t, err, "an unsupported runtime must fail closed") } -func TestAgentKeepsOnlyTheFailuresOfItsOutput(t *testing.T) { - h, err := harness.ByName("codex") - require.NoError(t, err) - a := &agent{Harness: h} - var events []harness.Event - - a.ParseStream(strings.NewReader(`{"type":"thread.started","thread_id":"thread-1"}`+"\n"+`{"type":"item.completed","item":{"type":"agent_message","text":"report content"}}`+"\n"+`{"type":"turn.failed","error":{"message":"usage limit"}}`), func(e harness.Event) { events = append(events, e) }) - - require.Equal(t, `{"type":"turn.failed","error":{"message":"usage limit"}}`, a.failures.String()) - require.Len(t, events, 3) - require.Equal(t, harness.Event{Kind: harness.KindSession, SessionID: "thread-1"}, events[0]) -} - -func TestAgentLetsHarnessRestartALostConversation(t *testing.T) { - h, err := harness.ByName("claude") - require.NoError(t, err) - a := &agent{Harness: h} - var jobs []harness.Job - run := func(_ context.Context, j harness.Job, emit func(harness.Event)) error { - jobs = append(jobs, j) - if j.ResumeSessionID != "" { - a.ParseStream(strings.NewReader("No conversation found with session ID: dead\n"), emit) - return errors.New("exit status 1") - } - a.ParseStream(strings.NewReader(`{"type":"system","subtype":"init","session_id":"fresh"}`+"\n"), emit) - return nil - } - - session, err := harness.RunSession(t.Context(), run, harness.Job{Prompt: "Answer", ResumeSessionID: "dead", ResumePrompt: "Fix it"}, nil) - - require.NoError(t, err) - require.Len(t, jobs, 2, "a resume whose conversation is lost must restart fresh") - require.Empty(t, jobs[1].ResumeSessionID) - require.Equal(t, "fresh", session.ID) - require.Empty(t, a.failures.String()) -} - -func TestAgentLetsHarnessSeeTheAccountError(t *testing.T) { - h, err := harness.ByName("claude") - require.NoError(t, err) - a := &agent{Harness: h} - var seen []harness.Event - - a.ParseStream(strings.NewReader(`{"type":"error","error":"Credit balance is too low"}`+"\n"), func(e harness.Event) { seen = append(seen, e) }) - - require.True(t, slices.ContainsFunc(seen, func(e harness.Event) bool { return e.Kind == harness.KindError && h.AccountErrorText(e.Text) != "" }), "events = %+v", seen) -} - -func TestReadOutputIsBounded(t *testing.T) { - path := filepath.Join(t.TempDir(), "report.json") - for name, size := range map[string]int{"valid": 1 << 20, "too large": 1<<20 + 1} { - t.Run(name, func(t *testing.T) { - require.NoError(t, os.WriteFile(path, make([]byte, size), 0o600)) - _, err := readOutput(path) - if name == "too large" { - require.Error(t, err) - return - } - require.NoError(t, err) - }) - } -} - func TestBackends(t *testing.T) { - for name, want := range map[string]struct{ harness, keyEnv, basePath, effort string }{ - "claude": {"claude", "ANTHROPIC_API_KEY", "/model-proxy", ""}, - "codex": {"codex", "CODEX_API_KEY", "/model-proxy/v1", "medium"}, - "codex-token": {"codex", "CODEX_ACCESS_TOKEN", "/model-proxy/backend-api/codex", "medium"}, + for name, want := range map[string]struct{ harness, keyEnv, effort string }{ + "claude": {"claude", "ANTHROPIC_API_KEY", ""}, + "codex": {"codex", "CODEX_API_KEY", "medium"}, + "codex-token": {"codex", "CODEX_ACCESS_TOKEN", "medium"}, } { b, ok := backends[name] assert.True(t, ok, name) assert.Equal(t, want.harness, b.harness, name) assert.Equal(t, want.keyEnv, b.keyEnv, name) - assert.Equal(t, want.basePath, b.basePath, name) assert.Equal(t, want.effort, b.effort, name) assert.Equal(t, name == "codex-token", b.accessToken, name) } @@ -260,9 +179,13 @@ func TestWriteConfig(t *testing.T) { } { t.Run(name, func(t *testing.T) { state := t.TempDir() - r := &Reader{backend: backends[tc.backend], Daybreak: tc.daybreak, ca: []byte("CA")} + server := &modelproxy.Server{Origin: "http://proxy:1"} + if tc.backend == "codex-token" { + server = &modelproxy.Server{Origin: "https://proxy:1", CA: []byte("CA")} + } + r := &Reader{backend: backends[tc.backend], Daybreak: tc.daybreak, server: server} - err := r.writeConfig(state, "https://proxy:1") + env, err := r.writeConfig(state) if tc.wantErr { require.Error(t, err) @@ -270,20 +193,18 @@ func TestWriteConfig(t *testing.T) { require.NoError(t, err) } got, _ := os.ReadFile(filepath.Join(state, "config.toml")) - if tc.wantConfig { - require.NotEmpty(t, got) - } else { + ca, _ := os.ReadFile(filepath.Join(state, caFile)) + if !tc.wantConfig { require.Empty(t, got) + require.Empty(t, env) + return } - for _, want := range []string{"model_provider = \"tamis\"", "supports_websockets = false", "chatgpt_base_url = \"https://proxy:1/model-proxy/backend-api\"", "base_url = \"https://proxy:1" + backends[tc.backend].basePath + "\""} { - if tc.wantConfig { - require.Contains(t, string(got), want) - } - } - ca, _ := os.ReadFile(filepath.Join(state, caFile)) + require.True(t, strings.HasPrefix(string(got), fmt.Sprintf("daybreak = %t\n", tc.daybreak)), "config = %q", got) + require.Contains(t, string(got), "base_url = \""+backends[tc.backend].provider.BaseURL(server.Origin)+"\"") tokenAuth := strings.Contains(string(got), "requires_openai_auth = true") && !strings.Contains(string(got), "env_key") - require.Equal(t, tc.backend == "codex-token", tokenAuth, "access token config = %q, CA = %q", got, ca) - require.Equal(t, tokenAuth, string(ca) == "CA", "access token config = %q, CA = %q", got, ca) + require.Equal(t, tc.backend == "codex-token", tokenAuth, "config = %q", got) + require.Equal(t, tokenAuth, string(ca) == "CA", "CA = %q", ca) + require.Equal(t, tokenAuth, slices.Contains(env, "CODEX_CA_CERTIFICATE="+container.StateMount+"/"+caFile), "env = %v", env) }) } } @@ -333,70 +254,74 @@ func TestAgentImagePinsAgree(t *testing.T) { require.Equal(t, module[1], proxy[1], "harness-proxy in the agent image must match the harness module") } -func TestKeepFailuresKeepsOnlyCodexErrorEvents(t *testing.T) { - failed := `{"type":"turn.failed","error":{"message":"usage limit reached"}}` +func TestFailuresKeepTheErrorsOfACodexRun(t *testing.T) { + h, err := harness.ByName("codex") + require.NoError(t, err) stream := strings.Join([]string{ `{"type":"thread.started","thread_id":"t"}`, `{"type":"item.completed","item":{"type":"agent_message","text":"report content"}}`, `{"type":"item.completed","item":{"type":"error","message":"tool failed"}}`, - `not json`, `{"type":"error","message":"stream disconnected"}`, - failed, + `{"type":"turn.failed","error":{"message":"usage limit reached"}}`, }, "\n") - var kept strings.Builder + var failed failures - keepFailures(strings.NewReader(stream), &kept) + h.ParseStream(strings.NewReader(stream), failed.keep) - require.Equal(t, `{"type":"item.completed","item":{"type":"error","message":"tool failed"}}`+"\n"+`{"type":"error","message":"stream disconnected"}`+"\n"+failed, kept.String()) + require.Equal(t, failures{"tool failed", "stream disconnected", "usage limit reached"}, failed) } -func TestKeepFailuresSkipsALineLongerThanTheOutputLimit(t *testing.T) { - failed := `{"type":"turn.failed","error":{"message":"usage limit"}}` - stream := `{"type":"error","message":"` + strings.Repeat("a", 2*maxAgentOutput) + `"}` + "\n" + failed - var kept strings.Builder - - keepFailures(strings.NewReader(stream), &kept) - - require.Equal(t, failed, kept.String()) -} - -func TestKeepFailuresKeepsNothingFromASuccessfulRun(t *testing.T) { - var kept strings.Builder +func TestFailuresKeepNothingFromASuccessfulRun(t *testing.T) { + h, err := harness.ByName("codex") + require.NoError(t, err) + var failed failures - keepFailures(strings.NewReader(`{"type":"turn.completed","usage":{"input_tokens":1}}`+"\nnot json\n"), &kept) + h.ParseStream(strings.NewReader(`{"type":"item.completed","item":{"type":"agent_message","text":"report content"}}`+"\n"+`{"type":"turn.completed","usage":{"input_tokens":1}}`), failed.keep) - require.Empty(t, kept.String()) + require.Empty(t, failed) } -func TestSalvageKeepsAValidReportFromAFailedRun(t *testing.T) { - path := filepath.Join(t.TempDir(), "report.json") - require.NoError(t, os.WriteFile(path, []byte(`{"ok":true}`), 0o600)) +func TestOutcomeKeepsAValidReportFromAFailedRun(t *testing.T) { failure := errors.New("exit status 1") - raw, err := salvage(path, func(json.RawMessage) error { return nil }, failure) + raw, err := outcome([]byte(`{"ok":true}`), failure, failures{"usage limit reached"}) require.JSONEq(t, `{"ok":true}`, string(raw)) var incomplete *Incomplete require.ErrorAs(t, err, &incomplete) require.ErrorIs(t, err, failure, "the agent error was lost") + require.ErrorContains(t, errors.Unwrap(errors.Unwrap(err)), "usage limit reached") } -func TestSalvageFailsWithoutAValidReport(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "report.json") - require.NoError(t, os.WriteFile(path, []byte(`{"ok":false}`), 0o600)) - invalid := func(json.RawMessage) error { return errors.New("invalid") } +func TestOutcomeFailsAFailedRunWithoutAValidReport(t *testing.T) { + failure := errors.New("exit status 1") - for name, path := range map[string]string{"invalid": path, "missing": filepath.Join(dir, "missing.json")} { - raw, err := salvage(path, invalid, errors.New("exit status 1")) + for name, err := range map[string]error{"run": failure, "run and repair": errors.Join(&harness.ValidationError{Detail: "invalid"}, failure)} { + raw, err := outcome(nil, err, nil) require.Nil(t, raw, name) var agent *Error require.ErrorAs(t, err, &agent, name) + require.ErrorIs(t, err, failure, name) + require.NotErrorIs(t, err, ErrNoReport, name) var incomplete *Incomplete require.False(t, errors.As(err, &incomplete), name) } } +func TestOutcomeReportsAnInvalidReport(t *testing.T) { + raw, err := outcome(nil, &harness.ValidationError{Detail: "invalid"}, nil) + + require.Nil(t, raw) + require.ErrorIs(t, err, ErrNoReport) +} + +func TestOutcomeReturnsAValidReport(t *testing.T) { + raw, err := outcome([]byte(`{"ok":true}`), nil, nil) + + require.NoError(t, err) + require.JSONEq(t, `{"ok":true}`, string(raw)) +} + func TestWriteWorkspaceWritesTheRequestFiles(t *testing.T) { workspace := t.TempDir() diff --git a/renovate.json b/renovate.json index 0f902d7..6a7f42c 100644 --- a/renovate.json +++ b/renovate.json @@ -1,34 +1,14 @@ { "$schema": "https://docs.renovatebot.com/renovate-schema.json", - "extends": ["config:recommended", ":disableDependencyDashboard"], + "extends": [ + "config:recommended", + ":disableDependencyDashboard", + "github>alpha-omega-security/harness//renovate/agent-clis#v0.1.20" + ], "enabledManagers": ["gomod", "github-actions", "dockerfile", "custom.regex"], "automerge": false, "internalChecksFilter": "strict", "customManagers": [ - { - "customType": "regex", - "description": "Update Claude Code release assets and their SHA-256 pins", - "managerFilePatterns": ["/^docker/agent/Dockerfile$/"], - "matchStrings": [ - "ARG CLAUDE_(?:AMD64|ARM64)_LOCK=(?v\\d+\\.\\d+\\.\\d+)@sha256:(?[a-f0-9]{64})" - ], - "packageNameTemplate": "anthropics/claude-code", - "depNameTemplate": "anthropics/claude-code", - "datasourceTemplate": "github-release-attachments", - "versioningTemplate": "semver-coerced" - }, - { - "customType": "regex", - "description": "Update Codex CLI package assets and their SHA-256 pins", - "managerFilePatterns": ["/^docker/agent/Dockerfile$/"], - "matchStrings": [ - "ARG CODEX_(?:AMD64|ARM64)_LOCK=(?rust-v\\d+\\.\\d+\\.\\d+)@sha256:(?[a-f0-9]{64})" - ], - "packageNameTemplate": "openai/codex", - "depNameTemplate": "openai/codex", - "datasourceTemplate": "github-release-attachments", - "versioningTemplate": "regex:^rust-v(?\\d+)\\.(?\\d+)\\.(?\\d+)(?:-(?.+))?$" - }, { "customType": "regex", "description": "Update the harness-proxy version built into the agent image", @@ -75,18 +55,6 @@ "matchManagers": ["custom.regex"], "minimumReleaseAge": "7 days" }, - { - "description": "Keep both Claude Code assets on one release", - "matchPackageNames": ["anthropics/claude-code"], - "groupName": "Claude Code", - "minimumGroupSize": 2 - }, - { - "description": "Keep both Codex assets on one release", - "matchPackageNames": ["openai/codex"], - "groupName": "Codex CLI", - "minimumGroupSize": 2 - }, { "description": "harness-proxy in the agent image must match the harness module tamis is built with", "matchPackageNames": ["github.com/alpha-omega-security/harness"],