Move to harness v0.1.20 and use what it now provides

docker/agent/Dockerfile

@@ -9,7 +9,7 @@ ARG CLAUDE_AMD64_LOCK=v2.1.306@sha256:9dc640d16d8228eff9620bde9e0f70d38385a2124f
 ARG CLAUDE_ARM64_LOCK=v2.1.306@sha256:871fd7f46210c13a95a38506e03ac90abfc5125fc25a1d4f840fbab551b5cd2e
 ARG CODEX_AMD64_LOCK=rust-v0.163.0@sha256:582646c6bf0561eb0de540f7fc85b9121b0091a826bc6e96830b7f1722122182
 ARG CODEX_ARM64_LOCK=rust-v0.163.0@sha256:9d8ecf2d19d3e184877983902d6c85de48b3888dcb72c382a8a0c975845927fb
-ARG HARNESS_VERSION=v0.1.19
+ARG HARNESS_VERSION=v0.1.20
 
 FROM golang:1.27.3-trixie@sha256:1896654cfc5c2b5ad74c07f29cb36f5d2d1cbed42e3054070fbfe444e521e83c AS proxy
 ARG HARNESS_VERSION

go.mod

@@ -3,7 +3,7 @@ module github.com/php/tamis
 go 1.27.3
 
 require (
-	github.com/alpha-omega-security/harness v0.1.19
+	github.com/alpha-omega-security/harness v0.1.20
 	github.com/anthropics/anthropic-sdk-go v1.84.0
 	github.com/git-pkgs/cwe v0.1.1
 	github.com/openai/openai-go/v3 v3.85.1
@@ -32,6 +32,7 @@ require (
 	github.com/mattn/go-isatty v0.0.24 // indirect
 	github.com/pb33f/go-yaml v0.1.0 // indirect
 	github.com/pb33f/ordered-map/v2 v2.3.2 // indirect
+	github.com/santhosh-tekuri/jsonschema/v6 v6.0.3 // indirect
 	github.com/standard-webhooks/standard-webhooks/libraries v0.0.1 // indirect
 	github.com/tidwall/gjson v1.20.0 // indirect
 	github.com/tidwall/match v1.2.0 // indirect

go.sum

@@ -10,8 +10,8 @@ github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 h1:fhqpLE3UEXi9lPaBRpQ6Xu
 github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0/go.mod h1:7dCRMLwisfRH3dBupKeNCioWYUZ4SS09Z14H+7i8ZoY=
 github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c h1:pxW6RcqyfI9/kWtOwnv/G+AzdKuy2ZrqINhenH4HyNs=
 github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
-github.com/alpha-omega-security/harness v0.1.19 h1:X/EI0++ki34CFCROOCqfmiUSzodumqlJ0LLJks7N2Ws=
-github.com/alpha-omega-security/harness v0.1.19/go.mod h1:4JI0CxVRwiWQ9zAtSTnM5p8l0A79bwREtps9BA9oLXY=
+github.com/alpha-omega-security/harness v0.1.20 h1:BmWRfH13mycUnzzu51mkYH0HZtoKrvkeTr82dfTsbnc=
+github.com/alpha-omega-security/harness v0.1.20/go.mod h1:A3YtZQo/uHk0t0BLNGLE8Ov9Lga7g5w0g2QYMu5z7oM=
 github.com/anthropics/anthropic-sdk-go v1.84.0 h1:mnAhggc5H5A98JYmhQELT+j4J89PYBFKnnSvUTNwNd8=
 github.com/anthropics/anthropic-sdk-go v1.84.0/go.mod h1:GThfYqPJoaQ/6pmibCI98Cr4y5su2FXMUHn3NrSSnIc=
 github.com/aws/aws-sdk-go-v2 v1.47.1 h1:uOIZnp4PK3ZhKI0dNrJrhTEsLxbpXHTAJlwoS1pvAtw=
@@ -54,6 +54,8 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UF
 github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/coder/websocket v1.8.15 h1:6B2JPeOGlpff2Uz6vOEH1Vzpi0iUz20A+lPVhPHtNUA=
 github.com/coder/websocket v1.8.15/go.mod h1:NX3SzP+inril6yawo5CQXx8+fk145lPDC6pumgx0mVg=
+github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
+github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
 github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI=
 github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ=
 github.com/dnephin/pflag v1.0.7 h1:oxONGlWxhmUct0YzKTgrpQv9AUA1wtPBn7zuSjJqptk=
@@ -109,6 +111,8 @@ github.com/pb33f/ordered-map/v2 v2.3.2 h1:wDyaZ2Pv9QLh64X4utCeD5Zoi9nIv2aW3lwv17
 github.com/pb33f/ordered-map/v2 v2.3.2/go.mod h1:1OhFrXu3OYw3kM+FXF+Ug3ugmmZ9LE8z0JfQMLvtV0w=
 github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
 github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
+github.com/santhosh-tekuri/jsonschema/v6 v6.0.3 h1:1EYB5IzjZawrrnELUi78f9fPu57HuXjmddZPjrls/28=
+github.com/santhosh-tekuri/jsonschema/v6 v6.0.3/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU=
 github.com/segmentio/asm v1.1.3 h1:WM03sfUOENvvKexOLp+pCqgb/WDjsi7EK8gIsICtzhc=
 github.com/segmentio/asm v1.1.3/go.mod h1:Ld3L4ZXGNcSLRg4JBsZ3//1+f/TjYl0Mzen/DQy1EJg=
 github.com/segmentio/encoding v0.5.4 h1:OW1VRern8Nw6ITAtwSZ7Idrl3MXCFwXHPgqESYfvNt0=

internal/sandbox/runner.go

@@ -1,53 +1,30 @@
 package sandbox
 
 import (
-	"bufio"
-	"bytes"
-	"encoding/json"
-	"errors"
-	"io"
-
 	"github.com/alpha-omega-security/harness"
 	"github.com/alpha-omega-security/harness/container"
 )
 
-const maxAgentOutput = 1 << 20
-
-// agent keeps the failure events of a run's output, which otherwise holds the report and the agent's
-// reasoning about it.
 type agent struct {
 	harness.Harness
-	failures boundedWriter
 }
 
 // EgressHosts is empty so the egress proxy allows the model proxy and the request's hosts only, not the
 // provider's own API.
 func (*agent) EgressHosts() []string { return nil }
 
-// ParseStream hands every event to harness, which reads the session, a lost conversation and the account
-// errors from them. Read emits none of them further.
-func (a *agent) ParseStream(r io.Reader, emit func(harness.Event)) {
-	pr, pw := io.Pipe()
-	done := make(chan struct{})
-	go func() {
-		a.Harness.ParseStream(pr, emit)
-		_, _ = io.Copy(io.Discard, pr)
-		close(done)
-	}()
-	keepFailures(io.TeeReader(r, pw), &a.failures)
-	_ = pw.Close()
-	<-done
+// failures keeps the errors a run reports among its events. Codex exec --json reports why it stopped only there.
+type failures []string
+
+func (f *failures) keep(e harness.Event) {
+	if e.Kind == harness.KindError {
+		*f = append(*f, e.Text)
+	}
 }
 
 // runner keeps the agent on a network whose only way out is the egress proxy, allowing the model proxy and
-// hosts. harness refuses tunnels to its API port, so a model proxy served over TLS is reached as a separate
-// host port.
+// hosts.
 func (r *Reader) runner(state, source, token string, hosts, env []string) container.Runner {
-	sidecar := container.SidecarConfig{Allow: hosts, APIPort: r.modelPort}
-	if r.backend.accessToken {
-		sidecar.APIPort, sidecar.HostPorts = "", []string{r.modelPort}
-	}
-
 	return container.Runner{
 		Runtime:  r.runtime,
 		Image:    r.image,
@@ -59,52 +36,11 @@ func (r *Reader) runner(state, source, token string, hosts, env []string) contai
 		ProcessEnv:        []string{r.backend.keyEnv + "=" + token},
 		Hardened:          true,
 		IsolateHost:       true,
-		Sidecar:           sidecar,
+		Sidecar:           container.SidecarConfig{Allow: hosts, APIPort: r.server.APIPort, HostPorts: r.server.HostPorts},
 		ReadOnlyWorkspace: true,
 		Memory:            "2g",
 		CPUs:              "2",
 		PIDsLimit:         256,
 		FileSizeLimit:     64 << 20,
 	}
 }
-
-// keepFailures copies the error events of a JSONL agent output and discards the rest, lines longer than
-// maxAgentOutput included. Codex exec --json
-// reports why it stopped only there, as "error" and "turn.failed" events or "error" items.
-func keepFailures(r io.Reader, w io.Writer) {
-	br := bufio.NewReaderSize(r, maxAgentOutput)
-	for {
-		line, err := br.ReadSlice('\n')
-		for errors.Is(err, bufio.ErrBufferFull) {
-			line = nil
-			_, err = br.ReadSlice('\n')
-		}
-		var event struct {
-			Type string `json:"type"`
-			Item struct {
-				Type string `json:"type"`
-			} `json:"item"`
-		}
-		if json.Unmarshal(line, &event) == nil && (event.Type == "error" || event.Type == "turn.failed" || event.Item.Type == "error") {
-			_, _ = w.Write(line)
-		}
-		if err != nil {
-			_, _ = io.Copy(io.Discard, r)
-			return
-		}
-	}
-}
-
-type boundedWriter struct {
-	b bytes.Buffer
-}
-
-func (w *boundedWriter) Write(p []byte) (int, error) {
-	if remaining := maxAgentOutput - w.b.Len(); remaining > 0 {
-		w.b.Write(p[:min(remaining, len(p))])
-	}
-
-	return len(p), nil
-}
-
-func (w *boundedWriter) String() string { return w.b.String() }

internal/sandbox/runner_integration_test.go

@@ -11,7 +11,6 @@ import (
 	"io"
 	"net"
 	"net/http"
-	"net/http/httptest"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -25,6 +24,8 @@ import (
 	"github.com/alpha-omega-security/harness"
 	"github.com/alpha-omega-security/harness/container"
 	"github.com/alpha-omega-security/harness/egress"
+	"github.com/alpha-omega-security/harness/modelproxy"
+	"github.com/alpha-omega-security/harness/modelproxy/modelproxytest"
 	"github.com/stretchr/testify/require"
 )
 
@@ -44,7 +45,7 @@ func TestIsolatedNetworkReachesTheModelButNotOtherHostServices(t *testing.T) {
 	require.NotEmpty(t, hostIP, "resolving the real host gateway")
 	h, err := harness.ByName("claude")
 	require.NoError(t, err)
-	reader := &Reader{backend: backends["claude"], harness: h, runtime: runtime, image: image, modelPort: modelPort}
+	reader := &Reader{backend: backends["claude"], harness: h, runtime: runtime, image: image, server: &modelproxy.Server{APIPort: modelPort}}
 	env := []string{"MODEL_URL=http://" + egress.HostGatewayAlias + ":" + modelPort, "SECRET_URL=http://" + net.JoinHostPort(hostIP, secretPort), "PROXIED_SECRET_URL=http://" + egress.HostGatewayAlias + ":" + secretPort}
 	scope, err := reader.runner(t.TempDir(), t.TempDir(), "proxy-token", allow, env).Open(ctx, &agent{Harness: h})
 	require.NoError(t, err)
@@ -84,93 +85,23 @@ func testAgentCanReachTheFilteredModelProxy(t *testing.T, name, model, imageSuff
 	var calls atomic.Int32
 	var first atomic.Value
 	upstream, port := testServerHandler(t, func(w http.ResponseWriter, r *http.Request) {
-		switch {
-		case strings.HasSuffix(r.URL.Path, "/v1/user-auth-credential/whoami"):
-			_, _ = io.WriteString(w, `{"email":"private@example.com","chatgpt_user_id":"private-user","chatgpt_account_id":"private-account","chatgpt_plan_type":"enterprise","chatgpt_account_is_fedramp":false}`)
-		case strings.HasSuffix(r.URL.Path, "/wham/config/bundle"):
-			_, _ = io.WriteString(w, `{}`)
-		case strings.HasSuffix(r.URL.Path, "/codex/models"):
-			_, _ = io.WriteString(w, `{"models":[]}`)
-		default:
-			if calls.Add(1) == 1 {
-				body, _ := io.ReadAll(r.Body)
-				first.Store(string(body))
-				w.Header().Set("Content-Type", "text/event-stream")
-				event := func(kind string, data any) {
-					raw, _ := json.Marshal(data)
-					_, _ = fmt.Fprintf(w, "event: %s\ndata: %s\n\n", kind, raw)
-				}
-				if name == "claude" {
-					event("message_start", map[string]any{"type": "message_start", "message": map[string]any{"id": "msg_fixture", "type": "message", "role": "assistant", "model": model, "content": []any{}, "stop_reason": nil, "usage": map[string]int{"input_tokens": 1, "output_tokens": 0}}})
-					event("content_block_start", map[string]any{"type": "content_block_start", "index": 0, "content_block": map[string]any{"type": "tool_use", "id": "tool_fixture", "name": "Bash", "input": map[string]any{}}})
-					event("content_block_delta", map[string]any{"type": "content_block_delta", "index": 0, "delta": map[string]any{"type": "input_json_delta", "partial_json": `{"command":"printf fixture"}`}})
-					event("content_block_stop", map[string]any{"type": "content_block_stop", "index": 0})
-					event("message_delta", map[string]any{"type": "message_delta", "delta": map[string]any{"stop_reason": "tool_use", "stop_sequence": nil}, "usage": map[string]int{"output_tokens": 1}})
-					event("message_stop", map[string]any{"type": "message_stop"})
-				} else {
-					tool := map[string]any{"type": "function_call", "id": "fc_fixture", "call_id": "call_fixture", "name": "exec_command", "arguments": `{"cmd":"printf fixture"}`}
-					event("response.created", map[string]any{"type": "response.created", "response": map[string]any{"id": "resp_fixture", "status": "in_progress", "output": []any{}}})
-					event("response.output_item.done", map[string]any{"type": "response.output_item.done", "output_index": 0, "item": tool})
-					event("response.completed", map[string]any{"type": "response.completed", "response": map[string]any{"id": "resp_fixture", "status": "completed", "model": model, "output": []any{tool}, "usage": map[string]int{"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}}})
-				}
-				return
-			}
+		if modelproxytest.CodexStartup(w, r) {
+			return
+		}
+		if calls.Add(1) != 1 {
 			http.Error(w, `{"type":"error","error":{"type":"invalid_request_error","message":"stop"}}`, http.StatusBadRequest)
+			return
 		}
-	})
-	t.Cleanup(func() { _ = upstream.Close() })
-	reader := testReader(t, name, model, image, port)
-	reader.server.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		body, _ := io.ReadAll(r.Body)
-		r.Body = io.NopCloser(bytes.NewReader(body))
-		recorder := httptest.NewRecorder()
-		reader.models.ServeHTTP(recorder, r)
-		if recorder.Code == http.StatusForbidden {
-			var value map[string]any
-			_ = json.Unmarshal(body, &value)
-			keys := func(m map[string]any) []string {
-				var keys []string
-				for key := range m {
-					keys = append(keys, key)
-				}
-				return keys
-			}
-			t.Logf("rejected fixture request fields: %v; context: %v; beta: %v; reason: %s", keys(value), value["context_management"], r.Header.Get("Anthropic-Beta"), recorder.Body.String())
-			if blocks, ok := value["system"].([]any); ok {
-				for _, block := range blocks {
-					if block, ok := block.(map[string]any); ok {
-						t.Logf("system block %v: %v", block["type"], keys(block))
-					}
-				}
-			}
-			if messages, ok := value["messages"].([]any); ok {
-				for _, message := range messages {
-					if message, ok := message.(map[string]any); ok {
-						t.Logf("message fields: %v", keys(message))
-						if blocks, ok := message["content"].([]any); ok {
-							for _, block := range blocks {
-								if block, ok := block.(map[string]any); ok {
-									t.Logf("message block %v: %v", block["type"], keys(block))
-								}
-							}
-						}
-					}
-				}
-			}
-			if ts, ok := value["tools"].([]any); ok {
-				for _, tool := range ts {
-					if tool, ok := tool.(map[string]any); ok {
-						t.Logf("tool type %v fields: %v", tool["type"], keys(tool))
-					}
-				}
-			}
-		}
-		for name, values := range recorder.Header() {
-			w.Header()[name] = values
+		first.Store(string(body))
+		if name == "claude" {
+			modelproxytest.AnthropicToolUse(w, model, "tool_fixture", "Bash", `{"command":"printf fixture"}`)
+			return
 		}
-		w.WriteHeader(recorder.Code)
-		_, _ = w.Write(recorder.Body.Bytes())
+		modelproxytest.ResponsesFunctionCall(w, model, "call_fixture", "exec_command", `{"cmd":"printf fixture"}`)
 	})
+	t.Cleanup(func() { _ = upstream.Close() })
+	reader := testReader(t, name, model, image, port)
 	_, readErr := reader.Read(t.Context(), Request{RepoPath: gitRepo(t), Prompt: "Answer", System: "System", Report: "Report", Schema: []byte(`{}`), Validate: func(json.RawMessage) error { return nil }})
 	require.GreaterOrEqual(t, calls.Load(), int32(2), "the legitimate %s request was blocked before the model provider: %v", name, errors.Unwrap(readErr))
 	if name != "claude" {
@@ -220,32 +151,18 @@ func readWithCorrection(t *testing.T, image, corrected string) (json.RawMessage,
 				break
 			}
 		}
-		w.Header().Set("Content-Type", "text/event-stream")
-		event := func(kind string, data any) {
-			raw, _ := json.Marshal(data)
-			_, _ = fmt.Fprintf(w, "event: %s\ndata: %s\n\n", kind, raw)
-		}
-		event("message_start", map[string]any{"type": "message_start", "message": map[string]any{"id": "msg_fixture", "type": "message", "role": "assistant", "model": "claude-opus-5-5", "content": []any{}, "stop_reason": nil, "usage": map[string]int{"input_tokens": 1, "output_tokens": 0}}})
-		stop := "end_turn"
 		if called {
-			event("content_block_start", map[string]any{"type": "content_block_start", "index": 0, "content_block": map[string]any{"type": "text", "text": ""}})
-			event("content_block_delta", map[string]any{"type": "content_block_delta", "index": 0, "delta": map[string]any{"type": "text_delta", "text": "Done."}})
-		} else {
-			output := "not json"
-			if bytes.Contains(body, []byte("did not produce valid output")) {
-				output = corrected
-				resumed = bytes.Contains(body, []byte(`"tool_use_id"`))
-			}
-			command, _ := json.Marshal(map[string]string{"command": "printf '%s' '" + output + "' > /work/" + outputFile})
-			stop = "tool_use"
-			// A resumed conversation already holds the first run's tool call, so each call needs its own id.
-			id := fmt.Sprintf("tool_fixture_%d", calls.Add(1))
-			event("content_block_start", map[string]any{"type": "content_block_start", "index": 0, "content_block": map[string]any{"type": "tool_use", "id": id, "name": "Bash", "input": map[string]any{}}})
-			event("content_block_delta", map[string]any{"type": "content_block_delta", "index": 0, "delta": map[string]any{"type": "input_json_delta", "partial_json": string(command)}})
+			modelproxytest.AnthropicText(w, "claude-opus-5-5", "Done.")
+			return
+		}
+		output := "not json"
+		if bytes.Contains(body, []byte("failed validation")) {
+			output = corrected
+			resumed = bytes.Contains(body, []byte(`"tool_use_id"`))
 		}
-		event("content_block_stop", map[string]any{"type": "content_block_stop", "index": 0})
-		event("message_delta", map[string]any{"type": "message_delta", "delta": map[string]any{"stop_reason": stop, "stop_sequence": nil}, "usage": map[string]int{"output_tokens": 1}})
-		event("message_stop", map[string]any{"type": "message_stop"})
+		command, _ := json.Marshal(map[string]string{"command": "printf '%s' '" + output + "' > /work/" + outputFile})
+		// A resumed conversation already holds the first run's tool call, so each call needs its own id.
+		modelproxytest.AnthropicToolUse(w, "claude-opus-5-5", fmt.Sprintf("tool_fixture_%d", calls.Add(1)), "Bash", string(command))
 	})
 	reader := testReader(t, "claude", "claude-opus-5-5", image, port)