diff --git a/README.md b/README.md index d36bab401d..553d1a2b17 100644 --- a/README.md +++ b/README.md @@ -180,7 +180,8 @@ it ran (role, model, a number, which model judged, door, size bucket, day) under a per-install nonce, never code, prompts, paths or an identity, and `codeaf pool status` shows exactly what is waiting to go. Turn it off with `model_pool = off` on the settings sheet or -`CODEAF_MODEL_POOL=off`; `read` uses the pool and sends nothing. The relay +`CODEAF_MODEL_POOL=off`; `read` uses the pool and sends nothing, and +`CODEAF_TELEMETRY=off` caps it at `read` along with the usage counts. The relay publishes a signed index the crew picker reads under `picked from = learn`. The index is mirrored on the `model-pool` branch at `pool/index.json`. The design is [Pareto Crewing](docs/design/model-pool/pareto-crewing.pdf); the relay's code is under `relay/`, with a [runbook](docs/design/model-pool/RUNBOOK.md) @@ -283,7 +284,7 @@ codeaf sends anonymous usage counts to AgentField. Sent: version, OS, mode (chat or task), how many sessions, how many errors. Never: anything about you or your work. No prompts, code, file names, paths, repo names, keys, email, IP, or machine name. - See exactly what leaves: codeaf telemetry show + What is collected: codeaf telemetry info Turn off: CODEAF_TELEMETRY=off ``` diff --git a/cmd/codeaf/main.go b/cmd/codeaf/main.go index 7f9019080a..9daac79708 100644 --- a/cmd/codeaf/main.go +++ b/cmd/codeaf/main.go @@ -490,7 +490,7 @@ Look at what happened — read-only, no key, nothing spent codeaf why [--db path] what one piece of work did — its turns, tools, arguments, how it ended codeaf telemetry - the anonymous usage counts: status, show, off, on + the anonymous usage counts: status, info, show, off, on codeaf logs [--tail 40] [--follow] [--path] [--json] [--run id] [--call id] [--tag t] [--model m] [--node n] [--body id] every model call codeaf made — what was asked, which lane answered, what diff --git a/cmd/codeaf/pool.go b/cmd/codeaf/pool.go index 4fdf763282..c1b8e486c4 100644 --- a/cmd/codeaf/pool.go +++ b/cmd/codeaf/pool.go @@ -123,7 +123,7 @@ func runPool(args []string) error { // emptiness is the state root's own profile, never a directory called "pool" // beside wherever the command happened to run. func runPoolWith(args []string, output io.Writer, profileDir string, now func() time.Time, lookup func(string) (string, bool)) error { - cfg := poolcfg.Resolve(config.ModelPoolSettingAt(profileDir), config.ModelPoolPublicKeySettingAt(profileDir), lookup) + cfg := config.ModelPoolResolved(profileDir, lookup) poolDir := config.ProfilePath(profileDir, "pool") if len(args) == 0 { args = []string{"show"} @@ -180,8 +180,8 @@ func statusPool(args []string, output io.Writer, poolDir string, cfg poolcfg.Con } // printPool is the reading form's whole answer. The config first — every value -// beside the word saying where it came from, one of default, setting, env or -// ci — then the cached index with its age, its cells under --cells, then the +// beside the word saying where it came from, one of default, setting, env, ci +// or telemetry — then the cached index with its age, its cells under --cells, then the // install's own sheet, then, for status, the outbox and the two doors the // mode opens. func printPool(output io.Writer, poolDir string, cfg poolcfg.Config, now time.Time, asJSON, withCells, withStatus bool, keys []ed25519.PublicKey) error { diff --git a/cmd/codeaf/telemetry.go b/cmd/codeaf/telemetry.go index 792ee28e72..8b76b04774 100644 --- a/cmd/codeaf/telemetry.go +++ b/cmd/codeaf/telemetry.go @@ -1,20 +1,28 @@ package main import ( + "bytes" + "encoding/json" "flag" "fmt" "os" + "path/filepath" "strings" + "time" "github.com/Agent-Field/codeaf/internal/config" + "github.com/Agent-Field/codeaf/internal/pool/outbox" + "github.com/Agent-Field/codeaf/internal/pool/poolcfg" + "github.com/Agent-Field/codeaf/internal/pool/record" "github.com/Agent-Field/codeaf/internal/telemetry" ) // The `telemetry` command: the person's door onto the anonymous-usage pipe. -// Four verbs, one per question a person arrives with — what is it doing, what -// exactly would leave, and the two ways of turning it off or back on. It -// EMITS NOTHING ITSELF: it is a command about telemetry, not a session, and -// the wiring in main.go's execute() looks at os.Args to make sure of it. +// Five verbs, one per question a person arrives with — what is it doing, what +// is collected, what is waiting to leave right now, and the two ways of +// turning it off or back on. It EMITS NOTHING ITSELF: it is a command about +// telemetry, not a session, and the wiring in main.go's execute() looks at +// os.Args to make sure of it. func runTelemetry(args []string) error { if len(args) == 0 { return runTelemetryStatus(nil) @@ -22,12 +30,14 @@ func runTelemetry(args []string) error { switch args[0] { case "status": return runTelemetryStatus(args[1:]) + case "info": + return runTelemetryInfo(args[1:]) case "show": return runTelemetryShow(args[1:]) case "on", "off": return runTelemetrySet(args[0], args[1:]) default: - return fmt.Errorf("telemetry takes one of: status, show, on, off") + return fmt.Errorf("telemetry takes one of: status, info, show, on, off") } } @@ -89,16 +99,269 @@ func telemetryInstallPrefix() string { return hash } -// runTelemetryShow prints exactly what is waiting to leave the machine, the -// package's own rendering so the command and `codeaf telemetry show` a person -// reads in the notice are one and the same thing. +// runTelemetryInfo prints what is collected — the shape of every row that can +// leave, in a person's words, for both streams. The notice promises "what is +// collected: codeaf telemetry info", and two streams leave: the anonymous +// usage counts this package spools, and the Model Pool's judged seat scores, +// which wait in the pool's own outbox under the profile and go to a different +// relay under a different switch. Until 2026-09-18 the only listing covered +// the first, so a person who read it and set CODEAF_TELEMETRY=off believed +// nothing more would leave while the pool went on sending. Both are described +// here, each under a line naming where it goes or why it does not. +func runTelemetryInfo(args []string) error { + flags := telemetryFlags("info") + if err := flags.Parse(args); err != nil { + return err + } + profileDir := config.ProfileDir() + telemetry.Configure(telemetryConfiguredOff()) + fmt.Fprintln(usageOut, infoText(profileDir, os.LookupEnv)) + return nil +} + +// runTelemetryShow prints exactly what is waiting to leave the machine — ALL +// of it, from both streams — as one JSON object a person can read and a +// script can parse: a key per destination, and under each where it goes, why +// it is not sent when it is not, and the rows waiting in the bytes the relay +// would receive. Indented by two, because the person who runs this is reading +// it, not piping it; a pipe reads indented JSON just as well. func runTelemetryShow(args []string) error { flags := telemetryFlags("show") if err := flags.Parse(args); err != nil { return err } - fmt.Fprintln(usageOut, telemetry.Show()) - return nil + profileDir := config.ProfileDir() + telemetry.Configure(telemetryConfiguredOff()) + report := waitingReport{ + Usage: waitingStream{ + Destination: telemetry.Endpoint(), + Off: telemetry.OffReason(), + Waiting: nonNil(telemetry.SpoolContents()), + }, + } + cfg := config.ModelPoolResolved(profileDir, os.LookupEnv) + report.ModelPool = waitingStream{ + Destination: cfg.SubmitURL, + Off: modelPoolOffReason(cfg), + Waiting: nonNil(poolRowsWaiting(config.ProfilePath(profileDir, "pool"))), + } + // The encoder, not json.MarshalIndent: a waiting row is written with + // HTML escaping off, as the outbox and the spool write it, so the bytes + // printed are the bytes a relay is sent. + enc := json.NewEncoder(usageOut) + enc.SetEscapeHTML(false) + enc.SetIndent("", " ") + return enc.Encode(report) +} + +// waitingReport is `codeaf telemetry show`'s whole answer: one entry per +// destination, keyed by the stream's short name, in the order the notice +// names them. +type waitingReport struct { + Usage waitingStream `json:"usage"` + ModelPool waitingStream `json:"model_pool"` +} + +// waitingStream is one destination: where its rows go, the reason nothing is +// sent there when that is so, and the rows waiting to leave, oldest first. +type waitingStream struct { + Destination string `json:"destination"` + Off string `json:"off,omitempty"` + Waiting []json.RawMessage `json:"waiting"` +} + +// nonNil renders an empty queue as `[]`, never `null`: a person reading the +// object should see an empty list where rows would be, not an absence. +func nonNil(rows []json.RawMessage) []json.RawMessage { + if rows == nil { + return []json.RawMessage{} + } + return rows +} + +// modelPoolOffReason names why the pool sends nothing, or "" when it sends: +// `read` uses the pool and sends nothing, `off` asks no judge at all, and a +// cap that came from the telemetry off switch says so, because the person who +// set that switch is the one reading this. +func modelPoolOffReason(cfg poolcfg.Config) string { + if cfg.CanSend() { + return "" + } + reason := fmt.Sprintf("model_pool %s", cfg.Mode) + if cfg.Source.Mode == "telemetry" { + reason += " (capped by the telemetry off switch)" + } + return reason +} + +// infoPreface is the first thing `codeaf telemetry info` says, before either +// stream: the one fact a person came to check. It is true of everything the +// binary sends to AgentField — the usage counts carry only the allowlisted +// fields below, and a Model Pool row carries model slugs, a score and a day. +// The judge that produces a score reads a clipped brief and deliverable, but +// that is a model call to your own provider, like any turn, and nothing it +// read rides in the row. +const infoPreface = `codeaf does NOT collect or share your chat. No prompts, replies, code, file names, +paths, repo names, keys, email, IP or machine name leave for AgentField. Only the +fields below do, as this machine would fill them.` + +// infoText composes the two streams, in the order the notice names them: the +// usage counts first, the Model Pool second. Each sits under a heading naming +// where it goes or why it does not, then WHAT A ROW LOOKS LIKE: the fields +// with this machine's own values where they are known before a run, and one +// example row per event where they are not, spelled from the contract's own +// constants. Only what is sent is listed; the never lists live in the notice +// and docs/TELEMETRY.md, because a person reading a shape wants the shape, +// not a second disclaimer. What is waiting right now is `show`'s answer. +func infoText(profileDir string, lookup func(string) (string, bool)) string { + var out strings.Builder + out.WriteString(infoPreface) + out.WriteString("\n\n") + out.WriteString(usageCountsHeading()) + out.WriteByte('\n') + writeUsageCountFields(&out) + out.WriteByte('\n') + cfg := config.ModelPoolResolved(profileDir, lookup) + out.WriteString(modelPoolHeading(cfg)) + out.WriteByte('\n') + writeModelPoolFields(&out) + return strings.TrimRight(out.String(), "\n") +} + +// showIndent is the two spaces every line under a stream heading starts with. +const showIndent = " " + +// showKeyWidth is the column the values start in: the widest key any row +// carries is model_calls_failed, eighteen characters, and two for air. +const showKeyWidth = 20 + +// writeUsageCountFields prints the usage-count row as this machine would fill +// it — the six every-event props with their live values and the four +// envelope fields — then one example row per event, then the stop reasons. +// The bands themselves are not listed: the example rows show one of each, +// and docs/TELEMETRY.md spells the rest. +func writeUsageCountFields(out *strings.Builder) { + fmt.Fprintf(out, "%severy event, as this machine would send it now\n", showIndent) + for _, prop := range telemetry.CommonPropValues() { + writeField(out, prop.Name, prop.Value) + } + install := "sha256 of a random id, minted on the first send" + if hash, ok := telemetry.InstallIDHashIfMinted(); ok { + install = hash[:12] + "…" + } + writeField(out, "install_id_hash", install) + writeField(out, "session_id_hash", "sha256 of the run id, one per session; absent on first_run") + writeField(out, "event_id", "16 random bytes as hex, one per event") + writeField(out, "event_time", time.Now().UTC().Format(time.RFC3339)) + out.WriteByte('\n') + fmt.Fprintf(out, "%swhat each event adds, for example\n", showIndent) + for _, event := range telemetry.AllowlistedEvents() { + names := telemetry.EventPropNames(event) + if len(names) == 0 { + writeField(out, event, "nothing; sent once per install") + continue + } + writeField(out, event, exampleRow(event, names)) + } + writeField(out, "stop_reason", "one of "+strings.Join(telemetry.StopReasons(), " · ")) +} + +// exampleRow spells one event's props as key=value pairs in the doc's order, +// wrapped so a session_ended row does not run past the terminal's edge. The +// values are [telemetry.ExampleProp]'s, from the contract's constants. +func exampleRow(event string, names []string) string { + var pairs []string + for _, name := range names { + pairs = append(pairs, name+"="+telemetry.ExampleProp(event, name)) + } + const perLine = 5 + var lines []string + for len(pairs) > 0 { + n := perLine + if n > len(pairs) { + n = len(pairs) + } + lines = append(lines, strings.Join(pairs[:n], " ")) + pairs = pairs[n:] + } + continuation := "\n" + showIndent + showIndent + strings.Repeat(" ", showKeyWidth+1) + return strings.Join(lines, continuation) +} + +// writeModelPoolFields prints what one pool row looks like: an example row in +// the bytes a relay would receive, indented so it reads, then the two +// identities a batch travels under. +func writeModelPoolFields(out *strings.Builder) { + fmt.Fprintf(out, "%sone row per judged seat, after a task lands, for example\n", showIndent) + // The row is indented by two, the way `show` prints a waiting one, and + // set in under the heading; json.Indent keeps the bytes the row's own. + var row bytes.Buffer + if err := json.Indent(&row, []byte(record.ExampleRowJSON(time.Now())), showIndent+showIndent, " "); err == nil { + fmt.Fprintf(out, "%s%s%s\n", showIndent, showIndent, row.String()) + } + writeField(out, "nonce", "16 random bytes as hex, one per row, so a resend is not a double count") + writeField(out, "X-Codeaf-Install", "a header: a random per-install id, minted on the first send; not the usage counts' id") +} + +// writeField prints one field line: the key in its column and the value. +func writeField(out *strings.Builder, key, value string) { + fmt.Fprintf(out, "%s%s%-*s %s\n", showIndent, showIndent, showKeyWidth, key, value) +} + +// usageCountsHeading names where the usage counts go, or the rung of the +// opt-out ladder that keeps them here. The two streams are numbered in the +// order the notice names them, so a person can say "the second one". It reads the same ladder `telemetry +// status` reads, so the two verbs cannot disagree about whether anything is +// sent. +func usageCountsHeading() string { + if reason := telemetry.OffReason(); reason != "" { + return fmt.Sprintf("1. Usage Counts (off: %s)", reason) + } + return fmt.Sprintf("1. Usage Counts (%s)", telemetry.Endpoint()) +} + +// modelPoolHeading names where the pool rows go, or the mode that keeps them +// here: `read` uses the pool and sends nothing, `off` asks no judge at all. +func modelPoolHeading(cfg poolcfg.Config) string { + if !cfg.CanSend() { + return fmt.Sprintf("2. Model Pool (model_pool %s, nothing is sent)", cfg.Mode) + } + return fmt.Sprintf("2. Model Pool (%s)", cfg.SubmitURL) +} + +// poolRowsWaiting reads the pool outbox's pending rows the way +// telemetry.SpoolContents reads the spool: raw JSON rows, oldest first, nil +// when nothing waits. It reads the file by path and stats it first, like +// [pendingRows], because [outbox.Open] creates an absent outbox and a reading +// form must not write. +func poolRowsWaiting(poolDir string) []json.RawMessage { + path := filepath.Join(poolDir, "outbox.jsonl") + if _, err := os.Stat(path); err != nil { + return nil + } + box, err := outbox.Open(path) + if err != nil { + return nil + } + defer box.Close() + rows := box.Pending() + if len(rows) == 0 { + return nil + } + out := make([]json.RawMessage, 0, len(rows)) + for _, row := range rows { + // The outbox stores a row compacted; encoding it again here, with + // HTML escaping off as the outbox writes it, answers the same bytes + // the relay is sent. + var line bytes.Buffer + enc := json.NewEncoder(&line) + enc.SetEscapeHTML(false) + if err := enc.Encode(row); err != nil { + continue + } + out = append(out, json.RawMessage(bytes.TrimRight(line.Bytes(), "\n"))) + } + return out } // runTelemetrySet writes the settings row from internal/config: `telemetry off` @@ -123,7 +386,7 @@ func runTelemetrySet(word string, args []string) error { return err } if value { - fmt.Fprintln(usageOut, "telemetry on — anonymous usage counts are sent (see `codeaf telemetry show`)") + fmt.Fprintln(usageOut, "telemetry on — anonymous usage counts are sent (see `codeaf telemetry info`)") return nil } fmt.Fprintln(usageOut, "telemetry off — nothing is sent; the session counters still count") diff --git a/cmd/codeaf/telemetry_test.go b/cmd/codeaf/telemetry_test.go index 14401f4dd2..f45c480e23 100644 --- a/cmd/codeaf/telemetry_test.go +++ b/cmd/codeaf/telemetry_test.go @@ -1,15 +1,20 @@ package main import ( + "bytes" + "encoding/json" "net/http" "net/http/httptest" "os" "path/filepath" + "runtime" "strings" "testing" "github.com/Agent-Field/codeaf/internal/config" "github.com/Agent-Field/codeaf/internal/home" + "github.com/Agent-Field/codeaf/internal/pool/outbox" + "github.com/Agent-Field/codeaf/internal/telemetry" ) // telemetryHome is the clean room every telemetry-verb test runs in: a @@ -84,17 +89,67 @@ func TestTelemetryStatusNamesTheReasonWhenOff(t *testing.T) { } } -func TestTelemetryShowPrintsTheSpool(t *testing.T) { - telemetryHome(t) - telemetrySink(t) +// telemetryShowReport is `codeaf telemetry show`'s object as the tests read it back. +type telemetryShowReport struct { + Usage telemetryShowStream `json:"usage"` + ModelPool telemetryShowStream `json:"model_pool"` +} + +type telemetryShowStream struct { + Destination string `json:"destination"` + Off string `json:"off,omitempty"` + Waiting []json.RawMessage `json:"waiting"` +} + +// runShow runs the verb and parses its answer, failing on anything that is +// not one JSON object indented by two — the shape the verb promises. +func runTelemetryShowJSON(t *testing.T) (telemetryShowReport, string) { + t.Helper() usageOut = &strings.Builder{} defer func() { usageOut = os.Stdout }() if err := runTelemetry([]string{"show"}); err != nil { t.Fatal(err) } got := usageOut.(*strings.Builder).String() - if strings.TrimSpace(got) == "" { - t.Fatal("show printed nothing") + var report telemetryShowReport + if err := json.Unmarshal([]byte(got), &report); err != nil { + t.Fatalf("show should print one JSON object, got %v:\n%s", err, got) + } + // Re-encoding the parsed report the way the verb encodes it — two-space + // indent, HTML escaping off, usage before model_pool — must give the same + // bytes back. + var again strings.Builder + enc := json.NewEncoder(&again) + enc.SetEscapeHTML(false) + enc.SetIndent("", " ") + if err := enc.Encode(report); err != nil { + t.Fatal(err) + } + if got != again.String() { + t.Errorf("show should be indented by two, got:\n%s\nwant:\n%s", got, again.String()) + } + return report, got +} + +func TestTelemetryShowIsOneObjectKeyedByDestination(t *testing.T) { + telemetryHome(t) + telemetrySink(t) + report, got := runTelemetryShowJSON(t) + if !strings.HasPrefix(got, "{\n \"usage\": {") { + t.Errorf("show should open on the usage stream, got:\n%s", got) + } + if !strings.Contains(got, "\n \"model_pool\": {") { + t.Errorf("show should carry the model_pool stream, got:\n%s", got) + } + if report.Usage.Destination == "" || report.ModelPool.Destination == "" { + t.Errorf("both streams should name a destination, got %+v", report) + } + // An empty queue is an empty list, never null. + if !strings.Contains(got, "\"waiting\": []") { + t.Errorf("an empty queue should print as [], got:\n%s", got) + } + if strings.Contains(got, "null") { + t.Errorf("show should never print null, got:\n%s", got) } } @@ -174,3 +229,221 @@ func TestTelemetryIsInUsageAndEnvironmentText(t *testing.T) { t.Error("the environment table should name DO_NOT_TRACK") } } + +// seedPoolOutbox writes one judged row into the profile's pool outbox the way +// a run would — through the outbox's own open and append, closed again — so +// the show verb reads what stands on disk. +func seedPoolOutbox(t *testing.T, root, payload string) { + t.Helper() + box, err := outbox.Open(filepath.Join(root, "pool", "outbox.jsonl")) + if err != nil { + t.Fatal(err) + } + if err := box.Append([]byte(payload)); err != nil { + t.Fatal(err) + } + if err := box.Close(); err != nil { + t.Fatal(err) + } +} + +// TestTelemetryShowPrintsBothStreams is the promise behind the notice's "what +// is collected": two streams leave this binary, under different switches, and +// a show that printed only the usage-count spool let a person believe +// CODEAF_TELEMETRY=off stopped everything. Both streams print under their own +// key, and the pool row's bytes are the bytes the relay would receive. +func TestTelemetryShowPrintsBothStreams(t *testing.T) { + root := telemetryHome(t) + telemetrySink(t) + t.Setenv("CODEAF_MODEL_POOL", "on") + seedPoolOutbox(t, root, `{"schema":1,"metric":"role_quality","role":"worker","model":"vendor/model-x","score":81}`) + report, got := runTelemetryShowJSON(t) + if !strings.HasPrefix(report.Usage.Destination, "http") { + t.Errorf("the usage stream should name its endpoint, got %+v", report.Usage) + } + // The suite's TestMain pins the relay at an unreachable local address so + // no test can post to the real one; the destination names whatever submit + // address is in force, and the path is the relay's own. + if !strings.HasPrefix(report.ModelPool.Destination, "http") || !strings.HasSuffix(report.ModelPool.Destination, "/v1/rows") { + t.Errorf("the pool stream should name the relay, got %+v", report.ModelPool) + } + if report.ModelPool.Off != "" { + t.Errorf("a pool on should not be off, got %q", report.ModelPool.Off) + } + if len(report.Usage.Waiting) != 0 { + t.Errorf("the usage spool is empty under go test, got %s", report.Usage.Waiting) + } + // The rows print indented inside the object; compacted, they are the + // outbox's own bytes: the envelope a relay receives, the row inside it. + if len(report.ModelPool.Waiting) != 1 { + t.Fatalf("the pool's one row should wait, got:\n%s", got) + } + row := compactWaitingRow(t, report.ModelPool.Waiting[0]) + for _, want := range []string{`"payload":{`, `"model":"vendor/model-x"`, `"score":81`, `"nonce":"`} { + if !strings.Contains(row, want) { + t.Errorf("the pool row should carry %s as the relay would receive it, got %s", want, row) + } + } +} + +// compactWaitingRow is one waiting row without the indent `show` prints it with, +// so a test can compare it to the bytes the outbox or spool holds. +func compactWaitingRow(t *testing.T, raw json.RawMessage) string { + t.Helper() + var out bytes.Buffer + if err := json.Compact(&out, raw); err != nil { + t.Fatalf("waiting row is not JSON: %v\n%s", err, raw) + } + return out.String() +} + +// TestTelemetryShowSaysWhenThePoolSendsNothing pins the off reason for a pool +// in `read`: the rows that wait are still printed, under a reason nothing is +// sent, so the person is not told a destination that nothing goes to. +func TestTelemetryShowSaysWhenThePoolSendsNothing(t *testing.T) { + root := telemetryHome(t) + telemetrySink(t) + t.Setenv("CODEAF_MODEL_POOL", "read") + seedPoolOutbox(t, root, `{"n":1}`) + report, _ := runTelemetryShowJSON(t) + if report.ModelPool.Off != "model_pool read" { + t.Errorf("a pool in read should say so, got %q", report.ModelPool.Off) + } + if len(report.ModelPool.Waiting) != 1 || !strings.Contains(compactWaitingRow(t, report.ModelPool.Waiting[0]), `"payload":{"n":1}`) { + t.Errorf("the waiting row should still print, got %s", report.ModelPool.Waiting) + } +} + +// TestTelemetryShowDoesNotCreateThePoolOutbox is the reading-form law: a show +// on a profile with no outbox prints `[]` for the pool and leaves no file +// behind, because [outbox.Open] creates an absent outbox and a reader must not. +func TestTelemetryShowDoesNotCreateThePoolOutbox(t *testing.T) { + root := telemetryHome(t) + telemetrySink(t) + report, _ := runTelemetryShowJSON(t) + if _, err := os.Stat(filepath.Join(root, "pool", "outbox.jsonl")); !os.IsNotExist(err) { + t.Fatalf("show must not create the pool outbox, stat: %v", err) + } + if len(report.Usage.Waiting) != 0 || len(report.ModelPool.Waiting) != 0 { + t.Fatalf("an empty machine should have nothing waiting for either stream, got %+v", report) + } +} + +// TestTelemetryInfoNamesEveryFieldOnAnEmptyMachine is the notice's "what is +// collected" read on the day a person installs: nothing has been sent yet, +// and the verb shows the shape of every row — this machine's own values where +// they are known before a run, one example row per event where they are not, +// and the pool row in the relay's bytes. It lists only what is sent: no line +// starts with "never", because a person reading a shape wants the shape and +// the notice already carries the disclaimer; and it says nothing about what +// is waiting, which is `show`'s answer. +func TestTelemetryInfoNamesEveryFieldOnAnEmptyMachine(t *testing.T) { + telemetryHome(t) + telemetrySink(t) + usageOut = &strings.Builder{} + defer func() { usageOut = os.Stdout }() + if err := runTelemetry([]string{"info"}); err != nil { + t.Fatal(err) + } + got := usageOut.(*strings.Builder).String() + // The first line is the fact a person came to check, before either stream. + if !strings.HasPrefix(got, "codeaf does NOT collect or share your chat.") { + t.Errorf("info should open on the chat-content fact, got:\n%s", got) + } + for _, want := range []string{ + "No prompts, replies, code, file names,\npaths, repo names, keys, email, IP or machine name leave for AgentField.", + "1. Usage Counts (", + "every event, as this machine would send it now", + "os " + runtime.GOOS, + "arch " + runtime.GOARCH, + "install_method unknown", + "install_id_hash sha256 of a random id, minted on the first send", + "what each event adds, for example", + "first_run nothing; sent once per install", + "session_started mode=chat resumed=false", + "session_ended mode=chat duration=5-30m turns=6-20", + "stop_reason=done exit_code=0", + // No blank line between the last event and the stop reasons. + "fingerprint=3fa9c1e2b7d04e85\n stop_reason one of done · error · incomplete", + "2. Model Pool (", + "one row per judged seat, after a task lands, for example", + " {\n \"schema\": 1,\n \"metric\": \"role_quality\",\n \"role\": \"worker\",", + "\n \"door\": \"task\",\n \"size\": \"M\",\n \"day\": \"", + "\n }\n nonce", + "nonce 16 random bytes as hex", + "X-Codeaf-Install", + } { + if !strings.Contains(got, want) { + t.Errorf("info should print %q, got:\n%s", want, got) + } + } + for _, line := range strings.Split(got, "\n") { + if strings.HasPrefix(strings.TrimSpace(line), "never") { + t.Errorf("info lists only what is sent; got a never line: %q", line) + } + } + // The bands are not spelled out: the example rows carry one of each and + // the doc lists the rest, so a listing of every count, dollar and + // duration band is text a person does not need here. + for _, absent := range []string{"bands ", "counts 0 ·", "dollars 0 ·", "duration <1m ·", "waiting"} { + if strings.Contains(got, absent) { + t.Errorf("info should not print %q, got:\n%s", absent, got) + } + } + for _, name := range telemetry.CommonPropNames() { + if !strings.Contains(got, name) { + t.Errorf("info should name the every-event prop %q", name) + } + } + if _, err := os.Stat(filepath.Join(home.Dir(), "telemetry", "install_id")); !os.IsNotExist(err) { + t.Fatalf("info must not mint an install id, stat: %v", err) + } +} + +// TestTelemetryOffQuietsThePoolFromTheEnvironment is the notice's promise read +// end to end: with CODEAF_TELEMETRY=off and the pool explicitly on, the show +// verb reports the pool sending nothing, from the telemetry switch. +func TestTelemetryOffQuietsThePoolFromTheEnvironment(t *testing.T) { + telemetryHome(t) + telemetrySink(t) + t.Setenv("CODEAF_TELEMETRY", "off") + t.Setenv("CODEAF_MODEL_POOL", "on") + report, _ := runTelemetryShowJSON(t) + if report.ModelPool.Off != "model_pool read (capped by the telemetry off switch)" { + t.Errorf("CODEAF_TELEMETRY=off should quiet the pool and say why, got %q", report.ModelPool.Off) + } + if report.Usage.Off != "CODEAF_TELEMETRY=off" { + t.Errorf("the counts should name the same rung, got %q", report.Usage.Off) + } +} + +// TestTelemetryOffCommandQuietsThePool is the profile rung: `codeaf telemetry +// off` writes a row no environment carries, and the pool's resolved config +// reads it off the disk and sends nothing, whatever the pool setting says. +func TestTelemetryOffCommandQuietsThePool(t *testing.T) { + root := telemetryHome(t) + usageOut = &strings.Builder{} + defer func() { usageOut = os.Stdout }() + poolOn := func(name string) (string, bool) { + if name == "CODEAF_MODEL_POOL" { + return "on", true + } + return "", false + } + if cfg := config.ModelPoolResolved(root, poolOn); !cfg.CanSend() { + t.Fatalf("before the command the pool should send, got mode %v from %q", cfg.Mode, cfg.Source.Mode) + } + if err := runTelemetry([]string{"off"}); err != nil { + t.Fatal(err) + } + cfg := config.ModelPoolResolved(root, poolOn) + if cfg.CanSend() || cfg.Source.Mode != "telemetry" || !cfg.CanRead() { + t.Fatalf("after `telemetry off` the pool should read and not send, got mode %v from %q", cfg.Mode, cfg.Source.Mode) + } + if err := runTelemetry([]string{"on"}); err != nil { + t.Fatal(err) + } + if cfg := config.ModelPoolResolved(root, poolOn); !cfg.CanSend() { + t.Fatalf("`telemetry on` should hand the pool back, got mode %v from %q", cfg.Mode, cfg.Source.Mode) + } +} diff --git a/docs/GUIDE.md b/docs/GUIDE.md index 2ad661d0c8..0401e7b948 100644 --- a/docs/GUIDE.md +++ b/docs/GUIDE.md @@ -121,10 +121,14 @@ curl -fsSL https://agentfield.ai/get/codeaf | VERSION= bash The script needs `curl` or `wget`, plus `sha256sum` or `shasum`. It downloads `checksums.txt` and refuses a sha256 mismatch. Unless `--no-modify-path` is set, it -appends one `export PATH=… # codeaf installer` line to the applicable shell file. Its -last action runs the installed file's `version`. The `/get/devaf` line selects the -dev channel and names the file `devaf`, installing it beside codeaf. Release builds -cover darwin, linux, and windows on amd64 and arm64. +appends one `export PATH=… # codeaf installer` line to the applicable shell file. On a +normal run it prints three things and nothing else: `installed codeaf v… built … · +go… os/arch` (the installed file naming itself), the three-line telemetry notice, and, +when the folder is not yet on `PATH`, the bare `export PATH=…` line to paste into the +current shell, bold green on a terminal, last, with a blank line above and below. +`--verbose` also reports the channel, the tag and the install path on stderr. The +`/get/devaf` line selects the dev channel and names the file `devaf`, installing it +beside codeaf. Release builds cover darwin, linux, and windows on amd64 and arm64. diff --git a/docs/TELEMETRY.md b/docs/TELEMETRY.md index c0f83720fc..fb71e98c7a 100644 --- a/docs/TELEMETRY.md +++ b/docs/TELEMETRY.md @@ -6,24 +6,34 @@ about you or your work ever leaves this machine. ## The notice -Before the first session's events are sent, codeaf prints this to stderr once; -the installer prints it too: +Before the first session's events are sent, codeaf prints this to stderr once: ``` codeaf sends anonymous usage counts to AgentField. Sent: version, OS, mode (chat or task), how many sessions, how many errors. Never: anything about you or your work. No prompts, code, file names, paths, repo names, keys, email, IP, or machine name. - See exactly what leaves: codeaf telemetry show + What is collected: codeaf telemetry info Turn off: CODEAF_TELEMETRY=off ``` +The installer prints a three-line form of the same notice, to stderr, after the +`installed codeaf …` receipt and before the `export PATH` line. The full notice +above still arrives at the first session: + +``` +codeaf shares anonymous performance data with AgentField +codeaf does NOT share your prompts, code, files, or any private information +see what is shared: codeaf telemetry info · turn off: CODEAF_TELEMETRY=off +``` + ## What is sent Exactly four events. Each carries the every-event properties; three of them add more. Every value is a count, a band, or a word from a fixed list. The -table is generated from the same allowlist the code is held to, and a test -fails the build if the two ever drift apart. +table's names come from the same allowlist the code is held to and its words +from the table `codeaf telemetry info` prints, and a test fails the build if +any of the three drift apart. | Event | Property | What it is | | --- | --- | --- | @@ -73,11 +83,12 @@ Events wait in ~/.codeaf/telemetry/spool.jsonl until they are sent: at most 50 per request, nothing older than 7 days, at most 1000 lines kept, and nothing sent before the notice has been shown. An event whose version is unknown is dropped at send time and never leaves the machine. `codeaf telemetry show` -prints exactly what has not left yet. +prints exactly what has not left yet, as JSON. ## Turning it off -Any one of these turns the counts off. They are checked in this order: +Any one of these turns the counts off, and every one of them also stops the +Model Pool from sending. They are checked in this order: 1. `CODEAF_TELEMETRY=off` — also `0` or `false`. 2. `DO_NOT_TRACK=1` — also `true`, the ecosystem's own word for it. @@ -90,13 +101,47 @@ Any one of these turns the counts off. They are checked in this order: A build that cannot name its own source — dirty or unstamped — never reports, and neither does a test binary. +## The Model Pool is a second stream, under its own switch + +The usage counts are not the only thing this binary sends to AgentField. With +`model_pool` set to `on` — the default — a judge scores each crew seat after a +task lands, and one row per seat leaves for +`https://codeaf.agentfield.ai/pool/v1/rows`: the model slug that held the +seat, the judge's slug, the seat (worker, high or mastermind), a 0-100 score, +the door the run came in by (task, do, exec or run), the crew size and the UTC +day, under a random per-install nonce in an `X-Codeaf-Install` header. No prompt, code, path or name rides in a row. **Every way of turning the +counts off turns this stream off too** — `CODEAF_TELEMETRY=off`, +`DO_NOT_TRACK=1`, the project file, `codeaf telemetry off` — by capping the +pool at `read`: the index is still read and the judge still scores into the +install's own sheet, but nothing is sent, and `codeaf pool status` says `mode +read · telemetry`. That cap wins over an explicit `model_pool = on`, because +the notice's "Turn off" line carries no exception. The pool's own switch, +`model_pool` in settings or `CODEAF_MODEL_POOL`, adds `off` (ask no judge at +all). `codeaf telemetry info` describes both streams and `codeaf telemetry +show` prints the rows waiting to leave from both, so "what is collected" is +answered for everything the binary sends. + ## The command `codeaf telemetry` reads the counts and never sends anything of its own. - `codeaf telemetry status` says whether the counts are on, and why not when they are off. -- `codeaf telemetry show` prints exactly what is waiting to leave the machine. +- `codeaf telemetry info` opens on the fact a person came to check — `codeaf + does NOT collect or share your chat`, with the never list — then prints what + is collected, from BOTH streams, shaped like the data: for the usage counts, + every every-event field with the value + this machine would send now, one example row per event (`mode=chat + duration=5-30m turns=6-20 …`, from the contract's own bands) and the stop + reasons a row can carry; for the Model Pool, one example row in the bytes + the relay receives and the two identities a batch travels under. Each stream + sits under a numbered heading, `1. Usage Counts` and `2. Model Pool`, naming + where it goes or why it is not sent. It lists only what is sent — the never lists are the notice's and this page's. +- `codeaf telemetry show` prints what is waiting to leave right now, as one + JSON object indented by two: a key per destination, `usage` and + `model_pool`, and under each its `destination`, an `off` reason when + nothing is sent there, and `waiting`, the rows in the bytes the relay would + receive, `[]` when none wait — which is what a fresh install shows. - `codeaf telemetry off` and `codeaf telemetry on` write the profile setting. ``` diff --git a/docs/changes/unreleased/1208-installer-path-line-last.md b/docs/changes/unreleased/1208-installer-path-line-last.md new file mode 100644 index 0000000000..764aa5ebd2 --- /dev/null +++ b/docs/changes/unreleased/1208-installer-path-line-last.md @@ -0,0 +1,19 @@ +--- +kind: changed +title: the installer prints a receipt, a three-line telemetry notice, and the bare export PATH line last +pr: 1208 +surface: [build, docs] +invalidates: + - "The installer printed `codeaf: add it to this shell with: export PATH=...` between `codeaf: installed` and `codeaf version`. That sentence is gone; the bare `export PATH=...` line is now the very last thing the script prints, with a blank line above and below, bold green on a terminal." + - "The installer opened with `codeaf: stable v0.3.0 for darwin/arm64` and `codeaf: installed `. Neither prints on a normal run any more; `--verbose` still reports both on stderr, and the receipt is `installed codeaf v… built … · go… os/arch`." + - "The installer printed the full six-line telemetry notice from docs/TELEMETRY.md. It prints a three-line form now (`codeaf shares anonymous performance data with AgentField` / `codeaf does NOT share your prompts, code, files, or any private information` / `see what is shared: codeaf telemetry show · turn off: CODEAF_TELEMETRY=off`); the full notice still prints from the binary before the first session's events leave." + - "`codeaf telemetry show` printed a field-by-field table with a meaning column and a `never:` line per stream. It prints the shape of the data now: the rows waiting first, then this machine's live every-event values, one example row per event from the contract's bands, the stop reasons, and the pool row in the relay's bytes; no `never` line, only what is sent." + - "`codeaf telemetry` had four verbs: status, show, on, off, and `show` printed both the field listing and the waiting rows. It has five: `info` opens on `codeaf does NOT collect or share your chat` and prints what is collected (the shape, live values, example rows) and `show` prints only what is waiting to leave, as one JSON object indented by two with a key per destination (`usage`, `model_pool`), each carrying `destination`, an `off` reason when nothing is sent, and `waiting`." + - "The notice's fifth line was `See exactly what leaves: codeaf telemetry show`, and the installer's third line pointed at `show` too. Both point at `codeaf telemetry info` now: `What is collected: codeaf telemetry info`, and `see what is shared: codeaf telemetry info · turn off: CODEAF_TELEMETRY=off`. The binary, the README, docs/TELEMETRY.md and the installer all carry the new bytes." + - "docs/TELEMETRY.md listed an empty `CODEAF_TELEMETRY_ENDPOINT` among the switches that also stop the Model Pool from sending, and the pool did not read it: the counts stopped and the pool kept sending. poolcfg reads it now, and a set-and-empty endpoint caps the pool at `read` like the other rungs." + - "A missing release failed with `could not download codeaf--; check the tag on the Releases page`, and the tag was only visible because the channel line above it named it. The failure names the tag itself now: `no codeaf-- in release ; check the tag on the Releases page`." +--- + +The install one-liner ends on the one line a person still has to paste, and +says as little as possible above it. `test/installer-telemetry.sh` pins the +receipt, the three-line notice against docs/TELEMETRY.md, and the hint's shape. diff --git a/docs/changes/unreleased/1214-telemetry-show-prints-the-model-pool-rows.md b/docs/changes/unreleased/1214-telemetry-show-prints-the-model-pool-rows.md new file mode 100644 index 0000000000..55271e5d2a --- /dev/null +++ b/docs/changes/unreleased/1214-telemetry-show-prints-the-model-pool-rows.md @@ -0,0 +1,10 @@ +--- +kind: changed +title: "The telemetry off switch quiets the Model Pool too, and `codeaf telemetry show` prints both streams" +pr: 1214 +surface: [chat, engine, docs] +invalidates: + - "`codeaf telemetry show` printed only the usage-count spool — two empty arrays on the day a person installs, when the notice had just told them to run it — so \"see exactly what leaves\" was untrue of the Model Pool rows and empty of meaning before the first run. It prints both streams now: every field with the value this machine would send, what each event adds, the never lists, then what is waiting, each under a line naming where it goes or why it is not sent." + - "`CODEAF_TELEMETRY=off` stopped the usage counts and nothing else; the Model Pool went on posting model slugs and scores under `model_pool = on`, the default. Every rung of the telemetry off ladder — the variable, `DO_NOT_TRACK=1`, the project file, `codeaf telemetry off` — now caps the pool at `read`, over an explicit `on`, and `codeaf pool status` names the cap as `mode read · telemetry`." + - "docs/TELEMETRY.md and the chat manual read as though the usage counts were the only thing sent to AgentField. The Model Pool is a second stream to codeaf.agentfield.ai, and both pages name it, its fields, its relay and its switches." +--- diff --git a/internal/config/derivation_test.go b/internal/config/derivation_test.go index 0ba5c24fc2..e462c3b219 100644 --- a/internal/config/derivation_test.go +++ b/internal/config/derivation_test.go @@ -126,7 +126,12 @@ var settingReaders = map[string]string{ KeyDocumentEngine: "DocumentEngine", KeyVisionModel: "VisionModel", KeyAttribution: "Attribution", - KeyModelPool: "ModelPoolSettingAt", + // The pool row is read through [ModelPoolAt] since the telemetry off switch + // started capping the pool at read: `codeaf pool` and `codeaf telemetry` + // call [ModelPoolResolved] with their injected environment, and every + // other verb calls [ModelPoolAt]. Nothing outside this package reads the + // stored word directly any more. + KeyModelPool: "ModelPoolAt", // The pool key row names the resolver that reads it: [ModelPoolAt] carries // the word into poolcfg beside the mode, and the puller's keys resolve // from there (cmd/codeaf's poolTrustedKeys). diff --git a/internal/config/settings.go b/internal/config/settings.go index 2309e47d78..cd85c1b2f2 100644 --- a/internal/config/settings.go +++ b/internal/config/settings.go @@ -3424,7 +3424,43 @@ func ModelPoolPublicKeySettingAt(profileDir string) string { // and [ModelPoolPublicKeySettingAt] against its own lookup rather than // calling this. func ModelPoolAt(profileDir string) poolcfg.Config { - return poolcfg.Resolve(ModelPoolSettingAt(profileDir), ModelPoolPublicKeySettingAt(profileDir), os.LookupEnv) + return ModelPoolResolved(profileDir, os.LookupEnv) +} + +// ModelPoolResolved is [ModelPoolAt] with the environment injected, for the +// verbs whose tests hand one in. It is where the telemetry off switch reaches +// the pool: the environment rungs (CODEAF_TELEMETRY, DO_NOT_TRACK) are read by +// the resolver through lookup, and the two rungs that live on disk — the +// project file and the profile row that `codeaf telemetry off` writes — are +// read here and applied with [poolcfg.Config.Quieted]. The rows, not the +// pin: a caller that injected an environment must get the answer for THAT +// environment's CODEAF_TELEMETRY, not the one the harness happens to export +// (the fall-through [telemetryRowsOff] describes is the one exception). +func ModelPoolResolved(profileDir string, lookup func(string) (string, bool)) poolcfg.Config { + cfg := poolcfg.Resolve(ModelPoolSettingAt(profileDir), ModelPoolPublicKeySettingAt(profileDir), lookup) + cwd, _ := os.Getwd() + if telemetryRowsOff(cwd, profileDir) { + cfg = cfg.Quieted() + } + return cfg +} + +// telemetryRowsOff is the disk half of [TelemetryOffReason]: the project file +// and the profile row. It does not read CODEAF_TELEMETRY itself — the caller +// has read that through its own lookup — but it is not blind to the process +// environment either: [ProjectBoolAt] falls through to [TelemetryAt] when the +// project file says nothing, and TelemetryAt reads the pin through +// internal/env, which honours the former AFORGE_TELEMETRY spelling. // legacy-name +// THAT FALL-THROUGH IS WHY THE FORMER SPELLING CAPS THE POOL; a rewrite that +// read the two rows directly would drop it. +func telemetryRowsOff(cwd, profileDir string) bool { + if cwd != "" { + if value, err := ProjectBoolAt(cwd, profileDir, KeyTelemetry); err == nil && !value { + return true + } + } + value, ok := persistedBool(profileDir, KeyTelemetry) + return ok && !value } // ExaKeyAt resolves the Exa credential: the environment first, then the sheet, diff --git a/internal/manual/chat/running-from-the-terminal.md b/internal/manual/chat/running-from-the-terminal.md index d98e5ac1d1..ea1c44b990 100644 --- a/internal/manual/chat/running-from-the-terminal.md +++ b/internal/manual/chat/running-from-the-terminal.md @@ -92,9 +92,11 @@ exactly once, or what it fetched is not a shell script, it answers 502. Building from source needs nothing published: clone the repository, run `make build`, then run `bin/codeaf` from the checkout. -The installer writes `~/.codeaf/bin/codeaf`; its last action runs that file's -`version`. `/update` in the chat or `codeaf update` in a terminal replaces it in -place; running the install line again works too. +The installer writes `~/.codeaf/bin/codeaf` and prints three things: one line +naming the installed file's `version`, the three-line telemetry notice, and last, +when the folder is not yet on `PATH`, the bare `export PATH=…` line to paste. +`/update` in the chat or `codeaf update` in a terminal replaces it in place; +running the install line again works too. ## What is devaf — dev build beside codeaf — side by side — two versions — install under a different file name — --name @@ -639,7 +641,10 @@ setting answers for all of it, `model_pool` in `/settings`, with three values: `on` reads and sends, `read` uses the pool and sends nothing, `off` does neither. It defaults to `on`. `CODEAF_MODEL_POOL` pins the same word from the shell, and on a CI machine with neither set codeaf reads but does -not send. +not send. **The telemetry off switch stops the pool sending too**: +`CODEAF_TELEMETRY=off`, `DO_NOT_TRACK=1` or `codeaf telemetry off` caps the +pool at `read` — it wins over an explicit `on` — and `codeaf pool status` +then says `mode read · telemetry`. ``` codeaf pool [show|status|verify] [--json] [--key key] @@ -647,7 +652,8 @@ codeaf pool [show|status|verify] [--json] [--key key] `show` — also what bare `codeaf pool` prints — is the reading form: the mode and the addresses in force with the word saying where each came from -(`default`, `setting`, `env` or `ci`), then what index is cached, how old +(`default`, `setting`, `env`, `ci`, or `telemetry` when the telemetry off switch capped +sending), then what index is cached, how old it is and how many cells it holds, or `no index cached yet · built-in seed of `. The binary carries a seed index of our own scored runs, read until a fresher signed one is cached. `--cells` lists the held @@ -656,8 +662,9 @@ spells, the measurement and the installs behind it — and `--json --cells` carries them as an array. Your install also keeps the scores its judge gave in `own.json` under the pool directory — `show` and `status` say what that sheet holds — and -the crew reads them beside the index. `status` adds what is waiting to be sent -and whether the mode allows sending and reading. `codeaf pool status` also +the crew reads them beside the index. `status` adds how many rows are waiting to be sent +and whether the mode allows sending and reading; `codeaf telemetry show` prints the +rows themselves, as JSON. `codeaf pool status` also says whether the relay answered, and whether the mirror did, and what the last judge did — which model, which seats it scored, or why it failed. `--json` prints the same answer as one object; `show` reads nothing off the network. @@ -834,8 +841,19 @@ retracted belief restores, a stopped service starts again, a revoked device pair ## What does it count about a run — the anonymous usage counts, and `codeaf telemetry` `codeaf telemetry` is the door onto the anonymous usage counts: `status` says whether -they are on and why not when they are off, `show` prints exactly what is waiting to -leave the machine, and `off` and `on` write the answer to your profile. It reads and +they are on and why not when they are off; `info` opens on `codeaf does NOT collect or +share your chat` and then says what is collected, shaped like the data — for each stream, under a line naming where it goes or why it is not sent, every +every-event field with the value this machine would send now and one example row per +event (`session_ended mode=chat duration=5-30m turns=6-20 …`) with the stop reasons a +row can carry, and for the Model Pool one example row in the relay's own bytes; `show` +prints what is waiting to leave right now as one JSON object, a key per destination +(`usage`, `model_pool`) with its `destination`, an `off` reason when nothing is sent +there, and `waiting`, the rows themselves, `[]` on the day you install; and `off` and +`on` write the answer to your profile. `info` lists only what is sent, never a disclaimer. +`CODEAF_TELEMETRY=off` — or `DO_NOT_TRACK=1`, or `codeaf telemetry off` — stops both: the +usage counts go quiet and the Model Pool is capped at `read`, so it still picks models +from the index and sends nothing. The pool's own switch, `model_pool` in `/settings` or +`CODEAF_MODEL_POOL`, adds `off`, which asks no judge at all. It reads and sends nothing of its own — it is a command about the counts, not a session. The notice the first session prints names the bargain before the first byte leaves, and `CODEAF_TELEMETRY=off` or `DO_NOT_TRACK=1` turns the counts off entirely. See diff --git a/internal/pool/poolcfg/poolcfg.go b/internal/pool/poolcfg/poolcfg.go index 4680f72c51..4886d71c25 100644 --- a/internal/pool/poolcfg/poolcfg.go +++ b/internal/pool/poolcfg/poolcfg.go @@ -54,7 +54,8 @@ func (m Mode) String() string { } // Sources records where each resolved value came from. Every field holds one -// of four words: "default", "setting", "env" or "ci". +// of four words: "default", "setting", "env" or "ci" — and Mode alone may hold +// a fifth, "telemetry", when the telemetry off switch capped it at Read. type Sources struct { Mode string RelayURL string @@ -86,16 +87,23 @@ type Config struct { Source Sources } -// The eight names Resolve reads, and no others. +// The eleven names Resolve reads, and no others. The first three are the +// telemetry off switch's environment rungs, spelled exactly as +// internal/telemetry's ladder spells them — the variable, the ecosystem's +// word, and an endpoint set to nothing — so the one switch the notice names +// turns off everything that leaves. const ( - envMode = "CODEAF_MODEL_POOL" - envCI = "CI" - envRelay = "CODEAF_MODEL_POOL_RELAY_URL" - envIndex = "CODEAF_MODEL_POOL_URL" - envSubmit = "CODEAF_MODEL_POOL_SUBMIT_URL" - envMirror = "CODEAF_MODEL_POOL_MIRROR_URL" - envTTL = "CODEAF_MODEL_POOL_TTL" - envKey = "CODEAF_MODEL_POOL_PUBLIC_KEY" + envTelemetry = "CODEAF_TELEMETRY" + envDoNotTrack = "DO_NOT_TRACK" + envEndpoint = "CODEAF_TELEMETRY_ENDPOINT" + envMode = "CODEAF_MODEL_POOL" + envCI = "CI" + envRelay = "CODEAF_MODEL_POOL_RELAY_URL" + envIndex = "CODEAF_MODEL_POOL_URL" + envSubmit = "CODEAF_MODEL_POOL_SUBMIT_URL" + envMirror = "CODEAF_MODEL_POOL_MIRROR_URL" + envTTL = "CODEAF_MODEL_POOL_TTL" + envKey = "CODEAF_MODEL_POOL_PUBLIC_KEY" ) // Where a resolved value came from. @@ -104,6 +112,9 @@ const ( srcSetting = "setting" srcEnv = "env" srcCI = "ci" + // srcTelemetry is the Mode source when the telemetry off switch capped + // an On at Read: the pool still reads, and sends nothing. + srcTelemetry = "telemetry" ) // TTL bounds, and the value held when no readable TTL is set. @@ -124,7 +135,7 @@ const ( // Config, it keeps nothing between calls, and many goroutines may call it at // once. func Resolve(setting, publicKey string, lookup func(name string) (value string, set bool)) Config { - // One injection for the whole environment: the eight names are read once, + // One injection for the whole environment: every name is read once, // here, in this order, and each value is trimmed as it is read. get := func(name string) (string, bool) { if lookup == nil { @@ -138,6 +149,9 @@ func Resolve(setting, publicKey string, lookup func(name string) (value string, } modeWord, _ := get(envMode) ciWord, _ := get(envCI) + telemetryWord, _ := get(envTelemetry) + doNotTrackWord, _ := get(envDoNotTrack) + endpointWord, endpointSet := get(envEndpoint) relayWord, _ := get(envRelay) indexWord, _ := get(envIndex) submitWord, submitSet := get(envSubmit) @@ -157,6 +171,16 @@ func Resolve(setting, publicKey string, lookup func(name string) (value string, } else if ciSaysYes(ciWord) { mode, modeSrc = Read, srcCI } + // THE TELEMETRY OFF SWITCH WINS OVER EVERY ANSWER ABOVE, an explicit + // `on` included. The notice says "Turn off: CODEAF_TELEMETRY=off" without + // qualification, and the pool's rows are the other thing this binary + // sends; a switch that stopped one stream and not the other would make + // that sentence untrue. It caps rather than turns off: the pool is still + // read, the judge still scores into the install's own sheet, and nothing + // leaves. + if telemetrySaysOff(telemetryWord, doNotTrackWord, endpointWord, endpointSet) && mode == On { + mode, modeSrc = Read, srcTelemetry + } // The relay is the base the other two addresses derive from, and its // source is theirs: a relay read from the environment hands down both of @@ -260,6 +284,36 @@ func parseMode(word string) (Mode, bool) { return On, false } +// telemetrySaysOff reads the telemetry off switch the way internal/telemetry's +// ladder reads it, spelling for spelling: CODEAF_TELEMETRY is off, 0 or false, +// DO_NOT_TRACK is 1 or true, or CODEAF_TELEMETRY_ENDPOINT is set and empty — +// docs/TELEMETRY.md's fifth rung, which stopped the counts and not the pool +// until 2026-09-21. Two readers of one switch must agree, and this is the +// second one. +func telemetrySaysOff(telemetryWord, doNotTrackWord, endpointWord string, endpointSet bool) bool { + switch strings.ToLower(telemetryWord) { + case "off", "0", "false": + return true + } + switch strings.ToLower(doNotTrackWord) { + case "1", "true": + return true + } + return endpointSet && endpointWord == "" +} + +// Quieted is the config with sending capped by the telemetry off switch's +// other two rungs — the project file and the profile row — which the resolver +// cannot see because it reads no disk. An On becomes Read from "telemetry"; +// a Read or an Off is already sending nothing and is answered unchanged. +func (c Config) Quieted() Config { + if c.Mode == On { + c.Mode = Read + c.Source.Mode = srcTelemetry + } + return c +} + // ciSaysYes reads the CI word: one of true, 1 or yes, without regard to case. func ciSaysYes(word string) bool { switch strings.ToLower(word) { diff --git a/internal/pool/poolcfg/poolcfg_test.go b/internal/pool/poolcfg/poolcfg_test.go index 87ffcc5eb6..285728f851 100644 --- a/internal/pool/poolcfg/poolcfg_test.go +++ b/internal/pool/poolcfg/poolcfg_test.go @@ -128,6 +128,63 @@ func TestResolveModeFromCI(t *testing.T) { } } +// TestResolveModeUnderTheTelemetryOffSwitch is the law the notice rests on: +// "Turn off: CODEAF_TELEMETRY=off" is true of everything that leaves, so the +// switch caps the pool at Read — over the default, over CI, and over an +// explicit `on` from the setting or the environment — and leaves Read and Off +// as they were, because they already send nothing. +func TestResolveModeUnderTheTelemetryOffSwitch(t *testing.T) { + cases := []struct { + name string + setting string + lookup func(string) (string, bool) + want Mode + wantSrc string + }{ + {"telemetry off caps the default", "", envOf(map[string]string{"CODEAF_TELEMETRY": "off"}), Read, "telemetry"}, + {"telemetry 0 caps the default", "", envOf(map[string]string{"CODEAF_TELEMETRY": "0"}), Read, "telemetry"}, + {"telemetry false, padded and mixed case", "", envOf(map[string]string{"CODEAF_TELEMETRY": " False "}), Read, "telemetry"}, + {"do not track 1 caps the default", "", envOf(map[string]string{"DO_NOT_TRACK": "1"}), Read, "telemetry"}, + {"do not track true caps the default", "", envOf(map[string]string{"DO_NOT_TRACK": "true"}), Read, "telemetry"}, + {"an empty endpoint caps the default", "", envOf(map[string]string{"CODEAF_TELEMETRY_ENDPOINT": ""}), Read, "telemetry"}, + {"a blank endpoint caps an explicit env on", "", envOf(map[string]string{"CODEAF_TELEMETRY_ENDPOINT": " ", "CODEAF_MODEL_POOL": "on"}), Read, "telemetry"}, + {"a set endpoint does not cap", "", envOf(map[string]string{"CODEAF_TELEMETRY_ENDPOINT": "https://example.test/t"}), On, "default"}, + {"telemetry off caps an explicit setting on", "on", envOf(map[string]string{"CODEAF_TELEMETRY": "off"}), Read, "telemetry"}, + {"telemetry off caps an explicit env on", "off", envOf(map[string]string{"CODEAF_TELEMETRY": "off", "CODEAF_MODEL_POOL": "on"}), Read, "telemetry"}, + {"telemetry off leaves read as read", "read", envOf(map[string]string{"CODEAF_TELEMETRY": "off"}), Read, "setting"}, + {"telemetry off leaves off as off", "off", envOf(map[string]string{"CODEAF_TELEMETRY": "off"}), Off, "setting"}, + {"telemetry off leaves ci as ci", "", envOf(map[string]string{"CODEAF_TELEMETRY": "off", "CI": "true"}), Read, "ci"}, + {"telemetry on is not an answer", "", envOf(map[string]string{"CODEAF_TELEMETRY": "on"}), On, "default"}, + {"telemetry set and empty is not an answer", "", envOf(map[string]string{"CODEAF_TELEMETRY": ""}), On, "default"}, + {"do not track 0 is not an answer", "", envOf(map[string]string{"DO_NOT_TRACK": "0"}), On, "default"}, + } + for _, c := range cases { + got := Resolve(c.setting, "", c.lookup) + if got.Mode != c.want || got.Source.Mode != c.wantSrc { + t.Errorf("%s: Mode = %v from %q, want %v from %q", c.name, got.Mode, got.Source.Mode, c.want, c.wantSrc) + } + if c.want != On && got.CanSend() { + t.Errorf("%s: CanSend must be false when the mode is %v", c.name, c.want) + } + } +} + +// TestQuietedCapsOnlyOn pins the disk half: Quieted turns On into Read from +// "telemetry" and answers Read and Off unchanged, source and all. +func TestQuietedCapsOnlyOn(t *testing.T) { + on := Resolve("on", "", envOf(nil)).Quieted() + if on.Mode != Read || on.Source.Mode != "telemetry" || on.CanSend() || !on.CanRead() { + t.Errorf("Quieted on = %v from %q, CanSend %v, CanRead %v", on.Mode, on.Source.Mode, on.CanSend(), on.CanRead()) + } + for _, word := range []string{"read", "off"} { + before := Resolve(word, "", envOf(nil)) + after := before.Quieted() + if after.Mode != before.Mode || after.Source.Mode != before.Source.Mode { + t.Errorf("Quieted %s = %v from %q, want unchanged %v from %q", word, after.Mode, after.Source.Mode, before.Mode, before.Source.Mode) + } + } +} + func TestResolveIndexURL(t *testing.T) { cases := []struct { name string @@ -489,14 +546,18 @@ func TestResolveNilLookupUsesSetting(t *testing.T) { } } -func TestResolveAsksEightNames(t *testing.T) { +// TestResolveAsksElevenNames pins the whole of what Resolve reads: the eight +// pool names, and the three environment rungs of the telemetry off switch, +// which the pool obeys so the one switch the notice names stops everything +// that leaves. +func TestResolveAsksElevenNames(t *testing.T) { var asked []string Resolve("", "", func(name string) (string, bool) { asked = append(asked, name) return "", false }) sort.Strings(asked) - want := []string{envMode, envCI, envRelay, envIndex, envSubmit, envMirror, envTTL, envKey} + want := []string{envMode, envCI, envRelay, envIndex, envSubmit, envMirror, envTTL, envKey, envTelemetry, envDoNotTrack, envEndpoint} sort.Strings(want) if !reflect.DeepEqual(asked, want) { t.Errorf("Resolve asked lookup for %v, want %v", asked, want) diff --git a/internal/pool/record/record.go b/internal/pool/record/record.go index 2fda401483..271627b726 100644 --- a/internal/pool/record/record.go +++ b/internal/pool/record/record.go @@ -17,11 +17,14 @@ package record import ( + "bytes" "encoding/json" "errors" "fmt" "os" "path/filepath" + "strings" + "time" "github.com/Agent-Field/codeaf/internal/crewpick" "github.com/Agent-Field/codeaf/internal/pool/judge" @@ -113,6 +116,32 @@ type Row struct { Day string `json:"day"` } +// ExampleRowJSON is one row as the relay would receive it, on the day given, +// with placeholder slugs where a real row carries the model that held the +// seat and the model that judged it. `codeaf telemetry show` prints it so a +// person sees the bytes before any row exists. It is marshalled from [Row], +// so it cannot spell a key a real row would not. +func ExampleRowJSON(now time.Time) string { + row := Row{ + Schema: rowSchema, + Metric: Metric, + Role: "worker", + Model: "", + Score: 81, + Judge: "", + Door: "task", + Size: "M", + Day: now.UTC().Format("2006-01-02"), + } + var out bytes.Buffer + enc := json.NewEncoder(&out) + enc.SetEscapeHTML(false) + if err := enc.Encode(row); err != nil { + return "" + } + return strings.TrimRight(out.String(), "\n") +} + // RowsOf reads judge scores into rows, one per score. Every row carries // schema 1 and the role_quality metric; the day is spelled by the caller, the // way the outbox spells its own. diff --git a/internal/pool/record/record_test.go b/internal/pool/record/record_test.go index 6508e8c266..1b639394a6 100644 --- a/internal/pool/record/record_test.go +++ b/internal/pool/record/record_test.go @@ -4,8 +4,10 @@ import ( "encoding/json" "os" "path/filepath" + "reflect" "strings" "testing" + "time" "github.com/Agent-Field/codeaf/internal/crewpick" "github.com/Agent-Field/codeaf/internal/pool/judge" @@ -326,3 +328,32 @@ func TestCellsAnswerSortedAndCarryTheMeanAndCount(t *testing.T) { } } } + +// TestExampleRowJSONIsARow holds the example `codeaf telemetry show` prints +// to the row itself: it parses back into a Row, carries every key the row +// spells and no other, and names the day it was asked for. +func TestExampleRowJSONIsARow(t *testing.T) { + day := time.Date(2026, 9, 19, 23, 59, 0, 0, time.UTC) + text := ExampleRowJSON(day) + var row Row + if err := json.Unmarshal([]byte(text), &row); err != nil { + t.Fatalf("example does not parse as a row: %v\n%s", err, text) + } + if row.Schema != rowSchema || row.Metric != Metric || row.Day != "2026-09-19" { + t.Errorf("example row = %+v", row) + } + var keys map[string]any + if err := json.Unmarshal([]byte(text), &keys); err != nil { + t.Fatal(err) + } + rt := reflect.TypeOf(Row{}) + for i := 0; i < rt.NumField(); i++ { + name, _, _ := strings.Cut(rt.Field(i).Tag.Get("json"), ",") + if _, ok := keys[name]; !ok { + t.Errorf("example row lacks %q", name) + } + } + if len(keys) != rt.NumField() { + t.Errorf("example row has %d keys, the row has %d", len(keys), rt.NumField()) + } +} diff --git a/internal/release/install_test.go b/internal/release/install_test.go index 3f6d26a678..b21b49f08f 100644 --- a/internal/release/install_test.go +++ b/internal/release/install_test.go @@ -263,9 +263,21 @@ func TestInstallerGetsLatestStableAndFinishesWithVersion(t *testing.T) { if run.code != 0 { t.Fatalf("exit %d:\n%s", run.code, run.output) } - for _, want := range []string{"stable v1.2.3", runtime.GOOS + "/" + runtime.GOARCH, "codeaf v1.2.3 · fake"} { - if !strings.Contains(run.output, want) { - t.Errorf("output does not contain %q:\n%s", want, run.output) + // A normal run says three things and nothing else: the installed binary + // naming itself, the notice, the line to paste. The channel, the tag and + // the platform are --verbose's to say. + if !strings.Contains(run.output, "installed codeaf v1.2.3 · fake") { + t.Errorf("output does not carry the receipt:\n%s", run.output) + } + for _, absent := range []string{"stable v1.2.3", "codeaf: installed"} { + if strings.Contains(run.output, absent) { + t.Errorf("a normal run should not say %q:\n%s", absent, run.output) + } + } + verbose := runInstaller(t, github, []string{"--verbose"}, "CODEAF_NO_MODIFY_PATH=1") + for _, want := range []string{"stable v1.2.3", runtime.GOOS + "/" + runtime.GOARCH, "codeaf: installed", "installed codeaf v1.2.3 · fake"} { + if verbose.code != 0 || !strings.Contains(verbose.output, want) { + t.Errorf("verbose output does not contain %q (exit %d):\n%s", want, verbose.code, verbose.output) } } if _, err := os.Stat(filepath.Join(run.installDir, "codeaf")); err != nil { @@ -474,7 +486,7 @@ func TestInstallerPicksTheNewestChannelBuildWhateverTheListOrder(t *testing.T) { valid: {published: "2026-09-15T15:00:00Z"}, malformed: {published: "2026-09-15T16:00:00Z"}, } - run := runInstaller(t, github, []string{"--dev"}, "CODEAF_NO_MODIFY_PATH=1") + run := runInstaller(t, github, []string{"--dev", "--verbose"}, "CODEAF_NO_MODIFY_PATH=1") if run.code != 0 || !strings.Contains(run.output, "dev "+valid) || strings.Contains(run.output, malformed) { t.Fatalf("exit %d, want valid tag %s:\n%s", run.code, valid, run.output) } @@ -490,7 +502,7 @@ func TestInstallerPicksTheNewestChannelBuildWhateverTheListOrder(t *testing.T) { valid: {published: "2026-09-15T15:00:00Z"}, malformed: {published: "2026-09-15T16:00:00Z"}, } - run := runInstaller(t, github, []string{"--rc"}, "CODEAF_NO_MODIFY_PATH=1") + run := runInstaller(t, github, []string{"--rc", "--verbose"}, "CODEAF_NO_MODIFY_PATH=1") if run.code != 0 || !strings.Contains(run.output, "rc "+valid) || strings.Contains(run.output, malformed) { t.Fatalf("exit %d, want valid tag %s:\n%s", run.code, valid, run.output) } @@ -508,7 +520,7 @@ func TestInstallerPicksTheNewestChannelBuildWhateverTheListOrder(t *testing.T) { middle: {published: "2026-09-15T14:56:00Z"}, newest: {published: "2026-09-15T15:21:00Z"}, } - run := runInstaller(t, github, []string{"--dev"}, "CODEAF_NO_MODIFY_PATH=1") + run := runInstaller(t, github, []string{"--dev", "--verbose"}, "CODEAF_NO_MODIFY_PATH=1") if run.code != 0 { t.Fatalf("exit %d:\n%s", run.code, run.output) } @@ -543,7 +555,7 @@ func TestInstallerPicksTheNewestChannelBuildWhateverTheListOrder(t *testing.T) { older: {created: "2026-09-15T13:00:00Z", published: "2026-09-15T14:00:00Z"}, newest: {created: "2026-09-15T16:00:00Z", published: json.RawMessage("null")}, } - run := runInstaller(t, github, []string{"--dev"}, "CODEAF_NO_MODIFY_PATH=1") + run := runInstaller(t, github, []string{"--dev", "--verbose"}, "CODEAF_NO_MODIFY_PATH=1") if run.code != 0 || !strings.Contains(run.output, "dev "+newest) { t.Fatalf("exit %d, want %s:\n%s", run.code, newest, run.output) } @@ -563,7 +575,7 @@ func TestInstallerPicksTheNewestChannelBuildWhateverTheListOrder(t *testing.T) { older: {published: "2026-09-15T12:00:00Z"}, newest: {published: "2026-09-15T17:00:00Z"}, } - run := runInstaller(t, github, []string{"--staging"}, "CODEAF_NO_MODIFY_PATH=1") + run := runInstaller(t, github, []string{"--staging", "--verbose"}, "CODEAF_NO_MODIFY_PATH=1") if run.code != 0 || !strings.Contains(run.output, "staging "+newest) { t.Fatalf("exit %d, want %s:\n%s", run.code, newest, run.output) } @@ -572,15 +584,17 @@ func TestInstallerPicksTheNewestChannelBuildWhateverTheListOrder(t *testing.T) { func TestInstallerPinsAReleaseAndNamesAMissingOne(t *testing.T) { github := newInstallGitHub(t, "v1.2.3", "build-legacy") - run := runInstaller(t, github, []string{"--version", "v1.2.3"}, "CODEAF_NO_MODIFY_PATH=1") + // The channel and tag are --verbose lines; a normal run's receipt is the + // installed binary naming itself, which the fake does with its tag. + run := runInstaller(t, github, []string{"--version", "v1.2.3", "--verbose"}, "CODEAF_NO_MODIFY_PATH=1") if run.code != 0 || !strings.Contains(run.output, "stable v1.2.3") { t.Fatalf("exit %d:\n%s", run.code, run.output) } fromEnvironment := runInstaller(t, github, nil, "VERSION=v1.2.3", "CODEAF_NO_MODIFY_PATH=1") - if fromEnvironment.code != 0 || !strings.Contains(fromEnvironment.output, "stable v1.2.3") { + if fromEnvironment.code != 0 || !strings.Contains(fromEnvironment.output, "installed codeaf v1.2.3 · fake") { t.Fatalf("VERSION install exit %d:\n%s", fromEnvironment.code, fromEnvironment.output) } - legacy := runInstaller(t, github, []string{"--version", "build-legacy"}, "CODEAF_NO_MODIFY_PATH=1") + legacy := runInstaller(t, github, []string{"--version", "build-legacy", "--verbose"}, "CODEAF_NO_MODIFY_PATH=1") wantLegacy := "codeaf: build-legacy for " + runtime.GOOS + "/" + runtime.GOARCH if legacy.code != 0 || !strings.Contains(legacy.output, wantLegacy) || strings.Contains(legacy.output, "codeaf: version ") { t.Fatalf("legacy-tag install exit %d:\n%s", legacy.code, legacy.output) @@ -619,7 +633,7 @@ func TestDocumentedVersionPinReachesThePipedInstaller(t *testing.T) { "SHELL=/bin/bash", } output, err := command.CombinedOutput() - if err != nil || !strings.Contains(string(output), "stable v1.2.3") || strings.Contains(string(output), "v9.9.9") { + if err != nil || !strings.Contains(string(output), "installed codeaf v1.2.3 · fake") || strings.Contains(string(output), "v9.9.9") { t.Fatalf("documented pin failed: %v\n%s", err, output) } } @@ -882,12 +896,19 @@ func TestV1InstallerName(t *testing.T) { if err != nil || string(kept) != string(original) { t.Fatalf("codeaf = %q, %v", kept, err) } - if !strings.Contains(run.output, "codeaf: installed "+devaf) { - t.Fatalf("output does not name %s:\n%s", devaf, run.output) + // A normal run's receipt is the installed file naming itself, and the + // last thing said is the line to paste; the path is --verbose's to say. + if !strings.Contains(run.output, "installed codeaf "+tag+" · fake") { + t.Fatalf("output does not carry the receipt:\n%s", run.output) } - if got := strings.Split(strings.TrimSpace(run.output), "\n"); got[len(got)-1] != "codeaf "+tag+" · fake" { + if got := strings.Split(strings.TrimSpace(run.output), "\n"); got[len(got)-1] != `export PATH="`+dir+`:$PATH"` { t.Fatalf("last line = %q:\n%s", got[len(got)-1], run.output) } + verbose := runInstaller(t, github, []string{"--name", "devaf", "--dev", "--verbose"}, + "CODEAF_INSTALL_DIR="+dir, "CODEAF_NO_MODIFY_PATH=1") + if verbose.code != 0 || !strings.Contains(verbose.output, "codeaf: installed "+devaf) { + t.Fatalf("verbose output does not name %s (exit %d):\n%s", devaf, verbose.code, verbose.output) + } }) t.Run("environment and flag precedence", func(t *testing.T) { diff --git a/internal/telemetry/doc_test.go b/internal/telemetry/doc_test.go index 740af9df0e..8ed8d96b97 100644 --- a/internal/telemetry/doc_test.go +++ b/internal/telemetry/doc_test.go @@ -25,8 +25,16 @@ func docBody(t *testing.T) string { // every-event props and the extras per event. Anything else on the page is // prose and not held to anything. func docProps(t *testing.T, body string) (common []string, perEvent map[string][]string) { + common, perEvent, _ = docPropsWithDocs(t, body) + return common, perEvent +} + +// docPropsWithDocs is docProps with the third column kept, keyed +// "\t", for the test that holds the doc's words to PropDoc. +func docPropsWithDocs(t *testing.T, body string) (common []string, perEvent map[string][]string, docs map[string]string) { t.Helper() perEvent = map[string][]string{} + docs = map[string]string{} inSection := false for _, line := range strings.Split(body, "\n") { trimmed := strings.TrimSpace(line) @@ -49,13 +57,59 @@ func docProps(t *testing.T, body string) (common []string, perEvent map[string][ } if cells[0] == "every event" { common = append(common, cells[1]) + docs[cells[0]+"\t"+cells[1]] = cells[2] continue } if cells[0] != "" { perEvent[cells[0]] = append(perEvent[cells[0]], cells[1]) + docs[cells[0]+"\t"+cells[1]] = cells[2] + } + } + return common, perEvent, docs +} + +// TestDocPropertyWordsMatchPropDoc holds the doc's third column to the table +// `codeaf telemetry show` prints from, word for word, and holds that table to +// the allowlist: every allowlisted prop has a description, and every +// description is of an allowlisted prop. +func TestDocPropertyWordsMatchPropDoc(t *testing.T) { + _, _, docs := docPropsWithDocs(t, docBody(t)) + for key, words := range docs { + event, prop, _ := strings.Cut(key, "\t") + if got := PropDoc(event, prop); got != words { + t.Errorf("%s %s: the doc says %q, PropDoc says %q", event, prop, words, got) + } + } + for _, name := range CommonPropNames() { + if PropDoc(EveryEvent, name) == "" { + t.Errorf("every-event prop %q has no PropDoc", name) + } + if _, ok := docs[EveryEvent+"\t"+name]; !ok { + t.Errorf("every-event prop %q has no row in the doc", name) + } + } + for _, event := range AllowlistedEvents() { + names := EventPropNames(event) + want := map[string]bool{} + for _, name := range AllowlistedProps(event) { + if PropDoc(EveryEvent, name) == "" { + want[name] = true + } + } + got := map[string]bool{} + for _, name := range names { + got[name] = true + if PropDoc(event, name) == "" { + t.Errorf("%s: %q has no PropDoc", event, name) + } + if _, ok := docs[event+"\t"+name]; !ok { + t.Errorf("%s: %q has no row in the doc", event, name) + } + } + if !reflect.DeepEqual(got, want) { + t.Errorf("%s: EventPropNames %v drift from the allowlist's own props %v", event, names, AllowlistedProps(event)) } } - return common, perEvent } func TestDocPropertyTableMatchesTheAllowlist(t *testing.T) { @@ -108,7 +162,7 @@ func TestDocCarriesTheNoticeAndTheSwitches(t *testing.T) { if !strings.Contains(body, Notice) { t.Error("docs/TELEMETRY.md must quote the notice byte for byte") } - for _, wanted := range []string{"CODEAF_TELEMETRY=off", "DO_NOT_TRACK=1", "telemetry show"} { + for _, wanted := range []string{"CODEAF_TELEMETRY=off", "DO_NOT_TRACK=1", "telemetry info", "telemetry show"} { if !strings.Contains(body, wanted) { t.Errorf("docs/TELEMETRY.md must mention %q", wanted) } diff --git a/internal/telemetry/events.go b/internal/telemetry/events.go index 05774eff04..b3dd0e0c5c 100644 --- a/internal/telemetry/events.go +++ b/internal/telemetry/events.go @@ -308,6 +308,122 @@ var allowedProps = map[string]map[string]bool{ "fault": merge(set(commonPropNames), set([]string{"mode", "scope", "fingerprint"})), } +// EveryEvent is the key under which [PropDoc] answers for the six props every +// event carries, and the exact words the doc's table spells in its first +// column for them. +const EveryEvent = "every event" + +// propDocs is what each allowlisted prop IS, in a person's words: the third +// column of docs/TELEMETRY.md's table, held here so that `codeaf telemetry +// show` and the doc read from one table and the doc test can fail the build +// when the two drift. Every allowlisted prop has a line, and the test holds +// that too. +var propDocs = map[string]map[string]string{ + EveryEvent: { + "codeaf_version": "the release tag this binary was built from, at most 64 characters", + "channel": "stable, rc, staging, dev, or unknown", + "os": "darwin, linux, windows, or other", + "arch": "amd64, arm64, or other", + "usage_context": "local (a person's machine), ci, or container", + "install_method": "script, source, or unknown", + }, + "session_started": { + "mode": "chat or task", + "resumed": "whether the session continued an earlier one", + }, + "session_ended": { + "mode": "chat or task", + "duration": "a band: under 1m, 1-5m, 5-30m, 30m-2h, 2h or more", + "turns": "a count band", + "model_calls": "a count band", + "model_calls_failed": "a count band", + "tool_calls": "a count band", + "tool_calls_failed": "a count band", + "cost_usd": "a dollar band", + "stop_reason": "done, error, incomplete, budget, turn-cap, deadline, price, question, interrupted, or unknown", + "exit_code": "0 to 5", + }, + "fault": { + "mode": "chat, task, or other", + "scope": "main, goroutine, or surface", + "fingerprint": "16 hex characters hashed from codeaf function names in the stack", + }, +} + +// PropDoc answers what one prop is, for the event named — [EveryEvent] for +// the six every event carries — or "" for a prop the table does not hold. +func PropDoc(event, prop string) string { return propDocs[event][prop] } + +// EventPropNames answers the props an event adds beyond the every-event six, +// in the doc's order, so a listing reads the way the doc reads. +func EventPropNames(event string) []string { + switch event { + case "session_started": + return []string{"mode", "resumed"} + case "session_ended": + return []string{"mode", "duration", "turns", "model_calls", "model_calls_failed", + "tool_calls", "tool_calls_failed", "cost_usd", "stop_reason", "exit_code"} + case "fault": + return []string{"mode", "scope", "fingerprint"} + } + return nil +} + +// The identity and envelope fields every event carries beside its props, and +// what each is, for a listing that shows a person the whole row. +const ( + InstallHashDoc = "sha256 of a random id kept on this machine; the id itself never leaves" + SessionHashDoc = "sha256 of the run id, one per session; absent on first_run" + EventIDDoc = "16 random bytes as hex, one per event" + EventTimeDoc = "when the event happened, UTC, to the second" +) + +// CountBandsDoc and CostBandsDoc spell the bands, as the doc spells them. +const ( + CountBandsDoc = "count bands are 0, 1, 2-5, 6-20, 21-100 and 100+" + CostBandsDoc = "dollar bands are 0, under 0.01, 0.01-0.1, 0.1-1, 1-10 and 10+" +) + +// StopReasons lists every stop reason, in the contract's order. +func StopReasons() []string { + return []string{StopDone, StopError, StopIncomplete, StopBudget, StopTurnCap, + StopDeadline, StopPrice, StopQuestion, StopInterrupted, StopUnknown} +} + +// exampleProps is one plausible value per event prop, spelled from the +// contract's own constants wherever the contract has one, so an example row +// can never show a value a real row could not carry. `codeaf telemetry show` +// prints one row per event from this table so a person sees the shape of +// what leaves before anything has. The fingerprint is the one invented value: +// sixteen hex characters, which is all a real one is. +var exampleProps = map[string]map[string]string{ + "session_started": { + "mode": string(ModeChat), + "resumed": "false", + }, + "session_ended": { + "mode": string(ModeChat), + "duration": Duration5To30m, + "turns": BucketSix, + "model_calls": BucketTwo1, + "model_calls_failed": BucketZero, + "tool_calls": BucketSix, + "tool_calls_failed": BucketZero, + "cost_usd": Cost10cTo1, + "stop_reason": StopDone, + "exit_code": "0", + }, + "fault": { + "mode": string(ModeChat), + "scope": ScopeMain, + "fingerprint": "3fa9c1e2b7d04e85", + }, +} + +// ExampleProp answers the example value for one event prop, or "" for a prop +// the table does not hold; a test holds the table to the allowlist. +func ExampleProp(event, prop string) string { return exampleProps[event][prop] } + // AllowlistedProps answers which key an event name accepts. It backs the doc // drift test and the privacy law: the table above is the allowlist, this is // its only reader outside this file's own tests. diff --git a/internal/telemetry/events_test.go b/internal/telemetry/events_test.go index 1d545f42c2..b60c084ecc 100644 --- a/internal/telemetry/events_test.go +++ b/internal/telemetry/events_test.go @@ -335,3 +335,31 @@ func TestNothingSentinelEverReachesTheWire(t *testing.T) { } } } + +// TestExamplePropsCoverTheAllowlistExactly holds the example table `codeaf +// telemetry show` prints to the allowlist: every prop an event adds has an +// example, no example names a prop the event cannot carry, and every example +// is a value the contract admits where the contract enumerates one. +func TestExamplePropsCoverTheAllowlistExactly(t *testing.T) { + common := set(CommonPropNames()) + for _, event := range AllowlistedEvents() { + added := EventPropNames(event) + for _, name := range added { + if ExampleProp(event, name) == "" { + t.Errorf("%s %s has no example value", event, name) + } + } + for name := range exampleProps[event] { + if common[name] || !allowedProps[event][name] { + t.Errorf("%s has an example for %q, which it does not add", event, name) + } + } + if len(added) == 0 && len(exampleProps[event]) != 0 { + t.Errorf("%s adds nothing but has examples", event) + } + } + if !ValidStopReason(ExampleProp("session_ended", "stop_reason")) { + t.Errorf("the example stop_reason is not one of the contract's") + } + +} diff --git a/internal/telemetry/notice.go b/internal/telemetry/notice.go index 588c6d5543..77bf08bb1c 100644 --- a/internal/telemetry/notice.go +++ b/internal/telemetry/notice.go @@ -14,7 +14,7 @@ const Notice = `codeaf sends anonymous usage counts to AgentField. Sent: version, OS, mode (chat or task), how many sessions, how many errors. Never: anything about you or your work. No prompts, code, file names, paths, repo names, keys, email, IP, or machine name. - See exactly what leaves: codeaf telemetry show + What is collected: codeaf telemetry info Turn off: CODEAF_TELEMETRY=off` // noticeOnce keeps the notice to one line per process even when several diff --git a/internal/telemetry/telemetry.go b/internal/telemetry/telemetry.go index 16b7f079d9..e45a71ccac 100644 --- a/internal/telemetry/telemetry.go +++ b/internal/telemetry/telemetry.go @@ -12,6 +12,7 @@ import ( "crypto/sha256" "encoding/hex" "encoding/json" + "fmt" "os" "runtime" "strings" @@ -359,6 +360,38 @@ func commonProps() map[string]any { } } +// PropValue is one every-event prop with the value this machine would send +// for it right now. +type PropValue struct { + Name string + Value string +} + +// CommonPropValues answers the six every-event props as this binary on this +// machine would fill them, in contract order — the same reader every event +// constructor uses, so what a listing shows is what an event would carry. It +// reads the install marker and the build; it writes nothing. +func CommonPropValues() []PropValue { + props := commonProps() + out := make([]PropValue, 0, len(commonPropNames)) + for _, name := range commonPropNames { + out = append(out, PropValue{Name: name, Value: fmt.Sprint(props[name])}) + } + return out +} + +// InstallIDHashIfMinted answers the install id hash when this machine has an +// install id already, and false when it has not: a reading verb must not mint +// one, because the id is written on the first SEND and a machine that has +// sent nothing has no identity to show. +func InstallIDHashIfMinted() (string, bool) { + stored := readStoredInstallID(telemetryFile("install_id")) + if len(stored) != 64 { + return "", false + } + return hashHex(stored), true +} + // hashHex is sha256 over one string, spelled once because three different // properties are hashes and none of them may drift into a different recipe. func hashHex(value string) string { diff --git a/internal/telemetry/telemetry_test.go b/internal/telemetry/telemetry_test.go index a96a2f9033..7a5cc83eed 100644 --- a/internal/telemetry/telemetry_test.go +++ b/internal/telemetry/telemetry_test.go @@ -360,7 +360,7 @@ const contractNotice = `codeaf sends anonymous usage counts to AgentField. Sent: version, OS, mode (chat or task), how many sessions, how many errors. Never: anything about you or your work. No prompts, code, file names, paths, repo names, keys, email, IP, or machine name. - See exactly what leaves: codeaf telemetry show + What is collected: codeaf telemetry info Turn off: CODEAF_TELEMETRY=off` func TestNoticeIsTheContractText(t *testing.T) { diff --git a/scripts/install.sh b/scripts/install.sh index 40d0a00e63..ee503684fb 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -7,16 +7,14 @@ LEGACY_REPOSITORY="Agent-Field/aforge-v2" # Remove after the one-release reposit CHANNEL="${CHANNEL:-stable}" INSTALL_NAME="${CODEAF_INSTALL_NAME:-codeaf}" VERSION="${VERSION:-}" -# The telemetry notice, verbatim from docs/TELEMETRY.md. -# The installer only writes a local install marker and prints this text; it -# never sends telemetry, and it makes no request that the download steps did -# not already make. -TELEMETRY_NOTICE='codeaf sends anonymous usage counts to AgentField. - Sent: version, OS, mode (chat or task), how many sessions, how many errors. - Never: anything about you or your work. No prompts, code, file names, - paths, repo names, keys, email, IP, or machine name. - See exactly what leaves: codeaf telemetry show - Turn off: CODEAF_TELEMETRY=off' +# The installer's three-line telemetry notice, verbatim from docs/TELEMETRY.md. +# The binary prints the full notice before the first session's events leave; +# the installer says the fact, the inspector and the switch. It writes a local +# install marker and prints this text, never sends telemetry, and makes no +# request that the download steps did not already make. +TELEMETRY_NOTICE='codeaf shares anonymous performance data with AgentField +codeaf does NOT share your prompts, code, files, or any private information +see what is shared: codeaf telemetry info · turn off: CODEAF_TELEMETRY=off' VERBOSE="${VERBOSE:-0}" NO_MODIFY_PATH="${CODEAF_NO_MODIFY_PATH:-${AFORGE_NO_MODIFY_PATH:-0}}" # legacy-name INSTALL_DIR="${CODEAF_INSTALL_DIR:-${AFORGE_INSTALL_DIR:-${HOME}/.codeaf/bin}}" # legacy-name @@ -110,6 +108,22 @@ write_install_marker() { # Printed once, at the very end of a successful install. The binary repeats it # before the first session's counts are ever sent. +# The one line a person still has to paste, printed last of all, between a +# blank line above and a blank line below, bold green on a terminal. Bare +# `export PATH=...` and nothing else, so it can be selected and pasted without +# trimming a prefix. Colour is skipped when stdout is not a terminal or +# NO_COLOR is set (https://no-color.org). +print_path_hint() { + local hint="$1" + [[ -n "$hint" ]] || return 0 + local on="" off="" + if [[ -t 1 && -z "${NO_COLOR:-}" ]]; then + on=$'\033[1;32m' + off=$'\033[0m' + fi + printf '\n%s%s%s\n\n' "$on" "$hint" "$off" +} + print_telemetry_notice() { if telemetry_off; then printf 'codeaf: anonymous usage counts are off (CODEAF_TELEMETRY=off or DO_NOT_TRACK=1)\n' >&2 @@ -408,10 +422,14 @@ download_release() { download_asset "$repository" "checksums.txt" "$TMP_ROOT/checksums.txt" } -if [[ -n "$DISPLAY_CHANNEL" ]]; then - printf 'codeaf: %s %s for %s/%s\n' "$DISPLAY_CHANNEL" "$TAG" "$OS" "$ARCH" -else - printf 'codeaf: %s for %s/%s\n' "$TAG" "$OS" "$ARCH" +# The channel and tag are not announced on a normal run: the installed +# binary names itself at the end, and that one line is the whole receipt. +if [[ "$VERBOSE" == "1" ]]; then + if [[ -n "$DISPLAY_CHANNEL" ]]; then + printf 'codeaf: %s %s for %s/%s\n' "$DISPLAY_CHANNEL" "$TAG" "$OS" "$ARCH" >&2 + else + printf 'codeaf: %s for %s/%s\n' "$TAG" "$OS" "$ARCH" >&2 + fi fi DOWNLOAD_REPOSITORY="$REPOSITORY" if ! download_release "$DOWNLOAD_REPOSITORY"; then @@ -419,10 +437,10 @@ if ! download_release "$DOWNLOAD_REPOSITORY"; then # Remove after the renamed repository has carried releases for one release. DOWNLOAD_REPOSITORY="$LEGACY_REPOSITORY" if ! download_release "$DOWNLOAD_REPOSITORY"; then - fail "could not download codeaf-${OS}-${ARCH}${extension}; check the tag on the Releases page" + fail "no codeaf-${OS}-${ARCH}${extension} in release ${TAG}; check the tag on the Releases page" fi else - fail "could not download codeaf-${OS}-${ARCH}${extension}; check the tag on the Releases page" + fail "no codeaf-${OS}-${ARCH}${extension} in release ${TAG}; check the tag on the Releases page" fi fi @@ -461,7 +479,9 @@ cp "$TMP_ROOT/$ASSET" "$INSTALL_TEMP" chmod 0755 "$INSTALL_TEMP" mv -f "$INSTALL_TEMP" "$INSTALL_DIR/$INSTALL_NAME${extension}" INSTALL_TEMP="" -printf 'codeaf: installed %s\n' "$INSTALL_DIR/$INSTALL_NAME${extension}" +if [[ "$VERBOSE" == "1" ]]; then + printf 'codeaf: installed %s\n' "$INSTALL_DIR/$INSTALL_NAME${extension}" >&2 +fi path_has_dir() { case ":${PATH}:" in @@ -488,9 +508,13 @@ append_path_line() { fi } +# The PATH line is not printed here. It is the last thing the installer says, +# after `codeaf version` and the telemetry notice, so the one line a person +# has to paste sits at the bottom of the screen where their eye already is. +PATH_HINT="" if [[ "$OS" != "windows" ]] && ! path_has_dir; then export_line="export PATH=\"$INSTALL_DIR:\$PATH\"" - printf 'codeaf: add it to this shell with: %s\n' "$export_line" + PATH_HINT="$export_line" if [[ "$NO_MODIFY_PATH" != "1" ]]; then shell_name=$(basename "${SHELL:-/bin/bash}") case "$shell_name" in @@ -510,6 +534,17 @@ if [[ "$OS" != "windows" ]] && ! path_has_dir; then fi fi +# The receipt is the installed binary naming itself: `codeaf version` is one +# line by law, so "installed " in front of it reads as one sentence. A file +# installed under another name (devaf) still says codeaf here, because the +# name is the file's and the product's is the sentence's. +if [[ "$RUN_BOOT_ADOPTION" == "1" ]]; then + version_line=$("$INSTALL_DIR/$INSTALL_NAME${extension}" version) +else + version_line=$(CODEAF_HOME="$STATE_ROOT" "$INSTALL_DIR/$INSTALL_NAME${extension}" version) +fi +printf 'installed %s\n' "$version_line" + # The install marker lives under the state root, and a custom install outside # it must not create the login's state folders: the marker is written when the # install is inside the state root or the root already exists, and skipped @@ -518,9 +553,4 @@ if [[ "$RUN_BOOT_ADOPTION" == "1" || -d "$STATE_ROOT" ]]; then write_install_marker "$STATE_ROOT" fi print_telemetry_notice - -if [[ "$RUN_BOOT_ADOPTION" == "1" ]]; then - "$INSTALL_DIR/$INSTALL_NAME${extension}" version -else - CODEAF_HOME="$STATE_ROOT" "$INSTALL_DIR/$INSTALL_NAME${extension}" version -fi +print_path_hint "$PATH_HINT" diff --git a/test/installer-telemetry.sh b/test/installer-telemetry.sh index b4d29871e8..2a4bd12f0e 100755 --- a/test/installer-telemetry.sh +++ b/test/installer-telemetry.sh @@ -1,9 +1,9 @@ #!/usr/bin/env bash # THE INSTALLER'S TELEMETRY DUTIES, PROVED WITHOUT A NETWORK. # -# The notice text lives once, byte for byte, in docs/TELEMETRY.md and is -# quoted in three places — the binary, this installer and the README — and -# only a test notices when one of them drifts. The installer's main body +# The notice text lives once, byte for byte, in docs/TELEMETRY.md: the full +# form quoted by the binary and the README, and the three-line form the +# installer prints. Only a test notices when one of them drifts. The installer's main body # downloads a release, so this test never sources it whole: it lifts out the # three telemetry functions and runs them against a temporary state root. # Nothing here opens a socket. @@ -15,10 +15,11 @@ doc=docs/TELEMETRY.md test -f "$doc" || { echo "docs/TELEMETRY.md is missing; this test reads the notice from it"; exit 1; } eval "$(awk '/^TELEMETRY_NOTICE=/{f=1} /^VERBOSE=/{f=0} f' "$script")" -eval "$(sed -n '/^telemetry_off()/,/^}/p; /^write_install_marker()/,/^}/p; /^print_telemetry_notice()/,/^}/p' "$script")" +eval "$(sed -n '/^telemetry_off()/,/^}/p; /^write_install_marker()/,/^}/p; /^print_telemetry_notice()/,/^}/p; /^print_path_hint()/,/^}/p' "$script")" type telemetry_off >/dev/null type write_install_marker >/dev/null type print_telemetry_notice >/dev/null +type print_path_hint >/dev/null pass=0 fail=0 @@ -82,14 +83,24 @@ ok "unwritable state root does not fail the install" 'write_install_marker /proc notice="$tmp/notice.txt" print_telemetry_notice 2> "$notice" +# The first fenced block under "The notice" is the binary's full notice, the +# second is the installer's three-line form; awk counts fences to tell them apart. expected=$(awk ' /^## The notice$/ {f=1; next} f && /^```$/ {f++; next} f == 2 {print} ' "$doc") +installer_expected=$(awk ' + /^## The notice$/ {f=1; next} + f && /^```$/ {f++; next} + f == 4 {print} +' "$doc") body=$(sed 1d "$notice") ok "one blank line before the notice" '[ -z "$(head -n 1 "$notice")" ]' -ok "notice matches docs/TELEMETRY.md verbatim" '[ "$body" = "$expected" ]' +ok "installer notice matches docs/TELEMETRY.md verbatim" '[ "$body" = "$installer_expected" ]' +ok "installer notice is three lines" '[ "$(printf "%s\n" "$body" | wc -l | tr -d " ")" = 3 ]' +ok "installer notice names the inspector and the switch" 'case "$body" in *"codeaf telemetry info"*CODEAF_TELEMETRY=off*) true;; *) false;; esac' +ok "installer notice names what is never shared" 'case "$body" in *"does NOT share your prompts, code, files"*) true;; *) false;; esac' readme_block=$(awk ' /^```text$/ {f = 1; buf = ""; next} /^```$/ {if (f && buf ~ /codeaf sends anonymous usage counts/) {print buf; exit} f = 0; next} @@ -102,7 +113,7 @@ for v in off 0 false OFF False; do export CODEAF_TELEMETRY="$v" out=$( print_telemetry_notice 2>&1 ) ok "CODEAF_TELEMETRY=$v opts out" 'case "$out" in *"off"*) true;; *) false;; esac' - ok "opt-out prints no notice body" 'case "$out" in *"anonymous usage counts to AgentField"*) false;; *) true;; esac' + ok "opt-out prints no notice body" 'case "$out" in *"anonymous performance data"*) false;; *) true;; esac' unset CODEAF_TELEMETRY done for v in 1 true TRUE; do @@ -112,12 +123,34 @@ for v in 1 true TRUE; do unset DO_NOT_TRACK done out=$( print_telemetry_notice 2>&1 ) -ok "unset prints the notice" 'case "$out" in *"anonymous usage counts to AgentField"*) true;; *) false;; esac' +ok "unset prints the notice" 'case "$out" in *"anonymous performance data with AgentField"*) true;; *) false;; esac' export CODEAF_TELEMETRY=1 out=$( print_telemetry_notice 2>&1 ) -ok "CODEAF_TELEMETRY=1 prints the notice" 'case "$out" in *"anonymous usage counts to AgentField"*) true;; *) false;; esac' +ok "CODEAF_TELEMETRY=1 prints the notice" 'case "$out" in *"anonymous performance data with AgentField"*) true;; *) false;; esac' unset CODEAF_TELEMETRY +# --- the PATH line comes last ------------------------------------------------- +# The line a person has to paste is the installer's final word: bare, after a +# blank line, and never prefixed with "codeaf: add it to this shell with:", +# which made it a sentence to trim rather than a line to select. + +hint='export PATH="/x/bin:$PATH"' +has_escape() { printf '%s' "$1" | grep -q "$(printf '\033')"; } +ok "an empty hint prints nothing" '[ -z "$(print_path_hint "" 2>&1)" ]' +out=$(print_path_hint "$hint"; printf x); out=${out%x} +expected_hint=$(printf '\n%s\n\nx' "$hint"); expected_hint=${expected_hint%x} +ok "the hint is one blank line, the bare export, one blank line" '[ "$out" = "$expected_hint" ]' +ok "channel and path announcements go to stderr, verbose only" '[ -z "$(grep -E "printf .codeaf: (installed|%s %s for|%s for)" "$script" | grep -v ">&2")" ]' +ok "the receipt is the installed binary naming itself" 'grep -q "printf .installed %s" "$script"' +ok "no colour when stdout is not a terminal" '! has_escape "$out"' +export NO_COLOR=1 +out=$(print_path_hint "$hint" 2>&1) +ok "NO_COLOR is respected" '! has_escape "$out"' +unset NO_COLOR +last_line=$(grep -v '^[[:space:]]*#' "$script" | grep -v '^[[:space:]]*$' | tail -n 1) +ok "the hint is the installer's last line" '[ "$last_line" = "print_path_hint \"\$PATH_HINT\"" ]' +ok "the old prefixed sentence is gone" '! grep -q "add it to this shell with" "$script"' + # --- nothing new on the wire -------------------------------------------------- ok "no telemetry request anywhere in the installer" '! grep -qE "agentfield\.ai/api|POST" "$script"'