From a0811c40141bd42f17a8ff3f5320f391c1f1197b Mon Sep 17 00:00:00 2001 From: Danil Pismenny Date: Mon, 7 Sep 2026 00:56:24 +0300 Subject: [PATCH 01/13] feat: gate legacy sources before installation --- .github/workflows/local-e2e.yml | 12 ++ CHANGELOG.md | 5 + README.md | 27 ++++ docs/component-delivery.md | 41 ++++++ docs/source-format-bridge.md | 89 ++++++++++++ internal/cli/capabilities.go | 53 ++++++++ internal/cli/capabilities_test.go | 71 ++++++++++ internal/cli/cli.go | 3 + internal/cli/cli_test.go | 5 + internal/doctor/doctor_test.go | 1 + internal/ownership/source.go | 2 +- internal/ownership/source_format.go | 164 +++++++++++++++++++++++ internal/ownership/source_format_test.go | 138 +++++++++++++++++++ internal/ownership/source_test.go | 5 + internal/push/push_test.go | 3 + scripts/e2e-init-update.sh | 1 + scripts/e2e-source-format.sh | 39 ++++++ 17 files changed, 658 insertions(+), 1 deletion(-) create mode 100644 docs/component-delivery.md create mode 100644 docs/source-format-bridge.md create mode 100644 internal/cli/capabilities.go create mode 100644 internal/cli/capabilities_test.go create mode 100644 internal/ownership/source_format.go create mode 100644 internal/ownership/source_format_test.go create mode 100755 scripts/e2e-source-format.sh diff --git a/.github/workflows/local-e2e.yml b/.github/workflows/local-e2e.yml index 5f58e09..ebbb682 100644 --- a/.github/workflows/local-e2e.yml +++ b/.github/workflows/local-e2e.yml @@ -25,3 +25,15 @@ jobs: env: E2E_BINARY: ${{ runner.temp }}/memory-bank-cli run: bash scripts/e2e-init-update.sh + - name: Check out supported manifestless legacy source + uses: actions/checkout@v7 + with: + repository: dapi/memory-bank + ref: f1f04de843aef45a2425d4a7351d577bbf89e940 + path: .fixtures/legacy-source + persist-credentials: false + - name: Verify real bridge source-format compatibility + env: + E2E_BINARY: ${{ runner.temp }}/memory-bank-cli + LEGACY_SOURCE: ${{ github.workspace }}/.fixtures/legacy-source + run: bash scripts/e2e-source-format.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 9bb4f32..50bae56 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ and this project follows [Semantic Versioning](https://semver.org/spec/v2.0.0.ht ## [Unreleased] +- Gate installation sources by a pinned legacy compatibility list or strict source-format + declaration; reject unknown formats before planning or changing downstream files. +- Add the JSON `capabilities --require` handshake for installation entrypoints. + + ## [2.3.0] - 2026-09-06 ### Added diff --git a/README.md b/README.md index f59455f..a34326b 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,33 @@ Existing locks from the legacy payload roots are migrated conservatively: unchanged files adopt canonical ownership, while local customization is preserved for explicit resolution. +## Source compatibility + +Before planning any `init`, `pull` or doctor repair, the CLI checks the pinned +source format. The supported manifestless source is Memory Bank commit +`f1f04de843aef45a2425d4a7351d577bbf89e940`. Unknown manifestless commits are +rejected before downstream writes, including saved pull plans. + +Custom legacy sources must commit `memory-bank-source.json` at the checkout +root (outside `template/`) and consumers must explicitly repin that commit: + +```json +{"schema_version":1,"payload_format":"legacy/v1","capabilities":["legacy/v1"]} +``` + +This bridge supports legacy payloads only. Component payloads require a later +CLI. Check capabilities before invoking an installer: + +```sh +memory-bank-cli capabilities --require legacy/v1 +``` + +The command emits JSON and returns nonzero for an unsupported capability. +Keep the previous CLI/source pair to continue using an undeclared custom +source without changing it. See the [bridge contract](docs/source-format-bridge.md) +for the supported-source boundary and wire format, and the +[component delivery plan](docs/component-delivery.md) for the subsequent work. + ## Analyse an execution handoff Inspect an Execution Handoff without changing the handoff or Memory Bank: diff --git a/docs/component-delivery.md b/docs/component-delivery.md new file mode 100644 index 0000000..9a73d13 --- /dev/null +++ b/docs/component-delivery.md @@ -0,0 +1,41 @@ +# Component installation delivery + +Tracker: [CLI #62](https://github.com/dapi/memory-bank-cli/issues/62). +Source: [memory-bank #141](https://github.com/dapi/memory-bank/issues/141). +CLI owner: memory-bank-cli. The shared protocol belongs to the +[template contract](https://github.com/dapi/memory-bank/blob/feat/141-component-adoption/docs/components.md). +The [FT-141 brief](https://github.com/dapi/memory-bank/blob/feat/141-component-adoption/memory-bank/features/FT-141/brief.md) +owns imported requirements and acceptance; this file owns CLI realization and evidence routing. + +Grounded CLI revision: `ac7101c307e65566787bdb32a1bdad40b9a8b995`. +`internal/ownership/update.go` provides a handle-relative payload/agent/lock transaction. +`internal/ownership/source.go` verifies pinned blobs but lacks a component source gate. +`internal/doctor/governance.go` applies feature lifecycle checks type-wide. +Baseline `env -u GOROOT go test ./...` passes all packages on 2026-09-07. + +## Gates and realization + +CLI-01 uses the independently reviewed [bridge plan](source-format-bridge.md). CLI-02/03 wait for the shared component design gate. CLI code, tests +and evidence stay here. The slice is independently verifiable with synthetic source fixtures; +the template PR supplies the final cross-repository source integration. + +| Step | Imported requirement | CLI surface | Verification | +| --- | --- | --- | --- | +| CLI-01 | REQ-06/08 | ownership/source_format.go; source verification; capabilities command | Real bridge/pre-bridge binary fixtures; incompatible sources cannot mutate | +| CLI-02 | REQ-01/02/04 | ownership/components.go; Lock/Options/run and resolution planning | Preset/adapter/ownership, no-op, stale-plan and rollback tests | +| CLI-03 | REQ-03/04/05 | ownership/documents.go; validator; doctor/lint; document commands | Creation/adoption, tampering, immutable rules, selectors and legacy migration | + +## Acceptance and evidence + +Imported SC-01…06 and NEG-01…05 are mandatory CLI checks; SC-07/08 additionally use the actual +template payload. Concrete test names, commands and CI results are recorded in the PR as they +are delivered. Required local checks: `env -u GOROOT go test ./...`, `env -u GOROOT go vet ./...`, existing hermetic +init/pull E2E and component fixtures. Required GitHub Actions must be green on the reviewed +commit. Independent code-converge implementation and simplify review must have no actionable +findings. Unexecuted checks are not evidence. + +## Rollout + +Prepare the bridge commit and record its actual binary identity before component support. +Component source must wait for supporting CLI. No live downstream mutation, merge or release +publication belongs to this task. Related PRs state the required release order explicitly. diff --git a/docs/source-format-bridge.md b/docs/source-format-bridge.md new file mode 100644 index 0000000..334b8d1 --- /dev/null +++ b/docs/source-format-bridge.md @@ -0,0 +1,89 @@ +# Source-format bridge + +This is W1 of [CLI #62](https://github.com/dapi/memory-bank-cli/issues/62), supporting +[memory-bank #141](https://github.com/dapi/memory-bank/issues/141). Scope is source format +classification before any installation plan or mutation. Component installation and adoption +remain later units governed by the shared component design. + +## Requirements and acceptance + +- BR-01: A bridge accepts the pinned manifestless template commit + `f1f04de843aef45a2425d4a7351d577bbf89e940`, and rejects an unknown manifestless source. +- BR-02: A declared source has `memory-bank-source.json` at its checkout root, schema 1, + `payload_format: legacy/v1`, and `capabilities: [legacy/v1]`. The file must be a tracked + regular Git blob at the selected commit. Unknown/duplicate fields, trailing JSON, + unsupported schema/format/capabilities, missing mandatory capabilities and a component + manifest inside a declared legacy source are errors. The component marker is + `/memory-bank/components.json` for template/, or `/components.json` + for the legacy payload roots. The bridge cannot process a component source. +- BR-03: Rejection leaves every downstream file and lock unchanged, including dry-run, + `pull --plan`, `pull --apply-plan` and doctor repair paths using ownership.Init. +- BR-04: `capabilities [--require CAP ...]` prints machine-readable supported capabilities + and returns nonzero when a required capability is unavailable. The bridge advertises + `source-format/v1` and `legacy/v1`; it does not advertise components or adoption. +- BR-05: The manifestless SHA explicitly listed in BR-01 retains its legacy installation + behavior. Declared legacy fixture sources retain the existing ownership/transaction behavior. + Other previously usable custom commits are intentionally unsupported by this bridge: + owners must keep their previous CLI/source pair, or add a declaration and explicitly repin. + No automatic repinning or rewriting of a source is performed. + +The declaration is an explicit supported legacy format for custom sources, not an allowlist +bypass for manifestless source. The built-in SHA list is the only way to accept an undeclared +legacy tree. Future formats must change the declaration. Pre-bridge programs remain unable to +read this gate: their direct execution on component payload is unsupported; the later template +entrypoint checks capabilities before calling installer. This is the explicit compatibility +boundary accepted in issue #141: no guarantee is made for direct pre-bridge invocation on +a component source. BR-01–BR-05 apply to the bridge binary. Retrofitting old executables is +out of scope and cannot be achieved by changing a source manifest. + +The capability wire response is exactly one JSON object on stdout followed by a newline: +`{"schema_version":1,"cli_version":"","capabilities":["source-format/v1","legacy/v1"],"unsupported":[]}`. +Arrays contain strings; capabilities have deterministic declared order. Repeated `--require CAP` +is allowed; unsupported contains the requested unavailable values in request order. Exit 0 means +all requirements are available, exit 1 means at least one is unavailable (same JSON response), +exit 2 is invalid command syntax (diagnostic on stderr). No prose is printed on stdout. +The future entrypoint relies on exit status of --require, not on parsing incidental output. + +## Design and execution plan + +Grounded revision: `ac7101c307e65566787bdb32a1bdad40b9a8b995`. +`internal/ownership/source.go#verifySourceCheckout` is called by both run and PlanPull and +already verifies clean checkout and pinned regular blobs. The complete repair call chain is +`cli.runDoctor --fix → ownership.Init → run → verifySourceCheckout`; it reaches the same gate. +Add an explicit doctor-repair rejection test asserting unchanged target files and no lock. +`cli.runOwnership --apply-plan → ApplyResolutionPlan → PlanPull → verifySourceCheckout` +revalidates the actual source before comparing a saved plan; the final +`ApplyResolutionPlan → Update → run → verifySourceCheckout` validates it again before writes. +Thus a plan produced by any earlier binary conveys no source-format exemption. The negative +apply-plan fixture supplies a matching source identity in an old-format plan against an +unsupported checkout and asserts a source-format error and byte-identical downstream state. +All source-consuming mutation routes listed in BR-03 therefore use the same gate. +Add sourceGate there after pinned +payload-root validation, using Git objects rather than mutable working-tree files. +`internal/ownership/source_format.go` owns declaration schema/classification and strict decoding. +`internal/cli/cli.go#Run` adds the capability command; no new mutating command is introduced. + +The only alternatives are accepting every undeclared tree (violates BR-01) or refusing all +legacy trees (violates compatibility). Use the compiled supported SHA plus strict declarations. +No state schema, transaction engine, target paths or template payload bytes change in W1. + +Test helpers in `internal/ownership/source_test.go#commitTestSource`, +`internal/cli/cli_test.go#commitCLISource` and `scripts/e2e-init-update.sh#setup_case` declare +synthetic legacy fixtures without disabling production gates. New source-format tests retain +explicit undeclared/invalid fixtures and verify no mutation. Any other fixture producer that +uses an actual Git source receives the same declaration; fixture changes are supporting BR-05. + +## Validation and readiness + +Validation profile: standard (CLI/source format contract). No live state or release mutation. +Before implementation: independent document review of this plan must be clean. After it: +1. Add strict source classification and capability reporting (BR-01/02/04). +2. Add positive/negative pinned Git fixtures and update existing fixture producers (BR-03/05). +3. Run `env -u GOROOT go test ./...`, `env -u GOROOT go vet ./...`, build the bridge and run hermetic E2E. +4. Independent code-converge review; fix findings and re-review. Record bridge commit and binary + SHA-256 before adding component capabilities in the subsequent unit. + +Baseline tests already pass with `env -u GOROOT go test ./...`; the inherited GOROOT mismatches +the PATH compiler, so use a consistent toolchain per command. Failed preflight preserves the +legacy tree. Code rollback is a revert before release. A bridge release is prepared as a +separate commit/PR; publishing it is outside the current implementation/PR task. diff --git a/internal/cli/capabilities.go b/internal/cli/capabilities.go new file mode 100644 index 0000000..bf8fd64 --- /dev/null +++ b/internal/cli/capabilities.go @@ -0,0 +1,53 @@ +package cli + +import ( + "encoding/json" + "errors" + "flag" + "fmt" + "io" + + "github.com/dapi/memory-bank-cli/internal/ownership" +) + +func runCapabilities(arguments []string, version string, stdout, stderr io.Writer) int { + flags := flag.NewFlagSet("memory-bank-cli capabilities", flag.ContinueOnError) + flags.SetOutput(stderr) + var required entrypointFlags + flags.Var(&required, "require", "required capability (repeatable)") + if err := flags.Parse(arguments); err != nil { + if errors.Is(err, flag.ErrHelp) { + return exitSuccess + } + return exitUsage + } + if flags.NArg() != 0 { + fmt.Fprintln(stderr, "capabilities: unexpected positional arguments") + return exitUsage + } + report := struct { + SchemaVersion int `json:"schema_version"` + CLIVersion string `json:"cli_version"` + Capabilities []string `json:"capabilities"` + Unsupported []string `json:"unsupported"` + }{1, version, ownership.SupportedCapabilities(), []string{}} + for _, requested := range required { + found := false + for _, available := range report.Capabilities { + if requested == available { + found = true + } + } + if !found { + report.Unsupported = append(report.Unsupported, requested) + } + } + if err := json.NewEncoder(stdout).Encode(report); err != nil { + fmt.Fprintln(stderr, err) + return exitFailure + } + if len(report.Unsupported) != 0 { + return exitFailure + } + return exitSuccess +} diff --git a/internal/cli/capabilities_test.go b/internal/cli/capabilities_test.go new file mode 100644 index 0000000..1a1e762 --- /dev/null +++ b/internal/cli/capabilities_test.go @@ -0,0 +1,71 @@ +package cli + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestCapabilitiesWireContract(t *testing.T) { + for _, tc := range []struct { + args []string + code int + unsupported string + }{ + {nil, 0, `[]`}, + {[]string{"--require", "legacy/v1", "--require", "source-format/v1"}, 0, `[]`}, + {[]string{"--require", "components/v1", "--require", "adoption/v1"}, 1, `["components/v1","adoption/v1"]`}, + } { + var out, err bytes.Buffer + code := Run(append([]string{"capabilities"}, tc.args...), "test-version", &out, &err) + want := `{"schema_version":1,"cli_version":"test-version","capabilities":["source-format/v1","legacy/v1"],"unsupported":` + tc.unsupported + "}\n" + if code != tc.code || out.String() != want || err.Len() != 0 { + t.Fatalf("code=%d stdout=%s stderr=%s", code, out.String(), err.String()) + } + } + var out, err bytes.Buffer + if code := Run([]string{"capabilities", "unexpected"}, "test", &out, &err); code != 2 || out.Len() != 0 { + t.Fatalf("invalid syntax code=%d stdout=%s", code, out.String()) + } +} + +func TestDoctorRepairRejectsUnsupportedSourceBeforeMutation(t *testing.T) { + for _, dry := range []bool{false, true} { + repo, source := t.TempDir(), t.TempDir() + readme := []byte("---\ndoc_function: index\npurpose: Fixture.\nstatus: active\n---\n# Memory Bank\n") + for _, root := range []string{repo, source} { + if err := os.MkdirAll(filepath.Join(root, "memory-bank"), 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "memory-bank/README.md"), readme, 0644); err != nil { + t.Fatal(err) + } + } + if err := os.WriteFile(filepath.Join(source, "memory-bank-source.json"), []byte(`{"schema_version":99}`), 0644); err != nil { + t.Fatal(err) + } + ref := commitCLISource(t, source, "unsupported") + args := []string{"doctor", "--fix", "--repo-root", repo, "--source", source, "--template-version", "fixture", "--source-ref", ref} + if dry { + args = append(args, "--dry-run") + } + var out, stderr bytes.Buffer + if code := Run(args, "test", &out, &stderr); code != 1 || !strings.Contains(stderr.String(), "unsupported source format") { + t.Fatalf("code=%d stderr=%s", code, stderr.String()) + } + got, err := os.ReadFile(filepath.Join(repo, "memory-bank/README.md")) + if err != nil || !bytes.Equal(got, readme) { + t.Fatal("repair changed README") + } + entries, err := os.ReadDir(filepath.Join(repo, "memory-bank")) + if err != nil || len(entries) != 1 { + t.Fatalf("repair changed tree: %v %v", entries, err) + } + entries, err = os.ReadDir(repo) + if err != nil || len(entries) != 1 { + t.Fatalf("repair changed root: %v %v", entries, err) + } + } +} diff --git a/internal/cli/cli.go b/internal/cli/cli.go index 0fff519..3b75842 100644 --- a/internal/cli/cli.go +++ b/internal/cli/cli.go @@ -58,6 +58,8 @@ func Run(arguments []string, version string, stdout, stderr io.Writer) int { } switch arguments[0] { + case "capabilities": + return runCapabilities(arguments[1:], version, stdout, stderr) case "analyze-graph": return runAnalyzeGraph(arguments[1:], stdout, stderr) case "lint": @@ -124,6 +126,7 @@ func printRootUsage(writer io.Writer) { fmt.Fprintln(writer, "Usage: memory-bank-cli [options]") fmt.Fprintln(writer) fmt.Fprintln(writer, "Commands:") + fmt.Fprintln(writer, " capabilities Report supported source formats and required capabilities") fmt.Fprintln(writer, " analyze-graph Analyse typed execution-context handoff evidence") fmt.Fprintln(writer, " init Adopt or install a template and create its ownership lock") fmt.Fprintln(writer, " pull Safely synchronize a template using its ownership lock") diff --git a/internal/cli/cli_test.go b/internal/cli/cli_test.go index e254f7f..fa8d833 100644 --- a/internal/cli/cli_test.go +++ b/internal/cli/cli_test.go @@ -28,6 +28,11 @@ func testRepository(t *testing.T) string { func commitCLISource(t *testing.T, root, message string) string { t.Helper() + if _, err := os.Lstat(filepath.Join(root, "memory-bank-source.json")); os.IsNotExist(err) { + if err := os.WriteFile(filepath.Join(root, "memory-bank-source.json"), []byte(`{"schema_version":1,"payload_format":"legacy/v1","capabilities":["legacy/v1"]}`), 0o644); err != nil { + t.Fatal(err) + } + } if _, err := os.Stat(filepath.Join(root, ".git")); os.IsNotExist(err) { runCLIGit(t, root, "init", "--quiet") } diff --git a/internal/doctor/doctor_test.go b/internal/doctor/doctor_test.go index b5d5501..37601f4 100644 --- a/internal/doctor/doctor_test.go +++ b/internal/doctor/doctor_test.go @@ -298,6 +298,7 @@ func TestTemplateOwnedAgentFileUsesLockDriftContract(t *testing.T) { t.Fatal(err) } } + writeSource("memory-bank-source.json", `{"schema_version":1,"payload_format":"legacy/v1","capabilities":["legacy/v1"]}`) writeSource("template/AGENTS.md", "canonical template instructions\n") writeSource("template/memory-bank/README.md", "---\ndoc_function: index\npurpose: Test Memory Bank.\nstatus: active\n---\n# Memory Bank\n") runGit(t, source, "init", "--quiet") diff --git a/internal/ownership/source.go b/internal/ownership/source.go index 0bb8753..634de83 100644 --- a/internal/ownership/source.go +++ b/internal/ownership/source.go @@ -114,7 +114,7 @@ func verifySourceCheckout(root, expectedRef string) error { if err := verifySourcePayload(root, expectedRef, payloadRoot); err != nil { return err } - return nil + return verifySourceFormat(root, expectedRef, payloadRoot) } func verifySourcePayload(root, expectedRef, payloadRoot string) error { diff --git a/internal/ownership/source_format.go b/internal/ownership/source_format.go new file mode 100644 index 0000000..f38b8e0 --- /dev/null +++ b/internal/ownership/source_format.go @@ -0,0 +1,164 @@ +package ownership + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "path" + "strings" +) + +// Source classification is specified in docs/source-format-bridge.md. +const SourceDeclarationFile = "memory-bank-source.json" + +// SupportedLegacySourceRefs returns the published immutable manifestless sources. +func SupportedLegacySourceRefs() []string { + return []string{"f1f04de843aef45a2425d4a7351d577bbf89e940"} +} + +// SupportedCapabilities is the versioned capability handshake with installers. +func SupportedCapabilities() []string { return []string{"source-format/v1", "legacy/v1"} } + +type sourceDeclaration struct { + SchemaVersion int `json:"schema_version"` + PayloadFormat string `json:"payload_format"` + Capabilities []string `json:"capabilities"` +} + +func verifySourceFormat(root, ref, payloadRoot string) error { + marker := path.Join(payloadRoot, "components.json") + if payloadRoot == CanonicalTemplateRoot { + marker = path.Join(payloadRoot, "memory-bank/components.json") + } + entry, err := gitOutput(root, "ls-tree", ref, "--", marker) + if err != nil { + return fmt.Errorf("inspect component marker: %w", err) + } + if entry != "" { + return errors.New("unsupported component source: this bridge supports only legacy/v1; upgrade the CLI before installing components") + } + data, exists, err := readSourceDeclaration(root, ref) + if err != nil { + return err + } + if !exists { + for _, allowed := range SupportedLegacySourceRefs() { + if strings.EqualFold(ref, allowed) { + return nil + } + } + return fmt.Errorf("unsupported manifestless source %s: use a published supported legacy commit or a declared source", ref) + } + var fields map[string]json.RawMessage + if err := decodeSourceJSON(data, &fields); err != nil { + return fmt.Errorf("invalid %s: %w", SourceDeclarationFile, err) + } + for key := range fields { + if key != "schema_version" && key != "payload_format" && key != "capabilities" { + return fmt.Errorf("invalid %s: unknown field %q", SourceDeclarationFile, key) + } + } + var declaration sourceDeclaration + if err := decodeSourceJSON(data, &declaration); err != nil { + return fmt.Errorf("invalid %s: %w", SourceDeclarationFile, err) + } + if declaration.SchemaVersion != 1 || declaration.PayloadFormat != "legacy/v1" { + return fmt.Errorf("unsupported source format: schema=%d payload_format=%q", declaration.SchemaVersion, declaration.PayloadFormat) + } + seen := map[string]bool{} + for _, capability := range declaration.Capabilities { + if seen[capability] { + return fmt.Errorf("duplicate source capability %q", capability) + } + seen[capability] = true + supported := false + for _, available := range SupportedCapabilities() { + if capability == available { + supported = true + } + } + if !supported { + return fmt.Errorf("unsupported source capability %q", capability) + } + } + if !seen["legacy/v1"] { + return errors.New("source declaration requires capability legacy/v1") + } + return nil +} + +func readSourceDeclaration(root, ref string) ([]byte, bool, error) { + tree, err := gitOutput(root, "ls-tree", "-z", ref, "--", SourceDeclarationFile) + if err != nil { + return nil, false, fmt.Errorf("inspect source declaration: %w", err) + } + if tree == "" { + return nil, false, nil + } + header, name, ok := strings.Cut(strings.TrimSuffix(tree, "\x00"), "\t") + fields := strings.Fields(header) + if !ok || name != SourceDeclarationFile || len(fields) != 3 || fields[1] != "blob" || (fields[0] != "100644" && fields[0] != "100755") { + return nil, false, errors.New("source declaration must be a tracked regular Git blob") + } + data, err := gitBytes(root, "cat-file", "blob", fields[2]) + return data, true, err +} + +// Reject duplicate keys before decoding: encoding/json otherwise accepts the last +// value, which makes the declared source format ambiguous across implementations. +func decodeSourceJSON(data []byte, target any) error { + decoder := json.NewDecoder(bytes.NewReader(data)) + var value func() error + value = func() error { + token, err := decoder.Token() + if err != nil { + return err + } + delim, container := token.(json.Delim) + if !container { + return nil + } + switch delim { + case '{': + keys := map[string]bool{} + for decoder.More() { + keyToken, err := decoder.Token() + if err != nil { + return err + } + key, ok := keyToken.(string) + if !ok { + return errors.New("invalid JSON object key") + } + if keys[key] { + return fmt.Errorf("duplicate JSON field %q", key) + } + keys[key] = true + if err := value(); err != nil { + return err + } + } + case '[': + for decoder.More() { + if err := value(); err != nil { + return err + } + } + default: + return errors.New("unexpected JSON delimiter") + } + _, err = decoder.Token() + return err + } + if err := value(); err != nil { + return err + } + if _, err := decoder.Token(); !errors.Is(err, io.EOF) { + return errors.New("trailing JSON data") + } + decoder = json.NewDecoder(bytes.NewReader(data)) + decoder.DisallowUnknownFields() + return decoder.Decode(target) +} diff --git a/internal/ownership/source_format_test.go b/internal/ownership/source_format_test.go new file mode 100644 index 0000000..d10ff14 --- /dev/null +++ b/internal/ownership/source_format_test.go @@ -0,0 +1,138 @@ +package ownership + +import ( + "os" + "path/filepath" + "reflect" + "strings" + "testing" +) + +const legacyDeclaration = `{"schema_version":1,"payload_format":"legacy/v1","capabilities":["legacy/v1"]}` + +func rawSourceCommit(t *testing.T, source string) string { + t.Helper() + runGitTest(t, source, "init", "--quiet") + runGitTest(t, source, "add", "--all") + runGitTest(t, source, "-c", "user.name=Test", "-c", "user.email=test@example.invalid", "commit", "--quiet", "-m", "source-format fixture") + return runGitTest(t, source, "rev-parse", "HEAD") +} + +func treeSnapshot(t *testing.T, root string) map[string]string { + t.Helper() + result := map[string]string{} + err := filepath.WalkDir(root, func(p string, d os.DirEntry, err error) error { + if err != nil { + return err + } + rel, _ := filepath.Rel(root, p) + if d.IsDir() { + result[rel] = "dir" + return nil + } + info, err := d.Info() + if err != nil { + return err + } + data, err := os.ReadFile(p) + if err != nil { + return err + } + result[rel] = info.Mode().String() + ":" + string(data) + return nil + }) + if err != nil { + t.Fatal(err) + } + return result +} + +func TestSourceFormatRejectionIsNonMutating(t *testing.T) { + cases := []struct{ name, declaration, marker, want string }{ + {"unknown manifestless", "", "", "unsupported manifestless"}, + {"unknown schema", `{"schema_version":2,"payload_format":"legacy/v1","capabilities":["legacy/v1"]}`, "", "unsupported source format"}, + {"component format", `{"schema_version":1,"payload_format":"components/v1","capabilities":["components/v1"]}`, "", "unsupported source format"}, + {"unknown field", `{"schema_version":1,"payload_format":"legacy/v1","capabilities":["legacy/v1"],"extra":true}`, "", "unknown field"}, + {"duplicate field", `{"schema_version":1,"schema_version":1,"payload_format":"legacy/v1","capabilities":["legacy/v1"]}`, "", "duplicate JSON field"}, + {"trailing JSON", legacyDeclaration + `{}`, "", "trailing JSON"}, + {"missing capability", `{"schema_version":1,"payload_format":"legacy/v1"}`, "", "requires capability"}, + {"unsupported capability", `{"schema_version":1,"payload_format":"legacy/v1","capabilities":["legacy/v1","components/v1"]}`, "", "unsupported source capability"}, + {"duplicate capability", `{"schema_version":1,"payload_format":"legacy/v1","capabilities":["legacy/v1","legacy/v1"]}`, "", "duplicate source capability"}, + {"legacy with component marker", legacyDeclaration, "template/memory-bank/components.json", "unsupported component source"}, + {"manifestless with marker", "", "template/memory-bank/components.json", "unsupported component source"}, + {"null", `null`, "", "unsupported source format"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + source, repo := t.TempDir(), t.TempDir() + write(t, source, "template/memory-bank/dna/rule.md", "incoming\n") + if tc.declaration != "" { + write(t, source, SourceDeclarationFile, tc.declaration) + } + if tc.marker != "" { + write(t, source, tc.marker, "{}") + } + ref := rawSourceCommit(t, source) + options := Options{RepoRoot: repo, SourceRoot: source, TemplateVersion: "fixture", SourceRef: ref} + write(t, repo, "user.md", "preserve me") + check := func(err error, before map[string]string) { + t.Helper() + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("want %q, got %v", tc.want, err) + } + if !reflect.DeepEqual(before, treeSnapshot(t, repo)) { + t.Fatal("rejected operation changed downstream") + } + } + for _, dry := range []bool{false, true} { + options.DryRun = dry + before := treeSnapshot(t, repo) + _, err := Init(options) + check(err, before) + } + validSource := t.TempDir() + write(t, validSource, "template/memory-bank/dna/rule.md", "base\n") + validRef := commitTestSource(t, validSource) + if _, err := Init(Options{RepoRoot: repo, SourceRoot: validSource, TemplateVersion: "base", SourceRef: validRef}); err != nil { + t.Fatal(err) + } + for _, dry := range []bool{false, true} { + options.DryRun = dry + before := treeSnapshot(t, repo) + _, err := Update(options) + check(err, before) + } + before := treeSnapshot(t, repo) + _, err := PlanPull(options) + check(err, before) + // A saved pre-bridge plan cannot exempt its source from revalidation. + _, err = ApplyResolutionPlan(options, ResolutionPlan{FormatVersion: 1, Template: Template{Version: "fixture", SourceRef: ref}}) + check(err, before) + }) + } +} + +func TestSourceDeclarationMustBeRegularTrackedBlob(t *testing.T) { + source := t.TempDir() + write(t, source, "template/memory-bank/README.md", "payload") + write(t, source, "declaration.json", legacyDeclaration) + symlinkForTest(t, "declaration.json", filepath.Join(source, SourceDeclarationFile)) + ref := rawSourceCommit(t, source) + if err := verifySourceCheckout(source, ref); err == nil || !strings.Contains(err.Error(), "regular Git blob") { + t.Fatalf("got %v", err) + } +} + +func TestComponentMarkerRejectedInLegacyPayloadRoots(t *testing.T) { + for _, root := range []string{"memory-bank", "memory-bank-template"} { + t.Run(root, func(t *testing.T) { + source := t.TempDir() + write(t, source, root+"/components.json", "{}") + write(t, source, SourceDeclarationFile, legacyDeclaration) + ref := rawSourceCommit(t, source) + if err := verifySourceCheckout(source, ref); err == nil || !strings.Contains(err.Error(), "unsupported component source") { + t.Fatalf("got %v", err) + } + }) + } +} diff --git a/internal/ownership/source_test.go b/internal/ownership/source_test.go index 0436bc8..5809597 100644 --- a/internal/ownership/source_test.go +++ b/internal/ownership/source_test.go @@ -404,6 +404,11 @@ func TestPinnedSourceExecutableModeIsInstalled(t *testing.T) { func commitTestSource(t *testing.T, root string) string { t.Helper() + if _, err := os.Lstat(filepath.Join(root, "memory-bank-source.json")); os.IsNotExist(err) { + if err := os.WriteFile(filepath.Join(root, "memory-bank-source.json"), []byte(`{"schema_version":1,"payload_format":"legacy/v1","capabilities":["legacy/v1"]}`), 0o644); err != nil { + t.Fatal(err) + } + } runGitTest(t, root, "init", "--quiet") runGitTest(t, root, "add", "--all") runGitTest(t, root, "-c", "user.name=Memory Bank Tests", "-c", "user.email=tests@example.invalid", "commit", "--quiet", "-m", "source") diff --git a/internal/push/push_test.go b/internal/push/push_test.go index 0c30046..d709e6b 100644 --- a/internal/push/push_test.go +++ b/internal/push/push_test.go @@ -423,6 +423,9 @@ func TestDryRunIncludesCanonicalTemplatePathsOutsideMemoryBank(t *testing.T) { if err := os.WriteFile(filepath.Join(source, "template", ".config", "hidden"), []byte("base\n"), 0o644); err != nil { t.Fatal(err) } + if err := os.WriteFile(filepath.Join(source, "memory-bank-source.json"), []byte(`{"schema_version":1,"payload_format":"legacy/v1","capabilities":["legacy/v1"]}`), 0o644); err != nil { + t.Fatal(err) + } git(t, source, "init", "--quiet") git(t, source, "add", ".") git(t, source, "-c", "user.name=Test", "-c", "user.email=test@example.invalid", "commit", "--quiet", "-m", "template") diff --git a/scripts/e2e-init-update.sh b/scripts/e2e-init-update.sh index 9fddc07..72dead9 100755 --- a/scripts/e2e-init-update.sh +++ b/scripts/e2e-init-update.sh @@ -73,6 +73,7 @@ setup_case() { require git -C "$template_work" config user.name 'E2E Fixture' require git -C "$template_work" config user.email 'fixture@example.invalid' write_template_v1 + printf '%s\n' '{"schema_version":1,"payload_format":"legacy/v1","capabilities":["legacy/v1"]}' >"$template_work/memory-bank-source.json" if [ "$include_project_local" = 1 ]; then mkdir -p "$template_work/memory-bank/dna" printf 'project local v1\n' >"$template_work/memory-bank/dna/managed.md" diff --git a/scripts/e2e-source-format.sh b/scripts/e2e-source-format.sh new file mode 100755 index 0000000..61bc874 --- /dev/null +++ b/scripts/e2e-source-format.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# Real-binary source-format contract. No mocking of ownership or Git verification. +set -euo pipefail +: "${E2E_BINARY:?point E2E_BINARY at the built bridge}" +: "${LEGACY_SOURCE:?point LEGACY_SOURCE at the clean pinned legacy checkout}" +legacy_ref=f1f04de843aef45a2425d4a7351d577bbf89e940 +work_root="$(mktemp -d)" +trap 'rm -rf "$work_root"' EXIT +mkdir "$work_root/downstream" "$work_root/unknown" "$work_root/component" +"$E2E_BINARY" capabilities --require source-format/v1 --require legacy/v1 +if "$E2E_BINARY" capabilities --require components/v1 >"$work_root/capabilities.json"; then + echo 'bridge advertised unsupported components' >&2; exit 1 +fi +"$E2E_BINARY" init --repo-root "$work_root/downstream" --source "$LEGACY_SOURCE" --source-ref "$legacy_ref" --template-version legacy-f1f04de --json >"$work_root/init.json" +"$E2E_BINARY" pull --repo-root "$work_root/downstream" --source "$LEGACY_SOURCE" --source-ref "$legacy_ref" --template-version legacy-f1f04de --json >"$work_root/pull.json" +cp -R "$work_root/downstream" "$work_root/before" +for format in unknown component; do + source_root="$work_root/$format" + mkdir -p "$source_root/template/memory-bank" + printf 'incoming\n' >"$source_root/template/memory-bank/README.md" + if [ "$format" = component ]; then + printf '%s\n' '{"schema_version":1,"payload_format":"components/v1","capabilities":["components/v1"]}' >"$source_root/memory-bank-source.json" + printf '{}\n' >"$source_root/template/memory-bank/components.json" + fi + git -C "$source_root" init --quiet + git -C "$source_root" add . + git -C "$source_root" -c user.name=Fixture -c user.email=fixture@example.invalid commit --quiet -m "$format" + source_ref="$(git -C "$source_root" rev-parse HEAD)" + if "$E2E_BINARY" pull --repo-root "$work_root/downstream" --source "$source_root" --source-ref "$source_ref" --template-version fixture >"$work_root/$format.out" 2>"$work_root/$format.err"; then + echo "bridge accepted $format source" >&2; exit 1 + fi + diff -r "$work_root/before" "$work_root/downstream" + mkdir "$work_root/new-$format" + if "$E2E_BINARY" init --repo-root "$work_root/new-$format" --source "$source_root" --source-ref "$source_ref" --template-version fixture >"$work_root/$format-init.out" 2>"$work_root/$format-init.err"; then + echo "bridge initialized $format source" >&2; exit 1 + fi + test -z "$(ls -A "$work_root/new-$format")" +done +printf 'Real bridge source-format fixtures passed\n' From fc5d91c67a8cb8db4749b7995952659fe3b0b279 Mon Sep 17 00:00:00 2001 From: Danil Pismenny Date: Mon, 7 Sep 2026 01:09:32 +0300 Subject: [PATCH 02/13] fix: declare the downstream canary legacy source --- README.md | 3 +++ memory-bank-source.json | 1 + 2 files changed, 4 insertions(+) create mode 100644 memory-bank-source.json diff --git a/README.md b/README.md index a34326b..dc04b21 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,9 @@ memory-bank-cli capabilities --require legacy/v1 ``` The command emits JSON and returns nonzero for an unsupported capability. +This repository also declares its legacy `memory-bank/` payload because the scheduled +downstream canary uses it as a pinned template fixture. + Keep the previous CLI/source pair to continue using an undeclared custom source without changing it. See the [bridge contract](docs/source-format-bridge.md) for the supported-source boundary and wire format, and the diff --git a/memory-bank-source.json b/memory-bank-source.json new file mode 100644 index 0000000..32441fa --- /dev/null +++ b/memory-bank-source.json @@ -0,0 +1 @@ +{"schema_version":1,"payload_format":"legacy/v1","capabilities":["legacy/v1"]} From 43c388dd493d34fd035c632b89c95b70ca913773 Mon Sep 17 00:00:00 2001 From: Danil Pismenny Date: Mon, 7 Sep 2026 01:17:57 +0300 Subject: [PATCH 03/13] docs: remove the canary fixture dependency cycle --- .../features/FT-023-push-upstream-publication/design.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/memory-bank/features/FT-023-push-upstream-publication/design.md b/memory-bank/features/FT-023-push-upstream-publication/design.md index b3aba81..555cc43 100644 --- a/memory-bank/features/FT-023-push-upstream-publication/design.md +++ b/memory-bank/features/FT-023-push-upstream-publication/design.md @@ -5,7 +5,6 @@ doc_function: canonical purpose: "Feature-local solution for safe upstream publication: managed-only selection, compensating Git/GitHub transaction and evidence-backed CLI contract." derived_from: - brief.md - - decision-log.md - "https://github.com/dapi/memory-bank-cli/issues/29" status: active audience: humans_and_agents @@ -26,7 +25,7 @@ must_not_define: ## Context -`REQ-01`–`REQ-05` cross the current repository, a nested upstream checkout, an upstream Git remote and GitHub PR creation. The accepted FPF decisions in `decision-log.md` prefer a small auditable publish set and compensation rather than fictional distributed atomicity. +`REQ-01`–`REQ-05` cross the current repository, a nested upstream checkout, an upstream Git remote and GitHub PR creation. The accepted decisions below prefer a small auditable publish set and compensation rather than fictional distributed atomicity. The [decision log](decision-log.md) retains their provenance. ## C4 Applicability From 25dc8a20c09de45fa309a4eda23d8daef604dcf2 Mon Sep 17 00:00:00 2001 From: Danil Pismenny Date: Mon, 7 Sep 2026 01:23:21 +0300 Subject: [PATCH 04/13] docs: retain provenance with an acyclic evidence owner --- .../features/FT-023-push-upstream-publication/decision-log.md | 3 +-- .../features/FT-023-push-upstream-publication/design.md | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/memory-bank/features/FT-023-push-upstream-publication/decision-log.md b/memory-bank/features/FT-023-push-upstream-publication/decision-log.md index 3220e28..5c2ef5a 100644 --- a/memory-bank/features/FT-023-push-upstream-publication/decision-log.md +++ b/memory-bank/features/FT-023-push-upstream-publication/decision-log.md @@ -5,7 +5,6 @@ doc_function: derived purpose: "Audit ledger for FT-023 provenance and FPF reasoning. It links to canonical owners and does not define requirements, selected solution or implementation sequence." derived_from: - brief.md - - design.md - ../../flows/feature.md status: active audience: humans_and_agents @@ -19,7 +18,7 @@ must_not_define: ## Ownership -`brief.md` owns problem-space facts, validation decision and verify. `design.md` owns accepted feature-local solution facts. This ledger records the evidence and FPF reasoning only; if it conflicts with a canonical owner, update that owner first and then this log. +`brief.md` owns problem-space facts, validation decision and verify. `design.md` owns accepted feature-local solution facts. This ledger owns the historical evidence and FPF reasoning used by the design. Links to design.md identify where decisions were formalized; they are navigation to the downstream solution owner, not an upstream dependency. Historical evidence remains factual; current selected-solution claims belong to design.md. ## Decisions and Open Questions diff --git a/memory-bank/features/FT-023-push-upstream-publication/design.md b/memory-bank/features/FT-023-push-upstream-publication/design.md index 555cc43..416b13f 100644 --- a/memory-bank/features/FT-023-push-upstream-publication/design.md +++ b/memory-bank/features/FT-023-push-upstream-publication/design.md @@ -5,6 +5,7 @@ doc_function: canonical purpose: "Feature-local solution for safe upstream publication: managed-only selection, compensating Git/GitHub transaction and evidence-backed CLI contract." derived_from: - brief.md + - decision-log.md - "https://github.com/dapi/memory-bank-cli/issues/29" status: active audience: humans_and_agents From f09f53c00cf766a847f3d2578e59e6033157a94b Mon Sep 17 00:00:00 2001 From: Danil Pismenny Date: Mon, 7 Sep 2026 01:26:50 +0300 Subject: [PATCH 05/13] docs: scope the decision provenance reference --- memory-bank/features/FT-023-push-upstream-publication/design.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/memory-bank/features/FT-023-push-upstream-publication/design.md b/memory-bank/features/FT-023-push-upstream-publication/design.md index 416b13f..a72228a 100644 --- a/memory-bank/features/FT-023-push-upstream-publication/design.md +++ b/memory-bank/features/FT-023-push-upstream-publication/design.md @@ -26,7 +26,7 @@ must_not_define: ## Context -`REQ-01`–`REQ-05` cross the current repository, a nested upstream checkout, an upstream Git remote and GitHub PR creation. The accepted decisions below prefer a small auditable publish set and compensation rather than fictional distributed atomicity. The [decision log](decision-log.md) retains their provenance. +`REQ-01`–`REQ-05` cross the current repository, a nested upstream checkout, an upstream Git remote and GitHub PR creation. The accepted decisions below prefer a small auditable publish set and compensation rather than fictional distributed atomicity. The [decision log](decision-log.md) retains the provenance of SD-01, SD-02 and the validation decision SD-03. ## C4 Applicability From a91e61b40b856afccd7809b673c5f03830197e72 Mon Sep 17 00:00:00 2001 From: Danil Pismenny Date: Mon, 7 Sep 2026 01:37:25 +0300 Subject: [PATCH 06/13] fix: pin the canary to the canonical legacy template --- .github/workflows/downstream-canary.yml | 7 ++++--- README.md | 3 --- memory-bank-source.json | 1 - .../decision-log.md | 3 ++- .../FT-023-push-upstream-publication/design.md | 2 +- scripts/downstream-smoke.sh | 14 ++++++++------ 6 files changed, 15 insertions(+), 15 deletions(-) delete mode 100644 memory-bank-source.json diff --git a/.github/workflows/downstream-canary.yml b/.github/workflows/downstream-canary.yml index 4f03149..ae84935 100644 --- a/.github/workflows/downstream-canary.yml +++ b/.github/workflows/downstream-canary.yml @@ -11,9 +11,9 @@ on: default: main type: string template_ref: - description: Template Git ref to test + description: Git ref in dapi/memory-bank to test required: false - default: main + default: f1f04de843aef45a2425d4a7351d577bbf89e940 type: string permissions: @@ -33,7 +33,8 @@ jobs: - name: Run compatibility fixture env: CLI_REF: ${{ inputs.cli_ref || 'main' }} - TEMPLATE_REF: ${{ inputs.template_ref || 'main' }} + TEMPLATE_REPOSITORY_URL: https://github.com/dapi/memory-bank.git + TEMPLATE_REF: ${{ inputs.template_ref || 'f1f04de843aef45a2425d4a7351d577bbf89e940' }} REPORT_DIR: ${{ runner.temp }}/downstream-canary run: bash scripts/downstream-smoke.sh - name: Upload canary evidence diff --git a/README.md b/README.md index dc04b21..a34326b 100644 --- a/README.md +++ b/README.md @@ -31,9 +31,6 @@ memory-bank-cli capabilities --require legacy/v1 ``` The command emits JSON and returns nonzero for an unsupported capability. -This repository also declares its legacy `memory-bank/` payload because the scheduled -downstream canary uses it as a pinned template fixture. - Keep the previous CLI/source pair to continue using an undeclared custom source without changing it. See the [bridge contract](docs/source-format-bridge.md) for the supported-source boundary and wire format, and the diff --git a/memory-bank-source.json b/memory-bank-source.json deleted file mode 100644 index 32441fa..0000000 --- a/memory-bank-source.json +++ /dev/null @@ -1 +0,0 @@ -{"schema_version":1,"payload_format":"legacy/v1","capabilities":["legacy/v1"]} diff --git a/memory-bank/features/FT-023-push-upstream-publication/decision-log.md b/memory-bank/features/FT-023-push-upstream-publication/decision-log.md index 5c2ef5a..3220e28 100644 --- a/memory-bank/features/FT-023-push-upstream-publication/decision-log.md +++ b/memory-bank/features/FT-023-push-upstream-publication/decision-log.md @@ -5,6 +5,7 @@ doc_function: derived purpose: "Audit ledger for FT-023 provenance and FPF reasoning. It links to canonical owners and does not define requirements, selected solution or implementation sequence." derived_from: - brief.md + - design.md - ../../flows/feature.md status: active audience: humans_and_agents @@ -18,7 +19,7 @@ must_not_define: ## Ownership -`brief.md` owns problem-space facts, validation decision and verify. `design.md` owns accepted feature-local solution facts. This ledger owns the historical evidence and FPF reasoning used by the design. Links to design.md identify where decisions were formalized; they are navigation to the downstream solution owner, not an upstream dependency. Historical evidence remains factual; current selected-solution claims belong to design.md. +`brief.md` owns problem-space facts, validation decision and verify. `design.md` owns accepted feature-local solution facts. This ledger records the evidence and FPF reasoning only; if it conflicts with a canonical owner, update that owner first and then this log. ## Decisions and Open Questions diff --git a/memory-bank/features/FT-023-push-upstream-publication/design.md b/memory-bank/features/FT-023-push-upstream-publication/design.md index a72228a..b3aba81 100644 --- a/memory-bank/features/FT-023-push-upstream-publication/design.md +++ b/memory-bank/features/FT-023-push-upstream-publication/design.md @@ -26,7 +26,7 @@ must_not_define: ## Context -`REQ-01`–`REQ-05` cross the current repository, a nested upstream checkout, an upstream Git remote and GitHub PR creation. The accepted decisions below prefer a small auditable publish set and compensation rather than fictional distributed atomicity. The [decision log](decision-log.md) retains the provenance of SD-01, SD-02 and the validation decision SD-03. +`REQ-01`–`REQ-05` cross the current repository, a nested upstream checkout, an upstream Git remote and GitHub PR creation. The accepted FPF decisions in `decision-log.md` prefer a small auditable publish set and compensation rather than fictional distributed atomicity. ## C4 Applicability diff --git a/scripts/downstream-smoke.sh b/scripts/downstream-smoke.sh index 03af649..c752194 100755 --- a/scripts/downstream-smoke.sh +++ b/scripts/downstream-smoke.sh @@ -2,6 +2,7 @@ set -euo pipefail repository_url="${REPOSITORY_URL:-https://github.com/dapi/memory-bank-cli.git}" +template_repository_url="${TEMPLATE_REPOSITORY_URL:-$repository_url}" cli_ref="${CLI_REF:?CLI_REF is required}" template_ref="${TEMPLATE_REF:?TEMPLATE_REF is required}" release_tag="${RELEASE_TAG:-}" @@ -89,19 +90,20 @@ git --version >/dev/null resolve_ref() { local ref="$1" + local source_url="$2" local sha if [[ "$ref" =~ ^[0-9a-f]{40}$ ]]; then local resolver="$workspace/ref-resolution" if [ ! -d "$resolver/.git" ]; then git init --quiet "$resolver" fi - git -C "$resolver" fetch --quiet --depth=1 "$repository_url" "$ref" + git -C "$resolver" fetch --quiet --depth=1 "$source_url" "$ref" sha="$(git -C "$resolver" rev-parse --verify 'FETCH_HEAD^{commit}')" test "$sha" = "$ref" || return 1 else - sha="$(git ls-remote "$repository_url" "${ref}^{}" | awk 'NR == 1 { print $1 }')" + sha="$(git ls-remote "$source_url" "${ref}^{}" | awk 'NR == 1 { print $1 }')" if [ -z "$sha" ]; then - sha="$(git ls-remote "$repository_url" "$ref" | awk 'NR == 1 { print $1 }')" + sha="$(git ls-remote "$source_url" "$ref" | awk 'NR == 1 { print $1 }')" fi fi test -n "$sha" || return 1 @@ -109,7 +111,7 @@ resolve_ref() { } step="resolve-cli-ref" -cli_sha="$(resolve_ref "$cli_ref")" +cli_sha="$(resolve_ref "$cli_ref" "$repository_url")" cli_install_ref="$cli_sha" if [ -n "$release_tag" ] && [ "$cli_ref" = "$release_tag" ]; then # Stable releases are installed by tag so the smoke test covers the released @@ -117,7 +119,7 @@ if [ -n "$release_tag" ] && [ "$cli_ref" = "$release_tag" ]; then cli_install_ref="$cli_ref" fi step="resolve-template-ref" -template_sha="$(resolve_ref "$template_ref")" +template_sha="$(resolve_ref "$template_ref" "$template_repository_url")" source_root="$workspace/template" downstream_root="$workspace/downstream" @@ -125,7 +127,7 @@ bin_root="$workspace/bin" phase="template" step="clone-template" -git clone --quiet "$repository_url" "$source_root" +git clone --quiet "$template_repository_url" "$source_root" step="checkout-template" git -C "$source_root" checkout --quiet --detach "$template_sha" step="verify-template-clean" From cd320d7938ea0144a545447629635bdc195adf94 Mon Sep 17 00:00:00 2001 From: Danil Pismenny Date: Mon, 7 Sep 2026 01:48:42 +0300 Subject: [PATCH 07/13] fix: drain CLI help when detecting the sync command --- scripts/downstream-smoke.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/downstream-smoke.sh b/scripts/downstream-smoke.sh index c752194..753842f 100755 --- a/scripts/downstream-smoke.sh +++ b/scripts/downstream-smoke.sh @@ -144,7 +144,9 @@ test -x "$cli" # installed command contract rather than pinning one legacy tag, so every # pre-`pull` release continues to use `update` and later releases use `pull`. step="detect-sync-command" -if "$cli" --help | grep -Eq '^[[:space:]]+pull[[:space:]]'; then +# Consume the full help stream: grep -q may close the pipe early and make the +# Go producer exit with SIGPIPE under pipefail, selecting the wrong command. +if "$cli" --help | grep -E '^[[:space:]]+pull[[:space:]]' >/dev/null; then sync_command="pull" else sync_command="update" From 8f91a5c514189a58a6a3597e947466bfbfc4ac73 Mon Sep 17 00:00:00 2001 From: Danil Pismenny Date: Mon, 7 Sep 2026 02:13:07 +0300 Subject: [PATCH 08/13] fix: persist adapted ownership and pin original Git objects --- docs/source-format-bridge.md | 7 +++- internal/ownership/source.go | 2 +- internal/ownership/source_format_test.go | 47 ++++++++++++++++++++++++ internal/ownership/update.go | 2 +- internal/ownership/update_test.go | 2 +- 5 files changed, 56 insertions(+), 4 deletions(-) diff --git a/docs/source-format-bridge.md b/docs/source-format-bridge.md index 334b8d1..f70a7ae 100644 --- a/docs/source-format-bridge.md +++ b/docs/source-format-bridge.md @@ -65,7 +65,12 @@ payload-root validation, using Git objects rather than mutable working-tree file The only alternatives are accepting every undeclared tree (violates BR-01) or refusing all legacy trees (violates compatibility). Use the compiled supported SHA plus strict declarations. -No state schema, transaction engine, target paths or template payload bytes change in W1. +No new state schema, transaction writer or target paths are introduced in W1. +The canonical-source canary exposed one existing bookkeeping bug: a preserve decision can +change ownership without a file mutation, so run must persist changed Files entries, not +only a changed file count. A regression fixture verifies persistence and the next no-op pull. +Git plumbing disables replacement objects so local replace refs cannot change a pinned +source commit's contents. A real Git fixture proved the previous substitution and now rejects it. Test helpers in `internal/ownership/source_test.go#commitTestSource`, `internal/cli/cli_test.go#commitCLISource` and `scripts/e2e-init-update.sh#setup_case` declare diff --git a/internal/ownership/source.go b/internal/ownership/source.go index 634de83..167a996 100644 --- a/internal/ownership/source.go +++ b/internal/ownership/source.go @@ -213,7 +213,7 @@ func gitOutput(root string, arguments ...string) (string, error) { func gitBytes(root string, arguments ...string) ([]byte, error) { commandArguments := append([]string{"-C", root}, arguments...) command := exec.Command("git", commandArguments...) - command.Env = append(os.Environ(), "GIT_OPTIONAL_LOCKS=0") + command.Env = append(os.Environ(), "GIT_OPTIONAL_LOCKS=0", "GIT_NO_REPLACE_OBJECTS=1") output, err := command.CombinedOutput() if err != nil { result := strings.TrimSpace(string(output)) diff --git a/internal/ownership/source_format_test.go b/internal/ownership/source_format_test.go index d10ff14..046fbc1 100644 --- a/internal/ownership/source_format_test.go +++ b/internal/ownership/source_format_test.go @@ -136,3 +136,50 @@ func TestComponentMarkerRejectedInLegacyPayloadRoots(t *testing.T) { }) } } + +func TestPinnedSourceIgnoresGitReplacementObjects(t *testing.T) { + source, repo := t.TempDir(), t.TempDir() + write(t, source, "template/memory-bank/README.md", "original pinned content\n") + write(t, source, SourceDeclarationFile, legacyDeclaration) + original := rawSourceCommit(t, source) + write(t, source, "template/memory-bank/README.md", "replacement content\n") + replacement := rawSourceCommit(t, source) + runGitTest(t, source, "replace", original, replacement) + runGitTest(t, source, "checkout", "--quiet", "--detach", original) + before := treeSnapshot(t, repo) + _, err := Init(Options{RepoRoot: repo, SourceRoot: source, TemplateVersion: "pinned", SourceRef: original}) + if err == nil { + t.Fatal("installed replacement objects under an unchanged source commit identity") + } + if !reflect.DeepEqual(before, treeSnapshot(t, repo)) { + t.Fatal("replaced source changed downstream") + } +} + +func TestUnchangedPullPersistsManagedAdaptation(t *testing.T) { + source, repo := t.TempDir(), t.TempDir() + target := "memory-bank/domain/model.md" + write(t, source, "template/"+target, "template model\n") + ref := commitTestSource(t, source) + options := Options{RepoRoot: repo, SourceRoot: source, TemplateVersion: "fixture", SourceRef: ref} + if _, err := Init(options); err != nil { + t.Fatal(err) + } + write(t, repo, target, "template model\nproject adaptation\n") + report, err := Update(options) + if err != nil { + t.Fatal(err) + } + lock, _, err := ReadLock(repo) + if err != nil { + t.Fatal(err) + } + if !report.Applied || lock.Files[target].Ownership != Adapted { + t.Fatalf("ownership change was not persisted: report=%#v file=%#v", report, lock.Files[target]) + } + before := treeSnapshot(t, repo) + report, err = Update(options) + if err != nil || report.Applied || !reflect.DeepEqual(before, treeSnapshot(t, repo)) { + t.Fatalf("repeat pull is not a no-op: report=%#v err=%v", report, err) + } +} diff --git a/internal/ownership/update.go b/internal/ownership/update.go index 72e7431..69cd017 100644 --- a/internal/ownership/update.go +++ b/internal/ownership/update.go @@ -157,7 +157,7 @@ func run(options Options, old Lock, hasLock bool, repo pinnedRepo, lockDigest st return report, nil } template := Template{Version: options.TemplateVersion, SourceRef: options.SourceRef} - needsLockWrite := !hasLock || templateMutationCount > 0 || old.SchemaVersion != CurrentSchemaVersion || old.Template != template || len(old.Files) != len(next.Files) + needsLockWrite := !hasLock || templateMutationCount > 0 || old.SchemaVersion != CurrentSchemaVersion || old.Template != template || !reflect.DeepEqual(old.Files, next.Files) if !needsLockWrite { if len(mutations) == 0 { return report, nil diff --git a/internal/ownership/update_test.go b/internal/ownership/update_test.go index 3e48024..ff7e590 100644 --- a/internal/ownership/update_test.go +++ b/internal/ownership/update_test.go @@ -364,7 +364,7 @@ func TestManagedContentDriftIsPreservedWhenTemplateIsUnchanged(t *testing.T) { for index := 0; index < 2; index++ { report, err := Update(opts(repo, source, "a")) - if err != nil || report.ConflictCount != 0 || decisionFor(t, report, path).Reason != "preserve local managed content while template is unchanged" { + if err != nil || report.ConflictCount != 0 || decisionFor(t, report, path).Action != Preserve || report.Applied != (index == 0) { t.Fatalf("drift run %d: report=%#v err=%v", index, report, err) } } From b3a8e3b35091c5f426f4ae97fc6996aa1f6c5f71 Mon Sep 17 00:00:00 2001 From: Danil Pismenny Date: Mon, 7 Sep 2026 02:22:27 +0300 Subject: [PATCH 09/13] refactor: avoid repeating source JSON validation --- internal/ownership/source_format.go | 4 +++- internal/ownership/source_format_test.go | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/internal/ownership/source_format.go b/internal/ownership/source_format.go index f38b8e0..33d7748 100644 --- a/internal/ownership/source_format.go +++ b/internal/ownership/source_format.go @@ -60,8 +60,10 @@ func verifySourceFormat(root, ref, payloadRoot string) error { return fmt.Errorf("invalid %s: unknown field %q", SourceDeclarationFile, key) } } + // The first pass validates exact key spelling (encoding/json accepts case + // aliases), duplicate keys and trailing bytes. Only typed decoding remains. var declaration sourceDeclaration - if err := decodeSourceJSON(data, &declaration); err != nil { + if err := json.Unmarshal(data, &declaration); err != nil { return fmt.Errorf("invalid %s: %w", SourceDeclarationFile, err) } if declaration.SchemaVersion != 1 || declaration.PayloadFormat != "legacy/v1" { diff --git a/internal/ownership/source_format_test.go b/internal/ownership/source_format_test.go index 046fbc1..7a29b33 100644 --- a/internal/ownership/source_format_test.go +++ b/internal/ownership/source_format_test.go @@ -53,6 +53,8 @@ func TestSourceFormatRejectionIsNonMutating(t *testing.T) { {"unknown schema", `{"schema_version":2,"payload_format":"legacy/v1","capabilities":["legacy/v1"]}`, "", "unsupported source format"}, {"component format", `{"schema_version":1,"payload_format":"components/v1","capabilities":["components/v1"]}`, "", "unsupported source format"}, {"unknown field", `{"schema_version":1,"payload_format":"legacy/v1","capabilities":["legacy/v1"],"extra":true}`, "", "unknown field"}, + {"case-aliased field", `{"Schema_Version":1,"payload_format":"legacy/v1","capabilities":["legacy/v1"]}`, "", "unknown field"}, + {"wrong field type", `{"schema_version":"1","payload_format":"legacy/v1","capabilities":["legacy/v1"]}`, "", "cannot unmarshal"}, {"duplicate field", `{"schema_version":1,"schema_version":1,"payload_format":"legacy/v1","capabilities":["legacy/v1"]}`, "", "duplicate JSON field"}, {"trailing JSON", legacyDeclaration + `{}`, "", "trailing JSON"}, {"missing capability", `{"schema_version":1,"payload_format":"legacy/v1"}`, "", "requires capability"}, From aae9a0f53b17ebc14920393aa03ea430eafdac8e Mon Sep 17 00:00:00 2001 From: Danil Pismenny Date: Mon, 7 Sep 2026 02:31:32 +0300 Subject: [PATCH 10/13] refactor: scope source envelope parsing to its schema --- internal/ownership/source_format.go | 82 ++++++++++++----------------- 1 file changed, 33 insertions(+), 49 deletions(-) diff --git a/internal/ownership/source_format.go b/internal/ownership/source_format.go index 33d7748..e6c944c 100644 --- a/internal/ownership/source_format.go +++ b/internal/ownership/source_format.go @@ -51,8 +51,8 @@ func verifySourceFormat(root, ref, payloadRoot string) error { } return fmt.Errorf("unsupported manifestless source %s: use a published supported legacy commit or a declared source", ref) } - var fields map[string]json.RawMessage - if err := decodeSourceJSON(data, &fields); err != nil { + fields, err := sourceFields(data) + if err != nil { return fmt.Errorf("invalid %s: %w", SourceDeclarationFile, err) } for key := range fields { @@ -108,59 +108,43 @@ func readSourceDeclaration(root, ref string) ([]byte, bool, error) { return data, true, err } -// Reject duplicate keys before decoding: encoding/json otherwise accepts the last -// value, which makes the declared source format ambiguous across implementations. -func decodeSourceJSON(data []byte, target any) error { +// The envelope is shallow: values are scalars or a string array. Typed decoding +// rejects nested objects, so only top-level keys need duplicate detection here. +func sourceFields(data []byte) (map[string]json.RawMessage, error) { decoder := json.NewDecoder(bytes.NewReader(data)) - var value func() error - value = func() error { - token, err := decoder.Token() - if err != nil { - return err - } - delim, container := token.(json.Delim) - if !container { - return nil + token, err := decoder.Token() + if err != nil { + return nil, err + } + fields := map[string]json.RawMessage{} + if token != nil { + if token != json.Delim('{') { + return nil, errors.New("source declaration must be an object") } - switch delim { - case '{': - keys := map[string]bool{} - for decoder.More() { - keyToken, err := decoder.Token() - if err != nil { - return err - } - key, ok := keyToken.(string) - if !ok { - return errors.New("invalid JSON object key") - } - if keys[key] { - return fmt.Errorf("duplicate JSON field %q", key) - } - keys[key] = true - if err := value(); err != nil { - return err - } + for decoder.More() { + keyToken, err := decoder.Token() + if err != nil { + return nil, err + } + key, ok := keyToken.(string) + if !ok { + return nil, errors.New("invalid JSON object key") } - case '[': - for decoder.More() { - if err := value(); err != nil { - return err - } + if _, exists := fields[key]; exists { + return nil, fmt.Errorf("duplicate JSON field %q", key) } - default: - return errors.New("unexpected JSON delimiter") + var value json.RawMessage + if err := decoder.Decode(&value); err != nil { + return nil, err + } + fields[key] = value + } + if _, err := decoder.Token(); err != nil { + return nil, err } - _, err = decoder.Token() - return err - } - if err := value(); err != nil { - return err } if _, err := decoder.Token(); !errors.Is(err, io.EOF) { - return errors.New("trailing JSON data") + return nil, errors.New("trailing JSON data") } - decoder = json.NewDecoder(bytes.NewReader(data)) - decoder.DisallowUnknownFields() - return decoder.Decode(target) + return fields, nil } From d5c563788fd170c611c6b59bc6e17cee068b6007 Mon Sep 17 00:00:00 2001 From: Danil Pismenny Date: Mon, 7 Sep 2026 02:38:03 +0300 Subject: [PATCH 11/13] refactor: decode source declarations in one pass --- internal/ownership/source_format.go | 58 ++++++++++++++--------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/internal/ownership/source_format.go b/internal/ownership/source_format.go index e6c944c..44effa3 100644 --- a/internal/ownership/source_format.go +++ b/internal/ownership/source_format.go @@ -51,21 +51,10 @@ func verifySourceFormat(root, ref, payloadRoot string) error { } return fmt.Errorf("unsupported manifestless source %s: use a published supported legacy commit or a declared source", ref) } - fields, err := sourceFields(data) + declaration, err := decodeSourceDeclaration(data) if err != nil { return fmt.Errorf("invalid %s: %w", SourceDeclarationFile, err) } - for key := range fields { - if key != "schema_version" && key != "payload_format" && key != "capabilities" { - return fmt.Errorf("invalid %s: unknown field %q", SourceDeclarationFile, key) - } - } - // The first pass validates exact key spelling (encoding/json accepts case - // aliases), duplicate keys and trailing bytes. Only typed decoding remains. - var declaration sourceDeclaration - if err := json.Unmarshal(data, &declaration); err != nil { - return fmt.Errorf("invalid %s: %w", SourceDeclarationFile, err) - } if declaration.SchemaVersion != 1 || declaration.PayloadFormat != "legacy/v1" { return fmt.Errorf("unsupported source format: schema=%d payload_format=%q", declaration.SchemaVersion, declaration.PayloadFormat) } @@ -108,43 +97,54 @@ func readSourceDeclaration(root, ref string) ([]byte, bool, error) { return data, true, err } -// The envelope is shallow: values are scalars or a string array. Typed decoding -// rejects nested objects, so only top-level keys need duplicate detection here. -func sourceFields(data []byte) (map[string]json.RawMessage, error) { +// A single shallow pass enforces exact key spelling, duplicate detection and +// field types; encoding/json's struct decoder alone accepts case aliases. +func decodeSourceDeclaration(data []byte) (sourceDeclaration, error) { + var declaration sourceDeclaration decoder := json.NewDecoder(bytes.NewReader(data)) token, err := decoder.Token() if err != nil { - return nil, err + return declaration, err } - fields := map[string]json.RawMessage{} + seen := map[string]bool{} if token != nil { if token != json.Delim('{') { - return nil, errors.New("source declaration must be an object") + return declaration, errors.New("source declaration must be an object") } for decoder.More() { keyToken, err := decoder.Token() if err != nil { - return nil, err + return declaration, err } key, ok := keyToken.(string) if !ok { - return nil, errors.New("invalid JSON object key") + return declaration, errors.New("invalid JSON object key") + } + if seen[key] { + return declaration, fmt.Errorf("duplicate JSON field %q", key) } - if _, exists := fields[key]; exists { - return nil, fmt.Errorf("duplicate JSON field %q", key) + seen[key] = true + var target any + switch key { + case "schema_version": + target = &declaration.SchemaVersion + case "payload_format": + target = &declaration.PayloadFormat + case "capabilities": + target = &declaration.Capabilities + default: + return declaration, fmt.Errorf("unknown field %q", key) } - var value json.RawMessage - if err := decoder.Decode(&value); err != nil { - return nil, err + if err := decoder.Decode(target); err != nil { + return declaration, err } - fields[key] = value } if _, err := decoder.Token(); err != nil { - return nil, err + return declaration, err } } if _, err := decoder.Token(); !errors.Is(err, io.EOF) { - return nil, errors.New("trailing JSON data") + return declaration, errors.New("trailing JSON data") } - return fields, nil + return declaration, nil } From 3b434fd93678c36447d10d4f308a39ce5d74b040 Mon Sep 17 00:00:00 2001 From: Danil Pismenny Date: Mon, 7 Sep 2026 02:47:13 +0300 Subject: [PATCH 12/13] fix: keep legacy declaration capabilities fixed --- docs/source-format-bridge.md | 4 +++- internal/ownership/source_format.go | 21 +++------------------ internal/ownership/source_format_test.go | 7 ++++--- 3 files changed, 10 insertions(+), 22 deletions(-) diff --git a/docs/source-format-bridge.md b/docs/source-format-bridge.md index f70a7ae..0dcf54e 100644 --- a/docs/source-format-bridge.md +++ b/docs/source-format-bridge.md @@ -11,7 +11,9 @@ remain later units governed by the shared component design. `f1f04de843aef45a2425d4a7351d577bbf89e940`, and rejects an unknown manifestless source. - BR-02: A declared source has `memory-bank-source.json` at its checkout root, schema 1, `payload_format: legacy/v1`, and `capabilities: [legacy/v1]`. The file must be a tracked - regular Git blob at the selected commit. Unknown/duplicate fields, trailing JSON, + regular Git blob at the selected commit. The declaration array is exactly [legacy/v1]; + source-format/v1 belongs to the CLI handshake, not this legacy declaration schema. + Unknown/duplicate fields, trailing JSON, unsupported schema/format/capabilities, missing mandatory capabilities and a component manifest inside a declared legacy source are errors. The component marker is `/memory-bank/components.json` for template/, or `/components.json` diff --git a/internal/ownership/source_format.go b/internal/ownership/source_format.go index 44effa3..df0d77f 100644 --- a/internal/ownership/source_format.go +++ b/internal/ownership/source_format.go @@ -58,24 +58,9 @@ func verifySourceFormat(root, ref, payloadRoot string) error { if declaration.SchemaVersion != 1 || declaration.PayloadFormat != "legacy/v1" { return fmt.Errorf("unsupported source format: schema=%d payload_format=%q", declaration.SchemaVersion, declaration.PayloadFormat) } - seen := map[string]bool{} - for _, capability := range declaration.Capabilities { - if seen[capability] { - return fmt.Errorf("duplicate source capability %q", capability) - } - seen[capability] = true - supported := false - for _, available := range SupportedCapabilities() { - if capability == available { - supported = true - } - } - if !supported { - return fmt.Errorf("unsupported source capability %q", capability) - } - } - if !seen["legacy/v1"] { - return errors.New("source declaration requires capability legacy/v1") + // Source schema capabilities are fixed independently of the CLI handshake. + if len(declaration.Capabilities) != 1 || declaration.Capabilities[0] != "legacy/v1" { + return errors.New("source declaration requires exactly capability legacy/v1") } return nil } diff --git a/internal/ownership/source_format_test.go b/internal/ownership/source_format_test.go index 7a29b33..87ad690 100644 --- a/internal/ownership/source_format_test.go +++ b/internal/ownership/source_format_test.go @@ -57,9 +57,10 @@ func TestSourceFormatRejectionIsNonMutating(t *testing.T) { {"wrong field type", `{"schema_version":"1","payload_format":"legacy/v1","capabilities":["legacy/v1"]}`, "", "cannot unmarshal"}, {"duplicate field", `{"schema_version":1,"schema_version":1,"payload_format":"legacy/v1","capabilities":["legacy/v1"]}`, "", "duplicate JSON field"}, {"trailing JSON", legacyDeclaration + `{}`, "", "trailing JSON"}, - {"missing capability", `{"schema_version":1,"payload_format":"legacy/v1"}`, "", "requires capability"}, - {"unsupported capability", `{"schema_version":1,"payload_format":"legacy/v1","capabilities":["legacy/v1","components/v1"]}`, "", "unsupported source capability"}, - {"duplicate capability", `{"schema_version":1,"payload_format":"legacy/v1","capabilities":["legacy/v1","legacy/v1"]}`, "", "duplicate source capability"}, + {"missing capability", `{"schema_version":1,"payload_format":"legacy/v1"}`, "", "requires exactly capability"}, + {"unsupported capability", `{"schema_version":1,"payload_format":"legacy/v1","capabilities":["legacy/v1","components/v1"]}`, "", "requires exactly capability"}, + {"CLI-only capability", `{"schema_version":1,"payload_format":"legacy/v1","capabilities":["legacy/v1","source-format/v1"]}`, "", "requires exactly capability"}, + {"duplicate capability", `{"schema_version":1,"payload_format":"legacy/v1","capabilities":["legacy/v1","legacy/v1"]}`, "", "requires exactly capability"}, {"legacy with component marker", legacyDeclaration, "template/memory-bank/components.json", "unsupported component source"}, {"manifestless with marker", "", "template/memory-bank/components.json", "unsupported component source"}, {"null", `null`, "", "unsupported source format"}, From eedfdc514929934b863a6740e3cd11b8ecd9d2c6 Mon Sep 17 00:00:00 2001 From: Danil Pismenny Date: Mon, 7 Sep 2026 11:55:36 +0300 Subject: [PATCH 13/13] feat: install document components and adopt versioned flows (#64) * docs: define component runtime execution plan * refactor: avoid repeating source JSON validation * docs: define durable component recovery execution * feat: add component contract validation primitives * fix: enforce frozen validation and projection boundaries * fix: distinguish historical replay from transition authorization * feat: compose component installations and explicit document transactions * feat: complete component migration and explicit adoption runtime * fix: reject unsupported URI schemes before relocation * ci: fetch legacy ancestry for component migration fixtures * fix: clarify template ownership with compatible renderer v3 * fix: preserve draft links and recognize source projections * refactor: remove duplicate evidence sorting --- .github/workflows/local-e2e.yml | 22 + CHANGELOG.md | 10 + docs/component-delivery.md | 25 +- docs/component-runtime-plan.md | 169 +++++++ go.mod | 1 + go.sum | 2 + internal/agentinstructions/block.go | 17 +- internal/cli/capabilities_test.go | 9 +- internal/cli/cli.go | 43 +- internal/cli/component_projection_test.go | 39 ++ internal/cli/documents.go | 70 +++ internal/contracts/common.go | 252 ++++++++++ internal/contracts/document.go | 315 ++++++++++++ internal/contracts/draft.go | 13 + internal/contracts/draft_test.go | 51 ++ internal/contracts/engine.go | 219 ++++++++ internal/contracts/engine_test.go | 167 +++++++ internal/contracts/engines/governance-v1.json | 1 + internal/contracts/legacy_parser.go | 100 ++++ internal/contracts/manifest.go | 371 ++++++++++++++ internal/contracts/manifest_test.go | 147 ++++++ internal/contracts/priming.go | 136 +++++ internal/contracts/priming_test.go | 21 + internal/contracts/registry.go | 301 +++++++++++ internal/contracts/registry_test.go | 163 ++++++ internal/contracts/relocate.go | 321 ++++++++++++ internal/contracts/relocate_test.go | 43 ++ internal/contracts/renderer.go | 47 ++ internal/contracts/renderer_test.go | 67 +++ internal/contracts/rules.go | 260 ++++++++++ internal/contracts/source_integration_test.go | 75 +++ internal/doctor/doctor.go | 36 +- internal/ownership/component_file_unix.go | 62 +++ internal/ownership/component_file_windows.go | 24 + .../ownership/component_integrity_test.go | 175 +++++++ internal/ownership/component_migration.go | 470 ++++++++++++++++++ .../ownership/component_migration_test.go | 198 ++++++++ internal/ownership/component_recovery.go | 440 ++++++++++++++++ internal/ownership/component_recovery_test.go | 112 +++++ internal/ownership/component_resolution.go | 150 ++++++ internal/ownership/component_state.go | 397 +++++++++++++++ internal/ownership/components.go | 376 ++++++++++++++ internal/ownership/components_test.go | 314 ++++++++++++ internal/ownership/documents.go | 440 ++++++++++++++++ internal/ownership/lock.go | 19 +- internal/ownership/resolution_plan.go | 9 + internal/ownership/source_format.go | 27 +- internal/ownership/types.go | 61 ++- internal/ownership/update.go | 93 +++- internal/projection/projection.go | 8 + internal/projection/projection_test.go | 17 + scripts/e2e-components.py | 77 +++ scripts/e2e-source-format.sh | 4 +- 53 files changed, 6931 insertions(+), 55 deletions(-) create mode 100644 docs/component-runtime-plan.md create mode 100644 internal/cli/component_projection_test.go create mode 100644 internal/cli/documents.go create mode 100644 internal/contracts/common.go create mode 100644 internal/contracts/document.go create mode 100644 internal/contracts/draft.go create mode 100644 internal/contracts/draft_test.go create mode 100644 internal/contracts/engine.go create mode 100644 internal/contracts/engine_test.go create mode 100644 internal/contracts/engines/governance-v1.json create mode 100644 internal/contracts/legacy_parser.go create mode 100644 internal/contracts/manifest.go create mode 100644 internal/contracts/manifest_test.go create mode 100644 internal/contracts/priming.go create mode 100644 internal/contracts/priming_test.go create mode 100644 internal/contracts/registry.go create mode 100644 internal/contracts/registry_test.go create mode 100644 internal/contracts/relocate.go create mode 100644 internal/contracts/relocate_test.go create mode 100644 internal/contracts/renderer.go create mode 100644 internal/contracts/renderer_test.go create mode 100644 internal/contracts/rules.go create mode 100644 internal/contracts/source_integration_test.go create mode 100644 internal/ownership/component_file_unix.go create mode 100644 internal/ownership/component_file_windows.go create mode 100644 internal/ownership/component_integrity_test.go create mode 100644 internal/ownership/component_migration.go create mode 100644 internal/ownership/component_migration_test.go create mode 100644 internal/ownership/component_recovery.go create mode 100644 internal/ownership/component_recovery_test.go create mode 100644 internal/ownership/component_resolution.go create mode 100644 internal/ownership/component_state.go create mode 100644 internal/ownership/components.go create mode 100644 internal/ownership/components_test.go create mode 100644 internal/ownership/documents.go create mode 100644 scripts/e2e-components.py diff --git a/.github/workflows/local-e2e.yml b/.github/workflows/local-e2e.yml index ebbb682..e01cf2d 100644 --- a/.github/workflows/local-e2e.yml +++ b/.github/workflows/local-e2e.yml @@ -37,3 +37,25 @@ jobs: E2E_BINARY: ${{ runner.temp }}/memory-bank-cli LEGACY_SOURCE: ${{ github.workspace }}/.fixtures/legacy-source run: bash scripts/e2e-source-format.sh + + - name: Check out reviewed component producer + uses: actions/checkout@v7 + with: + repository: dapi/memory-bank + ref: f695db6a703e5409f10c9988e6460b41068fe30c + fetch-depth: 0 + path: .fixtures/component-source + persist-credentials: false + - name: Verify component contracts and transaction fixtures + env: + MEMORY_BANK_COMPONENT_SOURCE: ${{ github.workspace }}/.fixtures/component-source + MEMORY_BANK_LEGACY_SOURCE: ${{ github.workspace }}/.fixtures/legacy-source + run: | + go test ./... + go vet ./... + - name: Verify actual-binary component acceptance + env: + E2E_BINARY: ${{ runner.temp }}/memory-bank-cli + MEMORY_BANK_COMPONENT_SOURCE: ${{ github.workspace }}/.fixtures/component-source + MEMORY_BANK_LEGACY_SOURCE: ${{ github.workspace }}/.fixtures/legacy-source + run: python3 scripts/e2e-components.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 50bae56..ae659a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,16 @@ and this project follows [Semantic Versioning](https://semver.org/spec/v2.0.0.ht ## [Unreleased] +- Install schema-2 component presets (`core`, `docs`, `full`, `legacy`) and additive + adapters on Linux/macOS, with composition-aware README and AGENTS blocks. +- Create base documents independently from explicit flow adoption; bind adopted + documents to immutable contract bundles and support evidence-backed transitions + and identity-preserving moves. +- Migrate the pinned legacy source through deterministic previews, explicit + classification/ownership resolutions and exact plan-digest consent. +- Validate component state before mutations and retain durable recovery journals, + including prepared-draft snapshots, when rollback or cleanup cannot finish. + - Gate installation sources by a pinned legacy compatibility list or strict source-format declaration; reject unknown formats before planning or changing downstream files. - Add the JSON `capabilities --require` handshake for installation entrypoints. diff --git a/docs/component-delivery.md b/docs/component-delivery.md index 9a73d13..a04d69d 100644 --- a/docs/component-delivery.md +++ b/docs/component-delivery.md @@ -15,7 +15,9 @@ Baseline `env -u GOROOT go test ./...` passes all packages on 2026-09-07. ## Gates and realization -CLI-01 uses the independently reviewed [bridge plan](source-format-bridge.md). CLI-02/03 wait for the shared component design gate. CLI code, tests +CLI-01 follows the independently reviewed [bridge plan](source-format-bridge.md). +The shared design and W2 execution-plan gates are complete; CLI-02/03 are implemented in +[PR 64](https://github.com/dapi/memory-bank-cli/pull/64), stacked on bridge PR 63. CLI code, tests and evidence stay here. The slice is independently verifiable with synthetic source fixtures; the template PR supplies the final cross-repository source integration. @@ -39,3 +41,24 @@ findings. Unexecuted checks are not evidence. Prepare the bridge commit and record its actual binary identity before component support. Component source must wait for supporting CLI. No live downstream mutation, merge or release publication belongs to this task. Related PRs state the required release order explicitly. + +The reviewed [component runtime plan](component-runtime-plan.md) owns W2 CLI execution +sequencing. CTR-01/ADR-002 and the shared Solution Ready gate were accepted before implementation. + +## Implemented verification surfaces + +`internal/contracts` validates portable paths, deterministic wire encodings, frozen bundles, +registry continuity, reference relocation and base/flow rules. `internal/ownership` integrates +those contracts with composition, document operations, migration previews, resolution plans +and the existing pinned transaction writer. A prepared `--from` draft receives a durable +private snapshot for recovery; it remains an unchanged read precondition through cleanup. + +The local acceptance run passes the complete Go suite and vet, 28 existing E2Es, source-format +fixtures and `scripts/e2e-components.py` against the actual producer. CI repeats these checks +using immutable producer and legacy source commits. Specific fixtures include all six base +types remaining unadopted, registry/marker/identity/type tampering, renderer-1 upgrade, +ambiguous migration resolution, unchanged legacy finding multisets, selector transition +rollback, adapter closure and complete recovery with restored read inputs. + +The PR records the final immutable review revisions and CI links. No merge, release or live +migration is claimed by this implementation checkpoint. diff --git a/docs/component-runtime-plan.md b/docs/component-runtime-plan.md new file mode 100644 index 0000000..1e9d9b0 --- /dev/null +++ b/docs/component-runtime-plan.md @@ -0,0 +1,169 @@ +# Component and document runtime plan + +This is CLI #62's implementation plan for W2, owned by memory-bank-cli. It imports +[CTR-01](https://github.com/dapi/memory-bank/blob/feat/141-component-adoption/docs/components.md) +and the parent issue's acceptance contract. The template feature owns payload declarations, +base templates and flow wrappers; this file owns CLI implementation sequencing only. + +Status: active. Shared Solution Ready is satisfied by the reviewed c22294b → b94560c +contract chain (2026-09-07T01:14:17Z). This execution plan passed independent review at +acbfb32, and the bridge checkpoint is complete. Bridge checkpoint: ready PR #63 at 3b434fd93678c36447d10d4f308a39ce5d74b040, with +required CI, actual-binary canary and independent functional/simplification reviews clean. No release or live downstream migration is included. + +## Grounding and boundaries + +Grounded CLI baseline ac7101c307e65566787bdb32a1bdad40b9a8b995 plus W1 a0811c4: + +- ownership/source.go verifies pinned regular Git blobs; W1's source_format.go classifies them. +- ownership/update.go owns run, buildPlan, mutation preconditions and applyAtomicallyPinned. +- ownership/lock.go reads schema 0/1 and validates ownership/digests/modes. +- ownership/resolution_plan.go revalidates sources in PlanPull and ApplyResolutionPlan. +- doctor/governance.go currently applies metadata and feature lifecycle checks type-wide. +- cli/cli.go dispatches init/pull/doctor/lint and keeps update reserved for the executable. +- agentinstructions/block.go preserves bytes outside the managed instruction block. + +Add a pure internal/contracts package for manifest/rule decoding and frozen validation semantics. +It must not import ownership, doctor or CLI. Ownership uses contracts during transaction preflight; +doctor/lint use the same validator, avoiding divergent activation rules. The existing legacy +validator remains available for legacy installations. Schema-2 installations use explicit adoption. + +## Steps + +1. **Composition** — internal/contracts/manifest.go and ownership/components.go validate the whole + source inventory, resolve preset/adapter dependencies, filter payload and compose README/AGENTS. + Extend Lock/Options and strict lock decoding for schema 2 without upgrading ordinary legacy + source pulls. Unknown fields/paths/capabilities, missing inventory, cycles and removals fail + before writes. Component-aware PlanPull and ApplyResolutionPlan are required in W2: + both use the same composed transaction plan and validate selection, state and current source. + Apply regenerates the entire plan before writes, so a saved pre-bridge plan receives no + exemption. A matching-identity old-format plan against a component source is a direct + negative fixture. Resolution planning is complete only after successful component preview, + application, stale-file/lock/source rejection and no-mutation tests pass. +2. **Frozen contracts** — internal/contracts/rules.go and engine.go decode base types and immutable + bundles. Each bundle embeds DNA, base and extension rules, plus the exact engine ID/digest. + The trusted implementation embeds its immutable behavior artifact and a positive/negative + corpus. Published bundles and engine behavior are never modified in place. A live DNA/base + change cannot affect an adopted document's verdict. Missing or changed required bundles fail + before payload mutation. Extensions cannot weaken their embedded base requirements. +3. **Adoption integrity** — ownership/adoption.go owns the registry snapshot, identity/path/type + reconciliation and checksum binding to lock. Scan regular Markdown only under memory-bank/ for orphan projection + fields, excluding .repo, dna, flows, templates, document-types, prompts and declared managed + template assets. Do not inspect unrelated Markdown outside that root. Reject unsafe + symlinks/aliases rather than following them. Recorded targets must resolve exactly once. Base documents have no record or marker. + Schema-2 full/legacy always have a registry, even when empty; schema-0/1 installations + remain on the old validator and have no registry requirement before explicit migration. + Missing/corrupt schema-2 state is an error, never + an invitation to recreate it. Read/validate both source and installed contracts. +4. **Document operations** — ownership/documents.go and cli/documents.go implement create, adopt, + transition and move using the same handle-relative transaction engine. Report a dry-run plan; + check applicable old/new gates and explicit evidence references; commit document, registry, + history and lock together. Same adoption/move is idempotent. Unsupported detach/delete or + transitions reject before writes. Base creation in full does not adopt implicitly. Legacy-flow + creation explicitly resolves a per-document compatibility ID rather than adding to a selector. +5. **Legacy migration** — ownership/component_migration.go verifies the source-specific supported + legacy map, ownership and local drift; freezes the prior document identities in selector + snapshots; and previews the creation/validation semantics change. Apply requires both explicit + --migrate-components and the current --migration-plan-digest. The digest binds old lock, source, + resolution map, observed document bytes/modes and proposed mutations; deterministic snapshot + identities/history ensure repeated preview is stable. Unsupported versions and incomplete, + incompatible or ambiguous owner maps conflict. Migration preserves existing invalid legacy + verdicts, while identity/integrity constraints still must hold. Selector transition atomically + adds an exclusion plus a new per-document record, with no implicit precedence. +6. **Validation entrypoints** — doctor and lint check component state, dependencies, base documents, + adoption and navigation; core/docs do not require absent Flows. Source-profile projection is + explicitly distinguished from a downstream with missing lock. Preflight validates the resulting + tree, including derived_from, Markdown paths, embedded frontmatter and priming manifest paths. + Existing project-owned content is preserved by pull; scaffold ownership transfers on creation. + Migration alone allows the same pre-existing legacy validation findings: compare multisets + of stable identity, finding code, rule ID and subject before/after under the frozen engine. + Any added or removed legacy finding, identity/integrity/path failure or new navigation violation blocks apply. + Normal validation still reports the preserved errors. Test invalid legacy migration succeeds + while an added violation fails without writes; ordinary pulls get no blanket exemption. + +Selection precedence is CTR-01's contract: fresh init without flags chooses legacy; a +flagless schema-2 pull preserves the locked preset/components/adapters exactly. An explicit +preset resolves together with retained/new adapters and dependencies, then rejects removal +of any installed component. Adapter flags are additions, never replacement. No selection +flag opts a schema-0/1 installation into component migration. With a legacy source, any +component-selection flag is rejected before writes; it is never silently ignored. With a +component source and schema-0/1 lock, selection flags without explicit migration consent +also reject before writes. The flagless case rejects identically: any schema-0/1 lock plus +a component source requires --migrate-components, including unattended pull with no preset +or adapter flags. It never implicitly changes validation semantics. A direct flagless +fixture checks byte/mode/lock preservation. Direct negative fixtures cover both source formats and preserve +every downstream byte/mode and lock. Tests repeat flagless pulls for +every preset and adapter variant and assert unchanged selection and no automatic Flows. + +Migration preview is `pull --migrate-components --dry-run --json` (with explicit source +inputs and optional --migration-resolution FILE). It writes no downstream state and returns +migration_plan_digest plus the exact proposed changes/semantics. Apply passes that digest +back as `pull --migrate-components --migration-plan-digest DIGEST` with the same source and +resolution input. Apply regenerates the preview and rejects changed observations or a stale +digest before writes. Neither unattended mode nor --preset legacy replaces this consent. + +CLI flags: init/pull --preset NAME, repeated --adapter NAME; pull --migrate-components, +--migration-plan-digest DIGEST and --migration-resolution FILE. Document commands use --type, +--path, --contract, optional --from for prepared creation, --to, --id (required for move), --dry-run and repeatable --evidence REF as applicable. An explicit +--legacy-flow chooses the installation's pinned compatibility contract. No adapter removal, +uninstall, contract composition, automatic adoption, arbitrary code execution or global service +is introduced. Exact serialized fields and encoding rules are owned by the shared +[CTR-01 wire format](https://github.com/dapi/memory-bank/blob/feat/141-component-adoption/docs/component-wire-format.md). +Go types and producer/consumer fixtures implement it; semantic changes return to design review. + +## Durable recovery extension + +The consolidated CTR-01 recovery predicate requires internal/ownership/component_recovery.go +and a component-only hook in the existing transaction engine. Before mutation, persist and +sync a versioned staging journal binding every observed/target path, before/after bytes and +modes, numbered backup mapping and created directories. Use a prepared/committed journal +state: first sync existing target-file contents and staged replacements, then sync the +prepared file, its staging directory and repository parent before target mutation. +The existing writer renames originals into numbered backups; it does not copy write-target originals before mutation. The optional read-only --from +input has a separate durable inputs/000000 snapshot, bound together with every ancestor +directory state through cleanup. Sync both directories after each original rename and before +installing its replacement, preserving the already synced original inode at target or backup. +The journal also records directory before/after existence and modes for complete restoration; after all replacements and lock-last, sync changed files and directories, then +atomically persist/sync the committed journal and staging directory. A crash before that +last durable marker is ambiguous and requires restoring the complete before state. No new +file writer or automatic rollback replay is introduced. Before subsequent component planning, retained staging blocks writes unless +its complete before state has been restored; committed cleanup retries instead check complete +after state and integrity. Unknown journals fail closed. The repository owner performs manual restoration using the +journal's exact path-to-backup mapping and before observations: restore originals from +numbered backups or a trusted pre-operation backup, restore modes, remove originally absent +targets and created empty directories, and preserve concurrent edits separately. Recovery +checking never edits target files; it checks the restored mixed-state fixture against all +observations before cleaning staging and permitting ordinary preflight. Tests inject rollback/cleanup failure and verify that restoring only the lock is +insufficient, complete restoration permits cleanup/re-entry and retry is idempotent. + +## Verification and failure boundaries + +Go contract/transaction fixtures cover every parent acceptance class: preset/default/adapter +matrix; repeated init/pull; docs-to-full and scaffold preservation; malicious paths and symlinks; +base vs adopted feature; registry/marker/identity/type/path tampering; frozen bundle/DNA/base/engine +drift and missing historical bundle; migration opt-in and stale digest; ambiguous moved documents +with valid/invalid owner maps; legacy selector exclusion plus rollback; fresh and migrated legacy +creation; unsupported transitions; failure during staged writes and concurrent changed lock. +Fixtures retain a byte/mode snapshot of the old tree and an external sentinel for negative paths. + +Run `env -u GOROOT go test ./...`, `env -u GOROOT go vet ./...`, hermetic ownership E2E, and a +real component binary against the exact template candidate commit. Keep a separately built bridge +binary at the reviewed W1 commit; its real-binary fixture must reject the component candidate. +A pre-bridge binary is tested through the template's minimum-capability entrypoint. Direct +pre-bridge execution on a component source remains explicitly unsupported by the parent issue. + +Before each implementation step, read its grounded owner files; update this plan when the exact +surface changes. Independent code-converge review uses a clean author commit, explicit baseline +and --max-cycles 0 so the run cannot fix, checkpoint or publish reviewed changes. Author fixes and +commits findings, then re-runs the review. Separate final code and simplification passes must be +clean. CLI and template PRs retain an explicit bridge-first release dependency; publication tags +are assigned by the release owner after review, not invented as already available binaries. + +## Re-evaluated execution boundary + +After five artifact review iterations, the implementation keeps one imported wire owner, +a bounded memory-bank document scan and an explicit digest-producing preview command. +The source-format bridge remains a separate delivery checkpoint; this plan does not +advertise component capability until its complete operation matrix is implemented. +Producer/consumer fixtures must cover CTR-01 selector grouping/IDs, context-root derivation, +canonical registry bytes and exact legacy finding multiset equality. Shared contract review +and this execution-plan review are separate gates; both completion checkpoints are recorded above. diff --git a/go.mod b/go.mod index 52709f7..08d6bd6 100644 --- a/go.mod +++ b/go.mod @@ -5,5 +5,6 @@ go 1.21 require ( golang.org/x/sys v0.17.0 golang.org/x/term v0.17.0 + golang.org/x/text v0.14.0 gopkg.in/yaml.v3 v3.0.1 ) diff --git a/go.sum b/go.sum index ab07185..f37a3ce 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,8 @@ golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/internal/agentinstructions/block.go b/internal/agentinstructions/block.go index c1b3993..712286b 100644 --- a/internal/agentinstructions/block.go +++ b/internal/agentinstructions/block.go @@ -45,7 +45,10 @@ type markerLine struct { // BuildPlan returns a safe whole-file replacement while preserving every byte // outside the exact managed markers. A newly appended block is separated from // existing content by one blank line; no existing newline is rewritten. -func BuildPlan(original []byte) Plan { +func BuildPlan(original []byte) Plan { return BuildPlanWithBlock(original, CurrentBlock) } + +// BuildPlanWithBlock shares the exact marker boundary rules with component renderers. +func BuildPlanWithBlock(original, block []byte) Plan { rawStarts := bytes.Count(original, []byte(StartMarker)) rawEnds := bytes.Count(original, []byte(EndMarker)) upper := bytes.ToUpper(original) @@ -63,8 +66,8 @@ func BuildPlan(original []byte) Plan { } data = append(data, '\n') } - data = append(data, CurrentBlock...) - return Plan{Status: Missing, Data: data, Diff: blockDiff(nil, CurrentBlock)} + data = append(data, block...) + return Plan{Status: Missing, Data: data, Diff: blockDiff(nil, block)} } if len(starts) != 1 || len(ends) != 1 { return Plan{Status: Ambiguous} @@ -75,14 +78,14 @@ func BuildPlan(original []byte) Plan { start := starts[0].start end := ends[0].end existing := original[start:end] - if bytes.Equal(existing, CurrentBlock) { + if bytes.Equal(existing, block) { return Plan{Status: Current, Data: append([]byte(nil), original...)} } - data := make([]byte, 0, len(original)-len(existing)+len(CurrentBlock)) + data := make([]byte, 0, len(original)-len(existing)+len(block)) data = append(data, original[:start]...) - data = append(data, CurrentBlock...) + data = append(data, block...) data = append(data, original[end:]...) - return Plan{Status: Outdated, Data: data, Diff: blockDiff(existing, CurrentBlock)} + return Plan{Status: Outdated, Data: data, Diff: blockDiff(existing, block)} } // standaloneMarkers recognizes ownership boundaries only when the marker is diff --git a/internal/cli/capabilities_test.go b/internal/cli/capabilities_test.go index 1a1e762..90f3fe6 100644 --- a/internal/cli/capabilities_test.go +++ b/internal/cli/capabilities_test.go @@ -4,11 +4,16 @@ import ( "bytes" "os" "path/filepath" + "runtime" "strings" "testing" ) func TestCapabilitiesWireContract(t *testing.T) { + componentCode, componentUnsupported, caps := 1, `["components/v1","adoption/v1"]`, `["source-format/v1","legacy/v1"]` + if runtime.GOOS == "linux" || runtime.GOOS == "darwin" { + componentCode, componentUnsupported, caps = 0, `[]`, `["source-format/v1","legacy/v1","components/v1","adoption/v1"]` + } for _, tc := range []struct { args []string code int @@ -16,11 +21,11 @@ func TestCapabilitiesWireContract(t *testing.T) { }{ {nil, 0, `[]`}, {[]string{"--require", "legacy/v1", "--require", "source-format/v1"}, 0, `[]`}, - {[]string{"--require", "components/v1", "--require", "adoption/v1"}, 1, `["components/v1","adoption/v1"]`}, + {[]string{"--require", "components/v1", "--require", "adoption/v1"}, componentCode, componentUnsupported}, } { var out, err bytes.Buffer code := Run(append([]string{"capabilities"}, tc.args...), "test-version", &out, &err) - want := `{"schema_version":1,"cli_version":"test-version","capabilities":["source-format/v1","legacy/v1"],"unsupported":` + tc.unsupported + "}\n" + want := `{"schema_version":1,"cli_version":"test-version","capabilities":` + caps + `,"unsupported":` + tc.unsupported + "}\n" if code != tc.code || out.String() != want || err.Len() != 0 { t.Fatalf("code=%d stdout=%s stderr=%s", code, out.String(), err.String()) } diff --git a/internal/cli/cli.go b/internal/cli/cli.go index 3b75842..bfd810b 100644 --- a/internal/cli/cli.go +++ b/internal/cli/cli.go @@ -16,11 +16,13 @@ import ( "strings" "github.com/dapi/memory-bank-cli/internal/analyzegraph" + "github.com/dapi/memory-bank-cli/internal/contracts" "github.com/dapi/memory-bank-cli/internal/doctor" "github.com/dapi/memory-bank-cli/internal/githubadapter" "github.com/dapi/memory-bank-cli/internal/handoff" "github.com/dapi/memory-bank-cli/internal/lint" "github.com/dapi/memory-bank-cli/internal/ownership" + "github.com/dapi/memory-bank-cli/internal/projection" "github.com/dapi/memory-bank-cli/internal/push" "github.com/dapi/memory-bank-cli/internal/repository" "github.com/dapi/memory-bank-cli/internal/selfupdate" @@ -58,6 +60,8 @@ func Run(arguments []string, version string, stdout, stderr io.Writer) int { } switch arguments[0] { + case "document": + return runDocument(arguments[1:], stdout, stderr) case "capabilities": return runCapabilities(arguments[1:], version, stdout, stderr) case "analyze-graph": @@ -126,6 +130,7 @@ func printRootUsage(writer io.Writer) { fmt.Fprintln(writer, "Usage: memory-bank-cli [options]") fmt.Fprintln(writer) fmt.Fprintln(writer, "Commands:") + fmt.Fprintln(writer, " document Create, adopt, transition, or move a project document") fmt.Fprintln(writer, " capabilities Report supported source formats and required capabilities") fmt.Fprintln(writer, " analyze-graph Analyse typed execution-context handoff evidence") fmt.Fprintln(writer, " init Adopt or install a template and create its ownership lock") @@ -382,6 +387,12 @@ func runOwnership(arguments []string, command string, stdin io.Reader, stdinIsTe applyPlan := flags.String("apply-plan", "", "apply a reviewed versioned pull resolution plan from FILE") ask := flags.Bool("ask", false, "interactively resolve user-owned managed-file collisions") agentFile := flags.String("agent-file", "AGENTS.md", "single repository-relative agent instruction file to manage") + preset := flags.String("preset", "", "component preset: core, docs, full, legacy") + var adapters entrypointFlags + flags.Var(&adapters, "adapter", "add an optional adapter (repeatable)") + migrate := flags.Bool("migrate-components", false, "explicitly migrate a supported legacy installation") + migrationDigest := flags.String("migration-plan-digest", "", "exact reviewed migration preview digest") + migrationResolution := flags.String("migration-resolution", "", "legacy classification resolution JSON file") jsonOutput := addJSONOutputFlag(flags) if err := flags.Parse(arguments); err != nil { if errors.Is(err, flag.ErrHelp) { @@ -442,7 +453,18 @@ func runOwnership(arguments []string, command string, stdin io.Reader, stdinIsTe options := ownership.Options{ RepoRoot: repoRoot, SourceRoot: sourceRoot, TemplateVersion: resolvedVersion, SourceRef: resolvedRef, DryRun: *dryRun, - AgentFile: *agentFile, + AgentFile: *agentFile, Preset: *preset, Adapters: adapters, MigrateComponents: *migrate, MigrationPlanDigest: *migrationDigest, + } + if *migrationResolution != "" { + options.MigrationResolution, err = os.ReadFile(*migrationResolution) + if err != nil { + fmt.Fprintln(stderr, err) + return exitFailure + } + } + if command == "init" && (*migrate || *migrationDigest != "" || *migrationResolution != "") { + fmt.Fprintln(stderr, "migration flags require pull") + return exitUsage } if *planOutput != "" { plan, planErr := ownership.PlanPull(options) @@ -981,6 +1003,25 @@ func runLint(arguments []string, commandName, version string, stdout, stderr io. return exitFailure } + if scopeRoot == "memory-bank" && !projection.IsUninstalledSourceProjection(repoRoot, contracts.ManifestPath) { + handled, findings, nav, e := ownership.ValidateComponents(repoRoot, "") + if handled { + if e != nil { + report.Errors.Config = append(report.Errors.Config, lint.ConfigError{Message: e.Error()}) + report.ExitCode = 1 + } else { + if len(configuredEntrypoints) == 0 { + report = nav + report.RepoRoot = repoRoot + } + for _, f := range findings { + report.Errors.Config = append(report.Errors.Config, lint.ConfigError{Message: fmt.Sprintf("%s: %s (%s)", f.Code, f.RuleID, f.Subject)}) + report.ExitCode = 1 + } + } + } + } + if err := writeResult(stdout, *jsonOutput, report, func(writer io.Writer) { lint.PrintTextReport(writer, report) }); err != nil { diff --git a/internal/cli/component_projection_test.go b/internal/cli/component_projection_test.go new file mode 100644 index 0000000..4a1df24 --- /dev/null +++ b/internal/cli/component_projection_test.go @@ -0,0 +1,39 @@ +package cli + +import ( + "bytes" + "os" + "path/filepath" + "testing" +) + +func TestLintRecognizesSourceComponentProjection(t *testing.T) { + root := t.TempDir() + for _, dir := range []string{"template/memory-bank", "memory-bank"} { + if err := os.MkdirAll(filepath.Join(root, dir), 0755); err != nil { + t.Fatal(err) + } + } + if err := os.WriteFile(filepath.Join(root, "template/memory-bank/components.json"), []byte("{}"), 0644); err != nil { + t.Fatal(err) + } + marker := filepath.Join(root, "memory-bank/components.json") + if err := os.Symlink("../template/memory-bank/components.json", marker); err != nil { + t.Skip(err) + } + if err := os.WriteFile(filepath.Join(root, "memory-bank/README.md"), []byte("---\nstatus: draft\ndoc_function: index\npurpose: Navigate project.\n---\n# Memory Bank\n"), 0644); err != nil { + t.Fatal(err) + } + var out, stderr bytes.Buffer + if code := Run([]string{"lint", "--repo-root", root}, "test", &out, &stderr); code != 0 { + t.Fatalf("source lint: %d %s %s", code, out.String(), stderr.String()) + } + if err := os.WriteFile(filepath.Join(root, "memory-bank/.lock"), []byte("{}"), 0644); err != nil { + t.Fatal(err) + } + out.Reset() + stderr.Reset() + if code := Run([]string{"lint", "--repo-root", root}, "test", &out, &stderr); code == 0 { + t.Fatal("installed projection bypassed component state") + } +} diff --git a/internal/cli/documents.go b/internal/cli/documents.go new file mode 100644 index 0000000..6ce2c65 --- /dev/null +++ b/internal/cli/documents.go @@ -0,0 +1,70 @@ +package cli + +import ( + "errors" + "flag" + "fmt" + "io" + + "github.com/dapi/memory-bank-cli/internal/ownership" + "github.com/dapi/memory-bank-cli/internal/repository" +) + +func runDocument(arguments []string, stdout, stderr io.Writer) int { + if len(arguments) == 0 { + fmt.Fprintln(stderr, "Usage: memory-bank-cli document [options]") + return exitUsage + } + op := arguments[0] + switch op { + case "create", "adopt", "transition", "move": + default: + fmt.Fprintln(stderr, "unknown document operation") + return exitUsage + } + flags := flag.NewFlagSet("memory-bank-cli document "+op, flag.ContinueOnError) + flags.SetOutput(stderr) + root := addRepoRootFlag(flags) + o := ownership.DocumentOptions{Operation: op} + flags.StringVar(&o.From, "from", "", "prepared local draft for atomic creation (repository-relative)") + flags.StringVar(&o.Type, "type", "", "installed document type") + flags.StringVar(&o.Path, "path", "", "project Markdown path under memory-bank") + flags.StringVar(&o.To, "to", "", "move destination within the original context") + flags.StringVar(&o.ID, "id", "", "required stable identity for move") + flags.StringVar(&o.Contract, "contract", "", "installed flow contract to adopt") + flags.BoolVar(&o.LegacyFlow, "legacy-flow", false, "use the installation's pinned compatibility contract") + flags.BoolVar(&o.DryRun, "dry-run", false, "validate and preview without mutation") + var evidence entrypointFlags + flags.Var(&evidence, "evidence", "transition evidence reference (repeatable)") + jsonOutput := addJSONOutputFlag(flags) + if err := flags.Parse(arguments[1:]); err != nil { + if errors.Is(err, flag.ErrHelp) { + return exitSuccess + } + return exitUsage + } + if flags.NArg() != 0 { + fmt.Fprintln(stderr, "unexpected document arguments") + return exitUsage + } + var err error + o.RepoRoot, err = repository.ResolveRoot(*root) + if err != nil { + fmt.Fprintln(stderr, err) + return exitFailure + } + o.Evidence = evidence + report, err := ownership.DocumentOperation(o) + if err != nil { + if report.Applied { + _ = writeResult(stdout, *jsonOutput, report, func(w io.Writer) { printOwnershipReport(w, report) }) + } + fmt.Fprintln(stderr, err) + return exitFailure + } + if err = writeResult(stdout, *jsonOutput, report, func(w io.Writer) { printOwnershipReport(w, report) }); err != nil { + fmt.Fprintln(stderr, err) + return exitFailure + } + return exitSuccess +} diff --git a/internal/contracts/common.go b/internal/contracts/common.go new file mode 100644 index 0000000..c3ae226 --- /dev/null +++ b/internal/contracts/common.go @@ -0,0 +1,252 @@ +package contracts + +import ( + "bytes" + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "golang.org/x/text/cases" + "golang.org/x/text/unicode/norm" + "io" + "path" + "reflect" + "regexp" + "sort" + "strings" + "unicode/utf8" +) + +func Digest(data []byte) string { + sum := sha256.Sum256(data) + return "sha256:" + hex.EncodeToString(sum[:]) +} +func ValidDigest(s string) bool { + if len(s) != 71 || !strings.HasPrefix(s, "sha256:") { + return false + } + _, err := hex.DecodeString(s[7:]) + return err == nil && strings.ToLower(s) == s +} +func ValidPath(s string) bool { + if s == "" || s == "." || path.IsAbs(s) || path.Clean(s) != s || strings.HasPrefix(s, "../") || strings.ContainsAny(s, "\\\x00") || !utf8.ValidString(s) { + return false + } + for _, part := range strings.Split(s, "/") { + if strings.EqualFold(part, ".git") || strings.ContainsAny(part, `<>:"|?*`) || strings.HasSuffix(part, ".") || strings.HasSuffix(part, " ") { + return false + } + for _, r := range part { + if r < 32 || r == 127 { + return false + } + } + stem := strings.ToUpper(strings.TrimRight(strings.SplitN(part, ".", 2)[0], ". ")) + if reservedName.MatchString(stem) { + return false + } + } + return true +} + +var reservedName = regexp.MustCompile(`^(CON|PRN|AUX|NUL|CONIN\$|CONOUT\$|COM[1-9¹²³]|LPT[1-9¹²³])$`) + +func PortableKey(s string) string { return norm.NFC.String(cases.Fold().String(norm.NFC.String(s))) } +func CheckPortable(paths []string) error { + seen := map[string]string{} + for _, p := range paths { + if !ValidPath(p) { + return fmt.Errorf("unsafe path %q", p) + } + parts := strings.Split(p, "/") + for i := range parts { + segment := strings.Join(parts[:i+1], "/") + key := PortableKey(segment) + if previous, ok := seen[key]; ok && previous != segment { + return fmt.Errorf("portable path collision: %s and %s", previous, segment) + } + seen[key] = segment + } + } + return nil +} +func SortedSet(values []string) bool { + for i, s := range values { + if s == "" || (i > 0 && values[i-1] >= s) { + return false + } + } + return true +} +func Keys[V any](m map[string]V) []string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} +func FramedID(domain string, values ...string) string { + h := sha256.New() + h.Write([]byte(domain)) + h.Write([]byte{0}) + for _, v := range values { + var length [8]byte + binary.BigEndian.PutUint64(length[:], uint64(len(v))) + h.Write(length[:]) + h.Write([]byte(v)) + } + return hex.EncodeToString(h.Sum(nil)) +} + +// Decode rejects duplicate and case-aliased fields, null typed values, unknown +// fields, trailing values and invalid UTF-8 before typed JSON decoding. +func Decode(data []byte, target any) error { + if !utf8.Valid(data) { + return errors.New("JSON must be UTF-8") + } + d := json.NewDecoder(bytes.NewReader(data)) + d.UseNumber() + value, err := readValue(d) + if err != nil { + return err + } + if _, err = d.Token(); err != io.EOF { + return errors.New("trailing JSON data") + } + typ := reflect.TypeOf(target) + if typ == nil || typ.Kind() != reflect.Pointer { + return errors.New("decode target must be a pointer") + } + if err = checkShape(value, typ.Elem()); err != nil { + return err + } + return json.Unmarshal(data, target) +} +func readValue(d *json.Decoder) (any, error) { + token, err := d.Token() + if err != nil { + return nil, err + } + delim, ok := token.(json.Delim) + if !ok { + return token, nil + } + switch delim { + case '{': + m := map[string]any{} + for d.More() { + key, err := d.Token() + if err != nil { + return nil, err + } + s, ok := key.(string) + if !ok { + return nil, errors.New("invalid object key") + } + if _, ok = m[s]; ok { + return nil, fmt.Errorf("duplicate JSON field %q", s) + } + v, err := readValue(d) + if err != nil { + return nil, err + } + m[s] = v + } + _, err = d.Token() + return m, err + case '[': + a := []any{} + for d.More() { + v, err := readValue(d) + if err != nil { + return nil, err + } + a = append(a, v) + } + _, err = d.Token() + return a, err + } + return nil, errors.New("unexpected JSON delimiter") +} +func checkShape(v any, t reflect.Type) error { + if t.Kind() == reflect.Pointer { + return checkShape(v, t.Elem()) + } + if v == nil { + return errors.New("null is not a typed value") + } + if reflect.PointerTo(t).Implements(reflect.TypeOf((*json.Unmarshaler)(nil)).Elem()) { + return nil + } + switch t.Kind() { + case reflect.Struct: + obj, ok := v.(map[string]any) + if !ok { + return errors.New("expected object") + } + fields := map[string]reflect.Type{} + for i := 0; i < t.NumField(); i++ { + f := t.Field(i) + name := strings.Split(f.Tag.Get("json"), ",")[0] + if name != "-" && f.IsExported() { + if name == "" { + name = f.Name + } + fields[name] = f.Type + if !strings.Contains(f.Tag.Get("json"), ",omitempty") { + if _, exists := obj[name]; !exists { + return fmt.Errorf("missing JSON field %q", name) + } + } + } + } + for key, value := range obj { + ft, ok := fields[key] + if !ok { + return fmt.Errorf("unknown JSON field %q", key) + } + if err := checkShape(value, ft); err != nil { + return fmt.Errorf("%s: %w", key, err) + } + } + case reflect.Map: + obj, ok := v.(map[string]any) + if !ok { + return errors.New("expected map") + } + for _, value := range obj { + if err := checkShape(value, t.Elem()); err != nil { + return err + } + } + case reflect.Slice: + a, ok := v.([]any) + if !ok { + return errors.New("expected array") + } + for _, value := range a { + if err := checkShape(value, t.Elem()); err != nil { + return err + } + } + } + return nil +} + +// Canonical sorts keys even for structs and retains integer precision. +func Canonical(v any) ([]byte, error) { + data, err := json.Marshal(v) + if err != nil { + return nil, err + } + d := json.NewDecoder(bytes.NewReader(data)) + d.UseNumber() + var obj any + if err = d.Decode(&obj); err != nil { + return nil, err + } + return json.Marshal(obj) +} diff --git a/internal/contracts/document.go b/internal/contracts/document.go new file mode 100644 index 0000000..a7ad914 --- /dev/null +++ b/internal/contracts/document.go @@ -0,0 +1,315 @@ +package contracts + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "path" + "regexp" + "strings" + "unicode/utf8" + + "gopkg.in/yaml.v3" +) + +type Document struct { + Path string + Raw, Body []byte + Fields map[string]any + HasFrontmatter bool + closing int + newline string + keys map[string][2]*yaml.Node +} + +func (d Document) String(key string) string { s, _ := d.Fields[key].(string); return s } +func (d Document) Has(key string) bool { _, ok := d.Fields[key]; return ok } +func ParseDocument(p string, data []byte) (Document, error) { + d := Document{Path: p, Raw: data, Body: data, Fields: map[string]any{}, newline: "\n", keys: map[string][2]*yaml.Node{}} + if !utf8.Valid(data) { + return d, errors.New("Markdown must be UTF-8") + } + start := 0 + if bytes.HasPrefix(data, []byte("---\r\n")) { + start = 5 + d.newline = "\r\n" + } else if bytes.HasPrefix(data, []byte("---\n")) { + start = 4 + } else { + return d, nil + } + d.HasFrontmatter = true + end := -1 + bodyStart := 0 + for offset := start; offset < len(data); { + next := bytes.IndexByte(data[offset:], '\n') + if next < 0 { + next = len(data) + } else { + next += offset + 1 + } + line := bytes.TrimSuffix(bytes.TrimSuffix(data[offset:next], []byte("\n")), []byte("\r")) + if bytes.Equal(line, []byte("---")) { + end = offset + bodyStart = next + break + } + offset = next + } + if end < 0 { + return d, errors.New("unterminated YAML frontmatter") + } + d.closing = end + d.Body = data[bodyStart:] + raw := data[start:end] + var node yaml.Node + decoder := yaml.NewDecoder(bytes.NewReader(raw)) + if err := decoder.Decode(&node); err != nil && err != io.EOF { + return d, fmt.Errorf("invalid YAML: %w", err) + } + if len(node.Content) == 0 { + return d, nil + } + root := node.Content[0] + if root.Kind != yaml.MappingNode { + return d, errors.New("frontmatter must be one mapping") + } + if err := root.Decode(&d.Fields); err != nil { + return d, fmt.Errorf("invalid YAML: %w", err) + } + var extra yaml.Node + if err := decoder.Decode(&extra); err != io.EOF { + return d, errors.New("multiple YAML documents are unsupported") + } + for i := 0; i < len(root.Content); i += 2 { + k, v := root.Content[i], root.Content[i+1] + if k.Kind != yaml.ScalarNode || k.Tag != "!!str" { + return d, errors.New("frontmatter keys must be strings") + } + d.keys[k.Value] = [2]*yaml.Node{k, v} + } + return d, nil +} + +// Project preserves every unrelated byte and applies the version-1 projection +// writer, rather than round-tripping an owner's YAML through a serializer. +func Project(data []byte, updates map[string]string) ([]byte, error) { + d, err := ParseDocument("", data) + if err != nil { + return nil, err + } + order := []string{"document_type", "document_id", "flow_contract"} + for key := range updates { + if !Contains(order, key) { + return nil, fmt.Errorf("unsupported projection %s", key) + } + } + replace := map[int][]byte{} + var appendLines []byte + for _, key := range order { + value, needed := updates[key] + if !needed { + continue + } + quoted, _ := json.Marshal(value) + line := append([]byte(key+": "), quoted...) + if nodes, exists := d.keys[key]; exists { + k, v := nodes[0], nodes[1] + if k.Style != 0 || k.Column != 1 || v.Kind != yaml.ScalarNode || v.Tag != "!!str" || v.Anchor != "" || v.Line != k.Line || v.Style&(yaml.TaggedStyle|yaml.LiteralStyle|yaml.FoldedStyle) != 0 { + return nil, fmt.Errorf("projection %s requires a plain key and single-line untagged string", key) + } + physicalLines := bytes.Split(d.Raw, []byte("\n")) + sourceLine := physicalLines[k.Line] // Node lines are relative to YAML after delimiter. + var oneLine map[string]any + if err := yaml.Unmarshal(sourceLine, &oneLine); err != nil || oneLine[key] != d.Fields[key] { + return nil, fmt.Errorf("projection %s spans physical lines", key) + } + if d.String(key) == value { + continue + } + replace[k.Line+1] = line // YAML line one follows the opening delimiter. + } else { + if d.Has(key) { + return nil, fmt.Errorf("projection %s must be a direct field", key) + } + appendLines = append(appendLines, append(line, []byte(d.newline)...)...) + } + } + if !d.HasFrontmatter { + out := append([]byte("---\n"), appendLines...) + out = append(out, []byte("---\n")...) + return append(out, data...), nil + } + var out []byte + for offset, lineNo := 0, 1; offset < len(data); lineNo++ { + if offset == d.closing { + out = append(out, appendLines...) + } + next := bytes.IndexByte(data[offset:], '\n') + if next < 0 { + next = len(data) + } else { + next += offset + 1 + } + if replacement, ok := replace[lineNo]; ok { + out = append(out, replacement...) + ending := "" + if next > offset && data[next-1] == '\n' { + ending = "\n" + if next-offset > 1 && data[next-2] == '\r' { + ending = "\r\n" + } + } + out = append(out, ending...) + } else { + out = append(out, data[offset:next]...) + } + offset = next + } + return out, nil +} + +func DerivedPaths(d Document) []string { + raw, exists := d.Fields["derived_from"] + if !exists { + return nil + } + values, ok := raw.([]any) + if !ok { + values = []any{raw} + } + out := []string{} + for _, item := range values { + value := "" + switch x := item.(type) { + case string: + value = x + case map[string]any: + value, _ = x["path"].(string) + } + if strings.TrimSpace(value) != "" { + out = append(out, value) + } + } + return out +} + +var headingPattern = regexp.MustCompile(`^ {0,3}(#{1,6})(?:[ \t]+(.*)|[ \t]*)$`) +var closingHeadingPattern = regexp.MustCompile(`[ \t]+#+[ \t]*$`) + +func headingParts(line string) (int, string, bool) { + m := headingPattern.FindStringSubmatch(line) + if len(m) == 0 { + return 0, "", false + } + title := strings.TrimSpace(closingHeadingPattern.ReplaceAllString(m[2], "")) + return len(m[1]), title, true +} + +// VisibleLines removes fenced blocks and HTML comments before rule matching. +func VisibleLines(data []byte) []string { + out := []string{} + fence := byte(0) + fenceLen := 0 + comment := false + fenceRun := func(line string) (byte, int, string) { + trim := strings.TrimSpace(line) + if len(trim) == 0 { + return 0, 0, "" + } + ch := trim[0] + n := 0 + if ch == '`' || ch == '~' { + for n < len(trim) && trim[n] == ch { + n++ + } + } + return ch, n, strings.TrimSpace(trim[n:]) + } + for _, line := range strings.Split(strings.ReplaceAll(string(data), "\r\n", "\n"), "\n") { + if fence != 0 { + ch, n, rest := fenceRun(line) + if ch == fence && n >= fenceLen && rest == "" { + fence = 0 + } + continue + } + if !comment && (strings.HasPrefix(line, " ") || strings.HasPrefix(line, "\t")) { + continue + } + var visible strings.Builder + for rest := line; rest != ""; { + if comment { + i := strings.Index(rest, "-->") + if i < 0 { + break + } + rest = rest[i+3:] + comment = false + continue + } + i := strings.Index(rest, "", true}, + {"---\nstatus: draft\nderived_from: [{path: /memory-bank/README.md, fit: exact}]\n---\n", true}, + {"---\nstatus: draft\nderived_from: &dep [/memory-bank/README.md]\n---\n", false}, + {"---\nstatus: draft\nderived_from: {path: [relative.md]}\n---\n", false}, + } { + d, err := ParseDocument("drafts/input.md", []byte(tc.text)) + if err != nil { + t.Fatal(err) + } + _, err = RelocateBaseDocument(d.Raw, d.Path, "memory-bank/features/FT-1/brief.md") + if (err == nil) != tc.valid { + t.Fatalf("%q valid=%v: %v", tc.text, tc.valid, err) + } + } +} diff --git a/internal/contracts/engine.go b/internal/contracts/engine.go new file mode 100644 index 0000000..2748fc9 --- /dev/null +++ b/internal/contracts/engine.go @@ -0,0 +1,219 @@ +package contracts + +import ( + "fmt" + "path" + "regexp" + "sort" + "strings" +) + +type Identity struct { + ID string `json:"id"` + Path string `json:"path"` + Type string `json:"type"` + ContextRoot string `json:"context_root"` +} +type Finding struct { + DocumentID string `json:"document_id"` + Code string `json:"code"` + RuleID string `json:"rule_id"` + Subject string `json:"subject"` +} + +func finding(id Identity, p, code, rule string) Finding { + subject := "@context" + if p != "" { + subject = strings.TrimPrefix(p, id.ContextRoot+"/") + } + return Finding{id.ID, code, rule, subject} +} +func SortFindings(f []Finding) []Finding { + sort.SliceStable(f, func(i, j int) bool { a, _ := Canonical(f[i]); b, _ := Canonical(f[j]); return string(a) < string(b) }) + return f +} +func ValidateRules(d Document, r Rules, id Identity, context map[string]Document) []Finding { + out := []Finding{} + if !d.HasFrontmatter { + out = append(out, finding(id, d.Path, "governance.frontmatter_missing", "frontmatter")) + } + for _, key := range Keys(r.Fields) { + values := r.Fields[key] + v, ok := d.Fields[key].(string) + if !ok || strings.TrimSpace(v) == "" || (len(values) > 0 && !Contains(values, v)) { + out = append(out, finding(id, d.Path, "contract.field_invalid", "field/"+key)) + } + } + headings := Headings(d) + for _, h := range r.Sections { + if !headings[h] { + out = append(out, finding(id, d.Path, "contract.section_missing", "section/"+h)) + } + } + if r.ActiveRequiresUpstream && d.String("status") == "active" && d.Path != "memory-bank/dna/principles.md" && len(DerivedPaths(d)) == 0 { + out = append(out, finding(id, d.Path, "contract.upstream_missing", "active-upstream")) + } + if r.FeatureLifecycle { + out = append(out, featureFindings(d, id, context, false)...) + } + return SortFindings(out) +} +func ValidateBundle(d Document, b Bundle, id Identity, context map[string]Document) []Finding { + if b.Legacy { + out := legacyMetadata(d, id, id.Type == "feature", id.Type == "adr") + validContext := map[string]Document{} + for _, p := range Keys(context) { + other := context[p] + if !strings.HasPrefix(p, id.ContextRoot+"/") { + continue + } + if p != d.Path && (id.Type == "feature" || id.Type == "research" || id.Type == "epic") { + out = append(out, legacyMetadata(other, id, false, false)...) + } + fields, found, err := parseLegacyFrontmatter(other.Raw) + if err == nil && found { + other.Fields = fields + validContext[p] = other + } + } + fields, found, err := parseLegacyFrontmatter(d.Raw) + if err == nil && found { + d.Fields = fields + validContext[d.Path] = d + } + if id.Type == "feature" { + if _, valid := validContext[d.Path]; valid { + out = append(out, featureFindings(d, id, validContext, true)...) + } else if len(validContext) > 0 { + out = append(out, finding(id, "", "lifecycle.feature_brief_missing", "lifecycle.feature_brief_missing")) + } + } + return SortFindings(out) + } + r, _ := MergeRules(b.DNA, b.Base, b.Extension) + return ValidateRules(d, r, id, context) +} +func LegacyBundle(id, typ string) (Bundle, error) { + if !Contains([]string{"adr", "feature", "prd", "use_case", "research", "epic"}, typ) { + return Bundle{}, fmt.Errorf("unsupported legacy type %s", typ) + } + b := Bundle{SchemaVersion: 1, ID: id, Type: typ, Engine: EngineRef{EngineID, EngineDigest()}, Legacy: true, DNA: Rules{Fields: map[string][]string{"status": {"active", "archived", "draft"}}, ActiveRequiresUpstream: true}} + if typ == "adr" { + b.Base = Rules{Fields: map[string][]string{"decision_status": {"accepted", "proposed", "rejected", "superseded"}}} + } + if typ == "feature" { + b.Extension = Rules{FeatureLifecycle: true} + } + return b, nil +} +func legacyMetadata(d Document, id Identity, featureOwner, adr bool) []Finding { + out := []Finding{} + add := func(code string) { out = append(out, finding(id, d.Path, code, code)) } + fields, found, err := parseLegacyFrontmatter(d.Raw) + if err != nil { + add("governance.frontmatter_invalid") + return out + } + d.Fields = fields + if !found { + add("governance.frontmatter_missing") + return out + } + if !Contains([]string{"draft", "active", "archived"}, d.String("status")) { + add("governance.status_invalid") + } + if d.Has("delivery_status") { + if !Contains([]string{"planned", "in_progress", "done", "cancelled"}, d.String("delivery_status")) { + add("governance.delivery_status_invalid") + } + if !featureOwner { + add("lifecycle.delivery_status_wrong_owner") + } + } + if d.Has("decision_status") { + if !Contains([]string{"proposed", "accepted", "superseded", "rejected"}, d.String("decision_status")) { + add("governance.decision_status_invalid") + } + if d.String("doc_kind") != "adr" { + add("lifecycle.decision_status_wrong_owner") + } + } + if d.String("status") == "active" && !d.Has("derived_from") { + add("governance.derived_from_missing") + } + if d.Has("derived_from") && len(DerivedPaths(d)) == 0 { + add("governance.derived_from_invalid") + } + if adr && !d.Has("decision_status") { + add("lifecycle.adr_decision_status_missing") + } + return out +} + +var designDecisionPattern = regexp.MustCompile("(?i)^\\s*(?:(?:[-+*]|\\d+[.)])\\s+)?(?:\\|\\s*)?`?design\\s+required\\s*:\\s*`?(yes|no)`?(?:\\s*`)?(?:\\s*\\|.*|\\s*[.,;:]?\\s*)$") + +func DesignDecision(d Document) (string, bool) { + inSection := false + depth := 0 + decision := "" + for _, line := range VisibleLines(d.Body) { + if level, title, ok := headingParts(line); ok { + if strings.EqualFold(title, "Design Requirement Decision") { + inSection = true + depth = level + continue + } + if inSection && level <= depth { + break + } + } + if inSection { + if m := designDecisionPattern.FindStringSubmatch(line); len(m) > 0 { + if decision != "" && decision != strings.ToLower(m[1]) { + return "", false + } + decision = strings.ToLower(m[1]) + } + } + } + return decision, decision != "" +} +func featureFindings(d Document, id Identity, context map[string]Document, legacy bool) []Finding { + out := []Finding{} + add := func(p, code string) { out = append(out, finding(id, p, code, code)) } + design, hasDesign := context[path.Join(id.ContextRoot, "design.md")] + plan, hasPlan := context[path.Join(id.ContextRoot, "implementation-plan.md")] + delivery := d.String("delivery_status") + if delivery == "" { + add(d.Path, "lifecycle.delivery_status_missing") + } + if (hasDesign || hasPlan) && d.String("status") != "active" { + add(d.Path, "lifecycle.plan_brief_not_active") + } + decision, valid := DesignDecision(d) + if legacy { + decision, valid = legacyDesignDecision(string(d.Raw)) + } + if (hasDesign || hasPlan) && !valid { + add(d.Path, "lifecycle.design_requirement_decision_invalid") + } + if hasDesign && valid && decision == "no" { + add(design.Path, "lifecycle.design_present_when_not_required") + } + if hasPlan && valid && decision == "yes" && !hasDesign { + add(plan.Path, "lifecycle.plan_without_design") + } + if hasPlan && hasDesign && valid && decision == "yes" && design.String("status") != "active" { + add(design.Path, "lifecycle.plan_design_not_active") + } + if delivery == "in_progress" && (!hasPlan || plan.String("status") != "active") { + add("", "lifecycle.execution_plan_not_active") + } + if delivery == "done" && (!hasPlan || plan.String("status") != "archived") { + add("", "lifecycle.done_plan_not_archived") + } + if delivery == "cancelled" && hasPlan && plan.String("status") != "archived" { + add("", "lifecycle.cancelled_plan_not_archived") + } + return out +} diff --git a/internal/contracts/engine_test.go b/internal/contracts/engine_test.go new file mode 100644 index 0000000..5a046bd --- /dev/null +++ b/internal/contracts/engine_test.go @@ -0,0 +1,167 @@ +package contracts + +import ( + "bytes" + "reflect" + "testing" +) + +func document(t *testing.T, p, s string) Document { + t.Helper() + d, e := ParseDocument(p, []byte(s)) + if e != nil { + t.Fatal(e) + } + return d +} +func TestProjectionPreservesUnrelatedBytes(t *testing.T) { + before := []byte("---\r\n# owner comment\r\nstatus: 'draft' # retain\r\ncustom: |\r\n document_id: body data\r\n---\r\n\r\n# Title\r\n") + updates := map[string]string{"document_type": "feature", "document_id": "doc-abc", "flow_contract": "feature/v1"} + after, err := Project(before, updates) + if err != nil { + t.Fatal(err) + } + want := bytes.Replace(before, []byte("---\r\n\r\n#"), []byte("document_type: \"feature\"\r\ndocument_id: \"doc-abc\"\r\nflow_contract: \"feature/v1\"\r\n---\r\n\r\n#"), 1) + if !bytes.Equal(after, want) { + t.Fatalf("unexpected bytes: %q", after) + } + again, err := Project(after, updates) + if err != nil || !bytes.Equal(again, after) { + t.Fatalf("projection not idempotent: %v", err) + } + changed, err := Project(after, map[string]string{"flow_contract": "feature/v2"}) + if err != nil || !bytes.Equal(changed, bytes.Replace(after, []byte("feature/v1"), []byte("feature/v2"), 1)) { + t.Fatalf("transition changed unrelated bytes: %v", err) + } +} +func TestProjectionRejectsAmbiguousYAML(t *testing.T) { + for _, raw := range []string{"---\nstatus: draft\nstatus: active\n---\n", "---\nstatus: draft\n", "---\ndocument_id: |\n old\n---\n", "---\n'document_id': old\n---\n"} { + if _, err := Project([]byte(raw), map[string]string{"document_id": "new"}); err == nil { + t.Fatalf("accepted %q", raw) + } + } +} +func TestRuleExtensionsCannotWeaken(t *testing.T) { + parent := Rules{Fields: map[string][]string{"status": {"active", "draft"}}, Sections: []string{"Problem"}, ActiveRequiresUpstream: true} + for _, child := range []Rules{{Fields: map[string][]string{"status": {}}}, {Fields: map[string][]string{"status": {"archived"}}}, {Fields: map[string][]string{"status": {"active", "archived", "draft"}}}} { + if _, err := MergeRules(parent, child); err == nil { + t.Fatal("accepted weakening") + } + } + merged, err := MergeRules(parent, Rules{Fields: map[string][]string{"status": {"draft"}}, Sections: []string{"Verify"}}) + if err != nil || !merged.ActiveRequiresUpstream || !reflect.DeepEqual(merged.Sections, []string{"Problem", "Verify"}) { + t.Fatalf("invalid strengthening: %#v %v", merged, err) + } +} +func TestBaseAndFlowGateDiffer(t *testing.T) { + d := document(t, "memory-bank/features/FT-1/brief.md", "---\nstatus: draft\ndocument_type: feature\n---\n# Feature\n## Problem\n") + id := Identity{"", d.Path, "feature", "memory-bank/features/FT-1"} + base := Rules{Fields: map[string][]string{"status": {"active", "archived", "draft"}}, Sections: []string{"Problem"}} + if got := ValidateRules(d, base, id, nil); len(got) != 0 { + t.Fatal(got) + } + flow, _ := MergeRules(base, Rules{Fields: map[string][]string{"delivery_status": {"planned"}}, Sections: []string{"Verify"}}) + if got := ValidateRules(d, flow, id, nil); len(got) != 2 { + t.Fatalf("missing flow gates: %v", got) + } +} +func TestLegacyFindingsSurviveProjectionAndPinnedDependencies(t *testing.T) { + p := "memory-bank/features/FT-1/brief.md" + d := document(t, p, "---\nstatus: active\nderived_from: ../../product/context.md\ndelivery_status: planned\n---\n# Feature\n") + design := document(t, "memory-bank/features/FT-1/design.md", "---\nstatus: draft\nderived_from: brief.md\n---\n# Design\n") + id := Identity{"doc-stable", p, "feature", "memory-bank/features/FT-1"} + bundle, _ := LegacyBundle("legacy/f1f04de/feature/v1", "feature") + ctx := map[string]Document{d.Path: d, design.Path: design} + before := ValidateBundle(d, bundle, id, ctx) + if len(before) != 1 || before[0].Code != "lifecycle.design_requirement_decision_invalid" { + t.Fatalf("expected missing mandatory legacy decision section: %v", before) + } + raw, err := Project(d.Raw, map[string]string{"document_id": id.ID, "document_type": "feature"}) + if err != nil { + t.Fatal(err) + } + after := document(t, p, string(raw)) + ctx[p] = after + if got := ValidateBundle(after, bundle, id, ctx); !reflect.DeepEqual(got, before) { + t.Fatalf("changed legacy verdict: %v / %v", before, got) + } + valid := document(t, p, string(raw)+"\n## Design Requirement Decision\nDesign required: yes\n") + if got := ValidateBundle(valid, bundle, id, ctx); len(got) != 0 { + t.Fatalf("valid legacy document failed: %v", got) + } + // Latest live rules are deliberately not supplied to ValidateBundle. + stricter := Rules{Sections: []string{"New gate"}} + if len(ValidateRules(valid, stricter, id, ctx)) == 0 || len(ValidateBundle(valid, bundle, id, ctx)) != 0 { + t.Fatal("live rules leaked into frozen verdict") + } +} +func TestHeadingsIgnoreCommentsAndFences(t *testing.T) { + d := document(t, "memory-bank/a.md", "---\nstatus: draft\n---\n\n```markdown\n## Hidden too\n```\n## Real ##\n") + if h := Headings(d); !reflect.DeepEqual(h, map[string]bool{"Real": true}) { + t.Fatal(h) + } +} +func TestWindowsAndUnicodePortableNames(t *testing.T) { + for _, p := range []string{"x/CON.md", "x/LPT¹.txt", "x/COM9", "x/file:stream", "x/name.", "x/name ", "x/aux .txt", "x/a\x7fb"} { + if ValidPath(p) { + t.Fatalf("accepted %q", p) + } + } + for input, want := range map[string]string{"Straße": "strasse", "Σ/ς/σ": "σ/σ/σ", "e\u0301.md": "é.md", "K": "k"} { + if got := PortableKey(input); got != want { + t.Fatalf("%q: %q", input, got) + } + } +} + +func TestFenceCommentsDoNotHideLaterHeadings(t *testing.T) { + d := document(t, "memory-bank/a.md", "---\nstatus: draft\n---\n```html\n\n") + if _, valid := DesignDecision(d); valid { + t.Fatal("modern engine treated a comment as a section") + } + if decision, valid := legacyDesignDecision(string(d.Raw)); !valid || decision != "yes" { + t.Fatal("compatibility parser changed historical behavior") + } +} + +func TestProjectionRejectsMultilineScalarContinuation(t *testing.T) { + for _, raw := range []string{"---\ndocument_id: \"old\n continued\"\nstatus: draft\n---\n", "---\ndocument_id: old\n continued\nstatus: draft\n---\n"} { + if _, err := Project([]byte(raw), map[string]string{"document_id": "replacement"}); err == nil { + t.Fatalf("accepted multiline projection: %q", raw) + } + } +} +func TestLegacyMissingFrontmatterCompanionDoesNotActivateLifecycle(t *testing.T) { + p := "memory-bank/features/FT-1/brief.md" + d := document(t, p, "---\nstatus: draft\ndelivery_status: planned\n---\n# Brief\n") + companion := document(t, "memory-bank/features/FT-1/design.md", "# No frontmatter\n") + id := Identity{"doc-stable", p, "feature", "memory-bank/features/FT-1"} + bundle, _ := LegacyBundle("legacy/f1f04de/feature/v1", "feature") + got := ValidateBundle(d, bundle, id, map[string]Document{p: d, companion.Path: companion}) + if len(got) != 1 || got[0].Code != "governance.frontmatter_missing" || got[0].Subject != "design.md" { + t.Fatalf("legacy presence semantics changed: %v", got) + } +} +func TestEmbeddedMetadataCRLFAndHorizontalRule(t *testing.T) { + if !hasEmbeddedFrontmatter([]byte("# Example\r\n---\r\nstatus: draft\r\n---\r\n")) { + t.Fatal("missed CRLF embedded metadata") + } + if hasEmbeddedFrontmatter([]byte("# Title\n---\nJust a paragraph.\n---\n")) { + t.Fatal("horizontal rule mistaken for metadata") + } +} + +func TestEmbeddedLifecycleAndDependencyFieldsRejected(t *testing.T) { + for _, field := range []string{"delivery_status", "research_status", "decision_status", "derived_from", "purpose", "doc_function"} { + if !hasEmbeddedFrontmatter([]byte("# Body\n---\n" + field + ": value\n---\n")) { + t.Fatalf("missed %s", field) + } + } +} diff --git a/internal/contracts/engines/governance-v1.json b/internal/contracts/engines/governance-v1.json new file mode 100644 index 0000000..2967736 --- /dev/null +++ b/internal/contracts/engines/governance-v1.json @@ -0,0 +1 @@ +{"id":"governance/v1","schema_version":1,"operators":["active_requires_upstream","feature_lifecycle","fields","sections"],"markdown":"atx-outside-fences-comments/v1","yaml":"single-top-level-mapping-duplicate-rejection/v1","legacy_classifier":"legacy-f1f04de/v1","finding_identity":"document-id-code-rule-context-relative-subject/v1","feature_lifecycle":"f1f04de-brief-context-and-design-decision/v1"} diff --git a/internal/contracts/legacy_parser.go b/internal/contracts/legacy_parser.go new file mode 100644 index 0000000..ffe5e74 --- /dev/null +++ b/internal/contracts/legacy_parser.go @@ -0,0 +1,100 @@ +// Legacy parser/operators are retained from CLI ac7101c for source f1f04de. +package contracts + +import ( + "bytes" + "fmt" + "gopkg.in/yaml.v3" + "regexp" + "strings" +) + +var ( + legacyDesignSectionHeading = regexp.MustCompile(`(?i)^\s*(#{1,6})\s+Design Requirement Decision\s*#*\s*$`) + legacyDesignHeading = regexp.MustCompile(`^\s*(#{1,6})\s+`) + legacyDecisionPattern = regexp.MustCompile("(?im)^\\s*(?:(?:[-+*]|\\d+[.)])\\s+)?(?:\\|\\s*)?`?design\\s+required\\s*:\\s*`?(yes|no)`?(?:\\s*`)?(?:\\s*\\|.*|\\s*[.,;:]?\\s*)$") +) + +func legacyDesignDecision(content string) (string, bool) { + section := legacyDesignSection(content) + matches := legacyDecisionPattern.FindAllStringSubmatch(section, -1) + if len(matches) == 0 { + return "", false + } + decision := matches[0][1] + if decision != "yes" && decision != "no" { + return "", false + } + for _, match := range matches[1:] { + if match[1] != decision { + return "", false + } + } + return decision, true +} + +func legacyDesignSection(content string) string { + lines := strings.Split(strings.ReplaceAll(content, "\r\n", "\n"), "\n") + sectionLines := []string{} + inSection := false + inFence := false + sectionDepth := 0 + for _, line := range lines { + trimmed := strings.TrimSpace(line) + if strings.HasPrefix(trimmed, "```") || strings.HasPrefix(trimmed, "~~~") { + inFence = !inFence + continue + } + if inFence { + continue + } + if matches := legacyDesignSectionHeading.FindStringSubmatch(line); len(matches) > 0 { + inSection = true + sectionDepth = len(matches[1]) + continue + } + if inSection { + if matches := legacyDesignHeading.FindStringSubmatch(line); len(matches) > 0 && len(matches[1]) <= sectionDepth { + return strings.Join(sectionLines, "\n") + } + sectionLines = append(sectionLines, line) + } + } + return strings.Join(sectionLines, "\n") +} + +func parseLegacyFrontmatter(data []byte) (map[string]any, bool, error) { + // YAML permits CRLF line endings. Normalize them before recognizing the + // Markdown delimiters so governed documents work consistently across + // platforms. + data = bytes.ReplaceAll(data, []byte("\r\n"), []byte("\n")) + if !bytes.HasPrefix(data, []byte("---\n")) { + return nil, false, nil + } + remainder := data[4:] + end := -1 + for offset := 0; offset < len(remainder); { + candidate := bytes.Index(remainder[offset:], []byte("\n---")) + if candidate < 0 { + break + } + candidate += offset + // A delimiter must occupy its entire line. Without this check a value + // such as "---not-a-delimiter" silently closes the frontmatter. + afterDelimiter := candidate + len("\n---") + if afterDelimiter == len(remainder) || remainder[afterDelimiter] == '\n' { + end = candidate + break + } + offset = afterDelimiter + } + if end < 0 { + return nil, true, fmt.Errorf("unterminated YAML frontmatter") + } + frontmatter := map[string]any{} + decoder := yaml.NewDecoder(bytes.NewReader(remainder[:end])) + if err := decoder.Decode(&frontmatter); err != nil { + return nil, true, fmt.Errorf("invalid YAML frontmatter: %w", err) + } + return frontmatter, true, nil +} diff --git a/internal/contracts/manifest.go b/internal/contracts/manifest.go new file mode 100644 index 0000000..67f8b86 --- /dev/null +++ b/internal/contracts/manifest.go @@ -0,0 +1,371 @@ +package contracts + +import ( + "errors" + "fmt" + "reflect" + "regexp" + "strings" +) + +const ManifestPath = "memory-bank/components.json" +const RegistryPath = "memory-bank/.adoption.json" + +var contractIDPattern = regexp.MustCompile(`^[a-z][a-z0-9_-]*(/[a-z0-9_-]+)*/v[1-9][0-9]*$`) + +func ValidContractID(s string) bool { return contractIDPattern.MatchString(s) } + +var sourceRefPattern = regexp.MustCompile(`^[0-9a-f]{40}([0-9a-f]{24})?$`) + +type Component struct { + Dependencies []string `json:"dependencies"` + Adapter bool `json:"adapter"` + Legacy bool `json:"legacy"` +} +type File struct { + Component string `json:"component"` + Ownership string `json:"ownership"` +} +type BundleRef struct { + Path string `json:"path"` + Digest string `json:"digest"` +} +type Compatibility struct { + Classifier string `json:"classifier"` + Contracts map[string]string `json:"contracts"` +} +type PathMigration struct { + To string `json:"to"` + Policy string `json:"policy"` +} +type Manifest struct { + SchemaVersion int `json:"schema_version"` + Capabilities []string `json:"capabilities"` + DNAContract string `json:"dna_contract"` + Components map[string]Component `json:"components"` + Presets map[string][]string `json:"presets"` + Files map[string]File `json:"files"` + DocumentTypes map[string]string `json:"document_types"` + Contracts map[string]BundleRef `json:"contracts"` + LegacySources map[string]Compatibility `json:"legacy_sources"` + LegacyDefaultSourceRef string `json:"legacy_default_source_ref"` + MigrationPaths map[string]PathMigration `json:"migration_paths,omitempty"` +} +type Installation struct { + RendererVersion *int `json:"renderer_version,omitempty"` + Preset string `json:"preset"` + Components []string `json:"components"` + Adapters []string `json:"adapters"` + ManifestDigest string `json:"manifest_digest"` + AdoptionDigest string `json:"adoption_digest,omitempty"` + LegacySourceRef string `json:"legacy_source_ref,omitempty"` +} + +func (s Installation) Has(id string) bool { + for _, v := range append(append([]string{}, s.Components...), s.Adapters...) { + if v == id { + return true + } + } + return false +} +func ReadManifest(data []byte, inventory map[string][]byte) (Manifest, error) { + var m Manifest + if err := Decode(data, &m); err != nil { + return m, err + } + if m.SchemaVersion != 1 { + return m, errors.New("unsupported component schema") + } + if !SortedSet(m.Capabilities) { + return m, errors.New("capabilities must be a sorted set") + } + caps := map[string]bool{} + for _, c := range m.Capabilities { + switch c { + case "components/v1", "adoption/v1", "source-format/v1", "legacy/v1": + caps[c] = true + default: + return m, fmt.Errorf("unsupported capability %q", c) + } + } + if !caps["components/v1"] || !caps["adoption/v1"] { + return m, errors.New("component capabilities missing") + } + required := map[string][]string{"dna": {}, "documents": {"dna"}, "flows": {"dna", "documents"}} + for name, deps := range required { + c, ok := m.Components[name] + if !ok || c.Adapter || c.Legacy || !reflect.DeepEqual(c.Dependencies, deps) { + return m, fmt.Errorf("invalid %s component dependencies", name) + } + } + for id, c := range m.Components { + if id == "" || !SortedSet(c.Dependencies) { + return m, errors.New("invalid component or dependencies") + } + if _, ok := required[id]; !ok && !c.Adapter { + return m, fmt.Errorf("extra non-adapter component %s", id) + } + if _, err := m.closure([]string{id}); err != nil { + return m, err + } + } + if len(m.Presets) != 4 { + return m, errors.New("exactly four presets are required") + } + for p, ids := range map[string][]string{"core": {"dna"}, "docs": {"dna", "documents"}, "full": {"dna", "documents", "flows"}} { + if !reflect.DeepEqual(m.Presets[p], ids) { + return m, fmt.Errorf("invalid preset %s", p) + } + } + if !SortedSet(m.Presets["legacy"]) { + return m, errors.New("invalid legacy preset") + } + legacy, err := m.closure(m.Presets["legacy"]) + if err != nil { + return m, err + } + expected := map[string]bool{"dna": true, "documents": true, "flows": true} + for id, c := range m.Components { + if c.Legacy { + expected[id] = true + } + } + expectedClosure, err := m.closure(Keys(expected)) + if err != nil || !reflect.DeepEqual(legacy, expectedClosure) { + return m, errors.New("legacy preset does not preserve declared legacy adapters") + } + if inventory != nil && len(m.Files) != len(inventory) { + return m, errors.New("inventory does not match payload") + } + if err := CheckPortable(Keys(m.Files)); err != nil { + return m, err + } + for p, f := range m.Files { + if !ValidPath(p) || p == "AGENTS.md" || p == RegistryPath || p == "memory-bank/.lock" || strings.HasPrefix(p, ".memory-bank-update-") || strings.HasPrefix(p, "memory-bank/.repo/") { + return m, fmt.Errorf("reserved or unsafe payload path %s", p) + } + if _, ok := inventory[p]; inventory != nil && !ok { + return m, fmt.Errorf("missing payload %s", p) + } + if _, ok := m.Components[f.Component]; !ok { + return m, fmt.Errorf("unknown file component %s", f.Component) + } + if f.Ownership != "managed" && f.Ownership != "user-owned" { + return m, fmt.Errorf("invalid ownership %s", p) + } + } + for _, p := range []string{ManifestPath, "memory-bank/README.md"} { + if m.Files[p] != (File{"dna", "managed"}) { + return m, fmt.Errorf("reserved composed path %s must be managed DNA", p) + } + } + if err := m.requireFile(m.DNAContract, "dna"); err != nil { + return m, err + } + if len(m.DocumentTypes) == 0 || len(m.Contracts) == 0 { + return m, errors.New("document types and contracts are required") + } + for typ, p := range m.DocumentTypes { + if typ == "" { + return m, errors.New("empty type") + } + if err := m.requireFile(p, "documents"); err != nil { + return m, err + } + } + for id, b := range m.Contracts { + if !ValidContractID(id) || !ValidDigest(b.Digest) { + return m, errors.New("invalid bundle reference") + } + if err := m.requireFile(b.Path, "flows"); err != nil { + return m, err + } + if inventory != nil && Digest(inventory[b.Path]) != b.Digest { + return m, fmt.Errorf("bundle digest mismatch: %s", id) + } + } + if _, ok := m.LegacySources[m.LegacyDefaultSourceRef]; !ok { + return m, errors.New("legacy creation baseline missing") + } + for ref, c := range m.LegacySources { + if !sourceRefPattern.MatchString(ref) || ref != "f1f04de843aef45a2425d4a7351d577bbf89e940" || c.Classifier != "legacy-f1f04de/v1" || len(c.Contracts) == 0 { + return m, errors.New("unsupported legacy classifier") + } + for typ, id := range c.Contracts { + if _, ok := m.DocumentTypes[typ]; !ok { + return m, errors.New("unknown compatibility type") + } + if _, ok := m.Contracts[id]; !ok { + return m, errors.New("missing compatibility bundle") + } + } + } + for from, migration := range m.MigrationPaths { + if migration.Policy != "retain-wrapper" || !strings.HasPrefix(from, "memory-bank/flows/templates/") { + return m, errors.New("unsupported path migration") + } + if err := m.requireFile(from, "flows"); err != nil { + return m, err + } + if err := m.requireFile(migration.To, "documents"); err != nil { + return m, err + } + } + return m, nil +} +func (m Manifest) requireFile(p, component string) error { + f, ok := m.Files[p] + if !ok || !ValidPath(p) || f.Component != component || f.Ownership != "managed" { + return fmt.Errorf("%s must be a managed %s file", p, component) + } + return nil +} +func (m Manifest) closure(ids []string) (map[string]bool, error) { + state := map[string]int{} + result := map[string]bool{} + var visit func(string) error + visit = func(id string) error { + c, ok := m.Components[id] + if !ok { + return fmt.Errorf("unknown component %q", id) + } + if state[id] == 1 { + return errors.New("component dependency cycle") + } + if state[id] == 2 { + return nil + } + state[id] = 1 + for _, dep := range c.Dependencies { + if err := visit(dep); err != nil { + return err + } + } + state[id] = 2 + result[id] = true + return nil + } + for _, id := range ids { + if err := visit(id); err != nil { + return nil, err + } + } + return result, nil +} +func (m Manifest) Select(preset string, adapters []string, old *Installation) (Installation, error) { + if old != nil && preset == "" && len(adapters) == 0 { + ids := append(append([]string{}, old.Components...), old.Adapters...) + closure, err := m.closure(ids) + if err != nil { + return Installation{}, err + } + if !reflect.DeepEqual(Keys(closure), sorted(ids)) { + return Installation{}, errors.New("source changed locked dependency closure") + } + return *old, nil + } + if preset == "" { + preset = "legacy" + if old != nil { + preset = old.Preset + } + } + ids, ok := m.Presets[preset] + if !ok { + return Installation{}, fmt.Errorf("unknown preset %q", preset) + } + ids = append([]string{}, ids...) + if old != nil { + adapters = append(append([]string{}, adapters...), old.Adapters...) + } + for _, id := range adapters { + c, ok := m.Components[id] + if !ok || !c.Adapter { + return Installation{}, fmt.Errorf("unknown adapter %q", id) + } + } + ids = append(ids, adapters...) + resolved, err := m.closure(ids) + if err != nil { + return Installation{}, err + } + if old != nil { + for _, id := range append(append([]string{}, old.Components...), old.Adapters...) { + if !resolved[id] { + return Installation{}, errors.New("component removal is unsupported") + } + } + } + s := Installation{Preset: preset, Components: []string{}, Adapters: []string{}} + for _, id := range Keys(resolved) { + if m.Components[id].Adapter { + s.Adapters = append(s.Adapters, id) + } else { + s.Components = append(s.Components, id) + } + } + if old != nil { + s.LegacySourceRef = old.LegacySourceRef + s.AdoptionDigest = old.AdoptionDigest + } + if preset == "legacy" && s.LegacySourceRef == "" { + s.LegacySourceRef = m.LegacyDefaultSourceRef + } + return s, nil +} +func sorted(values []string) []string { + seen := map[string]bool{} + for _, v := range values { + seen[v] = true + } + return Keys(seen) +} + +// ValidateInstallation checks a persisted closure rather than treating it as new opt-in. +func (m Manifest) ValidateInstallation(s Installation) error { + if s.RendererVersion != nil && *s.RendererVersion != 1 && *s.RendererVersion != 2 && *s.RendererVersion != 3 { + return errors.New("unsupported component renderer") + } + if !ValidDigest(s.ManifestDigest) || !SortedSet(s.Components) || !SortedSet(s.Adapters) { + return errors.New("invalid installation metadata") + } + if s.Has("flows") != ValidDigest(s.AdoptionDigest) || (!s.Has("flows") && s.AdoptionDigest != "") { + return errors.New("invalid adoption digest for selection") + } + for _, id := range s.Components { + c, ok := m.Components[id] + if !ok || c.Adapter { + return errors.New("invalid installed component") + } + } + for _, id := range s.Adapters { + c, ok := m.Components[id] + if !ok || !c.Adapter { + return errors.New("invalid installed adapter") + } + } + if s.Preset == "legacy" { + // The legacy preset is an installation default, not permission to adopt + // adapters newly marked legacy in a subsequent source. + if !reflect.DeepEqual(s.Components, []string{"dna", "documents", "flows"}) { + return errors.New("invalid legacy component closure") + } + ids := append(append([]string{}, s.Components...), s.Adapters...) + closure, e := m.closure(ids) + if e != nil || !reflect.DeepEqual(Keys(closure), sorted(ids)) { + return errors.New("invalid locked legacy adapter closure") + } + } else { + wanted, err := m.Select(s.Preset, s.Adapters, nil) + if err != nil || !reflect.DeepEqual(wanted.Components, s.Components) || !reflect.DeepEqual(wanted.Adapters, s.Adapters) { + return errors.New("installed closure does not match preset and adapters") + } + } + if s.LegacySourceRef != "" { + if _, ok := m.LegacySources[s.LegacySourceRef]; !ok || !s.Has("flows") { + return errors.New("unsupported legacy creation reference") + } + } + return nil +} diff --git a/internal/contracts/manifest_test.go b/internal/contracts/manifest_test.go new file mode 100644 index 0000000..62a521b --- /dev/null +++ b/internal/contracts/manifest_test.go @@ -0,0 +1,147 @@ +package contracts + +import ( + "encoding/json" + "reflect" + "testing" +) + +func fixture() (Manifest, map[string][]byte) { + const legacy = "f1f04de843aef45a2425d4a7351d577bbf89e940" + inventory := map[string][]byte{ManifestPath: []byte("manifest"), "memory-bank/README.md": []byte("index"), "memory-bank/dna/rules.json": []byte("rules"), "memory-bank/document-types/feature.json": []byte("type"), "memory-bank/flows/contracts/feature.json": []byte("bundle"), ".codex/agents/a.toml": []byte("adapter")} + m := Manifest{SchemaVersion: 1, Capabilities: []string{"adoption/v1", "components/v1"}, DNAContract: "memory-bank/dna/rules.json", Components: map[string]Component{"dna": {Dependencies: []string{}}, "documents": {Dependencies: []string{"dna"}}, "flows": {Dependencies: []string{"dna", "documents"}}, "codex": {Dependencies: []string{"flows"}, Adapter: true, Legacy: true}}, Presets: map[string][]string{"core": {"dna"}, "docs": {"dna", "documents"}, "full": {"dna", "documents", "flows"}, "legacy": {"codex", "dna", "documents", "flows"}}, Files: map[string]File{}, DocumentTypes: map[string]string{"feature": "memory-bank/document-types/feature.json"}, Contracts: map[string]BundleRef{"feature/v1": {"memory-bank/flows/contracts/feature.json", Digest([]byte("bundle"))}}, LegacySources: map[string]Compatibility{legacy: {"legacy-f1f04de/v1", map[string]string{"feature": "feature/v1"}}}, LegacyDefaultSourceRef: legacy} + for p := range inventory { + m.Files[p] = File{"dna", "managed"} + } + m.Files["memory-bank/document-types/feature.json"] = File{"documents", "managed"} + m.Files["memory-bank/flows/contracts/feature.json"] = File{"flows", "managed"} + m.Files[".codex/agents/a.toml"] = File{"codex", "managed"} + return m, inventory +} +func TestPresetMatrix(t *testing.T) { + m, inv := fixture() + data, _ := json.Marshal(m) + parsed, err := ReadManifest(data, inv) + if err != nil { + t.Fatal(err) + } + m = parsed + for _, tc := range []struct { + preset string + adapters []string + want []string + }{{"core", nil, []string{"dna"}}, {"docs", nil, []string{"dna", "documents"}}, {"full", nil, []string{"dna", "documents", "flows"}}, {"", nil, []string{"codex", "dna", "documents", "flows"}}, {"full", []string{"codex"}, []string{"codex", "dna", "documents", "flows"}}, {"core", []string{"codex"}, []string{"codex", "dna", "documents", "flows"}}} { + s, err := m.Select(tc.preset, tc.adapters, nil) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(sorted(append(append([]string{}, s.Components...), s.Adapters...)), tc.want) { + t.Fatalf("%s: %#v", tc.preset, s) + } + again, err := m.Select("", nil, &s) + if err != nil || !reflect.DeepEqual(again, s) { + t.Fatalf("selection changed: %#v %v", again, err) + } + } + docs, _ := m.Select("docs", nil, nil) + if _, err := m.Select("core", nil, &docs); err == nil { + t.Fatal("accepted downgrade") + } + if full, err := m.Select("full", nil, &docs); err != nil || !full.Has("flows") { + t.Fatal("upgrade failed") + } + withAdapter, _ := m.Select("full", []string{"codex"}, nil) + if s, err := m.Select("core", nil, &withAdapter); err != nil || !s.Has("flows") { + t.Fatalf("compared before retained adapter closure: %v", err) + } +} +func TestInventoryRejections(t *testing.T) { + for _, tc := range []struct { + name string + edit func(*Manifest, map[string][]byte) + }{ + {"unknown file", func(m *Manifest, i map[string][]byte) { i["extra"] = nil }}, + {"cycle", func(m *Manifest, i map[string][]byte) { + m.Components["codex"] = Component{Dependencies: []string{"codex"}, Adapter: true, Legacy: true} + }}, + {"manifest ownership", func(m *Manifest, i map[string][]byte) { m.Files[ManifestPath] = File{"documents", "managed"} }}, + {"bundle tamper", func(m *Manifest, i map[string][]byte) { + i["memory-bank/flows/contracts/feature.json"] = []byte("changed") + }}, + {"legacy omitted adapter", func(m *Manifest, i map[string][]byte) { m.Presets["legacy"] = []string{"dna", "documents", "flows"} }}, + {"unknown component", func(m *Manifest, i map[string][]byte) { m.Files["memory-bank/README.md"] = File{"unknown", "managed"} }}, + } { + t.Run(tc.name, func(t *testing.T) { + m, i := fixture() + tc.edit(&m, i) + data, _ := json.Marshal(m) + if _, err := ReadManifest(data, i); err == nil { + t.Fatal("accepted invalid manifest") + } + }) + } +} +func TestPortablePaths(t *testing.T) { + for _, paths := range [][]string{{"x/Foo.md", "x/foo.md"}, {"x/é.md", "x/e\u0301.md"}, {"x/straße.md", "x/STRASSE.md"}, {"A/x.md", "a/y.md"}, {"../outside"}, {"x/.GIT/config"}, {"/absolute"}, {"x\\y"}} { + if err := CheckPortable(paths); err == nil { + t.Fatalf("accepted %q", paths) + } + } + if err := CheckPortable([]string{"x/a.md", "x/b.md"}); err != nil { + t.Fatal(err) + } +} +func TestStrictJSON(t *testing.T) { + for _, s := range []string{`{"schema_version":1,"schema_version":1}`, `{"Schema_Version":1}`, `{"schema_version":1,"unknown":false}`, `{"schema_version":"1"}`, `null`, `{} {}`, `{"schema_version":null}`} { + var m Manifest + if err := Decode([]byte(s), &m); err == nil { + t.Fatalf("accepted %s", s) + } + } +} +func TestCanonicalStructsAndFrames(t *testing.T) { + type v struct { + Z string `json:"z"` + A int `json:"a"` + } + data, err := Canonical(v{"<\n", 1}) + if err != nil || string(data) != `{"a":1,"z":"\u003c\n"}` { + t.Fatalf("%s %v", data, err) + } + if FramedID("d", "ab", "c") == FramedID("d", "a", "bc") { + t.Fatal("ambiguous frames") + } +} + +func TestUnicodeConformance(t *testing.T) { + for input, want := range map[string]string{"Foo.md": "foo.md", "e\u0301.md": "é.md", "É.md": "é.md", "Straße.md": "strasse.md", "STRASSE.md": "strasse.md", "K.md": "k.md", "Σ.md": "σ.md", "ς.md": "σ.md"} { + if got := PortableKey(input); got != want { + t.Fatalf("%q: %q != %q", input, got, want) + } + } +} +func TestMissingRequiredField(t *testing.T) { + var c Component + if err := Decode([]byte(`{"dependencies":[],"adapter":false}`), &c); err == nil { + t.Fatal("missing legacy field was accepted") + } +} + +func TestLockedLegacyDoesNotAdoptNewDefaultAdapters(t *testing.T) { + m, _ := fixture() + s, err := m.Select("legacy", nil, nil) + if err != nil { + t.Fatal(err) + } + s.ManifestDigest = Digest([]byte("manifest")) + s.AdoptionDigest = Digest([]byte("registry")) + m.Components["newadapter"] = Component{Dependencies: []string{"flows"}, Adapter: true, Legacy: true} + m.Presets["legacy"] = []string{"codex", "dna", "documents", "flows", "newadapter"} + kept, err := m.Select("", nil, &s) + if err != nil || kept.Has("newadapter") { + t.Fatalf("implicit adapter: %+v %v", kept, err) + } + if err = m.ValidateInstallation(kept); err != nil { + t.Fatalf("old closure rejected: %v", err) + } +} diff --git a/internal/contracts/priming.go b/internal/contracts/priming.go new file mode 100644 index 0000000..51fd7f9 --- /dev/null +++ b/internal/contracts/priming.go @@ -0,0 +1,136 @@ +package contracts + +import ( + "bytes" + "errors" + "fmt" + "io" + "path" + "regexp" + "strings" + + "gopkg.in/yaml.v3" +) + +var primingIdentifier = regexp.MustCompile(`^[a-z][a-z0-9_]*$`) +var primingPlaceholder = regexp.MustCompile(`<[A-Z][A-Z0-9-]*>`) +var primingExternal = regexp.MustCompile(`(?i)^[a-z][a-z0-9+.-]*://`) + +// ValidatePriming mirrors the payload's version-1 priming schema without +// executing the manifests or consulting the human prompt catalog as workflow. +func ValidatePriming(files map[string][]byte) error { + processes := map[string]bool{} + for _, p := range Keys(files) { + template := p == "memory-bank/flows/templates/process/priming.yaml" + if !template && !(path.Dir(p) == "memory-bank/flows/priming" && strings.HasSuffix(p, ".yaml")) { + continue + } + var tree yaml.Node + if err := yaml.Unmarshal(files[p], &tree); err != nil { + return err + } + var ordinary func(*yaml.Node) bool + ordinary = func(n *yaml.Node) bool { + if n.Kind == yaml.AliasNode || n.Anchor != "" || n.Style&yaml.TaggedStyle != 0 { + return false + } + for _, child := range n.Content { + if !ordinary(child) { + return false + } + } + return true + } + if !ordinary(&tree) { + return errors.New("priming aliases, anchors and tags are unsupported") + } + var raw map[string]any + decoder := yaml.NewDecoder(bytes.NewReader(files[p])) + decoder.KnownFields(true) + if err := decoder.Decode(&raw); err != nil { + return fmt.Errorf("%s: %w", p, err) + } + if err := decoder.Decode(&struct{}{}); err != io.EOF { + return fmt.Errorf("%s: trailing YAML", p) + } + if len(raw) != 3 || raw["version"] != 1 { + return fmt.Errorf("%s: invalid priming schema", p) + } + for _, key := range []string{"version", "process", "stages"} { + if _, ok := raw[key]; !ok { + return fmt.Errorf("%s: priming field missing", p) + } + } + process, ok := raw["process"].(string) + if !ok || (!template && !primingIdentifier.MatchString(process)) { + return errors.New("invalid priming process") + } + if !template { + if processes[process] { + return errors.New("duplicate priming process") + } + processes[process] = true + } + stages, ok := raw["stages"].(map[string]any) + if !ok || len(stages) == 0 { + return errors.New("priming stages must be nonempty mapping") + } + for stage, value := range stages { + if !primingIdentifier.MatchString(stage) { + return errors.New("invalid priming stage") + } + values, ok := value.([]any) + if !ok || len(values) == 0 { + return errors.New("priming inputs must be nonempty list") + } + seen := map[string]bool{} + for _, value := range values { + input, ok := value.(string) + if !ok || input == "" || seen[input] { + return errors.New("invalid or duplicate priming input") + } + seen[input] = true + if primingExternal.MatchString(input) { + continue + } + if !strings.HasPrefix(input, "memory-bank/") || strings.Contains(input, "\\") || strings.Contains(strings.ToUpper(input), "TODO") || strings.Contains(input, "**") || strings.ContainsAny(input, "?[]{}") || path.Clean(input) != input || strings.Contains(input, "/../") { + return fmt.Errorf("unsafe priming input %s", input) + } + if template { + continue + } + remaining := primingPlaceholder.ReplaceAllString(input, "") + if strings.ContainsAny(remaining, "<>") { + return errors.New("invalid priming placeholder") + } + found := false + if strings.Contains(input, "<") { + prefix := strings.SplitN(input, "<", 2)[0] + for candidate := range files { + if strings.HasPrefix(candidate, prefix) { + found = true + break + } + } + } else if strings.Contains(input, "*") { + for candidate := range files { + match, err := path.Match(input, candidate) + if err != nil { + return err + } + if match { + found = true + break + } + } + } else { + _, found = files[input] + } + if !found { + return fmt.Errorf("%s: unresolved priming input %s", p, input) + } + } + } + } + return nil +} diff --git a/internal/contracts/priming_test.go b/internal/contracts/priming_test.go new file mode 100644 index 0000000..638c5da --- /dev/null +++ b/internal/contracts/priming_test.go @@ -0,0 +1,21 @@ +package contracts + +import "testing" + +func TestPrimingPreflightChecksWholeInventory(t *testing.T) { + p := "memory-bank/flows/priming/test.yaml" + good := map[string][]byte{p: []byte("version: 1\nprocess: test\nstages:\n entry:\n - memory-bank/dna/*.md\n"), "memory-bank/dna/README.md": []byte("index")} + if err := ValidatePriming(good); err != nil { + t.Fatal(err) + } + for _, input := range []string{"memory-bank/missing.md", "memory-bank/../outside.md", "memory-bank/dna/**", "memory-bank//x.md"} { + bad := map[string][]byte{} + for k, v := range good { + bad[k] = v + } + bad[p] = []byte("version: 1\nprocess: test\nstages:\n entry:\n - " + input + "\n") + if err := ValidatePriming(bad); err == nil { + t.Fatalf("invalid input accepted: %s", input) + } + } +} diff --git a/internal/contracts/registry.go b/internal/contracts/registry.go new file mode 100644 index 0000000..d8d5f03 --- /dev/null +++ b/internal/contracts/registry.go @@ -0,0 +1,301 @@ +package contracts + +import ( + "errors" + "fmt" + "regexp" + "strings" +) + +var documentIDPattern = regexp.MustCompile(`^doc-[0-9a-f]{64}$`) + +type Record struct { + ID string `json:"id"` + Path string `json:"path"` + Type string `json:"type"` + ContextRoot string `json:"context_root"` + ContractID string `json:"contract_id"` + BundleDigest string `json:"bundle_digest"` +} + +func (r Record) Identity() Identity { return Identity{r.ID, r.Path, r.Type, r.ContextRoot} } + +type Selector struct { + ID string `json:"id"` + SourceRef string `json:"source_ref"` + Type string `json:"type"` + ContractID string `json:"contract_id"` + BundleDigest string `json:"bundle_digest"` + Snapshot []Identity `json:"snapshot"` + Exclusions []string `json:"exclusions"` +} +type Event struct { + Operation string `json:"operation"` + DocumentID string `json:"document_id"` + FromPath string `json:"from_path"` + ToPath string `json:"to_path"` + FromContract string `json:"from_contract"` + ToContract string `json:"to_contract"` + Evidence []string `json:"evidence"` +} +type Registry struct { + SchemaVersion int `json:"schema_version"` + Records []Record `json:"records"` + Selectors []Selector `json:"selectors"` + History []Event `json:"history"` +} +type Binding struct { + Identity Identity + ContractID, BundleDigest, SelectorID string +} + +func EmptyRegistry() Registry { return Registry{1, []Record{}, []Selector{}, []Event{}} } +func RegistryBytes(r Registry) ([]byte, error) { + b, e := Canonical(r) + if e != nil { + return nil, e + } + return append(b, '\n'), nil +} +func SelectorID(source, typ, contract, bundle string) string { + return "sel-" + FramedID("memory-bank/selector-id/v1", source, typ, contract, bundle) +} +func MigrationID(lock, p, content string) string { + return "doc-" + FramedID("memory-bank/document-id/v1", lock, p, content) +} +func ReadRegistry(data []byte) (Registry, error) { + var r Registry + e := Decode(data, &r) + if e != nil { + return r, e + } + canonical, e := RegistryBytes(r) + if e != nil || string(canonical) != string(data) { + return r, errors.New("registry must use canonical JSON plus LF") + } + return r, nil +} + +// ValidateRegistry checks a registry whose exact bytes have already been authenticated +// against installation.adoption_digest. CTR-01 deliberately does not require retaining +// inactive historical bundles; the trusted lock protects previously checked history. +// New writes must additionally call Catalog.ValidateTransition before appending an event. +func ValidateRegistry(r Registry, c Catalog, docs map[string]Document) (map[string]Binding, error) { + active := map[string]Binding{} + all := map[string]Identity{} + records := map[string]Record{} + snapshots := map[string]Selector{} + if r.SchemaVersion != 1 { + return nil, errors.New("unsupported adoption registry schema") + } + previous := "" + checkIdentity := func(id Identity) error { + if !documentIDPattern.MatchString(id.ID) || !DocumentPath(id.Path) || !ValidPath(id.ContextRoot) || !strings.HasPrefix(id.Path, id.ContextRoot+"/") { + return errors.New("invalid document identity or context") + } + if _, ok := c.Types[id.Type]; !ok { + return errors.New("identity type is not installed") + } + return nil + } + checkBundle := func(typ, id, digest string) error { + b, ok := c.Bundles[id] + ref := c.Manifest.Contracts[id] + if !ok || b.Type != typ || ref.Digest != digest { + return fmt.Errorf("missing or changed pinned bundle %s", id) + } + return nil + } + for _, rec := range r.Records { + if rec.ID <= previous { + return nil, errors.New("records must be uniquely ID-sorted") + } + previous = rec.ID + if err := checkIdentity(rec.Identity()); err != nil { + return nil, err + } + if err := checkBundle(rec.Type, rec.ContractID, rec.BundleDigest); err != nil { + return nil, err + } + records[rec.ID] = rec + all[rec.ID] = rec.Identity() + active[rec.ID] = Binding{rec.Identity(), rec.ContractID, rec.BundleDigest, ""} + } + previous = "" + for _, sel := range r.Selectors { + if len(sel.Snapshot) == 0 || sel.ID <= previous || sel.ID != SelectorID(sel.SourceRef, sel.Type, sel.ContractID, sel.BundleDigest) || !SortedSet(sel.Exclusions) { + return nil, errors.New("invalid selector identity or ordering") + } + previous = sel.ID + compat, ok := c.Manifest.LegacySources[sel.SourceRef] + if !ok || compat.Contracts[sel.Type] != sel.ContractID { + return nil, errors.New("unsupported selector provenance") + } + if err := checkBundle(sel.Type, sel.ContractID, sel.BundleDigest); err != nil { + return nil, err + } + if !c.Bundles[sel.ContractID].Legacy { + return nil, errors.New("selector requires compatibility bundle") + } + members := map[string]bool{} + priorID := "" + for _, id := range sel.Snapshot { + if id.ID <= priorID || id.Type != sel.Type { + return nil, errors.New("invalid selector snapshot ordering/type") + } + priorID = id.ID + if err := checkIdentity(id); err != nil { + return nil, err + } + if _, duplicate := snapshots[id.ID]; duplicate { + return nil, errors.New("identity occurs in multiple selectors") + } + members[id.ID] = true + snapshots[id.ID] = sel + if rec, ok := records[id.ID]; ok { + if rec.Type != id.Type || rec.ContextRoot != id.ContextRoot || !Contains(sel.Exclusions, id.ID) { + return nil, errors.New("multiple applicable adoption records") + } + } else { + all[id.ID] = id + } + if !Contains(sel.Exclusions, id.ID) { + active[id.ID] = Binding{id, sel.ContractID, sel.BundleDigest, sel.ID} + } + } + for _, excluded := range sel.Exclusions { + if !members[excluded] { + return nil, errors.New("exclusion is not a snapshot member") + } + if _, ok := records[excluded]; !ok { + return nil, errors.New("exclusion lacks resulting record") + } + } + } + type state struct { + path, contract, initialPath, initialOperation string + selectorTransition bool + } + states := map[string]state{} + lastMigrated := "" + migrationEnded := false + for _, event := range r.History { + if event.Operation == "migrate" { + if migrationEnded || event.DocumentID <= lastMigrated { + return nil, errors.New("migration history must be an initial ID-sorted block") + } + lastMigrated = event.DocumentID + } else { + migrationEnded = true + } + if !documentIDPattern.MatchString(event.DocumentID) || !SortedSet(event.Evidence) || !DocumentPath(event.ToPath) || !ValidContractID(event.ToContract) { + return nil, errors.New("invalid history event") + } + if _, ok := all[event.DocumentID]; !ok { + return nil, errors.New("history refers to unknown identity") + } + prior, exists := states[event.DocumentID] + if !exists { + if event.FromContract != "" { + return nil, errors.New("initial event has a previous contract") + } + switch event.Operation { + case "create": + if event.FromPath != "" { + return nil, errors.New("create has a previous path") + } + case "adopt", "migrate": + if event.FromPath != event.ToPath { + return nil, errors.New("initial binding path mismatch") + } + default: + return nil, errors.New("history has no initial binding event") + } + prior.initialPath = event.ToPath + prior.initialOperation = event.Operation + sel, inSelector := snapshots[event.DocumentID] + if (event.Operation == "migrate") != inSelector || (inSelector && event.ToContract != sel.ContractID) { + return nil, errors.New("migration history does not match snapshot") + } + } else { + if event.FromPath != prior.path || event.FromContract != prior.contract { + return nil, errors.New("history continuity broken") + } + switch event.Operation { + case "move": + if event.ToPath == prior.path || event.ToContract != prior.contract { + return nil, errors.New("invalid move event") + } + case "transition": + oldBundle, oldAvailable := c.Bundles[event.FromContract] + newBundle, newAvailable := c.Bundles[event.ToContract] + if ((oldAvailable && oldBundle.TransitionEvidence) || (newAvailable && newBundle.TransitionEvidence)) && len(event.Evidence) == 0 { + return nil, errors.New("transition history lacks required evidence") + } + if event.ToPath != prior.path || event.ToContract == prior.contract { + return nil, errors.New("invalid transition event") + } + if sel, ok := snapshots[event.DocumentID]; ok && event.FromContract == sel.ContractID { + prior.selectorTransition = true + } + default: + return nil, errors.New("duplicate initial or unsupported event") + } + } + id := all[event.DocumentID] + if !strings.HasPrefix(event.ToPath, id.ContextRoot+"/") { + return nil, errors.New("event escaped identity context") + } + prior.path = event.ToPath + prior.contract = event.ToContract + states[event.DocumentID] = prior + } + paths := map[string]string{} + for id, binding := range active { + current, ok := states[id] + if !ok || current.path != binding.Identity.Path || current.contract != binding.ContractID { + return nil, errors.New("history final binding disagrees with registry") + } + expectedRoot, err := ContextRoot(current.initialPath, binding.Identity.Type) + if err != nil || expectedRoot != binding.Identity.ContextRoot { + return nil, errors.New("identity context does not match initial binding") + } + if sel, ok := snapshots[id]; ok && Contains(sel.Exclusions, id) && !current.selectorTransition { + return nil, errors.New("selector exclusion lacks explicit transition") + } + if other, duplicate := paths[binding.Identity.Path]; duplicate && other != id { + return nil, errors.New("multiple identities share a path") + } + paths[binding.Identity.Path] = id + d, present := docs[binding.Identity.Path] + if !present || d.String("document_id") != id || d.String("document_type") != binding.Identity.Type { + return nil, errors.New("missing target or identity/type projection") + } + if binding.SelectorID == "" { + if d.String("flow_contract") != binding.ContractID { + return nil, errors.New("missing or changed contract projection") + } + } else if d.Has("flow_contract") && d.String("flow_contract") != binding.ContractID { + return nil, errors.New("selector contract projection mismatch") + } + } + for p, d := range docs { + if d.Has("document_type") { + typ := d.String("document_type") + if _, ok := c.Types[typ]; !ok { + return nil, fmt.Errorf("%s: unknown document type", p) + } + if d.Has("doc_kind") && d.String("doc_kind") != typ { + return nil, fmt.Errorf("%s: contradictory document kind", p) + } + } + if d.Has("document_id") || d.Has("flow_contract") { + id := d.String("document_id") + binding, ok := active[id] + if !ok || binding.Identity.Path != p { + return nil, fmt.Errorf("%s: orphan adoption projection", p) + } + } + } + return active, nil +} diff --git a/internal/contracts/registry_test.go b/internal/contracts/registry_test.go new file mode 100644 index 0000000..ddfad32 --- /dev/null +++ b/internal/contracts/registry_test.go @@ -0,0 +1,163 @@ +package contracts + +import ( + "strings" + "testing" +) + +func registryFixture(t *testing.T) (Registry, Catalog, map[string]Document) { + t.Helper() + m, _ := fixture() + legacyID := "legacy/f1f04de/feature/v1" + legacy, _ := LegacyBundle(legacyID, "feature") + next := legacy + next.ID = "feature/v1" + next.Legacy = false + oldHash, newHash := Digest([]byte("legacy")), Digest([]byte("new")) + m.Contracts = map[string]BundleRef{legacyID: {"legacy.json", oldHash}, next.ID: {"new.json", newHash}} + m.LegacySources[m.LegacyDefaultSourceRef] = Compatibility{"legacy-f1f04de/v1", map[string]string{"feature": legacyID}} + c := Catalog{Manifest: m, Types: map[string]DocumentType{"feature": {Type: "feature"}}, Bundles: map[string]Bundle{legacyID: legacy, next.ID: next}} + first := Identity{"doc-" + strings.Repeat("1", 64), "memory-bank/features/FT-1/brief.md", "feature", "memory-bank/features/FT-1"} + second := Identity{"doc-" + strings.Repeat("2", 64), "memory-bank/features/FT-2/brief.md", "feature", "memory-bank/features/FT-2"} + sel := Selector{SelectorID(m.LegacyDefaultSourceRef, "feature", legacyID, oldHash), m.LegacyDefaultSourceRef, "feature", legacyID, oldHash, []Identity{first, second}, []string{}} + r := EmptyRegistry() + r.Selectors = []Selector{sel} + docs := map[string]Document{} + for _, id := range []Identity{first, second} { + r.History = append(r.History, Event{"migrate", id.ID, id.Path, id.Path, "", legacyID, []string{}}) + raw, _ := Project([]byte("---\nstatus: draft\n---\n# Feature\n"), map[string]string{"document_type": "feature", "document_id": id.ID}) + docs[id.Path] = document(t, id.Path, string(raw)) + } + return r, c, docs +} +func TestSelectorTransitionAndMoveHistory(t *testing.T) { + r, c, docs := registryFixture(t) + if active, err := ValidateRegistry(r, c, docs); err != nil || len(active) != 2 { + t.Fatalf("snapshot: %v", err) + } + id := r.Selectors[0].Snapshot[0] + old := r.Selectors[0].ContractID + next := "feature/v1" + r.Selectors[0].Exclusions = []string{id.ID} + r.Records = []Record{{id.ID, id.Path, id.Type, id.ContextRoot, next, c.Manifest.Contracts[next].Digest}} + r.History = append(r.History, Event{"transition", id.ID, id.Path, id.Path, old, next, []string{"review/1"}}) + raw, _ := Project(docs[id.Path].Raw, map[string]string{"flow_contract": next}) + docs[id.Path] = document(t, id.Path, string(raw)) + active, err := ValidateRegistry(r, c, docs) + if err != nil || active[id.ID].SelectorID != "" { + t.Fatalf("transition: %v", err) + } + moved := id.ContextRoot + "/renamed.md" + r.Records[0].Path = moved + r.History = append(r.History, Event{"move", id.ID, id.Path, moved, next, next, []string{}}) + docs[moved] = document(t, moved, string(raw)) + delete(docs, id.Path) + if active, err = ValidateRegistry(r, c, docs); err != nil || active[id.ID].Identity.Path != moved { + t.Fatalf("move: %v", err) + } + r.Selectors[0].Exclusions = []string{} + if _, err = ValidateRegistry(r, c, docs); err == nil { + t.Fatal("accepted selector plus record without exclusion") + } +} +func TestRegistryTamperingFails(t *testing.T) { + for _, name := range []string{"missing target", "removed id", "orphan", "changed type", "history", "selector id", "missing bundle"} { + t.Run(name, func(t *testing.T) { + r, c, docs := registryFixture(t) + id := r.Selectors[0].Snapshot[0] + switch name { + case "missing target": + delete(docs, id.Path) + case "removed id": + d := docs[id.Path] + delete(d.Fields, "document_id") + docs[id.Path] = d + case "orphan": + docs["memory-bank/features/FT-1/copy.md"] = docs[id.Path] + case "changed type": + d := docs[id.Path] + d.Fields["document_type"] = "adr" + docs[id.Path] = d + case "history": + r.History[0].ToPath = "memory-bank/features/FT-1/other.md" + case "selector id": + r.Selectors[0].ID = "sel-tampered" + case "missing bundle": + delete(c.Bundles, r.Selectors[0].ContractID) + } + if _, err := ValidateRegistry(r, c, docs); err == nil { + t.Fatal("accepted tampered state") + } + }) + } +} +func TestCanonicalRegistryAndBaseDocument(t *testing.T) { + r, c, docs := registryFixture(t) + base := document(t, "memory-bank/features/FT-3/brief.md", "---\nstatus: draft\ndocument_type: feature\n---\n# Base\n") + docs[base.Path] = base + active, err := ValidateRegistry(r, c, docs) + if err != nil || len(active) != 2 { + t.Fatalf("base joined snapshot: %v", err) + } + data, err := RegistryBytes(r) + if err != nil { + t.Fatal(err) + } + if _, err = ReadRegistry(data); err != nil { + t.Fatal(err) + } + if _, err = ReadRegistry(append([]byte(" "), data...)); err == nil { + t.Fatal("accepted noncanonical registry") + } +} + +func TestEmptySelectorRejected(t *testing.T) { + r, c, _ := registryFixture(t) + r.Selectors = r.Selectors[:1] + r.Selectors[0].Snapshot = []Identity{} + r.History = []Event{} + if _, err := ValidateRegistry(r, c, map[string]Document{}); err == nil { + t.Fatal("accepted empty selector group") + } +} +func TestTransitionReplayRequiresAvailableBundleEvidence(t *testing.T) { + r, c, docs := registryFixture(t) + id := r.Selectors[0].Snapshot[0] + next := "feature/v1" + bundle := c.Bundles[next] + bundle.TransitionEvidence = true + c.Bundles[next] = bundle + r.Selectors[0].Exclusions = []string{id.ID} + r.Records = []Record{{id.ID, id.Path, id.Type, id.ContextRoot, next, c.Manifest.Contracts[next].Digest}} + r.History = append(r.History, Event{"transition", id.ID, id.Path, id.Path, r.Selectors[0].ContractID, next, []string{}}) + raw, _ := Project(docs[id.Path].Raw, map[string]string{"flow_contract": next}) + docs[id.Path] = document(t, id.Path, string(raw)) + if _, err := ValidateRegistry(r, c, docs); err == nil { + t.Fatal("accepted transition without required evidence") + } + r.History[len(r.History)-1].Evidence = []string{"review/42"} + if _, err := ValidateRegistry(r, c, docs); err != nil { + t.Fatal(err) + } +} + +func TestRetiredHistoryIsReadableButCannotAuthorizeNewTransition(t *testing.T) { + old, c, docs := registryFixture(t) + id := old.Selectors[0].Snapshot[0] + current := "feature/v1" + retired := "retired/v1" + r := EmptyRegistry() + r.Records = []Record{{id.ID, id.Path, id.Type, id.ContextRoot, current, c.Manifest.Contracts[current].Digest}} + r.History = []Event{{"adopt", id.ID, id.Path, id.Path, "", retired, []string{}}, {"transition", id.ID, id.Path, id.Path, retired, current, []string{}}} + raw, _ := Project(docs[id.Path].Raw, map[string]string{"flow_contract": current}) + docs = map[string]Document{id.Path: document(t, id.Path, string(raw))} + if _, err := ValidateRegistry(r, c, docs); err != nil { + t.Fatalf("required an inactive historical bundle: %v", err) + } + if err := c.ValidateTransition(retired, current, id.Type, nil); err == nil { + t.Fatal("retired contract authorized a new transition") + } + if err := c.ValidateTransition(current, retired, id.Type, nil); err == nil { + t.Fatal("unknown target authorized a new transition") + } +} diff --git a/internal/contracts/relocate.go b/internal/contracts/relocate.go new file mode 100644 index 0000000..b6eba93 --- /dev/null +++ b/internal/contracts/relocate.go @@ -0,0 +1,321 @@ +package contracts + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "net/url" + "path" + "path/filepath" + "regexp" + "sort" + "strings" + + "gopkg.in/yaml.v3" +) + +var absoluteURIScheme = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9+.-]*:`) + +type textReplacement struct { + start, end int + value []byte +} + +// RelocateBaseDocument preserves the resolved target of supported Markdown and +// derived_from references while instantiating a base template at another path. +// It changes only reference tokens, never the source file or unrelated text. +func RelocateBaseDocument(data []byte, from, to string) ([]byte, error) { + if !ValidPath(from) || !ValidPath(to) { + return nil, errors.New("unsafe template relocation path") + } + if path.Dir(from) == path.Dir(to) { + return append([]byte{}, data...), nil + } + d, err := ParseDocument(from, data) + if err != nil { + return nil, err + } + relocate := func(ref string) (string, error) { + if strings.ContainsAny(ref, "\\\r\n") || referenceEntity.MatchString(ref) { + return "", errors.New("escaped references are unsupported") + } + if ref == "" || strings.HasPrefix(ref, "/") || strings.HasPrefix(ref, "#") || strings.HasPrefix(ref, "https://") || strings.HasPrefix(ref, "http://") || strings.HasPrefix(ref, "mailto:") { + return ref, nil + } + if absoluteURIScheme.MatchString(ref) { + return "", errors.New("unsupported absolute URI scheme; use lowercase http, https or mailto") + } + bare, suffix := ref, "" + if i := strings.IndexAny(ref, "?#"); i >= 0 { + bare, suffix = ref[:i], ref[i:] + } + decoded, e := url.PathUnescape(bare) + if e != nil || strings.HasPrefix(decoded, "/") || strings.ContainsAny(decoded, "\\\r\n") { + return "", errors.New("unsupported encoded relative destination") + } + target := path.Clean(path.Join(path.Dir(from), decoded)) + if !ValidPath(target) { + return "", fmt.Errorf("template reference escapes repository: %s", ref) + } + rel, e := filepath.Rel(filepath.FromSlash(path.Dir(to)), filepath.FromSlash(target)) + if e != nil { + return "", e + } + rel = filepath.ToSlash(rel) + if strings.HasSuffix(bare, "/") && !strings.HasSuffix(rel, "/") { + rel += "/" + } + parts := strings.Split(rel, "/") + for i := range parts { + parts[i] = url.PathEscape(parts[i]) + } + return strings.Join(parts, "/") + suffix, nil + } + changes := []textReplacement{} + // Node positions select the exact scalar token; aliases, folded scalars and + // unfamiliar derived_from shapes fail instead of rewriting a YAML subtree. + if nodes, ok := d.keys["derived_from"]; ok { + var visit func(*yaml.Node) error + visit = func(node *yaml.Node) error { + if node.Anchor != "" || node.Style&yaml.TaggedStyle != 0 { + return errors.New("dependency anchors and tags are unsupported") + } + switch node.Kind { + case yaml.SequenceNode: + for _, child := range node.Content { + if child.Kind != yaml.ScalarNode && child.Kind != yaml.MappingNode { + return errors.New("unsupported nested dependency sequence") + } + if e := visit(child); e != nil { + return e + } + } + case yaml.MappingNode: + found := false + for i := 0; i < len(node.Content); i += 2 { + if node.Content[i].Value != "path" && node.Content[i].Value != "fit" { + return errors.New("unsupported dependency field") + } + if node.Content[i].Value == "path" { + if node.Content[i+1].Kind != yaml.ScalarNode { + return errors.New("dependency path must be a string") + } + if found { + return errors.New("duplicate dependency path") + } + found = true + if e := visit(node.Content[i+1]); e != nil { + return e + } + } + } + if !found { + return errors.New("dependency mapping lacks path") + } + case yaml.ScalarNode: + if node.Tag != "!!str" || node.Anchor != "" || node.Style&(yaml.TaggedStyle|yaml.LiteralStyle|yaml.FoldedStyle) != 0 { + return errors.New("unsupported dependency scalar") + } + value, e := relocate(node.Value) + if e != nil { + return e + } + if value == node.Value { + return nil + } + start, end, e := yamlScalarRange(data, node) + if e != nil { + return e + } + b, _ := json.Marshal(value) + changes = append(changes, textReplacement{start, end, b}) + default: + return errors.New("unsupported dependency alias or node") + } + return nil + } + if err = visit(nodes[1]); err != nil { + return nil, err + } + } + masked := maskMarkdownCode(d.Body) + bodyOffset := len(data) - len(d.Body) + for _, pattern := range []*regexp.Regexp{draftInlineReference, draftReferenceDefinition} { + matches := pattern.FindAllSubmatchIndex(masked, -1) + for _, match := range matches { + start, end := match[2], match[3] + if start < 0 { + start, end = match[4], match[5] + } + ref := string(d.Body[start:end]) + value, e := relocate(ref) + if e != nil { + return nil, e + } + if value != ref { + changes = append(changes, textReplacement{bodyOffset + start, bodyOffset + end, []byte(value)}) + } + for i := match[0]; i < match[1]; i++ { + masked[i] = ' ' + } + } + } + if draftReferenceSyntax.Match(masked) || draftHTML.Match(draftAutolink.ReplaceAll(masked, nil)) { + return nil, errors.New("unsupported template reference syntax") + } + sort.Slice(changes, func(i, j int) bool { return changes[i].start < changes[j].start }) + result := []byte{} + offset := 0 + for _, change := range changes { + if change.start < offset { + return nil, errors.New("overlapping template references") + } + result = append(result, data[offset:change.start]...) + result = append(result, change.value...) + offset = change.end + } + return append(result, data[offset:]...), nil +} +func yamlScalarRange(data []byte, node *yaml.Node) (int, int, error) { + lines := bytes.SplitAfter(data, []byte("\n")) + if node.Line < 1 || node.Line >= len(lines) { + return 0, 0, errors.New("dependency position outside frontmatter") + } + offset := 0 + for _, line := range lines[:node.Line] { + offset += len(line) + } + runes := []rune(string(lines[node.Line])) + if node.Column < 1 || node.Column > len(runes) { + return 0, 0, errors.New("invalid dependency column") + } + offset += len(string(runes[:node.Column-1])) + end := offset + if node.Style == 0 { + end += len(node.Value) + if end > len(data) || string(data[offset:end]) != node.Value { + return 0, 0, errors.New("multiline dependency is unsupported") + } + } else { + quote := data[offset] + if quote != '\'' && quote != '"' { + return 0, 0, errors.New("unsupported dependency quoting") + } + end++ + for end < len(data) { + if data[end] == '\n' || data[end] == '\r' { + return 0, 0, errors.New("multiline dependency is unsupported") + } + if quote == '"' && data[end] == '\\' { + end += 2 + continue + } + if data[end] == quote { + if quote == '\'' && end+1 < len(data) && data[end+1] == quote { + end += 2 + continue + } + end++ + break + } + end++ + } + } + if end > len(data) { + return 0, 0, errors.New("unterminated dependency") + } + var value string + if e := yaml.Unmarshal(data[offset:end], &value); e != nil || value != node.Value { + return 0, 0, errors.New("dependency token mismatch") + } + return offset, end, nil +} + +// Masking keeps byte offsets intact, including CRLF. Examples inside code and +// comments are not navigation references and must remain verbatim. +func maskMarkdownCode(data []byte) []byte { + out := append([]byte{}, data...) + fence := byte(0) + fenceLength := 0 + comment := false + offset := 0 + blank := func(start, end int) { + for i := start; i < end; i++ { + if out[i] != '\n' && out[i] != '\r' { + out[i] = ' ' + } + } + } + for _, line := range bytes.SplitAfter(data, []byte("\n")) { + trim := strings.TrimSpace(string(line)) + run := 0 + var ch byte + if len(trim) > 0 { + ch = trim[0] + if ch == '`' || ch == '~' { + for run < len(trim) && trim[run] == ch { + run++ + } + } + } + if fence != 0 { + blank(offset, offset+len(line)) + if ch == fence && run >= fenceLength && strings.TrimSpace(trim[run:]) == "" { + fence = 0 + } + offset += len(line) + continue + } + if !comment && run >= 3 && (ch != '`' || !strings.Contains(trim[run:], "`")) { + fence = ch + fenceLength = run + blank(offset, offset+len(line)) + offset += len(line) + continue + } + if !comment && (bytes.HasPrefix(line, []byte(" ")) || bytes.HasPrefix(line, []byte("\t"))) { + blank(offset, offset+len(line)) + offset += len(line) + continue + } + for i := 0; i < len(line); { + if comment { + end := bytes.Index(line[i:], []byte("-->")) + if end < 0 { + blank(offset+i, offset+len(line)) + break + } + blank(offset+i, offset+i+end+3) + i += end + 3 + comment = false + continue + } + start := bytes.Index(line[i:], []byte("