feat: standalone inspect and migrate

.github/workflows/ci.yml

@@ -209,6 +209,8 @@ jobs:
       - uses: actions/setup-python@v5
         with:
           python-version: "3.12"
-      - run: pip install duckdb 'psycopg[binary]'
+      - run: pip install -r harness/requirements-execute.txt
+      - name: verify execution comparisons and isolation
+        run: python -m pytest harness -q
       - name: run it through both engines and compare the rows
         run: make oracle-exec

Makefile

@@ -18,7 +18,8 @@
 # "Working on Windows" in the README.
 SQLGLOT ?= $(HOME)/opensource/sqlglot
 SERVICE ?= $(HOME)/calvinchengx/emulators/data-agent-service
-GOLANGCI = golangci/golangci-lint:v2.13.1
+GOLANGCI ?= golangci-lint
+BINARY ?= .tmp/sqlglot
 # Windows spells it `python`; POSIX distributions increasingly only ship
 # `python3`. Override rather than edit.
 PYTHON ?= python3
@@ -42,7 +43,13 @@ doctor: ## Check the toolchain
 
 build: ## Build the sqlglot CLI into .tmp/
 	@mkdir -p .tmp
-	go build -o .tmp/sqlglot ./cmd/sqlglot
+	go build -o $(BINARY) ./cmd/sqlglot
+
+build-database-tests: ## Build the disposable database integration suite
+	@mkdir -p .tmp
+	go test -c -tags integration -o .tmp/database-review.test ./database/migrate/review
+	go test -c -tags integration -o .tmp/database-sqlserver.test ./database/migrate/sqlserver
+	go test -c -o .tmp/database-bundle.test ./database/migrate
 
 install: ## Install the sqlglot CLI to GOBIN (or GOPATH/bin)
 	go install ./cmd/sqlglot
@@ -111,15 +118,15 @@ coverage: test ## Per-dialect coverage, against the reference and against the se
 	@$(PYTHON) -c 'import json; c=json.load(open("testdata/service/coverage.json")); print("data agent service:"); [print("  %-24s %3d/%d" % (k, v["parsed"], v["total"])) for k, v in sorted(c["by_category"].items())]'
 	@$(PYTHON) -c 'import json; c=json.load(open("testdata/coverage.json")); print("reference %s  %d/%d" % (c["reference"][:12], c["matched"], c["total"])); [print("  %-10s %4d/%-4d unparsed %4d mismatched %d" % (d, v["matched"], v["total"], v["unparsed"], v["mismatched"])) for d, v in sorted(c["by_dialect"].items())]'
 
-lint: ## gofmt, vet and golangci-lint, in a container
+lint: ## gofmt, vet and native golangci-lint (v2.13.1)
 	@unformatted=$$(gofmt -l .); \
 	if [ -n "$$unformatted" ]; then \
 	  echo "not gofmt'd:"; echo "$$unformatted"; \
 	  echo "CI regenerates and diffs, so it fails there instead; run gofmt -w"; \
 	  exit 1; \
 	fi
 	go vet ./...
-	docker run --rm -v "$$(pwd):/src" -w /src $(GOLANGCI) golangci-lint run ./...
+	$(GOLANGCI) run ./...
 
 cover: ## Test coverage of the port
 	@go test ./... -coverpkg=./sqlglot/ -coverprofile=cover.out >/dev/null

README.md

@@ -217,6 +217,8 @@ worked around. PostgreSQL is skipped without `PGDSN`; CI supplies it as a
 service container and `make postgres` starts one locally. An engine it cannot
 reach is skipped with a note, not a failure.
 
+Install the execution harness dependencies with `python -m pip install -r harness/requirements-execute.txt`. The oracle uses DuckDB 1.4.4 and PostgreSQL 17 so engine upgrades cannot silently change its coverage baseline. Each statement runs in a rolled-back transaction to preserve the fixture tables and rows for subsequent comparisons. Comparisons preserve duplicate counts and explicit `ORDER BY`; unordered results are compared as multisets. Run `python -m pytest harness -q` to verify isolation and comparison semantics.
+
 Most of the corpus is a transpiler's test suite rather than a workload: it
 says `SELECT x FROM t` and never creates `t`. `testdata/fixtures/schema.sql`
 supplies those tables, and their names are not invented — they are harvested
@@ -282,6 +284,8 @@ make postgres   # start a PostgreSQL for it, and print the DSN to export
 
 The [migration packages](migrate/README.md) compare typed SQL Server snapshots and generate ordered DDL without a database connection. They also expose an Atlas adapter used by commons-db. Run `go run ./examples/sqlserver-migration` for a planning example that prints SQL without executing it.
 
+The [standalone database packages](database/README.md) own live schema inspection, HCL/script migration engines, and reviewed PostgreSQL, SQLite and SQL Server execution. The CLI exposes `schema inspect`, `migrate plan` and `migrate apply`. commons-db retains compatible APIs through shims. Run `go run ./examples/database` for a disposable SQLite plan/apply example.
+
 ## T-SQL Extended Events sessions
 
 `CREATE`, `ALTER` and `DROP EVENT SESSION` have no grammar in the reference, which warns and keeps each as a `Command` (the port refused CREATE and DROP outright). Under `tsql` the port reads them into nodes of its own, so a caller can build the statement as a tree and let `Generate` do the quoting: a session name is an `Identifier` (`[a]]b]` when quoted), a file path is a `National` or `Literal` (`N'it''s.xel'`), and a predicate is an ordinary condition. No statement in the reference corpus has this shape, so no compared tree changes; ALTER EVENT SESSION is the one place the port now returns a tree where the reference returns a `Command`, which is the same choice the T-SQL routine bodies made. Any other form is still refused.
@@ -343,9 +347,8 @@ ddl, err := sqlglot.Generate(session, "tsql")
 
 ## Working on Windows
 
-The port is pure Go with no dependencies, and the tests need nothing but the Go
-toolchain -- the expectations are committed, so there is no Python to install,
-no native library, nothing to fetch. CI runs the full differential on **Linux,
+The parser is pure Go; the database packages use pure Go drivers. The tests need the Go
+toolchain and module dependencies -- the expectations are committed, so there is no Python or native library to install. CI runs the full differential on **Linux,
 macOS and Windows, amd64 and arm64**, and it invokes `go` directly rather than
 `make`, so all five platforms are proven by the same commands you would type:
 
@@ -361,9 +364,7 @@ and a POSIX shell, so on Windows they want WSL or Git Bash. Nothing is lost by
 skipping it: `make test` is `go test ./...`, and `make gaps` is the last command
 above with the file-and-line prefix stripped.
 
-The remaining targets -- `oracle`, `service`, `lint` -- regenerate the committed
-expectations and the generated tables, or lint in a container. Those need
-Python, a checkout of the pinned sqlglot, and Docker. The asymmetry is
+The `oracle` and `service` targets regenerate committed expectations and generated tables and need Python and the pinned reference checkout. `lint` uses native golangci-lint v2.13.1; disposable database integration uses Docker. The asymmetry is
 deliberate: **contributing to the port needs a Go toolchain; maintaining the
 oracle needs more.** A contributor on Windows is held to exactly the same
 differential as everyone else without installing any of it.

cmd/sqlglot/database.go

@@ -0,0 +1,136 @@
+package main
+
+import (
+	"context"
+	"database/sql"
+	"encoding/json"
+	"errors"
+	"fmt"
+	_ "github.com/ClickHouse/clickhouse-go/v2"
+	_ "github.com/go-sql-driver/mysql"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strings"
+
+	inspection "github.com/flanksource/sqlglot-go/database/inspect"
+	"github.com/flanksource/sqlglot-go/database/inspect/snapshot"
+	"github.com/flanksource/sqlglot-go/database/migrate/review"
+	"github.com/flanksource/sqlglot-go/database/target"
+	"github.com/spf13/cobra"
+)
+
+type databaseFlags struct {
+	dsn     string
+	dialect string
+	schema  string
+}
+
+func (f *databaseFlags) bind(cmd *cobra.Command) {
+	cmd.Flags().StringVar(&f.dsn, "dsn", "", "Database connection string")
+	cmd.Flags().StringVar(&f.dialect, "driver", "", "Database driver (defaults to connection dialect)")
+	cmd.Flags().StringVar(&f.schema, "schema", "", "Schema to inspect or migrate (defaults to main, public or dbo)")
+}
+
+func (f *databaseFlags) open(ctx context.Context, readOnly bool) (*sql.DB, error) {
+	if strings.TrimSpace(f.dsn) == "" {
+		return nil, fmt.Errorf("database dsn is required")
+	}
+	dsn := f.dsn
+	f.dialect = inspection.NormalizeDriver(f.dialect)
+	if f.dialect == "" && strings.HasPrefix(dsn, "clickhouse://") {
+		f.dialect = "clickhouse"
+	}
+	if f.dialect == "" || f.dialect == "sqlite" || f.dialect == "sqlserver" || f.dialect == "postgres" {
+		t, err := dbtarget.Parse(dsn)
+		if err != nil {
+			return nil, err
+		}
+		if f.dialect != "" && f.dialect != string(t.Dialect) {
+			return nil, fmt.Errorf("driver %s does not match the connection dialect %s", f.dialect, t.Dialect)
+		}
+		f.dialect, dsn = string(t.Dialect), t.DSN
+	}
+	if f.dialect == "sqlite" {
+		u, err := url.Parse(dsn)
+		if err != nil {
+			return nil, err
+		}
+		q := u.Query()
+		q.Del("_pragma")
+		if readOnly {
+			q.Set("mode", "ro")
+		} else {
+			q.Set("mode", "rw")
+		}
+		u.RawQuery = q.Encode()
+		dsn = u.String()
+	}
+	if f.schema == "" {
+		switch f.dialect {
+		case "sqlite":
+			f.schema = "main"
+		case "sqlserver":
+			f.schema = "dbo"
+		default:
+			f.schema = "public"
+		}
+	}
+	db, err := sql.Open(f.dialect, dsn)
+	if err != nil {
+		return nil, err
+	}
+	if err := db.PingContext(ctx); err != nil {
+		return nil, errors.Join(err, db.Close())
+	}
+	return db, nil
+}
+
+func addDatabaseCommands(root *cobra.Command) {
+	schemaCmd := &cobra.Command{Use: "schema", Short: "Inspect database schemas"}
+	flags := &databaseFlags{}
+	var fullSnapshot bool
+	inspectCmd := &cobra.Command{Use: "inspect", Short: "Read a catalog or migration snapshot", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, _ []string) (returnErr error) {
+		db, err := flags.open(cmd.Context(), true)
+		if err != nil {
+			return err
+		}
+		defer func() { returnErr = errors.Join(returnErr, db.Close()) }()
+		var result any
+		if fullSnapshot && flags.dialect == "sqlserver" {
+			result, err = snapshot.Inspect(cmd.Context(), db, snapshot.Options{Schemas: []string{flags.schema}})
+		} else if fullSnapshot {
+			result, err = review.Inspect(cmd.Context(), db, flags.dialect, flags.schema)
+		} else {
+			result, err = inspection.Inspect(cmd.Context(), db, flags.dialect, inspection.Limits{})
+		}
+		if err != nil {
+			return err
+		}
+		return writeDatabaseJSON(cmd, "", result)
+	}}
+	flags.bind(inspectCmd)
+	inspectCmd.Flags().BoolVar(&fullSnapshot, "snapshot", false, "Return a complete migration snapshot for the selected schema")
+	schemaCmd.AddCommand(inspectCmd)
+	migrateCmd := &cobra.Command{Use: "migrate", Short: "Plan and apply reviewed database migrations"}
+	migrateCmd.AddCommand(newMigrationPlanCommand(), newMigrationApplyCommand())
+	root.AddCommand(schemaCmd, migrateCmd)
+}
+
+func writeDatabaseJSON(cmd *cobra.Command, path string, value any) (returnErr error) {
+	data, err := json.MarshalIndent(value, "", "  ")
+	if err != nil {
+		return err
+	}
+	data = append(data, '\n')
+	if path != "" {
+		root, err := os.OpenRoot(filepath.Dir(path))
+		if err != nil {
+			return err
+		}
+		defer func() { returnErr = errors.Join(returnErr, root.Close()) }()
+		return root.WriteFile(filepath.Base(path), data, 0600)
+	}
+	_, err = cmd.OutOrStdout().Write(data)
+	return err
+}

cmd/sqlglot/database_test.go

@@ -0,0 +1,66 @@
+package main
+
+import (
+	"bytes"
+	"database/sql"
+	"encoding/json"
+	"github.com/flanksource/sqlglot-go/database/migrate/review"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"github.com/zclconf/go-cty/cty"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+var _ = Describe("standalone database commands", func() {
+	It("parses typed bundle variables and rejects duplicate or invalid values", func() {
+		values, err := parseVariables([]string{`prefix="app"`, `limit=10`, `enabled=true`})
+		Expect(err).NotTo(HaveOccurred())
+		Expect(cty.ObjectVal(values).RawEquals(cty.ObjectVal(map[string]cty.Value{"prefix": cty.StringVal("app"), "limit": cty.NumberIntVal(10), "enabled": cty.BoolVal(true)}))).To(BeTrue())
+		_, err = parseVariables([]string{`prefix="app"`, `prefix="other"`})
+		Expect(err).To(MatchError(ContainSubstring("duplicate variable")))
+		_, err = parseVariables([]string{`limit=unknown`})
+		Expect(err).To(MatchError(ContainSubstring("evaluate variable limit")))
+	})
+	It("inspects, plans, applies and repeats a saved SQLite artifact", func() {
+		dir := GinkgoT().TempDir()
+		dbPath := filepath.Join(dir, "cli.db")
+		db, err := sql.Open("sqlite", dbPath)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(db.Ping()).To(Succeed())
+		Expect(db.Close()).To(Succeed())
+		desired := filepath.Join(dir, "desired.hcl")
+		Expect(os.WriteFile(desired, []byte("schema \"main\" {}\ntable \"items\" {\n schema = schema.main\n column \"id\" { type = integer }\n}"), 0600)).To(Succeed())
+		run := func(args ...string) (string, error) {
+			root := newRootCommand(strings.NewReader(""))
+			out := &bytes.Buffer{}
+			root.SetOut(out)
+			root.SetErr(out)
+			root.SetArgs(args)
+			err := root.Execute()
+			return out.String(), err
+		}
+		output, err := run("schema", "inspect", "--dsn", dbPath, "--snapshot")
+		Expect(err).NotTo(HaveOccurred())
+		Expect(output).To(ContainSubstring("\"dialect\": \"sqlite\""))
+		output, err = run("migrate", "plan", "--dsn", dbPath, "--desired", desired)
+		Expect(err).NotTo(HaveOccurred())
+		var artifact review.Artifact
+		Expect(json.Unmarshal([]byte(output), &artifact)).To(Succeed())
+		Expect(artifact.Hash).NotTo(BeEmpty())
+		artifactPath := filepath.Join(dir, "reviewed.json")
+		Expect(os.WriteFile(artifactPath, []byte(output), 0600)).To(Succeed())
+		output, err = run("migrate", "apply", "--dsn", dbPath, "--artifact", artifactPath, "--hash", artifact.Hash)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(output).To(ContainSubstring("\"outcome\": \"applied\""))
+		output, err = run("migrate", "apply", "--dsn", dbPath, "--artifact", artifactPath, "--hash", artifact.Hash)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(output).To(ContainSubstring("already-applied"))
+	})
+	It("requires an explicit reviewed hash before connecting", func() {
+		root := newRootCommand(strings.NewReader(""))
+		root.SetArgs([]string{"migrate", "apply", "--dsn", "missing.db", "--artifact", "missing.json"})
+		Expect(root.Execute()).To(MatchError(ContainSubstring("hash")))
+	})
+})

cmd/sqlglot/main.go

@@ -51,6 +51,7 @@ func newRootCommand(stdin io.Reader) *cobra.Command {
 		return version, nil
 	})
 	versionCmd.Short = "Print the CLI version"
+	addDatabaseCommands(root)
 	return root
 }