diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7505ecec..736409ac 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -209,6 +209,8 @@ jobs: - uses: actions/setup-python@v5 with: python-version: "3.12" - - run: pip install duckdb 'psycopg[binary]' + - run: pip install -r harness/requirements-execute.txt + - name: verify execution comparisons and isolation + run: python -m pytest harness -q - name: run it through both engines and compare the rows run: make oracle-exec diff --git a/Makefile b/Makefile index 1821f706..bbf33771 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,8 @@ # "Working on Windows" in the README. SQLGLOT ?= $(HOME)/opensource/sqlglot SERVICE ?= $(HOME)/calvinchengx/emulators/data-agent-service -GOLANGCI = golangci/golangci-lint:v2.13.1 +GOLANGCI ?= golangci-lint +BINARY ?= .tmp/sqlglot # Windows spells it `python`; POSIX distributions increasingly only ship # `python3`. Override rather than edit. PYTHON ?= python3 @@ -42,7 +43,13 @@ doctor: ## Check the toolchain build: ## Build the sqlglot CLI into .tmp/ @mkdir -p .tmp - go build -o .tmp/sqlglot ./cmd/sqlglot + go build -o $(BINARY) ./cmd/sqlglot + +build-database-tests: ## Build the disposable database integration suite + @mkdir -p .tmp + go test -c -tags integration -o .tmp/database-review.test ./database/migrate/review + go test -c -tags integration -o .tmp/database-sqlserver.test ./database/migrate/sqlserver + go test -c -o .tmp/database-bundle.test ./database/migrate install: ## Install the sqlglot CLI to GOBIN (or GOPATH/bin) go install ./cmd/sqlglot @@ -111,7 +118,7 @@ coverage: test ## Per-dialect coverage, against the reference and against the se @$(PYTHON) -c 'import json; c=json.load(open("testdata/service/coverage.json")); print("data agent service:"); [print(" %-24s %3d/%d" % (k, v["parsed"], v["total"])) for k, v in sorted(c["by_category"].items())]' @$(PYTHON) -c 'import json; c=json.load(open("testdata/coverage.json")); print("reference %s %d/%d" % (c["reference"][:12], c["matched"], c["total"])); [print(" %-10s %4d/%-4d unparsed %4d mismatched %d" % (d, v["matched"], v["total"], v["unparsed"], v["mismatched"])) for d, v in sorted(c["by_dialect"].items())]' -lint: ## gofmt, vet and golangci-lint, in a container +lint: ## gofmt, vet and native golangci-lint (v2.13.1) @unformatted=$$(gofmt -l .); \ if [ -n "$$unformatted" ]; then \ echo "not gofmt'd:"; echo "$$unformatted"; \ @@ -119,7 +126,7 @@ lint: ## gofmt, vet and golangci-lint, in a container exit 1; \ fi go vet ./... - docker run --rm -v "$$(pwd):/src" -w /src $(GOLANGCI) golangci-lint run ./... + $(GOLANGCI) run ./... cover: ## Test coverage of the port @go test ./... -coverpkg=./sqlglot/ -coverprofile=cover.out >/dev/null diff --git a/README.md b/README.md index 60d65019..26c5d3e1 100644 --- a/README.md +++ b/README.md @@ -217,6 +217,8 @@ worked around. PostgreSQL is skipped without `PGDSN`; CI supplies it as a service container and `make postgres` starts one locally. An engine it cannot reach is skipped with a note, not a failure. +Install the execution harness dependencies with `python -m pip install -r harness/requirements-execute.txt`. The oracle uses DuckDB 1.4.4 and PostgreSQL 17 so engine upgrades cannot silently change its coverage baseline. Each statement runs in a rolled-back transaction to preserve the fixture tables and rows for subsequent comparisons. Comparisons preserve duplicate counts and explicit `ORDER BY`; unordered results are compared as multisets. Run `python -m pytest harness -q` to verify isolation and comparison semantics. + Most of the corpus is a transpiler's test suite rather than a workload: it says `SELECT x FROM t` and never creates `t`. `testdata/fixtures/schema.sql` supplies those tables, and their names are not invented — they are harvested @@ -282,6 +284,8 @@ make postgres # start a PostgreSQL for it, and print the DSN to export The [migration packages](migrate/README.md) compare typed SQL Server snapshots and generate ordered DDL without a database connection. They also expose an Atlas adapter used by commons-db. Run `go run ./examples/sqlserver-migration` for a planning example that prints SQL without executing it. +The [standalone database packages](database/README.md) own live schema inspection, HCL/script migration engines, and reviewed PostgreSQL, SQLite and SQL Server execution. The CLI exposes `schema inspect`, `migrate plan` and `migrate apply`. commons-db retains compatible APIs through shims. Run `go run ./examples/database` for a disposable SQLite plan/apply example. + ## T-SQL Extended Events sessions `CREATE`, `ALTER` and `DROP EVENT SESSION` have no grammar in the reference, which warns and keeps each as a `Command` (the port refused CREATE and DROP outright). Under `tsql` the port reads them into nodes of its own, so a caller can build the statement as a tree and let `Generate` do the quoting: a session name is an `Identifier` (`[a]]b]` when quoted), a file path is a `National` or `Literal` (`N'it''s.xel'`), and a predicate is an ordinary condition. No statement in the reference corpus has this shape, so no compared tree changes; ALTER EVENT SESSION is the one place the port now returns a tree where the reference returns a `Command`, which is the same choice the T-SQL routine bodies made. Any other form is still refused. @@ -343,9 +347,8 @@ ddl, err := sqlglot.Generate(session, "tsql") ## Working on Windows -The port is pure Go with no dependencies, and the tests need nothing but the Go -toolchain -- the expectations are committed, so there is no Python to install, -no native library, nothing to fetch. CI runs the full differential on **Linux, +The parser is pure Go; the database packages use pure Go drivers. The tests need the Go +toolchain and module dependencies -- the expectations are committed, so there is no Python or native library to install. CI runs the full differential on **Linux, macOS and Windows, amd64 and arm64**, and it invokes `go` directly rather than `make`, so all five platforms are proven by the same commands you would type: @@ -361,9 +364,7 @@ and a POSIX shell, so on Windows they want WSL or Git Bash. Nothing is lost by skipping it: `make test` is `go test ./...`, and `make gaps` is the last command above with the file-and-line prefix stripped. -The remaining targets -- `oracle`, `service`, `lint` -- regenerate the committed -expectations and the generated tables, or lint in a container. Those need -Python, a checkout of the pinned sqlglot, and Docker. The asymmetry is +The `oracle` and `service` targets regenerate committed expectations and generated tables and need Python and the pinned reference checkout. `lint` uses native golangci-lint v2.13.1; disposable database integration uses Docker. The asymmetry is deliberate: **contributing to the port needs a Go toolchain; maintaining the oracle needs more.** A contributor on Windows is held to exactly the same differential as everyone else without installing any of it. diff --git a/cmd/sqlglot/database.go b/cmd/sqlglot/database.go new file mode 100644 index 00000000..b6b3a72a --- /dev/null +++ b/cmd/sqlglot/database.go @@ -0,0 +1,136 @@ +package main + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + _ "github.com/ClickHouse/clickhouse-go/v2" + _ "github.com/go-sql-driver/mysql" + "net/url" + "os" + "path/filepath" + "strings" + + inspection "github.com/flanksource/sqlglot-go/database/inspect" + "github.com/flanksource/sqlglot-go/database/inspect/snapshot" + "github.com/flanksource/sqlglot-go/database/migrate/review" + "github.com/flanksource/sqlglot-go/database/target" + "github.com/spf13/cobra" +) + +type databaseFlags struct { + dsn string + dialect string + schema string +} + +func (f *databaseFlags) bind(cmd *cobra.Command) { + cmd.Flags().StringVar(&f.dsn, "dsn", "", "Database connection string") + cmd.Flags().StringVar(&f.dialect, "driver", "", "Database driver (defaults to connection dialect)") + cmd.Flags().StringVar(&f.schema, "schema", "", "Schema to inspect or migrate (defaults to main, public or dbo)") +} + +func (f *databaseFlags) open(ctx context.Context, readOnly bool) (*sql.DB, error) { + if strings.TrimSpace(f.dsn) == "" { + return nil, fmt.Errorf("database dsn is required") + } + dsn := f.dsn + f.dialect = inspection.NormalizeDriver(f.dialect) + if f.dialect == "" && strings.HasPrefix(dsn, "clickhouse://") { + f.dialect = "clickhouse" + } + if f.dialect == "" || f.dialect == "sqlite" || f.dialect == "sqlserver" || f.dialect == "postgres" { + t, err := dbtarget.Parse(dsn) + if err != nil { + return nil, err + } + if f.dialect != "" && f.dialect != string(t.Dialect) { + return nil, fmt.Errorf("driver %s does not match the connection dialect %s", f.dialect, t.Dialect) + } + f.dialect, dsn = string(t.Dialect), t.DSN + } + if f.dialect == "sqlite" { + u, err := url.Parse(dsn) + if err != nil { + return nil, err + } + q := u.Query() + q.Del("_pragma") + if readOnly { + q.Set("mode", "ro") + } else { + q.Set("mode", "rw") + } + u.RawQuery = q.Encode() + dsn = u.String() + } + if f.schema == "" { + switch f.dialect { + case "sqlite": + f.schema = "main" + case "sqlserver": + f.schema = "dbo" + default: + f.schema = "public" + } + } + db, err := sql.Open(f.dialect, dsn) + if err != nil { + return nil, err + } + if err := db.PingContext(ctx); err != nil { + return nil, errors.Join(err, db.Close()) + } + return db, nil +} + +func addDatabaseCommands(root *cobra.Command) { + schemaCmd := &cobra.Command{Use: "schema", Short: "Inspect database schemas"} + flags := &databaseFlags{} + var fullSnapshot bool + inspectCmd := &cobra.Command{Use: "inspect", Short: "Read a catalog or migration snapshot", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, _ []string) (returnErr error) { + db, err := flags.open(cmd.Context(), true) + if err != nil { + return err + } + defer func() { returnErr = errors.Join(returnErr, db.Close()) }() + var result any + if fullSnapshot && flags.dialect == "sqlserver" { + result, err = snapshot.Inspect(cmd.Context(), db, snapshot.Options{Schemas: []string{flags.schema}}) + } else if fullSnapshot { + result, err = review.Inspect(cmd.Context(), db, flags.dialect, flags.schema) + } else { + result, err = inspection.Inspect(cmd.Context(), db, flags.dialect, inspection.Limits{}) + } + if err != nil { + return err + } + return writeDatabaseJSON(cmd, "", result) + }} + flags.bind(inspectCmd) + inspectCmd.Flags().BoolVar(&fullSnapshot, "snapshot", false, "Return a complete migration snapshot for the selected schema") + schemaCmd.AddCommand(inspectCmd) + migrateCmd := &cobra.Command{Use: "migrate", Short: "Plan and apply reviewed database migrations"} + migrateCmd.AddCommand(newMigrationPlanCommand(), newMigrationApplyCommand()) + root.AddCommand(schemaCmd, migrateCmd) +} + +func writeDatabaseJSON(cmd *cobra.Command, path string, value any) (returnErr error) { + data, err := json.MarshalIndent(value, "", " ") + if err != nil { + return err + } + data = append(data, '\n') + if path != "" { + root, err := os.OpenRoot(filepath.Dir(path)) + if err != nil { + return err + } + defer func() { returnErr = errors.Join(returnErr, root.Close()) }() + return root.WriteFile(filepath.Base(path), data, 0600) + } + _, err = cmd.OutOrStdout().Write(data) + return err +} diff --git a/cmd/sqlglot/database_test.go b/cmd/sqlglot/database_test.go new file mode 100644 index 00000000..bd319283 --- /dev/null +++ b/cmd/sqlglot/database_test.go @@ -0,0 +1,66 @@ +package main + +import ( + "bytes" + "database/sql" + "encoding/json" + "github.com/flanksource/sqlglot-go/database/migrate/review" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/zclconf/go-cty/cty" + "os" + "path/filepath" + "strings" +) + +var _ = Describe("standalone database commands", func() { + It("parses typed bundle variables and rejects duplicate or invalid values", func() { + values, err := parseVariables([]string{`prefix="app"`, `limit=10`, `enabled=true`}) + Expect(err).NotTo(HaveOccurred()) + Expect(cty.ObjectVal(values).RawEquals(cty.ObjectVal(map[string]cty.Value{"prefix": cty.StringVal("app"), "limit": cty.NumberIntVal(10), "enabled": cty.BoolVal(true)}))).To(BeTrue()) + _, err = parseVariables([]string{`prefix="app"`, `prefix="other"`}) + Expect(err).To(MatchError(ContainSubstring("duplicate variable"))) + _, err = parseVariables([]string{`limit=unknown`}) + Expect(err).To(MatchError(ContainSubstring("evaluate variable limit"))) + }) + It("inspects, plans, applies and repeats a saved SQLite artifact", func() { + dir := GinkgoT().TempDir() + dbPath := filepath.Join(dir, "cli.db") + db, err := sql.Open("sqlite", dbPath) + Expect(err).NotTo(HaveOccurred()) + Expect(db.Ping()).To(Succeed()) + Expect(db.Close()).To(Succeed()) + desired := filepath.Join(dir, "desired.hcl") + Expect(os.WriteFile(desired, []byte("schema \"main\" {}\ntable \"items\" {\n schema = schema.main\n column \"id\" { type = integer }\n}"), 0600)).To(Succeed()) + run := func(args ...string) (string, error) { + root := newRootCommand(strings.NewReader("")) + out := &bytes.Buffer{} + root.SetOut(out) + root.SetErr(out) + root.SetArgs(args) + err := root.Execute() + return out.String(), err + } + output, err := run("schema", "inspect", "--dsn", dbPath, "--snapshot") + Expect(err).NotTo(HaveOccurred()) + Expect(output).To(ContainSubstring("\"dialect\": \"sqlite\"")) + output, err = run("migrate", "plan", "--dsn", dbPath, "--desired", desired) + Expect(err).NotTo(HaveOccurred()) + var artifact review.Artifact + Expect(json.Unmarshal([]byte(output), &artifact)).To(Succeed()) + Expect(artifact.Hash).NotTo(BeEmpty()) + artifactPath := filepath.Join(dir, "reviewed.json") + Expect(os.WriteFile(artifactPath, []byte(output), 0600)).To(Succeed()) + output, err = run("migrate", "apply", "--dsn", dbPath, "--artifact", artifactPath, "--hash", artifact.Hash) + Expect(err).NotTo(HaveOccurred()) + Expect(output).To(ContainSubstring("\"outcome\": \"applied\"")) + output, err = run("migrate", "apply", "--dsn", dbPath, "--artifact", artifactPath, "--hash", artifact.Hash) + Expect(err).NotTo(HaveOccurred()) + Expect(output).To(ContainSubstring("already-applied")) + }) + It("requires an explicit reviewed hash before connecting", func() { + root := newRootCommand(strings.NewReader("")) + root.SetArgs([]string{"migrate", "apply", "--dsn", "missing.db", "--artifact", "missing.json"}) + Expect(root.Execute()).To(MatchError(ContainSubstring("hash"))) + }) +}) diff --git a/cmd/sqlglot/main.go b/cmd/sqlglot/main.go index c81e0c58..748df8f6 100644 --- a/cmd/sqlglot/main.go +++ b/cmd/sqlglot/main.go @@ -51,6 +51,7 @@ func newRootCommand(stdin io.Reader) *cobra.Command { return version, nil }) versionCmd.Short = "Print the CLI version" + addDatabaseCommands(root) return root } diff --git a/cmd/sqlglot/migrate.go b/cmd/sqlglot/migrate.go new file mode 100644 index 00000000..4783c618 --- /dev/null +++ b/cmd/sqlglot/migrate.go @@ -0,0 +1,285 @@ +package main + +import ( + "bytes" + "database/sql" + "encoding/json" + "errors" + "fmt" + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/hcl/v2/hclsyntax" + "github.com/zclconf/go-cty/cty" + "io" + "os" + "strings" + "time" + + "github.com/flanksource/sqlglot-go/database/inspect/snapshot" + bundle "github.com/flanksource/sqlglot-go/database/migrate" + "github.com/flanksource/sqlglot-go/database/migrate/review" + executor "github.com/flanksource/sqlglot-go/database/migrate/sqlserver" + native "github.com/flanksource/sqlglot-go/migrate" + planner "github.com/flanksource/sqlglot-go/migrate/sqlserver" + "github.com/spf13/cobra" +) + +type migrationFlags struct { + databaseFlags + bundle string + desired string + output string + artifact string + hash string + allowDrops bool + ack []string + lockTimeout time.Duration + routines string + variables []string +} + +func newMigrationPlanCommand() *cobra.Command { + f := &migrationFlags{} + cmd := &cobra.Command{Use: "plan", Short: "Generate a reviewed artifact without changing the database", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, _ []string) (returnErr error) { + if (f.bundle == "") == (f.desired == "") { + return fmt.Errorf("select exactly one of --bundle or --desired") + } + if err := f.validateBundleOptions(); err != nil { + return err + } + db, err := f.open(cmd.Context(), true) + if err != nil { + return err + } + defer func() { returnErr = errors.Join(returnErr, db.Close()) }() + result, planErr := f.plan(cmd, db) + if result != nil { + if err := writeDatabaseJSON(cmd, f.output, result); err != nil { + return err + } + } + return planErr + }} + f.bind(cmd) + cmd.Flags().StringVar(&f.bundle, "bundle", "", "Directory containing HCL and SQL migration files") + cmd.Flags().StringVar(&f.desired, "desired", "", "Desired migration snapshot JSON or backend HCL file") + cmd.Flags().StringVar(&f.output, "out", "", "Write the artifact JSON to this file") + cmd.Flags().BoolVar(&f.allowDrops, "allow-drops", false, "Include omitted tables and columns as removals in a bundle plan") + cmd.Flags().StringVar(&f.routines, "routines", "", "SQL Server routine directory within the bundle") + cmd.Flags().StringArrayVar(&f.variables, "var", nil, "Bundle variable as name=HCL-literal (repeatable)") + return cmd +} + +func (f *migrationFlags) plan(cmd *cobra.Command, db *sql.DB) (any, error) { + if f.bundle != "" { + opts := []bundle.Option{} + variables, err := parseVariables(f.variables) + if err != nil { + return nil, err + } + opts = append(opts, bundle.WithVariables(variables)) + if f.schema != "main" { + opts = append(opts, bundle.WithSchema(f.schema)) + } + if f.allowDrops { + opts = append(opts, bundle.WithDrops()) + } + if f.routines != "" { + opts = append(opts, bundle.WithRoutines(f.routines)) + } + preview, err := bundle.PreviewBundle(cmd.Context(), db, f.dialect, os.DirFS(f.bundle), opts...) + for _, warning := range preview.Warnings { + if _, writeErr := fmt.Fprintln(cmd.ErrOrStderr(), "warning:", warning); writeErr != nil { + return nil, writeErr + } + } + return preview, err + } + data, err := os.ReadFile(f.desired) + if err != nil { + return nil, err + } + if f.dialect == "sqlserver" { + var desired snapshot.Result + if err := decodeJSON(data, &desired); err != nil { + return nil, fmt.Errorf("SQL Server requires a complete snapshot JSON: %w", err) + } + current, err := snapshot.Inspect(cmd.Context(), db, snapshot.Options{Schemas: []string{f.schema}}) + if err != nil { + return nil, err + } + return planner.BuildArtifact(native.ArtifactRequest{Schemas: current.Schemas, Request: native.Request{Current: current.Schema, Desired: desired.Schema}}) + } + text := string(data) + if json.Valid(data) { + var desired review.Snapshot + if err := decodeJSON(data, &desired); err != nil { + return nil, err + } + if desired.Dialect != f.dialect || desired.Schema != f.schema { + return nil, fmt.Errorf("desired snapshot dialect or schema does not match target") + } + text = desired.HCL + } + return review.Build(cmd.Context(), db, review.Options{Dialect: f.dialect, Schema: f.schema, DesiredHCL: text}) +} + +func newMigrationApplyCommand() *cobra.Command { + f := &migrationFlags{} + cmd := &cobra.Command{Use: "apply", Short: "Apply exact reviewed SQL with hash and destructive acknowledgements", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, _ []string) (returnErr error) { + if f.hash == "" { + return fmt.Errorf("reviewed --hash is required") + } + if (f.artifact == "") == (f.bundle == "") { + return fmt.Errorf("select exactly one of --artifact or --bundle") + } + if err := f.validateBundleOptions(); err != nil { + return err + } + var data []byte + var err error + if f.artifact != "" { + data, err = os.ReadFile(f.artifact) + if err != nil { + return err + } + if err := f.validateArtifact(data); err != nil { + return err + } + } + db, err := f.open(cmd.Context(), false) + if err != nil { + return err + } + defer func() { returnErr = errors.Join(returnErr, db.Close()) }() + if f.bundle != "" { + planned, err := f.plan(cmd, db) + if err != nil { + return err + } + data, err = json.Marshal(planned) + if err != nil { + return err + } + } + result, err := f.apply(cmd, db, data) + if result != nil { + if outputErr := writeDatabaseJSON(cmd, "", result); outputErr != nil { + return outputErr + } + } + return err + }} + f.bind(cmd) + cmd.Flags().StringVar(&f.artifact, "artifact", "", "Saved reviewed artifact or bundle preview JSON") + cmd.Flags().StringVar(&f.bundle, "bundle", "", "Replan this bundle and require its hash to match the reviewed hash") + cmd.Flags().StringVar(&f.hash, "hash", "", "Explicitly reviewed artifact hash") + cmd.Flags().StringSliceVar(&f.ack, "ack", nil, "Acknowledge individual destructive step IDs (repeat or comma separate)") + cmd.Flags().BoolVar(&f.allowDrops, "allow-drops", false, "Include omitted objects when replanning a bundle") + cmd.Flags().DurationVar(&f.lockTimeout, "lock-timeout", 5*time.Second, "Maximum migration lock wait") + cmd.Flags().StringVar(&f.routines, "routines", "", "SQL Server routine directory within the bundle") + cmd.Flags().StringArrayVar(&f.variables, "var", nil, "Bundle variable as name=HCL-literal (repeatable)") + return cmd +} + +func (f *migrationFlags) apply(cmd *cobra.Command, db *sql.DB, data []byte) (any, error) { + var envelope struct { + Artifact json.RawMessage `json:"artifact"` + SQLServerArtifact json.RawMessage `json:"sqlServerArtifact"` + Version int `json:"version"` + } + if err := json.Unmarshal(data, &envelope); err != nil { + return nil, err + } + if envelope.Artifact != nil { + data = envelope.Artifact + } + if envelope.SQLServerArtifact != nil { + data = envelope.SQLServerArtifact + } + if f.dialect == "sqlserver" { + var artifact native.Artifact + if err := decodeJSON(data, &artifact); err != nil { + return nil, err + } + return executor.Apply(cmd.Context(), db, executor.Request{Artifact: artifact, Hash: f.hash, Acknowledgements: f.ack, LockTimeout: f.lockTimeout}) + } + var artifact review.Artifact + if err := decodeJSON(data, &artifact); err != nil { + return nil, err + } + if artifact.Baseline.Dialect != f.dialect || artifact.Baseline.Schema != f.schema { + return nil, fmt.Errorf("artifact dialect or schema does not match target") + } + return review.Apply(cmd.Context(), db, review.Request{Artifact: artifact, Hash: f.hash, Acknowledgements: f.ack, LockTimeout: f.lockTimeout}) +} + +func decodeJSON(data []byte, value any) error { + decoder := json.NewDecoder(bytes.NewReader(data)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(value); err != nil { + return err + } + if err := decoder.Decode(new(any)); err != io.EOF { + return fmt.Errorf("artifact must contain exactly one JSON value") + } + return nil +} + +func (f *migrationFlags) validateArtifact(data []byte) error { + var envelope struct { + Artifact json.RawMessage `json:"artifact"` + SQLServerArtifact json.RawMessage `json:"sqlServerArtifact"` + Version int `json:"version"` + } + if err := json.Unmarshal(data, &envelope); err != nil { + return err + } + if envelope.Artifact != nil { + return f.validateArtifact(envelope.Artifact) + } + if envelope.SQLServerArtifact != nil { + return f.validateArtifact(envelope.SQLServerArtifact) + } + if envelope.Version == 1 { + var a native.Artifact + if err := decodeJSON(data, &a); err != nil { + return err + } + return executor.Validate(executor.Request{Artifact: a, Hash: f.hash, Acknowledgements: f.ack, LockTimeout: f.lockTimeout}) + } + var a review.Artifact + if err := decodeJSON(data, &a); err != nil { + return err + } + return review.Validate(review.Request{Artifact: a, Hash: f.hash, Acknowledgements: f.ack, LockTimeout: f.lockTimeout}) +} + +func parseVariables(values []string) (map[string]cty.Value, error) { + result := map[string]cty.Value{} + for _, value := range values { + name, text, found := strings.Cut(value, "=") + if !found || !hclsyntax.ValidIdentifier(name) { + return nil, fmt.Errorf("variable must be name=HCL-literal") + } + if _, exists := result[name]; exists { + return nil, fmt.Errorf("duplicate variable %q", name) + } + expression, diagnostics := hclsyntax.ParseExpression([]byte(text), "variable "+name, hcl.Pos{Line: 1, Column: 1}) + if diagnostics.HasErrors() { + return nil, fmt.Errorf("parse variable %s: %s", name, diagnostics.Error()) + } + parsed, diagnostics := expression.Value(nil) + if diagnostics.HasErrors() { + return nil, fmt.Errorf("evaluate variable %s: %s", name, diagnostics.Error()) + } + result[name] = parsed + } + return result, nil +} + +func (f *migrationFlags) validateBundleOptions() error { + if f.bundle == "" && (len(f.variables) > 0 || f.routines != "" || f.allowDrops) { + return fmt.Errorf("--var, --routines and --allow-drops require --bundle") + } + return nil +} diff --git a/database/README.md b/database/README.md new file mode 100644 index 00000000..1ed30111 --- /dev/null +++ b/database/README.md @@ -0,0 +1,75 @@ +# Standalone database inspection and migrations + +These packages own the database implementation previously provided by commons-db. They import no commons-db packages. The parser and pure SQL Server planner remain independent of database drivers under `sqlglot` and `migrate`. + +| Package | Responsibility | +| --- | --- | +| `database/inspect` | Bounded, uncached PostgreSQL, SQLite, SQL Server, MySQL and ClickHouse catalogs | +| `database/capabilities` | Live backend version and feature probes | +| `database/inspect/snapshot` | Complete supported SQL Server snapshots, including modules and dependencies | +| `database/atlas/sqlserver` | SQL Server Atlas HCL codec and runtime driver | +| `database/migrate` | Existing HCL, scripts, security and provisioner API; read-only bundle preview | +| `database/migrate/review` | Reviewed PostgreSQL and SQLite schema artifacts and transactional execution | +| `database/migrate/sqlserver` | Existing SQL Server v1 artifact execution | +| `database/migrate/sqlite` | Programmatic SQLite table reconciliation | +| `database/migrate/viewdeps` | PostgreSQL dependent view capture, invalidation and restoration | + +commons-db preserves its migration and inspection APIs through aliases and forwarding functions. Its catalog cache, refresh policy and metadata remain in commons-db. Supported SQLite rebuilds are automatic; the previous `WithRebuilds` and `AllowRebuilds` names remain accepted. Column and table drops remain opt-in for bundle reconciliation. + +## CLI + +Build with `make build`. Database commands emit JSON. Set the database connection in your shell rather than writing credentials into artifacts. + +```sh +.tmp/sqlglot schema inspect --dsn "$DATABASE_URL" +.tmp/sqlglot schema inspect --dsn "$DATABASE_URL" --snapshot --schema public +.tmp/sqlglot migrate plan --dsn "$DATABASE_URL" --desired examples/database/schema.hcl --out .tmp/reviewed.json +.tmp/sqlglot migrate apply --dsn "$DATABASE_URL" --artifact .tmp/reviewed.json --hash "$REVIEWED_HASH" +``` + +`--dsn` is required. Migration connections support PostgreSQL URLs or keyword DSNs, SQL Server URLs/ADO/ODBC strings, and SQLite file paths or file URIs. `--driver` selects a catalog backend explicitly, including `mysql` with its native driver DSN and `clickhouse` with its native URL. `--schema` defaults to `public`, `dbo`, or `main`. Catalog inspection is bounded and may report truncation; `--snapshot` requests a migration snapshot and rejects unsupported features instead of truncating. + +`migrate plan` requires exactly one of `--desired` or `--bundle`. Desired input is backend HCL or a saved snapshot JSON; SQL Server uses its typed snapshot JSON. `--out` stores the JSON with owner-only permissions. Bundle directories use the existing portable HCL conventions and default to retaining omitted columns, tables and types. `--allow-drops` includes their removal in the reviewed plan. SQL Server routine files can be selected with `--routines` relative to the bundle directory. + +Supply bundle variables with repeatable `--var 'name=HCL-literal'`, for example `--var 'prefix="app"'` or `--var 'limit=10'`. Variables, routines and the drop option require a bundle input. SQLite apply opens an existing file with `mode=rw`; the reviewed executor owns its temporary connection pragmas. + +```sh +.tmp/sqlglot migrate plan --dsn "$DATABASE_URL" --bundle ./db --out .tmp/preview.json +.tmp/sqlglot migrate apply --dsn "$DATABASE_URL" --bundle ./db --hash "$REVIEWED_HASH" +.tmp/sqlglot migrate apply --dsn "$DATABASE_URL" --artifact .tmp/reviewed.json --hash "$REVIEWED_HASH" --ack "$DESTRUCTIVE_STEP_ID" +``` + +Applying a bundle replans it and requires the resulting hash to match the reviewed hash. Applying an artifact executes its saved SQL after validating it against the backend planner. `--hash` is mandatory. `--ack` accepts each required destructive ID, repeated or comma separated; missing, duplicate and unknown IDs fail. `--lock-timeout` bounds migration lock waiting and defaults to five seconds. PostgreSQL/SQLite zero selects the default; SQL Server zero means no wait. A commit error returns `commit-unknown` because loss of the commit response cannot prove rollback. + +SQLite inspection and preview open existing files in read-only mode. They do not create files or change journal mode. Bundle preview never runs pre/post scripts, creates migration metadata or creates a PostgreSQL schema. Pre-scripts are parsed with sqlglot and reported as DML, DDL, mixed or unknown, with parser completeness and diagnostics. DDL or unknown pre-scripts produce warnings, and schema planning is still attempted against the current database. Planning errors return the script analysis and a nonzero exit code. + +## Reviewed schema scope + +PostgreSQL and SQLite artifacts use version 2. They bind dialect, schema, baseline and desired HCL snapshots, ordered SQL, destructive IDs and advisory reasons to a SHA-256 hash. SQL Server retains its existing version 1 artifact and hash algorithm. All use the same CLI commands. + +PostgreSQL apply acquires a transaction-owned advisory lock before drift inspection. SQLite uses one pinned connection for pragma changes, `BEGIN IMMEDIATE`, drift inspection, exact SQL, foreign key checks, commit/rollback and pragma restoration. A current schema that matches the desired state reports `already-applied`; any other departure from the baseline is rejected. This shortcut applies only to schema artifacts. + +The PostgreSQL/SQLite reviewed path supports schema-only transactional table changes and PostgreSQL enums within one existing schema. Views, routines, user triggers, virtual tables, PostgreSQL partitioning/inheritance, domains, standalone sequences, row security and unsupported constraint states are refused. Snapshot round-trip checks reject metadata the HCL codec cannot preserve. Nontransactional plans are refused. Check expressions must compare to the backend-rendered form; unsupported normalization fails rather than accepting an unverified postcondition. + +Bundles containing scripts or security declarations produce advisory schema previews. Advisory artifacts cannot execute. Full reviewed PostgreSQL script/security phases, completion records and retry semantics are tracked separately. The existing library `migrate.Apply` retains its complete PostgreSQL script and security lifecycle; callers of that direct API are responsible for authorizing execution. + +## Library use + +`inspect.Inspect(ctx, db, driver, Limits{})` accepts a caller-owned `*sql.DB` and returns an uncached catalog. `review.Inspect` returns a supported migration snapshot. `review.Build` accepts `Options{Dialect, Schema, DesiredHCL}` and performs read-only planning. `review.Apply` requires `Request{Artifact, Hash, Acknowledgements, LockTimeout}`. None closes the caller's pool or changes its pool settings. + +Run the [SQLite example](../examples/database/main.go) with `go run ./examples/database`. It uses a disposable directory, inspects, plans, applies and repeats an artifact, then removes the directory. The adjacent `schema.hcl` is a PostgreSQL CLI example. + +## Verification + +```sh +GOWORK=off go test ./database/... ./cmd/sqlglot ./migrate/... +GOWORK=off make build +GOWORK=off make lint +GOWORK=off CGO_ENABLED=0 GOOS=linux GOARCH=amd64 make build-database-tests +bash hack/database-integration.sh postgres +bash hack/database-integration.sh sqlserver +``` + +Install golangci-lint v2.13.1 for the lint target. The integration harness uses disposable containers with no published ports, network access or host mounts, copies Linux test binaries into them and removes them on exit. It works with a remote Docker context. Build the test binaries for the Docker server architecture; the SQL Server image requires amd64. For an externally managed disposable server, run integration-tagged review tests with `SQLGLOT_TEST_POSTGRES_URL`, or SQL Server executor tests with `COMMONS_DB_SQLSERVER_ARTIFACT_TEST_URL`. These tests create and remove isolated databases. + +The harness rebuilds its test binaries automatically. Set `SQLGLOT_DOCKER_ARCH=arm64` for PostgreSQL on an ARM64 Docker server; the default is amd64. Migration advisory locks coordinate cooperating reviewed executors. Other DDL writers must be serialized with deployment execution. diff --git a/database/atlas/LICENSE.atlas b/database/atlas/LICENSE.atlas new file mode 100644 index 00000000..7a4a3ea2 --- /dev/null +++ b/database/atlas/LICENSE.atlas @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/database/atlas/NOTICE b/database/atlas/NOTICE new file mode 100644 index 00000000..9da1cf43 --- /dev/null +++ b/database/atlas/NOTICE @@ -0,0 +1,24 @@ +This directory contains code copied from Atlas (https://github.com/ariga/atlas). + + Upstream path: ariga.io/atlas/sql/internal/sqlx + Upstream version: ariga.io/atlas v0.38.0 + Copyright 2021-present The Atlas Authors. All rights reserved. + License: Apache License 2.0 (see LICENSE.atlas) + +Ported files (upstream license headers kept; local compatibility and error-handling changes): + atlas/internal/sqlx/sqlx.go + atlas/internal/sqlx/sqlx_oss.go + atlas/internal/sqlx/diff.go + atlas/internal/sqlx/plan.go + atlas/internal/sqlx/dev.go + atlas/internal/sqlx/sqlx_test.go + atlas/internal/sqlx/plan_test.go + atlas/internal/sqlx/dev_test.go + + Upstream path: ariga.io/atlas/sql/internal/specutil + Upstream version: ariga.io/atlas v0.38.0 + +Ported files (upstream license headers kept; imports, compatibility and error handling adapted): + atlas/internal/specutil/convert.go + atlas/internal/specutil/spec.go + atlas/internal/specutil/convert_test.go diff --git a/database/atlas/README.md b/database/atlas/README.md new file mode 100644 index 00000000..054b31ca --- /dev/null +++ b/database/atlas/README.md @@ -0,0 +1,16 @@ +# Atlas runtime extensions + +`sqlserver` supplies the SQL Server Atlas driver, HCL codec, inspection and connection management. Pure SQL generation remains in `migrate/atlas/sqlserver`. Raw catalog SQL lives in `database/inspect/catalog/sqlserver` and is shared with the catalog and complete snapshot inspectors. + +`internal/sqlx` and `internal/specutil` originate from Atlas v0.38.0 internal packages, which external modules cannot import. Local changes adapt imports, preserve errors and support the SQL Server codec. Retain these changes when updating the upstream baseline. Apache-2.0 headers, attribution and license are preserved in `NOTICE` and `LICENSE.atlas`. + +To inspect the upstream baseline without overwriting local changes: + +```sh +ATLAS="$(go env GOMODCACHE)/ariga.io/atlas@$(go list -m -f '{{.Version}}' ariga.io/atlas)" +diff -u "$ATLAS/sql/internal/sqlx/diff.go" database/atlas/internal/sqlx/diff.go +diff -u "$ATLAS/sql/internal/specutil/convert.go" database/atlas/internal/specutil/convert.go +GOWORK=off go test ./database/atlas/... +``` + +When updating Atlas, compare all helper files, review the local changes and update the attribution version. Run both pure planner and runtime driver tests. commons-db's `atlas/sqlserver` package is a compatibility shim and must not acquire a second driver registration. diff --git a/database/atlas/internal/specutil/convert.go b/database/atlas/internal/specutil/convert.go new file mode 100644 index 00000000..3ce7690e --- /dev/null +++ b/database/atlas/internal/specutil/convert.go @@ -0,0 +1,1454 @@ +// Copyright 2021-present The Atlas Authors. All rights reserved. +// This source code is licensed under the Apache 2.0 license found +// in the LICENSE file in the root directory of this source tree. + +package specutil + +import ( + "errors" + "fmt" + "slices" + "strconv" + "strings" + + "ariga.io/atlas/schemahcl" + "ariga.io/atlas/sql/schema" + "ariga.io/atlas/sql/sqlspec" + "github.com/flanksource/sqlglot-go/database/atlas/internal/sqlx" + "github.com/zclconf/go-cty/cty" +) + +// List of convert function types. +type ( + ConvertTableFunc func(*sqlspec.Table, *schema.Schema) (*schema.Table, error) + ConvertTableColumnFunc func(*sqlspec.Column, *schema.Table) (*schema.Column, error) + ConvertViewFunc func(*sqlspec.View, *schema.Schema) (*schema.View, error) + ConvertViewColumnFunc func(*sqlspec.Column, *schema.View) (*schema.Column, error) + ConvertTypeFunc func(*sqlspec.Column) (schema.Type, error) + ConvertPrimaryKeyFunc func(*sqlspec.PrimaryKey, *schema.Table) (*schema.Index, error) + ConvertIndexFunc func(*sqlspec.Index, *schema.Table) (*schema.Index, error) + ConvertViewIndexFunc func(*sqlspec.Index, *schema.View) (*schema.Index, error) + ConvertCheckFunc func(*sqlspec.Check) (*schema.Check, error) + ConvertFuncFunc func(*sqlspec.Func, *schema.Schema) (*schema.Func, error) + ConvertProcFunc func(*sqlspec.Func, *schema.Schema) (*schema.Proc, error) + ColumnTypeSpecFunc func(schema.Type) (*sqlspec.Column, error) + TableSpecFunc func(*schema.Table) (*sqlspec.Table, error) + TableColumnSpecFunc func(*schema.Column, *schema.Table) (*sqlspec.Column, error) + ViewSpecFunc func(*schema.View) (*sqlspec.View, error) + ViewColumnSpecFunc func(*schema.Column, *schema.View) (*sqlspec.Column, error) + PrimaryKeySpecFunc func(*schema.Index) (*sqlspec.PrimaryKey, error) + IndexSpecFunc func(*schema.Index) (*sqlspec.Index, error) + ForeignKeySpecFunc func(*schema.ForeignKey) (*sqlspec.ForeignKey, error) + CheckSpecFunc func(*schema.Check) *sqlspec.Check +) + +type ( + // ScanDoc represents a scanned HCL document. + ScanDoc struct { + Schemas []*sqlspec.Schema + Tables []*sqlspec.Table + Views []*sqlspec.View + Materialized []*sqlspec.View + Funcs []*sqlspec.Func + Procs []*sqlspec.Func + Triggers []*sqlspec.Trigger + } + + // ScanFuncs represents a set of scan functions + // used to convert the HCL document to the Realm. + ScanFuncs struct { + Table ConvertTableFunc + View ConvertViewFunc + Func ConvertFuncFunc + Proc ConvertProcFunc + // Triggers add themselves to the relevant tables/views. + Triggers func(*schema.Realm, []*sqlspec.Trigger) error + // Objects add themselves to the realm. + Objects func(*schema.Realm) error + // Optional function to extend the foreign keys. + ForeignKey func(*sqlspec.ForeignKey, *schema.ForeignKey) error + } + + // TableConvertFuncs holds the functions Table uses to convert the table's spec children. + TableConvertFuncs struct { + Column ConvertTableColumnFunc + PrimaryKey ConvertPrimaryKeyFunc + Index ConvertIndexFunc + Check ConvertCheckFunc + } + + // TableSpecFuncs holds the functions FromTable uses to convert the table's children to specs. + TableSpecFuncs struct { + Column TableColumnSpecFunc + PrimaryKey PrimaryKeySpecFunc + Index IndexSpecFunc + ForeignKey ForeignKeySpecFunc + Check CheckSpecFunc + } + + // SchemaFuncs represents a set of spec functions + // used to convert the Schema object to an HCL document. + SchemaFuncs struct { + Table TableSpecFunc + View ViewSpecFunc + Func func(*schema.Func) (*sqlspec.Func, error) + Proc func(*schema.Proc) (*sqlspec.Func, error) + } + // RefNamer is an interface for objects that can + // return their reference. + RefNamer interface { + // Ref returns the reference to the object. + Ref() *schemahcl.Ref + } + // SpecTypeNamer is an interface for objects that can + // return their spec type and name. + SpecTypeNamer interface { + SpecTyper + // SpecName returns the spec name of the object. + SpecName() string + } + // SpecTyper wraps the SpecType method. It allows objects + // to describe what their spec type is. + SpecTyper interface { + // SpecType returns the spec type of the object. + SpecType() string + } +) + +const ( + typeView = "view" + typeTable = "table" + typeColumn = "column" + typeIndex = "index" + typeSchema = "schema" + typeMaterialized = "materialized" + typeFunction = "function" + typeProcedure = "procedure" + typeTrigger = "trigger" +) + +// typeName returns the type name of the given object. +func typeName(o schema.Object) string { + attrOr := func(n string, attrs []schema.Attr) string { + for _, a := range attrs { + if t, ok := a.(SpecTyper); ok && t.SpecType() != "" { + n = t.SpecType() + } + } + return n + } + switch o := o.(type) { + case nil: + case *schema.Table: + if o == nil { + return typeTable + } + return attrOr(typeTable, o.Attrs) + case *schema.View: + switch { + case o == nil: + return typeView + case o.Materialized(): + return attrOr(typeMaterialized, o.Attrs) + default: + return attrOr(typeView, o.Attrs) + } + case *schema.Func: + if o == nil { + return typeFunction + } + return attrOr(typeFunction, o.Attrs) + case *schema.Proc: + if o == nil { + return typeProcedure + } + return attrOr(typeProcedure, o.Attrs) + default: + if ts, ok := o.(SpecTyper); ok && ts != nil && ts.SpecType() != "" { + return ts.SpecType() + } + } + return "object" +} + +// Scan populates the Realm from the schemas and table specs. +func Scan(r *schema.Realm, doc *ScanDoc, funcs *ScanFuncs) error { + st, err := newScanState(r, doc) + if err != nil { + return err + } + if err := st.scanTables(doc.Tables, funcs); err != nil { + return err + } + if err := st.scanViews(doc.Views, typeView, funcs.View); err != nil { + return err + } + if err := st.scanViews(doc.Materialized, typeMaterialized, funcs.View); err != nil { + return err + } + if err := st.scanFuncs(doc.Funcs, funcs.Func); err != nil { + return err + } + if err := st.scanProcs(doc.Procs, funcs.Proc); err != nil { + return err + } + if funcs.Triggers != nil { + if err := funcs.Triggers(r, doc.Triggers); err != nil { + return err + } + } + if funcs.Objects != nil { + if err := funcs.Objects(r); err != nil { + return err + } + } + return st.linkDeps() +} + +type scanState struct { + byName map[string]*schema.Schema + deps map[schema.Object][]*schemahcl.Ref + aliases map[string]string +} + +func newScanState(r *schema.Realm, doc *ScanDoc) (*scanState, error) { + st := &scanState{ + byName: make(map[string]*schema.Schema), + deps: make(map[schema.Object][]*schemahcl.Ref, len(doc.Views)), + aliases: make(map[string]string), + } + for _, s := range doc.Schemas { + s1 := schema.New(s.Name) + if err := convertCommentFromSpec(s, &s1.Attrs); err != nil { + return nil, err + } + schemahcl.AppendPos(&s1.Attrs, s.Range) + r.AddSchemas(s1) + st.byName[s.Name] = s1 + } + return st, nil +} + +func (st *scanState) schemaFor(kind, name string, ref *schemahcl.Ref) (*schema.Schema, error) { + n, err := SchemaName(ref) + if err != nil { + return nil, fmt.Errorf("cannot extract schema name for %s %q: %w", kind, name, err) + } + s, ok := st.byName[n] + if !ok { + return nil, fmt.Errorf("schema %q not found for %s %q", n, kind, name) + } + return s, nil +} + +func (st *scanState) alias(o schema.Object, kind string) { + if tn := typeName(o); tn != kind { + st.aliases[tn] = kind + } +} + +func (st *scanState) trackDeps(o schema.Object, kind, name string, spec Attrer) error { + d, ok := spec.Attr("depends_on") + if !ok { + return nil + } + refs, err := d.Refs() + if err != nil { + return fmt.Errorf("expect list of references for attribute %s.%s.depends_on: %w", kind, name, err) + } + st.deps[o] = refs + return nil +} + +func (st *scanState) scanTables(specs []*sqlspec.Table, funcs *ScanFuncs) error { + fks := make(map[*schema.Table][]*sqlspec.ForeignKey) + for _, spec := range specs { + s, err := st.schemaFor(typeTable, spec.Name, spec.Schema) + if err != nil { + return err + } + t, err := funcs.Table(spec, s) + if err != nil { + return fmt.Errorf("cannot convert table %q: %w", spec.Name, err) + } + st.alias(t, typeTable) + fks[t] = spec.ForeignKeys + s.AddTables(t) + if err := st.trackDeps(t, typeTable, spec.Name, spec); err != nil { + return err + } + } + for t, fks := range fks { + if err := linkForeignKeys(funcs, t, fks); err != nil { + return err + } + } + return nil +} + +func (st *scanState) scanViews(specs []*sqlspec.View, kind string, convert ConvertViewFunc) error { + for _, spec := range specs { + s, err := st.schemaFor(kind, spec.Name, spec.Schema) + if err != nil { + return err + } + v, err := convert(spec, s) + if err != nil { + return fmt.Errorf("cannot convert %s %q: %w", kind, spec.Name, err) + } + st.alias(v, kind) + if kind == typeMaterialized { + v.SetMaterialized(true) + } + s.AddViews(v) + if err := st.trackDeps(v, kind, spec.Name, spec); err != nil { + return err + } + } + return nil +} + +func (st *scanState) scanFuncs(specs []*sqlspec.Func, convert ConvertFuncFunc) error { + if convert == nil { + return nil + } + for _, spec := range specs { + s, err := st.schemaFor(typeFunction, spec.Name, spec.Schema) + if err != nil { + return err + } + f, err := convert(spec, s) + if err != nil { + return fmt.Errorf("cannot convert function %q: %w", spec.Name, err) + } + st.alias(f, typeFunction) + s.AddFuncs(f) + if err := st.trackDeps(f, typeFunction, f.Name, spec); err != nil { + return err + } + } + return nil +} + +func (st *scanState) scanProcs(specs []*sqlspec.Func, convert ConvertProcFunc) error { + if convert == nil { + return nil + } + for _, spec := range specs { + s, err := st.schemaFor(typeProcedure, spec.Name, spec.Schema) + if err != nil { + return err + } + p, err := convert(spec, s) + if err != nil { + return fmt.Errorf("cannot convert procedure %q: %w", spec.Name, err) + } + st.alias(p, typeProcedure) + s.AddProcs(p) + if err := st.trackDeps(p, typeProcedure, p.Name, spec); err != nil { + return err + } + } + return nil +} + +func (st *scanState) linkDeps() error { + for o, refs := range st.deps { + in := func(name string, ns *schema.Schema) dependsOnRefs { + return dependsOnRefs{Loc: fmt.Sprintf("%s.%s", typeName(o), name), Schema: ns, Refs: refs, Aliases: st.aliases} + } + var err error + switch o := o.(type) { + case *schema.Table: + err = fromDependsOn(o, in(o.Name, o.Schema)) + case *schema.View: + err = fromDependsOn(o, in(o.Name, o.Schema)) + case *schema.Func: + err = fromDependsOn(o, in(o.Name, o.Schema)) + case *schema.Proc: + err = fromDependsOn(o, in(o.Name, o.Schema)) + } + if err != nil { + return err + } + } + return nil +} + +// Table converts a sqlspec.Table to a schema.Table. Table conversion is done without converting +// ForeignKeySpecs into ForeignKeys, as the target tables do not necessarily exist in the schema +// at this point. Instead, the linking is done by the Schema function. +func Table(spec *sqlspec.Table, parent *schema.Schema, conv TableConvertFuncs) (*schema.Table, error) { + t := &schema.Table{ + Name: spec.Name, + Schema: parent, + } + schemahcl.AppendPos(&t.Attrs, spec.Range) + for _, cs := range spec.Columns { + c, err := conv.Column(cs, t) + if err != nil { + return nil, err + } + schemahcl.AppendPos(&c.Attrs, cs.Range) + t.AddColumns(c) + } + if spec.PrimaryKey != nil { + pk, err := conv.PrimaryKey(spec.PrimaryKey, t) + if err != nil { + return nil, err + } + schemahcl.AppendPos(&pk.Attrs, spec.PrimaryKey.Range) + t.SetPrimaryKey(pk) + } + for _, idx := range spec.Indexes { + i, err := conv.Index(idx, t) + if err != nil { + return nil, err + } + schemahcl.AppendPos(&i.Attrs, idx.Range) + t.AddIndexes(i) + } + for _, c := range spec.Checks { + ck, err := conv.Check(c) + if err != nil { + return nil, err + } + schemahcl.AppendPos(&ck.Attrs, c.Range) + t.AddChecks(ck) + } + if err := convertCommentFromSpec(spec, &t.Attrs); err != nil { + return nil, err + } + return t, nil +} + +// View converts a sqlspec.View to a schema.View. +func View(spec *sqlspec.View, parent *schema.Schema, convertC ConvertViewColumnFunc, convertI ConvertViewIndexFunc) (*schema.View, error) { + as, ok := spec.Extra.Attr("as") + if !ok { + return nil, fmt.Errorf("missing 'as' definition for view %q", spec.Name) + } + def, err := as.String() + if err != nil { + return nil, fmt.Errorf("expect string definition for attribute view.%s.as: %w", spec.Name, err) + } + v := schema.NewView(spec.Name, def).SetSchema(parent) + schemahcl.AppendPos(&v.Attrs, spec.Range) + for _, cs := range spec.Columns { + c, err := convertC(cs, v) + if err != nil { + return nil, err + } + schemahcl.AppendPos(&c.Attrs, cs.Range) + v.AddColumns(c) + } + for _, idx := range spec.Indexes { + i, err := convertI(idx, v) + if err != nil { + return nil, err + } + schemahcl.AppendPos(&i.Attrs, idx.Range) + v.AddIndexes(i) + } + if err := convertCommentFromSpec(spec, &v.Attrs); err != nil { + return nil, err + } + if c, ok := spec.Extra.Attr("check_option"); ok { + o, err := c.String() + if err != nil { + return nil, fmt.Errorf("expect string definition for attribute view.%s.check_option: %w", spec.Name, err) + } + v.SetCheckOption(o) + } + return v, nil +} + +// Column converts a sqlspec.Column into a schema.Column. +func Column(spec *sqlspec.Column, conv ConvertTypeFunc) (*schema.Column, error) { + out := &schema.Column{ + Name: spec.Name, + Type: &schema.ColumnType{ + Null: spec.Null, + }, + } + d, err := columnDefault(spec.Remain()) + if err != nil { + return nil, err + } + out.Default = d + ct, err := conv(spec) + if err != nil { + return nil, err + } + out.Type.Type = ct + if err := convertCommentFromSpec(spec, &out.Attrs); err != nil { + return nil, err + } + return out, err +} + +func columnDefault(r *schemahcl.Resource) (schema.Expr, error) { + defaultA, okA := r.Attr("default") + defaultR, okR := r.Resource("default") + switch { + case okA && okR: + return nil, errors.New("both default and default resource are set") + case okA: + v, err := Default(defaultA.V) + if err != nil { + return nil, err + } + return v, nil + case okR: + var spec struct { + Name string `spec:",name"` + As cty.Value `spec:"as"` + } + if err := defaultR.As(&spec); err != nil { + return nil, err + } + v, err := Default(spec.As) + if err != nil { + return nil, err + } + return &schema.NamedDefault{Name: spec.Name, Expr: v}, nil + default: + return nil, nil + } +} + +// Default converts a cty.Value (as defined in the spec) into a schema.Expr. +func Default(d cty.Value) (schema.Expr, error) { + if d.IsNull() { + return nil, nil // no default. + } + var x schema.Expr + switch { + case d.Type() == cty.String: + x = &schema.Literal{V: d.AsString()} + case d.Type() == cty.Number: + f := d.AsBigFloat() + // If the number is an integer, convert it to an integer. + if f.IsInt() { + x = &schema.Literal{V: f.Text('f', -1)} + } else { + x = &schema.Literal{V: f.String()} + } + case d.Type() == cty.Bool: + x = &schema.Literal{V: strconv.FormatBool(d.True())} + case d.Type().IsCapsuleType(): + raw, ok := d.EncapsulatedValue().(*schemahcl.RawExpr) + if !ok { + return nil, fmt.Errorf("invalid default value %q", d.Type().FriendlyName()) + } + x = &schema.RawExpr{X: raw.X} + default: + return nil, fmt.Errorf("unsupported type for default value: %T", d) + } + return x, nil +} + +// Index converts a sqlspec.Index to a schema.Index. The optional arguments allow +// passing functions for mutating the created index-part (e.g. add attributes). +func Index(spec *sqlspec.Index, parent *schema.Table, partFns ...func(*sqlspec.IndexPart, *schema.IndexPart) error) (*schema.Index, error) { + parts := make([]*schema.IndexPart, 0, len(spec.Columns)+len(spec.Parts)) + switch n, m := len(spec.Columns), len(spec.Parts); { + case n == 0 && m == 0: + return nil, fmt.Errorf("missing definition for index %q", spec.Name) + case n > 0 && m > 0: + return nil, fmt.Errorf(`multiple definitions for index %q, use "columns" or "on"`, spec.Name) + case n > 0: + for i, c := range spec.Columns { + c, err := ColumnByRef(parent, c) + if err != nil { + return nil, err + } + parts = append(parts, &schema.IndexPart{ + SeqNo: i, + C: c, + }) + } + case m > 0: + for i, p := range spec.Parts { + part := &schema.IndexPart{SeqNo: i, Desc: p.Desc} + switch { + case p.Column == nil && p.Expr == "": + return nil, fmt.Errorf(`"column" or "expr" are required for index %q at position %d`, spec.Name, i) + case p.Column != nil && p.Expr != "": + return nil, fmt.Errorf(`cannot use both "column" and "expr" in index %q at position %d`, spec.Name, i) + case p.Expr != "": + part.X = &schema.RawExpr{X: p.Expr} + case p.Column != nil: + c, err := ColumnByRef(parent, p.Column) + if err != nil { + return nil, err + } + part.C = c + } + for _, f := range partFns { + if err := f(p, part); err != nil { + return nil, err + } + } + parts = append(parts, part) + } + } + idx := &schema.Index{ + Name: spec.Name, + Unique: spec.Unique, + Table: parent, + Parts: parts, + } + if err := convertCommentFromSpec(spec, &idx.Attrs); err != nil { + return nil, err + } + for _, p := range idx.Parts { + if p.C != nil { + p.C.AddIndexes(idx) + } + } + return idx, nil +} + +// Check converts a sqlspec.Check to a schema.Check. +func Check(spec *sqlspec.Check) (*schema.Check, error) { + return &schema.Check{ + Name: spec.Name, + Expr: spec.Expr, + }, nil +} + +// PrimaryKey converts a sqlspec.PrimaryKey to a schema.Index. +func PrimaryKey(spec *sqlspec.PrimaryKey, parent *schema.Table) (*schema.Index, error) { + parts := make([]*schema.IndexPart, 0, len(spec.Columns)) + for seqno, c := range spec.Columns { + c, err := ColumnByRef(parent, c) + if err != nil { + return nil, nil + } + parts = append(parts, &schema.IndexPart{ + SeqNo: seqno, + C: c, + }) + } + pk := &schema.Index{ + Name: spec.Name, + Table: parent, + Parts: parts, + } + if err := convertCommentFromSpec(spec, &pk.Attrs); err != nil { + return nil, err + } + for _, p := range pk.Parts { + if p.C != nil { + p.C.AddIndexes(pk) + } + } + return pk, nil +} + +// linkForeignKeys creates the foreign keys defined in the Table's spec by creating references +// to column in the provided Schema. It is assumed that all tables referenced FK definitions in the spec +// are reachable from the provided schema or its connected realm. +func linkForeignKeys(funcs *ScanFuncs, tbl *schema.Table, fks []*sqlspec.ForeignKey) error { + for _, spec := range fks { + fk := &schema.ForeignKey{Symbol: spec.Symbol, Table: tbl} + schemahcl.AppendPos(&fk.Attrs, spec.Range) + if spec.OnUpdate != nil { + fk.OnUpdate = schema.ReferenceOption(FromVar(spec.OnUpdate.V)) + } + if spec.OnDelete != nil { + fk.OnDelete = schema.ReferenceOption(FromVar(spec.OnDelete.V)) + } + if n, m := len(spec.Columns), len(spec.RefColumns); n != m { + return fmt.Errorf("sqlspec: number of referencing and referenced columns do not match for foreign-key %q", fk.Symbol) + } + for _, ref := range spec.Columns { + c, err := ColumnByRef(tbl, ref) + if err != nil { + return err + } + fk.Columns = append(fk.Columns, c) + } + for i, ref := range spec.RefColumns { + t, c, err := externalRef(ref, tbl.Schema) + if isLocalRef(ref) { + t = fk.Table + c, err = ColumnByRef(fk.Table, ref) + } + if err != nil { + return err + } + if i > 0 && fk.RefTable != t { + return fmt.Errorf("sqlspec: more than 1 table was referenced for foreign-key %q", fk.Symbol) + } + fk.RefTable = t + fk.RefColumns = append(fk.RefColumns, c) + } + tbl.ForeignKeys = append(tbl.ForeignKeys, fk) + if funcs.ForeignKey != nil { + if err := funcs.ForeignKey(spec, fk); err != nil { + return err + } + } + } + return nil +} + +// FromSchema converts a schema.Schema into sqlspec.Schema and []sqlspec.Table. +func FromSchema(s *schema.Schema, funcs *SchemaFuncs) (*SchemaSpec, error) { + spec := &SchemaSpec{ + Schema: &sqlspec.Schema{ + Name: s.Name, + }, + Tables: make([]*sqlspec.Table, 0, len(s.Tables)), + Views: make([]*sqlspec.View, 0, len(s.Views)), + Materialized: make([]*sqlspec.View, 0, len(s.Views)), + } + for _, t := range s.Tables { + table, err := funcs.Table(t) + if err != nil { + return nil, err + } + if s.Name != "" { + table.Schema = SchemaRef(s.Name) + } + spec.Tables = append(spec.Tables, table) + spec.Triggers = append(spec.Triggers, t.Triggers...) + } + for _, v := range s.Views { + view, err := funcs.View(v) + if err != nil { + return nil, err + } + if s.Name != "" { + view.Schema = SchemaRef(s.Name) + } + if v.Materialized() { + spec.Materialized = append(spec.Materialized, view) + } else { + spec.Views = append(spec.Views, view) + } + spec.Triggers = append(spec.Triggers, v.Triggers...) + } + if funcs.Func != nil { + for _, f := range s.Funcs { + fn, err := funcs.Func(f) + if err != nil { + return nil, err + } + if s.Name != "" { + fn.Schema = SchemaRef(s.Name) + } + spec.Funcs = append(spec.Funcs, fn) + } + } + if funcs.Proc != nil { + for _, p := range s.Procs { + pr, err := funcs.Proc(p) + if err != nil { + return nil, err + } + if s.Name != "" { + pr.Schema = SchemaRef(s.Name) + } + spec.Procs = append(spec.Procs, pr) + } + } + convertCommentFromSchema(s.Attrs, &spec.Schema.Extra.Attrs) + return spec, nil +} + +// FromTable converts a schema.Table to a sqlspec.Table. +func FromTable(t *schema.Table, fns TableSpecFuncs) (*sqlspec.Table, error) { + spec := &sqlspec.Table{ + Name: t.Name, + } + for _, c := range t.Columns { + col, err := fns.Column(c, t) + if err != nil { + return nil, err + } + spec.Columns = append(spec.Columns, col) + } + if t.PrimaryKey != nil { + pk, err := fns.PrimaryKey(t.PrimaryKey) + if err != nil { + return nil, err + } + spec.PrimaryKey = pk + } + for _, idx := range t.Indexes { + i, err := fns.Index(idx) + if err != nil { + return nil, err + } + spec.Indexes = append(spec.Indexes, i) + } + for _, fk := range t.ForeignKeys { + f, err := fns.ForeignKey(fk) + if err != nil { + return nil, err + } + spec.ForeignKeys = append(spec.ForeignKeys, f) + } + for _, attr := range t.Attrs { + if c, ok := attr.(*schema.Check); ok { + spec.Checks = append(spec.Checks, fns.Check(c)) + } + } + if deps, ok := dependsOn(t.Schema.Realm, t.Deps); ok { + // Embedding a resource push its attributes to the end. + spec.Extra.Children = append(spec.Extra.Children, &schemahcl.Resource{Attrs: []*schemahcl.Attr{deps}}) + } + convertCommentFromSchema(t.Attrs, &spec.Extra.Attrs) + return spec, nil +} + +// FromView converts a schema.View to a sqlspec.View. +func FromView(v *schema.View, colFn ViewColumnSpecFunc, idxFn IndexSpecFunc) (*sqlspec.View, error) { + spec := &sqlspec.View{ + Name: v.Name, + } + for _, c := range v.Columns { + cs, err := colFn(c, v) + if err != nil { + return nil, err + } + spec.Columns = append(spec.Columns, cs) + } + for _, idx := range v.Indexes { + i, err := idxFn(idx) + if err != nil { + return nil, err + } + spec.Indexes = append(spec.Indexes, i) + } + // In case the view definition is multi-line, + // format it as indented heredoc with two spaces. + as := sqlspec.MightHeredoc(v.Def) + embed := &schemahcl.Resource{ + Attrs: []*schemahcl.Attr{ + schemahcl.StringAttr("as", as), + }, + } + if c := (schema.ViewCheckOption{}); sqlx.Has(v.Attrs, &c) { + switch strings.ToUpper(c.V) { + case schema.ViewCheckOptionNone, "": + case schema.ViewCheckOptionLocal, schema.ViewCheckOptionCascaded: + embed.Attrs = append(embed.Attrs, VarAttr("check_option", c.V)) + default: + embed.Attrs = append(embed.Attrs, schemahcl.StringAttr("check_option", c.V)) + } + } + if deps, ok := dependsOn(v.Schema.Realm, v.Deps); ok { + embed.Attrs = append(embed.Attrs, deps) + } + convertCommentFromSchema(v.Attrs, &embed.Attrs) + spec.Extra.Children = append(spec.Extra.Children, embed) + return spec, nil +} + +// dependsOn returns the depends_on attribute for the given objects. +func dependsOn(realm *schema.Realm, objects []schema.Object) (*schemahcl.Attr, bool) { + var ( + n2s = make(map[string][]*schema.Schema) + name = func(t schema.Object, n string) string { return typeName(t) + "/" + n } + ) + // Qualify references if there are objects with the same name. + if realm != nil { + for _, s := range realm.Schemas { + for _, t := range s.Tables { + n2s[name(t, t.Name)] = append(n2s[name(t, t.Name)], s) + } + for _, v := range s.Views { + n2s[name(v, v.Name)] = append(n2s[name(v, v.Name)], s) + } + for _, f := range s.Funcs { + if n := name(f, f.Name); !slices.Contains(n2s[n], s) { + n2s[n] = append(n2s[n], s) // Count overload once. + } + } + for _, p := range s.Procs { + if n := name(p, p.Name); !slices.Contains(n2s[n], s) { + n2s[n] = append(n2s[n], s) // Count overload once. + } + } + } + } + var ( + refs = make(map[string]bool, len(objects)) + deps = make([]*schemahcl.Ref, 0, len(objects)) + ) + for _, o := range objects { + path := make([]string, 0, 2) + var n, s string + switch d := o.(type) { + case *schema.Table: + n, s = d.Name, d.Schema.Name + case *schema.View: + n, s = d.Name, d.Schema.Name + case *schema.Func: + n, s = d.Name, d.Schema.Name + case *schema.Proc: + n, s = d.Name, d.Schema.Name + case RefNamer: + // If the object is a reference, add it to the depends_on list. + deps = append(deps, d.Ref()) + continue + } + if len(n2s[name(o, n)]) > 1 { + path = append(path, s) + } + if r := schemahcl.BuildRef([]schemahcl.PathIndex{ + {T: typeName(o), V: append(path, n)}, + }); !refs[r.V] { + refs[r.V] = true + deps = append(deps, r) + } + } + if len(deps) > 0 { + slices.SortFunc(deps, func(l, r *schemahcl.Ref) int { + return strings.Compare(l.V, r.V) + }) + return schemahcl.RefsAttr("depends_on", deps...), true + } + return nil, false +} + +// dependsOnRefs holds the depends_on references of an object at Loc, resolved against Schema. +type dependsOnRefs struct { + Loc string + Schema *schema.Schema + Refs []*schemahcl.Ref + Aliases map[string]string +} + +func fromDependsOn[T interface{ AddDeps(...schema.Object) T }](t T, in dependsOnRefs) error { + loc, ns, aliases := in.Loc, in.Schema, in.Aliases + for i, r := range in.Refs { + p, err := r.Path() + if err != nil { + return fmt.Errorf("extract %s.depends_on references: %w", loc, err) + } + if len(p) == 0 { + return fmt.Errorf("empty reference exists in %s.depends_on[%d]", loc, i) + } + q, n, err := RefName(r, p[0].T) + if err != nil { + return fmt.Errorf("extract %s name from %s.depends_on[%d]: %w", p[0].T, loc, i, err) + } + var o schema.Object + switch tn := p[0].T; { + case tn == typeTable, aliases[tn] == typeTable: + o, err = findT(ns, q, n, func(s *schema.Schema, name string) (*schema.Table, bool) { + if v, ok := s.Table(name); ok && typeName(v) == tn { + return v, true + } + return nil, false + }) + case tn == typeView, aliases[tn] == typeView: + o, err = findT(ns, q, n, func(s *schema.Schema, name string) (*schema.View, bool) { + if v, ok := s.View(name); ok && typeName(v) == tn { + return v, true + } + return nil, false + }) + case tn == typeMaterialized, aliases[tn] == typeMaterialized: + o, err = findT(ns, q, n, func(s *schema.Schema, name string) (*schema.View, bool) { + if v, ok := s.Materialized(name); ok && typeName(v) == tn { + return v, true + } + return nil, false + }) + case tn == typeFunction, aliases[tn] == typeFunction: + o, err = findT(ns, q, n, func(s *schema.Schema, name string) (*schema.Func, bool) { + if f, ok := s.Func(name); ok && typeName(f) == tn { + return f, true + } + return nil, false + }) + case tn == typeProcedure, aliases[tn] == typeProcedure: + o, err = findT(ns, q, n, func(s *schema.Schema, name string) (*schema.Proc, bool) { + if f, ok := s.Proc(name); ok && typeName(f) == tn { + return f, true + } + return nil, false + }) + default: + if o, err = findT(ns, q, n, func(s *schema.Schema, name string) (schema.Object, bool) { + return s.Object(func(o schema.Object) bool { + if o, ok := o.(SpecTypeNamer); ok { + return p[0].T == o.SpecType() && name == o.SpecName() + } + return false + }) + }); err != nil { + continue // Custom objects might be loaded in a different pass. + } + } + if err != nil { + return fmt.Errorf("find %s reference for %s.depends_on[%d]: %w", p[0].T, loc, i, err) + } + t.AddDeps(o) + } + return nil +} + +// FromPrimaryKey converts schema.Index to a sqlspec.PrimaryKey. +func FromPrimaryKey(s *schema.Index) (*sqlspec.PrimaryKey, error) { + c := make([]*schemahcl.Ref, 0, len(s.Parts)) + for _, v := range s.Parts { + c = append(c, ColumnRef(v.C.Name)) + } + return &sqlspec.PrimaryKey{ + Columns: c, + }, nil +} + +// FromColumn converts a *schema.Column into a *sqlspec.Column using the ColumnTypeSpecFunc. +func FromColumn(c *schema.Column, columnTypeSpec ColumnTypeSpecFunc) (*sqlspec.Column, error) { + ct, err := columnTypeSpec(c.Type.Type) + if err != nil { + return nil, err + } + spec := &sqlspec.Column{ + Name: c.Name, + Type: ct.Type, + Null: c.Type.Null, + DefaultExtension: schemahcl.DefaultExtension{ + Extra: schemahcl.Resource{Attrs: ct.Extra.Attrs}, + }, + } + switch v := c.Default.(type) { + case nil: + case *schema.NamedDefault: + lv, err := ColumnDefault(c) + if err != nil { + return nil, err + } + spec.Extra.Children = append(spec.Extra.Children, &schemahcl.Resource{ + Type: "default", + Name: v.Name, + Attrs: []*schemahcl.Attr{{K: "as", V: lv}}, + }) + default: + lv, err := ColumnDefault(c) + if err != nil { + return nil, err + } + spec.Extra.Attrs = slices.Insert(spec.Extra.Attrs, 0, &schemahcl.Attr{K: "default", V: lv}) + } + convertCommentFromSchema(c.Attrs, &spec.Extra.Attrs) + return spec, nil +} + +// FromGenExpr returns the spec for a generated expression. +func FromGenExpr(x schema.GeneratedExpr, t func(string) string) *schemahcl.Resource { + return &schemahcl.Resource{ + Type: "as", + Attrs: []*schemahcl.Attr{ + schemahcl.StringAttr("expr", x.Expr), + VarAttr("type", t(x.Type)), + }, + } +} + +// ConvertGenExpr converts the "as" attribute or the block under the given resource. +func ConvertGenExpr(r *schemahcl.Resource, c *schema.Column, t func(string) string) error { + asA, okA := r.Attr("as") + asR, okR := r.Resource("as") + switch { + case okA && okR: + return fmt.Errorf("multiple as definitions for column %q", c.Name) + case okA: + expr, err := asA.String() + if err != nil { + return err + } + c.Attrs = append(c.Attrs, &schema.GeneratedExpr{ + Type: t(""), // default type. + Expr: expr, + }) + case okR: + var spec struct { + Expr string `spec:"expr"` + Type string `spec:"type"` + } + if err := asR.As(&spec); err != nil { + return err + } + c.Attrs = append(c.Attrs, &schema.GeneratedExpr{ + Expr: spec.Expr, + Type: t(spec.Type), + }) + } + return nil +} + +// ColumnDefault converts the column default into cty.Value. +func ColumnDefault(c *schema.Column) (cty.Value, error) { + var textlike bool + if c.Type != nil { + switch c.Type.Type.(type) { + case *schema.StringType, *schema.EnumType: + textlike = true + } + } + switch x := schema.UnderlyingExpr(c.Default).(type) { + case nil: + return cty.NilVal, nil + case *schema.RawExpr: + return schemahcl.RawExprValue(&schemahcl.RawExpr{X: x.X}), nil + case *schema.Literal: + switch { + case oneOfPrefix(x.V, "0x", "0X", "0b", "0B", "b'", "B'", "x'", "X'"): + return schemahcl.RawExprValue(&schemahcl.RawExpr{X: x.V}), nil + case sqlx.IsQuoted(x.V, '\'', '"'): + // Normalize single quotes to double quotes. + s, err := sqlx.Unquote(x.V) + if err != nil { + return cty.NilVal, err + } + return cty.StringVal(s), nil + case strings.ToLower(x.V) == "true", strings.ToLower(x.V) == "false": + return cty.BoolVal(strings.ToLower(x.V) == "true"), nil + case sqlx.IsLiteralNumber(x.V) && !textlike: + if strings.Contains(x.V, ".") { + f, err := strconv.ParseFloat(x.V, 64) + if err != nil { + return cty.NilVal, err + } + return cty.NumberFloatVal(f), nil + } + switch i, err := strconv.ParseInt(x.V, 10, 64); { + case errors.Is(err, strconv.ErrRange): + u, err := strconv.ParseUint(x.V, 10, 64) + if err != nil { + return cty.NilVal, err + } + return cty.NumberUIntVal(u), nil + case err != nil: + return cty.NilVal, err + default: + return cty.NumberIntVal(i), nil + } + default: + // Literal values (non-expressions) are returned as strings. + return cty.StringVal(x.V), nil + } + default: + return cty.NilVal, fmt.Errorf("converting expr %T to literal value for column %s", x, c.Name) + } +} + +// FromIndex converts schema.Index to sqlspec.Index. +func FromIndex(idx *schema.Index, partFns ...func(*schema.Index, *schema.IndexPart, *sqlspec.IndexPart) error) (*sqlspec.Index, error) { + spec := &sqlspec.Index{Name: idx.Name, Unique: idx.Unique} + convertCommentFromSchema(idx.Attrs, &spec.Extra.Attrs) + spec.Parts = make([]*sqlspec.IndexPart, len(idx.Parts)) + for i, p := range idx.Parts { + part := &sqlspec.IndexPart{Desc: p.Desc} + switch { + case p.C == nil && p.X == nil: + return nil, fmt.Errorf("missing column or expression for key part of index %q", idx.Name) + case p.C != nil && p.X != nil: + return nil, fmt.Errorf("multiple key part definitions for index %q", idx.Name) + case p.C != nil: + part.Column = ColumnRef(p.C.Name) + case p.X != nil: + x, ok := p.X.(*schema.RawExpr) + if !ok { + return nil, fmt.Errorf("unexpected expression %T for index %q", p.X, idx.Name) + } + part.Expr = x.X + } + for _, f := range partFns { + if err := f(idx, p, part); err != nil { + return nil, err + } + } + spec.Parts[i] = part + } + if parts, ok := columnsOnly(spec.Parts); ok { + spec.Parts = nil + spec.Columns = parts + return spec, nil + } + return spec, nil +} + +func columnsOnly(parts []*sqlspec.IndexPart) ([]*schemahcl.Ref, bool) { + columns := make([]*schemahcl.Ref, len(parts)) + for i, p := range parts { + if p.Desc || p.Column == nil || len(p.Extra.Attrs) != 0 { + return nil, false + } + columns[i] = p.Column + } + return columns, true +} + +// FromForeignKey converts schema.ForeignKey to sqlspec.ForeignKey. +func FromForeignKey(s *schema.ForeignKey) (*sqlspec.ForeignKey, error) { + c := make([]*schemahcl.Ref, 0, len(s.Columns)) + for _, v := range s.Columns { + c = append(c, ColumnRef(v.Name)) + } + r := make([]*schemahcl.Ref, 0, len(s.RefColumns)) + for _, v := range s.RefColumns { + ref := ColumnRef(v.Name) + if s.Table != s.RefTable { + ref = ExternalColumnRef(v.Name, s.RefTable.Name) + } + r = append(r, ref) + } + fk := &sqlspec.ForeignKey{ + Symbol: s.Symbol, + Columns: c, + RefColumns: r, + } + if s.OnUpdate != "" { + fk.OnUpdate = &schemahcl.Ref{V: Var(string(s.OnUpdate))} + } + if s.OnDelete != "" { + fk.OnDelete = &schemahcl.Ref{V: Var(string(s.OnDelete))} + } + return fk, nil +} + +// FromCheck converts schema.Check to sqlspec.Check. +func FromCheck(s *schema.Check) *sqlspec.Check { + return &sqlspec.Check{ + Name: s.Name, + Expr: s.Expr, + } +} + +// SchemaName returns the name from a ref to a schema. +func SchemaName(ref *schemahcl.Ref) (string, error) { + vs, err := ref.ByType(typeSchema) + if err != nil { + return "", err + } + if len(vs) != 1 { + return "", fmt.Errorf("expected 1 schema ref, got %d", len(vs)) + } + return vs[0], nil +} + +// ColumnByRef returns a column from the table by its reference. +func ColumnByRef(tv interface { + Column(string) (*schema.Column, bool) +}, ref *schemahcl.Ref) (*schema.Column, error) { + vs, err := ref.ByType(typeColumn) + if err != nil { + return nil, err + } + if len(vs) != 1 { + return nil, fmt.Errorf("expected 1 column ref, got %d", len(vs)) + } + if c, ok := tv.Column(vs[0]); ok { + return c, nil + } + switch tv := tv.(type) { + case *schema.Table: + return nil, fmt.Errorf("column %q was not found in table %s", vs[0], tv.Name) + case *schema.View: + return nil, fmt.Errorf("column %q was not found in view %s", vs[0], tv.Name) + default: + return nil, fmt.Errorf("column %q was not found in %T", vs[0], tv) + } +} + +// IndexByRef returns a index from the table/view by its reference. +func IndexByRef(tv interface { + Index(string) (*schema.Index, bool) +}, ref *schemahcl.Ref) (*schema.Index, error) { + vs, err := ref.ByType(typeIndex) + if err != nil { + return nil, err + } + if len(vs) != 1 { + return nil, fmt.Errorf("expected 1 index ref, got %d", len(vs)) + } + if c, ok := tv.Index(vs[0]); ok { + return c, nil + } + switch tv := tv.(type) { + case *schema.Table: + return nil, fmt.Errorf("index %q was not found in table %s", vs[0], tv.Name) + case *schema.View: + return nil, fmt.Errorf("index %q was not found in view %s", vs[0], tv.Name) + default: + return nil, fmt.Errorf("index %q was not found in %T", vs[0], tv) + } +} + +func externalRef(ref *schemahcl.Ref, sch *schema.Schema) (*schema.Table, *schema.Column, error) { + qualifier, name, err := TableName(ref) + if err != nil { + return nil, nil, err + } + t, err := findT(sch, qualifier, name, func(s *schema.Schema, name string) (*schema.Table, bool) { + return s.Table(name) + }) + if err != nil { + return nil, nil, err + } + c, err := ColumnByRef(t, ref) + if err != nil { + return nil, nil, err + } + return t, c, nil +} + +// findT finds the table/view referenced by ref in the provided schema. If the table/view +// is not in the provided schema.Schema other schemas in the connected schema.Realm are +// searched as well. +func findT[T schema.Object](sch *schema.Schema, qualifier, name string, findT func(*schema.Schema, string) (T, bool)) (t T, err error) { + var ( + matches []T // Found references. + schemas []*schema.Schema // Schemas to search. + ) + switch { + case sch.Realm == nil || qualifier == sch.Name: + schemas = []*schema.Schema{sch} + case qualifier == "": + schemas = sch.Realm.Schemas + default: + s, ok := sch.Realm.Schema(qualifier) + if ok { + schemas = []*schema.Schema{s} + } + } + for _, s := range schemas { + t, ok := findT(s, name) + if ok { + matches = append(matches, t) + } + } + switch len(matches) { + case 1: + return matches[0], nil + case 0: + err = fmt.Errorf("referenced %s %q not found", typeName(t), name) + default: + err = fmt.Errorf("multiple reference %ss found for %q", typeName(t), name) + } + return +} + +// TableName returns the qualifier and name from a reference to a table. +func TableName(ref *schemahcl.Ref) (string, string, error) { + return RefName(ref, typeTable) +} + +// RefName returns the qualifier and name from a reference. +func RefName(ref *schemahcl.Ref, typeName string) (qualifier, name string, err error) { + vs, err := ref.ByType(typeName) + if err != nil { + return "", "", err + } + switch len(vs) { + case 1: + name = vs[0] + case 2: + qualifier, name = vs[0], vs[1] + default: + return "", "", fmt.Errorf("sqlspec: unexpected number of references in %q", vs) + } + return +} + +func isLocalRef(r *schemahcl.Ref) bool { + return strings.HasPrefix(r.V, "$column") +} + +// ColumnRef returns the reference of a column by its name. +func ColumnRef(cName string) *schemahcl.Ref { + return schemahcl.BuildRef([]schemahcl.PathIndex{ + {T: typeColumn, V: []string{cName}}, + }) +} + +// IndexRef returns the reference of a index by its name. +func IndexRef(name string) *schemahcl.Ref { + return schemahcl.BuildRef([]schemahcl.PathIndex{ + {T: typeIndex, V: []string{name}}, + }) +} + +// ExternalColumnRef returns the reference of a column by its name and table name. +func ExternalColumnRef(cName, tName string) *schemahcl.Ref { + return schemahcl.BuildRef([]schemahcl.PathIndex{ + {T: typeTable, V: []string{tName}}, + {T: typeColumn, V: []string{cName}}, + }) +} + +// QualifiedExternalColRef returns the reference of a column by its name and qualified table name. +func QualifiedExternalColRef(cName, tName, sName string) *schemahcl.Ref { + return schemahcl.BuildRef([]schemahcl.PathIndex{ + {T: typeTable, V: []string{sName, tName}}, + {T: typeColumn, V: []string{cName}}, + }) +} + +// SchemaRef returns the schemahcl.Ref to the schema with the given name. +func SchemaRef(name string) *schemahcl.Ref { + return schemahcl.BuildRef([]schemahcl.PathIndex{ + {T: typeSchema, V: []string{name}}, + }) +} + +// Attrer is the interface that wraps the Attr method. +type Attrer interface { + Attr(string) (*schemahcl.Attr, bool) +} + +// convertCommentFromSpec converts a spec comment attribute to a schema element attribute. +func convertCommentFromSpec(spec Attrer, attrs *[]schema.Attr) error { + if c, ok := spec.Attr("comment"); ok { + s, err := c.String() + if err != nil { + return err + } + *attrs = append(*attrs, &schema.Comment{Text: s}) + } + return nil +} + +// convertCommentFromSchema converts a schema element comment attribute to a spec comment attribute. +func convertCommentFromSchema(src []schema.Attr, target *[]*schemahcl.Attr) { + var c schema.Comment + if sqlx.Has(src, &c) { + *target = append(*target, schemahcl.StringAttr("comment", c.Text)) + } +} + +// ReferenceVars holds the HCL variables +// for foreign keys' referential-actions. +var ReferenceVars = []string{ + Var(string(schema.NoAction)), + Var(string(schema.Restrict)), + Var(string(schema.Cascade)), + Var(string(schema.SetNull)), + Var(string(schema.SetDefault)), +} + +// Var formats a string as variable to make it HCL compatible. +// The result is simple, replace each space with underscore. +func Var(s string) string { return strings.ReplaceAll(s, " ", "_") } + +// FromVar is the inverse function of Var. +func FromVar(s string) string { return strings.ReplaceAll(s, "_", " ") } + +func oneOfPrefix(s string, ps ...string) bool { + for _, p := range ps { + if strings.HasPrefix(s, p) { + return true + } + } + return false +} diff --git a/database/atlas/internal/specutil/convert_test.go b/database/atlas/internal/specutil/convert_test.go new file mode 100644 index 00000000..d7ef887d --- /dev/null +++ b/database/atlas/internal/specutil/convert_test.go @@ -0,0 +1,182 @@ +// Copyright 2021-present The Atlas Authors. All rights reserved. +// This source code is licensed under the Apache 2.0 license found +// in the LICENSE file in the root directory of this source tree. + +package specutil + +import ( + "math" + "testing" + + "ariga.io/atlas/schemahcl" + "ariga.io/atlas/sql/schema" + "ariga.io/atlas/sql/sqlspec" + + "github.com/stretchr/testify/require" + "github.com/zclconf/go-cty/cty" +) + +func TestFromSpec_SchemaName(t *testing.T) { + sc := &schema.Schema{ + Name: "schema", + Tables: []*schema.Table{ + {}, + }, + } + sc.Tables[0].Schema = sc + spec, err := FromSchema(sc, &SchemaFuncs{ + Table: func(*schema.Table) (*sqlspec.Table, error) { + return &sqlspec.Table{}, nil + }, + View: func(*schema.View) (*sqlspec.View, error) { + return &sqlspec.View{}, nil + }, + }) + require.NoError(t, err) + require.Equal(t, sc.Name, spec.Schema.Name) + require.Equal(t, "$schema."+sc.Name, spec.Tables[0].Schema.V) +} + +func TestFromForeignKey(t *testing.T) { + tbl := &schema.Table{ + Name: "users", + Columns: []*schema.Column{ + { + Name: "id", + Type: &schema.ColumnType{ + Type: &schema.IntegerType{ + T: "int", + }, + }, + }, + { + Name: "parent_id", + Type: &schema.ColumnType{ + Type: &schema.IntegerType{ + T: "int", + }, + }, + }, + }, + } + fk := &schema.ForeignKey{ + Symbol: "fk", + Table: tbl, + Columns: tbl.Columns[1:], + RefTable: tbl, + RefColumns: tbl.Columns[:1], + OnUpdate: schema.NoAction, + OnDelete: schema.Cascade, + } + key, err := FromForeignKey(fk) + require.NoError(t, err) + require.EqualValues(t, &sqlspec.ForeignKey{ + Symbol: "fk", + Columns: []*schemahcl.Ref{ + {V: "$column.parent_id"}, + }, + RefColumns: []*schemahcl.Ref{ + {V: "$column.id"}, + }, + OnUpdate: &schemahcl.Ref{V: "NO_ACTION"}, + OnDelete: &schemahcl.Ref{V: "CASCADE"}, + }, key) + + fk.OnDelete = "" + fk.OnUpdate = "" + key, err = FromForeignKey(fk) + require.NoError(t, err) + require.EqualValues(t, &sqlspec.ForeignKey{ + Symbol: "fk", + Columns: []*schemahcl.Ref{ + {V: "$column.parent_id"}, + }, + RefColumns: []*schemahcl.Ref{ + {V: "$column.id"}, + }, + }, key) +} + +func TestDefault(t *testing.T) { + for _, tt := range []struct { + v cty.Value + x string + }{ + { + v: cty.NumberUIntVal(1), + x: "1", + }, + { + v: cty.NumberIntVal(1), + x: "1", + }, + { + v: cty.NumberFloatVal(1), + x: "1", + }, + { + v: cty.NumberIntVal(-100), + x: "-100", + }, + { + v: cty.NumberFloatVal(-100), + x: "-100", + }, + { + v: cty.NumberUIntVal(math.MaxUint64), + x: "18446744073709551615", + }, + { + v: cty.NumberIntVal(math.MinInt64), + x: "-9223372036854775808", + }, + { + v: cty.NumberFloatVal(-1024.1024), + x: "-1024.1024", + }, + { + v: cty.StringVal("{}"), + x: "{}", + }, + { + v: cty.StringVal("1a.1"), + x: "1a.1", + }, + } { + // From cty.Value (HCL) to database literal. + x, err := Default(tt.v) + require.NoError(t, err) + require.Equal(t, tt.x, x.(*schema.Literal).V) + // From database literal to cty.Value (HCL). + v, err := ColumnDefault(schema.NewColumn("").SetDefault(&schema.Literal{V: tt.x})) + require.NoError(t, err) + require.True(t, tt.v.Equals(v).True()) + } +} + +func TestColumnDefault_LiteralNumber(t *testing.T) { + v, err := ColumnDefault( + schema.NewColumn("").SetDefault(&schema.Literal{V: "0"}), + ) + require.NoError(t, err) + require.True(t, cty.NumberIntVal(0).Equals(v).True()) + v, err = ColumnDefault( + schema.NewStringColumn("", "text").SetDefault(&schema.Literal{V: "0"}), + ) + require.NoError(t, err) + require.True(t, cty.StringVal("0").Equals(v).True()) +} + +func TestFromView(t *testing.T) { + spec, err := FromView(&schema.View{ + Name: "view", + Def: "SELECT * FROM users\r\n WHERE c NOT LIKE \"\\r\\n\"", + Schema: schema.New("public").SetRealm(schema.NewRealm()), + }, nil, nil) + require.NoError(t, err) + as, ok := spec.Attr("as") + require.True(t, ok) + s, err := as.String() + require.NoError(t, err) + require.Equal(t, "<<-SQL\n SELECT * FROM users\n WHERE c NOT LIKE \"\\r\\n\"\n SQL", s) +} diff --git a/database/atlas/internal/specutil/spec.go b/database/atlas/internal/specutil/spec.go new file mode 100644 index 00000000..9c9fb1de --- /dev/null +++ b/database/atlas/internal/specutil/spec.go @@ -0,0 +1,287 @@ +// Copyright 2021-present The Atlas Authors. All rights reserved. +// This source code is licensed under the Apache 2.0 license found +// in the LICENSE file in the root directory of this source tree. + +package specutil + +import ( + "fmt" + "slices" + + "ariga.io/atlas/schemahcl" + "ariga.io/atlas/sql/schema" + "ariga.io/atlas/sql/sqlspec" + + "github.com/hashicorp/hcl/v2/hclparse" + "github.com/zclconf/go-cty/cty" +) + +type ( + // SchemaSpec is returned by driver convert functions to + // marshal a *schema.Schema into top-level spec objects. + SchemaSpec struct { + Schema *sqlspec.Schema + Tables []*sqlspec.Table + Views []*sqlspec.View + Funcs []*sqlspec.Func + Procs []*sqlspec.Func + Materialized []*sqlspec.View + // Collected triggers to convert into spec. + Triggers []*schema.Trigger + } + // RealmFuncs represents the functions that used + // to convert the schema.Realm into HCL spec document. + RealmFuncs struct { + Schema func(*schema.Schema) (*SchemaSpec, error) + Triggers func([]*schema.Trigger, *Doc) ([]*sqlspec.Trigger, error) + } + // Doc represents the common HCL spec document. + Doc struct { + Tables []*sqlspec.Table `spec:"table"` + Views []*sqlspec.View `spec:"view"` + Materialized []*sqlspec.View `spec:"materialized"` + Funcs []*sqlspec.Func `spec:"function"` + Procs []*sqlspec.Func `spec:"procedure"` + Triggers []*sqlspec.Trigger `spec:"trigger"` + Schemas []*sqlspec.Schema `spec:"schema"` + } +) + +// Marshal marshals v into an Atlas DDL document using a schemahcl.Marshaler. Marshal uses the given +// schemaSpec function to convert a *schema.Schema into *sqlspec.Schema, []*sqlspec.Table and []*sqlspec.View. +func Marshal(v any, marshaler schemahcl.Marshaler, funcs RealmFuncs) ([]byte, error) { + var ( + d = &Doc{} + ts []*schema.Trigger + ) + switch s := v.(type) { + case *schema.Schema: + spec, err := funcs.Schema(s) + if err != nil { + return nil, fmt.Errorf("specutil: failed converting schema to spec: %w", err) + } + d.Tables = spec.Tables + d.Views = spec.Views + d.Materialized = spec.Materialized + d.Schemas = []*sqlspec.Schema{spec.Schema} + d.Funcs = spec.Funcs + d.Procs = spec.Procs + ts = spec.Triggers + case *schema.Realm: + for _, s := range s.Schemas { + spec, err := funcs.Schema(s) + if err != nil { + return nil, fmt.Errorf("specutil: failed converting schema to spec: %w", err) + } + d.Tables = append(d.Tables, spec.Tables...) + d.Views = append(d.Views, spec.Views...) + d.Materialized = append(d.Materialized, spec.Materialized...) + d.Schemas = append(d.Schemas, spec.Schema) + d.Funcs = append(d.Funcs, spec.Funcs...) + d.Procs = append(d.Procs, spec.Procs...) + ts = append(ts, spec.Triggers...) + } + if err := QualifyObjects(d.Tables); err != nil { + return nil, err + } + if err := QualifyObjects(d.Views); err != nil { + return nil, err + } + if err := QualifyObjects(d.Materialized); err != nil { + return nil, err + } + if err := QualifyObjects(d.Funcs); err != nil { + return nil, err + } + if err := QualifyObjects(d.Procs); err != nil { + return nil, err + } + if err := QualifyReferences(d.Tables, s); err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("specutil: failed marshaling spec. %T is not supported", v) + } + if funcs.Triggers != nil { + specs, err := funcs.Triggers(ts, d) + if err != nil { + return nil, err + } + d.Triggers = specs + } + return marshaler.MarshalSpec(d) +} + +// SchemaObject describes a top-level schema object +// that might be qualified, e.g. a table or a view. +type SchemaObject interface { + Label() string + QualifierLabel() string + SetQualifier(string) + SchemaRef() *schemahcl.Ref +} + +// QualifyObjects sets the Qualifier field equal to the schema +// name in any objects with duplicate names in the provided specs. +func QualifyObjects[T SchemaObject](specs []T) error { + var ( + schemas = make(map[string]bool, len(specs)) + byLabel = make(map[string]map[string][]T, len(specs)) + ) + // Loop first and qualify schema objects with the same label. + // For example, two tables named "users" reside in different + // schemas are converted to: ("s1", "users") and ("s2", "users"). + for _, v := range specs { + l := v.Label() + q, err := SchemaName(v.SchemaRef()) + if err != nil { + return err + } + if _, ok := byLabel[l]; !ok { + byLabel[l] = make(map[string][]T) + } + byLabel[l][q] = append(byLabel[l][q], v) + } + for _, v := range byLabel { + // Multiple objects with the same label on the same + // schema are not qualified (repeatable blocks). + if len(v) == 1 { + continue + } + for q, sv := range v { + for _, s := range sv { + s.SetQualifier(q) + schemas[q] = true + } + } + } + // After objects were qualified, they might be conflicted with different + // resources that labeled with the schema name. e.g., ("s1", "users") and + // ("s1"). To resolve this conflict, we qualify these objects as well. + for _, v := range specs { + if v.QualifierLabel() == "" && schemas[v.Label()] { + schemaName, err := SchemaName(v.SchemaRef()) + if err != nil { + return err + } + v.SetQualifier(schemaName) + } + } + return nil +} + +// QualifyReferences qualifies any reference with qualifier. +func QualifyReferences(tableSpecs []*sqlspec.Table, realm *schema.Realm) error { + type cref struct{ s, t string } + byRef := make(map[cref]*sqlspec.Table) + for _, t := range tableSpecs { + r := cref{s: t.Qualifier, t: t.Name} + if byRef[r] != nil { + return fmt.Errorf("duplicate references were found for: %v", r) + } + byRef[r] = t + } + for _, t := range tableSpecs { + sname, err := SchemaName(t.Schema) + if err != nil { + return err + } + s1, ok := realm.Schema(sname) + if !ok { + return fmt.Errorf("schema %q was not found in realm", sname) + } + t1, ok := s1.Table(t.Name) + if !ok { + return fmt.Errorf("table %q.%q was not found in realm", sname, t.Name) + } + for _, fk := range t.ForeignKeys { + fk1, ok := t1.ForeignKey(fk.Symbol) + if !ok { + return fmt.Errorf("table %q.%q.%q was not found in realm", sname, t.Name, fk.Symbol) + } + for i, c := range fk.RefColumns { + if r, ok := byRef[cref{s: fk1.RefTable.Schema.Name, t: fk1.RefTable.Name}]; ok && r.Qualifier != "" { + fk.RefColumns[i] = QualifiedExternalColRef(fk1.RefColumns[i].Name, r.Name, r.Qualifier) + } else if r, ok := byRef[cref{t: fk1.RefTable.Name}]; ok && r.Qualifier == "" { + fk.RefColumns[i] = ExternalColumnRef(fk1.RefColumns[i].Name, r.Name) + } else { + return fmt.Errorf("missing reference for column %q in %q.%q.%q", c.V, sname, t.Name, fk.Symbol) + } + } + } + } + return nil +} + +// ObjectRef returns a reference to the object. In case there is more than one object of +// this type with the same name, the reference will be qualified with the schema name. +func ObjectRef(s *schema.Schema, o SpecTypeNamer) *schemahcl.Ref { + typ, name := o.SpecType(), o.SpecName() + idx := schemahcl.PathIndex{T: typ, V: []string{name}} + if s != nil && s.Realm != nil && len(s.Realm.Schemas) > 1 && slices.ContainsFunc(s.Realm.Schemas, func(s1 *schema.Schema) bool { + return s1 != s && slices.ContainsFunc(s1.Objects, func(o1 schema.Object) bool { + if e, ok := o1.(SpecTypeNamer); ok { + return e.SpecType() == typ && e.SpecName() == name + } + return false + }) + }) { + idx.V = append([]string{s.Name}, idx.V...) + } + return schemahcl.BuildRef([]schemahcl.PathIndex{idx}) +} + +// TableSpecRef returns a reference to the table in the spec. In case there is more than +// one table with the same name, the reference will be qualified with the schema name. +func TableSpecRef(t *schema.Table) *schemahcl.Ref { + typ, name := typeTable, t.Name + idx := schemahcl.PathIndex{T: typ, V: []string{name}} + if s := t.Schema; s != nil && s.Realm != nil && len(s.Realm.Schemas) > 1 && slices.ContainsFunc(s.Realm.Schemas, func(s1 *schema.Schema) bool { + return s1 != s && slices.ContainsFunc(s1.Tables, func(t1 *schema.Table) bool { + return t1.Name == t.Name + }) + }) { + idx.V = append([]string{s.Name}, idx.V...) + } + return schemahcl.BuildRef([]schemahcl.PathIndex{idx}) +} + +// ViewSpecRef returns a reference to the view in the spec. In case there is more than +// one view with the same name, the reference will be qualified with the schema name. +func ViewSpecRef(v *schema.View) *schemahcl.Ref { + typ, name := typeView, v.Name + idx := schemahcl.PathIndex{T: typ, V: []string{name}} + if s := v.Schema; s != nil && s.Realm != nil && len(s.Realm.Schemas) > 1 && slices.ContainsFunc(s.Realm.Schemas, func(s1 *schema.Schema) bool { + return s1 != s && slices.ContainsFunc(s1.Views, func(v1 *schema.View) bool { + return v.Name == v1.Name + }) + }) { + idx.V = append([]string{s.Name}, idx.V...) + } + return schemahcl.BuildRef([]schemahcl.PathIndex{idx}) +} + +// HCLBytesFunc returns a helper that evaluates an HCL document from a byte slice instead +// of from an hclparse.Parser instance. +func HCLBytesFunc(ev schemahcl.Evaluator) func(b []byte, v any, inp map[string]cty.Value) error { + return func(b []byte, v any, inp map[string]cty.Value) error { + parser := hclparse.NewParser() + if _, diag := parser.ParseHCL(b, ""); diag.HasErrors() { + return diag + } + return ev.Eval(parser, v, inp) + } +} + +// VarAttr is a helper method for constructing *schemahcl.Attr instances that contain a variable reference. +func VarAttr(k, v string) *schemahcl.Attr { + return schemahcl.RefAttr(k, &schemahcl.Ref{V: v}) +} + +// TypeAttr is a helper method for constructing *schemahcl.Attr instances that contain a type reference. +func TypeAttr(k string, t *schemahcl.Type) *schemahcl.Attr { + return &schemahcl.Attr{ + K: k, + V: schemahcl.TypeValue(t), + } +} diff --git a/database/atlas/internal/sqlx/dev.go b/database/atlas/internal/sqlx/dev.go new file mode 100644 index 00000000..1cfaea03 --- /dev/null +++ b/database/atlas/internal/sqlx/dev.go @@ -0,0 +1,360 @@ +// Copyright 2021-present The Atlas Authors. All rights reserved. +// This source code is licensed under the Apache 2.0 license found +// in the LICENSE file in the root directory of this source tree. + +package sqlx + +import ( + "context" + "errors" + "strings" + + "ariga.io/atlas/sql/migrate" + "ariga.io/atlas/sql/schema" +) + +// DevDriver is a driver that provides additional functionality +// to interact with the development database. +type DevDriver struct { + // A Driver connected to the dev database. + Driver migrate.Driver + // PathObject allows providing a custom function to patch + // objects that hold a schema reference. + PatchObject func(*schema.Schema, schema.Object) +} + +// NormalizeRealm implements the schema.Normalizer interface. +// +// The implementation converts schema objects in "natural form" (e.g. HCL or DSL) +// to their "normal presentation" in the database, by creating them temporarily in +// a "dev database", and then inspects them from there. +func (d *DevDriver) NormalizeRealm(ctx context.Context, r *schema.Realm) (nr *schema.Realm, err error) { + restore, err := d.Driver.Snapshot(ctx) + if err != nil { + return nil, err + } + defer func() { + if rerr := restore(ctx); rerr != nil { + if err != nil { + rerr = errors.Join(err, rerr) + } + err = rerr + } + }() + var ( + changes []schema.Change + opts = &schema.InspectRealmOption{ + Schemas: make([]string, 0, len(r.Schemas)), + } + ) + name2pos := make(key2pos) + for _, o := range r.Objects { + changes = append(changes, &schema.AddObject{ + O: o, + Extra: []schema.Clause{ + &schema.IfNotExists{}, + }, + }) + name2pos.putObject(o) + } + for _, s := range r.Schemas { + k, _ := name2pos.put(s.Attrs, keyS, s.Name) + opts.Schemas = append(opts.Schemas, s.Name) + changes = append(changes, &schema.AddSchema{ + S: s, + Extra: []schema.Clause{ + &schema.IfNotExists{}, + }, + }) + for _, t := range s.Tables { + changes = append(changes, addTableChange(t)...) + // If the table was loaded with its position, + // record the position of its children. + if tk, ok := name2pos.put(t.Attrs, k, keyT, t.Name); ok { + name2pos.putTable(t, tk) + } + } + for _, v := range s.Views { + changes = append(changes, addViewChange(v)...) + // If the view was loaded with its position, + // record the position of its children. + if vk, ok := name2pos.put(v.Attrs, k, keyV, v.Name); ok { + name2pos.putView(v, vk) + } + } + for _, o := range s.Objects { + name2pos.putObject(o, k) + changes = append(changes, &schema.AddObject{O: o}) + } + for _, f := range s.Funcs { + name2pos.put(f.Attrs, k, keyFn, f.Name) + changes = append(changes, &schema.AddFunc{F: f}) + } + for _, p := range s.Procs { + name2pos.put(p.Attrs, k, keyPr, p.Name) + changes = append(changes, &schema.AddProc{P: p}) + } + } + if err := d.Driver.ApplyChanges(ctx, changes); err != nil { + return nil, err + } + if nr, err = d.Driver.InspectRealm(ctx, opts); err != nil { + return nil, err + } + if len(name2pos) > 0 { + name2pos.patchRealm(nr) + } + return nr, nil +} + +// NormalizeSchema returns the normal representation of the given database. See NormalizeRealm for more info. +func (d *DevDriver) NormalizeSchema(ctx context.Context, s *schema.Schema) (*schema.Schema, error) { + restore, err := d.Driver.Snapshot(ctx) + if err != nil { + return nil, err + } + defer func() { + if rerr := restore(ctx); rerr != nil { + if err != nil { + rerr = errors.Join(err, rerr) + } + err = rerr + } + }() + dev, err := d.Driver.InspectSchema(ctx, "", &schema.InspectOptions{ + Mode: schema.InspectSchemas, + }) + if err != nil { + return nil, err + } + // Modify dev-schema attributes if needed. + changes, err := d.Driver.SchemaDiff( + schema.New(dev.Name).AddAttrs(dev.Attrs...), + schema.New(dev.Name).AddAttrs(s.Attrs...), + ) + if err != nil { + return nil, err + } + prevName := s.Name + s.Name = dev.Name + name2pos := make(key2pos) + k, _ := name2pos.put(s.Attrs, keyS, s.Name) + for _, t := range s.Tables { + // If objects are not strongly connected. + if t.Schema != s { + t.Schema = s + } + for _, c := range t.Columns { + if e, ok := c.Type.Type.(*schema.EnumType); ok && e.Schema != s { + e.Schema = s + } + } + changes = append(changes, addTableChange(t)...) + if tk, ok := name2pos.put(t.Attrs, k, keyT, t.Name); ok { + name2pos.putTable(t, tk) + } + } + for _, v := range s.Views { + // If objects are not strongly connected. + if v.Schema != s { + v.Schema = s + } + changes = append(changes, addViewChange(v)...) + if vk, ok := name2pos.put(v.Attrs, k, keyV, v.Name); ok { + name2pos.putView(v, vk) + } + } + for _, o := range s.Objects { + if d.PatchObject != nil { + d.PatchObject(s, o) + } + name2pos.putObject(o, k) + changes = append(changes, &schema.AddObject{O: o}) + } + for _, f := range s.Funcs { + name2pos.put(f.Attrs, k, keyFn, f.Name) + changes = append(changes, &schema.AddFunc{F: f}) + } + for _, p := range s.Procs { + name2pos.put(p.Attrs, k, keyPr, p.Name) + changes = append(changes, &schema.AddProc{P: p}) + } + if err := d.Driver.ApplyChanges(ctx, changes, func(opts *migrate.PlanOptions) { + noQualifier := "" + opts.SchemaQualifier = &noQualifier + opts.Mode = migrate.PlanModeInPlace + }); err != nil { + return nil, err + } + ns, err := d.Driver.InspectSchema(ctx, "", nil) + if err != nil { + return nil, err + } + // Preserve the original schema name and attributes. + ns.Name = prevName + for _, a := range s.Attrs { + schema.ReplaceOrAppend(&ns.Attrs, a) + } + if len(name2pos) > 0 { + name2pos.patchSchema(ns) + } + return ns, err +} + +const ( + keyS = "schema" + keyV = "view" + keyT = "table" + keyC = "column" + keyI = "index" + keyP = "pk" + keyF = "fk" + keyK = "check" + keyTg = "trigger" + keyFn = "function" + keyPr = "procedure" +) + +type key2pos map[string]*schema.Pos + +func (k key2pos) putTable(t *schema.Table, tk string) { + for _, c := range t.Columns { + k.put(c.Attrs, tk, keyC, c.Name) + } + for _, i := range t.Indexes { + k.put(i.Attrs, tk, keyI, i.Name) + } + for _, f := range t.ForeignKeys { + k.put(f.Attrs, tk, keyF, f.Symbol) + } + for _, ck := range t.Checks() { + k.put(ck.Attrs, tk, keyK, ck.Name) + } + if t.PrimaryKey != nil { + k.put(t.PrimaryKey.Attrs, tk, keyP, t.PrimaryKey.Name) + } + for _, r := range t.Triggers { + k.put(r.Attrs, tk, keyTg, r.Name) + } +} + +func (k key2pos) putView(v *schema.View, vk string) { + for _, c := range v.Columns { + k.put(c.Attrs, vk, keyC, c.Name) + } + for _, i := range v.Indexes { + k.put(i.Attrs, vk, keyI, i.Name) + } + for _, r := range v.Triggers { + k.put(r.Attrs, vk, keyTg, r.Name) + } +} + +func (k key2pos) put(attrs []schema.Attr, typename ...string) (string, bool) { + n := poskey(typename...) + if p := (schema.Pos{}); Has(attrs, &p) { + k[n] = &p + return n, true + } + return n, false +} + +func (k key2pos) putObject(o schema.Object, prefix ...string) { + ns, ok := o.(schema.SpecTypeNamer) + if !ok { + return + } + t, n := ns.SpecType(), ns.SpecName() + if t == "" || n == "" { + return + } + nk := poskey(append(prefix, t, n)...) + if ps, ok := o.(schema.PosSetter); ok && ps.Pos() != nil { + k[nk] = ps.Pos() + } +} + +func (k key2pos) patchRealm(r *schema.Realm) { + k.patchObjects(r.Objects) + for _, s := range r.Schemas { + k.patchSchema(s) + } +} + +func (k key2pos) patchObjects(objs []schema.Object, prefix ...string) { + for _, o := range objs { + ns, ok := o.(schema.SpecTypeNamer) + if !ok { + continue + } + t, n := ns.SpecType(), ns.SpecName() + if t == "" || n == "" { + continue + } + st, ok := o.(schema.PosSetter) + if !ok { + continue + } + k.patch(st, append(prefix, t, n)...) + } +} + +func (k key2pos) patchSchema(s *schema.Schema) { + ks, _ := k.patch(s, keyS, s.Name) + k.patchObjects(s.Objects, ks) + for _, t := range s.Tables { + tk, ok := k.patch(t, ks, keyT, t.Name) + if !ok { + continue + } + for _, tr := range t.Triggers { + k.patch(tr, tk, keyTg, tr.Name) + } + for _, c := range t.Columns { + k.patch(c, tk, keyC, c.Name) + } + for _, i := range t.Indexes { + k.patch(i, tk, keyI, i.Name) + } + for _, f := range t.ForeignKeys { + k.patch(f, tk, keyF, f.Symbol) + } + for _, ck := range t.Checks() { + k.patch(ck, tk, keyK, ck.Name) + } + } + for _, v := range s.Views { + vk, ok := k.patch(v, ks, keyV, v.Name) + if !ok { + continue + } + for _, tr := range v.Triggers { + k.patch(tr, vk, keyTg, tr.Name) + } + for _, c := range v.Columns { + k.patch(c, vk, keyC, c.Name) + } + for _, i := range v.Indexes { + k.patch(i, vk, keyI, i.Name) + } + } + for _, f := range s.Funcs { + k.patch(f, ks, keyFn, f.Name) + } + for _, p := range s.Procs { + k.patch(p, ks, keyPr, p.Name) + } +} + +func (k key2pos) patch(ps schema.PosSetter, typename ...string) (string, bool) { + n := poskey(typename...) + if p, ok := k[n]; ok { + ps.SetPos(p) + return n, true + } + return n, false +} + +func poskey(typename ...string) string { + return strings.Join(typename, ".") +} diff --git a/database/atlas/internal/sqlx/dev_test.go b/database/atlas/internal/sqlx/dev_test.go new file mode 100644 index 00000000..ea417999 --- /dev/null +++ b/database/atlas/internal/sqlx/dev_test.go @@ -0,0 +1,99 @@ +// Copyright 2021-present The Atlas Authors. All rights reserved. +// This source code is licensed under the Apache 2.0 license found +// in the LICENSE file in the root directory of this source tree. + +package sqlx + +import ( + "context" + "testing" + + "ariga.io/atlas/sql/migrate" + "ariga.io/atlas/sql/schema" + + "github.com/hashicorp/hcl/v2" + "github.com/stretchr/testify/require" +) + +func TestDriver_NormalizeRealm(t *testing.T) { + var ( + drv = &mockDriver{ + realm: schema.NewRealm(schema.New("test").SetCharset("utf8mb4")), + } + dev = &DevDriver{ + Driver: drv, + } + r = schema.NewRealm(schema.New("test")) + ) + normal, err := dev.NormalizeRealm(context.Background(), r) + require.NoError(t, err) + require.Equal(t, normal, drv.realm) + + require.Len(t, drv.schemas, 1) + require.Len(t, drv.changes, 1, "expect 1 call for creating the schema") + require.Equal(t, &schema.AddSchema{ + S: r.Schemas[0], + // The "IF NOT EXISTS" clause is added to make the + // operation noop for schema like "public" in Postgres. + Extra: []schema.Clause{&schema.IfNotExists{}}, + }, drv.changes[0]) + + // Retain positions. + r.Schemas[0]. + AddAttrs( + schema.NewFilePos("schema.hcl"). + SetStart(hcl.Pos{Line: 1, Column: 1, Byte: 1}), + ). + AddTables( + schema.NewTable("t1"). + AddAttrs( + schema.NewFilePos("schema.hcl"). + SetStart(hcl.Pos{Line: 2, Column: 2, Byte: 2}), + ). + AddColumns( + schema.NewIntColumn("id", "int"). + AddAttrs( + schema.NewFilePos("schema.hcl"). + SetStart(hcl.Pos{Line: 3, Column: 3, Byte: 3}), + ), + ), + ) + drv.realm.Schemas[0].AddTables( + schema.NewTable("t1").AddColumns(schema.NewIntColumn("id", "int")), + ) + normal, err = dev.NormalizeRealm(context.Background(), r) + require.NoError(t, err) + p := normal.Schemas[0].Pos() + require.Equal(t, schema.NewFilePos("schema.hcl").SetStart(hcl.Pos{Line: 1, Column: 1, Byte: 1}), p) + p = normal.Schemas[0].Tables[0].Pos() + require.Equal(t, schema.NewFilePos("schema.hcl").SetStart(hcl.Pos{Line: 2, Column: 2, Byte: 2}), p) + p = normal.Schemas[0].Tables[0].Columns[0].Pos() + require.Equal(t, schema.NewFilePos("schema.hcl").SetStart(hcl.Pos{Line: 3, Column: 3, Byte: 3}), p) +} + +type mockDriver struct { + migrate.Driver + // Inspect. + schemas []string + realm *schema.Realm + // Apply. + changes []schema.Change +} + +func (m *mockDriver) InspectRealm(_ context.Context, opts *schema.InspectRealmOption) (*schema.Realm, error) { + m.schemas = append(m.schemas, opts.Schemas...) + return m.realm, nil +} + +func (m *mockDriver) ApplyChanges(_ context.Context, changes []schema.Change, _ ...migrate.PlanOption) error { + m.changes = append(m.changes, changes...) + return nil +} + +func (m *mockDriver) CheckClean(context.Context, *migrate.TableIdent) error { + return nil +} + +func (m *mockDriver) Snapshot(context.Context) (migrate.RestoreFunc, error) { + return func(context.Context) error { return nil }, nil +} diff --git a/database/atlas/internal/sqlx/diff.go b/database/atlas/internal/sqlx/diff.go new file mode 100644 index 00000000..eddd979b --- /dev/null +++ b/database/atlas/internal/sqlx/diff.go @@ -0,0 +1,1000 @@ +// Copyright 2021-present The Atlas Authors. All rights reserved. +// This source code is licensed under the Apache 2.0 license found +// in the LICENSE file in the root directory of this source tree. + +package sqlx + +import ( + "fmt" + "reflect" + "slices" + "sort" + "strconv" + "strings" + + "ariga.io/atlas/sql/schema" +) + +// NoChange can be returned by DiffDriver methods +// to indicate that no change is needed. +var NoChange = schema.Change(nil) + +type ( + // A Diff provides a generic schema.Differ for diffing schema elements. + // + // The DiffDriver is required for supporting database/dialect specific + // diff capabilities, like diffing custom types or attributes. + Diff struct { + DiffDriver + } + + // A DiffDriver wraps all required methods for diffing elements that may + // have database-specific diff logic. See sql/schema/mysql/diff.go for an + // implementation example. + DiffDriver interface { + // RealmObjectDiff returns a changeset for migrating realm (database) objects + // from one state to the other. For example, adding extensions or users. + RealmObjectDiff(from, to *schema.Realm) ([]schema.Change, error) + + // SchemaAttrDiff returns a changeset for migrating schema attributes + // from one state to the other. For example, changing schema collation. + SchemaAttrDiff(from, to *schema.Schema) []schema.Change + + // SchemaObjectDiff returns a changeset for migrating schema objects from + // one state to the other. For example, changing schema custom types. + SchemaObjectDiff(from, to *schema.Schema, _ *schema.DiffOptions) ([]schema.Change, error) + + // TableAttrDiff returns a changeset for migrating table attributes from + // one state to the other. For example, dropping or adding a `CHECK` constraint. + TableAttrDiff(from, to *schema.Table, _ *schema.DiffOptions) ([]schema.Change, error) + + // ViewAttrChanges returns the changes between the two view attributes. + ViewAttrChanges(from, to *schema.View) []schema.Change + + // ColumnChange returns the schema changes (if any) for migrating one column to the other. + ColumnChange(fromT *schema.Table, from, to *schema.Column, _ *schema.DiffOptions) (schema.Change, error) + + // IndexAttrChanged reports if the index attributes were changed. + // For example, an index type or predicate (for partial indexes). + IndexAttrChanged(from, to []schema.Attr) bool + + // IndexPartAttrChanged reports if the part's attributes at position "i" + // were changed. For example, an index-part collation. + IndexPartAttrChanged(from, to *schema.Index, i int) bool + + // IsGeneratedIndexName reports if the index name was generated by the database + // for unnamed INDEX or UNIQUE constraints. In such cases, the Differ will look + // for unnamed schema.Indexes on the desired state, before tagging the index as + // a candidate for deletion. + IsGeneratedIndexName(*schema.Table, *schema.Index) bool + + // ReferenceChanged reports if the foreign key referential action was + // changed. For example, action was changed from RESTRICT to CASCADE. + ReferenceChanged(from, to schema.ReferenceOption) bool + + // ForeignKeyAttrChanged reports if any of the foreign-key attributes were changed. + ForeignKeyAttrChanged(from, to []schema.Attr) bool + } + + // DropSchemaChanger is an optional interface allows DiffDriver to drop + // schema objects before dropping the schema itself. + DropSchemaChanger interface { + DropSchemaChange(*schema.Schema) []schema.Change + } + + // A Normalizer wraps the Normalize method for normalizing the from and to tables before + // running diffing. The "from" usually represents the inspected database state (current), + // and the second represents the desired state. + // + // If the DiffDriver implements the Normalizer interface, TableDiff normalizes its table + // inputs before starting the diff process. + Normalizer interface { + Normalize(from, to *schema.Table, opts *schema.DiffOptions) error + } + + // TableFinder wraps the FindTable method, providing more + // control to the DiffDriver on how tables are matched. + TableFinder interface { + FindTable(*schema.Schema, *schema.Table) (*schema.Table, error) + } + + // ChangesAnnotator is an optional interface allows DiffDriver to annotate + // changes with additional driver-specific attributes before they are returned. + ChangesAnnotator interface { + AnnotateChanges([]schema.Change, *schema.DiffOptions) ([]schema.Change, error) + } + + // ProcFuncsDiffer is an optional interface allows DiffDriver to diff + // functions and procedures. + ProcFuncsDiffer interface { + // ProcFuncsDiff returns a changeset for migrating functions and procedures + // from one schema state to the other. + ProcFuncsDiff(from, to *schema.Schema, opts *schema.DiffOptions) ([]schema.Change, error) + } + + // TriggerDiffer is an optional interface allows DiffDriver to diff triggers. + TriggerDiffer interface { + // TriggerDiff returns a changeset for migrating triggers from + // one state to the other. For example, changing action time. + TriggerDiff(from, to *schema.Trigger) ([]schema.Change, error) + } + + // ChangeSupporter wraps the single SupportChange method. + ChangeSupporter interface { + // SupportChange can be implemented to tell the Differ if they support + // a specific change type, or it should avoid suggesting it. + SupportChange(schema.Change) bool + } +) + +// RealmDiff implements the schema.Differ for Realm objects and returns a list of changes +// that need to be applied in order to move a database from the current state to the desired. +func (d *Diff) RealmDiff(from, to *schema.Realm, options ...schema.DiffOption) ([]schema.Change, error) { + var ( + changes schema.Changes + opts = schema.NewDiffOptions(options...) + ) + // Realm-level objects. + change, err := d.RealmObjectDiff(from, to) + if err != nil { + return nil, err + } + changes = opts.AddOrSkip(changes, change...) + // Drop or modify schema. + for _, s1 := range from.Schemas { + s2, ok := to.Schema(s1.Name) + if !ok { + if ds, ok := d.DiffDriver.(DropSchemaChanger); ok { + // The driver can drop other objects before dropping the schema. + changes = opts.AddOrSkip(changes, ds.DropSchemaChange(s1)...) + } else { + changes = opts.AddOrSkip(changes, &schema.DropSchema{S: s1}) + } + continue + } + change, err := d.schemaDiff(s1, s2, opts) + if err != nil { + return nil, err + } + changes = append(changes, change...) + } + // Add schemas. + for _, s1 := range to.Schemas { + if _, ok := from.Schema(s1.Name); ok { + continue + } + changes = opts.AddOrSkip(changes, &schema.AddSchema{S: s1}) + for _, o := range s1.Objects { + changes = opts.AddOrSkip(changes, &schema.AddObject{O: o}) + } + for _, f := range s1.Funcs { + changes = opts.AddOrSkip(changes, &schema.AddFunc{F: f}) + } + for _, p := range s1.Procs { + changes = opts.AddOrSkip(changes, &schema.AddProc{P: p}) + } + for _, t := range s1.Tables { + changes = opts.AddOrSkip(changes, addTableChange(t)...) + } + for _, v := range s1.Views { + changes = opts.AddOrSkip(changes, addViewChange(v)...) + } + } + return d.mayAnnotate(changes, opts) +} + +// SchemaDiff implements the schema.Differ interface and returns a list of +// changes that need to be applied in order to move from one state to the other. +func (d *Diff) SchemaDiff(from, to *schema.Schema, options ...schema.DiffOption) ([]schema.Change, error) { + opts := schema.NewDiffOptions(options...) + changes, err := d.schemaDiff(from, to, opts) + if err != nil { + return nil, err + } + return d.mayAnnotate(changes, opts) +} + +func (d *Diff) schemaDiff(from, to *schema.Schema, opts *schema.DiffOptions) ([]schema.Change, error) { + if from.Name != to.Name { + return nil, fmt.Errorf("mismatched schema names: %q != %q", from.Name, to.Name) + } + var changes []schema.Change + // Drop or modify attributes (collations, charset, etc). + if change := d.SchemaAttrDiff(from, to); len(change) > 0 { + changes = opts.AddOrSkip(changes, &schema.ModifySchema{ + S: to, + Changes: change, + }) + } + // Add, drop or modify objects. + change, err := d.SchemaObjectDiff(from, to, opts) + if err != nil { + return nil, err + } + changes = opts.AddOrSkip(changes, change...) + + // Drop or modify tables. + for _, t1 := range from.Tables { + switch t2, err := d.findTable(to, t1); { + case schema.IsNotExistError(err): + // Triggers should be dropped either by the driver or the database. + changes = opts.AddOrSkip(changes, &schema.DropTable{T: t1}) + case err != nil: + return nil, err + default: + if change, err := d.tableDiff(t1, t2, opts); err != nil { + return nil, err + } else if len(change) > 0 { + changes = opts.AddOrSkip(changes, &schema.ModifyTable{T: t2, Changes: change}) + } + if change, err := d.triggerDiff(triggerDiffInput{From: t1, To: t2, FromTriggers: t1.Triggers, ToTriggers: t2.Triggers}, opts); err != nil { + return nil, err + } else { + changes = append(changes, change...) + } + } + } + changes = d.fixRenames(changes) + // Add tables. + for _, t1 := range to.Tables { + switch _, err := d.findTable(from, t1); { + case schema.IsNotExistError(err): + changes = opts.AddOrSkip(changes, addTableChange(t1)...) + case err != nil: + return nil, err + } + } + + // Drop or modify views. + for _, v1 := range from.Views { + v2, ok := findView(to, v1) + if !ok { + // Changing a view to materialized (and vice versa) + // generates a drop and add. + changes = opts.AddOrSkip(changes, &schema.DropView{V: v1}) + continue + } + if change, err := d.viewDiff(v1, v2, opts); err != nil { + return nil, err + } else { + changes = append(changes, change...) + } + if change, err := d.triggerDiff(triggerDiffInput{From: v1, To: v2, FromTriggers: v1.Triggers, ToTriggers: v2.Triggers}, opts); err != nil { + return nil, err + } else { + changes = append(changes, change...) + } + } + // Add views. + for _, v1 := range to.Views { + if _, ok := findView(from, v1); !ok { + changes = opts.AddOrSkip(changes, addViewChange(v1)...) + } + } + // Add, drop and modify functions and procedures. + if pf, ok := d.DiffDriver.(ProcFuncsDiffer); ok { + change, err := pf.ProcFuncsDiff(from, to, opts) + if err != nil { + return nil, err + } + changes = append(changes, change...) + } + return changes, nil +} + +// TableDiff implements the schema.TableDiffer interface and returns a list of +// changes that need to be applied in order to move from one state to the other. +func (d *Diff) TableDiff(from, to *schema.Table, options ...schema.DiffOption) ([]schema.Change, error) { + opts := schema.NewDiffOptions(options...) + if from.Name != to.Name { + return nil, fmt.Errorf("mismatched table names: %q != %q", from.Name, to.Name) + } + changes, err := d.tableDiff(from, to, opts) + if err != nil { + return nil, err + } + if change, err := d.triggerDiff(triggerDiffInput{From: from, To: to, FromTriggers: from.Triggers, ToTriggers: to.Triggers}, opts); err != nil { + return nil, err + } else { + changes = append(changes, change...) + } + return d.mayAnnotate(changes, opts) +} + +// tableDiff implements the table diffing but skips the table name check. +func (d *Diff) tableDiff(from, to *schema.Table, opts *schema.DiffOptions) ([]schema.Change, error) { + // tableDiff can be called with non-identical + // names without affecting the diff process. + if name := from.Name; name != to.Name { + from.Name = to.Name + defer func() { from.Name = name }() + } + // Normalizing tables before starting the diff process. + if n, ok := d.DiffDriver.(Normalizer); ok { + if err := n.Normalize(from, to, opts); err != nil { + return nil, err + } + } + var changes []schema.Change + // Drop or modify attributes (collations, checks, etc). + change, err := d.TableAttrDiff(from, to, opts) + if err != nil { + return nil, err + } + changes = append(changes, change...) + + // Drop, add or modify columns. + if change, err = d.columnDiff(from, to, opts); err != nil { + return nil, err + } + changes = append(changes, change...) + + // Primary-key and index changes. + changes = append(changes, d.pkDiff(from, to, opts)...) + if change, err = d.indexDiffT(from, to, opts); err != nil { + return nil, err + } + changes = append(changes, change...) + + // Drop or modify foreign-keys. + for _, fk1 := range from.ForeignKeys { + fk2, ok := to.ForeignKey(fk1.Symbol) + if !ok { + changes = opts.AddOrSkip(changes, &schema.DropForeignKey{F: fk1}) + continue + } + if change := d.fkChange(fk1, fk2); change != schema.NoChange { + changes = opts.AddOrSkip(changes, &schema.ModifyForeignKey{ + From: fk1, + To: fk2, + Change: change, + }) + } + } + // Add foreign-keys. + for _, fk1 := range to.ForeignKeys { + if _, ok := from.ForeignKey(fk1.Symbol); !ok { + changes = opts.AddOrSkip(changes, &schema.AddForeignKey{F: fk1}) + } + } + return changes, nil +} + +func (d *Diff) mayAnnotate(changes []schema.Change, opts *schema.DiffOptions) (_ []schema.Change, err error) { + r, ok := d.DiffDriver.(ChangesAnnotator) + if ok { + if changes, err = r.AnnotateChanges(changes, opts); err != nil { + return nil, err + } + } + return changes, nil +} + +// addTableChange returns the changeset for creating the table. +func addTableChange(t *schema.Table) []schema.Change { + changes := make([]schema.Change, 0, 1+len(t.Triggers)) + changes = append(changes, &schema.AddTable{T: t}) + for _, r := range t.Triggers { + changes = append(changes, &schema.AddTrigger{T: r}) + } + return changes +} + +// addViewChange returns the changeset for creating the view. +func addViewChange(v *schema.View) []schema.Change { + changes := make([]schema.Change, 0, 1+len(v.Triggers)) + changes = append(changes, &schema.AddView{V: v}) + for _, r := range v.Triggers { + changes = append(changes, &schema.AddTrigger{T: r}) + } + return changes +} + +// columnDiff returns the schema changes (if any) for migrating table columns. +func (d *Diff) columnDiff(from, to *schema.Table, opts *schema.DiffOptions) ([]schema.Change, error) { + var all []schema.Change + // Drop or modify columns. + for _, c1 := range from.Columns { + c2, ok := to.Column(c1.Name) + if !ok { + all = append(all, &schema.DropColumn{C: c1}) + continue + } + change, err := d.ColumnChange(from, c1, c2, opts) + if err != nil { + return nil, err + } + if change != NoChange { + all = append(all, change) + } + } + // Add columns. + for _, c1 := range to.Columns { + if _, ok := from.Column(c1.Name); !ok { + all = append(all, &schema.AddColumn{ + C: c1, + }) + } + } + var ( + err error + changes = make([]schema.Change, 0, len(all)) + ) + if all, err = d.askForColumns(from, all, opts); err != nil { + return nil, err + } + for _, c := range all { + changes = opts.AddOrSkip(changes, c) + } + return changes, nil +} + +// pkDiff returns the schema changes (if any) for migrating table +// primary-key from current state to the desired state. +func (d *Diff) pkDiff(from, to *schema.Table, opts *schema.DiffOptions) (changes []schema.Change) { + switch pk1, pk2 := from.PrimaryKey, to.PrimaryKey; { + case pk1 == nil && pk2 != nil: + changes = opts.AddOrSkip(changes, &schema.AddPrimaryKey{P: pk2}) + case pk1 != nil && pk2 == nil: + changes = opts.AddOrSkip(changes, &schema.DropPrimaryKey{P: pk1}) + case pk1 != nil: + change := d.indexChange(pk1, pk2) + change &= ^schema.ChangeUnique + switch c, ok := d.DiffDriver.(ChangeSupporter); { + case change != schema.NoChange: + changes = opts.AddOrSkip(changes, &schema.ModifyPrimaryKey{ + From: pk1, To: pk2, Change: change, + }) + case (!ok || c.SupportChange((*schema.RenameConstraint)(nil))) && + pk1.Name != "" && pk2.Name != "" && pk1.Name != pk2.Name: + changes = opts.AddOrSkip(changes, &schema.RenameConstraint{ + From: pk1, To: pk2, + }) + } + } + return +} + +// indexDiffT returns the schema changes (if any) for migrating table +// indexes from current state to the desired state. +func (d *Diff) indexDiffT(from, to *schema.Table, opts *schema.DiffOptions) ([]schema.Change, error) { + var ( + all []schema.Change + exists = make(map[*schema.Index]bool) + ) + // Drop or modify indexes. + for _, idx1 := range from.Indexes { + idx2, ok := to.Index(idx1.Name) + // Found directly. + if ok { + if change := d.indexChange(idx1, idx2); change != schema.NoChange { + all = append(all, &schema.ModifyIndex{ + From: idx1, + To: idx2, + Change: change, + }) + } + exists[idx2] = true + continue + } + // Found indirectly. + if d.IsGeneratedIndexName(from, idx1) { + if idx2, ok := d.similarUnnamedIndex(to, idx1); ok { + exists[idx2] = true + continue + } + } + // Not found. + all = append(all, &schema.DropIndex{I: idx1}) + } + // Add indexes. + for _, idx := range to.Indexes { + if exists[idx] { + continue + } + if _, ok := from.Index(idx.Name); !ok { + all = append(all, &schema.AddIndex{I: idx}) + } + } + var ( + err error + changes = make([]schema.Change, 0, len(all)) + ) + if all, err = d.askForIndexes(from.Name, all, opts); err != nil { + return nil, err + } + for _, c := range all { + changes = opts.AddOrSkip(changes, c) + } + return changes, nil +} + +// viewDiff returns the schema changes (if any) for migrating view from +// current state to the desired state. +func (d *Diff) viewDiff(from, to *schema.View, opts *schema.DiffOptions) ([]schema.Change, error) { + c1, err := d.indexDiffV(from, to, opts) + if err != nil { + return nil, err + } + c2 := d.columnDiffV(from, to, opts) + var changes []schema.Change + if vs := append(d.ViewAttrChanges(from, to), append(c1, c2...)...); len(vs) > 0 || d.viewDefChanged(from, to) { + changes = opts.AddOrSkip(changes, &schema.ModifyView{From: from, To: to, Changes: vs}) + } + return changes, nil +} + +// viewDefChanged checks if the view definition has changed. +// It allows the DiffDriver to override the default implementation. +func (d *Diff) viewDefChanged(v1 *schema.View, v2 *schema.View) bool { + if vr, ok := d.DiffDriver.(interface { + ViewDefChanged(v1, v2 *schema.View) bool + }); ok { + return vr.ViewDefChanged(v1, v2) + } + return BodyDefChanged(v1.Def, v2.Def) +} + +// columnDiffV returns the schema changes (if any) for migrating view columns. +// Currently, only comment changes are supported. +func (d *Diff) columnDiffV(from, to *schema.View, opts *schema.DiffOptions) []schema.Change { + var changes []schema.Change + for _, c1 := range from.Columns { + c2, ok := to.Column(c1.Name) + if !ok { + continue + } + if change := CommentChange(c1.Attrs, c2.Attrs); change != schema.NoChange { + changes = opts.AddOrSkip(changes, &schema.ModifyColumn{ + From: c1, + To: c2, + Change: change, + }) + } + } + return changes +} + +// indexDiffV returns the schema changes (if any) for migrating view +// indexes from current state to the desired state. +func (d *Diff) indexDiffV(from, to *schema.View, opts *schema.DiffOptions) ([]schema.Change, error) { + var ( + changes []schema.Change + exists = make(map[*schema.Index]bool) + ) + // Drop or modify indexes. + for _, idx1 := range from.Indexes { + idx2, ok := to.Index(idx1.Name) + if ok { + if change := d.indexChange(idx1, idx2); change != schema.NoChange { + changes = opts.AddOrSkip(changes, &schema.ModifyIndex{ + From: idx1, + To: idx2, + Change: change, + }) + } + exists[idx2] = true + continue + } + // Not found. + changes = opts.AddOrSkip(changes, &schema.DropIndex{I: idx1}) + } + // Add indexes. + for _, idx := range to.Indexes { + if exists[idx] { + continue + } + if _, ok := from.Index(idx.Name); !ok { + changes = opts.AddOrSkip(changes, &schema.AddIndex{I: idx}) + } + } + return d.askForIndexes(from.Name, changes, opts) +} + +// indexChange returns the schema changes (if any) for migrating one index to the other. +func (d *Diff) indexChange(from, to *schema.Index) schema.ChangeKind { + var change schema.ChangeKind + if from.Unique != to.Unique { + change |= schema.ChangeUnique + } + if d.IndexAttrChanged(from.Attrs, to.Attrs) { + change |= schema.ChangeAttr + } + change |= d.partsChange(from, to, nil) + change |= CommentChange(from.Attrs, to.Attrs) + return change +} + +func (d *Diff) partsChange(fromI, toI *schema.Index, renames map[string]string) schema.ChangeKind { + from, to := fromI.Parts, toI.Parts + if len(from) != len(to) { + return schema.ChangeParts + } + sort.Slice(to, func(i, j int) bool { return to[i].SeqNo < to[j].SeqNo }) + sort.Slice(from, func(i, j int) bool { return from[i].SeqNo < from[j].SeqNo }) + for i := range from { + switch { + case from[i].Desc != to[i].Desc || d.IndexPartAttrChanged(fromI, toI, i): + return schema.ChangeParts + case from[i].C != nil && to[i].C != nil: + if from[i].C.Name != to[i].C.Name && renames[from[i].C.Name] != to[i].C.Name { + return schema.ChangeParts + } + case from[i].X != nil && to[i].X != nil: + x1, x2 := from[i].X.(*schema.RawExpr).X, to[i].X.(*schema.RawExpr).X + if x1 != x2 && x1 != MayWrap(x2) { + return schema.ChangeParts + } + default: // (C1 != nil) != (C2 != nil) || (X1 != nil) != (X2 != nil). + return schema.ChangeParts + } + } + return schema.NoChange +} + +// fkChange returns the schema changes (if any) for migrating one index to the other. +func (d *Diff) fkChange(from, to *schema.ForeignKey) schema.ChangeKind { + var change schema.ChangeKind + switch { + case from.RefTable.Name != to.RefTable.Name: + change |= schema.ChangeRefTable | schema.ChangeRefColumn + case len(from.RefColumns) != len(to.RefColumns): + change |= schema.ChangeRefColumn + default: + for i := range from.RefColumns { + if from.RefColumns[i].Name != to.RefColumns[i].Name { + change |= schema.ChangeRefColumn + } + } + } + switch { + case len(from.Columns) != len(to.Columns): + change |= schema.ChangeColumn + default: + for i := range from.Columns { + if from.Columns[i].Name != to.Columns[i].Name { + change |= schema.ChangeColumn + } + } + } + if d.ReferenceChanged(from.OnUpdate, to.OnUpdate) { + change |= schema.ChangeUpdateAction + } + if d.ReferenceChanged(from.OnDelete, to.OnDelete) { + change |= schema.ChangeDeleteAction + } + if d.ForeignKeyAttrChanged(from.Attrs, to.Attrs) { + change |= schema.ChangeAttr + } + return change +} + +// similarUnnamedIndex searches for an unnamed index with the same index-parts in the table. +func (d *Diff) similarUnnamedIndex(t *schema.Table, idx1 *schema.Index) (*schema.Index, bool) { + match := func(idx1, idx2 *schema.Index) bool { + return idx1.Unique == idx2.Unique && d.partsChange(idx1, idx2, nil) == schema.NoChange + } + if f, ok := d.DiffDriver.(interface { + FindGeneratedIndex(*schema.Table, *schema.Index) (*schema.Index, bool) + }); ok { + if idx2, ok := f.FindGeneratedIndex(t, idx1); ok && match(idx1, idx2) { + return idx2, true + } + } + for _, idx2 := range t.Indexes { + if idx2.Name == "" && match(idx1, idx2) { + return idx2, true + } + } + return nil, false +} + +func (d *Diff) findTable(s *schema.Schema, t1 *schema.Table) (*schema.Table, error) { + if f, ok := d.DiffDriver.(TableFinder); ok { + return f.FindTable(s, t1) + } + t2, ok := s.Table(t1.Name) + if !ok { + return nil, &schema.NotExistError{Err: fmt.Errorf("table %q was not found", t1.Name)} + } + return t2, nil +} + +// CommentChange reports if the element comment was changed. +func CommentChange(from, to []schema.Attr) schema.ChangeKind { + var c1, c2 schema.Comment + if Has(from, &c1) != Has(to, &c2) || c1.Text != c2.Text { + return schema.ChangeComment + } + return schema.NoChange +} + +// Charset reports if the attribute contains the "charset" attribute, +// and it needs to be defined explicitly on the schema. This is true, in +// case the element charset is different from its parent charset. +func Charset(attr, parent []schema.Attr) (string, bool) { + var c, p schema.Charset + if Has(attr, &c) && (parent == nil || Has(parent, &p) && c.V != p.V) { + return c.V, true + } + return "", false +} + +// Collate reports if the attribute contains the "collation"/"collate" attribute, +// and it needs to be defined explicitly on the schema. This is true, in +// case the element collation is different from its parent collation. +func Collate(attr, parent []schema.Attr) (string, bool) { + var c, p schema.Collation + if Has(attr, &c) && (parent == nil || Has(parent, &p) && c.V != p.V) { + return c.V, true + } + return "", false +} + +var ( + attrsType = reflect.TypeOf(([]schema.Attr)(nil)) + clausesType = reflect.TypeOf(([]schema.Clause)(nil)) + exprsType = reflect.TypeOf(([]schema.Expr)(nil)) +) + +// AttrOr returns the first attribute of the given type, +// or the given default value. +func AttrOr[T schema.Attr](attrs []schema.Attr, t T) T { + for _, attr := range attrs { + if a, ok := attr.(T); ok { + return a + } + } + return t +} + +// Has finds the first element in the elements list that +// matches target, and if so, sets target to that attribute +// value and returns true. +func Has(elements, target any) bool { + ev := reflect.ValueOf(elements) + if t := ev.Type(); t != attrsType && t != clausesType && t != exprsType { + panic(fmt.Sprintf("unexpected elements type: %T", elements)) + } + tv := reflect.ValueOf(target) + if tv.Kind() != reflect.Pointer || tv.IsNil() { + panic("target must be a non-nil pointer") + } + for i := 0; i < ev.Len(); i++ { + idx := ev.Index(i) + if idx.IsNil() { + continue + } + if e := idx.Elem(); e.Type().AssignableTo(tv.Type()) { + tv.Elem().Set(e.Elem()) + return true + } + } + return false +} + +// UnsupportedTypeError describes an unsupported type error. +type UnsupportedTypeError struct { + schema.Type +} + +func (e UnsupportedTypeError) Error() string { + return fmt.Sprintf("unsupported type %T", e.Type) +} + +// CommentDiff computes the comment diff between the 2 attribute list. +// Note that, the implementation relies on the fact that both PostgreSQL +// and MySQL treat empty comment as "no comment" and a way to clear comments. +func CommentDiff(from, to []schema.Attr) schema.Change { + var fromC, toC schema.Comment + switch fromHas, toHas := Has(from, &fromC), Has(to, &toC); { + case !fromHas && !toHas: + case !fromHas && toC.Text != "": + return &schema.AddAttr{ + A: &toC, + } + case !toHas: + // In MySQL, there is no way to DROP a comment. Instead, setting it to empty ('') + // will remove it from INFORMATION_SCHEMA. We use the same approach in PostgreSQL, + // because comments can be dropped either by setting them to NULL or empty string. + // See: postgres/backend/commands/comment.c#CreateComments. + return &schema.ModifyAttr{ + From: &fromC, + To: &toC, + } + default: + v1, err1 := Unquote(fromC.Text) + v2, err2 := Unquote(toC.Text) + if err1 == nil && err2 == nil && v1 != v2 { + return &schema.ModifyAttr{ + From: &fromC, + To: &toC, + } + } + } + return nil +} + +// CheckDiffMode is like CheckDiff, but compares also expressions +// if the schema.DiffMode is equal to schema.DiffModeNormalized. +func CheckDiffMode(from, to *schema.Table, mode schema.DiffMode, compare ...func(c1, c2 *schema.Check) bool) []schema.Change { + if !mode.Is(schema.DiffModeNormalized) { + return checksSimilarDiff(from, to, compare...) + } + return ChecksDiff(from, to, func(c1, c2 *schema.Check) bool { + if len(compare) == 1 && !compare[0](c1, c2) { + return false + } + return c1.Expr == c2.Expr || MayWrap(c1.Expr) == MayWrap(c2.Expr) + }) +} + +// ChecksDiff computes the change diff between the 2 tables. +func ChecksDiff(from, to *schema.Table, compare func(c1, c2 *schema.Check) bool) []schema.Change { + var ( + changes []schema.Change + fromC, toC = checks(from.Attrs), checks(to.Attrs) + compareTo = func(c1 *schema.Check) func(c2 *schema.Check) bool { + return func(c2 *schema.Check) bool { + if c1.Name != "" && c2.Name != "" { + // Only compare by name if both have a name. + return c1.Name == c2.Name + } + return compare(c1, c2) + } + } + ) + for _, c1 := range fromC { + switch idx := slices.IndexFunc(toC, compareTo(c1)); { + case idx == -1: + changes = append(changes, &schema.DropCheck{ + C: c1, + }) + case !compare(c1, toC[idx]): + changes = append(changes, &schema.ModifyCheck{ + From: c1, + To: toC[idx], + }) + } + } + for _, c1 := range toC { + if !slices.ContainsFunc(fromC, compareTo(c1)) { + changes = append(changes, &schema.AddCheck{ + C: c1, + }) + } + } + return changes +} + +// checksSimilarDiff computes the change diff between the 2 tables. +// Unlike ChecksDiff, it does not compare the constraint name, but +// determines if there is any similar constraint by its expression. +// This is an old implementation that is not used anymore by the CLI. +func checksSimilarDiff(from, to *schema.Table, compare ...func(c1, c2 *schema.Check) bool) []schema.Change { + var changes []schema.Change + // Drop or modify checks. + for _, c1 := range checks(from.Attrs) { + switch c2, ok := similarCheck(to.Attrs, c1); { + case !ok: + changes = append(changes, &schema.DropCheck{ + C: c1, + }) + case len(compare) == 1 && !compare[0](c1, c2): + changes = append(changes, &schema.ModifyCheck{ + From: c1, + To: c2, + }) + } + } + // Add checks. + for _, c1 := range checks(to.Attrs) { + if _, ok := similarCheck(from.Attrs, c1); !ok { + changes = append(changes, &schema.AddCheck{ + C: c1, + }) + } + } + return changes +} + +// checks extracts all constraints from table attributes. +func checks(attr []schema.Attr) (checks []*schema.Check) { + for i := range attr { + if c, ok := attr[i].(*schema.Check); ok { + checks = append(checks, c) + } + } + return checks +} + +// similarCheck returns a CHECK by its constraints name or expression. +func similarCheck(attrs []schema.Attr, c *schema.Check) (*schema.Check, bool) { + var byName, byExpr *schema.Check + for i := 0; i < len(attrs) && (byName == nil || byExpr == nil); i++ { + check, ok := attrs[i].(*schema.Check) + if !ok { + continue + } + if check.Name != "" && check.Name == c.Name { + byName = check + } + if check.Expr == c.Expr || MayWrap(check.Expr) == MayWrap(c.Expr) { + byExpr = check + } + } + // Give precedence to constraint name. + if byName != nil { + return byName, true + } + if byExpr != nil { + return byExpr, true + } + return nil, false +} + +// Unquote single or double quotes. +func Unquote(s string) (string, error) { + switch { + case IsQuoted(s, '"'): + return strconv.Unquote(s) + case IsQuoted(s, '\''): + return strings.ReplaceAll(s[1:len(s)-1], "''", "'"), nil + default: + return s, nil + } +} + +// SingleQuote quotes the given string with single quote. +func SingleQuote(s string) (string, error) { + switch { + case IsQuoted(s, '\''): + return s, nil + case IsQuoted(s, '"'): + v, err := strconv.Unquote(s) + if err != nil { + return "", err + } + s = v + fallthrough + default: + return "'" + strings.ReplaceAll(s, "'", "''") + "'", nil + } +} + +// TrimViewExtra trims the extra unnecessary +// characters from the view definition. +func TrimViewExtra(s string) string { + return strings.Trim(s, " \r\n\t;") +} + +// BodyDefChanged reports if the body definition of a function, procedure, view, or +// trigger has changed. There is a small task here that normalizes the indentation, +// which might be added during inspection or by the user. +func BodyDefChanged(from, to string) bool { + if from == to { + return false // Exact match. + } + if from, to = TrimViewExtra(from), TrimViewExtra(to); from == to { + return false // Match after trimming. + } + noident := func(v string) string { + var b strings.Builder + for i, s := range strings.Split(v, "\n") { + if s = TrimViewExtra(s); s != "" && i > 0 { + b.WriteByte(' ') + } + b.WriteString(s) + } + return b.String() + } + return noident(from) != noident(to) +} + +// findView finds the view by its name and its type. +func findView(s *schema.Schema, v1 *schema.View) (*schema.View, bool) { + if v1.Materialized() { + return s.Materialized(v1.Name) + } + return s.View(v1.Name) +} diff --git a/database/atlas/internal/sqlx/plan.go b/database/atlas/internal/sqlx/plan.go new file mode 100644 index 00000000..5c2c554c --- /dev/null +++ b/database/atlas/internal/sqlx/plan.go @@ -0,0 +1,632 @@ +// Copyright 2021-present The Atlas Authors. All rights reserved. +// This source code is licensed under the Apache 2.0 license found +// in the LICENSE file in the root directory of this source tree. + +package sqlx + +import ( + "context" + "database/sql" + "errors" + "fmt" + "slices" + "sort" + + "ariga.io/atlas/sql/migrate" + "ariga.io/atlas/sql/schema" +) + +type ( + execPlanner interface { + ExecContext(context.Context, string, ...any) (sql.Result, error) + PlanChanges(context.Context, string, []schema.Change, ...migrate.PlanOption) (*migrate.Plan, error) + } + // ApplyError is an error that exposes an information for getting + // how any changes were applied before encountering the failure. + ApplyError struct { + err string + applied int + } +) + +// Applied reports how many changes were applied before getting an error. +// In case the first change was failed, Applied() returns 0. +func (e *ApplyError) Applied() int { + return e.applied +} + +// Error implements the error interface. +func (e *ApplyError) Error() string { + return e.err +} + +// ApplyChanges is a helper used by the different drivers to apply changes. +func ApplyChanges(ctx context.Context, changes []schema.Change, p execPlanner, opts ...migrate.PlanOption) error { + plan, err := p.PlanChanges(ctx, "apply", changes, opts...) + if err != nil { + return err + } + for i, c := range plan.Changes { + if _, err := p.ExecContext(ctx, c.Cmd, c.Args...); err != nil { + if c.Comment != "" { + err = fmt.Errorf("%s: %w", c.Comment, err) + } + return &ApplyError{err: err.Error(), applied: i} + } + } + return nil +} + +// noRows implements the schema.ExecQuerier for migrate.Driver's without connections. +// This can be useful to always return no rows for queries, and block any execution. +type noRows struct{} + +// QueryContext implements the sqlx.ExecQuerier interface. +func (*noRows) QueryContext(context.Context, string, ...interface{}) (*sql.Rows, error) { + return nil, sql.ErrNoRows +} + +// ExecContext implements the sqlx.ExecQuerier interface. +func (*noRows) ExecContext(context.Context, string, ...interface{}) (sql.Result, error) { + return nil, errors.New("cannot execute statements without a database connection. use Open to create a new Driver") +} + +// NoRows to be used by differs and planners without a connection. +var NoRows schema.ExecQuerier = (*noRows)(nil) + +// SetReversible sets the Reversible field to +// true if all planned changes are reversible. +func SetReversible(p *migrate.Plan) error { + reversible := true + for _, c := range p.Changes { + stmts, err := c.ReverseStmts() + if err != nil { + return err + } + if len(stmts) == 0 { + reversible = false + } + } + p.Reversible = reversible + return nil +} + +// DetachCycles takes a list of schema changes, and detaches +// references between changes if there is at least one circular +// reference in the changeset. More explicitly, it postpones fks +// creation, or deletes fks before deletes their tables. +func DetachCycles(changes []schema.Change) ([]schema.Change, error) { + sorted, err := sortMap(changes) + if errors.Is(err, errCycle) { + return detachReferences(changes), nil + } + if err != nil { + return nil, err + } + planned := make([]schema.Change, len(changes)) + copy(planned, changes) + sort.Slice(planned, func(i, j int) bool { + return sorted[table(planned[i])] < sorted[table(planned[j])] + }) + return planned, nil +} + +// detachReferences detaches all table references. +func detachReferences(changes []schema.Change) []schema.Change { + var planned, deferred []schema.Change + for _, change := range changes { + switch change := change.(type) { + case *schema.AddTable: + var ( + ext []schema.Change + self []*schema.ForeignKey + ) + for _, fk := range change.T.ForeignKeys { + if fk.RefTable == change.T { + self = append(self, fk) + } else { + ext = append(ext, &schema.AddForeignKey{F: fk}) + } + } + if len(ext) > 0 { + deferred = append(deferred, &schema.ModifyTable{T: change.T, Changes: ext}) + t := *change.T + t.ForeignKeys = self + change = &schema.AddTable{T: &t, Extra: change.Extra} + } + planned = append(planned, change) + case *schema.DropTable: + var fks []schema.Change + for _, fk := range change.T.ForeignKeys { + if fk.RefTable != change.T { + fks = append(fks, &schema.DropForeignKey{F: fk}) + } + } + if len(fks) > 0 { + planned = append(planned, &schema.ModifyTable{T: change.T, Changes: fks}) + t := *change.T + t.ForeignKeys = nil + change = &schema.DropTable{T: &t, Extra: change.Extra} + } + deferred = append(deferred, change) + case *schema.ModifyTable: + var fks, rest []schema.Change + for _, c := range change.Changes { + switch c := c.(type) { + case *schema.AddForeignKey: + fks = append(fks, c) + default: + rest = append(rest, c) + } + } + if len(fks) > 0 { + deferred = append(deferred, &schema.ModifyTable{T: change.T, Changes: fks}) + } + if len(rest) > 0 { + planned = append(planned, &schema.ModifyTable{T: change.T, Changes: rest}) + } + default: + planned = append(planned, change) + } + } + return append(planned, deferred...) +} + +// errCycle is an internal error to indicate a case of a cycle. +var errCycle = errors.New("cycle detected") + +// sortMap returns an index-map indicates the position of table in a topological +// sort in reversed order based on its references, and a boolean indicate if there +// is a non-self loop. +func sortMap(changes []schema.Change) (map[string]int, error) { + var ( + visit func(string) bool + sorted = make(map[string]int) + progress = make(map[string]bool) + deps, err = dependencies(changes) + ) + if err != nil { + return nil, err + } + visit = func(name string) bool { + if _, done := sorted[name]; done { + return false + } + if progress[name] { + return true + } + progress[name] = true + for _, ref := range deps[name] { + if visit(ref.Name) { + return true + } + } + delete(progress, name) + sorted[name] = len(sorted) + return false + } + for _, node := range byKeys(deps) { + if visit(node.K) { + return nil, errCycle + } + } + return sorted, nil +} + +// dependencies returned an adjacency list of all tables and the tables they depend on. +func dependencies(changes []schema.Change) (map[string][]*schema.Table, error) { + deps := make(map[string][]*schema.Table) + for _, change := range changes { + switch change := change.(type) { + case *schema.AddTable: + for _, fk := range change.T.ForeignKeys { + if err := checkFK(fk); err != nil { + return nil, err + } + if fk.RefTable != change.T { + deps[change.T.Name] = append(deps[change.T.Name], fk.RefTable) + } + } + case *schema.DropTable: + for _, fk := range change.T.ForeignKeys { + if err := checkFK(fk); err != nil { + return nil, err + } + if isDropped(changes, fk.RefTable) { + deps[fk.RefTable.Name] = append(deps[fk.RefTable.Name], fk.Table) + } + } + case *schema.ModifyTable: + for _, c := range change.Changes { + switch c := c.(type) { + case *schema.AddForeignKey: + if err := checkFK(c.F); err != nil { + return nil, err + } + if c.F.RefTable != change.T { + deps[change.T.Name] = append(deps[change.T.Name], c.F.RefTable) + } + case *schema.ModifyForeignKey: + if err := checkFK(c.To); err != nil { + return nil, err + } + if c.To.RefTable != change.T { + deps[change.T.Name] = append(deps[change.T.Name], c.To.RefTable) + } + case *schema.DropForeignKey: + if err := checkFK(c.F); err != nil { + return nil, err + } + if isDropped(changes, c.F.RefTable) { + deps[c.F.RefTable.Name] = append(deps[c.F.RefTable.Name], c.F.Table) + } + } + } + } + } + return deps, nil +} + +func checkFK(fk *schema.ForeignKey) error { + var cause []string + if fk.Table == nil { + cause = append(cause, "child table") + } + if len(fk.Columns) == 0 { + cause = append(cause, "child columns") + } + if fk.RefTable == nil { + cause = append(cause, "parent table") + } + if len(fk.RefColumns) == 0 { + cause = append(cause, "parent columns") + } + if len(cause) != 0 { + return fmt.Errorf("missing %q for foreign key: %q", cause, fk.Symbol) + } + return nil +} + +// table extracts a table from the given change. +func table(change schema.Change) (t string) { + switch change := change.(type) { + case *schema.AddTable: + t = change.T.Name + case *schema.DropTable: + t = change.T.Name + case *schema.ModifyTable: + t = change.T.Name + } + return +} + +// isDropped checks if the given table is marked as a deleted in the changeset. +func isDropped(changes []schema.Change, t *schema.Table) bool { + for _, c := range changes { + if c, ok := c.(*schema.DropTable); ok && c.T.Name == t.Name { + return true + } + } + return false +} + +// CheckChangesScope checks that changes can be applied +// on a schema scope (connection). +func CheckChangesScope(opts migrate.PlanOptions, changes []schema.Change) error { + names := make(map[string]struct{}) + for _, c := range changes { + var t *schema.Table + switch c := c.(type) { + case *schema.ModifySchema: + switch scope := V(opts.SchemaQualifier); { + case !opts.Mode.Is(migrate.PlanModeInPlace): + // The migration plan is generated for deferred execution. + return fmt.Errorf("%T is not allowed when migration plan is scoped to one schema", c) + case scope != "" && scope != c.S.Name: + // Other schemas can not be modified when the migration plan is scoped to one schema. + return fmt.Errorf("modify schema %s is not allowed when migration plan is scoped to schema %s", c.S.Name, scope) + default: + names[c.S.Name] = struct{}{} + continue + } + case *schema.AddSchema, *schema.DropSchema: + return fmt.Errorf("%T is not allowed when migration plan is scoped to one schema", c) + case *schema.AddTable: + t = c.T + case *schema.ModifyTable: + t = c.T + case *schema.DropTable: + t = c.T + default: + continue + } + if t.Schema != nil && t.Schema.Name != "" { + names[t.Schema.Name] = struct{}{} + } + for _, c := range t.Columns { + e, ok := c.Type.Type.(*schema.EnumType) + if ok && e.Schema != nil && e.Schema.Name != "" { + names[t.Schema.Name] = struct{}{} + } + } + } + if len(names) > 1 { + ks := make([]string, 0, len(names)) + for k := range names { + ks = append(ks, k) + } + sort.Strings(ks) + return fmt.Errorf("found %d schemas when migration plan is scoped to one: %q", len(names), ks) + } + return nil +} + +// byKeys sorts a map by keys. +func byKeys[T any](m map[string]T) []struct { + K string + V T +} { + vs := make([]struct { + K string + V T + }, 0, len(m)) + for k, v := range m { + vs = append(vs, struct { + K string + V T + }{k, v}) + } + sort.Slice(vs, func(i, j int) bool { + return vs[i].K < vs[j].K + }) + return vs +} + +// SortOptions allows drivers to customize the behavior of the SortChanges function. +type SortOptions struct { + // FuncDepT reports if a function depends on the given table. + FuncDepT func(*schema.Func, *schema.Table) bool + // FuncDepV reports if a function depends on the given view. + FuncDepV func(*schema.Func, *schema.View) bool + // FuncDepO reports if a function depends on the given object. + FuncDepO func(*schema.Func, schema.Object) bool + // ProcDepT reports if a procedure depends on the given table. + ProcDepT func(*schema.Proc, *schema.Table) bool + // ProcDepO reports if a procedure depends on the given object. + ProcDepO func(*schema.Proc, schema.Object) bool + // ViewDepT reports if a view depends on the given table. + ViewDepT func(*schema.View, *schema.Table) bool + // CompareFuncArgs set to true to compare function arguments. + CompareFuncArgs bool + // DefaultSchema defines the default schema (also known as "search_path") that + // is used by the database to search for objects if no qualifier is provided. + DefaultSchema string +} + +// SortChanges is a helper function to sort to level changes based on their priority. +func SortChanges(changes []schema.Change, opts *SortOptions) []schema.Change { + var views, drop, other []schema.Change + for _, c := range changes { + switch c.(type) { + case *schema.AddView, *schema.DropView, *schema.ModifyView: + views = append(views, c) + case *schema.DropSchema, *schema.DropTable, *schema.DropFunc, *schema.DropProc, *schema.DropObject: + drop = append(drop, c) + default: + other = append(other, c) + } + } + if planned, err := sortViewChanges(views); err == nil { // no cycles. + views = planned + } + // To keep backwards compatibility with previous sorting and also in case we miss any dependency between changes + // (see, dependsOn function) we push views and drop changes to the end, unless there is a dependency requirement. + changes = append(other, append(views, drop...)...) + var ( + hasE = make(map[struct{ e1, e2 schema.Change }]bool) + edges = make(map[schema.Change][]schema.Change) + ) + for _, c1 := range changes { + for _, c2 := range changes { + // Skip checking dependencies between the same change. Also, if the inverse + // dependency is already added, skip it, as circular dependencies are not expected. + if c1 != c2 && !hasE[struct{ e1, e2 schema.Change }{c2, c1}] && dependsOn(c1, c2, V(opts)) { + edges[c1] = append(edges[c1], c2) + hasE[struct{ e1, e2 schema.Change }{c1, c2}] = true + } + } + } + var ( + add func(schema.Change) + added = make(map[schema.Change]bool) + planned = make([]schema.Change, 0, len(changes)) + ) + add = func(c schema.Change) { + if added[c] { + return + } + added[c] = true + for _, d := range edges[c] { + if !added[d] { + add(d) + } + } + planned = append(planned, c) + } + for _, c := range changes { + if !added[c] { + add(c) + } + } + return planned +} + +type ( + // Depender can be implemented by an object to determine if a change to it + // depends on other change, or if other change depends on it. For example: + // A table creation depends on type creation, and a type deletion depends on + // table deletion. + Depender interface { + DependsOn(change, other schema.Change) bool + DependencyOf(change, other schema.Change) bool + } + // RowTyper can be implemented by a type to determine if its source + // is a regular table (e.g., row types). + RowTyper interface { + RowTypeT() *schema.Table + RowTypeV() *schema.View + } +) + +// dependOnOf checks if the given change depends on the other change or +// vice versa based on their underlying object implementation. +func dependOnOf(change, other schema.Change) bool { + switch change := change.(type) { + case *schema.AddObject: + if d, ok := change.O.(Depender); ok && d.DependsOn(change, other) { + return true + } + case *schema.ModifyObject: + if d, ok := change.To.(Depender); ok && d.DependsOn(change, other) { + return true + } + case *schema.DropObject: + if d, ok := change.O.(Depender); ok && d.DependsOn(change, other) { + return true + } + } + switch other := other.(type) { + case *schema.AddObject: + if d, ok := other.O.(Depender); ok && d.DependencyOf(other, change) { + return true + } + case *schema.ModifyObject: + if d, ok := other.To.(Depender); ok && d.DependencyOf(other, change) { + return true + } + case *schema.DropObject: + if d, ok := other.O.(Depender); ok && d.DependencyOf(other, change) { + return true + } + } + return false +} + +// depOfDrops checks if the given object is a dependency of the given change. +func depOfDrop(o schema.Object, c schema.Change) bool { + var deps []schema.Object + switch c := c.(type) { + case *schema.DropTable: + deps = c.T.Deps + for _, t := range c.T.Triggers { + for _, d := range t.Deps { + // If the trigger depends on the table that has FK to its parent, + // this dependency should be ignored as the FK need be dropped first. + if t, ok := d.(*schema.Table); !ok || !refTo(t.ForeignKeys, c.T) { + deps = append(deps, d) + } + } + } + case *schema.DropView: + deps = c.V.Deps + for _, t := range c.V.Triggers { + deps = append(deps, t.Deps...) + } + case *schema.DropFunc: + deps = c.F.Deps + case *schema.DropProc: + deps = c.P.Deps + case *schema.DropTrigger: + deps = c.T.Deps + } + return slices.Contains(deps, o) +} + +// depOfAdd checks if the given change is a creation of a resource exists in the given list. +func depOfAdd(refs []schema.Object, c schema.Change) bool { + var o schema.Object + switch c := c.(type) { + case *schema.AddTable: + return slices.ContainsFunc(refs, func(o schema.Object) bool { + t, ok := o.(*schema.Table) + return ok && SameTable(c.T, t) + }) + case *schema.ModifyTable: + return slices.ContainsFunc(refs, func(o schema.Object) bool { + t, ok := o.(*schema.Table) + return ok && SameTable(c.T, t) + }) + case *schema.AddView: + return slices.ContainsFunc(refs, func(o schema.Object) bool { + v, ok := o.(*schema.View) + return ok && SameView(c.V, v) + }) + case *schema.ModifyView: + return slices.ContainsFunc(refs, func(o schema.Object) bool { + v, ok := o.(*schema.View) + return ok && SameView(c.To, v) + }) + case *schema.AddObject: + o = c.O + case *schema.AddTrigger: + o = c.T + // Check functions and procedures by + // names as they might have overloads. + case *schema.AddFunc: + return slices.ContainsFunc(refs, func(o schema.Object) bool { + f, ok := o.(*schema.Func) + return ok && c.F.Name == f.Name && SameSchema(c.F.Schema, f.Schema) + }) + case *schema.AddProc: + return slices.ContainsFunc(refs, func(o schema.Object) bool { + f, ok := o.(*schema.Proc) + return ok && c.P.Name == f.Name && SameSchema(c.P.Schema, f.Schema) + }) + default: + return false + } + return slices.Contains(refs, o) +} + +// refTo reports if the given foreign keys reference the given table. +func refTo(fks []*schema.ForeignKey, to *schema.Table) bool { + return slices.ContainsFunc(fks, func(fk *schema.ForeignKey) bool { + return SameTable(fk.RefTable, to) + }) +} + +// typeDependsOnT reports if the declaration of type "t" depends on the table. +func typeDependsOnT(t schema.Type, tt *schema.Table) bool { + rt, ok := schema.UnderlyingType(t).(RowTyper) + if !ok { + return false + } + rowT := rt.RowTypeT() + return rowT != nil && SameTable(rowT, tt) +} + +// SameView reports if the two objects represent the same view. +func SameView(v1, v2 *schema.View) bool { + if v1 == nil || v2 == nil { + return v1 == v2 + } + return v1.Name == v2.Name && SameSchema(v1.Schema, v2.Schema) +} + +// SameTable reports if the two objects represent the same table. +func SameTable(t1, t2 *schema.Table) bool { + if t1 == nil || t2 == nil { + return t1 == t2 + } + return t1.Name == t2.Name && SameSchema(t1.Schema, t2.Schema) +} + +// SameSchema reports if the given schemas are the same. +// Objects can be different as they might reside in two +// different states (current and desired). +func SameSchema(s1, s2 *schema.Schema) bool { + if s1 == nil || s2 == nil { + return s1 == s2 + } + return s1.Name == s2.Name +} diff --git a/database/atlas/internal/sqlx/plan_test.go b/database/atlas/internal/sqlx/plan_test.go new file mode 100644 index 00000000..078ba0e4 --- /dev/null +++ b/database/atlas/internal/sqlx/plan_test.go @@ -0,0 +1,199 @@ +// Copyright 2021-present The Atlas Authors. All rights reserved. +// This source code is licensed under the Apache 2.0 license found +// in the LICENSE file in the root directory of this source tree. + +package sqlx + +import ( + "fmt" + "testing" + + "ariga.io/atlas/sql/migrate" + "ariga.io/atlas/sql/schema" + + "github.com/stretchr/testify/require" +) + +func TestDetachCycles(t *testing.T) { + users := &schema.Table{ + Name: "users", + Columns: []*schema.Column{ + {Name: "id", Type: &schema.ColumnType{Raw: "bigint"}}, + {Name: "workplace_id", Type: &schema.ColumnType{Raw: "bigint", Null: true}}, + }, + } + workplaces := &schema.Table{ + Name: "workplaces", + Columns: []*schema.Column{ + {Name: "id", Type: &schema.ColumnType{Raw: "bigint"}}, + {Name: "owner_id", Type: &schema.ColumnType{Raw: "bigint", Null: true}}, + }, + } + users.ForeignKeys = []*schema.ForeignKey{ + {Symbol: "workplace", Table: users, Columns: users.Columns[1:2], RefTable: workplaces, RefColumns: workplaces.Columns[:1]}, + } + changes := []schema.Change{&schema.AddTable{T: workplaces}, &schema.AddTable{T: users}} + planned, err := DetachCycles(changes) + require.NoError(t, err) + require.Equal(t, changes, planned) + + deletion := []schema.Change{&schema.DropTable{T: users}, &schema.DropTable{T: workplaces}} + planned, err = DetachCycles(deletion) + require.NoError(t, err) + require.Equal(t, deletion, planned) + + // Create a circular reference. + workplaces.ForeignKeys = []*schema.ForeignKey{ + {Symbol: "owner", Table: workplaces, Columns: workplaces.Columns[1:], RefTable: users, RefColumns: users.Columns[:1]}, + } + // Add a self-ref foreign-key. + users.Columns = append(users.Columns, &schema.Column{Name: "spouse_id", Type: &schema.ColumnType{Raw: "bigint", Null: true}}) + users.ForeignKeys = append(users.ForeignKeys, &schema.ForeignKey{Symbol: "spouse", Table: users, Columns: users.Columns[2:], RefTable: users, RefColumns: users.Columns[:1]}) + + planned, err = DetachCycles(changes) + require.NoError(t, err) + require.Len(t, planned, 4) + require.Empty(t, planned[0].(*schema.AddTable).T.ForeignKeys) + require.NotEmpty(t, planned[1].(*schema.AddTable).T.ForeignKeys) + require.Equal(t, &schema.ModifyTable{ + T: workplaces, + Changes: []schema.Change{ + &schema.AddForeignKey{ + F: &schema.ForeignKey{Symbol: "owner", Table: workplaces, Columns: workplaces.Columns[1:], RefTable: users, RefColumns: users.Columns[:1]}, + }, + }, + }, planned[2]) + require.Equal(t, &schema.ModifyTable{ + T: users, + Changes: []schema.Change{ + &schema.AddForeignKey{ + F: &schema.ForeignKey{Symbol: "workplace", Table: users, Columns: users.Columns[1:2], RefTable: workplaces, RefColumns: workplaces.Columns[:1]}, + }, + }, + }, planned[3]) + + planned, err = DetachCycles(deletion) + require.NoError(t, err) + require.Equal(t, &schema.ModifyTable{ + T: users, + Changes: []schema.Change{ + &schema.DropForeignKey{ + F: &schema.ForeignKey{Symbol: "workplace", Table: users, Columns: users.Columns[1:2], RefTable: workplaces, RefColumns: workplaces.Columns[:1]}, + }, + }, + }, planned[0]) + require.Equal(t, &schema.ModifyTable{ + T: workplaces, + Changes: []schema.Change{ + &schema.DropForeignKey{ + F: &schema.ForeignKey{Symbol: "owner", Table: workplaces, Columns: workplaces.Columns[1:], RefTable: users, RefColumns: users.Columns[:1]}, + }, + }, + }, planned[1]) + users.ForeignKeys = nil + workplaces.ForeignKeys = nil + require.Equal(t, deletion, planned[2:]) + + // Delete associated table and foreign-key. + users.AddForeignKeys( + schema.NewForeignKey("workplace").AddColumns(users.Columns[1:2]...).SetRefTable(workplaces).AddRefColumns(workplaces.Columns[:1]...), + ) + changes = []schema.Change{&schema.DropTable{T: workplaces}, &schema.ModifyTable{ + T: users, + Changes: []schema.Change{ + &schema.DropForeignKey{F: users.ForeignKeys[0]}, + }, + }} + planned, err = DetachCycles(changes) + require.NoError(t, err) + require.Equal(t, []schema.Change{changes[1], changes[0]}, planned) +} + +func TestConsistentOrder(t *testing.T) { + newT := func(n string) *schema.Table { return schema.NewTable(n).AddColumns(schema.NewIntColumn("id", "int")) } + t1, t2, t3 := newT("t1"), newT("t2"), newT("t3") + t2.AddForeignKeys(schema.NewForeignKey("t1").AddColumns(t2.Columns[0]).SetRefTable(t1).AddRefColumns(t1.Columns[0])) + t3.AddForeignKeys(schema.NewForeignKey("t1").AddColumns(t3.Columns[0]).SetRefTable(t1).AddRefColumns(t1.Columns[0])) + order := func() string { + planned, err := DetachCycles([]schema.Change{ + &schema.AddTable{T: t1}, + &schema.AddTable{T: t2}, + &schema.AddTable{T: t3}, + }) + require.NoError(t, err) + return fmt.Sprintf("%s,%s,%s", planned[0].(*schema.AddTable).T.Name, planned[1].(*schema.AddTable).T.Name, planned[2].(*schema.AddTable).T.Name) + } + v := order() + for i := 0; i < 100; i++ { + require.Equal(t, v, order(), "inconsistent order") + } +} + +func TestCheckChangesScope(t *testing.T) { + err := CheckChangesScope(migrate.PlanOptions{}, []schema.Change{ + &schema.AddSchema{}, + }) + require.EqualError(t, err, "*schema.AddSchema is not allowed when migration plan is scoped to one schema") + err = CheckChangesScope(migrate.PlanOptions{}, []schema.Change{ + &schema.ModifySchema{ + S: schema.New("s1"), + Changes: []schema.Change{ + &schema.AddAttr{A: &schema.Collation{V: "utf8"}}, + }, + }, + }) + require.EqualError(t, err, "*schema.ModifySchema is not allowed when migration plan is scoped to one schema") + err = CheckChangesScope(migrate.PlanOptions{}, []schema.Change{ + &schema.DropSchema{}, + }) + require.EqualError(t, err, "*schema.DropSchema is not allowed when migration plan is scoped to one schema") + err = CheckChangesScope(migrate.PlanOptions{}, []schema.Change{ + &schema.AddTable{T: schema.NewTable("users").SetSchema(schema.New("s1"))}, + &schema.AddTable{T: schema.NewTable("users").SetSchema(schema.New("s2"))}, + }) + require.EqualError(t, err, "found 2 schemas when migration plan is scoped to one: [\"s1\" \"s2\"]") +} + +func TestSameTable(t *testing.T) { + t1 := schema.NewTable("t1") + require.True(t, SameTable(t1, t1)) + require.True(t, SameTable(t1, schema.NewTable("t1")), "same copy") + require.False(t, SameTable(t1, schema.NewTable("t2"))) + + t1.SetSchema(schema.New("public")) + require.True(t, SameTable(t1, t1)) + require.True(t, SameTable(t1, schema.NewTable("t1").SetSchema(schema.New("public"))), "same copy") + require.False(t, SameTable(t1, schema.NewTable("t1"))) + require.False(t, SameTable(t1, schema.NewTable("t1").SetSchema(schema.New("private")))) +} + +func TestSameView(t *testing.T) { + v1 := schema.NewView("v1", "") + require.True(t, SameView(v1, v1)) + require.True(t, SameView(v1, schema.NewView("v1", "")), "same copy") + require.False(t, SameView(v1, schema.NewView("v2", ""))) + + v1.SetSchema(schema.New("public")) + require.True(t, SameView(v1, v1)) + require.True(t, SameView(v1, schema.NewView("v1", "").SetSchema(schema.New("public"))), "same copy") + require.False(t, SameView(v1, schema.NewView("v1", ""))) + require.False(t, SameView(v1, schema.NewView("v1", "").SetSchema(schema.New("private")))) +} + +func TestSortDropTables_WithFK(t *testing.T) { + t1 := schema.NewTable("t1").AddColumns(schema.NewColumn("c1")) + t2 := schema.NewTable("t2").AddColumns(schema.NewColumn("c1"), schema.NewColumn("c2")) + t2.AddForeignKeys( + schema.NewForeignKey("FK_t2_t1"). + AddColumns(t2.Columns[1]). + AddRefColumns(t1.Columns[0]). + SetRefTable(t1), + ) + t1.Triggers = []*schema.Trigger{ + {Table: t1, Deps: []schema.Object{t2}}, + } + changes := []schema.Change{&schema.DropTable{T: t2}, &schema.DropTable{T: t1}} + require.Equal(t, []schema.Change{changes[0], changes[1]}, SortChanges(changes, nil)) + changes = []schema.Change{&schema.DropTable{T: t1}, &schema.DropTable{T: t2}} + require.Equal(t, []schema.Change{changes[1], changes[0]}, SortChanges(changes, nil)) +} diff --git a/database/atlas/internal/sqlx/sqlx.go b/database/atlas/internal/sqlx/sqlx.go new file mode 100644 index 00000000..e8d16a75 --- /dev/null +++ b/database/atlas/internal/sqlx/sqlx.go @@ -0,0 +1,734 @@ +// Copyright 2021-present The Atlas Authors. All rights reserved. +// This source code is licensed under the Apache 2.0 license found +// in the LICENSE file in the root directory of this source tree. + +package sqlx + +import ( + "bytes" + "context" + "database/sql" + "database/sql/driver" + "errors" + "fmt" + "io" + "reflect" + "slices" + "strconv" + "strings" + "unicode" + + "ariga.io/atlas/sql/schema" +) + +type ( + // ExecQueryCloser is the interface that groups + // Close with the schema.ExecQuerier methods. + ExecQueryCloser interface { + schema.ExecQuerier + io.Closer + } + nopCloser struct { + schema.ExecQuerier + } +) + +// Close implements the io.Closer interface. +func (nopCloser) Close() error { return nil } + +// SingleConn returns a closable single connection from the given ExecQuerier. +// If the ExecQuerier is already bound to a single connection (e.g. Tx, Conn), +// the connection will return as-is with a NopCloser. +func SingleConn(ctx context.Context, conn schema.ExecQuerier) (ExecQueryCloser, error) { + // A standard sql.DB or a wrapper of it. + if opener, ok := conn.(interface { + Conn(context.Context) (*sql.Conn, error) + }); ok { + return opener.Conn(ctx) + } + // Tx and Conn are bounded to a single connection. + // We use sql/driver.Tx to cover also custom Tx structs. + _, ok1 := conn.(driver.Tx) + _, ok2 := conn.(*sql.Conn) + if ok1 || ok2 { + return nopCloser{ExecQuerier: conn}, nil + } + return nil, fmt.Errorf("cannot obtain a single connection from %T", conn) +} + +// ValidString reports if the given string is not null and valid. +func ValidString(s sql.NullString) bool { + return s.Valid && s.String != "" && strings.ToLower(s.String) != "null" +} + +// ScanOne scans one record and closes the rows at the end. +func ScanOne(rows *sql.Rows, dest ...any) (returnErr error) { + defer func() { returnErr = errors.Join(returnErr, rows.Close()) }() + if !rows.Next() { + return sql.ErrNoRows + } + if err := rows.Scan(dest...); err != nil { + return err + } + return rows.Err() +} + +// ScanNullBool scans one sql.NullBool record and closes the rows at the end. +func ScanNullBool(rows *sql.Rows) (sql.NullBool, error) { + var b sql.NullBool + return b, ScanOne(rows, &b) +} + +type ( + // ScanStringer groups the fmt.Stringer and sql.Scanner interfaces. + ScanStringer interface { + fmt.Stringer + sql.Scanner + } + // nullString is a sql.NullString that implements the ScanStringer interface. + nullString struct{ sql.NullString } +) + +func (s nullString) String() string { return s.NullString.String } +func (s *nullString) Scan(v any) error { return s.NullString.Scan(v) } + +// SchemaFKs scans the rows and adds the foreign-key to the schema table. +// Reference elements are added as stubs and should be linked manually by the +// caller. +func SchemaFKs(s *schema.Schema, rows *sql.Rows) error { + return TypedSchemaFKs[*nullString](s, rows) +} + +// FKAttrScanner allows extending foreign-keys +// scanning with extra attributes. +type FKAttrScanner struct { + Columns func() []any + ScanFunc func(*schema.ForeignKey, []any) error +} + +// TypedSchemaFKs is a version of SchemaFKs that allows to specify the type of +// used to scan update and delete actions from the database. +func TypedSchemaFKs[T ScanStringer](s *schema.Schema, rows *sql.Rows, attr ...*FKAttrScanner) error { + for rows.Next() { + var ( + updateAction, deleteAction = V(new(T)), V(new(T)) + name, table, column, tSchema, refTable, refColumn, refSchema string + columns = []any{ + &name, &table, &column, &tSchema, &refTable, &refColumn, &refSchema, &updateAction, &deleteAction, + } + origin = len(columns) + ) + // Allows to scan extra attributes. + switch { + case len(attr) > 1: + return fmt.Errorf("unexpected number of attributes scanners: %d", len(attr)) + case len(attr) == 1: + columns = append(columns, attr[0].Columns()...) + } + if err := rows.Scan(columns...); err != nil { + return err + } + t, ok := s.Table(table) + if !ok { + return fmt.Errorf("table %q was not found in schema", table) + } + fk, ok := t.ForeignKey(name) + if !ok { + fk = &schema.ForeignKey{ + Symbol: name, + Table: t, + RefTable: t, + OnUpdate: schema.ReferenceOption(updateAction.String()), + OnDelete: schema.ReferenceOption(deleteAction.String()), + } + switch { + // Self reference. + case tSchema == refSchema && refTable == table: + // Reference to the same schema. + case tSchema == refSchema && refTable != table: + if fk.RefTable, ok = s.Table(refTable); !ok { + fk.RefTable = &schema.Table{Name: refTable, Schema: s} + } + // Reference to an external schema. + case tSchema != refSchema: + fk.RefTable = &schema.Table{Name: refTable, Schema: &schema.Schema{Name: refSchema}} + } + t.ForeignKeys = append(t.ForeignKeys, fk) + } + c, ok := t.Column(column) + if !ok { + return fmt.Errorf("column %q was not found for fk %q", column, fk.Symbol) + } + // Rows are ordered by ORDINAL_POSITION that specifies + // the position of the column in the FK definition. + if _, ok := fk.Column(c.Name); !ok { + fk.Columns = append(fk.Columns, c) + c.ForeignKeys = append(c.ForeignKeys, fk) + } + // Stub referenced columns or link if it's a self-reference. + var rc *schema.Column + if fk.Table != fk.RefTable { + rc = &schema.Column{Name: refColumn} + } else if c, ok := t.Column(refColumn); ok { + rc = c + } else { + return fmt.Errorf("referenced column %q was not found for fk %q", refColumn, fk.Symbol) + } + if _, ok := fk.RefColumn(rc.Name); !ok { + fk.RefColumns = append(fk.RefColumns, rc) + } + if len(attr) == 1 { + if err := attr[0].ScanFunc(fk, columns[origin:]); err != nil { + return err + } + } + } + return nil +} + +// LinkSchemaTables links foreign-key stub tables/columns to actual elements. +func LinkSchemaTables(schemas []*schema.Schema) { + byName := make(map[string]map[string]*schema.Table) + for _, s := range schemas { + byName[s.Name] = make(map[string]*schema.Table) + for _, t := range s.Tables { + t.Schema = s + byName[s.Name][t.Name] = t + } + } + for _, s := range schemas { + for _, t := range s.Tables { + for _, fk := range t.ForeignKeys { + rs, ok := byName[fk.RefTable.Schema.Name] + if !ok { + continue + } + ref, ok := rs[fk.RefTable.Name] + if !ok { + continue + } + fk.RefTable = ref + for i, c := range fk.RefColumns { + rc, ok := ref.Column(c.Name) + if ok { + fk.RefColumns[i] = rc + } + } + } + } + } +} + +// ValuesEqual checks if the 2 string slices are equal (including their order). +func ValuesEqual(v1, v2 []string) bool { + if len(v1) != len(v2) { + return false + } + for i := range v1 { + if v1[i] != v2[i] { + return false + } + } + return true +} + +// ModeInspectSchema returns the InspectMode or its default. +func ModeInspectSchema(o *schema.InspectOptions) schema.InspectMode { + if o != nil && o.Mode != 0 { + return o.Mode + } + return modeSchemaOrAll(V(o).Exclude, "*") +} + +// ModeInspectRealm returns the InspectMode or its default. +func ModeInspectRealm(o *schema.InspectRealmOption) schema.InspectMode { + if o != nil && o.Mode != 0 { + return o.Mode + } + return modeSchemaOrAll(V(o).Exclude, "*.*") +} + +// modeSchemaOrAll returns the inspect mode based on the exclude patterns. +func modeSchemaOrAll(exclude []string, match string) schema.InspectMode { + if slices.Contains(exclude, match) { + return schema.InspectSchemas + } + return schema.InspectSchemas | schema.InspectTables | schema.InspectViews | schema.InspectFuncs | + schema.InspectTypes | schema.InspectObjects | schema.InspectTriggers +} + +// A Builder provides a syntactic sugar API for writing SQL statements. +type Builder struct { + bytes.Buffer + QuoteOpening byte // quoting identifiers + QuoteClosing byte // quoting identifiers + Schema *string // schema qualifier + Indent string // indentation string + level int // current indentation level +} + +// P writes a list of phrases to the builder separated and +// suffixed with whitespace. +func (b *Builder) P(phrases ...string) *Builder { + for _, p := range phrases { + if p == "" { + continue + } + if b.Len() > 0 && !slices.Contains([]byte{' ', '(', '\n'}, b.lastByte()) { + _ = b.WriteByte(' ') + } + _, _ = b.WriteString(p) + if p[len(p)-1] != ' ' { + _ = b.WriteByte(' ') + } + } + return b +} + +// Int64 writes the given value to the builder in base 10. +func (b *Builder) Int64(v int64) *Builder { + return b.P(strconv.FormatInt(v, 10)) +} + +// Ident writes the given string quoted as an SQL identifier. +func (b *Builder) Ident(s string) *Builder { + if s != "" { + _ = b.WriteByte(b.QuoteOpening) + _, _ = b.WriteString(s) + _ = b.WriteByte(b.QuoteClosing) + _ = b.WriteByte(' ') + } + return b +} + +// View writes the view identifier to the builder, prefixed +// with the schema name if exists. +func (b *Builder) View(v *schema.View) *Builder { + return b.mayQualify(v.Schema, v.Name) +} + +// Table writes the table identifier to the builder, prefixed +// with the schema name if exists. +func (b *Builder) Table(t *schema.Table) *Builder { + return b.mayQualify(t.Schema, t.Name) +} + +// RefTable writes the referenced/parent table identifier to the builder. +// Unlike the Table method, RefTable prefix the table with its schema qualifier +// if the "Schema" is set to nil, or if the schema is set to empty (schema-scope), +// and the parent table is in a different schema. For example: +// +// CREATE TABLE child ( +// id INT PRIMARY KEY, +// parent_id1 INT REFERENCES other1.parent(id) +// parent_id2 INT REFERENCES other2.parent(id) +// ); +// +// This case is possible only if the child table (and its schema) was loaded with +// baseline schema (contains other1, other2) or with external references to resources +// not managed by this scope. +func (b *Builder) RefTable(childT, parentT *schema.Table) *Builder { + // Default schema-scope config (no schema qualifier), both schemas are known, and parent table resides in a different schema. + if b.Schema != nil && *b.Schema == "" && V(childT.Schema).Name != "" && V(parentT.Schema).Name != "" && !SameSchema(childT.Schema, parentT.Schema) { + b.Ident(parentT.Schema.Name) + b.rewriteLastByte('.') + b.Ident(parentT.Name) + return b + } + return b.Table(parentT) +} + +// TableColumn writes the table's resource identifier to the builder, prefixed +// with the schema name if exists. +func (b *Builder) TableColumn(t *schema.Table, c *schema.Column) *Builder { + return b.mayQualify(t.Schema, t.Name, c.Name) +} + +// Func writes the function identifier to the builder, prefixed +// with the schema name if exists. +func (b *Builder) Func(f *schema.Func) *Builder { + return b.mayQualify(f.Schema, f.Name) +} + +// FuncCall writes the function identifier to the builder as a function call, +func (b *Builder) FuncCall(f *schema.Func, args ...string) *Builder { + b.Func(f).rewriteLastByte('(') + b.MapComma(args, func(i int, b *Builder) { + _, _ = b.WriteString(args[i]) + }) + _ = b.WriteByte(')') + return b +} + +// Proc writes the procedure identifier to the builder, prefixed +// with the schema name if exists. +func (b *Builder) Proc(p *schema.Proc) *Builder { + return b.mayQualify(p.Schema, p.Name) +} + +// ProcCall writes the procedure identifier to the builder as a procedure call, +func (b *Builder) ProcCall(p *schema.Proc, args ...string) *Builder { + b.Proc(p).rewriteLastByte('(') + b.MapComma(args, func(i int, b *Builder) { + _, _ = b.WriteString(args[i]) + }) + _ = b.WriteByte(')') + return b +} + +// TableResource writes the table resource identifier to the builder, prefixed +// with the schema name if exists. +func (b *Builder) TableResource(t *schema.Table, r any) *Builder { + switch c := r.(type) { + case *schema.Column: + return b.TableColumn(t, c) + case *schema.Index: + return b.mayQualify(t.Schema, t.Name, c.Name) + default: + panic(fmt.Sprintf("unexpected table resource: %T", r)) + } +} + +// ViewResource writes the view resource identifier to the builder, prefixed +// with the schema name if exists. +func (b *Builder) ViewResource(v *schema.View, r any) *Builder { + switch c := r.(type) { + case *schema.Column: + return b.mayQualify(v.Schema, v.Name, c.Name) + case *schema.Index: + return b.mayQualify(v.Schema, v.Name, c.Name) + default: + panic(fmt.Sprintf("unexpected view resource: %T", r)) + } +} + +// SchemaResource writes the schema resource identifier to the builder, prefixed +// with the schema name if exists. +func (b *Builder) SchemaResource(s *schema.Schema, name string) *Builder { + return b.mayQualify(s, name) +} + +func (b *Builder) mayQualify(s *schema.Schema, top string, children ...string) *Builder { + switch { + // Custom qualifier. + case b.Schema != nil: + // Empty means skip prefix. + if *b.Schema != "" { + b.Ident(*b.Schema) + b.rewriteLastByte('.') + } + // Default schema qualifier. + case s != nil && s.Name != "": + b.Ident(s.Name) + b.rewriteLastByte('.') + } + b.Ident(top) + for _, ident := range children { + b.rewriteLastByte('.') + b.Ident(ident) + } + return b +} + +// IndentIn adds one indentation in. +func (b *Builder) IndentIn() *Builder { + b.level++ + return b +} + +// IndentOut removed one indentation level. +func (b *Builder) IndentOut() *Builder { + b.level-- + return b +} + +// NL adds line break and prefix the new line with +// indentation in case indentation is enabled. +func (b *Builder) NL() *Builder { + if b.Indent != "" { + if b.lastByte() == ' ' { + b.rewriteLastByte('\n') + } else { + _ = b.WriteByte('\n') + } + _, _ = b.WriteString(strings.Repeat(b.Indent, b.level)) + } + return b +} + +// Comma writes a comma in case the buffer is not empty, or +// replaces the last char if it is a whitespace. +func (b *Builder) Comma() *Builder { + switch { + case b.Len() == 0: + case b.lastByte() == ' ': + b.rewriteLastByte(',') + _ = b.WriteByte(' ') + default: + _, _ = b.WriteString(", ") + } + return b +} + +// MapComma maps the slice x using the function f and joins the result with +// a comma separating between the written elements. +func (b *Builder) MapComma(x any, f func(i int, b *Builder)) *Builder { + s := reflect.ValueOf(x) + for i := 0; i < s.Len(); i++ { + if i > 0 { + b.Comma() + } + f(i, b) + } + return b +} + +// Quote wraps the given function with a single quote and a prefix +func (b *Builder) Quote(prefix string, fn func(b *Builder)) *Builder { + _, _ = b.WriteString(prefix) + _ = b.WriteByte('\'') + fn(b) + if b.lastByte() != ' ' { + _ = b.WriteByte('\'') + } else { + b.rewriteLastByte('\'') + } + return b +} + +// MapIndent is like MapComma, but writes a new line before each element. +func (b *Builder) MapIndent(x any, f func(i int, b *Builder)) *Builder { + return b.MapComma(x, func(i int, b *Builder) { + f(i, b.NL()) + }) +} + +// MapCommaErr is like MapComma, but returns an error if f returns an error. +func (b *Builder) MapCommaErr(x any, f func(i int, b *Builder) error) error { + s := reflect.ValueOf(x) + for i := 0; i < s.Len(); i++ { + if i > 0 { + b.Comma() + } + if err := f(i, b); err != nil { + return err + } + } + return nil +} + +// MapIndentErr is like MapCommaErr, but writes a new line before each element. +func (b *Builder) MapIndentErr(x any, f func(i int, b *Builder) error) error { + return b.MapCommaErr(x, func(i int, b *Builder) error { + return f(i, b.NL()) + }) +} + +// Wrap wraps the written string with parentheses. +func (b *Builder) Wrap(f func(b *Builder)) *Builder { + _ = b.WriteByte('(') + f(b) + if b.lastByte() != ' ' { + _ = b.WriteByte(')') + } else { + b.rewriteLastByte(')') + } + return b +} + +// WrapErr wraps the written string with parentheses +func (b *Builder) WrapErr(f func(b *Builder) error) error { + var err error + b.Wrap(func(b *Builder) { err = f(b) }) + return err +} + +// WrapIndent is like Wrap but with extra level of indentation. +func (b *Builder) WrapIndent(f func(b *Builder)) *Builder { + return b.Wrap(func(b *Builder) { + b.IndentIn() + f(b) + b.IndentOut() + b.NL() + }) +} + +// WrapIndentErr is like WrapErr but with extra level of indentation. +func (b *Builder) WrapIndentErr(f func(b *Builder) error) error { + var err error + b.Wrap(func(b *Builder) { + b.IndentIn() + err = f(b) + b.IndentOut() + b.NL() + }) + return err +} + +// Clone returns a duplicate of the builder. +func (b *Builder) Clone() *Builder { + return &Builder{ + QuoteOpening: b.QuoteOpening, + QuoteClosing: b.QuoteClosing, + Buffer: *bytes.NewBufferString(b.Buffer.String()), + } +} + +// String overrides the Buffer.String method and ensure no spaces pad the returned statement. +func (b *Builder) String() string { + return strings.TrimSpace(b.Buffer.String()) +} + +func (b *Builder) lastByte() byte { + if b.Len() == 0 { + return 0 + } + buf := b.Bytes() + return buf[len(buf)-1] +} + +func (b *Builder) rewriteLastByte(c byte) { + if b.Len() == 0 { + return + } + buf := b.Bytes() + buf[len(buf)-1] = c +} + +// IsQuoted reports if the given string is quoted with one of the given quotes (e.g. ', ", `). +func IsQuoted(s string, q ...byte) bool { + last := len(s) - 1 + if last < 1 { + return false + } +Top: + for _, quote := range q { + if s[0] != quote || s[last] != quote { + continue + } + for i := 1; i < last-1; i++ { + switch c := s[i]; { + case c == '\\', c == quote && s[i+1] == quote: + i++ + // Accept only escaped quotes and reject otherwise. + case c == quote: + continue Top + } + } + return true + } + return false +} + +// IsLiteralBool reports if the given string is a valid literal bool. +func IsLiteralBool(s string) bool { + _, err := strconv.ParseBool(s) + return err == nil +} + +// IsLiteralNumber reports if the given string is a literal number. +func IsLiteralNumber(s string) bool { + // Hex digits. + if strings.HasPrefix(s, "0x") || strings.HasPrefix(s, "0X") { + // Some databases allow odd length hex string. + _, err := strconv.ParseUint(s[2:], 16, 64) + return err == nil + } + // Digits with optional exponent. + _, err := strconv.ParseFloat(s, 64) + return err == nil +} + +// DefaultValue returns the string represents the DEFAULT of a column. +func DefaultValue(c *schema.Column) (string, bool) { + switch x := schema.UnderlyingExpr(c.Default).(type) { + case nil: + return "", false + case *schema.Literal: + return x.V, true + case *schema.RawExpr: + return x.X, true + default: + panic(fmt.Sprintf("unexpected default value type: %T", x)) + } +} + +// MayWrap ensures the given string is wrapped with parentheses. +// Used by the different drivers to turn strings valid expressions. +func MayWrap(s string) string { + n := len(s) - 1 + if len(s) < 2 || s[0] != '(' || s[n] != ')' || !balanced(s[1:n]) { + return "(" + s + ")" + } + return s +} + +func balanced(expr string) bool { + return ExprLastIndex(expr) == len(expr)-1 +} + +// ExprLastIndex scans the first expression in the given string until +// its end and returns its last index. +func ExprLastIndex(expr string) int { + var l, r int + for i := 0; i < len(expr); i++ { + Top: + switch expr[i] { + case '(': + l++ + case ')': + r++ + // String or identifier. + case '\'', '"', '`': + for j := i + 1; j < len(expr); j++ { + switch expr[j] { + case '\\': + j++ + case expr[i]: + i = j + break Top + } + } + // Unexpected EOS. + return -1 + } + // Balanced parens and we reached EOS or a terminator. + if l == r && (i == len(expr)-1 || expr[i+1] == ',') { + return i + } else if r > l { + return -1 + } + } + return -1 +} + +// ReverseChanges reverses the order of the changes. +func ReverseChanges(c []schema.Change) { + for i, n := 0, len(c); i < n/2; i++ { + c[i], c[n-i-1] = c[n-i-1], c[i] + } +} + +// P returns a pointer to v. +func P[T any](v T) *T { + return &v +} + +// V returns the value p is pointing to. +// If p is nil, the zero value is returned. +func V[T any](p *T) (v T) { + if p != nil { + v = *p + } + return +} + +// IsUint reports whether the string represents an unsigned integer. +func IsUint(s string) bool { + for _, r := range s { + if !unicode.IsDigit(r) { + return false + } + } + return true +} diff --git a/database/atlas/internal/sqlx/sqlx_oss.go b/database/atlas/internal/sqlx/sqlx_oss.go new file mode 100644 index 00000000..b405be7e --- /dev/null +++ b/database/atlas/internal/sqlx/sqlx_oss.go @@ -0,0 +1,220 @@ +// Copyright 2021-present The Atlas Authors. All rights reserved. +// This source code is licensed under the Apache 2.0 license found +// in the LICENSE file in the root directory of this source tree. + +//go:build !ent + +package sqlx + +import ( + "slices" + + "ariga.io/atlas/sql/schema" +) + +// sortViewChanges is an optional function to sort to views by their dependencies. +func sortViewChanges(changes []schema.Change) ([]schema.Change, error) { + return changes, nil // unimplemented. +} + +type triggerOwner interface { + Trigger(string) (*schema.Trigger, bool) +} + +type triggerDiffInput struct { + From, To triggerOwner + FromTriggers, ToTriggers []*schema.Trigger +} + +func (*Diff) triggerDiff(_ triggerDiffInput, _ *schema.DiffOptions) ([]schema.Change, error) { + return nil, nil // unimplemented. +} + +func (*Diff) askForColumns(_ *schema.Table, changes []schema.Change, _ *schema.DiffOptions) ([]schema.Change, error) { + return changes, nil // unimplemented. +} + +func (*Diff) askForIndexes(_ string, changes []schema.Change, _ *schema.DiffOptions) ([]schema.Change, error) { + return changes, nil // unimplemented. +} + +func (*Diff) fixRenames(changes schema.Changes) schema.Changes { + return changes // unimplemented. +} + +// dependsOn reports if the given change depends on the other change. +func dependsOn(c1, c2 schema.Change, _ SortOptions) bool { + if dependOnOf(c1, c2) { + return true + } + switch c1 := c1.(type) { + case *schema.DropSchema: + return dropSchemaDependsOn(c1, c2) + case *schema.AddTable: + return addTableDependsOn(c1, c2) + case *schema.DropTable: + return dropTableDependsOn(c1, c2) + case *schema.ModifyTable: + return modifyTableDependsOn(c1, c2) + case *schema.DropObject: + return dropObjectDependsOn(c1, c2) + } + return false +} + +func dropSchemaDependsOn(c1 *schema.DropSchema, c2 schema.Change) bool { + switch c2 := c2.(type) { + case *schema.DropTable: + // Schema must be dropped after all its tables and references to them. + return SameSchema(c1.S, c2.T.Schema) || slices.ContainsFunc(c2.T.ForeignKeys, func(fk *schema.ForeignKey) bool { + return SameSchema(c1.S, fk.RefTable.Schema) + }) + case *schema.ModifyTable: + return SameSchema(c1.S, c2.T.Schema) || slices.ContainsFunc(c2.Changes, func(c schema.Change) bool { + fk, ok := c.(*schema.DropForeignKey) + return ok && SameSchema(c1.S, fk.F.RefTable.Schema) + }) + } + return false +} + +func addTableDependsOn(c1 *schema.AddTable, c2 schema.Change) bool { + switch c2 := c2.(type) { + case *schema.AddSchema: + return c1.T.Schema.Name == c2.S.Name + case *schema.DropTable: + // Table recreation. + return c1.T.Name == c2.T.Name && SameSchema(c1.T.Schema, c2.T.Schema) + case *schema.AddTable: + if refTo(c1.T.ForeignKeys, c2.T) { + return true + } + if slices.ContainsFunc(c1.T.Columns, func(c *schema.Column) bool { + return c.Type != nil && typeDependsOnT(c.Type.Type, c2.T) + }) { + return true + } + case *schema.ModifyTable: + if (c1.T.Name != c2.T.Name || !SameSchema(c1.T.Schema, c2.T.Schema)) && refTo(c1.T.ForeignKeys, c2.T) { + return true + } + case *schema.AddObject: + t, ok := c2.O.(schema.Type) + if ok && slices.ContainsFunc(c1.T.Columns, func(c *schema.Column) bool { + return schema.IsType(c.Type.Type, t) + }) { + return true + } + } + return depOfAdd(c1.T.Deps, c2) +} + +// dropTableDependsOn reports if a table drop must occur after +// all resources that rely on it are dropped. +func dropTableDependsOn(c1 *schema.DropTable, c2 schema.Change) bool { + switch c2 := c2.(type) { + case *schema.DropTable: + // References to this table, must be dropped first. + if refTo(c2.T.ForeignKeys, c1.T) { + return true + } + if slices.ContainsFunc(c2.T.Columns, func(c *schema.Column) bool { + return c.Type != nil && typeDependsOnT(c.Type.Type, c1.T) + }) { + return true + } + case *schema.ModifyTable: + if slices.ContainsFunc(c2.Changes, func(c schema.Change) bool { + switch c := c.(type) { + case *schema.DropForeignKey: + return refTo([]*schema.ForeignKey{c.F}, c1.T) + case *schema.DropColumn: + return c.C.Type != nil && typeDependsOnT(c.C.Type.Type, c1.T) + } + return false + }) { + return true + } + } + return depOfDrop(c1.T, c2) +} + +func modifyTableDependsOn(c1 *schema.ModifyTable, c2 schema.Change) bool { + switch c2 := c2.(type) { + case *schema.AddTable: + // Table modification relies on its creation. + if c1.T.Name == c2.T.Name && SameSchema(c1.T.Schema, c2.T.Schema) { + return true + } + // Tables need to be created before referencing them. + if slices.ContainsFunc(c1.Changes, func(c schema.Change) bool { + switch c := c.(type) { + case *schema.AddForeignKey: + return refTo([]*schema.ForeignKey{c.F}, c2.T) + case *schema.AddColumn: + return c.C.Type != nil && typeDependsOnT(c.C.Type.Type, c2.T) + case *schema.ModifyColumn: + return c.To.Type != nil && typeDependsOnT(c.To.Type.Type, c2.T) + } + return false + }) { + return true + } + case *schema.ModifyTable: + if c1.T != c2.T { + addC := make(map[*schema.Column]bool) + for _, c := range c2.Changes { + if add, ok := c.(*schema.AddColumn); ok { + addC[add.C] = true + } + } + return slices.ContainsFunc(c1.Changes, func(c schema.Change) bool { + fk, ok := c.(*schema.AddForeignKey) + return ok && refTo([]*schema.ForeignKey{fk.F}, c2.T) && slices.ContainsFunc(fk.F.Columns, func(c *schema.Column) bool { return addC[c] }) + }) + } + case *schema.AddObject: + t, ok := c2.O.(schema.Type) + if ok && slices.ContainsFunc(c1.Changes, func(c schema.Change) bool { + switch c := c.(type) { + case *schema.AddColumn: + return schema.IsType(c.C.Type.Type, t) + case *schema.ModifyColumn: + return schema.IsType(c.To.Type.Type, t) + default: + return false + } + }) { + return true + } + } + return depOfAdd(c1.T.Deps, c2) +} + +func dropObjectDependsOn(c1 *schema.DropObject, c2 schema.Change) bool { + t, ok := c1.O.(schema.Type) + if !ok { + return false + } + // Dropping a type must occur after all its usage were dropped. + switch c2 := c2.(type) { + case *schema.DropTable: + // Dropping a table also drops its triggers and might depend on the type. + if slices.ContainsFunc(c2.T.Triggers, func(tg *schema.Trigger) bool { + return slices.Contains(tg.Deps, c1.O) + }) { + return true + } + if slices.ContainsFunc(c2.T.Columns, func(c *schema.Column) bool { + return schema.IsType(c.Type.Type, t) + }) { + return true + } + case *schema.ModifyTable: + return slices.ContainsFunc(c2.Changes, func(c schema.Change) bool { + d, ok := c.(*schema.DropColumn) + return ok && schema.IsType(d.C.Type.Type, t) + }) + } + return false +} diff --git a/database/atlas/internal/sqlx/sqlx_test.go b/database/atlas/internal/sqlx/sqlx_test.go new file mode 100644 index 00000000..d8f0073b --- /dev/null +++ b/database/atlas/internal/sqlx/sqlx_test.go @@ -0,0 +1,301 @@ +// Copyright 2021-present The Atlas Authors. All rights reserved. +// This source code is licensed under the Apache 2.0 license found +// in the LICENSE file in the root directory of this source tree. + +package sqlx + +import ( + "errors" + "strconv" + "testing" + + "ariga.io/atlas/sql/schema" + + "github.com/stretchr/testify/require" +) + +func TestModeInspectRealm(t *testing.T) { + m := ModeInspectRealm(nil) + require.True(t, m.Is(schema.InspectSchemas)) + require.True(t, m.Is(schema.InspectTables)) + + m = ModeInspectRealm(&schema.InspectRealmOption{}) + require.True(t, m.Is(schema.InspectSchemas)) + require.True(t, m.Is(schema.InspectTables)) + + m = ModeInspectRealm(&schema.InspectRealmOption{Mode: schema.InspectSchemas}) + require.True(t, m.Is(schema.InspectSchemas)) + require.False(t, m.Is(schema.InspectTables)) + + m = ModeInspectRealm(&schema.InspectRealmOption{Exclude: []string{"*.*"}}) + require.Equal(t, schema.InspectSchemas, m) + m = ModeInspectRealm(&schema.InspectRealmOption{Exclude: []string{"*"}}) + require.NotEqual(t, schema.InspectSchemas, m) + require.True(t, m.Is(schema.InspectSchemas)) + require.True(t, m.Is(schema.InspectTables)) + m = ModeInspectRealm(&schema.InspectRealmOption{Mode: schema.InspectFuncs, Exclude: []string{"*.*"}}) + require.NotEqual(t, schema.InspectSchemas, m) + require.True(t, m.Is(schema.InspectFuncs)) + require.False(t, m.Is(schema.InspectTables)) +} + +func TestModeInspectSchema(t *testing.T) { + m := ModeInspectSchema(nil) + require.True(t, m.Is(schema.InspectSchemas)) + require.True(t, m.Is(schema.InspectTables)) + + m = ModeInspectSchema(&schema.InspectOptions{}) + require.True(t, m.Is(schema.InspectSchemas)) + require.True(t, m.Is(schema.InspectTables)) + + m = ModeInspectSchema(&schema.InspectOptions{Mode: schema.InspectSchemas}) + require.True(t, m.Is(schema.InspectSchemas)) + require.False(t, m.Is(schema.InspectTables)) + + m = ModeInspectSchema(&schema.InspectOptions{Exclude: []string{"*"}}) + require.Equal(t, schema.InspectSchemas, m) + m = ModeInspectSchema(&schema.InspectOptions{Exclude: []string{"*.*"}}) + require.NotEqual(t, schema.InspectSchemas, m) + require.True(t, m.Is(schema.InspectSchemas)) + require.True(t, m.Is(schema.InspectTables)) + m = ModeInspectSchema(&schema.InspectOptions{Mode: schema.InspectFuncs, Exclude: []string{"*"}}) + require.NotEqual(t, schema.InspectSchemas, m) + require.True(t, m.Is(schema.InspectFuncs)) + require.False(t, m.Is(schema.InspectTables)) +} + +func TestBuilder(t *testing.T) { + var ( + b = &Builder{QuoteOpening: '"', QuoteClosing: '"'} + columns = []string{"a", "b", "c"} + ) + b.P("CREATE TABLE"). + Table(&schema.Table{Name: "users"}). + Wrap(func(b *Builder) { + b.MapComma(columns, func(i int, b *Builder) { + b.Ident(columns[i]).P("int").P("NOT NULL") + }) + b.Comma().P("PRIMARY KEY").Wrap(func(b *Builder) { + b.MapComma(columns, func(i int, b *Builder) { + b.Ident(columns[i]) + }) + }) + }) + require.Equal(t, `CREATE TABLE "users" ("a" int NOT NULL, "b" int NOT NULL, "c" int NOT NULL, PRIMARY KEY ("a", "b", "c"))`, b.String()) + + // WrapErr. + require.EqualError(t, b.WrapErr(func(*Builder) error { return errors.New("oops") }), "oops") + require.EqualError(t, b.WrapIndentErr(func(*Builder) error { return errors.New("oops") }), "oops") +} + +func TestBuilder_Qualifier(t *testing.T) { + var ( + s = "other" + b = &Builder{QuoteOpening: '"', QuoteClosing: '"', Schema: &s} + ) + b.P("CREATE TABLE").Table(schema.NewTable("users")) + require.Equal(t, `CREATE TABLE "other"."users"`, b.String()) + + // Bypass table schema. + b.Reset() + b.P("CREATE TABLE").Table(schema.NewTable("users").SetSchema(schema.New("test"))) + require.Equal(t, `CREATE TABLE "other"."users"`, b.String()) + + // Empty qualifier, means skip. + s = "" + b.Reset() + b.P("CREATE TABLE").Table(schema.NewTable("users").SetSchema(schema.New("test"))) + require.Equal(t, `CREATE TABLE "users"`, b.String()) +} + +func TestQuote(t *testing.T) { + var ( + s = "s1" + b = &Builder{QuoteOpening: '[', QuoteClosing: ']', Schema: &s} + ) + b.P("EXECUTE sp_rename"). + P("@newname = N'c2'").Comma(). + P("@objtype = N'COLUMN'").Comma() + b.P("@objname = ").Quote("N", func(b *Builder) { + b.TableResource(schema.NewTable("t1"), &schema.Column{Name: "c1"}) + }) + require.Equal(t, `EXECUTE sp_rename @newname = N'c2', @objtype = N'COLUMN', @objname = N'[s1].[t1].[c1]'`, b.String()) +} +func TestMayWrap(t *testing.T) { + tests := []struct { + input string + wrapped bool + }{ + {"", true}, + {"()", false}, + {"('text')", false}, + {"('(')", false}, + {`('(\\')`, false}, + {`('\')(')`, false}, + {`(a) in (b)`, true}, + {`a in (b)`, true}, + {`("\\\\(((('")`, false}, + {`('(')||(')')`, true}, + // Test examples from SQLite. + {"b || 'x'", true}, + {"a+1", true}, + {"substr(x, 2)", true}, + {"(json_extract(x, '$.a'))", false}, + {"(substr(a, 2) COLLATE NOCASE)", false}, + {"(b+random())", false}, + } + for i, tt := range tests { + t.Run(strconv.Itoa(i), func(t *testing.T) { + expect := tt.input + if tt.wrapped { + expect = "(" + expect + ")" + } + require.Equal(t, expect, MayWrap(tt.input)) + }) + } +} + +func TestExprLastIndex(t *testing.T) { + tests := []struct { + input string + wantIdx int + }{ + {"", -1}, + {"()", 1}, + {"'('", 2}, + {"('(')", 4}, + {"('text')", 7}, + {"floor(x), y", 7}, + {"f(floor(x), y)", 13}, + {"f(floor(x), y, (z))", 18}, + {"f(x, (x*2)), y, (z)", 10}, + {"(a || ' ' || b)", 14}, + {"(a || ', ' || b)", 15}, + {"a || ', ' || b, x", 13}, + {"(a || ', ' || b), x", 15}, + } + for i, tt := range tests { + t.Run(strconv.Itoa(i), func(t *testing.T) { + idx := ExprLastIndex(tt.input) + require.Equal(t, tt.wantIdx, idx) + }) + } +} + +func TestIsQuoted(t *testing.T) { + tests := []struct { + input string + quotes []byte + want bool + }{ + {"''", []byte{'"', '\''}, true}, + {`""`, []byte{'"', '\''}, true}, + {"' '' \"\"'' '", []byte{'\''}, true}, + {"''''''''", []byte{'\''}, true}, + {"'foo'''", []byte{'\''}, true}, + {"'foo'''''", []byte{'\''}, true}, + {"'foo'', '''", []byte{'\''}, true}, + {"'foo bar'", []byte{'\''}, true}, + {`"never say \"never\""`, []byte{'"'}, true}, + {`"never say \"never\'"`, []byte{'"'}, true}, + {`'never say \"never\''`, []byte{'\''}, true}, + + {"'", []byte{'"', '\''}, false}, + {`"`, []byte{'"', '\''}, false}, + {"'foo' ''", []byte{'\''}, false}, + {"'foo' () ''", []byte{'\''}, false}, + {"'foo', ''", []byte{'\''}, false}, + {"'foo', 'bar'", []byte{'\''}, false}, + {"'foo',\" 'bar'", []byte{'\''}, false}, + } + for i, tt := range tests { + t.Run(strconv.Itoa(i), func(t *testing.T) { + quoted := IsQuoted(tt.input, tt.quotes...) + require.Equal(t, tt.want, quoted) + }) + } +} + +func TestReverseChanges(t *testing.T) { + tests := []struct { + input []schema.Change + expect []schema.Change + }{ + { + input: []schema.Change{ + (*schema.AddColumn)(nil), + }, + expect: []schema.Change{ + (*schema.AddColumn)(nil), + }, + }, + { + input: []schema.Change{ + (*schema.AddColumn)(nil), + (*schema.DropColumn)(nil), + }, + expect: []schema.Change{ + (*schema.DropColumn)(nil), + (*schema.AddColumn)(nil), + }, + }, + { + input: []schema.Change{ + (*schema.AddColumn)(nil), + (*schema.ModifyColumn)(nil), + (*schema.DropColumn)(nil), + }, + expect: []schema.Change{ + (*schema.DropColumn)(nil), + (*schema.ModifyColumn)(nil), + (*schema.AddColumn)(nil), + }, + }, + { + input: []schema.Change{ + (*schema.AddColumn)(nil), + (*schema.ModifyColumn)(nil), + (*schema.DropColumn)(nil), + (*schema.ModifyColumn)(nil), + }, + expect: []schema.Change{ + (*schema.ModifyColumn)(nil), + (*schema.DropColumn)(nil), + (*schema.ModifyColumn)(nil), + (*schema.AddColumn)(nil), + }, + }, + } + for i, tt := range tests { + t.Run(strconv.Itoa(i), func(t *testing.T) { + ReverseChanges(tt.input) + require.Equal(t, tt.expect, tt.input) + }) + } +} + +func TestIsUint(t *testing.T) { + require.True(t, IsUint("1")) + require.False(t, IsUint("-1")) + require.False(t, IsUint("1.2")) + require.False(t, IsUint("1.2.3")) +} + +func TestBodyDefChanged(t *testing.T) { + for i, tt := range []struct { + from, to string + changed bool + }{ + {"", "", false}, + {"a", "a", false}, + {"a", "b", true}, + {"select 1;", "select 1", false}, + {"\nselect 1\n", "\nselect 1;", false}, + {"\nselect \n \n1\n", "\nselect \n\n1;", false}, + {"\nselect \n \n1\n' '", "\nselect \n\n' ';", true}, + } { + t.Run(strconv.Itoa(i), func(t *testing.T) { + require.Equal(t, tt.changed, BodyDefChanged(tt.from, tt.to)) + }) + } +} diff --git a/database/atlas/sqlserver/convert_test.go b/database/atlas/sqlserver/convert_test.go new file mode 100644 index 00000000..4644ecc0 --- /dev/null +++ b/database/atlas/sqlserver/convert_test.go @@ -0,0 +1,121 @@ +package sqlserver_test + +import ( + "testing" + + "ariga.io/atlas/sql/schema" + + "github.com/flanksource/sqlglot-go/database/atlas/sqlserver" + "github.com/flanksource/sqlglot-go/database/inspect/catalog/sqlserver" + planner "github.com/flanksource/sqlglot-go/migrate/atlas/sqlserver" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("SQL Server column types", func() { + DescribeTable("ParseType reads a declared type and FormatType writes its canonical spelling", + func(declared, canonical string) { + parsed, err := planner.ParseType(declared) + Expect(err).ToNot(HaveOccurred()) + formatted, err := planner.FormatType(parsed) + Expect(err).ToNot(HaveOccurred()) + Expect(formatted).To(Equal(canonical)) + + reparsed, err := planner.ParseType(formatted) + Expect(err).ToNot(HaveOccurred()) + Expect(reparsed).To(Equal(parsed), "the canonical spelling is a fixed point") + }, + Entry("bit", "BIT", "bit"), + Entry("tinyint", "tinyint", "tinyint"), + Entry("smallint", "smallint", "smallint"), + Entry("int", "INT", "int"), + Entry("bigint", "bigint", "bigint"), + Entry("decimal with precision and scale", "decimal(18, 2)", "decimal(18,2)"), + Entry("decimal defaults to (18,0)", "decimal", "decimal(18,0)"), + Entry("numeric keeps its name", "NUMERIC(10)", "numeric(10,0)"), + Entry("money", "money", "money"), + Entry("smallmoney", "smallmoney", "smallmoney"), + Entry("float defaults to 53 bits", "float", "float"), + Entry("float(53) is float", "float(53)", "float"), + Entry("float(24) is stored as real", "float(24)", "real"), + Entry("real", "REAL", "real"), + Entry("char defaults to length 1", "char", "char(1)"), + Entry("varchar(max)", "varchar(MAX)", "varchar(max)"), + Entry("nchar", "nchar(10)", "nchar(10)"), + Entry("nvarchar", "NVARCHAR(50)", "nvarchar(50)"), + Entry("nvarchar(max)", "nvarchar(max)", "nvarchar(max)"), + Entry("text", "text", "text"), + Entry("ntext stays ntext", "NTEXT", "ntext"), + Entry("binary", "binary(16)", "binary(16)"), + Entry("varbinary(max)", "varbinary(max)", "varbinary(max)"), + Entry("image", "image", "image"), + Entry("date", "date", "date"), + Entry("time defaults to 7 digits", "time", "time(7)"), + Entry("time(3)", "TIME(3)", "time(3)"), + Entry("datetime", "datetime", "datetime"), + Entry("datetime2(0)", "datetime2(0)", "datetime2(0)"), + Entry("smalldatetime", "smalldatetime", "smalldatetime"), + Entry("datetimeoffset(2)", "datetimeoffset(2)", "datetimeoffset(2)"), + Entry("uniqueidentifier", "UNIQUEIDENTIFIER", "uniqueidentifier"), + Entry("xml", "xml", "xml"), + Entry("sql_variant", "sql_variant", "sql_variant"), + Entry("rowversion", "rowversion", "rowversion"), + Entry("timestamp is rowversion", "timestamp", "rowversion"), + ) + + DescribeTable("ParseType refuses types the driver does not manage", + func(declared, message string) { + _, err := planner.ParseType(declared) + Expect(err).To(MatchError(ContainSubstring(message))) + }, + Entry("an alias type", "sysname", `type "sysname" is not supported`), + Entry("a spatial type", "geography", `type "geography" is not supported`), + Entry("a float precision out of range", "float(54)", "float precision 54 is out of range 1-53"), + Entry("a length on an unsized type", "int(4)", "takes no size"), + Entry("trailing text", "int identity", "after the type"), + ) + + DescribeTable("the catalog's sys.columns widths map to the declared type", + func(column mssqlcatalog.Column, canonical string) { + t, err := sqlserver.CatalogType(column) + Expect(err).ToNot(HaveOccurred()) + formatted, err := planner.FormatType(t) + Expect(err).ToNot(HaveOccurred()) + Expect(formatted).To(Equal(canonical)) + }, + Entry("nvarchar bytes are halved", catalogColumn("nvarchar", 100, 0, 0), "nvarchar(50)"), + Entry("nvarchar(max) is -1", catalogColumn("nvarchar", -1, 0, 0), "nvarchar(max)"), + Entry("varchar bytes are characters", catalogColumn("varchar", 30, 0, 0), "varchar(30)"), + Entry("nchar bytes are halved", catalogColumn("nchar", 20, 0, 0), "nchar(10)"), + Entry("varbinary(max)", catalogColumn("varbinary", -1, 0, 0), "varbinary(max)"), + Entry("decimal precision and scale", catalogColumn("decimal", 9, 18, 2), "decimal(18,2)"), + Entry("float(53)", catalogColumn("float", 8, 53, 0), "float"), + Entry("real", catalogColumn("real", 4, 24, 0), "real"), + Entry("datetime2 fractional digits", catalogColumn("datetime2", 8, 27, 7), "datetime2(7)"), + Entry("datetimeoffset fractional digits", catalogColumn("datetimeoffset", 10, 34, 3), "datetimeoffset(3)"), + Entry("timestamp is rowversion", catalogColumn("timestamp", 8, 0, 0), "rowversion"), + Entry("int ignores its byte width", catalogColumn("int", 4, 10, 0), "int"), + ) + + It("refuses a catalog column without sys.columns widths", func() { + _, err := sqlserver.CatalogType(mssqlcatalog.Column{Table: "acme_orders", Name: "note", DataType: "nvarchar"}) + Expect(err).To(MatchError(ContainSubstring(`column "note" has no sys.columns width`))) + }) + + It("refuses to format a type that is not a SQL Server type", func() { + _, err := planner.FormatType(&schema.JSONType{T: "jsonb"}) + Expect(err).To(MatchError(ContainSubstring("unsupported type *schema.JSONType"))) + }) +}) + +func TestSQLServerDriver(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Atlas SQL Server Driver Suite") +} + +func catalogColumn(dataType string, maxLength, precision, scale int) mssqlcatalog.Column { + return mssqlcatalog.Column{ + Table: "acme_orders", Name: "value", DataType: dataType, + MaxLength: &maxLength, Precision: &precision, Scale: &scale, + } +} diff --git a/database/atlas/sqlserver/default_plan_test.go b/database/atlas/sqlserver/default_plan_test.go new file mode 100644 index 00000000..a75137aa --- /dev/null +++ b/database/atlas/sqlserver/default_plan_test.go @@ -0,0 +1,21 @@ +package sqlserver + +import ( + "ariga.io/atlas/sql/migrate" + "ariga.io/atlas/sql/schema" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ migrate.PlanApplier = DefaultPlan + +var _ = Describe("connectionless default planner", func() { + It("plans schema changes and refuses to execute them without a connection", func(ctx SpecContext) { + table := schema.NewTable("items").SetSchema(schema.New("dbo")).AddColumns(schema.NewIntColumn("id", "int")) + changes := []schema.Change{&schema.AddTable{T: table}} + plan, err := DefaultPlan.PlanChanges(ctx, "example", changes) + Expect(err).NotTo(HaveOccurred()) + Expect(plan.Changes).To(ContainElement(HaveField("Cmd", ContainSubstring("CREATE TABLE [dbo].[items]")))) + Expect(DefaultPlan.ApplyChanges(ctx, changes)).To(MatchError(ContainSubstring("cannot execute statements without a database connection"))) + }) +}) diff --git a/database/atlas/sqlserver/diff_test.go b/database/atlas/sqlserver/diff_test.go new file mode 100644 index 00000000..58ee1b28 --- /dev/null +++ b/database/atlas/sqlserver/diff_test.go @@ -0,0 +1,137 @@ +package sqlserver_test + +import ( + "fmt" + + "ariga.io/atlas/sql/schema" + + planner "github.com/flanksource/sqlglot-go/migrate/atlas/sqlserver" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +const touchOrderLive = `-- Marks an order as touched. +CREATE PROCEDURE dbo.acme_touch_order + @order_id INT -- the order +AS +BEGIN + /* one statement */ + UPDATE dbo.acme_orders SET touched_at = SYSUTCDATETIME(), label = N'touched' WHERE order_id = @order_id; +END` + +func routineSchema(definitions ...string) *schema.Schema { + s := schema.New("dbo") + for _, definition := range definitions { + routine, err := planner.ParseRoutine(definition) + Expect(err).ToNot(HaveOccurred()) + Expect(routine.AddTo(s)).To(Succeed()) + } + return s +} + +var _ = Describe("SQL Server routine diff", func() { + DescribeTable("compares routine definitions by their canonical form", + func(declared string, expected types) { + from, to := routineSchema(touchOrderLive), routineSchema(declared) + changes, err := planner.DefaultDiff.SchemaDiff(from, to) + Expect(err).ToNot(HaveOccurred()) + Expect(changeTypes(changes)).To(Equal(expected)) + }, + Entry("the same text", touchOrderLive, types{}), + Entry("comments, case, whitespace, brackets and OR ALTER differ", + "create or alter procedure [dbo].[acme_touch_order] @order_id int as begin update dbo.acme_orders set touched_at = sysutcdatetime(), label = N'touched' where order_id = @order_id; end", + types{}), + Entry("a string literal differs", + "CREATE PROCEDURE dbo.acme_touch_order @order_id INT AS BEGIN UPDATE dbo.acme_orders SET touched_at = SYSUTCDATETIME(), label = N'Touched' WHERE order_id = @order_id; END", + types{"*schema.ModifyProc"}), + Entry("a statement is added", + "CREATE PROCEDURE dbo.acme_touch_order @order_id INT AS BEGIN SET NOCOUNT ON; UPDATE dbo.acme_orders SET touched_at = SYSUTCDATETIME(), label = N'touched' WHERE order_id = @order_id; END", + types{"*schema.ModifyProc"}), + ) + + It("adds declared routines the database lacks and drops routines only the database has", func() { + from := routineSchema(touchOrderLive) + to := routineSchema(countOrdersProc, orderTotalFunc) + changes, err := planner.DefaultDiff.SchemaDiff(from, to) + Expect(err).ToNot(HaveOccurred()) + Expect(changeTypes(changes)).To(Equal(types{"*schema.AddProc", "*schema.AddFunc", "*schema.DropProc"})) + }) + + It("refuses to turn a procedure into a function", func() { + from := routineSchema("CREATE PROCEDURE dbo.acme_order_total AS BEGIN SELECT 1; END") + _, err := planner.DefaultDiff.SchemaDiff(from, routineSchema(orderTotalFunc)) + Expect(err).To(MatchError(ContainSubstring("acme_order_total is a procedure in the database and a function in the bundle"))) + }) + + It("fails on a live definition sqlglot cannot read instead of assuming a change", func() { + from := routineSchema(touchOrderLive) + from.Procs[0].Attrs = []schema.Attr{&planner.Definition{Text: "CREATE PROCEDURE dbo.acme_touch_order AS BEGIN"}} + _, err := planner.DefaultDiff.SchemaDiff(from, routineSchema(touchOrderLive)) + Expect(err).To(MatchError(ContainSubstring("live definition: sqlserver: routine acme_touch_order"))) + }) +}) + +var _ = Describe("SQL Server table diff", func() { + DescribeTable("compares column defaults as SQL Server stores them", + func(live, declared schema.Expr, expected types) { + changes, err := planner.DefaultDiff.TableDiff(ordersTable(live, nil), ordersTable(declared, nil)) + Expect(err).ToNot(HaveOccurred()) + Expect(changeTypes(changes)).To(Equal(expected)) + }, + Entry("a number wrapped twice", &schema.RawExpr{X: "((0))"}, &schema.Literal{V: "0"}, types{}), + Entry("a function call in another case", &schema.RawExpr{X: "(getdate())"}, &schema.RawExpr{X: "GETDATE()"}, types{}), + Entry("a string literal", &schema.RawExpr{X: "('open')"}, &schema.Literal{V: "'open'"}, types{}), + Entry("a different number", &schema.RawExpr{X: "((0))"}, &schema.Literal{V: "1"}, types{"*schema.ModifyColumn"}), + Entry("a removed default", &schema.RawExpr{X: "((0))"}, nil, types{"*schema.ModifyColumn"}), + ) + + It("matches an unquoted HCL string default to the N'...' literal SQL Server stores for nvarchar", func() { + column := func(def schema.Expr) *schema.Table { + c := schema.NewStringColumn("status", "nvarchar") + c.Type.Type = &schema.StringType{T: "nvarchar", Size: 20} + c.Default = def + t := schema.NewTable("acme_orders").AddColumns(c) + schema.New("dbo").AddTables(t) + return t + } + changes, err := planner.DefaultDiff.TableDiff(column(&schema.RawExpr{X: "(N'open')"}), column(&schema.Literal{V: "open"})) + Expect(err).ToNot(HaveOccurred()) + Expect(changes).To(BeEmpty()) + }) + + DescribeTable("compares check constraints by their canonical predicate", + func(live, declared string, expected types) { + changes, err := planner.DefaultDiff.TableDiff(ordersTable(nil, &schema.Check{Name: "CK_acme_orders_qty", Expr: live}), + ordersTable(nil, &schema.Check{Name: "CK_acme_orders_qty", Expr: declared})) + Expect(err).ToNot(HaveOccurred()) + Expect(changeTypes(changes)).To(Equal(expected)) + }, + Entry("stored with brackets and parentheses", "([qty]>(0))", "qty > 0", types{}), + Entry("grouping that changes the meaning", "(([qty]+[bonus])*(2)>(10))", "qty + bonus * 2 > 10", types{"*schema.ModifyCheck"}), + ) +}) + +type types []string + +func changeTypes(changes []schema.Change) types { + out := types{} + for _, change := range changes { + if modify, ok := change.(*schema.ModifyTable); ok { + out = append(out, changeTypes(modify.Changes)...) + continue + } + out = append(out, fmt.Sprintf("%T", change)) + } + return out +} + +func ordersTable(def schema.Expr, check *schema.Check) *schema.Table { + column := schema.NewIntColumn("qty", "int") + column.Default = def + t := schema.NewTable("acme_orders").AddColumns(column) + if check != nil { + t.AddChecks(check) + } + schema.New("dbo").AddTables(t) + return t +} diff --git a/database/atlas/sqlserver/driver.go b/database/atlas/sqlserver/driver.go new file mode 100644 index 00000000..3f799ee0 --- /dev/null +++ b/database/atlas/sqlserver/driver.go @@ -0,0 +1,227 @@ +// Package sqlserver is an Atlas driver for Microsoft SQL Server. +// +// It covers what declarative migrations of a shared database need: tables, +// columns, primary keys, indexes, foreign keys and checks from HCL, and stored +// procedures and functions from their T-SQL definitions, parsed and compared +// through sqlglot. Views, triggers, sequences, user-defined types, +// partitioning and computed columns are refused with an error, and versioned +// migration directories (revisions, snapshots, clean checks) are not +// supported. +// +// Pure diffing and planning live in sqlglot-go/migrate/atlas/sqlserver. This +// driver supplies inspection, connection management, locks and SQL execution. +// +// Inspection is scoped: InspectOptions.Tables names the objects to read -- +// tables and routines alike, since SQL Server keeps both in one namespace per +// schema -- and Mode picks tables (InspectTables) and/or routines +// (InspectFuncs). Generated SQL runs at compatibility level 100. +package sqlserver + +import ( + "context" + "database/sql" + "errors" + "fmt" + "net/url" + "time" + + "ariga.io/atlas/sql/migrate" + "ariga.io/atlas/sql/schema" + "ariga.io/atlas/sql/sqlclient" + "github.com/flanksource/sqlglot-go/database/atlas/internal/sqlx" + "github.com/flanksource/sqlglot-go/database/inspect/catalog/sqlserver" + planner "github.com/flanksource/sqlglot-go/migrate/atlas/sqlserver" + _ "github.com/microsoft/go-mssqldb" +) + +type ( + // Driver represents a SQL Server driver for introspecting database schemas, + // generating diff between schema elements and applying migration changes. + Driver struct { + *conn + schema.Differ + schema.Inspector + migrate.PlanApplier + } + + // conn is a database connection and what it runs on. + conn struct { + schema.ExecQuerier + version string + compatibility int + } +) + +// DriverName holds the name used for registration. +const DriverName = "sqlserver" + +// errUnsupported is wrapped by every refusal of a versioned-migration feature. +var errUnsupported = errors.New("not supported by the SQL Server driver") + +func init() { + sqlclient.Register( + DriverName, + sqlclient.OpenerFunc(opener), + sqlclient.RegisterDriverOpener(Open), + sqlclient.RegisterTxOpener(OpenTx), + sqlclient.RegisterCodec(codec, codec), + sqlclient.RegisterFlavours("mssql"), + sqlclient.RegisterURLParser(urlparse{}), + ) +} + +type urlparse struct{} + +// ParseURL implements the sqlclient.URLParser interface. The mssql flavour is +// rewritten to the sqlserver:// form go-mssqldb reads. +func (urlparse) ParseURL(u *url.URL) *sqlclient.URL { + dsn := *u + dsn.Scheme = DriverName + return &sqlclient.URL{URL: u, DSN: dsn.String()} +} + +func opener(ctx context.Context, u *url.URL) (*sqlclient.Client, error) { + ur := urlparse{}.ParseURL(u) + db, err := sql.Open(DriverName, ur.DSN) + if err != nil { + return nil, err + } + client, err := OpenClient(ctx, db) + if err != nil { + return nil, errors.Join(err, db.Close()) + } + client.URL = ur + return client, nil +} + +// OpenClient wraps an open database/sql handle, whatever DSN form it was +// opened with (URL, ADO or ODBC), in an Atlas client. Closing the client closes +// db. +func OpenClient(ctx context.Context, db *sql.DB) (*sqlclient.Client, error) { + drv, err := open(ctx, db) + if err != nil { + return nil, err + } + return &sqlclient.Client{Name: DriverName, DB: db, Driver: drv}, nil +} + +// Open opens a new SQL Server driver, reading the engine version and the +// database compatibility level. +func Open(db schema.ExecQuerier) (migrate.Driver, error) { + return open(context.Background(), db) +} + +func open(ctx context.Context, db schema.ExecQuerier) (*Driver, error) { + c := &conn{ExecQuerier: db} + rows, err := db.QueryContext(ctx, `SELECT CAST(SERVERPROPERTY('ProductVersion') AS nvarchar(128)), compatibility_level + FROM sys.databases WHERE database_id = DB_ID()`) + if err != nil { + return nil, fmt.Errorf("sqlserver: query server version: %w", err) + } + if err := sqlx.ScanOne(rows, &c.version, &c.compatibility); err != nil { + return nil, fmt.Errorf("sqlserver: scan server version: %w", err) + } + return &Driver{ + conn: c, + Differ: planner.DefaultDiff, + Inspector: &inspect{c}, + PlanApplier: &planApply{c}, + }, nil +} + +// OpenTx opens a transaction. +func OpenTx(ctx context.Context, db *sql.DB, opts *sql.TxOptions) (*sqlclient.Tx, error) { + tx, err := db.BeginTx(ctx, opts) + if err != nil { + return nil, err + } + return &sqlclient.Tx{Tx: tx, CommitFn: tx.Commit, RollbackFn: tx.Rollback}, nil +} + +// Version is the engine's ProductVersion, e.g. 16.0.4135.4. +func (d *Driver) Version() string { return d.version } + +// CompatibilityLevel is the connected database's compatibility_level. +func (d *Driver) CompatibilityLevel() int { return d.compatibility } + +// Snapshot implements migrate.Snapshoter and is not supported. +func (d *Driver) Snapshot(context.Context) (migrate.RestoreFunc, error) { + return nil, fmt.Errorf("sqlserver: snapshot: %w", errUnsupported) +} + +// CheckClean implements migrate.CleanChecker and is not supported. +func (d *Driver) CheckClean(context.Context, *migrate.TableIdent) error { + return fmt.Errorf("sqlserver: clean check: %w", errUnsupported) +} + +// FormatType converts schema type to its column form in the database. +func (*Driver) FormatType(t schema.Type) (string, error) { return planner.FormatType(t) } + +// ParseType returns the schema.Type value represented by the given string. +func (*Driver) ParseType(s string) (schema.Type, error) { return planner.ParseType(s) } + +// StmtBuilder is a helper method used to build statements with SQL Server +// formatting: bracket-quoted identifiers. +func (*Driver) StmtBuilder(opts migrate.PlanOptions) *sqlx.Builder { + return &sqlx.Builder{QuoteOpening: '[', QuoteClosing: ']', Schema: opts.SchemaQualifier, Indent: opts.Indent} +} + +// Lock implements schema.Locker with a session-owned sp_getapplock on a +// dedicated connection. A negative timeout waits forever and zero tries once. +func (d *Driver) Lock(ctx context.Context, name string, timeout time.Duration) (schema.UnlockFunc, error) { + if name == "" || len(name) > 255 { + return nil, fmt.Errorf("sqlserver: lock name must be 1-255 characters, got %d", len(name)) + } + lockConn, err := sqlx.SingleConn(ctx, d.ExecQuerier) + if err != nil { + return nil, fmt.Errorf("sqlserver: lock %q: %w", name, err) + } + status, err := applockStatus(ctx, lockConn, `EXEC @status = sp_getapplock @Resource = @p1, + @LockMode = 'Exclusive', @LockOwner = 'Session', @LockTimeout = @p2`, name, lockTimeout(timeout)) + switch { + case err != nil: + return nil, errors.Join(fmt.Errorf("sqlserver: lock %q: %w", name, err), lockConn.Close()) + case status == -1: + return nil, errors.Join(fmt.Errorf("sqlserver: lock %q: %w", name, schema.ErrLocked), lockConn.Close()) + case status < 0: + return nil, errors.Join(fmt.Errorf("sqlserver: lock %q: sp_getapplock returned %d", name, status), lockConn.Close()) + } + return func() error { + status, err := applockStatus(context.WithoutCancel(ctx), lockConn, + `EXEC @status = sp_releaseapplock @Resource = @p1, @LockOwner = 'Session'`, name) + if err == nil && status != 0 { + err = fmt.Errorf("sp_releaseapplock returned %d", status) + } + if err != nil { + err = fmt.Errorf("sqlserver: unlock %q: %w", name, err) + } + return errors.Join(err, lockConn.Close()) + }, nil +} + +func lockTimeout(timeout time.Duration) int64 { + if timeout < 0 { + return -1 + } + return timeout.Milliseconds() +} + +func applockStatus(ctx context.Context, conn schema.ExecQuerier, exec string, args ...any) (int, error) { + rows, err := conn.QueryContext(ctx, "DECLARE @status int; "+exec+"; SELECT @status", args...) + if err != nil { + return 0, err + } + var status int + return status, sqlx.ScanOne(rows, &status) +} + +// queryer adapts the driver connection to mssqlcatalog.Queryer. +type queryer struct{ schema.ExecQuerier } + +func (q queryer) QueryContext(ctx context.Context, query string, args ...any) (mssqlcatalog.Rows, error) { + rows, err := q.ExecQuerier.QueryContext(ctx, query, args...) + if err != nil { + return nil, err + } + return rows, nil +} diff --git a/database/atlas/sqlserver/inspect.go b/database/atlas/sqlserver/inspect.go new file mode 100644 index 00000000..c1dc7389 --- /dev/null +++ b/database/atlas/sqlserver/inspect.go @@ -0,0 +1,139 @@ +package sqlserver + +import ( + "context" + "fmt" + "strings" + + "ariga.io/atlas/sql/schema" + "github.com/flanksource/sqlglot-go/database/atlas/internal/sqlx" + "github.com/flanksource/sqlglot-go/database/inspect/catalog/sqlserver" + planner "github.com/flanksource/sqlglot-go/migrate/atlas/sqlserver" +) + +// inspect provides a SQL Server implementation for schema.Inspector. +type inspect struct{ *conn } + +var _ schema.Inspector = (*inspect)(nil) + +// managedModes is what the driver inspects. A zero mode means all of it. +const managedModes = schema.InspectSchemas | schema.InspectTables | schema.InspectFuncs + +// InspectSchema returns the tables and routines of one schema, the connection's +// default schema when name is empty. opts.Tables restricts both tables and +// routines to those names. +func (i *inspect) InspectSchema(ctx context.Context, name string, opts *schema.InspectOptions) (*schema.Schema, error) { + if opts == nil { + opts = &schema.InspectOptions{} + } + mode, err := inspectMode(opts.Mode) + if err != nil { + return nil, err + } + if name == "" { + if name, err = i.defaultSchema(ctx); err != nil { + return nil, err + } + } + schemas, err := mssqlcatalog.Schemas(ctx, queryer{i}, []string{name}) + if err != nil { + return nil, fmt.Errorf("sqlserver: query schemas: %w", err) + } + if len(schemas) == 0 { + return nil, &schema.NotExistError{Err: fmt.Errorf("sqlserver: schema %q was not found", name)} + } + s := schema.New(schemas[0]) + schema.NewRealm(s) + if err := i.inspectSchema(ctx, s, mode, opts.Tables); err != nil { + return nil, err + } + return schema.ExcludeSchema(s, opts.Exclude) +} + +// InspectRealm returns the tables and routines of every schema in opts.Schemas, +// or of every user schema when it is empty. +func (i *inspect) InspectRealm(ctx context.Context, opts *schema.InspectRealmOption) (*schema.Realm, error) { + if opts == nil { + opts = &schema.InspectRealmOption{} + } + mode, err := inspectMode(opts.Mode) + if err != nil { + return nil, err + } + names, err := mssqlcatalog.Schemas(ctx, queryer{i}, opts.Schemas) + if err != nil { + return nil, fmt.Errorf("sqlserver: query schemas: %w", err) + } + r := schema.NewRealm() + for _, name := range names { + s := schema.New(name) + r.AddSchemas(s) + if err := i.inspectSchema(ctx, s, mode, nil); err != nil { + return nil, err + } + } + return schema.ExcludeRealm(r, opts.Exclude) +} + +func inspectMode(mode schema.InspectMode) (schema.InspectMode, error) { + if mode == 0 { + return managedModes, nil + } + if unmanaged := mode &^ managedModes; unmanaged != 0 { + return 0, fmt.Errorf("sqlserver: inspect mode %d: views, triggers, types and objects are %w", unmanaged, errUnsupported) + } + return mode, nil +} + +func (i *inspect) defaultSchema(ctx context.Context) (string, error) { + rows, err := i.QueryContext(ctx, "SELECT SCHEMA_NAME()") + if err != nil { + return "", fmt.Errorf("sqlserver: query default schema: %w", err) + } + var name string + if err := sqlx.ScanOne(rows, &name); err != nil { + return "", fmt.Errorf("sqlserver: scan default schema: %w", err) + } + return name, nil +} + +func (i *inspect) inspectSchema(ctx context.Context, s *schema.Schema, mode schema.InspectMode, names []string) error { + scope := mssqlcatalog.Scope{Schema: s.Name, Names: names} + if mode.Is(schema.InspectTables) { + if err := i.inspectTables(ctx, s, scope); err != nil { + return err + } + } + if mode.Is(schema.InspectFuncs) { + if err := i.inspectRoutines(ctx, s, scope); err != nil { + return err + } + } + return nil +} + +// inspectRoutines reads each routine's definition and parses it. A routine +// whose definition is withheld or does not parse is an error. +func (i *inspect) inspectRoutines(ctx context.Context, s *schema.Schema, scope mssqlcatalog.Scope) error { + routines, err := mssqlcatalog.Routines(ctx, queryer{i}, scope) + if err != nil { + return fmt.Errorf("sqlserver: query routines: %w", err) + } + for _, live := range routines { + if live.Definition == nil { + return fmt.Errorf("sqlserver: routine %s.%s has no readable definition (encrypted or no VIEW DEFINITION permission)", live.Schema, live.Name) + } + routine, err := planner.ParseRoutine(*live.Definition) + if err != nil { + return fmt.Errorf("sqlserver: live routine %s.%s: %w", live.Schema, live.Name, err) + } + if !strings.EqualFold(routine.Name, live.Name) { + return fmt.Errorf("sqlserver: live routine %s.%s is defined as %q (renamed with sp_rename?)", live.Schema, live.Name, routine.Name) + } + routine.Name = live.Name + if err := routine.AddTo(s); err != nil { + return fmt.Errorf("sqlserver: live routine %s.%s: %w", live.Schema, live.Name, err) + } + } + return nil +} diff --git a/database/atlas/sqlserver/inspect_tables.go b/database/atlas/sqlserver/inspect_tables.go new file mode 100644 index 00000000..510cef8b --- /dev/null +++ b/database/atlas/sqlserver/inspect_tables.go @@ -0,0 +1,257 @@ +package sqlserver + +import ( + "context" + "fmt" + "strings" + + "ariga.io/atlas/sql/schema" + "github.com/flanksource/sqlglot-go/database/inspect/catalog/sqlserver" + planner "github.com/flanksource/sqlglot-go/migrate/atlas/sqlserver" +) + +// inspectTables reads the base tables in scope with their columns, keys, +// indexes, foreign keys and checks. A table the driver cannot reproduce -- +// partitioned, triggered, or with a computed or user-defined-type column -- is +// an error. +func (i *inspect) inspectTables(ctx context.Context, s *schema.Schema, scope mssqlcatalog.Scope) error { + q := queryer{i} + tables, err := mssqlcatalog.Tables(ctx, q, scope, mssqlcatalog.BaseTables) + if err != nil { + return fmt.Errorf("sqlserver: query tables: %w", err) + } + if len(tables) == 0 { + return nil + } + scope.Names = make([]string, len(tables)) + for n, t := range tables { + scope.Names[n] = t.Name + s.AddTables(schema.NewTable(t.Name)) + } + if err := refuseUnmanaged(ctx, q, scope); err != nil { + return err + } + columns, err := mssqlcatalog.Columns(ctx, q, scope) + if err != nil { + return fmt.Errorf("sqlserver: query columns: %w", err) + } + if err := addColumns(s, columns); err != nil { + return err + } + if err := i.inspectIndexes(ctx, s, scope); err != nil { + return err + } + return i.inspectConstraints(ctx, s, scope) +} + +func refuseUnmanaged(ctx context.Context, q mssqlcatalog.Queryer, scope mssqlcatalog.Scope) error { + features, err := mssqlcatalog.UnmanagedFeatures(ctx, q, scope) + if err != nil { + return fmt.Errorf("sqlserver: query table features: %w", err) + } + if len(features) == 0 { + return nil + } + found := make([]string, len(features)) + for n, f := range features { + found[n] = fmt.Sprintf("%s.%s (%s)", scope.Schema, f.Table, f.Feature) + } + return fmt.Errorf("sqlserver: partitioned tables and triggers are %w: %s", errUnsupported, strings.Join(found, ", ")) +} + +func (i *inspect) inspectIndexes(ctx context.Context, s *schema.Schema, scope mssqlcatalog.Scope) error { + indexes, err := mssqlcatalog.Indexes(ctx, queryer{i}, scope) + if err != nil { + return fmt.Errorf("sqlserver: query indexes: %w", err) + } + parts, err := mssqlcatalog.IndexColumns(ctx, queryer{i}, scope) + if err != nil { + return fmt.Errorf("sqlserver: query index columns: %w", err) + } + return addIndexes(s, indexes, parts) +} + +func (i *inspect) inspectConstraints(ctx context.Context, s *schema.Schema, scope mssqlcatalog.Scope) error { + fks, err := mssqlcatalog.ForeignKeys(ctx, queryer{i}, scope) + if err != nil { + return fmt.Errorf("sqlserver: query foreign keys: %w", err) + } + if err := addForeignKeys(s, fks); err != nil { + return err + } + checks, err := mssqlcatalog.CheckConstraints(ctx, queryer{i}, scope) + if err != nil { + return fmt.Errorf("sqlserver: query check constraints: %w", err) + } + for _, check := range checks { + t, err := tableOf(s, check.Table) + if err != nil { + return err + } + t.AddChecks(&schema.Check{Name: check.Name, Expr: check.Definition}) + } + return nil +} + +func tableOf(s *schema.Schema, name string) (*schema.Table, error) { + t, ok := s.Table(name) + if !ok { + return nil, fmt.Errorf("sqlserver: catalog row for table %s.%s, which was not inspected", s.Name, name) + } + return t, nil +} + +func addColumns(s *schema.Schema, columns []mssqlcatalog.Column) error { + for _, c := range columns { + t, err := tableOf(s, c.Table) + if err != nil { + return err + } + column, err := catalogColumn(c) + if err != nil { + return fmt.Errorf("sqlserver: column %s.%s.%s: %w", s.Name, c.Table, c.Name, err) + } + t.AddColumns(column) + } + return nil +} + +func catalogColumn(c mssqlcatalog.Column) (*schema.Column, error) { + switch { + case c.Computed: + return nil, fmt.Errorf("computed columns are %w", errUnsupported) + case c.UserDefinedType: + return nil, fmt.Errorf("user-defined types are %w", errUnsupported) + } + t, err := CatalogType(c) + if err != nil { + return nil, err + } + column := schema.NewColumn(c.Name).SetType(t).SetNull(c.Nullable) + column.Type.Raw = c.DataType + if c.Default != nil { + column.Default = &schema.RawExpr{X: *c.Default} + if c.DefaultConstraint == nil { + return nil, fmt.Errorf("default %s has no constraint name", *c.Default) + } + column.AddAttrs(&planner.DefaultConstraint{Name: *c.DefaultConstraint}) + } + if c.Identity { + if c.IdentitySeed == nil || c.IdentityIncrement == nil { + return nil, fmt.Errorf("identity column without seed and increment") + } + column.AddAttrs(&planner.Identity{Seed: *c.IdentitySeed, Increment: *c.IdentityIncrement}) + } + return column, nil +} + +func CatalogType(c mssqlcatalog.Column) (schema.Type, error) { + return planner.CatalogType(planner.CatalogColumn{Name: c.Name, DataType: c.DataType, MaxLength: c.MaxLength, Precision: c.Precision, Scale: c.Scale}) +} + +func addIndexes(s *schema.Schema, indexes []mssqlcatalog.Index, columns []mssqlcatalog.IndexColumn) error { + byIndex := make(map[[2]string][]mssqlcatalog.IndexColumn) + for _, c := range columns { + key := [2]string{c.Table, c.Index} + byIndex[key] = append(byIndex[key], c) + } + for _, idx := range indexes { + t, err := tableOf(s, idx.Table) + if err != nil { + return err + } + index, err := catalogIndex(t, idx, byIndex[[2]string{idx.Table, idx.Name}]) + if err != nil { + return fmt.Errorf("sqlserver: index %s.%s.%s: %w", s.Name, idx.Table, idx.Name, err) + } + if idx.Primary { + t.SetPrimaryKey(index) + } else { + t.AddIndexes(index) + } + } + return nil +} + +func catalogIndex(t *schema.Table, idx mssqlcatalog.Index, columns []mssqlcatalog.IndexColumn) (*schema.Index, error) { + index := schema.NewIndex(idx.Name).SetUnique(idx.Unique).SetTable(t) + switch idx.TypeDesc { + case "CLUSTERED": + index.AddAttrs(&planner.Clustered{}) + case "NONCLUSTERED": + default: + return nil, fmt.Errorf("%s indexes are %w", strings.ToLower(idx.TypeDesc), errUnsupported) + } + if idx.Filter != "" { + index.AddAttrs(&planner.IndexPredicate{P: idx.Filter}) + } + if idx.UniqueConstraint { + index.AddAttrs(&planner.UniqueConstraint{}) + } + var include []string + for _, c := range columns { + if c.Included { + include = append(include, c.Column) + continue + } + column, ok := t.Column(c.Column) + if !ok { + return nil, fmt.Errorf("key column %q was not inspected", c.Column) + } + index.AddParts(&schema.IndexPart{SeqNo: c.KeyOrdinal, C: column, Desc: c.Descending}) + } + if len(include) > 0 { + index.AddAttrs(&planner.IndexInclude{Columns: include}) + } + return index, nil +} + +func addForeignKeys(s *schema.Schema, rows []mssqlcatalog.ForeignKey) error { + byName := make(map[[2]string]*schema.ForeignKey) + for _, row := range rows { + t, err := tableOf(s, row.Table) + if err != nil { + return err + } + key := [2]string{row.Table, row.Name} + fk, ok := byName[key] + if !ok { + fk = &schema.ForeignKey{ + Symbol: row.Name, Table: t, RefTable: referencedTable(s, row.RefSchema, row.RefTable), + OnDelete: referenceOption(row.OnDelete), OnUpdate: referenceOption(row.OnUpdate), + } + byName[key] = fk + t.AddForeignKeys(fk) + } + column, ok := t.Column(row.Column) + if !ok { + return fmt.Errorf("sqlserver: foreign key %s.%s column %q was not inspected", row.Table, row.Name, row.Column) + } + fk.Columns = append(fk.Columns, column) + fk.RefColumns = append(fk.RefColumns, referencedColumn(fk.RefTable, row.RefColumn)) + } + return nil +} + +// referencedTable is the inspected table a foreign key references, or a +// stand-in naming it when the reference leaves the inspected scope. +func referencedTable(s *schema.Schema, refSchema, name string) *schema.Table { + if strings.EqualFold(refSchema, s.Name) { + if t, ok := s.Table(name); ok { + return t + } + return &schema.Table{Name: name, Schema: s} + } + return &schema.Table{Name: name, Schema: &schema.Schema{Name: refSchema}} +} + +func referencedColumn(t *schema.Table, name string) *schema.Column { + if c, ok := t.Column(name); ok { + return c + } + return &schema.Column{Name: name} +} + +func referenceOption(action string) schema.ReferenceOption { + return schema.ReferenceOption(strings.ReplaceAll(action, "_", " ")) +} diff --git a/database/atlas/sqlserver/inspect_test.go b/database/atlas/sqlserver/inspect_test.go new file mode 100644 index 00000000..5d4beaef --- /dev/null +++ b/database/atlas/sqlserver/inspect_test.go @@ -0,0 +1,73 @@ +package sqlserver_test + +import ( + "github.com/DATA-DOG/go-sqlmock" + + "ariga.io/atlas/sql/migrate" + "ariga.io/atlas/sql/schema" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/flanksource/sqlglot-go/database/atlas/sqlserver" + planner "github.com/flanksource/sqlglot-go/migrate/atlas/sqlserver" +) + +var _ = Describe("SQL Server inspection", func() { + var ( + mock sqlmock.Sqlmock + driver migrate.Driver + ) + + BeforeEach(func() { + db, m, err := sqlmock.New() + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { + Expect(m.ExpectationsWereMet()).To(Succeed()) + m.ExpectClose() + Expect(db.Close()).To(Succeed()) + }) + mock = m + mock.ExpectQuery(`SERVERPROPERTY\('ProductVersion'\).*compatibility_level`). + WillReturnRows(sqlmock.NewRows([]string{"version", "level"}).AddRow("16.0.4135.4", 100)) + driver, err = sqlserver.Open(db) + Expect(err).ToNot(HaveOccurred()) + Expect(driver.(*sqlserver.Driver).CompatibilityLevel()).To(Equal(100)) + }) + + expectSchema := func() { + mock.ExpectQuery(`INFORMATION_SCHEMA\.SCHEMATA`).WithArgs("dbo"). + WillReturnRows(sqlmock.NewRows([]string{"name"}).AddRow("dbo")) + } + expectRoutine := func(definition string) { + mock.ExpectQuery(`FROM sys\.objects o.*o\.name IN \(@p2\)`).WithArgs("dbo", "acme_touch_order"). + WillReturnRows(sqlmock.NewRows([]string{"schema", "name", "type", "definition", "return_type"}). + AddRow("dbo", "acme_touch_order", "P", definition, nil)) + } + + It("reads only the named routines, and no tables, when the mode asks for functions alone", func() { + expectSchema() + expectRoutine(touchOrderLive) + s, err := driver.InspectSchema(GinkgoT().Context(), "dbo", &schema.InspectOptions{ + Mode: schema.InspectFuncs, Tables: []string{"acme_touch_order"}, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(s.Tables).To(BeEmpty()) + Expect(s.Procs).To(HaveLen(1)) + Expect(s.Procs[0].Attrs).To(ConsistOf(&planner.Definition{Text: touchOrderLive})) + }) + + It("fails on a live definition that does not parse, naming the routine", func() { + expectSchema() + expectRoutine("CREATE PROCEDURE dbo.acme_touch_order AS BEGIN") + _, err := driver.InspectSchema(GinkgoT().Context(), "dbo", &schema.InspectOptions{ + Mode: schema.InspectFuncs, Tables: []string{"acme_touch_order"}, + }) + Expect(err).To(MatchError(ContainSubstring("sqlserver: live routine dbo.acme_touch_order: sqlserver: parse routine"))) + }) + + It("refuses to inspect views", func() { + _, err := driver.InspectSchema(GinkgoT().Context(), "dbo", &schema.InspectOptions{Mode: schema.InspectViews}) + Expect(err).To(MatchError(ContainSubstring("views, triggers, types and objects are not supported by the SQL Server driver"))) + }) +}) diff --git a/database/atlas/sqlserver/migrate.go b/database/atlas/sqlserver/migrate.go new file mode 100644 index 00000000..c08c0dcd --- /dev/null +++ b/database/atlas/sqlserver/migrate.go @@ -0,0 +1,24 @@ +package sqlserver + +import ( + "context" + + "ariga.io/atlas/sql/migrate" + "ariga.io/atlas/sql/schema" + + "github.com/flanksource/sqlglot-go/database/atlas/internal/sqlx" + planner "github.com/flanksource/sqlglot-go/migrate/atlas/sqlserver" +) + +// DefaultPlan plans without a connection and refuses statement execution. +var DefaultPlan migrate.PlanApplier = &planApply{conn: &conn{ExecQuerier: sqlx.NoRows}} + +type planApply struct{ *conn } + +func (*planApply) PlanChanges(ctx context.Context, name string, changes []schema.Change, options ...migrate.PlanOption) (*migrate.Plan, error) { + return planner.DefaultPlan.PlanChanges(ctx, name, changes, options...) +} + +func (p *planApply) ApplyChanges(ctx context.Context, changes []schema.Change, options ...migrate.PlanOption) error { + return sqlx.ApplyChanges(ctx, changes, p, options...) +} diff --git a/database/atlas/sqlserver/migrate_test.go b/database/atlas/sqlserver/migrate_test.go new file mode 100644 index 00000000..78bb8c07 --- /dev/null +++ b/database/atlas/sqlserver/migrate_test.go @@ -0,0 +1,129 @@ +package sqlserver_test + +import ( + "ariga.io/atlas/sql/migrate" + "ariga.io/atlas/sql/schema" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/flanksource/sqlglot-go/database/atlas/sqlserver" + planner "github.com/flanksource/sqlglot-go/migrate/atlas/sqlserver" +) + +func planCmds(changes ...schema.Change) []string { + plan, err := planner.DefaultPlan.PlanChanges(GinkgoT().Context(), "golden", changes) + Expect(err).ToNot(HaveOccurred()) + cmds := make([]string, len(plan.Changes)) + for i, c := range plan.Changes { + cmds[i] = c.Cmd + } + return cmds +} + +// declaredOrders is acme_orders as ordersHCL declares it. +func declaredOrders() *schema.Table { + var realm schema.Realm + Expect(sqlserver.EvalHCLBytes([]byte(ordersHCL), &realm, nil)).To(Succeed()) + t, ok := realm.Schemas[0].Table("acme_orders") + Expect(ok).To(BeTrue()) + return t +} + +func modify(t *schema.Table, changes ...schema.Change) *schema.ModifyTable { + return &schema.ModifyTable{T: t, Changes: changes} +} + +var _ = Describe("SQL Server plan", func() { + It("creates a table with identity, named default, clustered key and check, then its filtered covering index", func() { + Expect(planCmds(&schema.AddTable{T: declaredOrders()})).To(Equal([]string{ + "CREATE TABLE [dbo].[acme_orders] ([id] int IDENTITY(1,1) NOT NULL, " + + "[status] nvarchar(20) NOT NULL CONSTRAINT [DF_acme_orders_status] DEFAULT N'open', " + + "[note] nvarchar(max) NULL, [total] decimal(18,2) NOT NULL, " + + "PRIMARY KEY CLUSTERED ([id]), CONSTRAINT [CK_acme_orders_total] CHECK (total >= 0))", + "CREATE NONCLUSTERED INDEX [IX_acme_orders_status] ON [dbo].[acme_orders] ([status]) " + + "INCLUDE ([total]) WHERE status <> 'closed'", + })) + }) + + It("adds a column with its default", func() { + t := declaredOrders() + priority := schema.NewIntColumn("priority", "int").SetDefault(&schema.Literal{V: "0"}) + Expect(planCmds(modify(t, &schema.AddColumn{C: priority}))).To(Equal([]string{ + "ALTER TABLE [dbo].[acme_orders] ADD [priority] int NOT NULL CONSTRAINT [DF_acme_orders_priority] DEFAULT 0", + })) + }) + + It("rebinds a live default around a type change", func() { + t := declaredOrders() + live := schema.NewColumn("status").SetType(&schema.StringType{T: "nvarchar", Size: 10}). + SetDefault(&schema.RawExpr{X: "(N'new')"}).AddAttrs(&planner.DefaultConstraint{Name: "DF__acme_orde__statu__1A2B"}) + declared, _ := t.Column("status") + Expect(planCmds(modify(t, &schema.ModifyColumn{From: live, To: declared, Change: schema.ChangeType | schema.ChangeDefault}))).To(Equal([]string{ + "ALTER TABLE [dbo].[acme_orders] DROP CONSTRAINT [DF__acme_orde__statu__1A2B]", + "ALTER TABLE [dbo].[acme_orders] ALTER COLUMN [status] nvarchar(20) NOT NULL", + "ALTER TABLE [dbo].[acme_orders] ADD CONSTRAINT [DF_acme_orders_status] DEFAULT N'open' FOR [status]", + })) + }) + + It("changes nullability alone with ALTER COLUMN", func() { + t := declaredOrders() + declared, _ := t.Column("note") + live := schema.NewColumn("note").SetType(declared.Type.Type) + Expect(planCmds(modify(t, &schema.ModifyColumn{From: live, To: declared, Change: schema.ChangeNull}))).To(Equal([]string{ + "ALTER TABLE [dbo].[acme_orders] ALTER COLUMN [note] nvarchar(max) NULL", + })) + }) + + It("adds a unique clustered index and drops a constraint-backed unique index", func() { + t := declaredOrders() + status, _ := t.Column("status") + added := schema.NewUniqueIndex("UX_acme_orders_status").AddColumns(status).AddAttrs(&planner.Clustered{}) + dropped := schema.NewUniqueIndex("UQ_acme_orders_note").AddAttrs(&planner.UniqueConstraint{}) + Expect(planCmds(modify(t, &schema.AddIndex{I: added}, &schema.DropIndex{I: dropped}))).To(Equal([]string{ + "ALTER TABLE [dbo].[acme_orders] DROP CONSTRAINT [UQ_acme_orders_note]", + "CREATE UNIQUE CLUSTERED INDEX [UX_acme_orders_status] ON [dbo].[acme_orders] ([status])", + })) + }) + + It("replaces a check on the referenced table before adding the foreign key to it", func() { + t := declaredOrders() + lines := schema.NewTable("acme_order_lines").AddColumns(schema.NewIntColumn("order_id", "int")) + t.Schema.AddTables(lines) + orderID, _ := lines.Column("order_id") + id, _ := t.Column("id") + fk := schema.NewForeignKey("FK_acme_order_lines_order").AddColumns(orderID).SetRefTable(t).AddRefColumns(id). + SetOnDelete(schema.Cascade) + lines.AddForeignKeys(fk) + from := &schema.Check{Name: "CK_acme_orders_total", Expr: "([total]>=(0))"} + to := &schema.Check{Name: "CK_acme_orders_total", Expr: "total > 0"} + Expect(planCmds(modify(lines, &schema.AddForeignKey{F: fk}), modify(t, &schema.ModifyCheck{From: from, To: to}))).To(Equal([]string{ + "ALTER TABLE [dbo].[acme_orders] DROP CONSTRAINT [CK_acme_orders_total]", + "ALTER TABLE [dbo].[acme_orders] ADD CONSTRAINT [CK_acme_orders_total] CHECK (total > 0)", + "ALTER TABLE [dbo].[acme_order_lines] ADD CONSTRAINT [FK_acme_order_lines_order] FOREIGN KEY ([order_id]) " + + "REFERENCES [dbo].[acme_orders] ([id]) ON DELETE CASCADE", + })) + }) + + It("runs a declared routine as CREATE OR ALTER, byte for byte, and drops by name", func() { + declared := routineSchema(countOrdersProc, orderTotalFunc) + live := routineSchema(touchOrderLive) + Expect(planCmds( + &schema.AddProc{P: declared.Procs[0]}, + &schema.ModifyFunc{From: declared.Funcs[0], To: declared.Funcs[0]}, + &schema.DropProc{P: live.Procs[0]}, + )).To(Equal([]string{ + "-- Counts a customer's orders.\nCREATE OR ALTER PROCEDURE dbo.acme_count_orders\n @customer_id INT,\n" + + " @count INT OUTPUT\nAS\nBEGIN\n SET NOCOUNT ON;\n" + + " SET @count = (SELECT COUNT(*) FROM dbo.acme_orders WHERE customer_id = @customer_id);\nEND", + orderTotalFunc, + "DROP PROCEDURE [dbo].[acme_touch_order]", + })) + }) + + It("refuses a view without its complete T-SQL definition", func() { + _, err := planner.DefaultPlan.PlanChanges(GinkgoT().Context(), "golden", + []schema.Change{&schema.AddView{V: schema.NewView("acme_open", "SELECT 1")}}, func(*migrate.PlanOptions) {}) + Expect(err).To(MatchError("sqlserver: routine acme_open has no T-SQL definition")) + }) +}) diff --git a/database/atlas/sqlserver/routine_test.go b/database/atlas/sqlserver/routine_test.go new file mode 100644 index 00000000..ddd93de6 --- /dev/null +++ b/database/atlas/sqlserver/routine_test.go @@ -0,0 +1,86 @@ +package sqlserver_test + +import ( + "ariga.io/atlas/sql/schema" + + planner "github.com/flanksource/sqlglot-go/migrate/atlas/sqlserver" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +const countOrdersProc = `-- Counts a customer's orders. +CREATE PROCEDURE dbo.acme_count_orders + @customer_id INT, + @count INT OUTPUT +AS +BEGIN + SET NOCOUNT ON; + SET @count = (SELECT COUNT(*) FROM dbo.acme_orders WHERE customer_id = @customer_id); +END` + +const orderTotalFunc = `CREATE OR ALTER FUNCTION acme_order_total(@order_id INT) +RETURNS DECIMAL(18, 2) +AS +BEGIN + RETURN (SELECT SUM(quantity) FROM dbo.acme_order_lines WHERE order_id = @order_id); +END` + +var _ = Describe("SQL Server routine definitions", func() { + It("reads a procedure, its parameters and its definition into the schema", func() { + s := schema.New("dbo") + routine, err := planner.ParseRoutine(countOrdersProc) + Expect(err).ToNot(HaveOccurred()) + Expect(routine.AddTo(s)).To(Succeed()) + + Expect(s.Procs).To(HaveLen(1)) + proc := s.Procs[0] + Expect(proc.Name).To(Equal("acme_count_orders")) + Expect(proc.Schema).To(BeIdenticalTo(s)) + Expect(proc.Args).To(Equal([]*schema.FuncArg{ + {Name: "@customer_id", Type: &schema.IntegerType{T: "int"}, Mode: schema.FuncArgModeIn}, + {Name: "@count", Type: &schema.IntegerType{T: "int"}, Mode: schema.FuncArgModeInOut}, + })) + Expect(proc.Attrs).To(ConsistOf(&planner.Definition{Text: countOrdersProc})) + }) + + It("reads a scalar function and its return type", func() { + s := schema.New("dbo") + routine, err := planner.ParseRoutine(orderTotalFunc) + Expect(err).ToNot(HaveOccurred()) + Expect(routine.Name).To(Equal("acme_order_total")) + Expect(routine.AddTo(s)).To(Succeed()) + Expect(s.Funcs).To(HaveLen(1)) + Expect(s.Funcs[0].Ret).To(Equal(&schema.DecimalType{T: "decimal", Precision: 18, Scale: 2})) + }) + + DescribeTable("refuses a definition that is not exactly one routine", + func(definition, message string) { + _, err := planner.ParseRoutine(definition) + Expect(err).To(MatchError(ContainSubstring(message))) + }, + Entry("a table", "CREATE TABLE dbo.acme_orders (id INT)", "makes no routine"), + Entry("two routines", "CREATE PROCEDURE dbo.acme_one AS BEGIN SELECT 1; END\nCREATE PROCEDURE dbo.acme_two AS BEGIN SELECT 2; END", "sqlserver: parse routine"), + ) + + It("refuses a routine qualified with another schema", func() { + routine, err := planner.ParseRoutine("CREATE PROCEDURE tenant_x.acme_purge AS BEGIN SELECT 1; END") + Expect(err).ToNot(HaveOccurred()) + Expect(routine.AddTo(schema.New("dbo"))).To(MatchError(`sqlserver: routine tenant_x.acme_purge is declared in schema "tenant_x", not "dbo"`)) + }) + + DescribeTable("CreateOrAlter turns a definition into CREATE OR ALTER, keeping every other byte", + func(definition, expected string) { + Expect(planner.CreateOrAlter(definition)).To(Equal(expected)) + }, + Entry("leading comments and a CREATE PROCEDURE", + "-- note\n/* block */ create PROCEDURE dbo.acme_x AS SELECT 1", + "-- note\n/* block */ create OR ALTER PROCEDURE dbo.acme_x AS SELECT 1"), + Entry("already CREATE OR ALTER", "CREATE OR ALTER PROC dbo.acme_x AS SELECT 1", "CREATE OR ALTER PROC dbo.acme_x AS SELECT 1"), + Entry("a unicode literal before the body", "CREATE FUNCTION dbo.acme_f() RETURNS NVARCHAR(4) AS BEGIN RETURN N'ünï'; END", + "CREATE OR ALTER FUNCTION dbo.acme_f() RETURNS NVARCHAR(4) AS BEGIN RETURN N'ünï'; END"), + ) + + It("rewrites ALTER definitions from the live catalog", func() { + Expect(planner.CreateOrAlter("ALTER PROCEDURE dbo.acme_x AS SELECT 1")).To(Equal("CREATE OR ALTER PROCEDURE dbo.acme_x AS SELECT 1")) + }) +}) diff --git a/database/atlas/sqlserver/sqlspec.go b/database/atlas/sqlserver/sqlspec.go new file mode 100644 index 00000000..2db56b21 --- /dev/null +++ b/database/atlas/sqlserver/sqlspec.go @@ -0,0 +1,301 @@ +package sqlserver + +import ( + "fmt" + "slices" + + "ariga.io/atlas/schemahcl" + "ariga.io/atlas/sql/schema" + "ariga.io/atlas/sql/sqlspec" + "github.com/hashicorp/hcl/v2/hclparse" + "github.com/zclconf/go-cty/cty" + + "github.com/flanksource/sqlglot-go/database/atlas/internal/specutil" + planner "github.com/flanksource/sqlglot-go/migrate/atlas/sqlserver" +) + +// doc is the HCL document the codec reads. Only schemas and tables are +// converted; the other blocks are decoded so they can be refused by name. +// Procedures and functions come from their .sql definitions, never HCL. +type doc struct { + Tables []*sqlspec.Table `spec:"table"` + Schemas []*sqlspec.Schema `spec:"schema"` + Views []*sqlspec.View `spec:"view"` + Funcs []*sqlspec.Func `spec:"function"` + Procs []*sqlspec.Func `spec:"procedure"` + Triggers []*sqlspec.Trigger `spec:"trigger"` + Sequences []*sqlspec.Sequence `spec:"sequence"` +} + +func (d *doc) refuse() error { + for kind, n := range map[string]int{ + "view": len(d.Views), "function": len(d.Funcs), "procedure": len(d.Procs), + "trigger": len(d.Triggers), "sequence": len(d.Sequences), + } { + if n > 0 { + return fmt.Errorf("sqlserver: HCL %s blocks are %w (routines come from .sql files)", kind, errUnsupported) + } + } + return nil +} + +// Codec for schemahcl. +type Codec struct { + State *schemahcl.State +} + +// Eval evaluates an Atlas DDL document into v using the input. +func (c *Codec) Eval(p *hclparse.Parser, v any, input map[string]cty.Value) error { + return c.EvalOptions(p, v, &schemahcl.EvalOptions{Variables: input}) +} + +// EvalOptions decodes the HCL with the given options into a *schema.Realm or +// a *schema.Schema. +func (c *Codec) EvalOptions(p *hclparse.Parser, v any, opts *schemahcl.EvalOptions) error { + var d doc + if err := c.State.EvalOptions(p, &d, opts); err != nil { + return err + } + if err := d.refuse(); err != nil { + return err + } + r := &schema.Realm{} + if err := specutil.Scan(r, &specutil.ScanDoc{Schemas: d.Schemas, Tables: d.Tables}, scanFuncs); err != nil { + return fmt.Errorf("sqlserver: failed converting to *schema.Realm: %w", err) + } + switch v := v.(type) { + case *schema.Realm: + *v = *r + case *schema.Schema: + if len(r.Schemas) != 1 { + return fmt.Errorf("sqlserver: expecting document to contain a single schema, got %d", len(r.Schemas)) + } + *v = *r.Schemas[0] + default: + return fmt.Errorf("sqlserver: unexpected type %T", v) + } + return nil +} + +// MarshalSpec marshals v into an Atlas DDL document. Routines are left out: +// their single source is the .sql definition. +func (c *Codec) MarshalSpec(v any) ([]byte, error) { + return specutil.Marshal(v, c.State, specutil.RealmFuncs{Schema: schemaSpec}) +} + +var scanFuncs = &specutil.ScanFuncs{Table: convertTable} + +func convertTable(spec *sqlspec.Table, parent *schema.Schema) (*schema.Table, error) { + if err := onlyKeys(spec.Extra, nil, nil); err != nil { + return nil, fmt.Errorf("table %q: %w", spec.Name, err) + } + t, err := specutil.Table(spec, parent, specutil.TableConvertFuncs{ + Column: convertColumn, + PrimaryKey: convertPrimaryKey, + Index: convertIndex, + Check: specutil.Check, + }) + if err != nil { + return nil, err + } + if slices.ContainsFunc(t.Attrs, isComment) { + return nil, fmt.Errorf("table %q: comments are %w", spec.Name, errUnsupported) + } + return t, nil +} + +func isComment(a schema.Attr) bool { + _, ok := a.(*schema.Comment) + return ok +} + +// onlyKeys refuses any extra attribute or block outside the allowed sets, so +// an unsupported clause (a computed column's `as`, a partition) is an error +// rather than silently dropped. +func onlyKeys(r schemahcl.Resource, attrs, blocks []string) error { + for _, attr := range r.Attrs { + if !slices.Contains(attrs, attr.K) { + return fmt.Errorf("attribute %q is %w", attr.K, errUnsupported) + } + } + for _, child := range r.Children { + if !slices.Contains(blocks, child.Type) { + return fmt.Errorf("block %q is %w", child.Type, errUnsupported) + } + } + return nil +} + +func convertColumn(spec *sqlspec.Column, _ *schema.Table) (*schema.Column, error) { + if err := onlyKeys(spec.Extra, []string{"default"}, []string{"identity"}); err != nil { + return nil, fmt.Errorf("column %q: %w", spec.Name, err) + } + c, err := specutil.Column(spec, convertColumnType) + if err != nil { + return nil, err + } + identity, ok := spec.Extra.Resource("identity") + if !ok { + return c, nil + } + seed, err := int64Attr(identity, "seed") + if err != nil { + return nil, fmt.Errorf("column %q identity: %w", spec.Name, err) + } + increment, err := int64Attr(identity, "increment") + if err != nil { + return nil, fmt.Errorf("column %q identity: %w", spec.Name, err) + } + return c.AddAttrs(&planner.Identity{Seed: seed, Increment: increment}), nil +} + +func int64Attr(r *schemahcl.Resource, name string) (int64, error) { + attr, ok := r.Attr(name) + if !ok { + return 0, fmt.Errorf("missing %q", name) + } + return attr.Int64() +} + +func convertColumnType(spec *sqlspec.Column) (schema.Type, error) { + return TypeRegistry.Type(spec.Type, spec.Extra.Attrs) +} + +// convertPrimaryKey reads a primary key, clustered unless `clustered = false`. +func convertPrimaryKey(spec *sqlspec.PrimaryKey, t *schema.Table) (*schema.Index, error) { + if err := onlyKeys(spec.Extra, []string{"clustered"}, nil); err != nil { + return nil, fmt.Errorf("primary key of %q: %w", t.Name, err) + } + pk, err := specutil.PrimaryKey(spec, t) + if err != nil { + return nil, err + } + return pk, clusteredAttr(spec.Extra, true, pk) +} + +// convertIndex reads an index, nonclustered unless `clustered = true`, with +// optional `include = [column.x]` and `where = "..."`. +func convertIndex(spec *sqlspec.Index, t *schema.Table) (*schema.Index, error) { + if err := onlyKeys(spec.Extra, []string{"clustered", "include", "where"}, nil); err != nil { + return nil, fmt.Errorf("index %q: %w", spec.Name, err) + } + idx, err := specutil.Index(spec, t) + if err != nil { + return nil, err + } + if err := clusteredAttr(spec.Extra, false, idx); err != nil { + return nil, err + } + if attr, ok := spec.Extra.Attr("where"); ok { + p, err := attr.String() + if err != nil { + return nil, fmt.Errorf("index %q where: %w", spec.Name, err) + } + idx.AddAttrs(&planner.IndexPredicate{P: p}) + } + return idx, includeAttr(spec, t, idx) +} + +func clusteredAttr(r schemahcl.Resource, fallback bool, idx *schema.Index) error { + clustered := fallback + if attr, ok := r.Attr("clustered"); ok { + var err error + if clustered, err = attr.Bool(); err != nil { + return fmt.Errorf("index %q clustered: %w", idx.Name, err) + } + } + if clustered { + idx.AddAttrs(&planner.Clustered{}) + } + return nil +} + +func includeAttr(spec *sqlspec.Index, t *schema.Table, idx *schema.Index) error { + attr, ok := spec.Extra.Attr("include") + if !ok { + return nil + } + refs, err := attr.Refs() + if err != nil { + return fmt.Errorf("index %q include: %w", spec.Name, err) + } + include := &planner.IndexInclude{} + for _, ref := range refs { + c, err := specutil.ColumnByRef(t, ref) + if err != nil { + return fmt.Errorf("index %q include: %w", spec.Name, err) + } + include.Columns = append(include.Columns, c.Name) + } + idx.AddAttrs(include) + return nil +} + +func schemaSpec(s *schema.Schema) (*specutil.SchemaSpec, error) { + return specutil.FromSchema(s, &specutil.SchemaFuncs{Table: tableSpec}) +} + +func tableSpec(t *schema.Table) (*sqlspec.Table, error) { + indexSpec := func(idx *schema.Index) (*sqlspec.Index, error) { return specutil.FromIndex(idx) } + return specutil.FromTable(t, specutil.TableSpecFuncs{ + Column: columnSpec, + PrimaryKey: specutil.FromPrimaryKey, + Index: indexSpec, + ForeignKey: specutil.FromForeignKey, + Check: specutil.FromCheck, + }) +} + +func columnSpec(c *schema.Column, _ *schema.Table) (*sqlspec.Column, error) { + return specutil.FromColumn(c, func(t schema.Type) (*sqlspec.Column, error) { + st, err := TypeRegistry.Convert(t) + if err != nil { + return nil, err + } + return &sqlspec.Column{Type: st}, nil + }) +} + +func sizedSpec(name string) *schemahcl.TypeSpec { + return schemahcl.NewTypeSpec(name, schemahcl.WithAttributes(schemahcl.SizeTypeAttr(false))) +} + +// TypeRegistry holds the SQL Server HCL type specs. A (max) size is written +// with sql(), e.g. `type = sql("nvarchar(max)")`. +var TypeRegistry = schemahcl.NewRegistry( + schemahcl.WithFormatter(planner.FormatType), + schemahcl.WithParser(planner.ParseType), + schemahcl.WithSpecs( + schemahcl.NewTypeSpec("bit"), schemahcl.NewTypeSpec("tinyint"), schemahcl.NewTypeSpec("smallint"), + schemahcl.NewTypeSpec("int"), schemahcl.NewTypeSpec("bigint"), + schemahcl.NewTypeSpec("decimal", schemahcl.WithAttributes(schemahcl.PrecisionTypeAttr(), schemahcl.ScaleTypeAttr())), + schemahcl.NewTypeSpec("numeric", schemahcl.WithAttributes(schemahcl.PrecisionTypeAttr(), schemahcl.ScaleTypeAttr())), + schemahcl.NewTypeSpec("money"), schemahcl.NewTypeSpec("smallmoney"), + schemahcl.NewTypeSpec("float", schemahcl.WithAttributes(schemahcl.PrecisionTypeAttr())), schemahcl.NewTypeSpec("real"), + sizedSpec("char"), sizedSpec("varchar"), sizedSpec("nchar"), sizedSpec("nvarchar"), + schemahcl.NewTypeSpec("text"), schemahcl.NewTypeSpec("ntext"), + sizedSpec("binary"), sizedSpec("varbinary"), schemahcl.NewTypeSpec("image"), + schemahcl.NewTypeSpec("date"), schemahcl.NewTypeSpec("datetime"), schemahcl.NewTypeSpec("smalldatetime"), + schemahcl.NewTypeSpec("time", schemahcl.WithAttributes(schemahcl.PrecisionTypeAttr())), + schemahcl.NewTypeSpec("datetime2", schemahcl.WithAttributes(schemahcl.PrecisionTypeAttr())), + schemahcl.NewTypeSpec("datetimeoffset", schemahcl.WithAttributes(schemahcl.PrecisionTypeAttr())), + schemahcl.NewTypeSpec("uniqueidentifier"), schemahcl.NewTypeSpec("xml"), + schemahcl.NewTypeSpec("sql_variant"), schemahcl.NewTypeSpec("rowversion"), + ), +) + +var ( + codec = &Codec{ + State: schemahcl.New( + schemahcl.WithTypes("table.column.type", TypeRegistry.Specs()), + schemahcl.WithScopedEnums("table.foreign_key.on_update", specutil.ReferenceVars...), + schemahcl.WithScopedEnums("table.foreign_key.on_delete", specutil.ReferenceVars...), + ), + } + // MarshalHCL marshals v into an Atlas HCL DDL document. + MarshalHCL = schemahcl.MarshalerFunc(codec.MarshalSpec) + // EvalHCL implements the schemahcl.Evaluator interface. + EvalHCL = schemahcl.EvalFunc(codec.Eval) + // EvalHCLBytes evaluates an HCL document from a byte slice. + EvalHCLBytes = specutil.HCLBytesFunc(EvalHCL) +) diff --git a/database/atlas/sqlserver/sqlspec_test.go b/database/atlas/sqlserver/sqlspec_test.go new file mode 100644 index 00000000..6d223de0 --- /dev/null +++ b/database/atlas/sqlserver/sqlspec_test.go @@ -0,0 +1,110 @@ +package sqlserver_test + +import ( + "ariga.io/atlas/sql/schema" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/flanksource/sqlglot-go/database/atlas/sqlserver" + planner "github.com/flanksource/sqlglot-go/migrate/atlas/sqlserver" +) + +const ordersHCL = ` +schema "dbo" {} + +table "acme_orders" { + schema = schema.dbo + column "id" { + type = int + identity { + seed = 1 + increment = 1 + } + } + column "status" { + type = nvarchar(20) + default = "open" + } + column "note" { + type = sql("nvarchar(max)") + null = true + } + column "total" { + type = decimal(18, 2) + } + primary_key { + columns = [column.id] + } + index "IX_acme_orders_status" { + columns = [column.status] + include = [column.total] + where = "status <> 'closed'" + } + check "CK_acme_orders_total" { + expr = "total >= 0" + } +} +` + +var _ = Describe("SQL Server HCL codec", func() { + It("reads tables with identity, defaults, (max) types, clustering, included columns and filters", func() { + var realm schema.Realm + Expect(sqlserver.EvalHCLBytes([]byte(ordersHCL), &realm, nil)).To(Succeed()) + t, ok := realm.Schemas[0].Table("acme_orders") + Expect(ok).To(BeTrue()) + + id, _ := t.Column("id") + Expect(id.Type.Type).To(Equal(&schema.IntegerType{T: "int"})) + Expect(id.Attrs).To(ContainElement(&planner.Identity{Seed: 1, Increment: 1})) + + status, _ := t.Column("status") + Expect(status.Type.Type).To(Equal(&schema.StringType{T: "nvarchar", Size: 20})) + Expect(status.Default).To(Equal(&schema.Literal{V: "open"})) + + note, _ := t.Column("note") + Expect(note.Type.Type).To(Equal(&schema.StringType{T: "nvarchar", Size: -1})) + Expect(note.Type.Null).To(BeTrue()) + + Expect(t.PrimaryKey.Attrs).To(ContainElement(&planner.Clustered{})) + idx, _ := t.Index("IX_acme_orders_status") + Expect(idx.Attrs).ToNot(ContainElement(&planner.Clustered{})) + Expect(idx.Attrs).To(ContainElement(&planner.IndexInclude{Columns: []string{"total"}})) + Expect(idx.Attrs).To(ContainElement(&planner.IndexPredicate{P: "status <> 'closed'"})) + }) + + DescribeTable("refuses what the driver does not manage", + func(hcl, message string) { + var realm schema.Realm + err := sqlserver.EvalHCLBytes([]byte(hcl), &realm, nil) + Expect(err).To(MatchError(ContainSubstring(message))) + }, + Entry("a view", `schema "dbo" {} +view "acme_open" { + schema = schema.dbo + as = "SELECT 1" +}`, "HCL view blocks are not supported"), + Entry("a procedure block", `schema "dbo" {} +procedure "acme_purge" { + schema = schema.dbo + as = "SELECT 1" +}`, "HCL procedure blocks are not supported"), + Entry("a computed column", `schema "dbo" {} +table "acme_orders" { + schema = schema.dbo + column "total" { + type = int + as { + expr = "1 + 1" + } + } +}`, `column "total": block "as" is not supported`), + Entry("an alias type", `schema "dbo" {} +table "acme_orders" { + schema = schema.dbo + column "name" { + type = sql("sysname") + } +}`, `type "sysname" is not supported`), + ) +}) diff --git a/database/capabilities/capabilities.go b/database/capabilities/capabilities.go new file mode 100644 index 00000000..1b70fc7f --- /dev/null +++ b/database/capabilities/capabilities.go @@ -0,0 +1,279 @@ +package capabilities + +import ( + "context" + "crypto/sha256" + "database/sql" + "encoding/hex" + "encoding/json" + "fmt" + "sort" + "strconv" + "strings" +) + +type BackendCapability string + +const BackendCapabilityArrayFilters BackendCapability = "arrayFilters" + +type BackendCapabilityStatus struct { + Supported bool `json:"supported"` + Detail string `json:"detail,omitempty"` + MinimumVersion string `json:"minimumVersion,omitempty"` + MinimumCompatibilityLevel *int `json:"minimumCompatibilityLevel,omitempty"` +} + +type BackendCapabilities struct { + Backend string `json:"backend"` + Version string `json:"version"` + Database string `json:"database"` + CompatibilityLevel *int `json:"compatibilityLevel,omitempty"` + Features map[BackendCapability]BackendCapabilityStatus `json:"features"` +} + +func (c BackendCapabilities) Require(capability BackendCapability) error { + if err := c.Validate(); err != nil { + return err + } + status, ok := c.Features[capability] + if !ok { + return fmt.Errorf("backend %q did not report capability %q", c.Backend, capability) + } + if status.Supported { + return nil + } + requirements := make([]string, 0, 2) + if status.MinimumVersion != "" { + requirements = append(requirements, "version "+status.MinimumVersion+" or newer") + } + if status.MinimumCompatibilityLevel != nil { + requirements = append(requirements, fmt.Sprintf("compatibility level %d or newer", *status.MinimumCompatibilityLevel)) + } + reason := status.Detail + if len(requirements) > 0 { + if reason != "" { + reason += "; " + } + reason += "requires " + strings.Join(requirements, " and ") + } + if reason == "" { + reason = "the feature is unavailable" + } + return fmt.Errorf("backend %q does not support capability %q: %s", c.Backend, capability, reason) +} + +func (c BackendCapabilities) Fingerprint() (string, error) { + if err := c.Validate(); err != nil { + return "", err + } + type feature struct { + Name BackendCapability `json:"name"` + Status BackendCapabilityStatus `json:"status"` + } + names := make([]string, 0, len(c.Features)) + for name := range c.Features { + names = append(names, string(name)) + } + sort.Strings(names) + features := make([]feature, 0, len(names)) + for _, name := range names { + capability := BackendCapability(name) + features = append(features, feature{Name: capability, Status: c.Features[capability]}) + } + payload, err := json.Marshal(struct { + Backend string `json:"backend"` + Version string `json:"version"` + Database string `json:"database"` + CompatibilityLevel *int `json:"compatibilityLevel,omitempty"` + Features []feature `json:"features"` + }{c.Backend, c.Version, c.Database, c.CompatibilityLevel, features}) + if err != nil { + return "", fmt.Errorf("encode backend capability fingerprint: %w", err) + } + digest := sha256.Sum256(payload) + return "backend-capabilities-v1:" + hex.EncodeToString(digest[:]), nil +} + +func (c BackendCapabilities) Validate() error { + if strings.TrimSpace(c.Backend) == "" { + return fmt.Errorf("backend capability metadata backend is empty") + } + if len(c.Features) == 0 { + return fmt.Errorf("backend capability metadata features are missing") + } + return nil +} + +const ( + postgresCapabilitiesQuery = `SELECT current_database(), current_setting('server_version'), current_setting('server_version_num')::integer` + mysqlCapabilitiesQuery = `SELECT COALESCE(DATABASE(), ''), VERSION()` + sqlServerCapabilitiesQuery = `SELECT DB_NAME(), CAST(SERVERPROPERTY('ProductVersion') AS nvarchar(128)), compatibility_level +FROM sys.databases +WHERE name = DB_NAME()` + sqliteCapabilitiesQuery = `SELECT 'main', sqlite_version(), + EXISTS (SELECT 1 FROM json_each('["probe"]') WHERE type = 'text' AND value = 'probe')` + clickHouseCapabilitiesQuery = `SELECT currentDatabase(), version()` +) + +func ProbeBackendCapabilities(ctx context.Context, db *sql.DB, backend string) (BackendCapabilities, error) { + if db == nil { + return BackendCapabilities{}, fmt.Errorf("probe backend capabilities: nil sql database") + } + backend = normalizeCapabilityBackend(backend) + var capabilities BackendCapabilities + var err error + switch backend { + case "postgres": + capabilities, err = probePostgresCapabilities(ctx, db) + case "mysql": + capabilities, err = probeMySQLCapabilities(ctx, db) + case "sqlserver": + capabilities, err = probeSQLServerCapabilities(ctx, db) + case "sqlite": + capabilities, err = probeSQLiteCapabilities(ctx, db) + case "clickhouse": + capabilities, err = probeClickHouseCapabilities(ctx, db) + default: + return BackendCapabilities{}, fmt.Errorf("probe backend capabilities: unsupported SQL backend %q", backend) + } + if err != nil { + return BackendCapabilities{}, fmt.Errorf("probe %s backend capabilities: %w", backend, err) + } + if err := capabilities.Validate(); err != nil { + return BackendCapabilities{}, err + } + if strings.TrimSpace(capabilities.Version) == "" { + return BackendCapabilities{}, fmt.Errorf("backend capability metadata version is empty") + } + if strings.TrimSpace(capabilities.Database) == "" { + return BackendCapabilities{}, fmt.Errorf("backend capability metadata database is empty") + } + return capabilities, nil +} + +func probePostgresCapabilities(ctx context.Context, db *sql.DB) (BackendCapabilities, error) { + var database, version string + var versionNumber int + if err := db.QueryRowContext(ctx, postgresCapabilitiesQuery).Scan(&database, &version, &versionNumber); err != nil { + return BackendCapabilities{}, err + } + status := BackendCapabilityStatus{Detail: "native and JSON arrays", MinimumVersion: "9.5"} + if versionNumber >= 90500 { + status.Supported = true + } + return capabilities(BackendCapabilities{Backend: "postgres", Version: version, Database: database}, status), nil +} + +func probeMySQLCapabilities(ctx context.Context, db *sql.DB) (BackendCapabilities, error) { + var database, version string + if err := db.QueryRowContext(ctx, mysqlCapabilitiesQuery).Scan(&database, &version); err != nil { + return BackendCapabilities{}, err + } + minimum, minimumVersion, product := [3]int{8, 0, 4}, "8.0.4", "MySQL" + if strings.Contains(strings.ToLower(version), "mariadb") { + minimum, minimumVersion, product = [3]int{10, 6, 0}, "10.6.0", "MariaDB" + } + status := BackendCapabilityStatus{Detail: "JSON arrays via JSON_TABLE", MinimumVersion: minimumVersion} + parsed, err := parseBackendVersion(version) + if err != nil { + return BackendCapabilities{}, fmt.Errorf("parse %s version %q: %w", product, version, err) + } + if versionAtLeast(parsed, minimum) { + status.Supported = true + } + return capabilities(BackendCapabilities{Backend: "mysql", Version: version, Database: database}, status), nil +} + +func probeSQLServerCapabilities(ctx context.Context, db *sql.DB) (BackendCapabilities, error) { + var database, version string + var compatibilityLevel int + if err := db.QueryRowContext(ctx, sqlServerCapabilitiesQuery).Scan(&database, &version, &compatibilityLevel); err != nil { + return BackendCapabilities{}, err + } + minimum := intPointer(130) + status := BackendCapabilityStatus{ + Supported: compatibilityLevel >= *minimum, Detail: "JSON arrays via OPENJSON", MinimumCompatibilityLevel: minimum, + } + return capabilities(BackendCapabilities{ + Backend: "sqlserver", Version: version, Database: database, CompatibilityLevel: intPointer(compatibilityLevel), + }, status), nil +} + +func probeSQLiteCapabilities(ctx context.Context, db *sql.DB) (BackendCapabilities, error) { + var database, version string + var jsonArrays int + if err := db.QueryRowContext(ctx, sqliteCapabilitiesQuery).Scan(&database, &version, &jsonArrays); err != nil { + return BackendCapabilities{}, err + } + status := BackendCapabilityStatus{Supported: jsonArrays == 1, Detail: "JSON arrays via json_each"} + if !status.Supported { + status.Detail = "SQLite JSON functions are unavailable" + } + return capabilities(BackendCapabilities{Backend: "sqlite", Version: version, Database: database}, status), nil +} + +func probeClickHouseCapabilities(ctx context.Context, db *sql.DB) (BackendCapabilities, error) { + var database, version string + if err := db.QueryRowContext(ctx, clickHouseCapabilitiesQuery).Scan(&database, &version); err != nil { + return BackendCapabilities{}, err + } + return capabilities(BackendCapabilities{Backend: "clickhouse", Version: version, Database: database}, BackendCapabilityStatus{ + Supported: true, Detail: "native Array(String) values", + }), nil +} + +func capabilities(base BackendCapabilities, arrayFilters BackendCapabilityStatus) BackendCapabilities { + base.Features = map[BackendCapability]BackendCapabilityStatus{BackendCapabilityArrayFilters: arrayFilters} + return base +} + +func normalizeCapabilityBackend(backend string) string { + switch strings.ToLower(strings.TrimSpace(backend)) { + case "postgres", "postgresql", "pgx": + return "postgres" + case "mysql": + return "mysql" + case "sql_server", "sqlserver", "mssql": + return "sqlserver" + case "sqlite", "sqlite3": + return "sqlite" + case "clickhouse": + return "clickhouse" + default: + return strings.ToLower(strings.TrimSpace(backend)) + } +} + +func parseBackendVersion(version string) ([3]int, error) { + var result [3]int + parts := strings.SplitN(strings.TrimSpace(version), ".", len(result)) + if len(parts) != len(result) { + return result, fmt.Errorf("expected major.minor.patch") + } + for index, part := range parts { + digits := strings.TrimLeftFunc(part, func(char rune) bool { return char >= '0' && char <= '9' }) + digits = part[:len(part)-len(digits)] + if digits == "" { + return result, fmt.Errorf("component %d is not numeric", index+1) + } + value, err := strconv.Atoi(digits) + if err != nil { + return result, fmt.Errorf("component %d: %w", index+1, err) + } + result[index] = value + } + return result, nil +} + +func versionAtLeast(version, minimum [3]int) bool { + for index := range version { + if version[index] != minimum[index] { + return version[index] > minimum[index] + } + } + return true +} + +func intPointer(value int) *int { + return &value +} diff --git a/database/capabilities/capabilities_integration_test.go b/database/capabilities/capabilities_integration_test.go new file mode 100644 index 00000000..62b43e09 --- /dev/null +++ b/database/capabilities/capabilities_integration_test.go @@ -0,0 +1,21 @@ +package capabilities + +import ( + "context" + "database/sql" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("backend capability integration", func() { + It("probes the linked SQLite driver's real JSON implementation", func() { + db, err := sql.Open("sqlite", ":memory:") + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { _ = db.Close() }) + + capabilities, err := ProbeBackendCapabilities(context.Background(), db, "sqlite") + Expect(err).ToNot(HaveOccurred()) + Expect(capabilities.Require(BackendCapabilityArrayFilters)).To(Succeed()) + }) +}) diff --git a/database/capabilities/capabilities_test.go b/database/capabilities/capabilities_test.go new file mode 100644 index 00000000..80d22ff9 --- /dev/null +++ b/database/capabilities/capabilities_test.go @@ -0,0 +1,147 @@ +package capabilities + +import ( + "context" + "database/sql/driver" + "regexp" + + "github.com/DATA-DOG/go-sqlmock" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("backend capabilities", func() { + probe := func(backend, statement string, columns []string, values ...driver.Value) (BackendCapabilities, error) { + db, mock, err := sqlmock.New() + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { _ = db.Close() }) + mock.ExpectQuery(regexp.QuoteMeta(statement)).WillReturnRows(sqlmock.NewRows(columns).AddRow(values...)) + capabilities, err := ProbeBackendCapabilities(context.Background(), db, backend) + Expect(mock.ExpectationsWereMet()).To(Succeed()) + return capabilities, err + } + + It("reads PostgreSQL's numeric server version", func() { + capabilities, err := probe("postgres", postgresCapabilitiesQuery, + []string{"database", "version", "version_num"}, "warehouse", "17.4", 170004) + Expect(err).ToNot(HaveOccurred()) + Expect(capabilities.Database).To(Equal("warehouse")) + Expect(capabilities.Features[BackendCapabilityArrayFilters]).To(Equal(BackendCapabilityStatus{ + Supported: true, Detail: "native and JSON arrays", MinimumVersion: "9.5", + })) + }) + + DescribeTable("classifies MySQL array filtering from live version metadata", + func(version string, supported bool, detail, minimum string) { + capabilities, err := probe("mysql", mysqlCapabilitiesQuery, + []string{"database", "version"}, "warehouse", version) + Expect(err).ToNot(HaveOccurred()) + Expect(capabilities.Features[BackendCapabilityArrayFilters]).To(Equal(BackendCapabilityStatus{ + Supported: supported, Detail: detail, MinimumVersion: minimum, + })) + }, + Entry("the minimum MySQL release", "8.0.4", true, "JSON arrays via JSON_TABLE", "8.0.4"), + Entry("an older MySQL release", "8.0.3", false, "JSON arrays via JSON_TABLE", "8.0.4"), + Entry("the minimum MariaDB release", "10.6.0-MariaDB", true, "JSON arrays via JSON_TABLE", "10.6.0"), + Entry("an older MariaDB release", "10.5.27-MariaDB", false, "JSON arrays via JSON_TABLE", "10.6.0"), + ) + + It("uses SQL Server database compatibility rather than the Azure product version", func() { + capabilities, err := probe("sqlserver", sqlServerCapabilitiesQuery, + []string{"database", "version", "compatibility_level"}, "warehouse", "12.0.2000.8", 160) + Expect(err).ToNot(HaveOccurred()) + Expect(capabilities.Version).To(Equal("12.0.2000.8")) + Expect(capabilities.CompatibilityLevel).To(HaveValue(Equal(160))) + Expect(capabilities.Features[BackendCapabilityArrayFilters]).To(Equal(BackendCapabilityStatus{ + Supported: true, Detail: "JSON arrays via OPENJSON", MinimumCompatibilityLevel: new(130), + })) + }) + + It("rejects SQL Server databases below compatibility level 130", func() { + capabilities, err := probe("sql_server", sqlServerCapabilitiesQuery, + []string{"database", "version", "compatibility_level"}, "warehouse", "16.0.1000.6", 120) + Expect(err).ToNot(HaveOccurred()) + Expect(capabilities.Features[BackendCapabilityArrayFilters].Supported).To(BeFalse()) + Expect(capabilities.Require(BackendCapabilityArrayFilters)).To(MatchError(ContainSubstring("compatibility level 130"))) + }) + + DescribeTable("probes JSON expansion instead of assuming SQLite build features", + func(result int, supported bool) { + capabilities, err := probe("sqlite", sqliteCapabilitiesQuery, + []string{"database", "version", "json_arrays"}, "main", "3.50.4", result) + Expect(err).ToNot(HaveOccurred()) + Expect(capabilities.Features[BackendCapabilityArrayFilters].Supported).To(Equal(supported)) + }, + Entry("JSON functions are present", 1, true), + Entry("JSON functions return an unexpected result", 0, false), + ) + + It("reports ClickHouse array support as a static server feature", func() { + capabilities, err := probe("clickhouse", clickHouseCapabilitiesQuery, + []string{"database", "version"}, "default", "25.8.3.66") + Expect(err).ToNot(HaveOccurred()) + Expect(capabilities.Features[BackendCapabilityArrayFilters]).To(Equal(BackendCapabilityStatus{ + Supported: true, Detail: "native Array(String) values", + })) + }) + + It("fails when required metadata is empty", func() { + _, err := probe("postgres", postgresCapabilitiesQuery, + []string{"database", "version", "version_num"}, "warehouse", "", 170004) + Expect(err).To(MatchError(ContainSubstring("version is empty"))) + }) + + It("fails when a live probe has no selected database", func() { + _, err := probe("mysql", mysqlCapabilitiesQuery, + []string{"database", "version"}, "", "8.0.21") + Expect(err).To(MatchError(ContainSubstring("database is empty"))) + }) + + It("fails when version metadata cannot be interpreted", func() { + _, err := probe("mysql", mysqlCapabilitiesQuery, + []string{"database", "version"}, "warehouse", "not-a-version") + Expect(err).To(MatchError(ContainSubstring("parse MySQL version"))) + }) + + It("rejects capability metadata without a feature status", func() { + err := (BackendCapabilities{Backend: "postgres", Features: map[BackendCapability]BackendCapabilityStatus{}}).Validate() + Expect(err).To(MatchError(ContainSubstring("features are missing"))) + }) + + It("fingerprints feature metadata deterministically", func() { + left := BackendCapabilities{ + Backend: "postgres", Version: "17.4", Database: "warehouse", + Features: map[BackendCapability]BackendCapabilityStatus{ + BackendCapabilityArrayFilters: {Supported: true, MinimumVersion: "9.5"}, + }, + } + right := left + right.Features = map[BackendCapability]BackendCapabilityStatus{ + BackendCapabilityArrayFilters: {Supported: true, MinimumVersion: "9.5"}, + } + leftFingerprint, err := left.Fingerprint() + Expect(err).ToNot(HaveOccurred()) + rightFingerprint, err := right.Fingerprint() + Expect(err).ToNot(HaveOccurred()) + Expect(rightFingerprint).To(Equal(leftFingerprint)) + + right.Version = "17.5" + changed, err := right.Fingerprint() + Expect(err).ToNot(HaveOccurred()) + Expect(changed).ToNot(Equal(leftFingerprint)) + }) + + It("fingerprints static provider capabilities without invented server metadata", func() { + capabilities := BackendCapabilities{ + Backend: "opensearch", + Features: map[BackendCapability]BackendCapabilityStatus{ + BackendCapabilityArrayFilters: {Supported: true, Detail: "native keyword arrays"}, + }, + } + + fingerprint, err := capabilities.Fingerprint() + Expect(err).ToNot(HaveOccurred()) + Expect(fingerprint).To(HavePrefix("backend-capabilities-v1:")) + Expect(capabilities.Require(BackendCapabilityArrayFilters)).To(Succeed()) + }) +}) diff --git a/database/capabilities/suite_test.go b/database/capabilities/suite_test.go new file mode 100644 index 00000000..ebb8946d --- /dev/null +++ b/database/capabilities/suite_test.go @@ -0,0 +1,11 @@ +package capabilities + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + _ "modernc.org/sqlite" +) + +func TestCapabilities(t *testing.T) { RegisterFailHandler(Fail); RunSpecs(t, "Database capabilities") } diff --git a/database/inspect/capabilities_test.go b/database/inspect/capabilities_test.go new file mode 100644 index 00000000..a28ee176 --- /dev/null +++ b/database/inspect/capabilities_test.go @@ -0,0 +1,21 @@ +package sqlinspect + +import ( + connection "github.com/flanksource/sqlglot-go/database/capabilities" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("SQL catalog capabilities", func() { + It("carries the live backend metadata on the catalog", func() { + capabilities := connection.BackendCapabilities{ + Backend: "postgres", Version: "17.4", Database: "warehouse", + Features: map[connection.BackendCapability]connection.BackendCapabilityStatus{ + connection.BackendCapabilityArrayFilters: {Supported: true}, + }, + } + catalog := buildCatalog(catalogInput{Driver: "postgres", Database: "warehouse", DefaultSchema: "public", Databases: []string{"warehouse"}, Schemas: []string{"public"}, Capabilities: capabilities}) + + Expect(catalog.Capabilities).To(Equal(capabilities)) + }) +}) diff --git a/database/inspect/catalog.go b/database/inspect/catalog.go new file mode 100644 index 00000000..56938b65 --- /dev/null +++ b/database/inspect/catalog.go @@ -0,0 +1,459 @@ +// Package sqlinspect provides reusable, read-only SQL catalog inspection. +package sqlinspect + +import ( + "context" + "database/sql" + "errors" + "fmt" + "sort" + "strings" + + connection "github.com/flanksource/sqlglot-go/database/capabilities" +) + +const ( + DefaultMaxRelations = 5000 + DefaultMaxColumns = 50000 + DefaultMaxMetadata = 50000 + DefaultMaxRoutines = 2000 + DefaultMaxDefinitionBytes = 4 * 1024 * 1024 +) + +type Limits struct { + MaxRelations int + MaxColumns int + MaxMetadata int + MaxRoutines int + MaxDefinitionBytes int +} + +func (l Limits) withDefaults() Limits { + if l.MaxRelations <= 0 { + l.MaxRelations = DefaultMaxRelations + } + if l.MaxColumns <= 0 { + l.MaxColumns = DefaultMaxColumns + } + if l.MaxMetadata <= 0 { + l.MaxMetadata = DefaultMaxMetadata + } + if l.MaxRoutines <= 0 { + l.MaxRoutines = DefaultMaxRoutines + } + if l.MaxDefinitionBytes <= 0 { + l.MaxDefinitionBytes = DefaultMaxDefinitionBytes + } + return l +} + +type Catalog struct { + Driver string `json:"driver"` + Database string `json:"database,omitempty"` + Capabilities connection.BackendCapabilities `json:"capabilities"` + Databases []string `json:"databases,omitempty"` + DefaultSchema string `json:"defaultSchema,omitempty"` + Schemas []Schema `json:"schemas"` + Truncated bool `json:"truncated,omitempty"` + TruncateReason string `json:"truncateReason,omitempty"` +} + +type Schema struct { + Name string `json:"name"` + Relations []Relation `json:"relations"` + Routines []Routine `json:"routines,omitempty"` +} + +type Relation struct { + Name string `json:"name"` + Type string `json:"type"` + ViewDef string `json:"viewDefinition,omitempty"` + Columns []Column `json:"columns"` + Indexes []Index `json:"indexes,omitempty"` + ForeignKeys []ForeignKey `json:"foreignKeys,omitempty"` + Triggers []Trigger `json:"triggers,omitempty"` +} + +type Column struct { + Name string `json:"name"` + DataType string `json:"dataType,omitempty"` + Ordinal int `json:"ordinal,omitempty"` + Nullable *bool `json:"nullable,omitempty"` + Default *string `json:"default,omitempty"` + Identity bool `json:"identity,omitempty"` + PrimaryKey bool `json:"primaryKey,omitempty"` + Unique bool `json:"unique,omitempty"` + MaxLength *int `json:"maxLength,omitempty"` + NumericPrecision *int `json:"numericPrecision,omitempty"` + NumericScale *int `json:"numericScale,omitempty"` + Comment *string `json:"comment,omitempty"` + EnumValues []string `json:"enumValues,omitempty"` +} + +type Index struct { + Name string `json:"name"` + Unique bool `json:"unique,omitempty"` + Primary bool `json:"primary,omitempty"` + Columns []string `json:"columns"` + Type string `json:"type,omitempty"` + Filter string `json:"filter,omitempty"` +} + +type ForeignKey struct { + Name string `json:"name"` + Columns []string `json:"columns"` + ReferencedSchema string `json:"referencedSchema,omitempty"` + ReferencedTable string `json:"referencedTable"` + ReferencedColumns []string `json:"referencedColumns"` +} + +type Trigger struct { + Name string `json:"name"` + Event string `json:"event,omitempty"` + Timing string `json:"timing,omitempty"` + Disabled bool `json:"disabled,omitempty"` + SQL string `json:"sql,omitempty"` +} + +type Routine struct { + ID string `json:"id"` + Name string `json:"name"` + Type string `json:"type"` + SQL string `json:"sql,omitempty"` + ReturnType string `json:"returnType,omitempty"` + Parameters []RoutineParameter `json:"parameters,omitempty"` +} + +type RoutineParameter struct { + Name string `json:"name"` + DataType string `json:"dataType,omitempty"` + Ordinal int `json:"ordinal,omitempty"` +} + +type columnRow struct { + schema, relation, relationType, column, dataType string + ordinal int +} + +// inspect reads only the current database. Querying schemas independently keeps +// empty schemas visible even when there are no relations to discover. +func Inspect(ctx context.Context, db *sql.DB, driver string, limits Limits) (_ Catalog, returnErr error) { + if db == nil { + return Catalog{}, fmt.Errorf("nil sql database") + } + driver = normalizeDriver(driver) + capabilities, err := connection.ProbeBackendCapabilities(ctx, db, driver) + if err != nil { + return Catalog{}, fmt.Errorf("inspect SQL capabilities: %w", err) + } + if driver == "postgres" || driver == "sqlserver" { + catalog, err := inspectRich(ctx, db, driver, limits) + if err != nil { + return Catalog{}, err + } + catalog.Capabilities = capabilities + return catalog, nil + } + identity, statement, err := inspectionQueries(driver) + if err != nil { + return Catalog{}, err + } + + var database, defaultSchema string + if err := db.QueryRowContext(ctx, identity).Scan(&database, &defaultSchema); err != nil { + return Catalog{}, fmt.Errorf("inspect sql identity: %w", err) + } + databases, err := ListDatabases(ctx, db, driver) + if err != nil { + return Catalog{}, err + } + schemaRows, err := db.QueryContext(ctx, schemaInspectionQuery(driver)) + if err != nil { + return Catalog{}, fmt.Errorf("inspect sql schemas: %w", err) + } + var schemas []string + for schemaRows.Next() { + var schema string + if err := schemaRows.Scan(&schema); err != nil { + return Catalog{}, errors.Join(fmt.Errorf("scan sql schema: %w", err), schemaRows.Close()) + } + schemas = append(schemas, schema) + } + if err := schemaRows.Err(); err != nil { + return Catalog{}, errors.Join(fmt.Errorf("iterate sql schemas: %w", err), schemaRows.Close()) + } + if err := schemaRows.Close(); err != nil { + return Catalog{}, fmt.Errorf("close sql schema rows: %w", err) + } + rows, err := db.QueryContext(ctx, statement) + if err != nil { + return Catalog{}, fmt.Errorf("inspect sql catalog: %w", err) + } + defer func() { returnErr = errors.Join(returnErr, rows.Close()) }() + + items := make([]columnRow, 0) + for rows.Next() { + var item columnRow + if err := rows.Scan(&item.schema, &item.relation, &item.relationType, &item.column, &item.dataType, &item.ordinal); err != nil { + return Catalog{}, fmt.Errorf("scan sql catalog: %w", err) + } + items = append(items, item) + } + if err := rows.Err(); err != nil { + return Catalog{}, fmt.Errorf("iterate sql catalog: %w", err) + } + catalog := buildCatalog(catalogInput{ + Driver: driver, Database: database, DefaultSchema: defaultSchema, + Databases: databases, Schemas: schemas, Rows: items, + Limits: limits, Capabilities: capabilities, + }) + if driver == "sqlite" { + if err := enrichSQLite(ctx, db, &catalog, limits); err != nil { + return Catalog{}, err + } + } + return catalog, nil +} + +// ListDatabases returns databases accessible to the current connection user. +func ListDatabases(ctx context.Context, db *sql.DB, driver string) (_ []string, returnErr error) { + if db == nil { + return nil, fmt.Errorf("nil sql database") + } + statement := databaseInspectionQuery(driver) + if statement == "" { + return nil, fmt.Errorf("unsupported sql inspection driver %q", driver) + } + rows, err := db.QueryContext(ctx, statement) + if err != nil { + return nil, fmt.Errorf("inspect sql databases: %w", err) + } + defer func() { returnErr = errors.Join(returnErr, rows.Close()) }() + var databases []string + for rows.Next() { + var database string + if err := rows.Scan(&database); err != nil { + return nil, fmt.Errorf("scan sql database: %w", err) + } + databases = append(databases, database) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterate sql databases: %w", err) + } + return databases, nil +} + +func normalizeDriver(driver string) string { + switch strings.ToLower(driver) { + case "postgres", "postgresql", "pgx": + return "postgres" + case "mysql": + return "mysql" + case "sql_server", "sqlserver", "mssql": + return "sqlserver" + case "clickhouse": + return "clickhouse" + case "sqlite", "sqlite3": + return "sqlite" + default: + return strings.ToLower(driver) + } +} + +func inspectionQueries(driver string) (identity, catalog string, err error) { + switch normalizeDriver(driver) { + case "postgres": + return `SELECT current_database(), current_schema()`, ` +SELECT c.table_schema, c.table_name, t.table_type, c.column_name, + COALESCE(c.udt_name, c.data_type), c.ordinal_position +FROM information_schema.columns c +JOIN information_schema.tables t + ON t.table_schema = c.table_schema AND t.table_name = c.table_name +WHERE c.table_schema NOT IN ('information_schema','pg_catalog') + AND c.table_schema NOT LIKE 'pg_toast%' +ORDER BY c.table_schema, c.table_name, c.ordinal_position`, nil + case "mysql": + return `SELECT DATABASE(), DATABASE()`, ` +SELECT c.table_schema, c.table_name, t.table_type, c.column_name, + c.column_type, c.ordinal_position +FROM information_schema.columns c +JOIN information_schema.tables t + ON t.table_schema = c.table_schema AND t.table_name = c.table_name +WHERE c.table_schema = DATABASE() +ORDER BY c.table_schema, c.table_name, c.ordinal_position`, nil + case "sqlserver": + return `SELECT DB_NAME(), SCHEMA_NAME()`, ` +SELECT c.TABLE_SCHEMA, c.TABLE_NAME, t.TABLE_TYPE, c.COLUMN_NAME, + c.DATA_TYPE, c.ORDINAL_POSITION +FROM INFORMATION_SCHEMA.COLUMNS c +JOIN INFORMATION_SCHEMA.TABLES t + ON t.TABLE_SCHEMA = c.TABLE_SCHEMA AND t.TABLE_NAME = c.TABLE_NAME +WHERE c.TABLE_SCHEMA NOT IN ('INFORMATION_SCHEMA','sys') +ORDER BY c.TABLE_SCHEMA, c.TABLE_NAME, c.ORDINAL_POSITION`, nil + case "clickhouse": + return `SELECT currentDatabase(), currentDatabase()`, ` +SELECT c.database, c.table, + if(t.engine IN ('View', 'MaterializedView', 'LiveView', 'WindowView'), 'VIEW', 'BASE TABLE'), + c.name, c.type, c.position +FROM system.columns c +JOIN system.tables t ON t.database = c.database AND t.name = c.table +WHERE c.database = currentDatabase() +ORDER BY c.database, c.table, c.position`, nil + case "sqlite": + return `SELECT 'snapshot', 'main'`, ` +SELECT 'main', m.name, + CASE WHEN m.type = 'view' THEN 'VIEW' ELSE 'BASE TABLE' END, + p.name, p.type, p.cid + 1 +FROM sqlite_master m +JOIN pragma_table_info(m.name) p +WHERE m.type IN ('table', 'view') AND m.name NOT LIKE 'sqlite_%' +ORDER BY m.name, p.cid`, nil + default: + return "", "", fmt.Errorf("unsupported sql inspection driver %q", driver) + } +} + +func schemaInspectionQuery(driver string) string { + switch normalizeDriver(driver) { + case "postgres": + return `SELECT schema_name +FROM information_schema.schemata +WHERE schema_name NOT IN ('pg_catalog','information_schema','pg_toast') + AND schema_name NOT LIKE 'pg_temp_%' + AND schema_name NOT LIKE 'pg_toast_temp_%' +ORDER BY schema_name` + case "mysql": + return `SELECT schema_name +FROM information_schema.schemata +WHERE schema_name = DATABASE() +ORDER BY schema_name` + case "sqlserver": + return `SELECT schema_name +FROM information_schema.schemata +WHERE schema_name NOT IN ('sys','INFORMATION_SCHEMA','guest') + AND schema_name NOT LIKE 'db_%' +ORDER BY schema_name` + case "clickhouse": + return `SELECT name FROM system.databases WHERE name = currentDatabase() ORDER BY name` + case "sqlite": + return `SELECT 'main'` + default: + return "" + } +} + +func databaseInspectionQuery(driver string) string { + switch normalizeDriver(driver) { + case "postgres": + return `SELECT datname +FROM pg_database +WHERE datallowconn AND NOT datistemplate AND has_database_privilege(datname, 'CONNECT') +ORDER BY datname` + case "mysql": + return `SELECT schema_name +FROM information_schema.schemata +WHERE schema_name NOT IN ('information_schema','mysql','performance_schema','sys') +ORDER BY schema_name` + case "sqlserver": + return `SELECT name FROM sys.databases WHERE state = 0 AND HAS_DBACCESS(name) = 1 ORDER BY name` + case "clickhouse": + return `SELECT name +FROM system.databases +WHERE name NOT IN ('system','information_schema','INFORMATION_SCHEMA') +ORDER BY name` + case "sqlite": + return `SELECT 'snapshot'` + default: + return "" + } +} + +type catalogInput struct { + Driver, Database, DefaultSchema string + Databases, Schemas []string + Rows []columnRow + Limits Limits + Capabilities connection.BackendCapabilities +} + +func buildCatalog(in catalogInput) Catalog { + database, defaultSchema, databases, schemas, rows := in.Database, in.DefaultSchema, in.Databases, in.Schemas, in.Rows + limits := in.Limits.withDefaults() + type relationKey struct{ schema, relation string } + relations := make(map[relationKey]*Relation) + schemaNames := make(map[string]struct{}) + for _, schema := range schemas { + if schema != "" { + schemaNames[schema] = struct{}{} + } + } + if defaultSchema != "" { + schemaNames[defaultSchema] = struct{}{} + } + relationCount, columnCount := 0, 0 + truncated, reason := false, "" + + for _, row := range rows { + key := relationKey{row.schema, row.relation} + relation := relations[key] + if relation == nil { + if relationCount >= limits.MaxRelations { + truncated, reason = true, fmt.Sprintf("relation limit %d reached", limits.MaxRelations) + continue + } + relation = &Relation{Name: row.relation, Type: normalizeRelationType(row.relationType), Columns: []Column{}} + relations[key] = relation + schemaNames[row.schema] = struct{}{} + relationCount++ + } + if columnCount >= limits.MaxColumns { + truncated, reason = true, fmt.Sprintf("column limit %d reached", limits.MaxColumns) + continue + } + relation.Columns = append(relation.Columns, Column{Name: row.column, DataType: row.dataType, Ordinal: row.ordinal}) + columnCount++ + } + + names := make([]string, 0, len(schemaNames)) + for name := range schemaNames { + names = append(names, name) + } + sort.Strings(names) + databaseNames := make(map[string]struct{}, len(databases)+1) + for _, name := range databases { + if name != "" { + databaseNames[name] = struct{}{} + } + } + if database != "" { + databaseNames[database] = struct{}{} + } + orderedDatabases := make([]string, 0, len(databaseNames)) + for name := range databaseNames { + orderedDatabases = append(orderedDatabases, name) + } + sort.Strings(orderedDatabases) + catalog := Catalog{Driver: normalizeDriver(in.Driver), Database: database, Capabilities: in.Capabilities, Databases: orderedDatabases, DefaultSchema: defaultSchema, Schemas: []Schema{}, Truncated: truncated, TruncateReason: reason} + for _, schemaName := range names { + relationNames := make([]string, 0) + for key := range relations { + if key.schema == schemaName { + relationNames = append(relationNames, key.relation) + } + } + sort.Strings(relationNames) + schema := Schema{Name: schemaName, Relations: make([]Relation, 0, len(relationNames))} + for _, relationName := range relationNames { + schema.Relations = append(schema.Relations, *relations[relationKey{schemaName, relationName}]) + } + catalog.Schemas = append(catalog.Schemas, schema) + } + return catalog +} + +func normalizeRelationType(value string) string { + if strings.Contains(strings.ToLower(value), "view") { + return "view" + } + return "table" +} diff --git a/database/inspect/catalog/sqlserver/catalog.go b/database/inspect/catalog/sqlserver/catalog.go new file mode 100644 index 00000000..2b88e4ad --- /dev/null +++ b/database/inspect/catalog/sqlserver/catalog.go @@ -0,0 +1,148 @@ +// Package mssqlcatalog reads SQL Server catalog views into plain row structs. +// +// It is the single owner of the catalog SQL shared by the schema inspector and +// the Atlas SQL Server driver. Every query is schema-scoped, runs in one round +// trip, pushes name filters into SQL as ordinary binds, and returns rows in a +// deterministic owner-then-natural order. +// +// Queries must run on databases at compatibility level 100 (OIPA): no +// STRING_AGG, no OPENJSON. Ordered string aggregation uses FOR XML PATH, and +// name filters are plain `IN (@p1, @p2, ...)` lists. Each query text starts +// with its outer SELECT so a caller may prefix it with a TOP bound. +package mssqlcatalog + +import ( + "context" + "database/sql" + "fmt" + "strings" +) + +// Rows is the subset of *sql.Rows the catalog reads. +type Rows interface { + Next() bool + Scan(dest ...any) error + Err() error + Close() error +} + +// Queryer runs a catalog query. Callers that bound or instrument queries +// implement it; FromDB adapts database/sql handles. +type Queryer interface { + QueryContext(ctx context.Context, query string, args ...any) (Rows, error) +} + +// FromDB adapts db to Queryer. +func FromDB(db SQLQueryer) Queryer { return sqlDB{db} } + +type SQLQueryer interface { + QueryContext(context.Context, string, ...any) (*sql.Rows, error) +} + +type sqlDB struct{ db SQLQueryer } + +func (d sqlDB) QueryContext(ctx context.Context, query string, args ...any) (Rows, error) { + rows, err := d.db.QueryContext(ctx, query, args...) + if err != nil { + return nil, err + } + return rows, nil +} + +// Scope selects the catalog rows a query returns. +type Scope struct { + // Schema is the owning schema; required. + Schema string + // Names restricts rows to these owning objects (tables for table-level + // queries, routines for Routines and Parameters). Empty means all. + Names []string + // DefinitionLimit truncates object definitions server-side to this many + // characters. Zero returns definitions in full. + DefinitionLimit int +} + +func (s Scope) validate() error { + if strings.TrimSpace(s.Schema) == "" { + return fmt.Errorf("SQL Server catalog query: schema is required") + } + if s.DefinitionLimit < 0 { + return fmt.Errorf("SQL Server catalog query: definition limit must not be negative, got %d", s.DefinitionLimit) + } + return validateNames(s.Names) +} + +func validateNames(names []string) error { + for i, name := range names { + if strings.TrimSpace(name) == "" { + return fmt.Errorf("SQL Server catalog query: name filter entry %d is empty", i) + } + } + return nil +} + +// binds numbers SQL Server @pN placeholders in the order values are added. +type binds struct{ args []any } + +func (b *binds) add(value any) string { + b.args = append(b.args, value) + return fmt.Sprintf("@p%d", len(b.args)) +} + +// in renders ` AND IN (@pX, ...)`, or nothing for an empty filter. +func (b *binds) in(column string, names []string) string { + if len(names) == 0 { + return "" + } + placeholders := make([]string, len(names)) + for i, name := range names { + placeholders[i] = b.add(name) + } + return fmt.Sprintf(" AND %s IN (%s)", column, strings.Join(placeholders, ", ")) +} + +// definition wraps expr in LEFT(expr, @pN) when limit is set. +func (b *binds) definition(expr string, limit int) string { + if limit == 0 { + return expr + } + return fmt.Sprintf("LEFT(%s, %s)", expr, b.add(limit)) +} + +type catalogQuery struct { + Text string + Args []any +} + +func collect[T any](ctx context.Context, q Queryer, query catalogQuery, scan func(Rows) (T, error)) (out []T, err error) { + rows, err := q.QueryContext(ctx, query.Text, query.Args...) + if err != nil { + return nil, err + } + defer func() { + if closeErr := rows.Close(); err == nil { + err = closeErr + } + }() + for rows.Next() { + row, err := scan(rows) + if err != nil { + return nil, err + } + out = append(out, row) + } + return out, rows.Err() +} + +func nullableInt(v sql.NullInt64) *int { + if !v.Valid { + return nil + } + return new(int(v.Int64)) +} + +func nullableString(v sql.NullString) *string { + if !v.Valid { + return nil + } + return new(v.String) +} diff --git a/database/inspect/catalog/sqlserver/catalog_test.go b/database/inspect/catalog/sqlserver/catalog_test.go new file mode 100644 index 00000000..d73eca8a --- /dev/null +++ b/database/inspect/catalog/sqlserver/catalog_test.go @@ -0,0 +1,116 @@ +package mssqlcatalog_test + +import ( + "database/sql/driver" + + "github.com/DATA-DOG/go-sqlmock" + + "github.com/flanksource/sqlglot-go/database/inspect/catalog/sqlserver" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("SQL Server catalog queries", func() { + const schema = "dbo" + var ( + mock sqlmock.Sqlmock + queryer mssqlcatalog.Queryer + ) + + BeforeEach(func() { + db, m, err := sqlmock.New() + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { + Expect(m.ExpectationsWereMet()).To(Succeed()) + m.ExpectClose() + Expect(db.Close()).To(Succeed()) + }) + mock, queryer = m, mssqlcatalog.FromDB(db) + }) + + It("pushes the table-name filter into SQL as binds and maps default constraints and widths", func() { + const policyTable, clientTable, defaultName, defaultDef = "AsPolicy", "AsClient", "DF_AsPolicy_Status", "('08')" + mock.ExpectQuery(`WHERE c\.TABLE_SCHEMA = @p1 AND c\.TABLE_NAME IN \(@p2, @p3\) ORDER BY c\.TABLE_NAME, c\.ORDINAL_POSITION`). + WithArgs(schema, policyTable, clientTable). + WillReturnRows(sqlmock.NewRows([]string{ + "table", "column", "data_type", "nullable", "default", "position", "char_max", "num_precision", "num_scale", + "pk", "uq", "identity", "max_length", "precision", "scale", "default_name", + "seed", "increment", "computed", "user_defined", + }). + AddRow(policyTable, "Status", "nvarchar", 0, defaultDef, 2, 10, nil, nil, 0, 0, false, 20, 0, 0, defaultName, nil, nil, false, false). + AddRow(policyTable, "PolicyID", "int", 0, nil, 1, nil, 10, 0, 1, 0, true, 4, 10, 0, nil, 1000, 5, false, false)) + + columns, err := mssqlcatalog.Columns(GinkgoT().Context(), queryer, mssqlcatalog.Scope{ + Schema: schema, Names: []string{policyTable, clientTable}, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(columns).To(Equal([]mssqlcatalog.Column{ + { + Table: policyTable, Name: "Status", DataType: "nvarchar", Position: 2, + Default: new(defaultDef), DefaultConstraint: new(defaultName), + CharMaxLength: new(10), MaxLength: new(20), Precision: new(0), Scale: new(0), + }, + { + Table: policyTable, Name: "PolicyID", DataType: "int", Position: 1, PrimaryKey: true, Identity: true, + NumericPrecision: new(10), NumericScale: new(0), MaxLength: new(4), Precision: new(10), Scale: new(0), + IdentitySeed: new(int64(1000)), IdentityIncrement: new(int64(5)), + }, + })) + }) + + It("reports partitioned tables and table triggers in the scope", func() { + const ordersTable = "acme_orders" + mock.ExpectQuery(`sys\.partition_schemes.*t\.name IN \(@p2\).*sys\.triggers.*t\.name IN \(@p4\)`). + WithArgs(schema, ordersTable, schema, ordersTable). + WillReturnRows(sqlmock.NewRows([]string{"table", "feature"}). + AddRow(ordersTable, "partitioned"). + AddRow(ordersTable, "trigger tr_acme_orders_audit")) + + features, err := mssqlcatalog.UnmanagedFeatures(GinkgoT().Context(), queryer, mssqlcatalog.Scope{ + Schema: schema, Names: []string{ordersTable}, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(features).To(Equal([]mssqlcatalog.TableFeature{ + {Table: ordersTable, Feature: "partitioned"}, + {Table: ordersTable, Feature: "trigger tr_acme_orders_audit"}, + })) + }) + + DescribeTable("reads routine definitions from sys.sql_modules, truncated only when a limit is set", + func(limit int, definitionSQL string, args []driver.Value) { + mock.ExpectQuery(definitionSQL + `.*WHERE s\.name = @p1 AND o\.type IN \('P', 'FN', 'IF', 'TF'\) ORDER BY o\.name`). + WithArgs(args...). + WillReturnRows(sqlmock.NewRows([]string{"schema", "name", "type", "definition", "return_type"}). + AddRow(schema, "fn_age", "FN", "CREATE FUNCTION fn_age()", "int"). + AddRow(schema, "usp_purge", "P", nil, nil)) + + routines, err := mssqlcatalog.Routines(GinkgoT().Context(), queryer, mssqlcatalog.Scope{Schema: schema, DefinitionLimit: limit}) + Expect(err).ToNot(HaveOccurred()) + Expect(routines).To(Equal([]mssqlcatalog.Routine{ + {Schema: schema, Name: "fn_age", Type: mssqlcatalog.ScalarFunction, Definition: new("CREATE FUNCTION fn_age()"), ReturnType: new("int")}, + {Schema: schema, Name: "usp_purge", Type: mssqlcatalog.Procedure}, + })) + }, + Entry("full definition", 0, `RTRIM\(o\.type\), m\.definition,`, []driver.Value{schema}), + Entry("bounded definition", 4096, `RTRIM\(o\.type\), LEFT\(m\.definition, @p2\),`, []driver.Value{schema, 4096}), + ) + + DescribeTable("rejects an invalid scope before querying", + func(scope mssqlcatalog.Scope, message string) { + _, err := mssqlcatalog.Columns(GinkgoT().Context(), queryer, scope) + Expect(err).To(MatchError(ContainSubstring(message))) + }, + Entry("missing schema", mssqlcatalog.Scope{}, "schema is required"), + Entry("blank name in the filter", mssqlcatalog.Scope{Schema: schema, Names: []string{"AsPolicy", " "}}, "name filter entry 1 is empty"), + Entry("negative definition limit", mssqlcatalog.Scope{Schema: schema, DefinitionLimit: -1}, "definition limit must not be negative"), + ) + + It("fails loudly on a routine type outside the catalogued set", func() { + mock.ExpectQuery(`FROM sys\.objects o`). + WithArgs(schema). + WillReturnRows(sqlmock.NewRows([]string{"schema", "name", "type", "definition", "return_type"}). + AddRow(schema, "clr_proc", "PC", nil, nil)) + _, err := mssqlcatalog.Routines(GinkgoT().Context(), queryer, mssqlcatalog.Scope{Schema: schema}) + Expect(err).To(MatchError(ContainSubstring(`unexpected routine type "PC" for dbo.clr_proc`))) + }) +}) diff --git a/database/inspect/catalog/sqlserver/constraints.go b/database/inspect/catalog/sqlserver/constraints.go new file mode 100644 index 00000000..58414e04 --- /dev/null +++ b/database/inspect/catalog/sqlserver/constraints.go @@ -0,0 +1,182 @@ +package mssqlcatalog + +import ( + "context" + "database/sql" + "fmt" + "strings" +) + +// Index is one named index on a table. KeyColumns is the ordered key (included +// columns excluded); IndexColumns returns the full per-column detail. TypeDesc +// is sys.indexes.type_desc (CLUSTERED, NONCLUSTERED, ...); Filter is the +// filtered-index predicate, empty when unfiltered. +type Index struct { + Table string + Name string + Unique bool + Primary bool + UniqueConstraint bool + KeyColumns []string + TypeDesc string + Filter string +} + +// Indexes lists every named index on every table in the scope, ordered by +// table then index name. Views are absent: the join is to sys.tables. +// +// Key columns are aggregated with FOR XML PATH, not STRING_AGG ... WITHIN +// GROUP, which compatibility levels below 140 reject. The `, TYPE).value(...)` +// projection keeps XML metacharacters intact and yields nvarchar(max). +func Indexes(ctx context.Context, q Queryer, scope Scope) ([]Index, error) { + if err := scope.validate(); err != nil { + return nil, err + } + var b binds + query := fmt.Sprintf(`SELECT + t.name AS table_name, + i.name AS index_name, + i.is_unique, + i.is_primary_key, + STUFF(( + SELECT ',' + c.name + FROM sys.index_columns ic2 + INNER JOIN sys.columns c ON ic2.object_id = c.object_id AND ic2.column_id = c.column_id + WHERE ic2.object_id = i.object_id AND ic2.index_id = i.index_id AND ic2.key_ordinal > 0 + ORDER BY ic2.key_ordinal + FOR XML PATH(''), TYPE + ).value('.', 'nvarchar(max)'), 1, 1, '') AS columns, + i.type_desc, + COALESCE(i.filter_definition, ''), + i.is_unique_constraint + FROM sys.indexes i + INNER JOIN sys.tables t ON i.object_id = t.object_id + INNER JOIN sys.schemas s ON t.schema_id = s.schema_id + WHERE s.name = %s AND i.name IS NOT NULL%s + ORDER BY t.name, i.name`, b.add(scope.Schema), b.in("t.name", scope.Names)) + return collect(ctx, q, catalogQuery{Text: query, Args: b.args}, func(rows Rows) (Index, error) { + var idx Index + var columns sql.NullString + // Bit columns decode to Go bools in go-mssqldb; scan them as bools. + err := rows.Scan(&idx.Table, &idx.Name, &idx.Unique, &idx.Primary, &columns, &idx.TypeDesc, &idx.Filter, &idx.UniqueConstraint) + if columns.Valid && columns.String != "" { + idx.KeyColumns = strings.Split(columns.String, ",") + } + return idx, err + }) +} + +// IndexColumn is one column of a named index. KeyOrdinal is 0 for included +// columns. +type IndexColumn struct { + Table string + Index string + Column string + KeyOrdinal int + Included bool + Descending bool +} + +// IndexColumns lists the key and included columns of every named index in the +// scope, ordered by table, index, key columns by key ordinal, then included +// columns by index column id. +func IndexColumns(ctx context.Context, q Queryer, scope Scope) ([]IndexColumn, error) { + if err := scope.validate(); err != nil { + return nil, err + } + var b binds + query := fmt.Sprintf(`SELECT + t.name, i.name, c.name, ic.key_ordinal, ic.is_included_column, ic.is_descending_key + FROM sys.index_columns ic + INNER JOIN sys.indexes i ON i.object_id = ic.object_id AND i.index_id = ic.index_id + INNER JOIN sys.columns c ON c.object_id = ic.object_id AND c.column_id = ic.column_id + INNER JOIN sys.tables t ON t.object_id = i.object_id + INNER JOIN sys.schemas s ON s.schema_id = t.schema_id + WHERE s.name = %s AND i.name IS NOT NULL%s + ORDER BY t.name, i.name, ic.is_included_column, ic.key_ordinal, ic.index_column_id`, + b.add(scope.Schema), b.in("t.name", scope.Names)) + return collect(ctx, q, catalogQuery{Text: query, Args: b.args}, func(rows Rows) (IndexColumn, error) { + var col IndexColumn + return col, rows.Scan(&col.Table, &col.Index, &col.Column, &col.KeyOrdinal, &col.Included, &col.Descending) + }) +} + +// ForeignKey is one participating column of a foreign key. OnDelete and +// OnUpdate are the referential action descriptions (NO_ACTION, CASCADE, ...). +type ForeignKey struct { + Table string + Name string + Column string + RefSchema string + RefTable string + RefColumn string + OnDelete string + OnUpdate string +} + +// ForeignKeys lists every foreign key column in the scope. Rows are ordered by +// table, constraint, then constraint_column_id; that last key is load-bearing, +// because callers pair local and referenced columns positionally. +func ForeignKeys(ctx context.Context, q Queryer, scope Scope) ([]ForeignKey, error) { + if err := scope.validate(); err != nil { + return nil, err + } + var b binds + query := fmt.Sprintf(`SELECT + t1.name AS table_name, + fk.name AS constraint_name, + c1.name AS column_name, + s2.name AS ref_schema_name, + t2.name AS ref_table_name, + c2.name AS ref_column_name, + fk.delete_referential_action_desc, + fk.update_referential_action_desc + FROM sys.foreign_keys fk + INNER JOIN sys.foreign_key_columns fkc ON fk.object_id = fkc.constraint_object_id + INNER JOIN sys.tables t1 ON fk.parent_object_id = t1.object_id + INNER JOIN sys.schemas s1 ON t1.schema_id = s1.schema_id + INNER JOIN sys.columns c1 ON fkc.parent_object_id = c1.object_id AND fkc.parent_column_id = c1.column_id + INNER JOIN sys.tables t2 ON fk.referenced_object_id = t2.object_id + INNER JOIN sys.schemas s2 ON t2.schema_id = s2.schema_id + INNER JOIN sys.columns c2 ON fkc.referenced_object_id = c2.object_id AND fkc.referenced_column_id = c2.column_id + WHERE s1.name = %s%s + ORDER BY t1.name, fk.name, fkc.constraint_column_id`, b.add(scope.Schema), b.in("t1.name", scope.Names)) + return collect(ctx, q, catalogQuery{Text: query, Args: b.args}, func(rows Rows) (ForeignKey, error) { + var fk ForeignKey + return fk, rows.Scan(&fk.Table, &fk.Name, &fk.Column, &fk.RefSchema, &fk.RefTable, &fk.RefColumn, &fk.OnDelete, &fk.OnUpdate) + }) +} + +// CheckConstraint is one CHECK constraint. Column is set only for a +// column-level constraint. +type CheckConstraint struct { + Table string + Name string + Definition string + Column *string + Disabled bool +} + +// CheckConstraints lists every check constraint in the scope ordered by table +// then constraint name. +func CheckConstraints(ctx context.Context, q Queryer, scope Scope) ([]CheckConstraint, error) { + if err := scope.validate(); err != nil { + return nil, err + } + var b binds + query := fmt.Sprintf(`SELECT + t.name, cc.name, cc.definition, c.name, cc.is_disabled + FROM sys.check_constraints cc + INNER JOIN sys.tables t ON t.object_id = cc.parent_object_id + INNER JOIN sys.schemas s ON s.schema_id = t.schema_id + LEFT JOIN sys.columns c ON c.object_id = cc.parent_object_id AND c.column_id = cc.parent_column_id + WHERE s.name = %s%s + ORDER BY t.name, cc.name`, b.add(scope.Schema), b.in("t.name", scope.Names)) + return collect(ctx, q, catalogQuery{Text: query, Args: b.args}, func(rows Rows) (CheckConstraint, error) { + var check CheckConstraint + var column sql.NullString + err := rows.Scan(&check.Table, &check.Name, &check.Definition, &column, &check.Disabled) + check.Column = nullableString(column) + return check, err + }) +} diff --git a/database/inspect/catalog/sqlserver/features.go b/database/inspect/catalog/sqlserver/features.go new file mode 100644 index 00000000..118c1213 --- /dev/null +++ b/database/inspect/catalog/sqlserver/features.go @@ -0,0 +1,42 @@ +package mssqlcatalog + +import ( + "context" + "fmt" +) + +// TableFeature is a table property DDL generation does not manage: Feature is +// "partitioned", or "trigger " for each trigger on the table. +type TableFeature struct { + Table string + Feature string +} + +// UnmanagedFeatures lists the partitioned tables and table triggers in the +// scope, ordered by table then feature. +func UnmanagedFeatures(ctx context.Context, q Queryer, scope Scope) ([]TableFeature, error) { + if err := scope.validate(); err != nil { + return nil, err + } + var b binds + query := fmt.Sprintf(`SELECT table_name, feature FROM ( + SELECT t.name AS table_name, CAST('partitioned' AS nvarchar(300)) AS feature + FROM sys.tables t + INNER JOIN sys.schemas s ON s.schema_id = t.schema_id + INNER JOIN sys.indexes i ON i.object_id = t.object_id AND i.index_id IN (0, 1) + INNER JOIN sys.partition_schemes ps ON ps.data_space_id = i.data_space_id + WHERE s.name = %s%s + UNION ALL + SELECT t.name, CAST('trigger ' + tr.name AS nvarchar(300)) + FROM sys.triggers tr + INNER JOIN sys.tables t ON t.object_id = tr.parent_id + INNER JOIN sys.schemas s ON s.schema_id = t.schema_id + WHERE s.name = %s%s + ) features + ORDER BY table_name, feature`, + b.add(scope.Schema), b.in("t.name", scope.Names), b.add(scope.Schema), b.in("t.name", scope.Names)) + return collect(ctx, q, catalogQuery{Text: query, Args: b.args}, func(rows Rows) (TableFeature, error) { + var f TableFeature + return f, rows.Scan(&f.Table, &f.Feature) + }) +} diff --git a/database/inspect/catalog/sqlserver/migration_test.go b/database/inspect/catalog/sqlserver/migration_test.go new file mode 100644 index 00000000..49b9a1bb --- /dev/null +++ b/database/inspect/catalog/sqlserver/migration_test.go @@ -0,0 +1,31 @@ +package mssqlcatalog_test + +import ( + "github.com/DATA-DOG/go-sqlmock" + "github.com/flanksource/sqlglot-go/database/inspect/catalog/sqlserver" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Migration catalogue row integrity", func() { + It("returns schema, index-column and foreign-key values after scanning", func() { + db, mock, err := sqlmock.New() + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { mock.ExpectClose(); Expect(db.Close()).To(Succeed()) }) + q := mssqlcatalog.FromDB(db) + ctx, scope := GinkgoT().Context(), mssqlcatalog.Scope{Schema: "dbo"} + mock.ExpectQuery("INFORMATION_SCHEMA.SCHEMATA").WillReturnRows(sqlmock.NewRows([]string{"schema"}).AddRow("dbo")) + schemas, err := mssqlcatalog.Schemas(ctx, q, nil) + Expect(err).NotTo(HaveOccurred()) + Expect(schemas).To(Equal([]string{"dbo"})) + mock.ExpectQuery("FROM sys.index_columns").WillReturnRows(sqlmock.NewRows([]string{"table", "index", "column", "ordinal", "included", "descending"}).AddRow("orders", "IX_total", "total", 1, false, true)) + parts, err := mssqlcatalog.IndexColumns(ctx, q, scope) + Expect(err).NotTo(HaveOccurred()) + Expect(parts).To(Equal([]mssqlcatalog.IndexColumn{{Table: "orders", Index: "IX_total", Column: "total", KeyOrdinal: 1, Descending: true}})) + mock.ExpectQuery("FROM sys.foreign_keys").WillReturnRows(sqlmock.NewRows([]string{"table", "name", "column", "refSchema", "refTable", "refColumn", "delete", "update"}).AddRow("orders", "FK_tenant", "tenant_id", "tenant", "accounts", "id", "CASCADE", "NO_ACTION")) + keys, err := mssqlcatalog.ForeignKeys(ctx, q, scope) + Expect(err).NotTo(HaveOccurred()) + Expect(keys).To(Equal([]mssqlcatalog.ForeignKey{{Table: "orders", Name: "FK_tenant", Column: "tenant_id", RefSchema: "tenant", RefTable: "accounts", RefColumn: "id", OnDelete: "CASCADE", OnUpdate: "NO_ACTION"}})) + Expect(mock.ExpectationsWereMet()).To(Succeed()) + }) +}) diff --git a/database/inspect/catalog/sqlserver/routines.go b/database/inspect/catalog/sqlserver/routines.go new file mode 100644 index 00000000..93644355 --- /dev/null +++ b/database/inspect/catalog/sqlserver/routines.go @@ -0,0 +1,118 @@ +package mssqlcatalog + +import ( + "context" + "database/sql" + "fmt" +) + +// RoutineType is the sys.objects type code of a routine. +type RoutineType string + +const ( + Procedure RoutineType = "P" + ScalarFunction RoutineType = "FN" + InlineTableFunction RoutineType = "IF" + TableFunction RoutineType = "TF" +) + +// IsFunction reports whether t is one of the function types. +func (t RoutineType) IsFunction() bool { + return t == ScalarFunction || t == InlineTableFunction || t == TableFunction +} + +// Routine is one stored procedure or T-SQL function. Definition is the +// sys.sql_modules text (nil when withheld or encrypted). ReturnType is the +// scalar return type for FN, "TABLE" for IF/TF, and nil for procedures. +type Routine struct { + Schema string + Name string + Type RoutineType + Definition *string + ReturnType *string +} + +// Routines lists the procedures and T-SQL functions in the scope ordered by +// name. +func Routines(ctx context.Context, q Queryer, scope Scope) ([]Routine, error) { + if err := scope.validate(); err != nil { + return nil, err + } + var b binds + schema := b.add(scope.Schema) + definition := b.definition("m.definition", scope.DefinitionLimit) + query := fmt.Sprintf(`SELECT + s.name, o.name, RTRIM(o.type), %s, + CASE + WHEN o.type = 'FN' THEN ( + SELECT TOP 1 t.name + FROM sys.parameters p + INNER JOIN sys.types t ON p.user_type_id = t.user_type_id + WHERE p.object_id = o.object_id AND p.parameter_id = 0 + ) + WHEN o.type IN ('IF', 'TF') THEN 'TABLE' + ELSE NULL + END + FROM sys.objects o + INNER JOIN sys.schemas s ON o.schema_id = s.schema_id + LEFT JOIN sys.sql_modules m ON m.object_id = o.object_id + WHERE s.name = %s AND o.type IN ('P', 'FN', 'IF', 'TF')%s + ORDER BY o.name`, definition, schema, b.in("o.name", scope.Names)) + return collect(ctx, q, catalogQuery{Text: query, Args: b.args}, scanRoutine) +} + +func scanRoutine(rows Rows) (Routine, error) { + var r Routine + var definition, returnType sql.NullString + if err := rows.Scan(&r.Schema, &r.Name, &r.Type, &definition, &returnType); err != nil { + return r, err + } + if r.Type != Procedure && !r.Type.IsFunction() { + return r, fmt.Errorf("unexpected routine type %q for %s.%s", r.Type, r.Schema, r.Name) + } + r.Definition, r.ReturnType = nullableString(definition), nullableString(returnType) + return r, nil +} + +// Parameter is one declared routine parameter. A scalar function's return +// value (parameter_id 0) is excluded; Routine.ReturnType carries it. +type Parameter struct { + Routine string + Name string + DataType string + Position int + MaxLength int + Precision int + Scale int + Output bool +} + +// Parameters lists the parameters of every routine in the scope ordered by +// routine name then parameter id. +func Parameters(ctx context.Context, q Queryer, scope Scope) ([]Parameter, error) { + if err := scope.validate(); err != nil { + return nil, err + } + var b binds + query := fmt.Sprintf(`SELECT + o.name AS proc_name, + p.name AS param_name, + t.name AS data_type, + p.parameter_id AS position, + p.max_length, + p.precision, + p.scale, + p.is_output + FROM sys.parameters p + INNER JOIN sys.types t ON p.user_type_id = t.user_type_id + INNER JOIN sys.objects o ON p.object_id = o.object_id + INNER JOIN sys.schemas s ON o.schema_id = s.schema_id + WHERE s.name = %s + AND o.type IN ('P', 'FN', 'IF', 'TF') + AND p.parameter_id > 0%s + ORDER BY o.name, p.parameter_id`, b.add(scope.Schema), b.in("o.name", scope.Names)) + return collect(ctx, q, catalogQuery{Text: query, Args: b.args}, func(rows Rows) (Parameter, error) { + var p Parameter + return p, rows.Scan(&p.Routine, &p.Name, &p.DataType, &p.Position, &p.MaxLength, &p.Precision, &p.Scale, &p.Output) + }) +} diff --git a/database/inspect/catalog/sqlserver/suite_test.go b/database/inspect/catalog/sqlserver/suite_test.go new file mode 100644 index 00000000..62be4f37 --- /dev/null +++ b/database/inspect/catalog/sqlserver/suite_test.go @@ -0,0 +1,13 @@ +package mssqlcatalog_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestMSSQLCatalog(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "SQL Server Catalog Suite") +} diff --git a/database/inspect/catalog/sqlserver/tables.go b/database/inspect/catalog/sqlserver/tables.go new file mode 100644 index 00000000..e1cb946d --- /dev/null +++ b/database/inspect/catalog/sqlserver/tables.go @@ -0,0 +1,194 @@ +package mssqlcatalog + +import ( + "context" + "database/sql" + "fmt" +) + +// Schemas lists non-system schemas, optionally restricted to names. +func Schemas(ctx context.Context, q Queryer, names []string) ([]string, error) { + if err := validateNames(names); err != nil { + return nil, err + } + var b binds + query := fmt.Sprintf(`SELECT SCHEMA_NAME + FROM INFORMATION_SCHEMA.SCHEMATA + WHERE SCHEMA_NAME NOT IN ('sys', 'INFORMATION_SCHEMA', 'guest') + AND SCHEMA_NAME NOT LIKE 'db[_]%%'%s + ORDER BY SCHEMA_NAME`, b.in("SCHEMA_NAME", names)) + return collect(ctx, q, catalogQuery{Text: query, Args: b.args}, func(rows Rows) (string, error) { + var schema string + return schema, rows.Scan(&schema) + }) +} + +// TableType selects base tables, views, or (for any other value) both, +// matching the inspector's "table" / "view" vocabulary. +type TableType string + +const ( + BaseTables TableType = "table" + Views TableType = "view" +) + +// Table is one INFORMATION_SCHEMA.TABLES row. Type is the catalog's TABLE_TYPE +// ("BASE TABLE" or "VIEW"); Definition is the view body, empty for tables and +// nil when the catalog withholds it. +type Table struct { + Schema string + Name string + Type string + Definition *string +} + +// Tables lists the tables and/or views of a schema ordered by name. +func Tables(ctx context.Context, q Queryer, scope Scope, tableType TableType) ([]Table, error) { + if err := scope.validate(); err != nil { + return nil, err + } + var b binds + schema, kind := b.add(scope.Schema), b.add(string(tableType)) + query := fmt.Sprintf(`SELECT + t.TABLE_SCHEMA, + t.TABLE_NAME, + t.TABLE_TYPE, + CASE WHEN t.TABLE_TYPE = 'VIEW' THEN %s ELSE '' END AS VIEW_DEF + FROM INFORMATION_SCHEMA.TABLES t + JOIN sys.schemas s ON s.name = t.TABLE_SCHEMA + LEFT JOIN sys.objects o + ON o.schema_id = s.schema_id AND o.name = t.TABLE_NAME + WHERE t.TABLE_SCHEMA = %s + AND t.TABLE_TYPE = CASE %s + WHEN 'table' THEN 'BASE TABLE' + WHEN 'view' THEN 'VIEW' + ELSE t.TABLE_TYPE + END%s + ORDER BY t.TABLE_NAME`, + b.definition("OBJECT_DEFINITION(o.object_id)", scope.DefinitionLimit), schema, kind, b.in("t.TABLE_NAME", scope.Names)) + return collect(ctx, q, catalogQuery{Text: query, Args: b.args}, func(rows Rows) (Table, error) { + var t Table + var definition sql.NullString + err := rows.Scan(&t.Schema, &t.Name, &t.Type, &definition) + t.Definition = nullableString(definition) + return t, err + }) +} + +// Column is one table or view column. +// +// CharMaxLength, NumericPrecision and NumericScale come from INFORMATION_SCHEMA +// (character widths, -1 for max; nil for types without one). MaxLength, +// Precision and Scale are the raw sys.columns values (bytes, so nvarchar is +// doubled; -1 for max) that DDL generation needs; they are nil only when the +// sys.columns join misses. Default is the default expression and +// DefaultConstraint the name of the constraint holding it. IdentitySeed and +// IdentityIncrement are set for identity columns. Computed and +// UserDefinedType flag columns DDL generation cannot reproduce. +type Column struct { + Table string + Name string + DataType string + Nullable bool + Default *string + DefaultConstraint *string + Position int + CharMaxLength *int + NumericPrecision *int + NumericScale *int + MaxLength *int + Precision *int + Scale *int + PrimaryKey bool + Unique bool + Identity bool + IdentitySeed *int64 + IdentityIncrement *int64 + Computed bool + UserDefinedType bool +} + +// Columns lists every column of every table and view in the scope, ordered by +// table then ordinal position. sys.columns is joined only by name for +// identity, raw widths and the default constraint, which avoids a per-row +// COLUMNPROPERTY(OBJECT_ID(...)) call at schema scope. +func Columns(ctx context.Context, q Queryer, scope Scope) ([]Column, error) { + if err := scope.validate(); err != nil { + return nil, err + } + var b binds + schema := b.add(scope.Schema) + query := fmt.Sprintf(`SELECT + c.TABLE_NAME, + c.COLUMN_NAME, + c.DATA_TYPE, + CASE WHEN c.IS_NULLABLE = 'YES' THEN 1 ELSE 0 END, + c.COLUMN_DEFAULT, + c.ORDINAL_POSITION, + c.CHARACTER_MAXIMUM_LENGTH, + c.NUMERIC_PRECISION, + c.NUMERIC_SCALE, + CASE WHEN pk.COLUMN_NAME IS NULL THEN 0 ELSE 1 END, + CASE WHEN uq.COLUMN_NAME IS NULL THEN 0 ELSE 1 END, + sc.is_identity, + sc.max_length, + sc.precision, + sc.scale, + dc.name, + CAST(idc.seed_value AS bigint), + CAST(idc.increment_value AS bigint), + sc.is_computed, + ty.is_user_defined + FROM INFORMATION_SCHEMA.COLUMNS c + LEFT JOIN sys.schemas ss ON ss.name = c.TABLE_SCHEMA + LEFT JOIN sys.objects so ON so.schema_id = ss.schema_id AND so.name = c.TABLE_NAME + LEFT JOIN sys.columns sc ON sc.object_id = so.object_id AND sc.name = c.COLUMN_NAME + LEFT JOIN sys.types ty ON ty.user_type_id = sc.user_type_id + LEFT JOIN sys.identity_columns idc ON idc.object_id = sc.object_id AND idc.column_id = sc.column_id + LEFT JOIN sys.default_constraints dc ON dc.parent_object_id = sc.object_id AND dc.parent_column_id = sc.column_id + LEFT JOIN (%s) pk + ON pk.TABLE_SCHEMA = c.TABLE_SCHEMA AND pk.TABLE_NAME = c.TABLE_NAME AND pk.COLUMN_NAME = c.COLUMN_NAME + LEFT JOIN (%s) uq + ON uq.TABLE_SCHEMA = c.TABLE_SCHEMA AND uq.TABLE_NAME = c.TABLE_NAME AND uq.COLUMN_NAME = c.COLUMN_NAME + WHERE c.TABLE_SCHEMA = %s%s + ORDER BY c.TABLE_NAME, c.ORDINAL_POSITION`, + keyColumns("SELECT", "PRIMARY KEY", schema), keyColumns("SELECT DISTINCT", "UNIQUE", schema), + schema, b.in("c.TABLE_NAME", scope.Names)) + return collect(ctx, q, catalogQuery{Text: query, Args: b.args}, scanColumn) +} + +func keyColumns(selectClause, constraintType, schema string) string { + return fmt.Sprintf(`%s ku.TABLE_SCHEMA, ku.TABLE_NAME, ku.COLUMN_NAME + FROM INFORMATION_SCHEMA.TABLE_CONSTRAINTS tc + JOIN INFORMATION_SCHEMA.KEY_COLUMN_USAGE ku + ON ku.CONSTRAINT_NAME = tc.CONSTRAINT_NAME + AND ku.TABLE_SCHEMA = tc.TABLE_SCHEMA + AND ku.TABLE_NAME = tc.TABLE_NAME + WHERE tc.CONSTRAINT_TYPE = '%s' AND tc.TABLE_SCHEMA = %s`, selectClause, constraintType, schema) +} + +func scanColumn(rows Rows) (Column, error) { + var c Column + var nullable, primaryKey, unique int + var defaultValue, defaultName sql.NullString + var charMax, numPrecision, numScale, maxLength, precision, scale sql.NullInt64 + // is_identity is `bit`: go-mssqldb decodes it to a Go bool, and scanning + // a bool into an int fails. NULL when the sys.objects join misses. + var identity, computed, userDefined sql.NullBool + var seed, increment sql.NullInt64 + if err := rows.Scan(&c.Table, &c.Name, &c.DataType, &nullable, &defaultValue, &c.Position, + &charMax, &numPrecision, &numScale, &primaryKey, &unique, &identity, + &maxLength, &precision, &scale, &defaultName, &seed, &increment, &computed, &userDefined); err != nil { + return c, err + } + c.Nullable, c.PrimaryKey, c.Unique = nullable == 1, primaryKey == 1, unique == 1 + c.Identity = identity.Valid && identity.Bool + c.Computed, c.UserDefinedType = computed.Valid && computed.Bool, userDefined.Valid && userDefined.Bool + if seed.Valid && increment.Valid { + c.IdentitySeed, c.IdentityIncrement = new(seed.Int64), new(increment.Int64) + } + c.Default, c.DefaultConstraint = nullableString(defaultValue), nullableString(defaultName) + c.CharMaxLength, c.NumericPrecision, c.NumericScale = nullableInt(charMax), nullableInt(numPrecision), nullableInt(numScale) + c.MaxLength, c.Precision, c.Scale = nullableInt(maxLength), nullableInt(precision), nullableInt(scale) + return c, nil +} diff --git a/database/inspect/catalog_test.go b/database/inspect/catalog_test.go new file mode 100644 index 00000000..139cfbfa --- /dev/null +++ b/database/inspect/catalog_test.go @@ -0,0 +1,56 @@ +package sqlinspect + +import ( + "testing" +) + +func TestBuildCatalogOrdersAndBoundsMetadata(t *testing.T) { + rows := []columnRow{ + {schema: "z", relation: "events", relationType: "BASE TABLE", column: "id", dataType: "uuid", ordinal: 1}, + {schema: "public", relation: "users", relationType: "BASE TABLE", column: "name", dataType: "text", ordinal: 2}, + {schema: "public", relation: "users", relationType: "BASE TABLE", column: "id", dataType: "uuid", ordinal: 1}, + {schema: "public", relation: "active_users", relationType: "VIEW", column: "id", dataType: "uuid", ordinal: 1}, + } + catalog := buildCatalog(catalogInput{Driver: "postgres", Database: "app", DefaultSchema: "public", Databases: []string{"postgres", "app"}, Schemas: []string{"public", "z"}, Rows: rows}) + if catalog.Database != "app" || catalog.DefaultSchema != "public" || len(catalog.Schemas) != 2 { + t.Fatalf("catalog = %#v", catalog) + } + if len(catalog.Databases) != 2 || catalog.Databases[0] != "app" { + t.Fatalf("databases = %#v", catalog.Databases) + } + if catalog.Schemas[0].Name != "public" || catalog.Schemas[0].Relations[0].Name != "active_users" { + t.Fatalf("catalog is not deterministic: %#v", catalog.Schemas) + } + if catalog.Schemas[0].Relations[0].Type != "view" || len(catalog.Schemas[0].Relations[1].Columns) != 2 { + t.Fatalf("relation metadata = %#v", catalog.Schemas[0].Relations) + } + + truncated := buildCatalog(catalogInput{Driver: "postgres", Database: "app", DefaultSchema: "public", Databases: []string{"app"}, Schemas: []string{"public", "z"}, Rows: rows, Limits: Limits{MaxRelations: 1, MaxColumns: 1}}) + if !truncated.Truncated || truncated.TruncateReason == "" { + t.Fatalf("expected bounded catalog: %#v", truncated) + } +} + +func TestBuildCatalogPreservesEmptySchemas(t *testing.T) { + catalog := buildCatalog(catalogInput{Driver: "postgres", Database: "postgres", DefaultSchema: "public", Databases: []string{"postgres"}, Schemas: []string{"public"}}) + if len(catalog.Schemas) != 1 || catalog.Schemas[0].Name != "public" || len(catalog.Schemas[0].Relations) != 0 { + t.Fatalf("empty schema catalog = %#v", catalog) + } +} + +func TestInspectionQueriesSupportsBrowserDrivers(t *testing.T) { + for _, driver := range []string{"postgres", "mysql", "sql_server", "clickhouse"} { + if _, _, err := inspectionQueries(driver); err != nil { + t.Errorf("%s: %v", driver, err) + } + if schemaInspectionQuery(driver) == "" { + t.Errorf("%s: missing schema inspection query", driver) + } + if databaseInspectionQuery(driver) == "" { + t.Errorf("%s: missing database inspection query", driver) + } + } + if _, _, err := inspectionQueries("__unsupported__"); err == nil { + t.Fatal("unsupported driver must fail") + } +} diff --git a/database/inspect/inspector/bounds.go b/database/inspect/inspector/bounds.go new file mode 100644 index 00000000..1c64c489 --- /dev/null +++ b/database/inspect/inspector/bounds.go @@ -0,0 +1,72 @@ +package inspector + +import ( + "context" + "database/sql" + "fmt" + "strings" +) + +// Bounds is owned by one inspection. Rows caps each query; DefinitionBytes is +// consumed across queries and schemas. Oversized definitions are omitted rather +// than exposing incomplete SQL as a usable object definition. +type Bounds struct { + Rows int + DefinitionBytes int + Truncated bool +} + +type boundedDB struct { + *sql.DB + driver string + bounds *Bounds +} + +// QueryContext caps the database result before the driver buffers rows, with one +// lookahead row to distinguish an exact fit from a truncated catalog. +func (db *boundedDB) QueryContext(ctx context.Context, query string, args ...any) (*boundedRows, error) { + limit := max(db.bounds.Rows, 0) + query = strings.TrimSpace(query) + if db.driver == "sqlserver" { + query = strings.Replace(query, "SELECT", fmt.Sprintf("SELECT TOP (@p%d)", len(args)+1), 1) + } else { + query += fmt.Sprintf(" LIMIT $%d", len(args)+1) + } + rows, err := db.DB.QueryContext(ctx, query, append(args, limit+1)...) + if err != nil { + return nil, err + } + return &boundedRows{Rows: rows, remaining: limit, bounds: db.bounds}, nil +} + +func (db *boundedDB) definitionLimit() int { + return max(db.bounds.DefinitionBytes, 0) + 1 +} + +func (db *boundedDB) definition(value string) string { + if len(value) > db.bounds.DefinitionBytes { + db.bounds.Truncated = true + db.bounds.DefinitionBytes = 0 + return "" + } + db.bounds.DefinitionBytes -= len(value) + return value +} + +type boundedRows struct { + *sql.Rows + remaining int + bounds *Bounds +} + +func (rows *boundedRows) Next() bool { + if !rows.Rows.Next() { + return false + } + if rows.remaining == 0 { + rows.bounds.Truncated = true + return false + } + rows.remaining-- + return true +} diff --git a/database/inspect/inspector/models.go b/database/inspect/inspector/models.go new file mode 100644 index 00000000..4d232f50 --- /dev/null +++ b/database/inspect/inspector/models.go @@ -0,0 +1,135 @@ +package inspector + +import ( + "context" + "database/sql" + "fmt" +) + +// Table represents a database table or view +type Table struct { + Schema string + Name string + Type string // "table" or "view" + ViewDef string // SQL definition for views +} + +// Column represents a table column +type Column struct { + TableName string + ColumnName string + DataType string + IsNullable bool + DefaultValue *string + Position int + IsAutoIncrement bool + IsPrimaryKey bool + IsUnique bool + MaxLength *int // For varchar, char + NumericScale *int // For decimal, numeric + NumericPrecision *int // For decimal, numeric + Comment *string + EnumValues []string // For enum types +} + +// Index represents a table index +type Index struct { + TableName string + IndexName string + IsUnique bool + IsPrimary bool + Columns []string // Ordered list of column names + Type string // btree, hash, gin, gist, etc. + Condition string // WHERE clause for partial indexes +} + +// ForeignKey represents a foreign key constraint +type ForeignKey struct { + TableName string + ColumnName string + RefSchemaName string + RefTableName string + RefColumnName string + ConstraintName string +} + +// StoredProc represents a stored procedure or function +type StoredProc struct { + ID string + Schema string + Name string + Type string // "procedure" or "function" + SQL string // Complete SQL body/definition + ReturnType string // For functions: the SQL return type (e.g. "int", "TABLE"); empty for procedures +} + +// ProcParam represents a stored procedure parameter +type ProcParam struct { + ProcID string + ProcName string + ParamName string + DataType string + Position int +} + +// Trigger represents a database trigger attached to a table. +// +// Event is one or more trigger events, comma-joined when a single trigger +// fires on multiple DML actions (e.g. "INSERT,UPDATE"). Timing is "AFTER" +// or "INSTEAD OF" for SQL Server, which does not support BEFORE triggers. +type Trigger struct { + Schema string + Name string + TableName string + Event string + Timing string + IsDisabled bool + SQL string // CREATE TRIGGER body from OBJECT_DEFINITION +} + +// Inspector provides database introspection capabilities. +// +// Every catalogue method is schema-scoped: it returns a bounded set of rows +// in one round trip, and each row carries the name of the object that owns it +// (Column.TableName, Index.TableName, ForeignKey.TableName, ProcParam.ProcName). +// Callers group; dialects do not. Asking per object turned a schema dump into +// thousands of round trips for no extra information. +// +// Ordering is part of the contract, not an accident: every method returns rows +// ordered by owner and then by the object's own natural order (ordinal position, +// index name, constraint column position, parameter id). Consumers cache and +// compare catalogs, so row ordering must be deterministic. +type Inspector interface { + // Schema operations + GetSchemas(ctx context.Context) ([]string, error) + GetDefaultSchema(ctx context.Context) (string, error) + GetDatabaseName(ctx context.Context) (string, error) + + // Table operations + GetTables(ctx context.Context, schema string, tableType string) ([]*Table, error) + GetColumns(ctx context.Context, schema string) ([]*Column, error) + GetIndexes(ctx context.Context, schema string) ([]*Index, error) + GetForeignKeys(ctx context.Context, schema string) ([]*ForeignKey, error) + + // Stored procedure operations + GetStoredProcs(ctx context.Context, schema string) ([]*StoredProc, error) + GetProcParams(ctx context.Context, schema string) ([]*ProcParam, error) + + // Trigger operations. Dialects without trigger support return (nil, nil); + // only absence-of-feature is signalled by the nil slice, errors are real + // introspection failures. + GetTriggers(ctx context.Context, schema string) ([]*Trigger, error) +} + +// NewInspector creates an inspector for the given database driver +func NewInspector(db *sql.DB, driver string, bounds *Bounds) (Inspector, error) { + reader := &boundedDB{DB: db, driver: driver, bounds: bounds} + switch driver { + case "sqlserver": + return &SQLServerInspector{db: reader}, nil + case "postgres": + return &PostgresInspector{db: reader}, nil + default: + return nil, fmt.Errorf("unsupported database driver: %s", driver) + } +} diff --git a/database/inspect/inspector/postgres.go b/database/inspect/inspector/postgres.go new file mode 100644 index 00000000..25fead2f --- /dev/null +++ b/database/inspect/inspector/postgres.go @@ -0,0 +1,327 @@ +package inspector + +import ( + "context" + "database/sql" + "errors" + + "github.com/lib/pq" +) + +// PostgresInspector provides PostgreSQL database introspection +type PostgresInspector struct { + db *boundedDB +} + +// GetSchemas returns all non-system schemas in the database +func (i *PostgresInspector) GetSchemas(ctx context.Context) (_ []string, returnErr error) { + query := ` + SELECT schema_name + FROM information_schema.schemata + WHERE schema_name NOT IN ('pg_catalog', 'information_schema', 'pg_toast') + AND schema_name NOT LIKE 'pg_temp_%' + AND schema_name NOT LIKE 'pg_toast_temp_%' + ORDER BY schema_name + ` + + rows, err := i.db.QueryContext(ctx, query) + if err != nil { + return nil, err + } + defer func() { returnErr = errors.Join(returnErr, rows.Close()) }() + + var schemas []string + for rows.Next() { + var schema string + if err := rows.Scan(&schema); err != nil { + return nil, err + } + schemas = append(schemas, schema) + } + return schemas, rows.Err() +} + +// GetDefaultSchema returns the default schema for the current user +func (i *PostgresInspector) GetDefaultSchema(ctx context.Context) (string, error) { + var schema string + err := i.db.QueryRowContext(ctx, "SELECT current_schema()").Scan(&schema) + return schema, err +} + +// GetDatabaseName returns the current database name +func (i *PostgresInspector) GetDatabaseName(ctx context.Context) (string, error) { + var dbName string + err := i.db.QueryRowContext(ctx, "SELECT current_database()").Scan(&dbName) + return dbName, err +} + +// GetTables returns all tables or views in the specified schema +func (i *PostgresInspector) GetTables(ctx context.Context, schema string, tableType string) (_ []*Table, returnErr error) { + pgType := "BASE TABLE" + if tableType == "view" { + pgType = "VIEW" + } + + query := ` + SELECT + table_schema, + table_name, + table_type, + LEFT(COALESCE(view_definition, ''), $3) as view_def + FROM information_schema.tables + LEFT JOIN information_schema.views USING (table_schema, table_name) + WHERE table_schema = $1 + AND table_type = $2 + ORDER BY table_name + ` + + rows, err := i.db.QueryContext(ctx, query, schema, pgType, i.db.definitionLimit()) + if err != nil { + return nil, err + } + defer func() { returnErr = errors.Join(returnErr, rows.Close()) }() + + var tables []*Table + for rows.Next() { + var t Table + var viewDef sql.NullString + if err := rows.Scan(&t.Schema, &t.Name, &t.Type, &viewDef); err != nil { + return nil, err + } + if viewDef.Valid { + t.ViewDef = i.db.definition(viewDef.String) + } + tables = append(tables, &t) + } + return tables, rows.Err() +} + +// GetColumns returns every column of every table and view in the schema +func (i *PostgresInspector) GetColumns(ctx context.Context, schema string) (_ []*Column, returnErr error) { + query := ` + SELECT + table_name, + column_name, + CASE WHEN data_type IN ('ARRAY', 'USER-DEFINED') THEN udt_name ELSE data_type END, + CASE WHEN is_nullable = 'YES' THEN true ELSE false END, + column_default, + ordinal_position, + CASE WHEN is_identity = 'YES' THEN true ELSE false END, + character_maximum_length, + numeric_precision, + numeric_scale, + COALESCE(keys.primary_key, false), + COALESCE(keys.unique_key, false) + FROM information_schema.columns c + LEFT JOIN ( + SELECT t.relname, a.attname, + bool_or(con.contype = 'p') AS primary_key, + bool_or(con.contype = 'u') AS unique_key + FROM pg_constraint con + JOIN pg_class t ON t.oid = con.conrelid + JOIN pg_namespace n ON n.oid = t.relnamespace + CROSS JOIN LATERAL unnest(con.conkey) AS k(attnum) + JOIN pg_attribute a ON a.attrelid = t.oid AND a.attnum = k.attnum + WHERE n.nspname = $1 AND con.contype IN ('p', 'u') + GROUP BY t.relname, a.attname + ) keys ON keys.relname = c.table_name AND keys.attname = c.column_name + WHERE table_schema = $1 + ORDER BY table_name, ordinal_position + ` + + rows, err := i.db.QueryContext(ctx, query, schema) + if err != nil { + return nil, err + } + defer func() { returnErr = errors.Join(returnErr, rows.Close()) }() + + var columns []*Column + for rows.Next() { + var c Column + var defaultVal sql.NullString + var maxLength, precision, scale sql.NullInt64 + if err := rows.Scan(&c.TableName, &c.ColumnName, &c.DataType, &c.IsNullable, &defaultVal, &c.Position, + &c.IsAutoIncrement, &maxLength, &precision, &scale, &c.IsPrimaryKey, &c.IsUnique); err != nil { + return nil, err + } + if defaultVal.Valid { + c.DefaultValue = &defaultVal.String + } + c.MaxLength = nullableInt(maxLength) + c.NumericPrecision = nullableInt(precision) + c.NumericScale = nullableInt(scale) + columns = append(columns, &c) + } + return columns, rows.Err() +} + +// GetIndexes returns every index on every relation in the schema. +func (i *PostgresInspector) GetIndexes(ctx context.Context, schema string) (_ []*Index, returnErr error) { + query := ` + SELECT + t.relname as table_name, + i.relname as index_name, + ix.indisunique as is_unique, + ix.indisprimary as is_primary, + array_agg(CASE WHEN keys.attnum = 0 + THEN pg_get_indexdef(ix.indexrelid, keys.ordinality::int, true) + ELSE a.attname END ORDER BY keys.ordinality) as columns, + am.amname, + COALESCE(pg_get_expr(ix.indpred, ix.indrelid), '') + FROM pg_class t + INNER JOIN pg_namespace n ON t.relnamespace = n.oid + INNER JOIN pg_index ix ON t.oid = ix.indrelid + INNER JOIN pg_class i ON i.oid = ix.indexrelid + INNER JOIN pg_am am ON i.relam = am.oid + CROSS JOIN LATERAL unnest(ix.indkey) WITH ORDINALITY AS keys(attnum, ordinality) + LEFT JOIN pg_attribute a ON a.attrelid = t.oid AND a.attnum = keys.attnum + WHERE n.nspname = $1 AND keys.ordinality <= ix.indnkeyatts + GROUP BY t.relname, i.relname, ix.indisunique, ix.indisprimary, am.amname, ix.indpred, ix.indrelid + ORDER BY t.relname, i.relname + ` + + rows, err := i.db.QueryContext(ctx, query, schema) + if err != nil { + return nil, err + } + defer func() { returnErr = errors.Join(returnErr, rows.Close()) }() + + var indexes []*Index + for rows.Next() { + var idx Index + var columnsArr pq.StringArray + if err := rows.Scan(&idx.TableName, &idx.IndexName, &idx.IsUnique, &idx.IsPrimary, &columnsArr, &idx.Type, &idx.Condition); err != nil { + return nil, err + } + idx.Columns = []string(columnsArr) + indexes = append(indexes, &idx) + } + return indexes, rows.Err() +} + +// GetForeignKeys returns every foreign key column in the schema. +func (i *PostgresInspector) GetForeignKeys(ctx context.Context, schema string) (_ []*ForeignKey, returnErr error) { + query := ` + SELECT + src.relname, con.conname, src_col.attname, ref_ns.nspname, ref.relname, ref_col.attname + FROM pg_constraint con + JOIN pg_class src ON src.oid = con.conrelid + JOIN pg_namespace ns ON ns.oid = src.relnamespace + JOIN pg_class ref ON ref.oid = con.confrelid + JOIN pg_namespace ref_ns ON ref_ns.oid = ref.relnamespace + CROSS JOIN LATERAL unnest(con.conkey) WITH ORDINALITY src_key(attnum, ordinality) + JOIN LATERAL unnest(con.confkey) WITH ORDINALITY ref_key(attnum, ordinality) + ON ref_key.ordinality = src_key.ordinality + JOIN pg_attribute src_col ON src_col.attrelid = src.oid AND src_col.attnum = src_key.attnum + JOIN pg_attribute ref_col ON ref_col.attrelid = ref.oid AND ref_col.attnum = ref_key.attnum + WHERE con.contype = 'f' AND ns.nspname = $1 + ORDER BY src.relname, con.conname, src_key.ordinality + ` + + rows, err := i.db.QueryContext(ctx, query, schema) + if err != nil { + return nil, err + } + defer func() { returnErr = errors.Join(returnErr, rows.Close()) }() + + var fks []*ForeignKey + for rows.Next() { + var fk ForeignKey + if err := rows.Scan(&fk.TableName, &fk.ConstraintName, &fk.ColumnName, &fk.RefSchemaName, &fk.RefTableName, &fk.RefColumnName); err != nil { + return nil, err + } + fks = append(fks, &fk) + } + return fks, rows.Err() +} + +// GetStoredProcs returns all stored procedures and functions in the specified schema +func (i *PostgresInspector) GetStoredProcs(ctx context.Context, schema string) (_ []*StoredProc, returnErr error) { + query := ` + SELECT + p.proname || '_' || p.oid AS specific_name, + n.nspname AS schema_name, + p.proname AS proc_name, + CASE p.prokind + WHEN 'f' THEN 'function' + WHEN 'p' THEN 'procedure' + ELSE 'function' + END AS proc_type, + LEFT(pg_get_functiondef(p.oid), $2) AS proc_sql + FROM pg_proc p + INNER JOIN pg_namespace n ON p.pronamespace = n.oid + WHERE n.nspname = $1 + AND p.prokind IN ('f', 'p') + ORDER BY p.proname, p.oid + ` + + rows, err := i.db.QueryContext(ctx, query, schema, i.db.definitionLimit()) + if err != nil { + return nil, err + } + defer func() { returnErr = errors.Join(returnErr, rows.Close()) }() + + var procs []*StoredProc + for rows.Next() { + var p StoredProc + var procSQL sql.NullString + if err := rows.Scan(&p.ID, &p.Schema, &p.Name, &p.Type, &procSQL); err != nil { + return nil, err + } + if procSQL.Valid { + p.SQL = i.db.definition(procSQL.String) + } + procs = append(procs, &p) + } + return procs, rows.Err() +} + +// GetProcParams returns the parameters of every routine in the schema. +func (i *PostgresInspector) GetProcParams(ctx context.Context, schema string) (_ []*ProcParam, returnErr error) { + query := ` + SELECT + p.specific_name, + pr.proname AS proc_name, + COALESCE(p.parameter_name, ''), + p.data_type, + p.ordinal_position + FROM information_schema.parameters p + INNER JOIN pg_proc pr ON p.specific_name = pr.proname || '_' || pr.oid + INNER JOIN pg_namespace n ON pr.pronamespace = n.oid + WHERE n.nspname = $1 AND p.specific_schema = n.nspname + AND p.parameter_mode IN ('IN', 'INOUT') + ORDER BY pr.proname, pr.oid, p.ordinal_position + ` + + rows, err := i.db.QueryContext(ctx, query, schema) + if err != nil { + return nil, err + } + defer func() { returnErr = errors.Join(returnErr, rows.Close()) }() + + var params []*ProcParam + for rows.Next() { + var p ProcParam + if err := rows.Scan(&p.ProcID, &p.ProcName, &p.ParamName, &p.DataType, &p.Position); err != nil { + return nil, err + } + params = append(params, &p) + } + return params, rows.Err() +} + +// GetTriggers returns triggers in the schema. Not yet implemented for +// Postgres — returns nil, nil so the extractor can degrade gracefully. +// TODO: triggers — MSSQL-only for now. +func (i *PostgresInspector) GetTriggers(ctx context.Context, schema string) ([]*Trigger, error) { + return nil, nil +} + +// nullableInt keeps "the catalogue said nothing" distinct from "the catalogue +// said zero": a non-character column has no width, which is not a width of 0. +func nullableInt(v sql.NullInt64) *int { + if !v.Valid { + return nil + } + return new(int(v.Int64)) +} diff --git a/database/inspect/inspector/sqlserver.go b/database/inspect/inspector/sqlserver.go new file mode 100644 index 00000000..13e05eb6 --- /dev/null +++ b/database/inspect/inspector/sqlserver.go @@ -0,0 +1,220 @@ +package inspector + +import ( + "context" + "database/sql" + "errors" + + "github.com/flanksource/sqlglot-go/database/inspect/catalog/sqlserver" +) + +// SQLServerInspector provides SQL Server database introspection. Catalog SQL is +// owned by internal/mssqlcatalog; this type maps its rows onto the inspector +// models and applies the inspection's row and definition bounds. +type SQLServerInspector struct { + db *boundedDB +} + +// boundedCatalog runs mssqlcatalog queries through the inspection's bounds. +type boundedCatalog struct{ db *boundedDB } + +func (c boundedCatalog) QueryContext(ctx context.Context, query string, args ...any) (mssqlcatalog.Rows, error) { + rows, err := c.db.QueryContext(ctx, query, args...) + if err != nil { + return nil, err + } + return rows, nil +} + +func (i *SQLServerInspector) catalog() mssqlcatalog.Queryer { return boundedCatalog{i.db} } + +func (i *SQLServerInspector) scope(schema string) mssqlcatalog.Scope { + return mssqlcatalog.Scope{Schema: schema, DefinitionLimit: i.db.definitionLimit()} +} + +// GetSchemas returns all non-system schemas in the database +func (i *SQLServerInspector) GetSchemas(ctx context.Context) ([]string, error) { + return mssqlcatalog.Schemas(ctx, i.catalog(), nil) +} + +// GetDefaultSchema returns the default schema for the current user +func (i *SQLServerInspector) GetDefaultSchema(ctx context.Context) (string, error) { + var schema string + err := i.db.QueryRowContext(ctx, "SELECT SCHEMA_NAME()").Scan(&schema) + return schema, err +} + +// GetDatabaseName returns the current database name +func (i *SQLServerInspector) GetDatabaseName(ctx context.Context) (string, error) { + var dbName string + err := i.db.QueryRowContext(ctx, "SELECT DB_NAME()").Scan(&dbName) + return dbName, err +} + +// GetTables returns all tables or views in the specified schema +func (i *SQLServerInspector) GetTables(ctx context.Context, schema string, tableType string) ([]*Table, error) { + rows, err := mssqlcatalog.Tables(ctx, i.catalog(), i.scope(schema), mssqlcatalog.TableType(tableType)) + if err != nil { + return nil, err + } + var tables []*Table + for _, row := range rows { + t := &Table{Schema: row.Schema, Name: row.Name, Type: row.Type} + if row.Definition != nil { + t.ViewDef = i.db.definition(*row.Definition) + } + tables = append(tables, t) + } + return tables, nil +} + +// GetColumns returns every column of every table and view in the schema. +// +// Widths, precision and scale are the INFORMATION_SCHEMA character values, not +// sys.columns byte sizes: the browser needs character widths. +func (i *SQLServerInspector) GetColumns(ctx context.Context, schema string) ([]*Column, error) { + rows, err := mssqlcatalog.Columns(ctx, i.catalog(), i.scope(schema)) + if err != nil { + return nil, err + } + var columns []*Column + for _, row := range rows { + columns = append(columns, &Column{ + TableName: row.Table, ColumnName: row.Name, DataType: row.DataType, + IsNullable: row.Nullable, DefaultValue: row.Default, Position: row.Position, + IsAutoIncrement: row.Identity, IsPrimaryKey: row.PrimaryKey, IsUnique: row.Unique, + MaxLength: row.CharMaxLength, NumericPrecision: row.NumericPrecision, NumericScale: row.NumericScale, + }) + } + return columns, nil +} + +// GetIndexes returns every named index on every table in the schema. Columns +// is the ordered key; included columns are excluded. +func (i *SQLServerInspector) GetIndexes(ctx context.Context, schema string) ([]*Index, error) { + rows, err := mssqlcatalog.Indexes(ctx, i.catalog(), i.scope(schema)) + if err != nil { + return nil, err + } + var indexes []*Index + for _, row := range rows { + indexes = append(indexes, &Index{ + TableName: row.Table, IndexName: row.Name, IsUnique: row.Unique, IsPrimary: row.Primary, + Columns: row.KeyColumns, Type: row.TypeDesc, Condition: row.Filter, + }) + } + return indexes, nil +} + +// GetForeignKeys returns every foreign key column in the schema, one row per +// participating column, in constraint column order. +func (i *SQLServerInspector) GetForeignKeys(ctx context.Context, schema string) ([]*ForeignKey, error) { + rows, err := mssqlcatalog.ForeignKeys(ctx, i.catalog(), i.scope(schema)) + if err != nil { + return nil, err + } + var fks []*ForeignKey + for _, row := range rows { + fks = append(fks, &ForeignKey{ + TableName: row.Table, ConstraintName: row.Name, ColumnName: row.Column, + RefSchemaName: row.RefSchema, RefTableName: row.RefTable, RefColumnName: row.RefColumn, + }) + } + return fks, nil +} + +// GetStoredProcs returns all stored procedures and functions in the specified schema +func (i *SQLServerInspector) GetStoredProcs(ctx context.Context, schema string) ([]*StoredProc, error) { + rows, err := mssqlcatalog.Routines(ctx, i.catalog(), i.scope(schema)) + if err != nil { + return nil, err + } + var procs []*StoredProc + for _, row := range rows { + p := &StoredProc{ID: row.Name, Schema: row.Schema, Name: row.Name, Type: "procedure"} + if row.Type.IsFunction() { + p.Type = "function" + } + if row.Definition != nil { + p.SQL = i.db.definition(*row.Definition) + } + if row.ReturnType != nil { + p.ReturnType = *row.ReturnType + } + procs = append(procs, p) + } + return procs, nil +} + +// sqlServerTriggersQuery lists every DML trigger in a schema with its firing +// events comma-joined in type_desc order. Ordered string aggregation uses the +// FOR XML PATH idiom because STRING_AGG ... WITHIN GROUP is rejected below +// compatibility level 140 (OIPA databases run at level 100). +const sqlServerTriggersQuery = ` + SELECT + s.name AS schema_name, + tr.name AS trigger_name, + t.name AS table_name, + STUFF(( + SELECT ',' + te.type_desc + FROM sys.trigger_events te + WHERE te.object_id = tr.object_id + ORDER BY te.type_desc + FOR XML PATH(''), TYPE + ).value('.', 'nvarchar(max)'), 1, 1, '') AS events, + CASE WHEN tr.is_instead_of_trigger = 1 THEN 'INSTEAD OF' ELSE 'AFTER' END AS timing, + tr.is_disabled, + LEFT(OBJECT_DEFINITION(tr.object_id), @p2) AS trigger_sql + FROM sys.triggers tr + INNER JOIN sys.objects t ON tr.parent_id = t.object_id + INNER JOIN sys.schemas s ON t.schema_id = s.schema_id + WHERE s.name = @p1 + AND tr.parent_class = 1 + ORDER BY t.name, tr.name +` + +// GetTriggers returns all DML triggers in the given schema, grouped by +// (schema, name). A single trigger firing on multiple events (e.g. +// INSERT+UPDATE) is returned as one row with a comma-joined Event string. +func (i *SQLServerInspector) GetTriggers(ctx context.Context, schema string) (_ []*Trigger, returnErr error) { + rows, err := i.db.QueryContext(ctx, sqlServerTriggersQuery, schema, i.db.definitionLimit()) + if err != nil { + return nil, err + } + defer func() { returnErr = errors.Join(returnErr, rows.Close()) }() + + var triggers []*Trigger + for rows.Next() { + var tr Trigger + var events, body sql.NullString + // is_disabled is `bit`, which go-mssqldb decodes to a Go bool; scan it as a bool. + if err := rows.Scan(&tr.Schema, &tr.Name, &tr.TableName, &events, &tr.Timing, &tr.IsDisabled, &body); err != nil { + return nil, err + } + if events.Valid { + tr.Event = events.String + } + if body.Valid { + tr.SQL = i.db.definition(body.String) + } + triggers = append(triggers, &tr) + } + return triggers, rows.Err() +} + +// GetProcParams returns the parameters of every stored procedure and function +// in the schema. A scalar function's return value (parameter_id 0) is not a +// parameter; GetStoredProcs reports it as ReturnType. +func (i *SQLServerInspector) GetProcParams(ctx context.Context, schema string) ([]*ProcParam, error) { + rows, err := mssqlcatalog.Parameters(ctx, i.catalog(), i.scope(schema)) + if err != nil { + return nil, err + } + var params []*ProcParam + for _, row := range rows { + params = append(params, &ProcParam{ + ProcID: row.Routine, ProcName: row.Routine, ParamName: row.Name, DataType: row.DataType, Position: row.Position, + }) + } + return params, nil +} diff --git a/database/inspect/options.go b/database/inspect/options.go new file mode 100644 index 00000000..61e6346f --- /dev/null +++ b/database/inspect/options.go @@ -0,0 +1,13 @@ +package sqlinspect + +// NormalizeDriver resolves supported driver aliases to their catalog backend. +func NormalizeDriver(driver string) string { return normalizeDriver(driver) } + +// WithDefaults resolves zero catalog limits to the documented bounds. +func (l Limits) WithDefaults() Limits { return l.withDefaults() } + +// ValidateDriver rejects backends without catalog queries. +func ValidateDriver(driver string) error { + _, _, err := inspectionQueries(driver) + return err +} diff --git a/database/inspect/rich.go b/database/inspect/rich.go new file mode 100644 index 00000000..ef7fe070 --- /dev/null +++ b/database/inspect/rich.go @@ -0,0 +1,153 @@ +package sqlinspect + +import ( + "context" + "database/sql" + "fmt" + + "github.com/flanksource/sqlglot-go/database/inspect/inspector" +) + +// inspectRich assembles schema-scoped rows into the browser catalog. Optional +// metadata budgets must not suppress table/column browsing in later schemas. +func inspectRich(ctx context.Context, db *sql.DB, driver string, limits Limits) (Catalog, error) { + limits = limits.withDefaults() + bounds := &inspector.Bounds{Rows: limits.MaxRelations, DefinitionBytes: limits.MaxDefinitionBytes} + i, err := inspector.NewInspector(db, driver, bounds) + if err != nil { + return Catalog{}, err + } + database, err := i.GetDatabaseName(ctx) + if err != nil { + return Catalog{}, fmt.Errorf("inspect sql database: %w", err) + } + defaultSchema, err := i.GetDefaultSchema(ctx) + if err != nil { + return Catalog{}, fmt.Errorf("inspect sql default schema: %w", err) + } + databases, err := ListDatabases(ctx, db, driver) + if err != nil { + return Catalog{}, err + } + schemaNames, err := i.GetSchemas(ctx) + if err != nil { + return Catalog{}, fmt.Errorf("inspect sql schemas: %w", err) + } + + catalog := Catalog{Driver: driver, Database: database, Databases: databases, DefaultSchema: defaultSchema, Schemas: []Schema{}} + budget := richBudget{relations: limits.MaxRelations, columns: limits.MaxColumns, routines: limits.MaxRoutines, metadata: limits.MaxColumns, bounds: bounds} + for _, schemaName := range schemaNames { + if budget.relations == 0 || budget.columns == 0 { + bounds.Truncated = true + break + } + schema, err := inspectRichSchema(ctx, i, schemaName, &budget) + if err != nil { + return Catalog{}, fmt.Errorf("inspect sql schema %q: %w", schemaName, err) + } + catalog.Schemas = append(catalog.Schemas, schema) + } + catalog.Truncated = bounds.Truncated + if catalog.Truncated { + catalog.TruncateReason = fmt.Sprintf("catalog limits reached: %d relations, %d columns, %d routines, %d metadata rows, %d definition bytes", limits.MaxRelations, limits.MaxColumns, limits.MaxRoutines, limits.MaxColumns, limits.MaxDefinitionBytes) + } + return catalog, nil +} + +type richBudget struct { + relations, columns, routines, metadata int + bounds *inspector.Bounds +} + +// readCatalog bounds each query by the remaining global budget, not a fresh +// allowance per schema. The inspector detects overflow with a lookahead row. +func readCatalog[T any](bounds *inspector.Bounds, remaining *int, read func() ([]*T, error)) ([]*T, error) { + bounds.Rows = *remaining + items, err := read() + *remaining -= len(items) + return items, err +} + +func inspectRichSchema(ctx context.Context, i inspector.Inspector, schemaName string, budget *richBudget) (Schema, error) { + bounds := budget.bounds + tables, err := readCatalog(bounds, &budget.relations, func() ([]*inspector.Table, error) { return i.GetTables(ctx, schemaName, "table") }) + if err != nil { + return Schema{}, err + } + views, err := readCatalog(bounds, &budget.relations, func() ([]*inspector.Table, error) { return i.GetTables(ctx, schemaName, "view") }) + if err != nil { + return Schema{}, err + } + columns, err := readCatalog(bounds, &budget.columns, func() ([]*inspector.Column, error) { return i.GetColumns(ctx, schemaName) }) + if err != nil { + return Schema{}, err + } + indexes, err := readCatalog(bounds, &budget.metadata, func() ([]*inspector.Index, error) { return i.GetIndexes(ctx, schemaName) }) + if err != nil { + return Schema{}, err + } + foreignKeys, err := readCatalog(bounds, &budget.metadata, func() ([]*inspector.ForeignKey, error) { return i.GetForeignKeys(ctx, schemaName) }) + if err != nil { + return Schema{}, err + } + triggers, err := readCatalog(bounds, &budget.metadata, func() ([]*inspector.Trigger, error) { return i.GetTriggers(ctx, schemaName) }) + if err != nil { + return Schema{}, err + } + routines, err := readCatalog(bounds, &budget.routines, func() ([]*inspector.StoredProc, error) { return i.GetStoredProcs(ctx, schemaName) }) + if err != nil { + return Schema{}, err + } + params, err := readCatalog(bounds, &budget.metadata, func() ([]*inspector.ProcParam, error) { return i.GetProcParams(ctx, schemaName) }) + if err != nil { + return Schema{}, err + } + + schema := Schema{Name: schemaName, Relations: []Relation{}} + allTables := append(tables, views...) + for _, table := range allTables { + typeName := normalizeRelationType(table.Type) + schema.Relations = append(schema.Relations, Relation{Name: table.Name, Type: typeName, ViewDef: table.ViewDef, Columns: []Column{}}) + } + relations := make(map[string]*Relation, len(schema.Relations)) + for index := range schema.Relations { + relations[schema.Relations[index].Name] = &schema.Relations[index] + } + for _, column := range columns { + relation := relations[column.TableName] + if relation == nil { + continue + } + relation.Columns = append(relation.Columns, Column{Name: column.ColumnName, DataType: column.DataType, Ordinal: column.Position, Nullable: &column.IsNullable, Default: column.DefaultValue, Identity: column.IsAutoIncrement, PrimaryKey: column.IsPrimaryKey, Unique: column.IsUnique, MaxLength: column.MaxLength, NumericPrecision: column.NumericPrecision, NumericScale: column.NumericScale, Comment: column.Comment, EnumValues: column.EnumValues}) + } + for _, index := range indexes { + if relation := relations[index.TableName]; relation != nil { + relation.Indexes = append(relation.Indexes, Index{Name: index.IndexName, Unique: index.IsUnique, Primary: index.IsPrimary, Columns: index.Columns, Type: index.Type, Filter: index.Condition}) + } + } + for _, fk := range foreignKeys { + relation := relations[fk.TableName] + if relation == nil { + continue + } + if len(relation.ForeignKeys) == 0 || relation.ForeignKeys[len(relation.ForeignKeys)-1].Name != fk.ConstraintName { + relation.ForeignKeys = append(relation.ForeignKeys, ForeignKey{Name: fk.ConstraintName, ReferencedSchema: fk.RefSchemaName, ReferencedTable: fk.RefTableName}) + } + group := &relation.ForeignKeys[len(relation.ForeignKeys)-1] + group.Columns = append(group.Columns, fk.ColumnName) + group.ReferencedColumns = append(group.ReferencedColumns, fk.RefColumnName) + } + for _, trigger := range triggers { + if relation := relations[trigger.TableName]; relation != nil { + relation.Triggers = append(relation.Triggers, Trigger{Name: trigger.Name, Event: trigger.Event, Timing: trigger.Timing, Disabled: trigger.IsDisabled, SQL: trigger.SQL}) + } + } + paramsByRoutine := map[string][]RoutineParameter{} + for _, param := range params { + paramsByRoutine[param.ProcID] = append(paramsByRoutine[param.ProcID], RoutineParameter{Name: param.ParamName, DataType: param.DataType, Ordinal: param.Position}) + } + for _, routine := range routines { + schema.Routines = append(schema.Routines, Routine{ID: routine.ID, Name: routine.Name, Type: routine.Type, SQL: routine.SQL, ReturnType: routine.ReturnType, Parameters: paramsByRoutine[routine.ID]}) + } + return schema, nil +} diff --git a/database/inspect/snapshot/README.md b/database/inspect/snapshot/README.md new file mode 100644 index 00000000..1cae7a7c --- /dev/null +++ b/database/inspect/snapshot/README.md @@ -0,0 +1,17 @@ +# Complete migration snapshots + +`snapshot.Inspect(ctx, db, snapshot.Options{Schemas: []string{"dbo"}})` reads full SQL Server migration definitions without the row or definition limits used by the interactive explorer. Schemas are explicit. The connection must have database `VIEW DEFINITION` permission so hidden metadata cannot look like absent objects. + +Inspection reuses the shared SQL Server catalogue reader for tables, columns, indexes, keys, checks and routines. It also reads column collations, view and trigger definitions, trigger state and module dependencies. The result uses sqlglot's native migration model; this package does not generate or execute migrations. + +The current contract targets SQL Server 2016 or newer, including databases at compatibility level 100. Unsupported computed or user-defined columns, storage settings, indexed views, disabled or untrusted constraints, unsupported module session settings, and external or unresolved dependencies cause explicit errors. This prevents a partial catalogue from becoming a destructive desired state. + +Default, check or column expressions that depend on other database objects are also blocked: the native table model does not yet carry those dependencies for ordering and selection validation. + +Trigger schemas come from [sys.objects](https://learn.microsoft.com/en-us/sql/relational-databases/system-catalog-views/sys-objects-transact-sql), while parent and disabled state come from [sys.triggers](https://learn.microsoft.com/en-us/sql/relational-databases/system-catalog-views/sys-triggers-transact-sql). + +```sh +go test ./inspect/sql/snapshot -count=1 +``` + +The tests separate pure catalogue projection from database-driver boundary checks. Live SQL Server execution remains a separate integration gate. diff --git a/database/inspect/snapshot/build.go b/database/inspect/snapshot/build.go new file mode 100644 index 00000000..6cc57ceb --- /dev/null +++ b/database/inspect/snapshot/build.go @@ -0,0 +1,151 @@ +package snapshot + +import ( + "fmt" + "slices" + "strings" + + "github.com/flanksource/sqlglot-go/database/inspect/catalog/sqlserver" + "github.com/flanksource/sqlglot-go/migrate" + dialect "github.com/flanksource/sqlglot-go/migrate/atlas/sqlserver" +) + +func build(c catalog) (migrate.Schema, error) { + var out migrate.Schema + for _, t := range c.tables { + name := migrate.Name{Schema: t.Schema, Name: t.Name} + if t.Type == "VIEW" { + if t.Definition == nil || strings.TrimSpace(*t.Definition) == "" { + return out, fmt.Errorf("view %v has no readable definition", name) + } + out.Definitions = append(out.Definitions, migrate.Definition{Name: name, Kind: "VIEW", SQL: *t.Definition, Dependencies: c.dependencies[t.Name]}) + continue + } + table, err := buildTable(c, t) + if err != nil { + return out, fmt.Errorf("table %v: %w", name, err) + } + out.Tables = append(out.Tables, table) + } + for _, r := range c.routines { + if r.Definition == nil || strings.TrimSpace(*r.Definition) == "" { + return out, fmt.Errorf("routine %s.%s has no readable definition", r.Schema, r.Name) + } + kind := "PROCEDURE" + if r.Type.IsFunction() { + kind = "FUNCTION" + } + out.Definitions = append(out.Definitions, migrate.Definition{Name: migrate.Name{Schema: r.Schema, Name: r.Name}, Kind: kind, SQL: *r.Definition, Dependencies: c.dependencies[r.Name]}) + } + for _, d := range c.modules { + d.Dependencies = c.dependencies[d.Name.Name] + out.Definitions = append(out.Definitions, d) + } + return out, nil +} + +func buildTable(c catalog, input mssqlcatalog.Table) (migrate.Table, error) { + out := migrate.Table{Name: migrate.Name{Schema: input.Schema, Name: input.Name}} + for _, col := range c.columns { + if col.Table != input.Name { + continue + } + column, err := buildColumn(col) + if err != nil { + return out, fmt.Errorf("column %s: %w", col.Name, err) + } + column.Collation = c.collations[[2]string{col.Table, col.Name}] + out.Columns = append(out.Columns, column) + } + for _, index := range c.indexes { + if index.Table != input.Name { + continue + } + idx, err := buildIndex(index, c.parts) + if err != nil { + return out, err + } + out.Indexes = append(out.Indexes, idx) + } + for _, key := range c.keys { + if key.Table != input.Name { + continue + } + at := slices.IndexFunc(out.ForeignKeys, func(k migrate.ForeignKey) bool { return k.Name == key.Name }) + if at < 0 { + out.ForeignKeys = append(out.ForeignKeys, migrate.ForeignKey{Name: key.Name, References: migrate.Name{Schema: key.RefSchema, Name: key.RefTable}, OnDelete: strings.ReplaceAll(key.OnDelete, "_", " "), OnUpdate: strings.ReplaceAll(key.OnUpdate, "_", " ")}) + at = len(out.ForeignKeys) - 1 + } + out.ForeignKeys[at].Columns = append(out.ForeignKeys[at].Columns, key.Column) + out.ForeignKeys[at].ReferencedColumns = append(out.ForeignKeys[at].ReferencedColumns, key.RefColumn) + } + for _, check := range c.checks { + if check.Table != input.Name { + continue + } + if check.Disabled { + return out, fmt.Errorf("disabled check %s is unsupported", check.Name) + } + out.Checks = append(out.Checks, migrate.Check{Name: check.Name, Expression: check.Definition}) + } + return out, nil +} + +func buildColumn(c mssqlcatalog.Column) (migrate.Column, error) { + out := migrate.Column{Name: c.Name, Nullable: c.Nullable} + if c.Computed { + return out, fmt.Errorf("computed columns are unsupported") + } + if c.UserDefinedType { + return out, fmt.Errorf("user-defined column types are unsupported") + } + typ, err := dialect.CatalogType(dialect.CatalogColumn{Name: c.Name, DataType: c.DataType, MaxLength: c.MaxLength, Precision: c.Precision, Scale: c.Scale}) + if err != nil { + return out, err + } + out.Type, err = dialect.FormatType(typ) + if err != nil { + return out, err + } + if c.Identity { + if c.IdentitySeed == nil || c.IdentityIncrement == nil { + return out, fmt.Errorf("identity seed and increment are missing") + } + out.Identity = &migrate.Identity{Seed: *c.IdentitySeed, Increment: *c.IdentityIncrement} + } + if c.Default != nil { + if c.DefaultConstraint == nil || *c.DefaultConstraint == "" { + return out, fmt.Errorf("default constraint name is missing") + } + out.Default = &migrate.Default{Name: *c.DefaultConstraint, Expression: *c.Default} + } + return out, nil +} + +func buildIndex(index mssqlcatalog.Index, parts []mssqlcatalog.IndexColumn) (migrate.Index, error) { + out := migrate.Index{Name: index.Name, Unique: index.Unique, Primary: index.Primary, Constraint: index.UniqueConstraint, Filter: index.Filter} + switch index.TypeDesc { + case "CLUSTERED": + out.Clustered = true + case "NONCLUSTERED": + default: + return out, fmt.Errorf("index %s has unsupported type %q", index.Name, index.TypeDesc) + } + for _, part := range parts { + if part.Table != index.Table || part.Index != index.Name { + continue + } + if part.Included { + out.Included = append(out.Included, part.Column) + continue + } + if part.KeyOrdinal < 1 { + return out, fmt.Errorf("index %s has an implicit or unsupported key column %s", index.Name, part.Column) + } + out.Columns = append(out.Columns, migrate.IndexColumn{Name: part.Column, Descending: part.Descending}) + } + if len(out.Columns) == 0 { + return out, fmt.Errorf("index %s has no key columns", index.Name) + } + return out, nil +} diff --git a/database/inspect/snapshot/modules.go b/database/inspect/snapshot/modules.go new file mode 100644 index 00000000..18ce111c --- /dev/null +++ b/database/inspect/snapshot/modules.go @@ -0,0 +1,141 @@ +package snapshot + +import ( + "context" + "database/sql" + "fmt" + "slices" + + "github.com/flanksource/sqlglot-go/migrate" +) + +func readCollations(ctx context.Context, db Querier, c *catalog) error { + return visit(ctx, db, schemaQuery{SQL: `SELECT o.name, c.name, c.collation_name + FROM sys.columns c JOIN sys.objects o ON o.object_id=c.object_id + JOIN sys.schemas s ON s.schema_id=o.schema_id + WHERE s.name=@p1 AND o.type IN ('U','V') AND c.collation_name IS NOT NULL + ORDER BY o.name, c.column_id`, Schema: c.schema}, func(rows *sql.Rows) error { + var table, column, collation string + if err := rows.Scan(&table, &column, &collation); err != nil { + return err + } + c.collations[[2]string{table, column}] = collation + return nil + }) +} + +func readTriggers(ctx context.Context, db Querier, c *catalog) error { + return visit(ctx, db, schemaQuery{SQL: `SELECT SCHEMA_NAME(trigger_object.schema_id), tr.name, s.name, o.name, tr.is_disabled, m.definition + FROM sys.triggers tr JOIN sys.objects o ON o.object_id=tr.parent_id + JOIN sys.objects trigger_object ON trigger_object.object_id=tr.object_id + JOIN sys.schemas s ON s.schema_id=o.schema_id + LEFT JOIN sys.sql_modules m ON m.object_id=tr.object_id + WHERE s.name=@p1 AND tr.parent_class=1 ORDER BY tr.name`, Schema: c.schema}, func(rows *sql.Rows) error { + var d migrate.Definition + var parent migrate.Name + var definition sql.NullString + if err := rows.Scan(&d.Name.Schema, &d.Name.Name, &parent.Schema, &parent.Name, &d.Disabled, &definition); err != nil { + return err + } + if !definition.Valid || definition.String == "" { + return fmt.Errorf("trigger %v has no readable definition", d.Name) + } + d.Kind, d.SQL, d.Parent = "TRIGGER", definition.String, &parent + c.modules = append(c.modules, d) + return nil + }) +} + +func readDependencies(ctx context.Context, db Querier, c *catalog) error { + return visit(ctx, db, schemaQuery{SQL: `SELECT DISTINCT o.name, d.referenced_server_name, d.referenced_database_name, + COALESCE(SCHEMA_NAME(r.schema_id), d.referenced_schema_name), d.referenced_entity_name, d.referenced_id + FROM sys.sql_expression_dependencies d JOIN sys.objects o ON o.object_id=d.referencing_id + JOIN sys.schemas s ON s.schema_id=o.schema_id LEFT JOIN sys.objects r ON r.object_id=d.referenced_id + WHERE s.name=@p1 AND d.referenced_class=1 AND o.type IN ('V','P','FN','IF','TF','TR') + ORDER BY 1, 4, 5`, Schema: c.schema}, func(rows *sql.Rows) error { + var owner, entity string + var server, database, schema sql.NullString + var id sql.NullInt64 + if err := rows.Scan(&owner, &server, &database, &schema, &entity, &id); err != nil { + return err + } + if server.Valid || database.Valid || !schema.Valid || !id.Valid { + return fmt.Errorf("module %s.%s has an external or unresolved dependency %q", c.schema, owner, entity) + } + name := migrate.Name{Schema: schema.String, Name: entity} + if !slices.Contains(c.dependencies[owner], name) { + c.dependencies[owner] = append(c.dependencies[owner], name) + } + return nil + }) +} + +type schemaQuery struct { + SQL, Schema string +} + +func visit(ctx context.Context, db Querier, query schemaQuery, scan func(*sql.Rows) error) (err error) { + rows, err := db.QueryContext(ctx, query.SQL, query.Schema) + if err != nil { + return err + } + defer func() { + if closeErr := rows.Close(); err == nil { + err = closeErr + } + }() + for rows.Next() { + if err := scan(rows); err != nil { + return err + } + } + return rows.Err() +} + +// Every flag below changes DDL semantics that the native model cannot preserve. +const unsupportedQuery = `SELECT object_name, feature FROM ( + SELECT t.name AS object_name, 'table storage or temporal settings' AS feature + FROM sys.tables t JOIN sys.schemas s ON s.schema_id=t.schema_id + WHERE s.name=@p1 AND (t.temporal_type<>0 OR t.is_memory_optimized=1 OR t.is_filetable=1 OR COALESCE(t.filestream_data_space_id,0)<>0) + UNION ALL + SELECT o.name, 'special column settings' FROM sys.columns c JOIN sys.objects o ON o.object_id=c.object_id + JOIN sys.schemas s ON s.schema_id=o.schema_id + WHERE s.name=@p1 AND o.type='U' AND (c.is_sparse=1 OR c.is_column_set=1 OR c.is_filestream=1 OR c.generated_always_type<>0 OR c.is_rowguidcol=1 OR c.encryption_type IS NOT NULL OR c.is_hidden=1 OR c.rule_object_id<>0) + UNION ALL + SELECT t.name, 'index storage, state or compression' FROM sys.indexes i JOIN sys.tables t ON t.object_id=i.object_id + JOIN sys.schemas s ON s.schema_id=t.schema_id LEFT JOIN sys.data_spaces ds ON ds.data_space_id=i.data_space_id + WHERE s.name=@p1 AND (i.is_disabled=1 OR i.is_hypothetical=1 OR i.type NOT IN (0,1,2) OR ds.type='PS' + OR i.fill_factor<>0 OR i.is_padded=1 OR i.ignore_dup_key=1 OR i.allow_row_locks=0 OR i.allow_page_locks=0 + OR (ds.type='FG' AND NOT EXISTS (SELECT 1 FROM sys.filegroups fg WHERE fg.data_space_id=ds.data_space_id AND fg.is_default=1)) + OR EXISTS (SELECT 1 FROM sys.partitions p WHERE p.object_id=i.object_id AND p.index_id=i.index_id AND p.data_compression<>0)) + UNION ALL + SELECT t.name, 'untrusted or replication-sensitive check' FROM sys.check_constraints c JOIN sys.tables t ON t.object_id=c.parent_object_id + JOIN sys.schemas s ON s.schema_id=t.schema_id WHERE s.name=@p1 AND (c.is_disabled=1 OR c.is_not_trusted=1 OR c.is_not_for_replication=1) + UNION ALL + SELECT t.name, 'untrusted or replication-sensitive foreign key' FROM sys.foreign_keys f JOIN sys.tables t ON t.object_id=f.parent_object_id + JOIN sys.schemas s ON s.schema_id=t.schema_id WHERE s.name=@p1 AND (f.is_disabled=1 OR f.is_not_trusted=1 OR f.is_not_for_replication=1) + UNION ALL + SELECT o.name, 'module session settings' FROM sys.sql_modules m JOIN sys.objects o ON o.object_id=m.object_id + JOIN sys.schemas s ON s.schema_id=o.schema_id WHERE s.name=@p1 AND (m.uses_ansi_nulls=0 OR m.uses_quoted_identifier=0) + UNION ALL + SELECT v.name, 'indexed view' FROM sys.views v JOIN sys.schemas s ON s.schema_id=v.schema_id + WHERE s.name=@p1 AND EXISTS (SELECT 1 FROM sys.indexes i WHERE i.object_id=v.object_id AND i.index_id>0) + UNION ALL + SELECT o.name, 'unsupported schema object '+o.type_desc FROM sys.objects o JOIN sys.schemas s ON s.schema_id=o.schema_id + WHERE s.name=@p1 AND o.is_ms_shipped=0 AND o.type NOT IN ('U','V','P','FN','IF','TF','TR','PK','UQ','F','C','D') + UNION ALL + SELECT o.name, 'dependent default, check or column expression' FROM sys.objects o JOIN sys.schemas s ON s.schema_id=o.schema_id + JOIN sys.sql_expression_dependencies d ON d.referencing_id=o.object_id + WHERE s.name=@p1 AND o.type IN ('U','C','D') + AND (d.referenced_id IS NULL OR d.referenced_id<>COALESCE(NULLIF(o.parent_object_id,0),o.object_id)) +) unsupported ORDER BY object_name, feature` + +func refuseUnsupported(ctx context.Context, db Querier, schema string) error { + return visit(ctx, db, schemaQuery{SQL: unsupportedQuery, Schema: schema}, func(rows *sql.Rows) error { + var object, feature string + if err := rows.Scan(&object, &feature); err != nil { + return err + } + return fmt.Errorf("%s.%s has unsupported %s; migration snapshot is incomplete", schema, object, feature) + }) +} diff --git a/database/inspect/snapshot/querier.go b/database/inspect/snapshot/querier.go new file mode 100644 index 00000000..355826ea --- /dev/null +++ b/database/inspect/snapshot/querier.go @@ -0,0 +1,12 @@ +package snapshot + +import ( + "context" + "database/sql" +) + +// Querier is implemented by a database, pinned connection or transaction. +type Querier interface { + QueryContext(context.Context, string, ...any) (*sql.Rows, error) + QueryRowContext(context.Context, string, ...any) *sql.Row +} diff --git a/database/inspect/snapshot/read_test.go b/database/inspect/snapshot/read_test.go new file mode 100644 index 00000000..a8037a7d --- /dev/null +++ b/database/inspect/snapshot/read_test.go @@ -0,0 +1,66 @@ +package snapshot + +import ( + "errors" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/flanksource/sqlglot-go/migrate" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Migration inspection boundary", func() { + It("rejects expression dependencies that the table migration model cannot preserve", func() { + db, mock, err := sqlmock.New() + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { mock.ExpectClose(); Expect(db.Close()).To(Succeed()) }) + mock.ExpectQuery(`dependent default, check or column expression`).WithArgs("dbo").WillReturnRows(sqlmock.NewRows([]string{"object", "feature"}).AddRow("DF_orders_id", "dependent default, check or column expression")) + err = refuseUnsupported(GinkgoT().Context(), db, "dbo") + Expect(err).To(MatchError(And(ContainSubstring("dbo.DF_orders_id"), ContainSubstring("migration snapshot is incomplete")))) + Expect(mock.ExpectationsWereMet()).To(Succeed()) + }) + It("requires full definition visibility before enumerating objects", func() { + db, mock, err := sqlmock.New() + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { mock.ExpectClose(); Expect(db.Close()).To(Succeed()) }) + mock.ExpectQuery("HAS_PERMS_BY_NAME").WillReturnRows(sqlmock.NewRows([]string{"allowed"}).AddRow(0)) + _, err = Inspect(GinkgoT().Context(), db, Options{Schemas: []string{"dbo"}}) + Expect(err).To(MatchError(ContainSubstring("VIEW DEFINITION"))) + Expect(mock.ExpectationsWereMet()).To(Succeed()) + }) + + It("reads full view definitions and trigger state without explorer limits", func() { + db, mock, err := sqlmock.New() + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { mock.ExpectClose(); Expect(db.Close()).To(Succeed()) }) + mock.ExpectQuery("HAS_PERMS_BY_NAME").WillReturnRows(sqlmock.NewRows([]string{"allowed"}).AddRow(1)) + mock.ExpectQuery("INFORMATION_SCHEMA.SCHEMATA").WithArgs("dbo").WillReturnRows(sqlmock.NewRows([]string{"schema"}).AddRow("dbo")) + viewSQL := "CREATE VIEW dbo.orders AS SELECT 1 AS id" + triggerSQL := "CREATE TRIGGER dbo.audit_orders ON dbo.orders INSTEAD OF INSERT AS SELECT 1" + mock.ExpectQuery(`OBJECT_DEFINITION\(o.object_id\)`).WithArgs("dbo", "").WillReturnRows(sqlmock.NewRows([]string{"schema", "name", "type", "definition"}).AddRow("dbo", "orders", "VIEW", viewSQL)) + for _, query := range []string{`FROM INFORMATION_SCHEMA.COLUMNS`, `FROM sys.indexes`, `FROM sys.index_columns`, `FROM sys.foreign_keys`, `FROM sys.check_constraints`, `FROM sys.objects o`, `FROM sys.columns c`} { + mock.ExpectQuery(query).WithArgs("dbo").WillReturnRows(sqlmock.NewRows([]string{"empty"})) + } + mock.ExpectQuery(`SCHEMA_NAME\(trigger_object.schema_id\)`).WithArgs("dbo").WillReturnRows(sqlmock.NewRows([]string{"schema", "name", "parentSchema", "parent", "disabled", "definition"}).AddRow("dbo", "audit_orders", "dbo", "orders", true, triggerSQL)) + mock.ExpectQuery(`FROM sys.sql_expression_dependencies`).WithArgs("dbo").WillReturnRows(sqlmock.NewRows([]string{"empty"})) + mock.ExpectQuery(`SELECT object_name, feature`).WithArgs("dbo").WillReturnRows(sqlmock.NewRows([]string{"empty"})) + result, err := Inspect(GinkgoT().Context(), db, Options{Schemas: []string{"dbo"}}) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Schema.Definitions).To(Equal([]migrate.Definition{ + {Name: migrate.Name{Schema: "dbo", Name: "orders"}, Kind: "VIEW", SQL: viewSQL}, + {Name: migrate.Name{Schema: "dbo", Name: "audit_orders"}, Kind: "TRIGGER", SQL: triggerSQL, Parent: &migrate.Name{Schema: "dbo", Name: "orders"}, Disabled: true}, + })) + Expect(mock.ExpectationsWereMet()).To(Succeed()) + }) + + It("propagates catalogue read failures with their schema", func() { + db, mock, err := sqlmock.New() + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { mock.ExpectClose(); Expect(db.Close()).To(Succeed()) }) + mock.ExpectQuery("HAS_PERMS_BY_NAME").WillReturnRows(sqlmock.NewRows([]string{"allowed"}).AddRow(1)) + mock.ExpectQuery("INFORMATION_SCHEMA.SCHEMATA").WillReturnError(errors.New("catalogue unavailable")) + _, err = Inspect(GinkgoT().Context(), db, Options{Schemas: []string{"dbo"}}) + Expect(err).To(MatchError(And(ContainSubstring("dbo"), ContainSubstring("catalogue unavailable")))) + Expect(mock.ExpectationsWereMet()).To(Succeed()) + }) +}) diff --git a/database/inspect/snapshot/snapshot.go b/database/inspect/snapshot/snapshot.go new file mode 100644 index 00000000..8623e146 --- /dev/null +++ b/database/inspect/snapshot/snapshot.go @@ -0,0 +1,110 @@ +// Package snapshot inspects complete SQL Server schemas for migration planning. +// Query bounds used by the interactive explorer never apply to these snapshots. +package snapshot + +import ( + "context" + "fmt" + "slices" + + "github.com/flanksource/sqlglot-go/database/inspect/catalog/sqlserver" + "github.com/flanksource/sqlglot-go/migrate" + planner "github.com/flanksource/sqlglot-go/migrate/sqlserver" +) + +type Options struct{ Schemas []string } +type Result struct { + Schema migrate.Schema + Schemas []string +} + +type catalog struct { + schema string + tables []mssqlcatalog.Table + columns []mssqlcatalog.Column + indexes []mssqlcatalog.Index + parts []mssqlcatalog.IndexColumn + keys []mssqlcatalog.ForeignKey + checks []mssqlcatalog.CheckConstraint + routines []mssqlcatalog.Routine + modules []migrate.Definition + collations map[[2]string]string + dependencies map[string][]migrate.Name +} + +func Inspect(ctx context.Context, db Querier, options Options) (Result, error) { + if ctx == nil || db == nil || len(options.Schemas) == 0 { + return Result{}, fmt.Errorf("migration snapshot requires a context, database and explicit schema scope") + } + var permission int + if err := db.QueryRowContext(ctx, "SELECT COALESCE(HAS_PERMS_BY_NAME(DB_NAME(), 'DATABASE', 'VIEW DEFINITION'), 0)").Scan(&permission); err != nil { + return Result{}, fmt.Errorf("inspect migration permissions: %w", err) + } + if permission != 1 { + return Result{}, fmt.Errorf("complete migration inspection requires database VIEW DEFINITION permission") + } + result := Result{Schemas: slices.Clone(options.Schemas)} + seen := map[string]bool{} + dependencies := map[migrate.Name][]migrate.Name{} + for _, name := range options.Schemas { + if seen[name] { + return Result{}, fmt.Errorf("duplicate migration schema %q", name) + } + seen[name] = true + c, err := read(ctx, db, name) + if err != nil { + return Result{}, fmt.Errorf("inspect schema %q: %w", name, err) + } + schema, err := build(c) + if err != nil { + return Result{}, fmt.Errorf("inspect schema %q: %w", name, err) + } + result.Schema.Tables = append(result.Schema.Tables, schema.Tables...) + result.Schema.Definitions = append(result.Schema.Definitions, schema.Definitions...) + for owner, refs := range c.dependencies { + dependencies[migrate.Name{Schema: name, Name: owner}] = refs + } + } + for i := range result.Schema.Definitions { + d := &result.Schema.Definitions[i] + if !seen[d.Name.Schema] { + return Result{}, fmt.Errorf("definition %v is outside the inspected schema scope", d.Name) + } + d.Dependencies = dependencies[d.Name] + } + if _, err := planner.Plan(migrate.Request{Current: result.Schema, Desired: result.Schema}); err != nil { + return Result{}, fmt.Errorf("migration snapshot validation: %w", err) + } + return result, nil +} + +func read(ctx context.Context, db Querier, name string) (catalog, error) { + c := catalog{schema: name, collations: map[[2]string]string{}, dependencies: map[string][]migrate.Name{}} + q, scope := mssqlcatalog.FromDB(db), mssqlcatalog.Scope{Schema: name} + found, err := mssqlcatalog.Schemas(ctx, q, []string{name}) + if err != nil { + return c, err + } + if len(found) != 1 { + return c, fmt.Errorf("schema %q was not found", name) + } + steps := []func() error{ + func() error { var err error; c.tables, err = mssqlcatalog.Tables(ctx, q, scope, ""); return err }, + func() error { var err error; c.columns, err = mssqlcatalog.Columns(ctx, q, scope); return err }, + func() error { var err error; c.indexes, err = mssqlcatalog.Indexes(ctx, q, scope); return err }, + func() error { var err error; c.parts, err = mssqlcatalog.IndexColumns(ctx, q, scope); return err }, + func() error { var err error; c.keys, err = mssqlcatalog.ForeignKeys(ctx, q, scope); return err }, + func() error { var err error; c.checks, err = mssqlcatalog.CheckConstraints(ctx, q, scope); return err }, + func() error { var err error; c.routines, err = mssqlcatalog.Routines(ctx, q, scope); return err }, + func() error { return readCollations(ctx, db, &c) }, + func() error { return readTriggers(ctx, db, &c) }, + func() error { return readDependencies(ctx, db, &c) }, + func() error { return refuseUnsupported(ctx, db, name) }, + } + for _, step := range steps { + if err := step(); err != nil { + return c, err + } + } + return c, nil +} diff --git a/database/inspect/snapshot/snapshot_test.go b/database/inspect/snapshot/snapshot_test.go new file mode 100644 index 00000000..c51ffd75 --- /dev/null +++ b/database/inspect/snapshot/snapshot_test.go @@ -0,0 +1,56 @@ +package snapshot + +import ( + "testing" + + "github.com/flanksource/sqlglot-go/database/inspect/catalog/sqlserver" + "github.com/flanksource/sqlglot-go/migrate" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestSnapshot(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "SQL Server migration snapshots") +} + +func fixture() catalog { + return catalog{schema: "dbo", tables: []mssqlcatalog.Table{{Schema: "dbo", Name: "orders", Type: "BASE TABLE"}}, + columns: []mssqlcatalog.Column{ + {Table: "orders", Name: "id", DataType: "int", MaxLength: new(4), Precision: new(10), Scale: new(0), Identity: true, IdentitySeed: new(int64(100)), IdentityIncrement: new(int64(5))}, + {Table: "orders", Name: "label", DataType: "nvarchar", MaxLength: new(40), Precision: new(0), Scale: new(0), Default: new("(N'pending')"), DefaultConstraint: new("DF_label")}, + }, indexes: []mssqlcatalog.Index{{Table: "orders", Name: "IX_label", TypeDesc: "NONCLUSTERED", Filter: "[label] IS NOT NULL"}}, + parts: []mssqlcatalog.IndexColumn{{Table: "orders", Index: "IX_label", Column: "label", KeyOrdinal: 1, Descending: true}, {Table: "orders", Index: "IX_label", Column: "id", Included: true}}, + collations: map[[2]string]string{{"orders", "label"}: "Latin1_General_CI_AS"}, + } +} + +var _ = Describe("Complete catalogue projection", func() { + It("preserves identity, named defaults, collation and index parts", func() { + schema, err := build(fixture()) + Expect(err).NotTo(HaveOccurred()) + Expect(schema.Tables).To(Equal([]migrate.Table{{Name: migrate.Name{Schema: "dbo", Name: "orders"}, + Columns: []migrate.Column{{Name: "id", Type: "int", Identity: &migrate.Identity{Seed: 100, Increment: 5}}, {Name: "label", Type: "nvarchar(20)", Default: &migrate.Default{Name: "DF_label", Expression: "(N'pending')"}, Collation: "Latin1_General_CI_AS"}}, + Indexes: []migrate.Index{{Name: "IX_label", Columns: []migrate.IndexColumn{{Name: "label", Descending: true}}, Included: []string{"id"}, Filter: "[label] IS NOT NULL"}}, + }})) + }) + + DescribeTable("refuses definitions that cannot be reproduced", + func(change func(*catalog), message string) { + c := fixture() + change(&c) + _, err := build(c) + Expect(err).To(MatchError(ContainSubstring(message))) + }, + Entry("computed column", func(c *catalog) { c.columns[0].Computed = true }, "computed"), + Entry("user-defined type", func(c *catalog) { c.columns[0].UserDefinedType = true }, "user-defined"), + Entry("missing identity seed", func(c *catalog) { c.columns[0].IdentitySeed = nil }, "identity"), + Entry("unnamed default", func(c *catalog) { c.columns[1].DefaultConstraint = nil }, "constraint name"), + Entry("disabled check", func(c *catalog) { + c.checks = []mssqlcatalog.CheckConstraint{{Table: "orders", Name: "CK_id", Definition: "id > 0", Disabled: true}} + }, "disabled check"), + Entry("encrypted routine", func(c *catalog) { + c.routines = []mssqlcatalog.Routine{{Schema: "dbo", Name: "update_orders", Type: mssqlcatalog.Procedure}} + }, "definition"), + ) +}) diff --git a/database/inspect/snapshot/transaction_test.go b/database/inspect/snapshot/transaction_test.go new file mode 100644 index 00000000..046e9a64 --- /dev/null +++ b/database/inspect/snapshot/transaction_test.go @@ -0,0 +1,31 @@ +package snapshot + +import ( + "github.com/DATA-DOG/go-sqlmock" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Transaction-bound migration snapshots", func() { + It("uses the supplied transaction for permission and catalog reads", func() { + db, mock, err := sqlmock.New() + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { mock.ExpectClose(); Expect(db.Close()).To(Succeed()) }) + mock.ExpectBegin() + tx, err := db.BeginTx(GinkgoT().Context(), nil) + Expect(err).NotTo(HaveOccurred()) + mock.ExpectQuery("HAS_PERMS_BY_NAME").WillReturnRows(sqlmock.NewRows([]string{"allowed"}).AddRow(1)) + mock.ExpectQuery("INFORMATION_SCHEMA.SCHEMATA").WithArgs("dbo").WillReturnRows(sqlmock.NewRows([]string{"schema"}).AddRow("dbo")) + mock.ExpectQuery(`OBJECT_DEFINITION\(o.object_id\)`).WithArgs("dbo", "").WillReturnRows(sqlmock.NewRows([]string{"empty"})) + for _, query := range []string{`FROM INFORMATION_SCHEMA.COLUMNS`, `FROM sys.indexes`, `FROM sys.index_columns`, `FROM sys.foreign_keys`, `FROM sys.check_constraints`, `FROM sys.objects o`, `FROM sys.columns c`, `SCHEMA_NAME\(trigger_object.schema_id\)`, `FROM sys.sql_expression_dependencies`, `SELECT object_name, feature`} { + mock.ExpectQuery(query).WithArgs("dbo").WillReturnRows(sqlmock.NewRows([]string{"empty"})) + } + result, err := Inspect(GinkgoT().Context(), tx, Options{Schemas: []string{"dbo"}}) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Schemas).To(Equal([]string{"dbo"})) + Expect(result.Schema.Tables).To(BeEmpty()) + mock.ExpectRollback() + Expect(tx.Rollback()).To(Succeed()) + Expect(mock.ExpectationsWereMet()).To(Succeed()) + }) +}) diff --git a/database/inspect/sqlite.go b/database/inspect/sqlite.go new file mode 100644 index 00000000..3c995fd2 --- /dev/null +++ b/database/inspect/sqlite.go @@ -0,0 +1,349 @@ +package sqlinspect + +import ( + "context" + "database/sql" + "errors" + "fmt" + "strings" +) + +type sqliteEnrichment struct { + ctx context.Context + db *sql.DB + catalog *Catalog + limits Limits + columnsLeft int + metadataLeft int +} + +func enrichSQLite(ctx context.Context, db *sql.DB, catalog *Catalog, limits Limits) error { + limits = limits.withDefaults() + inspector := sqliteEnrichment{ctx: ctx, db: db, catalog: catalog, limits: limits, columnsLeft: limits.MaxColumns, metadataLeft: limits.MaxMetadata} + for _, schema := range catalog.Schemas { + for _, relation := range schema.Relations { + inspector.columnsLeft -= len(relation.Columns) + } + } + for schemaIndex := range catalog.Schemas { + for relationIndex := range catalog.Schemas[schemaIndex].Relations { + relation := &catalog.Schemas[schemaIndex].Relations[relationIndex] + if err := inspector.relation(relation); err != nil { + return fmt.Errorf("inspect SQLite relation %q: %w", relation.Name, err) + } + } + } + return nil +} + +func (inspector *sqliteEnrichment) truncate(reason string) { + if !inspector.catalog.Truncated { + inspector.catalog.Truncated = true + inspector.catalog.TruncateReason = reason + } +} + +func (inspector *sqliteEnrichment) takeMetadata() bool { + if inspector.metadataLeft == 0 { + inspector.truncate("SQLite metadata limit reached") + return false + } + inspector.metadataLeft-- + return true +} + +func (inspector *sqliteEnrichment) relation(relation *Relation) error { + var definition sql.NullString + if err := inspector.db.QueryRowContext(inspector.ctx, "SELECT sql FROM sqlite_master WHERE name = ? AND type IN ('table', 'view')", relation.Name).Scan(&definition); err != nil { + return err + } + if relation.Type == "view" && definition.Valid { + if len(definition.String) > inspector.limits.MaxDefinitionBytes { + inspector.truncate("SQLite definition limit reached") + } else { + relation.ViewDef = definition.String + } + } + if err := inspector.columns(relation); err != nil { + return err + } + if err := inspector.indexes(relation); err != nil { + return err + } + if err := inspector.foreignKeys(relation); err != nil { + return err + } + return inspector.triggers(relation) +} + +func (inspector *sqliteEnrichment) columns(relation *Relation) (returnErr error) { + var withoutRowID, strict bool + if err := inspector.db.QueryRowContext(inspector.ctx, `SELECT wr, strict FROM pragma_table_list WHERE schema = 'main' AND name = ?`, relation.Name).Scan(&withoutRowID, &strict); err != nil { + return err + } + rows, err := inspector.db.QueryContext(inspector.ctx, `SELECT name, type, "notnull", dflt_value, pk, cid, hidden FROM pragma_table_xinfo(?) ORDER BY cid`, relation.Name) + if err != nil { + return err + } + defer func() { returnErr = errors.Join(returnErr, rows.Close()) }() + type columnInfo struct { + column Column + notNull, pk, hidden int + } + var found []columnInfo + pkCount := 0 + for rows.Next() { + var entry columnInfo + var cid int + var value sql.NullString + if err := rows.Scan(&entry.column.Name, &entry.column.DataType, &entry.notNull, &value, &entry.pk, &cid, &entry.hidden); err != nil { + return err + } + entry.column.Ordinal = cid + 1 + entry.column.PrimaryKey = entry.pk > 0 + if value.Valid { + entry.column.Default = &value.String + } + if entry.pk > 0 { + pkCount++ + } + found = append(found, entry) + } + if err := rows.Err(); err != nil { + return err + } + retained := make(map[string]int, len(relation.Columns)) + for index, column := range relation.Columns { + retained[column.Name] = index + } + for _, entry := range found { + nonNullable := entry.notNull != 0 || entry.pk > 0 && (withoutRowID || strict || pkCount == 1 && strings.EqualFold(entry.column.DataType, "INTEGER")) + nullable := !nonNullable + entry.column.Nullable = &nullable + if index, ok := retained[entry.column.Name]; ok { + relation.Columns[index] = entry.column + } else if entry.hidden != 0 { + if inspector.columnsLeft == 0 { + inspector.truncate("column limit reached") + continue + } + relation.Columns = append(relation.Columns, entry.column) + inspector.columnsLeft-- + } + } + return nil +} + +func (inspector *sqliteEnrichment) indexes(relation *Relation) error { + rows, err := inspector.db.QueryContext(inspector.ctx, `SELECT name, "unique", origin, partial FROM pragma_index_list(?) ORDER BY seq`, relation.Name) + if err != nil { + return err + } + type indexInfo struct { + name, origin string + unique, partial bool + } + var found []indexInfo + for rows.Next() { + var entry indexInfo + if err := rows.Scan(&entry.name, &entry.unique, &entry.origin, &entry.partial); err != nil { + return errors.Join(err, rows.Close()) + } + found = append(found, entry) + } + if err := rows.Err(); err != nil { + return errors.Join(err, rows.Close()) + } + if err := rows.Close(); err != nil { + return err + } + for _, entry := range found { + if !inspector.takeMetadata() { + continue + } + columns, err := sqliteIndexColumns(inspector.ctx, inspector.db, entry.name) + if err != nil { + return err + } + index := Index{Name: entry.name, Unique: entry.unique, Primary: entry.origin == "pk", Columns: columns, Type: "btree"} + if entry.partial { + var definition string + if err := inspector.db.QueryRowContext(inspector.ctx, `SELECT sql FROM sqlite_master WHERE name = ? AND type = 'index'`, entry.name).Scan(&definition); err != nil { + return err + } + index.Filter = sqliteIndexFilter(definition) + if index.Filter == "" { + return fmt.Errorf("partial SQLite index %q has no WHERE predicate", entry.name) + } + } + relation.Indexes = append(relation.Indexes, index) + if entry.unique && !entry.partial && len(columns) == 1 { + for i := range relation.Columns { + if relation.Columns[i].Name == columns[0] { + relation.Columns[i].Unique = true + } + } + } + } + return nil +} + +func sqliteIndexFilter(definition string) string { + quoted, depth := byte(0), 0 + for index := 0; index < len(definition); index++ { + current := definition[index] + if quoted != 0 { + if current == quoted { + if index+1 < len(definition) && definition[index+1] == quoted { + index++ + } else { + quoted = 0 + } + } + continue + } + switch current { + case '\'', '"', '`': + quoted = current + case '(': + depth++ + case ')': + depth-- + default: + if depth == 0 && strings.EqualFold(definition[index:min(index+5, len(definition))], "WHERE") { + return strings.TrimSpace(definition[index+5:]) + } + } + } + return "" +} + +func sqliteIndexColumns(ctx context.Context, db *sql.DB, name string) (_ []string, returnErr error) { + rows, err := db.QueryContext(ctx, `SELECT name FROM pragma_index_info(?) ORDER BY seqno`, name) + if err != nil { + return nil, err + } + defer func() { returnErr = errors.Join(returnErr, rows.Close()) }() + var columns []string + for rows.Next() { + var column sql.NullString + if err := rows.Scan(&column); err != nil { + return nil, err + } + if column.Valid { + columns = append(columns, column.String) + } + } + return columns, rows.Err() +} + +type sqliteForeignKeyRow struct { + id, seq int + target, from string + to sql.NullString +} + +func (inspector *sqliteEnrichment) foreignKeys(relation *Relation) error { + rows, err := inspector.db.QueryContext(inspector.ctx, `SELECT id, seq, "table", "from", "to" FROM pragma_foreign_key_list(?) ORDER BY id, seq`, relation.Name) + if err != nil { + return err + } + var found []sqliteForeignKeyRow + for rows.Next() { + var entry sqliteForeignKeyRow + if err := rows.Scan(&entry.id, &entry.seq, &entry.target, &entry.from, &entry.to); err != nil { + return errors.Join(err, rows.Close()) + } + found = append(found, entry) + } + if err := rows.Err(); err != nil { + return errors.Join(err, rows.Close()) + } + if err := rows.Close(); err != nil { + return err + } + primaryColumns := map[string][]string{} + for offset := 0; offset < len(found); { + end := offset + 1 + for end < len(found) && found[end].id == found[offset].id { + end++ + } + if inspector.takeMetadata() { + key := ForeignKey{Name: fmt.Sprintf("fk_%s_%d", relation.Name, found[offset].id), ReferencedTable: found[offset].target} + for _, entry := range found[offset:end] { + key.Columns = append(key.Columns, entry.from) + to := entry.to.String + if !entry.to.Valid { + if _, ok := primaryColumns[entry.target]; !ok { + primaryColumns[entry.target], err = sqlitePrimaryColumns(inspector.ctx, inspector.db, entry.target) + if err != nil { + return err + } + } + if entry.seq >= len(primaryColumns[entry.target]) { + return fmt.Errorf("referenced table %q has no primary-key column at position %d", entry.target, entry.seq) + } + to = primaryColumns[entry.target][entry.seq] + } + key.ReferencedColumns = append(key.ReferencedColumns, to) + } + relation.ForeignKeys = append(relation.ForeignKeys, key) + } + offset = end + } + return nil +} + +func sqlitePrimaryColumns(ctx context.Context, db *sql.DB, table string) (_ []string, returnErr error) { + rows, err := db.QueryContext(ctx, `SELECT name FROM pragma_table_info(?) WHERE pk > 0 ORDER BY pk`, table) + if err != nil { + return nil, err + } + defer func() { returnErr = errors.Join(returnErr, rows.Close()) }() + var columns []string + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + return nil, err + } + columns = append(columns, name) + } + return columns, rows.Err() +} + +func (inspector *sqliteEnrichment) triggers(relation *Relation) (returnErr error) { + rows, err := inspector.db.QueryContext(inspector.ctx, `SELECT name, sql FROM sqlite_master WHERE type = 'trigger' AND tbl_name = ? ORDER BY name`, relation.Name) + if err != nil { + return err + } + defer func() { returnErr = errors.Join(returnErr, rows.Close()) }() + for rows.Next() { + var name, statement string + if err := rows.Scan(&name, &statement); err != nil { + return err + } + if !inspector.takeMetadata() { + continue + } + if len(statement) > inspector.limits.MaxDefinitionBytes { + inspector.truncate("SQLite definition limit reached") + continue + } + upper := strings.ToUpper(statement) + trigger := Trigger{Name: name, SQL: statement} + for _, timing := range []string{"BEFORE", "AFTER", "INSTEAD OF"} { + if strings.Contains(upper, timing) { + trigger.Timing = timing + break + } + } + for _, event := range []string{"INSERT", "UPDATE", "DELETE"} { + if strings.Contains(upper, event) { + trigger.Event = event + break + } + } + relation.Triggers = append(relation.Triggers, trigger) + } + return rows.Err() +} diff --git a/database/inspect/suite_test.go b/database/inspect/suite_test.go new file mode 100644 index 00000000..daf2737f --- /dev/null +++ b/database/inspect/suite_test.go @@ -0,0 +1,9 @@ +package sqlinspect + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "testing" +) + +func TestInspection(t *testing.T) { RegisterFailHandler(Fail); RunSpecs(t, "Database inspection") } diff --git a/database/migrate/README.md b/database/migrate/README.md new file mode 100644 index 00000000..7327660b --- /dev/null +++ b/database/migrate/README.md @@ -0,0 +1,328 @@ +# HCL migrations + +This package owns the HCL and script migration engine extracted from commons-db. See [standalone database documentation](../README.md) for the unified CLI, read-only bundle preview and reviewed PostgreSQL/SQLite schema artifacts. commons-db retains compatible API shims. + +`migrate.Apply` is the single filesystem-backed migration seam for PostgreSQL, SQLite and SQL Server. The connection string selects the target, while callers pass the same embedded Atlas HCL bundle and options. SQL Server bundles are SQL Server-shaped rather than portable; see [SQL Server](#sql-server). + +```go +err := migrate.Apply(ctx, dsn, schemaFS, + migrate.WithDir("migrations"), + migrate.WithName("runtime"), +) +``` + +## Target selection + +PostgreSQL accepts `postgres://`, `postgresql://`, and keyword connection strings such as `host=localhost dbname=runtime sslmode=disable`. + +SQLite accepts `sqlite://` URLs and plain file paths ending in `.db`: + +```go +err := migrate.Apply(ctx, "sqlite://state/runtime.sqlite", schemaFS, migrate.WithDir("migrations")) +err := migrate.Apply(ctx, "/var/lib/example/runtime.db", schemaFS, migrate.WithDir("migrations")) +``` + +SQL Server accepts `sqlserver://` URLs in the go-mssqldb form, e.g. `sqlserver://user:password@host:1433?database=app`. `mssql://` is rejected so that one spelling reaches the migrator. ADO connection strings (`;`-separated pairs that name the server with `server`, `data source`, `address`, `addr` or `network address`, e.g. `server=db;user id=app;password=secret;database=acme`) and `odbc:` connection strings are SQL Server too, and are passed to go-mssqldb unchanged. + +### Existing connections + +`WithDB` runs a SQL Server or SQLite migration on a `*sql.DB` the caller already holds, with the connection string left empty. The dialect comes from the handle's driver. Apply neither closes the handle nor changes its pool settings: + +```go +err := migrate.Apply(ctx, "", bundle, + migrate.WithDB(writeDB), + migrate.WithRoutines("routines"), + migrate.WithName("acme/routines")) +``` + +Passing both a connection string and `WithDB` is an error. A PostgreSQL handle is refused, because PostgreSQL migrations scope `search_path` and `lock_timeout` on connections they open themselves. + +Relative SQLite paths are resolved from the process working directory. SQLite is always file-backed; `:memory:` and `mode=memory` are rejected. The normalized connection enables foreign keys, a 5-second busy timeout, and WAL while preserving caller-supplied query parameters and unmanaged pragmas. + +Unknown URI schemes written as `scheme://...` fail instead of being passed to PostgreSQL accidentally. Scheme-like strings without `://` are treated as PostgreSQL DSNs unless they are SQL Server ADO/ODBC strings or their path ends in `.db`; use one of the explicit forms above rather than relying on that ambiguity. A PostgreSQL keyword DSN remains PostgreSQL even when its `dbname` ends in `.db`. + +`db.NewGorm` uses the same DSN resolver, so migration and application connections cannot disagree about the target. + +## Portable canonical HCL + +A cross-database bundle declares PostgreSQL-shaped Atlas HCL. PostgreSQL evaluates the declaration directly. SQLite evaluates the same declaration with the PostgreSQL evaluator, validates the resulting realm against the compatibility contract below, and then projects it to SQLite. Acceptance by the HCL parser alone does not mean an object is SQLite-compatible. + +All `.hcl` files below `WithDir` are loaded recursively in lexical path order and evaluated as one realm. `variable` blocks are supported on both targets when callers provide values through `WithVariables`; the resulting schema must still fit the portable set. + +### Supported cross-database blocks + +| HCL block | Supported portable form | SQLite behavior | +| --- | --- | --- | +| `variable` | Atlas variable declarations referenced by schema expressions | Evaluated before projection using `WithVariables` values | +| `schema` | Exactly one `schema "public" {}` | Projected to SQLite's `main` schema | +| `table` | One or more tables assigned to `schema.public` | Created with their declared portable columns and constraints | +| `column` | `type`, `null`, and a literal or `sql(...)` default | Type is mapped as documented below; default SQL is preserved verbatim | +| `primary_key` | Column references only | Preserved when the table is first created or safely rebuilt | +| `unique` | Named constraint over column references | Projected to a named unique index | +| `index` | Column references, optional `unique`, and optional `where`; default or explicit B-tree method only | Projected to a SQLite index; `where` is preserved verbatim | +| `foreign_key` | Local and referenced columns plus `on_update` and `on_delete` actions | Preserved when the table is first created or safely rebuilt | +| `check` | Named check with `expr` and no dialect attributes | Preserved verbatim when the table is first created or safely rebuilt | + +Index expressions, per-column index attributes such as sort direction or operator classes, foreign-key attributes, and table or check attributes outside the forms above are not in the portable set. + +### Supported portable column types + +The following table lists the PostgreSQL type families accepted after HCL evaluation, not every type accepted by Atlas's PostgreSQL parser. Underscored names such as `character_varying` and `double_precision` are direct HCL aliases. Space-containing SQL spellings and the less common aliases noted below are accepted through `sql("...")` and normalize to the same family. + +| PostgreSQL HCL type | SQLite declaration | Compatibility notes | +| --- | --- | --- | +| `uuid` | `text` | No SQLite UUID-format constraint is added | +| `text`, `char`, `character`, `varchar`, `character_varying`, `bpchar`, `name`; SQL spelling `character varying` | `text` | Length modifiers and PostgreSQL's internal `name` limit are not preserved | +| `json`, `jsonb` | `text` plus `json_valid(column)` | The generated check is named `__json` | +| `date`, `time`, `timetz`, `timestamp`, `timestamptz`; SQL spellings `time with time zone`, `time without time zone`, `timestamp with time zone`, `timestamp without time zone` | `datetime` | Precision and timezone enforcement are not preserved by SQLite | +| `int2`, `int4`, `int8`, `int`, `integer`, `smallint`, `bigint`, `xid`, `xid8` | `integer` | PostgreSQL width and transaction-ID distinctions collapse to SQLite integer affinity | +| `bool`, `boolean` | `bool` | SQLite gives the declared `bool` type NUMERIC affinity; it has no native boolean type | +| `bytea` | `blob` | Binary values remain binary | +| `numeric`, `decimal` | `numeric` | Precision and scale modifiers are not preserved | +| `real`, `float`, `float4`, `float8`, `double_precision` | `real` | Precision modifiers and PostgreSQL width distinctions are not preserved | + +Use only type modifiers whose loss is acceptable to the application, or enforce the invariant with a portable named `check`. Evaluated type families outside this table fail projection; they are not coerced to text. + +### Portable example + +```hcl +schema "public" {} + +table "projects" { + schema = schema.public + + column "id" { + type = uuid + null = false + } + column "payload" { + type = jsonb + null = false + default = sql("'{}'") + } + column "created_at" { + type = timestamptz + null = false + default = sql("CURRENT_TIMESTAMP") + } + + primary_key { + columns = [column.id] + } +} + +table "events" { + schema = schema.public + + column "id" { + type = uuid + null = false + } + column "project_id" { + type = uuid + null = false + } + column "event_key" { + type = text + null = false + } + column "active" { + type = bool + null = false + default = false + } + + primary_key { + columns = [column.id] + } + foreign_key "events_project_id_fkey" { + columns = [column.project_id] + ref_columns = [table.projects.column.id] + on_update = NO_ACTION + on_delete = CASCADE + } + index "events_active_key" { + unique = true + columns = [column.project_id, column.event_key] + where = "active" + } + check "events_event_key_nonempty" { + expr = "length(event_key) > 0" + } +} +``` + +Defaults, checks, and partial-index predicates are SQL strings, not a portable expression language. `commons-db` does not translate them. Every expression in a cross-database bundle must be accepted with the intended meaning by both PostgreSQL and SQLite. `CURRENT_TIMESTAMP` and `sql("'{}'")` are portable examples; PostgreSQL-only casts such as `::jsonb`, functions such as `now()`, operators, and JSON expressions must not be assumed portable. + +### SQLite rejection list + +SQLite rejects the following before schema mutation: + +| Incompatible declaration or option | Reason | +| --- | --- | +| No table declarations | The SQLite reconciler requires at least one declared table | +| More than one schema, or a schema other than `public` | A portable bundle maps exactly one canonical schema to SQLite `main` | +| Realm or schema attributes and objects, including enums, domains, extensions, sequences, views, functions, and procedures | There is no defined lossless projection | +| Table triggers | Trigger SQL is dialect-specific | +| Column types outside the supported type table | The projector never guesses an affinity | +| Column attributes such as generated or identity behavior | PostgreSQL attributes cannot be preserved by the current projection | +| Table attributes other than plain named checks | Storage parameters, partitioning, and other PostgreSQL metadata are not portable | +| Check attributes | Only the check name and expression are portable | +| Non-B-tree index methods | SQLite cannot preserve PostgreSQL access-method behavior | +| Expression indexes or index-part attributes | Only direct column references are portable | +| Foreign-key attributes beyond columns, references, and actions | PostgreSQL-only metadata would be lost | +| A user check named `
__json` with a different expression | That name is reserved for the generated `json_valid` check | +| Any `.sql` migration file | SQL phases are PostgreSQL-only; SQLite does not skip them silently | +| `role` or `permission` blocks | Managed security is PostgreSQL-only | +| `WithExclude(...)` | Partial inspection could hide SQLite drift | +| `WithSchema(...)` with a value other than `public` | SQLite has no PostgreSQL schema namespace | + +These declarations are parser-valid PostgreSQL HCL but are not SQLite-compatible: bit strings, network and MAC types, spatial types, intervals, serial and identity types, XML, money, arrays, ranges and multiranges, full-text search types, enums, domains, and other user-defined types. + +### Accepted declarations with different SQLite semantics + +Some declarations are accepted because their storage representation is useful on both targets, but SQLite cannot enforce every PostgreSQL semantic: + +- UUIDs are stored as text without a canonical-format check. +- String length, numeric precision and scale, floating-point width, and time precision are normalized to SQLite affinities. +- `json` and `jsonb` both become validated JSON text; PostgreSQL JSONB indexing and operators are not provided. +- Date and time families become `datetime`; SQLite does not enforce PostgreSQL timezone or temporal-type distinctions. +- Integer widths collapse to SQLite integer affinity. +- Boolean declarations have NUMERIC affinity and do not restrict stored values to `0` or `1`; add a portable check when that invariant matters. +- Defaults, checks, foreign-key actions, and partial-index predicates are carried as written. Syntax or semantics that differ between engines fail at apply time or behave according to the target engine. + +### SQLite migration-time limits + +The SQLite apply runs in one transaction. It reconciles supported indexes, constraints, column types, defaults and nullability, rebuilding populated tables when needed. Column and table drops are skipped by default; valid changes in the same migration still apply. Omitted columns remain in the effective desired table definition so a rebuild preserves their values. PostgreSQL also skips omitted columns and tables by default. + +`WithDrops()` opts into dropping columns and tables. On SQLite it inspects all user tables and treats the supplied HCL as the complete desired table schema: user tables omitted from the bundle are removed. Without this option, inspection stays limited to the declared tables. Index and constraint removal do not require `WithDrops()`. A rename represented by Atlas as a column removal and addition preserves the old column by default; this API does not infer rename mappings. + +The transaction disables foreign-key enforcement while applying changes, verifies that no new foreign-key violations appear before commit, and restores enforcement afterward. An incompatible value or constraint rolls the transaction back. A rebuild that would discard an unmanaged trigger is refused. SQLite's native column-addition rules still apply, and the migrator does not backfill values. Views remain outside the table bundle; callers must assess views that depend on changed columns. Rebuilds no longer require the removed `WithRebuilds()` option; remove it from existing callers. + +## PostgreSQL schema-scoped bundles + +`WithSchema` applies one bundle inside a normalized PostgreSQL schema and leaves the default behavior on `public` unchanged: + +```go +err := migrate.Apply(ctx, dsn, schemaFS, + migrate.WithName("runtime"), + migrate.WithSchema("agent_tenant_context"), +) +``` + +The schema name must be a lowercase PostgreSQL identifier no longer than 63 bytes. The migration creates the schema and scopes Atlas inspection, HCL objects, SQL migration metadata, dependent-view handling, and managed security to it. A reusable HCL bundle still declares only `schema "public" {}`; migration remaps that declaration to the selected schema. + +SQL files run with the selected schema as `search_path`. Keep objects owned by the bundle unqualified so the same bundle can target different schemas. Explicitly qualified SQL remains explicit and is not rewritten. + +## PostgreSQL SQL phases and security + +SQL files default to the post-Atlas phase. Directives are read from the leading comment header: + +```sql +-- phase: pre +-- dependsOn: 001_extensions.sql +-- runs: always +-- transaction: false +``` + +- `phase` is `pre` or `post`; omitted means `post`. +- `dependsOn` uses paths relative to the migration root. Dependencies are topologically ordered and changes rerun all transitive dependents. +- Scripts are transactional by default; the SQL and its SHA-256 migration log commit atomically. +- `runs: always` re-runs a script on every apply. It is an escape hatch for reconciliation that truly must occur at every boot, not a replacement for dependency tracking. + +Views are restored automatically when Atlas reshapes a base table they depend on. Prefer small hash-gated files, such as one file per view or shared dependency, so a table reshape re-creates only affected objects and steady-state startup remains DDL-free. + +Atlas-style PostgreSQL `role` and `permission` blocks may live in the same HCL files as tables. The OSS migration layer supports roles and memberships plus schema, table, column, and sequence permissions. Managed grants are reconciled exactly within the bundle's metadata scope; unrelated roles and grants remain untouched. Different scopes must not manage the same role or grantee/object pair. + +## Reusable PostgreSQL test templates + +`migrate.NewProvisioner` adapts an HCL and SQL bundle to `dbtest.Options.Provisioner`: + +```go +handle := dbtest.ForT(t, dbtest.Options{ + Name: "query_sessions", + LogName: "query-sessions-test", + Provisioner: migrate.NewProvisioner( + querySchema, + migrate.WithDir("migrations"), + migrate.WithName("query"), + ), +}) +``` + +The provisioner fingerprints the migration name, directory, Atlas options, variables, and every HCL and SQL file. `dbtest` prepares one sealed PostgreSQL template for that fingerprint, clones an isolated database for each call, and keeps the template for later tests. Normal hash-gated SQL is already present in the clone; `runs: always` scripts and security reconciliation still run for every instance. + +This option requires database creation. `dbtest` fails before invoking the provisioner when `COMMONS_DB_CREATE=false` because that mode targets the configured database directly. + +## Programmatic SQLite declarations + +The `migrate/sqlite` subpackage remains the lower-level seam for callers that already own a `*sql.DB` or `*sql.Tx` and construct Atlas `*schema.Table` declarations in Go. Its `ReconcileTables` function uses the same additive-only policy but does not load or project HCL. + +```go +declared, err := sqlitetable.Table{Name: "events", Columns: columns, StoredAs: stored, PrimaryKey: []string{"id"}}.Declare() +if err != nil { + return err +} +if err := sqlitemigrate.ReconcileTables(ctx, tx, sqlitemigrate.ReconcileOptions{}, declared); err != nil { + return err +} +``` + +The record store uses this lower-level API to add newly declared columns to tables that may already contain rows. New filesystem-backed migrations should call the target-aware parent `migrate.Apply` instead. + +## SQL Server + +`sqlserver://` targets reconcile through the Atlas SQL Server driver in `atlas/sqlserver`, which commons-db ships because Atlas OSS has none. A bundle declares tables in SQL Server-shaped HCL and stored procedures and functions as plain `.sql` files: + +The driver delegates its differ, SQL Server types and migration planner to `github.com/flanksource/sqlglot-go/migrate/atlas/sqlserver`. Commons-db owns catalog inspection, HCL loading, connections, locks, transactions and applying the resulting SQL. Consumers that need a database-independent typed plan can use sqlglot's `migrate/sqlserver` package directly; this does not expand the HCL or inspection support described below. + +```go +//go:embed db +var bundle embed.FS + +err := migrate.Apply(ctx, "sqlserver://app:secret@db:1433?database=acme", bundle, + migrate.WithDir("db"), + migrate.WithRoutines("db/routines"), + migrate.WithName("acme/orders"), +) +``` + +### Routines come from `.sql` files + +`WithRoutines(dir)` declares one routine per `.sql` file under `dir`. Each file holds exactly one `CREATE [OR ALTER] PROCEDURE|FUNCTION` batch, optionally followed by `GO`, and is named after the routine it creates (`acme_count_orders.sql` creates `acme_count_orders`). The file is the routine's only source: there are no procedure or function HCL blocks. It runs as written in SSMS or sqlcmd, and migrate runs the same text, with `CREATE` rewritten to `CREATE OR ALTER` and every other byte kept, as its own batch. + +A routine is compared with the live `sys.sql_modules` definition through sqlglot's canonical T-SQL: comments, whitespace, keyword case, `[brackets]` and `OR ALTER` do not count as changes, while a literal or any other token does and produces one `CREATE OR ALTER`. A definition, declared or live, that sqlglot cannot parse is an error naming the routine, never a guessed change. Routine files are never hash-gated SQL scripts. + +A routine that is removed from the bundle is left in the database; migrate never drops procedures or functions. + +### Tables from HCL + +`.hcl` files under `WithDir` are optional when the bundle has routines. They declare exactly `schema "dbo" {}` and its tables: + +| Block | Supported form | +| --- | --- | +| `column` | `type`, `null`, `default` (a number, a string, or `sql("getdate()")`), and an `identity { seed, increment }` block | +| `primary_key` | `columns`, optional name, `clustered = false` for a nonclustered key (clustered by default) | +| `index` | `columns` (with `desc` parts), `unique`, `clustered = true` (nonclustered by default), `include = [column.x]`, `where = "..."` | +| `foreign_key` | `columns`, `ref_columns`, `on_delete`, `on_update` | +| `check` | `expr` | + +Types: `bit`, `tinyint`, `smallint`, `int`, `bigint`, `decimal(p,s)`, `numeric(p,s)`, `money`, `smallmoney`, `float(n)` (stored as `real` up to 24 bits), `real`, `char(n)`, `varchar(n)`, `nchar(n)`, `nvarchar(n)`, `text`, `ntext`, `binary(n)`, `varbinary(n)`, `image`, `date`, `time(p)`, `datetime`, `datetime2(p)`, `smalldatetime`, `datetimeoffset(p)`, `uniqueidentifier`, `xml`, `sql_variant` and `rowversion` (`timestamp`). A `(max)` size is written with `sql()`: `type = sql("nvarchar(max)")`. Defaults are named `DF_
_`; changing one drops the live default constraint, whatever its name, and adds the new one. String defaults on `nchar`/`nvarchar`/`ntext` columns are written as `N'...'`. + +### Scoped inspection + +An OIPA-sized database holds thousands of objects, so migrate inspects only the tables and routines the bundle names. Undeclared objects are never read, and so never changed. Inside declared tables, columns, keys, foreign keys and checks are reconciled to the declaration, but indexes the bundle does not declare are kept, so tuning indexes added by a DBA survive. An apply that finds nothing to change runs no DDL. Every apply holds a session `sp_getapplock` named after `WithName` (one-minute wait), so concurrent applies of one bundle serialize. + +### Compatibility level 100 + +Generated DDL and the catalog queries avoid syntax newer than compatibility level 100 (no `STRING_AGG`, `OPENJSON` or `DROP ... IF EXISTS`). `CREATE OR ALTER` depends on the engine version (2016 SP1+), not the database's compatibility level. + +### Refused on SQL Server + +These fail with an error instead of being ignored: + +- hash-gated pre/post `.sql` scripts outside the routine directory; +- `WithSchema` other than `dbo` and `WithDrops`; +- `role` and `permission` blocks; +- `view`, `trigger`, `sequence`, `function` and `procedure` HCL blocks, computed columns, user-defined and alias types (including `sysname`), and any unknown column, index or table attribute; +- declared tables that the database has partitioned or carries triggers on, and IDENTITY changes on an existing column. diff --git a/database/migrate/check_changes.go b/database/migrate/check_changes.go new file mode 100644 index 00000000..8aaa7a65 --- /dev/null +++ b/database/migrate/check_changes.go @@ -0,0 +1,162 @@ +package migrate + +import ( + "context" + "database/sql" + "fmt" + + "ariga.io/atlas/sql/schema" + "github.com/lib/pq" +) + +// withCheckExpressionChanges adds a ModifyCheck for every CHECK the bundle edited +// in place. Atlas's non-normalized diff pairs checks by name and never compares +// their expressions, so without this an edited constraint never reaches a +// database that already holds the old one. Both sides are compared as +// PostgreSQL renders them, so HCL written in any equivalent form does not drop +// and re-add the constraint — re-validating every row — on each apply. +func withCheckExpressionChanges(ctx context.Context, db *sql.DB, opts checkExpressionOptions) ([]schema.Change, error) { + schemaName, changes := opts.Schema, opts.Changes + currentSchema, ok := opts.Current.Schema(schemaName) + if !ok { + return changes, nil + } + desiredSchema, ok := opts.Desired.Schema(schemaName) + if !ok { + return changes, nil + } + tx, err := db.BeginTx(ctx, nil) + if err != nil { + return nil, fmt.Errorf("begin check expression normalization: %w", err) + } + // Normalization only creates scratch temp tables; nothing it does is kept. + defer tx.Rollback() //nolint:errcheck + if _, err := tx.ExecContext(ctx, "SET LOCAL lock_timeout = '"+migrationLockTimeout+"'"); err != nil { + return nil, fmt.Errorf("bound check expression normalization lock wait: %w", err) + } + for i, to := range desiredSchema.Tables { + from, ok := currentSchema.Table(to.Name) + if !ok { + continue + } + modified, err := modifiedChecks(ctx, tx, modifiedChecksOptions{ + Schema: schemaName, From: from, To: to, TempTable: fmt.Sprintf("commons_db_check_%d", i), + }) + if err != nil { + return nil, err + } + if len(modified) > 0 { + changes = appendCheckChanges(changes, to, modified) + } + } + return changes, nil +} + +type checkExpressionOptions struct { + Schema string + Current, Desired *schema.Realm + Changes []schema.Change +} + +type modifiedChecksOptions struct { + Schema string + From, To *schema.Table + TempTable string +} + +// modifiedChecks returns the same-named checks whose normalized expressions +// differ. The desired expression is normalized by adding it to an empty copy of +// the table, which resolves columns, casts and functions exactly as the real +// constraint was resolved. +func modifiedChecks(ctx context.Context, tx *sql.Tx, opts modifiedChecksOptions) ([]*schema.ModifyCheck, error) { + var candidates []*schema.ModifyCheck + for _, desired := range tableChecks(opts.To) { + for _, existing := range tableChecks(opts.From) { + if existing.Name == desired.Name && existing.Expr != desired.Expr { + candidates = append(candidates, &schema.ModifyCheck{From: existing, To: desired}) + } + } + } + if len(candidates) == 0 { + return nil, nil + } + table := pq.QuoteIdentifier(opts.Schema) + "." + pq.QuoteIdentifier(opts.To.Name) + temp := "pg_temp." + pq.QuoteIdentifier(opts.TempTable) + if _, err := tx.ExecContext(ctx, "CREATE TEMP TABLE "+pq.QuoteIdentifier(opts.TempTable)+" (LIKE "+table+")"); err != nil { + return nil, fmt.Errorf("copy %s to normalize its checks: %w", table, err) + } + var modified []*schema.ModifyCheck + for _, candidate := range candidates { + name := candidate.To.Name + if _, err := tx.ExecContext(ctx, "ALTER TABLE "+temp+" ADD CONSTRAINT "+pq.QuoteIdentifier(name)+" CHECK ("+candidate.To.Expr+")"); err != nil { + return nil, fmt.Errorf("normalize check %s on %s: %w", name, table, err) + } + var normalized, live string + if err := tx.QueryRowContext(ctx, + `SELECT pg_get_constraintdef(oid) FROM pg_constraint WHERE conrelid = $1::regclass AND conname = $2`, + temp, name).Scan(&normalized); err != nil { + return nil, fmt.Errorf("read normalized check %s on %s: %w", name, table, err) + } + if err := tx.QueryRowContext(ctx, + `SELECT pg_get_constraintdef(oid) FROM pg_constraint WHERE conrelid = $1::regclass AND conname = $2`, + table, name).Scan(&live); err != nil { + return nil, fmt.Errorf("read live check %s on %s: %w", name, table, err) + } + if normalized != live { + modified = append(modified, candidate) + } + } + return modified, nil +} + +// appendCheckChanges folds the modified checks into the table's pending +// ModifyTable, skipping any check Atlas already changes, or adds one. +func appendCheckChanges(changes []schema.Change, table *schema.Table, modified []*schema.ModifyCheck) []schema.Change { + for _, change := range changes { + alter, ok := change.(*schema.ModifyTable) + if !ok || alter.T.Name != table.Name { + continue + } + for _, check := range modified { + if !changesCheck(alter.Changes, check.To.Name) { + alter.Changes = append(alter.Changes, check) + } + } + return changes + } + alter := &schema.ModifyTable{T: table} + for _, check := range modified { + alter.Changes = append(alter.Changes, check) + } + return append(changes, alter) +} + +func changesCheck(changes []schema.Change, name string) bool { + for _, change := range changes { + switch c := change.(type) { + case *schema.ModifyCheck: + if c.To.Name == name { + return true + } + case *schema.AddCheck: + if c.C.Name == name { + return true + } + case *schema.DropCheck: + if c.C.Name == name { + return true + } + } + } + return false +} + +func tableChecks(table *schema.Table) []*schema.Check { + var checks []*schema.Check + for _, attr := range table.Attrs { + if check, ok := attr.(*schema.Check); ok { + checks = append(checks, check) + } + } + return checks +} diff --git a/database/migrate/drop_policy_test.go b/database/migrate/drop_policy_test.go new file mode 100644 index 00000000..bacea076 --- /dev/null +++ b/database/migrate/drop_policy_test.go @@ -0,0 +1,20 @@ +package migrate + +import ( + "ariga.io/atlas/sql/schema" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Shared migration drop filtering", func() { + It("skips column and table drops while retaining other changes without mutating the input", func() { + columnDrop := &schema.DropColumn{C: schema.NewStringColumn("note", "text")} + indexDrop := &schema.DropIndex{I: schema.NewIndex("old_index")} + mixed := &schema.ModifyTable{T: schema.NewTable("events"), Changes: []schema.Change{columnDrop, indexDrop}} + onlyDrop := &schema.ModifyTable{T: schema.NewTable("other"), Changes: []schema.Change{columnDrop}} + filtered := withoutDrops([]schema.Change{mixed, onlyDrop, &schema.DropTable{T: schema.NewTable("omitted")}}) + Expect(filtered).To(HaveLen(1)) + Expect(filtered[0]).To(Equal(&schema.ModifyTable{T: mixed.T, Changes: []schema.Change{indexDrop}})) + Expect(mixed.Changes).To(Equal([]schema.Change{columnDrop, indexDrop})) + }) +}) diff --git a/database/migrate/migrate.go b/database/migrate/migrate.go new file mode 100644 index 00000000..b28fd3e4 --- /dev/null +++ b/database/migrate/migrate.go @@ -0,0 +1,411 @@ +// Package migrate applies declarative Atlas HCL schemas to PostgreSQL, SQLite +// and SQL Server databases, and on SQL Server also the stored procedures and +// functions declared as .sql files (WithRoutines). +package migrate + +import ( + "context" + "database/sql" + "errors" + "fmt" + "io/fs" + "net/url" + "path" + "strings" + + atlasmigrate "ariga.io/atlas/sql/migrate" + _ "ariga.io/atlas/sql/postgres" + "ariga.io/atlas/sql/schema" + "ariga.io/atlas/sql/sqlclient" + "github.com/flanksource/commons/logger" + "github.com/flanksource/sqlglot-go/database/target" + "github.com/hashicorp/hcl/v2/hclparse" + "github.com/lib/pq" + mssql "github.com/microsoft/go-mssqldb" + "github.com/zclconf/go-cty/cty" + "modernc.org/sqlite" +) + +type options struct { + dir string + name string + schema string + schemaSet bool + routines string + exclude []string + allowDrops bool + allowRebuilds bool + input map[string]cty.Value + db *sql.DB +} + +// Option configures an HCL migration. +type Option func(*options) + +// WithDir selects the root containing colocated HCL and SQL migrations. +func WithDir(dir string) Option { + return func(o *options) { o.dir = strings.Trim(path.Clean(dir), "/") } +} + +// WithName sets the metadata scope used for SQL hashes and managed security. +// It should be stable and unique for each migration bundle sharing a database. +func WithName(name string) Option { + return func(o *options) { o.name = strings.TrimSpace(name) } +} + +// WithSchema selects the PostgreSQL schema used for the complete migration lifecycle. +// SQLite accepts only the default public schema declared by portable HCL bundles, +// and SQL Server only dbo. +func WithSchema(name string) Option { + return func(o *options) { o.schema, o.schemaSet = name, true } +} + +// WithVariables supplies values for HCL variable blocks and security expressions. +func WithVariables(input map[string]cty.Value) Option { + return func(o *options) { o.input = input } +} + +// WithExclude excludes database objects from Atlas inspection. Values use +// Atlas's schema inspection patterns (for example "table.column"). +func WithExclude(patterns ...string) Option { + return func(o *options) { o.exclude = append(o.exclude, patterns...) } +} + +// WithDrops allows HCL files to remove columns, tables and types. Drops are suppressed by +// default so a partial schema bundle cannot delete objects owned by consumers. +// +// Tables and types share one switch because suppressing only tables is worse +// than suppressing neither: the surviving table keeps a column typed by the enum +// the diff still wants to drop, and PostgreSQL rejects the DROP TYPE. +func WithDrops() Option { + return func(o *options) { o.allowDrops = true } +} + +// WithRebuilds retains source compatibility. Supported SQLite rebuilds are now automatic. +func WithRebuilds() Option { + return func(o *options) { o.allowRebuilds = true } +} + +// WithDB runs the migration on an already open SQL Server or SQLite handle, +// with Apply's connection string left empty; the dialect comes from the +// handle's driver. Apply never closes db and never changes its pool settings. +// PostgreSQL handles are refused: those migrations scope search_path and +// lock_timeout on connections they open themselves. +func WithDB(db *sql.DB) Option { + return func(o *options) { o.db = db } +} + +// Apply resolves the database target from connection, or from the WithDB +// handle, and reconciles HCL schemas. +// PostgreSQL supports the complete HCL, SQL, security, and destructive-change +// lifecycle. SQLite projects portable PostgreSQL-shaped HCL to tables, +// columns, and indexes, rebuilding tables when needed and skipping column and +// table drops unless WithDrops is enabled. +// SQL Server (sqlserver:// URLs, ADO and odbc: strings) reconciles SQL Server HCL tables and the +// stored procedures and functions declared with WithRoutines, inspecting only +// the objects the bundle names; it refuses SQL scripts, security blocks, +// drops, rebuilds and schemas other than dbo. +func Apply(ctx context.Context, connection string, schemaFS fs.FS, opts ...Option) (returnErr error) { + config := resolveOptions(opts) + if config.db != nil { + if strings.TrimSpace(connection) != "" { + return errors.New("pass a connection string or WithDB, not both") + } + return applyDB(ctx, config.db, schemaFS, config) + } + if strings.TrimSpace(connection) == "" { + return errors.New("connection string is empty") + } + target, err := dbtarget.Parse(connection) + if err != nil { + return err + } + switch target.Dialect { + case dbtarget.SQLite: + db, err := openSQLite(target.DSN) + if err != nil { + return err + } + defer func() { returnErr = errors.Join(returnErr, db.Close()) }() + return applySQLite(ctx, db, schemaFS, config) + case dbtarget.SQLServer: + db, err := sql.Open("sqlserver", target.DSN) + if err != nil { + return fmt.Errorf("open SQL Server migration database: %w", err) + } + defer func() { returnErr = errors.Join(returnErr, db.Close()) }() + return applySQLServer(ctx, db, schemaFS, config) + case dbtarget.Postgres: + return applyPostgres(ctx, target.DSN, schemaFS, config) + default: + return fmt.Errorf("unsupported migration dialect %q", target.Dialect) + } +} + +// applyDB dispatches a WithDB handle on its driver. +func applyDB(ctx context.Context, db *sql.DB, schemaFS fs.FS, config options) error { + switch driver := db.Driver().(type) { + case *mssql.Driver: + return applySQLServer(ctx, db, schemaFS, config) + case *sqlite.Driver: + return applySQLite(ctx, db, schemaFS, config) + case *pq.Driver: + return errors.New("WithDB does not support PostgreSQL: its migrations scope search_path and lock_timeout on connections they open, so pass a connection string") + default: + return fmt.Errorf("WithDB does not support database driver %T; use a SQL Server or SQLite handle", driver) + } +} + +func resolveOptions(opts []Option) options { + cfg := options{dir: ".", schema: defaultSchema} + for _, opt := range opts { + opt(&cfg) + } + if cfg.dir == "" { + cfg.dir = "." + } + if cfg.name == "" { + cfg.name = cfg.dir + if cfg.name == "." { + cfg.name = "default" + } + } + cfg.exclude = append([]string(nil), cfg.exclude...) + cfg.input = cloneVariables(cfg.input) + return cfg +} + +func cloneVariables(input map[string]cty.Value) map[string]cty.Value { + if input == nil { + return nil + } + cloned := make(map[string]cty.Value, len(input)) + for name, value := range input { + cloned[name] = value + } + return cloned +} + +func validatePostgresApply(connection string, schemaFS fs.FS, cfg options) error { + if strings.TrimSpace(connection) == "" { + return errors.New("connection string is empty") + } + if schemaFS == nil { + return errors.New("schema filesystem is nil") + } + if err := ValidateSchemaName(cfg.schema); err != nil { + return fmt.Errorf("migration schema: %w", err) + } + return refuseRoutines(cfg, "PostgreSQL") +} + +func diffPostgresSchema( + ctx context.Context, + client *sqlclient.Client, + parser *hclparse.Parser, + cfg options, +) (current, desired *schema.Realm, changes []schema.Change, err error) { + current, err = atlasmigrate.SchemaConn(client.Driver, cfg.schema, &schema.InspectOptions{Exclude: cfg.exclude}).ReadState(ctx) + if err != nil { + return nil, nil, nil, fmt.Errorf("inspect current schema: %w", err) + } + desired = &schema.Realm{} + if err := client.Eval(parser, desired, cfg.input); err != nil { + return nil, nil, nil, fmt.Errorf("evaluate HCL schemas: %w", err) + } + if err := remapDesiredSchema(desired, cfg.schema); err != nil { + return nil, nil, nil, fmt.Errorf("scope desired schema: %w", err) + } + changes, err = client.RealmDiff(current, desired) + if err != nil { + return nil, nil, nil, fmt.Errorf("compute schema diff: %w", err) + } + if !cfg.allowDrops { + changes = withoutDrops(changes) + } + return current, desired, changes, nil +} + +func applyPlannedChanges(ctx context.Context, client *sqlclient.Client, changes []schema.Change) error { + plan, err := client.PlanChanges(ctx, "", changes) + if err != nil { + return fmt.Errorf("plan %d schema changes: %w", len(changes), err) + } + log := logger.GetLogger("migrate") + for _, change := range plan.Changes { + log.Tracef("%s", change.Cmd) + } + if err := client.ApplyChanges(ctx, changes); err != nil { + return fmt.Errorf("apply %d schema changes: %w", len(changes), err) + } + log.V(1).Infof("Applied %d schema changes", len(changes)) + return nil +} + +func applyPostgres(ctx context.Context, connection string, schemaFS fs.FS, cfg options) (returnErr error) { + if err := validatePostgresApply(connection, schemaFS, cfg); err != nil { + return err + } + + scripts, err := loadScripts(schemaFS, cfg.dir) + if err != nil { + return err + } + parser, security, err := loadHCL(schemaFS, cfg.dir, cfg.input) + if err != nil { + return err + } + security = remapSecuritySchema(security, cfg.schema) + + scopedConnection, err := prepareSchemaConnection(ctx, schemaConnectionOptions{Connection: connection, Schema: cfg.schema}) + if err != nil { + return err + } + db, err := sql.Open("postgres", scopedConnection) + if err != nil { + return fmt.Errorf("open SQL migration database: %w", err) + } + defer func() { returnErr = errors.Join(returnErr, db.Close()) }() + if err := db.PingContext(ctx); err != nil { + return fmt.Errorf("connect SQL migration database: %w", err) + } + if err := ensureMetadataTables(ctx, db); err != nil { + return err + } + scope := migrationScope(cfg.schema, cfg.name) + selected, err := selectScripts(ctx, db, scope, scripts) + if err != nil { + return err + } + ordered, err := topologicalScripts(scripts, selected) + if err != nil { + return err + } + if err := runScriptPhase(ctx, db, scriptPhaseOptions{Scope: scope, Ordered: ordered, Phase: phasePre}); err != nil { + return err + } + + client, err := sqlclient.Open(ctx, connectionWithLockTimeout(scopedConnection)) + if err != nil { + return fmt.Errorf("open database: %w", err) + } + defer func() { returnErr = errors.Join(returnErr, client.Close()) }() + + current, desired, changes, err := diffPostgresSchema(ctx, client, parser, cfg) + if err != nil { + return err + } + if changes, err = withCheckExpressionChanges(ctx, db, checkExpressionOptions{ + Schema: cfg.schema, Current: current, Desired: desired, Changes: changes, + }); err != nil { + return err + } + restoreViews := noRestore + if len(changes) == 0 { + logger.GetLogger("migrate").Debugf("No schema changes detected") + } else { + invalidated, restore, err := invalidateDependentViews(ctx, db, viewInvalidationOptions{ + Scope: scope, Schema: cfg.schema, Changes: changes, Scripts: scripts, + }) + if err != nil { + return err + } + restoreViews = restore + if len(invalidated) > 0 { + if selected, err = selectScripts(ctx, db, scope, scripts); err != nil { + return err + } + if ordered, err = topologicalScripts(scripts, selected); err != nil { + return err + } + } + if err := applyPlannedChanges(ctx, client, changes); err != nil { + return err + } + } + if err := runScriptPhase(ctx, db, scriptPhaseOptions{Scope: scope, Ordered: ordered, Phase: phasePost}); err != nil { + return err + } + // Restore after the post phase: a view this scope does not own may be built + // on top of one the post phase just recreated. + if err := restoreViews(ctx); err != nil { + return err + } + if err := retryOnLockContention(ctx, "reconcile database security", func() error { + return reconcileSecurity(ctx, db, scope, security) + }); err != nil { + return fmt.Errorf("reconcile database security: %w", err) + } + return nil +} + +// withoutDrops removes every destructive change a partial schema bundle would +// otherwise emit for objects it does not declare. Object drops are filtered +// alongside table drops because the two are load-bearing together: a bundle that +// declares neither a table nor the enum typing one of its columns emits both a +// DropTable and a DropObject, and keeping only the former leaves the column +// behind to block the DROP TYPE with "other objects depend on it". Procedure +// and function drops are filtered too: a routine missing from the bundle is +// left in place. +func withoutDrops(changes []schema.Change) []schema.Change { + log := logger.GetLogger("migrate") + filtered := make([]schema.Change, 0, len(changes)) + for _, change := range changes { + switch drop := change.(type) { + case *schema.DropTable: + log.Tracef("Skipping drop table of %s", drop.T.Name) + case *schema.DropObject: + log.Tracef("Skipping drop of %s", objectDescription(drop.O)) + case *schema.DropProc: + log.Tracef("Skipping drop procedure of %s", drop.P.Name) + case *schema.DropFunc: + log.Tracef("Skipping drop function of %s", drop.F.Name) + case *schema.ModifyTable: + modified := *drop + modified.Changes = make([]schema.Change, 0, len(drop.Changes)) + for _, inner := range drop.Changes { + if column, ok := inner.(*schema.DropColumn); ok { + log.Tracef("Skipping drop column of %s.%s", drop.T.Name, column.C.Name) + continue + } + modified.Changes = append(modified.Changes, inner) + } + if len(modified.Changes) > 0 { + filtered = append(filtered, &modified) + } + default: + filtered = append(filtered, change) + } + } + return filtered +} + +// objectDescription names a dropped object for the trace log. Enums are the only +// object Atlas's PostgreSQL driver actually plans drops for (see dropObject in +// ariga.io/atlas/sql/postgres), so anything else is reported by its Go type. +func objectDescription(object schema.Object) string { + if enum, ok := object.(*schema.EnumType); ok { + return fmt.Sprintf("enum type %s", enum.T) + } + return fmt.Sprintf("%T", object) +} + +// connectionWithLockTimeout appends a libpq `options` runtime parameter so every +// connection Atlas opens bounds how long its ALTER TABLE DDL waits for a lock. A +// migration that would otherwise block indefinitely against a live reader fails +// fast (55P03) and the next apply re-plans from a fresh inspect, instead of +// camping on an ACCESS EXCLUSIVE lock and starving concurrent traffic. The value +// is returned unchanged when it is not a URL-form DSN or already sets options. +func connectionWithLockTimeout(connection string) string { + u, err := url.Parse(connection) + if err != nil || u.Scheme == "" { + return connection + } + q := u.Query() + if q.Get("options") != "" { + return connection + } + q.Set("options", "-c lock_timeout="+migrationLockTimeout) + u.RawQuery = q.Encode() + return u.String() +} diff --git a/database/migrate/migrate_test.go b/database/migrate/migrate_test.go new file mode 100644 index 00000000..fa41291c --- /dev/null +++ b/database/migrate/migrate_test.go @@ -0,0 +1,67 @@ +package migrate + +import ( + "testing" + "testing/fstest" + + "ariga.io/atlas/sql/schema" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestWithoutDrops covers the pairing that makes suppression correct: a bundle +// that declares neither a consumer's table nor the enum typing one of its +// columns emits a DropTable and a DropObject, and filtering only the table would +// leave the column behind to block the DROP TYPE. +func TestWithoutDrops(t *testing.T) { + keep := &schema.AddTable{T: schema.NewTable("profiles")} + dropTable := &schema.DropTable{T: schema.NewTable("other")} + dropEnum := &schema.DropObject{O: &schema.EnumType{ + T: "other_status", + Values: []string{"pending", "done"}, + Schema: schema.New("public"), + }} + + got := withoutDrops([]schema.Change{keep, dropTable, dropEnum}) + require.Len(t, got, 1) + assert.Same(t, keep, got[0]) +} + +func TestObjectDescription(t *testing.T) { + assert.Equal(t, "enum type other_status", + objectDescription(&schema.EnumType{T: "other_status", Schema: schema.New("public")})) +} + +func TestApplyValidatesInputsBeforeConnecting(t *testing.T) { + assert.EqualError(t, Apply(t.Context(), "", fstest.MapFS{}), "connection string is empty") + assert.EqualError(t, Apply(t.Context(), "postgres://unused", nil), "schema filesystem is nil") +} + +func TestConnectionWithLockTimeout(t *testing.T) { + tests := []struct { + name string + connection string + want string + }{ + { + name: "url with sslmode but no options gains a bounded lock_timeout", + connection: "postgres://user@localhost/app?sslmode=disable", + want: "postgres://user@localhost/app?options=-c+lock_timeout%3D" + migrationLockTimeout + "&sslmode=disable", + }, + { + name: "existing options are preserved", + connection: "postgres://user@localhost/app?options=-c+statement_timeout%3D5s", + want: "postgres://user@localhost/app?options=-c+statement_timeout%3D5s", + }, + { + name: "keyword dsn is left untouched", + connection: "host=localhost user=app dbname=app sslmode=disable", + want: "host=localhost user=app dbname=app sslmode=disable", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, connectionWithLockTimeout(tt.connection)) + }) + } +} diff --git a/database/migrate/preview.go b/database/migrate/preview.go new file mode 100644 index 00000000..5ef468e1 --- /dev/null +++ b/database/migrate/preview.go @@ -0,0 +1,103 @@ +package migrate + +import ( + "context" + "database/sql" + "fmt" + "io/fs" + "slices" + + "ariga.io/atlas/sql/postgres" + "ariga.io/atlas/sql/schema" + "ariga.io/atlas/sql/sqlite" + "github.com/flanksource/sqlglot-go/database/migrate/review" + native "github.com/flanksource/sqlglot-go/migrate" +) + +type Preview struct { + Artifact *review.Artifact `json:"artifact,omitempty"` + SQLServerArtifact *native.Artifact `json:"sqlServerArtifact,omitempty"` + SQLServerPlan *native.Plan `json:"sqlServerPlan,omitempty"` + Scripts []ScriptAnalysis `json:"scripts,omitempty"` + Warnings []string `json:"warnings,omitempty"` + Error string `json:"error,omitempty"` +} + +// PreviewBundle inspects and plans without executing scripts or creating metadata. +// Script and security bundles yield advisory schema plans until phased artifacts are supported. +func PreviewBundle(ctx context.Context, db *sql.DB, dialect string, files fs.FS, opts ...Option) (result Preview, returnErr error) { + defer func() { + if returnErr != nil { + result.Error = returnErr.Error() + } + }() + if files == nil { + return result, fmt.Errorf("schema filesystem is nil") + } + cfg := resolveOptions(opts) + scripts, err := loadScripts(files, cfg.dir, cfg.routines) + if err != nil { + return result, err + } + for _, name := range scriptPaths(scripts) { + s := scripts[name] + a := AnalyzeScript(s.content, dialect) + a.Path, a.Phase = s.path, string(s.phase) + result.Scripts = append(result.Scripts, a) + if s.phase == phasePre && (a.Kind == "ddl" || a.Kind == "mixed" || a.Kind == "unknown") { + result.Warnings = append(result.Warnings, fmt.Sprintf("pre-script %s is %s; preview uses the current database without executing it", s.path, a.Kind)) + } + } + parser, security, err := loadHCL(files, cfg.dir, cfg.input) + if err != nil { + return result, err + } + var advisory []string + if len(scripts) > 0 { + advisory = append(advisory, "SQL scripts require phased reviewed artifacts") + } + if len(security.Roles)+len(security.Permissions) > 0 { + advisory = append(advisory, "security changes require phased reviewed artifacts") + } + if dialect == "sqlserver" { + return previewSQLServer(ctx, db, files, cfg, result, advisory) + } + if cfg.routines != "" || len(cfg.exclude) > 0 { + return result, fmt.Errorf("reviewed bundle previews do not support routines or inspection exclusions") + } + realm := &schema.Realm{} + if err = postgres.EvalHCL.Eval(parser, realm, cfg.input); err != nil { + return result, fmt.Errorf("evaluate preview HCL: %w", err) + } + schemaName := cfg.schema + var desired []byte + switch dialect { + case "sqlite": + if cfg.schema != defaultSchema { + return result, fmt.Errorf("SQLite bundle schema must be public") + } + if _, err = projectSQLiteRealm(realm); err != nil { + return result, err + } + schemaName = "main" + desired, err = sqlite.MarshalHCL.MarshalSpec(realm) + case "postgres": + if err = remapDesiredSchema(realm, cfg.schema); err != nil { + return result, err + } + desired, err = postgres.MarshalHCL.MarshalSpec(realm) + default: + return result, fmt.Errorf("unsupported reviewed bundle dialect %q", dialect) + } + if err != nil { + return result, err + } + artifact, err := review.Build(ctx, db, review.Options{Dialect: dialect, Schema: schemaName, DesiredHCL: string(desired), Advisory: advisory, PreserveOmitted: !cfg.allowDrops}) + if err != nil { + result.Error = err.Error() + return result, err + } + result.Artifact = &artifact + result.Warnings = append(result.Warnings, slices.Clone(advisory)...) + return result, nil +} diff --git a/database/migrate/preview_sqlserver.go b/database/migrate/preview_sqlserver.go new file mode 100644 index 00000000..7fce9903 --- /dev/null +++ b/database/migrate/preview_sqlserver.go @@ -0,0 +1,109 @@ +package migrate + +import ( + "context" + "database/sql" + "fmt" + "io/fs" + + "github.com/flanksource/sqlglot-go/database/inspect/snapshot" + native "github.com/flanksource/sqlglot-go/migrate" + planner "github.com/flanksource/sqlglot-go/migrate/sqlserver" +) + +func previewSQLServer(ctx context.Context, db *sql.DB, files fs.FS, cfg options, result Preview, advisory []string) (Preview, error) { + if cfg.schemaSet && cfg.schema != "dbo" { + return result, fmt.Errorf("SQL Server bundle scope must be dbo") + } + if len(cfg.exclude) > 0 { + return result, fmt.Errorf("reviewed SQL Server artifacts require complete schema inspection") + } + realm, err := sqlServerTables(files, cfg) + if err != nil { + return result, err + } + if cfg.routines != "" { + if err := addRoutines(files, cfg.routines, realm.Schemas[0]); err != nil { + return result, err + } + } + current, err := snapshot.Inspect(ctx, db, snapshot.Options{Schemas: []string{"dbo"}}) + if err != nil { + return result, err + } + if err := resolveSQLServerDefaults(ctx, db, realm, current.Schema); err != nil { + return result, err + } + desired, err := planner.FromAtlas(realm) + if err != nil { + return result, err + } + if !cfg.allowDrops { + preserveSQLServerOmitted(current.Schema, &desired) + } + artifact, err := planner.BuildArtifact(native.ArtifactRequest{Schemas: current.Schemas, Request: native.Request{Current: current.Schema, Desired: desired}}) + if err != nil { + result.Error = err.Error() + return result, err + } + result.Warnings = append(result.Warnings, advisory...) + if len(advisory) > 0 { + result.SQLServerPlan = &artifact.Plan + } else { + result.SQLServerArtifact = &artifact + } + return result, nil +} + +func preserveSQLServerOmitted(current native.Schema, desired *native.Schema) { + for _, before := range current.Tables { + found := false + for i := range desired.Tables { + after := &desired.Tables[i] + if after.Name != before.Name { + continue + } + found = true + for _, column := range before.Columns { + present := false + for _, candidate := range after.Columns { + if candidate.Name == column.Name { + present = true + break + } + } + if !present { + after.Columns = append(after.Columns, column) + } + } + for _, index := range before.Indexes { + present := false + for _, candidate := range after.Indexes { + if candidate.Name == index.Name { + present = true + break + } + } + if !present { + after.Indexes = append(after.Indexes, index) + } + } + break + } + if !found { + desired.Tables = append(desired.Tables, before) + } + } + for _, definition := range current.Definitions { + present := false + for _, candidate := range desired.Definitions { + if candidate.Name == definition.Name { + present = true + break + } + } + if !present { + desired.Definitions = append(desired.Definitions, definition) + } + } +} diff --git a/database/migrate/preview_sqlserver_defaults.go b/database/migrate/preview_sqlserver_defaults.go new file mode 100644 index 00000000..17ea3ba3 --- /dev/null +++ b/database/migrate/preview_sqlserver_defaults.go @@ -0,0 +1,56 @@ +package migrate + +import ( + "context" + "database/sql" + "fmt" + + "ariga.io/atlas/sql/schema" + native "github.com/flanksource/sqlglot-go/migrate" +) + +func resolveSQLServerDefaults(ctx context.Context, db *sql.DB, desired *schema.Realm, current native.Schema) error { + var collation string + if err := db.QueryRowContext(ctx, "SELECT CONVERT(nvarchar(128), DATABASEPROPERTYEX(DB_NAME(), 'Collation'))").Scan(&collation); err != nil { + return fmt.Errorf("inspect SQL Server database collation: %w", err) + } + if collation == "" { + return fmt.Errorf("SQL Server database collation is empty") + } + for _, s := range desired.Schemas { + for _, table := range s.Tables { + resolveSQLServerTableDefaults(table, current, collation) + } + } + return nil +} + +func resolveSQLServerTableDefaults(table *schema.Table, current native.Schema, collation string) { + var before native.Table + for _, candidate := range current.Tables { + if candidate.Name == (native.Name{Schema: table.Schema.Name, Name: table.Name}) { + before = candidate + break + } + } + if table.PrimaryKey != nil && table.PrimaryKey.Name == "" { + for _, index := range before.Indexes { + if index.Primary { + table.PrimaryKey.Name = index.Name + } + } + } + for _, column := range table.Columns { + if _, ok := column.Type.Type.(*schema.StringType); !ok { + continue + } + effective := collation + for _, old := range before.Columns { + if old.Name == column.Name && old.Collation != "" { + effective = old.Collation + break + } + } + column.SetCollation(effective) + } +} diff --git a/database/migrate/preview_sqlserver_defaults_test.go b/database/migrate/preview_sqlserver_defaults_test.go new file mode 100644 index 00000000..03b0d0fb --- /dev/null +++ b/database/migrate/preview_sqlserver_defaults_test.go @@ -0,0 +1,27 @@ +package migrate + +import ( + "ariga.io/atlas/sql/schema" + native "github.com/flanksource/sqlglot-go/migrate" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("SQL Server bundle defaults", func() { + It("retains catalog key names and collations and resolves new string columns to the database collation", func() { + realm, err := sqlServerTables(integrationBundle(), resolveOptions(integrationOptions)) + Expect(err).NotTo(HaveOccurred()) + table := realm.Schemas[0].Tables[0] + table.AddColumns(schema.NewStringColumn("note", "varchar")) + current := native.Schema{Tables: []native.Table{{ + Name: native.Name{Schema: "dbo", Name: "acme_orders"}, + Columns: []native.Column{{Name: "status", Collation: "Latin1_General_100_CI_AS"}}, + Indexes: []native.Index{{Name: "catalog_primary_key", Primary: true}}, + }}} + resolveSQLServerTableDefaults(table, current, "SQL_Latin1_General_CP1_CI_AS") + Expect(table.PrimaryKey.Name).To(Equal("catalog_primary_key")) + Expect(table.Columns[0].Attrs).NotTo(ContainElement(BeAssignableToTypeOf(&schema.Collation{}))) + Expect(table.Columns[1].Attrs).To(ContainElement(&schema.Collation{V: "Latin1_General_100_CI_AS"})) + Expect(table.Columns[len(table.Columns)-1].Attrs).To(ContainElement(&schema.Collation{V: "SQL_Latin1_General_CP1_CI_AS"})) + }) +}) diff --git a/database/migrate/preview_test.go b/database/migrate/preview_test.go new file mode 100644 index 00000000..7a21a68b --- /dev/null +++ b/database/migrate/preview_test.go @@ -0,0 +1,53 @@ +package migrate + +import ( + "database/sql" + planner "github.com/flanksource/sqlglot-go/migrate/sqlserver" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "path/filepath" + "testing/fstest" +) + +var _ = Describe("migration preview", func() { + It("projects SQL Server HCL and routine bundles into the native artifact model", func() { + files := integrationBundle() + cfg := resolveOptions(integrationOptions) + realm, err := sqlServerTables(files, cfg) + Expect(err).NotTo(HaveOccurred()) + Expect(addRoutines(files, cfg.routines, realm.Schemas[0])).To(Succeed()) + projected, err := planner.FromAtlas(realm) + Expect(err).NotTo(HaveOccurred()) + Expect(projected.Tables[0].Indexes).To(ContainElement(HaveField("Name", "PK_acme_orders"))) + Expect(projected.Tables[1].Indexes).To(ContainElement(HaveField("Name", "PK_acme_order_lines"))) + Expect(projected.Tables[0].Columns[1].Default.Expression).To(Equal("N'open'")) + }) + DescribeTable("classifies parsed scripts", func(text, kind string) { + analysis := AnalyzeScript(text, "postgres") + Expect(analysis.Kind).To(Equal(kind)) + }, + Entry("DML", "INSERT INTO items VALUES (1)", "dml"), + Entry("DDL", "ALTER TABLE items ADD COLUMN note text", "ddl"), + Entry("mixed", "CREATE TABLE items (id int); INSERT INTO items VALUES (1)", "mixed"), + Entry("unknown", "this is not valid SQL", "unknown"), + ) + It("warns about pre DDL and still produces a read-only advisory preview", func(ctx SpecContext) { + db, err := sql.Open("sqlite", filepath.Join(GinkgoT().TempDir(), "preview.db")) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(db.Close) + files := fstest.MapFS{ + "schema.hcl": {Data: []byte(portableSchema)}, + "pre.sql": {Data: []byte("-- phase: pre\nCREATE TABLE script_created (id integer)")}, + } + preview, err := PreviewBundle(ctx, db, "sqlite", files) + Expect(err).NotTo(HaveOccurred()) + Expect(preview.Scripts).To(HaveLen(1)) + Expect(preview.Scripts[0].Kind).To(Equal("ddl")) + Expect(preview.Warnings).NotTo(BeEmpty()) + Expect(preview.Artifact).NotTo(BeNil()) + Expect(preview.Artifact.Advisory).NotTo(BeEmpty()) + var count int + Expect(db.QueryRow("SELECT count(*) FROM sqlite_schema WHERE type = 'table'").Scan(&count)).To(Succeed()) + Expect(count).To(BeZero()) + }) +}) diff --git a/database/migrate/provisioner.go b/database/migrate/provisioner.go new file mode 100644 index 00000000..4084f719 --- /dev/null +++ b/database/migrate/provisioner.go @@ -0,0 +1,233 @@ +package migrate + +import ( + "context" + "crypto/sha256" + "database/sql" + "encoding/binary" + "encoding/hex" + "errors" + "fmt" + "hash" + "io/fs" + "path" + "sort" + "strings" + + "github.com/zclconf/go-cty/cty" + ctyjson "github.com/zclconf/go-cty/cty/json" +) + +const provisionerFingerprintVersion = "commons-db/migrate-provisioner/v1" + +// SchemaProvisioner prepares reusable dbtest templates and reconciles each +// cloned database using the same migration behavior as Apply. +type SchemaProvisioner struct { + schemaFS fs.FS + config options +} + +// NewProvisioner adapts an HCL migration bundle to dbtest.Provisioner without +// coupling the migrate package to dbtest. Options are frozen at construction. +func NewProvisioner(schemaFS fs.FS, opts ...Option) *SchemaProvisioner { + return &SchemaProvisioner{schemaFS: schemaFS, config: resolveOptions(opts)} +} + +// Fingerprint returns a deterministic identity for all inputs that affect the +// database produced by this provisioner. +func (p *SchemaProvisioner) Fingerprint(ctx context.Context) (string, error) { + if p == nil { + return "", errors.New("migration provisioner is nil") + } + if p.schemaFS == nil { + return "", errors.New("schema filesystem is nil") + } + if err := ctx.Err(); err != nil { + return "", err + } + if err := refuseRoutines(p.config, "PostgreSQL provisioner"); err != nil { + return "", err + } + if _, err := loadScripts(p.schemaFS, p.config.dir); err != nil { + return "", err + } + if _, _, err := loadHCL(p.schemaFS, p.config.dir, p.config.input); err != nil { + return "", err + } + + digest := sha256.New() + writeFingerprintField(digest, provisionerFingerprintVersion) + writeFingerprintField(digest, p.config.dir) + writeFingerprintField(digest, p.config.name) + writeFingerprintField(digest, "schema="+p.config.schema) + writeFingerprintField(digest, fmt.Sprintf("allow-drops=%t", p.config.allowDrops)) + for _, pattern := range sortedCopy(p.config.exclude) { + writeFingerprintField(digest, "exclude="+pattern) + } + if err := writeFingerprintVariables(digest, p.config.input); err != nil { + return "", err + } + if err := p.writeMigrationFiles(ctx, digest); err != nil { + return "", err + } + return hex.EncodeToString(digest.Sum(nil)), nil +} + +// PrepareTemplate applies the full migration bundle to a new template. +func (p *SchemaProvisioner) PrepareTemplate(ctx context.Context, connection string) error { + return p.prepare(ctx, connection) +} + +// PrepareInstance reconciles instance-specific and always-run migrations after +// cloning a prepared template. +func (p *SchemaProvisioner) PrepareInstance(ctx context.Context, connection string) error { + if p == nil { + return errors.New("migration provisioner is nil") + } + if strings.TrimSpace(connection) == "" { + return errors.New("connection string is empty") + } + if p.schemaFS == nil { + return errors.New("schema filesystem is nil") + } + if err := refuseRoutines(p.config, "PostgreSQL provisioner"); err != nil { + return err + } + scripts, err := loadScripts(p.schemaFS, p.config.dir) + if err != nil { + return err + } + _, security, err := loadHCL(p.schemaFS, p.config.dir, p.config.input) + if err != nil { + return err + } + if err := ValidateSchemaName(p.config.schema); err != nil { + return fmt.Errorf("migration schema: %w", err) + } + security = remapSecuritySchema(security, p.config.schema) + scopedConnection, err := prepareSchemaConnection(ctx, schemaConnectionOptions{Connection: connection, Schema: p.config.schema}) + if err != nil { + return err + } + database, err := sql.Open("postgres", scopedConnection) + if err != nil { + return fmt.Errorf("open SQL migration database: %w", err) + } + defer database.Close() //nolint:errcheck + if err := database.PingContext(ctx); err != nil { + return fmt.Errorf("connect SQL migration database: %w", err) + } + if err := ensureMetadataTables(ctx, database); err != nil { + return err + } + scope := migrationScope(p.config.schema, p.config.name) + selected, err := selectScripts(ctx, database, scope, scripts) + if err != nil { + return err + } + ordered, err := topologicalScripts(scripts, selected) + if err != nil { + return err + } + if err := runScriptPhase(ctx, database, scriptPhaseOptions{Scope: scope, Ordered: ordered, Phase: phasePre}); err != nil { + return err + } + if err := runScriptPhase(ctx, database, scriptPhaseOptions{Scope: scope, Ordered: ordered, Phase: phasePost}); err != nil { + return err + } + if err := retryOnLockContention(ctx, "reconcile database security", func() error { + return reconcileSecurity(ctx, database, scope, security) + }); err != nil { + return fmt.Errorf("reconcile database security: %w", err) + } + return nil +} + +func (p *SchemaProvisioner) prepare(ctx context.Context, connection string) error { + if p == nil { + return errors.New("migration provisioner is nil") + } + return applyPostgres(ctx, connection, p.schemaFS, p.config) +} + +func (p *SchemaProvisioner) writeMigrationFiles(ctx context.Context, digest hash.Hash) error { + type migrationFile struct { + name string + data []byte + } + var files []migrationFile + err := fs.WalkDir(p.schemaFS, p.config.dir, func(name string, entry fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if err := ctx.Err(); err != nil { + return err + } + extension := strings.ToLower(path.Ext(name)) + if entry.IsDir() || (extension != ".hcl" && extension != ".sql") { + return nil + } + data, err := fs.ReadFile(p.schemaFS, name) + if err != nil { + return fmt.Errorf("read migration fingerprint input %s: %w", name, err) + } + relative := strings.TrimPrefix(strings.TrimPrefix(name, p.config.dir), "/") + files = append(files, migrationFile{name: relative, data: data}) + return nil + }) + if err != nil { + return fmt.Errorf("load migration fingerprint inputs: %w", err) + } + sort.Slice(files, func(i, j int) bool { return files[i].name < files[j].name }) + for _, file := range files { + writeFingerprintField(digest, "file="+file.name) + writeFingerprintBytes(digest, file.data) + } + return nil +} + +func writeFingerprintVariables(digest hash.Hash, input map[string]cty.Value) error { + names := make([]string, 0, len(input)) + for name := range input { + names = append(names, name) + } + sort.Strings(names) + for _, name := range names { + value := input[name] + if value == cty.NilVal { + return fmt.Errorf("fingerprint migration variable %q: value is nil", name) + } + if !value.IsWhollyKnown() { + return fmt.Errorf("fingerprint migration variable %q: value is not wholly known", name) + } + typeJSON, err := ctyjson.MarshalType(value.Type()) + if err != nil { + return fmt.Errorf("fingerprint migration variable %q type: %w", name, err) + } + valueJSON, err := ctyjson.Marshal(value, value.Type()) + if err != nil { + return fmt.Errorf("fingerprint migration variable %q value: %w", name, err) + } + writeFingerprintField(digest, "variable="+name) + writeFingerprintBytes(digest, typeJSON) + writeFingerprintBytes(digest, valueJSON) + } + return nil +} + +func writeFingerprintField(digest hash.Hash, value string) { + writeFingerprintBytes(digest, []byte(value)) +} + +func writeFingerprintBytes(digest hash.Hash, value []byte) { + var size [8]byte + binary.BigEndian.PutUint64(size[:], uint64(len(value))) + _, _ = digest.Write(size[:]) + _, _ = digest.Write(value) +} + +func sortedCopy(values []string) []string { + result := append([]string(nil), values...) + sort.Strings(result) + return result +} diff --git a/database/migrate/provisioner_test.go b/database/migrate/provisioner_test.go new file mode 100644 index 00000000..9b9806db --- /dev/null +++ b/database/migrate/provisioner_test.go @@ -0,0 +1,117 @@ +package migrate + +import ( + "testing/fstest" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/zclconf/go-cty/cty" +) + +var _ = Describe("migration provisioner fingerprint", func() { + baseFS := func() fstest.MapFS { + return fstest.MapFS{ + "migrations/schema.hcl": &fstest.MapFile{Data: []byte(`schema "public" {}`)}, + "migrations/seed.sql": &fstest.MapFile{Data: []byte("SELECT 1")}, + "migrations/readme.txt": &fstest.MapFile{Data: []byte("ignored")}, + } + } + + It("is deterministic across variable and option order", func(ctx SpecContext) { + first := NewProvisioner(baseFS(), + WithDir("migrations"), + WithName("query"), + WithExclude("audit.*", "internal.*"), + WithVariables(map[string]cty.Value{ + "enabled": cty.True, + "name": cty.StringVal("example"), + }), + ) + secondFS := fstest.MapFS{ + "migrations/readme.txt": &fstest.MapFile{Data: []byte("changed but ignored")}, + "migrations/seed.sql": &fstest.MapFile{Data: []byte("SELECT 1")}, + "migrations/schema.hcl": &fstest.MapFile{Data: []byte(`schema "public" {}`)}, + } + second := NewProvisioner(secondFS, + WithVariables(map[string]cty.Value{ + "name": cty.StringVal("example"), + "enabled": cty.True, + }), + WithExclude("internal.*", "audit.*"), + WithName("query"), + WithDir("migrations"), + ) + + firstFingerprint, err := first.Fingerprint(ctx) + Expect(err).NotTo(HaveOccurred()) + secondFingerprint, err := second.Fingerprint(ctx) + Expect(err).NotTo(HaveOccurred()) + Expect(firstFingerprint).To(Equal(secondFingerprint)) + }) + + DescribeTable("changes when migration behavior changes", + func(ctx SpecContext, mutate func(fstest.MapFS) *SchemaProvisioner) { + baseline, err := NewProvisioner(baseFS(), WithDir("migrations"), WithName("query")).Fingerprint(ctx) + Expect(err).NotTo(HaveOccurred()) + changed, err := mutate(baseFS()).Fingerprint(ctx) + Expect(err).NotTo(HaveOccurred()) + Expect(changed).NotTo(Equal(baseline)) + }, + Entry("HCL content", func(files fstest.MapFS) *SchemaProvisioner { + files["migrations/schema.hcl"] = &fstest.MapFile{Data: []byte("schema \"other\" {}")} + return NewProvisioner(files, WithDir("migrations"), WithName("query")) + }), + Entry("SQL content", func(files fstest.MapFS) *SchemaProvisioner { + files["migrations/seed.sql"] = &fstest.MapFile{Data: []byte("SELECT 2")} + return NewProvisioner(files, WithDir("migrations"), WithName("query")) + }), + Entry("migration path", func(files fstest.MapFS) *SchemaProvisioner { + files["migrations/renamed.sql"] = files["migrations/seed.sql"] + delete(files, "migrations/seed.sql") + return NewProvisioner(files, WithDir("migrations"), WithName("query")) + }), + Entry("migration directory", func(files fstest.MapFS) *SchemaProvisioner { + files["schema.hcl"] = files["migrations/schema.hcl"] + files["seed.sql"] = files["migrations/seed.sql"] + delete(files, "migrations/schema.hcl") + delete(files, "migrations/seed.sql") + return NewProvisioner(files, WithName("query")) + }), + Entry("scope", func(files fstest.MapFS) *SchemaProvisioner { + return NewProvisioner(files, WithDir("migrations"), WithName("other")) + }), + Entry("schema", func(files fstest.MapFS) *SchemaProvisioner { + return NewProvisioner(files, WithDir("migrations"), WithName("query"), WithSchema("agent_namespace_context")) + }), + Entry("exclude patterns", func(files fstest.MapFS) *SchemaProvisioner { + return NewProvisioner(files, WithDir("migrations"), WithName("query"), WithExclude("audit.*")) + }), + Entry("table drop policy", func(files fstest.MapFS) *SchemaProvisioner { + return NewProvisioner(files, WithDir("migrations"), WithName("query"), WithDrops()) + }), + Entry("variables", func(files fstest.MapFS) *SchemaProvisioner { + return NewProvisioner(files, WithDir("migrations"), WithName("query"), WithVariables(map[string]cty.Value{ + "enabled": cty.True, + })) + }), + ) + + It("freezes options at construction", func(ctx SpecContext) { + excludes := []string{"audit.*"} + variables := map[string]cty.Value{"name": cty.StringVal("before")} + provisioner := NewProvisioner(baseFS(), + WithDir("migrations"), + WithName("query"), + WithExclude(excludes...), + WithVariables(variables), + ) + before, err := provisioner.Fingerprint(ctx) + Expect(err).NotTo(HaveOccurred()) + + excludes[0] = "changed.*" + variables["name"] = cty.StringVal("after") + after, err := provisioner.Fingerprint(ctx) + Expect(err).NotTo(HaveOccurred()) + Expect(after).To(Equal(before)) + }) +}) diff --git a/database/migrate/review/apply.go b/database/migrate/review/apply.go new file mode 100644 index 00000000..6c4cb84f --- /dev/null +++ b/database/migrate/review/apply.go @@ -0,0 +1,126 @@ +package review + +import ( + "context" + "database/sql" + "errors" + "fmt" + "time" + + "ariga.io/atlas/sql/schema" +) + +func Apply(ctx context.Context, db *sql.DB, request Request) (_ Result, returnErr error) { + result := Result{Hash: request.Hash, Outcome: Failed} + if err := validate(request); err != nil { + return result, err + } + if err := validateScope(request.Artifact.Baseline); err != nil { + return result, err + } + if db == nil { + return result, fmt.Errorf("database is nil") + } + if request.LockTimeout == 0 { + request.LockTimeout = 5 * time.Second + } + conn, err := db.Conn(ctx) + if err != nil { + return result, err + } + defer func() { returnErr = errors.Join(returnErr, conn.Close()) }() + if request.Artifact.Baseline.Dialect == "sqlite" { + return applySQLite(ctx, conn, request) + } + return applyPostgres(ctx, conn, request) +} + +func applyPostgres(ctx context.Context, conn *sql.Conn, request Request) (result Result, err error) { + result = Result{Hash: request.Hash, Outcome: Failed} + tx, err := conn.BeginTx(ctx, nil) + if err != nil { + return result, err + } + defer func() { + if e := tx.Rollback(); e != nil && !errors.Is(e, sql.ErrTxDone) { + err = errors.Join(err, e) + } + }() + if _, err = tx.ExecContext(ctx, fmt.Sprintf("SET LOCAL lock_timeout = '%dms'", max(1, request.LockTimeout.Milliseconds()))); err != nil { + return result, err + } + if _, err = tx.ExecContext(ctx, "SELECT pg_advisory_xact_lock(7561364747649)"); err != nil { + return result, fmt.Errorf("acquire migration lock: %w", err) + } + result, err = execute(ctx, tx, request) + if err != nil { + return result, err + } + if err = tx.Commit(); err != nil { + result.Outcome = CommitUnknown + return result, fmt.Errorf("commit outcome unknown: %w", err) + } + return result, nil +} + +func execute(ctx context.Context, q schema.ExecQuerier, request Request) (Result, error) { + result := Result{Hash: request.Hash, Outcome: Failed} + a := request.Artifact + b, err := open(q, a.Baseline.Dialect) + if err != nil { + return result, err + } + baseline, err := b.decode(a.Baseline) + if err != nil { + return result, err + } + desired, err := b.decode(a.Desired) + if err != nil { + return result, err + } + current, err := inspect(ctx, q, b, a.Baseline) + if err != nil { + return result, err + } + remaining, err := b.diff(current, desired) + if err != nil { + return result, err + } + if len(remaining) == 0 { + result.Outcome = AlreadyApplied + return result, nil + } + drift, err := b.diff(current, baseline) + if err != nil { + return result, err + } + if len(drift) > 0 { + return result, fmt.Errorf("schema drift: current schema matches neither reviewed baseline nor desired state") + } + steps, err := b.plan(ctx, baseline, desired) + if err != nil { + return result, err + } + if !sameSteps(steps, a.Steps) { + return result, fmt.Errorf("saved SQL differs from the current backend planner; review a fresh plan") + } + for _, step := range a.Steps { + if _, err := q.ExecContext(ctx, step.SQL); err != nil { + return result, fmt.Errorf("execute step %s: %w", step.ID, err) + } + result.Steps++ + } + current, err = inspect(ctx, q, b, a.Desired) + if err != nil { + return result, err + } + remaining, err = b.diff(current, desired) + if err != nil { + return result, err + } + if len(remaining) > 0 { + return result, fmt.Errorf("postcondition failed: %d schema differences remain", len(remaining)) + } + result.Outcome = Applied + return result, nil +} diff --git a/database/migrate/review/artifact.go b/database/migrate/review/artifact.go new file mode 100644 index 00000000..4f337bb8 --- /dev/null +++ b/database/migrate/review/artifact.go @@ -0,0 +1,114 @@ +// Package review plans and applies hash-bound, transactional schema migrations. +package review + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "slices" + "time" +) + +type Snapshot struct { + Dialect string `json:"dialect"` + Schema string `json:"schema"` + HCL string `json:"hcl"` +} + +type Step struct { + ID string `json:"id"` + SQL string `json:"sql"` + Comment string `json:"comment,omitempty"` + Destructive bool `json:"destructive,omitempty"` +} + +type Artifact struct { + Version int `json:"version"` + Baseline Snapshot `json:"baseline"` + Desired Snapshot `json:"desired"` + Steps []Step `json:"steps"` + Advisory []string `json:"advisory,omitempty"` + Hash string `json:"hash"` +} + +type Options struct { + Dialect string + Schema string + DesiredHCL string + Advisory []string + PreserveOmitted bool +} + +type Request struct { + Artifact Artifact + Hash string + Acknowledgements []string + LockTimeout time.Duration +} + +type Outcome string + +const ( + Applied Outcome = "applied" + AlreadyApplied Outcome = "already-applied" + Failed Outcome = "failed" + CommitUnknown Outcome = "commit-unknown" +) + +type Result struct { + Hash string `json:"hash"` + Outcome Outcome `json:"outcome"` + Steps int `json:"steps"` +} + +func (a Artifact) Acknowledgements() []string { + var ids []string + for _, step := range a.Steps { + if step.Destructive { + ids = append(ids, step.ID) + } + } + return ids +} + +func (a Artifact) digest() (string, error) { + a.Hash = "" + data, err := json.Marshal(a) + if err != nil { + return "", err + } + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:]), nil +} + +func validate(request Request) error { + a := request.Artifact + hash, err := a.digest() + if err != nil { + return err + } + if request.Hash == "" || a.Hash != request.Hash || hash != request.Hash { + return fmt.Errorf("reviewed artifact hash mismatch") + } + if a.Version != 2 { + return fmt.Errorf("unsupported schema artifact version %d", a.Version) + } + if a.Baseline.Dialect != a.Desired.Dialect || a.Baseline.Schema != a.Desired.Schema { + return fmt.Errorf("artifact baseline and desired scope differ") + } + if len(a.Advisory) > 0 { + return fmt.Errorf("advisory preview cannot be applied: %v", a.Advisory) + } + wanted := a.Acknowledgements() + got := slices.Clone(request.Acknowledgements) + slices.Sort(wanted) + slices.Sort(got) + if !slices.Equal(wanted, got) { + return fmt.Errorf("destructive acknowledgement mismatch: expected %v, received %v", wanted, got) + } + if request.LockTimeout < 0 || request.LockTimeout > 24*time.Hour { + return fmt.Errorf("lock timeout must be between zero and 24h") + } + return nil +} diff --git a/database/migrate/review/artifact_test.go b/database/migrate/review/artifact_test.go new file mode 100644 index 00000000..4249750d --- /dev/null +++ b/database/migrate/review/artifact_test.go @@ -0,0 +1,121 @@ +package review + +import ( + "database/sql" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + _ "modernc.org/sqlite" + "path/filepath" + "testing" +) + +func TestReviewedSchema(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Reviewed schema migrations") +} + +const itemsHCL = `schema "main" {} +table "items" { + schema = schema.main + column "id" { type = integer } + column "label" { type = text } +}` + +var _ = Describe("SQLite reviewed artifacts", func() { + var db *sql.DB + BeforeEach(func() { + var err error + db, err = sql.Open("sqlite", filepath.Join(GinkgoT().TempDir(), "review.db")) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(db.Close) + }) + It("produces the same reviewed hash for reordered named declarations", func(ctx SpecContext) { + first := `table "first" { + schema = schema.main + column "id" { type = integer } +}` + second := `table "second" { + schema = schema.main + column "id" { type = integer } +}` + a, err := Build(ctx, db, Options{Dialect: "sqlite", Schema: "main", DesiredHCL: "schema \"main\" {}\n" + first + "\n" + second}) + Expect(err).NotTo(HaveOccurred()) + b, err := Build(ctx, db, Options{Dialect: "sqlite", Schema: "main", DesiredHCL: "schema \"main\" {}\n" + second + "\n" + first}) + Expect(err).NotTo(HaveOccurred()) + Expect(a.Hash).To(Equal(b.Hash)) + }) + It("previews without writing, applies the exact saved plan and recognizes a repeat", func(ctx SpecContext) { + artifact, err := Build(ctx, db, Options{Dialect: "sqlite", Schema: "main", DesiredHCL: itemsHCL}) + Expect(err).NotTo(HaveOccurred()) + var count int + Expect(db.QueryRow("SELECT count(*) FROM sqlite_schema WHERE type = 'table'").Scan(&count)).To(Succeed()) + Expect(count).To(BeZero()) + result, err := Apply(ctx, db, Request{Artifact: artifact, Hash: artifact.Hash}) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Outcome).To(Equal(Applied)) + Expect(result.Steps).To(Equal(len(artifact.Steps))) + result, err = Apply(ctx, db, Request{Artifact: artifact, Hash: artifact.Hash}) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Outcome).To(Equal(AlreadyApplied)) + }) + It("rejects a changed hash before touching the database", func(ctx SpecContext) { + artifact, err := Build(ctx, db, Options{Dialect: "sqlite", Schema: "main", DesiredHCL: itemsHCL}) + Expect(err).NotTo(HaveOccurred()) + artifact.Steps[0].SQL = "DROP TABLE unrelated" + _, err = Apply(ctx, nil, Request{Artifact: artifact, Hash: artifact.Hash}) + Expect(err).To(MatchError(ContainSubstring("hash"))) + }) + It("rejects schema drift under the write lock", func(ctx SpecContext) { + artifact, err := Build(ctx, db, Options{Dialect: "sqlite", Schema: "main", DesiredHCL: itemsHCL}) + Expect(err).NotTo(HaveOccurred()) + _, err = db.Exec("CREATE TABLE intervening (id integer)") + Expect(err).NotTo(HaveOccurred()) + _, err = Apply(ctx, db, Request{Artifact: artifact, Hash: artifact.Hash}) + Expect(err).To(MatchError(ContainSubstring("drift"))) + }) + It("requires each destructive step acknowledgement and preserves rows through a rebuild", func(ctx SpecContext) { + _, err := db.Exec("CREATE TABLE items (id integer, label text, omitted text); INSERT INTO items VALUES (1, 'retained', 'removed')") + Expect(err).NotTo(HaveOccurred()) + artifact, err := Build(ctx, db, Options{Dialect: "sqlite", Schema: "main", DesiredHCL: itemsHCL}) + Expect(err).NotTo(HaveOccurred()) + Expect(artifact.Acknowledgements()).NotTo(BeEmpty()) + _, err = Apply(ctx, db, Request{Artifact: artifact, Hash: artifact.Hash}) + Expect(err).To(MatchError(ContainSubstring("acknowledgement"))) + result, err := Apply(ctx, db, Request{Artifact: artifact, Hash: artifact.Hash, Acknowledgements: artifact.Acknowledgements()}) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Outcome).To(Equal(Applied)) + var label string + Expect(db.QueryRow("SELECT label FROM items WHERE id = 1").Scan(&label)).To(Succeed()) + Expect(label).To(Equal("retained")) + }) + It("fails closed on unmanaged triggers", func(ctx SpecContext) { + _, err := db.Exec("CREATE TABLE items (id integer); CREATE TRIGGER items_audit AFTER INSERT ON items BEGIN SELECT 1; END") + Expect(err).NotTo(HaveOccurred()) + _, err = Build(ctx, db, Options{Dialect: "sqlite", Schema: "main", DesiredHCL: itemsHCL}) + Expect(err).To(MatchError(ContainSubstring("unsupported"))) + }) + It("rolls back a failed copy and restores connection pragmas", func(ctx SpecContext) { + db.SetMaxOpenConns(1) + _, err := db.Exec("PRAGMA foreign_keys = on; PRAGMA busy_timeout = 321; CREATE TABLE items (id integer, label text); INSERT INTO items VALUES (1, NULL)") + Expect(err).NotTo(HaveOccurred()) + desired := `schema "main" {} +table "items" { + schema = schema.main + column "id" { type = integer } + column "label" { + type = text + null = false + } +}` + artifact, err := Build(ctx, db, Options{Dialect: "sqlite", Schema: "main", DesiredHCL: desired}) + Expect(err).NotTo(HaveOccurred()) + _, err = Apply(ctx, db, Request{Artifact: artifact, Hash: artifact.Hash, Acknowledgements: artifact.Acknowledgements()}) + Expect(err).To(HaveOccurred()) + var foreignKeys, busyTimeout, count int + Expect(db.QueryRow("PRAGMA foreign_keys").Scan(&foreignKeys)).To(Succeed()) + Expect(db.QueryRow("PRAGMA busy_timeout").Scan(&busyTimeout)).To(Succeed()) + Expect([]int{foreignKeys, busyTimeout}).To(Equal([]int{1, 321})) + Expect(db.QueryRow("SELECT count(*) FROM items WHERE label IS NULL").Scan(&count)).To(Succeed()) + Expect(count).To(Equal(1)) + }) +}) diff --git a/database/migrate/review/checks.go b/database/migrate/review/checks.go new file mode 100644 index 00000000..f8ea5338 --- /dev/null +++ b/database/migrate/review/checks.go @@ -0,0 +1,42 @@ +package review + +import ( + "fmt" + "github.com/flanksource/sqlglot-go/sqlglot" + "strings" +) + +func sameCheck(left, right, dialect string) (bool, error) { + if strings.TrimSpace(left) == strings.TrimSpace(right) { + return true, nil + } + a, err := sqlglot.ParseOne("SELECT "+left, dialect) + if err != nil { + return false, fmt.Errorf("parse existing check expression: %w", err) + } + b, err := sqlglot.ParseOne("SELECT "+right, dialect) + if err != nil { + return false, fmt.Errorf("parse desired check expression: %w", err) + } + return withoutParens(a).Equal(withoutParens(b)), nil +} + +func withoutParens(e *sqlglot.Expression) *sqlglot.Expression { + if e == nil { + return nil + } + if e.Class == "Paren" { + return withoutParens(e.This()) + } + for _, key := range e.Keys { + switch value := e.Args[key].(type) { + case *sqlglot.Expression: + e.Set(key, withoutParens(value)) + case []*sqlglot.Expression: + for i := range value { + value[i] = withoutParens(value[i]) + } + } + } + return e +} diff --git a/database/migrate/review/checks_test.go b/database/migrate/review/checks_test.go new file mode 100644 index 00000000..b9995c20 --- /dev/null +++ b/database/migrate/review/checks_test.go @@ -0,0 +1,20 @@ +package review + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("reviewed check expression comparison", func() { + DescribeTable("compares parsed expression structure", func(left, right string, same bool) { + actual, err := sameCheck(left, right, "postgres") + Expect(err).NotTo(HaveOccurred()) + Expect(actual).To(Equal(same)) + }, + Entry("rendered outer parentheses", "id > 0", "(id > 0)", true), + Entry("nested redundant parentheses", "((id)) > 0", "id > 0", true), + Entry("different values", "id > 0", "id > 1", false), + Entry("operator grouping", "(a AND b) OR c", "a AND (b OR c)", false), + Entry("literal parentheses", "label = '(value)'", "label = 'value'", false), + ) +}) diff --git a/database/migrate/review/inspect.go b/database/migrate/review/inspect.go new file mode 100644 index 00000000..4df31bba --- /dev/null +++ b/database/migrate/review/inspect.go @@ -0,0 +1,93 @@ +package review + +import ( + "context" + "database/sql" + "errors" + "fmt" + "strings" + + "ariga.io/atlas/sql/schema" +) + +func validateScope(scope Snapshot) error { + if scope.Schema == "" { + return fmt.Errorf("schema scope is required") + } + if scope.Dialect == "sqlite" && scope.Schema != "main" { + return fmt.Errorf("SQLite reviewed migrations require schema main") + } + if scope.Dialect != "sqlite" && scope.Dialect != "postgres" { + return fmt.Errorf("unsupported reviewed schema dialect %q", scope.Dialect) + } + if strings.HasPrefix(scope.Schema, "pg_") || scope.Schema == "information_schema" { + return fmt.Errorf("system schema %q cannot be migrated", scope.Schema) + } + return nil +} + +func Inspect(ctx context.Context, db *sql.DB, dialect, schemaName string) (_ Snapshot, returnErr error) { + if db == nil { + return Snapshot{}, fmt.Errorf("database is nil") + } + conn, err := db.Conn(ctx) + if err != nil { + return Snapshot{}, err + } + defer func() { returnErr = errors.Join(returnErr, conn.Close()) }() + b, err := open(conn, dialect) + if err != nil { + return Snapshot{}, err + } + scope := Snapshot{Dialect: dialect, Schema: schemaName} + if err := validateScope(scope); err != nil { + return Snapshot{}, err + } + s, err := inspect(ctx, conn, b, scope) + if err != nil { + return Snapshot{}, err + } + return b.encode(s, scope) +} + +func inspect(ctx context.Context, q schema.ExecQuerier, b backend, scope Snapshot) (*schema.Schema, error) { + if err := unsupported(ctx, q, scope); err != nil { + return nil, err + } + s, err := b.driver.InspectSchema(ctx, scope.Schema, nil) + if err != nil { + return nil, fmt.Errorf("inspect schema %q: %w", scope.Schema, err) + } + return s, nil +} + +func unsupported(ctx context.Context, q schema.ExecQuerier, scope Snapshot) (returnErr error) { + statement := `SELECT name FROM sqlite_schema WHERE type IN ('trigger','view') OR (type = 'table' AND upper(sql) LIKE 'CREATE VIRTUAL%') LIMIT 1` + var args []any + if scope.Dialect == "postgres" { + statement = `SELECT name FROM ( +SELECT c.relname AS name FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace +WHERE n.nspname=$1 AND (c.relkind IN ('v','m','f','p') OR c.relispartition OR c.relrowsecurity OR c.relforcerowsecurity + OR (c.relkind='S' AND NOT EXISTS (SELECT 1 FROM pg_depend d WHERE d.objid=c.oid AND d.deptype IN ('a','i')))) +UNION ALL SELECT p.proname FROM pg_proc p JOIN pg_namespace n ON n.oid=p.pronamespace WHERE n.nspname=$1 +UNION ALL SELECT t.tgname FROM pg_trigger t JOIN pg_class c ON c.oid=t.tgrelid JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname=$1 AND NOT t.tgisinternal +UNION ALL SELECT c.conname FROM pg_constraint c JOIN pg_namespace n ON n.oid=c.connamespace WHERE n.nspname=$1 AND (c.contype='x' OR c.condeferrable OR NOT c.convalidated) +UNION ALL SELECT t.typname FROM pg_type t JOIN pg_namespace n ON n.oid=t.typnamespace WHERE n.nspname=$1 AND t.typtype='d' +UNION ALL SELECT c.relname FROM pg_inherits i JOIN pg_class c ON c.oid=i.inhrelid JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname=$1 +) unsupported LIMIT 1` + args = []any{scope.Schema} + } + rows, err := q.QueryContext(ctx, statement, args...) + if err != nil { + return fmt.Errorf("inspect unsupported schema features: %w", err) + } + defer func() { returnErr = errors.Join(returnErr, rows.Close()) }() + if rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + return err + } + return fmt.Errorf("unsupported schema object or feature %q in %s", name, scope.Schema) + } + return rows.Err() +} diff --git a/database/migrate/review/plan.go b/database/migrate/review/plan.go new file mode 100644 index 00000000..d3dcfbe7 --- /dev/null +++ b/database/migrate/review/plan.go @@ -0,0 +1,283 @@ +package review + +import ( + "context" + "crypto/sha256" + "database/sql" + "errors" + "fmt" + "reflect" + "slices" + "strings" + + "ariga.io/atlas/schemahcl" + atlas "ariga.io/atlas/sql/migrate" + "ariga.io/atlas/sql/postgres" + "ariga.io/atlas/sql/schema" + "ariga.io/atlas/sql/sqlite" + "github.com/hashicorp/hcl/v2/hclparse" +) + +type backend struct { + driver atlas.Driver + marshal schemahcl.Marshaler + eval schemahcl.Evaluator + dialect string +} + +func open(q schema.ExecQuerier, dialect string) (backend, error) { + b := backend{dialect: dialect} + var err error + switch dialect { + case "sqlite": + b.driver, err = sqlite.Open(q) + b.marshal, b.eval = sqlite.MarshalHCL, sqlite.EvalHCL + case "postgres": + b.driver, err = postgres.Open(q) + b.marshal, b.eval = postgres.MarshalHCL, postgres.EvalHCL + default: + return b, fmt.Errorf("unsupported reviewed schema dialect %q", dialect) + } + return b, err +} + +func (b backend) decode(snapshot Snapshot) (*schema.Schema, error) { + p := hclparse.NewParser() + if _, diag := p.ParseHCL([]byte(snapshot.HCL), "snapshot.hcl"); diag.HasErrors() { + return nil, fmt.Errorf("parse desired schema: %s", diag.Error()) + } + realm := &schema.Realm{} + if err := b.eval.Eval(p, realm, nil); err != nil { + return nil, fmt.Errorf("evaluate snapshot: %w", err) + } + if len(realm.Schemas) != 1 || realm.Schemas[0].Name != snapshot.Schema { + return nil, fmt.Errorf("snapshot must declare exactly schema %q", snapshot.Schema) + } + s := realm.Schemas[0] + if len(s.Views)+len(s.Funcs)+len(s.Procs) > 0 || len(realm.Objects) > 0 { + return nil, fmt.Errorf("unsupported non-table schema objects") + } + return s, nil +} + +func (b backend) encode(s *schema.Schema, scope Snapshot) (Snapshot, error) { + canonicalize(s) + data, err := b.marshal.MarshalSpec(s) + if err != nil { + return Snapshot{}, fmt.Errorf("serialize schema: %w", err) + } + scope.HCL = string(data) + roundtrip, err := b.decode(scope) + if err != nil { + return Snapshot{}, err + } + changes, err := b.diff(s, roundtrip) + if err != nil { + return Snapshot{}, err + } + if len(changes) > 0 { + return Snapshot{}, fmt.Errorf("unsupported lossy schema snapshot: %d round-trip changes", len(changes)) + } + return scope, nil +} + +func (b backend) diff(current, desired *schema.Schema) ([]schema.Change, error) { + changes, err := b.driver.SchemaDiff(current, desired) + if err != nil { + return nil, err + } + // Atlas ignores edited check expressions on some backends; bind them explicitly. + for _, table := range desired.Tables { + before, ok := current.Table(table.Name) + if !ok { + continue + } + var checks []schema.Change + for _, check := range table.Checks() { + if check.Name == "" { + continue + } + var old *schema.Check + for _, candidate := range before.Checks() { + if candidate.Name == check.Name { + old = candidate + break + } + } + if old == nil { + continue + } + equal, err := sameCheck(old.Expr, check.Expr, b.dialect) + if err != nil { + return nil, err + } + if !equal { + checks = append(checks, &schema.DropCheck{C: old}, &schema.AddCheck{C: check}) + } + } + if len(checks) == 0 { + continue + } + var modified *schema.ModifyTable + for _, change := range changes { + if m, ok := change.(*schema.ModifyTable); ok && m.T.Name == table.Name { + modified = m + break + } + } + if modified == nil { + modified = &schema.ModifyTable{T: before} + changes = append(changes, modified) + } + modified.Changes = append(modified.Changes, checks...) + } + return changes, nil +} + +func Build(ctx context.Context, db *sql.DB, opts Options) (_ Artifact, returnErr error) { + if db == nil { + return Artifact{}, fmt.Errorf("database is nil") + } + conn, err := db.Conn(ctx) + if err != nil { + return Artifact{}, err + } + defer func() { returnErr = errors.Join(returnErr, conn.Close()) }() + b, err := open(conn, opts.Dialect) + if err != nil { + return Artifact{}, err + } + scope := Snapshot{Dialect: opts.Dialect, Schema: opts.Schema} + if err := validateScope(scope); err != nil { + return Artifact{}, err + } + current, err := inspect(ctx, conn, b, scope) + if err != nil { + return Artifact{}, err + } + desired, err := b.decode(Snapshot{Dialect: opts.Dialect, Schema: opts.Schema, HCL: opts.DesiredHCL}) + if err != nil { + return Artifact{}, err + } + if opts.PreserveOmitted { + preserveOmitted(current, desired) + } + baseline, err := b.encode(current, scope) + if err != nil { + return Artifact{}, err + } + after, err := b.encode(desired, scope) + if err != nil { + return Artifact{}, err + } + steps, err := b.plan(ctx, current, desired) + if err != nil { + return Artifact{}, err + } + a := Artifact{Version: 2, Baseline: baseline, Desired: after, Steps: steps, Advisory: opts.Advisory} + a.Hash, err = a.digest() + return a, err +} + +func (b backend) plan(ctx context.Context, current, desired *schema.Schema) ([]Step, error) { + changes, err := b.diff(current, desired) + if err != nil { + return nil, err + } + plan, err := b.driver.PlanChanges(ctx, "reviewed-schema", changes) + if err != nil { + return nil, fmt.Errorf("plan schema: %w", err) + } + if !plan.Transactional { + return nil, fmt.Errorf("nontransactional plans require phased migration artifacts") + } + steps := make([]Step, 0, len(plan.Changes)) + for _, change := range plan.Changes { + if change.Cmd == "PRAGMA foreign_keys = off" || change.Cmd == "PRAGMA foreign_keys = on" { + continue + } + if len(change.Args) > 0 { + return nil, fmt.Errorf("unsupported parameterized schema migration") + } + id := fmt.Sprintf("%x", sha256.Sum256([]byte(fmt.Sprintf("%d:%s", len(steps), change.Cmd)))) + steps = append(steps, Step{ID: id, SQL: change.Cmd, Comment: change.Comment, Destructive: destructive(change.Source)}) + } + return steps, nil +} + +func destructive(change schema.Change) bool { + switch c := change.(type) { + case *schema.DropTable, *schema.DropColumn, *schema.DropObject, *schema.DropIndex, *schema.DropPrimaryKey, *schema.DropForeignKey, *schema.DropCheck: + return true + case *schema.ModifyColumn: + return c.Change.Is(schema.ChangeType) || c.Change.Is(schema.ChangeNull) + case *schema.ModifyTable: + for _, inner := range c.Changes { + if destructive(inner) { + return true + } + } + case *schema.ModifyObject, *schema.ModifyPrimaryKey, *schema.ModifyForeignKey, *schema.ModifyCheck: + return true + } + return false +} + +func sameSteps(a, b []Step) bool { return reflect.DeepEqual(a, b) } + +func canonicalize(s *schema.Schema) { + slices.SortFunc(s.Tables, func(a, b *schema.Table) int { return strings.Compare(a.Name, b.Name) }) + for _, t := range s.Tables { + slices.SortFunc(t.Indexes, func(a, b *schema.Index) int { return strings.Compare(a.Name, b.Name) }) + slices.SortFunc(t.ForeignKeys, func(a, b *schema.ForeignKey) int { return strings.Compare(a.Symbol, b.Symbol) }) + slices.SortStableFunc(t.Attrs, func(a, b schema.Attr) int { return strings.Compare(attributeKey(a), attributeKey(b)) }) + } + slices.SortStableFunc(s.Objects, func(a, b schema.Object) int { + left, right := fmt.Sprintf("%T", a), fmt.Sprintf("%T", b) + if e, ok := a.(*schema.EnumType); ok { + left += e.T + } + if e, ok := b.(*schema.EnumType); ok { + right += e.T + } + return strings.Compare(left, right) + }) +} + +func attributeKey(a schema.Attr) string { + if check, ok := a.(*schema.Check); ok { + return "check:" + check.Name + ":" + check.Expr + } + return fmt.Sprintf("%T", a) +} + +func preserveOmitted(current, desired *schema.Schema) { + for _, before := range current.Tables { + after, ok := desired.Table(before.Name) + if !ok { + desired.AddTables(before) + continue + } + for _, column := range before.Columns { + if _, ok := after.Column(column.Name); !ok { + after.AddColumns(column) + } + } + } + for _, object := range current.Objects { + enum, ok := object.(*schema.EnumType) + if !ok { + continue + } + found := false + for _, candidate := range desired.Objects { + if e, ok := candidate.(*schema.EnumType); ok && e.T == enum.T { + found = true + break + } + } + if !found { + desired.AddObjects(object) + } + } +} diff --git a/database/migrate/review/postgres_integration_test.go b/database/migrate/review/postgres_integration_test.go new file mode 100644 index 00000000..711075f3 --- /dev/null +++ b/database/migrate/review/postgres_integration_test.go @@ -0,0 +1,125 @@ +//go:build integration + +package review + +import ( + "database/sql" + "errors" + "fmt" + _ "github.com/lib/pq" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "net/url" + "os" + "time" +) + +var _ = Describe("PostgreSQL reviewed artifacts", func() { + var db *sql.DB + BeforeEach(func(ctx SpecContext) { + dsn := os.Getenv("SQLGLOT_TEST_POSTGRES_URL") + Expect(dsn).NotTo(BeEmpty(), "integration tests require a disposable PostgreSQL server") + admin, err := sql.Open("postgres", dsn) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(admin.Close) + name := fmt.Sprintf("sqlglot_review_%d", time.Now().UnixNano()) + _, err = admin.ExecContext(ctx, "CREATE DATABASE "+name) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { _, e := admin.Exec("DROP DATABASE " + name); Expect(e).NotTo(HaveOccurred()) }) + u, err := url.Parse(dsn) + Expect(err).NotTo(HaveOccurred()) + u.Path = "/" + name + db, err = sql.Open("postgres", u.String()) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(db.Close) + }) + It("recognizes backend-rendered parentheses in named checks", func(ctx SpecContext) { + desired := `schema "public" {} +table "items" { + schema = schema.public + column "id" { type = integer } + check "positive_id" { expr = "id > 0" } +}` + artifact, err := Build(ctx, db, Options{Dialect: "postgres", Schema: "public", DesiredHCL: desired}) + Expect(err).NotTo(HaveOccurred()) + result, err := Apply(ctx, db, Request{Artifact: artifact, Hash: artifact.Hash}) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Outcome).To(Equal(Applied)) + result, err = Apply(ctx, db, Request{Artifact: artifact, Hash: artifact.Hash}) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Outcome).To(Equal(AlreadyApplied)) + }) + It("plans read-only, applies exact SQL and recognizes the resulting schema", func(ctx SpecContext) { + desired := `schema "public" {} +table "items" { + schema = schema.public + column "id" { type = integer } + column "label" { type = text } +}` + artifact, err := Build(ctx, db, Options{Dialect: "postgres", Schema: "public", DesiredHCL: desired}) + Expect(err).NotTo(HaveOccurred()) + var tables int + Expect(db.QueryRow("SELECT count(*) FROM information_schema.tables WHERE table_schema = 'public'").Scan(&tables)).To(Succeed()) + Expect(tables).To(BeZero()) + result, err := Apply(ctx, db, Request{Artifact: artifact, Hash: artifact.Hash}) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Outcome).To(Equal(Applied)) + result, err = Apply(ctx, db, Request{Artifact: artifact, Hash: artifact.Hash}) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Outcome).To(Equal(AlreadyApplied)) + }) + It("preserves enum snapshots through reviewed creation", func(ctx SpecContext) { + desired := `schema "public" {} +enum "state" { + schema = schema.public + values = ["new", "done"] +} +table "items" { + schema = schema.public + column "state" { type = enum.state } +}` + artifact, err := Build(ctx, db, Options{Dialect: "postgres", Schema: "public", DesiredHCL: desired}) + Expect(err).NotTo(HaveOccurred()) + result, err := Apply(ctx, db, Request{Artifact: artifact, Hash: artifact.Hash}) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Outcome).To(Equal(Applied)) + }) + It("rolls back failed DDL and rejects drift", func(ctx SpecContext) { + _, err := db.Exec("CREATE TABLE items (label text); INSERT INTO items VALUES (NULL)") + Expect(err).NotTo(HaveOccurred()) + desired := `schema "public" {} +table "items" { + schema = schema.public + column "label" { type = text } +}` + artifact, err := Build(ctx, db, Options{Dialect: "postgres", Schema: "public", DesiredHCL: desired}) + Expect(err).NotTo(HaveOccurred()) + _, err = Apply(ctx, db, Request{Artifact: artifact, Hash: artifact.Hash, Acknowledgements: artifact.Acknowledgements()}) + Expect(err).To(HaveOccurred()) + var count int + Expect(db.QueryRow("SELECT count(*) FROM items WHERE label IS NULL").Scan(&count)).To(Succeed()) + Expect(count).To(Equal(1)) + _, err = db.Exec("ALTER TABLE items ADD COLUMN intervening text") + Expect(err).NotTo(HaveOccurred()) + _, err = Apply(ctx, db, Request{Artifact: artifact, Hash: artifact.Hash, Acknowledgements: artifact.Acknowledgements()}) + Expect(err).To(MatchError(ContainSubstring("drift"))) + }) + It("bounds waiting for another migration transaction", func(ctx SpecContext) { + snapshot, err := Inspect(ctx, db, "postgres", "public") + Expect(err).NotTo(HaveOccurred()) + artifact, err := Build(ctx, db, Options{Dialect: "postgres", Schema: "public", DesiredHCL: snapshot.HCL}) + Expect(err).NotTo(HaveOccurred()) + tx, err := db.BeginTx(ctx, nil) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { + if err := tx.Rollback(); !errors.Is(err, sql.ErrTxDone) { + Expect(err).NotTo(HaveOccurred()) + } + }) + _, err = tx.ExecContext(ctx, "SELECT pg_advisory_xact_lock(7561364747649)") + Expect(err).NotTo(HaveOccurred()) + _, err = Apply(ctx, db, Request{Artifact: artifact, Hash: artifact.Hash, LockTimeout: 25 * time.Millisecond}) + Expect(err).To(MatchError(ContainSubstring("acquire migration lock"))) + Expect(tx.Rollback()).To(Succeed()) + }) +}) diff --git a/database/migrate/review/sqlite.go b/database/migrate/review/sqlite.go new file mode 100644 index 00000000..78f7087f --- /dev/null +++ b/database/migrate/review/sqlite.go @@ -0,0 +1,82 @@ +package review + +import ( + "context" + "database/sql" + "database/sql/driver" + "errors" + "fmt" + "time" +) + +func applySQLite(ctx context.Context, conn *sql.Conn, request Request) (result Result, err error) { + result = Result{Hash: request.Hash, Outcome: Failed} + var foreignKeys, busyTimeout int + if err = conn.QueryRowContext(ctx, "PRAGMA foreign_keys").Scan(&foreignKeys); err != nil { + return result, err + } + if err = conn.QueryRowContext(ctx, "PRAGMA busy_timeout").Scan(&busyTimeout); err != nil { + return result, err + } + defer func() { + cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) + defer cancel() + _, foreignErr := conn.ExecContext(cleanup, fmt.Sprintf("PRAGMA foreign_keys = %d", foreignKeys)) + _, busyErr := conn.ExecContext(cleanup, fmt.Sprintf("PRAGMA busy_timeout = %d", busyTimeout)) + if foreignErr != nil || busyErr != nil { + err = errors.Join(err, fmt.Errorf("restore SQLite pragmas: %w", errors.Join(foreignErr, busyErr))) + _ = conn.Raw(func(any) error { return driver.ErrBadConn }) + } + }() + if _, err = conn.ExecContext(ctx, fmt.Sprintf("PRAGMA busy_timeout = %d", max(1, request.LockTimeout.Milliseconds()))); err != nil { + return result, err + } + if _, err = conn.ExecContext(ctx, "PRAGMA foreign_keys = off"); err != nil { + return result, err + } + if _, err = conn.ExecContext(ctx, "BEGIN IMMEDIATE"); err != nil { + return result, fmt.Errorf("acquire SQLite migration write lock: %w", err) + } + finished := false + defer func() { + if finished { + return + } + cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) + defer cancel() + _, rollbackErr := conn.ExecContext(cleanup, "ROLLBACK") + if rollbackErr != nil { + err = errors.Join(err, fmt.Errorf("rollback SQLite migration: %w", rollbackErr)) + _ = conn.Raw(func(any) error { return driver.ErrBadConn }) + } + }() + if err = foreignKeyCheck(ctx, conn); err != nil { + return result, err + } + result, err = execute(ctx, conn, request) + if err != nil { + return result, err + } + if err = foreignKeyCheck(ctx, conn); err != nil { + result.Outcome = Failed + return result, err + } + if _, err = conn.ExecContext(ctx, "COMMIT"); err != nil { + result.Outcome = CommitUnknown + return result, fmt.Errorf("commit outcome unknown: %w", err) + } + finished = true + return result, nil +} + +func foreignKeyCheck(ctx context.Context, conn *sql.Conn) (returnErr error) { + rows, err := conn.QueryContext(ctx, "PRAGMA foreign_key_check") + if err != nil { + return err + } + defer func() { returnErr = errors.Join(returnErr, rows.Close()) }() + if rows.Next() { + return fmt.Errorf("SQLite foreign key check failed") + } + return rows.Err() +} diff --git a/database/migrate/review/validate.go b/database/migrate/review/validate.go new file mode 100644 index 00000000..ad9acd81 --- /dev/null +++ b/database/migrate/review/validate.go @@ -0,0 +1,9 @@ +package review + +// Validate checks the reviewed hash, scope, acknowledgements and execution options without opening a database. +func Validate(request Request) error { + if err := validate(request); err != nil { + return err + } + return validateScope(request.Artifact.Baseline) +} diff --git a/database/migrate/routines.go b/database/migrate/routines.go new file mode 100644 index 00000000..61fdf266 --- /dev/null +++ b/database/migrate/routines.go @@ -0,0 +1,84 @@ +package migrate + +import ( + "fmt" + "io/fs" + "path" + "strings" + + planner "github.com/flanksource/sqlglot-go/migrate/atlas/sqlserver" + "github.com/flanksource/sqlglot-go/sqlglot" +) + +// WithRoutines declares stored procedures and functions from the .sql files +// under dir (relative to the schema filesystem root). Each file holds exactly +// one CREATE [OR ALTER] PROCEDURE|FUNCTION batch, optionally followed by GO, +// and is named after the routine it creates (mc_ClonePlan.sql creates +// mc_ClonePlan). The files are the routines' single source: they run as +// written, so the same file works in SSMS or sqlcmd, and they are never +// treated as hash-gated SQL scripts. Only SQL Server supports routines. +func WithRoutines(dir string) Option { + return func(o *options) { o.routines = strings.Trim(path.Clean(dir), "/") } +} + +// refuseRoutines rejects WithRoutines on a dialect that does not support it. +func refuseRoutines(cfg options, dialect string) error { + if cfg.routines != "" { + return fmt.Errorf("%s migrations do not support routines (WithRoutines); only SQL Server does", dialect) + } + return nil +} + +// loadRoutines parses every .sql file under dir into the routine it creates. +func loadRoutines(schemaFS fs.FS, dir string) ([]*planner.Routine, error) { + var routines []*planner.Routine + seen := map[string]string{} + err := fs.WalkDir(schemaFS, dir, func(name string, entry fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if entry.IsDir() || !strings.EqualFold(path.Ext(name), ".sql") { + return nil + } + routine, err := loadRoutine(schemaFS, name) + if err != nil { + return err + } + key := strings.ToLower(routine.Name) + if other, ok := seen[key]; ok { + return fmt.Errorf("routine %s is declared by both %s and %s", routine.Name, other, name) + } + seen[key] = name + routines = append(routines, routine) + return nil + }) + if err != nil { + return nil, fmt.Errorf("load routines from %q: %w", dir, err) + } + if len(routines) == 0 { + return nil, fmt.Errorf("load routines from %q: no .sql files found", dir) + } + return routines, nil +} + +func loadRoutine(schemaFS fs.FS, name string) (*planner.Routine, error) { + data, err := fs.ReadFile(schemaFS, name) + if err != nil { + return nil, fmt.Errorf("read routine %s: %w", name, err) + } + batches, err := sqlglot.SplitBatches(string(data), planner.Dialect) + if err != nil { + return nil, fmt.Errorf("routine %s: %w", name, err) + } + if len(batches) != 1 { + return nil, fmt.Errorf("routine %s: expected exactly one batch, found %d", name, len(batches)) + } + routine, err := planner.ParseRoutine(batches[0]) + if err != nil { + return nil, fmt.Errorf("routine %s: %w", name, err) + } + if base := strings.TrimSuffix(path.Base(name), path.Ext(name)); routine.Name != base { + return nil, fmt.Errorf("routine %s creates %q: the file must be named after its routine", name, routine.Name) + } + return routine, nil +} diff --git a/database/migrate/schema_option_ginkgo_test.go b/database/migrate/schema_option_ginkgo_test.go new file mode 100644 index 00000000..c17ddb97 --- /dev/null +++ b/database/migrate/schema_option_ginkgo_test.go @@ -0,0 +1,103 @@ +package migrate + +import ( + "net/url" + "strings" + "testing/fstest" + + "ariga.io/atlas/sql/schema" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("schema-scoped migrations", func() { + DescribeTable("validates normalized schema identifiers", + func(name string, valid bool) { + err := ValidateSchemaName(name) + if valid { + Expect(err).NotTo(HaveOccurred()) + } else { + Expect(err).To(HaveOccurred()) + } + }, + Entry("public", "public", true), + Entry("normalized context", "agent_namespace_context", true), + Entry("leading underscore", "_private", true), + Entry("63 bytes", "s"+strings.Repeat("a", 62), true), + Entry("empty", "", false), + Entry("uppercase", "AGENT", false), + Entry("hyphen", "agent-context", false), + Entry("quoted", `"agent"`, false), + Entry("qualified", "public.agent", false), + Entry("64 bytes", "s"+strings.Repeat("a", 63), false), + ) + + It("defaults options to public and retains a selected schema", func() { + Expect(resolveOptions(nil).schema).To(Equal(defaultSchema)) + cfg := resolveOptions([]Option{WithSchema("agent_namespace_context")}) + Expect(cfg.schema).To(Equal("agent_namespace_context")) + }) + + It("preserves default-schema keyword connections", func(ctx SpecContext) { + const connection = "host=localhost dbname=example" + got, err := prepareSchemaConnection(ctx, schemaConnectionOptions{Connection: connection, Schema: defaultSchema}) + Expect(err).NotTo(HaveOccurred()) + Expect(got).To(Equal(connection)) + }) + + It("rejects an invalid selected schema before connecting", func(ctx SpecContext) { + filesystem := fstest.MapFS{"schema.hcl": &fstest.MapFile{Data: []byte(`schema "public" {}`)}} + err := Apply(ctx, "postgres://unused", filesystem, WithSchema("")) + Expect(err).To(MatchError(ContainSubstring("migration schema: schema name is empty"))) + }) + + It("creates a schema-scoped URL without losing existing parameters", func() { + got, err := ConnectionForSchema("postgres://user:pass@localhost/database?sslmode=disable&application_name=migrate", "agent_namespace_context") + Expect(err).NotTo(HaveOccurred()) + parsed, err := url.Parse(got) + Expect(err).NotTo(HaveOccurred()) + Expect(parsed.Query().Get("search_path")).To(Equal("agent_namespace_context")) + Expect(parsed.Query().Get("sslmode")).To(Equal("disable")) + Expect(parsed.Query().Get("application_name")).To(Equal("migrate")) + }) + + It("rejects a non-URL connection when it cannot scope it safely", func() { + _, err := ConnectionForSchema("host=localhost dbname=example", "agent_namespace_context") + Expect(err).To(MatchError(ContainSubstring("URL-form"))) + }) + + It("includes non-public schemas in metadata scope without changing public scope", func() { + Expect(migrationScope(defaultSchema, "runtime")).To(Equal("runtime")) + Expect(migrationScope("agent_namespace_context", "runtime")).To(Equal("agent_namespace_context:runtime")) + }) + + It("remaps the desired public schema and its table references", func() { + public := schema.New("public") + table := schema.NewTable("threads").SetSchema(public) + public.AddTables(table) + realm := schema.NewRealm(public) + + Expect(remapDesiredSchema(realm, "agent_namespace_context")).To(Succeed()) + Expect(realm.Schemas).To(HaveLen(1)) + Expect(realm.Schemas[0].Name).To(Equal("agent_namespace_context")) + Expect(table.Schema).To(BeIdenticalTo(realm.Schemas[0])) + }) + + It("remaps security targets owned by the default schema", func() { + spec := securitySpec{Permissions: []permissionSpec{ + {Target: "schema:public"}, + {Target: "table:public.threads"}, + {Target: "column:public.threads.id"}, + {Target: "sequence:public.thread_seq"}, + }} + + got := remapSecuritySchema(spec, "agent_namespace_context") + Expect(got.Permissions).To(Equal([]permissionSpec{ + {Target: "schema:agent_namespace_context"}, + {Target: "table:agent_namespace_context.threads"}, + {Target: "column:agent_namespace_context.threads.id"}, + {Target: "sequence:agent_namespace_context.thread_seq"}, + })) + Expect(spec.Permissions[0].Target).To(Equal("schema:public")) + }) +}) diff --git a/database/migrate/schema_scope.go b/database/migrate/schema_scope.go new file mode 100644 index 00000000..b80b0bef --- /dev/null +++ b/database/migrate/schema_scope.go @@ -0,0 +1,124 @@ +package migrate + +import ( + "context" + "database/sql" + "fmt" + "net/url" + "regexp" + "strings" + + "ariga.io/atlas/sql/schema" + "github.com/lib/pq" +) + +const defaultSchema = "public" + +var normalizedSchemaName = regexp.MustCompile(`^[a-z_][a-z0-9_]*$`) + +// ValidateSchemaName validates an application-normalized PostgreSQL schema identifier. +func ValidateSchemaName(name string) error { + if name == "" { + return fmt.Errorf("schema name is empty") + } + if len(name) > 63 { + return fmt.Errorf("schema name %q is %d bytes; maximum is 63", name, len(name)) + } + if !normalizedSchemaName.MatchString(name) { + return fmt.Errorf("schema name %q must match %s", name, normalizedSchemaName.String()) + } + return nil +} + +// ConnectionForSchema returns a PostgreSQL URL whose connections select schemaName. +func ConnectionForSchema(connection, schemaName string) (string, error) { + if err := ValidateSchemaName(schemaName); err != nil { + return "", err + } + parsed, err := url.Parse(connection) + if err != nil || parsed.Scheme == "" || parsed.Host == "" { + return "", fmt.Errorf("schema-scoped migrations require a URL-form PostgreSQL connection") + } + switch strings.ToLower(parsed.Scheme) { + case "postgres", "postgresql": + default: + return "", fmt.Errorf("schema-scoped migrations require a PostgreSQL connection, got %q", parsed.Scheme) + } + query := parsed.Query() + query.Set("search_path", schemaName) + parsed.RawQuery = query.Encode() + return parsed.String(), nil +} + +func migrationScope(schemaName, bundleName string) string { + if schemaName != defaultSchema { + return schemaName + ":" + bundleName + } + return bundleName +} + +func remapDesiredSchema(realm *schema.Realm, schemaName string) error { + if realm == nil { + return fmt.Errorf("desired realm is nil") + } + if err := ValidateSchemaName(schemaName); err != nil { + return err + } + var source *schema.Schema + for _, candidate := range realm.Schemas { + if candidate.Name != defaultSchema { + return fmt.Errorf("desired realm contains schema %q; schema-scoped bundles must declare only %q", candidate.Name, defaultSchema) + } + source = candidate + } + if source == nil { + return fmt.Errorf("desired realm does not declare schema %q", defaultSchema) + } + source.Name = schemaName + return nil +} + +func remapSecuritySchema(spec securitySpec, schemaName string) securitySpec { + out := securitySpec{Roles: append([]roleSpec(nil), spec.Roles...), Permissions: make([]permissionSpec, len(spec.Permissions))} + for i, permission := range spec.Permissions { + permission.Privileges = append([]string(nil), permission.Privileges...) + for _, prefix := range []string{"table:", "column:", "sequence:"} { + permission.Target = strings.Replace(permission.Target, prefix+defaultSchema+".", prefix+schemaName+".", 1) + } + if permission.Target == "schema:"+defaultSchema { + permission.Target = "schema:" + schemaName + } + out.Permissions[i] = permission + } + return out +} + +type schemaConnectionOptions struct { + Connection string + Schema string +} + +func prepareSchemaConnection(ctx context.Context, opts schemaConnectionOptions) (string, error) { + if err := ValidateSchemaName(opts.Schema); err != nil { + return "", err + } + if opts.Schema == defaultSchema { + return opts.Connection, nil + } + scopedConnection, err := ConnectionForSchema(opts.Connection, opts.Schema) + if err != nil { + return "", err + } + database, err := sql.Open("postgres", opts.Connection) + if err != nil { + return "", fmt.Errorf("open schema migration database: %w", err) + } + defer database.Close() //nolint:errcheck + if err := database.PingContext(ctx); err != nil { + return "", fmt.Errorf("connect schema migration database: %w", err) + } + if _, err := database.ExecContext(ctx, "CREATE SCHEMA IF NOT EXISTS "+pq.QuoteIdentifier(opts.Schema)); err != nil { + return "", fmt.Errorf("create schema %q: %w", opts.Schema, err) + } + return scopedConnection, nil +} diff --git a/database/migrate/script_analysis.go b/database/migrate/script_analysis.go new file mode 100644 index 00000000..ef35316f --- /dev/null +++ b/database/migrate/script_analysis.go @@ -0,0 +1,52 @@ +package migrate + +import "github.com/flanksource/sqlglot-go/sqlglot" + +type ScriptAnalysis struct { + Path string `json:"path,omitempty"` + Phase string `json:"phase,omitempty"` + Kind string `json:"kind"` + Complete bool `json:"complete"` + Diagnostics []sqlglot.ParseDiagnostic `json:"diagnostics,omitempty"` + Error string `json:"error,omitempty"` +} + +func AnalyzeScript(text, dialect string) ScriptAnalysis { + if dialect == "sqlserver" { + dialect = "tsql" + } + result, err := sqlglot.ParseScript(text, dialect) + a := ScriptAnalysis{Kind: "unknown", Complete: result.Complete, Diagnostics: result.Diagnostics} + if err != nil { + a.Error = err.Error() + return a + } + if !result.Complete || len(result.Diagnostics) > 0 || result.Tree == nil { + return a + } + ddl, dml, unknown := false, false, false + result.Tree.Walk(func(e *sqlglot.Expression) bool { + switch e.Class { + case "Block": + return true + case "Create", "Alter", "Drop", "TruncateTable", "Grant", "Revoke": + ddl = true + case "Insert", "Update", "Delete", "Merge", "Select", "Union", "Except", "Intersect": + dml = true + default: + unknown = true + } + return false + }) + switch { + case unknown: + a.Kind = "unknown" + case ddl && dml: + a.Kind = "mixed" + case ddl: + a.Kind = "ddl" + case dml: + a.Kind = "dml" + } + return a +} diff --git a/database/migrate/scripts.go b/database/migrate/scripts.go new file mode 100644 index 00000000..15be99aa --- /dev/null +++ b/database/migrate/scripts.go @@ -0,0 +1,428 @@ +package migrate + +import ( + "bufio" + "context" + "crypto/sha256" + "database/sql" + "errors" + "fmt" + "io/fs" + "path" + "slices" + "sort" + "strings" + "time" + + "github.com/flanksource/commons/logger" + "github.com/lib/pq" +) + +const ( + // migrationLockTimeout bounds how long a transactional migration waits for a + // table lock before Postgres cancels it (55P03). This turns an indefinite + // block behind a live reader (e.g. a running server SELECT-ing the same + // captain_* tables) into a retryable timeout instead of a hang or deadlock. + migrationLockTimeout = "15s" + // migrationMaxAttempts is the total number of times a transactional migration + // is tried before giving up when it keeps hitting lock contention. + migrationMaxAttempts = 6 + // migrationRetryBackoff is the base delay between retries; it grows linearly + // with the attempt number so a busy server gets progressively longer gaps. + migrationRetryBackoff = 250 * time.Millisecond +) + +type scriptPhase string + +const ( + phasePre scriptPhase = "pre" + phasePost scriptPhase = "post" +) + +type script struct { + path string + content string + phase scriptPhase + dependsOn []string + always bool + transactional bool + hash []byte +} + +// loadScripts reads the hash-gated SQL scripts under dir, skipping the +// directories in skip (the WithRoutines directory). +func loadScripts(schemaFS fs.FS, dir string, skip ...string) (map[string]*script, error) { + scripts := map[string]*script{} + err := fs.WalkDir(schemaFS, dir, func(name string, entry fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if entry.IsDir() && slices.Contains(skip, name) { + return fs.SkipDir + } + if entry.IsDir() || strings.ToLower(path.Ext(name)) != ".sql" { + return nil + } + data, err := fs.ReadFile(schemaFS, name) + if err != nil { + return fmt.Errorf("read SQL %s: %w", name, err) + } + rel := strings.TrimPrefix(strings.TrimPrefix(name, dir), "/") + if rel == "" { + return fmt.Errorf("invalid SQL script path %q", name) + } + s, err := parseScript(rel, string(data)) + if err != nil { + return err + } + if _, exists := scripts[s.path]; exists { + return fmt.Errorf("duplicate SQL script path %q", s.path) + } + scripts[s.path] = s + return nil + }) + if err != nil { + return nil, fmt.Errorf("load SQL migrations: %w", err) + } + if err := validateScriptGraph(scripts); err != nil { + return nil, err + } + return scripts, nil +} + +func parseScript(name, content string) (*script, error) { + s := &script{path: path.Clean(name), content: content, phase: phasePost, transactional: true} + sum := sha256.Sum256([]byte(content)) + s.hash = sum[:] + + scanner := bufio.NewScanner(strings.NewReader(content)) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" { + continue + } + if !strings.HasPrefix(line, "--") { + break + } + switch { + case strings.HasPrefix(line, "-- phase:"): + value := strings.TrimSpace(strings.TrimPrefix(line, "-- phase:")) + switch scriptPhase(value) { + case phasePre, phasePost: + s.phase = scriptPhase(value) + default: + return nil, fmt.Errorf("SQL script %q has invalid phase %q", name, value) + } + case strings.HasPrefix(line, "-- dependsOn:"): + for _, dep := range strings.Split(strings.TrimSpace(strings.TrimPrefix(line, "-- dependsOn:")), ",") { + dep = path.Clean(strings.TrimSpace(dep)) + if dep != "." && dep != "" { + s.dependsOn = append(s.dependsOn, dep) + } + } + case line == "-- runs: always": + s.always = true + case strings.HasPrefix(line, "-- transaction:"): + value := strings.TrimSpace(strings.TrimPrefix(line, "-- transaction:")) + switch value { + case "true": + s.transactional = true + case "false": + s.transactional = false + default: + return nil, fmt.Errorf("SQL script %q has invalid transaction value %q", name, value) + } + } + } + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("scan SQL script %q: %w", name, err) + } + return s, nil +} + +func validateScriptGraph(scripts map[string]*script) error { + for _, s := range scripts { + seen := map[string]bool{} + for _, dep := range s.dependsOn { + if seen[dep] { + return fmt.Errorf("SQL script %q declares dependency %q more than once", s.path, dep) + } + seen[dep] = true + d, ok := scripts[dep] + if !ok { + return fmt.Errorf("SQL script %q depends on missing script %q", s.path, dep) + } + if s.phase == phasePre && d.phase == phasePost { + return fmt.Errorf("pre SQL script %q cannot depend on post script %q", s.path, dep) + } + } + } + _, err := topologicalScripts(scripts, nil) + return err +} + +func topologicalScripts(scripts map[string]*script, selected map[string]bool) ([]*script, error) { + indegree := map[string]int{} + dependents := map[string][]string{} + for name, s := range scripts { + if selected != nil && !selected[name] { + continue + } + for _, dep := range s.dependsOn { + if selected != nil && !selected[dep] { + continue + } + indegree[name]++ + dependents[dep] = append(dependents[dep], name) + } + if _, ok := indegree[name]; !ok { + indegree[name] = 0 + } + } + ready := make([]string, 0) + for name, degree := range indegree { + if degree == 0 { + ready = append(ready, name) + } + } + sort.Strings(ready) + ordered := make([]*script, 0, len(indegree)) + for len(ready) > 0 { + name := ready[0] + ready = ready[1:] + ordered = append(ordered, scripts[name]) + for _, dependent := range dependents[name] { + indegree[dependent]-- + if indegree[dependent] == 0 { + ready = append(ready, dependent) + sort.Strings(ready) + } + } + } + if len(ordered) != len(indegree) { + var cyclic []string + for name, degree := range indegree { + if degree > 0 { + cyclic = append(cyclic, name) + } + } + sort.Strings(cyclic) + return nil, fmt.Errorf("SQL migration dependency cycle: %s", strings.Join(cyclic, ", ")) + } + return ordered, nil +} + +func ensureMetadataTables(ctx context.Context, db *sql.DB) error { + _, err := db.ExecContext(ctx, ` +CREATE TABLE IF NOT EXISTS schema_migration_scripts ( + scope text NOT NULL, + path text NOT NULL, + hash bytea NOT NULL, + updated_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (scope, path) +); +CREATE TABLE IF NOT EXISTS schema_migration_security ( + scope text PRIMARY KEY, + state jsonb NOT NULL, + updated_at timestamptz NOT NULL DEFAULT now() +)`) + if err != nil { + return fmt.Errorf("create migration metadata tables: %w", err) + } + return nil +} + +func selectScripts(ctx context.Context, db *sql.DB, scope string, scripts map[string]*script) (_ map[string]bool, returnErr error) { + rows, err := db.QueryContext(ctx, `SELECT path, hash FROM schema_migration_scripts WHERE scope = $1`, scope) + if err != nil { + return nil, fmt.Errorf("read SQL migration hashes: %w", err) + } + defer func() { returnErr = errors.Join(returnErr, rows.Close()) }() + hashes := map[string][]byte{} + for rows.Next() { + var name string + var hash []byte + if err := rows.Scan(&name, &hash); err != nil { + return nil, err + } + hashes[name] = hash + } + if err := rows.Err(); err != nil { + return nil, err + } + return selectFromRecordedHashes(scripts, hashes), nil +} + +// selectFromRecordedHashes decides which scripts run. Hash-dirty scripts — +// genuine content changes — cascade to their dependents (a dependent may bind +// objects the changed script redefines). `runs: always` scripts are +// re-executed every Apply but must NOT cascade: their execution is free by +// contract (CREATE OR REPLACE), while their dependents replay heavyweight DDL +// (DROP/CREATE TRIGGER takes ACCESS EXCLUSIVE) that deadlocks concurrent +// readers when an Apply runs against a live database. An always script's hash +// is still recorded, so editing its content cascades exactly once. +func selectFromRecordedHashes(scripts map[string]*script, hashes map[string][]byte) map[string]bool { + dirty := map[string]bool{} + for name, s := range scripts { + dirty[name] = !equalBytes(hashes[name], s.hash) + } + changed := true + for changed { + changed = false + for name, s := range scripts { + if dirty[name] { + continue + } + for _, dep := range s.dependsOn { + if dirty[dep] { + dirty[name] = true + changed = true + break + } + } + } + } + selected := map[string]bool{} + for name, s := range scripts { + selected[name] = s.always || dirty[name] + } + return selected +} + +type scriptPhaseOptions struct { + Scope string + Ordered []*script + Phase scriptPhase +} + +func runScriptPhase(ctx context.Context, db *sql.DB, opts scriptPhaseOptions) error { + for _, s := range opts.Ordered { + if s.phase != opts.Phase { + continue + } + if s.transactional { + if err := runTransactionalScript(ctx, db, opts.Scope, s); err != nil { + return err + } + continue + } + if _, err := db.ExecContext(ctx, s.content); err != nil { + return fmt.Errorf("execute non-transactional SQL migration %s: %w", s.path, err) + } + if err := recordScript(ctx, db, opts.Scope, s); err != nil { + return err + } + } + return nil +} + +// retryOnLockContention runs fn, retrying on transient lock contention (a +// detected deadlock, a lock_timeout, or a serialization failure) with linear +// backoff. A DDL statement (a transactional script, a dependent-view DROP, or a +// security reconciliation) takes ACCESS EXCLUSIVE on tables a concurrently +// running process may be reading; the bounded lock_timeout converts the +// resulting hang into a retryable 55P03, and a deadlock (40P01) aborts one party +// — either way retrying lets the migration win once the reader's short query +// completes, instead of failing the whole run. fn MUST be idempotent. +func retryOnLockContention(ctx context.Context, desc string, fn func() error) error { + var lastErr error + for attempt := 1; attempt <= migrationMaxAttempts; attempt++ { + err := fn() + if err == nil { + return nil + } + if !isRetryableMigrationErr(err) { + return err + } + lastErr = err + if attempt < migrationMaxAttempts { + logger.Debugf("%s hit lock contention (attempt %d/%d): %v", desc, attempt, migrationMaxAttempts, err) + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(time.Duration(attempt) * migrationRetryBackoff): + } + } + } + return fmt.Errorf("gave up after %d attempts under lock contention: %w", migrationMaxAttempts, lastErr) +} + +// runTransactionalScript applies one transactional migration under a bounded +// lock_timeout, retrying on lock contention. The view/trigger scripts split out +// of the historical 50_views_and_triggers monolith are the canonical DDL that +// contends with live readers. +func runTransactionalScript(ctx context.Context, db *sql.DB, scope string, s *script) error { + return retryOnLockContention(ctx, "execute SQL migration "+s.path, func() error { + return applyTransactionalScript(ctx, db, scope, s) + }) +} + +// applyTransactionalScript runs the script (and records it) in a single +// transaction with a bounded lock_timeout so it never blocks indefinitely. +func applyTransactionalScript(ctx context.Context, db *sql.DB, scope string, s *script) error { + tx, err := db.BeginTx(ctx, nil) + if err != nil { + return fmt.Errorf("begin SQL migration %s: %w", s.path, err) + } + if _, err := tx.ExecContext(ctx, "SET LOCAL lock_timeout = '"+migrationLockTimeout+"'"); err != nil { + _ = tx.Rollback() + return fmt.Errorf("set lock_timeout for SQL migration %s: %w", s.path, err) + } + if _, err := tx.ExecContext(ctx, s.content); err != nil { + _ = tx.Rollback() + return fmt.Errorf("execute SQL migration %s: %w", s.path, err) + } + if err := recordScript(ctx, tx, scope, s); err != nil { + _ = tx.Rollback() + return err + } + if err := tx.Commit(); err != nil { + return fmt.Errorf("commit SQL migration %s: %w", s.path, err) + } + return nil +} + +// isRetryableMigrationErr reports whether err is transient lock contention that a +// retry can clear: a detected deadlock, a lock_timeout, or a serialization +// failure. A wrapped *pq.Error carries the SQLSTATE that classifies it. +func isRetryableMigrationErr(err error) bool { + var pqErr *pq.Error + if !errors.As(err, &pqErr) { + return false + } + switch pqErr.Code { + case "40P01", // deadlock_detected + "55P03", // lock_not_available (lock_timeout fired) + "40001": // serialization_failure + return true + default: + return false + } +} + +type execer interface { + ExecContext(context.Context, string, ...any) (sql.Result, error) +} + +func recordScript(ctx context.Context, db execer, scope string, s *script) error { + _, err := db.ExecContext(ctx, ` +INSERT INTO schema_migration_scripts(scope, path, hash, updated_at) +VALUES ($1, $2, $3, now()) +ON CONFLICT (scope, path) DO UPDATE SET hash = EXCLUDED.hash, updated_at = now()`, scope, s.path, s.hash) + if err != nil { + return fmt.Errorf("record SQL migration %s: %w", s.path, err) + } + return nil +} + +func equalBytes(a, b []byte) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} diff --git a/database/migrate/scripts_test.go b/database/migrate/scripts_test.go new file mode 100644 index 00000000..81fd723f --- /dev/null +++ b/database/migrate/scripts_test.go @@ -0,0 +1,145 @@ +package migrate + +import ( + "errors" + "fmt" + "strings" + "testing" + "testing/fstest" + + "github.com/lib/pq" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestIsRetryableMigrationErr(t *testing.T) { + cases := []struct { + name string + err error + want bool + }{ + {"deadlock", &pq.Error{Code: "40P01"}, true}, + {"lock_timeout", &pq.Error{Code: "55P03"}, true}, + {"serialization_failure", &pq.Error{Code: "40001"}, true}, + {"wrapped deadlock", fmt.Errorf("execute SQL migration x: %w", &pq.Error{Code: "40P01"}), true}, + {"undefined_table", &pq.Error{Code: "42P01"}, false}, + {"syntax_error", &pq.Error{Code: "42601"}, false}, + {"non-pq error", errors.New("connection refused"), false}, + {"nil", nil, false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, isRetryableMigrationErr(tc.err)) + }) + } +} + +// A `runs: always` script re-executes every Apply, but that must not drag its +// dependents (heavyweight trigger/view DDL) into every Apply with it — only a +// genuine content change cascades, and then exactly once. +func TestSelectFromRecordedHashesAlwaysDoesNotCascade(t *testing.T) { + filesystem := fstest.MapFS{ + "migrations/functions.sql": &fstest.MapFile{Data: []byte("-- phase: pre\n-- runs: always\nSELECT 1")}, + "migrations/triggers.sql": &fstest.MapFile{Data: []byte("-- dependsOn: functions.sql\nSELECT 2")}, + "migrations/views.sql": &fstest.MapFile{Data: []byte("-- dependsOn: triggers.sql\nSELECT 3")}, + } + scripts, err := loadScripts(filesystem, "migrations") + require.NoError(t, err) + recorded := func(names ...string) map[string][]byte { + hashes := map[string][]byte{} + for _, name := range names { + hashes[name] = scripts[name].hash + } + return hashes + } + + // Steady state: every hash recorded — only the always script runs. + selected := selectFromRecordedHashes(scripts, recorded("functions.sql", "triggers.sql", "views.sql")) + assert.Equal(t, map[string]bool{"functions.sql": true, "triggers.sql": false, "views.sql": false}, selected) + + // The always script's content changed since last Apply: cascades once. + stale := recorded("functions.sql", "triggers.sql", "views.sql") + stale["functions.sql"] = []byte("previous-content-hash") + selected = selectFromRecordedHashes(scripts, stale) + assert.Equal(t, map[string]bool{"functions.sql": true, "triggers.sql": true, "views.sql": true}, selected) + + // A mid-chain change cascades downstream only. + stale = recorded("functions.sql", "triggers.sql", "views.sql") + stale["triggers.sql"] = []byte("previous-content-hash") + selected = selectFromRecordedHashes(scripts, stale) + assert.Equal(t, map[string]bool{"functions.sql": true, "triggers.sql": true, "views.sql": true}, selected) + + // Fresh database: nothing recorded — everything runs. + selected = selectFromRecordedHashes(scripts, map[string][]byte{}) + assert.Equal(t, map[string]bool{"functions.sql": true, "triggers.sql": true, "views.sql": true}, selected) +} + +func TestLoadScriptsDefaultsToPostAndOrdersTransitively(t *testing.T) { + filesystem := fstest.MapFS{ + "migrations/a.sql": &fstest.MapFile{Data: []byte("-- phase: pre\nSELECT 1")}, + "migrations/b.sql": &fstest.MapFile{Data: []byte("-- phase: pre\n-- dependsOn: a.sql\nSELECT 2")}, + "migrations/c.sql": &fstest.MapFile{Data: []byte("-- dependsOn: b.sql\nSELECT 3")}, + } + scripts, err := loadScripts(filesystem, "migrations") + require.NoError(t, err) + assert.Equal(t, phasePre, scripts["a.sql"].phase) + assert.Equal(t, phasePost, scripts["c.sql"].phase) + + ordered, err := topologicalScripts(scripts, nil) + require.NoError(t, err) + require.Len(t, ordered, 3) + assert.Equal(t, []string{"a.sql", "b.sql", "c.sql"}, []string{ordered[0].path, ordered[1].path, ordered[2].path}) +} + +func TestScriptGraphValidation(t *testing.T) { + tests := []struct { + name string + files fstest.MapFS + err string + }{ + { + name: "missing", + files: fstest.MapFS{"migrations/a.sql": &fstest.MapFile{Data: []byte("-- dependsOn: missing.sql\nSELECT 1")}}, + err: "depends on missing script", + }, + { + name: "cycle", + files: fstest.MapFS{ + "migrations/a.sql": &fstest.MapFile{Data: []byte("-- dependsOn: b.sql\nSELECT 1")}, + "migrations/b.sql": &fstest.MapFile{Data: []byte("-- dependsOn: a.sql\nSELECT 2")}, + }, + err: "dependency cycle", + }, + { + name: "phase inversion", + files: fstest.MapFS{ + "migrations/a.sql": &fstest.MapFile{Data: []byte("SELECT 1")}, + "migrations/b.sql": &fstest.MapFile{Data: []byte("-- phase: pre\n-- dependsOn: a.sql\nSELECT 2")}, + }, + err: "cannot depend on post script", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := loadScripts(tt.files, "migrations") + require.Error(t, err) + assert.True(t, strings.Contains(err.Error(), tt.err), err.Error()) + }) + } +} + +func TestParseScriptDirectives(t *testing.T) { + s, err := parseScript("x.sql", ` +-- comment +-- phase: pre +-- runs: always +-- transaction: false +-- dependsOn: a.sql, nested/b.sql + +SELECT 1`) + require.NoError(t, err) + assert.Equal(t, phasePre, s.phase) + assert.True(t, s.always) + assert.False(t, s.transactional) + assert.Equal(t, []string{"a.sql", "nested/b.sql"}, s.dependsOn) +} diff --git a/database/migrate/security.go b/database/migrate/security.go new file mode 100644 index 00000000..5834f812 --- /dev/null +++ b/database/migrate/security.go @@ -0,0 +1,410 @@ +package migrate + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "sort" + "strings" + + "github.com/lib/pq" +) + +type managedSecurityState struct { + Permissions []permissionSpec `json:"permissions,omitempty"` + Memberships []membershipSpec `json:"memberships,omitempty"` +} + +type membershipSpec struct { + Member string `json:"member"` + Parent string `json:"parent"` +} + +type permissionTarget struct { + Kind, Schema, Object, Column string +} + +var allowedPrivileges = map[string]map[string]bool{ + "schema": {"USAGE": true, "CREATE": true}, + "table": {"SELECT": true, "INSERT": true, "UPDATE": true, "DELETE": true, "TRUNCATE": true, "REFERENCES": true, "TRIGGER": true, "MAINTAIN": true}, + "column": {"SELECT": true, "INSERT": true, "UPDATE": true, "REFERENCES": true}, + "sequence": {"USAGE": true, "SELECT": true, "UPDATE": true}, +} + +func validatePermission(p permissionSpec) error { + if p.Grantee == "" { + return fmt.Errorf("permission grantee is empty") + } + target, err := parsePermissionTarget(p.Target) + if err != nil { + return err + } + allowed := allowedPrivileges[target.Kind] + if len(p.Privileges) == 0 { + return fmt.Errorf("permission for %s has no privileges", p.Target) + } + for _, privilege := range p.Privileges { + if privilege != "ALL" && !allowed[privilege] { + return fmt.Errorf("privilege %q is not valid for %s", privilege, target.Kind) + } + } + return nil +} + +func parsePermissionTarget(value string) (permissionTarget, error) { + parts := strings.SplitN(value, ":", 2) + if len(parts) != 2 { + return permissionTarget{}, fmt.Errorf("unsupported permission target %q", value) + } + t := permissionTarget{Kind: parts[0]} + names := strings.Split(parts[1], ".") + switch t.Kind { + case "schema": + if len(names) != 1 { + return t, fmt.Errorf("invalid schema permission target %q", value) + } + t.Schema = names[0] + case "table", "sequence": + if len(names) != 2 { + return t, fmt.Errorf("invalid %s permission target %q", t.Kind, value) + } + t.Schema, t.Object = names[0], names[1] + case "column": + if len(names) != 3 { + return t, fmt.Errorf("invalid column permission target %q", value) + } + t.Schema, t.Object, t.Column = names[0], names[1], names[2] + default: + return t, fmt.Errorf("unsupported permission target kind %q", t.Kind) + } + return t, nil +} + +func reconcileSecurity(ctx context.Context, db *sql.DB, scope string, spec securitySpec) error { + tx, err := db.BeginTx(ctx, nil) + if err != nil { + return fmt.Errorf("begin security reconciliation: %w", err) + } + defer tx.Rollback() //nolint:errcheck + + // GRANT/REVOKE take strong locks on the targeted objects; bound the wait so + // reconciliation cannot camp against live traffic (the caller retries). + if _, err := tx.ExecContext(ctx, "SET LOCAL lock_timeout = '"+migrationLockTimeout+"'"); err != nil { + return fmt.Errorf("set lock_timeout for security reconciliation: %w", err) + } + + previous, err := readSecurityState(ctx, tx, scope) + if err != nil { + return err + } + if err := reconcileRoles(ctx, tx, spec.Roles); err != nil { + return err + } + desiredMemberships := roleMemberships(spec.Roles) + if err := reconcileMemberships(ctx, tx, previous.Memberships, desiredMemberships); err != nil { + return err + } + desiredPermissions, err := normalizePermissions(spec.Permissions) + if err != nil { + return err + } + if err := reconcilePermissions(ctx, tx, previous.Permissions, desiredPermissions); err != nil { + return err + } + state := managedSecurityState{Permissions: desiredPermissions, Memberships: desiredMemberships} + data, err := json.Marshal(state) + if err != nil { + return fmt.Errorf("marshal security state: %w", err) + } + if _, err := tx.ExecContext(ctx, ` +INSERT INTO schema_migration_security(scope, state, updated_at) +VALUES ($1, $2::jsonb, now()) +ON CONFLICT (scope) DO UPDATE SET state = EXCLUDED.state, updated_at = now()`, scope, string(data)); err != nil { + return fmt.Errorf("save security state: %w", err) + } + if err := tx.Commit(); err != nil { + return fmt.Errorf("commit security reconciliation: %w", err) + } + return nil +} + +func readSecurityState(ctx context.Context, tx *sql.Tx, scope string) (managedSecurityState, error) { + var raw []byte + err := tx.QueryRowContext(ctx, `SELECT state FROM schema_migration_security WHERE scope = $1`, scope).Scan(&raw) + if err == sql.ErrNoRows { + return managedSecurityState{}, nil + } + if err != nil { + return managedSecurityState{}, fmt.Errorf("read security state: %w", err) + } + var state managedSecurityState + if err := json.Unmarshal(raw, &state); err != nil { + return state, fmt.Errorf("decode security state: %w", err) + } + return state, nil +} + +func reconcileRoles(ctx context.Context, tx *sql.Tx, roles []roleSpec) error { + for _, role := range roles { + var exists bool + if err := tx.QueryRowContext(ctx, `SELECT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = $1)`, role.Name).Scan(&exists); err != nil { + return fmt.Errorf("inspect role %q: %w", role.Name, err) + } + if role.External { + if !exists { + return fmt.Errorf("external role %q does not exist", role.Name) + } + continue + } + quoted := pq.QuoteIdentifier(role.Name) + if !exists { + if _, err := tx.ExecContext(ctx, "CREATE ROLE "+quoted); err != nil { + return fmt.Errorf("create role %q: %w", role.Name, err) + } + } + attributes := []string{ + boolKeyword(role.Superuser, "SUPERUSER"), + boolKeyword(role.CreateDB, "CREATEDB"), + boolKeyword(role.CreateRole, "CREATEROLE"), + boolKeyword(role.Inherit, "INHERIT"), + boolKeyword(role.Login, "LOGIN"), + boolKeyword(role.Replication, "REPLICATION"), + boolKeyword(role.BypassRLS, "BYPASSRLS"), + fmt.Sprintf("CONNECTION LIMIT %d", role.ConnLimit), + } + if _, err := tx.ExecContext(ctx, "ALTER ROLE "+quoted+" WITH "+strings.Join(attributes, " ")); err != nil { + return fmt.Errorf("alter role %q: %w", role.Name, err) + } + comment := "NULL" + if role.Comment != "" { + comment = pq.QuoteLiteral(role.Comment) + } + if _, err := tx.ExecContext(ctx, "COMMENT ON ROLE "+quoted+" IS "+comment); err != nil { + return fmt.Errorf("comment role %q: %w", role.Name, err) + } + } + return nil +} + +func boolKeyword(value bool, enabled string) string { + if value { + return enabled + } + return "NO" + enabled +} + +func roleMemberships(roles []roleSpec) []membershipSpec { + var memberships []membershipSpec + for _, role := range roles { + if role.External { + continue + } + for _, parent := range role.MemberOf { + memberships = append(memberships, membershipSpec{Member: role.Name, Parent: parent}) + } + } + sort.Slice(memberships, func(i, j int) bool { + return memberships[i].Member+"\x00"+memberships[i].Parent < memberships[j].Member+"\x00"+memberships[j].Parent + }) + return memberships +} + +func reconcileMemberships(ctx context.Context, tx *sql.Tx, previous, desired []membershipSpec) error { + want := map[string]membershipSpec{} + for _, m := range desired { + want[m.Member+"\x00"+m.Parent] = m + if _, err := tx.ExecContext(ctx, "GRANT "+pq.QuoteIdentifier(m.Parent)+" TO "+pq.QuoteIdentifier(m.Member)); err != nil { + return fmt.Errorf("grant role %q to %q: %w", m.Parent, m.Member, err) + } + } + for _, m := range previous { + if _, ok := want[m.Member+"\x00"+m.Parent]; ok { + continue + } + if _, err := tx.ExecContext(ctx, "REVOKE "+pq.QuoteIdentifier(m.Parent)+" FROM "+pq.QuoteIdentifier(m.Member)); err != nil { + return fmt.Errorf("revoke role %q from %q: %w", m.Parent, m.Member, err) + } + } + return nil +} + +func normalizePermissions(raw []permissionSpec) ([]permissionSpec, error) { + grouped := map[string]permissionSpec{} + for _, p := range raw { + if err := validatePermission(p); err != nil { + return nil, err + } + key := p.Grantee + "\x00" + p.Target + current := grouped[key] + current.Grantee, current.Target = p.Grantee, p.Target + current.Grantable = current.Grantable || p.Grantable + allowed := allowedPrivileges[strings.SplitN(p.Target, ":", 2)[0]] + seen := map[string]bool{} + for _, privilege := range append(current.Privileges, p.Privileges...) { + if privilege == "ALL" { + for candidate := range allowed { + seen[candidate] = true + } + } else { + seen[privilege] = true + } + } + current.Privileges = current.Privileges[:0] + for privilege := range seen { + current.Privileges = append(current.Privileges, privilege) + } + sort.Strings(current.Privileges) + grouped[key] = current + } + out := make([]permissionSpec, 0, len(grouped)) + for _, p := range grouped { + out = append(out, p) + } + sort.Slice(out, func(i, j int) bool { return out[i].Grantee+"\x00"+out[i].Target < out[j].Grantee+"\x00"+out[j].Target }) + return out, nil +} + +func reconcilePermissions(ctx context.Context, tx *sql.Tx, previous, desired []permissionSpec) error { + want := map[string]permissionSpec{} + for _, p := range desired { + want[p.Grantee+"\x00"+p.Target] = p + current, err := currentPrivileges(ctx, tx, p) + if err != nil { + return err + } + desiredSet := map[string]bool{} + for _, privilege := range p.Privileges { + desiredSet[privilege] = true + grantable, exists := current[privilege] + switch { + case !exists || p.Grantable && !grantable: + if err := grantPrivilege(ctx, tx, p, privilege); err != nil { + return err + } + case !p.Grantable && grantable: + if err := revokeGrantOption(ctx, tx, p, privilege); err != nil { + return err + } + } + } + for privilege := range current { + if !desiredSet[privilege] && allowedPrivileges[strings.SplitN(p.Target, ":", 2)[0]][privilege] { + if err := revokePrivilege(ctx, tx, p, privilege); err != nil { + return err + } + } + } + } + for _, p := range previous { + if _, ok := want[p.Grantee+"\x00"+p.Target]; ok { + continue + } + for _, privilege := range p.Privileges { + if err := revokePrivilege(ctx, tx, p, privilege); err != nil { + return err + } + } + } + return nil +} + +func currentPrivileges(ctx context.Context, tx *sql.Tx, p permissionSpec) (_ map[string]bool, returnErr error) { + t, err := parsePermissionTarget(p.Target) + if err != nil { + return nil, err + } + var query string + var args []any + switch t.Kind { + case "schema": + query = `SELECT x.privilege_type, x.is_grantable FROM pg_namespace n CROSS JOIN LATERAL aclexplode(COALESCE(n.nspacl, acldefault('n', n.nspowner))) x LEFT JOIN pg_roles r ON r.oid = x.grantee WHERE n.nspname = $1 AND COALESCE(r.rolname, 'PUBLIC') = $2` + args = []any{t.Schema, p.Grantee} + case "table", "sequence": + query = `SELECT x.privilege_type, x.is_grantable FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace CROSS JOIN LATERAL aclexplode(COALESCE(c.relacl, acldefault(CASE WHEN c.relkind = 'S' THEN 'S'::"char" ELSE 'r'::"char" END, c.relowner))) x LEFT JOIN pg_roles r ON r.oid = x.grantee WHERE n.nspname = $1 AND c.relname = $2 AND COALESCE(r.rolname, 'PUBLIC') = $3` + args = []any{t.Schema, t.Object, p.Grantee} + case "column": + query = `SELECT x.privilege_type, x.is_grantable FROM pg_attribute a JOIN pg_class c ON c.oid = a.attrelid JOIN pg_namespace n ON n.oid = c.relnamespace CROSS JOIN LATERAL aclexplode(COALESCE(a.attacl, acldefault('c', c.relowner))) x LEFT JOIN pg_roles r ON r.oid = x.grantee WHERE n.nspname = $1 AND c.relname = $2 AND a.attname = $3 AND COALESCE(r.rolname, 'PUBLIC') = $4` + args = []any{t.Schema, t.Object, t.Column, p.Grantee} + } + rows, err := tx.QueryContext(ctx, query, args...) + if err != nil { + return nil, fmt.Errorf("inspect permissions for %s on %s: %w", p.Grantee, p.Target, err) + } + defer func() { returnErr = errors.Join(returnErr, rows.Close()) }() + out := map[string]bool{} + for rows.Next() { + var privilege string + var grantable bool + if err := rows.Scan(&privilege, &grantable); err != nil { + return nil, err + } + out[strings.ToUpper(privilege)] = grantable + } + return out, rows.Err() +} + +func grantPrivilege(ctx context.Context, tx *sql.Tx, p permissionSpec, privilege string) error { + statement, err := permissionStatement("GRANT", p, privilege) + if err != nil { + return err + } + if p.Grantable { + statement += " WITH GRANT OPTION" + } + if _, err := tx.ExecContext(ctx, statement); err != nil { + return fmt.Errorf("grant %s on %s to %s: %w", privilege, p.Target, p.Grantee, err) + } + return nil +} + +func revokePrivilege(ctx context.Context, tx *sql.Tx, p permissionSpec, privilege string) error { + statement, err := permissionStatement("REVOKE", p, privilege) + if err != nil { + return err + } + if _, err := tx.ExecContext(ctx, statement); err != nil { + return fmt.Errorf("revoke %s on %s from %s: %w", privilege, p.Target, p.Grantee, err) + } + return nil +} + +func revokeGrantOption(ctx context.Context, tx *sql.Tx, p permissionSpec, privilege string) error { + statement, err := permissionStatement("REVOKE GRANT OPTION FOR", p, privilege) + if err != nil { + return err + } + if _, err := tx.ExecContext(ctx, statement); err != nil { + return fmt.Errorf("revoke grant option for %s on %s from %s: %w", privilege, p.Target, p.Grantee, err) + } + return nil +} + +func permissionStatement(action string, p permissionSpec, privilege string) (string, error) { + t, err := parsePermissionTarget(p.Target) + if err != nil { + return "", err + } + grantee := pq.QuoteIdentifier(p.Grantee) + if p.Grantee == "PUBLIC" { + grantee = "PUBLIC" + } + preposition := " TO " + if strings.HasPrefix(action, "REVOKE") { + preposition = " FROM " + } + var object string + switch t.Kind { + case "schema": + object = "SCHEMA " + pq.QuoteIdentifier(t.Schema) + case "table": + object = "TABLE " + pq.QuoteIdentifier(t.Schema) + "." + pq.QuoteIdentifier(t.Object) + case "sequence": + object = "SEQUENCE " + pq.QuoteIdentifier(t.Schema) + "." + pq.QuoteIdentifier(t.Object) + case "column": + privilege += " (" + pq.QuoteIdentifier(t.Column) + ")" + object = "TABLE " + pq.QuoteIdentifier(t.Schema) + "." + pq.QuoteIdentifier(t.Object) + } + return action + " " + privilege + " ON " + object + preposition + grantee, nil +} diff --git a/database/migrate/security_hcl.go b/database/migrate/security_hcl.go new file mode 100644 index 00000000..ea613972 --- /dev/null +++ b/database/migrate/security_hcl.go @@ -0,0 +1,434 @@ +package migrate + +import ( + "fmt" + "io/fs" + "path" + "sort" + "strings" + + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/hcl/v2/hclparse" + "github.com/hashicorp/hcl/v2/hclsyntax" + "github.com/zclconf/go-cty/cty" + "github.com/zclconf/go-cty/cty/gocty" +) + +type roleSpec struct { + Name string + External bool + Login bool + Inherit bool + Superuser bool + CreateDB bool + CreateRole bool + Replication bool + BypassRLS bool + ConnLimit int64 + Comment string + MemberOf []string +} + +type permissionSpec struct { + Grantee string `json:"grantee"` + Target string `json:"target"` + Privileges []string `json:"privileges"` + Grantable bool `json:"grantable"` +} + +type securitySpec struct { + Roles []roleSpec + Permissions []permissionSpec +} + +type parsedHCLFile struct { + name string + data []byte + body *hclsyntax.Body +} + +func loadHCL(schemaFS fs.FS, dir string, input map[string]cty.Value) (*hclparse.Parser, securitySpec, error) { + var files []parsedHCLFile + err := fs.WalkDir(schemaFS, dir, func(name string, entry fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if entry.IsDir() || strings.ToLower(path.Ext(name)) != ".hcl" { + return nil + } + data, err := fs.ReadFile(schemaFS, name) + if err != nil { + return fmt.Errorf("read HCL %s: %w", name, err) + } + file, diagnostics := hclsyntax.ParseConfig(data, name, hcl.Pos{Line: 1, Column: 1}) + if diagnostics.HasErrors() { + return diagnostics + } + files = append(files, parsedHCLFile{name: name, data: data, body: file.Body.(*hclsyntax.Body)}) + return nil + }) + if err != nil { + return nil, securitySpec{}, fmt.Errorf("load HCL schemas: %w", err) + } + if len(files) == 0 { + return nil, securitySpec{}, fmt.Errorf("no HCL schema files found in %q", dir) + } + sort.Slice(files, func(i, j int) bool { return files[i].name < files[j].name }) + + ctx := securityEvalContext(files, input) + security, err := decodeSecurity(files, ctx) + if err != nil { + return nil, securitySpec{}, err + } + + parser := hclparse.NewParser() + for _, file := range files { + filtered := append([]byte(nil), file.data...) + for _, block := range file.body.Blocks { + if block.Type != "role" && block.Type != "permission" { + continue + } + r := block.Range() + for i := r.Start.Byte; i < r.End.Byte && i < len(filtered); i++ { + if filtered[i] != '\n' && filtered[i] != '\r' { + filtered[i] = ' ' + } + } + } + if _, diagnostics := parser.ParseHCL(filtered, file.name); diagnostics.HasErrors() { + return nil, securitySpec{}, diagnostics + } + } + return parser, security, nil +} + +func securityEvalContext(files []parsedHCLFile, input map[string]cty.Value) *hcl.EvalContext { + schemas := map[string]cty.Value{} + tables := map[string]cty.Value{} + qualifiedTables := map[string]map[string]cty.Value{} + sequences := map[string]cty.Value{} + qualifiedSequences := map[string]map[string]cty.Value{} + roles := map[string]cty.Value{} + for _, file := range files { + for _, block := range file.body.Blocks { + if len(block.Labels) == 0 { + continue + } + name := block.Labels[len(block.Labels)-1] + qualifier := "" + if len(block.Labels) > 1 { + qualifier = block.Labels[0] + } + switch block.Type { + case "schema": + schemas[name] = refValue("schema:" + name) + case "role": + roles[name] = refValue("role:" + name) + case "table": + schemaName := blockSchema(block, "public") + columns := map[string]cty.Value{} + for _, child := range block.Body.Blocks { + if child.Type == "column" && len(child.Labels) > 0 { + columns[child.Labels[0]] = refValue("column:" + schemaName + "." + name + "." + child.Labels[0]) + } + } + table := cty.ObjectVal(map[string]cty.Value{ + "__ref": cty.StringVal("table:" + schemaName + "." + name), + "column": objectValue(columns), + }) + if qualifier == "" { + tables[name] = table + } else { + if qualifiedTables[qualifier] == nil { + qualifiedTables[qualifier] = map[string]cty.Value{} + } + qualifiedTables[qualifier][name] = table + } + case "sequence": + schemaName := blockSchema(block, "public") + sequence := refValue("sequence:" + schemaName + "." + name) + if qualifier == "" { + sequences[name] = sequence + } else { + if qualifiedSequences[qualifier] == nil { + qualifiedSequences[qualifier] = map[string]cty.Value{} + } + qualifiedSequences[qualifier][name] = sequence + } + } + } + } + for qualifier, values := range qualifiedTables { + tables[qualifier] = objectValue(values) + } + for qualifier, values := range qualifiedSequences { + sequences[qualifier] = objectValue(values) + } + vars := map[string]cty.Value{ + "schema": objectValue(schemas), + "table": objectValue(tables), + "sequence": objectValue(sequences), + "role": objectValue(roles), + "var": objectValue(input), + "PUBLIC": cty.StringVal("PUBLIC"), + "ALL": cty.StringVal("ALL"), + "SELECT": cty.StringVal("SELECT"), + "INSERT": cty.StringVal("INSERT"), + "UPDATE": cty.StringVal("UPDATE"), + "DELETE": cty.StringVal("DELETE"), + "TRUNCATE": cty.StringVal("TRUNCATE"), + "REFERENCES": cty.StringVal("REFERENCES"), + "TRIGGER": cty.StringVal("TRIGGER"), + "CREATE": cty.StringVal("CREATE"), + "USAGE": cty.StringVal("USAGE"), + "MAINTAIN": cty.StringVal("MAINTAIN"), + } + return &hcl.EvalContext{Variables: vars} +} + +func blockSchema(block *hclsyntax.Block, fallback string) string { + attr, ok := block.Body.Attributes["schema"] + if !ok { + return fallback + } + traversal, diagnostics := hcl.AbsTraversalForExpr(attr.Expr) + if diagnostics.HasErrors() || len(traversal) < 2 { + return fallback + } + if root, ok := traversal[0].(hcl.TraverseRoot); !ok || root.Name != "schema" { + return fallback + } + if step, ok := traversal[1].(hcl.TraverseAttr); ok { + return step.Name + } + return fallback +} + +func decodeSecurity(files []parsedHCLFile, ctx *hcl.EvalContext) (securitySpec, error) { + var out securitySpec + seenRoles := map[string]bool{} + for _, file := range files { + for _, block := range file.body.Blocks { + switch block.Type { + case "role": + role, err := decodeRole(block, ctx) + if err != nil { + return out, fmt.Errorf("%s: %w", file.name, err) + } + if seenRoles[role.Name] { + return out, fmt.Errorf("duplicate role %q", role.Name) + } + seenRoles[role.Name] = true + out.Roles = append(out.Roles, role) + case "permission": + permissions, err := decodePermission(block, ctx) + if err != nil { + return out, fmt.Errorf("%s: %w", file.name, err) + } + out.Permissions = append(out.Permissions, permissions...) + } + } + } + return out, nil +} + +func decodeRole(block *hclsyntax.Block, ctx *hcl.EvalContext) (roleSpec, error) { + if len(block.Labels) != 1 { + return roleSpec{}, fmt.Errorf("role block requires exactly one label") + } + r := roleSpec{Name: block.Labels[0], Inherit: true, ConnLimit: -1} + allowed := map[string]bool{"external": true, "login": true, "inherit": true, "superuser": true, "create_db": true, "create_role": true, "replication": true, "bypass_rls": true, "conn_limit": true, "comment": true, "member_of": true} + for name, attr := range block.Body.Attributes { + if !allowed[name] { + return r, fmt.Errorf("role %q has unsupported attribute %q", r.Name, name) + } + value, diagnostics := attr.Expr.Value(ctx) + if diagnostics.HasErrors() { + return r, diagnostics + } + var err error + switch name { + case "external": + r.External, err = boolValue(value) + case "login": + r.Login, err = boolValue(value) + case "inherit": + r.Inherit, err = boolValue(value) + case "superuser": + r.Superuser, err = boolValue(value) + case "create_db": + r.CreateDB, err = boolValue(value) + case "create_role": + r.CreateRole, err = boolValue(value) + case "replication": + r.Replication, err = boolValue(value) + case "bypass_rls": + r.BypassRLS, err = boolValue(value) + case "conn_limit": + r.ConnLimit, err = intValue(value) + case "comment": + r.Comment, err = stringValue(value) + case "member_of": + r.MemberOf, err = refList(value, "role:") + } + if err != nil { + return r, fmt.Errorf("role %q attribute %s: %w", r.Name, name, err) + } + } + return r, nil +} + +func decodePermission(block *hclsyntax.Block, ctx *hcl.EvalContext) ([]permissionSpec, error) { + allowed := map[string]bool{"to": true, "for": true, "privileges": true, "grantable": true, "for_each": true} + for name := range block.Body.Attributes { + if !allowed[name] { + return nil, fmt.Errorf("permission has unsupported attribute %q", name) + } + } + for _, required := range []string{"to", "for", "privileges"} { + if _, ok := block.Body.Attributes[required]; !ok { + return nil, fmt.Errorf("permission is missing %q", required) + } + } + contexts := []*hcl.EvalContext{ctx} + if attr, ok := block.Body.Attributes["for_each"]; ok { + value, diagnostics := attr.Expr.Value(ctx) + if diagnostics.HasErrors() { + return nil, diagnostics + } + contexts = nil + it := value.ElementIterator() + for it.Next() { + _, item := it.Element() + child := ctx.NewChild() + child.Variables = map[string]cty.Value{"each": cty.ObjectVal(map[string]cty.Value{"value": item})} + contexts = append(contexts, child) + } + } + var out []permissionSpec + for _, evalCtx := range contexts { + to, diagnostics := block.Body.Attributes["to"].Expr.Value(evalCtx) + if diagnostics.HasErrors() { + return nil, diagnostics + } + grantee, err := reference(to) + if err != nil { + return nil, fmt.Errorf("permission to: %w", err) + } + grantee = strings.TrimPrefix(grantee, "role:") + targetValue, diagnostics := block.Body.Attributes["for"].Expr.Value(evalCtx) + if diagnostics.HasErrors() { + return nil, diagnostics + } + target, err := reference(targetValue) + if err != nil { + return nil, fmt.Errorf("permission for: %w", err) + } + privValue, diagnostics := block.Body.Attributes["privileges"].Expr.Value(evalCtx) + if diagnostics.HasErrors() { + return nil, diagnostics + } + privileges, err := stringList(privValue) + if err != nil { + return nil, fmt.Errorf("permission privileges: %w", err) + } + grantable := false + if attr, ok := block.Body.Attributes["grantable"]; ok { + value, diagnostics := attr.Expr.Value(evalCtx) + if diagnostics.HasErrors() { + return nil, diagnostics + } + grantable, err = boolValue(value) + if err != nil { + return nil, fmt.Errorf("permission grantable: %w", err) + } + } + p := permissionSpec{Grantee: grantee, Target: target, Privileges: privileges, Grantable: grantable} + if err := validatePermission(p); err != nil { + return nil, err + } + out = append(out, p) + } + return out, nil +} + +func refValue(ref string) cty.Value { + return cty.ObjectVal(map[string]cty.Value{"__ref": cty.StringVal(ref)}) +} + +func objectValue(values map[string]cty.Value) cty.Value { + if len(values) == 0 { + return cty.EmptyObjectVal + } + return cty.ObjectVal(values) +} + +func reference(value cty.Value) (string, error) { + if value.Type() == cty.String { + return value.AsString(), nil + } + if value.IsNull() || !value.IsKnown() || !value.Type().IsObjectType() { + return "", fmt.Errorf("expected an object reference or string") + } + ref := value.GetAttr("__ref") + if ref.Type() != cty.String { + return "", fmt.Errorf("invalid object reference") + } + return ref.AsString(), nil +} + +func refList(value cty.Value, prefix string) ([]string, error) { + var out []string + it := value.ElementIterator() + for it.Next() { + _, item := it.Element() + ref, err := reference(item) + if err != nil { + return nil, err + } + if !strings.HasPrefix(ref, prefix) { + return nil, fmt.Errorf("expected %s reference, got %q", prefix, ref) + } + out = append(out, strings.TrimPrefix(ref, prefix)) + } + return out, nil +} + +func stringList(value cty.Value) ([]string, error) { + var out []string + it := value.ElementIterator() + for it.Next() { + _, item := it.Element() + v, err := stringValue(item) + if err != nil { + return nil, err + } + out = append(out, strings.ToUpper(v)) + } + return out, nil +} + +func stringValue(value cty.Value) (string, error) { + if value.Type() != cty.String || value.IsNull() || !value.IsKnown() { + return "", fmt.Errorf("expected string") + } + return value.AsString(), nil +} + +func boolValue(value cty.Value) (bool, error) { + if value.Type() != cty.Bool || value.IsNull() || !value.IsKnown() { + return false, fmt.Errorf("expected boolean") + } + return value.True(), nil +} + +func intValue(value cty.Value) (int64, error) { + if value.Type() != cty.Number || value.IsNull() || !value.IsKnown() { + return 0, fmt.Errorf("expected number") + } + var result int64 + if err := gocty.FromCtyValue(value, &result); err != nil { + return 0, err + } + return result, nil +} diff --git a/database/migrate/security_hcl_test.go b/database/migrate/security_hcl_test.go new file mode 100644 index 00000000..7e5eadcd --- /dev/null +++ b/database/migrate/security_hcl_test.go @@ -0,0 +1,103 @@ +package migrate + +import ( + "testing" + "testing/fstest" + + "ariga.io/atlas/sql/postgres" + "ariga.io/atlas/sql/schema" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/zclconf/go-cty/cty" +) + +func TestLoadHCLSeparatesAtlasAndSecurityBlocks(t *testing.T) { + filesystem := fstest.MapFS{"migrations/schema.hcl": &fstest.MapFile{Data: []byte(` +schema "public" {} + +table "connections" { + schema = schema.public + column "id" { type = text } +} +table "profiles" { + schema = schema.public + column "id" { type = text } +} + +role "base" { external = true } +role "reader" { + comment = "Query reader" + member_of = [role.base] +} +permission { + for_each = [table.connections, table.profiles] + for = each.value + to = role.reader + privileges = [SELECT] +} +permission { + for = table.connections.column.id + to = PUBLIC + privileges = [SELECT] + grantable = true +} +`)}} + parser, securitySpec, err := loadHCL(filesystem, "migrations", nil) + require.NoError(t, err) + require.Len(t, securitySpec.Roles, 2) + assert.Equal(t, []string{"base"}, securitySpec.Roles[1].MemberOf) + require.Len(t, securitySpec.Permissions, 3) + assert.Equal(t, "table:public.connections", securitySpec.Permissions[0].Target) + assert.Equal(t, "table:public.profiles", securitySpec.Permissions[1].Target) + assert.Equal(t, "column:public.connections.id", securitySpec.Permissions[2].Target) + assert.Equal(t, "PUBLIC", securitySpec.Permissions[2].Grantee) + + realm := &schema.Realm{} + require.NoError(t, postgres.EvalHCL.Eval(parser, realm, nil), "filtered document must remain valid Atlas HCL") + require.Len(t, realm.Schemas, 1) + assert.Len(t, realm.Schemas[0].Tables, 2) +} + +func TestLoadHCLUsesSharedVariables(t *testing.T) { + filesystem := fstest.MapFS{"schema.hcl": &fstest.MapFile{Data: []byte(` +variable "role_comment" { type = string } +variable "can_grant" { type = bool } +schema "public" {} +table "items" { + schema = schema.public + column "id" { type = text } +} +role "reader" { comment = var.role_comment } +permission { + for = table.items + to = role.reader + privileges = [SELECT] + grantable = var.can_grant +} +`)}} + input := map[string]cty.Value{ + "role_comment": cty.StringVal("from input"), + "can_grant": cty.True, + } + parser, securitySpec, err := loadHCL(filesystem, ".", input) + require.NoError(t, err) + assert.Equal(t, "from input", securitySpec.Roles[0].Comment) + assert.True(t, securitySpec.Permissions[0].Grantable) + realm := &schema.Realm{} + require.NoError(t, postgres.EvalHCL.Eval(parser, realm, input)) +} + +func TestLoadHCLRejectsUnsupportedPermissionTarget(t *testing.T) { + filesystem := fstest.MapFS{"schema.hcl": &fstest.MapFile{Data: []byte(` +schema "public" {} +role "reader" {} +permission { + for = "view:public.summary" + to = role.reader + privileges = [SELECT] +} +`)}} + _, _, err := loadHCL(filesystem, ".", nil) + require.Error(t, err) + assert.Contains(t, err.Error(), "unsupported permission target kind") +} diff --git a/database/migrate/sqlite/apply.go b/database/migrate/sqlite/apply.go new file mode 100644 index 00000000..e17f3255 --- /dev/null +++ b/database/migrate/sqlite/apply.go @@ -0,0 +1,179 @@ +// Package sqlite reconciles SQLite tables with programmatic Atlas declarations. +// Filesystem-backed HCL migrations use the target-aware parent migrate package. +package sqlite + +import ( + "context" + "errors" + "fmt" + "slices" + "strings" + + atlasmigrate "ariga.io/atlas/sql/migrate" + "ariga.io/atlas/sql/schema" + atlas "ariga.io/atlas/sql/sqlite" + "github.com/flanksource/commons/logger" +) + +type ReconcileOptions struct { + AllowDrops bool + // AllowRebuilds retains source compatibility; supported rebuilds are automatic. + AllowRebuilds bool +} + +// ReconcileTables brings database's tables up to the tables declared: Atlas inspects +// each declared table, diffs it against its declaration, and applies the plan. +// Column and table drops are skipped unless AllowDrops is enabled. Other supported +// changes may rebuild tables. Views are outside the declared table schema. +// +// A column may be declared by its raw SQLite type alone (ColumnType.Raw); ReconcileTables +// parses it as inspection does, so a declaration and the table it created +// compare equal. +func ReconcileTables(ctx context.Context, database schema.ExecQuerier, options ReconcileOptions, tables ...*schema.Table) error { + driver, err := atlas.Open(database) + if err != nil { + return fmt.Errorf("sqlite migrate: open driver: %w", err) + } + changes, err := diffTables(ctx, driver, options, tables) + if err != nil { + return err + } + if len(changes) == 0 { + return nil + } + names := tableNames(tables) + plan, err := driver.PlanChanges(ctx, "", changes) + if err != nil { + return fmt.Errorf("sqlite migrate: plan %d changes: %w", len(changes), err) + } + if err := refuseUnmanagedTriggers(ctx, database, plan.Changes); err != nil { + return err + } + log := logger.GetLogger("migrate") + for _, change := range plan.Changes { + log.Tracef("%s", change.Cmd) + } + if err := driver.ApplyChanges(ctx, changes); err != nil { + return fmt.Errorf("sqlite migrate: apply %d changes: %w", len(changes), err) + } + log.V(1).Infof("Applied %d sqlite schema changes to %s", len(changes), strings.Join(names, ", ")) + return nil +} + +// PendingChanges reports the changes ReconcileTables would apply, without applying them. It only +// reads the schema, so a caller can skip opening a migration transaction when nothing is pending. +// It applies the same drop policy as ReconcileTables. +func PendingChanges(ctx context.Context, database schema.ExecQuerier, options ReconcileOptions, tables ...*schema.Table) ([]schema.Change, error) { + driver, err := atlas.Open(database) + if err != nil { + return nil, fmt.Errorf("sqlite migrate: open driver: %w", err) + } + return diffTables(ctx, driver, options, tables) +} + +func diffTables(ctx context.Context, driver atlasmigrate.Driver, options ReconcileOptions, tables []*schema.Table) ([]schema.Change, error) { + if len(tables) == 0 && !options.AllowDrops { + return nil, errors.New("sqlite migrate: no table declared") + } + for _, table := range tables { + if err := parseRawTypes(table); err != nil { + return nil, err + } + } + names := tableNames(tables) + if options.AllowDrops { + names = nil + } + current, err := driver.InspectSchema(ctx, "", &schema.InspectOptions{Mode: schema.InspectTables, Tables: names}) + if err != nil { + return nil, fmt.Errorf("sqlite migrate: inspect %s: %w", strings.Join(names, ", "), err) + } + desired := desiredSchema(current, options, tables) + changes, err := driver.SchemaDiff(current, desired) + if err != nil { + return nil, fmt.Errorf("sqlite migrate: diff %s: %w", strings.Join(names, ", "), err) + } + changes = withCheckExpressionChanges(current, desired, changes) + return changes, nil +} + +// desiredSchema retains omitted columns in the table definitions Atlas copies +// during rebuilds; filtering DropColumn changes alone would still lose them. +func desiredSchema(current *schema.Schema, options ReconcileOptions, tables []*schema.Table) *schema.Schema { + desired := schema.New(current.Name) + for _, table := range tables { + cloned := *table + cloned.Columns = slices.Clone(table.Columns) + if previous, found := current.Table(table.Name); found && !options.AllowDrops { + for _, column := range previous.Columns { + if _, declared := table.Column(column.Name); declared { + continue + } + preserved := *column + preserved.Indexes, preserved.ForeignKeys = nil, nil + cloned.AddColumns(&preserved) + logger.GetLogger("migrate").Tracef("Skipping drop column of %s.%s", table.Name, column.Name) + } + } + desired.AddTables(&cloned) + } + return desired +} + +func tableNames(tables []*schema.Table) []string { + names := make([]string, len(tables)) + for index, table := range tables { + names[index] = table.Name + } + return names +} + +// parseRawTypes gives every column declared by its raw type alone the type +// Atlas's own inspection parses that raw type into. +func parseRawTypes(table *schema.Table) error { + for _, column := range table.Columns { + if column.Type == nil { + return fmt.Errorf("sqlite migrate: table %q column %q declares no type", table.Name, column.Name) + } + if column.Type.Type != nil { + continue + } + parsed, err := atlas.ParseType(column.Type.Raw) + if err != nil { + return fmt.Errorf("sqlite migrate: table %q column %q type %q: %w", table.Name, column.Name, column.Type.Raw, err) + } + column.Type.Type = parsed + } + return nil +} + +func refuseUnmanagedTriggers(ctx context.Context, database schema.ExecQuerier, changes []*atlasmigrate.Change) error { + for _, change := range changes { + modified, ok := change.Source.(*schema.ModifyTable) + if !ok || !strings.HasPrefix(change.Cmd, "DROP TABLE ") { + continue + } + rows, err := database.QueryContext(ctx, `SELECT name FROM sqlite_schema WHERE type = 'trigger' AND tbl_name = ? LIMIT 1`, modified.T.Name) + if err != nil { + return fmt.Errorf("inspect triggers for table %q: %w", modified.T.Name, err) + } + var trigger string + if rows.Next() { + err = rows.Scan(&trigger) + } + if err == nil { + err = rows.Err() + } + closeErr := rows.Close() + if err != nil { + return fmt.Errorf("inspect triggers for table %q: %w", modified.T.Name, err) + } + if closeErr != nil { + return fmt.Errorf("close trigger inspection for table %q: %w", modified.T.Name, closeErr) + } + if trigger != "" { + return fmt.Errorf("sqlite migrate: table %q has unmanaged trigger %q; refusing a rebuild that would discard it", modified.T.Name, trigger) + } + } + return nil +} diff --git a/database/migrate/sqlite/apply_test.go b/database/migrate/sqlite/apply_test.go new file mode 100644 index 00000000..28f2d700 --- /dev/null +++ b/database/migrate/sqlite/apply_test.go @@ -0,0 +1,249 @@ +package sqlite_test + +import ( + "context" + "database/sql" + "path/filepath" + "testing" + + "ariga.io/atlas/sql/schema" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + _ "modernc.org/sqlite" + + sqlitemigrate "github.com/flanksource/sqlglot-go/database/migrate/sqlite" +) + +func TestSQLiteMigrate(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "SQLite Migrate Suite") +} + +// eventsTable is the table every spec starts from: keyed per stream by seq, +// with a unique index on (stream_id, name), holding one row. +const eventsTable = `CREATE TABLE "events" ("stream_id" TEXT, "seq" NUMERIC, "name" TEXT, "note" TEXT, PRIMARY KEY ("stream_id", "seq")); + CREATE UNIQUE INDEX "events_key" ON "events" ("stream_id", "name"); + INSERT INTO "events" VALUES ('run-1', 1, 'a', 'first');` + +func column(name, raw string) *schema.Column { + return &schema.Column{Name: name, Type: &schema.ColumnType{Raw: raw, Null: true}} +} + +// declaredEvents is eventsTable as declared, each column by its raw type alone. +func declaredEvents() *schema.Table { + table := schema.NewTable("events").AddColumns( + column("stream_id", "TEXT"), column("seq", "NUMERIC"), column("name", "TEXT"), column("note", "TEXT"), + ) + stream, _ := table.Column("stream_id") + seq, _ := table.Column("seq") + name, _ := table.Column("name") + table.SetPrimaryKey(schema.NewPrimaryKey(stream, seq)) + table.AddIndexes(schema.NewUniqueIndex("events_key").AddColumns(stream, name)) + return table +} + +var _ = Describe("sqlite migrate.ReconcileTables", func() { + var ( + ctx context.Context + database *sql.DB + ) + + BeforeEach(func() { + ctx = context.Background() + var err error + database, err = sql.Open("sqlite", filepath.Join(GinkgoT().TempDir(), "migrate.sqlite")) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(database.Close) + _, err = database.ExecContext(ctx, eventsTable) + Expect(err).ToNot(HaveOccurred()) + }) + + columnsOf := func(table string) []string { + rows, err := database.QueryContext(ctx, `SELECT name FROM pragma_table_info(?) ORDER BY cid`, table) + Expect(err).ToNot(HaveOccurred()) + defer func() { Expect(rows.Close()).To(Succeed()) }() + var names []string + for rows.Next() { + var name string + Expect(rows.Scan(&name)).To(Succeed()) + names = append(names, name) + } + Expect(rows.Err()).ToNot(HaveOccurred()) + return names + } + + schemaSQL := func() []string { + rows, err := database.QueryContext(ctx, `SELECT type || ' ' || name || ': ' || coalesce(sql, '') FROM sqlite_schema ORDER BY type, name`) + Expect(err).ToNot(HaveOccurred()) + defer func() { Expect(rows.Close()).To(Succeed()) }() + var objects []string + for rows.Next() { + var object string + Expect(rows.Scan(&object)).To(Succeed()) + objects = append(objects, object) + } + Expect(rows.Err()).ToNot(HaveOccurred()) + return objects + } + + It("leaves a table that matches its declaration exactly as it is", func() { + before := schemaSQL() + Expect(sqlitemigrate.ReconcileTables(ctx, database, sqlitemigrate.ReconcileOptions{}, declaredEvents())).To(Succeed()) + Expect(schemaSQL()).To(Equal(before)) + }) + + It("adds the columns a table lacks inside the caller's transaction, keeping its rows", func() { + declared := declaredEvents().AddColumns(column("object_type", "TEXT"), column("count", "NUMERIC")) + tx, err := database.BeginTx(ctx, nil) + Expect(err).ToNot(HaveOccurred()) + Expect(sqlitemigrate.ReconcileTables(ctx, tx, sqlitemigrate.ReconcileOptions{}, declared)).To(Succeed()) + Expect(tx.Commit()).To(Succeed()) + + var name, note string + var objectType, count sql.NullString + Expect(database.QueryRowContext(ctx, `SELECT "name", "note", "object_type", "count" FROM "events" WHERE "seq" = 1`). + Scan(&name, ¬e, &objectType, &count)).To(Succeed()) + Expect(map[string]any{ + "columns": columnsOf("events"), "name": name, "note": note, "objectType": objectType.Valid, "count": count.Valid, + }).To(Equal(map[string]any{ + "columns": []string{"stream_id", "seq", "name", "note", "object_type", "count"}, + "name": "a", "note": "first", "objectType": false, "count": false, + })) + }) + + It("creates a declared table and index the database lacks", func() { + audit := schema.NewTable("audit").AddColumns(column("at", "TEXT"), column("who", "TEXT")) + who, _ := audit.Column("who") + audit.AddIndexes(schema.NewIndex("audit_who").AddColumns(who)) + declared := declaredEvents() + note, _ := declared.Column("note") + declared.AddIndexes(schema.NewIndex("events_note").AddColumns(note)) + + Expect(sqlitemigrate.ReconcileTables(ctx, database, sqlitemigrate.ReconcileOptions{}, declared, audit)).To(Succeed()) + var indexes []string + rows, err := database.QueryContext(ctx, `SELECT name FROM sqlite_schema WHERE type = 'index' AND name IN ('audit_who', 'events_note') ORDER BY name`) + Expect(err).ToNot(HaveOccurred()) + defer func() { Expect(rows.Close()).To(Succeed()) }() + for rows.Next() { + var index string + Expect(rows.Scan(&index)).To(Succeed()) + indexes = append(indexes, index) + } + Expect(map[string]any{"audit": columnsOf("audit"), "indexes": indexes}).To(Equal(map[string]any{ + "audit": []string{"at", "who"}, "indexes": []string{"audit_who", "events_note"}, + })) + }) + + It("widens a named check while preserving existing rows", func() { + _, err := database.ExecContext(ctx, `CREATE TABLE roles (id INTEGER, role INTEGER, PRIMARY KEY (id), CONSTRAINT roles_role_check CHECK (role IN (0, 1, 2))); INSERT INTO roles VALUES (1, 2)`) + Expect(err).ToNot(HaveOccurred()) + declared := schema.NewTable("roles").AddColumns(column("id", "INTEGER"), column("role", "INTEGER")) + id, _ := declared.Column("id") + declared.SetPrimaryKey(schema.NewPrimaryKey(id)) + declared.AddChecks(&schema.Check{Name: "roles_role_check", Expr: "role IN (0, 1, 2, 3)"}) + + pending, err := sqlitemigrate.PendingChanges(ctx, database, sqlitemigrate.ReconcileOptions{}, declared) + Expect(err).ToNot(HaveOccurred()) + Expect(pending).To(HaveLen(1)) + Expect(sqlitemigrate.ReconcileTables(ctx, database, sqlitemigrate.ReconcileOptions{}, declared)).To(Succeed()) + var role int + Expect(database.QueryRowContext(ctx, `SELECT role FROM roles WHERE id = 1`).Scan(&role)).To(Succeed()) + Expect(role).To(Equal(2)) + _, err = database.ExecContext(ctx, `INSERT INTO roles VALUES (2, 3)`) + Expect(err).ToNot(HaveOccurred()) + before := schemaSQL() + pending, err = sqlitemigrate.PendingChanges(ctx, database, sqlitemigrate.ReconcileOptions{}, declared) + Expect(err).ToNot(HaveOccurred()) + Expect(pending).To(BeEmpty()) + Expect(sqlitemigrate.ReconcileTables(ctx, database, sqlitemigrate.ReconcileOptions{}, declared)).To(Succeed()) + Expect(schemaSQL()).To(Equal(before)) + }) + + It("leaves alone the tables and views nothing declares", func() { + _, err := database.ExecContext(ctx, `CREATE TABLE "other" ("x" TEXT); CREATE VIEW "named" AS SELECT "name" FROM "events"`) + Expect(err).ToNot(HaveOccurred()) + Expect(sqlitemigrate.ReconcileTables(ctx, database, sqlitemigrate.ReconcileOptions{}, declaredEvents().AddColumns(column("added", "TEXT")))).To(Succeed()) + Expect(schemaSQL()).To(ContainElements( + `table other: CREATE TABLE "other" ("x" TEXT)`, + `view named: CREATE VIEW "named" AS SELECT "name" FROM "events"`, + )) + }) + + DescribeTable("applies supported rewrites and index changes while preserving rows", + func(change func(declared *schema.Table)) { + declared := declaredEvents().AddColumns(column("added", "TEXT")) + change(declared) + Expect(sqlitemigrate.ReconcileTables(ctx, database, sqlitemigrate.ReconcileOptions{}, declared)).To(Succeed()) + Expect(columnsOf("events")).To(ContainElement("added")) + var name, note string + Expect(database.QueryRowContext(ctx, `SELECT name, note FROM events WHERE seq = 1`).Scan(&name, ¬e)).To(Succeed()) + Expect([]string{name, note}).To(Equal([]string{"a", "first"})) + }, + Entry("a dropped column", func(declared *schema.Table) { + declared.Columns = declared.Columns[:3] + declared.Columns = append(declared.Columns, column("added", "TEXT")) + }), + Entry("a retyped column", func(declared *schema.Table) { + note, _ := declared.Column("note") + note.Type.Raw = "NUMERIC" + }), + Entry("a column made NOT NULL", func(declared *schema.Table) { + note, _ := declared.Column("note") + note.Type.Null = false + }), + Entry("a changed primary key", func(declared *schema.Table) { + stream, _ := declared.Column("stream_id") + declared.SetPrimaryKey(schema.NewPrimaryKey(stream)) + }), + Entry("a dropped index", func(declared *schema.Table) { + declared.Indexes = nil + }), + Entry("a changed index", func(declared *schema.Table) { + stream, _ := declared.Column("stream_id") + note, _ := declared.Column("note") + declared.Indexes = nil + declared.AddIndexes(schema.NewUniqueIndex("events_key").AddColumns(stream, note)) + }), + ) + + It("drops a unique index without requiring the column and table drop option", func() { + declared := declaredEvents().AddColumns(column("added", "TEXT")) + declared.Indexes = nil + Expect(sqlitemigrate.ReconcileTables(ctx, database, sqlitemigrate.ReconcileOptions{}, declared)).To(Succeed()) + _, err := database.ExecContext(ctx, `INSERT INTO events (stream_id, seq, name) VALUES ('run-1', 2, 'a')`) + Expect(err).NotTo(HaveOccurred()) + }) + + It("preserves an unmanaged trigger when adding a column and index with rebuilds enabled", func() { + _, err := database.ExecContext(ctx, `CREATE TRIGGER events_audit AFTER INSERT ON events BEGIN SELECT 1; END`) + Expect(err).ToNot(HaveOccurred()) + declared := declaredEvents().AddColumns(column("added", "TEXT")) + note, _ := declared.Column("note") + declared.AddIndexes(schema.NewIndex("events_note").AddColumns(note)) + + Expect(sqlitemigrate.ReconcileTables(ctx, database, sqlitemigrate.ReconcileOptions{}, declared)).To(Succeed()) + Expect(columnsOf("events")).To(ContainElement("added")) + Expect(schemaSQL()).To(ContainElements( + `trigger events_audit: CREATE TRIGGER events_audit AFTER INSERT ON events BEGIN SELECT 1; END`, + "index events_note: CREATE INDEX `events_note` ON `events` (`note`)", + )) + }) + + It("refuses an additive plan when Atlas would rebuild and discard an unmanaged trigger", func() { + _, err := database.ExecContext(ctx, `CREATE TRIGGER events_audit AFTER INSERT ON events BEGIN SELECT 1; END`) + Expect(err).ToNot(HaveOccurred()) + declared := declaredEvents().AddColumns(column("added", "TEXT")) + added, _ := declared.Column("added") + declared.AddIndexes(schema.NewIndex("events_added").AddColumns(added)) + before := schemaSQL() + + for _, options := range []sqlitemigrate.ReconcileOptions{{}, {AllowDrops: true}} { + Expect(sqlitemigrate.ReconcileTables(ctx, database, options, declared)).To(MatchError(ContainSubstring("unmanaged trigger"))) + Expect(schemaSQL()).To(Equal(before)) + } + }) + + It("refuses a declaration naming no table", func() { + Expect(sqlitemigrate.ReconcileTables(ctx, database, sqlitemigrate.ReconcileOptions{})).To(MatchError(ContainSubstring("no table"))) + }) +}) diff --git a/database/migrate/sqlite/check_changes.go b/database/migrate/sqlite/check_changes.go new file mode 100644 index 00000000..17217b9e --- /dev/null +++ b/database/migrate/sqlite/check_changes.go @@ -0,0 +1,85 @@ +package sqlite + +import ( + "strings" + + "ariga.io/atlas/sql/schema" +) + +// Atlas pairs named checks without comparing their expressions, so add the missing modification. +func withCheckExpressionChanges(current, desired *schema.Schema, changes []schema.Change) []schema.Change { + for _, to := range desired.Tables { + from, found := current.Table(to.Name) + if !found { + continue + } + known := map[string]*schema.Check{} + for _, attr := range from.Attrs { + if check, ok := attr.(*schema.Check); ok && check.Name != "" { + known[check.Name] = check + } + } + var modified []schema.Change + for _, attr := range to.Attrs { + check, ok := attr.(*schema.Check) + if !ok || check.Name == "" { + continue + } + if existing := known[check.Name]; existing != nil && normalizedCheckExpression(existing.Expr) != normalizedCheckExpression(check.Expr) { + modified = append(modified, &schema.ModifyCheck{From: existing, To: check}) + } + } + if len(modified) == 0 { + continue + } + merged := false + for _, change := range changes { + if table, ok := change.(*schema.ModifyTable); ok && table.T.Name == to.Name { + table.Changes = append(table.Changes, modified...) + merged = true + break + } + } + if !merged { + changes = append(changes, &schema.ModifyTable{T: to, Changes: modified}) + } + } + return changes +} + +func normalizedCheckExpression(expression string) string { + expression = strings.TrimSpace(expression) + for len(expression) > 1 && expression[0] == '(' && expression[len(expression)-1] == ')' && wrappingParentheses(expression) { + expression = strings.TrimSpace(expression[1 : len(expression)-1]) + } + return expression +} + +func wrappingParentheses(expression string) bool { + depth := 0 + var quote byte + for i := 0; i < len(expression); i++ { + switch { + case quote != 0: + if expression[i] == quote { + if i+1 < len(expression) && expression[i+1] == quote { + i++ + } else { + quote = 0 + } + } + case expression[i] == '\'' || expression[i] == '"' || expression[i] == '`': + quote = expression[i] + case expression[i] == '[': + quote = ']' + case expression[i] == '(': + depth++ + case expression[i] == ')': + depth-- + if depth == 0 && i < len(expression)-1 { + return false + } + } + } + return depth == 0 && quote == 0 +} diff --git a/database/migrate/sqlite/check_changes_test.go b/database/migrate/sqlite/check_changes_test.go new file mode 100644 index 00000000..c683f98f --- /dev/null +++ b/database/migrate/sqlite/check_changes_test.go @@ -0,0 +1,23 @@ +package sqlite + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("normalizedCheckExpression", func() { + DescribeTable("strips only parentheses that wrap the whole expression", + func(expression, expected string) { + Expect(normalizedCheckExpression(expression)).To(Equal(expected)) + }, + Entry("wrapped", "(a > 0)", "a > 0"), + Entry("doubly wrapped", "((a > 0))", "a > 0"), + Entry("two groups", "(a > 0) AND (b > 0)", "(a > 0) AND (b > 0)"), + Entry("single-quoted parenthesis", "(name <> ')')", "name <> ')'"), + Entry("double-quoted identifier with parenthesis", `("a)" > 0)`, `"a)" > 0`), + Entry("backtick identifier with parenthesis", "(`a)` > 0)", "`a)` > 0"), + Entry("escaped backtick inside identifier", "(`a``)` > 0)", "`a``)` > 0"), + Entry("bracket identifier with parenthesis", "([a)] > 0)", "[a)] > 0"), + Entry("bracket identifier with opening parenthesis", "([(a] > 0) AND (b > 0)", "([(a] > 0) AND (b > 0)"), + ) +}) diff --git a/database/migrate/sqlite/drop_policy_test.go b/database/migrate/sqlite/drop_policy_test.go new file mode 100644 index 00000000..71dfda18 --- /dev/null +++ b/database/migrate/sqlite/drop_policy_test.go @@ -0,0 +1,89 @@ +package sqlite_test + +import ( + "context" + "database/sql" + "path/filepath" + + "ariga.io/atlas/sql/schema" + sqlitemigrate "github.com/flanksource/sqlglot-go/database/migrate/sqlite" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("SQLite drop policy", func() { + var database *sql.DB + var ctx context.Context + BeforeEach(func() { + ctx = context.Background() + var err error + database, err = sql.Open("sqlite", filepath.Join(GinkgoT().TempDir(), "policy.db")) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(database.Close) + _, err = database.ExecContext(ctx, eventsTable) + Expect(err).NotTo(HaveOccurred()) + _, err = database.ExecContext(ctx, `CREATE TABLE other (value TEXT); INSERT INTO other VALUES ('retained')`) + Expect(err).NotTo(HaveOccurred()) + }) + + DescribeTable("preserves omitted data through a rebuild unless drops are enabled", func(allowDrops bool) { + declared := declaredEvents() + declared.Columns = declared.Columns[:3] + declared.AddChecks(&schema.Check{Name: "events_seq_positive", Expr: "seq > 0"}) + options := sqlitemigrate.ReconcileOptions{AllowDrops: allowDrops} + for range 2 { + Expect(sqlitemigrate.ReconcileTables(ctx, database, options, declared)).To(Succeed()) + } + var noteColumns, otherTables int + Expect(database.QueryRowContext(ctx, `SELECT count(*) FROM pragma_table_info('events') WHERE name = 'note'`).Scan(¬eColumns)).To(Succeed()) + Expect(database.QueryRowContext(ctx, `SELECT count(*) FROM sqlite_schema WHERE type = 'table' AND name = 'other'`).Scan(&otherTables)).To(Succeed()) + if allowDrops { + Expect([]int{noteColumns, otherTables}).To(Equal([]int{0, 0})) + } else { + Expect([]int{noteColumns, otherTables}).To(Equal([]int{1, 1})) + var note, other string + Expect(database.QueryRowContext(ctx, `SELECT note FROM events WHERE seq = 1`).Scan(¬e)).To(Succeed()) + Expect(database.QueryRowContext(ctx, `SELECT value FROM other`).Scan(&other)).To(Succeed()) + Expect([]string{note, other}).To(Equal([]string{"first", "retained"})) + } + Expect(declared.Columns).NotTo(ContainElement(HaveField("Name", "note"))) + pending, err := sqlitemigrate.PendingChanges(ctx, database, options, declared) + Expect(err).NotTo(HaveOccurred()) + Expect(pending).To(BeEmpty()) + _, err = database.ExecContext(ctx, `INSERT INTO events (stream_id, seq, name) VALUES ('run-2', 0, 'invalid')`) + Expect(err).To(MatchError(ContainSubstring("CHECK constraint failed"))) + }, Entry("default", false), Entry("explicit opt-in", true)) + + It("reports no pending work when only a column and table are omitted", func() { + declared := declaredEvents() + declared.Columns = declared.Columns[:3] + pending, err := sqlitemigrate.PendingChanges(ctx, database, sqlitemigrate.ReconcileOptions{}, declared) + Expect(err).NotTo(HaveOccurred()) + Expect(pending).To(BeEmpty()) + }) + + It("allows an explicitly empty desired schema to remove all user tables only with drop opt-in", func() { + Expect(sqlitemigrate.ReconcileTables(ctx, database, sqlitemigrate.ReconcileOptions{})).To(MatchError(ContainSubstring("no table"))) + Expect(sqlitemigrate.ReconcileTables(ctx, database, sqlitemigrate.ReconcileOptions{AllowDrops: true})).To(Succeed()) + var count int + Expect(database.QueryRowContext(ctx, `SELECT count(*) FROM sqlite_schema WHERE type = 'table' AND name NOT LIKE 'sqlite_%'`).Scan(&count)).To(Succeed()) + Expect(count).To(BeZero()) + }) + + It("widens a unique key with a new Git path without enabling drops", func() { + declared := declaredEvents() + gitPath := column("git_path", "TEXT") + gitPath.Type.Null = false + gitPath.Default = &schema.Literal{V: "''"} + declared.AddColumns(gitPath) + declared.Indexes[0].AddColumns(gitPath) + Expect(sqlitemigrate.ReconcileTables(ctx, database, sqlitemigrate.ReconcileOptions{}, declared)).To(Succeed()) + var note, path string + Expect(database.QueryRowContext(ctx, `SELECT note, git_path FROM events WHERE seq = 1`).Scan(¬e, &path)).To(Succeed()) + Expect([]string{note, path}).To(Equal([]string{"first", ""})) + _, err := database.ExecContext(ctx, `INSERT INTO events (stream_id, seq, name, git_path) VALUES ('run-1', 2, 'a', 'other.go')`) + Expect(err).NotTo(HaveOccurred()) + _, err = database.ExecContext(ctx, `INSERT INTO events (stream_id, seq, name, git_path) VALUES ('run-1', 3, 'a', 'other.go')`) + Expect(err).To(MatchError(ContainSubstring("UNIQUE constraint failed"))) + }) +}) diff --git a/database/migrate/sqlite_drop_policy_test.go b/database/migrate/sqlite_drop_policy_test.go new file mode 100644 index 00000000..26fb0659 --- /dev/null +++ b/database/migrate/sqlite_drop_policy_test.go @@ -0,0 +1,58 @@ +package migrate + +import ( + "context" + "database/sql" + "path/filepath" + "strings" + "testing/fstest" + + "github.com/flanksource/sqlglot-go/database/target" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +const dropPolicySchema = `schema "public" {} +table "items" { + schema = schema.public + column "id" { type = text } + column "name" { type = text } + primary_key { columns = [column.id] } +}` + +var _ = Describe("SQLite HCL drop policy", func() { + DescribeTable("applies the drop option through HCL reconciliation", func(ctx context.Context, allowDrops bool) { + dsn := filepath.Join(GinkgoT().TempDir(), "policy.db") + files := fstest.MapFS{"schema.hcl": &fstest.MapFile{Data: []byte(dropPolicySchema)}} + Expect(Apply(ctx, dsn, files)).To(Succeed()) + target, err := dbtarget.Parse(dsn) + Expect(err).NotTo(HaveOccurred()) + database, err := sql.Open("sqlite", target.DSN) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(database.Close) + _, err = database.ExecContext(ctx, `ALTER TABLE items ADD COLUMN note TEXT; +INSERT INTO items VALUES ('item-1', 'retained', 'keep this'); +CREATE TABLE other (value TEXT); INSERT INTO other VALUES ('retained')`) + Expect(err).NotTo(HaveOccurred()) + files["schema.hcl"].Data = []byte(strings.Replace(dropPolicySchema, + " primary_key", " check \"items_name_nonempty\" { expr = \"length(name) > 0\" }\n primary_key", 1)) + var options []Option + if allowDrops { + options = append(options, WithDrops()) + } + for range 2 { + Expect(Apply(ctx, dsn, files, options...)).To(Succeed()) + } + var columns, tables int + Expect(database.QueryRowContext(ctx, `SELECT count(*) FROM pragma_table_info('items') WHERE name = 'note'`).Scan(&columns)).To(Succeed()) + Expect(database.QueryRowContext(ctx, `SELECT count(*) FROM sqlite_schema WHERE type = 'table' AND name = 'other'`).Scan(&tables)).To(Succeed()) + if allowDrops { + Expect([]int{columns, tables}).To(Equal([]int{0, 0})) + } else { + Expect([]int{columns, tables}).To(Equal([]int{1, 1})) + var note string + Expect(database.QueryRowContext(ctx, `SELECT note FROM items WHERE id = 'item-1'`).Scan(¬e)).To(Succeed()) + Expect(note).To(Equal("keep this")) + } + }, Entry("default", false), Entry("WithDrops", true)) +}) diff --git a/database/migrate/sqlite_target.go b/database/migrate/sqlite_target.go new file mode 100644 index 00000000..9c025ba3 --- /dev/null +++ b/database/migrate/sqlite_target.go @@ -0,0 +1,259 @@ +package migrate + +import ( + "context" + "database/sql" + "errors" + "fmt" + "io/fs" + "strings" + + "ariga.io/atlas/sql/postgres" + "ariga.io/atlas/sql/schema" + atlassqlite "ariga.io/atlas/sql/sqlite" + _ "modernc.org/sqlite" + + sqlitemigrate "github.com/flanksource/sqlglot-go/database/migrate/sqlite" +) + +// openSQLite opens the single-connection handle a DSN-driven SQLite migration +// runs on. +func openSQLite(connection string) (*sql.DB, error) { + database, err := sql.Open("sqlite", connection) + if err != nil { + return nil, fmt.Errorf("open SQLite migration database: %w", err) + } + database.SetMaxOpenConns(1) + database.SetMaxIdleConns(1) + return database, nil +} + +func applySQLite(ctx context.Context, database *sql.DB, schemaFS fs.FS, config options) error { + tables, err := sqliteTables(schemaFS, config) + if err != nil { + return err + } + if err := database.PingContext(ctx); err != nil { + return fmt.Errorf("connect SQLite migration database: %w", err) + } + return reconcileSQLite(ctx, database, tables, sqlitemigrate.ReconcileOptions{AllowDrops: config.allowDrops}) +} + +// sqliteTables evaluates the HCL bundle and projects it to the SQLite tables it declares. +func sqliteTables(schemaFS fs.FS, config options) ([]*schema.Table, error) { + if schemaFS == nil { + return nil, errors.New("schema filesystem is nil") + } + if config.schema != defaultSchema { + return nil, fmt.Errorf("SQLite migrations do not support schema %q", config.schema) + } + if len(config.exclude) > 0 { + return nil, errors.New("SQLite migrations do not support inspection exclusions") + } + if err := refuseRoutines(config, "SQLite"); err != nil { + return nil, err + } + scripts, err := loadScripts(schemaFS, config.dir) + if err != nil { + return nil, err + } + if len(scripts) > 0 { + return nil, errors.New("SQLite migrations do not support SQL migration files") + } + parser, security, err := loadHCL(schemaFS, config.dir, config.input) + if err != nil { + return nil, err + } + if len(security.Roles) > 0 || len(security.Permissions) > 0 { + return nil, errors.New("SQLite migrations do not support roles or permissions") + } + desired := &schema.Realm{} + if err := postgres.EvalHCL.Eval(parser, desired, config.input); err != nil { + return nil, fmt.Errorf("evaluate HCL schemas for SQLite: %w", err) + } + tables, err := projectSQLiteRealm(desired) + if err != nil { + return nil, fmt.Errorf("project HCL schema to SQLite: %w", err) + } + return tables, nil +} + +// reconcileSQLite checks for pending changes before opening a transaction, because Atlas's OpenTx +// runs PRAGMA foreign_key_check over the whole database when it opens and again when it commits. +// On an up-to-date database that pair of scans was the entire cost of a migration, and it grows with +// the data. The transaction re-inspects, so a change another process applied in between is not +// applied twice. +func reconcileSQLite(ctx context.Context, database *sql.DB, tables []*schema.Table, options sqlitemigrate.ReconcileOptions) error { + pending, err := sqlitemigrate.PendingChanges(ctx, database, options, tables...) + if err != nil { + return err + } + if len(pending) == 0 { + return nil + } + tx, err := atlassqlite.OpenTx(ctx, database, nil) + if err != nil { + return fmt.Errorf("begin SQLite migration: %w", err) + } + if err := sqlitemigrate.ReconcileTables(ctx, tx, options, tables...); err != nil { + return errors.Join(err, tx.Rollback()) + } + if err := tx.Commit(); err != nil { + return errors.Join(fmt.Errorf("commit SQLite migration: %w", err), tx.Rollback()) + } + return nil +} + +func projectSQLiteRealm(realm *schema.Realm) ([]*schema.Table, error) { + if realm == nil { + return nil, errors.New("desired realm is nil") + } + if len(realm.Schemas) != 1 { + return nil, fmt.Errorf("expected one schema, got %d", len(realm.Schemas)) + } + if len(realm.Attrs) > 0 || len(realm.Objects) > 0 { + return nil, errors.New("realm attributes and objects are not portable to SQLite") + } + scope := realm.Schemas[0] + if scope.Name != defaultSchema { + return nil, fmt.Errorf("canonical HCL must declare schema %q, got %q", defaultSchema, scope.Name) + } + if len(scope.Views) > 0 || len(scope.Funcs) > 0 || len(scope.Procs) > 0 || len(scope.Objects) > 0 || len(scope.Attrs) > 0 { + return nil, errors.New("schema views, functions, procedures, attributes, and objects are not portable to SQLite") + } + scope.Name = "main" + for _, table := range scope.Tables { + if err := projectSQLiteTable(table, scope); err != nil { + return nil, err + } + } + return scope.Tables, nil +} + +func projectSQLiteTable(table *schema.Table, scope *schema.Schema) error { + if table == nil { + return errors.New("canonical HCL contains a nil table") + } + if len(table.Triggers) > 0 { + return fmt.Errorf("table %q declares triggers, which are not portable to SQLite", table.Name) + } + table.Schema = scope + for _, column := range table.Columns { + jsonColumn, err := projectSQLiteColumn(table.Name, column) + if err != nil { + return err + } + if jsonColumn { + if err := addSQLiteJSONCheck(table, column.Name); err != nil { + return err + } + } + } + if table.PrimaryKey != nil { + if err := projectSQLiteIndex(table.Name, table.PrimaryKey); err != nil { + return err + } + } + for _, index := range table.Indexes { + if err := projectSQLiteIndex(table.Name, index); err != nil { + return err + } + } + for _, foreignKey := range table.ForeignKeys { + if len(foreignKey.Attrs) > 0 { + return fmt.Errorf("table %q foreign key %q has PostgreSQL-only attributes", table.Name, foreignKey.Symbol) + } + } + for _, attr := range table.Attrs { + check, ok := attr.(*schema.Check) + if !ok { + return fmt.Errorf("table %q has non-portable attribute %T", table.Name, attr) + } + if len(check.Attrs) > 0 { + return fmt.Errorf("table %q check %q has non-portable attributes", table.Name, check.Name) + } + } + return nil +} + +func projectSQLiteColumn(table string, column *schema.Column) (bool, error) { + if column == nil || column.Type == nil || column.Type.Type == nil { + return false, fmt.Errorf("table %q contains a column without a type", table) + } + if len(column.Attrs) > 0 { + return false, fmt.Errorf("table %q column %q has non-portable attributes", table, column.Name) + } + switch column.Type.Type.(type) { + case *schema.UUIDType: + column.Type.Type, column.Type.Raw = &schema.StringType{T: "text"}, "text" + case *schema.JSONType: + column.Type.Type, column.Type.Raw = &schema.StringType{T: "text"}, "text" + return true, nil + case *schema.TimeType: + column.Type.Type, column.Type.Raw = &schema.TimeType{T: "datetime"}, "datetime" + case *schema.IntegerType: + column.Type.Type, column.Type.Raw = &schema.IntegerType{T: "integer"}, "integer" + case *schema.StringType: + column.Type.Type, column.Type.Raw = &schema.StringType{T: "text"}, "text" + case *schema.BoolType: + column.Type.Raw = "bool" + case *schema.BinaryType: + column.Type.Type, column.Type.Raw = &schema.BinaryType{T: "blob"}, "blob" + case *schema.DecimalType: + column.Type.Type, column.Type.Raw = &schema.DecimalType{T: "numeric"}, "numeric" + case *schema.FloatType: + column.Type.Type, column.Type.Raw = &schema.FloatType{T: "real"}, "real" + default: + return false, fmt.Errorf("table %q column %q type %T is not portable to SQLite", table, column.Name, column.Type.Type) + } + return false, nil +} + +func projectSQLiteIndex(table string, index *schema.Index) error { + if index == nil { + return fmt.Errorf("table %q contains a nil index", table) + } + attrs := make([]schema.Attr, 0, len(index.Attrs)) + for _, attr := range index.Attrs { + switch attr := attr.(type) { + case *postgres.Constraint: + if !attr.IsUnique() { + return fmt.Errorf("table %q index %q has non-portable constraint", table, index.Name) + } + case *postgres.IndexPredicate: + attrs = append(attrs, &atlassqlite.IndexPredicate{P: attr.P}) + case *postgres.IndexType: + if attr.T != "" && !strings.EqualFold(attr.T, "btree") { + return fmt.Errorf("table %q index %q type %q is not portable to SQLite", table, index.Name, attr.T) + } + default: + return fmt.Errorf("table %q index %q has non-portable attribute %T", table, index.Name, attr) + } + } + for _, part := range index.Parts { + if part == nil || part.C == nil || part.X != nil { + return fmt.Errorf("table %q index %q contains a non-portable expression", table, index.Name) + } + if len(part.Attrs) > 0 { + return fmt.Errorf("table %q index %q has non-portable column attributes", table, index.Name) + } + } + index.Attrs = attrs + return nil +} + +func addSQLiteJSONCheck(table *schema.Table, column string) error { + name := table.Name + "_" + column + "_json" + quoted := `"` + strings.ReplaceAll(column, `"`, `""`) + `"` + expression := "json_valid(" + quoted + ")" + for _, attr := range table.Attrs { + if check, ok := attr.(*schema.Check); ok && check.Name == name { + if check.Expr != expression || len(check.Attrs) > 0 { + return fmt.Errorf("table %q check %q conflicts with generated SQLite JSON check", table.Name, name) + } + return nil + } + } + table.Attrs = append(table.Attrs, &schema.Check{Name: name, Expr: expression}) + return nil +} diff --git a/database/migrate/sqlite_target_test.go b/database/migrate/sqlite_target_test.go new file mode 100644 index 00000000..8dd5508d --- /dev/null +++ b/database/migrate/sqlite_target_test.go @@ -0,0 +1,260 @@ +package migrate + +import ( + "context" + "database/sql" + "path/filepath" + "strings" + "testing/fstest" + + "github.com/flanksource/sqlglot-go/database/target" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + _ "modernc.org/sqlite" +) + +const portableSchema = ` +schema "public" {} + +table "projects" { + schema = schema.public + column "id" { + type = uuid + null = false + } + column "external_key" { + type = text + null = false + } + column "payload" { + type = jsonb + null = false + default = sql("'{}'") + } + column "created_at" { + type = timestamptz + null = false + } + primary_key { columns = [column.id] } + unique "projects_external_key_key" { columns = [column.external_key] } +} + +table "events" { + schema = schema.public + column "id" { + type = uuid + null = false + } + column "project_id" { + type = uuid + null = false + } + column "event_key" { + type = text + null = false + } + column "active" { + type = bool + null = false + default = false + } + column "metadata" { + type = jsonb + null = false + default = sql("'{}'") + } + primary_key { columns = [column.id] } + foreign_key "events_project_id_fkey" { + columns = [column.project_id] + ref_columns = [table.projects.column.id] + on_update = NO_ACTION + on_delete = CASCADE + } + index "events_active_key" { + unique = true + columns = [column.project_id, column.event_key] + where = "active" + } +} +` + +var _ = Describe("SQLite HCL target", func() { + It("rebuilds a populated table for a new check while preserving referenced rows", func(ctx context.Context) { + dsn := filepath.Join(GinkgoT().TempDir(), "reshape.db") + files := fstest.MapFS{"migrations/schema.hcl": &fstest.MapFile{Data: []byte(portableSchema)}} + Expect(Apply(ctx, dsn, files, WithDir("migrations"))).To(Succeed()) + + target, err := dbtarget.Parse(dsn) + Expect(err).ToNot(HaveOccurred()) + database, err := sql.Open("sqlite", target.DSN) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(database.Close) + _, err = database.ExecContext(ctx, `INSERT INTO projects (id, external_key, payload, created_at) VALUES ('p1', 'existing', '{}', CURRENT_TIMESTAMP)`) + Expect(err).ToNot(HaveOccurred()) + _, err = database.ExecContext(ctx, `INSERT INTO events (id, project_id, event_key, active) VALUES ('e1', 'p1', 'build', true)`) + Expect(err).ToNot(HaveOccurred()) + + files["migrations/schema.hcl"] = &fstest.MapFile{Data: []byte(strings.Replace(portableSchema, + ` unique "projects_external_key_key" { columns = [column.external_key] }`, + ` unique "projects_external_key_key" { columns = [column.external_key] } + check "projects_external_key_nonempty" { expr = "length(external_key) > 0" }`, 1))} + Expect(Apply(ctx, dsn, files, WithDir("migrations"))).To(Succeed()) + + var key string + Expect(database.QueryRowContext(ctx, `SELECT external_key FROM projects WHERE id = 'p1'`).Scan(&key)).To(Succeed()) + Expect(key).To(Equal("existing")) + Expect(database.QueryRowContext(ctx, `SELECT project_id FROM events WHERE id = 'e1'`).Scan(&key)).To(Succeed()) + Expect(key).To(Equal("p1")) + _, err = database.ExecContext(ctx, `INSERT INTO projects (id, external_key, payload, created_at) VALUES ('p2', '', '{}', CURRENT_TIMESTAMP)`) + Expect(err).To(MatchError(ContainSubstring("CHECK constraint failed"))) + Expect(Apply(ctx, dsn, files, WithDir("migrations"))).To(Succeed()) + }) + + It("rolls back a rebuild when existing rows violate the new constraint", func(ctx context.Context) { + dsn := filepath.Join(GinkgoT().TempDir(), "rollback.db") + files := fstest.MapFS{"schema.hcl": &fstest.MapFile{Data: []byte(portableSchema)}} + Expect(Apply(ctx, dsn, files)).To(Succeed()) + target, err := dbtarget.Parse(dsn) + Expect(err).ToNot(HaveOccurred()) + database, err := sql.Open("sqlite", target.DSN) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(database.Close) + _, err = database.ExecContext(ctx, `INSERT INTO projects (id, external_key, payload, created_at) VALUES ('p1', '', '{}', CURRENT_TIMESTAMP)`) + Expect(err).ToNot(HaveOccurred()) + files["schema.hcl"] = &fstest.MapFile{Data: []byte(strings.Replace(portableSchema, + ` unique "projects_external_key_key" { columns = [column.external_key] }`, + ` unique "projects_external_key_key" { columns = [column.external_key] } + check "projects_external_key_nonempty" { expr = "length(external_key) > 0" }`, 1))} + Expect(Apply(ctx, dsn, files)).To(MatchError(ContainSubstring("CHECK constraint failed"))) + var count int + Expect(database.QueryRowContext(ctx, `SELECT count(*) FROM projects WHERE id = 'p1'`).Scan(&count)).To(Succeed()) + Expect(count).To(Equal(1)) + Expect(database.QueryRowContext(ctx, `SELECT count(*) FROM sqlite_schema WHERE name = 'new_projects'`).Scan(&count)).To(Succeed()) + Expect(count).To(BeZero()) + }) + + It("skips dropping a column omitted from the declaration", func(ctx context.Context) { + dsn := filepath.Join(GinkgoT().TempDir(), "drop.db") + files := fstest.MapFS{"schema.hcl": &fstest.MapFile{Data: []byte(portableSchema)}} + Expect(Apply(ctx, dsn, files)).To(Succeed()) + files["schema.hcl"] = &fstest.MapFile{Data: []byte(strings.Replace(portableSchema, + ` column "external_key" { + type = text + null = false + } +`, "", 1))} + files["schema.hcl"].Data = []byte(strings.Replace(string(files["schema.hcl"].Data), + ` unique "projects_external_key_key" { columns = [column.external_key] }`, "", 1)) + Expect(Apply(ctx, dsn, files)).To(Succeed()) + target, err := dbtarget.Parse(dsn) + Expect(err).NotTo(HaveOccurred()) + database, err := sql.Open("sqlite", target.DSN) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(database.Close) + var count int + Expect(database.QueryRowContext(ctx, `SELECT count(*) FROM pragma_table_info('projects') WHERE name = 'external_key'`).Scan(&count)).To(Succeed()) + Expect(count).To(Equal(1)) + }) + + It("refuses a rebuild that would discard an unmanaged trigger", func(ctx context.Context) { + dsn := filepath.Join(GinkgoT().TempDir(), "trigger.db") + files := fstest.MapFS{"schema.hcl": &fstest.MapFile{Data: []byte(portableSchema)}} + Expect(Apply(ctx, dsn, files)).To(Succeed()) + target, err := dbtarget.Parse(dsn) + Expect(err).ToNot(HaveOccurred()) + database, err := sql.Open("sqlite", target.DSN) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(database.Close) + _, err = database.ExecContext(ctx, `CREATE TRIGGER project_audit AFTER INSERT ON projects BEGIN SELECT 1; END`) + Expect(err).ToNot(HaveOccurred()) + files["schema.hcl"] = &fstest.MapFile{Data: []byte(strings.Replace(portableSchema, + ` unique "projects_external_key_key" { columns = [column.external_key] }`, + ` unique "projects_external_key_key" { columns = [column.external_key] } + check "projects_external_key_nonempty" { expr = "length(external_key) > 0" }`, 1))} + Expect(Apply(ctx, dsn, files)).To(MatchError(ContainSubstring("unmanaged trigger"))) + var count int + Expect(database.QueryRowContext(ctx, `SELECT count(*) FROM sqlite_schema WHERE type = 'trigger' AND name = 'project_audit'`).Scan(&count)).To(Succeed()) + Expect(count).To(Equal(1)) + }) + + It("applies one PostgreSQL-shaped HCL source as an idempotent SQLite schema", func(ctx context.Context) { + dsn := filepath.Join(GinkgoT().TempDir(), "portable.db") + files := fstest.MapFS{"migrations/schema.hcl": &fstest.MapFile{Data: []byte(portableSchema)}} + Expect(Apply(ctx, dsn, files, WithDir("migrations"), WithName("portable"))).To(Succeed()) + + target, err := dbtarget.Parse(dsn) + Expect(err).ToNot(HaveOccurred()) + database, err := sql.Open("sqlite", target.DSN) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(database.Close) + + _, err = database.ExecContext(ctx, `INSERT INTO projects (id, external_key, payload, created_at) VALUES ('p1', 'project', '{}', CURRENT_TIMESTAMP)`) + Expect(err).ToNot(HaveOccurred()) + + files["migrations/schema.hcl"] = &fstest.MapFile{Data: []byte(strings.Replace(portableSchema, + ` column "created_at" {`, + " column \"description\" {\n type = text\n null = true\n }\n column \"created_at\" {", + 1, + ))} + Expect(Apply(ctx, dsn, files, WithDir("migrations"), WithName("portable"))).To(Succeed()) + Expect(Apply(ctx, dsn, files, WithDir("migrations"), WithName("portable"))).To(Succeed()) + + var externalKey string + var description sql.NullString + Expect(database.QueryRowContext(ctx, `SELECT external_key, description FROM projects WHERE id = 'p1'`). + Scan(&externalKey, &description)).To(Succeed()) + Expect(map[string]any{"external_key": externalKey, "description_valid": description.Valid}).To(Equal(map[string]any{ + "external_key": "project", "description_valid": false, + })) + + _, err = database.ExecContext(ctx, `INSERT INTO events (id, project_id, event_key, active) VALUES ('e1', 'p1', 'build', true)`) + Expect(err).ToNot(HaveOccurred()) + _, err = database.ExecContext(ctx, `INSERT INTO events (id, project_id, event_key, active) VALUES ('e2', 'p1', 'build', true)`) + Expect(err).To(MatchError(ContainSubstring("UNIQUE constraint failed"))) + _, err = database.ExecContext(ctx, `INSERT INTO events (id, project_id, event_key, active) VALUES ('e3', 'missing', 'build', false)`) + Expect(err).To(MatchError(ContainSubstring("FOREIGN KEY constraint failed"))) + _, err = database.ExecContext(ctx, `INSERT INTO projects (id, external_key, payload, created_at) VALUES ('p2', 'invalid', '{', CURRENT_TIMESTAMP)`) + Expect(err).To(MatchError(ContainSubstring("CHECK constraint failed"))) + + var idType, payloadType, createdAtType string + Expect(database.QueryRowContext(ctx, `SELECT type FROM pragma_table_info('projects') WHERE name = 'id'`).Scan(&idType)).To(Succeed()) + Expect(database.QueryRowContext(ctx, `SELECT type FROM pragma_table_info('projects') WHERE name = 'payload'`).Scan(&payloadType)).To(Succeed()) + Expect(database.QueryRowContext(ctx, `SELECT type FROM pragma_table_info('projects') WHERE name = 'created_at'`).Scan(&createdAtType)).To(Succeed()) + Expect(map[string]string{"id": idType, "payload": payloadType, "created_at": createdAtType}).To(Equal(map[string]string{ + "id": "TEXT", "payload": "TEXT", "created_at": "datetime", + })) + }) + + It("rejects PostgreSQL-only bundle features before opening SQLite", func(ctx context.Context) { + dsn := filepath.Join(GinkgoT().TempDir(), "unsupported.db") + files := fstest.MapFS{ + "migrations/schema.hcl": &fstest.MapFile{Data: []byte(portableSchema)}, + "migrations/view.sql": &fstest.MapFile{Data: []byte("CREATE VIEW project_ids AS SELECT id FROM projects")}, + } + Expect(Apply(ctx, dsn, files, WithDir("migrations"))).To(MatchError(ContainSubstring("do not support SQL migration files"))) + }) + + It("rejects a canonical bundle with more than the public schema", func(ctx context.Context) { + dsn := filepath.Join(GinkgoT().TempDir(), "schemas.db") + files := fstest.MapFS{"schema.hcl": &fstest.MapFile{Data: []byte(portableSchema + `schema "audit" {}`)}} + Expect(Apply(ctx, dsn, files)).To(MatchError(ContainSubstring("expected one schema, got 2"))) + }) + + It("rejects a canonical check that collides with a projected JSON check", func(ctx context.Context) { + dsn := filepath.Join(GinkgoT().TempDir(), "check-collision.db") + files := fstest.MapFS{"schema.hcl": &fstest.MapFile{Data: []byte(` +schema "public" {} +table "documents" { + schema = schema.public + column "payload" { + type = jsonb + null = false + } + check "documents_payload_json" { + expr = "payload <> ''" + } +} +`)}} + Expect(Apply(ctx, dsn, files)).To(MatchError(ContainSubstring("conflicts with generated SQLite JSON check"))) + }) +}) diff --git a/database/migrate/sqlite_up_to_date_test.go b/database/migrate/sqlite_up_to_date_test.go new file mode 100644 index 00000000..35997e5d --- /dev/null +++ b/database/migrate/sqlite_up_to_date_test.go @@ -0,0 +1,122 @@ +package migrate + +import ( + "context" + "database/sql" + "database/sql/driver" + "path/filepath" + "strings" + "sync" + "testing/fstest" + + sqlitemigrate "github.com/flanksource/sqlglot-go/database/migrate/sqlite" + "github.com/flanksource/sqlglot-go/database/target" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + sqlite "modernc.org/sqlite" +) + +// recordingDriver wraps the modernc driver and records every statement and transaction a connection +// runs. Its connections expose only the prepare path, so database/sql routes every query through it. +type recordingDriver struct { + mu sync.Mutex + statements []string +} + +func (recorder *recordingDriver) Open(name string) (driver.Conn, error) { + conn, err := (&sqlite.Driver{}).Open(name) + if err != nil { + return nil, err + } + return &recordingConn{conn: conn, recorder: recorder}, nil +} + +func (recorder *recordingDriver) record(statement string) { + recorder.mu.Lock() + defer recorder.mu.Unlock() + recorder.statements = append(recorder.statements, statement) +} + +func (recorder *recordingDriver) recorded() []string { + recorder.mu.Lock() + defer recorder.mu.Unlock() + return append([]string(nil), recorder.statements...) +} + +type recordingConn struct { + conn driver.Conn + recorder *recordingDriver +} + +func (conn *recordingConn) Prepare(query string) (driver.Stmt, error) { + return conn.PrepareContext(context.Background(), query) +} + +func (conn *recordingConn) PrepareContext(ctx context.Context, query string) (driver.Stmt, error) { + conn.recorder.record(query) + return conn.conn.(driver.ConnPrepareContext).PrepareContext(ctx, query) +} + +func (conn *recordingConn) Close() error { return conn.conn.Close() } + +func (conn *recordingConn) Begin() (driver.Tx, error) { + return conn.BeginTx(context.Background(), driver.TxOptions{}) +} + +func (conn *recordingConn) BeginTx(ctx context.Context, options driver.TxOptions) (driver.Tx, error) { + conn.recorder.record("BEGIN") + return conn.conn.(driver.ConnBeginTx).BeginTx(ctx, options) +} + +var registerRecordingDriver = sync.OnceValue(func() *recordingDriver { + recorder := &recordingDriver{} + sql.Register("sqlite-recording", recorder) + return recorder +}) + +var _ = Describe("SQLite HCL target on an up-to-date database", func() { + reconcileRecorded := func(ctx context.Context, dsn string, files fstest.MapFS) []string { + tables, err := sqliteTables(files, resolveOptions(nil)) + Expect(err).ToNot(HaveOccurred()) + target, err := dbtarget.Parse(dsn) + Expect(err).ToNot(HaveOccurred()) + recorder := registerRecordingDriver() + database, err := sql.Open("sqlite-recording", target.DSN) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(database.Close) + before := len(recorder.recorded()) + Expect(reconcileSQLite(ctx, database, tables, sqlitemigrate.ReconcileOptions{})).To(Succeed()) + return recorder.recorded()[before:] + } + transactional := func(statements []string) []string { + var found []string + for _, statement := range statements { + if statement == "BEGIN" || strings.Contains(statement, "foreign_key_check") { + found = append(found, statement) + } + } + return found + } + + It("opens no migration transaction and runs no foreign key check when nothing changed", func(ctx context.Context) { + dsn := filepath.Join(GinkgoT().TempDir(), "current.db") + files := fstest.MapFS{"schema.hcl": &fstest.MapFile{Data: []byte(portableSchema)}} + Expect(Apply(ctx, dsn, files)).To(Succeed()) + + Expect(transactional(reconcileRecorded(ctx, dsn, files))).To(BeEmpty()) + }) + + It("still applies a pending change inside the checked transaction", func(ctx context.Context) { + dsn := filepath.Join(GinkgoT().TempDir(), "pending.db") + files := fstest.MapFS{"schema.hcl": &fstest.MapFile{Data: []byte(portableSchema)}} + Expect(Apply(ctx, dsn, files)).To(Succeed()) + files["schema.hcl"] = &fstest.MapFile{Data: []byte(strings.Replace(portableSchema, + ` column "created_at" {`, + " column \"description\" {\n type = text\n null = true\n }\n column \"created_at\" {", 1))} + + Expect(transactional(reconcileRecorded(ctx, dsn, files))).To(Equal([]string{ + "BEGIN", "PRAGMA foreign_key_check", "PRAGMA foreign_key_check", + })) + Expect(transactional(reconcileRecorded(ctx, dsn, files))).To(BeEmpty()) + }) +}) diff --git a/database/migrate/sqlserver.go b/database/migrate/sqlserver.go new file mode 100644 index 00000000..b9380d70 --- /dev/null +++ b/database/migrate/sqlserver.go @@ -0,0 +1,259 @@ +package migrate + +import ( + "context" + "database/sql" + "errors" + "fmt" + "io/fs" + "path" + "slices" + "strings" + "time" + + "ariga.io/atlas/sql/schema" + "ariga.io/atlas/sql/sqlclient" + "github.com/flanksource/commons/logger" + "github.com/flanksource/sqlglot-go/database/atlas/sqlserver" +) + +const ( + // sqlServerSchema is the only schema SQL Server migrations manage. + sqlServerSchema = "dbo" + // sqlServerLockWait bounds how long a migration waits for another + // migration of the same bundle to release its sp_getapplock. + sqlServerLockWait = time.Minute +) + +// applySQLServer reconciles the bundle's HCL tables and WithRoutines routines +// against the dbo schema. It inspects only the objects the bundle names, never +// drops a routine or an index the bundle does not declare, and runs no DDL +// when the database already matches. +// The caller owns db: closing the Atlas client would close it. +func applySQLServer(ctx context.Context, db *sql.DB, schemaFS fs.FS, cfg options) (returnErr error) { + desired, err := sqlServerDesired(schemaFS, cfg) + if err != nil { + return err + } + client, err := sqlserver.OpenClient(ctx, db) + if err != nil { + return fmt.Errorf("open SQL Server migration database: %w", err) + } + unlock, err := client.Lock(ctx, "commons-db/migrate:"+cfg.name, sqlServerLockWait) + if err != nil { + return fmt.Errorf("lock SQL Server migration %q: %w", cfg.name, err) + } + defer func() { returnErr = errors.Join(returnErr, unlock()) }() + return reconcileSQLServer(ctx, client, desired, cfg.exclude) +} + +// sqlServerChanges inspects the declared objects and returns the changes that +// bring them to the desired state, without drops. +func sqlServerChanges(ctx context.Context, client *sqlclient.Client, desired *schema.Realm, exclude []string) ([]schema.Change, error) { + target := desired.Schemas[0] + current, err := client.InspectSchema(ctx, target.Name, sqlServerInspectOptions(target, exclude)) + if err != nil { + return nil, fmt.Errorf("inspect current SQL Server schema: %w", err) + } + changes, err := client.RealmDiff(schema.NewRealm(current), desired) + if err != nil { + return nil, fmt.Errorf("compute schema diff: %w", err) + } + return withoutSQLServerDrops(changes), nil +} + +func reconcileSQLServer(ctx context.Context, client *sqlclient.Client, desired *schema.Realm, exclude []string) error { + changes, err := sqlServerChanges(ctx, client, desired, exclude) + if err != nil { + return err + } + log := logger.GetLogger("migrate") + if len(changes) == 0 { + log.Debugf("No schema changes detected") + return nil + } + plan, err := client.PlanChanges(ctx, "", changes) + if err != nil { + return fmt.Errorf("plan %d schema changes: %w", len(changes), err) + } + for _, change := range plan.Changes { + log.Tracef("%s", change.Cmd) + } + if err := client.ApplyChanges(ctx, changes); err != nil { + return fmt.Errorf("apply %d schema changes: %w", len(changes), err) + } + log.V(1).Infof("Applied %d schema changes", len(changes)) + return nil +} + +// sqlServerInspectOptions scopes inspection to the declared object names. +// SQL Server keeps tables and routines in one namespace per schema, so one +// name list filters both. +func sqlServerInspectOptions(target *schema.Schema, exclude []string) *schema.InspectOptions { + opts := &schema.InspectOptions{Mode: schema.InspectSchemas, Exclude: exclude} + for _, t := range target.Tables { + opts.Tables = append(opts.Tables, t.Name) + } + if len(target.Tables) > 0 { + opts.Mode |= schema.InspectTables + } + for _, p := range target.Procs { + opts.Tables = append(opts.Tables, p.Name) + } + for _, f := range target.Funcs { + opts.Tables = append(opts.Tables, f.Name) + } + if len(target.Procs)+len(target.Funcs) > 0 { + opts.Mode |= schema.InspectFuncs + } + return opts +} + +// withoutSQLServerDrops extends withoutDrops with the indexes of declared +// tables: indexes a DBA added to tune a shared database are left in place. +func withoutSQLServerDrops(changes []schema.Change) []schema.Change { + log := logger.GetLogger("migrate") + filtered := make([]schema.Change, 0, len(changes)) + for _, change := range withoutDrops(changes) { + modify, ok := change.(*schema.ModifyTable) + if !ok { + filtered = append(filtered, change) + continue + } + kept := make([]schema.Change, 0, len(modify.Changes)) + for _, c := range modify.Changes { + if drop, ok := c.(*schema.DropIndex); ok { + log.Tracef("Skipping drop index of %s.%s", modify.T.Name, drop.I.Name) + continue + } + kept = append(kept, c) + } + if len(kept) > 0 { + modify.Changes = kept + filtered = append(filtered, modify) + } + } + return filtered +} + +// sqlServerDesired builds the desired dbo schema from the HCL tables and the +// routine files, refusing what SQL Server migrations do not support. +func sqlServerDesired(schemaFS fs.FS, cfg options) (*schema.Realm, error) { + if schemaFS == nil { + return nil, errors.New("schema filesystem is nil") + } + if err := refuseSQLServerOptions(cfg); err != nil { + return nil, err + } + scripts, err := loadScripts(schemaFS, cfg.dir, cfg.routines) + if err != nil { + return nil, err + } + if len(scripts) > 0 { + return nil, fmt.Errorf("SQL Server migrations do not support SQL migration scripts, found %s; declare routines with WithRoutines", strings.Join(scriptPaths(scripts), ", ")) + } + realm, err := sqlServerTables(schemaFS, cfg) + if err != nil { + return nil, err + } + target := realm.Schemas[0] + if cfg.routines != "" { + if err := addRoutines(schemaFS, cfg.routines, target); err != nil { + return nil, err + } + } + if len(target.Tables)+len(target.Procs)+len(target.Funcs) == 0 { + return nil, fmt.Errorf("SQL Server migration bundle %q declares no tables or routines", cfg.name) + } + return realm, nil +} + +func refuseSQLServerOptions(cfg options) error { + switch { + case cfg.schemaSet && cfg.schema != sqlServerSchema: + return fmt.Errorf("SQL Server migrations support only schema %q, got %q", sqlServerSchema, cfg.schema) + case cfg.allowDrops: + return errors.New("SQL Server migrations do not support destructive changes (WithDrops)") + case cfg.allowRebuilds: + return errors.New("SQL Server migrations do not support SQLite rebuild options (WithRebuilds)") + } + return nil +} + +func addRoutines(schemaFS fs.FS, dir string, target *schema.Schema) error { + routines, err := loadRoutines(schemaFS, dir) + if err != nil { + return err + } + for _, routine := range routines { + if _, exists := target.Table(routine.Name); exists { + return fmt.Errorf("routine %s has the name of a declared table", routine.Name) + } + if err := routine.AddTo(target); err != nil { + return err + } + } + return nil +} + +// sqlServerTables evaluates the bundle's HCL, if it has any, into a realm +// holding exactly the dbo schema. +func sqlServerTables(schemaFS fs.FS, cfg options) (*schema.Realm, error) { + found, err := hasHCL(schemaFS, cfg.dir) + if err != nil { + return nil, err + } + if !found { + return schema.NewRealm(schema.New(sqlServerSchema)), nil + } + parser, security, err := loadHCL(schemaFS, cfg.dir, cfg.input) + if err != nil { + return nil, err + } + if len(security.Roles) > 0 || len(security.Permissions) > 0 { + return nil, errors.New("SQL Server migrations do not support roles or permissions") + } + realm := &schema.Realm{} + if err := sqlserver.EvalHCL.Eval(parser, realm, cfg.input); err != nil { + return nil, fmt.Errorf("evaluate HCL schemas for SQL Server: %w", err) + } + if len(realm.Schemas) != 1 || realm.Schemas[0].Name != sqlServerSchema { + return nil, fmt.Errorf("SQL Server HCL must declare exactly schema %q, got %s", sqlServerSchema, schemaNames(realm)) + } + return realm, nil +} + +func hasHCL(schemaFS fs.FS, dir string) (bool, error) { + found := false + err := fs.WalkDir(schemaFS, dir, func(name string, entry fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if !entry.IsDir() && strings.EqualFold(path.Ext(name), ".hcl") { + found = true + return fs.SkipAll + } + return nil + }) + if err != nil { + return false, fmt.Errorf("find HCL schemas in %q: %w", dir, err) + } + return found, nil +} + +func schemaNames(realm *schema.Realm) string { + names := make([]string, len(realm.Schemas)) + for i, s := range realm.Schemas { + names[i] = fmt.Sprintf("%q", s.Name) + } + return "[" + strings.Join(names, ", ") + "]" +} + +func scriptPaths(scripts map[string]*script) []string { + paths := make([]string, 0, len(scripts)) + for p := range scripts { + paths = append(paths, p) + } + slices.Sort(paths) + return paths +} diff --git a/database/migrate/sqlserver/README.md b/database/migrate/sqlserver/README.md new file mode 100644 index 00000000..692534f0 --- /dev/null +++ b/database/migrate/sqlserver/README.md @@ -0,0 +1,28 @@ +# Execute a reviewed SQL Server artifact + +This package executes a sqlglot artifact using complete catalog snapshots from `inspect/sql/snapshot`. It opens one transaction, takes a transaction-owned application lock, and checks whether the database matches the baseline or already matches the complete desired state. Only a matching baseline permits SQL execution. The executor runs the exact reviewed statements, inspects the desired postcondition in the same transaction, and commits after it matches. + +```go +result, err := sqlserver.Apply(ctx, db, sqlserver.Request{ + Artifact: artifact, + Hash: reviewedHash, + Acknowledgements: destructiveChangeIDs, + LockTimeout: 30 * time.Second, +}) +``` + +`Hash` must match the artifact reviewed by the caller. `Acknowledgements` must contain each destructive change ID exactly once; missing, unknown and duplicate IDs fail before the transaction opens. `LockTimeout` is bounded to SQL Server's nonnegative millisecond range; zero makes one attempt without waiting. + +Module definitions execute in separate nested batches so `CREATE OR ALTER` is first in its batch. The outer dynamic batch sets `XACT_ABORT` and the required module and filtered-index session options without leaving altered settings in the pooled session. Every pre-commit failure attempts rollback. The result distinguishes `applied`, `already-applied`, `failed` and `commit-unknown`; a commit error must be reconciled by inspecting the target before another decision. `Steps` counts statements that completed before the result, including statements subsequently rolled back. + +The fixed application-lock resource `commons-db/migrate/schema` serializes artifacts across different hashes and scopes within a database. Every cooperating migration runner must use this resource. Out-of-band schema writers must be quiesced during deployment. The application owns target identity, permissions, artifact storage and immutable attempt records. + +Run control-flow tests with `go test ./migrate/sqlserver ./inspect/sql/snapshot -count=1`. These tests use SQL mocks and do not prove SQL Server runtime behavior. + +The real engine suite is opt-in through the `integration` build tag. Set `COMMONS_DB_SQLSERVER_ARTIFACT_TEST_URL` to an admin URL for an explicitly isolated SQL Server test instance, with no database or with `database=master`, then run: + +```sh +go test -tags integration ./migrate/sqlserver -run '^TestArtifactExecution$' -ginkgo.focus 'Isolated SQL Server artifacts' -count=1 +``` + +The suite creates a uniquely named `commons_db_artifact_test_` database at compatibility level 100 per spec and drops only those databases during cleanup. It checks catalog round trips, selected changes, module batches, idempotence, data-conversion rollback, drift rejection and application-lock contention. With the integration tag enabled, missing configuration is a failure. No container is started automatically. diff --git a/database/migrate/sqlserver/apply.go b/database/migrate/sqlserver/apply.go new file mode 100644 index 00000000..e94d000c --- /dev/null +++ b/database/migrate/sqlserver/apply.go @@ -0,0 +1,105 @@ +// Package sqlserver executes immutable sqlglot artifacts against SQL Server. +package sqlserver + +import ( + "context" + "database/sql" + "errors" + "fmt" + "time" + + "github.com/flanksource/sqlglot-go/database/inspect/snapshot" + "github.com/flanksource/sqlglot-go/migrate" + planner "github.com/flanksource/sqlglot-go/migrate/sqlserver" +) + +type Request struct { + Artifact migrate.Artifact + Hash string + Acknowledgements []string + LockTimeout time.Duration +} + +type Outcome string + +const ( + Applied Outcome = "applied" + AlreadyApplied Outcome = "already-applied" + Failed Outcome = "failed" + CommitUnknown Outcome = "commit-unknown" +) + +type Result struct { + Hash string `json:"hash"` + Outcome Outcome `json:"outcome"` + Steps int `json:"steps"` +} + +type inspectFunc func(context.Context, *sql.Tx, snapshot.Options) (snapshot.Result, error) + +func Apply(ctx context.Context, db *sql.DB, request Request) (Result, error) { + return apply(ctx, db, request, func(ctx context.Context, tx *sql.Tx, options snapshot.Options) (snapshot.Result, error) { + return snapshot.Inspect(ctx, tx, options) + }) +} + +func apply(ctx context.Context, db *sql.DB, request Request, inspect inspectFunc) (result Result, returnErr error) { + result = Result{Hash: request.Hash, Outcome: Failed} + if ctx == nil || db == nil { + return result, fmt.Errorf("SQL Server artifact execution requires a context and database") + } + if err := validate(request); err != nil { + return result, err + } + tx, err := db.BeginTx(ctx, nil) + if err != nil { + return result, fmt.Errorf("begin migration transaction: %w", err) + } + defer func() { + if err := tx.Rollback(); err != nil && !errors.Is(err, sql.ErrTxDone) { + returnErr = errors.Join(returnErr, fmt.Errorf("rollback migration transaction: %w", err)) + if result.Outcome == AlreadyApplied { + result.Outcome = Failed + } + } + }() + if err := lock(ctx, tx, request.LockTimeout); err != nil { + return result, err + } + state, err := inspectState(ctx, tx, request.Artifact, inspect) + if err != nil { + return result, err + } + if state == migrate.ArtifactApplied { + result.Outcome = AlreadyApplied + return result, nil + } + for _, step := range request.Artifact.Plan.Steps { + if _, err := tx.ExecContext(ctx, executeBatch, step.SQL); err != nil { + return result, fmt.Errorf("migration step %s: %w", step.Change.ID, err) + } + result.Steps++ + } + if state, err := inspectState(ctx, tx, request.Artifact, inspect); err != nil || state != migrate.ArtifactApplied { + return result, fmt.Errorf("migration postcondition failed (state %q): %w", state, errors.Join(err, errPostcondition)) + } + if err := tx.Commit(); err != nil { + result.Outcome = CommitUnknown + return result, fmt.Errorf("migration commit outcome is unknown: %w", err) + } + result.Outcome = Applied + return result, nil +} + +var errPostcondition = errors.New("database does not match the complete desired snapshot") + +// Nested batches keep module DDL first in its batch and contain SET options. +const executeBatch = `EXEC sys.sp_executesql N'SET XACT_ABORT ON; SET ANSI_NULLS ON; SET ANSI_PADDING ON; SET ANSI_WARNINGS ON; SET ARITHABORT ON; SET CONCAT_NULL_YIELDS_NULL ON; SET QUOTED_IDENTIFIER ON; SET NUMERIC_ROUNDABORT OFF; EXEC sys.sp_executesql @ddl;', N'@ddl nvarchar(max)', @ddl=@p1` + +func inspectState(ctx context.Context, tx *sql.Tx, artifact migrate.Artifact, inspect inspectFunc) (migrate.ArtifactState, error) { + actual, err := inspect(ctx, tx, snapshot.Options{Schemas: artifact.Schemas}) + if err != nil { + return "", fmt.Errorf("inspect migration transaction: %w", err) + } + return planner.CheckArtifact(artifact, actual.Schema) +} diff --git a/database/migrate/sqlserver/apply_test.go b/database/migrate/sqlserver/apply_test.go new file mode 100644 index 00000000..d86a0541 --- /dev/null +++ b/database/migrate/sqlserver/apply_test.go @@ -0,0 +1,163 @@ +package sqlserver + +import ( + "context" + "database/sql" + "errors" + "regexp" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/flanksource/sqlglot-go/database/inspect/snapshot" + "github.com/flanksource/sqlglot-go/migrate" + planner "github.com/flanksource/sqlglot-go/migrate/sqlserver" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestArtifactExecution(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "SQL Server Artifact Execution") +} + +func testRequest() Request { + table := migrate.Table{Name: migrate.Name{Schema: "dbo", Name: "orders"}, Columns: []migrate.Column{{Name: "id", Type: "int"}}} + artifact, err := planner.BuildArtifact(migrate.ArtifactRequest{Schemas: []string{"dbo"}, Request: migrate.Request{Desired: migrate.Schema{Tables: []migrate.Table{table}}}}) + Expect(err).NotTo(HaveOccurred()) + return Request{Artifact: artifact, Hash: artifact.Hash} +} + +var _ = Describe("Reviewed artifact execution", func() { + var db *sql.DB + var mock sqlmock.Sqlmock + var request Request + var inspections int + var inspect inspectFunc + + BeforeEach(func() { + var err error + db, mock, err = sqlmock.New() + Expect(err).NotTo(HaveOccurred()) + request, inspections = testRequest(), 0 + inspect = func(_ context.Context, tx *sql.Tx, options snapshot.Options) (snapshot.Result, error) { + Expect(tx).NotTo(BeNil()) + Expect(options.Schemas).To(Equal(request.Artifact.Schemas)) + inspections++ + if inspections == 1 { + return snapshot.Result{Schema: request.Artifact.Request.Current}, nil + } + return snapshot.Result{Schema: request.Artifact.Request.Desired}, nil + } + }) + + AfterEach(func() { + Expect(mock.ExpectationsWereMet()).To(Succeed()) + mock.ExpectClose() + Expect(db.Close()).To(Succeed()) + }) + + begin := func(status int) { + mock.ExpectBegin() + mock.ExpectQuery("sp_getapplock").WithArgs(int64(0)).WillReturnRows(sqlmock.NewRows([]string{"status"}).AddRow(status)) + } + + It("executes the reviewed SQL in its own module batch, checks the postcondition and commits", func() { + begin(0) + mock.ExpectExec("sp_executesql").WithArgs(request.Artifact.Plan.Steps[0].SQL).WillReturnResult(sqlmock.NewResult(0, 0)) + mock.ExpectCommit() + result, err := apply(GinkgoT().Context(), db, request, inspect) + Expect(err).NotTo(HaveOccurred()) + Expect(result).To(Equal(Result{Hash: request.Hash, Outcome: Applied, Steps: 1})) + Expect(inspections).To(Equal(2)) + }) + + It("returns already-applied without executing or committing DDL", func() { + begin(1) + mock.ExpectRollback() + result, err := apply(GinkgoT().Context(), db, request, func(context.Context, *sql.Tx, snapshot.Options) (snapshot.Result, error) { + return snapshot.Result{Schema: request.Artifact.Request.Desired}, nil + }) + Expect(err).NotTo(HaveOccurred()) + Expect(result).To(Equal(Result{Hash: request.Hash, Outcome: AlreadyApplied})) + }) + + It("rolls back when the baseline has drifted", func() { + begin(0) + mock.ExpectRollback() + result, err := apply(GinkgoT().Context(), db, request, func(context.Context, *sql.Tx, snapshot.Options) (snapshot.Result, error) { + changed := testRequest().Artifact.Request.Desired + changed.Tables[0].Columns[0].Type = "bigint" + return snapshot.Result{Schema: changed}, nil + }) + Expect(err).To(MatchError(ContainSubstring("baseline drift"))) + Expect(result.Outcome).To(Equal(Failed)) + }) + + It("rolls back a statement failure and preserves its cause", func() { + begin(0) + cause := errors.New("DDL denied") + mock.ExpectExec("sp_executesql").WithArgs(request.Artifact.Plan.Steps[0].SQL).WillReturnError(cause) + mock.ExpectRollback() + result, err := apply(GinkgoT().Context(), db, request, inspect) + Expect(errors.Is(err, cause)).To(BeTrue()) + Expect(result.Outcome).To(Equal(Failed)) + Expect(inspections).To(Equal(1)) + }) + + It("rolls back when successful SQL did not produce the complete desired state", func() { + begin(0) + mock.ExpectExec("sp_executesql").WillReturnResult(sqlmock.NewResult(0, 0)) + mock.ExpectRollback() + result, err := apply(GinkgoT().Context(), db, request, func(context.Context, *sql.Tx, snapshot.Options) (snapshot.Result, error) { + return snapshot.Result{Schema: request.Artifact.Request.Current}, nil + }) + Expect(err).To(MatchError(ContainSubstring("postcondition"))) + Expect(result.Outcome).To(Equal(Failed)) + }) + + It("reports an uncertain commit separately from rollback failures", func() { + begin(0) + mock.ExpectExec("sp_executesql").WillReturnResult(sqlmock.NewResult(0, 0)) + cause := errors.New("connection lost while committing") + mock.ExpectCommit().WillReturnError(cause) + result, err := apply(GinkgoT().Context(), db, request, inspect) + Expect(errors.Is(err, cause)).To(BeTrue()) + Expect(result.Outcome).To(Equal(CommitUnknown)) + }) + + It("rejects lock failures before inspecting or changing the database", func() { + begin(-1) + mock.ExpectRollback() + result, err := apply(GinkgoT().Context(), db, request, inspect) + Expect(err).To(MatchError(ContainSubstring("sp_getapplock returned -1"))) + Expect(result.Outcome).To(Equal(Failed)) + Expect(inspections).To(BeZero()) + }) + + It("rejects a changed reviewed hash before opening a transaction", func() { + request.Hash = "changed" + _, err := Apply(GinkgoT().Context(), db, request) + Expect(err).To(MatchError(ContainSubstring("reviewed hash"))) + }) + + It("requires each destructive change ID and rejects unknown or repeated acknowledgements", func() { + request.Artifact.Request.Current, request.Artifact.Request.Desired = request.Artifact.Request.Desired, request.Artifact.Request.Current + var err error + request.Artifact, err = planner.BuildArtifact(migrate.ArtifactRequest{Schemas: request.Artifact.Schemas, Request: request.Artifact.Request}) + Expect(err).NotTo(HaveOccurred()) + request.Hash = request.Artifact.Hash + id := request.Artifact.Plan.Changes[0].ID + for _, acknowledgements := range [][]string{nil, {"unknown"}, {id, id}} { + request.Acknowledgements = acknowledgements + _, err := Apply(GinkgoT().Context(), db, request) + Expect(err).To(MatchError(ContainSubstring("acknowledg"))) + } + request.Acknowledgements = []string{id} + begin(0) + mock.ExpectExec(regexp.QuoteMeta("sp_executesql")).WithArgs(request.Artifact.Plan.Steps[0].SQL).WillReturnResult(sqlmock.NewResult(0, 0)) + mock.ExpectCommit() + result, err := apply(GinkgoT().Context(), db, request, inspect) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Outcome).To(Equal(Applied)) + }) +}) diff --git a/database/migrate/sqlserver/integration_test.go b/database/migrate/sqlserver/integration_test.go new file mode 100644 index 00000000..57f2915d --- /dev/null +++ b/database/migrate/sqlserver/integration_test.go @@ -0,0 +1,173 @@ +//go:build integration + +package sqlserver + +import ( + "database/sql" + "encoding/json" + "errors" + "fmt" + "net/url" + "os" + "slices" + "strings" + "time" + + "github.com/flanksource/sqlglot-go/database/inspect/snapshot" + "github.com/flanksource/sqlglot-go/migrate" + planner "github.com/flanksource/sqlglot-go/migrate/sqlserver" + mssql "github.com/microsoft/go-mssqldb" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +const integrationFixture = `CREATE TABLE dbo.orders ( + id int IDENTITY(1000,5) NOT NULL CONSTRAINT PK_orders PRIMARY KEY, + label nvarchar(20) NULL CONSTRAINT DF_orders_label DEFAULT (N'pending'), + total decimal(18,2) NOT NULL CONSTRAINT DF_orders_total DEFAULT ((0)), + CONSTRAINT CK_orders_total CHECK (total >= 0) +); +CREATE INDEX IX_orders_total ON dbo.orders(total DESC) INCLUDE(label) WHERE total > 0; +INSERT INTO dbo.orders(label,total) VALUES (N'cannot-convert',10);` + +var _ = Describe("Isolated SQL Server artifacts", func() { + var db *sql.DB + var baseline migrate.Schema + + BeforeEach(func(ctx SpecContext) { + dsn := os.Getenv("COMMONS_DB_SQLSERVER_ARTIFACT_TEST_URL") + Expect(dsn).NotTo(BeEmpty(), "provide an explicit admin URL for an isolated SQL Server test instance") + parsed, err := url.Parse(dsn) + Expect(err).NotTo(HaveOccurred()) + Expect(parsed.Scheme).To(Equal("sqlserver")) + Expect(parsed.Query().Get("database")).To(BeElementOf("", "master")) + admin, err := sql.Open("sqlserver", dsn) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(admin.Close) + name := fmt.Sprintf("commons_db_artifact_test_%d", time.Now().UnixNano()) + _, err = admin.ExecContext(ctx, "CREATE DATABASE ["+name+"]") + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func(ctx SpecContext) { + _, err := admin.ExecContext(ctx, "ALTER DATABASE ["+name+"] SET SINGLE_USER WITH ROLLBACK IMMEDIATE; DROP DATABASE ["+name+"]") + Expect(err).NotTo(HaveOccurred()) + }) + _, err = admin.ExecContext(ctx, "ALTER DATABASE ["+name+"] SET COMPATIBILITY_LEVEL = 100") + Expect(err).NotTo(HaveOccurred()) + query := parsed.Query() + query.Set("database", name) + parsed.RawQuery = query.Encode() + db, err = sql.Open("sqlserver", parsed.String()) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(db.Close) + _, err = db.ExecContext(ctx, integrationFixture) + Expect(err).NotTo(HaveOccurred()) + for _, ddl := range []string{ + "CREATE VIEW dbo.order_ids AS SELECT id FROM dbo.orders", + "CREATE FUNCTION dbo.order_scale(@amount decimal(18,2)) RETURNS decimal(18,2) AS BEGIN RETURN @amount; END", + "CREATE PROCEDURE dbo.order_count AS SELECT COUNT(*) FROM dbo.orders", + "CREATE TRIGGER dbo.orders_audit ON dbo.orders AFTER INSERT AS BEGIN SET NOCOUNT ON; END", + "DISABLE TRIGGER dbo.orders_audit ON dbo.orders", + } { + _, err = db.ExecContext(ctx, ddl) + Expect(err).NotTo(HaveOccurred()) + } + inspected, err := snapshot.Inspect(ctx, db, snapshot.Options{Schemas: []string{"dbo"}}) + Expect(err).NotTo(HaveOccurred()) + baseline = inspected.Schema + }) + + compile := func(desired migrate.Schema) Request { + artifact, err := planner.BuildArtifact(migrate.ArtifactRequest{Schemas: []string{"dbo"}, Request: migrate.Request{Current: baseline, Desired: desired}}) + Expect(err).NotTo(HaveOccurred()) + request := Request{Artifact: artifact, Hash: artifact.Hash} + for _, change := range artifact.Plan.Changes { + if change.Destructive { + request.Acknowledgements = append(request.Acknowledgements, change.ID) + } + } + return request + } + + copyBaseline := func() migrate.Schema { + body, err := json.Marshal(baseline) + Expect(err).NotTo(HaveOccurred()) + var desired migrate.Schema + Expect(json.Unmarshal(body, &desired)).To(Succeed()) + return desired + } + + It("changes a column, function and trigger state and then becomes a no-op", func(ctx SpecContext) { + desired := copyBaseline() + desired.Tables[0].Columns[1].Type = "nvarchar(100)" + desired.Tables[0].Indexes = append(desired.Tables[0].Indexes, migrate.Index{Name: "IX_orders_label", Columns: []migrate.IndexColumn{{Name: "label"}}, Filter: "[label] IS NOT NULL"}) + for i := range desired.Definitions { + d := &desired.Definitions[i] + if d.Kind == "FUNCTION" { + d.SQL = strings.Replace(d.SQL, "RETURN @amount;", "RETURN @amount * 2;", 1) + } + if d.Kind == "TRIGGER" { + d.Disabled = false + } + } + request := compile(desired) + result, err := Apply(ctx, db, request) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Outcome).To(Equal(Applied)) + var value int + Expect(db.QueryRowContext(ctx, "SELECT CAST(dbo.order_scale(5) AS int)").Scan(&value)).To(Succeed()) + Expect(value).To(Equal(10)) + result, err = Apply(ctx, db, request) + Expect(err).NotTo(HaveOccurred()) + Expect(result).To(Equal(Result{Hash: request.Hash, Outcome: AlreadyApplied})) + }) + + It("restores dropped defaults and indexes when a later conversion fails", func(ctx SpecContext) { + desired := copyBaseline() + desired.Tables[0].Columns[1].Type = "int" + desired.Tables[0].Columns[1].Collation = "" + desired.Tables[0].Columns[1].Default = nil + desired.Tables[0].Indexes = slices.DeleteFunc(desired.Tables[0].Indexes, func(index migrate.Index) bool { return !index.Primary }) + request := compile(desired) + result, err := Apply(ctx, db, request) + var conversion mssql.Error + Expect(errors.As(err, &conversion)).To(BeTrue()) + Expect(conversion.Number).To(Equal(int32(245))) + Expect(result.Outcome).To(Equal(Failed)) + Expect(result.Steps).To(BeNumerically(">", 0)) + var exists int + Expect(db.QueryRowContext(ctx, "SELECT COUNT(*) FROM sys.default_constraints WHERE name='DF_orders_label'").Scan(&exists)).To(Succeed()) + Expect(exists).To(Equal(1)) + Expect(db.QueryRowContext(ctx, "SELECT COUNT(*) FROM sys.indexes WHERE name='IX_orders_total'").Scan(&exists)).To(Succeed()) + Expect(exists).To(Equal(1)) + actual, err := snapshot.Inspect(ctx, db, snapshot.Options{Schemas: []string{"dbo"}}) + Expect(err).NotTo(HaveOccurred()) + state, err := planner.CheckArtifact(request.Artifact, actual.Schema) + Expect(err).NotTo(HaveOccurred()) + Expect(state).To(Equal(migrate.ArtifactPending)) + }) + + It("refuses target drift before applying the artifact", func(ctx SpecContext) { + desired := copyBaseline() + desired.Tables[0].Columns[1].Type = "nvarchar(100)" + request := compile(desired) + _, err := db.ExecContext(ctx, "ALTER TABLE dbo.orders ADD extra bit NULL") + Expect(err).NotTo(HaveOccurred()) + result, err := Apply(ctx, db, request) + Expect(err).To(MatchError(ContainSubstring("baseline drift"))) + Expect(result).To(Equal(Result{Hash: request.Hash, Outcome: Failed})) + }) + + It("serializes different artifacts with a transaction-owned database lock", func(ctx SpecContext) { + tx, err := db.BeginTx(ctx, nil) + Expect(err).NotTo(HaveOccurred()) + defer func() { Expect(tx.Rollback()).To(Succeed()) }() + Expect(lock(ctx, tx, 0)).To(Succeed()) + desired := copyBaseline() + desired.Tables[0].Columns[1].Type = "nvarchar(100)" + request := compile(desired) + request.LockTimeout = 10 * time.Millisecond + result, err := Apply(ctx, db, request) + Expect(err).To(MatchError(ContainSubstring("sp_getapplock returned -1"))) + Expect(result.Outcome).To(Equal(Failed)) + }) +}) diff --git a/database/migrate/sqlserver/validate.go b/database/migrate/sqlserver/validate.go new file mode 100644 index 00000000..f2be44c3 --- /dev/null +++ b/database/migrate/sqlserver/validate.go @@ -0,0 +1,56 @@ +package sqlserver + +import ( + "context" + "database/sql" + "fmt" + "math" + "time" + + planner "github.com/flanksource/sqlglot-go/migrate/sqlserver" +) + +func validate(request Request) error { + if err := planner.ValidateArtifact(request.Artifact); err != nil { + return err + } + if request.Hash != request.Artifact.Hash { + return fmt.Errorf("reviewed hash does not match migration artifact") + } + if request.LockTimeout < 0 || request.LockTimeout > math.MaxInt32*time.Millisecond { + return fmt.Errorf("migration lock timeout must be between zero and %s", math.MaxInt32*time.Millisecond) + } + required := map[string]bool{} + for _, change := range request.Artifact.Plan.Changes { + if change.Destructive { + required[change.ID] = true + } + } + for _, id := range request.Acknowledgements { + if !required[id] { + return fmt.Errorf("unknown or duplicate destructive acknowledgement %q", id) + } + delete(required, id) + } + if len(required) > 0 { + return fmt.Errorf("migration requires acknowledgements for %d destructive changes", len(required)) + } + return nil +} + +// Validate checks reviewed inputs without opening a database or starting a transaction. +func Validate(request Request) error { return validate(request) } + +func lock(ctx context.Context, tx *sql.Tx, timeout time.Duration) error { + var status int + if err := tx.QueryRowContext(ctx, `DECLARE @status int; + EXEC @status = sys.sp_getapplock @Resource=N'commons-db/migrate/schema', + @LockMode='Exclusive', @LockOwner='Transaction', @LockTimeout=@p1; + SELECT @status`, timeout.Milliseconds()).Scan(&status); err != nil { + return fmt.Errorf("lock migration transaction: %w", err) + } + if status < 0 { + return fmt.Errorf("lock migration transaction: sp_getapplock returned %d", status) + } + return nil +} diff --git a/database/migrate/sqlserver_integration_test.go b/database/migrate/sqlserver_integration_test.go new file mode 100644 index 00000000..28398a13 --- /dev/null +++ b/database/migrate/sqlserver_integration_test.go @@ -0,0 +1,477 @@ +package migrate + +import ( + "context" + "database/sql" + "fmt" + "net/url" + "os" + "strings" + "testing/fstest" + "time" + + "ariga.io/atlas/sql/schema" + "ariga.io/atlas/sql/sqlclient" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/flanksource/sqlglot-go/database/inspect/inspector" + executor "github.com/flanksource/sqlglot-go/database/migrate/sqlserver" +) + +// SQL Server integration uses an explicit disposable admin URL. The integration +// harness copies tests into a container, so local and remote Docker work alike. + +const integrationTablesHCL = `schema "dbo" {} + +table "acme_orders" { + schema = schema.dbo + column "id" { + type = int + identity { + seed = 1 + increment = 1 + } + } + column "status" { + type = nvarchar(20) + default = "open" + } + column "total" { + type = decimal(18, 2) + default = 0 + } + column "created_at" { + type = datetime2(3) + default = sql("sysutcdatetime()") + } + primary_key { + columns = [column.id] + } + index "IX_acme_orders_status" { + columns = [column.status] + include = [column.total] + where = "status <> 'closed'" + } + check "CK_acme_orders_total" { + expr = "total >= 0" + } +} + +table "acme_order_lines" { + schema = schema.dbo + column "line_id" { + type = uniqueidentifier + } + column "order_id" { + type = int + } + column "sku" { + type = sql("varchar(max)") + null = true + } + primary_key { + columns = [column.line_id] + clustered = false + } + foreign_key "FK_acme_order_lines_order" { + columns = [column.order_id] + ref_columns = [table.acme_orders.column.id] + on_delete = CASCADE + } +} +` + +const integrationCountOrders = `-- Counts a customer's orders. +CREATE PROCEDURE dbo.acme_count_orders + @status NVARCHAR(20), + @count INT OUTPUT +AS +BEGIN + SET NOCOUNT ON; + IF NOT EXISTS (SELECT 1 FROM dbo.acme_orders WHERE status = @status) + RETURN; + SET @count = (SELECT COUNT(*) FROM dbo.acme_orders WHERE status = @status); +END +GO +` + +const integrationOrderTotal = `CREATE OR ALTER FUNCTION dbo.acme_order_total(@order_id INT) +RETURNS DECIMAL(18, 2) +AS +BEGIN + RETURN (SELECT total FROM dbo.acme_orders WHERE id = @order_id); +END +` + +func integrationBundle() fstest.MapFS { + return fstest.MapFS{ + "db/schema.hcl": file(integrationTablesHCL), + "db/routines/acme_count_orders.sql": file(integrationCountOrders), + "db/routines/acme_order_total.sql": file(integrationOrderTotal), + } +} + +var integrationOptions = []Option{WithDir("db"), WithRoutines("db/routines"), WithName("acme/integration")} + +var _ = Describe("SQL Server migrations", Ordered, Label("integration"), func() { + var adminURL string + + BeforeAll(func(ctx SpecContext) { + adminURL = sqlServerAdminURL(ctx) + }) + + DescribeTable("reconcile tables and routines idempotently", + func(ctx SpecContext, compatibilityLevel int, via connectionForm) { + dsn, db := sqlServerDatabase(ctx, adminURL, compatibilityLevel) + reconcileScenario(ctx, sqlServerApplier(dsn, db, via), dsn, db) + }, + Entry("at the engine's default compatibility level", 0, viaURL), + Entry("at compatibility level 100", 100, viaURL), + Entry("through an ADO connection string", 0, viaADO), + Entry("through WithDB on the caller's handle, which stays open", 0, viaHandle), + ) + + It("reads the migrated database through the catalog inspector", func(ctx SpecContext) { + dsn, db := sqlServerDatabase(ctx, adminURL, 0) + Expect(Apply(ctx, dsn, integrationBundle(), integrationOptions...)).To(Succeed()) + inspectCatalog(ctx, db) + }) + + It("previews an HCL and routine bundle without mutation and applies its native artifact", func(ctx SpecContext) { + _, db := sqlServerDatabase(ctx, adminURL, 100) + preview, err := PreviewBundle(ctx, db, "sqlserver", integrationBundle(), integrationOptions...) + Expect(err).NotTo(HaveOccurred()) + Expect(preview.SQLServerArtifact).NotTo(BeNil()) + var count int + Expect(db.QueryRowContext(ctx, "SELECT count(*) FROM sys.tables WHERE is_ms_shipped = 0").Scan(&count)).To(Succeed()) + Expect(count).To(BeZero()) + artifact := *preview.SQLServerArtifact + result, err := executor.Apply(ctx, db, executor.Request{Artifact: artifact, Hash: artifact.Hash}) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Outcome).To(Equal(executor.Applied)) + result, err = executor.Apply(ctx, db, executor.Request{Artifact: artifact, Hash: artifact.Hash}) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Outcome).To(Equal(executor.AlreadyApplied)) + again, err := PreviewBundle(ctx, db, "sqlserver", integrationBundle(), integrationOptions...) + Expect(err).NotTo(HaveOccurred()) + Expect(again.SQLServerArtifact.Plan.Steps).To(BeEmpty()) + }) +}) + +// connectionForm is how a spec hands the database to Apply. +type connectionForm string + +const ( + viaURL connectionForm = "sqlserver:// URL" + viaADO connectionForm = "ADO connection string" + viaHandle connectionForm = "WithDB handle" +) + +// sqlServerApplier applies a bundle to the database at dsn, whose open +// handle is db, in the given connection form. +func sqlServerApplier(dsn string, db *sql.DB, via connectionForm) func(context.Context, fstest.MapFS) error { + return func(ctx context.Context, bundle fstest.MapFS) error { + switch via { + case viaADO: + return Apply(ctx, adoConnectionString(dsn), bundle, integrationOptions...) + case viaHandle: + return Apply(ctx, "", bundle, append([]Option{WithDB(db)}, integrationOptions...)...) + default: + return Apply(ctx, dsn, bundle, integrationOptions...) + } + } +} + +// adoConnectionString rewrites a sqlserver:// URL as the equivalent ADO +// key=value connection string. +func adoConnectionString(dsn string) string { + u, err := url.Parse(dsn) + Expect(err).ToNot(HaveOccurred()) + password, _ := u.User.Password() + pairs := []string{"server=" + u.Hostname(), "user id=" + u.User.Username(), "password=" + password, + "database=" + u.Query().Get("database")} + if port := u.Port(); port != "" { + pairs = append(pairs, "port="+port) + } + return strings.Join(pairs, ";") +} + +func reconcileScenario(ctx context.Context, apply func(context.Context, fstest.MapFS) error, dsn string, db *sql.DB) { + bundle := integrationBundle() + By("applying a table, an index, a foreign key and two routines") + Expect(apply(ctx, bundle)).To(Succeed()) + Expect(pendingSQLServerChanges(ctx, dsn, bundle)).To(BeEmpty()) + _, err := db.ExecContext(ctx, "CREATE INDEX IX_dba_tuning ON dbo.acme_orders (created_at)") + Expect(err).ToNot(HaveOccurred()) + before := modifyDates(ctx, db) + Expect(before).To(HaveLen(4)) + + By("re-applying the same bundle, which changes nothing") + Expect(pendingSQLServerChanges(ctx, dsn, bundle)).To(BeEmpty()) + Expect(apply(ctx, bundle)).To(Succeed()) + Expect(modifyDates(ctx, db)).To(Equal(before)) + Expect(objectExists(ctx, db, "IX_dba_tuning")).To(BeTrue(), "an index the bundle does not declare is kept") + + By("editing one routine's body, which alters exactly that routine") + bundle["db/routines/acme_order_total.sql"] = file(strings.Replace(integrationOrderTotal, "RETURN (SELECT total", "RETURN (SELECT total * 2", 1)) + Expect(changeKinds(pendingSQLServerChanges(ctx, dsn, bundle))).To(Equal([]string{"*schema.ModifyFunc"})) + Expect(apply(ctx, bundle)).To(Succeed()) + after := modifyDates(ctx, db) + Expect(after["acme_order_total"]).To(BeTemporally(">", before["acme_order_total"])) + delete(after, "acme_order_total") + delete(before, "acme_order_total") + Expect(after).To(Equal(before)) + Expect(pendingSQLServerChanges(ctx, dsn, bundle)).To(BeEmpty()) + + By("removing a routine from the bundle, which leaves it in place") + delete(bundle, "db/routines/acme_count_orders.sql") + Expect(apply(ctx, bundle)).To(Succeed()) + Expect(objectExists(ctx, db, "acme_count_orders")).To(BeTrue()) +} + +func pendingSQLServerChanges(ctx context.Context, dsn string, bundle fstest.MapFS) []schema.Change { + desired, err := sqlServerDesired(bundle, resolveOptions(integrationOptions)) + Expect(err).ToNot(HaveOccurred()) + client, err := sqlclient.Open(ctx, dsn) + Expect(err).ToNot(HaveOccurred()) + defer client.Close() //nolint:errcheck + changes, err := sqlServerChanges(ctx, client, desired, nil) + Expect(err).ToNot(HaveOccurred()) + return changes +} + +func changeKinds(changes []schema.Change) []string { + kinds := make([]string, len(changes)) + for i, change := range changes { + kinds[i] = fmt.Sprintf("%T", change) + } + return kinds +} + +func modifyDates(ctx context.Context, db *sql.DB) map[string]time.Time { + rows, err := db.QueryContext(ctx, `SELECT name, modify_date FROM sys.objects + WHERE name IN ('acme_orders', 'acme_order_lines', 'acme_count_orders', 'acme_order_total')`) + Expect(err).ToNot(HaveOccurred()) + defer rows.Close() //nolint:errcheck + dates := map[string]time.Time{} + for rows.Next() { + var name string + var modified time.Time + Expect(rows.Scan(&name, &modified)).To(Succeed()) + dates[name] = modified + } + Expect(rows.Err()).ToNot(HaveOccurred()) + return dates +} + +func objectExists(ctx context.Context, db *sql.DB, name string) bool { + var count int + Expect(db.QueryRowContext(ctx, `SELECT (SELECT COUNT(*) FROM sys.objects WHERE name = @p1) + + (SELECT COUNT(*) FROM sys.indexes WHERE name = @p1)`, name).Scan(&count)).To(Succeed()) + return count > 0 +} + +func inspectCatalog(ctx context.Context, db *sql.DB) { + bounds := &inspector.Bounds{Rows: 1000, DefinitionBytes: 1 << 20} + catalog, err := inspector.NewInspector(db, "sqlserver", bounds) + Expect(err).ToNot(HaveOccurred()) + tables, err := catalog.GetTables(ctx, "dbo", "table") + Expect(err).ToNot(HaveOccurred()) + Expect(tableNames(tables)).To(ConsistOf("acme_orders", "acme_order_lines")) + indexes, err := catalog.GetIndexes(ctx, "dbo") + Expect(err).ToNot(HaveOccurred()) + Expect(indexes).To(ContainElement(HaveField("IndexName", "IX_acme_orders_status"))) + fks, err := catalog.GetForeignKeys(ctx, "dbo") + Expect(err).ToNot(HaveOccurred()) + Expect(fks).To(ContainElement(HaveField("ConstraintName", "FK_acme_order_lines_order"))) + procs, err := catalog.GetStoredProcs(ctx, "dbo") + Expect(err).ToNot(HaveOccurred()) + Expect(procs).To(ContainElements(HaveField("Name", "acme_count_orders"), HaveField("Name", "acme_order_total"))) + params, err := catalog.GetProcParams(ctx, "dbo") + Expect(err).ToNot(HaveOccurred()) + Expect(params).ToNot(BeEmpty()) + Expect(bounds.Truncated).To(BeFalse()) +} + +func tableNames(tables []*inspector.Table) []string { + names := make([]string, len(tables)) + for i, t := range tables { + names[i] = t.Name + } + return names +} + +// sqlServerDatabase creates a database for one spec, at the given +// compatibility level when it is not zero, and drops it afterwards. +func sqlServerDatabase(ctx context.Context, adminURL string, compatibilityLevel int) (string, *sql.DB) { + admin, err := sql.Open("sqlserver", adminURL) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(admin.Close) + name := fmt.Sprintf("commons_db_test_%d", time.Now().UnixNano()) + _, err = admin.ExecContext(ctx, "CREATE DATABASE ["+name+"]") + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func(ctx SpecContext) { + _, err := admin.ExecContext(ctx, "ALTER DATABASE ["+name+"] SET SINGLE_USER WITH ROLLBACK IMMEDIATE; DROP DATABASE ["+name+"]") + Expect(err).ToNot(HaveOccurred()) + }) + if compatibilityLevel != 0 { + _, err = admin.ExecContext(ctx, fmt.Sprintf("ALTER DATABASE [%s] SET COMPATIBILITY_LEVEL = %d", name, compatibilityLevel)) + Expect(err).ToNot(HaveOccurred()) + } + u, err := url.Parse(adminURL) + Expect(err).ToNot(HaveOccurred()) + query := u.Query() + query.Set("database", name) + u.RawQuery = query.Encode() + db, err := sql.Open("sqlserver", u.String()) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(db.Close) + return u.String(), db +} + +// sqlServerAdminURL returns the configured disposable server. +func sqlServerAdminURL(ctx context.Context) string { + if configured := strings.TrimSpace(os.Getenv("COMMONS_DB_SQLSERVER_URL")); configured != "" { + waitForSQLServer(ctx, configured) + return configured + } + Skip("set COMMONS_DB_SQLSERVER_URL to a disposable SQL Server instance") + return "" +} + +func waitForSQLServer(ctx context.Context, adminURL string) { + db, err := sql.Open("sqlserver", adminURL) + Expect(err).ToNot(HaveOccurred()) + defer db.Close() //nolint:errcheck + Eventually(func(ctx context.Context) error { return db.PingContext(ctx) }). + WithContext(ctx).WithTimeout(2 * time.Minute).WithPolling(2 * time.Second).Should(Succeed()) +} + +const ( + sqlServerDSN = "sqlserver://sa:secret@localhost:1433?database=acme" + purgeRoutine = "CREATE OR ALTER PROCEDURE dbo.acme_purge_orders @before DATE AS BEGIN DELETE FROM dbo.acme_orders WHERE created_at < @before; END" + ordersHCLFile = `schema "dbo" {} +table "acme_orders" { + schema = schema.dbo + column "id" { + type = int + } + column "created_at" { + type = date + } + primary_key { + columns = [column.id] + } +} +` +) + +func file(content string) *fstest.MapFile { return &fstest.MapFile{Data: []byte(content)} } + +var _ = Describe("SQL Server routine files", func() { + DescribeTable("loads one routine per file, named after the file", + func(name, content string) { + routines, err := loadRoutines(fstest.MapFS{"routines/" + name: file(content)}, "routines") + Expect(err).ToNot(HaveOccurred()) + Expect(routines).To(HaveLen(1)) + Expect(routines[0].Name).To(Equal("acme_purge_orders")) + Expect(routines[0].Text).To(Equal(purgeRoutine)) + }, + Entry("a bare definition", "acme_purge_orders.sql", purgeRoutine), + Entry("a trailing GO", "acme_purge_orders.sql", purgeRoutine+"\nGO\n"), + ) + + DescribeTable("refuses a file that is not exactly one routine named after it", + func(name, content, message string) { + _, err := loadRoutines(fstest.MapFS{"routines/" + name: file(content)}, "routines") + Expect(err).To(MatchError(ContainSubstring(message))) + }, + Entry("a name mismatch", "acme_purge.sql", purgeRoutine, + `routines/acme_purge.sql creates "acme_purge_orders": the file must be named after its routine`), + Entry("two routines in two batches", "acme_purge_orders.sql", purgeRoutine+"\nGO\n"+purgeRoutine, + "expected exactly one batch, found 2"), + Entry("two routines in one batch", "acme_purge_orders.sql", purgeRoutine+"\n"+purgeRoutine, + "sqlserver: parse routine"), + Entry("a table", "acme_purge_orders.sql", "CREATE TABLE dbo.acme_purge_orders (id INT)", + "makes no routine"), + ) + + It("refuses an empty routine directory", func() { + _, err := loadRoutines(fstest.MapFS{"routines/README.md": file("acme")}, "routines") + Expect(err).To(MatchError(ContainSubstring("no .sql files found"))) + }) +}) + +var _ = Describe("SQL Server migration bundle", func() { + bundle := func() fstest.MapFS { + return fstest.MapFS{ + "db/schema.hcl": file(ordersHCLFile), + "db/routines/acme_purge_orders.sql": file(purgeRoutine), + "db/routines/acme_count_orders.sql": file("CREATE PROCEDURE acme_count_orders AS BEGIN SELECT COUNT(*) FROM dbo.acme_orders; END\nGO"), + } + } + + It("builds the desired dbo schema from HCL tables and routine files, which are not scripts", func() { + realm, err := sqlServerDesired(bundle(), resolveOptions([]Option{WithDir("db"), WithRoutines("db/routines")})) + Expect(err).ToNot(HaveOccurred()) + dbo := realm.Schemas[0] + Expect(dbo.Name).To(Equal("dbo")) + Expect(dbo.Tables).To(HaveLen(1)) + Expect([]string{dbo.Procs[0].Name, dbo.Procs[1].Name}).To(ConsistOf("acme_purge_orders", "acme_count_orders")) + Expect(sqlServerInspectOptions(dbo, nil)).To(Equal(&schema.InspectOptions{ + Mode: schema.InspectSchemas | schema.InspectTables | schema.InspectFuncs, + Tables: []string{"acme_orders", dbo.Procs[0].Name, dbo.Procs[1].Name}, + })) + }) + + It("builds routines alone when the bundle has no HCL", func() { + realm, err := sqlServerDesired(fstest.MapFS{"routines/acme_purge_orders.sql": file(purgeRoutine)}, + resolveOptions([]Option{WithRoutines("routines")})) + Expect(err).ToNot(HaveOccurred()) + Expect(realm.Schemas[0].Procs).To(HaveLen(1)) + Expect(sqlServerInspectOptions(realm.Schemas[0], nil).Mode).To(Equal(schema.InspectSchemas | schema.InspectFuncs)) + }) + + DescribeTable("refuses what SQL Server migrations do not support before connecting", + func(filesystem fstest.MapFS, opts []Option, message string) { + err := Apply(GinkgoT().Context(), sqlServerDSN, filesystem, opts...) + Expect(err).To(MatchError(ContainSubstring(message))) + }, + Entry("an empty bundle", fstest.MapFS{}, nil, `declares no tables or routines`), + Entry("a SQL script outside the routine directory", + fstest.MapFS{"db/routines/acme_purge_orders.sql": file(purgeRoutine), "db/001_seed.sql": file("SELECT 1")}, + []Option{WithDir("db"), WithRoutines("db/routines")}, "do not support SQL migration scripts, found 001_seed.sql"), + Entry("another schema", fstest.MapFS{"db/schema.hcl": file(ordersHCLFile)}, + []Option{WithDir("db"), WithSchema("tenant_x")}, `support only schema "dbo", got "tenant_x"`), + Entry("drops", fstest.MapFS{"db/schema.hcl": file(ordersHCLFile)}, + []Option{WithDir("db"), WithDrops()}, "do not support destructive changes"), + Entry("HCL in another schema", fstest.MapFS{"db/schema.hcl": file(`schema "tenant_x" {}`)}, + []Option{WithDir("db")}, `must declare exactly schema "dbo", got ["tenant_x"]`), + ) + + DescribeTable("WithRoutines is refused by the other dialects", + func(dsn, message string) { + err := Apply(GinkgoT().Context(), dsn, fstest.MapFS{"db/schema.hcl": file(ordersHCLFile)}, + WithDir("db"), WithRoutines("db/routines")) + Expect(err).To(MatchError(message)) + }, + Entry("PostgreSQL", "postgres://acme@localhost:5432/acme", + "PostgreSQL migrations do not support routines (WithRoutines); only SQL Server does"), + Entry("SQLite, before the file is opened", "sqlite://acme-never-created.db", + "SQLite migrations do not support routines (WithRoutines); only SQL Server does"), + ) + + It("keeps undeclared routines and indexes of declared tables", func() { + table := schema.NewTable("acme_orders") + kept := &schema.AddColumn{C: schema.NewIntColumn("priority", "int")} + changes := withoutSQLServerDrops([]schema.Change{ + &schema.DropProc{P: &schema.Proc{Name: "acme_legacy"}}, + &schema.DropFunc{F: &schema.Func{Name: "acme_legacy_total"}}, + &schema.ModifyTable{T: table, Changes: []schema.Change{&schema.DropIndex{I: schema.NewIndex("IX_dba_tuning")}, kept}}, + &schema.ModifyTable{T: table, Changes: []schema.Change{&schema.DropIndex{I: schema.NewIndex("IX_dba_other")}}}, + }) + Expect(changes).To(Equal([]schema.Change{&schema.ModifyTable{T: table, Changes: []schema.Change{kept}}})) + }) +}) diff --git a/database/migrate/suite_test.go b/database/migrate/suite_test.go new file mode 100644 index 00000000..044a28de --- /dev/null +++ b/database/migrate/suite_test.go @@ -0,0 +1,13 @@ +package migrate + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestMigrations(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Standalone Migration Suite") +} diff --git a/database/migrate/view_deps.go b/database/migrate/view_deps.go new file mode 100644 index 00000000..cfaa9eb4 --- /dev/null +++ b/database/migrate/view_deps.go @@ -0,0 +1,180 @@ +package migrate + +import ( + "context" + "database/sql" + "fmt" + + "ariga.io/atlas/sql/schema" + "github.com/flanksource/commons/logger" + "github.com/flanksource/sqlglot-go/database/migrate/viewdeps" + pgnodes "github.com/pgplex/pgparser/nodes" + pgparser "github.com/pgplex/pgparser/parser" +) + +// riskyModifiedTables returns the tables whose pending diff drops the table or drops/alters +// a column — the operations PostgreSQL refuses while a view depends on them. Pure additions +// never block a dependent view, so they are ignored. +func riskyModifiedTables(changes []schema.Change, schemaName string) []viewdeps.Table { + var refs []viewdeps.Table + seen := map[string]bool{} + add := func(t *schema.Table) { + if t == nil { + return + } + // Atlas always reports a schema; default to public so a table it left + // unqualified resolves the same way the diff did rather than through + // whatever search_path this connection happens to carry. + ref := viewdeps.Table{Schema: schemaName, Name: t.Name} + if t.Schema != nil && t.Schema.Name != "" { + ref.Schema = t.Schema.Name + } + if key := ref.Qualified(); !seen[key] { + seen[key] = true + refs = append(refs, ref) + } + } + for _, c := range changes { + switch ch := c.(type) { + case *schema.DropTable: + add(ch.T) + case *schema.ModifyTable: + for _, cc := range ch.Changes { + switch cc.(type) { + case *schema.DropColumn, *schema.ModifyColumn: + add(ch.T) + } + } + } + } + return refs +} + +// managedViews maps each fully-qualified view name to the post-phase SQL script that creates +// it, using pgparser to read CREATE [MATERIALIZED] VIEW targets. The SQL grammar treats +// PL/pgSQL bodies as opaque literals, so mixed view/function/trigger files parse cleanly; +// a script that fails to parse is logged and skipped (its views fall through as unmanaged). +func managedViews(scripts map[string]*script, schemaName string) map[string]string { + owners := map[string]string{} + for _, s := range scripts { + if s.phase != phasePost { + continue + } + stmts, err := pgparser.Parse(s.content) + if err != nil { + logger.GetLogger("migrate").Warnf("skip view attribution for %s: %v", s.path, err) + continue + } + for _, name := range viewNames(stmts, schemaName) { + owners[name] = s.path + } + } + return owners +} + +func viewNames(list *pgnodes.List, schemaName string) []string { + if list == nil { + return nil + } + var names []string + for _, item := range list.Items { + switch stmt := item.(type) { + case *pgnodes.ViewStmt: + names = append(names, qualifyRangeVar(stmt.View, schemaName)) + case *pgnodes.CreateTableAsStmt: + if stmt.Objtype == pgnodes.OBJECT_MATVIEW && stmt.Into != nil { + names = append(names, qualifyRangeVar(stmt.Into.Rel, schemaName)) + } + } + } + return names +} + +func qualifyRangeVar(rv *pgnodes.RangeVar, schemaName string) string { + if rv == nil { + return "" + } + resolvedSchema := rv.Schemaname + if resolvedSchema == "" { + resolvedSchema = schemaName + } + return resolvedSchema + "." + rv.Relname +} + +// invalidateDependentViews clears every view that depends on a table the pending diff drops +// or alters, and returns a func that restores the ones this scope does not own. +// +// A view created by a post-phase script is dropped and its recorded hash deleted, so the post +// phase recreates it from source in the same apply. A view no script owns — an operator's +// ad-hoc reporting view, or one left by an older binary — is captured from the catalog first +// and rebuilt by the returned func, so a schema change neither fails on it nor destroys it. +// +// The returned set of invalidated script paths is empty when nothing needed changing. The +// restore func is never nil. +type viewInvalidationOptions struct { + Scope string + Schema string + Changes []schema.Change + Scripts map[string]*script +} + +func invalidateDependentViews(ctx context.Context, db *sql.DB, opts viewInvalidationOptions) (map[string]bool, func(context.Context) error, error) { + risky := riskyModifiedTables(opts.Changes, opts.Schema) + if len(risky) == 0 { + return nil, noRestore, nil + } + owners := managedViews(opts.Scripts, opts.Schema) + log := logger.GetLogger("migrate") + + dropped, restore, err := viewdeps.Sweep(ctx, viewdeps.DropOptions{ + Tables: risky, + Query: db, + Exec: func(ctx context.Context, stmt string) error { + // DDL takes ACCESS EXCLUSIVE; bound the wait and retry so a live + // reader cannot deadlock reconciliation indefinitely. Drops are + // idempotent (IF EXISTS) and restores run once, so retry is safe. + return retryOnLockContention(ctx, "reconcile dependent view", func() error { + return execWithLockTimeout(ctx, db, stmt) + }) + }, + Owned: func(v viewdeps.View) bool { _, ok := owners[v.Qualified()]; return ok }, + Logf: func(format string, args ...any) { log.V(1).Infof(format, args...) }, + }) + if err != nil { + return nil, nil, err + } + + invalidated := map[string]bool{} + for _, v := range dropped { + if path, ok := owners[v.Qualified()]; ok { + invalidated[path] = true + } + } + for path := range invalidated { + if _, err := db.ExecContext(ctx, + `DELETE FROM schema_migration_scripts WHERE scope = $1 AND path = $2`, opts.Scope, path); err != nil { + return nil, nil, fmt.Errorf("invalidate SQL migration %s: %w", path, err) + } + } + return invalidated, restore, nil +} + +func noRestore(context.Context) error { return nil } + +// execWithLockTimeout runs a single statement in a transaction that first bounds +// its lock waits, so DDL cannot camp on an ACCESS EXCLUSIVE lock against live +// traffic. +func execWithLockTimeout(ctx context.Context, db *sql.DB, stmt string) error { + tx, err := db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() //nolint:errcheck + if _, err := tx.ExecContext(ctx, "SET LOCAL lock_timeout = '"+migrationLockTimeout+"'"); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, stmt); err != nil { + return err + } + return tx.Commit() +} diff --git a/database/migrate/view_deps_test.go b/database/migrate/view_deps_test.go new file mode 100644 index 00000000..ba0e2a06 --- /dev/null +++ b/database/migrate/view_deps_test.go @@ -0,0 +1,120 @@ +package migrate + +import ( + "testing" + + "ariga.io/atlas/sql/schema" + "github.com/stretchr/testify/assert" +) + +func postScript(path, content string) *script { + return &script{path: path, content: content, phase: phasePost} +} + +func TestManagedViews(t *testing.T) { + cases := []struct { + name string + scripts map[string]*script + want map[string]string + }{ + { + name: "create or replace view is attributed with its schema", + scripts: map[string]*script{"v.sql": postScript("v.sql", "CREATE OR REPLACE VIEW public.overview AS SELECT 1")}, + want: map[string]string{"public.overview": "v.sql"}, + }, + { + name: "unqualified view defaults to the public schema", + scripts: map[string]*script{"v.sql": postScript("v.sql", "CREATE VIEW overview AS SELECT 1")}, + want: map[string]string{"public.overview": "v.sql"}, + }, + { + name: "materialized view is attributed", + scripts: map[string]*script{"m.sql": postScript("m.sql", "CREATE MATERIALIZED VIEW reports.daily AS SELECT 1")}, + want: map[string]string{"reports.daily": "m.sql"}, + }, + { + name: "views alongside plpgsql functions and triggers are extracted without error", + scripts: map[string]*script{"mixed.sql": postScript("mixed.sql", ` +CREATE OR REPLACE FUNCTION public.touch() RETURNS trigger LANGUAGE plpgsql AS $$ +BEGIN + RETURN NEW; -- CREATE VIEW inside a body must not be picked up +END; +$$; +CREATE OR REPLACE VIEW public.transcript AS SELECT 1; +CREATE TRIGGER touch_trg BEFORE INSERT ON public.messages FOR EACH ROW EXECUTE FUNCTION public.touch();`)}, + want: map[string]string{"public.transcript": "mixed.sql"}, + }, + { + name: "pre-phase scripts are ignored", + scripts: map[string]*script{ + "pre.sql": {path: "pre.sql", content: "CREATE VIEW public.early AS SELECT 1", phase: phasePre}, + }, + want: map[string]string{}, + }, + { + name: "unparseable scripts are skipped rather than fatal", + scripts: map[string]*script{"bad.sql": postScript("bad.sql", "CREATE VIEW public.broken AS SELECT FROM WHERE ;;")}, + want: map[string]string{}, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, managedViews(tc.scripts, defaultSchema)) + }) + } +} + +func modifyTable(name string, changes ...schema.Change) *schema.ModifyTable { + return &schema.ModifyTable{T: &schema.Table{Name: name, Schema: &schema.Schema{Name: "public"}}, Changes: changes} +} + +func TestRiskyModifiedTables(t *testing.T) { + col := &schema.Column{Name: "c"} + cases := []struct { + name string + changes []schema.Change + want []string + }{ + { + name: "reports the actual table name", + changes: []schema.Change{modifyTable("orders", &schema.DropColumn{C: col})}, + want: []string{"public.orders"}, + }, + { + name: "dropping a column is risky", + changes: []schema.Change{modifyTable("messages", &schema.DropColumn{C: col})}, + want: []string{"public.messages"}, + }, + { + name: "altering a column type is risky", + changes: []schema.Change{modifyTable("messages", &schema.ModifyColumn{From: col, To: col})}, + want: []string{"public.messages"}, + }, + { + name: "dropping a table is risky", + changes: []schema.Change{&schema.DropTable{T: &schema.Table{Name: "messages", Schema: &schema.Schema{Name: "public"}}}}, + want: []string{"public.messages"}, + }, + { + name: "adding a column alone is not risky", + changes: []schema.Change{modifyTable("messages", &schema.AddColumn{C: col})}, + want: nil, + }, + { + name: "a table is reported once despite multiple risky changes", + changes: []schema.Change{ + modifyTable("messages", &schema.DropColumn{C: col}, &schema.ModifyColumn{From: col, To: col}), + }, + want: []string{"public.messages"}, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + var got []string + for _, ref := range riskyModifiedTables(tc.changes, defaultSchema) { + got = append(got, ref.Qualified()) + } + assert.Equal(t, tc.want, got) + }) + } +} diff --git a/database/migrate/viewdeps/capture.go b/database/migrate/viewdeps/capture.go new file mode 100644 index 00000000..f71b7c4d --- /dev/null +++ b/database/migrate/viewdeps/capture.go @@ -0,0 +1,214 @@ +package viewdeps + +import ( + "context" + "database/sql" + "errors" + "fmt" + "strings" +) + +// Definition is everything needed to recreate a view exactly as it was. It is +// read from the catalog before the view is dropped, so a view no migration owns +// survives a schema change instead of blocking it or being lost. +type Definition struct { + View + // SQL is the view body from pg_get_viewdef. Note this is the *expanded* + // definition: a view written as "SELECT a.*" comes back with every column + // listed, so restoring after a column was dropped fails loudly rather than + // silently changing the view's shape. + SQL string + // Indexes are complete CREATE INDEX statements. Materialized views only — + // and load-bearing, since REFRESH ... CONCURRENTLY requires a unique index. + Indexes []string + Owner string + Comment string + Grants []string +} + +const captureQuery = ` +SELECT + pg_get_viewdef(c.oid, true), + pg_get_userbyid(c.relowner), + COALESCE(obj_description(c.oid, 'pg_class'), '') +FROM pg_class c +JOIN pg_namespace n ON n.oid = c.relnamespace +WHERE n.nspname = $1 AND c.relname = $2` + +const captureIndexesQuery = ` +SELECT indexdef FROM pg_indexes WHERE schemaname = $1 AND tablename = $2 ORDER BY indexname` + +// captureGrantsQuery expands the ACL into one row per (grantee, privilege). +// A NULL relacl means default privileges only, and aclexplode yields no rows. +const captureGrantsQuery = ` +SELECT + CASE WHEN a.grantee = 0 THEN 'PUBLIC' ELSE pg_get_userbyid(a.grantee) END, + a.privilege_type, + a.is_grantable +FROM pg_class c +JOIN pg_namespace n ON n.oid = c.relnamespace +CROSS JOIN LATERAL aclexplode(c.relacl) a +WHERE n.nspname = $1 AND c.relname = $2 +ORDER BY 1, 2` + +// Capture reads full definitions for views, preserving the given order. +func Capture(ctx context.Context, q Querier, views []View) ([]Definition, error) { + if q == nil { + return nil, fmt.Errorf("viewdeps: Querier is nil") + } + defs := make([]Definition, 0, len(views)) + for _, v := range views { + def, err := captureOne(ctx, q, v) + if err != nil { + return nil, err + } + defs = append(defs, def) + } + return defs, nil +} + +func captureOne(ctx context.Context, q Querier, v View) (Definition, error) { + def := Definition{View: v} + rows, err := q.QueryContext(ctx, captureQuery, v.Schema, v.Name) + if err != nil { + return def, fmt.Errorf("capture view %s: %w", v.Qualified(), err) + } + if err := scanOne(rows, &def.SQL, &def.Owner, &def.Comment); err != nil { + return def, fmt.Errorf("capture view %s: %w", v.Qualified(), err) + } + if def.SQL == "" { + return def, fmt.Errorf("capture view %s: not found", v.Qualified()) + } + if v.Materialized() { + if def.Indexes, err = captureStrings(ctx, q, captureIndexesQuery, v); err != nil { + return def, fmt.Errorf("capture indexes for %s: %w", v.Qualified(), err) + } + } + if def.Grants, err = captureGrants(ctx, q, v); err != nil { + return def, fmt.Errorf("capture grants for %s: %w", v.Qualified(), err) + } + return def, nil +} + +func scanOne(rows *sql.Rows, dest ...any) (returnErr error) { + defer func() { returnErr = errors.Join(returnErr, rows.Close()) }() + if rows.Next() { + if err := rows.Scan(dest...); err != nil { + return err + } + } + return rows.Err() +} + +func captureStrings(ctx context.Context, q Querier, query string, v View) (_ []string, returnErr error) { + rows, err := q.QueryContext(ctx, query, v.Schema, v.Name) + if err != nil { + return nil, err + } + defer func() { returnErr = errors.Join(returnErr, rows.Close()) }() + var out []string + for rows.Next() { + var s string + if err := rows.Scan(&s); err != nil { + return nil, err + } + out = append(out, s) + } + return out, rows.Err() +} + +func captureGrants(ctx context.Context, q Querier, v View) (_ []string, returnErr error) { + rows, err := q.QueryContext(ctx, captureGrantsQuery, v.Schema, v.Name) + if err != nil { + return nil, err + } + defer func() { returnErr = errors.Join(returnErr, rows.Close()) }() + var grants []string + for rows.Next() { + var grantee, privilege string + var grantable bool + if err := rows.Scan(&grantee, &privilege, &grantable); err != nil { + return nil, err + } + grants = append(grants, grantStatement(v, grantee, privilege, grantable)) + } + return grants, rows.Err() +} + +func grantStatement(v View, grantee, privilege string, grantable bool) string { + // PUBLIC is a keyword, not an identifier, so it must not be quoted. + target := quoteIdent(grantee) + if grantee == "PUBLIC" { + target = grantee + } + stmt := fmt.Sprintf("GRANT %s ON %s.%s TO %s", + privilege, quoteIdent(v.Schema), quoteIdent(v.Name), target) + if grantable { + stmt += " WITH GRANT OPTION" + } + return stmt +} + +// CreateStatements returns the ordered DDL that recreates the view: the view +// itself, then owner, indexes, comment and grants. Materialized views are +// created WITH DATA so they are queryable immediately — an unpopulated +// materialized view errors on every read until someone refreshes it. +func (d Definition) CreateStatements() []string { + body := strings.TrimSuffix(strings.TrimSpace(d.SQL), ";") + kind, name := "VIEW", quoteIdent(d.Schema)+"."+quoteIdent(d.Name) + create := fmt.Sprintf("CREATE VIEW %s AS %s", name, body) + if d.Materialized() { + kind = "MATERIALIZED VIEW" + create = fmt.Sprintf("CREATE MATERIALIZED VIEW %s AS %s WITH DATA", name, body) + } + + stmts := []string{create} + if d.Owner != "" { + stmts = append(stmts, fmt.Sprintf("ALTER %s %s OWNER TO %s", kind, name, quoteIdent(d.Owner))) + } + stmts = append(stmts, d.Indexes...) + if d.Comment != "" { + stmts = append(stmts, fmt.Sprintf("COMMENT ON %s %s IS %s", kind, name, quoteLiteral(d.Comment))) + } + return append(stmts, d.Grants...) +} + +func quoteLiteral(s string) string { + return `'` + strings.ReplaceAll(s, `'`, `''`) + `'` +} + +// RestoreError reports a view that was dropped to allow a schema change and +// could not be recreated. It carries the full DDL so an operator can replay it +// by hand — the database is otherwise migrated correctly but missing the view. +type RestoreError struct { + View View + Statements []string + Err error +} + +func (e *RestoreError) Error() string { + return fmt.Sprintf("restore view %s: %v\n"+ + "the view was dropped to allow the schema change and could not be recreated.\n"+ + "Recreate it manually with:\n\n %s", + e.View.Qualified(), e.Err, strings.Join(e.Statements, ";\n ")+";") +} + +func (e *RestoreError) Unwrap() error { return e.Err } + +// Restore recreates definitions in the order given, which Dependents guarantees +// is dependency order. The first failure aborts and returns a *RestoreError +// naming the view and carrying its DDL. +func Restore(ctx context.Context, exec Exec, defs []Definition) error { + if exec == nil { + return fmt.Errorf("viewdeps: Exec is nil") + } + for _, def := range defs { + stmts := def.CreateStatements() + for _, stmt := range stmts { + if err := exec(ctx, stmt); err != nil { + return &RestoreError{View: def.View, Statements: stmts, Err: err} + } + } + } + return nil +} diff --git a/database/migrate/viewdeps/drop.go b/database/migrate/viewdeps/drop.go new file mode 100644 index 00000000..569e33b4 --- /dev/null +++ b/database/migrate/viewdeps/drop.go @@ -0,0 +1,81 @@ +package viewdeps + +import ( + "context" + "fmt" +) + +// DropOptions configures Sweep. +type DropOptions struct { + // Tables whose dependent views block the pending DDL. Empty is a no-op. + Tables []Table + // Query reads the dependency graph. Required. + Query Querier + // Exec runs each DROP and each restore statement. Required. + Exec Exec + // Owned reports whether the caller recreates this view itself. Owned views + // are dropped and left to the caller — restoring them from the captured + // definition would resurrect a stale one. Unowned views are captured before + // the drop and rebuilt by the returned restore func. Nil owns nothing. + Owned func(View) bool + // Logf, when set, receives one line per dropped view. + Logf func(format string, args ...any) +} + +// Sweep drops every view that transitively depends on opts.Tables and returns +// the views it dropped plus a func that restores the unowned ones. +// +// Call the restore func after the schema change, and after whatever recreates +// the owned views — an unowned view may be built on top of one of them. +// Restoring is not atomic with the DDL; a failure returns a *RestoreError +// carrying the DDL so the view can be replayed by hand. +func Sweep(ctx context.Context, opts DropOptions) ([]View, func(context.Context) error, error) { + if opts.Query == nil { + return nil, nil, fmt.Errorf("viewdeps: DropOptions.Query is nil") + } + if opts.Exec == nil { + return nil, nil, fmt.Errorf("viewdeps: DropOptions.Exec is nil") + } + dependents, err := Dependents(ctx, opts.Query, opts.Tables) + if err != nil { + return nil, nil, err + } + if len(dependents) == 0 { + return nil, noRestore, nil + } + + _, unowned := partition(dependents, opts.Owned) + captured, err := Capture(ctx, opts.Query, unowned) + if err != nil { + return nil, nil, err + } + + // Drop in reverse dependency order so a view is gone before the view it + // reads, keeping each CASCADE to the object actually named. + for i := len(dependents) - 1; i >= 0; i-- { + v := dependents[i] + if err := opts.Exec(ctx, v.DropStatement()); err != nil { + return nil, nil, fmt.Errorf("drop dependent view %s: %w", v.Qualified(), err) + } + if opts.Logf != nil { + opts.Logf("dropped dependent view %s", v.Qualified()) + } + } + return dependents, func(ctx context.Context) error { + return Restore(ctx, opts.Exec, captured) + }, nil +} + +func noRestore(context.Context) error { return nil } + +// partition splits views by ownership, preserving order within each group. +func partition(views []View, owned func(View) bool) (ownedViews, unowned []View) { + for _, v := range views { + if owned != nil && owned(v) { + ownedViews = append(ownedViews, v) + continue + } + unowned = append(unowned, v) + } + return ownedViews, unowned +} diff --git a/database/migrate/viewdeps/viewdeps.go b/database/migrate/viewdeps/viewdeps.go new file mode 100644 index 00000000..9d227b1e --- /dev/null +++ b/database/migrate/viewdeps/viewdeps.go @@ -0,0 +1,218 @@ +// Package viewdeps finds, drops, and restores the views and materialized views +// that transitively depend on a set of tables, so DDL PostgreSQL refuses while a +// dependent view exists — DROP TABLE, DROP COLUMN, ALTER COLUMN TYPE — can +// proceed. +// +// Discovery walks pg_depend/pg_rewrite rather than matching view names, so it is +// complete regardless of how a view is named or which schema it lives in. A +// name-convention sweep silently misses a view outside the connection's +// search_path — the DROP resolves to nothing and succeeds — and the blocked DDL +// then fails much later with a confusing error. +// +// The package deliberately depends only on the standard library so callers can +// use it with any driver: *sql.DB, *sql.Tx and gorm's connection pool all +// satisfy Querier as-is. +package viewdeps + +import ( + "context" + "database/sql" + "errors" + "fmt" + "strings" +) + +// Table identifies a relation whose dependents must be cleared. An empty Schema +// is resolved through the connection's search_path, which is usually what a +// caller migrating its own database wants; set it only to target a specific +// schema. +type Table struct { + Schema string + Name string +} + +// Qualified renders the table for to_regclass. An empty Schema is left +// unqualified deliberately: forcing "public" would target the wrong relation +// whenever the connection runs under a different search_path. +func (t Table) Qualified() string { + if t.Schema == "" { + return t.Name + } + return t.Schema + "." + t.Name +} + +// Tables builds refs for several relations in one schema. An empty schema +// resolves through search_path. +func Tables(schema string, names ...string) []Table { + tables := make([]Table, 0, len(names)) + for _, name := range names { + tables = append(tables, Table{Schema: schema, Name: name}) + } + return tables +} + +// View identifies a live view or materialized view. Kind is pg_class.relkind: +// "v" for a view, "m" for a materialized view. +type View struct { + Schema string + Name string + Kind string +} + +// Qualified renders the view schema-qualified. Unlike Table, a View always +// carries the schema the catalog reported, so this is never ambiguous. +func (v View) Qualified() string { return v.Schema + "." + v.Name } + +// Materialized reports whether the view holds its own copy of the data. +func (v View) Materialized() bool { return v.Kind == relkindMatview } + +// DropStatement returns the schema-qualified, identifier-quoted DROP. CASCADE is +// unconditional: a view built on this one is itself a dependent that has already +// been captured, so taking it here is both safe and necessary. +func (v View) DropStatement() string { + return fmt.Sprintf("DROP %s IF EXISTS %s.%s CASCADE", + v.dropKeyword(), quoteIdent(v.Schema), quoteIdent(v.Name)) +} + +func (v View) dropKeyword() string { + if v.Materialized() { + return "MATERIALIZED VIEW" + } + return "VIEW" +} + +const relkindMatview = "m" + +// Querier reads the dependency graph. *sql.DB, *sql.Tx, *sql.Conn and gorm's +// ConnPool all satisfy it without an adapter. +type Querier interface { + QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error) +} + +// Exec runs a single DDL statement. Callers supply the execution policy: the +// migrate package wraps each statement in a bounded lock_timeout with retry, and +// a gorm caller routes through *gorm.DB so an open transaction stays in scope. +type Exec func(ctx context.Context, stmt string) error + +// dependentsQuery returns every view and materialized view that transitively +// depends on any of the given relations, walking pg_rewrite rules through +// pg_depend. Relations that do not exist are skipped by the to_regclass filter +// rather than erroring, so a caller can pass its full table list before those +// tables have been created. +// +// Rows come back in dependency order. Each row carries the longest path from a +// target table (UNION ALL rather than UNION so depth accumulates; view +// dependencies are acyclic, so this terminates), and taking MAX per view gives +// an order in which every view appears after everything it reads. Restoring in +// that order means a view built on another view is always created second. +const dependentsQuery = ` +WITH RECURSIVE targets(oid) AS ( + SELECT to_regclass(t)::oid + FROM unnest(ARRAY[%s]::text[]) AS t + WHERE to_regclass(t) IS NOT NULL +), +deps(view_oid, depth) AS ( + SELECT r.ev_class, 1 + FROM pg_depend d + JOIN pg_rewrite r ON r.oid = d.objid AND d.classid = 'pg_rewrite'::regclass + JOIN targets t ON t.oid = d.refobjid + WHERE r.ev_class <> d.refobjid + UNION ALL + SELECT r.ev_class, deps.depth + 1 + FROM deps + JOIN pg_depend d ON d.refobjid = deps.view_oid AND d.refclassid = 'pg_class'::regclass + JOIN pg_rewrite r ON r.oid = d.objid AND d.classid = 'pg_rewrite'::regclass + WHERE r.ev_class <> d.refobjid +) +SELECT n.nspname, c.relname, c.relkind::text +FROM deps +JOIN pg_class c ON c.oid = deps.view_oid +JOIN pg_namespace n ON n.oid = c.relnamespace +WHERE c.relkind IN ('v', 'm') +GROUP BY 1, 2, 3 +ORDER BY MAX(deps.depth), 1, 2` + +// Dependents returns every view and materialized view that transitively depends +// on any of tables, in dependency order: a view always follows everything it +// reads. Tables that do not exist are ignored; the tables themselves are never +// returned. +func Dependents(ctx context.Context, q Querier, tables []Table) ([]View, error) { + if q == nil { + return nil, fmt.Errorf("viewdeps: Querier is nil") + } + if len(tables) == 0 { + return nil, nil + } + args := make([]any, len(tables)) + for i, t := range tables { + args[i] = t.Qualified() + } + rows, err := q.QueryContext(ctx, fmt.Sprintf(dependentsQuery, arrayPlaceholders(len(args))), args...) + if err != nil { + return nil, fmt.Errorf("inspect view dependencies: %w", err) + } + return scanViews(rows) +} + +// lookupQuery resolves relation names to their real schema. Bare names go +// through search_path, so this is the safe way to turn a name a caller +// constructed into something that can be dropped without relying on the +// session's search_path at DROP time. +const lookupQuery = ` +SELECT DISTINCT n.nspname, c.relname, c.relkind::text +FROM unnest(ARRAY[%s]::text[]) AS t +JOIN pg_class c ON c.oid = to_regclass(t) +JOIN pg_namespace n ON n.oid = c.relnamespace +WHERE c.relkind IN ('v', 'm') +ORDER BY 1, 2` + +// Lookup resolves relation names to schema-qualified view refs, skipping names +// that do not exist or name something that is not a view. +func Lookup(ctx context.Context, q Querier, names ...string) ([]View, error) { + if q == nil { + return nil, fmt.Errorf("viewdeps: Querier is nil") + } + if len(names) == 0 { + return nil, nil + } + args := make([]any, len(names)) + for i, name := range names { + args[i] = name + } + rows, err := q.QueryContext(ctx, fmt.Sprintf(lookupQuery, arrayPlaceholders(len(args))), args...) + if err != nil { + return nil, fmt.Errorf("resolve views: %w", err) + } + return scanViews(rows) +} + +func scanViews(rows *sql.Rows) (_ []View, returnErr error) { + defer func() { returnErr = errors.Join(returnErr, rows.Close()) }() + var views []View + for rows.Next() { + var v View + if err := rows.Scan(&v.Schema, &v.Name, &v.Kind); err != nil { + return nil, fmt.Errorf("read view row: %w", err) + } + views = append(views, v) + } + return views, rows.Err() +} + +// arrayPlaceholders renders "$1,$2,...,$n" for an n-element array literal. +// Building the array in SQL rather than binding a driver array type keeps the +// query portable across lib/pq, pgx and gorm. +func arrayPlaceholders(n int) string { + parts := make([]string, n) + for i := range parts { + parts[i] = fmt.Sprintf("$%d", i+1) + } + return strings.Join(parts, ",") +} + +// quoteIdent renders a PostgreSQL identifier safely, doubling any embedded +// quote. Always quoting means mixed-case and reserved names survive a +// drop/recreate round-trip unchanged. +func quoteIdent(s string) string { + return `"` + strings.ReplaceAll(s, `"`, `""`) + `"` +} diff --git a/database/migrate/viewdeps/viewdeps_test.go b/database/migrate/viewdeps/viewdeps_test.go new file mode 100644 index 00000000..45fde57e --- /dev/null +++ b/database/migrate/viewdeps/viewdeps_test.go @@ -0,0 +1,204 @@ +package viewdeps + +import ( + "context" + "database/sql" + "errors" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestViewdeps(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "viewdeps") +} + +// stubQuerier satisfies Querier for argument-validation specs that must fail +// before any query is issued. +type stubQuerier struct{} + +func (stubQuerier) QueryContext(context.Context, string, ...any) (*sql.Rows, error) { + return nil, errors.New("stubQuerier: unexpected query") +} + +const ( + reportingSchema = "reporting" + accountsMatview = "mv_accounts_x" +) + +var _ = Describe("Table", func() { + It("qualifies against the given schema", func() { + Expect(Table{Schema: reportingSchema, Name: "accounts"}.Qualified()).To(Equal("reporting.accounts")) + }) + + It("leaves an empty schema unqualified so search_path resolves it", func() { + // Defaulting to "public" here would target the wrong relation whenever + // the connection runs under a different search_path — the exact bug + // this package exists to prevent. + Expect(Table{Name: "accounts"}.Qualified()).To(Equal("accounts")) + }) + + It("builds refs for several names in one schema", func() { + Expect(Tables(reportingSchema, "a", "b")).To(Equal([]Table{ + {Schema: reportingSchema, Name: "a"}, + {Schema: reportingSchema, Name: "b"}, + })) + }) + + It("builds search_path-relative refs when the schema is empty", func() { + Expect(Tables("", "a")).To(Equal([]Table{{Name: "a"}})) + }) +}) + +var _ = Describe("View", func() { + It("drops a plain view schema-qualified with CASCADE", func() { + v := View{Schema: "public", Name: "user_report", Kind: "v"} + Expect(v.DropStatement()).To(Equal(`DROP VIEW IF EXISTS "public"."user_report" CASCADE`)) + Expect(v.Materialized()).To(BeFalse()) + }) + + It("drops a materialized view with the MATERIALIZED keyword", func() { + v := View{Schema: reportingSchema, Name: accountsMatview, Kind: "m"} + Expect(v.DropStatement()).To(Equal( + `DROP MATERIALIZED VIEW IF EXISTS "reporting"."mv_accounts_x" CASCADE`)) + Expect(v.Materialized()).To(BeTrue()) + }) + + It("escapes an embedded quote by doubling it", func() { + v := View{Schema: "public", Name: `we"ird`, Kind: "v"} + Expect(v.DropStatement()).To(Equal(`DROP VIEW IF EXISTS "public"."we""ird" CASCADE`)) + }) + + It("always qualifies, so a drop never depends on search_path", func() { + Expect(View{Schema: reportingSchema, Name: accountsMatview}.Qualified()). + To(Equal("reporting.mv_accounts_x")) + }) +}) + +var _ = Describe("arrayPlaceholders", func() { + It("numbers placeholders from one", func() { + Expect(arrayPlaceholders(3)).To(Equal("$1,$2,$3")) + }) + + It("renders a single placeholder without a separator", func() { + Expect(arrayPlaceholders(1)).To(Equal("$1")) + }) +}) + +var _ = Describe("partition", func() { + views := []View{ + {Schema: "public", Name: "owned", Kind: "v"}, + {Schema: "public", Name: "foreign", Kind: "v"}, + } + byName := func(want string) func(View) bool { + return func(v View) bool { return v.Name == want } + } + + It("splits views by the ownership predicate", func() { + owned, unowned := partition(views, byName("owned")) + Expect(owned).To(Equal(views[:1])) + Expect(unowned).To(Equal(views[1:])) + }) + + It("treats every view as unowned when the predicate is nil", func() { + owned, unowned := partition(views, nil) + Expect(owned).To(BeEmpty()) + Expect(unowned).To(Equal(views)) + }) +}) + +var _ = Describe("Definition.CreateStatements", func() { + base := Definition{ + View: View{Schema: "public", Name: "user_report", Kind: "v"}, + SQL: " SELECT accounts.code\n FROM accounts;", + Owner: "xero", + } + + It("recreates a plain view with the trailing semicolon stripped", func() { + Expect(base.CreateStatements()).To(Equal([]string{ + "CREATE VIEW \"public\".\"user_report\" AS SELECT accounts.code\n FROM accounts", + `ALTER VIEW "public"."user_report" OWNER TO "xero"`, + })) + }) + + It("creates a materialized view WITH DATA so it is queryable immediately", func() { + mv := base + mv.Kind = "m" + mv.Indexes = []string{`CREATE UNIQUE INDEX pk ON public.user_report USING btree (code)`} + Expect(mv.CreateStatements()).To(Equal([]string{ + "CREATE MATERIALIZED VIEW \"public\".\"user_report\" AS SELECT accounts.code\n FROM accounts WITH DATA", + `ALTER MATERIALIZED VIEW "public"."user_report" OWNER TO "xero"`, + `CREATE UNIQUE INDEX pk ON public.user_report USING btree (code)`, + })) + }) + + It("orders comment and grants after the indexes", func() { + d := base + d.Comment = "it's mine" + d.Grants = []string{`GRANT SELECT ON "public"."user_report" TO PUBLIC`} + Expect(d.CreateStatements()[2:]).To(Equal([]string{ + `COMMENT ON VIEW "public"."user_report" IS 'it''s mine'`, + `GRANT SELECT ON "public"."user_report" TO PUBLIC`, + })) + }) + + It("omits the owner statement when no owner was captured", func() { + d := base + d.Owner = "" + Expect(d.CreateStatements()).To(HaveLen(1)) + }) +}) + +var _ = Describe("grantStatement", func() { + v := View{Schema: "public", Name: "user_report", Kind: "v"} + + It("quotes a role but leaves PUBLIC as a bare keyword", func() { + Expect(grantStatement(v, "PUBLIC", "SELECT", false)).To(Equal( + `GRANT SELECT ON "public"."user_report" TO PUBLIC`)) + Expect(grantStatement(v, "reader", "SELECT", false)).To(Equal( + `GRANT SELECT ON "public"."user_report" TO "reader"`)) + }) + + It("appends WITH GRANT OPTION when the privilege is grantable", func() { + Expect(grantStatement(v, "reader", "UPDATE", true)).To(HaveSuffix(" WITH GRANT OPTION")) + }) +}) + +var _ = Describe("RestoreError", func() { + It("names the view and embeds replayable DDL", func() { + err := &RestoreError{ + View: View{Schema: reportingSchema, Name: "user_report", Kind: "v"}, + Statements: []string{"CREATE VIEW a AS SELECT 1", "ALTER VIEW a OWNER TO x"}, + Err: errors.New(`column "tenant_id" does not exist`), + } + Expect(err.Error()).To(ContainSubstring("reporting.user_report")) + Expect(err.Error()).To(ContainSubstring(`column "tenant_id" does not exist`)) + Expect(err.Error()).To(ContainSubstring("CREATE VIEW a AS SELECT 1;\n ALTER VIEW a OWNER TO x;")) + }) +}) + +var _ = Describe("fail-loud argument checks", func() { + ctx := context.Background() + table := Tables("", "accounts") + + It("rejects a nil Querier rather than panicking", func() { + _, err := Dependents(ctx, nil, table) + Expect(err).To(MatchError(ContainSubstring("Querier is nil"))) + _, err = Lookup(ctx, nil, "v") + Expect(err).To(MatchError(ContainSubstring("Querier is nil"))) + }) + + It("rejects a Sweep with no Query or no Exec", func() { + _, _, err := Sweep(ctx, DropOptions{Tables: table, Exec: func(context.Context, string) error { return nil }}) + Expect(err).To(MatchError(ContainSubstring("Query is nil"))) + + _, _, err = Sweep(ctx, DropOptions{Tables: table, Query: stubQuerier{}}) + Expect(err).To(MatchError(ContainSubstring("Exec is nil"))) + }) + + It("rejects a nil Exec on Restore", func() { + Expect(Restore(ctx, nil, []Definition{{}})).To(MatchError(ContainSubstring("Exec is nil"))) + }) +}) diff --git a/database/migrate/with_db_test.go b/database/migrate/with_db_test.go new file mode 100644 index 00000000..89507de4 --- /dev/null +++ b/database/migrate/with_db_test.go @@ -0,0 +1,56 @@ +package migrate + +import ( + "context" + "database/sql" + "path/filepath" + "testing/fstest" + + "github.com/flanksource/sqlglot-go/database/target" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Apply WithDB", func() { + openSQLiteHandle := func(name string) *sql.DB { + target, err := dbtarget.Parse(filepath.Join(GinkgoT().TempDir(), name)) + Expect(err).ToNot(HaveOccurred()) + database, err := sql.Open("sqlite", target.DSN) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(database.Close) + return database + } + files := fstest.MapFS{"schema.hcl": &fstest.MapFile{Data: []byte(portableSchema)}} + + It("migrates a caller's SQLite handle and leaves it open with its pool settings", func(ctx context.Context) { + database := openSQLiteHandle("handle.db") + database.SetMaxOpenConns(4) + + Expect(Apply(ctx, "", files, WithDB(database))).To(Succeed()) + + var tables int + Expect(database.QueryRowContext(ctx, + `SELECT count(*) FROM sqlite_schema WHERE type = 'table' AND name IN ('projects', 'events')`).Scan(&tables)).To(Succeed()) + Expect(tables).To(Equal(2)) + Expect(database.Stats().MaxOpenConnections).To(Equal(4)) + Expect(Apply(ctx, "", files, WithDB(database))).To(Succeed()) + }) + + It("refuses a connection string alongside a handle", func(ctx context.Context) { + database := openSQLiteHandle("both.db") + Expect(Apply(ctx, "state/other.db", files, WithDB(database))). + To(MatchError("pass a connection string or WithDB, not both")) + }) + + It("refuses an empty connection string without a handle", func(ctx context.Context) { + Expect(Apply(ctx, " ", files)).To(MatchError("connection string is empty")) + }) + + It("refuses a PostgreSQL handle", func(ctx context.Context) { + database, err := sql.Open("postgres", "postgres://localhost/acme?sslmode=disable") + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(database.Close) + Expect(Apply(ctx, "", files, WithDB(database))). + To(MatchError(ContainSubstring("WithDB does not support PostgreSQL"))) + }) +}) diff --git a/database/ownership_test.go b/database/ownership_test.go new file mode 100644 index 00000000..4ab9cc01 --- /dev/null +++ b/database/ownership_test.go @@ -0,0 +1,32 @@ +package database_test + +import ( + "context" + "database/sql" + "testing" + + inspection "github.com/flanksource/sqlglot-go/database/inspect" + "github.com/flanksource/sqlglot-go/database/migrate" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + _ "modernc.org/sqlite" +) + +func TestDatabase(t *testing.T) { RegisterFailHandler(Fail); RunSpecs(t, "Standalone database") } + +var _ = Describe("standalone database ownership", func() { + It("inspects SQLite without a commons-db connection or cache", func() { + db, err := sql.Open("sqlite", ":memory:") + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(db.Close) + _, err = db.Exec("CREATE TABLE example (id integer PRIMARY KEY)") + Expect(err).NotTo(HaveOccurred()) + catalog, err := inspection.Inspect(context.Background(), db, "sqlite", inspection.Limits{}) + Expect(err).NotTo(HaveOccurred()) + Expect(catalog.Schemas).To(HaveLen(1)) + Expect(catalog.Schemas[0].Relations[0].Name).To(Equal("example")) + }) + It("retains the standalone bundle apply API", func() { + Expect(migrate.Apply(context.Background(), "", nil)).To(MatchError(ContainSubstring("connection"))) + }) +}) diff --git a/database/target/file_uri_test.go b/database/target/file_uri_test.go new file mode 100644 index 00000000..7b0e7324 --- /dev/null +++ b/database/target/file_uri_test.go @@ -0,0 +1,13 @@ +package dbtarget + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = DescribeTable("SQLite file URI paths", func(path, expected string) { + Expect(sqliteFileURL(path).String()).To(Equal(expected)) +}, + Entry("Windows drive", "C:/temp/tenant #1.db", "file:///C:/temp/tenant%20%231.db"), + Entry("POSIX root", "/var/data/tenant #1.db", "file:///var/data/tenant%20%231.db"), +) diff --git a/database/target/target.go b/database/target/target.go new file mode 100644 index 00000000..e71d8c45 --- /dev/null +++ b/database/target/target.go @@ -0,0 +1,162 @@ +package dbtarget + +import ( + "fmt" + "net/url" + "path/filepath" + "strings" +) + +type Dialect string + +const ( + Postgres Dialect = "postgres" + SQLite Dialect = "sqlite" + SQLServer Dialect = "sqlserver" +) + +type Target struct { + Dialect Dialect + DSN string +} + +func Parse(raw string) (Target, error) { + dsn := strings.TrimSpace(raw) + if dsn == "" { + return Target{}, fmt.Errorf("database DSN is required") + } + if dsn == ":memory:" { + return Target{}, fmt.Errorf("ambiguous SQLite memory DSN %q; use a file-backed sqlite:// DSN", dsn) + } + if strings.HasPrefix(strings.ToLower(dsn), "sqlite://") { + return parseSQLite(dsn[len("sqlite://"):]) + } + if isSQLServerKeywordDSN(dsn) { + return Target{Dialect: SQLServer, DSN: dsn}, nil + } + if scheme := explicitScheme(dsn); scheme != "" { + switch scheme { + case "postgres", "postgresql": + return Target{Dialect: Postgres, DSN: dsn}, nil + case "sqlserver": + return Target{Dialect: SQLServer, DSN: dsn}, nil + case "mssql": + return Target{}, fmt.Errorf("unsupported database scheme %q; use sqlserver:// for SQL Server", scheme) + default: + return Target{}, fmt.Errorf("unsupported database scheme %q", scheme) + } + } + if !isPostgresKeywordDSN(dsn) && strings.EqualFold(filepath.Ext(pathWithoutQuery(dsn)), ".db") { + return parseSQLite(dsn) + } + return Target{Dialect: Postgres, DSN: dsn}, nil +} + +func parseSQLite(raw string) (Target, error) { + path, rawQuery, _ := strings.Cut(raw, "?") + if strings.TrimSpace(path) == "" { + return Target{}, fmt.Errorf("SQLite DSN path is required") + } + query, err := url.ParseQuery(rawQuery) + if err != nil { + return Target{}, fmt.Errorf("parse SQLite DSN query: %w", err) + } + fileURI := strings.HasPrefix(strings.ToLower(path), "file:") + filename := path + if fileURI { + if filename, err = url.PathUnescape(path[len("file:"):]); err != nil { + return Target{}, fmt.Errorf("decode SQLite file URI %q: %w", path, err) + } + } + if filename == ":memory:" || strings.EqualFold(query.Get("mode"), "memory") { + return Target{}, fmt.Errorf("SQLite migrations require a file-backed database") + } + + var parsed *url.URL + if fileURI { + parsed, err = url.Parse(path) + if err != nil { + return Target{}, fmt.Errorf("parse SQLite file URI: %w", err) + } + } else { + absolute, resolveErr := filepath.Abs(path) + if resolveErr != nil { + return Target{}, fmt.Errorf("resolve SQLite path %q: %w", path, resolveErr) + } + parsed = sqliteFileURL(filepath.Clean(absolute)) + } + parsed.RawQuery = managedSQLiteQuery(query).Encode() + return Target{Dialect: SQLite, DSN: parsed.String()}, nil +} + +func sqliteFileURL(path string) *url.URL { + path = filepath.ToSlash(path) + if !strings.HasPrefix(path, "/") { + path = "/" + path + } + return &url.URL{Scheme: "file", Path: path} +} + +func managedSQLiteQuery(query url.Values) url.Values { + managed := map[string]bool{"foreign_keys": true, "busy_timeout": true, "journal_mode": true} + pragmas := query["_pragma"][:0] + for _, pragma := range query["_pragma"] { + name, _, _ := strings.Cut(pragma, "(") + if !managed[strings.ToLower(strings.TrimSpace(name))] { + pragmas = append(pragmas, pragma) + } + } + query["_pragma"] = append(pragmas, "foreign_keys(1)", "busy_timeout(5000)", "journal_mode(WAL)") + return query +} + +func explicitScheme(dsn string) string { + index := strings.Index(dsn, "://") + if index <= 0 { + return "" + } + return strings.ToLower(dsn[:index]) +} + +func pathWithoutQuery(dsn string) string { + path, _, _ := strings.Cut(dsn, "?") + return path +} + +// isSQLServerKeywordDSN reports whether dsn is a go-mssqldb ODBC DSN or an ADO +// DSN: ';'-separated key=value pairs naming the server by one of the address +// keys msdsn.Parse reads. A space-separated PostgreSQL keyword DSN never +// matches. +func isSQLServerKeywordDSN(dsn string) bool { + if strings.HasPrefix(strings.ToLower(dsn), "odbc:") { + return true + } + if isPostgresKeywordDSN(dsn) { + return false + } + for _, pair := range strings.Split(dsn, ";") { + key, _, found := strings.Cut(pair, "=") + if !found { + continue + } + switch strings.ToLower(strings.TrimSpace(key)) { + case "server", "data source", "address", "addr", "network address": + return true + } + } + return false +} + +func isPostgresKeywordDSN(dsn string) bool { + for _, field := range strings.Fields(dsn) { + key, _, found := strings.Cut(field, "=") + if !found { + continue + } + switch strings.ToLower(key) { + case "host", "hostaddr", "port", "dbname", "user", "password", "sslmode", "service": + return true + } + } + return false +} diff --git a/database/target/target_test.go b/database/target/target_test.go new file mode 100644 index 00000000..59d57115 --- /dev/null +++ b/database/target/target_test.go @@ -0,0 +1,136 @@ +package dbtarget_test + +import ( + "database/sql" + "net/url" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/flanksource/sqlglot-go/database/target" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + _ "modernc.org/sqlite" +) + +var _ = Describe("database target DSNs", func() { + DescribeTable("classifies PostgreSQL without rewriting its DSN", + func(dsn string) { + target, err := dbtarget.Parse(dsn) + Expect(err).ToNot(HaveOccurred()) + Expect(target).To(Equal(dbtarget.Target{Dialect: dbtarget.Postgres, DSN: dsn})) + }, + Entry("postgres URL", "postgres://localhost/app?sslmode=disable"), + Entry("postgresql URL", "postgresql://localhost/app?sslmode=disable"), + Entry("keyword DSN whose database ends in db", "host=localhost dbname=example.db sslmode=disable"), + ) + + DescribeTable("classifies SQL Server without rewriting its DSN", + func(dsn string) { + target, err := dbtarget.Parse(dsn) + Expect(err).ToNot(HaveOccurred()) + Expect(target).To(Equal(dbtarget.Target{Dialect: dbtarget.SQLServer, DSN: dsn})) + }, + Entry("sqlserver URL with database query", "sqlserver://sa:secret@localhost:1433?database=oipa"), + Entry("upper-case scheme", "SQLServer://sa:secret@localhost?database=oipa"), + Entry("credentials with escaped special characters", "sqlserver://app%40corp:p%40ss%3Aw%2Frd%23@db.example:1433/instance?database=x&encrypt=disable"), + Entry("ADO keywords", "server=db.example;user id=app;password=secret;database=acme;encrypt=disable"), + Entry("ADO data source with initial catalog", "Data Source=db.example,1433;Initial Catalog=acme;User ID=app;Password=secret"), + Entry("ADO mixed-case keys and a trailing separator", "Server = db.example ; Database=acme;"), + Entry("ADO address key", "addr=db.example;database=acme"), + Entry("ODBC prefix", "odbc:server=db.example;user id=app;password={se;cret};database=acme"), + ) + + It("keeps a space-separated keyword DSN on PostgreSQL even when it names a server key", func() { + target, err := dbtarget.Parse("host=localhost dbname=acme server=ignored") + Expect(err).ToNot(HaveOccurred()) + Expect(target.Dialect).To(Equal(dbtarget.Postgres)) + }) + + DescribeTable("classifies and normalizes SQLite", + func(dsn, expectedPath string) { + target, err := dbtarget.Parse(dsn) + Expect(err).ToNot(HaveOccurred()) + Expect(target.Dialect).To(Equal(dbtarget.SQLite)) + + parsed, err := url.Parse(target.DSN) + Expect(err).ToNot(HaveOccurred()) + Expect(parsed.Scheme).To(Equal("file")) + Expect(parsed.Host).To(BeEmpty()) + path := parsed.Path + if filepath.VolumeName(expectedPath) != "" { + path = strings.TrimPrefix(path, "/") + } + Expect(filepath.FromSlash(path)).To(Equal(expectedPath)) + Expect(parsed.Query()["_pragma"]).To(ConsistOf("foreign_keys(1)", "busy_timeout(5000)", "journal_mode(WAL)")) + }, + Entry("absolute sqlite URL", "sqlite:///var/data/uir.db", absolutePath("/var/data/uir.db")), + Entry("explicit relative sqlite URL with any extension", "sqlite://state/uir.sqlite", absolutePath("state/uir.sqlite")), + Entry("bare relative db path", "state/uir.db", absolutePath("state/uir.db")), + ) + + It("opens the exact native filename with spaces and URL punctuation", func(ctx SpecContext) { + filename := filepath.Join(GinkgoT().TempDir(), "tenant #1.db") + target, err := dbtarget.Parse(filename) + Expect(err).NotTo(HaveOccurred()) + db, err := sql.Open("sqlite", target.DSN) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(db.Close) + _, err = db.ExecContext(ctx, "CREATE TABLE items (id integer)") + Expect(err).NotTo(HaveOccurred()) + info, err := os.Stat(filename) + Expect(err).NotTo(HaveOccurred()) + Expect(info.IsDir()).To(BeFalse()) + }) + + It("preserves SQLite query parameters and replaces managed pragmas", func() { + target, err := dbtarget.Parse("sqlite://state/uir.db?cache=shared&_pragma=foreign_keys(0)&_pragma=case_sensitive_like(1)") + Expect(err).ToNot(HaveOccurred()) + parsed, err := url.Parse(target.DSN) + Expect(err).ToNot(HaveOccurred()) + Expect(parsed.Query().Get("cache")).To(Equal("shared")) + Expect(parsed.Query()["_pragma"]).To(ConsistOf( + "case_sensitive_like(1)", "foreign_keys(1)", "busy_timeout(5000)", "journal_mode(WAL)", + )) + }) + + DescribeTable("rejects invalid or unsupported DSNs", + func(dsn, message string) { + _, err := dbtarget.Parse(dsn) + Expect(err).To(MatchError(ContainSubstring(message))) + }, + Entry("empty", " ", "DSN is required"), + Entry("unknown URL scheme", "mysql://localhost/app", `unsupported database scheme "mysql"`), + Entry("mssql alias scheme", "mssql://sa:secret@localhost?database=oipa", `use sqlserver://`), + Entry("explicit memory database", "sqlite://:memory:", "file-backed"), + Entry("file URI memory database", "sqlite://file::memory:?cache=shared", "file-backed"), + Entry("upper-case file URI scheme memory database", "sqlite://FILE::memory:", "file-backed"), + Entry("percent-encoded file URI memory database", "sqlite://file:%3Amemory%3A", "file-backed"), + Entry("partially percent-encoded file URI memory database", "sqlite://file::memory%3a?cache=shared", "file-backed"), + Entry("malformed file URI escape", "sqlite://file:state%zz.db", "decode SQLite file URI"), + Entry("bare memory database", ":memory:", "sqlite://"), + ) + + It("treats a file URI whose filename only case-folds to :memory: as a disk file", func() { + target, err := dbtarget.Parse("sqlite://file::MEMORY:") + Expect(err).ToNot(HaveOccurred()) + Expect(target.Dialect).To(Equal(dbtarget.SQLite)) + parsed, err := url.Parse(target.DSN) + Expect(err).ToNot(HaveOccurred()) + Expect(parsed.Opaque).To(Equal(":MEMORY:")) + }) +}) + +func TestDatabaseTarget(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Database Target Suite") +} + +func absolutePath(path string) string { + absolute, err := filepath.Abs(path) + if err != nil { + panic(err) + } + return filepath.Clean(strings.TrimSpace(absolute)) +} diff --git a/database/testdata/postgres.md b/database/testdata/postgres.md new file mode 100644 index 00000000..584ea624 --- /dev/null +++ b/database/testdata/postgres.md @@ -0,0 +1,10 @@ +--- +cwd: ../.. +timeout: 10m +--- + +Provide SQLGLOT_TEST_POSTGRES_URL for a disposable PostgreSQL admin connection. The suite creates and removes isolated databases. + +| Name | Command | CEL Validation | +| --- | --- | --- | +| Reviewed PostgreSQL schema execution | env GOWORK=off go test -tags integration ./database/migrate/review -count=1 | exitCode == 0 | diff --git a/database/testdata/standalone.md b/database/testdata/standalone.md new file mode 100644 index 00000000..0775457e --- /dev/null +++ b/database/testdata/standalone.md @@ -0,0 +1,12 @@ +--- +cwd: ../.. +timeout: 10m +--- + +| Name | Command | CEL Validation | +| --- | --- | --- | +| Standalone database and CLI behavior | env GOWORK=off go test ./database/... ./cmd/sqlglot -count=1 | exitCode == 0 | +| Pure planner regressions | env GOWORK=off go test ./migrate/... -count=1 | exitCode == 0 | +| No commons-db runtime dependency | env GOWORK=off go list -deps ./database/... | exitCode == 0 && !stdout.contains("github.com/flanksource/commons-db") | +| CLI build | env GOWORK=off make build | exitCode == 0 | +| Disposable SQLite example | env GOWORK=off go run ./examples/database | exitCode == 0 && stdout.contains("applied") && stdout.contains("already-applied") | diff --git a/examples/database/main.go b/examples/database/main.go new file mode 100644 index 00000000..4b6e4df0 --- /dev/null +++ b/examples/database/main.go @@ -0,0 +1,57 @@ +package main + +import ( + "context" + "database/sql" + "errors" + "fmt" + "os" + "path/filepath" + + "github.com/flanksource/sqlglot-go/database/migrate/review" + _ "modernc.org/sqlite" +) + +func main() { + if err := run(); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func run() (returnErr error) { + if err := os.MkdirAll(".tmp", 0750); err != nil { + return err + } + dir, err := os.MkdirTemp(".tmp", "database-example-") + if err != nil { + return err + } + defer func() { returnErr = errors.Join(returnErr, os.RemoveAll(dir)) }() + db, err := sql.Open("sqlite", filepath.Join(dir, "example.db")) + if err != nil { + return err + } + defer func() { returnErr = errors.Join(returnErr, db.Close()) }() + ctx := context.Background() + artifact, err := review.Build(ctx, db, review.Options{Dialect: "sqlite", Schema: "main", DesiredHCL: `schema "main" {} +table "items" { + schema = schema.main + column "id" { type = integer } +}`}) + if err != nil { + return err + } + fmt.Println("reviewed hash:", artifact.Hash) + for _, step := range artifact.Steps { + fmt.Println(step.SQL) + } + for range 2 { + result, err := review.Apply(ctx, db, review.Request{Artifact: artifact, Hash: artifact.Hash}) + if err != nil { + return err + } + fmt.Println(result.Outcome) + } + return nil +} diff --git a/examples/database/schema.hcl b/examples/database/schema.hcl new file mode 100644 index 00000000..c538890e --- /dev/null +++ b/examples/database/schema.hcl @@ -0,0 +1,15 @@ +schema "public" {} + +table "items" { + schema = schema.public + column "id" { + type = integer + } + column "label" { + type = text + null = true + } + primary_key { + columns = [column.id] + } +} diff --git a/go.mod b/go.mod index e0038d36..6d3cd18a 100644 --- a/go.mod +++ b/go.mod @@ -4,18 +4,35 @@ go 1.26.1 require ( ariga.io/atlas v0.38.0 + github.com/ClickHouse/clickhouse-go/v2 v2.46.0 + github.com/DATA-DOG/go-sqlmock v1.5.2 github.com/flanksource/clicky v1.21.80 + github.com/flanksource/commons v1.59.0 + github.com/go-sql-driver/mysql v1.10.0 + github.com/hashicorp/hcl/v2 v2.13.0 + github.com/lib/pq v1.12.3 github.com/mattn/go-runewidth v0.0.21 + github.com/microsoft/go-mssqldb v1.10.0 github.com/onsi/ginkgo/v2 v2.32.1 github.com/onsi/gomega v1.42.1 + github.com/pgplex/pgparser v0.2.0 github.com/spf13/cobra v1.10.2 + github.com/stretchr/testify v1.11.1 + github.com/zclconf/go-cty v1.14.4 + modernc.org/sqlite v1.51.0 ) require ( + filippo.io/edwards25519 v1.2.0 // indirect + github.com/ClickHouse/ch-go v0.71.0 // indirect github.com/Masterminds/semver/v3 v3.5.0 // indirect + github.com/agext/levenshtein v1.2.1 // indirect github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b // indirect github.com/alecthomas/chroma/v2 v2.23.1 // indirect github.com/alpkeskin/gotoon v0.1.1 // indirect + github.com/andybalholm/brotli v1.2.0 // indirect + github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect + github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect github.com/atotto/clipboard v0.1.4 // indirect github.com/aws/aws-sdk-go-v2 v1.42.0 // indirect github.com/aws/aws-sdk-go-v2/config v1.32.25 // indirect @@ -33,6 +50,7 @@ require ( github.com/aws/smithy-go v1.27.1 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/bmatcuk/doublestar v1.3.4 // indirect github.com/catppuccin/go v0.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/charmbracelet/bubbles v0.21.1-0.20250623103423-23b8fd6302d7 // indirect @@ -46,6 +64,7 @@ require ( github.com/charmbracelet/x/term v0.2.2 // indirect github.com/clipperhouse/displaywidth v0.11.0 // indirect github.com/clipperhouse/uax29/v2 v2.7.0 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/deckarep/golang-set/v2 v2.8.0 // indirect github.com/dlclark/regexp2 v1.11.5 // indirect github.com/dustin/go-humanize v1.0.1 // indirect @@ -53,20 +72,25 @@ require ( github.com/emirpasic/gods/v2 v2.0.0-alpha // indirect github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect github.com/fatih/color v1.18.0 // indirect - github.com/flanksource/commons v1.59.0 // indirect github.com/fsnotify/fsnotify v1.10.1 // indirect + github.com/go-faster/city v1.0.1 // indirect + github.com/go-faster/errors v0.7.1 // indirect github.com/go-jose/go-jose/v3 v3.0.5 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.3.0 // indirect + github.com/go-openapi/inflect v0.19.0 // indirect github.com/go-stack/stack v1.8.1 // indirect github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/go-xmlfmt/xmlfmt v1.1.3 // indirect github.com/goccy/go-yaml v1.19.2 // indirect + github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 // indirect + github.com/golang-sql/sqlexp v0.1.0 // indirect github.com/google/go-cmp v0.7.0 // indirect github.com/google/pprof v0.0.0-20260402051712-545e8a4df936 // indirect github.com/google/uuid v1.6.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/klauspost/compress v1.18.3 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect github.com/lmittmann/tint v1.1.3 // indirect @@ -76,13 +100,17 @@ require ( github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.22 // indirect github.com/mattn/go-localereader v0.0.1 // indirect + github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7 // indirect github.com/mitchellh/hashstructure/v2 v2.0.2 // indirect github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect github.com/muesli/cancelreader v0.2.2 // indirect github.com/muesli/termenv v0.16.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/ncruces/go-strftime v1.0.0 // indirect github.com/oklog/ulid/v2 v2.1.1 // indirect github.com/patrickmn/go-cache v2.1.0+incompatible // indirect + github.com/paulmach/orb v0.12.0 // indirect + github.com/pierrec/lz4/v4 v4.1.25 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/playwright-community/playwright-go v0.5700.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect @@ -91,6 +119,7 @@ require ( github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.68.1 // indirect github.com/prometheus/procfs v0.20.1 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/richardlehane/mscfb v1.0.7 // indirect github.com/richardlehane/msoleps v1.0.6 // indirect github.com/rivo/uniseg v0.4.7 // indirect @@ -98,9 +127,11 @@ require ( github.com/rogpeppe/go-internal v1.14.1 // indirect github.com/samber/lo v1.53.0 // indirect github.com/samber/oops v1.22.0 // indirect + github.com/segmentio/asm v1.2.1 // indirect github.com/shirou/gopsutil/v3 v3.24.5 // indirect github.com/shirou/gopsutil/v4 v4.26.7 // indirect github.com/shoenig/go-m1cpu v0.1.7 // indirect + github.com/shopspring/decimal v1.4.0 // indirect github.com/spf13/pflag v1.0.10 // indirect github.com/tiendc/go-deepcopy v1.7.2 // indirect github.com/timberio/go-datemath v0.1.0 // indirect @@ -116,6 +147,7 @@ require ( github.com/xuri/nfp v0.0.2-0.20250530014748-2ddeb826f9a9 // indirect github.com/yuin/goldmark v1.7.17 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect + github.com/zclconf/go-cty-yaml v1.1.0 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/otel v1.44.0 // indirect go.opentelemetry.io/otel/metric v1.44.0 // indirect @@ -133,5 +165,8 @@ require ( golang.org/x/tools v0.47.0 // indirect google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect gopkg.in/yaml.v3 v3.0.1 // indirect + modernc.org/libc v1.72.3 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.11.0 // indirect sigs.k8s.io/yaml v1.6.0 // indirect ) diff --git a/go.sum b/go.sum index 6a6ec7c6..f92ce637 100644 --- a/go.sum +++ b/go.sum @@ -2,13 +2,35 @@ ariga.io/atlas v0.38.0 h1:MwbtwVtDWJFq+ECyeTAz2ArvewDnpeiw/t/sgNdDsdo= ariga.io/atlas v0.38.0/go.mod h1:D7XMK6ei3GvfDqvzk+2VId78j77LdqHrqPOWamn51/s= cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4= cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= +filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo= +filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.1 h1:jHb/wfvRikGdxMXYV3QG/SzUOPYN9KEUUuC0Yd0/vC0= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.1/go.mod h1:pzBXCYn05zvYIrwLgtK8Ap8QcjRg+0i76tMQdWN6wOk= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 h1:Hk5QBxZQC1jb2Fwj6mpzme37xbCDdNTxU7O9eb5+LB4= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1/go.mod h1:IYus9qsFobWIc2YVwe/WPjcnyCkPKtnHAqUYeebc8z0= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 h1:fhqpLE3UEXi9lPaBRpQ6XuRW0nU7hgg4zlmZZa+a9q4= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0/go.mod h1:7dCRMLwisfRH3dBupKeNCioWYUZ4SS09Z14H+7i8ZoY= +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.4.0 h1:E4MgwLBGeVB5f2MdcIVD3ELVAWpr+WD6MUe1i+tM/PA= +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.4.0/go.mod h1:Y2b/1clN4zsAoUd/pgNAQHjLDnTis/6ROkUfyob6psM= +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.2.0 h1:nCYfgcSyHZXJI8J0IWE5MsCGlb2xp9fJiXyxWgmOFg4= +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.2.0/go.mod h1:ucUjca2JtSZboY8IoUqyQyuuXvwbMBVwFOm0vdQPNhA= +github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 h1:XRzhVemXdgvJqCH0sFfrBUTnUJSBrBf7++ypk+twtRs= +github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/ClickHouse/ch-go v0.71.0 h1:bUdZ/EZj/LcVHsMqaRUP2holqygrPWQKeMjc6nZoyRM= +github.com/ClickHouse/ch-go v0.71.0/go.mod h1:NwbNc+7jaqfY58dmdDUbG4Jl22vThgx1cYjBw0vtgXw= +github.com/ClickHouse/clickhouse-go/v2 v2.46.0 h1:s3eRy+hYmu5uzotB6ZhDofgHu8kDgGN/fpmjxRkqSpk= +github.com/ClickHouse/clickhouse-go/v2 v2.46.0/go.mod h1:giJfUVlMkcfUEPVfRpt51zZaGEx9i17gCos8gBl392c= +github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU= +github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU= github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= github.com/Masterminds/semver/v3 v3.5.0 h1:kQceYJfbupGfZOKZQg0kou0DgAKhzDg2NZPAwZ/2OOE= github.com/Masterminds/semver/v3 v3.5.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/agext/levenshtein v1.2.1 h1:QmvMAjj2aEICytGiWzmxoE0x2KZvE0fvmqMOfy2tjT8= +github.com/agext/levenshtein v1.2.1/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558= github.com/ajstarks/deck v0.0.0-20200831202436-30c9fc6549a9/go.mod h1:JynElWSGnm/4RlzPXRlREEwqTHAN3T56Bv2ITsFT3gY= github.com/ajstarks/deck/generate v0.0.0-20210309230005-c3f852c02e19/go.mod h1:T13YZdzov6OU0A1+RfKZiZN9ca6VeKdBdyDV+BY97Tk= github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b h1:slYM766cy2nI3BwyRiyQj/Ud48djTMtMebDqepE95rw= @@ -21,12 +43,18 @@ github.com/alecthomas/repr v0.5.2 h1:SU73FTI9D1P5UNtvseffFSGmdNci/O6RsqzeXJtP0Qs github.com/alecthomas/repr v0.5.2/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= github.com/alpkeskin/gotoon v0.1.1 h1:GQOVwMfWKINnfEA6slrXHJaJYDwnUFmrPlXOtnuja1w= github.com/alpkeskin/gotoon v0.1.1/go.mod h1:XRTz8RM4tz8M2nB37MNRN8rHF4YgeYd8nIXmoU0B0+M= +github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ= +github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= github.com/antchfx/xmlquery v1.5.1 h1:T9I4Ns1EXiWHy0IqKupGhnfTQtJwlGrpXtauYOoNv78= github.com/antchfx/xmlquery v1.5.1/go.mod h1:bVqnl7TaDXSReKINrhZz+2E/PbCu2tUahb+wZ7WZNT8= github.com/antchfx/xpath v1.3.6 h1:s0y+ElRRtTQdfHP609qFu0+c6bglDv20pqOViQjjdPI= github.com/antchfx/xpath v1.3.6/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw= +github.com/apparentlymart/go-textseg/v13 v13.0.0 h1:Y+KvPE1NYz0xl601PVImeQfFyEy6iT90AvPUL1NNfNw= +github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo= +github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew1u1fNQOlOtuGxQY= +github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmmsvpAG721bKi0joRfFdHIWJ4= github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4= github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI= github.com/aws/aws-sdk-go-v2 v1.42.0 h1:XvXMJTkFQtpBKIWZnmr9ZEOc2InWM2yldjXEJ/bymhA= @@ -63,6 +91,8 @@ github.com/aymanbagabas/go-udiff v0.4.1 h1:OEIrQ8maEeDBXQDoGCbbTTXYJMYRCRO1fnodZ github.com/aymanbagabas/go-udiff v0.4.1/go.mod h1:0L9PGwj20lrtmEMeyw4WKJ/TMyDtvAoK9bf2u/mNo3w= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bmatcuk/doublestar v1.3.4 h1:gPypJ5xD31uhX6Tf54sDPUOBXTqKH4c9aPY66CyQrS0= +github.com/bmatcuk/doublestar v1.3.4/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9MEoZQC/PmE= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= github.com/catppuccin/go v0.3.0 h1:d+0/YicIq+hSTo5oPuRi5kOpqkVA5tAsU6dNhvRu+aY= @@ -108,6 +138,7 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3 github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s= github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/deckarep/golang-set/v2 v2.8.0 h1:swm0rlPCmdWn9mESxKOjWk8hXSqoxOp+ZlfuyaAdFlQ= @@ -148,6 +179,10 @@ github.com/gkampitakis/go-diff v1.3.2 h1:Qyn0J9XJSDTgnsgHRdz9Zp24RaJeKMUHg2+PDZZ github.com/gkampitakis/go-diff v1.3.2/go.mod h1:LLgOrpqleQe26cte8s36HTWcTmMEur6OPYerdAAS9tk= github.com/gkampitakis/go-snaps v0.5.15 h1:amyJrvM1D33cPHwVrjo9jQxX8g/7E2wYdZ+01KS3zGE= github.com/gkampitakis/go-snaps v0.5.15/go.mod h1:HNpx/9GoKisdhw9AFOBT1N7DBs9DiHo/hGheFGBZ+mc= +github.com/go-faster/city v1.0.1 h1:4WAxSZ3V2Ws4QRDrscLEDcibJY8uf41H6AhXDrNDcGw= +github.com/go-faster/city v1.0.1/go.mod h1:jKcUJId49qdW3L1qKHH/3wPeUstCVpVSXTM6vO3VcTw= +github.com/go-faster/errors v0.7.1 h1:MkJTnDoEdi9pDabt1dpWf7AA8/BaSYZqibYyhZ20AYg= +github.com/go-faster/errors v0.7.1/go.mod h1:5ySTjWFiphBs07IKuiL69nxdfd5+fzh1u7FPGZP2quo= github.com/go-jose/go-jose/v3 v3.0.5 h1:BLLJWbC4nMZOfuPVxoZIxeYsn6Nl2r1fITaJ78UQlVQ= github.com/go-jose/go-jose/v3 v3.0.5/go.mod h1:5b+7YgP7ZICgJDBdfjZaIt+H/9L9T/YQrVfLAMboGkQ= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -158,20 +193,37 @@ github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= +github.com/go-openapi/inflect v0.19.0 h1:9jCH9scKIbHeV9m12SmPilScz6krDxKRasNNSNPXu/4= +github.com/go-openapi/inflect v0.19.0/go.mod h1:lHpZVlpIQqLyKwJ4N+YSc9hchQy/i12fJykb83CRBH4= +github.com/go-sql-driver/mysql v1.10.0 h1:Q+1LV8DkHJvSYAdR83XzuhDaTykuDx0l6fkXxoWCWfw= +github.com/go-sql-driver/mysql v1.10.0/go.mod h1:M+cqaI7+xxXGG9swrdeUIoPG3Y3KCkF0pZej+SK+nWk= github.com/go-stack/stack v1.8.1 h1:ntEHSVwIt7PNXNpgPmVfMrNhLtgjlmnZha2kOpuRiDw= github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP3XYfe4= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/go-test/deep v1.0.3 h1:ZrJSEWsXzPOxaZnFteGEfooLba+ju3FYIbOrS+rQd68= +github.com/go-test/deep v1.0.3/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= github.com/go-xmlfmt/xmlfmt v1.1.3 h1:t8Ey3Uy7jDSEisW2K3somuMKIpzktkWptA0iFCnRUWY= github.com/go-xmlfmt/xmlfmt v1.1.3/go.mod h1:aUCEOzzezBEjDBbFBoSiya/gduyIiWYRP6CnSFIV8AM= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM= github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= +github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 h1:au07oEsX2xN0ktxqI+Sida1w446QrXBRJ0nee3SNZlA= +github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= +github.com/golang-sql/sqlexp v0.1.0 h1:ZCD6MBpcuOVfGVqsEmY5/4FtYiKz6tSyUv9LPEDei6A= +github.com/golang-sql/sqlexp v0.1.0/go.mod h1:J4ad9Vo8ZCWQ2GMrC4UCQy1JpCbwU9m3EOqtpKwwwHI= github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ= github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/cel-go v0.31.0 h1:H0bhpFTqOvmHrBGrWKp7ZlhBm5Hh8PYUEXnwxT1LL7A= github.com/google/cel-go v0.31.0/go.mod h1:X0bD6iVNR8pkROSOoHVdgTkzmRcosof7WQqCD6wcMc8= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= @@ -187,6 +239,10 @@ github.com/hairyhenderson/toml v0.4.2-0.20210923231440-40456b8e66cf h1:I1sbT4ZbI github.com/hairyhenderson/toml v0.4.2-0.20210923231440-40456b8e66cf/go.mod h1:jDHmWDKZY6MIIYltYYfW4Rs7hQ50oS4qf/6spSiZAxY= github.com/hairyhenderson/yaml v0.0.0-20220618171115-2d35fca545ce h1:cVkYhlWAxwuS2/Yp6qPtcl0fGpcWxuZNonywHZ6/I+s= github.com/hairyhenderson/yaml v0.0.0-20220618171115-2d35fca545ce/go.mod h1:7TyiGlHI+IO+iJbqRZ82QbFtvgj/AIcFm5qc9DLn7Kc= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/hashicorp/hcl/v2 v2.13.0 h1:0Apadu1w6M11dyGFxWnmhhcMjkbAiKCv7G1r/2QgCNc= +github.com/hashicorp/hcl/v2 v2.13.0/go.mod h1:e4z5nxYlWNPdDSNYX+ph14EvWYMFm3eP0zIUqPc2jr0= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= @@ -203,11 +259,23 @@ github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE github.com/joshdk/go-junit v1.0.0/go.mod h1:TiiV0PqkaNfFXjEiyjWM3XXrhVyCa1K4Zfga6W52ung= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kisielk/sqlstruct v0.0.0-20201105191214-5f3e10d3ab46/go.mod h1:yyMNCyc/Ib3bDTKd379tNMpB/7/H5TjM2Y9QJ5THLbE= +github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/compress v1.18.3 h1:9PJRvfbmTabkOX8moIpXPbMMbYN60bWImDDU7L+/6zw= +github.com/klauspost/compress v1.18.3/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/lib/pq v1.12.3 h1:tTWxr2YLKwIvK90ZXEw8GP7UFHtcbTtty8zsI+YjrfQ= +github.com/lib/pq v1.12.3/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA= github.com/lmittmann/tint v1.1.3 h1:Hv4EaHWXQr+GTFnOU4VKf8UvAtZgn0VuKT+G0wFlO3I= github.com/lmittmann/tint v1.1.3/go.mod h1:HIS3gSy7qNwGCj+5oRjAutErFBl4BzdQP6cJZ0NfMwE= github.com/lrita/cmap v0.0.0-20231108122212-cb084a67f554 h1:a0+bIffIh/HdvvgtPQLRhOef1VDSxZ+8bQiyjQlJzqc= @@ -228,6 +296,10 @@ github.com/mattn/go-runewidth v0.0.21 h1:jJKAZiQH+2mIinzCJIaIG9Be1+0NR+5sz/lYEEj github.com/mattn/go-runewidth v0.0.21/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= github.com/mfridman/tparse v0.18.0 h1:wh6dzOKaIwkUGyKgOntDW4liXSo37qg5AXbIhkMV3vE= github.com/mfridman/tparse v0.18.0/go.mod h1:gEvqZTuCgEhPbYk/2lS3Kcxg1GmTxxU7kTC8DvP0i/A= +github.com/microsoft/go-mssqldb v1.10.0 h1:pHEt+Qz6YFPWqREq10mqSE524QQo+/QremwTCQht7TY= +github.com/microsoft/go-mssqldb v1.10.0/go.mod h1:mnG7lGa9iYJbzJqGCXyuQCegStKMr3kogDLD6+bmggg= +github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7 h1:DpOJ2HYzCv8LZP15IdmG+YdwD2luVPHITV96TkirNBM= +github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= github.com/mitchellh/hashstructure/v2 v2.0.2 h1:vGKWl0YJqUNxE8d+h8f6NJLcCJrgbhC4NcD46KavDd4= github.com/mitchellh/hashstructure/v2 v2.0.2/go.mod h1:MG3aRVU/N29oo/V/IhBX8GR/zz4kQkprJgF2EVszyDE= github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= @@ -236,6 +308,7 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8= github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 h1:ZK8zHtRHOkbHy6Mmr5D264iyp3TiX5OmNcI5cIARiQI= github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6/go.mod h1:CJlz5H+gyd6CUWT45Oy4q24RdLyn7Md9Vj2/ldJBSIo= github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELUXHmA= @@ -244,6 +317,8 @@ github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/ohler55/ojg v1.28.1 h1:Xy93DelhLSZNeWv8GPKtP6qMqkUlZlAxBP/AQcC5RfY= github.com/ohler55/ojg v1.28.1/go.mod h1:/Y5dGWkekv9ocnUixuETqiL58f+5pAsUfg5P8e7Pa2o= github.com/oklog/ulid/v2 v2.1.1 h1:suPZ4ARWLOJLegGFiZZ1dFAkqzhMjL3J1TzI+5wHz8s= @@ -256,7 +331,16 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= +github.com/paulmach/orb v0.12.0 h1:z+zOwjmG3MyEEqzv92UN49Lg1JFYx0L9GpGKNVDKk1s= +github.com/paulmach/orb v0.12.0/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU= +github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY= github.com/pborman/getopt v0.0.0-20170112200414-7148bc3a4c30/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o= +github.com/pgplex/pgparser v0.2.0 h1:BB7I/Mrwetiw07PTq/AlXcKfajlCwoJfwB7KZ7V9irI= +github.com/pgplex/pgparser v0.2.0/go.mod h1:DWhK7oZJn/16KTFeOQ+mk78+sRnEYya1XPghIoP9NFU= +github.com/pierrec/lz4/v4 v4.1.25 h1:kocOqRffaIbU5djlIBr7Wh+cx82C0vtFb0fOurZHqD0= +github.com/pierrec/lz4/v4 v4.1.25/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -275,6 +359,8 @@ github.com/prometheus/common v0.68.1 h1:omjRRl4QP4komogpXuhfeOiisQg7xdy8VM1UY+pS github.com/prometheus/common v0.68.1/go.mod h1:ZzL3f6u94qUxh9p+tJTrF+FvBS1XXbbRAZCQkytAL0Y= github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEycfc= github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/richardlehane/mscfb v1.0.7 h1:oeoiM0WE79vHwE8RpIYYvIAc8ajTH2mb6UZm55/+EB0= github.com/richardlehane/mscfb v1.0.7/go.mod h1:pe0+IUIc0AHh0+teNzBlJCtSyZdFOGgV4ZK9bsoV+Jo= github.com/richardlehane/msoleps v1.0.6 h1:9BvkpjvD+iUBalUY4esMwv6uBkfOip/Lzvd93jvR9gg= @@ -293,6 +379,10 @@ github.com/samber/lo v1.53.0 h1:t975lj2py4kJPQ6haz1QMgtId2gtmfktACxIXArw3HM= github.com/samber/lo v1.53.0/go.mod h1:4+MXEGsJzbKGaUEQFKBq2xtfuznW9oz/WrgyzMzRoM0= github.com/samber/oops v1.22.0 h1:fmWRC3YRUXFzpZ9Vs4nuckrsnCHv1cFV/bEm00Wvm0Y= github.com/samber/oops v1.22.0/go.mod h1:8ZDRxwQdphVhmLtEX9I6134LHJe5yeCV8cTfHz3m91Y= +github.com/segmentio/asm v1.2.1 h1:DTNbBqs57ioxAD4PrArqftgypG4/qNpXoJx8TVXxPR0= +github.com/segmentio/asm v1.2.1/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= +github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shirou/gopsutil/v3 v3.24.5 h1:i0t8kL+kQTvpAYToeuiVk3TgDeKOFioZO3Ztz/iZ9pI= github.com/shirou/gopsutil/v3 v3.24.5/go.mod h1:bsoOS1aStSs9ErQ1WWfxllSeS1K5D+U30r2NfcubMVk= github.com/shirou/gopsutil/v4 v4.26.7 h1:IXzpHz/dkMRYAhKkOXr1HB6SuzWU3eoyyeWe7g3bNZc= @@ -301,6 +391,8 @@ github.com/shoenig/go-m1cpu v0.1.7 h1:C76Yd0ObKR82W4vhfjZiCp0HxcSZ8Nqd84v+HZ0qyI github.com/shoenig/go-m1cpu v0.1.7/go.mod h1:KkDOw6m3ZJQAPHbrzkZki4hnx+pDRR1Lo+ldA56wD5w= github.com/shoenig/test v1.7.0 h1:eWcHtTXa6QLnBvm0jgEabMRN/uJ4DMV3M8xUGgRkZmk= github.com/shoenig/test v1.7.0/go.mod h1:UxJ6u/x2v/TNs/LoLxBNJRV9DiwBBKYxXSyczsBHFoI= +github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= +github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= @@ -310,6 +402,7 @@ github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= @@ -317,6 +410,7 @@ github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.2.0 h1:0pt8FlkOwjN2fPt4bIl4BoNxb98gGHN2ObFEDkrfZnM= github.com/tidwall/match v1.2.0/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= @@ -344,6 +438,9 @@ github.com/vadimi/go-ntlm v1.2.1 h1:y2xZf/a5+BJlYNJIIulP1q8F438H9bU7aGcYE53vghQ= github.com/vadimi/go-ntlm v1.2.1/go.mod h1:hPTY60eLSKGj9oUJAB+kZiLs2Cg5eKdH60aLczM9rMg= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g= +github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= github.com/xuri/efp v0.0.1 h1:fws5Rv3myXyYni8uwj2qKjVaRP30PdjeYe2Y6FDsCL8= @@ -352,6 +449,10 @@ github.com/xuri/excelize/v2 v2.11.0 h1:HxaEFl6sRN2+8J5a8HaKq+0M4FsjBGMnWWtjOCPSG github.com/xuri/excelize/v2 v2.11.0/go.mod h1:jxFLbzaIwGQ5ufFNvYfUOHqXhfPaNmP14KWfmNz2Uak= github.com/xuri/nfp v0.0.2-0.20250530014748-2ddeb826f9a9 h1:+C0TIdyyYmzadGaL/HBLbf3WdLgC29pgyhTjAT/0nuE= github.com/xuri/nfp v0.0.2-0.20250530014748-2ddeb826f9a9/go.mod h1:WwHg+CVyzlv/TX9xqBFXEZAuxOPxn2k1GNHwG41IIUQ= +github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= +github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= +github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yuin/goldmark v1.7.17 h1:p36OVWwRb246iHxA/U4p8OPEpOTESm4n+g+8t0EE5uA= @@ -360,6 +461,11 @@ github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +github.com/zclconf/go-cty v1.14.4 h1:uXXczd9QDGsgu0i/QFR/hzI5NYCHLf6NQw/atrbnhq8= +github.com/zclconf/go-cty v1.14.4/go.mod h1:VvMs5i0vgZdhYawQNq5kePSpLAoz8u1xvZgrPIxfnZE= +github.com/zclconf/go-cty-yaml v1.1.0 h1:nP+jp0qPHv2IhUVqmQSzjvqAWcObN0KBkUl2rWBdig0= +github.com/zclconf/go-cty-yaml v1.1.0/go.mod h1:9YLUH4g7lOhVWqUbctnVlZ5KLpg7JAprQNgxSZ1Gyxs= +go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= @@ -382,6 +488,7 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio= @@ -389,6 +496,7 @@ golang.org/x/exp v0.0.0-20260527015227-08cc5374adb3 h1:VHEvKbpgPXcPXn40t9cDTGK3J golang.org/x/exp v0.0.0-20260527015227-08cc5374adb3/go.mod h1:d2fgXJLVs4dYDHUk5lwMIfzRzSrWCfGZb0ZqeLa/Vcw= golang.org/x/image v0.43.0 h1:FLxcP4ec2350nTfOC8ysKtqYSIFbk/QGjw1ZHNP4tsY= golang.org/x/image v0.43.0/go.mod h1:rrpelvGFt+kLPAjPM4HeWPgrl0FtafueU//e5N0qk/Q= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= @@ -396,8 +504,10 @@ golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ= golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= @@ -406,7 +516,9 @@ golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= @@ -418,6 +530,7 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -437,6 +550,7 @@ golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc= golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= @@ -445,6 +559,8 @@ golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= @@ -452,14 +568,18 @@ golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q= golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa h1:Kjn0N0tCrDgiAFW+lGO4JZ3ck44CehvJQMAwj9QF0G8= google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:q4lMZS6kskjT5HvCPrnnypcDPVJqT/f4nfxmkE7gryY= google.golang.org/genproto/googleapis/rpc v0.0.0-20260622175928-b703f567277d h1:mpAgMyM9vQHxycBlDq50y1VHpfSfVwzXvrQKtYbXuUY= google.golang.org/genproto/googleapis/rpc v0.0.0-20260622175928-b703f567277d/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af h1:+5/Sw3GsDNlEmu7TfklWKPdQ0Ykja5VEmq2i817+jbI= google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= @@ -489,6 +609,34 @@ k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 h1:AZYQSJemyQB5eRxqcPky+/7EdBj0x k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk= layeh.com/gopher-json v0.0.0-20201124131017-552bb3c4c3bf h1:rRz0YsF7VXj9fXRF6yQgFI7DzST+hsI3TeFSGupntu0= layeh.com/gopher-json v0.0.0-20201124131017-552bb3c4c3bf/go.mod h1:ivKkcY8Zxw5ba0jldhZCYYQfGdb2K6u9tbYK1AwMIBc= +modernc.org/cc/v4 v4.28.2 h1:3tQ0lf2ADtoby2EtSP+J7IE2SHwEJdP8ioR59wx7XpY= +modernc.org/cc/v4 v4.28.2/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= +modernc.org/ccgo/v4 v4.34.0 h1:yRLPFZieg532OT4rp4JFNIVcquwalMX26G95WQDqwCQ= +modernc.org/ccgo/v4 v4.34.0/go.mod h1:AS5WYMyBakQ+fhsHhtP8mWB82KTGPkNNJDGfGQCe0/A= +modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= +modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/gc/v3 v3.1.2 h1:ZtDCnhonXSZexk/AYsegNRV1lJGgaNZJuKjJSWKyEqo= +modernc.org/gc/v3 v3.1.2/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= +modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= +modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= +modernc.org/libc v1.72.3 h1:ZnDF4tXn4NBXFutMMQC4vtbTFSXhhKzR73fv0beZEAU= +modernc.org/libc v1.72.3/go.mod h1:dn0dZNnnn1clLyvRxLxYExxiKRZIRENOfqQ8XEeg4Qs= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= +modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg= +modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= +modernc.org/sqlite v1.51.0 h1:aH/MMSoayAIhozZ7uJbVTT9QO/VhzBf0J9tymmmuC/U= +modernc.org/sqlite v1.51.0/go.mod h1:tcNzv5p84E0skkmJn038y+hWJbLQXQqEnQfeh5r2JLM= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= sigs.k8s.io/gateway-api v1.5.1 h1:RqVRIlkhLhUO8wOHKTLnTJA6o/1un4po4/6M1nRzdd0= sigs.k8s.io/gateway-api v1.5.1/go.mod h1:GvCETiaMAlLym5CovLxGjS0NysqFk3+Yuq3/rh6QL2o= sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= diff --git a/hack/database-integration.sh b/hack/database-integration.sh new file mode 100644 index 00000000..d399bec5 --- /dev/null +++ b/hack/database-integration.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd "$(dirname "$0")/.." +container_name="sqlglot-database-tests-$$" +started=false +cleanup() { + if [ "$started" = true ] + then + docker rm --force --volumes "$container_name" >/dev/null + fi +} +trap cleanup EXIT + +GOWORK=off CGO_ENABLED=0 GOOS=linux GOARCH="${SQLGLOT_DOCKER_ARCH:-amd64}" make build-database-tests + +case "${1:-}" in + postgres) + docker run --detach --name "$container_name" --network none --env POSTGRES_HOST_AUTH_METHOD=trust postgres:17-alpine >/dev/null + started=true + docker cp .tmp/database-review.test "$container_name:/database-review.test" + ready=false + for attempt in {1..60} + do + if docker exec "$container_name" pg_isready --quiet + then + ready=true + break + fi + sleep 1 + done + if [ "$ready" != true ] + then + docker logs "$container_name" + exit 1 + fi + docker exec --env 'SQLGLOT_TEST_POSTGRES_URL=postgres://postgres@127.0.0.1/postgres?sslmode=disable' "$container_name" /database-review.test -test.v + ;; + sqlserver) + test_password="Db$(openssl rand -hex 24)!" + docker run --detach --name "$container_name" --network none --env ACCEPT_EULA=Y --env "MSSQL_SA_PASSWORD=$test_password" mcr.microsoft.com/mssql/server:2022-latest >/dev/null + started=true + docker cp .tmp/database-sqlserver.test "$container_name:/database-sqlserver.test" + docker cp .tmp/database-bundle.test "$container_name:/database-bundle.test" + ready=false + for attempt in {1..60} + do + if docker exec --env "SQLCMDPASSWORD=$test_password" "$container_name" /opt/mssql-tools18/bin/sqlcmd -S tcp:127.0.0.1,1433 -U sa -C -l 2 -t 2 -Q 'SELECT 1' >/dev/null 2>&1 + then + ready=true + break + fi + sleep 1 + done + if [ "$ready" != true ] + then + docker logs "$container_name" + exit 1 + fi + docker exec --env "COMMONS_DB_SQLSERVER_ARTIFACT_TEST_URL=sqlserver://sa:$test_password@127.0.0.1:1433?database=master&encrypt=disable" "$container_name" /database-sqlserver.test -test.v + docker exec --env "COMMONS_DB_SQLSERVER_URL=sqlserver://sa:$test_password@127.0.0.1:1433?encrypt=disable" "$container_name" /database-bundle.test -test.v + ;; + *) + echo 'usage: bash hack/database-integration.sh postgres|sqlserver' >&2 + exit 2 + ;; +esac diff --git a/harness/execute.py b/harness/execute.py index 56ed98bc..29a3e2a0 100755 --- a/harness/execute.py +++ b/harness/execute.py @@ -23,10 +23,8 @@ The verdicts: - agree same rows, same order. What the port is held to. - reordered same rows, different order. Not a failure: without an - ORDER BY neither query promised an order. - DIVERGED different rows. The two queries ask different questions. + agree same rows and duplicate counts; order matters for ORDER BY. + DIVERGED different rows, duplicates, or explicitly requested order. nondeterministic the statement disagrees with ITSELF across two runs -- CURRENT_TIME, RANDOM. Excluded rather than denylisted by keyword, so the harness discovers the class instead of @@ -54,13 +52,9 @@ class Engine: """One database, and how to run a statement in it without keeping it. - Isolation is per engine because the mechanism is: DuckDB gets a scratch - working directory (its side effects are FILES -- ATTACH, COPY), PostgreSQL - gets a transaction that is always rolled back (its side effects are - CATALOG -- CREATE, DROP). Without either, a corpus of a transpiler's tests - leaves debris and later statements fail on it: the first run of this - harness left a file.db in the repository root, and the PostgreSQL run - failed with 'relation "t" already exists'. + Both engines roll back every statement so DDL and DML cannot change the + fixtures used by later comparisons. DuckDB also gets a scratch working + directory for filesystem effects such as ATTACH and COPY. """ name = "" @@ -98,13 +92,14 @@ def load_fixtures(self) -> None: self._con.execute(FIXTURES.read_text()) def rows(self, sql: str): - cur = self._con.execute(sql) + self._con.execute("BEGIN TRANSACTION") try: - return cur.fetchall() - except Exception: - return None + return self._con.execute(sql).fetchall() + finally: + self._con.execute("ROLLBACK") def close(self) -> None: + self._con.close() os.chdir(self._home) self._scratch.cleanup() @@ -163,6 +158,10 @@ def multiset(result): return sorted(result, key=repr) if isinstance(result, list) else result +def same_rows(left, right, ordered): + return left == right if ordered else multiset(left) == multiset(right) + + def open_engines(dsn: str | None) -> tuple[list[Engine], list[str]]: """Whatever can be reached. Skipping is reported, never silent.""" engines: list[Engine] = [] @@ -212,34 +211,26 @@ def compare(who: str, src: str, first, out: str) -> None: diverged.append({"engine": engine.name, "sql": src, "who": who, "written": out, "why": "does not run: " + str(e).split("\n")[0][:70]}) return - # The REWRITTEN side has to be stable too. Checking only the input was - # not enough: `USING SAMPLE 10%` returned the same thing twice and a - # different thing on the sixth run, so an unstable rewrite was - # reported as a divergence. A comparison is only meaningful if BOTH - # sides reproduce. + # Both sides must reproduce. Unordered row permutations are stable; + # changed values, duplicate counts, and requested order are not. try: - if engine.rows(out) != got: + if not same_rows(engine.rows(out), got, ordered): bump(who + ": nondeterministic") return except Exception: bump(who + ": nondeterministic") return - if got != first and multiset(got) != multiset(first): - # About to accuse. Look once more at BOTH sides first: a statement - # unstable enough to slip past the repeat checks above would - # otherwise be reported as a divergence, and a gate that cries - # wolf on a rerun is a gate people learn to ignore. Confirming - # costs two executions on the rare path and nothing on the common - # one. + if not same_rows(got, first, ordered): + # Confirm both sides once more before reporting a divergence. try: - if engine.rows(src) != first or engine.rows(out) != got: + if not same_rows(engine.rows(src), first, ordered) or not same_rows(engine.rows(out), got, ordered): bump(who + ": nondeterministic") return except Exception: bump(who + ": nondeterministic") return - if got == first: + if same_rows(got, first, ordered): # Two queries that both return NOTHING agree about nothing. With # fixture tables in place this is the cheap way to look busy -- # any pair of mistakes over an empty result set matches -- so it @@ -248,8 +239,6 @@ def compare(who: str, src: str, first, out: str) -> None: bump(who + ": both returned no rows") else: bump(who + ": agree") - elif multiset(got) == multiset(first): - bump(who + ": same rows, different order") else: bump("DIVERGED") diverged.append({"engine": engine.name, "sql": src, "who": who, "written": out, @@ -259,6 +248,7 @@ def compare(who: str, src: str, first, out: str) -> None: if rec.get("dialect") != engine.dialect: continue src = rec["sql"] + ordered = rec["ordered"] try: first = engine.rows(src) except Exception: @@ -268,7 +258,7 @@ def compare(who: str, src: str, first, out: str) -> None: # Three looks rather than two: sampling and hashing can repeat once by # luck, and every extra look is cheap next to a false divergence. try: - if any(engine.rows(src) != first for _ in range(2)): + if any(not same_rows(engine.rows(src), first, ordered) for _ in range(2)): bump("nondeterministic") continue except Exception: diff --git a/harness/execute_test.go b/harness/execute_test.go index 0962916e..9cea5538 100644 --- a/harness/execute_test.go +++ b/harness/execute_test.go @@ -8,6 +8,16 @@ import ( "github.com/flanksource/sqlglot-go/sqlglot" ) +func executionOrdered(tree *sqlglot.Expression) bool { + for tree.Class == "Subquery" || tree.Class == "Paren" { + if tree.Args["order"] != nil { + return true + } + tree = tree.This() + } + return tree.Args["order"] != nil +} + // TestEmitExecutionPairs writes what the EXECUTION oracle needs: for every // statement the port can read, the SQL that was written and the SQL the port // writes back. @@ -52,12 +62,13 @@ func TestEmitExecutionPairs(t *testing.T) { if !engines[c.Dialect] { continue } - rec := map[string]string{"dialect": c.Dialect, "sql": c.SQL, "reference": c.Rendered} + rec := map[string]any{"dialect": c.Dialect, "sql": c.SQL, "reference": c.Rendered, "ordered": true} tree, perr := sqlglot.ParseOne(c.SQL, c.Dialect) switch { case perr != nil: rec["port_error"] = perr.Error() default: + rec["ordered"] = executionOrdered(tree) got, gerr := sqlglot.Generate(tree, c.Dialect) if gerr != nil { rec["port_error"] = gerr.Error() diff --git a/harness/execution_order_ginkgo_test.go b/harness/execution_order_ginkgo_test.go new file mode 100644 index 00000000..60654e66 --- /dev/null +++ b/harness/execution_order_ginkgo_test.go @@ -0,0 +1,28 @@ +package harness + +import ( + "testing" + + "github.com/flanksource/sqlglot-go/sqlglot" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestExecutionOrder(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Execution order metadata") +} + +var _ = DescribeTable("Execution order metadata", func(sql string, ordered bool) { + tree, err := sqlglot.ParseOne(sql, "duckdb") + Expect(err).NotTo(HaveOccurred()) + Expect(executionOrdered(tree)).To(Equal(ordered)) +}, + Entry("unordered select", "SELECT a FROM t", false), + Entry("top-level order", "SELECT a FROM t ORDER BY a", true), + Entry("window order", "SELECT ROW_NUMBER() OVER (ORDER BY a) FROM t", false), + Entry("inner query order", "SELECT * FROM (SELECT a FROM t ORDER BY a) AS sub", false), + Entry("quoted order text", "SELECT 'ORDER BY a' FROM t", false), + Entry("set operation order", "SELECT a FROM t UNION SELECT a FROM x ORDER BY a", true), + Entry("parenthesized ordered query", "(SELECT a FROM t ORDER BY a)", true), +) diff --git a/harness/requirements-execute.txt b/harness/requirements-execute.txt new file mode 100644 index 00000000..af8ad9d6 --- /dev/null +++ b/harness/requirements-execute.txt @@ -0,0 +1,4 @@ +# The execution coverage gate is measured against this engine version. +duckdb==1.4.4 +psycopg[binary]>=3.2,<4 +pytest>=8,<10 diff --git a/harness/test_execute.py b/harness/test_execute.py new file mode 100644 index 00000000..2344b6d2 --- /dev/null +++ b/harness/test_execute.py @@ -0,0 +1,40 @@ +from itertools import cycle + +import pytest + +from execute import Engine, run + + +class ScriptedEngine(Engine): + name = "scripted" + dialect = "duckdb" + + def __init__(self, source, rewritten): + self.results = {"source": cycle(source), "rewritten": cycle(rewritten)} + + def rows(self, sql): + return next(self.results[sql]) + + +@pytest.mark.parametrize( + "ordered,source,rewritten,verdict", + [ + (False, [[(1,), (2,)], [(2,), (1,)]], [[(2,), (1,)]], "reference: agree"), + (False, [[(1,), (1,), (2,)]], [[(1,), (2,), (2,)]], "DIVERGED"), + (True, [[(1,), (2,)]], [[(2,), (1,)]], "DIVERGED"), + (True, [[(1,), (2,)], [(2,), (1,)]], [[(1,), (2,)]], "nondeterministic"), + (False, [[(1,)], [(2,)]], [[(1,)]], "nondeterministic"), + (False, [[(1,)]], [[(1,)], [(2,)]], "reference: nondeterministic"), + (False, [[([1, 2],)], [([2, 1],)]], [[([1, 2],)]], "nondeterministic"), + ], +) +def test_execution_respects_ordering_and_preserves_value_differences( + ordered, source, rewritten, verdict +): + pairs = [{"dialect": "duckdb", "sql": "source", "reference": "rewritten", "ordered": ordered}] + tally, divergences = run(ScriptedEngine(source, rewritten), pairs, set()) + expected = {verdict: 1} + if verdict != "nondeterministic": + expected["port refused it"] = 1 + assert tally == expected + assert len(divergences) == (1 if verdict == "DIVERGED" else 0) diff --git a/harness/test_execute_integration.py b/harness/test_execute_integration.py new file mode 100644 index 00000000..fc0c233d --- /dev/null +++ b/harness/test_execute_integration.py @@ -0,0 +1,34 @@ +import pytest + +from execute import DuckDB + + +@pytest.fixture +def engine(): + database = DuckDB() + try: + database.load_fixtures() + yield database + finally: + database.close() + + +def test_duckdb_does_not_keep_catalog_changes_between_statements(engine): + engine.rows("CREATE TABLE oracle_scratch (id INT)") + assert engine.rows( + "SELECT table_name FROM information_schema.tables " + "WHERE table_name = 'oracle_scratch'" + ) == [] + + +def test_duckdb_preserves_fixture_rows_between_statements(engine): + before = engine.rows("SELECT * FROM t ORDER BY a, b") + assert before + engine.rows("DELETE FROM t") + assert engine.rows("SELECT * FROM t ORDER BY a, b") == before + + +def test_duckdb_rolls_back_failed_statements(engine): + with pytest.raises(Exception, match="does not exist"): + engine.rows("SELECT * FROM missing_oracle_table") + assert engine.rows("SELECT COUNT(*) FROM t") == [(5,)] diff --git a/migrate/ARTIFACTS.md b/migrate/ARTIFACTS.md new file mode 100644 index 00000000..5294f070 --- /dev/null +++ b/migrate/ARTIFACTS.md @@ -0,0 +1,18 @@ +# Reviewed migration artifacts + +`sqlserver.BuildArtifact(migrate.ArtifactRequest{Schemas, Request})` binds complete current and desired snapshots, their explicit schema scope, and the exact generated plan to a SHA-256 hash. The artifact owns its snapshot data; later edits to the input cannot alter it. Named collections are sorted for deterministic output while column and key order is retained. + +```go +artifact, err := sqlserver.BuildArtifact(migrate.ArtifactRequest{ + Schemas: []string{"dbo"}, + Request: request, +}) +``` + +`ValidateArtifact` checks the version, content hash and consistency with the planner. It rejects edited SQL even when somebody recomputes the content hash. A planner upgrade that changes emitted SQL requires a new preview. The hash provides integrity and review identity; authorization and trusted storage remain the caller's responsibility. + +`CheckArtifact(artifact, actual)` uses native SQL semantics to compare a complete live snapshot. It returns `ArtifactApplied` when the entire desired state already exists, or `ArtifactPending` when the baseline still matches. Any other state fails with drift details. This includes changes to unselected target members captured in the baseline. Catalog expression formatting is compared through sqlglot rather than raw text hashes. + +These guards run in Go and do not open a database. They do not make the plan's individual SQL strings a self-guarding deployment script. A database executor must inspect under the same transaction and application lock used to execute the exact reviewed steps, check the complete postcondition, and commit only after it matches. Commons-db's `migrate/sqlserver` supplies this execution path. + +Run `go test ./migrate/... -count=1`. UIR's `go run ./examples/sql-migration` demonstrates compiling a selected member into an artifact without a database. diff --git a/migrate/README.md b/migrate/README.md index fc8ee4dd..060a529e 100644 --- a/migrate/README.md +++ b/migrate/README.md @@ -1,6 +1,6 @@ # SQL Server migrations -`migrate/sqlserver.Plan` compares typed current and desired schemas and generates ordered SQL without opening a database. `migrate/atlas/sqlserver` exposes the same differ and planner to Atlas consumers. Inspection, credentials, application locks, transactions, execution and deployment records belong to the caller. +`migrate/sqlserver.Plan` compares typed current and desired schemas and generates ordered SQL without opening a database. `migrate/atlas/sqlserver` exposes the same differ and planner to Atlas consumers. Sqlglot also compiles and checks immutable migration artifacts. Inspection, credentials, application locks, transactions, execution and deployment records belong to the caller. ```go import ( diff --git a/migrate/artifact.go b/migrate/artifact.go new file mode 100644 index 00000000..c16f035b --- /dev/null +++ b/migrate/artifact.go @@ -0,0 +1,22 @@ +package migrate + +// Artifact binds reviewed SQL to complete baseline and desired snapshots. +type Artifact struct { + Version int `json:"version"` + Schemas []string `json:"schemas"` + Request Request `json:"request"` + Plan Plan `json:"plan"` + Hash string `json:"hash"` +} + +type ArtifactRequest struct { + Schemas []string + Request Request +} + +type ArtifactState string + +const ( + ArtifactPending ArtifactState = "pending" + ArtifactApplied ArtifactState = "applied" +) diff --git a/migrate/atlas/sqlserver/collation.go b/migrate/atlas/sqlserver/collation.go new file mode 100644 index 00000000..5cd1eea2 --- /dev/null +++ b/migrate/atlas/sqlserver/collation.go @@ -0,0 +1,24 @@ +package sqlserver + +import ( + "fmt" + "regexp" + + "ariga.io/atlas/sql/schema" + "github.com/flanksource/sqlglot-go/migrate/internal/sqlx" +) + +var collationLiteral = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9_]*$`) + +func writeCollation(b *sqlx.Builder, name string) error { + if !collationLiteral.MatchString(name) { + return fmt.Errorf("sqlserver: invalid collation literal %q", name) + } + b.P("COLLATE", name) + return nil +} + +func supportsCollation(t schema.Type) bool { + _, ok := t.(*schema.StringType) + return ok +} diff --git a/migrate/atlas/sqlserver/collation_test.go b/migrate/atlas/sqlserver/collation_test.go new file mode 100644 index 00000000..841d489b --- /dev/null +++ b/migrate/atlas/sqlserver/collation_test.go @@ -0,0 +1,37 @@ +package sqlserver_test + +import ( + "ariga.io/atlas/sql/schema" + planner "github.com/flanksource/sqlglot-go/migrate/atlas/sqlserver" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("SQL Server collation syntax", func() { + It("writes collation literals in CREATE and ALTER COLUMN without identifier quoting", func() { + column := schema.NewStringColumn("label", "nvarchar").SetType(&schema.StringType{T: "nvarchar", Size: 100}).SetCollation("SQL_Latin1_General_CP1_CI_AS") + table := schema.NewTable("orders").AddColumns(column) + schema.New("dbo").AddTables(table) + Expect(planCmds(&schema.AddTable{T: table})).To(Equal([]string{"CREATE TABLE [dbo].[orders] ([label] nvarchar(100) COLLATE SQL_Latin1_General_CP1_CI_AS NOT NULL)"})) + from := schema.NewStringColumn("label", "nvarchar").SetType(&schema.StringType{T: "nvarchar", Size: 20}).SetCollation("SQL_Latin1_General_CP1_CI_AS") + Expect(planCmds(modify(table, &schema.ModifyColumn{From: from, To: column, Change: schema.ChangeType}))).To(Equal([]string{"ALTER TABLE [dbo].[orders] ALTER COLUMN [label] nvarchar(100) COLLATE SQL_Latin1_General_CP1_CI_AS NOT NULL"})) + }) + + It("rejects text that cannot be emitted as a collation literal", func() { + for _, name := range []string{"", "Latin1; SELECT 1", "[Latin1_General_CI_AS]", "Latin1--comment", "Latin1 General"} { + column := schema.NewStringColumn("label", "nvarchar").SetType(&schema.StringType{T: "nvarchar", Size: 100}).SetCollation(name) + table := schema.NewTable("orders").AddColumns(column) + schema.New("dbo").AddTables(table) + _, err := planner.DefaultPlan.PlanChanges(GinkgoT().Context(), "invalid", []schema.Change{&schema.AddTable{T: table}}) + Expect(err).To(MatchError(ContainSubstring("invalid collation literal"))) + } + }) + + It("omits COLLATE when a text column becomes an integer", func() { + from := schema.NewStringColumn("label", "nvarchar").SetType(&schema.StringType{T: "nvarchar", Size: 20}).SetCollation("SQL_Latin1_General_CP1_CI_AS") + to := schema.NewIntColumn("label", "int") + table := schema.NewTable("orders").AddColumns(to) + schema.New("dbo").AddTables(table) + Expect(planCmds(modify(table, &schema.ModifyColumn{From: from, To: to, Change: schema.ChangeType | schema.ChangeAttr}))).To(Equal([]string{"ALTER TABLE [dbo].[orders] ALTER COLUMN [label] int NOT NULL"})) + }) +}) diff --git a/migrate/atlas/sqlserver/diff.go b/migrate/atlas/sqlserver/diff.go index b521ef2c..6f5cdf65 100644 --- a/migrate/atlas/sqlserver/diff.go +++ b/migrate/atlas/sqlserver/diff.go @@ -150,7 +150,7 @@ func defaultChanged(from, to *schema.Column) (bool, error) { } func canonicalDefault(c *schema.Column) (string, error) { - value, err := defaultSQL(c) + value, err := DefaultSQL(c) if err != nil { return "", err } diff --git a/migrate/atlas/sqlserver/expr.go b/migrate/atlas/sqlserver/expr.go index 8df30d25..4dd363c6 100644 --- a/migrate/atlas/sqlserver/expr.go +++ b/migrate/atlas/sqlserver/expr.go @@ -80,10 +80,10 @@ func withoutParens(e *sqlglot.Expression, root bool) *sqlglot.Expression { return e } -// defaultSQL is the SQL text of a column default. An HCL string literal +// DefaultSQL is the SQL text of a column default. An HCL string literal // arrives unquoted and is quoted for the column's type, N'...' for the // Unicode types; numbers and raw sql() expressions are written as they are. -func defaultSQL(c *schema.Column) (string, error) { +func DefaultSQL(c *schema.Column) (string, error) { switch x := c.Default.(type) { case *schema.RawExpr: return x.X, nil diff --git a/migrate/atlas/sqlserver/migrate_modify.go b/migrate/atlas/sqlserver/migrate_modify.go index 169003b8..6c56721e 100644 --- a/migrate/atlas/sqlserver/migrate_modify.go +++ b/migrate/atlas/sqlserver/migrate_modify.go @@ -9,10 +9,14 @@ import ( // modifyTable plans each change of a ModifyTable as its own statement. func (s *state) modifyTable(modify *schema.ModifyTable) error { t := modify.T - for _, change := range modify.Changes { - if err := s.tableChange(t, change); err != nil { + for _, step := range orderedTableChanges(modify.Changes) { + start := len(s.Changes) + if err := s.tableChange(t, step.change); err != nil { return fmt.Errorf("sqlserver: modify table %q: %w", t.Name, err) } + for _, emitted := range s.Changes[start:] { + emitted.Source = step.source + } } return nil } @@ -32,9 +36,6 @@ func (s *state) tableChange(t *schema.Table, change schema.Change) error { return s.addIndex(c, t, c.I) case *schema.DropIndex: s.dropIndex(c, t, c.I) - case *schema.ModifyIndex: - s.dropIndex(c, t, c.From) - return s.addIndex(c, t, c.To) default: return s.constraintChange(t, change) } @@ -47,23 +48,14 @@ func (s *state) constraintChange(t *schema.Table, change schema.Change) error { s.addPrimaryKey(c, t, c.P) case *schema.DropPrimaryKey: s.dropConstraint(c, t, c.P.Name) - case *schema.ModifyPrimaryKey: - s.dropConstraint(c, t, c.From.Name) - s.addPrimaryKey(c, t, c.To) case *schema.AddForeignKey: s.addForeignKey(c, t, c.F) case *schema.DropForeignKey: s.dropConstraint(c, t, c.F.Symbol) - case *schema.ModifyForeignKey: - s.dropConstraint(c, t, c.From.Symbol) - s.addForeignKey(c, t, c.To) case *schema.AddCheck: s.addCheck(c, t, c.C) case *schema.DropCheck: s.dropConstraint(c, t, c.C.Name) - case *schema.ModifyCheck: - s.dropConstraint(c, t, c.From.Name) - s.addCheck(c, t, c.To) default: return fmt.Errorf("change %T is %w", change, errUnsupported) } @@ -97,15 +89,17 @@ func (s *state) modifyColumn(c *schema.ModifyColumn, t *schema.Table) error { } b := s.alterTable(t).P("ALTER COLUMN").Ident(c.To.Name).P(typ) if collation, ok := attrOf[*schema.Collation](c.To.Attrs); ok { - b.P("COLLATE").Ident(collation.V) - } else if c.Change.Is(schema.ChangeAttr) { + if err := writeCollation(b, collation.V); err != nil { + return err + } + } else if c.Change.Is(schema.ChangeAttr) && supportsCollation(c.To.Type.Type) { b.P("COLLATE DATABASE_DEFAULT") } b.P(nullability(c.To)) s.append(c, fmt.Sprintf("alter column %q", c.To.Name), b) } if rebind && c.To.Default != nil { - value, err := defaultSQL(c.To) + value, err := DefaultSQL(c.To) if err != nil { return err } diff --git a/migrate/atlas/sqlserver/migrate_table.go b/migrate/atlas/sqlserver/migrate_table.go index 64a12283..92b4c992 100644 --- a/migrate/atlas/sqlserver/migrate_table.go +++ b/migrate/atlas/sqlserver/migrate_table.go @@ -54,7 +54,9 @@ func (s *state) column(b *sqlx.Builder, t *schema.Table, c *schema.Column) error } b.Ident(c.Name).P(typ) if collation, ok := attrOf[*schema.Collation](c.Attrs); ok { - b.P("COLLATE").Ident(collation.V) + if err := writeCollation(b, collation.V); err != nil { + return err + } } if identity, ok := attrOf[*Identity](c.Attrs); ok { b.P(fmt.Sprintf("IDENTITY(%d,%d)", identity.Seed, identity.Increment)) @@ -65,7 +67,7 @@ func (s *state) column(b *sqlx.Builder, t *schema.Table, c *schema.Column) error b.P("NOT NULL") } if c.Default != nil { - value, err := defaultSQL(c) + value, err := DefaultSQL(c) if err != nil { return err } diff --git a/migrate/atlas/sqlserver/migrate_test.go b/migrate/atlas/sqlserver/migrate_test.go index 85ceae1c..02747795 100644 --- a/migrate/atlas/sqlserver/migrate_test.go +++ b/migrate/atlas/sqlserver/migrate_test.go @@ -84,8 +84,8 @@ var _ = Describe("SQL Server plan", func() { added := schema.NewUniqueIndex("UX_acme_orders_status").AddColumns(status).AddAttrs(&planner.Clustered{}) dropped := schema.NewUniqueIndex("UQ_acme_orders_note").AddAttrs(&planner.UniqueConstraint{}) Expect(planCmds(modify(t, &schema.AddIndex{I: added}, &schema.DropIndex{I: dropped}))).To(Equal([]string{ - "CREATE UNIQUE CLUSTERED INDEX [UX_acme_orders_status] ON [dbo].[acme_orders] ([status])", "ALTER TABLE [dbo].[acme_orders] DROP CONSTRAINT [UQ_acme_orders_note]", + "CREATE UNIQUE CLUSTERED INDEX [UX_acme_orders_status] ON [dbo].[acme_orders] ([status])", })) }) diff --git a/migrate/atlas/sqlserver/table_order.go b/migrate/atlas/sqlserver/table_order.go new file mode 100644 index 00000000..b7432f16 --- /dev/null +++ b/migrate/atlas/sqlserver/table_order.go @@ -0,0 +1,45 @@ +package sqlserver + +import ( + "cmp" + "slices" + + "ariga.io/atlas/sql/schema" +) + +type tableStep struct { + source schema.Change + change schema.Change + phase int +} + +func orderedTableChanges(changes []schema.Change) []tableStep { + steps := make([]tableStep, 0, len(changes)) + for _, change := range changes { + var drop, add schema.Change + switch c := change.(type) { + case *schema.ModifyIndex: + drop, add = &schema.DropIndex{I: c.From}, &schema.AddIndex{I: c.To} + case *schema.ModifyPrimaryKey: + drop, add = &schema.DropPrimaryKey{P: c.From}, &schema.AddPrimaryKey{P: c.To} + case *schema.ModifyForeignKey: + drop, add = &schema.DropForeignKey{F: c.From}, &schema.AddForeignKey{F: c.To} + case *schema.ModifyCheck: + drop, add = &schema.DropCheck{C: c.From}, &schema.AddCheck{C: c.To} + } + if drop != nil { + steps = append(steps, tableStep{source: change, change: drop, phase: 0}, tableStep{source: change, change: add, phase: 2}) + continue + } + phase := 1 + switch change.(type) { + case *schema.DropIndex, *schema.DropPrimaryKey, *schema.DropForeignKey, *schema.DropCheck: + phase = 0 + case *schema.AddIndex, *schema.AddPrimaryKey, *schema.AddForeignKey, *schema.AddCheck: + phase = 2 + } + steps = append(steps, tableStep{source: change, change: change, phase: phase}) + } + slices.SortStableFunc(steps, func(a, b tableStep) int { return cmp.Compare(a.phase, b.phase) }) + return steps +} diff --git a/migrate/sqlserver/artifact.go b/migrate/sqlserver/artifact.go new file mode 100644 index 00000000..4ec85f08 --- /dev/null +++ b/migrate/sqlserver/artifact.go @@ -0,0 +1,121 @@ +package sqlserver + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "slices" + + "github.com/flanksource/sqlglot-go/migrate" +) + +func BuildArtifact(input migrate.ArtifactRequest) (migrate.Artifact, error) { + body, err := json.Marshal(migrate.Artifact{Version: 1, Schemas: input.Schemas, Request: input.Request}) + if err != nil { + return migrate.Artifact{}, err + } + var artifact migrate.Artifact + if err := json.Unmarshal(body, &artifact); err != nil { + return migrate.Artifact{}, err + } + if err := artifactScope(artifact); err != nil { + return migrate.Artifact{}, err + } + slices.Sort(artifact.Schemas) + orderSnapshot(&artifact.Request.Current) + orderSnapshot(&artifact.Request.Desired) + artifact.Plan, err = Plan(artifact.Request) + if err != nil { + return migrate.Artifact{}, err + } + artifact.Hash, err = artifactHash(artifact) + return artifact, err +} + +func ValidateArtifact(artifact migrate.Artifact) error { + if artifact.Version != 1 { + return fmt.Errorf("sqlserver: unsupported migration artifact version %d", artifact.Version) + } + hash, err := artifactHash(artifact) + if err != nil { + return err + } + if artifact.Hash != hash { + return fmt.Errorf("sqlserver: migration artifact hash mismatch") + } + expected, err := BuildArtifact(migrate.ArtifactRequest{Schemas: artifact.Schemas, Request: artifact.Request}) + if err != nil { + return err + } + if expected.Hash != artifact.Hash { + return fmt.Errorf("sqlserver: migration artifact SQL does not match its snapshots and planner version") + } + return nil +} + +func CheckArtifact(artifact migrate.Artifact, actual migrate.Schema) (migrate.ArtifactState, error) { + if err := ValidateArtifact(artifact); err != nil { + return "", err + } + if err := artifactScope(migrate.Artifact{Schemas: artifact.Schemas, Request: migrate.Request{Current: actual}}); err != nil { + return "", err + } + desired, err := Plan(migrate.Request{Current: actual, Desired: artifact.Request.Desired}) + if err != nil { + return "", fmt.Errorf("sqlserver: inspect artifact state: %w", err) + } + if len(desired.Changes) == 0 && len(desired.Steps) == 0 { + return migrate.ArtifactApplied, nil + } + baseline, err := Plan(migrate.Request{Current: actual, Desired: artifact.Request.Current}) + if err != nil { + return "", fmt.Errorf("sqlserver: baseline drift: %w", err) + } + if len(baseline.Changes) != 0 || len(baseline.Steps) != 0 { + return "", &DriftError{Changes: baseline.Changes} + } + return migrate.ArtifactPending, nil +} + +type DriftError struct{ Changes []migrate.Change } + +func (e *DriftError) Error() string { + return fmt.Sprintf("sqlserver: baseline drift: %d catalog changes since preview", len(e.Changes)) +} + +func artifactHash(artifact migrate.Artifact) (string, error) { + artifact.Hash = "" + body, err := json.Marshal(artifact) + if err != nil { + return "", err + } + sum := sha256.Sum256(body) + return hex.EncodeToString(sum[:]), nil +} + +func artifactScope(artifact migrate.Artifact) error { + if len(artifact.Schemas) == 0 { + return fmt.Errorf("sqlserver: migration artifact requires explicit schema scope") + } + seen := map[string]bool{} + for _, name := range artifact.Schemas { + if name == "" || seen[name] { + return fmt.Errorf("sqlserver: empty or duplicate artifact schema %q", name) + } + seen[name] = true + } + for _, snapshot := range []migrate.Schema{artifact.Request.Current, artifact.Request.Desired} { + for _, table := range snapshot.Tables { + if !seen[table.Name.Schema] { + return fmt.Errorf("sqlserver: table %v is outside artifact schema scope", table.Name) + } + } + for _, definition := range snapshot.Definitions { + if !seen[definition.Name.Schema] { + return fmt.Errorf("sqlserver: definition %v is outside artifact schema scope", definition.Name) + } + } + } + return nil +} diff --git a/migrate/sqlserver/artifact_order.go b/migrate/sqlserver/artifact_order.go new file mode 100644 index 00000000..0e44d45d --- /dev/null +++ b/migrate/sqlserver/artifact_order.go @@ -0,0 +1,38 @@ +package sqlserver + +import ( + "cmp" + "slices" + + "github.com/flanksource/sqlglot-go/migrate" +) + +func compareName(a, b migrate.Name) int { + if c := cmp.Compare(a.Schema, b.Schema); c != 0 { + return c + } + return cmp.Compare(a.Name, b.Name) +} + +func orderSnapshot(snapshot *migrate.Schema) { + if snapshot.Tables == nil { + snapshot.Tables = []migrate.Table{} + } + if snapshot.Definitions == nil { + snapshot.Definitions = []migrate.Definition{} + } + slices.SortFunc(snapshot.Tables, func(a, b migrate.Table) int { return compareName(a.Name, b.Name) }) + slices.SortFunc(snapshot.Definitions, func(a, b migrate.Definition) int { return compareName(a.Name, b.Name) }) + for i := range snapshot.Tables { + t := &snapshot.Tables[i] + slices.SortFunc(t.Indexes, func(a, b migrate.Index) int { return cmp.Compare(a.Name, b.Name) }) + slices.SortFunc(t.ForeignKeys, func(a, b migrate.ForeignKey) int { return cmp.Compare(a.Name, b.Name) }) + slices.SortFunc(t.Checks, func(a, b migrate.Check) int { return cmp.Compare(a.Name, b.Name) }) + for j := range t.Indexes { + slices.Sort(t.Indexes[j].Included) + } + } + for i := range snapshot.Definitions { + slices.SortFunc(snapshot.Definitions[i].Dependencies, compareName) + } +} diff --git a/migrate/sqlserver/artifact_test.go b/migrate/sqlserver/artifact_test.go new file mode 100644 index 00000000..bcf59d6c --- /dev/null +++ b/migrate/sqlserver/artifact_test.go @@ -0,0 +1,95 @@ +package sqlserver_test + +import ( + "encoding/json" + "slices" + + "github.com/flanksource/sqlglot-go/migrate" + "github.com/flanksource/sqlglot-go/migrate/sqlserver" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func artifactRequest() migrate.ArtifactRequest { + orders := migrate.Table{Name: migrate.Name{Schema: "dbo", Name: "orders"}, Columns: []migrate.Column{{Name: "id", Type: "int"}, {Name: "label", Type: "nvarchar(20)", Nullable: true}}} + audit := migrate.Table{Name: migrate.Name{Schema: "audit", Name: "events"}, Columns: []migrate.Column{{Name: "id", Type: "int"}}} + desired := orders + desired.Columns = slices.Clone(orders.Columns) + desired.Columns[1].Type = "nvarchar(100)" + return migrate.ArtifactRequest{Schemas: []string{"dbo", "audit"}, Request: migrate.Request{Current: migrate.Schema{Tables: []migrate.Table{orders, audit}}, Desired: migrate.Schema{Tables: []migrate.Table{desired, audit}}}} +} + +var _ = Describe("Immutable migration artifacts", func() { + It("binds exact SQL and both snapshots to a deterministic hash without aliasing input", func() { + input := artifactRequest() + artifact, err := sqlserver.BuildArtifact(input) + Expect(err).NotTo(HaveOccurred()) + Expect(artifact.Hash).To(MatchRegexp("^[0-9a-f]{64}$")) + Expect(artifact.Schemas).To(Equal([]string{"audit", "dbo"})) + Expect(artifact.Plan.Steps).To(HaveLen(1)) + Expect(artifact.Plan.Steps[0].SQL).To(Equal("ALTER TABLE [dbo].[orders] ALTER COLUMN [label] nvarchar(100) NULL")) + slices.Reverse(input.Request.Current.Tables) + slices.Reverse(input.Request.Desired.Tables) + slices.Reverse(input.Schemas) + again, err := sqlserver.BuildArtifact(input) + Expect(err).NotTo(HaveOccurred()) + Expect(again).To(Equal(artifact)) + input.Request.Current.Tables[1].Columns[0].Type = "bigint" + Expect(sqlserver.ValidateArtifact(artifact)).To(Succeed()) + }) + + It("round trips JSON and rejects altered SQL, snapshots, scope and version", func() { + artifact, err := sqlserver.BuildArtifact(artifactRequest()) + Expect(err).NotTo(HaveOccurred()) + body, err := json.Marshal(artifact) + Expect(err).NotTo(HaveOccurred()) + for _, mutate := range []func(*migrate.Artifact){ + func(a *migrate.Artifact) { a.Plan.Steps[0].SQL = "DROP TABLE [dbo].[orders]" }, + func(a *migrate.Artifact) { a.Request.Desired.Tables[1].Columns[0].Type = "bigint" }, + func(a *migrate.Artifact) { a.Schemas[0] = "other" }, + func(a *migrate.Artifact) { a.Version++ }, + } { + var decoded migrate.Artifact + Expect(json.Unmarshal(body, &decoded)).To(Succeed()) + Expect(sqlserver.ValidateArtifact(decoded)).To(Succeed()) + mutate(&decoded) + Expect(sqlserver.ValidateArtifact(decoded)).NotTo(Succeed()) + } + }) + + It("accepts only the baseline or complete desired state and rejects drift in preserved members", func() { + input := artifactRequest() + artifact, err := sqlserver.BuildArtifact(input) + Expect(err).NotTo(HaveOccurred()) + state, err := sqlserver.CheckArtifact(artifact, input.Request.Current) + Expect(err).NotTo(HaveOccurred()) + Expect(state).To(Equal(migrate.ArtifactPending)) + state, err = sqlserver.CheckArtifact(artifact, input.Request.Desired) + Expect(err).NotTo(HaveOccurred()) + Expect(state).To(Equal(migrate.ArtifactApplied)) + input.Request.Current.Tables[0].Columns[0].Type = "bigint" + _, err = sqlserver.CheckArtifact(artifact, input.Request.Current) + Expect(err).To(MatchError(ContainSubstring("baseline drift"))) + }) + + It("compares catalog expressions semantically when checking a no-op", func() { + input := artifactRequest() + input.Request.Current.Tables[0].Columns[0].Default = &migrate.Default{Name: "DF_orders_id", Expression: "(0)"} + input.Request.Desired = input.Request.Current + artifact, err := sqlserver.BuildArtifact(input) + Expect(err).NotTo(HaveOccurred()) + input.Request.Current.Tables[0].Columns[0].Default.Expression = "((0))" + state, err := sqlserver.CheckArtifact(artifact, input.Request.Current) + Expect(err).NotTo(HaveOccurred()) + Expect(state).To(Equal(migrate.ArtifactApplied)) + }) + + It("requires a nonempty explicit scope containing both snapshots", func() { + input := artifactRequest() + for _, scope := range [][]string{nil, {""}, {"dbo"}, {"dbo", "dbo", "audit"}} { + input.Schemas = scope + _, err := sqlserver.BuildArtifact(input) + Expect(err).To(HaveOccurred()) + } + }) +}) diff --git a/migrate/sqlserver/convert.go b/migrate/sqlserver/convert.go index 180a95d0..4b33bc54 100644 --- a/migrate/sqlserver/convert.go +++ b/migrate/sqlserver/convert.go @@ -68,21 +68,52 @@ func realm(input migrate.Schema, namespaces []string) (*schema.Realm, error) { func uniqueObjects(input migrate.Schema) error { seen := make(map[migrate.Name]bool) + add := func(name migrate.Name) error { + if seen[name] { + return fmt.Errorf("sqlserver: duplicate object %+v", name) + } + seen[name] = true + return nil + } for _, table := range input.Tables { - if seen[table.Name] { - return fmt.Errorf("sqlserver: duplicate object %+v", table.Name) + if err := add(table.Name); err != nil { + return err + } + for _, name := range constraintNames(table) { + if err := add(migrate.Name{Schema: table.Name.Schema, Name: name}); err != nil { + return err + } } - seen[table.Name] = true } for _, d := range input.Definitions { - if seen[d.Name] { - return fmt.Errorf("sqlserver: duplicate object %+v", d.Name) + if err := add(d.Name); err != nil { + return err } - seen[d.Name] = true } return nil } +func constraintNames(table migrate.Table) []string { + var names []string + for _, c := range table.Columns { + if c.Default != nil && c.Default.Name != "" { + names = append(names, c.Default.Name) + } + } + for _, i := range table.Indexes { + if i.Primary || i.Constraint { + names = append(names, i.Name) + } + } + for _, f := range table.ForeignKeys { + names = append(names, f.Name) + } + for _, c := range table.Checks { + names = append(names, c.Name) + } + return names +} + func compareNames(a, b migrate.Name) int { if a.Schema != b.Schema { return strings.Compare(a.Schema, b.Schema) diff --git a/migrate/sqlserver/from_atlas.go b/migrate/sqlserver/from_atlas.go new file mode 100644 index 00000000..19baaf19 --- /dev/null +++ b/migrate/sqlserver/from_atlas.go @@ -0,0 +1,171 @@ +package sqlserver + +import ( + "ariga.io/atlas/sql/schema" + "fmt" + "github.com/flanksource/sqlglot-go/migrate" + atlas "github.com/flanksource/sqlglot-go/migrate/atlas/sqlserver" +) + +// FromAtlas projects supported declarative SQL Server tables and routines into the native model. +func FromAtlas(input *schema.Realm) (migrate.Schema, error) { + var output migrate.Schema + if input == nil || len(input.Objects)+len(input.Attrs) > 0 { + return output, fmt.Errorf("unsupported Atlas realm") + } + for _, s := range input.Schemas { + if len(s.Objects)+len(s.Views)+len(s.Attrs) > 0 { + return output, fmt.Errorf("unsupported Atlas schema objects in %q", s.Name) + } + for _, t := range s.Tables { + table, err := fromAtlasTable(t) + if err != nil { + return output, err + } + output.Tables = append(output.Tables, table) + } + for _, p := range s.Procs { + d, err := fromAtlasDefinition(migrate.Name{Schema: s.Name, Name: p.Name}, "PROCEDURE", p.Attrs, p.Deps) + if err != nil { + return output, err + } + output.Definitions = append(output.Definitions, d) + } + for _, f := range s.Funcs { + d, err := fromAtlasDefinition(migrate.Name{Schema: s.Name, Name: f.Name}, "FUNCTION", f.Attrs, f.Deps) + if err != nil { + return output, err + } + output.Definitions = append(output.Definitions, d) + } + } + _, err := Plan(migrate.Request{Current: output, Desired: output}) + return output, err +} + +func fromAtlasTable(t *schema.Table) (migrate.Table, error) { + if t.Schema == nil || len(t.Triggers) > 0 { + return migrate.Table{}, fmt.Errorf("unsupported or unscoped table %q", t.Name) + } + result := migrate.Table{Name: migrate.Name{Schema: t.Schema.Name, Name: t.Name}} + for _, c := range t.Columns { + if c.Type == nil || c.Type.Type == nil { + return result, fmt.Errorf("column %s has no parsed type", c.Name) + } + typ, err := atlas.FormatType(c.Type.Type) + if err != nil { + return result, err + } + column := migrate.Column{Name: c.Name, Type: typ, Nullable: c.Type.Null} + if c.Default != nil { + column.Default = &migrate.Default{} + column.Default.Expression, err = atlas.DefaultSQL(c) + if err != nil { + return result, err + } + } + for _, a := range c.Attrs { + switch a := a.(type) { + case *atlas.Identity: + column.Identity = &migrate.Identity{Seed: a.Seed, Increment: a.Increment} + case *atlas.DefaultConstraint: + if column.Default == nil { + return result, fmt.Errorf("default constraint without expression") + } + column.Default.Name = a.Name + case *schema.Collation: + column.Collation = a.V + default: + return result, fmt.Errorf("unsupported column attribute %T", a) + } + } + result.Columns = append(result.Columns, column) + } + indexes := append([]*schema.Index(nil), t.Indexes...) + if t.PrimaryKey != nil { + indexes = append(indexes, t.PrimaryKey) + } + for _, i := range indexes { + index, err := fromAtlasIndex(i, i == t.PrimaryKey) + if err != nil { + return result, err + } + if index.Primary && index.Name == "" { + index.Name = "PK_" + t.Name + } + result.Indexes = append(result.Indexes, index) + } + for _, f := range t.ForeignKeys { + if f.RefTable == nil || f.RefTable.Schema == nil || len(f.Attrs) > 0 { + return result, fmt.Errorf("unsupported foreign key %s", f.Symbol) + } + key := migrate.ForeignKey{Name: f.Symbol, References: migrate.Name{Schema: f.RefTable.Schema.Name, Name: f.RefTable.Name}, OnDelete: string(f.OnDelete), OnUpdate: string(f.OnUpdate)} + for _, c := range f.Columns { + key.Columns = append(key.Columns, c.Name) + } + for _, c := range f.RefColumns { + key.ReferencedColumns = append(key.ReferencedColumns, c.Name) + } + result.ForeignKeys = append(result.ForeignKeys, key) + } + for _, a := range t.Attrs { + check, ok := a.(*schema.Check) + if !ok || len(check.Attrs) > 0 { + return result, fmt.Errorf("unsupported table attribute %T", a) + } + result.Checks = append(result.Checks, migrate.Check{Name: check.Name, Expression: check.Expr}) + } + return result, nil +} + +func fromAtlasIndex(i *schema.Index, primary bool) (migrate.Index, error) { + result := migrate.Index{Name: i.Name, Unique: i.Unique, Primary: primary} + for _, p := range i.Parts { + if p.C == nil || p.X != nil || len(p.Attrs) > 0 { + return result, fmt.Errorf("unsupported index part on %s", i.Name) + } + result.Columns = append(result.Columns, migrate.IndexColumn{Name: p.C.Name, Descending: p.Desc}) + } + for _, a := range i.Attrs { + switch a := a.(type) { + case *atlas.Clustered: + result.Clustered = true + case *atlas.UniqueConstraint: + result.Constraint = true + case *atlas.IndexInclude: + result.Included = append([]string(nil), a.Columns...) + case *atlas.IndexPredicate: + result.Filter = a.P + default: + return result, fmt.Errorf("unsupported index attribute %T", a) + } + } + return result, nil +} + +func fromAtlasDefinition(name migrate.Name, kind string, attrs []schema.Attr, deps []schema.Object) (migrate.Definition, error) { + result := migrate.Definition{Name: name, Kind: kind} + for _, a := range attrs { + if definition, ok := a.(*atlas.Definition); ok { + result.SQL = definition.Text + } else { + return result, fmt.Errorf("unsupported routine attribute %T", a) + } + } + if result.SQL == "" { + return result, fmt.Errorf("routine %s requires its SQL definition", name.Name) + } + for _, dependency := range deps { + switch d := dependency.(type) { + case *schema.Table: + result.Dependencies = append(result.Dependencies, migrate.Name{Schema: d.Schema.Name, Name: d.Name}) + case *schema.Proc: + result.Dependencies = append(result.Dependencies, migrate.Name{Schema: d.Schema.Name, Name: d.Name}) + case *schema.Func: + result.Dependencies = append(result.Dependencies, migrate.Name{Schema: d.Schema.Name, Name: d.Name}) + default: + return result, fmt.Errorf("unsupported routine dependency %T", dependency) + } + } + return result, nil +} diff --git a/migrate/sqlserver/from_atlas_test.go b/migrate/sqlserver/from_atlas_test.go new file mode 100644 index 00000000..d517625b --- /dev/null +++ b/migrate/sqlserver/from_atlas_test.go @@ -0,0 +1,31 @@ +package sqlserver + +import ( + "ariga.io/atlas/sql/schema" + "github.com/flanksource/sqlglot-go/migrate" + atlas "github.com/flanksource/sqlglot-go/migrate/atlas/sqlserver" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Atlas snapshot projection", func() { + It("retains identity, defaults, clustered keys, constraints and routines", func() { + input := migrate.Schema{Tables: []migrate.Table{{Name: migrate.Name{Schema: "dbo", Name: "items"}, Columns: []migrate.Column{ + {Name: "id", Type: "int", Identity: &migrate.Identity{Seed: 1, Increment: 1}}, + {Name: "label", Type: "nvarchar(20)", Nullable: true, Default: &migrate.Default{Name: "DF_items_label", Expression: "N'new'"}}, + }, Indexes: []migrate.Index{{Name: "PK_items", Primary: true, Unique: true, Clustered: true, Columns: []migrate.IndexColumn{{Name: "id"}}}}, Checks: []migrate.Check{{Name: "positive_id", Expression: "id > 0"}}}}} + r, err := realm(input, []string{"dbo"}) + Expect(err).NotTo(HaveOccurred()) + proc, err := atlas.ParseRoutine("CREATE PROCEDURE dbo.example AS SELECT 1") + Expect(err).NotTo(HaveOccurred()) + Expect(proc.AddTo(r.Schemas[0])).To(Succeed()) + output, err := FromAtlas(r) + Expect(err).NotTo(HaveOccurred()) + input.Definitions = []migrate.Definition{{Name: migrate.Name{Schema: "dbo", Name: "example"}, Kind: "PROCEDURE", SQL: proc.Text}} + Expect(output).To(Equal(input)) + }) + It("rejects unsupported realm features", func() { + _, err := FromAtlas(schema.NewRealm(schema.New("dbo").AddViews(schema.NewView("v", "SELECT 1")))) + Expect(err).To(HaveOccurred()) + }) +}) diff --git a/migrate/sqlserver/namespace_test.go b/migrate/sqlserver/namespace_test.go new file mode 100644 index 00000000..6af9a911 --- /dev/null +++ b/migrate/sqlserver/namespace_test.go @@ -0,0 +1,24 @@ +package sqlserver_test + +import ( + "github.com/flanksource/sqlglot-go/migrate" + "github.com/flanksource/sqlglot-go/migrate/sqlserver" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("SQL Server object namespaces", func() { + It("rejects a selected default name that conflicts with an unselected table constraint", func() { + orders := migrate.Table{Name: migrate.Name{Schema: "dbo", Name: "orders"}, Columns: []migrate.Column{{Name: "id", Type: "int", Default: &migrate.Default{Name: "DF_shared", Expression: "(0)"}}}} + audit := migrate.Table{Name: migrate.Name{Schema: "dbo", Name: "audit"}, Columns: []migrate.Column{{Name: "id", Type: "int"}}, Checks: []migrate.Check{{Name: "DF_shared", Expression: "id >= 0"}}} + _, err := sqlserver.Plan(migrate.Request{Desired: migrate.Schema{Tables: []migrate.Table{orders, audit}}}) + Expect(err).To(MatchError(ContainSubstring("duplicate object"))) + }) + It("permits ordinary index names repeated on different tables", func() { + orders := migrate.Table{Name: migrate.Name{Schema: "dbo", Name: "orders"}, Columns: []migrate.Column{{Name: "id", Type: "int"}}, Indexes: []migrate.Index{{Name: "IX_id", Columns: []migrate.IndexColumn{{Name: "id"}}}}} + audit := orders + audit.Name.Name = "audit" + _, err := sqlserver.Plan(migrate.Request{Desired: migrate.Schema{Tables: []migrate.Table{orders, audit}}}) + Expect(err).NotTo(HaveOccurred()) + }) +}) diff --git a/migrate/sqlserver/planner_test.go b/migrate/sqlserver/planner_test.go index 366d3681..bf9a6a0b 100644 --- a/migrate/sqlserver/planner_test.go +++ b/migrate/sqlserver/planner_test.go @@ -40,7 +40,7 @@ var _ = Describe("Native migration contracts", func() { table := migrate.Table{Name: migrate.Name{Schema: "dbo", Name: "orders"}, Columns: []migrate.Column{{Name: "code", Type: "nvarchar(20)", Collation: "Latin1_General_100_BIN2", Default: &migrate.Default{Name: "DF_order_code", Expression: "N'new'"}}}, Indexes: []migrate.Index{{Name: "UQ_order_code", Unique: true, Constraint: true, Columns: []migrate.IndexColumn{{Name: "code"}}}}} plan, err := sqlserver.Plan(migrate.Request{Desired: migrate.Schema{Tables: []migrate.Table{table}}}) Expect(err).NotTo(HaveOccurred()) - Expect(plan.Steps[0].SQL).To(ContainSubstring("[code] nvarchar(20) COLLATE [Latin1_General_100_BIN2] NOT NULL CONSTRAINT [DF_order_code] DEFAULT N'new'")) + Expect(plan.Steps[0].SQL).To(ContainSubstring("[code] nvarchar(20) COLLATE Latin1_General_100_BIN2 NOT NULL CONSTRAINT [DF_order_code] DEFAULT N'new'")) Expect(plan.Steps[1].SQL).To(Equal("ALTER TABLE [dbo].[orders] ADD CONSTRAINT [UQ_order_code] UNIQUE NONCLUSTERED ([code])")) }) It("restores the database collation and preserves the named default", func() { diff --git a/migrate/sqlserver/table_order_test.go b/migrate/sqlserver/table_order_test.go new file mode 100644 index 00000000..0c898249 --- /dev/null +++ b/migrate/sqlserver/table_order_test.go @@ -0,0 +1,33 @@ +package sqlserver_test + +import ( + "github.com/flanksource/sqlglot-go/migrate" + "github.com/flanksource/sqlglot-go/migrate/sqlserver" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("SQL Server member dependency order", func() { + It("drops removed indexes before altering their column", func() { + current := migrate.Table{Name: migrate.Name{Schema: "dbo", Name: "orders"}, Columns: []migrate.Column{{Name: "label", Type: "nvarchar(20)"}}, Indexes: []migrate.Index{{Name: "IX_label", Columns: []migrate.IndexColumn{{Name: "label"}}}}} + desired := migrate.Table{Name: current.Name, Columns: []migrate.Column{{Name: "label", Type: "int"}}} + plan, err := sqlserver.Plan(migrate.Request{Current: migrate.Schema{Tables: []migrate.Table{current}}, Desired: migrate.Schema{Tables: []migrate.Table{desired}}}) + Expect(err).NotTo(HaveOccurred()) + Expect(plan.Steps[0].SQL).To(Equal("DROP INDEX [IX_label] ON [dbo].[orders]")) + Expect(plan.Steps[1].SQL).To(Equal("ALTER TABLE [dbo].[orders] ALTER COLUMN [label] int NOT NULL")) + }) + + It("splits index replacements around column changes while retaining their original review identity", func() { + current := migrate.Table{Name: migrate.Name{Schema: "dbo", Name: "orders"}, Columns: []migrate.Column{{Name: "label", Type: "nvarchar(20)"}}, Indexes: []migrate.Index{{Name: "IX_label", Columns: []migrate.IndexColumn{{Name: "label"}}}}} + desired := migrate.Table{Name: current.Name, Columns: []migrate.Column{{Name: "label", Type: "int"}}, Indexes: []migrate.Index{{Name: "IX_label", Unique: true, Columns: []migrate.IndexColumn{{Name: "label"}}}}} + plan, err := sqlserver.Plan(migrate.Request{Current: migrate.Schema{Tables: []migrate.Table{current}}, Desired: migrate.Schema{Tables: []migrate.Table{desired}}}) + Expect(err).NotTo(HaveOccurred()) + Expect(plan.Steps).To(HaveLen(3)) + Expect(plan.Steps[0].SQL).To(HavePrefix("DROP INDEX")) + Expect(plan.Steps[1].SQL).To(ContainSubstring("ALTER COLUMN")) + Expect(plan.Steps[2].SQL).To(HavePrefix("CREATE UNIQUE")) + Expect(plan.Steps[0].Change).To(Equal(plan.Steps[2].Change)) + Expect(plan.Steps[0].Change.Action).To(Equal("modify")) + Expect(plan.Changes).To(HaveLen(2)) + }) +}) diff --git a/sqlglot/generate_map_ginkgo_test.go b/sqlglot/generate_map_ginkgo_test.go new file mode 100644 index 00000000..f542acae --- /dev/null +++ b/sqlglot/generate_map_ginkgo_test.go @@ -0,0 +1,33 @@ +package sqlglot + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("MAP generation", func() { + DescribeTable("rejects mismatched key and value arrays without panicking", func(sql string, removeKey bool) { + tree, err := ParseOne(sql, "databricks") + Expect(err).NotTo(HaveOccurred()) + arg := "values" + if removeKey { + arg = "keys" + } + array := tree.Args[arg].(*Expression) + array.Set("expressions", []*Expression{}) + Expect(func() { + _, err = Generate(tree, "databricks") + }).NotTo(Panic()) + Expect(err).To(MatchError(ContainSubstring("mismatched key and value counts"))) + }, + Entry("missing values", "MAP(1, 'one')", false), + Entry("extra values", "MAP(1, 'one')", true), + ) + It("interleaves key and value pairs", func() { + tree, err := ParseOne("MAP(1, 'one', 2, 'two')", "databricks") + Expect(err).NotTo(HaveOccurred()) + written, err := Generate(tree, "databricks") + Expect(err).NotTo(HaveOccurred()) + Expect(written).To(Equal("MAP(1, 'one', 2, 'two')")) + }) +}) diff --git a/sqlglot/generate_nodes.go b/sqlglot/generate_nodes.go index 48a75e9e..b4106bc7 100644 --- a/sqlglot/generate_nodes.go +++ b/sqlglot/generate_nodes.go @@ -1075,9 +1075,12 @@ func (g *generator) writeVarMap(e *Expression) string { } keyItems, _ := keys.Args["expressions"].([]*Expression) valueItems, _ := values.Args["expressions"].([]*Expression) - parts := make([]string, 0, len(keyItems)+len(valueItems)) + if len(keyItems) != len(valueItems) { + return g.fail(e.Class + " with mismatched key and value counts") + } + parts := make([]string, len(keyItems)) for i := range keyItems { - parts = append(parts, g.node(keyItems[i]), g.node(valueItems[i])) + parts[i] = g.node(keyItems[i]) + ", " + g.node(valueItems[i]) } return "MAP(" + strings.Join(parts, ", ") + ")" }