diff --git a/internal/compiler/compile.go b/internal/compiler/compile.go index 84fbb20a3c..ad6922a792 100644 --- a/internal/compiler/compile.go +++ b/internal/compiler/compile.go @@ -1,6 +1,7 @@ package compiler import ( + "context" "errors" "fmt" "io" @@ -39,6 +40,13 @@ func (c *Compiler) parseCatalog(schemas []string) error { } contents := migrations.RemoveRollbackStatements(string(blob)) c.schema = append(c.schema, contents) + + // In accurate mode, we only need to collect schema files for migrations + // but don't build the internal catalog from them + if c.accurateMode { + continue + } + stmts, err := c.parser.Parse(strings.NewReader(contents)) if err != nil { merr.Add(filename, contents, 0, err) @@ -58,6 +66,15 @@ func (c *Compiler) parseCatalog(schemas []string) error { } func (c *Compiler) parseQueries(o opts.Parser) (*Result, error) { + ctx := context.Background() + + // In accurate mode, initialize the database connection pool before parsing queries + if c.accurateMode && c.pgAnalyzer != nil { + if err := c.pgAnalyzer.EnsurePool(ctx, c.schema); err != nil { + return nil, fmt.Errorf("failed to initialize database connection: %w", err) + } + } + var q []*Query merr := multierr.New() set := map[string]struct{}{} @@ -113,6 +130,18 @@ func (c *Compiler) parseQueries(o opts.Parser) (*Result, error) { if len(q) == 0 { return nil, fmt.Errorf("no queries contained in paths %s", strings.Join(c.conf.Queries, ",")) } + + // In accurate mode, build the catalog from the database after parsing all queries + if c.accurateMode && c.pgAnalyzer != nil { + // Default to "public" schema if no specific schemas are specified + schemas := []string{"public"} + cat, err := c.pgAnalyzer.IntrospectSchema(ctx, schemas) + if err != nil { + return nil, fmt.Errorf("failed to introspect database schema: %w", err) + } + c.catalog = cat + } + return &Result{ Catalog: c.catalog, Queries: q, diff --git a/internal/compiler/engine.go b/internal/compiler/engine.go index 75749cd6df..d5680ae85d 100644 --- a/internal/compiler/engine.go +++ b/internal/compiler/engine.go @@ -14,6 +14,7 @@ import ( sqliteanalyze "github.com/sqlc-dev/sqlc/internal/engine/sqlite/analyzer" "github.com/sqlc-dev/sqlc/internal/opts" "github.com/sqlc-dev/sqlc/internal/sql/catalog" + "github.com/sqlc-dev/sqlc/internal/x/expander" ) type Compiler struct { @@ -27,6 +28,15 @@ type Compiler struct { selector selector schema []string + + // accurateMode indicates that the compiler should use database-only analysis + // and skip building the internal catalog from schema files + accurateMode bool + // pgAnalyzer is the PostgreSQL-specific analyzer used in accurate mode + // for schema introspection + pgAnalyzer *pganalyze.Analyzer + // expander is used to expand SELECT * and RETURNING * in accurate mode + expander *expander.Expander } func NewCompiler(conf config.SQL, combo config.CombinedSettings) (*Compiler, error) { @@ -37,6 +47,9 @@ func NewCompiler(conf config.SQL, combo config.CombinedSettings) (*Compiler, err c.client = client } + // Check for accurate mode + accurateMode := conf.Analyzer.Accurate != nil && *conf.Analyzer.Accurate + switch conf.Engine { case config.EngineSQLite: c.parser = sqlite.NewParser() @@ -56,10 +69,32 @@ func NewCompiler(conf config.SQL, combo config.CombinedSettings) (*Compiler, err c.catalog = dolphin.NewCatalog() c.selector = newDefaultSelector() case config.EnginePostgreSQL: - c.parser = postgresql.NewParser() + parser := postgresql.NewParser() + c.parser = parser c.catalog = postgresql.NewCatalog() c.selector = newDefaultSelector() - if conf.Database != nil { + + if accurateMode { + // Accurate mode requires a database connection + if conf.Database == nil { + return nil, fmt.Errorf("accurate mode requires database configuration") + } + if conf.Database.URI == "" && !conf.Database.Managed { + return nil, fmt.Errorf("accurate mode requires database.uri or database.managed") + } + c.accurateMode = true + // Create the PostgreSQL analyzer for schema introspection + c.pgAnalyzer = pganalyze.New(c.client, *conf.Database) + // Use the analyzer wrapped with cache for query analysis + c.analyzer = analyzer.Cached( + c.pgAnalyzer, + combo.Global, + *conf.Database, + ) + // Create the expander using the pgAnalyzer as the column getter + // The parser implements both Parser and format.Dialect interfaces + c.expander = expander.New(c.pgAnalyzer, parser, parser) + } else if conf.Database != nil { if conf.Analyzer.Database == nil || *conf.Analyzer.Database { c.analyzer = analyzer.Cached( pganalyze.New(c.client, *conf.Database), diff --git a/internal/compiler/parse.go b/internal/compiler/parse.go index 681d291122..14cad21728 100644 --- a/internal/compiler/parse.go +++ b/internal/compiler/parse.go @@ -71,7 +71,56 @@ func (c *Compiler) parseQuery(stmt ast.Node, src string, o opts.Parser) (*Query, } var anlys *analysis - if c.analyzer != nil { + if c.accurateMode && c.expander != nil { + // In accurate mode, use the expander for star expansion + // and rely entirely on the database analyzer for type resolution + expandedQuery, err := c.expander.Expand(ctx, rawSQL) + if err != nil { + return nil, fmt.Errorf("star expansion failed: %w", err) + } + + // Parse named parameters from the expanded query + expandedStmts, err := c.parser.Parse(strings.NewReader(expandedQuery)) + if err != nil { + return nil, fmt.Errorf("parsing expanded query failed: %w", err) + } + if len(expandedStmts) == 0 { + return nil, errors.New("no statements in expanded query") + } + expandedRaw := expandedStmts[0].Raw + + // Use the analyzer to get type information from the database + result, err := c.analyzer.Analyze(ctx, expandedRaw, expandedQuery, c.schema, nil) + if err != nil { + return nil, err + } + + // Convert the analyzer result to the internal analysis format + var cols []*Column + for _, col := range result.Columns { + cols = append(cols, convertColumn(col)) + } + var params []Parameter + for _, p := range result.Params { + params = append(params, Parameter{ + Number: int(p.Number), + Column: convertColumn(p.Column), + }) + } + + // Determine the insert table if applicable + var table *ast.TableName + if insert, ok := expandedRaw.Stmt.(*ast.InsertStmt); ok { + table, _ = ParseTableName(insert.Relation) + } + + anlys = &analysis{ + Table: table, + Columns: cols, + Parameters: params, + Query: expandedQuery, + } + } else if c.analyzer != nil { inference, _ := c.inferQuery(raw, rawSQL) if inference == nil { inference = &analysis{} diff --git a/internal/config/config.go b/internal/config/config.go index 0ff805fccd..a86b5e726d 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -124,6 +124,7 @@ type SQL struct { type Analyzer struct { Database *bool `json:"database" yaml:"database"` + Accurate *bool `json:"accurate" yaml:"accurate"` } // TODO: Figure out a better name for this diff --git a/internal/config/v_one.json b/internal/config/v_one.json index a0667a7c9c..ed49bba365 100644 --- a/internal/config/v_one.json +++ b/internal/config/v_one.json @@ -80,6 +80,9 @@ "properties": { "database": { "type": "boolean" + }, + "accurate": { + "type": "boolean" } } }, diff --git a/internal/config/v_two.json b/internal/config/v_two.json index acf914997d..33003e6da8 100644 --- a/internal/config/v_two.json +++ b/internal/config/v_two.json @@ -83,6 +83,9 @@ "properties": { "database": { "type": "boolean" + }, + "accurate": { + "type": "boolean" } } }, diff --git a/internal/endtoend/testdata/accurate_cte/postgresql/stdlib/exec.json b/internal/endtoend/testdata/accurate_cte/postgresql/stdlib/exec.json new file mode 100644 index 0000000000..ee1b7ecd9e --- /dev/null +++ b/internal/endtoend/testdata/accurate_cte/postgresql/stdlib/exec.json @@ -0,0 +1,3 @@ +{ + "contexts": ["managed-db"] +} diff --git a/internal/endtoend/testdata/accurate_cte/postgresql/stdlib/go/db.go b/internal/endtoend/testdata/accurate_cte/postgresql/stdlib/go/db.go new file mode 100644 index 0000000000..3b320aa168 --- /dev/null +++ b/internal/endtoend/testdata/accurate_cte/postgresql/stdlib/go/db.go @@ -0,0 +1,31 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.30.0 + +package querytest + +import ( + "context" + "database/sql" +) + +type DBTX interface { + ExecContext(context.Context, string, ...interface{}) (sql.Result, error) + PrepareContext(context.Context, string) (*sql.Stmt, error) + QueryContext(context.Context, string, ...interface{}) (*sql.Rows, error) + QueryRowContext(context.Context, string, ...interface{}) *sql.Row +} + +func New(db DBTX) *Queries { + return &Queries{db: db} +} + +type Queries struct { + db DBTX +} + +func (q *Queries) WithTx(tx *sql.Tx) *Queries { + return &Queries{ + db: tx, + } +} diff --git a/internal/endtoend/testdata/accurate_cte/postgresql/stdlib/go/models.go b/internal/endtoend/testdata/accurate_cte/postgresql/stdlib/go/models.go new file mode 100644 index 0000000000..d54719e1ea --- /dev/null +++ b/internal/endtoend/testdata/accurate_cte/postgresql/stdlib/go/models.go @@ -0,0 +1,11 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.30.0 + +package querytest + +type Product struct { + ID int32 + Name string + Price interface{} +} diff --git a/internal/endtoend/testdata/accurate_cte/postgresql/stdlib/go/query.sql.go b/internal/endtoend/testdata/accurate_cte/postgresql/stdlib/go/query.sql.go new file mode 100644 index 0000000000..1a60961250 --- /dev/null +++ b/internal/endtoend/testdata/accurate_cte/postgresql/stdlib/go/query.sql.go @@ -0,0 +1,54 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.30.0 +// source: query.sql + +package querytest + +import ( + "context" + "database/sql" +) + +const getProductStats = `-- name: GetProductStats :one +WITH product_stats AS (SELECT count(*) AS total, avg(price) AS avg_price FROM products) SELECT total, avg_price FROM product_stats; +` + +type GetProductStatsRow struct { + Total sql.NullInt64 + AvgPrice sql.NullString +} + +func (q *Queries) GetProductStats(ctx context.Context) (GetProductStatsRow, error) { + row := q.db.QueryRowContext(ctx, getProductStats) + var i GetProductStatsRow + err := row.Scan(&i.Total, &i.AvgPrice) + return i, err +} + +const listExpensiveProducts = `-- name: ListExpensiveProducts :many +WITH expensive AS (SELECT id, name, price FROM products WHERE price > 100) SELECT id, name, price FROM expensive; +` + +func (q *Queries) ListExpensiveProducts(ctx context.Context) ([]Product, error) { + rows, err := q.db.QueryContext(ctx, listExpensiveProducts) + if err != nil { + return nil, err + } + defer rows.Close() + var items []Product + for rows.Next() { + var i Product + if err := rows.Scan(&i.ID, &i.Name, &i.Price); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} diff --git a/internal/endtoend/testdata/accurate_cte/postgresql/stdlib/query.sql b/internal/endtoend/testdata/accurate_cte/postgresql/stdlib/query.sql new file mode 100644 index 0000000000..4626fe0f04 --- /dev/null +++ b/internal/endtoend/testdata/accurate_cte/postgresql/stdlib/query.sql @@ -0,0 +1,11 @@ +-- name: ListExpensiveProducts :many +WITH expensive AS ( + SELECT * FROM products WHERE price > 100 +) +SELECT * FROM expensive; + +-- name: GetProductStats :one +WITH product_stats AS ( + SELECT COUNT(*) as total, AVG(price) as avg_price FROM products +) +SELECT * FROM product_stats; diff --git a/internal/endtoend/testdata/accurate_cte/postgresql/stdlib/schema.sql b/internal/endtoend/testdata/accurate_cte/postgresql/stdlib/schema.sql new file mode 100644 index 0000000000..17aaa6e650 --- /dev/null +++ b/internal/endtoend/testdata/accurate_cte/postgresql/stdlib/schema.sql @@ -0,0 +1,5 @@ +CREATE TABLE products ( + id SERIAL PRIMARY KEY, + name TEXT NOT NULL, + price NUMERIC(10,2) NOT NULL +); diff --git a/internal/endtoend/testdata/accurate_cte/postgresql/stdlib/sqlc.yaml b/internal/endtoend/testdata/accurate_cte/postgresql/stdlib/sqlc.yaml new file mode 100644 index 0000000000..10fe8da1d8 --- /dev/null +++ b/internal/endtoend/testdata/accurate_cte/postgresql/stdlib/sqlc.yaml @@ -0,0 +1,11 @@ +version: "2" +sql: + - engine: postgresql + schema: "schema.sql" + queries: "query.sql" + analyzer: + accurate: true + gen: + go: + package: "querytest" + out: "go" diff --git a/internal/endtoend/testdata/accurate_cte_values/postgresql/stdlib/exec.json b/internal/endtoend/testdata/accurate_cte_values/postgresql/stdlib/exec.json new file mode 100644 index 0000000000..ee1b7ecd9e --- /dev/null +++ b/internal/endtoend/testdata/accurate_cte_values/postgresql/stdlib/exec.json @@ -0,0 +1,3 @@ +{ + "contexts": ["managed-db"] +} diff --git a/internal/endtoend/testdata/accurate_cte_values/postgresql/stdlib/go/db.go b/internal/endtoend/testdata/accurate_cte_values/postgresql/stdlib/go/db.go new file mode 100644 index 0000000000..3b320aa168 --- /dev/null +++ b/internal/endtoend/testdata/accurate_cte_values/postgresql/stdlib/go/db.go @@ -0,0 +1,31 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.30.0 + +package querytest + +import ( + "context" + "database/sql" +) + +type DBTX interface { + ExecContext(context.Context, string, ...interface{}) (sql.Result, error) + PrepareContext(context.Context, string) (*sql.Stmt, error) + QueryContext(context.Context, string, ...interface{}) (*sql.Rows, error) + QueryRowContext(context.Context, string, ...interface{}) *sql.Row +} + +func New(db DBTX) *Queries { + return &Queries{db: db} +} + +type Queries struct { + db DBTX +} + +func (q *Queries) WithTx(tx *sql.Tx) *Queries { + return &Queries{ + db: tx, + } +} diff --git a/internal/endtoend/testdata/accurate_cte_values/postgresql/stdlib/go/models.go b/internal/endtoend/testdata/accurate_cte_values/postgresql/stdlib/go/models.go new file mode 100644 index 0000000000..333ea43ea3 --- /dev/null +++ b/internal/endtoend/testdata/accurate_cte_values/postgresql/stdlib/go/models.go @@ -0,0 +1,5 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.30.0 + +package querytest diff --git a/internal/endtoend/testdata/accurate_cte_values/postgresql/stdlib/go/query.sql.go b/internal/endtoend/testdata/accurate_cte_values/postgresql/stdlib/go/query.sql.go new file mode 100644 index 0000000000..d303845dcc --- /dev/null +++ b/internal/endtoend/testdata/accurate_cte_values/postgresql/stdlib/go/query.sql.go @@ -0,0 +1,38 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.30.0 +// source: query.sql + +package querytest + +import ( + "context" + "database/sql" +) + +const getPendingSaleStatuses = `-- name: GetPendingSaleStatuses :many +WITH w_pending_sale_status AS (SELECT status FROM (VALUES ('SAVED'), ('IDLE'), ('IN PROGRESS'), ('HELD')) AS pending_sale_status(status)) SELECT status FROM w_pending_sale_status; +` + +func (q *Queries) GetPendingSaleStatuses(ctx context.Context) ([]sql.NullString, error) { + rows, err := q.db.QueryContext(ctx, getPendingSaleStatuses) + if err != nil { + return nil, err + } + defer rows.Close() + var items []sql.NullString + for rows.Next() { + var status sql.NullString + if err := rows.Scan(&status); err != nil { + return nil, err + } + items = append(items, status) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} diff --git a/internal/endtoend/testdata/accurate_cte_values/postgresql/stdlib/query.sql b/internal/endtoend/testdata/accurate_cte_values/postgresql/stdlib/query.sql new file mode 100644 index 0000000000..607660da46 --- /dev/null +++ b/internal/endtoend/testdata/accurate_cte_values/postgresql/stdlib/query.sql @@ -0,0 +1,7 @@ +-- name: GetPendingSaleStatuses :many +WITH w_pending_sale_status as ( + select * from + (values ('SAVED'), ('IDLE'), ('IN PROGRESS'), ('HELD')) + as pending_sale_status(status) +) +SELECT status FROM w_pending_sale_status; diff --git a/internal/endtoend/testdata/accurate_cte_values/postgresql/stdlib/schema.sql b/internal/endtoend/testdata/accurate_cte_values/postgresql/stdlib/schema.sql new file mode 100644 index 0000000000..50de74bedf --- /dev/null +++ b/internal/endtoend/testdata/accurate_cte_values/postgresql/stdlib/schema.sql @@ -0,0 +1 @@ +-- Empty schema - CTE uses VALUES clause only diff --git a/internal/endtoend/testdata/accurate_cte_values/postgresql/stdlib/sqlc.yaml b/internal/endtoend/testdata/accurate_cte_values/postgresql/stdlib/sqlc.yaml new file mode 100644 index 0000000000..5f1405f536 --- /dev/null +++ b/internal/endtoend/testdata/accurate_cte_values/postgresql/stdlib/sqlc.yaml @@ -0,0 +1,13 @@ +version: "2" +sql: + - engine: "postgresql" + queries: "query.sql" + schema: "schema.sql" + database: + uri: "postgresql://postgres:mysecretpassword@localhost:5432/postgres" + gen: + go: + package: "querytest" + out: "go" + analyzer: + accurate: true diff --git a/internal/endtoend/testdata/accurate_enum/postgresql/stdlib/exec.json b/internal/endtoend/testdata/accurate_enum/postgresql/stdlib/exec.json new file mode 100644 index 0000000000..ee1b7ecd9e --- /dev/null +++ b/internal/endtoend/testdata/accurate_enum/postgresql/stdlib/exec.json @@ -0,0 +1,3 @@ +{ + "contexts": ["managed-db"] +} diff --git a/internal/endtoend/testdata/accurate_enum/postgresql/stdlib/go/db.go b/internal/endtoend/testdata/accurate_enum/postgresql/stdlib/go/db.go new file mode 100644 index 0000000000..3b320aa168 --- /dev/null +++ b/internal/endtoend/testdata/accurate_enum/postgresql/stdlib/go/db.go @@ -0,0 +1,31 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.30.0 + +package querytest + +import ( + "context" + "database/sql" +) + +type DBTX interface { + ExecContext(context.Context, string, ...interface{}) (sql.Result, error) + PrepareContext(context.Context, string) (*sql.Stmt, error) + QueryContext(context.Context, string, ...interface{}) (*sql.Rows, error) + QueryRowContext(context.Context, string, ...interface{}) *sql.Row +} + +func New(db DBTX) *Queries { + return &Queries{db: db} +} + +type Queries struct { + db DBTX +} + +func (q *Queries) WithTx(tx *sql.Tx) *Queries { + return &Queries{ + db: tx, + } +} diff --git a/internal/endtoend/testdata/accurate_enum/postgresql/stdlib/go/models.go b/internal/endtoend/testdata/accurate_enum/postgresql/stdlib/go/models.go new file mode 100644 index 0000000000..2b42787339 --- /dev/null +++ b/internal/endtoend/testdata/accurate_enum/postgresql/stdlib/go/models.go @@ -0,0 +1,59 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.30.0 + +package querytest + +import ( + "database/sql/driver" + "fmt" +) + +type Status string + +const ( + StatusPending Status = "pending" + StatusActive Status = "active" + StatusCompleted Status = "completed" +) + +func (e *Status) Scan(src interface{}) error { + switch s := src.(type) { + case []byte: + *e = Status(s) + case string: + *e = Status(s) + default: + return fmt.Errorf("unsupported scan type for Status: %T", src) + } + return nil +} + +type NullStatus struct { + Status Status + Valid bool // Valid is true if Status is not NULL +} + +// Scan implements the Scanner interface. +func (ns *NullStatus) Scan(value interface{}) error { + if value == nil { + ns.Status, ns.Valid = "", false + return nil + } + ns.Valid = true + return ns.Status.Scan(value) +} + +// Value implements the driver Valuer interface. +func (ns NullStatus) Value() (driver.Value, error) { + if !ns.Valid { + return nil, nil + } + return string(ns.Status), nil +} + +type Task struct { + ID int32 + Title string + Status Status +} diff --git a/internal/endtoend/testdata/accurate_enum/postgresql/stdlib/go/query.sql.go b/internal/endtoend/testdata/accurate_enum/postgresql/stdlib/go/query.sql.go new file mode 100644 index 0000000000..b78052fe87 --- /dev/null +++ b/internal/endtoend/testdata/accurate_enum/postgresql/stdlib/go/query.sql.go @@ -0,0 +1,81 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.30.0 +// source: query.sql + +package querytest + +import ( + "context" + "database/sql" +) + +const createTask = `-- name: CreateTask :one +INSERT INTO tasks (title, status) VALUES ($1, $2) RETURNING id, title, status; +` + +type CreateTaskParams struct { + Column1 sql.NullString + Column2 NullStatus +} + +func (q *Queries) CreateTask(ctx context.Context, arg CreateTaskParams) (Task, error) { + row := q.db.QueryRowContext(ctx, createTask, arg.Column1, arg.Column2) + var i Task + err := row.Scan(&i.ID, &i.Title, &i.Status) + return i, err +} + +const getTasksByStatus = `-- name: GetTasksByStatus :many +SELECT id, title, status FROM tasks WHERE status = $1; +` + +func (q *Queries) GetTasksByStatus(ctx context.Context, dollar_1 NullStatus) ([]Task, error) { + rows, err := q.db.QueryContext(ctx, getTasksByStatus, dollar_1) + if err != nil { + return nil, err + } + defer rows.Close() + var items []Task + for rows.Next() { + var i Task + if err := rows.Scan(&i.ID, &i.Title, &i.Status); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const listTasks = `-- name: ListTasks :many +SELECT id, title, status FROM tasks; +` + +func (q *Queries) ListTasks(ctx context.Context) ([]Task, error) { + rows, err := q.db.QueryContext(ctx, listTasks) + if err != nil { + return nil, err + } + defer rows.Close() + var items []Task + for rows.Next() { + var i Task + if err := rows.Scan(&i.ID, &i.Title, &i.Status); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} diff --git a/internal/endtoend/testdata/accurate_enum/postgresql/stdlib/query.sql b/internal/endtoend/testdata/accurate_enum/postgresql/stdlib/query.sql new file mode 100644 index 0000000000..11dcd9bf48 --- /dev/null +++ b/internal/endtoend/testdata/accurate_enum/postgresql/stdlib/query.sql @@ -0,0 +1,8 @@ +-- name: ListTasks :many +SELECT * FROM tasks; + +-- name: GetTasksByStatus :many +SELECT * FROM tasks WHERE status = $1; + +-- name: CreateTask :one +INSERT INTO tasks (title, status) VALUES ($1, $2) RETURNING *; diff --git a/internal/endtoend/testdata/accurate_enum/postgresql/stdlib/schema.sql b/internal/endtoend/testdata/accurate_enum/postgresql/stdlib/schema.sql new file mode 100644 index 0000000000..443ae9845f --- /dev/null +++ b/internal/endtoend/testdata/accurate_enum/postgresql/stdlib/schema.sql @@ -0,0 +1,7 @@ +CREATE TYPE status AS ENUM ('pending', 'active', 'completed'); + +CREATE TABLE tasks ( + id SERIAL PRIMARY KEY, + title TEXT NOT NULL, + status status NOT NULL DEFAULT 'pending' +); diff --git a/internal/endtoend/testdata/accurate_enum/postgresql/stdlib/sqlc.yaml b/internal/endtoend/testdata/accurate_enum/postgresql/stdlib/sqlc.yaml new file mode 100644 index 0000000000..10fe8da1d8 --- /dev/null +++ b/internal/endtoend/testdata/accurate_enum/postgresql/stdlib/sqlc.yaml @@ -0,0 +1,11 @@ +version: "2" +sql: + - engine: postgresql + schema: "schema.sql" + queries: "query.sql" + analyzer: + accurate: true + gen: + go: + package: "querytest" + out: "go" diff --git a/internal/endtoend/testdata/accurate_star_expansion/postgresql/stdlib/exec.json b/internal/endtoend/testdata/accurate_star_expansion/postgresql/stdlib/exec.json new file mode 100644 index 0000000000..ee1b7ecd9e --- /dev/null +++ b/internal/endtoend/testdata/accurate_star_expansion/postgresql/stdlib/exec.json @@ -0,0 +1,3 @@ +{ + "contexts": ["managed-db"] +} diff --git a/internal/endtoend/testdata/accurate_star_expansion/postgresql/stdlib/go/db.go b/internal/endtoend/testdata/accurate_star_expansion/postgresql/stdlib/go/db.go new file mode 100644 index 0000000000..3b320aa168 --- /dev/null +++ b/internal/endtoend/testdata/accurate_star_expansion/postgresql/stdlib/go/db.go @@ -0,0 +1,31 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.30.0 + +package querytest + +import ( + "context" + "database/sql" +) + +type DBTX interface { + ExecContext(context.Context, string, ...interface{}) (sql.Result, error) + PrepareContext(context.Context, string) (*sql.Stmt, error) + QueryContext(context.Context, string, ...interface{}) (*sql.Rows, error) + QueryRowContext(context.Context, string, ...interface{}) *sql.Row +} + +func New(db DBTX) *Queries { + return &Queries{db: db} +} + +type Queries struct { + db DBTX +} + +func (q *Queries) WithTx(tx *sql.Tx) *Queries { + return &Queries{ + db: tx, + } +} diff --git a/internal/endtoend/testdata/accurate_star_expansion/postgresql/stdlib/go/models.go b/internal/endtoend/testdata/accurate_star_expansion/postgresql/stdlib/go/models.go new file mode 100644 index 0000000000..ec1cb8d670 --- /dev/null +++ b/internal/endtoend/testdata/accurate_star_expansion/postgresql/stdlib/go/models.go @@ -0,0 +1,15 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.30.0 + +package querytest + +import ( + "database/sql" +) + +type Author struct { + ID int32 + Name string + Bio sql.NullString +} diff --git a/internal/endtoend/testdata/accurate_star_expansion/postgresql/stdlib/go/query.sql.go b/internal/endtoend/testdata/accurate_star_expansion/postgresql/stdlib/go/query.sql.go new file mode 100644 index 0000000000..0e81a90cf9 --- /dev/null +++ b/internal/endtoend/testdata/accurate_star_expansion/postgresql/stdlib/go/query.sql.go @@ -0,0 +1,93 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.30.0 +// source: query.sql + +package querytest + +import ( + "context" + "database/sql" +) + +const createAuthor = `-- name: CreateAuthor :one +INSERT INTO authors (name, bio) VALUES ($1, $2) RETURNING id, name, bio; +` + +type CreateAuthorParams struct { + Column1 sql.NullString + Column2 sql.NullString +} + +func (q *Queries) CreateAuthor(ctx context.Context, arg CreateAuthorParams) (Author, error) { + row := q.db.QueryRowContext(ctx, createAuthor, arg.Column1, arg.Column2) + var i Author + err := row.Scan(&i.ID, &i.Name, &i.Bio) + return i, err +} + +const deleteAuthor = `-- name: DeleteAuthor :one +DELETE FROM authors WHERE id = $1 RETURNING id, name, bio; +` + +func (q *Queries) DeleteAuthor(ctx context.Context, dollar_1 sql.NullInt32) (Author, error) { + row := q.db.QueryRowContext(ctx, deleteAuthor, dollar_1) + var i Author + err := row.Scan(&i.ID, &i.Name, &i.Bio) + return i, err +} + +const getAuthor = `-- name: GetAuthor :one +SELECT id, name, bio FROM authors WHERE id = $1; +` + +func (q *Queries) GetAuthor(ctx context.Context, dollar_1 sql.NullInt32) (Author, error) { + row := q.db.QueryRowContext(ctx, getAuthor, dollar_1) + var i Author + err := row.Scan(&i.ID, &i.Name, &i.Bio) + return i, err +} + +const listAuthors = `-- name: ListAuthors :many +SELECT id, name, bio FROM authors; +` + +func (q *Queries) ListAuthors(ctx context.Context) ([]Author, error) { + rows, err := q.db.QueryContext(ctx, listAuthors) + if err != nil { + return nil, err + } + defer rows.Close() + var items []Author + for rows.Next() { + var i Author + if err := rows.Scan(&i.ID, &i.Name, &i.Bio); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const updateAuthor = `-- name: UpdateAuthor :one +UPDATE authors SET name = $1, bio = $2 WHERE id = $3 RETURNING id, name, bio; +` + +type UpdateAuthorParams struct { + Column1 sql.NullString + Column2 sql.NullString + Column3 sql.NullInt32 +} + +func (q *Queries) UpdateAuthor(ctx context.Context, arg UpdateAuthorParams) (Author, error) { + row := q.db.QueryRowContext(ctx, updateAuthor, arg.Column1, arg.Column2, arg.Column3) + var i Author + err := row.Scan(&i.ID, &i.Name, &i.Bio) + return i, err +} diff --git a/internal/endtoend/testdata/accurate_star_expansion/postgresql/stdlib/query.sql b/internal/endtoend/testdata/accurate_star_expansion/postgresql/stdlib/query.sql new file mode 100644 index 0000000000..e091a5eaef --- /dev/null +++ b/internal/endtoend/testdata/accurate_star_expansion/postgresql/stdlib/query.sql @@ -0,0 +1,14 @@ +-- name: ListAuthors :many +SELECT * FROM authors; + +-- name: GetAuthor :one +SELECT * FROM authors WHERE id = $1; + +-- name: CreateAuthor :one +INSERT INTO authors (name, bio) VALUES ($1, $2) RETURNING *; + +-- name: UpdateAuthor :one +UPDATE authors SET name = $1, bio = $2 WHERE id = $3 RETURNING *; + +-- name: DeleteAuthor :one +DELETE FROM authors WHERE id = $1 RETURNING *; diff --git a/internal/endtoend/testdata/accurate_star_expansion/postgresql/stdlib/schema.sql b/internal/endtoend/testdata/accurate_star_expansion/postgresql/stdlib/schema.sql new file mode 100644 index 0000000000..ca6ad1e2cf --- /dev/null +++ b/internal/endtoend/testdata/accurate_star_expansion/postgresql/stdlib/schema.sql @@ -0,0 +1,5 @@ +CREATE TABLE authors ( + id SERIAL PRIMARY KEY, + name TEXT NOT NULL, + bio TEXT +); diff --git a/internal/endtoend/testdata/accurate_star_expansion/postgresql/stdlib/sqlc.yaml b/internal/endtoend/testdata/accurate_star_expansion/postgresql/stdlib/sqlc.yaml new file mode 100644 index 0000000000..10fe8da1d8 --- /dev/null +++ b/internal/endtoend/testdata/accurate_star_expansion/postgresql/stdlib/sqlc.yaml @@ -0,0 +1,11 @@ +version: "2" +sql: + - engine: postgresql + schema: "schema.sql" + queries: "query.sql" + analyzer: + accurate: true + gen: + go: + package: "querytest" + out: "go" diff --git a/internal/engine/postgresql/analyzer/analyze.go b/internal/engine/postgresql/analyzer/analyze.go index 5a08fa98ec..f12a2aa277 100644 --- a/internal/engine/postgresql/analyzer/analyze.go +++ b/internal/engine/postgresql/analyzer/analyze.go @@ -17,6 +17,7 @@ import ( "github.com/sqlc-dev/sqlc/internal/opts" "github.com/sqlc-dev/sqlc/internal/shfmt" "github.com/sqlc-dev/sqlc/internal/sql/ast" + "github.com/sqlc-dev/sqlc/internal/sql/catalog" "github.com/sqlc-dev/sqlc/internal/sql/named" "github.com/sqlc-dev/sqlc/internal/sql/sqlerr" ) @@ -320,3 +321,227 @@ func (a *Analyzer) Close(_ context.Context) error { } return nil } + +// SQL queries for schema introspection +const introspectTablesQuery = ` +SELECT + n.nspname AS schema_name, + c.relname AS table_name, + a.attname AS column_name, + pg_catalog.format_type(a.atttypid, a.atttypmod) AS data_type, + a.attnotnull AS not_null, + a.attndims AS array_dims, + COALESCE( + (SELECT true FROM pg_index i + WHERE i.indrelid = c.oid + AND i.indisprimary + AND a.attnum = ANY(i.indkey)), + false + ) AS is_primary_key +FROM + pg_catalog.pg_class c +JOIN + pg_catalog.pg_namespace n ON n.oid = c.relnamespace +JOIN + pg_catalog.pg_attribute a ON a.attrelid = c.oid +WHERE + c.relkind IN ('r', 'v', 'p') -- tables, views, partitioned tables + AND a.attnum > 0 -- skip system columns + AND NOT a.attisdropped + AND n.nspname = ANY($1) +ORDER BY + n.nspname, c.relname, a.attnum +` + +const introspectEnumsQuery = ` +SELECT + n.nspname AS schema_name, + t.typname AS type_name, + e.enumlabel AS enum_value +FROM + pg_catalog.pg_type t +JOIN + pg_catalog.pg_namespace n ON n.oid = t.typnamespace +JOIN + pg_catalog.pg_enum e ON e.enumtypid = t.oid +WHERE + t.typtype = 'e' + AND n.nspname = ANY($1) +ORDER BY + n.nspname, t.typname, e.enumsortorder +` + +type introspectedColumn struct { + SchemaName string `db:"schema_name"` + TableName string `db:"table_name"` + ColumnName string `db:"column_name"` + DataType string `db:"data_type"` + NotNull bool `db:"not_null"` + ArrayDims int `db:"array_dims"` + IsPrimaryKey bool `db:"is_primary_key"` +} + +type introspectedEnum struct { + SchemaName string `db:"schema_name"` + TypeName string `db:"type_name"` + EnumValue string `db:"enum_value"` +} + +// IntrospectSchema queries the database to build a catalog containing +// tables, columns, and enum types for the specified schemas. +func (a *Analyzer) IntrospectSchema(ctx context.Context, schemas []string) (*catalog.Catalog, error) { + if a.pool == nil { + return nil, fmt.Errorf("database connection not initialized") + } + + c, err := a.pool.Acquire(ctx) + if err != nil { + return nil, err + } + defer c.Release() + + // Query tables and columns + rows, err := c.Query(ctx, introspectTablesQuery, schemas) + if err != nil { + return nil, fmt.Errorf("introspect tables: %w", err) + } + columns, err := pgx.CollectRows(rows, pgx.RowToStructByName[introspectedColumn]) + if err != nil { + return nil, fmt.Errorf("collect table rows: %w", err) + } + + // Query enums + enumRows, err := c.Query(ctx, introspectEnumsQuery, schemas) + if err != nil { + return nil, fmt.Errorf("introspect enums: %w", err) + } + enums, err := pgx.CollectRows(enumRows, pgx.RowToStructByName[introspectedEnum]) + if err != nil { + return nil, fmt.Errorf("collect enum rows: %w", err) + } + + // Build catalog + cat := &catalog.Catalog{ + DefaultSchema: "public", + SearchPath: schemas, + } + + // Create schema map for quick lookup + schemaMap := make(map[string]*catalog.Schema) + for _, schemaName := range schemas { + schema := &catalog.Schema{Name: schemaName} + cat.Schemas = append(cat.Schemas, schema) + schemaMap[schemaName] = schema + } + + // Group columns by table + tableMap := make(map[string]*catalog.Table) + for _, col := range columns { + key := col.SchemaName + "." + col.TableName + tbl, exists := tableMap[key] + if !exists { + tbl = &catalog.Table{ + Rel: &ast.TableName{ + Schema: col.SchemaName, + Name: col.TableName, + }, + } + tableMap[key] = tbl + if schema, ok := schemaMap[col.SchemaName]; ok { + schema.Tables = append(schema.Tables, tbl) + } + } + + dt, isArray, dims := parseType(col.DataType) + tbl.Columns = append(tbl.Columns, &catalog.Column{ + Name: col.ColumnName, + Type: ast.TypeName{Name: dt}, + IsNotNull: col.NotNull, + IsArray: isArray || col.ArrayDims > 0, + ArrayDims: max(dims, col.ArrayDims), + }) + } + + // Group enum values by type + enumMap := make(map[string]*catalog.Enum) + for _, e := range enums { + key := e.SchemaName + "." + e.TypeName + enum, exists := enumMap[key] + if !exists { + enum = &catalog.Enum{ + Name: e.TypeName, + } + enumMap[key] = enum + if schema, ok := schemaMap[e.SchemaName]; ok { + schema.Types = append(schema.Types, enum) + } + } + enum.Vals = append(enum.Vals, e.EnumValue) + } + + return cat, nil +} + +// EnsurePool initializes the database connection pool if not already done. +// This is useful for accurate mode where we need to connect before analyzing queries. +func (a *Analyzer) EnsurePool(ctx context.Context, migrations []string) error { + if a.pool != nil { + return nil + } + + var uri string + if a.db.Managed { + if a.client == nil { + return fmt.Errorf("client is nil") + } + edb, err := a.client.CreateDatabase(ctx, &dbmanager.CreateDatabaseRequest{ + Engine: "postgresql", + Migrations: migrations, + }) + if err != nil { + return err + } + uri = edb.Uri + } else if a.dbg.OnlyManagedDatabases { + return fmt.Errorf("database: connections disabled via SQLCDEBUG=databases=managed") + } else { + uri = a.replacer.Replace(a.db.URI) + } + + conf, err := pgxpool.ParseConfig(uri) + if err != nil { + return err + } + pool, err := pgxpool.NewWithConfig(ctx, conf) + if err != nil { + return err + } + a.pool = pool + return nil +} + +// GetColumnNames implements the expander.ColumnGetter interface. +// It prepares a query and returns the column names from the result set description. +func (a *Analyzer) GetColumnNames(ctx context.Context, query string) ([]string, error) { + if a.pool == nil { + return nil, fmt.Errorf("database connection not initialized") + } + + conn, err := a.pool.Acquire(ctx) + if err != nil { + return nil, err + } + defer conn.Release() + + desc, err := conn.Conn().Prepare(ctx, "", query) + if err != nil { + return nil, err + } + + columns := make([]string, len(desc.Fields)) + for i, field := range desc.Fields { + columns[i] = field.Name + } + + return columns, nil +} diff --git a/internal/sql/ast/select_stmt.go b/internal/sql/ast/select_stmt.go index 8c3606dd4d..62e6f1c9cf 100644 --- a/internal/sql/ast/select_stmt.go +++ b/internal/sql/ast/select_stmt.go @@ -37,9 +37,16 @@ func (n *SelectStmt) Format(buf *TrackedBuffer, d format.Dialect) { } if items(n.ValuesLists) { - buf.WriteString("VALUES (") - buf.astFormat(n.ValuesLists, d) - buf.WriteString(")") + buf.WriteString("VALUES ") + // ValuesLists is a list of rows, where each row is a List of values + for i, row := range n.ValuesLists.Items { + if i > 0 { + buf.WriteString(", ") + } + buf.WriteString("(") + buf.astFormat(row, d) + buf.WriteString(")") + } return }