chore: use new database definitions and tooling

Author: ezdac

play/bb.edn

@@ -145,7 +145,10 @@
   (src-file "collator/oapi/oapi.yaml")
 
   -sqlc-files
-  (concat [(src-file "db/sqlc.yaml")] (fs/glob src-dir "**.sql"))
+  (fs/glob src-dir "**/sqlc.yaml")
+
+  -sql-files
+  (fs/glob src-dir "**.sql")
 
   -go-files
   (->> (p/process ["go" "list" "-m" "-f={{.Dir}}"]
@@ -252,11 +255,11 @@
 
   build:sqlc
   {:doc "Run sqlc"
-   :depends [-sqlc-files]
+   :depends [-sqlc-files -sql-files]
    :task (when (and (not windows?)
-                    (seq (fs/modified-since exe -sqlc-files)))
+                    (seq (fs/modified-since exe (concat -sqlc-files -sql-files))))
            (run '-install:sqlc)
-           (run-process '[sqlc generate] {:dir (-> src-dir (fs/path "db") str)}))}
+           (run-process '[go generate -run="sqlc generate" -x ./...] {:dir src-dir}))}
 
   build:protoc
   {:doc "Run protoc"

rolling-shutter/chainobserver/observer.go

@@ -13,9 +13,11 @@ import (
 	"github.com/rs/zerolog/log"
 	"golang.org/x/sync/errgroup"
 
+	cltrdb "github.com/shutter-network/rolling-shutter/rolling-shutter/chainobserver/db/collator"
+	kprdb "github.com/shutter-network/rolling-shutter/rolling-shutter/chainobserver/db/keyper"
+	syncdb "github.com/shutter-network/rolling-shutter/rolling-shutter/chainobserver/db/sync"
 	"github.com/shutter-network/rolling-shutter/rolling-shutter/contract"
 	"github.com/shutter-network/rolling-shutter/rolling-shutter/contract/deployment"
-	"github.com/shutter-network/rolling-shutter/rolling-shutter/db/chainobsdb"
 	"github.com/shutter-network/rolling-shutter/rolling-shutter/medley/eventsyncer"
 	"github.com/shutter-network/rolling-shutter/rolling-shutter/medley/retry"
 	"github.com/shutter-network/rolling-shutter/rolling-shutter/shdb"
@@ -50,7 +52,7 @@ func New(contracts *deployment.Contracts, dbpool *pgxpool.Pool) *ChainObserver {
 }
 
 func (chainobs *ChainObserver) Observe(ctx context.Context, eventTypes []*eventsyncer.EventType) error {
-	db := chainobsdb.New(chainobs.dbpool)
+	db := syncdb.New(chainobs.dbpool)
 	eventSyncProgress, err := db.GetEventSyncProgress(ctx)
 	if err != nil {
 		return errors.Wrap(err, "failed to get last synced event from db")
@@ -146,10 +148,12 @@ func (chainobs *ChainObserver) handleEventSyncUpdate(
 		return err
 	}
 	return chainobs.dbpool.BeginFunc(ctx, func(tx pgx.Tx) error {
-		db := chainobsdb.New(tx)
+		kdb := kprdb.New(tx)
+		cltdb := cltrdb.New(tx)
+		sncdb := syncdb.New(tx)
 
 		if eventSyncUpdate.Event != nil {
-			if err := chainobs.handleEvent(ctx, db, eventSyncUpdate.Event); err != nil {
+			if err := chainobs.handleEvent(ctx, kdb, cltdb, eventSyncUpdate.Event); err != nil {
 				return err
 			}
 		}
@@ -163,7 +167,7 @@ func (chainobs *ChainObserver) handleEventSyncUpdate(
 			nextBlockNumber = eventSyncUpdate.BlockNumber
 			nextLogIndex = eventSyncUpdate.LogIndex + 1
 		}
-		if err := db.UpdateEventSyncProgress(ctx, chainobsdb.UpdateEventSyncProgressParams{
+		if err := sncdb.UpdateEventSyncProgress(ctx, syncdb.UpdateEventSyncProgressParams{
 			NextBlockNumber: int32(nextBlockNumber),
 			NextLogIndex:    int32(nextLogIndex),
 		}); err != nil {
@@ -174,14 +178,14 @@ func (chainobs *ChainObserver) handleEventSyncUpdate(
 }
 
 func (chainobs *ChainObserver) handleEvent(
-	ctx context.Context, db *chainobsdb.Queries, event interface{},
+	ctx context.Context, kdb *kprdb.Queries, cltdb *cltrdb.Queries, event interface{},
 ) error {
 	var err error
 	switch event := event.(type) {
 	case newKeyperConfig:
-		err = chainobs.handleKeypersConfigsListNewConfigEvent(ctx, db, event)
+		err = chainobs.handleKeypersConfigsListNewConfigEvent(ctx, kdb, event)
 	case newCollatorConfig:
-		err = chainobs.handleCollatorConfigsListNewConfigEvent(ctx, db, event)
+		err = chainobs.handleCollatorConfigsListNewConfigEvent(ctx, cltdb, event)
 	default:
 		log.Info().Str("event-type", reflect.TypeOf(event).String()).Interface("event", event).
 			Msg("ignoring unknown event")
@@ -190,7 +194,7 @@ func (chainobs *ChainObserver) handleEvent(
 }
 
 func (chainobs *ChainObserver) handleKeypersConfigsListNewConfigEvent(
-	ctx context.Context, db *chainobsdb.Queries, event newKeyperConfig,
+	ctx context.Context, db *kprdb.Queries, event newKeyperConfig,
 ) error {
 	log.Info().
 		Uint64("block-number", event.Raw.BlockNumber).
@@ -203,7 +207,7 @@ func (chainobs *ChainObserver) handleKeypersConfigsListNewConfigEvent(
 			"activation block number %d from config contract would overflow int64",
 			event.ActivationBlockNumber)
 	}
-	err := db.InsertKeyperSet(ctx, chainobsdb.InsertKeyperSetParams{
+	err := db.InsertKeyperSet(ctx, kprdb.InsertKeyperSetParams{
 		KeyperConfigIndex:     int64(event.KeyperConfigIndex),
 		ActivationBlockNumber: int64(event.ActivationBlockNumber),
 		Keypers:               shdb.EncodeAddresses(event.addrs),
@@ -216,7 +220,7 @@ func (chainobs *ChainObserver) handleKeypersConfigsListNewConfigEvent(
 }
 
 func (chainobs *ChainObserver) handleCollatorConfigsListNewConfigEvent(
-	ctx context.Context, db *chainobsdb.Queries, event newCollatorConfig,
+	ctx context.Context, db *cltrdb.Queries, event newCollatorConfig,
 ) error {
 	log.Info().
 		Uint64("block-number", event.Raw.BlockNumber).
@@ -232,7 +236,7 @@ func (chainobs *ChainObserver) handleCollatorConfigsListNewConfigEvent(
 	if len(event.addrs) > 1 {
 		return errors.Errorf("got multiple collators from collator addrs set contract: %s", event.addrs)
 	} else if len(event.addrs) == 1 {
-		err := db.InsertChainCollator(ctx, chainobsdb.InsertChainCollatorParams{
+		err := db.InsertChainCollator(ctx, cltrdb.InsertChainCollatorParams{
 			ActivationBlockNumber: int64(event.ActivationBlockNumber),
 			Collator:              shdb.EncodeAddress(event.addrs[0]),
 		})

rolling-shutter/cmd/collator/collator.go

@@ -5,16 +5,14 @@ import (
 
 	"github.com/jackc/pgx/v4/pgxpool"
 	"github.com/pkg/errors"
-	"github.com/rs/zerolog/log"
 	"github.com/spf13/cobra"
 
 	"github.com/shutter-network/rolling-shutter/rolling-shutter/collator"
 	"github.com/shutter-network/rolling-shutter/rolling-shutter/collator/config"
-	"github.com/shutter-network/rolling-shutter/rolling-shutter/db/cltrdb"
-	"github.com/shutter-network/rolling-shutter/rolling-shutter/db/metadb"
+	"github.com/shutter-network/rolling-shutter/rolling-shutter/collator/database"
 	"github.com/shutter-network/rolling-shutter/rolling-shutter/medley/configuration/command"
+	"github.com/shutter-network/rolling-shutter/rolling-shutter/medley/db"
 	"github.com/shutter-network/rolling-shutter/rolling-shutter/medley/service"
-	"github.com/shutter-network/rolling-shutter/rolling-shutter/shdb"
 )
 
 func Cmd() *cobra.Command {
@@ -33,28 +31,12 @@ func Cmd() *cobra.Command {
 
 func initDB(cfg *config.Config) error {
 	ctx := context.Background()
-
 	dbpool, err := pgxpool.Connect(ctx, cfg.DatabaseURL)
 	if err != nil {
 		return errors.Wrap(err, "failed to connect to database")
 	}
 	defer dbpool.Close()
-
-	err = cltrdb.ValidateDB(ctx, dbpool)
-	if err == nil {
-		shdb.AddConnectionInfo(log.Info(), dbpool).Msg("database already exists")
-		return nil
-	} else if errors.Is(err, metadb.ErrSchemaMismatch) {
-		return err
-	}
-
-	// initialize the db
-	err = cltrdb.InitDB(ctx, dbpool)
-	if err != nil {
-		return err
-	}
-	shdb.AddConnectionInfo(log.Info(), dbpool).Msg("database initialized")
-	return nil
+	return db.InitDB(ctx, dbpool, database.Definition.Name(), database.Definition)
 }
 
 func main(cfg *config.Config) error {

rolling-shutter/cmd/keyper/keyper.go

@@ -9,12 +9,11 @@ import (
 	"github.com/spf13/cobra"
 
 	"github.com/shutter-network/rolling-shutter/rolling-shutter/cmd/shversion"
-	"github.com/shutter-network/rolling-shutter/rolling-shutter/db/kprdb"
-	"github.com/shutter-network/rolling-shutter/rolling-shutter/db/metadb"
 	"github.com/shutter-network/rolling-shutter/rolling-shutter/keyper"
+	"github.com/shutter-network/rolling-shutter/rolling-shutter/keyper/database"
 	"github.com/shutter-network/rolling-shutter/rolling-shutter/medley/configuration/command"
+	"github.com/shutter-network/rolling-shutter/rolling-shutter/medley/db"
 	"github.com/shutter-network/rolling-shutter/rolling-shutter/medley/service"
-	"github.com/shutter-network/rolling-shutter/rolling-shutter/shdb"
 )
 
 func Cmd() *cobra.Command {
@@ -41,28 +40,14 @@ func main(config *keyper.Config) error {
 	return service.RunWithSighandler(context.Background(), keyper.New(config))
 }
 
-func initDB(config *keyper.Config) error {
+func initDB(cfg *keyper.Config) error {
 	ctx := context.Background()
 
-	dbpool, err := pgxpool.Connect(ctx, config.DatabaseURL)
+	dbpool, err := pgxpool.Connect(ctx, cfg.DatabaseURL)
 	if err != nil {
 		return errors.Wrap(err, "failed to connect to database")
 	}
 	defer dbpool.Close()
 
-	err = kprdb.ValidateKeyperDB(ctx, dbpool)
-	if err == nil {
-		shdb.AddConnectionInfo(log.Info(), dbpool).Msg("database already exists")
-		return nil
-	} else if errors.Is(err, metadb.ErrSchemaMismatch) {
-		return err
-	}
-
-	// initialize the db
-	err = kprdb.InitDB(ctx, dbpool)
-	if err != nil {
-		return err
-	}
-	shdb.AddConnectionInfo(log.Info(), dbpool).Msg("database initialized")
-	return nil
+	return db.InitDB(ctx, dbpool, database.Definition.Name(), database.Definition)
 }

rolling-shutter/cmd/snapshot/snapshot.go

@@ -5,15 +5,13 @@ import (
 
 	"github.com/jackc/pgx/v4/pgxpool"
 	"github.com/pkg/errors"
-	"github.com/rs/zerolog/log"
 	"github.com/spf13/cobra"
 
-	"github.com/shutter-network/rolling-shutter/rolling-shutter/db/metadb"
-	"github.com/shutter-network/rolling-shutter/rolling-shutter/db/snpdb"
 	"github.com/shutter-network/rolling-shutter/rolling-shutter/medley/configuration/command"
+	"github.com/shutter-network/rolling-shutter/rolling-shutter/medley/db"
 	"github.com/shutter-network/rolling-shutter/rolling-shutter/medley/service"
-	"github.com/shutter-network/rolling-shutter/rolling-shutter/shdb"
 	"github.com/shutter-network/rolling-shutter/rolling-shutter/snapshot"
+	"github.com/shutter-network/rolling-shutter/rolling-shutter/snapshot/database"
 )
 
 var (
@@ -36,31 +34,14 @@ func Cmd() *cobra.Command {
 	return builder.Command()
 }
 
-func initDB(config *snapshot.Config) error {
+func initDB(cfg *snapshot.Config) error {
 	ctx := context.Background()
-
-	dbpool, err := pgxpool.Connect(ctx, config.DatabaseURL)
+	dbpool, err := pgxpool.Connect(ctx, cfg.DatabaseURL)
 	if err != nil {
 		return errors.Wrap(err, "failed to connect to database")
 	}
 	defer dbpool.Close()
-
-	err = snpdb.ValidateSnapshotDB(ctx, dbpool)
-	if err == nil {
-		shdb.AddConnectionInfo(log.Info(), dbpool).Msg("database already exists")
-		return nil
-	} else if errors.Is(err, metadb.ErrSchemaMismatch) {
-		return err
-	}
-
-	// initialize the db
-	err = snpdb.InitDB(ctx, dbpool)
-	if err != nil {
-		return err
-	}
-	shdb.AddConnectionInfo(log.Info(), dbpool).Msg("database initialized")
-
-	return nil
+	return db.InitDB(ctx, dbpool, database.Definition.Name(), database.Definition)
 }
 
 func main(config *snapshot.Config) error {

rolling-shutter/cmd/snapshotkeyper/snapshotkeyper.go

@@ -9,12 +9,11 @@ import (
 	"github.com/spf13/cobra"
 
 	"github.com/shutter-network/rolling-shutter/rolling-shutter/cmd/shversion"
-	"github.com/shutter-network/rolling-shutter/rolling-shutter/db/kprdb"
-	"github.com/shutter-network/rolling-shutter/rolling-shutter/db/metadb"
 	"github.com/shutter-network/rolling-shutter/rolling-shutter/keyper"
+	"github.com/shutter-network/rolling-shutter/rolling-shutter/keyper/database"
 	"github.com/shutter-network/rolling-shutter/rolling-shutter/medley/configuration/command"
+	"github.com/shutter-network/rolling-shutter/rolling-shutter/medley/db"
 	"github.com/shutter-network/rolling-shutter/rolling-shutter/medley/service"
-	"github.com/shutter-network/rolling-shutter/rolling-shutter/shdb"
 	"github.com/shutter-network/rolling-shutter/rolling-shutter/snapshotkeyper"
 )
 
@@ -52,20 +51,5 @@ func initDB(config *keyper.Config) error {
 		return errors.Wrap(err, "failed to connect to database")
 	}
 	defer dbpool.Close()
-
-	err = kprdb.ValidateKeyperDB(ctx, dbpool)
-	if err == nil {
-		shdb.AddConnectionInfo(log.Info(), dbpool).Msg("database already exists")
-		return nil
-	} else if errors.Is(err, metadb.ErrSchemaMismatch) {
-		return err
-	}
-
-	// initialize the db
-	err = kprdb.InitDB(ctx, dbpool)
-	if err != nil {
-		return err
-	}
-	shdb.AddConnectionInfo(log.Info(), dbpool).Msg("database initialized")
-	return nil
+	return db.InitDB(ctx, dbpool, database.Definition.Name(), database.Definition)
 }