diff --git a/README.md b/README.md index 8010cb7..f9c59a5 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ Settings come from ``settings.json``, see ``settings.json.template`` for an exam ``clientSecret`` is the clientSecret used to authenticate with the backend Etherpad instances. This should be a random string that is unique to this service. ``tokenURL`` is the URL of the OAuth2 token endpoint. This is normally `http://:/oidc/token` -## Basic Auth backed Etherpads +## Basic Auth-backed Etherpads ``username`` is the username used to authenticate with the backend Etherpad instances. This should be a random string that is unique to this service. @@ -52,5 +52,11 @@ Settings come from ``settings.json``, see ``settings.json.template`` for an exam Pads will be fetched every checkInterval * seconds whereas the normal checkInterval runs every checkInterval * milliseconds. If pads are deleted they are also deleted from the reverse proxy so it can be reassigned to another backend. + +## Database support + +- SQLite (default, file-based, no setup required) - specified by dbSettings.filename = "db/etherpad-proxy.db" +- Postgres - specified by dbSettings.postgresConnstr - e.g. postgres://user:password@localhost:5432/etherpad_proxy_db?sslmode=disable + # License Apache 2 diff --git a/admin.go b/admin.go index e4cb157..4c9d2ef 100644 --- a/admin.go +++ b/admin.go @@ -2,18 +2,20 @@ package main import ( "context" + "net/http" + + "github.com/ether/etherpad-proxy/databases/interfaces" "github.com/ether/etherpad-proxy/ui" "go.uber.org/zap" - "net/http" ) type AdminPanel struct { - DB *DB + DB interfaces.IDB logger *zap.SugaredLogger } func (a *AdminPanel) ServeHTTP(w http.ResponseWriter, _ *http.Request) { - padIDMap, err := a.DB.getAllPads() + padIDMap, err := a.DB.GetAllPads() if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return diff --git a/databases/database.go b/databases/database.go new file mode 100644 index 0000000..bc28ccf --- /dev/null +++ b/databases/database.go @@ -0,0 +1,36 @@ +package databases + +import ( + "fmt" + + "github.com/ether/etherpad-proxy/databases/interfaces" + "github.com/ether/etherpad-proxy/databases/postgres" + "github.com/ether/etherpad-proxy/databases/sqlite" + "github.com/ether/etherpad-proxy/models" +) + +type DBType string + +const ( + DBTypeSQLite DBType = "sqlite" + DBTypePostgres DBType = "postgres" +) + +func CreateNewDatabase(settings models.Settings) (interfaces.IDB, error) { + var dbType = DBTypePostgres + if settings.DBSettings.Filename != "" { + dbType = DBTypeSQLite + } + + if settings.DBSettings.Connstr != "" { + dbType = DBTypePostgres + } + + switch dbType { + case DBTypePostgres: + return postgres.NewPostgresDB(settings.DBSettings.Connstr) + case DBTypeSQLite: + return sqlite.NewSQLiteDB(settings.DBSettings.Filename) + } + return nil, fmt.Errorf("unknown database type: %s", dbType) +} diff --git a/databases/interfaces/iDB.go b/databases/interfaces/iDB.go new file mode 100644 index 0000000..eaab14d --- /dev/null +++ b/databases/interfaces/iDB.go @@ -0,0 +1,13 @@ +package interfaces + +import "github.com/ether/etherpad-proxy/models" + +type IDB interface { + Close() error + Get(id string) (*models.DBBackend, error) + CleanUpPads(padIds []string, padPrefix string) error + RecordClash(id string, data string) error + Set(id string, backend models.DBBackend) error + GetAllPads() (map[string]string, error) + GetClashByPadID(id string) ([]string, error) +} diff --git a/databases/postgres/postgres_db.go b/databases/postgres/postgres_db.go new file mode 100644 index 0000000..3d4bec0 --- /dev/null +++ b/databases/postgres/postgres_db.go @@ -0,0 +1,135 @@ +package postgres + +import ( + "database/sql" + + sq "github.com/Masterminds/squirrel" + "github.com/ether/etherpad-proxy/databases/interfaces" + "github.com/ether/etherpad-proxy/models" +) + +type Postgres struct { + Conn *sql.DB +} + +var _ interfaces.IDB = (*Postgres)(nil) + +func NewPostgresDB(connstr string) (*Postgres, error) { + conn, err := sql.Open("postgres", connstr) + if err != nil { + return nil, err + } + + db := &Postgres{ + Conn: conn, + } + + if _, err = db.Conn.Exec("CREATE TABLE IF NOT EXISTS pad (id TEXT, backend TEXT, PRIMARY KEY (id))"); err != nil { + return nil, err + } + + if _, err = db.Conn.Exec("CREATE TABLE IF NOT EXISTS clashes (id TEXT, data TEXT, PRIMARY KEY (id, data))"); err != nil { + return nil, err + } + + return db, nil +} + +func (db *Postgres) Close() error { + return db.Conn.Close() +} + +func (db *Postgres) Get(id string) (*models.DBBackend, error) { + var data string + var sqlGet, args, err = sq.Select("backend").From("pad").Where(sq.Eq{"id": id}).ToSql() + if err != nil { + return nil, err + } + err = db.Conn.QueryRow(sqlGet, args...).Scan(&data) + if err != nil { + return nil, err + } + + var actualData = models.DBBackend{ + Backend: data, + } + + return &actualData, nil +} + +func (db *Postgres) CleanUpPads(padIds []string, padPrefix string) error { + sqlDelete, args, err := sq.Delete("pad").Where(sq.And{sq.NotEq{"id": padIds}, + sq.Like{"backend": padPrefix}}).ToSql() + if err != nil { + return err + } + + _, err = db.Conn.Exec(sqlDelete, args...) + return err +} + +func (db *Postgres) RecordClash(id string, data string) error { + _, err := db.Conn.Exec("INSERT OR REPLACE INTO clashes (id, data) VALUES (?, ?)", id, data) + if err != nil { + return err + } + return nil +} + +func (db *Postgres) Set(id string, dbModel models.DBBackend) error { + + _, err := db.Conn.Exec("INSERT OR REPLACE INTO pad (id, backend) VALUES (?, ?)", id, dbModel.Backend) + if err != nil { + return err + } + return nil +} + +func (db *Postgres) GetAllPads() (map[string]string, error) { + var padIDMap = make(map[string]string) + var sqlGet, args, err = sq.Select("id, backend").From("pad").ToSql() + if err != nil { + return nil, err + } + rows, err := db.Conn.Query(sqlGet, args...) + if err != nil { + return nil, err + } + defer rows.Close() + + for rows.Next() { + var padID string + var backend string + if err := rows.Scan(&padID, &backend); err != nil { + return nil, err + } + padIDMap[padID] = backend + } + + return padIDMap, nil +} + +func (db *Postgres) GetClashByPadID(padId string) ([]string, error) { + var sqlGet, args, err = sq.Select("data").From("clashes").Where(sq.Eq{"id": padId}).ToSql() + + if err != nil { + return nil, err + } + + rows, err := db.Conn.Query(sqlGet, args...) + if err != nil { + return nil, err + } + + defer rows.Close() + var data = make([]string, 0) + for rows.Next() { + var clash string + if err := rows.Scan(&clash); err != nil { + return nil, err + } + data = append(data, clash) + } + + return data, nil +} diff --git a/db.go b/databases/sqlite/sqlite_db.go similarity index 90% rename from db.go rename to databases/sqlite/sqlite_db.go index 2d0ab87..e0f694f 100644 --- a/db.go +++ b/databases/sqlite/sqlite_db.go @@ -1,7 +1,9 @@ -package main +package sqlite import ( "database/sql" + + "github.com/ether/etherpad-proxy/databases/interfaces" "github.com/ether/etherpad-proxy/models" _ "modernc.org/sqlite" ) @@ -12,7 +14,7 @@ type DB struct { Conn *sql.DB } -func NewDB(filename string) (*DB, error) { +func NewSQLiteDB(filename string) (*DB, error) { conn, err := sql.Open("sqlite", filename) if err != nil { return nil, err @@ -86,7 +88,7 @@ func (db *DB) Set(id string, dbModel models.DBBackend) error { return nil } -func (db *DB) getAllPads() (map[string]string, error) { +func (db *DB) GetAllPads() (map[string]string, error) { var padIDMap = make(map[string]string) var sqlGet, args, err = sq.Select("id, backend").From("pad").ToSql() if err != nil { @@ -110,7 +112,7 @@ func (db *DB) getAllPads() (map[string]string, error) { return padIDMap, nil } -func (db *DB) getClashByPadID(padId string) ([]string, error) { +func (db *DB) GetClashByPadID(padId string) ([]string, error) { var sqlGet, args, err = sq.Select("data").From("clashes").Where(sq.Eq{"id": padId}).ToSql() if err != nil { @@ -134,3 +136,5 @@ func (db *DB) getClashByPadID(padId string) ([]string, error) { return data, nil } + +var _ interfaces.IDB = (*DB)(nil) diff --git a/go.mod b/go.mod index b4e6f7a..89138aa 100644 --- a/go.mod +++ b/go.mod @@ -17,6 +17,7 @@ require ( github.com/google/uuid v1.6.0 // indirect github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect + github.com/lib/pq v1.10.9 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/ncruces/go-strftime v0.1.9 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect diff --git a/go.sum b/go.sum index fd69980..c1a51c3 100644 --- a/go.sum +++ b/go.sum @@ -20,6 +20,8 @@ github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 h1:SOEGU9fKiNWd/HOJuq github.com/lann/builder v0.0.0-20180802200727-47ae307949d0/go.mod h1:dXGbAdH5GtBTC4WfIxhKZfyBF/HBFgRZSWwZ9g/He9o= github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 h1:P6pPBnrTSX3DEVR4fDembhRWSsG5rVo6hYhAB/ADZrk= github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0/go.mod h1:vmVJ0l/dxyfGW6FmdpVm2joNMFikkuWg0EoCKLGUMNw= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= diff --git a/models/Settings.go b/models/Settings.go index 7330382..e41305a 100644 --- a/models/Settings.go +++ b/models/Settings.go @@ -10,6 +10,7 @@ type Settings struct { type DBSettings struct { Filename string `json:"filename"` + Connstr string `json:"postgresConnstr"` } type Backend struct { diff --git a/proxyHandler.go b/proxyHandler.go index 4ec0aa2..6478cea 100644 --- a/proxyHandler.go +++ b/proxyHandler.go @@ -4,10 +4,6 @@ import ( "context" "database/sql" "errors" - "github.com/PuerkitoBio/goquery" - "github.com/ether/etherpad-proxy/models" - "github.com/ether/etherpad-proxy/ui" - "go.uber.org/zap" "log" "net/http" "net/http/httputil" @@ -15,6 +11,12 @@ import ( "strconv" "strings" "time" + + "github.com/PuerkitoBio/goquery" + "github.com/ether/etherpad-proxy/databases/interfaces" + "github.com/ether/etherpad-proxy/models" + "github.com/ether/etherpad-proxy/ui" + "go.uber.org/zap" ) import "math/rand/v2" import _ "github.com/ether/etherpad-proxy/ui" @@ -22,7 +24,7 @@ import _ "github.com/ether/etherpad-proxy/ui" type ProxyHandler struct { p map[string]httputil.ReverseProxy logger *zap.SugaredLogger - db DB + db interfaces.IDB } type StaticResource struct { @@ -126,7 +128,7 @@ func (ph *ProxyHandler) createRoute(padId *string, r *http.Request) (*httputil.R } if errors.Is(err, sql.ErrNoRows) { // if no backend is stored for this pad, create a new connection - result, err := ph.db.getClashByPadID(*padId) + result, err := ph.db.GetClashByPadID(*padId) if err != nil && errors.Is(err, sql.ErrNoRows) || len(result) == 0 { AvailableBackends.Mutex.Lock() diff --git a/runtime.go b/runtime.go index 6d2da3f..a90ca1e 100644 --- a/runtime.go +++ b/runtime.go @@ -4,11 +4,6 @@ import ( "context" "encoding/base64" "encoding/json" - "github.com/ether/etherpad-proxy/models" - "go.uber.org/zap" - "golang.org/x/oauth2" - "golang.org/x/oauth2/clientcredentials" - _ "golang.org/x/oauth2/clientcredentials" "io" "net/http" "net/http/httputil" @@ -16,6 +11,14 @@ import ( "strconv" "sync" "time" + + "github.com/ether/etherpad-proxy/databases" + "github.com/ether/etherpad-proxy/databases/interfaces" + "github.com/ether/etherpad-proxy/models" + "go.uber.org/zap" + "golang.org/x/oauth2" + "golang.org/x/oauth2/clientcredentials" + _ "golang.org/x/oauth2/clientcredentials" ) var AvailableBackends = models.AvailableBackends{ @@ -38,7 +41,7 @@ func checkAvailabilityLoop(settings models.Settings, _ *zap.SugaredLogger) { }() } -func cleanUpEtherpadsLoop(settings models.Settings, logger *zap.SugaredLogger, db DB) { +func cleanUpEtherpadsLoop(settings models.Settings, logger *zap.SugaredLogger, db interfaces.IDB) { var timerP = time.Duration(settings.CheckInterval) * time.Second var timerBefore = time.Duration(5) * time.Second go func() { @@ -50,7 +53,7 @@ func cleanUpEtherpadsLoop(settings models.Settings, logger *zap.SugaredLogger, d }() } -func cleanUpEtherpads(settings models.Settings, logger *zap.SugaredLogger, db DB) { +func cleanUpEtherpads(settings models.Settings, logger *zap.SugaredLogger, db interfaces.IDB) { AvailableBackends.Mutex.Lock() defer AvailableBackends.Mutex.Unlock() var mapOfPadsToBackends = make(map[string]string) @@ -149,14 +152,14 @@ func StartServer(settings models.Settings, logger *zap.SugaredLogger) { for key := range settings.Backends { backendIds = append(backendIds, key) } - db, err := NewDB(settings.DBSettings.Filename) + db, err := databases.CreateNewDatabase(settings) if err != nil { logger.Fatalf("Error opening database: %v", err) } proxies := make(map[string]httputil.ReverseProxy) checkAvailabilityLoop(settings, logger) - cleanUpEtherpadsLoop(settings, logger, *db) + cleanUpEtherpadsLoop(settings, logger, db) ScrapeJSFiles(settings) for key, backend := range settings.Backends { @@ -172,7 +175,7 @@ func StartServer(settings models.Settings, logger *zap.SugaredLogger) { handler := ProxyHandler{ p: proxies, logger: logger, - db: *db, + db: db, } http.HandleFunc("/", handler.ServeHTTP) diff --git a/settings.json.sqlite.template b/settings.json.sqlite.template new file mode 100644 index 0000000..ef40c4f --- /dev/null +++ b/settings.json.sqlite.template @@ -0,0 +1,27 @@ +{ + "port": 9000, + "backends" : { + "backend1": { + "host": "localhost", + "port": 9001, + "clientId": "admin_client", + "clientSecret": "admin", + "tokenURL": "http://localhost:9001/oidc/token" + }, + "backend2": { + "host": "localhost", + "port": 9002, + "username": "admin", + "password": "admin", + }, + "backend3": { + "host": "localhost", + "port": 9003 + } + }, + "maxPadsPerInstance": 5, + "checkInterval": 1000, + "dbSettings": { + "filename": "./db/dirty.db", + } +} diff --git a/settings.json.template b/settings.json.template index 3379970..ef40c4f 100644 --- a/settings.json.template +++ b/settings.json.template @@ -22,6 +22,6 @@ "maxPadsPerInstance": 5, "checkInterval": 1000, "dbSettings": { - "filename": "./db/dirty.db" + "filename": "./db/dirty.db", } }