From d605f434903d780921d0c7a45e8c81f52f15b9cd Mon Sep 17 00:00:00 2001 From: unknown Date: Sun, 6 Apr 2025 10:56:22 +0700 Subject: [PATCH 01/32] feat: direct write --- example/dump/main.go | 4 +- example/source/main.go | 4 +- go.mod | 2 +- mysqldump.go | 183 +++++++++++++++++++++-------------------- 4 files changed, 98 insertions(+), 95 deletions(-) diff --git a/example/dump/main.go b/example/dump/main.go index a3bb22a..51071fc 100644 --- a/example/dump/main.go +++ b/example/dump/main.go @@ -3,12 +3,12 @@ package main import ( "os" - "github.com/jarvanstack/mysqldump" + "github.com/notyusta/mysqldump" ) func main() { - dsn := "root:rootpasswd@tcp(localhost:3306)/dbname?charset=utf8mb4&parseTime=true&loc=Asia%2FShanghai" + dsn := "root:rootpasswd@tcp(localhost:3306)/dbname?charset=utf8mb4&parseTime=true&loc=Asia%2FJakarta" f, _ := os.Create("dump.sql") diff --git a/example/source/main.go b/example/source/main.go index c460e1c..032d13c 100644 --- a/example/source/main.go +++ b/example/source/main.go @@ -3,12 +3,12 @@ package main import ( "os" - "github.com/jarvanstack/mysqldump" + "github.com/notyusta/mysqldump" ) func main() { - dns := "root:rootpasswd@tcp(localhost:3306)/dbname?charset=utf8mb4&parseTime=true&loc=Asia%2FShanghai" + dns := "root:rootpasswd@tcp(localhost:3306)/dbname?charset=utf8mb4&parseTime=true&loc=Asia%2FJakarta" f, _ := os.Open("dump.sql") _ = mysqldump.Source( diff --git a/go.mod b/go.mod index fa0a6f3..8a7240b 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module github.com/jarvanstack/mysqldump +module github.com/notyusta/mysqldump go 1.18 diff --git a/mysqldump.go b/mysqldump.go index a26d21d..d106e37 100644 --- a/mysqldump.go +++ b/mysqldump.go @@ -247,13 +247,14 @@ func writeTableData(db *sql.DB, table string, buf *bufio.Writer) error { log.Printf("[error] %v \n", err) return err } + columnTypes, err := lineRows.ColumnTypes() if err != nil { log.Printf("[error] %v \n", err) return err } - var values [][]interface{} + // updated to write directly to prevent large mem for lineRows.Next() { row := make([]interface{}, len(columns)) rowPointers := make([]interface{}, len(columns)) @@ -265,99 +266,101 @@ func writeTableData(db *sql.DB, table string, buf *bufio.Writer) error { log.Printf("[error] %v \n", err) return err } - values = append(values, row) + + writeTableColumnData(buf, table, row, columnTypes) } - for _, row := range values { - ssql := "INSERT INTO `" + table + "` VALUES (" - - for i, col := range row { - if col == nil { - ssql += "NULL" - } else { - Type := columnTypes[i].DatabaseTypeName() - // 去除 UNSIGNED 和空格 - Type = strings.Replace(Type, "UNSIGNED", "", -1) - Type = strings.Replace(Type, " ", "", -1) - switch Type { - case "TINYINT", "SMALLINT", "MEDIUMINT", "INT", "INTEGER", "BIGINT": - if bs, ok := col.([]byte); ok { - ssql += string(bs) - } else { - ssql += fmt.Sprintf("%d", col) - } - case "FLOAT", "DOUBLE": - if bs, ok := col.([]byte); ok { - ssql += string(bs) - } else { - ssql += fmt.Sprintf("%f", col) - } - case "DECIMAL", "DEC": - ssql += fmt.Sprintf("%s", col) - - case "DATE": - t, ok := col.(time.Time) - if !ok { - log.Println("DATE 类型转换错误") - return err - } - ssql += fmt.Sprintf("'%s'", t.Format("2006-01-02")) - case "DATETIME": - t, ok := col.(time.Time) - if !ok { - log.Println("DATETIME 类型转换错误") - return err - } - ssql += fmt.Sprintf("'%s'", t.Format("2006-01-02 15:04:05")) - case "TIMESTAMP": - t, ok := col.(time.Time) - if !ok { - log.Println("TIMESTAMP 类型转换错误") - return err - } - ssql += fmt.Sprintf("'%s'", t.Format("2006-01-02 15:04:05")) - case "TIME": - t, ok := col.([]byte) - if !ok { - log.Println("TIME 类型转换错误") - return err - } - ssql += fmt.Sprintf("'%s'", string(t)) - case "YEAR": - t, ok := col.([]byte) - if !ok { - log.Println("YEAR 类型转换错误") - return err - } - ssql += string(t) - case "CHAR", "VARCHAR", "TINYTEXT", "TEXT", "MEDIUMTEXT", "LONGTEXT": - ssql += fmt.Sprintf("'%s'", strings.Replace(fmt.Sprintf("%s", col), "'", "''", -1)) - case "BIT", "BINARY", "VARBINARY", "TINYBLOB", "BLOB", "MEDIUMBLOB", "LONGBLOB": - ssql += fmt.Sprintf("0x%X", col) - case "ENUM", "SET": - ssql += fmt.Sprintf("'%s'", col) - case "BOOL", "BOOLEAN": - if col.(bool) { - ssql += "true" - } else { - ssql += "false" - } - case "JSON": - ssql += fmt.Sprintf("'%s'", col) - default: - // unsupported type - log.Printf("unsupported type: %s", Type) - return fmt.Errorf("unsupported type: %s", Type) + _, _ = buf.WriteString("\n\n") + return nil +} + +func writeTableColumnData(buf *bufio.Writer, table string, row []interface{}, columnTypes []*sql.ColumnType) (err error) { + ssql := "INSERT INTO `" + table + "` VALUES (" + + for i, col := range row { + if col == nil { + ssql += "NULL" + } else { + Type := columnTypes[i].DatabaseTypeName() + // 去除 UNSIGNED 和空格 + Type = strings.Replace(Type, "UNSIGNED", "", -1) + Type = strings.Replace(Type, " ", "", -1) + switch Type { + case "TINYINT", "SMALLINT", "MEDIUMINT", "INT", "INTEGER", "BIGINT": + if bs, ok := col.([]byte); ok { + ssql += string(bs) + } else { + ssql += fmt.Sprintf("%d", col) } - } - if i < len(row)-1 { - ssql += "," + case "FLOAT", "DOUBLE": + if bs, ok := col.([]byte); ok { + ssql += string(bs) + } else { + ssql += fmt.Sprintf("%f", col) + } + case "DECIMAL", "DEC": + ssql += fmt.Sprintf("%s", col) + + case "DATE": + t, ok := col.(time.Time) + if !ok { + log.Println("DATE 类型转换错误") + return err + } + ssql += fmt.Sprintf("'%s'", t.Format("2006-01-02")) + case "DATETIME": + t, ok := col.(time.Time) + if !ok { + log.Println("DATETIME 类型转换错误") + return err + } + ssql += fmt.Sprintf("'%s'", t.Format("2006-01-02 15:04:05")) + case "TIMESTAMP": + t, ok := col.(time.Time) + if !ok { + log.Println("TIMESTAMP 类型转换错误") + return err + } + ssql += fmt.Sprintf("'%s'", t.Format("2006-01-02 15:04:05")) + case "TIME": + t, ok := col.([]byte) + if !ok { + log.Println("TIME 类型转换错误") + return err + } + ssql += fmt.Sprintf("'%s'", string(t)) + case "YEAR": + t, ok := col.([]byte) + if !ok { + log.Println("YEAR 类型转换错误") + return err + } + ssql += string(t) + case "CHAR", "VARCHAR", "TINYTEXT", "TEXT", "MEDIUMTEXT", "LONGTEXT": + ssql += fmt.Sprintf("'%s'", strings.Replace(fmt.Sprintf("%s", col), "'", "''", -1)) + case "BIT", "BINARY", "VARBINARY", "TINYBLOB", "BLOB", "MEDIUMBLOB", "LONGBLOB": + ssql += fmt.Sprintf("0x%X", col) + case "ENUM", "SET": + ssql += fmt.Sprintf("'%s'", col) + case "BOOL", "BOOLEAN": + if col.(bool) { + ssql += "true" + } else { + ssql += "false" + } + case "JSON": + ssql += fmt.Sprintf("'%s'", col) + default: + // unsupported type + log.Printf("unsupported type: %s", Type) + return fmt.Errorf("unsupported type: %s", Type) } } - ssql += ");\n" - _, _ = buf.WriteString(ssql) + if i < len(row)-1 { + ssql += "," + } } - - _, _ = buf.WriteString("\n\n") - return nil + ssql += ");\n" + _, _ = buf.WriteString(ssql) + return } From a18c222dafacc0420395dba1d8380672d683679d Mon Sep 17 00:00:00 2001 From: unknown Date: Sun, 6 Apr 2025 11:00:43 +0700 Subject: [PATCH 02/32] feat: update package name --- example/dump/main.go | 2 +- example/source/main.go | 2 +- go.mod | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/example/dump/main.go b/example/dump/main.go index 51071fc..aa18773 100644 --- a/example/dump/main.go +++ b/example/dump/main.go @@ -3,7 +3,7 @@ package main import ( "os" - "github.com/notyusta/mysqldump" + "github.com/NotYusta/mysqldump" ) func main() { diff --git a/example/source/main.go b/example/source/main.go index 032d13c..348a01e 100644 --- a/example/source/main.go +++ b/example/source/main.go @@ -3,7 +3,7 @@ package main import ( "os" - "github.com/notyusta/mysqldump" + "github.com/NotYusta/mysqldump" ) func main() { diff --git a/go.mod b/go.mod index 8a7240b..d706e6b 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module github.com/notyusta/mysqldump +module github.com/NotYusta/mysqldump go 1.18 From ba4867414ef247ac713550e83bf0fcf527b19dbf Mon Sep 17 00:00:00 2001 From: unknown Date: Sun, 6 Apr 2025 11:15:36 +0700 Subject: [PATCH 03/32] feat: update db option --- example/dump/main.go | 11 +++++++++-- example/source/main.go | 12 ++++++++++-- mysqldump.go | 16 ++-------------- source.go | 20 ++------------------ util.go | 21 --------------------- 5 files changed, 23 insertions(+), 57 deletions(-) delete mode 100644 util.go diff --git a/example/dump/main.go b/example/dump/main.go index aa18773..a4d52b3 100644 --- a/example/dump/main.go +++ b/example/dump/main.go @@ -1,19 +1,26 @@ package main import ( + "database/sql" + "log" "os" "github.com/NotYusta/mysqldump" ) func main() { - dsn := "root:rootpasswd@tcp(localhost:3306)/dbname?charset=utf8mb4&parseTime=true&loc=Asia%2FJakarta" + db, err := sql.Open("mysql", dsn) + if err != nil { + log.Printf("[error] %v \n", err) + return + } + defer db.Close() f, _ := os.Create("dump.sql") _ = mysqldump.Dump( - dsn, // DSN + db, // DSN mysqldump.WithDropTable(), // Option: Delete table before create (Default: Not delete table) mysqldump.WithData(), // Option: Dump Data (Default: Only dump table schema) mysqldump.WithTables("test"), // Option: Dump Tables (Default: All tables) diff --git a/example/source/main.go b/example/source/main.go index 348a01e..153c0a1 100644 --- a/example/source/main.go +++ b/example/source/main.go @@ -1,6 +1,8 @@ package main import ( + "database/sql" + "log" "os" "github.com/NotYusta/mysqldump" @@ -8,11 +10,17 @@ import ( func main() { - dns := "root:rootpasswd@tcp(localhost:3306)/dbname?charset=utf8mb4&parseTime=true&loc=Asia%2FJakarta" + dsn := "root:rootpasswd@tcp(localhost:3306)/dbname?charset=utf8mb4&parseTime=true&loc=Asia%2FJakarta" + db, err := sql.Open("mysql", dsn) + if err != nil { + log.Printf("[error] %v \n", err) + return + } + f, _ := os.Open("dump.sql") _ = mysqldump.Source( - dns, + db, f, mysqldump.WithMergeInsert(1000), // Option: Merge insert 1000 (Default: Not merge insert) mysqldump.WithDebug(), // Option: Print execute sql (Default: Not print execute sql) diff --git a/mysqldump.go b/mysqldump.go index d106e37..1acacb4 100644 --- a/mysqldump.go +++ b/mysqldump.go @@ -70,7 +70,7 @@ func WithWriter(writer io.Writer) DumpOption { } } -func Dump(dsn string, opts ...DumpOption) error { +func Dump(db *sql.DB, opts ...DumpOption) error { // 打印开始 start := time.Now() log.Printf("[info] [dump] start at %s\n", start.Format("2006-01-02 15:04:05")) @@ -109,20 +109,8 @@ func Dump(dsn string, opts ...DumpOption) error { _, _ = buf.WriteString("\n\n") // 连接数据库 - db, err := sql.Open("mysql", dsn) - if err != nil { - log.Printf("[error] %v \n", err) - return err - } - defer db.Close() - // 1. 获取数据库 - dbName, err := GetDBNameFromDSN(dsn) - if err != nil { - log.Printf("[error] %v \n", err) - return err - } - _, err = db.Exec(fmt.Sprintf("USE `%s`", dbName)) + _, err = db.Exec("USE `mysql`") if err != nil { log.Printf("[error] %v \n", err) return err diff --git a/source.go b/source.go index 25eea72..2f85de7 100644 --- a/source.go +++ b/source.go @@ -4,7 +4,6 @@ import ( "bufio" "database/sql" "errors" - "fmt" "io" "log" "strings" @@ -66,7 +65,7 @@ func (db *dbWrapper) Exec(query string, args ...interface{}) (sql.Result, error) // Source 加载 // 禁止 golangci-lint 检查 // nolint: gocyclo -func Source(dsn string, reader io.Reader, opts ...SourceOption) error { +func Source(db *sql.DB, reader io.Reader, opts ...SourceOption) error { // 打印开始 start := time.Now() log.Printf("[info] [source] start at %s\n", start.Format("2006-01-02 15:04:05")) @@ -77,31 +76,16 @@ func Source(dsn string, reader io.Reader, opts ...SourceOption) error { }() var err error - var db *sql.DB var o sourceOption for _, opt := range opts { opt(&o) } - dbName, err := GetDBNameFromDSN(dsn) - if err != nil { - log.Printf("[error] %v\n", err) - return err - } - - // Open database - db, err = sql.Open("mysql", dsn) - if err != nil { - log.Printf("[error] %v\n", err) - return err - } - defer db.Close() - // DB Wrapper dbWrapper := newDBWrapper(db, o.dryRun, o.debug) // Use database - _, err = dbWrapper.Exec(fmt.Sprintf("USE %s;", dbName)) + _, err = dbWrapper.Exec("USE `mysql`") if err != nil { log.Printf("[error] %v\n", err) return err diff --git a/util.go b/util.go deleted file mode 100644 index ad084b4..0000000 --- a/util.go +++ /dev/null @@ -1,21 +0,0 @@ -package mysqldump - -import ( - "fmt" - "strings" -) - -//从dsn中提取出数据库名称,并将其作为结果返回。 -//如果无法解析出数据库名称,将返回一个错误。 - -func GetDBNameFromDSN(dsn string) (string, error) { - ss1 := strings.Split(dsn, "/") - if len(ss1) == 2 { - ss2 := strings.Split(ss1[1], "?") - if len(ss2) == 2 { - return ss2[0], nil - } - } - - return "", fmt.Errorf("dsn error: %s", dsn) -} From 3f6e1d79329303c723582f36707a0bec48521002 Mon Sep 17 00:00:00 2001 From: unknown Date: Sun, 6 Apr 2025 11:27:23 +0700 Subject: [PATCH 04/32] feat: remove logs --- mysqldump.go | 27 +-------------------------- source.go | 24 ------------------------ 2 files changed, 1 insertion(+), 50 deletions(-) diff --git a/mysqldump.go b/mysqldump.go index 1acacb4..c7c0dbe 100644 --- a/mysqldump.go +++ b/mysqldump.go @@ -5,7 +5,6 @@ import ( "database/sql" "fmt" "io" - "log" "os" "strings" "time" @@ -13,10 +12,7 @@ import ( _ "github.com/go-sql-driver/mysql" ) -func init() { - // 打印 日志 行数 - log.SetFlags(log.Lshortfile | log.LstdFlags) -} +func init() {} type dumpOption struct { // 导出表数据 @@ -73,13 +69,7 @@ func WithWriter(writer io.Writer) DumpOption { func Dump(db *sql.DB, opts ...DumpOption) error { // 打印开始 start := time.Now() - log.Printf("[info] [dump] start at %s\n", start.Format("2006-01-02 15:04:05")) // 打印结束 - defer func() { - end := time.Now() - log.Printf("[info] [dump] end at %s, cost %s\n", end.Format("2006-01-02 15:04:05"), end.Sub(start)) - }() - var err error var o dumpOption @@ -112,7 +102,6 @@ func Dump(db *sql.DB, opts ...DumpOption) error { _, err = db.Exec("USE `mysql`") if err != nil { - log.Printf("[error] %v \n", err) return err } @@ -121,7 +110,6 @@ func Dump(db *sql.DB, opts ...DumpOption) error { if o.isAllTable { tmp, err := getAllTables(db) if err != nil { - log.Printf("[error] %v \n", err) return err } tables = tmp @@ -139,7 +127,6 @@ func Dump(db *sql.DB, opts ...DumpOption) error { // 导出表结构 err = writeTableStruct(db, table, buf) if err != nil { - log.Printf("[error] %v \n", err) return err } @@ -147,7 +134,6 @@ func Dump(db *sql.DB, opts ...DumpOption) error { if o.isData { err = writeTableData(db, table, buf) if err != nil { - log.Printf("[error] %v \n", err) return err } } @@ -202,7 +188,6 @@ func writeTableStruct(db *sql.DB, table string, buf *bufio.Writer) error { createTableSQL, err := getCreateTableSQL(db, table) if err != nil { - log.Printf("[error] %v \n", err) return err } _, _ = buf.WriteString(createTableSQL) @@ -224,7 +209,6 @@ func writeTableData(db *sql.DB, table string, buf *bufio.Writer) error { lineRows, err := db.Query(fmt.Sprintf("SELECT * FROM `%s`", table)) if err != nil { - log.Printf("[error] %v \n", err) return err } defer lineRows.Close() @@ -232,13 +216,11 @@ func writeTableData(db *sql.DB, table string, buf *bufio.Writer) error { var columns []string columns, err = lineRows.Columns() if err != nil { - log.Printf("[error] %v \n", err) return err } columnTypes, err := lineRows.ColumnTypes() if err != nil { - log.Printf("[error] %v \n", err) return err } @@ -251,7 +233,6 @@ func writeTableData(db *sql.DB, table string, buf *bufio.Writer) error { } err = lineRows.Scan(rowPointers...) if err != nil { - log.Printf("[error] %v \n", err) return err } @@ -292,35 +273,30 @@ func writeTableColumnData(buf *bufio.Writer, table string, row []interface{}, co case "DATE": t, ok := col.(time.Time) if !ok { - log.Println("DATE 类型转换错误") return err } ssql += fmt.Sprintf("'%s'", t.Format("2006-01-02")) case "DATETIME": t, ok := col.(time.Time) if !ok { - log.Println("DATETIME 类型转换错误") return err } ssql += fmt.Sprintf("'%s'", t.Format("2006-01-02 15:04:05")) case "TIMESTAMP": t, ok := col.(time.Time) if !ok { - log.Println("TIMESTAMP 类型转换错误") return err } ssql += fmt.Sprintf("'%s'", t.Format("2006-01-02 15:04:05")) case "TIME": t, ok := col.([]byte) if !ok { - log.Println("TIME 类型转换错误") return err } ssql += fmt.Sprintf("'%s'", string(t)) case "YEAR": t, ok := col.([]byte) if !ok { - log.Println("YEAR 类型转换错误") return err } ssql += string(t) @@ -340,7 +316,6 @@ func writeTableColumnData(buf *bufio.Writer, table string, row []interface{}, co ssql += fmt.Sprintf("'%s'", col) default: // unsupported type - log.Printf("unsupported type: %s", Type) return fmt.Errorf("unsupported type: %s", Type) } } diff --git a/source.go b/source.go index 2f85de7..ad8e51e 100644 --- a/source.go +++ b/source.go @@ -5,9 +5,7 @@ import ( "database/sql" "errors" "io" - "log" "strings" - "time" ) type sourceOption struct { @@ -52,10 +50,6 @@ func newDBWrapper(db *sql.DB, dryRun, debug bool) *dbWrapper { } func (db *dbWrapper) Exec(query string, args ...interface{}) (sql.Result, error) { - if db.debug { - log.Printf("[debug] [query]\n%s\n", query) - } - if db.dryRun { return nil, nil } @@ -67,14 +61,6 @@ func (db *dbWrapper) Exec(query string, args ...interface{}) (sql.Result, error) // nolint: gocyclo func Source(db *sql.DB, reader io.Reader, opts ...SourceOption) error { // 打印开始 - start := time.Now() - log.Printf("[info] [source] start at %s\n", start.Format("2006-01-02 15:04:05")) - // 打印结束 - defer func() { - end := time.Now() - log.Printf("[info] [source] end at %s, cost %s\n", end.Format("2006-01-02 15:04:05"), end.Sub(start)) - }() - var err error var o sourceOption for _, opt := range opts { @@ -87,7 +73,6 @@ func Source(db *sql.DB, reader io.Reader, opts ...SourceOption) error { // Use database _, err = dbWrapper.Exec("USE `mysql`") if err != nil { - log.Printf("[error] %v\n", err) return err } @@ -99,7 +84,6 @@ func Source(db *sql.DB, reader io.Reader, opts ...SourceOption) error { // 关闭事务 _, err = dbWrapper.Exec("SET autocommit=0;") if err != nil { - log.Printf("[error] %v\n", err) return err } @@ -109,7 +93,6 @@ func Source(db *sql.DB, reader io.Reader, opts ...SourceOption) error { if err == io.EOF { break } - log.Printf("[error] %v\n", err) return err } @@ -118,7 +101,6 @@ func Source(db *sql.DB, reader io.Reader, opts ...SourceOption) error { // 删除末尾的换行符 ssql = trim(ssql) if err != nil { - log.Printf("[error] [trim] %v\n", err) return err } @@ -132,14 +114,12 @@ func Source(db *sql.DB, reader io.Reader, opts ...SourceOption) error { if err == io.EOF { break } - log.Printf("[error] %v\n", err) return err } ssql2 := string(line) ssql2 = trim(ssql2) if err != nil { - log.Printf("[error] [trim] %v\n", err) return err } if strings.HasPrefix(ssql2, "INSERT INTO") { @@ -152,14 +132,12 @@ func Source(db *sql.DB, reader io.Reader, opts ...SourceOption) error { // 合并 INSERT ssql, err = mergeInsert(insertSQLs) if err != nil { - log.Printf("[error] [mergeInsert] %v\n", err) return err } } _, err = dbWrapper.Exec(ssql) if err != nil { - log.Printf("[error] %v\n", err) return err } } @@ -167,14 +145,12 @@ func Source(db *sql.DB, reader io.Reader, opts ...SourceOption) error { // 提交事务 _, err = dbWrapper.Exec("COMMIT;") if err != nil { - log.Printf("[error] %v\n", err) return err } // 开启事务 _, err = dbWrapper.Exec("SET autocommit=1;") if err != nil { - log.Printf("[error] %v\n", err) return err } From 743a0479beac185163fca7bbc0d9aa6978f2a9a4 Mon Sep 17 00:00:00 2001 From: unknown Date: Sun, 6 Apr 2025 11:37:46 +0700 Subject: [PATCH 05/32] feat: add db name --- example/dump/main.go | 1 + example/source/main.go | 1 + mysqldump.go | 6 ++---- source.go | 5 +++-- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/example/dump/main.go b/example/dump/main.go index a4d52b3..6deed5f 100644 --- a/example/dump/main.go +++ b/example/dump/main.go @@ -21,6 +21,7 @@ func main() { _ = mysqldump.Dump( db, // DSN + "test", mysqldump.WithDropTable(), // Option: Delete table before create (Default: Not delete table) mysqldump.WithData(), // Option: Dump Data (Default: Only dump table schema) mysqldump.WithTables("test"), // Option: Dump Tables (Default: All tables) diff --git a/example/source/main.go b/example/source/main.go index 153c0a1..259562f 100644 --- a/example/source/main.go +++ b/example/source/main.go @@ -21,6 +21,7 @@ func main() { _ = mysqldump.Source( db, + "test", f, mysqldump.WithMergeInsert(1000), // Option: Merge insert 1000 (Default: Not merge insert) mysqldump.WithDebug(), // Option: Print execute sql (Default: Not print execute sql) diff --git a/mysqldump.go b/mysqldump.go index c7c0dbe..7a11bde 100644 --- a/mysqldump.go +++ b/mysqldump.go @@ -66,7 +66,7 @@ func WithWriter(writer io.Writer) DumpOption { } } -func Dump(db *sql.DB, opts ...DumpOption) error { +func Dump(db *sql.DB, dbName string, opts ...DumpOption) error { // 打印开始 start := time.Now() // 打印结束 @@ -98,9 +98,7 @@ func Dump(db *sql.DB, opts ...DumpOption) error { _, _ = buf.WriteString("-- ----------------------------\n") _, _ = buf.WriteString("\n\n") - // 连接数据库 - - _, err = db.Exec("USE `mysql`") + _, err = db.Exec(fmt.Sprintf("USE `%s`", dbName)) if err != nil { return err } diff --git a/source.go b/source.go index ad8e51e..0be1a1f 100644 --- a/source.go +++ b/source.go @@ -4,6 +4,7 @@ import ( "bufio" "database/sql" "errors" + "fmt" "io" "strings" ) @@ -59,7 +60,7 @@ func (db *dbWrapper) Exec(query string, args ...interface{}) (sql.Result, error) // Source 加载 // 禁止 golangci-lint 检查 // nolint: gocyclo -func Source(db *sql.DB, reader io.Reader, opts ...SourceOption) error { +func Source(db *sql.DB, dbName string, reader io.Reader, opts ...SourceOption) error { // 打印开始 var err error var o sourceOption @@ -71,7 +72,7 @@ func Source(db *sql.DB, reader io.Reader, opts ...SourceOption) error { dbWrapper := newDBWrapper(db, o.dryRun, o.debug) // Use database - _, err = dbWrapper.Exec("USE `mysql`") + _, err = dbWrapper.Exec(fmt.Sprintf("USE `%s`", dbName)) if err != nil { return err } From 6adcf42622f13757af9e1bedd6694cff2d24f809 Mon Sep 17 00:00:00 2001 From: unknown Date: Sun, 6 Apr 2025 11:42:59 +0700 Subject: [PATCH 06/32] feat: update dump --- mysqldump.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mysqldump.go b/mysqldump.go index 7a11bde..537564e 100644 --- a/mysqldump.go +++ b/mysqldump.go @@ -234,7 +234,10 @@ func writeTableData(db *sql.DB, table string, buf *bufio.Writer) error { return err } - writeTableColumnData(buf, table, row, columnTypes) + err = writeTableColumnData(buf, table, row, columnTypes) + if err != nil { + return err + } } _, _ = buf.WriteString("\n\n") From ec6c52a2e79ff142d2af91d1a08ac60f2e33b436 Mon Sep 17 00:00:00 2001 From: unknown Date: Sun, 6 Apr 2025 22:49:00 +0700 Subject: [PATCH 07/32] feat: add log --- mysqldump.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mysqldump.go b/mysqldump.go index 537564e..f77c687 100644 --- a/mysqldump.go +++ b/mysqldump.go @@ -247,7 +247,9 @@ func writeTableData(db *sql.DB, table string, buf *bufio.Writer) error { func writeTableColumnData(buf *bufio.Writer, table string, row []interface{}, columnTypes []*sql.ColumnType) (err error) { ssql := "INSERT INTO `" + table + "` VALUES (" + fmt.Printf("writing for %s\n", table) for i, col := range row { + fmt.Printf("searching row [%d] for %s\n", i, row) if col == nil { ssql += "NULL" } else { From 6c8c77bda388e11b24752d3f9e444de7ece5c7ec Mon Sep 17 00:00:00 2001 From: unknown Date: Sun, 6 Apr 2025 22:50:59 +0700 Subject: [PATCH 08/32] feat: add log --- mysqldump.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mysqldump.go b/mysqldump.go index f77c687..d8db369 100644 --- a/mysqldump.go +++ b/mysqldump.go @@ -249,7 +249,7 @@ func writeTableColumnData(buf *bufio.Writer, table string, row []interface{}, co fmt.Printf("writing for %s\n", table) for i, col := range row { - fmt.Printf("searching row [%d] for %s\n", i, row) + fmt.Printf("searching row [%d] for %s\n", i, table) if col == nil { ssql += "NULL" } else { From f2f641fdb5cf24b6391be2992294a978639e8a0a Mon Sep 17 00:00:00 2001 From: unknown Date: Sun, 6 Apr 2025 22:56:15 +0700 Subject: [PATCH 09/32] feat: add log --- mysqldump.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mysqldump.go b/mysqldump.go index d8db369..254eccd 100644 --- a/mysqldump.go +++ b/mysqldump.go @@ -128,8 +128,10 @@ func Dump(db *sql.DB, dbName string, opts ...DumpOption) error { return err } - // 导出表数据 - if o.isData { + } + + if o.isData { + for _, table := range tables { err = writeTableData(db, table, buf) if err != nil { return err From 91a9543c89d658a1fd8c2c6c50a67917e4760008 Mon Sep 17 00:00:00 2001 From: unknown Date: Sun, 6 Apr 2025 23:01:14 +0700 Subject: [PATCH 10/32] feat: add log --- mysqldump.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mysqldump.go b/mysqldump.go index 254eccd..a8ac977 100644 --- a/mysqldump.go +++ b/mysqldump.go @@ -201,10 +201,13 @@ func writeTableStruct(db *sql.DB, table string, buf *bufio.Writer) error { // 禁止 golangci-lint 检查 // nolint: gocyclo func writeTableData(db *sql.DB, table string, buf *bufio.Writer) error { + var totalRow uint64 + row := db.QueryRow(fmt.Sprintf("SELECT COUNT(*) FROM `%s`", table)) + row.Scan(&totalRow) // 导出表数据 _, _ = buf.WriteString("-- ----------------------------\n") - _, _ = buf.WriteString(fmt.Sprintf("-- Records of %s\n", table)) + _, _ = buf.WriteString(fmt.Sprintf("-- Records of %s (%d Rows)\n", table, totalRow)) _, _ = buf.WriteString("-- ----------------------------\n") lineRows, err := db.Query(fmt.Sprintf("SELECT * FROM `%s`", table)) From 5ea67c29f511e2501cc9674d93051947be822e72 Mon Sep 17 00:00:00 2001 From: unknown Date: Sun, 6 Apr 2025 23:05:04 +0700 Subject: [PATCH 11/32] feat: add log --- mysqldump.go | 125 +++++++++------------------------------------------ 1 file changed, 22 insertions(+), 103 deletions(-) diff --git a/mysqldump.go b/mysqldump.go index a8ac977..c380453 100644 --- a/mysqldump.go +++ b/mysqldump.go @@ -210,128 +210,47 @@ func writeTableData(db *sql.DB, table string, buf *bufio.Writer) error { _, _ = buf.WriteString(fmt.Sprintf("-- Records of %s (%d Rows)\n", table, totalRow)) _, _ = buf.WriteString("-- ----------------------------\n") - lineRows, err := db.Query(fmt.Sprintf("SELECT * FROM `%s`", table)) + rows, err := db.Query(fmt.Sprintf("SELECT * FROM `%s`", table)) if err != nil { return err } - defer lineRows.Close() + defer rows.Close() var columns []string - columns, err = lineRows.Columns() + columns, err = rows.Columns() if err != nil { return err } - columnTypes, err := lineRows.ColumnTypes() - if err != nil { - return err - } + // updated to write directly to prevent large mem - for lineRows.Next() { - row := make([]interface{}, len(columns)) - rowPointers := make([]interface{}, len(columns)) - for i := range columns { - rowPointers[i] = &row[i] + for rows.Next() { + data := make([]*sql.NullString, len(columns)) + ptrs := make([]interface{}, len(columns)) + for i, _ := range data { + ptrs[i] = &data[i] } - err = lineRows.Scan(rowPointers...) - if err != nil { + + // Read data + if err := rows.Scan(ptrs...); err != nil { return err } - err = writeTableColumnData(buf, table, row, columnTypes) - if err != nil { - return err + dataStrings := make([]string, len(columns)) + + for key, value := range data { + if value != nil && value.Valid { + dataStrings[key] = "'" + value.String + "'" + } else { + dataStrings[key] = "null" + } } + + buf.WriteString("("+strings.Join(dataStrings, ",")+")") } _, _ = buf.WriteString("\n\n") return nil } -func writeTableColumnData(buf *bufio.Writer, table string, row []interface{}, columnTypes []*sql.ColumnType) (err error) { - ssql := "INSERT INTO `" + table + "` VALUES (" - - fmt.Printf("writing for %s\n", table) - for i, col := range row { - fmt.Printf("searching row [%d] for %s\n", i, table) - if col == nil { - ssql += "NULL" - } else { - Type := columnTypes[i].DatabaseTypeName() - // 去除 UNSIGNED 和空格 - Type = strings.Replace(Type, "UNSIGNED", "", -1) - Type = strings.Replace(Type, " ", "", -1) - switch Type { - case "TINYINT", "SMALLINT", "MEDIUMINT", "INT", "INTEGER", "BIGINT": - if bs, ok := col.([]byte); ok { - ssql += string(bs) - } else { - ssql += fmt.Sprintf("%d", col) - } - case "FLOAT", "DOUBLE": - if bs, ok := col.([]byte); ok { - ssql += string(bs) - } else { - ssql += fmt.Sprintf("%f", col) - } - case "DECIMAL", "DEC": - ssql += fmt.Sprintf("%s", col) - - case "DATE": - t, ok := col.(time.Time) - if !ok { - return err - } - ssql += fmt.Sprintf("'%s'", t.Format("2006-01-02")) - case "DATETIME": - t, ok := col.(time.Time) - if !ok { - return err - } - ssql += fmt.Sprintf("'%s'", t.Format("2006-01-02 15:04:05")) - case "TIMESTAMP": - t, ok := col.(time.Time) - if !ok { - return err - } - ssql += fmt.Sprintf("'%s'", t.Format("2006-01-02 15:04:05")) - case "TIME": - t, ok := col.([]byte) - if !ok { - return err - } - ssql += fmt.Sprintf("'%s'", string(t)) - case "YEAR": - t, ok := col.([]byte) - if !ok { - return err - } - ssql += string(t) - case "CHAR", "VARCHAR", "TINYTEXT", "TEXT", "MEDIUMTEXT", "LONGTEXT": - ssql += fmt.Sprintf("'%s'", strings.Replace(fmt.Sprintf("%s", col), "'", "''", -1)) - case "BIT", "BINARY", "VARBINARY", "TINYBLOB", "BLOB", "MEDIUMBLOB", "LONGBLOB": - ssql += fmt.Sprintf("0x%X", col) - case "ENUM", "SET": - ssql += fmt.Sprintf("'%s'", col) - case "BOOL", "BOOLEAN": - if col.(bool) { - ssql += "true" - } else { - ssql += "false" - } - case "JSON": - ssql += fmt.Sprintf("'%s'", col) - default: - // unsupported type - return fmt.Errorf("unsupported type: %s", Type) - } - } - if i < len(row)-1 { - ssql += "," - } - } - ssql += ");\n" - _, _ = buf.WriteString(ssql) - return -} From 086eeda70776da6a6040868b0b200851e89940c2 Mon Sep 17 00:00:00 2001 From: unknown Date: Sun, 6 Apr 2025 23:10:09 +0700 Subject: [PATCH 12/32] feat: add log --- mysqldump.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/mysqldump.go b/mysqldump.go index c380453..aa4713e 100644 --- a/mysqldump.go +++ b/mysqldump.go @@ -222,13 +222,14 @@ func writeTableData(db *sql.DB, table string, buf *bufio.Writer) error { return err } + // Generate the column names for the INSERT statement + columnNames := strings.Join(columns, ",") - - // updated to write directly to prevent large mem + // Write the data for each row for rows.Next() { data := make([]*sql.NullString, len(columns)) ptrs := make([]interface{}, len(columns)) - for i, _ := range data { + for i := range data { ptrs[i] = &data[i] } @@ -239,18 +240,19 @@ func writeTableData(db *sql.DB, table string, buf *bufio.Writer) error { dataStrings := make([]string, len(columns)) + // Prepare the values for key, value := range data { if value != nil && value.Valid { dataStrings[key] = "'" + value.String + "'" } else { - dataStrings[key] = "null" + dataStrings[key] = "NULL" } } - buf.WriteString("("+strings.Join(dataStrings, ",")+")") + // Insert statement with column names + buf.WriteString(fmt.Sprintf("INSERT INTO %s (%s) VALUES (%s);\n", table, columnNames, strings.Join(dataStrings, ","))) } _, _ = buf.WriteString("\n\n") return nil } - From 7d54cd822140441d629b435adb234c63292cecd9 Mon Sep 17 00:00:00 2001 From: unknown Date: Sun, 6 Apr 2025 23:13:25 +0700 Subject: [PATCH 13/32] feat: add log --- mysqldump.go | 76 +++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 75 insertions(+), 1 deletion(-) diff --git a/mysqldump.go b/mysqldump.go index aa4713e..aeb0245 100644 --- a/mysqldump.go +++ b/mysqldump.go @@ -161,6 +161,79 @@ func getCreateTableSQL(db *sql.DB, table string) (string, error) { return createTableSQL, nil } +func getForeignKeyRelationships(db *sql.DB, table string) ([]string, error) { + var foreignKeys []string + query := ` + SELECT TABLE_NAME + FROM INFORMATION_SCHEMA.KEY_COLUMN_USAGE + WHERE REFERENCED_TABLE_NAME = ?` + rows, err := db.Query(query, table) + if err != nil { + return nil, err + } + defer rows.Close() + + for rows.Next() { + var dependentTable string + if err := rows.Scan(&dependentTable); err != nil { + return nil, err + } + foreignKeys = append(foreignKeys, dependentTable) + } + return foreignKeys, nil +} + +// Sort tables based on foreign key dependencies +func sortTablesByDependency(db *sql.DB, tables []string) ([]string, error) { + var sortedTables []string + visited := make(map[string]bool) + var result []string + + // Use depth-first search (DFS) to traverse the tables in dependency order + var visit func(table string) error + visit = func(table string) error { + if visited[table] { + return nil + } + visited[table] = true + + // Find tables that the current table is referenced by + foreignKeys, err := getForeignKeyRelationships(db, table) + if err != nil { + return err + } + + // Visit tables that the current table depends on + for _, foreignKey := range foreignKeys { + if !visited[foreignKey] { + if err := visit(foreignKey); err != nil { + return err + } + } + } + + // After visiting all dependencies, add the current table + result = append(result, table) + return nil + } + + // Visit each table to ensure all dependencies are traversed + for _, table := range tables { + if !visited[table] { + if err := visit(table); err != nil { + return nil, err + } + } + } + + // Reverse the result list to ensure the correct order + for i := len(result) - 1; i >= 0; i-- { + sortedTables = append(sortedTables, result[i]) + } + + return sortedTables, nil +} + func getAllTables(db *sql.DB) ([]string, error) { var tables []string rows, err := db.Query("SHOW TABLES") @@ -177,7 +250,8 @@ func getAllTables(db *sql.DB) ([]string, error) { } tables = append(tables, table) } - return tables, nil + + return sortTablesByDependency(db, tables) } func writeTableStruct(db *sql.DB, table string, buf *bufio.Writer) error { From bbc9172d47760390ff6835e72b29104d68faac6c Mon Sep 17 00:00:00 2001 From: unknown Date: Sun, 6 Apr 2025 23:20:55 +0700 Subject: [PATCH 14/32] feat: add log --- mysqldump.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mysqldump.go b/mysqldump.go index aeb0245..27d2ad6 100644 --- a/mysqldump.go +++ b/mysqldump.go @@ -317,11 +317,13 @@ func writeTableData(db *sql.DB, table string, buf *bufio.Writer) error { // Prepare the values for key, value := range data { if value != nil && value.Valid { - dataStrings[key] = "'" + value.String + "'" + escaped := strings.ReplaceAll(value.String, "'", "''") + dataStrings[key] = "'" + escaped + "'" } else { dataStrings[key] = "NULL" } } + // Insert statement with column names buf.WriteString(fmt.Sprintf("INSERT INTO %s (%s) VALUES (%s);\n", table, columnNames, strings.Join(dataStrings, ","))) From 786a058b013c4713bb8e68e9db035d44e7a2035b Mon Sep 17 00:00:00 2001 From: unknown Date: Sun, 6 Apr 2025 23:27:58 +0700 Subject: [PATCH 15/32] feat: add log --- mysqldump.go | 52 ++++++++++++++++++++++++++++++---------------------- 1 file changed, 30 insertions(+), 22 deletions(-) diff --git a/mysqldump.go b/mysqldump.go index 27d2ad6..fc67b8b 100644 --- a/mysqldump.go +++ b/mysqldump.go @@ -299,36 +299,44 @@ func writeTableData(db *sql.DB, table string, buf *bufio.Writer) error { // Generate the column names for the INSERT statement columnNames := strings.Join(columns, ",") - // Write the data for each row - for rows.Next() { - data := make([]*sql.NullString, len(columns)) - ptrs := make([]interface{}, len(columns)) - for i := range data { - ptrs[i] = &data[i] - } + if totalRow > 0 { + buf.WriteString(fmt.Sprintf("INSERT INTO %s (%s) VALUES\n", table, columnNames)) + + rowIndex := 0 + for rows.Next() { + data := make([]*sql.NullString, len(columns)) + ptrs := make([]interface{}, len(columns)) + for i := range data { + ptrs[i] = &data[i] + } - // Read data - if err := rows.Scan(ptrs...); err != nil { - return err - } + // Read data + if err := rows.Scan(ptrs...); err != nil { + return err + } - dataStrings := make([]string, len(columns)) + dataStrings := make([]string, len(columns)) + for key, value := range data { + if value != nil && value.Valid { + escaped := strings.ReplaceAll(value.String, "'", "''") + dataStrings[key] = "'" + escaped + "'" + } else { + dataStrings[key] = "NULL" + } + } - // Prepare the values - for key, value := range data { - if value != nil && value.Valid { - escaped := strings.ReplaceAll(value.String, "'", "''") - dataStrings[key] = "'" + escaped + "'" - } else { - dataStrings[key] = "NULL" + if rowIndex > 0 { + buf.WriteString(",\n") // comma for previous row } + buf.WriteString("(" + strings.Join(dataStrings, ",") + ")") + rowIndex++ } - - // Insert statement with column names - buf.WriteString(fmt.Sprintf("INSERT INTO %s (%s) VALUES (%s);\n", table, columnNames, strings.Join(dataStrings, ","))) + buf.WriteString(";\n\n") // terminate the statement } + buf.WriteString(");\n") + _, _ = buf.WriteString("\n\n") return nil } From 4b1a57d0f2681f5781e53f0bb4217baacb07f6c1 Mon Sep 17 00:00:00 2001 From: unknown Date: Sun, 6 Apr 2025 23:29:31 +0700 Subject: [PATCH 16/32] feat: add log --- mysqldump.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/mysqldump.go b/mysqldump.go index fc67b8b..2b08155 100644 --- a/mysqldump.go +++ b/mysqldump.go @@ -335,8 +335,6 @@ func writeTableData(db *sql.DB, table string, buf *bufio.Writer) error { buf.WriteString(";\n\n") // terminate the statement } - buf.WriteString(");\n") - _, _ = buf.WriteString("\n\n") return nil } From 2fb0e1c0d75eaa3fa8053b33aa633b3974f01948 Mon Sep 17 00:00:00 2001 From: unknown Date: Sun, 6 Apr 2025 23:31:04 +0700 Subject: [PATCH 17/32] feat: add log --- mysqldump.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mysqldump.go b/mysqldump.go index 2b08155..a806aef 100644 --- a/mysqldump.go +++ b/mysqldump.go @@ -300,7 +300,7 @@ func writeTableData(db *sql.DB, table string, buf *bufio.Writer) error { columnNames := strings.Join(columns, ",") if totalRow > 0 { - buf.WriteString(fmt.Sprintf("INSERT INTO %s (%s) VALUES\n", table, columnNames)) + buf.WriteString(fmt.Sprintf("INSERT INTO %s (%s) VALUES ", table, columnNames)) rowIndex := 0 for rows.Next() { @@ -326,7 +326,7 @@ func writeTableData(db *sql.DB, table string, buf *bufio.Writer) error { } if rowIndex > 0 { - buf.WriteString(",\n") // comma for previous row + buf.WriteString(",") // comma for previous row } buf.WriteString("(" + strings.Join(dataStrings, ",") + ")") rowIndex++ From cb3dbc6ed85af420347c037d553ad47f8b1d1cb7 Mon Sep 17 00:00:00 2001 From: unknown Date: Sun, 6 Apr 2025 23:32:59 +0700 Subject: [PATCH 18/32] feat: remove log --- mysqldump.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mysqldump.go b/mysqldump.go index a806aef..9de0e61 100644 --- a/mysqldump.go +++ b/mysqldump.go @@ -326,7 +326,7 @@ func writeTableData(db *sql.DB, table string, buf *bufio.Writer) error { } if rowIndex > 0 { - buf.WriteString(",") // comma for previous row + buf.WriteString(",\n") // comma for previous row } buf.WriteString("(" + strings.Join(dataStrings, ",") + ")") rowIndex++ From 704b6dbe9a2d4108dd74224bd11a93f77ebf5607 Mon Sep 17 00:00:00 2001 From: unknown Date: Sun, 6 Apr 2025 23:37:21 +0700 Subject: [PATCH 19/32] feat: remove log --- mysqldump.go | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/mysqldump.go b/mysqldump.go index 9de0e61..1f6e6e5 100644 --- a/mysqldump.go +++ b/mysqldump.go @@ -130,9 +130,11 @@ func Dump(db *sql.DB, dbName string, opts ...DumpOption) error { } + allTotalRows := uint64(0) if o.isData { for _, table := range tables { - err = writeTableData(db, table, buf) + totalRows, err := writeTableData(db, table, buf) + allTotalRows += totalRows if err != nil { return err } @@ -143,7 +145,10 @@ func Dump(db *sql.DB, dbName string, opts ...DumpOption) error { _, _ = buf.WriteString("-- ----------------------------\n") _, _ = buf.WriteString("-- Dumped by mysqldump\n") + _, _ = buf.WriteString("-- Maintained by Yusta (https://github.com/NotYusta)\n") _, _ = buf.WriteString("-- Cost Time: " + time.Since(start).String() + "\n") + _, _ = buf.WriteString("-- Table Counts: " + fmt.Sprintf("%d", len(tables)) + "\n") + _, _ = buf.WriteString("-- Table Rows: " + fmt.Sprintf("%d", allTotalRows) + "\n") _, _ = buf.WriteString("-- ----------------------------\n") buf.Flush() @@ -274,7 +279,7 @@ func writeTableStruct(db *sql.DB, table string, buf *bufio.Writer) error { // 禁止 golangci-lint 检查 // nolint: gocyclo -func writeTableData(db *sql.DB, table string, buf *bufio.Writer) error { +func writeTableData(db *sql.DB, table string, buf *bufio.Writer) (uint64, error) { var totalRow uint64 row := db.QueryRow(fmt.Sprintf("SELECT COUNT(*) FROM `%s`", table)) row.Scan(&totalRow) @@ -286,14 +291,14 @@ func writeTableData(db *sql.DB, table string, buf *bufio.Writer) error { rows, err := db.Query(fmt.Sprintf("SELECT * FROM `%s`", table)) if err != nil { - return err + return totalRow, err } defer rows.Close() var columns []string columns, err = rows.Columns() if err != nil { - return err + return totalRow, err } // Generate the column names for the INSERT statement @@ -312,7 +317,7 @@ func writeTableData(db *sql.DB, table string, buf *bufio.Writer) error { // Read data if err := rows.Scan(ptrs...); err != nil { - return err + return totalRow, err } dataStrings := make([]string, len(columns)) @@ -336,5 +341,5 @@ func writeTableData(db *sql.DB, table string, buf *bufio.Writer) error { } _, _ = buf.WriteString("\n\n") - return nil + return totalRow, nil } From e0fb8e2869ea219f0b6f32ff6046e8eb30081caa Mon Sep 17 00:00:00 2001 From: unknown Date: Sun, 6 Apr 2025 23:39:26 +0700 Subject: [PATCH 20/32] feat: add format --- mysqldump.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mysqldump.go b/mysqldump.go index 1f6e6e5..19a1de1 100644 --- a/mysqldump.go +++ b/mysqldump.go @@ -95,6 +95,7 @@ func Dump(db *sql.DB, dbName string, opts ...DumpOption) error { _, _ = buf.WriteString("-- ----------------------------\n") _, _ = buf.WriteString("-- MySQL Database Dump\n") _, _ = buf.WriteString("-- Start Time: " + start.Format("2006-01-02 15:04:05") + "\n") + _, _ = buf.WriteString("-- Database Name: " + dbName + "\n") _, _ = buf.WriteString("-- ----------------------------\n") _, _ = buf.WriteString("\n\n") @@ -147,6 +148,7 @@ func Dump(db *sql.DB, dbName string, opts ...DumpOption) error { _, _ = buf.WriteString("-- Dumped by mysqldump\n") _, _ = buf.WriteString("-- Maintained by Yusta (https://github.com/NotYusta)\n") _, _ = buf.WriteString("-- Cost Time: " + time.Since(start).String() + "\n") + _, _ = buf.WriteString("-- Completed Time: " + time.Now().Format("2006-01-02 15:04:05") + "\n") _, _ = buf.WriteString("-- Table Counts: " + fmt.Sprintf("%d", len(tables)) + "\n") _, _ = buf.WriteString("-- Table Rows: " + fmt.Sprintf("%d", allTotalRows) + "\n") _, _ = buf.WriteString("-- ----------------------------\n") From b49a9c165d3f484d77d5cb4ae80f0d7dc77edcaa Mon Sep 17 00:00:00 2001 From: unknown Date: Sun, 6 Apr 2025 23:39:52 +0700 Subject: [PATCH 21/32] feat: update time --- mysqldump.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mysqldump.go b/mysqldump.go index 19a1de1..cdbfb3a 100644 --- a/mysqldump.go +++ b/mysqldump.go @@ -148,7 +148,7 @@ func Dump(db *sql.DB, dbName string, opts ...DumpOption) error { _, _ = buf.WriteString("-- Dumped by mysqldump\n") _, _ = buf.WriteString("-- Maintained by Yusta (https://github.com/NotYusta)\n") _, _ = buf.WriteString("-- Cost Time: " + time.Since(start).String() + "\n") - _, _ = buf.WriteString("-- Completed Time: " + time.Now().Format("2006-01-02 15:04:05") + "\n") + _, _ = buf.WriteString("-- Complete Time: " + time.Now().Format("2006-01-02 15:04:05") + "\n") _, _ = buf.WriteString("-- Table Counts: " + fmt.Sprintf("%d", len(tables)) + "\n") _, _ = buf.WriteString("-- Table Rows: " + fmt.Sprintf("%d", allTotalRows) + "\n") _, _ = buf.WriteString("-- ----------------------------\n") From a3575e1a38faa036cea8576f6b0073e5a8bdc145 Mon Sep 17 00:00:00 2001 From: unknown Date: Mon, 7 Apr 2025 00:01:10 +0700 Subject: [PATCH 22/32] feat: update foreign key --- mysqldump.go | 79 ++-------------------------------------------------- 1 file changed, 3 insertions(+), 76 deletions(-) diff --git a/mysqldump.go b/mysqldump.go index cdbfb3a..3673997 100644 --- a/mysqldump.go +++ b/mysqldump.go @@ -98,6 +98,7 @@ func Dump(db *sql.DB, dbName string, opts ...DumpOption) error { _, _ = buf.WriteString("-- Database Name: " + dbName + "\n") _, _ = buf.WriteString("-- ----------------------------\n") _, _ = buf.WriteString("\n\n") + _, _ = buf.WriteString("SET FOREIGN_KEY_CHECKS=0;\n") _, err = db.Exec(fmt.Sprintf("USE `%s`", dbName)) if err != nil { @@ -143,7 +144,7 @@ func Dump(db *sql.DB, dbName string, opts ...DumpOption) error { } // 导出每个表的结构和数据 - + _, _ = buf.WriteString("SET FOREIGN_KEY_CHECKS=1;\n") _, _ = buf.WriteString("-- ----------------------------\n") _, _ = buf.WriteString("-- Dumped by mysqldump\n") _, _ = buf.WriteString("-- Maintained by Yusta (https://github.com/NotYusta)\n") @@ -168,79 +169,6 @@ func getCreateTableSQL(db *sql.DB, table string) (string, error) { return createTableSQL, nil } -func getForeignKeyRelationships(db *sql.DB, table string) ([]string, error) { - var foreignKeys []string - query := ` - SELECT TABLE_NAME - FROM INFORMATION_SCHEMA.KEY_COLUMN_USAGE - WHERE REFERENCED_TABLE_NAME = ?` - rows, err := db.Query(query, table) - if err != nil { - return nil, err - } - defer rows.Close() - - for rows.Next() { - var dependentTable string - if err := rows.Scan(&dependentTable); err != nil { - return nil, err - } - foreignKeys = append(foreignKeys, dependentTable) - } - return foreignKeys, nil -} - -// Sort tables based on foreign key dependencies -func sortTablesByDependency(db *sql.DB, tables []string) ([]string, error) { - var sortedTables []string - visited := make(map[string]bool) - var result []string - - // Use depth-first search (DFS) to traverse the tables in dependency order - var visit func(table string) error - visit = func(table string) error { - if visited[table] { - return nil - } - visited[table] = true - - // Find tables that the current table is referenced by - foreignKeys, err := getForeignKeyRelationships(db, table) - if err != nil { - return err - } - - // Visit tables that the current table depends on - for _, foreignKey := range foreignKeys { - if !visited[foreignKey] { - if err := visit(foreignKey); err != nil { - return err - } - } - } - - // After visiting all dependencies, add the current table - result = append(result, table) - return nil - } - - // Visit each table to ensure all dependencies are traversed - for _, table := range tables { - if !visited[table] { - if err := visit(table); err != nil { - return nil, err - } - } - } - - // Reverse the result list to ensure the correct order - for i := len(result) - 1; i >= 0; i-- { - sortedTables = append(sortedTables, result[i]) - } - - return sortedTables, nil -} - func getAllTables(db *sql.DB) ([]string, error) { var tables []string rows, err := db.Query("SHOW TABLES") @@ -258,7 +186,7 @@ func getAllTables(db *sql.DB) ([]string, error) { tables = append(tables, table) } - return sortTablesByDependency(db, tables) + return tables, nil } func writeTableStruct(db *sql.DB, table string, buf *bufio.Writer) error { @@ -266,7 +194,6 @@ func writeTableStruct(db *sql.DB, table string, buf *bufio.Writer) error { _, _ = buf.WriteString("-- ----------------------------\n") _, _ = buf.WriteString(fmt.Sprintf("-- Table structure for %s\n", table)) _, _ = buf.WriteString("-- ----------------------------\n") - createTableSQL, err := getCreateTableSQL(db, table) if err != nil { return err From 369eaf8767e628567ed6eadf3b48a85a6753d795 Mon Sep 17 00:00:00 2001 From: unknown Date: Mon, 7 Apr 2025 00:07:19 +0700 Subject: [PATCH 23/32] feat: quote columns --- mysqldump.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/mysqldump.go b/mysqldump.go index 3673997..67b9dc7 100644 --- a/mysqldump.go +++ b/mysqldump.go @@ -231,8 +231,11 @@ func writeTableData(db *sql.DB, table string, buf *bufio.Writer) (uint64, error) } // Generate the column names for the INSERT statement - columnNames := strings.Join(columns, ",") - + quotedColumns := make([]string, len(columns)) + for i, col := range columns { + quotedColumns[i] = "`" + col + "`" + } + columnNames := strings.Join(quotedColumns, ",") if totalRow > 0 { buf.WriteString(fmt.Sprintf("INSERT INTO %s (%s) VALUES ", table, columnNames)) From bb2befa5ea233349630d06087a4c50b7dd55281b Mon Sep 17 00:00:00 2001 From: unknown Date: Mon, 7 Apr 2025 00:15:50 +0700 Subject: [PATCH 24/32] feat: update format --- mysqldump.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mysqldump.go b/mysqldump.go index 67b9dc7..6e0af84 100644 --- a/mysqldump.go +++ b/mysqldump.go @@ -237,7 +237,7 @@ func writeTableData(db *sql.DB, table string, buf *bufio.Writer) (uint64, error) } columnNames := strings.Join(quotedColumns, ",") if totalRow > 0 { - buf.WriteString(fmt.Sprintf("INSERT INTO %s (%s) VALUES ", table, columnNames)) + buf.WriteString(fmt.Sprintf("INSERT INTO `%s` (%s) VALUES ", table, columnNames)) rowIndex := 0 for rows.Next() { From 2022b85838a379b15e501023e631dc79c412b9df Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 25 Apr 2025 09:36:17 +0700 Subject: [PATCH 25/32] feat: master --- mysqldump.go | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/mysqldump.go b/mysqldump.go index 6e0af84..7f48fa8 100644 --- a/mysqldump.go +++ b/mysqldump.go @@ -230,16 +230,7 @@ func writeTableData(db *sql.DB, table string, buf *bufio.Writer) (uint64, error) return totalRow, err } - // Generate the column names for the INSERT statement - quotedColumns := make([]string, len(columns)) - for i, col := range columns { - quotedColumns[i] = "`" + col + "`" - } - columnNames := strings.Join(quotedColumns, ",") if totalRow > 0 { - buf.WriteString(fmt.Sprintf("INSERT INTO `%s` (%s) VALUES ", table, columnNames)) - - rowIndex := 0 for rows.Next() { data := make([]*sql.NullString, len(columns)) ptrs := make([]interface{}, len(columns)) @@ -262,11 +253,7 @@ func writeTableData(db *sql.DB, table string, buf *bufio.Writer) (uint64, error) } } - if rowIndex > 0 { - buf.WriteString(",\n") // comma for previous row - } - buf.WriteString("(" + strings.Join(dataStrings, ",") + ")") - rowIndex++ + buf.WriteString(fmt.Sprintf("INSERT INTO `%s` VALUES %s;", table, "("+strings.Join(dataStrings, ",")+")")) } buf.WriteString(";\n\n") // terminate the statement From f70e68e16936a2545102fbfb8cd70c74f3d5c0b2 Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 25 Apr 2025 10:03:01 +0700 Subject: [PATCH 26/32] feat: update query --- mysqldump.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/mysqldump.go b/mysqldump.go index 7f48fa8..aea416f 100644 --- a/mysqldump.go +++ b/mysqldump.go @@ -253,10 +253,8 @@ func writeTableData(db *sql.DB, table string, buf *bufio.Writer) (uint64, error) } } - buf.WriteString(fmt.Sprintf("INSERT INTO `%s` VALUES %s;", table, "("+strings.Join(dataStrings, ",")+")")) + buf.WriteString(fmt.Sprintf("INSERT INTO `%s` VALUES %s;\n", table, "("+strings.Join(dataStrings, ",")+")")) } - - buf.WriteString(";\n\n") // terminate the statement } _, _ = buf.WriteString("\n\n") From bc973facece0709bcb5f6209916bfbe5d4e161be Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 25 Apr 2025 10:16:18 +0700 Subject: [PATCH 27/32] feat: reformat query --- mysqldump.go | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/mysqldump.go b/mysqldump.go index aea416f..89efac8 100644 --- a/mysqldump.go +++ b/mysqldump.go @@ -97,8 +97,7 @@ func Dump(db *sql.DB, dbName string, opts ...DumpOption) error { _, _ = buf.WriteString("-- Start Time: " + start.Format("2006-01-02 15:04:05") + "\n") _, _ = buf.WriteString("-- Database Name: " + dbName + "\n") _, _ = buf.WriteString("-- ----------------------------\n") - _, _ = buf.WriteString("\n\n") - _, _ = buf.WriteString("SET FOREIGN_KEY_CHECKS=0;\n") + _, _ = buf.WriteString("SET FOREIGN_KEY_CHECKS=0;\n\n") _, err = db.Exec(fmt.Sprintf("USE `%s`", dbName)) if err != nil { @@ -198,11 +197,7 @@ func writeTableStruct(db *sql.DB, table string, buf *bufio.Writer) error { if err != nil { return err } - _, _ = buf.WriteString(createTableSQL) - _, _ = buf.WriteString(";") - - _, _ = buf.WriteString("\n\n") - _, _ = buf.WriteString("\n\n") + _, _ = buf.WriteString(fmt.Sprintf("%s;\n\n", createTableSQL)) return nil } @@ -257,6 +252,6 @@ func writeTableData(db *sql.DB, table string, buf *bufio.Writer) (uint64, error) } } - _, _ = buf.WriteString("\n\n") + _, _ = buf.WriteString("\n") return totalRow, nil } From d1d235530732574ece2c20946d6a16a932d54000 Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 25 Apr 2025 10:19:51 +0700 Subject: [PATCH 28/32] feat: reformat query --- mysqldump.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/mysqldump.go b/mysqldump.go index 89efac8..0d77b4c 100644 --- a/mysqldump.go +++ b/mysqldump.go @@ -225,6 +225,13 @@ func writeTableData(db *sql.DB, table string, buf *bufio.Writer) (uint64, error) return totalRow, err } + quotedColumns := make([]string, len(columns)) + for i, col := range columns { + quotedColumns[i] = "`" + col + "`" + } + + columnNames := strings.Join(quotedColumns, ",") + if totalRow > 0 { for rows.Next() { data := make([]*sql.NullString, len(columns)) @@ -248,7 +255,7 @@ func writeTableData(db *sql.DB, table string, buf *bufio.Writer) (uint64, error) } } - buf.WriteString(fmt.Sprintf("INSERT INTO `%s` VALUES %s;\n", table, "("+strings.Join(dataStrings, ",")+")")) + buf.WriteString(fmt.Sprintf("INSERT INTO `%s` (%s) VALUES %s;\n", table, columnNames, "("+strings.Join(dataStrings, ",")+")")) } } From 028c6fe73a99507e65b0140587689789c940698a Mon Sep 17 00:00:00 2001 From: notyusta Date: Sat, 8 Nov 2025 12:46:00 +0700 Subject: [PATCH 29/32] feat: update batch size --- mysqldump.go | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/mysqldump.go b/mysqldump.go index 0d77b4c..09a0981 100644 --- a/mysqldump.go +++ b/mysqldump.go @@ -233,6 +233,10 @@ func writeTableData(db *sql.DB, table string, buf *bufio.Writer) (uint64, error) columnNames := strings.Join(quotedColumns, ",") if totalRow > 0 { + const batchSize = 1024 + var batch []string + count := 0 + for rows.Next() { data := make([]*sql.NullString, len(columns)) ptrs := make([]interface{}, len(columns)) @@ -240,7 +244,6 @@ func writeTableData(db *sql.DB, table string, buf *bufio.Writer) (uint64, error) ptrs[i] = &data[i] } - // Read data if err := rows.Scan(ptrs...); err != nil { return totalRow, err } @@ -255,8 +258,21 @@ func writeTableData(db *sql.DB, table string, buf *bufio.Writer) (uint64, error) } } - buf.WriteString(fmt.Sprintf("INSERT INTO `%s` (%s) VALUES %s;\n", table, columnNames, "("+strings.Join(dataStrings, ",")+")")) + batch = append(batch, "("+strings.Join(dataStrings, ",")+")") + count++ + + // Flush batch when full + if count%batchSize == 0 { + buf.WriteString(fmt.Sprintf("INSERT INTO `%s` (%s) VALUES %s;\n", table, columnNames, strings.Join(batch, ","))) + batch = batch[:0] // clear batch + } + } + + // Flush remaining rows + if len(batch) > 0 { + buf.WriteString(fmt.Sprintf("INSERT INTO `%s` (%s) VALUES %s;\n", table, columnNames, strings.Join(batch, ","))) } + } _, _ = buf.WriteString("\n") From f076314efdfd6f328598d19eb672f4ae23fc336b Mon Sep 17 00:00:00 2001 From: notyusta Date: Sat, 8 Nov 2025 12:59:58 +0700 Subject: [PATCH 30/32] feat: update query --- mysqldump.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/mysqldump.go b/mysqldump.go index 09a0981..5cd72c3 100644 --- a/mysqldump.go +++ b/mysqldump.go @@ -219,8 +219,7 @@ func writeTableData(db *sql.DB, table string, buf *bufio.Writer) (uint64, error) } defer rows.Close() - var columns []string - columns, err = rows.Columns() + columns, err := rows.Columns() if err != nil { return totalRow, err } @@ -229,7 +228,6 @@ func writeTableData(db *sql.DB, table string, buf *bufio.Writer) (uint64, error) for i, col := range columns { quotedColumns[i] = "`" + col + "`" } - columnNames := strings.Join(quotedColumns, ",") if totalRow > 0 { @@ -251,24 +249,27 @@ func writeTableData(db *sql.DB, table string, buf *bufio.Writer) (uint64, error) dataStrings := make([]string, len(columns)) for key, value := range data { if value != nil && value.Valid { - escaped := strings.ReplaceAll(value.String, "'", "''") + escaped := strings.ReplaceAll(value.String, "\\", "\\\\") + escaped = strings.ReplaceAll(escaped, "'", "''") dataStrings[key] = "'" + escaped + "'" } else { dataStrings[key] = "NULL" } } + if len(dataStrings) != len(columns) { + return totalRow, fmt.Errorf("row has mismatched column count") + } + batch = append(batch, "("+strings.Join(dataStrings, ",")+")") count++ - // Flush batch when full if count%batchSize == 0 { buf.WriteString(fmt.Sprintf("INSERT INTO `%s` (%s) VALUES %s;\n", table, columnNames, strings.Join(batch, ","))) - batch = batch[:0] // clear batch + batch = batch[:0] } } - // Flush remaining rows if len(batch) > 0 { buf.WriteString(fmt.Sprintf("INSERT INTO `%s` (%s) VALUES %s;\n", table, columnNames, strings.Join(batch, ","))) } From dd9c2ed0b2bac8a7b49a00b2ca047522fae7e13d Mon Sep 17 00:00:00 2001 From: notyusta Date: Sat, 8 Nov 2025 13:02:20 +0700 Subject: [PATCH 31/32] feat: update path --- example/dump/main.go | 4 ++-- example/source/main.go | 2 +- go.mod | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/example/dump/main.go b/example/dump/main.go index 6deed5f..4720f5a 100644 --- a/example/dump/main.go +++ b/example/dump/main.go @@ -5,7 +5,7 @@ import ( "log" "os" - "github.com/NotYusta/mysqldump" + "github.com/notyusta/mysqldump" ) func main() { @@ -20,7 +20,7 @@ func main() { f, _ := os.Create("dump.sql") _ = mysqldump.Dump( - db, // DSN + db, // DSN "test", mysqldump.WithDropTable(), // Option: Delete table before create (Default: Not delete table) mysqldump.WithData(), // Option: Dump Data (Default: Only dump table schema) diff --git a/example/source/main.go b/example/source/main.go index 259562f..2b4e40c 100644 --- a/example/source/main.go +++ b/example/source/main.go @@ -5,7 +5,7 @@ import ( "log" "os" - "github.com/NotYusta/mysqldump" + "github.com/notyusta/mysqldump" ) func main() { diff --git a/go.mod b/go.mod index d706e6b..8a7240b 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module github.com/NotYusta/mysqldump +module github.com/notyusta/mysqldump go 1.18 From 2e56758c8459fc0c7a25e2716deb5b7e7ddd89c6 Mon Sep 17 00:00:00 2001 From: notyusta Date: Sat, 8 Nov 2025 13:48:00 +0700 Subject: [PATCH 32/32] feat: update headers --- mysqldump.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/mysqldump.go b/mysqldump.go index 5cd72c3..b646639 100644 --- a/mysqldump.go +++ b/mysqldump.go @@ -97,7 +97,16 @@ func Dump(db *sql.DB, dbName string, opts ...DumpOption) error { _, _ = buf.WriteString("-- Start Time: " + start.Format("2006-01-02 15:04:05") + "\n") _, _ = buf.WriteString("-- Database Name: " + dbName + "\n") _, _ = buf.WriteString("-- ----------------------------\n") - _, _ = buf.WriteString("SET FOREIGN_KEY_CHECKS=0;\n\n") + + // Session / export settings + _, _ = buf.WriteString("/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;\n") + _, _ = buf.WriteString("/*!40101 SET NAMES utf8 */;\n") + _, _ = buf.WriteString("/*!50503 SET NAMES utf8mb4 */;\n") + _, _ = buf.WriteString("/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;\n") + _, _ = buf.WriteString("/*!40103 SET TIME_ZONE='+00:00' */;\n") + _, _ = buf.WriteString("/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;\n") + _, _ = buf.WriteString("/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;\n") + _, _ = buf.WriteString("/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;\n") _, err = db.Exec(fmt.Sprintf("USE `%s`", dbName)) if err != nil {