From 09653f9c0672e14981374144fa16833afbe7eb37 Mon Sep 17 00:00:00 2001 From: ssongliu <73214554+ssongliu@users.noreply.github.com> Date: Tue, 1 Aug 2023 11:20:16 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20=E4=BF=AE=E6=94=B9=E8=BF=9C=E7=A8=8B?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E5=BA=93=E4=BE=9D=E8=B5=96=20(#1794)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/utils/mysql/client/remote.go | 22 +- backend/utils/mysql/helper/dump.go | 327 ++++++++++++++++++ backend/utils/mysql/helper/source.go | 228 ++++++++++++ .../src/views/database/mysql/conn/index.vue | 1 + .../src/views/database/mysql/create/index.vue | 25 +- .../src/views/database/mysql/delete/index.vue | 2 +- frontend/src/views/database/mysql/index.vue | 1 + .../views/database/mysql/password/index.vue | 1 - .../database/mysql/setting/slow-log/index.vue | 2 +- go.mod | 1 - go.sum | 2 - 11 files changed, 577 insertions(+), 35 deletions(-) create mode 100644 backend/utils/mysql/helper/dump.go create mode 100644 backend/utils/mysql/helper/source.go diff --git a/backend/utils/mysql/client/remote.go b/backend/utils/mysql/client/remote.go index 4e3315b06..4307fd0e4 100644 --- a/backend/utils/mysql/client/remote.go +++ b/backend/utils/mysql/client/remote.go @@ -5,7 +5,7 @@ import ( "database/sql" "fmt" "os" - "path" + "os/exec" "strings" "time" @@ -14,8 +14,7 @@ import ( "github.com/1Panel-dev/1Panel/backend/global" "github.com/1Panel-dev/1Panel/backend/utils/common" "github.com/1Panel-dev/1Panel/backend/utils/files" - - "github.com/jarvanstack/mysqldump" + "github.com/1Panel-dev/1Panel/backend/utils/mysql/helper" ) type Remote struct { @@ -222,23 +221,26 @@ func (r *Remote) Backup(info BackupInfo) error { f, _ := os.OpenFile(fileNameItem, os.O_RDWR|os.O_CREATE, 0755) defer f.Close() - if err := mysqldump.Dump(dns, mysqldump.WithData(), mysqldump.WithDropTable(), mysqldump.WithWriter(f)); err != nil { + if err := helper.Dump(dns, helper.WithData(), helper.WithDropTable(), helper.WithWriter(f)); err != nil { return err } - if err := fileOp.Compress([]string{fileNameItem}, info.TargetDir, info.FileName, files.Gz); err != nil { - return err + gzipCmd := exec.Command("gzip", fileNameItem) + stdout, err := gzipCmd.CombinedOutput() + if err != nil { + return fmt.Errorf("gzip file %s failed, stdout: %v, err: %v", strings.TrimSuffix(info.FileName, ".gz"), string(stdout), err) } return nil } func (r *Remote) Recover(info RecoverInfo) error { - fileOp := files.NewFileOp() fileName := info.SourceFile if strings.HasSuffix(info.SourceFile, ".sql.gz") { fileName = strings.TrimSuffix(info.SourceFile, ".gz") - if err := fileOp.Decompress(info.SourceFile, path.Dir(fileName), files.Gz); err != nil { - return err + gzipCmd := exec.Command("gunzip", info.SourceFile) + stdout, err := gzipCmd.CombinedOutput() + if err != nil { + return fmt.Errorf("gunzip file %s failed, stdout: %v, err: %v", info.SourceFile, string(stdout), err) } } dns := fmt.Sprintf("%s:%s@tcp(%s:%v)/%s?charset=%s&parseTime=true&loc=Asia%sShanghai", r.User, r.Password, r.Address, r.Port, info.Name, info.Format, "%2F") @@ -247,7 +249,7 @@ func (r *Remote) Recover(info RecoverInfo) error { return err } defer f.Close() - if err := mysqldump.Source(dns, f, mysqldump.WithMergeInsert(1000)); err != nil { + if err := helper.Source(dns, f, helper.WithMergeInsert(1000)); err != nil { return err } return nil diff --git a/backend/utils/mysql/helper/dump.go b/backend/utils/mysql/helper/dump.go new file mode 100644 index 000000000..5c30008f7 --- /dev/null +++ b/backend/utils/mysql/helper/dump.go @@ -0,0 +1,327 @@ +package helper + +import ( + "bufio" + "database/sql" + "fmt" + "io" + "os" + "strings" + "time" + + "github.com/1Panel-dev/1Panel/backend/global" + _ "github.com/go-sql-driver/mysql" +) + +func init() {} + +type dumpOption struct { + isData bool + + tables []string + isAllTable bool + isDropTable bool + writer io.Writer +} + +type DumpOption func(*dumpOption) + +func WithDropTable() DumpOption { + return func(option *dumpOption) { + option.isDropTable = true + } +} + +func WithData() DumpOption { + return func(option *dumpOption) { + option.isData = true + } +} + +func WithTables(tables ...string) DumpOption { + return func(option *dumpOption) { + option.tables = tables + } +} + +func WithAllTable() DumpOption { + return func(option *dumpOption) { + option.isAllTable = true + } +} + +func WithWriter(writer io.Writer) DumpOption { + return func(option *dumpOption) { + option.writer = writer + } +} + +func Dump(dns string, opts ...DumpOption) error { + start := time.Now() + global.LOG.Infof("dump start at %s\n", start.Format("2006-01-02 15:04:05")) + defer func() { + end := time.Now() + global.LOG.Infof("dump end at %s, cost %s\n", end.Format("2006-01-02 15:04:05"), end.Sub(start)) + }() + + var err error + + var o dumpOption + + for _, opt := range opts { + opt(&o) + } + + if len(o.tables) == 0 { + o.isAllTable = true + } + + if o.writer == nil { + o.writer = os.Stdout + } + + buf := bufio.NewWriter(o.writer) + defer buf.Flush() + + _, _ = buf.WriteString("-- ----------------------------\n") + _, _ = buf.WriteString("-- MySQL Database Dump\n") + _, _ = buf.WriteString("-- Start Time: " + start.Format("2006-01-02 15:04:05") + "\n") + _, _ = buf.WriteString("-- ----------------------------\n") + _, _ = buf.WriteString("\n\n") + + db, err := sql.Open("mysql", dns) + if err != nil { + global.LOG.Errorf("open mysql db failed, err: %v", err) + return err + } + defer db.Close() + + dbName, err := getDBNameFromDNS(dns) + if err != nil { + global.LOG.Errorf("get db name from dns failed, err: %v", err) + return err + } + _, err = db.Exec(fmt.Sprintf("USE `%s`", dbName)) + if err != nil { + global.LOG.Errorf("exec `use %s` failed, err: %v", dbName, err) + return err + } + + var tables []string + if o.isAllTable { + tmp, err := getAllTables(db) + if err != nil { + global.LOG.Errorf("get all tables failed, err: %v", err) + return err + } + tables = tmp + } else { + tables = o.tables + } + + for _, table := range tables { + if o.isDropTable { + _, _ = buf.WriteString(fmt.Sprintf("DROP TABLE IF EXISTS `%s`;\n", table)) + } + + err = writeTableStruct(db, table, buf) + if err != nil { + global.LOG.Errorf("write table struct failed, err: %v", err) + return err + } + + if o.isData { + err = writeTableData(db, table, buf) + if err != nil { + global.LOG.Errorf("write table data failed, err: %v", err) + return err + } + } + } + + _, _ = buf.WriteString("-- ----------------------------\n") + _, _ = buf.WriteString("-- Dumped by mysqldump\n") + _, _ = buf.WriteString("-- Cost Time: " + time.Since(start).String() + "\n") + _, _ = buf.WriteString("-- ----------------------------\n") + _ = buf.Flush() + + return nil +} + +func getCreateTableSQL(db *sql.DB, table string) (string, error) { + var createTableSQL string + err := db.QueryRow(fmt.Sprintf("SHOW CREATE TABLE `%s`", table)).Scan(&table, &createTableSQL) + if err != nil { + return "", err + } + createTableSQL = strings.Replace(createTableSQL, "CREATE TABLE", "CREATE TABLE IF NOT EXISTS", 1) + return createTableSQL, nil +} + +func getAllTables(db *sql.DB) ([]string, error) { + var tables []string + rows, err := db.Query("SHOW TABLES") + if err != nil { + return nil, err + } + defer rows.Close() + + for rows.Next() { + var table string + err = rows.Scan(&table) + if err != nil { + return nil, err + } + tables = append(tables, table) + } + return tables, nil +} + +func writeTableStruct(db *sql.DB, table string, buf *bufio.Writer) error { + _, _ = buf.WriteString("-- ----------------------------\n") + _, _ = buf.WriteString(fmt.Sprintf("-- Table structure for %s\n", table)) + _, _ = buf.WriteString("-- ----------------------------\n") + + createTableSQL, err := getCreateTableSQL(db, table) + if err != nil { + global.LOG.Errorf("get create table sql failed, err: %v", err) + return err + } + _, _ = buf.WriteString(createTableSQL) + _, _ = buf.WriteString(";") + + _, _ = buf.WriteString("\n\n") + _, _ = buf.WriteString("\n\n") + return nil +} + +func writeTableData(db *sql.DB, table string, buf *bufio.Writer) error { + _, _ = buf.WriteString("-- ----------------------------\n") + _, _ = buf.WriteString(fmt.Sprintf("-- Records of %s\n", table)) + _, _ = buf.WriteString("-- ----------------------------\n") + + lineRows, err := db.Query(fmt.Sprintf("SELECT * FROM `%s`", table)) + if err != nil { + global.LOG.Errorf("exec `select * from %s` failed, err: %v", table, err) + return err + } + defer lineRows.Close() + + var columns []string + columns, err = lineRows.Columns() + if err != nil { + global.LOG.Errorf("get columes falied, err: %v", err) + return err + } + columnTypes, err := lineRows.ColumnTypes() + if err != nil { + global.LOG.Errorf("get colume types failed, err: %v", err) + return err + } + + var values [][]interface{} + for lineRows.Next() { + row := make([]interface{}, len(columns)) + rowPointers := make([]interface{}, len(columns)) + for i := range columns { + rowPointers[i] = &row[i] + } + err = lineRows.Scan(rowPointers...) + if err != nil { + global.LOG.Errorf("scan row data failed, err: %v", err) + return err + } + values = append(values, row) + } + + for _, row := range values { + ssql := "INSERT INTO `" + table + "` VALUES (" + + for i, col := range row { + if col == nil { + ssql += "NULL" + } else { + Type := columnTypes[i].DatabaseTypeName() + Type = strings.Replace(Type, "UNSIGNED", "", -1) + Type = strings.Replace(Type, " ", "", -1) + switch Type { + case "TINYINT", "SMALLINT", "MEDIUMINT", "INT", "INTEGER", "BIGINT": + if bs, ok := col.([]byte); ok { + ssql += string(bs) + } else { + ssql += fmt.Sprintf("%d", col) + } + case "FLOAT", "DOUBLE": + if bs, ok := col.([]byte); ok { + ssql += string(bs) + } else { + ssql += fmt.Sprintf("%f", col) + } + case "DECIMAL", "DEC": + ssql += fmt.Sprintf("%s", col) + + case "DATE": + t, ok := col.(time.Time) + if !ok { + global.LOG.Errorf("the DATE type conversion failed., err: %v", err) + return err + } + ssql += fmt.Sprintf("'%s'", t.Format("2006-01-02")) + case "DATETIME": + t, ok := col.(time.Time) + if !ok { + global.LOG.Errorf("the DATETIME type conversion failed., err: %v", err) + return err + } + ssql += fmt.Sprintf("'%s'", t.Format("2006-01-02 15:04:05")) + case "TIMESTAMP": + t, ok := col.(time.Time) + if !ok { + global.LOG.Errorf("the TIMESTAMP type conversion failed., err: %v", err) + return err + } + ssql += fmt.Sprintf("'%s'", t.Format("2006-01-02 15:04:05")) + case "TIME": + t, ok := col.([]byte) + if !ok { + global.LOG.Errorf("the TIME type conversion failed., err: %v", err) + return err + } + ssql += fmt.Sprintf("'%s'", string(t)) + case "YEAR": + t, ok := col.([]byte) + if !ok { + global.LOG.Errorf("the YEAR type conversion failed., err: %v", err) + return err + } + ssql += string(t) + case "CHAR", "VARCHAR", "TINYTEXT", "TEXT", "MEDIUMTEXT", "LONGTEXT": + ssql += fmt.Sprintf("'%s'", strings.Replace(fmt.Sprintf("%s", col), "'", "''", -1)) + case "BIT", "BINARY", "VARBINARY", "TINYBLOB", "BLOB", "MEDIUMBLOB", "LONGBLOB": + ssql += fmt.Sprintf("0x%X", col) + case "ENUM", "SET": + ssql += fmt.Sprintf("'%s'", col) + case "BOOL", "BOOLEAN": + if col.(bool) { + ssql += "true" + } else { + ssql += "false" + } + case "JSON": + ssql += fmt.Sprintf("'%s'", col) + default: + global.LOG.Errorf("unsupported colume type: %s", Type) + return fmt.Errorf("unsupported colume type: %s", Type) + } + } + if i < len(row)-1 { + ssql += "," + } + } + ssql += ");\n" + _, _ = buf.WriteString(ssql) + } + + _, _ = buf.WriteString("\n\n") + return nil +} diff --git a/backend/utils/mysql/helper/source.go b/backend/utils/mysql/helper/source.go new file mode 100644 index 000000000..a5890aafb --- /dev/null +++ b/backend/utils/mysql/helper/source.go @@ -0,0 +1,228 @@ +package helper + +import ( + "bufio" + "database/sql" + "errors" + "fmt" + "io" + "strings" + "time" + + "github.com/1Panel-dev/1Panel/backend/global" +) + +type sourceOption struct { + dryRun bool + mergeInsert int + debug bool +} +type SourceOption func(*sourceOption) + +func WithDryRun() SourceOption { + return func(o *sourceOption) { + o.dryRun = true + } +} + +func WithMergeInsert(size int) SourceOption { + return func(o *sourceOption) { + o.mergeInsert = size + } +} + +func WithDebug() SourceOption { + return func(o *sourceOption) { + o.debug = true + } +} + +type dbWrapper struct { + DB *sql.DB + debug bool + dryRun bool +} + +func newDBWrapper(db *sql.DB, dryRun, debug bool) *dbWrapper { + + return &dbWrapper{ + DB: db, + dryRun: dryRun, + debug: debug, + } +} + +func (db *dbWrapper) Exec(query string, args ...interface{}) (sql.Result, error) { + if db.debug { + global.LOG.Debugf("query %s", query) + } + + if db.dryRun { + return nil, nil + } + return db.DB.Exec(query, args...) +} + +func Source(dns string, reader io.Reader, opts ...SourceOption) error { + start := time.Now() + global.LOG.Infof("source start at %s", start.Format("2006-01-02 15:04:05")) + defer func() { + end := time.Now() + global.LOG.Infof("source end at %s, cost %s", end.Format("2006-01-02 15:04:05"), end.Sub(start)) + }() + + var err error + var db *sql.DB + var o sourceOption + for _, opt := range opts { + opt(&o) + } + + dbName, err := getDBNameFromDNS(dns) + if err != nil { + global.LOG.Errorf("get db name from dns failed, err: %v", err) + return err + } + + db, err = sql.Open("mysql", dns) + if err != nil { + global.LOG.Errorf("open mysql db failed, err: %v", err) + return err + } + defer db.Close() + + dbWrapper := newDBWrapper(db, o.dryRun, o.debug) + + _, err = dbWrapper.Exec(fmt.Sprintf("USE `%s`;", dbName)) + if err != nil { + global.LOG.Errorf("exec `use %s` failed, err: %v", dbName, err) + return err + } + + db.SetConnMaxLifetime(3600) + + r := bufio.NewReader(reader) + _, err = dbWrapper.Exec("SET autocommit=0;") + if err != nil { + global.LOG.Errorf("exec `set autocommit=0` failed, err: %v", err) + return err + } + + for { + line, err := r.ReadString(';') + if err != nil { + if err == io.EOF { + break + } + global.LOG.Errorf("read sql failed, err: %v", err) + return err + } + + ssql := string(line) + + ssql, err = trim(ssql) + if err != nil { + global.LOG.Errorf("trim sql failed, err: %v", err) + return err + } + + if o.mergeInsert > 1 && strings.HasPrefix(ssql, "INSERT INTO") { + var insertSQLs []string + insertSQLs = append(insertSQLs, ssql) + for i := 0; i < o.mergeInsert-1; i++ { + line, err := r.ReadString(';') + if err != nil { + if err == io.EOF { + break + } + global.LOG.Errorf("read merge insert sql failed, err: %v", err) + return err + } + + ssql2 := string(line) + ssql2, err = trim(ssql2) + if err != nil { + global.LOG.Errorf("trim merge insert sql failed, err: %v", err) + return err + } + if strings.HasPrefix(ssql2, "INSERT INTO") { + insertSQLs = append(insertSQLs, ssql2) + continue + } + + break + } + ssql, err = mergeInsert(insertSQLs) + if err != nil { + global.LOG.Errorf("do merge insert failed, err: %v", err) + return err + } + } + + _, err = dbWrapper.Exec(ssql) + if err != nil { + global.LOG.Errorf("exec sql failed, err: %v", err) + return err + } + } + + _, err = dbWrapper.Exec("COMMIT;") + if err != nil { + global.LOG.Errorf("exec `commit` failed, err: %v", err) + return err + } + + _, err = dbWrapper.Exec("SET autocommit=1;") + if err != nil { + global.LOG.Errorf("exec `autocommit=1` failed, err: %v", err) + return err + } + + return nil +} + +func mergeInsert(insertSQLs []string) (string, error) { + if len(insertSQLs) == 0 { + return "", errors.New("no input provided") + } + builder := strings.Builder{} + sql1 := insertSQLs[0] + sql1 = strings.TrimSuffix(sql1, ";") + builder.WriteString(sql1) + for i, insertSQL := range insertSQLs[1:] { + if i < len(insertSQLs)-1 { + builder.WriteString(",") + } + + valuesIdx := strings.Index(insertSQL, "VALUES") + if valuesIdx == -1 { + return "", errors.New("invalid SQL: missing VALUES keyword") + } + sqln := insertSQL[valuesIdx:] + sqln = strings.TrimPrefix(sqln, "VALUES") + sqln = strings.TrimSuffix(sqln, ";") + builder.WriteString(sqln) + + } + builder.WriteString(";") + + return builder.String(), nil +} + +func trim(s string) (string, error) { + s = strings.TrimLeft(s, "\n") + s = strings.TrimSpace(s) + return s, nil +} + +func getDBNameFromDNS(dns string) (string, error) { + ss1 := strings.Split(dns, "/") + if len(ss1) == 2 { + ss2 := strings.Split(ss1[1], "?") + if len(ss2) == 2 { + return ss2[0], nil + } + } + + return "", fmt.Errorf("dns error: %s", dns) +} diff --git a/frontend/src/views/database/mysql/conn/index.vue b/frontend/src/views/database/mysql/conn/index.vue index a6fb276da..78b249479 100644 --- a/frontend/src/views/database/mysql/conn/index.vue +++ b/frontend/src/views/database/mysql/conn/index.vue @@ -159,6 +159,7 @@ const loadPassword = async () => { const onSubmit = async () => { let param = { id: 0, + from: form.from, value: form.password, }; loading.value = true; diff --git a/frontend/src/views/database/mysql/create/index.vue b/frontend/src/views/database/mysql/create/index.vue index 2be1cfb1f..e51d86e4a 100644 --- a/frontend/src/views/database/mysql/create/index.vue +++ b/frontend/src/views/database/mysql/create/index.vue @@ -47,14 +47,7 @@ - - - + {{ loadLabel(form.from) }} @@ -83,13 +76,12 @@ import { reactive, ref } from 'vue'; import { Rules } from '@/global/form-rules'; import i18n from '@/lang'; import { ElForm } from 'element-plus'; -import { addMysqlDB, listRemoteDBs } from '@/api/modules/database'; +import { addMysqlDB } from '@/api/modules/database'; import DrawerHeader from '@/components/drawer-header/index.vue'; import { MsgSuccess } from '@/utils/message'; import { getRandomStr } from '@/utils/util'; const loading = ref(); -const dbOptions = ref(); const createVisiable = ref(false); const form = reactive({ name: '', @@ -108,16 +100,17 @@ const rules = reactive({ password: [Rules.requiredInput], permission: [Rules.requiredSelect], permissionIPs: [Rules.requiredInput], - from: [Rules.requiredSelect], }); type FormInstance = InstanceType; const formRef = ref(); interface DialogProps { + from: string; mysqlName: string; } const acceptParams = (params: DialogProps): void => { form.name = ''; + form.from = params.from; form.mysqlName = params.mysqlName; form.format = 'utf8mb4'; form.username = ''; @@ -125,20 +118,14 @@ const acceptParams = (params: DialogProps): void => { form.permissionIPs = ''; form.description = ''; random(); - loadDBOptions(); createVisiable.value = true; }; const handleClose = () => { createVisiable.value = false; }; -const loadDBOptions = async () => { - const res = await listRemoteDBs('mysql'); - dbOptions.value = res.data || []; -}; - -function loadLabel(item: any) { - return (item.name === 'local' ? i18n.global.t('database.localDB') : item.name) + '(' + item.address + ')'; +function loadLabel(from: any) { + return from === 'local' ? i18n.global.t('database.localDB') : from; } const random = async () => { diff --git a/frontend/src/views/database/mysql/delete/index.vue b/frontend/src/views/database/mysql/delete/index.vue index d2e9fed3d..aba38730a 100644 --- a/frontend/src/views/database/mysql/delete/index.vue +++ b/frontend/src/views/database/mysql/delete/index.vue @@ -5,7 +5,7 @@ width="30%" :close-on-click-modal="false" > - + diff --git a/frontend/src/views/database/mysql/index.vue b/frontend/src/views/database/mysql/index.vue index 389f9de80..52dd76317 100644 --- a/frontend/src/views/database/mysql/index.vue +++ b/frontend/src/views/database/mysql/index.vue @@ -264,6 +264,7 @@ const mysqlVersion = ref(); const dialogRef = ref(); const onOpenDialog = async () => { let params = { + from: paginationConfig.from, mysqlName: mysqlName.value, }; dialogRef.value!.acceptParams(params); diff --git a/frontend/src/views/database/mysql/password/index.vue b/frontend/src/views/database/mysql/password/index.vue index 347f86c63..f8213778a 100644 --- a/frontend/src/views/database/mysql/password/index.vue +++ b/frontend/src/views/database/mysql/password/index.vue @@ -115,7 +115,6 @@ const acceptParams = (params: DialogProps): void => { : i18n.global.t('database.permission'); changeForm.id = params.id; changeForm.from = params.from; - console.log(changeForm.from); changeForm.mysqlName = params.mysqlName; changeForm.userName = params.username; changeForm.password = params.password; diff --git a/frontend/src/views/database/mysql/setting/slow-log/index.vue b/frontend/src/views/database/mysql/setting/slow-log/index.vue index 75fe334d4..578794347 100644 --- a/frontend/src/views/database/mysql/setting/slow-log/index.vue +++ b/frontend/src/views/database/mysql/setting/slow-log/index.vue @@ -1,6 +1,6 @@