chore: greatly speed up migrator and lower memory usage (#2874)

* chore: add en-GB language

* chore: remove en-GB contents

* chore: prevent visitors from breaking demo
- prevent disabling password login
- prevent updating `memos-demo` user
- prevent setting additional style
- prevent setting additional script
- add some error feedback to system settings UI

* Revert "chore: add en-GB language"

This reverts commit 2716377b04.

* chore: speed-up migrator and lower memory usage
- remove all Store indirections
- query database directly with prepared statements

* chore: fix golangci-lint warnings

store/migrator.go

@@ -2,11 +2,11 @@ package store
 
 import (
 	"context"
+	"database/sql"
 	"fmt"
 	"os"
-	"time"
-
 	"strings"
+	"time"
 
 	"github.com/lithammer/shortuuid/v4"
 	"github.com/pkg/errors"
@@ -16,88 +16,242 @@ import (
 
 // MigrateResourceInternalPath migrates resource internal path from absolute path to relative path.
 func (s *Store) MigrateResourceInternalPath(ctx context.Context) error {
-	resources, err := s.ListResources(ctx, &FindResource{})
+	normalizedDataPath := strings.ReplaceAll(s.Profile.Data, `\`, "/")
-	if err != nil {
+	normalizedDataPath = strings.ReplaceAll(normalizedDataPath, `//`, "/")
-		return errors.Wrap(err, "failed to list resources")
+
+	db := s.driver.GetDB()
+	checkStmt := `
+		SELECT id FROM resource 
+		WHERE 
+			internal_path LIKE ?
+			OR internal_path LIKE ?
+			OR internal_path LIKE ?
+		LIMIT 1`
+	rows := 0
+	res := db.QueryRowContext(ctx, checkStmt, fmt.Sprintf("%s%%", s.Profile.Data), fmt.Sprintf("%s%%", normalizedDataPath), "%\\%")
+	if err := res.Scan(&rows); err != nil {
+		if rows == 0 || err == sql.ErrNoRows {
+			log.Debug("Resource internal path migration is not required.")
+			return nil
+		}
+		return errors.Wrap(err, "failed to check resource internal_path")
 	}
 
-	dataPath := strings.ReplaceAll(s.Profile.Data, `\`, "/")
+	log.Info("Migrating resource internal paths. This may take a while.")
+
+	listResourcesStmt := `
+		SELECT id, internal_path FROM resource
+		WHERE
+			internal_path IS NOT ''
+			AND internal_path LIKE ?
+			OR internal_path LIKE ?
+	`
+	resources, err := db.QueryContext(ctx, listResourcesStmt, fmt.Sprintf("%s%%", s.Profile.Data), fmt.Sprintf("%s%%", normalizedDataPath))
+	if err != nil || resources.Err() != nil {
+		return errors.Wrap(err, "failed to list resources")
+	}
+	defer resources.Close()
+
+	tx, err := db.BeginTx(ctx, nil)
+	if err != nil {
+		return errors.Wrap(err, "failed to start transaction")
+	}
+	defer tx.Rollback()
+
+	updateStmt := `UPDATE resource SET internal_path = ? WHERE id = ?`
+	preparedResourceUpdate, err := tx.PrepareContext(ctx, updateStmt)
+	if err != nil {
+		return errors.Wrap(err, "failed to prepare update statement")
+	}
+	defer preparedResourceUpdate.Close()
+
 	migrateStartTime := time.Now()
 	migratedCount := 0
-	for _, resource := range resources {
+	for resources.Next() {
+		resource := Resource{}
+		if err := resources.Scan(&resource.ID, &resource.InternalPath); err != nil {
+			return errors.Wrap(err, "failed to parse resource data")
+		}
+
 		if resource.InternalPath == "" {
 			continue
 		}
 
 		internalPath := strings.ReplaceAll(resource.InternalPath, `\`, "/")
-		if !strings.HasPrefix(internalPath, dataPath) {
+		if !strings.HasPrefix(internalPath, normalizedDataPath) {
 			continue
 		}
 
-		internalPath = strings.TrimPrefix(internalPath, dataPath)
+		internalPath = strings.TrimPrefix(internalPath, normalizedDataPath)
 
 		for os.IsPathSeparator(internalPath[0]) {
 			internalPath = internalPath[1:]
 		}
 
-		_, err := s.UpdateResource(ctx, &UpdateResource{
+		_, err := preparedResourceUpdate.ExecContext(ctx, internalPath, resource.ID)
-			ID:           resource.ID,
-			InternalPath: &internalPath,
-		})
 		if err != nil {
-			return errors.Wrap(err, "failed to update local resource path")
+			return errors.Wrap(err, "failed to update resource internal_path")
 		}
+
+		if migratedCount%500 == 0 {
+			log.Info(fmt.Sprintf("[Running] Migrated %d local resource paths", migratedCount))
+		}
+
 		migratedCount++
 	}
+	if err := tx.Commit(); err != nil {
+		return errors.Wrap(err, "failed to commit transaction")
+	}
 
-	if migratedCount > 0 && s.Profile.Mode == "prod" {
+	if migratedCount > 0 {
-		log.Info(fmt.Sprintf("migrated %d local resource paths in %s", migratedCount, time.Since(migrateStartTime)))
+		log.Info(fmt.Sprintf("Migrated %d local resource paths in %s", migratedCount, time.Since(migrateStartTime)))
 	}
 	return nil
 }
 
 // MigrateResourceName migrates resource name from other format to short UUID.
 func (s *Store) MigrateResourceName(ctx context.Context) error {
-	memos, err := s.ListMemos(ctx, &FindMemo{
+	db := s.driver.GetDB()
-		ExcludeContent: true,
-	})
-	if err != nil {
-		return errors.Wrap(err, "failed to list memos")
-	}
-	for _, memo := range memos {
-		if checkResourceName(memo.ResourceName) {
-			continue
-		}
 
-		resourceName := shortuuid.New()
+	checkStmt := `
-		err := s.UpdateMemo(ctx, &UpdateMemo{
+		SELECT resource_name FROM resource 
-			ID:           memo.ID,
+		WHERE 
-			ResourceName: &resourceName,
+			resource_name = ''
-		})
+		LIMIT 1`
-		if err != nil {
+	rows := 0
-			return errors.Wrap(err, "failed to update memo")
+	res := db.QueryRowContext(ctx, checkStmt)
+	if err := res.Scan(&rows); err != nil {
+		if rows == 0 || err == sql.ErrNoRows {
+			log.Debug("Resource migration to UUIDs is not required.")
+			return nil
 		}
+		return errors.Wrap(err, "failed to check resource.resource_name")
 	}
 
-	resources, err := s.ListResources(ctx, &FindResource{})
+	log.Info("Migrating resource IDs to UUIDs. This may take a while.")
-	if err != nil {
+
+	listResourceStmt := "SELECT `id`, `resource_name` FROM `resource` WHERE `resource_name` = ''"
+	resources, err := db.QueryContext(ctx, listResourceStmt)
+	if err != nil || resources.Err() != nil {
 		return errors.Wrap(err, "failed to list resources")
 	}
-	for _, resource := range resources {
+	defer resources.Close()
+
+	tx, err := db.BeginTx(ctx, nil)
+	if err != nil {
+		return errors.Wrap(err, "failed to start transaction")
+	}
+	defer tx.Rollback()
+
+	updateResourceStmt := "UPDATE `resource` SET `resource_name` = ? WHERE `id` = ?"
+	preparedResourceUpdate, err := tx.PrepareContext(ctx, updateResourceStmt)
+	if err != nil {
+		return errors.Wrap(err, "failed to prepare update statement")
+	}
+	defer preparedResourceUpdate.Close()
+
+	migrateStartTime := time.Now()
+	migratedCount := 0
+	for resources.Next() {
+		resource := Resource{}
+		if err := resources.Scan(&resource.ID, &resource.ResourceName); err != nil {
+			return errors.Wrap(err, "failed to parse resource data")
+		}
+
 		if checkResourceName(resource.ResourceName) {
 			continue
 		}
 
 		resourceName := shortuuid.New()
-		_, err := s.UpdateResource(ctx, &UpdateResource{
+		if _, err := preparedResourceUpdate.ExecContext(ctx, resourceName, resource.ID); err != nil {
-			ID:           resource.ID,
-			ResourceName: &resourceName,
-		})
-		if err != nil {
 			return errors.Wrap(err, "failed to update resource")
 		}
+
+		if migratedCount%500 == 0 {
+			log.Info(fmt.Sprintf("[Running] Migrated %d local resources IDs", migratedCount))
+		}
+		migratedCount++
+	}
+	if err := tx.Commit(); err != nil {
+		return errors.Wrap(err, "failed to commit transaction")
 	}
 
+	if migratedCount > 0 {
+		log.Info(fmt.Sprintf("Migrated %d resource IDs to UUIDs in %s", migratedCount, time.Since(migrateStartTime)))
+	}
+	return nil
+}
+
+// MigrateResourceName migrates memo name from other format to short UUID.
+func (s *Store) MigrateMemoName(ctx context.Context) error {
+	db := s.driver.GetDB()
+
+	checkStmt := `
+		SELECT resource_name FROM memo 
+		WHERE 
+			resource_name = ''
+		LIMIT 1`
+	rows := 0
+	res := db.QueryRowContext(ctx, checkStmt)
+	if err := res.Scan(&rows); err != nil {
+		if rows == 0 || err == sql.ErrNoRows {
+			log.Debug("Memo migration to UUIDs is not required.")
+			return nil
+		}
+		return errors.Wrap(err, "failed to check memo.resource_name")
+	}
+
+	log.Info("Migrating memo ids to uuids. This may take a while.")
+
+	listMemoStmt := "SELECT `id`, `resource_name` FROM `memo` WHERE `resource_name` = ''"
+	memos, err := db.QueryContext(ctx, listMemoStmt)
+	if err != nil || memos.Err() != nil {
+		return errors.Wrap(err, "failed to list memos")
+	}
+	defer memos.Close()
+
+	tx, err := db.BeginTx(ctx, nil)
+	if err != nil {
+		return errors.Wrap(err, "failed to start transaction")
+	}
+	defer tx.Rollback()
+
+	updateMemoStmt := "UPDATE `memo` SET `resource_name` = ? WHERE `id` = ?"
+	preparedMemoUpdate, err := tx.PrepareContext(ctx, updateMemoStmt)
+	if err != nil {
+		return errors.Wrap(err, "failed to prepare update statement")
+	}
+	defer preparedMemoUpdate.Close()
+
+	migrateStartTime := time.Now()
+	migratedCount := 0
+	for memos.Next() {
+		memo := Memo{}
+		if err := memos.Scan(&memo.ID, &memo.ResourceName); err != nil {
+			return errors.Wrap(err, "failed to parse memo data")
+		}
+
+		if checkResourceName(memo.ResourceName) {
+			continue
+		}
+
+		resourceName := shortuuid.New()
+		if _, err := preparedMemoUpdate.ExecContext(ctx, resourceName, memo.ID); err != nil {
+			return errors.Wrap(err, "failed to update memo")
+		}
+
+		if migratedCount%500 == 0 {
+			log.Info(fmt.Sprintf("[Running] Migrated %d local resources IDs", migratedCount))
+		}
+		migratedCount++
+	}
+	if err := tx.Commit(); err != nil {
+		return errors.Wrap(err, "failed to commit transaction")
+	}
+
+	if migratedCount > 0 {
+		log.Info(fmt.Sprintf("Migrated %d memo ids to uuids in %s", migratedCount, time.Since(migrateStartTime)))
+	}
 	return nil
 }
 

store/store.go

@@ -32,6 +32,9 @@ func (s *Store) MigrateManually(ctx context.Context) error {
 	if err := s.MigrateResourceName(ctx); err != nil {
 		return err
 	}
+	if err := s.MigrateMemoName(ctx); err != nil {
+		return err
+	}
 	return nil
 }