feat: schema migration handler (#100)

* chore: update about site dialog

* feat: schema migration

* chore: lint with golangci

common/version.go

@@ -1,4 +1,61 @@
 package common
 
+import (
+	"strconv"
+	"strings"
+)
+
 // Version is the service current released version.
+// Semantic versioning: https://semver.org/
 var Version = "0.1.3"
+
+// DevVersion is the service current development version.
+var DevVersion = "0.2.0"
+
+func GetCurrentVersion(mode string) string {
+	if mode == "dev" {
+		return DevVersion
+	}
+	return Version
+}
+
+func GetMinorVersion(version string) string {
+	versionList := strings.Split(version, ".")
+	if len(versionList) < 3 {
+		return ""
+	}
+	return versionList[0] + "." + versionList[1]
+}
+
+// convSemanticVersionToInt converts version string to int.
+func convSemanticVersionToInt(version string) int {
+	versionList := strings.Split(version, ".")
+
+	if len(versionList) < 3 {
+		return 0
+	}
+	major, err := strconv.Atoi(versionList[0])
+	if err != nil {
+		return 0
+	}
+	minor, err := strconv.Atoi(versionList[1])
+	if err != nil {
+		return 0
+	}
+	patch, err := strconv.Atoi(versionList[2])
+	if err != nil {
+		return 0
+	}
+
+	return major*10000 + minor*100 + patch
+}
+
+// IsVersionGreaterThanOrEqualTo returns true if version is greater than or equal to target.
+func IsVersionGreaterOrEqualThan(version, target string) bool {
+	return convSemanticVersionToInt(version) >= convSemanticVersionToInt(target)
+}
+
+// IsVersionGreaterThan returns true if version is greater than target.
+func IsVersionGreaterThan(version, target string) bool {
+	return convSemanticVersionToInt(version) > convSemanticVersionToInt(target)
+}

server/memo.go

@@ -176,8 +176,7 @@ func (s *Server) registerMemoRoutes(g *echo.Group) {
 			return echo.NewHTTPError(http.StatusInternalServerError, fmt.Sprintf("Failed to delete memo ID: %v", memoID)).SetInternal(err)
 		}
 
-		c.JSON(http.StatusOK, true)
-		return nil
+		return c.JSON(http.StatusOK, true)
 	})
 
 	g.GET("/memo/amount", func(c echo.Context) error {

server/profile/profile.go

@@ -72,6 +72,6 @@ func GetProfile() *Profile {
 		Mode:    mode,
 		Port:    port,
 		DSN:     dsn,
-		Version: common.Version,
+		Version: common.GetCurrentVersion(mode),
 	}
 }

server/resource.go

@@ -118,7 +118,10 @@ func (s *Server) registerResourceRoutes(g *echo.Group) {
 
 		c.Response().Writer.WriteHeader(http.StatusOK)
 		c.Response().Writer.Header().Set("Content-Type", resource.Type)
-		c.Response().Writer.Write(resource.Blob)
+		if _, err := c.Response().Writer.Write(resource.Blob); err != nil {
+			return echo.NewHTTPError(http.StatusInternalServerError, "Failed to write resource blob").SetInternal(err)
+		}
+
 		return nil
 	})
 
@@ -135,7 +138,6 @@ func (s *Server) registerResourceRoutes(g *echo.Group) {
 			return echo.NewHTTPError(http.StatusInternalServerError, "Failed to delete resource").SetInternal(err)
 		}
 
-		c.JSON(http.StatusOK, true)
-		return nil
+		return c.JSON(http.StatusOK, true)
 	})
 }

server/shortcut.go

@@ -109,7 +109,6 @@ func (s *Server) registerShortcutRoutes(g *echo.Group) {
 			return echo.NewHTTPError(http.StatusInternalServerError, "Failed to delete shortcut").SetInternal(err)
 		}
 
-		c.JSON(http.StatusOK, true)
-		return nil
+		return c.JSON(http.StatusOK, true)
 	})
 }

server/webhook.go

@@ -261,7 +261,10 @@ func (s *Server) registerWebhookRoutes(g *echo.Group) {
 
 		c.Response().Writer.WriteHeader(http.StatusOK)
 		c.Response().Writer.Header().Set("Content-Type", resource.Type)
-		c.Response().Writer.Write(resource.Blob)
+		if _, err := c.Response().Writer.Write(resource.Blob); err != nil {
+			return echo.NewHTTPError(http.StatusInternalServerError, "Failed to write response").SetInternal(err)
+		}
+
 		return nil
 	})
 }

store/db/db.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"io/fs"
 	"os"
+	"regexp"
 	"sort"
 
 	"github.com/usememos/memos/common"
@@ -52,51 +53,108 @@ func (db *DB) Open() (err error) {
 	}
 
 	db.Db = sqlDB
-	// If db file not exists, we should migrate and seed the database.
-	if _, err := os.Stat(db.DSN); errors.Is(err, os.ErrNotExist) {
-		if err := db.migrate(); err != nil {
-			return fmt.Errorf("failed to migrate: %w", err)
+	// If mode is dev, we should migrate and seed the database.
+	if db.mode == "dev" {
+		if err := db.applyLatestSchema(); err != nil {
+			return fmt.Errorf("failed to apply latest schema: %w", err)
 		}
-		// If mode is dev, then seed the database.
-		if db.mode == "dev" {
-			if err := db.seed(); err != nil {
-				return fmt.Errorf("failed to seed: %w", err)
-			}
+		if err := db.seed(); err != nil {
+			return fmt.Errorf("failed to seed: %w", err)
 		}
 	} else {
-		// If db file exists and mode is dev, we should migrate and seed the database.
-		if db.mode == "dev" {
-			if err := db.migrate(); err != nil {
-				return fmt.Errorf("failed to migrate: %w", err)
+		// If db file not exists, we should migrate the database.
+		if _, err := os.Stat(db.DSN); errors.Is(err, os.ErrNotExist) {
+			err := db.applyLatestSchema()
+			if err != nil {
+				return fmt.Errorf("failed to apply latest schema: %w", err)
 			}
-			if err := db.seed(); err != nil {
-				return fmt.Errorf("failed to seed: %w", err)
+		} else {
+			err := db.createMigrationHistoryTable()
+			if err != nil {
+				return fmt.Errorf("failed to create migration_history table: %w", err)
+			}
+
+			currentVersion := common.GetCurrentVersion(db.mode)
+			migrationHistory, err := findMigrationHistory(db.Db, &MigrationHistoryFind{})
+			if err != nil {
+				return err
+			}
+			if migrationHistory == nil {
+				migrationHistory, err = upsertMigrationHistory(db.Db, &MigrationHistoryCreate{
+					Version:   currentVersion,
+					Statement: "",
+				})
+				if err != nil {
+					return err
+				}
+			}
+
+			if common.IsVersionGreaterThan(currentVersion, migrationHistory.Version) {
+				minorVersionList := getMinorVersionList()
+
+				for _, minorVersion := range minorVersionList {
+					normalizedVersion := minorVersion + ".0"
+					if common.IsVersionGreaterThan(normalizedVersion, migrationHistory.Version) && common.IsVersionGreaterOrEqualThan(currentVersion, normalizedVersion) {
+						err := db.applyMigrationForMinorVersion(minorVersion)
+						if err != nil {
+							return fmt.Errorf("failed to apply minor version migration: %w", err)
+						}
+					}
+				}
 			}
 		}
 	}
 
-	err = db.compareMigrationHistory()
-	if err != nil {
-		return fmt.Errorf("failed to compare migration history, err=%w", err)
-	}
-
 	return err
 }
 
-func (db *DB) migrate() error {
-	filenames, err := fs.Glob(migrationFS, fmt.Sprintf("%s/*.sql", "migration"))
+const (
+	latestSchemaFileName = "LATEST__SCHEMA.sql"
+)
+
+func (db *DB) applyLatestSchema() error {
+	latestSchemaPath := fmt.Sprintf("%s/%s", "migration", latestSchemaFileName)
+	buf, err := migrationFS.ReadFile(latestSchemaPath)
+	if err != nil {
+		return fmt.Errorf("failed to read latest schema %q, error %w", latestSchemaPath, err)
+	}
+	stmt := string(buf)
+	if err := db.execute(stmt); err != nil {
+		return fmt.Errorf("migrate error: statement:%s err=%w", stmt, err)
+	}
+	return nil
+}
+
+func (db *DB) applyMigrationForMinorVersion(minorVersion string) error {
+	filenames, err := fs.Glob(migrationFS, fmt.Sprintf("%s/%s/*.sql", "migration", minorVersion))
 	if err != nil {
 		return err
 	}
 
 	sort.Strings(filenames)
+	migrationStmt := ""
 
 	// Loop over all migration files and execute them in order.
 	for _, filename := range filenames {
-		if err := db.executeFile(migrationFS, filename); err != nil {
-			return fmt.Errorf("migrate error: name=%q err=%w", filename, err)
+		buf, err := migrationFS.ReadFile(filename)
+		if err != nil {
+			return fmt.Errorf("failed to read minor version migration file, filename=%s err=%w", filename, err)
+		}
+		stmt := string(buf)
+		migrationStmt += stmt
+		if err := db.execute(stmt); err != nil {
+			return fmt.Errorf("migrate error: statement:%s err=%w", stmt, err)
 		}
 	}
+
+	// upsert the newest version to migration_history
+	if _, err = upsertMigrationHistory(db.Db, &MigrationHistoryCreate{
+		Version:   minorVersion + ".0",
+		Statement: migrationStmt,
+	}); err != nil {
+		return err
+	}
+
 	return nil
 }
 
@@ -110,64 +168,84 @@ func (db *DB) seed() error {
 
 	// Loop over all seed files and execute them in order.
 	for _, filename := range filenames {
-		if err := db.executeFile(seedFS, filename); err != nil {
-			return fmt.Errorf("seed error: name=%q err=%w", filename, err)
+		buf, err := seedFS.ReadFile(filename)
+		if err != nil {
+			return fmt.Errorf("failed to read seed file, filename=%s err=%w", filename, err)
+		}
+		stmt := string(buf)
+		if err := db.execute(stmt); err != nil {
+			return fmt.Errorf("seed error: statement:%s err=%w", stmt, err)
 		}
 	}
 	return nil
 }
 
-// executeFile runs a single seed file within a transaction.
-func (db *DB) executeFile(FS embed.FS, name string) error {
+// excecute runs a single SQL statement within a transaction.
+func (db *DB) execute(stmt string) error {
 	tx, err := db.Db.Begin()
 	if err != nil {
 		return err
 	}
 	defer tx.Rollback()
 
-	// Read and execute SQL file.
-	if buf, err := fs.ReadFile(FS, name); err != nil {
-		return err
-	} else if _, err := tx.Exec(string(buf)); err != nil {
+	if _, err := tx.Exec(stmt); err != nil {
 		return err
 	}
 
 	return tx.Commit()
 }
 
-// compareMigrationHistory compares migration history data
-func (db *DB) compareMigrationHistory() error {
+// minorDirRegexp is a regular expression for minor version directory.
+var minorDirRegexp = regexp.MustCompile(`^migration/[0-9]+\.[0-9]+$`)
+
+func getMinorVersionList() []string {
+	minorVersionList := []string{}
+
+	if err := fs.WalkDir(migrationFS, "migration", func(path string, file fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if file.IsDir() && minorDirRegexp.MatchString(path) {
+			minorVersionList = append(minorVersionList, file.Name())
+		}
+
+		return nil
+	}); err != nil {
+		panic(err)
+	}
+
+	sort.Strings(minorVersionList)
+
+	return minorVersionList
+}
+
+// createMigrationHistoryTable creates the migration_history table if it doesn't exist.
+func (db *DB) createMigrationHistoryTable() error {
 	table, err := findTable(db.Db, "migration_history")
 	if err != nil {
 		return err
 	}
-	if table == nil {
-		if err := createTable(db.Db, `
-		CREATE TABLE migration_history (
-			version TEXT NOT NULL PRIMARY KEY,
-			created_ts BIGINT NOT NULL DEFAULT (strftime('%s', 'now'))
-		);
-		`); err != nil {
-			return err
-		}
-	}
 
-	currentVersion := common.Version
-	migrationHistoryFind := MigrationHistoryFind{
-		Version: currentVersion,
-	}
-	migrationHistory, err := findMigrationHistory(db.Db, &migrationHistoryFind)
-	if err != nil {
-		return err
-	}
-	if migrationHistory == nil {
-		// ...do schema migration,
-		// then upsert the newest version to migration_history
-		_, err := upsertMigrationHistory(db.Db, currentVersion)
+	// TODO(steven): Drop the migration_history table if it exists temporarily.
+	if table != nil {
+		err = db.execute(`
+		DROP TABLE IF EXISTS migration_history;
+		`)
 		if err != nil {
 			return err
 		}
 	}
 
+	err = createTable(db.Db, `
+	CREATE TABLE migration_history (
+		version TEXT NOT NULL PRIMARY KEY,
+		statement TEXT NOT NULL DEFAULT '',
+		created_ts BIGINT NOT NULL DEFAULT (strftime('%s', 'now'))
+	);
+	`)
+	if err != nil {
+		return err
+	}
+
 	return nil
 }

store/db/migration/00000__reset.sql

@@ -1,5 +0,0 @@
-DROP TABLE IF EXISTS `memo_organizer`;
-DROP TABLE IF EXISTS `memo`;
-DROP TABLE IF EXISTS `shortcut`;
-DROP TABLE IF EXISTS `resource`;
-DROP TABLE IF EXISTS `user`;

store/db/migration/LATEST__SCHEMA.sql

@@ -1,3 +1,10 @@
+-- drop all tables
+DROP TABLE IF EXISTS `memo_organizer`;
+DROP TABLE IF EXISTS `memo`;
+DROP TABLE IF EXISTS `shortcut`;
+DROP TABLE IF EXISTS `resource`;
+DROP TABLE IF EXISTS `user`;
+
 -- user
 CREATE TABLE user (
   id INTEGER PRIMARY KEY AUTOINCREMENT,

store/db/migration_history.go

@@ -6,18 +6,26 @@ import (
 )
 
 type MigrationHistory struct {
-	CreatedTs int64
 	Version   string
+	Statement string
+	CreatedTs int64
+}
+
+type MigrationHistoryCreate struct {
+	Version   string
+	Statement string
 }
 
 type MigrationHistoryFind struct {
-	Version string
+	Version *string
 }
 
 func findMigrationHistoryList(db *sql.DB, find *MigrationHistoryFind) ([]*MigrationHistory, error) {
 	where, args := []string{"1 = 1"}, []interface{}{}
 
-	where, args = append(where, "version = ?"), append(args, find.Version)
+	if v := find.Version; v != nil {
+		where, args = append(where, "version = ?"), append(args, *v)
+	}
 
 	rows, err := db.Query(`
 		SELECT 
@@ -63,18 +71,21 @@ func findMigrationHistory(db *sql.DB, find *MigrationHistoryFind) (*MigrationHis
 	}
 }
 
-func upsertMigrationHistory(db *sql.DB, version string) (*MigrationHistory, error) {
+func upsertMigrationHistory(db *sql.DB, create *MigrationHistoryCreate) (*MigrationHistory, error) {
 	row, err := db.Query(`
 		INSERT INTO migration_history (
-			version
+			version, 
+			statement
 		)
-		VALUES (?)
+		VALUES (?, ?)
 		ON CONFLICT(version) DO UPDATE
 		SET
-			version=EXCLUDED.version
-		RETURNING version, created_ts
+			version=EXCLUDED.version,
+			statement=EXCLUDED.statement
+		RETURNING version, statement, created_ts
 	`,
-		version,
+		create.Version,
+		create.Statement,
 	)
 	if err != nil {
 		return nil, err
@@ -82,9 +93,10 @@ func upsertMigrationHistory(db *sql.DB, version string) (*MigrationHistory, erro
 	defer row.Close()
 
 	row.Next()
-	migrationHistory := MigrationHistory{}
+	var migrationHistory MigrationHistory
 	if err := row.Scan(
 		&migrationHistory.Version,
+		&migrationHistory.Statement,
 		&migrationHistory.CreatedTs,
 	); err != nil {
 		return nil, err

web/src/components/AboutSiteDialog.tsx

@@ -45,13 +45,19 @@ const AboutSiteDialog: React.FC<Props> = ({ destroy }: Props) => {
         </p>
         <br />
         <p>
-          <a href="https://github.com/usememos/memos">🏗 Source code</a>, and built by <a href="https://github.com/boojack">Steven 🐯</a>.
+          <a className="mr-2" href="https://github.com/usememos/memos">
+            🏗 Source code
+          </a>
+          <Only when={profile !== undefined}>
+            <>
+              version:
+              <span className="pre-text">
+                {profile?.version}-{profile?.mode}
+              </span>
+              🎉
+            </>
+          </Only>
         </p>
-        <Only when={profile !== undefined}>
-          <p className="updated-time-text">
-            version: <span className="pre-text">{profile?.version}</span> 🎉
-          </p>
-        </Only>
       </div>
     </>
   );

web/src/less/about-site-dialog.less

@@ -11,14 +11,10 @@
 
       > p {
         @apply my-1;
-
-        &.updated-time-text {
-          @apply flex flex-row justify-start items-center w-full text-sm mt-3 pt-2 border-t-2 border-t-gray-200 text-gray-600 whitespace-pre-wrap font-mono;
-        }
       }
 
       .pre-text {
-        @apply font-mono;
+        @apply font-mono mx-1;
       }
 
       a {