Skip to content

C
canifa_note
Commit 95de5cc7 authored by Steven's avatar Steven
Browse files
Options

refactor: update migration history methods

parent bc7decf6
Hide whitespace changes
Inline Side-by-side
Showing with 208 additions and 49 deletions
internal/version/version.go
View file @ 95de5cc7
......@@ -21,11 +21,15 @@ func GetCurrentVersion(mode string) string {
return Version
}
// GetMinorVersion extracts the minor version (e.g., "0.25") from a full version string (e.g., "0.25.1").
// Returns the minor version string or empty string if the version format is invalid.
// Version format should be "major.minor.patch" (e.g., "0.25.1").
func GetMinorVersion(version string) string {
versionList := strings.Split(version, ".")
if len(versionList) < 3 {
if len(versionList) < 2 {
return ""
}
// Return major.minor only (first two components)
return versionList[0] + "." + versionList[1]
}
......
store/db/mysql/migration_history.go
View file @ 95de5cc7
......@@ -6,6 +6,8 @@ import (
"github.com/usememos/memos/store"
)
// FindMigrationHistoryList retrieves all migration history records.
// NOTE: This method is deprecated along with the migration_history table.
func (d *DB) FindMigrationHistoryList(ctx context.Context, _ *store.FindMigrationHistory) ([]*store.MigrationHistory, error) {
query := "SELECT `version`, UNIX_TIMESTAMP(`created_ts`) FROM `migration_history` ORDER BY `created_ts` DESC"
rows, err := d.db.QueryContext(ctx, query)
......@@ -34,6 +36,11 @@ func (d *DB) FindMigrationHistoryList(ctx context.Context, _ *store.FindMigratio
return list, nil
}
// UpsertMigrationHistory inserts or updates a migration history record.
// NOTE: This method is deprecated along with the migration_history table.
// This uses separate INSERT and SELECT queries instead of INSERT...RETURNING because
// MySQL doesn't support RETURNING clause in the same way as PostgreSQL/SQLite.
// This could have race conditions but is acceptable for deprecated transition code.
func (d *DB) UpsertMigrationHistory(ctx context.Context, upsert *store.UpsertMigrationHistory) (*store.MigrationHistory, error) {
stmt := "INSERT INTO `migration_history` (`version`) VALUES (?) ON DUPLICATE KEY UPDATE `version` = ?"
_, err := d.db.ExecContext(ctx, stmt, upsert.Version, upsert.Version)
......
store/db/postgres/migration_history.go
View file @ 95de5cc7
......@@ -6,6 +6,8 @@ import (
"github.com/usememos/memos/store"
)
// FindMigrationHistoryList retrieves all migration history records.
// NOTE: This method is deprecated along with the migration_history table.
func (d *DB) FindMigrationHistoryList(ctx context.Context, _ *store.FindMigrationHistory) ([]*store.MigrationHistory, error) {
query := "SELECT version, created_ts FROM migration_history ORDER BY created_ts DESC"
rows, err := d.db.QueryContext(ctx, query)
......@@ -34,6 +36,8 @@ func (d *DB) FindMigrationHistoryList(ctx context.Context, _ *store.FindMigratio
return list, nil
}
// UpsertMigrationHistory inserts or updates a migration history record.
// NOTE: This method is deprecated along with the migration_history table.
func (d *DB) UpsertMigrationHistory(ctx context.Context, upsert *store.UpsertMigrationHistory) (*store.MigrationHistory, error) {
stmt := `
INSERT INTO migration_history (
......
store/db/sqlite/migration_history.go
View file @ 95de5cc7
......@@ -6,6 +6,8 @@ import (
"github.com/usememos/memos/store"
)
// FindMigrationHistoryList retrieves all migration history records.
// NOTE: This method is deprecated along with the migration_history table.
func (d *DB) FindMigrationHistoryList(ctx context.Context, _ *store.FindMigrationHistory) ([]*store.MigrationHistory, error) {
query := "SELECT `version`, `created_ts` FROM `migration_history` ORDER BY `created_ts` DESC"
rows, err := d.db.QueryContext(ctx, query)
......@@ -34,6 +36,8 @@ func (d *DB) FindMigrationHistoryList(ctx context.Context, _ *store.FindMigratio
return list, nil
}
// UpsertMigrationHistory inserts or updates a migration history record.
// NOTE: This method is deprecated along with the migration_history table.
func (d *DB) UpsertMigrationHistory(ctx context.Context, upsert *store.UpsertMigrationHistory) (*store.MigrationHistory, error) {
stmt := `
INSERT INTO migration_history (
......
store/driver.go
View file @ 95de5cc7
......@@ -14,6 +14,9 @@ type Driver interface {
IsInitialized(ctx context.Context) (bool, error)
// MigrationHistory model related methods.
// NOTE: These methods are deprecated. The migration_history table is no longer used
// for tracking schema versions. Schema version is now stored in workspace_setting.
// These methods are kept for backward compatibility to migrate existing installations.
FindMigrationHistoryList(ctx context.Context, find *FindMigrationHistory) ([]*MigrationHistory, error)
UpsertMigrationHistory(ctx context.Context, upsert *UpsertMigrationHistory) (*MigrationHistory, error)
......
store/migration_history.go
View file @ 95de5cc7
package store
// MigrationHistory represents a record in the migration_history table.
// NOTE: The migration_history table is deprecated in favor of storing schema version
// in workspace_setting (BASIC setting). This is kept for backward compatibility only.
// Migration from migration_history to workspace_setting happens automatically during startup.
type MigrationHistory struct {
Version string
CreatedTs int64
}
// UpsertMigrationHistory is used to insert or update a migration history record.
// NOTE: This is deprecated along with the migration_history table.
type UpsertMigrationHistory struct {
Version string
}
// FindMigrationHistory is used to query migration history records.
// NOTE: This is deprecated along with the migration_history table.
type FindMigrationHistory struct {
}
store/migrator.go
View file @ 95de5cc7
......@@ -18,6 +18,30 @@ import (
storepb "github.com/usememos/memos/proto/gen/store"
)
// Migration System Overview:
//
// The migration system handles database schema versioning and upgrades.
// Schema version is stored in workspace_setting (the new system).
// The old migration_history table is deprecated but still supported for backward compatibility.
//
// Migration Flow:
// 1. preMigrate: Check if DB is initialized. If not, apply LATEST.sql
// 2. normalizeMigrationHistoryList: Normalize old migration_history records (for pre-0.22 installations)
// 3. migrateSchemaVersionToSetting: Migrate version from migration_history to workspace_setting
// 4. Migrate (prod mode): Apply incremental migrations from current to target version
// 5. Migrate (demo mode): Seed database with demo data
//
// Version Tracking:
// - New installations: Schema version set in workspace_setting immediately
// - Old installations: Version migrated from migration_history to workspace_setting automatically
// - Empty version: Treated as 0.0.0 and all migrations applied
//
// Migration Files:
// - Location: store/migration/{driver}/{version}/NN__description.sql
// - Naming: NN is zero-padded patch number, description is human-readable
// - Ordering: Files sorted lexicographically and applied in order
// - LATEST.sql: Full schema for new installations (faster than incremental migrations)
//go:embed migration
var migrationFS embed.FS
......@@ -31,8 +55,59 @@ const (
// LatestSchemaFileName is the name of the latest schema file.
// This file is used to apply the latest schema when no migration history is found.
LatestSchemaFileName = "LATEST.sql"
// defaultSchemaVersion is used when schema version is empty or not set.
// This handles edge cases for old installations without version tracking.
defaultSchemaVersion = "0.0.0"
// migrationHistoryNormalizedVersion is the version where migration_history normalization was completed.
// Before 0.22, migration history had inconsistent versioning that needed normalization.
migrationHistoryNormalizedVersion = "0.22"
// Mode constants for profile mode
modeProd = "prod"
modeDemo = "demo"
)
// getSchemaVersionOrDefault returns the schema version or default if empty.
// This ensures safe version comparisons and handles old installations.
func getSchemaVersionOrDefault(schemaVersion string) string {
if schemaVersion == "" {
return defaultSchemaVersion
}
return schemaVersion
}
// isVersionEmpty checks if the schema version is empty or the default value.
func isVersionEmpty(schemaVersion string) bool {
return schemaVersion == "" || schemaVersion == defaultSchemaVersion
}
// shouldApplyMigration determines if a migration file should be applied.
// It checks if the file's version is between the current DB version and target version.
func shouldApplyMigration(fileVersion, currentDBVersion, targetVersion string) bool {
currentDBVersionSafe := getSchemaVersionOrDefault(currentDBVersion)
return version.IsVersionGreaterThan(fileVersion, currentDBVersionSafe) &&
version.IsVersionGreaterOrEqualThan(targetVersion, fileVersion)
}
// validateMigrationFileName checks if a migration file follows the expected naming convention.
// Expected format: "NN__description.sql" where NN is a zero-padded number.
func validateMigrationFileName(filename string) error {
if !strings.Contains(filename, MigrateFileNameSplit) {
return errors.Errorf("invalid migration filename format (missing %s): %s", MigrateFileNameSplit, filename)
}
parts := strings.Split(filename, MigrateFileNameSplit)
if len(parts) < 2 {
return errors.Errorf("invalid migration filename format: %s", filename)
}
// Check if first part is a number
if _, err := strconv.Atoi(parts[0]); err != nil {
return errors.Errorf("migration filename must start with a number: %s", filename)
}
return nil
}
// Migrate migrates the database schema to the latest version.
// It checks the current schema version and applies any necessary migrations.
// It also seeds the database with initial data if in demo mode.
......@@ -42,7 +117,7 @@ func (s *Store) Migrate(ctx context.Context) error {
}
switch s.profile.Mode {
case "prod":
case modeProd:
workspaceBasicSetting, err := s.GetWorkspaceBasicSetting(ctx)
if err != nil {
return errors.Wrap(err, "failed to get workspace basic setting")
......@@ -51,53 +126,21 @@ func (s *Store) Migrate(ctx context.Context) error {
if err != nil {
return errors.Wrap(err, "failed to get current schema version")
}
if version.IsVersionGreaterThan(workspaceBasicSetting.SchemaVersion, currentSchemaVersion) {
// Check for downgrade (but skip if schema version is empty - that means fresh/old installation)
if !isVersionEmpty(workspaceBasicSetting.SchemaVersion) && version.IsVersionGreaterThan(workspaceBasicSetting.SchemaVersion, currentSchemaVersion) {
slog.Error("cannot downgrade schema version",
slog.String("databaseVersion", workspaceBasicSetting.SchemaVersion),
slog.String("currentVersion", currentSchemaVersion),
)
return errors.Errorf("cannot downgrade schema version from %s to %s", workspaceBasicSetting.SchemaVersion, currentSchemaVersion)
}
if version.IsVersionGreaterThan(currentSchemaVersion, workspaceBasicSetting.SchemaVersion) {
filePaths, err := fs.Glob(migrationFS, fmt.Sprintf("%s*/*.sql", s.getMigrationBasePath()))
if err != nil {
return errors.Wrap(err, "failed to read migration files")
}
sort.Strings(filePaths)
// Start a transaction to apply the latest schema.
tx, err := s.driver.GetDB().Begin()
if err != nil {
return errors.Wrap(err, "failed to start transaction")
}
defer tx.Rollback()
slog.Info("start migration", slog.String("currentSchemaVersion", workspaceBasicSetting.SchemaVersion), slog.String("targetSchemaVersion", currentSchemaVersion))
for _, filePath := range filePaths {
fileSchemaVersion, err := s.getSchemaVersionOfMigrateScript(filePath)
if err != nil {
return errors.Wrap(err, "failed to get schema version of migrate script")
}
if version.IsVersionGreaterThan(fileSchemaVersion, workspaceBasicSetting.SchemaVersion) && version.IsVersionGreaterOrEqualThan(currentSchemaVersion, fileSchemaVersion) {
bytes, err := migrationFS.ReadFile(filePath)
if err != nil {
return errors.Wrapf(err, "failed to read minor version migration file: %s", filePath)
}
stmt := string(bytes)
if err := s.execute(ctx, tx, stmt); err != nil {
return errors.Wrapf(err, "migrate error: %s", stmt)
}
}
}
if err := tx.Commit(); err != nil {
return errors.Wrap(err, "failed to commit transaction")
}
slog.Info("end migrate")
if err := s.updateCurrentSchemaVersion(ctx, currentSchemaVersion); err != nil {
return errors.Wrap(err, "failed to update current schema version")
// Apply migrations if needed (including when schema version is empty)
if isVersionEmpty(workspaceBasicSetting.SchemaVersion) || version.IsVersionGreaterThan(currentSchemaVersion, workspaceBasicSetting.SchemaVersion) {
if err := s.applyMigrations(ctx, workspaceBasicSetting.SchemaVersion, currentSchemaVersion); err != nil {
return errors.Wrap(err, "failed to apply migrations")
}
}
case "demo":
case modeDemo:
// In demo mode, we should seed the database.
if err := s.seed(ctx); err != nil {
return errors.Wrap(err, "failed to seed")
......@@ -108,6 +151,78 @@ func (s *Store) Migrate(ctx context.Context) error {
return nil
}
// applyMigrations applies all necessary migration files between current and target schema versions.
// It runs all migrations in a single transaction for atomicity.
func (s *Store) applyMigrations(ctx context.Context, currentSchemaVersion, targetSchemaVersion string) error {
filePaths, err := fs.Glob(migrationFS, fmt.Sprintf("%s*/*.sql", s.getMigrationBasePath()))
if err != nil {
return errors.Wrap(err, "failed to read migration files")
}
sort.Strings(filePaths)
// Start a transaction to apply migrations atomically
tx, err := s.driver.GetDB().Begin()
if err != nil {
return errors.Wrap(err, "failed to start transaction")
}
defer tx.Rollback()
// Use safe version for comparison (handles empty version case)
schemaVersionForComparison := getSchemaVersionOrDefault(currentSchemaVersion)
if isVersionEmpty(currentSchemaVersion) {
slog.Warn("schema version is empty, treating as default for migration comparison",
slog.String("defaultVersion", defaultSchemaVersion))
}
slog.Info("start migration",
slog.String("currentSchemaVersion", schemaVersionForComparison),
slog.String("targetSchemaVersion", targetSchemaVersion))
migrationsApplied := 0
for _, filePath := range filePaths {
fileSchemaVersion, err := s.getSchemaVersionOfMigrateScript(filePath)
if err != nil {
return errors.Wrap(err, "failed to get schema version of migrate script")
}
if shouldApplyMigration(fileSchemaVersion, currentSchemaVersion, targetSchemaVersion) {
// Validate migration filename before applying
filename := filepath.Base(filePath)
if err := validateMigrationFileName(filename); err != nil {
slog.Warn("migration file has invalid name but will be applied", slog.String("file", filePath), slog.String("error", err.Error()))
}
slog.Info("applying migration",
slog.String("file", filePath),
slog.String("version", fileSchemaVersion))
bytes, err := migrationFS.ReadFile(filePath)
if err != nil {
return errors.Wrapf(err, "failed to read migration file: %s", filePath)
}
stmt := string(bytes)
if err := s.execute(ctx, tx, stmt); err != nil {
return errors.Wrapf(err, "failed to execute migration %s: %s", filePath, err)
}
migrationsApplied++
}
}
if err := tx.Commit(); err != nil {
return errors.Wrap(err, "failed to commit migration transaction")
}
slog.Info("migration completed", slog.Int("migrationsApplied", migrationsApplied))
// Update schema version after successful migration
if err := s.updateCurrentSchemaVersion(ctx, targetSchemaVersion); err != nil {
return errors.Wrap(err, "failed to update current schema version")
}
return nil
}
// preMigrate checks if the database is initialized and applies the latest schema if not.
func (s *Store) preMigrate(ctx context.Context) error {
initialized, err := s.driver.IsInitialized(ctx)
......@@ -127,6 +242,7 @@ func (s *Store) preMigrate(ctx context.Context) error {
return errors.Wrap(err, "failed to start transaction")
}
defer tx.Rollback()
slog.Info("initializing new database with latest schema", slog.String("file", filePath))
if err := s.execute(ctx, tx, string(bytes)); err != nil {
return errors.Errorf("failed to execute SQL file %s, err %s", filePath, err)
}
......@@ -139,12 +255,13 @@ func (s *Store) preMigrate(ctx context.Context) error {
if err != nil {
return errors.Wrap(err, "failed to get current schema version")
}
slog.Info("database initialized successfully", slog.String("schemaVersion", schemaVersion))
if err := s.updateCurrentSchemaVersion(ctx, schemaVersion); err != nil {
return errors.Wrap(err, "failed to update current schema version")
}
}
if s.profile.Mode == "prod" {
if s.profile.Mode == modeProd {
if err := s.normalizeMigrationHistoryList(ctx); err != nil {
return errors.Wrap(err, "failed to normalize migration history list")
}
......@@ -165,11 +282,11 @@ func (s *Store) getSeedBasePath() string {
// seed seeds the database with initial data.
// It reads all seed files from the embedded filesystem and executes them in order.
// This is only supported for SQLite databases.
// This is only supported for SQLite databases and is used in demo mode.
func (s *Store) seed(ctx context.Context) error {
// Only seed for SQLite.
// Only seed for SQLite - other databases should use production data
if s.profile.Driver != "sqlite" {
slog.Warn("seed is only supported for SQLite")
slog.Warn("seed is only supported for SQLite, skipping for other databases")
return nil
}
......@@ -265,6 +382,8 @@ func (s *Store) updateCurrentSchemaVersion(ctx context.Context, schemaVersion st
// normalizeMigrationHistoryList normalizes the migration history list.
// It checks the existing migration history and updates it to the latest schema version if necessary.
// NOTE: This is a transition function for backward compatibility with the deprecated migration_history table.
// This ensures that old installations (< 0.22) have their migration_history normalized before migrating to workspace_setting.
func (s *Store) normalizeMigrationHistoryList(ctx context.Context) error {
migrationHistoryList, err := s.driver.FindMigrationHistoryList(ctx, &FindMigrationHistory{})
if err != nil {
......@@ -281,9 +400,9 @@ func (s *Store) normalizeMigrationHistoryList(ctx context.Context) error {
latestVersion := versions[len(versions)-1]
latestMinorVersion := version.GetMinorVersion(latestVersion)
// If the latest version is greater than 0.22, return.
// As of 0.22, the migration history is already normalized.
if version.IsVersionGreaterThan(latestMinorVersion, "0.22") {
// If the latest version is greater than migrationHistoryNormalizedVersion, return.
// As of that version, the migration history is already normalized.
if version.IsVersionGreaterThan(latestMinorVersion, migrationHistoryNormalizedVersion) {
return nil
}
......@@ -318,6 +437,9 @@ func (s *Store) normalizeMigrationHistoryList(ctx context.Context) error {
// migrateSchemaVersionToSetting migrates the schema version from the migration history to the workspace basic setting.
// It retrieves the migration history, sorts the versions, and updates the workspace basic setting if necessary.
// NOTE: This is a transition function for backward compatibility with the deprecated migration_history table.
// The migration_history table is deprecated in favor of storing schema version in workspace_setting.
// This handles upgrades from old installations that only have migration_history but no workspace_setting.
func (s *Store) migrateSchemaVersionToSetting(ctx context.Context) error {
migrationHistoryList, err := s.driver.FindMigrationHistoryList(ctx, &FindMigrationHistory{})
if err != nil {
......@@ -337,7 +459,14 @@ func (s *Store) migrateSchemaVersionToSetting(ctx context.Context) error {
if err != nil {
return errors.Wrap(err, "failed to get workspace basic setting")
}
if version.IsVersionGreaterOrEqualThan(workspaceBasicSetting.SchemaVersion, latestVersion) {
// If workspace_setting has no schema version (empty), or migration_history has a newer version, update workspace_setting.
// This handles upgrades from old installations where schema version was only tracked in migration_history.
if isVersionEmpty(workspaceBasicSetting.SchemaVersion) || version.IsVersionGreaterThan(latestVersion, workspaceBasicSetting.SchemaVersion) {
slog.Info("migrating schema version from migration_history to workspace_setting",
slog.String("from", workspaceBasicSetting.SchemaVersion),
slog.String("to", latestVersion),
)
if err := s.updateCurrentSchemaVersion(ctx, latestVersion); err != nil {
return errors.Wrap(err, "failed to update current schema version")
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment