chore: store vacuum and clean (#2293)

* Move all vacuum code into driver

* Remove db from Store

cmd/memos.go

@@ -56,8 +56,7 @@ var (
 			}
 
 			driver := sqlite.NewDriver(db.DBInstance)
-
-			store := store.New(db.DBInstance, driver, profile)
+			store := store.New(driver, profile)
 			s, err := server.NewServer(ctx, profile, store)
 			if err != nil {
 				cancel()

cmd/mvrss.go

@@ -51,8 +51,7 @@ var (
 			}
 
 			driver := sqlite.NewDriver(db.DBInstance)
-
-			s := store.New(db.DBInstance, driver, profile)
+			s := store.New(driver, profile)
 			resources, err := s.ListResources(ctx, &store.FindResource{})
 			if err != nil {
 				fmt.Printf("failed to list resources, error: %+v\n", err)

cmd/setup.go

@@ -48,8 +48,7 @@ var (
 			}
 
 			driver := sqlite.NewDriver(db.DBInstance)
-
-			store := store.New(db.DBInstance, driver, profile)
+			store := store.New(driver, profile)
 			if err := ExecuteSetup(ctx, store, hostUsername, hostPassword); err != nil {
 				fmt.Printf("failed to setup, error: %+v\n", err)
 				return

server/server.go

@@ -162,7 +162,7 @@ func (s *Server) Shutdown(ctx context.Context) {
 	}
 
 	// Close database connection
-	if err := s.Store.GetDB().Close(); err != nil {
+	if err := s.Store.Close(); err != nil {
 		fmt.Printf("failed to close database, error: %v\n", err)
 	}
 

store/driver.go

@@ -7,6 +7,10 @@ import (
 )
 
 type Driver interface {
+	Vacuum(ctx context.Context) error
+	BackupTo(ctx context.Context, filename string) error
+	Close() error
+
 	CreateActivity(ctx context.Context, create *Activity) (*Activity, error)
 
 	CreateResource(ctx context.Context, create *Resource) (*Resource, error)

store/memo.go

@@ -2,7 +2,6 @@ package store
 
 import (
 	"context"
-	"database/sql"
 )
 
 // Visibility is the type of a visibility.
@@ -107,35 +106,9 @@ func (s *Store) UpdateMemo(ctx context.Context, update *UpdateMemo) error {
 }
 
 func (s *Store) DeleteMemo(ctx context.Context, delete *DeleteMemo) error {
-	if err := s.driver.DeleteMemo(ctx, delete); err != nil {
-		return err
-	}
-	if err := s.Vacuum(ctx); err != nil {
-		// Prevent linter warning.
-		return err
-	}
-	return nil
+	return s.driver.DeleteMemo(ctx, delete)
 }
 
 func (s *Store) FindMemosVisibilityList(ctx context.Context, memoIDs []int32) ([]Visibility, error) {
 	return s.driver.FindMemosVisibilityList(ctx, memoIDs)
 }
-
-func vacuumMemo(ctx context.Context, tx *sql.Tx) error {
-	stmt := `
-	DELETE FROM 
-		memo 
-	WHERE 
-		creator_id NOT IN (
-			SELECT 
-				id 
-			FROM 
-				user
-		)`
-	_, err := tx.ExecContext(ctx, stmt)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}

store/memo_organizer.go

@@ -2,7 +2,6 @@ package store
 
 import (
 	"context"
-	"database/sql"
 )
 
 type MemoOrganizer struct {
@@ -32,28 +31,3 @@ func (s *Store) GetMemoOrganizer(ctx context.Context, find *FindMemoOrganizer) (
 func (s *Store) DeleteMemoOrganizer(ctx context.Context, delete *DeleteMemoOrganizer) error {
 	return s.driver.DeleteMemoOrganizer(ctx, delete)
 }
-
-func vacuumMemoOrganizer(ctx context.Context, tx *sql.Tx) error {
-	stmt := `
-	DELETE FROM 
-		memo_organizer 
-	WHERE 
-		memo_id NOT IN (
-			SELECT 
-				id 
-			FROM 
-				memo
-		)
-		OR user_id NOT IN (
-			SELECT 
-				id 
-			FROM 
-				user
-		)`
-	_, err := tx.ExecContext(ctx, stmt)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}

store/memo_relation.go

@@ -2,7 +2,6 @@ package store
 
 import (
 	"context"
-	"database/sql"
 )
 
 type MemoRelationType string
@@ -54,13 +53,3 @@ func (s *Store) GetMemoRelation(ctx context.Context, find *FindMemoRelation) (*M
 func (s *Store) DeleteMemoRelation(ctx context.Context, delete *DeleteMemoRelation) error {
 	return s.driver.DeleteMemoRelation(ctx, delete)
 }
-
-func vacuumMemoRelations(ctx context.Context, tx *sql.Tx) error {
-	if _, err := tx.ExecContext(ctx, `
-		DELETE FROM memo_relation
-		WHERE memo_id NOT IN (SELECT id FROM memo) OR related_memo_id NOT IN (SELECT id FROM memo)
-	`); err != nil {
-		return err
-	}
-	return nil
-}

store/resource.go

@@ -2,7 +2,6 @@ package store
 
 import (
 	"context"
-	"database/sql"
 )
 
 type Resource struct {
@@ -75,33 +74,5 @@ func (s *Store) UpdateResource(ctx context.Context, update *UpdateResource) (*Re
 }
 
 func (s *Store) DeleteResource(ctx context.Context, delete *DeleteResource) error {
-	err := s.driver.DeleteResource(ctx, delete)
-	if err != nil {
-		return err
-	}
-
-	if err := s.Vacuum(ctx); err != nil {
-		// Prevent linter warning.
-		return err
-	}
-	return nil
-}
-
-func vacuumResource(ctx context.Context, tx *sql.Tx) error {
-	stmt := `
-	DELETE FROM 
-		resource 
-	WHERE 
-		creator_id NOT IN (
-			SELECT 
-				id 
-			FROM 
-				user
-		)`
-	_, err := tx.ExecContext(ctx, stmt)
-	if err != nil {
-		return err
-	}
-
-	return nil
+	return s.driver.DeleteResource(ctx, delete)
 }

store/sqlite/memo.go

@@ -234,6 +234,10 @@ func (d *Driver) DeleteMemo(ctx context.Context, delete *store.DeleteMemo) error
 		return err
 	}
 
+	if err := d.Vacuum(ctx); err != nil {
+		// Prevent linter warning.
+		return err
+	}
 	return nil
 }
 
@@ -268,3 +272,22 @@ func (d *Driver) FindMemosVisibilityList(ctx context.Context, memoIDs []int32) (
 
 	return visibilityList, nil
 }
+
+func vacuumMemo(ctx context.Context, tx *sql.Tx) error {
+	stmt := `
+	DELETE FROM 
+		memo 
+	WHERE 
+		creator_id NOT IN (
+			SELECT 
+				id 
+			FROM 
+				user
+		)`
+	_, err := tx.ExecContext(ctx, stmt)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

store/sqlite/memo_organizer.go

@@ -2,6 +2,7 @@ package sqlite
 
 import (
 	"context"
+	"database/sql"
 	"fmt"
 	"strings"
 
@@ -80,3 +81,28 @@ func (d *Driver) DeleteMemoOrganizer(ctx context.Context, delete *store.DeleteMe
 	}
 	return nil
 }
+
+func vacuumMemoOrganizer(ctx context.Context, tx *sql.Tx) error {
+	stmt := `
+	DELETE FROM 
+		memo_organizer 
+	WHERE 
+		memo_id NOT IN (
+			SELECT 
+				id 
+			FROM 
+				memo
+		)
+		OR user_id NOT IN (
+			SELECT 
+				id 
+			FROM 
+				user
+		)`
+	_, err := tx.ExecContext(ctx, stmt)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

store/sqlite/memo_relation.go

@@ -2,6 +2,7 @@ package sqlite
 
 import (
 	"context"
+	"database/sql"
 	"strings"
 
 	"github.com/usememos/memos/store"
@@ -104,3 +105,13 @@ func (d *Driver) DeleteMemoRelation(ctx context.Context, delete *store.DeleteMem
 	}
 	return nil
 }
+
+func vacuumMemoRelations(ctx context.Context, tx *sql.Tx) error {
+	if _, err := tx.ExecContext(ctx, `
+		DELETE FROM memo_relation
+		WHERE memo_id NOT IN (SELECT id FROM memo) OR related_memo_id NOT IN (SELECT id FROM memo)
+	`); err != nil {
+		return err
+	}
+	return nil
+}

store/sqlite/resource.go

@@ -181,5 +181,29 @@ func (d *Driver) DeleteResource(ctx context.Context, delete *store.DeleteResourc
 		return err
 	}
 
+	if err := d.Vacuum(ctx); err != nil {
+		// Prevent linter warning.
+		return err
+	}
+
+	return nil
+}
+
+func vacuumResource(ctx context.Context, tx *sql.Tx) error {
+	stmt := `
+	DELETE FROM 
+		resource 
+	WHERE 
+		creator_id NOT IN (
+			SELECT 
+				id 
+			FROM 
+				user
+		)`
+	_, err := tx.ExecContext(ctx, stmt)
+	if err != nil {
+		return err
+	}
+
 	return nil
 }

store/sqlite/sqlite.go

 package sqlite
 
 import (
+	"context"
 	"database/sql"
 
+	"github.com/pkg/errors"
+	"modernc.org/sqlite"
+
 	"github.com/usememos/memos/store"
 )
 
@@ -15,3 +19,91 @@ func NewDriver(db *sql.DB) store.Driver {
 		db: db,
 	}
 }
+
+func (d *Driver) Vacuum(ctx context.Context) error {
+	tx, err := d.db.BeginTx(ctx, nil)
+	if err != nil {
+		return err
+	}
+	defer tx.Rollback()
+
+	if err := vacuumImpl(ctx, tx); err != nil {
+		return err
+	}
+
+	if err := tx.Commit(); err != nil {
+		return err
+	}
+
+	// Vacuum sqlite database file size after deleting resource.
+	if _, err := d.db.Exec("VACUUM"); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func vacuumImpl(ctx context.Context, tx *sql.Tx) error {
+	if err := vacuumMemo(ctx, tx); err != nil {
+		return err
+	}
+	if err := vacuumResource(ctx, tx); err != nil {
+		return err
+	}
+	if err := vacuumUserSetting(ctx, tx); err != nil {
+		return err
+	}
+	if err := vacuumMemoOrganizer(ctx, tx); err != nil {
+		return err
+	}
+	if err := vacuumMemoRelations(ctx, tx); err != nil {
+		return err
+	}
+	if err := vacuumTag(ctx, tx); err != nil {
+		// Prevent revive warning.
+		return err
+	}
+
+	return nil
+}
+
+func (d *Driver) BackupTo(ctx context.Context, filename string) error {
+	conn, err := d.db.Conn(ctx)
+	if err != nil {
+		return errors.Errorf("fail to get conn %s", err)
+	}
+	defer conn.Close()
+
+	err = conn.Raw(func(driverConn any) error {
+		type backuper interface {
+			NewBackup(string) (*sqlite.Backup, error)
+		}
+		backupConn, ok := driverConn.(backuper)
+		if !ok {
+			return errors.Errorf("db connection is not a sqlite backuper")
+		}
+
+		bck, err := backupConn.NewBackup(filename)
+		if err != nil {
+			return errors.Errorf("fail to create sqlite backup %s", err)
+		}
+
+		for more := true; more; {
+			more, err = bck.Step(-1)
+			if err != nil {
+				return errors.Errorf("fail to execute sqlite backup %s", err)
+			}
+		}
+
+		return bck.Finish()
+	})
+	if err != nil {
+		return errors.Errorf("fail to backup %s", err)
+	}
+
+	return nil
+}
+
+func (d *Driver) Close() error {
+	return d.db.Close()
+}

store/sqlite/tag.go

@@ -2,6 +2,7 @@ package sqlite
 
 import (
 	"context"
+	"database/sql"
 	"strings"
 
 	"github.com/usememos/memos/store"
@@ -73,3 +74,22 @@ func (d *Driver) DeleteTag(ctx context.Context, delete *store.DeleteTag) error {
 	}
 	return nil
 }
+
+func vacuumTag(ctx context.Context, tx *sql.Tx) error {
+	stmt := `
+	DELETE FROM 
+		tag 
+	WHERE 
+		creator_id NOT IN (
+			SELECT 
+				id 
+			FROM 
+				user
+		)`
+	_, err := tx.ExecContext(ctx, stmt)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

store/sqlite/user.go

@@ -168,5 +168,11 @@ func (d *Driver) DeleteUser(ctx context.Context, delete *store.DeleteUser) error
 	if _, err := result.RowsAffected(); err != nil {
 		return err
 	}
+
+	if err := d.Vacuum(ctx); err != nil {
+		// Prevent linter warning.
+		return err
+	}
+
 	return nil
 }

store/sqlite/user_setting.go

@@ -2,6 +2,7 @@ package sqlite
 
 import (
 	"context"
+	"database/sql"
 	"errors"
 	"strings"
 
@@ -153,3 +154,22 @@ func (d *Driver) ListUserSettingsV1(ctx context.Context, find *store.FindUserSet
 
 	return userSettingList, nil
 }
+
+func vacuumUserSetting(ctx context.Context, tx *sql.Tx) error {
+	stmt := `
+	DELETE FROM 
+		user_setting 
+	WHERE 
+		user_id NOT IN (
+			SELECT 
+				id 
+			FROM 
+				user
+		)`
+	_, err := tx.ExecContext(ctx, stmt)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

store/store.go

@@ -2,20 +2,14 @@ package store
 
 import (
 	"context"
-	"database/sql"
 	"sync"
 
-	"modernc.org/sqlite"
-
-	"github.com/pkg/errors"
-
 	"github.com/usememos/memos/server/profile"
 )
 
 // Store provides database access to all raw objects.
 type Store struct {
 	Profile            *profile.Profile
-	db                 *sql.DB
 	driver             Driver
 	systemSettingCache sync.Map // map[string]*SystemSetting
 	userCache          sync.Map // map[int]*User
@@ -24,98 +18,21 @@ type Store struct {
 }
 
 // New creates a new instance of Store.
-func New(db *sql.DB, driver Driver, profile *profile.Profile) *Store {
+func New(driver Driver, profile *profile.Profile) *Store {
 	return &Store{
 		Profile: profile,
-		db:      db,
 		driver:  driver,
 	}
 }
 
-func (s *Store) GetDB() *sql.DB {
-	return s.db
-}
-
 func (s *Store) BackupTo(ctx context.Context, filename string) error {
-	conn, err := s.db.Conn(ctx)
-	if err != nil {
-		return errors.Errorf("fail to get conn %s", err)
-	}
-	defer conn.Close()
-
-	err = conn.Raw(func(driverConn any) error {
-		type backuper interface {
-			NewBackup(string) (*sqlite.Backup, error)
-		}
-		backupConn, ok := driverConn.(backuper)
-		if !ok {
-			return errors.Errorf("db connection is not a sqlite backuper")
-		}
-
-		bck, err := backupConn.NewBackup(filename)
-		if err != nil {
-			return errors.Errorf("fail to create sqlite backup %s", err)
-		}
-
-		for more := true; more; {
-			more, err = bck.Step(-1)
-			if err != nil {
-				return errors.Errorf("fail to execute sqlite backup %s", err)
-			}
-		}
-
-		return bck.Finish()
-	})
-	if err != nil {
-		return errors.Errorf("fail to backup %s", err)
-	}
-
-	return nil
+	return s.driver.BackupTo(ctx, filename)
 }
 
 func (s *Store) Vacuum(ctx context.Context) error {
-	tx, err := s.db.BeginTx(ctx, nil)
-	if err != nil {
-		return err
-	}
-	defer tx.Rollback()
-
-	if err := s.vacuumImpl(ctx, tx); err != nil {
-		return err
-	}
-
-	if err := tx.Commit(); err != nil {
-		return err
-	}
-
-	// Vacuum sqlite database file size after deleting resource.
-	if _, err := s.db.Exec("VACUUM"); err != nil {
-		return err
-	}
-
-	return nil
+	return s.driver.Vacuum(ctx)
 }
 
-func (*Store) vacuumImpl(ctx context.Context, tx *sql.Tx) error {
-	if err := vacuumMemo(ctx, tx); err != nil {
-		return err
-	}
-	if err := vacuumResource(ctx, tx); err != nil {
-		return err
-	}
-	if err := vacuumUserSetting(ctx, tx); err != nil {
-		return err
-	}
-	if err := vacuumMemoOrganizer(ctx, tx); err != nil {
-		return err
-	}
-	if err := vacuumMemoRelations(ctx, tx); err != nil {
-		return err
-	}
-	if err := vacuumTag(ctx, tx); err != nil {
-		// Prevent revive warning.
-		return err
-	}
-
-	return nil
+func (s *Store) Close() error {
+	return s.driver.Close()
 }

store/tag.go

@@ -2,7 +2,6 @@ package store
 
 import (
 	"context"
-	"database/sql"
 )
 
 type Tag struct {
@@ -30,22 +29,3 @@ func (s *Store) ListTags(ctx context.Context, find *FindTag) ([]*Tag, error) {
 func (s *Store) DeleteTag(ctx context.Context, delete *DeleteTag) error {
 	return s.driver.DeleteTag(ctx, delete)
 }
-
-func vacuumTag(ctx context.Context, tx *sql.Tx) error {
-	stmt := `
-	DELETE FROM 
-		tag 
-	WHERE 
-		creator_id NOT IN (
-			SELECT 
-				id 
-			FROM 
-				user
-		)`
-	_, err := tx.ExecContext(ctx, stmt)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}

store/user.go

@@ -130,10 +130,6 @@ func (s *Store) DeleteUser(ctx context.Context, delete *DeleteUser) error {
 		return err
 	}
 
-	if err := s.Vacuum(ctx); err != nil {
-		// Prevent linter warning.
-		return err
-	}
 	s.userCache.Delete(delete.ID)
 	return nil
 }

store/user_setting.go

@@ -2,7 +2,6 @@ package store
 
 import (
 	"context"
-	"database/sql"
 
 	storepb "github.com/usememos/memos/proto/gen/store"
 )
@@ -125,22 +124,3 @@ func (s *Store) GetUserAccessTokens(ctx context.Context, userID int32) ([]*store
 	accessTokensUserSetting := userSetting.GetAccessTokens()
 	return accessTokensUserSetting.AccessTokens, nil
 }
-
-func vacuumUserSetting(ctx context.Context, tx *sql.Tx) error {
-	stmt := `
-	DELETE FROM 
-		user_setting 
-	WHERE 
-		user_id NOT IN (
-			SELECT 
-				id 
-			FROM 
-				user
-		)`
-	_, err := tx.ExecContext(ctx, stmt)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}

test/server/server.go

@@ -41,8 +41,7 @@ func NewTestingServer(ctx context.Context, t *testing.T) (*TestingServer, error)
 	}
 
 	driver := sqlite.NewDriver(db.DBInstance)
-
-	store := store.New(db.DBInstance, driver, profile)
+	store := store.New(driver, profile)
 	server, err := server.NewServer(ctx, profile, store)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to create server")

test/store/store.go

@@ -25,7 +25,6 @@ func NewTestingStore(ctx context.Context, t *testing.T) *store.Store {
 	}
 
 	driver := sqlite.NewDriver(db.DBInstance)
-
-	store := store.New(db.DBInstance, driver, profile)
+	store := store.New(driver, profile)
 	return store
 }