refactor: implement s3 storage

docs/apidocs.swagger.yaml

@@ -1755,36 +1755,7 @@ paths:
                 format: date-time
       tags:
         - UserService
-  /o/r/{uid}:
-    get:
-      summary: GetResourceBinary returns a resource binary by name.
-      operationId: ResourceService_GetResourceBinary2
-      responses:
-        "200":
-          description: A successful response.
-          schema:
-            $ref: '#/definitions/apiHttpBody'
-        default:
-          description: An unexpected error response.
-          schema:
-            $ref: '#/definitions/googlerpcStatus'
-      parameters:
-        - name: uid
-          description: The user defined id of the resource.
-          in: path
-          required: true
-          type: string
-        - name: name
-          description: |-
-            The name of the resource.
-            Format: resources/{id}
-            id is the system generated unique identifier.
-          in: query
-          required: false
-          type: string
-      tags:
-        - ResourceService
-  /o/{name}:
+  /file/{name}:
     get:
       summary: GetResourceBinary returns a resource binary by name.
       operationId: ResourceService_GetResourceBinary
@@ -1814,7 +1785,7 @@ paths:
           type: string
       tags:
         - ResourceService
-  /o/{name}/avatar:
+  /file/{name}/avatar:
     get:
       summary: GetUserAvatarBinary gets the avatar of a user.
       operationId: UserService_GetUserAvatarBinary
@@ -1849,6 +1820,35 @@ paths:
           format: byte
       tags:
         - UserService
+  /o/r/{uid}:
+    get:
+      summary: GetResourceBinary returns a resource binary by name.
+      operationId: ResourceService_GetResourceBinary2
+      responses:
+        "200":
+          description: A successful response.
+          schema:
+            $ref: '#/definitions/apiHttpBody'
+        default:
+          description: An unexpected error response.
+          schema:
+            $ref: '#/definitions/googlerpcStatus'
+      parameters:
+        - name: uid
+          description: The user defined id of the resource.
+          in: path
+          required: true
+          type: string
+        - name: name
+          description: |-
+            The name of the resource.
+            Format: resources/{id}
+            id is the system generated unique identifier.
+          in: query
+          required: false
+          type: string
+      tags:
+        - ResourceService
 definitions:
   MemoServiceSetMemoRelationsBody:
     type: object

plugin/storage/s3/s3.go

@@ -6,66 +6,76 @@ import (
 	"time"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
-	s3config "github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/config"
 	"github.com/aws/aws-sdk-go-v2/credentials"
 	"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
-	awss3 "github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/pkg/errors"
-)
 
-const LinkLifetime = 24 * time.Hour
+	storepb "github.com/usememos/memos/proto/gen/store"
+)
 
-type Config struct {
-	AccessKeyID     string
-	AcesssKeySecret string
-	Endpoint        string
-	Region          string
-	Bucket          string
-}
+const presignLifetimeSecs = 7 * 24 * 60 * 60
 
 type Client struct {
-	Client *awss3.Client
-	Config *Config
+	Client *s3.Client
+	Bucket *string
 }
 
-func NewClient(ctx context.Context, config *Config) (*Client, error) {
+func NewClient(ctx context.Context, s3Config *storepb.WorkspaceStorageSetting_S3Config) (*Client, error) {
 	resolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...any) (aws.Endpoint, error) {
 		return aws.Endpoint{
-			URL: config.Endpoint,
+			URL: s3Config.Endpoint,
 		}, nil
 	})
-	s3Config, err := s3config.LoadDefaultConfig(ctx,
-		s3config.WithEndpointResolverWithOptions(resolver),
-		s3config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(config.AccessKeyID, config.AcesssKeySecret, "")),
-		s3config.WithRegion(config.Region),
+	cfg, err := config.LoadDefaultConfig(ctx,
+		config.WithEndpointResolverWithOptions(resolver),
+		config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(s3Config.AccessKeyId, s3Config.AccessKeySecret, "")),
+		config.WithRegion(s3Config.Region),
 	)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to load s3 config")
 	}
 
-	client := awss3.NewFromConfig(s3Config)
+	client := s3.NewFromConfig(cfg)
 	return &Client{
 		Client: client,
-		Config: config,
+		Bucket: aws.String(s3Config.Bucket),
 	}, nil
 }
 
-func (client *Client) UploadFile(ctx context.Context, filename string, fileType string, src io.Reader) (string, error) {
+// UploadObject uploads an object to S3.
+func (client *Client) UploadObject(ctx context.Context, key string, fileType string, content io.Reader) (string, error) {
 	uploader := manager.NewUploader(client.Client)
-	putInput := awss3.PutObjectInput{
-		Bucket:      aws.String(client.Config.Bucket),
-		Key:         aws.String(filename),
-		Body:        src,
+	putInput := s3.PutObjectInput{
+		Bucket:      client.Bucket,
+		Key:         aws.String(key),
 		ContentType: aws.String(fileType),
+		Body:        content,
 	}
-	uploadOutput, err := uploader.Upload(ctx, &putInput)
+	result, err := uploader.Upload(ctx, &putInput)
 	if err != nil {
 		return "", err
 	}
 
-	link := uploadOutput.Location
-	if link == "" {
-		return "", errors.New("failed to get file link")
+	resultKey := result.Key
+	if resultKey == nil || *resultKey == "" {
+		return "", errors.New("failed to get file key")
+	}
+	return *resultKey, nil
+}
+
+// PresignGetObject presigns an object in S3.
+func (client *Client) PresignGetObject(ctx context.Context, bucket, key string) (string, error) {
+	presignClient := s3.NewPresignClient(client.Client)
+	presignResult, err := presignClient.PresignGetObject(context.TODO(), &s3.GetObjectInput{
+		Bucket: aws.String(bucket),
+		Key:    aws.String(key),
+	}, func(opts *s3.PresignOptions) {
+		opts.Expires = time.Duration(presignLifetimeSecs * int64(time.Second))
+	})
+	if err != nil {
+		return "", errors.Wrap(err, "failed to presign put object")
 	}
-	return link, nil
+	return presignResult.URL, nil
 }

proto/api/v1/resource_service.proto

@@ -36,9 +36,12 @@ service ResourceService {
   // GetResourceBinary returns a resource binary by name.
   rpc GetResourceBinary(GetResourceBinaryRequest) returns (google.api.HttpBody) {
     option (google.api.http) = {
-      get: "/o/{name=resources/*}"
+      get: "/file/{name=resources/*}"
 
-      additional_bindings {get: "/o/r/{uid}"}
+      additional_bindings {
+        // DEPRECATED: Will be removed in the future. Use `/file/{name}` instead.
+        get: "/o/r/{uid}"
+      }
     };
     option (google.api.method_signature) = "name,uid";
   }

proto/api/v1/user_service.proto

@@ -29,7 +29,7 @@ service UserService {
   }
   // GetUserAvatarBinary gets the avatar of a user.
   rpc GetUserAvatarBinary(GetUserAvatarBinaryRequest) returns (google.api.HttpBody) {
-    option (google.api.http) = {get: "/o/{name=users/*}/avatar"};
+    option (google.api.http) = {get: "/file/{name=users/*}/avatar"};
     option (google.api.method_signature) = "name";
   }
   // CreateUser creates a new user.

proto/gen/api/v1/resource_service.pb.go

@@ -656,7 +656,7 @@ var file_api_v1_resource_service_proto_rawDesc = []byte{
 	0x61, 0x73, 0x6b, 0x52, 0x0a, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x4d, 0x61, 0x73, 0x6b, 0x22,
 	0x2b, 0x0a, 0x15, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63,
 	0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65,
-	0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x32, 0x95, 0x07, 0x0a,
+	0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x32, 0x98, 0x07, 0x0a,
 	0x0f, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65,
 	0x12, 0x72, 0x0a, 0x0e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72,
 	0x63, 0x65, 0x12, 0x23, 0x2e, 0x6d, 0x65, 0x6d, 0x6f, 0x73, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76,
@@ -688,44 +688,44 @@ var file_api_v1_resource_service_proto_rawDesc = []byte{
 	0x6f, 0x75, 0x72, 0x63, 0x65, 0x22, 0x29, 0xda, 0x41, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x82, 0xd3,
 	0xe4, 0x93, 0x02, 0x1c, 0x12, 0x1a, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x76, 0x31, 0x2f, 0x7b, 0x6e,
 	0x61, 0x6d, 0x65, 0x3d, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x2f, 0x2a, 0x7d,
-	0x12, 0x89, 0x01, 0x0a, 0x11, 0x47, 0x65, 0x74, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65,
+	0x12, 0x8c, 0x01, 0x0a, 0x11, 0x47, 0x65, 0x74, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65,
 	0x42, 0x69, 0x6e, 0x61, 0x72, 0x79, 0x12, 0x26, 0x2e, 0x6d, 0x65, 0x6d, 0x6f, 0x73, 0x2e, 0x61,
 	0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63,
 	0x65, 0x42, 0x69, 0x6e, 0x61, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14,
 	0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x48, 0x74, 0x74, 0x70,
-	0x42, 0x6f, 0x64, 0x79, 0x22, 0x36, 0xda, 0x41, 0x08, 0x6e, 0x61, 0x6d, 0x65, 0x2c, 0x75, 0x69,
-	0x64, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x25, 0x5a, 0x0c, 0x12, 0x0a, 0x2f, 0x6f, 0x2f, 0x72, 0x2f,
-	0x7b, 0x75, 0x69, 0x64, 0x7d, 0x12, 0x15, 0x2f, 0x6f, 0x2f, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x3d,
-	0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x2f, 0x2a, 0x7d, 0x12, 0x9b, 0x01, 0x0a,
-	0x0e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12,
-	0x23, 0x2e, 0x6d, 0x65, 0x6d, 0x6f, 0x73, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x55,
-	0x70, 0x64, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, 0x65, 0x71,
-	0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x6d, 0x65, 0x6d, 0x6f, 0x73, 0x2e, 0x61, 0x70, 0x69,
-	0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x22, 0x4c, 0xda, 0x41,
-	0x14, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2c, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65,
-	0x5f, 0x6d, 0x61, 0x73, 0x6b, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x2f, 0x3a, 0x08, 0x72, 0x65, 0x73,
-	0x6f, 0x75, 0x72, 0x63, 0x65, 0x32, 0x23, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x76, 0x31, 0x2f, 0x7b,
-	0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x6e, 0x61, 0x6d, 0x65, 0x3d, 0x72, 0x65,
-	0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x2f, 0x2a, 0x7d, 0x12, 0x78, 0x0a, 0x0e, 0x44, 0x65,
-	0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x23, 0x2e, 0x6d,
-	0x65, 0x6d, 0x6f, 0x73, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x65, 0x6c, 0x65,
-	0x74, 0x65, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
-	0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f,
-	0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x29, 0xda, 0x41, 0x04, 0x6e, 0x61,
-	0x6d, 0x65, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x1c, 0x2a, 0x1a, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x76,
-	0x31, 0x2f, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x3d, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65,
-	0x73, 0x2f, 0x2a, 0x7d, 0x42, 0xac, 0x01, 0x0a, 0x10, 0x63, 0x6f, 0x6d, 0x2e, 0x6d, 0x65, 0x6d,
-	0x6f, 0x73, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x42, 0x14, 0x52, 0x65, 0x73, 0x6f, 0x75,
-	0x72, 0x63, 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50,
-	0x01, 0x5a, 0x30, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x75, 0x73,
-	0x65, 0x6d, 0x65, 0x6d, 0x6f, 0x73, 0x2f, 0x6d, 0x65, 0x6d, 0x6f, 0x73, 0x2f, 0x70, 0x72, 0x6f,
-	0x74, 0x6f, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x76, 0x31, 0x3b, 0x61, 0x70,
-	0x69, 0x76, 0x31, 0xa2, 0x02, 0x03, 0x4d, 0x41, 0x58, 0xaa, 0x02, 0x0c, 0x4d, 0x65, 0x6d, 0x6f,
-	0x73, 0x2e, 0x41, 0x70, 0x69, 0x2e, 0x56, 0x31, 0xca, 0x02, 0x0c, 0x4d, 0x65, 0x6d, 0x6f, 0x73,
-	0x5c, 0x41, 0x70, 0x69, 0x5c, 0x56, 0x31, 0xe2, 0x02, 0x18, 0x4d, 0x65, 0x6d, 0x6f, 0x73, 0x5c,
-	0x41, 0x70, 0x69, 0x5c, 0x56, 0x31, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61,
-	0x74, 0x61, 0xea, 0x02, 0x0e, 0x4d, 0x65, 0x6d, 0x6f, 0x73, 0x3a, 0x3a, 0x41, 0x70, 0x69, 0x3a,
-	0x3a, 0x56, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+	0x42, 0x6f, 0x64, 0x79, 0x22, 0x39, 0xda, 0x41, 0x08, 0x6e, 0x61, 0x6d, 0x65, 0x2c, 0x75, 0x69,
+	0x64, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x28, 0x5a, 0x0c, 0x12, 0x0a, 0x2f, 0x6f, 0x2f, 0x72, 0x2f,
+	0x7b, 0x75, 0x69, 0x64, 0x7d, 0x12, 0x18, 0x2f, 0x66, 0x69, 0x6c, 0x65, 0x2f, 0x7b, 0x6e, 0x61,
+	0x6d, 0x65, 0x3d, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x2f, 0x2a, 0x7d, 0x12,
+	0x9b, 0x01, 0x0a, 0x0e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72,
+	0x63, 0x65, 0x12, 0x23, 0x2e, 0x6d, 0x65, 0x6d, 0x6f, 0x73, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76,
+	0x31, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65,
+	0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x6d, 0x65, 0x6d, 0x6f, 0x73, 0x2e,
+	0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x22,
+	0x4c, 0xda, 0x41, 0x14, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2c, 0x75, 0x70, 0x64,
+	0x61, 0x74, 0x65, 0x5f, 0x6d, 0x61, 0x73, 0x6b, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x2f, 0x3a, 0x08,
+	0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x32, 0x23, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x76,
+	0x31, 0x2f, 0x7b, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x6e, 0x61, 0x6d, 0x65,
+	0x3d, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x2f, 0x2a, 0x7d, 0x12, 0x78, 0x0a,
+	0x0e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12,
+	0x23, 0x2e, 0x6d, 0x65, 0x6d, 0x6f, 0x73, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x44,
+	0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, 0x65, 0x71,
+	0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72,
+	0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x29, 0xda, 0x41,
+	0x04, 0x6e, 0x61, 0x6d, 0x65, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x1c, 0x2a, 0x1a, 0x2f, 0x61, 0x70,
+	0x69, 0x2f, 0x76, 0x31, 0x2f, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x3d, 0x72, 0x65, 0x73, 0x6f, 0x75,
+	0x72, 0x63, 0x65, 0x73, 0x2f, 0x2a, 0x7d, 0x42, 0xac, 0x01, 0x0a, 0x10, 0x63, 0x6f, 0x6d, 0x2e,
+	0x6d, 0x65, 0x6d, 0x6f, 0x73, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x42, 0x14, 0x52, 0x65,
+	0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x50, 0x72, 0x6f,
+	0x74, 0x6f, 0x50, 0x01, 0x5a, 0x30, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d,
+	0x2f, 0x75, 0x73, 0x65, 0x6d, 0x65, 0x6d, 0x6f, 0x73, 0x2f, 0x6d, 0x65, 0x6d, 0x6f, 0x73, 0x2f,
+	0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x76, 0x31,
+	0x3b, 0x61, 0x70, 0x69, 0x76, 0x31, 0xa2, 0x02, 0x03, 0x4d, 0x41, 0x58, 0xaa, 0x02, 0x0c, 0x4d,
+	0x65, 0x6d, 0x6f, 0x73, 0x2e, 0x41, 0x70, 0x69, 0x2e, 0x56, 0x31, 0xca, 0x02, 0x0c, 0x4d, 0x65,
+	0x6d, 0x6f, 0x73, 0x5c, 0x41, 0x70, 0x69, 0x5c, 0x56, 0x31, 0xe2, 0x02, 0x18, 0x4d, 0x65, 0x6d,
+	0x6f, 0x73, 0x5c, 0x41, 0x70, 0x69, 0x5c, 0x56, 0x31, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74,
+	0x61, 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, 0x0e, 0x4d, 0x65, 0x6d, 0x6f, 0x73, 0x3a, 0x3a, 0x41,
+	0x70, 0x69, 0x3a, 0x3a, 0x56, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
 }
 
 var (

proto/gen/api/v1/resource_service.pb.gw.go

@@ -569,7 +569,7 @@ func RegisterResourceServiceHandlerServer(ctx context.Context, mux *runtime.Serv
 		inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req)
 		var err error
 		var annotatedContext context.Context
-		annotatedContext, err = runtime.AnnotateIncomingContext(ctx, mux, req, "/memos.api.v1.ResourceService/GetResourceBinary", runtime.WithHTTPPathPattern("/o/{name=resources/*}"))
+		annotatedContext, err = runtime.AnnotateIncomingContext(ctx, mux, req, "/memos.api.v1.ResourceService/GetResourceBinary", runtime.WithHTTPPathPattern("/file/{name=resources/*}"))
 		if err != nil {
 			runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err)
 			return
@@ -796,7 +796,7 @@ func RegisterResourceServiceHandlerClient(ctx context.Context, mux *runtime.Serv
 		inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req)
 		var err error
 		var annotatedContext context.Context
-		annotatedContext, err = runtime.AnnotateContext(ctx, mux, req, "/memos.api.v1.ResourceService/GetResourceBinary", runtime.WithHTTPPathPattern("/o/{name=resources/*}"))
+		annotatedContext, err = runtime.AnnotateContext(ctx, mux, req, "/memos.api.v1.ResourceService/GetResourceBinary", runtime.WithHTTPPathPattern("/file/{name=resources/*}"))
 		if err != nil {
 			runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err)
 			return
@@ -890,7 +890,7 @@ var (
 
 	pattern_ResourceService_GetResource_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2, 1, 0, 4, 2, 5, 3}, []string{"api", "v1", "resources", "name"}, ""))
 
-	pattern_ResourceService_GetResourceBinary_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 1, 0, 4, 2, 5, 2}, []string{"o", "resources", "name"}, ""))
+	pattern_ResourceService_GetResourceBinary_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 1, 0, 4, 2, 5, 2}, []string{"file", "resources", "name"}, ""))
 
 	pattern_ResourceService_GetResourceBinary_1 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 1, 0, 4, 1, 5, 2}, []string{"o", "r", "uid"}, ""))
 

proto/gen/api/v1/user_service.pb.gw.go

@@ -810,7 +810,7 @@ func RegisterUserServiceHandlerServer(ctx context.Context, mux *runtime.ServeMux
 		inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req)
 		var err error
 		var annotatedContext context.Context
-		annotatedContext, err = runtime.AnnotateIncomingContext(ctx, mux, req, "/memos.api.v1.UserService/GetUserAvatarBinary", runtime.WithHTTPPathPattern("/o/{name=users/*}/avatar"))
+		annotatedContext, err = runtime.AnnotateIncomingContext(ctx, mux, req, "/memos.api.v1.UserService/GetUserAvatarBinary", runtime.WithHTTPPathPattern("/file/{name=users/*}/avatar"))
 		if err != nil {
 			runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err)
 			return
@@ -1140,7 +1140,7 @@ func RegisterUserServiceHandlerClient(ctx context.Context, mux *runtime.ServeMux
 		inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req)
 		var err error
 		var annotatedContext context.Context
-		annotatedContext, err = runtime.AnnotateContext(ctx, mux, req, "/memos.api.v1.UserService/GetUserAvatarBinary", runtime.WithHTTPPathPattern("/o/{name=users/*}/avatar"))
+		annotatedContext, err = runtime.AnnotateContext(ctx, mux, req, "/memos.api.v1.UserService/GetUserAvatarBinary", runtime.WithHTTPPathPattern("/file/{name=users/*}/avatar"))
 		if err != nil {
 			runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err)
 			return
@@ -1342,7 +1342,7 @@ var (
 
 	pattern_UserService_GetUser_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2, 1, 0, 4, 2, 5, 3}, []string{"api", "v1", "users", "name"}, ""))
 
-	pattern_UserService_GetUserAvatarBinary_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 1, 0, 4, 2, 5, 2, 2, 3}, []string{"o", "users", "name", "avatar"}, ""))
+	pattern_UserService_GetUserAvatarBinary_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 1, 0, 4, 2, 5, 2, 2, 3}, []string{"file", "users", "name", "avatar"}, ""))
 
 	pattern_UserService_CreateUser_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2}, []string{"api", "v1", "users"}, ""))
 

proto/store/resource.proto

+syntax = "proto3";
+
+package memos.store;
+
+import "google/protobuf/timestamp.proto";
+
+option go_package = "gen/store";
+
+enum ResourceStorageType {
+  RESOURCE_STORAGE_TYPE_UNSPECIFIED = 0;
+  LOCAL = 1;
+  S3 = 2;
+  EXTERNAL = 3;
+}
+
+message ResourcePayload {
+  oneof payload {
+    S3Object s3_object = 1;
+  }
+
+  message S3Object {
+    string bucket = 1;
+    string key = 2;
+    google.protobuf.Timestamp last_presigned_time = 3;
+  }
+}

server/router/api/v1/memo_resource_service.go

@@ -52,7 +52,7 @@ func (s *APIV1Service) SetMemoResources(ctx context.Context, request *v1pb.SetMe
 			return nil, status.Errorf(codes.InvalidArgument, "invalid resource name: %v", err)
 		}
 		updatedTs := time.Now().Unix() + int64(index)
-		if _, err := s.Store.UpdateResource(ctx, &store.UpdateResource{
+		if err := s.Store.UpdateResource(ctx, &store.UpdateResource{
 			ID:        id,
 			MemoID:    &memoID,
 			UpdatedTs: &updatedTs,

server/router/api/v1/resource_service.go

@@ -6,7 +6,6 @@ import (
 	"encoding/binary"
 	"fmt"
 	"io"
-	"net/url"
 	"os"
 	"path/filepath"
 	"regexp"
@@ -50,34 +49,23 @@ func (s *APIV1Service) CreateResource(ctx context.Context, request *v1pb.CreateR
 		Filename:  request.Resource.Filename,
 		Type:      request.Resource.Type,
 	}
-	if request.Resource.ExternalLink != "" {
-		// Only allow those external links scheme with http/https
-		linkURL, err := url.Parse(request.Resource.ExternalLink)
-		if err != nil {
-			return nil, status.Errorf(codes.InvalidArgument, "invalid external link: %v", err)
-		}
-		if linkURL.Scheme != "http" && linkURL.Scheme != "https" {
-			return nil, status.Errorf(codes.InvalidArgument, "invalid external link scheme: %v", linkURL.Scheme)
-		}
-		create.ExternalLink = request.Resource.ExternalLink
-	} else {
-		workspaceStorageSetting, err := s.Store.GetWorkspaceStorageSetting(ctx)
-		if err != nil {
-			return nil, status.Errorf(codes.Internal, "failed to get workspace storage setting: %v", err)
-		}
-		size := binary.Size(request.Resource.Content)
-		uploadSizeLimit := int(workspaceStorageSetting.UploadSizeLimitMb) * MebiByte
-		if uploadSizeLimit == 0 {
-			uploadSizeLimit = MaxUploadBufferSizeBytes
-		}
-		if size > uploadSizeLimit {
-			return nil, status.Errorf(codes.InvalidArgument, "file size exceeds the limit")
-		}
-		create.Size = int64(size)
-		create.Blob = request.Resource.Content
-		if err := SaveResourceBlob(ctx, s.Store, create); err != nil {
-			return nil, status.Errorf(codes.Internal, "failed to save resource blob: %v", err)
-		}
+
+	workspaceStorageSetting, err := s.Store.GetWorkspaceStorageSetting(ctx)
+	if err != nil {
+		return nil, status.Errorf(codes.Internal, "failed to get workspace storage setting: %v", err)
+	}
+	size := binary.Size(request.Resource.Content)
+	uploadSizeLimit := int(workspaceStorageSetting.UploadSizeLimitMb) * MebiByte
+	if uploadSizeLimit == 0 {
+		uploadSizeLimit = MaxUploadBufferSizeBytes
+	}
+	if size > uploadSizeLimit {
+		return nil, status.Errorf(codes.InvalidArgument, "file size exceeds the limit")
+	}
+	create.Size = int64(size)
+	create.Blob = request.Resource.Content
+	if err := SaveResourceBlob(ctx, s.Store, create); err != nil {
+		return nil, status.Errorf(codes.Internal, "failed to save resource blob: %v", err)
 	}
 
 	if request.Resource.Memo != nil {
@@ -202,8 +190,8 @@ func (s *APIV1Service) GetResourceBinary(ctx context.Context, request *v1pb.GetR
 	}
 
 	blob := resource.Blob
-	if resource.InternalPath != "" {
-		resourcePath := filepath.FromSlash(resource.InternalPath)
+	if resource.StorageType == storepb.ResourceStorageType_LOCAL {
+		resourcePath := filepath.FromSlash(resource.Reference)
 		if !filepath.IsAbs(resourcePath) {
 			resourcePath = filepath.Join(s.Profile.Data, resourcePath)
 		}
@@ -255,11 +243,12 @@ func (s *APIV1Service) UpdateResource(ctx context.Context, request *v1pb.UpdateR
 		}
 	}
 
-	resource, err := s.Store.UpdateResource(ctx, update)
-	if err != nil {
+	if err := s.Store.UpdateResource(ctx, update); err != nil {
 		return nil, status.Errorf(codes.Internal, "failed to update resource: %v", err)
 	}
-	return s.convertResourceFromStore(ctx, resource), nil
+	return s.GetResource(ctx, &v1pb.GetResourceRequest{
+		Name: request.Resource.Name,
+	})
 }
 
 func (s *APIV1Service) DeleteResource(ctx context.Context, request *v1pb.DeleteResourceRequest) (*emptypb.Empty, error) {
@@ -292,13 +281,15 @@ func (s *APIV1Service) DeleteResource(ctx context.Context, request *v1pb.DeleteR
 
 func (s *APIV1Service) convertResourceFromStore(ctx context.Context, resource *store.Resource) *v1pb.Resource {
 	resourceMessage := &v1pb.Resource{
-		Name:         fmt.Sprintf("%s%d", ResourceNamePrefix, resource.ID),
-		Uid:          resource.UID,
-		CreateTime:   timestamppb.New(time.Unix(resource.CreatedTs, 0)),
-		Filename:     resource.Filename,
-		ExternalLink: resource.ExternalLink,
-		Type:         resource.Type,
-		Size:         resource.Size,
+		Name:       fmt.Sprintf("%s%d", ResourceNamePrefix, resource.ID),
+		Uid:        resource.UID,
+		CreateTime: timestamppb.New(time.Unix(resource.CreatedTs, 0)),
+		Filename:   resource.Filename,
+		Type:       resource.Type,
+		Size:       resource.Size,
+	}
+	if resource.StorageType == storepb.ResourceStorageType_EXTERNAL || resource.StorageType == storepb.ResourceStorageType_S3 {
+		resourceMessage.ExternalLink = resource.Reference
 	}
 	if resource.MemoID != nil {
 		memo, _ := s.Store.GetMemo(ctx, &store.FindMemo{
@@ -330,7 +321,7 @@ func SaveResourceBlob(ctx context.Context, s *store.Store, create *store.Resourc
 		if !strings.Contains(internalPath, "{filename}") {
 			internalPath = filepath.Join(internalPath, "{filename}")
 		}
-		internalPath = replacePathTemplate(internalPath, create.Filename)
+		internalPath = replaceFilenameWithPathTemplate(internalPath, create.Filename)
 		internalPath = filepath.ToSlash(internalPath)
 
 		// Ensure the directory exists.
@@ -352,20 +343,15 @@ func SaveResourceBlob(ctx context.Context, s *store.Store, create *store.Resourc
 		if err := os.WriteFile(osPath, create.Blob, 0644); err != nil {
 			return errors.Wrap(err, "Failed to write file")
 		}
-		create.InternalPath = internalPath
+		create.Reference = internalPath
 		create.Blob = nil
+		create.StorageType = storepb.ResourceStorageType_LOCAL
 	} else if workspaceStorageSetting.StorageType == storepb.WorkspaceStorageSetting_STORAGE_TYPE_S3 {
 		s3Config := workspaceStorageSetting.S3Config
 		if s3Config == nil {
 			return errors.Errorf("No actived external storage found")
 		}
-		s3Client, err := s3.NewClient(ctx, &s3.Config{
-			AccessKeyID:     s3Config.AccessKeyId,
-			AcesssKeySecret: s3Config.AccessKeySecret,
-			Endpoint:        s3Config.Endpoint,
-			Region:          s3Config.Region,
-			Bucket:          s3Config.Bucket,
-		})
+		s3Client, err := s3.NewClient(ctx, s3Config)
 		if err != nil {
 			return errors.Wrap(err, "Failed to create s3 client")
 		}
@@ -374,15 +360,28 @@ func SaveResourceBlob(ctx context.Context, s *store.Store, create *store.Resourc
 		if !strings.Contains(filepathTemplate, "{filename}") {
 			filepathTemplate = filepath.Join(filepathTemplate, "{filename}")
 		}
-		filepathTemplate = replacePathTemplate(filepathTemplate, create.Filename)
-		r := bytes.NewReader(create.Blob)
-		link, err := s3Client.UploadFile(ctx, filepathTemplate, create.Type, r)
+		filepathTemplate = replaceFilenameWithPathTemplate(filepathTemplate, create.Filename)
+		key, err := s3Client.UploadObject(ctx, filepathTemplate, create.Type, bytes.NewReader(create.Blob))
 		if err != nil {
 			return errors.Wrap(err, "Failed to upload via s3 client")
 		}
+		presignURL, err := s3Client.PresignGetObject(ctx, s3Config.Bucket, key)
+		if err != nil {
+			return errors.Wrap(err, "Failed to presign via s3 client")
+		}
 
-		create.ExternalLink = link
+		create.Reference = presignURL
 		create.Blob = nil
+		create.StorageType = storepb.ResourceStorageType_S3
+		create.Payload = &storepb.ResourcePayload{
+			Payload: &storepb.ResourcePayload_S3Object_{
+				S3Object: &storepb.ResourcePayload_S3Object{
+					Bucket:            s3Config.Bucket,
+					Key:               key,
+					LastPresignedTime: timestamppb.New(time.Now()),
+				},
+			},
+		}
 	}
 
 	return nil
@@ -390,7 +389,7 @@ func SaveResourceBlob(ctx context.Context, s *store.Store, create *store.Resourc
 
 var fileKeyPattern = regexp.MustCompile(`\{[a-z]{1,9}\}`)
 
-func replacePathTemplate(path, filename string) string {
+func replaceFilenameWithPathTemplate(path, filename string) string {
 	t := time.Now()
 	path = fileKeyPattern.ReplaceAllStringFunc(path, func(s string) string {
 		switch s {

server/router/api/v1/user_service.go

@@ -520,7 +520,7 @@ func convertUserFromStore(user *store.User) *v1pb.User {
 	}
 	// Use the avatar URL instead of raw base64 image data to reduce the response size.
 	if user.AvatarURL != "" {
-		userpb.AvatarUrl = fmt.Sprintf("/o/%s/avatar", userpb.Name)
+		userpb.AvatarUrl = fmt.Sprintf("/file/%s/avatar", userpb.Name)
 	}
 	return userpb
 }

server/router/api/v1/v1.go

@@ -112,7 +112,7 @@ func (s *APIV1Service) RegisterGateway(ctx context.Context, echoServer *echo.Ech
 		return err
 	}
 	echoServer.Any("/api/v1/*", echo.WrapHandler(gwMux))
-	echoServer.Any("/o/*", echo.WrapHandler(gwMux))
+	echoServer.Any("/file/*", echo.WrapHandler(gwMux))
 
 	// GRPC web proxy.
 	options := []grpcweb.Option{

server/router/rss/rss.go

@@ -2,6 +2,7 @@ package rss
 
 import (
 	"context"
+	"fmt"
 	"net/http"
 	"strconv"
 	"strings"
@@ -13,6 +14,7 @@ import (
 	"github.com/yourselfhosted/gomark/ast"
 	"github.com/yourselfhosted/gomark/renderer"
 
+	storepb "github.com/usememos/memos/proto/gen/store"
 	"github.com/usememos/memos/server/profile"
 	"github.com/usememos/memos/store"
 )
@@ -124,10 +126,10 @@ func (s *RSSService) generateRSSFromMemoList(ctx context.Context, memoList []*st
 		if len(resources) > 0 {
 			resource := resources[0]
 			enclosure := feeds.Enclosure{}
-			if resource.ExternalLink != "" {
-				enclosure.Url = resource.ExternalLink
+			if resource.StorageType == storepb.ResourceStorageType_EXTERNAL || resource.StorageType == storepb.ResourceStorageType_S3 {
+				enclosure.Url = resource.Reference
 			} else {
-				enclosure.Url = baseURL + "/o/r/" + resource.UID
+				enclosure.Url = fmt.Sprintf("%s/file/resources/%d", baseURL, resource.ID)
 			}
 			enclosure.Length = strconv.Itoa(int(resource.Size))
 			enclosure.Type = resource.Type

store/db/mysql/inbox.go

@@ -2,7 +2,6 @@ package mysql
 
 import (
 	"context"
-	"database/sql"
 	"strings"
 
 	"github.com/pkg/errors"
@@ -133,13 +132,3 @@ func (d *DB) DeleteInbox(ctx context.Context, delete *store.DeleteInbox) error {
 	}
 	return nil
 }
-
-func vacuumInbox(ctx context.Context, tx *sql.Tx) error {
-	stmt := "DELETE FROM `inbox` WHERE `sender_id` NOT IN (SELECT `id` FROM `user`)"
-	_, err := tx.ExecContext(ctx, stmt)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}

store/db/mysql/memo.go

@@ -2,7 +2,6 @@ package mysql
 
 import (
 	"context"
-	"database/sql"
 	"fmt"
 	"strings"
 
@@ -206,20 +205,5 @@ func (d *DB) DeleteMemo(ctx context.Context, delete *store.DeleteMemo) error {
 	if _, err := result.RowsAffected(); err != nil {
 		return err
 	}
-
-	if err := d.Vacuum(ctx); err != nil {
-		// Prevent linter warning.
-		return err
-	}
-	return nil
-}
-
-func vacuumMemo(ctx context.Context, tx *sql.Tx) error {
-	stmt := "DELETE FROM `memo` WHERE `creator_id` NOT IN (SELECT `id` FROM `user`)"
-	_, err := tx.ExecContext(ctx, stmt)
-	if err != nil {
-		return err
-	}
-
 	return nil
 }

store/db/mysql/memo_organizer.go

@@ -2,7 +2,6 @@ package mysql
 
 import (
 	"context"
-	"database/sql"
 	"strings"
 
 	"github.com/usememos/memos/store"
@@ -69,12 +68,3 @@ func (d *DB) DeleteMemoOrganizer(ctx context.Context, delete *store.DeleteMemoOr
 	}
 	return nil
 }
-
-func vacuumMemoOrganizer(ctx context.Context, tx *sql.Tx) error {
-	stmt := "DELETE FROM `memo_organizer` WHERE `memo_id` NOT IN (SELECT `id` FROM `memo`) OR `user_id` NOT IN (SELECT `id` FROM `user`)"
-	_, err := tx.ExecContext(ctx, stmt)
-	if err != nil {
-		return err
-	}
-	return nil
-}

store/db/mysql/memo_relation.go

@@ -2,7 +2,6 @@ package mysql
 
 import (
 	"context"
-	"database/sql"
 	"strings"
 
 	"github.com/usememos/memos/store"
@@ -89,10 +88,3 @@ func (d *DB) DeleteMemoRelation(ctx context.Context, delete *store.DeleteMemoRel
 	}
 	return nil
 }
-
-func vacuumMemoRelations(ctx context.Context, tx *sql.Tx) error {
-	if _, err := tx.ExecContext(ctx, "DELETE FROM `memo_relation` WHERE `memo_id` NOT IN (SELECT `id` FROM `memo`) OR `related_memo_id` NOT IN (SELECT `id` FROM `memo`)"); err != nil {
-		return err
-	}
-	return nil
-}

store/db/mysql/migration/dev/LATEST__SCHEMA.sql

@@ -71,11 +71,12 @@ CREATE TABLE `resource` (
   `updated_ts` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
   `filename` TEXT NOT NULL,
   `blob` MEDIUMBLOB,
-  `external_link` TEXT NOT NULL,
   `type` VARCHAR(256) NOT NULL DEFAULT '',
   `size` INT NOT NULL DEFAULT '0',
-  `internal_path` VARCHAR(256) NOT NULL DEFAULT '',
-  `memo_id` INT DEFAULT NULL
+  `memo_id` INT DEFAULT NULL,
+  `storage_type` VARCHAR(256) NOT NULL DEFAULT '',
+  `reference` VARCHAR(256) NOT NULL DEFAULT '',
+  `payload` TEXT NOT NULL
 );
 
 -- tag
@@ -95,14 +96,6 @@ CREATE TABLE `activity` (
   `payload` TEXT NOT NULL
 );
 
--- storage
-CREATE TABLE `storage` (
-  `id` INT NOT NULL AUTO_INCREMENT PRIMARY KEY,
-  `name` VARCHAR(256) NOT NULL,
-  `type` VARCHAR(256) NOT NULL,
-  `config` TEXT NOT NULL
-);
-
 -- idp
 CREATE TABLE `idp` (
   `id` INT NOT NULL AUTO_INCREMENT PRIMARY KEY,

store/db/mysql/migration/prod/0.22/00__resource_storage_type.sql

+ALTER TABLE `resource`
+  ADD COLUMN `storage_type` VARCHAR(256) NOT NULL DEFAULT '',
+  ADD COLUMN `reference` VARCHAR(256) NOT NULL DEFAULT '',
+  ADD COLUMN `payload` TEXT NOT NULL;
+
+UPDATE `resource`
+SET `storage_type` = 'LOCAL', `reference` = `internal_path`
+WHERE `internal_path` IS NOT NULL AND `internal_path` != '';
+
+UPDATE `resource`
+SET `storage_type` = 'EXTERNAL', `reference` = `external_link`
+WHERE `external_link` IS NOT NULL AND `external_link` != '';
+
+ALTER TABLE `resource`
+  DROP COLUMN `internal_path`,
+  DROP COLUMN `external_link`;

store/db/mysql/mysql.go

@@ -45,39 +45,6 @@ func (d *DB) GetDB() *sql.DB {
 	return d.db
 }
 
-func (d *DB) Vacuum(ctx context.Context) error {
-	tx, err := d.db.BeginTx(ctx, nil)
-	if err != nil {
-		return err
-	}
-	defer tx.Rollback()
-
-	if err := vacuumMemo(ctx, tx); err != nil {
-		return err
-	}
-	if err := vacuumResource(ctx, tx); err != nil {
-		return err
-	}
-	if err := vacuumUserSetting(ctx, tx); err != nil {
-		return err
-	}
-	if err := vacuumMemoOrganizer(ctx, tx); err != nil {
-		return err
-	}
-	if err := vacuumMemoRelations(ctx, tx); err != nil {
-		return err
-	}
-	if err := vacuumInbox(ctx, tx); err != nil {
-		return err
-	}
-	if err := vacuumTag(ctx, tx); err != nil {
-		// Prevent revive warning.
-		return err
-	}
-
-	return tx.Commit()
-}
-
 func (d *DB) GetCurrentDBSize(ctx context.Context) (int64, error) {
 	query := "SELECT SUM(`data_length` + `index_length`) AS `size` " +
 		" FROM information_schema.TABLES" +

store/db/mysql/resource.go

@@ -6,13 +6,29 @@ import (
 	"fmt"
 	"strings"
 
+	"github.com/pkg/errors"
+	"google.golang.org/protobuf/encoding/protojson"
+
+	storepb "github.com/usememos/memos/proto/gen/store"
 	"github.com/usememos/memos/store"
 )
 
 func (d *DB) CreateResource(ctx context.Context, create *store.Resource) (*store.Resource, error) {
-	fields := []string{"`uid`", "`filename`", "`blob`", "`external_link`", "`type`", "`size`", "`creator_id`", "`internal_path`", "`memo_id`"}
-	placeholder := []string{"?", "?", "?", "?", "?", "?", "?", "?", "?"}
-	args := []any{create.UID, create.Filename, create.Blob, create.ExternalLink, create.Type, create.Size, create.CreatorID, create.InternalPath, create.MemoID}
+	fields := []string{"`uid`", "`filename`", "`blob`", "`type`", "`size`", "`creator_id`", "`memo_id`", "`storage_type`", "`reference`", "`payload`"}
+	placeholder := []string{"?", "?", "?", "?", "?", "?", "?", "?", "?", "?"}
+	storageType := ""
+	if create.StorageType != storepb.ResourceStorageType_RESOURCE_STORAGE_TYPE_UNSPECIFIED {
+		storageType = create.StorageType.String()
+	}
+	payloadString := "{}"
+	if create.Payload != nil {
+		bytes, err := protojson.Marshal(create.Payload)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to marshal resource payload")
+		}
+		payloadString = string(bytes)
+	}
+	args := []any{create.UID, create.Filename, create.Blob, create.Type, create.Size, create.CreatorID, create.MemoID, storageType, create.Reference, payloadString}
 
 	stmt := "INSERT INTO `resource` (" + strings.Join(fields, ", ") + ") VALUES (" + strings.Join(placeholder, ", ") + ")"
 	result, err := d.db.ExecContext(ctx, stmt, args...)
@@ -51,12 +67,12 @@ func (d *DB) ListResources(ctx context.Context, find *store.FindResource) ([]*st
 		where = append(where, "`memo_id` IS NOT NULL")
 	}
 
-	fields := []string{"`id`", "`uid`", "`filename`", "`external_link`", "`type`", "`size`", "`creator_id`", "UNIX_TIMESTAMP(`created_ts`)", "UNIX_TIMESTAMP(`updated_ts`)", "`internal_path`", "`memo_id`"}
+	fields := []string{"`id`", "`uid`", "`filename`", "`type`", "`size`", "`creator_id`", "`created_ts`", "`updated_ts`", "`memo_id`", "`storage_type`", "`reference`", "`payload`"}
 	if find.GetBlob {
 		fields = append(fields, "`blob`")
 	}
 
-	query := fmt.Sprintf("SELECT %s FROM `resource` WHERE %s ORDER BY `updated_ts` DESC, `created_ts` DESC", strings.Join(fields, ", "), strings.Join(where, " AND "))
+	query := fmt.Sprintf("SELECT %s FROM `resource` WHERE %s ORDER BY `created_ts` DESC", strings.Join(fields, ", "), strings.Join(where, " AND "))
 	if find.Limit != nil {
 		query = fmt.Sprintf("%s LIMIT %d", query, *find.Limit)
 		if find.Offset != nil {
@@ -74,18 +90,21 @@ func (d *DB) ListResources(ctx context.Context, find *store.FindResource) ([]*st
 	for rows.Next() {
 		resource := store.Resource{}
 		var memoID sql.NullInt32
+		var storageType string
+		var payloadBytes []byte
 		dests := []any{
 			&resource.ID,
 			&resource.UID,
 			&resource.Filename,
-			&resource.ExternalLink,
 			&resource.Type,
 			&resource.Size,
 			&resource.CreatorID,
 			&resource.CreatedTs,
 			&resource.UpdatedTs,
-			&resource.InternalPath,
 			&memoID,
+			&storageType,
+			&resource.Reference,
+			&payloadBytes,
 		}
 		if find.GetBlob {
 			dests = append(dests, &resource.Blob)
@@ -93,9 +112,16 @@ func (d *DB) ListResources(ctx context.Context, find *store.FindResource) ([]*st
 		if err := rows.Scan(dests...); err != nil {
 			return nil, err
 		}
+
 		if memoID.Valid {
 			resource.MemoID = &memoID.Int32
 		}
+		resource.StorageType = storepb.ResourceStorageType(storepb.ResourceStorageType_value[storageType])
+		payload := &storepb.ResourcePayload{}
+		if err := protojsonUnmarshaler.Unmarshal(payloadBytes, payload); err != nil {
+			return nil, err
+		}
+		resource.Payload = payload
 		list = append(list, &resource)
 	}
 
@@ -118,7 +144,7 @@ func (d *DB) GetResource(ctx context.Context, find *store.FindResource) (*store.
 	return list[0], nil
 }
 
-func (d *DB) UpdateResource(ctx context.Context, update *store.UpdateResource) (*store.Resource, error) {
+func (d *DB) UpdateResource(ctx context.Context, update *store.UpdateResource) error {
 	set, args := []string{}, []any{}
 
 	if v := update.UID; v != nil {
@@ -130,52 +156,31 @@ func (d *DB) UpdateResource(ctx context.Context, update *store.UpdateResource) (
 	if v := update.Filename; v != nil {
 		set, args = append(set, "`filename` = ?"), append(args, *v)
 	}
-	if v := update.InternalPath; v != nil {
-		set, args = append(set, "`internal_path` = ?"), append(args, *v)
-	}
-	if v := update.ExternalLink; v != nil {
-		set, args = append(set, "`external_link` = ?"), append(args, *v)
-	}
 	if v := update.MemoID; v != nil {
 		set, args = append(set, "`memo_id` = ?"), append(args, *v)
 	}
-	if v := update.Blob; v != nil {
-		set, args = append(set, "`blob` = ?"), append(args, v)
-	}
 
 	args = append(args, update.ID)
 	stmt := "UPDATE `resource` SET " + strings.Join(set, ", ") + " WHERE `id` = ?"
-	if _, err := d.db.ExecContext(ctx, stmt, args...); err != nil {
-		return nil, err
-	}
-
-	return d.GetResource(ctx, &store.FindResource{ID: &update.ID})
-}
-
-func (d *DB) DeleteResource(ctx context.Context, delete *store.DeleteResource) error {
-	stmt := "DELETE FROM `resource` WHERE `id` = ?"
-	result, err := d.db.ExecContext(ctx, stmt, delete.ID)
+	result, err := d.db.ExecContext(ctx, stmt, args...)
 	if err != nil {
 		return err
 	}
 	if _, err := result.RowsAffected(); err != nil {
 		return err
 	}
-
-	if err := d.Vacuum(ctx); err != nil {
-		// Prevent linter warning.
-		return err
-	}
-
 	return nil
 }
 
-func vacuumResource(ctx context.Context, tx *sql.Tx) error {
-	stmt := "DELETE FROM `resource` WHERE `creator_id` NOT IN (SELECT `id` FROM `user`)"
-	_, err := tx.ExecContext(ctx, stmt)
+func (d *DB) DeleteResource(ctx context.Context, delete *store.DeleteResource) error {
+	stmt := "DELETE FROM `resource` WHERE `id` = ?"
+	result, err := d.db.ExecContext(ctx, stmt, delete.ID)
 	if err != nil {
 		return err
 	}
+	if _, err := result.RowsAffected(); err != nil {
+		return err
+	}
 
 	return nil
 }

store/db/mysql/tag.go

@@ -2,7 +2,6 @@ package mysql
 
 import (
 	"context"
-	"database/sql"
 	"strings"
 
 	"github.com/usememos/memos/store"
@@ -63,13 +62,3 @@ func (d *DB) DeleteTag(ctx context.Context, delete *store.DeleteTag) error {
 	}
 	return nil
 }
-
-func vacuumTag(ctx context.Context, tx *sql.Tx) error {
-	stmt := "DELETE FROM `tag` WHERE `creator_id` NOT IN (SELECT `id` FROM `user`)"
-	_, err := tx.ExecContext(ctx, stmt)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}

store/db/mysql/user.go

@@ -160,11 +160,5 @@ func (d *DB) DeleteUser(ctx context.Context, delete *store.DeleteUser) error {
 	if _, err := result.RowsAffected(); err != nil {
 		return err
 	}
-
-	if err := d.Vacuum(ctx); err != nil {
-		// Prevent linter warning.
-		return err
-	}
-
 	return nil
 }

store/db/mysql/user_setting.go

@@ -2,7 +2,6 @@ package mysql
 
 import (
 	"context"
-	"database/sql"
 	"strings"
 
 	storepb "github.com/usememos/memos/proto/gen/store"
@@ -55,13 +54,3 @@ func (d *DB) ListUserSettings(ctx context.Context, find *store.FindUserSetting)
 
 	return userSettingList, nil
 }
-
-func vacuumUserSetting(ctx context.Context, tx *sql.Tx) error {
-	stmt := "DELETE FROM `user_setting` WHERE `user_id` NOT IN (SELECT `id` FROM `user`)"
-	_, err := tx.ExecContext(ctx, stmt)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}

store/db/postgres/inbox.go

@@ -2,7 +2,6 @@ package postgres
 
 import (
 	"context"
-	"database/sql"
 	"strings"
 
 	"github.com/pkg/errors"
@@ -133,13 +132,3 @@ func (d *DB) DeleteInbox(ctx context.Context, delete *store.DeleteInbox) error {
 	}
 	return nil
 }
-
-func vacuumInbox(ctx context.Context, tx *sql.Tx) error {
-	stmt := `DELETE FROM inbox WHERE sender_id NOT IN (SELECT id FROM "user")`
-	_, err := tx.ExecContext(ctx, stmt)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}

store/db/postgres/memo.go

@@ -2,7 +2,6 @@ package postgres
 
 import (
 	"context"
-	"database/sql"
 	"fmt"
 	"strings"
 
@@ -203,9 +202,3 @@ func (d *DB) DeleteMemo(ctx context.Context, delete *store.DeleteMemo) error {
 	}
 	return nil
 }
-
-func vacuumMemo(ctx context.Context, tx *sql.Tx) error {
-	stmt := `DELETE FROM memo WHERE creator_id NOT IN (SELECT id FROM "user")`
-	_, err := tx.ExecContext(ctx, stmt)
-	return err
-}

store/db/postgres/memo_organizer.go

@@ -2,7 +2,6 @@ package postgres
 
 import (
 	"context"
-	"database/sql"
 	"fmt"
 	"strings"
 
@@ -90,18 +89,3 @@ func (d *DB) DeleteMemoOrganizer(ctx context.Context, delete *store.DeleteMemoOr
 	}
 	return nil
 }
-
-func vacuumMemoOrganizer(ctx context.Context, tx *sql.Tx) error {
-	stmt := `
-	DELETE FROM 
-		memo_organizer 
-	WHERE 
-		memo_id NOT IN (SELECT id FROM memo)
-		OR user_id NOT IN (SELECT id FROM "user")`
-	_, err := tx.ExecContext(ctx, stmt)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}

store/db/postgres/memo_relation.go

@@ -2,7 +2,6 @@ package postgres
 
 import (
 	"context"
-	"database/sql"
 	"strings"
 
 	"github.com/usememos/memos/store"
@@ -101,13 +100,3 @@ func (d *DB) DeleteMemoRelation(ctx context.Context, delete *store.DeleteMemoRel
 	}
 	return nil
 }
-
-func vacuumMemoRelations(ctx context.Context, tx *sql.Tx) error {
-	if _, err := tx.ExecContext(ctx, `
-		DELETE FROM memo_relation
-		WHERE memo_id NOT IN (SELECT id FROM memo) OR related_memo_id NOT IN (SELECT id FROM memo)
-	`); err != nil {
-		return err
-	}
-	return nil
-}

store/db/postgres/migration/dev/LATEST__SCHEMA.sql

@@ -71,11 +71,12 @@ CREATE TABLE resource (
   updated_ts BIGINT NOT NULL DEFAULT EXTRACT(EPOCH FROM NOW()),
   filename TEXT NOT NULL,
   blob BYTEA,
-  external_link TEXT NOT NULL,
   type TEXT NOT NULL DEFAULT '',
   size INTEGER NOT NULL DEFAULT 0,
-  internal_path TEXT NOT NULL DEFAULT '',
-  memo_id INTEGER DEFAULT NULL
+  memo_id INTEGER DEFAULT NULL,
+  storage_type TEXT NOT NULL DEFAULT '',
+  reference TEXT NOT NULL DEFAULT '',
+  payload TEXT NOT NULL DEFAULT '{}'
 );
 
 -- tag
@@ -95,14 +96,6 @@ CREATE TABLE activity (
   payload JSONB NOT NULL DEFAULT '{}'
 );
 
--- storage
-CREATE TABLE storage (
-  id SERIAL PRIMARY KEY,
-  name TEXT NOT NULL,
-  type TEXT NOT NULL,
-  config JSONB NOT NULL DEFAULT '{}'
-);
-
 -- idp
 CREATE TABLE idp (
   id SERIAL PRIMARY KEY,

store/db/postgres/migration/prod/0.22/00__resource_storage_type.sql

+ALTER TABLE `resource`
+  ADD COLUMN `storage_type` VARCHAR(256) NOT NULL DEFAULT '',
+  ADD COLUMN `reference` VARCHAR(256) NOT NULL DEFAULT '',
+  ADD COLUMN `payload` TEXT NOT NULL;
+
+UPDATE `resource`
+SET `storage_type` = 'LOCAL', `reference` = `internal_path`
+WHERE `internal_path` IS NOT NULL AND `internal_path` != '';
+
+UPDATE `resource`
+SET `storage_type` = 'EXTERNAL', `reference` = `external_link`
+WHERE `external_link` IS NOT NULL AND `external_link` != '';
+
+ALTER TABLE `resource`
+  DROP COLUMN `internal_path`,
+  DROP COLUMN `external_link`;

store/db/postgres/postgres.go

@@ -44,39 +44,6 @@ func (d *DB) GetDB() *sql.DB {
 	return d.db
 }
 
-func (d *DB) Vacuum(ctx context.Context) error {
-	tx, err := d.db.BeginTx(ctx, nil)
-	if err != nil {
-		return err
-	}
-	defer tx.Rollback()
-
-	if err := vacuumMemo(ctx, tx); err != nil {
-		return err
-	}
-	if err := vacuumResource(ctx, tx); err != nil {
-		return err
-	}
-	if err := vacuumUserSetting(ctx, tx); err != nil {
-		return err
-	}
-	if err := vacuumMemoOrganizer(ctx, tx); err != nil {
-		return err
-	}
-	if err := vacuumMemoRelations(ctx, tx); err != nil {
-		return err
-	}
-	if err := vacuumInbox(ctx, tx); err != nil {
-		return err
-	}
-	if err := vacuumTag(ctx, tx); err != nil {
-		// Prevent revive warning.
-		return err
-	}
-
-	return tx.Commit()
-}
-
 func (*DB) GetCurrentDBSize(context.Context) (int64, error) {
 	return 0, errors.New("unimplemented")
 }

store/db/postgres/resource.go

@@ -6,12 +6,28 @@ import (
 	"fmt"
 	"strings"
 
+	"github.com/pkg/errors"
+	"google.golang.org/protobuf/encoding/protojson"
+
+	storepb "github.com/usememos/memos/proto/gen/store"
 	"github.com/usememos/memos/store"
 )
 
 func (d *DB) CreateResource(ctx context.Context, create *store.Resource) (*store.Resource, error) {
-	fields := []string{"uid", "filename", "blob", "external_link", "type", "size", "creator_id", "internal_path", "memo_id"}
-	args := []any{create.UID, create.Filename, create.Blob, create.ExternalLink, create.Type, create.Size, create.CreatorID, create.InternalPath, create.MemoID}
+	fields := []string{"uid", "filename", "blob", "type", "size", "creator_id", "memo_id", "storage_type", "reference", "payload"}
+	storageType := ""
+	if create.StorageType != storepb.ResourceStorageType_RESOURCE_STORAGE_TYPE_UNSPECIFIED {
+		storageType = create.StorageType.String()
+	}
+	payloadString := "{}"
+	if create.Payload != nil {
+		bytes, err := protojson.Marshal(create.Payload)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to marshal resource payload")
+		}
+		payloadString = string(bytes)
+	}
+	args := []any{create.UID, create.Filename, create.Blob, create.Type, create.Size, create.CreatorID, create.MemoID, storageType, create.Reference, payloadString}
 
 	stmt := "INSERT INTO resource (" + strings.Join(fields, ", ") + ") VALUES (" + placeholders(len(args)) + ") RETURNING id, created_ts, updated_ts"
 	if err := d.db.QueryRowContext(ctx, stmt, args...).Scan(&create.ID, &create.CreatedTs, &create.UpdatedTs); err != nil {
@@ -42,7 +58,7 @@ func (d *DB) ListResources(ctx context.Context, find *store.FindResource) ([]*st
 		where = append(where, "memo_id IS NOT NULL")
 	}
 
-	fields := []string{"id", "uid", "filename", "external_link", "type", "size", "creator_id", "created_ts", "updated_ts", "internal_path", "memo_id"}
+	fields := []string{"id", "uid", "filename", "type", "size", "creator_id", "created_ts", "updated_ts", "memo_id", "storage_type", "reference", "payload"}
 	if find.GetBlob {
 		fields = append(fields, "blob")
 	}
@@ -52,7 +68,7 @@ func (d *DB) ListResources(ctx context.Context, find *store.FindResource) ([]*st
 			%s
 		FROM resource
 		WHERE %s
-		ORDER BY updated_ts DESC, created_ts DESC
+		ORDER BY created_ts DESC
 	`, strings.Join(fields, ", "), strings.Join(where, " AND "))
 	if find.Limit != nil {
 		query = fmt.Sprintf("%s LIMIT %d", query, *find.Limit)
@@ -71,18 +87,21 @@ func (d *DB) ListResources(ctx context.Context, find *store.FindResource) ([]*st
 	for rows.Next() {
 		resource := store.Resource{}
 		var memoID sql.NullInt32
+		var storageType string
+		var payloadBytes []byte
 		dests := []any{
 			&resource.ID,
 			&resource.UID,
 			&resource.Filename,
-			&resource.ExternalLink,
 			&resource.Type,
 			&resource.Size,
 			&resource.CreatorID,
 			&resource.CreatedTs,
 			&resource.UpdatedTs,
-			&resource.InternalPath,
 			&memoID,
+			&storageType,
+			&resource.Reference,
+			&payloadBytes,
 		}
 		if find.GetBlob {
 			dests = append(dests, &resource.Blob)
@@ -90,9 +109,16 @@ func (d *DB) ListResources(ctx context.Context, find *store.FindResource) ([]*st
 		if err := rows.Scan(dests...); err != nil {
 			return nil, err
 		}
+
 		if memoID.Valid {
 			resource.MemoID = &memoID.Int32
 		}
+		resource.StorageType = storepb.ResourceStorageType(storepb.ResourceStorageType_value[storageType])
+		payload := &storepb.ResourcePayload{}
+		if err := protojsonUnmarshaler.Unmarshal(payloadBytes, payload); err != nil {
+			return nil, err
+		}
+		resource.Payload = payload
 		list = append(list, &resource)
 	}
 
@@ -103,7 +129,7 @@ func (d *DB) ListResources(ctx context.Context, find *store.FindResource) ([]*st
 	return list, nil
 }
 
-func (d *DB) UpdateResource(ctx context.Context, update *store.UpdateResource) (*store.Resource, error) {
+func (d *DB) UpdateResource(ctx context.Context, update *store.UpdateResource) error {
 	set, args := []string{}, []any{}
 
 	if v := update.UID; v != nil {
@@ -115,45 +141,13 @@ func (d *DB) UpdateResource(ctx context.Context, update *store.UpdateResource) (
 	if v := update.Filename; v != nil {
 		set, args = append(set, "filename = "+placeholder(len(args)+1)), append(args, *v)
 	}
-	if v := update.InternalPath; v != nil {
-		set, args = append(set, "internal_path = "+placeholder(len(args)+1)), append(args, *v)
-	}
-	if v := update.ExternalLink; v != nil {
-		set, args = append(set, "external_link = "+placeholder(len(args)+1)), append(args, *v)
-	}
 	if v := update.MemoID; v != nil {
 		set, args = append(set, "memo_id = "+placeholder(len(args)+1)), append(args, *v)
 	}
-	if v := update.Blob; v != nil {
-		set, args = append(set, "blob = "+placeholder(len(args)+1)), append(args, v)
-	}
 
-	fields := []string{"id", "uid", "filename", "external_link", "type", "size", "creator_id", "created_ts", "updated_ts", "internal_path"}
-	stmt := `UPDATE resource SET ` + strings.Join(set, ", ") + ` WHERE id = ` + placeholder(len(args)+1) + ` RETURNING ` + strings.Join(fields, ", ")
+	stmt := `UPDATE resource SET ` + strings.Join(set, ", ") + ` WHERE id = ` + placeholder(len(args)+1)
 	args = append(args, update.ID)
-	resource := store.Resource{}
-	dests := []any{
-		&resource.ID,
-		&resource.UID,
-		&resource.Filename,
-		&resource.ExternalLink,
-		&resource.Type,
-		&resource.Size,
-		&resource.CreatorID,
-		&resource.CreatedTs,
-		&resource.UpdatedTs,
-		&resource.InternalPath,
-	}
-	if err := d.db.QueryRowContext(ctx, stmt, args...).Scan(dests...); err != nil {
-		return nil, err
-	}
-
-	return &resource, nil
-}
-
-func (d *DB) DeleteResource(ctx context.Context, delete *store.DeleteResource) error {
-	stmt := `DELETE FROM resource WHERE id = $1`
-	result, err := d.db.ExecContext(ctx, stmt, delete.ID)
+	result, err := d.db.ExecContext(ctx, stmt, args...)
 	if err != nil {
 		return err
 	}
@@ -163,12 +157,14 @@ func (d *DB) DeleteResource(ctx context.Context, delete *store.DeleteResource) e
 	return nil
 }
 
-func vacuumResource(ctx context.Context, tx *sql.Tx) error {
-	stmt := `DELETE FROM resource WHERE creator_id NOT IN (SELECT id FROM "user")`
-	_, err := tx.ExecContext(ctx, stmt)
+func (d *DB) DeleteResource(ctx context.Context, delete *store.DeleteResource) error {
+	stmt := `DELETE FROM resource WHERE id = $1`
+	result, err := d.db.ExecContext(ctx, stmt, delete.ID)
 	if err != nil {
 		return err
 	}
-
+	if _, err := result.RowsAffected(); err != nil {
+		return err
+	}
 	return nil
 }

store/db/postgres/tag.go

@@ -2,7 +2,6 @@ package postgres
 
 import (
 	"context"
-	"database/sql"
 	"strings"
 
 	"github.com/usememos/memos/store"
@@ -69,13 +68,3 @@ func (d *DB) DeleteTag(ctx context.Context, delete *store.DeleteTag) error {
 	}
 	return nil
 }
-
-func vacuumTag(ctx context.Context, tx *sql.Tx) error {
-	stmt := `DELETE FROM tag WHERE creator_id NOT IN (SELECT id FROM "user")`
-	_, err := tx.ExecContext(ctx, stmt)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}

store/db/postgres/user.go

@@ -165,11 +165,5 @@ func (d *DB) DeleteUser(ctx context.Context, delete *store.DeleteUser) error {
 	if _, err := result.RowsAffected(); err != nil {
 		return err
 	}
-
-	if err := d.Vacuum(ctx); err != nil {
-		// Prevent linter warning.
-		return err
-	}
-
 	return nil
 }

store/db/postgres/user_setting.go

@@ -2,7 +2,6 @@ package postgres
 
 import (
 	"context"
-	"database/sql"
 	"strings"
 
 	storepb "github.com/usememos/memos/proto/gen/store"
@@ -68,13 +67,3 @@ func (d *DB) ListUserSettings(ctx context.Context, find *store.FindUserSetting)
 
 	return userSettingList, nil
 }
-
-func vacuumUserSetting(ctx context.Context, tx *sql.Tx) error {
-	stmt := `DELETE FROM user_setting WHERE user_id NOT IN (SELECT id FROM "user")`
-	_, err := tx.ExecContext(ctx, stmt)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}

store/db/sqlite/inbox.go

@@ -2,7 +2,6 @@ package sqlite
 
 import (
 	"context"
-	"database/sql"
 	"strings"
 
 	"github.com/pkg/errors"
@@ -124,22 +123,3 @@ func (d *DB) DeleteInbox(ctx context.Context, delete *store.DeleteInbox) error {
 	}
 	return nil
 }
-
-func vacuumInbox(ctx context.Context, tx *sql.Tx) error {
-	stmt := `
-	DELETE FROM
-		inbox
-	WHERE
-		sender_id NOT IN (
-			SELECT
-				id
-			FROM
-				user
-		)`
-	_, err := tx.ExecContext(ctx, stmt)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}

store/db/sqlite/memo.go

@@ -2,7 +2,6 @@ package sqlite
 
 import (
 	"context"
-	"database/sql"
 	"fmt"
 	"strings"
 
@@ -187,20 +186,5 @@ func (d *DB) DeleteMemo(ctx context.Context, delete *store.DeleteMemo) error {
 	if _, err := result.RowsAffected(); err != nil {
 		return err
 	}
-
-	if err := d.Vacuum(ctx); err != nil {
-		// Prevent linter warning.
-		return err
-	}
-	return nil
-}
-
-func vacuumMemo(ctx context.Context, tx *sql.Tx) error {
-	stmt := "DELETE FROM `memo` WHERE `creator_id` NOT IN (SELECT `id` FROM `user`)"
-	_, err := tx.ExecContext(ctx, stmt)
-	if err != nil {
-		return err
-	}
-
 	return nil
 }

store/db/sqlite/memo_organizer.go

@@ -2,7 +2,6 @@ package sqlite
 
 import (
 	"context"
-	"database/sql"
 	"fmt"
 	"strings"
 
@@ -88,28 +87,3 @@ func (d *DB) DeleteMemoOrganizer(ctx context.Context, delete *store.DeleteMemoOr
 	}
 	return nil
 }
-
-func vacuumMemoOrganizer(ctx context.Context, tx *sql.Tx) error {
-	stmt := `
-	DELETE FROM 
-		memo_organizer 
-	WHERE 
-		memo_id NOT IN (
-			SELECT 
-				id 
-			FROM 
-				memo
-		)
-		OR user_id NOT IN (
-			SELECT 
-				id 
-			FROM 
-				user
-		)`
-	_, err := tx.ExecContext(ctx, stmt)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}

store/db/sqlite/memo_relation.go

@@ -2,7 +2,6 @@ package sqlite
 
 import (
 	"context"
-	"database/sql"
 	"strings"
 
 	"github.com/usememos/memos/store"
@@ -103,13 +102,3 @@ func (d *DB) DeleteMemoRelation(ctx context.Context, delete *store.DeleteMemoRel
 	}
 	return nil
 }
-
-func vacuumMemoRelations(ctx context.Context, tx *sql.Tx) error {
-	if _, err := tx.ExecContext(ctx, `
-		DELETE FROM memo_relation
-		WHERE memo_id NOT IN (SELECT id FROM memo) OR related_memo_id NOT IN (SELECT id FROM memo)
-	`); err != nil {
-		return err
-	}
-	return nil
-}

store/db/sqlite/migration/dev/LATEST__SCHEMA.sql

@@ -78,11 +78,12 @@ CREATE TABLE resource (
   updated_ts BIGINT NOT NULL DEFAULT (strftime('%s', 'now')),
   filename TEXT NOT NULL DEFAULT '',
   blob BLOB DEFAULT NULL,
-  external_link TEXT NOT NULL DEFAULT '',
   type TEXT NOT NULL DEFAULT '',
   size INTEGER NOT NULL DEFAULT 0,
-  internal_path TEXT NOT NULL DEFAULT '',
-  memo_id INTEGER
+  memo_id INTEGER,
+  storage_type TEXT NOT NULL DEFAULT '',
+  reference TEXT NOT NULL DEFAULT '',
+  payload TEXT NOT NULL DEFAULT '{}'
 );
 
 CREATE INDEX idx_resource_creator_id ON resource (creator_id);
@@ -106,14 +107,6 @@ CREATE TABLE activity (
   payload TEXT NOT NULL DEFAULT '{}'
 );
 
--- storage
-CREATE TABLE storage (
-  id INTEGER PRIMARY KEY AUTOINCREMENT,
-  name TEXT NOT NULL,
-  type TEXT NOT NULL,
-  config TEXT NOT NULL DEFAULT '{}'
-);
-
 -- idp
 CREATE TABLE idp (
   id INTEGER PRIMARY KEY AUTOINCREMENT,

store/db/sqlite/migration/prod/0.22/00__resource_storage_type.sql

+ALTER TABLE resource ADD COLUMN storage_type TEXT NOT NULL DEFAULT '';
+
+ALTER TABLE resource ADD COLUMN reference TEXT NOT NULL DEFAULT '';
+
+ALTER TABLE resource ADD COLUMN payload TEXT NOT NULL DEFAULT '{}';
+
+UPDATE resource
+SET storage_type = 'LOCAL', reference = internal_path
+WHERE internal_path IS NOT NULL AND internal_path != '';
+
+UPDATE resource
+SET storage_type = 'EXTERNAL', reference = external_link
+WHERE external_link IS NOT NULL AND external_link != '';
+
+ALTER TABLE resource
+  DROP COLUMN internal_path,
+  DROP COLUMN external_link;

store/db/sqlite/resource.go

@@ -6,13 +6,29 @@ import (
 	"fmt"
 	"strings"
 
+	"github.com/pkg/errors"
+	"google.golang.org/protobuf/encoding/protojson"
+
+	storepb "github.com/usememos/memos/proto/gen/store"
 	"github.com/usememos/memos/store"
 )
 
 func (d *DB) CreateResource(ctx context.Context, create *store.Resource) (*store.Resource, error) {
-	fields := []string{"`uid`", "`filename`", "`blob`", "`external_link`", "`type`", "`size`", "`creator_id`", "`internal_path`", "`memo_id`"}
-	placeholder := []string{"?", "?", "?", "?", "?", "?", "?", "?", "?"}
-	args := []any{create.UID, create.Filename, create.Blob, create.ExternalLink, create.Type, create.Size, create.CreatorID, create.InternalPath, create.MemoID}
+	fields := []string{"`uid`", "`filename`", "`blob`", "`type`", "`size`", "`creator_id`", "`memo_id`", "`storage_type`", "`reference`", "`payload`"}
+	placeholder := []string{"?", "?", "?", "?", "?", "?", "?", "?", "?", "?"}
+	storageType := ""
+	if create.StorageType != storepb.ResourceStorageType_RESOURCE_STORAGE_TYPE_UNSPECIFIED {
+		storageType = create.StorageType.String()
+	}
+	payloadString := "{}"
+	if create.Payload != nil {
+		bytes, err := protojson.Marshal(create.Payload)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to marshal resource payload")
+		}
+		payloadString = string(bytes)
+	}
+	args := []any{create.UID, create.Filename, create.Blob, create.Type, create.Size, create.CreatorID, create.MemoID, storageType, create.Reference, payloadString}
 
 	stmt := "INSERT INTO `resource` (" + strings.Join(fields, ", ") + ") VALUES (" + strings.Join(placeholder, ", ") + ") RETURNING `id`, `created_ts`, `updated_ts`"
 	if err := d.db.QueryRowContext(ctx, stmt, args...).Scan(&create.ID, &create.CreatedTs, &create.UpdatedTs); err != nil {
@@ -44,12 +60,12 @@ func (d *DB) ListResources(ctx context.Context, find *store.FindResource) ([]*st
 		where = append(where, "`memo_id` IS NOT NULL")
 	}
 
-	fields := []string{"`id`", "`uid`", "`filename`", "`external_link`", "`type`", "`size`", "`creator_id`", "`created_ts`", "`updated_ts`", "`internal_path`", "`memo_id`"}
+	fields := []string{"`id`", "`uid`", "`filename`", "`type`", "`size`", "`creator_id`", "`created_ts`", "`updated_ts`", "`memo_id`", "`storage_type`", "`reference`", "`payload`"}
 	if find.GetBlob {
 		fields = append(fields, "`blob`")
 	}
 
-	query := fmt.Sprintf("SELECT %s FROM `resource` WHERE %s ORDER BY `updated_ts` DESC, `created_ts` DESC", strings.Join(fields, ", "), strings.Join(where, " AND "))
+	query := fmt.Sprintf("SELECT %s FROM `resource` WHERE %s ORDER BY `created_ts` DESC", strings.Join(fields, ", "), strings.Join(where, " AND "))
 	if find.Limit != nil {
 		query = fmt.Sprintf("%s LIMIT %d", query, *find.Limit)
 		if find.Offset != nil {
@@ -67,18 +83,21 @@ func (d *DB) ListResources(ctx context.Context, find *store.FindResource) ([]*st
 	for rows.Next() {
 		resource := store.Resource{}
 		var memoID sql.NullInt32
+		var storageType string
+		var payloadBytes []byte
 		dests := []any{
 			&resource.ID,
 			&resource.UID,
 			&resource.Filename,
-			&resource.ExternalLink,
 			&resource.Type,
 			&resource.Size,
 			&resource.CreatorID,
 			&resource.CreatedTs,
 			&resource.UpdatedTs,
-			&resource.InternalPath,
 			&memoID,
+			&storageType,
+			&resource.Reference,
+			&payloadBytes,
 		}
 		if find.GetBlob {
 			dests = append(dests, &resource.Blob)
@@ -86,9 +105,16 @@ func (d *DB) ListResources(ctx context.Context, find *store.FindResource) ([]*st
 		if err := rows.Scan(dests...); err != nil {
 			return nil, err
 		}
+
 		if memoID.Valid {
 			resource.MemoID = &memoID.Int32
 		}
+		resource.StorageType = storepb.ResourceStorageType(storepb.ResourceStorageType_value[storageType])
+		payload := &storepb.ResourcePayload{}
+		if err := protojsonUnmarshaler.Unmarshal(payloadBytes, payload); err != nil {
+			return nil, err
+		}
+		resource.Payload = payload
 		list = append(list, &resource)
 	}
 
@@ -99,7 +125,7 @@ func (d *DB) ListResources(ctx context.Context, find *store.FindResource) ([]*st
 	return list, nil
 }
 
-func (d *DB) UpdateResource(ctx context.Context, update *store.UpdateResource) (*store.Resource, error) {
+func (d *DB) UpdateResource(ctx context.Context, update *store.UpdateResource) error {
 	set, args := []string{}, []any{}
 
 	if v := update.UID; v != nil {
@@ -111,40 +137,20 @@ func (d *DB) UpdateResource(ctx context.Context, update *store.UpdateResource) (
 	if v := update.Filename; v != nil {
 		set, args = append(set, "`filename` = ?"), append(args, *v)
 	}
-	if v := update.InternalPath; v != nil {
-		set, args = append(set, "`internal_path` = ?"), append(args, *v)
-	}
-	if v := update.ExternalLink; v != nil {
-		set, args = append(set, "`external_link` = ?"), append(args, *v)
-	}
 	if v := update.MemoID; v != nil {
 		set, args = append(set, "`memo_id` = ?"), append(args, *v)
 	}
-	if v := update.Blob; v != nil {
-		set, args = append(set, "`blob` = ?"), append(args, v)
-	}
 
 	args = append(args, update.ID)
-	fields := []string{"`id`", "`uid`", "`filename`", "`external_link`", "`type`", "`size`", "`creator_id`", "`created_ts`", "`updated_ts`", "`internal_path`"}
-	stmt := "UPDATE `resource` SET " + strings.Join(set, ", ") + " WHERE `id` = ? RETURNING " + strings.Join(fields, ", ")
-	resource := store.Resource{}
-	dests := []any{
-		&resource.ID,
-		&resource.UID,
-		&resource.Filename,
-		&resource.ExternalLink,
-		&resource.Type,
-		&resource.Size,
-		&resource.CreatorID,
-		&resource.CreatedTs,
-		&resource.UpdatedTs,
-		&resource.InternalPath,
-	}
-	if err := d.db.QueryRowContext(ctx, stmt, args...).Scan(dests...); err != nil {
-		return nil, err
+	stmt := "UPDATE `resource` SET " + strings.Join(set, ", ") + " WHERE `id` = ?"
+	result, err := d.db.ExecContext(ctx, stmt, args...)
+	if err != nil {
+		return errors.Wrap(err, "failed to update resource")
 	}
-
-	return &resource, nil
+	if _, err := result.RowsAffected(); err != nil {
+		return err
+	}
+	return nil
 }
 
 func (d *DB) DeleteResource(ctx context.Context, delete *store.DeleteResource) error {
@@ -156,21 +162,5 @@ func (d *DB) DeleteResource(ctx context.Context, delete *store.DeleteResource) e
 	if _, err := result.RowsAffected(); err != nil {
 		return err
 	}
-
-	if err := d.Vacuum(ctx); err != nil {
-		// Prevent linter warning.
-		return err
-	}
-
-	return nil
-}
-
-func vacuumResource(ctx context.Context, tx *sql.Tx) error {
-	stmt := "DELETE FROM `resource` WHERE `creator_id` NOT IN (SELECT `id` FROM `user`)"
-	_, err := tx.ExecContext(ctx, stmt)
-	if err != nil {
-		return err
-	}
-
 	return nil
 }

store/db/sqlite/sqlite.go

@@ -58,56 +58,6 @@ func (d *DB) GetDB() *sql.DB {
 	return d.db
 }
 
-func (d *DB) Vacuum(ctx context.Context) error {
-	tx, err := d.db.BeginTx(ctx, nil)
-	if err != nil {
-		return err
-	}
-	defer tx.Rollback()
-
-	if err := vacuumImpl(ctx, tx); err != nil {
-		return err
-	}
-
-	if err := tx.Commit(); err != nil {
-		return err
-	}
-
-	// Vacuum sqlite database file size after deleting resource.
-	if _, err := d.db.Exec("VACUUM"); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func vacuumImpl(ctx context.Context, tx *sql.Tx) error {
-	if err := vacuumMemo(ctx, tx); err != nil {
-		return err
-	}
-	if err := vacuumResource(ctx, tx); err != nil {
-		return err
-	}
-	if err := vacuumUserSetting(ctx, tx); err != nil {
-		return err
-	}
-	if err := vacuumMemoOrganizer(ctx, tx); err != nil {
-		return err
-	}
-	if err := vacuumMemoRelations(ctx, tx); err != nil {
-		return err
-	}
-	if err := vacuumInbox(ctx, tx); err != nil {
-		return err
-	}
-	if err := vacuumTag(ctx, tx); err != nil {
-		// Prevent revive warning.
-		return err
-	}
-
-	return nil
-}
-
 func (d *DB) GetCurrentDBSize(context.Context) (int64, error) {
 	fi, err := os.Stat(d.profile.DSN)
 	if err != nil {

store/db/sqlite/tag.go

@@ -2,7 +2,6 @@ package sqlite
 
 import (
 	"context"
-	"database/sql"
 	"strings"
 
 	"github.com/usememos/memos/store"
@@ -79,22 +78,3 @@ func (d *DB) DeleteTag(ctx context.Context, delete *store.DeleteTag) error {
 	}
 	return nil
 }
-
-func vacuumTag(ctx context.Context, tx *sql.Tx) error {
-	stmt := `
-	DELETE FROM 
-		tag 
-	WHERE 
-		creator_id NOT IN (
-			SELECT 
-				id 
-			FROM 
-				user
-		)`
-	_, err := tx.ExecContext(ctx, stmt)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}

store/db/sqlite/user.go

@@ -169,11 +169,5 @@ func (d *DB) DeleteUser(ctx context.Context, delete *store.DeleteUser) error {
 	if _, err := result.RowsAffected(); err != nil {
 		return err
 	}
-
-	if err := d.Vacuum(ctx); err != nil {
-		// Prevent linter warning.
-		return err
-	}
-
 	return nil
 }

store/db/sqlite/user_setting.go

@@ -2,7 +2,6 @@ package sqlite
 
 import (
 	"context"
-	"database/sql"
 	"strings"
 
 	storepb "github.com/usememos/memos/proto/gen/store"
@@ -67,22 +66,3 @@ func (d *DB) ListUserSettings(ctx context.Context, find *store.FindUserSetting)
 
 	return userSettingList, nil
 }
-
-func vacuumUserSetting(ctx context.Context, tx *sql.Tx) error {
-	stmt := `
-	DELETE FROM 
-		user_setting 
-	WHERE 
-		user_id NOT IN (
-			SELECT 
-				id 
-			FROM 
-				user
-		)`
-	_, err := tx.ExecContext(ctx, stmt)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}

store/driver.go

@@ -12,7 +12,6 @@ type Driver interface {
 	Close() error
 
 	Migrate(ctx context.Context) error
-	Vacuum(ctx context.Context) error
 
 	// current file is driver
 	GetCurrentDBSize(ctx context.Context) (int64, error)
@@ -28,7 +27,7 @@ type Driver interface {
 	// Resource model related methods.
 	CreateResource(ctx context.Context, create *Resource) (*Resource, error)
 	ListResources(ctx context.Context, find *FindResource) ([]*Resource, error)
-	UpdateResource(ctx context.Context, update *UpdateResource) (*Resource, error)
+	UpdateResource(ctx context.Context, update *UpdateResource) error
 	DeleteResource(ctx context.Context, delete *DeleteResource) error
 
 	// Memo model related methods.

store/resource.go

@@ -2,18 +2,13 @@ package store
 
 import (
 	"context"
-	"fmt"
 	"os"
 	"path/filepath"
 
 	"github.com/pkg/errors"
 
 	"github.com/usememos/memos/internal/util"
-)
-
-const (
-	// thumbnailImagePath is the directory to store image thumbnails.
-	thumbnailImagePath = ".thumbnail_cache"
+	storepb "github.com/usememos/memos/proto/gen/store"
 )
 
 type Resource struct {
@@ -28,13 +23,16 @@ type Resource struct {
 	UpdatedTs int64
 
 	// Domain specific fields
-	Filename     string
-	Blob         []byte
-	InternalPath string
-	ExternalLink string
-	Type         string
-	Size         int64
-	MemoID       *int32
+	Filename    string
+	Blob        []byte
+	Type        string
+	Size        int64
+	StorageType storepb.ResourceStorageType
+	Reference   string
+	Payload     *storepb.ResourcePayload
+
+	// The related memo ID.
+	MemoID *int32
 }
 
 type FindResource struct {
@@ -50,14 +48,11 @@ type FindResource struct {
 }
 
 type UpdateResource struct {
-	ID           int32
-	UID          *string
-	UpdatedTs    *int64
-	Filename     *string
-	InternalPath *string
-	ExternalLink *string
-	MemoID       *int32
-	Blob         []byte
+	ID        int32
+	UID       *string
+	UpdatedTs *int64
+	Filename  *string
+	MemoID    *int32
 }
 
 type DeleteResource struct {
@@ -89,9 +84,9 @@ func (s *Store) GetResource(ctx context.Context, find *FindResource) (*Resource,
 	return resources[0], nil
 }
 
-func (s *Store) UpdateResource(ctx context.Context, update *UpdateResource) (*Resource, error) {
+func (s *Store) UpdateResource(ctx context.Context, update *UpdateResource) error {
 	if update.UID != nil && !util.UIDMatcher.MatchString(*update.UID) {
-		return nil, errors.New("invalid uid")
+		return errors.New("invalid uid")
 	}
 	return s.driver.UpdateResource(ctx, update)
 }
@@ -106,19 +101,13 @@ func (s *Store) DeleteResource(ctx context.Context, delete *DeleteResource) erro
 	}
 
 	// Delete the local file.
-	if resource.InternalPath != "" {
-		resourcePath := filepath.FromSlash(resource.InternalPath)
-		if !filepath.IsAbs(resourcePath) {
-			resourcePath = filepath.Join(s.Profile.Data, resourcePath)
+	if resource.StorageType == storepb.ResourceStorageType_LOCAL {
+		p := filepath.FromSlash(resource.Reference)
+		if !filepath.IsAbs(p) {
+			p = filepath.Join(s.Profile.Data, p)
 		}
-		_ = os.Remove(resourcePath)
+		_ = os.Remove(p)
 	}
 
-	// Delete the thumbnail.
-	if util.HasPrefixes(resource.Type, "image/png", "image/jpeg") {
-		ext := filepath.Ext(resource.Filename)
-		thumbnailPath := filepath.Join(s.Profile.Data, thumbnailImagePath, fmt.Sprintf("%d%s", resource.ID, ext))
-		_ = os.Remove(thumbnailPath)
-	}
 	return s.driver.DeleteResource(ctx, delete)
 }

store/store.go

@@ -29,10 +29,6 @@ func (*Store) MigrateManually(context.Context) error {
 	return nil
 }
 
-func (s *Store) Vacuum(ctx context.Context) error {
-	return s.driver.Vacuum(ctx)
-}
-
 func (s *Store) Close() error {
 	return s.driver.Close()
 }

test/store/resource_test.go

@@ -14,14 +14,12 @@ func TestResourceStore(t *testing.T) {
 	ctx := context.Background()
 	ts := NewTestingStore(ctx, t)
 	_, err := ts.CreateResource(ctx, &store.Resource{
-		UID:          shortuuid.New(),
-		CreatorID:    101,
-		Filename:     "test.epub",
-		Blob:         []byte("test"),
-		InternalPath: "",
-		ExternalLink: "",
-		Type:         "application/epub+zip",
-		Size:         637607,
+		UID:       shortuuid.New(),
+		CreatorID: 101,
+		Filename:  "test.epub",
+		Blob:      []byte("test"),
+		Type:      "application/epub+zip",
+		Size:      637607,
 	})
 	require.NoError(t, err)
 

web/src/labs/html2image/getCloneStyledElement.ts

@@ -13,13 +13,8 @@ const applyStyles = async (sourceElement: HTMLElement, clonedElement: HTMLElemen
     } catch (error) {
       covertFailed = true;
     }
-    // NOTE: Get image blob from backend to avoid CORS error.
     if (covertFailed) {
-      try {
-        (clonedElement as HTMLImageElement).src = await convertResourceToDataURL(`/o/get/image?url=${url}`);
-      } catch (error) {
-        // do nth
-      }
+      throw new Error(`Failed to convert image to data URL: ${url}`);
     }
   }
 

web/src/utils/resource.ts

@@ -5,7 +5,7 @@ export const getResourceUrl = (resource: Resource) => {
     return resource.externalLink;
   }
 
-  return `${import.meta.env.VITE_API_BASE_URL || window.location.origin}/o/r/${resource.uid}`;
+  return `${import.meta.env.VITE_API_BASE_URL || window.location.origin}/file/${resource.name}`;
 };
 
 export const getResourceType = (resource: Resource) => {

web/vite.config.ts

@@ -23,7 +23,7 @@ export default defineConfig({
         target: devProxyServer,
         xfwd: true,
       },
-      "^/o/": {
+      "^/file": {
         target: devProxyServer,
         xfwd: true,
       },