Commit a4920d46 authored by Steven's avatar Steven

refactor: attachment service part2

parent bb5809ca
// Package httpgetter is using to get resources from url.
// * Get metadata for website;
// * Get image blob to avoid CORS;
package httpgetter
......@@ -7,17 +7,17 @@ import "store/workspace_setting.proto";
option go_package = "gen/store";
enum ResourceStorageType {
RESOURCE_STORAGE_TYPE_UNSPECIFIED = 0;
// Resource is stored locally. AKA, local file system.
enum AttachmentStorageType {
ATTACHMENT_STORAGE_TYPE_UNSPECIFIED = 0;
// Attachment is stored locally. AKA, local file system.
LOCAL = 1;
// Resource is stored in S3.
// Attachment is stored in S3.
S3 = 2;
// Resource is stored in an external storage. The reference is a URL.
// Attachment is stored in an external storage. The reference is a URL.
EXTERNAL = 3;
}
message ResourcePayload {
message AttachmentPayload {
oneof payload {
S3Object s3_object = 1;
}
......
......@@ -18,7 +18,7 @@ var authenticationAllowlistMethods = map[string]bool{
"/memos.api.v1.MemoService/GetMemo": true,
"/memos.api.v1.MemoService/ListMemos": true,
"/memos.api.v1.MarkdownService/GetLinkMetadata": true,
"/memos.api.v1.ResourceService/GetResourceBinary": true,
"/memos.api.v1.AttachmentService/GetAttachmentBinary": true,
}
// isUnauthorizeAllowedMethod returns whether the method is exempted from authentication.
......
This diff is collapsed.
......@@ -22,54 +22,54 @@ func (s *APIV1Service) SetMemoAttachments(ctx context.Context, request *v1pb.Set
if err != nil {
return nil, status.Errorf(codes.Internal, "failed to get memo")
}
resources, err := s.Store.ListResources(ctx, &store.FindResource{
attachments, err := s.Store.ListAttachments(ctx, &store.FindAttachment{
MemoID: &memo.ID,
})
if err != nil {
return nil, status.Errorf(codes.Internal, "failed to list resources")
return nil, status.Errorf(codes.Internal, "failed to list attachments")
}
// Delete resources that are not in the request.
for _, resource := range resources {
// Delete attachments that are not in the request.
for _, attachment := range attachments {
found := false
for _, requestResource := range request.Attachments {
requestResourceUID, err := ExtractAttachmentUIDFromName(requestResource.Name)
for _, requestAttachment := range request.Attachments {
requestAttachmentUID, err := ExtractAttachmentUIDFromName(requestAttachment.Name)
if err != nil {
return nil, status.Errorf(codes.InvalidArgument, "invalid attachment name: %v", err)
}
if resource.UID == requestResourceUID {
if attachment.UID == requestAttachmentUID {
found = true
break
}
}
if !found {
if err = s.Store.DeleteResource(ctx, &store.DeleteResource{
ID: int32(resource.ID),
if err = s.Store.DeleteAttachment(ctx, &store.DeleteAttachment{
ID: int32(attachment.ID),
MemoID: &memo.ID,
}); err != nil {
return nil, status.Errorf(codes.Internal, "failed to delete resource")
return nil, status.Errorf(codes.Internal, "failed to delete attachment")
}
}
}
slices.Reverse(request.Attachments)
// Update resources' memo_id in the request.
for index, resource := range request.Attachments {
resourceUID, err := ExtractAttachmentUIDFromName(resource.Name)
// Update attachments' memo_id in the request.
for index, attachment := range request.Attachments {
attachmentUID, err := ExtractAttachmentUIDFromName(attachment.Name)
if err != nil {
return nil, status.Errorf(codes.InvalidArgument, "invalid attachment name: %v", err)
}
tempResource, err := s.Store.GetResource(ctx, &store.FindResource{UID: &resourceUID})
tempAttachment, err := s.Store.GetAttachment(ctx, &store.FindAttachment{UID: &attachmentUID})
if err != nil {
return nil, status.Errorf(codes.Internal, "failed to get resource: %v", err)
return nil, status.Errorf(codes.Internal, "failed to get attachment: %v", err)
}
updatedTs := time.Now().Unix() + int64(index)
if err := s.Store.UpdateResource(ctx, &store.UpdateResource{
ID: tempResource.ID,
if err := s.Store.UpdateAttachment(ctx, &store.UpdateAttachment{
ID: tempAttachment.ID,
MemoID: &memo.ID,
UpdatedTs: &updatedTs,
}); err != nil {
return nil, status.Errorf(codes.Internal, "failed to update resource: %v", err)
return nil, status.Errorf(codes.Internal, "failed to update attachment: %v", err)
}
}
......@@ -85,18 +85,18 @@ func (s *APIV1Service) ListMemoAttachments(ctx context.Context, request *v1pb.Li
if err != nil {
return nil, status.Errorf(codes.Internal, "failed to get memo: %v", err)
}
resources, err := s.Store.ListResources(ctx, &store.FindResource{
attachments, err := s.Store.ListAttachments(ctx, &store.FindAttachment{
MemoID: &memo.ID,
})
if err != nil {
return nil, status.Errorf(codes.Internal, "failed to list resources: %v", err)
return nil, status.Errorf(codes.Internal, "failed to list attachments: %v", err)
}
response := &v1pb.ListMemoAttachmentsResponse{
Attachments: []*v1pb.Attachment{},
}
for _, resource := range resources {
response.Attachments = append(response.Attachments, s.convertAttachmentFromStore(ctx, resource))
for _, attachment := range attachments {
response.Attachments = append(response.Attachments, s.convertAttachmentFromStore(ctx, attachment))
}
return response, nil
}
......@@ -399,14 +399,14 @@ func (s *APIV1Service) DeleteMemo(ctx context.Context, request *v1pb.DeleteMemoR
return nil, status.Errorf(codes.Internal, "failed to delete memo relations")
}
// Delete related resources.
resources, err := s.Store.ListResources(ctx, &store.FindResource{MemoID: &memo.ID})
// Delete related attachments.
attachments, err := s.Store.ListAttachments(ctx, &store.FindAttachment{MemoID: &memo.ID})
if err != nil {
return nil, status.Errorf(codes.Internal, "failed to list resources")
return nil, status.Errorf(codes.Internal, "failed to list attachments")
}
for _, resource := range resources {
if err := s.Store.DeleteResource(ctx, &store.DeleteResource{ID: resource.ID}); err != nil {
return nil, status.Errorf(codes.Internal, "failed to delete resource")
for _, attachment := range attachments {
if err := s.Store.DeleteAttachment(ctx, &store.DeleteAttachment{ID: attachment.ID}); err != nil {
return nil, status.Errorf(codes.Internal, "failed to delete attachment")
}
}
......
......@@ -124,22 +124,22 @@ func (s *RSSService) generateRSSFromMemoList(ctx context.Context, memoList []*st
Created: time.Unix(memo.CreatedTs, 0),
Id: link.Href,
}
resources, err := s.Store.ListResources(ctx, &store.FindResource{
attachments, err := s.Store.ListAttachments(ctx, &store.FindAttachment{
MemoID: &memo.ID,
})
if err != nil {
return "", err
}
if len(resources) > 0 {
resource := resources[0]
if len(attachments) > 0 {
attachment := attachments[0]
enclosure := feeds.Enclosure{}
if resource.StorageType == storepb.ResourceStorageType_EXTERNAL || resource.StorageType == storepb.ResourceStorageType_S3 {
enclosure.Url = resource.Reference
if attachment.StorageType == storepb.AttachmentStorageType_EXTERNAL || attachment.StorageType == storepb.AttachmentStorageType_S3 {
enclosure.Url = attachment.Reference
} else {
enclosure.Url = fmt.Sprintf("%s/file/attachments/%s/%s", baseURL, resource.UID, resource.Filename)
enclosure.Url = fmt.Sprintf("%s/file/attachments/%s/%s", baseURL, attachment.UID, attachment.Filename)
}
enclosure.Length = strconv.Itoa(int(resource.Size))
enclosure.Type = resource.Type
enclosure.Length = strconv.Itoa(int(attachment.Size))
enclosure.Type = attachment.Type
feed.Items[i].Enclosure = &enclosure
}
}
......
......@@ -49,33 +49,33 @@ func (r *Runner) CheckAndPresign(ctx context.Context) {
return
}
s3StorageType := storepb.ResourceStorageType_S3
// Limit resources to a reasonable batch size
s3StorageType := storepb.AttachmentStorageType_S3
// Limit attachments to a reasonable batch size
const batchSize = 100
offset := 0
for {
limit := batchSize
resources, err := r.Store.ListResources(ctx, &store.FindResource{
attachments, err := r.Store.ListAttachments(ctx, &store.FindAttachment{
GetBlob: false,
StorageType: &s3StorageType,
Limit: &limit,
Offset: &offset,
})
if err != nil {
slog.Error("Failed to list resources for presigning", "error", err)
slog.Error("Failed to list attachments for presigning", "error", err)
return
}
// Break if no more resources
if len(resources) == 0 {
// Break if no more attachments
if len(attachments) == 0 {
break
}
// Process batch of resources
// Process batch of attachments
presignCount := 0
for _, resource := range resources {
s3ObjectPayload := resource.Payload.GetS3Object()
for _, attachment := range attachments {
s3ObjectPayload := attachment.Payload.GetS3Object()
if s3ObjectPayload == nil {
continue
}
......@@ -105,30 +105,30 @@ func (r *Runner) CheckAndPresign(ctx context.Context) {
presignURL, err := s3Client.PresignGetObject(ctx, s3ObjectPayload.Key)
if err != nil {
slog.Error("Failed to presign URL", "error", err, "resourceID", resource.ID)
slog.Error("Failed to presign URL", "error", err, "attachmentID", attachment.ID)
continue
}
s3ObjectPayload.S3Config = s3Config
s3ObjectPayload.LastPresignedTime = timestamppb.New(time.Now())
if err := r.Store.UpdateResource(ctx, &store.UpdateResource{
ID: resource.ID,
if err := r.Store.UpdateAttachment(ctx, &store.UpdateAttachment{
ID: attachment.ID,
Reference: &presignURL,
Payload: &storepb.ResourcePayload{
Payload: &storepb.ResourcePayload_S3Object_{
Payload: &storepb.AttachmentPayload{
Payload: &storepb.AttachmentPayload_S3Object_{
S3Object: s3ObjectPayload,
},
},
}); err != nil {
slog.Error("Failed to update resource", "error", err, "resourceID", resource.ID)
slog.Error("Failed to update attachment", "error", err, "attachmentID", attachment.ID)
continue
}
presignCount++
}
slog.Info("Presigned batch of S3 resources", "batchSize", len(resources), "presigned", presignCount)
slog.Info("Presigned batch of S3 attachments", "batchSize", len(attachments), "presigned", presignCount)
// Move to next batch
offset += len(resources)
offset += len(attachments)
}
}
......@@ -13,10 +13,10 @@ import (
storepb "github.com/usememos/memos/proto/gen/store"
)
type Resource struct {
// ID is the system generated unique identifier for the resource.
type Attachment struct {
// ID is the system generated unique identifier for the attachment.
ID int32
// UID is the user defined unique identifier for the resource.
// UID is the user defined unique identifier for the attachment.
UID string
// Standard fields
......@@ -29,15 +29,15 @@ type Resource struct {
Blob []byte
Type string
Size int64
StorageType storepb.ResourceStorageType
StorageType storepb.AttachmentStorageType
Reference string
Payload *storepb.ResourcePayload
Payload *storepb.AttachmentPayload
// The related memo ID.
MemoID *int32
}
type FindResource struct {
type FindAttachment struct {
GetBlob bool
ID *int32
UID *string
......@@ -46,35 +46,35 @@ type FindResource struct {
FilenameSearch *string
MemoID *int32
HasRelatedMemo bool
StorageType *storepb.ResourceStorageType
StorageType *storepb.AttachmentStorageType
Limit *int
Offset *int
}
type UpdateResource struct {
type UpdateAttachment struct {
ID int32
UID *string
UpdatedTs *int64
Filename *string
MemoID *int32
Reference *string
Payload *storepb.ResourcePayload
Payload *storepb.AttachmentPayload
}
type DeleteResource struct {
type DeleteAttachment struct {
ID int32
MemoID *int32
}
func (s *Store) CreateResource(ctx context.Context, create *Resource) (*Resource, error) {
func (s *Store) CreateAttachment(ctx context.Context, create *Attachment) (*Attachment, error) {
if !base.UIDMatcher.MatchString(create.UID) {
return nil, errors.New("invalid uid")
}
return s.driver.CreateResource(ctx, create)
return s.driver.CreateAttachment(ctx, create)
}
func (s *Store) ListResources(ctx context.Context, find *FindResource) ([]*Resource, error) {
// Set default limits to prevent loading too many resources at once
func (s *Store) ListAttachments(ctx context.Context, find *FindAttachment) ([]*Attachment, error) {
// Set default limits to prevent loading too many attachments at once
if find.Limit == nil && find.GetBlob {
// When fetching blobs, we should be especially careful with limits
defaultLimit := 10
......@@ -85,41 +85,41 @@ func (s *Store) ListResources(ctx context.Context, find *FindResource) ([]*Resou
find.Limit = &defaultLimit
}
return s.driver.ListResources(ctx, find)
return s.driver.ListAttachments(ctx, find)
}
func (s *Store) GetResource(ctx context.Context, find *FindResource) (*Resource, error) {
resources, err := s.ListResources(ctx, find)
func (s *Store) GetAttachment(ctx context.Context, find *FindAttachment) (*Attachment, error) {
attachments, err := s.ListAttachments(ctx, find)
if err != nil {
return nil, err
}
if len(resources) == 0 {
if len(attachments) == 0 {
return nil, nil
}
return resources[0], nil
return attachments[0], nil
}
func (s *Store) UpdateResource(ctx context.Context, update *UpdateResource) error {
func (s *Store) UpdateAttachment(ctx context.Context, update *UpdateAttachment) error {
if update.UID != nil && !base.UIDMatcher.MatchString(*update.UID) {
return errors.New("invalid uid")
}
return s.driver.UpdateResource(ctx, update)
return s.driver.UpdateAttachment(ctx, update)
}
func (s *Store) DeleteResource(ctx context.Context, delete *DeleteResource) error {
resource, err := s.GetResource(ctx, &FindResource{ID: &delete.ID})
func (s *Store) DeleteAttachment(ctx context.Context, delete *DeleteAttachment) error {
attachment, err := s.GetAttachment(ctx, &FindAttachment{ID: &delete.ID})
if err != nil {
return errors.Wrap(err, "failed to get resource")
return errors.Wrap(err, "failed to get attachment")
}
if resource == nil {
return errors.New("resource not found")
if attachment == nil {
return errors.New("attachment not found")
}
if resource.StorageType == storepb.ResourceStorageType_LOCAL {
if attachment.StorageType == storepb.AttachmentStorageType_LOCAL {
if err := func() error {
p := filepath.FromSlash(resource.Reference)
p := filepath.FromSlash(attachment.Reference)
if !filepath.IsAbs(p) {
p = filepath.Join(s.profile.Data, p)
}
......@@ -131,9 +131,9 @@ func (s *Store) DeleteResource(ctx context.Context, delete *DeleteResource) erro
}(); err != nil {
return errors.Wrap(err, "failed to delete local file")
}
} else if resource.StorageType == storepb.ResourceStorageType_S3 {
} else if attachment.StorageType == storepb.AttachmentStorageType_S3 {
if err := func() error {
s3ObjectPayload := resource.Payload.GetS3Object()
s3ObjectPayload := attachment.Payload.GetS3Object()
if s3ObjectPayload == nil {
return errors.Errorf("No s3 object found")
}
......@@ -162,5 +162,5 @@ func (s *Store) DeleteResource(ctx context.Context, delete *DeleteResource) erro
}
}
return s.driver.DeleteResource(ctx, delete)
return s.driver.DeleteAttachment(ctx, delete)
}
......@@ -13,18 +13,18 @@ import (
"github.com/usememos/memos/store"
)
func (d *DB) CreateResource(ctx context.Context, create *store.Resource) (*store.Resource, error) {
func (d *DB) CreateAttachment(ctx context.Context, create *store.Attachment) (*store.Attachment, error) {
fields := []string{"`uid`", "`filename`", "`blob`", "`type`", "`size`", "`creator_id`", "`memo_id`", "`storage_type`", "`reference`", "`payload`"}
placeholder := []string{"?", "?", "?", "?", "?", "?", "?", "?", "?", "?"}
storageType := ""
if create.StorageType != storepb.ResourceStorageType_RESOURCE_STORAGE_TYPE_UNSPECIFIED {
if create.StorageType != storepb.AttachmentStorageType_ATTACHMENT_STORAGE_TYPE_UNSPECIFIED {
storageType = create.StorageType.String()
}
payloadString := "{}"
if create.Payload != nil {
bytes, err := protojson.Marshal(create.Payload)
if err != nil {
return nil, errors.Wrap(err, "failed to marshal resource payload")
return nil, errors.Wrap(err, "failed to marshal attachment payload")
}
payloadString = string(bytes)
}
......@@ -42,10 +42,10 @@ func (d *DB) CreateResource(ctx context.Context, create *store.Resource) (*store
}
id32 := int32(id)
return d.GetResource(ctx, &store.FindResource{ID: &id32})
return d.GetAttachment(ctx, &store.FindAttachment{ID: &id32})
}
func (d *DB) ListResources(ctx context.Context, find *store.FindResource) ([]*store.Resource, error) {
func (d *DB) ListAttachments(ctx context.Context, find *store.FindAttachment) ([]*store.Attachment, error) {
where, args := []string{"1 = 1"}, []any{}
if v := find.ID; v != nil {
......@@ -92,43 +92,43 @@ func (d *DB) ListResources(ctx context.Context, find *store.FindResource) ([]*st
}
defer rows.Close()
list := make([]*store.Resource, 0)
list := make([]*store.Attachment, 0)
for rows.Next() {
resource := store.Resource{}
attachment := store.Attachment{}
var memoID sql.NullInt32
var storageType string
var payloadBytes []byte
dests := []any{
&resource.ID,
&resource.UID,
&resource.Filename,
&resource.Type,
&resource.Size,
&resource.CreatorID,
&resource.CreatedTs,
&resource.UpdatedTs,
&attachment.ID,
&attachment.UID,
&attachment.Filename,
&attachment.Type,
&attachment.Size,
&attachment.CreatorID,
&attachment.CreatedTs,
&attachment.UpdatedTs,
&memoID,
&storageType,
&resource.Reference,
&attachment.Reference,
&payloadBytes,
}
if find.GetBlob {
dests = append(dests, &resource.Blob)
dests = append(dests, &attachment.Blob)
}
if err := rows.Scan(dests...); err != nil {
return nil, err
}
if memoID.Valid {
resource.MemoID = &memoID.Int32
attachment.MemoID = &memoID.Int32
}
resource.StorageType = storepb.ResourceStorageType(storepb.ResourceStorageType_value[storageType])
payload := &storepb.ResourcePayload{}
attachment.StorageType = storepb.AttachmentStorageType(storepb.AttachmentStorageType_value[storageType])
payload := &storepb.AttachmentPayload{}
if err := protojsonUnmarshaler.Unmarshal(payloadBytes, payload); err != nil {
return nil, err
}
resource.Payload = payload
list = append(list, &resource)
attachment.Payload = payload
list = append(list, &attachment)
}
if err := rows.Err(); err != nil {
......@@ -138,8 +138,8 @@ func (d *DB) ListResources(ctx context.Context, find *store.FindResource) ([]*st
return list, nil
}
func (d *DB) GetResource(ctx context.Context, find *store.FindResource) (*store.Resource, error) {
list, err := d.ListResources(ctx, find)
func (d *DB) GetAttachment(ctx context.Context, find *store.FindAttachment) (*store.Attachment, error) {
list, err := d.ListAttachments(ctx, find)
if err != nil {
return nil, err
}
......@@ -150,7 +150,7 @@ func (d *DB) GetResource(ctx context.Context, find *store.FindResource) (*store.
return list[0], nil
}
func (d *DB) UpdateResource(ctx context.Context, update *store.UpdateResource) error {
func (d *DB) UpdateAttachment(ctx context.Context, update *store.UpdateAttachment) error {
set, args := []string{}, []any{}
if v := update.UID; v != nil {
......@@ -171,7 +171,7 @@ func (d *DB) UpdateResource(ctx context.Context, update *store.UpdateResource) e
if v := update.Payload; v != nil {
bytes, err := protojson.Marshal(v)
if err != nil {
return errors.Wrap(err, "failed to marshal resource payload")
return errors.Wrap(err, "failed to marshal attachment payload")
}
set, args = append(set, "`payload` = ?"), append(args, string(bytes))
}
......@@ -188,7 +188,7 @@ func (d *DB) UpdateResource(ctx context.Context, update *store.UpdateResource) e
return nil
}
func (d *DB) DeleteResource(ctx context.Context, delete *store.DeleteResource) error {
func (d *DB) DeleteAttachment(ctx context.Context, delete *store.DeleteAttachment) error {
stmt := "DELETE FROM `resource` WHERE `id` = ?"
result, err := d.db.ExecContext(ctx, stmt, delete.ID)
if err != nil {
......
......@@ -13,17 +13,17 @@ import (
"github.com/usememos/memos/store"
)
func (d *DB) CreateResource(ctx context.Context, create *store.Resource) (*store.Resource, error) {
func (d *DB) CreateAttachment(ctx context.Context, create *store.Attachment) (*store.Attachment, error) {
fields := []string{"uid", "filename", "blob", "type", "size", "creator_id", "memo_id", "storage_type", "reference", "payload"}
storageType := ""
if create.StorageType != storepb.ResourceStorageType_RESOURCE_STORAGE_TYPE_UNSPECIFIED {
if create.StorageType != storepb.AttachmentStorageType_ATTACHMENT_STORAGE_TYPE_UNSPECIFIED {
storageType = create.StorageType.String()
}
payloadString := "{}"
if create.Payload != nil {
bytes, err := protojson.Marshal(create.Payload)
if err != nil {
return nil, errors.Wrap(err, "failed to marshal resource payload")
return nil, errors.Wrap(err, "failed to marshal attachment payload")
}
payloadString = string(bytes)
}
......@@ -36,7 +36,7 @@ func (d *DB) CreateResource(ctx context.Context, create *store.Resource) (*store
return create, nil
}
func (d *DB) ListResources(ctx context.Context, find *store.FindResource) ([]*store.Resource, error) {
func (d *DB) ListAttachments(ctx context.Context, find *store.FindAttachment) ([]*store.Attachment, error) {
where, args := []string{"1 = 1"}, []any{}
if v := find.ID; v != nil {
......@@ -89,43 +89,43 @@ func (d *DB) ListResources(ctx context.Context, find *store.FindResource) ([]*st
}
defer rows.Close()
list := make([]*store.Resource, 0)
list := make([]*store.Attachment, 0)
for rows.Next() {
resource := store.Resource{}
attachment := store.Attachment{}
var memoID sql.NullInt32
var storageType string
var payloadBytes []byte
dests := []any{
&resource.ID,
&resource.UID,
&resource.Filename,
&resource.Type,
&resource.Size,
&resource.CreatorID,
&resource.CreatedTs,
&resource.UpdatedTs,
&attachment.ID,
&attachment.UID,
&attachment.Filename,
&attachment.Type,
&attachment.Size,
&attachment.CreatorID,
&attachment.CreatedTs,
&attachment.UpdatedTs,
&memoID,
&storageType,
&resource.Reference,
&attachment.Reference,
&payloadBytes,
}
if find.GetBlob {
dests = append(dests, &resource.Blob)
dests = append(dests, &attachment.Blob)
}
if err := rows.Scan(dests...); err != nil {
return nil, err
}
if memoID.Valid {
resource.MemoID = &memoID.Int32
attachment.MemoID = &memoID.Int32
}
resource.StorageType = storepb.ResourceStorageType(storepb.ResourceStorageType_value[storageType])
payload := &storepb.ResourcePayload{}
attachment.StorageType = storepb.AttachmentStorageType(storepb.AttachmentStorageType_value[storageType])
payload := &storepb.AttachmentPayload{}
if err := protojsonUnmarshaler.Unmarshal(payloadBytes, payload); err != nil {
return nil, err
}
resource.Payload = payload
list = append(list, &resource)
attachment.Payload = payload
list = append(list, &attachment)
}
if err := rows.Err(); err != nil {
......@@ -135,7 +135,7 @@ func (d *DB) ListResources(ctx context.Context, find *store.FindResource) ([]*st
return list, nil
}
func (d *DB) UpdateResource(ctx context.Context, update *store.UpdateResource) error {
func (d *DB) UpdateAttachment(ctx context.Context, update *store.UpdateAttachment) error {
set, args := []string{}, []any{}
if v := update.UID; v != nil {
......@@ -156,7 +156,7 @@ func (d *DB) UpdateResource(ctx context.Context, update *store.UpdateResource) e
if v := update.Payload; v != nil {
bytes, err := protojson.Marshal(v)
if err != nil {
return errors.Wrap(err, "failed to marshal resource payload")
return errors.Wrap(err, "failed to marshal attachment payload")
}
set, args = append(set, "payload = "+placeholder(len(args)+1)), append(args, string(bytes))
}
......@@ -173,7 +173,7 @@ func (d *DB) UpdateResource(ctx context.Context, update *store.UpdateResource) e
return nil
}
func (d *DB) DeleteResource(ctx context.Context, delete *store.DeleteResource) error {
func (d *DB) DeleteAttachment(ctx context.Context, delete *store.DeleteAttachment) error {
stmt := `DELETE FROM resource WHERE id = $1`
result, err := d.db.ExecContext(ctx, stmt, delete.ID)
if err != nil {
......
......@@ -13,18 +13,18 @@ import (
"github.com/usememos/memos/store"
)
func (d *DB) CreateResource(ctx context.Context, create *store.Resource) (*store.Resource, error) {
func (d *DB) CreateAttachment(ctx context.Context, create *store.Attachment) (*store.Attachment, error) {
fields := []string{"`uid`", "`filename`", "`blob`", "`type`", "`size`", "`creator_id`", "`memo_id`", "`storage_type`", "`reference`", "`payload`"}
placeholder := []string{"?", "?", "?", "?", "?", "?", "?", "?", "?", "?"}
storageType := ""
if create.StorageType != storepb.ResourceStorageType_RESOURCE_STORAGE_TYPE_UNSPECIFIED {
if create.StorageType != storepb.AttachmentStorageType_ATTACHMENT_STORAGE_TYPE_UNSPECIFIED {
storageType = create.StorageType.String()
}
payloadString := "{}"
if create.Payload != nil {
bytes, err := protojson.Marshal(create.Payload)
if err != nil {
return nil, errors.Wrap(err, "failed to marshal resource payload")
return nil, errors.Wrap(err, "failed to marshal attachment payload")
}
payloadString = string(bytes)
}
......@@ -38,7 +38,7 @@ func (d *DB) CreateResource(ctx context.Context, create *store.Resource) (*store
return create, nil
}
func (d *DB) ListResources(ctx context.Context, find *store.FindResource) ([]*store.Resource, error) {
func (d *DB) ListAttachments(ctx context.Context, find *store.FindAttachment) ([]*store.Attachment, error) {
where, args := []string{"1 = 1"}, []any{}
if v := find.ID; v != nil {
......@@ -85,43 +85,43 @@ func (d *DB) ListResources(ctx context.Context, find *store.FindResource) ([]*st
}
defer rows.Close()
list := make([]*store.Resource, 0)
list := make([]*store.Attachment, 0)
for rows.Next() {
resource := store.Resource{}
attachment := store.Attachment{}
var memoID sql.NullInt32
var storageType string
var payloadBytes []byte
dests := []any{
&resource.ID,
&resource.UID,
&resource.Filename,
&resource.Type,
&resource.Size,
&resource.CreatorID,
&resource.CreatedTs,
&resource.UpdatedTs,
&attachment.ID,
&attachment.UID,
&attachment.Filename,
&attachment.Type,
&attachment.Size,
&attachment.CreatorID,
&attachment.CreatedTs,
&attachment.UpdatedTs,
&memoID,
&storageType,
&resource.Reference,
&attachment.Reference,
&payloadBytes,
}
if find.GetBlob {
dests = append(dests, &resource.Blob)
dests = append(dests, &attachment.Blob)
}
if err := rows.Scan(dests...); err != nil {
return nil, err
}
if memoID.Valid {
resource.MemoID = &memoID.Int32
attachment.MemoID = &memoID.Int32
}
resource.StorageType = storepb.ResourceStorageType(storepb.ResourceStorageType_value[storageType])
payload := &storepb.ResourcePayload{}
attachment.StorageType = storepb.AttachmentStorageType(storepb.AttachmentStorageType_value[storageType])
payload := &storepb.AttachmentPayload{}
if err := protojsonUnmarshaler.Unmarshal(payloadBytes, payload); err != nil {
return nil, err
}
resource.Payload = payload
list = append(list, &resource)
attachment.Payload = payload
list = append(list, &attachment)
}
if err := rows.Err(); err != nil {
......@@ -131,7 +131,7 @@ func (d *DB) ListResources(ctx context.Context, find *store.FindResource) ([]*st
return list, nil
}
func (d *DB) UpdateResource(ctx context.Context, update *store.UpdateResource) error {
func (d *DB) UpdateAttachment(ctx context.Context, update *store.UpdateAttachment) error {
set, args := []string{}, []any{}
if v := update.UID; v != nil {
......@@ -152,7 +152,7 @@ func (d *DB) UpdateResource(ctx context.Context, update *store.UpdateResource) e
if v := update.Payload; v != nil {
bytes, err := protojson.Marshal(v)
if err != nil {
return errors.Wrap(err, "failed to marshal resource payload")
return errors.Wrap(err, "failed to marshal attachment payload")
}
set, args = append(set, "`payload` = ?"), append(args, string(bytes))
}
......@@ -161,7 +161,7 @@ func (d *DB) UpdateResource(ctx context.Context, update *store.UpdateResource) e
stmt := "UPDATE `resource` SET " + strings.Join(set, ", ") + " WHERE `id` = ?"
result, err := d.db.ExecContext(ctx, stmt, args...)
if err != nil {
return errors.Wrap(err, "failed to update resource")
return errors.Wrap(err, "failed to update attachment")
}
if _, err := result.RowsAffected(); err != nil {
return err
......@@ -169,7 +169,7 @@ func (d *DB) UpdateResource(ctx context.Context, update *store.UpdateResource) e
return nil
}
func (d *DB) DeleteResource(ctx context.Context, delete *store.DeleteResource) error {
func (d *DB) DeleteAttachment(ctx context.Context, delete *store.DeleteAttachment) error {
stmt := "DELETE FROM `resource` WHERE `id` = ?"
result, err := d.db.ExecContext(ctx, stmt, delete.ID)
if err != nil {
......
......@@ -25,11 +25,11 @@ type Driver interface {
CreateActivity(ctx context.Context, create *Activity) (*Activity, error)
ListActivities(ctx context.Context, find *FindActivity) ([]*Activity, error)
// Resource model related methods.
CreateResource(ctx context.Context, create *Resource) (*Resource, error)
ListResources(ctx context.Context, find *FindResource) ([]*Resource, error)
UpdateResource(ctx context.Context, update *UpdateResource) error
DeleteResource(ctx context.Context, delete *DeleteResource) error
// Attachment model related methods.
CreateAttachment(ctx context.Context, create *Attachment) (*Attachment, error)
ListAttachments(ctx context.Context, find *FindAttachment) ([]*Attachment, error)
UpdateAttachment(ctx context.Context, update *UpdateAttachment) error
DeleteAttachment(ctx context.Context, delete *DeleteAttachment) error
// Memo model related methods.
CreateMemo(ctx context.Context, create *Memo) (*Memo, error)
......
......@@ -10,10 +10,10 @@ import (
"github.com/usememos/memos/store"
)
func TestResourceStore(t *testing.T) {
func TestAttachmentStore(t *testing.T) {
ctx := context.Background()
ts := NewTestingStore(ctx, t)
_, err := ts.CreateResource(ctx, &store.Resource{
_, err := ts.CreateAttachment(ctx, &store.Attachment{
UID: shortuuid.New(),
CreatorID: 101,
Filename: "test.epub",
......@@ -25,39 +25,39 @@ func TestResourceStore(t *testing.T) {
correctFilename := "test.epub"
incorrectFilename := "test.png"
resource, err := ts.GetResource(ctx, &store.FindResource{
attachment, err := ts.GetAttachment(ctx, &store.FindAttachment{
Filename: &correctFilename,
})
require.NoError(t, err)
require.Equal(t, correctFilename, resource.Filename)
require.Equal(t, int32(1), resource.ID)
require.Equal(t, correctFilename, attachment.Filename)
require.Equal(t, int32(1), attachment.ID)
notFoundResource, err := ts.GetResource(ctx, &store.FindResource{
notFoundAttachment, err := ts.GetAttachment(ctx, &store.FindAttachment{
Filename: &incorrectFilename,
})
require.NoError(t, err)
require.Nil(t, notFoundResource)
require.Nil(t, notFoundAttachment)
var correctCreatorID int32 = 101
var incorrectCreatorID int32 = 102
_, err = ts.GetResource(ctx, &store.FindResource{
_, err = ts.GetAttachment(ctx, &store.FindAttachment{
CreatorID: &correctCreatorID,
})
require.NoError(t, err)
notFoundResource, err = ts.GetResource(ctx, &store.FindResource{
notFoundAttachment, err = ts.GetAttachment(ctx, &store.FindAttachment{
CreatorID: &incorrectCreatorID,
})
require.NoError(t, err)
require.Nil(t, notFoundResource)
require.Nil(t, notFoundAttachment)
err = ts.DeleteResource(ctx, &store.DeleteResource{
err = ts.DeleteAttachment(ctx, &store.DeleteAttachment{
ID: 1,
})
require.NoError(t, err)
err = ts.DeleteResource(ctx, &store.DeleteResource{
err = ts.DeleteAttachment(ctx, &store.DeleteAttachment{
ID: 2,
})
require.ErrorContains(t, err, "resource not found")
require.ErrorContains(t, err, "attachment not found")
ts.Close()
}
......@@ -17,15 +17,15 @@ import showPreviewImageDialog from "./PreviewImageDialog";
import SquareDiv from "./kit/SquareDiv";
interface Props {
resource: Attachment;
attachment: Attachment;
className?: string;
strokeWidth?: number;
}
const ResourceIcon = (props: Props) => {
const { resource } = props;
const resourceType = getAttachmentType(resource);
const resourceUrl = getAttachmentUrl(resource);
const AttachmentIcon = (props: Props) => {
const { attachment } = props;
const resourceType = getAttachmentType(attachment);
const resourceUrl = getAttachmentUrl(attachment);
const className = cn("w-full h-auto", props.className);
const strokeWidth = props.strokeWidth;
......@@ -38,7 +38,7 @@ const ResourceIcon = (props: Props) => {
<SquareDiv className={cn(className, "flex items-center justify-center overflow-clip")}>
<img
className="min-w-full min-h-full object-cover"
src={resource.externalLink ? resourceUrl : resourceUrl + "?thumbnail=true"}
src={attachment.externalLink ? resourceUrl : resourceUrl + "?thumbnail=true"}
onClick={() => showPreviewImageDialog(resourceUrl)}
decoding="async"
loading="lazy"
......@@ -47,7 +47,7 @@ const ResourceIcon = (props: Props) => {
);
}
const getResourceIcon = () => {
const getAttachmentIcon = () => {
switch (resourceType) {
case "video/*":
return <FileVideo2Icon strokeWidth={strokeWidth} className="w-full h-auto" />;
......@@ -74,9 +74,9 @@ const ResourceIcon = (props: Props) => {
return (
<div onClick={previewResource} className={cn(className, "max-w-16 opacity-50")}>
{getResourceIcon()}
{getAttachmentIcon()}
</div>
);
};
export default React.memo(ResourceIcon);
export default React.memo(AttachmentIcon);
import { Attachment } from "@/types/proto/api/v1/attachment_service";
import { getAttachmentUrl } from "@/utils/attachment";
import AttachmentIcon from "./AttachmentIcon";
interface Props {
attachment: Attachment;
className?: string;
}
const MemoAttachment: React.FC<Props> = (props: Props) => {
const { className, attachment } = props;
const attachmentUrl = getAttachmentUrl(attachment);
const handlePreviewBtnClick = () => {
window.open(attachmentUrl);
};
return (
<div className={`w-auto flex flex-row justify-start items-center text-gray-500 dark:text-gray-400 hover:opacity-80 ${className}`}>
{attachment.type.startsWith("audio") ? (
<audio src={attachmentUrl} controls></audio>
) : (
<>
<AttachmentIcon className="w-4! h-4! mr-1" attachment={attachment} />
<span className="text-sm max-w-[256px] truncate cursor-pointer" onClick={handlePreviewBtnClick}>
{attachment.filename}
</span>
</>
)}
</div>
);
};
export default MemoAttachment;
......@@ -2,7 +2,7 @@ import { memo } from "react";
import { Attachment } from "@/types/proto/api/v1/attachment_service";
import { cn } from "@/utils";
import { getAttachmentType, getAttachmentUrl } from "@/utils/attachment";
import MemoResource from "./MemoResource";
import MemoAttachment from "./MemoAttachment";
import showPreviewImageDialog from "./PreviewImageDialog";
const MemoAttachmentListView = ({ attachments = [] }: { attachments: Attachment[] }) => {
......@@ -78,7 +78,7 @@ const MemoAttachmentListView = ({ attachments = [] }: { attachments: Attachment[
return (
<div className="w-full flex flex-row justify-start overflow-auto gap-2">
{otherAttachments.map((attachment) => (
<MemoResource key={attachment.name} resource={attachment} />
<MemoAttachment key={attachment.name} attachment={attachment} />
))}
</div>
);
......
import { Button } from "@usememos/mui";
import { LoaderIcon, PaperclipIcon } from "lucide-react";
import { observer } from "mobx-react-lite";
import { useContext, useRef, useState } from "react";
import toast from "react-hot-toast";
import { attachmentStore } from "@/store/v2";
import { Attachment } from "@/types/proto/api/v1/attachment_service";
import { MemoEditorContext } from "../types";
interface Props {
isUploading?: boolean;
}
interface State {
uploadingFlag: boolean;
}
const UploadAttachmentButton = observer((props: Props) => {
const context = useContext(MemoEditorContext);
const [state, setState] = useState<State>({
uploadingFlag: false,
});
const fileInputRef = useRef<HTMLInputElement>(null);
const handleFileInputChange = async () => {
if (!fileInputRef.current || !fileInputRef.current.files || fileInputRef.current.files.length === 0) {
return;
}
if (state.uploadingFlag) {
return;
}
setState((state) => {
return {
...state,
uploadingFlag: true,
};
});
const createdAttachmentList: Attachment[] = [];
try {
if (!fileInputRef.current || !fileInputRef.current.files) {
return;
}
for (const file of fileInputRef.current.files) {
const { name: filename, size, type } = file;
const buffer = new Uint8Array(await file.arrayBuffer());
const attachment = await attachmentStore.createAttachment({
attachment: Attachment.fromPartial({
filename,
size,
type,
content: buffer,
}),
attachmentId: "",
});
createdAttachmentList.push(attachment);
}
} catch (error: any) {
console.error(error);
toast.error(error.details);
}
context.setAttachmentList([...context.attachmentList, ...createdAttachmentList]);
setState((state) => {
return {
...state,
uploadingFlag: false,
};
});
};
const isUploading = state.uploadingFlag || props.isUploading;
return (
<Button className="relative p-0" variant="plain" disabled={isUploading}>
{isUploading ? <LoaderIcon className="w-5 h-5 animate-spin" /> : <PaperclipIcon className="w-5 h-5" />}
<input
className="absolute inset-0 w-full h-full opacity-0 cursor-pointer"
ref={fileInputRef}
disabled={isUploading}
onChange={handleFileInputChange}
type="file"
id="files"
multiple={true}
accept="*"
/>
</Button>
);
});
export default UploadAttachmentButton;
import { Button } from "@usememos/mui";
import { LoaderIcon, PaperclipIcon } from "lucide-react";
import { observer } from "mobx-react-lite";
import { useContext, useRef, useState } from "react";
import toast from "react-hot-toast";
import { attachmentStore } from "@/store/v2";
import { Attachment } from "@/types/proto/api/v1/attachment_service";
import { MemoEditorContext } from "../types";
interface Props {
isUploadingResource?: boolean;
}
interface State {
uploadingFlag: boolean;
}
const UploadResourceButton = observer((props: Props) => {
const context = useContext(MemoEditorContext);
const [state, setState] = useState<State>({
uploadingFlag: false,
});
const fileInputRef = useRef<HTMLInputElement>(null);
const handleFileInputChange = async () => {
if (!fileInputRef.current || !fileInputRef.current.files || fileInputRef.current.files.length === 0) {
return;
}
if (state.uploadingFlag) {
return;
}
setState((state) => {
return {
...state,
uploadingFlag: true,
};
});
const createdAttachmentList: Attachment[] = [];
try {
if (!fileInputRef.current || !fileInputRef.current.files) {
return;
}
for (const file of fileInputRef.current.files) {
const { name: filename, size, type } = file;
const buffer = new Uint8Array(await file.arrayBuffer());
const attachment = await attachmentStore.createAttachment({
attachment: Attachment.fromPartial({
filename,
size,
type,
content: buffer,
}),
attachmentId: "",
});
createdAttachmentList.push(attachment);
}
} catch (error: any) {
console.error(error);
toast.error(error.details);
}
context.setAttachmentList([...context.attachmentList, ...createdAttachmentList]);
setState((state) => {
return {
...state,
uploadingFlag: false,
};
});
};
const isUploading = state.uploadingFlag || props.isUploadingResource;
return (
<Button className="relative p-0" variant="plain" disabled={isUploading}>
{isUploading ? <LoaderIcon className="w-5 h-5 animate-spin" /> : <PaperclipIcon className="w-5 h-5" />}
<input
className="absolute inset-0 w-full h-full opacity-0 cursor-pointer"
ref={fileInputRef}
disabled={isUploading}
onChange={handleFileInputChange}
type="file"
id="files"
multiple={true}
accept="*"
/>
</Button>
);
});
export default UploadResourceButton;
......@@ -2,7 +2,7 @@ import { DndContext, closestCenter, MouseSensor, TouchSensor, useSensor, useSens
import { arrayMove, SortableContext, verticalListSortingStrategy } from "@dnd-kit/sortable";
import { XIcon } from "lucide-react";
import { Attachment } from "@/types/proto/api/v1/attachment_service";
import ResourceIcon from "../ResourceIcon";
import AttachmentIcon from "../AttachmentIcon";
import SortableItem from "./SortableItem";
interface Props {
......@@ -41,7 +41,7 @@ const AttachmentListView = (props: Props) => {
className="max-w-full w-auto flex flex-row justify-start items-center flex-nowrap gap-x-1 bg-zinc-100 dark:bg-zinc-900 px-2 py-1 rounded hover:shadow-sm text-gray-500 dark:text-gray-400"
>
<SortableItem id={attachment.name} className="flex flex-row justify-start items-center gap-x-1">
<ResourceIcon resource={attachment} className="w-4! h-4! opacity-100!" />
<AttachmentIcon attachment={attachment} className="w-4! h-4! opacity-100!" />
<span className="text-sm max-w-32 truncate">{attachment.filename}</span>
</SortableItem>
<button className="shrink-0" onClick={() => handleDeleteAttachment(attachment.name)}>
......
......@@ -25,7 +25,7 @@ import AddMemoRelationPopover from "./ActionButton/AddMemoRelationPopover";
import LocationSelector from "./ActionButton/LocationSelector";
import MarkdownMenu from "./ActionButton/MarkdownMenu";
import TagSelector from "./ActionButton/TagSelector";
import UploadResourceButton from "./ActionButton/UploadResourceButton";
import UploadAttachmentButton from "./ActionButton/UploadAttachmentButton";
import VisibilitySelector from "./ActionButton/VisibilitySelector";
import AttachmentListView from "./AttachmentListView";
import Editor, { EditorRefActions } from "./Editor";
......@@ -51,7 +51,7 @@ interface State {
attachmentList: Attachment[];
relationList: MemoRelation[];
location: Location | undefined;
isUploadingResource: boolean;
isUploadingAttachment: boolean;
isRequesting: boolean;
isComposing: boolean;
isDraggingFile: boolean;
......@@ -67,7 +67,7 @@ const MemoEditor = observer((props: Props) => {
attachmentList: [],
relationList: [],
location: undefined,
isUploadingResource: false,
isUploadingAttachment: false,
isRequesting: false,
isComposing: false,
isDraggingFile: false,
......@@ -203,7 +203,7 @@ const MemoEditor = observer((props: Props) => {
setState((state) => {
return {
...state,
isUploadingResource: true,
isUploadingAttachment: true,
};
});
......@@ -223,7 +223,7 @@ const MemoEditor = observer((props: Props) => {
setState((state) => {
return {
...state,
isUploadingResource: false,
isUploadingAttachment: false,
};
});
return attachment;
......@@ -233,7 +233,7 @@ const MemoEditor = observer((props: Props) => {
setState((state) => {
return {
...state,
isUploadingResource: false,
isUploadingAttachment: false,
};
});
}
......@@ -456,7 +456,7 @@ const MemoEditor = observer((props: Props) => {
[i18n.language],
);
const allowSave = (hasContent || state.attachmentList.length > 0) && !state.isUploadingResource && !state.isRequesting;
const allowSave = (hasContent || state.attachmentList.length > 0) && !state.isUploadingAttachment && !state.isRequesting;
return (
<MemoEditorContext.Provider
......@@ -502,7 +502,7 @@ const MemoEditor = observer((props: Props) => {
<div className="flex flex-row justify-start items-center opacity-80 dark:opacity-60 space-x-2">
<TagSelector editorRef={editorRef} />
<MarkdownMenu editorRef={editorRef} />
<UploadResourceButton isUploadingResource={state.isUploadingResource} />
<UploadAttachmentButton isUploading={state.isUploadingAttachment} />
<AddMemoRelationPopover editorRef={editorRef} />
<LocationSelector
location={state.location}
......
import { Attachment } from "@/types/proto/api/v1/attachment_service";
import { getAttachmentUrl } from "@/utils/attachment";
import ResourceIcon from "./ResourceIcon";
interface Props {
resource: Attachment;
className?: string;
}
const MemoResource: React.FC<Props> = (props: Props) => {
const { className, resource } = props;
const resourceUrl = getAttachmentUrl(resource);
const handlePreviewBtnClick = () => {
window.open(resourceUrl);
};
return (
<div className={`w-auto flex flex-row justify-start items-center text-gray-500 dark:text-gray-400 hover:opacity-80 ${className}`}>
{resource.type.startsWith("audio") ? (
<audio src={resourceUrl} controls></audio>
) : (
<>
<ResourceIcon className="w-4! h-4! mr-1" resource={resource} />
<span className="text-sm max-w-[256px] truncate cursor-pointer" onClick={handlePreviewBtnClick}>
{resource.filename}
</span>
</>
)}
</div>
);
};
export default MemoResource;
......@@ -5,9 +5,9 @@ import { includes } from "lodash-es";
import { PaperclipIcon, SearchIcon, TrashIcon } from "lucide-react";
import { observer } from "mobx-react-lite";
import { useEffect, useState } from "react";
import AttachmentIcon from "@/components/AttachmentIcon";
import Empty from "@/components/Empty";
import MobileHeader from "@/components/MobileHeader";
import ResourceIcon from "@/components/ResourceIcon";
import { attachmentServiceClient } from "@/grpcweb";
import useLoading from "@/hooks/useLoading";
import useResponsiveWidth from "@/hooks/useResponsiveWidth";
......@@ -112,7 +112,7 @@ const Attachments = observer(() => {
return (
<div key={attachment.name} className="w-24 sm:w-32 h-auto flex flex-col justify-start items-start">
<div className="w-24 h-24 flex justify-center items-center sm:w-32 sm:h-32 border border-zinc-200 dark:border-zinc-900 overflow-clip rounded-xl cursor-pointer hover:shadow hover:opacity-80">
<ResourceIcon resource={attachment} strokeWidth={0.5} />
<AttachmentIcon attachment={attachment} strokeWidth={0.5} />
</div>
<div className="w-full max-w-full flex flex-row justify-between items-center mt-1 px-1">
<p className="text-xs shrink text-gray-400 truncate">{attachment.filename}</p>
......@@ -144,7 +144,7 @@ const Attachments = observer(() => {
return (
<div key={attachment.name} className="w-24 sm:w-32 h-auto flex flex-col justify-start items-start">
<div className="w-24 h-24 flex justify-center items-center sm:w-32 sm:h-32 border border-zinc-200 dark:border-zinc-900 overflow-clip rounded-xl cursor-pointer hover:shadow hover:opacity-80">
<ResourceIcon resource={attachment} strokeWidth={0.5} />
<AttachmentIcon attachment={attachment} strokeWidth={0.5} />
</div>
<div className="w-full max-w-full flex flex-row justify-between items-center mt-1 px-1">
<p className="text-xs shrink text-gray-400 truncate">{attachment.filename}</p>
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment