Commit 78aa4133 authored by Johnny's avatar Johnny

feat: implement attachment filtering functionality

parent 955ff0ca
...@@ -99,6 +99,9 @@ var ( ...@@ -99,6 +99,9 @@ var (
defaultOnce sync.Once defaultOnce sync.Once
defaultInst *Engine defaultInst *Engine
defaultErr error defaultErr error
defaultAttachmentOnce sync.Once
defaultAttachmentInst *Engine
defaultAttachmentErr error
) )
// DefaultEngine returns the process-wide memo filter engine. // DefaultEngine returns the process-wide memo filter engine.
...@@ -109,6 +112,14 @@ func DefaultEngine() (*Engine, error) { ...@@ -109,6 +112,14 @@ func DefaultEngine() (*Engine, error) {
return defaultInst, defaultErr return defaultInst, defaultErr
} }
// DefaultAttachmentEngine returns the process-wide attachment filter engine.
func DefaultAttachmentEngine() (*Engine, error) {
defaultAttachmentOnce.Do(func() {
defaultAttachmentInst, defaultAttachmentErr = NewEngine(NewAttachmentSchema())
})
return defaultAttachmentInst, defaultAttachmentErr
}
func normalizeLegacyFilter(expr string) string { func normalizeLegacyFilter(expr string) string {
expr = rewriteNumericLogicalOperand(expr, "&&") expr = rewriteNumericLogicalOperand(expr, "&&")
expr = rewriteNumericLogicalOperand(expr, "||") expr = rewriteNumericLogicalOperand(expr, "||")
......
...@@ -243,6 +243,62 @@ func NewSchema() Schema { ...@@ -243,6 +243,62 @@ func NewSchema() Schema {
} }
} }
// NewAttachmentSchema constructs the attachment filter schema and CEL environment.
func NewAttachmentSchema() Schema {
fields := map[string]Field{
"filename": {
Name: "filename",
Kind: FieldKindScalar,
Type: FieldTypeString,
Column: Column{Table: "resource", Name: "filename"},
SupportsContains: true,
Expressions: map[DialectName]string{},
},
"mime_type": {
Name: "mime_type",
Kind: FieldKindScalar,
Type: FieldTypeString,
Column: Column{Table: "resource", Name: "type"},
Expressions: map[DialectName]string{},
},
"create_time": {
Name: "create_time",
Kind: FieldKindScalar,
Type: FieldTypeTimestamp,
Column: Column{Table: "resource", Name: "created_ts"},
Expressions: map[DialectName]string{
DialectMySQL: "UNIX_TIMESTAMP(%s)",
DialectPostgres: "EXTRACT(EPOCH FROM TO_TIMESTAMP(%s))",
},
},
"memo": {
Name: "memo",
Kind: FieldKindScalar,
Type: FieldTypeString,
Column: Column{Table: "resource", Name: "memo_uid"},
Expressions: map[DialectName]string{},
AllowedComparisonOps: map[ComparisonOperator]bool{
CompareEq: true,
CompareNeq: true,
},
},
}
envOptions := []cel.EnvOption{
cel.Variable("filename", cel.StringType),
cel.Variable("mime_type", cel.StringType),
cel.Variable("create_time", cel.IntType),
cel.Variable("memo", cel.StringType),
nowFunction,
}
return Schema{
Name: "attachment",
Fields: fields,
EnvOptions: envOptions,
}
}
// columnExpr returns the field expression for the given dialect, applying // columnExpr returns the field expression for the given dialect, applying
// any schema-specific overrides (e.g. UNIX timestamp conversions). // any schema-specific overrides (e.g. UNIX timestamp conversions).
func (f Field) columnExpr(d DialectName) string { func (f Field) columnExpr(d DialectName) string {
......
...@@ -101,9 +101,9 @@ message ListAttachmentsRequest { ...@@ -101,9 +101,9 @@ message ListAttachmentsRequest {
string page_token = 2 [(google.api.field_behavior) = OPTIONAL]; string page_token = 2 [(google.api.field_behavior) = OPTIONAL];
// Optional. Filter to apply to the list results. // Optional. Filter to apply to the list results.
// Example: "type=image/png" or "filename:*.jpg" // Example: "mime_type==\"image/png\"" or "filename.contains(\"test\")"
// Supported operators: =, !=, <, <=, >, >=, : // Supported operators: =, !=, <, <=, >, >=, : (contains), in
// Supported fields: filename, type, size, create_time, memo // Supported fields: filename, mime_type, create_time, memo
string filter = 3 [(google.api.field_behavior) = OPTIONAL]; string filter = 3 [(google.api.field_behavior) = OPTIONAL];
// Optional. The order to sort results by. // Optional. The order to sort results by.
......
...@@ -201,9 +201,9 @@ type ListAttachmentsRequest struct { ...@@ -201,9 +201,9 @@ type ListAttachmentsRequest struct {
// Provide this to retrieve the subsequent page. // Provide this to retrieve the subsequent page.
PageToken string `protobuf:"bytes,2,opt,name=page_token,json=pageToken,proto3" json:"page_token,omitempty"` PageToken string `protobuf:"bytes,2,opt,name=page_token,json=pageToken,proto3" json:"page_token,omitempty"`
// Optional. Filter to apply to the list results. // Optional. Filter to apply to the list results.
// Example: "type=image/png" or "filename:*.jpg" // Example: "mime_type==\"image/png\"" or "filename.contains(\"test\")"
// Supported operators: =, !=, <, <=, >, >=, : // Supported operators: =, !=, <, <=, >, >=, : (contains), in
// Supported fields: filename, type, size, create_time, memo // Supported fields: filename, mime_type, create_time, memo
Filter string `protobuf:"bytes,3,opt,name=filter,proto3" json:"filter,omitempty"` Filter string `protobuf:"bytes,3,opt,name=filter,proto3" json:"filter,omitempty"`
// Optional. The order to sort results by. // Optional. The order to sort results by.
// Example: "create_time desc" or "filename asc" // Example: "create_time desc" or "filename asc"
......
...@@ -97,9 +97,9 @@ paths: ...@@ -97,9 +97,9 @@ paths:
in: query in: query
description: |- description: |-
Optional. Filter to apply to the list results. Optional. Filter to apply to the list results.
Example: "type=image/png" or "filename:*.jpg" Example: "mime_type==\"image/png\"" or "filename.contains(\"test\")"
Supported operators: =, !=, <, <=, >, >=, : Supported operators: =, !=, <, <=, >, >=, : (contains), in
Supported fields: filename, type, size, create_time, memo Supported fields: filename, mime_type, create_time, memo
schema: schema:
type: string type: string
- name: orderBy - name: orderBy
......
...@@ -21,6 +21,7 @@ import ( ...@@ -21,6 +21,7 @@ import (
"github.com/usememos/memos/internal/profile" "github.com/usememos/memos/internal/profile"
"github.com/usememos/memos/internal/util" "github.com/usememos/memos/internal/util"
"github.com/usememos/memos/plugin/filter"
"github.com/usememos/memos/plugin/storage/s3" "github.com/usememos/memos/plugin/storage/s3"
v1pb "github.com/usememos/memos/proto/gen/api/v1" v1pb "github.com/usememos/memos/proto/gen/api/v1"
storepb "github.com/usememos/memos/proto/gen/store" storepb "github.com/usememos/memos/proto/gen/store"
...@@ -156,6 +157,14 @@ func (s *APIV1Service) ListAttachments(ctx context.Context, request *v1pb.ListAt ...@@ -156,6 +157,14 @@ func (s *APIV1Service) ListAttachments(ctx context.Context, request *v1pb.ListAt
Offset: &offset, Offset: &offset,
} }
// Parse filter if provided
if request.Filter != "" {
if err := s.validateAttachmentFilter(ctx, request.Filter); err != nil {
return nil, status.Errorf(codes.InvalidArgument, "invalid filter: %v", err)
}
findAttachment.Filters = append(findAttachment.Filters, request.Filter)
}
attachments, err := s.Store.ListAttachments(ctx, findAttachment) attachments, err := s.Store.ListAttachments(ctx, findAttachment)
if err != nil { if err != nil {
return nil, status.Errorf(codes.Internal, "failed to list attachments: %v", err) return nil, status.Errorf(codes.Internal, "failed to list attachments: %v", err)
...@@ -472,3 +481,29 @@ func isValidMimeType(mimeType string) bool { ...@@ -472,3 +481,29 @@ func isValidMimeType(mimeType string) bool {
matched, _ := regexp.MatchString(`^[a-zA-Z0-9][a-zA-Z0-9!#$&^_.+-]{0,126}/[a-zA-Z0-9][a-zA-Z0-9!#$&^_.+-]{0,126}$`, mimeType) matched, _ := regexp.MatchString(`^[a-zA-Z0-9][a-zA-Z0-9!#$&^_.+-]{0,126}/[a-zA-Z0-9][a-zA-Z0-9!#$&^_.+-]{0,126}$`, mimeType)
return matched return matched
} }
func (s *APIV1Service) validateAttachmentFilter(ctx context.Context, filterStr string) error {
if filterStr == "" {
return errors.New("filter cannot be empty")
}
engine, err := filter.DefaultAttachmentEngine()
if err != nil {
return err
}
var dialect filter.DialectName
switch s.Profile.Driver {
case "mysql":
dialect = filter.DialectMySQL
case "postgres":
dialect = filter.DialectPostgres
default:
dialect = filter.DialectSQLite
}
if _, err := engine.CompileToStatement(ctx, filterStr, filter.RenderOptions{Dialect: dialect}); err != nil {
return errors.Wrap(err, "failed to compile filter")
}
return nil
}
...@@ -51,6 +51,7 @@ type FindAttachment struct { ...@@ -51,6 +51,7 @@ type FindAttachment struct {
MemoIDList []int32 MemoIDList []int32
HasRelatedMemo bool HasRelatedMemo bool
StorageType *storepb.AttachmentStorageType StorageType *storepb.AttachmentStorageType
Filters []string
Limit *int Limit *int
Offset *int Offset *int
} }
......
...@@ -9,6 +9,7 @@ import ( ...@@ -9,6 +9,7 @@ import (
"github.com/pkg/errors" "github.com/pkg/errors"
"google.golang.org/protobuf/encoding/protojson" "google.golang.org/protobuf/encoding/protojson"
"github.com/usememos/memos/plugin/filter"
storepb "github.com/usememos/memos/proto/gen/store" storepb "github.com/usememos/memos/proto/gen/store"
"github.com/usememos/memos/store" "github.com/usememos/memos/store"
) )
...@@ -83,6 +84,16 @@ func (d *DB) ListAttachments(ctx context.Context, find *store.FindAttachment) ([ ...@@ -83,6 +84,16 @@ func (d *DB) ListAttachments(ctx context.Context, find *store.FindAttachment) ([
where, args = append(where, "`resource`.`storage_type` = ?"), append(args, find.StorageType.String()) where, args = append(where, "`resource`.`storage_type` = ?"), append(args, find.StorageType.String())
} }
if len(find.Filters) > 0 {
engine, err := filter.DefaultAttachmentEngine()
if err != nil {
return nil, errors.Wrap(err, "failed to get filter engine")
}
if err := filter.AppendConditions(ctx, engine, find.Filters, filter.DialectMySQL, &where, &args); err != nil {
return nil, errors.Wrap(err, "failed to append filter conditions")
}
}
fields := []string{ fields := []string{
"`resource`.`id` AS `id`", "`resource`.`id` AS `id`",
"`resource`.`uid` AS `uid`", "`resource`.`uid` AS `uid`",
......
...@@ -9,6 +9,7 @@ import ( ...@@ -9,6 +9,7 @@ import (
"github.com/pkg/errors" "github.com/pkg/errors"
"google.golang.org/protobuf/encoding/protojson" "google.golang.org/protobuf/encoding/protojson"
"github.com/usememos/memos/plugin/filter"
storepb "github.com/usememos/memos/proto/gen/store" storepb "github.com/usememos/memos/proto/gen/store"
"github.com/usememos/memos/store" "github.com/usememos/memos/store"
) )
...@@ -72,6 +73,16 @@ func (d *DB) ListAttachments(ctx context.Context, find *store.FindAttachment) ([ ...@@ -72,6 +73,16 @@ func (d *DB) ListAttachments(ctx context.Context, find *store.FindAttachment) ([
where, args = append(where, "resource.storage_type = "+placeholder(len(args)+1)), append(args, v.String()) where, args = append(where, "resource.storage_type = "+placeholder(len(args)+1)), append(args, v.String())
} }
if len(find.Filters) > 0 {
engine, err := filter.DefaultAttachmentEngine()
if err != nil {
return nil, errors.Wrap(err, "failed to get filter engine")
}
if err := filter.AppendConditions(ctx, engine, find.Filters, filter.DialectPostgres, &where, &args); err != nil {
return nil, errors.Wrap(err, "failed to append filter conditions")
}
}
fields := []string{ fields := []string{
"resource.id AS id", "resource.id AS id",
"resource.uid AS uid", "resource.uid AS uid",
......
...@@ -9,6 +9,7 @@ import ( ...@@ -9,6 +9,7 @@ import (
"github.com/pkg/errors" "github.com/pkg/errors"
"google.golang.org/protobuf/encoding/protojson" "google.golang.org/protobuf/encoding/protojson"
"github.com/usememos/memos/plugin/filter"
storepb "github.com/usememos/memos/proto/gen/store" storepb "github.com/usememos/memos/proto/gen/store"
"github.com/usememos/memos/store" "github.com/usememos/memos/store"
) )
...@@ -76,6 +77,16 @@ func (d *DB) ListAttachments(ctx context.Context, find *store.FindAttachment) ([ ...@@ -76,6 +77,16 @@ func (d *DB) ListAttachments(ctx context.Context, find *store.FindAttachment) ([
where, args = append(where, "`resource`.`storage_type` = ?"), append(args, find.StorageType.String()) where, args = append(where, "`resource`.`storage_type` = ?"), append(args, find.StorageType.String())
} }
if len(find.Filters) > 0 {
engine, err := filter.DefaultAttachmentEngine()
if err != nil {
return nil, errors.Wrap(err, "failed to get filter engine")
}
if err := filter.AppendConditions(ctx, engine, find.Filters, filter.DialectSQLite, &where, &args); err != nil {
return nil, errors.Wrap(err, "failed to append filter conditions")
}
}
fields := []string{ fields := []string{
"`resource`.`id` AS `id`", "`resource`.`id` AS `id`",
"`resource`.`uid` AS `uid`", "`resource`.`uid` AS `uid`",
......
...@@ -61,3 +61,62 @@ func TestAttachmentStore(t *testing.T) { ...@@ -61,3 +61,62 @@ func TestAttachmentStore(t *testing.T) {
require.ErrorContains(t, err, "attachment not found") require.ErrorContains(t, err, "attachment not found")
ts.Close() ts.Close()
} }
func TestAttachmentStoreWithFilter(t *testing.T) {
ctx := context.Background()
ts := NewTestingStore(ctx, t)
_, err := ts.CreateAttachment(ctx, &store.Attachment{
UID: shortuuid.New(),
CreatorID: 101,
Filename: "test.png",
Blob: []byte("test"),
Type: "image/png",
Size: 1000,
})
require.NoError(t, err)
_, err = ts.CreateAttachment(ctx, &store.Attachment{
UID: shortuuid.New(),
CreatorID: 101,
Filename: "test.jpg",
Blob: []byte("test"),
Type: "image/jpeg",
Size: 2000,
})
require.NoError(t, err)
_, err = ts.CreateAttachment(ctx, &store.Attachment{
UID: shortuuid.New(),
CreatorID: 101,
Filename: "test.pdf",
Blob: []byte("test"),
Type: "application/pdf",
Size: 3000,
})
require.NoError(t, err)
attachments, err := ts.ListAttachments(ctx, &store.FindAttachment{
CreatorID: &[]int32{101}[0],
Filters: []string{`mime_type == "image/png"`},
})
require.NoError(t, err)
require.Len(t, attachments, 1)
require.Equal(t, "image/png", attachments[0].Type)
attachments, err = ts.ListAttachments(ctx, &store.FindAttachment{
CreatorID: &[]int32{101}[0],
Filters: []string{`mime_type in ["image/png", "image/jpeg"]`},
})
require.NoError(t, err)
require.Len(t, attachments, 2)
attachments, err = ts.ListAttachments(ctx, &store.FindAttachment{
CreatorID: &[]int32{101}[0],
Filters: []string{`filename.contains("test")`},
})
require.NoError(t, err)
require.Len(t, attachments, 3)
ts.Close()
}
...@@ -139,9 +139,9 @@ export type ListAttachmentsRequest = Message<"memos.api.v1.ListAttachmentsReques ...@@ -139,9 +139,9 @@ export type ListAttachmentsRequest = Message<"memos.api.v1.ListAttachmentsReques
/** /**
* Optional. Filter to apply to the list results. * Optional. Filter to apply to the list results.
* Example: "type=image/png" or "filename:*.jpg" * Example: "mime_type==\"image/png\"" or "filename.contains(\"test\")"
* Supported operators: =, !=, <, <=, >, >=, : * Supported operators: =, !=, <, <=, >, >=, : (contains), in
* Supported fields: filename, type, size, create_time, memo * Supported fields: filename, mime_type, create_time, memo
* *
* @generated from field: string filter = 3; * @generated from field: string filter = 3;
*/ */
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment