Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
C
canifa_note
Project
Project
Details
Activity
Releases
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Vũ Hoàng Anh
canifa_note
Commits
32096309
Commit
32096309
authored
Apr 28, 2024
by
Steven
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
refactor: update storage setting
parent
f25c7d9b
Changes
14
Expand all
Hide whitespace changes
Inline
Side-by-side
Showing
14 changed files
with
599 additions
and
852 deletions
+599
-852
apidocs.swagger.yaml
docs/apidocs.swagger.yaml
+21
-8
s3.go
plugin/storage/s3/s3.go
+10
-87
workspace_setting_service.proto
proto/api/v1/workspace_setting_service.proto
+19
-12
workspace_setting_service.pb.go
proto/gen/api/v1/workspace_setting_service.pb.go
+216
-115
workspace_setting.pb.go
proto/gen/store/workspace_setting.pb.go
+181
-81
workspace_setting.proto
proto/store/workspace_setting.proto
+19
-12
resource_service.go
server/route/api/v1/resource_service.go
+17
-34
workspace_setting_service.go
server/route/api/v1/workspace_setting_service.go
+30
-12
server.go
server/server.go
+0
-2
resource_presign.go
server/service/resource_presign/resource_presign.go
+0
-110
workspace_setting.go
store/workspace_setting.go
+5
-5
CreateStorageServiceDialog.tsx
web/src/components/CreateStorageServiceDialog.tsx
+0
-257
StorageSection.tsx
web/src/components/Settings/StorageSection.tsx
+81
-114
grpcweb.ts
web/src/grpcweb.ts
+0
-3
No files found.
docs/apidocs.swagger.yaml
View file @
32096309
...
...
@@ -2059,31 +2059,44 @@ definitions:
storageType
:
$ref
:
'
#/definitions/apiv1WorkspaceStorageSettingStorageType'
description
:
storage_type is the storage type.
activedExternalStorageId
:
type
:
integer
format
:
int32
description
:
The id of actived external storage.
localStoragePathTemplate
:
filepathTemplate
:
type
:
string
title
:
|-
The template of
local storag
e path.
The template of
fil
e path.
e.g. assets/{timestamp}_{filename}
uploadSizeLimitMb
:
type
:
string
format
:
int64
description
:
The max upload size in megabytes.
s3Config
:
$ref
:
'
#/definitions/apiv1WorkspaceStorageSettingS3Config'
description
:
The S3 config.
apiv1WorkspaceStorageSettingS3Config
:
type
:
object
properties
:
accessKeyId
:
type
:
string
accessKeySecret
:
type
:
string
endpoint
:
type
:
string
region
:
type
:
string
bucket
:
type
:
string
title
:
'
Reference:
https://developers.cloudflare.com/r2/examples/aws/aws-sdk-go/'
apiv1WorkspaceStorageSettingStorageType
:
type
:
string
enum
:
-
STORAGE_TYPE_UNSPECIFIED
-
STORAGE_TYPE_DATABASE
-
STORAGE_TYPE_LOCAL
-
STORAGE_TYPE_
EXTERNAL
-
STORAGE_TYPE_
S3
default
:
STORAGE_TYPE_UNSPECIFIED
description
:
|2-
- STORAGE_TYPE_DATABASE: STORAGE_TYPE_DATABASE is the database storage type.
- STORAGE_TYPE_LOCAL: STORAGE_TYPE_LOCAL is the local storage type.
- STORAGE_TYPE_
EXTERNAL: STORAGE_TYPE_EXTERNAL is the external
storage type.
- STORAGE_TYPE_
S3: STORAGE_TYPE_S3 is the S3
storage type.
googlerpcStatus
:
type
:
object
properties
:
...
...
plugin/storage/s3/s3.go
View file @
32096309
...
...
@@ -2,10 +2,7 @@ package s3
import
(
"context"
"fmt"
"io"
"net/url"
"strings"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
...
...
@@ -13,21 +10,17 @@ import (
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
awss3
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/pkg/errors"
)
const
LinkLifetime
=
24
*
time
.
Hour
type
Config
struct
{
AccessKey
string
SecretKey
string
Bucket
string
EndPoint
string
Region
string
URLPrefix
string
URLSuffix
string
PreSign
bool
AccessKeyID
string
AcesssKeySecret
string
Endpoint
string
Region
string
Bucket
string
}
type
Client
struct
{
...
...
@@ -36,32 +29,21 @@ type Client struct {
}
func
NewClient
(
ctx
context
.
Context
,
config
*
Config
)
(
*
Client
,
error
)
{
// For some s3-compatible object stores, converting the hostname is not required,
// and not setting this option will result in not being able to access the corresponding object store address.
// But Aliyun OSS should disable this option
hostnameImmutable
:=
true
if
strings
.
HasSuffix
(
config
.
EndPoint
,
"aliyuncs.com"
)
{
hostnameImmutable
=
false
}
resolver
:=
aws
.
EndpointResolverWithOptionsFunc
(
func
(
service
,
region
string
,
options
...
any
)
(
aws
.
Endpoint
,
error
)
{
return
aws
.
Endpoint
{
URL
:
config
.
EndPoint
,
SigningRegion
:
config
.
Region
,
HostnameImmutable
:
hostnameImmutable
,
URL
:
config
.
Endpoint
,
},
nil
})
awsConfig
,
err
:=
s3config
.
LoadDefaultConfig
(
ctx
,
s3Config
,
err
:=
s3config
.
LoadDefaultConfig
(
ctx
,
s3config
.
WithEndpointResolverWithOptions
(
resolver
),
s3config
.
WithCredentialsProvider
(
credentials
.
NewStaticCredentialsProvider
(
config
.
AccessKey
,
config
.
SecretKey
,
""
)),
s3config
.
WithCredentialsProvider
(
credentials
.
NewStaticCredentialsProvider
(
config
.
AccessKey
ID
,
config
.
AcesssKeySecret
,
""
)),
s3config
.
WithRegion
(
config
.
Region
),
)
if
err
!=
nil
{
return
nil
,
err
return
nil
,
err
ors
.
Wrap
(
err
,
"failed to load s3 config"
)
}
client
:=
awss3
.
NewFromConfig
(
awsConfig
)
client
:=
awss3
.
NewFromConfig
(
s3Config
)
return
&
Client
{
Client
:
client
,
Config
:
config
,
...
...
@@ -76,73 +58,14 @@ func (client *Client) UploadFile(ctx context.Context, filename string, fileType
Body
:
src
,
ContentType
:
aws
.
String
(
fileType
),
}
// Set ACL according to if url prefix is set.
if
client
.
Config
.
URLPrefix
==
""
&&
!
client
.
Config
.
PreSign
{
putInput
.
ACL
=
types
.
ObjectCannedACL
(
*
aws
.
String
(
"public-read"
))
}
uploadOutput
,
err
:=
uploader
.
Upload
(
ctx
,
&
putInput
)
if
err
!=
nil
{
return
""
,
err
}
link
:=
uploadOutput
.
Location
// If url prefix is set, use it as the file link.
if
client
.
Config
.
URLPrefix
!=
""
{
parts
:=
strings
.
Split
(
filename
,
"/"
)
for
i
:=
range
parts
{
parts
[
i
]
=
url
.
PathEscape
(
parts
[
i
])
}
link
=
fmt
.
Sprintf
(
"%s/%s%s"
,
client
.
Config
.
URLPrefix
,
strings
.
Join
(
parts
,
"/"
),
client
.
Config
.
URLSuffix
)
}
if
link
==
""
{
return
""
,
errors
.
New
(
"failed to get file link"
)
}
if
client
.
Config
.
PreSign
{
return
client
.
PreSignLink
(
ctx
,
link
)
}
return
link
,
nil
}
// PreSignLink generates a pre-signed URL for the given sourceLink.
// If the link does not belong to the configured storage endpoint, it is returned as-is.
// If the link belongs to the storage, the function generates a pre-signed URL using the AWS S3 client.
func
(
client
*
Client
)
PreSignLink
(
ctx
context
.
Context
,
sourceLink
string
)
(
string
,
error
)
{
u
,
err
:=
url
.
Parse
(
sourceLink
)
if
err
!=
nil
{
return
""
,
errors
.
Wrapf
(
err
,
"parse URL"
)
}
// if link doesn't belong to storage, then return as-is.
// the empty hostname is corner-case for AWS native endpoint.
endpointURL
,
err
:=
url
.
Parse
(
client
.
Config
.
EndPoint
)
if
err
!=
nil
{
return
""
,
errors
.
Wrapf
(
err
,
"parse Endpoint URL"
)
}
endpointHost
:=
endpointURL
.
Hostname
()
if
client
.
Config
.
Bucket
!=
""
&&
!
strings
.
Contains
(
endpointHost
,
client
.
Config
.
Bucket
)
{
endpointHost
=
fmt
.
Sprintf
(
"%s.%s"
,
client
.
Config
.
Bucket
,
endpointHost
)
}
if
client
.
Config
.
EndPoint
!=
""
&&
!
strings
.
Contains
(
endpointHost
,
u
.
Hostname
())
{
return
sourceLink
,
nil
}
filename
:=
u
.
Path
if
prefixLen
:=
len
(
client
.
Config
.
URLPrefix
);
len
(
filename
)
>=
prefixLen
{
filename
=
filename
[
prefixLen
:
]
}
if
suffixLen
:=
len
(
client
.
Config
.
URLSuffix
);
len
(
filename
)
>=
suffixLen
{
filename
=
filename
[
:
len
(
filename
)
-
suffixLen
]
}
filename
=
strings
.
Trim
(
filename
,
"/"
)
if
strings
.
HasPrefix
(
filename
,
client
.
Config
.
Bucket
)
{
filename
=
strings
.
Trim
(
filename
[
len
(
client
.
Config
.
Bucket
)
:
],
"/"
)
}
req
,
err
:=
awss3
.
NewPresignClient
(
client
.
Client
)
.
PresignGetObject
(
ctx
,
&
awss3
.
GetObjectInput
{
Bucket
:
aws
.
String
(
client
.
Config
.
Bucket
),
Key
:
aws
.
String
(
filename
),
},
awss3
.
WithPresignExpires
(
LinkLifetime
))
if
err
!=
nil
{
return
""
,
errors
.
Wrapf
(
err
,
"pre-sign link"
)
}
return
req
.
URL
,
nil
}
proto/api/v1/workspace_setting_service.proto
View file @
32096309
...
...
@@ -63,25 +63,32 @@ message WorkspaceCustomProfile {
}
message
WorkspaceStorageSetting
{
// storage_type is the storage type.
StorageType
storage_type
=
1
;
// The id of actived external storage.
optional
int32
actived_external_storage_id
=
2
;
// The template of local storage path.
// e.g. assets/{timestamp}_{filename}
string
local_storage_path_template
=
3
;
// The max upload size in megabytes.
int64
upload_size_limit_mb
=
4
;
enum
StorageType
{
STORAGE_TYPE_UNSPECIFIED
=
0
;
// STORAGE_TYPE_DATABASE is the database storage type.
STORAGE_TYPE_DATABASE
=
1
;
// STORAGE_TYPE_LOCAL is the local storage type.
STORAGE_TYPE_LOCAL
=
2
;
// STORAGE_TYPE_EXTERNAL is the external storage type.
STORAGE_TYPE_EXTERNAL
=
3
;
// STORAGE_TYPE_S3 is the S3 storage type.
STORAGE_TYPE_S3
=
3
;
}
// storage_type is the storage type.
StorageType
storage_type
=
1
;
// The template of file path.
// e.g. assets/{timestamp}_{filename}
string
filepath_template
=
2
;
// The max upload size in megabytes.
int64
upload_size_limit_mb
=
3
;
// Reference: https://developers.cloudflare.com/r2/examples/aws/aws-sdk-go/
message
S3Config
{
string
access_key_id
=
1
;
string
access_key_secret
=
2
;
string
endpoint
=
3
;
string
region
=
4
;
string
bucket
=
5
;
}
// The S3 config.
S3Config
s3_config
=
4
;
}
message
WorkspaceMemoRelatedSetting
{
...
...
proto/gen/api/v1/workspace_setting_service.pb.go
View file @
32096309
This diff is collapsed.
Click to expand it.
proto/gen/store/workspace_setting.pb.go
View file @
32096309
This diff is collapsed.
Click to expand it.
proto/store/workspace_setting.proto
View file @
32096309
...
...
@@ -55,25 +55,32 @@ message WorkspaceCustomProfile {
}
message
WorkspaceStorageSetting
{
// storage_type is the storage type.
StorageType
storage_type
=
1
;
// The id of actived external storage.
optional
int32
actived_external_storage_id
=
2
;
// The template of local storage path.
// e.g. assets/{timestamp}_{filename}
string
local_storage_path_template
=
3
;
// The max upload size in megabytes.
int64
upload_size_limit_mb
=
4
;
enum
StorageType
{
STORAGE_TYPE_UNSPECIFIED
=
0
;
// STORAGE_TYPE_DATABASE is the database storage type.
STORAGE_TYPE_DATABASE
=
1
;
// STORAGE_TYPE_LOCAL is the local storage type.
STORAGE_TYPE_LOCAL
=
2
;
// STORAGE_TYPE_EXTERNAL is the external storage type.
STORAGE_TYPE_EXTERNAL
=
3
;
// STORAGE_TYPE_S3 is the S3 storage type.
STORAGE_TYPE_S3
=
3
;
}
// storage_type is the storage type.
StorageType
storage_type
=
1
;
// The template of file path.
// e.g. assets/{timestamp}_{filename}
string
filepath_template
=
2
;
// The max upload size in megabytes.
int64
upload_size_limit_mb
=
3
;
// Reference: https://developers.cloudflare.com/r2/examples/aws/aws-sdk-go/
message
S3Config
{
string
access_key_id
=
1
;
string
access_key_secret
=
2
;
string
endpoint
=
3
;
string
region
=
4
;
string
bucket
=
5
;
}
// The S3 config.
S3Config
s3_config
=
4
;
}
message
WorkspaceMemoRelatedSetting
{
...
...
server/route/api/v1/resource_service.go
View file @
32096309
...
...
@@ -254,12 +254,12 @@ func SaveResourceBlob(ctx context.Context, s *store.Store, create *store.Resourc
}
if
workspaceStorageSetting
.
StorageType
==
storepb
.
WorkspaceStorageSetting_STORAGE_TYPE_LOCAL
{
localStoragePath
:=
"assets/{timestamp}_{filename}"
if
workspaceStorageSetting
.
LocalStorageP
athTemplate
!=
""
{
localStoragePath
=
workspaceStorageSetting
.
LocalStorageP
athTemplate
filepathTemplate
:=
"assets/{timestamp}_{filename}"
if
workspaceStorageSetting
.
Filep
athTemplate
!=
""
{
filepathTemplate
=
workspaceStorageSetting
.
Filep
athTemplate
}
internalPath
:=
localStoragePath
internalPath
:=
filepathTemplate
if
!
strings
.
Contains
(
internalPath
,
"{filename}"
)
{
internalPath
=
filepath
.
Join
(
internalPath
,
"{filename}"
)
}
...
...
@@ -287,46 +287,29 @@ func SaveResourceBlob(ctx context.Context, s *store.Store, create *store.Resourc
}
create
.
InternalPath
=
internalPath
create
.
Blob
=
nil
}
else
if
workspaceStorageSetting
.
StorageType
==
storepb
.
WorkspaceStorageSetting_STORAGE_TYPE_EXTERNAL
{
if
workspaceStorageSetting
.
ActivedExternalStorageId
==
nil
{
return
errors
.
Errorf
(
"No actived external storage found"
)
}
storage
,
err
:=
s
.
GetStorage
(
ctx
,
&
store
.
FindStorage
{
ID
:
workspaceStorageSetting
.
ActivedExternalStorageId
})
if
err
!=
nil
{
return
errors
.
Wrap
(
err
,
"Failed to find actived external storage"
)
}
if
storage
==
nil
{
return
errors
.
Errorf
(
"Storage %d not found"
,
*
workspaceStorageSetting
.
ActivedExternalStorageId
)
}
if
storage
.
Type
!=
storepb
.
Storage_S3
{
return
errors
.
Errorf
(
"Unsupported storage type: %s"
,
storage
.
Type
.
String
())
}
s3Config
:=
storage
.
Config
.
GetS3Config
()
}
else
if
workspaceStorageSetting
.
StorageType
==
storepb
.
WorkspaceStorageSetting_STORAGE_TYPE_S3
{
s3Config
:=
workspaceStorageSetting
.
S3Config
if
s3Config
==
nil
{
return
errors
.
Errorf
(
"
S3 config not
found"
)
return
errors
.
Errorf
(
"
No actived external storage
found"
)
}
s3Client
,
err
:=
s3
.
NewClient
(
ctx
,
&
s3
.
Config
{
AccessKey
:
s3Config
.
AccessKey
,
SecretKey
:
s3Config
.
SecretKey
,
EndPoint
:
s3Config
.
EndPoint
,
Region
:
s3Config
.
Region
,
Bucket
:
s3Config
.
Bucket
,
URLPrefix
:
s3Config
.
UrlPrefix
,
URLSuffix
:
s3Config
.
UrlSuffix
,
PreSign
:
s3Config
.
PreSign
,
AccessKeyID
:
s3Config
.
AccessKeyId
,
AcesssKeySecret
:
s3Config
.
AccessKeySecret
,
Endpoint
:
s3Config
.
Endpoint
,
Region
:
s3Config
.
Region
,
Bucket
:
s3Config
.
Bucket
,
})
if
err
!=
nil
{
return
errors
.
Wrap
(
err
,
"Failed to create s3 client"
)
}
file
Path
:=
s3Config
.
Path
if
!
strings
.
Contains
(
file
Path
,
"{filename}"
)
{
file
Path
=
filepath
.
Join
(
filePath
,
"{filename}"
)
file
pathTemplate
:=
workspaceStorageSetting
.
FilepathTemplate
if
!
strings
.
Contains
(
file
pathTemplate
,
"{filename}"
)
{
file
pathTemplate
=
filepath
.
Join
(
filepathTemplate
,
"{filename}"
)
}
file
Path
=
replacePathTemplate
(
filePath
,
create
.
Filename
)
file
pathTemplate
=
replacePathTemplate
(
filepathTemplate
,
create
.
Filename
)
r
:=
bytes
.
NewReader
(
create
.
Blob
)
link
,
err
:=
s3Client
.
UploadFile
(
ctx
,
file
Path
,
create
.
Type
,
r
)
link
,
err
:=
s3Client
.
UploadFile
(
ctx
,
file
pathTemplate
,
create
.
Type
,
r
)
if
err
!=
nil
{
return
errors
.
Wrap
(
err
,
"Failed to upload via s3 client"
)
}
...
...
server/route/api/v1/workspace_setting_service.go
View file @
32096309
...
...
@@ -162,28 +162,46 @@ func convertWorkspaceGeneralSettingToStore(setting *v1pb.WorkspaceGeneralSetting
return
generalSetting
}
func
convertWorkspaceStorageSettingFromStore
(
setting
*
storepb
.
WorkspaceStorageSetting
)
*
v1pb
.
WorkspaceStorageSetting
{
if
setting
==
nil
{
func
convertWorkspaceStorageSettingFromStore
(
setting
pb
*
storepb
.
WorkspaceStorageSetting
)
*
v1pb
.
WorkspaceStorageSetting
{
if
setting
pb
==
nil
{
return
nil
}
return
&
v1pb
.
WorkspaceStorageSetting
{
StorageType
:
v1pb
.
WorkspaceStorageSetting_StorageType
(
setting
.
StorageType
),
LocalStoragePathTemplate
:
setting
.
LocalStoragePathTemplate
,
UploadSizeLimitMb
:
setting
.
UploadSizeLimitMb
,
ActivedExternalStorageId
:
setting
.
ActivedExternalStorageId
,
setting
:=
&
v1pb
.
WorkspaceStorageSetting
{
StorageType
:
v1pb
.
WorkspaceStorageSetting_StorageType
(
settingpb
.
StorageType
),
FilepathTemplate
:
settingpb
.
FilepathTemplate
,
UploadSizeLimitMb
:
settingpb
.
UploadSizeLimitMb
,
}
if
settingpb
.
S3Config
!=
nil
{
setting
.
S3Config
=
&
v1pb
.
WorkspaceStorageSetting_S3Config
{
AccessKeyId
:
settingpb
.
S3Config
.
AccessKeyId
,
AccessKeySecret
:
settingpb
.
S3Config
.
AccessKeySecret
,
Endpoint
:
settingpb
.
S3Config
.
Endpoint
,
Region
:
settingpb
.
S3Config
.
Region
,
Bucket
:
settingpb
.
S3Config
.
Bucket
,
}
}
return
setting
}
func
convertWorkspaceStorageSettingToStore
(
setting
*
v1pb
.
WorkspaceStorageSetting
)
*
storepb
.
WorkspaceStorageSetting
{
if
setting
==
nil
{
return
nil
}
return
&
storepb
.
WorkspaceStorageSetting
{
StorageType
:
storepb
.
WorkspaceStorageSetting_StorageType
(
setting
.
StorageType
),
LocalStoragePathTemplate
:
setting
.
LocalStoragePathTemplate
,
UploadSizeLimitMb
:
setting
.
UploadSizeLimitMb
,
ActivedExternalStorageId
:
setting
.
ActivedExternalStorageId
,
settingpb
:=
&
storepb
.
WorkspaceStorageSetting
{
StorageType
:
storepb
.
WorkspaceStorageSetting_StorageType
(
setting
.
StorageType
),
FilepathTemplate
:
setting
.
FilepathTemplate
,
UploadSizeLimitMb
:
setting
.
UploadSizeLimitMb
,
}
if
setting
.
S3Config
!=
nil
{
settingpb
.
S3Config
=
&
storepb
.
WorkspaceStorageSetting_S3Config
{
AccessKeyId
:
setting
.
S3Config
.
AccessKeyId
,
AccessKeySecret
:
setting
.
S3Config
.
AccessKeySecret
,
Endpoint
:
setting
.
S3Config
.
Endpoint
,
Region
:
setting
.
S3Config
.
Region
,
Bucket
:
setting
.
S3Config
.
Bucket
,
}
}
return
settingpb
}
func
convertWorkspaceMemoRelatedSettingFromStore
(
setting
*
storepb
.
WorkspaceMemoRelatedSetting
)
*
v1pb
.
WorkspaceMemoRelatedSetting
{
...
...
server/server.go
View file @
32096309
...
...
@@ -22,7 +22,6 @@ import (
"github.com/usememos/memos/server/route/frontend"
"github.com/usememos/memos/server/route/resource"
"github.com/usememos/memos/server/route/rss"
resourcepresign
"github.com/usememos/memos/server/service/resource_presign"
versionchecker
"github.com/usememos/memos/server/service/version_checker"
"github.com/usememos/memos/store"
)
...
...
@@ -148,7 +147,6 @@ func (s *Server) Shutdown(ctx context.Context) {
}
func
(
s
*
Server
)
StartBackgroundRunners
(
ctx
context
.
Context
)
{
go
resourcepresign
.
RunPreSignLinks
(
ctx
,
s
.
Store
)
go
versionchecker
.
NewVersionChecker
(
s
.
Store
,
s
.
Profile
)
.
Start
(
ctx
)
}
...
...
server/service/resource_presign/resource_presign.go
deleted
100644 → 0
View file @
f25c7d9b
package
resourcepresign
import
(
"context"
"log/slog"
"strings"
"time"
"github.com/pkg/errors"
"github.com/usememos/memos/plugin/storage/s3"
storepb
"github.com/usememos/memos/proto/gen/store"
"github.com/usememos/memos/store"
)
// RunPreSignLinks is a background runner that pre-signs external links stored in the database.
// It uses S3 client to generate presigned URLs and updates the corresponding resources in the store.
func
RunPreSignLinks
(
ctx
context
.
Context
,
dataStore
*
store
.
Store
)
{
for
{
if
err
:=
signExternalLinks
(
ctx
,
dataStore
);
err
!=
nil
{
slog
.
Error
(
"failed to pre-sign links"
,
err
)
}
else
{
slog
.
Debug
(
"pre-signed links"
)
}
select
{
case
<-
time
.
After
(
s3
.
LinkLifetime
/
2
)
:
case
<-
ctx
.
Done
()
:
return
}
}
}
func
signExternalLinks
(
ctx
context
.
Context
,
dataStore
*
store
.
Store
)
error
{
objectStore
,
err
:=
findObjectStorage
(
ctx
,
dataStore
)
if
err
!=
nil
{
return
errors
.
Wrapf
(
err
,
"find object storage"
)
}
if
objectStore
==
nil
||
!
objectStore
.
Config
.
PreSign
{
// object storage not set or not supported
return
nil
}
resources
,
err
:=
dataStore
.
ListResources
(
ctx
,
&
store
.
FindResource
{
GetBlob
:
false
,
})
if
err
!=
nil
{
return
errors
.
Wrapf
(
err
,
"list resources"
)
}
for
_
,
resource
:=
range
resources
{
if
resource
.
ExternalLink
==
""
{
// not for object store
continue
}
if
strings
.
Contains
(
resource
.
ExternalLink
,
"?"
)
&&
time
.
Since
(
time
.
Unix
(
resource
.
UpdatedTs
,
0
))
<
s3
.
LinkLifetime
/
2
{
// resource not signed (hack for migration)
// resource was recently updated - skipping
continue
}
newLink
,
err
:=
objectStore
.
PreSignLink
(
ctx
,
resource
.
ExternalLink
)
if
err
!=
nil
{
slog
.
Error
(
"failed to pre-sign link"
,
err
)
continue
}
now
:=
time
.
Now
()
.
Unix
()
if
_
,
err
:=
dataStore
.
UpdateResource
(
ctx
,
&
store
.
UpdateResource
{
ID
:
resource
.
ID
,
UpdatedTs
:
&
now
,
ExternalLink
:
&
newLink
,
});
err
!=
nil
{
return
errors
.
Wrapf
(
err
,
"update resource %d link to %q"
,
resource
.
ID
,
newLink
)
}
}
return
nil
}
// findObjectStorage returns current default storage if it's S3-compatible or nil otherwise.
// Returns error only in case of internal problems (ie: database or configuration issues).
// May return nil client and nil error.
func
findObjectStorage
(
ctx
context
.
Context
,
dataStore
*
store
.
Store
)
(
*
s3
.
Client
,
error
)
{
workspaceStorageSetting
,
err
:=
dataStore
.
GetWorkspaceStorageSetting
(
ctx
)
if
err
!=
nil
{
return
nil
,
errors
.
Wrap
(
err
,
"Failed to find workspaceStorageSetting"
)
}
if
workspaceStorageSetting
.
StorageType
!=
storepb
.
WorkspaceStorageSetting_STORAGE_TYPE_EXTERNAL
||
workspaceStorageSetting
.
ActivedExternalStorageId
==
nil
{
return
nil
,
nil
}
storage
,
err
:=
dataStore
.
GetStorage
(
ctx
,
&
store
.
FindStorage
{
ID
:
workspaceStorageSetting
.
ActivedExternalStorageId
})
if
err
!=
nil
{
return
nil
,
errors
.
Wrap
(
err
,
"Failed to find storage"
)
}
if
storage
==
nil
||
storage
.
Type
!=
storepb
.
Storage_S3
{
return
nil
,
nil
}
s3Config
:=
storage
.
Config
.
GetS3Config
()
return
s3
.
NewClient
(
ctx
,
&
s3
.
Config
{
AccessKey
:
s3Config
.
AccessKey
,
SecretKey
:
s3Config
.
SecretKey
,
EndPoint
:
s3Config
.
EndPoint
,
Region
:
s3Config
.
Region
,
Bucket
:
s3Config
.
Bucket
,
URLPrefix
:
s3Config
.
UrlPrefix
,
URLSuffix
:
s3Config
.
UrlSuffix
,
PreSign
:
s3Config
.
PreSign
,
})
}
store/workspace_setting.go
View file @
32096309
...
...
@@ -139,9 +139,9 @@ func (s *Store) GetWorkspaceMemoRelatedSetting(ctx context.Context) (*storepb.Wo
}
const
(
defaultWorkspaceStorageType
=
storepb
.
WorkspaceStorageSetting_STORAGE_TYPE_DATABASE
defaultWorkspaceUploadSizeLimitMb
=
30
defaultWorkspace
LocalStoragePathTemplate
=
"assets/{timestamp}_{filename}"
defaultWorkspaceStorageType
=
storepb
.
WorkspaceStorageSetting_STORAGE_TYPE_DATABASE
defaultWorkspaceUploadSizeLimitMb
=
30
defaultWorkspace
FilepathTemplate
=
"assets/{timestamp}_{filename}"
)
func
(
s
*
Store
)
GetWorkspaceStorageSetting
(
ctx
context
.
Context
)
(
*
storepb
.
WorkspaceStorageSetting
,
error
)
{
...
...
@@ -162,8 +162,8 @@ func (s *Store) GetWorkspaceStorageSetting(ctx context.Context) (*storepb.Worksp
if
workspaceStorageSetting
.
UploadSizeLimitMb
==
0
{
workspaceStorageSetting
.
UploadSizeLimitMb
=
defaultWorkspaceUploadSizeLimitMb
}
if
workspaceStorageSetting
.
LocalStorageP
athTemplate
==
""
{
workspaceStorageSetting
.
LocalStoragePathTemplate
=
defaultWorkspaceLocalStorageP
athTemplate
if
workspaceStorageSetting
.
Filep
athTemplate
==
""
{
workspaceStorageSetting
.
FilepathTemplate
=
defaultWorkspaceFilep
athTemplate
}
return
workspaceStorageSetting
,
nil
}
...
...
web/src/components/CreateStorageServiceDialog.tsx
deleted
100644 → 0
View file @
f25c7d9b
import
{
Button
,
IconButton
,
Input
,
Checkbox
,
Typography
}
from
"@mui/joy"
;
import
React
,
{
useEffect
,
useState
}
from
"react"
;
import
{
toast
}
from
"react-hot-toast"
;
import
{
storageServiceClient
}
from
"@/grpcweb"
;
import
{
S3Config
,
Storage
,
Storage_Type
}
from
"@/types/proto/api/v1/storage_service"
;
import
{
useTranslate
}
from
"@/utils/i18n"
;
import
{
generateDialog
}
from
"./Dialog"
;
import
Icon
from
"./Icon"
;
import
LearnMore
from
"./LearnMore"
;
import
RequiredBadge
from
"./RequiredBadge"
;
interface
Props
extends
DialogProps
{
storage
?:
Storage
;
confirmCallback
?:
()
=>
void
;
}
const
CreateStorageServiceDialog
:
React
.
FC
<
Props
>
=
(
props
:
Props
)
=>
{
const
t
=
useTranslate
();
const
{
destroy
,
storage
,
confirmCallback
}
=
props
;
const
[
basicInfo
,
setBasicInfo
]
=
useState
({
title
:
""
,
});
const
[
type
]
=
useState
<
Storage_Type
>
(
Storage_Type
.
S3
);
const
[
s3Config
,
setS3Config
]
=
useState
<
S3Config
>
({
endPoint
:
""
,
region
:
""
,
accessKey
:
""
,
secretKey
:
""
,
path
:
""
,
bucket
:
""
,
urlPrefix
:
""
,
urlSuffix
:
""
,
preSign
:
false
,
});
const
isCreating
=
storage
===
undefined
;
useEffect
(()
=>
{
if
(
storage
)
{
setBasicInfo
({
title
:
storage
.
title
,
});
if
(
storage
.
type
===
"S3"
)
{
setS3Config
(
S3Config
.
fromPartial
(
storage
.
config
?.
s3Config
||
{}));
}
}
},
[]);
const
handleCloseBtnClick
=
()
=>
{
destroy
();
};
const
allowConfirmAction
=
()
=>
{
if
(
basicInfo
.
title
===
""
)
{
return
false
;
}
if
(
type
===
"S3"
)
{
if
(
s3Config
.
endPoint
===
""
||
s3Config
.
region
===
""
||
s3Config
.
accessKey
===
""
||
s3Config
.
secretKey
===
""
||
s3Config
.
bucket
===
""
)
{
return
false
;
}
}
return
true
;
};
const
handleConfirmBtnClick
=
async
()
=>
{
try
{
if
(
isCreating
)
{
await
storageServiceClient
.
createStorage
({
storage
:
Storage
.
fromPartial
({
title
:
basicInfo
.
title
,
type
:
type
,
config
:
{
s3Config
:
s3Config
,
},
}),
});
}
else
{
await
storageServiceClient
.
updateStorage
({
storage
:
Storage
.
fromPartial
({
id
:
storage
?.
id
,
title
:
basicInfo
.
title
,
type
:
type
,
config
:
{
s3Config
:
s3Config
,
},
}),
updateMask
:
[
"title"
,
"config"
],
});
}
}
catch
(
error
:
any
)
{
console
.
error
(
error
);
toast
.
error
(
error
.
response
.
data
.
message
);
}
if
(
confirmCallback
)
{
confirmCallback
();
}
destroy
();
};
const
setPartialS3Config
=
(
state
:
Partial
<
S3Config
>
)
=>
{
setS3Config
({
...
s3Config
,
...
state
,
});
};
return
(
<>
<
div
className=
"dialog-header-container"
>
<
span
>
{
t
(
isCreating
?
"setting.storage-section.create-storage"
:
"setting.storage-section.update-storage"
)
}
</
span
>
<
IconButton
size=
"sm"
onClick=
{
handleCloseBtnClick
}
>
<
Icon
.
X
className=
"w-5 h-auto"
/>
</
IconButton
>
</
div
>
<
div
className=
"dialog-content-container min-w-[19rem]"
>
<
Typography
className=
"!mb-1"
level=
"body-md"
>
{
t
(
"common.name"
)
}
<
RequiredBadge
/>
</
Typography
>
<
Input
className=
"mb-2"
placeholder=
{
t
(
"common.name"
)
}
value=
{
basicInfo
.
title
}
onChange=
{
(
e
)
=>
setBasicInfo
({
...
basicInfo
,
title
:
e
.
target
.
value
,
})
}
fullWidth
/>
<
Typography
className=
"!mb-1"
level=
"body-md"
>
{
t
(
"setting.storage-section.endpoint"
)
}
<
RequiredBadge
/>
</
Typography
>
<
Input
className=
"mb-2"
placeholder=
{
t
(
"setting.storage-section.s3-compatible-url"
)
}
value=
{
s3Config
.
endPoint
}
onChange=
{
(
e
)
=>
setPartialS3Config
({
endPoint
:
e
.
target
.
value
})
}
fullWidth
/>
<
Typography
className=
"!mb-1"
level=
"body-md"
>
{
t
(
"setting.storage-section.region"
)
}
<
RequiredBadge
/>
</
Typography
>
<
Input
className=
"mb-2"
placeholder=
{
t
(
"setting.storage-section.region-placeholder"
)
}
value=
{
s3Config
.
region
}
onChange=
{
(
e
)
=>
setPartialS3Config
({
region
:
e
.
target
.
value
})
}
fullWidth
/>
<
Typography
className=
"!mb-1"
level=
"body-md"
>
{
t
(
"setting.storage-section.accesskey"
)
}
<
RequiredBadge
/>
</
Typography
>
<
Input
className=
"mb-2"
placeholder=
{
t
(
"setting.storage-section.accesskey-placeholder"
)
}
value=
{
s3Config
.
accessKey
}
onChange=
{
(
e
)
=>
setPartialS3Config
({
accessKey
:
e
.
target
.
value
})
}
fullWidth
/>
<
Typography
className=
"!mb-1"
level=
"body-md"
>
{
t
(
"setting.storage-section.secretkey"
)
}
<
RequiredBadge
/>
</
Typography
>
<
Input
className=
"mb-2"
placeholder=
{
t
(
"setting.storage-section.secretkey-placeholder"
)
}
value=
{
s3Config
.
secretKey
}
onChange=
{
(
e
)
=>
setPartialS3Config
({
secretKey
:
e
.
target
.
value
})
}
fullWidth
/>
<
Typography
className=
"!mb-1"
level=
"body-md"
>
{
t
(
"setting.storage-section.bucket"
)
}
<
RequiredBadge
/>
</
Typography
>
<
Input
className=
"mb-2"
placeholder=
{
t
(
"setting.storage-section.bucket-placeholder"
)
}
value=
{
s3Config
.
bucket
}
onChange=
{
(
e
)
=>
setPartialS3Config
({
bucket
:
e
.
target
.
value
})
}
fullWidth
/>
<
div
className=
"flex flex-row items-center mb-1"
>
<
Typography
level=
"body-md"
>
{
t
(
"setting.storage-section.path"
)
}
</
Typography
>
<
LearnMore
className=
"ml-1"
title=
{
t
(
"setting.storage-section.path-description"
)
}
url=
"https://usememos.com/docs/advanced-settings/local-storage"
/>
</
div
>
<
Input
className=
"mb-2"
placeholder=
{
t
(
"setting.storage-section.path-placeholder"
)
+
"/{year}/{month}/{filename}"
}
value=
{
s3Config
.
path
}
onChange=
{
(
e
)
=>
setPartialS3Config
({
path
:
e
.
target
.
value
})
}
fullWidth
/>
<
Typography
className=
"!mb-1"
level=
"body-md"
>
{
t
(
"setting.storage-section.url-prefix"
)
}
</
Typography
>
<
Input
className=
"mb-2"
placeholder=
{
t
(
"setting.storage-section.url-prefix-placeholder"
)
}
value=
{
s3Config
.
urlPrefix
}
onChange=
{
(
e
)
=>
setPartialS3Config
({
urlPrefix
:
e
.
target
.
value
})
}
fullWidth
/>
<
Typography
className=
"!mb-1"
level=
"body-md"
>
{
t
(
"setting.storage-section.url-suffix"
)
}
</
Typography
>
<
Input
className=
"mb-2"
placeholder=
{
t
(
"setting.storage-section.url-suffix-placeholder"
)
}
value=
{
s3Config
.
urlSuffix
}
onChange=
{
(
e
)
=>
setPartialS3Config
({
urlSuffix
:
e
.
target
.
value
})
}
fullWidth
/>
<
Checkbox
className=
"mb-2"
label=
{
t
(
"setting.storage-section.presign-placeholder"
)
}
checked=
{
s3Config
.
preSign
}
onChange=
{
(
e
)
=>
setPartialS3Config
({
preSign
:
e
.
target
.
checked
})
}
/>
<
div
className=
"mt-2 w-full flex flex-row justify-end items-center space-x-1"
>
<
Button
variant=
"plain"
color=
"neutral"
onClick=
{
handleCloseBtnClick
}
>
{
t
(
"common.cancel"
)
}
</
Button
>
<
Button
onClick=
{
handleConfirmBtnClick
}
disabled=
{
!
allowConfirmAction
()
}
>
{
t
(
isCreating
?
"common.create"
:
"common.update"
)
}
</
Button
>
</
div
>
</
div
>
</>
);
};
function
showCreateStorageServiceDialog
(
storage
?:
Storage
,
confirmCallback
?:
()
=>
void
)
{
generateDialog
(
{
className
:
"create-storage-service-dialog"
,
dialogName
:
"create-storage-service-dialog"
,
},
CreateStorageServiceDialog
,
{
storage
,
confirmCallback
},
);
}
export
default
showCreateStorageServiceDialog
;
web/src/components/Settings/StorageSection.tsx
View file @
32096309
This diff is collapsed.
Click to expand it.
web/src/grpcweb.ts
View file @
32096309
...
...
@@ -6,7 +6,6 @@ import { InboxServiceDefinition } from "./types/proto/api/v1/inbox_service";
import
{
LinkServiceDefinition
}
from
"./types/proto/api/v1/link_service"
;
import
{
MemoServiceDefinition
}
from
"./types/proto/api/v1/memo_service"
;
import
{
ResourceServiceDefinition
}
from
"./types/proto/api/v1/resource_service"
;
import
{
StorageServiceDefinition
}
from
"./types/proto/api/v1/storage_service"
;
import
{
TagServiceDefinition
}
from
"./types/proto/api/v1/tag_service"
;
import
{
UserServiceDefinition
}
from
"./types/proto/api/v1/user_service"
;
import
{
WebhookServiceDefinition
}
from
"./types/proto/api/v1/webhook_service"
;
...
...
@@ -44,6 +43,4 @@ export const webhookServiceClient = clientFactory.create(WebhookServiceDefinitio
export
const
linkServiceClient
=
clientFactory
.
create
(
LinkServiceDefinition
,
channel
);
export
const
storageServiceClient
=
clientFactory
.
create
(
StorageServiceDefinition
,
channel
);
export
const
identityProviderServiceClient
=
clientFactory
.
create
(
IdentityProviderServiceDefinition
,
channel
);
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment