feat: support local storage (#1383)

* feat: support local storage

* update

* update

* update

* update

api/resource.go

@@ -11,6 +11,7 @@ type Resource struct {
 	// Domain specific fields
 	Filename     string     `json:"filename"`
 	Blob         []byte     `json:"-"`
+	InternalPath string     `json:"internalPath"`
 	ExternalLink string     `json:"externalLink"`
 	Type         string     `json:"type"`
 	Size         int64      `json:"size"`
@@ -27,6 +28,7 @@ type ResourceCreate struct {
 	// Domain specific fields
 	Filename     string     `json:"filename"`
 	Blob         []byte     `json:"-"`
+	InternalPath string     `json:"internalPath"`
 	ExternalLink string     `json:"externalLink"`
 	Type         string     `json:"type"`
 	Size         int64      `json:"-"`

api/system.go

@@ -18,5 +18,8 @@ type SystemStatus struct {
 	AdditionalScript string `json:"additionalScript"`
 	// Customized server profile, including server name and external url.
 	CustomizedProfile CustomizedProfile `json:"customizedProfile"`
+	// Storage service ID.
 	StorageServiceID int `json:"storageServiceId"`
+	// Local storage path
+	LocalStoragePath string `json:"localStoragePath"`
 }

api/system_setting.go

@@ -27,6 +27,8 @@ const (
 	SystemSettingCustomizedProfileName SystemSettingName = "customizedProfile"
 	// SystemSettingStorageServiceIDName is the key type of storage service ID.
 	SystemSettingStorageServiceIDName SystemSettingName = "storageServiceId"
+	// SystemSettingLocalStoragePathName is the key type of local storage path.
+	SystemSettingLocalStoragePathName SystemSettingName = "localStoragePath"
 	// SystemSettingOpenAIConfigName is the key type of OpenAI config.
 	SystemSettingOpenAIConfigName SystemSettingName = "openAIConfig"
 )
@@ -70,6 +72,8 @@ func (key SystemSettingName) String() string {
 		return "customizedProfile"
 	case SystemSettingStorageServiceIDName:
 		return "storageServiceId"
+	case SystemSettingLocalStoragePathName:
+		return "localStoragePath"
 	case SystemSettingOpenAIConfigName:
 		return "openAIConfig"
 	}
@@ -142,6 +146,12 @@ func (upsert SystemSettingUpsert) Validate() error {
 			return fmt.Errorf("failed to unmarshal system setting storage service id value")
 		}
 		return nil
+	} else if upsert.Name == SystemSettingLocalStoragePathName {
+		value := ""
+		err := json.Unmarshal([]byte(upsert.Value), &value)
+		if err != nil {
+			return fmt.Errorf("failed to unmarshal system setting local storage path value")
+		}
 	} else if upsert.Name == SystemSettingOpenAIConfigName {
 		value := OpenAIConfig{}
 		err := json.Unmarshal([]byte(upsert.Value), &value)

server/resource.go

@@ -7,7 +7,9 @@ import (
 	"io"
 	"net/http"
 	"net/url"
+	"os"
 	"path"
+	"path/filepath"
 	"regexp"
 	"strconv"
 	"strings"
@@ -105,13 +107,13 @@ func (s *Server) registerResourceRoutes(g *echo.Group) {
 		}
 		defer src.Close()
 
-		systemSetting, err := s.Store.FindSystemSetting(ctx, &api.SystemSettingFind{Name: api.SystemSettingStorageServiceIDName})
+		systemSettingStorageServiceID, err := s.Store.FindSystemSetting(ctx, &api.SystemSettingFind{Name: api.SystemSettingStorageServiceIDName})
 		if err != nil && common.ErrorCode(err) != common.NotFound {
 			return echo.NewHTTPError(http.StatusInternalServerError, "Failed to find storage").SetInternal(err)
 		}
 		storageServiceID := 0
-		if systemSetting != nil {
-			err = json.Unmarshal([]byte(systemSetting.Value), &storageServiceID)
+		if systemSettingStorageServiceID != nil {
+			err = json.Unmarshal([]byte(systemSettingStorageServiceID.Value), &storageServiceID)
 			if err != nil {
 				return echo.NewHTTPError(http.StatusInternalServerError, "Failed to unmarshal storage service id").SetInternal(err)
 			}
@@ -119,6 +121,7 @@ func (s *Server) registerResourceRoutes(g *echo.Group) {
 
 		var resourceCreate *api.ResourceCreate
 		if storageServiceID == 0 {
+			// Database storage.
 			fileBytes, err := io.ReadAll(src)
 			if err != nil {
 				return echo.NewHTTPError(http.StatusInternalServerError, "Failed to read file").SetInternal(err)
@@ -130,6 +133,47 @@ func (s *Server) registerResourceRoutes(g *echo.Group) {
 				Size:      size,
 				Blob:      fileBytes,
 			}
+		} else if storageServiceID == -1 {
+			// Local storage.
+			systemSettingLocalStoragePath, err := s.Store.FindSystemSetting(ctx, &api.SystemSettingFind{Name: api.SystemSettingLocalStoragePathName})
+			if err != nil && common.ErrorCode(err) != common.NotFound {
+				return echo.NewHTTPError(http.StatusInternalServerError, "Failed to find storage").SetInternal(err)
+			}
+			localStoragePath := ""
+			if systemSettingLocalStoragePath != nil {
+				err = json.Unmarshal([]byte(systemSettingLocalStoragePath.Value), &localStoragePath)
+				if err != nil {
+					return echo.NewHTTPError(http.StatusInternalServerError, "Failed to unmarshal storage service id").SetInternal(err)
+				}
+			}
+			filePath := localStoragePath
+			if !strings.Contains(filePath, "{filename}") {
+				filePath = path.Join(filePath, "{filename}")
+			}
+			filePath = path.Join(s.Profile.Data, replacePathTemplate(filePath, filename))
+			dirPath := filepath.Dir(filePath)
+			err = os.MkdirAll(dirPath, os.ModePerm)
+			if err != nil {
+				return echo.NewHTTPError(http.StatusInternalServerError, "Failed to create directory").SetInternal(err)
+			}
+			dst, err := os.Create(filePath)
+			if err != nil {
+				return echo.NewHTTPError(http.StatusInternalServerError, "Failed to create file").SetInternal(err)
+			}
+			defer dst.Close()
+
+			_, err = io.Copy(dst, src)
+			if err != nil {
+				return echo.NewHTTPError(http.StatusInternalServerError, "Failed to copy file").SetInternal(err)
+			}
+
+			resourceCreate = &api.ResourceCreate{
+				CreatorID:    userID,
+				Filename:     filename,
+				Type:         filetype,
+				Size:         size,
+				InternalPath: filePath,
+			}
 		} else {
 			storage, err := s.Store.FindStorage(ctx, &api.StorageFind{ID: &storageServiceID})
 			if err != nil {
@@ -138,38 +182,11 @@ func (s *Server) registerResourceRoutes(g *echo.Group) {
 
 			if storage.Type == api.StorageS3 {
 				s3Config := storage.Config.S3Config
-				t := time.Now()
 				var s3FileKey string
-				if s3Config.Path == "" {
-					s3FileKey = filename
-				} else {
-					s3FileKey = fileKeyPattern.ReplaceAllStringFunc(s3Config.Path, func(s string) string {
-						switch s {
-						case "{filename}":
-							return filename
-						case "{timestamp}":
-							return fmt.Sprintf("%d", t.Unix())
-						case "{year}":
-							return fmt.Sprintf("%d", t.Year())
-						case "{month}":
-							return fmt.Sprintf("%02d", t.Month())
-						case "{day}":
-							return fmt.Sprintf("%02d", t.Day())
-						case "{hour}":
-							return fmt.Sprintf("%02d", t.Hour())
-						case "{minute}":
-							return fmt.Sprintf("%02d", t.Minute())
-						case "{second}":
-							return fmt.Sprintf("%02d", t.Second())
-						}
-						return s
-					})
-
 				if !strings.Contains(s3Config.Path, "{filename}") {
-						s3FileKey = path.Join(s3FileKey, filename)
-					}
+					s3FileKey = path.Join(s3Config.Path, "{filename}")
 				}
-
+				s3FileKey = replacePathTemplate(s3FileKey, filename)
 				s3client, err := s3.NewClient(ctx, &s3.Config{
 					AccessKey: s3Config.AccessKey,
 					SecretKey: s3Config.SecretKey,
@@ -387,16 +404,29 @@ func (s *Server) registerResourcePublicRoutes(g *echo.Group) {
 			return echo.NewHTTPError(http.StatusInternalServerError, fmt.Sprintf("Failed to find resource by ID: %v", resourceID)).SetInternal(err)
 		}
 
+		blob := resource.Blob
+		if resource.InternalPath != "" {
+			src, err := os.Open(resource.InternalPath)
+			if err != nil {
+				return echo.NewHTTPError(http.StatusInternalServerError, fmt.Sprintf("Failed to open the local resource: %s", resource.InternalPath)).SetInternal(err)
+			}
+			defer src.Close()
+			blob, err = io.ReadAll(src)
+			if err != nil {
+				return echo.NewHTTPError(http.StatusInternalServerError, fmt.Sprintf("Failed to read the local resource: %s", resource.InternalPath)).SetInternal(err)
+			}
+		}
+
 		c.Response().Writer.Header().Set(echo.HeaderCacheControl, "max-age=31536000, immutable")
 		c.Response().Writer.Header().Set(echo.HeaderContentSecurityPolicy, "default-src 'self'")
 		resourceType := strings.ToLower(resource.Type)
 		if strings.HasPrefix(resourceType, "text") {
 			resourceType = echo.MIMETextPlainCharsetUTF8
 		} else if strings.HasPrefix(resourceType, "video") || strings.HasPrefix(resourceType, "audio") {
-			http.ServeContent(c.Response(), c.Request(), resource.Filename, time.Unix(resource.UpdatedTs, 0), bytes.NewReader(resource.Blob))
+			http.ServeContent(c.Response(), c.Request(), resource.Filename, time.Unix(resource.UpdatedTs, 0), bytes.NewReader(blob))
 			return nil
 		}
-		return c.Stream(http.StatusOK, resourceType, bytes.NewReader(resource.Blob))
+		return c.Stream(http.StatusOK, resourceType, bytes.NewReader(blob))
 	})
 }
 
@@ -422,3 +452,29 @@ func (s *Server) createResourceCreateActivity(c echo.Context, resource *api.Reso
 	}
 	return err
 }
+
+func replacePathTemplate(path string, filename string) string {
+	t := time.Now()
+	path = fileKeyPattern.ReplaceAllStringFunc(path, func(s string) string {
+		switch s {
+		case "{filename}":
+			return filename
+		case "{timestamp}":
+			return fmt.Sprintf("%d", t.Unix())
+		case "{year}":
+			return fmt.Sprintf("%d", t.Year())
+		case "{month}":
+			return fmt.Sprintf("%02d", t.Month())
+		case "{day}":
+			return fmt.Sprintf("%02d", t.Day())
+		case "{hour}":
+			return fmt.Sprintf("%02d", t.Hour())
+		case "{minute}":
+			return fmt.Sprintf("%02d", t.Minute())
+		case "{second}":
+			return fmt.Sprintf("%02d", t.Second())
+		}
+		return s
+	})
+	return path
+}

server/system.go

@@ -52,6 +52,7 @@ func (s *Server) registerSystemRoutes(g *echo.Group) {
 				ExternalURL: "",
 			},
 			StorageServiceID: 0,
+			LocalStoragePath: "",
 		}
 
 		systemSettingList, err := s.Store.FindSystemSettingList(ctx, &api.SystemSettingFind{})
@@ -86,6 +87,8 @@ func (s *Server) registerSystemRoutes(g *echo.Group) {
 				systemStatus.CustomizedProfile = customizedProfile
 			} else if systemSetting.Name == api.SystemSettingStorageServiceIDName {
 				systemStatus.StorageServiceID = int(baseValue.(float64))
+			} else if systemSetting.Name == api.SystemSettingLocalStoragePathName {
+				systemStatus.LocalStoragePath = baseValue.(string)
 			}
 		}
 

store/db/migration/dev/LATEST__SCHEMA.sql

@@ -74,6 +74,7 @@ CREATE TABLE resource (
   updated_ts BIGINT NOT NULL DEFAULT (strftime('%s', 'now')),
   filename TEXT NOT NULL DEFAULT '',
   blob BLOB DEFAULT NULL,
+  internal_path TEXT NOT NULL DEFAULT '',
   external_link TEXT NOT NULL DEFAULT '',
   type TEXT NOT NULL DEFAULT '',
   size INTEGER NOT NULL DEFAULT 0,

store/resource.go

@@ -24,6 +24,7 @@ type resourceRaw struct {
 	// Domain specific fields
 	Filename         string
 	Blob             []byte
+	InternalPath     string
 	ExternalLink     string
 	Type             string
 	Size             int64
@@ -43,6 +44,7 @@ func (raw *resourceRaw) toResource() *api.Resource {
 		// Domain specific fields
 		Filename:         raw.Filename,
 		Blob:             raw.Blob,
+		InternalPath:     raw.InternalPath,
 		ExternalLink:     raw.ExternalLink,
 		Type:             raw.Type,
 		Size:             raw.Size,
@@ -195,9 +197,9 @@ func (s *Store) createResourceImpl(ctx context.Context, tx *sql.Tx, create *api.
 	values := []any{create.Filename, create.Blob, create.ExternalLink, create.Type, create.Size, create.CreatorID}
 	placeholders := []string{"?", "?", "?", "?", "?", "?"}
 	if s.profile.IsDev() {
-		fields = append(fields, "visibility")
-		values = append(values, create.Visibility)
-		placeholders = append(placeholders, "?")
+		fields = append(fields, "visibility", "internal_path")
+		values = append(values, create.Visibility, create.InternalPath)
+		placeholders = append(placeholders, "?", "?")
 	}
 
 	query := `
@@ -218,7 +220,7 @@ func (s *Store) createResourceImpl(ctx context.Context, tx *sql.Tx, create *api.
 		&resourceRaw.CreatorID,
 	}
 	if s.profile.IsDev() {
-		dests = append(dests, &resourceRaw.Visibility)
+		dests = append(dests, &resourceRaw.Visibility, &resourceRaw.InternalPath)
 	}
 	dests = append(dests, []any{&resourceRaw.CreatedTs, &resourceRaw.UpdatedTs}...)
 	if err := tx.QueryRowContext(ctx, query, values...).Scan(dests...); err != nil {
@@ -247,7 +249,7 @@ func (s *Store) patchResourceImpl(ctx context.Context, tx *sql.Tx, patch *api.Re
 
 	fields := []string{"id", "filename", "external_link", "type", "size", "creator_id", "created_ts", "updated_ts"}
 	if s.profile.IsDev() {
-		fields = append(fields, "visibility")
+		fields = append(fields, "visibility", "internal_path")
 	}
 
 	query := `
@@ -267,7 +269,7 @@ func (s *Store) patchResourceImpl(ctx context.Context, tx *sql.Tx, patch *api.Re
 		&resourceRaw.UpdatedTs,
 	}
 	if s.profile.IsDev() {
-		dests = append(dests, &resourceRaw.Visibility)
+		dests = append(dests, &resourceRaw.Visibility, &resourceRaw.InternalPath)
 	}
 	if err := tx.QueryRowContext(ctx, query, args...).Scan(dests...); err != nil {
 		return nil, FormatError(err)
@@ -297,7 +299,7 @@ func (s *Store) findResourceListImpl(ctx context.Context, tx *sql.Tx, find *api.
 		fields = append(fields, "resource.blob")
 	}
 	if s.profile.IsDev() {
-		fields = append(fields, "resource.visibility")
+		fields = append(fields, "visibility", "internal_path")
 	}
 
 	query := fmt.Sprintf(`
@@ -334,7 +336,7 @@ func (s *Store) findResourceListImpl(ctx context.Context, tx *sql.Tx, find *api.
 			dests = append(dests, &resourceRaw.Blob)
 		}
 		if s.profile.IsDev() {
-			dests = append(dests, &resourceRaw.Visibility)
+			dests = append(dests, &resourceRaw.Visibility, &resourceRaw.InternalPath)
 		}
 		if err := rows.Scan(dests...); err != nil {
 			return nil, FormatError(err)

web/src/components/Settings/StorageSection.tsx

@@ -5,6 +5,7 @@ import { useTranslation } from "react-i18next";
 import { useGlobalStore } from "../../store/module";
 import * as api from "../../helpers/api";
 import showCreateStorageServiceDialog from "../CreateStorageServiceDialog";
+import showUpdateLocalStorageDialog from "../UpdateLocalStorageDialog";
 import Dropdown from "../base/Dropdown";
 import { showCommonDialog } from "../Dialog/CommonDialog";
 
@@ -27,10 +28,6 @@ const StorageSection = () => {
   };
 
   const handleActiveStorageServiceChanged = async (storageId: StorageId) => {
-    if (storageList.length === 0) {
-      return;
-    }
-
     await api.upsertSystemSetting({
       name: "storageServiceId",
       value: JSON.stringify(storageId),
@@ -70,6 +67,7 @@ const StorageSection = () => {
         }}
       >
         <Option value={0}>Database</Option>
+        <Option value={-1}>Local</Option>
         {storageList.map((storage) => (
           <Option key={storage.id} value={storage.id}>
             {storage.name}
@@ -84,6 +82,26 @@ const StorageSection = () => {
         </button>
       </div>
       <div className="mt-2 w-full flex flex-col">
+        <div className="py-2 w-full border-t last:border-b flex flex-row items-center justify-between">
+          <div className="flex flex-row items-center">
+            <p className="ml-2">Local</p>
+          </div>
+          <div className="flex flex-row items-center">
+            <Dropdown
+              actionsClassName="!w-28"
+              actions={
+                <>
+                  <button
+                    className="w-full text-left text-sm leading-6 py-1 px-3 cursor-pointer rounded hover:bg-gray-100 dark:hover:bg-zinc-600"
+                    onClick={() => showUpdateLocalStorageDialog(systemStatus.localStoragePath)}
+                  >
+                    Edit
+                  </button>
+                </>
+              }
+            />
+          </div>
+        </div>
         {storageList.map((storage) => (
           <div key={storage.id} className="py-2 w-full border-t last:border-b flex flex-row items-center justify-between">
             <div className="flex flex-row items-center">

web/src/components/UpdateLocalStorageDialog.tsx

+import { useState } from "react";
+import { Button, Input, Typography } from "@mui/joy";
+import { toast } from "react-hot-toast";
+import * as api from "../helpers/api";
+import { generateDialog } from "./Dialog";
+import Icon from "./Icon";
+import { useGlobalStore } from "../store/module";
+
+interface Props extends DialogProps {
+  localStoragePath?: string;
+  confirmCallback?: () => void;
+}
+
+const UpdateLocalStorageDialog: React.FC<Props> = (props: Props) => {
+  const { destroy, localStoragePath, confirmCallback } = props;
+  const globalStore = useGlobalStore();
+  const [path, setPath] = useState(localStoragePath || "");
+
+  const handleCloseBtnClick = () => {
+    destroy();
+  };
+
+  const handleConfirmBtnClick = async () => {
+    try {
+      await api.upsertSystemSetting({
+        name: "localStoragePath",
+        value: JSON.stringify(path),
+      });
+      await globalStore.fetchSystemStatus();
+    } catch (error: any) {
+      console.error(error);
+      toast.error(error.response.data.message);
+    }
+    if (confirmCallback) {
+      confirmCallback();
+    }
+    destroy();
+  };
+
+  return (
+    <>
+      <div className="dialog-header-container">
+        <p className="title-text">Update local storage path</p>
+        <button className="btn close-btn" onClick={handleCloseBtnClick}>
+          <Icon.X />
+        </button>
+      </div>
+      <div className="dialog-content-container">
+        <div className="py-2">
+          <Typography className="!mb-1" level="body2">
+            Local Path
+          </Typography>
+          <Typography className="!mb-1" level="body2">
+            <span className="text-sm text-gray-400 ml-1">{"e.g., {year}/{month}/{day}/your/path/{timestamp}_{filename}"}</span>
+          </Typography>
+          <Input className="mb-2" placeholder="Path" value={path} onChange={(e) => setPath(e.target.value)} fullWidth />
+        </div>
+        <div className="mt-2 w-full flex flex-row justify-end items-center space-x-1">
+          <Button variant="plain" color="neutral" onClick={handleCloseBtnClick}>
+            Cancel
+          </Button>
+          <Button onClick={handleConfirmBtnClick}>Update</Button>
+        </div>
+      </div>
+    </>
+  );
+};
+
+function showUpdateLocalStorageDialog(localStoragePath?: string, confirmCallback?: () => void) {
+  generateDialog(
+    {
+      className: "update-local-storage-dialog",
+      dialogName: "update-local-storage-dialog",
+    },
+    UpdateLocalStorageDialog,
+    { localStoragePath, confirmCallback }
+  );
+}
+
+export default showUpdateLocalStorageDialog;

web/src/types/modules/system.d.ts

@@ -28,6 +28,7 @@ interface SystemStatus {
   additionalScript: string;
   customizedProfile: CustomizedProfile;
   storageServiceId: number;
+  localStoragePath: string;
 }
 
 interface SystemSetting {