fix(frontend): restore sitemap and robots routes

server/router/frontend/frontend.go

@@ -3,10 +3,14 @@ package frontend
 import (
 	"context"
 	"embed"
+	"encoding/xml"
 	"io/fs"
+	"net/http"
+	"strings"
 
 	"github.com/labstack/echo/v5"
 	"github.com/labstack/echo/v5/middleware"
+	"github.com/pkg/errors"
 
 	"github.com/usememos/memos/internal/profile"
 	"github.com/usememos/memos/internal/util"
@@ -28,10 +32,10 @@ func NewFrontendService(profile *profile.Profile, store *store.Store) *FrontendS
 	}
 }
 
-func (*FrontendService) Serve(_ context.Context, e *echo.Echo) {
+func (s *FrontendService) Serve(_ context.Context, e *echo.Echo) {
 	skipper := func(c *echo.Context) bool {
 		// Skip API routes.
-		if util.HasPrefixes(c.Path(), "/api", "/memos.api.v1") {
+		if util.HasPrefixes(c.Path(), "/api", "/memos.api.v1", "/robots.txt", "/sitemap.xml") {
 			return true
 		}
 		// For index.html and root path, set no-cache headers to prevent browser caching
@@ -57,6 +61,8 @@ func (*FrontendService) Serve(_ context.Context, e *echo.Echo) {
 		HTML5:      true, // Enable fallback to index.html
 		Skipper:    skipper,
 	}))
+
+	s.registerRoutes(e)
 }
 
 func getFileSystem(path string) fs.FS {
@@ -66,3 +72,70 @@ func getFileSystem(path string) fs.FS {
 	}
 	return sub
 }
+
+func (s *FrontendService) registerRoutes(e *echo.Echo) {
+	e.GET("/robots.txt", s.getRobotsTXT)
+	e.GET("/sitemap.xml", s.getSitemapXML)
+}
+
+func (s *FrontendService) getRobotsTXT(c *echo.Context) error {
+	instanceURL, err := normalizeInstanceURL(s.Profile.InstanceURL)
+	if err != nil {
+		return err
+	}
+
+	robotsTXT := strings.Join([]string{
+		"User-agent: *",
+		"Allow: /",
+		"Host: " + instanceURL,
+		"Sitemap: " + instanceURL + "/sitemap.xml",
+	}, "\n")
+	return c.String(http.StatusOK, robotsTXT)
+}
+
+func (s *FrontendService) getSitemapXML(c *echo.Context) error {
+	instanceURL, err := normalizeInstanceURL(s.Profile.InstanceURL)
+	if err != nil {
+		return err
+	}
+
+	memos, err := s.Store.ListMemos(c.Request().Context(), &store.FindMemo{
+		VisibilityList: []store.Visibility{store.Public},
+	})
+	if err != nil {
+		return errors.Wrap(err, "failed to list public memos for sitemap")
+	}
+
+	urls := make([]sitemapURL, 0, len(memos))
+	for _, memo := range memos {
+		urls = append(urls, sitemapURL{
+			Loc: instanceURL + "/m/" + memo.UID,
+		})
+	}
+
+	return c.XML(http.StatusOK, sitemapURLSet{
+		XMLNS: sitemapXMLNamespace,
+		URLs:  urls,
+	})
+}
+
+func normalizeInstanceURL(instanceURL string) (string, error) {
+	instanceURL = strings.TrimRight(instanceURL, "/")
+	if instanceURL == "" {
+		return "", echo.NewHTTPError(http.StatusNotFound, "instance URL is not configured")
+	}
+	return instanceURL, nil
+}
+
+type sitemapURLSet struct {
+	XMLName xml.Name     `xml:"urlset"`
+	XMLNS   string       `xml:"xmlns,attr"`
+	URLs    []sitemapURL `xml:"url"`
+}
+
+type sitemapURL struct {
+	Loc string `xml:"loc"`
+}
+
+//nolint:revive
+const sitemapXMLNamespace = "http://www.sitemaps.org/schemas/sitemap/0.9"

server/router/frontend/frontend_test.go

+package frontend
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/labstack/echo/v5"
+	"github.com/stretchr/testify/require"
+
+	"github.com/usememos/memos/internal/profile"
+	"github.com/usememos/memos/store"
+	teststore "github.com/usememos/memos/store/test"
+)
+
+func TestFrontendService_RobotsTXT(t *testing.T) {
+	ctx := context.Background()
+	testStore := teststore.NewTestingStore(ctx, t)
+	profile := &profile.Profile{
+		InstanceURL: "https://demo.usememos.com/",
+	}
+
+	e := echo.New()
+	NewFrontendService(profile, testStore).Serve(ctx, e)
+
+	req := httptest.NewRequest(http.MethodGet, "/robots.txt", nil)
+	rec := httptest.NewRecorder()
+	e.ServeHTTP(rec, req)
+
+	require.Equal(t, http.StatusOK, rec.Code)
+	require.Equal(t, "text/plain; charset=UTF-8", rec.Header().Get("Content-Type"))
+	require.Equal(t, "User-agent: *\nAllow: /\nHost: https://demo.usememos.com\nSitemap: https://demo.usememos.com/sitemap.xml", rec.Body.String())
+}
+
+func TestFrontendService_SitemapXML(t *testing.T) {
+	ctx := context.Background()
+	testStore := teststore.NewTestingStore(ctx, t)
+	profile := &profile.Profile{
+		InstanceURL: "https://demo.usememos.com",
+	}
+
+	user, err := testStore.CreateUser(ctx, &store.User{
+		Username: "sitemap-owner",
+		Role:     store.RoleUser,
+		Email:    "sitemap-owner@example.com",
+	})
+	require.NoError(t, err)
+
+	_, err = testStore.CreateMemo(ctx, &store.Memo{
+		UID:        "publicmemo",
+		CreatorID:  user.ID,
+		Content:    "public memo",
+		Visibility: store.Public,
+	})
+	require.NoError(t, err)
+
+	_, err = testStore.CreateMemo(ctx, &store.Memo{
+		UID:        "privatememo",
+		CreatorID:  user.ID,
+		Content:    "private memo",
+		Visibility: store.Private,
+	})
+	require.NoError(t, err)
+
+	e := echo.New()
+	NewFrontendService(profile, testStore).Serve(ctx, e)
+
+	req := httptest.NewRequest(http.MethodGet, "/sitemap.xml", nil)
+	rec := httptest.NewRecorder()
+	e.ServeHTTP(rec, req)
+
+	require.Equal(t, http.StatusOK, rec.Code)
+	require.Contains(t, rec.Header().Get("Content-Type"), "application/xml")
+	require.Contains(t, rec.Body.String(), `<loc>https://demo.usememos.com/m/publicmemo</loc>`)
+	require.NotContains(t, rec.Body.String(), "privatememo")
+}
+
+func TestFrontendService_SitemapRoutesRequireInstanceURL(t *testing.T) {
+	ctx := context.Background()
+	testStore := teststore.NewTestingStore(ctx, t)
+
+	e := echo.New()
+	NewFrontendService(&profile.Profile{}, testStore).Serve(ctx, e)
+
+	for _, path := range []string{"/robots.txt", "/sitemap.xml"} {
+		req := httptest.NewRequest(http.MethodGet, path, nil)
+		rec := httptest.NewRecorder()
+		e.ServeHTTP(rec, req)
+
+		require.Equal(t, http.StatusNotFound, rec.Code)
+	}
+}