feat: implement math expression parser

api/v2/markdown_service.go

@@ -61,6 +61,8 @@ func convertFromASTNode(rawNode ast.Node) *apiv2pb.Node {
 	case *ast.TaskList:
 		children := convertFromASTNodes(n.Children)
 		node.Node = &apiv2pb.Node_TaskListNode{TaskListNode: &apiv2pb.TaskListNode{Symbol: n.Symbol, Complete: n.Complete, Children: children}}
+	case *ast.MathBlock:
+		node.Node = &apiv2pb.Node_MathBlockNode{MathBlockNode: &apiv2pb.MathBlockNode{Content: n.Content}}
 	case *ast.Text:
 		node.Node = &apiv2pb.Node_TextNode{TextNode: &apiv2pb.TextNode{Content: n.Content}}
 	case *ast.Bold:
@@ -84,6 +86,8 @@ func convertFromASTNode(rawNode ast.Node) *apiv2pb.Node {
 		node.Node = &apiv2pb.Node_StrikethroughNode{StrikethroughNode: &apiv2pb.StrikethroughNode{Content: n.Content}}
 	case *ast.EscapingCharacter:
 		node.Node = &apiv2pb.Node_EscapingCharacterNode{EscapingCharacterNode: &apiv2pb.EscapingCharacterNode{Symbol: n.Symbol}}
+	case *ast.Math:
+		node.Node = &apiv2pb.Node_MathNode{MathNode: &apiv2pb.MathNode{Content: n.Content}}
 	default:
 		node.Node = &apiv2pb.Node_TextNode{TextNode: &apiv2pb.TextNode{}}
 	}

plugin/gomark/ast/ast.go

@@ -14,6 +14,7 @@ const (
 	OrderedListNode
 	UnorderedListNode
 	TaskListNode
+	MathBlockNode
 	// Inline nodes.
 	TextNode
 	BoldNode
@@ -26,6 +27,7 @@ const (
 	TagNode
 	StrikethroughNode
 	EscapingCharacterNode
+	MathNode
 )
 
 type Node interface {

plugin/gomark/ast/block.go

@@ -170,3 +170,17 @@ func (n *TaskList) Restore() string {
 	}
 	return fmt.Sprintf("%s [%s] %s", n.Symbol, complete, result)
 }
+
+type MathBlock struct {
+	BaseBlock
+
+	Content string
+}
+
+func (*MathBlock) Type() NodeType {
+	return MathBlockNode
+}
+
+func (n *MathBlock) Restore() string {
+	return fmt.Sprintf("$$\n%s\n$$", n.Content)
+}

plugin/gomark/ast/inline.go

@@ -177,3 +177,17 @@ func (*EscapingCharacter) Type() NodeType {
 func (n *EscapingCharacter) Restore() string {
 	return fmt.Sprintf("\\%s", n.Symbol)
 }
+
+type Math struct {
+	BaseInline
+
+	Content string
+}
+
+func (*Math) Type() NodeType {
+	return MathNode
+}
+
+func (n *Math) Restore() string {
+	return fmt.Sprintf("$%s$", n.Content)
+}

plugin/gomark/parser/math.go

+package parser
+
+import (
+	"errors"
+
+	"github.com/usememos/memos/plugin/gomark/ast"
+	"github.com/usememos/memos/plugin/gomark/parser/tokenizer"
+)
+
+type MathParser struct{}
+
+func NewMathParser() *MathParser {
+	return &MathParser{}
+}
+
+func (*MathParser) Match(tokens []*tokenizer.Token) (int, bool) {
+	if len(tokens) < 3 {
+		return 0, false
+	}
+
+	if tokens[0].Type != tokenizer.DollarSign {
+		return 0, false
+	}
+
+	contentTokens := []*tokenizer.Token{}
+	for _, token := range tokens[1:] {
+		if token.Type == tokenizer.Newline {
+			return 0, false
+		}
+		if token.Type == tokenizer.DollarSign {
+			break
+		}
+		contentTokens = append(contentTokens, token)
+	}
+	if len(contentTokens) == 0 {
+		return 0, false
+	}
+	if len(contentTokens)+2 > len(tokens) {
+		return 0, false
+	}
+	if tokens[len(contentTokens)+1].Type != tokenizer.DollarSign {
+		return 0, false
+	}
+	return len(contentTokens) + 2, true
+}
+
+func (p *MathParser) Parse(tokens []*tokenizer.Token) (ast.Node, error) {
+	size, ok := p.Match(tokens)
+	if size == 0 || !ok {
+		return nil, errors.New("not matched")
+	}
+
+	return &ast.Math{
+		Content: tokenizer.Stringify(tokens[1 : size-1]),
+	}, nil
+}

plugin/gomark/parser/math_block.go

+package parser
+
+import (
+	"errors"
+
+	"github.com/usememos/memos/plugin/gomark/ast"
+	"github.com/usememos/memos/plugin/gomark/parser/tokenizer"
+)
+
+type MathBlockParser struct{}
+
+func NewMathBlockParser() *MathBlockParser {
+	return &MathBlockParser{}
+}
+
+func (*MathBlockParser) Match(tokens []*tokenizer.Token) (int, bool) {
+	if len(tokens) < 7 {
+		return 0, false
+	}
+
+	if tokens[0].Type != tokenizer.DollarSign && tokens[1].Type != tokenizer.DollarSign && tokens[2].Type != tokenizer.Newline {
+		return 0, false
+	}
+
+	cursor := 3
+	matched := false
+	for ; cursor < len(tokens)-2; cursor++ {
+		if tokens[cursor].Type == tokenizer.Newline && tokens[cursor+1].Type == tokenizer.DollarSign && tokens[cursor+2].Type == tokenizer.DollarSign {
+			if cursor+2 == len(tokens)-1 {
+				cursor += 3
+				matched = true
+				break
+			} else if tokens[cursor+3].Type == tokenizer.Newline {
+				cursor += 3
+				matched = true
+				break
+			}
+		}
+	}
+	if !matched {
+		return 0, false
+	}
+
+	return cursor, true
+}
+
+func (p *MathBlockParser) Parse(tokens []*tokenizer.Token) (ast.Node, error) {
+	size, ok := p.Match(tokens)
+	if size == 0 || !ok {
+		return nil, errors.New("not matched")
+	}
+
+	return &ast.MathBlock{
+		Content: tokenizer.Stringify(tokens[3 : size-3]),
+	}, nil
+}

plugin/gomark/parser/math_block_test.go

+package parser
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/usememos/memos/plugin/gomark/ast"
+	"github.com/usememos/memos/plugin/gomark/parser/tokenizer"
+	"github.com/usememos/memos/plugin/gomark/restore"
+)
+
+func TestMathBlockParser(t *testing.T) {
+	tests := []struct {
+		text string
+		link ast.Node
+	}{
+		{
+			text: "$$\n(1+x)^2\n$$",
+			link: &ast.MathBlock{
+				Content: "(1+x)^2",
+			},
+		},
+	}
+	for _, test := range tests {
+		tokens := tokenizer.Tokenize(test.text)
+		node, _ := NewMathBlockParser().Parse(tokens)
+		require.Equal(t, restore.Restore([]ast.Node{test.link}), restore.Restore([]ast.Node{node}))
+	}
+}

plugin/gomark/parser/math_test.go

+package parser
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/usememos/memos/plugin/gomark/ast"
+	"github.com/usememos/memos/plugin/gomark/parser/tokenizer"
+	"github.com/usememos/memos/plugin/gomark/restore"
+)
+
+func TestMathParser(t *testing.T) {
+	tests := []struct {
+		text string
+		link ast.Node
+	}{
+		{
+			text: "$\\sqrt{3x-1}+(1+x)^2$",
+			link: &ast.Math{
+				Content: "\\sqrt{3x-1}+(1+x)^2",
+			},
+		},
+	}
+	for _, test := range tests {
+		tokens := tokenizer.Tokenize(test.text)
+		node, _ := NewMathParser().Parse(tokens)
+		require.Equal(t, restore.Restore([]ast.Node{test.link}), restore.Restore([]ast.Node{node}))
+	}
+}

plugin/gomark/parser/parser.go

@@ -37,6 +37,7 @@ var defaultBlockParsers = []BlockParser{
 	NewTaskListParser(),
 	NewUnorderedListParser(),
 	NewOrderedListParser(),
+	NewMathBlockParser(),
 	NewParagraphParser(),
 	NewLineBreakParser(),
 }
@@ -90,6 +91,7 @@ var defaultInlineParsers = []InlineParser{
 	NewBoldParser(),
 	NewItalicParser(),
 	NewCodeParser(),
+	NewMathParser(),
 	NewTagParser(),
 	NewStrikethroughParser(),
 	NewLineBreakParser(),

plugin/gomark/parser/tokenizer/tokenizer.go

@@ -18,6 +18,7 @@ const (
 	Dot                TokenType = "."
 	LessThan           TokenType = "<"
 	GreaterThan        TokenType = ">"
+	DollarSign         TokenType = "$"
 	Backslash          TokenType = "\\"
 	Newline            TokenType = "\n"
 	Space              TokenType = " "
@@ -74,6 +75,8 @@ func Tokenize(text string) []*Token {
 			tokens = append(tokens, NewToken(PlusSign, "+"))
 		case '.':
 			tokens = append(tokens, NewToken(Dot, "."))
+		case '$':
+			tokens = append(tokens, NewToken(DollarSign, "$"))
 		case '\\':
 			tokens = append(tokens, NewToken(Backslash, `\`))
 		case '\n':

proto/api/v2/markdown_service.proto

@@ -34,17 +34,19 @@ enum NodeType {
   ORDERED_LIST = 7;
   UNORDERED_LIST = 8;
   TASK_LIST = 9;
-  TEXT = 10;
-  BOLD = 11;
-  ITALIC = 12;
-  BOLD_ITALIC = 13;
-  CODE = 14;
-  IMAGE = 15;
-  LINK = 16;
-  AUTO_LINK = 17;
-  TAG = 18;
-  STRIKETHROUGH = 19;
-  ESCAPING_CHARACTER = 20;
+  MATH_BLOCK = 10;
+  TEXT = 11;
+  BOLD = 12;
+  ITALIC = 13;
+  BOLD_ITALIC = 14;
+  CODE = 15;
+  IMAGE = 16;
+  LINK = 17;
+  AUTO_LINK = 18;
+  TAG = 19;
+  STRIKETHROUGH = 20;
+  ESCAPING_CHARACTER = 21;
+  MATH = 22;
 }
 
 message Node {
@@ -59,17 +61,19 @@ message Node {
     OrderedListNode ordered_list_node = 8;
     UnorderedListNode unordered_list_node = 9;
     TaskListNode task_list_node = 10;
-    TextNode text_node = 11;
-    BoldNode bold_node = 12;
-    ItalicNode italic_node = 13;
-    BoldItalicNode bold_italic_node = 14;
-    CodeNode code_node = 15;
-    ImageNode image_node = 16;
-    LinkNode link_node = 17;
-    AutoLinkNode auto_link_node = 18;
-    TagNode tag_node = 19;
-    StrikethroughNode strikethrough_node = 20;
-    EscapingCharacterNode escaping_character_node = 21;
+    MathBlockNode math_block_node = 11;
+    TextNode text_node = 12;
+    BoldNode bold_node = 13;
+    ItalicNode italic_node = 14;
+    BoldItalicNode bold_italic_node = 15;
+    CodeNode code_node = 16;
+    ImageNode image_node = 17;
+    LinkNode link_node = 18;
+    AutoLinkNode auto_link_node = 19;
+    TagNode tag_node = 20;
+    StrikethroughNode strikethrough_node = 21;
+    EscapingCharacterNode escaping_character_node = 22;
+    MathNode math_node = 23;
   }
 }
 
@@ -113,6 +117,10 @@ message TaskListNode {
   repeated Node children = 3;
 }
 
+message MathBlockNode {
+  string content = 1;
+}
+
 message TextNode {
   string content = 1;
 }
@@ -161,3 +169,7 @@ message StrikethroughNode {
 message EscapingCharacterNode {
   string symbol = 1;
 }
+
+message MathNode {
+  string content = 1;
+}

proto/gen/api/v2/README.md

@@ -79,6 +79,8 @@
     - [ItalicNode](#memos-api-v2-ItalicNode)
     - [LineBreakNode](#memos-api-v2-LineBreakNode)
     - [LinkNode](#memos-api-v2-LinkNode)
+    - [MathBlockNode](#memos-api-v2-MathBlockNode)
+    - [MathNode](#memos-api-v2-MathNode)
     - [Node](#memos-api-v2-Node)
     - [OrderedListNode](#memos-api-v2-OrderedListNode)
     - [ParagraphNode](#memos-api-v2-ParagraphNode)
@@ -1154,6 +1156,36 @@
 
 
 
+<a name="memos-api-v2-MathBlockNode"></a>
+
+### MathBlockNode
+
+
+
+| Field | Type | Label | Description |
+| ----- | ---- | ----- | ----------- |
+| content | [string](#string) |  |  |
+
+
+
+
+
+
+<a name="memos-api-v2-MathNode"></a>
+
+### MathNode
+
+
+
+| Field | Type | Label | Description |
+| ----- | ---- | ----- | ----------- |
+| content | [string](#string) |  |  |
+
+
+
+
+
+
 <a name="memos-api-v2-Node"></a>
 
 ### Node
@@ -1172,6 +1204,7 @@
 | ordered_list_node | [OrderedListNode](#memos-api-v2-OrderedListNode) |  |  |
 | unordered_list_node | [UnorderedListNode](#memos-api-v2-UnorderedListNode) |  |  |
 | task_list_node | [TaskListNode](#memos-api-v2-TaskListNode) |  |  |
+| math_block_node | [MathBlockNode](#memos-api-v2-MathBlockNode) |  |  |
 | text_node | [TextNode](#memos-api-v2-TextNode) |  |  |
 | bold_node | [BoldNode](#memos-api-v2-BoldNode) |  |  |
 | italic_node | [ItalicNode](#memos-api-v2-ItalicNode) |  |  |
@@ -1183,6 +1216,7 @@
 | tag_node | [TagNode](#memos-api-v2-TagNode) |  |  |
 | strikethrough_node | [StrikethroughNode](#memos-api-v2-StrikethroughNode) |  |  |
 | escaping_character_node | [EscapingCharacterNode](#memos-api-v2-EscapingCharacterNode) |  |  |
+| math_node | [MathNode](#memos-api-v2-MathNode) |  |  |
 
 
 
@@ -1347,17 +1381,19 @@
 | ORDERED_LIST | 7 |  |
 | UNORDERED_LIST | 8 |  |
 | TASK_LIST | 9 |  |
-| TEXT | 10 |  |
-| BOLD | 11 |  |
-| ITALIC | 12 |  |
-| BOLD_ITALIC | 13 |  |
-| CODE | 14 |  |
-| IMAGE | 15 |  |
-| LINK | 16 |  |
-| AUTO_LINK | 17 |  |
-| TAG | 18 |  |
-| STRIKETHROUGH | 19 |  |
-| ESCAPING_CHARACTER | 20 |  |
+| MATH_BLOCK | 10 |  |
+| TEXT | 11 |  |
+| BOLD | 12 |  |
+| ITALIC | 13 |  |
+| BOLD_ITALIC | 14 |  |
+| CODE | 15 |  |
+| IMAGE | 16 |  |
+| LINK | 17 |  |
+| AUTO_LINK | 18 |  |
+| TAG | 19 |  |
+| STRIKETHROUGH | 20 |  |
+| ESCAPING_CHARACTER | 21 |  |
+| MATH | 22 |  |
 
 
  

web/package.json

@@ -12,6 +12,7 @@
   "dependencies": {
     "@emotion/react": "^11.11.3",
     "@emotion/styled": "^11.11.0",
+    "@matejmazur/react-katex": "^3.1.3",
     "@mui/joy": "5.0.0-beta.20",
     "@reduxjs/toolkit": "^1.9.7",
     "axios": "^1.6.3",
@@ -20,6 +21,7 @@
     "highlight.js": "^11.9.0",
     "i18next": "^21.10.0",
     "i18next-browser-languagedetector": "^7.2.0",
+    "katex": "^0.16.9",
     "lodash-es": "^4.17.21",
     "long": "^5.2.3",
     "lucide-react": "^0.263.1",

web/pnpm-lock.yaml

@@ -14,6 +14,9 @@ dependencies:
   '@emotion/styled':
     specifier: ^11.11.0
     version: 11.11.0(@emotion/react@11.11.3)(@types/react@18.2.45)(react@18.2.0)
+  '@matejmazur/react-katex':
+    specifier: ^3.1.3
+    version: 3.1.3(katex@0.16.9)(react@18.2.0)
   '@mui/joy':
     specifier: 5.0.0-beta.20
     version: 5.0.0-beta.20(@emotion/react@11.11.3)(@emotion/styled@11.11.0)(@types/react@18.2.45)(react-dom@18.2.0)(react@18.2.0)
@@ -38,6 +41,9 @@ dependencies:
   i18next-browser-languagedetector:
     specifier: ^7.2.0
     version: 7.2.0
+  katex:
+    specifier: ^0.16.9
+    version: 0.16.9
   lodash-es:
     specifier: ^4.17.21
     version: 4.17.21
@@ -923,6 +929,17 @@ packages:
       '@jridgewell/resolve-uri': 3.1.1
       '@jridgewell/sourcemap-codec': 1.4.15
 
+  /@matejmazur/react-katex@3.1.3(katex@0.16.9)(react@18.2.0):
+    resolution: {integrity: sha512-rBp7mJ9An7ktNoU653BWOYdO4FoR4YNwofHZi+vaytX/nWbIlmHVIF+X8VFOn6c3WYmrLT5FFBjKqCZ1sjR5uQ==}
+    engines: {node: '>=12', yarn: '>=1.1'}
+    peerDependencies:
+      katex: '>=0.9'
+      react: '>=16'
+    dependencies:
+      katex: 0.16.9
+      react: 18.2.0
+    dev: false
+
   /@mui/base@5.0.0-beta.29(@types/react@18.2.45)(react-dom@18.2.0)(react@18.2.0):
     resolution: {integrity: sha512-OXfUssYrB6ch/xpBVHMKAjThPlI9VyGGKdvQLMXef2j39wXfcxPlUVQlwia/lmE3rxWIGvbwkZsDtNYzLMsDUg==}
     engines: {node: '>=12.0.0'}
@@ -2012,6 +2029,11 @@ packages:
     engines: {node: '>= 6'}
     dev: false
 
+  /commander@8.3.0:
+    resolution: {integrity: sha512-OkTL9umf+He2DZkUq8f8J9of7yL6RJKI24dVITBmNfZBmri9zYZQrKkuXiKhyfPSu8tUhnVBB1iKXevvnlR4Ww==}
+    engines: {node: '>= 12'}
+    dev: false
+
   /concat-map@0.0.1:
     resolution: {integrity: sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==}
     dev: true
@@ -3124,6 +3146,13 @@ packages:
       object.values: 1.1.7
     dev: true
 
+  /katex@0.16.9:
+    resolution: {integrity: sha512-fsSYjWS0EEOwvy81j3vRA8TEAhQhKiqO+FQaKWp0m39qwOzHVBgAUBIXWj1pB+O2W3fIpNa6Y9KSKCVbfPhyAQ==}
+    hasBin: true
+    dependencies:
+      commander: 8.3.0
+    dev: false
+
   /keyv@4.5.4:
     resolution: {integrity: sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==}
     dependencies:

web/src/components/MemoContent/Math.tsx

+import TeX from "@matejmazur/react-katex";
+import "katex/dist/katex.min.css";
+
+interface Props {
+  content: string;
+  block?: boolean;
+}
+
+const Math: React.FC<Props> = ({ content, block }: Props) => {
+  return <TeX block={block} math={content}></TeX>;
+};
+
+export default Math;

web/src/components/MemoContent/Renderer.tsx

@@ -11,6 +11,7 @@ import {
   ImageNode,
   ItalicNode,
   LinkNode,
+  MathNode,
   Node,
   NodeType,
   OrderedListNode,
@@ -34,6 +35,7 @@ import Image from "./Image";
 import Italic from "./Italic";
 import LineBreak from "./LineBreak";
 import Link from "./Link";
+import Math from "./Math";
 import OrderedList from "./OrderedList";
 import Paragraph from "./Paragraph";
 import Strikethrough from "./Strikethrough";
@@ -66,6 +68,8 @@ const Renderer: React.FC<Props> = ({ node }: Props) => {
       return <UnorderedList {...(node.unorderedListNode as UnorderedListNode)} />;
     case NodeType.TASK_LIST:
       return <TaskList {...(node.taskListNode as TaskListNode)} />;
+    case NodeType.MATH_BLOCK:
+      return <Math {...(node.mathBlockNode as MathNode)} block={true} />;
     case NodeType.TEXT:
       return <Text {...(node.textNode as TextNode)} />;
     case NodeType.BOLD:
@@ -86,6 +90,8 @@ const Renderer: React.FC<Props> = ({ node }: Props) => {
       return <Tag {...(node.tagNode as TagNode)} />;
     case NodeType.STRIKETHROUGH:
       return <Strikethrough {...(node.strikethroughNode as StrikethroughNode)} />;
+    case NodeType.MATH:
+      return <Math {...(node.mathNode as MathNode)} />;
     case NodeType.ESCAPING_CHARACTER:
       return <EscapingCharacter {...(node.escapingCharacterNode as EscapingCharacterNode)} />;
     default: