chore: implement list nodes

plugin/gomark/ast/block.go

@@ -64,3 +64,26 @@ type Blockquote struct {
 func (*Blockquote) Type() NodeType {
 	return BlockquoteNode
 }
+
+type OrderedList struct {
+	BaseBlock
+
+	Number   string
+	Children []Node
+}
+
+func (*OrderedList) Type() NodeType {
+	return OrderedListNode
+}
+
+type UnorderedList struct {
+	BaseBlock
+
+	// Symbol is "*" or "-" or "+".
+	Symbol   string
+	Children []Node
+}
+
+func (*UnorderedList) Type() NodeType {
+	return UnorderedListNode
+}

plugin/gomark/ast/node.go

@@ -11,6 +11,8 @@ const (
 	HeadingNode
 	HorizontalRuleNode
 	BlockquoteNode
+	OrderedListNode
+	UnorderedListNode
 	// Inline nodes.
 	TextNode
 	BoldNode

plugin/gomark/parser/bold.go

@@ -23,7 +23,7 @@ func (*BoldParser) Match(tokens []*tokenizer.Token) (int, bool) {
 		return 0, false
 	}
 	prefixTokenType := prefixTokens[0].Type
-	if prefixTokenType != tokenizer.Asterisk && prefixTokenType != tokenizer.Underline {
+	if prefixTokenType != tokenizer.Asterisk && prefixTokenType != tokenizer.Underscore {
 		return 0, false
 	}
 

plugin/gomark/parser/bold_italic.go

@@ -23,7 +23,7 @@ func (*BoldItalicParser) Match(tokens []*tokenizer.Token) (int, bool) {
 		return 0, false
 	}
 	prefixTokenType := prefixTokens[0].Type
-	if prefixTokenType != tokenizer.Asterisk && prefixTokenType != tokenizer.Underline {
+	if prefixTokenType != tokenizer.Asterisk && prefixTokenType != tokenizer.Underscore {
 		return 0, false
 	}
 

plugin/gomark/parser/heading.go

@@ -16,7 +16,7 @@ func NewHeadingParser() *HeadingParser {
 func (*HeadingParser) Match(tokens []*tokenizer.Token) (int, bool) {
 	cursor := 0
 	for _, token := range tokens {
-		if token.Type == tokenizer.Hash {
+		if token.Type == tokenizer.PoundSign {
 			cursor++
 		} else {
 			break
@@ -57,7 +57,7 @@ func (p *HeadingParser) Parse(tokens []*tokenizer.Token) (ast.Node, error) {
 
 	level := 0
 	for _, token := range tokens {
-		if token.Type == tokenizer.Hash {
+		if token.Type == tokenizer.PoundSign {
 			level++
 		} else {
 			break

plugin/gomark/parser/horizontal_rule.go

@@ -20,7 +20,7 @@ func (*HorizontalRuleParser) Match(tokens []*tokenizer.Token) (int, bool) {
 	if tokens[0].Type != tokens[1].Type || tokens[0].Type != tokens[2].Type || tokens[1].Type != tokens[2].Type {
 		return 0, false
 	}
-	if tokens[0].Type != tokenizer.Dash && tokens[0].Type != tokenizer.Underline && tokens[0].Type != tokenizer.Asterisk {
+	if tokens[0].Type != tokenizer.Hyphen && tokens[0].Type != tokenizer.Underscore && tokens[0].Type != tokenizer.Asterisk {
 		return 0, false
 	}
 	if len(tokens) > 3 && tokens[3].Type != tokenizer.Newline {

plugin/gomark/parser/italic.go

@@ -21,7 +21,7 @@ func (*ItalicParser) Match(tokens []*tokenizer.Token) (int, bool) {
 	}
 
 	prefixTokens := tokens[:1]
-	if prefixTokens[0].Type != tokenizer.Asterisk && prefixTokens[0].Type != tokenizer.Underline {
+	if prefixTokens[0].Type != tokenizer.Asterisk && prefixTokens[0].Type != tokenizer.Underscore {
 		return 0, false
 	}
 	prefixTokenType := prefixTokens[0].Type

plugin/gomark/parser/ordered_list.go

+package parser
+
+import (
+	"errors"
+
+	"github.com/usememos/memos/plugin/gomark/ast"
+	"github.com/usememos/memos/plugin/gomark/parser/tokenizer"
+)
+
+type OrderedListParser struct{}
+
+func NewOrderedListParser() *OrderedListParser {
+	return &OrderedListParser{}
+}
+
+func (*OrderedListParser) Match(tokens []*tokenizer.Token) (int, bool) {
+	if len(tokens) < 4 {
+		return 0, false
+	}
+	if tokens[0].Type != tokenizer.Number || tokens[1].Type != tokenizer.Dot || tokens[2].Type != tokenizer.Space {
+		return 0, false
+	}
+
+	contentTokens := []*tokenizer.Token{}
+	for _, token := range tokens[3:] {
+		contentTokens = append(contentTokens, token)
+		if token.Type == tokenizer.Newline {
+			break
+		}
+	}
+
+	if len(contentTokens) == 0 {
+		return 0, false
+	}
+
+	return len(contentTokens) + 3, true
+}
+
+func (p *OrderedListParser) Parse(tokens []*tokenizer.Token) (ast.Node, error) {
+	size, ok := p.Match(tokens)
+	if size == 0 || !ok {
+		return nil, errors.New("not matched")
+	}
+
+	contentTokens := tokens[3:size]
+	children, err := ParseInline(contentTokens)
+	if err != nil {
+		return nil, err
+	}
+	return &ast.OrderedList{
+		Number:   tokens[0].Value,
+		Children: children,
+	}, nil
+}

plugin/gomark/parser/ordered_list_test.go

+package parser
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/usememos/memos/plugin/gomark/ast"
+	"github.com/usememos/memos/plugin/gomark/parser/tokenizer"
+)
+
+func TestOrderedListParser(t *testing.T) {
+	tests := []struct {
+		text string
+		node ast.Node
+	}{
+		{
+			text: "1.asd",
+			node: nil,
+		},
+		{
+			text: "1. Hello World",
+			node: &ast.OrderedList{
+				Number: "1",
+				Children: []ast.Node{
+					&ast.Text{
+						Content: "Hello World",
+					},
+				},
+			},
+		},
+		{
+			text: "1aa. Hello World",
+			node: nil,
+		},
+		{
+			text: "22. Hello *World*",
+			node: &ast.OrderedList{
+				Number: "22",
+				Children: []ast.Node{
+					&ast.Text{
+						Content: "Hello ",
+					},
+					&ast.Italic{
+						Symbol:  "*",
+						Content: "World",
+					},
+				},
+			},
+		},
+	}
+
+	for _, test := range tests {
+		tokens := tokenizer.Tokenize(test.text)
+		node, _ := NewOrderedListParser().Parse(tokens)
+		require.Equal(t, StringifyNodes([]ast.Node{test.node}), StringifyNodes([]ast.Node{node}))
+	}
+}

plugin/gomark/parser/tag.go

@@ -17,12 +17,12 @@ func (*TagParser) Match(tokens []*tokenizer.Token) (int, bool) {
 	if len(tokens) < 2 {
 		return 0, false
 	}
-	if tokens[0].Type != tokenizer.Hash {
+	if tokens[0].Type != tokenizer.PoundSign {
 		return 0, false
 	}
 	contentTokens := []*tokenizer.Token{}
 	for _, token := range tokens[1:] {
-		if token.Type == tokenizer.Newline || token.Type == tokenizer.Space || token.Type == tokenizer.Hash {
+		if token.Type == tokenizer.Newline || token.Type == tokenizer.Space || token.Type == tokenizer.PoundSign {
 			break
 		}
 		contentTokens = append(contentTokens, token)

plugin/gomark/parser/tokenizer/tokenizer.go

@@ -3,9 +3,9 @@ package tokenizer
 type TokenType = string
 
 const (
-	Underline          TokenType = "_"
+	Underscore         TokenType = "_"
 	Asterisk           TokenType = "*"
-	Hash               TokenType = "#"
+	PoundSign          TokenType = "#"
 	Backtick           TokenType = "`"
 	LeftSquareBracket  TokenType = "["
 	RightSquareBracket TokenType = "]"
@@ -13,13 +13,16 @@ const (
 	RightParenthesis   TokenType = ")"
 	ExclamationMark    TokenType = "!"
 	Tilde              TokenType = "~"
-	Dash               TokenType = "-"
+	Hyphen             TokenType = "-"
+	PlusSign           TokenType = "+"
+	Dot                TokenType = "."
 	GreaterThan        TokenType = ">"
 	Newline            TokenType = "\n"
 	Space              TokenType = " "
 )
 
 const (
+	Number TokenType = "number"
 	Text   TokenType = ""
 )
 
@@ -40,11 +43,11 @@ func Tokenize(text string) []*Token {
 	for _, c := range text {
 		switch c {
 		case '_':
-			tokens = append(tokens, NewToken(Underline, "_"))
+			tokens = append(tokens, NewToken(Underscore, "_"))
 		case '*':
 			tokens = append(tokens, NewToken(Asterisk, "*"))
 		case '#':
-			tokens = append(tokens, NewToken(Hash, "#"))
+			tokens = append(tokens, NewToken(PoundSign, "#"))
 		case '`':
 			tokens = append(tokens, NewToken(Backtick, "`"))
 		case '[':
@@ -60,9 +63,13 @@ func Tokenize(text string) []*Token {
 		case '~':
 			tokens = append(tokens, NewToken(Tilde, "~"))
 		case '-':
-			tokens = append(tokens, NewToken(Dash, "-"))
+			tokens = append(tokens, NewToken(Hyphen, "-"))
 		case '>':
 			tokens = append(tokens, NewToken(GreaterThan, ">"))
+		case '+':
+			tokens = append(tokens, NewToken(PlusSign, "+"))
+		case '.':
+			tokens = append(tokens, NewToken(Dot, "."))
 		case '\n':
 			tokens = append(tokens, NewToken(Newline, "\n"))
 		case ' ':
@@ -72,10 +79,19 @@ func Tokenize(text string) []*Token {
 			if len(tokens) > 0 {
 				prevToken = tokens[len(tokens)-1]
 			}
-			if prevToken == nil || prevToken.Type != Text {
-				tokens = append(tokens, NewToken(Text, string(c)))
-			} else {
+
+			isNumber := c >= '0' && c <= '9'
+			if prevToken != nil {
+				if (prevToken.Type == Text && !isNumber) || (prevToken.Type == Number && isNumber) {
 					prevToken.Value += string(c)
+					continue
+				}
+			}
+
+			if isNumber {
+				tokens = append(tokens, NewToken(Number, string(c)))
+			} else {
+				tokens = append(tokens, NewToken(Text, string(c)))
 			}
 		}
 	}

plugin/gomark/parser/tokenizer/tokenizer_test.go

@@ -41,7 +41,7 @@ func TestTokenize(t *testing.T) {
  world`,
 			tokens: []*Token{
 				{
-					Type:  Hash,
+					Type:  PoundSign,
 					Value: "#",
 				},
 				{

plugin/gomark/parser/unordered_list.go

+package parser
+
+import (
+	"errors"
+
+	"github.com/usememos/memos/plugin/gomark/ast"
+	"github.com/usememos/memos/plugin/gomark/parser/tokenizer"
+)
+
+type UnorderedListParser struct{}
+
+func NewUnorderedListParser() *UnorderedListParser {
+	return &UnorderedListParser{}
+}
+
+func (*UnorderedListParser) Match(tokens []*tokenizer.Token) (int, bool) {
+	if len(tokens) < 3 {
+		return 0, false
+	}
+	symbolToken := tokens[0]
+	if (symbolToken.Type != tokenizer.Hyphen && symbolToken.Type != tokenizer.Asterisk && symbolToken.Type != tokenizer.PlusSign) || tokens[1].Type != tokenizer.Space {
+		return 0, false
+	}
+
+	contentTokens := []*tokenizer.Token{}
+	for _, token := range tokens[2:] {
+		if token.Type == tokenizer.Newline {
+			break
+		}
+		contentTokens = append(contentTokens, token)
+	}
+	if len(contentTokens) == 0 {
+		return 0, false
+	}
+
+	return len(contentTokens) + 2, true
+}
+
+func (p *UnorderedListParser) Parse(tokens []*tokenizer.Token) (ast.Node, error) {
+	size, ok := p.Match(tokens)
+	if size == 0 || !ok {
+		return nil, errors.New("not matched")
+	}
+
+	symbolToken := tokens[0]
+	contentTokens := tokens[2:size]
+	children, err := ParseInline(contentTokens)
+	if err != nil {
+		return nil, err
+	}
+	return &ast.UnorderedList{
+		Symbol:   symbolToken.Type,
+		Children: children,
+	}, nil
+}

plugin/gomark/parser/unordered_list_test.go

+package parser
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/usememos/memos/plugin/gomark/ast"
+	"github.com/usememos/memos/plugin/gomark/parser/tokenizer"
+)
+
+func TestUnorderedListParser(t *testing.T) {
+	tests := []struct {
+		text string
+		node ast.Node
+	}{
+		{
+			text: "*asd",
+			node: nil,
+		},
+		{
+			text: "+ Hello World",
+			node: &ast.UnorderedList{
+				Symbol: tokenizer.PlusSign,
+				Children: []ast.Node{
+					&ast.Text{
+						Content: "Hello World",
+					},
+				},
+			},
+		},
+		{
+			text: "* **Hello**",
+			node: &ast.UnorderedList{
+				Symbol: tokenizer.Asterisk,
+				Children: []ast.Node{
+					&ast.Bold{
+						Symbol:  "*",
+						Content: "Hello",
+					},
+				},
+			},
+		},
+	}
+
+	for _, test := range tests {
+		tokens := tokenizer.Tokenize(test.text)
+		node, _ := NewUnorderedListParser().Parse(tokens)
+		require.Equal(t, StringifyNodes([]ast.Node{test.node}), StringifyNodes([]ast.Node{node}))
+	}
+}

plugin/gomark/render/html/html.go

@@ -159,7 +159,7 @@ func (r *HTMLRender) renderLink(node *ast.Link) {
 
 func (r *HTMLRender) renderTag(node *ast.Tag) {
 	r.output.WriteString(`<span>`)
-	r.output.WriteString(`# `)
+	r.output.WriteString(`#`)
 	r.output.WriteString(node.Content)
 	r.output.WriteString(`</span>`)
 }

plugin/gomark/render/html/html_test.go

@@ -30,6 +30,10 @@ func TestHTMLRender(t *testing.T) {
 			text:     "**Hello** world!",
 			expected: `<p><strong>Hello</strong> world!</p>`,
 		},
+		{
+			text:     "#article #memo",
+			expected: `<p><span>#article</span> <span>#memo</span></p>`,
+		},
 	}
 
 	for _, test := range tests {