feat: image and link parser (#1744)

* feat: image and link parser

* chore: update

plugin/gomark/parser/heading_test.go

@@ -13,11 +13,11 @@ func TestHeadingParser(t *testing.T) {
 		heading *HeadingParser
 	}{
 		{
-			text:    "*Hello world!",
+			text:    "*Hello world",
 			heading: nil,
 		},
 		{
-			text: "## Hello World!",
+			text: "## Hello World",
 			heading: &HeadingParser{
 				Level: 2,
 				ContentTokens: []*tokenizer.Token{
@@ -31,7 +31,7 @@ func TestHeadingParser(t *testing.T) {
 					},
 					{
 						Type:  tokenizer.Text,
-						Value: "World!",
+						Value: "World",
 					},
 				},
 			},
@@ -65,12 +65,12 @@ func TestHeadingParser(t *testing.T) {
 			},
 		},
 		{
-			text:    " # 123123 Hello World!",
+			text:    " # 123123 Hello World",
 			heading: nil,
 		},
 		{
 			text: `# 123 
-Hello World!`,
+Hello World`,
 			heading: &HeadingParser{
 				Level: 1,
 				ContentTokens: []*tokenizer.Token{

plugin/gomark/parser/image.go

+package parser
+
+import "github.com/usememos/memos/plugin/gomark/parser/tokenizer"
+
+type ImageParser struct {
+	AltText string
+	URL     string
+}
+
+func NewImageParser() *ImageParser {
+	return &ImageParser{}
+}
+
+func (*ImageParser) Match(tokens []*tokenizer.Token) *ImageParser {
+	if len(tokens) < 5 {
+		return nil
+	}
+	if tokens[0].Type != tokenizer.ExclamationMark {
+		return nil
+	}
+	if tokens[1].Type != tokenizer.LeftSquareBracket {
+		return nil
+	}
+	cursor, altText := 2, ""
+	for ; cursor < len(tokens)-2; cursor++ {
+		if tokens[cursor].Type == tokenizer.Newline {
+			return nil
+		}
+		if tokens[cursor].Type == tokenizer.RightSquareBracket {
+			break
+		}
+		altText += tokens[cursor].Value
+	}
+	if tokens[cursor+1].Type != tokenizer.LeftParenthesis {
+		return nil
+	}
+	matched, url := false, ""
+	for _, token := range tokens[cursor+2:] {
+		if token.Type == tokenizer.Newline || token.Type == tokenizer.Space {
+			return nil
+		}
+		if token.Type == tokenizer.RightParenthesis {
+			matched = true
+			break
+		}
+		url += token.Value
+	}
+	if !matched || url == "" {
+		return nil
+	}
+	return &ImageParser{
+		AltText: altText,
+		URL:     url,
+	}
+}

plugin/gomark/parser/image_test.go

+package parser
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"github.com/usememos/memos/plugin/gomark/parser/tokenizer"
+)
+
+func TestImageParser(t *testing.T) {
+	tests := []struct {
+		text  string
+		image *ImageParser
+	}{
+		{
+			text: "![](https://example.com)",
+			image: &ImageParser{
+				AltText: "",
+				URL:     "https://example.com",
+			},
+		},
+		{
+			text:  "! [](https://example.com)",
+			image: nil,
+		},
+		{
+			text:  "![alte]( htt ps :/ /example.com)",
+			image: nil,
+		},
+		{
+			text: "![al te](https://example.com)",
+			image: &ImageParser{
+				AltText: "al te",
+				URL:     "https://example.com",
+			},
+		},
+	}
+	for _, test := range tests {
+		tokens := tokenizer.Tokenize(test.text)
+		require.Equal(t, test.image, NewImageParser().Match(tokens))
+	}
+}

plugin/gomark/parser/link.go

+package parser
+
+import "github.com/usememos/memos/plugin/gomark/parser/tokenizer"
+
+type LinkParser struct {
+	ContentTokens []*tokenizer.Token
+	URL           string
+}
+
+func NewLinkParser() *LinkParser {
+	return &LinkParser{}
+}
+
+func (*LinkParser) Match(tokens []*tokenizer.Token) *LinkParser {
+	if len(tokens) < 4 {
+		return nil
+	}
+	if tokens[0].Type != tokenizer.LeftSquareBracket {
+		return nil
+	}
+	cursor, contentTokens := 1, []*tokenizer.Token{}
+	for ; cursor < len(tokens)-2; cursor++ {
+		if tokens[cursor].Type == tokenizer.Newline {
+			return nil
+		}
+		if tokens[cursor].Type == tokenizer.RightSquareBracket {
+			break
+		}
+		contentTokens = append(contentTokens, tokens[cursor])
+	}
+	if tokens[cursor+1].Type != tokenizer.LeftParenthesis {
+		return nil
+	}
+	matched, url := false, ""
+	for _, token := range tokens[cursor+2:] {
+		if token.Type == tokenizer.Newline || token.Type == tokenizer.Space {
+			return nil
+		}
+		if token.Type == tokenizer.RightParenthesis {
+			matched = true
+			break
+		}
+		url += token.Value
+	}
+	if !matched || url == "" {
+		return nil
+	}
+	if len(contentTokens) == 0 {
+		contentTokens = append(contentTokens, &tokenizer.Token{
+			Type:  tokenizer.Text,
+			Value: url,
+		})
+	}
+	return &LinkParser{
+		ContentTokens: contentTokens,
+		URL:           url,
+	}
+}

plugin/gomark/parser/link_test.go

+package parser
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"github.com/usememos/memos/plugin/gomark/parser/tokenizer"
+)
+
+func TestLinkParser(t *testing.T) {
+	tests := []struct {
+		text string
+		link *LinkParser
+	}{
+		{
+			text: "[](https://example.com)",
+			link: &LinkParser{
+				ContentTokens: []*tokenizer.Token{
+					{
+						Type:  tokenizer.Text,
+						Value: "https://example.com",
+					},
+				},
+				URL: "https://example.com",
+			},
+		},
+		{
+			text: "! [](https://example.com)",
+			link: nil,
+		},
+		{
+			text: "[alte]( htt ps :/ /example.com)",
+			link: nil,
+		},
+		{
+			text: "[hello world](https://example.com)",
+			link: &LinkParser{
+				ContentTokens: []*tokenizer.Token{
+					{
+						Type:  tokenizer.Text,
+						Value: "hello",
+					},
+					{
+						Type:  tokenizer.Space,
+						Value: " ",
+					},
+					{
+						Type:  tokenizer.Text,
+						Value: "world",
+					},
+				},
+				URL: "https://example.com",
+			},
+		},
+	}
+	for _, test := range tests {
+		tokens := tokenizer.Tokenize(test.text)
+		require.Equal(t, test.link, NewLinkParser().Match(tokens))
+	}
+}

plugin/gomark/parser/paragraph_test.go

@@ -17,7 +17,7 @@ func TestParagraphParser(t *testing.T) {
 			paragraph: nil,
 		},
 		{
-			text: "Hello world!",
+			text: "Hello world",
 			paragraph: &ParagraphParser{
 				ContentTokens: []*tokenizer.Token{
 					{
@@ -30,14 +30,14 @@ func TestParagraphParser(t *testing.T) {
 					},
 					{
 						Type:  tokenizer.Text,
-						Value: "world!",
+						Value: "world",
 					},
 				},
 			},
 		},
 		{
 			text: `Hello 
-world!`,
+world`,
 			paragraph: &ParagraphParser{
 				ContentTokens: []*tokenizer.Token{
 					{
@@ -53,7 +53,7 @@ world!`,
 		},
 		{
 			text: `Hello \n 
-world!`,
+world`,
 			paragraph: &ParagraphParser{
 				ContentTokens: []*tokenizer.Token{
 					{

plugin/gomark/parser/tokenizer/tokenizer.go

@@ -7,6 +7,11 @@ const (
 	Star               TokenType = "*"
 	Hash               TokenType = "#"
 	Backtick           TokenType = "`"
+	LeftSquareBracket  TokenType = "["
+	RightSquareBracket TokenType = "]"
+	LeftParenthesis    TokenType = "("
+	RightParenthesis   TokenType = ")"
+	ExclamationMark    TokenType = "!"
 	Newline            TokenType = "\n"
 	Space              TokenType = " "
 )
@@ -37,10 +42,20 @@ func Tokenize(text string) []*Token {
 			tokens = append(tokens, NewToken(Star, "*"))
 		case '#':
 			tokens = append(tokens, NewToken(Hash, "#"))
-		case '\n':
-			tokens = append(tokens, NewToken(Newline, "\n"))
 		case '`':
 			tokens = append(tokens, NewToken(Backtick, "`"))
+		case '[':
+			tokens = append(tokens, NewToken(LeftSquareBracket, "["))
+		case ']':
+			tokens = append(tokens, NewToken(RightSquareBracket, "]"))
+		case '(':
+			tokens = append(tokens, NewToken(LeftParenthesis, "("))
+		case ')':
+			tokens = append(tokens, NewToken(RightParenthesis, ")"))
+		case '!':
+			tokens = append(tokens, NewToken(ExclamationMark, "!"))
+		case '\n':
+			tokens = append(tokens, NewToken(Newline, "\n"))
 		case ' ':
 			tokens = append(tokens, NewToken(Space, " "))
 		default:

plugin/gomark/parser/tokenizer/tokenizer_test.go

@@ -28,7 +28,11 @@ func TestTokenize(t *testing.T) {
 				},
 				{
 					Type:  Text,
-					Value: "world!",
+					Value: "world",
+				},
+				{
+					Type:  ExclamationMark,
+					Value: "!",
 				},
 			},
 		},