From 3f4b361fad1b5e1088fb7f9c26dc04af480e2067 Mon Sep 17 00:00:00 2001 From: Steven Date: Mon, 15 Jan 2024 22:30:06 +0800 Subject: [PATCH] feat: implement highlight parser --- plugin/gomark/ast/ast.go | 1 + plugin/gomark/ast/inline.go | 14 +++++ plugin/gomark/parser/highlight.go | 58 +++++++++++++++++++++ plugin/gomark/parser/highlight_test.go | 41 +++++++++++++++ plugin/gomark/parser/parser.go | 1 + plugin/gomark/parser/tokenizer/tokenizer.go | 3 ++ 6 files changed, 118 insertions(+) create mode 100644 plugin/gomark/parser/highlight.go create mode 100644 plugin/gomark/parser/highlight_test.go diff --git a/plugin/gomark/ast/ast.go b/plugin/gomark/ast/ast.go index 9bf2fba7..fdcdd9ae 100644 --- a/plugin/gomark/ast/ast.go +++ b/plugin/gomark/ast/ast.go @@ -28,6 +28,7 @@ const ( StrikethroughNode EscapingCharacterNode MathNode + HighlightNode ) type Node interface { diff --git a/plugin/gomark/ast/inline.go b/plugin/gomark/ast/inline.go index 075c6b6d..0631c747 100644 --- a/plugin/gomark/ast/inline.go +++ b/plugin/gomark/ast/inline.go @@ -191,3 +191,17 @@ func (*Math) Type() NodeType { func (n *Math) Restore() string { return fmt.Sprintf("$%s$", n.Content) } + +type Highlight struct { + BaseInline + + Content string +} + +func (*Highlight) Type() NodeType { + return HighlightNode +} + +func (n *Highlight) Restore() string { + return fmt.Sprintf("==%s==", n.Content) +} diff --git a/plugin/gomark/parser/highlight.go b/plugin/gomark/parser/highlight.go new file mode 100644 index 00000000..37bebbf5 --- /dev/null +++ b/plugin/gomark/parser/highlight.go @@ -0,0 +1,58 @@ +package parser + +import ( + "errors" + + "github.com/usememos/memos/plugin/gomark/ast" + "github.com/usememos/memos/plugin/gomark/parser/tokenizer" +) + +type HighlightParser struct{} + +func NewHighlightParser() InlineParser { + return &HighlightParser{} +} + +func (*HighlightParser) Match(tokens []*tokenizer.Token) (int, bool) { + if len(tokens) < 5 { + return 0, false + } + + prefixTokens := tokens[:2] + if prefixTokens[0].Type != prefixTokens[1].Type { + return 0, false + } + prefixTokenType := prefixTokens[0].Type + if prefixTokenType != tokenizer.EqualSign { + return 0, false + } + + cursor, matched := 2, false + for ; cursor < len(tokens)-1; cursor++ { + token, nextToken := tokens[cursor], tokens[cursor+1] + if token.Type == tokenizer.Newline || nextToken.Type == tokenizer.Newline { + return 0, false + } + if token.Type == prefixTokenType && nextToken.Type == prefixTokenType { + matched = true + break + } + } + if !matched { + return 0, false + } + + return cursor + 2, true +} + +func (p *HighlightParser) Parse(tokens []*tokenizer.Token) (ast.Node, error) { + size, ok := p.Match(tokens) + if size == 0 || !ok { + return nil, errors.New("not matched") + } + + contentTokens := tokens[2 : size-2] + return &ast.Highlight{ + Content: tokenizer.Stringify(contentTokens), + }, nil +} diff --git a/plugin/gomark/parser/highlight_test.go b/plugin/gomark/parser/highlight_test.go new file mode 100644 index 00000000..346b6844 --- /dev/null +++ b/plugin/gomark/parser/highlight_test.go @@ -0,0 +1,41 @@ +package parser + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/usememos/memos/plugin/gomark/ast" + "github.com/usememos/memos/plugin/gomark/parser/tokenizer" + "github.com/usememos/memos/plugin/gomark/restore" +) + +func TestHighlightParser(t *testing.T) { + tests := []struct { + text string + bold ast.Node + }{ + { + text: "==Hello world!", + bold: nil, + }, + { + text: "==Hello==", + bold: &ast.Highlight{ + Content: "Hello", + }, + }, + { + text: "==Hello world==", + bold: &ast.Highlight{ + Content: "Hello world", + }, + }, + } + + for _, test := range tests { + tokens := tokenizer.Tokenize(test.text) + node, _ := NewHighlightParser().Parse(tokens) + require.Equal(t, restore.Restore([]ast.Node{test.bold}), restore.Restore([]ast.Node{node})) + } +} diff --git a/plugin/gomark/parser/parser.go b/plugin/gomark/parser/parser.go index bf67a46f..4f1de8a7 100644 --- a/plugin/gomark/parser/parser.go +++ b/plugin/gomark/parser/parser.go @@ -80,6 +80,7 @@ var defaultInlineParsers = []InlineParser{ NewAutoLinkParser(), NewBoldParser(), NewItalicParser(), + NewHighlightParser(), NewCodeParser(), NewMathParser(), NewTagParser(), diff --git a/plugin/gomark/parser/tokenizer/tokenizer.go b/plugin/gomark/parser/tokenizer/tokenizer.go index 8322a23a..56435554 100644 --- a/plugin/gomark/parser/tokenizer/tokenizer.go +++ b/plugin/gomark/parser/tokenizer/tokenizer.go @@ -19,6 +19,7 @@ const ( LessThan TokenType = "<" GreaterThan TokenType = ">" DollarSign TokenType = "$" + EqualSign TokenType = "=" Backslash TokenType = "\\" Newline TokenType = "\n" Space TokenType = " " @@ -77,6 +78,8 @@ func Tokenize(text string) []*Token { tokens = append(tokens, NewToken(Dot, ".")) case '$': tokens = append(tokens, NewToken(DollarSign, "$")) + case '=': + tokens = append(tokens, NewToken(EqualSign, "=")) case '\\': tokens = append(tokens, NewToken(Backslash, `\`)) case '\n':