From 2d733288bc78e37c7ccb6ce1e588debb1bdf3912 Mon Sep 17 00:00:00 2001 From: gobwas Date: Fri, 27 May 2016 20:47:19 +0300 Subject: [PATCH 1/3] refactoring --- lexer.go => lexer/lexer.go | 155 +++---------- lexer/lexer_test.go | 192 ++++++++++++++++ lexer/token.go | 88 +++++++ lexer_test.go | 192 ---------------- parser.go | 230 ------------------- parser/ast.go | 48 ++++ parser/parser.go | 144 ++++++++++++ parser/parser_test.go | 332 +++++++++++++++++++++++++++ parser_test.go | 324 -------------------------- {runes => util/runes}/runes.go | 0 {runes => util/runes}/runes_test.go | 0 {strings => util/strings}/strings.go | 0 12 files changed, 832 insertions(+), 873 deletions(-) rename lexer.go => lexer/lexer.go (59%) create mode 100644 lexer/lexer_test.go create mode 100644 lexer/token.go delete mode 100644 lexer_test.go delete mode 100644 parser.go create mode 100644 parser/ast.go create mode 100644 parser/parser.go create mode 100644 parser/parser_test.go delete mode 100644 parser_test.go rename {runes => util/runes}/runes.go (100%) rename {runes => util/runes}/runes_test.go (100%) rename {strings => util/strings}/strings.go (100%) diff --git a/lexer.go b/lexer/lexer.go similarity index 59% rename from lexer.go rename to lexer/lexer.go index 9b756dc..e074b09 100644 --- a/lexer.go +++ b/lexer/lexer.go @@ -1,9 +1,9 @@ -package glob +package lexer import ( "bytes" "fmt" - "github.com/gobwas/glob/runes" + "github.com/gobwas/glob/util/runes" "unicode/utf8" ) @@ -30,123 +30,24 @@ var specials = []byte{ char_terms_close, } -func special(c byte) bool { +func Special(c byte) bool { return bytes.IndexByte(specials, c) != -1 } -type itemType int +type tokens []Token -const ( - item_eof itemType = iota - item_error - item_text - item_char - item_any - item_super - item_single - item_not - item_separator - item_range_open - item_range_close - item_range_lo - item_range_hi - item_range_between - item_terms_open - item_terms_close -) - -func (i itemType) String() string { - switch i { - case item_eof: - return "eof" - - case item_error: - return "error" - - case item_text: - return "text" - - case item_char: - return "char" - - case item_any: - return "any" - - case item_super: - return "super" - - case item_single: - return "single" - - case item_not: - return "not" - - case item_separator: - return "separator" - - case item_range_open: - return "range_open" - - case item_range_close: - return "range_close" - - case item_range_lo: - return "range_lo" - - case item_range_hi: - return "range_hi" - - case item_range_between: - return "range_between" - - case item_terms_open: - return "terms_open" - - case item_terms_close: - return "terms_close" - - default: - return "undef" - } -} - -type item struct { - t itemType - s string -} - -func (i item) String() string { - return fmt.Sprintf("%v<%q>", i.t, i.s) -} - -type stubLexer struct { - Items []item - pos int -} - -func (s *stubLexer) nextItem() (ret item) { - if s.pos == len(s.Items) { - return item{item_eof, ""} - } - ret = s.Items[s.pos] - s.pos++ - return -} - -type items []item - -func (i *items) shift() (ret item) { +func (i *tokens) shift() (ret Token) { ret = (*i)[0] copy(*i, (*i)[1:]) *i = (*i)[:len(*i)-1] return } -func (i *items) push(v item) { +func (i *tokens) push(v Token) { *i = append(*i, v) } -func (i *items) empty() bool { +func (i *tokens) empty() bool { return len(*i) == 0 } @@ -157,7 +58,7 @@ type lexer struct { pos int err error - items items + tokens tokens termsLevel int lastRune rune @@ -167,8 +68,8 @@ type lexer struct { func newLexer(source string) *lexer { l := &lexer{ - data: source, - items: items(make([]item, 0, 4)), + data: source, + tokens: tokens(make([]Token, 0, 4)), } return l } @@ -233,12 +134,12 @@ func (l *lexer) termsLeave() { l.termsLevel-- } -func (l *lexer) nextItem() item { +func (l *lexer) nextItem() Token { if l.err != nil { - return item{item_error, l.err.Error()} + return Token{Error, l.err.Error()} } - if !l.items.empty() { - return l.items.shift() + if !l.tokens.empty() { + return l.tokens.shift() } l.fetchItem() @@ -252,32 +153,32 @@ func (l *lexer) fetchItem() { r := l.read() switch { case r == eof: - l.items.push(item{item_eof, ""}) + l.tokens.push(Token{EOF, ""}) case r == char_terms_open: l.termsEnter() - l.items.push(item{item_terms_open, string(r)}) + l.tokens.push(Token{TermsOpen, string(r)}) case r == char_comma && l.inTerms(): - l.items.push(item{item_separator, string(r)}) + l.tokens.push(Token{Separator, string(r)}) case r == char_terms_close && l.inTerms(): - l.items.push(item{item_terms_close, string(r)}) + l.tokens.push(Token{TermsClose, string(r)}) l.termsLeave() case r == char_range_open: - l.items.push(item{item_range_open, string(r)}) + l.tokens.push(Token{RangeOpen, string(r)}) l.fetchRange() case r == char_single: - l.items.push(item{item_single, string(r)}) + l.tokens.push(Token{Single, string(r)}) case r == char_any: if l.read() == char_any { - l.items.push(item{item_super, string(r) + string(r)}) + l.tokens.push(Token{Super, string(r) + string(r)}) } else { l.unread() - l.items.push(item{item_any, string(r)}) + l.tokens.push(Token{Any, string(r)}) } default: @@ -308,27 +209,27 @@ func (l *lexer) fetchRange() { if r != char_range_close { l.errorf("expected close range character") } else { - l.items.push(item{item_range_close, string(r)}) + l.tokens.push(Token{RangeClose, string(r)}) } return } if wantHi { - l.items.push(item{item_range_hi, string(r)}) + l.tokens.push(Token{RangeHi, string(r)}) wantClose = true continue } if !seenNot && r == char_range_not { - l.items.push(item{item_not, string(r)}) + l.tokens.push(Token{Not, string(r)}) seenNot = true continue } if n, w := l.peek(); n == char_range_between { l.seek(w) - l.items.push(item{item_range_lo, string(r)}) - l.items.push(item{item_range_between, string(n)}) + l.tokens.push(Token{RangeLo, string(r)}) + l.tokens.push(Token{RangeBetween, string(n)}) wantHi = true continue } @@ -367,6 +268,6 @@ reading: } if len(data) > 0 { - l.items.push(item{item_text, string(data)}) + l.tokens.push(Token{Text, string(data)}) } } diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go new file mode 100644 index 0000000..8af3aa6 --- /dev/null +++ b/lexer/lexer_test.go @@ -0,0 +1,192 @@ +package lexer + +import ( + "testing" +) + +func TestLexGood(t *testing.T) { + for id, test := range []struct { + pattern string + items []Token + }{ + { + pattern: "", + items: []Token{ + Token{EOF, ""}, + }, + }, + { + pattern: "hello", + items: []Token{ + Token{Text, "hello"}, + Token{EOF, ""}, + }, + }, + { + pattern: "/{rate,[0-9]]}*", + items: []Token{ + Token{Text, "/"}, + Token{TermsOpen, "{"}, + Token{Text, "rate"}, + Token{Separator, ","}, + Token{RangeOpen, "["}, + Token{RangeLo, "0"}, + Token{RangeBetween, "-"}, + Token{RangeHi, "9"}, + Token{RangeClose, "]"}, + Token{Text, "]"}, + Token{TermsClose, "}"}, + Token{Any, "*"}, + Token{EOF, ""}, + }, + }, + { + pattern: "hello,world", + items: []Token{ + Token{Text, "hello,world"}, + Token{EOF, ""}, + }, + }, + { + pattern: "hello\\,world", + items: []Token{ + Token{Text, "hello,world"}, + Token{EOF, ""}, + }, + }, + { + pattern: "hello\\{world", + items: []Token{ + Token{Text, "hello{world"}, + Token{EOF, ""}, + }, + }, + { + pattern: "hello?", + items: []Token{ + Token{Text, "hello"}, + Token{Single, "?"}, + Token{EOF, ""}, + }, + }, + { + pattern: "hellof*", + items: []Token{ + Token{Text, "hellof"}, + Token{Any, "*"}, + Token{EOF, ""}, + }, + }, + { + pattern: "hello**", + items: []Token{ + Token{Text, "hello"}, + Token{Super, "**"}, + Token{EOF, ""}, + }, + }, + { + pattern: "[日-語]", + items: []Token{ + Token{RangeOpen, "["}, + Token{RangeLo, "日"}, + Token{RangeBetween, "-"}, + Token{RangeHi, "語"}, + Token{RangeClose, "]"}, + Token{EOF, ""}, + }, + }, + { + pattern: "[!日-語]", + items: []Token{ + Token{RangeOpen, "["}, + Token{Not, "!"}, + Token{RangeLo, "日"}, + Token{RangeBetween, "-"}, + Token{RangeHi, "語"}, + Token{RangeClose, "]"}, + Token{EOF, ""}, + }, + }, + { + pattern: "[日本語]", + items: []Token{ + Token{RangeOpen, "["}, + Token{Text, "日本語"}, + Token{RangeClose, "]"}, + Token{EOF, ""}, + }, + }, + { + pattern: "[!日本語]", + items: []Token{ + Token{RangeOpen, "["}, + Token{Not, "!"}, + Token{Text, "日本語"}, + Token{RangeClose, "]"}, + Token{EOF, ""}, + }, + }, + { + pattern: "{a,b}", + items: []Token{ + Token{TermsOpen, "{"}, + Token{Text, "a"}, + Token{Separator, ","}, + Token{Text, "b"}, + Token{TermsClose, "}"}, + Token{EOF, ""}, + }, + }, + { + pattern: "/{z,ab}*", + items: []Token{ + Token{Text, "/"}, + Token{TermsOpen, "{"}, + Token{Text, "z"}, + Token{Separator, ","}, + Token{Text, "ab"}, + Token{TermsClose, "}"}, + Token{Any, "*"}, + Token{EOF, ""}, + }, + }, + { + pattern: "{[!日-語],*,?,{a,b,\\c}}", + items: []Token{ + Token{TermsOpen, "{"}, + Token{RangeOpen, "["}, + Token{Not, "!"}, + Token{RangeLo, "日"}, + Token{RangeBetween, "-"}, + Token{RangeHi, "語"}, + Token{RangeClose, "]"}, + Token{Separator, ","}, + Token{Any, "*"}, + Token{Separator, ","}, + Token{Single, "?"}, + Token{Separator, ","}, + Token{TermsOpen, "{"}, + Token{Text, "a"}, + Token{Separator, ","}, + Token{Text, "b"}, + Token{Separator, ","}, + Token{Text, "c"}, + Token{TermsClose, "}"}, + Token{TermsClose, "}"}, + Token{EOF, ""}, + }, + }, + } { + lexer := newLexer(test.pattern) + for i, exp := range test.items { + act := lexer.nextItem() + if act.Type != exp.Type { + t.Errorf("#%d %q: wrong %d-th item type: exp: %q; act: %q\n\t(%s vs %s)", id, test.pattern, i, exp.Type, act.Type, exp, act) + } + if act.Raw != exp.Raw { + t.Errorf("#%d %q: wrong %d-th item contents: exp: %q; act: %q\n\t(%s vs %s)", id, test.pattern, i, exp.Raw, act.Raw, exp, act) + } + } + } +} diff --git a/lexer/token.go b/lexer/token.go new file mode 100644 index 0000000..2797c4e --- /dev/null +++ b/lexer/token.go @@ -0,0 +1,88 @@ +package lexer + +import "fmt" + +type TokenType int + +const ( + EOF TokenType = iota + Error + Text + Char + Any + Super + Single + Not + Separator + RangeOpen + RangeClose + RangeLo + RangeHi + RangeBetween + TermsOpen + TermsClose +) + +func (tt TokenType) String() string { + switch tt { + case EOF: + return "eof" + + case Error: + return "error" + + case Text: + return "text" + + case Char: + return "char" + + case Any: + return "any" + + case Super: + return "super" + + case Single: + return "single" + + case Not: + return "not" + + case Separator: + return "separator" + + case RangeOpen: + return "range_open" + + case RangeClose: + return "range_close" + + case RangeLo: + return "range_lo" + + case RangeHi: + return "range_hi" + + case RangeBetween: + return "range_between" + + case TermsOpen: + return "terms_open" + + case TermsClose: + return "terms_close" + + default: + return "undef" + } +} + +type Token struct { + Type TokenType + Raw string +} + +func (t Token) String() string { + return fmt.Sprintf("%v<%q>", t.Type, t.Raw) +} diff --git a/lexer_test.go b/lexer_test.go deleted file mode 100644 index 8ede767..0000000 --- a/lexer_test.go +++ /dev/null @@ -1,192 +0,0 @@ -package glob - -import ( - "testing" -) - -func TestLexGood(t *testing.T) { - for id, test := range []struct { - pattern string - items []item - }{ - { - pattern: "", - items: []item{ - item{item_eof, ""}, - }, - }, - { - pattern: "hello", - items: []item{ - item{item_text, "hello"}, - item{item_eof, ""}, - }, - }, - { - pattern: "/{rate,[0-9]]}*", - items: []item{ - item{item_text, "/"}, - item{item_terms_open, "{"}, - item{item_text, "rate"}, - item{item_separator, ","}, - item{item_range_open, "["}, - item{item_range_lo, "0"}, - item{item_range_between, "-"}, - item{item_range_hi, "9"}, - item{item_range_close, "]"}, - item{item_text, "]"}, - item{item_terms_close, "}"}, - item{item_any, "*"}, - item{item_eof, ""}, - }, - }, - { - pattern: "hello,world", - items: []item{ - item{item_text, "hello,world"}, - item{item_eof, ""}, - }, - }, - { - pattern: "hello\\,world", - items: []item{ - item{item_text, "hello,world"}, - item{item_eof, ""}, - }, - }, - { - pattern: "hello\\{world", - items: []item{ - item{item_text, "hello{world"}, - item{item_eof, ""}, - }, - }, - { - pattern: "hello?", - items: []item{ - item{item_text, "hello"}, - item{item_single, "?"}, - item{item_eof, ""}, - }, - }, - { - pattern: "hellof*", - items: []item{ - item{item_text, "hellof"}, - item{item_any, "*"}, - item{item_eof, ""}, - }, - }, - { - pattern: "hello**", - items: []item{ - item{item_text, "hello"}, - item{item_super, "**"}, - item{item_eof, ""}, - }, - }, - { - pattern: "[日-語]", - items: []item{ - item{item_range_open, "["}, - item{item_range_lo, "日"}, - item{item_range_between, "-"}, - item{item_range_hi, "語"}, - item{item_range_close, "]"}, - item{item_eof, ""}, - }, - }, - { - pattern: "[!日-語]", - items: []item{ - item{item_range_open, "["}, - item{item_not, "!"}, - item{item_range_lo, "日"}, - item{item_range_between, "-"}, - item{item_range_hi, "語"}, - item{item_range_close, "]"}, - item{item_eof, ""}, - }, - }, - { - pattern: "[日本語]", - items: []item{ - item{item_range_open, "["}, - item{item_text, "日本語"}, - item{item_range_close, "]"}, - item{item_eof, ""}, - }, - }, - { - pattern: "[!日本語]", - items: []item{ - item{item_range_open, "["}, - item{item_not, "!"}, - item{item_text, "日本語"}, - item{item_range_close, "]"}, - item{item_eof, ""}, - }, - }, - { - pattern: "{a,b}", - items: []item{ - item{item_terms_open, "{"}, - item{item_text, "a"}, - item{item_separator, ","}, - item{item_text, "b"}, - item{item_terms_close, "}"}, - item{item_eof, ""}, - }, - }, - { - pattern: "/{z,ab}*", - items: []item{ - item{item_text, "/"}, - item{item_terms_open, "{"}, - item{item_text, "z"}, - item{item_separator, ","}, - item{item_text, "ab"}, - item{item_terms_close, "}"}, - item{item_any, "*"}, - item{item_eof, ""}, - }, - }, - { - pattern: "{[!日-語],*,?,{a,b,\\c}}", - items: []item{ - item{item_terms_open, "{"}, - item{item_range_open, "["}, - item{item_not, "!"}, - item{item_range_lo, "日"}, - item{item_range_between, "-"}, - item{item_range_hi, "語"}, - item{item_range_close, "]"}, - item{item_separator, ","}, - item{item_any, "*"}, - item{item_separator, ","}, - item{item_single, "?"}, - item{item_separator, ","}, - item{item_terms_open, "{"}, - item{item_text, "a"}, - item{item_separator, ","}, - item{item_text, "b"}, - item{item_separator, ","}, - item{item_text, "c"}, - item{item_terms_close, "}"}, - item{item_terms_close, "}"}, - item{item_eof, ""}, - }, - }, - } { - lexer := newLexer(test.pattern) - for i, exp := range test.items { - act := lexer.nextItem() - if act.t != exp.t { - t.Errorf("#%d %q: wrong %d-th item type: exp: %q; act: %q\n\t(%s vs %s)", id, test.pattern, i, exp.t, act.t, exp, act) - } - if act.s != exp.s { - t.Errorf("#%d %q: wrong %d-th item contents: exp: %q; act: %q\n\t(%s vs %s)", id, test.pattern, i, exp.s, act.s, exp, act) - } - } - } -} diff --git a/parser.go b/parser.go deleted file mode 100644 index 760ec96..0000000 --- a/parser.go +++ /dev/null @@ -1,230 +0,0 @@ -package glob - -import ( - "errors" - "fmt" - "unicode/utf8" -) - -type node interface { - children() []node - append(node) -} - -// todo may be split it into another package -type lexerIface interface { - nextItem() item -} - -type nodeImpl struct { - desc []node -} - -func (n *nodeImpl) append(c node) { - n.desc = append(n.desc, c) -} -func (n *nodeImpl) children() []node { - return n.desc -} - -type nodeList struct { - nodeImpl - not bool - chars string -} -type nodeRange struct { - nodeImpl - not bool - lo, hi rune -} -type nodeText struct { - nodeImpl - text string -} - -type nodePattern struct{ nodeImpl } -type nodeAny struct{ nodeImpl } -type nodeSuper struct{ nodeImpl } -type nodeSingle struct{ nodeImpl } -type nodeAnyOf struct{ nodeImpl } - -type tree struct { - root node - current node - path []node -} - -func (t *tree) enter(c node) { - if t.root == nil { - t.root = c - t.current = c - return - } - - t.current.append(c) - t.path = append(t.path, c) - t.current = c -} - -func (t *tree) leave() { - if len(t.path)-1 <= 0 { - t.current = t.root - t.path = nil - return - } - - t.path = t.path[:len(t.path)-1] - t.current = t.path[len(t.path)-1] -} - -type parseFn func(*tree, lexerIface) (parseFn, error) - -func parse(lexer lexerIface) (*nodePattern, error) { - var parser parseFn - - root := &nodePattern{} - tree := &tree{} - tree.enter(root) - - for parser = parserMain; ; { - next, err := parser(tree, lexer) - if err != nil { - return nil, err - } - - if next == nil { - break - } - - parser = next - } - - return root, nil -} - -func parserMain(tree *tree, lexer lexerIface) (parseFn, error) { - for stop := false; !stop; { - item := lexer.nextItem() - - switch item.t { - case item_eof: - stop = true - continue - - case item_error: - return nil, errors.New(item.s) - - case item_text: - tree.current.append(&nodeText{text: item.s}) - return parserMain, nil - - case item_any: - tree.current.append(&nodeAny{}) - return parserMain, nil - - case item_super: - tree.current.append(&nodeSuper{}) - return parserMain, nil - - case item_single: - tree.current.append(&nodeSingle{}) - return parserMain, nil - - case item_range_open: - return parserRange, nil - - case item_terms_open: - tree.enter(&nodeAnyOf{}) - tree.enter(&nodePattern{}) - return parserMain, nil - - case item_separator: - tree.leave() - tree.enter(&nodePattern{}) - return parserMain, nil - - case item_terms_close: - tree.leave() - tree.leave() - return parserMain, nil - - default: - return nil, fmt.Errorf("unexpected token: %s", item) - } - } - - return nil, nil -} - -func parserRange(tree *tree, lexer lexerIface) (parseFn, error) { - var ( - not bool - lo rune - hi rune - chars string - ) - - for { - item := lexer.nextItem() - - switch item.t { - case item_eof: - return nil, errors.New("unexpected end") - - case item_error: - return nil, errors.New(item.s) - - case item_not: - not = true - - case item_range_lo: - r, w := utf8.DecodeRuneInString(item.s) - if len(item.s) > w { - return nil, fmt.Errorf("unexpected length of lo character") - } - - lo = r - - case item_range_between: - // - - case item_range_hi: - r, w := utf8.DecodeRuneInString(item.s) - if len(item.s) > w { - return nil, fmt.Errorf("unexpected length of lo character") - } - - hi = r - - if hi < lo { - return nil, fmt.Errorf("hi character '%s' should be greater than lo '%s'", string(hi), string(lo)) - } - - case item_text: - chars = item.s - - case item_range_close: - isRange := lo != 0 && hi != 0 - isChars := chars != "" - - if isChars == isRange { - return nil, fmt.Errorf("could not parse range") - } - - if isRange { - tree.current.append(&nodeRange{ - lo: lo, - hi: hi, - not: not, - }) - } else { - tree.current.append(&nodeList{ - chars: chars, - not: not, - }) - } - - return parserMain, nil - } - } -} diff --git a/parser/ast.go b/parser/ast.go new file mode 100644 index 0000000..588a45c --- /dev/null +++ b/parser/ast.go @@ -0,0 +1,48 @@ +package parser + +type Node interface { + Children() []Node + Parent() Node + append(Node) Node +} + +type node struct { + parent Node + children []Node +} + +func (n *node) Children() []Node { + return n.children +} + +func (n *node) Parent() Node { + return n.parent +} + +func (n *node) append(c Node) Node { + n.children = append(n.children, c) + return c +} + +type ListNode struct { + node + Not bool + Chars string +} + +type RangeNode struct { + node + Not bool + Lo, Hi rune +} + +type TextNode struct { + node + Text string +} + +type PatternNode struct{ node } +type AnyNode struct{ node } +type SuperNode struct{ node } +type SingleNode struct{ node } +type AnyOfNode struct{ node } diff --git a/parser/parser.go b/parser/parser.go new file mode 100644 index 0000000..6adea39 --- /dev/null +++ b/parser/parser.go @@ -0,0 +1,144 @@ +package parser + +import ( + "errors" + "fmt" + "github.com/gobwas/glob/lexer" + "unicode/utf8" +) + +type Lexer interface { + Next() lexer.Token +} + +type parseFn func(Node, Lexer) (parseFn, Node, error) + +func Parse(lexer Lexer) (*PatternNode, error) { + var parser parseFn + + root := &PatternNode{} + + var ( + tree Node + err error + ) + for parser, tree = parserMain, root; parser != nil; { + parser, tree, err = parser(tree, lexer) + if err != nil { + return nil, err + } + } + + return root, nil +} + +func parserMain(tree Node, lex Lexer) (parseFn, Node, error) { + for { + token := lex.Next() + switch token.Type { + case lexer.EOF: + return nil, tree, nil + + case lexer.Error: + return nil, tree, errors.New(token.Raw) + + case lexer.Text: + return parserMain, tree.append(&TextNode{Text: token.Raw}), nil + + case lexer.Any: + return parserMain, tree.append(&AnyNode{}), nil + + case lexer.Super: + return parserMain, tree.append(&SuperNode{}), nil + + case lexer.Single: + return parserMain, tree.append(&SingleNode{}), nil + + case lexer.RangeOpen: + return parserRange, tree, nil + + case lexer.TermsOpen: + return parserMain, tree.append(&AnyOfNode{}).append(&PatternNode{}), nil + + case lexer.Separator: + return parserMain, tree.Parent().append(&PatternNode{}), nil + + case lexer.TermsClose: + return parserMain, tree.Parent().Parent(), nil + + default: + return nil, tree, fmt.Errorf("unexpected token: %s", token) + } + } + return nil, tree, fmt.Errorf("unknown error") +} + +func parserRange(tree Node, lex Lexer) (parseFn, Node, error) { + var ( + not bool + lo rune + hi rune + chars string + ) + for { + token := lex.Next() + switch token.Type { + case lexer.EOF: + return nil, tree, errors.New("unexpected end") + + case lexer.Error: + return nil, tree, errors.New(token.Raw) + + case lexer.Not: + not = true + + case lexer.RangeLo: + r, w := utf8.DecodeRuneInString(token.Raw) + if len(token.Raw) > w { + return nil, tree, fmt.Errorf("unexpected length of lo character") + } + lo = r + + case lexer.RangeBetween: + // + + case lexer.RangeHi: + r, w := utf8.DecodeRuneInString(token.Raw) + if len(token.Raw) > w { + return nil, tree, fmt.Errorf("unexpected length of lo character") + } + + hi = r + + if hi < lo { + return nil, tree, fmt.Errorf("hi character '%s' should be greater than lo '%s'", string(hi), string(lo)) + } + + case lexer.Text: + chars = token.Raw + + case lexer.RangeClose: + isRange := lo != 0 && hi != 0 + isChars := chars != "" + + if isChars == isRange { + return nil, tree, fmt.Errorf("could not parse range") + } + + if isRange { + tree = tree.append(&RangeNode{ + Lo: lo, + Hi: hi, + Not: not, + }) + } else { + tree = tree.append(&ListNode{ + Chars: chars, + Not: not, + }) + } + + return parserMain, tree, nil + } + } +} diff --git a/parser/parser_test.go b/parser/parser_test.go new file mode 100644 index 0000000..177d7bf --- /dev/null +++ b/parser/parser_test.go @@ -0,0 +1,332 @@ +package parser + +import ( + "fmt" + "github.com/gobwas/glob/lexer" + "reflect" + "testing" +) + +type stubLexer struct { + tokens []lexer.Token + pos int +} + +func (s *stubLexer) Next() (ret lexer.Token) { + if s.pos == len(s.tokens) { + return lexer.Token{lexer.EOF, ""} + } + ret = s.tokens[s.pos] + s.pos++ + return +} + +func TestParseString(t *testing.T) { + for id, test := range []struct { + tokens []lexer.Token + tree Node + }{ + { + //pattern: "abc", + tokens: []lexer.Token{ + lexer.Token{lexer.Text, "abc"}, + lexer.Token{lexer.EOF, ""}, + }, + tree: &PatternNode{ + node: node{ + children: []Node{ + &TextNode{Text: "abc"}, + }, + }, + }, + }, + { + //pattern: "a*c", + tokens: []lexer.Token{ + lexer.Token{lexer.Text, "a"}, + lexer.Token{lexer.Any, "*"}, + lexer.Token{lexer.Text, "c"}, + lexer.Token{lexer.EOF, ""}, + }, + tree: &PatternNode{ + node: node{ + children: []Node{ + &TextNode{Text: "a"}, + &AnyNode{}, + &TextNode{Text: "c"}, + }, + }, + }, + }, + { + //pattern: "a**c", + tokens: []lexer.Token{ + lexer.Token{lexer.Text, "a"}, + lexer.Token{lexer.Super, "**"}, + lexer.Token{lexer.Text, "c"}, + lexer.Token{lexer.EOF, ""}, + }, + tree: &PatternNode{ + node: node{ + children: []Node{ + &TextNode{Text: "a"}, + &SuperNode{}, + &TextNode{Text: "c"}, + }, + }, + }, + }, + { + //pattern: "a?c", + tokens: []lexer.Token{ + lexer.Token{lexer.Text, "a"}, + lexer.Token{lexer.Single, "?"}, + lexer.Token{lexer.Text, "c"}, + lexer.Token{lexer.EOF, ""}, + }, + tree: &PatternNode{ + node: node{ + children: []Node{ + &TextNode{Text: "a"}, + &SingleNode{}, + &TextNode{Text: "c"}, + }, + }, + }, + }, + { + //pattern: "[!a-z]", + tokens: []lexer.Token{ + lexer.Token{lexer.RangeOpen, "["}, + lexer.Token{lexer.Not, "!"}, + lexer.Token{lexer.RangeLo, "a"}, + lexer.Token{lexer.RangeBetween, "-"}, + lexer.Token{lexer.RangeHi, "z"}, + lexer.Token{lexer.RangeClose, "]"}, + lexer.Token{lexer.EOF, ""}, + }, + tree: &PatternNode{ + node: node{ + children: []Node{ + &RangeNode{Lo: 'a', Hi: 'z', Not: true}, + }, + }, + }, + }, + { + //pattern: "[az]", + tokens: []lexer.Token{ + lexer.Token{lexer.RangeOpen, "["}, + lexer.Token{lexer.Text, "az"}, + lexer.Token{lexer.RangeClose, "]"}, + lexer.Token{lexer.EOF, ""}, + }, + tree: &PatternNode{ + node: node{ + children: []Node{ + &ListNode{Chars: "az"}, + }, + }, + }, + }, + { + //pattern: "{a,z}", + tokens: []lexer.Token{ + lexer.Token{lexer.TermsOpen, "{"}, + lexer.Token{lexer.Text, "a"}, + lexer.Token{lexer.Separator, ","}, + lexer.Token{lexer.Text, "z"}, + lexer.Token{lexer.TermsClose, "}"}, + lexer.Token{lexer.EOF, ""}, + }, + tree: &PatternNode{ + node: node{ + children: []Node{ + &AnyOfNode{node: node{children: []Node{ + &PatternNode{ + node: node{children: []Node{ + &TextNode{Text: "a"}, + }}, + }, + &PatternNode{ + node: node{children: []Node{ + &TextNode{Text: "z"}, + }}, + }, + }}}, + }, + }, + }, + }, + { + //pattern: "/{z,ab}*", + tokens: []lexer.Token{ + lexer.Token{lexer.Text, "/"}, + lexer.Token{lexer.TermsOpen, "{"}, + lexer.Token{lexer.Text, "z"}, + lexer.Token{lexer.Separator, ","}, + lexer.Token{lexer.Text, "ab"}, + lexer.Token{lexer.TermsClose, "}"}, + lexer.Token{lexer.Any, "*"}, + lexer.Token{lexer.EOF, ""}, + }, + tree: &PatternNode{ + node: node{ + children: []Node{ + &TextNode{Text: "/"}, + &AnyOfNode{node: node{children: []Node{ + &PatternNode{ + node: node{children: []Node{ + &TextNode{Text: "z"}, + }}, + }, + &PatternNode{ + node: node{children: []Node{ + &TextNode{Text: "ab"}, + }}, + }, + }}}, + &AnyNode{}, + }, + }, + }, + }, + { + //pattern: "{a,{x,y},?,[a-z],[!qwe]}", + tokens: []lexer.Token{ + lexer.Token{lexer.TermsOpen, "{"}, + lexer.Token{lexer.Text, "a"}, + lexer.Token{lexer.Separator, ","}, + lexer.Token{lexer.TermsOpen, "{"}, + lexer.Token{lexer.Text, "x"}, + lexer.Token{lexer.Separator, ","}, + lexer.Token{lexer.Text, "y"}, + lexer.Token{lexer.TermsClose, "}"}, + lexer.Token{lexer.Separator, ","}, + lexer.Token{lexer.Single, "?"}, + lexer.Token{lexer.Separator, ","}, + lexer.Token{lexer.RangeOpen, "["}, + lexer.Token{lexer.RangeLo, "a"}, + lexer.Token{lexer.RangeBetween, "-"}, + lexer.Token{lexer.RangeHi, "z"}, + lexer.Token{lexer.RangeClose, "]"}, + lexer.Token{lexer.Separator, ","}, + lexer.Token{lexer.RangeOpen, "["}, + lexer.Token{lexer.Not, "!"}, + lexer.Token{lexer.Text, "qwe"}, + lexer.Token{lexer.RangeClose, "]"}, + lexer.Token{lexer.TermsClose, "}"}, + lexer.Token{lexer.EOF, ""}, + }, + tree: &PatternNode{ + node: node{ + children: []Node{ + &AnyOfNode{node: node{children: []Node{ + &PatternNode{ + node: node{children: []Node{ + &TextNode{Text: "a"}, + }}, + }, + &PatternNode{ + node: node{children: []Node{ + &AnyOfNode{node: node{children: []Node{ + &PatternNode{ + node: node{children: []Node{ + &TextNode{Text: "x"}, + }}, + }, + &PatternNode{ + node: node{children: []Node{ + &TextNode{Text: "y"}, + }}, + }, + }}}, + }}, + }, + &PatternNode{ + node: node{children: []Node{ + &SingleNode{}, + }}, + }, + &PatternNode{ + node: node{ + children: []Node{ + &RangeNode{Lo: 'a', Hi: 'z', Not: false}, + }, + }, + }, + &PatternNode{ + node: node{ + children: []Node{ + &ListNode{Chars: "qwe", Not: true}, + }, + }, + }, + }}}, + }, + }, + }, + }, + } { + lexer := &stubLexer{tokens: test.tokens} + result, err := Parse(lexer) + if err != nil { + t.Errorf("[%d] unexpected error: %s", id, err) + } + if !reflect.DeepEqual(test.tree, result) { + t.Errorf("[%d] Parse():\nact:\t%s\nexp:\t%s\n", id, result, test.tree) + } + } +} + +const abstractNodeImpl = "nodeImpl" + +func nodeEqual(a, b Node) error { + if (a == nil || b == nil) && a != b { + return fmt.Errorf("nodes are not equal: exp %s, act %s", a, b) + } + + aValue, bValue := reflect.Indirect(reflect.ValueOf(a)), reflect.Indirect(reflect.ValueOf(b)) + aType, bType := aValue.Type(), bValue.Type() + if aType != bType { + return fmt.Errorf("nodes are not equal: exp %s, act %s", aValue.Type(), bValue.Type()) + } + + for i := 0; i < aType.NumField(); i++ { + var eq bool + + f := aType.Field(i).Name + if f == abstractNodeImpl { + continue + } + + af, bf := aValue.FieldByName(f), bValue.FieldByName(f) + + switch af.Kind() { + case reflect.String: + eq = af.String() == bf.String() + case reflect.Bool: + eq = af.Bool() == bf.Bool() + default: + eq = fmt.Sprint(af) == fmt.Sprint(bf) + } + + if !eq { + return fmt.Errorf("nodes<%s> %q fields are not equal: exp %q, act %q", aType, f, af, bf) + } + } + + for i, aDesc := range a.Children() { + if len(b.Children())-1 < i { + return fmt.Errorf("node does not have enough children (got %d children, wanted %d-th token)", len(b.Children()), i) + } + + bDesc := b.Children()[i] + + if err := nodeEqual(aDesc, bDesc); err != nil { + return err + } + } + + return nil +} diff --git a/parser_test.go b/parser_test.go deleted file mode 100644 index 0e9312f..0000000 --- a/parser_test.go +++ /dev/null @@ -1,324 +0,0 @@ -package glob - -import ( - "fmt" - "reflect" - "testing" -) - -func TestParseString(t *testing.T) { - for id, test := range []struct { - items []item - tree node - }{ - { - //pattern: "abc", - items: []item{ - item{item_text, "abc"}, - item{item_eof, ""}, - }, - tree: &nodePattern{ - nodeImpl: nodeImpl{ - desc: []node{ - &nodeText{text: "abc"}, - }, - }, - }, - }, - { - //pattern: "a*c", - items: []item{ - item{item_text, "a"}, - item{item_any, "*"}, - item{item_text, "c"}, - item{item_eof, ""}, - }, - tree: &nodePattern{ - nodeImpl: nodeImpl{ - desc: []node{ - &nodeText{text: "a"}, - &nodeAny{}, - &nodeText{text: "c"}, - }, - }, - }, - }, - { - //pattern: "a**c", - items: []item{ - item{item_text, "a"}, - item{item_super, "**"}, - item{item_text, "c"}, - item{item_eof, ""}, - }, - tree: &nodePattern{ - nodeImpl: nodeImpl{ - desc: []node{ - &nodeText{text: "a"}, - &nodeSuper{}, - &nodeText{text: "c"}, - }, - }, - }, - }, - { - //pattern: "a?c", - items: []item{ - item{item_text, "a"}, - item{item_single, "?"}, - item{item_text, "c"}, - item{item_eof, ""}, - }, - tree: &nodePattern{ - nodeImpl: nodeImpl{ - desc: []node{ - &nodeText{text: "a"}, - &nodeSingle{}, - &nodeText{text: "c"}, - }, - }, - }, - }, - { - //pattern: "[!a-z]", - items: []item{ - item{item_range_open, "["}, - item{item_not, "!"}, - item{item_range_lo, "a"}, - item{item_range_between, "-"}, - item{item_range_hi, "z"}, - item{item_range_close, "]"}, - item{item_eof, ""}, - }, - tree: &nodePattern{ - nodeImpl: nodeImpl{ - desc: []node{ - &nodeRange{lo: 'a', hi: 'z', not: true}, - }, - }, - }, - }, - { - //pattern: "[az]", - items: []item{ - item{item_range_open, "["}, - item{item_text, "az"}, - item{item_range_close, "]"}, - item{item_eof, ""}, - }, - tree: &nodePattern{ - nodeImpl: nodeImpl{ - desc: []node{ - &nodeList{chars: "az"}, - }, - }, - }, - }, - { - //pattern: "{a,z}", - items: []item{ - item{item_terms_open, "{"}, - item{item_text, "a"}, - item{item_separator, ","}, - item{item_text, "z"}, - item{item_terms_close, "}"}, - item{item_eof, ""}, - }, - tree: &nodePattern{ - nodeImpl: nodeImpl{ - desc: []node{ - &nodeAnyOf{nodeImpl: nodeImpl{desc: []node{ - &nodePattern{ - nodeImpl: nodeImpl{desc: []node{ - &nodeText{text: "a"}, - }}, - }, - &nodePattern{ - nodeImpl: nodeImpl{desc: []node{ - &nodeText{text: "z"}, - }}, - }, - }}}, - }, - }, - }, - }, - { - //pattern: "/{z,ab}*", - items: []item{ - item{item_text, "/"}, - item{item_terms_open, "{"}, - item{item_text, "z"}, - item{item_separator, ","}, - item{item_text, "ab"}, - item{item_terms_close, "}"}, - item{item_any, "*"}, - item{item_eof, ""}, - }, - tree: &nodePattern{ - nodeImpl: nodeImpl{ - desc: []node{ - &nodeText{text: "/"}, - &nodeAnyOf{nodeImpl: nodeImpl{desc: []node{ - &nodePattern{ - nodeImpl: nodeImpl{desc: []node{ - &nodeText{text: "z"}, - }}, - }, - &nodePattern{ - nodeImpl: nodeImpl{desc: []node{ - &nodeText{text: "ab"}, - }}, - }, - }}}, - &nodeAny{}, - }, - }, - }, - }, - { - //pattern: "{a,{x,y},?,[a-z],[!qwe]}", - items: []item{ - item{item_terms_open, "{"}, - item{item_text, "a"}, - item{item_separator, ","}, - item{item_terms_open, "{"}, - item{item_text, "x"}, - item{item_separator, ","}, - item{item_text, "y"}, - item{item_terms_close, "}"}, - item{item_separator, ","}, - item{item_single, "?"}, - item{item_separator, ","}, - item{item_range_open, "["}, - item{item_range_lo, "a"}, - item{item_range_between, "-"}, - item{item_range_hi, "z"}, - item{item_range_close, "]"}, - item{item_separator, ","}, - item{item_range_open, "["}, - item{item_not, "!"}, - item{item_text, "qwe"}, - item{item_range_close, "]"}, - item{item_terms_close, "}"}, - item{item_eof, ""}, - }, - tree: &nodePattern{ - nodeImpl: nodeImpl{ - desc: []node{ - &nodeAnyOf{nodeImpl: nodeImpl{desc: []node{ - &nodePattern{ - nodeImpl: nodeImpl{desc: []node{ - &nodeText{text: "a"}, - }}, - }, - &nodePattern{ - nodeImpl: nodeImpl{desc: []node{ - &nodeAnyOf{nodeImpl: nodeImpl{desc: []node{ - &nodePattern{ - nodeImpl: nodeImpl{desc: []node{ - &nodeText{text: "x"}, - }}, - }, - &nodePattern{ - nodeImpl: nodeImpl{desc: []node{ - &nodeText{text: "y"}, - }}, - }, - }}}, - }}, - }, - &nodePattern{ - nodeImpl: nodeImpl{desc: []node{ - &nodeSingle{}, - }}, - }, - &nodePattern{ - nodeImpl: nodeImpl{ - desc: []node{ - &nodeRange{lo: 'a', hi: 'z', not: false}, - }, - }, - }, - &nodePattern{ - nodeImpl: nodeImpl{ - desc: []node{ - &nodeList{chars: "qwe", not: true}, - }, - }, - }, - }}}, - }, - }, - }, - }, - } { - lexer := &stubLexer{Items: test.items} - pattern, err := parse(lexer) - - if err != nil { - t.Errorf("#%d %s", id, err) - continue - } - - if !reflect.DeepEqual(test.tree, pattern) { - t.Errorf("#%d tries are not equal", id) - if err = nodeEqual(test.tree, pattern); err != nil { - t.Errorf("#%d %s", id, err) - continue - } - } - } -} - -const abstractNodeImpl = "nodeImpl" - -func nodeEqual(a, b node) error { - if (a == nil || b == nil) && a != b { - return fmt.Errorf("nodes are not equal: exp %s, act %s", a, b) - } - - aValue, bValue := reflect.Indirect(reflect.ValueOf(a)), reflect.Indirect(reflect.ValueOf(b)) - aType, bType := aValue.Type(), bValue.Type() - if aType != bType { - return fmt.Errorf("nodes are not equal: exp %s, act %s", aValue.Type(), bValue.Type()) - } - - for i := 0; i < aType.NumField(); i++ { - var eq bool - - f := aType.Field(i).Name - if f == abstractNodeImpl { - continue - } - - af, bf := aValue.FieldByName(f), bValue.FieldByName(f) - - switch af.Kind() { - case reflect.String: - eq = af.String() == bf.String() - case reflect.Bool: - eq = af.Bool() == bf.Bool() - default: - eq = fmt.Sprint(af) == fmt.Sprint(bf) - } - - if !eq { - return fmt.Errorf("nodes<%s> %q fields are not equal: exp %q, act %q", aType, f, af, bf) - } - } - - for i, aDesc := range a.children() { - if len(b.children())-1 < i { - return fmt.Errorf("node does not have enough children (got %d children, wanted %d-th token)", len(b.children()), i) - } - - bDesc := b.children()[i] - - if err := nodeEqual(aDesc, bDesc); err != nil { - return err - } - } - - return nil -} diff --git a/runes/runes.go b/util/runes/runes.go similarity index 100% rename from runes/runes.go rename to util/runes/runes.go diff --git a/runes/runes_test.go b/util/runes/runes_test.go similarity index 100% rename from runes/runes_test.go rename to util/runes/runes_test.go diff --git a/strings/strings.go b/util/strings/strings.go similarity index 100% rename from strings/strings.go rename to util/strings/strings.go From 510a1756cf2f1425445d369012ec9e3434dbec65 Mon Sep 17 00:00:00 2001 From: gobwas Date: Mon, 30 May 2016 19:35:53 +0300 Subject: [PATCH 2/3] refactoring --- compiler.go => compiler/compiler.go | 449 ++++++++++---------- compiler/compiler_test.go | 573 ++++++++++++++++++++++++++ compiler_test.go | 548 ------------------------ glob.go | 9 +- match/any.go | 2 +- match/list.go | 2 +- match/single.go | 2 +- parser/ast.go | 48 --- parser/parser_test.go | 332 --------------- syntax/ast/ast.go | 85 ++++ {parser => syntax/ast}/parser.go | 51 ++- syntax/ast/parser_test.go | 256 ++++++++++++ {lexer => syntax/lexer}/lexer.go | 26 +- {lexer => syntax/lexer}/lexer_test.go | 4 +- {lexer => syntax/lexer}/token.go | 0 syntax/syntax.go | 14 + 16 files changed, 1207 insertions(+), 1194 deletions(-) rename compiler.go => compiler/compiler.go (51%) create mode 100644 compiler/compiler_test.go delete mode 100644 compiler_test.go delete mode 100644 parser/ast.go delete mode 100644 parser/parser_test.go create mode 100644 syntax/ast/ast.go rename {parser => syntax/ast}/parser.go (66%) create mode 100644 syntax/ast/parser_test.go rename {lexer => syntax/lexer}/lexer.go (97%) rename {lexer => syntax/lexer}/lexer_test.go (98%) rename {lexer => syntax/lexer}/token.go (100%) create mode 100644 syntax/syntax.go diff --git a/compiler.go b/compiler/compiler.go similarity index 51% rename from compiler.go rename to compiler/compiler.go index bc48559..0278b57 100644 --- a/compiler.go +++ b/compiler/compiler.go @@ -1,4 +1,4 @@ -package glob +package compiler // TODO use constructor with all matchers, and to their structs private // TODO glue multiple Text nodes (like after QuoteMeta) @@ -6,11 +6,12 @@ package glob import ( "fmt" "github.com/gobwas/glob/match" - "github.com/gobwas/glob/runes" + "github.com/gobwas/glob/syntax/ast" + "github.com/gobwas/glob/util/runes" "reflect" ) -func optimize(matcher match.Matcher) match.Matcher { +func optimizeMatcher(matcher match.Matcher) match.Matcher { switch m := matcher.(type) { case match.Any: @@ -33,8 +34,8 @@ func optimize(matcher match.Matcher) match.Matcher { return m case match.BTree: - m.Left = optimize(m.Left) - m.Right = optimize(m.Right) + m.Left = optimizeMatcher(m.Left) + m.Right = optimizeMatcher(m.Right) r, ok := m.Value.(match.Text) if !ok { @@ -80,34 +81,72 @@ func optimize(matcher match.Matcher) match.Matcher { return matcher } -func glueMatchers(matchers []match.Matcher) match.Matcher { - var ( - glued []match.Matcher - winner match.Matcher - ) +func compileMatchers(matchers []match.Matcher) (match.Matcher, error) { + if len(matchers) == 0 { + return nil, fmt.Errorf("compile error: need at least one matcher") + } + if len(matchers) == 1 { + return matchers[0], nil + } + if m := glueMatchers(matchers); m != nil { + return m, nil + } + + idx := -1 maxLen := -1 - - if m := glueAsEvery(matchers); m != nil { - glued = append(glued, m) - return m - } - - if m := glueAsRow(matchers); m != nil { - glued = append(glued, m) - return m - } - - for _, g := range glued { - if l := g.Len(); l > maxLen { + var val match.Matcher + for i, matcher := range matchers { + if l := matcher.Len(); l != -1 && l >= maxLen { maxLen = l - winner = g + idx = i + val = matcher } } - return winner + if val == nil { // not found matcher with static length + r, err := compileMatchers(matchers[1:]) + if err != nil { + return nil, err + } + return match.NewBTree(matchers[0], nil, r), nil + } + + left := matchers[:idx] + var right []match.Matcher + if len(matchers) > idx+1 { + right = matchers[idx+1:] + } + + var l, r match.Matcher + var err error + if len(left) > 0 { + l, err = compileMatchers(left) + if err != nil { + return nil, err + } + } + + if len(right) > 0 { + r, err = compileMatchers(right) + if err != nil { + return nil, err + } + } + + return match.NewBTree(val, l, r), nil } -func glueAsRow(matchers []match.Matcher) match.Matcher { +func glueMatchers(matchers []match.Matcher) match.Matcher { + if m := glueMatchersAsEvery(matchers); m != nil { + return m + } + if m := glueMatchersAsRow(matchers); m != nil { + return m + } + return nil +} + +func glueMatchersAsRow(matchers []match.Matcher) match.Matcher { if len(matchers) <= 1 { return nil } @@ -124,11 +163,10 @@ func glueAsRow(matchers []match.Matcher) match.Matcher { l += ml } } - return match.NewRow(l, c...) } -func glueAsEvery(matchers []match.Matcher) match.Matcher { +func glueMatchersAsEvery(matchers []match.Matcher) match.Matcher { if len(matchers) <= 1 { return nil } @@ -254,255 +292,212 @@ func minimizeMatchers(matchers []match.Matcher) []match.Matcher { return minimizeMatchers(next) } -func minimizeAnyOf(children []node) node { - var nodes [][]node - var min int - var idx int - for i, desc := range children { - pat, ok := desc.(*nodePattern) - if !ok { - return nil - } - - n := pat.children() - ln := len(n) - if len(nodes) == 0 || (ln < min) { - min = ln - idx = i - } - - nodes = append(nodes, pat.children()) +// minimizeAnyOf tries to apply some heuristics to minimize number of nodes in given tree +func minimizeTree(tree *ast.Node) *ast.Node { + switch tree.Kind { + case ast.KindAnyOf: + return minimizeTreeAnyOf(tree) + default: + return nil } +} - minNodes := nodes[idx] - if idx+1 < len(nodes) { - nodes = append(nodes[:idx], nodes[idx+1:]...) - } else { - nodes = nodes[:idx] - } - - var commonLeft []node - var commonLeftCount int - for i, n := range minNodes { - has := true - for _, t := range nodes { - if !reflect.DeepEqual(n, t[i]) { - has = false - break - } - } - - if has { - commonLeft = append(commonLeft, n) - commonLeftCount++ - } else { - break - } - } - - var commonRight []node - var commonRightCount int - for i := min - 1; i > commonLeftCount-1; i-- { - n := minNodes[i] - has := true - for _, t := range nodes { - if !reflect.DeepEqual(n, t[len(t)-(min-i)]) { - has = false - break - } - } - - if has { - commonRight = append(commonRight, n) - commonRightCount++ - } else { - break - } - } - - if commonLeftCount == 0 && commonRightCount == 0 { +// minimizeAnyOf tries to find common children of given node of AnyOf pattern +// it searches for common children from left and from right +// if any common children are found – then it returns new optimized ast tree +// else it returns nil +func minimizeTreeAnyOf(tree *ast.Node) *ast.Node { + if !areOfSameKind(tree.Children, ast.KindPattern) { return nil } - nodes = append(nodes, minNodes) - nodes[len(nodes)-1], nodes[idx] = nodes[idx], nodes[len(nodes)-1] + commonLeft, commonRight := commonChildren(tree.Children) + commonLeftCount, commonRightCount := len(commonLeft), len(commonRight) + if commonLeftCount == 0 && commonRightCount == 0 { // there are no common parts + return nil + } - var result []node + var result []*ast.Node if commonLeftCount > 0 { - result = append(result, &nodePattern{nodeImpl: nodeImpl{desc: commonLeft}}) + result = append(result, ast.NewNode(ast.KindPattern, nil, commonLeft...)) } - var anyOf []node - for _, n := range nodes { - if commonLeftCount+commonRightCount == len(n) { - anyOf = append(anyOf, nil) + var anyOf []*ast.Node + for _, child := range tree.Children { + reuse := child.Children[commonLeftCount : len(child.Children)-commonRightCount] + var node *ast.Node + if len(reuse) == 0 { + // this pattern is completely reduced by commonLeft and commonRight patterns + // so it become nothing + node = ast.NewNode(ast.KindNothing, nil) } else { - anyOf = append(anyOf, &nodePattern{nodeImpl: nodeImpl{desc: n[commonLeftCount : len(n)-commonRightCount]}}) + node = ast.NewNode(ast.KindPattern, nil, reuse...) } + anyOf = appendIfUnique(anyOf, node) } - - anyOf = uniqueNodes(anyOf) - if len(anyOf) == 1 { - if anyOf[0] != nil { - result = append(result, &nodePattern{nodeImpl: nodeImpl{desc: anyOf}}) - } - } else { - result = append(result, &nodeAnyOf{nodeImpl: nodeImpl{desc: anyOf}}) + switch { + case len(anyOf) == 1 && anyOf[0].Kind != ast.KindNothing: + result = append(result, anyOf[0]) + case len(anyOf) > 1: + result = append(result, ast.NewNode(ast.KindAnyOf, nil, anyOf...)) } if commonRightCount > 0 { - result = append(result, &nodePattern{nodeImpl: nodeImpl{desc: commonRight}}) + result = append(result, ast.NewNode(ast.KindPattern, nil, commonRight...)) } - return &nodePattern{nodeImpl: nodeImpl{desc: result}} + return ast.NewNode(ast.KindPattern, nil, result...) } -func uniqueNodes(nodes []node) (result []node) { -head: - for _, n := range nodes { - for _, e := range result { - if reflect.DeepEqual(e, n) { - continue head - } - } - - result = append(result, n) +func commonChildren(nodes []*ast.Node) (commonLeft, commonRight []*ast.Node) { + // find node that has least number of children + idx := leastChildren(nodes) + if idx == -1 { + return nil, nil } + tree := nodes[idx] - return -} + var ( + breakLeft bool + breakRight bool + ) + for i, j := 0, len(tree.Children)-1; j >= 0 && !(breakLeft && breakLeft); i, j = i+1, j-1 { + treeLeft := tree.Children[i] + treeRight := tree.Children[j] -func compileMatchers(matchers []match.Matcher) (match.Matcher, error) { - if len(matchers) == 0 { - return nil, fmt.Errorf("compile error: need at least one matcher") - } + fmt.Println(i, j) - if len(matchers) == 1 { - return matchers[0], nil - } - - if m := glueMatchers(matchers); m != nil { - return m, nil - } - - idx := -1 - maxLen := -1 - var val match.Matcher - for i, matcher := range matchers { - if l := matcher.Len(); l != -1 && l >= maxLen { - maxLen = l - idx = i - val = matcher - } - } - - if val == nil { // not found matcher with static length - r, err := compileMatchers(matchers[1:]) - if err != nil { - return nil, err - } - return match.NewBTree(matchers[0], nil, r), nil - } - - left := matchers[:idx] - var right []match.Matcher - if len(matchers) > idx+1 { - right = matchers[idx+1:] - } - - var l, r match.Matcher - var err error - if len(left) > 0 { - l, err = compileMatchers(left) - if err != nil { - return nil, err - } - } - - if len(right) > 0 { - r, err = compileMatchers(right) - if err != nil { - return nil, err - } - } - - return match.NewBTree(val, l, r), nil -} - -func do(leaf node, s []rune) (m match.Matcher, err error) { - switch n := leaf.(type) { - case *nodeAnyOf: - // todo this could be faster on pattern_alternatives_combine_lite - if n := minimizeAnyOf(n.children()); n != nil { - return do(n, s) - } - - var matchers []match.Matcher - for _, desc := range n.children() { - if desc == nil { - matchers = append(matchers, match.NewNothing()) + for k := 0; k < len(nodes) && !(breakLeft && breakLeft); k++ { + // skip least children node + if k == idx { continue } - m, err := do(desc, s) - if err != nil { - return nil, err - } - matchers = append(matchers, optimize(m)) - } + restLeft := nodes[k].Children[i] + restRight := nodes[k].Children[j+len(nodes[k].Children)-len(tree.Children)] + breakLeft = breakLeft || !treeLeft.Equal(restLeft) + + // disable searching for right common parts, if left part is already overlapping + breakRight = breakRight || (!breakLeft && j <= i) + breakRight = breakRight || !treeRight.Equal(restRight) + } + if !breakLeft { + fmt.Println("left app") + commonLeft = append(commonLeft, treeLeft) + } + if !breakRight { + fmt.Println("right app") + commonRight = append(commonRight, treeRight) + } + } + return +} + +func appendIfUnique(target []*ast.Node, val *ast.Node) []*ast.Node { + for _, n := range target { + if reflect.DeepEqual(n, val) { + return target + } + } + return append(target, val) +} + +func areOfSameKind(nodes []*ast.Node, kind ast.Kind) bool { + for _, n := range nodes { + if n.Kind != kind { + return false + } + } + return true +} + +func leastChildren(nodes []*ast.Node) int { + min := -1 + idx := -1 + for i, n := range nodes { + if idx == -1 || (len(n.Children) < min) { + min = len(n.Children) + idx = i + } + } + return idx +} + +func compileTreeChildren(tree *ast.Node, sep []rune) ([]match.Matcher, error) { + var matchers []match.Matcher + for _, desc := range tree.Children { + m, err := compile(desc, sep) + if err != nil { + return nil, err + } + matchers = append(matchers, optimizeMatcher(m)) + } + return matchers, nil +} + +func compile(tree *ast.Node, sep []rune) (m match.Matcher, err error) { + switch tree.Kind { + case ast.KindAnyOf: + // todo this could be faster on pattern_alternatives_combine_lite (see glob_test.go) + if n := minimizeTree(tree); n != nil { + return compile(n, sep) + } + matchers, err := compileTreeChildren(tree, sep) + if err != nil { + return nil, err + } return match.NewAnyOf(matchers...), nil - case *nodePattern: - nodes := leaf.children() - if len(nodes) == 0 { + case ast.KindPattern: + if len(tree.Children) == 0 { return match.NewNothing(), nil } - - var matchers []match.Matcher - for _, desc := range nodes { - m, err := do(desc, s) - if err != nil { - return nil, err - } - matchers = append(matchers, optimize(m)) + matchers, err := compileTreeChildren(tree, sep) + if err != nil { + return nil, err } - m, err = compileMatchers(minimizeMatchers(matchers)) if err != nil { return nil, err } - case *nodeList: - m = match.NewList([]rune(n.chars), n.not) + case ast.KindAny: + m = match.NewAny(sep) - case *nodeRange: - m = match.NewRange(n.lo, n.hi, n.not) - - case *nodeAny: - m = match.NewAny(s) - - case *nodeSuper: + case ast.KindSuper: m = match.NewSuper() - case *nodeSingle: - m = match.NewSingle(s) + case ast.KindSingle: + m = match.NewSingle(sep) - case *nodeText: - m = match.NewText(n.text) + case ast.KindNothing: + m = match.NewNothing() + + case ast.KindList: + l := tree.Value.(ast.List) + m = match.NewList([]rune(l.Chars), l.Not) + + case ast.KindRange: + r := tree.Value.(ast.Range) + m = match.NewRange(r.Lo, r.Hi, r.Not) + + case ast.KindText: + t := tree.Value.(ast.Text) + m = match.NewText(t.Text) default: return nil, fmt.Errorf("could not compile tree: unknown node type") } - return optimize(m), nil + return optimizeMatcher(m), nil } -func compile(ast *nodePattern, s []rune) (Glob, error) { - g, err := do(ast, s) +func Compile(tree *ast.Node, sep []rune) (match.Matcher, error) { + m, err := compile(tree, sep) if err != nil { return nil, err } - return g, nil + return m, nil } diff --git a/compiler/compiler_test.go b/compiler/compiler_test.go new file mode 100644 index 0000000..a93b918 --- /dev/null +++ b/compiler/compiler_test.go @@ -0,0 +1,573 @@ +package compiler + +import ( + "github.com/gobwas/glob/match" + "github.com/gobwas/glob/match/debug" + "github.com/gobwas/glob/syntax/ast" + "reflect" + "testing" +) + +var separators = []rune{'.'} + +func TestCommonChildren(t *testing.T) { + for i, test := range []struct { + nodes []*ast.Node + left []*ast.Node + right []*ast.Node + }{ + // { + // nodes: []*ast.Node{ + // ast.NewNode(ast.KindNothing, nil, + // ast.NewNode(ast.KindText, ast.Text{"a"}), + // ast.NewNode(ast.KindText, ast.Text{"z"}), + // ast.NewNode(ast.KindText, ast.Text{"c"}), + // ), + // ast.NewNode(ast.KindNothing, nil, + // ast.NewNode(ast.KindText, ast.Text{"a"}), + // ast.NewNode(ast.KindText, ast.Text{"b"}), + // ast.NewNode(ast.KindText, ast.Text{"c"}), + // ), + // }, + // left: []*ast.Node{ + // ast.NewNode(ast.KindText, ast.Text{"a"}), + // }, + // right: []*ast.Node{ + // ast.NewNode(ast.KindText, ast.Text{"c"}), + // }, + // }, + { + nodes: []*ast.Node{ + ast.NewNode(ast.KindNothing, nil, + ast.NewNode(ast.KindText, ast.Text{"a"}), + ast.NewNode(ast.KindText, ast.Text{"b"}), + ast.NewNode(ast.KindText, ast.Text{"c"}), + ast.NewNode(ast.KindText, ast.Text{"d"}), + ), + ast.NewNode(ast.KindNothing, nil, + ast.NewNode(ast.KindText, ast.Text{"a"}), + ast.NewNode(ast.KindText, ast.Text{"b"}), + ast.NewNode(ast.KindText, ast.Text{"c"}), + ast.NewNode(ast.KindText, ast.Text{"c"}), + ast.NewNode(ast.KindText, ast.Text{"d"}), + ), + }, + left: []*ast.Node{ + ast.NewNode(ast.KindText, ast.Text{"a"}), + ast.NewNode(ast.KindText, ast.Text{"b"}), + ast.NewNode(ast.KindText, ast.Text{"c"}), + }, + right: []*ast.Node{ + ast.NewNode(ast.KindText, ast.Text{"d"}), + }, + }, + } { + left, right := commonChildren(test.nodes) + if !nodesEqual(left, test.left) { + t.Errorf("[%d] left, right := commonChildren(); left = %v; want %v", i, left, test.left) + } + if !nodesEqual(right, test.right) { + t.Errorf("[%d] left, right := commonChildren(); right = %v; want %v", i, right, test.right) + } + } +} + +func nodesEqual(a, b []*ast.Node) bool { + if len(a) != len(b) { + return false + } + for i, av := range a { + if !av.Equal(b[i]) { + return false + } + } + return true +} + +func TestGlueMatchers(t *testing.T) { + for id, test := range []struct { + in []match.Matcher + exp match.Matcher + }{ + { + []match.Matcher{ + match.NewSuper(), + match.NewSingle(nil), + }, + match.NewMin(1), + }, + { + []match.Matcher{ + match.NewAny(separators), + match.NewSingle(separators), + }, + match.EveryOf{match.Matchers{ + match.NewMin(1), + match.NewContains(string(separators), true), + }}, + }, + { + []match.Matcher{ + match.NewSingle(nil), + match.NewSingle(nil), + match.NewSingle(nil), + }, + match.EveryOf{match.Matchers{ + match.NewMin(3), + match.NewMax(3), + }}, + }, + { + []match.Matcher{ + match.NewList([]rune{'a'}, true), + match.NewAny([]rune{'a'}), + }, + match.EveryOf{match.Matchers{ + match.NewMin(1), + match.NewContains("a", true), + }}, + }, + } { + act, err := compileMatchers(test.in) + if err != nil { + t.Errorf("#%d convert matchers error: %s", id, err) + continue + } + + if !reflect.DeepEqual(act, test.exp) { + t.Errorf("#%d unexpected convert matchers result:\nact: %#v;\nexp: %#v", id, act, test.exp) + continue + } + } +} + +func TestCompileMatchers(t *testing.T) { + for id, test := range []struct { + in []match.Matcher + exp match.Matcher + }{ + { + []match.Matcher{ + match.NewSuper(), + match.NewSingle(separators), + match.NewText("c"), + }, + match.NewBTree( + match.NewText("c"), + match.NewBTree( + match.NewSingle(separators), + match.NewSuper(), + nil, + ), + nil, + ), + }, + { + []match.Matcher{ + match.NewAny(nil), + match.NewText("c"), + match.NewAny(nil), + }, + match.NewBTree( + match.NewText("c"), + match.NewAny(nil), + match.NewAny(nil), + ), + }, + { + []match.Matcher{ + match.NewRange('a', 'c', true), + match.NewList([]rune{'z', 't', 'e'}, false), + match.NewText("c"), + match.NewSingle(nil), + }, + match.NewRow( + 4, + match.Matchers{ + match.NewRange('a', 'c', true), + match.NewList([]rune{'z', 't', 'e'}, false), + match.NewText("c"), + match.NewSingle(nil), + }..., + ), + }, + } { + act, err := compileMatchers(test.in) + if err != nil { + t.Errorf("#%d convert matchers error: %s", id, err) + continue + } + + if !reflect.DeepEqual(act, test.exp) { + t.Errorf("#%d unexpected convert matchers result:\nact: %#v\nexp: %#v", id, act, test.exp) + continue + } + } +} + +func TestConvertMatchers(t *testing.T) { + for id, test := range []struct { + in, exp []match.Matcher + }{ + { + []match.Matcher{ + match.NewRange('a', 'c', true), + match.NewList([]rune{'z', 't', 'e'}, false), + match.NewText("c"), + match.NewSingle(nil), + match.NewAny(nil), + }, + []match.Matcher{ + match.NewRow( + 4, + []match.Matcher{ + match.NewRange('a', 'c', true), + match.NewList([]rune{'z', 't', 'e'}, false), + match.NewText("c"), + match.NewSingle(nil), + }..., + ), + match.NewAny(nil), + }, + }, + { + []match.Matcher{ + match.NewRange('a', 'c', true), + match.NewList([]rune{'z', 't', 'e'}, false), + match.NewText("c"), + match.NewSingle(nil), + match.NewAny(nil), + match.NewSingle(nil), + match.NewSingle(nil), + match.NewAny(nil), + }, + []match.Matcher{ + match.NewRow( + 3, + match.Matchers{ + match.NewRange('a', 'c', true), + match.NewList([]rune{'z', 't', 'e'}, false), + match.NewText("c"), + }..., + ), + match.NewMin(3), + }, + }, + } { + act := minimizeMatchers(test.in) + if !reflect.DeepEqual(act, test.exp) { + t.Errorf("#%d unexpected convert matchers 2 result:\nact: %#v\nexp: %#v", id, act, test.exp) + continue + } + } +} + +func TestCompiler(t *testing.T) { + for id, test := range []struct { + ast *ast.Node + result match.Matcher + sep []rune + }{ + // { + // ast: ast.NewNode(ast.KindPattern, nil, + // ast.NewNode(ast.KindText, &ast.Text{"abc"}), + // ), + // result: match.NewText("abc"), + // }, + // { + // ast: ast.NewNode(ast.KindPattern, nil, + // ast.NewNode(ast.KindAny, nil), + // ), + // sep: separators, + // result: match.NewAny(separators), + // }, + // { + // ast: ast.NewNode(ast.KindPattern, nil, + // ast.NewNode(ast.KindAny, nil), + // ), + // result: match.NewSuper(), + // }, + // { + // ast: ast.NewNode(ast.KindPattern, nil, + // ast.NewNode(ast.KindSuper, nil), + // ), + // result: match.NewSuper(), + // }, + // { + // ast: ast.NewNode(ast.KindPattern, nil, + // ast.NewNode(ast.KindSingle, nil), + // ), + // sep: separators, + // result: match.NewSingle(separators), + // }, + // { + // ast: ast.NewNode(ast.KindPattern, nil, + // ast.NewNode(ast.KindRange, &ast.Range{ + // Lo: 'a', + // Hi: 'z', + // Not: true, + // }), + // ), + // result: match.NewRange('a', 'z', true), + // }, + // { + // ast: ast.NewNode(ast.KindPattern, nil, + // ast.NewNode(ast.KindList, &ast.List{ + // Chars: "abc", + // Not: true, + // }), + // ), + // result: match.NewList([]rune{'a', 'b', 'c'}, true), + // }, + // { + // ast: ast.NewNode(ast.KindPattern, nil, + // ast.NewNode(ast.KindAny, nil), + // ast.NewNode(ast.KindSingle, nil), + // ast.NewNode(ast.KindSingle, nil), + // ast.NewNode(ast.KindSingle, nil), + // ), + // sep: separators, + // result: match.EveryOf{Matchers: match.Matchers{ + // match.NewMin(3), + // match.NewContains(string(separators), true), + // }}, + // }, + // { + // ast: ast.NewNode(ast.KindPattern, nil, + // ast.NewNode(ast.KindAny, nil), + // ast.NewNode(ast.KindSingle, nil), + // ast.NewNode(ast.KindSingle, nil), + // ast.NewNode(ast.KindSingle, nil), + // ), + // result: match.NewMin(3), + // }, + // { + // ast: ast.NewNode(ast.KindPattern, nil, + // ast.NewNode(ast.KindAny, nil), + // ast.NewNode(ast.KindText, &ast.Text{"abc"}), + // ast.NewNode(ast.KindSingle, nil), + // ), + // sep: separators, + // result: match.NewBTree( + // match.NewRow( + // 4, + // match.Matchers{ + // match.NewText("abc"), + // match.NewSingle(separators), + // }..., + // ), + // match.NewAny(separators), + // nil, + // ), + // }, + // { + // ast: ast.NewNode(ast.KindPattern, nil, + // ast.NewNode(ast.KindText, &ast.Text{"/"}), + // ast.NewNode(ast.KindAnyOf, nil, + // ast.NewNode(ast.KindText, &ast.Text{"z"}), + // ast.NewNode(ast.KindText, &ast.Text{"ab"}), + // ), + // ast.NewNode(ast.KindSuper, nil), + // ), + // sep: separators, + // result: match.NewBTree( + // match.NewText("/"), + // nil, + // match.NewBTree( + // match.NewAnyOf(match.NewText("z"), match.NewText("ab")), + // nil, + // match.NewSuper(), + // ), + // ), + // }, + // { + // ast: ast.NewNode(ast.KindPattern, nil, + // ast.NewNode(ast.KindSuper, nil), + // ast.NewNode(ast.KindSingle, nil), + // ast.NewNode(ast.KindText, &ast.Text{"abc"}), + // ast.NewNode(ast.KindSingle, nil), + // ), + // sep: separators, + // result: match.NewBTree( + // match.NewRow( + // 5, + // match.Matchers{ + // match.NewSingle(separators), + // match.NewText("abc"), + // match.NewSingle(separators), + // }..., + // ), + // match.NewSuper(), + // nil, + // ), + // }, + // { + // ast: ast.NewNode(ast.KindPattern, nil, + // ast.NewNode(ast.KindAny, nil), + // ast.NewNode(ast.KindText, &ast.Text{"abc"}), + // ), + // result: match.NewSuffix("abc"), + // }, + // { + // ast: ast.NewNode(ast.KindPattern, nil, + // ast.NewNode(ast.KindText, &ast.Text{"abc"}), + // ast.NewNode(ast.KindAny, nil), + // ), + // result: match.NewPrefix("abc"), + // }, + // { + // ast: ast.NewNode(ast.KindPattern, nil, + // ast.NewNode(ast.KindText, &ast.Text{"abc"}), + // ast.NewNode(ast.KindAny, nil), + // ast.NewNode(ast.KindText, &ast.Text{"def"}), + // ), + // result: match.NewPrefixSuffix("abc", "def"), + // }, + // { + // ast: ast.NewNode(ast.KindPattern, nil, + // ast.NewNode(ast.KindAny, nil), + // ast.NewNode(ast.KindAny, nil), + // ast.NewNode(ast.KindAny, nil), + // ast.NewNode(ast.KindText, &ast.Text{"abc"}), + // ast.NewNode(ast.KindAny, nil), + // ast.NewNode(ast.KindAny, nil), + // ), + // result: match.NewContains("abc", false), + // }, + // { + // ast: ast.NewNode(ast.KindPattern, nil, + // ast.NewNode(ast.KindAny, nil), + // ast.NewNode(ast.KindAny, nil), + // ast.NewNode(ast.KindAny, nil), + // ast.NewNode(ast.KindText, &ast.Text{"abc"}), + // ast.NewNode(ast.KindAny, nil), + // ast.NewNode(ast.KindAny, nil), + // ), + // sep: separators, + // result: match.NewBTree( + // match.NewText("abc"), + // match.NewAny(separators), + // match.NewAny(separators), + // ), + // }, + // { + // ast: ast.NewNode(ast.KindPattern, nil, + // ast.NewNode(ast.KindSuper, nil), + // ast.NewNode(ast.KindSingle, nil), + // ast.NewNode(ast.KindText, &ast.Text{"abc"}), + // ast.NewNode(ast.KindSuper, nil), + // ast.NewNode(ast.KindSingle, nil), + // ), + // result: match.NewBTree( + // match.NewText("abc"), + // match.NewMin(1), + // match.NewMin(1), + // ), + // }, + // { + // ast: ast.NewNode(ast.KindPattern, nil, + // ast.NewNode(ast.KindText, &ast.Text{"abc"}), + // ), + // result: match.NewText("abc"), + // }, + // { + // ast: ast.NewNode(ast.KindPattern, nil, + // ast.NewNode(ast.KindAnyOf, nil, + // ast.NewNode(ast.KindPattern, nil, + // ast.NewNode(ast.KindAnyOf, nil, + // ast.NewNode(ast.KindPattern, nil, + // ast.NewNode(ast.KindText, &ast.Text{"abc"}), + // ), + // ), + // ), + // ), + // ), + // result: match.NewText("abc"), + // }, + { + ast: ast.NewNode(ast.KindPattern, nil, + ast.NewNode(ast.KindAnyOf, nil, + ast.NewNode(ast.KindPattern, nil, + ast.NewNode(ast.KindText, &ast.Text{"abc"}), + ast.NewNode(ast.KindSingle, nil), + ), + ast.NewNode(ast.KindPattern, nil, + ast.NewNode(ast.KindText, &ast.Text{"abc"}), + ast.NewNode(ast.KindList, &ast.List{Chars: "def"}), + ), + ast.NewNode(ast.KindPattern, nil, + ast.NewNode(ast.KindText, &ast.Text{"abc"}), + ), + ast.NewNode(ast.KindPattern, nil, + ast.NewNode(ast.KindText, &ast.Text{"abc"}), + ), + ), + ), + result: match.NewBTree( + match.NewText("abc"), + nil, + match.AnyOf{Matchers: match.Matchers{ + match.NewSingle(nil), + match.NewList([]rune{'d', 'e', 'f'}, false), + match.NewNothing(), + }}, + ), + }, + // { + // ast: ast.NewNode(ast.KindPattern, nil, + // ast.NewNode(ast.KindRange, &ast.Range{Lo: 'a', Hi: 'z'}), + // ast.NewNode(ast.KindRange, &ast.Range{Lo: 'a', Hi: 'x', Not: true}), + // ast.NewNode(ast.KindAny, nil), + // ), + // result: match.NewBTree( + // match.NewRow( + // 2, + // match.Matchers{ + // match.NewRange('a', 'z', false), + // match.NewRange('a', 'x', true), + // }..., + // ), + // nil, + // match.NewSuper(), + // ), + // }, + // { + // ast: ast.NewNode(ast.KindPattern, nil, + // ast.NewNode(ast.KindAnyOf, nil, + // ast.NewNode(ast.KindPattern, nil, + // ast.NewNode(ast.KindText, &ast.Text{"abc"}), + // ast.NewNode(ast.KindList, &ast.List{Chars: "abc"}), + // ast.NewNode(ast.KindText, &ast.Text{"ghi"}), + // ), + // ast.NewNode(ast.KindPattern, nil, + // ast.NewNode(ast.KindText, &ast.Text{"abc"}), + // ast.NewNode(ast.KindList, &ast.List{Chars: "def"}), + // ast.NewNode(ast.KindText, &ast.Text{"ghi"}), + // ), + // ), + // ), + // result: match.NewRow( + // 7, + // match.Matchers{ + // match.NewText("abc"), + // match.AnyOf{Matchers: match.Matchers{ + // match.NewList([]rune{'a', 'b', 'c'}, false), + // match.NewList([]rune{'d', 'e', 'f'}, false), + // }}, + // match.NewText("ghi"), + // }..., + // ), + // }, + } { + m, err := Compile(test.ast, test.sep) + if err != nil { + t.Errorf("compilation error: %s", err) + continue + } + + if !reflect.DeepEqual(m, test.result) { + t.Errorf("[%d] Compile():\nexp: %#v\nact: %#v\n\ngraphviz:\nexp:\n%s\nact:\n%s\n", id, test.result, m, debug.Graphviz("", test.result.(match.Matcher)), debug.Graphviz("", m.(match.Matcher))) + continue + } + } +} diff --git a/compiler_test.go b/compiler_test.go deleted file mode 100644 index 4c20db2..0000000 --- a/compiler_test.go +++ /dev/null @@ -1,548 +0,0 @@ -package glob - -import ( - "github.com/gobwas/glob/match" - "github.com/gobwas/glob/match/debug" - "reflect" - "testing" -) - -var separators = []rune{'.'} - -func TestGlueMatchers(t *testing.T) { - for id, test := range []struct { - in []match.Matcher - exp match.Matcher - }{ - { - []match.Matcher{ - match.NewSuper(), - match.NewSingle(nil), - }, - match.NewMin(1), - }, - { - []match.Matcher{ - match.NewAny(separators), - match.NewSingle(separators), - }, - match.EveryOf{match.Matchers{ - match.NewMin(1), - match.NewContains(string(separators), true), - }}, - }, - { - []match.Matcher{ - match.NewSingle(nil), - match.NewSingle(nil), - match.NewSingle(nil), - }, - match.EveryOf{match.Matchers{ - match.NewMin(3), - match.NewMax(3), - }}, - }, - { - []match.Matcher{ - match.NewList([]rune{'a'}, true), - match.NewAny([]rune{'a'}), - }, - match.EveryOf{match.Matchers{ - match.NewMin(1), - match.NewContains("a", true), - }}, - }, - } { - act, err := compileMatchers(test.in) - if err != nil { - t.Errorf("#%d convert matchers error: %s", id, err) - continue - } - - if !reflect.DeepEqual(act, test.exp) { - t.Errorf("#%d unexpected convert matchers result:\nact: %#v;\nexp: %#v", id, act, test.exp) - continue - } - } -} - -func TestCompileMatchers(t *testing.T) { - for id, test := range []struct { - in []match.Matcher - exp match.Matcher - }{ - { - []match.Matcher{ - match.NewSuper(), - match.NewSingle(separators), - match.NewText("c"), - }, - match.NewBTree( - match.NewText("c"), - match.NewBTree( - match.NewSingle(separators), - match.NewSuper(), - nil, - ), - nil, - ), - }, - { - []match.Matcher{ - match.NewAny(nil), - match.NewText("c"), - match.NewAny(nil), - }, - match.NewBTree( - match.NewText("c"), - match.NewAny(nil), - match.NewAny(nil), - ), - }, - { - []match.Matcher{ - match.NewRange('a', 'c', true), - match.NewList([]rune{'z', 't', 'e'}, false), - match.NewText("c"), - match.NewSingle(nil), - }, - match.NewRow( - 4, - match.Matchers{ - match.NewRange('a', 'c', true), - match.NewList([]rune{'z', 't', 'e'}, false), - match.NewText("c"), - match.NewSingle(nil), - }..., - ), - }, - } { - act, err := compileMatchers(test.in) - if err != nil { - t.Errorf("#%d convert matchers error: %s", id, err) - continue - } - - if !reflect.DeepEqual(act, test.exp) { - t.Errorf("#%d unexpected convert matchers result:\nact: %#v\nexp: %#v", id, act, test.exp) - continue - } - } -} - -func TestConvertMatchers(t *testing.T) { - for id, test := range []struct { - in, exp []match.Matcher - }{ - { - []match.Matcher{ - match.NewRange('a', 'c', true), - match.NewList([]rune{'z', 't', 'e'}, false), - match.NewText("c"), - match.NewSingle(nil), - match.NewAny(nil), - }, - []match.Matcher{ - match.NewRow( - 4, - []match.Matcher{ - match.NewRange('a', 'c', true), - match.NewList([]rune{'z', 't', 'e'}, false), - match.NewText("c"), - match.NewSingle(nil), - }..., - ), - match.NewAny(nil), - }, - }, - { - []match.Matcher{ - match.NewRange('a', 'c', true), - match.NewList([]rune{'z', 't', 'e'}, false), - match.NewText("c"), - match.NewSingle(nil), - match.NewAny(nil), - match.NewSingle(nil), - match.NewSingle(nil), - match.NewAny(nil), - }, - []match.Matcher{ - match.NewRow( - 3, - match.Matchers{ - match.NewRange('a', 'c', true), - match.NewList([]rune{'z', 't', 'e'}, false), - match.NewText("c"), - }..., - ), - match.NewMin(3), - }, - }, - } { - act := minimizeMatchers(test.in) - if !reflect.DeepEqual(act, test.exp) { - t.Errorf("#%d unexpected convert matchers 2 result:\nact: %#v\nexp: %#v", id, act, test.exp) - continue - } - } -} - -func pattern(nodes ...node) *nodePattern { - return &nodePattern{ - nodeImpl: nodeImpl{ - desc: nodes, - }, - } -} -func anyOf(nodes ...node) *nodeAnyOf { - return &nodeAnyOf{ - nodeImpl: nodeImpl{ - desc: nodes, - }, - } -} -func TestCompiler(t *testing.T) { - for id, test := range []struct { - ast *nodePattern - result Glob - sep []rune - }{ - { - ast: pattern(&nodeText{text: "abc"}), - result: match.NewText("abc"), - }, - { - ast: pattern(&nodeAny{}), - sep: separators, - result: match.NewAny(separators), - }, - { - ast: pattern(&nodeAny{}), - result: match.NewSuper(), - }, - { - ast: pattern(&nodeSuper{}), - result: match.NewSuper(), - }, - { - ast: pattern(&nodeSingle{}), - sep: separators, - result: match.NewSingle(separators), - }, - { - ast: pattern(&nodeRange{ - lo: 'a', - hi: 'z', - not: true, - }), - result: match.NewRange('a', 'z', true), - }, - { - ast: pattern(&nodeList{ - chars: "abc", - not: true, - }), - result: match.NewList([]rune{'a', 'b', 'c'}, true), - }, - { - ast: pattern(&nodeAny{}, &nodeSingle{}, &nodeSingle{}, &nodeSingle{}), - sep: separators, - result: match.EveryOf{Matchers: match.Matchers{ - match.NewMin(3), - match.NewContains(string(separators), true), - }}, - }, - { - ast: pattern(&nodeAny{}, &nodeSingle{}, &nodeSingle{}, &nodeSingle{}), - result: match.NewMin(3), - }, - { - ast: pattern(&nodeAny{}, &nodeText{text: "abc"}, &nodeSingle{}), - sep: separators, - result: match.NewBTree( - match.NewRow( - 4, - match.Matchers{ - match.NewText("abc"), - match.NewSingle(separators), - }..., - ), - match.NewAny(separators), - nil, - ), - }, - { - ast: pattern(&nodeText{text: "/"}, anyOf(&nodeText{text: "z"}, &nodeText{text: "ab"}), &nodeSuper{}), - sep: separators, - result: match.NewBTree( - match.NewText("/"), - nil, - match.NewBTree( - match.NewAnyOf(match.NewText("z"), match.NewText("ab")), - nil, - match.NewSuper(), - ), - ), - }, - { - ast: pattern(&nodeSuper{}, &nodeSingle{}, &nodeText{text: "abc"}, &nodeSingle{}), - sep: separators, - result: match.NewBTree( - match.NewRow( - 5, - match.Matchers{ - match.NewSingle(separators), - match.NewText("abc"), - match.NewSingle(separators), - }..., - ), - match.NewSuper(), - nil, - ), - }, - { - ast: pattern(&nodeAny{}, &nodeText{text: "abc"}), - result: match.NewSuffix("abc"), - }, - { - ast: pattern(&nodeText{text: "abc"}, &nodeAny{}), - result: match.NewPrefix("abc"), - }, - { - ast: pattern(&nodeText{text: "abc"}, &nodeAny{}, &nodeText{text: "def"}), - result: match.NewPrefixSuffix("abc", "def"), - }, - { - ast: pattern(&nodeAny{}, &nodeAny{}, &nodeAny{}, &nodeText{text: "abc"}, &nodeAny{}, &nodeAny{}), - result: match.NewContains("abc", false), - }, - { - ast: pattern(&nodeAny{}, &nodeAny{}, &nodeAny{}, &nodeText{text: "abc"}, &nodeAny{}, &nodeAny{}), - sep: separators, - result: match.NewBTree( - match.NewText("abc"), - match.NewAny(separators), - match.NewAny(separators), - ), - }, - { - ast: pattern(&nodeSuper{}, &nodeSingle{}, &nodeText{text: "abc"}, &nodeSuper{}, &nodeSingle{}), - result: match.NewBTree( - match.NewText("abc"), - match.NewMin(1), - match.NewMin(1), - ), - }, - { - ast: pattern(anyOf(&nodeText{text: "abc"})), - result: match.NewText("abc"), - }, - { - ast: pattern(anyOf(pattern(anyOf(pattern(&nodeText{text: "abc"}))))), - result: match.NewText("abc"), - }, - { - ast: pattern(anyOf( - pattern( - &nodeText{text: "abc"}, - &nodeSingle{}, - ), - pattern( - &nodeText{text: "abc"}, - &nodeList{chars: "def"}, - ), - pattern( - &nodeText{text: "abc"}, - ), - pattern( - &nodeText{text: "abc"}, - ), - )), - result: match.NewBTree( - match.NewText("abc"), - nil, - match.AnyOf{Matchers: match.Matchers{ - match.NewSingle(nil), - match.NewList([]rune{'d', 'e', 'f'}, false), - match.NewNothing(), - }}, - ), - }, - { - ast: pattern( - &nodeRange{lo: 'a', hi: 'z'}, - &nodeRange{lo: 'a', hi: 'x', not: true}, - &nodeAny{}, - ), - result: match.NewBTree( - match.NewRow( - 2, - match.Matchers{ - match.NewRange('a', 'z', false), - match.NewRange('a', 'x', true), - }..., - ), - nil, - match.NewSuper(), - ), - }, - { - ast: pattern(anyOf( - pattern( - &nodeText{text: "abc"}, - &nodeList{chars: "abc"}, - &nodeText{text: "ghi"}, - ), - pattern( - &nodeText{text: "abc"}, - &nodeList{chars: "def"}, - &nodeText{text: "ghi"}, - ), - )), - result: match.NewRow( - 7, - match.Matchers{ - match.NewText("abc"), - match.AnyOf{Matchers: match.Matchers{ - match.NewList([]rune{'a', 'b', 'c'}, false), - match.NewList([]rune{'d', 'e', 'f'}, false), - }}, - match.NewText("ghi"), - }..., - ), - }, - } { - m, err := compile(test.ast, test.sep) - if err != nil { - t.Errorf("compilation error: %s", err) - continue - } - - if !reflect.DeepEqual(m, test.result) { - t.Errorf("#%d results are not equal:\nexp: %#v\nact: %#v\nexp:\n%s\nact:\n%s\n", id, test.result, m, debug.Graphviz("", test.result.(match.Matcher)), debug.Graphviz("", m.(match.Matcher))) - continue - } - } -} - -const complexityString = "abcd" - -//func BenchmarkComplexityAny(b *testing.B) { -// m := match.NewAny(nil) -// for i := 0; i < b.N; i++ { -// _ = m.Match(complexityString) -// _, _ = m.Index(complexityString) -// } -//} -//func BenchmarkComplexityContains(b *testing.B) { -// m := match.NewContains() -// for i := 0; i < b.N; i++ { -// _ = m.Match(complexityString) -// _, _ = m.Index(complexityString) -// } -//} -//func BenchmarkComplexityList(b *testing.B) { -// m := match.NewList() -// for i := 0; i < b.N; i++ { -// _ = m.Match(complexityString) -// _, _ = m.Index(complexityString) -// } -//} -//func BenchmarkComplexityMax(b *testing.B) { -// m := match.NewMax() -// for i := 0; i < b.N; i++ { -// _ = m.Match(complexityString) -// _, _ = m.Index(complexityString) -// } -//} -//func BenchmarkComplexityMin(b *testing.B) { -// m := match.NewMin() -// for i := 0; i < b.N; i++ { -// _ = m.Match(complexityString) -// _, _ = m.Index(complexityString) -// } -//} -//func BenchmarkComplexityNothing(b *testing.B) { -// m := match.NewNothing() -// for i := 0; i < b.N; i++ { -// _ = m.Match(complexityString) -// _, _ = m.Index(complexityString) -// } -//} -//func BenchmarkComplexityPrefix(b *testing.B) { -// m := match.NewPrefix() -// for i := 0; i < b.N; i++ { -// _ = m.Match(complexityString) -// _, _ = m.Index(complexityString) -// } -//} -//func BenchmarkComplexityPrefixSuffix(b *testing.B) { -// m := match.NewPrefixSuffix() -// for i := 0; i < b.N; i++ { -// _ = m.Match(complexityString) -// _, _ = m.Index(complexityString) -// } -//} -//func BenchmarkComplexityRange(b *testing.B) { -// m := match.NewRange() -// for i := 0; i < b.N; i++ { -// _ = m.Match(complexityString) -// _, _ = m.Index(complexityString) -// } -//} -//func BenchmarkComplexityRow(b *testing.B) { -// m := match.NewRow() -// for i := 0; i < b.N; i++ { -// _ = m.Match(complexityString) -// _, _ = m.Index(complexityString) -// } -//} -//func BenchmarkComplexitySingle(b *testing.B) { -// m := match.NewSingle(nil) -// for i := 0; i < b.N; i++ { -// _ = m.Match(complexityString) -// _, _ = m.Index(complexityString) -// } -//} -//func BenchmarkComplexitySuffix(b *testing.B) { -// m := match.NewSuffix() -// for i := 0; i < b.N; i++ { -// _ = m.Match(complexityString) -// _, _ = m.Index(complexityString) -// } -//} -//func BenchmarkComplexitySuper(b *testing.B) { -// m := match.NewSuper() -// for i := 0; i < b.N; i++ { -// _ = m.Match(complexityString) -// _, _ = m.Index(complexityString) -// } -//} -//func BenchmarkComplexityText(b *testing.B) { -// m := match.NewText() -// for i := 0; i < b.N; i++ { -// _ = m.Match(complexityString) -// _, _ = m.Index(complexityString) -// } -//} -//func BenchmarkComplexityAnyOf(b *testing.B) { -// m := match.NewAnyOf() -// for i := 0; i < b.N; i++ { -// _ = m.Match(complexityString) -// _, _ = m.Index(complexityString) -// } -//} -//func BenchmarkComplexityBTree(b *testing.B) { -// m := match.NewBTree(match.NewText("abc"), match.NewText("d"), match.NewText("e")) -// for i := 0; i < b.N; i++ { -// _ = m.Match(complexityString) -// _, _ = m.Index(complexityString) -// } -//} -//func BenchmarkComplexityEveryOf(b *testing.B) { -// m := match.NewEveryOf() -// for i := 0; i < b.N; i++ { -// _ = m.Match(complexityString) -// _, _ = m.Index(complexityString) -// } -//} diff --git a/glob.go b/glob.go index 58f45c9..9eb9d58 100644 --- a/glob.go +++ b/glob.go @@ -1,5 +1,10 @@ package glob +import ( + "github.com/gobwas/glob/parser" + "github.com/gobwas/glob/syntax" +) + // Glob represents compiled glob pattern. type Glob interface { Match(string) bool @@ -32,7 +37,7 @@ type Glob interface { // comma-separated (without spaces) patterns // func Compile(pattern string, separators ...rune) (Glob, error) { - ast, err := parse(newLexer(pattern)) + ast, err := syntax.Parse(pattern) if err != nil { return nil, err } @@ -63,7 +68,7 @@ func QuoteMeta(s string) string { // a byte loop is correct because all meta characters are ASCII j := 0 for i := 0; i < len(s); i++ { - if special(s[i]) { + if syntax.Special(s[i]) { b[j] = '\\' j++ } diff --git a/match/any.go b/match/any.go index 1d2d12b..514a9a5 100644 --- a/match/any.go +++ b/match/any.go @@ -2,7 +2,7 @@ package match import ( "fmt" - "github.com/gobwas/glob/strings" + "github.com/gobwas/glob/util/strings" ) type Any struct { diff --git a/match/list.go b/match/list.go index fe0841f..7fd763e 100644 --- a/match/list.go +++ b/match/list.go @@ -2,7 +2,7 @@ package match import ( "fmt" - "github.com/gobwas/glob/runes" + "github.com/gobwas/glob/util/runes" "unicode/utf8" ) diff --git a/match/single.go b/match/single.go index 33e926d..ee6e395 100644 --- a/match/single.go +++ b/match/single.go @@ -2,7 +2,7 @@ package match import ( "fmt" - "github.com/gobwas/glob/runes" + "github.com/gobwas/glob/util/runes" "unicode/utf8" ) diff --git a/parser/ast.go b/parser/ast.go deleted file mode 100644 index 588a45c..0000000 --- a/parser/ast.go +++ /dev/null @@ -1,48 +0,0 @@ -package parser - -type Node interface { - Children() []Node - Parent() Node - append(Node) Node -} - -type node struct { - parent Node - children []Node -} - -func (n *node) Children() []Node { - return n.children -} - -func (n *node) Parent() Node { - return n.parent -} - -func (n *node) append(c Node) Node { - n.children = append(n.children, c) - return c -} - -type ListNode struct { - node - Not bool - Chars string -} - -type RangeNode struct { - node - Not bool - Lo, Hi rune -} - -type TextNode struct { - node - Text string -} - -type PatternNode struct{ node } -type AnyNode struct{ node } -type SuperNode struct{ node } -type SingleNode struct{ node } -type AnyOfNode struct{ node } diff --git a/parser/parser_test.go b/parser/parser_test.go deleted file mode 100644 index 177d7bf..0000000 --- a/parser/parser_test.go +++ /dev/null @@ -1,332 +0,0 @@ -package parser - -import ( - "fmt" - "github.com/gobwas/glob/lexer" - "reflect" - "testing" -) - -type stubLexer struct { - tokens []lexer.Token - pos int -} - -func (s *stubLexer) Next() (ret lexer.Token) { - if s.pos == len(s.tokens) { - return lexer.Token{lexer.EOF, ""} - } - ret = s.tokens[s.pos] - s.pos++ - return -} - -func TestParseString(t *testing.T) { - for id, test := range []struct { - tokens []lexer.Token - tree Node - }{ - { - //pattern: "abc", - tokens: []lexer.Token{ - lexer.Token{lexer.Text, "abc"}, - lexer.Token{lexer.EOF, ""}, - }, - tree: &PatternNode{ - node: node{ - children: []Node{ - &TextNode{Text: "abc"}, - }, - }, - }, - }, - { - //pattern: "a*c", - tokens: []lexer.Token{ - lexer.Token{lexer.Text, "a"}, - lexer.Token{lexer.Any, "*"}, - lexer.Token{lexer.Text, "c"}, - lexer.Token{lexer.EOF, ""}, - }, - tree: &PatternNode{ - node: node{ - children: []Node{ - &TextNode{Text: "a"}, - &AnyNode{}, - &TextNode{Text: "c"}, - }, - }, - }, - }, - { - //pattern: "a**c", - tokens: []lexer.Token{ - lexer.Token{lexer.Text, "a"}, - lexer.Token{lexer.Super, "**"}, - lexer.Token{lexer.Text, "c"}, - lexer.Token{lexer.EOF, ""}, - }, - tree: &PatternNode{ - node: node{ - children: []Node{ - &TextNode{Text: "a"}, - &SuperNode{}, - &TextNode{Text: "c"}, - }, - }, - }, - }, - { - //pattern: "a?c", - tokens: []lexer.Token{ - lexer.Token{lexer.Text, "a"}, - lexer.Token{lexer.Single, "?"}, - lexer.Token{lexer.Text, "c"}, - lexer.Token{lexer.EOF, ""}, - }, - tree: &PatternNode{ - node: node{ - children: []Node{ - &TextNode{Text: "a"}, - &SingleNode{}, - &TextNode{Text: "c"}, - }, - }, - }, - }, - { - //pattern: "[!a-z]", - tokens: []lexer.Token{ - lexer.Token{lexer.RangeOpen, "["}, - lexer.Token{lexer.Not, "!"}, - lexer.Token{lexer.RangeLo, "a"}, - lexer.Token{lexer.RangeBetween, "-"}, - lexer.Token{lexer.RangeHi, "z"}, - lexer.Token{lexer.RangeClose, "]"}, - lexer.Token{lexer.EOF, ""}, - }, - tree: &PatternNode{ - node: node{ - children: []Node{ - &RangeNode{Lo: 'a', Hi: 'z', Not: true}, - }, - }, - }, - }, - { - //pattern: "[az]", - tokens: []lexer.Token{ - lexer.Token{lexer.RangeOpen, "["}, - lexer.Token{lexer.Text, "az"}, - lexer.Token{lexer.RangeClose, "]"}, - lexer.Token{lexer.EOF, ""}, - }, - tree: &PatternNode{ - node: node{ - children: []Node{ - &ListNode{Chars: "az"}, - }, - }, - }, - }, - { - //pattern: "{a,z}", - tokens: []lexer.Token{ - lexer.Token{lexer.TermsOpen, "{"}, - lexer.Token{lexer.Text, "a"}, - lexer.Token{lexer.Separator, ","}, - lexer.Token{lexer.Text, "z"}, - lexer.Token{lexer.TermsClose, "}"}, - lexer.Token{lexer.EOF, ""}, - }, - tree: &PatternNode{ - node: node{ - children: []Node{ - &AnyOfNode{node: node{children: []Node{ - &PatternNode{ - node: node{children: []Node{ - &TextNode{Text: "a"}, - }}, - }, - &PatternNode{ - node: node{children: []Node{ - &TextNode{Text: "z"}, - }}, - }, - }}}, - }, - }, - }, - }, - { - //pattern: "/{z,ab}*", - tokens: []lexer.Token{ - lexer.Token{lexer.Text, "/"}, - lexer.Token{lexer.TermsOpen, "{"}, - lexer.Token{lexer.Text, "z"}, - lexer.Token{lexer.Separator, ","}, - lexer.Token{lexer.Text, "ab"}, - lexer.Token{lexer.TermsClose, "}"}, - lexer.Token{lexer.Any, "*"}, - lexer.Token{lexer.EOF, ""}, - }, - tree: &PatternNode{ - node: node{ - children: []Node{ - &TextNode{Text: "/"}, - &AnyOfNode{node: node{children: []Node{ - &PatternNode{ - node: node{children: []Node{ - &TextNode{Text: "z"}, - }}, - }, - &PatternNode{ - node: node{children: []Node{ - &TextNode{Text: "ab"}, - }}, - }, - }}}, - &AnyNode{}, - }, - }, - }, - }, - { - //pattern: "{a,{x,y},?,[a-z],[!qwe]}", - tokens: []lexer.Token{ - lexer.Token{lexer.TermsOpen, "{"}, - lexer.Token{lexer.Text, "a"}, - lexer.Token{lexer.Separator, ","}, - lexer.Token{lexer.TermsOpen, "{"}, - lexer.Token{lexer.Text, "x"}, - lexer.Token{lexer.Separator, ","}, - lexer.Token{lexer.Text, "y"}, - lexer.Token{lexer.TermsClose, "}"}, - lexer.Token{lexer.Separator, ","}, - lexer.Token{lexer.Single, "?"}, - lexer.Token{lexer.Separator, ","}, - lexer.Token{lexer.RangeOpen, "["}, - lexer.Token{lexer.RangeLo, "a"}, - lexer.Token{lexer.RangeBetween, "-"}, - lexer.Token{lexer.RangeHi, "z"}, - lexer.Token{lexer.RangeClose, "]"}, - lexer.Token{lexer.Separator, ","}, - lexer.Token{lexer.RangeOpen, "["}, - lexer.Token{lexer.Not, "!"}, - lexer.Token{lexer.Text, "qwe"}, - lexer.Token{lexer.RangeClose, "]"}, - lexer.Token{lexer.TermsClose, "}"}, - lexer.Token{lexer.EOF, ""}, - }, - tree: &PatternNode{ - node: node{ - children: []Node{ - &AnyOfNode{node: node{children: []Node{ - &PatternNode{ - node: node{children: []Node{ - &TextNode{Text: "a"}, - }}, - }, - &PatternNode{ - node: node{children: []Node{ - &AnyOfNode{node: node{children: []Node{ - &PatternNode{ - node: node{children: []Node{ - &TextNode{Text: "x"}, - }}, - }, - &PatternNode{ - node: node{children: []Node{ - &TextNode{Text: "y"}, - }}, - }, - }}}, - }}, - }, - &PatternNode{ - node: node{children: []Node{ - &SingleNode{}, - }}, - }, - &PatternNode{ - node: node{ - children: []Node{ - &RangeNode{Lo: 'a', Hi: 'z', Not: false}, - }, - }, - }, - &PatternNode{ - node: node{ - children: []Node{ - &ListNode{Chars: "qwe", Not: true}, - }, - }, - }, - }}}, - }, - }, - }, - }, - } { - lexer := &stubLexer{tokens: test.tokens} - result, err := Parse(lexer) - if err != nil { - t.Errorf("[%d] unexpected error: %s", id, err) - } - if !reflect.DeepEqual(test.tree, result) { - t.Errorf("[%d] Parse():\nact:\t%s\nexp:\t%s\n", id, result, test.tree) - } - } -} - -const abstractNodeImpl = "nodeImpl" - -func nodeEqual(a, b Node) error { - if (a == nil || b == nil) && a != b { - return fmt.Errorf("nodes are not equal: exp %s, act %s", a, b) - } - - aValue, bValue := reflect.Indirect(reflect.ValueOf(a)), reflect.Indirect(reflect.ValueOf(b)) - aType, bType := aValue.Type(), bValue.Type() - if aType != bType { - return fmt.Errorf("nodes are not equal: exp %s, act %s", aValue.Type(), bValue.Type()) - } - - for i := 0; i < aType.NumField(); i++ { - var eq bool - - f := aType.Field(i).Name - if f == abstractNodeImpl { - continue - } - - af, bf := aValue.FieldByName(f), bValue.FieldByName(f) - - switch af.Kind() { - case reflect.String: - eq = af.String() == bf.String() - case reflect.Bool: - eq = af.Bool() == bf.Bool() - default: - eq = fmt.Sprint(af) == fmt.Sprint(bf) - } - - if !eq { - return fmt.Errorf("nodes<%s> %q fields are not equal: exp %q, act %q", aType, f, af, bf) - } - } - - for i, aDesc := range a.Children() { - if len(b.Children())-1 < i { - return fmt.Errorf("node does not have enough children (got %d children, wanted %d-th token)", len(b.Children()), i) - } - - bDesc := b.Children()[i] - - if err := nodeEqual(aDesc, bDesc); err != nil { - return err - } - } - - return nil -} diff --git a/syntax/ast/ast.go b/syntax/ast/ast.go new file mode 100644 index 0000000..3e499bc --- /dev/null +++ b/syntax/ast/ast.go @@ -0,0 +1,85 @@ +package ast + +type Visitor interface { + Visit(*Node) Visitor +} + +func Walk(v Visitor, n *Node) { + if v = v.Visit(n); v == nil { + return + } + for _, c := range n.Children { + Walk(v, c) + } +} + +type Node struct { + Parent *Node + Children []*Node + Value interface{} + Kind Kind +} + +func NewNode(k Kind, v interface{}, ch ...*Node) *Node { + n := &Node{ + Kind: k, + Value: v, + } + for _, c := range ch { + Insert(n, c) + } + return n +} + +func (a *Node) Equal(b *Node) bool { + if a.Kind != b.Kind { + return false + } + if a.Value != b.Value { + return false + } + if len(a.Children) != len(b.Children) { + return false + } + for i, c := range a.Children { + if !c.Equal(b.Children[i]) { + return false + } + } + return true +} + +func Insert(parent *Node, children ...*Node) { + parent.Children = append(parent.Children, children...) + for _, ch := range children { + ch.Parent = parent + } +} + +type List struct { + Not bool + Chars string +} + +type Range struct { + Not bool + Lo, Hi rune +} + +type Text struct { + Text string +} + +type Kind int + +const ( + KindNothing Kind = iota + KindPattern + KindList + KindRange + KindText + KindAny + KindSuper + KindSingle + KindAnyOf +) diff --git a/parser/parser.go b/syntax/ast/parser.go similarity index 66% rename from parser/parser.go rename to syntax/ast/parser.go index 6adea39..93caa33 100644 --- a/parser/parser.go +++ b/syntax/ast/parser.go @@ -1,9 +1,9 @@ -package parser +package ast import ( "errors" "fmt" - "github.com/gobwas/glob/lexer" + "github.com/gobwas/glob/syntax/lexer" "unicode/utf8" ) @@ -11,15 +11,15 @@ type Lexer interface { Next() lexer.Token } -type parseFn func(Node, Lexer) (parseFn, Node, error) +type parseFn func(*Node, Lexer) (parseFn, *Node, error) -func Parse(lexer Lexer) (*PatternNode, error) { +func Parse(lexer Lexer) (*Node, error) { var parser parseFn - root := &PatternNode{} + root := NewNode(KindPattern, nil) var ( - tree Node + tree *Node err error ) for parser, tree = parserMain, root; parser != nil; { @@ -32,7 +32,7 @@ func Parse(lexer Lexer) (*PatternNode, error) { return root, nil } -func parserMain(tree Node, lex Lexer) (parseFn, Node, error) { +func parserMain(tree *Node, lex Lexer) (parseFn, *Node, error) { for { token := lex.Next() switch token.Type { @@ -43,28 +43,41 @@ func parserMain(tree Node, lex Lexer) (parseFn, Node, error) { return nil, tree, errors.New(token.Raw) case lexer.Text: - return parserMain, tree.append(&TextNode{Text: token.Raw}), nil + Insert(tree, NewNode(KindText, &Text{token.Raw})) + return parserMain, tree, nil case lexer.Any: - return parserMain, tree.append(&AnyNode{}), nil + Insert(tree, NewNode(KindAny, nil)) + return parserMain, tree, nil case lexer.Super: - return parserMain, tree.append(&SuperNode{}), nil + Insert(tree, NewNode(KindSuper, nil)) + return parserMain, tree, nil case lexer.Single: - return parserMain, tree.append(&SingleNode{}), nil + Insert(tree, NewNode(KindSingle, nil)) + return parserMain, tree, nil case lexer.RangeOpen: return parserRange, tree, nil case lexer.TermsOpen: - return parserMain, tree.append(&AnyOfNode{}).append(&PatternNode{}), nil + a := NewNode(KindAnyOf, nil) + Insert(tree, a) + + p := NewNode(KindPattern, nil) + Insert(a, p) + + return parserMain, p, nil case lexer.Separator: - return parserMain, tree.Parent().append(&PatternNode{}), nil + p := NewNode(KindPattern, nil) + Insert(tree.Parent, p) + + return parserMain, p, nil case lexer.TermsClose: - return parserMain, tree.Parent().Parent(), nil + return parserMain, tree.Parent.Parent, nil default: return nil, tree, fmt.Errorf("unexpected token: %s", token) @@ -73,7 +86,7 @@ func parserMain(tree Node, lex Lexer) (parseFn, Node, error) { return nil, tree, fmt.Errorf("unknown error") } -func parserRange(tree Node, lex Lexer) (parseFn, Node, error) { +func parserRange(tree *Node, lex Lexer) (parseFn, *Node, error) { var ( not bool lo rune @@ -126,16 +139,16 @@ func parserRange(tree Node, lex Lexer) (parseFn, Node, error) { } if isRange { - tree = tree.append(&RangeNode{ + Insert(tree, NewNode(KindRange, &Range{ Lo: lo, Hi: hi, Not: not, - }) + })) } else { - tree = tree.append(&ListNode{ + Insert(tree, NewNode(KindList, &List{ Chars: chars, Not: not, - }) + })) } return parserMain, tree, nil diff --git a/syntax/ast/parser_test.go b/syntax/ast/parser_test.go new file mode 100644 index 0000000..84e8d45 --- /dev/null +++ b/syntax/ast/parser_test.go @@ -0,0 +1,256 @@ +package ast + +import ( + "github.com/gobwas/glob/syntax" + "reflect" + "testing" +) + +type stubLexer struct { + tokens []syntax.Token + pos int +} + +func (s *stubLexer) Next() (ret syntax.Token) { + if s.pos == len(s.tokens) { + return syntax.Token{syntax.EOF, ""} + } + ret = s.tokens[s.pos] + s.pos++ + return +} + +func TestParseString(t *testing.T) { + for id, test := range []struct { + tokens []syntax.Token + tree Node + }{ + { + //pattern: "abc", + tokens: []syntax.Token{ + syntax.Token{syntax.Text, "abc"}, + syntax.Token{syntax.EOF, ""}, + }, + tree: NewNode(KindPattern, nil, + NewNode(KindText, &Text{Text: "abc"}), + ), + }, + { + //pattern: "a*c", + tokens: []syntax.Token{ + syntax.Token{syntax.Text, "a"}, + syntax.Token{syntax.Any, "*"}, + syntax.Token{syntax.Text, "c"}, + syntax.Token{syntax.EOF, ""}, + }, + tree: NewNode(KindPattern, nil, + NewNode(KindText, &Text{Text: "a"}), + NewNode(KindAny, nil), + NewNode(KindText, &Text{Text: "c"}), + ), + }, + { + //pattern: "a**c", + tokens: []syntax.Token{ + syntax.Token{syntax.Text, "a"}, + syntax.Token{syntax.Super, "**"}, + syntax.Token{syntax.Text, "c"}, + syntax.Token{syntax.EOF, ""}, + }, + tree: NewNode(KindPattern, nil, + NewNode(KindText, &Text{Text: "a"}), + NewNode(KindSuper, nil), + NewNode(KindText, &Text{Text: "c"}), + ), + }, + { + //pattern: "a?c", + tokens: []syntax.Token{ + syntax.Token{syntax.Text, "a"}, + syntax.Token{syntax.Single, "?"}, + syntax.Token{syntax.Text, "c"}, + syntax.Token{syntax.EOF, ""}, + }, + tree: NewNode(KindPattern, nil, + NewNode(KindText, &Text{Text: "a"}), + NewNode(KindSingle, nil), + NewNode(KindText, &Text{Text: "c"}), + ), + }, + { + //pattern: "[!a-z]", + tokens: []syntax.Token{ + syntax.Token{syntax.RangeOpen, "["}, + syntax.Token{syntax.Not, "!"}, + syntax.Token{syntax.RangeLo, "a"}, + syntax.Token{syntax.RangeBetween, "-"}, + syntax.Token{syntax.RangeHi, "z"}, + syntax.Token{syntax.RangeClose, "]"}, + syntax.Token{syntax.EOF, ""}, + }, + tree: NewNode(KindPattern, nil, + NewNode(KindRange, &Range{Lo: 'a', Hi: 'z', Not: true}), + ), + }, + { + //pattern: "[az]", + tokens: []syntax.Token{ + syntax.Token{syntax.RangeOpen, "["}, + syntax.Token{syntax.Text, "az"}, + syntax.Token{syntax.RangeClose, "]"}, + syntax.Token{syntax.EOF, ""}, + }, + tree: NewNode(KindPattern, nil, + NewNode(KindList, &List{Chars: "az"}), + ), + }, + { + //pattern: "{a,z}", + tokens: []syntax.Token{ + syntax.Token{syntax.TermsOpen, "{"}, + syntax.Token{syntax.Text, "a"}, + syntax.Token{syntax.Separator, ","}, + syntax.Token{syntax.Text, "z"}, + syntax.Token{syntax.TermsClose, "}"}, + syntax.Token{syntax.EOF, ""}, + }, + tree: NewNode(KindPattern, nil, + NewNode(KindAnyOf, nil, + NewNode(KindPattern, nil, + NewNode(KindText, &Text{Text: "a"}), + ), + NewNode(KindPattern, nil, + NewNode(KindText, &Text{Text: "z"}), + ), + ), + ), + }, + { + //pattern: "/{z,ab}*", + tokens: []syntax.Token{ + syntax.Token{syntax.Text, "/"}, + syntax.Token{syntax.TermsOpen, "{"}, + syntax.Token{syntax.Text, "z"}, + syntax.Token{syntax.Separator, ","}, + syntax.Token{syntax.Text, "ab"}, + syntax.Token{syntax.TermsClose, "}"}, + syntax.Token{syntax.Any, "*"}, + syntax.Token{syntax.EOF, ""}, + }, + tree: NewNode(KindPattern, nil, + NewNode(KindText, &Text{Text: "/"}), + NewNode(KindAnyOf, nil, + NewNode(KindPattern, nil, + NewNode(KindText, &Text{Text: "z"}), + ), + NewNode(KindPattern, nil, + NewNode(KindText, &Text{Text: "ab"}), + ), + ), + NewNode(KindAny, nil), + ), + }, + { + //pattern: "{a,{x,y},?,[a-z],[!qwe]}", + tokens: []syntax.Token{ + syntax.Token{syntax.TermsOpen, "{"}, + syntax.Token{syntax.Text, "a"}, + syntax.Token{syntax.Separator, ","}, + syntax.Token{syntax.TermsOpen, "{"}, + syntax.Token{syntax.Text, "x"}, + syntax.Token{syntax.Separator, ","}, + syntax.Token{syntax.Text, "y"}, + syntax.Token{syntax.TermsClose, "}"}, + syntax.Token{syntax.Separator, ","}, + syntax.Token{syntax.Single, "?"}, + syntax.Token{syntax.Separator, ","}, + syntax.Token{syntax.RangeOpen, "["}, + syntax.Token{syntax.RangeLo, "a"}, + syntax.Token{syntax.RangeBetween, "-"}, + syntax.Token{syntax.RangeHi, "z"}, + syntax.Token{syntax.RangeClose, "]"}, + syntax.Token{syntax.Separator, ","}, + syntax.Token{syntax.RangeOpen, "["}, + syntax.Token{syntax.Not, "!"}, + syntax.Token{syntax.Text, "qwe"}, + syntax.Token{syntax.RangeClose, "]"}, + syntax.Token{syntax.TermsClose, "}"}, + syntax.Token{syntax.EOF, ""}, + }, + tree: NewNode(KindPattern, nil, + NewNode(KindAnyOf, nil, + NewNode(KindPattern, nil, + NewNode(KindText, &Text{Text: "a"}), + ), + NewNode(KindPattern, nil, + NewNode(KindAnyOf, nil, + NewNode(KindPattern, nil, + NewNode(KindText, &Text{Text: "x"}), + ), + NewNode(KindPattern, nil, + NewNode(KindText, &Text{Text: "y"}), + ), + ), + ), + NewNode(KindPattern, nil, + NewNode(KindSingle, nil), + ), + NewNode(KindPattern, nil, + NewNode(KindRange, &Range{Lo: 'a', Hi: 'z', Not: false}), + ), + NewNode(KindPattern, nil, + NewNode(KindList, &List{Chars: "qwe", Not: true}), + ), + ), + ), + }, + } { + lexer := &stubLexer{tokens: test.tokens} + result, err := Parse(lexer) + if err != nil { + t.Errorf("[%d] unexpected error: %s", id, err) + } + if !reflect.DeepEqual(test.tree, result) { + t.Errorf("[%d] Parse():\nact:\t%s\nexp:\t%s\n", id, result, test.tree) + } + } +} + +type kv struct { + kind Kind + value interface{} +} + +type visitor struct { + visited []kv +} + +func (v *visitor) Visit(n Node) Visitor { + v.visited = append(v.visited, kv{n.Kind(), n.Value()}) + return v +} + +func TestWalkTree(t *testing.T) { + + for i, test := range []struct { + tree *Node + visited []kv + }{ + { + tree: NewNode(KindPattern, nil, + NewNode(KindSingle, nil), + ), + visited: []kv{ + kv{KindPattern, nil}, + kv{KindSingle, nil}, + }, + }, + } { + v := &visitor{} + Walk(v, test.tree) + + if !reflect.DeepEqual(test.visited, v.visited) { + t.Errorf("[%d] unexpected result of Walk():\nvisited:\t%v\nwant:\t\t%v", i, v.visited, test.visited) + } + } +} diff --git a/lexer/lexer.go b/syntax/lexer/lexer.go similarity index 97% rename from lexer/lexer.go rename to syntax/lexer/lexer.go index e074b09..a1c8d19 100644 --- a/lexer/lexer.go +++ b/syntax/lexer/lexer.go @@ -66,7 +66,7 @@ type lexer struct { hasRune bool } -func newLexer(source string) *lexer { +func NewLexer(source string) *lexer { l := &lexer{ data: source, tokens: tokens(make([]Token, 0, 4)), @@ -74,6 +74,18 @@ func newLexer(source string) *lexer { return l } +func (l *lexer) Next() Token { + if l.err != nil { + return Token{Error, l.err.Error()} + } + if !l.tokens.empty() { + return l.tokens.shift() + } + + l.fetchItem() + return l.Next() +} + func (l *lexer) peek() (r rune, w int) { if l.pos == len(l.data) { return eof, 0 @@ -134,18 +146,6 @@ func (l *lexer) termsLeave() { l.termsLevel-- } -func (l *lexer) nextItem() Token { - if l.err != nil { - return Token{Error, l.err.Error()} - } - if !l.tokens.empty() { - return l.tokens.shift() - } - - l.fetchItem() - return l.nextItem() -} - var inTextBreakers = []rune{char_single, char_any, char_range_open, char_terms_open} var inTermsBreakers = append(inTextBreakers, char_terms_close, char_comma) diff --git a/lexer/lexer_test.go b/syntax/lexer/lexer_test.go similarity index 98% rename from lexer/lexer_test.go rename to syntax/lexer/lexer_test.go index 8af3aa6..26c8983 100644 --- a/lexer/lexer_test.go +++ b/syntax/lexer/lexer_test.go @@ -178,9 +178,9 @@ func TestLexGood(t *testing.T) { }, }, } { - lexer := newLexer(test.pattern) + lexer := NewLexer(test.pattern) for i, exp := range test.items { - act := lexer.nextItem() + act := lexer.Next() if act.Type != exp.Type { t.Errorf("#%d %q: wrong %d-th item type: exp: %q; act: %q\n\t(%s vs %s)", id, test.pattern, i, exp.Type, act.Type, exp, act) } diff --git a/lexer/token.go b/syntax/lexer/token.go similarity index 100% rename from lexer/token.go rename to syntax/lexer/token.go diff --git a/syntax/syntax.go b/syntax/syntax.go new file mode 100644 index 0000000..1d168b1 --- /dev/null +++ b/syntax/syntax.go @@ -0,0 +1,14 @@ +package syntax + +import ( + "github.com/gobwas/glob/syntax/ast" + "github.com/gobwas/glob/syntax/lexer" +) + +func Parse(s string) (*ast.Node, error) { + return ast.Parse(lexer.NewLexer(s)) +} + +func Special(b byte) bool { + return lexer.Special(b) +} From 36eb5476d5057e0993a49f11457822704eb88e6b Mon Sep 17 00:00:00 2001 From: gobwas Date: Tue, 31 May 2016 11:28:02 +0300 Subject: [PATCH 3/3] refactoring done --- compiler/compiler.go | 35 ++- compiler/compiler_test.go | 625 +++++++++++++++++++++----------------- glob.go | 4 +- glob_test.go | 14 +- syntax/ast/ast.go | 13 - syntax/ast/parser.go | 6 +- syntax/ast/parser_test.go | 231 ++++++-------- 7 files changed, 468 insertions(+), 460 deletions(-) diff --git a/compiler/compiler.go b/compiler/compiler.go index 0278b57..ce04b5d 100644 --- a/compiler/compiler.go +++ b/compiler/compiler.go @@ -350,23 +350,33 @@ func minimizeTreeAnyOf(tree *ast.Node) *ast.Node { } func commonChildren(nodes []*ast.Node) (commonLeft, commonRight []*ast.Node) { + if len(nodes) <= 1 { + return + } + // find node that has least number of children idx := leastChildren(nodes) if idx == -1 { - return nil, nil + return } tree := nodes[idx] + treeLength := len(tree.Children) + + // allocate max able size for rightCommon slice + // to get ability insert elements in reverse order (from end to start) + // without sorting + commonRight = make([]*ast.Node, treeLength) + lastRight := treeLength // will use this to get results as commonRight[lastRight:] var ( - breakLeft bool - breakRight bool + breakLeft bool + breakRight bool + commonTotal int ) - for i, j := 0, len(tree.Children)-1; j >= 0 && !(breakLeft && breakLeft); i, j = i+1, j-1 { + for i, j := 0, treeLength-1; commonTotal < treeLength && j >= 0 && !(breakLeft && breakLeft); i, j = i+1, j-1 { treeLeft := tree.Children[i] treeRight := tree.Children[j] - fmt.Println(i, j) - for k := 0; k < len(nodes) && !(breakLeft && breakLeft); k++ { // skip least children node if k == idx { @@ -374,7 +384,7 @@ func commonChildren(nodes []*ast.Node) (commonLeft, commonRight []*ast.Node) { } restLeft := nodes[k].Children[i] - restRight := nodes[k].Children[j+len(nodes[k].Children)-len(tree.Children)] + restRight := nodes[k].Children[j+len(nodes[k].Children)-treeLength] breakLeft = breakLeft || !treeLeft.Equal(restLeft) @@ -382,15 +392,20 @@ func commonChildren(nodes []*ast.Node) (commonLeft, commonRight []*ast.Node) { breakRight = breakRight || (!breakLeft && j <= i) breakRight = breakRight || !treeRight.Equal(restRight) } + if !breakLeft { - fmt.Println("left app") + commonTotal++ commonLeft = append(commonLeft, treeLeft) } if !breakRight { - fmt.Println("right app") - commonRight = append(commonRight, treeRight) + commonTotal++ + lastRight = j + commonRight[j] = treeRight } } + + commonRight = commonRight[lastRight:] + return } diff --git a/compiler/compiler_test.go b/compiler/compiler_test.go index a93b918..b58b1eb 100644 --- a/compiler/compiler_test.go +++ b/compiler/compiler_test.go @@ -16,26 +16,35 @@ func TestCommonChildren(t *testing.T) { left []*ast.Node right []*ast.Node }{ - // { - // nodes: []*ast.Node{ - // ast.NewNode(ast.KindNothing, nil, - // ast.NewNode(ast.KindText, ast.Text{"a"}), - // ast.NewNode(ast.KindText, ast.Text{"z"}), - // ast.NewNode(ast.KindText, ast.Text{"c"}), - // ), - // ast.NewNode(ast.KindNothing, nil, - // ast.NewNode(ast.KindText, ast.Text{"a"}), - // ast.NewNode(ast.KindText, ast.Text{"b"}), - // ast.NewNode(ast.KindText, ast.Text{"c"}), - // ), - // }, - // left: []*ast.Node{ - // ast.NewNode(ast.KindText, ast.Text{"a"}), - // }, - // right: []*ast.Node{ - // ast.NewNode(ast.KindText, ast.Text{"c"}), - // }, - // }, + { + nodes: []*ast.Node{ + ast.NewNode(ast.KindNothing, nil, + ast.NewNode(ast.KindText, ast.Text{"a"}), + ast.NewNode(ast.KindText, ast.Text{"z"}), + ast.NewNode(ast.KindText, ast.Text{"c"}), + ), + }, + }, + { + nodes: []*ast.Node{ + ast.NewNode(ast.KindNothing, nil, + ast.NewNode(ast.KindText, ast.Text{"a"}), + ast.NewNode(ast.KindText, ast.Text{"z"}), + ast.NewNode(ast.KindText, ast.Text{"c"}), + ), + ast.NewNode(ast.KindNothing, nil, + ast.NewNode(ast.KindText, ast.Text{"a"}), + ast.NewNode(ast.KindText, ast.Text{"b"}), + ast.NewNode(ast.KindText, ast.Text{"c"}), + ), + }, + left: []*ast.Node{ + ast.NewNode(ast.KindText, ast.Text{"a"}), + }, + right: []*ast.Node{ + ast.NewNode(ast.KindText, ast.Text{"c"}), + }, + }, { nodes: []*ast.Node{ ast.NewNode(ast.KindNothing, nil, @@ -55,12 +64,54 @@ func TestCommonChildren(t *testing.T) { left: []*ast.Node{ ast.NewNode(ast.KindText, ast.Text{"a"}), ast.NewNode(ast.KindText, ast.Text{"b"}), - ast.NewNode(ast.KindText, ast.Text{"c"}), }, right: []*ast.Node{ + ast.NewNode(ast.KindText, ast.Text{"c"}), ast.NewNode(ast.KindText, ast.Text{"d"}), }, }, + { + nodes: []*ast.Node{ + ast.NewNode(ast.KindNothing, nil, + ast.NewNode(ast.KindText, ast.Text{"a"}), + ast.NewNode(ast.KindText, ast.Text{"b"}), + ast.NewNode(ast.KindText, ast.Text{"c"}), + ), + ast.NewNode(ast.KindNothing, nil, + ast.NewNode(ast.KindText, ast.Text{"a"}), + ast.NewNode(ast.KindText, ast.Text{"b"}), + ast.NewNode(ast.KindText, ast.Text{"b"}), + ast.NewNode(ast.KindText, ast.Text{"c"}), + ), + }, + left: []*ast.Node{ + ast.NewNode(ast.KindText, ast.Text{"a"}), + ast.NewNode(ast.KindText, ast.Text{"b"}), + }, + right: []*ast.Node{ + ast.NewNode(ast.KindText, ast.Text{"c"}), + }, + }, + { + nodes: []*ast.Node{ + ast.NewNode(ast.KindNothing, nil, + ast.NewNode(ast.KindText, ast.Text{"a"}), + ast.NewNode(ast.KindText, ast.Text{"d"}), + ), + ast.NewNode(ast.KindNothing, nil, + ast.NewNode(ast.KindText, ast.Text{"a"}), + ast.NewNode(ast.KindText, ast.Text{"d"}), + ), + ast.NewNode(ast.KindNothing, nil, + ast.NewNode(ast.KindText, ast.Text{"a"}), + ast.NewNode(ast.KindText, ast.Text{"e"}), + ), + }, + left: []*ast.Node{ + ast.NewNode(ast.KindText, ast.Text{"a"}), + }, + right: []*ast.Node{}, + }, } { left, right := commonChildren(test.nodes) if !nodesEqual(left, test.left) { @@ -268,238 +319,238 @@ func TestCompiler(t *testing.T) { result match.Matcher sep []rune }{ - // { - // ast: ast.NewNode(ast.KindPattern, nil, - // ast.NewNode(ast.KindText, &ast.Text{"abc"}), - // ), - // result: match.NewText("abc"), - // }, - // { - // ast: ast.NewNode(ast.KindPattern, nil, - // ast.NewNode(ast.KindAny, nil), - // ), - // sep: separators, - // result: match.NewAny(separators), - // }, - // { - // ast: ast.NewNode(ast.KindPattern, nil, - // ast.NewNode(ast.KindAny, nil), - // ), - // result: match.NewSuper(), - // }, - // { - // ast: ast.NewNode(ast.KindPattern, nil, - // ast.NewNode(ast.KindSuper, nil), - // ), - // result: match.NewSuper(), - // }, - // { - // ast: ast.NewNode(ast.KindPattern, nil, - // ast.NewNode(ast.KindSingle, nil), - // ), - // sep: separators, - // result: match.NewSingle(separators), - // }, - // { - // ast: ast.NewNode(ast.KindPattern, nil, - // ast.NewNode(ast.KindRange, &ast.Range{ - // Lo: 'a', - // Hi: 'z', - // Not: true, - // }), - // ), - // result: match.NewRange('a', 'z', true), - // }, - // { - // ast: ast.NewNode(ast.KindPattern, nil, - // ast.NewNode(ast.KindList, &ast.List{ - // Chars: "abc", - // Not: true, - // }), - // ), - // result: match.NewList([]rune{'a', 'b', 'c'}, true), - // }, - // { - // ast: ast.NewNode(ast.KindPattern, nil, - // ast.NewNode(ast.KindAny, nil), - // ast.NewNode(ast.KindSingle, nil), - // ast.NewNode(ast.KindSingle, nil), - // ast.NewNode(ast.KindSingle, nil), - // ), - // sep: separators, - // result: match.EveryOf{Matchers: match.Matchers{ - // match.NewMin(3), - // match.NewContains(string(separators), true), - // }}, - // }, - // { - // ast: ast.NewNode(ast.KindPattern, nil, - // ast.NewNode(ast.KindAny, nil), - // ast.NewNode(ast.KindSingle, nil), - // ast.NewNode(ast.KindSingle, nil), - // ast.NewNode(ast.KindSingle, nil), - // ), - // result: match.NewMin(3), - // }, - // { - // ast: ast.NewNode(ast.KindPattern, nil, - // ast.NewNode(ast.KindAny, nil), - // ast.NewNode(ast.KindText, &ast.Text{"abc"}), - // ast.NewNode(ast.KindSingle, nil), - // ), - // sep: separators, - // result: match.NewBTree( - // match.NewRow( - // 4, - // match.Matchers{ - // match.NewText("abc"), - // match.NewSingle(separators), - // }..., - // ), - // match.NewAny(separators), - // nil, - // ), - // }, - // { - // ast: ast.NewNode(ast.KindPattern, nil, - // ast.NewNode(ast.KindText, &ast.Text{"/"}), - // ast.NewNode(ast.KindAnyOf, nil, - // ast.NewNode(ast.KindText, &ast.Text{"z"}), - // ast.NewNode(ast.KindText, &ast.Text{"ab"}), - // ), - // ast.NewNode(ast.KindSuper, nil), - // ), - // sep: separators, - // result: match.NewBTree( - // match.NewText("/"), - // nil, - // match.NewBTree( - // match.NewAnyOf(match.NewText("z"), match.NewText("ab")), - // nil, - // match.NewSuper(), - // ), - // ), - // }, - // { - // ast: ast.NewNode(ast.KindPattern, nil, - // ast.NewNode(ast.KindSuper, nil), - // ast.NewNode(ast.KindSingle, nil), - // ast.NewNode(ast.KindText, &ast.Text{"abc"}), - // ast.NewNode(ast.KindSingle, nil), - // ), - // sep: separators, - // result: match.NewBTree( - // match.NewRow( - // 5, - // match.Matchers{ - // match.NewSingle(separators), - // match.NewText("abc"), - // match.NewSingle(separators), - // }..., - // ), - // match.NewSuper(), - // nil, - // ), - // }, - // { - // ast: ast.NewNode(ast.KindPattern, nil, - // ast.NewNode(ast.KindAny, nil), - // ast.NewNode(ast.KindText, &ast.Text{"abc"}), - // ), - // result: match.NewSuffix("abc"), - // }, - // { - // ast: ast.NewNode(ast.KindPattern, nil, - // ast.NewNode(ast.KindText, &ast.Text{"abc"}), - // ast.NewNode(ast.KindAny, nil), - // ), - // result: match.NewPrefix("abc"), - // }, - // { - // ast: ast.NewNode(ast.KindPattern, nil, - // ast.NewNode(ast.KindText, &ast.Text{"abc"}), - // ast.NewNode(ast.KindAny, nil), - // ast.NewNode(ast.KindText, &ast.Text{"def"}), - // ), - // result: match.NewPrefixSuffix("abc", "def"), - // }, - // { - // ast: ast.NewNode(ast.KindPattern, nil, - // ast.NewNode(ast.KindAny, nil), - // ast.NewNode(ast.KindAny, nil), - // ast.NewNode(ast.KindAny, nil), - // ast.NewNode(ast.KindText, &ast.Text{"abc"}), - // ast.NewNode(ast.KindAny, nil), - // ast.NewNode(ast.KindAny, nil), - // ), - // result: match.NewContains("abc", false), - // }, - // { - // ast: ast.NewNode(ast.KindPattern, nil, - // ast.NewNode(ast.KindAny, nil), - // ast.NewNode(ast.KindAny, nil), - // ast.NewNode(ast.KindAny, nil), - // ast.NewNode(ast.KindText, &ast.Text{"abc"}), - // ast.NewNode(ast.KindAny, nil), - // ast.NewNode(ast.KindAny, nil), - // ), - // sep: separators, - // result: match.NewBTree( - // match.NewText("abc"), - // match.NewAny(separators), - // match.NewAny(separators), - // ), - // }, - // { - // ast: ast.NewNode(ast.KindPattern, nil, - // ast.NewNode(ast.KindSuper, nil), - // ast.NewNode(ast.KindSingle, nil), - // ast.NewNode(ast.KindText, &ast.Text{"abc"}), - // ast.NewNode(ast.KindSuper, nil), - // ast.NewNode(ast.KindSingle, nil), - // ), - // result: match.NewBTree( - // match.NewText("abc"), - // match.NewMin(1), - // match.NewMin(1), - // ), - // }, - // { - // ast: ast.NewNode(ast.KindPattern, nil, - // ast.NewNode(ast.KindText, &ast.Text{"abc"}), - // ), - // result: match.NewText("abc"), - // }, - // { - // ast: ast.NewNode(ast.KindPattern, nil, - // ast.NewNode(ast.KindAnyOf, nil, - // ast.NewNode(ast.KindPattern, nil, - // ast.NewNode(ast.KindAnyOf, nil, - // ast.NewNode(ast.KindPattern, nil, - // ast.NewNode(ast.KindText, &ast.Text{"abc"}), - // ), - // ), - // ), - // ), - // ), - // result: match.NewText("abc"), - // }, + { + ast: ast.NewNode(ast.KindPattern, nil, + ast.NewNode(ast.KindText, ast.Text{"abc"}), + ), + result: match.NewText("abc"), + }, + { + ast: ast.NewNode(ast.KindPattern, nil, + ast.NewNode(ast.KindAny, nil), + ), + sep: separators, + result: match.NewAny(separators), + }, + { + ast: ast.NewNode(ast.KindPattern, nil, + ast.NewNode(ast.KindAny, nil), + ), + result: match.NewSuper(), + }, + { + ast: ast.NewNode(ast.KindPattern, nil, + ast.NewNode(ast.KindSuper, nil), + ), + result: match.NewSuper(), + }, + { + ast: ast.NewNode(ast.KindPattern, nil, + ast.NewNode(ast.KindSingle, nil), + ), + sep: separators, + result: match.NewSingle(separators), + }, + { + ast: ast.NewNode(ast.KindPattern, nil, + ast.NewNode(ast.KindRange, ast.Range{ + Lo: 'a', + Hi: 'z', + Not: true, + }), + ), + result: match.NewRange('a', 'z', true), + }, + { + ast: ast.NewNode(ast.KindPattern, nil, + ast.NewNode(ast.KindList, ast.List{ + Chars: "abc", + Not: true, + }), + ), + result: match.NewList([]rune{'a', 'b', 'c'}, true), + }, + { + ast: ast.NewNode(ast.KindPattern, nil, + ast.NewNode(ast.KindAny, nil), + ast.NewNode(ast.KindSingle, nil), + ast.NewNode(ast.KindSingle, nil), + ast.NewNode(ast.KindSingle, nil), + ), + sep: separators, + result: match.EveryOf{Matchers: match.Matchers{ + match.NewMin(3), + match.NewContains(string(separators), true), + }}, + }, + { + ast: ast.NewNode(ast.KindPattern, nil, + ast.NewNode(ast.KindAny, nil), + ast.NewNode(ast.KindSingle, nil), + ast.NewNode(ast.KindSingle, nil), + ast.NewNode(ast.KindSingle, nil), + ), + result: match.NewMin(3), + }, + { + ast: ast.NewNode(ast.KindPattern, nil, + ast.NewNode(ast.KindAny, nil), + ast.NewNode(ast.KindText, ast.Text{"abc"}), + ast.NewNode(ast.KindSingle, nil), + ), + sep: separators, + result: match.NewBTree( + match.NewRow( + 4, + match.Matchers{ + match.NewText("abc"), + match.NewSingle(separators), + }..., + ), + match.NewAny(separators), + nil, + ), + }, + { + ast: ast.NewNode(ast.KindPattern, nil, + ast.NewNode(ast.KindText, ast.Text{"/"}), + ast.NewNode(ast.KindAnyOf, nil, + ast.NewNode(ast.KindText, ast.Text{"z"}), + ast.NewNode(ast.KindText, ast.Text{"ab"}), + ), + ast.NewNode(ast.KindSuper, nil), + ), + sep: separators, + result: match.NewBTree( + match.NewText("/"), + nil, + match.NewBTree( + match.NewAnyOf(match.NewText("z"), match.NewText("ab")), + nil, + match.NewSuper(), + ), + ), + }, + { + ast: ast.NewNode(ast.KindPattern, nil, + ast.NewNode(ast.KindSuper, nil), + ast.NewNode(ast.KindSingle, nil), + ast.NewNode(ast.KindText, ast.Text{"abc"}), + ast.NewNode(ast.KindSingle, nil), + ), + sep: separators, + result: match.NewBTree( + match.NewRow( + 5, + match.Matchers{ + match.NewSingle(separators), + match.NewText("abc"), + match.NewSingle(separators), + }..., + ), + match.NewSuper(), + nil, + ), + }, + { + ast: ast.NewNode(ast.KindPattern, nil, + ast.NewNode(ast.KindAny, nil), + ast.NewNode(ast.KindText, ast.Text{"abc"}), + ), + result: match.NewSuffix("abc"), + }, + { + ast: ast.NewNode(ast.KindPattern, nil, + ast.NewNode(ast.KindText, ast.Text{"abc"}), + ast.NewNode(ast.KindAny, nil), + ), + result: match.NewPrefix("abc"), + }, + { + ast: ast.NewNode(ast.KindPattern, nil, + ast.NewNode(ast.KindText, ast.Text{"abc"}), + ast.NewNode(ast.KindAny, nil), + ast.NewNode(ast.KindText, ast.Text{"def"}), + ), + result: match.NewPrefixSuffix("abc", "def"), + }, + { + ast: ast.NewNode(ast.KindPattern, nil, + ast.NewNode(ast.KindAny, nil), + ast.NewNode(ast.KindAny, nil), + ast.NewNode(ast.KindAny, nil), + ast.NewNode(ast.KindText, ast.Text{"abc"}), + ast.NewNode(ast.KindAny, nil), + ast.NewNode(ast.KindAny, nil), + ), + result: match.NewContains("abc", false), + }, + { + ast: ast.NewNode(ast.KindPattern, nil, + ast.NewNode(ast.KindAny, nil), + ast.NewNode(ast.KindAny, nil), + ast.NewNode(ast.KindAny, nil), + ast.NewNode(ast.KindText, ast.Text{"abc"}), + ast.NewNode(ast.KindAny, nil), + ast.NewNode(ast.KindAny, nil), + ), + sep: separators, + result: match.NewBTree( + match.NewText("abc"), + match.NewAny(separators), + match.NewAny(separators), + ), + }, + { + ast: ast.NewNode(ast.KindPattern, nil, + ast.NewNode(ast.KindSuper, nil), + ast.NewNode(ast.KindSingle, nil), + ast.NewNode(ast.KindText, ast.Text{"abc"}), + ast.NewNode(ast.KindSuper, nil), + ast.NewNode(ast.KindSingle, nil), + ), + result: match.NewBTree( + match.NewText("abc"), + match.NewMin(1), + match.NewMin(1), + ), + }, + { + ast: ast.NewNode(ast.KindPattern, nil, + ast.NewNode(ast.KindText, ast.Text{"abc"}), + ), + result: match.NewText("abc"), + }, { ast: ast.NewNode(ast.KindPattern, nil, ast.NewNode(ast.KindAnyOf, nil, ast.NewNode(ast.KindPattern, nil, - ast.NewNode(ast.KindText, &ast.Text{"abc"}), + ast.NewNode(ast.KindAnyOf, nil, + ast.NewNode(ast.KindPattern, nil, + ast.NewNode(ast.KindText, ast.Text{"abc"}), + ), + ), + ), + ), + ), + result: match.NewText("abc"), + }, + { + ast: ast.NewNode(ast.KindPattern, nil, + ast.NewNode(ast.KindAnyOf, nil, + ast.NewNode(ast.KindPattern, nil, + ast.NewNode(ast.KindText, ast.Text{"abc"}), ast.NewNode(ast.KindSingle, nil), ), ast.NewNode(ast.KindPattern, nil, - ast.NewNode(ast.KindText, &ast.Text{"abc"}), - ast.NewNode(ast.KindList, &ast.List{Chars: "def"}), + ast.NewNode(ast.KindText, ast.Text{"abc"}), + ast.NewNode(ast.KindList, ast.List{Chars: "def"}), ), ast.NewNode(ast.KindPattern, nil, - ast.NewNode(ast.KindText, &ast.Text{"abc"}), + ast.NewNode(ast.KindText, ast.Text{"abc"}), ), ast.NewNode(ast.KindPattern, nil, - ast.NewNode(ast.KindText, &ast.Text{"abc"}), + ast.NewNode(ast.KindText, ast.Text{"abc"}), ), ), ), @@ -513,51 +564,51 @@ func TestCompiler(t *testing.T) { }}, ), }, - // { - // ast: ast.NewNode(ast.KindPattern, nil, - // ast.NewNode(ast.KindRange, &ast.Range{Lo: 'a', Hi: 'z'}), - // ast.NewNode(ast.KindRange, &ast.Range{Lo: 'a', Hi: 'x', Not: true}), - // ast.NewNode(ast.KindAny, nil), - // ), - // result: match.NewBTree( - // match.NewRow( - // 2, - // match.Matchers{ - // match.NewRange('a', 'z', false), - // match.NewRange('a', 'x', true), - // }..., - // ), - // nil, - // match.NewSuper(), - // ), - // }, - // { - // ast: ast.NewNode(ast.KindPattern, nil, - // ast.NewNode(ast.KindAnyOf, nil, - // ast.NewNode(ast.KindPattern, nil, - // ast.NewNode(ast.KindText, &ast.Text{"abc"}), - // ast.NewNode(ast.KindList, &ast.List{Chars: "abc"}), - // ast.NewNode(ast.KindText, &ast.Text{"ghi"}), - // ), - // ast.NewNode(ast.KindPattern, nil, - // ast.NewNode(ast.KindText, &ast.Text{"abc"}), - // ast.NewNode(ast.KindList, &ast.List{Chars: "def"}), - // ast.NewNode(ast.KindText, &ast.Text{"ghi"}), - // ), - // ), - // ), - // result: match.NewRow( - // 7, - // match.Matchers{ - // match.NewText("abc"), - // match.AnyOf{Matchers: match.Matchers{ - // match.NewList([]rune{'a', 'b', 'c'}, false), - // match.NewList([]rune{'d', 'e', 'f'}, false), - // }}, - // match.NewText("ghi"), - // }..., - // ), - // }, + { + ast: ast.NewNode(ast.KindPattern, nil, + ast.NewNode(ast.KindRange, ast.Range{Lo: 'a', Hi: 'z'}), + ast.NewNode(ast.KindRange, ast.Range{Lo: 'a', Hi: 'x', Not: true}), + ast.NewNode(ast.KindAny, nil), + ), + result: match.NewBTree( + match.NewRow( + 2, + match.Matchers{ + match.NewRange('a', 'z', false), + match.NewRange('a', 'x', true), + }..., + ), + nil, + match.NewSuper(), + ), + }, + { + ast: ast.NewNode(ast.KindPattern, nil, + ast.NewNode(ast.KindAnyOf, nil, + ast.NewNode(ast.KindPattern, nil, + ast.NewNode(ast.KindText, ast.Text{"abc"}), + ast.NewNode(ast.KindList, ast.List{Chars: "abc"}), + ast.NewNode(ast.KindText, ast.Text{"ghi"}), + ), + ast.NewNode(ast.KindPattern, nil, + ast.NewNode(ast.KindText, ast.Text{"abc"}), + ast.NewNode(ast.KindList, ast.List{Chars: "def"}), + ast.NewNode(ast.KindText, ast.Text{"ghi"}), + ), + ), + ), + result: match.NewRow( + 7, + match.Matchers{ + match.NewText("abc"), + match.AnyOf{Matchers: match.Matchers{ + match.NewList([]rune{'a', 'b', 'c'}, false), + match.NewList([]rune{'d', 'e', 'f'}, false), + }}, + match.NewText("ghi"), + }..., + ), + }, } { m, err := Compile(test.ast, test.sep) if err != nil { diff --git a/glob.go b/glob.go index 9eb9d58..2afde34 100644 --- a/glob.go +++ b/glob.go @@ -1,7 +1,7 @@ package glob import ( - "github.com/gobwas/glob/parser" + "github.com/gobwas/glob/compiler" "github.com/gobwas/glob/syntax" ) @@ -42,7 +42,7 @@ func Compile(pattern string, separators ...rune) (Glob, error) { return nil, err } - matcher, err := compile(ast, separators) + matcher, err := compiler.Compile(ast, separators) if err != nil { return nil, err } diff --git a/glob_test.go b/glob_test.go index 7cd52af..071ccb3 100644 --- a/glob_test.go +++ b/glob_test.go @@ -159,12 +159,6 @@ func TestGlob(t *testing.T) { } func TestQuoteMeta(t *testing.T) { - specialsQuoted := make([]byte, len(specials)*2) - for i, j := 0, 0; i < len(specials); i, j = i+1, j+2 { - specialsQuoted[j] = '\\' - specialsQuoted[j+1] = specials[i] - } - for id, test := range []struct { in, out string }{ @@ -177,12 +171,12 @@ func TestQuoteMeta(t *testing.T) { out: `\{foo\*\}`, }, { - in: string(specials), - out: string(specialsQuoted), + in: `*?\[]{}`, + out: `\*\?\\\[\]\{\}`, }, { - in: string(append([]byte("some text and"), specials...)), - out: string(append([]byte("some text and"), specialsQuoted...)), + in: `some text and *?\[]{}`, + out: `some text and \*\?\\\[\]\{\}`, }, } { act := QuoteMeta(test.in) diff --git a/syntax/ast/ast.go b/syntax/ast/ast.go index 3e499bc..a903a20 100644 --- a/syntax/ast/ast.go +++ b/syntax/ast/ast.go @@ -1,18 +1,5 @@ package ast -type Visitor interface { - Visit(*Node) Visitor -} - -func Walk(v Visitor, n *Node) { - if v = v.Visit(n); v == nil { - return - } - for _, c := range n.Children { - Walk(v, c) - } -} - type Node struct { Parent *Node Children []*Node diff --git a/syntax/ast/parser.go b/syntax/ast/parser.go index 93caa33..429b409 100644 --- a/syntax/ast/parser.go +++ b/syntax/ast/parser.go @@ -43,7 +43,7 @@ func parserMain(tree *Node, lex Lexer) (parseFn, *Node, error) { return nil, tree, errors.New(token.Raw) case lexer.Text: - Insert(tree, NewNode(KindText, &Text{token.Raw})) + Insert(tree, NewNode(KindText, Text{token.Raw})) return parserMain, tree, nil case lexer.Any: @@ -139,13 +139,13 @@ func parserRange(tree *Node, lex Lexer) (parseFn, *Node, error) { } if isRange { - Insert(tree, NewNode(KindRange, &Range{ + Insert(tree, NewNode(KindRange, Range{ Lo: lo, Hi: hi, Not: not, })) } else { - Insert(tree, NewNode(KindList, &List{ + Insert(tree, NewNode(KindList, List{ Chars: chars, Not: not, })) diff --git a/syntax/ast/parser_test.go b/syntax/ast/parser_test.go index 84e8d45..d8440fd 100644 --- a/syntax/ast/parser_test.go +++ b/syntax/ast/parser_test.go @@ -1,19 +1,19 @@ package ast import ( - "github.com/gobwas/glob/syntax" + "github.com/gobwas/glob/syntax/lexer" "reflect" "testing" ) type stubLexer struct { - tokens []syntax.Token + tokens []lexer.Token pos int } -func (s *stubLexer) Next() (ret syntax.Token) { +func (s *stubLexer) Next() (ret lexer.Token) { if s.pos == len(s.tokens) { - return syntax.Token{syntax.EOF, ""} + return lexer.Token{lexer.EOF, ""} } ret = s.tokens[s.pos] s.pos++ @@ -22,129 +22,129 @@ func (s *stubLexer) Next() (ret syntax.Token) { func TestParseString(t *testing.T) { for id, test := range []struct { - tokens []syntax.Token - tree Node + tokens []lexer.Token + tree *Node }{ { //pattern: "abc", - tokens: []syntax.Token{ - syntax.Token{syntax.Text, "abc"}, - syntax.Token{syntax.EOF, ""}, + tokens: []lexer.Token{ + lexer.Token{lexer.Text, "abc"}, + lexer.Token{lexer.EOF, ""}, }, tree: NewNode(KindPattern, nil, - NewNode(KindText, &Text{Text: "abc"}), + NewNode(KindText, Text{Text: "abc"}), ), }, { //pattern: "a*c", - tokens: []syntax.Token{ - syntax.Token{syntax.Text, "a"}, - syntax.Token{syntax.Any, "*"}, - syntax.Token{syntax.Text, "c"}, - syntax.Token{syntax.EOF, ""}, + tokens: []lexer.Token{ + lexer.Token{lexer.Text, "a"}, + lexer.Token{lexer.Any, "*"}, + lexer.Token{lexer.Text, "c"}, + lexer.Token{lexer.EOF, ""}, }, tree: NewNode(KindPattern, nil, - NewNode(KindText, &Text{Text: "a"}), + NewNode(KindText, Text{Text: "a"}), NewNode(KindAny, nil), - NewNode(KindText, &Text{Text: "c"}), + NewNode(KindText, Text{Text: "c"}), ), }, { //pattern: "a**c", - tokens: []syntax.Token{ - syntax.Token{syntax.Text, "a"}, - syntax.Token{syntax.Super, "**"}, - syntax.Token{syntax.Text, "c"}, - syntax.Token{syntax.EOF, ""}, + tokens: []lexer.Token{ + lexer.Token{lexer.Text, "a"}, + lexer.Token{lexer.Super, "**"}, + lexer.Token{lexer.Text, "c"}, + lexer.Token{lexer.EOF, ""}, }, tree: NewNode(KindPattern, nil, - NewNode(KindText, &Text{Text: "a"}), + NewNode(KindText, Text{Text: "a"}), NewNode(KindSuper, nil), - NewNode(KindText, &Text{Text: "c"}), + NewNode(KindText, Text{Text: "c"}), ), }, { //pattern: "a?c", - tokens: []syntax.Token{ - syntax.Token{syntax.Text, "a"}, - syntax.Token{syntax.Single, "?"}, - syntax.Token{syntax.Text, "c"}, - syntax.Token{syntax.EOF, ""}, + tokens: []lexer.Token{ + lexer.Token{lexer.Text, "a"}, + lexer.Token{lexer.Single, "?"}, + lexer.Token{lexer.Text, "c"}, + lexer.Token{lexer.EOF, ""}, }, tree: NewNode(KindPattern, nil, - NewNode(KindText, &Text{Text: "a"}), + NewNode(KindText, Text{Text: "a"}), NewNode(KindSingle, nil), - NewNode(KindText, &Text{Text: "c"}), + NewNode(KindText, Text{Text: "c"}), ), }, { //pattern: "[!a-z]", - tokens: []syntax.Token{ - syntax.Token{syntax.RangeOpen, "["}, - syntax.Token{syntax.Not, "!"}, - syntax.Token{syntax.RangeLo, "a"}, - syntax.Token{syntax.RangeBetween, "-"}, - syntax.Token{syntax.RangeHi, "z"}, - syntax.Token{syntax.RangeClose, "]"}, - syntax.Token{syntax.EOF, ""}, + tokens: []lexer.Token{ + lexer.Token{lexer.RangeOpen, "["}, + lexer.Token{lexer.Not, "!"}, + lexer.Token{lexer.RangeLo, "a"}, + lexer.Token{lexer.RangeBetween, "-"}, + lexer.Token{lexer.RangeHi, "z"}, + lexer.Token{lexer.RangeClose, "]"}, + lexer.Token{lexer.EOF, ""}, }, tree: NewNode(KindPattern, nil, - NewNode(KindRange, &Range{Lo: 'a', Hi: 'z', Not: true}), + NewNode(KindRange, Range{Lo: 'a', Hi: 'z', Not: true}), ), }, { //pattern: "[az]", - tokens: []syntax.Token{ - syntax.Token{syntax.RangeOpen, "["}, - syntax.Token{syntax.Text, "az"}, - syntax.Token{syntax.RangeClose, "]"}, - syntax.Token{syntax.EOF, ""}, + tokens: []lexer.Token{ + lexer.Token{lexer.RangeOpen, "["}, + lexer.Token{lexer.Text, "az"}, + lexer.Token{lexer.RangeClose, "]"}, + lexer.Token{lexer.EOF, ""}, }, tree: NewNode(KindPattern, nil, - NewNode(KindList, &List{Chars: "az"}), + NewNode(KindList, List{Chars: "az"}), ), }, { //pattern: "{a,z}", - tokens: []syntax.Token{ - syntax.Token{syntax.TermsOpen, "{"}, - syntax.Token{syntax.Text, "a"}, - syntax.Token{syntax.Separator, ","}, - syntax.Token{syntax.Text, "z"}, - syntax.Token{syntax.TermsClose, "}"}, - syntax.Token{syntax.EOF, ""}, + tokens: []lexer.Token{ + lexer.Token{lexer.TermsOpen, "{"}, + lexer.Token{lexer.Text, "a"}, + lexer.Token{lexer.Separator, ","}, + lexer.Token{lexer.Text, "z"}, + lexer.Token{lexer.TermsClose, "}"}, + lexer.Token{lexer.EOF, ""}, }, tree: NewNode(KindPattern, nil, NewNode(KindAnyOf, nil, NewNode(KindPattern, nil, - NewNode(KindText, &Text{Text: "a"}), + NewNode(KindText, Text{Text: "a"}), ), NewNode(KindPattern, nil, - NewNode(KindText, &Text{Text: "z"}), + NewNode(KindText, Text{Text: "z"}), ), ), ), }, { //pattern: "/{z,ab}*", - tokens: []syntax.Token{ - syntax.Token{syntax.Text, "/"}, - syntax.Token{syntax.TermsOpen, "{"}, - syntax.Token{syntax.Text, "z"}, - syntax.Token{syntax.Separator, ","}, - syntax.Token{syntax.Text, "ab"}, - syntax.Token{syntax.TermsClose, "}"}, - syntax.Token{syntax.Any, "*"}, - syntax.Token{syntax.EOF, ""}, + tokens: []lexer.Token{ + lexer.Token{lexer.Text, "/"}, + lexer.Token{lexer.TermsOpen, "{"}, + lexer.Token{lexer.Text, "z"}, + lexer.Token{lexer.Separator, ","}, + lexer.Token{lexer.Text, "ab"}, + lexer.Token{lexer.TermsClose, "}"}, + lexer.Token{lexer.Any, "*"}, + lexer.Token{lexer.EOF, ""}, }, tree: NewNode(KindPattern, nil, - NewNode(KindText, &Text{Text: "/"}), + NewNode(KindText, Text{Text: "/"}), NewNode(KindAnyOf, nil, NewNode(KindPattern, nil, - NewNode(KindText, &Text{Text: "z"}), + NewNode(KindText, Text{Text: "z"}), ), NewNode(KindPattern, nil, - NewNode(KindText, &Text{Text: "ab"}), + NewNode(KindText, Text{Text: "ab"}), ), ), NewNode(KindAny, nil), @@ -152,43 +152,43 @@ func TestParseString(t *testing.T) { }, { //pattern: "{a,{x,y},?,[a-z],[!qwe]}", - tokens: []syntax.Token{ - syntax.Token{syntax.TermsOpen, "{"}, - syntax.Token{syntax.Text, "a"}, - syntax.Token{syntax.Separator, ","}, - syntax.Token{syntax.TermsOpen, "{"}, - syntax.Token{syntax.Text, "x"}, - syntax.Token{syntax.Separator, ","}, - syntax.Token{syntax.Text, "y"}, - syntax.Token{syntax.TermsClose, "}"}, - syntax.Token{syntax.Separator, ","}, - syntax.Token{syntax.Single, "?"}, - syntax.Token{syntax.Separator, ","}, - syntax.Token{syntax.RangeOpen, "["}, - syntax.Token{syntax.RangeLo, "a"}, - syntax.Token{syntax.RangeBetween, "-"}, - syntax.Token{syntax.RangeHi, "z"}, - syntax.Token{syntax.RangeClose, "]"}, - syntax.Token{syntax.Separator, ","}, - syntax.Token{syntax.RangeOpen, "["}, - syntax.Token{syntax.Not, "!"}, - syntax.Token{syntax.Text, "qwe"}, - syntax.Token{syntax.RangeClose, "]"}, - syntax.Token{syntax.TermsClose, "}"}, - syntax.Token{syntax.EOF, ""}, + tokens: []lexer.Token{ + lexer.Token{lexer.TermsOpen, "{"}, + lexer.Token{lexer.Text, "a"}, + lexer.Token{lexer.Separator, ","}, + lexer.Token{lexer.TermsOpen, "{"}, + lexer.Token{lexer.Text, "x"}, + lexer.Token{lexer.Separator, ","}, + lexer.Token{lexer.Text, "y"}, + lexer.Token{lexer.TermsClose, "}"}, + lexer.Token{lexer.Separator, ","}, + lexer.Token{lexer.Single, "?"}, + lexer.Token{lexer.Separator, ","}, + lexer.Token{lexer.RangeOpen, "["}, + lexer.Token{lexer.RangeLo, "a"}, + lexer.Token{lexer.RangeBetween, "-"}, + lexer.Token{lexer.RangeHi, "z"}, + lexer.Token{lexer.RangeClose, "]"}, + lexer.Token{lexer.Separator, ","}, + lexer.Token{lexer.RangeOpen, "["}, + lexer.Token{lexer.Not, "!"}, + lexer.Token{lexer.Text, "qwe"}, + lexer.Token{lexer.RangeClose, "]"}, + lexer.Token{lexer.TermsClose, "}"}, + lexer.Token{lexer.EOF, ""}, }, tree: NewNode(KindPattern, nil, NewNode(KindAnyOf, nil, NewNode(KindPattern, nil, - NewNode(KindText, &Text{Text: "a"}), + NewNode(KindText, Text{Text: "a"}), ), NewNode(KindPattern, nil, NewNode(KindAnyOf, nil, NewNode(KindPattern, nil, - NewNode(KindText, &Text{Text: "x"}), + NewNode(KindText, Text{Text: "x"}), ), NewNode(KindPattern, nil, - NewNode(KindText, &Text{Text: "y"}), + NewNode(KindText, Text{Text: "y"}), ), ), ), @@ -196,10 +196,10 @@ func TestParseString(t *testing.T) { NewNode(KindSingle, nil), ), NewNode(KindPattern, nil, - NewNode(KindRange, &Range{Lo: 'a', Hi: 'z', Not: false}), + NewNode(KindRange, Range{Lo: 'a', Hi: 'z', Not: false}), ), NewNode(KindPattern, nil, - NewNode(KindList, &List{Chars: "qwe", Not: true}), + NewNode(KindList, List{Chars: "qwe", Not: true}), ), ), ), @@ -215,42 +215,3 @@ func TestParseString(t *testing.T) { } } } - -type kv struct { - kind Kind - value interface{} -} - -type visitor struct { - visited []kv -} - -func (v *visitor) Visit(n Node) Visitor { - v.visited = append(v.visited, kv{n.Kind(), n.Value()}) - return v -} - -func TestWalkTree(t *testing.T) { - - for i, test := range []struct { - tree *Node - visited []kv - }{ - { - tree: NewNode(KindPattern, nil, - NewNode(KindSingle, nil), - ), - visited: []kv{ - kv{KindPattern, nil}, - kv{KindSingle, nil}, - }, - }, - } { - v := &visitor{} - Walk(v, test.tree) - - if !reflect.DeepEqual(test.visited, v.visited) { - t.Errorf("[%d] unexpected result of Walk():\nvisited:\t%v\nwant:\t\t%v", i, v.visited, test.visited) - } - } -}