From 1550dd349cb281435687c1e16e812134ae3bb2b2 Mon Sep 17 00:00:00 2001 From: gobwas Date: Thu, 12 May 2016 10:46:16 +0300 Subject: [PATCH 1/3] detection of the bug --- compiler.go | 2 +- compiler_test.go | 396 ++++++++++++++++++++++++----------------------- glob_test.go | 4 + lexer.go | 14 ++ lexer_test.go | 21 +++ parser.go | 13 +- parser_test.go | 127 +++++++++++++-- 7 files changed, 364 insertions(+), 213 deletions(-) diff --git a/compiler.go b/compiler.go index 16c241b..0bb745e 100644 --- a/compiler.go +++ b/compiler.go @@ -490,8 +490,8 @@ func doAnyOf(n *nodeAnyOf, s []rune) (match.Matcher, error) { } func do(leaf node, s []rune) (m match.Matcher, err error) { + fmt.Println("node", reflect.Indirect(reflect.ValueOf(leaf)).Type().Name()) switch n := leaf.(type) { - case *nodeAnyOf: // todo this could be faster on pattern_alternatives_combine_lite if n := minimizeAnyOf(n.children()); n != nil { diff --git a/compiler_test.go b/compiler_test.go index f2d0c70..8b6b34c 100644 --- a/compiler_test.go +++ b/compiler_test.go @@ -2,6 +2,7 @@ package glob import ( "github.com/gobwas/glob/match" + "github.com/gobwas/glob/match/debug" "reflect" "testing" ) @@ -206,209 +207,210 @@ func TestCompiler(t *testing.T) { result Glob sep []rune }{ + //{ + // ast: pattern(&nodeText{text: "abc"}), + // result: match.NewText("abc"), + //}, + //{ + // ast: pattern(&nodeAny{}), + // sep: separators, + // result: match.NewAny(separators), + //}, + //{ + // ast: pattern(&nodeAny{}), + // result: match.NewSuper(), + //}, + //{ + // ast: pattern(&nodeSuper{}), + // result: match.NewSuper(), + //}, + //{ + // ast: pattern(&nodeSingle{}), + // sep: separators, + // result: match.NewSingle(separators), + //}, + //{ + // ast: pattern(&nodeRange{ + // lo: 'a', + // hi: 'z', + // not: true, + // }), + // result: match.NewRange('a', 'z', true), + //}, + //{ + // ast: pattern(&nodeList{ + // chars: "abc", + // not: true, + // }), + // result: match.NewList([]rune{'a', 'b', 'c'}, true), + //}, + //{ + // ast: pattern(&nodeAny{}, &nodeSingle{}, &nodeSingle{}, &nodeSingle{}), + // sep: separators, + // result: match.EveryOf{Matchers: match.Matchers{ + // match.NewMin(3), + // match.NewContains(string(separators), true), + // }}, + //}, + //{ + // ast: pattern(&nodeAny{}, &nodeSingle{}, &nodeSingle{}, &nodeSingle{}), + // result: match.NewMin(3), + //}, + //{ + // ast: pattern(&nodeAny{}, &nodeText{text: "abc"}, &nodeSingle{}), + // sep: separators, + // result: match.NewBTree( + // match.NewRow( + // 4, + // match.Matchers{ + // match.NewText("abc"), + // match.NewSingle(separators), + // }..., + // ), + // match.NewAny(separators), + // nil, + // ), + //}, { - ast: pattern(&nodeText{text: "abc"}), - result: match.NewText("abc"), - }, - { - ast: pattern(&nodeAny{}), - sep: separators, - result: match.NewAny(separators), - }, - { - ast: pattern(&nodeAny{}), - result: match.NewSuper(), - }, - { - ast: pattern(&nodeSuper{}), - result: match.NewSuper(), - }, - { - ast: pattern(&nodeSingle{}), - sep: separators, - result: match.NewSingle(separators), - }, - { - ast: pattern(&nodeRange{ - lo: 'a', - hi: 'z', - not: true, - }), - result: match.NewRange('a', 'z', true), - }, - { - ast: pattern(&nodeList{ - chars: "abc", - not: true, - }), - result: match.NewList([]rune{'a', 'b', 'c'}, true), - }, - { - ast: pattern(&nodeAny{}, &nodeSingle{}, &nodeSingle{}, &nodeSingle{}), - sep: separators, - result: match.EveryOf{Matchers: match.Matchers{ - match.NewMin(3), - match.NewContains(string(separators), true), - }}, - }, - { - ast: pattern(&nodeAny{}, &nodeSingle{}, &nodeSingle{}, &nodeSingle{}), - result: match.NewMin(3), - }, - { - ast: pattern(&nodeAny{}, &nodeText{text: "abc"}, &nodeSingle{}), + ast: pattern(&nodeText{text: "/"}, anyOf(&nodeText{text: "z"}, &nodeText{text: "ab"}), &nodeSuper{}), sep: separators, result: match.NewBTree( - match.NewRow( - 4, - match.Matchers{ - match.NewText("abc"), - match.NewSingle(separators), - }..., - ), - match.NewAny(separators), + match.NewText("/"), nil, + match.NewBTree( + match.NewAnyOf(match.NewText("z"), match.NewText("ab")), + nil, + match.NewSuper(), + ), ), }, - { - ast: pattern(&nodeSuper{}, &nodeSingle{}, &nodeText{text: "abc"}, &nodeSingle{}), - sep: separators, - result: match.NewBTree( - match.NewRow( - 5, - match.Matchers{ - match.NewSingle(separators), - match.NewText("abc"), - match.NewSingle(separators), - }..., - ), - match.NewSuper(), - nil, - ), - }, - { - ast: pattern(&nodeAny{}, &nodeText{text: "abc"}), - result: match.NewSuffix("abc"), - }, - { - ast: pattern(&nodeText{text: "abc"}, &nodeAny{}), - result: match.NewPrefix("abc"), - }, - { - ast: pattern(&nodeText{text: "abc"}, &nodeAny{}, &nodeText{text: "def"}), - result: match.NewPrefixSuffix("abc", "def"), - }, - { - ast: pattern(&nodeAny{}, &nodeAny{}, &nodeAny{}, &nodeText{text: "abc"}, &nodeAny{}, &nodeAny{}), - result: match.NewContains("abc", false), - }, - { - ast: pattern(&nodeAny{}, &nodeAny{}, &nodeAny{}, &nodeText{text: "abc"}, &nodeAny{}, &nodeAny{}), - sep: separators, - result: match.NewBTree( - match.NewText("abc"), - match.NewAny(separators), - match.NewAny(separators), - ), - }, - { - ast: pattern(&nodeSuper{}, &nodeSingle{}, &nodeText{text: "abc"}, &nodeSuper{}, &nodeSingle{}), - result: match.NewBTree( - match.NewText("abc"), - match.NewMin(1), - match.NewMin(1), - ), - }, - { - ast: pattern(anyOf(&nodeText{text: "abc"})), - result: match.NewText("abc"), - }, - { - ast: pattern(anyOf(pattern(anyOf(pattern(&nodeText{text: "abc"}))))), - result: match.NewText("abc"), - }, - { - ast: pattern(anyOf( - pattern( - &nodeText{text: "abc"}, - &nodeSingle{}, - ), - pattern( - &nodeText{text: "abc"}, - &nodeList{chars: "def"}, - ), - pattern( - &nodeText{text: "abc"}, - ), - pattern( - &nodeText{text: "abc"}, - ), - )), - result: match.NewBTree( - match.NewText("abc"), - nil, - match.AnyOf{Matchers: match.Matchers{ - match.NewSingle(nil), - match.NewList([]rune{'d', 'e', 'f'}, false), - match.NewNothing(), - }}, - ), - }, - { - ast: pattern( - &nodeRange{lo: 'a', hi: 'z'}, - &nodeRange{lo: 'a', hi: 'x', not: true}, - &nodeAny{}, - ), - result: match.NewBTree( - match.NewRow( - 2, - match.Matchers{ - match.NewRange('a', 'z', false), - match.NewRange('a', 'x', true), - }..., - ), - nil, - match.NewSuper(), - ), - }, - { - ast: pattern(anyOf( - pattern( - &nodeText{text: "abc"}, - &nodeList{chars: "abc"}, - &nodeText{text: "ghi"}, - ), - pattern( - &nodeText{text: "abc"}, - &nodeList{chars: "def"}, - &nodeText{text: "ghi"}, - ), - )), - result: match.NewRow( - 7, - match.Matchers{ - match.NewText("abc"), - match.AnyOf{Matchers: match.Matchers{ - match.NewList([]rune{'a', 'b', 'c'}, false), - match.NewList([]rune{'d', 'e', 'f'}, false), - }}, - match.NewText("ghi"), - }..., - ), - }, - // { - // ast: pattern( - // anyOf(&nodeText{text: "a"}, &nodeText{text: "b"}), - // anyOf(&nodeText{text: "c"}, &nodeText{text: "d"}), - // ), - // result: match.AnyOf{Matchers: match.Matchers{ - // match.NewRow(Matchers: match.Matchers{match.Raw{"a"}, match.Raw{"c", 1}}), - // match.NewRow(Matchers: match.Matchers{match.Raw{"a"}, match.Raw{"d"}}), - // match.NewRow(Matchers: match.Matchers{match.Raw{"b"}, match.Raw{"c", 1}}), - // match.NewRow(Matchers: match.Matchers{match.Raw{"b"}, match.Raw{"d"}}), + //{ + // ast: pattern(&nodeSuper{}, &nodeSingle{}, &nodeText{text: "abc"}, &nodeSingle{}), + // sep: separators, + // result: match.NewBTree( + // match.NewRow( + // 5, + // match.Matchers{ + // match.NewSingle(separators), + // match.NewText("abc"), + // match.NewSingle(separators), + // }..., + // ), + // match.NewSuper(), + // nil, + // ), + //}, + //{ + // ast: pattern(&nodeAny{}, &nodeText{text: "abc"}), + // result: match.NewSuffix("abc"), + //}, + //{ + // ast: pattern(&nodeText{text: "abc"}, &nodeAny{}), + // result: match.NewPrefix("abc"), + //}, + //{ + // ast: pattern(&nodeText{text: "abc"}, &nodeAny{}, &nodeText{text: "def"}), + // result: match.NewPrefixSuffix("abc", "def"), + //}, + //{ + // ast: pattern(&nodeAny{}, &nodeAny{}, &nodeAny{}, &nodeText{text: "abc"}, &nodeAny{}, &nodeAny{}), + // result: match.NewContains("abc", false), + //}, + //{ + // ast: pattern(&nodeAny{}, &nodeAny{}, &nodeAny{}, &nodeText{text: "abc"}, &nodeAny{}, &nodeAny{}), + // sep: separators, + // result: match.NewBTree( + // match.NewText("abc"), + // match.NewAny(separators), + // match.NewAny(separators), + // ), + //}, + //{ + // ast: pattern(&nodeSuper{}, &nodeSingle{}, &nodeText{text: "abc"}, &nodeSuper{}, &nodeSingle{}), + // result: match.NewBTree( + // match.NewText("abc"), + // match.NewMin(1), + // match.NewMin(1), + // ), + //}, + //{ + // ast: pattern(anyOf(&nodeText{text: "abc"})), + // result: match.NewText("abc"), + //}, + //{ + // ast: pattern(anyOf(pattern(anyOf(pattern(&nodeText{text: "abc"}))))), + // result: match.NewText("abc"), + //}, + //{ + // ast: pattern(anyOf( + // pattern( + // &nodeText{text: "abc"}, + // &nodeSingle{}, + // ), + // pattern( + // &nodeText{text: "abc"}, + // &nodeList{chars: "def"}, + // ), + // pattern( + // &nodeText{text: "abc"}, + // ), + // pattern( + // &nodeText{text: "abc"}, + // ), + // )), + // result: match.NewBTree( + // match.NewText("abc"), + // nil, + // match.AnyOf{Matchers: match.Matchers{ + // match.NewSingle(nil), + // match.NewList([]rune{'d', 'e', 'f'}, false), + // match.NewNothing(), + // }}, + // ), + //}, + //{ + // ast: pattern( + // &nodeRange{lo: 'a', hi: 'z'}, + // &nodeRange{lo: 'a', hi: 'x', not: true}, + // &nodeAny{}, + // ), + // result: match.NewBTree( + // match.NewRow( + // 2, + // match.Matchers{ + // match.NewRange('a', 'z', false), + // match.NewRange('a', 'x', true), + // }..., + // ), + // nil, + // match.NewSuper(), + // ), + //}, + //{ + // ast: pattern(anyOf( + // pattern( + // &nodeText{text: "abc"}, + // &nodeList{chars: "abc"}, + // &nodeText{text: "ghi"}, + // ), + // pattern( + // &nodeText{text: "abc"}, + // &nodeList{chars: "def"}, + // &nodeText{text: "ghi"}, + // ), + // )), + // result: match.NewRow( + // 7, + // match.Matchers{ + // match.NewText("abc"), + // match.AnyOf{Matchers: match.Matchers{ + // match.NewList([]rune{'a', 'b', 'c'}, false), + // match.NewList([]rune{'d', 'e', 'f'}, false), // }}, - // }, + // match.NewText("ghi"), + // }..., + // ), + //}, } { m, err := compile(test.ast, test.sep) if err != nil { @@ -417,7 +419,7 @@ func TestCompiler(t *testing.T) { } if !reflect.DeepEqual(m, test.result) { - t.Errorf("#%d results are not equal:\nexp: %#v\nact: %#v", id, test.result, m) + t.Errorf("#%d results are not equal:\nexp: %#v\nact: %#v\nexp:\n%s\nact:\n%s\n", id, test.result, m, debug.Graphviz("", test.result.(match.Matcher)), debug.Graphviz("", m.(match.Matcher))) continue } } diff --git a/glob_test.go b/glob_test.go index 6fe73a6..ff28e2f 100644 --- a/glob_test.go +++ b/glob_test.go @@ -108,6 +108,10 @@ func TestGlob(t *testing.T) { glob(true, "{*,**}{a,b}", "ab"), glob(false, "{*,**}{a,b}", "ac"), + glob(true, "/{rate,[a-z][a-z][a-z]}*", "/rate"), + glob(true, "/{rate,[0-9][0-9][0-9]}*", "/rate"), + glob(true, "/{rate,[a-z][a-z][a-z]}*", "/usd"), + glob(true, pattern_all, fixture_all_match), glob(false, pattern_all, fixture_all_mismatch), diff --git a/lexer.go b/lexer.go index 81cfb22..75f970a 100644 --- a/lexer.go +++ b/lexer.go @@ -123,6 +123,20 @@ func (i item) String() string { return fmt.Sprintf("%v<%s>", i.t, i.s) } +type stubLexer struct { + Items []item + pos int +} + +func (s *stubLexer) nextItem() (ret item) { + if s.pos == len(s.Items) { + return item{item_eof, ""} + } + ret = s.Items[s.pos] + s.pos++ + return +} + type lexer struct { input string start int diff --git a/lexer_test.go b/lexer_test.go index 664dd71..48fbcf3 100644 --- a/lexer_test.go +++ b/lexer_test.go @@ -16,6 +16,14 @@ func TestLexGood(t *testing.T) { item{item_eof, ""}, }, }, + //{ + // TODO(gobwas): this is freezing on globtest/globdraw (`]]` syntax error) + //pattern: "/{rate,[0-9]]}*", + //items: []item{ + // item{item_text, "hello"}, + // item{item_eof, ""}, + //}, + //}, { pattern: "hello,world", items: []item{ @@ -114,6 +122,19 @@ func TestLexGood(t *testing.T) { item{item_eof, ""}, }, }, + { + pattern: "/{z,ab}*", + items: []item{ + item{item_text, "/"}, + item{item_terms_open, "{"}, + item{item_text, "z"}, + item{item_separator, ","}, + item{item_text, "ab"}, + item{item_terms_close, "}"}, + item{item_any, "*"}, + item{item_eof, ""}, + }, + }, { pattern: "{[!日-語],*,?,{a,b,\\c}}", items: []item{ diff --git a/parser.go b/parser.go index 43da13a..760ec96 100644 --- a/parser.go +++ b/parser.go @@ -11,6 +11,11 @@ type node interface { append(node) } +// todo may be split it into another package +type lexerIface interface { + nextItem() item +} + type nodeImpl struct { desc []node } @@ -72,9 +77,9 @@ func (t *tree) leave() { t.current = t.path[len(t.path)-1] } -type parseFn func(*tree, *lexer) (parseFn, error) +type parseFn func(*tree, lexerIface) (parseFn, error) -func parse(lexer *lexer) (*nodePattern, error) { +func parse(lexer lexerIface) (*nodePattern, error) { var parser parseFn root := &nodePattern{} @@ -97,7 +102,7 @@ func parse(lexer *lexer) (*nodePattern, error) { return root, nil } -func parserMain(tree *tree, lexer *lexer) (parseFn, error) { +func parserMain(tree *tree, lexer lexerIface) (parseFn, error) { for stop := false; !stop; { item := lexer.nextItem() @@ -151,7 +156,7 @@ func parserMain(tree *tree, lexer *lexer) (parseFn, error) { return nil, nil } -func parserRange(tree *tree, lexer *lexer) (parseFn, error) { +func parserRange(tree *tree, lexer lexerIface) (parseFn, error) { var ( not bool lo rune diff --git a/parser_test.go b/parser_test.go index 159c1ff..0e9312f 100644 --- a/parser_test.go +++ b/parser_test.go @@ -8,11 +8,15 @@ import ( func TestParseString(t *testing.T) { for id, test := range []struct { - pattern string - tree node + items []item + tree node }{ { - pattern: "abc", + //pattern: "abc", + items: []item{ + item{item_text, "abc"}, + item{item_eof, ""}, + }, tree: &nodePattern{ nodeImpl: nodeImpl{ desc: []node{ @@ -22,7 +26,13 @@ func TestParseString(t *testing.T) { }, }, { - pattern: "a*c", + //pattern: "a*c", + items: []item{ + item{item_text, "a"}, + item{item_any, "*"}, + item{item_text, "c"}, + item{item_eof, ""}, + }, tree: &nodePattern{ nodeImpl: nodeImpl{ desc: []node{ @@ -34,7 +44,13 @@ func TestParseString(t *testing.T) { }, }, { - pattern: "a**c", + //pattern: "a**c", + items: []item{ + item{item_text, "a"}, + item{item_super, "**"}, + item{item_text, "c"}, + item{item_eof, ""}, + }, tree: &nodePattern{ nodeImpl: nodeImpl{ desc: []node{ @@ -46,7 +62,13 @@ func TestParseString(t *testing.T) { }, }, { - pattern: "a?c", + //pattern: "a?c", + items: []item{ + item{item_text, "a"}, + item{item_single, "?"}, + item{item_text, "c"}, + item{item_eof, ""}, + }, tree: &nodePattern{ nodeImpl: nodeImpl{ desc: []node{ @@ -58,7 +80,16 @@ func TestParseString(t *testing.T) { }, }, { - pattern: "[!a-z]", + //pattern: "[!a-z]", + items: []item{ + item{item_range_open, "["}, + item{item_not, "!"}, + item{item_range_lo, "a"}, + item{item_range_between, "-"}, + item{item_range_hi, "z"}, + item{item_range_close, "]"}, + item{item_eof, ""}, + }, tree: &nodePattern{ nodeImpl: nodeImpl{ desc: []node{ @@ -68,7 +99,13 @@ func TestParseString(t *testing.T) { }, }, { - pattern: "[az]", + //pattern: "[az]", + items: []item{ + item{item_range_open, "["}, + item{item_text, "az"}, + item{item_range_close, "]"}, + item{item_eof, ""}, + }, tree: &nodePattern{ nodeImpl: nodeImpl{ desc: []node{ @@ -78,7 +115,15 @@ func TestParseString(t *testing.T) { }, }, { - pattern: "{a,z}", + //pattern: "{a,z}", + items: []item{ + item{item_terms_open, "{"}, + item{item_text, "a"}, + item{item_separator, ","}, + item{item_text, "z"}, + item{item_terms_close, "}"}, + item{item_eof, ""}, + }, tree: &nodePattern{ nodeImpl: nodeImpl{ desc: []node{ @@ -99,7 +144,65 @@ func TestParseString(t *testing.T) { }, }, { - pattern: "{a,{x,y},?,[a-z],[!qwe]}", + //pattern: "/{z,ab}*", + items: []item{ + item{item_text, "/"}, + item{item_terms_open, "{"}, + item{item_text, "z"}, + item{item_separator, ","}, + item{item_text, "ab"}, + item{item_terms_close, "}"}, + item{item_any, "*"}, + item{item_eof, ""}, + }, + tree: &nodePattern{ + nodeImpl: nodeImpl{ + desc: []node{ + &nodeText{text: "/"}, + &nodeAnyOf{nodeImpl: nodeImpl{desc: []node{ + &nodePattern{ + nodeImpl: nodeImpl{desc: []node{ + &nodeText{text: "z"}, + }}, + }, + &nodePattern{ + nodeImpl: nodeImpl{desc: []node{ + &nodeText{text: "ab"}, + }}, + }, + }}}, + &nodeAny{}, + }, + }, + }, + }, + { + //pattern: "{a,{x,y},?,[a-z],[!qwe]}", + items: []item{ + item{item_terms_open, "{"}, + item{item_text, "a"}, + item{item_separator, ","}, + item{item_terms_open, "{"}, + item{item_text, "x"}, + item{item_separator, ","}, + item{item_text, "y"}, + item{item_terms_close, "}"}, + item{item_separator, ","}, + item{item_single, "?"}, + item{item_separator, ","}, + item{item_range_open, "["}, + item{item_range_lo, "a"}, + item{item_range_between, "-"}, + item{item_range_hi, "z"}, + item{item_range_close, "]"}, + item{item_separator, ","}, + item{item_range_open, "["}, + item{item_not, "!"}, + item{item_text, "qwe"}, + item{item_range_close, "]"}, + item{item_terms_close, "}"}, + item{item_eof, ""}, + }, tree: &nodePattern{ nodeImpl: nodeImpl{ desc: []node{ @@ -150,7 +253,9 @@ func TestParseString(t *testing.T) { }, }, } { - pattern, err := parse(newLexer(test.pattern)) + lexer := &stubLexer{Items: test.items} + pattern, err := parse(lexer) + if err != nil { t.Errorf("#%d %s", id, err) continue From eabde343bdb505425c4ad501d85f0915d95dacc6 Mon Sep 17 00:00:00 2001 From: gobwas Date: Sat, 14 May 2016 21:38:00 +0300 Subject: [PATCH 2/3] bugfix, cleaning up --- compiler.go | 196 +++--------------------------------------------- match/any_of.go | 13 ++-- 2 files changed, 17 insertions(+), 192 deletions(-) diff --git a/compiler.go b/compiler.go index 0bb745e..bc48559 100644 --- a/compiler.go +++ b/compiler.go @@ -385,20 +385,25 @@ func compileMatchers(matchers []match.Matcher) (match.Matcher, error) { return m, nil } - var ( - val match.Matcher - idx int - ) + idx := -1 maxLen := -1 + var val match.Matcher for i, matcher := range matchers { - l := matcher.Len() - if l >= maxLen { + if l := matcher.Len(); l != -1 && l >= maxLen { maxLen = l idx = i val = matcher } } + if val == nil { // not found matcher with static length + r, err := compileMatchers(matchers[1:]) + if err != nil { + return nil, err + } + return match.NewBTree(matchers[0], nil, r), nil + } + left := matchers[:idx] var right []match.Matcher if len(matchers) > idx+1 { @@ -424,73 +429,7 @@ func compileMatchers(matchers []match.Matcher) (match.Matcher, error) { return match.NewBTree(val, l, r), nil } -//func complexity(m match.Matcher) int { -// var matchers []match.Matcher -// var k int -// -// switch matcher := m.(type) { -// -// case match.Nothing: -// return 0 -// -// case match.Max, match.Range, match.Suffix, match.Text: -// return 1 -// -// case match.PrefixSuffix, match.Single, match.Row: -// return 2 -// -// case match.Any, match.Contains, match.List, match.Min, match.Prefix, match.Super: -// return 4 -// -// case match.BTree: -// matchers = append(matchers, matcher.Value) -// if matcher.Left != nil { -// matchers = append(matchers, matcher.Left) -// } -// if matcher.Right != nil { -// matchers = append(matchers, matcher.Right) -// } -// k = 1 -// -// case match.AnyOf: -// matchers = matcher.Matchers -// k = 1 -// case match.EveryOf: -// matchers = matcher.Matchers -// k = 1 -// -// default: -// return 0 -// } -// -// var sum int -// for _, m := range matchers { -// sum += complexity(m) -// } -// -// return sum * k -//} - -func doAnyOf(n *nodeAnyOf, s []rune) (match.Matcher, error) { - var matchers []match.Matcher - for _, desc := range n.children() { - if desc == nil { - matchers = append(matchers, match.NewNothing()) - continue - } - - m, err := do(desc, s) - if err != nil { - return nil, err - } - matchers = append(matchers, optimize(m)) - } - - return match.NewAnyOf(matchers...), nil -} - func do(leaf node, s []rune) (m match.Matcher, err error) { - fmt.Println("node", reflect.Indirect(reflect.ValueOf(leaf)).Type().Name()) switch n := leaf.(type) { case *nodeAnyOf: // todo this could be faster on pattern_alternatives_combine_lite @@ -559,120 +498,7 @@ func do(leaf node, s []rune) (m match.Matcher, err error) { return optimize(m), nil } -func do2(node node, s []rune) ([]match.Matcher, error) { - var result []match.Matcher - - switch n := node.(type) { - - case *nodePattern: - ways := [][]match.Matcher{[]match.Matcher{}} - - for _, desc := range node.children() { - variants, err := do2(desc, s) - if err != nil { - return nil, err - } - - fmt.Println("variants pat", variants) - - for i, l := 0, len(ways); i < l; i++ { - for i := 0; i < len(variants); i++ { - o := optimize(variants[i]) - if i == len(variants)-1 { - ways[i] = append(ways[i], o) - } else { - var w []match.Matcher - copy(w, ways[i]) - ways = append(ways, append(w, o)) - } - } - } - - fmt.Println("ways pat", ways) - } - - for _, matchers := range ways { - c, err := compileMatchers(minimizeMatchers(matchers)) - if err != nil { - return nil, err - } - result = append(result, c) - } - - case *nodeAnyOf: - ways := make([][]match.Matcher, len(node.children())) - for _, desc := range node.children() { - variants, err := do2(desc, s) - if err != nil { - return nil, err - } - - fmt.Println("variants any", variants) - - for x, l := 0, len(ways); x < l; x++ { - for i := 0; i < len(variants); i++ { - o := optimize(variants[i]) - if i == len(variants)-1 { - ways[x] = append(ways[x], o) - } else { - var w []match.Matcher - copy(w, ways[x]) - ways = append(ways, append(w, o)) - } - } - } - - fmt.Println("ways any", ways) - } - - for _, matchers := range ways { - c, err := compileMatchers(minimizeMatchers(matchers)) - if err != nil { - return nil, err - } - result = append(result, c) - } - - case *nodeList: - result = append(result, match.NewList([]rune(n.chars), n.not)) - - case *nodeRange: - result = append(result, match.NewRange(n.lo, n.hi, n.not)) - - case *nodeAny: - result = append(result, match.NewAny(s)) - - case *nodeSuper: - result = append(result, match.NewSuper()) - - case *nodeSingle: - result = append(result, match.NewSingle(s)) - - case *nodeText: - result = append(result, match.NewText(n.text)) - - default: - return nil, fmt.Errorf("could not compile tree: unknown node type") - } - - for i, m := range result { - result[i] = optimize(m) - } - - return result, nil -} - func compile(ast *nodePattern, s []rune) (Glob, error) { - // ms, err := do2(ast, s) - // if err != nil { - // return nil, err - // } - // if len(ms) == 1 { - // return ms[0], nil - // } else { - // return match.NewAnyOf(ms), nil - // } - g, err := do(ast, s) if err != nil { return nil, err diff --git a/match/any_of.go b/match/any_of.go index ffae2f6..6037043 100644 --- a/match/any_of.go +++ b/match/any_of.go @@ -63,16 +63,15 @@ func (self AnyOf) Len() (l int) { l = -1 for _, m := range self.Matchers { ml := m.Len() - if ml == -1 { - return -1 - } - - if l == -1 { + switch { + case l == -1: l = ml continue - } - if l != ml { + case ml == -1: + return -1 + + case l != ml: return -1 } } From 994ba33cd955c0fe082a5d5e6e6f7366e608c6b1 Mon Sep 17 00:00:00 2001 From: gobwas Date: Sat, 14 May 2016 22:08:32 +0300 Subject: [PATCH 3/3] enable tests, fix lexer --- compiler_test.go | 382 +++++++++++++++++++++++------------------------ glob_test.go | 6 +- lexer.go | 22 +-- lexer_test.go | 26 +++- 4 files changed, 226 insertions(+), 210 deletions(-) diff --git a/compiler_test.go b/compiler_test.go index 8b6b34c..4c20db2 100644 --- a/compiler_test.go +++ b/compiler_test.go @@ -207,70 +207,70 @@ func TestCompiler(t *testing.T) { result Glob sep []rune }{ - //{ - // ast: pattern(&nodeText{text: "abc"}), - // result: match.NewText("abc"), - //}, - //{ - // ast: pattern(&nodeAny{}), - // sep: separators, - // result: match.NewAny(separators), - //}, - //{ - // ast: pattern(&nodeAny{}), - // result: match.NewSuper(), - //}, - //{ - // ast: pattern(&nodeSuper{}), - // result: match.NewSuper(), - //}, - //{ - // ast: pattern(&nodeSingle{}), - // sep: separators, - // result: match.NewSingle(separators), - //}, - //{ - // ast: pattern(&nodeRange{ - // lo: 'a', - // hi: 'z', - // not: true, - // }), - // result: match.NewRange('a', 'z', true), - //}, - //{ - // ast: pattern(&nodeList{ - // chars: "abc", - // not: true, - // }), - // result: match.NewList([]rune{'a', 'b', 'c'}, true), - //}, - //{ - // ast: pattern(&nodeAny{}, &nodeSingle{}, &nodeSingle{}, &nodeSingle{}), - // sep: separators, - // result: match.EveryOf{Matchers: match.Matchers{ - // match.NewMin(3), - // match.NewContains(string(separators), true), - // }}, - //}, - //{ - // ast: pattern(&nodeAny{}, &nodeSingle{}, &nodeSingle{}, &nodeSingle{}), - // result: match.NewMin(3), - //}, - //{ - // ast: pattern(&nodeAny{}, &nodeText{text: "abc"}, &nodeSingle{}), - // sep: separators, - // result: match.NewBTree( - // match.NewRow( - // 4, - // match.Matchers{ - // match.NewText("abc"), - // match.NewSingle(separators), - // }..., - // ), - // match.NewAny(separators), - // nil, - // ), - //}, + { + ast: pattern(&nodeText{text: "abc"}), + result: match.NewText("abc"), + }, + { + ast: pattern(&nodeAny{}), + sep: separators, + result: match.NewAny(separators), + }, + { + ast: pattern(&nodeAny{}), + result: match.NewSuper(), + }, + { + ast: pattern(&nodeSuper{}), + result: match.NewSuper(), + }, + { + ast: pattern(&nodeSingle{}), + sep: separators, + result: match.NewSingle(separators), + }, + { + ast: pattern(&nodeRange{ + lo: 'a', + hi: 'z', + not: true, + }), + result: match.NewRange('a', 'z', true), + }, + { + ast: pattern(&nodeList{ + chars: "abc", + not: true, + }), + result: match.NewList([]rune{'a', 'b', 'c'}, true), + }, + { + ast: pattern(&nodeAny{}, &nodeSingle{}, &nodeSingle{}, &nodeSingle{}), + sep: separators, + result: match.EveryOf{Matchers: match.Matchers{ + match.NewMin(3), + match.NewContains(string(separators), true), + }}, + }, + { + ast: pattern(&nodeAny{}, &nodeSingle{}, &nodeSingle{}, &nodeSingle{}), + result: match.NewMin(3), + }, + { + ast: pattern(&nodeAny{}, &nodeText{text: "abc"}, &nodeSingle{}), + sep: separators, + result: match.NewBTree( + match.NewRow( + 4, + match.Matchers{ + match.NewText("abc"), + match.NewSingle(separators), + }..., + ), + match.NewAny(separators), + nil, + ), + }, { ast: pattern(&nodeText{text: "/"}, anyOf(&nodeText{text: "z"}, &nodeText{text: "ab"}), &nodeSuper{}), sep: separators, @@ -284,133 +284,133 @@ func TestCompiler(t *testing.T) { ), ), }, - //{ - // ast: pattern(&nodeSuper{}, &nodeSingle{}, &nodeText{text: "abc"}, &nodeSingle{}), - // sep: separators, - // result: match.NewBTree( - // match.NewRow( - // 5, - // match.Matchers{ - // match.NewSingle(separators), - // match.NewText("abc"), - // match.NewSingle(separators), - // }..., - // ), - // match.NewSuper(), - // nil, - // ), - //}, - //{ - // ast: pattern(&nodeAny{}, &nodeText{text: "abc"}), - // result: match.NewSuffix("abc"), - //}, - //{ - // ast: pattern(&nodeText{text: "abc"}, &nodeAny{}), - // result: match.NewPrefix("abc"), - //}, - //{ - // ast: pattern(&nodeText{text: "abc"}, &nodeAny{}, &nodeText{text: "def"}), - // result: match.NewPrefixSuffix("abc", "def"), - //}, - //{ - // ast: pattern(&nodeAny{}, &nodeAny{}, &nodeAny{}, &nodeText{text: "abc"}, &nodeAny{}, &nodeAny{}), - // result: match.NewContains("abc", false), - //}, - //{ - // ast: pattern(&nodeAny{}, &nodeAny{}, &nodeAny{}, &nodeText{text: "abc"}, &nodeAny{}, &nodeAny{}), - // sep: separators, - // result: match.NewBTree( - // match.NewText("abc"), - // match.NewAny(separators), - // match.NewAny(separators), - // ), - //}, - //{ - // ast: pattern(&nodeSuper{}, &nodeSingle{}, &nodeText{text: "abc"}, &nodeSuper{}, &nodeSingle{}), - // result: match.NewBTree( - // match.NewText("abc"), - // match.NewMin(1), - // match.NewMin(1), - // ), - //}, - //{ - // ast: pattern(anyOf(&nodeText{text: "abc"})), - // result: match.NewText("abc"), - //}, - //{ - // ast: pattern(anyOf(pattern(anyOf(pattern(&nodeText{text: "abc"}))))), - // result: match.NewText("abc"), - //}, - //{ - // ast: pattern(anyOf( - // pattern( - // &nodeText{text: "abc"}, - // &nodeSingle{}, - // ), - // pattern( - // &nodeText{text: "abc"}, - // &nodeList{chars: "def"}, - // ), - // pattern( - // &nodeText{text: "abc"}, - // ), - // pattern( - // &nodeText{text: "abc"}, - // ), - // )), - // result: match.NewBTree( - // match.NewText("abc"), - // nil, - // match.AnyOf{Matchers: match.Matchers{ - // match.NewSingle(nil), - // match.NewList([]rune{'d', 'e', 'f'}, false), - // match.NewNothing(), - // }}, - // ), - //}, - //{ - // ast: pattern( - // &nodeRange{lo: 'a', hi: 'z'}, - // &nodeRange{lo: 'a', hi: 'x', not: true}, - // &nodeAny{}, - // ), - // result: match.NewBTree( - // match.NewRow( - // 2, - // match.Matchers{ - // match.NewRange('a', 'z', false), - // match.NewRange('a', 'x', true), - // }..., - // ), - // nil, - // match.NewSuper(), - // ), - //}, - //{ - // ast: pattern(anyOf( - // pattern( - // &nodeText{text: "abc"}, - // &nodeList{chars: "abc"}, - // &nodeText{text: "ghi"}, - // ), - // pattern( - // &nodeText{text: "abc"}, - // &nodeList{chars: "def"}, - // &nodeText{text: "ghi"}, - // ), - // )), - // result: match.NewRow( - // 7, - // match.Matchers{ - // match.NewText("abc"), - // match.AnyOf{Matchers: match.Matchers{ - // match.NewList([]rune{'a', 'b', 'c'}, false), - // match.NewList([]rune{'d', 'e', 'f'}, false), - // }}, - // match.NewText("ghi"), - // }..., - // ), - //}, + { + ast: pattern(&nodeSuper{}, &nodeSingle{}, &nodeText{text: "abc"}, &nodeSingle{}), + sep: separators, + result: match.NewBTree( + match.NewRow( + 5, + match.Matchers{ + match.NewSingle(separators), + match.NewText("abc"), + match.NewSingle(separators), + }..., + ), + match.NewSuper(), + nil, + ), + }, + { + ast: pattern(&nodeAny{}, &nodeText{text: "abc"}), + result: match.NewSuffix("abc"), + }, + { + ast: pattern(&nodeText{text: "abc"}, &nodeAny{}), + result: match.NewPrefix("abc"), + }, + { + ast: pattern(&nodeText{text: "abc"}, &nodeAny{}, &nodeText{text: "def"}), + result: match.NewPrefixSuffix("abc", "def"), + }, + { + ast: pattern(&nodeAny{}, &nodeAny{}, &nodeAny{}, &nodeText{text: "abc"}, &nodeAny{}, &nodeAny{}), + result: match.NewContains("abc", false), + }, + { + ast: pattern(&nodeAny{}, &nodeAny{}, &nodeAny{}, &nodeText{text: "abc"}, &nodeAny{}, &nodeAny{}), + sep: separators, + result: match.NewBTree( + match.NewText("abc"), + match.NewAny(separators), + match.NewAny(separators), + ), + }, + { + ast: pattern(&nodeSuper{}, &nodeSingle{}, &nodeText{text: "abc"}, &nodeSuper{}, &nodeSingle{}), + result: match.NewBTree( + match.NewText("abc"), + match.NewMin(1), + match.NewMin(1), + ), + }, + { + ast: pattern(anyOf(&nodeText{text: "abc"})), + result: match.NewText("abc"), + }, + { + ast: pattern(anyOf(pattern(anyOf(pattern(&nodeText{text: "abc"}))))), + result: match.NewText("abc"), + }, + { + ast: pattern(anyOf( + pattern( + &nodeText{text: "abc"}, + &nodeSingle{}, + ), + pattern( + &nodeText{text: "abc"}, + &nodeList{chars: "def"}, + ), + pattern( + &nodeText{text: "abc"}, + ), + pattern( + &nodeText{text: "abc"}, + ), + )), + result: match.NewBTree( + match.NewText("abc"), + nil, + match.AnyOf{Matchers: match.Matchers{ + match.NewSingle(nil), + match.NewList([]rune{'d', 'e', 'f'}, false), + match.NewNothing(), + }}, + ), + }, + { + ast: pattern( + &nodeRange{lo: 'a', hi: 'z'}, + &nodeRange{lo: 'a', hi: 'x', not: true}, + &nodeAny{}, + ), + result: match.NewBTree( + match.NewRow( + 2, + match.Matchers{ + match.NewRange('a', 'z', false), + match.NewRange('a', 'x', true), + }..., + ), + nil, + match.NewSuper(), + ), + }, + { + ast: pattern(anyOf( + pattern( + &nodeText{text: "abc"}, + &nodeList{chars: "abc"}, + &nodeText{text: "ghi"}, + ), + pattern( + &nodeText{text: "abc"}, + &nodeList{chars: "def"}, + &nodeText{text: "ghi"}, + ), + )), + result: match.NewRow( + 7, + match.Matchers{ + match.NewText("abc"), + match.AnyOf{Matchers: match.Matchers{ + match.NewList([]rune{'a', 'b', 'c'}, false), + match.NewList([]rune{'d', 'e', 'f'}, false), + }}, + match.NewText("ghi"), + }..., + ), + }, } { m, err := compile(test.ast, test.sep) if err != nil { diff --git a/glob_test.go b/glob_test.go index ff28e2f..c850ce0 100644 --- a/glob_test.go +++ b/glob_test.go @@ -166,7 +166,11 @@ func TestQuoteMeta(t *testing.T) { }{ { in: `[foo*]`, - out: `\[foo\*\]`, + out: `\[foo\*]`, + }, + { + in: `{foo*}`, + out: `\{foo\*\}`, }, { in: string(specials), diff --git a/lexer.go b/lexer.go index 75f970a..37c1ec4 100644 --- a/lexer.go +++ b/lexer.go @@ -25,7 +25,6 @@ var specials = []byte{ char_single, char_escape, char_range_open, - char_range_close, char_terms_open, char_terms_close, } @@ -283,20 +282,20 @@ func lexRaw(l *lexer) stateFn { return lexTermsOpen case char_terms_close: - l.unread() - return lexTermsClose + if l.inTerms() { // if we are in terms + l.unread() + return lexTermsClose + } case char_comma: - if l.inTerms() { // if we are not in terms + if l.inTerms() { // if we are in terms l.unread() return lexSeparator } - fallthrough - - default: - l.unread() - return lexText } + + l.unread() + return lexText } if l.pos > l.start { @@ -339,7 +338,10 @@ scan: escaped = false } - l.emit(item_text, string(data)) + if len(data) > 0 { + l.emit(item_text, string(data)) + } + return lexRaw } diff --git a/lexer_test.go b/lexer_test.go index 48fbcf3..7e5f226 100644 --- a/lexer_test.go +++ b/lexer_test.go @@ -16,14 +16,24 @@ func TestLexGood(t *testing.T) { item{item_eof, ""}, }, }, - //{ - // TODO(gobwas): this is freezing on globtest/globdraw (`]]` syntax error) - //pattern: "/{rate,[0-9]]}*", - //items: []item{ - // item{item_text, "hello"}, - // item{item_eof, ""}, - //}, - //}, + { + pattern: "/{rate,[0-9]]}*", + items: []item{ + item{item_text, "/"}, + item{item_terms_open, "{"}, + item{item_text, "rate"}, + item{item_separator, ","}, + item{item_range_open, "["}, + item{item_range_lo, "0"}, + item{item_range_between, "-"}, + item{item_range_hi, "9"}, + item{item_range_close, "]"}, + item{item_text, "]"}, + item{item_terms_close, "}"}, + item{item_any, "*"}, + item{item_eof, ""}, + }, + }, { pattern: "hello,world", items: []item{