refactoring

This commit is contained in:
gobwas 2016-05-30 19:35:53 +03:00
parent 2d733288bc
commit 510a1756cf
16 changed files with 1207 additions and 1194 deletions

View File

@ -1,4 +1,4 @@
package glob
package compiler
// TODO use constructor with all matchers, and to their structs private
// TODO glue multiple Text nodes (like after QuoteMeta)
@ -6,11 +6,12 @@ package glob
import (
"fmt"
"github.com/gobwas/glob/match"
"github.com/gobwas/glob/runes"
"github.com/gobwas/glob/syntax/ast"
"github.com/gobwas/glob/util/runes"
"reflect"
)
func optimize(matcher match.Matcher) match.Matcher {
func optimizeMatcher(matcher match.Matcher) match.Matcher {
switch m := matcher.(type) {
case match.Any:
@ -33,8 +34,8 @@ func optimize(matcher match.Matcher) match.Matcher {
return m
case match.BTree:
m.Left = optimize(m.Left)
m.Right = optimize(m.Right)
m.Left = optimizeMatcher(m.Left)
m.Right = optimizeMatcher(m.Right)
r, ok := m.Value.(match.Text)
if !ok {
@ -80,34 +81,72 @@ func optimize(matcher match.Matcher) match.Matcher {
return matcher
}
func glueMatchers(matchers []match.Matcher) match.Matcher {
var (
glued []match.Matcher
winner match.Matcher
)
func compileMatchers(matchers []match.Matcher) (match.Matcher, error) {
if len(matchers) == 0 {
return nil, fmt.Errorf("compile error: need at least one matcher")
}
if len(matchers) == 1 {
return matchers[0], nil
}
if m := glueMatchers(matchers); m != nil {
return m, nil
}
idx := -1
maxLen := -1
if m := glueAsEvery(matchers); m != nil {
glued = append(glued, m)
return m
}
if m := glueAsRow(matchers); m != nil {
glued = append(glued, m)
return m
}
for _, g := range glued {
if l := g.Len(); l > maxLen {
var val match.Matcher
for i, matcher := range matchers {
if l := matcher.Len(); l != -1 && l >= maxLen {
maxLen = l
winner = g
idx = i
val = matcher
}
}
return winner
if val == nil { // not found matcher with static length
r, err := compileMatchers(matchers[1:])
if err != nil {
return nil, err
}
return match.NewBTree(matchers[0], nil, r), nil
}
func glueAsRow(matchers []match.Matcher) match.Matcher {
left := matchers[:idx]
var right []match.Matcher
if len(matchers) > idx+1 {
right = matchers[idx+1:]
}
var l, r match.Matcher
var err error
if len(left) > 0 {
l, err = compileMatchers(left)
if err != nil {
return nil, err
}
}
if len(right) > 0 {
r, err = compileMatchers(right)
if err != nil {
return nil, err
}
}
return match.NewBTree(val, l, r), nil
}
func glueMatchers(matchers []match.Matcher) match.Matcher {
if m := glueMatchersAsEvery(matchers); m != nil {
return m
}
if m := glueMatchersAsRow(matchers); m != nil {
return m
}
return nil
}
func glueMatchersAsRow(matchers []match.Matcher) match.Matcher {
if len(matchers) <= 1 {
return nil
}
@ -124,11 +163,10 @@ func glueAsRow(matchers []match.Matcher) match.Matcher {
l += ml
}
}
return match.NewRow(l, c...)
}
func glueAsEvery(matchers []match.Matcher) match.Matcher {
func glueMatchersAsEvery(matchers []match.Matcher) match.Matcher {
if len(matchers) <= 1 {
return nil
}
@ -254,255 +292,212 @@ func minimizeMatchers(matchers []match.Matcher) []match.Matcher {
return minimizeMatchers(next)
}
func minimizeAnyOf(children []node) node {
var nodes [][]node
var min int
var idx int
for i, desc := range children {
pat, ok := desc.(*nodePattern)
if !ok {
// minimizeAnyOf tries to apply some heuristics to minimize number of nodes in given tree
func minimizeTree(tree *ast.Node) *ast.Node {
switch tree.Kind {
case ast.KindAnyOf:
return minimizeTreeAnyOf(tree)
default:
return nil
}
}
// minimizeAnyOf tries to find common children of given node of AnyOf pattern
// it searches for common children from left and from right
// if any common children are found then it returns new optimized ast tree
// else it returns nil
func minimizeTreeAnyOf(tree *ast.Node) *ast.Node {
if !areOfSameKind(tree.Children, ast.KindPattern) {
return nil
}
n := pat.children()
ln := len(n)
if len(nodes) == 0 || (ln < min) {
min = ln
idx = i
}
nodes = append(nodes, pat.children())
}
minNodes := nodes[idx]
if idx+1 < len(nodes) {
nodes = append(nodes[:idx], nodes[idx+1:]...)
} else {
nodes = nodes[:idx]
}
var commonLeft []node
var commonLeftCount int
for i, n := range minNodes {
has := true
for _, t := range nodes {
if !reflect.DeepEqual(n, t[i]) {
has = false
break
}
}
if has {
commonLeft = append(commonLeft, n)
commonLeftCount++
} else {
break
}
}
var commonRight []node
var commonRightCount int
for i := min - 1; i > commonLeftCount-1; i-- {
n := minNodes[i]
has := true
for _, t := range nodes {
if !reflect.DeepEqual(n, t[len(t)-(min-i)]) {
has = false
break
}
}
if has {
commonRight = append(commonRight, n)
commonRightCount++
} else {
break
}
}
if commonLeftCount == 0 && commonRightCount == 0 {
commonLeft, commonRight := commonChildren(tree.Children)
commonLeftCount, commonRightCount := len(commonLeft), len(commonRight)
if commonLeftCount == 0 && commonRightCount == 0 { // there are no common parts
return nil
}
nodes = append(nodes, minNodes)
nodes[len(nodes)-1], nodes[idx] = nodes[idx], nodes[len(nodes)-1]
var result []node
var result []*ast.Node
if commonLeftCount > 0 {
result = append(result, &nodePattern{nodeImpl: nodeImpl{desc: commonLeft}})
result = append(result, ast.NewNode(ast.KindPattern, nil, commonLeft...))
}
var anyOf []node
for _, n := range nodes {
if commonLeftCount+commonRightCount == len(n) {
anyOf = append(anyOf, nil)
var anyOf []*ast.Node
for _, child := range tree.Children {
reuse := child.Children[commonLeftCount : len(child.Children)-commonRightCount]
var node *ast.Node
if len(reuse) == 0 {
// this pattern is completely reduced by commonLeft and commonRight patterns
// so it become nothing
node = ast.NewNode(ast.KindNothing, nil)
} else {
anyOf = append(anyOf, &nodePattern{nodeImpl: nodeImpl{desc: n[commonLeftCount : len(n)-commonRightCount]}})
node = ast.NewNode(ast.KindPattern, nil, reuse...)
}
anyOf = appendIfUnique(anyOf, node)
}
anyOf = uniqueNodes(anyOf)
if len(anyOf) == 1 {
if anyOf[0] != nil {
result = append(result, &nodePattern{nodeImpl: nodeImpl{desc: anyOf}})
}
} else {
result = append(result, &nodeAnyOf{nodeImpl: nodeImpl{desc: anyOf}})
switch {
case len(anyOf) == 1 && anyOf[0].Kind != ast.KindNothing:
result = append(result, anyOf[0])
case len(anyOf) > 1:
result = append(result, ast.NewNode(ast.KindAnyOf, nil, anyOf...))
}
if commonRightCount > 0 {
result = append(result, &nodePattern{nodeImpl: nodeImpl{desc: commonRight}})
result = append(result, ast.NewNode(ast.KindPattern, nil, commonRight...))
}
return &nodePattern{nodeImpl: nodeImpl{desc: result}}
return ast.NewNode(ast.KindPattern, nil, result...)
}
func uniqueNodes(nodes []node) (result []node) {
head:
for _, n := range nodes {
for _, e := range result {
if reflect.DeepEqual(e, n) {
continue head
}
func commonChildren(nodes []*ast.Node) (commonLeft, commonRight []*ast.Node) {
// find node that has least number of children
idx := leastChildren(nodes)
if idx == -1 {
return nil, nil
}
tree := nodes[idx]
result = append(result, n)
}
var (
breakLeft bool
breakRight bool
)
for i, j := 0, len(tree.Children)-1; j >= 0 && !(breakLeft && breakLeft); i, j = i+1, j-1 {
treeLeft := tree.Children[i]
treeRight := tree.Children[j]
return
}
fmt.Println(i, j)
func compileMatchers(matchers []match.Matcher) (match.Matcher, error) {
if len(matchers) == 0 {
return nil, fmt.Errorf("compile error: need at least one matcher")
}
if len(matchers) == 1 {
return matchers[0], nil
}
if m := glueMatchers(matchers); m != nil {
return m, nil
}
idx := -1
maxLen := -1
var val match.Matcher
for i, matcher := range matchers {
if l := matcher.Len(); l != -1 && l >= maxLen {
maxLen = l
idx = i
val = matcher
}
}
if val == nil { // not found matcher with static length
r, err := compileMatchers(matchers[1:])
if err != nil {
return nil, err
}
return match.NewBTree(matchers[0], nil, r), nil
}
left := matchers[:idx]
var right []match.Matcher
if len(matchers) > idx+1 {
right = matchers[idx+1:]
}
var l, r match.Matcher
var err error
if len(left) > 0 {
l, err = compileMatchers(left)
if err != nil {
return nil, err
}
}
if len(right) > 0 {
r, err = compileMatchers(right)
if err != nil {
return nil, err
}
}
return match.NewBTree(val, l, r), nil
}
func do(leaf node, s []rune) (m match.Matcher, err error) {
switch n := leaf.(type) {
case *nodeAnyOf:
// todo this could be faster on pattern_alternatives_combine_lite
if n := minimizeAnyOf(n.children()); n != nil {
return do(n, s)
}
var matchers []match.Matcher
for _, desc := range n.children() {
if desc == nil {
matchers = append(matchers, match.NewNothing())
for k := 0; k < len(nodes) && !(breakLeft && breakLeft); k++ {
// skip least children node
if k == idx {
continue
}
m, err := do(desc, s)
restLeft := nodes[k].Children[i]
restRight := nodes[k].Children[j+len(nodes[k].Children)-len(tree.Children)]
breakLeft = breakLeft || !treeLeft.Equal(restLeft)
// disable searching for right common parts, if left part is already overlapping
breakRight = breakRight || (!breakLeft && j <= i)
breakRight = breakRight || !treeRight.Equal(restRight)
}
if !breakLeft {
fmt.Println("left app")
commonLeft = append(commonLeft, treeLeft)
}
if !breakRight {
fmt.Println("right app")
commonRight = append(commonRight, treeRight)
}
}
return
}
func appendIfUnique(target []*ast.Node, val *ast.Node) []*ast.Node {
for _, n := range target {
if reflect.DeepEqual(n, val) {
return target
}
}
return append(target, val)
}
func areOfSameKind(nodes []*ast.Node, kind ast.Kind) bool {
for _, n := range nodes {
if n.Kind != kind {
return false
}
}
return true
}
func leastChildren(nodes []*ast.Node) int {
min := -1
idx := -1
for i, n := range nodes {
if idx == -1 || (len(n.Children) < min) {
min = len(n.Children)
idx = i
}
}
return idx
}
func compileTreeChildren(tree *ast.Node, sep []rune) ([]match.Matcher, error) {
var matchers []match.Matcher
for _, desc := range tree.Children {
m, err := compile(desc, sep)
if err != nil {
return nil, err
}
matchers = append(matchers, optimize(m))
matchers = append(matchers, optimizeMatcher(m))
}
return matchers, nil
}
func compile(tree *ast.Node, sep []rune) (m match.Matcher, err error) {
switch tree.Kind {
case ast.KindAnyOf:
// todo this could be faster on pattern_alternatives_combine_lite (see glob_test.go)
if n := minimizeTree(tree); n != nil {
return compile(n, sep)
}
matchers, err := compileTreeChildren(tree, sep)
if err != nil {
return nil, err
}
return match.NewAnyOf(matchers...), nil
case *nodePattern:
nodes := leaf.children()
if len(nodes) == 0 {
case ast.KindPattern:
if len(tree.Children) == 0 {
return match.NewNothing(), nil
}
var matchers []match.Matcher
for _, desc := range nodes {
m, err := do(desc, s)
matchers, err := compileTreeChildren(tree, sep)
if err != nil {
return nil, err
}
matchers = append(matchers, optimize(m))
}
m, err = compileMatchers(minimizeMatchers(matchers))
if err != nil {
return nil, err
}
case *nodeList:
m = match.NewList([]rune(n.chars), n.not)
case ast.KindAny:
m = match.NewAny(sep)
case *nodeRange:
m = match.NewRange(n.lo, n.hi, n.not)
case *nodeAny:
m = match.NewAny(s)
case *nodeSuper:
case ast.KindSuper:
m = match.NewSuper()
case *nodeSingle:
m = match.NewSingle(s)
case ast.KindSingle:
m = match.NewSingle(sep)
case *nodeText:
m = match.NewText(n.text)
case ast.KindNothing:
m = match.NewNothing()
case ast.KindList:
l := tree.Value.(ast.List)
m = match.NewList([]rune(l.Chars), l.Not)
case ast.KindRange:
r := tree.Value.(ast.Range)
m = match.NewRange(r.Lo, r.Hi, r.Not)
case ast.KindText:
t := tree.Value.(ast.Text)
m = match.NewText(t.Text)
default:
return nil, fmt.Errorf("could not compile tree: unknown node type")
}
return optimize(m), nil
return optimizeMatcher(m), nil
}
func compile(ast *nodePattern, s []rune) (Glob, error) {
g, err := do(ast, s)
func Compile(tree *ast.Node, sep []rune) (match.Matcher, error) {
m, err := compile(tree, sep)
if err != nil {
return nil, err
}
return g, nil
return m, nil
}

573
compiler/compiler_test.go Normal file
View File

@ -0,0 +1,573 @@
package compiler
import (
"github.com/gobwas/glob/match"
"github.com/gobwas/glob/match/debug"
"github.com/gobwas/glob/syntax/ast"
"reflect"
"testing"
)
var separators = []rune{'.'}
func TestCommonChildren(t *testing.T) {
for i, test := range []struct {
nodes []*ast.Node
left []*ast.Node
right []*ast.Node
}{
// {
// nodes: []*ast.Node{
// ast.NewNode(ast.KindNothing, nil,
// ast.NewNode(ast.KindText, ast.Text{"a"}),
// ast.NewNode(ast.KindText, ast.Text{"z"}),
// ast.NewNode(ast.KindText, ast.Text{"c"}),
// ),
// ast.NewNode(ast.KindNothing, nil,
// ast.NewNode(ast.KindText, ast.Text{"a"}),
// ast.NewNode(ast.KindText, ast.Text{"b"}),
// ast.NewNode(ast.KindText, ast.Text{"c"}),
// ),
// },
// left: []*ast.Node{
// ast.NewNode(ast.KindText, ast.Text{"a"}),
// },
// right: []*ast.Node{
// ast.NewNode(ast.KindText, ast.Text{"c"}),
// },
// },
{
nodes: []*ast.Node{
ast.NewNode(ast.KindNothing, nil,
ast.NewNode(ast.KindText, ast.Text{"a"}),
ast.NewNode(ast.KindText, ast.Text{"b"}),
ast.NewNode(ast.KindText, ast.Text{"c"}),
ast.NewNode(ast.KindText, ast.Text{"d"}),
),
ast.NewNode(ast.KindNothing, nil,
ast.NewNode(ast.KindText, ast.Text{"a"}),
ast.NewNode(ast.KindText, ast.Text{"b"}),
ast.NewNode(ast.KindText, ast.Text{"c"}),
ast.NewNode(ast.KindText, ast.Text{"c"}),
ast.NewNode(ast.KindText, ast.Text{"d"}),
),
},
left: []*ast.Node{
ast.NewNode(ast.KindText, ast.Text{"a"}),
ast.NewNode(ast.KindText, ast.Text{"b"}),
ast.NewNode(ast.KindText, ast.Text{"c"}),
},
right: []*ast.Node{
ast.NewNode(ast.KindText, ast.Text{"d"}),
},
},
} {
left, right := commonChildren(test.nodes)
if !nodesEqual(left, test.left) {
t.Errorf("[%d] left, right := commonChildren(); left = %v; want %v", i, left, test.left)
}
if !nodesEqual(right, test.right) {
t.Errorf("[%d] left, right := commonChildren(); right = %v; want %v", i, right, test.right)
}
}
}
func nodesEqual(a, b []*ast.Node) bool {
if len(a) != len(b) {
return false
}
for i, av := range a {
if !av.Equal(b[i]) {
return false
}
}
return true
}
func TestGlueMatchers(t *testing.T) {
for id, test := range []struct {
in []match.Matcher
exp match.Matcher
}{
{
[]match.Matcher{
match.NewSuper(),
match.NewSingle(nil),
},
match.NewMin(1),
},
{
[]match.Matcher{
match.NewAny(separators),
match.NewSingle(separators),
},
match.EveryOf{match.Matchers{
match.NewMin(1),
match.NewContains(string(separators), true),
}},
},
{
[]match.Matcher{
match.NewSingle(nil),
match.NewSingle(nil),
match.NewSingle(nil),
},
match.EveryOf{match.Matchers{
match.NewMin(3),
match.NewMax(3),
}},
},
{
[]match.Matcher{
match.NewList([]rune{'a'}, true),
match.NewAny([]rune{'a'}),
},
match.EveryOf{match.Matchers{
match.NewMin(1),
match.NewContains("a", true),
}},
},
} {
act, err := compileMatchers(test.in)
if err != nil {
t.Errorf("#%d convert matchers error: %s", id, err)
continue
}
if !reflect.DeepEqual(act, test.exp) {
t.Errorf("#%d unexpected convert matchers result:\nact: %#v;\nexp: %#v", id, act, test.exp)
continue
}
}
}
func TestCompileMatchers(t *testing.T) {
for id, test := range []struct {
in []match.Matcher
exp match.Matcher
}{
{
[]match.Matcher{
match.NewSuper(),
match.NewSingle(separators),
match.NewText("c"),
},
match.NewBTree(
match.NewText("c"),
match.NewBTree(
match.NewSingle(separators),
match.NewSuper(),
nil,
),
nil,
),
},
{
[]match.Matcher{
match.NewAny(nil),
match.NewText("c"),
match.NewAny(nil),
},
match.NewBTree(
match.NewText("c"),
match.NewAny(nil),
match.NewAny(nil),
),
},
{
[]match.Matcher{
match.NewRange('a', 'c', true),
match.NewList([]rune{'z', 't', 'e'}, false),
match.NewText("c"),
match.NewSingle(nil),
},
match.NewRow(
4,
match.Matchers{
match.NewRange('a', 'c', true),
match.NewList([]rune{'z', 't', 'e'}, false),
match.NewText("c"),
match.NewSingle(nil),
}...,
),
},
} {
act, err := compileMatchers(test.in)
if err != nil {
t.Errorf("#%d convert matchers error: %s", id, err)
continue
}
if !reflect.DeepEqual(act, test.exp) {
t.Errorf("#%d unexpected convert matchers result:\nact: %#v\nexp: %#v", id, act, test.exp)
continue
}
}
}
func TestConvertMatchers(t *testing.T) {
for id, test := range []struct {
in, exp []match.Matcher
}{
{
[]match.Matcher{
match.NewRange('a', 'c', true),
match.NewList([]rune{'z', 't', 'e'}, false),
match.NewText("c"),
match.NewSingle(nil),
match.NewAny(nil),
},
[]match.Matcher{
match.NewRow(
4,
[]match.Matcher{
match.NewRange('a', 'c', true),
match.NewList([]rune{'z', 't', 'e'}, false),
match.NewText("c"),
match.NewSingle(nil),
}...,
),
match.NewAny(nil),
},
},
{
[]match.Matcher{
match.NewRange('a', 'c', true),
match.NewList([]rune{'z', 't', 'e'}, false),
match.NewText("c"),
match.NewSingle(nil),
match.NewAny(nil),
match.NewSingle(nil),
match.NewSingle(nil),
match.NewAny(nil),
},
[]match.Matcher{
match.NewRow(
3,
match.Matchers{
match.NewRange('a', 'c', true),
match.NewList([]rune{'z', 't', 'e'}, false),
match.NewText("c"),
}...,
),
match.NewMin(3),
},
},
} {
act := minimizeMatchers(test.in)
if !reflect.DeepEqual(act, test.exp) {
t.Errorf("#%d unexpected convert matchers 2 result:\nact: %#v\nexp: %#v", id, act, test.exp)
continue
}
}
}
func TestCompiler(t *testing.T) {
for id, test := range []struct {
ast *ast.Node
result match.Matcher
sep []rune
}{
// {
// ast: ast.NewNode(ast.KindPattern, nil,
// ast.NewNode(ast.KindText, &ast.Text{"abc"}),
// ),
// result: match.NewText("abc"),
// },
// {
// ast: ast.NewNode(ast.KindPattern, nil,
// ast.NewNode(ast.KindAny, nil),
// ),
// sep: separators,
// result: match.NewAny(separators),
// },
// {
// ast: ast.NewNode(ast.KindPattern, nil,
// ast.NewNode(ast.KindAny, nil),
// ),
// result: match.NewSuper(),
// },
// {
// ast: ast.NewNode(ast.KindPattern, nil,
// ast.NewNode(ast.KindSuper, nil),
// ),
// result: match.NewSuper(),
// },
// {
// ast: ast.NewNode(ast.KindPattern, nil,
// ast.NewNode(ast.KindSingle, nil),
// ),
// sep: separators,
// result: match.NewSingle(separators),
// },
// {
// ast: ast.NewNode(ast.KindPattern, nil,
// ast.NewNode(ast.KindRange, &ast.Range{
// Lo: 'a',
// Hi: 'z',
// Not: true,
// }),
// ),
// result: match.NewRange('a', 'z', true),
// },
// {
// ast: ast.NewNode(ast.KindPattern, nil,
// ast.NewNode(ast.KindList, &ast.List{
// Chars: "abc",
// Not: true,
// }),
// ),
// result: match.NewList([]rune{'a', 'b', 'c'}, true),
// },
// {
// ast: ast.NewNode(ast.KindPattern, nil,
// ast.NewNode(ast.KindAny, nil),
// ast.NewNode(ast.KindSingle, nil),
// ast.NewNode(ast.KindSingle, nil),
// ast.NewNode(ast.KindSingle, nil),
// ),
// sep: separators,
// result: match.EveryOf{Matchers: match.Matchers{
// match.NewMin(3),
// match.NewContains(string(separators), true),
// }},
// },
// {
// ast: ast.NewNode(ast.KindPattern, nil,
// ast.NewNode(ast.KindAny, nil),
// ast.NewNode(ast.KindSingle, nil),
// ast.NewNode(ast.KindSingle, nil),
// ast.NewNode(ast.KindSingle, nil),
// ),
// result: match.NewMin(3),
// },
// {
// ast: ast.NewNode(ast.KindPattern, nil,
// ast.NewNode(ast.KindAny, nil),
// ast.NewNode(ast.KindText, &ast.Text{"abc"}),
// ast.NewNode(ast.KindSingle, nil),
// ),
// sep: separators,
// result: match.NewBTree(
// match.NewRow(
// 4,
// match.Matchers{
// match.NewText("abc"),
// match.NewSingle(separators),
// }...,
// ),
// match.NewAny(separators),
// nil,
// ),
// },
// {
// ast: ast.NewNode(ast.KindPattern, nil,
// ast.NewNode(ast.KindText, &ast.Text{"/"}),
// ast.NewNode(ast.KindAnyOf, nil,
// ast.NewNode(ast.KindText, &ast.Text{"z"}),
// ast.NewNode(ast.KindText, &ast.Text{"ab"}),
// ),
// ast.NewNode(ast.KindSuper, nil),
// ),
// sep: separators,
// result: match.NewBTree(
// match.NewText("/"),
// nil,
// match.NewBTree(
// match.NewAnyOf(match.NewText("z"), match.NewText("ab")),
// nil,
// match.NewSuper(),
// ),
// ),
// },
// {
// ast: ast.NewNode(ast.KindPattern, nil,
// ast.NewNode(ast.KindSuper, nil),
// ast.NewNode(ast.KindSingle, nil),
// ast.NewNode(ast.KindText, &ast.Text{"abc"}),
// ast.NewNode(ast.KindSingle, nil),
// ),
// sep: separators,
// result: match.NewBTree(
// match.NewRow(
// 5,
// match.Matchers{
// match.NewSingle(separators),
// match.NewText("abc"),
// match.NewSingle(separators),
// }...,
// ),
// match.NewSuper(),
// nil,
// ),
// },
// {
// ast: ast.NewNode(ast.KindPattern, nil,
// ast.NewNode(ast.KindAny, nil),
// ast.NewNode(ast.KindText, &ast.Text{"abc"}),
// ),
// result: match.NewSuffix("abc"),
// },
// {
// ast: ast.NewNode(ast.KindPattern, nil,
// ast.NewNode(ast.KindText, &ast.Text{"abc"}),
// ast.NewNode(ast.KindAny, nil),
// ),
// result: match.NewPrefix("abc"),
// },
// {
// ast: ast.NewNode(ast.KindPattern, nil,
// ast.NewNode(ast.KindText, &ast.Text{"abc"}),
// ast.NewNode(ast.KindAny, nil),
// ast.NewNode(ast.KindText, &ast.Text{"def"}),
// ),
// result: match.NewPrefixSuffix("abc", "def"),
// },
// {
// ast: ast.NewNode(ast.KindPattern, nil,
// ast.NewNode(ast.KindAny, nil),
// ast.NewNode(ast.KindAny, nil),
// ast.NewNode(ast.KindAny, nil),
// ast.NewNode(ast.KindText, &ast.Text{"abc"}),
// ast.NewNode(ast.KindAny, nil),
// ast.NewNode(ast.KindAny, nil),
// ),
// result: match.NewContains("abc", false),
// },
// {
// ast: ast.NewNode(ast.KindPattern, nil,
// ast.NewNode(ast.KindAny, nil),
// ast.NewNode(ast.KindAny, nil),
// ast.NewNode(ast.KindAny, nil),
// ast.NewNode(ast.KindText, &ast.Text{"abc"}),
// ast.NewNode(ast.KindAny, nil),
// ast.NewNode(ast.KindAny, nil),
// ),
// sep: separators,
// result: match.NewBTree(
// match.NewText("abc"),
// match.NewAny(separators),
// match.NewAny(separators),
// ),
// },
// {
// ast: ast.NewNode(ast.KindPattern, nil,
// ast.NewNode(ast.KindSuper, nil),
// ast.NewNode(ast.KindSingle, nil),
// ast.NewNode(ast.KindText, &ast.Text{"abc"}),
// ast.NewNode(ast.KindSuper, nil),
// ast.NewNode(ast.KindSingle, nil),
// ),
// result: match.NewBTree(
// match.NewText("abc"),
// match.NewMin(1),
// match.NewMin(1),
// ),
// },
// {
// ast: ast.NewNode(ast.KindPattern, nil,
// ast.NewNode(ast.KindText, &ast.Text{"abc"}),
// ),
// result: match.NewText("abc"),
// },
// {
// ast: ast.NewNode(ast.KindPattern, nil,
// ast.NewNode(ast.KindAnyOf, nil,
// ast.NewNode(ast.KindPattern, nil,
// ast.NewNode(ast.KindAnyOf, nil,
// ast.NewNode(ast.KindPattern, nil,
// ast.NewNode(ast.KindText, &ast.Text{"abc"}),
// ),
// ),
// ),
// ),
// ),
// result: match.NewText("abc"),
// },
{
ast: ast.NewNode(ast.KindPattern, nil,
ast.NewNode(ast.KindAnyOf, nil,
ast.NewNode(ast.KindPattern, nil,
ast.NewNode(ast.KindText, &ast.Text{"abc"}),
ast.NewNode(ast.KindSingle, nil),
),
ast.NewNode(ast.KindPattern, nil,
ast.NewNode(ast.KindText, &ast.Text{"abc"}),
ast.NewNode(ast.KindList, &ast.List{Chars: "def"}),
),
ast.NewNode(ast.KindPattern, nil,
ast.NewNode(ast.KindText, &ast.Text{"abc"}),
),
ast.NewNode(ast.KindPattern, nil,
ast.NewNode(ast.KindText, &ast.Text{"abc"}),
),
),
),
result: match.NewBTree(
match.NewText("abc"),
nil,
match.AnyOf{Matchers: match.Matchers{
match.NewSingle(nil),
match.NewList([]rune{'d', 'e', 'f'}, false),
match.NewNothing(),
}},
),
},
// {
// ast: ast.NewNode(ast.KindPattern, nil,
// ast.NewNode(ast.KindRange, &ast.Range{Lo: 'a', Hi: 'z'}),
// ast.NewNode(ast.KindRange, &ast.Range{Lo: 'a', Hi: 'x', Not: true}),
// ast.NewNode(ast.KindAny, nil),
// ),
// result: match.NewBTree(
// match.NewRow(
// 2,
// match.Matchers{
// match.NewRange('a', 'z', false),
// match.NewRange('a', 'x', true),
// }...,
// ),
// nil,
// match.NewSuper(),
// ),
// },
// {
// ast: ast.NewNode(ast.KindPattern, nil,
// ast.NewNode(ast.KindAnyOf, nil,
// ast.NewNode(ast.KindPattern, nil,
// ast.NewNode(ast.KindText, &ast.Text{"abc"}),
// ast.NewNode(ast.KindList, &ast.List{Chars: "abc"}),
// ast.NewNode(ast.KindText, &ast.Text{"ghi"}),
// ),
// ast.NewNode(ast.KindPattern, nil,
// ast.NewNode(ast.KindText, &ast.Text{"abc"}),
// ast.NewNode(ast.KindList, &ast.List{Chars: "def"}),
// ast.NewNode(ast.KindText, &ast.Text{"ghi"}),
// ),
// ),
// ),
// result: match.NewRow(
// 7,
// match.Matchers{
// match.NewText("abc"),
// match.AnyOf{Matchers: match.Matchers{
// match.NewList([]rune{'a', 'b', 'c'}, false),
// match.NewList([]rune{'d', 'e', 'f'}, false),
// }},
// match.NewText("ghi"),
// }...,
// ),
// },
} {
m, err := Compile(test.ast, test.sep)
if err != nil {
t.Errorf("compilation error: %s", err)
continue
}
if !reflect.DeepEqual(m, test.result) {
t.Errorf("[%d] Compile():\nexp: %#v\nact: %#v\n\ngraphviz:\nexp:\n%s\nact:\n%s\n", id, test.result, m, debug.Graphviz("", test.result.(match.Matcher)), debug.Graphviz("", m.(match.Matcher)))
continue
}
}
}

View File

@ -1,548 +0,0 @@
package glob
import (
"github.com/gobwas/glob/match"
"github.com/gobwas/glob/match/debug"
"reflect"
"testing"
)
var separators = []rune{'.'}
func TestGlueMatchers(t *testing.T) {
for id, test := range []struct {
in []match.Matcher
exp match.Matcher
}{
{
[]match.Matcher{
match.NewSuper(),
match.NewSingle(nil),
},
match.NewMin(1),
},
{
[]match.Matcher{
match.NewAny(separators),
match.NewSingle(separators),
},
match.EveryOf{match.Matchers{
match.NewMin(1),
match.NewContains(string(separators), true),
}},
},
{
[]match.Matcher{
match.NewSingle(nil),
match.NewSingle(nil),
match.NewSingle(nil),
},
match.EveryOf{match.Matchers{
match.NewMin(3),
match.NewMax(3),
}},
},
{
[]match.Matcher{
match.NewList([]rune{'a'}, true),
match.NewAny([]rune{'a'}),
},
match.EveryOf{match.Matchers{
match.NewMin(1),
match.NewContains("a", true),
}},
},
} {
act, err := compileMatchers(test.in)
if err != nil {
t.Errorf("#%d convert matchers error: %s", id, err)
continue
}
if !reflect.DeepEqual(act, test.exp) {
t.Errorf("#%d unexpected convert matchers result:\nact: %#v;\nexp: %#v", id, act, test.exp)
continue
}
}
}
func TestCompileMatchers(t *testing.T) {
for id, test := range []struct {
in []match.Matcher
exp match.Matcher
}{
{
[]match.Matcher{
match.NewSuper(),
match.NewSingle(separators),
match.NewText("c"),
},
match.NewBTree(
match.NewText("c"),
match.NewBTree(
match.NewSingle(separators),
match.NewSuper(),
nil,
),
nil,
),
},
{
[]match.Matcher{
match.NewAny(nil),
match.NewText("c"),
match.NewAny(nil),
},
match.NewBTree(
match.NewText("c"),
match.NewAny(nil),
match.NewAny(nil),
),
},
{
[]match.Matcher{
match.NewRange('a', 'c', true),
match.NewList([]rune{'z', 't', 'e'}, false),
match.NewText("c"),
match.NewSingle(nil),
},
match.NewRow(
4,
match.Matchers{
match.NewRange('a', 'c', true),
match.NewList([]rune{'z', 't', 'e'}, false),
match.NewText("c"),
match.NewSingle(nil),
}...,
),
},
} {
act, err := compileMatchers(test.in)
if err != nil {
t.Errorf("#%d convert matchers error: %s", id, err)
continue
}
if !reflect.DeepEqual(act, test.exp) {
t.Errorf("#%d unexpected convert matchers result:\nact: %#v\nexp: %#v", id, act, test.exp)
continue
}
}
}
func TestConvertMatchers(t *testing.T) {
for id, test := range []struct {
in, exp []match.Matcher
}{
{
[]match.Matcher{
match.NewRange('a', 'c', true),
match.NewList([]rune{'z', 't', 'e'}, false),
match.NewText("c"),
match.NewSingle(nil),
match.NewAny(nil),
},
[]match.Matcher{
match.NewRow(
4,
[]match.Matcher{
match.NewRange('a', 'c', true),
match.NewList([]rune{'z', 't', 'e'}, false),
match.NewText("c"),
match.NewSingle(nil),
}...,
),
match.NewAny(nil),
},
},
{
[]match.Matcher{
match.NewRange('a', 'c', true),
match.NewList([]rune{'z', 't', 'e'}, false),
match.NewText("c"),
match.NewSingle(nil),
match.NewAny(nil),
match.NewSingle(nil),
match.NewSingle(nil),
match.NewAny(nil),
},
[]match.Matcher{
match.NewRow(
3,
match.Matchers{
match.NewRange('a', 'c', true),
match.NewList([]rune{'z', 't', 'e'}, false),
match.NewText("c"),
}...,
),
match.NewMin(3),
},
},
} {
act := minimizeMatchers(test.in)
if !reflect.DeepEqual(act, test.exp) {
t.Errorf("#%d unexpected convert matchers 2 result:\nact: %#v\nexp: %#v", id, act, test.exp)
continue
}
}
}
func pattern(nodes ...node) *nodePattern {
return &nodePattern{
nodeImpl: nodeImpl{
desc: nodes,
},
}
}
func anyOf(nodes ...node) *nodeAnyOf {
return &nodeAnyOf{
nodeImpl: nodeImpl{
desc: nodes,
},
}
}
func TestCompiler(t *testing.T) {
for id, test := range []struct {
ast *nodePattern
result Glob
sep []rune
}{
{
ast: pattern(&nodeText{text: "abc"}),
result: match.NewText("abc"),
},
{
ast: pattern(&nodeAny{}),
sep: separators,
result: match.NewAny(separators),
},
{
ast: pattern(&nodeAny{}),
result: match.NewSuper(),
},
{
ast: pattern(&nodeSuper{}),
result: match.NewSuper(),
},
{
ast: pattern(&nodeSingle{}),
sep: separators,
result: match.NewSingle(separators),
},
{
ast: pattern(&nodeRange{
lo: 'a',
hi: 'z',
not: true,
}),
result: match.NewRange('a', 'z', true),
},
{
ast: pattern(&nodeList{
chars: "abc",
not: true,
}),
result: match.NewList([]rune{'a', 'b', 'c'}, true),
},
{
ast: pattern(&nodeAny{}, &nodeSingle{}, &nodeSingle{}, &nodeSingle{}),
sep: separators,
result: match.EveryOf{Matchers: match.Matchers{
match.NewMin(3),
match.NewContains(string(separators), true),
}},
},
{
ast: pattern(&nodeAny{}, &nodeSingle{}, &nodeSingle{}, &nodeSingle{}),
result: match.NewMin(3),
},
{
ast: pattern(&nodeAny{}, &nodeText{text: "abc"}, &nodeSingle{}),
sep: separators,
result: match.NewBTree(
match.NewRow(
4,
match.Matchers{
match.NewText("abc"),
match.NewSingle(separators),
}...,
),
match.NewAny(separators),
nil,
),
},
{
ast: pattern(&nodeText{text: "/"}, anyOf(&nodeText{text: "z"}, &nodeText{text: "ab"}), &nodeSuper{}),
sep: separators,
result: match.NewBTree(
match.NewText("/"),
nil,
match.NewBTree(
match.NewAnyOf(match.NewText("z"), match.NewText("ab")),
nil,
match.NewSuper(),
),
),
},
{
ast: pattern(&nodeSuper{}, &nodeSingle{}, &nodeText{text: "abc"}, &nodeSingle{}),
sep: separators,
result: match.NewBTree(
match.NewRow(
5,
match.Matchers{
match.NewSingle(separators),
match.NewText("abc"),
match.NewSingle(separators),
}...,
),
match.NewSuper(),
nil,
),
},
{
ast: pattern(&nodeAny{}, &nodeText{text: "abc"}),
result: match.NewSuffix("abc"),
},
{
ast: pattern(&nodeText{text: "abc"}, &nodeAny{}),
result: match.NewPrefix("abc"),
},
{
ast: pattern(&nodeText{text: "abc"}, &nodeAny{}, &nodeText{text: "def"}),
result: match.NewPrefixSuffix("abc", "def"),
},
{
ast: pattern(&nodeAny{}, &nodeAny{}, &nodeAny{}, &nodeText{text: "abc"}, &nodeAny{}, &nodeAny{}),
result: match.NewContains("abc", false),
},
{
ast: pattern(&nodeAny{}, &nodeAny{}, &nodeAny{}, &nodeText{text: "abc"}, &nodeAny{}, &nodeAny{}),
sep: separators,
result: match.NewBTree(
match.NewText("abc"),
match.NewAny(separators),
match.NewAny(separators),
),
},
{
ast: pattern(&nodeSuper{}, &nodeSingle{}, &nodeText{text: "abc"}, &nodeSuper{}, &nodeSingle{}),
result: match.NewBTree(
match.NewText("abc"),
match.NewMin(1),
match.NewMin(1),
),
},
{
ast: pattern(anyOf(&nodeText{text: "abc"})),
result: match.NewText("abc"),
},
{
ast: pattern(anyOf(pattern(anyOf(pattern(&nodeText{text: "abc"}))))),
result: match.NewText("abc"),
},
{
ast: pattern(anyOf(
pattern(
&nodeText{text: "abc"},
&nodeSingle{},
),
pattern(
&nodeText{text: "abc"},
&nodeList{chars: "def"},
),
pattern(
&nodeText{text: "abc"},
),
pattern(
&nodeText{text: "abc"},
),
)),
result: match.NewBTree(
match.NewText("abc"),
nil,
match.AnyOf{Matchers: match.Matchers{
match.NewSingle(nil),
match.NewList([]rune{'d', 'e', 'f'}, false),
match.NewNothing(),
}},
),
},
{
ast: pattern(
&nodeRange{lo: 'a', hi: 'z'},
&nodeRange{lo: 'a', hi: 'x', not: true},
&nodeAny{},
),
result: match.NewBTree(
match.NewRow(
2,
match.Matchers{
match.NewRange('a', 'z', false),
match.NewRange('a', 'x', true),
}...,
),
nil,
match.NewSuper(),
),
},
{
ast: pattern(anyOf(
pattern(
&nodeText{text: "abc"},
&nodeList{chars: "abc"},
&nodeText{text: "ghi"},
),
pattern(
&nodeText{text: "abc"},
&nodeList{chars: "def"},
&nodeText{text: "ghi"},
),
)),
result: match.NewRow(
7,
match.Matchers{
match.NewText("abc"),
match.AnyOf{Matchers: match.Matchers{
match.NewList([]rune{'a', 'b', 'c'}, false),
match.NewList([]rune{'d', 'e', 'f'}, false),
}},
match.NewText("ghi"),
}...,
),
},
} {
m, err := compile(test.ast, test.sep)
if err != nil {
t.Errorf("compilation error: %s", err)
continue
}
if !reflect.DeepEqual(m, test.result) {
t.Errorf("#%d results are not equal:\nexp: %#v\nact: %#v\nexp:\n%s\nact:\n%s\n", id, test.result, m, debug.Graphviz("", test.result.(match.Matcher)), debug.Graphviz("", m.(match.Matcher)))
continue
}
}
}
const complexityString = "abcd"
//func BenchmarkComplexityAny(b *testing.B) {
// m := match.NewAny(nil)
// for i := 0; i < b.N; i++ {
// _ = m.Match(complexityString)
// _, _ = m.Index(complexityString)
// }
//}
//func BenchmarkComplexityContains(b *testing.B) {
// m := match.NewContains()
// for i := 0; i < b.N; i++ {
// _ = m.Match(complexityString)
// _, _ = m.Index(complexityString)
// }
//}
//func BenchmarkComplexityList(b *testing.B) {
// m := match.NewList()
// for i := 0; i < b.N; i++ {
// _ = m.Match(complexityString)
// _, _ = m.Index(complexityString)
// }
//}
//func BenchmarkComplexityMax(b *testing.B) {
// m := match.NewMax()
// for i := 0; i < b.N; i++ {
// _ = m.Match(complexityString)
// _, _ = m.Index(complexityString)
// }
//}
//func BenchmarkComplexityMin(b *testing.B) {
// m := match.NewMin()
// for i := 0; i < b.N; i++ {
// _ = m.Match(complexityString)
// _, _ = m.Index(complexityString)
// }
//}
//func BenchmarkComplexityNothing(b *testing.B) {
// m := match.NewNothing()
// for i := 0; i < b.N; i++ {
// _ = m.Match(complexityString)
// _, _ = m.Index(complexityString)
// }
//}
//func BenchmarkComplexityPrefix(b *testing.B) {
// m := match.NewPrefix()
// for i := 0; i < b.N; i++ {
// _ = m.Match(complexityString)
// _, _ = m.Index(complexityString)
// }
//}
//func BenchmarkComplexityPrefixSuffix(b *testing.B) {
// m := match.NewPrefixSuffix()
// for i := 0; i < b.N; i++ {
// _ = m.Match(complexityString)
// _, _ = m.Index(complexityString)
// }
//}
//func BenchmarkComplexityRange(b *testing.B) {
// m := match.NewRange()
// for i := 0; i < b.N; i++ {
// _ = m.Match(complexityString)
// _, _ = m.Index(complexityString)
// }
//}
//func BenchmarkComplexityRow(b *testing.B) {
// m := match.NewRow()
// for i := 0; i < b.N; i++ {
// _ = m.Match(complexityString)
// _, _ = m.Index(complexityString)
// }
//}
//func BenchmarkComplexitySingle(b *testing.B) {
// m := match.NewSingle(nil)
// for i := 0; i < b.N; i++ {
// _ = m.Match(complexityString)
// _, _ = m.Index(complexityString)
// }
//}
//func BenchmarkComplexitySuffix(b *testing.B) {
// m := match.NewSuffix()
// for i := 0; i < b.N; i++ {
// _ = m.Match(complexityString)
// _, _ = m.Index(complexityString)
// }
//}
//func BenchmarkComplexitySuper(b *testing.B) {
// m := match.NewSuper()
// for i := 0; i < b.N; i++ {
// _ = m.Match(complexityString)
// _, _ = m.Index(complexityString)
// }
//}
//func BenchmarkComplexityText(b *testing.B) {
// m := match.NewText()
// for i := 0; i < b.N; i++ {
// _ = m.Match(complexityString)
// _, _ = m.Index(complexityString)
// }
//}
//func BenchmarkComplexityAnyOf(b *testing.B) {
// m := match.NewAnyOf()
// for i := 0; i < b.N; i++ {
// _ = m.Match(complexityString)
// _, _ = m.Index(complexityString)
// }
//}
//func BenchmarkComplexityBTree(b *testing.B) {
// m := match.NewBTree(match.NewText("abc"), match.NewText("d"), match.NewText("e"))
// for i := 0; i < b.N; i++ {
// _ = m.Match(complexityString)
// _, _ = m.Index(complexityString)
// }
//}
//func BenchmarkComplexityEveryOf(b *testing.B) {
// m := match.NewEveryOf()
// for i := 0; i < b.N; i++ {
// _ = m.Match(complexityString)
// _, _ = m.Index(complexityString)
// }
//}

View File

@ -1,5 +1,10 @@
package glob
import (
"github.com/gobwas/glob/parser"
"github.com/gobwas/glob/syntax"
)
// Glob represents compiled glob pattern.
type Glob interface {
Match(string) bool
@ -32,7 +37,7 @@ type Glob interface {
// comma-separated (without spaces) patterns
//
func Compile(pattern string, separators ...rune) (Glob, error) {
ast, err := parse(newLexer(pattern))
ast, err := syntax.Parse(pattern)
if err != nil {
return nil, err
}
@ -63,7 +68,7 @@ func QuoteMeta(s string) string {
// a byte loop is correct because all meta characters are ASCII
j := 0
for i := 0; i < len(s); i++ {
if special(s[i]) {
if syntax.Special(s[i]) {
b[j] = '\\'
j++
}

View File

@ -2,7 +2,7 @@ package match
import (
"fmt"
"github.com/gobwas/glob/strings"
"github.com/gobwas/glob/util/strings"
)
type Any struct {

View File

@ -2,7 +2,7 @@ package match
import (
"fmt"
"github.com/gobwas/glob/runes"
"github.com/gobwas/glob/util/runes"
"unicode/utf8"
)

View File

@ -2,7 +2,7 @@ package match
import (
"fmt"
"github.com/gobwas/glob/runes"
"github.com/gobwas/glob/util/runes"
"unicode/utf8"
)

View File

@ -1,48 +0,0 @@
package parser
type Node interface {
Children() []Node
Parent() Node
append(Node) Node
}
type node struct {
parent Node
children []Node
}
func (n *node) Children() []Node {
return n.children
}
func (n *node) Parent() Node {
return n.parent
}
func (n *node) append(c Node) Node {
n.children = append(n.children, c)
return c
}
type ListNode struct {
node
Not bool
Chars string
}
type RangeNode struct {
node
Not bool
Lo, Hi rune
}
type TextNode struct {
node
Text string
}
type PatternNode struct{ node }
type AnyNode struct{ node }
type SuperNode struct{ node }
type SingleNode struct{ node }
type AnyOfNode struct{ node }

View File

@ -1,332 +0,0 @@
package parser
import (
"fmt"
"github.com/gobwas/glob/lexer"
"reflect"
"testing"
)
type stubLexer struct {
tokens []lexer.Token
pos int
}
func (s *stubLexer) Next() (ret lexer.Token) {
if s.pos == len(s.tokens) {
return lexer.Token{lexer.EOF, ""}
}
ret = s.tokens[s.pos]
s.pos++
return
}
func TestParseString(t *testing.T) {
for id, test := range []struct {
tokens []lexer.Token
tree Node
}{
{
//pattern: "abc",
tokens: []lexer.Token{
lexer.Token{lexer.Text, "abc"},
lexer.Token{lexer.EOF, ""},
},
tree: &PatternNode{
node: node{
children: []Node{
&TextNode{Text: "abc"},
},
},
},
},
{
//pattern: "a*c",
tokens: []lexer.Token{
lexer.Token{lexer.Text, "a"},
lexer.Token{lexer.Any, "*"},
lexer.Token{lexer.Text, "c"},
lexer.Token{lexer.EOF, ""},
},
tree: &PatternNode{
node: node{
children: []Node{
&TextNode{Text: "a"},
&AnyNode{},
&TextNode{Text: "c"},
},
},
},
},
{
//pattern: "a**c",
tokens: []lexer.Token{
lexer.Token{lexer.Text, "a"},
lexer.Token{lexer.Super, "**"},
lexer.Token{lexer.Text, "c"},
lexer.Token{lexer.EOF, ""},
},
tree: &PatternNode{
node: node{
children: []Node{
&TextNode{Text: "a"},
&SuperNode{},
&TextNode{Text: "c"},
},
},
},
},
{
//pattern: "a?c",
tokens: []lexer.Token{
lexer.Token{lexer.Text, "a"},
lexer.Token{lexer.Single, "?"},
lexer.Token{lexer.Text, "c"},
lexer.Token{lexer.EOF, ""},
},
tree: &PatternNode{
node: node{
children: []Node{
&TextNode{Text: "a"},
&SingleNode{},
&TextNode{Text: "c"},
},
},
},
},
{
//pattern: "[!a-z]",
tokens: []lexer.Token{
lexer.Token{lexer.RangeOpen, "["},
lexer.Token{lexer.Not, "!"},
lexer.Token{lexer.RangeLo, "a"},
lexer.Token{lexer.RangeBetween, "-"},
lexer.Token{lexer.RangeHi, "z"},
lexer.Token{lexer.RangeClose, "]"},
lexer.Token{lexer.EOF, ""},
},
tree: &PatternNode{
node: node{
children: []Node{
&RangeNode{Lo: 'a', Hi: 'z', Not: true},
},
},
},
},
{
//pattern: "[az]",
tokens: []lexer.Token{
lexer.Token{lexer.RangeOpen, "["},
lexer.Token{lexer.Text, "az"},
lexer.Token{lexer.RangeClose, "]"},
lexer.Token{lexer.EOF, ""},
},
tree: &PatternNode{
node: node{
children: []Node{
&ListNode{Chars: "az"},
},
},
},
},
{
//pattern: "{a,z}",
tokens: []lexer.Token{
lexer.Token{lexer.TermsOpen, "{"},
lexer.Token{lexer.Text, "a"},
lexer.Token{lexer.Separator, ","},
lexer.Token{lexer.Text, "z"},
lexer.Token{lexer.TermsClose, "}"},
lexer.Token{lexer.EOF, ""},
},
tree: &PatternNode{
node: node{
children: []Node{
&AnyOfNode{node: node{children: []Node{
&PatternNode{
node: node{children: []Node{
&TextNode{Text: "a"},
}},
},
&PatternNode{
node: node{children: []Node{
&TextNode{Text: "z"},
}},
},
}}},
},
},
},
},
{
//pattern: "/{z,ab}*",
tokens: []lexer.Token{
lexer.Token{lexer.Text, "/"},
lexer.Token{lexer.TermsOpen, "{"},
lexer.Token{lexer.Text, "z"},
lexer.Token{lexer.Separator, ","},
lexer.Token{lexer.Text, "ab"},
lexer.Token{lexer.TermsClose, "}"},
lexer.Token{lexer.Any, "*"},
lexer.Token{lexer.EOF, ""},
},
tree: &PatternNode{
node: node{
children: []Node{
&TextNode{Text: "/"},
&AnyOfNode{node: node{children: []Node{
&PatternNode{
node: node{children: []Node{
&TextNode{Text: "z"},
}},
},
&PatternNode{
node: node{children: []Node{
&TextNode{Text: "ab"},
}},
},
}}},
&AnyNode{},
},
},
},
},
{
//pattern: "{a,{x,y},?,[a-z],[!qwe]}",
tokens: []lexer.Token{
lexer.Token{lexer.TermsOpen, "{"},
lexer.Token{lexer.Text, "a"},
lexer.Token{lexer.Separator, ","},
lexer.Token{lexer.TermsOpen, "{"},
lexer.Token{lexer.Text, "x"},
lexer.Token{lexer.Separator, ","},
lexer.Token{lexer.Text, "y"},
lexer.Token{lexer.TermsClose, "}"},
lexer.Token{lexer.Separator, ","},
lexer.Token{lexer.Single, "?"},
lexer.Token{lexer.Separator, ","},
lexer.Token{lexer.RangeOpen, "["},
lexer.Token{lexer.RangeLo, "a"},
lexer.Token{lexer.RangeBetween, "-"},
lexer.Token{lexer.RangeHi, "z"},
lexer.Token{lexer.RangeClose, "]"},
lexer.Token{lexer.Separator, ","},
lexer.Token{lexer.RangeOpen, "["},
lexer.Token{lexer.Not, "!"},
lexer.Token{lexer.Text, "qwe"},
lexer.Token{lexer.RangeClose, "]"},
lexer.Token{lexer.TermsClose, "}"},
lexer.Token{lexer.EOF, ""},
},
tree: &PatternNode{
node: node{
children: []Node{
&AnyOfNode{node: node{children: []Node{
&PatternNode{
node: node{children: []Node{
&TextNode{Text: "a"},
}},
},
&PatternNode{
node: node{children: []Node{
&AnyOfNode{node: node{children: []Node{
&PatternNode{
node: node{children: []Node{
&TextNode{Text: "x"},
}},
},
&PatternNode{
node: node{children: []Node{
&TextNode{Text: "y"},
}},
},
}}},
}},
},
&PatternNode{
node: node{children: []Node{
&SingleNode{},
}},
},
&PatternNode{
node: node{
children: []Node{
&RangeNode{Lo: 'a', Hi: 'z', Not: false},
},
},
},
&PatternNode{
node: node{
children: []Node{
&ListNode{Chars: "qwe", Not: true},
},
},
},
}}},
},
},
},
},
} {
lexer := &stubLexer{tokens: test.tokens}
result, err := Parse(lexer)
if err != nil {
t.Errorf("[%d] unexpected error: %s", id, err)
}
if !reflect.DeepEqual(test.tree, result) {
t.Errorf("[%d] Parse():\nact:\t%s\nexp:\t%s\n", id, result, test.tree)
}
}
}
const abstractNodeImpl = "nodeImpl"
func nodeEqual(a, b Node) error {
if (a == nil || b == nil) && a != b {
return fmt.Errorf("nodes are not equal: exp %s, act %s", a, b)
}
aValue, bValue := reflect.Indirect(reflect.ValueOf(a)), reflect.Indirect(reflect.ValueOf(b))
aType, bType := aValue.Type(), bValue.Type()
if aType != bType {
return fmt.Errorf("nodes are not equal: exp %s, act %s", aValue.Type(), bValue.Type())
}
for i := 0; i < aType.NumField(); i++ {
var eq bool
f := aType.Field(i).Name
if f == abstractNodeImpl {
continue
}
af, bf := aValue.FieldByName(f), bValue.FieldByName(f)
switch af.Kind() {
case reflect.String:
eq = af.String() == bf.String()
case reflect.Bool:
eq = af.Bool() == bf.Bool()
default:
eq = fmt.Sprint(af) == fmt.Sprint(bf)
}
if !eq {
return fmt.Errorf("nodes<%s> %q fields are not equal: exp %q, act %q", aType, f, af, bf)
}
}
for i, aDesc := range a.Children() {
if len(b.Children())-1 < i {
return fmt.Errorf("node does not have enough children (got %d children, wanted %d-th token)", len(b.Children()), i)
}
bDesc := b.Children()[i]
if err := nodeEqual(aDesc, bDesc); err != nil {
return err
}
}
return nil
}

85
syntax/ast/ast.go Normal file
View File

@ -0,0 +1,85 @@
package ast
type Visitor interface {
Visit(*Node) Visitor
}
func Walk(v Visitor, n *Node) {
if v = v.Visit(n); v == nil {
return
}
for _, c := range n.Children {
Walk(v, c)
}
}
type Node struct {
Parent *Node
Children []*Node
Value interface{}
Kind Kind
}
func NewNode(k Kind, v interface{}, ch ...*Node) *Node {
n := &Node{
Kind: k,
Value: v,
}
for _, c := range ch {
Insert(n, c)
}
return n
}
func (a *Node) Equal(b *Node) bool {
if a.Kind != b.Kind {
return false
}
if a.Value != b.Value {
return false
}
if len(a.Children) != len(b.Children) {
return false
}
for i, c := range a.Children {
if !c.Equal(b.Children[i]) {
return false
}
}
return true
}
func Insert(parent *Node, children ...*Node) {
parent.Children = append(parent.Children, children...)
for _, ch := range children {
ch.Parent = parent
}
}
type List struct {
Not bool
Chars string
}
type Range struct {
Not bool
Lo, Hi rune
}
type Text struct {
Text string
}
type Kind int
const (
KindNothing Kind = iota
KindPattern
KindList
KindRange
KindText
KindAny
KindSuper
KindSingle
KindAnyOf
)

View File

@ -1,9 +1,9 @@
package parser
package ast
import (
"errors"
"fmt"
"github.com/gobwas/glob/lexer"
"github.com/gobwas/glob/syntax/lexer"
"unicode/utf8"
)
@ -11,15 +11,15 @@ type Lexer interface {
Next() lexer.Token
}
type parseFn func(Node, Lexer) (parseFn, Node, error)
type parseFn func(*Node, Lexer) (parseFn, *Node, error)
func Parse(lexer Lexer) (*PatternNode, error) {
func Parse(lexer Lexer) (*Node, error) {
var parser parseFn
root := &PatternNode{}
root := NewNode(KindPattern, nil)
var (
tree Node
tree *Node
err error
)
for parser, tree = parserMain, root; parser != nil; {
@ -32,7 +32,7 @@ func Parse(lexer Lexer) (*PatternNode, error) {
return root, nil
}
func parserMain(tree Node, lex Lexer) (parseFn, Node, error) {
func parserMain(tree *Node, lex Lexer) (parseFn, *Node, error) {
for {
token := lex.Next()
switch token.Type {
@ -43,28 +43,41 @@ func parserMain(tree Node, lex Lexer) (parseFn, Node, error) {
return nil, tree, errors.New(token.Raw)
case lexer.Text:
return parserMain, tree.append(&TextNode{Text: token.Raw}), nil
Insert(tree, NewNode(KindText, &Text{token.Raw}))
return parserMain, tree, nil
case lexer.Any:
return parserMain, tree.append(&AnyNode{}), nil
Insert(tree, NewNode(KindAny, nil))
return parserMain, tree, nil
case lexer.Super:
return parserMain, tree.append(&SuperNode{}), nil
Insert(tree, NewNode(KindSuper, nil))
return parserMain, tree, nil
case lexer.Single:
return parserMain, tree.append(&SingleNode{}), nil
Insert(tree, NewNode(KindSingle, nil))
return parserMain, tree, nil
case lexer.RangeOpen:
return parserRange, tree, nil
case lexer.TermsOpen:
return parserMain, tree.append(&AnyOfNode{}).append(&PatternNode{}), nil
a := NewNode(KindAnyOf, nil)
Insert(tree, a)
p := NewNode(KindPattern, nil)
Insert(a, p)
return parserMain, p, nil
case lexer.Separator:
return parserMain, tree.Parent().append(&PatternNode{}), nil
p := NewNode(KindPattern, nil)
Insert(tree.Parent, p)
return parserMain, p, nil
case lexer.TermsClose:
return parserMain, tree.Parent().Parent(), nil
return parserMain, tree.Parent.Parent, nil
default:
return nil, tree, fmt.Errorf("unexpected token: %s", token)
@ -73,7 +86,7 @@ func parserMain(tree Node, lex Lexer) (parseFn, Node, error) {
return nil, tree, fmt.Errorf("unknown error")
}
func parserRange(tree Node, lex Lexer) (parseFn, Node, error) {
func parserRange(tree *Node, lex Lexer) (parseFn, *Node, error) {
var (
not bool
lo rune
@ -126,16 +139,16 @@ func parserRange(tree Node, lex Lexer) (parseFn, Node, error) {
}
if isRange {
tree = tree.append(&RangeNode{
Insert(tree, NewNode(KindRange, &Range{
Lo: lo,
Hi: hi,
Not: not,
})
}))
} else {
tree = tree.append(&ListNode{
Insert(tree, NewNode(KindList, &List{
Chars: chars,
Not: not,
})
}))
}
return parserMain, tree, nil

256
syntax/ast/parser_test.go Normal file
View File

@ -0,0 +1,256 @@
package ast
import (
"github.com/gobwas/glob/syntax"
"reflect"
"testing"
)
type stubLexer struct {
tokens []syntax.Token
pos int
}
func (s *stubLexer) Next() (ret syntax.Token) {
if s.pos == len(s.tokens) {
return syntax.Token{syntax.EOF, ""}
}
ret = s.tokens[s.pos]
s.pos++
return
}
func TestParseString(t *testing.T) {
for id, test := range []struct {
tokens []syntax.Token
tree Node
}{
{
//pattern: "abc",
tokens: []syntax.Token{
syntax.Token{syntax.Text, "abc"},
syntax.Token{syntax.EOF, ""},
},
tree: NewNode(KindPattern, nil,
NewNode(KindText, &Text{Text: "abc"}),
),
},
{
//pattern: "a*c",
tokens: []syntax.Token{
syntax.Token{syntax.Text, "a"},
syntax.Token{syntax.Any, "*"},
syntax.Token{syntax.Text, "c"},
syntax.Token{syntax.EOF, ""},
},
tree: NewNode(KindPattern, nil,
NewNode(KindText, &Text{Text: "a"}),
NewNode(KindAny, nil),
NewNode(KindText, &Text{Text: "c"}),
),
},
{
//pattern: "a**c",
tokens: []syntax.Token{
syntax.Token{syntax.Text, "a"},
syntax.Token{syntax.Super, "**"},
syntax.Token{syntax.Text, "c"},
syntax.Token{syntax.EOF, ""},
},
tree: NewNode(KindPattern, nil,
NewNode(KindText, &Text{Text: "a"}),
NewNode(KindSuper, nil),
NewNode(KindText, &Text{Text: "c"}),
),
},
{
//pattern: "a?c",
tokens: []syntax.Token{
syntax.Token{syntax.Text, "a"},
syntax.Token{syntax.Single, "?"},
syntax.Token{syntax.Text, "c"},
syntax.Token{syntax.EOF, ""},
},
tree: NewNode(KindPattern, nil,
NewNode(KindText, &Text{Text: "a"}),
NewNode(KindSingle, nil),
NewNode(KindText, &Text{Text: "c"}),
),
},
{
//pattern: "[!a-z]",
tokens: []syntax.Token{
syntax.Token{syntax.RangeOpen, "["},
syntax.Token{syntax.Not, "!"},
syntax.Token{syntax.RangeLo, "a"},
syntax.Token{syntax.RangeBetween, "-"},
syntax.Token{syntax.RangeHi, "z"},
syntax.Token{syntax.RangeClose, "]"},
syntax.Token{syntax.EOF, ""},
},
tree: NewNode(KindPattern, nil,
NewNode(KindRange, &Range{Lo: 'a', Hi: 'z', Not: true}),
),
},
{
//pattern: "[az]",
tokens: []syntax.Token{
syntax.Token{syntax.RangeOpen, "["},
syntax.Token{syntax.Text, "az"},
syntax.Token{syntax.RangeClose, "]"},
syntax.Token{syntax.EOF, ""},
},
tree: NewNode(KindPattern, nil,
NewNode(KindList, &List{Chars: "az"}),
),
},
{
//pattern: "{a,z}",
tokens: []syntax.Token{
syntax.Token{syntax.TermsOpen, "{"},
syntax.Token{syntax.Text, "a"},
syntax.Token{syntax.Separator, ","},
syntax.Token{syntax.Text, "z"},
syntax.Token{syntax.TermsClose, "}"},
syntax.Token{syntax.EOF, ""},
},
tree: NewNode(KindPattern, nil,
NewNode(KindAnyOf, nil,
NewNode(KindPattern, nil,
NewNode(KindText, &Text{Text: "a"}),
),
NewNode(KindPattern, nil,
NewNode(KindText, &Text{Text: "z"}),
),
),
),
},
{
//pattern: "/{z,ab}*",
tokens: []syntax.Token{
syntax.Token{syntax.Text, "/"},
syntax.Token{syntax.TermsOpen, "{"},
syntax.Token{syntax.Text, "z"},
syntax.Token{syntax.Separator, ","},
syntax.Token{syntax.Text, "ab"},
syntax.Token{syntax.TermsClose, "}"},
syntax.Token{syntax.Any, "*"},
syntax.Token{syntax.EOF, ""},
},
tree: NewNode(KindPattern, nil,
NewNode(KindText, &Text{Text: "/"}),
NewNode(KindAnyOf, nil,
NewNode(KindPattern, nil,
NewNode(KindText, &Text{Text: "z"}),
),
NewNode(KindPattern, nil,
NewNode(KindText, &Text{Text: "ab"}),
),
),
NewNode(KindAny, nil),
),
},
{
//pattern: "{a,{x,y},?,[a-z],[!qwe]}",
tokens: []syntax.Token{
syntax.Token{syntax.TermsOpen, "{"},
syntax.Token{syntax.Text, "a"},
syntax.Token{syntax.Separator, ","},
syntax.Token{syntax.TermsOpen, "{"},
syntax.Token{syntax.Text, "x"},
syntax.Token{syntax.Separator, ","},
syntax.Token{syntax.Text, "y"},
syntax.Token{syntax.TermsClose, "}"},
syntax.Token{syntax.Separator, ","},
syntax.Token{syntax.Single, "?"},
syntax.Token{syntax.Separator, ","},
syntax.Token{syntax.RangeOpen, "["},
syntax.Token{syntax.RangeLo, "a"},
syntax.Token{syntax.RangeBetween, "-"},
syntax.Token{syntax.RangeHi, "z"},
syntax.Token{syntax.RangeClose, "]"},
syntax.Token{syntax.Separator, ","},
syntax.Token{syntax.RangeOpen, "["},
syntax.Token{syntax.Not, "!"},
syntax.Token{syntax.Text, "qwe"},
syntax.Token{syntax.RangeClose, "]"},
syntax.Token{syntax.TermsClose, "}"},
syntax.Token{syntax.EOF, ""},
},
tree: NewNode(KindPattern, nil,
NewNode(KindAnyOf, nil,
NewNode(KindPattern, nil,
NewNode(KindText, &Text{Text: "a"}),
),
NewNode(KindPattern, nil,
NewNode(KindAnyOf, nil,
NewNode(KindPattern, nil,
NewNode(KindText, &Text{Text: "x"}),
),
NewNode(KindPattern, nil,
NewNode(KindText, &Text{Text: "y"}),
),
),
),
NewNode(KindPattern, nil,
NewNode(KindSingle, nil),
),
NewNode(KindPattern, nil,
NewNode(KindRange, &Range{Lo: 'a', Hi: 'z', Not: false}),
),
NewNode(KindPattern, nil,
NewNode(KindList, &List{Chars: "qwe", Not: true}),
),
),
),
},
} {
lexer := &stubLexer{tokens: test.tokens}
result, err := Parse(lexer)
if err != nil {
t.Errorf("[%d] unexpected error: %s", id, err)
}
if !reflect.DeepEqual(test.tree, result) {
t.Errorf("[%d] Parse():\nact:\t%s\nexp:\t%s\n", id, result, test.tree)
}
}
}
type kv struct {
kind Kind
value interface{}
}
type visitor struct {
visited []kv
}
func (v *visitor) Visit(n Node) Visitor {
v.visited = append(v.visited, kv{n.Kind(), n.Value()})
return v
}
func TestWalkTree(t *testing.T) {
for i, test := range []struct {
tree *Node
visited []kv
}{
{
tree: NewNode(KindPattern, nil,
NewNode(KindSingle, nil),
),
visited: []kv{
kv{KindPattern, nil},
kv{KindSingle, nil},
},
},
} {
v := &visitor{}
Walk(v, test.tree)
if !reflect.DeepEqual(test.visited, v.visited) {
t.Errorf("[%d] unexpected result of Walk():\nvisited:\t%v\nwant:\t\t%v", i, v.visited, test.visited)
}
}
}

View File

@ -66,7 +66,7 @@ type lexer struct {
hasRune bool
}
func newLexer(source string) *lexer {
func NewLexer(source string) *lexer {
l := &lexer{
data: source,
tokens: tokens(make([]Token, 0, 4)),
@ -74,6 +74,18 @@ func newLexer(source string) *lexer {
return l
}
func (l *lexer) Next() Token {
if l.err != nil {
return Token{Error, l.err.Error()}
}
if !l.tokens.empty() {
return l.tokens.shift()
}
l.fetchItem()
return l.Next()
}
func (l *lexer) peek() (r rune, w int) {
if l.pos == len(l.data) {
return eof, 0
@ -134,18 +146,6 @@ func (l *lexer) termsLeave() {
l.termsLevel--
}
func (l *lexer) nextItem() Token {
if l.err != nil {
return Token{Error, l.err.Error()}
}
if !l.tokens.empty() {
return l.tokens.shift()
}
l.fetchItem()
return l.nextItem()
}
var inTextBreakers = []rune{char_single, char_any, char_range_open, char_terms_open}
var inTermsBreakers = append(inTextBreakers, char_terms_close, char_comma)

View File

@ -178,9 +178,9 @@ func TestLexGood(t *testing.T) {
},
},
} {
lexer := newLexer(test.pattern)
lexer := NewLexer(test.pattern)
for i, exp := range test.items {
act := lexer.nextItem()
act := lexer.Next()
if act.Type != exp.Type {
t.Errorf("#%d %q: wrong %d-th item type: exp: %q; act: %q\n\t(%s vs %s)", id, test.pattern, i, exp.Type, act.Type, exp, act)
}

14
syntax/syntax.go Normal file
View File

@ -0,0 +1,14 @@
package syntax
import (
"github.com/gobwas/glob/syntax/ast"
"github.com/gobwas/glob/syntax/lexer"
)
func Parse(s string) (*ast.Node, error) {
return ast.Parse(lexer.NewLexer(s))
}
func Special(b byte) bool {
return lexer.Special(b)
}