mirror
/
glob
mirror of https://github.com/gobwas/glob.git
1
0
Fork 1
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #16 from gobwas/newlexer_more 

Newlexer more
This commit is contained in:
Sergey Kamardin 2016-05-31 11:55:06 +03:00
parent 0b50f259da 36eb5476d5
commit ce6abff517
22 changed files with 1934 additions and 1954 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

508
compiler.go
View File

@ -1,508 +0,0 @@
package glob
// TODO use constructor with all matchers, and to their structs private
// TODO glue multiple Text nodes (like after QuoteMeta)
import (
"fmt"
"github.com/gobwas/glob/match"
"github.com/gobwas/glob/runes"
"reflect"
)
func optimize(matcher match.Matcher) match.Matcher {
switch m := matcher.(type) {
case match.Any:
if len(m.Separators) == 0 {
return match.NewSuper()
}
case match.AnyOf:
if len(m.Matchers) == 1 {
return m.Matchers[0]
}
return m
case match.List:
if m.Not == false && len(m.List) == 1 {
return match.NewText(string(m.List))
}
return m
case match.BTree:
m.Left = optimize(m.Left)
m.Right = optimize(m.Right)
r, ok := m.Value.(match.Text)
if !ok {
return m
}
leftNil := m.Left == nil
rightNil := m.Right == nil
if leftNil && rightNil {
return match.NewText(r.Str)
}
_, leftSuper := m.Left.(match.Super)
lp, leftPrefix := m.Left.(match.Prefix)
_, rightSuper := m.Right.(match.Super)
rs, rightSuffix := m.Right.(match.Suffix)
if leftSuper && rightSuper {
return match.NewContains(r.Str, false)
}
if leftSuper && rightNil {
return match.NewSuffix(r.Str)
}
if rightSuper && leftNil {
return match.NewPrefix(r.Str)
}
if leftNil && rightSuffix {
return match.NewPrefixSuffix(r.Str, rs.Suffix)
}
if rightNil && leftPrefix {
return match.NewPrefixSuffix(lp.Prefix, r.Str)
}
return m
}
return matcher
}
func glueMatchers(matchers []match.Matcher) match.Matcher {
var (
glued []match.Matcher
winner match.Matcher
)
maxLen := -1
if m := glueAsEvery(matchers); m != nil {
glued = append(glued, m)
return m
}
if m := glueAsRow(matchers); m != nil {
glued = append(glued, m)
return m
}
for _, g := range glued {
if l := g.Len(); l > maxLen {
maxLen = l
winner = g
}
}
return winner
}
func glueAsRow(matchers []match.Matcher) match.Matcher {
if len(matchers) <= 1 {
return nil
}
var (
c []match.Matcher
l int
)
for _, matcher := range matchers {
if ml := matcher.Len(); ml == -1 {
return nil
} else {
c = append(c, matcher)
l += ml
}
}
return match.NewRow(l, c...)
}
func glueAsEvery(matchers []match.Matcher) match.Matcher {
if len(matchers) <= 1 {
return nil
}
var (
hasAny bool
hasSuper bool
hasSingle bool
min int
separator []rune
)
for i, matcher := range matchers {
var sep []rune
switch m := matcher.(type) {
case match.Super:
sep = []rune{}
hasSuper = true
case match.Any:
sep = m.Separators
hasAny = true
case match.Single:
sep = m.Separators
hasSingle = true
min++
case match.List:
if !m.Not {
return nil
}
sep = m.List
hasSingle = true
min++
default:
return nil
}
// initialize
if i == 0 {
separator = sep
}
if runes.Equal(sep, separator) {
continue
}
return nil
}
if hasSuper && !hasAny && !hasSingle {
return match.NewSuper()
}
if hasAny && !hasSuper && !hasSingle {
return match.NewAny(separator)
}
if (hasAny || hasSuper) && min > 0 && len(separator) == 0 {
return match.NewMin(min)
}
every := match.NewEveryOf()
if min > 0 {
every.Add(match.NewMin(min))
if !hasAny && !hasSuper {
every.Add(match.NewMax(min))
}
}
if len(separator) > 0 {
every.Add(match.NewContains(string(separator), true))
}
return every
}
func minimizeMatchers(matchers []match.Matcher) []match.Matcher {
var done match.Matcher
var left, right, count int
for l := 0; l < len(matchers); l++ {
for r := len(matchers); r > l; r-- {
if glued := glueMatchers(matchers[l:r]); glued != nil {
var swap bool
if done == nil {
swap = true
} else {
cl, gl := done.Len(), glued.Len()
swap = cl > -1 && gl > -1 && gl > cl
swap = swap || count < r-l
}
if swap {
done = glued
left = l
right = r
count = r - l
}
}
}
}
if done == nil {
return matchers
}
next := append(append([]match.Matcher{}, matchers[:left]...), done)
if right < len(matchers) {
next = append(next, matchers[right:]...)
}
if len(next) == len(matchers) {
return next
}
return minimizeMatchers(next)
}
func minimizeAnyOf(children []node) node {
var nodes [][]node
var min int
var idx int
for i, desc := range children {
pat, ok := desc.(*nodePattern)
if !ok {
return nil
}
n := pat.children()
ln := len(n)
if len(nodes) == 0 || (ln < min) {
min = ln
idx = i
}
nodes = append(nodes, pat.children())
}
minNodes := nodes[idx]
if idx+1 < len(nodes) {
nodes = append(nodes[:idx], nodes[idx+1:]...)
} else {
nodes = nodes[:idx]
}
var commonLeft []node
var commonLeftCount int
for i, n := range minNodes {
has := true
for _, t := range nodes {
if !reflect.DeepEqual(n, t[i]) {
has = false
break
}
}
if has {
commonLeft = append(commonLeft, n)
commonLeftCount++
} else {
break
}
}
var commonRight []node
var commonRightCount int
for i := min - 1; i > commonLeftCount-1; i-- {
n := minNodes[i]
has := true
for _, t := range nodes {
if !reflect.DeepEqual(n, t[len(t)-(min-i)]) {
has = false
break
}
}
if has {
commonRight = append(commonRight, n)
commonRightCount++
} else {
break
}
}
if commonLeftCount == 0 && commonRightCount == 0 {
return nil
}
nodes = append(nodes, minNodes)
nodes[len(nodes)-1], nodes[idx] = nodes[idx], nodes[len(nodes)-1]
var result []node
if commonLeftCount > 0 {
result = append(result, &nodePattern{nodeImpl: nodeImpl{desc: commonLeft}})
}
var anyOf []node
for _, n := range nodes {
if commonLeftCount+commonRightCount == len(n) {
anyOf = append(anyOf, nil)
} else {
anyOf = append(anyOf, &nodePattern{nodeImpl: nodeImpl{desc: n[commonLeftCount : len(n)-commonRightCount]}})
}
}
anyOf = uniqueNodes(anyOf)
if len(anyOf) == 1 {
if anyOf[0] != nil {
result = append(result, &nodePattern{nodeImpl: nodeImpl{desc: anyOf}})
}
} else {
result = append(result, &nodeAnyOf{nodeImpl: nodeImpl{desc: anyOf}})
}
if commonRightCount > 0 {
result = append(result, &nodePattern{nodeImpl: nodeImpl{desc: commonRight}})
}
return &nodePattern{nodeImpl: nodeImpl{desc: result}}
}
func uniqueNodes(nodes []node) (result []node) {
head:
for _, n := range nodes {
for _, e := range result {
if reflect.DeepEqual(e, n) {
continue head
}
}
result = append(result, n)
}
return
}
func compileMatchers(matchers []match.Matcher) (match.Matcher, error) {
if len(matchers) == 0 {
return nil, fmt.Errorf("compile error: need at least one matcher")
}
if len(matchers) == 1 {
return matchers[0], nil
}
if m := glueMatchers(matchers); m != nil {
return m, nil
}
idx := -1
maxLen := -1
var val match.Matcher
for i, matcher := range matchers {
if l := matcher.Len(); l != -1 && l >= maxLen {
maxLen = l
idx = i
val = matcher
}
}
if val == nil { // not found matcher with static length
r, err := compileMatchers(matchers[1:])
if err != nil {
return nil, err
}
return match.NewBTree(matchers[0], nil, r), nil
}
left := matchers[:idx]
var right []match.Matcher
if len(matchers) > idx+1 {
right = matchers[idx+1:]
}
var l, r match.Matcher
var err error
if len(left) > 0 {
l, err = compileMatchers(left)
if err != nil {
return nil, err
}
}
if len(right) > 0 {
r, err = compileMatchers(right)
if err != nil {
return nil, err
}
}
return match.NewBTree(val, l, r), nil
}
func do(leaf node, s []rune) (m match.Matcher, err error) {
switch n := leaf.(type) {
case *nodeAnyOf:
// todo this could be faster on pattern_alternatives_combine_lite
if n := minimizeAnyOf(n.children()); n != nil {
return do(n, s)
}
var matchers []match.Matcher
for _, desc := range n.children() {
if desc == nil {
matchers = append(matchers, match.NewNothing())
continue
}
m, err := do(desc, s)
if err != nil {
return nil, err
}
matchers = append(matchers, optimize(m))
}
return match.NewAnyOf(matchers...), nil
case *nodePattern:
nodes := leaf.children()
if len(nodes) == 0 {
return match.NewNothing(), nil
}
var matchers []match.Matcher
for _, desc := range nodes {
m, err := do(desc, s)
if err != nil {
return nil, err
}
matchers = append(matchers, optimize(m))
}
m, err = compileMatchers(minimizeMatchers(matchers))
if err != nil {
return nil, err
}
case *nodeList:
m = match.NewList([]rune(n.chars), n.not)
case *nodeRange:
m = match.NewRange(n.lo, n.hi, n.not)
case *nodeAny:
m = match.NewAny(s)
case *nodeSuper:
m = match.NewSuper()
case *nodeSingle:
m = match.NewSingle(s)
case *nodeText:
m = match.NewText(n.text)
default:
return nil, fmt.Errorf("could not compile tree: unknown node type")
}
return optimize(m), nil
}
func compile(ast *nodePattern, s []rune) (Glob, error) {
g, err := do(ast, s)
if err != nil {
return nil, err
}
return g, nil
}

518
compiler/compiler.go Normal file
View File

@ -0,0 +1,518 @@
package compiler
// TODO use constructor with all matchers, and to their structs private
// TODO glue multiple Text nodes (like after QuoteMeta)
import (
"fmt"
"github.com/gobwas/glob/match"
"github.com/gobwas/glob/syntax/ast"
"github.com/gobwas/glob/util/runes"
"reflect"
)
func optimizeMatcher(matcher match.Matcher) match.Matcher {
switch m := matcher.(type) {
case match.Any:
if len(m.Separators) == 0 {
return match.NewSuper()
}
case match.AnyOf:
if len(m.Matchers) == 1 {
return m.Matchers[0]
}
return m
case match.List:
if m.Not == false && len(m.List) == 1 {
return match.NewText(string(m.List))
}
return m
case match.BTree:
m.Left = optimizeMatcher(m.Left)
m.Right = optimizeMatcher(m.Right)
r, ok := m.Value.(match.Text)
if !ok {
return m
}
leftNil := m.Left == nil
rightNil := m.Right == nil
if leftNil && rightNil {
return match.NewText(r.Str)
}
_, leftSuper := m.Left.(match.Super)
lp, leftPrefix := m.Left.(match.Prefix)
_, rightSuper := m.Right.(match.Super)
rs, rightSuffix := m.Right.(match.Suffix)
if leftSuper && rightSuper {
return match.NewContains(r.Str, false)
}
if leftSuper && rightNil {
return match.NewSuffix(r.Str)
}
if rightSuper && leftNil {
return match.NewPrefix(r.Str)
}
if leftNil && rightSuffix {
return match.NewPrefixSuffix(r.Str, rs.Suffix)
}
if rightNil && leftPrefix {
return match.NewPrefixSuffix(lp.Prefix, r.Str)
}
return m
}
return matcher
}
func compileMatchers(matchers []match.Matcher) (match.Matcher, error) {
if len(matchers) == 0 {
return nil, fmt.Errorf("compile error: need at least one matcher")
}
if len(matchers) == 1 {
return matchers[0], nil
}
if m := glueMatchers(matchers); m != nil {
return m, nil
}
idx := -1
maxLen := -1
var val match.Matcher
for i, matcher := range matchers {
if l := matcher.Len(); l != -1 && l >= maxLen {
maxLen = l
idx = i
val = matcher
}
}
if val == nil { // not found matcher with static length
r, err := compileMatchers(matchers[1:])
if err != nil {
return nil, err
}
return match.NewBTree(matchers[0], nil, r), nil
}
left := matchers[:idx]
var right []match.Matcher
if len(matchers) > idx+1 {
right = matchers[idx+1:]
}
var l, r match.Matcher
var err error
if len(left) > 0 {
l, err = compileMatchers(left)
if err != nil {
return nil, err
}
}
if len(right) > 0 {
r, err = compileMatchers(right)
if err != nil {
return nil, err
}
}
return match.NewBTree(val, l, r), nil
}
func glueMatchers(matchers []match.Matcher) match.Matcher {
if m := glueMatchersAsEvery(matchers); m != nil {
return m
}
if m := glueMatchersAsRow(matchers); m != nil {
return m
}
return nil
}
func glueMatchersAsRow(matchers []match.Matcher) match.Matcher {
if len(matchers) <= 1 {
return nil
}
var (
c []match.Matcher
l int
)
for _, matcher := range matchers {
if ml := matcher.Len(); ml == -1 {
return nil
} else {
c = append(c, matcher)
l += ml
}
}
return match.NewRow(l, c...)
}
func glueMatchersAsEvery(matchers []match.Matcher) match.Matcher {
if len(matchers) <= 1 {
return nil
}
var (
hasAny bool
hasSuper bool
hasSingle bool
min int
separator []rune
)
for i, matcher := range matchers {
var sep []rune
switch m := matcher.(type) {
case match.Super:
sep = []rune{}
hasSuper = true
case match.Any:
sep = m.Separators
hasAny = true
case match.Single:
sep = m.Separators
hasSingle = true
min++
case match.List:
if !m.Not {
return nil
}
sep = m.List
hasSingle = true
min++
default:
return nil
}
// initialize
if i == 0 {
separator = sep
}
if runes.Equal(sep, separator) {
continue
}
return nil
}
if hasSuper && !hasAny && !hasSingle {
return match.NewSuper()
}
if hasAny && !hasSuper && !hasSingle {
return match.NewAny(separator)
}
if (hasAny || hasSuper) && min > 0 && len(separator) == 0 {
return match.NewMin(min)
}
every := match.NewEveryOf()
if min > 0 {
every.Add(match.NewMin(min))
if !hasAny && !hasSuper {
every.Add(match.NewMax(min))
}
}
if len(separator) > 0 {
every.Add(match.NewContains(string(separator), true))
}
return every
}
func minimizeMatchers(matchers []match.Matcher) []match.Matcher {
var done match.Matcher
var left, right, count int
for l := 0; l < len(matchers); l++ {
for r := len(matchers); r > l; r-- {
if glued := glueMatchers(matchers[l:r]); glued != nil {
var swap bool
if done == nil {
swap = true
} else {
cl, gl := done.Len(), glued.Len()
swap = cl > -1 && gl > -1 && gl > cl
swap = swap || count < r-l
}
if swap {
done = glued
left = l
right = r
count = r - l
}
}
}
}
if done == nil {
return matchers
}
next := append(append([]match.Matcher{}, matchers[:left]...), done)
if right < len(matchers) {
next = append(next, matchers[right:]...)
}
if len(next) == len(matchers) {
return next
}
return minimizeMatchers(next)
}
// minimizeAnyOf tries to apply some heuristics to minimize number of nodes in given tree
func minimizeTree(tree *ast.Node) *ast.Node {
switch tree.Kind {
case ast.KindAnyOf:
return minimizeTreeAnyOf(tree)
default:
return nil
}
}
// minimizeAnyOf tries to find common children of given node of AnyOf pattern
// it searches for common children from left and from right
// if any common children are found – then it returns new optimized ast tree
// else it returns nil
func minimizeTreeAnyOf(tree *ast.Node) *ast.Node {
if !areOfSameKind(tree.Children, ast.KindPattern) {
return nil
}
commonLeft, commonRight := commonChildren(tree.Children)
commonLeftCount, commonRightCount := len(commonLeft), len(commonRight)
if commonLeftCount == 0 && commonRightCount == 0 { // there are no common parts
return nil
}
var result []*ast.Node
if commonLeftCount > 0 {
result = append(result, ast.NewNode(ast.KindPattern, nil, commonLeft...))
}
var anyOf []*ast.Node
for _, child := range tree.Children {
reuse := child.Children[commonLeftCount : len(child.Children)-commonRightCount]
var node *ast.Node
if len(reuse) == 0 {
// this pattern is completely reduced by commonLeft and commonRight patterns
// so it become nothing
node = ast.NewNode(ast.KindNothing, nil)
} else {
node = ast.NewNode(ast.KindPattern, nil, reuse...)
}
anyOf = appendIfUnique(anyOf, node)
}
switch {
case len(anyOf) == 1 && anyOf[0].Kind != ast.KindNothing:
result = append(result, anyOf[0])
case len(anyOf) > 1:
result = append(result, ast.NewNode(ast.KindAnyOf, nil, anyOf...))
}
if commonRightCount > 0 {
result = append(result, ast.NewNode(ast.KindPattern, nil, commonRight...))
}
return ast.NewNode(ast.KindPattern, nil, result...)
}
func commonChildren(nodes []*ast.Node) (commonLeft, commonRight []*ast.Node) {
if len(nodes) <= 1 {
return
}
// find node that has least number of children
idx := leastChildren(nodes)
if idx == -1 {
return
}
tree := nodes[idx]
treeLength := len(tree.Children)
// allocate max able size for rightCommon slice
// to get ability insert elements in reverse order (from end to start)
// without sorting
commonRight = make([]*ast.Node, treeLength)
lastRight := treeLength // will use this to get results as commonRight[lastRight:]
var (
breakLeft bool
breakRight bool
commonTotal int
)
for i, j := 0, treeLength-1; commonTotal < treeLength && j >= 0 && !(breakLeft && breakLeft); i, j = i+1, j-1 {
treeLeft := tree.Children[i]
treeRight := tree.Children[j]
for k := 0; k < len(nodes) && !(breakLeft && breakLeft); k++ {
// skip least children node
if k == idx {
continue
}
restLeft := nodes[k].Children[i]
restRight := nodes[k].Children[j+len(nodes[k].Children)-treeLength]
breakLeft = breakLeft || !treeLeft.Equal(restLeft)
// disable searching for right common parts, if left part is already overlapping
breakRight = breakRight || (!breakLeft && j <= i)
breakRight = breakRight || !treeRight.Equal(restRight)
}
if !breakLeft {
commonTotal++
commonLeft = append(commonLeft, treeLeft)
}
if !breakRight {
commonTotal++
lastRight = j
commonRight[j] = treeRight
}
}
commonRight = commonRight[lastRight:]
return
}
func appendIfUnique(target []*ast.Node, val *ast.Node) []*ast.Node {
for _, n := range target {
if reflect.DeepEqual(n, val) {
return target
}
}
return append(target, val)
}
func areOfSameKind(nodes []*ast.Node, kind ast.Kind) bool {
for _, n := range nodes {
if n.Kind != kind {
return false
}
}
return true
}
func leastChildren(nodes []*ast.Node) int {
min := -1
idx := -1
for i, n := range nodes {
if idx == -1 || (len(n.Children) < min) {
min = len(n.Children)
idx = i
}
}
return idx
}
func compileTreeChildren(tree *ast.Node, sep []rune) ([]match.Matcher, error) {
var matchers []match.Matcher
for _, desc := range tree.Children {
m, err := compile(desc, sep)
if err != nil {
return nil, err
}
matchers = append(matchers, optimizeMatcher(m))
}
return matchers, nil
}
func compile(tree *ast.Node, sep []rune) (m match.Matcher, err error) {
switch tree.Kind {
case ast.KindAnyOf:
// todo this could be faster on pattern_alternatives_combine_lite (see glob_test.go)
if n := minimizeTree(tree); n != nil {
return compile(n, sep)
}
matchers, err := compileTreeChildren(tree, sep)
if err != nil {
return nil, err
}
return match.NewAnyOf(matchers...), nil
case ast.KindPattern:
if len(tree.Children) == 0 {
return match.NewNothing(), nil
}
matchers, err := compileTreeChildren(tree, sep)
if err != nil {
return nil, err
}
m, err = compileMatchers(minimizeMatchers(matchers))
if err != nil {
return nil, err
}
case ast.KindAny:
m = match.NewAny(sep)
case ast.KindSuper:
m = match.NewSuper()
case ast.KindSingle:
m = match.NewSingle(sep)
case ast.KindNothing:
m = match.NewNothing()
case ast.KindList:
l := tree.Value.(ast.List)
m = match.NewList([]rune(l.Chars), l.Not)
case ast.KindRange:
r := tree.Value.(ast.Range)
m = match.NewRange(r.Lo, r.Hi, r.Not)
case ast.KindText:
t := tree.Value.(ast.Text)
m = match.NewText(t.Text)
default:
return nil, fmt.Errorf("could not compile tree: unknown node type")
}
return optimizeMatcher(m), nil
}
func Compile(tree *ast.Node, sep []rune) (match.Matcher, error) {
m, err := compile(tree, sep)
if err != nil {
return nil, err
}
return m, nil
}

624
compiler/compiler_test.go Normal file
View File

@ -0,0 +1,624 @@
package compiler
import (
"github.com/gobwas/glob/match"
"github.com/gobwas/glob/match/debug"
"github.com/gobwas/glob/syntax/ast"
"reflect"
"testing"
)
var separators = []rune{'.'}
func TestCommonChildren(t *testing.T) {
for i, test := range []struct {
nodes []*ast.Node
left []*ast.Node
right []*ast.Node
}{
{
nodes: []*ast.Node{
ast.NewNode(ast.KindNothing, nil,
ast.NewNode(ast.KindText, ast.Text{"a"}),
ast.NewNode(ast.KindText, ast.Text{"z"}),
ast.NewNode(ast.KindText, ast.Text{"c"}),
),
},
},
{
nodes: []*ast.Node{
ast.NewNode(ast.KindNothing, nil,
ast.NewNode(ast.KindText, ast.Text{"a"}),
ast.NewNode(ast.KindText, ast.Text{"z"}),
ast.NewNode(ast.KindText, ast.Text{"c"}),
),
ast.NewNode(ast.KindNothing, nil,
ast.NewNode(ast.KindText, ast.Text{"a"}),
ast.NewNode(ast.KindText, ast.Text{"b"}),
ast.NewNode(ast.KindText, ast.Text{"c"}),
),
},
left: []*ast.Node{
ast.NewNode(ast.KindText, ast.Text{"a"}),
},
right: []*ast.Node{
ast.NewNode(ast.KindText, ast.Text{"c"}),
},
},
{
nodes: []*ast.Node{
ast.NewNode(ast.KindNothing, nil,
ast.NewNode(ast.KindText, ast.Text{"a"}),
ast.NewNode(ast.KindText, ast.Text{"b"}),
ast.NewNode(ast.KindText, ast.Text{"c"}),
ast.NewNode(ast.KindText, ast.Text{"d"}),
),
ast.NewNode(ast.KindNothing, nil,
ast.NewNode(ast.KindText, ast.Text{"a"}),
ast.NewNode(ast.KindText, ast.Text{"b"}),
ast.NewNode(ast.KindText, ast.Text{"c"}),
ast.NewNode(ast.KindText, ast.Text{"c"}),
ast.NewNode(ast.KindText, ast.Text{"d"}),
),
},
left: []*ast.Node{
ast.NewNode(ast.KindText, ast.Text{"a"}),
ast.NewNode(ast.KindText, ast.Text{"b"}),
},
right: []*ast.Node{
ast.NewNode(ast.KindText, ast.Text{"c"}),
ast.NewNode(ast.KindText, ast.Text{"d"}),
},
},
{
nodes: []*ast.Node{
ast.NewNode(ast.KindNothing, nil,
ast.NewNode(ast.KindText, ast.Text{"a"}),
ast.NewNode(ast.KindText, ast.Text{"b"}),
ast.NewNode(ast.KindText, ast.Text{"c"}),
),
ast.NewNode(ast.KindNothing, nil,
ast.NewNode(ast.KindText, ast.Text{"a"}),
ast.NewNode(ast.KindText, ast.Text{"b"}),
ast.NewNode(ast.KindText, ast.Text{"b"}),
ast.NewNode(ast.KindText, ast.Text{"c"}),
),
},
left: []*ast.Node{
ast.NewNode(ast.KindText, ast.Text{"a"}),
ast.NewNode(ast.KindText, ast.Text{"b"}),
},
right: []*ast.Node{
ast.NewNode(ast.KindText, ast.Text{"c"}),
},
},
{
nodes: []*ast.Node{
ast.NewNode(ast.KindNothing, nil,
ast.NewNode(ast.KindText, ast.Text{"a"}),
ast.NewNode(ast.KindText, ast.Text{"d"}),
),
ast.NewNode(ast.KindNothing, nil,
ast.NewNode(ast.KindText, ast.Text{"a"}),
ast.NewNode(ast.KindText, ast.Text{"d"}),
),
ast.NewNode(ast.KindNothing, nil,
ast.NewNode(ast.KindText, ast.Text{"a"}),
ast.NewNode(ast.KindText, ast.Text{"e"}),
),
},
left: []*ast.Node{
ast.NewNode(ast.KindText, ast.Text{"a"}),
},
right: []*ast.Node{},
},
} {
left, right := commonChildren(test.nodes)
if !nodesEqual(left, test.left) {
t.Errorf("[%d] left, right := commonChildren(); left = %v; want %v", i, left, test.left)
}
if !nodesEqual(right, test.right) {
t.Errorf("[%d] left, right := commonChildren(); right = %v; want %v", i, right, test.right)
}
}
}
func nodesEqual(a, b []*ast.Node) bool {
if len(a) != len(b) {
return false
}
for i, av := range a {
if !av.Equal(b[i]) {
return false
}
}
return true
}
func TestGlueMatchers(t *testing.T) {
for id, test := range []struct {
in []match.Matcher
exp match.Matcher
}{
{
[]match.Matcher{
match.NewSuper(),
match.NewSingle(nil),
},
match.NewMin(1),
},
{
[]match.Matcher{
match.NewAny(separators),
match.NewSingle(separators),
},
match.EveryOf{match.Matchers{
match.NewMin(1),
match.NewContains(string(separators), true),
}},
},
{
[]match.Matcher{
match.NewSingle(nil),
match.NewSingle(nil),
match.NewSingle(nil),
},
match.EveryOf{match.Matchers{
match.NewMin(3),
match.NewMax(3),
}},
},
{
[]match.Matcher{
match.NewList([]rune{'a'}, true),
match.NewAny([]rune{'a'}),
},
match.EveryOf{match.Matchers{
match.NewMin(1),
match.NewContains("a", true),
}},
},
} {
act, err := compileMatchers(test.in)
if err != nil {
t.Errorf("#%d convert matchers error: %s", id, err)
continue
}
if !reflect.DeepEqual(act, test.exp) {
t.Errorf("#%d unexpected convert matchers result:\nact: %#v;\nexp: %#v", id, act, test.exp)
continue
}
}
}
func TestCompileMatchers(t *testing.T) {
for id, test := range []struct {
in []match.Matcher
exp match.Matcher
}{
{
[]match.Matcher{
match.NewSuper(),
match.NewSingle(separators),
match.NewText("c"),
},
match.NewBTree(
match.NewText("c"),
match.NewBTree(
match.NewSingle(separators),
match.NewSuper(),
nil,
),
nil,
),
},
{
[]match.Matcher{
match.NewAny(nil),
match.NewText("c"),
match.NewAny(nil),
},
match.NewBTree(
match.NewText("c"),
match.NewAny(nil),
match.NewAny(nil),
),
},
{
[]match.Matcher{
match.NewRange('a', 'c', true),
match.NewList([]rune{'z', 't', 'e'}, false),
match.NewText("c"),
match.NewSingle(nil),
},
match.NewRow(
4,
match.Matchers{
match.NewRange('a', 'c', true),
match.NewList([]rune{'z', 't', 'e'}, false),
match.NewText("c"),
match.NewSingle(nil),
}...,
),
},
} {
act, err := compileMatchers(test.in)
if err != nil {
t.Errorf("#%d convert matchers error: %s", id, err)
continue
}
if !reflect.DeepEqual(act, test.exp) {
t.Errorf("#%d unexpected convert matchers result:\nact: %#v\nexp: %#v", id, act, test.exp)
continue
}
}
}
func TestConvertMatchers(t *testing.T) {
for id, test := range []struct {
in, exp []match.Matcher
}{
{
[]match.Matcher{
match.NewRange('a', 'c', true),
match.NewList([]rune{'z', 't', 'e'}, false),
match.NewText("c"),
match.NewSingle(nil),
match.NewAny(nil),
},
[]match.Matcher{
match.NewRow(
4,
[]match.Matcher{
match.NewRange('a', 'c', true),
match.NewList([]rune{'z', 't', 'e'}, false),
match.NewText("c"),
match.NewSingle(nil),
}...,
),
match.NewAny(nil),
},
},
{
[]match.Matcher{
match.NewRange('a', 'c', true),
match.NewList([]rune{'z', 't', 'e'}, false),
match.NewText("c"),
match.NewSingle(nil),
match.NewAny(nil),
match.NewSingle(nil),
match.NewSingle(nil),
match.NewAny(nil),
},
[]match.Matcher{
match.NewRow(
3,
match.Matchers{
match.NewRange('a', 'c', true),
match.NewList([]rune{'z', 't', 'e'}, false),
match.NewText("c"),
}...,
),
match.NewMin(3),
},
},
} {
act := minimizeMatchers(test.in)
if !reflect.DeepEqual(act, test.exp) {
t.Errorf("#%d unexpected convert matchers 2 result:\nact: %#v\nexp: %#v", id, act, test.exp)
continue
}
}
}
func TestCompiler(t *testing.T) {
for id, test := range []struct {
ast *ast.Node
result match.Matcher
sep []rune
}{
{
ast: ast.NewNode(ast.KindPattern, nil,
ast.NewNode(ast.KindText, ast.Text{"abc"}),
),
result: match.NewText("abc"),
},
{
ast: ast.NewNode(ast.KindPattern, nil,
ast.NewNode(ast.KindAny, nil),
),
sep: separators,
result: match.NewAny(separators),
},
{
ast: ast.NewNode(ast.KindPattern, nil,
ast.NewNode(ast.KindAny, nil),
),
result: match.NewSuper(),
},
{
ast: ast.NewNode(ast.KindPattern, nil,
ast.NewNode(ast.KindSuper, nil),
),
result: match.NewSuper(),
},
{
ast: ast.NewNode(ast.KindPattern, nil,
ast.NewNode(ast.KindSingle, nil),
),
sep: separators,
result: match.NewSingle(separators),
},
{
ast: ast.NewNode(ast.KindPattern, nil,
ast.NewNode(ast.KindRange, ast.Range{
Lo: 'a',
Hi: 'z',
Not: true,
}),
),
result: match.NewRange('a', 'z', true),
},
{
ast: ast.NewNode(ast.KindPattern, nil,
ast.NewNode(ast.KindList, ast.List{
Chars: "abc",
Not: true,
}),
),
result: match.NewList([]rune{'a', 'b', 'c'}, true),
},
{
ast: ast.NewNode(ast.KindPattern, nil,
ast.NewNode(ast.KindAny, nil),
ast.NewNode(ast.KindSingle, nil),
ast.NewNode(ast.KindSingle, nil),
ast.NewNode(ast.KindSingle, nil),
),
sep: separators,
result: match.EveryOf{Matchers: match.Matchers{
match.NewMin(3),
match.NewContains(string(separators), true),
}},
},
{
ast: ast.NewNode(ast.KindPattern, nil,
ast.NewNode(ast.KindAny, nil),
ast.NewNode(ast.KindSingle, nil),
ast.NewNode(ast.KindSingle, nil),
ast.NewNode(ast.KindSingle, nil),
),
result: match.NewMin(3),
},
{
ast: ast.NewNode(ast.KindPattern, nil,
ast.NewNode(ast.KindAny, nil),
ast.NewNode(ast.KindText, ast.Text{"abc"}),
ast.NewNode(ast.KindSingle, nil),
),
sep: separators,
result: match.NewBTree(
match.NewRow(
4,
match.Matchers{
match.NewText("abc"),
match.NewSingle(separators),
}...,
),
match.NewAny(separators),
nil,
),
},
{
ast: ast.NewNode(ast.KindPattern, nil,
ast.NewNode(ast.KindText, ast.Text{"/"}),
ast.NewNode(ast.KindAnyOf, nil,
ast.NewNode(ast.KindText, ast.Text{"z"}),
ast.NewNode(ast.KindText, ast.Text{"ab"}),
),
ast.NewNode(ast.KindSuper, nil),
),
sep: separators,
result: match.NewBTree(
match.NewText("/"),
nil,
match.NewBTree(
match.NewAnyOf(match.NewText("z"), match.NewText("ab")),
nil,
match.NewSuper(),
),
),
},
{
ast: ast.NewNode(ast.KindPattern, nil,
ast.NewNode(ast.KindSuper, nil),
ast.NewNode(ast.KindSingle, nil),
ast.NewNode(ast.KindText, ast.Text{"abc"}),
ast.NewNode(ast.KindSingle, nil),
),
sep: separators,
result: match.NewBTree(
match.NewRow(
5,
match.Matchers{
match.NewSingle(separators),
match.NewText("abc"),
match.NewSingle(separators),
}...,
),
match.NewSuper(),
nil,
),
},
{
ast: ast.NewNode(ast.KindPattern, nil,
ast.NewNode(ast.KindAny, nil),
ast.NewNode(ast.KindText, ast.Text{"abc"}),
),
result: match.NewSuffix("abc"),
},
{
ast: ast.NewNode(ast.KindPattern, nil,
ast.NewNode(ast.KindText, ast.Text{"abc"}),
ast.NewNode(ast.KindAny, nil),
),
result: match.NewPrefix("abc"),
},
{
ast: ast.NewNode(ast.KindPattern, nil,
ast.NewNode(ast.KindText, ast.Text{"abc"}),
ast.NewNode(ast.KindAny, nil),
ast.NewNode(ast.KindText, ast.Text{"def"}),
),
result: match.NewPrefixSuffix("abc", "def"),
},
{
ast: ast.NewNode(ast.KindPattern, nil,
ast.NewNode(ast.KindAny, nil),
ast.NewNode(ast.KindAny, nil),
ast.NewNode(ast.KindAny, nil),
ast.NewNode(ast.KindText, ast.Text{"abc"}),
ast.NewNode(ast.KindAny, nil),
ast.NewNode(ast.KindAny, nil),
),
result: match.NewContains("abc", false),
},
{
ast: ast.NewNode(ast.KindPattern, nil,
ast.NewNode(ast.KindAny, nil),
ast.NewNode(ast.KindAny, nil),
ast.NewNode(ast.KindAny, nil),
ast.NewNode(ast.KindText, ast.Text{"abc"}),
ast.NewNode(ast.KindAny, nil),
ast.NewNode(ast.KindAny, nil),
),
sep: separators,
result: match.NewBTree(
match.NewText("abc"),
match.NewAny(separators),
match.NewAny(separators),
),
},
{
ast: ast.NewNode(ast.KindPattern, nil,
ast.NewNode(ast.KindSuper, nil),
ast.NewNode(ast.KindSingle, nil),
ast.NewNode(ast.KindText, ast.Text{"abc"}),
ast.NewNode(ast.KindSuper, nil),
ast.NewNode(ast.KindSingle, nil),
),
result: match.NewBTree(
match.NewText("abc"),
match.NewMin(1),
match.NewMin(1),
),
},
{
ast: ast.NewNode(ast.KindPattern, nil,
ast.NewNode(ast.KindText, ast.Text{"abc"}),
),
result: match.NewText("abc"),
},
{
ast: ast.NewNode(ast.KindPattern, nil,
ast.NewNode(ast.KindAnyOf, nil,
ast.NewNode(ast.KindPattern, nil,
ast.NewNode(ast.KindAnyOf, nil,
ast.NewNode(ast.KindPattern, nil,
ast.NewNode(ast.KindText, ast.Text{"abc"}),
),
),
),
),
),
result: match.NewText("abc"),
},
{
ast: ast.NewNode(ast.KindPattern, nil,
ast.NewNode(ast.KindAnyOf, nil,
ast.NewNode(ast.KindPattern, nil,
ast.NewNode(ast.KindText, ast.Text{"abc"}),
ast.NewNode(ast.KindSingle, nil),
),
ast.NewNode(ast.KindPattern, nil,
ast.NewNode(ast.KindText, ast.Text{"abc"}),
ast.NewNode(ast.KindList, ast.List{Chars: "def"}),
),
ast.NewNode(ast.KindPattern, nil,
ast.NewNode(ast.KindText, ast.Text{"abc"}),
),
ast.NewNode(ast.KindPattern, nil,
ast.NewNode(ast.KindText, ast.Text{"abc"}),
),
),
),
result: match.NewBTree(
match.NewText("abc"),
nil,
match.AnyOf{Matchers: match.Matchers{
match.NewSingle(nil),
match.NewList([]rune{'d', 'e', 'f'}, false),
match.NewNothing(),
}},
),
},
{
ast: ast.NewNode(ast.KindPattern, nil,
ast.NewNode(ast.KindRange, ast.Range{Lo: 'a', Hi: 'z'}),
ast.NewNode(ast.KindRange, ast.Range{Lo: 'a', Hi: 'x', Not: true}),
ast.NewNode(ast.KindAny, nil),
),
result: match.NewBTree(
match.NewRow(
2,
match.Matchers{
match.NewRange('a', 'z', false),
match.NewRange('a', 'x', true),
}...,
),
nil,
match.NewSuper(),
),
},
{
ast: ast.NewNode(ast.KindPattern, nil,
ast.NewNode(ast.KindAnyOf, nil,
ast.NewNode(ast.KindPattern, nil,
ast.NewNode(ast.KindText, ast.Text{"abc"}),
ast.NewNode(ast.KindList, ast.List{Chars: "abc"}),
ast.NewNode(ast.KindText, ast.Text{"ghi"}),
),
ast.NewNode(ast.KindPattern, nil,
ast.NewNode(ast.KindText, ast.Text{"abc"}),
ast.NewNode(ast.KindList, ast.List{Chars: "def"}),
ast.NewNode(ast.KindText, ast.Text{"ghi"}),
),
),
),
result: match.NewRow(
7,
match.Matchers{
match.NewText("abc"),
match.AnyOf{Matchers: match.Matchers{
match.NewList([]rune{'a', 'b', 'c'}, false),
match.NewList([]rune{'d', 'e', 'f'}, false),
}},
match.NewText("ghi"),
}...,
),
},
} {
m, err := Compile(test.ast, test.sep)
if err != nil {
t.Errorf("compilation error: %s", err)
continue
}
if !reflect.DeepEqual(m, test.result) {
t.Errorf("[%d] Compile():\nexp: %#v\nact: %#v\n\ngraphviz:\nexp:\n%s\nact:\n%s\n", id, test.result, m, debug.Graphviz("", test.result.(match.Matcher)), debug.Graphviz("", m.(match.Matcher)))
continue
}
}
}

548
compiler_test.go
View File

@ -1,548 +0,0 @@
package glob
import (
"github.com/gobwas/glob/match"
"github.com/gobwas/glob/match/debug"
"reflect"
"testing"
)
var separators = []rune{'.'}
func TestGlueMatchers(t *testing.T) {
for id, test := range []struct {
in []match.Matcher
exp match.Matcher
}{
{
[]match.Matcher{
match.NewSuper(),
match.NewSingle(nil),
},
match.NewMin(1),
},
{
[]match.Matcher{
match.NewAny(separators),
match.NewSingle(separators),
},
match.EveryOf{match.Matchers{
match.NewMin(1),
match.NewContains(string(separators), true),
}},
},
{
[]match.Matcher{
match.NewSingle(nil),
match.NewSingle(nil),
match.NewSingle(nil),
},
match.EveryOf{match.Matchers{
match.NewMin(3),
match.NewMax(3),
}},
},
{
[]match.Matcher{
match.NewList([]rune{'a'}, true),
match.NewAny([]rune{'a'}),
},
match.EveryOf{match.Matchers{
match.NewMin(1),
match.NewContains("a", true),
}},
},
} {
act, err := compileMatchers(test.in)
if err != nil {
t.Errorf("#%d convert matchers error: %s", id, err)
continue
}
if !reflect.DeepEqual(act, test.exp) {
t.Errorf("#%d unexpected convert matchers result:\nact: %#v;\nexp: %#v", id, act, test.exp)
continue
}
}
}
func TestCompileMatchers(t *testing.T) {
for id, test := range []struct {
in []match.Matcher
exp match.Matcher
}{
{
[]match.Matcher{
match.NewSuper(),
match.NewSingle(separators),
match.NewText("c"),
},
match.NewBTree(
match.NewText("c"),
match.NewBTree(
match.NewSingle(separators),
match.NewSuper(),
nil,
),
nil,
),
},
{
[]match.Matcher{
match.NewAny(nil),
match.NewText("c"),
match.NewAny(nil),
},
match.NewBTree(
match.NewText("c"),
match.NewAny(nil),
match.NewAny(nil),
),
},
{
[]match.Matcher{
match.NewRange('a', 'c', true),
match.NewList([]rune{'z', 't', 'e'}, false),
match.NewText("c"),
match.NewSingle(nil),
},
match.NewRow(
4,
match.Matchers{
match.NewRange('a', 'c', true),
match.NewList([]rune{'z', 't', 'e'}, false),
match.NewText("c"),
match.NewSingle(nil),
}...,
),
},
} {
act, err := compileMatchers(test.in)
if err != nil {
t.Errorf("#%d convert matchers error: %s", id, err)
continue
}
if !reflect.DeepEqual(act, test.exp) {
t.Errorf("#%d unexpected convert matchers result:\nact: %#v\nexp: %#v", id, act, test.exp)
continue
}
}
}
func TestConvertMatchers(t *testing.T) {
for id, test := range []struct {
in, exp []match.Matcher
}{
{
[]match.Matcher{
match.NewRange('a', 'c', true),
match.NewList([]rune{'z', 't', 'e'}, false),
match.NewText("c"),
match.NewSingle(nil),
match.NewAny(nil),
},
[]match.Matcher{
match.NewRow(
4,
[]match.Matcher{
match.NewRange('a', 'c', true),
match.NewList([]rune{'z', 't', 'e'}, false),
match.NewText("c"),
match.NewSingle(nil),
}...,
),
match.NewAny(nil),
},
},
{
[]match.Matcher{
match.NewRange('a', 'c', true),
match.NewList([]rune{'z', 't', 'e'}, false),
match.NewText("c"),
match.NewSingle(nil),
match.NewAny(nil),
match.NewSingle(nil),
match.NewSingle(nil),
match.NewAny(nil),
},
[]match.Matcher{
match.NewRow(
3,
match.Matchers{
match.NewRange('a', 'c', true),
match.NewList([]rune{'z', 't', 'e'}, false),
match.NewText("c"),
}...,
),
match.NewMin(3),
},
},
} {
act := minimizeMatchers(test.in)
if !reflect.DeepEqual(act, test.exp) {
t.Errorf("#%d unexpected convert matchers 2 result:\nact: %#v\nexp: %#v", id, act, test.exp)
continue
}
}
}
func pattern(nodes ...node) *nodePattern {
return &nodePattern{
nodeImpl: nodeImpl{
desc: nodes,
},
}
}
func anyOf(nodes ...node) *nodeAnyOf {
return &nodeAnyOf{
nodeImpl: nodeImpl{
desc: nodes,
},
}
}
func TestCompiler(t *testing.T) {
for id, test := range []struct {
ast *nodePattern
result Glob
sep []rune
}{
{
ast: pattern(&nodeText{text: "abc"}),
result: match.NewText("abc"),
},
{
ast: pattern(&nodeAny{}),
sep: separators,
result: match.NewAny(separators),
},
{
ast: pattern(&nodeAny{}),
result: match.NewSuper(),
},
{
ast: pattern(&nodeSuper{}),
result: match.NewSuper(),
},
{
ast: pattern(&nodeSingle{}),
sep: separators,
result: match.NewSingle(separators),
},
{
ast: pattern(&nodeRange{
lo: 'a',
hi: 'z',
not: true,
}),
result: match.NewRange('a', 'z', true),
},
{
ast: pattern(&nodeList{
chars: "abc",
not: true,
}),
result: match.NewList([]rune{'a', 'b', 'c'}, true),
},
{
ast: pattern(&nodeAny{}, &nodeSingle{}, &nodeSingle{}, &nodeSingle{}),
sep: separators,
result: match.EveryOf{Matchers: match.Matchers{
match.NewMin(3),
match.NewContains(string(separators), true),
}},
},
{
ast: pattern(&nodeAny{}, &nodeSingle{}, &nodeSingle{}, &nodeSingle{}),
result: match.NewMin(3),
},
{
ast: pattern(&nodeAny{}, &nodeText{text: "abc"}, &nodeSingle{}),
sep: separators,
result: match.NewBTree(
match.NewRow(
4,
match.Matchers{
match.NewText("abc"),
match.NewSingle(separators),
}...,
),
match.NewAny(separators),
nil,
),
},
{
ast: pattern(&nodeText{text: "/"}, anyOf(&nodeText{text: "z"}, &nodeText{text: "ab"}), &nodeSuper{}),
sep: separators,
result: match.NewBTree(
match.NewText("/"),
nil,
match.NewBTree(
match.NewAnyOf(match.NewText("z"), match.NewText("ab")),
nil,
match.NewSuper(),
),
),
},
{
ast: pattern(&nodeSuper{}, &nodeSingle{}, &nodeText{text: "abc"}, &nodeSingle{}),
sep: separators,
result: match.NewBTree(
match.NewRow(
5,
match.Matchers{
match.NewSingle(separators),
match.NewText("abc"),
match.NewSingle(separators),
}...,
),
match.NewSuper(),
nil,
),
},
{
ast: pattern(&nodeAny{}, &nodeText{text: "abc"}),
result: match.NewSuffix("abc"),
},
{
ast: pattern(&nodeText{text: "abc"}, &nodeAny{}),
result: match.NewPrefix("abc"),
},
{
ast: pattern(&nodeText{text: "abc"}, &nodeAny{}, &nodeText{text: "def"}),
result: match.NewPrefixSuffix("abc", "def"),
},
{
ast: pattern(&nodeAny{}, &nodeAny{}, &nodeAny{}, &nodeText{text: "abc"}, &nodeAny{}, &nodeAny{}),
result: match.NewContains("abc", false),
},
{
ast: pattern(&nodeAny{}, &nodeAny{}, &nodeAny{}, &nodeText{text: "abc"}, &nodeAny{}, &nodeAny{}),
sep: separators,
result: match.NewBTree(
match.NewText("abc"),
match.NewAny(separators),
match.NewAny(separators),
),
},
{
ast: pattern(&nodeSuper{}, &nodeSingle{}, &nodeText{text: "abc"}, &nodeSuper{}, &nodeSingle{}),
result: match.NewBTree(
match.NewText("abc"),
match.NewMin(1),
match.NewMin(1),
),
},
{
ast: pattern(anyOf(&nodeText{text: "abc"})),
result: match.NewText("abc"),
},
{
ast: pattern(anyOf(pattern(anyOf(pattern(&nodeText{text: "abc"}))))),
result: match.NewText("abc"),
},
{
ast: pattern(anyOf(
pattern(
&nodeText{text: "abc"},
&nodeSingle{},
),
pattern(
&nodeText{text: "abc"},
&nodeList{chars: "def"},
),
pattern(
&nodeText{text: "abc"},
),
pattern(
&nodeText{text: "abc"},
),
)),
result: match.NewBTree(
match.NewText("abc"),
nil,
match.AnyOf{Matchers: match.Matchers{
match.NewSingle(nil),
match.NewList([]rune{'d', 'e', 'f'}, false),
match.NewNothing(),
}},
),
},
{
ast: pattern(
&nodeRange{lo: 'a', hi: 'z'},
&nodeRange{lo: 'a', hi: 'x', not: true},
&nodeAny{},
),
result: match.NewBTree(
match.NewRow(
2,
match.Matchers{
match.NewRange('a', 'z', false),
match.NewRange('a', 'x', true),
}...,
),
nil,
match.NewSuper(),
),
},
{
ast: pattern(anyOf(
pattern(
&nodeText{text: "abc"},
&nodeList{chars: "abc"},
&nodeText{text: "ghi"},
),
pattern(
&nodeText{text: "abc"},
&nodeList{chars: "def"},
&nodeText{text: "ghi"},
),
)),
result: match.NewRow(
7,
match.Matchers{
match.NewText("abc"),
match.AnyOf{Matchers: match.Matchers{
match.NewList([]rune{'a', 'b', 'c'}, false),
match.NewList([]rune{'d', 'e', 'f'}, false),
}},
match.NewText("ghi"),
}...,
),
},
} {
m, err := compile(test.ast, test.sep)
if err != nil {
t.Errorf("compilation error: %s", err)
continue
}
if !reflect.DeepEqual(m, test.result) {
t.Errorf("#%d results are not equal:\nexp: %#v\nact: %#v\nexp:\n%s\nact:\n%s\n", id, test.result, m, debug.Graphviz("", test.result.(match.Matcher)), debug.Graphviz("", m.(match.Matcher)))
continue
}
}
}
const complexityString = "abcd"
//func BenchmarkComplexityAny(b *testing.B) {
// m := match.NewAny(nil)
// for i := 0; i < b.N; i++ {
// _ = m.Match(complexityString)
// _, _ = m.Index(complexityString)
// }
//}
//func BenchmarkComplexityContains(b *testing.B) {
// m := match.NewContains()
// for i := 0; i < b.N; i++ {
// _ = m.Match(complexityString)
// _, _ = m.Index(complexityString)
// }
//}
//func BenchmarkComplexityList(b *testing.B) {
// m := match.NewList()
// for i := 0; i < b.N; i++ {
// _ = m.Match(complexityString)
// _, _ = m.Index(complexityString)
// }
//}
//func BenchmarkComplexityMax(b *testing.B) {
// m := match.NewMax()
// for i := 0; i < b.N; i++ {
// _ = m.Match(complexityString)
// _, _ = m.Index(complexityString)
// }
//}
//func BenchmarkComplexityMin(b *testing.B) {
// m := match.NewMin()
// for i := 0; i < b.N; i++ {
// _ = m.Match(complexityString)
// _, _ = m.Index(complexityString)
// }
//}
//func BenchmarkComplexityNothing(b *testing.B) {
// m := match.NewNothing()
// for i := 0; i < b.N; i++ {
// _ = m.Match(complexityString)
// _, _ = m.Index(complexityString)
// }
//}
//func BenchmarkComplexityPrefix(b *testing.B) {
// m := match.NewPrefix()
// for i := 0; i < b.N; i++ {
// _ = m.Match(complexityString)
// _, _ = m.Index(complexityString)
// }
//}
//func BenchmarkComplexityPrefixSuffix(b *testing.B) {
// m := match.NewPrefixSuffix()
// for i := 0; i < b.N; i++ {
// _ = m.Match(complexityString)
// _, _ = m.Index(complexityString)
// }
//}
//func BenchmarkComplexityRange(b *testing.B) {
// m := match.NewRange()
// for i := 0; i < b.N; i++ {
// _ = m.Match(complexityString)
// _, _ = m.Index(complexityString)
// }
//}
//func BenchmarkComplexityRow(b *testing.B) {
// m := match.NewRow()
// for i := 0; i < b.N; i++ {
// _ = m.Match(complexityString)
// _, _ = m.Index(complexityString)
// }
//}
//func BenchmarkComplexitySingle(b *testing.B) {
// m := match.NewSingle(nil)
// for i := 0; i < b.N; i++ {
// _ = m.Match(complexityString)
// _, _ = m.Index(complexityString)
// }
//}
//func BenchmarkComplexitySuffix(b *testing.B) {
// m := match.NewSuffix()
// for i := 0; i < b.N; i++ {
// _ = m.Match(complexityString)
// _, _ = m.Index(complexityString)
// }
//}
//func BenchmarkComplexitySuper(b *testing.B) {
// m := match.NewSuper()
// for i := 0; i < b.N; i++ {
// _ = m.Match(complexityString)
// _, _ = m.Index(complexityString)
// }
//}
//func BenchmarkComplexityText(b *testing.B) {
// m := match.NewText()
// for i := 0; i < b.N; i++ {
// _ = m.Match(complexityString)
// _, _ = m.Index(complexityString)
// }
//}
//func BenchmarkComplexityAnyOf(b *testing.B) {
// m := match.NewAnyOf()
// for i := 0; i < b.N; i++ {
// _ = m.Match(complexityString)
// _, _ = m.Index(complexityString)
// }
//}
//func BenchmarkComplexityBTree(b *testing.B) {
// m := match.NewBTree(match.NewText("abc"), match.NewText("d"), match.NewText("e"))
// for i := 0; i < b.N; i++ {
// _ = m.Match(complexityString)
// _, _ = m.Index(complexityString)
// }
//}
//func BenchmarkComplexityEveryOf(b *testing.B) {
// m := match.NewEveryOf()
// for i := 0; i < b.N; i++ {
// _ = m.Match(complexityString)
// _, _ = m.Index(complexityString)
// }
//}

11
glob.go
View File

@ -1,5 +1,10 @@
package glob package glob
import (
"github.com/gobwas/glob/compiler"
"github.com/gobwas/glob/syntax"
)
// Glob represents compiled glob pattern. // Glob represents compiled glob pattern.
type Glob interface { type Glob interface {
Match(string) bool Match(string) bool
@ -32,12 +37,12 @@ type Glob interface {
// comma-separated (without spaces) patterns // comma-separated (without spaces) patterns
// //
func Compile(pattern string, separators ...rune) (Glob, error) { func Compile(pattern string, separators ...rune) (Glob, error) {
ast, err := parse(newLexer(pattern)) ast, err := syntax.Parse(pattern)
if err != nil { if err != nil {
return nil, err return nil, err
} }
matcher, err := compile(ast, separators) matcher, err := compiler.Compile(ast, separators)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -63,7 +68,7 @@ func QuoteMeta(s string) string {
// a byte loop is correct because all meta characters are ASCII // a byte loop is correct because all meta characters are ASCII
j := 0 j := 0
for i := 0; i < len(s); i++ { for i := 0; i < len(s); i++ {
if special(s[i]) { if syntax.Special(s[i]) {
b[j] = '\\' b[j] = '\\'
j++ j++
} }

14
glob_test.go
View File

@ -159,12 +159,6 @@ func TestGlob(t *testing.T) {
} }
func TestQuoteMeta(t *testing.T) { func TestQuoteMeta(t *testing.T) {
specialsQuoted := make([]byte, len(specials)*2)
for i, j := 0, 0; i < len(specials); i, j = i+1, j+2 {
specialsQuoted[j] = '\\'
specialsQuoted[j+1] = specials[i]
}
for id, test := range []struct { for id, test := range []struct {
in, out string in, out string
}{ }{
@ -177,12 +171,12 @@ func TestQuoteMeta(t *testing.T) {
out: `\{foo\*\}`, out: `\{foo\*\}`,
}, },
{ {
in: string(specials), in: `*?\[]{}`,
out: string(specialsQuoted), out: `\*\?\\\[\]\{\}`,
}, },
{ {
in: string(append([]byte("some text and"), specials...)), in: `some text and *?\[]{}`,
out: string(append([]byte("some text and"), specialsQuoted...)), out: `some text and \*\?\\\[\]\{\}`,
}, },
} { } {
act := QuoteMeta(test.in) act := QuoteMeta(test.in)

192
lexer_test.go
View File

@ -1,192 +0,0 @@
package glob
import (
"testing"
)
func TestLexGood(t *testing.T) {
for id, test := range []struct {
pattern string
items []item
}{
{
pattern: "",
items: []item{
item{item_eof, ""},
},
},
{
pattern: "hello",
items: []item{
item{item_text, "hello"},
item{item_eof, ""},
},
},
{
pattern: "/{rate,[0-9]]}*",
items: []item{
item{item_text, "/"},
item{item_terms_open, "{"},
item{item_text, "rate"},
item{item_separator, ","},
item{item_range_open, "["},
item{item_range_lo, "0"},
item{item_range_between, "-"},
item{item_range_hi, "9"},
item{item_range_close, "]"},
item{item_text, "]"},
item{item_terms_close, "}"},
item{item_any, "*"},
item{item_eof, ""},
},
},
{
pattern: "hello,world",
items: []item{
item{item_text, "hello,world"},
item{item_eof, ""},
},
},
{
pattern: "hello\\,world",
items: []item{
item{item_text, "hello,world"},
item{item_eof, ""},
},
},
{
pattern: "hello\\{world",
items: []item{
item{item_text, "hello{world"},
item{item_eof, ""},
},
},
{
pattern: "hello?",
items: []item{
item{item_text, "hello"},
item{item_single, "?"},
item{item_eof, ""},
},
},
{
pattern: "hellof*",
items: []item{
item{item_text, "hellof"},
item{item_any, "*"},
item{item_eof, ""},
},
},
{
pattern: "hello**",
items: []item{
item{item_text, "hello"},
item{item_super, "**"},
item{item_eof, ""},
},
},
{
pattern: "[日-語]",
items: []item{
item{item_range_open, "["},
item{item_range_lo, "日"},
item{item_range_between, "-"},
item{item_range_hi, "語"},
item{item_range_close, "]"},
item{item_eof, ""},
},
},
{
pattern: "[!日-語]",
items: []item{
item{item_range_open, "["},
item{item_not, "!"},
item{item_range_lo, "日"},
item{item_range_between, "-"},
item{item_range_hi, "語"},
item{item_range_close, "]"},
item{item_eof, ""},
},
},
{
pattern: "[日本語]",
items: []item{
item{item_range_open, "["},
item{item_text, "日本語"},
item{item_range_close, "]"},
item{item_eof, ""},
},
},
{
pattern: "[!日本語]",
items: []item{
item{item_range_open, "["},
item{item_not, "!"},
item{item_text, "日本語"},
item{item_range_close, "]"},
item{item_eof, ""},
},
},
{
pattern: "{a,b}",
items: []item{
item{item_terms_open, "{"},
item{item_text, "a"},
item{item_separator, ","},
item{item_text, "b"},
item{item_terms_close, "}"},
item{item_eof, ""},
},
},
{
pattern: "/{z,ab}*",
items: []item{
item{item_text, "/"},
item{item_terms_open, "{"},
item{item_text, "z"},
item{item_separator, ","},
item{item_text, "ab"},
item{item_terms_close, "}"},
item{item_any, "*"},
item{item_eof, ""},
},
},
{
pattern: "{[!日-語],*,?,{a,b,\\c}}",
items: []item{
item{item_terms_open, "{"},
item{item_range_open, "["},
item{item_not, "!"},
item{item_range_lo, "日"},
item{item_range_between, "-"},
item{item_range_hi, "語"},
item{item_range_close, "]"},
item{item_separator, ","},
item{item_any, "*"},
item{item_separator, ","},
item{item_single, "?"},
item{item_separator, ","},
item{item_terms_open, "{"},
item{item_text, "a"},
item{item_separator, ","},
item{item_text, "b"},
item{item_separator, ","},
item{item_text, "c"},
item{item_terms_close, "}"},
item{item_terms_close, "}"},
item{item_eof, ""},
},
},
} {
lexer := newLexer(test.pattern)
for i, exp := range test.items {
act := lexer.nextItem()
if act.t != exp.t {
t.Errorf("#%d %q: wrong %d-th item type: exp: %q; act: %q\n\t(%s vs %s)", id, test.pattern, i, exp.t, act.t, exp, act)
}
if act.s != exp.s {
t.Errorf("#%d %q: wrong %d-th item contents: exp: %q; act: %q\n\t(%s vs %s)", id, test.pattern, i, exp.s, act.s, exp, act)
}
}
}
}

2
match/any.go
View File

@ -2,7 +2,7 @@ package match
import ( import (
"fmt" "fmt"
"github.com/gobwas/glob/strings" "github.com/gobwas/glob/util/strings"
) )
type Any struct { type Any struct {

2
match/list.go
View File

@ -2,7 +2,7 @@ package match
import ( import (
"fmt" "fmt"
"github.com/gobwas/glob/runes" "github.com/gobwas/glob/util/runes"
"unicode/utf8" "unicode/utf8"
) )

2
match/single.go
View File

@ -2,7 +2,7 @@ package match
import ( import (
"fmt" "fmt"
"github.com/gobwas/glob/runes" "github.com/gobwas/glob/util/runes"
"unicode/utf8" "unicode/utf8"
) )

230
parser.go
View File

@ -1,230 +0,0 @@
package glob
import (
"errors"
"fmt"
"unicode/utf8"
)
type node interface {
children() []node
append(node)
}
// todo may be split it into another package
type lexerIface interface {
nextItem() item
}
type nodeImpl struct {
desc []node
}
func (n *nodeImpl) append(c node) {
n.desc = append(n.desc, c)
}
func (n *nodeImpl) children() []node {
return n.desc
}
type nodeList struct {
nodeImpl
not bool
chars string
}
type nodeRange struct {
nodeImpl
not bool
lo, hi rune
}
type nodeText struct {
nodeImpl
text string
}
type nodePattern struct{ nodeImpl }
type nodeAny struct{ nodeImpl }
type nodeSuper struct{ nodeImpl }
type nodeSingle struct{ nodeImpl }
type nodeAnyOf struct{ nodeImpl }
type tree struct {
root node
current node
path []node
}
func (t *tree) enter(c node) {
if t.root == nil {
t.root = c
t.current = c
return
}
t.current.append(c)
t.path = append(t.path, c)
t.current = c
}
func (t *tree) leave() {
if len(t.path)-1 <= 0 {
t.current = t.root
t.path = nil
return
}
t.path = t.path[:len(t.path)-1]
t.current = t.path[len(t.path)-1]
}
type parseFn func(*tree, lexerIface) (parseFn, error)
func parse(lexer lexerIface) (*nodePattern, error) {
var parser parseFn
root := &nodePattern{}
tree := &tree{}
tree.enter(root)
for parser = parserMain; ; {
next, err := parser(tree, lexer)
if err != nil {
return nil, err
}
if next == nil {
break
}
parser = next
}
return root, nil
}
func parserMain(tree *tree, lexer lexerIface) (parseFn, error) {
for stop := false; !stop; {
item := lexer.nextItem()
switch item.t {
case item_eof:
stop = true
continue
case item_error:
return nil, errors.New(item.s)
case item_text:
tree.current.append(&nodeText{text: item.s})
return parserMain, nil
case item_any:
tree.current.append(&nodeAny{})
return parserMain, nil
case item_super:
tree.current.append(&nodeSuper{})
return parserMain, nil
case item_single:
tree.current.append(&nodeSingle{})
return parserMain, nil
case item_range_open:
return parserRange, nil
case item_terms_open:
tree.enter(&nodeAnyOf{})
tree.enter(&nodePattern{})
return parserMain, nil
case item_separator:
tree.leave()
tree.enter(&nodePattern{})
return parserMain, nil
case item_terms_close:
tree.leave()
tree.leave()
return parserMain, nil
default:
return nil, fmt.Errorf("unexpected token: %s", item)
}
}
return nil, nil
}
func parserRange(tree *tree, lexer lexerIface) (parseFn, error) {
var (
not bool
lo rune
hi rune
chars string
)
for {
item := lexer.nextItem()
switch item.t {
case item_eof:
return nil, errors.New("unexpected end")
case item_error:
return nil, errors.New(item.s)
case item_not:
not = true
case item_range_lo:
r, w := utf8.DecodeRuneInString(item.s)
if len(item.s) > w {
return nil, fmt.Errorf("unexpected length of lo character")
}
lo = r
case item_range_between:
//
case item_range_hi:
r, w := utf8.DecodeRuneInString(item.s)
if len(item.s) > w {
return nil, fmt.Errorf("unexpected length of lo character")
}
hi = r
if hi < lo {
return nil, fmt.Errorf("hi character '%s' should be greater than lo '%s'", string(hi), string(lo))
}
case item_text:
chars = item.s
case item_range_close:
isRange := lo != 0 && hi != 0
isChars := chars != ""
if isChars == isRange {
return nil, fmt.Errorf("could not parse range")
}
if isRange {
tree.current.append(&nodeRange{
lo: lo,
hi: hi,
not: not,
})
} else {
tree.current.append(&nodeList{
chars: chars,
not: not,
})
}
return parserMain, nil
}
}
}

324
parser_test.go
View File

@ -1,324 +0,0 @@
package glob
import (
"fmt"
"reflect"
"testing"
)
func TestParseString(t *testing.T) {
for id, test := range []struct {
items []item
tree node
}{
{
//pattern: "abc",
items: []item{
item{item_text, "abc"},
item{item_eof, ""},
},
tree: &nodePattern{
nodeImpl: nodeImpl{
desc: []node{
&nodeText{text: "abc"},
},
},
},
},
{
//pattern: "a*c",
items: []item{
item{item_text, "a"},
item{item_any, "*"},
item{item_text, "c"},
item{item_eof, ""},
},
tree: &nodePattern{
nodeImpl: nodeImpl{
desc: []node{
&nodeText{text: "a"},
&nodeAny{},
&nodeText{text: "c"},
},
},
},
},
{
//pattern: "a**c",
items: []item{
item{item_text, "a"},
item{item_super, "**"},
item{item_text, "c"},
item{item_eof, ""},
},
tree: &nodePattern{
nodeImpl: nodeImpl{
desc: []node{
&nodeText{text: "a"},
&nodeSuper{},
&nodeText{text: "c"},
},
},
},
},
{
//pattern: "a?c",
items: []item{
item{item_text, "a"},
item{item_single, "?"},
item{item_text, "c"},
item{item_eof, ""},
},
tree: &nodePattern{
nodeImpl: nodeImpl{
desc: []node{
&nodeText{text: "a"},
&nodeSingle{},
&nodeText{text: "c"},
},
},
},
},
{
//pattern: "[!a-z]",
items: []item{
item{item_range_open, "["},
item{item_not, "!"},
item{item_range_lo, "a"},
item{item_range_between, "-"},
item{item_range_hi, "z"},
item{item_range_close, "]"},
item{item_eof, ""},
},
tree: &nodePattern{
nodeImpl: nodeImpl{
desc: []node{
&nodeRange{lo: 'a', hi: 'z', not: true},
},
},
},
},
{
//pattern: "[az]",
items: []item{
item{item_range_open, "["},
item{item_text, "az"},
item{item_range_close, "]"},
item{item_eof, ""},
},
tree: &nodePattern{
nodeImpl: nodeImpl{
desc: []node{
&nodeList{chars: "az"},
},
},
},
},
{
//pattern: "{a,z}",
items: []item{
item{item_terms_open, "{"},
item{item_text, "a"},
item{item_separator, ","},
item{item_text, "z"},
item{item_terms_close, "}"},
item{item_eof, ""},
},
tree: &nodePattern{
nodeImpl: nodeImpl{
desc: []node{
&nodeAnyOf{nodeImpl: nodeImpl{desc: []node{
&nodePattern{
nodeImpl: nodeImpl{desc: []node{
&nodeText{text: "a"},
}},
},
&nodePattern{
nodeImpl: nodeImpl{desc: []node{
&nodeText{text: "z"},
}},
},
}}},
},
},
},
},
{
//pattern: "/{z,ab}*",
items: []item{
item{item_text, "/"},
item{item_terms_open, "{"},
item{item_text, "z"},
item{item_separator, ","},
item{item_text, "ab"},
item{item_terms_close, "}"},
item{item_any, "*"},
item{item_eof, ""},
},
tree: &nodePattern{
nodeImpl: nodeImpl{
desc: []node{
&nodeText{text: "/"},
&nodeAnyOf{nodeImpl: nodeImpl{desc: []node{
&nodePattern{
nodeImpl: nodeImpl{desc: []node{
&nodeText{text: "z"},
}},
},
&nodePattern{
nodeImpl: nodeImpl{desc: []node{
&nodeText{text: "ab"},
}},
},
}}},
&nodeAny{},
},
},
},
},
{
//pattern: "{a,{x,y},?,[a-z],[!qwe]}",
items: []item{
item{item_terms_open, "{"},
item{item_text, "a"},
item{item_separator, ","},
item{item_terms_open, "{"},
item{item_text, "x"},
item{item_separator, ","},
item{item_text, "y"},
item{item_terms_close, "}"},
item{item_separator, ","},
item{item_single, "?"},
item{item_separator, ","},
item{item_range_open, "["},
item{item_range_lo, "a"},
item{item_range_between, "-"},
item{item_range_hi, "z"},
item{item_range_close, "]"},
item{item_separator, ","},
item{item_range_open, "["},
item{item_not, "!"},
item{item_text, "qwe"},
item{item_range_close, "]"},
item{item_terms_close, "}"},
item{item_eof, ""},
},
tree: &nodePattern{
nodeImpl: nodeImpl{
desc: []node{
&nodeAnyOf{nodeImpl: nodeImpl{desc: []node{
&nodePattern{
nodeImpl: nodeImpl{desc: []node{
&nodeText{text: "a"},
}},
},
&nodePattern{
nodeImpl: nodeImpl{desc: []node{
&nodeAnyOf{nodeImpl: nodeImpl{desc: []node{
&nodePattern{
nodeImpl: nodeImpl{desc: []node{
&nodeText{text: "x"},
}},
},
&nodePattern{
nodeImpl: nodeImpl{desc: []node{
&nodeText{text: "y"},
}},
},
}}},
}},
},
&nodePattern{
nodeImpl: nodeImpl{desc: []node{
&nodeSingle{},
}},
},
&nodePattern{
nodeImpl: nodeImpl{
desc: []node{
&nodeRange{lo: 'a', hi: 'z', not: false},
},
},
},
&nodePattern{
nodeImpl: nodeImpl{
desc: []node{
&nodeList{chars: "qwe", not: true},
},
},
},
}}},
},
},
},
},
} {
lexer := &stubLexer{Items: test.items}
pattern, err := parse(lexer)
if err != nil {
t.Errorf("#%d %s", id, err)
continue
}
if !reflect.DeepEqual(test.tree, pattern) {
t.Errorf("#%d tries are not equal", id)
if err = nodeEqual(test.tree, pattern); err != nil {
t.Errorf("#%d %s", id, err)
continue
}
}
}
}
const abstractNodeImpl = "nodeImpl"
func nodeEqual(a, b node) error {
if (a == nil || b == nil) && a != b {
return fmt.Errorf("nodes are not equal: exp %s, act %s", a, b)
}
aValue, bValue := reflect.Indirect(reflect.ValueOf(a)), reflect.Indirect(reflect.ValueOf(b))
aType, bType := aValue.Type(), bValue.Type()
if aType != bType {
return fmt.Errorf("nodes are not equal: exp %s, act %s", aValue.Type(), bValue.Type())
}
for i := 0; i < aType.NumField(); i++ {
var eq bool
f := aType.Field(i).Name
if f == abstractNodeImpl {
continue
}
af, bf := aValue.FieldByName(f), bValue.FieldByName(f)
switch af.Kind() {
case reflect.String:
eq = af.String() == bf.String()
case reflect.Bool:
eq = af.Bool() == bf.Bool()
default:
eq = fmt.Sprint(af) == fmt.Sprint(bf)
}
if !eq {
return fmt.Errorf("nodes<%s> %q fields are not equal: exp %q, act %q", aType, f, af, bf)
}
}
for i, aDesc := range a.children() {
if len(b.children())-1 < i {
return fmt.Errorf("node does not have enough children (got %d children, wanted %d-th token)", len(b.children()), i)
}
bDesc := b.children()[i]
if err := nodeEqual(aDesc, bDesc); err != nil {
return err
}
}
return nil
}

72
syntax/ast/ast.go Normal file
View File

@ -0,0 +1,72 @@
package ast
type Node struct {
Parent *Node
Children []*Node
Value interface{}
Kind Kind
}
func NewNode(k Kind, v interface{}, ch ...*Node) *Node {
n := &Node{
Kind: k,
Value: v,
}
for _, c := range ch {
Insert(n, c)
}
return n
}
func (a *Node) Equal(b *Node) bool {
if a.Kind != b.Kind {
return false
}
if a.Value != b.Value {
return false
}
if len(a.Children) != len(b.Children) {
return false
}
for i, c := range a.Children {
if !c.Equal(b.Children[i]) {
return false
}
}
return true
}
func Insert(parent *Node, children ...*Node) {
parent.Children = append(parent.Children, children...)
for _, ch := range children {
ch.Parent = parent
}
}
type List struct {
Not bool
Chars string
}
type Range struct {
Not bool
Lo, Hi rune
}
type Text struct {
Text string
}
type Kind int
const (
KindNothing Kind = iota
KindPattern
KindList
KindRange
KindText
KindAny
KindSuper
KindSingle
KindAnyOf
)

157
syntax/ast/parser.go Normal file
View File

@ -0,0 +1,157 @@
package ast
import (
"errors"
"fmt"
"github.com/gobwas/glob/syntax/lexer"
"unicode/utf8"
)
type Lexer interface {
Next() lexer.Token
}
type parseFn func(*Node, Lexer) (parseFn, *Node, error)
func Parse(lexer Lexer) (*Node, error) {
var parser parseFn
root := NewNode(KindPattern, nil)
var (
tree *Node
err error
)
for parser, tree = parserMain, root; parser != nil; {
parser, tree, err = parser(tree, lexer)
if err != nil {
return nil, err
}
}
return root, nil
}
func parserMain(tree *Node, lex Lexer) (parseFn, *Node, error) {
for {
token := lex.Next()
switch token.Type {
case lexer.EOF:
return nil, tree, nil
case lexer.Error:
return nil, tree, errors.New(token.Raw)
case lexer.Text:
Insert(tree, NewNode(KindText, Text{token.Raw}))
return parserMain, tree, nil
case lexer.Any:
Insert(tree, NewNode(KindAny, nil))
return parserMain, tree, nil
case lexer.Super:
Insert(tree, NewNode(KindSuper, nil))
return parserMain, tree, nil
case lexer.Single:
Insert(tree, NewNode(KindSingle, nil))
return parserMain, tree, nil
case lexer.RangeOpen:
return parserRange, tree, nil
case lexer.TermsOpen:
a := NewNode(KindAnyOf, nil)
Insert(tree, a)
p := NewNode(KindPattern, nil)
Insert(a, p)
return parserMain, p, nil
case lexer.Separator:
p := NewNode(KindPattern, nil)
Insert(tree.Parent, p)
return parserMain, p, nil
case lexer.TermsClose:
return parserMain, tree.Parent.Parent, nil
default:
return nil, tree, fmt.Errorf("unexpected token: %s", token)
}
}
return nil, tree, fmt.Errorf("unknown error")
}
func parserRange(tree *Node, lex Lexer) (parseFn, *Node, error) {
var (
not bool
lo rune
hi rune
chars string
)
for {
token := lex.Next()
switch token.Type {
case lexer.EOF:
return nil, tree, errors.New("unexpected end")
case lexer.Error:
return nil, tree, errors.New(token.Raw)
case lexer.Not:
not = true
case lexer.RangeLo:
r, w := utf8.DecodeRuneInString(token.Raw)
if len(token.Raw) > w {
return nil, tree, fmt.Errorf("unexpected length of lo character")
}
lo = r
case lexer.RangeBetween:
//
case lexer.RangeHi:
r, w := utf8.DecodeRuneInString(token.Raw)
if len(token.Raw) > w {
return nil, tree, fmt.Errorf("unexpected length of lo character")
}
hi = r
if hi < lo {
return nil, tree, fmt.Errorf("hi character '%s' should be greater than lo '%s'", string(hi), string(lo))
}
case lexer.Text:
chars = token.Raw
case lexer.RangeClose:
isRange := lo != 0 && hi != 0
isChars := chars != ""
if isChars == isRange {
return nil, tree, fmt.Errorf("could not parse range")
}
if isRange {
Insert(tree, NewNode(KindRange, Range{
Lo: lo,
Hi: hi,
Not: not,
}))
} else {
Insert(tree, NewNode(KindList, List{
Chars: chars,
Not: not,
}))
}
return parserMain, tree, nil
}
}
}

217
syntax/ast/parser_test.go Normal file
View File

@ -0,0 +1,217 @@
package ast
import (
"github.com/gobwas/glob/syntax/lexer"
"reflect"
"testing"
)
type stubLexer struct {
tokens []lexer.Token
pos int
}
func (s *stubLexer) Next() (ret lexer.Token) {
if s.pos == len(s.tokens) {
return lexer.Token{lexer.EOF, ""}
}
ret = s.tokens[s.pos]
s.pos++
return
}
func TestParseString(t *testing.T) {
for id, test := range []struct {
tokens []lexer.Token
tree *Node
}{
{
//pattern: "abc",
tokens: []lexer.Token{
lexer.Token{lexer.Text, "abc"},
lexer.Token{lexer.EOF, ""},
},
tree: NewNode(KindPattern, nil,
NewNode(KindText, Text{Text: "abc"}),
),
},
{
//pattern: "a*c",
tokens: []lexer.Token{
lexer.Token{lexer.Text, "a"},
lexer.Token{lexer.Any, "*"},
lexer.Token{lexer.Text, "c"},
lexer.Token{lexer.EOF, ""},
},
tree: NewNode(KindPattern, nil,
NewNode(KindText, Text{Text: "a"}),
NewNode(KindAny, nil),
NewNode(KindText, Text{Text: "c"}),
),
},
{
//pattern: "a**c",
tokens: []lexer.Token{
lexer.Token{lexer.Text, "a"},
lexer.Token{lexer.Super, "**"},
lexer.Token{lexer.Text, "c"},
lexer.Token{lexer.EOF, ""},
},
tree: NewNode(KindPattern, nil,
NewNode(KindText, Text{Text: "a"}),
NewNode(KindSuper, nil),
NewNode(KindText, Text{Text: "c"}),
),
},
{
//pattern: "a?c",
tokens: []lexer.Token{
lexer.Token{lexer.Text, "a"},
lexer.Token{lexer.Single, "?"},
lexer.Token{lexer.Text, "c"},
lexer.Token{lexer.EOF, ""},
},
tree: NewNode(KindPattern, nil,
NewNode(KindText, Text{Text: "a"}),
NewNode(KindSingle, nil),
NewNode(KindText, Text{Text: "c"}),
),
},
{
//pattern: "[!a-z]",
tokens: []lexer.Token{
lexer.Token{lexer.RangeOpen, "["},
lexer.Token{lexer.Not, "!"},
lexer.Token{lexer.RangeLo, "a"},
lexer.Token{lexer.RangeBetween, "-"},
lexer.Token{lexer.RangeHi, "z"},
lexer.Token{lexer.RangeClose, "]"},
lexer.Token{lexer.EOF, ""},
},
tree: NewNode(KindPattern, nil,
NewNode(KindRange, Range{Lo: 'a', Hi: 'z', Not: true}),
),
},
{
//pattern: "[az]",
tokens: []lexer.Token{
lexer.Token{lexer.RangeOpen, "["},
lexer.Token{lexer.Text, "az"},
lexer.Token{lexer.RangeClose, "]"},
lexer.Token{lexer.EOF, ""},
},
tree: NewNode(KindPattern, nil,
NewNode(KindList, List{Chars: "az"}),
),
},
{
//pattern: "{a,z}",
tokens: []lexer.Token{
lexer.Token{lexer.TermsOpen, "{"},
lexer.Token{lexer.Text, "a"},
lexer.Token{lexer.Separator, ","},
lexer.Token{lexer.Text, "z"},
lexer.Token{lexer.TermsClose, "}"},
lexer.Token{lexer.EOF, ""},
},
tree: NewNode(KindPattern, nil,
NewNode(KindAnyOf, nil,
NewNode(KindPattern, nil,
NewNode(KindText, Text{Text: "a"}),
),
NewNode(KindPattern, nil,
NewNode(KindText, Text{Text: "z"}),
),
),
),
},
{
//pattern: "/{z,ab}*",
tokens: []lexer.Token{
lexer.Token{lexer.Text, "/"},
lexer.Token{lexer.TermsOpen, "{"},
lexer.Token{lexer.Text, "z"},
lexer.Token{lexer.Separator, ","},
lexer.Token{lexer.Text, "ab"},
lexer.Token{lexer.TermsClose, "}"},
lexer.Token{lexer.Any, "*"},
lexer.Token{lexer.EOF, ""},
},
tree: NewNode(KindPattern, nil,
NewNode(KindText, Text{Text: "/"}),
NewNode(KindAnyOf, nil,
NewNode(KindPattern, nil,
NewNode(KindText, Text{Text: "z"}),
),
NewNode(KindPattern, nil,
NewNode(KindText, Text{Text: "ab"}),
),
),
NewNode(KindAny, nil),
),
},
{
//pattern: "{a,{x,y},?,[a-z],[!qwe]}",
tokens: []lexer.Token{
lexer.Token{lexer.TermsOpen, "{"},
lexer.Token{lexer.Text, "a"},
lexer.Token{lexer.Separator, ","},
lexer.Token{lexer.TermsOpen, "{"},
lexer.Token{lexer.Text, "x"},
lexer.Token{lexer.Separator, ","},
lexer.Token{lexer.Text, "y"},
lexer.Token{lexer.TermsClose, "}"},
lexer.Token{lexer.Separator, ","},
lexer.Token{lexer.Single, "?"},
lexer.Token{lexer.Separator, ","},
lexer.Token{lexer.RangeOpen, "["},
lexer.Token{lexer.RangeLo, "a"},
lexer.Token{lexer.RangeBetween, "-"},
lexer.Token{lexer.RangeHi, "z"},
lexer.Token{lexer.RangeClose, "]"},
lexer.Token{lexer.Separator, ","},
lexer.Token{lexer.RangeOpen, "["},
lexer.Token{lexer.Not, "!"},
lexer.Token{lexer.Text, "qwe"},
lexer.Token{lexer.RangeClose, "]"},
lexer.Token{lexer.TermsClose, "}"},
lexer.Token{lexer.EOF, ""},
},
tree: NewNode(KindPattern, nil,
NewNode(KindAnyOf, nil,
NewNode(KindPattern, nil,
NewNode(KindText, Text{Text: "a"}),
),
NewNode(KindPattern, nil,
NewNode(KindAnyOf, nil,
NewNode(KindPattern, nil,
NewNode(KindText, Text{Text: "x"}),
),
NewNode(KindPattern, nil,
NewNode(KindText, Text{Text: "y"}),
),
),
),
NewNode(KindPattern, nil,
NewNode(KindSingle, nil),
),
NewNode(KindPattern, nil,
NewNode(KindRange, Range{Lo: 'a', Hi: 'z', Not: false}),
),
NewNode(KindPattern, nil,
NewNode(KindList, List{Chars: "qwe", Not: true}),
),
),
),
},
} {
lexer := &stubLexer{tokens: test.tokens}
result, err := Parse(lexer)
if err != nil {
t.Errorf("[%d] unexpected error: %s", id, err)
}
if !reflect.DeepEqual(test.tree, result) {
t.Errorf("[%d] Parse():\nact:\t%s\nexp:\t%s\n", id, result, test.tree)
}
}
}

171
lexer.go → syntax/lexer/lexer.go
View File

@ -1,9 +1,9 @@
package glob package lexer
import ( import (
"bytes" "bytes"
"fmt" "fmt"
"github.com/gobwas/glob/runes" "github.com/gobwas/glob/util/runes"
"unicode/utf8" "unicode/utf8"
) )
@ -30,123 +30,24 @@ var specials = []byte{
char_terms_close, char_terms_close,
} }
func special(c byte) bool { func Special(c byte) bool {
return bytes.IndexByte(specials, c) != -1 return bytes.IndexByte(specials, c) != -1
} }
type itemType int type tokens []Token
const ( func (i *tokens) shift() (ret Token) {
item_eof itemType = iota
item_error
item_text
item_char
item_any
item_super
item_single
item_not
item_separator
item_range_open
item_range_close
item_range_lo
item_range_hi
item_range_between
item_terms_open
item_terms_close
)
func (i itemType) String() string {
switch i {
case item_eof:
return "eof"
case item_error:
return "error"
case item_text:
return "text"
case item_char:
return "char"
case item_any:
return "any"
case item_super:
return "super"
case item_single:
return "single"
case item_not:
return "not"
case item_separator:
return "separator"
case item_range_open:
return "range_open"
case item_range_close:
return "range_close"
case item_range_lo:
return "range_lo"
case item_range_hi:
return "range_hi"
case item_range_between:
return "range_between"
case item_terms_open:
return "terms_open"
case item_terms_close:
return "terms_close"
default:
return "undef"
}
}
type item struct {
t itemType
s string
}
func (i item) String() string {
return fmt.Sprintf("%v<%q>", i.t, i.s)
}
type stubLexer struct {
Items []item
pos int
}
func (s *stubLexer) nextItem() (ret item) {
if s.pos == len(s.Items) {
return item{item_eof, ""}
}
ret = s.Items[s.pos]
s.pos++
return
}
type items []item
func (i *items) shift() (ret item) {
ret = (*i)[0] ret = (*i)[0]
copy(*i, (*i)[1:]) copy(*i, (*i)[1:])
*i = (*i)[:len(*i)-1] *i = (*i)[:len(*i)-1]
return return
} }
func (i *items) push(v item) { func (i *tokens) push(v Token) {
*i = append(*i, v) *i = append(*i, v)
} }
func (i *items) empty() bool { func (i *tokens) empty() bool {
return len(*i) == 0 return len(*i) == 0
} }
@ -157,7 +58,7 @@ type lexer struct {
pos int pos int
err error err error
items items tokens tokens
termsLevel int termsLevel int
lastRune rune lastRune rune
@ -165,14 +66,26 @@ type lexer struct {
hasRune bool hasRune bool
} }
func newLexer(source string) *lexer { func NewLexer(source string) *lexer {
l := &lexer{ l := &lexer{
data: source, data: source,
items: items(make([]item, 0, 4)), tokens: tokens(make([]Token, 0, 4)),
} }
return l return l
} }
func (l *lexer) Next() Token {
if l.err != nil {
return Token{Error, l.err.Error()}
}
if !l.tokens.empty() {
return l.tokens.shift()
}
l.fetchItem()
return l.Next()
}
func (l *lexer) peek() (r rune, w int) { func (l *lexer) peek() (r rune, w int) {
if l.pos == len(l.data) { if l.pos == len(l.data) {
return eof, 0 return eof, 0
@ -233,18 +146,6 @@ func (l *lexer) termsLeave() {
l.termsLevel-- l.termsLevel--
} }
func (l *lexer) nextItem() item {
if l.err != nil {
return item{item_error, l.err.Error()}
}
if !l.items.empty() {
return l.items.shift()
}
l.fetchItem()
return l.nextItem()
}
var inTextBreakers = []rune{char_single, char_any, char_range_open, char_terms_open} var inTextBreakers = []rune{char_single, char_any, char_range_open, char_terms_open}
var inTermsBreakers = append(inTextBreakers, char_terms_close, char_comma) var inTermsBreakers = append(inTextBreakers, char_terms_close, char_comma)
@ -252,32 +153,32 @@ func (l *lexer) fetchItem() {
r := l.read() r := l.read()
switch { switch {
case r == eof: case r == eof:
l.items.push(item{item_eof, ""}) l.tokens.push(Token{EOF, ""})
case r == char_terms_open: case r == char_terms_open:
l.termsEnter() l.termsEnter()
l.items.push(item{item_terms_open, string(r)}) l.tokens.push(Token{TermsOpen, string(r)})
case r == char_comma && l.inTerms(): case r == char_comma && l.inTerms():
l.items.push(item{item_separator, string(r)}) l.tokens.push(Token{Separator, string(r)})
case r == char_terms_close && l.inTerms(): case r == char_terms_close && l.inTerms():
l.items.push(item{item_terms_close, string(r)}) l.tokens.push(Token{TermsClose, string(r)})
l.termsLeave() l.termsLeave()
case r == char_range_open: case r == char_range_open:
l.items.push(item{item_range_open, string(r)}) l.tokens.push(Token{RangeOpen, string(r)})
l.fetchRange() l.fetchRange()
case r == char_single: case r == char_single:
l.items.push(item{item_single, string(r)}) l.tokens.push(Token{Single, string(r)})
case r == char_any: case r == char_any:
if l.read() == char_any { if l.read() == char_any {
l.items.push(item{item_super, string(r) + string(r)}) l.tokens.push(Token{Super, string(r) + string(r)})
} else { } else {
l.unread() l.unread()
l.items.push(item{item_any, string(r)}) l.tokens.push(Token{Any, string(r)})
} }
default: default:
@ -308,27 +209,27 @@ func (l *lexer) fetchRange() {
if r != char_range_close { if r != char_range_close {
l.errorf("expected close range character") l.errorf("expected close range character")
} else { } else {
l.items.push(item{item_range_close, string(r)}) l.tokens.push(Token{RangeClose, string(r)})
} }
return return
} }
if wantHi { if wantHi {
l.items.push(item{item_range_hi, string(r)}) l.tokens.push(Token{RangeHi, string(r)})
wantClose = true wantClose = true
continue continue
} }
if !seenNot && r == char_range_not { if !seenNot && r == char_range_not {
l.items.push(item{item_not, string(r)}) l.tokens.push(Token{Not, string(r)})
seenNot = true seenNot = true
continue continue
} }
if n, w := l.peek(); n == char_range_between { if n, w := l.peek(); n == char_range_between {
l.seek(w) l.seek(w)
l.items.push(item{item_range_lo, string(r)}) l.tokens.push(Token{RangeLo, string(r)})
l.items.push(item{item_range_between, string(n)}) l.tokens.push(Token{RangeBetween, string(n)})
wantHi = true wantHi = true
continue continue
} }
@ -367,6 +268,6 @@ reading:
} }
if len(data) > 0 { if len(data) > 0 {
l.items.push(item{item_text, string(data)}) l.tokens.push(Token{Text, string(data)})
} }
} }

192
syntax/lexer/lexer_test.go Normal file
View File

@ -0,0 +1,192 @@
package lexer
import (
"testing"
)
func TestLexGood(t *testing.T) {
for id, test := range []struct {
pattern string
items []Token
}{
{
pattern: "",
items: []Token{
Token{EOF, ""},
},
},
{
pattern: "hello",
items: []Token{
Token{Text, "hello"},
Token{EOF, ""},
},
},
{
pattern: "/{rate,[0-9]]}*",
items: []Token{
Token{Text, "/"},
Token{TermsOpen, "{"},
Token{Text, "rate"},
Token{Separator, ","},
Token{RangeOpen, "["},
Token{RangeLo, "0"},
Token{RangeBetween, "-"},
Token{RangeHi, "9"},
Token{RangeClose, "]"},
Token{Text, "]"},
Token{TermsClose, "}"},
Token{Any, "*"},
Token{EOF, ""},
},
},
{
pattern: "hello,world",
items: []Token{
Token{Text, "hello,world"},
Token{EOF, ""},
},
},
{
pattern: "hello\\,world",
items: []Token{
Token{Text, "hello,world"},
Token{EOF, ""},
},
},
{
pattern: "hello\\{world",
items: []Token{
Token{Text, "hello{world"},
Token{EOF, ""},
},
},
{
pattern: "hello?",
items: []Token{
Token{Text, "hello"},
Token{Single, "?"},
Token{EOF, ""},
},
},
{
pattern: "hellof*",
items: []Token{
Token{Text, "hellof"},
Token{Any, "*"},
Token{EOF, ""},
},
},
{
pattern: "hello**",
items: []Token{
Token{Text, "hello"},
Token{Super, "**"},
Token{EOF, ""},
},
},
{
pattern: "[日-語]",
items: []Token{
Token{RangeOpen, "["},
Token{RangeLo, "日"},
Token{RangeBetween, "-"},
Token{RangeHi, "語"},
Token{RangeClose, "]"},
Token{EOF, ""},
},
},
{
pattern: "[!日-語]",
items: []Token{
Token{RangeOpen, "["},
Token{Not, "!"},
Token{RangeLo, "日"},
Token{RangeBetween, "-"},
Token{RangeHi, "語"},
Token{RangeClose, "]"},
Token{EOF, ""},
},
},
{
pattern: "[日本語]",
items: []Token{
Token{RangeOpen, "["},
Token{Text, "日本語"},
Token{RangeClose, "]"},
Token{EOF, ""},
},
},
{
pattern: "[!日本語]",
items: []Token{
Token{RangeOpen, "["},
Token{Not, "!"},
Token{Text, "日本語"},
Token{RangeClose, "]"},
Token{EOF, ""},
},
},
{
pattern: "{a,b}",
items: []Token{
Token{TermsOpen, "{"},
Token{Text, "a"},
Token{Separator, ","},
Token{Text, "b"},
Token{TermsClose, "}"},
Token{EOF, ""},
},
},
{
pattern: "/{z,ab}*",
items: []Token{
Token{Text, "/"},
Token{TermsOpen, "{"},
Token{Text, "z"},
Token{Separator, ","},
Token{Text, "ab"},
Token{TermsClose, "}"},
Token{Any, "*"},
Token{EOF, ""},
},
},
{
pattern: "{[!日-語],*,?,{a,b,\\c}}",
items: []Token{
Token{TermsOpen, "{"},
Token{RangeOpen, "["},
Token{Not, "!"},
Token{RangeLo, "日"},
Token{RangeBetween, "-"},
Token{RangeHi, "語"},
Token{RangeClose, "]"},
Token{Separator, ","},
Token{Any, "*"},
Token{Separator, ","},
Token{Single, "?"},
Token{Separator, ","},
Token{TermsOpen, "{"},
Token{Text, "a"},
Token{Separator, ","},
Token{Text, "b"},
Token{Separator, ","},
Token{Text, "c"},
Token{TermsClose, "}"},
Token{TermsClose, "}"},
Token{EOF, ""},
},
},
} {
lexer := NewLexer(test.pattern)
for i, exp := range test.items {
act := lexer.Next()
if act.Type != exp.Type {
t.Errorf("#%d %q: wrong %d-th item type: exp: %q; act: %q\n\t(%s vs %s)", id, test.pattern, i, exp.Type, act.Type, exp, act)
}
if act.Raw != exp.Raw {
t.Errorf("#%d %q: wrong %d-th item contents: exp: %q; act: %q\n\t(%s vs %s)", id, test.pattern, i, exp.Raw, act.Raw, exp, act)
}
}
}
}

88
syntax/lexer/token.go Normal file
View File

@ -0,0 +1,88 @@
package lexer
import "fmt"
type TokenType int
const (
EOF TokenType = iota
Error
Text
Char
Any
Super
Single
Not
Separator
RangeOpen
RangeClose
RangeLo
RangeHi
RangeBetween
TermsOpen
TermsClose
)
func (tt TokenType) String() string {
switch tt {
case EOF:
return "eof"
case Error:
return "error"
case Text:
return "text"
case Char:
return "char"
case Any:
return "any"
case Super:
return "super"
case Single:
return "single"
case Not:
return "not"
case Separator:
return "separator"
case RangeOpen:
return "range_open"
case RangeClose:
return "range_close"
case RangeLo:
return "range_lo"
case RangeHi:
return "range_hi"
case RangeBetween:
return "range_between"
case TermsOpen:
return "terms_open"
case TermsClose:
return "terms_close"
default:
return "undef"
}
}
type Token struct {
Type TokenType
Raw string
}
func (t Token) String() string {
return fmt.Sprintf("%v<%q>", t.Type, t.Raw)
}

14
syntax/syntax.go Normal file
View File

@ -0,0 +1,14 @@
package syntax
import (
"github.com/gobwas/glob/syntax/ast"
"github.com/gobwas/glob/syntax/lexer"
)
func Parse(s string) (*ast.Node, error) {
return ast.Parse(lexer.NewLexer(s))
}
func Special(b byte) bool {
return lexer.Special(b)
}

0
runes/runes.go → util/runes/runes.go
View File

0
runes/runes_test.go → util/runes/runes_test.go
View File

0
strings/strings.go → util/strings/strings.go
View File