Merge pull request #17 from markbates/declan-macmanus

skip ignoreable directories
This commit is contained in:
Mark Bates 2019-11-05 15:16:12 -05:00 committed by GitHub
commit 08fb31c710
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 392 additions and 462 deletions

View File

@ -13,6 +13,7 @@ steps:
mv !(gopath) "$(modulePath)"
displayName: "Setup Go Workspace"
- script: |
go get github.com/gobuffalo/buffalo
go get -t -v ./...
go test -race ./...
workingDirectory: "$(modulePath)"

View File

@ -1,6 +1,7 @@
package parser
import (
"fmt"
"go/token"
"sort"
)
@ -39,11 +40,12 @@ func (decls Decls) Files() ([]*File, error) {
files, err := fl.Files(v)
if err != nil {
return nil, err
return nil, fmt.Errorf("%w: %s", err, d)
}
for _, f := range files {
m[f.Abs] = f
v[f.Abs] = f.Abs
}
}

View File

@ -2,11 +2,6 @@ package parser
import (
"encoding/json"
"go/ast"
"go/parser"
"go/token"
"io"
"io/ioutil"
"github.com/markbates/pkger/here"
)
@ -21,36 +16,3 @@ func (f File) String() string {
b, _ := json.MarshalIndent(f, "", " ")
return string(b)
}
type parsedFile struct {
File string
FileSet *token.FileSet
Ast *ast.File
}
// parseFileMode ...
func parseFileMode(f string, mode parser.Mode) (parsedFile, error) {
pf := parsedFile{
File: f,
FileSet: token.NewFileSet(),
}
b, err := ioutil.ReadFile(f)
if err != nil {
return pf, err
}
src := string(b)
pff, err := parser.ParseFile(pf.FileSet, f, src, mode)
if err != nil && err != io.EOF {
return pf, err
}
pf.Ast = pff
return pf, nil
}
// parseFile ...
func parseFile(f string) (parsedFile, error) {
return parseFileMode(f, 0)
}

View File

@ -71,11 +71,7 @@ func (d OpenDecl) Files(virtual map[string]string) ([]*File, error) {
return wd.Files(virtual)
}
var files []*File
files = append(files, &File{
Abs: filepath.Join(her.Module.Dir, pt.Name),
Path: pt,
Here: her,
})
files = append(files, d.file)
return files, nil
}

View File

@ -4,68 +4,212 @@ import (
"fmt"
"go/parser"
"go/token"
"io"
"io/ioutil"
"os"
"path/filepath"
"strings"
"sync"
"github.com/markbates/pkger/here"
)
// var DefaultIgnoredFolders = []string{".", "_", "vendor", "node_modules", "_fixtures", "testdata"}
var defaultIgnoredFolders = []string{".", "_", "vendor", "node_modules", "testdata"}
func Parse(her here.Info) (Decls, error) {
src, err := fromSource(her)
p, err := New(her)
if err != nil {
return nil, err
}
return src, nil
return p.Decls()
}
func fromSource(her here.Info) (Decls, error) {
root := her.Dir
fi, err := os.Stat(root)
func ParseSource(source Source, mode parser.Mode) (*ParsedSource, error) {
pf := &ParsedSource{
Source: source,
FileSet: token.NewFileSet(),
}
b, err := ioutil.ReadFile(source.Abs)
if err != nil {
return nil, err
}
if !fi.IsDir() {
return nil, fmt.Errorf("%q is not a directory", root)
src := string(b)
pff, err := parser.ParseFile(pf.FileSet, source.Abs, src, mode)
if err != nil && err != io.EOF {
return nil, err
}
pf.Ast = pff
return pf, nil
}
func ParseFile(abs string, mode parser.Mode) (*ParsedSource, error) {
s := Source{
Abs: abs,
}
info, err := os.Stat(abs)
if err != nil {
return nil, err
}
if info.IsDir() {
return nil, fmt.Errorf("%s is a directory", abs)
}
dir := filepath.Dir(abs)
s.Here, err = here.Dir(dir)
if err != nil {
return nil, err
}
s.Path, err = s.Here.Parse(strings.TrimPrefix(abs, dir))
return ParseSource(s, 0)
}
func ParseDir(abs string, mode parser.Mode) ([]*ParsedSource, error) {
info, err := os.Stat(abs)
if err != nil {
return nil, err
}
if !info.IsDir() {
return nil, fmt.Errorf("%s is not a directory", abs)
}
dir := filepath.Dir(abs)
her, err := here.Dir(dir)
if err != nil {
return nil, err
}
pt, err := her.Parse(strings.TrimPrefix(abs, dir))
if err != nil {
return nil, err
}
fset := token.NewFileSet()
pkgs, err := parser.ParseDir(fset, abs, nil, 0)
if err != nil {
return nil, err
}
var srcs []*ParsedSource
for _, pkg := range pkgs {
for name, pf := range pkg.Files {
s := &ParsedSource{
Source: Source{
Abs: name,
Path: pt,
Here: her,
},
FileSet: fset,
Ast: pf,
}
srcs = append(srcs, s)
}
}
return srcs, nil
}
func New(her here.Info) (*Parser, error) {
return &Parser{
Info: her,
decls: map[string]Decls{},
}, nil
}
type Parser struct {
here.Info
decls map[string]Decls
once sync.Once
err error
}
func (p *Parser) Decls() (Decls, error) {
if err := p.parse(); err != nil {
return nil, err
}
var decls Decls
orderedNames := []string{
"MkdirAll",
"Create",
"Stat",
"Open",
"Dir",
"Walk",
}
for _, n := range orderedNames {
decls = append(decls, p.decls[n]...)
}
return decls, nil
}
func (p *Parser) DeclsMap() (map[string]Decls, error) {
err := p.Parse()
return p.decls, err
}
func (p *Parser) Parse() error {
(&p.once).Do(func() {
p.err = p.parse()
})
return p.err
}
func (p *Parser) parse() error {
p.decls = map[string]Decls{}
root := p.Dir
fi, err := os.Stat(root)
if err != nil {
return err
}
if !fi.IsDir() {
return fmt.Errorf("%q is not a directory", root)
}
err = filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
fset := token.NewFileSet()
if !info.IsDir() {
return nil
}
pkgs, err := parser.ParseDir(fset, path, nil, 0)
if err != nil {
return err
}
for _, pkg := range pkgs {
for name, pf := range pkg.Files {
f := &file{
fset: fset,
astFile: pf,
filename: name,
current: her,
}
x, err := f.find()
if err != nil {
return err
}
decls = append(decls, x...)
base := filepath.Base(path)
for _, x := range defaultIgnoredFolders {
if strings.HasPrefix(base, x) {
return filepath.SkipDir
}
}
srcs, err := ParseDir(path, 0)
if err != nil {
return fmt.Errorf("%w: %s", err, path)
}
for _, src := range srcs {
mm, err := src.DeclsMap()
if err != nil {
return fmt.Errorf("%w: %s", err, src.Abs)
}
for k, v := range mm {
p.decls[k] = append(p.decls[k], v...)
}
}
return nil
})
return decls, err
return err
}

View File

@ -1,19 +1,23 @@
package parser_test
package parser
import (
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
"github.com/markbates/pkger/here"
"github.com/markbates/pkger/parser"
"github.com/markbates/pkger/pkging/pkgtest"
"github.com/markbates/pkger/pkging/stdos"
"github.com/stretchr/testify/require"
)
func Test_Parser_Ref(t *testing.T) {
defer func() {
c := exec.Command("go", "mod", "tidy", "-v")
c.Run()
}()
r := require.New(t)
ref, err := pkgtest.NewRef()
@ -26,16 +30,20 @@ func Test_Parser_Ref(t *testing.T) {
_, err = pkgtest.LoadFiles("/", ref, disk)
r.NoError(err)
res, err := parser.Parse(ref.Info)
res, err := Parse(ref.Info)
r.NoError(err)
files, err := res.Files()
r.NoError(err)
r.Len(files, 22)
r.Len(files, 23)
for _, f := range files {
r.True(strings.HasPrefix(f.Abs, ref.Dir), "%q %q", f.Abs, ref.Dir)
if f.Path.Pkg == ref.Module.Path {
r.True(strings.HasPrefix(f.Abs, ref.Dir), "%q %q", f.Abs, ref.Dir)
} else {
r.False(strings.HasPrefix(f.Abs, ref.Dir), "%q %q", f.Abs, ref.Dir)
}
}
}
@ -51,11 +59,15 @@ func Test_Parser_Example_HTTP(t *testing.T) {
root := filepath.Join(cur.Dir, "examples", "http", "pkger")
r.NoError(os.Chdir(root))
defer func() {
c := exec.Command("go", "mod", "tidy", "-v")
c.Run()
}()
her, err := here.Dir(".")
r.NoError(err)
res, err := parser.Parse(her)
res, err := Parse(her)
r.NoError(err)
files, err := res.Files()

185
parser/source.go Normal file
View File

@ -0,0 +1,185 @@
package parser
import (
"fmt"
"go/ast"
"go/token"
"path/filepath"
"strconv"
"sync"
"github.com/markbates/pkger/here"
)
type Source struct {
Abs string // full path on disk to file
Path here.Path
Here here.Info
}
type ParsedSource struct {
Source
FileSet *token.FileSet
Ast *ast.File
decls map[string]Decls
once sync.Once
err error
}
func (p *ParsedSource) Parse() error {
(&p.once).Do(func() {
p.err = p.parse()
})
return p.err
}
func (p *ParsedSource) value(node ast.Node) (string, error) {
var s string
switch x := node.(type) {
case *ast.BasicLit:
s = x.Value
case *ast.Ident:
s = x.Name
}
return strconv.Unquote(s)
}
func (p *ParsedSource) parse() error {
p.decls = map[string]Decls{}
var fn walker = func(node ast.Node) bool {
ce, ok := node.(*ast.CallExpr)
if !ok {
return true
}
sel, ok := ce.Fun.(*ast.SelectorExpr)
if !ok {
return true
}
pkg, ok := sel.X.(*ast.Ident)
if !ok {
return true
}
if pkg.Name != "pkger" {
return true
}
var fn func(f File, pos token.Position, value string) Decl
name := sel.Sel.Name
switch name {
case "MkdirAll":
fn = func(f File, pos token.Position, value string) Decl {
return MkdirAllDecl{
file: &f,
pos: pos,
value: value,
}
}
case "Create":
fn = func(f File, pos token.Position, value string) Decl {
return CreateDecl{
file: &f,
pos: pos,
value: value,
}
}
case "Stat":
fn = func(f File, pos token.Position, value string) Decl {
return StatDecl{
file: &f,
pos: pos,
value: value,
}
}
case "Open":
fn = func(f File, pos token.Position, value string) Decl {
return OpenDecl{
file: &f,
pos: pos,
value: value,
}
}
case "Dir":
fn = func(f File, pos token.Position, value string) Decl {
return HTTPDecl{
file: &f,
pos: pos,
value: value,
}
}
case "Walk":
fn = func(f File, pos token.Position, value string) Decl {
return WalkDecl{
file: &f,
pos: pos,
value: value,
}
}
default:
return true
}
if len(ce.Args) < 1 {
p.err = fmt.Errorf("declarations require at least one argument")
return false
}
n := ce.Args[0]
val, err := p.value(n)
if err != nil {
p.err = fmt.Errorf("%w: %s", err, n)
return false
}
info, err := here.Dir(filepath.Dir(p.Abs))
if err != nil {
p.err = fmt.Errorf("%w: %s", err, p.Abs)
return false
}
pt, err := info.Parse(val)
if err != nil {
p.err = fmt.Errorf("%w: %s", err, p.Abs)
return false
}
if pt.Pkg != info.Module.Path {
info, err = here.Package(pt.Pkg)
if err != nil {
p.err = fmt.Errorf("%w: %s", err, p.Abs)
return false
}
}
f := File{
Abs: filepath.Join(info.Module.Dir, pt.Name),
Here: info,
Path: pt,
}
p.decls[name] = append(p.decls[name], fn(f, p.FileSet.Position(n.Pos()), val))
return true
}
ast.Walk(fn, p.Ast)
return nil
}
func (p *ParsedSource) DeclsMap() (map[string]Decls, error) {
err := p.Parse()
return p.decls, err
}
// wrap a function to fulfill ast.Visitor interface
type walker func(ast.Node) bool
func (w walker) Visit(node ast.Node) ast.Visitor {
if w(node) {
return w
}
return nil
}

View File

@ -2,6 +2,7 @@ package parser
import (
"encoding/json"
"fmt"
"go/token"
"os"
)
@ -15,8 +16,7 @@ type StatDecl struct {
}
func (d StatDecl) String() string {
b, _ := json.Marshal(d)
return string(b)
return fmt.Sprintf("pkger.Stat(%q)", d.value)
}
func (d StatDecl) MarshalJSON() ([]byte, error) {

View File

@ -1,374 +0,0 @@
package parser
import (
"go/ast"
"go/token"
"path/filepath"
"strconv"
"github.com/markbates/pkger/here"
)
type visitor func(node ast.Node) (w ast.Visitor)
func (v visitor) Visit(node ast.Node) ast.Visitor {
return v(node)
}
// inspired by https://gist.github.com/cryptix/d1b129361cea51a59af2
type file struct {
fset *token.FileSet
astFile *ast.File
filename string
decls Decls
current here.Info
}
func (f *file) walk(fn func(ast.Node) bool) {
ast.Walk(walker(fn), f.astFile)
}
func (f *file) find() (Decls, error) {
// --- virtual calls first ---
if err := f.findMkdirAllCalls(); err != nil {
return nil, err
}
if err := f.findCreateCalls(); err != nil {
return nil, err
}
// -- physical calls second ---
if err := f.findStatCalls(); err != nil {
return nil, err
}
if err := f.findOpenCalls(); err != nil {
return nil, err
}
if err := f.findHTTPCalls(); err != nil {
return nil, err
}
if err := f.findWalkCalls(); err != nil {
return nil, err
}
return f.decls, nil
}
func (f *file) asValue(node ast.Node) (string, error) {
var s string
switch x := node.(type) {
case *ast.BasicLit:
s = x.Value
case *ast.Ident:
s = x.Name
}
return strconv.Unquote(s)
}
func (f *file) findMkdirAllCalls() error {
var err error
f.walk(func(node ast.Node) bool {
ce, ok := node.(*ast.CallExpr)
if !ok {
return true
}
exists := isPkgDot(ce.Fun, "pkger", "MkdirAll")
if !(exists) || len(ce.Args) != 2 {
return true
}
n := ce.Args[0]
s, err := f.asValue(n)
if err != nil {
return false
}
info, err := here.Dir(filepath.Dir(f.filename))
if err != nil {
return false
}
pf := &File{
Abs: f.filename,
Here: info,
Path: here.Path{
Pkg: info.Module.Path,
Name: s,
},
}
decl := MkdirAllDecl{
file: pf,
pos: f.fset.Position(n.Pos()),
value: s,
}
f.decls = append(f.decls, decl)
return true
})
return err
}
func (f *file) findStatCalls() error {
var err error
f.walk(func(node ast.Node) bool {
ce, ok := node.(*ast.CallExpr)
if !ok {
return true
}
exists := isPkgDot(ce.Fun, "pkger", "Stat")
if !(exists) || len(ce.Args) != 1 {
return true
}
n := ce.Args[0]
s, err := f.asValue(n)
if err != nil {
return false
}
info, err := here.Dir(filepath.Dir(f.filename))
if err != nil {
return false
}
pf := &File{
Abs: f.filename,
Here: info,
Path: here.Path{
Pkg: info.Module.Path,
Name: s,
},
}
decl := StatDecl{
file: pf,
pos: f.fset.Position(n.Pos()),
value: s,
}
f.decls = append(f.decls, decl)
return true
})
return err
}
func (f *file) findCreateCalls() error {
var err error
f.walk(func(node ast.Node) bool {
ce, ok := node.(*ast.CallExpr)
if !ok {
return true
}
exists := isPkgDot(ce.Fun, "pkger", "Create")
if !(exists) || len(ce.Args) != 1 {
return true
}
n := ce.Args[0]
s, err := f.asValue(n)
if err != nil {
return false
}
info, err := here.Dir(filepath.Dir(f.filename))
if err != nil {
return false
}
pf := &File{
Abs: f.filename,
Here: info,
Path: here.Path{
Pkg: info.Module.Path,
Name: s,
},
}
decl := CreateDecl{
file: pf,
pos: f.fset.Position(n.Pos()),
value: s,
}
f.decls = append(f.decls, decl)
return true
})
return err
}
func (f *file) findOpenCalls() error {
var err error
f.walk(func(node ast.Node) bool {
ce, ok := node.(*ast.CallExpr)
if !ok {
return true
}
exists := isPkgDot(ce.Fun, "pkger", "Open")
if !(exists) || len(ce.Args) != 1 {
return true
}
n := ce.Args[0]
s, err := f.asValue(n)
if err != nil {
return false
}
info, err := here.Dir(filepath.Dir(f.filename))
if err != nil {
return false
}
pf := &File{
Abs: f.filename,
Here: info,
Path: here.Path{
Pkg: info.Module.Path,
Name: s,
},
}
decl := OpenDecl{
file: pf,
pos: f.fset.Position(n.Pos()),
value: s,
}
f.decls = append(f.decls, decl)
return true
})
return err
}
func (f *file) findWalkCalls() error {
var err error
f.walk(func(node ast.Node) bool {
ce, ok := node.(*ast.CallExpr)
if !ok {
return true
}
exists := isPkgDot(ce.Fun, "pkger", "Walk")
if !(exists) || len(ce.Args) != 2 {
return true
}
n := ce.Args[0]
s, err := f.asValue(n)
if err != nil {
err = err
return false
}
dir := filepath.Dir(f.filename)
info, err := here.Dir(dir)
if err != nil {
err = err
return false
}
pf := &File{
Abs: f.filename,
Here: info,
Path: here.Path{
Pkg: info.Module.Path,
Name: s,
},
}
decl := WalkDecl{
file: pf,
pos: f.fset.Position(n.Pos()),
value: s,
}
f.decls = append(f.decls, decl)
return true
})
return err
}
func (f *file) findHTTPCalls() error {
var err error
f.walk(func(node ast.Node) bool {
ce, ok := node.(*ast.CallExpr)
if !ok {
return true
}
exists := isPkgDot(ce.Fun, "pkger", "Dir")
if !(exists) || len(ce.Args) != 1 {
return true
}
n := ce.Args[0]
s, err := f.asValue(n)
if err != nil {
return false
}
info, err := here.Dir(filepath.Dir(f.filename))
if err != nil {
return false
}
pf := &File{
Abs: f.filename,
Here: info,
Path: here.Path{
Pkg: info.Module.Path,
Name: s,
},
}
pos := f.fset.Position(n.Pos())
decl := HTTPDecl{
file: pf,
pos: pos,
value: s,
}
f.decls = append(f.decls, decl)
return true
})
return err
}
// helpers
// =======
func isPkgDot(expr ast.Expr, pkg, name string) bool {
sel, ok := expr.(*ast.SelectorExpr)
return ok && isIdent(sel.X, pkg) && isIdent(sel.Sel, name)
}
func isIdent(expr ast.Expr, ident string) bool {
id, ok := expr.(*ast.Ident)
return ok && id.Name == ident
}
// wrap a function to fulfill ast.Visitor interface
type walker func(ast.Node) bool
func (w walker) Visit(node ast.Node) ast.Visitor {
if w(node) {
return w
}
return nil
}

View File

@ -1,10 +1,8 @@
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
@ -12,7 +10,6 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=

View File

@ -7,6 +7,7 @@ import (
)
func Serve() {
pkger.Stat("github.com/gobuffalo/buffalo:/logo.svg")
dir := http.FileServer(pkger.Dir("/public"))
http.ListenAndServe(":3000", dir)
}

View File

@ -29,16 +29,20 @@ func Test_Stuff(t *testing.T) {
decls, err := parser.Parse(ref.Info)
r.NoError(err)
r.Len(decls, 9)
r.Len(decls, 10)
files, err := decls.Files()
r.NoError(err)
for _, f := range files {
r.Equal("app", f.Path.Pkg)
if f.Path.Pkg == ref.Module.Path {
r.Equal("app", f.Path.Pkg)
} else {
r.NotEqual("app", f.Path.Pkg)
}
}
r.Len(files, 22)
r.Len(files, 23)
bb := &bytes.Buffer{}