This commit is contained in:
Mark Bates 2019-09-23 09:53:30 -04:00
parent 9bd3853a21
commit 18375ac12e
8 changed files with 1297 additions and 394 deletions

View File

@ -2,6 +2,9 @@ module github.com/markbates/pkger/examples/app
go 1.13 go 1.13
require github.com/markbates/pkger v0.0.0 require (
github.com/gobuffalo/buffalo v0.14.10
github.com/markbates/pkger v0.0.0
)
replace github.com/markbates/pkger => ../../ replace github.com/markbates/pkger => ../../

File diff suppressed because it is too large Load Diff

View File

@ -6,6 +6,7 @@ import (
"net/http" "net/http"
"os" "os"
_ "github.com/gobuffalo/buffalo"
"github.com/markbates/pkger" "github.com/markbates/pkger"
) )
@ -24,7 +25,7 @@ func main() {
fmt.Printf("Walking files for %s\n", current.ImportPath) fmt.Printf("Walking files for %s\n", current.ImportPath)
// walk the files in this module. "/" is where the `go.mod` for this module is // walk the files in this module. "/" is where the `go.mod` for this module is
err = pkger.Walk("github.com/markbates/pkger/examples/app:/", func(path string, info os.FileInfo, err error) error { err = pkger.Walk("/", func(path string, info os.FileInfo, err error) error {
if err != nil { if err != nil {
return err return err
} }

1
go.mod
View File

@ -4,5 +4,6 @@ go 1.13
require ( require (
github.com/markbates/errx v1.1.0 github.com/markbates/errx v1.1.0
github.com/markbates/oncer v1.0.0
github.com/stretchr/testify v1.4.0 github.com/stretchr/testify v1.4.0
) )

5
go.sum
View File

@ -1,10 +1,15 @@
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/markbates/errx v1.1.0 h1:QDFeR+UP95dO12JgW+tgi2UVfo0V8YBHiUIOaeBPiEI= github.com/markbates/errx v1.1.0 h1:QDFeR+UP95dO12JgW+tgi2UVfo0V8YBHiUIOaeBPiEI=
github.com/markbates/errx v1.1.0/go.mod h1:PLa46Oex9KNbVDZhKel8v1OT7hD5JZ2eI7AHhA0wswc= github.com/markbates/errx v1.1.0/go.mod h1:PLa46Oex9KNbVDZhKel8v1OT7hD5JZ2eI7AHhA0wswc=
github.com/markbates/oncer v1.0.0 h1:E83IaVAHygyndzPimgUYJjbshhDTALZyXxvk9FOlQRY=
github.com/markbates/oncer v1.0.0/go.mod h1:Z59JA581E9GP6w96jai+TGqafHPW+cPfRxz2aSZ0mcI=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=

View File

@ -2,12 +2,11 @@ package parser
import ( import (
"fmt" "fmt"
"go/parser"
"go/token"
"os" "os"
"path/filepath"
"sort"
"strings"
"github.com/markbates/pkger" "github.com/markbates/oncer"
"github.com/markbates/pkger/here" "github.com/markbates/pkger/here"
"github.com/markbates/pkger/pkging" "github.com/markbates/pkger/pkging"
) )
@ -16,165 +15,117 @@ var DefaultIgnoredFolders = []string{".", "_", "vendor", "node_modules", "_fixtu
func Parse(her here.Info) (Results, error) { func Parse(her here.Info) (Results, error) {
var r Results var r Results
var err error
name := her.ImportPath oncer.Do(her.ImportPath, func() {
pwd, err := os.Getwd()
pt, err := pkger.Parse(name)
if err != nil {
return r, err
}
r.Path = pt
m := map[pkging.Path]bool{}
root := r.Path.Name
if !strings.HasPrefix(root, string(filepath.Separator)) {
root = string(filepath.Separator) + root
}
if !strings.HasPrefix(root, her.Dir) {
root = filepath.Join(her.Dir, root)
}
err = filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
if err != nil { if err != nil {
return err return
} }
defer os.Chdir(pwd)
base := filepath.Base(path) fmt.Println("cd: ", her.Dir, her.ImportPath)
os.Chdir(her.Dir)
for _, ig := range DefaultIgnoredFolders { // 1. search for .go files in/imported by `her.ImportPath`
if strings.HasPrefix(base, ig) { src, err := fromSource(her)
if info.IsDir() {
return filepath.SkipDir
}
return nil
}
}
if info.IsDir() {
if _, err := os.Stat(filepath.Join(path, "go.mod")); err == nil {
her, err = here.Dir(path)
if err != nil {
return err
}
}
n := fmt.Sprintf("%s:%s", her.ImportPath, strings.TrimPrefix(path, her.Dir))
pt, err := pkger.Parse(n)
if err != nil {
return err
}
m[pt] = true
return nil
}
ext := filepath.Ext(path)
if ext != ".go" {
return nil
}
v, err := newVisitor(path, her)
if err != nil { if err != nil {
return err return
} }
fmt.Println(">>>TODO parser/parser.go:30: src ", src)
found, err := v.Run() // 2. parse .go ast's for `pkger.*` calls
if err != nil { // 3. find path's in those files
return err // 4. walk folders in those paths and add to results
}
for _, p := range found {
if p.Pkg == "." {
p.Pkg = her.ImportPath
}
if _, ok := m[p]; ok {
continue
}
m[p] = true
found, err := sourceFiles(p)
if err != nil {
return err
}
for _, pf := range found {
pf.Pkg = p.Pkg
m[pf] = true
}
}
return nil
}) })
var found []pkging.Path
for k := range m {
if len(k.String()) == 0 {
continue
}
found = append(found, k)
}
sort.Slice(found, func(a, b int) bool {
return found[a].String() <= found[b].String()
})
r.Paths = found
return r, err return r, err
} }
func sourceFiles(pt pkging.Path) ([]pkging.Path, error) { func fromSource(her here.Info) ([]pkging.Path, error) {
var res []pkging.Path fmt.Println(">>>TODO parser/parser.go:201: her.ImportPath ", her.ImportPath)
fmt.Println(her)
her, err := pkger.Info(pt.Pkg) root := her.Dir
fi, err := os.Stat(root)
if err != nil { if err != nil {
return res, err return nil, err
}
fp := her.FilePath(pt.Name)
fi, err := os.Stat(fp)
if err != nil {
return res, err
} }
if !fi.IsDir() { if !fi.IsDir() {
return res, nil return nil, fmt.Errorf("%q is not a directory", root)
} }
err = filepath.Walk(fp, func(p string, info os.FileInfo, err error) error { fset := token.NewFileSet()
if err != nil {
return err
}
base := filepath.Base(p) pkgs, err := parser.ParseDir(fset, root, nil, 0)
if err != nil {
return nil, err
}
if base == "." { var paths []pkging.Path
return nil for _, pkg := range pkgs {
} for _, pf := range pkg.Files {
f := &file{fset: fset, astFile: pf, filename: pf.Name.Name}
f.decls = make(map[string]string)
for _, ig := range DefaultIgnoredFolders { x, err := f.find()
if strings.HasPrefix(base, ig) { if err != nil {
if info.IsDir() { return nil, err
return filepath.SkipDir }
for i, pt := range x {
if pt.Pkg == "/" || pt.Pkg == "" {
pt.Pkg = her.ImportPath
x[i] = pt
} }
return nil paths = append(paths, x[i])
} }
} }
}
if info.IsDir() { for _, i := range her.Imports {
return nil fmt.Println(">>>TODO parser/parser.go:237: i ", i)
} }
n := strings.TrimPrefix(p, her.Dir)
n = strings.Replace(n, "\\", "/", -1)
pt := pkging.Path{
Name: n,
}
res = append(res, pt)
return nil
})
return res, err
return paths, nil
} }
// func importName(pkg *ast.File) (string, error) {
// var v visitor
// var name string
// var err error
// v = func(node ast.Node) ast.Visitor {
// if node == nil {
// return v
// }
// switch t := node.(type) {
// case *ast.ImportSpec:
// s, err := strconv.Unquote(t.Path.Value)
// if err != nil {
// err = err
// return nil
// }
// if s != "github.com/markbates/pkger" {
// if t.Name == nil {
// name = "pkger"
// return v
// }
// }
// default:
// // fmt.Printf("%#v\n", node)
// }
// return v
// }
// ast.Walk(v, pkg)
//
// if err != nil {
// return "", err
// }
//
// if len(name) == 0 {
// return "", io.EOF
// }
// return name, nil
// }
type Results struct { type Results struct {
Paths []pkging.Path Paths []pkging.Path
Path pkging.Path Path pkging.Path

View File

@ -1,7 +1,7 @@
package parser package parser
import ( import (
"fmt" "os"
"path/filepath" "path/filepath"
"sort" "sort"
"testing" "testing"
@ -13,13 +13,17 @@ import (
func Test_Parser(t *testing.T) { func Test_Parser(t *testing.T) {
r := require.New(t) r := require.New(t)
ch := filepath.Join("..", pwd, err := os.Getwd()
r.NoError(err)
ch := filepath.Join(pwd, "..",
"examples", "examples",
"app") "app")
info := here.Info{ info := here.Info{
Dir: ch, Dir: ch,
ImportPath: "github.com/markbates/pkger/examples/app", ImportPath: "github.com/markbates/pkger/examples/app",
} }
res, err := Parse(info) res, err := Parse(info)
r.NoError(err) r.NoError(err)
@ -41,6 +45,6 @@ func Test_Parser(t *testing.T) {
} }
sort.Strings(act) sort.Strings(act)
fmt.Printf("%#v\n", act) // fmt.Printf("%#v\n", act)
r.Equal(exp, act) r.Equal(exp, act)
} }

View File

@ -3,298 +3,248 @@ package parser
import ( import (
"fmt" "fmt"
"go/ast" "go/ast"
"go/token"
"strconv" "strconv"
"strings"
"github.com/markbates/pkger" "github.com/markbates/pkger"
"github.com/markbates/pkger/here" "github.com/markbates/pkger/here"
"github.com/markbates/pkger/pkging" "github.com/markbates/pkger/pkging"
) )
type visitor struct { type visitor func(node ast.Node) (w ast.Visitor)
File string
Found map[pkging.Path]bool func (v visitor) Visit(node ast.Node) ast.Visitor {
info here.Info return v(node)
errors []error
} }
func newVisitor(p string, info here.Info) (*visitor, error) { // inspired by https://gist.github.com/cryptix/d1b129361cea51a59af2
return &visitor{ type file struct {
File: p, fset *token.FileSet
Found: map[pkging.Path]bool{}, astFile *ast.File
info: info, filename string
}, nil decls map[string]string
paths []pkging.Path
} }
func (v *visitor) Run() ([]pkging.Path, error) { func (f *file) walk(fn func(ast.Node) bool) {
pf, err := parseFile(v.File) ast.Walk(walker(fn), f.astFile)
if err != nil {
return nil, fmt.Errorf("%s: %v", v.File, err)
}
ast.Walk(v, pf.Ast)
var found []pkging.Path
for k := range v.Found {
found = append(found, k)
}
return found, nil
} }
func (v *visitor) addPath(p string) error { func (f *file) find() ([]pkging.Path, error) {
p, _ = strconv.Unquote(p) if err := f.findDecals(); err != nil {
pt, err := pkger.Parse(p) return nil, err
if err != nil {
return err
} }
if strings.HasPrefix(p, ":") { if err := f.findOpenCalls(); err != nil {
pt.Pkg = v.info.ImportPath return nil, err
} }
v.Found[pt] = true if err := f.findWalkCalls(); err != nil {
return nil, err
}
return nil if err := f.findImportCalls(); err != nil {
return nil, err
}
return f.paths, nil
} }
func (v *visitor) Visit(node ast.Node) ast.Visitor { func (f *file) findDecals() error {
if node == nil { // iterate over all declarations
return v for _, d := range f.astFile.Decls {
}
if err := v.eval(node); err != nil {
v.errors = append(v.errors, err)
}
return v // log.Printf("#%d Decl: %+v\n", i, d)
}
func (v *visitor) eval(node ast.Node) error { // only interested in generic declarations
switch t := node.(type) { if genDecl, ok := d.(*ast.GenDecl); ok {
case *ast.CallExpr:
return v.evalExpr(t)
case *ast.Ident:
return v.evalIdent(t)
case *ast.GenDecl:
for _, n := range t.Specs {
if err := v.eval(n); err != nil {
return err
}
}
case *ast.FuncDecl:
if t.Body == nil {
return nil
}
for _, b := range t.Body.List {
if err := v.evalStmt(b); err != nil {
return err
}
}
return nil
case *ast.ValueSpec:
for _, e := range t.Values {
if err := v.evalExpr(e); err != nil {
return err
}
}
}
return nil
}
func (v *visitor) evalStmt(stmt ast.Stmt) error { // handle const's and vars
switch t := stmt.(type) { if genDecl.Tok == token.CONST || genDecl.Tok == token.VAR {
case *ast.ExprStmt:
return v.evalExpr(t.X)
case *ast.AssignStmt:
for _, e := range t.Rhs {
if err := v.evalArgs(e); err != nil {
return err
}
}
}
return nil
}
func (v *visitor) evalExpr(expr ast.Expr) error { // there may be multiple
switch t := expr.(type) { // i.e. const ( ... )
case *ast.CallExpr: for _, cDecl := range genDecl.Specs {
if t.Fun == nil {
return nil
}
for _, a := range t.Args {
switch at := a.(type) {
case *ast.CallExpr:
if sel, ok := t.Fun.(*ast.SelectorExpr); ok {
return v.evalSelector(at, sel)
}
if err := v.evalArgs(at); err != nil { // havn't find another kind of spec then value but better check
return err if vSpec, ok := cDecl.(*ast.ValueSpec); ok {
} // log.Printf("const ValueSpec: %+v\n", vSpec)
case *ast.CompositeLit:
for _, e := range at.Elts {
if err := v.evalExpr(e); err != nil {
return err
}
}
}
}
if ft, ok := t.Fun.(*ast.SelectorExpr); ok {
return v.evalSelector(t, ft)
}
case *ast.KeyValueExpr:
return v.evalExpr(t.Value)
}
return nil
}
func (v *visitor) evalArgs(expr ast.Expr) error { // iterate over Name/Value pair
switch at := expr.(type) { for i := 0; i < len(vSpec.Names); i++ {
case *ast.CompositeLit: // TODO: only basic literals work currently
for _, e := range at.Elts { if i > len(vSpec.Values) || len(vSpec.Values) == 0 {
if err := v.evalExpr(e); err != nil { break
return err }
} switch v := vSpec.Values[i].(type) {
} case *ast.BasicLit:
case *ast.CallExpr: f.decls[vSpec.Names[i].Name] = v.Value
if at.Fun == nil { default:
return nil // log.Printf("Name: %s - Unsupported ValueSpec: %+v\n", vSpec.Names[i].Name, v)
} }
switch st := at.Fun.(type) {
case *ast.SelectorExpr:
if err := v.evalSelector(at, st); err != nil {
return err
}
case *ast.Ident:
return v.evalIdent(st)
}
for _, a := range at.Args {
if err := v.evalArgs(a); err != nil {
return err
}
}
}
return nil
}
func (v *visitor) evalSelector(expr *ast.CallExpr, sel *ast.SelectorExpr) error {
x, ok := sel.X.(*ast.Ident)
if !ok {
return nil
}
if x.Name == "pkger" {
switch sel.Sel.Name {
case "Walk":
if len(expr.Args) != 2 {
return fmt.Errorf("`Walk` requires two arguments")
}
zz := func(e ast.Expr) (string, error) {
switch at := e.(type) {
case *ast.Ident:
switch at.Obj.Kind {
case ast.Var:
if as, ok := at.Obj.Decl.(*ast.AssignStmt); ok {
return v.fromVariable(as)
}
case ast.Con:
if vs, ok := at.Obj.Decl.(*ast.ValueSpec); ok {
return v.fromConstant(vs)
} }
} }
return "", v.evalIdent(at)
case *ast.BasicLit:
return at.Value, nil
case *ast.CallExpr:
return "", v.evalExpr(at)
} }
return "", fmt.Errorf("can't handle %T", e)
} }
k1, err := zz(expr.Args[0]) }
}
return nil
}
func (f *file) findOpenCalls() error {
var err error
f.walk(func(node ast.Node) bool {
ce, ok := node.(*ast.CallExpr)
if !ok {
return true
}
exists := isPkgDot(ce.Fun, "pkger", "Open")
if !(exists) || len(ce.Args) != 1 {
return true
}
switch x := ce.Args[0].(type) {
case *ast.BasicLit:
s, err := strconv.Unquote(x.Value)
if err != nil { if err != nil {
return err err = nil
return false
} }
if err := v.addPath(k1); err != nil { pt, err := pkger.Parse(s)
return err if err != nil {
err = err
return false
} }
f.paths = append(f.paths, pt)
case *ast.Ident:
val, ok := f.decls[x.Name]
if !ok {
//TODO: Add ERRORs list to file type and return after iteration!
// log.Printf("Could not find identifier[%s] in decls map\n", x.Name)
return true
}
s, err := strconv.Unquote(val)
if err != nil {
err = nil
return false
}
pt, err := pkger.Parse(s)
if err != nil {
err = err
return false
}
f.paths = append(f.paths, pt)
return nil default:
case "Open":
for _, e := range expr.Args {
switch at := e.(type) {
case *ast.Ident:
switch at.Obj.Kind {
case ast.Var:
if as, ok := at.Obj.Decl.(*ast.AssignStmt); ok {
v.addVariable("", as)
}
case ast.Con:
if vs, ok := at.Obj.Decl.(*ast.ValueSpec); ok {
v.addConstant("", vs)
}
}
return v.evalIdent(at)
case *ast.BasicLit:
return v.addPath(at.Value)
case *ast.CallExpr:
return v.evalExpr(at)
}
}
} }
}
return nil return true
})
return err
} }
func (v *visitor) evalIdent(i *ast.Ident) error { func (f *file) findWalkCalls() error {
if i.Obj == nil { var err error
return nil f.walk(func(node ast.Node) bool {
} ce, ok := node.(*ast.CallExpr)
if s, ok := i.Obj.Decl.(*ast.AssignStmt); ok { if !ok {
return v.evalStmt(s) return true
} }
return nil
} exists := isPkgDot(ce.Fun, "pkger", "Walk")
if !(exists) || len(ce.Args) != 2 {
func (v *visitor) fromVariable(as *ast.AssignStmt) (string, error) { return true
if len(as.Rhs) == 1 { }
if bs, ok := as.Rhs[0].(*ast.BasicLit); ok {
return bs.Value, nil switch x := ce.Args[0].(type) {
}
} case *ast.BasicLit:
return "", fmt.Errorf("unable to find value from variable %v", as) s, err := strconv.Unquote(x.Value)
} if err != nil {
err = nil
func (v *visitor) addVariable(bn string, as *ast.AssignStmt) error { return false
bv, err := v.fromVariable(as) }
if err != nil { pt, err := pkger.Parse(s)
return nil if err != nil {
} err = err
if len(bn) == 0 { return false
bn = bv }
} f.paths = append(f.paths, pt)
return v.addPath(bn) case *ast.Ident:
} val, ok := f.decls[x.Name]
if !ok {
func (v *visitor) fromConstant(vs *ast.ValueSpec) (string, error) { //TODO: Add ERRORs list to file type and return after iteration!
if len(vs.Values) == 1 { // log.Printf("Could not find identifier[%s] in decls map\n", x.Name)
if bs, ok := vs.Values[0].(*ast.BasicLit); ok { return true
return bs.Value, nil }
} s, err := strconv.Unquote(val)
} if err != nil {
return "", fmt.Errorf("unable to find value from constant %v", vs) err = nil
} return false
}
func (v *visitor) addConstant(bn string, vs *ast.ValueSpec) error { pt, err := pkger.Parse(s)
if len(vs.Values) == 1 { if err != nil {
if bs, ok := vs.Values[0].(*ast.BasicLit); ok { err = err
bv := bs.Value return false
if len(bn) == 0 { }
bn = bv f.paths = append(f.paths, pt)
}
return v.addPath(bn) default:
} }
return true
})
return err
}
func (f *file) findImportCalls() error {
var err error
f.walk(func(node ast.Node) bool {
ce, ok := node.(*ast.ImportSpec)
if !ok {
return true
}
s, err := strconv.Unquote(ce.Path.Value)
if err != nil {
return false
}
info, err := here.Package(s)
if err != nil {
return false
}
fmt.Println(">>>TODO parser/visitor.go:216: info ", info)
res, err := Parse(info)
if err != nil {
return false
}
fmt.Println(">>>TODO parser/visitor.go:224: res ", res)
return true
})
return err
}
// helpers
// =======
func isPkgDot(expr ast.Expr, pkg, name string) bool {
sel, ok := expr.(*ast.SelectorExpr)
return ok && isIdent(sel.X, pkg) && isIdent(sel.Sel, name)
}
func isIdent(expr ast.Expr, ident string) bool {
id, ok := expr.(*ast.Ident)
return ok && id.Name == ident
}
// wrap a function to fulfill ast.Visitor interface
type walker func(ast.Node) bool
func (w walker) Visit(node ast.Node) ast.Visitor {
if w(node) {
return w
} }
return nil return nil
} }