This commit is contained in:
Mark Bates 2019-09-23 09:53:30 -04:00
parent 9bd3853a21
commit 18375ac12e
8 changed files with 1297 additions and 394 deletions

View File

@ -2,6 +2,9 @@ module github.com/markbates/pkger/examples/app
go 1.13
require github.com/markbates/pkger v0.0.0
require (
github.com/gobuffalo/buffalo v0.14.10
github.com/markbates/pkger v0.0.0
)
replace github.com/markbates/pkger => ../../

File diff suppressed because it is too large Load Diff

View File

@ -6,6 +6,7 @@ import (
"net/http"
"os"
_ "github.com/gobuffalo/buffalo"
"github.com/markbates/pkger"
)
@ -24,7 +25,7 @@ func main() {
fmt.Printf("Walking files for %s\n", current.ImportPath)
// walk the files in this module. "/" is where the `go.mod` for this module is
err = pkger.Walk("github.com/markbates/pkger/examples/app:/", func(path string, info os.FileInfo, err error) error {
err = pkger.Walk("/", func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}

1
go.mod
View File

@ -4,5 +4,6 @@ go 1.13
require (
github.com/markbates/errx v1.1.0
github.com/markbates/oncer v1.0.0
github.com/stretchr/testify v1.4.0
)

5
go.sum
View File

@ -1,10 +1,15 @@
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/markbates/errx v1.1.0 h1:QDFeR+UP95dO12JgW+tgi2UVfo0V8YBHiUIOaeBPiEI=
github.com/markbates/errx v1.1.0/go.mod h1:PLa46Oex9KNbVDZhKel8v1OT7hD5JZ2eI7AHhA0wswc=
github.com/markbates/oncer v1.0.0 h1:E83IaVAHygyndzPimgUYJjbshhDTALZyXxvk9FOlQRY=
github.com/markbates/oncer v1.0.0/go.mod h1:Z59JA581E9GP6w96jai+TGqafHPW+cPfRxz2aSZ0mcI=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=

View File

@ -2,12 +2,11 @@ package parser
import (
"fmt"
"go/parser"
"go/token"
"os"
"path/filepath"
"sort"
"strings"
"github.com/markbates/pkger"
"github.com/markbates/oncer"
"github.com/markbates/pkger/here"
"github.com/markbates/pkger/pkging"
)
@ -16,165 +15,117 @@ var DefaultIgnoredFolders = []string{".", "_", "vendor", "node_modules", "_fixtu
func Parse(her here.Info) (Results, error) {
var r Results
var err error
name := her.ImportPath
pt, err := pkger.Parse(name)
if err != nil {
return r, err
}
r.Path = pt
m := map[pkging.Path]bool{}
root := r.Path.Name
if !strings.HasPrefix(root, string(filepath.Separator)) {
root = string(filepath.Separator) + root
}
if !strings.HasPrefix(root, her.Dir) {
root = filepath.Join(her.Dir, root)
}
err = filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
oncer.Do(her.ImportPath, func() {
pwd, err := os.Getwd()
if err != nil {
return err
return
}
defer os.Chdir(pwd)
base := filepath.Base(path)
fmt.Println("cd: ", her.Dir, her.ImportPath)
os.Chdir(her.Dir)
for _, ig := range DefaultIgnoredFolders {
if strings.HasPrefix(base, ig) {
if info.IsDir() {
return filepath.SkipDir
}
return nil
}
}
if info.IsDir() {
if _, err := os.Stat(filepath.Join(path, "go.mod")); err == nil {
her, err = here.Dir(path)
if err != nil {
return err
}
}
n := fmt.Sprintf("%s:%s", her.ImportPath, strings.TrimPrefix(path, her.Dir))
pt, err := pkger.Parse(n)
if err != nil {
return err
}
m[pt] = true
return nil
}
ext := filepath.Ext(path)
if ext != ".go" {
return nil
}
v, err := newVisitor(path, her)
// 1. search for .go files in/imported by `her.ImportPath`
src, err := fromSource(her)
if err != nil {
return err
return
}
fmt.Println(">>>TODO parser/parser.go:30: src ", src)
found, err := v.Run()
if err != nil {
return err
}
for _, p := range found {
if p.Pkg == "." {
p.Pkg = her.ImportPath
}
if _, ok := m[p]; ok {
continue
}
m[p] = true
found, err := sourceFiles(p)
if err != nil {
return err
}
for _, pf := range found {
pf.Pkg = p.Pkg
m[pf] = true
}
}
return nil
// 2. parse .go ast's for `pkger.*` calls
// 3. find path's in those files
// 4. walk folders in those paths and add to results
})
var found []pkging.Path
for k := range m {
if len(k.String()) == 0 {
continue
}
found = append(found, k)
}
sort.Slice(found, func(a, b int) bool {
return found[a].String() <= found[b].String()
})
r.Paths = found
return r, err
}
func sourceFiles(pt pkging.Path) ([]pkging.Path, error) {
var res []pkging.Path
her, err := pkger.Info(pt.Pkg)
func fromSource(her here.Info) ([]pkging.Path, error) {
fmt.Println(">>>TODO parser/parser.go:201: her.ImportPath ", her.ImportPath)
fmt.Println(her)
root := her.Dir
fi, err := os.Stat(root)
if err != nil {
return res, err
}
fp := her.FilePath(pt.Name)
fi, err := os.Stat(fp)
if err != nil {
return res, err
return nil, err
}
if !fi.IsDir() {
return res, nil
return nil, fmt.Errorf("%q is not a directory", root)
}
err = filepath.Walk(fp, func(p string, info os.FileInfo, err error) error {
if err != nil {
return err
}
fset := token.NewFileSet()
base := filepath.Base(p)
pkgs, err := parser.ParseDir(fset, root, nil, 0)
if err != nil {
return nil, err
}
if base == "." {
return nil
}
var paths []pkging.Path
for _, pkg := range pkgs {
for _, pf := range pkg.Files {
f := &file{fset: fset, astFile: pf, filename: pf.Name.Name}
f.decls = make(map[string]string)
for _, ig := range DefaultIgnoredFolders {
if strings.HasPrefix(base, ig) {
if info.IsDir() {
return filepath.SkipDir
x, err := f.find()
if err != nil {
return nil, err
}
for i, pt := range x {
if pt.Pkg == "/" || pt.Pkg == "" {
pt.Pkg = her.ImportPath
x[i] = pt
}
return nil
paths = append(paths, x[i])
}
}
}
if info.IsDir() {
return nil
}
n := strings.TrimPrefix(p, her.Dir)
n = strings.Replace(n, "\\", "/", -1)
pt := pkging.Path{
Name: n,
}
res = append(res, pt)
return nil
})
return res, err
for _, i := range her.Imports {
fmt.Println(">>>TODO parser/parser.go:237: i ", i)
}
return paths, nil
}
// func importName(pkg *ast.File) (string, error) {
// var v visitor
// var name string
// var err error
// v = func(node ast.Node) ast.Visitor {
// if node == nil {
// return v
// }
// switch t := node.(type) {
// case *ast.ImportSpec:
// s, err := strconv.Unquote(t.Path.Value)
// if err != nil {
// err = err
// return nil
// }
// if s != "github.com/markbates/pkger" {
// if t.Name == nil {
// name = "pkger"
// return v
// }
// }
// default:
// // fmt.Printf("%#v\n", node)
// }
// return v
// }
// ast.Walk(v, pkg)
//
// if err != nil {
// return "", err
// }
//
// if len(name) == 0 {
// return "", io.EOF
// }
// return name, nil
// }
type Results struct {
Paths []pkging.Path
Path pkging.Path

View File

@ -1,7 +1,7 @@
package parser
import (
"fmt"
"os"
"path/filepath"
"sort"
"testing"
@ -13,13 +13,17 @@ import (
func Test_Parser(t *testing.T) {
r := require.New(t)
ch := filepath.Join("..",
pwd, err := os.Getwd()
r.NoError(err)
ch := filepath.Join(pwd, "..",
"examples",
"app")
info := here.Info{
Dir: ch,
ImportPath: "github.com/markbates/pkger/examples/app",
}
res, err := Parse(info)
r.NoError(err)
@ -41,6 +45,6 @@ func Test_Parser(t *testing.T) {
}
sort.Strings(act)
fmt.Printf("%#v\n", act)
// fmt.Printf("%#v\n", act)
r.Equal(exp, act)
}

View File

@ -3,298 +3,248 @@ package parser
import (
"fmt"
"go/ast"
"go/token"
"strconv"
"strings"
"github.com/markbates/pkger"
"github.com/markbates/pkger/here"
"github.com/markbates/pkger/pkging"
)
type visitor struct {
File string
Found map[pkging.Path]bool
info here.Info
errors []error
type visitor func(node ast.Node) (w ast.Visitor)
func (v visitor) Visit(node ast.Node) ast.Visitor {
return v(node)
}
func newVisitor(p string, info here.Info) (*visitor, error) {
return &visitor{
File: p,
Found: map[pkging.Path]bool{},
info: info,
}, nil
// inspired by https://gist.github.com/cryptix/d1b129361cea51a59af2
type file struct {
fset *token.FileSet
astFile *ast.File
filename string
decls map[string]string
paths []pkging.Path
}
func (v *visitor) Run() ([]pkging.Path, error) {
pf, err := parseFile(v.File)
if err != nil {
return nil, fmt.Errorf("%s: %v", v.File, err)
}
ast.Walk(v, pf.Ast)
var found []pkging.Path
for k := range v.Found {
found = append(found, k)
}
return found, nil
func (f *file) walk(fn func(ast.Node) bool) {
ast.Walk(walker(fn), f.astFile)
}
func (v *visitor) addPath(p string) error {
p, _ = strconv.Unquote(p)
pt, err := pkger.Parse(p)
if err != nil {
return err
func (f *file) find() ([]pkging.Path, error) {
if err := f.findDecals(); err != nil {
return nil, err
}
if strings.HasPrefix(p, ":") {
pt.Pkg = v.info.ImportPath
if err := f.findOpenCalls(); err != nil {
return nil, err
}
v.Found[pt] = true
if err := f.findWalkCalls(); err != nil {
return nil, err
}
return nil
if err := f.findImportCalls(); err != nil {
return nil, err
}
return f.paths, nil
}
func (v *visitor) Visit(node ast.Node) ast.Visitor {
if node == nil {
return v
}
if err := v.eval(node); err != nil {
v.errors = append(v.errors, err)
}
func (f *file) findDecals() error {
// iterate over all declarations
for _, d := range f.astFile.Decls {
return v
}
// log.Printf("#%d Decl: %+v\n", i, d)
func (v *visitor) eval(node ast.Node) error {
switch t := node.(type) {
case *ast.CallExpr:
return v.evalExpr(t)
case *ast.Ident:
return v.evalIdent(t)
case *ast.GenDecl:
for _, n := range t.Specs {
if err := v.eval(n); err != nil {
return err
}
}
case *ast.FuncDecl:
if t.Body == nil {
return nil
}
for _, b := range t.Body.List {
if err := v.evalStmt(b); err != nil {
return err
}
}
return nil
case *ast.ValueSpec:
for _, e := range t.Values {
if err := v.evalExpr(e); err != nil {
return err
}
}
}
return nil
}
// only interested in generic declarations
if genDecl, ok := d.(*ast.GenDecl); ok {
func (v *visitor) evalStmt(stmt ast.Stmt) error {
switch t := stmt.(type) {
case *ast.ExprStmt:
return v.evalExpr(t.X)
case *ast.AssignStmt:
for _, e := range t.Rhs {
if err := v.evalArgs(e); err != nil {
return err
}
}
}
return nil
}
// handle const's and vars
if genDecl.Tok == token.CONST || genDecl.Tok == token.VAR {
func (v *visitor) evalExpr(expr ast.Expr) error {
switch t := expr.(type) {
case *ast.CallExpr:
if t.Fun == nil {
return nil
}
for _, a := range t.Args {
switch at := a.(type) {
case *ast.CallExpr:
if sel, ok := t.Fun.(*ast.SelectorExpr); ok {
return v.evalSelector(at, sel)
}
// there may be multiple
// i.e. const ( ... )
for _, cDecl := range genDecl.Specs {
if err := v.evalArgs(at); err != nil {
return err
}
case *ast.CompositeLit:
for _, e := range at.Elts {
if err := v.evalExpr(e); err != nil {
return err
}
}
}
}
if ft, ok := t.Fun.(*ast.SelectorExpr); ok {
return v.evalSelector(t, ft)
}
case *ast.KeyValueExpr:
return v.evalExpr(t.Value)
}
return nil
}
// havn't find another kind of spec then value but better check
if vSpec, ok := cDecl.(*ast.ValueSpec); ok {
// log.Printf("const ValueSpec: %+v\n", vSpec)
func (v *visitor) evalArgs(expr ast.Expr) error {
switch at := expr.(type) {
case *ast.CompositeLit:
for _, e := range at.Elts {
if err := v.evalExpr(e); err != nil {
return err
}
}
case *ast.CallExpr:
if at.Fun == nil {
return nil
}
switch st := at.Fun.(type) {
case *ast.SelectorExpr:
if err := v.evalSelector(at, st); err != nil {
return err
}
case *ast.Ident:
return v.evalIdent(st)
}
for _, a := range at.Args {
if err := v.evalArgs(a); err != nil {
return err
}
}
}
return nil
}
func (v *visitor) evalSelector(expr *ast.CallExpr, sel *ast.SelectorExpr) error {
x, ok := sel.X.(*ast.Ident)
if !ok {
return nil
}
if x.Name == "pkger" {
switch sel.Sel.Name {
case "Walk":
if len(expr.Args) != 2 {
return fmt.Errorf("`Walk` requires two arguments")
}
zz := func(e ast.Expr) (string, error) {
switch at := e.(type) {
case *ast.Ident:
switch at.Obj.Kind {
case ast.Var:
if as, ok := at.Obj.Decl.(*ast.AssignStmt); ok {
return v.fromVariable(as)
}
case ast.Con:
if vs, ok := at.Obj.Decl.(*ast.ValueSpec); ok {
return v.fromConstant(vs)
// iterate over Name/Value pair
for i := 0; i < len(vSpec.Names); i++ {
// TODO: only basic literals work currently
if i > len(vSpec.Values) || len(vSpec.Values) == 0 {
break
}
switch v := vSpec.Values[i].(type) {
case *ast.BasicLit:
f.decls[vSpec.Names[i].Name] = v.Value
default:
// log.Printf("Name: %s - Unsupported ValueSpec: %+v\n", vSpec.Names[i].Name, v)
}
}
}
return "", v.evalIdent(at)
case *ast.BasicLit:
return at.Value, nil
case *ast.CallExpr:
return "", v.evalExpr(at)
}
return "", fmt.Errorf("can't handle %T", e)
}
k1, err := zz(expr.Args[0])
}
}
return nil
}
func (f *file) findOpenCalls() error {
var err error
f.walk(func(node ast.Node) bool {
ce, ok := node.(*ast.CallExpr)
if !ok {
return true
}
exists := isPkgDot(ce.Fun, "pkger", "Open")
if !(exists) || len(ce.Args) != 1 {
return true
}
switch x := ce.Args[0].(type) {
case *ast.BasicLit:
s, err := strconv.Unquote(x.Value)
if err != nil {
return err
err = nil
return false
}
if err := v.addPath(k1); err != nil {
return err
pt, err := pkger.Parse(s)
if err != nil {
err = err
return false
}
f.paths = append(f.paths, pt)
case *ast.Ident:
val, ok := f.decls[x.Name]
if !ok {
//TODO: Add ERRORs list to file type and return after iteration!
// log.Printf("Could not find identifier[%s] in decls map\n", x.Name)
return true
}
s, err := strconv.Unquote(val)
if err != nil {
err = nil
return false
}
pt, err := pkger.Parse(s)
if err != nil {
err = err
return false
}
f.paths = append(f.paths, pt)
return nil
case "Open":
for _, e := range expr.Args {
switch at := e.(type) {
case *ast.Ident:
switch at.Obj.Kind {
case ast.Var:
if as, ok := at.Obj.Decl.(*ast.AssignStmt); ok {
v.addVariable("", as)
}
case ast.Con:
if vs, ok := at.Obj.Decl.(*ast.ValueSpec); ok {
v.addConstant("", vs)
}
}
return v.evalIdent(at)
case *ast.BasicLit:
return v.addPath(at.Value)
case *ast.CallExpr:
return v.evalExpr(at)
}
}
default:
}
}
return nil
return true
})
return err
}
func (v *visitor) evalIdent(i *ast.Ident) error {
if i.Obj == nil {
return nil
}
if s, ok := i.Obj.Decl.(*ast.AssignStmt); ok {
return v.evalStmt(s)
}
return nil
}
func (v *visitor) fromVariable(as *ast.AssignStmt) (string, error) {
if len(as.Rhs) == 1 {
if bs, ok := as.Rhs[0].(*ast.BasicLit); ok {
return bs.Value, nil
}
}
return "", fmt.Errorf("unable to find value from variable %v", as)
}
func (v *visitor) addVariable(bn string, as *ast.AssignStmt) error {
bv, err := v.fromVariable(as)
if err != nil {
return nil
}
if len(bn) == 0 {
bn = bv
}
return v.addPath(bn)
}
func (v *visitor) fromConstant(vs *ast.ValueSpec) (string, error) {
if len(vs.Values) == 1 {
if bs, ok := vs.Values[0].(*ast.BasicLit); ok {
return bs.Value, nil
}
}
return "", fmt.Errorf("unable to find value from constant %v", vs)
}
func (v *visitor) addConstant(bn string, vs *ast.ValueSpec) error {
if len(vs.Values) == 1 {
if bs, ok := vs.Values[0].(*ast.BasicLit); ok {
bv := bs.Value
if len(bn) == 0 {
bn = bv
}
return v.addPath(bn)
}
func (f *file) findWalkCalls() error {
var err error
f.walk(func(node ast.Node) bool {
ce, ok := node.(*ast.CallExpr)
if !ok {
return true
}
exists := isPkgDot(ce.Fun, "pkger", "Walk")
if !(exists) || len(ce.Args) != 2 {
return true
}
switch x := ce.Args[0].(type) {
case *ast.BasicLit:
s, err := strconv.Unquote(x.Value)
if err != nil {
err = nil
return false
}
pt, err := pkger.Parse(s)
if err != nil {
err = err
return false
}
f.paths = append(f.paths, pt)
case *ast.Ident:
val, ok := f.decls[x.Name]
if !ok {
//TODO: Add ERRORs list to file type and return after iteration!
// log.Printf("Could not find identifier[%s] in decls map\n", x.Name)
return true
}
s, err := strconv.Unquote(val)
if err != nil {
err = nil
return false
}
pt, err := pkger.Parse(s)
if err != nil {
err = err
return false
}
f.paths = append(f.paths, pt)
default:
}
return true
})
return err
}
func (f *file) findImportCalls() error {
var err error
f.walk(func(node ast.Node) bool {
ce, ok := node.(*ast.ImportSpec)
if !ok {
return true
}
s, err := strconv.Unquote(ce.Path.Value)
if err != nil {
return false
}
info, err := here.Package(s)
if err != nil {
return false
}
fmt.Println(">>>TODO parser/visitor.go:216: info ", info)
res, err := Parse(info)
if err != nil {
return false
}
fmt.Println(">>>TODO parser/visitor.go:224: res ", res)
return true
})
return err
}
// helpers
// =======
func isPkgDot(expr ast.Expr, pkg, name string) bool {
sel, ok := expr.(*ast.SelectorExpr)
return ok && isIdent(sel.X, pkg) && isIdent(sel.Sel, name)
}
func isIdent(expr ast.Expr, ident string) bool {
id, ok := expr.(*ast.Ident)
return ok && id.Name == ident
}
// wrap a function to fulfill ast.Visitor interface
type walker func(ast.Node) bool
func (w walker) Visit(node ast.Node) ast.Visitor {
if w(node) {
return w
}
return nil
}