giggling crew

parser/decl.go

@@ -1,6 +1,7 @@
 package parser
 
 import (
+	"fmt"
 	"go/token"
 	"sort"
 )
@@ -39,11 +40,12 @@ func (decls Decls) Files() ([]*File, error) {
 
 		files, err := fl.Files(v)
 		if err != nil {
-			return nil, err
+			return nil, fmt.Errorf("%w: %s", err, d)
 		}
 
 		for _, f := range files {
 			m[f.Abs] = f
+			v[f.Abs] = f.Abs
 		}
 	}
 

parser/file.go

@@ -2,11 +2,6 @@ package parser
 
 import (
 	"encoding/json"
-	"go/ast"
-	"go/parser"
-	"go/token"
-	"io"
-	"io/ioutil"
 
 	"github.com/markbates/pkger/here"
 )
@@ -21,36 +16,3 @@ func (f File) String() string {
 	b, _ := json.MarshalIndent(f, "", " ")
 	return string(b)
 }
-
-type parsedFile struct {
-	File    string
-	FileSet *token.FileSet
-	Ast     *ast.File
-}
-
-// parseFileMode ...
-func parseFileMode(f string, mode parser.Mode) (parsedFile, error) {
-	pf := parsedFile{
-		File:    f,
-		FileSet: token.NewFileSet(),
-	}
-
-	b, err := ioutil.ReadFile(f)
-	if err != nil {
-		return pf, err
-	}
-	src := string(b)
-
-	pff, err := parser.ParseFile(pf.FileSet, f, src, mode)
-	if err != nil && err != io.EOF {
-		return pf, err
-	}
-	pf.Ast = pff
-
-	return pf, nil
-}
-
-// parseFile ...
-func parseFile(f string) (parsedFile, error) {
-	return parseFileMode(f, 0)
-}

parser/open.go

@@ -71,11 +71,7 @@ func (d OpenDecl) Files(virtual map[string]string) ([]*File, error) {
 		return wd.Files(virtual)
 	}
 	var files []*File
-	files = append(files, &File{
+	files = append(files, d.file)
-		Abs:  filepath.Join(her.Module.Dir, pt.Name),
-		Path: pt,
-		Here: her,
-	})
 
 	return files, nil
 }

parser/parser.go

@@ -4,9 +4,12 @@ import (
 	"fmt"
 	"go/parser"
 	"go/token"
+	"io"
+	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strings"
+	"sync"
 
 	"github.com/markbates/pkger/here"
 )
@@ -14,33 +17,171 @@ import (
 var defaultIgnoredFolders = []string{".", "_", "vendor", "node_modules", "testdata"}
 
 func Parse(her here.Info) (Decls, error) {
-	src, err := fromSource(her)
+	p, err := New(her)
+	if err != nil {
+		return nil, err
+	}
+	return p.Decls()
+}
+
+func ParseSource(source Source, mode parser.Mode) (*ParsedSource, error) {
+	pf := &ParsedSource{
+		Source:  source,
+		FileSet: token.NewFileSet(),
+	}
+
+	b, err := ioutil.ReadFile(source.Abs)
+	if err != nil {
+		return nil, err
+	}
+	src := string(b)
+
+	pff, err := parser.ParseFile(pf.FileSet, source.Abs, src, mode)
+	if err != nil && err != io.EOF {
+		return nil, err
+	}
+	pf.Ast = pff
+
+	return pf, nil
+}
+
+func ParseFile(abs string, mode parser.Mode) (*ParsedSource, error) {
+	s := Source{
+		Abs: abs,
+	}
+
+	info, err := os.Stat(abs)
 	if err != nil {
 		return nil, err
 	}
 
-	return src, nil
+	if info.IsDir() {
+		return nil, fmt.Errorf("%s is a directory", abs)
 	}
 
-func fromSource(her here.Info) (Decls, error) {
+	dir := filepath.Dir(abs)
-	root := her.Dir
+
-	fi, err := os.Stat(root)
+	s.Here, err = here.Dir(dir)
 	if err != nil {
 		return nil, err
 	}
-	if !fi.IsDir() {
+
-		return nil, fmt.Errorf("%q is not a directory", root)
+	s.Path, err = s.Here.Parse(strings.TrimPrefix(abs, dir))
+
+	return ParseSource(s, 0)
+}
+
+func ParseDir(abs string, mode parser.Mode) ([]*ParsedSource, error) {
+	info, err := os.Stat(abs)
+	if err != nil {
+		return nil, err
+	}
+
+	if !info.IsDir() {
+		return nil, fmt.Errorf("%s is not a directory", abs)
+	}
+	dir := filepath.Dir(abs)
+
+	her, err := here.Dir(dir)
+	if err != nil {
+		return nil, err
+	}
+
+	pt, err := her.Parse(strings.TrimPrefix(abs, dir))
+	if err != nil {
+		return nil, err
+	}
+
+	fset := token.NewFileSet()
+
+	pkgs, err := parser.ParseDir(fset, abs, nil, 0)
+	if err != nil {
+		return nil, err
+	}
+
+	var srcs []*ParsedSource
+	for _, pkg := range pkgs {
+		for name, pf := range pkg.Files {
+			s := &ParsedSource{
+				Source: Source{
+					Abs:  name,
+					Path: pt,
+					Here: her,
+				},
+				FileSet: fset,
+				Ast:     pf,
+			}
+			srcs = append(srcs, s)
+		}
+	}
+
+	return srcs, nil
+}
+func New(her here.Info) (*Parser, error) {
+	return &Parser{
+		Info:  her,
+		decls: map[string]Decls{},
+	}, nil
+}
+
+type Parser struct {
+	here.Info
+	decls map[string]Decls
+	once  sync.Once
+	err   error
+}
+
+func (p *Parser) Decls() (Decls, error) {
+	if err := p.parse(); err != nil {
+		return nil, err
 	}
 
 	var decls Decls
+	orderedNames := []string{
+		"MkdirAll",
+		"Create",
+		"Stat",
+		"Open",
+		"Dir",
+		"Walk",
+	}
+
+	for _, n := range orderedNames {
+		decls = append(decls, p.decls[n]...)
+	}
+
+	return decls, nil
+}
+
+func (p *Parser) DeclsMap() (map[string]Decls, error) {
+	err := p.Parse()
+	return p.decls, err
+}
+
+func (p *Parser) Parse() error {
+	(&p.once).Do(func() {
+		p.err = p.parse()
+	})
+	return p.err
+}
+
+func (p *Parser) parse() error {
+	p.decls = map[string]Decls{}
+	root := p.Dir
+
+	fi, err := os.Stat(root)
+	if err != nil {
+		return err
+	}
+	if !fi.IsDir() {
+		return fmt.Errorf("%q is not a directory", root)
+	}
 
 	err = filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
 
-		fset := token.NewFileSet()
-
 		if !info.IsDir() {
 			return nil
 		}
@@ -52,29 +193,23 @@ func fromSource(her here.Info) (Decls, error) {
 			}
 		}
 
-		pkgs, err := parser.ParseDir(fset, path, nil, 0)
+		srcs, err := ParseDir(path, 0)
 		if err != nil {
-			return err
+			return fmt.Errorf("%w: %s", err, path)
 		}
 
-		for _, pkg := range pkgs {
+		for _, src := range srcs {
-			for name, pf := range pkg.Files {
+			mm, err := src.DeclsMap()
-				f := &file{
+			if err != nil {
-					fset:     fset,
+				return fmt.Errorf("%w: %s", err, src.Abs)
-					astFile:  pf,
+			}
-					filename: name,
+			for k, v := range mm {
-					current:  her,
+				p.decls[k] = append(p.decls[k], v...)
+			}
 		}
 
-				x, err := f.find()
-				if err != nil {
-					return err
-				}
-				decls = append(decls, x...)
-			}
-		}
 		return nil
 	})
 
-	return decls, err
+	return err
 }

parser/parser_test.go

@@ -1,4 +1,4 @@
-package parser_test
+package parser
 
 import (
 	"os"
@@ -8,7 +8,6 @@ import (
 	"testing"
 
 	"github.com/markbates/pkger/here"
-	"github.com/markbates/pkger/parser"
 	"github.com/markbates/pkger/pkging/pkgtest"
 	"github.com/markbates/pkger/pkging/stdos"
 	"github.com/stretchr/testify/require"
@@ -31,7 +30,7 @@ func Test_Parser_Ref(t *testing.T) {
 	_, err = pkgtest.LoadFiles("/", ref, disk)
 	r.NoError(err)
 
-	res, err := parser.Parse(ref.Info)
+	res, err := Parse(ref.Info)
 
 	r.NoError(err)
 
@@ -68,7 +67,7 @@ func Test_Parser_Example_HTTP(t *testing.T) {
 	her, err := here.Dir(".")
 	r.NoError(err)
 
-	res, err := parser.Parse(her)
+	res, err := Parse(her)
 	r.NoError(err)
 
 	files, err := res.Files()

parser/source.go

@@ -0,0 +1,185 @@
+package parser
+
+import (
+	"fmt"
+	"go/ast"
+	"go/token"
+	"path/filepath"
+	"strconv"
+	"sync"
+
+	"github.com/markbates/pkger/here"
+)
+
+type Source struct {
+	Abs  string // full path on disk to file
+	Path here.Path
+	Here here.Info
+}
+
+type ParsedSource struct {
+	Source
+	FileSet *token.FileSet
+	Ast     *ast.File
+	decls   map[string]Decls
+	once    sync.Once
+	err     error
+}
+
+func (p *ParsedSource) Parse() error {
+	(&p.once).Do(func() {
+		p.err = p.parse()
+	})
+	return p.err
+}
+
+func (p *ParsedSource) value(node ast.Node) (string, error) {
+	var s string
+
+	switch x := node.(type) {
+	case *ast.BasicLit:
+		s = x.Value
+	case *ast.Ident:
+		s = x.Name
+	}
+
+	return strconv.Unquote(s)
+}
+
+func (p *ParsedSource) parse() error {
+	p.decls = map[string]Decls{}
+	var fn walker = func(node ast.Node) bool {
+		ce, ok := node.(*ast.CallExpr)
+		if !ok {
+			return true
+		}
+
+		sel, ok := ce.Fun.(*ast.SelectorExpr)
+		if !ok {
+			return true
+		}
+
+		pkg, ok := sel.X.(*ast.Ident)
+		if !ok {
+			return true
+		}
+
+		if pkg.Name != "pkger" {
+			return true
+		}
+
+		var fn func(f File, pos token.Position, value string) Decl
+
+		name := sel.Sel.Name
+		switch name {
+		case "MkdirAll":
+			fn = func(f File, pos token.Position, value string) Decl {
+				return MkdirAllDecl{
+					file:  &f,
+					pos:   pos,
+					value: value,
+				}
+			}
+		case "Create":
+			fn = func(f File, pos token.Position, value string) Decl {
+				return CreateDecl{
+					file:  &f,
+					pos:   pos,
+					value: value,
+				}
+			}
+		case "Stat":
+			fn = func(f File, pos token.Position, value string) Decl {
+				return StatDecl{
+					file:  &f,
+					pos:   pos,
+					value: value,
+				}
+			}
+		case "Open":
+			fn = func(f File, pos token.Position, value string) Decl {
+				return OpenDecl{
+					file:  &f,
+					pos:   pos,
+					value: value,
+				}
+			}
+		case "Dir":
+			fn = func(f File, pos token.Position, value string) Decl {
+				return HTTPDecl{
+					file:  &f,
+					pos:   pos,
+					value: value,
+				}
+			}
+		case "Walk":
+			fn = func(f File, pos token.Position, value string) Decl {
+				return WalkDecl{
+					file:  &f,
+					pos:   pos,
+					value: value,
+				}
+			}
+		default:
+			return true
+		}
+
+		if len(ce.Args) < 1 {
+			p.err = fmt.Errorf("declarations require at least one argument")
+			return false
+		}
+
+		n := ce.Args[0]
+		val, err := p.value(n)
+		if err != nil {
+			p.err = fmt.Errorf("%w: %s", err, n)
+			return false
+		}
+
+		info, err := here.Dir(filepath.Dir(p.Abs))
+		if err != nil {
+			p.err = fmt.Errorf("%w: %s", err, p.Abs)
+			return false
+		}
+
+		pt, err := info.Parse(val)
+		if err != nil {
+			p.err = fmt.Errorf("%w: %s", err, p.Abs)
+			return false
+		}
+
+		if pt.Pkg != info.Module.Path {
+			info, err = here.Package(pt.Pkg)
+			if err != nil {
+				p.err = fmt.Errorf("%w: %s", err, p.Abs)
+				return false
+			}
+		}
+
+		f := File{
+			Abs:  filepath.Join(info.Module.Dir, pt.Name),
+			Here: info,
+			Path: pt,
+		}
+
+		p.decls[name] = append(p.decls[name], fn(f, p.FileSet.Position(n.Pos()), val))
+		return true
+	}
+	ast.Walk(fn, p.Ast)
+	return nil
+}
+
+func (p *ParsedSource) DeclsMap() (map[string]Decls, error) {
+	err := p.Parse()
+	return p.decls, err
+}
+
+// wrap a function to fulfill ast.Visitor interface
+type walker func(ast.Node) bool
+
+func (w walker) Visit(node ast.Node) ast.Visitor {
+	if w(node) {
+		return w
+	}
+	return nil
+}

parser/stat.go

@@ -2,6 +2,7 @@ package parser
 
 import (
 	"encoding/json"
+	"fmt"
 	"go/token"
 	"os"
 )
@@ -15,8 +16,7 @@ type StatDecl struct {
 }
 
 func (d StatDecl) String() string {
-	b, _ := json.Marshal(d)
+	return fmt.Sprintf("pkger.Stat(%q)", d.value)
-	return string(b)
 }
 
 func (d StatDecl) MarshalJSON() ([]byte, error) {

parser/visitor.go

@@ -1,386 +0,0 @@
-package parser
-
-import (
-	"fmt"
-	"go/ast"
-	"go/token"
-	"path/filepath"
-	"strconv"
-
-	"github.com/markbates/pkger/here"
-)
-
-type visitor func(node ast.Node) (w ast.Visitor)
-
-func (v visitor) Visit(node ast.Node) ast.Visitor {
-	return v(node)
-}
-
-// inspired by https://gist.github.com/cryptix/d1b129361cea51a59af2
-type file struct {
-	fset     *token.FileSet
-	astFile  *ast.File
-	filename string
-	decls    Decls
-	current  here.Info
-}
-
-func (f *file) walk(fn func(ast.Node) bool) {
-	ast.Walk(walker(fn), f.astFile)
-}
-
-func (f *file) find() (Decls, error) {
-	// --- virtual calls first ---
-	if err := f.findMkdirAllCalls(); err != nil {
-		return nil, err
-	}
-
-	if err := f.findCreateCalls(); err != nil {
-		return nil, err
-	}
-
-	// -- physical calls second ---
-	if err := f.findStatCalls(); err != nil {
-		return nil, err
-	}
-
-	if err := f.findOpenCalls(); err != nil {
-		return nil, err
-	}
-
-	if err := f.findHTTPCalls(); err != nil {
-		return nil, err
-	}
-
-	if err := f.findWalkCalls(); err != nil {
-		return nil, err
-	}
-
-	return f.decls, nil
-}
-
-func (f *file) asValue(node ast.Node) (string, error) {
-	var s string
-
-	switch x := node.(type) {
-	case *ast.BasicLit:
-		s = x.Value
-	case *ast.Ident:
-		s = x.Name
-	}
-
-	return strconv.Unquote(s)
-}
-
-func (f *file) findMkdirAllCalls() error {
-	var err error
-	f.walk(func(node ast.Node) bool {
-		ce, ok := node.(*ast.CallExpr)
-		if !ok {
-			return true
-		}
-
-		exists := isPkgDot(ce.Fun, "pkger", "MkdirAll")
-		if !(exists) || len(ce.Args) != 2 {
-			return true
-		}
-
-		n := ce.Args[0]
-
-		s, err := f.asValue(n)
-		if err != nil {
-			return false
-		}
-
-		info, err := here.Dir(filepath.Dir(f.filename))
-		if err != nil {
-			return false
-		}
-
-		pf := &File{
-			Abs:  f.filename,
-			Here: info,
-			Path: here.Path{
-				Pkg:  info.Module.Path,
-				Name: s,
-			},
-		}
-
-		decl := MkdirAllDecl{
-			file:  pf,
-			pos:   f.fset.Position(n.Pos()),
-			value: s,
-		}
-
-		f.decls = append(f.decls, decl)
-		return true
-	})
-	return err
-}
-
-func (f *file) findStatCalls() error {
-	var err error
-	f.walk(func(node ast.Node) bool {
-		ce, ok := node.(*ast.CallExpr)
-		if !ok {
-			return true
-		}
-
-		exists := isPkgDot(ce.Fun, "pkger", "Stat")
-		if !(exists) || len(ce.Args) != 1 {
-			return true
-		}
-
-		n := ce.Args[0]
-
-		s, err := f.asValue(n)
-		if err != nil {
-			return false
-		}
-
-		info, err := here.Dir(filepath.Dir(f.filename))
-		if err != nil {
-			return false
-		}
-
-		pt, err := info.Parse(s)
-		if err != nil {
-			return false
-		}
-
-		fmt.Println(">>>TODO parser/visitor.go:151: pt ", pt)
-		fmt.Println(">>>TODO parser/visitor.go:151: info.Module ", info.Module)
-		if pt.Pkg != info.Module.Path {
-			info, err = here.Package(pt.Pkg)
-			if err != nil {
-				return false
-			}
-		}
-
-		pf := &File{
-			Abs:  f.filename,
-			Here: info,
-			Path: pt,
-		}
-
-		decl := StatDecl{
-			file:  pf,
-			pos:   f.fset.Position(n.Pos()),
-			value: s,
-		}
-
-		f.decls = append(f.decls, decl)
-		return true
-	})
-	return err
-}
-
-func (f *file) findCreateCalls() error {
-	var err error
-	f.walk(func(node ast.Node) bool {
-		ce, ok := node.(*ast.CallExpr)
-		if !ok {
-			return true
-		}
-
-		exists := isPkgDot(ce.Fun, "pkger", "Create")
-		if !(exists) || len(ce.Args) != 1 {
-			return true
-		}
-
-		n := ce.Args[0]
-
-		s, err := f.asValue(n)
-		if err != nil {
-			return false
-		}
-
-		info, err := here.Dir(filepath.Dir(f.filename))
-		if err != nil {
-			return false
-		}
-
-		pf := &File{
-			Abs:  f.filename,
-			Here: info,
-			Path: here.Path{
-				Pkg:  info.Module.Path,
-				Name: s,
-			},
-		}
-
-		decl := CreateDecl{
-			file:  pf,
-			pos:   f.fset.Position(n.Pos()),
-			value: s,
-		}
-
-		f.decls = append(f.decls, decl)
-		return true
-	})
-	return err
-}
-
-func (f *file) findOpenCalls() error {
-	var err error
-	f.walk(func(node ast.Node) bool {
-		ce, ok := node.(*ast.CallExpr)
-		if !ok {
-			return true
-		}
-
-		exists := isPkgDot(ce.Fun, "pkger", "Open")
-		if !(exists) || len(ce.Args) != 1 {
-			return true
-		}
-
-		n := ce.Args[0]
-
-		s, err := f.asValue(n)
-		if err != nil {
-			return false
-		}
-
-		info, err := here.Dir(filepath.Dir(f.filename))
-		if err != nil {
-			return false
-		}
-
-		pf := &File{
-			Abs:  f.filename,
-			Here: info,
-			Path: here.Path{
-				Pkg:  info.Module.Path,
-				Name: s,
-			},
-		}
-
-		decl := OpenDecl{
-			file:  pf,
-			pos:   f.fset.Position(n.Pos()),
-			value: s,
-		}
-
-		f.decls = append(f.decls, decl)
-		return true
-	})
-	return err
-}
-
-func (f *file) findWalkCalls() error {
-	var err error
-	f.walk(func(node ast.Node) bool {
-		ce, ok := node.(*ast.CallExpr)
-		if !ok {
-			return true
-		}
-
-		exists := isPkgDot(ce.Fun, "pkger", "Walk")
-		if !(exists) || len(ce.Args) != 2 {
-			return true
-		}
-
-		n := ce.Args[0]
-
-		s, err := f.asValue(n)
-		if err != nil {
-			err = err
-			return false
-		}
-
-		dir := filepath.Dir(f.filename)
-		info, err := here.Dir(dir)
-		if err != nil {
-			err = err
-			return false
-		}
-
-		pf := &File{
-			Abs:  f.filename,
-			Here: info,
-			Path: here.Path{
-				Pkg:  info.Module.Path,
-				Name: s,
-			},
-		}
-
-		decl := WalkDecl{
-			file:  pf,
-			pos:   f.fset.Position(n.Pos()),
-			value: s,
-		}
-
-		f.decls = append(f.decls, decl)
-		return true
-	})
-	return err
-}
-
-func (f *file) findHTTPCalls() error {
-	var err error
-	f.walk(func(node ast.Node) bool {
-		ce, ok := node.(*ast.CallExpr)
-		if !ok {
-			return true
-		}
-
-		exists := isPkgDot(ce.Fun, "pkger", "Dir")
-		if !(exists) || len(ce.Args) != 1 {
-			return true
-		}
-
-		n := ce.Args[0]
-
-		s, err := f.asValue(n)
-		if err != nil {
-			return false
-		}
-
-		info, err := here.Dir(filepath.Dir(f.filename))
-		if err != nil {
-			return false
-		}
-
-		pf := &File{
-			Abs:  f.filename,
-			Here: info,
-			Path: here.Path{
-				Pkg:  info.Module.Path,
-				Name: s,
-			},
-		}
-
-		pos := f.fset.Position(n.Pos())
-		decl := HTTPDecl{
-			file:  pf,
-			pos:   pos,
-			value: s,
-		}
-
-		f.decls = append(f.decls, decl)
-		return true
-	})
-	return err
-}
-
-// helpers
-// =======
-func isPkgDot(expr ast.Expr, pkg, name string) bool {
-	sel, ok := expr.(*ast.SelectorExpr)
-	return ok && isIdent(sel.X, pkg) && isIdent(sel.Sel, name)
-}
-
-func isIdent(expr ast.Expr, ident string) bool {
-	id, ok := expr.(*ast.Ident)
-	return ok && id.Name == ident
-}
-
-// wrap a function to fulfill ast.Visitor interface
-type walker func(ast.Node) bool
-
-func (w walker) Visit(node ast.Node) ast.Visitor {
-	if w(node) {
-		return w
-	}
-	return nil
-}