From e7599507eb3050ccabd2b55674177af4b7fd6fd6 Mon Sep 17 00:00:00 2001 From: Mark Bates Date: Tue, 5 Nov 2019 15:04:38 -0500 Subject: [PATCH] giggling crew --- parser/decl.go | 4 +- parser/file.go | 38 ----- parser/open.go | 6 +- parser/parser.go | 189 ++++++++++++++++++--- parser/parser_test.go | 7 +- parser/source.go | 185 ++++++++++++++++++++ parser/stat.go | 4 +- parser/visitor.go | 386 ------------------------------------------ 8 files changed, 356 insertions(+), 463 deletions(-) create mode 100644 parser/source.go delete mode 100644 parser/visitor.go diff --git a/parser/decl.go b/parser/decl.go index 270c5e9..762806c 100644 --- a/parser/decl.go +++ b/parser/decl.go @@ -1,6 +1,7 @@ package parser import ( + "fmt" "go/token" "sort" ) @@ -39,11 +40,12 @@ func (decls Decls) Files() ([]*File, error) { files, err := fl.Files(v) if err != nil { - return nil, err + return nil, fmt.Errorf("%w: %s", err, d) } for _, f := range files { m[f.Abs] = f + v[f.Abs] = f.Abs } } diff --git a/parser/file.go b/parser/file.go index 8149178..4b626ed 100644 --- a/parser/file.go +++ b/parser/file.go @@ -2,11 +2,6 @@ package parser import ( "encoding/json" - "go/ast" - "go/parser" - "go/token" - "io" - "io/ioutil" "github.com/markbates/pkger/here" ) @@ -21,36 +16,3 @@ func (f File) String() string { b, _ := json.MarshalIndent(f, "", " ") return string(b) } - -type parsedFile struct { - File string - FileSet *token.FileSet - Ast *ast.File -} - -// parseFileMode ... -func parseFileMode(f string, mode parser.Mode) (parsedFile, error) { - pf := parsedFile{ - File: f, - FileSet: token.NewFileSet(), - } - - b, err := ioutil.ReadFile(f) - if err != nil { - return pf, err - } - src := string(b) - - pff, err := parser.ParseFile(pf.FileSet, f, src, mode) - if err != nil && err != io.EOF { - return pf, err - } - pf.Ast = pff - - return pf, nil -} - -// parseFile ... -func parseFile(f string) (parsedFile, error) { - return parseFileMode(f, 0) -} diff --git a/parser/open.go b/parser/open.go index c9a94b3..21efaf6 100644 --- a/parser/open.go +++ b/parser/open.go @@ -71,11 +71,7 @@ func (d OpenDecl) Files(virtual map[string]string) ([]*File, error) { return wd.Files(virtual) } var files []*File - files = append(files, &File{ - Abs: filepath.Join(her.Module.Dir, pt.Name), - Path: pt, - Here: her, - }) + files = append(files, d.file) return files, nil } diff --git a/parser/parser.go b/parser/parser.go index 5052a9a..5c82410 100644 --- a/parser/parser.go +++ b/parser/parser.go @@ -4,9 +4,12 @@ import ( "fmt" "go/parser" "go/token" + "io" + "io/ioutil" "os" "path/filepath" "strings" + "sync" "github.com/markbates/pkger/here" ) @@ -14,33 +17,171 @@ import ( var defaultIgnoredFolders = []string{".", "_", "vendor", "node_modules", "testdata"} func Parse(her here.Info) (Decls, error) { - src, err := fromSource(her) + p, err := New(her) if err != nil { return nil, err } - - return src, nil + return p.Decls() } -func fromSource(her here.Info) (Decls, error) { - root := her.Dir - fi, err := os.Stat(root) +func ParseSource(source Source, mode parser.Mode) (*ParsedSource, error) { + pf := &ParsedSource{ + Source: source, + FileSet: token.NewFileSet(), + } + + b, err := ioutil.ReadFile(source.Abs) if err != nil { return nil, err } - if !fi.IsDir() { - return nil, fmt.Errorf("%q is not a directory", root) + src := string(b) + + pff, err := parser.ParseFile(pf.FileSet, source.Abs, src, mode) + if err != nil && err != io.EOF { + return nil, err + } + pf.Ast = pff + + return pf, nil +} + +func ParseFile(abs string, mode parser.Mode) (*ParsedSource, error) { + s := Source{ + Abs: abs, + } + + info, err := os.Stat(abs) + if err != nil { + return nil, err + } + + if info.IsDir() { + return nil, fmt.Errorf("%s is a directory", abs) + } + + dir := filepath.Dir(abs) + + s.Here, err = here.Dir(dir) + if err != nil { + return nil, err + } + + s.Path, err = s.Here.Parse(strings.TrimPrefix(abs, dir)) + + return ParseSource(s, 0) +} + +func ParseDir(abs string, mode parser.Mode) ([]*ParsedSource, error) { + info, err := os.Stat(abs) + if err != nil { + return nil, err + } + + if !info.IsDir() { + return nil, fmt.Errorf("%s is not a directory", abs) + } + dir := filepath.Dir(abs) + + her, err := here.Dir(dir) + if err != nil { + return nil, err + } + + pt, err := her.Parse(strings.TrimPrefix(abs, dir)) + if err != nil { + return nil, err + } + + fset := token.NewFileSet() + + pkgs, err := parser.ParseDir(fset, abs, nil, 0) + if err != nil { + return nil, err + } + + var srcs []*ParsedSource + for _, pkg := range pkgs { + for name, pf := range pkg.Files { + s := &ParsedSource{ + Source: Source{ + Abs: name, + Path: pt, + Here: her, + }, + FileSet: fset, + Ast: pf, + } + srcs = append(srcs, s) + } + } + + return srcs, nil +} +func New(her here.Info) (*Parser, error) { + return &Parser{ + Info: her, + decls: map[string]Decls{}, + }, nil +} + +type Parser struct { + here.Info + decls map[string]Decls + once sync.Once + err error +} + +func (p *Parser) Decls() (Decls, error) { + if err := p.parse(); err != nil { + return nil, err } var decls Decls + orderedNames := []string{ + "MkdirAll", + "Create", + "Stat", + "Open", + "Dir", + "Walk", + } + + for _, n := range orderedNames { + decls = append(decls, p.decls[n]...) + } + + return decls, nil +} + +func (p *Parser) DeclsMap() (map[string]Decls, error) { + err := p.Parse() + return p.decls, err +} + +func (p *Parser) Parse() error { + (&p.once).Do(func() { + p.err = p.parse() + }) + return p.err +} + +func (p *Parser) parse() error { + p.decls = map[string]Decls{} + root := p.Dir + + fi, err := os.Stat(root) + if err != nil { + return err + } + if !fi.IsDir() { + return fmt.Errorf("%q is not a directory", root) + } err = filepath.Walk(root, func(path string, info os.FileInfo, err error) error { if err != nil { return err } - fset := token.NewFileSet() - if !info.IsDir() { return nil } @@ -52,29 +193,23 @@ func fromSource(her here.Info) (Decls, error) { } } - pkgs, err := parser.ParseDir(fset, path, nil, 0) + srcs, err := ParseDir(path, 0) if err != nil { - return err + return fmt.Errorf("%w: %s", err, path) } - for _, pkg := range pkgs { - for name, pf := range pkg.Files { - f := &file{ - fset: fset, - astFile: pf, - filename: name, - current: her, - } - - x, err := f.find() - if err != nil { - return err - } - decls = append(decls, x...) + for _, src := range srcs { + mm, err := src.DeclsMap() + if err != nil { + return fmt.Errorf("%w: %s", err, src.Abs) + } + for k, v := range mm { + p.decls[k] = append(p.decls[k], v...) } } + return nil }) - return decls, err + return err } diff --git a/parser/parser_test.go b/parser/parser_test.go index 4ec3512..073bd3e 100644 --- a/parser/parser_test.go +++ b/parser/parser_test.go @@ -1,4 +1,4 @@ -package parser_test +package parser import ( "os" @@ -8,7 +8,6 @@ import ( "testing" "github.com/markbates/pkger/here" - "github.com/markbates/pkger/parser" "github.com/markbates/pkger/pkging/pkgtest" "github.com/markbates/pkger/pkging/stdos" "github.com/stretchr/testify/require" @@ -31,7 +30,7 @@ func Test_Parser_Ref(t *testing.T) { _, err = pkgtest.LoadFiles("/", ref, disk) r.NoError(err) - res, err := parser.Parse(ref.Info) + res, err := Parse(ref.Info) r.NoError(err) @@ -68,7 +67,7 @@ func Test_Parser_Example_HTTP(t *testing.T) { her, err := here.Dir(".") r.NoError(err) - res, err := parser.Parse(her) + res, err := Parse(her) r.NoError(err) files, err := res.Files() diff --git a/parser/source.go b/parser/source.go new file mode 100644 index 0000000..662dc2d --- /dev/null +++ b/parser/source.go @@ -0,0 +1,185 @@ +package parser + +import ( + "fmt" + "go/ast" + "go/token" + "path/filepath" + "strconv" + "sync" + + "github.com/markbates/pkger/here" +) + +type Source struct { + Abs string // full path on disk to file + Path here.Path + Here here.Info +} + +type ParsedSource struct { + Source + FileSet *token.FileSet + Ast *ast.File + decls map[string]Decls + once sync.Once + err error +} + +func (p *ParsedSource) Parse() error { + (&p.once).Do(func() { + p.err = p.parse() + }) + return p.err +} + +func (p *ParsedSource) value(node ast.Node) (string, error) { + var s string + + switch x := node.(type) { + case *ast.BasicLit: + s = x.Value + case *ast.Ident: + s = x.Name + } + + return strconv.Unquote(s) +} + +func (p *ParsedSource) parse() error { + p.decls = map[string]Decls{} + var fn walker = func(node ast.Node) bool { + ce, ok := node.(*ast.CallExpr) + if !ok { + return true + } + + sel, ok := ce.Fun.(*ast.SelectorExpr) + if !ok { + return true + } + + pkg, ok := sel.X.(*ast.Ident) + if !ok { + return true + } + + if pkg.Name != "pkger" { + return true + } + + var fn func(f File, pos token.Position, value string) Decl + + name := sel.Sel.Name + switch name { + case "MkdirAll": + fn = func(f File, pos token.Position, value string) Decl { + return MkdirAllDecl{ + file: &f, + pos: pos, + value: value, + } + } + case "Create": + fn = func(f File, pos token.Position, value string) Decl { + return CreateDecl{ + file: &f, + pos: pos, + value: value, + } + } + case "Stat": + fn = func(f File, pos token.Position, value string) Decl { + return StatDecl{ + file: &f, + pos: pos, + value: value, + } + } + case "Open": + fn = func(f File, pos token.Position, value string) Decl { + return OpenDecl{ + file: &f, + pos: pos, + value: value, + } + } + case "Dir": + fn = func(f File, pos token.Position, value string) Decl { + return HTTPDecl{ + file: &f, + pos: pos, + value: value, + } + } + case "Walk": + fn = func(f File, pos token.Position, value string) Decl { + return WalkDecl{ + file: &f, + pos: pos, + value: value, + } + } + default: + return true + } + + if len(ce.Args) < 1 { + p.err = fmt.Errorf("declarations require at least one argument") + return false + } + + n := ce.Args[0] + val, err := p.value(n) + if err != nil { + p.err = fmt.Errorf("%w: %s", err, n) + return false + } + + info, err := here.Dir(filepath.Dir(p.Abs)) + if err != nil { + p.err = fmt.Errorf("%w: %s", err, p.Abs) + return false + } + + pt, err := info.Parse(val) + if err != nil { + p.err = fmt.Errorf("%w: %s", err, p.Abs) + return false + } + + if pt.Pkg != info.Module.Path { + info, err = here.Package(pt.Pkg) + if err != nil { + p.err = fmt.Errorf("%w: %s", err, p.Abs) + return false + } + } + + f := File{ + Abs: filepath.Join(info.Module.Dir, pt.Name), + Here: info, + Path: pt, + } + + p.decls[name] = append(p.decls[name], fn(f, p.FileSet.Position(n.Pos()), val)) + return true + } + ast.Walk(fn, p.Ast) + return nil +} + +func (p *ParsedSource) DeclsMap() (map[string]Decls, error) { + err := p.Parse() + return p.decls, err +} + +// wrap a function to fulfill ast.Visitor interface +type walker func(ast.Node) bool + +func (w walker) Visit(node ast.Node) ast.Visitor { + if w(node) { + return w + } + return nil +} diff --git a/parser/stat.go b/parser/stat.go index fc0fb0b..d533839 100644 --- a/parser/stat.go +++ b/parser/stat.go @@ -2,6 +2,7 @@ package parser import ( "encoding/json" + "fmt" "go/token" "os" ) @@ -15,8 +16,7 @@ type StatDecl struct { } func (d StatDecl) String() string { - b, _ := json.Marshal(d) - return string(b) + return fmt.Sprintf("pkger.Stat(%q)", d.value) } func (d StatDecl) MarshalJSON() ([]byte, error) { diff --git a/parser/visitor.go b/parser/visitor.go deleted file mode 100644 index ad81813..0000000 --- a/parser/visitor.go +++ /dev/null @@ -1,386 +0,0 @@ -package parser - -import ( - "fmt" - "go/ast" - "go/token" - "path/filepath" - "strconv" - - "github.com/markbates/pkger/here" -) - -type visitor func(node ast.Node) (w ast.Visitor) - -func (v visitor) Visit(node ast.Node) ast.Visitor { - return v(node) -} - -// inspired by https://gist.github.com/cryptix/d1b129361cea51a59af2 -type file struct { - fset *token.FileSet - astFile *ast.File - filename string - decls Decls - current here.Info -} - -func (f *file) walk(fn func(ast.Node) bool) { - ast.Walk(walker(fn), f.astFile) -} - -func (f *file) find() (Decls, error) { - // --- virtual calls first --- - if err := f.findMkdirAllCalls(); err != nil { - return nil, err - } - - if err := f.findCreateCalls(); err != nil { - return nil, err - } - - // -- physical calls second --- - if err := f.findStatCalls(); err != nil { - return nil, err - } - - if err := f.findOpenCalls(); err != nil { - return nil, err - } - - if err := f.findHTTPCalls(); err != nil { - return nil, err - } - - if err := f.findWalkCalls(); err != nil { - return nil, err - } - - return f.decls, nil -} - -func (f *file) asValue(node ast.Node) (string, error) { - var s string - - switch x := node.(type) { - case *ast.BasicLit: - s = x.Value - case *ast.Ident: - s = x.Name - } - - return strconv.Unquote(s) -} - -func (f *file) findMkdirAllCalls() error { - var err error - f.walk(func(node ast.Node) bool { - ce, ok := node.(*ast.CallExpr) - if !ok { - return true - } - - exists := isPkgDot(ce.Fun, "pkger", "MkdirAll") - if !(exists) || len(ce.Args) != 2 { - return true - } - - n := ce.Args[0] - - s, err := f.asValue(n) - if err != nil { - return false - } - - info, err := here.Dir(filepath.Dir(f.filename)) - if err != nil { - return false - } - - pf := &File{ - Abs: f.filename, - Here: info, - Path: here.Path{ - Pkg: info.Module.Path, - Name: s, - }, - } - - decl := MkdirAllDecl{ - file: pf, - pos: f.fset.Position(n.Pos()), - value: s, - } - - f.decls = append(f.decls, decl) - return true - }) - return err -} - -func (f *file) findStatCalls() error { - var err error - f.walk(func(node ast.Node) bool { - ce, ok := node.(*ast.CallExpr) - if !ok { - return true - } - - exists := isPkgDot(ce.Fun, "pkger", "Stat") - if !(exists) || len(ce.Args) != 1 { - return true - } - - n := ce.Args[0] - - s, err := f.asValue(n) - if err != nil { - return false - } - - info, err := here.Dir(filepath.Dir(f.filename)) - if err != nil { - return false - } - - pt, err := info.Parse(s) - if err != nil { - return false - } - - fmt.Println(">>>TODO parser/visitor.go:151: pt ", pt) - fmt.Println(">>>TODO parser/visitor.go:151: info.Module ", info.Module) - if pt.Pkg != info.Module.Path { - info, err = here.Package(pt.Pkg) - if err != nil { - return false - } - } - - pf := &File{ - Abs: f.filename, - Here: info, - Path: pt, - } - - decl := StatDecl{ - file: pf, - pos: f.fset.Position(n.Pos()), - value: s, - } - - f.decls = append(f.decls, decl) - return true - }) - return err -} - -func (f *file) findCreateCalls() error { - var err error - f.walk(func(node ast.Node) bool { - ce, ok := node.(*ast.CallExpr) - if !ok { - return true - } - - exists := isPkgDot(ce.Fun, "pkger", "Create") - if !(exists) || len(ce.Args) != 1 { - return true - } - - n := ce.Args[0] - - s, err := f.asValue(n) - if err != nil { - return false - } - - info, err := here.Dir(filepath.Dir(f.filename)) - if err != nil { - return false - } - - pf := &File{ - Abs: f.filename, - Here: info, - Path: here.Path{ - Pkg: info.Module.Path, - Name: s, - }, - } - - decl := CreateDecl{ - file: pf, - pos: f.fset.Position(n.Pos()), - value: s, - } - - f.decls = append(f.decls, decl) - return true - }) - return err -} - -func (f *file) findOpenCalls() error { - var err error - f.walk(func(node ast.Node) bool { - ce, ok := node.(*ast.CallExpr) - if !ok { - return true - } - - exists := isPkgDot(ce.Fun, "pkger", "Open") - if !(exists) || len(ce.Args) != 1 { - return true - } - - n := ce.Args[0] - - s, err := f.asValue(n) - if err != nil { - return false - } - - info, err := here.Dir(filepath.Dir(f.filename)) - if err != nil { - return false - } - - pf := &File{ - Abs: f.filename, - Here: info, - Path: here.Path{ - Pkg: info.Module.Path, - Name: s, - }, - } - - decl := OpenDecl{ - file: pf, - pos: f.fset.Position(n.Pos()), - value: s, - } - - f.decls = append(f.decls, decl) - return true - }) - return err -} - -func (f *file) findWalkCalls() error { - var err error - f.walk(func(node ast.Node) bool { - ce, ok := node.(*ast.CallExpr) - if !ok { - return true - } - - exists := isPkgDot(ce.Fun, "pkger", "Walk") - if !(exists) || len(ce.Args) != 2 { - return true - } - - n := ce.Args[0] - - s, err := f.asValue(n) - if err != nil { - err = err - return false - } - - dir := filepath.Dir(f.filename) - info, err := here.Dir(dir) - if err != nil { - err = err - return false - } - - pf := &File{ - Abs: f.filename, - Here: info, - Path: here.Path{ - Pkg: info.Module.Path, - Name: s, - }, - } - - decl := WalkDecl{ - file: pf, - pos: f.fset.Position(n.Pos()), - value: s, - } - - f.decls = append(f.decls, decl) - return true - }) - return err -} - -func (f *file) findHTTPCalls() error { - var err error - f.walk(func(node ast.Node) bool { - ce, ok := node.(*ast.CallExpr) - if !ok { - return true - } - - exists := isPkgDot(ce.Fun, "pkger", "Dir") - if !(exists) || len(ce.Args) != 1 { - return true - } - - n := ce.Args[0] - - s, err := f.asValue(n) - if err != nil { - return false - } - - info, err := here.Dir(filepath.Dir(f.filename)) - if err != nil { - return false - } - - pf := &File{ - Abs: f.filename, - Here: info, - Path: here.Path{ - Pkg: info.Module.Path, - Name: s, - }, - } - - pos := f.fset.Position(n.Pos()) - decl := HTTPDecl{ - file: pf, - pos: pos, - value: s, - } - - f.decls = append(f.decls, decl) - return true - }) - return err -} - -// helpers -// ======= -func isPkgDot(expr ast.Expr, pkg, name string) bool { - sel, ok := expr.(*ast.SelectorExpr) - return ok && isIdent(sel.X, pkg) && isIdent(sel.Sel, name) -} - -func isIdent(expr ast.Expr, ident string) bool { - id, ok := expr.(*ast.Ident) - return ok && id.Name == ident -} - -// wrap a function to fulfill ast.Visitor interface -type walker func(ast.Node) bool - -func (w walker) Visit(node ast.Node) ast.Visitor { - if w(node) { - return w - } - return nil -}