giggling crew

This commit is contained in:
Mark Bates 2019-11-05 15:04:38 -05:00
parent 5a8d085fb8
commit e7599507eb
8 changed files with 356 additions and 463 deletions

View File

@ -1,6 +1,7 @@
package parser
import (
"fmt"
"go/token"
"sort"
)
@ -39,11 +40,12 @@ func (decls Decls) Files() ([]*File, error) {
files, err := fl.Files(v)
if err != nil {
return nil, err
return nil, fmt.Errorf("%w: %s", err, d)
}
for _, f := range files {
m[f.Abs] = f
v[f.Abs] = f.Abs
}
}

View File

@ -2,11 +2,6 @@ package parser
import (
"encoding/json"
"go/ast"
"go/parser"
"go/token"
"io"
"io/ioutil"
"github.com/markbates/pkger/here"
)
@ -21,36 +16,3 @@ func (f File) String() string {
b, _ := json.MarshalIndent(f, "", " ")
return string(b)
}
type parsedFile struct {
File string
FileSet *token.FileSet
Ast *ast.File
}
// parseFileMode ...
func parseFileMode(f string, mode parser.Mode) (parsedFile, error) {
pf := parsedFile{
File: f,
FileSet: token.NewFileSet(),
}
b, err := ioutil.ReadFile(f)
if err != nil {
return pf, err
}
src := string(b)
pff, err := parser.ParseFile(pf.FileSet, f, src, mode)
if err != nil && err != io.EOF {
return pf, err
}
pf.Ast = pff
return pf, nil
}
// parseFile ...
func parseFile(f string) (parsedFile, error) {
return parseFileMode(f, 0)
}

View File

@ -71,11 +71,7 @@ func (d OpenDecl) Files(virtual map[string]string) ([]*File, error) {
return wd.Files(virtual)
}
var files []*File
files = append(files, &File{
Abs: filepath.Join(her.Module.Dir, pt.Name),
Path: pt,
Here: her,
})
files = append(files, d.file)
return files, nil
}

View File

@ -4,9 +4,12 @@ import (
"fmt"
"go/parser"
"go/token"
"io"
"io/ioutil"
"os"
"path/filepath"
"strings"
"sync"
"github.com/markbates/pkger/here"
)
@ -14,33 +17,171 @@ import (
var defaultIgnoredFolders = []string{".", "_", "vendor", "node_modules", "testdata"}
func Parse(her here.Info) (Decls, error) {
src, err := fromSource(her)
p, err := New(her)
if err != nil {
return nil, err
}
return src, nil
return p.Decls()
}
func fromSource(her here.Info) (Decls, error) {
root := her.Dir
fi, err := os.Stat(root)
func ParseSource(source Source, mode parser.Mode) (*ParsedSource, error) {
pf := &ParsedSource{
Source: source,
FileSet: token.NewFileSet(),
}
b, err := ioutil.ReadFile(source.Abs)
if err != nil {
return nil, err
}
if !fi.IsDir() {
return nil, fmt.Errorf("%q is not a directory", root)
src := string(b)
pff, err := parser.ParseFile(pf.FileSet, source.Abs, src, mode)
if err != nil && err != io.EOF {
return nil, err
}
pf.Ast = pff
return pf, nil
}
func ParseFile(abs string, mode parser.Mode) (*ParsedSource, error) {
s := Source{
Abs: abs,
}
info, err := os.Stat(abs)
if err != nil {
return nil, err
}
if info.IsDir() {
return nil, fmt.Errorf("%s is a directory", abs)
}
dir := filepath.Dir(abs)
s.Here, err = here.Dir(dir)
if err != nil {
return nil, err
}
s.Path, err = s.Here.Parse(strings.TrimPrefix(abs, dir))
return ParseSource(s, 0)
}
func ParseDir(abs string, mode parser.Mode) ([]*ParsedSource, error) {
info, err := os.Stat(abs)
if err != nil {
return nil, err
}
if !info.IsDir() {
return nil, fmt.Errorf("%s is not a directory", abs)
}
dir := filepath.Dir(abs)
her, err := here.Dir(dir)
if err != nil {
return nil, err
}
pt, err := her.Parse(strings.TrimPrefix(abs, dir))
if err != nil {
return nil, err
}
fset := token.NewFileSet()
pkgs, err := parser.ParseDir(fset, abs, nil, 0)
if err != nil {
return nil, err
}
var srcs []*ParsedSource
for _, pkg := range pkgs {
for name, pf := range pkg.Files {
s := &ParsedSource{
Source: Source{
Abs: name,
Path: pt,
Here: her,
},
FileSet: fset,
Ast: pf,
}
srcs = append(srcs, s)
}
}
return srcs, nil
}
func New(her here.Info) (*Parser, error) {
return &Parser{
Info: her,
decls: map[string]Decls{},
}, nil
}
type Parser struct {
here.Info
decls map[string]Decls
once sync.Once
err error
}
func (p *Parser) Decls() (Decls, error) {
if err := p.parse(); err != nil {
return nil, err
}
var decls Decls
orderedNames := []string{
"MkdirAll",
"Create",
"Stat",
"Open",
"Dir",
"Walk",
}
for _, n := range orderedNames {
decls = append(decls, p.decls[n]...)
}
return decls, nil
}
func (p *Parser) DeclsMap() (map[string]Decls, error) {
err := p.Parse()
return p.decls, err
}
func (p *Parser) Parse() error {
(&p.once).Do(func() {
p.err = p.parse()
})
return p.err
}
func (p *Parser) parse() error {
p.decls = map[string]Decls{}
root := p.Dir
fi, err := os.Stat(root)
if err != nil {
return err
}
if !fi.IsDir() {
return fmt.Errorf("%q is not a directory", root)
}
err = filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
fset := token.NewFileSet()
if !info.IsDir() {
return nil
}
@ -52,29 +193,23 @@ func fromSource(her here.Info) (Decls, error) {
}
}
pkgs, err := parser.ParseDir(fset, path, nil, 0)
srcs, err := ParseDir(path, 0)
if err != nil {
return err
return fmt.Errorf("%w: %s", err, path)
}
for _, pkg := range pkgs {
for name, pf := range pkg.Files {
f := &file{
fset: fset,
astFile: pf,
filename: name,
current: her,
for _, src := range srcs {
mm, err := src.DeclsMap()
if err != nil {
return fmt.Errorf("%w: %s", err, src.Abs)
}
for k, v := range mm {
p.decls[k] = append(p.decls[k], v...)
}
}
x, err := f.find()
if err != nil {
return err
}
decls = append(decls, x...)
}
}
return nil
})
return decls, err
return err
}

View File

@ -1,4 +1,4 @@
package parser_test
package parser
import (
"os"
@ -8,7 +8,6 @@ import (
"testing"
"github.com/markbates/pkger/here"
"github.com/markbates/pkger/parser"
"github.com/markbates/pkger/pkging/pkgtest"
"github.com/markbates/pkger/pkging/stdos"
"github.com/stretchr/testify/require"
@ -31,7 +30,7 @@ func Test_Parser_Ref(t *testing.T) {
_, err = pkgtest.LoadFiles("/", ref, disk)
r.NoError(err)
res, err := parser.Parse(ref.Info)
res, err := Parse(ref.Info)
r.NoError(err)
@ -68,7 +67,7 @@ func Test_Parser_Example_HTTP(t *testing.T) {
her, err := here.Dir(".")
r.NoError(err)
res, err := parser.Parse(her)
res, err := Parse(her)
r.NoError(err)
files, err := res.Files()

185
parser/source.go Normal file
View File

@ -0,0 +1,185 @@
package parser
import (
"fmt"
"go/ast"
"go/token"
"path/filepath"
"strconv"
"sync"
"github.com/markbates/pkger/here"
)
type Source struct {
Abs string // full path on disk to file
Path here.Path
Here here.Info
}
type ParsedSource struct {
Source
FileSet *token.FileSet
Ast *ast.File
decls map[string]Decls
once sync.Once
err error
}
func (p *ParsedSource) Parse() error {
(&p.once).Do(func() {
p.err = p.parse()
})
return p.err
}
func (p *ParsedSource) value(node ast.Node) (string, error) {
var s string
switch x := node.(type) {
case *ast.BasicLit:
s = x.Value
case *ast.Ident:
s = x.Name
}
return strconv.Unquote(s)
}
func (p *ParsedSource) parse() error {
p.decls = map[string]Decls{}
var fn walker = func(node ast.Node) bool {
ce, ok := node.(*ast.CallExpr)
if !ok {
return true
}
sel, ok := ce.Fun.(*ast.SelectorExpr)
if !ok {
return true
}
pkg, ok := sel.X.(*ast.Ident)
if !ok {
return true
}
if pkg.Name != "pkger" {
return true
}
var fn func(f File, pos token.Position, value string) Decl
name := sel.Sel.Name
switch name {
case "MkdirAll":
fn = func(f File, pos token.Position, value string) Decl {
return MkdirAllDecl{
file: &f,
pos: pos,
value: value,
}
}
case "Create":
fn = func(f File, pos token.Position, value string) Decl {
return CreateDecl{
file: &f,
pos: pos,
value: value,
}
}
case "Stat":
fn = func(f File, pos token.Position, value string) Decl {
return StatDecl{
file: &f,
pos: pos,
value: value,
}
}
case "Open":
fn = func(f File, pos token.Position, value string) Decl {
return OpenDecl{
file: &f,
pos: pos,
value: value,
}
}
case "Dir":
fn = func(f File, pos token.Position, value string) Decl {
return HTTPDecl{
file: &f,
pos: pos,
value: value,
}
}
case "Walk":
fn = func(f File, pos token.Position, value string) Decl {
return WalkDecl{
file: &f,
pos: pos,
value: value,
}
}
default:
return true
}
if len(ce.Args) < 1 {
p.err = fmt.Errorf("declarations require at least one argument")
return false
}
n := ce.Args[0]
val, err := p.value(n)
if err != nil {
p.err = fmt.Errorf("%w: %s", err, n)
return false
}
info, err := here.Dir(filepath.Dir(p.Abs))
if err != nil {
p.err = fmt.Errorf("%w: %s", err, p.Abs)
return false
}
pt, err := info.Parse(val)
if err != nil {
p.err = fmt.Errorf("%w: %s", err, p.Abs)
return false
}
if pt.Pkg != info.Module.Path {
info, err = here.Package(pt.Pkg)
if err != nil {
p.err = fmt.Errorf("%w: %s", err, p.Abs)
return false
}
}
f := File{
Abs: filepath.Join(info.Module.Dir, pt.Name),
Here: info,
Path: pt,
}
p.decls[name] = append(p.decls[name], fn(f, p.FileSet.Position(n.Pos()), val))
return true
}
ast.Walk(fn, p.Ast)
return nil
}
func (p *ParsedSource) DeclsMap() (map[string]Decls, error) {
err := p.Parse()
return p.decls, err
}
// wrap a function to fulfill ast.Visitor interface
type walker func(ast.Node) bool
func (w walker) Visit(node ast.Node) ast.Visitor {
if w(node) {
return w
}
return nil
}

View File

@ -2,6 +2,7 @@ package parser
import (
"encoding/json"
"fmt"
"go/token"
"os"
)
@ -15,8 +16,7 @@ type StatDecl struct {
}
func (d StatDecl) String() string {
b, _ := json.Marshal(d)
return string(b)
return fmt.Sprintf("pkger.Stat(%q)", d.value)
}
func (d StatDecl) MarshalJSON() ([]byte, error) {

View File

@ -1,386 +0,0 @@
package parser
import (
"fmt"
"go/ast"
"go/token"
"path/filepath"
"strconv"
"github.com/markbates/pkger/here"
)
type visitor func(node ast.Node) (w ast.Visitor)
func (v visitor) Visit(node ast.Node) ast.Visitor {
return v(node)
}
// inspired by https://gist.github.com/cryptix/d1b129361cea51a59af2
type file struct {
fset *token.FileSet
astFile *ast.File
filename string
decls Decls
current here.Info
}
func (f *file) walk(fn func(ast.Node) bool) {
ast.Walk(walker(fn), f.astFile)
}
func (f *file) find() (Decls, error) {
// --- virtual calls first ---
if err := f.findMkdirAllCalls(); err != nil {
return nil, err
}
if err := f.findCreateCalls(); err != nil {
return nil, err
}
// -- physical calls second ---
if err := f.findStatCalls(); err != nil {
return nil, err
}
if err := f.findOpenCalls(); err != nil {
return nil, err
}
if err := f.findHTTPCalls(); err != nil {
return nil, err
}
if err := f.findWalkCalls(); err != nil {
return nil, err
}
return f.decls, nil
}
func (f *file) asValue(node ast.Node) (string, error) {
var s string
switch x := node.(type) {
case *ast.BasicLit:
s = x.Value
case *ast.Ident:
s = x.Name
}
return strconv.Unquote(s)
}
func (f *file) findMkdirAllCalls() error {
var err error
f.walk(func(node ast.Node) bool {
ce, ok := node.(*ast.CallExpr)
if !ok {
return true
}
exists := isPkgDot(ce.Fun, "pkger", "MkdirAll")
if !(exists) || len(ce.Args) != 2 {
return true
}
n := ce.Args[0]
s, err := f.asValue(n)
if err != nil {
return false
}
info, err := here.Dir(filepath.Dir(f.filename))
if err != nil {
return false
}
pf := &File{
Abs: f.filename,
Here: info,
Path: here.Path{
Pkg: info.Module.Path,
Name: s,
},
}
decl := MkdirAllDecl{
file: pf,
pos: f.fset.Position(n.Pos()),
value: s,
}
f.decls = append(f.decls, decl)
return true
})
return err
}
func (f *file) findStatCalls() error {
var err error
f.walk(func(node ast.Node) bool {
ce, ok := node.(*ast.CallExpr)
if !ok {
return true
}
exists := isPkgDot(ce.Fun, "pkger", "Stat")
if !(exists) || len(ce.Args) != 1 {
return true
}
n := ce.Args[0]
s, err := f.asValue(n)
if err != nil {
return false
}
info, err := here.Dir(filepath.Dir(f.filename))
if err != nil {
return false
}
pt, err := info.Parse(s)
if err != nil {
return false
}
fmt.Println(">>>TODO parser/visitor.go:151: pt ", pt)
fmt.Println(">>>TODO parser/visitor.go:151: info.Module ", info.Module)
if pt.Pkg != info.Module.Path {
info, err = here.Package(pt.Pkg)
if err != nil {
return false
}
}
pf := &File{
Abs: f.filename,
Here: info,
Path: pt,
}
decl := StatDecl{
file: pf,
pos: f.fset.Position(n.Pos()),
value: s,
}
f.decls = append(f.decls, decl)
return true
})
return err
}
func (f *file) findCreateCalls() error {
var err error
f.walk(func(node ast.Node) bool {
ce, ok := node.(*ast.CallExpr)
if !ok {
return true
}
exists := isPkgDot(ce.Fun, "pkger", "Create")
if !(exists) || len(ce.Args) != 1 {
return true
}
n := ce.Args[0]
s, err := f.asValue(n)
if err != nil {
return false
}
info, err := here.Dir(filepath.Dir(f.filename))
if err != nil {
return false
}
pf := &File{
Abs: f.filename,
Here: info,
Path: here.Path{
Pkg: info.Module.Path,
Name: s,
},
}
decl := CreateDecl{
file: pf,
pos: f.fset.Position(n.Pos()),
value: s,
}
f.decls = append(f.decls, decl)
return true
})
return err
}
func (f *file) findOpenCalls() error {
var err error
f.walk(func(node ast.Node) bool {
ce, ok := node.(*ast.CallExpr)
if !ok {
return true
}
exists := isPkgDot(ce.Fun, "pkger", "Open")
if !(exists) || len(ce.Args) != 1 {
return true
}
n := ce.Args[0]
s, err := f.asValue(n)
if err != nil {
return false
}
info, err := here.Dir(filepath.Dir(f.filename))
if err != nil {
return false
}
pf := &File{
Abs: f.filename,
Here: info,
Path: here.Path{
Pkg: info.Module.Path,
Name: s,
},
}
decl := OpenDecl{
file: pf,
pos: f.fset.Position(n.Pos()),
value: s,
}
f.decls = append(f.decls, decl)
return true
})
return err
}
func (f *file) findWalkCalls() error {
var err error
f.walk(func(node ast.Node) bool {
ce, ok := node.(*ast.CallExpr)
if !ok {
return true
}
exists := isPkgDot(ce.Fun, "pkger", "Walk")
if !(exists) || len(ce.Args) != 2 {
return true
}
n := ce.Args[0]
s, err := f.asValue(n)
if err != nil {
err = err
return false
}
dir := filepath.Dir(f.filename)
info, err := here.Dir(dir)
if err != nil {
err = err
return false
}
pf := &File{
Abs: f.filename,
Here: info,
Path: here.Path{
Pkg: info.Module.Path,
Name: s,
},
}
decl := WalkDecl{
file: pf,
pos: f.fset.Position(n.Pos()),
value: s,
}
f.decls = append(f.decls, decl)
return true
})
return err
}
func (f *file) findHTTPCalls() error {
var err error
f.walk(func(node ast.Node) bool {
ce, ok := node.(*ast.CallExpr)
if !ok {
return true
}
exists := isPkgDot(ce.Fun, "pkger", "Dir")
if !(exists) || len(ce.Args) != 1 {
return true
}
n := ce.Args[0]
s, err := f.asValue(n)
if err != nil {
return false
}
info, err := here.Dir(filepath.Dir(f.filename))
if err != nil {
return false
}
pf := &File{
Abs: f.filename,
Here: info,
Path: here.Path{
Pkg: info.Module.Path,
Name: s,
},
}
pos := f.fset.Position(n.Pos())
decl := HTTPDecl{
file: pf,
pos: pos,
value: s,
}
f.decls = append(f.decls, decl)
return true
})
return err
}
// helpers
// =======
func isPkgDot(expr ast.Expr, pkg, name string) bool {
sel, ok := expr.(*ast.SelectorExpr)
return ok && isIdent(sel.X, pkg) && isIdent(sel.Sel, name)
}
func isIdent(expr ast.Expr, ident string) bool {
id, ok := expr.(*ast.Ident)
return ok && id.Name == ident
}
// wrap a function to fulfill ast.Visitor interface
type walker func(ast.Node) bool
func (w walker) Visit(node ast.Node) ast.Visitor {
if w(node) {
return w
}
return nil
}