diff --git a/cmd/pkger/cmds/pack.go b/cmd/pkger/cmds/pack.go index 161c11b..d99617f 100644 --- a/cmd/pkger/cmds/pack.go +++ b/cmd/pkger/cmds/pack.go @@ -7,9 +7,8 @@ import ( "sort" "github.com/markbates/pkger" - "github.com/markbates/pkger/here" "github.com/markbates/pkger/parser" - "github.com/markbates/pkger/pkging/stdos" + "github.com/markbates/pkger/pkging/pkgutil" ) const outName = "pkged.go" @@ -118,7 +117,7 @@ func (e *packCmd) Flags() *flag.FlagSet { return e.FlagSet } -func Package(out string, paths []here.Path) error { +func Package(out string, decls parser.Decls) error { os.RemoveAll(out) f, err := os.Create(out) @@ -135,17 +134,12 @@ func Package(out string, paths []here.Path) error { fmt.Fprintf(f, "import \"github.com/markbates/pkger\"\n\n") fmt.Fprintf(f, "import \"github.com/markbates/pkger/pkging/mem\"\n\n") fmt.Fprintf(f, "// packing:\n") - for _, p := range paths { - fmt.Fprintf(f, "// %s\n", p) - } + // for _, p := range paths { + // fmt.Fprintf(f, "// %s\n", p) + // } fmt.Fprintf(f, "\nvar _ = pkger.Apply(mem.UnmarshalEmbed([]byte(`") - disk, err := stdos.New(c) - if err != nil { - return err - } - - if err := disk.Stuff(f, paths); err != nil { + if err := pkgutil.Stuff(f, c, decls); err != nil { return err } diff --git a/examples/walk/pkger/Makefile b/examples/walk/pkger/Makefile index d12579d..dd14cf3 100644 --- a/examples/walk/pkger/Makefile +++ b/examples/walk/pkger/Makefile @@ -1,5 +1,4 @@ default: - go get github.com/markbates/pkger/cmd/pkger pkger GOOS=linux go build -v -o example docker build -t pkger:example . diff --git a/examples/walk/pkger/go.mod b/examples/walk/pkger/go.mod index 44a475e..4c41bc1 100644 --- a/examples/walk/pkger/go.mod +++ b/examples/walk/pkger/go.mod @@ -2,6 +2,6 @@ module app go 1.13 -require github.com/markbates/pkger v0.1.0 +require github.com/markbates/pkger v0.2.0 replace github.com/markbates/pkger => ../../../ diff --git a/parser/decl.go b/parser/decl.go new file mode 100644 index 0000000..762893d --- /dev/null +++ b/parser/decl.go @@ -0,0 +1,47 @@ +package parser + +import ( + "go/token" + "sort" +) + +type Decl interface { + File() (*File, error) + Pos() (token.Pos, error) + Value() (string, error) +} + +type Filer interface { + Files() ([]*File, error) +} + +type Decls []Decl + +func (decls Decls) Files() ([]*File, error) { + m := map[string]*File{} + + for _, d := range decls { + fl, ok := d.(Filer) + if !ok { + continue + } + + files, err := fl.Files() + if err != nil { + return nil, err + } + + for _, f := range files { + m[f.Path.String()] = f + } + } + + var files []*File + for _, f := range m { + files = append(files, f) + } + sort.Slice(files, func(i, j int) bool { + return files[i].Path.String() < files[j].Path.String() + }) + return files, nil +} diff --git a/parser/file.go b/parser/file.go index e1d738b..8149178 100644 --- a/parser/file.go +++ b/parser/file.go @@ -1,13 +1,27 @@ package parser import ( + "encoding/json" "go/ast" "go/parser" "go/token" "io" "io/ioutil" + + "github.com/markbates/pkger/here" ) +type File struct { + Abs string // full path on disk to file + Path here.Path + Here here.Info +} + +func (f File) String() string { + b, _ := json.MarshalIndent(f, "", " ") + return string(b) +} + type parsedFile struct { File string FileSet *token.FileSet diff --git a/parser/open.go b/parser/open.go new file mode 100644 index 0000000..53b20cf --- /dev/null +++ b/parser/open.go @@ -0,0 +1,77 @@ +package parser + +import ( + "go/token" + "os" + "path/filepath" + + "github.com/markbates/pkger" + "github.com/markbates/pkger/here" +) + +var _ Decl = OpenDecl{} + +type OpenDecl struct { + file *File + pos token.Pos + value string +} + +func (d OpenDecl) File() (*File, error) { + if d.file == nil { + return nil, os.ErrNotExist + } + return d.file, nil +} + +func (d OpenDecl) Pos() (token.Pos, error) { + if d.pos <= 0 { + return -1, os.ErrNotExist + } + return d.pos, nil +} + +func (d OpenDecl) Value() (string, error) { + if d.value == "" { + return "", os.ErrNotExist + } + return d.value, nil +} + +func (d OpenDecl) Files() ([]*File, error) { + + pt, err := pkger.Parse(d.value) + if err != nil { + return nil, err + } + + her, err := here.Package(pt.Pkg) + if err != nil { + return nil, err + } + + fp := filepath.Join(her.Dir, pt.Name) + + osf, err := os.Stat(fp) + if err != nil { + return nil, err + } + + if osf.IsDir() { + wd := WalkDecl{ + file: d.file, + pos: d.pos, + value: d.value, + } + return wd.Files() + } + + var files []*File + files = append(files, &File{ + Abs: filepath.Join(her.Dir, pt.Name), + Path: pt, + Here: her, + }) + + return files, nil +} diff --git a/parser/parser.go b/parser/parser.go index 29fd89f..d5104ed 100644 --- a/parser/parser.go +++ b/parser/parser.go @@ -5,18 +5,13 @@ import ( "go/parser" "go/token" "os" - "path/filepath" - "sort" - "strings" "github.com/markbates/pkger/here" - "github.com/markbates/pkger/pkging/stdos" ) // var DefaultIgnoredFolders = []string{".", "_", "vendor", "node_modules", "_fixtures", "testdata"} -func Parse(her here.Info) ([]here.Path, error) { - +func Parse(her here.Info) (Decls, error) { src, err := fromSource(her) if err != nil { return nil, err @@ -25,7 +20,7 @@ func Parse(her here.Info) ([]here.Path, error) { return src, nil } -func fromSource(her here.Info) ([]here.Path, error) { +func fromSource(her here.Info) (Decls, error) { root := her.Dir fi, err := os.Stat(root) if err != nil { @@ -41,78 +36,24 @@ func fromSource(her here.Info) ([]here.Path, error) { if err != nil { return nil, err } - pm := map[string]here.Path{} + + var decls Decls + for _, pkg := range pkgs { - for _, pf := range pkg.Files { + for name, pf := range pkg.Files { f := &file{ fset: fset, astFile: pf, - filename: pf.Name.Name, - decls: map[string]string{}, + filename: name, } x, err := f.find() if err != nil { return nil, err } - for _, dl := range x { - pt, err := her.Parse(dl) - if err != nil { - return nil, err - } - res, err := fromPath(pt) - if err != nil { - return nil, err - } - for _, p := range res { - pm[p.String()] = p - } - } + decls = append(decls, x...) } } - var paths []here.Path - for _, v := range pm { - paths = append(paths, v) - } - - sort.Slice(paths, func(i, j int) bool { - return paths[i].String() < paths[j].String() - }) - - return paths, nil -} - -func fromPath(pt here.Path) ([]here.Path, error) { - var paths []here.Path - - her, err := here.Package(pt.Pkg) - if err != nil { - return nil, err - } - - pkg, err := stdos.New(her) - if err != nil { - return nil, err - } - - root := here.Path{ - Pkg: pt.Pkg, - Name: strings.Replace(filepath.Dir(pt.Name), "\\", "/", -1), - } - paths = append(paths, root) - err = pkg.Walk(pt.Name, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - p, err := her.Parse(path) - if err != nil { - return err - } - paths = append(paths, p) - - return nil - }) - - return paths, nil + return decls, nil } diff --git a/parser/parser_test.go b/parser/parser_test.go index f7a169c..bc98f9c 100644 --- a/parser/parser_test.go +++ b/parser/parser_test.go @@ -1,9 +1,11 @@ -package parser +package parser_test import ( + "fmt" "sort" "testing" + "github.com/markbates/pkger/parser" "github.com/markbates/pkger/pkging/pkgtest" "github.com/stretchr/testify/require" ) @@ -14,15 +16,22 @@ func Test_Parser_App(t *testing.T) { app, err := pkgtest.App() r.NoError(err) - res, err := Parse(app.Info) + res, err := parser.Parse(app.Info) r.NoError(err) - act := make([]string, len(res)) - for i := 0; i < len(res); i++ { - act[i] = res[i].String() + files, err := res.Files() + r.NoError(err) + + act := make([]string, len(files)) + for i := 0; i < len(files); i++ { + act[i] = files[i].Path.String() } sort.Strings(act) + + for _, a := range act { + fmt.Println(a) + } r.Equal(app.Paths.Parser, act) } diff --git a/parser/visitor.go b/parser/visitor.go index f61ee1c..d6ea0f4 100644 --- a/parser/visitor.go +++ b/parser/visitor.go @@ -3,8 +3,10 @@ package parser import ( "go/ast" "go/token" - "sort" + "path/filepath" "strconv" + + "github.com/markbates/pkger/here" ) type visitor func(node ast.Node) (w ast.Visitor) @@ -18,17 +20,14 @@ type file struct { fset *token.FileSet astFile *ast.File filename string - decls map[string]string + decls Decls } func (f *file) walk(fn func(ast.Node) bool) { ast.Walk(walker(fn), f.astFile) } -func (f *file) find() ([]string, error) { - // if err := f.findDecals(); err != nil { - // return nil, err - // } +func (f *file) find() (Decls, error) { if err := f.findOpenCalls(); err != nil { return nil, err } @@ -37,88 +36,20 @@ func (f *file) find() ([]string, error) { return nil, err } - if err := f.findImportCalls(); err != nil { - return nil, err - } - - var paths []string - for _, p := range f.decls { - paths = append(paths, p) - } - sort.Slice(paths, func(a, b int) bool { - return paths[a] < paths[b] - }) - return paths, nil + return f.decls, nil } -func (f *file) findDecals() error { - // iterate over all declarations - for _, d := range f.astFile.Decls { +func (f *file) asValue(node ast.Node) (string, error) { + var s string - // log.Printf("#%d Decl: %+v\n", i, d) - - // only interested in generic declarations - if genDecl, ok := d.(*ast.GenDecl); ok { - - // handle const's and vars - if genDecl.Tok == token.CONST || genDecl.Tok == token.VAR { - - // there may be multiple - // i.e. const ( ... ) - for _, cDecl := range genDecl.Specs { - - // havn't find another kind of spec then value but better check - if vSpec, ok := cDecl.(*ast.ValueSpec); ok { - // log.Printf("const ValueSpec: %+v\n", vSpec) - - // iterate over Name/Value pair - for i := 0; i < len(vSpec.Names); i++ { - // TODO: only basic literals work currently - if i > len(vSpec.Values) || len(vSpec.Values) == 0 { - break - } - switch v := vSpec.Values[i].(type) { - case *ast.BasicLit: - - if e := f.addNode(v); e != nil { - return e - } - // f.decls[vSpec.Names[i].Name] = v.Value - default: - // log.Printf("Name: %s - Unsupported ValueSpec: %+v\n", vSpec.Names[i].Name, v) - } - } - } - } - } - - } - } - - return nil -} - -func (f *file) addNode(node ast.Node) error { switch x := node.(type) { case *ast.BasicLit: - return f.add(x.Value) + s = x.Value case *ast.Ident: - return f.add(x.Name) - default: + s = x.Name } - return nil -} -func (f *file) add(s string) error { - s, err := strconv.Unquote(s) - if err != nil { - return err - } - if _, ok := f.decls[s]; !ok { - // fmt.Println(">>>TODO parser/visitor.go:98: s ", s) - f.decls[s] = s - } - return nil + return strconv.Unquote(s) } func (f *file) findOpenCalls() error { @@ -134,12 +65,30 @@ func (f *file) findOpenCalls() error { return true } - // fmt.Println(">>>TODO parser/visitor.go:138: findOpenCalls ", ce.Args[0]) - if e := f.addNode(ce.Args[0]); e != nil { - err = e + n := ce.Args[0] + + s, err := f.asValue(n) + if err != nil { return false } + info, err := here.Dir(filepath.Dir(f.filename)) + if err != nil { + return false + } + + pf := &File{ + Abs: f.filename, + Here: info, + } + + decl := OpenDecl{ + file: pf, + pos: n.Pos(), + value: s, + } + + f.decls = append(f.decls, decl) return true }) return err @@ -158,40 +107,30 @@ func (f *file) findWalkCalls() error { return true } - // fmt.Println(">>>TODO parser/visitor.go:138: findWalkCalls ", ce.Args[0]) - if e := f.addNode(ce.Args[0]); e != nil { - err = e + n := ce.Args[0] + + s, err := f.asValue(n) + if err != nil { return false } - return true - }) - return err -} + info, err := here.Dir(filepath.Dir(f.filename)) + if err != nil { + return false + } -func (f *file) findImportCalls() error { - var err error - f.walk(func(node ast.Node) bool { - // ce, ok := node.(*ast.ImportSpec) - // if !ok { - // return true - // } + pf := &File{ + Abs: f.filename, + Here: info, + } - // s, err := strconv.Unquote(ce.Path.Value) - // if err != nil { - // return false - // } - // fmt.Println(">>>TODO parser/visitor.go:215: s ", s) - // info, err := here.Package(s) - // if err != nil { - // return false - // } - // fmt.Println(">>>TODO parser/visitor.go:216: info ", info) - // res, err := Parse(info) - // if err != nil { - // return false - // } - // fmt.Println(">>>TODO parser/visitor.go:224: res ", res) + decl := WalkDecl{ + file: pf, + pos: n.Pos(), + value: s, + } + + f.decls = append(f.decls, decl) return true }) return err diff --git a/parser/walk.go b/parser/walk.go new file mode 100644 index 0000000..8ee84e6 --- /dev/null +++ b/parser/walk.go @@ -0,0 +1,78 @@ +package parser + +import ( + "go/token" + "os" + "path/filepath" + "strings" + + "github.com/markbates/pkger" + "github.com/markbates/pkger/here" +) + +var _ Decl = WalkDecl{} + +type WalkDecl struct { + file *File + pos token.Pos + value string +} + +func (d WalkDecl) File() (*File, error) { + if d.file == nil { + return nil, os.ErrNotExist + } + return d.file, nil +} + +func (d WalkDecl) Pos() (token.Pos, error) { + if d.pos <= 0 { + return -1, os.ErrNotExist + } + return d.pos, nil +} + +func (d WalkDecl) Value() (string, error) { + if d.value == "" { + return "", os.ErrNotExist + } + return d.value, nil +} + +func (d WalkDecl) Files() ([]*File, error) { + pt, err := pkger.Parse(d.value) + if err != nil { + return nil, err + } + + cur, err := here.Package(pt.Pkg) + if err != nil { + return nil, err + } + + root := filepath.Join(cur.Dir, pt.Name) + + var files []*File + err = filepath.Walk(root, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + n := strings.TrimPrefix(path, cur.Dir) + + pt, err := pkger.Parse(n) + if err != nil { + return err + } + pt.Pkg = cur.ImportPath + + files = append(files, &File{ + Abs: path, + Path: pt, + Here: cur, + }) + return nil + }) + + return files, err +} diff --git a/pkging/embed.go b/pkging/embed.go deleted file mode 100644 index 4f1c21b..0000000 --- a/pkging/embed.go +++ /dev/null @@ -1 +0,0 @@ -package pkging diff --git a/pkging/faces.go b/pkging/faces.go index f56e4cf..192ebb0 100644 --- a/pkging/faces.go +++ b/pkging/faces.go @@ -1,15 +1,5 @@ package pkging -import ( - "io" - - "github.com/markbates/pkger/here" -) - type Adder interface { Add(files ...File) error } - -type Stuffer interface { - Stuff(w io.Writer, paths []here.Path) error -} diff --git a/pkging/mem/embed_test.go b/pkging/mem/embed_test.go index 56ed778..f3fa114 100644 --- a/pkging/mem/embed_test.go +++ b/pkging/mem/embed_test.go @@ -3,12 +3,12 @@ package mem_test import ( "bytes" "os" - "sort" "testing" "github.com/markbates/pkger/parser" "github.com/markbates/pkger/pkging/mem" "github.com/markbates/pkger/pkging/pkgtest" + "github.com/markbates/pkger/pkging/pkgutil" "github.com/markbates/pkger/pkging/stdos" "github.com/stretchr/testify/require" ) @@ -19,12 +19,15 @@ func Test_Pkger_Embedding(t *testing.T) { app, err := pkgtest.App() r.NoError(err) - paths, err := parser.Parse(app.Info) + res, err := parser.Parse(app.Info) r.NoError(err) - ps := make([]string, len(paths)) - for i, p := range paths { - ps[i] = p.String() + files, err := res.Files() + r.NoError(err) + + ps := make([]string, len(files)) + for i, f := range files { + ps[i] = f.Path.String() } r.Equal(app.Paths.Parser, ps) @@ -53,63 +56,61 @@ func Test_Pkger_Embedding(t *testing.T) { }) r.NoError(err) - var res []string + var act []string err = base.Walk("/", func(path string, info os.FileInfo, err error) error { if err != nil { return err } - res = append(res, path) + act = append(act, path) return nil }) r.NoError(err) - r.Equal(app.Paths.Root, res) + r.Equal(app.Paths.Root, act) - res = []string{} + act = []string{} err = base.Walk("/public", func(path string, info os.FileInfo, err error) error { if err != nil { return err } - res = append(res, path) + act = append(act, path) return nil }) r.NoError(err) - r.Equal(app.Paths.Public, res) + r.Equal(app.Paths.Public, act) bb := &bytes.Buffer{} - err = disk.Stuff(bb, paths) + err = pkgutil.Stuff(bb, app.Info, res) r.NoError(err) pkg := &mem.Pkger{} err = pkg.UnmarshalEmbed(bb.Bytes()) r.NoError(err) - res = []string{} + act = []string{} err = pkg.Walk("/", func(path string, info os.FileInfo, err error) error { if err != nil { return err } - res = append(res, path) + act = append(act, path) return nil }) r.NoError(err) - exp := append(app.Paths.Public, "app:/") - sort.Strings(exp) - r.Equal(exp, res) + r.Equal(app.Paths.Public, act) - res = []string{} + act = []string{} err = pkg.Walk("/public", func(path string, info os.FileInfo, err error) error { if err != nil { return err } - res = append(res, path) + act = append(act, path) return nil }) r.NoError(err) - r.Equal(app.Paths.Public, res) + r.Equal(app.Paths.Public, act) } diff --git a/pkging/mem/mem.go b/pkging/mem/mem.go index 5b31e0b..f2eb499 100644 --- a/pkging/mem/mem.go +++ b/pkging/mem/mem.go @@ -58,12 +58,12 @@ func (p *Pkger) MarshalJSON() ([]byte, error) { infos[key] = info return true }) - - return json.Marshal(embed.Data{ + ed := embed.Data{ Infos: infos, Files: files, Here: p.Here, - }) + } + return json.Marshal(ed) } // UnmarshalJSON re-hydrates the *Pkger diff --git a/pkging/mem/mem_test.go b/pkging/mem/mem_test.go index 2daea58..3b8aed2 100644 --- a/pkging/mem/mem_test.go +++ b/pkging/mem/mem_test.go @@ -1,9 +1,10 @@ -package mem +package mem_test import ( "testing" "github.com/markbates/pkger/pkging" + "github.com/markbates/pkger/pkging/mem" "github.com/markbates/pkger/pkging/pkgtest" ) @@ -14,7 +15,7 @@ func Test_Pkger(t *testing.T) { return nil, err } - pkg, err := New(app.Info) + pkg, err := mem.New(app.Info) if err != nil { return nil, err } diff --git a/pkging/pkgtest/app.go b/pkging/pkgtest/app.go index e34f8ab..dd41085 100644 --- a/pkging/pkgtest/app.go +++ b/pkging/pkgtest/app.go @@ -65,6 +65,7 @@ func App() (AppDetails, error) { var rootPaths = []string{ "app:/", "app:/go.mod", + "app:/go.sum", "app:/main.go", "app:/public", "app:/public/images", @@ -86,13 +87,11 @@ var publicPaths = []string{ } var parserPaths = []string{ - "app:/", "app:/public", "app:/public/images", "app:/public/images/img1.png", "app:/public/images/img2.png", "app:/public/index.html", - "github.com/gobuffalo/buffalo:/", "github.com/gobuffalo/buffalo:/render", "github.com/gobuffalo/buffalo:/render/auto.go", "github.com/gobuffalo/buffalo:/render/auto_test.go", diff --git a/pkging/pkgutil/stuff.go b/pkging/pkgutil/stuff.go new file mode 100644 index 0000000..aeb6b2b --- /dev/null +++ b/pkging/pkgutil/stuff.go @@ -0,0 +1,85 @@ +package pkgutil + +import ( + "io" + "os" + + "github.com/markbates/pkger/here" + "github.com/markbates/pkger/parser" + "github.com/markbates/pkger/pkging/mem" + "github.com/markbates/pkger/pkging/stdos" +) + +func Stuff(w io.Writer, c here.Info, decls parser.Decls) error { + disk, err := stdos.New(c) + if err != nil { + return err + } + + pkg, err := mem.New(c) + if err != nil { + return err + } + + files, err := decls.Files() + if err != nil { + return err + } + + for _, pf := range files { + err = func() error { + df, err := disk.Open(pf.Path.String()) + if err != nil { + return err + } + defer df.Close() + + info, err := df.Stat() + if err != nil { + return err + } + + if err := pkg.Add(df); err != nil { + return err + } + + if !info.IsDir() { + return nil + } + + err = disk.Walk(df.Path().String(), func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + if info.IsDir() { + return nil + } + + f, err := disk.Open(path) + if err != nil { + return err + } + defer f.Close() + + if err := pkg.Add(f); err != nil { + return err + } + + return nil + }) + + return err + }() + + if err != nil { + return err + } + b, err := pkg.MarshalEmbed() + if err != nil { + return err + } + _, err = w.Write(b) + } + return nil +} diff --git a/pkging/stdos/stdos.go b/pkging/stdos/stdos.go index 85f6f77..787aac2 100644 --- a/pkging/stdos/stdos.go +++ b/pkging/stdos/stdos.go @@ -2,7 +2,6 @@ package stdos import ( "fmt" - "io" "os" "path/filepath" "strings" @@ -10,7 +9,6 @@ import ( "github.com/markbates/pkger/here" "github.com/markbates/pkger/internal/maps" "github.com/markbates/pkger/pkging" - "github.com/markbates/pkger/pkging/mem" ) var _ pkging.Pkger = &Pkger{} @@ -20,38 +18,6 @@ type Pkger struct { infos *maps.Infos } -func (disk *Pkger) Stuff(w io.Writer, paths []here.Path) error { - pkg, err := mem.New(disk.Here) - if err != nil { - return err - } - - for _, pt := range paths { - err = func() error { - f, err := disk.Open(pt.String()) - if err != nil { - return err - } - defer f.Close() - if err := pkg.Add(f); err != nil { - return err - } - - return nil - }() - if err != nil { - return err - } - } - - b, err := pkg.MarshalEmbed() - if err != nil { - return err - } - _, err = w.Write(b) - return err -} - // Abs returns an absolute representation of path. If the path is not absolute it will be joined with the current working directory to turn it into an absolute path. The absolute path name for a given file is not guaranteed to be unique. Abs calls Clean on the result. func (f *Pkger) Abs(p string) (string, error) { pt, err := f.Parse(p) diff --git a/pkging/stdos/stdos_test.go b/pkging/stdos/stdos_test.go index 0c60a9c..09629b1 100644 --- a/pkging/stdos/stdos_test.go +++ b/pkging/stdos/stdos_test.go @@ -1,4 +1,4 @@ -package stdos +package stdos_test import ( "io/ioutil" @@ -6,6 +6,7 @@ import ( "github.com/markbates/pkger/pkging" "github.com/markbates/pkger/pkging/pkgtest" + "github.com/markbates/pkger/pkging/stdos" ) func Test_Pkger(t *testing.T) { @@ -22,7 +23,7 @@ func Test_Pkger(t *testing.T) { app.Dir = dir - mypkging, err := New(app.Info) + mypkging, err := stdos.New(app.Info) if err != nil { return nil, err }