mirror
/
pkger
mirror of https://github.com/markbates/pkger.git
1
0
Fork 1
Code Issues Packages Projects Releases Wiki Activity

until i see the light

This commit is contained in:
Mark Bates 2019-10-21 10:27:04 -04:00
parent 31d0eef850
commit 2978645721
19 changed files with 413 additions and 273 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
cmd/pkger/cmds/pack.go
View File

@ -7,9 +7,8 @@ import (
"sort"
"github.com/markbates/pkger"
"github.com/markbates/pkger/here"
"github.com/markbates/pkger/parser"
"github.com/markbates/pkger/pkging/stdos"
"github.com/markbates/pkger/pkging/pkgutil"
)
const outName = "pkged.go"
@ -118,7 +117,7 @@ func (e *packCmd) Flags() *flag.FlagSet {
return e.FlagSet
}
func Package(out string, paths []here.Path) error {
func Package(out string, decls parser.Decls) error {
os.RemoveAll(out)
f, err := os.Create(out)
@ -135,17 +134,12 @@ func Package(out string, paths []here.Path) error {
fmt.Fprintf(f, "import \"github.com/markbates/pkger\"\n\n")
fmt.Fprintf(f, "import \"github.com/markbates/pkger/pkging/mem\"\n\n")
fmt.Fprintf(f, "// packing:\n")
for _, p := range paths {
fmt.Fprintf(f, "// %s\n", p)
}
// for _, p := range paths {
// fmt.Fprintf(f, "// %s\n", p)
// }
fmt.Fprintf(f, "\nvar _ = pkger.Apply(mem.UnmarshalEmbed([]byte(`")
disk, err := stdos.New(c)
if err != nil {
return err
}
if err := disk.Stuff(f, paths); err != nil {
if err := pkgutil.Stuff(f, c, decls); err != nil {
return err
}

1
examples/walk/pkger/Makefile
View File

@ -1,5 +1,4 @@
default:
go get github.com/markbates/pkger/cmd/pkger
pkger
GOOS=linux go build -v -o example
docker build -t pkger:example .

2
examples/walk/pkger/go.mod
View File

@ -2,6 +2,6 @@ module app
go 1.13
require github.com/markbates/pkger v0.1.0
require github.com/markbates/pkger v0.2.0
replace github.com/markbates/pkger => ../../../

47
parser/decl.go Normal file
View File

@ -0,0 +1,47 @@
package parser
import (
"go/token"
"sort"
)
type Decl interface {
File() (*File, error)
Pos() (token.Pos, error)
Value() (string, error)
}
type Filer interface {
Files() ([]*File, error)
}
type Decls []Decl
func (decls Decls) Files() ([]*File, error) {
m := map[string]*File{}
for _, d := range decls {
fl, ok := d.(Filer)
if !ok {
continue
}
files, err := fl.Files()
if err != nil {
return nil, err
}
for _, f := range files {
m[f.Path.String()] = f
}
}
var files []*File
for _, f := range m {
files = append(files, f)
}
sort.Slice(files, func(i, j int) bool {
return files[i].Path.String() < files[j].Path.String()
})
return files, nil
}

14
parser/file.go
View File

@ -1,13 +1,27 @@
package parser
import (
"encoding/json"
"go/ast"
"go/parser"
"go/token"
"io"
"io/ioutil"
"github.com/markbates/pkger/here"
)
type File struct {
Abs string // full path on disk to file
Path here.Path
Here here.Info
}
func (f File) String() string {
b, _ := json.MarshalIndent(f, "", " ")
return string(b)
}
type parsedFile struct {
File string
FileSet *token.FileSet

77
parser/open.go Normal file
View File

@ -0,0 +1,77 @@
package parser
import (
"go/token"
"os"
"path/filepath"
"github.com/markbates/pkger"
"github.com/markbates/pkger/here"
)
var _ Decl = OpenDecl{}
type OpenDecl struct {
file *File
pos token.Pos
value string
}
func (d OpenDecl) File() (*File, error) {
if d.file == nil {
return nil, os.ErrNotExist
}
return d.file, nil
}
func (d OpenDecl) Pos() (token.Pos, error) {
if d.pos <= 0 {
return -1, os.ErrNotExist
}
return d.pos, nil
}
func (d OpenDecl) Value() (string, error) {
if d.value == "" {
return "", os.ErrNotExist
}
return d.value, nil
}
func (d OpenDecl) Files() ([]*File, error) {
pt, err := pkger.Parse(d.value)
if err != nil {
return nil, err
}
her, err := here.Package(pt.Pkg)
if err != nil {
return nil, err
}
fp := filepath.Join(her.Dir, pt.Name)
osf, err := os.Stat(fp)
if err != nil {
return nil, err
}
if osf.IsDir() {
wd := WalkDecl{
file: d.file,
pos: d.pos,
value: d.value,
}
return wd.Files()
}
var files []*File
files = append(files, &File{
Abs: filepath.Join(her.Dir, pt.Name),
Path: pt,
Here: her,
})
return files, nil
}

77
parser/parser.go
View File

@ -5,18 +5,13 @@ import (
"go/parser"
"go/token"
"os"
"path/filepath"
"sort"
"strings"
"github.com/markbates/pkger/here"
"github.com/markbates/pkger/pkging/stdos"
)
// var DefaultIgnoredFolders = []string{".", "_", "vendor", "node_modules", "_fixtures", "testdata"}
func Parse(her here.Info) ([]here.Path, error) {
func Parse(her here.Info) (Decls, error) {
src, err := fromSource(her)
if err != nil {
return nil, err
@ -25,7 +20,7 @@ func Parse(her here.Info) ([]here.Path, error) {
return src, nil
}
func fromSource(her here.Info) ([]here.Path, error) {
func fromSource(her here.Info) (Decls, error) {
root := her.Dir
fi, err := os.Stat(root)
if err != nil {
@ -41,78 +36,24 @@ func fromSource(her here.Info) ([]here.Path, error) {
if err != nil {
return nil, err
}
pm := map[string]here.Path{}
var decls Decls
for _, pkg := range pkgs {
for _, pf := range pkg.Files {
for name, pf := range pkg.Files {
f := &file{
fset: fset,
astFile: pf,
filename: pf.Name.Name,
decls: map[string]string{},
filename: name,
}
x, err := f.find()
if err != nil {
return nil, err
}
for _, dl := range x {
pt, err := her.Parse(dl)
if err != nil {
return nil, err
decls = append(decls, x...)
}
res, err := fromPath(pt)
if err != nil {
return nil, err
}
for _, p := range res {
pm[p.String()] = p
}
}
}
}
var paths []here.Path
for _, v := range pm {
paths = append(paths, v)
}
sort.Slice(paths, func(i, j int) bool {
return paths[i].String() < paths[j].String()
})
return paths, nil
}
func fromPath(pt here.Path) ([]here.Path, error) {
var paths []here.Path
her, err := here.Package(pt.Pkg)
if err != nil {
return nil, err
}
pkg, err := stdos.New(her)
if err != nil {
return nil, err
}
root := here.Path{
Pkg: pt.Pkg,
Name: strings.Replace(filepath.Dir(pt.Name), "\\", "/", -1),
}
paths = append(paths, root)
err = pkg.Walk(pt.Name, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
p, err := her.Parse(path)
if err != nil {
return err
}
paths = append(paths, p)
return nil
})
return paths, nil
return decls, nil
}

19
parser/parser_test.go
View File

@ -1,9 +1,11 @@
package parser
package parser_test
import (
"fmt"
"sort"
"testing"
"github.com/markbates/pkger/parser"
"github.com/markbates/pkger/pkging/pkgtest"
"github.com/stretchr/testify/require"
)
@ -14,15 +16,22 @@ func Test_Parser_App(t *testing.T) {
app, err := pkgtest.App()
r.NoError(err)
res, err := Parse(app.Info)
res, err := parser.Parse(app.Info)
r.NoError(err)
act := make([]string, len(res))
for i := 0; i < len(res); i++ {
act[i] = res[i].String()
files, err := res.Files()
r.NoError(err)
act := make([]string, len(files))
for i := 0; i < len(files); i++ {
act[i] = files[i].Path.String()
}
sort.Strings(act)
for _, a := range act {
fmt.Println(a)
}
r.Equal(app.Paths.Parser, act)
}

163
parser/visitor.go
View File

@ -3,8 +3,10 @@ package parser
import (
"go/ast"
"go/token"
"sort"
"path/filepath"
"strconv"
"github.com/markbates/pkger/here"
)
type visitor func(node ast.Node) (w ast.Visitor)
@ -18,17 +20,14 @@ type file struct {
fset *token.FileSet
astFile *ast.File
filename string
decls map[string]string
decls Decls
}
func (f *file) walk(fn func(ast.Node) bool) {
ast.Walk(walker(fn), f.astFile)
}
func (f *file) find() ([]string, error) {
// if err := f.findDecals(); err != nil {
// return nil, err
// }
func (f *file) find() (Decls, error) {
if err := f.findOpenCalls(); err != nil {
return nil, err
}
@ -37,88 +36,20 @@ func (f *file) find() ([]string, error) {
return nil, err
}
if err := f.findImportCalls(); err != nil {
return nil, err
}
var paths []string
for _, p := range f.decls {
paths = append(paths, p)
}
sort.Slice(paths, func(a, b int) bool {
return paths[a] < paths[b]
})
return paths, nil
return f.decls, nil
}
func (f *file) findDecals() error {
// iterate over all declarations
for _, d := range f.astFile.Decls {
func (f *file) asValue(node ast.Node) (string, error) {
var s string
// log.Printf("#%d Decl: %+v\n", i, d)
// only interested in generic declarations
if genDecl, ok := d.(*ast.GenDecl); ok {
// handle const's and vars
if genDecl.Tok == token.CONST || genDecl.Tok == token.VAR {
// there may be multiple
// i.e. const ( ... )
for _, cDecl := range genDecl.Specs {
// havn't find another kind of spec then value but better check
if vSpec, ok := cDecl.(*ast.ValueSpec); ok {
// log.Printf("const ValueSpec: %+v\n", vSpec)
// iterate over Name/Value pair
for i := 0; i < len(vSpec.Names); i++ {
// TODO: only basic literals work currently
if i > len(vSpec.Values) || len(vSpec.Values) == 0 {
break
}
switch v := vSpec.Values[i].(type) {
case *ast.BasicLit:
if e := f.addNode(v); e != nil {
return e
}
// f.decls[vSpec.Names[i].Name] = v.Value
default:
// log.Printf("Name: %s - Unsupported ValueSpec: %+v\n", vSpec.Names[i].Name, v)
}
}
}
}
}
}
}
return nil
}
func (f *file) addNode(node ast.Node) error {
switch x := node.(type) {
case *ast.BasicLit:
return f.add(x.Value)
s = x.Value
case *ast.Ident:
return f.add(x.Name)
default:
s = x.Name
}
return nil
}
func (f *file) add(s string) error {
s, err := strconv.Unquote(s)
if err != nil {
return err
}
if _, ok := f.decls[s]; !ok {
// fmt.Println(">>>TODO parser/visitor.go:98: s ", s)
f.decls[s] = s
}
return nil
return strconv.Unquote(s)
}
func (f *file) findOpenCalls() error {
@ -134,12 +65,30 @@ func (f *file) findOpenCalls() error {
return true
}
// fmt.Println(">>>TODO parser/visitor.go:138: findOpenCalls ", ce.Args[0])
if e := f.addNode(ce.Args[0]); e != nil {
err = e
n := ce.Args[0]
s, err := f.asValue(n)
if err != nil {
return false
}
info, err := here.Dir(filepath.Dir(f.filename))
if err != nil {
return false
}
pf := &File{
Abs: f.filename,
Here: info,
}
decl := OpenDecl{
file: pf,
pos: n.Pos(),
value: s,
}
f.decls = append(f.decls, decl)
return true
})
return err
@ -158,40 +107,30 @@ func (f *file) findWalkCalls() error {
return true
}
// fmt.Println(">>>TODO parser/visitor.go:138: findWalkCalls ", ce.Args[0])
if e := f.addNode(ce.Args[0]); e != nil {
err = e
n := ce.Args[0]
s, err := f.asValue(n)
if err != nil {
return false
}
return true
})
return err
}
info, err := here.Dir(filepath.Dir(f.filename))
if err != nil {
return false
}
func (f *file) findImportCalls() error {
var err error
f.walk(func(node ast.Node) bool {
// ce, ok := node.(*ast.ImportSpec)
// if !ok {
// return true
// }
pf := &File{
Abs: f.filename,
Here: info,
}
// s, err := strconv.Unquote(ce.Path.Value)
// if err != nil {
// return false
// }
// fmt.Println(">>>TODO parser/visitor.go:215: s ", s)
// info, err := here.Package(s)
// if err != nil {
// return false
// }
// fmt.Println(">>>TODO parser/visitor.go:216: info ", info)
// res, err := Parse(info)
// if err != nil {
// return false
// }
// fmt.Println(">>>TODO parser/visitor.go:224: res ", res)
decl := WalkDecl{
file: pf,
pos: n.Pos(),
value: s,
}
f.decls = append(f.decls, decl)
return true
})
return err

78
parser/walk.go Normal file
View File

@ -0,0 +1,78 @@
package parser
import (
"go/token"
"os"
"path/filepath"
"strings"
"github.com/markbates/pkger"
"github.com/markbates/pkger/here"
)
var _ Decl = WalkDecl{}
type WalkDecl struct {
file *File
pos token.Pos
value string
}
func (d WalkDecl) File() (*File, error) {
if d.file == nil {
return nil, os.ErrNotExist
}
return d.file, nil
}
func (d WalkDecl) Pos() (token.Pos, error) {
if d.pos <= 0 {
return -1, os.ErrNotExist
}
return d.pos, nil
}
func (d WalkDecl) Value() (string, error) {
if d.value == "" {
return "", os.ErrNotExist
}
return d.value, nil
}
func (d WalkDecl) Files() ([]*File, error) {
pt, err := pkger.Parse(d.value)
if err != nil {
return nil, err
}
cur, err := here.Package(pt.Pkg)
if err != nil {
return nil, err
}
root := filepath.Join(cur.Dir, pt.Name)
var files []*File
err = filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
n := strings.TrimPrefix(path, cur.Dir)
pt, err := pkger.Parse(n)
if err != nil {
return err
}
pt.Pkg = cur.ImportPath
files = append(files, &File{
Abs: path,
Path: pt,
Here: cur,
})
return nil
})
return files, err
}

1
pkging/embed.go
View File

@ -1 +0,0 @@
package pkging

10
pkging/faces.go
View File

@ -1,15 +1,5 @@
package pkging
import (
"io"
"github.com/markbates/pkger/here"
)
type Adder interface {
Add(files ...File) error
}
type Stuffer interface {
Stuff(w io.Writer, paths []here.Path) error
}

41
pkging/mem/embed_test.go
View File

@ -3,12 +3,12 @@ package mem_test
import (
"bytes"
"os"
"sort"
"testing"
"github.com/markbates/pkger/parser"
"github.com/markbates/pkger/pkging/mem"
"github.com/markbates/pkger/pkging/pkgtest"
"github.com/markbates/pkger/pkging/pkgutil"
"github.com/markbates/pkger/pkging/stdos"
"github.com/stretchr/testify/require"
)
@ -19,12 +19,15 @@ func Test_Pkger_Embedding(t *testing.T) {
app, err := pkgtest.App()
r.NoError(err)
paths, err := parser.Parse(app.Info)
res, err := parser.Parse(app.Info)
r.NoError(err)
ps := make([]string, len(paths))
for i, p := range paths {
ps[i] = p.String()
files, err := res.Files()
r.NoError(err)
ps := make([]string, len(files))
for i, f := range files {
ps[i] = f.Path.String()
}
r.Equal(app.Paths.Parser, ps)
@ -53,63 +56,61 @@ func Test_Pkger_Embedding(t *testing.T) {
})
r.NoError(err)
var res []string
var act []string
err = base.Walk("/", func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
res = append(res, path)
act = append(act, path)
return nil
})
r.NoError(err)
r.Equal(app.Paths.Root, res)
r.Equal(app.Paths.Root, act)
res = []string{}
act = []string{}
err = base.Walk("/public", func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
res = append(res, path)
act = append(act, path)
return nil
})
r.NoError(err)
r.Equal(app.Paths.Public, res)
r.Equal(app.Paths.Public, act)
bb := &bytes.Buffer{}
err = disk.Stuff(bb, paths)
err = pkgutil.Stuff(bb, app.Info, res)
r.NoError(err)
pkg := &mem.Pkger{}
err = pkg.UnmarshalEmbed(bb.Bytes())
r.NoError(err)
res = []string{}
act = []string{}
err = pkg.Walk("/", func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
res = append(res, path)
act = append(act, path)
return nil
})
r.NoError(err)
exp := append(app.Paths.Public, "app:/")
sort.Strings(exp)
r.Equal(exp, res)
r.Equal(app.Paths.Public, act)
res = []string{}
act = []string{}
err = pkg.Walk("/public", func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
res = append(res, path)
act = append(act, path)
return nil
})
r.NoError(err)
r.Equal(app.Paths.Public, res)
r.Equal(app.Paths.Public, act)
}

6
pkging/mem/mem.go
View File

@ -58,12 +58,12 @@ func (p *Pkger) MarshalJSON() ([]byte, error) {
infos[key] = info
return true
})
return json.Marshal(embed.Data{
ed := embed.Data{
Infos: infos,
Files: files,
Here: p.Here,
})
}
return json.Marshal(ed)
}
// UnmarshalJSON re-hydrates the *Pkger

5
pkging/mem/mem_test.go
View File

@ -1,9 +1,10 @@
package mem
package mem_test
import (
"testing"
"github.com/markbates/pkger/pkging"
"github.com/markbates/pkger/pkging/mem"
"github.com/markbates/pkger/pkging/pkgtest"
)
@ -14,7 +15,7 @@ func Test_Pkger(t *testing.T) {
return nil, err
}
pkg, err := New(app.Info)
pkg, err := mem.New(app.Info)
if err != nil {
return nil, err
}

3
pkging/pkgtest/app.go
View File

@ -65,6 +65,7 @@ func App() (AppDetails, error) {
var rootPaths = []string{
"app:/",
"app:/go.mod",
"app:/go.sum",
"app:/main.go",
"app:/public",
"app:/public/images",
@ -86,13 +87,11 @@ var publicPaths = []string{
}
var parserPaths = []string{
"app:/",
"app:/public",
"app:/public/images",
"app:/public/images/img1.png",
"app:/public/images/img2.png",
"app:/public/index.html",
"github.com/gobuffalo/buffalo:/",
"github.com/gobuffalo/buffalo:/render",
"github.com/gobuffalo/buffalo:/render/auto.go",
"github.com/gobuffalo/buffalo:/render/auto_test.go",

85
pkging/pkgutil/stuff.go Normal file
View File

@ -0,0 +1,85 @@
package pkgutil
import (
"io"
"os"
"github.com/markbates/pkger/here"
"github.com/markbates/pkger/parser"
"github.com/markbates/pkger/pkging/mem"
"github.com/markbates/pkger/pkging/stdos"
)
func Stuff(w io.Writer, c here.Info, decls parser.Decls) error {
disk, err := stdos.New(c)
if err != nil {
return err
}
pkg, err := mem.New(c)
if err != nil {
return err
}
files, err := decls.Files()
if err != nil {
return err
}
for _, pf := range files {
err = func() error {
df, err := disk.Open(pf.Path.String())
if err != nil {
return err
}
defer df.Close()
info, err := df.Stat()
if err != nil {
return err
}
if err := pkg.Add(df); err != nil {
return err
}
if !info.IsDir() {
return nil
}
err = disk.Walk(df.Path().String(), func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if info.IsDir() {
return nil
}
f, err := disk.Open(path)
if err != nil {
return err
}
defer f.Close()
if err := pkg.Add(f); err != nil {
return err
}
return nil
})
return err
}()
if err != nil {
return err
}
b, err := pkg.MarshalEmbed()
if err != nil {
return err
}
_, err = w.Write(b)
}
return nil
}

34
pkging/stdos/stdos.go
View File

@ -2,7 +2,6 @@ package stdos
import (
"fmt"
"io"
"os"
"path/filepath"
"strings"
@ -10,7 +9,6 @@ import (
"github.com/markbates/pkger/here"
"github.com/markbates/pkger/internal/maps"
"github.com/markbates/pkger/pkging"
"github.com/markbates/pkger/pkging/mem"
)
var _ pkging.Pkger = &Pkger{}
@ -20,38 +18,6 @@ type Pkger struct {
infos *maps.Infos
}
func (disk *Pkger) Stuff(w io.Writer, paths []here.Path) error {
pkg, err := mem.New(disk.Here)
if err != nil {
return err
}
for _, pt := range paths {
err = func() error {
f, err := disk.Open(pt.String())
if err != nil {
return err
}
defer f.Close()
if err := pkg.Add(f); err != nil {
return err
}
return nil
}()
if err != nil {
return err
}
}
b, err := pkg.MarshalEmbed()
if err != nil {
return err
}
_, err = w.Write(b)
return err
}
// Abs returns an absolute representation of path. If the path is not absolute it will be joined with the current working directory to turn it into an absolute path. The absolute path name for a given file is not guaranteed to be unique. Abs calls Clean on the result.
func (f *Pkger) Abs(p string) (string, error) {
pt, err := f.Parse(p)

5
pkging/stdos/stdos_test.go
View File

@ -1,4 +1,4 @@
package stdos
package stdos_test
import (
"io/ioutil"
@ -6,6 +6,7 @@ import (
"github.com/markbates/pkger/pkging"
"github.com/markbates/pkger/pkging/pkgtest"
"github.com/markbates/pkger/pkging/stdos"
)
func Test_Pkger(t *testing.T) {
@ -22,7 +23,7 @@ func Test_Pkger(t *testing.T) {
app.Dir = dir
mypkging, err := New(app.Info)
mypkging, err := stdos.New(app.Info)
if err != nil {
return nil, err
}