Merge pull request #8 from markbates/remove-stuffing

jumpin' jack flash its a gas, gas, gas

cmd/pkger/cmds/pack.go

@@ -9,7 +9,7 @@ import (
 	"github.com/markbates/pkger"
 	"github.com/markbates/pkger/here"
 	"github.com/markbates/pkger/parser"
-	"github.com/markbates/pkger/pkging/stuffing"
+	"github.com/markbates/pkger/pkging/stdos"
 )
 
 const outName = "pkged.go"
@@ -140,8 +140,12 @@ func Package(out string, paths []here.Path) error {
 	}
 	fmt.Fprintf(f, "\nvar _ = pkger.Apply(mem.UnmarshalEmbed([]byte(`")
 
-	if err := stuffing.Stuff(f, c, paths); err != nil {
+	disk, err := stdos.New(c)
-		os.RemoveAll(out)
+	if err != nil {
+		return err
+	}
+
+	if err := disk.Stuff(f, paths); err != nil {
 		return err
 	}
 

examples/walk/pkger/go.mod

@@ -2,4 +2,6 @@ module app
 
 go 1.13
 
-require github.com/markbates/pkger v0.0.0-20191016200917-09e9684b656b
+require github.com/markbates/pkger v0.1.0
+
+replace github.com/markbates/pkger => ../../../

examples/walk/pkger/go.sum

@@ -6,8 +6,6 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
-github.com/markbates/pkger v0.0.0-20191016200917-09e9684b656b h1:cXYQ3JZQkRLFyN8m22Q58mOhdKyTooPqvGt8pyWR8eA=
-github.com/markbates/pkger v0.0.0-20191016200917-09e9684b656b/go.mod h1:0nBNvgA9jJk9gWhO/BcIYz1qJ8BJ0MWCSyKFPApdWNs=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -18,3 +16,5 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
+gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

pkging/embed.go

@@ -0,0 +1 @@
+package pkging

pkging/embed/embed.go

@@ -0,0 +1,58 @@
+package embed
+
+import (
+	"bytes"
+	"compress/gzip"
+	"encoding/hex"
+	"encoding/json"
+
+	"github.com/markbates/pkger/here"
+)
+
+type Embedder interface {
+	MarshalEmbed() ([]byte, error)
+}
+
+type Unembedder interface {
+	UnmarshalEmbed([]byte) error
+}
+
+type Data struct {
+	Infos map[string]here.Info `json:"infos"`
+	Files map[string]File      `json:"files"`
+	Here  here.Info            `json:"here"`
+}
+
+func (d *Data) MarshalEmbed() ([]byte, error) {
+	bb := &bytes.Buffer{}
+	gz := gzip.NewWriter(bb)
+	defer gz.Close()
+	if err := json.NewEncoder(gz).Encode(d); err != nil {
+		return nil, err
+	}
+	if err := gz.Close(); err != nil {
+		return nil, err
+	}
+	s := hex.EncodeToString(bb.Bytes())
+	return []byte(s), nil
+}
+
+func (d *Data) UnmarshalEmbed(in []byte) error {
+	b := make([]byte, len(in))
+	if _, err := hex.Decode(b, in); err != nil {
+		return err
+	}
+
+	gz, err := gzip.NewReader(bytes.NewReader(b))
+	if err != nil {
+		return err
+	}
+	defer gz.Close()
+
+	p := &Data{}
+	if err := json.NewDecoder(gz).Decode(p); err != nil {
+		return err
+	}
+	(*d) = *p
+	return nil
+}

pkging/embed/file.go

@@ -0,0 +1,14 @@
+package embed
+
+import (
+	"github.com/markbates/pkger/here"
+	"github.com/markbates/pkger/pkging"
+)
+
+type File struct {
+	Info   *pkging.FileInfo `json:"info"`
+	Here   here.Info        `json:"her"`
+	Path   here.Path        `json:"path"`
+	Data   []byte           `json:"data"`
+	Parent here.Path        `json:"parent"`
+}

pkging/faces.go

@@ -0,0 +1,15 @@
+package pkging
+
+import (
+	"io"
+
+	"github.com/markbates/pkger/here"
+)
+
+type Adder interface {
+	Add(files ...File) error
+}
+
+type Stuffer interface {
+	Stuff(w io.Writer, paths []here.Path) error
+}

pkging/mem/add.go

@@ -0,0 +1,48 @@
+package mem
+
+import (
+	"bytes"
+	"io"
+	"path/filepath"
+
+	"github.com/markbates/pkger/pkging"
+)
+
+// Add copies the pkging.File into the *Pkger
+func (fx *Pkger) Add(files ...pkging.File) error {
+	for _, f := range files {
+		info, err := f.Stat()
+		if err != nil {
+			return err
+		}
+
+		if f.Path().Pkg == fx.Here.ImportPath {
+			dir := filepath.Dir(f.Name())
+			if dir != "/" {
+				if err := fx.MkdirAll(dir, 0755); err != nil {
+					return err
+				}
+			}
+		}
+
+		mf := &File{
+			Here:   f.Info(),
+			info:   pkging.NewFileInfo(info),
+			path:   f.Path(),
+			pkging: fx,
+		}
+
+		if !info.IsDir() {
+			bb := &bytes.Buffer{}
+			_, err = io.Copy(bb, f)
+			if err != nil {
+				return err
+			}
+			mf.data = bb.Bytes()
+		}
+
+		fx.files.Store(mf.Path(), mf)
+	}
+
+	return nil
+}

pkging/mem/embed.go

@@ -21,6 +21,26 @@ func (pkg *Pkger) MarshalEmbed() ([]byte, error) {
 	return []byte(s), nil
 }
 
+func (pkg *Pkger) UnmarshalEmbed(in []byte) error {
+	b := make([]byte, len(in))
+	if _, err := hex.Decode(b, in); err != nil {
+		return err
+	}
+
+	gz, err := gzip.NewReader(bytes.NewReader(b))
+	if err != nil {
+		return err
+	}
+	defer gz.Close()
+
+	p := &Pkger{}
+	if err := json.NewDecoder(gz).Decode(p); err != nil {
+		return err
+	}
+	(*pkg) = *p
+	return nil
+}
+
 func UnmarshalEmbed(in []byte) (*Pkger, error) {
 	b := make([]byte, len(in))
 	if _, err := hex.Decode(b, in); err != nil {
@@ -33,10 +53,9 @@ func UnmarshalEmbed(in []byte) (*Pkger, error) {
 	}
 	defer gz.Close()
 
-	pkg := &Pkger{}
+	p := &Pkger{}
-	if err := json.NewDecoder(gz).Decode(pkg); err != nil {
+	if err := json.NewDecoder(gz).Decode(p); err != nil {
 		return nil, err
 	}
-
+	return p, nil
-	return pkg, nil
 }

pkging/mem/embed_test.go

@@ -10,7 +10,6 @@ import (
 	"github.com/markbates/pkger/pkging/mem"
 	"github.com/markbates/pkger/pkging/pkgtest"
 	"github.com/markbates/pkger/pkging/stdos"
-	"github.com/markbates/pkger/pkging/stuffing"
 	"github.com/stretchr/testify/require"
 )
 
@@ -80,10 +79,11 @@ func Test_Pkger_Embedding(t *testing.T) {
 
 	bb := &bytes.Buffer{}
 
-	err = stuffing.Stuff(bb, app.Info, paths)
+	err = disk.Stuff(bb, paths)
 	r.NoError(err)
 
-	pkg, err := mem.UnmarshalEmbed(bb.Bytes())
+	pkg := &mem.Pkger{}
+	err = pkg.UnmarshalEmbed(bb.Bytes())
 	r.NoError(err)
 
 	res = []string{}

pkging/mem/file.go

@@ -2,7 +2,6 @@ package mem
 
 import (
 	"bytes"
-	"encoding/json"
 	"fmt"
 	"io"
 	"net/http"
@@ -21,8 +20,8 @@ const timeFmt = time.RFC3339Nano
 var _ pkging.File = &File{}
 
 type File struct {
+	Here   here.Info
 	info   *pkging.FileInfo
-	her    here.Info
 	path   here.Path
 	data   []byte
 	parent here.Path
@@ -31,37 +30,6 @@ type File struct {
 	pkging pkging.Pkger
 }
 
-type fJay struct {
-	Info   *pkging.FileInfo `json:"info"`
-	Her    here.Info        `json:"her"`
-	Path   here.Path        `json:"path"`
-	Data   []byte           `json:"data"`
-	Parent here.Path        `json:"parent"`
-}
-
-func (f File) MarshalJSON() ([]byte, error) {
-	return json.Marshal(fJay{
-		Info:   f.info,
-		Her:    f.her,
-		Path:   f.path,
-		Data:   f.data,
-		Parent: f.parent,
-	})
-}
-
-func (f *File) UnmarshalJSON(b []byte) error {
-	var y fJay
-	if err := json.Unmarshal(b, &y); err != nil {
-		return err
-	}
-	f.info = y.Info
-	f.her = y.Her
-	f.path = y.Path
-	f.data = y.Data
-	f.parent = y.Parent
-	return nil
-}
-
 // Seek sets the offset for the next Read or Write on file to offset, interpreted according to whence: 0 means relative to the origin of the file, 1 means relative to the current offset, and 2 means relative to the end. It returns the new offset and an error, if any.
 func (f *File) Seek(ofpkginget int64, whence int) (int64, error) {
 	if sk, ok := f.reader.(io.Seeker); ok {
@@ -121,7 +89,7 @@ func (f *File) Write(b []byte) (int, error) {
 
 // Info returns the here.Info of the file
 func (f File) Info() here.Info {
-	return f.her
+	return f.Here
 }
 
 // Stat returns the FileInfo structure describing file. If there is an error, it will be of type *PathError.
@@ -203,7 +171,7 @@ func (f *File) Readdir(count int) ([]os.FileInfo, error) {
 
 // Open implements the http.FileSystem interface. A FileSystem implements access to a collection of named files. The elements in a file path are separated by slash ('/', U+002F) characters, regardless of host operating system convention.
 func (f *File) Open(name string) (http.File, error) {
-	pt, err := f.her.Parse(name)
+	pt, err := f.Here.Parse(name)
 	if err != nil {
 		return nil, err
 	}
@@ -227,7 +195,7 @@ func (f *File) Open(name string) (http.File, error) {
 	if fi.IsDir() {
 		d2 := &File{
 			info:   pkging.NewFileInfo(fi),
-			her:    di.Info(),
+			Here:   di.Info(),
 			path:   pt,
 			parent: f.path,
 			pkging: f.pkging,

pkging/mem/mem.go

@@ -1,10 +1,8 @@
 package mem
 
 import (
-	"bytes"
 	"encoding/json"
 	"fmt"
-	"io"
 	"os"
 	"path/filepath"
 	"strings"
@@ -13,6 +11,7 @@ import (
 	"github.com/markbates/pkger/here"
 	"github.com/markbates/pkger/internal/maps"
 	"github.com/markbates/pkger/pkging"
+	"github.com/markbates/pkger/pkging/embed"
 )
 
 var _ pkging.Pkger = &Pkger{}
@@ -34,43 +33,55 @@ type Pkger struct {
 	files *maps.Files
 }
 
-type jay struct {
-	Infos   *maps.Infos      `json:"infos"`
-	Files   map[string]*File `json:"files"`
-	Current here.Info        `json:"current"`
-}
-
 // MarshalJSON creates a fully re-hydratable JSON representation of *Pkger
 func (p *Pkger) MarshalJSON() ([]byte, error) {
-	files := map[string]*File{}
+	files := map[string]embed.File{}
 
 	p.files.Range(func(key here.Path, file pkging.File) bool {
 		f, ok := file.(*File)
 		if !ok {
 			return true
 		}
-		files[key.String()] = f
+		ef := embed.File{
+			Info:   f.info,
+			Here:   f.Here,
+			Path:   f.path,
+			Parent: f.parent,
+			Data:   f.data,
+		}
+		files[key.String()] = ef
 		return true
 	})
 
-	return json.Marshal(jay{
+	infos := map[string]here.Info{}
-		Infos:   p.infos,
+	p.infos.Range(func(key string, info here.Info) bool {
-		Files:   files,
+		infos[key] = info
-		Current: p.Here,
+		return true
+	})
+
+	return json.Marshal(embed.Data{
+		Infos: infos,
+		Files: files,
+		Here:  p.Here,
 	})
 }
 
 // UnmarshalJSON re-hydrates the *Pkger
 func (p *Pkger) UnmarshalJSON(b []byte) error {
-	y := jay{}
+	y := &embed.Data{
+		Infos: map[string]here.Info{},
+		Files: map[string]embed.File{},
+	}
 
 	if err := json.Unmarshal(b, &y); err != nil {
 		return err
 	}
 
-	p.Here = y.Current
+	p.Here = y.Here
-
+	p.infos = &maps.Infos{}
-	p.infos = y.Infos
+	for k, v := range y.Infos {
+		p.infos.Store(k, v)
+	}
 
 	p.files = &maps.Files{}
 	for k, v := range y.Files {
@@ -78,7 +89,15 @@ func (p *Pkger) UnmarshalJSON(b []byte) error {
 		if err != nil {
 			return err
 		}
-		p.files.Store(pt, v)
+
+		f := &File{
+			Here:   v.Here,
+			info:   v.Info,
+			path:   v.Path,
+			data:   v.Data,
+			parent: v.Parent,
+		}
+		p.files.Store(pt, f)
 	}
 	return nil
 }
@@ -149,43 +168,6 @@ func (fx *Pkger) RemoveAll(name string) error {
 	return nil
 }
 
-// Add copies the pkging.File into the *Pkger
-func (fx *Pkger) Add(f pkging.File) error {
-	info, err := f.Stat()
-	if err != nil {
-		return err
-	}
-
-	if f.Path().Pkg == fx.Here.ImportPath {
-		dir := filepath.Dir(f.Name())
-		if dir != "/" {
-			if err := fx.MkdirAll(dir, 0755); err != nil {
-				return err
-			}
-		}
-	}
-
-	mf := &File{
-		her:    f.Info(),
-		info:   pkging.NewFileInfo(info),
-		path:   f.Path(),
-		pkging: fx,
-	}
-
-	if !info.IsDir() {
-		bb := &bytes.Buffer{}
-		_, err = io.Copy(bb, f)
-		if err != nil {
-			return err
-		}
-		mf.data = bb.Bytes()
-	}
-
-	fx.files.Store(mf.Path(), mf)
-
-	return nil
-}
-
 // Create creates the named file with mode 0666 (before umask) - It's actually 0644, truncating it if it already exists. If successful, methods on the returned File can be used for I/O; the associated file descriptor has mode O_RDWR.
 func (fx *Pkger) Create(name string) (pkging.File, error) {
 	fx.MkdirAll("/", 0755)
@@ -207,8 +189,8 @@ func (fx *Pkger) Create(name string) (pkging.File, error) {
 	}
 
 	f := &File{
+		Here: her,
 		path: pt,
-		her:  her,
 		info: &pkging.FileInfo{
 			Details: pkging.Details{
 				Name:    pt.Name,
@@ -249,9 +231,9 @@ func (fx *Pkger) MkdirAll(p string, perm os.FileMode) error {
 			continue
 		}
 		f := &File{
+			Here:   cur,
 			pkging: fx,
 			path:   pt,
-			her:    cur,
 			info: &pkging.FileInfo{
 				Details: pkging.Details{
 					Name:    pt.Name,
@@ -301,7 +283,7 @@ func (fx *Pkger) Open(name string) (pkging.File, error) {
 		info:   pkging.WithName(f.info.Name(), f.info),
 		path:   f.path,
 		data:   f.data,
-		her:    f.her,
+		Here:   f.Here,
 	}
 
 	return nf, nil

pkging/stdos/stdos.go

@@ -2,6 +2,7 @@ package stdos
 
 import (
 	"fmt"
+	"io"
 	"os"
 	"path/filepath"
 	"strings"
@@ -9,6 +10,7 @@ import (
 	"github.com/markbates/pkger/here"
 	"github.com/markbates/pkger/internal/maps"
 	"github.com/markbates/pkger/pkging"
+	"github.com/markbates/pkger/pkging/mem"
 )
 
 var _ pkging.Pkger = &Pkger{}
@@ -18,6 +20,38 @@ type Pkger struct {
 	infos *maps.Infos
 }
 
+func (disk *Pkger) Stuff(w io.Writer, paths []here.Path) error {
+	pkg, err := mem.New(disk.Here)
+	if err != nil {
+		return err
+	}
+
+	for _, pt := range paths {
+		err = func() error {
+			f, err := disk.Open(pt.String())
+			if err != nil {
+				return err
+			}
+			defer f.Close()
+			if err := pkg.Add(f); err != nil {
+				return err
+			}
+
+			return nil
+		}()
+		if err != nil {
+			return err
+		}
+	}
+
+	b, err := pkg.MarshalEmbed()
+	if err != nil {
+		return err
+	}
+	_, err = w.Write(b)
+	return err
+}
+
 // Abs returns an absolute representation of path. If the path is not absolute it will be joined with the current working directory to turn it into an absolute path. The absolute path name for a given file is not guaranteed to be unique. Abs calls Clean on the result.
 func (f *Pkger) Abs(p string) (string, error) {
 	pt, err := f.Parse(p)

pkging/stuffing/stuffing.go

@@ -1,46 +0,0 @@
-package stuffing
-
-import (
-	"io"
-
-	"github.com/markbates/pkger/here"
-	"github.com/markbates/pkger/pkging/mem"
-	"github.com/markbates/pkger/pkging/stdos"
-)
-
-func Stuff(w io.Writer, cur here.Info, paths []here.Path) error {
-	disk, err := stdos.New(cur)
-	if err != nil {
-		return err
-	}
-
-	pkg, err := mem.New(cur)
-	if err != nil {
-		return err
-	}
-
-	for _, pt := range paths {
-		err = func() error {
-			f, err := disk.Open(pt.String())
-			if err != nil {
-				return err
-			}
-			defer f.Close()
-			if err := pkg.Add(f); err != nil {
-				return err
-			}
-
-			return nil
-		}()
-		if err != nil {
-			return err
-		}
-	}
-
-	b, err := pkg.MarshalEmbed()
-	if err != nil {
-		return err
-	}
-	_, err = w.Write(b)
-	return err
-}