From e58748472479d1931dc1cc49c4ac96351d3daa66 Mon Sep 17 00:00:00 2001 From: Mark Bates Date: Sun, 20 Oct 2019 17:52:16 -0400 Subject: [PATCH] jumpin' jack flash its a gas, gas, gas --- cmd/pkger/cmds/pack.go | 10 ++-- examples/walk/pkger/go.mod | 4 +- examples/walk/pkger/go.sum | 4 +- pkging/embed.go | 1 + pkging/embed/embed.go | 58 +++++++++++++++++++++ pkging/embed/file.go | 14 +++++ pkging/faces.go | 15 ++++++ pkging/mem/add.go | 48 +++++++++++++++++ pkging/mem/embed.go | 27 ++++++++-- pkging/mem/embed_test.go | 6 +-- pkging/mem/file.go | 40 ++------------- pkging/mem/mem.go | 100 +++++++++++++++--------------------- pkging/stdos/stdos.go | 34 ++++++++++++ pkging/stuffing/stuffing.go | 46 ----------------- 14 files changed, 253 insertions(+), 154 deletions(-) create mode 100644 pkging/embed.go create mode 100644 pkging/embed/embed.go create mode 100644 pkging/embed/file.go create mode 100644 pkging/faces.go create mode 100644 pkging/mem/add.go delete mode 100644 pkging/stuffing/stuffing.go diff --git a/cmd/pkger/cmds/pack.go b/cmd/pkger/cmds/pack.go index 89b8e7e..161c11b 100644 --- a/cmd/pkger/cmds/pack.go +++ b/cmd/pkger/cmds/pack.go @@ -9,7 +9,7 @@ import ( "github.com/markbates/pkger" "github.com/markbates/pkger/here" "github.com/markbates/pkger/parser" - "github.com/markbates/pkger/pkging/stuffing" + "github.com/markbates/pkger/pkging/stdos" ) const outName = "pkged.go" @@ -140,8 +140,12 @@ func Package(out string, paths []here.Path) error { } fmt.Fprintf(f, "\nvar _ = pkger.Apply(mem.UnmarshalEmbed([]byte(`") - if err := stuffing.Stuff(f, c, paths); err != nil { - os.RemoveAll(out) + disk, err := stdos.New(c) + if err != nil { + return err + } + + if err := disk.Stuff(f, paths); err != nil { return err } diff --git a/examples/walk/pkger/go.mod b/examples/walk/pkger/go.mod index 8550ba4..44a475e 100644 --- a/examples/walk/pkger/go.mod +++ b/examples/walk/pkger/go.mod @@ -2,4 +2,6 @@ module app go 1.13 -require github.com/markbates/pkger v0.0.0-20191016200917-09e9684b656b +require github.com/markbates/pkger v0.1.0 + +replace github.com/markbates/pkger => ../../../ diff --git a/examples/walk/pkger/go.sum b/examples/walk/pkger/go.sum index 66386d4..3bc4a9a 100644 --- a/examples/walk/pkger/go.sum +++ b/examples/walk/pkger/go.sum @@ -6,8 +6,6 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/markbates/pkger v0.0.0-20191016200917-09e9684b656b h1:cXYQ3JZQkRLFyN8m22Q58mOhdKyTooPqvGt8pyWR8eA= -github.com/markbates/pkger v0.0.0-20191016200917-09e9684b656b/go.mod h1:0nBNvgA9jJk9gWhO/BcIYz1qJ8BJ0MWCSyKFPApdWNs= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -18,3 +16,5 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/pkging/embed.go b/pkging/embed.go new file mode 100644 index 0000000..4f1c21b --- /dev/null +++ b/pkging/embed.go @@ -0,0 +1 @@ +package pkging diff --git a/pkging/embed/embed.go b/pkging/embed/embed.go new file mode 100644 index 0000000..187202a --- /dev/null +++ b/pkging/embed/embed.go @@ -0,0 +1,58 @@ +package embed + +import ( + "bytes" + "compress/gzip" + "encoding/hex" + "encoding/json" + + "github.com/markbates/pkger/here" +) + +type Embedder interface { + MarshalEmbed() ([]byte, error) +} + +type Unembedder interface { + UnmarshalEmbed([]byte) error +} + +type Data struct { + Infos map[string]here.Info `json:"infos"` + Files map[string]File `json:"files"` + Here here.Info `json:"here"` +} + +func (d *Data) MarshalEmbed() ([]byte, error) { + bb := &bytes.Buffer{} + gz := gzip.NewWriter(bb) + defer gz.Close() + if err := json.NewEncoder(gz).Encode(d); err != nil { + return nil, err + } + if err := gz.Close(); err != nil { + return nil, err + } + s := hex.EncodeToString(bb.Bytes()) + return []byte(s), nil +} + +func (d *Data) UnmarshalEmbed(in []byte) error { + b := make([]byte, len(in)) + if _, err := hex.Decode(b, in); err != nil { + return err + } + + gz, err := gzip.NewReader(bytes.NewReader(b)) + if err != nil { + return err + } + defer gz.Close() + + p := &Data{} + if err := json.NewDecoder(gz).Decode(p); err != nil { + return err + } + (*d) = *p + return nil +} diff --git a/pkging/embed/file.go b/pkging/embed/file.go new file mode 100644 index 0000000..af239f8 --- /dev/null +++ b/pkging/embed/file.go @@ -0,0 +1,14 @@ +package embed + +import ( + "github.com/markbates/pkger/here" + "github.com/markbates/pkger/pkging" +) + +type File struct { + Info *pkging.FileInfo `json:"info"` + Here here.Info `json:"her"` + Path here.Path `json:"path"` + Data []byte `json:"data"` + Parent here.Path `json:"parent"` +} diff --git a/pkging/faces.go b/pkging/faces.go new file mode 100644 index 0000000..f56e4cf --- /dev/null +++ b/pkging/faces.go @@ -0,0 +1,15 @@ +package pkging + +import ( + "io" + + "github.com/markbates/pkger/here" +) + +type Adder interface { + Add(files ...File) error +} + +type Stuffer interface { + Stuff(w io.Writer, paths []here.Path) error +} diff --git a/pkging/mem/add.go b/pkging/mem/add.go new file mode 100644 index 0000000..8dd8650 --- /dev/null +++ b/pkging/mem/add.go @@ -0,0 +1,48 @@ +package mem + +import ( + "bytes" + "io" + "path/filepath" + + "github.com/markbates/pkger/pkging" +) + +// Add copies the pkging.File into the *Pkger +func (fx *Pkger) Add(files ...pkging.File) error { + for _, f := range files { + info, err := f.Stat() + if err != nil { + return err + } + + if f.Path().Pkg == fx.Here.ImportPath { + dir := filepath.Dir(f.Name()) + if dir != "/" { + if err := fx.MkdirAll(dir, 0755); err != nil { + return err + } + } + } + + mf := &File{ + Here: f.Info(), + info: pkging.NewFileInfo(info), + path: f.Path(), + pkging: fx, + } + + if !info.IsDir() { + bb := &bytes.Buffer{} + _, err = io.Copy(bb, f) + if err != nil { + return err + } + mf.data = bb.Bytes() + } + + fx.files.Store(mf.Path(), mf) + } + + return nil +} diff --git a/pkging/mem/embed.go b/pkging/mem/embed.go index 81fbe2e..27a8617 100644 --- a/pkging/mem/embed.go +++ b/pkging/mem/embed.go @@ -21,6 +21,26 @@ func (pkg *Pkger) MarshalEmbed() ([]byte, error) { return []byte(s), nil } +func (pkg *Pkger) UnmarshalEmbed(in []byte) error { + b := make([]byte, len(in)) + if _, err := hex.Decode(b, in); err != nil { + return err + } + + gz, err := gzip.NewReader(bytes.NewReader(b)) + if err != nil { + return err + } + defer gz.Close() + + p := &Pkger{} + if err := json.NewDecoder(gz).Decode(p); err != nil { + return err + } + (*pkg) = *p + return nil +} + func UnmarshalEmbed(in []byte) (*Pkger, error) { b := make([]byte, len(in)) if _, err := hex.Decode(b, in); err != nil { @@ -33,10 +53,9 @@ func UnmarshalEmbed(in []byte) (*Pkger, error) { } defer gz.Close() - pkg := &Pkger{} - if err := json.NewDecoder(gz).Decode(pkg); err != nil { + p := &Pkger{} + if err := json.NewDecoder(gz).Decode(p); err != nil { return nil, err } - - return pkg, nil + return p, nil } diff --git a/pkging/mem/embed_test.go b/pkging/mem/embed_test.go index 51893e9..56ed778 100644 --- a/pkging/mem/embed_test.go +++ b/pkging/mem/embed_test.go @@ -10,7 +10,6 @@ import ( "github.com/markbates/pkger/pkging/mem" "github.com/markbates/pkger/pkging/pkgtest" "github.com/markbates/pkger/pkging/stdos" - "github.com/markbates/pkger/pkging/stuffing" "github.com/stretchr/testify/require" ) @@ -80,10 +79,11 @@ func Test_Pkger_Embedding(t *testing.T) { bb := &bytes.Buffer{} - err = stuffing.Stuff(bb, app.Info, paths) + err = disk.Stuff(bb, paths) r.NoError(err) - pkg, err := mem.UnmarshalEmbed(bb.Bytes()) + pkg := &mem.Pkger{} + err = pkg.UnmarshalEmbed(bb.Bytes()) r.NoError(err) res = []string{} diff --git a/pkging/mem/file.go b/pkging/mem/file.go index 64c6791..bac0af5 100644 --- a/pkging/mem/file.go +++ b/pkging/mem/file.go @@ -2,7 +2,6 @@ package mem import ( "bytes" - "encoding/json" "fmt" "io" "net/http" @@ -21,8 +20,8 @@ const timeFmt = time.RFC3339Nano var _ pkging.File = &File{} type File struct { + Here here.Info info *pkging.FileInfo - her here.Info path here.Path data []byte parent here.Path @@ -31,37 +30,6 @@ type File struct { pkging pkging.Pkger } -type fJay struct { - Info *pkging.FileInfo `json:"info"` - Her here.Info `json:"her"` - Path here.Path `json:"path"` - Data []byte `json:"data"` - Parent here.Path `json:"parent"` -} - -func (f File) MarshalJSON() ([]byte, error) { - return json.Marshal(fJay{ - Info: f.info, - Her: f.her, - Path: f.path, - Data: f.data, - Parent: f.parent, - }) -} - -func (f *File) UnmarshalJSON(b []byte) error { - var y fJay - if err := json.Unmarshal(b, &y); err != nil { - return err - } - f.info = y.Info - f.her = y.Her - f.path = y.Path - f.data = y.Data - f.parent = y.Parent - return nil -} - // Seek sets the offset for the next Read or Write on file to offset, interpreted according to whence: 0 means relative to the origin of the file, 1 means relative to the current offset, and 2 means relative to the end. It returns the new offset and an error, if any. func (f *File) Seek(ofpkginget int64, whence int) (int64, error) { if sk, ok := f.reader.(io.Seeker); ok { @@ -121,7 +89,7 @@ func (f *File) Write(b []byte) (int, error) { // Info returns the here.Info of the file func (f File) Info() here.Info { - return f.her + return f.Here } // Stat returns the FileInfo structure describing file. If there is an error, it will be of type *PathError. @@ -203,7 +171,7 @@ func (f *File) Readdir(count int) ([]os.FileInfo, error) { // Open implements the http.FileSystem interface. A FileSystem implements access to a collection of named files. The elements in a file path are separated by slash ('/', U+002F) characters, regardless of host operating system convention. func (f *File) Open(name string) (http.File, error) { - pt, err := f.her.Parse(name) + pt, err := f.Here.Parse(name) if err != nil { return nil, err } @@ -227,7 +195,7 @@ func (f *File) Open(name string) (http.File, error) { if fi.IsDir() { d2 := &File{ info: pkging.NewFileInfo(fi), - her: di.Info(), + Here: di.Info(), path: pt, parent: f.path, pkging: f.pkging, diff --git a/pkging/mem/mem.go b/pkging/mem/mem.go index 8815d5a..5b31e0b 100644 --- a/pkging/mem/mem.go +++ b/pkging/mem/mem.go @@ -1,10 +1,8 @@ package mem import ( - "bytes" "encoding/json" "fmt" - "io" "os" "path/filepath" "strings" @@ -13,6 +11,7 @@ import ( "github.com/markbates/pkger/here" "github.com/markbates/pkger/internal/maps" "github.com/markbates/pkger/pkging" + "github.com/markbates/pkger/pkging/embed" ) var _ pkging.Pkger = &Pkger{} @@ -34,43 +33,55 @@ type Pkger struct { files *maps.Files } -type jay struct { - Infos *maps.Infos `json:"infos"` - Files map[string]*File `json:"files"` - Current here.Info `json:"current"` -} - // MarshalJSON creates a fully re-hydratable JSON representation of *Pkger func (p *Pkger) MarshalJSON() ([]byte, error) { - files := map[string]*File{} + files := map[string]embed.File{} p.files.Range(func(key here.Path, file pkging.File) bool { f, ok := file.(*File) if !ok { return true } - files[key.String()] = f + ef := embed.File{ + Info: f.info, + Here: f.Here, + Path: f.path, + Parent: f.parent, + Data: f.data, + } + files[key.String()] = ef return true }) - return json.Marshal(jay{ - Infos: p.infos, - Files: files, - Current: p.Here, + infos := map[string]here.Info{} + p.infos.Range(func(key string, info here.Info) bool { + infos[key] = info + return true + }) + + return json.Marshal(embed.Data{ + Infos: infos, + Files: files, + Here: p.Here, }) } // UnmarshalJSON re-hydrates the *Pkger func (p *Pkger) UnmarshalJSON(b []byte) error { - y := jay{} + y := &embed.Data{ + Infos: map[string]here.Info{}, + Files: map[string]embed.File{}, + } if err := json.Unmarshal(b, &y); err != nil { return err } - p.Here = y.Current - - p.infos = y.Infos + p.Here = y.Here + p.infos = &maps.Infos{} + for k, v := range y.Infos { + p.infos.Store(k, v) + } p.files = &maps.Files{} for k, v := range y.Files { @@ -78,7 +89,15 @@ func (p *Pkger) UnmarshalJSON(b []byte) error { if err != nil { return err } - p.files.Store(pt, v) + + f := &File{ + Here: v.Here, + info: v.Info, + path: v.Path, + data: v.Data, + parent: v.Parent, + } + p.files.Store(pt, f) } return nil } @@ -149,43 +168,6 @@ func (fx *Pkger) RemoveAll(name string) error { return nil } -// Add copies the pkging.File into the *Pkger -func (fx *Pkger) Add(f pkging.File) error { - info, err := f.Stat() - if err != nil { - return err - } - - if f.Path().Pkg == fx.Here.ImportPath { - dir := filepath.Dir(f.Name()) - if dir != "/" { - if err := fx.MkdirAll(dir, 0755); err != nil { - return err - } - } - } - - mf := &File{ - her: f.Info(), - info: pkging.NewFileInfo(info), - path: f.Path(), - pkging: fx, - } - - if !info.IsDir() { - bb := &bytes.Buffer{} - _, err = io.Copy(bb, f) - if err != nil { - return err - } - mf.data = bb.Bytes() - } - - fx.files.Store(mf.Path(), mf) - - return nil -} - // Create creates the named file with mode 0666 (before umask) - It's actually 0644, truncating it if it already exists. If successful, methods on the returned File can be used for I/O; the associated file descriptor has mode O_RDWR. func (fx *Pkger) Create(name string) (pkging.File, error) { fx.MkdirAll("/", 0755) @@ -207,8 +189,8 @@ func (fx *Pkger) Create(name string) (pkging.File, error) { } f := &File{ + Here: her, path: pt, - her: her, info: &pkging.FileInfo{ Details: pkging.Details{ Name: pt.Name, @@ -249,9 +231,9 @@ func (fx *Pkger) MkdirAll(p string, perm os.FileMode) error { continue } f := &File{ + Here: cur, pkging: fx, path: pt, - her: cur, info: &pkging.FileInfo{ Details: pkging.Details{ Name: pt.Name, @@ -301,7 +283,7 @@ func (fx *Pkger) Open(name string) (pkging.File, error) { info: pkging.WithName(f.info.Name(), f.info), path: f.path, data: f.data, - her: f.her, + Here: f.Here, } return nf, nil diff --git a/pkging/stdos/stdos.go b/pkging/stdos/stdos.go index 787aac2..85f6f77 100644 --- a/pkging/stdos/stdos.go +++ b/pkging/stdos/stdos.go @@ -2,6 +2,7 @@ package stdos import ( "fmt" + "io" "os" "path/filepath" "strings" @@ -9,6 +10,7 @@ import ( "github.com/markbates/pkger/here" "github.com/markbates/pkger/internal/maps" "github.com/markbates/pkger/pkging" + "github.com/markbates/pkger/pkging/mem" ) var _ pkging.Pkger = &Pkger{} @@ -18,6 +20,38 @@ type Pkger struct { infos *maps.Infos } +func (disk *Pkger) Stuff(w io.Writer, paths []here.Path) error { + pkg, err := mem.New(disk.Here) + if err != nil { + return err + } + + for _, pt := range paths { + err = func() error { + f, err := disk.Open(pt.String()) + if err != nil { + return err + } + defer f.Close() + if err := pkg.Add(f); err != nil { + return err + } + + return nil + }() + if err != nil { + return err + } + } + + b, err := pkg.MarshalEmbed() + if err != nil { + return err + } + _, err = w.Write(b) + return err +} + // Abs returns an absolute representation of path. If the path is not absolute it will be joined with the current working directory to turn it into an absolute path. The absolute path name for a given file is not guaranteed to be unique. Abs calls Clean on the result. func (f *Pkger) Abs(p string) (string, error) { pt, err := f.Parse(p) diff --git a/pkging/stuffing/stuffing.go b/pkging/stuffing/stuffing.go deleted file mode 100644 index 9204a33..0000000 --- a/pkging/stuffing/stuffing.go +++ /dev/null @@ -1,46 +0,0 @@ -package stuffing - -import ( - "io" - - "github.com/markbates/pkger/here" - "github.com/markbates/pkger/pkging/mem" - "github.com/markbates/pkger/pkging/stdos" -) - -func Stuff(w io.Writer, cur here.Info, paths []here.Path) error { - disk, err := stdos.New(cur) - if err != nil { - return err - } - - pkg, err := mem.New(cur) - if err != nil { - return err - } - - for _, pt := range paths { - err = func() error { - f, err := disk.Open(pt.String()) - if err != nil { - return err - } - defer f.Close() - if err := pkg.Add(f); err != nil { - return err - } - - return nil - }() - if err != nil { - return err - } - } - - b, err := pkg.MarshalEmbed() - if err != nil { - return err - } - _, err = w.Write(b) - return err -}