From a63d5995a31aa0763d4f7fc40eb666b0aa1b82d1 Mon Sep 17 00:00:00 2001 From: Mark Bates Date: Thu, 1 Aug 2019 22:34:32 -0400 Subject: [PATCH] compress packages --- cmd/pkger/list.go | 4 +- cmd/pkger/pack.go | 6 +-- cmd/pkger/walk.go | 3 +- pkgs/current.go => current.go | 2 +- file.go | 60 +++++++++++------------ file_test.go | 3 +- index.go | 73 +++++++++------------------- index_test.go | 5 +- internal/examples/walk/go.sum | 2 + internal/examples/walk/main.go | 3 +- paths/parse.go => parse.go | 6 +-- paths/parse_test.go => parse_test.go | 2 +- parser/parser.go | 27 +++++----- parser/visitor.go | 12 ++--- paths/path.go => path.go | 2 +- paths/path_test.go => path_test.go | 2 +- paths/paths_map.go => paths_map.go | 2 +- pkger.go | 6 +-- pkger_test.go | 9 +--- walk.go | 6 +-- 20 files changed, 93 insertions(+), 142 deletions(-) rename pkgs/current.go => current.go (96%) rename paths/parse.go => parse.go (92%) rename paths/parse_test.go => parse_test.go (97%) rename paths/path.go => path.go (97%) rename paths/path_test.go => path_test.go (97%) rename paths/paths_map.go => paths_map.go (99%) diff --git a/cmd/pkger/list.go b/cmd/pkger/list.go index ceb6f26..911da05 100644 --- a/cmd/pkger/list.go +++ b/cmd/pkger/list.go @@ -3,12 +3,12 @@ package main import ( "fmt" + "github.com/markbates/pkger" "github.com/markbates/pkger/parser" - "github.com/markbates/pkger/pkgs" ) func list(args []string) error { - info, err := pkgs.Current() + info, err := pkger.Current() if err != nil { return err } diff --git a/cmd/pkger/pack.go b/cmd/pkger/pack.go index d37fa93..705ad78 100644 --- a/cmd/pkger/pack.go +++ b/cmd/pkger/pack.go @@ -11,14 +11,12 @@ import ( "github.com/markbates/pkger" "github.com/markbates/pkger/parser" - "github.com/markbates/pkger/paths" - "github.com/markbates/pkger/pkgs" ) const outName = "pkged.go" func pack(args []string) error { - info, err := pkgs.Current() + info, err := pkger.Current() if err != nil { return err } @@ -41,7 +39,7 @@ func pack(args []string) error { return nil } -func Package(p string, paths []paths.Path) error { +func Package(p string, paths []pkger.Path) error { os.RemoveAll(p) var files []*pkger.File diff --git a/cmd/pkger/walk.go b/cmd/pkger/walk.go index 0f8294f..59c642c 100644 --- a/cmd/pkger/walk.go +++ b/cmd/pkger/walk.go @@ -5,11 +5,10 @@ import ( "os" "github.com/markbates/pkger" - "github.com/markbates/pkger/paths" ) func walk(args []string) error { - err := pkger.Walk(".", func(path paths.Path, info os.FileInfo) error { + err := pkger.Walk(".", func(path pkger.Path, info os.FileInfo) error { fmt.Println(path) return nil }) diff --git a/pkgs/current.go b/current.go similarity index 96% rename from pkgs/current.go rename to current.go index 1d916d4..2aa5575 100644 --- a/pkgs/current.go +++ b/current.go @@ -1,6 +1,6 @@ // TODO: need to populate in memory cache when packed. // you can't use go list, etc... in prod -package pkgs +package pkger import ( "github.com/gobuffalo/here" diff --git a/file.go b/file.go index c119ddf..6c11602 100644 --- a/file.go +++ b/file.go @@ -8,11 +8,9 @@ import ( "io/ioutil" "net/http" "os" - "path" "time" "github.com/gobuffalo/here" - "github.com/markbates/pkger/paths" ) const timeFmt = time.RFC3339Nano @@ -20,9 +18,8 @@ const timeFmt = time.RFC3339Nano type File struct { info *FileInfo her here.Info - path paths.Path + path Path data []byte - index *index writer io.ReadWriter Source io.ReadCloser } @@ -95,7 +92,6 @@ func (f File) MarshalJSON() ([]byte, error) { m["info"] = f.info m["her"] = f.her m["path"] = f.path - m["index"] = f.index m["data"] = f.data if len(f.data) == 0 { b, err := ioutil.ReadAll(&f) @@ -138,30 +134,16 @@ func (f *File) UnmarshalJSON(b []byte) error { return err } - ind, ok := m["index"] - if !ok { - return fmt.Errorf("missing index") - } - f.index = newIndex() - if err := json.Unmarshal(ind, f.index); err != nil { - return err - } return nil } func (f *File) Open(name string) (http.File, error) { - if f.index == nil { - f.index = newIndex() - } - // name = strings.TrimPrefix(name, "/") - pt, err := paths.Parse(name) + pt, err := Parse(name) if err != nil { return nil, err } - pt.Name = path.Join(f.Path().Name, pt.Name) - if len(pt.Pkg) == 0 { pt.Pkg = f.path.Pkg } @@ -170,22 +152,36 @@ func (f *File) Open(name string) (http.File, error) { crs: &byteCRS{bytes.NewReader(f.data)}, } - if pt == f.path { - h.File = f - } else { - of, err := f.index.Open(pt) - if err != nil { - return nil, err - } - defer of.Close() - h.File = of - } - if len(f.data) > 0 { return h, nil } + if pt == f.path { + h.File = f + } + + if h.File == nil { + of, err := rootIndex.Open(pt) + if err != nil { + return nil, err + } + h.File = of + } + bf, err := f.her.Open(h.File.FilePath()) + if err != nil { + if _, ok := err.(*os.PathError); ok { + return h, nil + } + + b, err := ioutil.ReadAll(h.File) + if err != nil { + return h, err + } + h.crs = &byteCRS{bytes.NewReader(b)} + return h, err + } + if err != nil { return h, err } @@ -222,7 +218,7 @@ func (f File) FilePath() string { return f.her.FilePath(f.Name()) } -func (f File) Path() paths.Path { +func (f File) Path() Path { return f.path } diff --git a/file_test.go b/file_test.go index 2597daa..0a7447e 100644 --- a/file_test.go +++ b/file_test.go @@ -6,7 +6,6 @@ import ( "strings" "testing" - "github.com/markbates/pkger/paths" "github.com/stretchr/testify/require" ) @@ -55,7 +54,7 @@ func Test_File_Write(t *testing.T) { i := newIndex() - f, err := i.Create(paths.Path{ + f, err := i.Create(Path{ Name: "/hello.txt", }) r.NoError(err) diff --git a/index.go b/index.go index 181a1fb..147c1eb 100644 --- a/index.go +++ b/index.go @@ -9,24 +9,20 @@ import ( "time" "github.com/gobuffalo/here" - "github.com/markbates/pkger/paths" - "github.com/markbates/pkger/pkgs" ) type index struct { - Pkg string - Files map[paths.Path]*File + Files map[Path]*File } -func (i *index) Create(pt paths.Path) (*File, error) { - her, err := pkgs.Pkg(pt.Pkg) +func (i *index) Create(pt Path) (*File, error) { + her, err := Pkg(pt.Pkg) if err != nil { return nil, err } f := &File{ - path: pt, - index: newIndex(), - her: her, + path: pt, + her: her, info: &FileInfo{ name: strings.TrimPrefix(pt.Name, "/"), mode: 0666, @@ -39,9 +35,7 @@ func (i *index) Create(pt paths.Path) (*File, error) { } func (i index) MarshalJSON() ([]byte, error) { - m := map[string]interface{}{ - "pkg": i.Pkg, - } + m := map[string]interface{}{} fm := map[string]File{} @@ -54,10 +48,7 @@ func (i index) MarshalJSON() ([]byte, error) { return json.Marshal(m) } -func (i index) Walk(pt paths.Path, wf WalkFunc) error { - if len(pt.Pkg) == 0 { - pt.Pkg = i.Pkg - } +func (i index) Walk(pt Path, wf WalkFunc) error { if len(i.Files) > 0 { for k, v := range i.Files { if k.Pkg != pt.Pkg { @@ -72,7 +63,7 @@ func (i index) Walk(pt paths.Path, wf WalkFunc) error { var info here.Info var err error if pt.Pkg == "." { - info, err = pkgs.Current() + info, err = Current() if err != nil { return err } @@ -80,7 +71,7 @@ func (i index) Walk(pt paths.Path, wf WalkFunc) error { } if info.IsZero() { - info, err = pkgs.Pkg(pt.Pkg) + info, err = Pkg(pt.Pkg) if err != nil { return fmt.Errorf("%s: %s", pt, err) } @@ -92,7 +83,7 @@ func (i index) Walk(pt paths.Path, wf WalkFunc) error { } path = strings.TrimPrefix(path, info.Dir) - pt, err := paths.Parse(fmt.Sprintf("%s:%s", pt.Pkg, path)) + pt, err := Parse(fmt.Sprintf("%s:%s", pt.Pkg, path)) if err != nil { return err } @@ -102,28 +93,21 @@ func (i index) Walk(pt paths.Path, wf WalkFunc) error { return err } -func (i index) Open(pt paths.Path) (*File, error) { - if len(pt.Pkg) == 0 { - pt.Pkg = i.Pkg - } +func (i *index) Open(pt Path) (*File, error) { f, ok := i.Files[pt] if !ok { return i.openDisk(pt) } return &File{ - info: f.info, - path: f.path, - data: f.data, - her: f.her, - index: newIndex(), + info: f.info, + path: f.path, + data: f.data, + her: f.her, }, nil } -func (i index) openDisk(pt paths.Path) (*File, error) { - if len(pt.Pkg) == 0 { - pt.Pkg = i.Pkg - } - info, err := pkgs.Pkg(pt.Pkg) +func (i index) openDisk(pt Path) (*File, error) { + info, err := Pkg(pt.Pkg) if err != nil { return nil, err } @@ -140,30 +124,17 @@ func (i index) openDisk(pt paths.Path) (*File, error) { info: WithName(strings.TrimPrefix(pt.Name, "/"), NewFileInfo(fi)), her: info, path: pt, - index: &index{ - Files: map[paths.Path]*File{}, - }, } return f, nil } -func (i index) Parse(p string) (paths.Path, error) { - pt, err := paths.Parse(p) - if err != nil { - return pt, err - } - - if len(pt.Pkg) == 0 { - pt.Pkg = i.Pkg - } - - return pt, nil -} - func newIndex() *index { return &index{ - Files: map[paths.Path]*File{}, + Files: map[Path]*File{}, } } -var rootIndex = newIndex() +var rootIndex = func() *index { + i := newIndex() + return i +}() diff --git a/index_test.go b/index_test.go index 776bbd8..0286b12 100644 --- a/index_test.go +++ b/index_test.go @@ -6,7 +6,6 @@ import ( "strings" "testing" - "github.com/markbates/pkger/paths" "github.com/stretchr/testify/require" ) @@ -15,7 +14,7 @@ func Test_index_Create(t *testing.T) { i := newIndex() - f, err := i.Create(paths.Path{ + f, err := i.Create(Path{ Name: "/hello.txt", }) r.NoError(err) @@ -38,7 +37,7 @@ func Test_index_Create_Write(t *testing.T) { i := newIndex() - f, err := i.Create(paths.Path{ + f, err := i.Create(Path{ Name: "/hello.txt", }) r.NoError(err) diff --git a/internal/examples/walk/go.sum b/internal/examples/walk/go.sum index 3af4d7a..fd60a48 100644 --- a/internal/examples/walk/go.sum +++ b/internal/examples/walk/go.sum @@ -5,6 +5,8 @@ github.com/gobuffalo/envy v1.7.0 h1:GlXgaiBkmrYMHco6t4j7SacKO4XUjvh5pwXh0f4uxXU= github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= github.com/gobuffalo/here v0.2.1 h1:YWZUvrHnxNCIY2nnHPnF5Ob99Z5Iq29wHioLgcY+2G0= github.com/gobuffalo/here v0.2.1/go.mod h1:2a6G14FaAKOGJMK/5UNa4Og/+iyFS5cq3MnlvFR7YDk= +github.com/gobuffalo/here v0.2.2 h1:AXEK2ApOb4F5cKZ46Ofi8inGWa0qy5ChmJXAK5/IDmo= +github.com/gobuffalo/here v0.2.2/go.mod h1:2a6G14FaAKOGJMK/5UNa4Og/+iyFS5cq3MnlvFR7YDk= github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= diff --git a/internal/examples/walk/main.go b/internal/examples/walk/main.go index cb3e426..0eb385a 100644 --- a/internal/examples/walk/main.go +++ b/internal/examples/walk/main.go @@ -6,11 +6,10 @@ import ( "os" "github.com/markbates/pkger" - "github.com/markbates/pkger/paths" ) func main() { - err := pkger.Walk("github.com/gobuffalo/envy", func(path paths.Path, info os.FileInfo) error { + err := pkger.Walk("github.com/gobuffalo/envy", func(path pkger.Path, info os.FileInfo) error { fmt.Println(path) return nil }) diff --git a/paths/parse.go b/parse.go similarity index 92% rename from paths/parse.go rename to parse.go index d3abdb6..451c0b7 100644 --- a/paths/parse.go +++ b/parse.go @@ -1,12 +1,10 @@ -package paths +package pkger import ( "fmt" "regexp" "strings" "sync" - - "github.com/markbates/pkger/pkgs" ) var cache = pathsMap{ @@ -44,7 +42,7 @@ func build(p, pkg, name string) (Path, error) { Name: name, } - info, err := pkgs.Current() + info, err := Current() if err != nil { return pt, err } diff --git a/paths/parse_test.go b/parse_test.go similarity index 97% rename from paths/parse_test.go rename to parse_test.go index aa9551b..30b6673 100644 --- a/paths/parse_test.go +++ b/parse_test.go @@ -1,4 +1,4 @@ -package paths +package pkger import ( "testing" diff --git a/parser/parser.go b/parser/parser.go index e4d2665..f02d15e 100644 --- a/parser/parser.go +++ b/parser/parser.go @@ -6,15 +6,14 @@ import ( "sort" "strings" - "github.com/markbates/pkger/paths" - "github.com/markbates/pkger/pkgs" + "github.com/markbates/pkger" ) var DefaultIgnoredFolders = []string{".", "_", "vendor", "node_modules", "_fixtures", "testdata"} func Parse(name string) (Results, error) { var r Results - c, err := pkgs.Current() + c, err := pkger.Current() if err != nil { return r, err } @@ -23,19 +22,19 @@ func Parse(name string) (Results, error) { name = c.ImportPath } - pt, err := paths.Parse(name) + pt, err := pkger.Parse(name) if err != nil { return r, err } r.Path = pt - her, err := pkgs.Pkg(r.Path.Pkg) + her, err := pkger.Pkg(r.Path.Pkg) if err != nil { return r, err } - m := map[paths.Path]bool{} + m := map[pkger.Path]bool{} root := r.Path.Name if !strings.HasPrefix(root, string(filepath.Separator)) { @@ -116,7 +115,7 @@ func Parse(name string) (Results, error) { return nil }) - var found []paths.Path + var found []pkger.Path for k := range m { if len(k.String()) == 0 { @@ -132,10 +131,10 @@ func Parse(name string) (Results, error) { return r, err } -func sourceFiles(pt paths.Path) ([]paths.Path, error) { - var res []paths.Path +func sourceFiles(pt pkger.Path) ([]pkger.Path, error) { + var res []pkger.Path - her, err := pkgs.Pkg(pt.Pkg) + her, err := pkger.Pkg(pt.Pkg) if err != nil { return res, err @@ -150,7 +149,7 @@ func sourceFiles(pt paths.Path) ([]paths.Path, error) { return res, nil } - c, err := pkgs.Current() + c, err := pkger.Current() if err != nil { return res, err } @@ -189,7 +188,7 @@ func sourceFiles(pt paths.Path) ([]paths.Path, error) { } n := strings.TrimPrefix(strings.TrimPrefix(p, her.Dir), "/") - pt := paths.Path{ + pt := pkger.Path{ Name: n, } res = append(res, pt) @@ -200,6 +199,6 @@ func sourceFiles(pt paths.Path) ([]paths.Path, error) { } type Results struct { - Paths []paths.Path - Path paths.Path + Paths []pkger.Path + Path pkger.Path } diff --git a/parser/visitor.go b/parser/visitor.go index 0767465..7b776f4 100644 --- a/parser/visitor.go +++ b/parser/visitor.go @@ -5,23 +5,23 @@ import ( "go/ast" "strconv" - "github.com/markbates/pkger/paths" + "github.com/markbates/pkger" ) type Visitor struct { File string - Found map[paths.Path]bool + Found map[pkger.Path]bool errors []error } func NewVisitor(p string) (*Visitor, error) { return &Visitor{ File: p, - Found: map[paths.Path]bool{}, + Found: map[pkger.Path]bool{}, }, nil } -func (v *Visitor) Run() ([]paths.Path, error) { +func (v *Visitor) Run() ([]pkger.Path, error) { pf, err := parseFile(v.File) if err != nil { return nil, err @@ -29,7 +29,7 @@ func (v *Visitor) Run() ([]paths.Path, error) { ast.Walk(v, pf.Ast) - var found []paths.Path + var found []pkger.Path for k := range v.Found { found = append(found, k) @@ -40,7 +40,7 @@ func (v *Visitor) Run() ([]paths.Path, error) { func (v *Visitor) addPath(p string) error { p, _ = strconv.Unquote(p) - pt, err := paths.Parse(p) + pt, err := pkger.Parse(p) if err != nil { return err } diff --git a/paths/path.go b/path.go similarity index 97% rename from paths/path.go rename to path.go index 2c03c8d..5349e76 100644 --- a/paths/path.go +++ b/path.go @@ -1,4 +1,4 @@ -package paths +package pkger import ( "encoding/json" diff --git a/paths/path_test.go b/path_test.go similarity index 97% rename from paths/path_test.go rename to path_test.go index 7204c57..90aaae8 100644 --- a/paths/path_test.go +++ b/path_test.go @@ -1,4 +1,4 @@ -package paths +package pkger import ( "testing" diff --git a/paths/paths_map.go b/paths_map.go similarity index 99% rename from paths/paths_map.go rename to paths_map.go index dc4148c..d82f02b 100644 --- a/paths/paths_map.go +++ b/paths_map.go @@ -1,6 +1,6 @@ // Code generated by github.com/gobuffalo/mapgen. DO NOT EDIT. -package paths +package pkger import ( "sort" diff --git a/pkger.go b/pkger.go index 7a91384..607d799 100644 --- a/pkger.go +++ b/pkger.go @@ -1,10 +1,8 @@ package pkger -import "github.com/markbates/pkger/paths" - // Open opens the named file for reading. func Open(p string) (*File, error) { - pt, err := paths.Parse(p) + pt, err := Parse(p) if err != nil { return nil, err } @@ -13,7 +11,7 @@ func Open(p string) (*File, error) { // Create creates the named file with mode 0666 (before umask), truncating it if it already exists. If successful, methods on the returned File can be used for I/O; the associated file descriptor has mode O_RDWR. If there is an error, it will be of type *PathError. func Create(p string) (*File, error) { - pt, err := paths.Parse(p) + pt, err := Parse(p) if err != nil { return nil, err } diff --git a/pkger_test.go b/pkger_test.go index 030c6b4..5d7b877 100644 --- a/pkger_test.go +++ b/pkger_test.go @@ -5,17 +5,12 @@ import ( "strings" ) -func createFile(i *index, p string, body ...string) (*File, error) { - pt, err := i.Parse(p) - if err != nil { - return nil, err - } - +func createFile(p string, body ...string) (*File, error) { if len(body) == 0 { body = append(body, radio) } - f, err := i.Create(pt) + f, err := Create(p) if err != nil { return nil, err } diff --git a/walk.go b/walk.go index c8f3cef..9501bb0 100644 --- a/walk.go +++ b/walk.go @@ -2,14 +2,12 @@ package pkger import ( "os" - - "github.com/markbates/pkger/paths" ) -type WalkFunc func(paths.Path, os.FileInfo) error +type WalkFunc func(Path, os.FileInfo) error func Walk(p string, wf WalkFunc) error { - pt, err := paths.Parse(p) + pt, err := Parse(p) if err != nil { return err }