diff --git a/Makefile b/Makefile index 8ac1886..380c6d7 100644 --- a/Makefile +++ b/Makefile @@ -17,7 +17,7 @@ build: tidy make tidy test: tidy - $(GO_BIN) test -count 1 -cover -tags ${TAGS} -timeout 10s ./... + $(GO_BIN) test -count 1 -cover -tags ${TAGS} -timeout 1m ./... make tidy cov: diff --git a/README.md b/README.md index 68914d4..27fefd0 100644 --- a/README.md +++ b/README.md @@ -63,11 +63,54 @@ package // Pkger stuff here ``` -## Usage +## Reference Application + +The reference application for the `README` examples, as well as all testing, can be found at [https://github.com/markbates/pkger/tree/master/pkging/pkgtest/testdata/ref](https://github.com/markbates/pkger/tree/master/pkging/pkgtest/testdata/ref). + +``` +├── actions +│   └── actions.go +├── assets +│   ├── css +│   │   ├── _buffalo.scss +│   │   └── application.scss +│   ├── images +│   │   ├── favicon.ico +│   │   └── logo.svg +│   └── js +│   └── application.js +├── go.mod +├── go.sum +├── locales +│   └── all.en-us.yaml +├── main.go +├── mod +│   └── mod.go +├── models +│   └── models.go +├── public +│   ├── assets +│   │   └── app.css +│   ├── images +│   │   └── img1.png +│   ├── index.html +│   └── robots.txt +├── templates +│   ├── _flash.plush.html +│   ├── application.plush.html +│   └── index.plush.html +└── web + └── web.go + +13 directories, 20 files +``` + + +## API Usage Pkger's API is modeled on that of the [`os`](https://godoc.org/os) package in Go's standard library. This makes Pkger usage familiar to Go developers. - +The two most important interfaces are [`github.com/markbates/pkger/pkging#Pkger`](https://godoc.org/github.com/markbates/pkger/pkging#Pkger) and [`github.com/markbates/pkger/pkging#File`](https://godoc.org/github.com/markbates/pkger/pkging#File). ```go type Pkger interface { @@ -97,86 +140,3 @@ type File interface { } ``` -```bash -├── go.mod -├── go.sum -├── main.go -├── public -│   ├── images -│   │   ├── mark-small.png -│   │   ├── img1.png -│   │   ├── mark_250px.png -│   │   └── mark_400px.png -│   └── index.html -``` - -```go -package main - -import ( - "fmt" - "log" - "os" - "text/tabwriter" - "time" - - "github.com/markbates/pkger" -) - -func main() { - if err := run(); err != nil { - log.Fatal(err) - } -} - -func run() error { - w := tabwriter.NewWriter(os.Stdout, 0, 0, 0, ' ', tabwriter.Debug) - defer w.Flush() - - return pkger.Walk("/public", func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - - fmt.Fprintf(w, - "%s \t %d \t %s \t %s \t\n", - info.Name(), - info.Size(), - info.Mode(), - info.ModTime().Format(time.RFC3339), - ) - - return nil - }) - -} -``` - -### Output Without Packing - -```bash -# compile the go binary as usual and run the app: -$ go build -v; ./app - -public | 128 | drwxr-xr-x | 2019-10-18T16:24:55-04:00 | -images | 128 | drwxr-xr-x | 2019-10-18T16:24:55-04:00 | -img1.png | 27718 | -rw-r--r-- | 2019-10-18T16:24:55-04:00 | -img2.png | 27718 | -rw-r--r-- | 2019-10-18T16:24:55-04:00 | -index.html | 257 | -rw-r--r-- | 2019-10-18T16:24:55-04:00 | -``` - -### Output With Packing - -```bash -# run the pkger cli to generate a pkged.go file: -$ pkger - -# compile the go binary as usual and run the app: -$ go build -v; ./app - -public | 128 | drwxr-xr-x | 2019-10-18T16:24:55-04:00 | -images | 128 | drwxr-xr-x | 2019-10-18T16:24:55-04:00 | -img1.png | 27718 | -rw-r--r-- | 2019-10-18T16:24:55-04:00 | -img2.png | 27718 | -rw-r--r-- | 2019-10-18T16:24:55-04:00 | -index.html | 257 | -rw-r--r-- | 2019-10-18T16:24:55-04:00 | -``` diff --git a/apply.go b/apply.go index fcf954f..e59b19c 100644 --- a/apply.go +++ b/apply.go @@ -1,5 +1,3 @@ -// +build !debug - package pkger import ( diff --git a/cmd/pkger/cmds/list.go b/cmd/pkger/cmds/list.go index 4d0e085..b0d3f6f 100644 --- a/cmd/pkger/cmds/list.go +++ b/cmd/pkger/cmds/list.go @@ -16,9 +16,10 @@ import ( type listCmd struct { *flag.FlagSet - help bool - json bool - subs []command + help bool + json bool + include slice + subs []command } func (e *listCmd) Name() string { @@ -43,7 +44,7 @@ func (e *listCmd) Exec(args []string) error { fp := filepath.Join(info.Dir, outName) os.RemoveAll(fp) - decls, err := parser.Parse(info) + decls, err := parser.Parse(info, e.include...) if err != nil { return err } @@ -93,6 +94,7 @@ func (e *listCmd) Flags() *flag.FlagSet { if e.FlagSet == nil { e.FlagSet = flag.NewFlagSet("list", flag.ExitOnError) e.BoolVar(&e.json, "json", false, "prints in JSON format") + e.Var(&e.include, "include", "packages the specified file or directory") } e.Usage = Usage(os.Stderr, e.FlagSet) return e.FlagSet diff --git a/cmd/pkger/cmds/pack.go b/cmd/pkger/cmds/pack.go index 5feec33..bc056a9 100644 --- a/cmd/pkger/cmds/pack.go +++ b/cmd/pkger/cmds/pack.go @@ -14,13 +14,25 @@ import ( "github.com/markbates/pkger/pkging/pkgutil" ) +type slice []string + +func (i slice) String() string { + return fmt.Sprintf("%s", []string(i)) +} + +func (i *slice) Set(value string) error { + *i = append(*i, value) + return nil +} + const outName = "pkged.go" type packCmd struct { *flag.FlagSet - out string - help bool - subs []command + out string + help bool + include slice + subs []command } func (e *packCmd) Name() string { @@ -36,7 +48,7 @@ func (e *packCmd) Exec(args []string) error { fp := filepath.Join(info.Dir, e.out, outName) os.RemoveAll(fp) - decls, err := parser.Parse(info) + decls, err := parser.Parse(info, e.include...) if err != nil { return err } @@ -92,6 +104,7 @@ func New() (*packCmd, error) { c.FlagSet = flag.NewFlagSet("pkger", flag.ExitOnError) c.BoolVar(&c.help, "h", false, "prints help information") c.StringVar(&c.out, "o", "", "output directory for pkged.go") + c.Var(&c.include, "include", "packages the specified file or directory") c.Usage = func() { fmt.Fprintf(os.Stderr, "Usage:\n\n") Usage(os.Stderr, c.FlagSet)() @@ -112,6 +125,11 @@ func (e *packCmd) Flags() *flag.FlagSet { func Package(out string, decls parser.Decls) error { os.RemoveAll(out) + defer func() { + if err := recover(); err != nil { + os.RemoveAll(out) + } + }() f, err := os.Create(out) if err != nil { diff --git a/cmd/pkger/main.go b/cmd/pkger/main.go index f4b94f4..a3aa5ea 100644 --- a/cmd/pkger/main.go +++ b/cmd/pkger/main.go @@ -10,7 +10,7 @@ import ( func main() { clean := func() { - c := exec.Command("go", "mod", "tidy", "-v") + c := exec.Command("go", "mod", "tidy") c.Stdout = os.Stdout c.Stderr = os.Stderr c.Stdin = os.Stdin @@ -18,6 +18,13 @@ func main() { } defer clean() + defer func() { + if err := recover(); err != nil { + clean() + log.Fatal(err) + } + }() + if err := run(); err != nil { clean() log.Fatal(err) diff --git a/here/here.go b/here/here.go index ad81774..f2b3452 100644 --- a/here/here.go +++ b/here/here.go @@ -5,6 +5,7 @@ import ( "fmt" "os/exec" "regexp" + "strings" "sync" ) @@ -20,7 +21,7 @@ func run(n string, args ...string) ([]byte, error) { c.Stderr = bb err := c.Run() if err != nil { - return nil, fmt.Errorf("%w: %s", err, bb) + return nil, fmt.Errorf("%w: %q: %s", err, strings.Join(c.Args, " "), bb) } return bb.Bytes(), nil diff --git a/here/pkg.go b/here/pkg.go index c810b77..9d51c07 100644 --- a/here/pkg.go +++ b/here/pkg.go @@ -2,8 +2,7 @@ package here import ( "encoding/json" - "path" - "strings" + "fmt" ) // Package attempts to gather info for the requested package. @@ -18,14 +17,12 @@ import ( func Package(p string) (Info, error) { i, err := Cache(p, func(p string) (Info, error) { var i Info + if len(p) == 0 || p == "." { + return i, fmt.Errorf("missing package name") + } b, err := run("go", "list", "-json", "-find", p) if err != nil { - if !strings.Contains(err.Error(), "can't load package: package") { - return i, err - } - - p, _ = path.Split(p) - return Package(p) + return i, err } if err := json.Unmarshal(b, &i); err != nil { return i, err diff --git a/parser/include.go b/parser/include.go new file mode 100644 index 0000000..ef5409f --- /dev/null +++ b/parser/include.go @@ -0,0 +1,57 @@ +package parser + +import ( + "encoding/json" + "fmt" + "go/token" + "os" +) + +var _ Decl = IncludeDecl{} + +type IncludeDecl struct { + file *File + pos token.Position + value string +} + +func (d IncludeDecl) String() string { + return fmt.Sprintf("pkger.Include(%q)", d.value) +} + +func (d IncludeDecl) MarshalJSON() ([]byte, error) { + return json.Marshal(map[string]interface{}{ + "type": "pkger.Include", + "file": d.file, + "pos": d.pos, + "value": d.value, + }) +} + +func (d IncludeDecl) File() (*File, error) { + if d.file == nil { + return nil, os.ErrNotExist + } + return d.file, nil +} + +func (d IncludeDecl) Position() (token.Position, error) { + return d.pos, nil +} + +func (d IncludeDecl) Value() (string, error) { + if d.value == "" { + return "", os.ErrNotExist + } + return d.value, nil +} + +func (d IncludeDecl) Files(virtual map[string]string) ([]*File, error) { + od := OpenDecl{ + file: d.file, + pos: d.pos, + value: d.value, + } + + return od.Files(virtual) +} diff --git a/parser/open.go b/parser/open.go index 21efaf6..85ec71e 100644 --- a/parser/open.go +++ b/parser/open.go @@ -70,6 +70,7 @@ func (d OpenDecl) Files(virtual map[string]string) ([]*File, error) { } return wd.Files(virtual) } + var files []*File files = append(files, d.file) diff --git a/parser/parser.go b/parser/parser.go index 032fa5d..99f6ce2 100644 --- a/parser/parser.go +++ b/parser/parser.go @@ -16,15 +16,32 @@ import ( var defaultIgnoredFolders = []string{".", "_", "vendor", "node_modules", "testdata"} -func Parse(her here.Info) (Decls, error) { +func New(her here.Info) (*Parser, error) { + return &Parser{ + Info: her, + decls: map[string]Decls{}, + }, nil +} + +type Parser struct { + here.Info + decls map[string]Decls + once sync.Once + includes []string + err error +} + +func Parse(her here.Info, includes ...string) (Decls, error) { p, err := New(her) if err != nil { return nil, err } + p.includes = includes + return p.Decls() } -func ParseSource(source Source, mode parser.Mode) (*ParsedSource, error) { +func (p *Parser) ParseSource(source Source, mode parser.Mode) (*ParsedSource, error) { pf := &ParsedSource{ Source: source, FileSet: token.NewFileSet(), @@ -45,7 +62,7 @@ func ParseSource(source Source, mode parser.Mode) (*ParsedSource, error) { return pf, nil } -func ParseFile(abs string, mode parser.Mode) (*ParsedSource, error) { +func (p *Parser) ParseFile(abs string, mode parser.Mode) (*ParsedSource, error) { s := Source{ Abs: abs, } @@ -68,10 +85,10 @@ func ParseFile(abs string, mode parser.Mode) (*ParsedSource, error) { s.Path, err = s.Here.Parse(strings.TrimPrefix(abs, dir)) - return ParseSource(s, 0) + return p.ParseSource(s, 0) } -func ParseDir(abs string, mode parser.Mode) ([]*ParsedSource, error) { +func (p *Parser) ParseDir(abs string, mode parser.Mode) ([]*ParsedSource, error) { info, err := os.Stat(abs) if err != nil { return nil, err @@ -116,19 +133,6 @@ func ParseDir(abs string, mode parser.Mode) ([]*ParsedSource, error) { return srcs, nil } -func New(her here.Info) (*Parser, error) { - return &Parser{ - Info: her, - decls: map[string]Decls{}, - }, nil -} - -type Parser struct { - here.Info - decls map[string]Decls - once sync.Once - err error -} func (p *Parser) Decls() (Decls, error) { if err := p.parse(); err != nil { @@ -139,6 +143,7 @@ func (p *Parser) Decls() (Decls, error) { orderedNames := []string{ "MkdirAll", "Create", + "Include", "Stat", "Open", "Dir", @@ -166,12 +171,18 @@ func (p *Parser) Parse() error { func (p *Parser) parse() error { p.decls = map[string]Decls{} + root := p.Dir + if err := p.parseIncludes(); err != nil { + return err + } + fi, err := os.Stat(root) if err != nil { return err } + if !fi.IsDir() { return fmt.Errorf("%q is not a directory", root) } @@ -192,7 +203,7 @@ func (p *Parser) parse() error { } } - srcs, err := ParseDir(path, 0) + srcs, err := p.ParseDir(path, 0) if err != nil { return fmt.Errorf("%w: %s", err, path) } @@ -212,3 +223,33 @@ func (p *Parser) parse() error { return err } + +func (p *Parser) parseIncludes() error { + for _, i := range p.includes { + pt, err := p.Info.Parse(i) + if err != nil { + return err + } + + her := p.Info + if pt.Pkg != her.ImportPath { + her, err = here.Package(pt.Pkg) + if err != nil { + return err + } + } + + abs := filepath.Join(her.Module.Dir, pt.Name) + + f := &File{ + Abs: abs, + Path: pt, + Here: her, + } + p.decls["Include"] = append(p.decls["Include"], IncludeDecl{ + value: i, + file: f, + }) + } + return nil +} diff --git a/parser/parser_test.go b/parser/parser_test.go index 7539dbd..d47e0bb 100644 --- a/parser/parser_test.go +++ b/parser/parser_test.go @@ -36,7 +36,46 @@ func Test_Parser_Ref(t *testing.T) { files, err := res.Files() r.NoError(err) - r.Len(files, 23) + r.Len(files, 25) + + for _, f := range files { + if f.Path.Pkg == ref.Module.Path { + r.True(strings.HasPrefix(f.Abs, ref.Dir), "%q %q", f.Abs, ref.Dir) + } else { + r.False(strings.HasPrefix(f.Abs, ref.Dir), "%q %q", f.Abs, ref.Dir) + } + } +} + +func Test_Parser_Ref_Include(t *testing.T) { + defer func() { + c := exec.Command("go", "mod", "tidy", "-v") + c.Run() + }() + r := require.New(t) + + here.ClearCache() + ref, err := pkgtest.NewRef() + r.NoError(err) + defer os.RemoveAll(ref.Dir) + + disk, err := stdos.New(ref.Info) + r.NoError(err) + + _, err = pkgtest.LoadFiles("/", ref, disk) + r.NoError(err) + + res, err := Parse(ref.Info, "github.com/gobuffalo/buffalo:/app.go") + + r.NoError(err) + + files, err := res.Files() + r.NoError(err) + // t.FailNow() + + l := len(files) + r.Equal(26, l) + // r.Len(files, 27) for _, f := range files { if f.Path.Pkg == ref.Module.Path { diff --git a/parser/source.go b/parser/source.go index 662dc2d..276f632 100644 --- a/parser/source.go +++ b/parser/source.go @@ -88,6 +88,14 @@ func (p *ParsedSource) parse() error { value: value, } } + case "Include": + fn = func(f File, pos token.Position, value string) Decl { + return IncludeDecl{ + file: &f, + pos: pos, + value: value, + } + } case "Stat": fn = func(f File, pos token.Position, value string) Decl { return StatDecl{ diff --git a/pkger.go b/pkger.go index 2cf1dc6..6b4daa8 100644 --- a/pkger.go +++ b/pkger.go @@ -95,3 +95,6 @@ func Remove(name string) error { func RemoveAll(name string) error { return impl().RemoveAll(name) } + +// Include is a no-op that directs the pkger tool to include the desired file or folder. +func Include(name string) {} diff --git a/pkging/pkger.go b/pkging/pkger.go index 0022db2..c510ff8 100644 --- a/pkging/pkger.go +++ b/pkging/pkger.go @@ -15,26 +15,38 @@ type Pkger interface { Current() (here.Info, error) // Info returns the here.Info of the here.Path + // @Parser Directive Info(p string) (here.Info, error) // Create creates the named file with mode 0666 (before umask) - It's actually 0644, truncating it if it already exists. If successful, methods on the returned File can be used for I/O; the associated file descriptor has mode O_RDWR. + // @Parser Directive Create(name string) (File, error) // MkdirAll creates a directory named path, along with any necessary parents, and returns nil, or else returns an error. The permission bits perm (before umask) are used for all directories that MkdirAll creates. If path is already a directory, MkdirAll does nothing and returns nil. + // @Parser Directive MkdirAll(p string, perm os.FileMode) error // Open opens the named file for reading. If successful, methods on the returned file can be used for reading; the associated file descriptor has mode O_RDONLY. + // @Parser Directive Open(name string) (File, error) // Stat returns a FileInfo describing the named file. + // @Parser Directive Stat(name string) (os.FileInfo, error) // Walk walks the file tree rooted at root, calling walkFn for each file or directory in the tree, including root. All errors that arise visiting files and directories are filtered by walkFn. The files are walked in lexical order, which makes the output deterministic but means that for very large directories Walk can be inefficient. Walk does not follow symbolic links. - That is from the standard library. I know. Their grammar teachers can not be happy with them right now. + // @Parser Directive Walk(p string, wf filepath.WalkFunc) error // Remove removes the named file or (empty) directory. + // @Parser Directive Remove(name string) error // RemoveAll removes path and any children it contains. It removes everything it can but returns the first error it encounters. If the path does not exist, RemoveAll returns nil (no error). + // @Parser Directive RemoveAll(path string) error + + // Include is a no-op that directs the pkger tool to include the desired file or folder. + // @Parser Directive + // pkger.Include } diff --git a/pkging/pkgtest/testdata/ref/go.mod b/pkging/pkgtest/testdata/ref/go.mod index a335439..18f4f20 100644 --- a/pkging/pkgtest/testdata/ref/go.mod +++ b/pkging/pkgtest/testdata/ref/go.mod @@ -2,9 +2,6 @@ module app go 1.13 -require ( - github.com/gobuffalo/buffalo v0.15.0 // indirect - github.com/markbates/pkger v0.0.0 -) +require github.com/markbates/pkger v0.0.0 replace github.com/markbates/pkger => ../../../../ diff --git a/pkging/pkgtest/testdata/ref/main.go b/pkging/pkgtest/testdata/ref/main.go index d28c351..d3e41a3 100644 --- a/pkging/pkgtest/testdata/ref/main.go +++ b/pkging/pkgtest/testdata/ref/main.go @@ -12,6 +12,7 @@ import ( ) func main() { + pkger.Include("/web") if err := run(); err != nil { log.Fatal(err) } diff --git a/pkging/pkgutil/stuff_test.go b/pkging/pkgutil/stuff_test.go index e4975ee..681d3f4 100644 --- a/pkging/pkgutil/stuff_test.go +++ b/pkging/pkgutil/stuff_test.go @@ -29,7 +29,7 @@ func Test_Stuff(t *testing.T) { decls, err := parser.Parse(ref.Info) r.NoError(err) - r.Len(decls, 10) + r.Len(decls, 11) files, err := decls.Files() r.NoError(err) @@ -42,7 +42,7 @@ func Test_Stuff(t *testing.T) { } } - r.Len(files, 23) + r.Len(files, 25) bb := &bytes.Buffer{}