Merge pull request #29 from markbates/minotaur

Adds -include flag and pkger.Include directive fixes #28

Makefile

@@ -17,7 +17,7 @@ build: tidy
 	make tidy
 
 test: tidy
-	$(GO_BIN) test -count 1 -cover -tags ${TAGS} -timeout 10s ./...
+	$(GO_BIN) test -count 1 -cover -tags ${TAGS} -timeout 1m ./...
 	make tidy
 
 cov:

README.md

@@ -63,11 +63,54 @@ package <reader>
 // Pkger stuff here
 ```
 
-## Usage
+## Reference Application
+
+The reference application for the `README` examples, as well as all testing, can be found at [https://github.com/markbates/pkger/tree/master/pkging/pkgtest/testdata/ref](https://github.com/markbates/pkger/tree/master/pkging/pkgtest/testdata/ref).
+
+```
+├── actions
+│   └── actions.go
+├── assets
+│   ├── css
+│   │   ├── _buffalo.scss
+│   │   └── application.scss
+│   ├── images
+│   │   ├── favicon.ico
+│   │   └── logo.svg
+│   └── js
+│       └── application.js
+├── go.mod
+├── go.sum
+├── locales
+│   └── all.en-us.yaml
+├── main.go
+├── mod
+│   └── mod.go
+├── models
+│   └── models.go
+├── public
+│   ├── assets
+│   │   └── app.css
+│   ├── images
+│   │   └── img1.png
+│   ├── index.html
+│   └── robots.txt
+├── templates
+│   ├── _flash.plush.html
+│   ├── application.plush.html
+│   └── index.plush.html
+└── web
+    └── web.go
+
+13 directories, 20 files
+```
+
+
+## API Usage
 
 Pkger's API is modeled on that of the [`os`](https://godoc.org/os) package in Go's standard library. This makes Pkger usage familiar to Go developers.
 
-
+The two most important interfaces are [`github.com/markbates/pkger/pkging#Pkger`](https://godoc.org/github.com/markbates/pkger/pkging#Pkger) and [`github.com/markbates/pkger/pkging#File`](https://godoc.org/github.com/markbates/pkger/pkging#File).
 
 ```go
 type Pkger interface {
@@ -97,86 +140,3 @@ type File interface {
 }
 ```
 
-```bash
-├── go.mod
-├── go.sum
-├── main.go
-├── public
-│   ├── images
-│   │   ├── mark-small.png
-│   │   ├── img1.png
-│   │   ├── mark_250px.png
-│   │   └── mark_400px.png
-│   └── index.html
-```
-
-```go
-package main
-
-import (
-	"fmt"
-	"log"
-	"os"
-	"text/tabwriter"
-	"time"
-
-	"github.com/markbates/pkger"
-)
-
-func main() {
-	if err := run(); err != nil {
-		log.Fatal(err)
-	}
-}
-
-func run() error {
-	w := tabwriter.NewWriter(os.Stdout, 0, 0, 0, ' ', tabwriter.Debug)
-	defer w.Flush()
-
-	return pkger.Walk("/public", func(path string, info os.FileInfo, err error) error {
-		if err != nil {
-			return err
-		}
-
-		fmt.Fprintf(w,
-			"%s \t %d \t %s \t %s \t\n",
-			info.Name(),
-			info.Size(),
-			info.Mode(),
-			info.ModTime().Format(time.RFC3339),
-		)
-
-		return nil
-	})
-
-}
-```
-
-### Output Without Packing
-
-```bash
-# compile the go binary as usual and run the app:
-$ go build -v; ./app
-
-public     | 128   | drwxr-xr-x | 2019-10-18T16:24:55-04:00 |
-images     | 128   | drwxr-xr-x | 2019-10-18T16:24:55-04:00 |
-img1.png   | 27718 | -rw-r--r-- | 2019-10-18T16:24:55-04:00 |
-img2.png   | 27718 | -rw-r--r-- | 2019-10-18T16:24:55-04:00 |
-index.html | 257   | -rw-r--r-- | 2019-10-18T16:24:55-04:00 |
-```
-
-### Output With Packing
-
-```bash
-# run the pkger cli to generate a pkged.go file:
-$ pkger
-
-# compile the go binary as usual and run the app:
-$ go build -v; ./app
-
-public     | 128   | drwxr-xr-x | 2019-10-18T16:24:55-04:00 |
-images     | 128   | drwxr-xr-x | 2019-10-18T16:24:55-04:00 |
-img1.png   | 27718 | -rw-r--r-- | 2019-10-18T16:24:55-04:00 |
-img2.png   | 27718 | -rw-r--r-- | 2019-10-18T16:24:55-04:00 |
-index.html | 257   | -rw-r--r-- | 2019-10-18T16:24:55-04:00 |
-```

apply.go

@@ -1,5 +1,3 @@
-// +build !debug
-
 package pkger
 
 import (

cmd/pkger/cmds/list.go

@@ -18,6 +18,7 @@ type listCmd struct {
 	*flag.FlagSet
 	help    bool
 	json    bool
+	include slice
 	subs    []command
 }
 
@@ -43,7 +44,7 @@ func (e *listCmd) Exec(args []string) error {
 	fp := filepath.Join(info.Dir, outName)
 	os.RemoveAll(fp)
 
-	decls, err := parser.Parse(info)
+	decls, err := parser.Parse(info, e.include...)
 	if err != nil {
 		return err
 	}
@@ -93,6 +94,7 @@ func (e *listCmd) Flags() *flag.FlagSet {
 	if e.FlagSet == nil {
 		e.FlagSet = flag.NewFlagSet("list", flag.ExitOnError)
 		e.BoolVar(&e.json, "json", false, "prints in JSON format")
+		e.Var(&e.include, "include", "packages the specified file or directory")
 	}
 	e.Usage = Usage(os.Stderr, e.FlagSet)
 	return e.FlagSet

cmd/pkger/cmds/pack.go

@@ -14,12 +14,24 @@ import (
 	"github.com/markbates/pkger/pkging/pkgutil"
 )
 
+type slice []string
+
+func (i slice) String() string {
+	return fmt.Sprintf("%s", []string(i))
+}
+
+func (i *slice) Set(value string) error {
+	*i = append(*i, value)
+	return nil
+}
+
 const outName = "pkged.go"
 
 type packCmd struct {
 	*flag.FlagSet
 	out     string
 	help    bool
+	include slice
 	subs    []command
 }
 
@@ -36,7 +48,7 @@ func (e *packCmd) Exec(args []string) error {
 	fp := filepath.Join(info.Dir, e.out, outName)
 	os.RemoveAll(fp)
 
-	decls, err := parser.Parse(info)
+	decls, err := parser.Parse(info, e.include...)
 	if err != nil {
 		return err
 	}
@@ -92,6 +104,7 @@ func New() (*packCmd, error) {
 	c.FlagSet = flag.NewFlagSet("pkger", flag.ExitOnError)
 	c.BoolVar(&c.help, "h", false, "prints help information")
 	c.StringVar(&c.out, "o", "", "output directory for pkged.go")
+	c.Var(&c.include, "include", "packages the specified file or directory")
 	c.Usage = func() {
 		fmt.Fprintf(os.Stderr, "Usage:\n\n")
 		Usage(os.Stderr, c.FlagSet)()
@@ -112,6 +125,11 @@ func (e *packCmd) Flags() *flag.FlagSet {
 
 func Package(out string, decls parser.Decls) error {
 	os.RemoveAll(out)
+	defer func() {
+		if err := recover(); err != nil {
+			os.RemoveAll(out)
+		}
+	}()
 
 	f, err := os.Create(out)
 	if err != nil {

cmd/pkger/main.go

@@ -10,7 +10,7 @@ import (
 
 func main() {
 	clean := func() {
-		c := exec.Command("go", "mod", "tidy", "-v")
+		c := exec.Command("go", "mod", "tidy")
 		c.Stdout = os.Stdout
 		c.Stderr = os.Stderr
 		c.Stdin = os.Stdin
@@ -18,6 +18,13 @@ func main() {
 	}
 	defer clean()
 
+	defer func() {
+		if err := recover(); err != nil {
+			clean()
+			log.Fatal(err)
+		}
+	}()
+
 	if err := run(); err != nil {
 		clean()
 		log.Fatal(err)

here/here.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"os/exec"
 	"regexp"
+	"strings"
 	"sync"
 )
 
@@ -20,7 +21,7 @@ func run(n string, args ...string) ([]byte, error) {
 	c.Stderr = bb
 	err := c.Run()
 	if err != nil {
-		return nil, fmt.Errorf("%w: %s", err, bb)
+		return nil, fmt.Errorf("%w: %q: %s", err, strings.Join(c.Args, " "), bb)
 	}
 
 	return bb.Bytes(), nil

here/pkg.go

@@ -2,8 +2,7 @@ package here
 
 import (
 	"encoding/json"
-	"path"
+	"fmt"
-	"strings"
 )
 
 // Package attempts to gather info for the requested package.
@@ -18,15 +17,13 @@ import (
 func Package(p string) (Info, error) {
 	i, err := Cache(p, func(p string) (Info, error) {
 		var i Info
+		if len(p) == 0 || p == "." {
+			return i, fmt.Errorf("missing package name")
+		}
 		b, err := run("go", "list", "-json", "-find", p)
 		if err != nil {
-			if !strings.Contains(err.Error(), "can't load package: package") {
 			return i, err
 		}
-
-			p, _ = path.Split(p)
-			return Package(p)
-		}
 		if err := json.Unmarshal(b, &i); err != nil {
 			return i, err
 		}

parser/include.go

@@ -0,0 +1,57 @@
+package parser
+
+import (
+	"encoding/json"
+	"fmt"
+	"go/token"
+	"os"
+)
+
+var _ Decl = IncludeDecl{}
+
+type IncludeDecl struct {
+	file  *File
+	pos   token.Position
+	value string
+}
+
+func (d IncludeDecl) String() string {
+	return fmt.Sprintf("pkger.Include(%q)", d.value)
+}
+
+func (d IncludeDecl) MarshalJSON() ([]byte, error) {
+	return json.Marshal(map[string]interface{}{
+		"type":  "pkger.Include",
+		"file":  d.file,
+		"pos":   d.pos,
+		"value": d.value,
+	})
+}
+
+func (d IncludeDecl) File() (*File, error) {
+	if d.file == nil {
+		return nil, os.ErrNotExist
+	}
+	return d.file, nil
+}
+
+func (d IncludeDecl) Position() (token.Position, error) {
+	return d.pos, nil
+}
+
+func (d IncludeDecl) Value() (string, error) {
+	if d.value == "" {
+		return "", os.ErrNotExist
+	}
+	return d.value, nil
+}
+
+func (d IncludeDecl) Files(virtual map[string]string) ([]*File, error) {
+	od := OpenDecl{
+		file:  d.file,
+		pos:   d.pos,
+		value: d.value,
+	}
+
+	return od.Files(virtual)
+}

parser/open.go

@@ -70,6 +70,7 @@ func (d OpenDecl) Files(virtual map[string]string) ([]*File, error) {
 		}
 		return wd.Files(virtual)
 	}
+
 	var files []*File
 	files = append(files, d.file)
 

parser/parser.go

@@ -16,15 +16,32 @@ import (
 
 var defaultIgnoredFolders = []string{".", "_", "vendor", "node_modules", "testdata"}
 
-func Parse(her here.Info) (Decls, error) {
+func New(her here.Info) (*Parser, error) {
+	return &Parser{
+		Info:  her,
+		decls: map[string]Decls{},
+	}, nil
+}
+
+type Parser struct {
+	here.Info
+	decls    map[string]Decls
+	once     sync.Once
+	includes []string
+	err      error
+}
+
+func Parse(her here.Info, includes ...string) (Decls, error) {
 	p, err := New(her)
 	if err != nil {
 		return nil, err
 	}
+	p.includes = includes
+
 	return p.Decls()
 }
 
-func ParseSource(source Source, mode parser.Mode) (*ParsedSource, error) {
+func (p *Parser) ParseSource(source Source, mode parser.Mode) (*ParsedSource, error) {
 	pf := &ParsedSource{
 		Source:  source,
 		FileSet: token.NewFileSet(),
@@ -45,7 +62,7 @@ func ParseSource(source Source, mode parser.Mode) (*ParsedSource, error) {
 	return pf, nil
 }
 
-func ParseFile(abs string, mode parser.Mode) (*ParsedSource, error) {
+func (p *Parser) ParseFile(abs string, mode parser.Mode) (*ParsedSource, error) {
 	s := Source{
 		Abs: abs,
 	}
@@ -68,10 +85,10 @@ func ParseFile(abs string, mode parser.Mode) (*ParsedSource, error) {
 
 	s.Path, err = s.Here.Parse(strings.TrimPrefix(abs, dir))
 
-	return ParseSource(s, 0)
+	return p.ParseSource(s, 0)
 }
 
-func ParseDir(abs string, mode parser.Mode) ([]*ParsedSource, error) {
+func (p *Parser) ParseDir(abs string, mode parser.Mode) ([]*ParsedSource, error) {
 	info, err := os.Stat(abs)
 	if err != nil {
 		return nil, err
@@ -116,19 +133,6 @@ func ParseDir(abs string, mode parser.Mode) ([]*ParsedSource, error) {
 
 	return srcs, nil
 }
-func New(her here.Info) (*Parser, error) {
-	return &Parser{
-		Info:  her,
-		decls: map[string]Decls{},
-	}, nil
-}
-
-type Parser struct {
-	here.Info
-	decls map[string]Decls
-	once  sync.Once
-	err   error
-}
 
 func (p *Parser) Decls() (Decls, error) {
 	if err := p.parse(); err != nil {
@@ -139,6 +143,7 @@ func (p *Parser) Decls() (Decls, error) {
 	orderedNames := []string{
 		"MkdirAll",
 		"Create",
+		"Include",
 		"Stat",
 		"Open",
 		"Dir",
@@ -166,12 +171,18 @@ func (p *Parser) Parse() error {
 
 func (p *Parser) parse() error {
 	p.decls = map[string]Decls{}
+
 	root := p.Dir
 
+	if err := p.parseIncludes(); err != nil {
+		return err
+	}
+
 	fi, err := os.Stat(root)
 	if err != nil {
 		return err
 	}
+
 	if !fi.IsDir() {
 		return fmt.Errorf("%q is not a directory", root)
 	}
@@ -192,7 +203,7 @@ func (p *Parser) parse() error {
 			}
 		}
 
-		srcs, err := ParseDir(path, 0)
+		srcs, err := p.ParseDir(path, 0)
 		if err != nil {
 			return fmt.Errorf("%w: %s", err, path)
 		}
@@ -212,3 +223,33 @@ func (p *Parser) parse() error {
 
 	return err
 }
+
+func (p *Parser) parseIncludes() error {
+	for _, i := range p.includes {
+		pt, err := p.Info.Parse(i)
+		if err != nil {
+			return err
+		}
+
+		her := p.Info
+		if pt.Pkg != her.ImportPath {
+			her, err = here.Package(pt.Pkg)
+			if err != nil {
+				return err
+			}
+		}
+
+		abs := filepath.Join(her.Module.Dir, pt.Name)
+
+		f := &File{
+			Abs:  abs,
+			Path: pt,
+			Here: her,
+		}
+		p.decls["Include"] = append(p.decls["Include"], IncludeDecl{
+			value: i,
+			file:  f,
+		})
+	}
+	return nil
+}

parser/parser_test.go

@@ -36,7 +36,46 @@ func Test_Parser_Ref(t *testing.T) {
 
 	files, err := res.Files()
 	r.NoError(err)
-	r.Len(files, 23)
+	r.Len(files, 25)
+
+	for _, f := range files {
+		if f.Path.Pkg == ref.Module.Path {
+			r.True(strings.HasPrefix(f.Abs, ref.Dir), "%q %q", f.Abs, ref.Dir)
+		} else {
+			r.False(strings.HasPrefix(f.Abs, ref.Dir), "%q %q", f.Abs, ref.Dir)
+		}
+	}
+}
+
+func Test_Parser_Ref_Include(t *testing.T) {
+	defer func() {
+		c := exec.Command("go", "mod", "tidy", "-v")
+		c.Run()
+	}()
+	r := require.New(t)
+
+	here.ClearCache()
+	ref, err := pkgtest.NewRef()
+	r.NoError(err)
+	defer os.RemoveAll(ref.Dir)
+
+	disk, err := stdos.New(ref.Info)
+	r.NoError(err)
+
+	_, err = pkgtest.LoadFiles("/", ref, disk)
+	r.NoError(err)
+
+	res, err := Parse(ref.Info, "github.com/gobuffalo/buffalo:/app.go")
+
+	r.NoError(err)
+
+	files, err := res.Files()
+	r.NoError(err)
+	// t.FailNow()
+
+	l := len(files)
+	r.Equal(26, l)
+	// r.Len(files, 27)
 
 	for _, f := range files {
 		if f.Path.Pkg == ref.Module.Path {

parser/source.go

@@ -88,6 +88,14 @@ func (p *ParsedSource) parse() error {
 					value: value,
 				}
 			}
+		case "Include":
+			fn = func(f File, pos token.Position, value string) Decl {
+				return IncludeDecl{
+					file:  &f,
+					pos:   pos,
+					value: value,
+				}
+			}
 		case "Stat":
 			fn = func(f File, pos token.Position, value string) Decl {
 				return StatDecl{

pkger.go

@@ -95,3 +95,6 @@ func Remove(name string) error {
 func RemoveAll(name string) error {
 	return impl().RemoveAll(name)
 }
+
+// Include is a no-op that directs the pkger tool to include the desired file or folder.
+func Include(name string) {}

pkging/pkger.go

@@ -15,26 +15,38 @@ type Pkger interface {
 	Current() (here.Info, error)
 
 	// Info returns the here.Info of the here.Path
+	// @Parser Directive
 	Info(p string) (here.Info, error)
 
 	// Create creates the named file with mode 0666 (before umask) - It's actually 0644, truncating it if it already exists. If successful, methods on the returned File can be used for I/O; the associated file descriptor has mode O_RDWR.
+	// @Parser Directive
 	Create(name string) (File, error)
 
 	// MkdirAll creates a directory named path, along with any necessary parents, and returns nil, or else returns an error. The permission bits perm (before umask) are used for all directories that MkdirAll creates. If path is already a directory, MkdirAll does nothing and returns nil.
+	// @Parser Directive
 	MkdirAll(p string, perm os.FileMode) error
 
 	// Open opens the named file for reading. If successful, methods on the returned file can be used for reading; the associated file descriptor has mode O_RDONLY.
+	// @Parser Directive
 	Open(name string) (File, error)
 
 	// Stat returns a FileInfo describing the named file.
+	// @Parser Directive
 	Stat(name string) (os.FileInfo, error)
 
 	// Walk walks the file tree rooted at root, calling walkFn for each file or directory in the tree, including root. All errors that arise visiting files and directories are filtered by walkFn. The files are walked in lexical order, which makes the output deterministic but means that for very large directories Walk can be inefficient. Walk does not follow symbolic links. - That is from the standard library. I know. Their grammar teachers can not be happy with them right now.
+	// @Parser Directive
 	Walk(p string, wf filepath.WalkFunc) error
 
 	// Remove removes the named file or (empty) directory.
+	// @Parser Directive
 	Remove(name string) error
 
 	// RemoveAll removes path and any children it contains. It removes everything it can but returns the first error it encounters. If the path does not exist, RemoveAll returns nil (no error).
+	// @Parser Directive
 	RemoveAll(path string) error
+
+	// Include is a no-op that directs the pkger tool to include the desired file or folder.
+	// @Parser Directive
+	// 	pkger.Include
 }

pkging/pkgtest/testdata/ref/go.mod

@@ -2,9 +2,6 @@ module app
 
 go 1.13
 
-require (
+require github.com/markbates/pkger v0.0.0
-	github.com/gobuffalo/buffalo v0.15.0 // indirect
-	github.com/markbates/pkger v0.0.0
-)
 
 replace github.com/markbates/pkger => ../../../../

pkging/pkgtest/testdata/ref/main.go

@@ -12,6 +12,7 @@ import (
 )
 
 func main() {
+	pkger.Include("/web")
 	if err := run(); err != nil {
 		log.Fatal(err)
 	}

pkging/pkgutil/stuff_test.go

@@ -29,7 +29,7 @@ func Test_Stuff(t *testing.T) {
 	decls, err := parser.Parse(ref.Info)
 	r.NoError(err)
 
-	r.Len(decls, 10)
+	r.Len(decls, 11)
 
 	files, err := decls.Files()
 	r.NoError(err)
@@ -42,7 +42,7 @@ func Test_Stuff(t *testing.T) {
 		}
 	}
 
-	r.Len(files, 23)
+	r.Len(files, 25)
 
 	bb := &bytes.Buffer{}