re
/
afero
forked from mirror/afero
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix Chown() pull request errors

This commit is contained in:
Mike Futerko 2020-12-07 15:19:45 +02:00
parent 30c3c32863 736b98e92c
commit cb1d580bf4
41 changed files with 2112 additions and 260 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored Normal file
View File

@ -0,0 +1,2 @@
sftpfs/file1
sftpfs/test/

14
.travis.yml
View File

@ -1,9 +1,12 @@
sudo: false
language: go
arch:
- amd64
- ppc64e
go:
- 1.9
- "1.10"
- "1.13"
- "1.14"
- tip
os:
@ -16,6 +19,7 @@ matrix:
fast_finish: true
script:
- go build
- go test -race -v ./...
- go build -v ./...
- go test -count=1 -cover -race -v ./...
- go vet ./...
- FILES=$(gofmt -s -l . zipfs sftpfs mem tarfs); if [[ -n "${FILES}" ]]; then echo "You have go format errors; gofmt your changes"; exit 1; fi

39
README.md
View File

@ -6,7 +6,7 @@ A FileSystem Abstraction System for Go
# Overview
Afero is an filesystem framework providing a simple, uniform and universal API
Afero is a filesystem framework providing a simple, uniform and universal API
interacting with any filesystem, as an abstraction layer providing interfaces,
types and methods. Afero has an exceptionally clean interface and simple design
without needless constructors or initialization methods.
@ -18,7 +18,7 @@ and benefit of the os and ioutil packages.
Afero provides significant improvements over using the os package alone, most
notably the ability to create mock and testing filesystems without relying on the disk.
It is suitable for use in a any situation where you would consider using the OS
It is suitable for use in any situation where you would consider using the OS
package as it provides an additional abstraction that makes it easy to use a
memory backed file system during testing. It also adds support for the http
filesystem for full interoperability.
@ -41,8 +41,8 @@ Afero is easy to use and easier to adopt.
A few different ways you could use Afero:
* Use the interfaces alone to define you own file system.
* Wrap for the OS packages.
* Use the interfaces alone to define your own file system.
* Wrapper for the OS packages.
* Define different filesystems for different parts of your application.
* Use Afero for mock filesystems while testing
@ -227,7 +227,7 @@ operation and a mock filesystem during testing or as needed.
```go
appfs := afero.NewOsFs()
appfs.MkdirAll("src/a", 0755))
appfs.MkdirAll("src/a", 0755)
```
## Memory Backed Storage
@ -241,7 +241,7 @@ safely.
```go
mm := afero.NewMemMapFs()
mm.MkdirAll("src/a", 0755))
mm.MkdirAll("src/a", 0755)
```
#### InMemoryFile
@ -306,7 +306,7 @@ Any Afero FileSystem can be used as an httpFs.
```go
httpFs := afero.NewHttpFs(<ExistingFS>)
fileserver := http.FileServer(httpFs.Dir(<PATH>)))
fileserver := http.FileServer(httpFs.Dir(<PATH>))
http.Handle("/", fileserver)
```
@ -380,8 +380,6 @@ The following is a short list of possible backends we hope someone will
implement:
* SSH
* ZIP
* TAR
* S3
# About the project
@ -406,28 +404,7 @@ Googles very well.
## Release Notes
* **0.10.0** 2015.12.10
* Full compatibility with Windows
* Introduction of afero utilities
* Test suite rewritten to work cross platform
* Normalize paths for MemMapFs
* Adding Sync to the file interface
* **Breaking Change** Walk and ReadDir have changed parameter order
* Moving types used by MemMapFs to a subpackage
* General bugfixes and improvements
* **0.9.0** 2015.11.05
* New Walk function similar to filepath.Walk
* MemMapFs.OpenFile handles O_CREATE, O_APPEND, O_TRUNC
* MemMapFs.Remove now really deletes the file
* InMemoryFile.Readdir and Readdirnames work correctly
* InMemoryFile functions lock it for concurrent access
* Test suite improvements
* **0.8.0** 2014.10.28
* First public version
* Interfaces feel ready for people to build using
* Interfaces satisfy all known uses
* MemMapFs passes the majority of the OS test suite
* OsFs passes the majority of the OS test suite
See the [Releases Page](https://github.com/spf13/afero/releases).
## Contributing

8
afero.go
View File

@ -91,14 +91,14 @@ type Fs interface {
// The name of this FileSystem
Name() string
//Chmod changes the mode of the named file to mode.
// Chmod changes the mode of the named file to mode.
Chmod(name string, mode os.FileMode) error
// Chown changes the uid and gid of the named file.
Chown(name string, uid, gid int) error
//Chtimes changes the access and modification times of the named file
Chtimes(name string, atime time.Time, mtime time.Time) error
// Chown changes uid and gid of the named file.
Chown(name string, uid, gid int) error
}
var (

4
appveyor.yml
View File

@ -10,6 +10,6 @@ build_script:
go get -v github.com/spf13/afero/...
go build github.com/spf13/afero
go build -v github.com/spf13/afero/...
test_script:
- cmd: go test -race -v github.com/spf13/afero/...
- cmd: go test -count=1 -cover -race -v github.com/spf13/afero/...

36
basepath.go
View File

@ -83,6 +83,13 @@ func (b *BasePathFs) Chmod(name string, mode os.FileMode) (err error) {
return b.source.Chmod(name, mode)
}
func (b *BasePathFs) Chown(name string, uid, gid int) (err error) {
if name, err = b.RealPath(name); err != nil {
return &os.PathError{Op: "chown", Path: name, Err: err}
}
return b.source.Chown(name, uid, gid)
}
func (b *BasePathFs) Name() string {
return "BasePathFs"
}
@ -177,11 +184,28 @@ func (b *BasePathFs) LstatIfPossible(name string) (os.FileInfo, bool, error) {
return fi, false, err
}
func (b *BasePathFs) Chown(name string, uid, gid int) (err error) {
if name, err = b.RealPath(name); err != nil {
return &os.PathError{Op: "chown", Path: name, Err: err}
func (b *BasePathFs) SymlinkIfPossible(oldname, newname string) error {
oldname, err := b.RealPath(oldname)
if err != nil {
return &os.LinkError{Op: "symlink", Old: oldname, New: newname, Err: err}
}
return b.source.Chown(name, uid, gid)
newname, err = b.RealPath(newname)
if err != nil {
return &os.LinkError{Op: "symlink", Old: oldname, New: newname, Err: err}
}
if linker, ok := b.source.(Linker); ok {
return linker.SymlinkIfPossible(oldname, newname)
}
return &os.LinkError{Op: "symlink", Old: oldname, New: newname, Err: ErrNoSymlink}
}
func (b *BasePathFs) ReadlinkIfPossible(name string) (string, error) {
name, err := b.RealPath(name)
if err != nil {
return "", &os.PathError{Op: "readlink", Path: name, Err: err}
}
if reader, ok := b.source.(LinkReader); ok {
return reader.ReadlinkIfPossible(name)
}
return "", &os.PathError{Op: "readlink", Path: name, Err: ErrNoReadlink}
}
// vim: ts=4 sw=4 noexpandtab nolist syn=go

14
basepath_test.go
View File

@ -96,10 +96,10 @@ func TestNestedBasePaths(t *testing.T) {
Dir1, Dir2, Dir3 string
}
dirSpecs := []dirSpec{
dirSpec{Dir1: "/", Dir2: "/", Dir3: "/"},
dirSpec{Dir1: "/", Dir2: "/path2", Dir3: "/"},
dirSpec{Dir1: "/path1/dir", Dir2: "/path2/dir/", Dir3: "/path3/dir"},
dirSpec{Dir1: "C:/path1", Dir2: "path2/dir", Dir3: "/path3/dir/"},
{Dir1: "/", Dir2: "/", Dir3: "/"},
{Dir1: "/", Dir2: "/path2", Dir3: "/"},
{Dir1: "/path1/dir", Dir2: "/path2/dir/", Dir3: "/path3/dir"},
{Dir1: "C:/path1", Dir2: "path2/dir", Dir3: "/path3/dir/"},
}
for _, ds := range dirSpecs {
@ -113,9 +113,9 @@ func TestNestedBasePaths(t *testing.T) {
FileName string
}
specs := []spec{
spec{BaseFs: level3Fs, FileName: "f.txt"},
spec{BaseFs: level2Fs, FileName: "f.txt"},
spec{BaseFs: level1Fs, FileName: "f.txt"},
{BaseFs: level3Fs, FileName: "f.txt"},
{BaseFs: level2Fs, FileName: "f.txt"},
{BaseFs: level1Fs, FileName: "f.txt"},
}
for _, s := range specs {

42
cacheOnReadFs.go
View File

@ -117,6 +117,27 @@ func (u *CacheOnReadFs) Chmod(name string, mode os.FileMode) error {
return u.layer.Chmod(name, mode)
}
func (u *CacheOnReadFs) Chown(name string, uid, gid int) error {
st, _, err := u.cacheStatus(name)
if err != nil {
return err
}
switch st {
case cacheLocal:
case cacheHit:
err = u.base.Chown(name, uid, gid)
case cacheStale, cacheMiss:
if err := u.copyToLayer(name); err != nil {
return err
}
err = u.base.Chown(name, uid, gid)
}
if err != nil {
return err
}
return u.layer.Chown(name, uid, gid)
}
func (u *CacheOnReadFs) Stat(name string) (os.FileInfo, error) {
st, fi, err := u.cacheStatus(name)
if err != nil {
@ -288,24 +309,3 @@ func (u *CacheOnReadFs) Create(name string) (File, error) {
}
return &UnionFile{Base: bfh, Layer: lfh}, nil
}
func (u *CacheOnReadFs) Chown(name string, uid, gid int) error {
st, _, err := u.cacheStatus(name)
if err != nil {
return err
}
switch st {
case cacheLocal:
case cacheHit:
err = u.base.Chown(name, uid, gid)
case cacheStale, cacheMiss:
if err := u.copyToLayer(name); err != nil {
return err
}
err = u.base.Chown(name, uid, gid)
}
if err != nil {
return err
}
return u.layer.Chown(name, uid, gid)
}

20
composite_test.go
View File

@ -477,7 +477,8 @@ func TestUnionFileReaddirAskForTooMany(t *testing.T) {
base := &MemMapFs{}
overlay := &MemMapFs{}
for i := 0; i < 5; i++ {
const testFiles = 5
for i := 0; i < testFiles; i++ {
WriteFile(base, fmt.Sprintf("file%d.txt", i), []byte("afero"), 0777)
}
@ -490,13 +491,24 @@ func TestUnionFileReaddirAskForTooMany(t *testing.T) {
defer f.Close()
names, err := f.Readdirnames(6)
// Read part of all files
wantNames := 3
names, err := f.Readdirnames(wantNames)
if err != nil {
t.Fatal(err)
}
if len(names) != wantNames {
t.Fatalf("got %d names %v, want %d", len(names), names, wantNames)
}
if len(names) != 5 {
t.Fatal(names)
// Try to read more files than remaining
wantNames = testFiles - len(names)
names, err = f.Readdirnames(wantNames + 1)
if err != nil {
t.Fatal(err)
}
if len(names) != wantNames {
t.Fatalf("got %d names %v, want %d", len(names), names, wantNames)
}
// End of directory

2
const_bsds.go
View File

@ -11,7 +11,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
// +build darwin openbsd freebsd netbsd dragonfly
// +build aix darwin openbsd freebsd netbsd dragonfly
package afero

1
const_win_unix.go
View File

@ -15,6 +15,7 @@
// +build !freebsd
// +build !dragonfly
// +build !netbsd
// +build !aix
package afero

48
copyOnWriteFs.go
View File

@ -14,7 +14,7 @@ var _ Lstater = (*CopyOnWriteFs)(nil)
// a possibly writeable layer on top. Changes to the file system will only
// be made in the overlay: Changing an existing file in the base layer which
// is not present in the overlay will copy the file to the overlay ("changing"
// includes also calls to e.g. Chtimes() and Chmod()).
// includes also calls to e.g. Chtimes(), Chmod() and Chown()).
//
// Reading directories is currently only supported via Open(), not OpenFile().
type CopyOnWriteFs struct {
@ -75,6 +75,19 @@ func (u *CopyOnWriteFs) Chmod(name string, mode os.FileMode) error {
return u.layer.Chmod(name, mode)
}
func (u *CopyOnWriteFs) Chown(name string, uid, gid int) error {
b, err := u.isBaseFile(name)
if err != nil {
return err
}
if b {
if err := u.copyToLayer(name); err != nil {
return err
}
}
return u.layer.Chown(name, uid, gid)
}
func (u *CopyOnWriteFs) Stat(name string) (os.FileInfo, error) {
fi, err := u.layer.Stat(name)
if err != nil {
@ -117,6 +130,26 @@ func (u *CopyOnWriteFs) LstatIfPossible(name string) (os.FileInfo, bool, error)
return fi, false, err
}
func (u *CopyOnWriteFs) SymlinkIfPossible(oldname, newname string) error {
if slayer, ok := u.layer.(Linker); ok {
return slayer.SymlinkIfPossible(oldname, newname)
}
return &os.LinkError{Op: "symlink", Old: oldname, New: newname, Err: ErrNoSymlink}
}
func (u *CopyOnWriteFs) ReadlinkIfPossible(name string) (string, error) {
if rlayer, ok := u.layer.(LinkReader); ok {
return rlayer.ReadlinkIfPossible(name)
}
if rbase, ok := u.base.(LinkReader); ok {
return rbase.ReadlinkIfPossible(name)
}
return "", &os.PathError{Op: "readlink", Path: name, Err: ErrNoReadlink}
}
func (u *CopyOnWriteFs) isNotExist(err error) bool {
if e, ok := err.(*os.PathError); ok {
err = e.Err
@ -291,16 +324,3 @@ func (u *CopyOnWriteFs) MkdirAll(name string, perm os.FileMode) error {
func (u *CopyOnWriteFs) Create(name string) (File, error) {
return u.OpenFile(name, os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0666)
}
func (u *CopyOnWriteFs) Chown(name string, uid, gid int) error {
b, err := u.isBaseFile(name)
if err != nil {
return err
}
if b {
if err := u.copyToLayer(name); err != nil {
return err
}
}
return u.layer.Chown(name, uid, gid)
}

8
go.mod
View File

@ -1,3 +1,9 @@
module github.com/spf13/afero
require golang.org/x/text v0.3.0
require (
github.com/pkg/sftp v1.10.1
golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586
golang.org/x/text v0.3.3
)
go 1.13

27
go.sum
View File

@ -1,2 +1,29 @@
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8=
github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/sftp v1.10.1 h1:VasscCm72135zRysgrJDKsntdmPN+OuU3+nnHYA9wyc=
github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586 h1:7KByu05hhLed2MO29w7p1XfZvZ13m8mub3shuVftRs0=
golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

32
ioutil.go
View File

@ -22,6 +22,7 @@ import (
"path/filepath"
"sort"
"strconv"
"strings"
"sync"
"time"
)
@ -147,7 +148,7 @@ func reseed() uint32 {
return uint32(time.Now().UnixNano() + int64(os.Getpid()))
}
func nextSuffix() string {
func nextRandom() string {
randmu.Lock()
r := rand
if r == 0 {
@ -159,27 +160,36 @@ func nextSuffix() string {
return strconv.Itoa(int(1e9 + r%1e9))[1:]
}
// TempFile creates a new temporary file in the directory dir
// with a name beginning with prefix, opens the file for reading
// and writing, and returns the resulting *File.
// TempFile creates a new temporary file in the directory dir,
// opens the file for reading and writing, and returns the resulting *os.File.
// The filename is generated by taking pattern and adding a random
// string to the end. If pattern includes a "*", the random string
// replaces the last "*".
// If dir is the empty string, TempFile uses the default directory
// for temporary files (see os.TempDir).
// Multiple programs calling TempFile simultaneously
// will not choose the same file. The caller can use f.Name()
// to find the pathname of the file. It is the caller's responsibility
// will not choose the same file. The caller can use f.Name()
// to find the pathname of the file. It is the caller's responsibility
// to remove the file when no longer needed.
func (a Afero) TempFile(dir, prefix string) (f File, err error) {
return TempFile(a.Fs, dir, prefix)
func (a Afero) TempFile(dir, pattern string) (f File, err error) {
return TempFile(a.Fs, dir, pattern)
}
func TempFile(fs Fs, dir, prefix string) (f File, err error) {
func TempFile(fs Fs, dir, pattern string) (f File, err error) {
if dir == "" {
dir = os.TempDir()
}
var prefix, suffix string
if pos := strings.LastIndex(pattern, "*"); pos != -1 {
prefix, suffix = pattern[:pos], pattern[pos+1:]
} else {
prefix = pattern
}
nconflict := 0
for i := 0; i < 10000; i++ {
name := filepath.Join(dir, prefix+nextSuffix())
name := filepath.Join(dir, prefix+nextRandom()+suffix)
f, err = fs.OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0600)
if os.IsExist(err) {
if nconflict++; nconflict > 10 {
@ -211,7 +221,7 @@ func TempDir(fs Fs, dir, prefix string) (name string, err error) {
nconflict := 0
for i := 0; i < 10000; i++ {
try := filepath.Join(dir, prefix+nextSuffix())
try := filepath.Join(dir, prefix+nextRandom())
err = fs.Mkdir(try, 0700)
if os.IsExist(err) {
if nconflict++; nconflict > 10 {

66
ioutil_test.go
View File

@ -15,7 +15,11 @@
package afero
import "testing"
import (
"path/filepath"
"strings"
"testing"
)
func checkSizePath(t *testing.T, path string, size int64) {
dir, err := testFS.Stat(path)
@ -110,3 +114,63 @@ func TestReadDir(t *testing.T) {
t.Fatalf("ReadDir %s: i-am-a-dir directory not found", dirname)
}
}
func TestTempFile(t *testing.T) {
type args struct {
dir string
pattern string
}
tests := map[string]struct {
args args
want func(*testing.T, string)
}{
"foo": { // simple file name
args: args{
dir: "",
pattern: "foo",
},
want: func(t *testing.T, base string) {
if !strings.HasPrefix(base, "foo") || len(base) <= len("foo") {
t.Errorf("TempFile() file = %s, invalid file name", base)
}
},
},
"foo.bar": { // file name w/ ext
args: args{
dir: "",
pattern: "foo.bar",
},
want: func(t *testing.T, base string) {
if !strings.HasPrefix(base, "foo.bar") || len(base) <= len("foo.bar") {
t.Errorf("TempFile() file = %v, invalid file name", base)
}
},
},
"foo-*.bar": { // file name with wild card
args: args{
dir: "",
pattern: "foo-*.bar",
},
want: func(t *testing.T, base string) {
if !(strings.HasPrefix(base, "foo-") || strings.HasPrefix(base, "bar")) ||
len(base) <= len("foo-*.bar") {
t.Errorf("TempFile() file = %v, invalid file name", base)
}
},
},
}
for name, tt := range tests {
t.Run(name, func(t *testing.T) {
file, err := TempFile(NewMemMapFs(), tt.args.dir, tt.args.pattern)
if err != nil {
t.Errorf("TempFile() error = %v, none expected", err)
return
}
if file == nil {
t.Errorf("TempFile() file = %v, should not be nil", file)
return
}
tt.want(t, filepath.Base(file.Name()))
})
}
}

2
match.go
View File

@ -106,5 +106,5 @@ func glob(fs Fs, dir, pattern string, matches []string) (m []string, e error) {
// recognized by Match.
func hasMeta(path string) bool {
// TODO(niemeyer): Should other magic characters be added here?
return strings.IndexAny(path, "*?[") >= 0
return strings.ContainsAny(path, "*?[")
}

1
match_test.go
View File

@ -172,7 +172,6 @@ func TestGlobSymlink(t *testing.T) {
}
}
func TestGlobError(t *testing.T) {
for _, fs := range Fss {
_, err := Glob(fs, "[7]")

20
mem/file.go
View File

@ -207,8 +207,11 @@ func (f *File) Read(b []byte) (n int, err error) {
}
func (f *File) ReadAt(b []byte, off int64) (n int, err error) {
prev := atomic.LoadInt64(&f.at)
atomic.StoreInt64(&f.at, off)
return f.Read(b)
n, err = f.Read(b)
atomic.StoreInt64(&f.at, prev)
return
}
func (f *File) Truncate(size int64) error {
@ -221,6 +224,8 @@ func (f *File) Truncate(size int64) error {
if size < 0 {
return ErrOutOfRange
}
f.fileData.Lock()
defer f.fileData.Unlock()
if size > int64(len(f.fileData.data)) {
diff := size - int64(len(f.fileData.data))
f.fileData.data = append(f.fileData.data, bytes.Repeat([]byte{00}, int(diff))...)
@ -236,17 +241,20 @@ func (f *File) Seek(offset int64, whence int) (int64, error) {
return 0, ErrFileClosed
}
switch whence {
case 0:
case io.SeekStart:
atomic.StoreInt64(&f.at, offset)
case 1:
atomic.AddInt64(&f.at, int64(offset))
case 2:
case io.SeekCurrent:
atomic.AddInt64(&f.at, offset)
case io.SeekEnd:
atomic.StoreInt64(&f.at, int64(len(f.fileData.data))+offset)
}
return f.at, nil
}
func (f *File) Write(b []byte) (n int, err error) {
if f.closed == true {
return 0, ErrFileClosed
}
if f.readOnly {
return 0, &os.PathError{Op: "write", Path: f.fileData.name, Err: errors.New("file handle is read only")}
}
@ -268,7 +276,7 @@ func (f *File) Write(b []byte) (n int, err error) {
}
setModTime(f.fileData, time.Now())
atomic.StoreInt64(&f.at, int64(len(f.fileData.data)))
atomic.AddInt64(&f.at, int64(n))
return
}

93
mem/file_test.go
View File

@ -1,6 +1,8 @@
package mem
import (
"bytes"
"io"
"testing"
"time"
)
@ -152,3 +154,94 @@ func TestFileDataSizeRace(t *testing.T) {
t.Errorf("Failed to read correct value for dir, was %v", s.Size())
}
}
func TestFileReadAtSeekOffset(t *testing.T) {
t.Parallel()
fd := CreateFile("foo")
f := NewFileHandle(fd)
_, err := f.WriteString("TEST")
if err != nil {
t.Fatal(err)
}
offset, err := f.Seek(0, io.SeekStart)
if err != nil {
t.Fatal(err)
}
if offset != 0 {
t.Fail()
}
offsetBeforeReadAt, err := f.Seek(0, io.SeekCurrent)
if err != nil {
t.Fatal(err)
}
if offsetBeforeReadAt != 0 {
t.Fatal("expected 0")
}
b := make([]byte, 4)
n, err := f.ReadAt(b, 0)
if err != nil {
t.Fatal(err)
}
if n != 4 {
t.Fail()
}
if string(b) != "TEST" {
t.Fail()
}
offsetAfterReadAt, err := f.Seek(0, io.SeekCurrent)
if err != nil {
t.Fatal(err)
}
if offsetAfterReadAt != offsetBeforeReadAt {
t.Fatal("ReadAt should not affect offset")
}
err = f.Close()
if err != nil {
t.Fatal(err)
}
}
func TestFileWriteAndSeek(t *testing.T) {
fd := CreateFile("foo")
f := NewFileHandle(fd)
assert := func(expected bool, v ...interface{}) {
if !expected {
t.Helper()
t.Fatal(v...)
}
}
data4 := []byte{0, 1, 2, 3}
data20 := bytes.Repeat(data4, 5)
var off int64
for i := 0; i < 100; i++ {
// write 20 bytes
n, err := f.Write(data20)
assert(err == nil, err)
off += int64(n)
assert(n == len(data20), n)
assert(off == int64((i+1)*len(data20)), off)
// rewind to start and write 4 bytes there
cur, err := f.Seek(-off, io.SeekCurrent)
assert(err == nil, err)
assert(cur == 0, cur)
n, err = f.Write(data4)
assert(err == nil, err)
assert(n == len(data4), n)
// back at the end
cur, err = f.Seek(off-int64(n), io.SeekCurrent)
assert(err == nil, err)
assert(cur == off, cur, off)
}
}

90
memmap.go
View File

@ -25,6 +25,8 @@ import (
"github.com/spf13/afero/mem"
)
const chmodBits = os.ModePerm | os.ModeSetuid | os.ModeSetgid | os.ModeSticky // Only a subset of bits are allowed to be changed. Documented under os.Chmod()
type MemMapFs struct {
mu sync.RWMutex
data map[string]*mem.FileData
@ -40,7 +42,9 @@ func (m *MemMapFs) getData() map[string]*mem.FileData {
m.data = make(map[string]*mem.FileData)
// Root should always exist, right?
// TODO: what about windows?
m.data[FilePathSeparator] = mem.CreateDir(FilePathSeparator)
root := mem.CreateDir(FilePathSeparator)
mem.SetMode(root, os.ModeDir|0755)
m.data[FilePathSeparator] = root
})
return m.data
}
@ -52,7 +56,7 @@ func (m *MemMapFs) Create(name string) (File, error) {
m.mu.Lock()
file := mem.CreateFile(name)
m.getData()[name] = file
m.registerWithParent(file)
m.registerWithParent(file, 0)
m.mu.Unlock()
return mem.NewFileHandle(file), nil
}
@ -83,14 +87,14 @@ func (m *MemMapFs) findParent(f *mem.FileData) *mem.FileData {
return pfile
}
func (m *MemMapFs) registerWithParent(f *mem.FileData) {
func (m *MemMapFs) registerWithParent(f *mem.FileData, perm os.FileMode) {
if f == nil {
return
}
parent := m.findParent(f)
if parent == nil {
pdir := filepath.Dir(filepath.Clean(f.Name()))
err := m.lockfreeMkdir(pdir, 0777)
err := m.lockfreeMkdir(pdir, perm)
if err != nil {
//log.Println("Mkdir error:", err)
return
@ -119,13 +123,15 @@ func (m *MemMapFs) lockfreeMkdir(name string, perm os.FileMode) error {
}
} else {
item := mem.CreateDir(name)
mem.SetMode(item, os.ModeDir|perm)
m.getData()[name] = item
m.registerWithParent(item)
m.registerWithParent(item, perm)
}
return nil
}
func (m *MemMapFs) Mkdir(name string, perm os.FileMode) error {
perm &= chmodBits
name = normalizePath(name)
m.mu.RLock()
@ -137,13 +143,12 @@ func (m *MemMapFs) Mkdir(name string, perm os.FileMode) error {
m.mu.Lock()
item := mem.CreateDir(name)
mem.SetMode(item, os.ModeDir|perm)
m.getData()[name] = item
m.registerWithParent(item)
m.registerWithParent(item, perm)
m.mu.Unlock()
m.Chmod(name, perm|os.ModeDir)
return nil
return m.setFileMode(name, perm|os.ModeDir)
}
func (m *MemMapFs) MkdirAll(path string, perm os.FileMode) error {
@ -210,8 +215,12 @@ func (m *MemMapFs) lockfreeOpen(name string) (*mem.FileData, error) {
}
func (m *MemMapFs) OpenFile(name string, flag int, perm os.FileMode) (File, error) {
perm &= chmodBits
chmod := false
file, err := m.openWrite(name)
if err == nil && (flag&os.O_EXCL > 0) {
return nil, &os.PathError{Op: "open", Path: name, Err: ErrFileExists}
}
if os.IsNotExist(err) && (flag&os.O_CREATE > 0) {
file, err = m.Create(name)
chmod = true
@ -237,7 +246,7 @@ func (m *MemMapFs) OpenFile(name string, flag int, perm os.FileMode) (File, erro
}
}
if chmod {
m.Chmod(name, perm)
return file, m.setFileMode(name, perm)
}
return file, nil
}
@ -269,7 +278,7 @@ func (m *MemMapFs) RemoveAll(path string) error {
m.mu.RLock()
defer m.mu.RUnlock()
for p, _ := range m.getData() {
for p := range m.getData() {
if strings.HasPrefix(p, path) {
m.mu.RUnlock()
m.mu.Lock()
@ -299,7 +308,7 @@ func (m *MemMapFs) Rename(oldname, newname string) error {
delete(m.getData(), oldname)
mem.ChangeFileName(fileData, newname)
m.getData()[newname] = fileData
m.registerWithParent(fileData)
m.registerWithParent(fileData, 0)
m.mu.Unlock()
m.mu.RLock()
} else {
@ -308,6 +317,11 @@ func (m *MemMapFs) Rename(oldname, newname string) error {
return nil
}
func (m *MemMapFs) LstatIfPossible(name string) (os.FileInfo, bool, error) {
fileInfo, err := m.Stat(name)
return fileInfo, false, err
}
func (m *MemMapFs) Stat(name string) (os.FileInfo, error) {
f, err := m.Open(name)
if err != nil {
@ -318,6 +332,21 @@ func (m *MemMapFs) Stat(name string) (os.FileInfo, error) {
}
func (m *MemMapFs) Chmod(name string, mode os.FileMode) error {
mode &= chmodBits
m.mu.RLock()
f, ok := m.getData()[name]
m.mu.RUnlock()
if !ok {
return &os.PathError{Op: "chmod", Path: name, Err: ErrFileNotFound}
}
prevOtherBits := mem.GetFileInfo(f).Mode() & ^chmodBits
mode = prevOtherBits | mode
return m.setFileMode(name, mode)
}
func (m *MemMapFs) setFileMode(name string, mode os.FileMode) error {
name = normalizePath(name)
m.mu.RLock()
@ -334,6 +363,22 @@ func (m *MemMapFs) Chmod(name string, mode os.FileMode) error {
return nil
}
func (m *MemMapFs) Chown(name string, uid, gid int) error {
name = normalizePath(name)
m.mu.RLock()
f, ok := m.getData()[name]
m.mu.RUnlock()
if !ok {
return &os.PathError{Op: "chown", Path: name, Err: ErrFileNotFound}
}
mem.SetUID(f, uid)
mem.SetGID(f, gid)
return nil
}
func (m *MemMapFs) Chtimes(name string, atime time.Time, mtime time.Time) error {
name = normalizePath(name)
@ -357,24 +402,3 @@ func (m *MemMapFs) List() {
fmt.Println(x.Name(), y.Size())
}
}
func (m *MemMapFs) Chown(name string, uid, gid int) error {
name = normalizePath(name)
m.mu.RLock()
f, ok := m.getData()[name]
m.mu.RUnlock()
if !ok {
return &os.PathError{Op: "chown", Path: name, Err: ErrFileNotFound}
}
mem.SetUID(f, uid)
mem.SetGID(f, gid)
return nil
}
// func debugMemMapList(fs Fs) {
// if x, ok := fs.(*MemMapFs); ok {
// x.List()
// }
// }

234
memmap_test.go
View File

@ -38,6 +38,8 @@ func TestPathErrors(t *testing.T) {
path2 := filepath.Join(".", "different", "path")
fs := NewMemMapFs()
perm := os.FileMode(0755)
uid := 1000
gid := 1000
// relevant functions:
// func (m *MemMapFs) Chmod(name string, mode os.FileMode) error
@ -54,6 +56,9 @@ func TestPathErrors(t *testing.T) {
err := fs.Chmod(path, perm)
checkPathError(t, err, "Chmod")
err = fs.Chown(path, uid, gid)
checkPathError(t, err, "Chown")
err = fs.Chtimes(path, time.Now(), time.Now())
checkPathError(t, err, "Chtimes")
@ -104,6 +109,29 @@ func checkPathError(t *testing.T, err error, op string) {
}
}
// Ensure os.O_EXCL is correctly handled.
func TestOpenFileExcl(t *testing.T) {
const fileName = "/myFileTest"
const fileMode = os.FileMode(0765)
fs := NewMemMapFs()
// First creation should succeed.
f, err := fs.OpenFile(fileName, os.O_CREATE|os.O_EXCL, fileMode)
if err != nil {
t.Errorf("OpenFile Create Excl failed: %s", err)
return
}
f.Close()
// Second creation should fail.
_, err = fs.OpenFile(fileName, os.O_CREATE|os.O_EXCL, fileMode)
if err == nil {
t.Errorf("OpenFile Create Excl should have failed, but it didn't")
}
checkPathError(t, err, "Open")
}
// Ensure Permissions are set on OpenFile/Mkdir/MkdirAll
func TestPermSet(t *testing.T) {
const fileName = "/myFileTest"
@ -389,6 +417,116 @@ loop:
}
}
// root is a directory
func TestMemFsRootDirMode(t *testing.T) {
t.Parallel()
fs := NewMemMapFs()
info, err := fs.Stat("/")
if err != nil {
t.Fatal(err)
}
if !info.IsDir() {
t.Error("should be a directory")
}
if !info.Mode().IsDir() {
t.Errorf("FileMode is not directory, is %s", info.Mode().String())
}
}
// MkdirAll creates intermediate directories with correct mode
func TestMemFsMkdirAllMode(t *testing.T) {
t.Parallel()
fs := NewMemMapFs()
err := fs.MkdirAll("/a/b/c", 0755)
if err != nil {
t.Fatal(err)
}
info, err := fs.Stat("/a")
if err != nil {
t.Fatal(err)
}
if !info.Mode().IsDir() {
t.Error("/a: mode is not directory")
}
if info.Mode() != os.FileMode(os.ModeDir|0755) {
t.Errorf("/a: wrong permissions, expected drwxr-xr-x, got %s", info.Mode())
}
info, err = fs.Stat("/a/b")
if err != nil {
t.Fatal(err)
}
if !info.Mode().IsDir() {
t.Error("/a/b: mode is not directory")
}
if info.Mode() != os.FileMode(os.ModeDir|0755) {
t.Errorf("/a/b: wrong permissions, expected drwxr-xr-x, got %s", info.Mode())
}
info, err = fs.Stat("/a/b/c")
if err != nil {
t.Fatal(err)
}
if !info.Mode().IsDir() {
t.Error("/a/b/c: mode is not directory")
}
if info.Mode() != os.FileMode(os.ModeDir|0755) {
t.Errorf("/a/b/c: wrong permissions, expected drwxr-xr-x, got %s", info.Mode())
}
}
// MkdirAll does not change permissions of already-existing directories
func TestMemFsMkdirAllNoClobber(t *testing.T) {
t.Parallel()
fs := NewMemMapFs()
err := fs.MkdirAll("/a/b/c", 0755)
if err != nil {
t.Fatal(err)
}
info, err := fs.Stat("/a/b")
if err != nil {
t.Fatal(err)
}
if info.Mode() != os.FileMode(os.ModeDir|0755) {
t.Errorf("/a/b: wrong permissions, expected drwxr-xr-x, got %s", info.Mode())
}
err = fs.MkdirAll("/a/b/c/d/e/f", 0710)
// '/a/b' is unchanged
if err != nil {
t.Fatal(err)
}
info, err = fs.Stat("/a/b")
if err != nil {
t.Fatal(err)
}
if info.Mode() != os.FileMode(os.ModeDir|0755) {
t.Errorf("/a/b: wrong permissions, expected drwxr-xr-x, got %s", info.Mode())
}
// new directories created with proper permissions
info, err = fs.Stat("/a/b/c/d")
if err != nil {
t.Fatal(err)
}
if info.Mode() != os.FileMode(os.ModeDir|0710) {
t.Errorf("/a/b/c/d: wrong permissions, expected drwx--x---, got %s", info.Mode())
}
info, err = fs.Stat("/a/b/c/d/e")
if err != nil {
t.Fatal(err)
}
if info.Mode() != os.FileMode(os.ModeDir|0710) {
t.Errorf("/a/b/c/d/e: wrong permissions, expected drwx--x---, got %s", info.Mode())
}
info, err = fs.Stat("/a/b/c/d/e/f")
if err != nil {
t.Fatal(err)
}
if info.Mode() != os.FileMode(os.ModeDir|0710) {
t.Errorf("/a/b/c/d/e/f: wrong permissions, expected drwx--x---, got %s", info.Mode())
}
}
func TestMemFsDirMode(t *testing.T) {
fs := NewMemMapFs()
err := fs.Mkdir("/testDir1", 0644)
@ -449,3 +587,99 @@ func TestMemFsUnexpectedEOF(t *testing.T) {
t.Fatal("Expected ErrUnexpectedEOF")
}
}
func TestMemFsChmod(t *testing.T) {
t.Parallel()
fs := NewMemMapFs()
const file = "hello"
if err := fs.Mkdir(file, 0700); err != nil {
t.Fatal(err)
}
info, err := fs.Stat(file)
if err != nil {
t.Fatal(err)
}
if info.Mode().String() != "drwx------" {
t.Fatal("mkdir failed to create a directory: mode =", info.Mode())
}
err = fs.Chmod(file, 0)
if err != nil {
t.Error("Failed to run chmod:", err)
}
info, err = fs.Stat(file)
if err != nil {
t.Fatal(err)
}
if info.Mode().String() != "d---------" {
t.Error("chmod should not change file type. New mode =", info.Mode())
}
}
// can't use Mkdir to get around which permissions we're allowed to set
func TestMemFsMkdirModeIllegal(t *testing.T) {
t.Parallel()
fs := NewMemMapFs()
err := fs.Mkdir("/a", os.ModeSocket|0755)
if err != nil {
t.Fatal(err)
}
info, err := fs.Stat("/a")
if err != nil {
t.Fatal(err)
}
if info.Mode() != os.FileMode(os.ModeDir|0755) {
t.Fatalf("should not be able to use Mkdir to set illegal mode: %s", info.Mode().String())
}
}
// can't use OpenFile to get around which permissions we're allowed to set
func TestMemFsOpenFileModeIllegal(t *testing.T) {
t.Parallel()
fs := NewMemMapFs()
file, err := fs.OpenFile("/a", os.O_CREATE, os.ModeSymlink|0644)
if err != nil {
t.Fatal(err)
}
defer file.Close()
info, err := fs.Stat("/a")
if err != nil {
t.Fatal(err)
}
if info.Mode() != os.FileMode(0644) {
t.Fatalf("should not be able to use OpenFile to set illegal mode: %s", info.Mode().String())
}
}
// LstatIfPossible should always return false, since MemMapFs does not
// support symlinks.
func TestMemFsLstatIfPossible(t *testing.T) {
t.Parallel()
fs := NewMemMapFs()
// We assert that fs implements Lstater
fsAsserted, ok := fs.(Lstater)
if !ok {
t.Fatalf("The filesytem does not implement Lstater")
}
file, err := fs.OpenFile("/a.txt", os.O_CREATE, 0o644)
if err != nil {
t.Fatalf("Error when opening file: %v", err)
}
defer file.Close()
_, lstatCalled, err := fsAsserted.LstatIfPossible("/a.txt")
if err != nil {
t.Fatalf("Function returned err: %v", err)
}
if lstatCalled {
t.Fatalf("Function indicated lstat was called. This should never be true.")
}
}

12
os.go
View File

@ -91,6 +91,10 @@ func (OsFs) Chmod(name string, mode os.FileMode) error {
return os.Chmod(name, mode)
}
func (OsFs) Chown(name string, uid, gid int) error {
return os.Chown(name, uid, gid)
}
func (OsFs) Chtimes(name string, atime time.Time, mtime time.Time) error {
return os.Chtimes(name, atime, mtime)
}
@ -100,6 +104,10 @@ func (OsFs) LstatIfPossible(name string) (os.FileInfo, bool, error) {
return fi, true, err
}
func (OsFs) Chown(name string, uid, gid int) error {
return os.Chown(name, uid, gid)
func (OsFs) SymlinkIfPossible(oldname, newname string) error {
return os.Symlink(oldname, newname)
}
func (OsFs) ReadlinkIfPossible(name string) (string, error) {
return os.Readlink(name)
}

20
readonlyfs.go
View File

@ -28,6 +28,10 @@ func (r *ReadOnlyFs) Chmod(n string, m os.FileMode) error {
return syscall.EPERM
}
func (r *ReadOnlyFs) Chown(n string, uid, gid int) error {
return syscall.EPERM
}
func (r *ReadOnlyFs) Name() string {
return "ReadOnlyFilter"
}
@ -44,6 +48,18 @@ func (r *ReadOnlyFs) LstatIfPossible(name string) (os.FileInfo, bool, error) {
return fi, false, err
}
func (r *ReadOnlyFs) SymlinkIfPossible(oldname, newname string) error {
return &os.LinkError{Op: "symlink", Old: oldname, New: newname, Err: ErrNoSymlink}
}
func (r *ReadOnlyFs) ReadlinkIfPossible(name string) (string, error) {
if srdr, ok := r.source.(LinkReader); ok {
return srdr.ReadlinkIfPossible(name)
}
return "", &os.PathError{Op: "readlink", Path: name, Err: ErrNoReadlink}
}
func (r *ReadOnlyFs) Rename(o, n string) error {
return syscall.EPERM
}
@ -78,7 +94,3 @@ func (r *ReadOnlyFs) MkdirAll(n string, p os.FileMode) error {
func (r *ReadOnlyFs) Create(n string) (File, error) {
return nil, syscall.EPERM
}
func (r *ReadOnlyFs) Chown(n string, uid, gid int) error {
return syscall.EPERM
}

17
regexpfs.go
View File

@ -60,6 +60,13 @@ func (r *RegexpFs) Chmod(name string, mode os.FileMode) error {
return r.source.Chmod(name, mode)
}
func (r *RegexpFs) Chown(name string, uid, gid int) error {
if err := r.dirOrMatches(name); err != nil {
return err
}
return r.source.Chown(name, uid, gid)
}
func (r *RegexpFs) Name() string {
return "RegexpFs"
}
@ -126,6 +133,9 @@ func (r *RegexpFs) Open(name string) (File, error) {
}
}
f, err := r.source.Open(name)
if err != nil {
return nil, err
}
return &RegexpFile{f: f, re: r.re}, nil
}
@ -212,10 +222,3 @@ func (f *RegexpFile) Truncate(s int64) error {
func (f *RegexpFile) WriteString(s string) (int, error) {
return f.f.WriteString(s)
}
func (r *RegexpFs) Chown(name string, uid, gid int) error {
if err := r.dirOrMatches(name); err != nil {
return err
}
return r.source.Chown(name, uid, gid)
}

16
sftpfs/sftp.go
View File

@ -94,8 +94,14 @@ func (s Fs) Open(name string) (afero.File, error) {
return FileOpen(s.client, name)
}
// OpenFile calls the OpenFile method on the SSHFS connection. The mode argument
// is ignored because it's ignored by the github.com/pkg/sftp implementation.
func (s Fs) OpenFile(name string, flag int, perm os.FileMode) (afero.File, error) {
return nil, nil
sshfsFile, err := s.client.OpenFile(name, flag)
if err != nil {
return nil, err
}
return &File{fd: sshfsFile}, nil
}
func (s Fs) Remove(name string) error {
@ -124,10 +130,10 @@ func (s Fs) Chmod(name string, mode os.FileMode) error {
return s.client.Chmod(name, mode)
}
func (s Fs) Chtimes(name string, atime time.Time, mtime time.Time) error {
return s.client.Chtimes(name, atime, mtime)
}
func (s Fs) Chown(name string, uid, gid int) error {
return s.client.Chown(name, uid, gid)
}
func (s Fs) Chtimes(name string, atime time.Time, mtime time.Time) error {
return s.client.Chtimes(name, atime, mtime)
}

107
sftpfs/sftp_test_go → sftpfs/sftp_test.go
View File

@ -11,24 +11,24 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package afero
package sftpfs
import (
"testing"
"os"
"log"
"fmt"
"net"
"flag"
"time"
"io/ioutil"
"crypto/rsa"
_rand "crypto/rand"
"encoding/pem"
"crypto/rsa"
"crypto/x509"
"encoding/pem"
"flag"
"fmt"
"io/ioutil"
"log"
"net"
"os"
"testing"
"time"
"golang.org/x/crypto/ssh"
"github.com/pkg/sftp"
"golang.org/x/crypto/ssh"
)
type SftpFsContext struct {
@ -40,50 +40,51 @@ type SftpFsContext struct {
// TODO we only connect with hardcoded user+pass for now
// it should be possible to use $HOME/.ssh/id_rsa to login into the stub sftp server
func SftpConnect(user, password, host string) (*SftpFsContext, error) {
/*
pemBytes, err := ioutil.ReadFile(os.Getenv("HOME") + "/.ssh/id_rsa")
if err != nil {
return nil,err
}
/*
pemBytes, err := ioutil.ReadFile(os.Getenv("HOME") + "/.ssh/id_rsa")
if err != nil {
return nil,err
}
signer, err := ssh.ParsePrivateKey(pemBytes)
if err != nil {
return nil,err
}
signer, err := ssh.ParsePrivateKey(pemBytes)
if err != nil {
return nil,err
}
sshcfg := &ssh.ClientConfig{
User: user,
Auth: []ssh.AuthMethod{
ssh.Password(password),
ssh.PublicKeys(signer),
},
}
*/
sshcfg := &ssh.ClientConfig{
User: user,
Auth: []ssh.AuthMethod{
ssh.Password(password),
ssh.PublicKeys(signer),
},
}
*/
sshcfg := &ssh.ClientConfig{
User: user,
Auth: []ssh.AuthMethod{
ssh.Password(password),
},
HostKeyCallback: ssh.InsecureIgnoreHostKey(),
}
sshc, err := ssh.Dial("tcp", host, sshcfg)
if err != nil {
return nil,err
return nil, err
}
sftpc, err := sftp.NewClient(sshc)
if err != nil {
return nil,err
return nil, err
}
ctx := &SftpFsContext{
sshc: sshc,
sshc: sshc,
sshcfg: sshcfg,
sftpc: sftpc,
sftpc: sftpc,
}
return ctx,nil
return ctx, nil
}
func (ctx *SftpFsContext) Disconnect() error {
@ -97,7 +98,6 @@ func RunSftpServer(rootpath string) {
var (
readOnly bool
debugLevelStr string
debugLevel int
debugStderr bool
rootDir string
)
@ -109,10 +109,6 @@ func RunSftpServer(rootpath string) {
flag.Parse()
debugStream := ioutil.Discard
if debugStderr {
debugStream = os.Stderr
debugLevel = 1
}
// An SSH server is represented by a ServerConfig, which holds
// certificate details and handles authentication of ServerConns.
@ -146,7 +142,6 @@ func RunSftpServer(rootpath string) {
if err != nil {
log.Fatal("failed to listen for connection", err)
}
fmt.Printf("Listening on %v\n", listener.Addr())
nConn, err := listener.Accept()
if err != nil {
@ -155,11 +150,11 @@ func RunSftpServer(rootpath string) {
// Before use, a handshake must be performed on the incoming
// net.Conn.
_, chans, reqs, err := ssh.NewServerConn(nConn, config)
conn, chans, reqs, err := ssh.NewServerConn(nConn, config)
if err != nil {
log.Fatal("failed to handshake", err)
}
fmt.Fprintf(debugStream, "SSH server established\n")
defer conn.Close()
// The incoming Request channel must be serviced.
go ssh.DiscardRequests(reqs)
@ -200,13 +195,12 @@ func RunSftpServer(rootpath string) {
}
}(requests)
server, err := sftp.NewServer(channel, channel, debugStream, debugLevel, readOnly, rootpath)
server, err := sftp.NewServer(channel, sftp.WithDebug(debugStream))
if err != nil {
log.Fatal(err)
}
if err := server.Serve(); err != nil {
log.Fatal("sftp server completed with error:", err)
}
_ = server.Serve()
return
}
}
@ -253,25 +247,23 @@ func TestSftpCreate(t *testing.T) {
}
defer ctx.Disconnect()
var AppFs Fs = SftpFs{
SftpClient: ctx.sftpc,
}
var fs = New(ctx.sftpc)
AppFs.MkdirAll("test/dir1/dir2/dir3", os.FileMode(0777))
AppFs.Mkdir("test/foo", os.FileMode(0000))
AppFs.Chmod("test/foo", os.FileMode(0700))
AppFs.Mkdir("test/bar", os.FileMode(0777))
fs.MkdirAll("test/dir1/dir2/dir3", os.FileMode(0777))
fs.Mkdir("test/foo", os.FileMode(0000))
fs.Chmod("test/foo", os.FileMode(0700))
fs.Mkdir("test/bar", os.FileMode(0777))
file, err := AppFs.Create("file1")
file, err := fs.Create("file1")
if err != nil {
t.Error(err)
}
defer file.Close()
file.Write([]byte("hello\t"))
file.Write([]byte("hello "))
file.WriteString("world!\n")
f1, err := AppFs.Open("file1")
f1, err := fs.Open("file1")
if err != nil {
log.Fatalf("open: %v", err)
}
@ -279,8 +271,9 @@ func TestSftpCreate(t *testing.T) {
b := make([]byte, 100)
_, err = f1.Read(b)
_, _ = f1.Read(b)
fmt.Println(string(b))
fmt.Println("done")
// TODO check here if "hello\tworld\n" is in buffer b
}

55
symlink.go Normal file
View File

@ -0,0 +1,55 @@
// Copyright © 2018 Steve Francia <spf@spf13.com>.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package afero
import (
"errors"
)
// Symlinker is an optional interface in Afero. It is only implemented by the
// filesystems saying so.
// It indicates support for 3 symlink related interfaces that implement the
// behaviors of the os methods:
// - Lstat
// - Symlink, and
// - Readlink
type Symlinker interface {
Lstater
Linker
LinkReader
}
// Linker is an optional interface in Afero. It is only implemented by the
// filesystems saying so.
// It will call Symlink if the filesystem itself is, or it delegates to, the os filesystem,
// or the filesystem otherwise supports Symlink's.
type Linker interface {
SymlinkIfPossible(oldname, newname string) error
}
// ErrNoSymlink is the error that will be wrapped in an os.LinkError if a file system
// does not support Symlink's either directly or through its delegated filesystem.
// As expressed by support for the Linker interface.
var ErrNoSymlink = errors.New("symlink not supported")
// LinkReader is an optional interface in Afero. It is only implemented by the
// filesystems saying so.
type LinkReader interface {
ReadlinkIfPossible(name string) (string, error)
}
// ErrNoReadlink is the error that will be wrapped in an os.Path if a file system
// does not support the readlink operation either directly or through its delegated filesystem.
// As expressed by support for the LinkReader interface.
var ErrNoReadlink = errors.New("readlink not supported")

160
symlink_test.go Normal file
View File

@ -0,0 +1,160 @@
package afero
import (
"os"
"path/filepath"
"strings"
"testing"
)
func TestSymlinkIfPossible(t *testing.T) {
wd, _ := os.Getwd()
defer func() {
os.Chdir(wd)
}()
osFs := &OsFs{}
workDir, err := TempDir(osFs, "", "afero-symlink")
if err != nil {
t.Fatal(err)
}
defer func() {
osFs.RemoveAll(workDir)
}()
memWorkDir := "/sym"
memFs := NewMemMapFs()
overlayFs1 := &CopyOnWriteFs{base: osFs, layer: memFs}
overlayFs2 := &CopyOnWriteFs{base: memFs, layer: osFs}
overlayFsMemOnly := &CopyOnWriteFs{base: memFs, layer: NewMemMapFs()}
basePathFs := &BasePathFs{source: osFs, path: workDir}
basePathFsMem := &BasePathFs{source: memFs, path: memWorkDir}
roFs := &ReadOnlyFs{source: osFs}
roFsMem := &ReadOnlyFs{source: memFs}
pathFileMem := filepath.Join(memWorkDir, "aferom.txt")
osPath := filepath.Join(workDir, "afero.txt")
WriteFile(osFs, osPath, []byte("Hi, Afero!"), 0777)
WriteFile(memFs, filepath.Join(pathFileMem), []byte("Hi, Afero!"), 0777)
testLink := func(l Linker, source, destination string, output *string) {
if fs, ok := l.(Fs); ok {
dir := filepath.Dir(destination)
if dir != "" {
fs.MkdirAll(dir, 0777)
}
}
err := l.SymlinkIfPossible(source, destination)
if (err == nil) && (output != nil) {
t.Fatalf("Error creating symlink, succeeded when expecting error %v", *output)
} else if (err != nil) && (output == nil) {
t.Fatalf("Error creating symlink, expected success, got %v", err)
} else if err != nil && err.Error() != *output && !strings.HasSuffix(err.Error(), *output) {
t.Fatalf("Error creating symlink, expected error '%v', instead got output '%v'", *output, err)
} else {
// test passed, if expecting a successful link, check the link with lstat if able
if output == nil {
if lst, ok := l.(Lstater); ok {
_, ok, err := lst.LstatIfPossible(destination)
if !ok {
if err != nil {
t.Fatalf("Error calling lstat on file after successful link, got: %v", err)
} else {
t.Fatalf("Error calling lstat on file after successful link, result didn't use lstat (not link)")
}
return
}
}
}
}
}
notSupported := ErrNoSymlink.Error()
testLink(osFs, osPath, filepath.Join(workDir, "os/link.txt"), nil)
testLink(overlayFs1, osPath, filepath.Join(workDir, "overlay/link1.txt"), &notSupported)
testLink(overlayFs2, pathFileMem, filepath.Join(workDir, "overlay2/link2.txt"), nil)
testLink(overlayFsMemOnly, pathFileMem, filepath.Join(memWorkDir, "overlay3/link.txt"), &notSupported)
testLink(basePathFs, "afero.txt", "basepath/link.txt", nil)
testLink(basePathFsMem, pathFileMem, "link/file.txt", &notSupported)
testLink(roFs, osPath, filepath.Join(workDir, "ro/link.txt"), &notSupported)
testLink(roFsMem, pathFileMem, filepath.Join(memWorkDir, "ro/link.txt"), &notSupported)
}
func TestReadlinkIfPossible(t *testing.T) {
wd, _ := os.Getwd()
defer func() {
os.Chdir(wd)
}()
osFs := &OsFs{}
workDir, err := TempDir(osFs, "", "afero-readlink")
if err != nil {
t.Fatal(err)
}
defer func() {
osFs.RemoveAll(workDir)
}()
memWorkDir := "/read"
memFs := NewMemMapFs()
overlayFs1 := &CopyOnWriteFs{base: osFs, layer: memFs}
overlayFs2 := &CopyOnWriteFs{base: memFs, layer: osFs}
overlayFsMemOnly := &CopyOnWriteFs{base: memFs, layer: NewMemMapFs()}
basePathFs := &BasePathFs{source: osFs, path: workDir}
basePathFsMem := &BasePathFs{source: memFs, path: memWorkDir}
roFs := &ReadOnlyFs{source: osFs}
roFsMem := &ReadOnlyFs{source: memFs}
pathFileMem := filepath.Join(memWorkDir, "aferom.txt")
osPath := filepath.Join(workDir, "afero.txt")
WriteFile(osFs, osPath, []byte("Hi, Afero!"), 0777)
WriteFile(memFs, filepath.Join(pathFileMem), []byte("Hi, Afero!"), 0777)
createLink := func(l Linker, source, destination string) error {
if fs, ok := l.(Fs); ok {
dir := filepath.Dir(destination)
if dir != "" {
fs.MkdirAll(dir, 0777)
}
}
return l.SymlinkIfPossible(source, destination)
}
testRead := func(r LinkReader, name string, output *string) {
_, err := r.ReadlinkIfPossible(name)
if (err != nil) && (output == nil) {
t.Fatalf("Error reading link, expected success, got error: %v", err)
} else if (err == nil) && (output != nil) {
t.Fatalf("Error reading link, succeeded when expecting error: %v", *output)
} else if err != nil && err.Error() != *output && !strings.HasSuffix(err.Error(), *output) {
t.Fatalf("Error reading link, expected error '%v', instead received '%v'", *output, err)
}
}
notSupported := ErrNoReadlink.Error()
err = createLink(osFs, osPath, filepath.Join(workDir, "os/link.txt"))
if err != nil {
t.Fatal("Error creating test link: ", err)
}
testRead(osFs, filepath.Join(workDir, "os/link.txt"), nil)
testRead(overlayFs1, filepath.Join(workDir, "os/link.txt"), nil)
testRead(overlayFs2, filepath.Join(workDir, "os/link.txt"), nil)
testRead(overlayFsMemOnly, pathFileMem, &notSupported)
testRead(basePathFs, "os/link.txt", nil)
testRead(basePathFsMem, pathFileMem, &notSupported)
testRead(roFs, filepath.Join(workDir, "os/link.txt"), nil)
testRead(roFsMem, pathFileMem, &notSupported)
}

144
tarfs/file.go Normal file
View File

@ -0,0 +1,144 @@
package tarfs
import (
"archive/tar"
"bytes"
"os"
"path/filepath"
"sort"
"syscall"
"github.com/spf13/afero"
)
type File struct {
h *tar.Header
data *bytes.Reader
closed bool
fs *Fs
}
func (f *File) Close() error {
if f.closed {
return afero.ErrFileClosed
}
f.closed = true
f.h = nil
f.data = nil
f.fs = nil
return nil
}
func (f *File) Read(p []byte) (n int, err error) {
if f.closed {
return 0, afero.ErrFileClosed
}
if f.h.Typeflag == tar.TypeDir {
return 0, syscall.EISDIR
}
return f.data.Read(p)
}
func (f *File) ReadAt(p []byte, off int64) (n int, err error) {
if f.closed {
return 0, afero.ErrFileClosed
}
if f.h.Typeflag == tar.TypeDir {
return 0, syscall.EISDIR
}
return f.data.ReadAt(p, off)
}
func (f *File) Seek(offset int64, whence int) (int64, error) {
if f.closed {
return 0, afero.ErrFileClosed
}
if f.h.Typeflag == tar.TypeDir {
return 0, syscall.EISDIR
}
return f.data.Seek(offset, whence)
}
func (f *File) Write(p []byte) (n int, err error) { return 0, syscall.EROFS }
func (f *File) WriteAt(p []byte, off int64) (n int, err error) { return 0, syscall.EROFS }
func (f *File) Name() string {
return filepath.Join(splitpath(f.h.Name))
}
func (f *File) getDirectoryNames() ([]string, error) {
d, ok := f.fs.files[f.Name()]
if !ok {
return nil, &os.PathError{Op: "readdir", Path: f.Name(), Err: syscall.ENOENT}
}
var names []string
for n := range d {
names = append(names, n)
}
sort.Strings(names)
return names, nil
}
func (f *File) Readdir(count int) ([]os.FileInfo, error) {
if f.closed {
return nil, afero.ErrFileClosed
}
if !f.h.FileInfo().IsDir() {
return nil, syscall.ENOTDIR
}
names, err := f.getDirectoryNames()
if err != nil {
return nil, err
}
d := f.fs.files[f.Name()]
var fi []os.FileInfo
for _, n := range names {
if n == "" {
continue
}
f := d[n]
fi = append(fi, f.h.FileInfo())
if count > 0 && len(fi) >= count {
break
}
}
return fi, nil
}
func (f *File) Readdirnames(n int) ([]string, error) {
fi, err := f.Readdir(n)
if err != nil {
return nil, err
}
var names []string
for _, f := range fi {
names = append(names, f.Name())
}
return names, nil
}
func (f *File) Stat() (os.FileInfo, error) { return f.h.FileInfo(), nil }
func (f *File) Sync() error { return nil }
func (f *File) Truncate(size int64) error { return syscall.EROFS }
func (f *File) WriteString(s string) (ret int, err error) { return 0, syscall.EROFS }

139
tarfs/fs.go Normal file
View File

@ -0,0 +1,139 @@
// package tarfs implements a read-only in-memory representation of a tar archive
package tarfs
import (
"archive/tar"
"bytes"
"io"
"os"
"path/filepath"
"syscall"
"time"
"github.com/spf13/afero"
)
type Fs struct {
files map[string]map[string]*File
}
func splitpath(name string) (dir, file string) {
name = filepath.ToSlash(name)
if len(name) == 0 || name[0] != '/' {
name = "/" + name
}
name = filepath.Clean(name)
dir, file = filepath.Split(name)
dir = filepath.Clean(dir)
return
}
func New(t *tar.Reader) *Fs {
fs := &Fs{files: make(map[string]map[string]*File)}
for {
hdr, err := t.Next()
if err == io.EOF {
break
}
if err != nil {
return nil
}
d, f := splitpath(hdr.Name)
if _, ok := fs.files[d]; !ok {
fs.files[d] = make(map[string]*File)
}
var buf bytes.Buffer
size, err := buf.ReadFrom(t)
if err != nil {
panic("tarfs: reading from tar:" + err.Error())
}
if size != hdr.Size {
panic("tarfs: size mismatch")
}
file := &File{
h: hdr,
data: bytes.NewReader(buf.Bytes()),
fs: fs,
}
fs.files[d][f] = file
}
if fs.files[afero.FilePathSeparator] == nil {
fs.files[afero.FilePathSeparator] = make(map[string]*File)
}
// Add a pseudoroot
fs.files[afero.FilePathSeparator][""] = &File{
h: &tar.Header{
Name: afero.FilePathSeparator,
Typeflag: tar.TypeDir,
Size: 0,
},
data: bytes.NewReader(nil),
fs: fs,
}
return fs
}
func (fs *Fs) Open(name string) (afero.File, error) {
d, f := splitpath(name)
if _, ok := fs.files[d]; !ok {
return nil, &os.PathError{Op: "open", Path: name, Err: syscall.ENOENT}
}
file, ok := fs.files[d][f]
if !ok {
return nil, &os.PathError{Op: "open", Path: name, Err: syscall.ENOENT}
}
nf := *file
return &nf, nil
}
func (fs *Fs) Name() string { return "tarfs" }
func (fs *Fs) Create(name string) (afero.File, error) { return nil, syscall.EROFS }
func (fs *Fs) Mkdir(name string, perm os.FileMode) error { return syscall.EROFS }
func (fs *Fs) MkdirAll(path string, perm os.FileMode) error { return syscall.EROFS }
func (fs *Fs) OpenFile(name string, flag int, perm os.FileMode) (afero.File, error) {
if flag != os.O_RDONLY {
return nil, &os.PathError{Op: "open", Path: name, Err: syscall.EPERM}
}
return fs.Open(name)
}
func (fs *Fs) Remove(name string) error { return syscall.EROFS }
func (fs *Fs) RemoveAll(path string) error { return syscall.EROFS }
func (fs *Fs) Rename(oldname string, newname string) error { return syscall.EROFS }
func (fs *Fs) Stat(name string) (os.FileInfo, error) {
d, f := splitpath(name)
if _, ok := fs.files[d]; !ok {
return nil, &os.PathError{Op: "stat", Path: name, Err: syscall.ENOENT}
}
file, ok := fs.files[d][f]
if !ok {
return nil, &os.PathError{Op: "stat", Path: name, Err: syscall.ENOENT}
}
return file.h.FileInfo(), nil
}
func (fs *Fs) Chmod(name string, mode os.FileMode) error { return syscall.EROFS }
func (fs *Fs) Chown(name string, uid, gid int) error { return syscall.EROFS }
func (fs *Fs) Chtimes(name string, atime time.Time, mtime time.Time) error { return syscall.EROFS }

406
tarfs/tarfs_test.go Normal file
View File

@ -0,0 +1,406 @@
// Most of the tests are stolen from the zipfs implementation
package tarfs
import (
"archive/tar"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"reflect"
"strings"
"syscall"
"testing"
"github.com/spf13/afero"
)
var files = []struct {
name string
exists bool
isdir bool
size int64
content string
contentAt4k string
}{
{"/", true, true, 0, "", ""},
{"/sub", true, true, 0, "", ""},
{"/sub/testDir2", true, true, 0, "", ""},
{"/sub/testDir2/testFile", true, false, 8192, "cccccccc", "ccccdddd"},
{"/testFile", true, false, 8192, "aaaaaaaa", "aaaabbbb"},
{"/testDir1/testFile", true, false, 8192, "bbbbbbbb", "bbbbcccc"},
{"/nonExisting", false, false, 0, "", ""},
}
var dirs = []struct {
name string
children []string
}{
{"/", []string{"sub", "testDir1", "testFile"}},
{"/sub", []string{"testDir2"}},
{"/sub/testDir2", []string{"testFile"}},
{"/testDir1", []string{"testFile"}},
}
var afs *afero.Afero
func TestMain(m *testing.M) {
tf, err := os.Open("testdata/t.tar")
if err != nil {
fmt.Print(err)
os.Exit(1)
}
tfs := New(tar.NewReader(tf))
afs = &afero.Afero{Fs: tfs}
// Check that an empty reader does not panic.
_ = New(tar.NewReader(strings.NewReader("")))
os.Exit(m.Run())
}
func TestFsOpen(t *testing.T) {
for _, f := range files {
file, err := afs.Open(f.name)
if (err == nil) != f.exists {
t.Errorf("%v exists = %v, but got err = %v", f.name, f.exists, err)
}
if !f.exists {
continue
}
if err != nil {
t.Fatalf("%v: %v", f.name, err)
}
if file.Name() != filepath.FromSlash(f.name) {
t.Errorf("Name(), got %v, expected %v", file.Name(), filepath.FromSlash(f.name))
}
s, err := file.Stat()
if err != nil {
t.Fatalf("stat %v: got error '%v'", file.Name(), err)
}
if isdir := s.IsDir(); isdir != f.isdir {
t.Errorf("%v directory, got: %v, expected: %v", file.Name(), isdir, f.isdir)
}
if size := s.Size(); size != f.size {
t.Errorf("%v size, got: %v, expected: %v", file.Name(), size, f.size)
}
}
}
func TestRead(t *testing.T) {
for _, f := range files {
if !f.exists {
continue
}
file, err := afs.Open(f.name)
if err != nil {
t.Fatalf("opening %v: %v", f.name, err)
}
buf := make([]byte, 8)
n, err := file.Read(buf)
if err != nil {
if f.isdir && (err != syscall.EISDIR) {
t.Errorf("%v got error %v, expected EISDIR", f.name, err)
} else if !f.isdir {
t.Errorf("%v: %v", f.name, err)
}
} else if n != 8 {
t.Errorf("%v: got %d read bytes, expected 8", f.name, n)
} else if string(buf) != f.content {
t.Errorf("%v: got <%s>, expected <%s>", f.name, f.content, string(buf))
}
}
}
func TestReadAt(t *testing.T) {
for _, f := range files {
if !f.exists {
continue
}
file, err := afs.Open(f.name)
if err != nil {
t.Fatalf("opening %v: %v", f.name, err)
}
buf := make([]byte, 8)
n, err := file.ReadAt(buf, 4092)
if err != nil {
if f.isdir && (err != syscall.EISDIR) {
t.Errorf("%v got error %v, expected EISDIR", f.name, err)
} else if !f.isdir {
t.Errorf("%v: %v", f.name, err)
}
} else if n != 8 {
t.Errorf("%v: got %d read bytes, expected 8", f.name, n)
} else if string(buf) != f.contentAt4k {
t.Errorf("%v: got <%s>, expected <%s>", f.name, f.contentAt4k, string(buf))
}
}
}
func TestSeek(t *testing.T) {
for _, f := range files {
if !f.exists {
continue
}
file, err := afs.Open(f.name)
if err != nil {
t.Fatalf("opening %v: %v", f.name, err)
}
var tests = []struct {
offin int64
whence int
offout int64
}{
{0, io.SeekStart, 0},
{10, io.SeekStart, 10},
{1, io.SeekCurrent, 11},
{10, io.SeekCurrent, 21},
{0, io.SeekEnd, f.size},
{-1, io.SeekEnd, f.size - 1},
}
for _, s := range tests {
n, err := file.Seek(s.offin, s.whence)
if err != nil {
if f.isdir && err == syscall.EISDIR {
continue
}
t.Errorf("%v: %v", f.name, err)
}
if n != s.offout {
t.Errorf("%v: (off: %v, whence: %v): got %v, expected %v", f.name, s.offin, s.whence, n, s.offout)
}
}
}
}
func TestName(t *testing.T) {
for _, f := range files {
if !f.exists {
continue
}
file, err := afs.Open(f.name)
if err != nil {
t.Fatalf("opening %v: %v", f.name, err)
}
n := file.Name()
if n != filepath.FromSlash(f.name) {
t.Errorf("got: %v, expected: %v", n, filepath.FromSlash(f.name))
}
}
}
func TestClose(t *testing.T) {
for _, f := range files {
if !f.exists {
continue
}
file, err := afs.Open(f.name)
if err != nil {
t.Fatalf("opening %v: %v", f.name, err)
}
err = file.Close()
if err != nil {
t.Errorf("%v: %v", f.name, err)
}
err = file.Close()
if err == nil {
t.Errorf("%v: closing twice should return an error", f.name)
}
buf := make([]byte, 8)
n, err := file.Read(buf)
if n != 0 || err == nil {
t.Errorf("%v: could read from a closed file", f.name)
}
n, err = file.ReadAt(buf, 256)
if n != 0 || err == nil {
t.Errorf("%v: could readAt from a closed file", f.name)
}
off, err := file.Seek(0, io.SeekStart)
if off != 0 || err == nil {
t.Errorf("%v: could seek from a closed file", f.name)
}
}
}
func TestOpenFile(t *testing.T) {
for _, f := range files {
file, err := afs.OpenFile(f.name, os.O_RDONLY, 0400)
if !f.exists {
if !errors.Is(err, syscall.ENOENT) {
t.Errorf("%v: got %v, expected%v", f.name, err, syscall.ENOENT)
}
continue
}
if err != nil {
t.Fatalf("%v: %v", f.name, err)
}
file.Close()
file, err = afs.OpenFile(f.name, os.O_CREATE, 0600)
if !errors.Is(err, syscall.EPERM) {
t.Errorf("%v: open for write: got %v, expected %v", f.name, err, syscall.EPERM)
}
}
}
func TestFsStat(t *testing.T) {
for _, f := range files {
fi, err := afs.Stat(f.name)
if !f.exists {
if !errors.Is(err, syscall.ENOENT) {
t.Errorf("%v: got %v, expected%v", f.name, err, syscall.ENOENT)
}
continue
}
if err != nil {
t.Fatalf("stat %v: got error '%v'", f.name, err)
}
if isdir := fi.IsDir(); isdir != f.isdir {
t.Errorf("%v directory, got: %v, expected: %v", f.name, isdir, f.isdir)
}
if size := fi.Size(); size != f.size {
t.Errorf("%v size, got: %v, expected: %v", f.name, size, f.size)
}
}
}
func TestReaddir(t *testing.T) {
for _, d := range dirs {
dir, err := afs.Open(d.name)
if err != nil {
t.Fatal(err)
}
fi, err := dir.Readdir(0)
if err != nil {
t.Fatal(err)
}
var names []string
for _, f := range fi {
names = append(names, f.Name())
}
if !reflect.DeepEqual(names, d.children) {
t.Errorf("%v: children, got '%v', expected '%v'", d.name, names, d.children)
}
fi, err = dir.Readdir(1)
if err != nil {
t.Fatal(err)
}
names = []string{}
for _, f := range fi {
names = append(names, f.Name())
}
if !reflect.DeepEqual(names, d.children[0:1]) {
t.Errorf("%v: children, got '%v', expected '%v'", d.name, names, d.children[0:1])
}
}
dir, err := afs.Open("/testFile")
if err != nil {
t.Fatal(err)
}
_, err = dir.Readdir(-1)
if err != syscall.ENOTDIR {
t.Fatal("Expected error")
}
}
func TestReaddirnames(t *testing.T) {
for _, d := range dirs {
dir, err := afs.Open(d.name)
if err != nil {
t.Fatal(err)
}
names, err := dir.Readdirnames(0)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(names, d.children) {
t.Errorf("%v: children, got '%v', expected '%v'", d.name, names, d.children)
}
names, err = dir.Readdirnames(1)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(names, d.children[0:1]) {
t.Errorf("%v: children, got '%v', expected '%v'", d.name, names, d.children[0:1])
}
}
dir, err := afs.Open("/testFile")
if err != nil {
t.Fatal(err)
}
_, err = dir.Readdir(-1)
if err != syscall.ENOTDIR {
t.Fatal("Expected error")
}
}
func TestGlob(t *testing.T) {
for _, s := range []struct {
glob string
entries []string
}{
{filepath.FromSlash("/*"), []string{filepath.FromSlash("/sub"), filepath.FromSlash("/testDir1"), filepath.FromSlash("/testFile")}},
{filepath.FromSlash("*"), []string{filepath.FromSlash("sub"), filepath.FromSlash("testDir1"), filepath.FromSlash("testFile")}},
{filepath.FromSlash("sub/*"), []string{filepath.FromSlash("sub/testDir2")}},
{filepath.FromSlash("sub/testDir2/*"), []string{filepath.FromSlash("sub/testDir2/testFile")}},
{filepath.FromSlash("testDir1/*"), []string{filepath.FromSlash("testDir1/testFile")}},
} {
entries, err := afero.Glob(afs.Fs, s.glob)
if err != nil {
t.Error(err)
}
if reflect.DeepEqual(entries, s.entries) {
t.Logf("glob: %s: glob ok", s.glob)
} else {
t.Errorf("glob: %s: got %#v, expected %#v", s.glob, entries, s.entries)
}
}
}

BIN
tarfs/testdata/t.tar vendored Normal file
View File

Binary file not shown.

17
unionFile.go
View File

@ -186,25 +186,22 @@ func (f *UnionFile) Readdir(c int) (ofi []os.FileInfo, err error) {
}
f.files = append(f.files, merged...)
}
files := f.files[f.off:]
if c <= 0 && len(f.files) == 0 {
return f.files, nil
if c <= 0 {
return files, nil
}
if f.off >= len(f.files) {
if len(files) == 0 {
return nil, io.EOF
}
if c <= 0 {
return f.files[f.off:], nil
}
if c > len(f.files) {
c = len(f.files)
if c > len(files) {
c = len(files)
}
defer func() { f.off += c }()
return f.files[f.off:c], nil
return files[:c], nil
}
func (f *UnionFile) Readdirnames(c int) ([]string, error) {

20
util_test.go
View File

@ -415,10 +415,10 @@ func TestFullBaseFsPath(t *testing.T) {
Dir1, Dir2, Dir3 string
}
dirSpecs := []dirSpec{
dirSpec{Dir1: "/", Dir2: "/", Dir3: "/"},
dirSpec{Dir1: "/", Dir2: "/path2", Dir3: "/"},
dirSpec{Dir1: "/path1/dir", Dir2: "/path2/dir/", Dir3: "/path3/dir"},
dirSpec{Dir1: "C:/path1", Dir2: "path2/dir", Dir3: "/path3/dir/"},
{Dir1: "/", Dir2: "/", Dir3: "/"},
{Dir1: "/", Dir2: "/path2", Dir3: "/"},
{Dir1: "/path1/dir", Dir2: "/path2/dir/", Dir3: "/path3/dir"},
{Dir1: "C:/path1", Dir2: "path2/dir", Dir3: "/path3/dir/"},
}
for _, ds := range dirSpecs {
@ -433,12 +433,12 @@ func TestFullBaseFsPath(t *testing.T) {
ExpectedPath string
}
specs := []spec{
spec{BaseFs: level3Fs, FileName: "f.txt", ExpectedPath: filepath.Join(ds.Dir1, ds.Dir2, ds.Dir3, "f.txt")},
spec{BaseFs: level3Fs, FileName: "", ExpectedPath: filepath.Join(ds.Dir1, ds.Dir2, ds.Dir3, "")},
spec{BaseFs: level2Fs, FileName: "f.txt", ExpectedPath: filepath.Join(ds.Dir1, ds.Dir2, "f.txt")},
spec{BaseFs: level2Fs, FileName: "", ExpectedPath: filepath.Join(ds.Dir1, ds.Dir2, "")},
spec{BaseFs: level1Fs, FileName: "f.txt", ExpectedPath: filepath.Join(ds.Dir1, "f.txt")},
spec{BaseFs: level1Fs, FileName: "", ExpectedPath: filepath.Join(ds.Dir1, "")},
{BaseFs: level3Fs, FileName: "f.txt", ExpectedPath: filepath.Join(ds.Dir1, ds.Dir2, ds.Dir3, "f.txt")},
{BaseFs: level3Fs, FileName: "", ExpectedPath: filepath.Join(ds.Dir1, ds.Dir2, ds.Dir3, "")},
{BaseFs: level2Fs, FileName: "f.txt", ExpectedPath: filepath.Join(ds.Dir1, ds.Dir2, "f.txt")},
{BaseFs: level2Fs, FileName: "", ExpectedPath: filepath.Join(ds.Dir1, ds.Dir2, "")},
{BaseFs: level1Fs, FileName: "f.txt", ExpectedPath: filepath.Join(ds.Dir1, "f.txt")},
{BaseFs: level1Fs, FileName: "", ExpectedPath: filepath.Join(ds.Dir1, "")},
}
for _, s := range specs {

165
zipfs/file.go Normal file
View File

@ -0,0 +1,165 @@
package zipfs
import (
"archive/zip"
"io"
"os"
"path/filepath"
"syscall"
"github.com/spf13/afero"
)
type File struct {
fs *Fs
zipfile *zip.File
reader io.ReadCloser
offset int64
isdir, closed bool
buf []byte
}
func (f *File) fillBuffer(offset int64) (err error) {
if f.reader == nil {
if f.reader, err = f.zipfile.Open(); err != nil {
return
}
}
if offset > int64(f.zipfile.UncompressedSize64) {
offset = int64(f.zipfile.UncompressedSize64)
err = io.EOF
}
if len(f.buf) >= int(offset) {
return
}
buf := make([]byte, int(offset)-len(f.buf))
if n, readErr := io.ReadFull(f.reader, buf); n > 0 {
f.buf = append(f.buf, buf[:n]...)
} else if readErr != nil {
err = readErr
}
return
}
func (f *File) Close() (err error) {
f.zipfile = nil
f.closed = true
f.buf = nil
if f.reader != nil {
err = f.reader.Close()
f.reader = nil
}
return
}
func (f *File) Read(p []byte) (n int, err error) {
if f.isdir {
return 0, syscall.EISDIR
}
if f.closed {
return 0, afero.ErrFileClosed
}
err = f.fillBuffer(f.offset + int64(len(p)))
n = copy(p, f.buf[f.offset:])
f.offset += int64(n)
return
}
func (f *File) ReadAt(p []byte, off int64) (n int, err error) {
if f.isdir {
return 0, syscall.EISDIR
}
if f.closed {
return 0, afero.ErrFileClosed
}
err = f.fillBuffer(off + int64(len(p)))
n = copy(p, f.buf[int(off):])
return
}
func (f *File) Seek(offset int64, whence int) (int64, error) {
if f.isdir {
return 0, syscall.EISDIR
}
if f.closed {
return 0, afero.ErrFileClosed
}
switch whence {
case os.SEEK_SET:
case os.SEEK_CUR:
offset += f.offset
case os.SEEK_END:
offset += int64(f.zipfile.UncompressedSize64)
default:
return 0, syscall.EINVAL
}
if offset < 0 || offset > int64(f.zipfile.UncompressedSize64) {
return 0, afero.ErrOutOfRange
}
f.offset = offset
return offset, nil
}
func (f *File) Write(p []byte) (n int, err error) { return 0, syscall.EPERM }
func (f *File) WriteAt(p []byte, off int64) (n int, err error) { return 0, syscall.EPERM }
func (f *File) Name() string {
if f.zipfile == nil {
return string(filepath.Separator)
}
return filepath.Join(splitpath(f.zipfile.Name))
}
func (f *File) getDirEntries() (map[string]*zip.File, error) {
if !f.isdir {
return nil, syscall.ENOTDIR
}
name := f.Name()
entries, ok := f.fs.files[name]
if !ok {
return nil, &os.PathError{Op: "readdir", Path: name, Err: syscall.ENOENT}
}
return entries, nil
}
func (f *File) Readdir(count int) (fi []os.FileInfo, err error) {
zipfiles, err := f.getDirEntries()
if err != nil {
return nil, err
}
for _, zipfile := range zipfiles {
fi = append(fi, zipfile.FileInfo())
if count > 0 && len(fi) >= count {
break
}
}
return
}
func (f *File) Readdirnames(count int) (names []string, err error) {
zipfiles, err := f.getDirEntries()
if err != nil {
return nil, err
}
for filename := range zipfiles {
names = append(names, filename)
if count > 0 && len(names) >= count {
break
}
}
return
}
func (f *File) Stat() (os.FileInfo, error) {
if f.zipfile == nil {
return &pseudoRoot{}, nil
}
return f.zipfile.FileInfo(), nil
}
func (f *File) Sync() error { return nil }
func (f *File) Truncate(size int64) error { return syscall.EPERM }
func (f *File) WriteString(s string) (ret int, err error) { return 0, syscall.EPERM }

43
zipfs/file_test.go Normal file
View File

@ -0,0 +1,43 @@
package zipfs
import (
"archive/zip"
"io"
"testing"
)
func TestFileRead(t *testing.T) {
zrc, err := zip.OpenReader("testdata/small.zip")
if err != nil {
t.Fatal(err)
}
zfs := New(&zrc.Reader)
f, err := zfs.Open("smallFile")
if err != nil {
t.Fatal(err)
}
info, err := f.Stat()
if err != nil {
t.Fatal(err)
}
chunkSize := info.Size() * 2 // read with extra large buffer
buf := make([]byte, chunkSize)
n, err := f.Read(buf)
if err != io.EOF {
t.Fatal("Failed to read file to completion:", err)
}
if n != int(info.Size()) {
t.Errorf("Expected read length to be %d, found: %d", info.Size(), n)
}
// read a second time to check f.offset and f.buf are correct
buf = make([]byte, chunkSize)
n, err = f.Read(buf)
if err != io.EOF {
t.Fatal("Failed to read a fully read file:", err)
}
if n != 0 {
t.Errorf("Expected read length to be 0, found: %d", n)
}
}

113
zipfs/fs.go Normal file
View File

@ -0,0 +1,113 @@
package zipfs
import (
"archive/zip"
"os"
"path/filepath"
"syscall"
"time"
"github.com/spf13/afero"
)
type Fs struct {
r *zip.Reader
files map[string]map[string]*zip.File
}
func splitpath(name string) (dir, file string) {
name = filepath.ToSlash(name)
if len(name) == 0 || name[0] != '/' {
name = "/" + name
}
name = filepath.Clean(name)
dir, file = filepath.Split(name)
dir = filepath.Clean(dir)
return
}
func New(r *zip.Reader) afero.Fs {
fs := &Fs{r: r, files: make(map[string]map[string]*zip.File)}
for _, file := range r.File {
d, f := splitpath(file.Name)
if _, ok := fs.files[d]; !ok {
fs.files[d] = make(map[string]*zip.File)
}
if _, ok := fs.files[d][f]; !ok {
fs.files[d][f] = file
}
if file.FileInfo().IsDir() {
dirname := filepath.Join(d, f)
if _, ok := fs.files[dirname]; !ok {
fs.files[dirname] = make(map[string]*zip.File)
}
}
}
return fs
}
func (fs *Fs) Create(name string) (afero.File, error) { return nil, syscall.EPERM }
func (fs *Fs) Mkdir(name string, perm os.FileMode) error { return syscall.EPERM }
func (fs *Fs) MkdirAll(path string, perm os.FileMode) error { return syscall.EPERM }
func (fs *Fs) Open(name string) (afero.File, error) {
d, f := splitpath(name)
if f == "" {
return &File{fs: fs, isdir: true}, nil
}
if _, ok := fs.files[d]; !ok {
return nil, &os.PathError{Op: "stat", Path: name, Err: syscall.ENOENT}
}
file, ok := fs.files[d][f]
if !ok {
return nil, &os.PathError{Op: "stat", Path: name, Err: syscall.ENOENT}
}
return &File{fs: fs, zipfile: file, isdir: file.FileInfo().IsDir()}, nil
}
func (fs *Fs) OpenFile(name string, flag int, perm os.FileMode) (afero.File, error) {
if flag != os.O_RDONLY {
return nil, syscall.EPERM
}
return fs.Open(name)
}
func (fs *Fs) Remove(name string) error { return syscall.EPERM }
func (fs *Fs) RemoveAll(path string) error { return syscall.EPERM }
func (fs *Fs) Rename(oldname, newname string) error { return syscall.EPERM }
type pseudoRoot struct{}
func (p *pseudoRoot) Name() string { return string(filepath.Separator) }
func (p *pseudoRoot) Size() int64 { return 0 }
func (p *pseudoRoot) Mode() os.FileMode { return os.ModeDir | os.ModePerm }
func (p *pseudoRoot) ModTime() time.Time { return time.Now() }
func (p *pseudoRoot) IsDir() bool { return true }
func (p *pseudoRoot) Sys() interface{} { return nil }
func (fs *Fs) Stat(name string) (os.FileInfo, error) {
d, f := splitpath(name)
if f == "" {
return &pseudoRoot{}, nil
}
if _, ok := fs.files[d]; !ok {
return nil, &os.PathError{Op: "stat", Path: name, Err: syscall.ENOENT}
}
file, ok := fs.files[d][f]
if !ok {
return nil, &os.PathError{Op: "stat", Path: name, Err: syscall.ENOENT}
}
return file.FileInfo(), nil
}
func (fs *Fs) Name() string { return "zipfs" }
func (fs *Fs) Chmod(name string, mode os.FileMode) error { return syscall.EPERM }
func (fs *Fs) Chown(name string, uid, gid int) error { return syscall.EPERM }
func (fs *Fs) Chtimes(name string, atime time.Time, mtime time.Time) error { return syscall.EPERM }

BIN
zipfs/testdata/small.zip vendored Normal file
View File

Binary file not shown.

BIN
zipfs/testdata/t.zip vendored Normal file
View File

Binary file not shown.

103
zipfs/zipfs_test.go Normal file
View File

@ -0,0 +1,103 @@
package zipfs
import (
"github.com/spf13/afero"
"archive/zip"
"path/filepath"
"reflect"
"testing"
)
func TestZipFS(t *testing.T) {
zrc, err := zip.OpenReader("testdata/t.zip")
if err != nil {
t.Fatal(err)
}
zfs := New(&zrc.Reader)
a := &afero.Afero{Fs: zfs}
buf, err := a.ReadFile("testFile")
if err != nil {
t.Error(err)
}
if len(buf) != 8192 {
t.Errorf("short read: %d != 8192", len(buf))
}
buf = make([]byte, 8)
f, err := a.Open("testFile")
if err != nil {
t.Error(err)
}
if n, err := f.ReadAt(buf, 4092); err != nil {
t.Error(err)
} else if n != 8 {
t.Errorf("expected to read 8 bytes, got %d", n)
} else if string(buf) != "aaaabbbb" {
t.Errorf("expected to get <aaaabbbb>, got <%s>", string(buf))
}
d, err := a.Open("/")
if d == nil {
t.Error(`Open("/") returns nil`)
}
if err != nil {
t.Errorf(`Open("/"): err = %v`, err)
}
if s, _ := d.Stat(); !s.IsDir() {
t.Error(`expected root ("/") to be a directory`)
}
if n := d.Name(); n != string(filepath.Separator) {
t.Errorf("Wrong Name() of root directory: Expected: '%c', got '%s'", filepath.Separator, n)
}
buf = make([]byte, 8192)
if n, err := f.Read(buf); err != nil {
t.Error(err)
} else if n != 8192 {
t.Errorf("expected to read 8192 bytes, got %d", n)
} else if buf[4095] != 'a' || buf[4096] != 'b' {
t.Error("got wrong contents")
}
for _, s := range []struct {
path string
dir bool
}{
{"/", true},
{"testDir1", true},
{"testDir1/testFile", false},
{"testFile", false},
{"sub", true},
{"sub/testDir2", true},
{"sub/testDir2/testFile", false},
} {
if dir, _ := a.IsDir(s.path); dir == s.dir {
t.Logf("%s: directory check ok", s.path)
} else {
t.Errorf("%s: directory check NOT ok: %t, expected %t", s.path, dir, s.dir)
}
}
for _, s := range []struct {
glob string
entries []string
}{
{filepath.FromSlash("/*"), []string{filepath.FromSlash("/sub"), filepath.FromSlash("/testDir1"), filepath.FromSlash("/testFile")}},
{filepath.FromSlash("*"), []string{filepath.FromSlash("sub"), filepath.FromSlash("testDir1"), filepath.FromSlash("testFile")}},
{filepath.FromSlash("sub/*"), []string{filepath.FromSlash("sub/testDir2")}},
{filepath.FromSlash("sub/testDir2/*"), []string{filepath.FromSlash("sub/testDir2/testFile")}},
{filepath.FromSlash("testDir1/*"), []string{filepath.FromSlash("testDir1/testFile")}},
} {
entries, err := afero.Glob(zfs, s.glob)
if err != nil {
t.Error(err)
}
if reflect.DeepEqual(entries, s.entries) {
t.Logf("glob: %s: glob ok", s.glob)
} else {
t.Errorf("glob: %s: got %#v, expected %#v", s.glob, entries, s.entries)
}
}
}