From 4f00b0640091abe9ff896b3f3fd4d58fdb0f3e8b Mon Sep 17 00:00:00 2001 From: Hilko Bengen Date: Wed, 15 Nov 2017 16:50:29 +0100 Subject: [PATCH 1/5] Add zipfs, an archive/zip-based read-only filesystem --- README.md | 1 - zipfs/file.go | 157 +++++++++++++++++++++++++++++++++++++++++++ zipfs/fs.go | 111 ++++++++++++++++++++++++++++++ zipfs/testdata/t.zip | Bin 0 -> 1015 bytes zipfs/zipfs_test.go | 88 ++++++++++++++++++++++++ 5 files changed, 356 insertions(+), 1 deletion(-) create mode 100644 zipfs/file.go create mode 100644 zipfs/fs.go create mode 100644 zipfs/testdata/t.zip create mode 100644 zipfs/zipfs_test.go diff --git a/README.md b/README.md index 80e9927..f1793fb 100644 --- a/README.md +++ b/README.md @@ -380,7 +380,6 @@ The following is a short list of possible backends we hope someone will implement: * SSH -* ZIP * TAR * S3 diff --git a/zipfs/file.go b/zipfs/file.go new file mode 100644 index 0000000..2841227 --- /dev/null +++ b/zipfs/file.go @@ -0,0 +1,157 @@ +package zipfs + +import ( + "archive/zip" + "io" + "os" + "path/filepath" + "syscall" + + "github.com/spf13/afero" +) + +type File struct { + fs *Fs + zipfile *zip.File + reader io.ReadCloser + offset int64 + isdir, closed bool + buf []byte +} + +func (f *File) fillBuffer(offset int64) (err error) { + if f.reader == nil { + if f.reader, err = f.zipfile.Open(); err != nil { + return + } + } + if offset > int64(f.zipfile.UncompressedSize64) { + offset = int64(f.zipfile.UncompressedSize64) + err = io.EOF + } + if len(f.buf) >= int(offset) { + return + } + buf := make([]byte, int(offset)-len(f.buf)) + n, _ := io.ReadFull(f.reader, buf) + if n > 0 { + f.buf = append(f.buf, buf[:n]...) + } + return +} + +func (f *File) Close() (err error) { + f.zipfile = nil + f.closed = true + f.buf = nil + if f.reader != nil { + err = f.reader.Close() + f.reader = nil + } + return +} + +func (f *File) Read(p []byte) (n int, err error) { + if f.isdir { + return 0, syscall.EISDIR + } + if f.closed { + return 0, afero.ErrFileClosed + } + err = f.fillBuffer(f.offset + int64(len(p))) + n = copy(p, f.buf[f.offset:]) + f.offset += int64(len(p)) + return +} + +func (f *File) ReadAt(p []byte, off int64) (n int, err error) { + if f.isdir { + return 0, syscall.EISDIR + } + if f.closed { + return 0, afero.ErrFileClosed + } + err = f.fillBuffer(off + int64(len(p))) + n = copy(p, f.buf[int(off):]) + return +} + +func (f *File) Seek(offset int64, whence int) (int64, error) { + if f.isdir { + return 0, syscall.EISDIR + } + if f.closed { + return 0, afero.ErrFileClosed + } + switch whence { + case os.SEEK_SET: + case os.SEEK_CUR: + offset += f.offset + case os.SEEK_END: + offset += int64(f.zipfile.UncompressedSize64) + default: + return 0, syscall.EINVAL + } + if offset < 0 || offset > int64(f.zipfile.UncompressedSize64) { + return 0, afero.ErrOutOfRange + } + f.offset = offset + return offset, nil +} + +func (f *File) Write(p []byte) (n int, err error) { return 0, syscall.EPERM } + +func (f *File) WriteAt(p []byte, off int64) (n int, err error) { return 0, syscall.EPERM } + +func (f *File) Name() string { return f.zipfile.Name } + +func (f *File) getDirEntries() (map[string]*zip.File, error) { + if !f.isdir { + return nil, syscall.ENOTDIR + } + name := string(filepath.Separator) + if f.zipfile != nil { + name = filepath.Join(splitpath(f.zipfile.Name)) + } + entries, ok := f.fs.files[name] + if !ok { + return nil, &os.PathError{Op: "readdir", Path: name, Err: syscall.ENOENT} + } + return entries, nil +} + +func (f *File) Readdir(count int) (fi []os.FileInfo, err error) { + zipfiles, err := f.getDirEntries() + if err != nil { + return nil, err + } + for _, zipfile := range zipfiles { + fi = append(fi, zipfile.FileInfo()) + if count >= 0 && len(fi) >= count { + break + } + } + return +} + +func (f *File) Readdirnames(count int) (names []string, err error) { + zipfiles, err := f.getDirEntries() + if err != nil { + return nil, err + } + for filename, _ := range zipfiles { + names = append(names, filename) + if count >= 0 && len(names) >= count { + break + } + } + return +} + +func (f *File) Stat() (os.FileInfo, error) { return f.zipfile.FileInfo(), nil } + +func (f *File) Sync() error { return nil } + +func (f *File) Truncate(size int64) error { return syscall.EPERM } + +func (f *File) WriteString(s string) (ret int, err error) { return 0, syscall.EPERM } diff --git a/zipfs/fs.go b/zipfs/fs.go new file mode 100644 index 0000000..72887f9 --- /dev/null +++ b/zipfs/fs.go @@ -0,0 +1,111 @@ +package zipfs + +import ( + "archive/zip" + "os" + "path/filepath" + "syscall" + "time" + + "github.com/spf13/afero" +) + +type Fs struct { + r *zip.Reader + files map[string]map[string]*zip.File +} + +func splitpath(name string) (dir, file string) { + name = filepath.ToSlash(name) + if len(name) == 0 || name[0] != '/' { + name = "/" + name + } + name = filepath.Clean(name) + dir, file = filepath.Split(name) + dir = filepath.Clean(dir) + return +} + +func New(r *zip.Reader) afero.Fs { + fs := &Fs{r: r, files: make(map[string]map[string]*zip.File)} + for _, file := range r.File { + d, f := splitpath(file.Name) + if _, ok := fs.files[d]; !ok { + fs.files[d] = make(map[string]*zip.File) + } + if _, ok := fs.files[d][f]; !ok { + fs.files[d][f] = file + } + if file.FileInfo().IsDir() { + dirname := filepath.Join(d, f) + if _, ok := fs.files[dirname]; !ok { + fs.files[dirname] = make(map[string]*zip.File) + } + } + } + return fs +} + +func (fs *Fs) Create(name string) (afero.File, error) { return nil, syscall.EPERM } + +func (fs *Fs) Mkdir(name string, perm os.FileMode) error { return syscall.EPERM } + +func (fs *Fs) MkdirAll(path string, perm os.FileMode) error { return syscall.EPERM } + +func (fs *Fs) Open(name string) (afero.File, error) { + d, f := splitpath(name) + if f == "" { + return &File{fs: fs, isdir: true}, nil + } + if _, ok := fs.files[d]; !ok { + return nil, &os.PathError{Op: "stat", Path: name, Err: syscall.ENOENT} + } + file, ok := fs.files[d][f] + if !ok { + return nil, &os.PathError{Op: "stat", Path: name, Err: syscall.ENOENT} + } + return &File{fs: fs, zipfile: file, isdir: file.FileInfo().IsDir()}, nil +} + +func (fs *Fs) OpenFile(name string, flag int, perm os.FileMode) (afero.File, error) { + if flag != os.O_RDONLY { + return nil, syscall.EPERM + } + return fs.Open(name) +} + +func (fs *Fs) Remove(name string) error { return syscall.EPERM } + +func (fs *Fs) RemoveAll(path string) error { return syscall.EPERM } + +func (fs *Fs) Rename(oldname, newname string) error { return syscall.EPERM } + +type pseudoRoot struct{} + +func (p *pseudoRoot) Name() string { return string(filepath.Separator) } +func (p *pseudoRoot) Size() int64 { return 0 } +func (p *pseudoRoot) Mode() os.FileMode { return os.ModeDir | os.ModePerm } +func (p *pseudoRoot) ModTime() time.Time { return time.Now() } +func (p *pseudoRoot) IsDir() bool { return true } +func (p *pseudoRoot) Sys() interface{} { return nil } + +func (fs *Fs) Stat(name string) (os.FileInfo, error) { + d, f := splitpath(name) + if f == "" { + return &pseudoRoot{}, nil + } + if _, ok := fs.files[d]; !ok { + return nil, &os.PathError{Op: "stat", Path: name, Err: syscall.ENOENT} + } + file, ok := fs.files[d][f] + if !ok { + return nil, &os.PathError{Op: "stat", Path: name, Err: syscall.ENOENT} + } + return file.FileInfo(), nil +} + +func (fs *Fs) Name() string { return "zipfs" } + +func (fs *Fs) Chmod(name string, mode os.FileMode) error { return syscall.EPERM } + +func (fs *Fs) Chtimes(name string, atime time.Time, mtime time.Time) error { return syscall.EPERM } diff --git a/zipfs/testdata/t.zip b/zipfs/testdata/t.zip new file mode 100644 index 0000000000000000000000000000000000000000..99e39a7d2dd910afa2cdad03e4bc99aa108a8589 GIT binary patch literal 1015 zcmWIWW@h1H0E6;;ZwLcIvM|Un6qhFHhlX%6Fc-MX$3a}Bg+eB1_m$@0M~6$ zfvTGqs=Fk$xWpy1$OvN09uJ-<5XNGR2+&pz1~H%`*q+5DNrT*=z`!7iZVbpgx6GVW zu&ZVRU9=U`xVHxxIf2@l7Hs@l?`-en3uHHxvjWL?ya)$tqdAxpXcN@2h7iYY0-6lM zI33Fabgb+ek?AnU3L+bU;?Q=WLqQnLD7Zt{fBlcep^zYHD9-mj>l1JRW(x<@7I-M! zci@QvVO)l<{aWvW9tw<1a?H4*T>=, got <%s>", string(buf)) + } + + buf = make([]byte, 8192) + if n, err := f.Read(buf); err != nil { + t.Error(err) + } else if n != 8192 { + t.Errorf("expected to read 8192 bytes, got %d", n) + } else if buf[4095] != 'a' || buf[4096] != 'b' { + t.Error("got wrong contents") + } + + for _, s := range []struct { + path string + dir bool + }{ + {"testDir1", true}, + {"testDir1/testFile", false}, + {"testFile", false}, + {"sub", true}, + {"sub/testDir2", true}, + {"sub/testDir2/testFile", false}, + } { + if dir, _ := a.IsDir(s.path); dir == s.dir { + t.Logf("%s: directory check ok", s.path) + } else { + t.Errorf("%s: directory check NOT ok: %t, expected %t", s.path, dir, s.dir) + } + } + + for _, s := range []struct { + glob string + entries []string + }{ + {filepath.FromSlash("/*"), []string{filepath.FromSlash("/sub"), filepath.FromSlash("/testDir1"), filepath.FromSlash("/testFile")}}, + {filepath.FromSlash("*"), []string{filepath.FromSlash("sub"), filepath.FromSlash("testDir1"), filepath.FromSlash("testFile")}}, + {filepath.FromSlash("sub/*"), []string{filepath.FromSlash("sub/testDir2")}}, + {filepath.FromSlash("sub/testDir2/*"), []string{filepath.FromSlash("sub/testDir2/testFile")}}, + {filepath.FromSlash("testDir1/*"), []string{filepath.FromSlash("testDir1/testFile")}}, + } { + entries, err := afero.Glob(zfs, s.glob) + if err != nil { + t.Error(err) + } + if reflect.DeepEqual(entries, s.entries) { + t.Logf("glob: %s: glob ok", s.glob) + } else { + t.Errorf("glob: %s: got %#v, expected %#v", s.glob, entries, s.entries) + } + } +} From d5bfeca89b128cf5061264419543ee1c120e0173 Mon Sep 17 00:00:00 2001 From: Hilko Bengen Date: Mon, 11 Mar 2019 19:00:04 +0100 Subject: [PATCH 2/5] Work around root directory Open/Stat corner case leading to panic See https://github.com/spf13/afero/pull/146#issuecomment-470840725 --- zipfs/file.go | 7 ++++++- zipfs/zipfs_test.go | 12 ++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/zipfs/file.go b/zipfs/file.go index 2841227..36c4ce9 100644 --- a/zipfs/file.go +++ b/zipfs/file.go @@ -148,7 +148,12 @@ func (f *File) Readdirnames(count int) (names []string, err error) { return } -func (f *File) Stat() (os.FileInfo, error) { return f.zipfile.FileInfo(), nil } +func (f *File) Stat() (os.FileInfo, error) { + if f.zipfile == nil { + return &pseudoRoot{}, nil + } + return f.zipfile.FileInfo(), nil +} func (f *File) Sync() error { return nil } diff --git a/zipfs/zipfs_test.go b/zipfs/zipfs_test.go index 02ac6b6..54db4aa 100644 --- a/zipfs/zipfs_test.go +++ b/zipfs/zipfs_test.go @@ -38,6 +38,17 @@ func TestZipFS(t *testing.T) { t.Errorf("expected to get , got <%s>", string(buf)) } + d, err := a.Open("/") + if d == nil { + t.Error(`Open("/") returns nil`) + } + if err != nil { + t.Errorf(`Open("/"): err = %v`, err) + } + if s, _ := d.Stat(); !s.IsDir() { + t.Error(`expected root ("/") to be a directory`) + } + buf = make([]byte, 8192) if n, err := f.Read(buf); err != nil { t.Error(err) @@ -51,6 +62,7 @@ func TestZipFS(t *testing.T) { path string dir bool }{ + {"/", true}, {"testDir1", true}, {"testDir1/testFile", false}, {"testFile", false}, From 72cabd552e74467703e3f90b84a48939a7bae1c7 Mon Sep 17 00:00:00 2001 From: Hilko Bengen Date: Mon, 11 Mar 2019 19:14:41 +0100 Subject: [PATCH 3/5] Eliminate another root-path-related corner case --- zipfs/file.go | 12 +++++++----- zipfs/zipfs_test.go | 3 +++ 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/zipfs/file.go b/zipfs/file.go index 36c4ce9..a81ea06 100644 --- a/zipfs/file.go +++ b/zipfs/file.go @@ -103,16 +103,18 @@ func (f *File) Write(p []byte) (n int, err error) { return 0, syscall.EPERM } func (f *File) WriteAt(p []byte, off int64) (n int, err error) { return 0, syscall.EPERM } -func (f *File) Name() string { return f.zipfile.Name } +func (f *File) Name() string { + if f.zipfile == nil { + return string(filepath.Separator) + } + return filepath.Join(splitpath(f.zipfile.Name)) +} func (f *File) getDirEntries() (map[string]*zip.File, error) { if !f.isdir { return nil, syscall.ENOTDIR } - name := string(filepath.Separator) - if f.zipfile != nil { - name = filepath.Join(splitpath(f.zipfile.Name)) - } + name := f.Name() entries, ok := f.fs.files[name] if !ok { return nil, &os.PathError{Op: "readdir", Path: name, Err: syscall.ENOENT} diff --git a/zipfs/zipfs_test.go b/zipfs/zipfs_test.go index 54db4aa..7a141cb 100644 --- a/zipfs/zipfs_test.go +++ b/zipfs/zipfs_test.go @@ -48,6 +48,9 @@ func TestZipFS(t *testing.T) { if s, _ := d.Stat(); !s.IsDir() { t.Error(`expected root ("/") to be a directory`) } + if n := d.Name(); n != "/" { + t.Errorf("Wrong Name() of root directory: Expected: '/', got '%s'", n) + } buf = make([]byte, 8192) if n, err := f.Read(buf); err != nil { From 9b520d08216160cf4ac57820fc3f3a2c6634916f Mon Sep 17 00:00:00 2001 From: Hilko Bengen Date: Mon, 11 Mar 2019 20:20:35 +0100 Subject: [PATCH 4/5] Use filepath.Separator in test code, to fix tests on Windows --- zipfs/zipfs_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zipfs/zipfs_test.go b/zipfs/zipfs_test.go index 7a141cb..bf781cc 100644 --- a/zipfs/zipfs_test.go +++ b/zipfs/zipfs_test.go @@ -48,8 +48,8 @@ func TestZipFS(t *testing.T) { if s, _ := d.Stat(); !s.IsDir() { t.Error(`expected root ("/") to be a directory`) } - if n := d.Name(); n != "/" { - t.Errorf("Wrong Name() of root directory: Expected: '/', got '%s'", n) + if n := d.Name(); n != string(filepath.Separator) { + t.Errorf("Wrong Name() of root directory: Expected: '%c', got '%s'", filepath.Separator, n) } buf = make([]byte, 8192) From 344ad9d197b9a029919060b3a66b8a191d9a69a6 Mon Sep 17 00:00:00 2001 From: Hilko Bengen Date: Mon, 30 Mar 2020 15:28:04 +0200 Subject: [PATCH 5/5] Don't hide errors while reading from ZIP files --- zipfs/file.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/zipfs/file.go b/zipfs/file.go index a81ea06..94f9ddc 100644 --- a/zipfs/file.go +++ b/zipfs/file.go @@ -33,9 +33,10 @@ func (f *File) fillBuffer(offset int64) (err error) { return } buf := make([]byte, int(offset)-len(f.buf)) - n, _ := io.ReadFull(f.reader, buf) - if n > 0 { + if n, readErr := io.ReadFull(f.reader, buf); n > 0 { f.buf = append(f.buf, buf[:n]...) + } else if readErr != nil { + err = readErr } return }