From 90555c18942059e61125b32f572e41a8a8ebc02f Mon Sep 17 00:00:00 2001 From: Steve Francia Date: Sat, 5 Dec 2015 11:43:38 -0500 Subject: [PATCH] reorganize utils into afero package --- fs.go => afero.go | 110 +------------------------- fs_test.go => afero_test.go | 40 ---------- util/ioutil.go => ioutil.go | 26 +++--- util/ioutil_test.go => ioutil_test.go | 22 +++--- path.go | 108 +++++++++++++++++++++++++ path_test.go | 74 +++++++++++++++++ util/util.go => util.go | 40 ++++------ util/util_test.go => util_test.go | 20 +++-- 8 files changed, 233 insertions(+), 207 deletions(-) rename fs.go => afero.go (54%) rename fs_test.go => afero_test.go (94%) rename util/ioutil.go => ioutil.go (89%) rename util/ioutil_test.go => ioutil_test.go (89%) create mode 100644 path.go create mode 100644 path_test.go rename util/util.go => util.go (82%) rename util/util_test.go => util_test.go (96%) diff --git a/fs.go b/afero.go similarity index 54% rename from fs.go rename to afero.go index 380b3ef..9e5fa3c 100644 --- a/fs.go +++ b/afero.go @@ -26,11 +26,13 @@ import ( "errors" "io" "os" - "path/filepath" - "sort" "time" ) +type Afero struct { + fs Fs +} + // File represents a file in the filesystem. type File interface { io.Closer @@ -104,107 +106,3 @@ var ( ErrFileExists = os.ErrExist ErrDestinationExists = os.ErrExist ) - -// ReadDir reads the directory named by dirname and returns -// a list of sorted directory entries. -func ReadDir(dirname string, fs Fs) ([]os.FileInfo, error) { - f, err := fs.Open(dirname) - if err != nil { - return nil, err - } - list, err := f.Readdir(-1) - f.Close() - if err != nil { - return nil, err - } - sort.Sort(byName(list)) - return list, nil -} - -// readDirNames reads the directory named by dirname and returns -// a sorted list of directory entries. -// adapted from https://golang.org/src/path/filepath/path.go -func readDirNames(dirname string, fs Fs) ([]string, error) { - f, err := fs.Open(dirname) - if err != nil { - return nil, err - } - names, err := f.Readdirnames(-1) - f.Close() - if err != nil { - return nil, err - } - sort.Strings(names) - return names, nil -} - -// walk recursively descends path, calling walkFn -// adapted from https://golang.org/src/path/filepath/path.go -func walk(path string, info os.FileInfo, walkFn filepath.WalkFunc, fs Fs) error { - err := walkFn(path, info, nil) - if err != nil { - if info.IsDir() && err == filepath.SkipDir { - return nil - } - return err - } - - if !info.IsDir() { - return nil - } - - names, err := readDirNames(path, fs) - if err != nil { - return walkFn(path, info, err) - } - - for _, name := range names { - filename := filepath.Join(path, name) - fileInfo, err := lstatIfOs(filename, fs) - if err != nil { - if err := walkFn(filename, fileInfo, err); err != nil && err != filepath.SkipDir { - return err - } - } else { - err = walk(filename, fileInfo, walkFn, fs) - if err != nil { - if !fileInfo.IsDir() || err != filepath.SkipDir { - return err - } - } - } - } - return nil -} - -// if the filesystem is OsFs use Lstat, else use fs.Stat -func lstatIfOs(path string, fs Fs) (info os.FileInfo, err error) { - _, ok := fs.(*OsFs) - if ok { - info, err = os.Lstat(path) - } else { - info, err = fs.Stat(path) - } - return -} - -// Walk walks the file tree rooted at root, calling walkFn for each file or -// directory in the tree, including root. All errors that arise visiting files -// and directories are filtered by walkFn. The files are walked in lexical -// order, which makes the output deterministic but means that for very -// large directories Walk can be inefficient. -// Walk does not follow symbolic links. -func Walk(root string, walkFn filepath.WalkFunc, fs Fs) error { - info, err := lstatIfOs(root, fs) - if err != nil { - return walkFn(root, nil, err) - } - return walk(root, info, walkFn, fs) -} - -// byName implements sort.Interface. -type byName []os.FileInfo - -func (f byName) Len() int { return len(f) } -func (f byName) Less(i, j int) bool { return f[i].Name() < f[j].Name() } -func (f byName) Swap(i, j int) { f[i], f[j] = f[j], f[i] } diff --git a/fs_test.go b/afero_test.go similarity index 94% rename from fs_test.go rename to afero_test.go index b96de20..6f925cb 100644 --- a/fs_test.go +++ b/afero_test.go @@ -458,46 +458,6 @@ func (m myFileInfo) String() string { return out } -func TestWalk(t *testing.T) { - outputs := make([]string, len(Fss)) - for i, fs := range Fss { - walkFn := func(path string, info os.FileInfo, err error) error { - if err != nil { - t.Error("walkFn error:", err) - return nil - } - var size int64 - if !info.IsDir() { - size = info.Size() - } - outputs[i] += fmt.Sprintln(path, info.Name(), size, info.IsDir(), err) - return nil - } - err := Walk(testDir, walkFn, fs) - if err != nil { - t.Error(err) - } - } - fail := false - for i, o := range outputs { - if i == 0 { - continue - } - if o != outputs[i-1] { - fail = true - break - } - } - if fail { - t.Log("Walk outputs not equal!") - for i, o := range outputs { - t.Log(Fss[i].Name()) - t.Log(o) - } - t.Fail() - } -} - func TestReaddirAll(t *testing.T) { defer removeTestDir(t) for _, fs := range Fss { diff --git a/util/ioutil.go b/ioutil.go similarity index 89% rename from util/ioutil.go rename to ioutil.go index ada8b12..2d3c68b 100644 --- a/util/ioutil.go +++ b/ioutil.go @@ -13,7 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package util +package afero import ( "bytes" @@ -24,8 +24,6 @@ import ( "strconv" "sync" "time" - - "github.com/spf13/afero" ) // byName implements sort.Interface. @@ -37,11 +35,11 @@ func (f byName) Swap(i, j int) { f[i], f[j] = f[j], f[i] } // ReadDir reads the directory named by dirname and returns // a list of sorted directory entries. -func (a AferoUtilFS) ReadDir(dirname string) ([]os.FileInfo, error) { +func (a Afero) ReadDir(dirname string) ([]os.FileInfo, error) { return ReadDir(dirname, a.fs) } -func ReadDir(dirname string, fs afero.Fs) ([]os.FileInfo, error) { +func ReadDir(dirname string, fs Fs) ([]os.FileInfo, error) { f, err := fs.Open(dirname) if err != nil { return nil, err @@ -59,11 +57,11 @@ func ReadDir(dirname string, fs afero.Fs) ([]os.FileInfo, error) { // A successful call returns err == nil, not err == EOF. Because ReadFile // reads the whole file, it does not treat an EOF from Read as an error // to be reported. -func (a AferoUtilFS) ReadFile(filename string) ([]byte, error) { +func (a Afero) ReadFile(filename string) ([]byte, error) { return ReadFile(filename, a.fs) } -func ReadFile(filename string, fs afero.Fs) ([]byte, error) { +func ReadFile(filename string, fs Fs) ([]byte, error) { f, err := fs.Open(filename) if err != nil { return nil, err @@ -119,11 +117,11 @@ func ReadAll(r io.Reader) ([]byte, error) { // WriteFile writes data to a file named by filename. // If the file does not exist, WriteFile creates it with permissions perm; // otherwise WriteFile truncates it before writing. -func (a AferoUtilFS) WriteFile(filename string, data []byte, perm os.FileMode) error { +func (a Afero) WriteFile(filename string, data []byte, perm os.FileMode) error { return WriteFile(filename, data, perm, a.fs) } -func WriteFile(filename string, data []byte, perm os.FileMode, fs afero.Fs) error { +func WriteFile(filename string, data []byte, perm os.FileMode, fs Fs) error { f, err := fs.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm) if err != nil { return err @@ -163,18 +161,18 @@ func nextSuffix() string { // TempFile creates a new temporary file in the directory dir // with a name beginning with prefix, opens the file for reading -// and writing, and returns the resulting *afero.File. +// and writing, and returns the resulting *File. // If dir is the empty string, TempFile uses the default directory // for temporary files (see os.TempDir). // Multiple programs calling TempFile simultaneously // will not choose the same file. The caller can use f.Name() // to find the pathname of the file. It is the caller's responsibility // to remove the file when no longer needed. -func (a AferoUtilFS) TempFile(dir, prefix string) (f afero.File, err error) { +func (a Afero) TempFile(dir, prefix string) (f File, err error) { return TempFile(dir, prefix, a.fs) } -func TempFile(dir, prefix string, fs afero.Fs) (f afero.File, err error) { +func TempFile(dir, prefix string, fs Fs) (f File, err error) { if dir == "" { dir = os.TempDir() } @@ -203,10 +201,10 @@ func TempFile(dir, prefix string, fs afero.Fs) (f afero.File, err error) { // Multiple programs calling TempDir simultaneously // will not choose the same directory. It is the caller's responsibility // to remove the directory when no longer needed. -func (a AferoUtilFS) TempDir(dir, prefix string) (name string, err error) { +func (a Afero) TempDir(dir, prefix string) (name string, err error) { return TempDir(dir, prefix, a.fs) } -func TempDir(dir, prefix string, fs afero.Fs) (name string, err error) { +func TempDir(dir, prefix string, fs Fs) (name string, err error) { if dir == "" { dir = os.TempDir() } diff --git a/util/ioutil_test.go b/ioutil_test.go similarity index 89% rename from util/ioutil_test.go rename to ioutil_test.go index ab2efbf..ed05977 100644 --- a/util/ioutil_test.go +++ b/ioutil_test.go @@ -13,15 +13,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -package util +package afero -import ( - "testing" +import "testing" - "github.com/spf13/afero" -) - -func checkSize(t *testing.T, path string, size int64) { +func checkSizePath(t *testing.T, path string, size int64) { dir, err := testFS.Stat(path) if err != nil { t.Fatalf("Stat %q (looking for size %d): %s", path, size, err) @@ -32,8 +28,8 @@ func checkSize(t *testing.T, path string, size int64) { } func TestReadFile(t *testing.T) { - testFS = &afero.MemMapFs{} - fsutil := &AferoUtilFS{fs: testFS} + testFS = &MemMapFs{} + fsutil := &Afero{fs: testFS} testFS.Create("this_exists.go") filename := "rumpelstilzchen" @@ -48,12 +44,12 @@ func TestReadFile(t *testing.T) { t.Fatalf("ReadFile %s: %v", filename, err) } - checkSize(t, filename, int64(len(contents))) + checkSizePath(t, filename, int64(len(contents))) } func TestWriteFile(t *testing.T) { - testFS = &afero.MemMapFs{} - fsutil := &AferoUtilFS{fs: testFS} + testFS = &MemMapFs{} + fsutil := &Afero{fs: testFS} f, err := fsutil.TempFile("", "ioutil-test") if err != nil { t.Fatal(err) @@ -82,7 +78,7 @@ func TestWriteFile(t *testing.T) { } func TestReadDir(t *testing.T) { - testFS = &afero.MemMapFs{} + testFS = &MemMapFs{} testFS.Mkdir("/i-am-a-dir", 0777) testFS.Create("/this_exists.go") dirname := "rumpelstilzchen" diff --git a/path.go b/path.go new file mode 100644 index 0000000..6ae81eb --- /dev/null +++ b/path.go @@ -0,0 +1,108 @@ +// Copyright ©2015 The Go Authors +// Copyright ©2015 Steve Francia +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package afero + +import ( + "os" + "path/filepath" + "sort" +) + +// readDirNames reads the directory named by dirname and returns +// a sorted list of directory entries. +// adapted from https://golang.org/src/path/filepath/path.go +func readDirNames(dirname string, fs Fs) ([]string, error) { + f, err := fs.Open(dirname) + if err != nil { + return nil, err + } + names, err := f.Readdirnames(-1) + f.Close() + if err != nil { + return nil, err + } + sort.Strings(names) + return names, nil +} + +// walk recursively descends path, calling walkFn +// adapted from https://golang.org/src/path/filepath/path.go +func walk(path string, info os.FileInfo, walkFn filepath.WalkFunc, fs Fs) error { + err := walkFn(path, info, nil) + if err != nil { + if info.IsDir() && err == filepath.SkipDir { + return nil + } + return err + } + + if !info.IsDir() { + return nil + } + + names, err := readDirNames(path, fs) + if err != nil { + return walkFn(path, info, err) + } + + for _, name := range names { + filename := filepath.Join(path, name) + fileInfo, err := lstatIfOs(filename, fs) + if err != nil { + if err := walkFn(filename, fileInfo, err); err != nil && err != filepath.SkipDir { + return err + } + } else { + err = walk(filename, fileInfo, walkFn, fs) + if err != nil { + if !fileInfo.IsDir() || err != filepath.SkipDir { + return err + } + } + } + } + return nil +} + +// if the filesystem is OsFs use Lstat, else use fs.Stat +func lstatIfOs(path string, fs Fs) (info os.FileInfo, err error) { + _, ok := fs.(*OsFs) + if ok { + info, err = os.Lstat(path) + } else { + info, err = fs.Stat(path) + } + return +} + +// Walk walks the file tree rooted at root, calling walkFn for each file or +// directory in the tree, including root. All errors that arise visiting files +// and directories are filtered by walkFn. The files are walked in lexical +// order, which makes the output deterministic but means that for very +// large directories Walk can be inefficient. +// Walk does not follow symbolic links. + +func (a Afero) Walk(root string, walkFn filepath.WalkFunc) error { + return Walk(root, walkFn, a.fs) +} + +func Walk(root string, walkFn filepath.WalkFunc, fs Fs) error { + info, err := lstatIfOs(root, fs) + if err != nil { + return walkFn(root, nil, err) + } + return walk(root, info, walkFn, fs) +} diff --git a/path_test.go b/path_test.go new file mode 100644 index 0000000..3c62525 --- /dev/null +++ b/path_test.go @@ -0,0 +1,74 @@ +// Copyright © 2014 Steve Francia . +// Copyright 2009 The Go Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package afero + +import ( + "fmt" + "os" + "testing" +) + +func TestWalk(t *testing.T) { + var Fss = []Fs{&MemMapFs{}, &OsFs{}} + var testDir = "/tmp/afero" + var testName = "test.txt" + for _, fs := range Fss { + path := testDir + "/" + testName + if err := fs.MkdirAll(testDir, 0777); err != nil { + t.Fatal(fs.Name(), "unable to create dir", err) + } + + f, err := fs.Create(path) + if err != nil { + t.Fatal(fs.Name(), "create failed:", err) + } + defer f.Close() + f.WriteString("Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.") + } + + outputs := make([]string, len(Fss)) + for i, fs := range Fss { + walkFn := func(path string, info os.FileInfo, err error) error { + var size int64 + if !info.IsDir() { + size = info.Size() + } + outputs[i] += fmt.Sprintln(path, info.Name(), size, info.IsDir(), err) + return nil + } + err := Walk(testDir, walkFn, fs) + if err != nil { + t.Error(err) + } + } + fail := false + for i, o := range outputs { + if i == 0 { + continue + } + if o != outputs[i-1] { + fail = true + break + } + } + if fail { + t.Log("Walk outputs not equal!") + for i, o := range outputs { + t.Log(Fss[i].Name()) + t.Log(o) + } + t.Fail() + } +} diff --git a/util/util.go b/util.go similarity index 82% rename from util/util.go rename to util.go index a0be68c..d80f986 100644 --- a/util/util.go +++ b/util.go @@ -14,7 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package util +package afero import ( "bytes" @@ -28,23 +28,17 @@ import ( "golang.org/x/text/transform" "golang.org/x/text/unicode/norm" - - "github.com/spf13/afero" ) -type AferoUtilFS struct { - fs afero.Fs -} - // Filepath separator defined by os.Separator. const FilePathSeparator = string(filepath.Separator) // Takes a reader and a path and writes the content -func (a AferoUtilFS) WriteReader(path string, r io.Reader) (err error) { +func (a Afero) WriteReader(path string, r io.Reader) (err error) { return WriteReader(path, r, a.fs) } -func WriteReader(path string, r io.Reader, fs afero.Fs) (err error) { +func WriteReader(path string, r io.Reader, fs Fs) (err error) { dir, _ := filepath.Split(path) ospath := filepath.FromSlash(dir) @@ -68,11 +62,11 @@ func WriteReader(path string, r io.Reader, fs afero.Fs) (err error) { } // Same as WriteReader but checks to see if file/directory already exists. -func (a AferoUtilFS) SafeWriteReader(path string, r io.Reader) (err error) { +func (a Afero) SafeWriteReader(path string, r io.Reader) (err error) { return SafeWriteReader(path, r, a.fs) } -func SafeWriteReader(path string, r io.Reader, fs afero.Fs) (err error) { +func SafeWriteReader(path string, r io.Reader, fs Fs) (err error) { dir, _ := filepath.Split(path) ospath := filepath.FromSlash(dir) @@ -101,13 +95,13 @@ func SafeWriteReader(path string, r io.Reader, fs afero.Fs) (err error) { return } -func (a AferoUtilFS) GetTempDir(subPath string) string { +func (a Afero) GetTempDir(subPath string) string { return GetTempDir(subPath, a.fs) } // GetTempDir returns the default temp directory with trailing slash // if subPath is not empty then it will be created recursively with mode 777 rwx rwx rwx -func GetTempDir(subPath string, fs afero.Fs) string { +func GetTempDir(subPath string, fs Fs) string { addSlash := func(p string) string { if FilePathSeparator != p[len(p)-1:] { p = p + FilePathSeparator @@ -175,12 +169,12 @@ func isMn(r rune) bool { return unicode.Is(unicode.Mn, r) // Mn: nonspacing marks } -func (a AferoUtilFS) FileContainsBytes(filename string, subslice []byte) (bool, error) { +func (a Afero) FileContainsBytes(filename string, subslice []byte) (bool, error) { return FileContainsBytes(filename, subslice, a.fs) } // Check if a file contains a specified string. -func FileContainsBytes(filename string, subslice []byte, fs afero.Fs) (bool, error) { +func FileContainsBytes(filename string, subslice []byte, fs Fs) (bool, error) { f, err := fs.Open(filename) if err != nil { return false, err @@ -226,12 +220,12 @@ func readerContains(r io.Reader, subslice []byte) bool { return false } -func (a AferoUtilFS) DirExists(path string) (bool, error) { +func (a Afero) DirExists(path string) (bool, error) { return DirExists(path, a.fs) } // DirExists checks if a path exists and is a directory. -func DirExists(path string, fs afero.Fs) (bool, error) { +func DirExists(path string, fs Fs) (bool, error) { fi, err := fs.Stat(path) if err == nil && fi.IsDir() { return true, nil @@ -242,12 +236,12 @@ func DirExists(path string, fs afero.Fs) (bool, error) { return false, err } -func (a AferoUtilFS) IsDir(path string) (bool, error) { +func (a Afero) IsDir(path string) (bool, error) { return IsDir(path, a.fs) } // IsDir checks if a given path is a directory. -func IsDir(path string, fs afero.Fs) (bool, error) { +func IsDir(path string, fs Fs) (bool, error) { fi, err := fs.Stat(path) if err != nil { return false, err @@ -255,12 +249,12 @@ func IsDir(path string, fs afero.Fs) (bool, error) { return fi.IsDir(), nil } -func (a AferoUtilFS) IsEmpty(path string) (bool, error) { +func (a Afero) IsEmpty(path string) (bool, error) { return IsEmpty(path, a.fs) } // IsEmpty checks if a given file or directory is empty. -func IsEmpty(path string, fs afero.Fs) (bool, error) { +func IsEmpty(path string, fs Fs) (bool, error) { if b, _ := Exists(path, fs); !b { return false, fmt.Errorf("%q path does not exist", path) } @@ -280,12 +274,12 @@ func IsEmpty(path string, fs afero.Fs) (bool, error) { return fi.Size() == 0, nil } -func (a AferoUtilFS) Exists(path string) (bool, error) { +func (a Afero) Exists(path string) (bool, error) { return Exists(path, a.fs) } // Check if a file or directory exists. -func Exists(path string, fs afero.Fs) (bool, error) { +func Exists(path string, fs Fs) (bool, error) { _, err := fs.Stat(path) if err == nil { return true, nil diff --git a/util/util_test.go b/util_test.go similarity index 96% rename from util/util_test.go rename to util_test.go index 447fdf6..7ff3f84 100644 --- a/util/util_test.go +++ b/util_test.go @@ -15,7 +15,7 @@ // limitations under the License. // -package util +package afero import ( "fmt" @@ -25,11 +25,9 @@ import ( "strings" "testing" "time" - - "github.com/spf13/afero" ) -var testFS = new(afero.MemMapFs) +var testFS = new(MemMapFs) func TestDirExists(t *testing.T) { type test struct { @@ -38,7 +36,7 @@ func TestDirExists(t *testing.T) { } // First create a couple directories so there is something in the filesystem - //testFS := new(afero.MemMapFs) + //testFS := new(MemMapFs) testFS.MkdirAll("/foo/bar", 0777) data := []test{ @@ -68,7 +66,7 @@ func TestDirExists(t *testing.T) { } func TestIsDir(t *testing.T) { - testFS = new(afero.MemMapFs) + testFS = new(MemMapFs) type test struct { input string @@ -91,7 +89,7 @@ func TestIsDir(t *testing.T) { } func TestIsEmpty(t *testing.T) { - testFS = new(afero.MemMapFs) + testFS = new(MemMapFs) zeroSizedFile, _ := createZeroSizedFileInTempDir() defer deleteFileInTempDir(zeroSizedFile) @@ -141,7 +139,7 @@ func TestIsEmpty(t *testing.T) { } } -func createZeroSizedFileInTempDir() (afero.File, error) { +func createZeroSizedFileInTempDir() (File, error) { filePrefix := "_path_test_" f, e := TempFile("", filePrefix, testFS) // dir is os.TempDir() if e != nil { @@ -152,7 +150,7 @@ func createZeroSizedFileInTempDir() (afero.File, error) { return f, nil } -func createNonZeroSizedFileInTempDir() (afero.File, error) { +func createNonZeroSizedFileInTempDir() (File, error) { f, err := createZeroSizedFileInTempDir() if err != nil { // no file ?? @@ -167,7 +165,7 @@ func createNonZeroSizedFileInTempDir() (afero.File, error) { return f, nil } -func deleteFileInTempDir(f afero.File) { +func deleteFileInTempDir(f File) { err := testFS.Remove(f.Name()) if err != nil { // now what? @@ -365,7 +363,7 @@ func TestGetTempDir(t *testing.T) { } for _, test := range tests { - output := GetTempDir(test.input, new(afero.MemMapFs)) + output := GetTempDir(test.input, new(MemMapFs)) if output != test.expected { t.Errorf("Expected %#v, got %#v\n", test.expected, output) }