From 0bc61700a35431737e8309576d2da4005504f22e Mon Sep 17 00:00:00 2001 From: spf13 Date: Tue, 28 Oct 2014 10:29:28 -0400 Subject: [PATCH] afero is born --- LICENSE.txt | 174 ++++++++++++++++++++++++++++ README.md | 172 ++++++++++++++++++++++++++- fs.go | 120 +++++++++++++++++++ fs_test.go | 325 ++++++++++++++++++++++++++++++++++++++++++++++++++++ httpFs.go | 96 ++++++++++++++++ memfile.go | 190 ++++++++++++++++++++++++++++++ memmap.go | 219 +++++++++++++++++++++++++++++++++++ memradix.go | 14 +++ os.go | 61 ++++++++++ 9 files changed, 1369 insertions(+), 2 deletions(-) create mode 100644 LICENSE.txt create mode 100644 fs.go create mode 100644 fs_test.go create mode 100644 httpFs.go create mode 100644 memfile.go create mode 100644 memmap.go create mode 100644 memradix.go create mode 100644 os.go diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..298f0e2 --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,174 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. diff --git a/README.md b/README.md index ac5858c..276af7a 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,172 @@ -afero -===== +# Afero A FileSystem Abstraction System for Go + +[![Build Status](https://travis-ci.org/spf13/fs.png)](https://travis-ci.org/spf13/fs) + +## Overview + +Package Afero provides types and methods for interacting with the filesystem, +as an abstraction layer. + +It provides a few implementations that are largely interoperable. One that +uses the operating system filesystem, one that uses memory to store files +(cross platform) and an interface that should be implemented if you want to +provide your own filesystem. + +It is suitable for use in a any situation where you would consider using +the OS package as it provides an additional abstraction that makes it +easy to use a memory backed file system during testing. It also adds +support for the http filesystem for full interoperability. + +Afero has an exceptionally clean interface and simple design without needless +constructors or initialization methods. + +## The name + +Initially this project was called fs. Unfortunately as I used it, the +name proved confusing, there were too many fs’. In looking for +alternatives I looked up the word 'abstract' in a variety of different +languages. Afero is the Greek word for abstract and it seemed to be a +fitting name for the project. It also means ‘to do’ or ‘thing’ in +Esperanto which is also fitting. + +## Interface + +Afero simply takes the interfaces already defined throughout the standard +library and unifies them into a pair of interfaces that satisfy all +known uses. One interface for a file and one for a filesystem. + +## Filesystems + +Afero additionally comes with a few filesystems and file implementations +ready to use. + +### OsFs + +The first is simply a wrapper around the native OS calls. This makes it +very easy to use as all of the calls are the same as the existing OS +calls. It also makes it trivial to have your code use the OS during +operation and a mock filesystem during testing or as needed. + +### MemMapFs + +Fs also provides a fully atomic memory backed filesystem perfect for use in +mocking and to speed up unnecessary disk io when persistence isn’t +necessary. It is fully concurrent and will work within go routines +safely. + +### MemFile + +As part of MemMapFs, fs also provides an atomic, fully concurrent memory +backed file implementation. This can be used in other memory backed file +systems with ease. Plans are to add a radix tree memory stored file +system using MemFile. + +## Usage + + +### Installing +Using Afero is easy. First use go get to install the latest version +of the library. + + $ go get github.com/spf13/afero + +Next include Afero in your application. + + import "github.com/spf13/afero" + +## Using Afero + +There are a few different ways to use Afero. You could use the +interfaces alone to define you own file system. You could use it as a +wrapper for the OS packages. You could use it to define different +filesystems for different parts of your application. Here we will +demonstrate a basic usage. + +First define a package variable and set it to a pointer to a filesystem. + + var AppFs afero.Fs = &afero.MemMapFs{} + + or + + var AppFs afero.Fs = &afero.OsFs{} + +It is important to note that if you repeat the composite literal you +will be using a completely new and isolated filesystem. In the case of +OsFs it will still use the same underlying filesystem but will reduce +the ability to drop in other filesystems as desired. + +Then throughout your functions and methods use the methods familiar +already from the OS package. + +Methods Available: + + Create(name string) : File, error + Mkdir(name string, perm os.FileMode) : error + MkdirAll(path string, perm os.FileMode) : error + Name() : string + Open(name string) : File, error + OpenFile(name string, flag int, perm os.FileMode) : File, error + Remove(name string) : error + RemoveAll(path string) : error + Rename(oldname, newname string) : error + Stat(name string) : os.FileInfo, error + +In our case we would call `AppFs.Open()` as an example because that is how we’ve defined to +access our filesystem. + +In some applications it may make sense to define a new package that +simply exports the file system variable for easy access from anywhere. + + +### Using a Mock Filesystem for Testing + +There is a large benefit to using a mock filesystem for testing. It has +a completely blank state every time it is initialized and can be easily +reproducible regardless of OS. It is also faster than disk which makes +the tests run faster. Lastly it doesn’t require any clean up after tests +are run. + +One way to accomplish this is to define a variable as mentioned above. +In your application this will be set to &afero.OsFs{} during testing you +can set it to &afero.MemMapFs{}. + +### Using with Http + +The Http package requires a slightly specific version of Open which +returns an http.File type. + +Afero provides an httpFs file system which satisfies this requirement. +Any Afero FileSystem can be used as an httpFs. + + httpFs := &fs.HttpFs{SourceFs: } + fileserver := http.FileServer(httpFs.Dir())) + http.Handle("/", fileserver) + + +## Release Notes +* **0.8.0** 2014.10.28 + * First public version + * Interfaces feel ready for people to build using + * Interfaces satisfy all known uses + * MemMapFs passes the majority of the OS test suite + * OsFs passes the majority of the OS test suite + +## Contributing + +1. Fork it +2. Create your feature branch (`git checkout -b my-new-feature`) +3. Commit your changes (`git commit -am 'Add some feature'`) +4. Push to the branch (`git push origin my-new-feature`) +5. Create new Pull Request + +## Contributors + +Names in no particular order: + +* [spf13](https://github.com/spf13) + +## License + +Afero is released under the Apache 2.0 license. See [LICENSE.txt](https://github.com/spf13/afero/blob/master/LICENSE.txt) diff --git a/fs.go b/fs.go new file mode 100644 index 0000000..e9b9cbf --- /dev/null +++ b/fs.go @@ -0,0 +1,120 @@ +// Copyright © 2014 Steve Francia . +// Copyright 2013 tsuru authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package afero provides types and methods for interacting with the filesystem, +// as an abstraction layer. + +// Afero also provides a few implementations that are mostly interoperable. One that +// uses the operating system filesystem, one that uses memory to store files +// (cross platform) and an interface that should be implemented if you want to +// provide your own filesystem. + +package afero + +import ( + "errors" + "io" + "os" + "sort" +) + +// File represents a file in the filesystem. +type File interface { + io.Closer + io.Reader + io.ReaderAt + io.Seeker + io.Writer + io.WriterAt + //Fd() uintptr + Stat() (os.FileInfo, error) + Readdir(count int) ([]os.FileInfo, error) + Readdirnames(n int) ([]string, error) + WriteString(s string) (ret int, err error) + Truncate(size int64) error + Name() string +} + +// Fs is the filesystem interface. +// +// Any simulated or real filesystem should implement this interface. +type Fs interface { + // Create creates a file in the filesystem, returning the file and an + // error, if any happens. + Create(name string) (File, error) + + // Mkdir creates a directory in the filesystem, return an error if any + // happens. + Mkdir(name string, perm os.FileMode) error + + // MkdirAll creates a directory path and all parents that does not exist + // yet. + MkdirAll(path string, perm os.FileMode) error + + // Open opens a file, returning it or an error, if any happens. + Open(name string) (File, error) + + // OpenFile opens a file using the given flags and the given mode. + OpenFile(name string, flag int, perm os.FileMode) (File, error) + + // Remove removes a file identified by name, returning an error, if any + // happens. + Remove(name string) error + + // RemoveAll removes a directory path and all any children it contains. It + // does not fail if the path does not exist (return nil). + RemoveAll(path string) error + + // Rename renames a file. + Rename(oldname, newname string) error + + // Stat returns a FileInfo describing the named file, or an error, if any + // happens. + Stat(name string) (os.FileInfo, error) + + // The name of this FileSystem + Name() string +} + +var ( + ErrFileClosed = errors.New("File is closed") + ErrOutOfRange = errors.New("Out of range") + ErrTooLarge = errors.New("Too large") + ErrFileNotFound = os.ErrNotExist + ErrFileExists = os.ErrExist + ErrDestinationExists = os.ErrExist +) + +// ReadDir reads the directory named by dirname and returns +// a list of sorted directory entries. +func ReadDir(dirname string, fs Fs) ([]os.FileInfo, error) { + f, err := fs.Open(dirname) + if err != nil { + return nil, err + } + list, err := f.Readdir(-1) + f.Close() + if err != nil { + return nil, err + } + sort.Sort(byName(list)) + return list, nil +} + +// byName implements sort.Interface. +type byName []os.FileInfo + +func (f byName) Len() int { return len(f) } +func (f byName) Less(i, j int) bool { return f[i].Name() < f[j].Name() } +func (f byName) Swap(i, j int) { f[i], f[j] = f[j], f[i] } diff --git a/fs_test.go b/fs_test.go new file mode 100644 index 0000000..9246220 --- /dev/null +++ b/fs_test.go @@ -0,0 +1,325 @@ +// Copyright © 2014 Steve Francia . +// Copyright 2009 The Go Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package afero + +import ( + "bytes" + "io" + "io/ioutil" + "os" + "path" + "runtime" + "strings" + "syscall" + "testing" +) + +var dot = []string{ + "fs.go", + "fs_test.go", + "httpFs.go", + "memfile.go", + "memmap.go", +} + +var testDir = "/tmp/fun" +var testName = "test.txt" +var Fss = []Fs{&MemMapFs{}, &OsFs{}} + +//var Fss = []Fs{OsFs{}} + +//Read with length 0 should not return EOF. +func TestRead0(t *testing.T) { + for _, fs := range Fss { + path := testDir + "/" + testName + if err := fs.MkdirAll(testDir, 777); err != nil { + t.Fatal(fs.Name(), "unable to create dir", err) + } + + f, err := fs.Create(path) + if err != nil { + t.Fatal(fs.Name(), "create failed:", err) + } + defer f.Close() + f.WriteString("Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.") + + b := make([]byte, 0) + n, err := f.Read(b) + if n != 0 || err != nil { + t.Errorf("%v: Read(0) = %d, %v, want 0, nil", fs.Name(), n, err) + } + f.Seek(0, 0) + b = make([]byte, 100) + n, err = f.Read(b) + if n <= 0 || err != nil { + t.Errorf("%v: Read(100) = %d, %v, want >0, nil", fs.Name(), n, err) + } + } +} + +func TestRename(t *testing.T) { + for _, fs := range Fss { + from, to := testDir+"/renamefrom", testDir+"/renameto" + fs.Remove(to) // Just in case. + fs.MkdirAll(testDir, 0777) // Just in case. + file, err := fs.Create(from) + if err != nil { + t.Fatalf("open %q failed: %v", to, err) + } + if err = file.Close(); err != nil { + t.Errorf("close %q failed: %v", to, err) + } + err = fs.Rename(from, to) + if err != nil { + t.Fatalf("rename %q, %q failed: %v", to, from, err) + } + defer fs.Remove(to) + _, err = fs.Stat(to) + if err != nil { + t.Errorf("stat %q failed: %v", to, err) + } + } +} + +func TestTruncate(t *testing.T) { + for _, fs := range Fss { + f := newFile("TestTruncate", fs, t) + defer fs.Remove(f.Name()) + defer f.Close() + + checkSize(t, f, 0) + f.Write([]byte("hello, world\n")) + checkSize(t, f, 13) + f.Truncate(10) + checkSize(t, f, 10) + f.Truncate(1024) + checkSize(t, f, 1024) + f.Truncate(0) + checkSize(t, f, 0) + _, err := f.Write([]byte("surprise!")) + if err == nil { + checkSize(t, f, 13+9) // wrote at offset past where hello, world was. + } + } +} + +func TestSeek(t *testing.T) { + for _, fs := range Fss { + f := newFile("TestSeek", fs, t) + defer fs.Remove(f.Name()) + defer f.Close() + + const data = "hello, world\n" + io.WriteString(f, data) + + type test struct { + in int64 + whence int + out int64 + } + var tests = []test{ + {0, 1, int64(len(data))}, + {0, 0, 0}, + {5, 0, 5}, + {0, 2, int64(len(data))}, + {0, 0, 0}, + {-1, 2, int64(len(data)) - 1}, + {1 << 33, 0, 1 << 33}, + {1 << 33, 2, 1<<33 + int64(len(data))}, + } + for i, tt := range tests { + off, err := f.Seek(tt.in, tt.whence) + if off != tt.out || err != nil { + if e, ok := err.(*os.PathError); ok && e.Err == syscall.EINVAL && tt.out > 1<<32 { + // Reiserfs rejects the big seeks. + // http://code.google.com/p/go/issues/detail?id=91 + break + } + t.Errorf("#%d: Seek(%v, %v) = %v, %v want %v, nil", i, tt.in, tt.whence, off, err, tt.out) + } + } + } +} + +func TestReadAt(t *testing.T) { + for _, fs := range Fss { + f := newFile("TestReadAt", fs, t) + defer fs.Remove(f.Name()) + defer f.Close() + + const data = "hello, world\n" + io.WriteString(f, data) + + b := make([]byte, 5) + n, err := f.ReadAt(b, 7) + if err != nil || n != len(b) { + t.Fatalf("ReadAt 7: %d, %v", n, err) + } + if string(b) != "world" { + t.Fatalf("ReadAt 7: have %q want %q", string(b), "world") + } + } +} + +func TestWriteAt(t *testing.T) { + for _, fs := range Fss { + f := newFile("TestWriteAt", fs, t) + defer fs.Remove(f.Name()) + defer f.Close() + + const data = "hello, world\n" + io.WriteString(f, data) + + n, err := f.WriteAt([]byte("WORLD"), 7) + if err != nil || n != 5 { + t.Fatalf("WriteAt 7: %d, %v", n, err) + } + + f2, err := fs.Open(f.Name()) + defer f2.Close() + buf := new(bytes.Buffer) + buf.ReadFrom(f2) + b := buf.Bytes() + if err != nil { + t.Fatalf("%v: ReadFile %s: %v", fs.Name(), f.Name(), err) + } + if string(b) != "hello, WORLD\n" { + t.Fatalf("after write: have %q want %q", string(b), "hello, WORLD\n") + } + + } +} + +//func TestReaddirnames(t *testing.T) { +//for _, fs := range Fss { +//testReaddirnames(fs, ".", dot, t) +////testReaddirnames(sysdir.name, fs, sysdir.files, t) +//} +//} + +//func TestReaddir(t *testing.T) { +//for _, fs := range Fss { +//testReaddir(fs, ".", dot, t) +////testReaddir(sysdir.name, fs, sysdir.files, t) +//} +//} + +func newFile(testName string, fs Fs, t *testing.T) (f File) { + // Use a local file system, not NFS. + // On Unix, override $TMPDIR in case the user + // has it set to an NFS-mounted directory. + dir := "" + if runtime.GOOS != "windows" { + dir = "/tmp" + } + fs.MkdirAll(dir, 0777) + f, err := fs.Create(path.Join(dir, testName)) + if err != nil { + t.Fatalf("%v: open %s: %s", fs.Name(), testName, err) + } + return f +} + +func writeFile(t *testing.T, fs Fs, fname string, flag int, text string) string { + f, err := fs.OpenFile(fname, flag, 0666) + if err != nil { + t.Fatalf("Open: %v", err) + } + n, err := io.WriteString(f, text) + if err != nil { + t.Fatalf("WriteString: %d, %v", n, err) + } + f.Close() + data, err := ioutil.ReadFile(fname) + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + return string(data) +} + +func testReaddirnames(fs Fs, dir string, contents []string, t *testing.T) { + file, err := fs.Open(dir) + if err != nil { + t.Fatalf("open %q failed: %v", dir, err) + } + defer file.Close() + s, err2 := file.Readdirnames(-1) + if err2 != nil { + t.Fatalf("readdirnames %q failed: %v", dir, err2) + } + for _, m := range contents { + found := false + for _, n := range s { + if n == "." || n == ".." { + t.Errorf("got %s in directory", n) + } + if equal(m, n) { + if found { + t.Error("present twice:", m) + } + found = true + } + } + if !found { + t.Error("could not find", m) + } + } +} + +func testReaddir(fs Fs, dir string, contents []string, t *testing.T) { + file, err := fs.Open(dir) + if err != nil { + t.Fatalf("open %q failed: %v", dir, err) + } + defer file.Close() + s, err2 := file.Readdir(-1) + if err2 != nil { + t.Fatalf("readdir %q failed: %v", dir, err2) + } + for _, m := range contents { + found := false + for _, n := range s { + if equal(m, n.Name()) { + if found { + t.Error("present twice:", m) + } + found = true + } + } + if !found { + t.Error("could not find", m) + } + } +} + +func equal(name1, name2 string) (r bool) { + switch runtime.GOOS { + case "windows": + r = strings.ToLower(name1) == strings.ToLower(name2) + default: + r = name1 == name2 + } + return +} + +func checkSize(t *testing.T, f File, size int64) { + dir, err := f.Stat() + if err != nil { + t.Fatalf("Stat %q (looking for size %d): %s", f.Name(), size, err) + } + if dir.Size() != size { + t.Errorf("Stat %q: size %d want %d", f.Name(), dir.Size(), size) + } +} diff --git a/httpFs.go b/httpFs.go new file mode 100644 index 0000000..af44344 --- /dev/null +++ b/httpFs.go @@ -0,0 +1,96 @@ +// Copyright © 2014 Steve Francia . +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package afero + +import ( + "errors" + "net/http" + "os" + "path" + "path/filepath" + "strings" +) + +type httpDir struct { + basePath string + fs HttpFs +} + +func (d httpDir) Open(name string) (http.File, error) { + if filepath.Separator != '/' && strings.IndexRune(name, filepath.Separator) >= 0 || + strings.Contains(name, "\x00") { + return nil, errors.New("http: invalid character in file path") + } + dir := string(d.basePath) + if dir == "" { + dir = "." + } + f, err := d.fs.Open(filepath.Join(dir, filepath.FromSlash(path.Clean("/"+name)))) + if err != nil { + return nil, err + } + return f, nil +} + +type HttpFs struct { + SourceFs Fs +} + +func (h HttpFs) Dir(s string) *httpDir { + return &httpDir{basePath: s, fs: h} +} + +func (h HttpFs) Name() string { return "h HttpFs" } + +func (h HttpFs) Create(name string) (File, error) { + return h.SourceFs.Create(name) +} + +func (h HttpFs) Mkdir(name string, perm os.FileMode) error { + return h.SourceFs.Mkdir(name, perm) +} + +func (h HttpFs) MkdirAll(path string, perm os.FileMode) error { + return h.SourceFs.MkdirAll(path, perm) +} + +func (h HttpFs) Open(name string) (http.File, error) { + f, err := h.SourceFs.Open(name) + if err == nil { + if httpfile, ok := f.(http.File); ok { + return httpfile, nil + } + } + return nil, err +} + +func (h HttpFs) OpenFile(name string, flag int, perm os.FileMode) (File, error) { + return h.SourceFs.OpenFile(name, flag, perm) +} + +func (h HttpFs) Remove(name string) error { + return h.SourceFs.Remove(name) +} + +func (h HttpFs) RemoveAll(path string) error { + return h.SourceFs.RemoveAll(path) +} + +func (h HttpFs) Rename(oldname, newname string) error { + return h.SourceFs.Rename(oldname, newname) +} + +func (h HttpFs) Stat(name string) (os.FileInfo, error) { + return h.SourceFs.Stat(name) +} diff --git a/memfile.go b/memfile.go new file mode 100644 index 0000000..ef6fdaf --- /dev/null +++ b/memfile.go @@ -0,0 +1,190 @@ +// Copyright © 2014 Steve Francia . +// Copyright 2013 tsuru authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package afero + +import ( + "bytes" + "io" + "os" + "sync/atomic" +) + +import "time" + +type MemDir interface { + Len() int + Names() []string + Files() []File + Add(File) + Remove(File) +} + +type InMemoryFile struct { + at int64 + name string + data []byte + memDir MemDir + dir bool + closed bool +} + +func MemFileCreate(name string) *InMemoryFile { + return &InMemoryFile{name: name} +} + +func (f *InMemoryFile) Close() error { + atomic.StoreInt64(&f.at, 0) + f.closed = true + return nil +} + +func (f *InMemoryFile) Name() string { + return f.name +} + +func (f *InMemoryFile) Stat() (os.FileInfo, error) { + return &InMemoryFileInfo{f}, nil +} + +func (f *InMemoryFile) Readdir(count int) (res []os.FileInfo, err error) { + files := f.memDir.Files() + limit := len(files) + + if len(files) == 0 { + return + } + + if count > 0 { + limit = count + } + + if len(files) < limit { + err = io.EOF + } + + res = make([]os.FileInfo, f.memDir.Len()) + + i := 0 + for _, file := range f.memDir.Files() { + res[i], _ = file.Stat() + i++ + } + return res, nil +} + +func (f *InMemoryFile) Readdirnames(n int) (names []string, err error) { + fi, err := f.Readdir(n) + names = make([]string, len(fi)) + for i, f := range fi { + names[i] = f.Name() + } + return names, err +} + +func (f *InMemoryFile) Read(b []byte) (n int, err error) { + if f.closed == true { + return 0, ErrFileClosed + } + if len(f.data)-int(f.at) >= len(b) { + n = len(b) + } else { + n = len(f.data) - int(f.at) + err = io.EOF + } + copy(b, f.data[f.at:f.at+int64(n)]) + atomic.AddInt64(&f.at, int64(n)) + return +} + +func (f *InMemoryFile) ReadAt(b []byte, off int64) (n int, err error) { + atomic.StoreInt64(&f.at, off) + return f.Read(b) +} + +func (f *InMemoryFile) Truncate(size int64) error { + if f.closed == true { + return ErrFileClosed + } + if size < 0 { + return ErrOutOfRange + } + if size > int64(len(f.data)) { + diff := size - int64(len(f.data)) + f.data = append(f.data, bytes.Repeat([]byte{00}, int(diff))...) + } else { + f.data = f.data[0:size] + } + return nil +} + +func (f *InMemoryFile) Seek(offset int64, whence int) (int64, error) { + if f.closed == true { + return 0, ErrFileClosed + } + switch whence { + case 0: + atomic.StoreInt64(&f.at, offset) + case 1: + atomic.AddInt64(&f.at, int64(offset)) + case 2: + atomic.StoreInt64(&f.at, int64(len(f.data))+offset) + } + return f.at, nil +} + +func (f *InMemoryFile) Write(b []byte) (n int, err error) { + n = len(b) + cur := atomic.LoadInt64(&f.at) + diff := cur - int64(len(f.data)) + var tail []byte + if n+int(cur) < len(f.data) { + tail = f.data[n+int(cur):] + } + if diff > 0 { + f.data = append(bytes.Repeat([]byte{00}, int(diff)), b...) + f.data = append(f.data, tail...) + } else { + f.data = append(f.data[:cur], b...) + f.data = append(f.data, tail...) + } + + atomic.StoreInt64(&f.at, int64(len(f.data))) + return +} + +func (f *InMemoryFile) WriteAt(b []byte, off int64) (n int, err error) { + atomic.StoreInt64(&f.at, off) + return f.Write(b) +} + +func (f *InMemoryFile) WriteString(s string) (ret int, err error) { + return f.Write([]byte(s)) +} + +func (f *InMemoryFile) Info() *InMemoryFileInfo { + return &InMemoryFileInfo{file: f} +} + +type InMemoryFileInfo struct { + file *InMemoryFile +} + +// Implements os.FileInfo +func (s *InMemoryFileInfo) Name() string { return s.file.Name() } +func (s *InMemoryFileInfo) Size() int64 { return int64(len(s.file.data)) } +func (s *InMemoryFileInfo) Mode() os.FileMode { return os.ModeTemporary } +func (s *InMemoryFileInfo) ModTime() time.Time { return time.Time{} } +func (s *InMemoryFileInfo) IsDir() bool { return s.file.dir } +func (s *InMemoryFileInfo) Sys() interface{} { return nil } diff --git a/memmap.go b/memmap.go new file mode 100644 index 0000000..9b4d0a0 --- /dev/null +++ b/memmap.go @@ -0,0 +1,219 @@ +// Copyright © 2014 Steve Francia . +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package afero + +import ( + "os" + "path" + "path/filepath" + "strings" + "sync" +) + +var mux = &sync.Mutex{} + +type MemMapFs struct { + data map[string]File + mutex *sync.RWMutex +} + +func (m *MemMapFs) lock() { + mx := m.getMutex() + mx.Lock() +} +func (m *MemMapFs) unlock() { m.getMutex().Unlock() } +func (m *MemMapFs) rlock() { m.getMutex().RLock() } +func (m *MemMapFs) runlock() { m.getMutex().RUnlock() } + +func (m *MemMapFs) getData() map[string]File { + if m.data == nil { + m.data = make(map[string]File) + } + return m.data +} + +func (m *MemMapFs) getMutex() *sync.RWMutex { + mux.Lock() + if m.mutex == nil { + m.mutex = &sync.RWMutex{} + } + mux.Unlock() + return m.mutex +} + +type MemDirMap map[string]File + +func (m MemDirMap) Len() int { return len(m) } +func (m MemDirMap) Add(f File) { m[f.Name()] = f } +func (m MemDirMap) Remove(f File) { delete(m, f.Name()) } +func (m MemDirMap) Files() (files []File) { + for _, f := range m { + files = append(files, f) + } + return files +} + +func (m MemDirMap) Names() (names []string) { + for x := range m { + names = append(names, x) + } + return names +} + +func (MemMapFs) Name() string { return "MemMapFS" } + +func (m *MemMapFs) Create(name string) (File, error) { + m.lock() + m.getData()[name] = MemFileCreate(name) + m.unlock() + m.registerDirs(m.getData()[name]) + return m.getData()[name], nil +} + +func (m *MemMapFs) registerDirs(f File) { + var x = f.Name() + for x != "/" { + f := m.registerWithParent(f) + if f == nil { + break + } + x = f.Name() + } +} + +func (m *MemMapFs) unRegisterWithParent(f File) File { + parent := m.findParent(f) + pmem := parent.(*InMemoryFile) + pmem.memDir.Remove(f) + return parent +} + +func (m *MemMapFs) findParent(f File) File { + dirs, _ := path.Split(f.Name()) + if len(dirs) > 1 { + _, parent := path.Split(path.Clean(dirs)) + if len(parent) > 0 { + pfile, err := m.Open(parent) + if err != nil { + return pfile + } + } + } + return nil +} + +func (m *MemMapFs) registerWithParent(f File) File { + if f == nil { + return nil + } + parent := m.findParent(f) + if parent != nil { + pmem := parent.(*InMemoryFile) + pmem.memDir.Add(f) + } else { + pdir := filepath.Dir(path.Clean(f.Name())) + m.Mkdir(pdir, 0777) + } + return parent +} + +func (m *MemMapFs) Mkdir(name string, perm os.FileMode) error { + m.rlock() + _, ok := m.getData()[name] + m.runlock() + if ok { + return ErrFileExists + } else { + m.lock() + m.getData()[name] = &InMemoryFile{name: name, memDir: &MemDirMap{}, dir: true} + m.unlock() + m.registerDirs(m.getData()[name]) + } + return nil +} + +func (m *MemMapFs) MkdirAll(path string, perm os.FileMode) error { + return m.Mkdir(path, 0777) +} + +func (m *MemMapFs) Open(name string) (File, error) { + m.rlock() + f, ok := m.getData()[name] + m.runlock() + if ok { + f.Seek(0, 0) + return f, nil + } else { + return nil, ErrFileNotFound + } +} + +func (m *MemMapFs) OpenFile(name string, flag int, perm os.FileMode) (File, error) { + return m.Open(name) +} + +func (m *MemMapFs) Remove(name string) error { + m.rlock() + defer m.runlock() + + if _, ok := m.getData()["name"]; ok { + m.lock() + delete(m.getData(), name) + m.unlock() + } + return nil +} + +func (m *MemMapFs) RemoveAll(path string) error { + m.rlock() + defer m.runlock() + for p, _ := range m.getData() { + if strings.HasPrefix(p, path) { + m.runlock() + m.lock() + delete(m.getData(), p) + m.unlock() + m.rlock() + } + } + return nil +} + +func (m *MemMapFs) Rename(oldname, newname string) error { + m.rlock() + defer m.runlock() + if _, ok := m.getData()[oldname]; ok { + if _, ok := m.getData()[newname]; !ok { + m.runlock() + m.lock() + m.getData()[newname] = m.getData()[oldname] + delete(m.getData(), oldname) + m.unlock() + m.rlock() + } else { + return ErrDestinationExists + } + } else { + return ErrFileNotFound + } + return nil +} + +func (m *MemMapFs) Stat(name string) (os.FileInfo, error) { + f, err := m.Open(name) + if err != nil { + return nil, err + } + return &InMemoryFileInfo{file: f.(*InMemoryFile)}, nil +} diff --git a/memradix.go b/memradix.go new file mode 100644 index 0000000..87527f3 --- /dev/null +++ b/memradix.go @@ -0,0 +1,14 @@ +// Copyright © 2014 Steve Francia . +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package afero diff --git a/os.go b/os.go new file mode 100644 index 0000000..4f472fc --- /dev/null +++ b/os.go @@ -0,0 +1,61 @@ +// Copyright © 2014 Steve Francia . +// Copyright 2013 tsuru authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package afero + +import "os" + +// OsFs is a Fs implementation that uses functions provided by the os package. +// +// For details in any method, check the documentation of the os package +// (http://golang.org/pkg/os/). +type OsFs struct{} + +func (OsFs) Name() string { return "OsFs" } + +func (OsFs) Create(name string) (File, error) { + return os.Create(name) +} + +func (OsFs) Mkdir(name string, perm os.FileMode) error { + return os.Mkdir(name, perm) +} + +func (OsFs) MkdirAll(path string, perm os.FileMode) error { + return os.MkdirAll(path, perm) +} + +func (OsFs) Open(name string) (File, error) { + return os.Open(name) +} + +func (OsFs) OpenFile(name string, flag int, perm os.FileMode) (File, error) { + return os.OpenFile(name, flag, perm) +} + +func (OsFs) Remove(name string) error { + return os.Remove(name) +} + +func (OsFs) RemoveAll(path string) error { + return os.RemoveAll(path) +} + +func (OsFs) Rename(oldname, newname string) error { + return os.Rename(oldname, newname) +} + +func (OsFs) Stat(name string) (os.FileInfo, error) { + return os.Stat(name) +}