From 7096d68458e787d1121f2b895fbc47d90587bf22 Mon Sep 17 00:00:00 2001 From: Martin Bertschler Date: Sun, 3 Jan 2016 23:59:42 +0100 Subject: [PATCH] mem.File is now mem.FileData and mem.File is a file handle with a pointer to a mem.FileData --- mem/dir.go | 16 ++++---- mem/dirmap.go | 16 ++++---- mem/file.go | 105 +++++++++++++++++++++++++++---------------------- memmap.go | 95 +++++++++++++++++++------------------------- memmap_test.go | 43 ++++++++++++++------ 5 files changed, 144 insertions(+), 131 deletions(-) diff --git a/mem/dir.go b/mem/dir.go index 79cba7a..d7b7bdf 100644 --- a/mem/dir.go +++ b/mem/dir.go @@ -16,22 +16,22 @@ package mem type Dir interface { Len() int Names() []string - Files() []File - Add(File) - Remove(File) + Files() []*File + Add(*File) + Remove(*File) } func RemoveFromMemDir(dir *File, f *File) { - dir.memDir.Remove(*f) + dir.fileData.memDir.Remove(f) } func AddToMemDir(dir *File, f *File) { - dir.memDir.Add(*f) + dir.fileData.memDir.Add(f) } func InitializeDir(d *File) { - if d.memDir == nil { - d.dir = true - d.memDir = &DirMap{} + if d.fileData.memDir == nil { + d.fileData.dir = true + d.fileData.memDir = &DirMap{} } } diff --git a/mem/dirmap.go b/mem/dirmap.go index 59c5d8d..f18718c 100644 --- a/mem/dirmap.go +++ b/mem/dirmap.go @@ -15,25 +15,25 @@ package mem import "sort" -type DirMap map[string]File +type DirMap map[string]*FileData -func (m DirMap) Len() int { return len(m) } -func (m DirMap) Add(f File) { m[f.Name()] = f } -func (m DirMap) Remove(f File) { delete(m, f.Name()) } -func (m DirMap) Files() (files []File) { +func (m DirMap) Len() int { return len(m) } +func (m DirMap) Add(f *File) { m[f.fileData.name] = f.fileData } +func (m DirMap) Remove(f *File) { delete(m, f.fileData.name) } +func (m DirMap) Files() (files []*File) { for _, f := range m { - files = append(files, f) + files = append(files, NewFileHandle(f)) } sort.Sort(filesSorter(files)) return files } -type filesSorter []File +type filesSorter []*File // implement sort.Interface for []File func (s filesSorter) Len() int { return len(s) } func (s filesSorter) Swap(i, j int) { s[i], s[j] = s[j], s[i] } -func (s filesSorter) Less(i, j int) bool { return s[i].Name() < s[j].Name() } +func (s filesSorter) Less(i, j int) bool { return s[i].fileData.name < s[j].fileData.name } func (m DirMap) Names() (names []string) { for x := range m { diff --git a/mem/file.go b/mem/file.go index 33093f5..16f13a0 100644 --- a/mem/file.go +++ b/mem/file.go @@ -32,63 +32,74 @@ type File struct { // atomic requires 64-bit alignment for struct field access at int64 readDirCount int64 + closed bool + fileData *FileData +} +func NewFileHandle(data *FileData) *File { + return &File{fileData: data} +} + +func (f File) Data() *FileData { + return f.fileData +} + +type FileData struct { sync.Mutex name string data []byte memDir Dir dir bool - closed bool mode os.FileMode modtime time.Time } -func CreateFile(name string) *File { - return &File{name: name, mode: os.ModeTemporary, modtime: time.Now()} +func CreateFile(name string) *FileData { + return &FileData{name: name, mode: os.ModeTemporary, modtime: time.Now()} } -func CreateDir(name string) *File { - return &File{name: name, memDir: &DirMap{}, dir: true} +func CreateDir(name string) *FileData { + return &FileData{name: name, memDir: &DirMap{}, dir: true} } func ChangeFileName(f *File, newname string) { - f.name = newname + f.fileData.name = newname } func SetMode(f *File, mode os.FileMode) { - f.mode = mode + f.fileData.mode = mode } func SetModTime(f *File, mtime time.Time) { - f.modtime = mtime + f.fileData.modtime = mtime } func GetFileInfo(f *File) *FileInfo { - return &FileInfo{file: f} + return &FileInfo{f.fileData} } func (f *File) Open() error { atomic.StoreInt64(&f.at, 0) atomic.StoreInt64(&f.readDirCount, 0) - f.Lock() + f.fileData.Lock() f.closed = false - f.Unlock() + f.fileData.Unlock() return nil } func (f *File) Close() error { - f.Lock() + f.fileData.Lock() f.closed = true - f.Unlock() + f.fileData.Unlock() return nil } func (f *File) Name() string { - return f.name + return f.fileData.name } func (f *File) Stat() (os.FileInfo, error) { - return &FileInfo{f}, nil + return &FileInfo{f.fileData}, nil } func (f *File) Sync() error { @@ -98,8 +109,8 @@ func (f *File) Sync() error { func (f *File) Readdir(count int) (res []os.FileInfo, err error) { var outLength int64 - f.Lock() - files := f.memDir.Files()[f.readDirCount:] + f.fileData.Lock() + files := f.fileData.memDir.Files()[f.readDirCount:] if count > 0 { if len(files) < count { outLength = int64(len(files)) @@ -113,11 +124,11 @@ func (f *File) Readdir(count int) (res []os.FileInfo, err error) { outLength = int64(len(files)) } f.readDirCount += outLength - f.Unlock() + f.fileData.Unlock() res = make([]os.FileInfo, outLength) for i := range res { - res[i], _ = files[i].Stat() + res[i] = &FileInfo{files[i].fileData} } return res, err @@ -133,20 +144,20 @@ func (f *File) Readdirnames(n int) (names []string, err error) { } func (f *File) Read(b []byte) (n int, err error) { - f.Lock() - defer f.Unlock() + f.fileData.Lock() + defer f.fileData.Unlock() if f.closed == true { return 0, ErrFileClosed } - if len(b) > 0 && int(f.at) == len(f.data) { + if len(b) > 0 && int(f.at) == len(f.fileData.data) { return 0, io.EOF } - if len(f.data)-int(f.at) >= len(b) { + if len(f.fileData.data)-int(f.at) >= len(b) { n = len(b) } else { - n = len(f.data) - int(f.at) + n = len(f.fileData.data) - int(f.at) } - copy(b, f.data[f.at:f.at+int64(n)]) + copy(b, f.fileData.data[f.at:f.at+int64(n)]) atomic.AddInt64(&f.at, int64(n)) return } @@ -163,11 +174,11 @@ func (f *File) Truncate(size int64) error { if size < 0 { return ErrOutOfRange } - if size > int64(len(f.data)) { - diff := size - int64(len(f.data)) - f.data = append(f.data, bytes.Repeat([]byte{00}, int(diff))...) + if size > int64(len(f.fileData.data)) { + diff := size - int64(len(f.fileData.data)) + f.fileData.data = append(f.fileData.data, bytes.Repeat([]byte{00}, int(diff))...) } else { - f.data = f.data[0:size] + f.fileData.data = f.fileData.data[0:size] } return nil } @@ -182,7 +193,7 @@ func (f *File) Seek(offset int64, whence int) (int64, error) { case 1: atomic.AddInt64(&f.at, int64(offset)) case 2: - atomic.StoreInt64(&f.at, int64(len(f.data))+offset) + atomic.StoreInt64(&f.at, int64(len(f.fileData.data))+offset) } return f.at, nil } @@ -190,22 +201,22 @@ func (f *File) Seek(offset int64, whence int) (int64, error) { func (f *File) Write(b []byte) (n int, err error) { n = len(b) cur := atomic.LoadInt64(&f.at) - f.Lock() - defer f.Unlock() - diff := cur - int64(len(f.data)) + f.fileData.Lock() + defer f.fileData.Unlock() + diff := cur - int64(len(f.fileData.data)) var tail []byte - if n+int(cur) < len(f.data) { - tail = f.data[n+int(cur):] + if n+int(cur) < len(f.fileData.data) { + tail = f.fileData.data[n+int(cur):] } if diff > 0 { - f.data = append(bytes.Repeat([]byte{00}, int(diff)), b...) - f.data = append(f.data, tail...) + f.fileData.data = append(bytes.Repeat([]byte{00}, int(diff)), b...) + f.fileData.data = append(f.fileData.data, tail...) } else { - f.data = append(f.data[:cur], b...) - f.data = append(f.data, tail...) + f.fileData.data = append(f.fileData.data[:cur], b...) + f.fileData.data = append(f.fileData.data, tail...) } - atomic.StoreInt64(&f.at, int64(len(f.data))) + atomic.StoreInt64(&f.at, int64(len(f.fileData.data))) return } @@ -219,27 +230,27 @@ func (f *File) WriteString(s string) (ret int, err error) { } func (f *File) Info() *FileInfo { - return &FileInfo{file: f} + return &FileInfo{f.fileData} } type FileInfo struct { - file *File + *FileData } // Implements os.FileInfo func (s *FileInfo) Name() string { - _, name := filepath.Split(s.file.Name()) + _, name := filepath.Split(s.name) return name } -func (s *FileInfo) Mode() os.FileMode { return s.file.mode } -func (s *FileInfo) ModTime() time.Time { return s.file.modtime } -func (s *FileInfo) IsDir() bool { return s.file.dir } +func (s *FileInfo) Mode() os.FileMode { return s.mode } +func (s *FileInfo) ModTime() time.Time { return s.modtime } +func (s *FileInfo) IsDir() bool { return s.dir } func (s *FileInfo) Sys() interface{} { return nil } func (s *FileInfo) Size() int64 { if s.IsDir() { return int64(42) } - return int64(len(s.file.data)) + return int64(len(s.data)) } var ( diff --git a/memmap.go b/memmap.go index ff9da99..c8ab75a 100644 --- a/memmap.go +++ b/memmap.go @@ -14,7 +14,6 @@ package afero import ( - "errors" "fmt" "log" "os" @@ -28,15 +27,15 @@ import ( type MemMapFs struct { mu sync.RWMutex - data map[string]File + data map[string]*mem.FileData init sync.Once } var memfsInit sync.Once -func (m *MemMapFs) getData() map[string]File { +func (m *MemMapFs) getData() map[string]*mem.FileData { m.init.Do(func() { - m.data = make(map[string]File) + m.data = make(map[string]*mem.FileData) // Root should always exist, right? // TODO: what about windows? m.data[FilePathSeparator] = mem.CreateDir(FilePathSeparator) @@ -51,9 +50,11 @@ func (m *MemMapFs) Create(name string) (File, error) { m.mu.Lock() file := mem.CreateFile(name) m.getData()[name] = file - m.registerWithParent(file) + + handle := mem.NewFileHandle(file) + m.registerWithParent(handle) m.mu.Unlock() - return file, nil + return handle, nil } func (m *MemMapFs) unRegisterWithParent(fileName string) error { @@ -65,13 +66,11 @@ func (m *MemMapFs) unRegisterWithParent(fileName string) error { if parent == nil { log.Fatal("parent of ", f.Name(), " is nil") } - pmem := parent.(*mem.File) - fmem := f.(*mem.File) - mem.RemoveFromMemDir(pmem, fmem) + mem.RemoveFromMemDir(parent, f) return nil } -func (m *MemMapFs) findParent(f File) File { +func (m *MemMapFs) findParent(f *mem.File) *mem.File { pdir, _ := filepath.Split(f.Name()) pdir = filepath.Clean(pdir) pfile, err := m.lockfreeOpen(pdir) @@ -81,7 +80,7 @@ func (m *MemMapFs) findParent(f File) File { return pfile } -func (m *MemMapFs) registerWithParent(f File) { +func (m *MemMapFs) registerWithParent(f *mem.File) { if f == nil { return } @@ -99,15 +98,9 @@ func (m *MemMapFs) registerWithParent(f File) { return } } - pmem := parent.(*mem.File) - fmem := f.(*mem.File) - // TODO(mbertschler): memDir is only nil when it was not made with Mkdir - // or lockfreeMkdir. In this case the parent is also not a real directory. - // This currently only happens for the file ".". - // This is a quick hack to make the library usable with relative paths. - mem.InitializeDir(pmem) - mem.AddToMemDir(pmem, fmem) + mem.InitializeDir(parent) + mem.AddToMemDir(parent, f) } func (m *MemMapFs) lockfreeMkdir(name string, perm os.FileMode) error { @@ -115,7 +108,7 @@ func (m *MemMapFs) lockfreeMkdir(name string, perm os.FileMode) error { x, ok := m.getData()[name] if ok { // Only return ErrFileExists if it's a file, not a directory. - i, err := x.Stat() + i, err := mem.NewFileHandle(x).Stat() if !i.IsDir() { return ErrFileExists } @@ -125,7 +118,7 @@ func (m *MemMapFs) lockfreeMkdir(name string, perm os.FileMode) error { } else { item := mem.CreateDir(name) m.getData()[name] = item - m.registerWithParent(item) + m.registerWithParent(mem.NewFileHandle(item)) } return nil } @@ -142,7 +135,7 @@ func (m *MemMapFs) Mkdir(name string, perm os.FileMode) error { m.mu.Lock() item := mem.CreateDir(name) m.getData()[name] = item - m.registerWithParent(item) + m.registerWithParent(mem.NewFileHandle(item)) m.mu.Unlock() } return nil @@ -179,27 +172,23 @@ func (m *MemMapFs) Open(name string) (File, error) { m.mu.RLock() f, ok := m.getData()[name] - ff, ok := f.(*mem.File) - - if ok { - ff.Open() - } m.mu.RUnlock() - - if ok { - return f, nil - } else { + if !ok { return nil, &os.PathError{"open", name, ErrFileNotFound} } + ff := mem.NewFileHandle(f) + ff.Open() + return ff, nil } -func (m *MemMapFs) lockfreeOpen(name string) (File, error) { +func (m *MemMapFs) lockfreeOpen(name string) (*mem.File, error) { name = normalizePath(name) f, ok := m.getData()[name] - ff, ok := f.(*mem.File) if ok { - ff.Open() - return f, nil + // TODO (mbertschler) + // not used anymore? + //ff.Open() + return mem.NewFileHandle(f), nil } else { return nil, ErrFileNotFound } @@ -284,10 +273,11 @@ func (m *MemMapFs) Rename(oldname, newname string) error { m.mu.RUnlock() m.mu.Lock() m.unRegisterWithParent(oldname) - file := m.getData()[oldname].(*mem.File) + fileData := m.getData()[oldname] + file := mem.NewFileHandle(fileData) delete(m.getData(), oldname) mem.ChangeFileName(file, newname) - m.getData()[newname] = file + m.getData()[newname] = fileData m.registerWithParent(file) m.mu.Unlock() m.mu.RLock() @@ -316,14 +306,11 @@ func (m *MemMapFs) Chmod(name string, mode os.FileMode) error { return &os.PathError{"chmod", name, ErrFileNotFound} } - ff, ok := f.(*mem.File) - if ok { - m.mu.Lock() - mem.SetMode(ff, mode) - m.mu.Unlock() - } else { - return errors.New("Unable to Chmod Memory File") - } + ff := mem.NewFileHandle(f) + m.mu.Lock() + mem.SetMode(ff, mode) + m.mu.Unlock() + return nil } @@ -334,21 +321,19 @@ func (m *MemMapFs) Chtimes(name string, atime time.Time, mtime time.Time) error return &os.PathError{"chtimes", name, ErrFileNotFound} } - ff, ok := f.(*mem.File) - if ok { - m.mu.Lock() - mem.SetModTime(ff, mtime) - m.mu.Unlock() - } else { - return errors.New("Unable to Chtime Memory File") - } + ff := mem.NewFileHandle(f) + m.mu.Lock() + mem.SetModTime(ff, mtime) + m.mu.Unlock() + return nil } func (m *MemMapFs) List() { for _, x := range m.data { - y, _ := x.Stat() - fmt.Println(x.Name(), y.Size()) + f := mem.NewFileHandle(x) + y, _ := f.Stat() + fmt.Println(f.Name(), y.Size()) } } diff --git a/memmap_test.go b/memmap_test.go index b739927..d8de243 100644 --- a/memmap_test.go +++ b/memmap_test.go @@ -106,10 +106,9 @@ func TestMultipleOpenFiles(t *testing.T) { defer removeAllTestFiles(t) const fileName = "./afero-demo2.txt" - var fss = []Fs{&OsFs{}, &MemMapFs{}} - var data = make([][]byte, len(fss)) + var data = make([][]byte, len(Fss)) - for i, fs := range fss { + for i, fs := range Fss { dir := testDir(fs) path := filepath.Join(dir, fileName) fh1, err := fs.Create(path) @@ -120,18 +119,36 @@ func TestMultipleOpenFiles(t *testing.T) { if err != nil { t.Error("fh.Write failed: " + err.Error()) } - fh1.Seek(0, os.SEEK_SET) + _, err = fh1.Seek(0, os.SEEK_SET) + if err != nil { + t.Error(err) + } fh2, err := fs.OpenFile(path, os.O_RDWR, 0777) if err != nil { t.Error("fs.OpenFile failed: " + err.Error()) } - fh2.Seek(0, os.SEEK_END) - fh2.Write([]byte("data")) - fh2.Close() + _, err = fh2.Seek(0, os.SEEK_END) + if err != nil { + t.Error(err) + } + _, err = fh2.Write([]byte("data")) + if err != nil { + t.Error(err) + } + err = fh2.Close() + if err != nil { + t.Error(err) + } - fh1.Write([]byte("data")) - fh1.Close() + _, err = fh1.Write([]byte("data")) + if err != nil { + t.Error(err) + } + err = fh1.Close() + if err != nil { + t.Error(err) + } // the file now should contain "datadata" data[i], err = ReadFile(fs, path) if err != nil { @@ -139,14 +156,14 @@ func TestMultipleOpenFiles(t *testing.T) { } } - for i, fs := range fss { + for i, fs := range Fss { if i == 0 { continue } if string(data[0]) != string(data[i]) { - t.Errorf("OsFs and %s don't behave the same\n"+ - "OsFs: \"%s\"\n%s: \"%s\"\n", - fs.Name(), data[0], fs.Name(), data[i]) + t.Errorf("%s and %s don't behave the same\n"+ + "%s: \"%s\"\n%s: \"%s\"\n", + Fss[0].Name(), fs.Name(), Fss[0].Name(), data[0], fs.Name(), data[i]) } } }