re
/
afero
forked from mirror/afero
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #55 from spf13/multiple-mem-file-handles 

Seperate MemMapFs file data and file handles
This commit is contained in:
Martin Bertschler 2016-01-04 00:59:32 +01:00
parent e7cb826c1f a5f01f1106
commit 023640316a
5 changed files with 173 additions and 120 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
mem/dir.go
View File

@ -16,20 +16,20 @@ package mem
type Dir interface {
Len() int
Names() []string
Files() []File
Add(File)
Remove(File)
Files() []*FileData
Add(*FileData)
Remove(*FileData)
}
func RemoveFromMemDir(dir *File, f *File) {
dir.memDir.Remove(*f)
func RemoveFromMemDir(dir *FileData, f *FileData) {
dir.memDir.Remove(f)
}
func AddToMemDir(dir *File, f *File) {
dir.memDir.Add(*f)
func AddToMemDir(dir *FileData, f *FileData) {
dir.memDir.Add(f)
}
func InitializeDir(d *File) {
func InitializeDir(d *FileData) {
if d.memDir == nil {
d.dir = true
d.memDir = &DirMap{}

16
mem/dirmap.go
View File

@ -15,12 +15,12 @@ package mem
import "sort"
type DirMap map[string]File
type DirMap map[string]*FileData
func (m DirMap) Len() int { return len(m) }
func (m DirMap) Add(f File) { m[f.Name()] = f }
func (m DirMap) Remove(f File) { delete(m, f.Name()) }
func (m DirMap) Files() (files []File) {
func (m DirMap) Len() int { return len(m) }
func (m DirMap) Add(f *FileData) { m[f.name] = f }
func (m DirMap) Remove(f *FileData) { delete(m, f.name) }
func (m DirMap) Files() (files []*FileData) {
for _, f := range m {
files = append(files, f)
}
@ -28,12 +28,12 @@ func (m DirMap) Files() (files []File) {
return files
}
type filesSorter []File
// implement sort.Interface for []*FileData
type filesSorter []*FileData
// implement sort.Interface for []File
func (s filesSorter) Len() int { return len(s) }
func (s filesSorter) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
func (s filesSorter) Less(i, j int) bool { return s[i].Name() < s[j].Name() }
func (s filesSorter) Less(i, j int) bool { return s[i].name < s[j].name }
func (m DirMap) Names() (names []string) {
for x := range m {

111
mem/file.go
View File

@ -32,63 +32,78 @@ type File struct {
// atomic requires 64-bit alignment for struct field access
at int64
readDirCount int64
closed bool
fileData *FileData
}
func NewFileHandle(data *FileData) *File {
return &File{fileData: data}
}
func (f File) Data() *FileData {
return f.fileData
}
type FileData struct {
sync.Mutex
name string
data []byte
memDir Dir
dir bool
closed bool
mode os.FileMode
modtime time.Time
}
func CreateFile(name string) *File {
return &File{name: name, mode: os.ModeTemporary, modtime: time.Now()}
func (d FileData) Name() string {
return d.name
}
func CreateDir(name string) *File {
return &File{name: name, memDir: &DirMap{}, dir: true}
func CreateFile(name string) *FileData {
return &FileData{name: name, mode: os.ModeTemporary, modtime: time.Now()}
}
func ChangeFileName(f *File, newname string) {
func CreateDir(name string) *FileData {
return &FileData{name: name, memDir: &DirMap{}, dir: true}
}
func ChangeFileName(f *FileData, newname string) {
f.name = newname
}
func SetMode(f *File, mode os.FileMode) {
func SetMode(f *FileData, mode os.FileMode) {
f.mode = mode
}
func SetModTime(f *File, mtime time.Time) {
func SetModTime(f *FileData, mtime time.Time) {
f.modtime = mtime
}
func GetFileInfo(f *File) *FileInfo {
return &FileInfo{file: f}
func GetFileInfo(f *FileData) *FileInfo {
return &FileInfo{f}
}
func (f *File) Open() error {
atomic.StoreInt64(&f.at, 0)
atomic.StoreInt64(&f.readDirCount, 0)
f.Lock()
f.fileData.Lock()
f.closed = false
f.Unlock()
f.fileData.Unlock()
return nil
}
func (f *File) Close() error {
f.Lock()
f.fileData.Lock()
f.closed = true
f.Unlock()
f.fileData.Unlock()
return nil
}
func (f *File) Name() string {
return f.name
return f.fileData.name
}
func (f *File) Stat() (os.FileInfo, error) {
return &FileInfo{f}, nil
return &FileInfo{f.fileData}, nil
}
func (f *File) Sync() error {
@ -98,8 +113,8 @@ func (f *File) Sync() error {
func (f *File) Readdir(count int) (res []os.FileInfo, err error) {
var outLength int64
f.Lock()
files := f.memDir.Files()[f.readDirCount:]
f.fileData.Lock()
files := f.fileData.memDir.Files()[f.readDirCount:]
if count > 0 {
if len(files) < count {
outLength = int64(len(files))
@ -113,11 +128,11 @@ func (f *File) Readdir(count int) (res []os.FileInfo, err error) {
outLength = int64(len(files))
}
f.readDirCount += outLength
f.Unlock()
f.fileData.Unlock()
res = make([]os.FileInfo, outLength)
for i := range res {
res[i], _ = files[i].Stat()
res[i] = &FileInfo{files[i]}
}
return res, err
@ -133,20 +148,20 @@ func (f *File) Readdirnames(n int) (names []string, err error) {
}
func (f *File) Read(b []byte) (n int, err error) {
f.Lock()
defer f.Unlock()
f.fileData.Lock()
defer f.fileData.Unlock()
if f.closed == true {
return 0, ErrFileClosed
}
if len(b) > 0 && int(f.at) == len(f.data) {
if len(b) > 0 && int(f.at) == len(f.fileData.data) {
return 0, io.EOF
}
if len(f.data)-int(f.at) >= len(b) {
if len(f.fileData.data)-int(f.at) >= len(b) {
n = len(b)
} else {
n = len(f.data) - int(f.at)
n = len(f.fileData.data) - int(f.at)
}
copy(b, f.data[f.at:f.at+int64(n)])
copy(b, f.fileData.data[f.at:f.at+int64(n)])
atomic.AddInt64(&f.at, int64(n))
return
}
@ -163,11 +178,11 @@ func (f *File) Truncate(size int64) error {
if size < 0 {
return ErrOutOfRange
}
if size > int64(len(f.data)) {
diff := size - int64(len(f.data))
f.data = append(f.data, bytes.Repeat([]byte{00}, int(diff))...)
if size > int64(len(f.fileData.data)) {
diff := size - int64(len(f.fileData.data))
f.fileData.data = append(f.fileData.data, bytes.Repeat([]byte{00}, int(diff))...)
} else {
f.data = f.data[0:size]
f.fileData.data = f.fileData.data[0:size]
}
return nil
}
@ -182,7 +197,7 @@ func (f *File) Seek(offset int64, whence int) (int64, error) {
case 1:
atomic.AddInt64(&f.at, int64(offset))
case 2:
atomic.StoreInt64(&f.at, int64(len(f.data))+offset)
atomic.StoreInt64(&f.at, int64(len(f.fileData.data))+offset)
}
return f.at, nil
}
@ -190,22 +205,22 @@ func (f *File) Seek(offset int64, whence int) (int64, error) {
func (f *File) Write(b []byte) (n int, err error) {
n = len(b)
cur := atomic.LoadInt64(&f.at)
f.Lock()
defer f.Unlock()
diff := cur - int64(len(f.data))
f.fileData.Lock()
defer f.fileData.Unlock()
diff := cur - int64(len(f.fileData.data))
var tail []byte
if n+int(cur) < len(f.data) {
tail = f.data[n+int(cur):]
if n+int(cur) < len(f.fileData.data) {
tail = f.fileData.data[n+int(cur):]
}
if diff > 0 {
f.data = append(bytes.Repeat([]byte{00}, int(diff)), b...)
f.data = append(f.data, tail...)
f.fileData.data = append(bytes.Repeat([]byte{00}, int(diff)), b...)
f.fileData.data = append(f.fileData.data, tail...)
} else {
f.data = append(f.data[:cur], b...)
f.data = append(f.data, tail...)
f.fileData.data = append(f.fileData.data[:cur], b...)
f.fileData.data = append(f.fileData.data, tail...)
}
atomic.StoreInt64(&f.at, int64(len(f.data)))
atomic.StoreInt64(&f.at, int64(len(f.fileData.data)))
return
}
@ -219,27 +234,27 @@ func (f *File) WriteString(s string) (ret int, err error) {
}
func (f *File) Info() *FileInfo {
return &FileInfo{file: f}
return &FileInfo{f.fileData}
}
type FileInfo struct {
file *File
*FileData
}
// Implements os.FileInfo
func (s *FileInfo) Name() string {
_, name := filepath.Split(s.file.Name())
_, name := filepath.Split(s.name)
return name
}
func (s *FileInfo) Mode() os.FileMode { return s.file.mode }
func (s *FileInfo) ModTime() time.Time { return s.file.modtime }
func (s *FileInfo) IsDir() bool { return s.file.dir }
func (s *FileInfo) Mode() os.FileMode { return s.mode }
func (s *FileInfo) ModTime() time.Time { return s.modtime }
func (s *FileInfo) IsDir() bool { return s.dir }
func (s *FileInfo) Sys() interface{} { return nil }
func (s *FileInfo) Size() int64 {
if s.IsDir() {
return int64(42)
}
return int64(len(s.file.data))
return int64(len(s.data))
}
var (

83
memmap.go
View File

@ -14,7 +14,6 @@
package afero
import (
"errors"
"fmt"
"log"
"os"
@ -28,15 +27,15 @@ import (
type MemMapFs struct {
mu sync.RWMutex
data map[string]File
data map[string]*mem.FileData
init sync.Once
}
var memfsInit sync.Once
func (m *MemMapFs) getData() map[string]File {
func (m *MemMapFs) getData() map[string]*mem.FileData {
m.init.Do(func() {
m.data = make(map[string]File)
m.data = make(map[string]*mem.FileData)
// Root should always exist, right?
// TODO: what about windows?
m.data[FilePathSeparator] = mem.CreateDir(FilePathSeparator)
@ -53,7 +52,7 @@ func (m *MemMapFs) Create(name string) (File, error) {
m.getData()[name] = file
m.registerWithParent(file)
m.mu.Unlock()
return file, nil
return mem.NewFileHandle(file), nil
}
func (m *MemMapFs) unRegisterWithParent(fileName string) error {
@ -65,13 +64,11 @@ func (m *MemMapFs) unRegisterWithParent(fileName string) error {
if parent == nil {
log.Fatal("parent of ", f.Name(), " is nil")
}
pmem := parent.(*mem.File)
fmem := f.(*mem.File)
mem.RemoveFromMemDir(pmem, fmem)
mem.RemoveFromMemDir(parent, f)
return nil
}
func (m *MemMapFs) findParent(f File) File {
func (m *MemMapFs) findParent(f *mem.FileData) *mem.FileData {
pdir, _ := filepath.Split(f.Name())
pdir = filepath.Clean(pdir)
pfile, err := m.lockfreeOpen(pdir)
@ -81,7 +78,7 @@ func (m *MemMapFs) findParent(f File) File {
return pfile
}
func (m *MemMapFs) registerWithParent(f File) {
func (m *MemMapFs) registerWithParent(f *mem.FileData) {
if f == nil {
return
}
@ -99,15 +96,9 @@ func (m *MemMapFs) registerWithParent(f File) {
return
}
}
pmem := parent.(*mem.File)
fmem := f.(*mem.File)
// TODO(mbertschler): memDir is only nil when it was not made with Mkdir
// or lockfreeMkdir. In this case the parent is also not a real directory.
// This currently only happens for the file ".".
// This is a quick hack to make the library usable with relative paths.
mem.InitializeDir(pmem)
mem.AddToMemDir(pmem, fmem)
mem.InitializeDir(parent)
mem.AddToMemDir(parent, f)
}
func (m *MemMapFs) lockfreeMkdir(name string, perm os.FileMode) error {
@ -115,13 +106,10 @@ func (m *MemMapFs) lockfreeMkdir(name string, perm os.FileMode) error {
x, ok := m.getData()[name]
if ok {
// Only return ErrFileExists if it's a file, not a directory.
i, err := x.Stat()
i := mem.FileInfo{x}
if !i.IsDir() {
return ErrFileExists
}
if err != nil {
return err
}
} else {
item := mem.CreateDir(name)
m.getData()[name] = item
@ -179,26 +167,17 @@ func (m *MemMapFs) Open(name string) (File, error) {
m.mu.RLock()
f, ok := m.getData()[name]
ff, ok := f.(*mem.File)
if ok {
ff.Open()
}
m.mu.RUnlock()
if ok {
return f, nil
} else {
if !ok {
return nil, &os.PathError{"open", name, ErrFileNotFound}
}
return mem.NewFileHandle(f), nil
}
func (m *MemMapFs) lockfreeOpen(name string) (File, error) {
func (m *MemMapFs) lockfreeOpen(name string) (*mem.FileData, error) {
name = normalizePath(name)
f, ok := m.getData()[name]
ff, ok := f.(*mem.File)
if ok {
ff.Open()
return f, nil
} else {
return nil, ErrFileNotFound
@ -284,11 +263,11 @@ func (m *MemMapFs) Rename(oldname, newname string) error {
m.mu.RUnlock()
m.mu.Lock()
m.unRegisterWithParent(oldname)
file := m.getData()[oldname].(*mem.File)
fileData := m.getData()[oldname]
delete(m.getData(), oldname)
mem.ChangeFileName(file, newname)
m.getData()[newname] = file
m.registerWithParent(file)
mem.ChangeFileName(fileData, newname)
m.getData()[newname] = fileData
m.registerWithParent(fileData)
m.mu.Unlock()
m.mu.RLock()
} else {
@ -305,7 +284,7 @@ func (m *MemMapFs) Stat(name string) (os.FileInfo, error) {
if err != nil {
return nil, err
}
fi := mem.GetFileInfo(f.(*mem.File))
fi := mem.GetFileInfo(f.(*mem.File).Data())
return fi, nil
}
@ -316,14 +295,10 @@ func (m *MemMapFs) Chmod(name string, mode os.FileMode) error {
return &os.PathError{"chmod", name, ErrFileNotFound}
}
ff, ok := f.(*mem.File)
if ok {
m.mu.Lock()
mem.SetMode(ff, mode)
m.mu.Unlock()
} else {
return errors.New("Unable to Chmod Memory File")
}
m.mu.Lock()
mem.SetMode(f, mode)
m.mu.Unlock()
return nil
}
@ -334,20 +309,16 @@ func (m *MemMapFs) Chtimes(name string, atime time.Time, mtime time.Time) error
return &os.PathError{"chtimes", name, ErrFileNotFound}
}
ff, ok := f.(*mem.File)
if ok {
m.mu.Lock()
mem.SetModTime(ff, mtime)
m.mu.Unlock()
} else {
return errors.New("Unable to Chtime Memory File")
}
m.mu.Lock()
mem.SetModTime(f, mtime)
m.mu.Unlock()
return nil
}
func (m *MemMapFs) List() {
for _, x := range m.data {
y, _ := x.Stat()
y := mem.FileInfo{x}
fmt.Println(x.Name(), y.Size())
}
}

67
memmap_test.go
View File

@ -100,3 +100,70 @@ func checkPathError(t *testing.T, err error, op string) {
t.Error(op+":", err, "contains another os.PathError")
}
}
// Fails if multiple file objects use the same file.at counter in MemMapFs
func TestMultipleOpenFiles(t *testing.T) {
defer removeAllTestFiles(t)
const fileName = "./afero-demo2.txt"
var data = make([][]byte, len(Fss))
for i, fs := range Fss {
dir := testDir(fs)
path := filepath.Join(dir, fileName)
fh1, err := fs.Create(path)
if err != nil {
t.Error("os.Create failed: " + err.Error())
}
_, err = fh1.Write([]byte("test"))
if err != nil {
t.Error("fh.Write failed: " + err.Error())
}
_, err = fh1.Seek(0, os.SEEK_SET)
if err != nil {
t.Error(err)
}
fh2, err := fs.OpenFile(path, os.O_RDWR, 0777)
if err != nil {
t.Error("fs.OpenFile failed: " + err.Error())
}
_, err = fh2.Seek(0, os.SEEK_END)
if err != nil {
t.Error(err)
}
_, err = fh2.Write([]byte("data"))
if err != nil {
t.Error(err)
}
err = fh2.Close()
if err != nil {
t.Error(err)
}
_, err = fh1.Write([]byte("data"))
if err != nil {
t.Error(err)
}
err = fh1.Close()
if err != nil {
t.Error(err)
}
// the file now should contain "datadata"
data[i], err = ReadFile(fs, path)
if err != nil {
t.Error(err)
}
}
for i, fs := range Fss {
if i == 0 {
continue
}
if string(data[0]) != string(data[i]) {
t.Errorf("%s and %s don't behave the same\n"+
"%s: \"%s\"\n%s: \"%s\"\n",
Fss[0].Name(), fs.Name(), Fss[0].Name(), data[0], fs.Name(), data[i])
}
}
}