mirror
/
afero
mirror of https://github.com/spf13/afero.git
1
0
Fork 1
Code Issues Packages Projects Releases Wiki Activity

Make the merge func in UnionFile configureable

This commit is contained in:
Bjørn Erik Pedersen 2018-03-28 17:26:24 +02:00
parent 8902da1e4d
commit b6dc11ece0
No known key found for this signature in database
GPG Key ID: 330E6E2BD4859D8F
5 changed files with 111 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
cacheOnReadFs.go
View File

@ -205,7 +205,7 @@ func (u *CacheOnReadFs) OpenFile(name string, flag int, perm os.FileMode) (File,
bfi.Close() // oops, what if O_TRUNC was set and file opening in the layer failed...? bfi.Close() // oops, what if O_TRUNC was set and file opening in the layer failed...?
return nil, err return nil, err
} }
return &UnionFile{base: bfi, layer: lfi}, nil return &UnionFile{Base: bfi, Layer: lfi}, nil
} }
return u.layer.OpenFile(name, flag, perm) return u.layer.OpenFile(name, flag, perm)
} }
@ -251,7 +251,7 @@ func (u *CacheOnReadFs) Open(name string) (File, error) {
if err != nil && bfile == nil { if err != nil && bfile == nil {
return nil, err return nil, err
} }
return &UnionFile{base: bfile, layer: lfile}, nil return &UnionFile{Base: bfile, Layer: lfile}, nil
} }
func (u *CacheOnReadFs) Mkdir(name string, perm os.FileMode) error { func (u *CacheOnReadFs) Mkdir(name string, perm os.FileMode) error {
@ -286,5 +286,5 @@ func (u *CacheOnReadFs) Create(name string) (File, error) {
bfh.Close() bfh.Close()
return nil, err return nil, err
} }
return &UnionFile{base: bfh, layer: lfh}, nil return &UnionFile{Base: bfh, Layer: lfh}, nil
} }

1
composite_test.go
View File

@ -51,7 +51,6 @@ func CleanupTempDirs(t *testing.T) {
func TestUnionCreateExisting(t *testing.T) { func TestUnionCreateExisting(t *testing.T) {
base := &MemMapFs{} base := &MemMapFs{}
roBase := &ReadOnlyFs{source: base} roBase := &ReadOnlyFs{source: base}
ufs := NewCopyOnWriteFs(roBase, &MemMapFs{}) ufs := NewCopyOnWriteFs(roBase, &MemMapFs{})
base.MkdirAll("/home/test", 0777) base.MkdirAll("/home/test", 0777)

2
copyOnWriteFs.go
View File

@ -258,7 +258,7 @@ func (u *CopyOnWriteFs) Open(name string) (File, error) {
return nil, fmt.Errorf("BaseErr: %v\nOverlayErr: %v", bErr, lErr) return nil, fmt.Errorf("BaseErr: %v\nOverlayErr: %v", bErr, lErr)
} }
return &UnionFile{base: bfile, layer: lfile}, nil return &UnionFile{Base: bfile, Layer: lfile}, nil
} }
func (u *CopyOnWriteFs) Mkdir(name string, perm os.FileMode) error { func (u *CopyOnWriteFs) Mkdir(name string, perm os.FileMode) error {

1
copyOnWriteFs_test.go
View File

@ -8,6 +8,7 @@ func TestCopyOnWrite(t *testing.T) {
base := NewOsFs() base := NewOsFs()
roBase := NewReadOnlyFs(base) roBase := NewReadOnlyFs(base)
ufs := NewCopyOnWriteFs(roBase, NewMemMapFs()) ufs := NewCopyOnWriteFs(roBase, NewMemMapFs())
fs = ufs fs = ufs
err = fs.MkdirAll("nonexistent/directory/", 0744) err = fs.MkdirAll("nonexistent/directory/", 0744)
if err != nil { if err != nil {

199
unionFile.go
View File

@ -21,32 +21,33 @@ import (
// successful read in the overlay will move the cursor position in the base layer // successful read in the overlay will move the cursor position in the base layer
// by the number of bytes read. // by the number of bytes read.
type UnionFile struct { type UnionFile struct {
base File Base File
layer File Layer File
off int Merger DirsMerger
files []os.FileInfo off int
files []os.FileInfo
} }
func (f *UnionFile) Close() error { func (f *UnionFile) Close() error {
// first close base, so we have a newer timestamp in the overlay. If we'd close // first close base, so we have a newer timestamp in the overlay. If we'd close
// the overlay first, we'd get a cacheStale the next time we access this file // the overlay first, we'd get a cacheStale the next time we access this file
// -> cache would be useless ;-) // -> cache would be useless ;-)
if f.base != nil { if f.Base != nil {
f.base.Close() f.Base.Close()
} }
if f.layer != nil { if f.Layer != nil {
return f.layer.Close() return f.Layer.Close()
} }
return BADFD return BADFD
} }
func (f *UnionFile) Read(s []byte) (int, error) { func (f *UnionFile) Read(s []byte) (int, error) {
if f.layer != nil { if f.Layer != nil {
n, err := f.layer.Read(s) n, err := f.Layer.Read(s)
if (err == nil || err == io.EOF) && f.base != nil { if (err == nil || err == io.EOF) && f.Base != nil {
// advance the file position also in the base file, the next // advance the file position also in the base file, the next
// call may be a write at this position (or a seek with SEEK_CUR) // call may be a write at this position (or a seek with SEEK_CUR)
if _, seekErr := f.base.Seek(int64(n), os.SEEK_CUR); seekErr != nil { if _, seekErr := f.Base.Seek(int64(n), os.SEEK_CUR); seekErr != nil {
// only overwrite err in case the seek fails: we need to // only overwrite err in case the seek fails: we need to
// report an eventual io.EOF to the caller // report an eventual io.EOF to the caller
err = seekErr err = seekErr
@ -54,123 +55,135 @@ func (f *UnionFile) Read(s []byte) (int, error) {
} }
return n, err return n, err
} }
if f.base != nil { if f.Base != nil {
return f.base.Read(s) return f.Base.Read(s)
} }
return 0, BADFD return 0, BADFD
} }
func (f *UnionFile) ReadAt(s []byte, o int64) (int, error) { func (f *UnionFile) ReadAt(s []byte, o int64) (int, error) {
if f.layer != nil { if f.Layer != nil {
n, err := f.layer.ReadAt(s, o) n, err := f.Layer.ReadAt(s, o)
if (err == nil || err == io.EOF) && f.base != nil { if (err == nil || err == io.EOF) && f.Base != nil {
_, err = f.base.Seek(o+int64(n), os.SEEK_SET) _, err = f.Base.Seek(o+int64(n), os.SEEK_SET)
} }
return n, err return n, err
} }
if f.base != nil { if f.Base != nil {
return f.base.ReadAt(s, o) return f.Base.ReadAt(s, o)
} }
return 0, BADFD return 0, BADFD
} }
func (f *UnionFile) Seek(o int64, w int) (pos int64, err error) { func (f *UnionFile) Seek(o int64, w int) (pos int64, err error) {
if f.layer != nil { if f.Layer != nil {
pos, err = f.layer.Seek(o, w) pos, err = f.Layer.Seek(o, w)
if (err == nil || err == io.EOF) && f.base != nil { if (err == nil || err == io.EOF) && f.Base != nil {
_, err = f.base.Seek(o, w) _, err = f.Base.Seek(o, w)
} }
return pos, err return pos, err
} }
if f.base != nil { if f.Base != nil {
return f.base.Seek(o, w) return f.Base.Seek(o, w)
} }
return 0, BADFD return 0, BADFD
} }
func (f *UnionFile) Write(s []byte) (n int, err error) { func (f *UnionFile) Write(s []byte) (n int, err error) {
if f.layer != nil { if f.Layer != nil {
n, err = f.layer.Write(s) n, err = f.Layer.Write(s)
if err == nil && f.base != nil { // hmm, do we have fixed size files where a write may hit the EOF mark? if err == nil && f.Base != nil { // hmm, do we have fixed size files where a write may hit the EOF mark?
_, err = f.base.Write(s) _, err = f.Base.Write(s)
} }
return n, err return n, err
} }
if f.base != nil { if f.Base != nil {
return f.base.Write(s) return f.Base.Write(s)
} }
return 0, BADFD return 0, BADFD
} }
func (f *UnionFile) WriteAt(s []byte, o int64) (n int, err error) { func (f *UnionFile) WriteAt(s []byte, o int64) (n int, err error) {
if f.layer != nil { if f.Layer != nil {
n, err = f.layer.WriteAt(s, o) n, err = f.Layer.WriteAt(s, o)
if err == nil && f.base != nil { if err == nil && f.Base != nil {
_, err = f.base.WriteAt(s, o) _, err = f.Base.WriteAt(s, o)
} }
return n, err return n, err
} }
if f.base != nil { if f.Base != nil {
return f.base.WriteAt(s, o) return f.Base.WriteAt(s, o)
} }
return 0, BADFD return 0, BADFD
} }
func (f *UnionFile) Name() string { func (f *UnionFile) Name() string {
if f.layer != nil { if f.Layer != nil {
return f.layer.Name() return f.Layer.Name()
} }
return f.base.Name() return f.Base.Name()
} }
// UnionKey can be implemented to use an alternate key than FileInfo.Name when // DirsMerger is how UnionFile weaves two directories together.
// weaving the two directories together. The use case(s) are uncommon and special, // It takes the FileInfo slices from the layer and the base and returns a
// but this will allow a union file system with multiple files with the same filename. // single view.
// This needs to be implemented by your own implementation of os.FileInfo as type DirsMerger func(lofi, bofi []os.FileInfo) ([]os.FileInfo, error)
// returned in Readdir.
type UnionKey interface {
AferoUnionKey() string
}
func (f *UnionFile) key(ofi os.FileInfo) string { var defaultUnionMergeDirsFn = func(lofi, bofi []os.FileInfo) ([]os.FileInfo, error) {
switch tp := ofi.(type) { var files = make(map[string]os.FileInfo)
case UnionKey:
return tp.AferoUnionKey() for _, fi := range lofi {
default: files[fi.Name()] = fi
return tp.Name()
} }
for _, fi := range bofi {
if _, exists := files[fi.Name()]; !exists {
files[fi.Name()] = fi
}
}
rfi := make([]os.FileInfo, len(files))
i := 0
for _, fi := range files {
rfi[i] = fi
i++
}
return rfi, nil
} }
// Readdir will weave the two directories together and // Readdir will weave the two directories together and
// return a single view of the overlayed directories // return a single view of the overlayed directories
func (f *UnionFile) Readdir(c int) (ofi []os.FileInfo, err error) { func (f *UnionFile) Readdir(c int) (ofi []os.FileInfo, err error) {
var merge DirsMerger = f.Merger
if merge == nil {
merge = defaultUnionMergeDirsFn
}
if f.off == 0 { if f.off == 0 {
var files = make(map[string]os.FileInfo) var lfi []os.FileInfo
var rfi []os.FileInfo if f.Layer != nil {
if f.layer != nil { lfi, err = f.Layer.Readdir(-1)
rfi, err = f.layer.Readdir(-1)
if err != nil { if err != nil {
return nil, err return nil, err
} }
for _, fi := range rfi {
files[f.key(fi)] = fi
}
} }
if f.base != nil { var bfi []os.FileInfo
rfi, err = f.base.Readdir(-1) if f.Base != nil {
bfi, err = f.Base.Readdir(-1)
if err != nil { if err != nil {
return nil, err return nil, err
} }
for _, fi := range rfi {
if _, exists := files[fi.Name()]; !exists {
files[f.key(fi)] = fi
}
}
} }
for _, fi := range files { merged, err := merge(lfi, bfi)
f.files = append(f.files, fi) if err != nil {
return nil, err
} }
f.files = append(f.files, merged...)
} }
if c == -1 { if c == -1 {
return f.files[f.off:], nil return f.files[f.off:], nil
@ -192,53 +205,53 @@ func (f *UnionFile) Readdirnames(c int) ([]string, error) {
} }
func (f *UnionFile) Stat() (os.FileInfo, error) { func (f *UnionFile) Stat() (os.FileInfo, error) {
if f.layer != nil { if f.Layer != nil {
return f.layer.Stat() return f.Layer.Stat()
} }
if f.base != nil { if f.Base != nil {
return f.base.Stat() return f.Base.Stat()
} }
return nil, BADFD return nil, BADFD
} }
func (f *UnionFile) Sync() (err error) { func (f *UnionFile) Sync() (err error) {
if f.layer != nil { if f.Layer != nil {
err = f.layer.Sync() err = f.Layer.Sync()
if err == nil && f.base != nil { if err == nil && f.Base != nil {
err = f.base.Sync() err = f.Base.Sync()
} }
return err return err
} }
if f.base != nil { if f.Base != nil {
return f.base.Sync() return f.Base.Sync()
} }
return BADFD return BADFD
} }
func (f *UnionFile) Truncate(s int64) (err error) { func (f *UnionFile) Truncate(s int64) (err error) {
if f.layer != nil { if f.Layer != nil {
err = f.layer.Truncate(s) err = f.Layer.Truncate(s)
if err == nil && f.base != nil { if err == nil && f.Base != nil {
err = f.base.Truncate(s) err = f.Base.Truncate(s)
} }
return err return err
} }
if f.base != nil { if f.Base != nil {
return f.base.Truncate(s) return f.Base.Truncate(s)
} }
return BADFD return BADFD
} }
func (f *UnionFile) WriteString(s string) (n int, err error) { func (f *UnionFile) WriteString(s string) (n int, err error) {
if f.layer != nil { if f.Layer != nil {
n, err = f.layer.WriteString(s) n, err = f.Layer.WriteString(s)
if err == nil && f.base != nil { if err == nil && f.Base != nil {
_, err = f.base.WriteString(s) _, err = f.Base.WriteString(s)
} }
return n, err return n, err
} }
if f.base != nil { if f.Base != nil {
return f.base.WriteString(s) return f.Base.WriteString(s)
} }
return 0, BADFD return 0, BADFD
} }