Compare commits

..

9 Commits

Author SHA1 Message Date
re 1707ad0dd8 fix repos 2022-12-12 18:01:24 +03:00
Bjørn Erik Pedersen a800a9de53 Fix concurrency issue in MemMapFs.Mkdir/MkdirAll
* The backing map is protected by a RWMutex
* This commit double checks for the existence of the directory inside the write lock to avoid potential data races when multiple goroutines tries to create
the same directory.

Fixes #361
Fixes #298
2022-11-14 18:51:03 +01:00
Bjørn Erik Pedersen 2a70f2bb2d Make mem.File implement fs.ReadDirFile 2022-07-19 12:29:51 +02:00
Bjørn Erik Pedersen 0aa65edf44 Fix sorting in IOFS.ReadDir
We recently added a check for fs.ReadDirFile in IOFS.ReadDir, but forgot to apply a sort to the
result as defined in the spec.

This fixes that and adds a test case for it.
2022-07-19 11:13:10 +02:00
Bjørn Erik Pedersen b0a534a781
Update README.md 2022-07-14 20:00:13 +02:00
Bjørn Erik Pedersen c92ae364de Make IOFS.ReadDir check for fs.ReadDirFile
And implement `fs.ReadDirFile` for `BasePathFile`.

There are other `File` implementations that could also benefit from the above, but
this is a start.

The primary motivation behind this is to allow `fs.WalkDir` use the native
implementation whenever possible, as that new function was added in Go 1.16 with
speed as its main selling point.

This commit also adds a benchmark for `fs.WalkDir` that, when compared to the main branch:

```bash
name        old time/op    new time/op    delta
WalkDir-10     369µs ± 1%     196µs ± 3%  -46.89%  (p=0.029 n=4+4)

name        old alloc/op   new alloc/op   delta
WalkDir-10    85.3kB ± 0%    40.2kB ± 0%  -52.87%  (p=0.029 n=4+4)

name        old allocs/op  new allocs/op  delta
WalkDir-10       826 ± 0%       584 ± 0%  -29.30%  (p=0.029 n=4+4)
```

Fixes #365
2022-07-14 15:11:09 +02:00
Bjørn Erik Pedersen 52b64170ec Fix staticcheck lint errors 2022-07-14 14:33:54 +02:00
Bjørn Erik Pedersen 939bf3d6b2 Fix test failures on Windows
And also enable the CI build for Windows.
2022-07-14 13:05:08 +02:00
Bjørn Erik Pedersen 9439436a4d all: Run gofmt -s -w 2022-07-14 13:05:08 +02:00
36 changed files with 479 additions and 141 deletions

View File

@ -8,8 +8,7 @@ jobs:
strategy: strategy:
matrix: matrix:
go-version: [1.16.x,1.17.x,1.18.x] go-version: [1.16.x,1.17.x,1.18.x]
# TODO(bep) fix windows-latest platform: [ubuntu-latest, macos-latest, windows-latest]
platform: [ubuntu-latest, macos-latest]
runs-on: ${{ matrix.platform }} runs-on: ${{ matrix.platform }}
steps: steps:
- name: Install Go - name: Install Go
@ -25,14 +24,14 @@ jobs:
shell: bash shell: bash
- name: Checkout code - name: Checkout code
uses: actions/checkout@v1 uses: actions/checkout@v1
#- name: Fmt - name: Fmt
# if: matrix.platform != 'windows-latest' # :( if: matrix.platform != 'windows-latest' # :(
# run: "diff <(gofmt -d .) <(printf '')" run: "diff <(gofmt -d .) <(printf '')"
# shell: bash shell: bash
- name: Vet - name: Vet
run: go vet ./... run: go vet ./...
#- name: Staticcheck - name: Staticcheck
# if: matrix.go-version != '1.16.x' if: matrix.go-version != '1.16.x'
# run: staticcheck ./... run: staticcheck ./...
- name: Test - name: Test
run: go test -race ./... run: go test -race ./...

View File

@ -2,7 +2,7 @@
A FileSystem Abstraction System for Go A FileSystem Abstraction System for Go
[![Build Status](https://travis-ci.org/spf13/afero.svg)](https://travis-ci.org/spf13/afero) [![Build status](https://ci.appveyor.com/api/projects/status/github/spf13/afero?branch=master&svg=true)](https://ci.appveyor.com/project/spf13/afero) [![GoDoc](https://godoc.org/github.com/spf13/afero?status.svg)](https://godoc.org/github.com/spf13/afero) [![Join the chat at https://gitter.im/spf13/afero](https://badges.gitter.im/Dev%20Chat.svg)](https://gitter.im/spf13/afero?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) [![Test](https://git.internal/re/afero/actions/workflows/test.yml/badge.svg)](https://git.internal/re/afero/actions/workflows/test.yml) [![GoDoc](https://godoc.org/git.internal/re/afero?status.svg)](https://godoc.org/git.internal/re/afero) [![Join the chat at https://gitter.im/spf13/afero](https://badges.gitter.im/Dev%20Chat.svg)](https://gitter.im/spf13/afero?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
# Overview # Overview
@ -50,11 +50,11 @@ A few different ways you could use Afero:
First use go get to install the latest version of the library. First use go get to install the latest version of the library.
$ go get github.com/spf13/afero $ go get git.internal/re/afero
Next include Afero in your application. Next include Afero in your application.
```go ```go
import "github.com/spf13/afero" import "git.internal/re/afero"
``` ```
## Step 2: Declare a backend ## Step 2: Declare a backend
@ -152,7 +152,7 @@ Walk(root string, walkFn filepath.WalkFunc) error
WriteFile(filename string, data []byte, perm os.FileMode) error WriteFile(filename string, data []byte, perm os.FileMode) error
WriteReader(path string, r io.Reader) (err error) WriteReader(path string, r io.Reader) (err error)
``` ```
For a complete list see [Afero's GoDoc](https://godoc.org/github.com/spf13/afero) For a complete list see [Afero's GoDoc](https://godoc.org/git.internal/re/afero)
They are available under two different approaches to use. You can either call They are available under two different approaches to use. You can either call
them directly where the first parameter of each function will be the file them directly where the first parameter of each function will be the file
@ -417,7 +417,7 @@ Googles very well.
## Release Notes ## Release Notes
See the [Releases Page](https://github.com/spf13/afero/releases). See the [Releases Page](https://git.internal/re/afero/releases).
## Contributing ## Contributing
@ -439,4 +439,4 @@ Names in no particular order:
## License ## License
Afero is released under the Apache 2.0 license. See Afero is released under the Apache 2.0 license. See
[LICENSE.txt](https://github.com/spf13/afero/blob/master/LICENSE.txt) [LICENSE.txt](https://git.internal/re/afero/blob/master/LICENSE.txt)

View File

@ -103,8 +103,8 @@ type Fs interface {
var ( var (
ErrFileClosed = errors.New("File is closed") ErrFileClosed = errors.New("File is closed")
ErrOutOfRange = errors.New("Out of range") ErrOutOfRange = errors.New("out of range")
ErrTooLarge = errors.New("Too large") ErrTooLarge = errors.New("too large")
ErrFileNotFound = os.ErrNotExist ErrFileNotFound = os.ErrNotExist
ErrFileExists = os.ErrExist ErrFileExists = os.ErrExist
ErrDestinationExists = os.ErrExist ErrDestinationExists = os.ErrExist

View File

@ -18,6 +18,7 @@ import (
"bytes" "bytes"
"fmt" "fmt"
"io" "io"
iofs "io/fs"
"io/ioutil" "io/ioutil"
"os" "os"
"path/filepath" "path/filepath"
@ -27,8 +28,10 @@ import (
"testing" "testing"
) )
var testName = "test.txt" var (
var Fss = []Fs{&MemMapFs{}, &OsFs{}} testName = "test.txt"
Fss = []Fs{&MemMapFs{}, &OsFs{}}
)
var testRegistry map[Fs][]string = make(map[Fs][]string) var testRegistry map[Fs][]string = make(map[Fs][]string)
@ -44,7 +47,6 @@ func testDir(fs Fs) string {
func tmpFile(fs Fs) File { func tmpFile(fs Fs) File {
x, err := TempFile(fs, "", "afero") x, err := TempFile(fs, "", "afero")
if err != nil { if err != nil {
panic(fmt.Sprint("unable to work with temp file", err)) panic(fmt.Sprint("unable to work with temp file", err))
} }
@ -54,7 +56,7 @@ func tmpFile(fs Fs) File {
return x return x
} }
//Read with length 0 should not return EOF. // Read with length 0 should not return EOF.
func TestRead0(t *testing.T) { func TestRead0(t *testing.T) {
for _, fs := range Fss { for _, fs := range Fss {
f := tmpFile(fs) f := tmpFile(fs)
@ -82,7 +84,7 @@ func TestOpenFile(t *testing.T) {
tmp := testDir(fs) tmp := testDir(fs)
path := filepath.Join(tmp, testName) path := filepath.Join(tmp, testName)
f, err := fs.OpenFile(path, os.O_RDWR|os.O_CREATE, 0600) f, err := fs.OpenFile(path, os.O_RDWR|os.O_CREATE, 0o600)
if err != nil { if err != nil {
t.Error(fs.Name(), "OpenFile (O_CREATE) failed:", err) t.Error(fs.Name(), "OpenFile (O_CREATE) failed:", err)
continue continue
@ -90,7 +92,7 @@ func TestOpenFile(t *testing.T) {
io.WriteString(f, "initial") io.WriteString(f, "initial")
f.Close() f.Close()
f, err = fs.OpenFile(path, os.O_WRONLY|os.O_APPEND, 0600) f, err = fs.OpenFile(path, os.O_WRONLY|os.O_APPEND, 0o600)
if err != nil { if err != nil {
t.Error(fs.Name(), "OpenFile (O_APPEND) failed:", err) t.Error(fs.Name(), "OpenFile (O_APPEND) failed:", err)
continue continue
@ -98,7 +100,7 @@ func TestOpenFile(t *testing.T) {
io.WriteString(f, "|append") io.WriteString(f, "|append")
f.Close() f.Close()
f, err = fs.OpenFile(path, os.O_RDONLY, 0600) f, _ = fs.OpenFile(path, os.O_RDONLY, 0o600)
contents, _ := ioutil.ReadAll(f) contents, _ := ioutil.ReadAll(f)
expectedContents := "initial|append" expectedContents := "initial|append"
if string(contents) != expectedContents { if string(contents) != expectedContents {
@ -106,7 +108,7 @@ func TestOpenFile(t *testing.T) {
} }
f.Close() f.Close()
f, err = fs.OpenFile(path, os.O_RDWR|os.O_TRUNC, 0600) f, err = fs.OpenFile(path, os.O_RDWR|os.O_TRUNC, 0o600)
if err != nil { if err != nil {
t.Error(fs.Name(), "OpenFile (O_TRUNC) failed:", err) t.Error(fs.Name(), "OpenFile (O_TRUNC) failed:", err)
continue continue
@ -332,7 +334,7 @@ func TestSeek(t *testing.T) {
whence int whence int
out int64 out int64
} }
var tests = []test{ tests := []test{
{0, 1, int64(len(data))}, {0, 1, int64(len(data))},
{0, 0, 0}, {0, 0, 0},
{5, 0, 5}, {5, 0, 5},
@ -423,7 +425,7 @@ func setupTestDirReusePath(t *testing.T, fs Fs, path string) string {
func setupTestFiles(t *testing.T, fs Fs, path string) string { func setupTestFiles(t *testing.T, fs Fs, path string) string {
testSubDir := filepath.Join(path, "more", "subdirectories", "for", "testing", "we") testSubDir := filepath.Join(path, "more", "subdirectories", "for", "testing", "we")
err := fs.MkdirAll(testSubDir, 0700) err := fs.MkdirAll(testSubDir, 0o700)
if err != nil && !os.IsExist(err) { if err != nil && !os.IsExist(err) {
t.Fatal(err) t.Fatal(err)
} }
@ -530,22 +532,43 @@ func TestReaddirSimple(t *testing.T) {
func TestReaddir(t *testing.T) { func TestReaddir(t *testing.T) {
defer removeAllTestFiles(t) defer removeAllTestFiles(t)
for num := 0; num < 6; num++ { const nums = 6
for num := 0; num < nums; num++ {
outputs := make([]string, len(Fss)) outputs := make([]string, len(Fss))
infos := make([]string, len(Fss)) infos := make([]string, len(Fss))
for i, fs := range Fss { for i, fs := range Fss {
testSubDir := setupTestDir(t, fs) testSubDir := setupTestDir(t, fs)
//tDir := filepath.Dir(testSubDir)
root, err := fs.Open(testSubDir) root, err := fs.Open(testSubDir)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
infosn := make([]string, nums)
for j := 0; j < nums; j++ {
info, err := root.Readdir(num)
outputs[i] += fmt.Sprintf("%v Error: %v\n", myFileInfo(info), err)
s := fmt.Sprintln(len(info), err)
infosn[j] = s
infos[i] += s
}
root.Close()
// Also check fs.ReadDirFile interface if implemented
if _, ok := root.(iofs.ReadDirFile); ok {
root, err = fs.Open(testSubDir)
if err != nil {
t.Fatal(err)
}
defer root.Close() defer root.Close()
for j := 0; j < 6; j++ { for j := 0; j < nums; j++ {
info, err := root.Readdir(num) dirEntries, err := root.(iofs.ReadDirFile).ReadDir(num)
outputs[i] += fmt.Sprintf("%v Error: %v\n", myFileInfo(info), err) s := fmt.Sprintln(len(dirEntries), err)
infos[i] += fmt.Sprintln(len(info), err) if s != infosn[j] {
t.Fatalf("%s: %s != %s", fs.Name(), s, infosn[j])
}
}
} }
} }
@ -570,7 +593,7 @@ func TestReaddir(t *testing.T) {
} }
} }
// https://github.com/spf13/afero/issues/169 // https://git.internal/re/afero/issues/169
func TestReaddirRegularFile(t *testing.T) { func TestReaddirRegularFile(t *testing.T) {
defer removeAllTestFiles(t) defer removeAllTestFiles(t)
for _, fs := range Fss { for _, fs := range Fss {
@ -615,7 +638,7 @@ func TestReaddirAll(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
var namesRoot = []string{} namesRoot := []string{}
for _, e := range rootInfo { for _, e := range rootInfo {
namesRoot = append(namesRoot, e.Name()) namesRoot = append(namesRoot, e.Name())
} }
@ -630,7 +653,7 @@ func TestReaddirAll(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
var namesSub = []string{} namesSub := []string{}
for _, e := range subInfo { for _, e := range subInfo {
namesSub = append(namesSub, e.Name()) namesSub = append(namesSub, e.Name())
} }
@ -700,7 +723,7 @@ func removeAllTestFiles(t *testing.T) {
func equal(name1, name2 string) (r bool) { func equal(name1, name2 string) (r bool) {
switch runtime.GOOS { switch runtime.GOOS {
case "windows": case "windows":
r = strings.ToLower(name1) == strings.ToLower(name2) r = strings.EqualFold(name1, name2)
default: default:
r = name1 == name2 r = name1 == name2
} }

View File

@ -1,6 +1,7 @@
package afero package afero
import ( import (
"io/fs"
"os" "os"
"path/filepath" "path/filepath"
"runtime" "runtime"
@ -8,7 +9,10 @@ import (
"time" "time"
) )
var _ Lstater = (*BasePathFs)(nil) var (
_ Lstater = (*BasePathFs)(nil)
_ fs.ReadDirFile = (*BasePathFile)(nil)
)
// The BasePathFs restricts all operations to a given path within an Fs. // The BasePathFs restricts all operations to a given path within an Fs.
// The given file name to the operations on this Fs will be prepended with // The given file name to the operations on this Fs will be prepended with
@ -33,6 +37,14 @@ func (f *BasePathFile) Name() string {
return strings.TrimPrefix(sourcename, filepath.Clean(f.path)) return strings.TrimPrefix(sourcename, filepath.Clean(f.path))
} }
func (f *BasePathFile) ReadDir(n int) ([]fs.DirEntry, error) {
if rdf, ok := f.File.(fs.ReadDirFile); ok {
return rdf.ReadDir(n)
}
return readDirFile{f.File}.ReadDir(n)
}
func NewBasePathFs(source Fs, path string) Fs { func NewBasePathFs(source Fs, path string) Fs {
return &BasePathFs{source: source, path: path} return &BasePathFs{source: source, path: path}
} }

View File

@ -79,7 +79,7 @@ func TestUnionCreateExisting(t *testing.T) {
fh.Close() fh.Close()
fh, _ = base.Open("/home/test/file.txt") fh, _ = base.Open("/home/test/file.txt")
data, err = ioutil.ReadAll(fh) data, _ = ioutil.ReadAll(fh)
if string(data) != "This is a test" { if string(data) != "This is a test" {
t.Errorf("Got wrong data in base file") t.Errorf("Got wrong data in base file")
} }
@ -326,9 +326,9 @@ func TestUnionCacheWrite(t *testing.T) {
t.Errorf("Failed to write file") t.Errorf("Failed to write file")
} }
fh.Seek(0, os.SEEK_SET) fh.Seek(0, io.SeekStart)
buf := make([]byte, 4) buf := make([]byte, 4)
_, err = fh.Read(buf) _, _ = fh.Read(buf)
fh.Write([]byte(" IS A")) fh.Write([]byte(" IS A"))
fh.Close() fh.Close()

View File

@ -11,6 +11,7 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
//go:build aix || darwin || openbsd || freebsd || netbsd || dragonfly
// +build aix darwin openbsd freebsd netbsd dragonfly // +build aix darwin openbsd freebsd netbsd dragonfly
package afero package afero

View File

@ -10,12 +10,8 @@
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
// +build !darwin //go:build !darwin && !openbsd && !freebsd && !dragonfly && !netbsd && !aix
// +build !openbsd // +build !darwin,!openbsd,!freebsd,!dragonfly,!netbsd,!aix
// +build !freebsd
// +build !dragonfly
// +build !netbsd
// +build !aix
package afero package afero

View File

@ -16,9 +16,9 @@ func TestCopyOnWrite(t *testing.T) {
compositeFs := NewCopyOnWriteFs(NewReadOnlyFs(NewOsFs()), osFs) compositeFs := NewCopyOnWriteFs(NewReadOnlyFs(NewOsFs()), osFs)
var dir = filepath.Join(writeDir, "some/path") dir := filepath.Join(writeDir, "some/path")
err = compositeFs.MkdirAll(dir, 0744) err = compositeFs.MkdirAll(dir, 0o744)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -27,17 +27,17 @@ func TestCopyOnWrite(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
// https://github.com/spf13/afero/issues/189 // https://git.internal/re/afero/issues/189
// We want the composite file system to behave like the OS file system // We want the composite file system to behave like the OS file system
// on Mkdir and MkdirAll // on Mkdir and MkdirAll
for _, fs := range []Fs{osFs, compositeFs} { for _, fs := range []Fs{osFs, compositeFs} {
err = fs.Mkdir(dir, 0744) err = fs.Mkdir(dir, 0o744)
if err == nil || !os.IsExist(err) { if err == nil || !os.IsExist(err) {
t.Errorf("Mkdir: Got %q for %T", err, fs) t.Errorf("Mkdir: Got %q for %T", err, fs)
} }
// MkdirAll does not return an error when the directory already exists // MkdirAll does not return an error when the directory already exists
err = fs.MkdirAll(dir, 0744) err = fs.MkdirAll(dir, 0o744)
if err != nil { if err != nil {
t.Errorf("MkdirAll: Got %q for %T", err, fs) t.Errorf("MkdirAll: Got %q for %T", err, fs)
} }
@ -49,7 +49,7 @@ func TestCopyOnWriteFileInMemMapBase(t *testing.T) {
base := &MemMapFs{} base := &MemMapFs{}
layer := &MemMapFs{} layer := &MemMapFs{}
if err := WriteFile(base, "base.txt", []byte("base"), 0755); err != nil { if err := WriteFile(base, "base.txt", []byte("base"), 0o755); err != nil {
t.Fatalf("Failed to write file: %s", err) t.Fatalf("Failed to write file: %s", err)
} }

View File

@ -302,6 +302,9 @@ func (fs *Fs) Remove(name string) error {
} }
var infos []os.FileInfo var infos []os.FileInfo
infos, err = dir.Readdir(0) infos, err = dir.Readdir(0)
if err != nil {
return err
}
if len(infos) > 0 { if len(infos) > 0 {
return syscall.ENOTEMPTY return syscall.ENOTEMPTY
} }
@ -345,6 +348,9 @@ func (fs *Fs) RemoveAll(path string) error {
var infos []os.FileInfo var infos []os.FileInfo
infos, err = dir.Readdir(0) infos, err = dir.Readdir(0)
if err != nil {
return err
}
for _, info := range infos { for _, info := range infos {
nameToRemove := fs.normSeparators(info.Name()) nameToRemove := fs.normSeparators(info.Name())
err = fs.RemoveAll(path + fs.separator + nameToRemove) err = fs.RemoveAll(path + fs.separator + nameToRemove)

View File

@ -22,8 +22,8 @@ import (
"time" "time"
"cloud.google.com/go/storage" "cloud.google.com/go/storage"
"git.internal/re/afero"
"github.com/googleapis/google-cloud-go-testing/storage/stiface" "github.com/googleapis/google-cloud-go-testing/storage/stiface"
"github.com/spf13/afero"
"google.golang.org/api/option" "google.golang.org/api/option"
) )
@ -76,39 +76,51 @@ func NewGcsFSFromClientWithSeparator(ctx context.Context, client *storage.Client
func (fs *GcsFs) Name() string { func (fs *GcsFs) Name() string {
return fs.source.Name() return fs.source.Name()
} }
func (fs *GcsFs) Create(name string) (afero.File, error) { func (fs *GcsFs) Create(name string) (afero.File, error) {
return fs.source.Create(name) return fs.source.Create(name)
} }
func (fs *GcsFs) Mkdir(name string, perm os.FileMode) error { func (fs *GcsFs) Mkdir(name string, perm os.FileMode) error {
return fs.source.Mkdir(name, perm) return fs.source.Mkdir(name, perm)
} }
func (fs *GcsFs) MkdirAll(path string, perm os.FileMode) error { func (fs *GcsFs) MkdirAll(path string, perm os.FileMode) error {
return fs.source.MkdirAll(path, perm) return fs.source.MkdirAll(path, perm)
} }
func (fs *GcsFs) Open(name string) (afero.File, error) { func (fs *GcsFs) Open(name string) (afero.File, error) {
return fs.source.Open(name) return fs.source.Open(name)
} }
func (fs *GcsFs) OpenFile(name string, flag int, perm os.FileMode) (afero.File, error) { func (fs *GcsFs) OpenFile(name string, flag int, perm os.FileMode) (afero.File, error) {
return fs.source.OpenFile(name, flag, perm) return fs.source.OpenFile(name, flag, perm)
} }
func (fs *GcsFs) Remove(name string) error { func (fs *GcsFs) Remove(name string) error {
return fs.source.Remove(name) return fs.source.Remove(name)
} }
func (fs *GcsFs) RemoveAll(path string) error { func (fs *GcsFs) RemoveAll(path string) error {
return fs.source.RemoveAll(path) return fs.source.RemoveAll(path)
} }
func (fs *GcsFs) Rename(oldname, newname string) error { func (fs *GcsFs) Rename(oldname, newname string) error {
return fs.source.Rename(oldname, newname) return fs.source.Rename(oldname, newname)
} }
func (fs *GcsFs) Stat(name string) (os.FileInfo, error) { func (fs *GcsFs) Stat(name string) (os.FileInfo, error) {
return fs.source.Stat(name) return fs.source.Stat(name)
} }
func (fs *GcsFs) Chmod(name string, mode os.FileMode) error { func (fs *GcsFs) Chmod(name string, mode os.FileMode) error {
return fs.source.Chmod(name, mode) return fs.source.Chmod(name, mode)
} }
func (fs *GcsFs) Chtimes(name string, atime time.Time, mtime time.Time) error { func (fs *GcsFs) Chtimes(name string, atime time.Time, mtime time.Time) error {
return fs.source.Chtimes(name, atime, mtime) return fs.source.Chtimes(name, atime, mtime)
} }
func (fs *GcsFs) Chown(name string, uid, gid int) error { func (fs *GcsFs) Chown(name string, uid, gid int) error {
return fs.source.Chown(name, uid, gid) return fs.source.Chown(name, uid, gid)
} }

View File

@ -18,8 +18,8 @@ import (
"strings" "strings"
"cloud.google.com/go/storage" "cloud.google.com/go/storage"
"git.internal/re/afero"
"github.com/googleapis/google-cloud-go-testing/storage/stiface" "github.com/googleapis/google-cloud-go-testing/storage/stiface"
"github.com/spf13/afero"
"google.golang.org/api/iterator" "google.golang.org/api/iterator"
) )
@ -166,7 +166,7 @@ func (w *writerMock) Close() error {
if w.file == nil { if w.file == nil {
var err error var err error
if strings.HasSuffix(w.name, "/") { if strings.HasSuffix(w.name, "/") {
err = w.fs.Mkdir(w.name, 0755) err = w.fs.Mkdir(w.name, 0o755)
if err != nil { if err != nil {
return err return err
} }

View File

@ -20,8 +20,8 @@ import (
"golang.org/x/oauth2/google" "golang.org/x/oauth2/google"
"cloud.google.com/go/storage" "cloud.google.com/go/storage"
"git.internal/re/afero"
"github.com/googleapis/google-cloud-go-testing/storage/stiface" "github.com/googleapis/google-cloud-go-testing/storage/stiface"
"github.com/spf13/afero"
) )
const ( const (
@ -122,7 +122,7 @@ func TestMain(m *testing.M) {
gcsAfs = &afero.Afero{Fs: &GcsFs{NewGcsFs(ctx, mockClient)}} gcsAfs = &afero.Afero{Fs: &GcsFs{NewGcsFs(ctx, mockClient)}}
// Uncomment to use the real, not mocked, client // Uncomment to use the real, not mocked, client
//gcsAfs = &Afero{Fs: &GcsFs{gcsfs.NewGcsFs(ctx, client)}} // gcsAfs = &Afero{Fs: &GcsFs{gcsfs.NewGcsFs(ctx, client)}}
exitCode = m.Run() exitCode = m.Run()
} }
@ -341,7 +341,7 @@ func TestGcsSeek(t *testing.T) {
t.Fatalf("opening %v: %v", name, err) t.Fatalf("opening %v: %v", name, err)
} }
var tests = []struct { tests := []struct {
offIn int64 offIn int64
whence int whence int
offOut int64 offOut int64
@ -477,7 +477,7 @@ func TestGcsOpenFile(t *testing.T) {
} }
for _, name := range names { for _, name := range names {
file, err := gcsAfs.OpenFile(name, os.O_RDONLY, 0400) file, err := gcsAfs.OpenFile(name, os.O_RDONLY, 0o400)
if !f.exists { if !f.exists {
if (f.name != "" && !errors.Is(err, syscall.ENOENT)) || if (f.name != "" && !errors.Is(err, syscall.ENOENT)) ||
(f.name == "" && !errors.Is(err, ErrNoBucketInName)) { (f.name == "" && !errors.Is(err, ErrNoBucketInName)) {
@ -496,7 +496,7 @@ func TestGcsOpenFile(t *testing.T) {
t.Fatalf("failed to close a file \"%s\": %s", name, err) t.Fatalf("failed to close a file \"%s\": %s", name, err)
} }
file, err = gcsAfs.OpenFile(name, os.O_CREATE, 0600) _, err = gcsAfs.OpenFile(name, os.O_CREATE, 0o600)
if !errors.Is(err, syscall.EPERM) { if !errors.Is(err, syscall.EPERM) {
t.Errorf("%v: open for write: got %v, expected %v", name, err, syscall.EPERM) t.Errorf("%v: open for write: got %v, expected %v", name, err, syscall.EPERM)
} }
@ -714,7 +714,7 @@ func TestGcsMkdir(t *testing.T) {
t.Run("empty", func(t *testing.T) { t.Run("empty", func(t *testing.T) {
emptyDirName := bucketName emptyDirName := bucketName
err := gcsAfs.Mkdir(emptyDirName, 0755) err := gcsAfs.Mkdir(emptyDirName, 0o755)
if err == nil { if err == nil {
t.Fatal("did not fail upon creation of an empty folder") t.Fatal("did not fail upon creation of an empty folder")
} }
@ -723,7 +723,7 @@ func TestGcsMkdir(t *testing.T) {
dirName := filepath.Join(bucketName, "a-test-dir") dirName := filepath.Join(bucketName, "a-test-dir")
var err error var err error
err = gcsAfs.Mkdir(dirName, 0755) err = gcsAfs.Mkdir(dirName, 0o755)
if err != nil { if err != nil {
t.Fatal("failed to create a folder with error", err) t.Fatal("failed to create a folder with error", err)
} }
@ -739,7 +739,7 @@ func TestGcsMkdir(t *testing.T) {
t.Errorf("%s: mode is not directory", dirName) t.Errorf("%s: mode is not directory", dirName)
} }
if info.Mode() != os.ModeDir|0755 { if info.Mode() != os.ModeDir|0o755 {
t.Errorf("%s: wrong permissions, expected drwxr-xr-x, got %s", dirName, info.Mode()) t.Errorf("%s: wrong permissions, expected drwxr-xr-x, got %s", dirName, info.Mode())
} }
@ -754,7 +754,7 @@ func TestGcsMkdirAll(t *testing.T) {
t.Run("empty", func(t *testing.T) { t.Run("empty", func(t *testing.T) {
emptyDirName := bucketName emptyDirName := bucketName
err := gcsAfs.MkdirAll(emptyDirName, 0755) err := gcsAfs.MkdirAll(emptyDirName, 0o755)
if err == nil { if err == nil {
t.Fatal("did not fail upon creation of an empty folder") t.Fatal("did not fail upon creation of an empty folder")
} }
@ -762,7 +762,7 @@ func TestGcsMkdirAll(t *testing.T) {
t.Run("success", func(t *testing.T) { t.Run("success", func(t *testing.T) {
dirName := filepath.Join(bucketName, "a/b/c") dirName := filepath.Join(bucketName, "a/b/c")
err := gcsAfs.MkdirAll(dirName, 0755) err := gcsAfs.MkdirAll(dirName, 0o755)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -774,7 +774,7 @@ func TestGcsMkdirAll(t *testing.T) {
if !info.Mode().IsDir() { if !info.Mode().IsDir() {
t.Errorf("%s: mode is not directory", filepath.Join(bucketName, "a")) t.Errorf("%s: mode is not directory", filepath.Join(bucketName, "a"))
} }
if info.Mode() != os.ModeDir|0755 { if info.Mode() != os.ModeDir|0o755 {
t.Errorf("%s: wrong permissions, expected drwxr-xr-x, got %s", filepath.Join(bucketName, "a"), info.Mode()) t.Errorf("%s: wrong permissions, expected drwxr-xr-x, got %s", filepath.Join(bucketName, "a"), info.Mode())
} }
info, err = gcsAfs.Stat(filepath.Join(bucketName, "a/b")) info, err = gcsAfs.Stat(filepath.Join(bucketName, "a/b"))
@ -784,7 +784,7 @@ func TestGcsMkdirAll(t *testing.T) {
if !info.Mode().IsDir() { if !info.Mode().IsDir() {
t.Errorf("%s: mode is not directory", filepath.Join(bucketName, "a/b")) t.Errorf("%s: mode is not directory", filepath.Join(bucketName, "a/b"))
} }
if info.Mode() != os.ModeDir|0755 { if info.Mode() != os.ModeDir|0o755 {
t.Errorf("%s: wrong permissions, expected drwxr-xr-x, got %s", filepath.Join(bucketName, "a/b"), info.Mode()) t.Errorf("%s: wrong permissions, expected drwxr-xr-x, got %s", filepath.Join(bucketName, "a/b"), info.Mode())
} }
info, err = gcsAfs.Stat(dirName) info, err = gcsAfs.Stat(dirName)
@ -794,7 +794,7 @@ func TestGcsMkdirAll(t *testing.T) {
if !info.Mode().IsDir() { if !info.Mode().IsDir() {
t.Errorf("%s: mode is not directory", dirName) t.Errorf("%s: mode is not directory", dirName)
} }
if info.Mode() != os.ModeDir|0755 { if info.Mode() != os.ModeDir|0o755 {
t.Errorf("%s: wrong permissions, expected drwxr-xr-x, got %s", dirName, info.Mode()) t.Errorf("%s: wrong permissions, expected drwxr-xr-x, got %s", dirName, info.Mode())
} }
@ -816,7 +816,7 @@ func TestGcsRemoveAll(t *testing.T) {
aDir := filepath.Join(bucketName, "a") aDir := filepath.Join(bucketName, "a")
bDir := filepath.Join(aDir, "b") bDir := filepath.Join(aDir, "b")
err := gcsAfs.MkdirAll(bDir, 0755) err := gcsAfs.MkdirAll(bDir, 0o755)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }

2
go.mod
View File

@ -1,4 +1,4 @@
module github.com/spf13/afero module git.internal/re/afero
require ( require (
cloud.google.com/go/storage v1.14.0 cloud.google.com/go/storage v1.14.0

View File

@ -29,7 +29,7 @@ type httpDir struct {
} }
func (d httpDir) Open(name string) (http.File, error) { func (d httpDir) Open(name string) (http.File, error) {
if filepath.Separator != '/' && strings.IndexRune(name, filepath.Separator) >= 0 || if filepath.Separator != '/' && strings.ContainsRune(name, filepath.Separator) ||
strings.Contains(name, "\x00") { strings.Contains(name, "\x00") {
return nil, errors.New("http: invalid character in file path") return nil, errors.New("http: invalid character in file path")
} }

View File

@ -0,0 +1,27 @@
// Copyright © 2022 Steve Francia <spf@spf13.com>.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package common
import "io/fs"
// FileInfoDirEntry provides an adapter from os.FileInfo to fs.DirEntry
type FileInfoDirEntry struct {
fs.FileInfo
}
var _ fs.DirEntry = FileInfoDirEntry{}
func (d FileInfoDirEntry) Type() fs.FileMode { return d.FileInfo.Mode().Type() }
func (d FileInfoDirEntry) Info() (fs.FileInfo, error) { return d.FileInfo, nil }

38
iofs.go
View File

@ -1,3 +1,4 @@
//go:build go1.16
// +build go1.16 // +build go1.16
package afero package afero
@ -7,7 +8,10 @@ import (
"io/fs" "io/fs"
"os" "os"
"path" "path"
"sort"
"time" "time"
"git.internal/re/afero/internal/common"
) )
// IOFS adopts afero.Fs to stdlib io/fs.FS // IOFS adopts afero.Fs to stdlib io/fs.FS
@ -66,14 +70,31 @@ func (iofs IOFS) Glob(pattern string) ([]string, error) {
} }
func (iofs IOFS) ReadDir(name string) ([]fs.DirEntry, error) { func (iofs IOFS) ReadDir(name string) ([]fs.DirEntry, error) {
items, err := ReadDir(iofs.Fs, name) f, err := iofs.Fs.Open(name)
if err != nil { if err != nil {
return nil, iofs.wrapError("readdir", name, err) return nil, iofs.wrapError("readdir", name, err)
} }
defer f.Close()
if rdf, ok := f.(fs.ReadDirFile); ok {
items, err := rdf.ReadDir(-1)
if err != nil {
return nil, iofs.wrapError("readdir", name, err)
}
sort.Slice(items, func(i, j int) bool { return items[i].Name() < items[j].Name() })
return items, nil
}
items, err := f.Readdir(-1)
if err != nil {
return nil, iofs.wrapError("readdir", name, err)
}
sort.Sort(byName(items))
ret := make([]fs.DirEntry, len(items)) ret := make([]fs.DirEntry, len(items))
for i := range items { for i := range items {
ret[i] = dirEntry{items[i]} ret[i] = common.FileInfoDirEntry{FileInfo: items[i]}
} }
return ret, nil return ret, nil
@ -108,17 +129,6 @@ func (IOFS) wrapError(op, path string, err error) error {
} }
} }
// dirEntry provides adapter from os.FileInfo to fs.DirEntry
type dirEntry struct {
fs.FileInfo
}
var _ fs.DirEntry = dirEntry{}
func (d dirEntry) Type() fs.FileMode { return d.FileInfo.Mode().Type() }
func (d dirEntry) Info() (fs.FileInfo, error) { return d.FileInfo, nil }
// readDirFile provides adapter from afero.File to fs.ReadDirFile needed for correct Open // readDirFile provides adapter from afero.File to fs.ReadDirFile needed for correct Open
type readDirFile struct { type readDirFile struct {
File File
@ -134,7 +144,7 @@ func (r readDirFile) ReadDir(n int) ([]fs.DirEntry, error) {
ret := make([]fs.DirEntry, len(items)) ret := make([]fs.DirEntry, len(items))
for i := range items { for i := range items {
ret[i] = dirEntry{items[i]} ret[i] = common.FileInfoDirEntry{FileInfo: items[i]}
} }
return ret, nil return ret, nil

View File

@ -1,3 +1,4 @@
//go:build go1.16
// +build go1.16 // +build go1.16
package afero package afero
@ -5,15 +6,25 @@ package afero
import ( import (
"bytes" "bytes"
"errors" "errors"
"fmt"
"io" "io"
"io/fs" "io/fs"
"math/rand"
"os" "os"
"path/filepath"
"runtime"
"testing" "testing"
"testing/fstest" "testing/fstest"
"time" "time"
"git.internal/re/afero/internal/common"
) )
func TestIOFS(t *testing.T) { func TestIOFS(t *testing.T) {
if runtime.GOOS == "windows" {
// TODO(bep): some of the "bad path" tests in fstest.TestFS fail on Windows
t.Skip("Skipping on Windows")
}
t.Parallel() t.Parallel()
t.Run("use MemMapFs", func(t *testing.T) { t.Run("use MemMapFs", func(t *testing.T) {
@ -57,6 +68,93 @@ func TestIOFS(t *testing.T) {
}) })
} }
func TestIOFSNativeDirEntryWhenPossible(t *testing.T) {
t.Parallel()
osfs := NewBasePathFs(NewOsFs(), t.TempDir())
err := osfs.MkdirAll("dir1/dir2", os.ModePerm)
if err != nil {
t.Fatal(err)
}
const numFiles = 10
var fileNumbers []int
for i := 0; i < numFiles; i++ {
fileNumbers = append(fileNumbers, i)
}
rand.Shuffle(len(fileNumbers), func(i, j int) {
fileNumbers[i], fileNumbers[j] = fileNumbers[j], fileNumbers[i]
})
for _, i := range fileNumbers {
f, err := osfs.Create(fmt.Sprintf("dir1/dir2/test%d.txt", i))
if err != nil {
t.Fatal(err)
}
f.Close()
}
dir2, err := osfs.Open("dir1/dir2")
if err != nil {
t.Fatal(err)
}
assertDirEntries := func(entries []fs.DirEntry, ordered bool) {
if len(entries) != numFiles {
t.Fatalf("expected %d, got %d", numFiles, len(entries))
}
for i, entry := range entries {
if _, ok := entry.(common.FileInfoDirEntry); ok {
t.Fatal("DirEntry not native")
}
if ordered && entry.Name() != fmt.Sprintf("test%d.txt", i) {
t.Fatalf("expected %s, got %s", fmt.Sprintf("test%d.txt", i), entry.Name())
}
}
}
dirEntries, err := dir2.(fs.ReadDirFile).ReadDir(-1)
if err != nil {
t.Fatal(err)
}
assertDirEntries(dirEntries, false)
iofs := NewIOFS(osfs)
dirEntries, err = iofs.ReadDir("dir1/dir2")
if err != nil {
t.Fatal(err)
}
assertDirEntries(dirEntries, true)
fileCount := 0
err = fs.WalkDir(iofs, "", func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if !d.IsDir() {
fileCount++
}
if _, ok := d.(common.FileInfoDirEntry); ok {
t.Fatal("DirEntry not native")
}
return nil
})
if err != nil {
t.Fatal(err)
}
if fileCount != numFiles {
t.Fatalf("expected %d, got %d", numFiles, fileCount)
}
}
func TestFromIOFS(t *testing.T) { func TestFromIOFS(t *testing.T) {
t.Parallel() t.Parallel()
@ -259,7 +357,6 @@ func TestFromIOFS_File(t *testing.T) {
// MapFS files implements io.ReaderAt // MapFS files implements io.ReaderAt
b := make([]byte, 2) b := make([]byte, 2)
_, err := file.ReadAt(b, 2) _, err := file.ReadAt(b, 2)
if err != nil { if err != nil {
t.Errorf("ReadAt failed: %v", err) t.Errorf("ReadAt failed: %v", err)
return return
@ -319,7 +416,7 @@ func TestFromIOFS_File(t *testing.T) {
return return
} }
var expectedItems = []struct { expectedItems := []struct {
Name string Name string
IsDir bool IsDir bool
Size int64 Size int64
@ -371,7 +468,7 @@ func TestFromIOFS_File(t *testing.T) {
return return
} }
var expectedItems = []string{"dir1", "dir2", "test.txt"} expectedItems := []string{"dir1", "dir2", "test.txt"}
if len(expectedItems) != len(items) { if len(expectedItems) != len(items) {
t.Errorf("Items count mismatch, expected %d, got %d", len(expectedItems), len(items)) t.Errorf("Items count mismatch, expected %d, got %d", len(expectedItems), len(items))
@ -410,3 +507,45 @@ func assertPermissionError(t *testing.T, err error) {
t.Errorf("Expected (*fs.PathError).Err == fs.ErrPermisson, got %[1]T (%[1]v)", err) t.Errorf("Expected (*fs.PathError).Err == fs.ErrPermisson, got %[1]T (%[1]v)", err)
} }
} }
func BenchmarkWalkDir(b *testing.B) {
osfs := NewBasePathFs(NewOsFs(), b.TempDir())
createSomeFiles := func(dirname string) {
for i := 0; i < 10; i++ {
f, err := osfs.Create(filepath.Join(dirname, fmt.Sprintf("test%d.txt", i)))
if err != nil {
b.Fatal(err)
}
f.Close()
}
}
depth := 10
for level := depth; level > 0; level-- {
dirname := ""
for i := 0; i < level; i++ {
dirname = filepath.Join(dirname, fmt.Sprintf("dir%d", i))
err := osfs.MkdirAll(dirname, 0o755)
if err != nil && !os.IsExist(err) {
b.Fatal(err)
}
}
createSomeFiles(dirname)
}
iofs := NewIOFS(osfs)
b.ResetTimer()
for i := 0; i < b.N; i++ {
err := fs.WalkDir(iofs, "", func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
return nil
})
if err != nil {
b.Fatal(err)
}
}
}

View File

@ -141,7 +141,7 @@ func WriteFile(fs Fs, filename string, data []byte, perm os.FileMode) error {
// We generate random temporary file names so that there's a good // We generate random temporary file names so that there's a good
// chance the file doesn't exist yet - keeps the number of tries in // chance the file doesn't exist yet - keeps the number of tries in
// TempFile to a minimum. // TempFile to a minimum.
var rand uint32 var randNum uint32
var randmu sync.Mutex var randmu sync.Mutex
func reseed() uint32 { func reseed() uint32 {
@ -150,12 +150,12 @@ func reseed() uint32 {
func nextRandom() string { func nextRandom() string {
randmu.Lock() randmu.Lock()
r := rand r := randNum
if r == 0 { if r == 0 {
r = reseed() r = reseed()
} }
r = r*1664525 + 1013904223 // constants from Numerical Recipes r = r*1664525 + 1013904223 // constants from Numerical Recipes
rand = r randNum = r
randmu.Unlock() randmu.Unlock()
return strconv.Itoa(int(1e9 + r%1e9))[1:] return strconv.Itoa(int(1e9 + r%1e9))[1:]
} }
@ -194,7 +194,7 @@ func TempFile(fs Fs, dir, pattern string) (f File, err error) {
if os.IsExist(err) { if os.IsExist(err) {
if nconflict++; nconflict > 10 { if nconflict++; nconflict > 10 {
randmu.Lock() randmu.Lock()
rand = reseed() randNum = reseed()
randmu.Unlock() randmu.Unlock()
} }
continue continue
@ -226,7 +226,7 @@ func TempDir(fs Fs, dir, prefix string) (name string, err error) {
if os.IsExist(err) { if os.IsExist(err) {
if nconflict++; nconflict > 10 { if nconflict++; nconflict > 10 {
randmu.Lock() randmu.Lock()
rand = reseed() randNum = reseed()
randmu.Unlock() randmu.Unlock()
} }
continue continue

View File

@ -37,13 +37,13 @@ func TestReadFile(t *testing.T) {
testFS.Create("this_exists.go") testFS.Create("this_exists.go")
filename := "rumpelstilzchen" filename := "rumpelstilzchen"
contents, err := fsutil.ReadFile(filename) _, err := fsutil.ReadFile(filename)
if err == nil { if err == nil {
t.Fatalf("ReadFile %s: error expected, none found", filename) t.Fatalf("ReadFile %s: error expected, none found", filename)
} }
filename = "this_exists.go" filename = "this_exists.go"
contents, err = fsutil.ReadFile(filename) contents, err := fsutil.ReadFile(filename)
if err != nil { if err != nil {
t.Fatalf("ReadFile %s: %v", filename, err) t.Fatalf("ReadFile %s: %v", filename, err)
} }

View File

@ -18,15 +18,20 @@ import (
"bytes" "bytes"
"errors" "errors"
"io" "io"
"io/fs"
"os" "os"
"path/filepath" "path/filepath"
"sync" "sync"
"sync/atomic" "sync/atomic"
"time" "time"
"git.internal/re/afero/internal/common"
) )
const FilePathSeparator = string(filepath.Separator) const FilePathSeparator = string(filepath.Separator)
var _ fs.ReadDirFile = &File{}
type File struct { type File struct {
// atomic requires 64-bit alignment for struct field access // atomic requires 64-bit alignment for struct field access
at int64 at int64
@ -183,10 +188,23 @@ func (f *File) Readdirnames(n int) (names []string, err error) {
return names, err return names, err
} }
// Implements fs.ReadDirFile
func (f *File) ReadDir(n int) ([]fs.DirEntry, error) {
fi, err := f.Readdir(n)
if err != nil {
return nil, err
}
di := make([]fs.DirEntry, len(fi))
for i, f := range fi {
di[i] = common.FileInfoDirEntry{FileInfo: f}
}
return di, nil
}
func (f *File) Read(b []byte) (n int, err error) { func (f *File) Read(b []byte) (n int, err error) {
f.fileData.Lock() f.fileData.Lock()
defer f.fileData.Unlock() defer f.fileData.Unlock()
if f.closed == true { if f.closed {
return 0, ErrFileClosed return 0, ErrFileClosed
} }
if len(b) > 0 && int(f.at) == len(f.fileData.data) { if len(b) > 0 && int(f.at) == len(f.fileData.data) {
@ -214,7 +232,7 @@ func (f *File) ReadAt(b []byte, off int64) (n int, err error) {
} }
func (f *File) Truncate(size int64) error { func (f *File) Truncate(size int64) error {
if f.closed == true { if f.closed {
return ErrFileClosed return ErrFileClosed
} }
if f.readOnly { if f.readOnly {
@ -227,7 +245,7 @@ func (f *File) Truncate(size int64) error {
defer f.fileData.Unlock() defer f.fileData.Unlock()
if size > int64(len(f.fileData.data)) { if size > int64(len(f.fileData.data)) {
diff := size - int64(len(f.fileData.data)) diff := size - int64(len(f.fileData.data))
f.fileData.data = append(f.fileData.data, bytes.Repeat([]byte{00}, int(diff))...) f.fileData.data = append(f.fileData.data, bytes.Repeat([]byte{0o0}, int(diff))...)
} else { } else {
f.fileData.data = f.fileData.data[0:size] f.fileData.data = f.fileData.data[0:size]
} }
@ -236,7 +254,7 @@ func (f *File) Truncate(size int64) error {
} }
func (f *File) Seek(offset int64, whence int) (int64, error) { func (f *File) Seek(offset int64, whence int) (int64, error) {
if f.closed == true { if f.closed {
return 0, ErrFileClosed return 0, ErrFileClosed
} }
switch whence { switch whence {
@ -251,7 +269,7 @@ func (f *File) Seek(offset int64, whence int) (int64, error) {
} }
func (f *File) Write(b []byte) (n int, err error) { func (f *File) Write(b []byte) (n int, err error) {
if f.closed == true { if f.closed {
return 0, ErrFileClosed return 0, ErrFileClosed
} }
if f.readOnly { if f.readOnly {
@ -267,7 +285,7 @@ func (f *File) Write(b []byte) (n int, err error) {
tail = f.fileData.data[n+int(cur):] tail = f.fileData.data[n+int(cur):]
} }
if diff > 0 { if diff > 0 {
f.fileData.data = append(f.fileData.data, append(bytes.Repeat([]byte{00}, int(diff)), b...)...) f.fileData.data = append(f.fileData.data, append(bytes.Repeat([]byte{0o0}, int(diff)), b...)...)
f.fileData.data = append(f.fileData.data, tail...) f.fileData.data = append(f.fileData.data, tail...)
} else { } else {
f.fileData.data = append(f.fileData.data[:cur], b...) f.fileData.data = append(f.fileData.data[:cur], b...)
@ -303,16 +321,19 @@ func (s *FileInfo) Name() string {
s.Unlock() s.Unlock()
return name return name
} }
func (s *FileInfo) Mode() os.FileMode { func (s *FileInfo) Mode() os.FileMode {
s.Lock() s.Lock()
defer s.Unlock() defer s.Unlock()
return s.mode return s.mode
} }
func (s *FileInfo) ModTime() time.Time { func (s *FileInfo) ModTime() time.Time {
s.Lock() s.Lock()
defer s.Unlock() defer s.Unlock()
return s.modtime return s.modtime
} }
func (s *FileInfo) IsDir() bool { func (s *FileInfo) IsDir() bool {
s.Lock() s.Lock()
defer s.Unlock() defer s.Unlock()
@ -330,8 +351,8 @@ func (s *FileInfo) Size() int64 {
var ( var (
ErrFileClosed = errors.New("File is closed") ErrFileClosed = errors.New("File is closed")
ErrOutOfRange = errors.New("Out of range") ErrOutOfRange = errors.New("out of range")
ErrTooLarge = errors.New("Too large") ErrTooLarge = errors.New("too large")
ErrFileNotFound = os.ErrNotExist ErrFileNotFound = os.ErrNotExist
ErrFileExists = os.ErrExist ErrFileExists = os.ErrExist
ErrDestinationExists = os.ErrExist ErrDestinationExists = os.ErrExist

View File

@ -66,8 +66,8 @@ func TestFileDataModTimeRace(t *testing.T) {
func TestFileDataModeRace(t *testing.T) { func TestFileDataModeRace(t *testing.T) {
t.Parallel() t.Parallel()
const someMode = 0777 const someMode = 0o777
const someOtherMode = 0660 const someOtherMode = 0o660
d := FileData{ d := FileData{
mode: someMode, mode: someMode,
@ -95,7 +95,7 @@ func TestFileDataModeRace(t *testing.T) {
} }
} }
// See https://github.com/spf13/afero/issues/286. // See https://git.internal/re/afero/issues/286.
func TestFileWriteAt(t *testing.T) { func TestFileWriteAt(t *testing.T) {
t.Parallel() t.Parallel()
@ -167,7 +167,7 @@ func TestFileDataIsDirRace(t *testing.T) {
s.Unlock() s.Unlock()
}() }()
//just logging the value to trigger a read: // just logging the value to trigger a read:
t.Logf("Value is %v", s.IsDir()) t.Logf("Value is %v", s.IsDir())
} }
@ -196,10 +196,10 @@ func TestFileDataSizeRace(t *testing.T) {
s.Unlock() s.Unlock()
}() }()
//just logging the value to trigger a read: // just logging the value to trigger a read:
t.Logf("Value is %v", s.Size()) t.Logf("Value is %v", s.Size())
//Testing the Dir size case // Testing the Dir size case
d.dir = true d.dir = true
if s.Size() != int64(42) { if s.Size() != int64(42) {
t.Errorf("Failed to read correct value for dir, was %v", s.Size()) t.Errorf("Failed to read correct value for dir, was %v", s.Size())

View File

@ -22,7 +22,7 @@ import (
"sync" "sync"
"time" "time"
"github.com/spf13/afero/mem" "git.internal/re/afero/mem"
) )
const chmodBits = os.ModePerm | os.ModeSetuid | os.ModeSetgid | os.ModeSticky // Only a subset of bits are allowed to be changed. Documented under os.Chmod() const chmodBits = os.ModePerm | os.ModeSetuid | os.ModeSetgid | os.ModeSticky // Only a subset of bits are allowed to be changed. Documented under os.Chmod()
@ -43,7 +43,7 @@ func (m *MemMapFs) getData() map[string]*mem.FileData {
// Root should always exist, right? // Root should always exist, right?
// TODO: what about windows? // TODO: what about windows?
root := mem.CreateDir(FilePathSeparator) root := mem.CreateDir(FilePathSeparator)
mem.SetMode(root, os.ModeDir|0755) mem.SetMode(root, os.ModeDir|0o755)
m.data[FilePathSeparator] = root m.data[FilePathSeparator] = root
}) })
return m.data return m.data
@ -96,12 +96,12 @@ func (m *MemMapFs) registerWithParent(f *mem.FileData, perm os.FileMode) {
pdir := filepath.Dir(filepath.Clean(f.Name())) pdir := filepath.Dir(filepath.Clean(f.Name()))
err := m.lockfreeMkdir(pdir, perm) err := m.lockfreeMkdir(pdir, perm)
if err != nil { if err != nil {
//log.Println("Mkdir error:", err) // log.Println("Mkdir error:", err)
return return
} }
parent, err = m.lockfreeOpen(pdir) parent, err = m.lockfreeOpen(pdir)
if err != nil { if err != nil {
//log.Println("Open after Mkdir error:", err) // log.Println("Open after Mkdir error:", err)
return return
} }
} }
@ -142,6 +142,11 @@ func (m *MemMapFs) Mkdir(name string, perm os.FileMode) error {
} }
m.mu.Lock() m.mu.Lock()
// Dobule check that it doesn't exist.
if _, ok := m.getData()[name]; ok {
m.mu.Unlock()
return &os.PathError{Op: "mkdir", Path: name, Err: ErrFileExists}
}
item := mem.CreateDir(name) item := mem.CreateDir(name)
mem.SetMode(item, os.ModeDir|perm) mem.SetMode(item, os.ModeDir|perm)
m.getData()[name] = item m.getData()[name] = item

View File

@ -3,9 +3,12 @@ package afero
import ( import (
"fmt" "fmt"
"io" "io"
"io/fs"
"os" "os"
"path/filepath" "path/filepath"
"runtime" "runtime"
"strings"
"sync"
"testing" "testing"
"time" "time"
) )
@ -214,7 +217,7 @@ func TestMultipleOpenFiles(t *testing.T) {
if err != nil { if err != nil {
t.Error("fh.Write failed: " + err.Error()) t.Error("fh.Write failed: " + err.Error())
} }
_, err = fh1.Seek(0, os.SEEK_SET) _, err = fh1.Seek(0, io.SeekStart)
if err != nil { if err != nil {
t.Error(err) t.Error(err)
} }
@ -692,3 +695,84 @@ func TestMemFsLstatIfPossible(t *testing.T) {
t.Fatalf("Function indicated lstat was called. This should never be true.") t.Fatalf("Function indicated lstat was called. This should never be true.")
} }
} }
func TestMemMapFsConfurrentMkdir(t *testing.T) {
const dir = "test_dir"
const n = 1000
mfs := NewMemMapFs().(*MemMapFs)
allFilePaths := make([]string, 0, n)
// run concurrency test
var wg sync.WaitGroup
for i := 0; i < n; i++ {
fp := filepath.Join(
dir,
fmt.Sprintf("%02d", n%10),
fmt.Sprintf("%d.txt", i),
)
allFilePaths = append(allFilePaths, fp)
wg.Add(1)
go func() {
defer wg.Done()
if err := mfs.MkdirAll(filepath.Dir(fp), 0755); err != nil {
t.Error(err)
}
wt, err := mfs.Create(fp)
if err != nil {
t.Error(err)
}
defer func() {
if err := wt.Close(); err != nil {
t.Error(err)
}
}()
// write 30 bytes
for j := 0; j < 10; j++ {
_, err := wt.Write([]byte("000"))
if err != nil {
t.Error(err)
}
}
}()
}
wg.Wait()
// Test1: find all files by full path access
for _, fp := range allFilePaths {
info, err := mfs.Stat(fp)
if err != nil {
t.Error(err)
}
if info.Size() != 30 {
t.Errorf("file size should be 30, but got %d", info.Size())
}
}
// Test2: find all files by walk
foundFiles := make([]string, 0, n)
wErr := Walk(mfs, dir, func(path string, info fs.FileInfo, err error) error {
if err != nil {
t.Error(err)
}
if info.IsDir() {
return nil // skip dir
}
if strings.HasSuffix(info.Name(), ".txt") {
foundFiles = append(foundFiles, path)
}
return nil
})
if wErr != nil {
t.Error(wErr)
}
if len(foundFiles) != n {
t.Errorf("found %d files, but expect %d", len(foundFiles), n)
}
}

View File

@ -16,12 +16,12 @@ func TestFilterReadOnly(t *testing.T) {
func TestFilterReadonlyRemoveAndRead(t *testing.T) { func TestFilterReadonlyRemoveAndRead(t *testing.T) {
mfs := &MemMapFs{} mfs := &MemMapFs{}
fh, err := mfs.Create("/file.txt") fh, _ := mfs.Create("/file.txt")
fh.Write([]byte("content here")) fh.Write([]byte("content here"))
fh.Close() fh.Close()
fs := NewReadOnlyFs(mfs) fs := NewReadOnlyFs(mfs)
err = fs.Remove("/file.txt") err := fs.Remove("/file.txt")
if err == nil { if err == nil {
t.Errorf("Did not fail to remove file") t.Errorf("Did not fail to remove file")
} }

View File

@ -17,8 +17,8 @@ import (
"os" "os"
"time" "time"
"git.internal/re/afero"
"github.com/pkg/sftp" "github.com/pkg/sftp"
"github.com/spf13/afero"
) )
// Fs is a afero.Fs implementation that uses functions provided by the sftp package. // Fs is a afero.Fs implementation that uses functions provided by the sftp package.

View File

@ -215,6 +215,9 @@ func MakeSSHKeyPair(bits int, pubKeyPath, privateKeyPath string) error {
// generate and write private key as PEM // generate and write private key as PEM
privateKeyFile, err := os.Create(privateKeyPath) privateKeyFile, err := os.Create(privateKeyPath)
if err != nil {
return err
}
defer privateKeyFile.Close() defer privateKeyFile.Close()
if err != nil { if err != nil {
return err return err

View File

@ -8,7 +8,7 @@ import (
"sort" "sort"
"syscall" "syscall"
"github.com/spf13/afero" "git.internal/re/afero"
) )
type File struct { type File struct {

View File

@ -10,7 +10,7 @@ import (
"syscall" "syscall"
"time" "time"
"github.com/spf13/afero" "git.internal/re/afero"
) )
type Fs struct { type Fs struct {

View File

@ -13,7 +13,7 @@ import (
"syscall" "syscall"
"testing" "testing"
"github.com/spf13/afero" "git.internal/re/afero"
) )
var files = []struct { var files = []struct {
@ -161,7 +161,7 @@ func TestSeek(t *testing.T) {
t.Fatalf("opening %v: %v", f.name, err) t.Fatalf("opening %v: %v", f.name, err)
} }
var tests = []struct { tests := []struct {
offin int64 offin int64
whence int whence int
offout int64 offout int64
@ -252,7 +252,7 @@ func TestClose(t *testing.T) {
func TestOpenFile(t *testing.T) { func TestOpenFile(t *testing.T) {
for _, f := range files { for _, f := range files {
file, err := afs.OpenFile(f.name, os.O_RDONLY, 0400) file, err := afs.OpenFile(f.name, os.O_RDONLY, 0o400)
if !f.exists { if !f.exists {
if !errors.Is(err, syscall.ENOENT) { if !errors.Is(err, syscall.ENOENT) {
t.Errorf("%v: got %v, expected%v", f.name, err, syscall.ENOENT) t.Errorf("%v: got %v, expected%v", f.name, err, syscall.ENOENT)
@ -266,7 +266,7 @@ func TestOpenFile(t *testing.T) {
} }
file.Close() file.Close()
file, err = afs.OpenFile(f.name, os.O_CREATE, 0600) _, err = afs.OpenFile(f.name, os.O_CREATE, 0o600)
if !errors.Is(err, syscall.EPERM) { if !errors.Is(err, syscall.EPERM) {
t.Errorf("%v: open for write: got %v, expected %v", f.name, err, syscall.EPERM) t.Errorf("%v: open for write: got %v, expected %v", f.name, err, syscall.EPERM)
} }

View File

@ -65,7 +65,7 @@ func (f *UnionFile) ReadAt(s []byte, o int64) (int, error) {
if f.Layer != nil { if f.Layer != nil {
n, err := f.Layer.ReadAt(s, o) n, err := f.Layer.ReadAt(s, o)
if (err == nil || err == io.EOF) && f.Base != nil { if (err == nil || err == io.EOF) && f.Base != nil {
_, err = f.Base.Seek(o+int64(n), os.SEEK_SET) _, err = f.Base.Seek(o+int64(n), io.SeekStart)
} }
return n, err return n, err
} }

10
util.go
View File

@ -25,6 +25,7 @@ import (
"strings" "strings"
"unicode" "unicode"
"golang.org/x/text/runes"
"golang.org/x/text/transform" "golang.org/x/text/transform"
"golang.org/x/text/unicode/norm" "golang.org/x/text/unicode/norm"
) )
@ -158,16 +159,12 @@ func UnicodeSanitize(s string) string {
// Transform characters with accents into plain forms. // Transform characters with accents into plain forms.
func NeuterAccents(s string) string { func NeuterAccents(s string) string {
t := transform.Chain(norm.NFD, transform.RemoveFunc(isMn), norm.NFC) t := transform.Chain(norm.NFD, runes.Remove(runes.In(unicode.Mn)), norm.NFC)
result, _, _ := transform.String(t, string(s)) result, _, _ := transform.String(t, string(s))
return result return result
} }
func isMn(r rune) bool {
return unicode.Is(unicode.Mn, r) // Mn: nonspacing marks
}
func (a Afero) FileContainsBytes(filename string, subslice []byte) (bool, error) { func (a Afero) FileContainsBytes(filename string, subslice []byte) (bool, error) {
return FileContainsBytes(a.Fs, filename, subslice) return FileContainsBytes(a.Fs, filename, subslice)
} }
@ -299,6 +296,9 @@ func IsEmpty(fs Fs, path string) (bool, error) {
} }
defer f.Close() defer f.Close()
list, err := f.Readdir(-1) list, err := f.Readdir(-1)
if err != nil {
return false, err
}
return len(list) == 0, nil return len(list) == 0, nil
} }
return fi.Size() == 0, nil return fi.Size() == 0, nil

View File

@ -185,7 +185,7 @@ func createZeroSizedFileInTempDir() (File, error) {
func createNonZeroSizedFileInTempDir() (File, error) { func createNonZeroSizedFileInTempDir() (File, error) {
f, err := createZeroSizedFileInTempDir() f, err := createZeroSizedFileInTempDir()
if err != nil { if err != nil {
// no file ?? return nil, err
} }
byteString := []byte("byteString") byteString := []byte("byteString")
err = WriteFile(testFS, f.Name(), byteString, 0644) err = WriteFile(testFS, f.Name(), byteString, 0644)
@ -200,7 +200,7 @@ func createNonZeroSizedFileInTempDir() (File, error) {
func deleteFileInTempDir(f File) { func deleteFileInTempDir(f File) {
err := testFS.Remove(f.Name()) err := testFS.Remove(f.Name())
if err != nil { if err != nil {
// now what? panic(err)
} }
} }
@ -217,7 +217,7 @@ func createEmptyTempDir() (string, error) {
func createTempDirWithZeroLengthFiles() (string, error) { func createTempDirWithZeroLengthFiles() (string, error) {
d, dirErr := createEmptyTempDir() d, dirErr := createEmptyTempDir()
if dirErr != nil { if dirErr != nil {
//now what? return "", dirErr
} }
filePrefix := "_path_test_" filePrefix := "_path_test_"
_, fileErr := TempFile(testFS, d, filePrefix) // dir is os.TempDir() _, fileErr := TempFile(testFS, d, filePrefix) // dir is os.TempDir()
@ -235,7 +235,7 @@ func createTempDirWithZeroLengthFiles() (string, error) {
func createTempDirWithNonZeroLengthFiles() (string, error) { func createTempDirWithNonZeroLengthFiles() (string, error) {
d, dirErr := createEmptyTempDir() d, dirErr := createEmptyTempDir()
if dirErr != nil { if dirErr != nil {
//now what? return "", dirErr
} }
filePrefix := "_path_test_" filePrefix := "_path_test_"
f, fileErr := TempFile(testFS, d, filePrefix) // dir is os.TempDir() f, fileErr := TempFile(testFS, d, filePrefix) // dir is os.TempDir()
@ -406,7 +406,7 @@ func TestGetTempDir(t *testing.T) {
func deleteTempDir(d string) { func deleteTempDir(d string) {
err := os.RemoveAll(d) err := os.RemoveAll(d)
if err != nil { if err != nil {
// now what? panic(err)
} }
} }

View File

@ -7,7 +7,7 @@ import (
"path/filepath" "path/filepath"
"syscall" "syscall"
"github.com/spf13/afero" "git.internal/re/afero"
) )
type File struct { type File struct {
@ -85,10 +85,10 @@ func (f *File) Seek(offset int64, whence int) (int64, error) {
return 0, afero.ErrFileClosed return 0, afero.ErrFileClosed
} }
switch whence { switch whence {
case os.SEEK_SET: case io.SeekStart:
case os.SEEK_CUR: case io.SeekCurrent:
offset += f.offset offset += f.offset
case os.SEEK_END: case io.SeekEnd:
offset += int64(f.zipfile.UncompressedSize64) offset += int64(f.zipfile.UncompressedSize64)
default: default:
return 0, syscall.EINVAL return 0, syscall.EINVAL

View File

@ -7,7 +7,7 @@ import (
"syscall" "syscall"
"time" "time"
"github.com/spf13/afero" "git.internal/re/afero"
) )
type Fs struct { type Fs struct {

View File

@ -1,12 +1,12 @@
package zipfs package zipfs
import ( import (
"github.com/spf13/afero"
"archive/zip" "archive/zip"
"path/filepath" "path/filepath"
"reflect" "reflect"
"testing" "testing"
"git.internal/re/afero"
) )
func TestZipFS(t *testing.T) { func TestZipFS(t *testing.T) {