re
/
afero
forked from mirror/afero
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Changed abstraction to operate on client, not bucket

This commit is contained in:
Vasily Ovchinnikov 2021-04-12 20:01:12 +02:00
parent 1a0f20777f
commit 78acf3b074
8 changed files with 568 additions and 356 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
gcs.go
View File

@ -21,9 +21,11 @@ import (
"os"
"time"
"github.com/spf13/afero/gcsfs"
"cloud.google.com/go/storage"
"github.com/googleapis/google-cloud-go-testing/storage/stiface"
"github.com/spf13/afero/gcsfs"
"google.golang.org/api/option"
)
@ -31,47 +33,44 @@ type GcsFs struct {
source *gcsfs.GcsFs
}
// Creates a GCS file system, automatically instantiating and decorating the storage client.
// NewGcsFS creates a GCS file system, automatically instantiating and decorating the storage client.
// You can provide additional options to be passed to the client creation, as per
// cloud.google.com/go/storage documentation
func NewGcsFS(ctx context.Context, bucketName string, opts ...option.ClientOption) (Fs, error) {
func NewGcsFS(ctx context.Context, opts ...option.ClientOption) (Fs, error) {
client, err := storage.NewClient(ctx, opts...)
if err != nil {
return nil, err
}
return NewGcsFSFromClient(ctx, client, bucketName)
return NewGcsFSFromClient(ctx, client)
}
// The same as NewGcsFS, but the files system will use the provided folder separator.
func NewGcsFSWithSeparator(ctx context.Context, bucketName, folderSeparator string, opts ...option.ClientOption) (Fs, error) {
// NewGcsFSWithSeparator is the same as NewGcsFS, but the files system will use the provided folder separator.
func NewGcsFSWithSeparator(ctx context.Context, folderSeparator string, opts ...option.ClientOption) (Fs, error) {
client, err := storage.NewClient(ctx, opts...)
if err != nil {
return nil, err
}
return NewGcsFSFromClientWithSeparator(ctx, client, bucketName, folderSeparator)
return NewGcsFSFromClientWithSeparator(ctx, client, folderSeparator)
}
// Creates a GCS file system from a given storage client
func NewGcsFSFromClient(ctx context.Context, client *storage.Client, bucketName string) (Fs, error) {
// NewGcsFSFromClient creates a GCS file system from a given storage client
func NewGcsFSFromClient(ctx context.Context, client *storage.Client) (Fs, error) {
c := stiface.AdaptClient(client)
bucket := c.Bucket(bucketName)
return &GcsFs{gcsfs.NewGcsFs(ctx, bucket)}, nil
return &GcsFs{gcsfs.NewGcsFs(ctx, c)}, nil
}
// Same as NewGcsFSFromClient, but the file system will use the provided folder separator.
func NewGcsFSFromClientWithSeparator(ctx context.Context, client *storage.Client, bucketName, folderSeparator string) (Fs, error) {
// NewGcsFSFromClientWithSeparator is the same as NewGcsFSFromClient, but the file system will use the provided folder separator.
func NewGcsFSFromClientWithSeparator(ctx context.Context, client *storage.Client, folderSeparator string) (Fs, error) {
c := stiface.AdaptClient(client)
bucket := c.Bucket(bucketName)
return &GcsFs{gcsfs.NewGcsFsWithSeparator(ctx, bucket, folderSeparator)}, nil
return &GcsFs{gcsfs.NewGcsFsWithSeparator(ctx, c, folderSeparator)}, nil
}
// Wraps gcs.GcsFs and convert some return types to afero interfaces.
func (fs *GcsFs) Name() string {
return fs.source.Name()
}

8
gcs_mocks.go
View File

@ -50,6 +50,10 @@ type bucketMock struct {
fs Fs
}
func (m *bucketMock) Attrs(context.Context) (*storage.BucketAttrs, error) {
return &storage.BucketAttrs{}, nil
}
func (m *bucketMock) Object(name string) stiface.ObjectHandle {
return &objectMock{name: name, fs: m.fs}
}
@ -193,6 +197,10 @@ func (r *readerMock) Read(p []byte) (int, error) {
return r.file.Read(p)
}
func (r *readerMock) Close() error {
return r.file.Close()
}
type objectItMock struct {
stiface.ObjectIterator

647
gcs_test.go
View File

@ -17,6 +17,8 @@ import (
"syscall"
"testing"
"golang.org/x/oauth2/google"
"github.com/spf13/afero/gcsfs"
"cloud.google.com/go/storage"
@ -28,6 +30,8 @@ const (
dirSize = 42
)
var bucketName = "a-test-bucket"
var files = []struct {
name string
exists bool
@ -37,13 +41,14 @@ var files = []struct {
offset int64
contentAtOffset string
}{
{"", true, true, dirSize, "", 0, ""}, // this is NOT a valid path for GCS, so we do some magic here
{"sub", true, true, dirSize, "", 0, ""},
{"sub/testDir2", true, true, dirSize, "", 0, ""},
{"sub/testDir2/testFile", true, false, 8 * 1024, "c", 4 * 1024, "d"},
{"testFile", true, false, 12 * 1024, "a", 7 * 1024, "b"},
{"testDir1/testFile", true, false, 3 * 512, "b", 512, "c"},
{"", false, true, dirSize, "", 0, ""}, // special case
{"nonExisting", false, false, dirSize, "", 0, ""},
}
@ -51,7 +56,7 @@ var dirs = []struct {
name string
children []string
}{
{"", []string{"sub", "testDir1", "testFile"}},
{"", []string{"sub", "testDir1", "testFile"}}, // in this case it will be prepended with bucket name
{"sub", []string{"testDir2"}},
{"sub/testDir2", []string{"testFile"}},
{"testDir1", []string{"testFile"}},
@ -63,20 +68,35 @@ func TestMain(m *testing.M) {
ctx := context.Background()
var err error
// Check if GOOGLE_APPLICATION_CREDENTIALS are present. If not, then a fake service account
// in order to respect deferring
var exitCode int
defer os.Exit(exitCode)
defer func() {
err := recover()
if err != nil {
fmt.Print(err)
exitCode = 2
}
}()
// Check if any credentials are present. If not, a fake service account, taken from the link
// would be used: https://github.com/google/oauth2l/blob/master/integration/fixtures/fake-service-account.json
if os.Getenv("GOOGLE_APPLICATION_CREDENTIALS") == "" {
cred, err := google.FindDefaultCredentials(ctx)
if err != nil && !strings.HasPrefix(err.Error(), "google: could not find default credentials") {
panic(err)
}
if cred == nil {
var fakeCredentialsAbsPath string
fakeCredentialsAbsPath, err = filepath.Abs("gcs-fake-service-account.json")
if err != nil {
fmt.Print(err)
os.Exit(1)
panic(err)
}
err = os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", fakeCredentialsAbsPath)
if err != nil {
fmt.Print(err)
os.Exit(1)
panic(err)
}
// reset it after the run
@ -92,8 +112,7 @@ func TestMain(m *testing.M) {
var c *storage.Client
c, err = storage.NewClient(ctx)
if err != nil {
fmt.Print(err)
os.Exit(1)
panic(err)
}
client := stiface.AdaptClient(c)
@ -101,18 +120,12 @@ func TestMain(m *testing.M) {
mockClient := newClientMock()
mockClient.Client = client
bucket := mockClient.Bucket("a-test-bucket")
gcsAfs = &Afero{Fs: &GcsFs{gcsfs.NewGcsFs(ctx, mockClient)}}
// If you want to run the test suite on a LIVE bucket, comment the previous
// block and uncomment the line below and put your bucket name there.
// Keep in mind, that GCS will likely rate limit you, so it would be impossible
// to run the entire suite at once, only test by test.
//bucket := client.Bucket("a-test-bucket")
// Uncomment to use the real, not mocked, client
//gcsAfs = &Afero{Fs: &GcsFs{gcsfs.NewGcsFs(ctx, client)}}
gcsAfs = &Afero{Fs: &GcsFs{gcsfs.NewGcsFs(ctx, bucket)}}
// defer here to assure our Env cleanup happens, if the mock was used
defer os.Exit(m.Run())
exitCode = m.Run()
}
func createFiles(t *testing.T) {
@ -122,8 +135,10 @@ func createFiles(t *testing.T) {
// the files have to be created first
for _, f := range files {
if !f.isdir && f.exists {
name := filepath.Join(bucketName, f.name)
var freshFile File
freshFile, err = gcsAfs.Create(f.name)
freshFile, err = gcsAfs.Create(name)
if err != nil {
t.Fatalf("failed to create a file \"%s\": %s", f.name, err)
}
@ -160,7 +175,9 @@ func removeFiles(t *testing.T) {
// the files have to be created first
for _, f := range files {
if !f.isdir && f.exists {
err = gcsAfs.Remove(f.name)
name := filepath.Join(bucketName, f.name)
err = gcsAfs.Remove(name)
if err != nil && err == syscall.ENOENT {
t.Errorf("failed to remove file \"%s\": %s", f.name, err)
}
@ -168,43 +185,55 @@ func removeFiles(t *testing.T) {
}
}
func TestFsOpen(t *testing.T) {
func TestGcsFsOpen(t *testing.T) {
createFiles(t)
defer removeFiles(t)
for _, f := range files {
file, err := gcsAfs.Open(f.name)
if (err == nil) != f.exists {
t.Errorf("%v exists = %v, but got err = %v", f.name, f.exists, err)
nameBase := filepath.Join(bucketName, f.name)
names := []string{
nameBase,
string(os.PathSeparator) + nameBase,
}
if f.name == "" {
names = []string{f.name}
}
if !f.exists {
continue
}
if err != nil {
t.Fatalf("%v: %v", f.name, err)
}
for _, name := range names {
file, err := gcsAfs.Open(name)
if (err == nil) != f.exists {
t.Errorf("%v exists = %v, but got err = %v", name, f.exists, err)
}
if file.Name() != filepath.FromSlash(f.name) {
t.Errorf("Name(), got %v, expected %v", file.Name(), filepath.FromSlash(f.name))
}
if !f.exists {
continue
}
if err != nil {
t.Fatalf("%v: %v", name, err)
}
s, err := file.Stat()
if err != nil {
t.Fatalf("stat %v: got error '%v'", file.Name(), err)
}
if file.Name() != filepath.FromSlash(nameBase) {
t.Errorf("Name(), got %v, expected %v", file.Name(), filepath.FromSlash(nameBase))
}
if isdir := s.IsDir(); isdir != f.isdir {
t.Errorf("%v directory, got: %v, expected: %v", file.Name(), isdir, f.isdir)
}
s, err := file.Stat()
if err != nil {
t.Fatalf("stat %v: got error '%v'", file.Name(), err)
}
if size := s.Size(); size != f.size {
t.Errorf("%v size, got: %v, expected: %v", file.Name(), size, f.size)
if isdir := s.IsDir(); isdir != f.isdir {
t.Errorf("%v directory, got: %v, expected: %v", file.Name(), isdir, f.isdir)
}
if size := s.Size(); size != f.size {
t.Errorf("%v size, got: %v, expected: %v", file.Name(), size, f.size)
}
}
}
}
func TestRead(t *testing.T) {
func TestGcsRead(t *testing.T) {
createFiles(t)
defer removeFiles(t)
@ -213,25 +242,36 @@ func TestRead(t *testing.T) {
continue
}
file, err := gcsAfs.Open(f.name)
if err != nil {
t.Fatalf("opening %v: %v", f.name, err)
nameBase := filepath.Join(bucketName, f.name)
names := []string{
nameBase,
string(os.PathSeparator) + nameBase,
}
if f.name == "" {
names = []string{f.name}
}
buf := make([]byte, 8)
n, err := file.Read(buf)
if err != nil {
if f.isdir && (err != syscall.EISDIR) {
t.Errorf("%v got error %v, expected EISDIR", f.name, err)
} else if !f.isdir {
t.Errorf("%v: %v", f.name, err)
for _, name := range names {
file, err := gcsAfs.Open(name)
if err != nil {
t.Fatalf("opening %v: %v", name, err)
}
} else if n != 8 {
t.Errorf("%v: got %d read bytes, expected 8", f.name, n)
} else if string(buf) != strings.Repeat(f.content, testBytes) {
t.Errorf("%v: got <%s>, expected <%s>", f.name, f.content, string(buf))
}
buf := make([]byte, 8)
n, err := file.Read(buf)
if err != nil {
if f.isdir && (err != syscall.EISDIR) {
t.Errorf("%v got error %v, expected EISDIR", name, err)
} else if !f.isdir {
t.Errorf("%v: %v", name, err)
}
} else if n != 8 {
t.Errorf("%v: got %d read bytes, expected 8", name, n)
} else if string(buf) != strings.Repeat(f.content, testBytes) {
t.Errorf("%v: got <%s>, expected <%s>", f.name, f.content, string(buf))
}
}
}
}
@ -244,25 +284,36 @@ func TestGcsReadAt(t *testing.T) {
continue
}
file, err := gcsAfs.Open(f.name)
if err != nil {
t.Fatalf("opening %v: %v", f.name, err)
nameBase := filepath.Join(bucketName, f.name)
names := []string{
nameBase,
string(os.PathSeparator) + nameBase,
}
if f.name == "" {
names = []string{f.name}
}
buf := make([]byte, testBytes)
n, err := file.ReadAt(buf, f.offset-testBytes/2)
if err != nil {
if f.isdir && (err != syscall.EISDIR) {
t.Errorf("%v got error %v, expected EISDIR", f.name, err)
} else if !f.isdir {
t.Errorf("%v: %v", f.name, err)
for _, name := range names {
file, err := gcsAfs.Open(name)
if err != nil {
t.Fatalf("opening %v: %v", name, err)
}
} else if n != 8 {
t.Errorf("%v: got %d read bytes, expected 8", f.name, n)
} else if string(buf) != strings.Repeat(f.content, testBytes/2)+strings.Repeat(f.contentAtOffset, testBytes/2) {
t.Errorf("%v: got <%s>, expected <%s>", f.name, f.contentAtOffset, string(buf))
}
buf := make([]byte, testBytes)
n, err := file.ReadAt(buf, f.offset-testBytes/2)
if err != nil {
if f.isdir && (err != syscall.EISDIR) {
t.Errorf("%v got error %v, expected EISDIR", name, err)
} else if !f.isdir {
t.Errorf("%v: %v", name, err)
}
} else if n != 8 {
t.Errorf("%v: got %d read bytes, expected 8", f.name, n)
} else if string(buf) != strings.Repeat(f.content, testBytes/2)+strings.Repeat(f.contentAtOffset, testBytes/2) {
t.Errorf("%v: got <%s>, expected <%s>", f.name, f.contentAtOffset, string(buf))
}
}
}
}
@ -275,43 +326,55 @@ func TestGcsSeek(t *testing.T) {
continue
}
file, err := gcsAfs.Open(f.name)
if err != nil {
t.Fatalf("opening %v: %v", f.name, err)
nameBase := filepath.Join(bucketName, f.name)
names := []string{
nameBase,
string(os.PathSeparator) + nameBase,
}
if f.name == "" {
names = []string{f.name}
}
var tests = []struct {
offIn int64
whence int
offOut int64
}{
{0, io.SeekStart, 0},
{10, io.SeekStart, 10},
{1, io.SeekCurrent, 11},
{10, io.SeekCurrent, 21},
{0, io.SeekEnd, f.size},
{-1, io.SeekEnd, f.size - 1},
}
for _, s := range tests {
n, err := file.Seek(s.offIn, s.whence)
for _, name := range names {
file, err := gcsAfs.Open(name)
if err != nil {
if f.isdir && err == syscall.EISDIR {
continue
t.Fatalf("opening %v: %v", name, err)
}
var tests = []struct {
offIn int64
whence int
offOut int64
}{
{0, io.SeekStart, 0},
{10, io.SeekStart, 10},
{1, io.SeekCurrent, 11},
{10, io.SeekCurrent, 21},
{0, io.SeekEnd, f.size},
{-1, io.SeekEnd, f.size - 1},
}
for _, s := range tests {
n, err := file.Seek(s.offIn, s.whence)
if err != nil {
if f.isdir && err == syscall.EISDIR {
continue
}
t.Errorf("%v: %v", name, err)
}
t.Errorf("%v: %v", f.name, err)
}
if n != s.offOut {
t.Errorf("%v: (off: %v, whence: %v): got %v, expected %v", f.name, s.offIn, s.whence, n, s.offOut)
if n != s.offOut {
t.Errorf("%v: (off: %v, whence: %v): got %v, expected %v", f.name, s.offIn, s.whence, n, s.offOut)
}
}
}
}
}
func TestName(t *testing.T) {
func TestGcsName(t *testing.T) {
createFiles(t)
defer removeFiles(t)
@ -320,20 +383,32 @@ func TestName(t *testing.T) {
continue
}
file, err := gcsAfs.Open(f.name)
if err != nil {
t.Fatalf("opening %v: %v", f.name, err)
nameBase := filepath.Join(bucketName, f.name)
names := []string{
nameBase,
string(os.PathSeparator) + nameBase,
}
if f.name == "" {
names = []string{f.name}
}
n := file.Name()
if n != filepath.FromSlash(f.name) {
t.Errorf("got: %v, expected: %v", n, filepath.FromSlash(f.name))
for _, name := range names {
file, err := gcsAfs.Open(name)
if err != nil {
t.Fatalf("opening %v: %v", name, err)
}
n := file.Name()
if n != filepath.FromSlash(nameBase) {
t.Errorf("got: %v, expected: %v", n, filepath.FromSlash(nameBase))
}
}
}
}
func TestClose(t *testing.T) {
func TestGcsClose(t *testing.T) {
createFiles(t)
defer removeFiles(t)
@ -342,35 +417,47 @@ func TestClose(t *testing.T) {
continue
}
file, err := gcsAfs.Open(f.name)
if err != nil {
t.Fatalf("opening %v: %v", f.name, err)
nameBase := filepath.Join(bucketName, f.name)
names := []string{
nameBase,
string(os.PathSeparator) + nameBase,
}
if f.name == "" {
names = []string{f.name}
}
err = file.Close()
if err != nil {
t.Errorf("%v: %v", f.name, err)
}
for _, name := range names {
file, err := gcsAfs.Open(name)
if err != nil {
t.Fatalf("opening %v: %v", name, err)
}
err = file.Close()
if err == nil {
t.Errorf("%v: closing twice should return an error", f.name)
}
err = file.Close()
if err != nil {
t.Errorf("%v: %v", name, err)
}
buf := make([]byte, 8)
n, err := file.Read(buf)
if n != 0 || err == nil {
t.Errorf("%v: could read from a closed file", f.name)
}
err = file.Close()
if err == nil {
t.Errorf("%v: closing twice should return an error", name)
}
n, err = file.ReadAt(buf, 256)
if n != 0 || err == nil {
t.Errorf("%v: could readAt from a closed file", f.name)
}
buf := make([]byte, 8)
n, err := file.Read(buf)
if n != 0 || err == nil {
t.Errorf("%v: could read from a closed file", name)
}
off, err := file.Seek(0, io.SeekStart)
if off != 0 || err == nil {
t.Errorf("%v: could seek from a closed file", f.name)
n, err = file.ReadAt(buf, 256)
if n != 0 || err == nil {
t.Errorf("%v: could readAt from a closed file", name)
}
off, err := file.Seek(0, io.SeekStart)
if off != 0 || err == nil {
t.Errorf("%v: could seek from a closed file", name)
}
}
}
}
@ -380,56 +467,81 @@ func TestGcsOpenFile(t *testing.T) {
defer removeFiles(t)
for _, f := range files {
file, err := gcsAfs.OpenFile(f.name, os.O_RDONLY, 0400)
if !f.exists {
if !errors.Is(err, syscall.ENOENT) {
t.Errorf("%v: got %v, expected%v", f.name, err, syscall.ENOENT)
nameBase := filepath.Join(bucketName, f.name)
names := []string{
nameBase,
string(os.PathSeparator) + nameBase,
}
if f.name == "" {
names = []string{f.name}
}
for _, name := range names {
file, err := gcsAfs.OpenFile(name, os.O_RDONLY, 0400)
if !f.exists {
if (f.name != "" && !errors.Is(err, syscall.ENOENT)) ||
(f.name == "" && !errors.Is(err, gcsfs.ErrNoBucketInName)) {
t.Errorf("%v: got %v, expected%v", name, err, syscall.ENOENT)
}
continue
}
continue
}
if err != nil {
t.Fatalf("%v: %v", name, err)
}
if err != nil {
t.Fatalf("%v: %v", f.name, err)
}
err = file.Close()
if err != nil {
t.Fatalf("failed to close a file \"%s\": %s", name, err)
}
err = file.Close()
if err != nil {
t.Fatalf("failed to close a file \"%s\": %s", f.name, err)
file, err = gcsAfs.OpenFile(name, os.O_CREATE, 0600)
if !errors.Is(err, syscall.EPERM) {
t.Errorf("%v: open for write: got %v, expected %v", name, err, syscall.EPERM)
}
}
file, err = gcsAfs.OpenFile(f.name, os.O_CREATE, 0600)
if !errors.Is(err, syscall.EPERM) {
t.Errorf("%v: open for write: got %v, expected %v", f.name, err, syscall.EPERM)
}
}
}
func TestFsStat(t *testing.T) {
func TestGcsFsStat(t *testing.T) {
createFiles(t)
defer removeFiles(t)
for _, f := range files {
fi, err := gcsAfs.Stat(f.name)
if !f.exists {
if !errors.Is(err, syscall.ENOENT) {
t.Errorf("%v: got %v, expected%v", f.name, err, syscall.ENOENT)
nameBase := filepath.Join(bucketName, f.name)
names := []string{
nameBase,
string(os.PathSeparator) + nameBase,
}
if f.name == "" {
names = []string{f.name}
}
for _, name := range names {
fi, err := gcsAfs.Stat(name)
if !f.exists {
if (f.name != "" && !errors.Is(err, syscall.ENOENT)) ||
(f.name == "" && !errors.Is(err, gcsfs.ErrNoBucketInName)) {
t.Errorf("%v: got %v, expected%v", name, err, syscall.ENOENT)
}
continue
}
continue
}
if err != nil {
t.Fatalf("stat %v: got error '%v'", name, err)
}
if err != nil {
t.Fatalf("stat %v: got error '%v'", f.name, err)
}
if isdir := fi.IsDir(); isdir != f.isdir {
t.Errorf("%v directory, got: %v, expected: %v", name, isdir, f.isdir)
}
if isdir := fi.IsDir(); isdir != f.isdir {
t.Errorf("%v directory, got: %v, expected: %v", f.name, isdir, f.isdir)
}
if size := fi.Size(); size != f.size {
t.Errorf("%v size, got: %v, expected: %v", f.name, size, f.size)
if size := fi.Size(); size != f.size {
t.Errorf("%v size, got: %v, expected: %v", name, size, f.size)
}
}
}
}
@ -439,47 +551,65 @@ func TestGcsReaddir(t *testing.T) {
defer removeFiles(t)
for _, d := range dirs {
dir, err := gcsAfs.Open(d.name)
if err != nil {
t.Fatal(err)
nameBase := filepath.Join(bucketName, d.name)
names := []string{
nameBase,
string(os.PathSeparator) + nameBase,
}
fi, err := dir.Readdir(0)
if err != nil {
t.Fatal(err)
}
var names []string
for _, f := range fi {
names = append(names, f.Name())
}
for _, name := range names {
dir, err := gcsAfs.Open(name)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(names, d.children) {
t.Errorf("%v: children, got '%v', expected '%v'", d.name, names, d.children)
}
fi, err := dir.Readdir(0)
if err != nil {
t.Fatal(err)
}
var fileNames []string
for _, f := range fi {
fileNames = append(fileNames, f.Name())
}
fi, err = dir.Readdir(1)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(fileNames, d.children) {
t.Errorf("%v: children, got '%v', expected '%v'", name, fileNames, d.children)
}
names = []string{}
for _, f := range fi {
names = append(names, f.Name())
}
fi, err = dir.Readdir(1)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(names, d.children[0:1]) {
t.Errorf("%v: children, got '%v', expected '%v'", d.name, names, d.children[0:1])
fileNames = []string{}
for _, f := range fi {
fileNames = append(fileNames, f.Name())
}
if !reflect.DeepEqual(fileNames, d.children[0:1]) {
t.Errorf("%v: children, got '%v', expected '%v'", name, fileNames, d.children[0:1])
}
}
}
dir, err := gcsAfs.Open("testFile")
if err != nil {
t.Fatal(err)
nameBase := filepath.Join(bucketName, "testFile")
names := []string{
nameBase,
string(os.PathSeparator) + nameBase,
}
_, err = dir.Readdir(-1)
if err != syscall.ENOTDIR {
t.Fatal("Expected error")
for _, name := range names {
dir, err := gcsAfs.Open(name)
if err != nil {
t.Fatal(err)
}
_, err = dir.Readdir(-1)
if err != syscall.ENOTDIR {
t.Fatal("Expected error")
}
}
}
@ -488,38 +618,56 @@ func TestGcsReaddirnames(t *testing.T) {
defer removeFiles(t)
for _, d := range dirs {
dir, err := gcsAfs.Open(d.name)
if err != nil {
t.Fatal(err)
nameBase := filepath.Join(bucketName, d.name)
names := []string{
nameBase,
string(os.PathSeparator) + nameBase,
}
names, err := dir.Readdirnames(0)
if err != nil {
t.Fatal(err)
}
for _, name := range names {
dir, err := gcsAfs.Open(name)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(names, d.children) {
t.Errorf("%v: children, got '%v', expected '%v'", d.name, names, d.children)
}
fileNames, err := dir.Readdirnames(0)
if err != nil {
t.Fatal(err)
}
names, err = dir.Readdirnames(1)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(fileNames, d.children) {
t.Errorf("%v: children, got '%v', expected '%v'", name, fileNames, d.children)
}
if !reflect.DeepEqual(names, d.children[0:1]) {
t.Errorf("%v: children, got '%v', expected '%v'", d.name, names, d.children[0:1])
fileNames, err = dir.Readdirnames(1)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(fileNames, d.children[0:1]) {
t.Errorf("%v: children, got '%v', expected '%v'", name, fileNames, d.children[0:1])
}
}
}
dir, err := gcsAfs.Open("testFile")
if err != nil {
t.Fatal(err)
nameBase := filepath.Join(bucketName, "testFile")
names := []string{
nameBase,
string(os.PathSeparator) + nameBase,
}
_, err = dir.Readdir(-1)
if err != syscall.ENOTDIR {
t.Fatal("Expected error")
for _, name := range names {
dir, err := gcsAfs.Open(name)
if err != nil {
t.Fatal(err)
}
_, err = dir.Readdirnames(-1)
if err != syscall.ENOTDIR {
t.Fatal("Expected error")
}
}
}
@ -531,29 +679,40 @@ func TestGcsGlob(t *testing.T) {
glob string
entries []string
}{
{filepath.FromSlash("/*"), []string{filepath.FromSlash("/sub"), filepath.FromSlash("/testDir1"), filepath.FromSlash("/testFile")}},
{filepath.FromSlash("*"), []string{filepath.FromSlash("sub"), filepath.FromSlash("testDir1"), filepath.FromSlash("testFile")}},
{filepath.FromSlash("/sub/*"), []string{filepath.FromSlash("/sub/testDir2")}},
{filepath.FromSlash("/sub/testDir2/*"), []string{filepath.FromSlash("/sub/testDir2/testFile")}},
{filepath.FromSlash("/testDir1/*"), []string{filepath.FromSlash("/testDir1/testFile")}},
{filepath.FromSlash("sub/*"), []string{filepath.FromSlash("sub/testDir2")}},
{filepath.FromSlash("sub/testDir2/*"), []string{filepath.FromSlash("sub/testDir2/testFile")}},
{filepath.FromSlash("testDir1/*"), []string{filepath.FromSlash("testDir1/testFile")}},
} {
entries, err := Glob(gcsAfs.Fs, s.glob)
if err != nil {
t.Error(err)
nameBase := filepath.Join(bucketName, s.glob)
prefixedGlobs := []string{
nameBase,
string(os.PathSeparator) + nameBase,
}
if reflect.DeepEqual(entries, s.entries) {
t.Logf("glob: %s: glob ok", s.glob)
} else {
t.Errorf("glob: %s: got %#v, expected %#v", s.glob, entries, s.entries)
prefixedEntries := [][]string{{}, {}}
for _, entry := range s.entries {
prefixedEntries[0] = append(prefixedEntries[0], filepath.Join(bucketName, entry))
prefixedEntries[1] = append(prefixedEntries[1], string(os.PathSeparator)+filepath.Join(bucketName, entry))
}
for i, prefixedGlob := range prefixedGlobs {
entries, err := Glob(gcsAfs.Fs, prefixedGlob)
if err != nil {
t.Error(err)
}
if reflect.DeepEqual(entries, prefixedEntries[i]) {
t.Logf("glob: %s: glob ok", prefixedGlob)
} else {
t.Errorf("glob: %s: got %#v, expected %#v", prefixedGlob, entries, prefixedEntries)
}
}
}
}
func TestMkdir(t *testing.T) {
dirName := "/a-test-dir"
func TestGcsMkdir(t *testing.T) {
dirName := filepath.Join(bucketName, "a-test-dir")
var err error
err = gcsAfs.Mkdir(dirName, 0755)
@ -582,45 +741,47 @@ func TestMkdir(t *testing.T) {
}
}
func TestMkdirAll(t *testing.T) {
err := gcsAfs.MkdirAll("/a/b/c", 0755)
func TestGcsMkdirAll(t *testing.T) {
dirName := filepath.Join(bucketName, "a/b/c")
err := gcsAfs.MkdirAll(dirName, 0755)
if err != nil {
t.Fatal(err)
}
info, err := gcsAfs.Stat("/a")
info, err := gcsAfs.Stat(filepath.Join(bucketName, "a"))
if err != nil {
t.Fatal(err)
}
if !info.Mode().IsDir() {
t.Error("/a: mode is not directory")
t.Errorf("%s: mode is not directory", filepath.Join(bucketName, "a"))
}
if info.Mode() != os.ModeDir|0755 {
t.Errorf("/a: wrong permissions, expected drwxr-xr-x, got %s", info.Mode())
t.Errorf("%s: wrong permissions, expected drwxr-xr-x, got %s", filepath.Join(bucketName, "a"), info.Mode())
}
info, err = gcsAfs.Stat("/a/b")
info, err = gcsAfs.Stat(filepath.Join(bucketName, "a/b"))
if err != nil {
t.Fatal(err)
}
if !info.Mode().IsDir() {
t.Error("/a/b: mode is not directory")
t.Errorf("%s: mode is not directory", filepath.Join(bucketName, "a/b"))
}
if info.Mode() != os.ModeDir|0755 {
t.Errorf("/a/b: wrong permissions, expected drwxr-xr-x, got %s", info.Mode())
t.Errorf("%s: wrong permissions, expected drwxr-xr-x, got %s", filepath.Join(bucketName, "a/b"), info.Mode())
}
info, err = gcsAfs.Stat("/a/b/c")
info, err = gcsAfs.Stat(dirName)
if err != nil {
t.Fatal(err)
}
if !info.Mode().IsDir() {
t.Error("/a/b/c: mode is not directory")
t.Errorf("%s: mode is not directory", dirName)
}
if info.Mode() != os.ModeDir|0755 {
t.Errorf("/a/b/c: wrong permissions, expected drwxr-xr-x, got %s", info.Mode())
t.Errorf("%s: wrong permissions, expected drwxr-xr-x, got %s", dirName, info.Mode())
}
err = gcsAfs.RemoveAll("/a")
err = gcsAfs.RemoveAll(filepath.Join(bucketName, "a"))
if err != nil {
t.Fatalf("failed to remove the folder /a with error: %s", err)
t.Fatalf("failed to remove the folder %s with error: %s", filepath.Join(bucketName, "a"), err)
}
}

1
gcsfs/errors.go
View File

@ -22,6 +22,7 @@ import (
)
var (
ErrNoBucketInName = errors.New("no bucket name found in the name")
ErrFileClosed = errors.New("file is closed")
ErrOutOfRange = errors.New("out of range")
ErrObjectDoesNotExist = errors.New("storage: object doesn't exist")

10
gcsfs/file.go
View File

@ -195,11 +195,13 @@ func (o *GcsFile) readdirImpl(count int) ([]*FileInfo, error) {
return nil, syscall.ENOTDIR
}
path := o.resource.fs.ensureTrailingSeparator(o.Name())
path := o.resource.fs.ensureTrailingSeparator(o.resource.name)
if o.ReadDirIt == nil {
//log.Printf("Querying path : %s\n", path)
o.ReadDirIt = o.resource.fs.bucket.Objects(
o.resource.ctx, &storage.Query{Delimiter: o.resource.fs.separator, Prefix: path, Versions: false})
bucketName, bucketPath := o.resource.fs.splitName(path)
o.ReadDirIt = o.resource.fs.client.Bucket(bucketName).Objects(
o.resource.ctx, &storage.Query{Delimiter: o.resource.fs.separator, Prefix: bucketPath, Versions: false})
}
var res []*FileInfo
for {
@ -283,7 +285,7 @@ func (o *GcsFile) Stat() (os.FileInfo, error) {
return nil, err
}
return newFileInfo(o.Name(), o.resource.fs, o.resource.fileMode)
return newFileInfo(o.resource.name, o.resource.fs, o.resource.fileMode)
}
func (o *GcsFile) Sync() error {

12
gcsfs/file_info.go
View File

@ -46,7 +46,10 @@ func newFileInfo(name string, fs *GcsFs, fileMode os.FileMode) (*FileInfo, error
fileMode: fileMode,
}
obj := fs.getObj(name)
obj, err := fs.getObj(name)
if err != nil {
return nil, err
}
objAttrs, err := obj.Attrs(fs.ctx)
if err != nil {
@ -58,8 +61,9 @@ func newFileInfo(name string, fs *GcsFs, fileMode os.FileMode) (*FileInfo, error
} else if err.Error() == ErrObjectDoesNotExist.Error() {
// Folders do not actually "exist" in GCloud, so we have to check, if something exists with
// such a prefix
it := fs.bucket.Objects(
fs.ctx, &storage.Query{Delimiter: fs.separator, Prefix: name, Versions: false})
bucketName, bucketPath := fs.splitName(name)
it := fs.client.Bucket(bucketName).Objects(
fs.ctx, &storage.Query{Delimiter: fs.separator, Prefix: bucketPath, Versions: false})
if _, err = it.Next(); err == nil {
res.name = fs.ensureTrailingSeparator(res.name)
res.isDir = true
@ -100,7 +104,7 @@ func newFileInfoFromAttrs(objAttrs *storage.ObjectAttrs, fileMode os.FileMode) *
}
func (fi *FileInfo) Name() string {
return filepath.Base(fi.name)
return filepath.Base(filepath.FromSlash(fi.name))
}
func (fi *FileInfo) Size() int64 {

212
gcsfs/fs.go
View File

@ -19,7 +19,6 @@ package gcsfs
import (
"context"
"errors"
"log"
"os"
"path/filepath"
"strings"
@ -31,26 +30,29 @@ import (
const (
defaultFileMode = 0755
gsPrefix = "gs://"
)
// GcsFs is a Fs implementation that uses functions provided by google cloud storage
type GcsFs struct {
ctx context.Context
bucket stiface.BucketHandle
separator string
ctx context.Context
client stiface.Client
separator string
buckets map[string]stiface.BucketHandle
rawGcsObjects map[string]*GcsFile
autoRemoveEmptyFolders bool //trigger for creating "virtual folders" (not required by GCSs)
}
func NewGcsFs(ctx context.Context, bucket stiface.BucketHandle) *GcsFs {
return NewGcsFsWithSeparator(ctx, bucket, "/")
func NewGcsFs(ctx context.Context, client stiface.Client) *GcsFs {
return NewGcsFsWithSeparator(ctx, client, "/")
}
func NewGcsFsWithSeparator(ctx context.Context, bucket stiface.BucketHandle, folderSep string) *GcsFs {
func NewGcsFsWithSeparator(ctx context.Context, client stiface.Client, folderSep string) *GcsFs {
return &GcsFs{
ctx: ctx,
bucket: bucket,
client: client,
separator: folderSep,
rawGcsObjects: make(map[string]*GcsFile),
@ -69,39 +71,65 @@ func (fs *GcsFs) ensureTrailingSeparator(s string) string {
}
return s
}
func (fs *GcsFs) ensureNoLeadingSeparators(s string) string {
// GCS does REALLY not like the names, that begin with a separator
func (fs *GcsFs) ensureNoLeadingSeparator(s string) string {
if len(s) > 0 && strings.HasPrefix(s, fs.separator) {
log.Printf(
"WARNING: the provided path \"%s\" starts with a separator \"%s\", which is not supported by "+
"GCloud. The separator will be automatically trimmed",
s,
fs.separator,
)
return s[len(fs.separator):]
s = s[len(fs.separator):]
}
return s
}
func ensureNoPrefix(s string) string {
if len(s) > 0 && strings.HasPrefix(s, gsPrefix) {
return s[len(gsPrefix):]
}
return s
}
func correctTheDot(s string) string {
// So, Afero's Glob likes to give "." as a name - that to list the "empty" dir name.
// GCS _really_ dislikes the dot and gives no entries for it - so we should rather replace the dot
// with an empty string
if s == "." {
return ""
func validateName(s string) error {
if len(s) == 0 {
return ErrNoBucketInName
}
return s
return nil
}
func (fs *GcsFs) getObj(name string) stiface.ObjectHandle {
return fs.bucket.Object(name)
// Splits provided name into bucket name and path
func (fs *GcsFs) splitName(name string) (bucketName string, path string) {
splitName := strings.Split(name, fs.separator)
return splitName[0], strings.Join(splitName[1:], fs.separator)
}
func (fs *GcsFs) getBucket(name string) (stiface.BucketHandle, error) {
bucket := fs.buckets[name]
if bucket == nil {
bucket = fs.client.Bucket(name)
_, err := bucket.Attrs(fs.ctx)
if err != nil {
return nil, err
}
}
return bucket, nil
}
func (fs *GcsFs) getObj(name string) (stiface.ObjectHandle, error) {
bucketName, path := fs.splitName(name)
bucket, err := fs.getBucket(bucketName)
if err != nil {
return nil, err
}
return bucket.Object(path), nil
}
func (fs *GcsFs) Name() string { return "GcsFs" }
func (fs *GcsFs) Create(name string) (*GcsFile, error) {
name = fs.ensureNoLeadingSeparators(fs.normSeparators(correctTheDot(name)))
name = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(name)))
if err := validateName(name); err != nil {
return nil, err
}
if !fs.autoRemoveEmptyFolders {
baseDir := filepath.Base(name)
@ -113,9 +141,11 @@ func (fs *GcsFs) Create(name string) (*GcsFile, error) {
}
}
obj := fs.getObj(name)
obj, err := fs.getObj(name)
if err != nil {
return nil, err
}
w := obj.NewWriter(fs.ctx)
var err error
err = w.Close()
if err != nil {
return nil, err
@ -127,15 +157,24 @@ func (fs *GcsFs) Create(name string) (*GcsFile, error) {
}
func (fs *GcsFs) Mkdir(name string, _ os.FileMode) error {
name = fs.ensureTrailingSeparator(fs.ensureNoLeadingSeparators(fs.normSeparators(correctTheDot(name))))
name = fs.ensureNoLeadingSeparator(fs.ensureTrailingSeparator(fs.normSeparators(ensureNoPrefix(name))))
if err := validateName(name); err != nil {
return err
}
obj := fs.getObj(name)
obj, err := fs.getObj(name)
if err != nil {
return err
}
w := obj.NewWriter(fs.ctx)
return w.Close()
}
func (fs *GcsFs) MkdirAll(path string, perm os.FileMode) error {
path = fs.ensureTrailingSeparator(fs.ensureNoLeadingSeparators(fs.normSeparators(correctTheDot(path))))
path = fs.ensureNoLeadingSeparator(fs.ensureTrailingSeparator(fs.normSeparators(ensureNoPrefix(path))))
if err := validateName(path); err != nil {
return err
}
root := ""
folders := strings.Split(path, fs.separator)
@ -165,13 +204,21 @@ func (fs *GcsFs) OpenFile(name string, flag int, fileMode os.FileMode) (*GcsFile
var file *GcsFile
var err error
name = fs.ensureNoLeadingSeparators(fs.normSeparators(correctTheDot(name)))
name = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(name)))
if err = validateName(name); err != nil {
return nil, err
}
obj, found := fs.rawGcsObjects[name]
f, found := fs.rawGcsObjects[name]
if found {
file = NewGcsFileFromOldFH(flag, fileMode, obj.resource)
file = NewGcsFileFromOldFH(flag, fileMode, f.resource)
} else {
file = NewGcsFile(fs.ctx, fs, fs.getObj(name), flag, fileMode, name)
var obj stiface.ObjectHandle
obj, err = fs.getObj(name)
if err != nil {
return nil, err
}
file = NewGcsFile(fs.ctx, fs, obj, flag, fileMode, name)
}
if flag == os.O_RDONLY {
@ -211,9 +258,15 @@ func (fs *GcsFs) OpenFile(name string, flag int, fileMode os.FileMode) (*GcsFile
}
func (fs *GcsFs) Remove(name string) error {
name = fs.ensureNoLeadingSeparators(fs.normSeparators(correctTheDot(name)))
name = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(name)))
if err := validateName(name); err != nil {
return err
}
obj := fs.getObj(name)
obj, err := fs.getObj(name)
if err != nil {
return err
}
info, err := fs.Stat(name)
if err != nil {
return err
@ -235,7 +288,10 @@ func (fs *GcsFs) Remove(name string) error {
// it's an empty folder, we can continue
name = fs.ensureTrailingSeparator(name)
obj = fs.getObj(name)
obj, err = fs.getObj(name)
if err != nil {
return err
}
return obj.Delete(fs.ctx)
}
@ -243,7 +299,10 @@ func (fs *GcsFs) Remove(name string) error {
}
func (fs *GcsFs) RemoveAll(path string) error {
path = fs.ensureNoLeadingSeparators(fs.normSeparators(correctTheDot(path)))
path = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(path)))
if err := validateName(path); err != nil {
return err
}
pathInfo, err := fs.Stat(path)
if err != nil {
@ -262,63 +321,37 @@ func (fs *GcsFs) RemoveAll(path string) error {
var infos []os.FileInfo
infos, err = dir.Readdir(0)
for _, info := range infos {
err = fs.RemoveAll(path + fs.separator + info.Name())
nameToRemove := fs.normSeparators(info.Name())
err = fs.RemoveAll(path + fs.separator + nameToRemove)
if err != nil {
return err
}
}
return fs.Remove(path)
//it := fs.bucket.Objects(fs.ctx, &storage.Query{Delimiter: fs.separator, Prefix: path, Versions: false})
//for {
// objAttrs, err := it.Next()
// if err == iterator.Done {
// break
// }
// if err != nil {
// return err
// }
//
// name := objAttrs.Name
// if name == "" {
// name = objAttrs.Prefix
// }
//
// if name == path {
// // somehow happens
// continue
// }
// if objAttrs.Name == "" && objAttrs.Prefix != "" {
// // it's a folder, let's try to remove it normally first
// err = fs.Remove(path + fs.separator + objAttrs.Name)
// if err != nil {
// if err == syscall.ENOTEMPTY {
// err = fs.RemoveAll(path + fs.separator + objAttrs.Name)
// }
// }
// if err != nil {
// return err
// }
//
// } else {
// err = fs.Remove(objAttrs.Name)
// if err != nil {
// return err
// }
// }
//}
//return nil
}
func (fs *GcsFs) Rename(oldName, newName string) error {
oldName = fs.ensureNoLeadingSeparators(fs.normSeparators(correctTheDot(oldName)))
newName = fs.ensureNoLeadingSeparators(fs.normSeparators(correctTheDot(newName)))
oldName = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(oldName)))
if err := validateName(oldName); err != nil {
return err
}
src := fs.bucket.Object(oldName)
dst := fs.bucket.Object(newName)
newName = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(newName)))
if err := validateName(newName); err != nil {
return err
}
if _, err := dst.CopierFrom(src).Run(fs.ctx); err != nil {
src, err := fs.getObj(oldName)
if err != nil {
return err
}
dst, err := fs.getObj(newName)
if err != nil {
return err
}
if _, err = dst.CopierFrom(src).Run(fs.ctx); err != nil {
return err
}
delete(fs.rawGcsObjects, oldName)
@ -326,7 +359,10 @@ func (fs *GcsFs) Rename(oldName, newName string) error {
}
func (fs *GcsFs) Stat(name string) (os.FileInfo, error) {
name = fs.ensureNoLeadingSeparators(fs.normSeparators(correctTheDot(name)))
name = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(name)))
if err := validateName(name); err != nil {
return nil, err
}
return newFileInfo(name, fs, defaultFileMode)
}

1
go.mod
View File

@ -6,6 +6,7 @@ require (
github.com/pkg/sftp v1.10.1
github.com/stretchr/testify v1.5.1
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9
golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99 // indirect
golang.org/x/text v0.3.4
google.golang.org/api v0.40.0
)