Changed abstraction to operate on client, not bucket

gcs.go

@@ -21,9 +21,11 @@ import (
 	"os"
 	"time"
 
+	"github.com/spf13/afero/gcsfs"
+
 	"cloud.google.com/go/storage"
 	"github.com/googleapis/google-cloud-go-testing/storage/stiface"
-	"github.com/spf13/afero/gcsfs"
+
 	"google.golang.org/api/option"
 )
 
@@ -31,47 +33,44 @@ type GcsFs struct {
 	source *gcsfs.GcsFs
 }
 
-// Creates a GCS file system, automatically instantiating and decorating the storage client.
+// NewGcsFS creates a GCS file system, automatically instantiating and decorating the storage client.
 // You can provide additional options to be passed to the client creation, as per
 // cloud.google.com/go/storage documentation
-func NewGcsFS(ctx context.Context, bucketName string, opts ...option.ClientOption) (Fs, error) {
+func NewGcsFS(ctx context.Context, opts ...option.ClientOption) (Fs, error) {
 	client, err := storage.NewClient(ctx, opts...)
 	if err != nil {
 		return nil, err
 	}
 
-	return NewGcsFSFromClient(ctx, client, bucketName)
+	return NewGcsFSFromClient(ctx, client)
 }
 
-// The same as NewGcsFS, but the files system will use the provided folder separator.
+// NewGcsFSWithSeparator is the same as NewGcsFS, but the files system will use the provided folder separator.
-func NewGcsFSWithSeparator(ctx context.Context, bucketName, folderSeparator string, opts ...option.ClientOption) (Fs, error) {
+func NewGcsFSWithSeparator(ctx context.Context, folderSeparator string, opts ...option.ClientOption) (Fs, error) {
 	client, err := storage.NewClient(ctx, opts...)
 	if err != nil {
 		return nil, err
 	}
 
-	return NewGcsFSFromClientWithSeparator(ctx, client, bucketName, folderSeparator)
+	return NewGcsFSFromClientWithSeparator(ctx, client, folderSeparator)
 }
 
-// Creates a GCS file system from a given storage client
+// NewGcsFSFromClient creates a GCS file system from a given storage client
-func NewGcsFSFromClient(ctx context.Context, client *storage.Client, bucketName string) (Fs, error) {
+func NewGcsFSFromClient(ctx context.Context, client *storage.Client) (Fs, error) {
 	c := stiface.AdaptClient(client)
 
-	bucket := c.Bucket(bucketName)
+	return &GcsFs{gcsfs.NewGcsFs(ctx, c)}, nil
-
-	return &GcsFs{gcsfs.NewGcsFs(ctx, bucket)}, nil
 }
 
-// Same as NewGcsFSFromClient, but the file system will use the provided folder separator.
+// NewGcsFSFromClientWithSeparator is the same as NewGcsFSFromClient, but the file system will use the provided folder separator.
-func NewGcsFSFromClientWithSeparator(ctx context.Context, client *storage.Client, bucketName, folderSeparator string) (Fs, error) {
+func NewGcsFSFromClientWithSeparator(ctx context.Context, client *storage.Client, folderSeparator string) (Fs, error) {
 	c := stiface.AdaptClient(client)
 
-	bucket := c.Bucket(bucketName)
+	return &GcsFs{gcsfs.NewGcsFsWithSeparator(ctx, c, folderSeparator)}, nil
-
-	return &GcsFs{gcsfs.NewGcsFsWithSeparator(ctx, bucket, folderSeparator)}, nil
 }
 
 // Wraps gcs.GcsFs and convert some return types to afero interfaces.
+
 func (fs *GcsFs) Name() string {
 	return fs.source.Name()
 }

gcs_mocks.go

@@ -50,6 +50,10 @@ type bucketMock struct {
 	fs Fs
 }
 
+func (m *bucketMock) Attrs(context.Context) (*storage.BucketAttrs, error) {
+	return &storage.BucketAttrs{}, nil
+}
+
 func (m *bucketMock) Object(name string) stiface.ObjectHandle {
 	return &objectMock{name: name, fs: m.fs}
 }
@@ -193,6 +197,10 @@ func (r *readerMock) Read(p []byte) (int, error) {
 	return r.file.Read(p)
 }
 
+func (r *readerMock) Close() error {
+	return r.file.Close()
+}
+
 type objectItMock struct {
 	stiface.ObjectIterator
 

gcs_test.go

@@ -17,6 +17,8 @@ import (
 	"syscall"
 	"testing"
 
+	"golang.org/x/oauth2/google"
+
 	"github.com/spf13/afero/gcsfs"
 
 	"cloud.google.com/go/storage"
@@ -28,6 +30,8 @@ const (
 	dirSize   = 42
 )
 
+var bucketName = "a-test-bucket"
+
 var files = []struct {
 	name            string
 	exists          bool
@@ -37,13 +41,14 @@ var files = []struct {
 	offset          int64
 	contentAtOffset string
 }{
-	{"", true, true, dirSize, "", 0, ""}, // this is NOT a valid path for GCS, so we do some magic here
 	{"sub", true, true, dirSize, "", 0, ""},
 	{"sub/testDir2", true, true, dirSize, "", 0, ""},
 	{"sub/testDir2/testFile", true, false, 8 * 1024, "c", 4 * 1024, "d"},
 	{"testFile", true, false, 12 * 1024, "a", 7 * 1024, "b"},
 	{"testDir1/testFile", true, false, 3 * 512, "b", 512, "c"},
 
+	{"", false, true, dirSize, "", 0, ""}, // special case
+
 	{"nonExisting", false, false, dirSize, "", 0, ""},
 }
 
@@ -51,7 +56,7 @@ var dirs = []struct {
 	name     string
 	children []string
 }{
-	{"", []string{"sub", "testDir1", "testFile"}},
+	{"", []string{"sub", "testDir1", "testFile"}}, // in this case it will be prepended with bucket name
 	{"sub", []string{"testDir2"}},
 	{"sub/testDir2", []string{"testFile"}},
 	{"testDir1", []string{"testFile"}},
@@ -63,20 +68,35 @@ func TestMain(m *testing.M) {
 	ctx := context.Background()
 	var err error
 
-	// Check if GOOGLE_APPLICATION_CREDENTIALS are present. If not, then a fake service account
+	// in order to respect deferring
+	var exitCode int
+	defer os.Exit(exitCode)
+
+	defer func() {
+		err := recover()
+		if err != nil {
+			fmt.Print(err)
+			exitCode = 2
+		}
+	}()
+
+	// Check if any credentials are present. If not, a fake service account, taken from the link
 	// would be used: https://github.com/google/oauth2l/blob/master/integration/fixtures/fake-service-account.json
-	if os.Getenv("GOOGLE_APPLICATION_CREDENTIALS") == "" {
+	cred, err := google.FindDefaultCredentials(ctx)
+	if err != nil && !strings.HasPrefix(err.Error(), "google: could not find default credentials") {
+		panic(err)
+	}
+
+	if cred == nil {
 		var fakeCredentialsAbsPath string
 		fakeCredentialsAbsPath, err = filepath.Abs("gcs-fake-service-account.json")
 		if err != nil {
-			fmt.Print(err)
+			panic(err)
-			os.Exit(1)
 		}
 
 		err = os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", fakeCredentialsAbsPath)
 		if err != nil {
-			fmt.Print(err)
+			panic(err)
-			os.Exit(1)
 		}
 
 		// reset it after the run
@@ -92,8 +112,7 @@ func TestMain(m *testing.M) {
 	var c *storage.Client
 	c, err = storage.NewClient(ctx)
 	if err != nil {
-		fmt.Print(err)
+		panic(err)
-		os.Exit(1)
 	}
 	client := stiface.AdaptClient(c)
 
@@ -101,18 +120,12 @@ func TestMain(m *testing.M) {
 	mockClient := newClientMock()
 	mockClient.Client = client
 
-	bucket := mockClient.Bucket("a-test-bucket")
+	gcsAfs = &Afero{Fs: &GcsFs{gcsfs.NewGcsFs(ctx, mockClient)}}
 
-	// If you want to run the test suite on a LIVE bucket, comment the previous
+	// Uncomment to use the real, not mocked, client
-	// block and uncomment the line below and put your bucket name there.
+	//gcsAfs = &Afero{Fs: &GcsFs{gcsfs.NewGcsFs(ctx, client)}}
-	// Keep in mind, that GCS will likely rate limit you, so it would be impossible
-	// to run the entire suite at once, only test by test.
-	//bucket := client.Bucket("a-test-bucket")
 
-	gcsAfs = &Afero{Fs: &GcsFs{gcsfs.NewGcsFs(ctx, bucket)}}
+	exitCode = m.Run()
-
-	// defer here to assure our Env cleanup happens, if the mock was used
-	defer os.Exit(m.Run())
 }
 
 func createFiles(t *testing.T) {
@@ -122,8 +135,10 @@ func createFiles(t *testing.T) {
 	// the files have to be created first
 	for _, f := range files {
 		if !f.isdir && f.exists {
+			name := filepath.Join(bucketName, f.name)
+
 			var freshFile File
-			freshFile, err = gcsAfs.Create(f.name)
+			freshFile, err = gcsAfs.Create(name)
 			if err != nil {
 				t.Fatalf("failed to create a file \"%s\": %s", f.name, err)
 			}
@@ -160,7 +175,9 @@ func removeFiles(t *testing.T) {
 	// the files have to be created first
 	for _, f := range files {
 		if !f.isdir && f.exists {
-			err = gcsAfs.Remove(f.name)
+			name := filepath.Join(bucketName, f.name)
+
+			err = gcsAfs.Remove(name)
 			if err != nil && err == syscall.ENOENT {
 				t.Errorf("failed to remove file \"%s\": %s", f.name, err)
 			}
@@ -168,25 +185,36 @@ func removeFiles(t *testing.T) {
 	}
 }
 
-func TestFsOpen(t *testing.T) {
+func TestGcsFsOpen(t *testing.T) {
 	createFiles(t)
 	defer removeFiles(t)
 
 	for _, f := range files {
-		file, err := gcsAfs.Open(f.name)
+		nameBase := filepath.Join(bucketName, f.name)
+
+		names := []string{
+			nameBase,
+			string(os.PathSeparator) + nameBase,
+		}
+		if f.name == "" {
+			names = []string{f.name}
+		}
+
+		for _, name := range names {
+			file, err := gcsAfs.Open(name)
 			if (err == nil) != f.exists {
-			t.Errorf("%v exists = %v, but got err = %v", f.name, f.exists, err)
+				t.Errorf("%v exists = %v, but got err = %v", name, f.exists, err)
 			}
 
 			if !f.exists {
 				continue
 			}
 			if err != nil {
-			t.Fatalf("%v: %v", f.name, err)
+				t.Fatalf("%v: %v", name, err)
 			}
 
-		if file.Name() != filepath.FromSlash(f.name) {
+			if file.Name() != filepath.FromSlash(nameBase) {
-			t.Errorf("Name(), got %v, expected %v", file.Name(), filepath.FromSlash(f.name))
+				t.Errorf("Name(), got %v, expected %v", file.Name(), filepath.FromSlash(nameBase))
 			}
 
 			s, err := file.Stat()
@@ -203,8 +231,9 @@ func TestFsOpen(t *testing.T) {
 			}
 		}
 	}
+}
 
-func TestRead(t *testing.T) {
+func TestGcsRead(t *testing.T) {
 	createFiles(t)
 	defer removeFiles(t)
 
@@ -213,25 +242,36 @@ func TestRead(t *testing.T) {
 			continue
 		}
 
-		file, err := gcsAfs.Open(f.name)
+		nameBase := filepath.Join(bucketName, f.name)
+
+		names := []string{
+			nameBase,
+			string(os.PathSeparator) + nameBase,
+		}
+		if f.name == "" {
+			names = []string{f.name}
+		}
+
+		for _, name := range names {
+			file, err := gcsAfs.Open(name)
 			if err != nil {
-			t.Fatalf("opening %v: %v", f.name, err)
+				t.Fatalf("opening %v: %v", name, err)
 			}
 
 			buf := make([]byte, 8)
 			n, err := file.Read(buf)
 			if err != nil {
 				if f.isdir && (err != syscall.EISDIR) {
-				t.Errorf("%v got error %v, expected EISDIR", f.name, err)
+					t.Errorf("%v got error %v, expected EISDIR", name, err)
 				} else if !f.isdir {
-				t.Errorf("%v: %v", f.name, err)
+					t.Errorf("%v: %v", name, err)
 				}
 			} else if n != 8 {
-			t.Errorf("%v: got %d read bytes, expected 8", f.name, n)
+				t.Errorf("%v: got %d read bytes, expected 8", name, n)
 			} else if string(buf) != strings.Repeat(f.content, testBytes) {
 				t.Errorf("%v: got <%s>, expected <%s>", f.name, f.content, string(buf))
 			}
-
+		}
 	}
 }
 
@@ -244,25 +284,36 @@ func TestGcsReadAt(t *testing.T) {
 			continue
 		}
 
-		file, err := gcsAfs.Open(f.name)
+		nameBase := filepath.Join(bucketName, f.name)
+
+		names := []string{
+			nameBase,
+			string(os.PathSeparator) + nameBase,
+		}
+		if f.name == "" {
+			names = []string{f.name}
+		}
+
+		for _, name := range names {
+			file, err := gcsAfs.Open(name)
 			if err != nil {
-			t.Fatalf("opening %v: %v", f.name, err)
+				t.Fatalf("opening %v: %v", name, err)
 			}
 
 			buf := make([]byte, testBytes)
 			n, err := file.ReadAt(buf, f.offset-testBytes/2)
 			if err != nil {
 				if f.isdir && (err != syscall.EISDIR) {
-				t.Errorf("%v got error %v, expected EISDIR", f.name, err)
+					t.Errorf("%v got error %v, expected EISDIR", name, err)
 				} else if !f.isdir {
-				t.Errorf("%v: %v", f.name, err)
+					t.Errorf("%v: %v", name, err)
 				}
 			} else if n != 8 {
 				t.Errorf("%v: got %d read bytes, expected 8", f.name, n)
 			} else if string(buf) != strings.Repeat(f.content, testBytes/2)+strings.Repeat(f.contentAtOffset, testBytes/2) {
 				t.Errorf("%v: got <%s>, expected <%s>", f.name, f.contentAtOffset, string(buf))
 			}
-
+		}
 	}
 }
 
@@ -275,9 +326,20 @@ func TestGcsSeek(t *testing.T) {
 			continue
 		}
 
-		file, err := gcsAfs.Open(f.name)
+		nameBase := filepath.Join(bucketName, f.name)
+
+		names := []string{
+			nameBase,
+			string(os.PathSeparator) + nameBase,
+		}
+		if f.name == "" {
+			names = []string{f.name}
+		}
+
+		for _, name := range names {
+			file, err := gcsAfs.Open(name)
 			if err != nil {
-			t.Fatalf("opening %v: %v", f.name, err)
+				t.Fatalf("opening %v: %v", name, err)
 			}
 
 			var tests = []struct {
@@ -300,18 +362,19 @@ func TestGcsSeek(t *testing.T) {
 						continue
 					}
 
-				t.Errorf("%v: %v", f.name, err)
+					t.Errorf("%v: %v", name, err)
 				}
 
 				if n != s.offOut {
 					t.Errorf("%v: (off: %v, whence: %v): got %v, expected %v", f.name, s.offIn, s.whence, n, s.offOut)
 				}
 			}
+		}
 
 	}
 }
 
-func TestName(t *testing.T) {
+func TestGcsName(t *testing.T) {
 	createFiles(t)
 	defer removeFiles(t)
 
@@ -320,20 +383,32 @@ func TestName(t *testing.T) {
 			continue
 		}
 
-		file, err := gcsAfs.Open(f.name)
+		nameBase := filepath.Join(bucketName, f.name)
+
+		names := []string{
+			nameBase,
+			string(os.PathSeparator) + nameBase,
+		}
+		if f.name == "" {
+			names = []string{f.name}
+		}
+
+		for _, name := range names {
+			file, err := gcsAfs.Open(name)
 			if err != nil {
-			t.Fatalf("opening %v: %v", f.name, err)
+				t.Fatalf("opening %v: %v", name, err)
 			}
 
 			n := file.Name()
-		if n != filepath.FromSlash(f.name) {
+			if n != filepath.FromSlash(nameBase) {
-			t.Errorf("got: %v, expected: %v", n, filepath.FromSlash(f.name))
+				t.Errorf("got: %v, expected: %v", n, filepath.FromSlash(nameBase))
+			}
 		}
 
 	}
 }
 
-func TestClose(t *testing.T) {
+func TestGcsClose(t *testing.T) {
 	createFiles(t)
 	defer removeFiles(t)
 
@@ -342,35 +417,47 @@ func TestClose(t *testing.T) {
 			continue
 		}
 
-		file, err := gcsAfs.Open(f.name)
+		nameBase := filepath.Join(bucketName, f.name)
+
+		names := []string{
+			nameBase,
+			string(os.PathSeparator) + nameBase,
+		}
+		if f.name == "" {
+			names = []string{f.name}
+		}
+
+		for _, name := range names {
+			file, err := gcsAfs.Open(name)
 			if err != nil {
-			t.Fatalf("opening %v: %v", f.name, err)
+				t.Fatalf("opening %v: %v", name, err)
 			}
 
 			err = file.Close()
 			if err != nil {
-			t.Errorf("%v: %v", f.name, err)
+				t.Errorf("%v: %v", name, err)
 			}
 
 			err = file.Close()
 			if err == nil {
-			t.Errorf("%v: closing twice should return an error", f.name)
+				t.Errorf("%v: closing twice should return an error", name)
 			}
 
 			buf := make([]byte, 8)
 			n, err := file.Read(buf)
 			if n != 0 || err == nil {
-			t.Errorf("%v: could read from a closed file", f.name)
+				t.Errorf("%v: could read from a closed file", name)
 			}
 
 			n, err = file.ReadAt(buf, 256)
 			if n != 0 || err == nil {
-			t.Errorf("%v: could readAt from a closed file", f.name)
+				t.Errorf("%v: could readAt from a closed file", name)
 			}
 
 			off, err := file.Seek(0, io.SeekStart)
 			if off != 0 || err == nil {
-			t.Errorf("%v: could seek from a closed file", f.name)
+				t.Errorf("%v: could seek from a closed file", name)
+			}
 		}
 	}
 }
@@ -380,56 +467,81 @@ func TestGcsOpenFile(t *testing.T) {
 	defer removeFiles(t)
 
 	for _, f := range files {
-		file, err := gcsAfs.OpenFile(f.name, os.O_RDONLY, 0400)
+		nameBase := filepath.Join(bucketName, f.name)
+
+		names := []string{
+			nameBase,
+			string(os.PathSeparator) + nameBase,
+		}
+		if f.name == "" {
+			names = []string{f.name}
+		}
+
+		for _, name := range names {
+			file, err := gcsAfs.OpenFile(name, os.O_RDONLY, 0400)
 			if !f.exists {
-			if !errors.Is(err, syscall.ENOENT) {
+				if (f.name != "" && !errors.Is(err, syscall.ENOENT)) ||
-				t.Errorf("%v: got %v, expected%v", f.name, err, syscall.ENOENT)
+					(f.name == "" && !errors.Is(err, gcsfs.ErrNoBucketInName)) {
+					t.Errorf("%v: got %v, expected%v", name, err, syscall.ENOENT)
 				}
 
 				continue
 			}
 
 			if err != nil {
-			t.Fatalf("%v: %v", f.name, err)
+				t.Fatalf("%v: %v", name, err)
 			}
 
 			err = file.Close()
 			if err != nil {
-			t.Fatalf("failed to close a file \"%s\": %s", f.name, err)
+				t.Fatalf("failed to close a file \"%s\": %s", name, err)
 			}
 
-		file, err = gcsAfs.OpenFile(f.name, os.O_CREATE, 0600)
+			file, err = gcsAfs.OpenFile(name, os.O_CREATE, 0600)
 			if !errors.Is(err, syscall.EPERM) {
-			t.Errorf("%v: open for write: got %v, expected %v", f.name, err, syscall.EPERM)
+				t.Errorf("%v: open for write: got %v, expected %v", name, err, syscall.EPERM)
+			}
 		}
-
 	}
 }
 
-func TestFsStat(t *testing.T) {
+func TestGcsFsStat(t *testing.T) {
 	createFiles(t)
 	defer removeFiles(t)
 
 	for _, f := range files {
-		fi, err := gcsAfs.Stat(f.name)
+		nameBase := filepath.Join(bucketName, f.name)
+
+		names := []string{
+			nameBase,
+			string(os.PathSeparator) + nameBase,
+		}
+		if f.name == "" {
+			names = []string{f.name}
+		}
+
+		for _, name := range names {
+			fi, err := gcsAfs.Stat(name)
 			if !f.exists {
-			if !errors.Is(err, syscall.ENOENT) {
+				if (f.name != "" && !errors.Is(err, syscall.ENOENT)) ||
-				t.Errorf("%v: got %v, expected%v", f.name, err, syscall.ENOENT)
+					(f.name == "" && !errors.Is(err, gcsfs.ErrNoBucketInName)) {
+					t.Errorf("%v: got %v, expected%v", name, err, syscall.ENOENT)
 				}
 
 				continue
 			}
 
 			if err != nil {
-			t.Fatalf("stat %v: got error '%v'", f.name, err)
+				t.Fatalf("stat %v: got error '%v'", name, err)
 			}
 
 			if isdir := fi.IsDir(); isdir != f.isdir {
-			t.Errorf("%v directory, got: %v, expected: %v", f.name, isdir, f.isdir)
+				t.Errorf("%v directory, got: %v, expected: %v", name, isdir, f.isdir)
 			}
 
 			if size := fi.Size(); size != f.size {
-			t.Errorf("%v size, got: %v, expected: %v", f.name, size, f.size)
+				t.Errorf("%v size, got: %v, expected: %v", name, size, f.size)
+			}
 		}
 	}
 }
@@ -439,7 +551,15 @@ func TestGcsReaddir(t *testing.T) {
 	defer removeFiles(t)
 
 	for _, d := range dirs {
-		dir, err := gcsAfs.Open(d.name)
+		nameBase := filepath.Join(bucketName, d.name)
+
+		names := []string{
+			nameBase,
+			string(os.PathSeparator) + nameBase,
+		}
+
+		for _, name := range names {
+			dir, err := gcsAfs.Open(name)
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -448,13 +568,13 @@ func TestGcsReaddir(t *testing.T) {
 			if err != nil {
 				t.Fatal(err)
 			}
-		var names []string
+			var fileNames []string
 			for _, f := range fi {
-			names = append(names, f.Name())
+				fileNames = append(fileNames, f.Name())
 			}
 
-		if !reflect.DeepEqual(names, d.children) {
+			if !reflect.DeepEqual(fileNames, d.children) {
-			t.Errorf("%v: children, got '%v', expected '%v'", d.name, names, d.children)
+				t.Errorf("%v: children, got '%v', expected '%v'", name, fileNames, d.children)
 			}
 
 			fi, err = dir.Readdir(1)
@@ -462,17 +582,26 @@ func TestGcsReaddir(t *testing.T) {
 				t.Fatal(err)
 			}
 
-		names = []string{}
+			fileNames = []string{}
 			for _, f := range fi {
-			names = append(names, f.Name())
+				fileNames = append(fileNames, f.Name())
 			}
 
-		if !reflect.DeepEqual(names, d.children[0:1]) {
+			if !reflect.DeepEqual(fileNames, d.children[0:1]) {
-			t.Errorf("%v: children, got '%v', expected '%v'", d.name, names, d.children[0:1])
+				t.Errorf("%v: children, got '%v', expected '%v'", name, fileNames, d.children[0:1])
+			}
 		}
 	}
 
-	dir, err := gcsAfs.Open("testFile")
+	nameBase := filepath.Join(bucketName, "testFile")
+
+	names := []string{
+		nameBase,
+		string(os.PathSeparator) + nameBase,
+	}
+
+	for _, name := range names {
+		dir, err := gcsAfs.Open(name)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -482,46 +611,65 @@ func TestGcsReaddir(t *testing.T) {
 			t.Fatal("Expected error")
 		}
 	}
+}
 
 func TestGcsReaddirnames(t *testing.T) {
 	createFiles(t)
 	defer removeFiles(t)
 
 	for _, d := range dirs {
-		dir, err := gcsAfs.Open(d.name)
+		nameBase := filepath.Join(bucketName, d.name)
+
+		names := []string{
+			nameBase,
+			string(os.PathSeparator) + nameBase,
+		}
+
+		for _, name := range names {
+			dir, err := gcsAfs.Open(name)
 			if err != nil {
 				t.Fatal(err)
 			}
 
-		names, err := dir.Readdirnames(0)
+			fileNames, err := dir.Readdirnames(0)
 			if err != nil {
 				t.Fatal(err)
 			}
 
-		if !reflect.DeepEqual(names, d.children) {
+			if !reflect.DeepEqual(fileNames, d.children) {
-			t.Errorf("%v: children, got '%v', expected '%v'", d.name, names, d.children)
+				t.Errorf("%v: children, got '%v', expected '%v'", name, fileNames, d.children)
 			}
 
-		names, err = dir.Readdirnames(1)
+			fileNames, err = dir.Readdirnames(1)
 			if err != nil {
 				t.Fatal(err)
 			}
 
-		if !reflect.DeepEqual(names, d.children[0:1]) {
+			if !reflect.DeepEqual(fileNames, d.children[0:1]) {
-			t.Errorf("%v: children, got '%v', expected '%v'", d.name, names, d.children[0:1])
+				t.Errorf("%v: children, got '%v', expected '%v'", name, fileNames, d.children[0:1])
+			}
 		}
 	}
 
-	dir, err := gcsAfs.Open("testFile")
+	nameBase := filepath.Join(bucketName, "testFile")
+
+	names := []string{
+		nameBase,
+		string(os.PathSeparator) + nameBase,
+	}
+
+	for _, name := range names {
+		dir, err := gcsAfs.Open(name)
 		if err != nil {
 			t.Fatal(err)
 		}
 
-	_, err = dir.Readdir(-1)
+		_, err = dir.Readdirnames(-1)
 		if err != syscall.ENOTDIR {
 			t.Fatal("Expected error")
 		}
 	}
+}
 
 func TestGcsGlob(t *testing.T) {
 	createFiles(t)
@@ -531,29 +679,40 @@ func TestGcsGlob(t *testing.T) {
 		glob    string
 		entries []string
 	}{
-		{filepath.FromSlash("/*"), []string{filepath.FromSlash("/sub"), filepath.FromSlash("/testDir1"), filepath.FromSlash("/testFile")}},
 		{filepath.FromSlash("*"), []string{filepath.FromSlash("sub"), filepath.FromSlash("testDir1"), filepath.FromSlash("testFile")}},
-		{filepath.FromSlash("/sub/*"), []string{filepath.FromSlash("/sub/testDir2")}},
-		{filepath.FromSlash("/sub/testDir2/*"), []string{filepath.FromSlash("/sub/testDir2/testFile")}},
-		{filepath.FromSlash("/testDir1/*"), []string{filepath.FromSlash("/testDir1/testFile")}},
 		{filepath.FromSlash("sub/*"), []string{filepath.FromSlash("sub/testDir2")}},
 		{filepath.FromSlash("sub/testDir2/*"), []string{filepath.FromSlash("sub/testDir2/testFile")}},
 		{filepath.FromSlash("testDir1/*"), []string{filepath.FromSlash("testDir1/testFile")}},
 	} {
-		entries, err := Glob(gcsAfs.Fs, s.glob)
+		nameBase := filepath.Join(bucketName, s.glob)
+
+		prefixedGlobs := []string{
+			nameBase,
+			string(os.PathSeparator) + nameBase,
+		}
+
+		prefixedEntries := [][]string{{}, {}}
+		for _, entry := range s.entries {
+			prefixedEntries[0] = append(prefixedEntries[0], filepath.Join(bucketName, entry))
+			prefixedEntries[1] = append(prefixedEntries[1], string(os.PathSeparator)+filepath.Join(bucketName, entry))
+		}
+
+		for i, prefixedGlob := range prefixedGlobs {
+			entries, err := Glob(gcsAfs.Fs, prefixedGlob)
 			if err != nil {
 				t.Error(err)
 			}
-		if reflect.DeepEqual(entries, s.entries) {
+			if reflect.DeepEqual(entries, prefixedEntries[i]) {
-			t.Logf("glob: %s: glob ok", s.glob)
+				t.Logf("glob: %s: glob ok", prefixedGlob)
 			} else {
-			t.Errorf("glob: %s: got %#v, expected %#v", s.glob, entries, s.entries)
+				t.Errorf("glob: %s: got %#v, expected %#v", prefixedGlob, entries, prefixedEntries)
+			}
 		}
 	}
 }
 
-func TestMkdir(t *testing.T) {
+func TestGcsMkdir(t *testing.T) {
-	dirName := "/a-test-dir"
+	dirName := filepath.Join(bucketName, "a-test-dir")
 	var err error
 
 	err = gcsAfs.Mkdir(dirName, 0755)
@@ -582,45 +741,47 @@ func TestMkdir(t *testing.T) {
 	}
 }
 
-func TestMkdirAll(t *testing.T) {
+func TestGcsMkdirAll(t *testing.T) {
-	err := gcsAfs.MkdirAll("/a/b/c", 0755)
+	dirName := filepath.Join(bucketName, "a/b/c")
+
+	err := gcsAfs.MkdirAll(dirName, 0755)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	info, err := gcsAfs.Stat("/a")
+	info, err := gcsAfs.Stat(filepath.Join(bucketName, "a"))
 	if err != nil {
 		t.Fatal(err)
 	}
 	if !info.Mode().IsDir() {
-		t.Error("/a: mode is not directory")
+		t.Errorf("%s: mode is not directory", filepath.Join(bucketName, "a"))
 	}
 	if info.Mode() != os.ModeDir|0755 {
-		t.Errorf("/a: wrong permissions, expected drwxr-xr-x, got %s", info.Mode())
+		t.Errorf("%s: wrong permissions, expected drwxr-xr-x, got %s", filepath.Join(bucketName, "a"), info.Mode())
 	}
-	info, err = gcsAfs.Stat("/a/b")
+	info, err = gcsAfs.Stat(filepath.Join(bucketName, "a/b"))
 	if err != nil {
 		t.Fatal(err)
 	}
 	if !info.Mode().IsDir() {
-		t.Error("/a/b: mode is not directory")
+		t.Errorf("%s: mode is not directory", filepath.Join(bucketName, "a/b"))
 	}
 	if info.Mode() != os.ModeDir|0755 {
-		t.Errorf("/a/b: wrong permissions, expected drwxr-xr-x, got %s", info.Mode())
+		t.Errorf("%s: wrong permissions, expected drwxr-xr-x, got %s", filepath.Join(bucketName, "a/b"), info.Mode())
 	}
-	info, err = gcsAfs.Stat("/a/b/c")
+	info, err = gcsAfs.Stat(dirName)
 	if err != nil {
 		t.Fatal(err)
 	}
 	if !info.Mode().IsDir() {
-		t.Error("/a/b/c: mode is not directory")
+		t.Errorf("%s: mode is not directory", dirName)
 	}
 	if info.Mode() != os.ModeDir|0755 {
-		t.Errorf("/a/b/c: wrong permissions, expected drwxr-xr-x, got %s", info.Mode())
+		t.Errorf("%s: wrong permissions, expected drwxr-xr-x, got %s", dirName, info.Mode())
 	}
 
-	err = gcsAfs.RemoveAll("/a")
+	err = gcsAfs.RemoveAll(filepath.Join(bucketName, "a"))
 	if err != nil {
-		t.Fatalf("failed to remove the folder /a with error: %s", err)
+		t.Fatalf("failed to remove the folder %s with error: %s", filepath.Join(bucketName, "a"), err)
 	}
 }

gcsfs/errors.go

@@ -22,6 +22,7 @@ import (
 )
 
 var (
+	ErrNoBucketInName     = errors.New("no bucket name found in the name")
 	ErrFileClosed         = errors.New("file is closed")
 	ErrOutOfRange         = errors.New("out of range")
 	ErrObjectDoesNotExist = errors.New("storage: object doesn't exist")

gcsfs/file.go

@@ -195,11 +195,13 @@ func (o *GcsFile) readdirImpl(count int) ([]*FileInfo, error) {
 		return nil, syscall.ENOTDIR
 	}
 
-	path := o.resource.fs.ensureTrailingSeparator(o.Name())
+	path := o.resource.fs.ensureTrailingSeparator(o.resource.name)
 	if o.ReadDirIt == nil {
 		//log.Printf("Querying path : %s\n", path)
-		o.ReadDirIt = o.resource.fs.bucket.Objects(
+		bucketName, bucketPath := o.resource.fs.splitName(path)
-			o.resource.ctx, &storage.Query{Delimiter: o.resource.fs.separator, Prefix: path, Versions: false})
+
+		o.ReadDirIt = o.resource.fs.client.Bucket(bucketName).Objects(
+			o.resource.ctx, &storage.Query{Delimiter: o.resource.fs.separator, Prefix: bucketPath, Versions: false})
 	}
 	var res []*FileInfo
 	for {
@@ -283,7 +285,7 @@ func (o *GcsFile) Stat() (os.FileInfo, error) {
 		return nil, err
 	}
 
-	return newFileInfo(o.Name(), o.resource.fs, o.resource.fileMode)
+	return newFileInfo(o.resource.name, o.resource.fs, o.resource.fileMode)
 }
 
 func (o *GcsFile) Sync() error {

gcsfs/file_info.go

@@ -46,7 +46,10 @@ func newFileInfo(name string, fs *GcsFs, fileMode os.FileMode) (*FileInfo, error
 		fileMode: fileMode,
 	}
 
-	obj := fs.getObj(name)
+	obj, err := fs.getObj(name)
+	if err != nil {
+		return nil, err
+	}
 
 	objAttrs, err := obj.Attrs(fs.ctx)
 	if err != nil {
@@ -58,8 +61,9 @@ func newFileInfo(name string, fs *GcsFs, fileMode os.FileMode) (*FileInfo, error
 		} else if err.Error() == ErrObjectDoesNotExist.Error() {
 			// Folders do not actually "exist" in GCloud, so we have to check, if something exists with
 			// such a prefix
-			it := fs.bucket.Objects(
+			bucketName, bucketPath := fs.splitName(name)
-				fs.ctx, &storage.Query{Delimiter: fs.separator, Prefix: name, Versions: false})
+			it := fs.client.Bucket(bucketName).Objects(
+				fs.ctx, &storage.Query{Delimiter: fs.separator, Prefix: bucketPath, Versions: false})
 			if _, err = it.Next(); err == nil {
 				res.name = fs.ensureTrailingSeparator(res.name)
 				res.isDir = true
@@ -100,7 +104,7 @@ func newFileInfoFromAttrs(objAttrs *storage.ObjectAttrs, fileMode os.FileMode) *
 }
 
 func (fi *FileInfo) Name() string {
-	return filepath.Base(fi.name)
+	return filepath.Base(filepath.FromSlash(fi.name))
 }
 
 func (fi *FileInfo) Size() int64 {

gcsfs/fs.go

@@ -19,7 +19,6 @@ package gcsfs
 import (
 	"context"
 	"errors"
-	"log"
 	"os"
 	"path/filepath"
 	"strings"
@@ -31,26 +30,29 @@ import (
 
 const (
 	defaultFileMode = 0755
+	gsPrefix        = "gs://"
 )
 
 // GcsFs is a Fs implementation that uses functions provided by google cloud storage
 type GcsFs struct {
 	ctx       context.Context
-	bucket        stiface.BucketHandle
+	client    stiface.Client
 	separator string
+
+	buckets       map[string]stiface.BucketHandle
 	rawGcsObjects map[string]*GcsFile
 
 	autoRemoveEmptyFolders bool //trigger for creating "virtual folders" (not required by GCSs)
 }
 
-func NewGcsFs(ctx context.Context, bucket stiface.BucketHandle) *GcsFs {
+func NewGcsFs(ctx context.Context, client stiface.Client) *GcsFs {
-	return NewGcsFsWithSeparator(ctx, bucket, "/")
+	return NewGcsFsWithSeparator(ctx, client, "/")
 }
 
-func NewGcsFsWithSeparator(ctx context.Context, bucket stiface.BucketHandle, folderSep string) *GcsFs {
+func NewGcsFsWithSeparator(ctx context.Context, client stiface.Client, folderSep string) *GcsFs {
 	return &GcsFs{
 		ctx:           ctx,
-		bucket:        bucket,
+		client:        client,
 		separator:     folderSep,
 		rawGcsObjects: make(map[string]*GcsFile),
 
@@ -69,39 +71,65 @@ func (fs *GcsFs) ensureTrailingSeparator(s string) string {
 	}
 	return s
 }
-
+func (fs *GcsFs) ensureNoLeadingSeparator(s string) string {
-func (fs *GcsFs) ensureNoLeadingSeparators(s string) string {
-	// GCS does REALLY not like the names, that begin with a separator
 	if len(s) > 0 && strings.HasPrefix(s, fs.separator) {
-		log.Printf(
+		s = s[len(fs.separator):]
-			"WARNING: the provided path \"%s\" starts with a separator \"%s\", which is not supported by "+
+	}
-				"GCloud. The separator will be automatically trimmed",
+
-			s,
+	return s
-			fs.separator,
+}
-		)
+
-		return s[len(fs.separator):]
+func ensureNoPrefix(s string) string {
+	if len(s) > 0 && strings.HasPrefix(s, gsPrefix) {
+		return s[len(gsPrefix):]
 	}
 	return s
 }
 
-func correctTheDot(s string) string {
+func validateName(s string) error {
-	// So, Afero's Glob likes to give "." as a name - that to list the "empty" dir name.
+	if len(s) == 0 {
-	// GCS _really_ dislikes the dot and gives no entries for it - so we should rather replace the dot
+		return ErrNoBucketInName
-	// with an empty string
-	if s == "." {
-		return ""
 	}
-	return s
+	return nil
 }
 
-func (fs *GcsFs) getObj(name string) stiface.ObjectHandle {
+// Splits provided name into bucket name and path
-	return fs.bucket.Object(name)
+func (fs *GcsFs) splitName(name string) (bucketName string, path string) {
+	splitName := strings.Split(name, fs.separator)
+
+	return splitName[0], strings.Join(splitName[1:], fs.separator)
+}
+
+func (fs *GcsFs) getBucket(name string) (stiface.BucketHandle, error) {
+	bucket := fs.buckets[name]
+	if bucket == nil {
+		bucket = fs.client.Bucket(name)
+		_, err := bucket.Attrs(fs.ctx)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return bucket, nil
+}
+
+func (fs *GcsFs) getObj(name string) (stiface.ObjectHandle, error) {
+	bucketName, path := fs.splitName(name)
+
+	bucket, err := fs.getBucket(bucketName)
+	if err != nil {
+		return nil, err
+	}
+
+	return bucket.Object(path), nil
 }
 
 func (fs *GcsFs) Name() string { return "GcsFs" }
 
 func (fs *GcsFs) Create(name string) (*GcsFile, error) {
-	name = fs.ensureNoLeadingSeparators(fs.normSeparators(correctTheDot(name)))
+	name = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(name)))
+	if err := validateName(name); err != nil {
+		return nil, err
+	}
 
 	if !fs.autoRemoveEmptyFolders {
 		baseDir := filepath.Base(name)
@@ -113,9 +141,11 @@ func (fs *GcsFs) Create(name string) (*GcsFile, error) {
 		}
 	}
 
-	obj := fs.getObj(name)
+	obj, err := fs.getObj(name)
+	if err != nil {
+		return nil, err
+	}
 	w := obj.NewWriter(fs.ctx)
-	var err error
 	err = w.Close()
 	if err != nil {
 		return nil, err
@@ -127,15 +157,24 @@ func (fs *GcsFs) Create(name string) (*GcsFile, error) {
 }
 
 func (fs *GcsFs) Mkdir(name string, _ os.FileMode) error {
-	name = fs.ensureTrailingSeparator(fs.ensureNoLeadingSeparators(fs.normSeparators(correctTheDot(name))))
+	name = fs.ensureNoLeadingSeparator(fs.ensureTrailingSeparator(fs.normSeparators(ensureNoPrefix(name))))
+	if err := validateName(name); err != nil {
+		return err
+	}
 
-	obj := fs.getObj(name)
+	obj, err := fs.getObj(name)
+	if err != nil {
+		return err
+	}
 	w := obj.NewWriter(fs.ctx)
 	return w.Close()
 }
 
 func (fs *GcsFs) MkdirAll(path string, perm os.FileMode) error {
-	path = fs.ensureTrailingSeparator(fs.ensureNoLeadingSeparators(fs.normSeparators(correctTheDot(path))))
+	path = fs.ensureNoLeadingSeparator(fs.ensureTrailingSeparator(fs.normSeparators(ensureNoPrefix(path))))
+	if err := validateName(path); err != nil {
+		return err
+	}
 
 	root := ""
 	folders := strings.Split(path, fs.separator)
@@ -165,13 +204,21 @@ func (fs *GcsFs) OpenFile(name string, flag int, fileMode os.FileMode) (*GcsFile
 	var file *GcsFile
 	var err error
 
-	name = fs.ensureNoLeadingSeparators(fs.normSeparators(correctTheDot(name)))
+	name = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(name)))
+	if err = validateName(name); err != nil {
+		return nil, err
+	}
 
-	obj, found := fs.rawGcsObjects[name]
+	f, found := fs.rawGcsObjects[name]
 	if found {
-		file = NewGcsFileFromOldFH(flag, fileMode, obj.resource)
+		file = NewGcsFileFromOldFH(flag, fileMode, f.resource)
 	} else {
-		file = NewGcsFile(fs.ctx, fs, fs.getObj(name), flag, fileMode, name)
+		var obj stiface.ObjectHandle
+		obj, err = fs.getObj(name)
+		if err != nil {
+			return nil, err
+		}
+		file = NewGcsFile(fs.ctx, fs, obj, flag, fileMode, name)
 	}
 
 	if flag == os.O_RDONLY {
@@ -211,9 +258,15 @@ func (fs *GcsFs) OpenFile(name string, flag int, fileMode os.FileMode) (*GcsFile
 }
 
 func (fs *GcsFs) Remove(name string) error {
-	name = fs.ensureNoLeadingSeparators(fs.normSeparators(correctTheDot(name)))
+	name = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(name)))
+	if err := validateName(name); err != nil {
+		return err
+	}
 
-	obj := fs.getObj(name)
+	obj, err := fs.getObj(name)
+	if err != nil {
+		return err
+	}
 	info, err := fs.Stat(name)
 	if err != nil {
 		return err
@@ -235,7 +288,10 @@ func (fs *GcsFs) Remove(name string) error {
 
 		// it's an empty folder, we can continue
 		name = fs.ensureTrailingSeparator(name)
-		obj = fs.getObj(name)
+		obj, err = fs.getObj(name)
+		if err != nil {
+			return err
+		}
 
 		return obj.Delete(fs.ctx)
 	}
@@ -243,7 +299,10 @@ func (fs *GcsFs) Remove(name string) error {
 }
 
 func (fs *GcsFs) RemoveAll(path string) error {
-	path = fs.ensureNoLeadingSeparators(fs.normSeparators(correctTheDot(path)))
+	path = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(path)))
+	if err := validateName(path); err != nil {
+		return err
+	}
 
 	pathInfo, err := fs.Stat(path)
 	if err != nil {
@@ -262,63 +321,37 @@ func (fs *GcsFs) RemoveAll(path string) error {
 	var infos []os.FileInfo
 	infos, err = dir.Readdir(0)
 	for _, info := range infos {
-		err = fs.RemoveAll(path + fs.separator + info.Name())
+		nameToRemove := fs.normSeparators(info.Name())
+		err = fs.RemoveAll(path + fs.separator + nameToRemove)
 		if err != nil {
 			return err
 		}
 	}
 
 	return fs.Remove(path)
-
-	//it := fs.bucket.Objects(fs.ctx, &storage.Query{Delimiter: fs.separator, Prefix: path, Versions: false})
-	//for {
-	//	objAttrs, err := it.Next()
-	//	if err == iterator.Done {
-	//		break
-	//	}
-	//	if err != nil {
-	//		return err
-	//	}
-	//
-	//	name := objAttrs.Name
-	//	if name == "" {
-	//		name = objAttrs.Prefix
-	//	}
-	//
-	//	if name == path {
-	//		// somehow happens
-	//		continue
-	//	}
-	//	if objAttrs.Name == "" && objAttrs.Prefix != "" {
-	//		// it's a folder, let's try to remove it normally first
-	//		err = fs.Remove(path + fs.separator + objAttrs.Name)
-	//		if err != nil {
-	//			if err == syscall.ENOTEMPTY {
-	//				err = fs.RemoveAll(path + fs.separator + objAttrs.Name)
-	//			}
-	//		}
-	//		if err != nil {
-	//			return err
-	//		}
-	//
-	//	} else {
-	//		err = fs.Remove(objAttrs.Name)
-	//		if err != nil {
-	//			return err
-	//		}
-	//	}
-	//}
-	//return nil
 }
 
 func (fs *GcsFs) Rename(oldName, newName string) error {
-	oldName = fs.ensureNoLeadingSeparators(fs.normSeparators(correctTheDot(oldName)))
+	oldName = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(oldName)))
-	newName = fs.ensureNoLeadingSeparators(fs.normSeparators(correctTheDot(newName)))
+	if err := validateName(oldName); err != nil {
+		return err
+	}
 
-	src := fs.bucket.Object(oldName)
+	newName = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(newName)))
-	dst := fs.bucket.Object(newName)
+	if err := validateName(newName); err != nil {
+		return err
+	}
 
-	if _, err := dst.CopierFrom(src).Run(fs.ctx); err != nil {
+	src, err := fs.getObj(oldName)
+	if err != nil {
+		return err
+	}
+	dst, err := fs.getObj(newName)
+	if err != nil {
+		return err
+	}
+
+	if _, err = dst.CopierFrom(src).Run(fs.ctx); err != nil {
 		return err
 	}
 	delete(fs.rawGcsObjects, oldName)
@@ -326,7 +359,10 @@ func (fs *GcsFs) Rename(oldName, newName string) error {
 }
 
 func (fs *GcsFs) Stat(name string) (os.FileInfo, error) {
-	name = fs.ensureNoLeadingSeparators(fs.normSeparators(correctTheDot(name)))
+	name = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(name)))
+	if err := validateName(name); err != nil {
+		return nil, err
+	}
 
 	return newFileInfo(name, fs, defaultFileMode)
 }

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/pkg/sftp v1.10.1
 	github.com/stretchr/testify v1.5.1
 	golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9
+	golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99 // indirect
 	golang.org/x/text v0.3.4
 	google.golang.org/api v0.40.0
 )