re
/
afero
forked from mirror/afero
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

GCS fs: move all gcsfs related implementations to its own package 

this way we don't force any application that import afero to include
gcfs deps in its binary
This commit is contained in:
Nicola Murino 2021-12-27 18:55:57 +01:00
parent d70f944720
commit 165e3dc3a5
8 changed files with 65 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
gcsfs/file.go
View File

@ -44,7 +44,7 @@ type GcsFile struct {
func NewGcsFile(
ctx context.Context,
fs *GcsFs,
fs *Fs,
obj stiface.ObjectHandle,
openFlags int,
// Unused: there is no use to the file mode in GCloud just yet - but we keep it here, just in case we need it

2
gcsfs/file_info.go
View File

@ -37,7 +37,7 @@ type FileInfo struct {
fileMode os.FileMode
}
func newFileInfo(name string, fs *GcsFs, fileMode os.FileMode) (*FileInfo, error) {
func newFileInfo(name string, fs *Fs, fileMode os.FileMode) (*FileInfo, error) {
res := &FileInfo{
name: name,
size: folderSize,

2
gcsfs/file_resource.go
View File

@ -40,7 +40,7 @@ const (
type gcsFileResource struct {
ctx context.Context
fs *GcsFs
fs *Fs
obj stiface.ObjectHandle
name string

48
gcsfs/fs.go
View File

@ -33,8 +33,8 @@ const (
gsPrefix = "gs://"
)
// GcsFs is a Fs implementation that uses functions provided by google cloud storage
type GcsFs struct {
// Fs is a Fs implementation that uses functions provided by google cloud storage
type Fs struct {
ctx context.Context
client stiface.Client
separator string
@ -45,12 +45,12 @@ type GcsFs struct {
autoRemoveEmptyFolders bool //trigger for creating "virtual folders" (not required by GCSs)
}
func NewGcsFs(ctx context.Context, client stiface.Client) *GcsFs {
func NewGcsFs(ctx context.Context, client stiface.Client) *Fs {
return NewGcsFsWithSeparator(ctx, client, "/")
}
func NewGcsFsWithSeparator(ctx context.Context, client stiface.Client, folderSep string) *GcsFs {
return &GcsFs{
func NewGcsFsWithSeparator(ctx context.Context, client stiface.Client, folderSep string) *Fs {
return &Fs{
ctx: ctx,
client: client,
separator: folderSep,
@ -61,17 +61,17 @@ func NewGcsFsWithSeparator(ctx context.Context, client stiface.Client, folderSep
}
// normSeparators will normalize all "\\" and "/" to the provided separator
func (fs *GcsFs) normSeparators(s string) string {
func (fs *Fs) normSeparators(s string) string {
return strings.Replace(strings.Replace(s, "\\", fs.separator, -1), "/", fs.separator, -1)
}
func (fs *GcsFs) ensureTrailingSeparator(s string) string {
func (fs *Fs) ensureTrailingSeparator(s string) string {
if len(s) > 0 && !strings.HasSuffix(s, fs.separator) {
return s + fs.separator
}
return s
}
func (fs *GcsFs) ensureNoLeadingSeparator(s string) string {
func (fs *Fs) ensureNoLeadingSeparator(s string) string {
if len(s) > 0 && strings.HasPrefix(s, fs.separator) {
s = s[len(fs.separator):]
}
@ -94,13 +94,13 @@ func validateName(s string) error {
}
// Splits provided name into bucket name and path
func (fs *GcsFs) splitName(name string) (bucketName string, path string) {
func (fs *Fs) splitName(name string) (bucketName string, path string) {
splitName := strings.Split(name, fs.separator)
return splitName[0], strings.Join(splitName[1:], fs.separator)
}
func (fs *GcsFs) getBucket(name string) (stiface.BucketHandle, error) {
func (fs *Fs) getBucket(name string) (stiface.BucketHandle, error) {
bucket := fs.buckets[name]
if bucket == nil {
bucket = fs.client.Bucket(name)
@ -112,7 +112,7 @@ func (fs *GcsFs) getBucket(name string) (stiface.BucketHandle, error) {
return bucket, nil
}
func (fs *GcsFs) getObj(name string) (stiface.ObjectHandle, error) {
func (fs *Fs) getObj(name string) (stiface.ObjectHandle, error) {
bucketName, path := fs.splitName(name)
bucket, err := fs.getBucket(bucketName)
@ -123,9 +123,9 @@ func (fs *GcsFs) getObj(name string) (stiface.ObjectHandle, error) {
return bucket.Object(path), nil
}
func (fs *GcsFs) Name() string { return "GcsFs" }
func (fs *Fs) Name() string { return "GcsFs" }
func (fs *GcsFs) Create(name string) (*GcsFile, error) {
func (fs *Fs) Create(name string) (*GcsFile, error) {
name = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(name)))
if err := validateName(name); err != nil {
return nil, err
@ -156,7 +156,7 @@ func (fs *GcsFs) Create(name string) (*GcsFile, error) {
return file, nil
}
func (fs *GcsFs) Mkdir(name string, _ os.FileMode) error {
func (fs *Fs) Mkdir(name string, _ os.FileMode) error {
name = fs.ensureNoLeadingSeparator(fs.ensureTrailingSeparator(fs.normSeparators(ensureNoPrefix(name))))
if err := validateName(name); err != nil {
return err
@ -179,7 +179,7 @@ func (fs *GcsFs) Mkdir(name string, _ os.FileMode) error {
return w.Close()
}
func (fs *GcsFs) MkdirAll(path string, perm os.FileMode) error {
func (fs *Fs) MkdirAll(path string, perm os.FileMode) error {
path = fs.ensureNoLeadingSeparator(fs.ensureTrailingSeparator(fs.normSeparators(ensureNoPrefix(path))))
if err := validateName(path); err != nil {
return err
@ -216,11 +216,11 @@ func (fs *GcsFs) MkdirAll(path string, perm os.FileMode) error {
return nil
}
func (fs *GcsFs) Open(name string) (*GcsFile, error) {
func (fs *Fs) Open(name string) (*GcsFile, error) {
return fs.OpenFile(name, os.O_RDONLY, 0)
}
func (fs *GcsFs) OpenFile(name string, flag int, fileMode os.FileMode) (*GcsFile, error) {
func (fs *Fs) OpenFile(name string, flag int, fileMode os.FileMode) (*GcsFile, error) {
var file *GcsFile
var err error
@ -277,7 +277,7 @@ func (fs *GcsFs) OpenFile(name string, flag int, fileMode os.FileMode) (*GcsFile
return file, nil
}
func (fs *GcsFs) Remove(name string) error {
func (fs *Fs) Remove(name string) error {
name = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(name)))
if err := validateName(name); err != nil {
return err
@ -318,7 +318,7 @@ func (fs *GcsFs) Remove(name string) error {
return obj.Delete(fs.ctx)
}
func (fs *GcsFs) RemoveAll(path string) error {
func (fs *Fs) RemoveAll(path string) error {
path = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(path)))
if err := validateName(path); err != nil {
return err
@ -351,7 +351,7 @@ func (fs *GcsFs) RemoveAll(path string) error {
return fs.Remove(path)
}
func (fs *GcsFs) Rename(oldName, newName string) error {
func (fs *Fs) Rename(oldName, newName string) error {
oldName = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(oldName)))
if err := validateName(oldName); err != nil {
return err
@ -378,7 +378,7 @@ func (fs *GcsFs) Rename(oldName, newName string) error {
return src.Delete(fs.ctx)
}
func (fs *GcsFs) Stat(name string) (os.FileInfo, error) {
func (fs *Fs) Stat(name string) (os.FileInfo, error) {
name = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(name)))
if err := validateName(name); err != nil {
return nil, err
@ -387,14 +387,14 @@ func (fs *GcsFs) Stat(name string) (os.FileInfo, error) {
return newFileInfo(name, fs, defaultFileMode)
}
func (fs *GcsFs) Chmod(_ string, _ os.FileMode) error {
func (fs *Fs) Chmod(_ string, _ os.FileMode) error {
return errors.New("method Chmod is not implemented in GCS")
}
func (fs *GcsFs) Chtimes(_ string, _, _ time.Time) error {
func (fs *Fs) Chtimes(_ string, _, _ time.Time) error {
return errors.New("method Chtimes is not implemented. Create, Delete, Updated times are read only fields in GCS and set implicitly")
}
func (fs *GcsFs) Chown(_ string, _, _ int) error {
func (fs *Fs) Chown(_ string, _, _ int) error {
return errors.New("method Chown is not implemented for GCS")
}

0
gcs-fake-service-account.json → gcsfs/gcs-fake-service-account.json
View File

25
gcs.go → gcsfs/gcs.go
View File

@ -14,29 +14,28 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package afero
package gcsfs
import (
"context"
"os"
"time"
"github.com/spf13/afero/gcsfs"
"cloud.google.com/go/storage"
"github.com/googleapis/google-cloud-go-testing/storage/stiface"
"github.com/spf13/afero"
"google.golang.org/api/option"
)
type GcsFs struct {
source *gcsfs.GcsFs
source *Fs
}
// NewGcsFS creates a GCS file system, automatically instantiating and decorating the storage client.
// You can provide additional options to be passed to the client creation, as per
// cloud.google.com/go/storage documentation
func NewGcsFS(ctx context.Context, opts ...option.ClientOption) (Fs, error) {
func NewGcsFS(ctx context.Context, opts ...option.ClientOption) (afero.Fs, error) {
if json := os.Getenv("GOOGLE_APPLICATION_CREDENTIALS_JSON"); json != "" {
opts = append(opts, option.WithCredentialsJSON([]byte(json)))
}
@ -49,7 +48,7 @@ func NewGcsFS(ctx context.Context, opts ...option.ClientOption) (Fs, error) {
}
// NewGcsFSWithSeparator is the same as NewGcsFS, but the files system will use the provided folder separator.
func NewGcsFSWithSeparator(ctx context.Context, folderSeparator string, opts ...option.ClientOption) (Fs, error) {
func NewGcsFSWithSeparator(ctx context.Context, folderSeparator string, opts ...option.ClientOption) (afero.Fs, error) {
client, err := storage.NewClient(ctx, opts...)
if err != nil {
return nil, err
@ -59,17 +58,17 @@ func NewGcsFSWithSeparator(ctx context.Context, folderSeparator string, opts ...
}
// NewGcsFSFromClient creates a GCS file system from a given storage client
func NewGcsFSFromClient(ctx context.Context, client *storage.Client) (Fs, error) {
func NewGcsFSFromClient(ctx context.Context, client *storage.Client) (afero.Fs, error) {
c := stiface.AdaptClient(client)
return &GcsFs{gcsfs.NewGcsFs(ctx, c)}, nil
return &GcsFs{NewGcsFs(ctx, c)}, nil
}
// NewGcsFSFromClientWithSeparator is the same as NewGcsFSFromClient, but the file system will use the provided folder separator.
func NewGcsFSFromClientWithSeparator(ctx context.Context, client *storage.Client, folderSeparator string) (Fs, error) {
func NewGcsFSFromClientWithSeparator(ctx context.Context, client *storage.Client, folderSeparator string) (afero.Fs, error) {
c := stiface.AdaptClient(client)
return &GcsFs{gcsfs.NewGcsFsWithSeparator(ctx, c, folderSeparator)}, nil
return &GcsFs{NewGcsFsWithSeparator(ctx, c, folderSeparator)}, nil
}
// Wraps gcs.GcsFs and convert some return types to afero interfaces.
@ -77,7 +76,7 @@ func NewGcsFSFromClientWithSeparator(ctx context.Context, client *storage.Client
func (fs *GcsFs) Name() string {
return fs.source.Name()
}
func (fs *GcsFs) Create(name string) (File, error) {
func (fs *GcsFs) Create(name string) (afero.File, error) {
return fs.source.Create(name)
}
func (fs *GcsFs) Mkdir(name string, perm os.FileMode) error {
@ -86,10 +85,10 @@ func (fs *GcsFs) Mkdir(name string, perm os.FileMode) error {
func (fs *GcsFs) MkdirAll(path string, perm os.FileMode) error {
return fs.source.MkdirAll(path, perm)
}
func (fs *GcsFs) Open(name string) (File, error) {
func (fs *GcsFs) Open(name string) (afero.File, error) {
return fs.source.Open(name)
}
func (fs *GcsFs) OpenFile(name string, flag int, perm os.FileMode) (File, error) {
func (fs *GcsFs) OpenFile(name string, flag int, perm os.FileMode) (afero.File, error) {
return fs.source.OpenFile(name, flag, perm)
}
func (fs *GcsFs) Remove(name string) error {

37
gcs_mocks.go → gcsfs/gcs_mocks.go
View File

@ -9,7 +9,7 @@
// switching over to a real bucket - and then the mocks have to be adjusted to match the
// implementation.
package afero
package gcsfs
import (
"context"
@ -17,10 +17,9 @@ import (
"os"
"strings"
"github.com/spf13/afero/gcsfs"
"cloud.google.com/go/storage"
"github.com/googleapis/google-cloud-go-testing/storage/stiface"
"github.com/spf13/afero"
"google.golang.org/api/iterator"
)
@ -31,11 +30,11 @@ func normSeparators(s string) string {
type clientMock struct {
stiface.Client
fs Fs
fs afero.Fs
}
func newClientMock() *clientMock {
return &clientMock{fs: NewMemMapFs()}
return &clientMock{fs: afero.NewMemMapFs()}
}
func (m *clientMock) Bucket(name string) stiface.BucketHandle {
@ -47,7 +46,7 @@ type bucketMock struct {
bucketName string
fs Fs
fs afero.Fs
}
func (m *bucketMock) Attrs(context.Context) (*storage.BucketAttrs, error) {
@ -66,7 +65,7 @@ type objectMock struct {
stiface.ObjectHandle
name string
fs Fs
fs afero.Fs
}
func (o *objectMock) NewWriter(_ context.Context) stiface.Writer {
@ -75,7 +74,7 @@ func (o *objectMock) NewWriter(_ context.Context) stiface.Writer {
func (o *objectMock) NewRangeReader(_ context.Context, offset, length int64) (stiface.Reader, error) {
if o.name == "" {
return nil, gcsfs.ErrEmptyObjectName
return nil, ErrEmptyObjectName
}
file, err := o.fs.Open(o.name)
@ -104,14 +103,14 @@ func (o *objectMock) NewRangeReader(_ context.Context, offset, length int64) (st
func (o *objectMock) Delete(_ context.Context) error {
if o.name == "" {
return gcsfs.ErrEmptyObjectName
return ErrEmptyObjectName
}
return o.fs.Remove(o.name)
}
func (o *objectMock) Attrs(_ context.Context) (*storage.ObjectAttrs, error) {
if o.name == "" {
return nil, gcsfs.ErrEmptyObjectName
return nil, ErrEmptyObjectName
}
info, err := o.fs.Stat(o.name)
@ -130,7 +129,7 @@ func (o *objectMock) Attrs(_ context.Context) (*storage.ObjectAttrs, error) {
if info.IsDir() {
// we have to mock it here, because of FileInfo logic
return nil, gcsfs.ErrObjectDoesNotExist
return nil, ErrObjectDoesNotExist
}
return res, nil
@ -140,14 +139,14 @@ type writerMock struct {
stiface.Writer
name string
fs Fs
fs afero.Fs
file File
file afero.File
}
func (w *writerMock) Write(p []byte) (n int, err error) {
if w.name == "" {
return 0, gcsfs.ErrEmptyObjectName
return 0, ErrEmptyObjectName
}
if w.file == nil {
@ -162,7 +161,7 @@ func (w *writerMock) Write(p []byte) (n int, err error) {
func (w *writerMock) Close() error {
if w.name == "" {
return gcsfs.ErrEmptyObjectName
return ErrEmptyObjectName
}
if w.file == nil {
var err error
@ -187,7 +186,7 @@ func (w *writerMock) Close() error {
type readerMock struct {
stiface.Reader
file File
file afero.File
buf []byte
}
@ -212,9 +211,9 @@ type objectItMock struct {
stiface.ObjectIterator
name string
fs Fs
fs afero.Fs
dir File
dir afero.File
infos []*storage.ObjectAttrs
}
@ -227,7 +226,7 @@ func (it *objectItMock) Next() (*storage.ObjectAttrs, error) {
}
var isDir bool
isDir, err = IsDir(it.fs, it.name)
isDir, err = afero.IsDir(it.fs, it.name)
if err != nil {
return nil, err
}

17
gcs_test.go → gcsfs/gcs_test.go
View File

@ -3,7 +3,7 @@
// Most of the tests are "derived" from the Afero's own tarfs implementation.
// Write-oriented tests and/or checks have been added on top of that
package afero
package gcsfs
import (
"context"
@ -19,10 +19,9 @@ import (
"golang.org/x/oauth2/google"
"github.com/spf13/afero/gcsfs"
"cloud.google.com/go/storage"
"github.com/googleapis/google-cloud-go-testing/storage/stiface"
"github.com/spf13/afero"
)
const (
@ -62,7 +61,7 @@ var dirs = []struct {
{"testDir1", []string{"testFile"}},
}
var gcsAfs *Afero
var gcsAfs *afero.Afero
func TestMain(m *testing.M) {
ctx := context.Background()
@ -120,7 +119,7 @@ func TestMain(m *testing.M) {
mockClient := newClientMock()
mockClient.Client = client
gcsAfs = &Afero{Fs: &GcsFs{gcsfs.NewGcsFs(ctx, mockClient)}}
gcsAfs = &afero.Afero{Fs: &GcsFs{NewGcsFs(ctx, mockClient)}}
// Uncomment to use the real, not mocked, client
//gcsAfs = &Afero{Fs: &GcsFs{gcsfs.NewGcsFs(ctx, client)}}
@ -137,7 +136,7 @@ func createFiles(t *testing.T) {
if !f.isdir && f.exists {
name := filepath.Join(bucketName, f.name)
var freshFile File
var freshFile afero.File
freshFile, err = gcsAfs.Create(name)
if err != nil {
t.Fatalf("failed to create a file \"%s\": %s", f.name, err)
@ -481,7 +480,7 @@ func TestGcsOpenFile(t *testing.T) {
file, err := gcsAfs.OpenFile(name, os.O_RDONLY, 0400)
if !f.exists {
if (f.name != "" && !errors.Is(err, syscall.ENOENT)) ||
(f.name == "" && !errors.Is(err, gcsfs.ErrNoBucketInName)) {
(f.name == "" && !errors.Is(err, ErrNoBucketInName)) {
t.Errorf("%v: got %v, expected%v", name, err, syscall.ENOENT)
}
@ -524,7 +523,7 @@ func TestGcsFsStat(t *testing.T) {
fi, err := gcsAfs.Stat(name)
if !f.exists {
if (f.name != "" && !errors.Is(err, syscall.ENOENT)) ||
(f.name == "" && !errors.Is(err, gcsfs.ErrNoBucketInName)) {
(f.name == "" && !errors.Is(err, ErrNoBucketInName)) {
t.Errorf("%v: got %v, expected%v", name, err, syscall.ENOENT)
}
@ -698,7 +697,7 @@ func TestGcsGlob(t *testing.T) {
}
for i, prefixedGlob := range prefixedGlobs {
entries, err := Glob(gcsAfs.Fs, prefixedGlob)
entries, err := afero.Glob(gcsAfs.Fs, prefixedGlob)
if err != nil {
t.Error(err)
}