re
/
afero
forked from mirror/afero
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #333 from drakkan/gcsfs 

GCS fs: move all gcsfs related implementations to its own package
This commit is contained in:
Michalis Kargakis 2021-12-28 23:53:39 +01:00 committed by GitHub
parent d70f944720 165e3dc3a5
commit d8a4ef9d05
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 65 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
gcsfs/file.go
View File

@ -44,7 +44,7 @@ type GcsFile struct {
func NewGcsFile(
ctx context.Context,
fs *GcsFs,
fs *Fs,
obj stiface.ObjectHandle,
openFlags int,
// Unused: there is no use to the file mode in GCloud just yet - but we keep it here, just in case we need it

2
gcsfs/file_info.go
View File

@ -37,7 +37,7 @@ type FileInfo struct {
fileMode os.FileMode
}
func newFileInfo(name string, fs *GcsFs, fileMode os.FileMode) (*FileInfo, error) {
func newFileInfo(name string, fs *Fs, fileMode os.FileMode) (*FileInfo, error) {
res := &FileInfo{
name: name,
size: folderSize,

2
gcsfs/file_resource.go
View File

@ -40,7 +40,7 @@ const (
type gcsFileResource struct {
ctx context.Context
fs *GcsFs
fs *Fs
obj stiface.ObjectHandle
name string

48
gcsfs/fs.go
View File

@ -33,8 +33,8 @@ const (
gsPrefix = "gs://"
)
// GcsFs is a Fs implementation that uses functions provided by google cloud storage
type GcsFs struct {
// Fs is a Fs implementation that uses functions provided by google cloud storage
type Fs struct {
ctx context.Context
client stiface.Client
separator string
@ -45,12 +45,12 @@ type GcsFs struct {
autoRemoveEmptyFolders bool //trigger for creating "virtual folders" (not required by GCSs)
}
func NewGcsFs(ctx context.Context, client stiface.Client) *GcsFs {
func NewGcsFs(ctx context.Context, client stiface.Client) *Fs {
return NewGcsFsWithSeparator(ctx, client, "/")
}
func NewGcsFsWithSeparator(ctx context.Context, client stiface.Client, folderSep string) *GcsFs {
return &GcsFs{
func NewGcsFsWithSeparator(ctx context.Context, client stiface.Client, folderSep string) *Fs {
return &Fs{
ctx: ctx,
client: client,
separator: folderSep,
@ -61,17 +61,17 @@ func NewGcsFsWithSeparator(ctx context.Context, client stiface.Client, folderSep
}
// normSeparators will normalize all "\\" and "/" to the provided separator
func (fs *GcsFs) normSeparators(s string) string {
func (fs *Fs) normSeparators(s string) string {
return strings.Replace(strings.Replace(s, "\\", fs.separator, -1), "/", fs.separator, -1)
}
func (fs *GcsFs) ensureTrailingSeparator(s string) string {
func (fs *Fs) ensureTrailingSeparator(s string) string {
if len(s) > 0 && !strings.HasSuffix(s, fs.separator) {
return s + fs.separator
}
return s
}
func (fs *GcsFs) ensureNoLeadingSeparator(s string) string {
func (fs *Fs) ensureNoLeadingSeparator(s string) string {
if len(s) > 0 && strings.HasPrefix(s, fs.separator) {
s = s[len(fs.separator):]
}
@ -94,13 +94,13 @@ func validateName(s string) error {
}
// Splits provided name into bucket name and path
func (fs *GcsFs) splitName(name string) (bucketName string, path string) {
func (fs *Fs) splitName(name string) (bucketName string, path string) {
splitName := strings.Split(name, fs.separator)
return splitName[0], strings.Join(splitName[1:], fs.separator)
}
func (fs *GcsFs) getBucket(name string) (stiface.BucketHandle, error) {
func (fs *Fs) getBucket(name string) (stiface.BucketHandle, error) {
bucket := fs.buckets[name]
if bucket == nil {
bucket = fs.client.Bucket(name)
@ -112,7 +112,7 @@ func (fs *GcsFs) getBucket(name string) (stiface.BucketHandle, error) {
return bucket, nil
}
func (fs *GcsFs) getObj(name string) (stiface.ObjectHandle, error) {
func (fs *Fs) getObj(name string) (stiface.ObjectHandle, error) {
bucketName, path := fs.splitName(name)
bucket, err := fs.getBucket(bucketName)
@ -123,9 +123,9 @@ func (fs *GcsFs) getObj(name string) (stiface.ObjectHandle, error) {
return bucket.Object(path), nil
}
func (fs *GcsFs) Name() string { return "GcsFs" }
func (fs *Fs) Name() string { return "GcsFs" }
func (fs *GcsFs) Create(name string) (*GcsFile, error) {
func (fs *Fs) Create(name string) (*GcsFile, error) {
name = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(name)))
if err := validateName(name); err != nil {
return nil, err
@ -156,7 +156,7 @@ func (fs *GcsFs) Create(name string) (*GcsFile, error) {
return file, nil
}
func (fs *GcsFs) Mkdir(name string, _ os.FileMode) error {
func (fs *Fs) Mkdir(name string, _ os.FileMode) error {
name = fs.ensureNoLeadingSeparator(fs.ensureTrailingSeparator(fs.normSeparators(ensureNoPrefix(name))))
if err := validateName(name); err != nil {
return err
@ -179,7 +179,7 @@ func (fs *GcsFs) Mkdir(name string, _ os.FileMode) error {
return w.Close()
}
func (fs *GcsFs) MkdirAll(path string, perm os.FileMode) error {
func (fs *Fs) MkdirAll(path string, perm os.FileMode) error {
path = fs.ensureNoLeadingSeparator(fs.ensureTrailingSeparator(fs.normSeparators(ensureNoPrefix(path))))
if err := validateName(path); err != nil {
return err
@ -216,11 +216,11 @@ func (fs *GcsFs) MkdirAll(path string, perm os.FileMode) error {
return nil
}
func (fs *GcsFs) Open(name string) (*GcsFile, error) {
func (fs *Fs) Open(name string) (*GcsFile, error) {
return fs.OpenFile(name, os.O_RDONLY, 0)
}
func (fs *GcsFs) OpenFile(name string, flag int, fileMode os.FileMode) (*GcsFile, error) {
func (fs *Fs) OpenFile(name string, flag int, fileMode os.FileMode) (*GcsFile, error) {
var file *GcsFile
var err error
@ -277,7 +277,7 @@ func (fs *GcsFs) OpenFile(name string, flag int, fileMode os.FileMode) (*GcsFile
return file, nil
}
func (fs *GcsFs) Remove(name string) error {
func (fs *Fs) Remove(name string) error {
name = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(name)))
if err := validateName(name); err != nil {
return err
@ -318,7 +318,7 @@ func (fs *GcsFs) Remove(name string) error {
return obj.Delete(fs.ctx)
}
func (fs *GcsFs) RemoveAll(path string) error {
func (fs *Fs) RemoveAll(path string) error {
path = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(path)))
if err := validateName(path); err != nil {
return err
@ -351,7 +351,7 @@ func (fs *GcsFs) RemoveAll(path string) error {
return fs.Remove(path)
}
func (fs *GcsFs) Rename(oldName, newName string) error {
func (fs *Fs) Rename(oldName, newName string) error {
oldName = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(oldName)))
if err := validateName(oldName); err != nil {
return err
@ -378,7 +378,7 @@ func (fs *GcsFs) Rename(oldName, newName string) error {
return src.Delete(fs.ctx)
}
func (fs *GcsFs) Stat(name string) (os.FileInfo, error) {
func (fs *Fs) Stat(name string) (os.FileInfo, error) {
name = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(name)))
if err := validateName(name); err != nil {
return nil, err
@ -387,14 +387,14 @@ func (fs *GcsFs) Stat(name string) (os.FileInfo, error) {
return newFileInfo(name, fs, defaultFileMode)
}
func (fs *GcsFs) Chmod(_ string, _ os.FileMode) error {
func (fs *Fs) Chmod(_ string, _ os.FileMode) error {
return errors.New("method Chmod is not implemented in GCS")
}
func (fs *GcsFs) Chtimes(_ string, _, _ time.Time) error {
func (fs *Fs) Chtimes(_ string, _, _ time.Time) error {
return errors.New("method Chtimes is not implemented. Create, Delete, Updated times are read only fields in GCS and set implicitly")
}
func (fs *GcsFs) Chown(_ string, _, _ int) error {
func (fs *Fs) Chown(_ string, _, _ int) error {
return errors.New("method Chown is not implemented for GCS")
}

0
gcs-fake-service-account.json → gcsfs/gcs-fake-service-account.json
View File

25
gcs.go → gcsfs/gcs.go
View File

@ -14,29 +14,28 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package afero
package gcsfs
import (
"context"
"os"
"time"
"github.com/spf13/afero/gcsfs"
"cloud.google.com/go/storage"
"github.com/googleapis/google-cloud-go-testing/storage/stiface"
"github.com/spf13/afero"
"google.golang.org/api/option"
)
type GcsFs struct {
source *gcsfs.GcsFs
source *Fs
}
// NewGcsFS creates a GCS file system, automatically instantiating and decorating the storage client.
// You can provide additional options to be passed to the client creation, as per
// cloud.google.com/go/storage documentation
func NewGcsFS(ctx context.Context, opts ...option.ClientOption) (Fs, error) {
func NewGcsFS(ctx context.Context, opts ...option.ClientOption) (afero.Fs, error) {
if json := os.Getenv("GOOGLE_APPLICATION_CREDENTIALS_JSON"); json != "" {
opts = append(opts, option.WithCredentialsJSON([]byte(json)))
}
@ -49,7 +48,7 @@ func NewGcsFS(ctx context.Context, opts ...option.ClientOption) (Fs, error) {
}
// NewGcsFSWithSeparator is the same as NewGcsFS, but the files system will use the provided folder separator.
func NewGcsFSWithSeparator(ctx context.Context, folderSeparator string, opts ...option.ClientOption) (Fs, error) {
func NewGcsFSWithSeparator(ctx context.Context, folderSeparator string, opts ...option.ClientOption) (afero.Fs, error) {
client, err := storage.NewClient(ctx, opts...)
if err != nil {
return nil, err
@ -59,17 +58,17 @@ func NewGcsFSWithSeparator(ctx context.Context, folderSeparator string, opts ...
}
// NewGcsFSFromClient creates a GCS file system from a given storage client
func NewGcsFSFromClient(ctx context.Context, client *storage.Client) (Fs, error) {
func NewGcsFSFromClient(ctx context.Context, client *storage.Client) (afero.Fs, error) {
c := stiface.AdaptClient(client)
return &GcsFs{gcsfs.NewGcsFs(ctx, c)}, nil
return &GcsFs{NewGcsFs(ctx, c)}, nil
}
// NewGcsFSFromClientWithSeparator is the same as NewGcsFSFromClient, but the file system will use the provided folder separator.
func NewGcsFSFromClientWithSeparator(ctx context.Context, client *storage.Client, folderSeparator string) (Fs, error) {
func NewGcsFSFromClientWithSeparator(ctx context.Context, client *storage.Client, folderSeparator string) (afero.Fs, error) {
c := stiface.AdaptClient(client)
return &GcsFs{gcsfs.NewGcsFsWithSeparator(ctx, c, folderSeparator)}, nil
return &GcsFs{NewGcsFsWithSeparator(ctx, c, folderSeparator)}, nil
}
// Wraps gcs.GcsFs and convert some return types to afero interfaces.
@ -77,7 +76,7 @@ func NewGcsFSFromClientWithSeparator(ctx context.Context, client *storage.Client
func (fs *GcsFs) Name() string {
return fs.source.Name()
}
func (fs *GcsFs) Create(name string) (File, error) {
func (fs *GcsFs) Create(name string) (afero.File, error) {
return fs.source.Create(name)
}
func (fs *GcsFs) Mkdir(name string, perm os.FileMode) error {
@ -86,10 +85,10 @@ func (fs *GcsFs) Mkdir(name string, perm os.FileMode) error {
func (fs *GcsFs) MkdirAll(path string, perm os.FileMode) error {
return fs.source.MkdirAll(path, perm)
}
func (fs *GcsFs) Open(name string) (File, error) {
func (fs *GcsFs) Open(name string) (afero.File, error) {
return fs.source.Open(name)
}
func (fs *GcsFs) OpenFile(name string, flag int, perm os.FileMode) (File, error) {
func (fs *GcsFs) OpenFile(name string, flag int, perm os.FileMode) (afero.File, error) {
return fs.source.OpenFile(name, flag, perm)
}
func (fs *GcsFs) Remove(name string) error {

37
gcs_mocks.go → gcsfs/gcs_mocks.go
View File

@ -9,7 +9,7 @@
// switching over to a real bucket - and then the mocks have to be adjusted to match the
// implementation.
package afero
package gcsfs
import (
"context"
@ -17,10 +17,9 @@ import (
"os"
"strings"
"github.com/spf13/afero/gcsfs"
"cloud.google.com/go/storage"
"github.com/googleapis/google-cloud-go-testing/storage/stiface"
"github.com/spf13/afero"
"google.golang.org/api/iterator"
)
@ -31,11 +30,11 @@ func normSeparators(s string) string {
type clientMock struct {
stiface.Client
fs Fs
fs afero.Fs
}
func newClientMock() *clientMock {
return &clientMock{fs: NewMemMapFs()}
return &clientMock{fs: afero.NewMemMapFs()}
}
func (m *clientMock) Bucket(name string) stiface.BucketHandle {
@ -47,7 +46,7 @@ type bucketMock struct {
bucketName string
fs Fs
fs afero.Fs
}
func (m *bucketMock) Attrs(context.Context) (*storage.BucketAttrs, error) {
@ -66,7 +65,7 @@ type objectMock struct {
stiface.ObjectHandle
name string
fs Fs
fs afero.Fs
}
func (o *objectMock) NewWriter(_ context.Context) stiface.Writer {
@ -75,7 +74,7 @@ func (o *objectMock) NewWriter(_ context.Context) stiface.Writer {
func (o *objectMock) NewRangeReader(_ context.Context, offset, length int64) (stiface.Reader, error) {
if o.name == "" {
return nil, gcsfs.ErrEmptyObjectName
return nil, ErrEmptyObjectName
}
file, err := o.fs.Open(o.name)
@ -104,14 +103,14 @@ func (o *objectMock) NewRangeReader(_ context.Context, offset, length int64) (st
func (o *objectMock) Delete(_ context.Context) error {
if o.name == "" {
return gcsfs.ErrEmptyObjectName
return ErrEmptyObjectName
}
return o.fs.Remove(o.name)
}
func (o *objectMock) Attrs(_ context.Context) (*storage.ObjectAttrs, error) {
if o.name == "" {
return nil, gcsfs.ErrEmptyObjectName
return nil, ErrEmptyObjectName
}
info, err := o.fs.Stat(o.name)
@ -130,7 +129,7 @@ func (o *objectMock) Attrs(_ context.Context) (*storage.ObjectAttrs, error) {
if info.IsDir() {
// we have to mock it here, because of FileInfo logic
return nil, gcsfs.ErrObjectDoesNotExist
return nil, ErrObjectDoesNotExist
}
return res, nil
@ -140,14 +139,14 @@ type writerMock struct {
stiface.Writer
name string
fs Fs
fs afero.Fs
file File
file afero.File
}
func (w *writerMock) Write(p []byte) (n int, err error) {
if w.name == "" {
return 0, gcsfs.ErrEmptyObjectName
return 0, ErrEmptyObjectName
}
if w.file == nil {
@ -162,7 +161,7 @@ func (w *writerMock) Write(p []byte) (n int, err error) {
func (w *writerMock) Close() error {
if w.name == "" {
return gcsfs.ErrEmptyObjectName
return ErrEmptyObjectName
}
if w.file == nil {
var err error
@ -187,7 +186,7 @@ func (w *writerMock) Close() error {
type readerMock struct {
stiface.Reader
file File
file afero.File
buf []byte
}
@ -212,9 +211,9 @@ type objectItMock struct {
stiface.ObjectIterator
name string
fs Fs
fs afero.Fs
dir File
dir afero.File
infos []*storage.ObjectAttrs
}
@ -227,7 +226,7 @@ func (it *objectItMock) Next() (*storage.ObjectAttrs, error) {
}
var isDir bool
isDir, err = IsDir(it.fs, it.name)
isDir, err = afero.IsDir(it.fs, it.name)
if err != nil {
return nil, err
}

17
gcs_test.go → gcsfs/gcs_test.go
View File

@ -3,7 +3,7 @@
// Most of the tests are "derived" from the Afero's own tarfs implementation.
// Write-oriented tests and/or checks have been added on top of that
package afero
package gcsfs
import (
"context"
@ -19,10 +19,9 @@ import (
"golang.org/x/oauth2/google"
"github.com/spf13/afero/gcsfs"
"cloud.google.com/go/storage"
"github.com/googleapis/google-cloud-go-testing/storage/stiface"
"github.com/spf13/afero"
)
const (
@ -62,7 +61,7 @@ var dirs = []struct {
{"testDir1", []string{"testFile"}},
}
var gcsAfs *Afero
var gcsAfs *afero.Afero
func TestMain(m *testing.M) {
ctx := context.Background()
@ -120,7 +119,7 @@ func TestMain(m *testing.M) {
mockClient := newClientMock()
mockClient.Client = client
gcsAfs = &Afero{Fs: &GcsFs{gcsfs.NewGcsFs(ctx, mockClient)}}
gcsAfs = &afero.Afero{Fs: &GcsFs{NewGcsFs(ctx, mockClient)}}
// Uncomment to use the real, not mocked, client
//gcsAfs = &Afero{Fs: &GcsFs{gcsfs.NewGcsFs(ctx, client)}}
@ -137,7 +136,7 @@ func createFiles(t *testing.T) {
if !f.isdir && f.exists {
name := filepath.Join(bucketName, f.name)
var freshFile File
var freshFile afero.File
freshFile, err = gcsAfs.Create(name)
if err != nil {
t.Fatalf("failed to create a file \"%s\": %s", f.name, err)
@ -481,7 +480,7 @@ func TestGcsOpenFile(t *testing.T) {
file, err := gcsAfs.OpenFile(name, os.O_RDONLY, 0400)
if !f.exists {
if (f.name != "" && !errors.Is(err, syscall.ENOENT)) ||
(f.name == "" && !errors.Is(err, gcsfs.ErrNoBucketInName)) {
(f.name == "" && !errors.Is(err, ErrNoBucketInName)) {
t.Errorf("%v: got %v, expected%v", name, err, syscall.ENOENT)
}
@ -524,7 +523,7 @@ func TestGcsFsStat(t *testing.T) {
fi, err := gcsAfs.Stat(name)
if !f.exists {
if (f.name != "" && !errors.Is(err, syscall.ENOENT)) ||
(f.name == "" && !errors.Is(err, gcsfs.ErrNoBucketInName)) {
(f.name == "" && !errors.Is(err, ErrNoBucketInName)) {
t.Errorf("%v: got %v, expected%v", name, err, syscall.ENOENT)
}
@ -698,7 +697,7 @@ func TestGcsGlob(t *testing.T) {
}
for i, prefixedGlob := range prefixedGlobs {
entries, err := Glob(gcsAfs.Fs, prefixedGlob)
entries, err := afero.Glob(gcsAfs.Fs, prefixedGlob)
if err != nil {
t.Error(err)
}