diff --git a/go.mod b/go.mod index b77ae41..251deb2 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,9 @@ module github.com/spf13/afero require ( cloud.google.com/go/storage v1.14.0 github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8 + github.com/ncw/swift v1.0.47 github.com/pkg/sftp v1.13.1 + github.com/stretchr/testify v1.4.0 golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99 golang.org/x/text v0.3.4 diff --git a/swift/file.go b/swift/file.go new file mode 100644 index 0000000..4b8fd4b --- /dev/null +++ b/swift/file.go @@ -0,0 +1,254 @@ +package swift + +import ( + "bytes" + "fmt" + "github.com/ncw/swift" + "io" + "log" + "os" + "path/filepath" + "time" +) + +type File struct { + objectCreateFile *swift.ObjectCreateFile + objectOpenFile *swift.ObjectOpenFile + tmpContent []byte + fileInfo *FileInfo + + seekStart int64 + + conn *swift.Connection // The connection to be able to do things like updating the file. This is probably not the best way to do it. + containerName string +} + +func (file *File) Close() error { + if file.objectCreateFile != nil { + return file.objectCreateFile.Close() + } + + if file.objectOpenFile != nil { + return file.objectOpenFile.Close() + } + + log.Println("nothing to close") + return nil +} + +func (file *File) Read(p []byte) (n int, err error) { + return file.ReadAt(p, file.seekStart) +} + +func (file *File) ReadAt(p []byte, off int64) (n int, err error) { + if file.objectOpenFile != nil { + return file.objectOpenFile.Read(p) + } + + if file.tmpContent != nil { + // If we have an offset > 0, we just strip the first offset n bytes from the + // input and put that into p + if off > 0 { + file.tmpContent = file.tmpContent[off:] + } + + if len(file.tmpContent) > len(p) { + return 0, io.EOF + } + return copy(p, file.tmpContent), nil + } + + return 0, fmt.Errorf("nothing to read") +} + +func (file *File) Seek(offset int64, whence int) (int64, error) { + if file.tmpContent == nil { + return 0, fmt.Errorf("no open file") + } + + switch whence { + case io.SeekStart: + file.seekStart = offset + case io.SeekCurrent: + file.seekStart += offset + case io.SeekEnd: + file.seekStart = int64(len(file.tmpContent)) - offset + default: + return 0, fmt.Errorf("unkown whence") + } + + return file.seekStart, nil +} + +func (file *File) Write(p []byte) (n int, err error) { + return file.WriteAt(p, file.seekStart) +} + +func (file *File) WriteAt(p []byte, off int64) (n int, err error) { + if file.objectCreateFile != nil { + return file.objectCreateFile.Write(p) + } + + // If offset is > 0, we need to modify our p slightly and add the len(off) bytes to p before it, since it is not + // possible to write to a swift file with an offset + if off > 0 { + tmp := make([]byte, off) + _, err := file.Read(tmp) + if err != nil { + return 0, err + } + p = append(tmp, p...) + } + + // If we have an open file, write to it. + if file.objectOpenFile != nil { + _, err = file.conn.ObjectPut(file.containerName, file.Name(), bytes.NewReader(p), true, "", "", swift.Headers{}) + return int(len(p)), err // Apparently swift does not return the length it wrote. So we just return the size to not break anything. + } + + // If we're at this point, that means no other file type thing (like objectCreateFile) exists. + // This usually happens when we want to get a file and then get its content. + // In that case put that content in here temporarily so we can access it later. + file.tmpContent = p + return len(p), nil +} + +func (file *File) Name() string { + return file.fileInfo.Name() +} + +func (file *File) Readdir(count int) (res []os.FileInfo, err error) { + if !file.fileInfo.IsDir() { + return nil, fmt.Errorf("%s is not a dir", file.fileInfo.Name()) + } + + files, err := file.conn.ObjectsAll(file.containerName, &swift.ObjectsOpts{ + Limit: count, + Headers: map[string]string{"Content-Type": "application/directory"}, + }) + if err != nil { + return nil, err + } + + // This could probably be optimized slightly to consume less memory + res = make([]os.FileInfo, count) + for i, f := range files { + // Sometimes filtering directly in the request does not work, so we need to manually filder here + if f.ContentType != "application/directory" { + continue + } + res[i] = FileInfo{ + name: f.Name, + size: f.Bytes, + modTime: f.LastModified, + mode: os.ModeDir, // At this point we can be sure this is always a folder + } + } + + return res, nil +} + +func (file *File) Readdirnames(n int) (names []string, err error) { + infs, err := file.Readdir(n) + if err != nil { + return + } + + names = make([]string, len(infs)) + for i, f := range infs { + _, names[i] = filepath.Split(f.Name()) + } + + return +} + +func (file *File) Stat() (os.FileInfo, error) { + return file.fileInfo, nil +} + +func (file *File) Sync() error { + return nil +} + +func (file *File) Truncate(size int64) error { + _, err := file.conn.ObjectPut(file.containerName, file.Name(), bytes.NewReader([]byte{}), true, "", "", swift.Headers{}) + return err +} + +func (file *File) WriteString(s string) (ret int, err error) { + return file.Write([]byte(s)) +} + +func (file *File) putFileInfoTogetherFromOpenObject(f *swift.ObjectOpenFile, headers swift.Headers, name string) (err error) { + file.objectOpenFile = f + + // Put the file info together + size, err := f.Length() + if err != nil { + return + } + var modtime time.Time + modtime, err = headers.ObjectMetadata().GetModTime() + if err != nil { + // Try parsing it directly if the method fails + modtime, err = time.Parse("Mon, 02 Jan 2006 15:04:05 MST", headers["Last-Modified"]) + if err == nil { + goto putFileInfoTogether + } + + return err + } + +putFileInfoTogether: + + var mode os.FileMode = os.ModePerm + if headers["Content-Type"] == "application/directory" { + mode = os.ModeDir + } + + file.fileInfo = &FileInfo{ + name: name, + size: size, + modTime: modtime, + mode: mode, // Apparently there is no way to get the mode of an object. Or am I missing something? + } + return nil +} + +// ===================== +// File Info starts here +// ===================== + +type FileInfo struct { + name string + size int64 + mode os.FileMode + modTime time.Time +} + +func (fi FileInfo) Name() string { + return fi.name +} + +func (fi FileInfo) Size() int64 { + if fi.IsDir() { + return int64(42) + } + return fi.size +} + +func (fi FileInfo) Mode() os.FileMode { + return fi.mode +} + +func (fi FileInfo) ModTime() time.Time { + return fi.modTime +} + +func (fi FileInfo) IsDir() bool { + return fi.Mode().IsDir() +} + +func (fi FileInfo) Sys() interface{} { + return &fi +} diff --git a/swift/swift.go b/swift/swift.go new file mode 100644 index 0000000..288aacf --- /dev/null +++ b/swift/swift.go @@ -0,0 +1,142 @@ +package swift + +import ( + "bytes" + "fmt" + "github.com/ncw/swift" + "log" + "os" + "time" +) + +type Fs struct { + Connection *swift.Connection + containerName string +} + +func NewSwiftFs(conn *swift.Connection, containerName string) (fs *Fs, err error) { + if containerName == "" { + return fs, fmt.Errorf("container name cannot be empty") + } + fs = &Fs{ + Connection: conn, + containerName: containerName, + } + err = fs.Connection.Authenticate() + return +} + +func (s *Fs) Name() string { + return "swiftfs" +} + +func (s *Fs) Create(name string) (file File, err error) { + objectfile, err := s.Connection.ObjectCreate(s.containerName, name, true, "", "", swift.Headers{}) + if err != nil { + return file, err + } + file.objectCreateFile = objectfile + + return +} + +func (s *Fs) Mkdir(name string, perm os.FileMode) (err error) { + return s.MkdirAll(name, perm) +} + +func (s *Fs) MkdirAll(path string, perm os.FileMode) (err error) { + // Swift usually creates all directories deep down, no matter if any of these directories already exist + // Objects created with application/directory are so called "pseudo-directories". + _, err = s.Connection.ObjectPut(s.containerName, path+"/", bytes.NewReader([]byte{}), true, "", "application/directory", swift.Headers{}) + return +} + +func (s *Fs) Open(name string) (file *File, err error) { + file = new(File) + file.conn = s.Connection + file.containerName = s.containerName + + f, headers, err := s.Connection.ObjectOpen(s.containerName, name, true, swift.Headers{}) + if err != nil { + return + } + err = file.putFileInfoTogetherFromOpenObject(f, headers, name) + return +} + +func (*Fs) OpenFile(name string, flag int, perm os.FileMode) (*File, error) { + return nil, nil +} + +func (s *Fs) Remove(name string) error { + return s.Connection.ObjectDelete(s.containerName, name) +} + +func (s *Fs) RemoveAll(path string) (err error) { + sinfo, err := s.Connection.QueryInfo() + if err != nil { + return + } + + // Get all objects we want to delte + objects, err := s.Connection.ObjectsAll(s.containerName, &swift.ObjectsOpts{Prefix: path}) + if err != nil { + return + } + + var objectnames []string + for _, o := range objects { + // If the server supports bulk delete, add it to a list to bulkdelete the items later, + // otherwise delete them directly + if sinfo.SupportsBulkDelete() { + objectnames = append(objectnames, o.Name) + } else { + err = s.Connection.ObjectDelete(s.containerName, o.Name) + if err != nil { + return + } + } + } + if sinfo.SupportsBulkDelete() { + _, err = s.Connection.BulkDelete(s.containerName, objectnames) + if err != nil { + return + } + } + + return err +} + +func (s *Fs) Rename(oldname, newname string) error { + return s.Connection.ObjectMove(s.containerName, oldname, s.containerName, newname) +} + +func (s *Fs) Stat(name string) (os.FileInfo, error) { + file, err := s.Open(name) + if err != nil { + return nil, err + } + return file.Stat() +} + +func (*Fs) Chmod(name string, mode os.FileMode) error { + log.Println("Swift does not support file modes.") + return nil +} + +func (s *Fs) Chtimes(name string, _, mtime time.Time) error { + // While the uncommented code should work, the problem is that we have no way to update the time on + // the server without modifying the time. Whenever we send a request to swift to update the last modified time, + // we modify the file thus setting the last modified time to now... + // Maybe I'll find a solution for this in the future, but for now I'll just diable it because of the reasons above. + return nil + /* + _, headers, err := s.Connection.ObjectOpen(s.containerName, name, true, swift.Headers{}) + + if err != nil { + return err + } + headers.ObjectMetadata().SetModTime(mtime) // Swift does not support access time so we won't use that. + return s.Connection.ObjectUpdate(s.containerName, name, headers) + */ +} diff --git a/swift/swift_test.go b/swift/swift_test.go new file mode 100644 index 0000000..603ff48 --- /dev/null +++ b/swift/swift_test.go @@ -0,0 +1,309 @@ +package swift + +import ( + "bytes" + "fmt" + "github.com/ncw/swift" + "github.com/stretchr/testify/assert" + "log" + "os" + "testing" + "time" +) + +type testFile struct { + name string + typ string +} + +func (s *Fs) prepareTestContainer() { + // Get all existing objects to clear the container + objects, err := s.Connection.ObjectsAll(s.containerName, nil) + if err != nil { + log.Fatal(err) + } + info, err := s.Connection.QueryInfo() + if err != nil { + log.Fatal(err) + } + + var objectnames []string + for _, o := range objects { + if info.SupportsBulkDelete() { + objectnames = append(objectnames, o.Name) + } else { + err = s.Connection.ObjectDelete(s.containerName, o.Name) + if err != nil { + log.Fatal(err) + } + log.Println("Deleted object", o.Name) + } + } + // Bulk delete all in one go if the server supports it + if info.SupportsBulkDelete() { + _, err = s.Connection.BulkDelete(s.containerName, objectnames) + if err != nil { + log.Fatal(err) + } + log.Println("Deleted", objectnames) + } + // Create some files we're expecting + for _, newObject := range []testFile{ + {"existingTestfile", "application/octet-stream"}, + {"fileToDelete", "application/octet-stream"}, + {"testfolder/", "application/directory"}, + {"testfolder/fileintestfolder1", "application/octet-stream"}, + {"testfolder/fileintestfolder2", "application/octet-stream"}, + {"testfolder/subfolder/", "application/directory"}, + {"testfolder/subfolder/fileinsubtestfolder1", "application/octet-stream"}, + {"testfolder/subfolder/fileinsubtestfolder2", "application/octet-stream"}, + } { + _, err := s.Connection.ObjectPut(s.containerName, newObject.name, bytes.NewReader([]byte{}), true, "", newObject.typ, swift.Headers{}) + if err != nil { + log.Fatal(err) + } + log.Println("Created object", newObject) + } +} + +func (s *Fs) checkExpectedContent(files []testFile) (notmatching []testFile) { + objects, err := s.Connection.ObjectsAll(s.containerName, nil) + if err != nil { + log.Fatal(err) + } + + // We use this map to keep track of files we already passed so we don't count them twice + filesfound := make(map[string]testFile) + // We use this map to keep track of all files we wanted, but didn't exist. + // When all files we want to exist exist, this map should be empty at the end of this function. + filesnotpassed := make(map[string]testFile) + for _, o := range objects { + for _, ob := range files { + filesnotpassed[ob.name] = ob + if o.Name == ob.name && o.ContentType == ob.typ { + filesfound[o.Name] = ob + delete(filesnotpassed, o.Name) + continue + } + if _, ok := filesfound[o.Name]; ok { + continue + } + } + } + + for _, o := range objects { + if _, ok := filesfound[o.Name]; !ok { + delete(filesnotpassed, o.Name) + // Notmatching will contain all files which exist, but we didn't want to exist. + notmatching = append(notmatching, testFile{o.Name, o.ContentType}) + } + } + + // Append all files we did not found + for _, f := range filesnotpassed { + if _, ok := filesfound[f.name]; !ok { + notmatching = append(notmatching, f) + } + } + + return +} + +// Helper method to get a testinstance +func createSwiftfsTestInstance() (fs *Fs, err error) { + // Maybe we can cache this somehow? (aka not creating a new connection for every test function) + // Also we need a good way to set up fixtures before each test. + fs, err = NewSwiftFs(&swift.Connection{ + UserName: os.Getenv("SWIFT_APIUSER"), + ApiKey: os.Getenv("SWIFT_APIKEY"), + AuthUrl: os.Getenv("SWIFT_AUTHURL"), + Domain: os.Getenv("SWIFT_DOMAIN"), + }, "testcontainer") + if err != nil { + return + } + fs.prepareTestContainer() + return +} + +var testStringToWrite = "lorem ipsum" + +func TestFs_Name(t *testing.T) { + swiftfs, err := createSwiftfsTestInstance() + assert.NoError(t, err) + + assert.Equal(t, "swiftfs", swiftfs.Name()) +} + +func TestFsCreate(t *testing.T) { + swiftfs, err := createSwiftfsTestInstance() + assert.NoError(t, err) + + file, err := swiftfs.Create("testfile") + assert.NoError(t, err) + + // Test writing to a created file + written, err := file.WriteString(testStringToWrite) + assert.NoError(t, err) + assert.Equal(t, len(testStringToWrite), written) + + err = file.Close() + assert.NoError(t, err) +} + +func TestFs_Open(t *testing.T) { + swiftfs, err := createSwiftfsTestInstance() + assert.NoError(t, err) + + // Try opening a file which does not exist + _, err = swiftfs.Open("notExistingTestfile") + assert.Error(t, err) + + // Open the real file + file, err := swiftfs.Open("existingTestfile") // This file needs to exist previously + assert.NoError(t, err) + + // Test writing to an open file + written, err := file.WriteString(testStringToWrite) + assert.NoError(t, err) + assert.Equal(t, len(testStringToWrite), written) + + err = file.Close() + assert.NoError(t, err) +} + +func TestFs_Mkdir(t *testing.T) { + swiftfs, err := createSwiftfsTestInstance() + assert.NoError(t, err) + + err = swiftfs.Mkdir("testdir", os.ModePerm) + assert.NoError(t, err) +} + +// MkdirAll should do exactly the same +func TestFs_MkdirAll(t *testing.T) { + swiftfs, err := createSwiftfsTestInstance() + assert.NoError(t, err) + + err = swiftfs.MkdirAll("testdir", os.ModePerm) + assert.NoError(t, err) +} + +func TestFs_Remove(t *testing.T) { + swiftfs, err := createSwiftfsTestInstance() + assert.NoError(t, err) + + err = swiftfs.Remove("fileToDelete") + assert.NoError(t, err) + + // Testing removing a second time should return an error as that file does not exist now + err = swiftfs.Remove("fileToDelete") + assert.Error(t, err) +} + +func TestFs_RemoveAll(t *testing.T) { + swiftfs, err := createSwiftfsTestInstance() + assert.NoError(t, err) + + err = swiftfs.RemoveAll("testfolder") + assert.NoError(t, err) + notmatching := swiftfs.checkExpectedContent([]testFile{ + {"existingTestfile", "application/octet-stream"}, + {"fileToDelete", "application/octet-stream"}, + }) + assert.Len(t, notmatching, 0) + + // Deleting something with removeAll should return nil if it does not exist + err = swiftfs.RemoveAll("testfolder") + assert.NoError(t, err) +} + +func TestFs_Rename(t *testing.T) { + swiftfs, err := createSwiftfsTestInstance() + assert.NoError(t, err) + + err = swiftfs.Rename("existingTestfile", "otherExistingTestfile") + assert.NoError(t, err) + notmatching := swiftfs.checkExpectedContent([]testFile{ + {"otherExistingTestfile", "application/octet-stream"}, + {"fileToDelete", "application/octet-stream"}, + {"testfolder/", "application/directory"}, + {"testfolder/fileintestfolder1", "application/octet-stream"}, + {"testfolder/fileintestfolder2", "application/octet-stream"}, + {"testfolder/subfolder/", "application/directory"}, + {"testfolder/subfolder/fileinsubtestfolder1", "application/octet-stream"}, + {"testfolder/subfolder/fileinsubtestfolder2", "application/octet-stream"}, + }) + assert.Len(t, notmatching, 0) + + // Moving a nonexisting file should not work + err = swiftfs.Rename("existingTestfile", "otherExistingTestfile") + assert.Error(t, err) +} + +func TestFs_Stat(t *testing.T) { + swiftfs, err := createSwiftfsTestInstance() + assert.NoError(t, err) + + finfo, err := swiftfs.Stat("existingTestfile") + assert.NoError(t, err) + assert.Equal(t, "existingTestfile", finfo.Name()) + assert.Equal(t, int64(0), finfo.Size()) + assert.Equal(t, os.ModePerm, finfo.Mode()) + assert.False(t, finfo.IsDir()) + assert.Equal(t, finfo, finfo.Sys()) +} + +func TestFs_Chmod(t *testing.T) { + swiftfs, err := createSwiftfsTestInstance() + assert.NoError(t, err) + + err = swiftfs.Chmod("existingTestfile", os.ModePerm) + assert.NoError(t, err) +} + +func TestFs_Chtimes(t *testing.T) { + swiftfs, err := createSwiftfsTestInstance() + assert.NoError(t, err) + + filename := "existingTestfile" + newtime := time.Date(2019, time.January, 31, 14, 0, 0, 0, time.UTC) + err = swiftfs.Chtimes(filename, newtime, newtime) + assert.NoError(t, err) + + // See the function on why this check is diabled. + /* + file, err := swiftfs.Open(filename) + assert.NoError(t, err) + finfo, err := file.Stat() + assert.NoError(t, err) + assert.True(t, newtime.Equal(finfo.ModTime())) + */ +} + +func TestReaddir(t *testing.T) { + swiftfs, err := createSwiftfsTestInstance() + assert.NoError(t, err) + + root, err := swiftfs.Open("testfolder/") + assert.NoError(t, err) + + folders, err := root.Readdir(10) // Random number + assert.NoError(t, err) + + numfolders := 2 + for _, f := range folders { + if f == nil { + continue + } + if f.IsDir() { + numfolders-- + continue + } + + fmt.Printf("%s is not a folder", f.Name()) + t.FailNow() + } + + assert.Equal(t, 0, numfolders) +}