Added Support for openstack swift object storage

go.mod

@@ -3,7 +3,9 @@ module github.com/spf13/afero
 require (
 	cloud.google.com/go/storage v1.14.0
 	github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8
+	github.com/ncw/swift v1.0.47
 	github.com/pkg/sftp v1.13.1
+	github.com/stretchr/testify v1.4.0
 	golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa
 	golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99
 	golang.org/x/text v0.3.4

swift/file.go

@@ -0,0 +1,254 @@
+package swift
+
+import (
+	"bytes"
+	"fmt"
+	"github.com/ncw/swift"
+	"io"
+	"log"
+	"os"
+	"path/filepath"
+	"time"
+)
+
+type File struct {
+	objectCreateFile *swift.ObjectCreateFile
+	objectOpenFile   *swift.ObjectOpenFile
+	tmpContent       []byte
+	fileInfo         *FileInfo
+
+	seekStart int64
+
+	conn          *swift.Connection // The connection to be able to do things like updating the file. This is probably not the best way to do it.
+	containerName string
+}
+
+func (file *File) Close() error {
+	if file.objectCreateFile != nil {
+		return file.objectCreateFile.Close()
+	}
+
+	if file.objectOpenFile != nil {
+		return file.objectOpenFile.Close()
+	}
+
+	log.Println("nothing to close")
+	return nil
+}
+
+func (file *File) Read(p []byte) (n int, err error) {
+	return file.ReadAt(p, file.seekStart)
+}
+
+func (file *File) ReadAt(p []byte, off int64) (n int, err error) {
+	if file.objectOpenFile != nil {
+		return file.objectOpenFile.Read(p)
+	}
+
+	if file.tmpContent != nil {
+		// If we have an offset > 0, we just strip the first offset n bytes from the
+		// input and put that into p
+		if off > 0 {
+			file.tmpContent = file.tmpContent[off:]
+		}
+
+		if len(file.tmpContent) > len(p) {
+			return 0, io.EOF
+		}
+		return copy(p, file.tmpContent), nil
+	}
+
+	return 0, fmt.Errorf("nothing to read")
+}
+
+func (file *File) Seek(offset int64, whence int) (int64, error) {
+	if file.tmpContent == nil {
+		return 0, fmt.Errorf("no open file")
+	}
+
+	switch whence {
+	case io.SeekStart:
+		file.seekStart = offset
+	case io.SeekCurrent:
+		file.seekStart += offset
+	case io.SeekEnd:
+		file.seekStart = int64(len(file.tmpContent)) - offset
+	default:
+		return 0, fmt.Errorf("unkown whence")
+	}
+
+	return file.seekStart, nil
+}
+
+func (file *File) Write(p []byte) (n int, err error) {
+	return file.WriteAt(p, file.seekStart)
+}
+
+func (file *File) WriteAt(p []byte, off int64) (n int, err error) {
+	if file.objectCreateFile != nil {
+		return file.objectCreateFile.Write(p)
+	}
+
+	// If offset is > 0, we need to modify our p slightly and add the len(off) bytes to p before it, since it is not
+	// possible to write to a swift file with an offset
+	if off > 0 {
+		tmp := make([]byte, off)
+		_, err := file.Read(tmp)
+		if err != nil {
+			return 0, err
+		}
+		p = append(tmp, p...)
+	}
+
+	// If we have an open file, write to it.
+	if file.objectOpenFile != nil {
+		_, err = file.conn.ObjectPut(file.containerName, file.Name(), bytes.NewReader(p), true, "", "", swift.Headers{})
+		return int(len(p)), err // Apparently swift does not return the length it wrote. So we just return the size to not break anything.
+	}
+
+	// If we're at this point, that means no other file type thing (like objectCreateFile) exists.
+	// This usually happens when we want to get a file and then get its content.
+	// In that case put that content in here temporarily so we can access it later.
+	file.tmpContent = p
+	return len(p), nil
+}
+
+func (file *File) Name() string {
+	return file.fileInfo.Name()
+}
+
+func (file *File) Readdir(count int) (res []os.FileInfo, err error) {
+	if !file.fileInfo.IsDir() {
+		return nil, fmt.Errorf("%s is not a dir", file.fileInfo.Name())
+	}
+
+	files, err := file.conn.ObjectsAll(file.containerName, &swift.ObjectsOpts{
+		Limit:   count,
+		Headers: map[string]string{"Content-Type": "application/directory"},
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	// This could probably be optimized slightly to consume less memory
+	res = make([]os.FileInfo, count)
+	for i, f := range files {
+		// Sometimes filtering directly in the request does not work, so we need to manually filder here
+		if f.ContentType != "application/directory" {
+			continue
+		}
+		res[i] = FileInfo{
+			name:    f.Name,
+			size:    f.Bytes,
+			modTime: f.LastModified,
+			mode:    os.ModeDir, // At this point we can be sure this is always a folder
+		}
+	}
+
+	return res, nil
+}
+
+func (file *File) Readdirnames(n int) (names []string, err error) {
+	infs, err := file.Readdir(n)
+	if err != nil {
+		return
+	}
+
+	names = make([]string, len(infs))
+	for i, f := range infs {
+		_, names[i] = filepath.Split(f.Name())
+	}
+
+	return
+}
+
+func (file *File) Stat() (os.FileInfo, error) {
+	return file.fileInfo, nil
+}
+
+func (file *File) Sync() error {
+	return nil
+}
+
+func (file *File) Truncate(size int64) error {
+	_, err := file.conn.ObjectPut(file.containerName, file.Name(), bytes.NewReader([]byte{}), true, "", "", swift.Headers{})
+	return err
+}
+
+func (file *File) WriteString(s string) (ret int, err error) {
+	return file.Write([]byte(s))
+}
+
+func (file *File) putFileInfoTogetherFromOpenObject(f *swift.ObjectOpenFile, headers swift.Headers, name string) (err error) {
+	file.objectOpenFile = f
+
+	// Put the file info together
+	size, err := f.Length()
+	if err != nil {
+		return
+	}
+	var modtime time.Time
+	modtime, err = headers.ObjectMetadata().GetModTime()
+	if err != nil {
+		// Try parsing it directly if the method fails
+		modtime, err = time.Parse("Mon, 02 Jan 2006 15:04:05 MST", headers["Last-Modified"])
+		if err == nil {
+			goto putFileInfoTogether
+		}
+
+		return err
+	}
+
+putFileInfoTogether:
+
+	var mode os.FileMode = os.ModePerm
+	if headers["Content-Type"] == "application/directory" {
+		mode = os.ModeDir
+	}
+
+	file.fileInfo = &FileInfo{
+		name:    name,
+		size:    size,
+		modTime: modtime,
+		mode:    mode, // Apparently there is no way to get the mode of an object. Or am I missing something?
+	}
+	return nil
+}
+
+// =====================
+// File Info starts here
+// =====================
+
+type FileInfo struct {
+	name    string
+	size    int64
+	mode    os.FileMode
+	modTime time.Time
+}
+
+func (fi FileInfo) Name() string {
+	return fi.name
+}
+
+func (fi FileInfo) Size() int64 {
+	if fi.IsDir() {
+		return int64(42)
+	}
+	return fi.size
+}
+
+func (fi FileInfo) Mode() os.FileMode {
+	return fi.mode
+}
+
+func (fi FileInfo) ModTime() time.Time {
+	return fi.modTime
+}
+
+func (fi FileInfo) IsDir() bool {
+	return fi.Mode().IsDir()
+}
+
+func (fi FileInfo) Sys() interface{} {
+	return &fi
+}

swift/swift.go

@@ -0,0 +1,142 @@
+package swift
+
+import (
+	"bytes"
+	"fmt"
+	"github.com/ncw/swift"
+	"log"
+	"os"
+	"time"
+)
+
+type Fs struct {
+	Connection    *swift.Connection
+	containerName string
+}
+
+func NewSwiftFs(conn *swift.Connection, containerName string) (fs *Fs, err error) {
+	if containerName == "" {
+		return fs, fmt.Errorf("container name cannot be empty")
+	}
+	fs = &Fs{
+		Connection:    conn,
+		containerName: containerName,
+	}
+	err = fs.Connection.Authenticate()
+	return
+}
+
+func (s *Fs) Name() string {
+	return "swiftfs"
+}
+
+func (s *Fs) Create(name string) (file File, err error) {
+	objectfile, err := s.Connection.ObjectCreate(s.containerName, name, true, "", "", swift.Headers{})
+	if err != nil {
+		return file, err
+	}
+	file.objectCreateFile = objectfile
+
+	return
+}
+
+func (s *Fs) Mkdir(name string, perm os.FileMode) (err error) {
+	return s.MkdirAll(name, perm)
+}
+
+func (s *Fs) MkdirAll(path string, perm os.FileMode) (err error) {
+	// Swift usually creates all directories deep down, no matter if any of these directories already exist
+	// Objects created with application/directory are so called "pseudo-directories".
+	_, err = s.Connection.ObjectPut(s.containerName, path+"/", bytes.NewReader([]byte{}), true, "", "application/directory", swift.Headers{})
+	return
+}
+
+func (s *Fs) Open(name string) (file *File, err error) {
+	file = new(File)
+	file.conn = s.Connection
+	file.containerName = s.containerName
+
+	f, headers, err := s.Connection.ObjectOpen(s.containerName, name, true, swift.Headers{})
+	if err != nil {
+		return
+	}
+	err = file.putFileInfoTogetherFromOpenObject(f, headers, name)
+	return
+}
+
+func (*Fs) OpenFile(name string, flag int, perm os.FileMode) (*File, error) {
+	return nil, nil
+}
+
+func (s *Fs) Remove(name string) error {
+	return s.Connection.ObjectDelete(s.containerName, name)
+}
+
+func (s *Fs) RemoveAll(path string) (err error) {
+	sinfo, err := s.Connection.QueryInfo()
+	if err != nil {
+		return
+	}
+
+	// Get all objects we want to delte
+	objects, err := s.Connection.ObjectsAll(s.containerName, &swift.ObjectsOpts{Prefix: path})
+	if err != nil {
+		return
+	}
+
+	var objectnames []string
+	for _, o := range objects {
+		// If the server supports bulk delete, add it to a list to bulkdelete the items later,
+		// otherwise delete them directly
+		if sinfo.SupportsBulkDelete() {
+			objectnames = append(objectnames, o.Name)
+		} else {
+			err = s.Connection.ObjectDelete(s.containerName, o.Name)
+			if err != nil {
+				return
+			}
+		}
+	}
+	if sinfo.SupportsBulkDelete() {
+		_, err = s.Connection.BulkDelete(s.containerName, objectnames)
+		if err != nil {
+			return
+		}
+	}
+
+	return err
+}
+
+func (s *Fs) Rename(oldname, newname string) error {
+	return s.Connection.ObjectMove(s.containerName, oldname, s.containerName, newname)
+}
+
+func (s *Fs) Stat(name string) (os.FileInfo, error) {
+	file, err := s.Open(name)
+	if err != nil {
+		return nil, err
+	}
+	return file.Stat()
+}
+
+func (*Fs) Chmod(name string, mode os.FileMode) error {
+	log.Println("Swift does not support file modes.")
+	return nil
+}
+
+func (s *Fs) Chtimes(name string, _, mtime time.Time) error {
+	// While the uncommented code should work, the problem is that we have no way to update the time on
+	// the server without modifying the time. Whenever we send a request to swift to update the last modified time,
+	// we modify the file thus setting the last modified time to now...
+	// Maybe I'll find a solution for this in the future, but for now I'll just diable it because of the reasons above.
+	return nil
+	/*
+		_, headers, err := s.Connection.ObjectOpen(s.containerName, name, true, swift.Headers{})
+
+		if err != nil {
+			return err
+		}
+		headers.ObjectMetadata().SetModTime(mtime) // Swift does not support access time so we won't use that.
+		return s.Connection.ObjectUpdate(s.containerName, name, headers)
+	*/
+}

swift/swift_test.go

@@ -0,0 +1,309 @@
+package swift
+
+import (
+	"bytes"
+	"fmt"
+	"github.com/ncw/swift"
+	"github.com/stretchr/testify/assert"
+	"log"
+	"os"
+	"testing"
+	"time"
+)
+
+type testFile struct {
+	name string
+	typ  string
+}
+
+func (s *Fs) prepareTestContainer() {
+	// Get all existing objects to clear the container
+	objects, err := s.Connection.ObjectsAll(s.containerName, nil)
+	if err != nil {
+		log.Fatal(err)
+	}
+	info, err := s.Connection.QueryInfo()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	var objectnames []string
+	for _, o := range objects {
+		if info.SupportsBulkDelete() {
+			objectnames = append(objectnames, o.Name)
+		} else {
+			err = s.Connection.ObjectDelete(s.containerName, o.Name)
+			if err != nil {
+				log.Fatal(err)
+			}
+			log.Println("Deleted object", o.Name)
+		}
+	}
+	// Bulk delete all in one go if the server supports it
+	if info.SupportsBulkDelete() {
+		_, err = s.Connection.BulkDelete(s.containerName, objectnames)
+		if err != nil {
+			log.Fatal(err)
+		}
+		log.Println("Deleted", objectnames)
+	}
+	// Create some files we're expecting
+	for _, newObject := range []testFile{
+		{"existingTestfile", "application/octet-stream"},
+		{"fileToDelete", "application/octet-stream"},
+		{"testfolder/", "application/directory"},
+		{"testfolder/fileintestfolder1", "application/octet-stream"},
+		{"testfolder/fileintestfolder2", "application/octet-stream"},
+		{"testfolder/subfolder/", "application/directory"},
+		{"testfolder/subfolder/fileinsubtestfolder1", "application/octet-stream"},
+		{"testfolder/subfolder/fileinsubtestfolder2", "application/octet-stream"},
+	} {
+		_, err := s.Connection.ObjectPut(s.containerName, newObject.name, bytes.NewReader([]byte{}), true, "", newObject.typ, swift.Headers{})
+		if err != nil {
+			log.Fatal(err)
+		}
+		log.Println("Created object", newObject)
+	}
+}
+
+func (s *Fs) checkExpectedContent(files []testFile) (notmatching []testFile) {
+	objects, err := s.Connection.ObjectsAll(s.containerName, nil)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// We use this map to keep track of files we already passed so we don't count them twice
+	filesfound := make(map[string]testFile)
+	// We use this map to keep track of all files we wanted, but didn't exist.
+	// When all files we want to exist exist, this map should be empty at the end of this function.
+	filesnotpassed := make(map[string]testFile)
+	for _, o := range objects {
+		for _, ob := range files {
+			filesnotpassed[ob.name] = ob
+			if o.Name == ob.name && o.ContentType == ob.typ {
+				filesfound[o.Name] = ob
+				delete(filesnotpassed, o.Name)
+				continue
+			}
+			if _, ok := filesfound[o.Name]; ok {
+				continue
+			}
+		}
+	}
+
+	for _, o := range objects {
+		if _, ok := filesfound[o.Name]; !ok {
+			delete(filesnotpassed, o.Name)
+			// Notmatching will contain all files which exist, but we didn't want to exist.
+			notmatching = append(notmatching, testFile{o.Name, o.ContentType})
+		}
+	}
+
+	// Append all files we did not found
+	for _, f := range filesnotpassed {
+		if _, ok := filesfound[f.name]; !ok {
+			notmatching = append(notmatching, f)
+		}
+	}
+
+	return
+}
+
+// Helper method to get a testinstance
+func createSwiftfsTestInstance() (fs *Fs, err error) {
+	// Maybe we can cache this somehow? (aka not creating a new connection for every test function)
+	// Also we need a good way to set up fixtures before each test.
+	fs, err = NewSwiftFs(&swift.Connection{
+		UserName: os.Getenv("SWIFT_APIUSER"),
+		ApiKey:   os.Getenv("SWIFT_APIKEY"),
+		AuthUrl:  os.Getenv("SWIFT_AUTHURL"),
+		Domain:   os.Getenv("SWIFT_DOMAIN"),
+	}, "testcontainer")
+	if err != nil {
+		return
+	}
+	fs.prepareTestContainer()
+	return
+}
+
+var testStringToWrite = "lorem ipsum"
+
+func TestFs_Name(t *testing.T) {
+	swiftfs, err := createSwiftfsTestInstance()
+	assert.NoError(t, err)
+
+	assert.Equal(t, "swiftfs", swiftfs.Name())
+}
+
+func TestFsCreate(t *testing.T) {
+	swiftfs, err := createSwiftfsTestInstance()
+	assert.NoError(t, err)
+
+	file, err := swiftfs.Create("testfile")
+	assert.NoError(t, err)
+
+	// Test writing to a created file
+	written, err := file.WriteString(testStringToWrite)
+	assert.NoError(t, err)
+	assert.Equal(t, len(testStringToWrite), written)
+
+	err = file.Close()
+	assert.NoError(t, err)
+}
+
+func TestFs_Open(t *testing.T) {
+	swiftfs, err := createSwiftfsTestInstance()
+	assert.NoError(t, err)
+
+	// Try opening a file which does not exist
+	_, err = swiftfs.Open("notExistingTestfile")
+	assert.Error(t, err)
+
+	// Open the real file
+	file, err := swiftfs.Open("existingTestfile") // This file needs to exist previously
+	assert.NoError(t, err)
+
+	// Test writing to an open file
+	written, err := file.WriteString(testStringToWrite)
+	assert.NoError(t, err)
+	assert.Equal(t, len(testStringToWrite), written)
+
+	err = file.Close()
+	assert.NoError(t, err)
+}
+
+func TestFs_Mkdir(t *testing.T) {
+	swiftfs, err := createSwiftfsTestInstance()
+	assert.NoError(t, err)
+
+	err = swiftfs.Mkdir("testdir", os.ModePerm)
+	assert.NoError(t, err)
+}
+
+// MkdirAll should do exactly the same
+func TestFs_MkdirAll(t *testing.T) {
+	swiftfs, err := createSwiftfsTestInstance()
+	assert.NoError(t, err)
+
+	err = swiftfs.MkdirAll("testdir", os.ModePerm)
+	assert.NoError(t, err)
+}
+
+func TestFs_Remove(t *testing.T) {
+	swiftfs, err := createSwiftfsTestInstance()
+	assert.NoError(t, err)
+
+	err = swiftfs.Remove("fileToDelete")
+	assert.NoError(t, err)
+
+	// Testing removing a second time should return an error as that file does not exist now
+	err = swiftfs.Remove("fileToDelete")
+	assert.Error(t, err)
+}
+
+func TestFs_RemoveAll(t *testing.T) {
+	swiftfs, err := createSwiftfsTestInstance()
+	assert.NoError(t, err)
+
+	err = swiftfs.RemoveAll("testfolder")
+	assert.NoError(t, err)
+	notmatching := swiftfs.checkExpectedContent([]testFile{
+		{"existingTestfile", "application/octet-stream"},
+		{"fileToDelete", "application/octet-stream"},
+	})
+	assert.Len(t, notmatching, 0)
+
+	// Deleting something with removeAll should return nil if it does not exist
+	err = swiftfs.RemoveAll("testfolder")
+	assert.NoError(t, err)
+}
+
+func TestFs_Rename(t *testing.T) {
+	swiftfs, err := createSwiftfsTestInstance()
+	assert.NoError(t, err)
+
+	err = swiftfs.Rename("existingTestfile", "otherExistingTestfile")
+	assert.NoError(t, err)
+	notmatching := swiftfs.checkExpectedContent([]testFile{
+		{"otherExistingTestfile", "application/octet-stream"},
+		{"fileToDelete", "application/octet-stream"},
+		{"testfolder/", "application/directory"},
+		{"testfolder/fileintestfolder1", "application/octet-stream"},
+		{"testfolder/fileintestfolder2", "application/octet-stream"},
+		{"testfolder/subfolder/", "application/directory"},
+		{"testfolder/subfolder/fileinsubtestfolder1", "application/octet-stream"},
+		{"testfolder/subfolder/fileinsubtestfolder2", "application/octet-stream"},
+	})
+	assert.Len(t, notmatching, 0)
+
+	// Moving a nonexisting file should not work
+	err = swiftfs.Rename("existingTestfile", "otherExistingTestfile")
+	assert.Error(t, err)
+}
+
+func TestFs_Stat(t *testing.T) {
+	swiftfs, err := createSwiftfsTestInstance()
+	assert.NoError(t, err)
+
+	finfo, err := swiftfs.Stat("existingTestfile")
+	assert.NoError(t, err)
+	assert.Equal(t, "existingTestfile", finfo.Name())
+	assert.Equal(t, int64(0), finfo.Size())
+	assert.Equal(t, os.ModePerm, finfo.Mode())
+	assert.False(t, finfo.IsDir())
+	assert.Equal(t, finfo, finfo.Sys())
+}
+
+func TestFs_Chmod(t *testing.T) {
+	swiftfs, err := createSwiftfsTestInstance()
+	assert.NoError(t, err)
+
+	err = swiftfs.Chmod("existingTestfile", os.ModePerm)
+	assert.NoError(t, err)
+}
+
+func TestFs_Chtimes(t *testing.T) {
+	swiftfs, err := createSwiftfsTestInstance()
+	assert.NoError(t, err)
+
+	filename := "existingTestfile"
+	newtime := time.Date(2019, time.January, 31, 14, 0, 0, 0, time.UTC)
+	err = swiftfs.Chtimes(filename, newtime, newtime)
+	assert.NoError(t, err)
+
+	// See the function on why this check is diabled.
+	/*
+		file, err := swiftfs.Open(filename)
+		assert.NoError(t, err)
+		finfo, err := file.Stat()
+		assert.NoError(t, err)
+		assert.True(t, newtime.Equal(finfo.ModTime()))
+	*/
+}
+
+func TestReaddir(t *testing.T) {
+	swiftfs, err := createSwiftfsTestInstance()
+	assert.NoError(t, err)
+
+	root, err := swiftfs.Open("testfolder/")
+	assert.NoError(t, err)
+
+	folders, err := root.Readdir(10) // Random number
+	assert.NoError(t, err)
+
+	numfolders := 2
+	for _, f := range folders {
+		if f == nil {
+			continue
+		}
+		if f.IsDir() {
+			numfolders--
+			continue
+		}
+
+		fmt.Printf("%s is not a folder", f.Name())
+		t.FailNow()
+	}
+
+	assert.Equal(t, 0, numfolders)
+}