Added Support for openstack swift object storage

This commit is contained in:
kolaente 2020-09-26 12:38:44 +02:00
parent 450b30f2bf
commit 9be4eec9c9
No known key found for this signature in database
GPG Key ID: F40E70337AB24C9B
4 changed files with 707 additions and 0 deletions

2
go.mod
View File

@ -3,7 +3,9 @@ module github.com/spf13/afero
require (
cloud.google.com/go/storage v1.14.0
github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8
github.com/ncw/swift v1.0.47
github.com/pkg/sftp v1.13.1
github.com/stretchr/testify v1.4.0
golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa
golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99
golang.org/x/text v0.3.4

254
swift/file.go Normal file
View File

@ -0,0 +1,254 @@
package swift
import (
"bytes"
"fmt"
"github.com/ncw/swift"
"io"
"log"
"os"
"path/filepath"
"time"
)
type File struct {
objectCreateFile *swift.ObjectCreateFile
objectOpenFile *swift.ObjectOpenFile
tmpContent []byte
fileInfo *FileInfo
seekStart int64
conn *swift.Connection // The connection to be able to do things like updating the file. This is probably not the best way to do it.
containerName string
}
func (file *File) Close() error {
if file.objectCreateFile != nil {
return file.objectCreateFile.Close()
}
if file.objectOpenFile != nil {
return file.objectOpenFile.Close()
}
log.Println("nothing to close")
return nil
}
func (file *File) Read(p []byte) (n int, err error) {
return file.ReadAt(p, file.seekStart)
}
func (file *File) ReadAt(p []byte, off int64) (n int, err error) {
if file.objectOpenFile != nil {
return file.objectOpenFile.Read(p)
}
if file.tmpContent != nil {
// If we have an offset > 0, we just strip the first offset n bytes from the
// input and put that into p
if off > 0 {
file.tmpContent = file.tmpContent[off:]
}
if len(file.tmpContent) > len(p) {
return 0, io.EOF
}
return copy(p, file.tmpContent), nil
}
return 0, fmt.Errorf("nothing to read")
}
func (file *File) Seek(offset int64, whence int) (int64, error) {
if file.tmpContent == nil {
return 0, fmt.Errorf("no open file")
}
switch whence {
case io.SeekStart:
file.seekStart = offset
case io.SeekCurrent:
file.seekStart += offset
case io.SeekEnd:
file.seekStart = int64(len(file.tmpContent)) - offset
default:
return 0, fmt.Errorf("unkown whence")
}
return file.seekStart, nil
}
func (file *File) Write(p []byte) (n int, err error) {
return file.WriteAt(p, file.seekStart)
}
func (file *File) WriteAt(p []byte, off int64) (n int, err error) {
if file.objectCreateFile != nil {
return file.objectCreateFile.Write(p)
}
// If offset is > 0, we need to modify our p slightly and add the len(off) bytes to p before it, since it is not
// possible to write to a swift file with an offset
if off > 0 {
tmp := make([]byte, off)
_, err := file.Read(tmp)
if err != nil {
return 0, err
}
p = append(tmp, p...)
}
// If we have an open file, write to it.
if file.objectOpenFile != nil {
_, err = file.conn.ObjectPut(file.containerName, file.Name(), bytes.NewReader(p), true, "", "", swift.Headers{})
return int(len(p)), err // Apparently swift does not return the length it wrote. So we just return the size to not break anything.
}
// If we're at this point, that means no other file type thing (like objectCreateFile) exists.
// This usually happens when we want to get a file and then get its content.
// In that case put that content in here temporarily so we can access it later.
file.tmpContent = p
return len(p), nil
}
func (file *File) Name() string {
return file.fileInfo.Name()
}
func (file *File) Readdir(count int) (res []os.FileInfo, err error) {
if !file.fileInfo.IsDir() {
return nil, fmt.Errorf("%s is not a dir", file.fileInfo.Name())
}
files, err := file.conn.ObjectsAll(file.containerName, &swift.ObjectsOpts{
Limit: count,
Headers: map[string]string{"Content-Type": "application/directory"},
})
if err != nil {
return nil, err
}
// This could probably be optimized slightly to consume less memory
res = make([]os.FileInfo, count)
for i, f := range files {
// Sometimes filtering directly in the request does not work, so we need to manually filder here
if f.ContentType != "application/directory" {
continue
}
res[i] = FileInfo{
name: f.Name,
size: f.Bytes,
modTime: f.LastModified,
mode: os.ModeDir, // At this point we can be sure this is always a folder
}
}
return res, nil
}
func (file *File) Readdirnames(n int) (names []string, err error) {
infs, err := file.Readdir(n)
if err != nil {
return
}
names = make([]string, len(infs))
for i, f := range infs {
_, names[i] = filepath.Split(f.Name())
}
return
}
func (file *File) Stat() (os.FileInfo, error) {
return file.fileInfo, nil
}
func (file *File) Sync() error {
return nil
}
func (file *File) Truncate(size int64) error {
_, err := file.conn.ObjectPut(file.containerName, file.Name(), bytes.NewReader([]byte{}), true, "", "", swift.Headers{})
return err
}
func (file *File) WriteString(s string) (ret int, err error) {
return file.Write([]byte(s))
}
func (file *File) putFileInfoTogetherFromOpenObject(f *swift.ObjectOpenFile, headers swift.Headers, name string) (err error) {
file.objectOpenFile = f
// Put the file info together
size, err := f.Length()
if err != nil {
return
}
var modtime time.Time
modtime, err = headers.ObjectMetadata().GetModTime()
if err != nil {
// Try parsing it directly if the method fails
modtime, err = time.Parse("Mon, 02 Jan 2006 15:04:05 MST", headers["Last-Modified"])
if err == nil {
goto putFileInfoTogether
}
return err
}
putFileInfoTogether:
var mode os.FileMode = os.ModePerm
if headers["Content-Type"] == "application/directory" {
mode = os.ModeDir
}
file.fileInfo = &FileInfo{
name: name,
size: size,
modTime: modtime,
mode: mode, // Apparently there is no way to get the mode of an object. Or am I missing something?
}
return nil
}
// =====================
// File Info starts here
// =====================
type FileInfo struct {
name string
size int64
mode os.FileMode
modTime time.Time
}
func (fi FileInfo) Name() string {
return fi.name
}
func (fi FileInfo) Size() int64 {
if fi.IsDir() {
return int64(42)
}
return fi.size
}
func (fi FileInfo) Mode() os.FileMode {
return fi.mode
}
func (fi FileInfo) ModTime() time.Time {
return fi.modTime
}
func (fi FileInfo) IsDir() bool {
return fi.Mode().IsDir()
}
func (fi FileInfo) Sys() interface{} {
return &fi
}

142
swift/swift.go Normal file
View File

@ -0,0 +1,142 @@
package swift
import (
"bytes"
"fmt"
"github.com/ncw/swift"
"log"
"os"
"time"
)
type Fs struct {
Connection *swift.Connection
containerName string
}
func NewSwiftFs(conn *swift.Connection, containerName string) (fs *Fs, err error) {
if containerName == "" {
return fs, fmt.Errorf("container name cannot be empty")
}
fs = &Fs{
Connection: conn,
containerName: containerName,
}
err = fs.Connection.Authenticate()
return
}
func (s *Fs) Name() string {
return "swiftfs"
}
func (s *Fs) Create(name string) (file File, err error) {
objectfile, err := s.Connection.ObjectCreate(s.containerName, name, true, "", "", swift.Headers{})
if err != nil {
return file, err
}
file.objectCreateFile = objectfile
return
}
func (s *Fs) Mkdir(name string, perm os.FileMode) (err error) {
return s.MkdirAll(name, perm)
}
func (s *Fs) MkdirAll(path string, perm os.FileMode) (err error) {
// Swift usually creates all directories deep down, no matter if any of these directories already exist
// Objects created with application/directory are so called "pseudo-directories".
_, err = s.Connection.ObjectPut(s.containerName, path+"/", bytes.NewReader([]byte{}), true, "", "application/directory", swift.Headers{})
return
}
func (s *Fs) Open(name string) (file *File, err error) {
file = new(File)
file.conn = s.Connection
file.containerName = s.containerName
f, headers, err := s.Connection.ObjectOpen(s.containerName, name, true, swift.Headers{})
if err != nil {
return
}
err = file.putFileInfoTogetherFromOpenObject(f, headers, name)
return
}
func (*Fs) OpenFile(name string, flag int, perm os.FileMode) (*File, error) {
return nil, nil
}
func (s *Fs) Remove(name string) error {
return s.Connection.ObjectDelete(s.containerName, name)
}
func (s *Fs) RemoveAll(path string) (err error) {
sinfo, err := s.Connection.QueryInfo()
if err != nil {
return
}
// Get all objects we want to delte
objects, err := s.Connection.ObjectsAll(s.containerName, &swift.ObjectsOpts{Prefix: path})
if err != nil {
return
}
var objectnames []string
for _, o := range objects {
// If the server supports bulk delete, add it to a list to bulkdelete the items later,
// otherwise delete them directly
if sinfo.SupportsBulkDelete() {
objectnames = append(objectnames, o.Name)
} else {
err = s.Connection.ObjectDelete(s.containerName, o.Name)
if err != nil {
return
}
}
}
if sinfo.SupportsBulkDelete() {
_, err = s.Connection.BulkDelete(s.containerName, objectnames)
if err != nil {
return
}
}
return err
}
func (s *Fs) Rename(oldname, newname string) error {
return s.Connection.ObjectMove(s.containerName, oldname, s.containerName, newname)
}
func (s *Fs) Stat(name string) (os.FileInfo, error) {
file, err := s.Open(name)
if err != nil {
return nil, err
}
return file.Stat()
}
func (*Fs) Chmod(name string, mode os.FileMode) error {
log.Println("Swift does not support file modes.")
return nil
}
func (s *Fs) Chtimes(name string, _, mtime time.Time) error {
// While the uncommented code should work, the problem is that we have no way to update the time on
// the server without modifying the time. Whenever we send a request to swift to update the last modified time,
// we modify the file thus setting the last modified time to now...
// Maybe I'll find a solution for this in the future, but for now I'll just diable it because of the reasons above.
return nil
/*
_, headers, err := s.Connection.ObjectOpen(s.containerName, name, true, swift.Headers{})
if err != nil {
return err
}
headers.ObjectMetadata().SetModTime(mtime) // Swift does not support access time so we won't use that.
return s.Connection.ObjectUpdate(s.containerName, name, headers)
*/
}

309
swift/swift_test.go Normal file
View File

@ -0,0 +1,309 @@
package swift
import (
"bytes"
"fmt"
"github.com/ncw/swift"
"github.com/stretchr/testify/assert"
"log"
"os"
"testing"
"time"
)
type testFile struct {
name string
typ string
}
func (s *Fs) prepareTestContainer() {
// Get all existing objects to clear the container
objects, err := s.Connection.ObjectsAll(s.containerName, nil)
if err != nil {
log.Fatal(err)
}
info, err := s.Connection.QueryInfo()
if err != nil {
log.Fatal(err)
}
var objectnames []string
for _, o := range objects {
if info.SupportsBulkDelete() {
objectnames = append(objectnames, o.Name)
} else {
err = s.Connection.ObjectDelete(s.containerName, o.Name)
if err != nil {
log.Fatal(err)
}
log.Println("Deleted object", o.Name)
}
}
// Bulk delete all in one go if the server supports it
if info.SupportsBulkDelete() {
_, err = s.Connection.BulkDelete(s.containerName, objectnames)
if err != nil {
log.Fatal(err)
}
log.Println("Deleted", objectnames)
}
// Create some files we're expecting
for _, newObject := range []testFile{
{"existingTestfile", "application/octet-stream"},
{"fileToDelete", "application/octet-stream"},
{"testfolder/", "application/directory"},
{"testfolder/fileintestfolder1", "application/octet-stream"},
{"testfolder/fileintestfolder2", "application/octet-stream"},
{"testfolder/subfolder/", "application/directory"},
{"testfolder/subfolder/fileinsubtestfolder1", "application/octet-stream"},
{"testfolder/subfolder/fileinsubtestfolder2", "application/octet-stream"},
} {
_, err := s.Connection.ObjectPut(s.containerName, newObject.name, bytes.NewReader([]byte{}), true, "", newObject.typ, swift.Headers{})
if err != nil {
log.Fatal(err)
}
log.Println("Created object", newObject)
}
}
func (s *Fs) checkExpectedContent(files []testFile) (notmatching []testFile) {
objects, err := s.Connection.ObjectsAll(s.containerName, nil)
if err != nil {
log.Fatal(err)
}
// We use this map to keep track of files we already passed so we don't count them twice
filesfound := make(map[string]testFile)
// We use this map to keep track of all files we wanted, but didn't exist.
// When all files we want to exist exist, this map should be empty at the end of this function.
filesnotpassed := make(map[string]testFile)
for _, o := range objects {
for _, ob := range files {
filesnotpassed[ob.name] = ob
if o.Name == ob.name && o.ContentType == ob.typ {
filesfound[o.Name] = ob
delete(filesnotpassed, o.Name)
continue
}
if _, ok := filesfound[o.Name]; ok {
continue
}
}
}
for _, o := range objects {
if _, ok := filesfound[o.Name]; !ok {
delete(filesnotpassed, o.Name)
// Notmatching will contain all files which exist, but we didn't want to exist.
notmatching = append(notmatching, testFile{o.Name, o.ContentType})
}
}
// Append all files we did not found
for _, f := range filesnotpassed {
if _, ok := filesfound[f.name]; !ok {
notmatching = append(notmatching, f)
}
}
return
}
// Helper method to get a testinstance
func createSwiftfsTestInstance() (fs *Fs, err error) {
// Maybe we can cache this somehow? (aka not creating a new connection for every test function)
// Also we need a good way to set up fixtures before each test.
fs, err = NewSwiftFs(&swift.Connection{
UserName: os.Getenv("SWIFT_APIUSER"),
ApiKey: os.Getenv("SWIFT_APIKEY"),
AuthUrl: os.Getenv("SWIFT_AUTHURL"),
Domain: os.Getenv("SWIFT_DOMAIN"),
}, "testcontainer")
if err != nil {
return
}
fs.prepareTestContainer()
return
}
var testStringToWrite = "lorem ipsum"
func TestFs_Name(t *testing.T) {
swiftfs, err := createSwiftfsTestInstance()
assert.NoError(t, err)
assert.Equal(t, "swiftfs", swiftfs.Name())
}
func TestFsCreate(t *testing.T) {
swiftfs, err := createSwiftfsTestInstance()
assert.NoError(t, err)
file, err := swiftfs.Create("testfile")
assert.NoError(t, err)
// Test writing to a created file
written, err := file.WriteString(testStringToWrite)
assert.NoError(t, err)
assert.Equal(t, len(testStringToWrite), written)
err = file.Close()
assert.NoError(t, err)
}
func TestFs_Open(t *testing.T) {
swiftfs, err := createSwiftfsTestInstance()
assert.NoError(t, err)
// Try opening a file which does not exist
_, err = swiftfs.Open("notExistingTestfile")
assert.Error(t, err)
// Open the real file
file, err := swiftfs.Open("existingTestfile") // This file needs to exist previously
assert.NoError(t, err)
// Test writing to an open file
written, err := file.WriteString(testStringToWrite)
assert.NoError(t, err)
assert.Equal(t, len(testStringToWrite), written)
err = file.Close()
assert.NoError(t, err)
}
func TestFs_Mkdir(t *testing.T) {
swiftfs, err := createSwiftfsTestInstance()
assert.NoError(t, err)
err = swiftfs.Mkdir("testdir", os.ModePerm)
assert.NoError(t, err)
}
// MkdirAll should do exactly the same
func TestFs_MkdirAll(t *testing.T) {
swiftfs, err := createSwiftfsTestInstance()
assert.NoError(t, err)
err = swiftfs.MkdirAll("testdir", os.ModePerm)
assert.NoError(t, err)
}
func TestFs_Remove(t *testing.T) {
swiftfs, err := createSwiftfsTestInstance()
assert.NoError(t, err)
err = swiftfs.Remove("fileToDelete")
assert.NoError(t, err)
// Testing removing a second time should return an error as that file does not exist now
err = swiftfs.Remove("fileToDelete")
assert.Error(t, err)
}
func TestFs_RemoveAll(t *testing.T) {
swiftfs, err := createSwiftfsTestInstance()
assert.NoError(t, err)
err = swiftfs.RemoveAll("testfolder")
assert.NoError(t, err)
notmatching := swiftfs.checkExpectedContent([]testFile{
{"existingTestfile", "application/octet-stream"},
{"fileToDelete", "application/octet-stream"},
})
assert.Len(t, notmatching, 0)
// Deleting something with removeAll should return nil if it does not exist
err = swiftfs.RemoveAll("testfolder")
assert.NoError(t, err)
}
func TestFs_Rename(t *testing.T) {
swiftfs, err := createSwiftfsTestInstance()
assert.NoError(t, err)
err = swiftfs.Rename("existingTestfile", "otherExistingTestfile")
assert.NoError(t, err)
notmatching := swiftfs.checkExpectedContent([]testFile{
{"otherExistingTestfile", "application/octet-stream"},
{"fileToDelete", "application/octet-stream"},
{"testfolder/", "application/directory"},
{"testfolder/fileintestfolder1", "application/octet-stream"},
{"testfolder/fileintestfolder2", "application/octet-stream"},
{"testfolder/subfolder/", "application/directory"},
{"testfolder/subfolder/fileinsubtestfolder1", "application/octet-stream"},
{"testfolder/subfolder/fileinsubtestfolder2", "application/octet-stream"},
})
assert.Len(t, notmatching, 0)
// Moving a nonexisting file should not work
err = swiftfs.Rename("existingTestfile", "otherExistingTestfile")
assert.Error(t, err)
}
func TestFs_Stat(t *testing.T) {
swiftfs, err := createSwiftfsTestInstance()
assert.NoError(t, err)
finfo, err := swiftfs.Stat("existingTestfile")
assert.NoError(t, err)
assert.Equal(t, "existingTestfile", finfo.Name())
assert.Equal(t, int64(0), finfo.Size())
assert.Equal(t, os.ModePerm, finfo.Mode())
assert.False(t, finfo.IsDir())
assert.Equal(t, finfo, finfo.Sys())
}
func TestFs_Chmod(t *testing.T) {
swiftfs, err := createSwiftfsTestInstance()
assert.NoError(t, err)
err = swiftfs.Chmod("existingTestfile", os.ModePerm)
assert.NoError(t, err)
}
func TestFs_Chtimes(t *testing.T) {
swiftfs, err := createSwiftfsTestInstance()
assert.NoError(t, err)
filename := "existingTestfile"
newtime := time.Date(2019, time.January, 31, 14, 0, 0, 0, time.UTC)
err = swiftfs.Chtimes(filename, newtime, newtime)
assert.NoError(t, err)
// See the function on why this check is diabled.
/*
file, err := swiftfs.Open(filename)
assert.NoError(t, err)
finfo, err := file.Stat()
assert.NoError(t, err)
assert.True(t, newtime.Equal(finfo.ModTime()))
*/
}
func TestReaddir(t *testing.T) {
swiftfs, err := createSwiftfsTestInstance()
assert.NoError(t, err)
root, err := swiftfs.Open("testfolder/")
assert.NoError(t, err)
folders, err := root.Readdir(10) // Random number
assert.NoError(t, err)
numfolders := 2
for _, f := range folders {
if f == nil {
continue
}
if f.IsDir() {
numfolders--
continue
}
fmt.Printf("%s is not a folder", f.Name())
t.FailNow()
}
assert.Equal(t, 0, numfolders)
}