From 90ff561ddabb8a8684af99c06930ab0caba7f51f Mon Sep 17 00:00:00 2001 From: Jerry Jacobs Date: Sat, 26 Dec 2015 20:36:25 +0100 Subject: [PATCH 1/6] Add SftpFs beta backend --- README.md | 7 ++- sftp.go | 128 +++++++++++++++++++++++++++++++++++++++++++++++++++ sftp/file.go | 95 ++++++++++++++++++++++++++++++++++++++ sftp_test.go | 92 ++++++++++++++++++++++++++++++++++++ 4 files changed, 321 insertions(+), 1 deletion(-) create mode 100644 sftp.go create mode 100644 sftp/file.go create mode 100644 sftp_test.go diff --git a/README.md b/README.md index 3a624ea..1309153 100644 --- a/README.md +++ b/README.md @@ -263,12 +263,17 @@ backed file implementation. This can be used in other memory backed file systems with ease. Plans are to add a radix tree memory stored file system using InMemoryFile. +## SftpFs + +Afero has experimental support for secure file transfer protocol (sftp). Which can +be used to perform file operations over a encrypted channel. + ## Desired/possible backends The following is a short list of possible backends we hope someone will implement: -* SSH/SCP +* SSH * ZIP * TAR * S3 diff --git a/sftp.go b/sftp.go new file mode 100644 index 0000000..a095a54 --- /dev/null +++ b/sftp.go @@ -0,0 +1,128 @@ +// Copyright © 2015 Jerry Jacobs . +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package afero + +import ( + "os" + "time" + + "github.com/spf13/afero/sftp" + + "github.com/pkg/sftp" +) + +// SftpFs is a Fs implementation that uses functions provided by the sftp package. +// +// For details in any method, check the documentation of the sftp package +// (github.com/pkg/sftp). +type SftpFs struct{ + SftpClient *sftp.Client +} + +func (s SftpFs) Name() string { return "SftpFs" } + +func (s SftpFs) Create(name string) (File, error) { + f, err := sftpfs.FileCreate(s.SftpClient, name) + return f, err +} + +func (s SftpFs) Mkdir(name string, perm os.FileMode) error { + err := s.SftpClient.Mkdir(name) + if err != nil { + return err + } + return s.SftpClient.Chmod(name, perm) +} + +func (s SftpFs) MkdirAll(path string, perm os.FileMode) error { + // Fast path: if we can tell whether path is a directory or file, stop with success or error. + dir, err := s.Stat(path) + if err == nil { + if dir.IsDir() { + return nil + } + return err + } + + // Slow path: make sure parent exists and then call Mkdir for path. + i := len(path) + for i > 0 && os.IsPathSeparator(path[i-1]) { // Skip trailing path separator. + i-- + } + + j := i + for j > 0 && !os.IsPathSeparator(path[j-1]) { // Scan backward over element. + j-- + } + + if j > 1 { + // Create parent + err = s.MkdirAll(path[0:j-1], perm) + if err != nil { + return err + } + } + + // Parent now exists; invoke Mkdir and use its result. + err = s.Mkdir(path, perm) + if err != nil { + // Handle arguments like "foo/." by + // double-checking that directory doesn't exist. + dir, err1 := s.Lstat(path) + if err1 == nil && dir.IsDir() { + return nil + } + return err + } + return nil +} + +func (s SftpFs) Open(name string) (File, error) { + f, err := sftpfs.FileOpen(s.SftpClient, name) + return f, err +} + +func (s SftpFs) OpenFile(name string, flag int, perm os.FileMode) (File, error) { + return nil,nil +} + +func (s SftpFs) Remove(name string) error { + return s.SftpClient.Remove(name) +} + +func (s SftpFs) RemoveAll(path string) error { + // TODO have a look at os.RemoveAll + // https://github.com/golang/go/blob/master/src/os/path.go#L66 + return nil +} + +func (s SftpFs) Rename(oldname, newname string) error { + return s.SftpClient.Rename(oldname, newname) +} + +func (s SftpFs) Stat(name string) (os.FileInfo, error) { + return s.SftpClient.Stat(name) +} + +func (s SftpFs) Lstat(p string) (os.FileInfo, error) { + return s.SftpClient.Lstat(p) +} + +func (s SftpFs) Chmod(name string, mode os.FileMode) error { + return s.SftpClient.Chmod(name, mode) +} + +func (s SftpFs) Chtimes(name string, atime time.Time, mtime time.Time) error { + return s.SftpClient.Chtimes(name, atime, mtime) +} diff --git a/sftp/file.go b/sftp/file.go new file mode 100644 index 0000000..bab4abc --- /dev/null +++ b/sftp/file.go @@ -0,0 +1,95 @@ +// Copyright © 2015 Jerry Jacobs . +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sftpfs + +import ( + "os" + "github.com/pkg/sftp" +) + +type File struct { + fd *sftp.File +} + +func FileOpen(s *sftp.Client, name string) (*File, error) { + fd, err := s.Open(name) + if err != nil { + return &File{}, err + } + return &File{fd: fd}, nil +} + +func FileCreate(s *sftp.Client, name string) (*File, error) { + fd, err := s.Create(name) + if err != nil { + return &File{}, err + } + return &File{fd: fd}, nil +} + +func (f *File) Close() error { + return f.fd.Close() +} + +func (f *File) Name() string { + return f.fd.Name() +} + +func (f *File) Stat() (os.FileInfo, error) { + return f.fd.Stat() +} + +func (f *File) Sync() error { + return nil +} + +func (f *File) Truncate(size int64) error { + return f.fd.Truncate(size) +} + +func (f *File) Read(b []byte) (n int, err error) { + return f.fd.Read(b) +} + +// TODO +func (f *File) ReadAt(b []byte, off int64) (n int, err error) { + return 0,nil +} + +// TODO +func (f *File) Readdir(count int) (res []os.FileInfo, err error) { + return nil,nil +} + +// TODO +func (f *File) Readdirnames(n int) (names []string, err error) { + return nil,nil +} + +func (f *File) Seek(offset int64, whence int) (int64, error) { + return f.fd.Seek(offset, whence) +} + +func (f *File) Write(b []byte) (n int, err error) { + return f.fd.Write(b) +} + +// TODO +func (f *File) WriteAt(b []byte, off int64) (n int, err error) { + return 0,nil +} + +func (f *File) WriteString(s string) (ret int, err error) { + return f.fd.Write([]byte(s)) +} diff --git a/sftp_test.go b/sftp_test.go new file mode 100644 index 0000000..49bdda8 --- /dev/null +++ b/sftp_test.go @@ -0,0 +1,92 @@ +// Copyright © 2015 Jerry Jacobs . +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package afero + +import ( + "os" + "io/ioutil" + "testing" + + "github.com/xor-gate/afero" + + "golang.org/x/crypto/ssh" + "github.com/pkg/sftp" +) + +type SftpFsContext struct { + sshc *ssh.Client + sshcfg *ssh.ClientConfig + sftpc *sftp.Client +} + +func SftpConnect(user string, host string) (*SftpFsContext, error) { + pemBytes, err := ioutil.ReadFile(os.Getenv("HOME") + "/.ssh/id_rsa") + if err != nil { + return nil,err + } + + signer, err := ssh.ParsePrivateKey(pemBytes) + if err != nil { + return nil,err + } + + sshcfg := &ssh.ClientConfig{ + User: user, + Auth: []ssh.AuthMethod{ + ssh.PublicKeys(signer), + }, + } + + sshc, err := ssh.Dial("tcp", host, sshcfg) + if err != nil { + return nil,err + } + + sftpc, err := sftp.NewClient(sshc) + if err != nil { + return nil,err + } + + ctx := &SftpFsContext{ + sshc: sshc, + sshcfg: sshcfg, + sftpc: sftpc, + } + + return ctx,nil +} + +func (ctx *SftpFsContext) Disconnect() error { + ctx.sftpc.Close() + ctx.sshc.Close() + return nil +} + +func TestSftpCreate(t *testing.T) { + ctx, err := SftpConnect("user", "host:port") + if err != nil { + t.Fatal(err) + } + defer ctx.Disconnect() + + var AppFs afero.Fs = afero.SftpFs{ + SftpClient: ctx.sftpc, + } + + file, err := AppFs.Create("aferoSftpFsTestFile") + if err != nil { + t.Error(err) + } + defer file.Close() +} From 3f46b4f31fe22c9b572145aa94974bc01a0ca2ed Mon Sep 17 00:00:00 2001 From: Jerry Jacobs Date: Sat, 26 Dec 2015 20:43:55 +0100 Subject: [PATCH 2/6] sftp_test.go: Fixup import path --- sftp_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sftp_test.go b/sftp_test.go index 49bdda8..8c48652 100644 --- a/sftp_test.go +++ b/sftp_test.go @@ -18,7 +18,7 @@ import ( "io/ioutil" "testing" - "github.com/xor-gate/afero" + "github.com/spf13/afero" "golang.org/x/crypto/ssh" "github.com/pkg/sftp" From 7d5dacc912daf4125151430ab72d7febf738dc18 Mon Sep 17 00:00:00 2001 From: Jerry Jacobs Date: Sat, 2 Jan 2016 14:00:20 +0100 Subject: [PATCH 3/6] * Generate RSA key and spawn SFTP server for test --- sftp_test.go | 198 ++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 190 insertions(+), 8 deletions(-) diff --git a/sftp_test.go b/sftp_test.go index 8c48652..839a19d 100644 --- a/sftp_test.go +++ b/sftp_test.go @@ -14,11 +14,18 @@ package afero import ( - "os" - "io/ioutil" "testing" - - "github.com/spf13/afero" + "os" + "log" + "fmt" + "net" + "flag" + "time" + "io/ioutil" + "crypto/rsa" + _rand "crypto/rand" + "encoding/pem" + "crypto/x509" "golang.org/x/crypto/ssh" "github.com/pkg/sftp" @@ -30,7 +37,7 @@ type SftpFsContext struct { sftpc *sftp.Client } -func SftpConnect(user string, host string) (*SftpFsContext, error) { +func SftpConnect(user, password, host string) (*SftpFsContext, error) { pemBytes, err := ioutil.ReadFile(os.Getenv("HOME") + "/.ssh/id_rsa") if err != nil { return nil,err @@ -44,6 +51,7 @@ func SftpConnect(user string, host string) (*SftpFsContext, error) { sshcfg := &ssh.ClientConfig{ User: user, Auth: []ssh.AuthMethod{ + ssh.Password(password), ssh.PublicKeys(signer), }, } @@ -73,20 +81,194 @@ func (ctx *SftpFsContext) Disconnect() error { return nil } +func RunSftpServer() { + var ( + readOnly bool + debugLevelStr string + debugLevel int + debugStderr bool + rootDir string + ) + + flag.BoolVar(&readOnly, "R", false, "read-only server") + flag.BoolVar(&debugStderr, "e", false, "debug to stderr") + flag.StringVar(&debugLevelStr, "l", "none", "debug level") + flag.StringVar(&rootDir, "root", ".", "root directory") + flag.Parse() + + debugStream := ioutil.Discard + if debugStderr { + debugStream = os.Stderr + debugLevel = 1 + } + + // An SSH server is represented by a ServerConfig, which holds + // certificate details and handles authentication of ServerConns. + config := &ssh.ServerConfig{ + PasswordCallback: func(c ssh.ConnMetadata, pass []byte) (*ssh.Permissions, error) { + // Should use constant-time compare (or better, salt+hash) in + // a production setting. + fmt.Fprintf(debugStream, "Login: %s\n", c.User()) + if c.User() == "test" && string(pass) == "test" { + return nil, nil + } + return nil, fmt.Errorf("password rejected for %q", c.User()) + }, + } + + privateBytes, err := ioutil.ReadFile("./test/id_rsa") + if err != nil { + log.Fatal("Failed to load private key", err) + } + + private, err := ssh.ParsePrivateKey(privateBytes) + if err != nil { + log.Fatal("Failed to parse private key", err) + } + + config.AddHostKey(private) + + // Once a ServerConfig has been configured, connections can be + // accepted. + listener, err := net.Listen("tcp", "0.0.0.0:2022") + if err != nil { + log.Fatal("failed to listen for connection", err) + } + fmt.Printf("Listening on %v\n", listener.Addr()) + + nConn, err := listener.Accept() + if err != nil { + log.Fatal("failed to accept incoming connection", err) + } + + // Before use, a handshake must be performed on the incoming + // net.Conn. + _, chans, reqs, err := ssh.NewServerConn(nConn, config) + if err != nil { + log.Fatal("failed to handshake", err) + } + fmt.Fprintf(debugStream, "SSH server established\n") + + // The incoming Request channel must be serviced. + go ssh.DiscardRequests(reqs) + + // Service the incoming Channel channel. + for newChannel := range chans { + // Channels have a type, depending on the application level + // protocol intended. In the case of an SFTP session, this is "subsystem" + // with a payload string of "sftp" + fmt.Fprintf(debugStream, "Incoming channel: %s\n", newChannel.ChannelType()) + if newChannel.ChannelType() != "session" { + newChannel.Reject(ssh.UnknownChannelType, "unknown channel type") + fmt.Fprintf(debugStream, "Unknown channel type: %s\n", newChannel.ChannelType()) + continue + } + channel, requests, err := newChannel.Accept() + if err != nil { + log.Fatal("could not accept channel.", err) + } + fmt.Fprintf(debugStream, "Channel accepted\n") + + // Sessions have out-of-band requests such as "shell", + // "pty-req" and "env". Here we handle only the + // "subsystem" request. + go func(in <-chan *ssh.Request) { + for req := range in { + fmt.Fprintf(debugStream, "Request: %v\n", req.Type) + ok := false + switch req.Type { + case "subsystem": + fmt.Fprintf(debugStream, "Subsystem: %s\n", req.Payload[4:]) + if string(req.Payload[4:]) == "sftp" { + ok = true + } + } + fmt.Fprintf(debugStream, " - accepted: %v\n", ok) + req.Reply(ok, nil) + } + }(requests) + + server, err := sftp.NewServer(channel, channel, debugStream, debugLevel, readOnly, rootDir) + if err != nil { + log.Fatal(err) + } + if err := server.Serve(); err != nil { + log.Fatal("sftp server completed with error:", err) + } + } +} + + +// MakeSSHKeyPair make a pair of public and private keys for SSH access. +// Public key is encoded in the format for inclusion in an OpenSSH authorized_keys file. +// Private Key generated is PEM encoded +func MakeSSHKeyPair(bits int, pubKeyPath, privateKeyPath string) error { + privateKey, err := rsa.GenerateKey(_rand.Reader, bits) + if err != nil { + return err + } + + // generate and write private key as PEM + privateKeyFile, err := os.Create(privateKeyPath) + defer privateKeyFile.Close() + if err != nil { + return err + } + + privateKeyPEM := &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(privateKey)} + if err := pem.Encode(privateKeyFile, privateKeyPEM); err != nil { + return err + } + + // generate and write public key + pub, err := ssh.NewPublicKey(&privateKey.PublicKey) + if err != nil { + return err + } + + return ioutil.WriteFile(pubKeyPath, ssh.MarshalAuthorizedKey(pub), 0655) +} + func TestSftpCreate(t *testing.T) { - ctx, err := SftpConnect("user", "host:port") + os.Mkdir("./test", 0777) + MakeSSHKeyPair(1024, "./test/id_rsa.pub", "./test/id_rsa") + + go RunSftpServer() + time.Sleep(2 * time.Second) + + ctx, err := SftpConnect("test", "test", "localhost:2022") if err != nil { t.Fatal(err) } defer ctx.Disconnect() - var AppFs afero.Fs = afero.SftpFs{ + var AppFs Fs = SftpFs{ SftpClient: ctx.sftpc, } - file, err := AppFs.Create("aferoSftpFsTestFile") + AppFs.MkdirAll("test/dir1/dir2/dir3", os.FileMode(0777)) + AppFs.Mkdir("test/foo", os.FileMode(0000)) + AppFs.Mkdir("test/bar", os.FileMode(0777)) + + file, err := AppFs.Create("file1") if err != nil { t.Error(err) } defer file.Close() + + file.Write([]byte("whohoo\n")) + file.WriteString("sdsdsdsdsddssdsd") + + f1, err := AppFs.Open("file1") + if err != nil { + log.Fatalf("open: %v", err) + } + defer f1.Close() + + b := make([]byte, 100) + + _, err = f1.Read(b) + fmt.Println(string(b)) + + AppFs.Remove("test") } From acdc8aab04669eb2b89c4607ede69c870ee0919a Mon Sep 17 00:00:00 2001 From: Jerry Jacobs Date: Sat, 2 Jan 2016 14:12:59 +0100 Subject: [PATCH 4/6] * Cleanup and add some todos --- sftp_test.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/sftp_test.go b/sftp_test.go index 839a19d..3530f68 100644 --- a/sftp_test.go +++ b/sftp_test.go @@ -81,7 +81,8 @@ func (ctx *SftpFsContext) Disconnect() error { return nil } -func RunSftpServer() { +// TODO for such a weird reason rootpath is "." when writing "file1" with afero sftp backend +func RunSftpServer(rootpath string) { var ( readOnly bool debugLevelStr string @@ -91,15 +92,15 @@ func RunSftpServer() { ) flag.BoolVar(&readOnly, "R", false, "read-only server") - flag.BoolVar(&debugStderr, "e", false, "debug to stderr") + flag.BoolVar(&debugStderr, "e", true, "debug to stderr") flag.StringVar(&debugLevelStr, "l", "none", "debug level") - flag.StringVar(&rootDir, "root", ".", "root directory") + flag.StringVar(&rootDir, "root", rootpath, "root directory") flag.Parse() debugStream := ioutil.Discard if debugStderr { debugStream = os.Stderr - debugLevel = 1 + debugLevel = 5 } // An SSH server is represented by a ServerConfig, which holds @@ -188,7 +189,7 @@ func RunSftpServer() { } }(requests) - server, err := sftp.NewServer(channel, channel, debugStream, debugLevel, readOnly, rootDir) + server, err := sftp.NewServer(channel, channel, debugStream, debugLevel, readOnly, rootpath) if err != nil { log.Fatal(err) } @@ -198,7 +199,6 @@ func RunSftpServer() { } } - // MakeSSHKeyPair make a pair of public and private keys for SSH access. // Public key is encoded in the format for inclusion in an OpenSSH authorized_keys file. // Private Key generated is PEM encoded @@ -233,7 +233,7 @@ func TestSftpCreate(t *testing.T) { os.Mkdir("./test", 0777) MakeSSHKeyPair(1024, "./test/id_rsa.pub", "./test/id_rsa") - go RunSftpServer() + go RunSftpServer("./test/") time.Sleep(2 * time.Second) ctx, err := SftpConnect("test", "test", "localhost:2022") @@ -256,8 +256,8 @@ func TestSftpCreate(t *testing.T) { } defer file.Close() - file.Write([]byte("whohoo\n")) - file.WriteString("sdsdsdsdsddssdsd") + file.Write([]byte("hello\t")) + file.WriteString("world!\n") f1, err := AppFs.Open("file1") if err != nil { @@ -270,5 +270,5 @@ func TestSftpCreate(t *testing.T) { _, err = f1.Read(b) fmt.Println(string(b)) - AppFs.Remove("test") + // TODO check here if "hello\tworld\n" is in buffer b } From 719e24f1e2596085b7162f8255562414d59e9444 Mon Sep 17 00:00:00 2001 From: Jerry Jacobs Date: Sat, 2 Jan 2016 14:24:58 +0100 Subject: [PATCH 5/6] Dont read HOME/.ssh/id_rsa, create todo --- sftp_test.go | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/sftp_test.go b/sftp_test.go index 3530f68..bb00535 100644 --- a/sftp_test.go +++ b/sftp_test.go @@ -37,7 +37,10 @@ type SftpFsContext struct { sftpc *sftp.Client } +// TODO we only connect with hardcoded user+pass for now +// it should be possible to use $HOME/.ssh/id_rsa to login into the stub sftp server func SftpConnect(user, password, host string) (*SftpFsContext, error) { +/* pemBytes, err := ioutil.ReadFile(os.Getenv("HOME") + "/.ssh/id_rsa") if err != nil { return nil,err @@ -55,6 +58,14 @@ func SftpConnect(user, password, host string) (*SftpFsContext, error) { ssh.PublicKeys(signer), }, } +*/ + + sshcfg := &ssh.ClientConfig{ + User: user, + Auth: []ssh.AuthMethod{ + ssh.Password(password), + }, + } sshc, err := ssh.Dial("tcp", host, sshcfg) if err != nil { @@ -100,7 +111,7 @@ func RunSftpServer(rootpath string) { debugStream := ioutil.Discard if debugStderr { debugStream = os.Stderr - debugLevel = 5 + debugLevel = 1 } // An SSH server is represented by a ServerConfig, which holds @@ -234,7 +245,7 @@ func TestSftpCreate(t *testing.T) { MakeSSHKeyPair(1024, "./test/id_rsa.pub", "./test/id_rsa") go RunSftpServer("./test/") - time.Sleep(2 * time.Second) + time.Sleep(5 * time.Second) ctx, err := SftpConnect("test", "test", "localhost:2022") if err != nil { @@ -248,6 +259,7 @@ func TestSftpCreate(t *testing.T) { AppFs.MkdirAll("test/dir1/dir2/dir3", os.FileMode(0777)) AppFs.Mkdir("test/foo", os.FileMode(0000)) + AppFs.Chmod("test/foo", os.FileMode(0700)) AppFs.Mkdir("test/bar", os.FileMode(0777)) file, err := AppFs.Create("file1") From a34aa3adc40a5d5fc6904cb0ed08bbcbb9509c26 Mon Sep 17 00:00:00 2001 From: Jerry Jacobs Date: Sat, 2 Jan 2016 14:38:54 +0100 Subject: [PATCH 6/6] README: Add xor-gate as author --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 1309153..83310e5 100644 --- a/README.md +++ b/README.md @@ -366,6 +366,7 @@ Names in no particular order: * [spf13](https://github.com/spf13) * [jaqx0r](https://github.com/jaqx0r) * [mbertschler](https://github.com/mbertschler) +* [xor-gate](https://github.com/xor-gate) ## License