From 33dd3593c9681cd77ed3ec7909322b88b29b1250 Mon Sep 17 00:00:00 2001 From: John Adams Date: Mon, 14 May 2018 11:39:27 -0700 Subject: [PATCH] fork and merge in my chnages to support vault --- remote/remote.go | 22 +++-- remote/vault_stub.go | 29 +++++++ vault/vault.go | 194 +++++++++++++++++++++++++++++++++++++++++++ viper.go | 18 ++-- 4 files changed, 249 insertions(+), 14 deletions(-) create mode 100644 remote/vault_stub.go create mode 100644 vault/vault.go diff --git a/remote/remote.go b/remote/remote.go index 810d070..ccc673e 100644 --- a/remote/remote.go +++ b/remote/remote.go @@ -82,16 +82,22 @@ func getConfigManager(rp viper.RemoteProvider) (crypt.ConfigManager, error) { if err != nil { return nil, err } - if rp.Provider() == "etcd" { - cm, err = crypt.NewEtcdConfigManager([]string{rp.Endpoint()}, kr) - } else { - cm, err = crypt.NewConsulConfigManager([]string{rp.Endpoint()}, kr) + switch(rp.Provider()) { + case "etcd": + cm, err = crypt.NewEtcdConfigManager([]string{rp.Endpoint()}, kr) + case "consul": + cm, err = crypt.NewConsulConfigManager([]string{rp.Endpoint()}, kr) + case "vault": + cm, err = NewVaultConfigManager([]string{rp.Endpoint()}, kr) } } else { - if rp.Provider() == "etcd" { - cm, err = crypt.NewStandardEtcdConfigManager([]string{rp.Endpoint()}) - } else { - cm, err = crypt.NewStandardConsulConfigManager([]string{rp.Endpoint()}) + switch(rp.Provider()) { + case "etcd": + cm, err = crypt.NewStandardEtcdConfigManager([]string{rp.Endpoint()}) + case "consul": + cm, err = crypt.NewStandardConsulConfigManager([]string{rp.Endpoint()}) + case "vault": + cm, err = NewStandardVaultConfigManager([]string{rp.Endpoint()}) } } if err != nil { diff --git a/remote/vault_stub.go b/remote/vault_stub.go new file mode 100644 index 0000000..603aa79 --- /dev/null +++ b/remote/vault_stub.go @@ -0,0 +1,29 @@ +package remote + +// jna -- This stub will connect our vault provider with the +// crypt.ConfigManager interface. The main code is in vault/vault.go +// +// Until I can get the maintainers of the crypt package to support +// vault, we can make vault work this way. + +import ( + "io" + crypt "github.com/xordataexchange/crypt/config" + vault "github.com/spf13/viper/vault" +) + +func NewStandardVaultConfigManager(machines []string) (crypt.ConfigManager, error) { + store, err := vault.New(machines) + if err != nil { + return nil, err + } + return crypt.NewStandardConfigManager(store) +} + +func NewVaultConfigManager(machines []string, keystore io.Reader) (crypt.ConfigManager, error) { + store, err := vault.New(machines) + if err != nil { + return nil, err + } + return crypt.NewConfigManager(store, keystore) +} diff --git a/vault/vault.go b/vault/vault.go new file mode 100644 index 0000000..2bdad21 --- /dev/null +++ b/vault/vault.go @@ -0,0 +1,194 @@ +package vault + +/* Vault implements Hashicorp-vault based storage for configurations + * which is substaintally more secure than storing configs in + * consul or flat files. + * + * If using approle authentication. set your environment variables + * as follows to use this backend + * + * export VAULT_SECRET_ID= ... secret ... + * export VAULT_ROLE_ID= ... role id ... + * -- or -- + * export VAULT_TOKEN = .... + * + * If you are using SSL with vault, you can set + * export VAULT_CACERT= ... pem file containing ca cert ... + * and/or + * export VAULT_SSL_VERIFY=no +*/ + +import ( + "os" + "fmt" + "time" + + "github.com/xordataexchange/crypt/backend" + + vaultapi "github.com/hashicorp/vault/api" +) + +type Client struct { + client *vaultapi.Client + secret string // used only with role authentication, nil if using env-VAULT_TOKEN + secret_ttl time.Duration // if non-zero, it expires at this time + secret_acq_at float64 // when we got the secret + secret_expires bool +} + +func (c *Client) acquireToken(role string, secret string) (string, error) { + secretData := map[string]interface{}{ + "role_id" : role, + "secret_id" : secret, + } + + data, err := c.client.Logical().Write("auth/approle/login", secretData) + if data == nil { + return "", err + } + /* data is now of type *api.Secret and we can use it to set the client up */ + token,err := data.TokenID() + if err == nil { + c.client.SetToken(token) + } + + /* handle expiry */ + ttl, err := data.TokenTTL() + if err == nil { + c.secret_ttl = ttl + if ttl != 0 { + c.secret_expires = true + } + } + + c.secret_acq_at = float64(time.Now().Unix()) + + fmt.Println("Got token %s with expiry %d and acquired at %v", token, c.secret_ttl, c.secret_acq_at) + return token, err +} + +// this can be called before operations to ensure token is currentfg +func (c *Client) renewToken() (string, error) { + if c.secret_expires { + if ((c.secret_ttl.Seconds() + c.secret_acq_at > float64(time.Now().Unix())) && c.secret_ttl != 0) { + return c.acquireToken(os.Getenv("VAULT_ROLE_ID"), os.Getenv("VAULT_SECRET_ID")) + } else { + return "", nil + } + } else { + return "", nil + } +} + +func New(machines []string) (*Client, error) { + /* default config reads from the environment and sets defaults */ + /* a call to vaultapi.ReadEnvironment is not necessary here. */ + /* + * vault environment variables are required to proceed. + * either VAULT_TOKEN or VAULT_ROLE_ID and VAULT_SECRET_ID must be set + * see: https://github.com/hashicorp/vault/blob/master/api/client.go + */ + + conf := vaultapi.DefaultConfig() + + if len(machines) > 0 { + conf.Address = machines[0] + } + + // from the vault docs - + // https://godoc.org/github.com/hashicorp/vault/api#Secret + // If the environment variable `VAULT_TOKEN` is present, the token + // will be automatically added to the client. Otherwise, you must + // manually call `SetToken()`. + var returnval *Client + + client, err := vaultapi.NewClient(conf) + + if err != nil { + return nil, err + } + + /* what token are we using? */ + if v := os.Getenv(vaultapi.EnvVaultToken); v == "" { + /* not using VAULT_TOKEN! */ + if v := os.Getenv("VAULT_ROLE_ID"); v == "" { + fmt.Fprintf(os.Stderr, "neither VAULT_TOKEN or a VAULT_ROLE_ID/VAULT_SECRET_ID are set. Can't auth to vault.\n") + return nil, fmt.Errorf("Can't Auth to Vault") + } + if v := os.Getenv("VAULT_SECRET_ID"); v == "" { + fmt.Fprintf(os.Stderr, "VAULT_ROLE_ID set but VAULT_SECRET_ID is empty. Can't auth to vault.\n") + return nil, fmt.Errorf("Can't Auth to Vault") + } + + returnval = &Client{client, "", 0, float64(time.Now().Unix()), false} + + /* using the approle secrets, try to acquire a token */ + _, err := returnval.acquireToken(os.Getenv("VAULT_ROLE_ID"), os.Getenv("VAULT_SECRET_ID")) + if err != nil { + fmt.Fprintf(os.Stderr, "Vault ROLE/SECRET authentication failed - %v\n", err) + return nil, fmt.Errorf("Can't Auth to Vault") + } + } else { + /* we'll just go ahead with VAULT_TOKEN for auth */ + returnval = &Client{client, os.Getenv(vaultapi.EnvVaultToken), 0, float64(time.Now().Unix()), false} + } + + return returnval, nil +} + +func (c *Client) Get(key string) ([]byte, error) { + /* note that the vault client only connects when Get is issued if + * you are using VAULT_TOKEN authentication (set in the environment) + * + * If using role authentication, we'll try to acquire a token at init. + * + * This interface returns only one value from a secret. It expects that the + * referenced secret will have the data in the "value" key. + */ + data, err := c.client.Logical().Read(key) + if err != nil { + fmt.Println("Error during Vault Get -", err) + return []byte{}, err + } + if data.Data == nil { + return []byte{}, fmt.Errorf("Key ( %s ) was not found.", key) + } + + v := data.Data["value"].(string) + return []byte(v) , nil +} + +func (c *Client) List(key string) (backend.KVPairs, error) { + // TODO: NOT IMPLEMENTED + //pairs, err := c.client.Logical().List(key) + return nil, nil +} + +func (c *Client) Set(key string, value []byte) error { + secretData := map[string]interface{}{ + "value": value, + } + _, err := c.client.Logical().Write(key, secretData) + + return err +} + +func (c *Client) Watch(key string, stop chan bool) <-chan *backend.Response { + respChan := make(chan *backend.Response, 0) + go func() { + for { + data, err := c.client.Logical().Read(key) + if data == nil && err == nil { + err = fmt.Errorf("Key ( %s ) was not found.", key) + } + if err != nil { + respChan <- &backend.Response{nil, err} + time.Sleep(time.Second * 5) + continue + } + + respChan <- &backend.Response{data.Data["value"].([]byte), nil} + } + }() + return respChan +} diff --git a/viper.go b/viper.go index 907a102..2693f80 100644 --- a/viper.go +++ b/viper.go @@ -210,7 +210,7 @@ func New() *Viper { func Reset() { v = New() SupportedExts = []string{"json", "toml", "yaml", "yml", "properties", "props", "prop", "hcl"} - SupportedRemoteProviders = []string{"etcd", "consul"} + SupportedRemoteProviders = []string{"etcd", "consul", "vault"} } type defaultRemoteProvider struct { @@ -251,7 +251,7 @@ type RemoteProvider interface { var SupportedExts = []string{"json", "toml", "yaml", "yml", "properties", "props", "prop", "hcl"} // SupportedRemoteProviders are universally supported remote providers. -var SupportedRemoteProviders = []string{"etcd", "consul"} +var SupportedRemoteProviders = []string{"etcd", "consul", "vault"} func OnConfigChange(run func(in fsnotify.Event)) { v.OnConfigChange(run) } func (v *Viper) OnConfigChange(run func(in fsnotify.Event)) { @@ -363,12 +363,17 @@ func (v *Viper) AddConfigPath(in string) { // AddRemoteProvider adds a remote configuration source. // Remote Providers are searched in the order they are added. -// provider is a string value, "etcd" or "consul" are currently supported. +// provider is a string value, "etcd", "vault", or "consul" are currently supported. // endpoint is the url. etcd requires http://ip:port consul requires ip:port // path is the path in the k/v store to retrieve configuration // To retrieve a config file called myapp.json from /configs/myapp.json // you should set path to /configs and set config name (SetConfigName()) to // "myapp" +// +// "vault" allows access to Hashicorp's Vault. Two environment variables +// must be set to allow Vault to work, VAULT_ROLE_ID and VAULT_SECRET_ID. +// + func AddRemoteProvider(provider, endpoint, path string) error { return v.AddRemoteProvider(provider, endpoint, path) } @@ -1528,7 +1533,7 @@ func (v *Viper) getKeyValueConfig() error { v.kvstore = val return nil } - return RemoteConfigError("No Files Found") + return RemoteConfigError("No Files Found or remote error") } func (v *Viper) getRemoteConfig(provider RemoteProvider) (map[string]interface{}, error) { @@ -1536,6 +1541,7 @@ func (v *Viper) getRemoteConfig(provider RemoteProvider) (map[string]interface{} if err != nil { return nil, err } + err = v.unmarshalReader(reader, v.kvstore) return v.kvstore, err } @@ -1554,7 +1560,7 @@ func (v *Viper) watchKeyValueConfigOnChannel() error { }(respc) return nil } - return RemoteConfigError("No Files Found") + return RemoteConfigError("No Files Found or remote error") } // Retrieve the first found remote configuration. @@ -1567,7 +1573,7 @@ func (v *Viper) watchKeyValueConfig() error { v.kvstore = val return nil } - return RemoteConfigError("No Files Found") + return RemoteConfigError("No Files Found or remote error") } func (v *Viper) watchRemoteConfig(provider RemoteProvider) (map[string]interface{}, error) {