2015-05-25 16:22:27 +03:00
|
|
|
package redis
|
|
|
|
|
|
|
|
import (
|
|
|
|
"errors"
|
|
|
|
"fmt"
|
2016-10-09 13:30:45 +03:00
|
|
|
"math/rand"
|
|
|
|
"strconv"
|
2015-05-25 16:22:27 +03:00
|
|
|
"sync"
|
2016-10-02 15:44:01 +03:00
|
|
|
"sync/atomic"
|
2015-05-25 16:22:27 +03:00
|
|
|
"time"
|
|
|
|
|
2016-10-09 13:49:28 +03:00
|
|
|
"gopkg.in/redis.v5/internal"
|
|
|
|
"gopkg.in/redis.v5/internal/consistenthash"
|
|
|
|
"gopkg.in/redis.v5/internal/hashtag"
|
|
|
|
"gopkg.in/redis.v5/internal/pool"
|
2015-05-25 16:22:27 +03:00
|
|
|
)
|
|
|
|
|
2016-08-09 16:32:08 +03:00
|
|
|
var errRingShardsDown = errors.New("redis: all ring shards are down")
|
2015-05-25 16:22:27 +03:00
|
|
|
|
|
|
|
// RingOptions are used to configure a ring client and should be
|
|
|
|
// passed to NewRing.
|
|
|
|
type RingOptions struct {
|
2016-08-09 16:32:08 +03:00
|
|
|
// Map of name => host:port addresses of ring shards.
|
2015-05-25 16:22:27 +03:00
|
|
|
Addrs map[string]string
|
|
|
|
|
2016-08-09 16:32:08 +03:00
|
|
|
// Frequency of PING commands sent to check shards availability.
|
|
|
|
// Shard is considered down after 3 subsequent failed checks.
|
|
|
|
HeartbeatFrequency time.Duration
|
|
|
|
|
2015-05-25 16:22:27 +03:00
|
|
|
// Following options are copied from Options struct.
|
|
|
|
|
2016-06-05 12:45:39 +03:00
|
|
|
DB int
|
2015-05-25 16:22:27 +03:00
|
|
|
Password string
|
|
|
|
|
2015-06-04 11:50:24 +03:00
|
|
|
MaxRetries int
|
|
|
|
|
2015-05-25 16:22:27 +03:00
|
|
|
DialTimeout time.Duration
|
|
|
|
ReadTimeout time.Duration
|
|
|
|
WriteTimeout time.Duration
|
|
|
|
|
2016-03-17 19:00:47 +03:00
|
|
|
PoolSize int
|
|
|
|
PoolTimeout time.Duration
|
|
|
|
IdleTimeout time.Duration
|
|
|
|
IdleCheckFrequency time.Duration
|
2015-05-25 16:22:27 +03:00
|
|
|
}
|
|
|
|
|
2016-08-09 16:32:08 +03:00
|
|
|
func (opt *RingOptions) init() {
|
|
|
|
if opt.HeartbeatFrequency == 0 {
|
|
|
|
opt.HeartbeatFrequency = 500 * time.Millisecond
|
|
|
|
}
|
|
|
|
}
|
2016-06-05 14:10:30 +03:00
|
|
|
|
2015-05-25 16:22:27 +03:00
|
|
|
func (opt *RingOptions) clientOptions() *Options {
|
|
|
|
return &Options{
|
|
|
|
DB: opt.DB,
|
|
|
|
Password: opt.Password,
|
|
|
|
|
|
|
|
DialTimeout: opt.DialTimeout,
|
|
|
|
ReadTimeout: opt.ReadTimeout,
|
|
|
|
WriteTimeout: opt.WriteTimeout,
|
|
|
|
|
2016-03-17 19:00:47 +03:00
|
|
|
PoolSize: opt.PoolSize,
|
|
|
|
PoolTimeout: opt.PoolTimeout,
|
|
|
|
IdleTimeout: opt.IdleTimeout,
|
|
|
|
IdleCheckFrequency: opt.IdleCheckFrequency,
|
2015-05-25 16:22:27 +03:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
type ringShard struct {
|
|
|
|
Client *Client
|
2016-10-02 15:44:01 +03:00
|
|
|
down int32
|
2015-05-25 16:22:27 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
func (shard *ringShard) String() string {
|
|
|
|
var state string
|
|
|
|
if shard.IsUp() {
|
|
|
|
state = "up"
|
|
|
|
} else {
|
|
|
|
state = "down"
|
|
|
|
}
|
|
|
|
return fmt.Sprintf("%s is %s", shard.Client, state)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (shard *ringShard) IsDown() bool {
|
2016-08-09 16:32:08 +03:00
|
|
|
const threshold = 3
|
2016-10-02 15:44:01 +03:00
|
|
|
return atomic.LoadInt32(&shard.down) >= threshold
|
2015-05-25 16:22:27 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
func (shard *ringShard) IsUp() bool {
|
|
|
|
return !shard.IsDown()
|
|
|
|
}
|
|
|
|
|
|
|
|
// Vote votes to set shard state and returns true if state was changed.
|
|
|
|
func (shard *ringShard) Vote(up bool) bool {
|
|
|
|
if up {
|
|
|
|
changed := shard.IsDown()
|
2016-10-02 15:44:01 +03:00
|
|
|
atomic.StoreInt32(&shard.down, 0)
|
2015-05-25 16:22:27 +03:00
|
|
|
return changed
|
|
|
|
}
|
|
|
|
|
|
|
|
if shard.IsDown() {
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
2016-10-02 15:44:01 +03:00
|
|
|
atomic.AddInt32(&shard.down, 1)
|
2015-05-25 16:22:27 +03:00
|
|
|
return shard.IsDown()
|
|
|
|
}
|
|
|
|
|
|
|
|
// Ring is a Redis client that uses constistent hashing to distribute
|
2015-09-12 09:36:03 +03:00
|
|
|
// keys across multiple Redis servers (shards). It's safe for
|
|
|
|
// concurrent use by multiple goroutines.
|
2015-05-25 16:22:27 +03:00
|
|
|
//
|
2015-11-21 11:20:01 +03:00
|
|
|
// Ring monitors the state of each shard and removes dead shards from
|
2016-07-02 11:07:27 +03:00
|
|
|
// the ring. When shard comes online it is added back to the ring. This
|
2015-05-25 16:22:27 +03:00
|
|
|
// gives you maximum availability and partition tolerance, but no
|
|
|
|
// consistency between different shards or even clients. Each client
|
|
|
|
// uses shards that are available to the client and does not do any
|
|
|
|
// coordination when shard state is changed.
|
|
|
|
//
|
2016-08-09 16:32:08 +03:00
|
|
|
// Ring should be used when you need multiple Redis servers for caching
|
2015-05-25 16:22:27 +03:00
|
|
|
// and can tolerate losing data when one of the servers dies.
|
|
|
|
// Otherwise you should use Redis Cluster.
|
|
|
|
type Ring struct {
|
2016-06-05 12:45:39 +03:00
|
|
|
cmdable
|
2015-05-25 16:22:27 +03:00
|
|
|
|
2015-06-04 11:50:24 +03:00
|
|
|
opt *RingOptions
|
2015-05-25 16:22:27 +03:00
|
|
|
nreplicas int
|
|
|
|
|
2016-05-06 21:12:31 +03:00
|
|
|
mu sync.RWMutex
|
2015-05-25 16:22:27 +03:00
|
|
|
hash *consistenthash.Map
|
|
|
|
shards map[string]*ringShard
|
|
|
|
|
2016-05-06 21:12:31 +03:00
|
|
|
cmdsInfoOnce *sync.Once
|
2016-10-09 13:30:45 +03:00
|
|
|
cmdsInfo map[string]*CommandInfo
|
2016-05-06 21:12:31 +03:00
|
|
|
|
2015-05-25 16:22:27 +03:00
|
|
|
closed bool
|
|
|
|
}
|
|
|
|
|
2016-09-27 12:24:14 +03:00
|
|
|
var _ Cmdable = (*Ring)(nil)
|
|
|
|
|
2015-05-25 16:22:27 +03:00
|
|
|
func NewRing(opt *RingOptions) *Ring {
|
|
|
|
const nreplicas = 100
|
2016-06-05 14:10:30 +03:00
|
|
|
opt.init()
|
2015-05-25 16:22:27 +03:00
|
|
|
ring := &Ring{
|
2015-06-04 11:50:24 +03:00
|
|
|
opt: opt,
|
2015-05-25 16:22:27 +03:00
|
|
|
nreplicas: nreplicas,
|
2015-06-04 11:50:24 +03:00
|
|
|
|
|
|
|
hash: consistenthash.New(nreplicas, nil),
|
|
|
|
shards: make(map[string]*ringShard),
|
2016-05-06 21:12:31 +03:00
|
|
|
|
|
|
|
cmdsInfoOnce: new(sync.Once),
|
2015-05-25 16:22:27 +03:00
|
|
|
}
|
2016-06-05 12:45:39 +03:00
|
|
|
ring.cmdable.process = ring.Process
|
2015-05-25 16:22:27 +03:00
|
|
|
for name, addr := range opt.Addrs {
|
|
|
|
clopt := opt.clientOptions()
|
|
|
|
clopt.Addr = addr
|
|
|
|
ring.addClient(name, NewClient(clopt))
|
|
|
|
}
|
|
|
|
go ring.heartbeat()
|
|
|
|
return ring
|
|
|
|
}
|
|
|
|
|
2016-10-02 15:44:01 +03:00
|
|
|
// PoolStats returns accumulated connection pool stats.
|
|
|
|
func (c *Ring) PoolStats() *PoolStats {
|
|
|
|
var acc PoolStats
|
|
|
|
for _, shard := range c.shards {
|
|
|
|
s := shard.Client.connPool.Stats()
|
|
|
|
acc.Requests += s.Requests
|
|
|
|
acc.Hits += s.Hits
|
|
|
|
acc.Timeouts += s.Timeouts
|
|
|
|
acc.TotalConns += s.TotalConns
|
|
|
|
acc.FreeConns += s.FreeConns
|
|
|
|
}
|
|
|
|
return &acc
|
|
|
|
}
|
|
|
|
|
|
|
|
// ForEachShard concurrently calls the fn on each live shard in the ring.
|
|
|
|
// It returns the first error if any.
|
|
|
|
func (c *Ring) ForEachShard(fn func(client *Client) error) error {
|
|
|
|
var wg sync.WaitGroup
|
|
|
|
errCh := make(chan error, 1)
|
|
|
|
for _, shard := range c.shards {
|
|
|
|
if shard.IsDown() {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
|
|
|
wg.Add(1)
|
|
|
|
go func(shard *ringShard) {
|
|
|
|
defer wg.Done()
|
|
|
|
err := fn(shard.Client)
|
|
|
|
if err != nil {
|
|
|
|
select {
|
|
|
|
case errCh <- err:
|
|
|
|
default:
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}(shard)
|
|
|
|
}
|
|
|
|
wg.Wait()
|
|
|
|
|
|
|
|
select {
|
|
|
|
case err := <-errCh:
|
|
|
|
return err
|
|
|
|
default:
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-07-01 15:25:28 +03:00
|
|
|
func (c *Ring) cmdInfo(name string) *CommandInfo {
|
|
|
|
c.cmdsInfoOnce.Do(func() {
|
|
|
|
for _, shard := range c.shards {
|
2016-05-06 21:12:31 +03:00
|
|
|
cmdsInfo, err := shard.Client.Command().Result()
|
|
|
|
if err == nil {
|
2016-07-01 15:25:28 +03:00
|
|
|
c.cmdsInfo = cmdsInfo
|
2016-05-06 21:12:31 +03:00
|
|
|
return
|
|
|
|
}
|
|
|
|
}
|
2016-07-01 15:25:28 +03:00
|
|
|
c.cmdsInfoOnce = &sync.Once{}
|
2016-05-06 21:12:31 +03:00
|
|
|
})
|
2016-07-01 15:25:28 +03:00
|
|
|
if c.cmdsInfo == nil {
|
2016-05-06 21:12:31 +03:00
|
|
|
return nil
|
|
|
|
}
|
2016-07-01 15:25:28 +03:00
|
|
|
return c.cmdsInfo[name]
|
2016-05-06 21:12:31 +03:00
|
|
|
}
|
|
|
|
|
2016-07-01 15:25:28 +03:00
|
|
|
func (c *Ring) addClient(name string, cl *Client) {
|
|
|
|
c.mu.Lock()
|
|
|
|
c.hash.Add(name)
|
|
|
|
c.shards[name] = &ringShard{Client: cl}
|
|
|
|
c.mu.Unlock()
|
2015-05-25 16:22:27 +03:00
|
|
|
}
|
|
|
|
|
2016-11-30 13:39:14 +03:00
|
|
|
func (c *Ring) shardByKey(key string) (*ringShard, error) {
|
2016-10-09 13:30:45 +03:00
|
|
|
key = hashtag.Key(key)
|
|
|
|
|
2016-07-01 15:25:28 +03:00
|
|
|
c.mu.RLock()
|
2015-05-25 16:22:27 +03:00
|
|
|
|
2016-07-01 15:25:28 +03:00
|
|
|
if c.closed {
|
2016-10-09 13:30:45 +03:00
|
|
|
c.mu.RUnlock()
|
2016-03-14 14:17:33 +03:00
|
|
|
return nil, pool.ErrClosed
|
2015-05-25 16:22:27 +03:00
|
|
|
}
|
|
|
|
|
2016-10-09 13:30:45 +03:00
|
|
|
name := c.hash.Get(key)
|
2015-05-25 16:22:27 +03:00
|
|
|
if name == "" {
|
2016-07-01 15:25:28 +03:00
|
|
|
c.mu.RUnlock()
|
2015-05-25 16:22:27 +03:00
|
|
|
return nil, errRingShardsDown
|
|
|
|
}
|
|
|
|
|
2016-11-30 13:39:14 +03:00
|
|
|
shard := c.shards[name]
|
2016-07-01 15:25:28 +03:00
|
|
|
c.mu.RUnlock()
|
2016-11-30 13:39:14 +03:00
|
|
|
return shard, nil
|
2015-05-25 16:22:27 +03:00
|
|
|
}
|
|
|
|
|
2016-11-30 13:39:14 +03:00
|
|
|
func (c *Ring) randomShard() (*ringShard, error) {
|
2016-10-09 13:30:45 +03:00
|
|
|
return c.shardByKey(strconv.Itoa(rand.Int()))
|
|
|
|
}
|
|
|
|
|
2016-11-30 13:39:14 +03:00
|
|
|
func (c *Ring) shardByName(name string) (*ringShard, error) {
|
2016-10-09 13:30:45 +03:00
|
|
|
if name == "" {
|
|
|
|
return c.randomShard()
|
|
|
|
}
|
|
|
|
|
|
|
|
c.mu.RLock()
|
2016-11-30 13:39:14 +03:00
|
|
|
shard := c.shards[name]
|
2016-10-09 13:30:45 +03:00
|
|
|
c.mu.RUnlock()
|
2016-11-30 13:39:14 +03:00
|
|
|
return shard, nil
|
2016-10-09 13:30:45 +03:00
|
|
|
}
|
|
|
|
|
2016-11-30 13:39:14 +03:00
|
|
|
func (c *Ring) cmdShard(cmd Cmder) (*ringShard, error) {
|
2016-10-09 13:30:45 +03:00
|
|
|
cmdInfo := c.cmdInfo(cmd.arg(0))
|
|
|
|
firstKey := cmd.arg(cmdFirstKeyPos(cmd, cmdInfo))
|
|
|
|
if firstKey == "" {
|
|
|
|
return c.randomShard()
|
|
|
|
}
|
|
|
|
return c.shardByKey(firstKey)
|
|
|
|
}
|
|
|
|
|
2016-07-01 15:25:28 +03:00
|
|
|
func (c *Ring) Process(cmd Cmder) error {
|
2016-11-30 13:39:14 +03:00
|
|
|
shard, err := c.cmdShard(cmd)
|
2015-05-25 16:22:27 +03:00
|
|
|
if err != nil {
|
|
|
|
cmd.setErr(err)
|
2016-06-17 15:09:38 +03:00
|
|
|
return err
|
2015-05-25 16:22:27 +03:00
|
|
|
}
|
2016-11-30 13:39:14 +03:00
|
|
|
return shard.Client.Process(cmd)
|
2015-05-25 16:22:27 +03:00
|
|
|
}
|
|
|
|
|
2016-10-09 13:30:45 +03:00
|
|
|
// rebalance removes dead shards from the Ring.
|
2016-07-01 15:25:28 +03:00
|
|
|
func (c *Ring) rebalance() {
|
2016-10-09 13:30:45 +03:00
|
|
|
hash := consistenthash.New(c.nreplicas, nil)
|
2016-07-01 15:25:28 +03:00
|
|
|
for name, shard := range c.shards {
|
2015-05-25 16:22:27 +03:00
|
|
|
if shard.IsUp() {
|
2016-10-09 13:30:45 +03:00
|
|
|
hash.Add(name)
|
2015-05-25 16:22:27 +03:00
|
|
|
}
|
|
|
|
}
|
2016-10-09 13:30:45 +03:00
|
|
|
|
|
|
|
c.mu.Lock()
|
|
|
|
c.hash = hash
|
|
|
|
c.mu.Unlock()
|
2015-05-25 16:22:27 +03:00
|
|
|
}
|
|
|
|
|
2016-07-02 11:07:27 +03:00
|
|
|
// heartbeat monitors state of each shard in the ring.
|
2016-07-01 15:25:28 +03:00
|
|
|
func (c *Ring) heartbeat() {
|
2016-08-09 16:32:08 +03:00
|
|
|
ticker := time.NewTicker(c.opt.HeartbeatFrequency)
|
2015-05-25 16:22:27 +03:00
|
|
|
defer ticker.Stop()
|
|
|
|
for _ = range ticker.C {
|
|
|
|
var rebalance bool
|
|
|
|
|
2016-07-01 15:25:28 +03:00
|
|
|
c.mu.RLock()
|
2015-05-25 16:22:27 +03:00
|
|
|
|
2016-07-01 15:25:28 +03:00
|
|
|
if c.closed {
|
|
|
|
c.mu.RUnlock()
|
2015-05-25 16:22:27 +03:00
|
|
|
break
|
|
|
|
}
|
|
|
|
|
2016-07-01 15:25:28 +03:00
|
|
|
for _, shard := range c.shards {
|
2015-05-25 16:22:27 +03:00
|
|
|
err := shard.Client.Ping().Err()
|
2016-03-12 11:52:13 +03:00
|
|
|
if shard.Vote(err == nil || err == pool.ErrPoolTimeout) {
|
2016-04-09 14:52:01 +03:00
|
|
|
internal.Logf("ring shard state changed: %s", shard)
|
2015-05-25 16:22:27 +03:00
|
|
|
rebalance = true
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-07-01 15:25:28 +03:00
|
|
|
c.mu.RUnlock()
|
2015-05-25 16:22:27 +03:00
|
|
|
|
|
|
|
if rebalance {
|
2016-07-01 15:25:28 +03:00
|
|
|
c.rebalance()
|
2015-05-25 16:22:27 +03:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Close closes the ring client, releasing any open resources.
|
|
|
|
//
|
2015-09-12 09:36:03 +03:00
|
|
|
// It is rare to Close a Ring, as the Ring is meant to be long-lived
|
|
|
|
// and shared between many goroutines.
|
2016-12-03 18:30:13 +03:00
|
|
|
func (c *Ring) Close() error {
|
2016-07-01 15:25:28 +03:00
|
|
|
defer c.mu.Unlock()
|
|
|
|
c.mu.Lock()
|
2015-05-25 16:22:27 +03:00
|
|
|
|
2016-07-01 15:25:28 +03:00
|
|
|
if c.closed {
|
2015-05-25 16:22:27 +03:00
|
|
|
return nil
|
|
|
|
}
|
2016-07-01 15:25:28 +03:00
|
|
|
c.closed = true
|
2015-05-25 16:22:27 +03:00
|
|
|
|
2016-12-03 18:30:13 +03:00
|
|
|
var firstErr error
|
2016-07-01 15:25:28 +03:00
|
|
|
for _, shard := range c.shards {
|
2016-12-03 18:30:13 +03:00
|
|
|
if err := shard.Client.Close(); err != nil && firstErr == nil {
|
|
|
|
firstErr = err
|
2015-05-25 16:22:27 +03:00
|
|
|
}
|
|
|
|
}
|
2016-07-01 15:25:28 +03:00
|
|
|
c.hash = nil
|
|
|
|
c.shards = nil
|
2015-05-25 16:22:27 +03:00
|
|
|
|
2016-12-03 18:30:13 +03:00
|
|
|
return firstErr
|
2015-05-25 16:22:27 +03:00
|
|
|
}
|
2015-06-04 11:50:24 +03:00
|
|
|
|
2016-07-01 15:25:28 +03:00
|
|
|
func (c *Ring) Pipeline() *Pipeline {
|
2016-06-05 12:45:39 +03:00
|
|
|
pipe := Pipeline{
|
2016-07-01 15:25:28 +03:00
|
|
|
exec: c.pipelineExec,
|
2015-06-04 11:50:24 +03:00
|
|
|
}
|
2016-06-05 12:45:39 +03:00
|
|
|
pipe.cmdable.process = pipe.Process
|
|
|
|
pipe.statefulCmdable.process = pipe.Process
|
|
|
|
return &pipe
|
2015-06-04 11:50:24 +03:00
|
|
|
}
|
|
|
|
|
2016-07-01 15:25:28 +03:00
|
|
|
func (c *Ring) Pipelined(fn func(*Pipeline) error) ([]Cmder, error) {
|
|
|
|
return c.Pipeline().pipelined(fn)
|
2015-06-04 11:50:24 +03:00
|
|
|
}
|
|
|
|
|
2016-10-13 13:56:24 +03:00
|
|
|
func (c *Ring) pipelineExec(cmds []Cmder) (firstErr error) {
|
2015-06-04 11:50:24 +03:00
|
|
|
cmdsMap := make(map[string][]Cmder)
|
|
|
|
for _, cmd := range cmds {
|
2016-10-09 13:30:45 +03:00
|
|
|
cmdInfo := c.cmdInfo(cmd.arg(0))
|
|
|
|
name := cmd.arg(cmdFirstKeyPos(cmd, cmdInfo))
|
|
|
|
if name != "" {
|
|
|
|
name = c.hash.Get(hashtag.Key(name))
|
2015-06-24 15:37:41 +03:00
|
|
|
}
|
2015-06-04 11:50:24 +03:00
|
|
|
cmdsMap[name] = append(cmdsMap[name], cmd)
|
|
|
|
}
|
|
|
|
|
2016-07-01 15:25:28 +03:00
|
|
|
for i := 0; i <= c.opt.MaxRetries; i++ {
|
2016-10-13 13:56:24 +03:00
|
|
|
var failedCmdsMap map[string][]Cmder
|
2015-06-04 11:50:24 +03:00
|
|
|
|
|
|
|
for name, cmds := range cmdsMap {
|
2016-11-30 13:39:14 +03:00
|
|
|
shard, err := c.shardByName(name)
|
2016-10-09 13:30:45 +03:00
|
|
|
if err != nil {
|
|
|
|
setCmdsErr(cmds, err)
|
2016-10-13 13:56:24 +03:00
|
|
|
if firstErr == nil {
|
|
|
|
firstErr = err
|
2016-10-09 13:30:45 +03:00
|
|
|
}
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
2016-11-30 13:39:14 +03:00
|
|
|
cn, _, err := shard.Client.conn()
|
2015-06-04 11:50:24 +03:00
|
|
|
if err != nil {
|
|
|
|
setCmdsErr(cmds, err)
|
2016-10-13 13:56:24 +03:00
|
|
|
if firstErr == nil {
|
|
|
|
firstErr = err
|
2015-06-04 11:50:24 +03:00
|
|
|
}
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
2016-12-13 18:28:39 +03:00
|
|
|
canRetry, err := shard.Client.pipelineProcessCmds(cn, cmds)
|
2016-11-30 13:39:14 +03:00
|
|
|
shard.Client.putConn(cn, err, false)
|
2016-10-13 13:56:24 +03:00
|
|
|
if err == nil {
|
|
|
|
continue
|
2015-06-04 11:50:24 +03:00
|
|
|
}
|
2016-10-13 13:56:24 +03:00
|
|
|
if firstErr == nil {
|
|
|
|
firstErr = err
|
|
|
|
}
|
2016-12-13 18:28:39 +03:00
|
|
|
if canRetry && internal.IsRetryableError(err) {
|
2016-10-13 13:56:24 +03:00
|
|
|
if failedCmdsMap == nil {
|
|
|
|
failedCmdsMap = make(map[string][]Cmder)
|
|
|
|
}
|
|
|
|
failedCmdsMap[name] = cmds
|
2015-06-04 11:50:24 +03:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if len(failedCmdsMap) == 0 {
|
|
|
|
break
|
|
|
|
}
|
|
|
|
cmdsMap = failedCmdsMap
|
|
|
|
}
|
|
|
|
|
2016-10-13 13:56:24 +03:00
|
|
|
return firstErr
|
2015-06-04 11:50:24 +03:00
|
|
|
}
|