Merge pull request from go-redis/fix/do-not-use-random-node

Use master / another slave node when current node is failing
This commit is contained in:
Vladimir Mihailenco 2019-06-19 11:02:25 +03:00 committed by GitHub
commit fa2f6ee00b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 49 additions and 49 deletions

View File

@ -155,7 +155,7 @@ type clusterNode struct {
latency uint32 // atomic latency uint32 // atomic
generation uint32 // atomic generation uint32 // atomic
loading uint32 // atomic failing uint32 // atomic
} }
func newClusterNode(clOpt *ClusterOptions, addr string) *clusterNode { func newClusterNode(clOpt *ClusterOptions, addr string) *clusterNode {
@ -203,21 +203,21 @@ func (n *clusterNode) Latency() time.Duration {
return time.Duration(latency) * time.Microsecond return time.Duration(latency) * time.Microsecond
} }
func (n *clusterNode) MarkAsLoading() { func (n *clusterNode) MarkAsFailing() {
atomic.StoreUint32(&n.loading, uint32(time.Now().Unix())) atomic.StoreUint32(&n.failing, uint32(time.Now().Unix()))
} }
func (n *clusterNode) Loading() bool { func (n *clusterNode) Failing() bool {
const minute = int64(time.Minute / time.Second) const timeout = 15 // 15 seconds
loading := atomic.LoadUint32(&n.loading) failing := atomic.LoadUint32(&n.failing)
if loading == 0 { if failing == 0 {
return false return false
} }
if time.Now().Unix()-int64(loading) < minute { if time.Now().Unix()-int64(failing) < timeout {
return true return true
} }
atomic.StoreUint32(&n.loading, 0) atomic.StoreUint32(&n.failing, 0)
return false return false
} }
@ -522,7 +522,7 @@ func (c *clusterState) slotSlaveNode(slot int) (*clusterNode, error) {
case 1: case 1:
return nodes[0], nil return nodes[0], nil
case 2: case 2:
if slave := nodes[1]; !slave.Loading() { if slave := nodes[1]; !slave.Failing() {
return slave, nil return slave, nil
} }
return nodes[0], nil return nodes[0], nil
@ -531,7 +531,7 @@ func (c *clusterState) slotSlaveNode(slot int) (*clusterNode, error) {
for i := 0; i < 10; i++ { for i := 0; i < 10; i++ {
n := rand.Intn(len(nodes)-1) + 1 n := rand.Intn(len(nodes)-1) + 1
slave = nodes[n] slave = nodes[n]
if !slave.Loading() { if !slave.Failing() {
return slave, nil return slave, nil
} }
} }
@ -551,7 +551,7 @@ func (c *clusterState) slotClosestNode(slot int) (*clusterNode, error) {
var node *clusterNode var node *clusterNode
for _, n := range nodes { for _, n := range nodes {
if n.Loading() { if n.Failing() {
continue continue
} }
if node == nil || node.Latency()-n.Latency() > threshold { if node == nil || node.Latency()-n.Latency() > threshold {
@ -561,10 +561,13 @@ func (c *clusterState) slotClosestNode(slot int) (*clusterNode, error) {
return node, nil return node, nil
} }
func (c *clusterState) slotRandomNode(slot int) *clusterNode { func (c *clusterState) slotRandomNode(slot int) (*clusterNode, error) {
nodes := c.slotNodes(slot) nodes := c.slotNodes(slot)
if len(nodes) == 0 {
return c.nodes.Random()
}
n := rand.Intn(len(nodes)) n := rand.Intn(len(nodes))
return nodes[n] return nodes[n], nil
} }
func (c *clusterState) slotNodes(slot int) []*clusterNode { func (c *clusterState) slotNodes(slot int) []*clusterNode {
@ -742,23 +745,26 @@ func (c *ClusterClient) ProcessContext(ctx context.Context, cmd Cmder) error {
} }
func (c *ClusterClient) process(ctx context.Context, cmd Cmder) error { func (c *ClusterClient) process(ctx context.Context, cmd Cmder) error {
cmdInfo := c.cmdInfo(cmd.Name())
slot := c.cmdSlot(cmd)
var node *clusterNode var node *clusterNode
var ask bool var ask bool
for attempt := 0; attempt <= c.opt.MaxRedirects; attempt++ { for attempt := 0; attempt <= c.opt.MaxRedirects; attempt++ {
var err error
if attempt > 0 { if attempt > 0 {
time.Sleep(c.retryBackoff(attempt)) time.Sleep(c.retryBackoff(attempt))
} }
if node == nil { if node == nil {
var err error node, err = c.cmdNode(cmdInfo, slot)
_, node, err = c.cmdSlotAndNode(cmd)
if err != nil { if err != nil {
cmd.setErr(err) cmd.setErr(err)
break break
} }
} }
var err error
if ask { if ask {
pipe := node.Client.Pipeline() pipe := node.Client.Pipeline()
_ = pipe.Process(NewCmd("ASKING")) _ = pipe.Process(NewCmd("ASKING"))
@ -780,7 +786,7 @@ func (c *ClusterClient) process(ctx context.Context, cmd Cmder) error {
// If slave is loading - pick another node. // If slave is loading - pick another node.
if c.opt.ReadOnly && internal.IsLoadingError(err) { if c.opt.ReadOnly && internal.IsLoadingError(err) {
node.MarkAsLoading() node.MarkAsFailing()
node = nil node = nil
continue continue
} }
@ -807,11 +813,9 @@ func (c *ClusterClient) process(ctx context.Context, cmd Cmder) error {
continue continue
} }
// Second try random node. // Second try another node.
node, err = c.nodes.Random() node.MarkAsFailing()
if err != nil { node = nil
break
}
continue continue
} }
@ -1100,17 +1104,20 @@ func (c *ClusterClient) mapCmdsByNode(cmds []Cmder, cmdsMap *cmdsMap) error {
cmdsAreReadOnly := c.opt.ReadOnly && c.cmdsAreReadOnly(cmds) cmdsAreReadOnly := c.opt.ReadOnly && c.cmdsAreReadOnly(cmds)
for _, cmd := range cmds { for _, cmd := range cmds {
slot := c.cmdSlot(cmd)
var node *clusterNode var node *clusterNode
var err error var err error
if cmdsAreReadOnly { if cmdsAreReadOnly {
_, node, err = c.cmdSlotAndNode(cmd) cmdInfo := c.cmdInfo(cmd.Name())
node, err = c.cmdNode(cmdInfo, slot)
} else { } else {
slot := c.cmdSlot(cmd)
node, err = state.slotMasterNode(slot) node, err = state.slotMasterNode(slot)
} }
if err != nil { if err != nil {
return err return err
} }
cmdsMap.mu.Lock() cmdsMap.mu.Lock()
cmdsMap.m[node] = append(cmdsMap.m[node], cmd) cmdsMap.m[node] = append(cmdsMap.m[node], cmd)
cmdsMap.mu.Unlock() cmdsMap.mu.Unlock()
@ -1162,7 +1169,7 @@ func (c *ClusterClient) pipelineReadCmds(
} }
if c.opt.ReadOnly && internal.IsLoadingError(err) { if c.opt.ReadOnly && internal.IsLoadingError(err) {
node.MarkAsLoading() node.MarkAsFailing()
} else if internal.IsRedisError(err) { } else if internal.IsRedisError(err) {
continue continue
} }
@ -1529,14 +1536,6 @@ func (c *ClusterClient) cmdInfo(name string) *CommandInfo {
return info return info
} }
func cmdSlot(cmd Cmder, pos int) int {
if pos == 0 {
return hashtag.RandomSlot()
}
firstKey := cmd.stringArg(pos)
return hashtag.Slot(firstKey)
}
func (c *ClusterClient) cmdSlot(cmd Cmder) int { func (c *ClusterClient) cmdSlot(cmd Cmder) int {
args := cmd.Args() args := cmd.Args()
if args[0] == "cluster" && args[1] == "getkeysinslot" { if args[0] == "cluster" && args[1] == "getkeysinslot" {
@ -1547,32 +1546,31 @@ func (c *ClusterClient) cmdSlot(cmd Cmder) int {
return cmdSlot(cmd, cmdFirstKeyPos(cmd, cmdInfo)) return cmdSlot(cmd, cmdFirstKeyPos(cmd, cmdInfo))
} }
func (c *ClusterClient) cmdSlotAndNode(cmd Cmder) (int, *clusterNode, error) { func cmdSlot(cmd Cmder, pos int) int {
if pos == 0 {
return hashtag.RandomSlot()
}
firstKey := cmd.stringArg(pos)
return hashtag.Slot(firstKey)
}
func (c *ClusterClient) cmdNode(cmdInfo *CommandInfo, slot int) (*clusterNode, error) {
state, err := c.state.Get() state, err := c.state.Get()
if err != nil { if err != nil {
return 0, nil, err return nil, err
} }
cmdInfo := c.cmdInfo(cmd.Name())
slot := c.cmdSlot(cmd)
if c.opt.ReadOnly && cmdInfo != nil && cmdInfo.ReadOnly { if c.opt.ReadOnly && cmdInfo != nil && cmdInfo.ReadOnly {
if c.opt.RouteByLatency { if c.opt.RouteByLatency {
node, err := state.slotClosestNode(slot) return state.slotClosestNode(slot)
return slot, node, err
} }
if c.opt.RouteRandomly { if c.opt.RouteRandomly {
node := state.slotRandomNode(slot) return state.slotRandomNode(slot)
return slot, node, nil }
return state.slotSlaveNode(slot)
} }
node, err := state.slotSlaveNode(slot) return state.slotMasterNode(slot)
return slot, node, err
}
node, err := state.slotMasterNode(slot)
return slot, node, err
} }
func (c *ClusterClient) slotMasterNode(slot int) (*clusterNode, error) { func (c *ClusterClient) slotMasterNode(slot int) (*clusterNode, error) {

View File

@ -379,6 +379,8 @@ var _ = Describe("Ring watch", func() {
}) })
ring.ForEachShard(func(cl *redis.Client) error { ring.ForEachShard(func(cl *redis.Client) error {
defer GinkgoRecover()
pool := cl.Pool() pool := cl.Pool()
Expect(pool.Len()).To(BeNumerically("<=", 10)) Expect(pool.Len()).To(BeNumerically("<=", 10))
Expect(pool.IdleLen()).To(BeNumerically("<=", 10)) Expect(pool.IdleLen()).To(BeNumerically("<=", 10))