Retry timeout and retryable error

This commit is contained in:
Vladimir Mihailenco 2020-07-24 14:57:12 +03:00
parent 16e62e05a1
commit 21a1f58caf
5 changed files with 23 additions and 12 deletions

View File

@ -810,7 +810,7 @@ func (c *ClusterClient) _process(ctx context.Context, cmd Cmder) error {
continue continue
} }
if isRetryableError(lastErr, cmd.readTimeout() == nil) { if shouldRetry(lastErr, cmd.readTimeout() == nil) {
// First retry the same node. // First retry the same node.
if attempt == 0 { if attempt == 0 {
continue continue
@ -1466,7 +1466,7 @@ func (c *ClusterClient) Watch(ctx context.Context, fn func(*Tx) error, keys ...s
continue continue
} }
if isRetryableError(err, true) { if shouldRetry(err, true) {
continue continue
} }

View File

@ -24,15 +24,16 @@ type Error interface {
var _ Error = proto.RedisError("") var _ Error = proto.RedisError("")
func isRetryableError(err error, retryTimeout bool) bool { func shouldRetry(err error, retryTimeout bool) bool {
switch err { switch err {
case io.EOF, io.ErrUnexpectedEOF:
return true
case nil, context.Canceled, context.DeadlineExceeded: case nil, context.Canceled, context.DeadlineExceeded:
return false return false
case io.EOF:
return true
} }
if netErr, ok := err.(net.Error); ok {
if netErr.Timeout() { if v, ok := err.(timeoutError); ok {
if v.Timeout() {
return retryTimeout return retryTimeout
} }
return true return true
@ -51,6 +52,7 @@ func isRetryableError(err error, retryTimeout bool) bool {
if strings.HasPrefix(s, "CLUSTERDOWN ") { if strings.HasPrefix(s, "CLUSTERDOWN ") {
return true return true
} }
return false return false
} }
@ -63,16 +65,19 @@ func isBadConn(err error, allowTimeout bool) bool {
if err == nil { if err == nil {
return false return false
} }
if isRedisError(err) { if isRedisError(err) {
// Close connections in read only state in case domain addr is used // Close connections in read only state in case domain addr is used
// and domain resolves to a different Redis Server. See #790. // and domain resolves to a different Redis Server. See #790.
return isReadOnlyError(err) return isReadOnlyError(err)
} }
if allowTimeout { if allowTimeout {
if netErr, ok := err.(net.Error); ok && netErr.Timeout() { if netErr, ok := err.(net.Error); ok && netErr.Timeout() {
return false return !netErr.Temporary()
} }
} }
return true return true
} }
@ -106,3 +111,9 @@ func isLoadingError(err error) bool {
func isReadOnlyError(err error) bool { func isReadOnlyError(err error) bool {
return strings.HasPrefix(err.Error(), "READONLY ") return strings.HasPrefix(err.Error(), "READONLY ")
} }
//------------------------------------------------------------------------------
type timeoutError interface {
Timeout() bool
}

View File

@ -332,7 +332,7 @@ func startSentinel(port, masterName, masterPort string) (*redisProcess, error) {
type badConnError string type badConnError string
func (e badConnError) Error() string { return string(e) } func (e badConnError) Error() string { return string(e) }
func (e badConnError) Timeout() bool { return false } func (e badConnError) Timeout() bool { return true }
func (e badConnError) Temporary() bool { return false } func (e badConnError) Temporary() bool { return false }
type badConn struct { type badConn struct {

View File

@ -345,7 +345,7 @@ func (c *baseClient) _process(ctx context.Context, cmd Cmder) error {
if err == nil { if err == nil {
return nil return nil
} }
retry = isRetryableError(err, retryTimeout) retry = shouldRetry(err, retryTimeout)
return err return err
}) })
if err == nil || !retry { if err == nil || !retry {
@ -430,7 +430,7 @@ func (c *baseClient) _generalProcessPipeline(
canRetry, err = p(ctx, cn, cmds) canRetry, err = p(ctx, cn, cmds)
return err return err
}) })
if lastErr == nil || !canRetry || !isRetryableError(lastErr, true) { if lastErr == nil || !canRetry || !shouldRetry(lastErr, true) {
return lastErr return lastErr
} }
} }

View File

@ -597,7 +597,7 @@ func (c *Ring) _process(ctx context.Context, cmd Cmder) error {
} }
lastErr = shard.Client.Process(ctx, cmd) lastErr = shard.Client.Process(ctx, cmd)
if lastErr == nil || !isRetryableError(lastErr, cmd.readTimeout() == nil) { if lastErr == nil || !shouldRetry(lastErr, cmd.readTimeout() == nil) {
return lastErr return lastErr
} }
} }