Use PubSub.Channel to not burn CPU on errors

This commit is contained in:
Vladimir Mihailenco 2018-10-01 11:38:21 +03:00
parent 69445c6e87
commit 48e9afe2a7
3 changed files with 108 additions and 96 deletions

View File

@ -20,7 +20,6 @@ func init() {
PoolSize: 10, PoolSize: 10,
PoolTimeout: 30 * time.Second, PoolTimeout: 30 * time.Second,
}) })
// redisdb.FlushDB()
} }
func ExampleNewClient() { func ExampleNewClient() {

View File

@ -51,7 +51,6 @@ func (c *PubSub) _conn(newChannels []string) (*pool.Conn, error) {
if c.closed { if c.closed {
return nil, pool.ErrClosed return nil, pool.ErrClosed
} }
if c.cn != nil { if c.cn != nil {
return c.cn, nil return c.cn, nil
} }

View File

@ -119,7 +119,7 @@ func NewSentinelClient(opt *Options) *SentinelClient {
return c return c
} }
func (c *SentinelClient) PubSub() *PubSub { func (c *SentinelClient) pubSub() *PubSub {
pubsub := &PubSub{ pubsub := &PubSub{
opt: c.opt, opt: c.opt,
@ -132,14 +132,34 @@ func (c *SentinelClient) PubSub() *PubSub {
return pubsub return pubsub
} }
// Subscribe subscribes the client to the specified channels.
// Channels can be omitted to create empty subscription.
func (c *SentinelClient) Subscribe(channels ...string) *PubSub {
pubsub := c.pubSub()
if len(channels) > 0 {
_ = pubsub.Subscribe(channels...)
}
return pubsub
}
// PSubscribe subscribes the client to the given patterns.
// Patterns can be omitted to create empty subscription.
func (c *SentinelClient) PSubscribe(channels ...string) *PubSub {
pubsub := c.pubSub()
if len(channels) > 0 {
_ = pubsub.PSubscribe(channels...)
}
return pubsub
}
func (c *SentinelClient) GetMasterAddrByName(name string) *StringSliceCmd { func (c *SentinelClient) GetMasterAddrByName(name string) *StringSliceCmd {
cmd := NewStringSliceCmd("SENTINEL", "get-master-addr-by-name", name) cmd := NewStringSliceCmd("sentinel", "get-master-addr-by-name", name)
c.Process(cmd) c.Process(cmd)
return cmd return cmd
} }
func (c *SentinelClient) Sentinels(name string) *SliceCmd { func (c *SentinelClient) Sentinels(name string) *SliceCmd {
cmd := NewSliceCmd("SENTINEL", "sentinels", name) cmd := NewSliceCmd("sentinel", "sentinels", name)
c.Process(cmd) c.Process(cmd)
return cmd return cmd
} }
@ -158,77 +178,70 @@ type sentinelFailover struct {
sentinel *SentinelClient sentinel *SentinelClient
} }
func (d *sentinelFailover) Close() error { func (c *sentinelFailover) Close() error {
return d.resetSentinel() c.mu.Lock()
defer c.mu.Unlock()
return c.closeSentinel()
} }
func (d *sentinelFailover) Pool() *pool.ConnPool { func (c *sentinelFailover) Pool() *pool.ConnPool {
d.poolOnce.Do(func() { c.poolOnce.Do(func() {
d.opt.Dialer = d.dial c.opt.Dialer = c.dial
d.pool = newConnPool(d.opt) c.pool = newConnPool(c.opt)
}) })
return d.pool return c.pool
} }
func (d *sentinelFailover) dial() (net.Conn, error) { func (c *sentinelFailover) dial() (net.Conn, error) {
addr, err := d.MasterAddr() addr, err := c.MasterAddr()
if err != nil { if err != nil {
return nil, err return nil, err
} }
return net.DialTimeout("tcp", addr, d.opt.DialTimeout) return net.DialTimeout("tcp", addr, c.opt.DialTimeout)
} }
func (d *sentinelFailover) MasterAddr() (string, error) { func (c *sentinelFailover) MasterAddr() (string, error) {
d.mu.Lock() addr, err := c.masterAddr()
defer d.mu.Unlock()
addr, err := d.masterAddr()
if err != nil { if err != nil {
return "", err return "", err
} }
d._switchMaster(addr) c.switchMaster(addr)
return addr, nil return addr, nil
} }
func (d *sentinelFailover) masterAddr() (string, error) { func (c *sentinelFailover) masterAddr() (string, error) {
// Try last working sentinel. addr := c.getMasterAddr()
if d.sentinel != nil { if addr != "" {
addr, err := d.sentinel.GetMasterAddrByName(d.masterName).Result()
if err == nil {
addr := net.JoinHostPort(addr[0], addr[1])
return addr, nil return addr, nil
} }
internal.Logf("sentinel: GetMasterAddrByName name=%q failed: %s", c.mu.Lock()
d.masterName, err) defer c.mu.Unlock()
d._resetSentinel()
}
for i, sentinelAddr := range d.sentinelAddrs { for i, sentinelAddr := range c.sentinelAddrs {
sentinel := NewSentinelClient(&Options{ sentinel := NewSentinelClient(&Options{
Addr: sentinelAddr, Addr: sentinelAddr,
DialTimeout: d.opt.DialTimeout, DialTimeout: c.opt.DialTimeout,
ReadTimeout: d.opt.ReadTimeout, ReadTimeout: c.opt.ReadTimeout,
WriteTimeout: d.opt.WriteTimeout, WriteTimeout: c.opt.WriteTimeout,
PoolSize: d.opt.PoolSize, PoolSize: c.opt.PoolSize,
PoolTimeout: d.opt.PoolTimeout, PoolTimeout: c.opt.PoolTimeout,
IdleTimeout: d.opt.IdleTimeout, IdleTimeout: c.opt.IdleTimeout,
}) })
masterAddr, err := sentinel.GetMasterAddrByName(d.masterName).Result() masterAddr, err := sentinel.GetMasterAddrByName(c.masterName).Result()
if err != nil { if err != nil {
internal.Logf("sentinel: GetMasterAddrByName master=%q failed: %s", internal.Logf("sentinel: GetMasterAddrByName master=%q failed: %s",
d.masterName, err) c.masterName, err)
sentinel.Close() sentinel.Close()
continue continue
} }
// Push working sentinel to the top. // Push working sentinel to the top.
d.sentinelAddrs[0], d.sentinelAddrs[i] = d.sentinelAddrs[i], d.sentinelAddrs[0] c.sentinelAddrs[0], c.sentinelAddrs[i] = c.sentinelAddrs[i], c.sentinelAddrs[0]
d.setSentinel(sentinel) c.setSentinel(sentinel)
addr := net.JoinHostPort(masterAddr[0], masterAddr[1]) addr := net.JoinHostPort(masterAddr[0], masterAddr[1])
return addr, nil return addr, nil
@ -237,17 +250,41 @@ func (d *sentinelFailover) masterAddr() (string, error) {
return "", errors.New("redis: all sentinels are unreachable") return "", errors.New("redis: all sentinels are unreachable")
} }
func (c *sentinelFailover) switchMaster(addr string) { func (c *sentinelFailover) getMasterAddr() string {
c.mu.RLock()
sentinel := c.sentinel
c.mu.RUnlock()
if sentinel == nil {
return ""
}
addr, err := sentinel.GetMasterAddrByName(c.masterName).Result()
if err != nil {
internal.Logf("sentinel: GetMasterAddrByName name=%q failed: %s",
c.masterName, err)
c.mu.Lock() c.mu.Lock()
c._switchMaster(addr) if c.sentinel == sentinel {
c.closeSentinel()
}
c.mu.Unlock() c.mu.Unlock()
return ""
}
return net.JoinHostPort(addr[0], addr[1])
} }
func (c *sentinelFailover) _switchMaster(addr string) { func (c *sentinelFailover) switchMaster(addr string) {
if c._masterAddr == addr { c.mu.RLock()
masterAddr := c._masterAddr
c.mu.RUnlock()
if masterAddr == addr {
return return
} }
c.mu.Lock()
defer c.mu.Unlock()
internal.Logf("sentinel: new master=%q addr=%q", internal.Logf("sentinel: new master=%q addr=%q",
c.masterName, addr) c.masterName, addr)
_ = c.Pool().Filter(func(cn *pool.Conn) bool { _ = c.Pool().Filter(func(cn *pool.Conn) bool {
@ -256,32 +293,22 @@ func (c *sentinelFailover) _switchMaster(addr string) {
c._masterAddr = addr c._masterAddr = addr
} }
func (d *sentinelFailover) setSentinel(sentinel *SentinelClient) { func (c *sentinelFailover) setSentinel(sentinel *SentinelClient) {
d.discoverSentinels(sentinel) c.discoverSentinels(sentinel)
d.sentinel = sentinel c.sentinel = sentinel
go d.listen(sentinel) go c.listen(sentinel)
} }
func (d *sentinelFailover) resetSentinel() error { func (c *sentinelFailover) closeSentinel() error {
var err error err := c.sentinel.Close()
d.mu.Lock() c.sentinel = nil
if d.sentinel != nil {
err = d._resetSentinel()
}
d.mu.Unlock()
return err return err
} }
func (d *sentinelFailover) _resetSentinel() error { func (c *sentinelFailover) discoverSentinels(sentinel *SentinelClient) {
err := d.sentinel.Close() sentinels, err := sentinel.Sentinels(c.masterName).Result()
d.sentinel = nil
return err
}
func (d *sentinelFailover) discoverSentinels(sentinel *SentinelClient) {
sentinels, err := sentinel.Sentinels(d.masterName).Result()
if err != nil { if err != nil {
internal.Logf("sentinel: Sentinels master=%q failed: %s", d.masterName, err) internal.Logf("sentinel: Sentinels master=%q failed: %s", c.masterName, err)
return return
} }
for _, sentinel := range sentinels { for _, sentinel := range sentinels {
@ -290,49 +317,36 @@ func (d *sentinelFailover) discoverSentinels(sentinel *SentinelClient) {
key := vals[i].(string) key := vals[i].(string)
if key == "name" { if key == "name" {
sentinelAddr := vals[i+1].(string) sentinelAddr := vals[i+1].(string)
if !contains(d.sentinelAddrs, sentinelAddr) { if !contains(c.sentinelAddrs, sentinelAddr) {
internal.Logf( internal.Logf("sentinel: discovered new sentinel=%q for master=%q",
"sentinel: discovered new sentinel=%q for master=%q", sentinelAddr, c.masterName)
sentinelAddr, d.masterName, c.sentinelAddrs = append(c.sentinelAddrs, sentinelAddr)
)
d.sentinelAddrs = append(d.sentinelAddrs, sentinelAddr)
} }
} }
} }
} }
} }
func (d *sentinelFailover) listen(sentinel *SentinelClient) { func (c *sentinelFailover) listen(sentinel *SentinelClient) {
pubsub := sentinel.PubSub() pubsub := sentinel.Subscribe("+switch-master")
defer pubsub.Close() defer pubsub.Close()
err := pubsub.Subscribe("+switch-master") ch := pubsub.Channel()
if err != nil {
internal.Logf("sentinel: Subscribe failed: %s", err)
d.resetSentinel()
return
}
for { for {
msg, err := pubsub.ReceiveMessage() msg, ok := <-ch
if err != nil { if !ok {
if err == pool.ErrClosed { break
d.resetSentinel()
return
}
internal.Logf("sentinel: ReceiveMessage failed: %s", err)
continue
} }
switch msg.Channel { switch msg.Channel {
case "+switch-master": case "+switch-master":
parts := strings.Split(msg.Payload, " ") parts := strings.Split(msg.Payload, " ")
if parts[0] != d.masterName { if parts[0] != c.masterName {
internal.Logf("sentinel: ignore addr for master=%q", parts[0]) internal.Logf("sentinel: ignore addr for master=%q", parts[0])
continue continue
} }
addr := net.JoinHostPort(parts[3], parts[4]) addr := net.JoinHostPort(parts[3], parts[4])
d.switchMaster(addr) c.switchMaster(addr)
} }
} }
} }