From f11aedb948175b0de3b3ec8fa8ef387951abdd9b Mon Sep 17 00:00:00 2001 From: Chris Rice Date: Wed, 28 Jun 2023 14:59:28 -0700 Subject: [PATCH 1/5] To properly support kubernetes & Sentinel. replica_announce properties should be supported #692 --- internal/server/client.go | 1 + internal/server/config.go | 58 ++++++++++++++++++++++++++++++++++++++- internal/server/follow.go | 33 +++++++++++++++++++++- internal/server/stats.go | 3 ++ 4 files changed, 93 insertions(+), 2 deletions(-) diff --git a/internal/server/client.go b/internal/server/client.go index 67fedf75..e6153333 100644 --- a/internal/server/client.go +++ b/internal/server/client.go @@ -16,6 +16,7 @@ import ( type Client struct { id int // unique id replPort int // the known replication port for follower connections + replAddr string // the known replication addr for follower connections authd bool // client has been authenticated outputType Type // Null, JSON, or RESP remoteAddr string // original remote address diff --git a/internal/server/config.go b/internal/server/config.go index 7ec0bcad..9632da8e 100644 --- a/internal/server/config.go +++ b/internal/server/config.go @@ -34,9 +34,11 @@ const ( AutoGC = "autogc" KeepAlive = "keepalive" LogConfig = "logconfig" + AnnounceIP = "replica_announce_ip" + AnnouncePort = "replica_announce_port" ) -var validProperties = []string{RequirePass, LeaderAuth, ProtectedMode, MaxMemory, AutoGC, KeepAlive, LogConfig, ReplicaPriority} +var validProperties = []string{RequirePass, LeaderAuth, ProtectedMode, MaxMemory, AutoGC, KeepAlive, LogConfig, ReplicaPriority, AnnouncePort, AnnounceIP} // Config is a tile38 config type Config struct { @@ -66,6 +68,10 @@ type Config struct { _keepAlive int64 _logConfigP interface{} _logConfig string + _announceIPP string + _announceIP string + _announcePortP string + _announcePort int64 } func loadConfig(path string) (*Config, error) { @@ -94,6 +100,8 @@ func loadConfig(path string) (*Config, error) { _autoGCP: gjson.Get(json, AutoGC).String(), _keepAliveP: gjson.Get(json, KeepAlive).String(), _logConfig: gjson.Get(json, LogConfig).String(), + _announceIPP: gjson.Get(json, AnnounceIP).String(), + _announcePortP: gjson.Get(json, AnnouncePort).String(), } if config._serverID == "" { @@ -131,6 +139,12 @@ func loadConfig(path string) (*Config, error) { if err := config.setProperty(LogConfig, config._logConfig, true); err != nil { return nil, err } + if err := config.setProperty(AnnounceIP, config._announceIPP, true); err != nil { + return nil, err + } + if err := config.setProperty(AnnouncePort, config._announcePortP, true); err != nil { + return nil, err + } config.write(false) return config, nil } @@ -162,6 +176,14 @@ func (config *Config) write(writeProperties bool) { if config._logConfig != "" { config._logConfigP = config._logConfig } + if config._announceIP != "" { + config._announceIPP = config._announceIP + } + if config._announcePort == 0 { + config._announcePortP = "" + } else { + config._announcePortP = strconv.FormatUint(uint64(config._announcePort), 10) + } } m := make(map[string]interface{}) @@ -211,6 +233,12 @@ func (config *Config) write(writeProperties bool) { m[LogConfig] = lcfg } } + if config._announceIPP != "" { + m[AutoGC] = config._announceIPP + } + if config._announcePortP != "" { + m[AutoGC] = config._announcePortP + } data, err := json.MarshalIndent(m, "", "\t") if err != nil { panic(err) @@ -332,6 +360,18 @@ func (config *Config) setProperty(name, value string, fromLoad bool) error { } else { config._replicaPriority = replicaPriority } + case AnnounceIP: + config._announceIP = value + case AnnouncePort: + if value == "" { + config._announcePort = 0 + } else { + announcePort, err := strconv.ParseUint(value, 10, 64) + if err != nil { + invalid = true + } + config._announcePort = int64(announcePort) + } } if invalid { @@ -377,6 +417,10 @@ func (config *Config) getProperty(name string) string { } else { return strconv.FormatUint(uint64(config._replicaPriority), 10) } + case AnnounceIP: + return config._announceIP + case AnnouncePort: + return strconv.FormatUint(uint64(config._announcePort), 10) } } @@ -509,6 +553,18 @@ func (config *Config) keepAlive() int64 { config.mu.RUnlock() return v } +func (config *Config) announceIP() string { + config.mu.RLock() + v := config._announceIP + config.mu.RUnlock() + return v +} +func (config *Config) announcePort() int { + config.mu.RLock() + v := config._announcePort + config.mu.RUnlock() + return int(v) +} func (config *Config) setFollowHost(v string) { config.mu.Lock() config._followHost = v diff --git a/internal/server/follow.go b/internal/server/follow.go index e61759f9..af785eef 100644 --- a/internal/server/follow.go +++ b/internal/server/follow.go @@ -130,6 +130,18 @@ func (s *Server) cmdReplConf(msg *Message, client *Client) (res resp.Value, err return OKMessage(msg, start), nil } } + case "ip-address": + // Apply the replication ip to the client and return + s.connsmu.RLock() + defer s.connsmu.RUnlock() + for _, c := range s.conns { + if c.remoteAddr == client.remoteAddr { + c.mu.Lock() + c.replAddr = val + c.mu.Unlock() + return OKMessage(msg, start), nil + } + } } return NOMessage, fmt.Errorf("cannot find follower") } @@ -231,7 +243,11 @@ func (s *Server) followStep(host string, port int, followc int) error { } // Send the replication port to the leader - v, err := conn.Do("replconf", "listening-port", s.port) + p := s.config.announcePort() + if p == 0 { + p = s.port + } + v, err := conn.Do("replconf", "listening-port", p) if err != nil { return err } @@ -241,6 +257,21 @@ func (s *Server) followStep(host string, port int, followc int) error { if v.String() != "OK" { return errors.New("invalid response to replconf request") } + + // Send the replication ip to the leader + ip := s.config.announceIP() + if ip != "" { + v, err := conn.Do("replconf", "ip-address", ip) + if err != nil { + return err + } + if v.Error() != nil { + return v.Error() + } + if v.String() != "OK" { + return errors.New("invalid response to replconf request") + } + } if s.opts.ShowDebugMessages { log.Debug("follow:", addr, ":replconf") } diff --git a/internal/server/stats.go b/internal/server/stats.go index 9a0ae287..dcba16d5 100644 --- a/internal/server/stats.go +++ b/internal/server/stats.go @@ -430,6 +430,9 @@ func (s *Server) writeInfoStats(w *bytes.Buffer) { func replicaIPAndPort(cc *Client) (ip string, port int) { ip = cc.remoteAddr + if cc.replAddr != "" { + ip = cc.replAddr + } i := strings.LastIndex(ip, ":") if i != -1 { ip = ip[:i] From b554c4c28f989f43e78d719a7f9ae8a4d65dc23f Mon Sep 17 00:00:00 2001 From: Chris Rice Date: Wed, 28 Jun 2023 15:18:23 -0700 Subject: [PATCH 2/5] Update docs to match actual build behavior --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 997d580f..d8a7c869 100644 --- a/README.md +++ b/README.md @@ -70,6 +70,7 @@ Tile38 can be compiled and used on Linux, OSX, Windows, FreeBSD, and probably ot To build everything simply: ``` +$ git tag .. $ make ``` From 9d072ea247bf3c212d2c640ccb1a75974d1fd11b Mon Sep 17 00:00:00 2001 From: Chris Rice Date: Wed, 28 Jun 2023 15:18:23 -0700 Subject: [PATCH 3/5] Bugfix --- README.md | 1 + internal/server/config.go | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 997d580f..d8a7c869 100644 --- a/README.md +++ b/README.md @@ -70,6 +70,7 @@ Tile38 can be compiled and used on Linux, OSX, Windows, FreeBSD, and probably ot To build everything simply: ``` +$ git tag .. $ make ``` diff --git a/internal/server/config.go b/internal/server/config.go index 9632da8e..d8982f57 100644 --- a/internal/server/config.go +++ b/internal/server/config.go @@ -234,10 +234,10 @@ func (config *Config) write(writeProperties bool) { } } if config._announceIPP != "" { - m[AutoGC] = config._announceIPP + m[AnnounceIP] = config._announceIPP } if config._announcePortP != "" { - m[AutoGC] = config._announcePortP + m[AnnouncePort] = config._announcePortP } data, err := json.MarshalIndent(m, "", "\t") if err != nil { From 86d95577904ea9b436cc5cced63f9c9bc625dae3 Mon Sep 17 00:00:00 2001 From: Chris Rice Date: Thu, 29 Jun 2023 08:03:47 -0700 Subject: [PATCH 4/5] Fix --- internal/server/config.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/server/config.go b/internal/server/config.go index d8982f57..7e87f2cd 100644 --- a/internal/server/config.go +++ b/internal/server/config.go @@ -369,8 +369,9 @@ func (config *Config) setProperty(name, value string, fromLoad bool) error { announcePort, err := strconv.ParseUint(value, 10, 64) if err != nil { invalid = true + } else { + config._announcePort = int64(announcePort) } - config._announcePort = int64(announcePort) } } From d898f77388a528b49039ee02a666f894fa97deb5 Mon Sep 17 00:00:00 2001 From: Chris Rice Date: Thu, 29 Jun 2023 11:46:11 -0700 Subject: [PATCH 5/5] Revert "Update docs to match actual build behavior" This reverts commit b554c4c28f989f43e78d719a7f9ae8a4d65dc23f. --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index d8a7c869..997d580f 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,6 @@ Tile38 can be compiled and used on Linux, OSX, Windows, FreeBSD, and probably ot To build everything simply: ``` -$ git tag .. $ make ```