diff --git a/cmd/tile38-server/main.go b/cmd/tile38-server/main.go index cbc3d241..58a1fd54 100644 --- a/cmd/tile38-server/main.go +++ b/cmd/tile38-server/main.go @@ -141,12 +141,33 @@ Developer Options: } var ( - devMode bool nohup bool showEvioDisabled bool showThreadsDisabled bool ) + var ( + // use to be in core/options.go + + // DevMode puts application in to dev mode + devMode = false + + // ShowDebugMessages allows for log.Debug to print to console. + showDebugMessages = false + + // ProtectedMode forces Tile38 to default in protected mode. + protectedMode = "no" + + // AppendOnly allows for disabling the appendonly file. + appendOnly = true + + // AppendFileName allows for custom appendonly file path + appendFileName = "" + + // QueueFileName allows for custom queue.db file path + queueFileName = "" + ) + // parse non standard args. nargs := []string{os.Args[0]} for i := 1; i < len(os.Args); i++ { @@ -163,10 +184,10 @@ Developer Options: if i < len(os.Args) { switch strings.ToLower(os.Args[i]) { case "no": - core.ProtectedMode = "no" + protectedMode = "no" continue case "yes": - core.ProtectedMode = "yes" + protectedMode = "yes" continue } } @@ -183,10 +204,10 @@ Developer Options: if i < len(os.Args) { switch strings.ToLower(os.Args[i]) { case "no": - core.AppendOnly = false + appendOnly = false continue case "yes": - core.AppendOnly = true + appendOnly = true continue } } @@ -198,14 +219,14 @@ Developer Options: fmt.Fprintf(os.Stderr, "appendfilename must have a value\n") os.Exit(1) } - core.AppendFileName = os.Args[i] + appendFileName = os.Args[i] case "--queuefilename", "-queuefilename": i++ if i == len(os.Args) || os.Args[i] == "" { fmt.Fprintf(os.Stderr, "queuefilename must have a value\n") os.Exit(1) } - core.QueueFileName = os.Args[i] + queueFileName = os.Args[i] case "--http-transport", "-http-transport": i++ if i < len(os.Args) { @@ -281,7 +302,7 @@ Developer Options: flag.Parse() if logEncoding == "json" { - log.LogJSON = true + log.SetLogJSON(true) data, _ := os.ReadFile(filepath.Join(dir, "config")) if gjson.GetBytes(data, "logconfig.encoding").String() == "json" { c := gjson.GetBytes(data, "logconfig").String() @@ -299,17 +320,16 @@ Developer Options: log.SetOutput(logw) if quiet { - log.Level = 0 + log.SetLevel(0) } else if veryVerbose { - log.Level = 3 + log.SetLevel(3) } else if verbose { - log.Level = 2 + log.SetLevel(2) } else { - log.Level = 1 + log.SetLevel(1) } - core.DevMode = devMode - core.ShowDebugMessages = veryVerbose + showDebugMessages = veryVerbose hostd := "" if host != "" { @@ -425,7 +445,7 @@ Developer Options: saddr = fmt.Sprintf("Port: %d", port) } - if log.LogJSON { + if log.LogJSON() { log.Printf(`Tile38 %s%s %d bit (%s/%s) %s%s, PID: %d. Visit tile38.com/sponsor to support the project`, core.Version, gitsha, strconv.IntSize, runtime.GOARCH, runtime.GOOS, hostd, saddr, os.Getpid()) } else { @@ -449,12 +469,18 @@ Developer Options: log.Warnf("thread flag is deprecated use GOMAXPROCS to set number of threads instead") } opts := server.Options{ - Host: host, - Port: port, - Dir: dir, - UseHTTP: httpTransport, - MetricsAddr: *metricsAddr, - UnixSocketPath: unixSocket, + Host: host, + Port: port, + Dir: dir, + UseHTTP: httpTransport, + MetricsAddr: *metricsAddr, + UnixSocketPath: unixSocket, + DevMode: devMode, + ShowDebugMessages: showDebugMessages, + ProtectedMode: protectedMode, + AppendOnly: appendOnly, + AppendFileName: appendFileName, + QueueFileName: queueFileName, } if err := server.Serve(opts); err != nil { log.Fatal(err) diff --git a/core/options.go b/core/options.go deleted file mode 100644 index 76e2fcfd..00000000 --- a/core/options.go +++ /dev/null @@ -1,19 +0,0 @@ -package core - -// DevMode puts application in to dev mode -var DevMode = false - -// ShowDebugMessages allows for log.Debug to print to console. -var ShowDebugMessages = false - -// ProtectedMode forces Tile38 to default in protected mode. -var ProtectedMode = "no" - -// AppendOnly allows for disabling the appendonly file. -var AppendOnly = true - -// AppendFileName allows for custom appendonly file path -var AppendFileName = "" - -// QueueFileName allows for custom queue.db file path -var QueueFileName = "" diff --git a/go.mod b/go.mod index 34bc5063..cdd6d812 100644 --- a/go.mod +++ b/go.mod @@ -22,6 +22,7 @@ require ( github.com/tidwall/geojson v1.3.6 github.com/tidwall/gjson v1.14.3 github.com/tidwall/hashmap v1.6.1 + github.com/tidwall/limiter v0.4.0 github.com/tidwall/match v1.1.1 github.com/tidwall/pretty v1.2.0 github.com/tidwall/redbench v0.1.0 @@ -31,6 +32,7 @@ require ( github.com/tidwall/sjson v1.2.4 github.com/xdg/scram v1.0.5 github.com/yuin/gopher-lua v0.0.0-20200816102855-ee81675732da + go.uber.org/atomic v1.5.0 go.uber.org/zap v1.13.0 golang.org/x/net v0.0.0-20220809184613-07c6da5e1ced golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 @@ -95,7 +97,6 @@ require ( github.com/tidwall/tinyqueue v0.1.1 // indirect github.com/xdg/stringprep v1.0.3 // indirect go.opencensus.io v0.22.4 // indirect - go.uber.org/atomic v1.5.0 // indirect go.uber.org/multierr v1.3.0 // indirect go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee // indirect golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa // indirect diff --git a/go.sum b/go.sum index 8674dd65..2c0705b3 100644 --- a/go.sum +++ b/go.sum @@ -368,6 +368,8 @@ github.com/tidwall/grect v0.1.4 h1:dA3oIgNgWdSspFzn1kS4S/RDpZFLrIxAZOdJKjYapOg= github.com/tidwall/grect v0.1.4/go.mod h1:9FBsaYRaR0Tcy4UwefBX/UDcDcDy9V5jUcxHzv2jd5Q= github.com/tidwall/hashmap v1.6.1 h1:FIAHjKwcyOo1Y3/orsQO08floKhInbEX2VQv7CQRNuw= github.com/tidwall/hashmap v1.6.1/go.mod h1:hX452N3VtFD8okD3/6q/yOquJvJmYxmZ1H0nLtwkaxM= +github.com/tidwall/limiter v0.4.0 h1:nj+7mS6aMDRzp15QTVDrgkun0def5/PfB4ogs5NlIVQ= +github.com/tidwall/limiter v0.4.0/go.mod h1:n+qBGuSOgAvgcq1xUvo+mXWg8oBLQC8wkkheN9KZou0= github.com/tidwall/lotsa v1.0.2 h1:dNVBH5MErdaQ/xd9s769R31/n2dXavsQ0Yf4TMEHHw8= github.com/tidwall/lotsa v1.0.2/go.mod h1:X6NiU+4yHA3fE3Puvpnn1XMDrFZrE9JO2/w+UMuqgR8= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= diff --git a/internal/endpoint/amqp.go b/internal/endpoint/amqp.go index 546dee78..b69a22ef 100644 --- a/internal/endpoint/amqp.go +++ b/internal/endpoint/amqp.go @@ -27,13 +27,21 @@ func (conn *AMQPConn) Expired() bool { defer conn.mu.Unlock() if !conn.ex { if time.Since(conn.t) > amqpExpiresAfter { - conn.ex = true conn.close() + conn.ex = true } } return conn.ex } +// ExpireNow forces the connection to expire +func (conn *AMQPConn) ExpireNow() { + conn.mu.Lock() + defer conn.mu.Unlock() + conn.close() + conn.ex = true +} + func (conn *AMQPConn) close() { if conn.conn != nil { conn.conn.Close() diff --git a/internal/endpoint/disque.go b/internal/endpoint/disque.go index 35300ec6..664262f8 100644 --- a/internal/endpoint/disque.go +++ b/internal/endpoint/disque.go @@ -33,15 +33,21 @@ func (conn *DisqueConn) Expired() bool { defer conn.mu.Unlock() if !conn.ex { if time.Since(conn.t) > disqueExpiresAfter { - if conn.conn != nil { - conn.close() - } + conn.close() conn.ex = true } } return conn.ex } +// ExpireNow forces the connection to expire +func (conn *DisqueConn) ExpireNow() { + conn.mu.Lock() + defer conn.mu.Unlock() + conn.close() + conn.ex = true +} + func (conn *DisqueConn) close() { if conn.conn != nil { conn.conn.Close() diff --git a/internal/endpoint/endpoint.go b/internal/endpoint/endpoint.go index 8e5d1274..f13feeb7 100644 --- a/internal/endpoint/endpoint.go +++ b/internal/endpoint/endpoint.go @@ -6,6 +6,7 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "time" "github.com/streadway/amqp" @@ -136,6 +137,7 @@ type Endpoint struct { // Conn is an endpoint connection type Conn interface { + ExpireNow() Expired() bool Send(val string) error } @@ -145,6 +147,8 @@ type Manager struct { mu sync.RWMutex conns map[string]Conn publisher LocalPublisher + shutdown int32 // atomic bool + wg sync.WaitGroup // run wait group } // NewManager returns a new manager @@ -153,13 +157,29 @@ func NewManager(publisher LocalPublisher) *Manager { conns: make(map[string]Conn), publisher: publisher, } - go epc.Run() + epc.wg.Add(1) + go epc.run() return epc } +func (epc *Manager) Shutdown() { + defer epc.wg.Wait() + atomic.StoreInt32(&epc.shutdown, 1) + // expire the connections + epc.mu.Lock() + defer epc.mu.Unlock() + for _, conn := range epc.conns { + conn.ExpireNow() + } +} + // Run starts the managing of endpoints -func (epc *Manager) Run() { +func (epc *Manager) run() { + defer epc.wg.Done() for { + if atomic.LoadInt32(&epc.shutdown) != 0 { + return + } time.Sleep(time.Second) func() { epc.mu.Lock() diff --git a/internal/endpoint/eventHub.go b/internal/endpoint/eventHub.go index 5de50616..83ea0775 100644 --- a/internal/endpoint/eventHub.go +++ b/internal/endpoint/eventHub.go @@ -28,6 +28,10 @@ func (conn *EvenHubConn) Expired() bool { return false } +// ExpireNow forces the connection to expire +func (conn *EvenHubConn) ExpireNow() { +} + // Send sends a message func (conn *EvenHubConn) Send(msg string) error { hub, err := eventhub.NewHubFromConnectionString(conn.ep.EventHub.ConnectionString) diff --git a/internal/endpoint/grpc.go b/internal/endpoint/grpc.go index 12de3d7e..d40a53fa 100644 --- a/internal/endpoint/grpc.go +++ b/internal/endpoint/grpc.go @@ -36,14 +36,21 @@ func (conn *GRPCConn) Expired() bool { defer conn.mu.Unlock() if !conn.ex { if time.Since(conn.t) > grpcExpiresAfter { - if conn.conn != nil { - conn.close() - } + conn.close() conn.ex = true } } return conn.ex } + +// ExpireNow forces the connection to expire +func (conn *GRPCConn) ExpireNow() { + conn.mu.Lock() + defer conn.mu.Unlock() + conn.close() + conn.ex = true +} + func (conn *GRPCConn) close() { if conn.conn != nil { conn.conn.Close() diff --git a/internal/endpoint/http.go b/internal/endpoint/http.go index 52deba24..c89bb986 100644 --- a/internal/endpoint/http.go +++ b/internal/endpoint/http.go @@ -38,6 +38,10 @@ func (conn *HTTPConn) Expired() bool { return false } +// ExpireNow forces the connection to expire +func (conn *HTTPConn) ExpireNow() { +} + // Send sends a message func (conn *HTTPConn) Send(msg string) error { req, err := http.NewRequest("POST", conn.ep.Original, bytes.NewBufferString(msg)) diff --git a/internal/endpoint/kafka.go b/internal/endpoint/kafka.go index ef083380..4379d24d 100644 --- a/internal/endpoint/kafka.go +++ b/internal/endpoint/kafka.go @@ -34,15 +34,21 @@ func (conn *KafkaConn) Expired() bool { defer conn.mu.Unlock() if !conn.ex { if time.Since(conn.t) > kafkaExpiresAfter { - if conn.conn != nil { - conn.close() - } + conn.close() conn.ex = true } } return conn.ex } +// ExpireNow forces the connection to expire +func (conn *KafkaConn) ExpireNow() { + conn.mu.Lock() + defer conn.mu.Unlock() + conn.close() + conn.ex = true +} + func (conn *KafkaConn) close() { if conn.conn != nil { conn.conn.Close() @@ -62,7 +68,7 @@ func (conn *KafkaConn) Send(msg string) error { } conn.t = time.Now() - if log.Level > 2 { + if log.Level() > 2 { sarama.Logger = lg.New(log.Output(), "[sarama] ", 0) } diff --git a/internal/endpoint/local.go b/internal/endpoint/local.go index 1038fd79..2c1c7d6d 100644 --- a/internal/endpoint/local.go +++ b/internal/endpoint/local.go @@ -23,6 +23,10 @@ func (conn *LocalConn) Expired() bool { return false } +// ExpireNow forces the connection to expire +func (conn *LocalConn) ExpireNow() { +} + // Send sends a message func (conn *LocalConn) Send(msg string) error { conn.publisher.Publish(conn.ep.Local.Channel, msg) diff --git a/internal/endpoint/mqtt.go b/internal/endpoint/mqtt.go index 5f80765c..d7410649 100644 --- a/internal/endpoint/mqtt.go +++ b/internal/endpoint/mqtt.go @@ -40,12 +40,19 @@ func (conn *MQTTConn) Expired() bool { return conn.ex } +// ExpireNow forces the connection to expire +func (conn *MQTTConn) ExpireNow() { + conn.mu.Lock() + defer conn.mu.Unlock() + conn.close() + conn.ex = true +} + func (conn *MQTTConn) close() { if conn.conn != nil { if conn.conn.IsConnected() { conn.conn.Disconnect(250) } - conn.conn = nil } } diff --git a/internal/endpoint/nats.go b/internal/endpoint/nats.go index 6c0c7815..ae254802 100644 --- a/internal/endpoint/nats.go +++ b/internal/endpoint/nats.go @@ -32,15 +32,21 @@ func (conn *NATSConn) Expired() bool { defer conn.mu.Unlock() if !conn.ex { if time.Since(conn.t) > natsExpiresAfter { - if conn.conn != nil { - conn.close() - } + conn.close() conn.ex = true } } return conn.ex } +// ExpireNow forces the connection to expire +func (conn *NATSConn) ExpireNow() { + conn.mu.Lock() + defer conn.mu.Unlock() + conn.close() + conn.ex = true +} + func (conn *NATSConn) close() { if conn.conn != nil { conn.conn.Close() diff --git a/internal/endpoint/pubsub.go b/internal/endpoint/pubsub.go index db6dac33..b4926474 100644 --- a/internal/endpoint/pubsub.go +++ b/internal/endpoint/pubsub.go @@ -83,13 +83,21 @@ func (conn *PubSubConn) Expired() bool { defer conn.mu.Unlock() if !conn.ex { if time.Since(conn.t) > pubsubExpiresAfter { - conn.ex = true conn.close() + conn.ex = true } } return conn.ex } +// ExpireNow forces the connection to expire +func (conn *PubSubConn) ExpireNow() { + conn.mu.Lock() + defer conn.mu.Unlock() + conn.close() + conn.ex = true +} + func newPubSubConn(ep Endpoint) *PubSubConn { return &PubSubConn{ ep: ep, diff --git a/internal/endpoint/redis.go b/internal/endpoint/redis.go index 8064d166..d5101b57 100644 --- a/internal/endpoint/redis.go +++ b/internal/endpoint/redis.go @@ -32,15 +32,21 @@ func (conn *RedisConn) Expired() bool { defer conn.mu.Unlock() if !conn.ex { if time.Since(conn.t) > redisExpiresAfter { - if conn.conn != nil { - conn.close() - } + conn.close() conn.ex = true } } return conn.ex } +// ExpireNow forces the connection to expire +func (conn *RedisConn) ExpireNow() { + conn.mu.Lock() + defer conn.mu.Unlock() + conn.close() + conn.ex = true +} + func (conn *RedisConn) close() { if conn.conn != nil { conn.conn.Close() diff --git a/internal/endpoint/sqs.go b/internal/endpoint/sqs.go index 8a1098b7..fae871e5 100644 --- a/internal/endpoint/sqs.go +++ b/internal/endpoint/sqs.go @@ -39,13 +39,21 @@ func (conn *SQSConn) Expired() bool { defer conn.mu.Unlock() if !conn.ex { if time.Since(conn.t) > sqsExpiresAfter { - conn.ex = true conn.close() + conn.ex = true } } return conn.ex } +// ExpireNow forces the connection to expire +func (conn *SQSConn) ExpireNow() { + conn.mu.Lock() + defer conn.mu.Unlock() + conn.close() + conn.ex = true +} + func (conn *SQSConn) close() { if conn.svc != nil { conn.svc = nil @@ -82,7 +90,7 @@ func (conn *SQSConn) Send(msg string) error { sess := session.Must(session.NewSession(&aws.Config{ Region: ®ion, Credentials: creds, - CredentialsChainVerboseErrors: aws.Bool(log.Level >= 3), + CredentialsChainVerboseErrors: aws.Bool(log.Level() >= 3), MaxRetries: aws.Int(5), })) svc := sqs.New(sess) diff --git a/internal/log/log.go b/internal/log/log.go index 4250d84c..16d8df87 100644 --- a/internal/log/log.go +++ b/internal/log/log.go @@ -6,91 +6,120 @@ import ( "io" "os" "sync" + "sync/atomic" "time" "go.uber.org/zap" "golang.org/x/term" ) -var mu sync.Mutex +var wmu sync.Mutex var wr io.Writer -var tty bool -var LogJSON = false -var logger *zap.SugaredLogger + +var zmu sync.Mutex +var zlogger *zap.SugaredLogger + +var tty atomic.Bool +var ljson atomic.Bool +var llevel atomic.Int32 + +func init() { + SetOutput(os.Stderr) + SetLevel(1) +} // Level is the log level // 0: silent - do not log // 1: normal - show everything except debug and warn // 2: verbose - show everything except debug // 3: very verbose - show everything -var Level = 1 +func SetLevel(level int) { + if level < 0 { + level = 0 + } else if level > 3 { + level = 3 + } + llevel.Store(int32(level)) +} + +// Level returns the log level +func Level() int { + return int(llevel.Load()) +} + +func SetLogJSON(logJSON bool) { + ljson.Store(logJSON) +} + +func LogJSON() bool { + return ljson.Load() +} // SetOutput sets the output of the logger func SetOutput(w io.Writer) { f, ok := w.(*os.File) - tty = ok && term.IsTerminal(int(f.Fd())) + tty.Store(ok && term.IsTerminal(int(f.Fd()))) + wmu.Lock() wr = w + wmu.Unlock() } // Build a zap logger from default or custom config func Build(c string) error { + var zcfg zap.Config if c == "" { - zcfg := zap.NewProductionConfig() + zcfg = zap.NewProductionConfig() // to be able to filter with Tile38 levels zcfg.Level.SetLevel(zap.DebugLevel) // disable caller because caller is always log.go zcfg.DisableCaller = true - core, err := zcfg.Build() - if err != nil { - return err - } - defer core.Sync() - logger = core.Sugar() } else { - var zcfg zap.Config err := json.Unmarshal([]byte(c), &zcfg) if err != nil { return err } - // to be able to filter with Tile38 levels zcfg.Level.SetLevel(zap.DebugLevel) // disable caller because caller is always log.go zcfg.DisableCaller = true - - core, err := zcfg.Build() - if err != nil { - return err - } - defer core.Sync() - logger = core.Sugar() } + core, err := zcfg.Build() + if err != nil { + return err + } + defer core.Sync() + zmu.Lock() + zlogger = core.Sugar() + zmu.Unlock() return nil } // Set a zap logger func Set(sl *zap.SugaredLogger) { - logger = sl + zmu.Lock() + zlogger = sl + zmu.Unlock() } // Get a zap logger func Get() *zap.SugaredLogger { - return logger + zmu.Lock() + sl := zlogger + zmu.Unlock() + return sl } -func init() { - SetOutput(os.Stderr) -} - -// Output retuns the output writer +// Output returns the output writer func Output() io.Writer { + wmu.Lock() + defer wmu.Unlock() return wr } func log(level int, tag, color string, formatted bool, format string, args ...interface{}) { - if Level < level { + if llevel.Load() < int32(level) { return } var msg string @@ -99,30 +128,32 @@ func log(level int, tag, color string, formatted bool, format string, args ...in } else { msg = fmt.Sprint(args...) } - if LogJSON { + if ljson.Load() { + zmu.Lock() + defer zmu.Unlock() switch tag { case "ERRO": - logger.Error(msg) + zlogger.Error(msg) case "FATA": - logger.Fatal(msg) + zlogger.Fatal(msg) case "WARN": - logger.Warn(msg) + zlogger.Warn(msg) case "DEBU": - logger.Debug(msg) + zlogger.Debug(msg) default: - logger.Info(msg) + zlogger.Info(msg) } return } s := []byte(time.Now().Format("2006/01/02 15:04:05")) s = append(s, ' ') - if tty { + if tty.Load() { s = append(s, color...) } s = append(s, '[') s = append(s, tag...) s = append(s, ']') - if tty { + if tty.Load() { s = append(s, "\x1b[0m"...) } s = append(s, ' ') @@ -130,79 +161,79 @@ func log(level int, tag, color string, formatted bool, format string, args ...in if s[len(s)-1] != '\n' { s = append(s, '\n') } - mu.Lock() + wmu.Lock() wr.Write(s) - mu.Unlock() + wmu.Unlock() } var emptyFormat string // Infof ... func Infof(format string, args ...interface{}) { - if Level >= 1 { + if llevel.Load() >= 1 { log(1, "INFO", "\x1b[36m", true, format, args...) } } // Info ... func Info(args ...interface{}) { - if Level >= 1 { + if llevel.Load() >= 1 { log(1, "INFO", "\x1b[36m", false, emptyFormat, args...) } } // HTTPf ... func HTTPf(format string, args ...interface{}) { - if Level >= 1 { + if llevel.Load() >= 1 { log(1, "HTTP", "\x1b[1m\x1b[30m", true, format, args...) } } // HTTP ... func HTTP(args ...interface{}) { - if Level >= 1 { + if llevel.Load() >= 1 { log(1, "HTTP", "\x1b[1m\x1b[30m", false, emptyFormat, args...) } } // Errorf ... func Errorf(format string, args ...interface{}) { - if Level >= 1 { + if llevel.Load() >= 1 { log(1, "ERRO", "\x1b[1m\x1b[31m", true, format, args...) } } // Error .. func Error(args ...interface{}) { - if Level >= 1 { + if llevel.Load() >= 1 { log(1, "ERRO", "\x1b[1m\x1b[31m", false, emptyFormat, args...) } } // Warnf ... func Warnf(format string, args ...interface{}) { - if Level >= 1 { + if llevel.Load() >= 1 { log(2, "WARN", "\x1b[33m", true, format, args...) } } // Warn ... func Warn(args ...interface{}) { - if Level >= 1 { + if llevel.Load() >= 1 { log(2, "WARN", "\x1b[33m", false, emptyFormat, args...) } } // Debugf ... func Debugf(format string, args ...interface{}) { - if Level >= 3 { + if llevel.Load() >= 3 { log(3, "DEBU", "\x1b[35m", true, format, args...) } } // Debug ... func Debug(args ...interface{}) { - if Level >= 3 { + if llevel.Load() >= 3 { log(3, "DEBU", "\x1b[35m", false, emptyFormat, args...) } } diff --git a/internal/log/log_test.go b/internal/log/log_test.go index 6c86128d..75f30be9 100644 --- a/internal/log/log_test.go +++ b/internal/log/log_test.go @@ -13,7 +13,7 @@ import ( func TestLog(t *testing.T) { f := &bytes.Buffer{} - LogJSON = false + SetLogJSON(false) SetOutput(f) Printf("hello %v", "everyone") if !strings.HasSuffix(f.String(), "hello everyone\n") { @@ -23,7 +23,7 @@ func TestLog(t *testing.T) { func TestLogJSON(t *testing.T) { - LogJSON = true + SetLogJSON(true) Build("") type tcase struct { @@ -40,7 +40,7 @@ func TestLogJSON(t *testing.T) { return func(t *testing.T) { observedZapCore, observedLogs := observer.New(zap.DebugLevel) Set(zap.New(observedZapCore).Sugar()) - Level = tc.level + SetLevel(tc.level) if tc.format != "" { tc.fops(tc.format, tc.args) @@ -187,8 +187,8 @@ func TestLogJSON(t *testing.T) { } func BenchmarkLogPrintf(t *testing.B) { - LogJSON = false - Level = 1 + SetLogJSON(false) + SetLevel(1) SetOutput(io.Discard) t.ResetTimer() for i := 0; i < t.N; i++ { @@ -197,8 +197,8 @@ func BenchmarkLogPrintf(t *testing.B) { } func BenchmarkLogJSONPrintf(t *testing.B) { - LogJSON = true - Level = 1 + SetLogJSON(true) + SetLevel(1) ec := zap.NewProductionEncoderConfig() ec.EncodeDuration = zapcore.NanosDurationEncoder diff --git a/internal/server/aof.go b/internal/server/aof.go index 6d083e11..250080b1 100644 --- a/internal/server/aof.go +++ b/internal/server/aof.go @@ -41,10 +41,7 @@ func (s *Server) loadAOF() (err error) { ps := float64(count) / (float64(d) / float64(time.Second)) suf := []string{"bytes/s", "KB/s", "MB/s", "GB/s", "TB/s"} bps := float64(fi.Size()) / (float64(d) / float64(time.Second)) - for i := 0; bps > 1024; i++ { - if len(suf) == 1 { - break - } + for i := 0; bps > 1024 && len(suf) > 1; i++ { bps /= 1024 suf = suf[1:] } @@ -123,11 +120,7 @@ func commandErrIsFatal(err error) bool { // FSET (and other writable commands) may return errors that we need // to ignore during the loading process. These errors may occur (though unlikely) // due to the aof rewrite operation. - switch err { - case errKeyNotFound, errIDNotFound: - return false - } - return true + return !(err == errKeyNotFound || err == errIDNotFound) } // flushAOF flushes all aof buffer data to disk. Set sync to true to sync the @@ -406,73 +399,80 @@ func (s liveAOFSwitches) Error() string { return goingLive } -func (s *Server) cmdAOFMD5(msg *Message) (res resp.Value, err error) { +// AOFMD5 pos size +func (s *Server) cmdAOFMD5(msg *Message) (resp.Value, error) { start := time.Now() - vs := msg.Args[1:] - var ok bool - var spos, ssize string - if vs, spos, ok = tokenval(vs); !ok || spos == "" { - return NOMessage, errInvalidNumberOfArguments + // >> Args + + args := msg.Args + if len(args) != 3 { + return retrerr(errInvalidNumberOfArguments) } - if vs, ssize, ok = tokenval(vs); !ok || ssize == "" { - return NOMessage, errInvalidNumberOfArguments - } - if len(vs) != 0 { - return NOMessage, errInvalidNumberOfArguments - } - pos, err := strconv.ParseInt(spos, 10, 64) + pos, err := strconv.ParseInt(args[1], 10, 64) if err != nil || pos < 0 { - return NOMessage, errInvalidArgument(spos) + return retrerr(errInvalidArgument(args[1])) } - size, err := strconv.ParseInt(ssize, 10, 64) + size, err := strconv.ParseInt(args[2], 10, 64) if err != nil || size < 0 { - return NOMessage, errInvalidArgument(ssize) + return retrerr(errInvalidArgument(args[2])) } + + // >> Operation + sum, err := s.checksum(pos, size) if err != nil { - return NOMessage, err + return retrerr(err) } - switch msg.OutputType { - case JSON: - res = resp.StringValue( - fmt.Sprintf(`{"ok":true,"md5":"%s","elapsed":"%s"}`, sum, time.Since(start))) - case RESP: - res = resp.SimpleStringValue(sum) + + // >> Response + + if msg.OutputType == JSON { + return resp.StringValue(fmt.Sprintf( + `{"ok":true,"md5":"%s","elapsed":"%s"}`, + sum, time.Since(start))), nil } - return res, nil + return resp.SimpleStringValue(sum), nil } -func (s *Server) cmdAOF(msg *Message) (res resp.Value, err error) { +// AOF pos +func (s *Server) cmdAOF(msg *Message) (resp.Value, error) { if s.aof == nil { - return NOMessage, errors.New("aof disabled") + return retrerr(errors.New("aof disabled")) } - vs := msg.Args[1:] - var ok bool - var spos string - if vs, spos, ok = tokenval(vs); !ok || spos == "" { - return NOMessage, errInvalidNumberOfArguments + // >> Args + + args := msg.Args + if len(args) != 2 { + return retrerr(errInvalidNumberOfArguments) } - if len(vs) != 0 { - return NOMessage, errInvalidNumberOfArguments - } - pos, err := strconv.ParseInt(spos, 10, 64) + + pos, err := strconv.ParseInt(args[1], 10, 64) if err != nil || pos < 0 { - return NOMessage, errInvalidArgument(spos) + return retrerr(errInvalidArgument(args[1])) } + + // >> Operation + f, err := os.Open(s.aof.Name()) if err != nil { - return NOMessage, err + return retrerr(err) } defer f.Close() + n, err := f.Seek(0, 2) if err != nil { - return NOMessage, err + return retrerr(err) } + if n < pos { - return NOMessage, errors.New("pos is too big, must be less that the aof_size of leader") + return retrerr(errors.New( + "pos is too big, must be less that the aof_size of leader")) } + + // >> Response + var ls liveAOFSwitches ls.pos = pos return NOMessage, ls @@ -485,8 +485,6 @@ func (s *Server) liveAOF(pos int64, conn net.Conn, rd *PipelineReader, msg *Mess if err != nil { return err } - defer f.Close() - s.mu.Lock() s.aofconnM[conn] = f s.mu.Unlock() @@ -495,91 +493,42 @@ func (s *Server) liveAOF(pos int64, conn net.Conn, rd *PipelineReader, msg *Mess delete(s.aofconnM, conn) s.mu.Unlock() conn.Close() + f.Close() }() if _, err := conn.Write([]byte("+OK\r\n")); err != nil { return err } - if _, err := f.Seek(pos, 0); err != nil { return err } - cond := sync.NewCond(&sync.Mutex{}) - var mustQuit bool + var wg sync.WaitGroup + wg.Add(1) go func() { defer func() { - cond.L.Lock() - mustQuit = true - cond.Broadcast() - cond.L.Unlock() + f.Close() + conn.Close() + wg.Done() }() - for { - vs, err := rd.ReadMessages() - if err != nil { - if err != io.EOF { - log.Error(err) - } - return - } - for _, v := range vs { - switch v.Command() { - default: - log.Error("received a live command that was not QUIT") - return - case "quit", "": - return - } - } - } + // Any incoming message should end the connection + rd.ReadMessages() }() - go func() { - defer func() { - cond.L.Lock() - mustQuit = true - cond.Broadcast() - cond.L.Unlock() - }() - err := func() error { - _, err := io.Copy(conn, f) - if err != nil { + _, err = io.Copy(conn, f) + if err != nil { + return err + } + b := make([]byte, 4096*2) + for { + n, err := f.Read(b) + if n > 0 { + if _, err := conn.Write(b[:n]); err != nil { return err } - - b := make([]byte, 4096) - // The reader needs to be OK with the eof not - for { - n, err := f.Read(b) - if n > 0 { - if _, err := conn.Write(b[:n]); err != nil { - return err - } - } - if err != io.EOF { - if err != nil { - return err - } - continue - } - s.fcond.L.Lock() - s.fcond.Wait() - s.fcond.L.Unlock() - } - }() - if err != nil { - if !strings.Contains(err.Error(), "use of closed network connection") && - !strings.Contains(err.Error(), "bad file descriptor") { - log.Error(err) - } - return } - }() - for { - cond.L.Lock() - if mustQuit { - cond.L.Unlock() - return nil + if err == io.EOF { + time.Sleep(time.Second / 4) + } else if err != nil { + return err } - cond.Wait() - cond.L.Unlock() } } diff --git a/internal/server/aofshrink.go b/internal/server/aofshrink.go index c47693c1..257b102b 100644 --- a/internal/server/aofshrink.go +++ b/internal/server/aofshrink.go @@ -8,7 +8,6 @@ import ( "time" "github.com/tidwall/btree" - "github.com/tidwall/tile38/core" "github.com/tidwall/tile38/internal/collection" "github.com/tidwall/tile38/internal/field" "github.com/tidwall/tile38/internal/log" @@ -20,12 +19,9 @@ const maxids = 32 const maxchunk = 4 * 1024 * 1024 func (s *Server) aofshrink() { - if s.aof == nil { - return - } start := time.Now() s.mu.Lock() - if s.shrinking { + if s.aof == nil || s.shrinking { s.mu.Unlock() return } @@ -42,7 +38,7 @@ func (s *Server) aofshrink() { }() err := func() error { - f, err := os.Create(core.AppendFileName + "-shrink") + f, err := os.Create(s.opts.AppendFileName + "-shrink") if err != nil { return err } @@ -279,13 +275,13 @@ func (s *Server) aofshrink() { if err := f.Close(); err != nil { log.Fatalf("shrink new aof close fatal operation: %v", err) } - if err := os.Rename(core.AppendFileName, core.AppendFileName+"-bak"); err != nil { + if err := os.Rename(s.opts.AppendFileName, s.opts.AppendFileName+"-bak"); err != nil { log.Fatalf("shrink backup fatal operation: %v", err) } - if err := os.Rename(core.AppendFileName+"-shrink", core.AppendFileName); err != nil { + if err := os.Rename(s.opts.AppendFileName+"-shrink", s.opts.AppendFileName); err != nil { log.Fatalf("shrink rename fatal operation: %v", err) } - s.aof, err = os.OpenFile(core.AppendFileName, os.O_CREATE|os.O_RDWR, 0600) + s.aof, err = os.OpenFile(s.opts.AppendFileName, os.O_CREATE|os.O_RDWR, 0600) if err != nil { log.Fatalf("shrink openfile fatal operation: %v", err) } @@ -296,7 +292,7 @@ func (s *Server) aofshrink() { } s.aofsz = int(n) - os.Remove(core.AppendFileName + "-bak") // ignore error + os.Remove(s.opts.AppendFileName + "-bak") // ignore error return nil }() diff --git a/internal/server/atomic.go b/internal/server/atomic.go deleted file mode 100644 index 97e9a4cb..00000000 --- a/internal/server/atomic.go +++ /dev/null @@ -1,32 +0,0 @@ -package server - -import ( - "sync/atomic" -) - -type aint struct{ v uintptr } - -func (a *aint) add(d int) int { - if d < 0 { - return int(atomic.AddUintptr(&a.v, ^uintptr((d*-1)-1))) - } - return int(atomic.AddUintptr(&a.v, uintptr(d))) -} -func (a *aint) get() int { - return int(atomic.LoadUintptr(&a.v)) -} -func (a *aint) set(i int) int { - return int(atomic.SwapUintptr(&a.v, uintptr(i))) -} - -type abool struct{ v uint32 } - -func (a *abool) on() bool { - return atomic.LoadUint32(&a.v) != 0 -} -func (a *abool) set(t bool) bool { - if t { - return atomic.SwapUint32(&a.v, 1) != 0 - } - return atomic.SwapUint32(&a.v, 0) != 0 -} diff --git a/internal/server/atomic_test.go b/internal/server/atomic_test.go deleted file mode 100644 index 6ed7cc97..00000000 --- a/internal/server/atomic_test.go +++ /dev/null @@ -1,19 +0,0 @@ -package server - -import "testing" - -func TestAtomicInt(t *testing.T) { - var x aint - x.set(10) - if x.get() != 10 { - t.Fatalf("expected %v, got %v", 10, x.get()) - } - x.add(-9) - if x.get() != 1 { - t.Fatalf("expected %v, got %v", 1, x.get()) - } - x.add(-1) - if x.get() != 0 { - t.Fatalf("expected %v, got %v", 0, x.get()) - } -} diff --git a/internal/server/bson.go b/internal/server/bson.go index bcc9d2ea..c65adcbb 100644 --- a/internal/server/bson.go +++ b/internal/server/bson.go @@ -5,7 +5,6 @@ import ( "crypto/rand" "encoding/binary" "encoding/hex" - "io" "os" "sync/atomic" "time" @@ -23,23 +22,17 @@ func bsonID() string { var ( bsonProcess = uint16(os.Getpid()) bsonMachine = func() []byte { - host, err := os.Hostname() - if err != nil { - b := make([]byte, 3) - if _, err := io.ReadFull(rand.Reader, b); err != nil { - panic("random error: " + err.Error()) - } - return b - } + host, _ := os.Hostname() + b := make([]byte, 3) + Must(rand.Read(b)) + host = Default(host, string(b)) hw := md5.New() hw.Write([]byte(host)) return hw.Sum(nil)[:3] }() bsonCounter = func() uint32 { b := make([]byte, 4) - if _, err := io.ReadFull(rand.Reader, b); err != nil { - panic("random error: " + err.Error()) - } + Must(rand.Read(b)) return binary.BigEndian.Uint32(b) }() ) diff --git a/internal/server/bson_test.go b/internal/server/bson_test.go new file mode 100644 index 00000000..ca157446 --- /dev/null +++ b/internal/server/bson_test.go @@ -0,0 +1,10 @@ +package server + +import "testing" + +func TestBSON(t *testing.T) { + id := bsonID() + if len(id) != 24 { + t.Fail() + } +} diff --git a/internal/server/checksum.go b/internal/server/checksum.go index 3538be2b..3b612923 100644 --- a/internal/server/checksum.go +++ b/internal/server/checksum.go @@ -9,7 +9,6 @@ import ( "time" "github.com/tidwall/resp" - "github.com/tidwall/tile38/core" "github.com/tidwall/tile38/internal/log" ) @@ -140,12 +139,12 @@ func getEndOfLastValuePositionInFile(fname string, startPos int64) (int64, error // We will do some various checksums on the leader until we find the correct position to start at. func (s *Server) followCheckSome(addr string, followc int, auth string, ) (pos int64, err error) { - if core.ShowDebugMessages { + if s.opts.ShowDebugMessages { log.Debug("follow:", addr, ":check some") } s.mu.Lock() defer s.mu.Unlock() - if s.followc.get() != followc { + if int(s.followc.Load()) != followc { return 0, errNoLongerFollowing } if s.aofsz < checksumsz { @@ -211,7 +210,7 @@ func (s *Server) followCheckSome(addr string, followc int, auth string, return 0, err } if pos == fullpos { - if core.ShowDebugMessages { + if s.opts.ShowDebugMessages { log.Debug("follow: aof fully intact") } return pos, nil diff --git a/internal/server/client.go b/internal/server/client.go index 6af0a108..67fedf75 100644 --- a/internal/server/client.go +++ b/internal/server/client.go @@ -2,7 +2,6 @@ package server import ( "encoding/json" - "errors" "fmt" "io" "sort" @@ -32,6 +31,8 @@ type Client struct { name string // optional defined name opened time.Time // when the client was created/opened, unix nano last time.Time // last client request/response, unix nano + + closer io.Closer // used to close the connection } // Write ... @@ -40,32 +41,19 @@ func (client *Client) Write(b []byte) (n int, err error) { return len(b), nil } -type byID []*Client - -func (arr byID) Len() int { - return len(arr) -} -func (arr byID) Less(a, b int) bool { - return arr[a].id < arr[b].id -} -func (arr byID) Swap(a, b int) { - arr[a], arr[b] = arr[b], arr[a] -} - -func (s *Server) cmdClient(msg *Message, client *Client) (resp.Value, error) { +// CLIENT (LIST | KILL | GETNAME | SETNAME) +func (s *Server) cmdCLIENT(msg *Message, client *Client) (resp.Value, error) { start := time.Now() - if len(msg.Args) == 1 { - return NOMessage, errInvalidNumberOfArguments + args := msg.Args + if len(args) == 1 { + return retrerr(errInvalidNumberOfArguments) } - switch strings.ToLower(msg.Args[1]) { - default: - return NOMessage, clientErrorf( - "Syntax error, try CLIENT (LIST | KILL | GETNAME | SETNAME)", - ) + + switch strings.ToLower(args[1]) { case "list": - if len(msg.Args) != 2 { - return NOMessage, errInvalidNumberOfArguments + if len(args) != 2 { + return retrerr(errInvalidNumberOfArguments) } var list []*Client s.connsmu.RLock() @@ -73,7 +61,9 @@ func (s *Server) cmdClient(msg *Message, client *Client) (resp.Value, error) { list = append(list, cc) } s.connsmu.RUnlock() - sort.Sort(byID(list)) + sort.Slice(list, func(i, j int) bool { + return list[i].id < list[j].id + }) now := time.Now() var buf []byte for _, client := range list { @@ -89,8 +79,7 @@ func (s *Server) cmdClient(msg *Message, client *Client) (resp.Value, error) { ) client.mu.Unlock() } - switch msg.OutputType { - case JSON: + if msg.OutputType == JSON { // Create a map of all key/value info fields var cmap []map[string]interface{} clients := strings.Split(string(buf), "\n") @@ -110,124 +99,113 @@ func (s *Server) cmdClient(msg *Message, client *Client) (resp.Value, error) { } } - // Marshal the map and use the output in the JSON response - data, err := json.Marshal(cmap) - if err != nil { - return NOMessage, err - } - return resp.StringValue(`{"ok":true,"list":` + string(data) + `,"elapsed":"` + time.Since(start).String() + "\"}"), nil - case RESP: - return resp.BytesValue(buf), nil - } - return NOMessage, nil - case "getname": - if len(msg.Args) != 2 { - return NOMessage, errInvalidNumberOfArguments - } - name := "" - switch msg.OutputType { - case JSON: - client.mu.Lock() - name := client.name - client.mu.Unlock() - return resp.StringValue(`{"ok":true,"name":` + - jsonString(name) + + data, _ := json.Marshal(cmap) + return resp.StringValue(`{"ok":true,"list":` + string(data) + `,"elapsed":"` + time.Since(start).String() + "\"}"), nil - case RESP: - return resp.StringValue(name), nil } + return resp.BytesValue(buf), nil + case "getname": + if len(args) != 2 { + return retrerr(errInvalidNumberOfArguments) + } + client.mu.Lock() + name := client.name + client.mu.Unlock() + if msg.OutputType == JSON { + return resp.StringValue(`{"ok":true,"name":` + jsonString(name) + + `,"elapsed":"` + time.Since(start).String() + "\"}"), nil + } + return resp.StringValue(name), nil case "setname": - if len(msg.Args) != 3 { - return NOMessage, errInvalidNumberOfArguments + if len(args) != 3 { + return retrerr(errInvalidNumberOfArguments) } name := msg.Args[2] for i := 0; i < len(name); i++ { if name[i] < '!' || name[i] > '~' { - return NOMessage, clientErrorf( + return retrerr(clientErrorf( "Client names cannot contain spaces, newlines or special characters.", - ) + )) } } client.mu.Lock() client.name = name client.mu.Unlock() - switch msg.OutputType { - case JSON: - return resp.StringValue(`{"ok":true,"elapsed":"` + time.Since(start).String() + "\"}"), nil - case RESP: - return resp.SimpleStringValue("OK"), nil + if msg.OutputType == JSON { + return resp.StringValue(`{"ok":true,"elapsed":"` + + time.Since(start).String() + "\"}"), nil } + return resp.SimpleStringValue("OK"), nil case "kill": - if len(msg.Args) < 3 { - return NOMessage, errInvalidNumberOfArguments + if len(args) < 3 { + return retrerr(errInvalidNumberOfArguments) } var useAddr bool var addr string var useID bool var id string - for i := 2; i < len(msg.Args); i++ { - arg := msg.Args[i] + for i := 2; i < len(args); i++ { + if useAddr || useID { + return retrerr(errInvalidNumberOfArguments) + } + arg := args[i] if strings.Contains(arg, ":") { addr = arg useAddr = true - break - } - switch strings.ToLower(arg) { - default: - return NOMessage, clientErrorf("No such client") - case "addr": - i++ - if i == len(msg.Args) { - return NOMessage, errors.New("syntax error") + } else { + switch strings.ToLower(arg) { + case "addr": + i++ + if i == len(args) { + return retrerr(errInvalidNumberOfArguments) + } + addr = args[i] + useAddr = true + case "id": + i++ + if i == len(args) { + return retrerr(errInvalidNumberOfArguments) + } + id = args[i] + useID = true + default: + return retrerr(clientErrorf("No such client")) } - addr = msg.Args[i] - useAddr = true - case "id": - i++ - if i == len(msg.Args) { - return NOMessage, errors.New("syntax error") - } - id = msg.Args[i] - useID = true } } - var cclose *Client + var closing []io.Closer s.connsmu.RLock() for _, cc := range s.conns { if useID && fmt.Sprintf("%d", cc.id) == id { - cclose = cc - break - } else if useAddr && client.remoteAddr == addr { - cclose = cc - break + if cc.closer != nil { + closing = append(closing, cc.closer) + } + } else if useAddr { + if cc.remoteAddr == addr { + if cc.closer != nil { + closing = append(closing, cc.closer) + } + } } } s.connsmu.RUnlock() - if cclose == nil { - return NOMessage, clientErrorf("No such client") + if len(closing) == 0 { + return retrerr(clientErrorf("No such client")) } - - var res resp.Value - switch msg.OutputType { - case JSON: - res = resp.StringValue(`{"ok":true,"elapsed":"` + time.Since(start).String() + "\"}") - case RESP: - res = resp.SimpleStringValue("OK") + // go func() { + // close the connections behind the scene + for _, closer := range closing { + closer.Close() } - - client.conn.Close() - // closing self, return response now - // NOTE: This is the only exception where we do convert response to a string - var outBytes []byte - switch msg.OutputType { - case JSON: - outBytes = res.Bytes() - case RESP: - outBytes, _ = res.MarshalRESP() + // }() + if msg.OutputType == JSON { + return resp.StringValue(`{"ok":true,"elapsed":"` + + time.Since(start).String() + "\"}"), nil } - cclose.conn.Write(outBytes) - cclose.conn.Close() - return res, nil + return resp.SimpleStringValue("OK"), nil + default: + return retrerr(clientErrorf( + "Syntax error, try CLIENT (LIST | KILL | GETNAME | SETNAME)", + )) } - return NOMessage, errors.New("invalid output type") } diff --git a/internal/server/config.go b/internal/server/config.go index b29cfc1d..7ec0bcad 100644 --- a/internal/server/config.go +++ b/internal/server/config.go @@ -427,6 +427,9 @@ func (s *Server) cmdConfigSet(msg *Message) (res resp.Value, err error) { if err := s.config.setProperty(name, value, false); err != nil { return NOMessage, err } + if name == MaxMemory { + s.checkOutOfMemory() + } return OKMessage(msg, start), nil } func (s *Server) cmdConfigRewrite(msg *Message) (res resp.Value, err error) { diff --git a/internal/server/crud.go b/internal/server/crud.go index eef2a2b9..d5658edd 100644 --- a/internal/server/crud.go +++ b/internal/server/crud.go @@ -2,7 +2,7 @@ package server import ( "bytes" - "errors" + "math" "strconv" "strings" "time" @@ -17,27 +17,30 @@ import ( "github.com/tidwall/tile38/internal/object" ) -func (s *Server) cmdBounds(msg *Message) (resp.Value, error) { +// BOUNDS key +func (s *Server) cmdBOUNDS(msg *Message) (resp.Value, error) { start := time.Now() - vs := msg.Args[1:] - var ok bool - var key string - if vs, key, ok = tokenval(vs); !ok || key == "" { - return NOMessage, errInvalidNumberOfArguments - } - if len(vs) != 0 { - return NOMessage, errInvalidNumberOfArguments + // >> Args + + args := msg.Args + if len(args) != 2 { + return retrerr(errInvalidNumberOfArguments) } + key := args[1] + + // >> Operation col, _ := s.cols.Get(key) if col == nil { if msg.OutputType == RESP { return resp.NullValue(), nil } - return NOMessage, errKeyNotFound + return retrerr(errKeyNotFound) } + // >> Response + vals := make([]resp.Value, 0, 2) var buf bytes.Buffer if msg.OutputType == JSON { @@ -52,105 +55,125 @@ func (s *Server) cmdBounds(msg *Message) (resp.Value, error) { if msg.OutputType == JSON { buf.WriteString(`,"bounds":`) buf.WriteString(string(bbox.AppendJSON(nil))) - } else { - vals = append(vals, resp.ArrayValue([]resp.Value{ - resp.ArrayValue([]resp.Value{ - resp.FloatValue(minX), - resp.FloatValue(minY), - }), - resp.ArrayValue([]resp.Value{ - resp.FloatValue(maxX), - resp.FloatValue(maxY), - }), - })) - } - switch msg.OutputType { - case JSON: buf.WriteString(`,"elapsed":"` + time.Since(start).String() + "\"}") return resp.StringValue(buf.String()), nil - case RESP: - return vals[0], nil } - return NOMessage, nil + + // RESP + vals = append(vals, resp.ArrayValue([]resp.Value{ + resp.ArrayValue([]resp.Value{ + resp.FloatValue(minX), + resp.FloatValue(minY), + }), + resp.ArrayValue([]resp.Value{ + resp.FloatValue(maxX), + resp.FloatValue(maxY), + }), + })) + return vals[0], nil } -func (s *Server) cmdType(msg *Message) (resp.Value, error) { +// TYPE key +// undocumented return "none" or "hash" +func (s *Server) cmdTYPE(msg *Message) (resp.Value, error) { start := time.Now() - vs := msg.Args[1:] - var ok bool - var key string - if _, key, ok = tokenval(vs); !ok || key == "" { - return NOMessage, errInvalidNumberOfArguments + // >> Args + + args := msg.Args + if len(args) != 2 { + return retrerr(errInvalidNumberOfArguments) } + key := args[1] + + // >> Operation col, _ := s.cols.Get(key) if col == nil { if msg.OutputType == RESP { return resp.SimpleStringValue("none"), nil } - return NOMessage, errKeyNotFound + return retrerr(errKeyNotFound) } + // >> Response + typ := "hash" - switch msg.OutputType { - case JSON: - return resp.StringValue(`{"ok":true,"type":` + string(typ) + `,"elapsed":"` + time.Since(start).String() + "\"}"), nil - case RESP: - return resp.SimpleStringValue(typ), nil + if msg.OutputType == JSON { + return resp.StringValue(`{"ok":true,"type":` + jsonString(typ) + + `,"elapsed":"` + time.Since(start).String() + "\"}"), nil } - return NOMessage, nil + return resp.SimpleStringValue(typ), nil } -func (s *Server) cmdGet(msg *Message) (resp.Value, error) { +// GET key id [WITHFIELDS] [OBJECT|POINT|BOUNDS|(HASH geohash)] +func (s *Server) cmdGET(msg *Message) (resp.Value, error) { start := time.Now() - vs := msg.Args[1:] - var ok bool - var key, id, typ, sprecision string - if vs, key, ok = tokenval(vs); !ok || key == "" { - return NOMessage, errInvalidNumberOfArguments - } - if vs, id, ok = tokenval(vs); !ok || id == "" { - return NOMessage, errInvalidNumberOfArguments + // >> Args + + args := msg.Args + + if len(args) < 3 { + return retrerr(errInvalidNumberOfArguments) } + key, id := args[1], args[2] withfields := false - if _, peek, ok := tokenval(vs); ok && strings.ToLower(peek) == "withfields" { - withfields = true - vs = vs[1:] + kind := "object" + var precision int64 + for i := 3; i < len(args); i++ { + switch strings.ToLower(args[i]) { + case "withfields": + withfields = true + case "object": + kind = "object" + case "point": + kind = "point" + case "bounds": + kind = "bounds" + case "hash": + kind = "hash" + i++ + if i == len(args) { + return retrerr(errInvalidNumberOfArguments) + } + var err error + precision, err = strconv.ParseInt(args[i], 10, 64) + if err != nil || precision < 1 || precision > 12 { + return retrerr(errInvalidArgument(args[i])) + } + default: + return retrerr(errInvalidNumberOfArguments) + } } + // >> Operation + col, _ := s.cols.Get(key) if col == nil { if msg.OutputType == RESP { return resp.NullValue(), nil } - return NOMessage, errKeyNotFound + return retrerr(errKeyNotFound) } o := col.Get(id) - ok = o != nil - if !ok { + if o == nil { if msg.OutputType == RESP { return resp.NullValue(), nil } - return NOMessage, errIDNotFound + return retrerr(errIDNotFound) } + // >> Response + vals := make([]resp.Value, 0, 2) var buf bytes.Buffer if msg.OutputType == JSON { buf.WriteString(`{"ok":true`) } - vs, typ, ok = tokenval(vs) - typ = strings.ToLower(typ) - if !ok { - typ = "object" - } - switch typ { - default: - return NOMessage, errInvalidArgument(typ) + switch kind { case "object": if msg.OutputType == JSON { buf.WriteString(`,"object":`) @@ -179,16 +202,9 @@ func (s *Server) cmdGet(msg *Message) (resp.Value, error) { } } case "hash": - if vs, sprecision, ok = tokenval(vs); !ok || sprecision == "" { - return NOMessage, errInvalidNumberOfArguments - } if msg.OutputType == JSON { buf.WriteString(`,"hash":`) } - precision, err := strconv.ParseInt(sprecision, 10, 64) - if err != nil || precision < 1 || precision > 12 { - return NOMessage, errInvalidArgument(sprecision) - } center := o.Geo().Center() p := geohash.EncodeWithPrecision(center.Y, center.X, uint(precision)) if msg.OutputType == JSON { @@ -215,9 +231,6 @@ func (s *Server) cmdGet(msg *Message) (resp.Value, error) { } } - if len(vs) != 0 { - return NOMessage, errInvalidNumberOfArguments - } if withfields { nfields := o.Fields().Len() if nfields > 0 { @@ -231,10 +244,11 @@ func (s *Server) cmdGet(msg *Message) (resp.Value, error) { if i > 0 { buf.WriteString(`,`) } - buf.WriteString(jsonString(f.Name()) + ":" + f.Value().JSON()) + buf.WriteString(jsonString(f.Name()) + ":" + + f.Value().JSON()) } else { - fvals = append(fvals, - resp.StringValue(f.Name()), resp.StringValue(f.Value().Data())) + fvals = append(fvals, resp.StringValue(f.Name()), + resp.StringValue(f.Value().Data())) } i++ return true @@ -246,24 +260,21 @@ func (s *Server) cmdGet(msg *Message) (resp.Value, error) { } } } - switch msg.OutputType { - case JSON: + if msg.OutputType == JSON { buf.WriteString(`,"elapsed":"` + time.Since(start).String() + "\"}") return resp.StringValue(buf.String()), nil - case RESP: - var oval resp.Value - if withfields { - oval = resp.ArrayValue(vals) - } else { - oval = vals[0] - } - return oval, nil } - return NOMessage, nil + var oval resp.Value + if withfields { + oval = resp.ArrayValue(vals) + } else { + oval = vals[0] + } + return oval, nil } // DEL key id [ERRON404] -func (s *Server) cmdDel(msg *Message) (resp.Value, commandDetails, error) { +func (s *Server) cmdDEL(msg *Message) (resp.Value, commandDetails, error) { start := time.Now() // >> Args @@ -330,113 +341,109 @@ func (s *Server) cmdDel(msg *Message) (resp.Value, commandDetails, error) { return res, d, nil } -func (s *Server) cmdPdel(msg *Message) (res resp.Value, d commandDetails, err error) { +// PDEL key pattern +func (s *Server) cmdPDEL(msg *Message) (resp.Value, commandDetails, error) { start := time.Now() - vs := msg.Args[1:] - var ok bool - if vs, d.key, ok = tokenval(vs); !ok || d.key == "" { - err = errInvalidNumberOfArguments - return - } - if vs, d.pattern, ok = tokenval(vs); !ok || d.pattern == "" { - err = errInvalidNumberOfArguments - return - } - if len(vs) != 0 { - err = errInvalidNumberOfArguments - return - } - now := time.Now() - iter := func(o *object.Object) bool { - if match, _ := glob.Match(d.pattern, o.ID()); match { - d.children = append(d.children, &commandDetails{ - command: "del", - updated: true, - timestamp: now, - key: d.key, - obj: o, - }) - } - return true - } - var expired int - col, _ := s.cols.Get(d.key) + // >> Args + + args := msg.Args + if len(args) != 3 { + return retwerr(errInvalidNumberOfArguments) + } + key := args[1] + pattern := args[2] + + // >> Operation + + now := time.Now() + var children []*commandDetails + col, _ := s.cols.Get(key) if col != nil { - g := glob.Parse(d.pattern, false) + g := glob.Parse(pattern, false) + var ids []string + iter := func(o *object.Object) bool { + if match, _ := glob.Match(pattern, o.ID()); match { + ids = append(ids, o.ID()) + } + return true + } if g.Limits[0] == "" && g.Limits[1] == "" { col.Scan(false, nil, msg.Deadline, iter) } else { - col.ScanRange(g.Limits[0], g.Limits[1], false, nil, msg.Deadline, iter) + col.ScanRange(g.Limits[0], g.Limits[1], + false, nil, msg.Deadline, iter) } - var atLeastOneNotDeleted bool - for i, dc := range d.children { - old := col.Delete(dc.obj.ID()) - if old == nil { - d.children[i].command = "?" - atLeastOneNotDeleted = true - } else { - dc.obj = old - d.children[i] = dc - } - s.groupDisconnectObject(dc.key, dc.obj.ID()) - } - if atLeastOneNotDeleted { - var nchildren []*commandDetails - for _, dc := range d.children { - if dc.command == "del" { - nchildren = append(nchildren, dc) - } - } - d.children = nchildren + for _, id := range ids { + obj := col.Delete(id) + children = append(children, &commandDetails{ + command: "del", + updated: true, + timestamp: now, + key: key, + obj: obj, + }) + s.groupDisconnectObject(key, id) } if col.Count() == 0 { - s.cols.Delete(d.key) + s.cols.Delete(key) } } + + // >> Response + + var d commandDetails + var res resp.Value + d.command = "pdel" + d.children = children + d.key = key + d.pattern = pattern d.updated = len(d.children) > 0 d.timestamp = now d.parent = true switch msg.OutputType { case JSON: - res = resp.StringValue(`{"ok":true,"elapsed":"` + time.Since(start).String() + "\"}") + res = resp.StringValue(`{"ok":true,"elapsed":"` + + time.Since(start).String() + "\"}") case RESP: - total := len(d.children) - expired - if total < 0 { - total = 0 - } - res = resp.IntegerValue(total) + res = resp.IntegerValue(len(d.children)) } - return + return res, d, nil } -func (s *Server) cmdDrop(msg *Message) (res resp.Value, d commandDetails, err error) { +// DROP key +func (s *Server) cmdDROP(msg *Message) (resp.Value, commandDetails, error) { start := time.Now() - vs := msg.Args[1:] - var ok bool - if vs, d.key, ok = tokenval(vs); !ok || d.key == "" { - err = errInvalidNumberOfArguments - return + + // >> Args + + args := msg.Args + if len(args) != 2 { + return retwerr(errInvalidNumberOfArguments) } - if len(vs) != 0 { - err = errInvalidNumberOfArguments - return - } - col, _ := s.cols.Get(d.key) + key := args[1] + + // >> Operation + + col, _ := s.cols.Get(key) if col != nil { - s.cols.Delete(d.key) - d.updated = true - } else { - d.key = "" // ignore the details - d.updated = false + s.cols.Delete(key) } - s.groupDisconnectCollection(d.key) + s.groupDisconnectCollection(key) + + // >> Response + + var res resp.Value + var d commandDetails + d.key = key + d.updated = col != nil d.command = "drop" d.timestamp = time.Now() switch msg.OutputType { case JSON: - res = resp.StringValue(`{"ok":true,"elapsed":"` + time.Since(start).String() + "\"}") + res = resp.StringValue(`{"ok":true,"elapsed":"` + + time.Since(start).String() + "\"}") case RESP: if d.updated { res = resp.IntegerValue(1) @@ -444,57 +451,70 @@ func (s *Server) cmdDrop(msg *Message) (res resp.Value, d commandDetails, err er res = resp.IntegerValue(0) } } - return + return res, d, nil } -func (s *Server) cmdRename(msg *Message) (res resp.Value, d commandDetails, err error) { - nx := msg.Command() == "renamenx" +// RENAME key newkey +// RENAMENX key newkey +func (s *Server) cmdRENAME(msg *Message) (resp.Value, commandDetails, error) { start := time.Now() - vs := msg.Args[1:] - var ok bool - if vs, d.key, ok = tokenval(vs); !ok || d.key == "" { - err = errInvalidNumberOfArguments - return + + // >> Args + + args := msg.Args + if len(args) != 3 { + return retwerr(errInvalidNumberOfArguments) } - if vs, d.newKey, ok = tokenval(vs); !ok || d.newKey == "" { - err = errInvalidNumberOfArguments - return - } - if len(vs) != 0 { - err = errInvalidNumberOfArguments - return - } - col, _ := s.cols.Get(d.key) + nx := strings.ToLower(args[0]) == "renamenx" + key := args[1] + newKey := args[2] + + // >> Operation + + col, _ := s.cols.Get(key) if col == nil { - err = errKeyNotFound - return + return retwerr(errKeyNotFound) } + var ierr error s.hooks.Ascend(nil, func(v interface{}) bool { h := v.(*Hook) - if h.Key == d.key || h.Key == d.newKey { - err = errKeyHasHooksSet + if h.Key == key || h.Key == newKey { + ierr = errKeyHasHooksSet return false } return true }) - d.command = "rename" - newCol, _ := s.cols.Get(d.newKey) + if ierr != nil { + return retwerr(ierr) + } + var updated bool + newCol, _ := s.cols.Get(newKey) if newCol == nil { - d.updated = true - } else if nx { - d.updated = false - } else { - s.cols.Delete(d.newKey) - d.updated = true + updated = true + } else if !nx { + s.cols.Delete(newKey) + updated = true } - if d.updated { - s.cols.Delete(d.key) - s.cols.Set(d.newKey, col) + if updated { + s.cols.Delete(key) + s.cols.Set(newKey, col) } + + // >> Response + + var d commandDetails + var res resp.Value + + d.command = "rename" + d.key = key + d.newKey = newKey + d.updated = updated d.timestamp = time.Now() + switch msg.OutputType { case JSON: - res = resp.StringValue(`{"ok":true,"elapsed":"` + time.Since(start).String() + "\"}") + res = resp.StringValue(`{"ok":true,"elapsed":"` + + time.Since(start).String() + "\"}") case RESP: if !nx { res = resp.SimpleStringValue("OK") @@ -504,17 +524,23 @@ func (s *Server) cmdRename(msg *Message) (res resp.Value, d commandDetails, err res = resp.IntegerValue(0) } } - return + return res, d, nil } -func (s *Server) cmdFLUSHDB(msg *Message) (res resp.Value, d commandDetails, err error) { +// FLUSHDB +func (s *Server) cmdFLUSHDB(msg *Message) (resp.Value, commandDetails, error) { start := time.Now() - vs := msg.Args[1:] - if len(vs) != 0 { - err = errInvalidNumberOfArguments - return + + // >> Args + + args := msg.Args + + if len(args) != 1 { + return retwerr(errInvalidNumberOfArguments) } + // >> Operation + // clear the entire database s.cols.Clear() s.groupHooks.Clear() @@ -525,23 +551,29 @@ func (s *Server) cmdFLUSHDB(msg *Message) (res resp.Value, d commandDetails, err s.hookTree.Clear() s.hookCross.Clear() + // >> Response + + var d commandDetails d.command = "flushdb" d.updated = true d.timestamp = time.Now() - switch msg.OutputType { - case JSON: - res = resp.StringValue(`{"ok":true,"elapsed":"` + time.Since(start).String() + "\"}") - case RESP: + + var res resp.Value + if msg.OutputType == JSON { + res = resp.StringValue(`{"ok":true,"elapsed":"` + + time.Since(start).String() + "\"}") + } else { res = resp.SimpleStringValue("OK") } - return + return res, d, nil } // SET key id [FIELD name value ...] [EX seconds] [NX|XX] -// (OBJECT geojson)|(POINT lat lon z)|(BOUNDS minlat minlon maxlat maxlon)|(HASH geohash)|(STRING value) +// (OBJECT geojson)|(POINT lat lon z)|(BOUNDS minlat minlon maxlat maxlon)| +// (HASH geohash)|(STRING value) func (s *Server) cmdSET(msg *Message) (resp.Value, commandDetails, error) { start := time.Now() - if s.config.maxMemory() > 0 && s.outOfMemory.on() { + if s.config.maxMemory() > 0 && s.outOfMemory.Load() { return retwerr(errOOM) } @@ -674,23 +706,22 @@ func (s *Server) cmdSET(msg *Message) (resp.Value, commandDetails, error) { return retwerr(errInvalidArgument(args[i])) } } + if oobj == nil { + return retwerr(errInvalidNumberOfArguments) + } // >> Operation nada := func() (resp.Value, commandDetails, error) { // exclude operation due to 'xx' or 'nx' match - switch msg.OutputType { - default: - case JSON: + if msg.OutputType == JSON { if nx { return retwerr(errIDAlreadyExists) } else { return retwerr(errIDNotFound) } - case RESP: - return resp.NullValue(), commandDetails{}, nil } - return retwerr(errors.New("nada unknown output")) + return resp.NullValue(), commandDetails{}, nil } col, ok := s.cols.Get(key) @@ -749,7 +780,7 @@ func retrerr(err error) (resp.Value, error) { // FSET key id [XX] field value [field value...] func (s *Server) cmdFSET(msg *Message) (resp.Value, commandDetails, error) { start := time.Now() - if s.config.maxMemory() > 0 && s.outOfMemory.on() { + if s.config.maxMemory() > 0 && s.outOfMemory.Load() { return retwerr(errOOM) } @@ -835,6 +866,9 @@ func (s *Server) cmdFSET(msg *Message) (resp.Value, commandDetails, error) { // EXPIRE key id seconds func (s *Server) cmdEXPIRE(msg *Message) (resp.Value, commandDetails, error) { start := time.Now() + + // >> Args + args := msg.Args if len(args) != 4 { return retwerr(errInvalidNumberOfArguments) @@ -844,12 +878,16 @@ func (s *Server) cmdEXPIRE(msg *Message) (resp.Value, commandDetails, error) { if err != nil { return retwerr(errInvalidArgument(svalue)) } + + // >> Operation + var ok bool var obj *object.Object col, _ := s.cols.Get(key) if col != nil { - // replace the expiration by getting the old objec - ex := time.Now().Add(time.Duration(float64(time.Second) * value)).UnixNano() + // replace the expiration by getting the old object + ex := time.Now().Add( + time.Duration(float64(time.Second) * value)).UnixNano() o := col.Get(id) ok = o != nil if ok { @@ -857,6 +895,9 @@ func (s *Server) cmdEXPIRE(msg *Message) (resp.Value, commandDetails, error) { col.Set(obj) } } + + // >> Response + var d commandDetails if ok { d.key = key @@ -889,11 +930,17 @@ func (s *Server) cmdEXPIRE(msg *Message) (resp.Value, commandDetails, error) { // PERSIST key id func (s *Server) cmdPERSIST(msg *Message) (resp.Value, commandDetails, error) { start := time.Now() + + // >> Args + args := msg.Args if len(args) != 3 { return retwerr(errInvalidNumberOfArguments) } key, id := args[1], args[2] + + // >> Operation + col, _ := s.cols.Get(key) if col == nil { if msg.OutputType == RESP { @@ -917,6 +964,8 @@ func (s *Server) cmdPERSIST(msg *Message) (resp.Value, commandDetails, error) { cleared = true } + // >> Response + var res resp.Value var d commandDetails @@ -928,7 +977,8 @@ func (s *Server) cmdPERSIST(msg *Message) (resp.Value, commandDetails, error) { switch msg.OutputType { case JSON: - res = resp.SimpleStringValue(`{"ok":true,"elapsed":"` + time.Since(start).String() + "\"}") + res = resp.SimpleStringValue(`{"ok":true,"elapsed":"` + + time.Since(start).String() + "\"}") case RESP: if cleared { res = resp.IntegerValue(1) @@ -942,62 +992,47 @@ func (s *Server) cmdPERSIST(msg *Message) (resp.Value, commandDetails, error) { // TTL key id func (s *Server) cmdTTL(msg *Message) (resp.Value, error) { start := time.Now() + + // >> Args + args := msg.Args if len(args) != 3 { return retrerr(errInvalidNumberOfArguments) } key, id := args[1], args[2] - var v float64 - var ok bool - var ok2 bool + + // >> Operation + col, _ := s.cols.Get(key) - if col != nil { - o := col.Get(id) - ok = o != nil - if ok { - if o.Expires() != 0 { - now := start.UnixNano() - if now > o.Expires() { - ok2 = false - } else { - v = float64(o.Expires()-now) / float64(time.Second) - if v < 0 { - v = 0 - } - ok2 = true - } - } + if col == nil { + if msg.OutputType == JSON { + return retrerr(errKeyNotFound) } + return resp.IntegerValue(-2), nil } - var res resp.Value - switch msg.OutputType { - case JSON: - if ok { - var ttl string - if ok2 { - ttl = strconv.FormatFloat(v, 'f', -1, 64) - } else { - ttl = "-1" - } - res = resp.SimpleStringValue( - `{"ok":true,"ttl":` + ttl + `,"elapsed":"` + - time.Since(start).String() + "\"}") - } else { - if col == nil { - return retrerr(errKeyNotFound) - } + + o := col.Get(id) + if o == nil { + if msg.OutputType == JSON { return retrerr(errIDNotFound) } - case RESP: - if ok { - if ok2 { - res = resp.IntegerValue(int(v)) - } else { - res = resp.IntegerValue(-1) - } - } else { - res = resp.IntegerValue(-2) - } + return resp.IntegerValue(-2), nil } - return res, nil + + var ttl float64 + if o.Expires() == 0 { + ttl = -1 + } else { + now := start.UnixNano() + ttl = math.Max(float64(o.Expires()-now)/float64(time.Second), 0) + } + + // >> Response + + if msg.OutputType == JSON { + return resp.SimpleStringValue( + `{"ok":true,"ttl":` + strconv.Itoa(int(ttl)) + `,"elapsed":"` + + time.Since(start).String() + "\"}"), nil + } + return resp.IntegerValue(int(ttl)), nil } diff --git a/internal/server/expire.go b/internal/server/expire.go index cd67fd79..5f812913 100644 --- a/internal/server/expire.go +++ b/internal/server/expire.go @@ -1,6 +1,7 @@ package server import ( + "sync" "time" "github.com/tidwall/tile38/internal/collection" @@ -12,20 +13,15 @@ const bgExpireDelay = time.Second / 10 // backgroundExpiring deletes expired items from the database. // It's executes every 1/10 of a second. -func (s *Server) backgroundExpiring() { - for { - if s.stopServer.on() { - return - } - func() { - s.mu.Lock() - defer s.mu.Unlock() - now := time.Now() - s.backgroundExpireObjects(now) - s.backgroundExpireHooks(now) - }() - time.Sleep(bgExpireDelay) - } +func (s *Server) backgroundExpiring(wg *sync.WaitGroup) { + defer wg.Done() + s.loopUntilServerStops(bgExpireDelay, func() { + s.mu.Lock() + defer s.mu.Unlock() + now := time.Now() + s.backgroundExpireObjects(now) + s.backgroundExpireHooks(now) + }) } func (s *Server) backgroundExpireObjects(now time.Time) { @@ -42,7 +38,7 @@ func (s *Server) backgroundExpireObjects(now time.Time) { return true }) for _, msg := range msgs { - _, d, err := s.cmdDel(msg) + _, d, err := s.cmdDEL(msg) if err != nil { log.Fatal(err) } diff --git a/internal/server/follow.go b/internal/server/follow.go index 3b2a6772..21c39575 100644 --- a/internal/server/follow.go +++ b/internal/server/follow.go @@ -9,7 +9,6 @@ import ( "time" "github.com/tidwall/resp" - "github.com/tidwall/tile38/core" "github.com/tidwall/tile38/internal/log" ) @@ -84,10 +83,11 @@ func (s *Server) cmdFollow(msg *Message) (res resp.Value, err error) { } s.config.write(false) if update { - s.followc.add(1) + s.followc.Add(1) if s.config.followHost() != "" { log.Infof("following new host '%s' '%s'.", host, sport) - go s.follow(s.config.followHost(), s.config.followPort(), s.followc.get()) + go s.follow(s.config.followHost(), s.config.followPort(), + int(s.followc.Load())) } else { log.Infof("following no one") } @@ -153,7 +153,7 @@ func doServer(conn *RESPConn) (map[string]string, error) { func (s *Server) followHandleCommand(args []string, followc int, w io.Writer) (int, error) { s.mu.Lock() defer s.mu.Unlock() - if s.followc.get() != followc { + if int(s.followc.Load()) != followc { return s.aofsz, errNoLongerFollowing } msg := &Message{Args: args} @@ -188,7 +188,7 @@ func (s *Server) followDoLeaderAuth(conn *RESPConn, auth string) error { } func (s *Server) followStep(host string, port int, followc int) error { - if s.followc.get() != followc { + if int(s.followc.Load()) != followc { return errNoLongerFollowing } s.mu.Lock() @@ -240,7 +240,7 @@ func (s *Server) followStep(host string, port int, followc int) error { if v.String() != "OK" { return errors.New("invalid response to replconf request") } - if core.ShowDebugMessages { + if s.opts.ShowDebugMessages { log.Debug("follow:", addr, ":replconf") } @@ -254,7 +254,7 @@ func (s *Server) followStep(host string, port int, followc int) error { if v.String() != "OK" { return errors.New("invalid response to aof live request") } - if core.ShowDebugMessages { + if s.opts.ShowDebugMessages { log.Debug("follow:", addr, ":read aof") } diff --git a/internal/server/hooks.go b/internal/server/hooks.go index bebcaef2..6d13d005 100644 --- a/internal/server/hooks.go +++ b/internal/server/hooks.go @@ -7,6 +7,7 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "time" "github.com/tidwall/buntdb" @@ -501,7 +502,7 @@ type Hook struct { query string epm *endpoint.Manager expires time.Time - counter *aint // counter that grows when a message was sent + counter *atomic.Int64 // counter that grows when a message was sent sig int } @@ -701,7 +702,7 @@ func (h *Hook) proc() (ok bool) { } log.Debugf("Endpoint send ok: %v: %v: %v", idx, endpoint, err) sent = true - h.counter.add(1) + h.counter.Add(1) break } if !sent { diff --git a/internal/server/keys.go b/internal/server/keys.go index 94eaf199..095a0686 100644 --- a/internal/server/keys.go +++ b/internal/server/keys.go @@ -1,8 +1,7 @@ package server import ( - "bytes" - "strings" + "encoding/json" "time" "github.com/tidwall/resp" @@ -10,86 +9,59 @@ import ( "github.com/tidwall/tile38/internal/glob" ) -func (s *Server) cmdKeys(msg *Message) (res resp.Value, err error) { +// KEYS pattern +func (s *Server) cmdKEYS(msg *Message) (resp.Value, error) { var start = time.Now() - vs := msg.Args[1:] - var pattern string - var ok bool - if vs, pattern, ok = tokenval(vs); !ok || pattern == "" { - return NOMessage, errInvalidNumberOfArguments - } - if len(vs) != 0 { - return NOMessage, errInvalidNumberOfArguments - } + // >> Args - var wr = &bytes.Buffer{} - var once bool - if msg.OutputType == JSON { - wr.WriteString(`{"ok":true,"keys":[`) + args := msg.Args + if len(args) != 2 { + return retrerr(errInvalidNumberOfArguments) } - var wild bool - if strings.Contains(pattern, "*") { - wild = true - } - var everything bool - var greater bool - var greaterPivot string - var vals []resp.Value + pattern := args[1] - iter := func(key string, col *collection.Collection) bool { - var match bool - if everything { - match = true - } else if greater { - if !strings.HasPrefix(key, greaterPivot) { - return false - } - match = true - } else { - match, _ = glob.Match(pattern, key) - } - if match { - if once { - if msg.OutputType == JSON { - wr.WriteByte(',') + // >> Operation + + keys := []string{} + g := glob.Parse(pattern, false) + everything := g.Limits[0] == "" && g.Limits[1] == "" + if everything { + s.cols.Scan( + func(key string, _ *collection.Collection) bool { + match, _ := glob.Match(pattern, key) + if match { + keys = append(keys, key) } - } else { - once = true - } - switch msg.OutputType { - case JSON: - wr.WriteString(jsonString(key)) - case RESP: - vals = append(vals, resp.StringValue(key)) - } - - // If no more than one match is expected, stop searching - if !wild { - return false - } - } - return true - } - - // TODO: This can be further optimized by using glob.Parse and limits - if pattern == "*" { - everything = true - s.cols.Scan(iter) - } else if strings.HasSuffix(pattern, "*") { - greaterPivot = pattern[:len(pattern)-1] - if glob.IsGlob(greaterPivot) { - s.cols.Scan(iter) - } else { - greater = true - s.cols.Ascend(greaterPivot, iter) - } + return true + }, + ) } else { - s.cols.Scan(iter) + s.cols.Ascend(g.Limits[0], + func(key string, _ *collection.Collection) bool { + if key > g.Limits[1] { + return false + } + match, _ := glob.Match(pattern, key) + if match { + keys = append(keys, key) + } + return true + }, + ) } + + // >> Response + if msg.OutputType == JSON { - wr.WriteString(`],"elapsed":"` + time.Since(start).String() + "\"}") - return resp.StringValue(wr.String()), nil + data, _ := json.Marshal(keys) + return resp.StringValue(`{"ok":true,"keys":` + string(data) + + `,"elapsed":"` + time.Since(start).String() + `"}`), nil + } + + var vals []resp.Value + for _, key := range keys { + vals = append(vals, resp.StringValue(key)) } return resp.ArrayValue(vals), nil } diff --git a/internal/server/live.go b/internal/server/live.go index eb66ec64..2e694d40 100644 --- a/internal/server/live.go +++ b/internal/server/live.go @@ -11,6 +11,7 @@ import ( "github.com/tidwall/redcon" "github.com/tidwall/tile38/internal/log" + "go.uber.org/atomic" ) type liveBuffer struct { @@ -21,10 +22,16 @@ type liveBuffer struct { cond *sync.Cond } -func (s *Server) processLives() { - defer s.lwait.Done() +func (s *Server) processLives(wg *sync.WaitGroup) { + defer wg.Done() + var done atomic.Bool + wg.Add(1) go func() { + defer wg.Done() for { + if done.Load() { + break + } s.lcond.Broadcast() time.Sleep(time.Second / 4) } @@ -32,7 +39,8 @@ func (s *Server) processLives() { s.lcond.L.Lock() defer s.lcond.L.Unlock() for { - if s.stopServer.on() { + if s.stopServer.Load() { + done.Store(true) return } for len(s.lstack) > 0 { @@ -204,7 +212,7 @@ func (s *Server) goLive( return nil // nil return is fine here } } - s.statsTotalMsgsSent.add(len(msgs)) + s.statsTotalMsgsSent.Add(int64(len(msgs))) lb.cond.L.Lock() } diff --git a/internal/server/metrics.go b/internal/server/metrics.go index a5cd4623..78133ceb 100644 --- a/internal/server/metrics.go +++ b/internal/server/metrics.go @@ -92,9 +92,14 @@ func (s *Server) Collect(ch chan<- prometheus.Metric) { s.extStats(m) for metric, descr := range metricDescriptions { - if val, ok := m[metric].(int); ok { - ch <- prometheus.MustNewConstMetric(descr, prometheus.GaugeValue, float64(val)) - } else if val, ok := m[metric].(float64); ok { + val, ok := m[metric].(float64) + if !ok { + val2, ok2 := m[metric].(int) + if ok2 { + val, ok = float64(val2), true + } + } + if ok { ch <- prometheus.MustNewConstMetric(descr, prometheus.GaugeValue, val) } } diff --git a/internal/server/must.go b/internal/server/must.go new file mode 100644 index 00000000..b50ba653 --- /dev/null +++ b/internal/server/must.go @@ -0,0 +1,16 @@ +package server + +func Must[T any](a T, err error) T { + if err != nil { + panic(err) + } + return a +} + +func Default[T comparable](a, b T) T { + var c T + if a == c { + return b + } + return a +} diff --git a/internal/server/must_test.go b/internal/server/must_test.go new file mode 100644 index 00000000..de3bcade --- /dev/null +++ b/internal/server/must_test.go @@ -0,0 +1,38 @@ +package server + +import ( + "errors" + "testing" +) + +func TestMust(t *testing.T) { + if Must(1, nil) != 1 { + t.Fail() + } + func() { + var ended bool + defer func() { + if ended { + t.Fail() + } + err, ok := recover().(error) + if !ok { + t.Fail() + } + if err.Error() != "ok" { + t.Fail() + } + }() + Must(1, errors.New("ok")) + ended = true + }() +} + +func TestDefault(t *testing.T) { + if Default("", "2") != "2" { + t.Fail() + } + if Default("1", "2") != "1" { + t.Fail() + } +} diff --git a/internal/server/output.go b/internal/server/output.go index 65824ada..51476a1e 100644 --- a/internal/server/output.go +++ b/internal/server/output.go @@ -7,35 +7,31 @@ import ( "github.com/tidwall/resp" ) -func (s *Server) cmdOutput(msg *Message) (res resp.Value, err error) { +// OUTPUT [resp|json] +func (s *Server) cmdOUTPUT(msg *Message) (resp.Value, error) { start := time.Now() - vs := msg.Args[1:] - var arg string - var ok bool - if len(vs) != 0 { - if _, arg, ok = tokenval(vs); !ok || arg == "" { - return NOMessage, errInvalidNumberOfArguments + args := msg.Args + switch len(args) { + case 1: + if msg.OutputType == JSON { + return resp.StringValue(`{"ok":true,"output":"json","elapsed":` + + time.Since(start).String() + `}`), nil } + return resp.StringValue("resp"), nil + case 2: // Setting the original message output type will be picked up by the // server prior to the next command being executed. - switch strings.ToLower(arg) { + switch strings.ToLower(args[1]) { default: - return NOMessage, errInvalidArgument(arg) + return retrerr(errInvalidArgument(args[1])) case "json": msg.OutputType = JSON case "resp": msg.OutputType = RESP } return OKMessage(msg, start), nil - } - // return the output - switch msg.OutputType { default: - return NOMessage, nil - case JSON: - return resp.StringValue(`{"ok":true,"output":"json","elapsed":` + time.Since(start).String() + `}`), nil - case RESP: - return resp.StringValue("resp"), nil + return retrerr(errInvalidNumberOfArguments) } } diff --git a/internal/server/pubsub.go b/internal/server/pubsub.go index 0a22831f..4016347f 100644 --- a/internal/server/pubsub.go +++ b/internal/server/pubsub.go @@ -280,7 +280,7 @@ func (s *Server) liveSubscription( write(b) } } - s.statsTotalMsgsSent.add(1) + s.statsTotalMsgsSent.Add(1) } m := [2]map[string]bool{ diff --git a/internal/server/readonly.go b/internal/server/readonly.go index 586a9c0c..94067289 100644 --- a/internal/server/readonly.go +++ b/internal/server/readonly.go @@ -1,44 +1,50 @@ package server import ( - "strings" "time" "github.com/tidwall/resp" "github.com/tidwall/tile38/internal/log" ) -func (s *Server) cmdReadOnly(msg *Message) (res resp.Value, err error) { +// READONLY yes|no +func (s *Server) cmdREADONLY(msg *Message) (resp.Value, error) { start := time.Now() - vs := msg.Args[1:] - var arg string - var ok bool - if vs, arg, ok = tokenval(vs); !ok || arg == "" { - return NOMessage, errInvalidNumberOfArguments + // >> Args + + args := msg.Args + if len(args) != 2 { + return retrerr(errInvalidNumberOfArguments) } - if len(vs) != 0 { - return NOMessage, errInvalidNumberOfArguments - } - update := false - switch strings.ToLower(arg) { + + switch args[1] { + case "yes", "no": default: - return NOMessage, errInvalidArgument(arg) - case "yes": + return retrerr(errInvalidArgument(args[1])) + } + + // >> Operation + + var updated bool + if args[1] == "yes" { if !s.config.readOnly() { - update = true + updated = true s.config.setReadOnly(true) log.Info("read only") } - case "no": + } else { if s.config.readOnly() { - update = true + updated = true s.config.setReadOnly(false) log.Info("read write") } } - if update { + if updated { s.config.write(false) } + + // >> Response + return OKMessage(msg, start), nil } diff --git a/internal/server/scripts.go b/internal/server/scripts.go index 8b8b5fca..3aae4114 100644 --- a/internal/server/scripts.go +++ b/internal/server/scripts.go @@ -596,23 +596,23 @@ func (s *Server) commandInScript(msg *Message) ( case "fset": res, d, err = s.cmdFSET(msg) case "del": - res, d, err = s.cmdDel(msg) + res, d, err = s.cmdDEL(msg) case "pdel": - res, d, err = s.cmdPdel(msg) + res, d, err = s.cmdPDEL(msg) case "drop": - res, d, err = s.cmdDrop(msg) + res, d, err = s.cmdDROP(msg) case "expire": res, d, err = s.cmdEXPIRE(msg) case "rename": - res, d, err = s.cmdRename(msg) + res, d, err = s.cmdRENAME(msg) case "renamenx": - res, d, err = s.cmdRename(msg) + res, d, err = s.cmdRENAME(msg) case "persist": res, d, err = s.cmdPERSIST(msg) case "ttl": res, err = s.cmdTTL(msg) case "stats": - res, err = s.cmdStats(msg) + res, err = s.cmdSTATS(msg) case "scan": res, err = s.cmdScan(msg) case "nearby": @@ -624,9 +624,9 @@ func (s *Server) commandInScript(msg *Message) ( case "search": res, err = s.cmdSearch(msg) case "bounds": - res, err = s.cmdBounds(msg) + res, err = s.cmdBOUNDS(msg) case "get": - res, err = s.cmdGet(msg) + res, err = s.cmdGET(msg) case "jget": res, err = s.cmdJget(msg) case "jset": @@ -634,13 +634,13 @@ func (s *Server) commandInScript(msg *Message) ( case "jdel": res, d, err = s.cmdJdel(msg) case "type": - res, err = s.cmdType(msg) + res, err = s.cmdTYPE(msg) case "keys": - res, err = s.cmdKeys(msg) + res, err = s.cmdKEYS(msg) case "test": - res, err = s.cmdTest(msg) + res, err = s.cmdTEST(msg) case "server": - res, err = s.cmdServer(msg) + res, err = s.cmdSERVER(msg) } s.sendMonitor(err, msg, nil, true) return diff --git a/internal/server/server.go b/internal/server/server.go index 50c24353..bf792724 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -80,21 +80,24 @@ type Server struct { config *Config epc *endpoint.Manager + lnmu sync.Mutex + ln net.Listener // server listener + // env opts geomParseOpts geojson.ParseOptions geomIndexOpts geometry.IndexOptions http500Errors bool // atomics - followc aint // counter increases when follow property changes - statsTotalConns aint // counter for total connections - statsTotalCommands aint // counter for total commands - statsTotalMsgsSent aint // counter for total sent webhook messages - statsExpired aint // item expiration counter - lastShrinkDuration aint - stopServer abool - outOfMemory abool - loadedAndReady abool // server is loaded and ready for commands + followc atomic.Int64 // counter when follow property changes + statsTotalConns atomic.Int64 // counter for total connections + statsTotalCommands atomic.Int64 // counter for total commands + statsTotalMsgsSent atomic.Int64 // counter for total sent webhook messages + statsExpired atomic.Int64 // item expiration counter + lastShrinkDuration atomic.Int64 + stopServer atomic.Bool + outOfMemory atomic.Bool + loadedAndReady atomic.Bool // server is loaded and ready for commands connsmu sync.RWMutex conns map[int]*Client @@ -114,7 +117,6 @@ type Server struct { lstack []*commandDetails lives map[*liveBuffer]bool lcond *sync.Cond - lwait sync.WaitGroup fcup bool // follow caught up fcuponce bool // follow caught up once shrinking bool // aof shrinking flag @@ -135,6 +137,8 @@ type Server struct { monconnsMu sync.RWMutex monconns map[net.Conn]bool // monitor connections + + opts Options } // Options for Serve() @@ -145,18 +149,51 @@ type Options struct { UseHTTP bool MetricsAddr string UnixSocketPath string // path for unix socket + + // DevMode puts application in to dev mode + DevMode bool + + // ShowDebugMessages allows for log.Debug to print to console. + ShowDebugMessages bool + + // ProtectedMode forces Tile38 to default in protected mode. + ProtectedMode string + + // AppendOnly allows for disabling the appendonly file. + AppendOnly bool + + // AppendFileName allows for custom appendonly file path + AppendFileName string + + // QueueFileName allows for custom queue.db file path + QueueFileName string + + // Shutdown allows for shutting down the server. + Shutdown <-chan bool } // Serve starts a new tile38 server func Serve(opts Options) error { - if core.AppendFileName == "" { - core.AppendFileName = path.Join(opts.Dir, "appendonly.aof") + if opts.AppendFileName == "" { + opts.AppendFileName = path.Join(opts.Dir, "appendonly.aof") } - if core.QueueFileName == "" { - core.QueueFileName = path.Join(opts.Dir, "queue.db") + if opts.QueueFileName == "" { + opts.QueueFileName = path.Join(opts.Dir, "queue.db") + } + if opts.ProtectedMode == "" { + opts.ProtectedMode = "no" } log.Infof("Server started, Tile38 version %s, git %s", core.Version, core.GitSHA) + defer func() { + log.Warn("Server has shutdown, bye now") + if false { + // prints the stack, looking for running goroutines. + buf := make([]byte, 10000) + n := runtime.Stack(buf, true) + println(string(buf[:n])) + } + }() // Initialize the s s := &Server{ @@ -183,9 +220,11 @@ func Serve(opts Options) error { groupHooks: btree.NewNonConcurrent(byGroupHook), groupObjects: btree.NewNonConcurrent(byGroupObject), hookExpires: btree.NewNonConcurrent(byHookExpires), + opts: opts, } s.epc = endpoint.NewManager(s) + defer s.epc.Shutdown() s.luascripts = s.newScriptMap() s.luapool = s.newPool() defer s.luapool.Shutdown() @@ -255,8 +294,25 @@ func Serve(opts Options) error { nerr <- s.netServe() }() + go func() { + <-opts.Shutdown + s.stopServer.Store(true) + log.Warnf("Shutting down...") + s.lnmu.Lock() + ln := s.ln + s.ln = nil + s.lnmu.Unlock() + if ln != nil { + ln.Close() + } + for conn, f := range s.aofconnM { + conn.Close() + f.Close() + } + }() + // Load the queue before the aof - qdb, err := buntdb.Open(core.QueueFileName) + qdb, err := buntdb.Open(opts.QueueFileName) if err != nil { return err } @@ -284,8 +340,8 @@ func Serve(opts Options) error { if err := s.migrateAOF(); err != nil { return err } - if core.AppendOnly { - f, err := os.OpenFile(core.AppendFileName, os.O_CREATE|os.O_RDWR, 0600) + if opts.AppendOnly { + f, err := os.OpenFile(opts.AppendFileName, os.O_CREATE|os.O_RDWR, 0600) if err != nil { return err } @@ -300,41 +356,69 @@ func Serve(opts Options) error { } // Start background routines - if s.config.followHost() != "" { - go s.follow(s.config.followHost(), s.config.followPort(), - s.followc.get()) - } + var bgwg sync.WaitGroup - if opts.MetricsAddr != "" { - log.Infof("Listening for metrics at: %s", opts.MetricsAddr) + if s.config.followHost() != "" { + bgwg.Add(1) go func() { - http.HandleFunc("/", s.MetricsIndexHandler) - http.HandleFunc("/metrics", s.MetricsHandler) - log.Fatal(http.ListenAndServe(opts.MetricsAddr, nil)) + defer bgwg.Done() + s.follow(s.config.followHost(), s.config.followPort(), + int(s.followc.Load())) }() } - s.lwait.Add(1) - go s.processLives() - go s.watchOutOfMemory() - go s.watchLuaStatePool() - go s.watchAutoGC() - go s.backgroundExpiring() - go s.backgroundSyncAOF() + var mln net.Listener + if opts.MetricsAddr != "" { + log.Infof("Listening for metrics at: %s", opts.MetricsAddr) + mln, err = net.Listen("tcp", opts.MetricsAddr) + if err != nil { + return err + } + bgwg.Add(1) + go func() { + defer bgwg.Done() + smux := http.NewServeMux() + smux.HandleFunc("/", s.MetricsIndexHandler) + smux.HandleFunc("/metrics", s.MetricsHandler) + err := http.Serve(mln, smux) + if err != nil { + if !s.stopServer.Load() { + log.Fatalf("metrics server: %s", err) + } + } + }() + } + + bgwg.Add(1) + go s.processLives(&bgwg) + bgwg.Add(1) + go s.watchOutOfMemory(&bgwg) + bgwg.Add(1) + go s.watchLuaStatePool(&bgwg) + bgwg.Add(1) + go s.watchAutoGC(&bgwg) + bgwg.Add(1) + go s.backgroundExpiring(&bgwg) + bgwg.Add(1) + go s.backgroundSyncAOF(&bgwg) defer func() { + log.Debug("Stopping background routines") // Stop background routines - s.followc.add(1) // this will force any follow communication to die - s.stopServer.set(true) - s.lwait.Wait() + s.followc.Add(1) // this will force any follow communication to die + s.stopServer.Store(true) + if mln != nil { + mln.Close() // Stop the metrics server + } + bgwg.Wait() }() // Server is now loaded and ready. Wait for network error messages. - s.loadedAndReady.set(true) + s.loadedAndReady.Store(true) return <-nerr } func (s *Server) isProtected() bool { - if core.ProtectedMode == "no" { + if s.opts.ProtectedMode == "no" { // --protected-mode no return false } @@ -360,28 +444,52 @@ func (s *Server) netServe() error { if err != nil { return err } - defer ln.Close() + s.lnmu.Lock() + s.ln = ln + s.lnmu.Unlock() + + var wg sync.WaitGroup + defer func() { + log.Debug("Closing client connections...") + s.connsmu.RLock() + for _, c := range s.conns { + c.closer.Close() + } + s.connsmu.RUnlock() + wg.Wait() + ln.Close() + log.Debug("Client connection closed") + }() + log.Infof("Ready to accept connections at %s", ln.Addr()) var clientID int64 for { conn, err := ln.Accept() if err != nil { - return err + if s.stopServer.Load() { + return nil + } + log.Warn(err) + time.Sleep(time.Second / 5) + continue } - + wg.Add(1) go func(conn net.Conn) { + defer wg.Done() + // open connection // create the client client := new(Client) client.id = int(atomic.AddInt64(&clientID, 1)) client.opened = time.Now() client.remoteAddr = conn.RemoteAddr().String() + client.closer = conn // add client to server map s.connsmu.Lock() s.conns[client.id] = client s.connsmu.Unlock() - s.statsTotalConns.add(1) + s.statsTotalConns.Add(1) // set the client keep-alive, if needed if s.config.keepAlive() > 0 { @@ -460,7 +568,7 @@ func (s *Server) netServe() error { client.mu.Unlock() // update total command count - s.statsTotalCommands.add(1) + s.statsTotalCommands.Add(1) // handle the command err := s.handleInputCommand(client, msg) @@ -592,20 +700,16 @@ func (conn *liveConn) SetWriteDeadline(deadline time.Time) error { panic("not supported") } -func (s *Server) watchAutoGC() { - t := time.NewTicker(time.Second) - defer t.Stop() +func (s *Server) watchAutoGC(wg *sync.WaitGroup) { + defer wg.Done() start := time.Now() - for range t.C { - if s.stopServer.on() { - return - } + s.loopUntilServerStops(time.Second, func() { autoGC := s.config.autoGC() if autoGC == 0 { - continue + return } if time.Since(start) < time.Second*time.Duration(autoGC) { - continue + return } var mem1, mem2 runtime.MemStats runtime.ReadMemStats(&mem1) @@ -620,58 +724,65 @@ func (s *Server) watchAutoGC() { "alloc: %v, heap_alloc: %v, heap_released: %v", mem2.Alloc, mem2.HeapAlloc, mem2.HeapReleased) start = time.Now() - } + }) } -func (s *Server) watchOutOfMemory() { - t := time.NewTicker(time.Second * 2) - defer t.Stop() +func (s *Server) checkOutOfMemory() { + if s.stopServer.Load() { + return + } + oom := s.outOfMemory.Load() var mem runtime.MemStats - for range t.C { - func() { - if s.stopServer.on() { - return - } - oom := s.outOfMemory.on() - if s.config.maxMemory() == 0 { - if oom { - s.outOfMemory.set(false) - } - return - } - if oom { - runtime.GC() - } - runtime.ReadMemStats(&mem) - s.outOfMemory.set(int(mem.HeapAlloc) > s.config.maxMemory()) - }() + if s.config.maxMemory() == 0 { + if oom { + s.outOfMemory.Store(false) + } + return + } + if oom { + runtime.GC() + } + runtime.ReadMemStats(&mem) + s.outOfMemory.Store(int(mem.HeapAlloc) > s.config.maxMemory()) +} + +func (s *Server) loopUntilServerStops(dur time.Duration, op func()) { + var last time.Time + for { + if s.stopServer.Load() { + return + } + now := time.Now() + if now.Sub(last) > dur { + op() + last = now + } + time.Sleep(time.Second / 5) } } -func (s *Server) watchLuaStatePool() { - t := time.NewTicker(time.Second * 10) - defer t.Stop() - for range t.C { - func() { - s.luapool.Prune() - }() - } +func (s *Server) watchOutOfMemory(wg *sync.WaitGroup) { + defer wg.Done() + s.loopUntilServerStops(time.Second*4, func() { + s.checkOutOfMemory() + }) +} + +func (s *Server) watchLuaStatePool(wg *sync.WaitGroup) { + defer wg.Done() + s.loopUntilServerStops(time.Second*10, func() { + s.luapool.Prune() + }) } // backgroundSyncAOF ensures that the aof buffer is does not grow too big. -func (s *Server) backgroundSyncAOF() { - t := time.NewTicker(time.Second) - defer t.Stop() - for range t.C { - if s.stopServer.on() { - return - } - func() { - s.mu.Lock() - defer s.mu.Unlock() - s.flushAOF(true) - }() - } +func (s *Server) backgroundSyncAOF(wg *sync.WaitGroup) { + defer wg.Done() + s.loopUntilServerStops(time.Second, func() { + s.mu.Lock() + defer s.mu.Unlock() + s.flushAOF(true) + }) } func isReservedFieldName(field string) bool { @@ -812,7 +923,7 @@ func (s *Server) handleInputCommand(client *Client, msg *Message) error { return nil } - if !s.loadedAndReady.on() { + if !s.loadedAndReady.Load() { switch msg.Command() { case "output", "ping", "echo": default: @@ -1014,17 +1125,17 @@ func (s *Server) command(msg *Message, client *Client) ( case "fset": res, d, err = s.cmdFSET(msg) case "del": - res, d, err = s.cmdDel(msg) + res, d, err = s.cmdDEL(msg) case "pdel": - res, d, err = s.cmdPdel(msg) + res, d, err = s.cmdPDEL(msg) case "drop": - res, d, err = s.cmdDrop(msg) + res, d, err = s.cmdDROP(msg) case "flushdb": res, d, err = s.cmdFLUSHDB(msg) case "rename": - res, d, err = s.cmdRename(msg) + res, d, err = s.cmdRENAME(msg) case "renamenx": - res, d, err = s.cmdRename(msg) + res, d, err = s.cmdRENAME(msg) case "sethook": res, d, err = s.cmdSetHook(msg) case "delhook": @@ -1048,19 +1159,19 @@ func (s *Server) command(msg *Message, client *Client) ( case "ttl": res, err = s.cmdTTL(msg) case "shutdown": - if !core.DevMode { + if !s.opts.DevMode { err = fmt.Errorf("unknown command '%s'", msg.Args[0]) return } log.Fatal("shutdown requested by developer") case "massinsert": - if !core.DevMode { + if !s.opts.DevMode { err = fmt.Errorf("unknown command '%s'", msg.Args[0]) return } res, err = s.cmdMassInsert(msg) case "sleep": - if !core.DevMode { + if !s.opts.DevMode { err = fmt.Errorf("unknown command '%s'", msg.Args[0]) return } @@ -1070,15 +1181,15 @@ func (s *Server) command(msg *Message, client *Client) ( case "replconf": res, err = s.cmdReplConf(msg, client) case "readonly": - res, err = s.cmdReadOnly(msg) + res, err = s.cmdREADONLY(msg) case "stats": - res, err = s.cmdStats(msg) + res, err = s.cmdSTATS(msg) case "server": - res, err = s.cmdServer(msg) + res, err = s.cmdSERVER(msg) case "healthz": - res, err = s.cmdHealthz(msg) + res, err = s.cmdHEALTHZ(msg) case "info": - res, err = s.cmdInfo(msg) + res, err = s.cmdINFO(msg) case "scan": res, err = s.cmdScan(msg) case "nearby": @@ -1090,9 +1201,9 @@ func (s *Server) command(msg *Message, client *Client) ( case "search": res, err = s.cmdSearch(msg) case "bounds": - res, err = s.cmdBounds(msg) + res, err = s.cmdBOUNDS(msg) case "get": - res, err = s.cmdGet(msg) + res, err = s.cmdGET(msg) case "jget": res, err = s.cmdJget(msg) case "jset": @@ -1100,11 +1211,11 @@ func (s *Server) command(msg *Message, client *Client) ( case "jdel": res, d, err = s.cmdJdel(msg) case "type": - res, err = s.cmdType(msg) + res, err = s.cmdTYPE(msg) case "keys": - res, err = s.cmdKeys(msg) + res, err = s.cmdKEYS(msg) case "output": - res, err = s.cmdOutput(msg) + res, err = s.cmdOUTPUT(msg) case "aof": res, err = s.cmdAOF(msg) case "aofmd5": @@ -1132,7 +1243,7 @@ func (s *Server) command(msg *Message, client *Client) ( return s.command(msg, client) } case "client": - res, err = s.cmdClient(msg, client) + res, err = s.cmdCLIENT(msg, client) case "eval", "evalro", "evalna": res, err = s.cmdEvalUnified(false, msg) case "evalsha", "evalrosha", "evalnasha": @@ -1150,10 +1261,11 @@ func (s *Server) command(msg *Message, client *Client) ( case "publish": res, err = s.cmdPublish(msg) case "test": - res, err = s.cmdTest(msg) + res, err = s.cmdTEST(msg) case "monitor": res, err = s.cmdMonitor(msg) } + s.sendMonitor(err, msg, client, false) return } diff --git a/internal/server/stats.go b/internal/server/stats.go index 87238561..0d324f02 100644 --- a/internal/server/stats.go +++ b/internal/server/stats.go @@ -45,22 +45,23 @@ func readMemStats() runtime.MemStats { return ms } -func (s *Server) cmdStats(msg *Message) (res resp.Value, err error) { +// STATS key [key...] +func (s *Server) cmdSTATS(msg *Message) (resp.Value, error) { start := time.Now() - vs := msg.Args[1:] - var ms = []map[string]interface{}{} - if len(vs) == 0 { - return NOMessage, errInvalidNumberOfArguments + // >> Args + + args := msg.Args + if len(args) < 2 { + return retrerr(errInvalidNumberOfArguments) } + + // >> Operation + var vals []resp.Value - var key string - var ok bool - for { - vs, key, ok = tokenval(vs) - if !ok { - break - } + var ms = []map[string]interface{}{} + for i := 1; i < len(args); i++ { + key := args[i] col, _ := s.cols.Get(key) if col != nil { m := make(map[string]interface{}) @@ -83,67 +84,82 @@ func (s *Server) cmdStats(msg *Message) (res resp.Value, err error) { } } } - switch msg.OutputType { - case JSON: - data, err := json.Marshal(ms) - if err != nil { - return NOMessage, err - } - res = resp.StringValue(`{"ok":true,"stats":` + string(data) + `,"elapsed":"` + time.Since(start).String() + "\"}") - case RESP: - res = resp.ArrayValue(vals) + // >> Response + + if msg.OutputType == JSON { + data, _ := json.Marshal(ms) + return resp.StringValue(`{"ok":true,"stats":` + string(data) + + `,"elapsed":"` + time.Since(start).String() + "\"}"), nil } - return res, nil + return resp.ArrayValue(vals), nil } -func (s *Server) cmdHealthz(msg *Message) (res resp.Value, err error) { +// HEALTHZ +func (s *Server) cmdHEALTHZ(msg *Message) (resp.Value, error) { start := time.Now() + + // >> Args + + args := msg.Args + if len(args) != 1 { + return retrerr(errInvalidNumberOfArguments) + } + + // >> Operation + if s.config.followHost() != "" { m := make(map[string]interface{}) s.basicStats(m) if fmt.Sprintf("%v", m["caught_up"]) != "true" { - return NOMessage, errors.New("not caught up") + return retrerr(errors.New("not caught up")) } } - switch msg.OutputType { - case JSON: - res = resp.StringValue(`{"ok":true,"elapsed":"` + time.Since(start).String() + "\"}") - case RESP: - res = resp.SimpleStringValue("OK") + + // >> Response + + if msg.OutputType == JSON { + return resp.StringValue(`{"ok":true,"elapsed":"` + + time.Since(start).String() + "\"}"), nil } - return res, nil + return resp.SimpleStringValue("OK"), nil } -func (s *Server) cmdServer(msg *Message) (res resp.Value, err error) { +// SERVER [ext] +func (s *Server) cmdSERVER(msg *Message) (resp.Value, error) { start := time.Now() + + // >> Args + + args := msg.Args + var ext bool + for i := 1; i < len(args); i++ { + switch strings.ToLower(args[i]) { + case "ext": + ext = true + default: + return retrerr(errInvalidArgument(args[i])) + } + } + + // >> Operation + m := make(map[string]interface{}) - args := msg.Args[1:] - // Switch on the type of stats requested - switch len(args) { - case 0: + if ext { + s.extStats(m) + } else { s.basicStats(m) - case 1: - if strings.ToLower(args[0]) == "ext" { - s.extStats(m) - } - default: - return NOMessage, errInvalidNumberOfArguments } - switch msg.OutputType { - case JSON: - data, err := json.Marshal(m) - if err != nil { - return NOMessage, err - } - res = resp.StringValue(`{"ok":true,"stats":` + string(data) + `,"elapsed":"` + time.Since(start).String() + "\"}") - case RESP: - vals := respValuesSimpleMap(m) - res = resp.ArrayValue(vals) + // >> Response + + if msg.OutputType == JSON { + data, _ := json.Marshal(m) + return resp.StringValue(`{"ok":true,"stats":` + string(data) + + `,"elapsed":"` + time.Since(start).String() + "\"}"), nil } - return res, nil + return resp.ArrayValue(respValuesSimpleMap(m)), nil } // basicStats populates the passed map with basic system/go/tile38 statistics @@ -302,11 +318,11 @@ func (s *Server) extStats(m map[string]interface{}) { // Whether or not a cluster is enabled m["tile38_cluster_enabled"] = false // Whether or not the Tile38 AOF is enabled - m["tile38_aof_enabled"] = core.AppendOnly + m["tile38_aof_enabled"] = s.opts.AppendOnly // Whether or not an AOF shrink is currently in progress m["tile38_aof_rewrite_in_progress"] = s.shrinking // Length of time the last AOF shrink took - m["tile38_aof_last_rewrite_time_sec"] = s.lastShrinkDuration.get() / int(time.Second) + m["tile38_aof_last_rewrite_time_sec"] = s.lastShrinkDuration.Load() / int64(time.Second) // Duration of the on-going AOF rewrite operation if any var currentShrinkStart time.Time if currentShrinkStart.IsZero() { @@ -319,13 +335,13 @@ func (s *Server) extStats(m map[string]interface{}) { // Whether or no the HTTP transport is being served m["tile38_http_transport"] = s.http // Number of connections accepted by the server - m["tile38_total_connections_received"] = s.statsTotalConns.get() + m["tile38_total_connections_received"] = s.statsTotalConns.Load() // Number of commands processed by the server - m["tile38_total_commands_processed"] = s.statsTotalCommands.get() + m["tile38_total_commands_processed"] = s.statsTotalCommands.Load() // Number of webhook messages sent by server - m["tile38_total_messages_sent"] = s.statsTotalMsgsSent.get() + m["tile38_total_messages_sent"] = s.statsTotalMsgsSent.Load() // Number of key expiration events - m["tile38_expired_keys"] = s.statsExpired.get() + m["tile38_expired_keys"] = s.statsExpired.Load() // Number of connected slaves m["tile38_connected_slaves"] = len(s.aofconnM) @@ -393,9 +409,9 @@ func boolInt(t bool) int { return 0 } func (s *Server) writeInfoPersistence(w *bytes.Buffer) { - fmt.Fprintf(w, "aof_enabled:%d\r\n", boolInt(core.AppendOnly)) - fmt.Fprintf(w, "aof_rewrite_in_progress:%d\r\n", boolInt(s.shrinking)) // Flag indicating a AOF rewrite operation is on-going - fmt.Fprintf(w, "aof_last_rewrite_time_sec:%d\r\n", s.lastShrinkDuration.get()/int(time.Second)) // Duration of the last AOF rewrite operation in seconds + fmt.Fprintf(w, "aof_enabled:%d\r\n", boolInt(s.opts.AppendOnly)) + fmt.Fprintf(w, "aof_rewrite_in_progress:%d\r\n", boolInt(s.shrinking)) // Flag indicating a AOF rewrite operation is on-going + fmt.Fprintf(w, "aof_last_rewrite_time_sec:%d\r\n", s.lastShrinkDuration.Load()/int64(time.Second)) // Duration of the last AOF rewrite operation in seconds var currentShrinkStart time.Time // c.currentShrinkStart.get() if currentShrinkStart.IsZero() { @@ -406,10 +422,10 @@ func (s *Server) writeInfoPersistence(w *bytes.Buffer) { } func (s *Server) writeInfoStats(w *bytes.Buffer) { - fmt.Fprintf(w, "total_connections_received:%d\r\n", s.statsTotalConns.get()) // Total number of connections accepted by the server - fmt.Fprintf(w, "total_commands_processed:%d\r\n", s.statsTotalCommands.get()) // Total number of commands processed by the server - fmt.Fprintf(w, "total_messages_sent:%d\r\n", s.statsTotalMsgsSent.get()) // Total number of commands processed by the server - fmt.Fprintf(w, "expired_keys:%d\r\n", s.statsExpired.get()) // Total number of key expiration events + fmt.Fprintf(w, "total_connections_received:%d\r\n", s.statsTotalConns.Load()) // Total number of connections accepted by the server + fmt.Fprintf(w, "total_commands_processed:%d\r\n", s.statsTotalCommands.Load()) // Total number of commands processed by the server + fmt.Fprintf(w, "total_messages_sent:%d\r\n", s.statsTotalMsgsSent.Load()) // Total number of commands processed by the server + fmt.Fprintf(w, "expired_keys:%d\r\n", s.statsExpired.Load()) // Total number of key expiration events } // writeInfoReplication writes all replication data to the 'info' response @@ -441,27 +457,52 @@ func (s *Server) writeInfoCluster(w *bytes.Buffer) { fmt.Fprintf(w, "cluster_enabled:0\r\n") } -func (s *Server) cmdInfo(msg *Message) (res resp.Value, err error) { +// INFO [section ...] +func (s *Server) cmdINFO(msg *Message) (res resp.Value, err error) { start := time.Now() - sections := []string{"server", "clients", "memory", "persistence", "stats", "replication", "cpu", "cluster", "keyspace"} - switch len(msg.Args) { - default: - return NOMessage, errInvalidNumberOfArguments - case 1: - case 2: - section := strings.ToLower(msg.Args[1]) + // >> Args + + args := msg.Args + + msects := make(map[string]bool) + allsects := []string{ + "server", "clients", "memory", "persistence", "stats", + "replication", "cpu", "cluster", "keyspace", + } + + if len(args) == 1 { + for _, s := range allsects { + msects[s] = true + } + } + for i := 1; i < len(args); i++ { + section := strings.ToLower(args[i]) switch section { + case "all", "default": + for _, s := range allsects { + msects[s] = true + } default: - sections = []string{section} - case "all": - sections = []string{"server", "clients", "memory", "persistence", "stats", "replication", "cpu", "commandstats", "cluster", "keyspace"} - case "default": + for _, s := range allsects { + if s == section { + msects[section] = true + } + } + } + } + + // >> Operation + + var sects []string + for _, s := range allsects { + if msects[s] { + sects = append(sects, s) } } w := &bytes.Buffer{} - for i, section := range sections { + for i, section := range sects { if i > 0 { w.WriteString("\r\n") } @@ -495,8 +536,9 @@ func (s *Server) cmdInfo(msg *Message) (res resp.Value, err error) { } } - switch msg.OutputType { - case JSON: + // >> Response + + if msg.OutputType == JSON { // Create a map of all key/value info fields m := make(map[string]interface{}) for _, kv := range strings.Split(w.String(), "\r\n") { @@ -509,15 +551,11 @@ func (s *Server) cmdInfo(msg *Message) (res resp.Value, err error) { } // Marshal the map and use the output in the JSON response - data, err := json.Marshal(m) - if err != nil { - return NOMessage, err - } - res = resp.StringValue(`{"ok":true,"info":` + string(data) + `,"elapsed":"` + time.Since(start).String() + "\"}") - case RESP: - res = resp.BytesValue(w.Bytes()) + data, _ := json.Marshal(m) + return resp.StringValue(`{"ok":true,"info":` + string(data) + + `,"elapsed":"` + time.Since(start).String() + "\"}"), nil } - return res, nil + return resp.BytesValue(w.Bytes()), nil } // tryParseType attempts to parse the passed string as an integer, float64 and diff --git a/internal/server/test.go b/internal/server/test.go index 53e0d889..225f5786 100644 --- a/internal/server/test.go +++ b/internal/server/test.go @@ -50,7 +50,7 @@ func (s *Server) parseArea(ovs []string, doClip bool) (vs []string, o geojson.Ob o = geojson.NewPoint(geometry.Point{X: lon, Y: lat}) case "sector": if doClip { - err = errInvalidArgument("cannot clip with " + ltyp) + err = fmt.Errorf("invalid clip type '%s'", typ) return } var slat, slon, smeters, sb1, sb2 string @@ -288,8 +288,15 @@ func (s *Server) parseArea(ovs []string, doClip bool) (vs []string, o geojson.Ob return } -func (s *Server) cmdTest(msg *Message) (res resp.Value, err error) { +// TEST (POINT lat lon)|(GET key id)|(BOUNDS minlat minlon maxlat maxlon)| +// (OBJECT geojson)|(CIRCLE lat lon meters)|(TILE x y z)|(QUADKEY quadkey)| +// (HASH geohash) INTERSECTS|WITHIN [CLIP] (POINT lat lon)|(GET key id)| +// (BOUNDS minlat minlon maxlat maxlon)|(OBJECT geojson)| +// (CIRCLE lat lon meters)|(TILE x y z)|(QUADKEY quadkey)|(HASH geohash)| +// (SECTOR lat lon meters bearing1 bearing2) +func (s *Server) cmdTEST(msg *Message) (res resp.Value, err error) { start := time.Now() + vs := msg.Args[1:] var ok bool @@ -348,8 +355,7 @@ func (s *Server) cmdTest(msg *Message) (res resp.Value, err error) { } } } - switch msg.OutputType { - case JSON: + if msg.OutputType == JSON { var buf bytes.Buffer buf.WriteString(`{"ok":true`) if result != 0 { @@ -362,13 +368,11 @@ func (s *Server) cmdTest(msg *Message) (res resp.Value, err error) { } buf.WriteString(`,"elapsed":"` + time.Since(start).String() + "\"}") return resp.StringValue(buf.String()), nil - case RESP: - if clipped != nil { - return resp.ArrayValue([]resp.Value{ - resp.IntegerValue(result), - resp.StringValue(clipped.JSON())}), nil - } - return resp.IntegerValue(result), nil } - return NOMessage, nil + if clipped != nil { + return resp.ArrayValue([]resp.Value{ + resp.IntegerValue(result), + resp.StringValue(clipped.JSON())}), nil + } + return resp.IntegerValue(result), nil } diff --git a/scripts/test.sh b/scripts/test.sh index 1fc9288b..acfc97f0 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -5,10 +5,17 @@ cd $(dirname "${BASH_SOURCE[0]}")/.. export CGO_ENABLED=0 -# if [ "$NOMODULES" != "1" ]; then -# export GO111MODULE=on -# export GOFLAGS=-mod=vendor -# fi +cd tests +go test -coverpkg=../internal/server -coverprofile=/tmp/coverage.out $GOTEST -cd tests && go test && cd .. -go test $(go list ./... | grep -v /vendor/ | grep -v /tests) + +# go test -coverpkg=../internal/... -coverprofile=/tmp/coverage.out \ +# -v ./... $GOTEST + +go tool cover -html=/tmp/coverage.out -o /tmp/coverage.html +echo "details: file:///tmp/coverage.html" +cd .. + +if [[ "$GOTEST" == "" ]]; then + go test $(go list ./... | grep -v /vendor/ | grep -v /tests) +fi diff --git a/tests/aof_legacy b/tests/aof_legacy new file mode 100644 index 00000000..b0fd1d5e Binary files /dev/null and b/tests/aof_legacy differ diff --git a/tests/aof_test.go b/tests/aof_test.go new file mode 100644 index 00000000..b0734f84 --- /dev/null +++ b/tests/aof_test.go @@ -0,0 +1,347 @@ +package tests + +import ( + "bytes" + "crypto/md5" + "encoding/hex" + "errors" + "fmt" + "io" + "math/rand" + "net" + "net/http" + "sync/atomic" + "time" + + "github.com/gomodule/redigo/redis" + + _ "embed" +) + +func subTestAOF(g *testGroup) { + g.regSubTest("loading", aof_loading_test) + g.regSubTest("migrate", aof_migrate_test) + g.regSubTest("AOF", aof_AOF_test) + g.regSubTest("AOFMD5", aof_AOFMD5_test) + g.regSubTest("AOFSHRINK", aof_AOFSHRINK_test) + g.regSubTest("READONLY", aof_READONLY_test) +} + +func loadAOFAndClose(aof any) error { + mc, err := loadAOF(aof) + if mc != nil { + mc.Close() + } + return err +} + +func loadAOF(aof any) (*mockServer, error) { + var aofb []byte + switch aof := aof.(type) { + case []byte: + aofb = []byte(aof) + case string: + aofb = []byte(aof) + default: + return nil, errors.New("aof is not string or bytes") + } + return mockOpenServer(MockServerOptions{ + Silent: true, + Metrics: false, + AOFData: aofb, + }) +} + +func aof_loading_test(mc *mockServer) error { + + var err error + // invalid command + err = loadAOFAndClose("asdfasdf\r\n") + if err == nil || err.Error() != "unknown command 'asdfasdf'" { + return fmt.Errorf("expected '%v', got '%v'", + "unknown command 'asdfasdf'", err) + } + + // incomplete command + err = loadAOFAndClose("set fleet truck point 10 10\r\nasdfasdf") + if err != nil { + return err + } + + // big aof file + var aof string + for i := 0; i < 10000; i++ { + aof += fmt.Sprintf("SET fleet truck%d POINT 10 10\r\n", i) + } + err = loadAOFAndClose(aof) + if err != nil { + return err + } + + // extra zeros at various places + aof = "" + for i := 0; i < 1000; i++ { + if i%10 == 0 { + aof += string(bytes.Repeat([]byte{0}, 100)) + } + aof += fmt.Sprintf("SET fleet truck%d POINT 10 10\r\n", i) + } + aof += string(bytes.Repeat([]byte{0}, 5000)) + err = loadAOFAndClose(aof) + if err != nil { + return err + } + + // bad protocol + aof = "*2\r\n$1\r\nh\r\n+OK\r\n" + err = loadAOFAndClose(aof) + if fmt.Sprintf("%v", err) != "Protocol error: expected '$', got '+'" { + return fmt.Errorf("expected '%v', got '%v'", + "Protocol error: expected '$', got '+'", err) + } + return nil +} + +func aof_AOFMD5_test(mc *mockServer) error { + for i := 0; i < 10000; i++ { + _, err := mc.Do("SET", "fleet", rand.Int(), + "POINT", rand.Float64()*180-90, rand.Float64()*360-180) + if err != nil { + return err + } + } + aof, err := mc.readAOF() + if err != nil { + return err + } + check := func(start, size int) func(s string) error { + return func(s string) error { + sum := md5.Sum(aof[start : start+size]) + val := hex.EncodeToString(sum[:]) + if s != val { + return fmt.Errorf("expected '%s', got '%s'", val, s) + } + return nil + } + } + return mc.DoBatch( + Do("AOFMD5").Err("wrong number of arguments for 'aofmd5' command"), + Do("AOFMD5", 0).Err("wrong number of arguments for 'aofmd5' command"), + Do("AOFMD5", 0, 0, 1).Err("wrong number of arguments for 'aofmd5' command"), + Do("AOFMD5", -1, 0).Err("invalid argument '-1'"), + Do("AOFMD5", 1, -1).Err("invalid argument '-1'"), + Do("AOFMD5", 0, 100000000000).Err("EOF"), + Do("AOFMD5", 0, 0).Str("d41d8cd98f00b204e9800998ecf8427e"), + Do("AOFMD5", 0, 0).JSON().Str(`{"ok":true,"md5":"d41d8cd98f00b204e9800998ecf8427e"}`), + Do("AOFMD5", 0, 0).Func(check(0, 0)), + Do("AOFMD5", 0, 1).Func(check(0, 1)), + Do("AOFMD5", 0, 100).Func(check(0, 100)), + Do("AOFMD5", 1002, 4321).Func(check(1002, 4321)), + ) +} + +func openFollower(mc *mockServer) (conn redis.Conn, err error) { + conn, err = redis.Dial("tcp", fmt.Sprintf(":%d", mc.port), + redis.DialReadTimeout(time.Second)) + if err != nil { + return nil, err + } + defer func() { + if err != nil { + conn.Close() + conn = nil + } + }() + if err := conn.Send("AOF", 0); err != nil { + return nil, err + } + if err := conn.Flush(); err != nil { + return nil, err + } + str, err := redis.String(conn.Receive()) + if err != nil { + return nil, err + } + if str != "OK" { + return nil, fmt.Errorf("expected '%s', got '%s'", "OK", str) + } + return conn, nil +} + +func aof_AOF_test(mc *mockServer) error { + var argss [][]interface{} + for i := 0; i < 10000; i++ { + args := []interface{}{"SET", "fleet", fmt.Sprint(rand.Int()), + "POINT", fmt.Sprint(rand.Float64()*180 - 90), + fmt.Sprint(rand.Float64()*360 - 180)} + argss = append(argss, args) + _, err := mc.Do(fmt.Sprint(args[0]), args[1:]...) + if err != nil { + return err + } + } + readAll := func() (conn redis.Conn, err error) { + conn, err = openFollower(mc) + if err != nil { + return + } + defer func() { + if err != nil { + conn.Close() + conn = nil + } + }() + var t bool + for i := 0; i < len(argss); i++ { + args, err := redis.Values(conn.Receive()) + if err != nil { + return nil, err + } + if t || (len(args) == len(argss[0]) && + fmt.Sprintf("%s", args[2]) == fmt.Sprintf("%s", argss[0][2])) { + t = true + if fmt.Sprintf("%s", args[2]) != + fmt.Sprintf("%s", argss[i][2]) { + return nil, fmt.Errorf("expected '%s', got '%s'", + argss[i][2], args[2]) + } + } else { + i-- + } + } + return conn, nil + } + + conn, err := readAll() + if err != nil { + return err + } + defer conn.Close() + _, err = conn.Do("fancy") // non-existent error + if err == nil || err.Error() != "EOF" { + return fmt.Errorf("expected '%v', got '%v'", "EOF", err) + } + + conn, err = readAll() + if err != nil { + return err + } + defer conn.Close() + _, err = conn.Do("quit") + if err == nil || err.Error() != "EOF" { + return fmt.Errorf("expected '%v', got '%v'", "EOF", err) + } + + return mc.DoBatch( + Do("AOF").Err("wrong number of arguments for 'aof' command"), + Do("AOF", 0, 0).Err("wrong number of arguments for 'aof' command"), + Do("AOF", -1).Err("invalid argument '-1'"), + Do("AOF", 1000000000000).Err("pos is too big, must be less that the aof_size of leader"), + ) +} + +func aof_AOFSHRINK_test(mc *mockServer) error { + var err error + haddr := fmt.Sprintf("localhost:%d", getNextPort()) + ln, err := net.Listen("tcp", haddr) + if err != nil { + return err + } + defer ln.Close() + var msgs atomic.Int32 + go func() { + mux := http.NewServeMux() + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + msgs.Add(1) + // println(r.URL.Path) + }) + http.Serve(ln, mux) + }() + err = mc.DoBatch( + Do("SETCHAN", "mychan", "INTERSECTS", "mi:0", "BOUNDS", -10, -10, 10, 10).Str("1"), + Do("SETHOOK", "myhook", "http://"+haddr, "INTERSECTS", "mi:0", "BOUNDS", -10, -10, 10, 10).Str("1"), + Do("MASSINSERT", 5, 10000).OK(), + ) + if err != nil { + return err + } + err = mc.DoBatch( + Do("AOFSHRINK").OK(), + Do("MASSINSERT", 5, 10000).OK(), + ) + if err != nil { + return err + } + nmsgs := msgs.Load() + if nmsgs == 0 { + return fmt.Errorf("expected > 0, got %d", nmsgs) + } + return err +} + +func aof_READONLY_test(mc *mockServer) error { + return mc.DoBatch( + Do("SET", "mykey", "myid", "POINT", "10", "10").OK(), + Do("READONLY", "yes").OK(), + Do("SET", "mykey", "myid", "POINT", "10", "10").Err("read only"), + Do("READONLY", "no").OK(), + Do("SET", "mykey", "myid", "POINT", "10", "10").OK(), + Do("READONLY").Err("wrong number of arguments for 'readonly' command"), + Do("READONLY", "maybe").Err("invalid argument 'maybe'"), + ) +} + +//go:embed aof_legacy +var aofLegacy []byte + +func aof_migrate_test(mc *mockServer) error { + var aof []byte + for i := 0; i < 10000; i++ { + aof = append(aof, aofLegacy...) + } + var mc2 *mockServer + var err error + defer func() { + mc2.Close() + }() + mc2, err = mockOpenServer(MockServerOptions{ + AOFFileName: "aof", + AOFData: aof, + Silent: true, + Metrics: true, + }) + if err != nil { + return err + } + err = mc2.DoBatch( + Do("GET", "1", "2").Str(`{"type":"Point","coordinates":[20,10]}`), + ) + if err != nil { + return err + } + mc2.Close() + + mc2, err = mockOpenServer(MockServerOptions{ + AOFFileName: "aof", + AOFData: aofLegacy[:len(aofLegacy)-1], + Silent: true, + Metrics: true, + }) + if err != io.ErrUnexpectedEOF { + return fmt.Errorf("expected '%v', got '%v'", io.ErrUnexpectedEOF, err) + } + mc2.Close() + + mc2, err = mockOpenServer(MockServerOptions{ + AOFFileName: "aof", + AOFData: aofLegacy[1:], + Silent: true, + Metrics: true, + }) + if err != io.ErrUnexpectedEOF { + return fmt.Errorf("expected '%v', got '%v'", io.ErrUnexpectedEOF, err) + } + mc2.Close() + + return nil +} diff --git a/tests/client_test.go b/tests/client_test.go index 016b88aa..2c8d3b7d 100644 --- a/tests/client_test.go +++ b/tests/client_test.go @@ -3,25 +3,33 @@ package tests import ( "errors" "fmt" - "testing" + "strings" "github.com/gomodule/redigo/redis" "github.com/tidwall/gjson" + "github.com/tidwall/pretty" ) -func subTestClient(t *testing.T, mc *mockServer) { - runStep(t, mc, "valid json", client_valid_json_test) - runStep(t, mc, "valid client count", info_valid_client_count_test) +func subTestClient(g *testGroup) { + g.regSubTest("OUTPUT", client_OUTPUT_test) + g.regSubTest("CLIENT", client_CLIENT_test) } -func client_valid_json_test(mc *mockServer) error { - if err := mc.DoBatch([][]interface{}{ +func client_OUTPUT_test(mc *mockServer) error { + if err := mc.DoBatch( // tests removal of "elapsed" member. - {"OUTPUT", "json"}, {`{"ok":true}`}, - {"OUTPUT", "resp"}, {`OK`}, - }); err != nil { + Do("OUTPUT", "json", "yaml").Err(`wrong number of arguments for 'output' command`), + Do("OUTPUT", "json").Str(`{"ok":true}`), + Do("OUTPUT").JSON().Str(`{"ok":true,"output":"json"}`), + Do("OUTPUT").Str(`resp`), // this is due to the internal Do test + Do("OUTPUT", "resp").OK(), + Do("OUTPUT", "yaml").Err(`invalid argument 'yaml'`), + Do("OUTPUT").Str(`resp`), + Do("OUTPUT").JSON().Str(`{"ok":true,"output":"json"}`), + ); err != nil { return err } + // run direct commands if _, err := mc.Do("OUTPUT", "json"); err != nil { return err @@ -45,9 +53,14 @@ func client_valid_json_test(mc *mockServer) error { return nil } -func info_valid_client_count_test(mc *mockServer) error { +func client_CLIENT_test(mc *mockServer) error { numConns := 20 var conns []redis.Conn + defer func() { + for i := range conns { + conns[i].Close() + } + }() for i := 0; i <= numConns; i++ { conn, err := redis.Dial("tcp", fmt.Sprintf(":%d", mc.port)) if err != nil { @@ -55,9 +68,16 @@ func info_valid_client_count_test(mc *mockServer) error { } conns = append(conns, conn) } - for i := range conns { - defer conns[i].Close() + + _, err := conns[1].Do("CLIENT", "setname", "cl1") + if err != nil { + return err } + _, err = conns[2].Do("CLIENT", "setname", "cl2") + if err != nil { + return err + } + if _, err := mc.Do("OUTPUT", "JSON"); err != nil { return err } @@ -69,9 +89,53 @@ func info_valid_client_count_test(mc *mockServer) error { if !ok { return errors.New("Failed to type assert CLIENT response") } - sres := string(bres) - if len(gjson.Get(sres, "list").Array()) < numConns { + sres := string(pretty.Pretty(bres)) + if int(gjson.Get(sres, "list.#").Int()) < numConns { return errors.New("Invalid number of connections") } - return nil + + client13ID := gjson.Get(sres, "list.13.id").String() + client14Addr := gjson.Get(sres, "list.14.addr").String() + client15Addr := gjson.Get(sres, "list.15.addr").String() + + return mc.DoBatch( + Do("CLIENT", "list").JSON().Func(func(s string) error { + if int(gjson.Get(s, "list.#").Int()) < numConns { + return errors.New("Invalid number of connections") + } + return nil + }), + Do("CLIENT", "list").Func(func(s string) error { + if len(strings.Split(strings.TrimSpace(s), "\n")) < numConns { + return errors.New("Invalid number of connections") + } + return nil + }), + Do("CLIENT").Err(`wrong number of arguments for 'client' command`), + Do("CLIENT", "hello").Err(`Syntax error, try CLIENT (LIST | KILL | GETNAME | SETNAME)`), + Do("CLIENT", "list", "arg3").Err(`wrong number of arguments for 'client' command`), + Do("CLIENT", "getname", "arg3").Err(`wrong number of arguments for 'client' command`), + Do("CLIENT", "getname").JSON().Str(`{"ok":true,"name":""}`), + Do("CLIENT", "getname").Str(``), + Do("CLIENT", "setname", "abc").OK(), + Do("CLIENT", "getname").Str(`abc`), + Do("CLIENT", "getname").JSON().Str(`{"ok":true,"name":"abc"}`), + Do("CLIENT", "setname", "abc", "efg").Err(`wrong number of arguments for 'client' command`), + Do("CLIENT", "setname", " abc ").Err(`Client names cannot contain spaces, newlines or special characters.`), + Do("CLIENT", "setname", "abcd").JSON().OK(), + Do("CLIENT", "kill", "name", "abcd").Err("No such client"), + Do("CLIENT", "getname").Str(`abcd`), + Do("CLIENT", "kill").Err(`wrong number of arguments for 'client' command`), + Do("CLIENT", "kill", "").Err(`No such client`), + Do("CLIENT", "kill", "abcd").Err(`No such client`), + Do("CLIENT", "kill", "id", client13ID).OK(), + Do("CLIENT", "kill", "id").Err("wrong number of arguments for 'client' command"), + Do("CLIENT", "kill", client14Addr).OK(), + Do("CLIENT", "kill", client14Addr, "yikes").Err("wrong number of arguments for 'client' command"), + Do("CLIENT", "kill", "addr").Err("wrong number of arguments for 'client' command"), + Do("CLIENT", "kill", "addr", client15Addr).JSON().OK(), + Do("CLIENT", "kill", "addr", client14Addr, "yikes").Err("wrong number of arguments for 'client' command"), + Do("CLIENT", "kill", "id", "1000").Err("No such client"), + ) + } diff --git a/tests/fence_roaming_test.go b/tests/fence_roaming_test.go index e96084a3..4ba6796d 100644 --- a/tests/fence_roaming_test.go +++ b/tests/fence_roaming_test.go @@ -2,7 +2,7 @@ package tests import ( "fmt" - "io/ioutil" + "io" "net/http" "net/http/httptest" "sync" @@ -29,7 +29,7 @@ func fence_roaming_webhook_test(mc *mockServer) error { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if err := func() error { // Read the request body - body, err := ioutil.ReadAll(r.Body) + body, err := io.ReadAll(r.Body) if err != nil { return err } @@ -115,8 +115,10 @@ func fence_roaming_live_test(mc *mockServer) error { liveReady.Add(1) return goMultiFunc(mc, func() error { - sc, err := redis.DialTimeout("tcp", fmt.Sprintf(":%d", mc.port), - 0, time.Second*5, time.Second*5) + sc, err := redis.Dial("tcp", fmt.Sprintf(":%d", mc.port), + redis.DialConnectTimeout(0), + redis.DialReadTimeout(time.Second*5), + redis.DialWriteTimeout(time.Second*5)) if err != nil { liveReady.Done() return err diff --git a/tests/fence_test.go b/tests/fence_test.go index 013d4b19..d668dbb7 100644 --- a/tests/fence_test.go +++ b/tests/fence_test.go @@ -12,30 +12,29 @@ import ( "strings" "sync" "sync/atomic" - "testing" "time" "github.com/gomodule/redigo/redis" "github.com/tidwall/gjson" ) -func subTestFence(t *testing.T, mc *mockServer) { +func subTestFence(g *testGroup) { // Standard - runStep(t, mc, "basic", fence_basic_test) - runStep(t, mc, "channel message order", fence_channel_message_order_test) - runStep(t, mc, "detect inside,outside", fence_detect_inside_test) + g.regSubTest("basic", fence_basic_test) + g.regSubTest("channel message order", fence_channel_message_order_test) + g.regSubTest("detect inside,outside", fence_detect_inside_test) // Roaming - runStep(t, mc, "roaming live", fence_roaming_live_test) - runStep(t, mc, "roaming channel", fence_roaming_channel_test) - runStep(t, mc, "roaming webhook", fence_roaming_webhook_test) + g.regSubTest("roaming live", fence_roaming_live_test) + g.regSubTest("roaming channel", fence_roaming_channel_test) + g.regSubTest("roaming webhook", fence_roaming_webhook_test) // channel meta - runStep(t, mc, "channel meta", fence_channel_meta_test) + g.regSubTest("channel meta", fence_channel_meta_test) // various - runStep(t, mc, "detect eecio", fence_eecio_test) + g.regSubTest("detect eecio", fence_eecio_test) } type fenceReader struct { @@ -205,7 +204,7 @@ func fence_channel_message_order_test(mc *mockServer) error { break loop } case error: - fmt.Printf(err.Error()) + fmt.Printf("%s\n", err.Error()) } } @@ -230,10 +229,10 @@ func fence_channel_message_order_test(mc *mockServer) error { // Fire all setup commands on the base client for _, cmd := range []string{ "SET points point POINT 33.412529053733444 -111.93368911743164", - fmt.Sprintf(`SETCHAN A WITHIN points FENCE OBJECT {"type":"Polygon","coordinates":[[[-111.95205688476562,33.400491820565236],[-111.92630767822266,33.400491820565236],[-111.92630767822266,33.422272258866045],[-111.95205688476562,33.422272258866045],[-111.95205688476562,33.400491820565236]]]}`), - fmt.Sprintf(`SETCHAN B WITHIN points FENCE OBJECT {"type":"Polygon","coordinates":[[[-111.93952560424803,33.403501285221594],[-111.92630767822266,33.403501285221594],[-111.92630767822266,33.41997983836345],[-111.93952560424803,33.41997983836345],[-111.93952560424803,33.403501285221594]]]}`), - fmt.Sprintf(`SETCHAN C WITHIN points FENCE OBJECT {"type":"Polygon","coordinates":[[[-111.9255781173706,33.40342963251261],[-111.91201686859131,33.40342963251261],[-111.91201686859131,33.41994401881284],[-111.9255781173706,33.41994401881284],[-111.9255781173706,33.40342963251261]]]}`), - fmt.Sprintf(`SETCHAN D WITHIN points FENCE OBJECT {"type":"Polygon","coordinates":[[[-111.92562103271484,33.40063513076968],[-111.90021514892578,33.40063513076968],[-111.90021514892578,33.42212898435788],[-111.92562103271484,33.42212898435788],[-111.92562103271484,33.40063513076968]]]}`), + `SETCHAN A WITHIN points FENCE OBJECT {"type":"Polygon","coordinates":[[[-111.95205688476562,33.400491820565236],[-111.92630767822266,33.400491820565236],[-111.92630767822266,33.422272258866045],[-111.95205688476562,33.422272258866045],[-111.95205688476562,33.400491820565236]]]}`, + `SETCHAN B WITHIN points FENCE OBJECT {"type":"Polygon","coordinates":[[[-111.93952560424803,33.403501285221594],[-111.92630767822266,33.403501285221594],[-111.92630767822266,33.41997983836345],[-111.93952560424803,33.41997983836345],[-111.93952560424803,33.403501285221594]]]}`, + `SETCHAN C WITHIN points FENCE OBJECT {"type":"Polygon","coordinates":[[[-111.9255781173706,33.40342963251261],[-111.91201686859131,33.40342963251261],[-111.91201686859131,33.41994401881284],[-111.9255781173706,33.41994401881284],[-111.9255781173706,33.40342963251261]]]}`, + `SETCHAN D WITHIN points FENCE OBJECT {"type":"Polygon","coordinates":[[[-111.92562103271484,33.40063513076968],[-111.90021514892578,33.40063513076968],[-111.90021514892578,33.42212898435788],[-111.92562103271484,33.42212898435788],[-111.92562103271484,33.40063513076968]]]}`, "SET points point POINT 33.412529053733444 -111.91909790039062", } { if _, err := do(bc, cmd); err != nil { diff --git a/tests/follower_test.go b/tests/follower_test.go new file mode 100644 index 00000000..97d44add --- /dev/null +++ b/tests/follower_test.go @@ -0,0 +1,68 @@ +package tests + +import "time" + +func subTestFollower(g *testGroup) { + g.regSubTest("follow", follower_follow_test) +} + +func follower_follow_test(mc *mockServer) error { + mc2, err := mockOpenServer(MockServerOptions{ + Silent: true, Metrics: false, + }) + if err != nil { + return err + } + defer mc2.Close() + err = mc.DoBatch( + Do("SET", "mykey", "truck1", "POINT", 10, 10).OK(), + Do("SET", "mykey", "truck2", "POINT", 10, 10).OK(), + Do("SET", "mykey", "truck3", "POINT", 10, 10).OK(), + Do("SET", "mykey", "truck4", "POINT", 10, 10).OK(), + Do("SET", "mykey", "truck5", "POINT", 10, 10).OK(), + Do("SET", "mykey", "truck6", "POINT", 10, 10).OK(), + Do("CONFIG", "SET", "requirepass", "1234").OK(), + Do("AUTH", "1234").OK(), + ) + if err != nil { + return err + } + err = mc2.DoBatch( + Do("SET", "mykey2", "truck1", "POINT", 10, 10).OK(), + Do("SET", "mykey2", "truck2", "POINT", 10, 10).OK(), + Do("GET", "mykey2", "truck1").Str(`{"type":"Point","coordinates":[10,10]}`), + Do("GET", "mykey2", "truck2").Str(`{"type":"Point","coordinates":[10,10]}`), + + Do("CONFIG", "SET", "leaderauth", "1234").OK(), + Do("FOLLOW", "localhost", mc.port).OK(), + Do("GET", "mykey", "truck1").Err("catching up to leader"), + Sleep(time.Second/2), + + Do("GET", "mykey", "truck1").Err(`{"type":"Point","coordinates":[10,10]}`), + Do("GET", "mykey", "truck2").Err(`{"type":"Point","coordinates":[10,10]}`), + ) + if err != nil { + return err + } + + err = mc.DoBatch( + Do("SET", "mykey", "truck7", "POINT", 10, 10).OK(), + Do("SET", "mykey", "truck8", "POINT", 10, 10).OK(), + Do("SET", "mykey", "truck9", "POINT", 10, 10).OK(), + ) + if err != nil { + return err + } + + err = mc2.DoBatch( + Sleep(time.Second/2), + Do("GET", "mykey", "truck7").Str(`{"type":"Point","coordinates":[10,10]}`), + Do("GET", "mykey", "truck8").Str(`{"type":"Point","coordinates":[10,10]}`), + Do("GET", "mykey", "truck9").Str(`{"type":"Point","coordinates":[10,10]}`), + ) + if err != nil { + return err + } + + return nil +} diff --git a/tests/json_test.go b/tests/json_test.go index 863ac55d..3dda9667 100644 --- a/tests/json_test.go +++ b/tests/json_test.go @@ -1,11 +1,9 @@ package tests -import "testing" - -func subTestJSON(t *testing.T, mc *mockServer) { - runStep(t, mc, "basic", json_JSET_basic_test) - runStep(t, mc, "geojson", json_JSET_geojson_test) - runStep(t, mc, "number", json_JSET_number_test) +func subTestJSON(g *testGroup) { + g.regSubTest("basic", json_JSET_basic_test) + g.regSubTest("geojson", json_JSET_geojson_test) + g.regSubTest("number", json_JSET_number_test) } func json_JSET_basic_test(mc *mockServer) error { diff --git a/tests/keys_search_test.go b/tests/keys_search_test.go index e68b9236..606bb0e9 100644 --- a/tests/keys_search_test.go +++ b/tests/keys_search_test.go @@ -13,26 +13,26 @@ import ( "github.com/tidwall/gjson" ) -func subTestSearch(t *testing.T, mc *mockServer) { - runStep(t, mc, "KNN_BASIC", keys_KNN_basic_test) - runStep(t, mc, "KNN_RANDOM", keys_KNN_random_test) - runStep(t, mc, "KNN_CURSOR", keys_KNN_cursor_test) - runStep(t, mc, "NEARBY_SPARSE", keys_NEARBY_SPARSE_test) - runStep(t, mc, "WITHIN_CIRCLE", keys_WITHIN_CIRCLE_test) - runStep(t, mc, "WITHIN_SECTOR", keys_WITHIN_SECTOR_test) - runStep(t, mc, "INTERSECTS_CIRCLE", keys_INTERSECTS_CIRCLE_test) - runStep(t, mc, "INTERSECTS_SECTOR", keys_INTERSECTS_SECTOR_test) - runStep(t, mc, "WITHIN", keys_WITHIN_test) - runStep(t, mc, "WITHIN_CURSOR", keys_WITHIN_CURSOR_test) - runStep(t, mc, "WITHIN_CLIPBY", keys_WITHIN_CLIPBY_test) - runStep(t, mc, "INTERSECTS", keys_INTERSECTS_test) - runStep(t, mc, "INTERSECTS_CURSOR", keys_INTERSECTS_CURSOR_test) - runStep(t, mc, "INTERSECTS_CLIPBY", keys_INTERSECTS_CLIPBY_test) - runStep(t, mc, "SCAN_CURSOR", keys_SCAN_CURSOR_test) - runStep(t, mc, "SEARCH_CURSOR", keys_SEARCH_CURSOR_test) - runStep(t, mc, "MATCH", keys_MATCH_test) - runStep(t, mc, "FIELDS", keys_FIELDS_search_test) - runStep(t, mc, "BUFFER", keys_BUFFER_search_test) +func subTestSearch(g *testGroup) { + g.regSubTest("KNN_BASIC", keys_KNN_basic_test) + g.regSubTest("KNN_RANDOM", keys_KNN_random_test) + g.regSubTest("KNN_CURSOR", keys_KNN_cursor_test) + g.regSubTest("NEARBY_SPARSE", keys_NEARBY_SPARSE_test) + g.regSubTest("WITHIN_CIRCLE", keys_WITHIN_CIRCLE_test) + g.regSubTest("WITHIN_SECTOR", keys_WITHIN_SECTOR_test) + g.regSubTest("INTERSECTS_CIRCLE", keys_INTERSECTS_CIRCLE_test) + g.regSubTest("INTERSECTS_SECTOR", keys_INTERSECTS_SECTOR_test) + g.regSubTest("WITHIN", keys_WITHIN_test) + g.regSubTest("WITHIN_CURSOR", keys_WITHIN_CURSOR_test) + g.regSubTest("WITHIN_CLIPBY", keys_WITHIN_CLIPBY_test) + g.regSubTest("INTERSECTS", keys_INTERSECTS_test) + g.regSubTest("INTERSECTS_CURSOR", keys_INTERSECTS_CURSOR_test) + g.regSubTest("INTERSECTS_CLIPBY", keys_INTERSECTS_CLIPBY_test) + g.regSubTest("SCAN_CURSOR", keys_SCAN_CURSOR_test) + g.regSubTest("SEARCH_CURSOR", keys_SEARCH_CURSOR_test) + g.regSubTest("MATCH", keys_MATCH_test) + g.regSubTest("FIELDS", keys_FIELDS_search_test) + g.regSubTest("BUFFER", keys_BUFFER_search_test) } func keys_KNN_basic_test(mc *mockServer) error { @@ -122,9 +122,7 @@ func keys_KNN_random_test(mc *mockServer) error { mc.Do("OUTPUT", "json") defer mc.Do("OUTPUT", "resp") - start := time.Now() res, err := redis.String(mc.Do("NEARBY", "points", "LIMIT", N, "POINT", target[1], target[0])) - println(time.Since(start).String()) if err != nil { return err } diff --git a/tests/keys_test.go b/tests/keys_test.go index aef23ded..d172f501 100644 --- a/tests/keys_test.go +++ b/tests/keys_test.go @@ -4,314 +4,403 @@ import ( "errors" "fmt" "math/rand" - "os/exec" - "strconv" "strings" - "testing" "time" "github.com/gomodule/redigo/redis" "github.com/tidwall/gjson" ) -func subTestKeys(t *testing.T, mc *mockServer) { - runStep(t, mc, "BOUNDS", keys_BOUNDS_test) - runStep(t, mc, "DEL", keys_DEL_test) - runStep(t, mc, "DROP", keys_DROP_test) - runStep(t, mc, "RENAME", keys_RENAME_test) - runStep(t, mc, "RENAMENX", keys_RENAMENX_test) - runStep(t, mc, "EXPIRE", keys_EXPIRE_test) - runStep(t, mc, "FSET", keys_FSET_test) - runStep(t, mc, "GET", keys_GET_test) - runStep(t, mc, "KEYS", keys_KEYS_test) - runStep(t, mc, "PERSIST", keys_PERSIST_test) - runStep(t, mc, "SET", keys_SET_test) - runStep(t, mc, "STATS", keys_STATS_test) - runStep(t, mc, "TTL", keys_TTL_test) - runStep(t, mc, "SET EX", keys_SET_EX_test) - runStep(t, mc, "PDEL", keys_PDEL_test) - runStep(t, mc, "FIELDS", keys_FIELDS_test) - runStep(t, mc, "WHEREIN", keys_WHEREIN_test) - runStep(t, mc, "WHEREEVAL", keys_WHEREEVAL_test) +func subTestKeys(g *testGroup) { + g.regSubTest("BOUNDS", keys_BOUNDS_test) + g.regSubTest("DEL", keys_DEL_test) + g.regSubTest("DROP", keys_DROP_test) + g.regSubTest("RENAME", keys_RENAME_test) + g.regSubTest("RENAMENX", keys_RENAMENX_test) + g.regSubTest("EXPIRE", keys_EXPIRE_test) + g.regSubTest("FSET", keys_FSET_test) + g.regSubTest("GET", keys_GET_test) + g.regSubTest("KEYS", keys_KEYS_test) + g.regSubTest("PERSIST", keys_PERSIST_test) + g.regSubTest("SET", keys_SET_test) + g.regSubTest("STATS", keys_STATS_test) + g.regSubTest("TTL", keys_TTL_test) + g.regSubTest("SET EX", keys_SET_EX_test) + g.regSubTest("PDEL", keys_PDEL_test) + g.regSubTest("FIELDS", keys_FIELDS_test) + g.regSubTest("WHEREIN", keys_WHEREIN_test) + g.regSubTest("WHEREEVAL", keys_WHEREEVAL_test) + g.regSubTest("TYPE", keys_TYPE_test) + g.regSubTest("FLUSHDB", keys_FLUSHDB_test) + g.regSubTest("HEALTHZ", keys_HEALTHZ_test) + g.regSubTest("SERVER", keys_SERVER_test) + g.regSubTest("INFO", keys_INFO_test) } func keys_BOUNDS_test(mc *mockServer) error { - return mc.DoBatch([][]interface{}{ - {"SET", "mykey", "myid1", "POINT", 33, -115}, {"OK"}, - {"BOUNDS", "mykey"}, {"[[-115 33] [-115 33]]"}, - {"SET", "mykey", "myid2", "POINT", 34, -112}, {"OK"}, - {"BOUNDS", "mykey"}, {"[[-115 33] [-112 34]]"}, - {"DEL", "mykey", "myid2"}, {1}, - {"BOUNDS", "mykey"}, {"[[-115 33] [-115 33]]"}, - {"SET", "mykey", "myid3", "OBJECT", `{"type":"Point","coordinates":[-130,38,10]}`}, {"OK"}, - {"SET", "mykey", "myid4", "OBJECT", `{"type":"Point","coordinates":[-110,25,-8]}`}, {"OK"}, - {"BOUNDS", "mykey"}, {"[[-130 25] [-110 38]]"}, - }) + return mc.DoBatch( + Do("BOUNDS", "mykey").Str(""), + Do("BOUNDS", "mykey").JSON().Err("key not found"), + Do("SET", "mykey", "myid1", "POINT", 33, -115).OK(), + Do("BOUNDS", "mykey").Str("[[-115 33] [-115 33]]"), + Do("BOUNDS", "mykey").JSON().Str(`{"ok":true,"bounds":{"type":"Point","coordinates":[-115,33]}}`), + Do("SET", "mykey", "myid2", "POINT", 34, -112).OK(), + Do("BOUNDS", "mykey").Str("[[-115 33] [-112 34]]"), + Do("DEL", "mykey", "myid2").Str("1"), + Do("BOUNDS", "mykey").Str("[[-115 33] [-115 33]]"), + Do("SET", "mykey", "myid3", "OBJECT", `{"type":"Point","coordinates":[-130,38,10]}`).OK(), + Do("SET", "mykey", "myid4", "OBJECT", `{"type":"Point","coordinates":[-110,25,-8]}`).OK(), + Do("BOUNDS", "mykey").Str("[[-130 25] [-110 38]]"), + Do("BOUNDS", "mykey", "hello").Err("wrong number of arguments for 'bounds' command"), + Do("BOUNDS", "nada").Str(""), + Do("BOUNDS", "nada").JSON().Err("key not found"), + Do("BOUNDS", "").Str(""), + Do("BOUNDS", "mykey").JSON().Str(`{"ok":true,"bounds":{"type":"Polygon","coordinates":[[[-130,25],[-110,25],[-110,38],[-130,38],[-130,25]]]}}`), + ) } + func keys_DEL_test(mc *mockServer) error { - return mc.DoBatch([][]interface{}{ - {"SET", "mykey", "myid", "POINT", 33, -115}, {"OK"}, - {"GET", "mykey", "myid", "POINT"}, {"[33 -115]"}, - {"DEL", "mykey", "myid"}, {"1"}, - {"GET", "mykey", "myid"}, {nil}, - }) + return mc.DoBatch( + Do("SET", "mykey", "myid", "POINT", 33, -115).OK(), + Do("GET", "mykey", "myid", "POINT").Str("[33 -115]"), + Do("DEL", "mykey", "myid2", "ERRON404").Err("id not found"), + Do("DEL", "mykey", "myid").Str("1"), + Do("DEL", "mykey", "myid").Str("0"), + Do("DEL", "mykey").Err("wrong number of arguments for 'del' command"), + Do("GET", "mykey", "myid").Str(""), + Do("DEL", "mykey", "myid", "ERRON404").Err("key not found"), + Do("DEL", "mykey", "myid", "invalid-arg").Err("invalid argument 'invalid-arg'"), + Do("SET", "mykey", "myid", "POINT", 33, -115).OK(), + Do("DEL", "mykey", "myid2", "ERRON404").JSON().Err("id not found"), + Do("DEL", "mykey", "myid").JSON().OK(), + Do("DEL", "mykey", "myid").JSON().OK(), + Do("DEL", "mykey", "myid", "ERRON404").JSON().Err("key not found"), + ) } + func keys_DROP_test(mc *mockServer) error { - return mc.DoBatch([][]interface{}{ - {"SET", "mykey", "myid1", "HASH", "9my5xp7"}, {"OK"}, - {"SET", "mykey", "myid2", "HASH", "9my5xp8"}, {"OK"}, - {"SCAN", "mykey", "COUNT"}, {2}, - {"DROP", "mykey"}, {1}, - {"SCAN", "mykey", "COUNT"}, {0}, - {"DROP", "mykey"}, {0}, - {"SCAN", "mykey", "COUNT"}, {0}, - }) + return mc.DoBatch( + Do("SET", "mykey", "myid1", "HASH", "9my5xp7").OK(), + Do("SET", "mykey", "myid2", "HASH", "9my5xp8").OK(), + Do("SCAN", "mykey", "COUNT").Str("2"), + Do("DROP").Err("wrong number of arguments for 'drop' command"), + Do("DROP", "mykey", "arg3").Err("wrong number of arguments for 'drop' command"), + Do("DROP", "mykey").Str("1"), + Do("SCAN", "mykey", "COUNT").Str("0"), + Do("DROP", "mykey").Str("0"), + Do("SCAN", "mykey", "COUNT").Str("0"), + Do("SET", "mykey", "myid1", "HASH", "9my5xp7").OK(), + Do("DROP", "mykey").JSON().OK(), + Do("DROP", "mykey").JSON().OK(), + ) } func keys_RENAME_test(mc *mockServer) error { - return mc.DoBatch([][]interface{}{ - {"SET", "mykey", "myid1", "HASH", "9my5xp7"}, {"OK"}, - {"SET", "mykey", "myid2", "HASH", "9my5xp8"}, {"OK"}, - {"SCAN", "mykey", "COUNT"}, {2}, - {"RENAME", "mykey", "mynewkey"}, {"OK"}, - {"SCAN", "mykey", "COUNT"}, {0}, - {"SCAN", "mynewkey", "COUNT"}, {2}, - {"SET", "mykey", "myid3", "HASH", "9my5xp7"}, {"OK"}, - {"RENAME", "mykey", "mynewkey"}, {"OK"}, - {"SCAN", "mykey", "COUNT"}, {0}, - {"SCAN", "mynewkey", "COUNT"}, {1}, - {"RENAME", "foo", "mynewkey"}, {"ERR key not found"}, - {"SCAN", "mynewkey", "COUNT"}, {1}, - }) + return mc.DoBatch( + Do("SET", "mykey", "myid1", "HASH", "9my5xp7").OK(), + Do("SET", "mykey", "myid2", "HASH", "9my5xp8").OK(), + Do("SCAN", "mykey", "COUNT").Str("2"), + Do("RENAME", "foo", "mynewkey", "arg3").Err("wrong number of arguments for 'rename' command"), + Do("RENAME", "mykey", "mynewkey").OK(), + Do("SCAN", "mykey", "COUNT").Str("0"), + Do("SCAN", "mynewkey", "COUNT").Str("2"), + Do("SET", "mykey", "myid3", "HASH", "9my5xp7").OK(), + Do("RENAME", "mykey", "mynewkey").OK(), + Do("SCAN", "mykey", "COUNT").Str("0"), + Do("SCAN", "mynewkey", "COUNT").Str("1"), + Do("RENAME", "foo", "mynewkey").Err("key not found"), + Do("SCAN", "mynewkey", "COUNT").Str("1"), + Do("SETCHAN", "mychan", "INTERSECTS", "mynewkey", "BOUNDS", 10, 10, 20, 20).Str("1"), + Do("RENAME", "mynewkey", "foo2").Err("key has hooks set"), + Do("RENAMENX", "mynewkey", "foo2").Err("key has hooks set"), + Do("SET", "mykey", "myid1", "HASH", "9my5xp7").OK(), + Do("RENAME", "mykey", "foo2").OK(), + Do("RENAMENX", "foo2", "foo3").Str("1"), + Do("RENAMENX", "foo2", "foo3").Err("key not found"), + Do("RENAME", "foo2", "foo3").JSON().Err("key not found"), + Do("SET", "mykey", "myid1", "HASH", "9my5xp7").OK(), + Do("RENAMENX", "mykey", "foo3").Str("0"), + Do("RENAME", "foo3", "foo4").JSON().OK(), + ) } func keys_RENAMENX_test(mc *mockServer) error { - return mc.DoBatch([][]interface{}{ - {"SET", "mykey", "myid1", "HASH", "9my5xp7"}, {"OK"}, - {"SET", "mykey", "myid2", "HASH", "9my5xp8"}, {"OK"}, - {"SCAN", "mykey", "COUNT"}, {2}, - {"RENAMENX", "mykey", "mynewkey"}, {1}, - {"SCAN", "mykey", "COUNT"}, {0}, - {"DROP", "mykey"}, {0}, - {"SCAN", "mykey", "COUNT"}, {0}, - {"SCAN", "mynewkey", "COUNT"}, {2}, - {"SET", "mykey", "myid3", "HASH", "9my5xp7"}, {"OK"}, - {"RENAMENX", "mykey", "mynewkey"}, {0}, - {"SCAN", "mykey", "COUNT"}, {1}, - {"SCAN", "mynewkey", "COUNT"}, {2}, - {"RENAMENX", "foo", "mynewkey"}, {"ERR key not found"}, - {"SCAN", "mynewkey", "COUNT"}, {2}, - }) + return mc.DoBatch( + Do("SET", "mykey", "myid1", "HASH", "9my5xp7").OK(), + Do("SET", "mykey", "myid2", "HASH", "9my5xp8").OK(), + Do("SCAN", "mykey", "COUNT").Str("2"), + Do("RENAMENX", "mykey", "mynewkey").Str("1"), + Do("SCAN", "mykey", "COUNT").Str("0"), + Do("DROP", "mykey").Str("0"), + Do("SCAN", "mykey", "COUNT").Str("0"), + Do("SCAN", "mynewkey", "COUNT").Str("2"), + Do("SET", "mykey", "myid3", "HASH", "9my5xp7").OK(), + Do("RENAMENX", "mykey", "mynewkey").Str("0"), + Do("SCAN", "mykey", "COUNT").Str("1"), + Do("SCAN", "mynewkey", "COUNT").Str("2"), + Do("RENAMENX", "foo", "mynewkey").Str("ERR key not found"), + Do("SCAN", "mynewkey", "COUNT").Str("2"), + ) } func keys_EXPIRE_test(mc *mockServer) error { - return mc.DoBatch([][]interface{}{ - {"SET", "mykey", "myid", "STRING", "value"}, {"OK"}, - {"EXPIRE", "mykey", "myid", 1}, {1}, - {time.Second / 4}, {}, // sleep - {"GET", "mykey", "myid"}, {"value"}, - {time.Second}, {}, // sleep - {"GET", "mykey", "myid"}, {nil}, - }) + return mc.DoBatch( + Do("SET", "mykey", "myid", "STRING", "value").OK(), + Do("EXPIRE", "mykey", "myid").Err("wrong number of arguments for 'expire' command"), + Do("EXPIRE", "mykey", "myid", "y").Err("invalid argument 'y'"), + Do("EXPIRE", "mykey", "myid", 1).Str("1"), + Do("EXPIRE", "mykey", "myid", 1).JSON().OK(), + Sleep(time.Second/4), + Do("GET", "mykey", "myid").Str("value"), + Sleep(time.Second), + Do("GET", "mykey", "myid").Str(""), + Do("EXPIRE", "mykey", "myid", 1).JSON().Err("key not found"), + Do("SET", "mykey", "myid1", "STRING", "value1").OK(), + Do("SET", "mykey", "myid2", "STRING", "value2").OK(), + Do("EXPIRE", "mykey", "myid1", 1).Str("1"), + Sleep(time.Second/4), + Do("GET", "mykey", "myid1").Str("value1"), + Sleep(time.Second), + Do("EXPIRE", "mykey", "myid1", 1).Str("0"), + Do("EXPIRE", "mykey", "myid1", 1).JSON().Err("id not found"), + ) } func keys_FSET_test(mc *mockServer) error { - return mc.DoBatch([][]interface{}{ - {"SET", "mykey", "myid", "HASH", "9my5xp7"}, {"OK"}, - {"GET", "mykey", "myid", "WITHFIELDS", "HASH", 7}, {"[9my5xp7]"}, - {"FSET", "mykey", "myid", "f1", 105.6}, {1}, - {"GET", "mykey", "myid", "WITHFIELDS", "HASH", 7}, {"[9my5xp7 [f1 105.6]]"}, - {"FSET", "mykey", "myid", "f1", 1.1, "f2", 2.2}, {2}, - {"GET", "mykey", "myid", "WITHFIELDS", "HASH", 7}, {"[9my5xp7 [f1 1.1 f2 2.2]]"}, - {"FSET", "mykey", "myid", "f1", 1.1, "f2", 22.22}, {1}, - {"GET", "mykey", "myid", "WITHFIELDS", "HASH", 7}, {"[9my5xp7 [f1 1.1 f2 22.22]]"}, - {"FSET", "mykey", "myid", "f1", 0}, {1}, - {"GET", "mykey", "myid", "WITHFIELDS", "HASH", 7}, {"[9my5xp7 [f2 22.22]]"}, - {"FSET", "mykey", "myid", "f2", 0}, {1}, - {"GET", "mykey", "myid", "WITHFIELDS", "HASH", 7}, {"[9my5xp7]"}, - {"FSET", "mykey", "myid2", "xx", "f1", 1.1, "f2", 2.2}, {0}, - {"GET", "mykey", "myid2"}, {nil}, - {"DEL", "mykey", "myid"}, {"1"}, - {"GET", "mykey", "myid"}, {nil}, - }) + return mc.DoBatch( + Do("SET", "mykey", "myid", "HASH", "9my5xp7").OK(), + Do("GET", "mykey", "myid", "WITHFIELDS", "HASH", 7).Str("[9my5xp7]"), + Do("FSET", "mykey", "myid", "f1", 105.6).Str("1"), + Do("GET", "mykey", "myid", "WITHFIELDS", "HASH", 7).Str("[9my5xp7 [f1 105.6]]"), + Do("FSET", "mykey", "myid", "f1", 1.1, "f2", 2.2).Str("2"), + Do("GET", "mykey", "myid", "WITHFIELDS", "HASH", 7).Str("[9my5xp7 [f1 1.1 f2 2.2]]"), + Do("FSET", "mykey", "myid", "f1", 1.1, "f2", 22.22).Str("1"), + Do("GET", "mykey", "myid", "WITHFIELDS", "HASH", 7).Str("[9my5xp7 [f1 1.1 f2 22.22]]"), + Do("FSET", "mykey", "myid", "f1", 0).Str("1"), + Do("GET", "mykey", "myid", "WITHFIELDS", "HASH", 7).Str("[9my5xp7 [f2 22.22]]"), + Do("FSET", "mykey", "myid", "f2", 0).Str("1"), + Do("GET", "mykey", "myid", "WITHFIELDS", "HASH", 7).Str("[9my5xp7]"), + Do("FSET", "mykey", "myid2", "xx", "f1", 1.1, "f2", 2.2).Str("0"), + Do("GET", "mykey", "myid2").Str(""), + Do("DEL", "mykey", "myid").Str("1"), + Do("GET", "mykey", "myid").Str(""), + Do("SET", "mykey", "myid", "HASH", "9my5xp7").OK(), + Do("CONFIG", "SET", "maxmemory", "1").OK(), + Do("FSET", "mykey", "myid", "xx", "f1", 1.1, "f2", 2.2).Err(`OOM command not allowed when used memory > 'maxmemory'`), + Do("CONFIG", "SET", "maxmemory", "0").OK(), + Do("FSET", "mykey", "myid", "xx").Err("wrong number of arguments for 'fset' command"), + Do("FSET", "mykey", "myid", "f1", "a", "f2").Err("wrong number of arguments for 'fset' command"), + Do("FSET", "mykey", "myid", "z", "a").Err("invalid argument 'z'"), + Do("FSET", "mykey2", "myid", "a", "b").Err("key not found"), + Do("FSET", "mykey", "myid2", "a", "b").Err("id not found"), + Do("FSET", "mykey", "myid", "f2", 0).JSON().OK(), + ) } func keys_GET_test(mc *mockServer) error { - return mc.DoBatch([][]interface{}{ - {"SET", "mykey", "myid", "STRING", "value"}, {"OK"}, - {"GET", "mykey", "myid"}, {"value"}, - {"SET", "mykey", "myid", "STRING", "value2"}, {"OK"}, - {"GET", "mykey", "myid"}, {"value2"}, - {"DEL", "mykey", "myid"}, {"1"}, - {"GET", "mykey", "myid"}, {nil}, - }) + return mc.DoBatch( + Do("SET", "mykey", "myid", "STRING", "value").OK(), + Do("GET", "mykey", "myid").Str("value"), + Do("SET", "mykey", "myid", "STRING", "value2").OK(), + Do("GET", "mykey", "myid").Str("value2"), + Do("DEL", "mykey", "myid").Str("1"), + Do("GET", "mykey", "myid").Str(""), + Do("GET", "mykey").Err("wrong number of arguments for 'get' command"), + Do("GET", "mykey", "myid", "hash").Err("wrong number of arguments for 'get' command"), + Do("GET", "mykey", "myid", "hash", "0").Err("invalid argument '0'"), + Do("GET", "mykey", "myid", "hash", "-1").Err("invalid argument '-1'"), + Do("GET", "mykey", "myid", "hash", "13").Err("invalid argument '13'"), + Do("SET", "mykey", "myid", "field", "hello", "world", "field", "hiya", 55, "point", 33, -112).OK(), + Do("GET", "mykey", "myid", "hash", "1").Str("9"), + Do("GET", "mykey", "myid", "point").Str("[33 -112]"), + Do("GET", "mykey", "myid", "bounds").Str("[[33 -112] [33 -112]]"), + Do("GET", "mykey", "myid", "object").Str(`{"type":"Point","coordinates":[-112,33]}`), + Do("GET", "mykey", "myid", "object").Str(`{"type":"Point","coordinates":[-112,33]}`), + Do("GET", "mykey", "myid", "withfields", "point").Str(`[[33 -112] [hello world hiya 55]]`), + Do("GET", "mykey", "myid", "joint").Err("wrong number of arguments for 'get' command"), + Do("GET", "mykey2", "myid").Str(""), + Do("GET", "mykey2", "myid").JSON().Err("key not found"), + Do("GET", "mykey", "myid2").Str(""), + Do("GET", "mykey", "myid2").JSON().Err("id not found"), + Do("GET", "mykey", "myid", "point").JSON().Str(`{"ok":true,"point":{"lat":33,"lon":-112}}`), + Do("GET", "mykey", "myid", "object").JSON().Str(`{"ok":true,"object":{"type":"Point","coordinates":[-112,33]}}`), + Do("GET", "mykey", "myid", "hash", "1").JSON().Str(`{"ok":true,"hash":"9"}`), + Do("GET", "mykey", "myid", "bounds").JSON().Str(`{"ok":true,"bounds":{"sw":{"lat":33,"lon":-112},"ne":{"lat":33,"lon":-112}}}`), + Do("SET", "mykey", "myid2", "point", 33, -112, 55).OK(), + Do("GET", "mykey", "myid2", "point").Str("[33 -112 55]"), + Do("GET", "mykey", "myid2", "point").JSON().Str(`{"ok":true,"point":{"lat":33,"lon":-112,"z":55}}`), + Do("GET", "mykey", "myid", "withfields").JSON().Str(`{"ok":true,"object":{"type":"Point","coordinates":[-112,33]},"fields":{"hello":"world","hiya":55}}`), + ) } func keys_KEYS_test(mc *mockServer) error { - return mc.DoBatch([][]interface{}{ - {"SET", "mykey11", "myid4", "STRING", "value"}, {"OK"}, - {"SET", "mykey22", "myid2", "HASH", "9my5xp7"}, {"OK"}, - {"SET", "mykey22", "myid1", "OBJECT", `{"type":"Point","coordinates":[-130,38,10]}`}, {"OK"}, - {"SET", "mykey11", "myid3", "OBJECT", `{"type":"Point","coordinates":[-110,25,-8]}`}, {"OK"}, - {"SET", "mykey42", "myid2", "HASH", "9my5xp7"}, {"OK"}, - {"SET", "mykey31", "myid4", "STRING", "value"}, {"OK"}, - {"SET", "mykey310", "myid5", "STRING", "value"}, {"OK"}, - {"KEYS", "*"}, {"[mykey11 mykey22 mykey31 mykey310 mykey42]"}, - {"KEYS", "*key*"}, {"[mykey11 mykey22 mykey31 mykey310 mykey42]"}, - {"KEYS", "mykey*"}, {"[mykey11 mykey22 mykey31 mykey310 mykey42]"}, - {"KEYS", "mykey4*"}, {"[mykey42]"}, - {"KEYS", "mykey*1"}, {"[mykey11 mykey31]"}, - {"KEYS", "mykey*1*"}, {"[mykey11 mykey31 mykey310]"}, - {"KEYS", "mykey*10"}, {"[mykey310]"}, - {"KEYS", "mykey*2"}, {"[mykey22 mykey42]"}, - {"KEYS", "*2"}, {"[mykey22 mykey42]"}, - {"KEYS", "*1*"}, {"[mykey11 mykey31 mykey310]"}, - {"KEYS", "mykey"}, {"[]"}, - {"KEYS", "mykey31"}, {"[mykey31]"}, - {"KEYS", "mykey[^3]*"}, {"[mykey11 mykey22 mykey42]"}, - }) + return mc.DoBatch( + Do("SET", "mykey11", "myid4", "STRING", "value").OK(), + Do("SET", "mykey22", "myid2", "HASH", "9my5xp7").OK(), + Do("SET", "mykey22", "myid1", "OBJECT", `{"type":"Point","coordinates":[-130,38,10]}`).OK(), + Do("SET", "mykey11", "myid3", "OBJECT", `{"type":"Point","coordinates":[-110,25,-8]}`).OK(), + Do("SET", "mykey42", "myid2", "HASH", "9my5xp7").OK(), + Do("SET", "mykey31", "myid4", "STRING", "value").OK(), + Do("SET", "mykey310", "myid5", "STRING", "value").OK(), + Do("KEYS", "*").Str("[mykey11 mykey22 mykey31 mykey310 mykey42]"), + Do("KEYS", "*key*").Str("[mykey11 mykey22 mykey31 mykey310 mykey42]"), + Do("KEYS", "mykey*").Str("[mykey11 mykey22 mykey31 mykey310 mykey42]"), + Do("KEYS", "mykey4*").Str("[mykey42]"), + Do("KEYS", "mykey*1").Str("[mykey11 mykey31]"), + Do("KEYS", "mykey*1*").Str("[mykey11 mykey31 mykey310]"), + Do("KEYS", "mykey*10").Str("[mykey310]"), + Do("KEYS", "mykey*2").Str("[mykey22 mykey42]"), + Do("KEYS", "*2").Str("[mykey22 mykey42]"), + Do("KEYS", "*1*").Str("[mykey11 mykey31 mykey310]"), + Do("KEYS", "mykey").Str("[]"), + Do("KEYS", "mykey31").Str("[mykey31]"), + Do("KEYS", "mykey[^3]*").Str("[mykey11 mykey22 mykey42]"), + Do("KEYS").Err("wrong number of arguments for 'keys' command"), + Do("KEYS", "*").JSON().Str(`{"ok":true,"keys":["mykey11","mykey22","mykey31","mykey310","mykey42"]}`), + ) } func keys_PERSIST_test(mc *mockServer) error { - return mc.DoBatch([][]interface{}{ - {"SET", "mykey", "myid", "STRING", "value"}, {"OK"}, - {"EXPIRE", "mykey", "myid", 2}, {1}, - {"PERSIST", "mykey", "myid"}, {1}, - {"PERSIST", "mykey", "myid"}, {0}, - }) + return mc.DoBatch( + Do("SET", "mykey", "myid", "STRING", "value").OK(), + Do("EXPIRE", "mykey", "myid", 2).Str("1"), + Do("PERSIST", "mykey", "myid").Str("1"), + Do("PERSIST", "mykey", "myid").Str("0"), + Do("PERSIST", "mykey").Err("wrong number of arguments for 'persist' command"), + Do("PERSIST", "mykey2", "myid").Str("0"), + Do("PERSIST", "mykey2", "myid").JSON().Err("key not found"), + Do("PERSIST", "mykey", "myid2").Str("0"), + Do("PERSIST", "mykey", "myid2").JSON().Err("id not found"), + Do("EXPIRE", "mykey", "myid", 2).Str("1"), + Do("PERSIST", "mykey", "myid").JSON().OK(), + ) } func keys_SET_test(mc *mockServer) error { return mc.DoBatch( - "point", [][]interface{}{ - {"SET", "mykey", "myid", "POINT", 33, -115}, {"OK"}, - {"GET", "mykey", "myid", "POINT"}, {"[33 -115]"}, - {"GET", "mykey", "myid", "BOUNDS"}, {"[[33 -115] [33 -115]]"}, - {"GET", "mykey", "myid", "OBJECT"}, {`{"type":"Point","coordinates":[-115,33]}`}, - {"GET", "mykey", "myid", "HASH", 7}, {"9my5xp7"}, - {"DEL", "mykey", "myid"}, {"1"}, - {"GET", "mykey", "myid"}, {nil}, - }, - "object", [][]interface{}{ - {"SET", "mykey", "myid", "OBJECT", `{"type":"Point","coordinates":[-115,33]}`}, {"OK"}, - {"GET", "mykey", "myid", "POINT"}, {"[33 -115]"}, - {"GET", "mykey", "myid", "BOUNDS"}, {"[[33 -115] [33 -115]]"}, - {"GET", "mykey", "myid", "OBJECT"}, {`{"type":"Point","coordinates":[-115,33]}`}, - {"GET", "mykey", "myid", "HASH", 7}, {"9my5xp7"}, - {"DEL", "mykey", "myid"}, {"1"}, - {"GET", "mykey", "myid"}, {nil}, - }, - "bounds", [][]interface{}{ - {"SET", "mykey", "myid", "BOUNDS", 33, -115, 33, -115}, {"OK"}, - {"GET", "mykey", "myid", "POINT"}, {"[33 -115]"}, - {"GET", "mykey", "myid", "BOUNDS"}, {"[[33 -115] [33 -115]]"}, - {"GET", "mykey", "myid", "OBJECT"}, {`{"type":"Point","coordinates":[-115,33]}`}, - {"GET", "mykey", "myid", "HASH", 7}, {"9my5xp7"}, - {"DEL", "mykey", "myid"}, {"1"}, - {"GET", "mykey", "myid"}, {nil}, - }, - "hash", [][]interface{}{ - {"SET", "mykey", "myid", "HASH", "9my5xp7"}, {"OK"}, - {"GET", "mykey", "myid", "HASH", 7}, {"9my5xp7"}, - {"DEL", "mykey", "myid"}, {"1"}, - {"GET", "mykey", "myid"}, {nil}, - }, - "field", [][]interface{}{ - {"SET", "mykey", "myid", "FIELD", "f1", 33, "FIELD", "a2", 44.5, "HASH", "9my5xp7"}, {"OK"}, - {"GET", "mykey", "myid", "WITHFIELDS", "HASH", 7}, {"[9my5xp7 [a2 44.5 f1 33]]"}, - {"FSET", "mykey", "myid", "f1", 0}, {1}, - {"FSET", "mykey", "myid", "f1", 0}, {0}, - {"GET", "mykey", "myid", "WITHFIELDS", "HASH", 7}, {"[9my5xp7 [a2 44.5]]"}, - {"DEL", "mykey", "myid"}, {"1"}, - {"GET", "mykey", "myid"}, {nil}, - }, - "string", [][]interface{}{ - {"SET", "mykey", "myid", "STRING", "value"}, {"OK"}, - {"GET", "mykey", "myid"}, {"value"}, - {"SET", "mykey", "myid", "STRING", "value2"}, {"OK"}, - {"GET", "mykey", "myid"}, {"value2"}, - {"DEL", "mykey", "myid"}, {"1"}, - {"GET", "mykey", "myid"}, {nil}, - }, + // Section: point + Do("SET", "mykey", "myid", "POINT", 33, -115).OK(), + Do("GET", "mykey", "myid", "POINT").Str("[33 -115]"), + Do("GET", "mykey", "myid", "BOUNDS").Str("[[33 -115] [33 -115]]"), + Do("GET", "mykey", "myid", "OBJECT").Str(`{"type":"Point","coordinates":[-115,33]}`), + Do("GET", "mykey", "myid", "HASH", 7).Str("9my5xp7"), + Do("DEL", "mykey", "myid").Str("1"), + Do("GET", "mykey", "myid").Str(""), + Do("SET", "mykey", "myid", "point", "33", "-112", "99").OK(), + + // Section: object + Do("SET", "mykey", "myid", "OBJECT", `{"type":"Point","coordinates":[-115,33]}`).OK(), + Do("GET", "mykey", "myid", "POINT").Str("[33 -115]"), + Do("GET", "mykey", "myid", "BOUNDS").Str("[[33 -115] [33 -115]]"), + Do("GET", "mykey", "myid", "OBJECT").Str(`{"type":"Point","coordinates":[-115,33]}`), + Do("GET", "mykey", "myid", "HASH", 7).Str("9my5xp7"), + Do("DEL", "mykey", "myid").Str("1"), + Do("GET", "mykey", "myid").Str(""), + + // Section: bounds + Do("SET", "mykey", "myid", "BOUNDS", 33, -115, 33, -115).OK(), + Do("GET", "mykey", "myid", "POINT").Str("[33 -115]"), + Do("GET", "mykey", "myid", "BOUNDS").Str("[[33 -115] [33 -115]]"), + Do("GET", "mykey", "myid", "OBJECT").Str(`{"type":"Point","coordinates":[-115,33]}`), + Do("GET", "mykey", "myid", "HASH", 7).Str("9my5xp7"), + Do("DEL", "mykey", "myid").Str("1"), + Do("GET", "mykey", "myid").Str(""), + + // Section: hash + Do("SET", "mykey", "myid", "HASH", "9my5xp7").OK(), + Do("GET", "mykey", "myid", "HASH", 7).Str("9my5xp7"), + Do("DEL", "mykey", "myid").Str("1"), + Do("GET", "mykey", "myid").Str(""), + Do("SET", "mykey", "myid", "HASH", "9my5xp7").JSON().OK(), + + // Section: field + Do("SET", "mykey", "myid", "FIELD", "f1", 33, "FIELD", "a2", 44.5, "HASH", "9my5xp7").OK(), + Do("GET", "mykey", "myid", "WITHFIELDS", "HASH", 7).Str("[9my5xp7 [a2 44.5 f1 33]]"), + Do("FSET", "mykey", "myid", "f1", 0).Str("1"), + Do("FSET", "mykey", "myid", "f1", 0).Str("0"), + Do("GET", "mykey", "myid", "WITHFIELDS", "HASH", 7).Str("[9my5xp7 [a2 44.5]]"), + Do("DEL", "mykey", "myid").Str("1"), + Do("GET", "mykey", "myid").Str(""), + + // Section: string + Do("SET", "mykey", "myid", "STRING", "value").OK(), + Do("GET", "mykey", "myid").Str("value"), + Do("SET", "mykey", "myid", "STRING", "value2").OK(), + Do("GET", "mykey", "myid").Str("value2"), + Do("DEL", "mykey", "myid").Str("1"), + Do("GET", "mykey", "myid").Str(""), + + // Test error conditions + Do("CONFIG", "SET", "maxmemory", "1").OK(), + Do("SET", "mykey", "myid", "STRING", "value2").Err("OOM command not allowed when used memory > 'maxmemory'"), + Do("CONFIG", "SET", "maxmemory", "0").OK(), + Do("SET").Err("wrong number of arguments for 'set' command"), + Do("SET", "mykey", "myid", "FIELD", "f1").Err("wrong number of arguments for 'set' command"), + Do("SET", "mykey", "myid", "FIELD", "z", "1").Err("invalid argument 'z'"), + Do("SET", "mykey", "myid", "EX").Err("wrong number of arguments for 'set' command"), + Do("SET", "mykey", "myid", "EX", "yyy").Err("invalid argument 'yyy'"), + Do("SET", "mykey", "myid", "EX", "123").Err("wrong number of arguments for 'set' command"), + Do("SET", "mykey", "myid", "nx").Err("wrong number of arguments for 'set' command"), + Do("SET", "mykey", "myid", "nx", "xx").Err("invalid argument 'xx'"), + Do("SET", "mykey", "myid", "xx", "nx").Err("invalid argument 'nx'"), + Do("SET", "mykey", "myid", "string").Err("wrong number of arguments for 'set' command"), + Do("SET", "mykey", "myid", "point").Err("wrong number of arguments for 'set' command"), + Do("SET", "mykey", "myid", "point", "33").Err("wrong number of arguments for 'set' command"), + Do("SET", "mykey", "myid", "point", "33f", "-112").Err("invalid argument '33f'"), + Do("SET", "mykey", "myid", "point", "33", "-112f").Err("invalid argument '-112f'"), + Do("SET", "mykey", "myid", "point", "33", "-112f", "99").Err("invalid argument '-112f'"), + Do("SET", "mykey", "myid", "bounds").Err("wrong number of arguments for 'set' command"), + Do("SET", "mykey", "myid", "bounds", "fff", "1", "2", "3").Err("invalid argument 'fff'"), + Do("SET", "mykey", "myid", "hash").Err("wrong number of arguments for 'set' command"), + Do("SET", "mykey", "myid", "object").Err("wrong number of arguments for 'set' command"), + Do("SET", "mykey", "myid", "object", "asd").Err("invalid data"), + Do("SET", "mykey", "myid", "joint").Err("invalid argument 'joint'"), + Do("SET", "mykey", "myid", "XX", "HASH", "9my5xp7").Err(""), + Do("SET", "mykey", "myid", "XX", "HASH", "9my5xp7").JSON().Err("id not found"), + Do("SET", "mykey", "myid1", "HASH", "9my5xp7").OK(), + Do("SET", "mykey", "myid", "XX", "HASH", "9my5xp7").Err(""), + Do("SET", "mykey", "myid", "NX", "HASH", "9my5xp7").OK(), + Do("SET", "mykey", "myid", "XX", "HASH", "9my5xp7").OK(), + Do("SET", "mykey", "myid", "NX", "HASH", "9my5xp7").Err(""), + Do("SET", "mykey", "myid", "NX", "HASH", "9my5xp7").JSON().Err("id already exists"), ) } func keys_STATS_test(mc *mockServer) error { - return mc.DoBatch([][]interface{}{ - {"STATS", "mykey"}, {"[nil]"}, - {"SET", "mykey", "myid", "STRING", "value"}, {"OK"}, - {"STATS", "mykey"}, {"[[in_memory_size 9 num_objects 1 num_points 0 num_strings 1]]"}, - {"SET", "mykey", "myid2", "STRING", "value"}, {"OK"}, - {"STATS", "mykey"}, {"[[in_memory_size 19 num_objects 2 num_points 0 num_strings 2]]"}, - {"SET", "mykey", "myid3", "OBJECT", `{"type":"Point","coordinates":[-115,33]}`}, {"OK"}, - {"STATS", "mykey"}, {"[[in_memory_size 40 num_objects 3 num_points 1 num_strings 2]]"}, - {"DEL", "mykey", "myid"}, {1}, - {"STATS", "mykey"}, {"[[in_memory_size 31 num_objects 2 num_points 1 num_strings 1]]"}, - {"DEL", "mykey", "myid3"}, {1}, - {"STATS", "mykey"}, {"[[in_memory_size 10 num_objects 1 num_points 0 num_strings 1]]"}, - {"STATS", "mykey", "mykey2"}, {"[[in_memory_size 10 num_objects 1 num_points 0 num_strings 1] nil]"}, - {"DEL", "mykey", "myid2"}, {1}, - {"STATS", "mykey"}, {"[nil]"}, - {"STATS", "mykey", "mykey2"}, {"[nil nil]"}, - }) + return mc.DoBatch( + Do("STATS", "mykey").Str("[nil]"), + Do("SET", "mykey", "myid", "STRING", "value").OK(), + Do("STATS", "mykey").Str("[[in_memory_size 9 num_objects 1 num_points 0 num_strings 1]]"), + Do("STATS", "mykey", "hello").JSON().Str(`{"ok":true,"stats":[{"in_memory_size":9,"num_objects":1,"num_points":0,"num_strings":1},null]}`), + Do("SET", "mykey", "myid2", "STRING", "value").OK(), + Do("STATS", "mykey").Str("[[in_memory_size 19 num_objects 2 num_points 0 num_strings 2]]"), + Do("SET", "mykey", "myid3", "OBJECT", `{"type":"Point","coordinates":[-115,33]}`).OK(), + Do("STATS", "mykey").Str("[[in_memory_size 40 num_objects 3 num_points 1 num_strings 2]]"), + Do("DEL", "mykey", "myid").Str("1"), + Do("STATS", "mykey").Str("[[in_memory_size 31 num_objects 2 num_points 1 num_strings 1]]"), + Do("DEL", "mykey", "myid3").Str("1"), + Do("STATS", "mykey").Str("[[in_memory_size 10 num_objects 1 num_points 0 num_strings 1]]"), + Do("STATS", "mykey", "mykey2").Str("[[in_memory_size 10 num_objects 1 num_points 0 num_strings 1] nil]"), + Do("DEL", "mykey", "myid2").Str("1"), + Do("STATS", "mykey").Str("[nil]"), + Do("STATS", "mykey", "mykey2").Str("[nil nil]"), + Do("STATS", "mykey").Str("[nil]"), + Do("STATS").Err(`wrong number of arguments for 'stats' command`), + ) } func keys_TTL_test(mc *mockServer) error { - return mc.DoBatch([][]interface{}{ - {"SET", "mykey", "myid", "STRING", "value"}, {"OK"}, - {"EXPIRE", "mykey", "myid", 2}, {1}, - {time.Second / 4}, {}, // sleep - {"TTL", "mykey", "myid"}, {1}, - }) + return mc.DoBatch( + Do("SET", "mykey", "myid", "STRING", "value").OK(), + Do("EXPIRE", "mykey", "myid", 2).Str("1"), + Do("EXPIRE", "mykey", "myid", 2).JSON().OK(), + Sleep(time.Millisecond*10), + Do("TTL", "mykey", "myid").Str("1"), + Do("EXPIRE", "mykey", "myid", 1).Str("1"), + Sleep(time.Millisecond*10), + Do("TTL", "mykey", "myid").Str("0"), + Do("TTL", "mykey", "myid").JSON().Str(`{"ok":true,"ttl":0}`), + Do("TTL", "mykey2", "myid").Str("-2"), + Do("TTL", "mykey", "myid2").Str("-2"), + Do("TTL", "mykey").Err("wrong number of arguments for 'ttl' command"), + Do("SET", "mykey", "myid", "STRING", "value").OK(), + Do("TTL", "mykey", "myid").Str("-1"), + Do("TTL", "mykey2", "myid").JSON().Err("key not found"), + Do("TTL", "mykey", "myid2").JSON().Err("id not found"), + ) } -type PSAUX struct { - User string - PID int - CPU float64 - Mem float64 - VSZ int - RSS int - TTY string - Stat string - Start string - Time string - Command string -} - -func atoi(s string) int { - n, _ := strconv.ParseInt(s, 10, 64) - return int(n) -} -func atof(s string) float64 { - n, _ := strconv.ParseFloat(s, 64) - return float64(n) -} -func psaux(pid int) PSAUX { - var res []byte - res, err := exec.Command("ps", "aux").CombinedOutput() - if err != nil { - return PSAUX{} - } - pids := strconv.FormatInt(int64(pid), 10) - for _, line := range strings.Split(string(res), "\n") { - var words []string - for _, word := range strings.Split(line, " ") { - if word != "" { - words = append(words, word) - } - if len(words) > 11 { - if words[1] == pids { - return PSAUX{ - User: words[0], - PID: atoi(words[1]), - CPU: atof(words[2]), - Mem: atof(words[3]), - VSZ: atoi(words[4]), - RSS: atoi(words[5]), - TTY: words[6], - Stat: words[7], - Start: words[8], - Time: words[9], - Command: words[10], - } - } - } - } - } - return PSAUX{} -} func keys_SET_EX_test(mc *mockServer) (err error) { rand.Seed(time.Now().UnixNano()) @@ -364,52 +453,195 @@ func keys_FIELDS_test(mc *mockServer) error { } func keys_PDEL_test(mc *mockServer) error { - return mc.DoBatch([][]interface{}{ - {"SET", "mykey", "myid1a", "POINT", 33, -115}, {"OK"}, - {"SET", "mykey", "myid1b", "POINT", 33, -115}, {"OK"}, - {"SET", "mykey", "myid2a", "POINT", 33, -115}, {"OK"}, - {"SET", "mykey", "myid2b", "POINT", 33, -115}, {"OK"}, - {"SET", "mykey", "myid3a", "POINT", 33, -115}, {"OK"}, - {"SET", "mykey", "myid3b", "POINT", 33, -115}, {"OK"}, - {"SET", "mykey", "myid4a", "POINT", 33, -115}, {"OK"}, - {"SET", "mykey", "myid4b", "POINT", 33, -115}, {"OK"}, - {"PDEL", "mykeyNA", "*"}, {0}, - {"PDEL", "mykey", "myid1a"}, {1}, - {"PDEL", "mykey", "myid1a"}, {0}, - {"PDEL", "mykey", "myid1*"}, {1}, - {"PDEL", "mykey", "myid2*"}, {2}, - {"PDEL", "mykey", "*b"}, {2}, - {"PDEL", "mykey", "*"}, {2}, - {"PDEL", "mykey", "*"}, {0}, - }) + return mc.DoBatch( + Do("SET", "mykey", "myid1a", "POINT", 33, -115).OK(), + Do("SET", "mykey", "myid1b", "POINT", 33, -115).OK(), + Do("SET", "mykey", "myid2a", "POINT", 33, -115).OK(), + Do("SET", "mykey", "myid2b", "POINT", 33, -115).OK(), + Do("SET", "mykey", "myid3a", "POINT", 33, -115).OK(), + Do("SET", "mykey", "myid3b", "POINT", 33, -115).OK(), + Do("SET", "mykey", "myid4a", "POINT", 33, -115).OK(), + Do("SET", "mykey", "myid4b", "POINT", 33, -115).OK(), + Do("PDEL", "mykey").Err("wrong number of arguments for 'pdel' command"), + Do("PDEL", "mykeyNA", "*").Str("0"), + Do("PDEL", "mykey", "myid1a").Str("1"), + Do("PDEL", "mykey", "myid1a").Str("0"), + Do("PDEL", "mykey", "myid1*").Str("1"), + Do("PDEL", "mykey", "myid2*").Str("2"), + Do("PDEL", "mykey", "*b").Str("2"), + Do("PDEL", "mykey", "*").Str("2"), + Do("PDEL", "mykey", "*").Str("0"), + Do("SET", "mykey", "myid1a", "POINT", 33, -115).OK(), + Do("SET", "mykey", "myid1b", "POINT", 33, -115).OK(), + Do("SET", "mykey", "myid2a", "POINT", 33, -115).OK(), + Do("SET", "mykey", "myid2b", "POINT", 33, -115).OK(), + Do("SET", "mykey", "myid3a", "POINT", 33, -115).OK(), + Do("PDEL", "mykey", "*").JSON().OK(), + ) } func keys_WHEREIN_test(mc *mockServer) error { - return mc.DoBatch([][]interface{}{ - {"SET", "mykey", "myid_a1", "FIELD", "a", 1, "POINT", 33, -115}, {"OK"}, - {"WITHIN", "mykey", "WHEREIN", "a", 3, 0, 1, 2, "BOUNDS", 32.8, -115.2, 33.2, -114.8}, {`[0 [[myid_a1 {"type":"Point","coordinates":[-115,33]} [a 1]]]]`}, - {"WITHIN", "mykey", "WHEREIN", "a", "a", 0, 1, 2, "BOUNDS", 32.8, -115.2, 33.2, -114.8}, {"ERR invalid argument 'a'"}, - {"WITHIN", "mykey", "WHEREIN", "a", 1, 0, 1, 2, "BOUNDS", 32.8, -115.2, 33.2, -114.8}, {"ERR invalid argument '1'"}, - {"WITHIN", "mykey", "WHEREIN", "a", 3, 0, "a", 2, "BOUNDS", 32.8, -115.2, 33.2, -114.8}, {"[0 []]"}, - {"WITHIN", "mykey", "WHEREIN", "a", 4, 0, "a", 1, 2, "BOUNDS", 32.8, -115.2, 33.2, -114.8}, {`[0 [[myid_a1 {"type":"Point","coordinates":[-115,33]} [a 1]]]]`}, - {"SET", "mykey", "myid_a2", "FIELD", "a", 2, "POINT", 32.99, -115}, {"OK"}, - {"SET", "mykey", "myid_a3", "FIELD", "a", 3, "POINT", 33, -115.02}, {"OK"}, - {"WITHIN", "mykey", "WHEREIN", "a", 3, 0, 1, 2, "BOUNDS", 32.8, -115.2, 33.2, -114.8}, { - `[0 [[myid_a2 {"type":"Point","coordinates":[-115,32.99]} [a 2]] [myid_a1 {"type":"Point","coordinates":[-115,33]} [a 1]]]]`}, + return mc.DoBatch( + Do("SET", "mykey", "myid_a1", "FIELD", "a", 1, "POINT", 33, -115).OK(), + Do("WITHIN", "mykey", "WHEREIN", "a", 3, 0, 1, 2, "BOUNDS", 32.8, -115.2, 33.2, -114.8).Str(`[0 [[myid_a1 {"type":"Point","coordinates":[-115,33]} [a 1]]]]`), + Do("WITHIN", "mykey", "WHEREIN", "a", "a", 0, 1, 2, "BOUNDS", 32.8, -115.2, 33.2, -114.8).Err("invalid argument 'a'"), + Do("WITHIN", "mykey", "WHEREIN", "a", 1, 0, 1, 2, "BOUNDS", 32.8, -115.2, 33.2, -114.8).Err("invalid argument '1'"), + Do("WITHIN", "mykey", "WHEREIN", "a", 3, 0, "a", 2, "BOUNDS", 32.8, -115.2, 33.2, -114.8).Str("[0 []]"), + Do("WITHIN", "mykey", "WHEREIN", "a", 4, 0, "a", 1, 2, "BOUNDS", 32.8, -115.2, 33.2, -114.8).Str(`[0 [[myid_a1 {"type":"Point","coordinates":[-115,33]} [a 1]]]]`), + Do("SET", "mykey", "myid_a2", "FIELD", "a", 2, "POINT", 32.99, -115).OK(), + Do("SET", "mykey", "myid_a3", "FIELD", "a", 3, "POINT", 33, -115.02).OK(), + Do("WITHIN", "mykey", "WHEREIN", "a", 3, 0, 1, 2, "BOUNDS", 32.8, -115.2, 33.2, -114.8).Str(`[0 [[myid_a2 {"type":"Point","coordinates":[-115,32.99]} [a 2]] [myid_a1 {"type":"Point","coordinates":[-115,33]} [a 1]]]]`), // zero value should not match 1 and 2 - {"SET", "mykey", "myid_a0", "FIELD", "a", 0, "POINT", 33, -115.02}, {"OK"}, - {"WITHIN", "mykey", "WHEREIN", "a", 2, 1, 2, "BOUNDS", 32.8, -115.2, 33.2, -114.8}, {`[0 [[myid_a2 {"type":"Point","coordinates":[-115,32.99]} [a 2]] [myid_a1 {"type":"Point","coordinates":[-115,33]} [a 1]]]]`}, - }) + Do("SET", "mykey", "myid_a0", "FIELD", "a", 0, "POINT", 33, -115.02).OK(), + Do("WITHIN", "mykey", "WHEREIN", "a", 2, 1, 2, "BOUNDS", 32.8, -115.2, 33.2, -114.8).Str(`[0 [[myid_a2 {"type":"Point","coordinates":[-115,32.99]} [a 2]] [myid_a1 {"type":"Point","coordinates":[-115,33]} [a 1]]]]`), + ) } func keys_WHEREEVAL_test(mc *mockServer) error { - return mc.DoBatch([][]interface{}{ - {"SET", "mykey", "myid_a1", "FIELD", "a", 1, "POINT", 33, -115}, {"OK"}, - {"WITHIN", "mykey", "WHEREEVAL", "return FIELDS.a > tonumber(ARGV[1])", 1, 0.5, "BOUNDS", 32.8, -115.2, 33.2, -114.8}, {`[0 [[myid_a1 {"type":"Point","coordinates":[-115,33]} [a 1]]]]`}, - {"WITHIN", "mykey", "WHEREEVAL", "return FIELDS.a > tonumber(ARGV[1])", "a", 0.5, "BOUNDS", 32.8, -115.2, 33.2, -114.8}, {"ERR invalid argument 'a'"}, - {"WITHIN", "mykey", "WHEREEVAL", "return FIELDS.a > tonumber(ARGV[1])", 1, 0.5, 4, "BOUNDS", 32.8, -115.2, 33.2, -114.8}, {"ERR invalid argument '4'"}, - {"SET", "mykey", "myid_a2", "FIELD", "a", 2, "POINT", 32.99, -115}, {"OK"}, - {"SET", "mykey", "myid_a3", "FIELD", "a", 3, "POINT", 33, -115.02}, {"OK"}, - {"WITHIN", "mykey", "WHEREEVAL", "return FIELDS.a > tonumber(ARGV[1]) and FIELDS.a ~= tonumber(ARGV[2])", 2, 0.5, 3, "BOUNDS", 32.8, -115.2, 33.2, -114.8}, {`[0 [[myid_a2 {"type":"Point","coordinates":[-115,32.99]} [a 2]] [myid_a1 {"type":"Point","coordinates":[-115,33]} [a 1]]]]`}, - }) + return mc.DoBatch( + Do("SET", "mykey", "myid_a1", "FIELD", "a", 1, "POINT", 33, -115).OK(), + Do("WITHIN", "mykey", "WHEREEVAL", "return FIELDS.a > tonumber(ARGV[1])", 1, 0.5, "BOUNDS", 32.8, -115.2, 33.2, -114.8).Str(`[0 [[myid_a1 {"type":"Point","coordinates":[-115,33]} [a 1]]]]`), + Do("WITHIN", "mykey", "WHEREEVAL", "return FIELDS.a > tonumber(ARGV[1])", "a", 0.5, "BOUNDS", 32.8, -115.2, 33.2, -114.8).Err("invalid argument 'a'"), + Do("WITHIN", "mykey", "WHEREEVAL", "return FIELDS.a > tonumber(ARGV[1])", 1, 0.5, 4, "BOUNDS", 32.8, -115.2, 33.2, -114.8).Err("invalid argument '4'"), + Do("SET", "mykey", "myid_a2", "FIELD", "a", 2, "POINT", 32.99, -115).OK(), + Do("SET", "mykey", "myid_a3", "FIELD", "a", 3, "POINT", 33, -115.02).OK(), + Do("WITHIN", "mykey", "WHEREEVAL", "return FIELDS.a > tonumber(ARGV[1]) and FIELDS.a ~= tonumber(ARGV[2])", 2, 0.5, 3, "BOUNDS", 32.8, -115.2, 33.2, -114.8).Str(`[0 [[myid_a2 {"type":"Point","coordinates":[-115,32.99]} [a 2]] [myid_a1 {"type":"Point","coordinates":[-115,33]} [a 1]]]]`), + ) +} + +func keys_TYPE_test(mc *mockServer) error { + return mc.DoBatch( + Do("SET", "mykey", "myid1", "POINT", 33, -115).OK(), + Do("TYPE", "mykey").Str("hash"), + Do("TYPE", "mykey", "hello").Err("wrong number of arguments for 'type' command"), + Do("TYPE", "mykey2").Str("none"), + Do("TYPE", "mykey2").JSON().Err("key not found"), + Do("TYPE", "mykey").JSON().Str(`{"ok":true,"type":"hash"}`), + ) +} + +func keys_FLUSHDB_test(mc *mockServer) error { + return mc.DoBatch( + Do("SET", "mykey1", "myid1", "POINT", 33, -115).OK(), + Do("SET", "mykey2", "myid1", "POINT", 33, -115).OK(), + Do("SETCHAN", "mychan", "INTERSECTS", "mykey1", "BOUNDS", 10, 10, 10, 10).Str("1"), + Do("KEYS", "*").Str("[mykey1 mykey2]"), + Do("CHANS", "*").JSON().Func(func(s string) error { + if gjson.Get(s, "chans.#").Int() != 1 { + return fmt.Errorf("expected '%d', got '%d'", 1, gjson.Get(s, "chans.#").Int()) + } + return nil + }), + Do("FLUSHDB", "arg2").Err("wrong number of arguments for 'flushdb' command"), + Do("FLUSHDB").OK(), + Do("KEYS", "*").Str("[]"), + Do("CHANS", "*").Str("[]"), + Do("SET", "mykey1", "myid1", "POINT", 33, -115).OK(), + Do("SET", "mykey2", "myid1", "POINT", 33, -115).OK(), + Do("SETCHAN", "mychan", "INTERSECTS", "mykey1", "BOUNDS", 10, 10, 10, 10).Str("1"), + Do("FLUSHDB").JSON().OK(), + ) +} + +func keys_HEALTHZ_test(mc *mockServer) error { + return mc.DoBatch( + Do("HEALTHZ").OK(), + Do("HEALTHZ").JSON().OK(), + Do("HEALTHZ", "arg").Err(`wrong number of arguments for 'healthz' command`), + ) +} + +func keys_SERVER_test(mc *mockServer) error { + return mc.DoBatch( + Do("SERVER").Func(func(s string) error { + valid := strings.HasPrefix(s, "[") && strings.HasSuffix(s, "]") && + strings.Contains(s, "cpus") && strings.Contains(s, "mem_alloc") + if !valid { + return errors.New("looks invalid") + } + return nil + }), + Do("SERVER").JSON().Func(func(s string) error { + if !gjson.Get(s, "ok").Bool() { + return errors.New("not ok") + } + valid := gjson.Get(s, "stats.cpus").Exists() && + gjson.Get(s, "stats.mem_alloc").Exists() + if !valid { + return errors.New("looks invalid") + } + return nil + }), + Do("SERVER", "ext").Func(func(s string) error { + valid := strings.HasPrefix(s, "[") && strings.HasSuffix(s, "]") && + strings.Contains(s, "sys_cpus") && + strings.Contains(s, "tile38_connected_clients") + + if !valid { + return errors.New("looks invalid") + } + return nil + }), + Do("SERVER", "ext").JSON().Func(func(s string) error { + if !gjson.Get(s, "ok").Bool() { + return errors.New("not ok") + } + valid := gjson.Get(s, "stats.sys_cpus").Exists() && + gjson.Get(s, "stats.tile38_connected_clients").Exists() + if !valid { + return errors.New("looks invalid") + } + return nil + }), + Do("SERVER", "ett").Err(`invalid argument 'ett'`), + Do("SERVER", "ett").JSON().Err(`invalid argument 'ett'`), + ) +} + +func keys_INFO_test(mc *mockServer) error { + return mc.DoBatch( + Do("INFO").Func(func(s string) error { + if !strings.Contains(s, "# Clients") || + !strings.Contains(s, "# Stats") { + return errors.New("looks invalid") + } + return nil + }), + Do("INFO", "all").Func(func(s string) error { + if !strings.Contains(s, "# Clients") || + !strings.Contains(s, "# Stats") { + return errors.New("looks invalid") + } + return nil + }), + Do("INFO", "default").Func(func(s string) error { + if !strings.Contains(s, "# Clients") || + !strings.Contains(s, "# Stats") { + return errors.New("looks invalid") + } + return nil + }), + Do("INFO", "cpu").Func(func(s string) error { + if !strings.Contains(s, "# CPU") || + strings.Contains(s, "# Clients") || + strings.Contains(s, "# Stats") { + return errors.New("looks invalid") + } + return nil + }), + Do("INFO", "cpu", "clients").Func(func(s string) error { + if !strings.Contains(s, "# CPU") || + !strings.Contains(s, "# Clients") || + strings.Contains(s, "# Stats") { + return errors.New("looks invalid") + } + return nil + }), + Do("INFO").JSON().Func(func(s string) error { + if gjson.Get(s, "info.tile38_version").String() == "" { + return errors.New("looks invalid") + } + return nil + }), + ) } diff --git a/tests/metrics_test.go b/tests/metrics_test.go index b7fa4055..963b83d4 100644 --- a/tests/metrics_test.go +++ b/tests/metrics_test.go @@ -1,42 +1,55 @@ package tests import ( - "io/ioutil" + "fmt" + "io" "net/http" "strings" - "testing" ) -func downloadURLWithStatusCode(t *testing.T, u string) (int, string) { - resp, err := http.Get(u) - if err != nil { - t.Fatal(err) - } - defer resp.Body.Close() - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - t.Fatal(err) - } - return resp.StatusCode, string(body) +func subTestMetrics(g *testGroup) { + g.regSubTest("basic", metrics_basic_test) } -func subTestMetrics(t *testing.T, mc *mockServer) { +func downloadURLWithStatusCode(u string) (int, string, error) { + resp, err := http.Get(u) + if err != nil { + return 0, "", err + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + return 0, "", err + } + return resp.StatusCode, string(body), nil +} + +func metrics_basic_test(mc *mockServer) error { + + maddr := fmt.Sprintf("http://127.0.0.1:%d/", mc.metricsPort()) + mc.Do("SET", "metrics_test_1", "1", "FIELD", "foo", 5.5, "POINT", 5, 5) mc.Do("SET", "metrics_test_2", "2", "FIELD", "foo", 19.19, "POINT", 19, 19) mc.Do("SET", "metrics_test_2", "3", "FIELD", "foo", 19.19, "POINT", 19, 19) mc.Do("SET", "metrics_test_2", "truck1:driver", "STRING", "John Denton") - status, index := downloadURLWithStatusCode(t, "http://127.0.0.1:4321/") + status, index, err := downloadURLWithStatusCode(maddr) + if err != nil { + return err + } if status != 200 { - t.Fatalf("Expected status code 200, got: %d", status) + return fmt.Errorf("Expected status code 200, got: %d", status) } if !strings.Contains(index, " 0 { + if opts.AOFFileName == "" { + opts.AOFFileName = "appendonly.aof" } - if err := server.Serve(opts); err != nil { - log.Fatal(err) + if err := os.MkdirAll(dir, 0777); err != nil { + return nil, err + } + err := os.WriteFile(filepath.Join(dir, opts.AOFFileName), + opts.AOFData, 0666) + if err != nil { + return nil, err + } + } + + shutdown := make(chan bool) + s := &mockServer{port: port, dir: dir, shutdown: shutdown} + if opts.Metrics { + s.mport = getNextPort() + } + var ferrt int32 // atomic flag for when ferr has been set + var ferr error // ferr for when the server fails to start + go func() { + sopts := server.Options{ + Host: "localhost", + Port: port, + Dir: dir, + UseHTTP: true, + DevMode: true, + AppendOnly: true, + Shutdown: shutdown, + ShowDebugMessages: true, + } + if opts.Metrics { + sopts.MetricsAddr = fmt.Sprintf(":%d", s.mport) + } + err := server.Serve(sopts) + if err != nil { + ferr = err + atomic.StoreInt32(&ferrt, 1) } }() - if err := s.waitForStartup(); err != nil { + if err := s.waitForStartup(&ferr, &ferrt); err != nil { s.Close() return nil, err } return s, nil } -func (s *mockServer) waitForStartup() error { +func (s *mockServer) waitForStartup(ferr *error, ferrt *int32) error { var lerr error start := time.Now() for { - if time.Now().Sub(start) > time.Second*5 { + if atomic.LoadInt32(ferrt) != 0 { + return *ferr + } + if time.Since(start) > time.Second*5 { if lerr != nil { return lerr } @@ -106,9 +170,17 @@ func (s *mockServer) waitForStartup() error { } func (mc *mockServer) Close() { + if mc == nil || mc.closed { + return + } + mc.closed = true + mc.shutdown <- true if mc.conn != nil { mc.conn.Close() } + if mc.dir != "" { + os.RemoveAll(mc.dir) + } } func (mc *mockServer) ResetConn() { @@ -159,7 +231,28 @@ func (s *mockServer) Do(commandName string, args ...interface{}) (interface{}, e return resps[0], nil } -func (mc *mockServer) DoBatch(commands ...interface{}) error { //[][]interface{}) error { +func (mc *mockServer) DoBatch(commands ...interface{}) error { + // Probe for I/O tests + if len(commands) > 0 { + if _, ok := commands[0].(*IO); ok { + var cmds []*IO + // If the first is an I/O test then all must be + for _, cmd := range commands { + if cmd, ok := cmd.(*IO); ok { + cmds = append(cmds, cmd) + } else { + return errors.New("DoBatch cannot mix I/O tests with other kinds") + } + } + for i, cmd := range cmds { + if err := mc.doIOTest(i, cmd); err != nil { + return err + } + } + return nil + } + } + var tag string for _, commands := range commands { switch commands := commands.(type) { @@ -181,6 +274,10 @@ func (mc *mockServer) DoBatch(commands ...interface{}) error { //[][]interface{} } } tag = "" + case *IO: + return errors.New("DoBatch cannot mix I/O tests with other kinds") + default: + return fmt.Errorf("Unknown command input") } } return nil @@ -281,27 +378,3 @@ func (mc *mockServer) DoExpect(expect interface{}, commandName string, args ...i } return nil } -func round(v float64, decimals int) float64 { - var pow float64 = 1 - for i := 0; i < decimals; i++ { - pow *= 10 - } - return float64(int((v*pow)+0.5)) / pow -} - -func exfloat(v float64, decimals int) func(v interface{}) (resp, expect interface{}) { - ex := round(v, decimals) - return func(v interface{}) (resp, expect interface{}) { - var s string - if b, ok := v.([]uint8); ok { - s = string(b) - } else { - s = fmt.Sprintf("%v", v) - } - n, err := strconv.ParseFloat(s, 64) - if err != nil { - return v, ex - } - return round(n, decimals), ex - } -} diff --git a/tests/monitor_test.go b/tests/monitor_test.go new file mode 100644 index 00000000..32818430 --- /dev/null +++ b/tests/monitor_test.go @@ -0,0 +1,77 @@ +package tests + +import ( + "fmt" + "strings" + "sync" + + "github.com/gomodule/redigo/redis" +) + +func subTestMonitor(g *testGroup) { + g.regSubTest("monitor", follower_monitor_test) +} + +func follower_monitor_test(mc *mockServer) error { + N := 1000 + ch := make(chan error) + var wg sync.WaitGroup + wg.Add(1) + go func() { + ch <- func() error { + conn, err := redis.Dial("tcp", fmt.Sprintf("localhost:%d", mc.port)) + if err != nil { + wg.Done() + return err + } + defer conn.Close() + s, err := redis.String(conn.Do("MONITOR")) + if err != nil { + wg.Done() + return err + } + if s != "OK" { + wg.Done() + return fmt.Errorf("expected '%s', got '%s'", "OK", s) + } + wg.Done() + + for i := 0; i < N; i++ { + s, err := redis.String(conn.Receive()) + if err != nil { + return err + } + ex := fmt.Sprintf(`"mykey" "%d"`, i) + if !strings.Contains(s, ex) { + return fmt.Errorf("expected '%s', got '%s'", ex, s) + } + } + return nil + }() + }() + + wg.Wait() + + conn, err := redis.Dial("tcp", fmt.Sprintf("localhost:%d", mc.port)) + if err != nil { + return err + } + defer conn.Close() + + for i := 0; i < N; i++ { + s, err := redis.String(conn.Do("SET", "mykey", i, "POINT", 10, 10)) + if err != nil { + return err + } + if s != "OK" { + return fmt.Errorf("expected '%s', got '%s'", "OK", s) + } + } + + err = <-ch + if err != nil { + err = fmt.Errorf("monitor client: %w", err) + } + + return err +} diff --git a/tests/scripts_test.go b/tests/scripts_test.go index e879cedf..8dbefc7f 100644 --- a/tests/scripts_test.go +++ b/tests/scripts_test.go @@ -3,14 +3,13 @@ package tests import ( "fmt" "strings" - "testing" ) -func subTestScripts(t *testing.T, mc *mockServer) { - runStep(t, mc, "BASIC", scripts_BASIC_test) - runStep(t, mc, "ATOMIC", scripts_ATOMIC_test) - runStep(t, mc, "READONLY", scripts_READONLY_test) - runStep(t, mc, "NONATOMIC", scripts_NONATOMIC_test) +func subTestScripts(g *testGroup) { + g.regSubTest("BASIC", scripts_BASIC_test) + g.regSubTest("ATOMIC", scripts_ATOMIC_test) + g.regSubTest("READONLY", scripts_READONLY_test) + g.regSubTest("NONATOMIC", scripts_NONATOMIC_test) } func scripts_BASIC_test(mc *mockServer) error { diff --git a/tests/stats_test.go b/tests/stats_test.go index 7cc466ae..30670d66 100644 --- a/tests/stats_test.go +++ b/tests/stats_test.go @@ -2,13 +2,12 @@ package tests import ( "errors" - "testing" "github.com/tidwall/gjson" ) -func subTestInfo(t *testing.T, mc *mockServer) { - runStep(t, mc, "valid json", info_valid_json_test) +func subTestInfo(g *testGroup) { + g.regSubTest("valid json", info_valid_json_test) } func info_valid_json_test(mc *mockServer) error { diff --git a/tests/testcmd_test.go b/tests/testcmd_test.go index 5556b084..2aff3df2 100644 --- a/tests/testcmd_test.go +++ b/tests/testcmd_test.go @@ -1,15 +1,11 @@ package tests -import ( - "testing" -) - -func subTestTestCmd(t *testing.T, mc *mockServer) { - runStep(t, mc, "WITHIN", testcmd_WITHIN_test) - runStep(t, mc, "INTERSECTS", testcmd_INTERSECTS_test) - runStep(t, mc, "INTERSECTS_CLIP", testcmd_INTERSECTS_CLIP_test) - runStep(t, mc, "ExpressionErrors", testcmd_expressionErrors_test) - runStep(t, mc, "Expressions", testcmd_expression_test) +func subTestTestCmd(g *testGroup) { + g.regSubTest("WITHIN", testcmd_WITHIN_test) + g.regSubTest("INTERSECTS", testcmd_INTERSECTS_test) + g.regSubTest("INTERSECTS_CLIP", testcmd_INTERSECTS_CLIP_test) + g.regSubTest("ExpressionErrors", testcmd_expressionErrors_test) + g.regSubTest("Expressions", testcmd_expression_test) } func testcmd_WITHIN_test(mc *mockServer) error { @@ -29,30 +25,100 @@ func testcmd_WITHIN_test(mc *mockServer) error { poly9 := `{"type":"Polygon","coordinates":[[[-122.44037926197052,37.73313523548048],[-122.44017541408539,37.73313523548048],[-122.44017541408539,37.73336857568778],[-122.44037926197052,37.73336857568778],[-122.44037926197052,37.73313523548048]]]}` poly10 := `{"type":"Polygon","coordinates":[[[-122.44040071964262,37.73359343010089],[-122.4402666091919,37.73359343010089],[-122.4402666091919,37.73373767596864],[-122.44040071964262,37.73373767596864],[-122.44040071964262,37.73359343010089]]]}` - return mc.DoBatch([][]interface{}{ - {"SET", "mykey", "point1", "POINT", 37.7335, -122.4412}, {"OK"}, - {"SET", "mykey", "point2", "POINT", 37.7335, -122.44121}, {"OK"}, - {"SET", "mykey", "line3", "OBJECT", `{"type":"LineString","coordinates":[[-122.4408378,37.7341129],[-122.4408378,37.733]]}`}, {"OK"}, - {"SET", "mykey", "poly4", "OBJECT", `{"type":"Polygon","coordinates":[[[-122.4408378,37.7341129],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.7341129],[-122.4408378,37.7341129]]]}`}, {"OK"}, - {"SET", "mykey", "multipoly5", "OBJECT", `{"type":"MultiPolygon","coordinates":[[[[-122.4408378,37.7341129],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.7341129],[-122.4408378,37.7341129]]],[[[-122.44091033935547,37.731981251280985],[-122.43994474411011,37.731981251280985],[-122.43994474411011,37.73254976045042],[-122.44091033935547,37.73254976045042],[-122.44091033935547,37.731981251280985]]]]}`}, {"OK"}, - {"SET", "mykey", "point6", "POINT", -5, 5}, {"OK"}, - {"SET", "mykey", "point7", "POINT", 33, 21}, {"OK"}, - {"SET", "mykey", "poly8", "OBJECT", poly8}, {"OK"}, + return mc.DoBatch( + Do("SET", "mykey", "point1", "POINT", 37.7335, -122.4412).OK(), + Do("SET", "mykey", "point2", "POINT", 37.7335, -122.44121).OK(), + Do("SET", "mykey", "line3", "OBJECT", `{"type":"LineString","coordinates":[[-122.4408378,37.7341129],[-122.4408378,37.733]]}`).OK(), + Do("SET", "mykey", "poly4", "OBJECT", `{"type":"Polygon","coordinates":[[[-122.4408378,37.7341129],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.7341129],[-122.4408378,37.7341129]]]}`).OK(), + Do("SET", "mykey", "multipoly5", "OBJECT", `{"type":"MultiPolygon","coordinates":[[[[-122.4408378,37.7341129],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.7341129],[-122.4408378,37.7341129]]],[[[-122.44091033935547,37.731981251280985],[-122.43994474411011,37.731981251280985],[-122.43994474411011,37.73254976045042],[-122.44091033935547,37.73254976045042],[-122.44091033935547,37.731981251280985]]]]}`).OK(), + Do("SET", "mykey", "point6", "POINT", -5, 5).OK(), + Do("SET", "mykey", "point7", "POINT", 33, 21).OK(), + Do("SET", "mykey", "poly8", "OBJECT", poly8).OK(), - {"TEST", "GET", "mykey", "point1", "WITHIN", "OBJECT", poly}, {"1"}, - {"TEST", "GET", "mykey", "point1", "WITHIN", "SECTOR", "37.72999", "-122.44760", "1000", "0", "90"}, {"1"}, - {"TEST", "GET", "mykey", "line3", "WITHIN", "OBJECT", poly}, {"1"}, - {"TEST", "GET", "mykey", "poly4", "WITHIN", "OBJECT", poly}, {"1"}, - {"TEST", "GET", "mykey", "multipoly5", "WITHIN", "OBJECT", poly}, {"1"}, - {"TEST", "GET", "mykey", "poly8", "WITHIN", "OBJECT", poly}, {"1"}, - {"TEST", "GET", "mykey", "poly8", "WITHIN", "SECTOR", "37.72999", "-122.44760", "1000", "0", "90"}, {"1"}, + Do("TEST", "GET", "mykey", "point1", "WITHIN", "OBJECT", poly).Str("1"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "SECTOR", "37.72999", "-122.44760", "1000", "0", "90").Str("1"), + Do("TEST", "GET", "mykey", "line3", "WITHIN", "OBJECT", poly).Str("1"), + Do("TEST", "GET", "mykey", "poly4", "WITHIN", "OBJECT", poly).Str("1"), + Do("TEST", "GET", "mykey", "multipoly5", "WITHIN", "OBJECT", poly).Str("1"), + Do("TEST", "GET", "mykey", "poly8", "WITHIN", "OBJECT", poly).Str("1"), + Do("TEST", "GET", "mykey", "poly8", "WITHIN", "SECTOR", "37.72999", "-122.44760", "1000", "0", "90").Str("1"), + Do("TEST", "GET", "mykey", "poly8", "WITHIN", "SECTOR", "37.72999", "-122.44760", "1000", "0", "90").JSON().Str(`{"ok":true,"result":true}`), - {"TEST", "GET", "mykey", "point6", "WITHIN", "OBJECT", poly}, {"0"}, - {"TEST", "GET", "mykey", "point7", "WITHIN", "OBJECT", poly}, {"0"}, + Do("TEST", "GET", "mykey", "point6", "WITHIN", "OBJECT", poly).Str("0"), + Do("TEST", "GET", "mykey", "point6", "WITHIN", "OBJECT", poly).JSON().Str(`{"ok":true,"result":false}`), + Do("TEST", "GET", "mykey", "point7", "WITHIN", "OBJECT", poly).Str("0"), - {"TEST", "OBJECT", poly9, "WITHIN", "OBJECT", poly8}, {"1"}, - {"TEST", "OBJECT", poly10, "WITHIN", "OBJECT", poly8}, {"0"}, - }) + Do("TEST", "OBJECT", poly9, "WITHIN", "OBJECT", poly8).Str("1"), + Do("TEST", "OBJECT", poly10, "WITHIN", "OBJECT", poly8).Str("0"), + + Do("TEST", "GET", "mykey", "point1", "WITHIN", "SECTOR", "37.72999", "-122.44760", "1000", "0", "ff").Err("invalid argument 'ff'"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "SECTOR", "37.72999", "-122.44760", "1000", "ee", "ff").Err("invalid argument 'ee'"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "SECTOR", "37.72999", "-122.44760", "dd", "ee", "ff").Err("invalid argument 'dd'"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "SECTOR", "37.72999", "cc", "dd", "ee", "ff").Err("invalid argument 'cc'"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "SECTOR", "bb", "cc", "dd", "ee", "ff").Err("invalid argument 'bb'"), + + Do("TEST", "GET", "mykey", "point1", "WITHIN", "SECTOR", "37.72999", "-122.44760", "1000", "0").Err("wrong number of arguments for 'test' command"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "SECTOR", "37.72999", "-122.44760", "1000").Err("wrong number of arguments for 'test' command"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "SECTOR", "37.72999", "-122.44760").Err("wrong number of arguments for 'test' command"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "SECTOR", "37.72999").Err("wrong number of arguments for 'test' command"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "SECTOR").Err("wrong number of arguments for 'test' command"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "SECTOR", "37.72999", "-122.44760", "1000", "1", "1").Err("equal bearings (1 == 1), use CIRCLE instead"), + + Do("TEST", "GET", "mykey", "point1", "WITHIN", "CIRCLE", "37.72999", "-122.44760", "10000").Str("1"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "CIRCLE", "37.72999", "-122.44760", "10000", "10000").Err("wrong number of arguments for 'test' command"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "CIRCLE", "37.72999", "-122.44760").Err("wrong number of arguments for 'test' command"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "CIRCLE", "37.72999").Err("wrong number of arguments for 'test' command"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "CIRCLE").Err("wrong number of arguments for 'test' command"), + + Do("TEST", "GET", "mykey", "point1", "WITHIN", "CIRCLE", "37.72999", "-122.44760", "cc").Err("invalid argument 'cc'"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "CIRCLE", "37.72999", "bb", "cc").Err("invalid argument 'bb'"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "CIRCLE", "aa", "bb", "cc").Err("invalid argument 'aa'"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "CIRCLE", "37.72999", "-122.44760", "-10000").Err("invalid argument '-10000'"), + + Do("TEST", "GET", "mykey", "point1", "WITHIN", "hash").Err("wrong number of arguments for 'test' command"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "hash", "123").Str("0"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "hash", "123", "asdf").Err("wrong number of arguments for 'test' command"), + + Do("TEST", "GET", "mykey", "point1", "WITHIN", "quadkey").Err("wrong number of arguments for 'test' command"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "quadkey", "123").Str("0"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "quadkey", "pqowie").Err("invalid argument 'pqowie'"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "quadkey", "123", "asdf").Err("wrong number of arguments for 'test' command"), + + Do("TEST", "GET", "mykey", "point1", "WITHIN", "tile").Err("wrong number of arguments for 'test' command"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "tile", "1").Err("wrong number of arguments for 'test' command"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "tile", "1", "2").Err("wrong number of arguments for 'test' command"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "tile", "1", "2", "3").Str("0"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "tile", "1", "2", "cc").Err("invalid argument 'cc'"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "tile", "1", "bb", "cc").Err("invalid argument 'bb'"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "tile", "aa", "bb", "cc").Err("invalid argument 'aa'"), + + Do("TEST", "GET", "mykey", "point1", "WITHIN", "point", "1", "2").Str("0"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "point").Err("wrong number of arguments for 'test' command"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "point", "1").Err("wrong number of arguments for 'test' command"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "point", "1", "2", "3").Err("wrong number of arguments for 'test' command"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "point", "1", "bb").Err("invalid argument 'bb'"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "point", "aa", "bb").Err("invalid argument 'aa'"), + + Do("TEST", "GET", "mykey", "point1", "WITHIN").Err("wrong number of arguments for 'test' command"), + + Do("TEST", "GET", "mykey", "point1", "WITHIN", "bounds", "1", "2", "3", "4").Str("0"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "bounds", "1", "2", "3", "4", "5").Err("wrong number of arguments for 'test' command"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "bounds", "1", "2", "3").Err("wrong number of arguments for 'test' command"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "bounds", "1", "2").Err("wrong number of arguments for 'test' command"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "bounds", "1").Err("wrong number of arguments for 'test' command"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "bounds").Err("wrong number of arguments for 'test' command"), + + Do("TEST", "GET", "mykey", "point1", "WITHIN", "bounds", "1", "2", "3", "dd").Err("invalid argument 'dd'"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "bounds", "1", "2", "cc", "dd").Err("invalid argument 'cc'"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "bounds", "1", "bb", "cc", "dd").Err("invalid argument 'bb'"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "bounds", "aa", "bb", "cc", "dd").Err("invalid argument 'aa'"), + + Do("TEST", "GET", "mykey", "point6", "WITHIN", "GET").Err("wrong number of arguments for 'test' command"), + Do("TEST", "GET", "mykey", "point6", "WITHIN", "GET", "mykey").Err("wrong number of arguments for 'test' command"), + Do("TEST", "GET", "mykey", "point6", "WITHIN", "GET", "mykey", "point6").Str("1"), + Do("TEST", "GET", "mykey", "point6", "WITHIN", "GET", "mykey__", "point6").Err("key not found"), + Do("TEST", "GET", "mykey", "point6", "WITHIN", "GET", "mykey", "point6__").Err("id not found"), + ) } func testcmd_INTERSECTS_test(mc *mockServer) error { @@ -73,30 +139,30 @@ func testcmd_INTERSECTS_test(mc *mockServer) error { poly10 := `{"type": "Polygon","coordinates": [[[-122.44040071964262,37.73359343010089],[-122.4402666091919,37.73359343010089],[-122.4402666091919,37.73373767596864],[-122.44040071964262,37.73373767596864],[-122.44040071964262,37.73359343010089]]]}` poly101 := `{"type":"Polygon","coordinates":[[[-122.44051605463028,37.73375464605226],[-122.44028002023695,37.73375464605226],[-122.44028002023695,37.733903134117966],[-122.44051605463028,37.733903134117966],[-122.44051605463028,37.73375464605226]]]}` - return mc.DoBatch([][]interface{}{ - {"SET", "mykey", "point1", "POINT", 37.7335, -122.4412}, {"OK"}, - {"SET", "mykey", "point2", "POINT", 37.7335, -122.44121}, {"OK"}, - {"SET", "mykey", "line3", "OBJECT", `{"type":"LineString","coordinates":[[-122.4408378,37.7341129],[-122.4408378,37.733]]}`}, {"OK"}, - {"SET", "mykey", "poly4", "OBJECT", `{"type":"Polygon","coordinates":[[[-122.4408378,37.7341129],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.7341129],[-122.4408378,37.7341129]]]}`}, {"OK"}, - {"SET", "mykey", "multipoly5", "OBJECT", `{"type":"MultiPolygon","coordinates":[[[[-122.4408378,37.7341129],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.7341129],[-122.4408378,37.7341129]]],[[[-122.44091033935547,37.731981251280985],[-122.43994474411011,37.731981251280985],[-122.43994474411011,37.73254976045042],[-122.44091033935547,37.73254976045042],[-122.44091033935547,37.731981251280985]]]]}`}, {"OK"}, - {"SET", "mykey", "point6", "POINT", -5, 5}, {"OK"}, - {"SET", "mykey", "point7", "POINT", 33, 21}, {"OK"}, - {"SET", "mykey", "poly8", "OBJECT", poly8}, {"OK"}, + return mc.DoBatch( + Do("SET", "mykey", "point1", "POINT", 37.7335, -122.4412).OK(), + Do("SET", "mykey", "point2", "POINT", 37.7335, -122.44121).OK(), + Do("SET", "mykey", "line3", "OBJECT", `{"type":"LineString","coordinates":[[-122.4408378,37.7341129],[-122.4408378,37.733]]}`).OK(), + Do("SET", "mykey", "poly4", "OBJECT", `{"type":"Polygon","coordinates":[[[-122.4408378,37.7341129],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.7341129],[-122.4408378,37.7341129]]]}`).OK(), + Do("SET", "mykey", "multipoly5", "OBJECT", `{"type":"MultiPolygon","coordinates":[[[[-122.4408378,37.7341129],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.7341129],[-122.4408378,37.7341129]]],[[[-122.44091033935547,37.731981251280985],[-122.43994474411011,37.731981251280985],[-122.43994474411011,37.73254976045042],[-122.44091033935547,37.73254976045042],[-122.44091033935547,37.731981251280985]]]]}`).OK(), + Do("SET", "mykey", "point6", "POINT", -5, 5).OK(), + Do("SET", "mykey", "point7", "POINT", 33, 21).OK(), + Do("SET", "mykey", "poly8", "OBJECT", poly8).OK(), - {"TEST", "GET", "mykey", "point1", "INTERSECTS", "OBJECT", poly}, {"1"}, - {"TEST", "GET", "mykey", "point2", "INTERSECTS", "OBJECT", poly}, {"1"}, - {"TEST", "GET", "mykey", "line3", "INTERSECTS", "OBJECT", poly}, {"1"}, - {"TEST", "GET", "mykey", "poly4", "INTERSECTS", "OBJECT", poly}, {"1"}, - {"TEST", "GET", "mykey", "multipoly5", "INTERSECTS", "OBJECT", poly}, {"1"}, - {"TEST", "GET", "mykey", "poly8", "INTERSECTS", "OBJECT", poly}, {"1"}, + Do("TEST", "GET", "mykey", "point1", "INTERSECTS", "OBJECT", poly).Str("1"), + Do("TEST", "GET", "mykey", "point2", "INTERSECTS", "OBJECT", poly).Str("1"), + Do("TEST", "GET", "mykey", "line3", "INTERSECTS", "OBJECT", poly).Str("1"), + Do("TEST", "GET", "mykey", "poly4", "INTERSECTS", "OBJECT", poly).Str("1"), + Do("TEST", "GET", "mykey", "multipoly5", "INTERSECTS", "OBJECT", poly).Str("1"), + Do("TEST", "GET", "mykey", "poly8", "INTERSECTS", "OBJECT", poly).Str("1"), - {"TEST", "GET", "mykey", "point6", "INTERSECTS", "OBJECT", poly}, {"0"}, - {"TEST", "GET", "mykey", "point7", "INTERSECTS", "OBJECT", poly}, {"0"}, + Do("TEST", "GET", "mykey", "point6", "INTERSECTS", "OBJECT", poly).Str("0"), + Do("TEST", "GET", "mykey", "point7", "INTERSECTS", "OBJECT", poly).Str("0"), - {"TEST", "OBJECT", poly9, "INTERSECTS", "OBJECT", poly8}, {"1"}, - {"TEST", "OBJECT", poly10, "INTERSECTS", "OBJECT", poly8}, {"1"}, - {"TEST", "OBJECT", poly101, "INTERSECTS", "OBJECT", poly8}, {"0"}, - }) + Do("TEST", "OBJECT", poly9, "INTERSECTS", "OBJECT", poly8).Str("1"), + Do("TEST", "OBJECT", poly10, "INTERSECTS", "OBJECT", poly8).Str("1"), + Do("TEST", "OBJECT", poly101, "INTERSECTS", "OBJECT", poly8).Str("0"), + ) } func testcmd_INTERSECTS_CLIP_test(mc *mockServer) error { @@ -105,53 +171,48 @@ func testcmd_INTERSECTS_CLIP_test(mc *mockServer) error { multipoly5 := `{"type":"MultiPolygon","coordinates":[[[[-122.4408378,37.7341129],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.7341129],[-122.4408378,37.7341129]]],[[[-122.44091033935547,37.731981251280985],[-122.43994474411011,37.731981251280985],[-122.43994474411011,37.73254976045042],[-122.44091033935547,37.73254976045042],[-122.44091033935547,37.731981251280985]]]]}` poly101 := `{"type":"Polygon","coordinates":[[[-122.44051605463028,37.73375464605226],[-122.44028002023695,37.73375464605226],[-122.44028002023695,37.733903134117966],[-122.44051605463028,37.733903134117966],[-122.44051605463028,37.73375464605226]]]}` - return mc.DoBatch([][]interface{}{ - {"SET", "mykey", "point1", "POINT", 37.7335, -122.4412}, {"OK"}, + return mc.DoBatch( + Do("SET", "mykey", "point1", "POINT", 37.7335, -122.4412).OK(), - {"TEST", "OBJECT", poly9, "INTERSECTS", "CLIP", "OBJECT", "{}"}, {"ERR invalid clip type 'OBJECT'"}, - {"TEST", "OBJECT", poly9, "INTERSECTS", "CLIP", "CIRCLE", "1", "2", "3"}, {"ERR invalid clip type 'CIRCLE'"}, - {"TEST", "OBJECT", poly9, "INTERSECTS", "CLIP", "GET", "mykey", "point1"}, {"ERR invalid clip type 'GET'"}, - {"TEST", "OBJECT", poly9, "WITHIN", "CLIP", "BOUNDS", 10, 10, 20, 20}, {"ERR invalid argument 'CLIP'"}, + Do("TEST", "OBJECT", poly9, "INTERSECTS", "CLIP", "OBJECT", "{}").Err("invalid clip type 'OBJECT'"), + Do("TEST", "OBJECT", poly9, "INTERSECTS", "CLIP", "CIRCLE", "1", "2", "3").Err("invalid clip type 'CIRCLE'"), + Do("TEST", "OBJECT", poly9, "INTERSECTS", "CLIP", "GET", "mykey", "point1").Err("invalid clip type 'GET'"), + Do("TEST", "OBJECT", poly9, "WITHIN", "CLIP", "BOUNDS", 10, 10, 20, 20).Err("invalid argument 'CLIP'"), - {"TEST", "OBJECT", poly9, "INTERSECTS", "CLIP", "BOUNDS", 37.732906137107, -122.44126439094543, 37.73421283683962, -122.43980526924135}, {"[1 " + poly9 + "]"}, - {"TEST", "OBJECT", poly8, "INTERSECTS", "CLIP", "BOUNDS", 37.733, -122.4408378, 37.7341129, -122.44}, {"[1 " + poly8 + "]"}, - {"TEST", "OBJECT", multipoly5, "INTERSECTS", "CLIP", "BOUNDS", 37.73227823422744, -122.44120001792908, 37.73319038868677, -122.43955314159392}, {"[1 " + `{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[-122.4408378,37.73319038868677],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.73319038868677],[-122.4408378,37.73319038868677]]]},"properties":{}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[-122.44091033935547,37.73227823422744],[-122.43994474411011,37.73227823422744],[-122.43994474411011,37.73254976045042],[-122.44091033935547,37.73254976045042],[-122.44091033935547,37.73227823422744]]]},"properties":{}}]}` + "]"}, - {"TEST", "OBJECT", poly101, "INTERSECTS", "CLIP", "BOUNDS", 37.73315644825698, -122.44054287672043, 37.73349585185455, -122.44008690118788}, {"0"}, - }) + Do("TEST", "OBJECT", poly9, "INTERSECTS", "CLIP", "SECTOR").Err("invalid clip type 'SECTOR'"), + + Do("TEST", "OBJECT", poly9, "INTERSECTS", "CLIP", "BOUNDS", 37.732906137107, -122.44126439094543, 37.73421283683962, -122.43980526924135).Str("[1 "+poly9+"]"), + Do("TEST", "OBJECT", poly9, "INTERSECTS", "CLIP", "BOUNDS", 37.732906137107, -122.44126439094543, 37.73421283683962, -122.43980526924135).JSON().Str(`{"ok":true,"result":true,"object":`+poly9+`}`), + Do("TEST", "OBJECT", poly8, "INTERSECTS", "CLIP", "BOUNDS", 37.733, -122.4408378, 37.7341129, -122.44).Str("[1 "+poly8+"]"), + Do("TEST", "OBJECT", multipoly5, "INTERSECTS", "CLIP", "BOUNDS", 37.73227823422744, -122.44120001792908, 37.73319038868677, -122.43955314159392).Str("[1 "+`{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[-122.4408378,37.73319038868677],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.73319038868677],[-122.4408378,37.73319038868677]]]},"properties":{}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[-122.44091033935547,37.73227823422744],[-122.43994474411011,37.73227823422744],[-122.43994474411011,37.73254976045042],[-122.44091033935547,37.73254976045042],[-122.44091033935547,37.73227823422744]]]},"properties":{}}]}`+"]"), + Do("TEST", "OBJECT", poly101, "INTERSECTS", "CLIP", "BOUNDS", 37.73315644825698, -122.44054287672043, 37.73349585185455, -122.44008690118788).Str("0"), + ) } func testcmd_expressionErrors_test(mc *mockServer) error { - return mc.DoBatch([][]interface{}{ - {"SET", "mykey", "foo", "OBJECT", `{"type":"LineString","coordinates":[[-122.4408378,37.7341129],[-122.4408378,37.733]]}`}, {"OK"}, - {"SET", "mykey", "bar", "OBJECT", `{"type":"LineString","coordinates":[[-122.4408378,37.7341129],[-122.4408378,37.733]]}`}, {"OK"}, - {"SET", "mykey", "baz", "OBJECT", `{"type":"LineString","coordinates":[[-122.4408378,37.7341129],[-122.4408378,37.733]]}`}, {"OK"}, + return mc.DoBatch( + Do("SET", "mykey", "foo", "OBJECT", `{"type":"LineString","coordinates":[[-122.4408378,37.7341129],[-122.4408378,37.733]]}`).OK(), + Do("SET", "mykey", "bar", "OBJECT", `{"type":"LineString","coordinates":[[-122.4408378,37.7341129],[-122.4408378,37.733]]}`).OK(), + Do("SET", "mykey", "baz", "OBJECT", `{"type":"LineString","coordinates":[[-122.4408378,37.7341129],[-122.4408378,37.733]]}`).OK(), - {"TEST", "GET", "mykey", "foo", "INTERSECTS", "(", "GET", "mykey", "bar"}, { - "ERR wrong number of arguments for 'test' command"}, - {"TEST", "GET", "mykey", "foo", "INTERSECTS", "GET", "mykey", "bar", ")"}, { - "ERR invalid argument ')'"}, + Do("TEST", "GET", "mykey", "foo", "INTERSECTS", "(", "GET", "mykey", "bar").Err("wrong number of arguments for 'test' command"), + Do("TEST", "GET", "mykey", "foo", "INTERSECTS", "GET", "mykey", "bar", ")").Err("invalid argument ')'"), - {"TEST", "GET", "mykey", "foo", "INTERSECTS", "OR", "GET", "mykey", "bar"}, { - "ERR invalid argument 'or'"}, - {"TEST", "GET", "mykey", "foo", "INTERSECTS", "AND", "GET", "mykey", "bar"}, { - "ERR invalid argument 'and'"}, - {"TEST", "GET", "mykey", "foo", "INTERSECTS", "GET", "mykey", "bar", "OR", "AND", "GET", "mykey", "baz"}, { - "ERR invalid argument 'and'"}, - {"TEST", "GET", "mykey", "foo", "INTERSECTS", "GET", "mykey", "bar", "AND", "OR", "GET", "mykey", "baz"}, { - "ERR invalid argument 'or'"}, - {"TEST", "GET", "mykey", "foo", "INTERSECTS", "GET", "mykey", "bar", "OR", "OR", "GET", "mykey", "baz"}, { - "ERR invalid argument 'or'"}, - {"TEST", "GET", "mykey", "foo", "INTERSECTS", "GET", "mykey", "bar", "AND", "AND", "GET", "mykey", "baz"}, { - "ERR invalid argument 'and'"}, - {"TEST", "GET", "mykey", "foo", "INTERSECTS", "GET", "mykey", "bar", "OR"}, { - "ERR wrong number of arguments for 'test' command"}, - {"TEST", "GET", "mykey", "foo", "INTERSECTS", "GET", "mykey", "bar", "AND"}, { - "ERR wrong number of arguments for 'test' command"}, - {"TEST", "GET", "mykey", "foo", "INTERSECTS", "GET", "mykey", "bar", "NOT"}, { - "ERR wrong number of arguments for 'test' command"}, - {"TEST", "GET", "mykey", "foo", "INTERSECTS", "GET", "mykey", "bar", "NOT", "AND", "GET", "mykey", "baz"}, { - "ERR invalid argument 'and'"}, - }) + Do("TEST", "GET", "mykey", "foo", "INTERSECTS", "OR", "GET", "mykey", "bar").Err("invalid argument 'or'"), + Do("TEST", "GET", "mykey", "foo", "INTERSECTS", "AND", "GET", "mykey", "bar").Err("invalid argument 'and'"), + Do("TEST", "GET", "mykey", "foo", "INTERSECTS", "GET", "mykey", "bar", "OR", "AND", "GET", "mykey", "baz").Err("invalid argument 'and'"), + Do("TEST", "GET", "mykey", "foo", "INTERSECTS", "GET", "mykey", "bar", "AND", "OR", "GET", "mykey", "baz").Err("invalid argument 'or'"), + Do("TEST", "GET", "mykey", "foo", "INTERSECTS", "GET", "mykey", "bar", "OR", "OR", "GET", "mykey", "baz").Err("invalid argument 'or'"), + Do("TEST", "GET", "mykey", "foo", "INTERSECTS", "GET", "mykey", "bar", "AND", "AND", "GET", "mykey", "baz").Err("invalid argument 'and'"), + Do("TEST", "GET", "mykey", "foo", "INTERSECTS", "GET", "mykey", "bar", "OR").Err("wrong number of arguments for 'test' command"), + Do("TEST", "GET", "mykey", "foo", "INTERSECTS", "GET", "mykey", "bar", "AND").Err("wrong number of arguments for 'test' command"), + Do("TEST", "GET", "mykey", "foo", "INTERSECTS", "GET", "mykey", "bar", "NOT").Err("wrong number of arguments for 'test' command"), + Do("TEST", "GET", "mykey", "foo", "INTERSECTS", "GET", "mykey", "bar", "NOT", "AND", "GET", "mykey", "baz").Err("invalid argument 'and'"), + + Do("TEST").Err("wrong number of arguments for 'test' command"), + Do("TEST", "GET", "mykey", "foo").Err("wrong number of arguments for 'test' command"), + Do("TEST", "GET", "mykey", "foo", "jello").Err("invalid argument 'jello'"), + ) } func testcmd_expression_test(mc *mockServer) error { @@ -170,46 +231,36 @@ func testcmd_expression_test(mc *mockServer) error { poly8 := `{"type":"Polygon","coordinates":[[[-122.4408378,37.7341129],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.7341129],[-122.4408378,37.7341129]],[[-122.44060993194579,37.73345766902749],[-122.44044363498686,37.73345766902749],[-122.44044363498686,37.73355524732416],[-122.44060993194579,37.73355524732416],[-122.44060993194579,37.73345766902749]],[[-122.44060724973677,37.7336888869566],[-122.4402102828026,37.7336888869566],[-122.4402102828026,37.7339752567853],[-122.44060724973677,37.7339752567853],[-122.44060724973677,37.7336888869566]]]}` poly9 := `{"type": "Polygon","coordinates": [[[-122.44037926197052,37.73313523548048],[-122.44017541408539,37.73313523548048],[-122.44017541408539,37.73336857568778],[-122.44037926197052,37.73336857568778],[-122.44037926197052,37.73313523548048]]]}` - return mc.DoBatch([][]interface{}{ - {"SET", "mykey", "line3", "OBJECT", `{"type":"LineString","coordinates":[[-122.4408378,37.7341129],[-122.4408378,37.733]]}`}, {"OK"}, - {"SET", "mykey", "poly8", "OBJECT", poly8}, {"OK"}, + return mc.DoBatch( + Do("SET", "mykey", "line3", "OBJECT", `{"type":"LineString","coordinates":[[-122.4408378,37.7341129],[-122.4408378,37.733]]}`).OK(), + Do("SET", "mykey", "poly8", "OBJECT", poly8).OK(), - {"TEST", "OBJECT", poly9, "INTERSECTS", "NOT", "OBJECT", poly}, {"0"}, - {"TEST", "OBJECT", poly9, "INTERSECTS", "NOT", "NOT", "OBJECT", poly}, {"1"}, - {"TEST", "OBJECT", poly9, "INTERSECTS", "NOT", "NOT", "NOT", "OBJECT", poly}, {"0"}, + Do("TEST", "OBJECT", poly9, "INTERSECTS", "NOT", "OBJECT", poly).Str("0"), + Do("TEST", "OBJECT", poly9, "INTERSECTS", "NOT", "NOT", "OBJECT", poly).Str("1"), + Do("TEST", "OBJECT", poly9, "INTERSECTS", "NOT", "NOT", "NOT", "OBJECT", poly).Str("0"), - {"TEST", "OBJECT", poly9, "INTERSECTS", "OBJECT", poly8, "OR", "OBJECT", poly}, {"1"}, - {"TEST", "OBJECT", poly9, "INTERSECTS", "OBJECT", poly8, "AND", "OBJECT", poly}, {"1"}, - {"TEST", "OBJECT", poly9, "INTERSECTS", "GET", "mykey", "poly8", "OR", "OBJECT", poly}, {"1"}, + Do("TEST", "OBJECT", poly9, "INTERSECTS", "OBJECT", poly8, "OR", "OBJECT", poly).Str("1"), + Do("TEST", "OBJECT", poly9, "INTERSECTS", "OBJECT", poly8, "AND", "OBJECT", poly).Str("1"), + Do("TEST", "OBJECT", poly9, "INTERSECTS", "GET", "mykey", "poly8", "OR", "OBJECT", poly).Str("1"), - {"TEST", "OBJECT", poly9, "INTERSECTS", "GET", "mykey", "line3"}, {"0"}, - {"TEST", "OBJECT", poly9, "INTERSECTS", "GET", "mykey", "poly8", "AND", - "(", "OBJECT", poly, "AND", "GET", "mykey", "line3", ")"}, {"0"}, - {"TEST", "OBJECT", poly9, "INTERSECTS", "GET", "mykey", "poly8", "AND", - "(", "OBJECT", poly, "OR", "GET", "mykey", "line3", ")"}, {"1"}, - {"TEST", "OBJECT", poly9, "INTERSECTS", "GET", "mykey", "poly8", "AND", - "(", "OBJECT", poly, "AND", "NOT", "GET", "mykey", "line3", ")"}, {"1"}, - {"TEST", "OBJECT", poly9, "INTERSECTS", "NOT", "GET", "mykey", "line3"}, {"1"}, - {"TEST", "NOT", "OBJECT", poly9, "INTERSECTS", "GET", "mykey", "line3"}, {"1"}, - {"TEST", "OBJECT", poly9, "INTERSECTS", "GET", "mykey", "line3", - "OR", "OBJECT", poly8, "AND", "OBJECT", poly}, {"1"}, - {"TEST", "OBJECT", poly9, "INTERSECTS", "OBJECT", poly8, "AND", "OBJECT", poly, - "OR", "GET", "mykey", "line3"}, {"1"}, - {"TEST", "OBJECT", poly9, "INTERSECTS", "GET", "mykey", "line3", "OR", - "(", "OBJECT", poly8, "AND", "OBJECT", poly, ")"}, {"1"}, - {"TEST", "OBJECT", poly9, "INTERSECTS", - "(", "GET", "mykey", "line3", "OR", "OBJECT", poly8, ")", "AND", "OBJECT", poly}, {"1"}, + Do("TEST", "OBJECT", poly9, "INTERSECTS", "GET", "mykey", "line3").Str("0"), + Do("TEST", "OBJECT", poly9, "INTERSECTS", "GET", "mykey", "poly8", "AND", "(", "OBJECT", poly, "AND", "GET", "mykey", "line3", ")").Str("0"), + Do("TEST", "OBJECT", poly9, "INTERSECTS", "GET", "mykey", "poly8", "AND", "(", "OBJECT", poly, "OR", "GET", "mykey", "line3", ")").Str("1"), + Do("TEST", "OBJECT", poly9, "INTERSECTS", "GET", "mykey", "poly8", "AND", "(", "OBJECT", poly, "AND", "NOT", "GET", "mykey", "line3", ")").Str("1"), + Do("TEST", "OBJECT", poly9, "INTERSECTS", "NOT", "GET", "mykey", "line3").Str("1"), + Do("TEST", "NOT", "OBJECT", poly9, "INTERSECTS", "GET", "mykey", "line3").Str("1"), + Do("TEST", "OBJECT", poly9, "INTERSECTS", "GET", "mykey", "line3", "OR", "OBJECT", poly8, "AND", "OBJECT", poly).Str("1"), + Do("TEST", "OBJECT", poly9, "INTERSECTS", "OBJECT", poly8, "AND", "OBJECT", poly, "OR", "GET", "mykey", "line3").Str("1"), + Do("TEST", "OBJECT", poly9, "INTERSECTS", "GET", "mykey", "line3", "OR", "(", "OBJECT", poly8, "AND", "OBJECT", poly, ")").Str("1"), + Do("TEST", "OBJECT", poly9, "INTERSECTS", "(", "GET", "mykey", "line3", "OR", "OBJECT", poly8, ")", "AND", "OBJECT", poly).Str("1"), - {"TEST", "OBJECT", poly9, "WITHIN", "OBJECT", poly8, "OR", "OBJECT", poly}, {"1"}, - {"TEST", "OBJECT", poly9, "WITHIN", "OBJECT", poly8, "AND", "OBJECT", poly}, {"1"}, + Do("TEST", "OBJECT", poly9, "WITHIN", "OBJECT", poly8, "OR", "OBJECT", poly).Str("1"), + Do("TEST", "OBJECT", poly9, "WITHIN", "OBJECT", poly8, "AND", "OBJECT", poly).Str("1"), - {"TEST", "OBJECT", poly9, "WITHIN", "GET", "mykey", "line3"}, {"0"}, - {"TEST", "OBJECT", poly9, "WITHIN", "GET", "mykey", "poly8", "AND", - "(", "OBJECT", poly, "AND", "GET", "mykey", "line3", ")"}, {"0"}, - {"TEST", "OBJECT", poly9, "WITHIN", "GET", "mykey", "poly8", "AND", - "(", "OBJECT", poly, "OR", "GET", "mykey", "line3", ")"}, {"1"}, - {"TEST", "OBJECT", poly9, "WITHIN", "GET", "mykey", "poly8", "AND", - "(", "OBJECT", poly, "AND", "NOT", "GET", "mykey", "line3", ")"}, {"1"}, - {"TEST", "OBJECT", poly9, "WITHIN", "NOT", "GET", "mykey", "line3"}, {"1"}, - }) + Do("TEST", "OBJECT", poly9, "WITHIN", "GET", "mykey", "line3").Str("0"), + Do("TEST", "OBJECT", poly9, "WITHIN", "GET", "mykey", "poly8", "AND", "(", "OBJECT", poly, "AND", "GET", "mykey", "line3", ")").Str("0"), + Do("TEST", "OBJECT", poly9, "WITHIN", "GET", "mykey", "poly8", "AND", "(", "OBJECT", poly, "OR", "GET", "mykey", "line3", ")").Str("1"), + Do("TEST", "OBJECT", poly9, "WITHIN", "GET", "mykey", "poly8", "AND", "(", "OBJECT", poly, "AND", "NOT", "GET", "mykey", "line3", ")").Str("1"), + Do("TEST", "OBJECT", poly9, "WITHIN", "NOT", "GET", "mykey", "line3").Str("1"), + ) } diff --git a/tests/tests_test.go b/tests/tests_test.go index b4f1b3ee..7298426c 100644 --- a/tests/tests_test.go +++ b/tests/tests_test.go @@ -5,11 +5,16 @@ import ( "math/rand" "os" "os/signal" + "runtime" + "strings" + "sync" "syscall" "testing" "time" "github.com/gomodule/redigo/redis" + "github.com/tidwall/limiter" + "go.uber.org/atomic" ) const ( @@ -26,11 +31,12 @@ const ( white = "\x1b[37m" ) -func TestAll(t *testing.T) { - mockCleanup(false) - defer mockCleanup(false) +func TestIntegration(t *testing.T) { - ch := make(chan os.Signal) + mockCleanup(true) + defer mockCleanup(true) + + ch := make(chan os.Signal, 1) signal.Notify(ch, os.Interrupt, syscall.SIGTERM) go func() { <-ch @@ -38,62 +44,190 @@ func TestAll(t *testing.T) { os.Exit(1) }() - mc, err := mockOpenServer(false) - if err != nil { - t.Fatal(err) + regTestGroup("keys", subTestKeys) + regTestGroup("json", subTestJSON) + regTestGroup("search", subTestSearch) + regTestGroup("testcmd", subTestTestCmd) + regTestGroup("client", subTestClient) + regTestGroup("scripts", subTestScripts) + regTestGroup("fence", subTestFence) + regTestGroup("info", subTestInfo) + regTestGroup("timeouts", subTestTimeout) + regTestGroup("metrics", subTestMetrics) + regTestGroup("follower", subTestFollower) + regTestGroup("aof", subTestAOF) + regTestGroup("monitor", subTestMonitor) + runTestGroups(t) +} + +var allGroups []*testGroup + +func runTestGroups(t *testing.T) { + limit := runtime.NumCPU() + if limit > 16 { + limit = 16 } - defer mc.Close() - runSubTest(t, "keys", mc, subTestKeys) - runSubTest(t, "json", mc, subTestJSON) - runSubTest(t, "search", mc, subTestSearch) - runSubTest(t, "testcmd", mc, subTestTestCmd) - runSubTest(t, "fence", mc, subTestFence) - runSubTest(t, "scripts", mc, subTestScripts) - runSubTest(t, "info", mc, subTestInfo) - runSubTest(t, "client", mc, subTestClient) - runSubTest(t, "timeouts", mc, subTestTimeout) - runSubTest(t, "metrics", mc, subTestMetrics) -} + l := limiter.New(limit) -func runSubTest(t *testing.T, name string, mc *mockServer, test func(t *testing.T, mc *mockServer)) { - t.Run(name, func(t *testing.T) { - fmt.Printf(bright+"Testing %s\n"+clear, name) - test(t, mc) - }) -} - -func runStep(t *testing.T, mc *mockServer, name string, step func(mc *mockServer) error) { - t.Helper() - t.Run(name, func(t *testing.T) { - t.Helper() - if err := func() error { - // reset the current server - mc.ResetConn() - defer mc.ResetConn() - // clear the database so the test is consistent - if err := mc.DoBatch([][]interface{}{ - {"OUTPUT", "resp"}, {"OK"}, - {"FLUSHDB"}, {"OK"}, - }); err != nil { - return err - } - if err := step(mc); err != nil { - return err - } - return nil - }(); err != nil { - fmt.Printf("["+red+"fail"+clear+"]: %s\n", name) - t.Fatal(err) + // Initialize all stores as "skipped", but they'll be unset if the test is + // not actually skipped. + for _, g := range allGroups { + for _, s := range g.subs { + s.skipped.Store(true) } - fmt.Printf("["+green+"ok"+clear+"]: %s\n", name) - }) + } + for _, g := range allGroups { + func(g *testGroup) { + t.Run(g.name, func(t *testing.T) { + for _, s := range g.subs { + func(s *testGroupSub) { + t.Run(s.name, func(t *testing.T) { + s.skipped.Store(false) + var wg sync.WaitGroup + wg.Add(1) + var err error + go func() { + l.Begin() + defer func() { + l.End() + wg.Done() + }() + err = s.run() + }() + if false { + t.Parallel() + t.Run("bg", func(t *testing.T) { + wg.Wait() + if err != nil { + t.Fatal(err) + } + }) + } + }) + }(s) + } + }) + }(g) + } + + done := make(chan bool) + go func() { + defer func() { done <- true }() + for { + finished := true + for _, g := range allGroups { + skipped := true + for _, s := range g.subs { + if !s.skipped.Load() { + skipped = false + break + } + } + if !skipped && !g.printed.Load() { + fmt.Printf(bright+"Testing %s\n"+clear, g.name) + g.printed.Store(true) + } + const frtmp = "[" + magenta + " " + clear + "] %s (running) " + for _, s := range g.subs { + if !s.skipped.Load() && !s.printedName.Load() { + fmt.Printf(frtmp, s.name) + s.printedName.Store(true) + } + if s.done.Load() && !s.printedResult.Load() { + fmt.Printf("\r") + msg := fmt.Sprintf(frtmp, s.name) + fmt.Print(strings.Repeat(" ", len(msg))) + fmt.Printf("\r") + if s.err != nil { + fmt.Printf("["+red+"fail"+clear+"] %s\n", s.name) + } else { + fmt.Printf("["+green+"ok"+clear+"] %s\n", s.name) + } + s.printedResult.Store(true) + } + if !s.skipped.Load() && !s.done.Load() { + finished = false + break + } + } + if !finished { + break + } + } + if finished { + break + } + time.Sleep(time.Second / 4) + } + }() + <-done + var fail bool + for _, g := range allGroups { + for _, s := range g.subs { + if s.err != nil { + t.Errorf("%s/%s/%s\n%s", t.Name(), g.name, s.name, s.err) + fail = true + } + } + } + if fail { + t.Fail() + } + +} + +type testGroup struct { + name string + subs []*testGroupSub + printed atomic.Bool +} + +type testGroupSub struct { + g *testGroup + name string + fn func(mc *mockServer) error + err error + skipped atomic.Bool + done atomic.Bool + printedName atomic.Bool + printedResult atomic.Bool +} + +func regTestGroup(name string, fn func(g *testGroup)) { + g := &testGroup{name: name} + allGroups = append(allGroups, g) + fn(g) +} + +func (g *testGroup) regSubTest(name string, fn func(mc *mockServer) error) { + s := &testGroupSub{g: g, name: name, fn: fn} + g.subs = append(g.subs, s) +} + +func (s *testGroupSub) run() (err error) { + // This all happens in a background routine. + defer func() { + s.err = err + s.done.Store(true) + }() + return func() error { + mc, err := mockOpenServer(MockServerOptions{ + Silent: true, + Metrics: true, + }) + if err != nil { + return err + } + defer mc.Close() + return s.fn(mc) + }() } func BenchmarkAll(b *testing.B) { mockCleanup(true) defer mockCleanup(true) - ch := make(chan os.Signal) + ch := make(chan os.Signal, 1) signal.Notify(ch, os.Interrupt, syscall.SIGTERM) go func() { <-ch @@ -101,7 +235,9 @@ func BenchmarkAll(b *testing.B) { os.Exit(1) }() - mc, err := mockOpenServer(true) + mc, err := mockOpenServer(MockServerOptions{ + Silent: true, Metrics: true, + }) if err != nil { b.Fatal(err) } diff --git a/tests/timeout_test.go b/tests/timeout_test.go index aa8d4263..816e820c 100644 --- a/tests/timeout_test.go +++ b/tests/timeout_test.go @@ -4,19 +4,18 @@ import ( "fmt" "math/rand" "strings" - "testing" "time" "github.com/gomodule/redigo/redis" ) -func subTestTimeout(t *testing.T, mc *mockServer) { - runStep(t, mc, "spatial", timeout_spatial_test) - runStep(t, mc, "search", timeout_search_test) - runStep(t, mc, "scripts", timeout_scripts_test) - runStep(t, mc, "no writes", timeout_no_writes_test) - runStep(t, mc, "within scripts", timeout_within_scripts_test) - runStep(t, mc, "no writes within scripts", timeout_no_writes_within_scripts_test) +func subTestTimeout(g *testGroup) { + g.regSubTest("spatial", timeout_spatial_test) + g.regSubTest("search", timeout_search_test) + g.regSubTest("scripts", timeout_scripts_test) + g.regSubTest("no writes", timeout_no_writes_test) + g.regSubTest("within scripts", timeout_within_scripts_test) + g.regSubTest("no writes within scripts", timeout_no_writes_within_scripts_test) } func setup(mc *mockServer, count int, points bool) (err error) { @@ -54,22 +53,27 @@ func setup(mc *mockServer, count int, points bool) (err error) { return } -func timeout_spatial_test(mc *mockServer) (err error) { - err = setup(mc, 10000, true) +func timeout_spatial_test(mc *mockServer) error { + err := setup(mc, 10000, true) + if err != nil { + return err + } + return mc.DoBatch( + Do("SCAN", "mykey", "WHERE", "foo", -1, 2, "COUNT").Str("10000"), + Do("INTERSECTS", "mykey", "WHERE", "foo", -1, 2, "COUNT", "BOUNDS", -90, -180, 90, 180).Str("10000"), + Do("WITHIN", "mykey", "WHERE", "foo", -1, 2, "COUNT", "BOUNDS", -90, -180, 90, 180).Str("10000"), - return mc.DoBatch([][]interface{}{ - {"SCAN", "mykey", "WHERE", "foo", -1, 2, "COUNT"}, {"10000"}, - {"INTERSECTS", "mykey", "WHERE", "foo", -1, 2, "COUNT", "BOUNDS", -90, -180, 90, 180}, {"10000"}, - {"WITHIN", "mykey", "WHERE", "foo", -1, 2, "COUNT", "BOUNDS", -90, -180, 90, 180}, {"10000"}, - - {"TIMEOUT", "0.000001", "SCAN", "mykey", "WHERE", "foo", -1, 2, "COUNT"}, {"ERR timeout"}, - {"TIMEOUT", "0.000001", "INTERSECTS", "mykey", "WHERE", "foo", -1, 2, "COUNT", "BOUNDS", -90, -180, 90, 180}, {"ERR timeout"}, - {"TIMEOUT", "0.000001", "WITHIN", "mykey", "WHERE", "foo", -1, 2, "COUNT", "BOUNDS", -90, -180, 90, 180}, {"ERR timeout"}, - }) + Do("TIMEOUT", "0.000001", "SCAN", "mykey", "WHERE", "foo", -1, 2, "COUNT").Err("timeout"), + Do("TIMEOUT", "0.000001", "INTERSECTS", "mykey", "WHERE", "foo", -1, 2, "COUNT", "BOUNDS", -90, -180, 90, 180).Err("timeout"), + Do("TIMEOUT", "0.000001", "WITHIN", "mykey", "WHERE", "foo", -1, 2, "COUNT", "BOUNDS", -90, -180, 90, 180).Err("timeout"), + ) } func timeout_search_test(mc *mockServer) (err error) { err = setup(mc, 10000, false) + if err != nil { + return err + } return mc.DoBatch([][]interface{}{ {"SEARCH", "mykey", "MATCH", "val:*", "COUNT"}, {"10000"}, @@ -122,6 +126,9 @@ func scriptTimeoutErr(v interface{}) (resp, expect interface{}) { func timeout_within_scripts_test(mc *mockServer) (err error) { err = setup(mc, 10000, true) + if err != nil { + return err + } script1 := "return tile38.call('timeout', 10, 'SCAN', 'mykey', 'WHERE', 'foo', -1, 2, 'COUNT')" script2 := "return tile38.call('timeout', 0.000001, 'SCAN', 'mykey', 'WHERE', 'foo', -1, 2, 'COUNT')"