From d61f0bc6c849f3e618934655f83aeec6a18a8472 Mon Sep 17 00:00:00 2001 From: tidwall Date: Fri, 23 Sep 2022 07:30:03 -0700 Subject: [PATCH 01/33] wip - better tests --- scripts/test.sh | 13 ++- tests/fence_roaming_test.go | 10 +- tests/fence_test.go | 10 +- tests/keys_test.go | 82 ++----------- tests/metrics_test.go | 4 +- tests/mock_io_test.go | 227 ++++++++++++++++++++++++++++++++++++ tests/mock_test.go | 68 ++++++----- tests/tests_test.go | 7 +- tests/timeout_test.go | 9 ++ 9 files changed, 305 insertions(+), 125 deletions(-) create mode 100644 tests/mock_io_test.go diff --git a/scripts/test.sh b/scripts/test.sh index 1fc9288b..9b2b83a8 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -5,10 +5,11 @@ cd $(dirname "${BASH_SOURCE[0]}")/.. export CGO_ENABLED=0 -# if [ "$NOMODULES" != "1" ]; then -# export GO111MODULE=on -# export GOFLAGS=-mod=vendor -# fi +cd tests +go test -coverpkg=../internal/server -coverprofile=/tmp/coverage.out +go tool cover -html=/tmp/coverage.out -o /tmp/coverage.html +echo "details: file:///tmp/coverage.html" +cd .. -cd tests && go test && cd .. -go test $(go list ./... | grep -v /vendor/ | grep -v /tests) +# go test -coverpkg=internal/ \ +# $(go list ./... | grep -v /vendor/ | grep -v /tests) diff --git a/tests/fence_roaming_test.go b/tests/fence_roaming_test.go index e96084a3..4ba6796d 100644 --- a/tests/fence_roaming_test.go +++ b/tests/fence_roaming_test.go @@ -2,7 +2,7 @@ package tests import ( "fmt" - "io/ioutil" + "io" "net/http" "net/http/httptest" "sync" @@ -29,7 +29,7 @@ func fence_roaming_webhook_test(mc *mockServer) error { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if err := func() error { // Read the request body - body, err := ioutil.ReadAll(r.Body) + body, err := io.ReadAll(r.Body) if err != nil { return err } @@ -115,8 +115,10 @@ func fence_roaming_live_test(mc *mockServer) error { liveReady.Add(1) return goMultiFunc(mc, func() error { - sc, err := redis.DialTimeout("tcp", fmt.Sprintf(":%d", mc.port), - 0, time.Second*5, time.Second*5) + sc, err := redis.Dial("tcp", fmt.Sprintf(":%d", mc.port), + redis.DialConnectTimeout(0), + redis.DialReadTimeout(time.Second*5), + redis.DialWriteTimeout(time.Second*5)) if err != nil { liveReady.Done() return err diff --git a/tests/fence_test.go b/tests/fence_test.go index 013d4b19..192ead53 100644 --- a/tests/fence_test.go +++ b/tests/fence_test.go @@ -205,7 +205,7 @@ func fence_channel_message_order_test(mc *mockServer) error { break loop } case error: - fmt.Printf(err.Error()) + fmt.Printf("%s\n", err.Error()) } } @@ -230,10 +230,10 @@ func fence_channel_message_order_test(mc *mockServer) error { // Fire all setup commands on the base client for _, cmd := range []string{ "SET points point POINT 33.412529053733444 -111.93368911743164", - fmt.Sprintf(`SETCHAN A WITHIN points FENCE OBJECT {"type":"Polygon","coordinates":[[[-111.95205688476562,33.400491820565236],[-111.92630767822266,33.400491820565236],[-111.92630767822266,33.422272258866045],[-111.95205688476562,33.422272258866045],[-111.95205688476562,33.400491820565236]]]}`), - fmt.Sprintf(`SETCHAN B WITHIN points FENCE OBJECT {"type":"Polygon","coordinates":[[[-111.93952560424803,33.403501285221594],[-111.92630767822266,33.403501285221594],[-111.92630767822266,33.41997983836345],[-111.93952560424803,33.41997983836345],[-111.93952560424803,33.403501285221594]]]}`), - fmt.Sprintf(`SETCHAN C WITHIN points FENCE OBJECT {"type":"Polygon","coordinates":[[[-111.9255781173706,33.40342963251261],[-111.91201686859131,33.40342963251261],[-111.91201686859131,33.41994401881284],[-111.9255781173706,33.41994401881284],[-111.9255781173706,33.40342963251261]]]}`), - fmt.Sprintf(`SETCHAN D WITHIN points FENCE OBJECT {"type":"Polygon","coordinates":[[[-111.92562103271484,33.40063513076968],[-111.90021514892578,33.40063513076968],[-111.90021514892578,33.42212898435788],[-111.92562103271484,33.42212898435788],[-111.92562103271484,33.40063513076968]]]}`), + `SETCHAN A WITHIN points FENCE OBJECT {"type":"Polygon","coordinates":[[[-111.95205688476562,33.400491820565236],[-111.92630767822266,33.400491820565236],[-111.92630767822266,33.422272258866045],[-111.95205688476562,33.422272258866045],[-111.95205688476562,33.400491820565236]]]}`, + `SETCHAN B WITHIN points FENCE OBJECT {"type":"Polygon","coordinates":[[[-111.93952560424803,33.403501285221594],[-111.92630767822266,33.403501285221594],[-111.92630767822266,33.41997983836345],[-111.93952560424803,33.41997983836345],[-111.93952560424803,33.403501285221594]]]}`, + `SETCHAN C WITHIN points FENCE OBJECT {"type":"Polygon","coordinates":[[[-111.9255781173706,33.40342963251261],[-111.91201686859131,33.40342963251261],[-111.91201686859131,33.41994401881284],[-111.9255781173706,33.41994401881284],[-111.9255781173706,33.40342963251261]]]}`, + `SETCHAN D WITHIN points FENCE OBJECT {"type":"Polygon","coordinates":[[[-111.92562103271484,33.40063513076968],[-111.90021514892578,33.40063513076968],[-111.90021514892578,33.42212898435788],[-111.92562103271484,33.42212898435788],[-111.92562103271484,33.40063513076968]]]}`, "SET points point POINT 33.412529053733444 -111.91909790039062", } { if _, err := do(bc, cmd); err != nil { diff --git a/tests/keys_test.go b/tests/keys_test.go index aef23ded..9bf47e77 100644 --- a/tests/keys_test.go +++ b/tests/keys_test.go @@ -4,9 +4,6 @@ import ( "errors" "fmt" "math/rand" - "os/exec" - "strconv" - "strings" "testing" "time" @@ -36,18 +33,19 @@ func subTestKeys(t *testing.T, mc *mockServer) { } func keys_BOUNDS_test(mc *mockServer) error { - return mc.DoBatch([][]interface{}{ - {"SET", "mykey", "myid1", "POINT", 33, -115}, {"OK"}, - {"BOUNDS", "mykey"}, {"[[-115 33] [-115 33]]"}, - {"SET", "mykey", "myid2", "POINT", 34, -112}, {"OK"}, - {"BOUNDS", "mykey"}, {"[[-115 33] [-112 34]]"}, - {"DEL", "mykey", "myid2"}, {1}, - {"BOUNDS", "mykey"}, {"[[-115 33] [-115 33]]"}, - {"SET", "mykey", "myid3", "OBJECT", `{"type":"Point","coordinates":[-130,38,10]}`}, {"OK"}, - {"SET", "mykey", "myid4", "OBJECT", `{"type":"Point","coordinates":[-110,25,-8]}`}, {"OK"}, - {"BOUNDS", "mykey"}, {"[[-130 25] [-110 38]]"}, - }) + return mc.DoBatch( + Do("SET", "mykey", "myid1", "POINT", 33, -115).OK(), + Do("BOUNDS", "mykey").String("[[-115 33] [-115 33]]"), + Do("SET", "mykey", "myid2", "POINT", 34, -112).OK(), + Do("BOUNDS", "mykey").String("[[-115 33] [-112 34]]"), + Do("DEL", "mykey", "myid2").String("1"), + Do("BOUNDS", "mykey").String("[[-115 33] [-115 33]]"), + Do("SET", "mykey", "myid3", "OBJECT", `{"type":"Point","coordinates":[-130,38,10]}`).OK(), + Do("SET", "mykey", "myid4", "OBJECT", `{"type":"Point","coordinates":[-110,25,-8]}`).OK(), + Do("BOUNDS", "mykey").String("[[-130 25] [-110 38]]"), + ) } + func keys_DEL_test(mc *mockServer) error { return mc.DoBatch([][]interface{}{ {"SET", "mykey", "myid", "POINT", 33, -115}, {"OK"}, @@ -256,62 +254,6 @@ func keys_TTL_test(mc *mockServer) error { }) } -type PSAUX struct { - User string - PID int - CPU float64 - Mem float64 - VSZ int - RSS int - TTY string - Stat string - Start string - Time string - Command string -} - -func atoi(s string) int { - n, _ := strconv.ParseInt(s, 10, 64) - return int(n) -} -func atof(s string) float64 { - n, _ := strconv.ParseFloat(s, 64) - return float64(n) -} -func psaux(pid int) PSAUX { - var res []byte - res, err := exec.Command("ps", "aux").CombinedOutput() - if err != nil { - return PSAUX{} - } - pids := strconv.FormatInt(int64(pid), 10) - for _, line := range strings.Split(string(res), "\n") { - var words []string - for _, word := range strings.Split(line, " ") { - if word != "" { - words = append(words, word) - } - if len(words) > 11 { - if words[1] == pids { - return PSAUX{ - User: words[0], - PID: atoi(words[1]), - CPU: atof(words[2]), - Mem: atof(words[3]), - VSZ: atoi(words[4]), - RSS: atoi(words[5]), - TTY: words[6], - Stat: words[7], - Start: words[8], - Time: words[9], - Command: words[10], - } - } - } - } - } - return PSAUX{} -} func keys_SET_EX_test(mc *mockServer) (err error) { rand.Seed(time.Now().UnixNano()) diff --git a/tests/metrics_test.go b/tests/metrics_test.go index b7fa4055..14461bb3 100644 --- a/tests/metrics_test.go +++ b/tests/metrics_test.go @@ -1,7 +1,7 @@ package tests import ( - "io/ioutil" + "io" "net/http" "strings" "testing" @@ -13,7 +13,7 @@ func downloadURLWithStatusCode(t *testing.T, u string) (int, string) { t.Fatal(err) } defer resp.Body.Close() - body, err := ioutil.ReadAll(resp.Body) + body, err := io.ReadAll(resp.Body) if err != nil { t.Fatal(err) } diff --git a/tests/mock_io_test.go b/tests/mock_io_test.go new file mode 100644 index 00000000..f4502aa1 --- /dev/null +++ b/tests/mock_io_test.go @@ -0,0 +1,227 @@ +package tests + +import ( + "errors" + "fmt" + "go/ast" + "go/parser" + "go/token" + "os" + "path/filepath" + "runtime" + "strings" + + "github.com/tidwall/gjson" +) + +type IO struct { + args []any + json bool + out any +} + +func Do(args ...any) *IO { + return &IO{args: args} +} +func (cmd *IO) JSON() *IO { + cmd.json = true + return cmd +} +func (cmd *IO) String(s string) *IO { + cmd.out = s + return cmd +} +func (cmd *IO) Custom(fn func(s string) error) *IO { + cmd.out = func(s string) error { + if cmd.json { + if !gjson.Valid(s) { + return errors.New("invalid json") + } + } + return fn(s) + } + return cmd +} + +func (cmd *IO) OK() *IO { + return cmd.Custom(func(s string) error { + if cmd.json { + if gjson.Get(s, "ok").Type != gjson.True { + return errors.New("not ok") + } + } else if s != "OK" { + return errors.New("not ok") + } + return nil + }) +} + +type ioVisitor struct { + fset *token.FileSet + ln int + pos int + got bool + data string + end int + done bool + index int + nidx int + frag string + fpos int +} + +func (v *ioVisitor) Visit(n ast.Node) ast.Visitor { + if n == nil || v.done { + return nil + } + + if v.got { + if int(n.Pos()) > v.end { + v.done = true + return v + } + if n, ok := n.(*ast.CallExpr); ok { + frag := strings.TrimSpace(v.data[int(n.Pos())-1 : int(n.End())]) + if _, ok := n.Fun.(*ast.Ident); ok { + if v.index == v.nidx { + frag = strings.TrimSpace(strings.TrimSuffix(frag, ".")) + idx := strings.IndexByte(frag, '(') + if idx != -1 { + frag = frag[idx:] + } + v.frag = frag + v.done = true + v.fpos = int(n.Pos()) + return v + } + v.nidx++ + } + } + return v + } + if int(n.Pos()) == v.pos { + if n, ok := n.(*ast.CallExpr); ok { + v.end = int(n.Rparen) + v.got = true + return v + } + } + return v +} + +func (cmd *IO) deepError(index int, err error) error { + oerr := err + werr := func(err error) error { + return fmt.Errorf("batch[%d]: %v: %v", index, oerr, err) + } + + // analyse stack + _, file, ln, ok := runtime.Caller(3) + if !ok { + return werr(errors.New("runtime.Caller failed")) + } + + // get the character position from line + bdata, err := os.ReadFile(file) + if err != nil { + return werr(err) + } + data := string(bdata) + + var pos int + var iln int + var pln int + for i := 0; i < len(data); i++ { + if data[i] == '\n' { + j := pln + line := data[pln:i] + pln = i + 1 + iln++ + if iln == ln { + line = strings.TrimSpace(line) + if !strings.HasPrefix(line, "return mc.DoBatch(") { + return oerr + } + for ; j < len(data); j++ { + if data[j] == 'm' { + break + } + } + pos = j + 1 + break + } + } + } + if pos == 0 { + return oerr + } + + fset := token.NewFileSet() + pfile, err := parser.ParseFile(fset, file, nil, 0) + if err != nil { + return werr(err) + } + v := &ioVisitor{ + fset: fset, + ln: ln, + pos: pos, + data: string(data), + index: index, + } + ast.Walk(v, pfile) + + if v.fpos == 0 { + return oerr + } + + pln = 1 + for i := 0; i < len(data); i++ { + if data[i] == '\n' { + if i > v.fpos { + break + } + pln++ + } + } + + fsig := fmt.Sprintf("%s:%d", filepath.Base(file), pln) + emsg := oerr.Error() + if strings.HasPrefix(emsg, "expected ") && + strings.Contains(emsg, ", got ") { + emsg = "" + + " EXPECTED: " + strings.Split(emsg, ", got ")[0][9:] + "\n" + + " GOT: " + + strings.Split(emsg, ", got ")[1] + } else { + emsg = "" + + " ERROR: " + emsg + } + + return fmt.Errorf("\n%s: entry[%d]\n COMMAND: %s\n%s", + fsig, index+1, v.frag, emsg) +} + +func (mc *mockServer) doIOTest(index int, cmd *IO) error { + // switch json mode if desired + if cmd.json { + if !mc.ioJSON { + if _, err := mc.Do("OUTPUT", "json"); err != nil { + return err + } + mc.ioJSON = true + } + } else { + if mc.ioJSON { + if _, err := mc.Do("OUTPUT", "resp"); err != nil { + return err + } + mc.ioJSON = false + } + } + + err := mc.DoExpect(cmd.out, cmd.args[0].(string), cmd.args[1:]...) + if err != nil { + return cmd.deepError(index, err) + } + return nil +} diff --git a/tests/mock_test.go b/tests/mock_test.go index f3a07cf2..a1aec6fb 100644 --- a/tests/mock_test.go +++ b/tests/mock_test.go @@ -3,11 +3,10 @@ package tests import ( "errors" "fmt" - "io/ioutil" + "io" "log" "math/rand" "os" - "strconv" "strings" "time" @@ -24,7 +23,7 @@ func mockCleanup(silent bool) { if !silent { fmt.Printf("Cleanup: may take some time... ") } - files, _ := ioutil.ReadDir(".") + files, _ := os.ReadDir(".") for _, file := range files { if strings.HasPrefix(file.Name(), "data-mock-") { os.RemoveAll(file.Name()) @@ -36,11 +35,9 @@ func mockCleanup(silent bool) { } type mockServer struct { - port int - //join string - //n *finn.Node - //m *Machine - conn redis.Conn + port int + conn redis.Conn + ioJSON bool } func mockOpenServer(silent bool) (*mockServer, error) { @@ -50,7 +47,7 @@ func mockOpenServer(silent bool) (*mockServer, error) { if !silent { fmt.Printf("Starting test server at port %d\n", port) } - logOutput := ioutil.Discard + logOutput := io.Discard if os.Getenv("PRINTLOG") == "1" { logOutput = os.Stderr } @@ -80,7 +77,7 @@ func (s *mockServer) waitForStartup() error { var lerr error start := time.Now() for { - if time.Now().Sub(start) > time.Second*5 { + if time.Since(start) > time.Second*5 { if lerr != nil { return lerr } @@ -159,7 +156,28 @@ func (s *mockServer) Do(commandName string, args ...interface{}) (interface{}, e return resps[0], nil } -func (mc *mockServer) DoBatch(commands ...interface{}) error { //[][]interface{}) error { +func (mc *mockServer) DoBatch(commands ...interface{}) error { + // Probe for I/O tests + if len(commands) > 0 { + if _, ok := commands[0].(*IO); ok { + var cmds []*IO + // If the first is an I/O test then all must be + for _, cmd := range commands { + if cmd, ok := cmd.(*IO); ok { + cmds = append(cmds, cmd) + } else { + return errors.New("DoBatch cannot mix I/O tests with other kinds") + } + } + for i, cmd := range cmds { + if err := mc.doIOTest(i, cmd); err != nil { + return err + } + } + return nil + } + } + var tag string for _, commands := range commands { switch commands := commands.(type) { @@ -181,6 +199,10 @@ func (mc *mockServer) DoBatch(commands ...interface{}) error { //[][]interface{} } } tag = "" + case *IO: + return errors.New("DoBatch cannot mix I/O tests with other kinds") + default: + return fmt.Errorf("Unknown command input") } } return nil @@ -281,27 +303,3 @@ func (mc *mockServer) DoExpect(expect interface{}, commandName string, args ...i } return nil } -func round(v float64, decimals int) float64 { - var pow float64 = 1 - for i := 0; i < decimals; i++ { - pow *= 10 - } - return float64(int((v*pow)+0.5)) / pow -} - -func exfloat(v float64, decimals int) func(v interface{}) (resp, expect interface{}) { - ex := round(v, decimals) - return func(v interface{}) (resp, expect interface{}) { - var s string - if b, ok := v.([]uint8); ok { - s = string(b) - } else { - s = fmt.Sprintf("%v", v) - } - n, err := strconv.ParseFloat(s, 64) - if err != nil { - return v, ex - } - return round(n, decimals), ex - } -} diff --git a/tests/tests_test.go b/tests/tests_test.go index b4f1b3ee..5f20937d 100644 --- a/tests/tests_test.go +++ b/tests/tests_test.go @@ -30,7 +30,7 @@ func TestAll(t *testing.T) { mockCleanup(false) defer mockCleanup(false) - ch := make(chan os.Signal) + ch := make(chan os.Signal, 1) signal.Notify(ch, os.Interrupt, syscall.SIGTERM) go func() { <-ch @@ -82,8 +82,9 @@ func runStep(t *testing.T, mc *mockServer, name string, step func(mc *mockServer } return nil }(); err != nil { - fmt.Printf("["+red+"fail"+clear+"]: %s\n", name) + fmt.Fprintf(os.Stderr, "["+red+"fail"+clear+"]: %s\n", name) t.Fatal(err) + // t.Fatal(err) } fmt.Printf("["+green+"ok"+clear+"]: %s\n", name) }) @@ -93,7 +94,7 @@ func BenchmarkAll(b *testing.B) { mockCleanup(true) defer mockCleanup(true) - ch := make(chan os.Signal) + ch := make(chan os.Signal, 1) signal.Notify(ch, os.Interrupt, syscall.SIGTERM) go func() { <-ch diff --git a/tests/timeout_test.go b/tests/timeout_test.go index aa8d4263..1663e47b 100644 --- a/tests/timeout_test.go +++ b/tests/timeout_test.go @@ -56,6 +56,9 @@ func setup(mc *mockServer, count int, points bool) (err error) { func timeout_spatial_test(mc *mockServer) (err error) { err = setup(mc, 10000, true) + if err != nil { + return err + } return mc.DoBatch([][]interface{}{ {"SCAN", "mykey", "WHERE", "foo", -1, 2, "COUNT"}, {"10000"}, @@ -70,6 +73,9 @@ func timeout_spatial_test(mc *mockServer) (err error) { func timeout_search_test(mc *mockServer) (err error) { err = setup(mc, 10000, false) + if err != nil { + return err + } return mc.DoBatch([][]interface{}{ {"SEARCH", "mykey", "MATCH", "val:*", "COUNT"}, {"10000"}, @@ -122,6 +128,9 @@ func scriptTimeoutErr(v interface{}) (resp, expect interface{}) { func timeout_within_scripts_test(mc *mockServer) (err error) { err = setup(mc, 10000, true) + if err != nil { + return err + } script1 := "return tile38.call('timeout', 10, 'SCAN', 'mykey', 'WHERE', 'foo', -1, 2, 'COUNT')" script2 := "return tile38.call('timeout', 0.000001, 'SCAN', 'mykey', 'WHERE', 'foo', -1, 2, 'COUNT')" From ef95f04acad3930eb661c06d122a977e6ef08975 Mon Sep 17 00:00:00 2001 From: tidwall Date: Fri, 23 Sep 2022 07:51:05 -0700 Subject: [PATCH 02/33] Better coverage BOUNDS --- internal/server/crud.go | 53 +++++++++++++++++++------------------- internal/server/scripts.go | 2 +- internal/server/server.go | 2 +- scripts/test.sh | 5 ++-- tests/keys_test.go | 8 ++++++ tests/mock_io_test.go | 20 ++++++++++++++ 6 files changed, 58 insertions(+), 32 deletions(-) diff --git a/internal/server/crud.go b/internal/server/crud.go index eef2a2b9..a9a37877 100644 --- a/internal/server/crud.go +++ b/internal/server/crud.go @@ -17,27 +17,30 @@ import ( "github.com/tidwall/tile38/internal/object" ) -func (s *Server) cmdBounds(msg *Message) (resp.Value, error) { +// BOUNDS key +func (s *Server) cmdBOUNDS(msg *Message) (resp.Value, error) { start := time.Now() - vs := msg.Args[1:] - var ok bool - var key string - if vs, key, ok = tokenval(vs); !ok || key == "" { - return NOMessage, errInvalidNumberOfArguments - } - if len(vs) != 0 { - return NOMessage, errInvalidNumberOfArguments + // >> Args + + args := msg.Args + if len(args) != 2 { + return retrerr(errInvalidNumberOfArguments) } + key := args[1] + + // >> Operation col, _ := s.cols.Get(key) if col == nil { if msg.OutputType == RESP { return resp.NullValue(), nil } - return NOMessage, errKeyNotFound + return retrerr(errKeyNotFound) } + // >> Response + vals := make([]resp.Value, 0, 2) var buf bytes.Buffer if msg.OutputType == JSON { @@ -52,26 +55,22 @@ func (s *Server) cmdBounds(msg *Message) (resp.Value, error) { if msg.OutputType == JSON { buf.WriteString(`,"bounds":`) buf.WriteString(string(bbox.AppendJSON(nil))) - } else { - vals = append(vals, resp.ArrayValue([]resp.Value{ - resp.ArrayValue([]resp.Value{ - resp.FloatValue(minX), - resp.FloatValue(minY), - }), - resp.ArrayValue([]resp.Value{ - resp.FloatValue(maxX), - resp.FloatValue(maxY), - }), - })) - } - switch msg.OutputType { - case JSON: buf.WriteString(`,"elapsed":"` + time.Since(start).String() + "\"}") return resp.StringValue(buf.String()), nil - case RESP: - return vals[0], nil } - return NOMessage, nil + + // RESP + vals = append(vals, resp.ArrayValue([]resp.Value{ + resp.ArrayValue([]resp.Value{ + resp.FloatValue(minX), + resp.FloatValue(minY), + }), + resp.ArrayValue([]resp.Value{ + resp.FloatValue(maxX), + resp.FloatValue(maxY), + }), + })) + return vals[0], nil } func (s *Server) cmdType(msg *Message) (resp.Value, error) { diff --git a/internal/server/scripts.go b/internal/server/scripts.go index 8b8b5fca..e97d8066 100644 --- a/internal/server/scripts.go +++ b/internal/server/scripts.go @@ -624,7 +624,7 @@ func (s *Server) commandInScript(msg *Message) ( case "search": res, err = s.cmdSearch(msg) case "bounds": - res, err = s.cmdBounds(msg) + res, err = s.cmdBOUNDS(msg) case "get": res, err = s.cmdGet(msg) case "jget": diff --git a/internal/server/server.go b/internal/server/server.go index 66a116a1..4cdc385f 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -1096,7 +1096,7 @@ func (s *Server) command(msg *Message, client *Client) ( case "search": res, err = s.cmdSearch(msg) case "bounds": - res, err = s.cmdBounds(msg) + res, err = s.cmdBOUNDS(msg) case "get": res, err = s.cmdGet(msg) case "jget": diff --git a/scripts/test.sh b/scripts/test.sh index 9b2b83a8..a5997b74 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -6,10 +6,9 @@ cd $(dirname "${BASH_SOURCE[0]}")/.. export CGO_ENABLED=0 cd tests -go test -coverpkg=../internal/server -coverprofile=/tmp/coverage.out +go test -coverpkg=../internal/server -coverprofile=/tmp/coverage.out $GOTEST go tool cover -html=/tmp/coverage.out -o /tmp/coverage.html echo "details: file:///tmp/coverage.html" cd .. -# go test -coverpkg=internal/ \ -# $(go list ./... | grep -v /vendor/ | grep -v /tests) +go test $(go list ./... | grep -v /vendor/ | grep -v /tests) diff --git a/tests/keys_test.go b/tests/keys_test.go index 9bf47e77..29e4625d 100644 --- a/tests/keys_test.go +++ b/tests/keys_test.go @@ -34,8 +34,11 @@ func subTestKeys(t *testing.T, mc *mockServer) { func keys_BOUNDS_test(mc *mockServer) error { return mc.DoBatch( + Do("BOUNDS", "mykey").String(""), + Do("BOUNDS", "mykey").JSON().Error("key not found"), Do("SET", "mykey", "myid1", "POINT", 33, -115).OK(), Do("BOUNDS", "mykey").String("[[-115 33] [-115 33]]"), + Do("BOUNDS", "mykey").JSON().String(`{"ok":true,"bounds":{"type":"Point","coordinates":[-115,33]}}`), Do("SET", "mykey", "myid2", "POINT", 34, -112).OK(), Do("BOUNDS", "mykey").String("[[-115 33] [-112 34]]"), Do("DEL", "mykey", "myid2").String("1"), @@ -43,6 +46,11 @@ func keys_BOUNDS_test(mc *mockServer) error { Do("SET", "mykey", "myid3", "OBJECT", `{"type":"Point","coordinates":[-130,38,10]}`).OK(), Do("SET", "mykey", "myid4", "OBJECT", `{"type":"Point","coordinates":[-110,25,-8]}`).OK(), Do("BOUNDS", "mykey").String("[[-130 25] [-110 38]]"), + Do("BOUNDS", "mykey", "hello").Error("wrong number of arguments for 'bounds' command"), + Do("BOUNDS", "nada").String(""), + Do("BOUNDS", "nada").JSON().Error("key not found"), + Do("BOUNDS", "").String(""), + Do("BOUNDS", "mykey").JSON().String(`{"ok":true,"bounds":{"type":"Polygon","coordinates":[[[-130,25],[-110,25],[-110,38],[-130,38],[-130,25]]]}}`), ) } diff --git a/tests/mock_io_test.go b/tests/mock_io_test.go index f4502aa1..3f07fd91 100644 --- a/tests/mock_io_test.go +++ b/tests/mock_io_test.go @@ -56,6 +56,26 @@ func (cmd *IO) OK() *IO { }) } +func (cmd *IO) Error(msg string) *IO { + return cmd.Custom(func(s string) error { + if cmd.json { + if gjson.Get(s, "ok").Type != gjson.False { + return errors.New("ok=true") + } + if gjson.Get(s, "err").String() != msg { + return fmt.Errorf("expected '%s', got '%s'", + msg, gjson.Get(s, "err").String()) + } + } else { + s = strings.TrimPrefix(s, "ERR ") + if s != msg { + return fmt.Errorf("expected '%s', got '%s'", msg, s) + } + } + return nil + }) +} + type ioVisitor struct { fset *token.FileSet ln int From db380a4fee472e64cd5d5dce1cb1cfba9aee59ef Mon Sep 17 00:00:00 2001 From: tidwall Date: Fri, 23 Sep 2022 09:04:01 -0700 Subject: [PATCH 03/33] Better DEL/PDEL/TYPE tests --- internal/server/crud.go | 143 ++++++++++++++++++------------------- internal/server/expire.go | 2 +- internal/server/scripts.go | 6 +- internal/server/server.go | 6 +- scripts/test.sh | 4 +- tests/keys_test.go | 120 +++++++++++++++++++------------ tests/mock_io_test.go | 4 +- 7 files changed, 156 insertions(+), 129 deletions(-) diff --git a/internal/server/crud.go b/internal/server/crud.go index a9a37877..65702cd3 100644 --- a/internal/server/crud.go +++ b/internal/server/crud.go @@ -73,33 +73,38 @@ func (s *Server) cmdBOUNDS(msg *Message) (resp.Value, error) { return vals[0], nil } -func (s *Server) cmdType(msg *Message) (resp.Value, error) { +// TYPE key +// undocumented return "none" or "hash" +func (s *Server) cmdTYPE(msg *Message) (resp.Value, error) { start := time.Now() - vs := msg.Args[1:] - var ok bool - var key string - if _, key, ok = tokenval(vs); !ok || key == "" { - return NOMessage, errInvalidNumberOfArguments + // >> Args + + args := msg.Args + if len(args) != 2 { + return retrerr(errInvalidNumberOfArguments) } + key := args[1] + + // >> Operation col, _ := s.cols.Get(key) if col == nil { if msg.OutputType == RESP { return resp.SimpleStringValue("none"), nil } - return NOMessage, errKeyNotFound + return retrerr(errKeyNotFound) } + // >> Response + typ := "hash" - switch msg.OutputType { - case JSON: - return resp.StringValue(`{"ok":true,"type":` + string(typ) + `,"elapsed":"` + time.Since(start).String() + "\"}"), nil - case RESP: - return resp.SimpleStringValue(typ), nil + if msg.OutputType == JSON { + return resp.StringValue(`{"ok":true,"type":` + jsonString(typ) + + `,"elapsed":"` + time.Since(start).String() + "\"}"), nil } - return NOMessage, nil + return resp.SimpleStringValue(typ), nil } func (s *Server) cmdGet(msg *Message) (resp.Value, error) { @@ -262,7 +267,7 @@ func (s *Server) cmdGet(msg *Message) (resp.Value, error) { } // DEL key id [ERRON404] -func (s *Server) cmdDel(msg *Message) (resp.Value, commandDetails, error) { +func (s *Server) cmdDEL(msg *Message) (resp.Value, commandDetails, error) { start := time.Now() // >> Args @@ -329,85 +334,75 @@ func (s *Server) cmdDel(msg *Message) (resp.Value, commandDetails, error) { return res, d, nil } -func (s *Server) cmdPdel(msg *Message) (res resp.Value, d commandDetails, err error) { +// PDEL key pattern +func (s *Server) cmdPDEL(msg *Message) (resp.Value, commandDetails, error) { start := time.Now() - vs := msg.Args[1:] - var ok bool - if vs, d.key, ok = tokenval(vs); !ok || d.key == "" { - err = errInvalidNumberOfArguments - return - } - if vs, d.pattern, ok = tokenval(vs); !ok || d.pattern == "" { - err = errInvalidNumberOfArguments - return - } - if len(vs) != 0 { - err = errInvalidNumberOfArguments - return - } - now := time.Now() - iter := func(o *object.Object) bool { - if match, _ := glob.Match(d.pattern, o.ID()); match { - d.children = append(d.children, &commandDetails{ - command: "del", - updated: true, - timestamp: now, - key: d.key, - obj: o, - }) - } - return true - } - var expired int - col, _ := s.cols.Get(d.key) + // >> Args + + args := msg.Args + if len(args) != 3 { + return retwerr(errInvalidNumberOfArguments) + } + key := args[1] + pattern := args[2] + + // >> Operation + + now := time.Now() + var children []*commandDetails + col, _ := s.cols.Get(key) if col != nil { - g := glob.Parse(d.pattern, false) + g := glob.Parse(pattern, false) + var ids []string + iter := func(o *object.Object) bool { + if match, _ := glob.Match(pattern, o.ID()); match { + ids = append(ids, o.ID()) + } + return true + } if g.Limits[0] == "" && g.Limits[1] == "" { col.Scan(false, nil, msg.Deadline, iter) } else { - col.ScanRange(g.Limits[0], g.Limits[1], false, nil, msg.Deadline, iter) + col.ScanRange(g.Limits[0], g.Limits[1], + false, nil, msg.Deadline, iter) } - var atLeastOneNotDeleted bool - for i, dc := range d.children { - old := col.Delete(dc.obj.ID()) - if old == nil { - d.children[i].command = "?" - atLeastOneNotDeleted = true - } else { - dc.obj = old - d.children[i] = dc - } - s.groupDisconnectObject(dc.key, dc.obj.ID()) - } - if atLeastOneNotDeleted { - var nchildren []*commandDetails - for _, dc := range d.children { - if dc.command == "del" { - nchildren = append(nchildren, dc) - } - } - d.children = nchildren + for _, id := range ids { + obj := col.Delete(id) + children = append(children, &commandDetails{ + command: "del", + updated: true, + timestamp: now, + key: key, + obj: obj, + }) + s.groupDisconnectObject(key, id) } if col.Count() == 0 { - s.cols.Delete(d.key) + s.cols.Delete(key) } } + + // >> Response + + var d commandDetails + var res resp.Value + d.command = "pdel" + d.children = children + d.key = key + d.pattern = pattern d.updated = len(d.children) > 0 d.timestamp = now d.parent = true switch msg.OutputType { case JSON: - res = resp.StringValue(`{"ok":true,"elapsed":"` + time.Since(start).String() + "\"}") + res = resp.StringValue(`{"ok":true,"elapsed":"` + + time.Since(start).String() + "\"}") case RESP: - total := len(d.children) - expired - if total < 0 { - total = 0 - } - res = resp.IntegerValue(total) + res = resp.IntegerValue(len(d.children)) } - return + return res, d, nil } func (s *Server) cmdDrop(msg *Message) (res resp.Value, d commandDetails, err error) { diff --git a/internal/server/expire.go b/internal/server/expire.go index cd67fd79..39abd412 100644 --- a/internal/server/expire.go +++ b/internal/server/expire.go @@ -42,7 +42,7 @@ func (s *Server) backgroundExpireObjects(now time.Time) { return true }) for _, msg := range msgs { - _, d, err := s.cmdDel(msg) + _, d, err := s.cmdDEL(msg) if err != nil { log.Fatal(err) } diff --git a/internal/server/scripts.go b/internal/server/scripts.go index e97d8066..1c3a5fc5 100644 --- a/internal/server/scripts.go +++ b/internal/server/scripts.go @@ -596,9 +596,9 @@ func (s *Server) commandInScript(msg *Message) ( case "fset": res, d, err = s.cmdFSET(msg) case "del": - res, d, err = s.cmdDel(msg) + res, d, err = s.cmdDEL(msg) case "pdel": - res, d, err = s.cmdPdel(msg) + res, d, err = s.cmdPDEL(msg) case "drop": res, d, err = s.cmdDrop(msg) case "expire": @@ -634,7 +634,7 @@ func (s *Server) commandInScript(msg *Message) ( case "jdel": res, d, err = s.cmdJdel(msg) case "type": - res, err = s.cmdType(msg) + res, err = s.cmdTYPE(msg) case "keys": res, err = s.cmdKeys(msg) case "test": diff --git a/internal/server/server.go b/internal/server/server.go index 4cdc385f..b27ac477 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -1020,9 +1020,9 @@ func (s *Server) command(msg *Message, client *Client) ( case "fset": res, d, err = s.cmdFSET(msg) case "del": - res, d, err = s.cmdDel(msg) + res, d, err = s.cmdDEL(msg) case "pdel": - res, d, err = s.cmdPdel(msg) + res, d, err = s.cmdPDEL(msg) case "drop": res, d, err = s.cmdDrop(msg) case "flushdb": @@ -1106,7 +1106,7 @@ func (s *Server) command(msg *Message, client *Client) ( case "jdel": res, d, err = s.cmdJdel(msg) case "type": - res, err = s.cmdType(msg) + res, err = s.cmdTYPE(msg) case "keys": res, err = s.cmdKeys(msg) case "output": diff --git a/scripts/test.sh b/scripts/test.sh index a5997b74..36086708 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -11,4 +11,6 @@ go tool cover -html=/tmp/coverage.out -o /tmp/coverage.html echo "details: file:///tmp/coverage.html" cd .. -go test $(go list ./... | grep -v /vendor/ | grep -v /tests) +if [[ "$GOTEST" == "" ]]; then + go test $(go list ./... | grep -v /vendor/ | grep -v /tests) +fi diff --git a/tests/keys_test.go b/tests/keys_test.go index 29e4625d..ed5b3998 100644 --- a/tests/keys_test.go +++ b/tests/keys_test.go @@ -30,38 +30,50 @@ func subTestKeys(t *testing.T, mc *mockServer) { runStep(t, mc, "FIELDS", keys_FIELDS_test) runStep(t, mc, "WHEREIN", keys_WHEREIN_test) runStep(t, mc, "WHEREEVAL", keys_WHEREEVAL_test) + runStep(t, mc, "TYPE", keys_TYPE_test) } func keys_BOUNDS_test(mc *mockServer) error { return mc.DoBatch( - Do("BOUNDS", "mykey").String(""), - Do("BOUNDS", "mykey").JSON().Error("key not found"), + Do("BOUNDS", "mykey").Str(""), + Do("BOUNDS", "mykey").JSON().Err("key not found"), Do("SET", "mykey", "myid1", "POINT", 33, -115).OK(), - Do("BOUNDS", "mykey").String("[[-115 33] [-115 33]]"), - Do("BOUNDS", "mykey").JSON().String(`{"ok":true,"bounds":{"type":"Point","coordinates":[-115,33]}}`), + Do("BOUNDS", "mykey").Str("[[-115 33] [-115 33]]"), + Do("BOUNDS", "mykey").JSON().Str(`{"ok":true,"bounds":{"type":"Point","coordinates":[-115,33]}}`), Do("SET", "mykey", "myid2", "POINT", 34, -112).OK(), - Do("BOUNDS", "mykey").String("[[-115 33] [-112 34]]"), - Do("DEL", "mykey", "myid2").String("1"), - Do("BOUNDS", "mykey").String("[[-115 33] [-115 33]]"), + Do("BOUNDS", "mykey").Str("[[-115 33] [-112 34]]"), + Do("DEL", "mykey", "myid2").Str("1"), + Do("BOUNDS", "mykey").Str("[[-115 33] [-115 33]]"), Do("SET", "mykey", "myid3", "OBJECT", `{"type":"Point","coordinates":[-130,38,10]}`).OK(), Do("SET", "mykey", "myid4", "OBJECT", `{"type":"Point","coordinates":[-110,25,-8]}`).OK(), - Do("BOUNDS", "mykey").String("[[-130 25] [-110 38]]"), - Do("BOUNDS", "mykey", "hello").Error("wrong number of arguments for 'bounds' command"), - Do("BOUNDS", "nada").String(""), - Do("BOUNDS", "nada").JSON().Error("key not found"), - Do("BOUNDS", "").String(""), - Do("BOUNDS", "mykey").JSON().String(`{"ok":true,"bounds":{"type":"Polygon","coordinates":[[[-130,25],[-110,25],[-110,38],[-130,38],[-130,25]]]}}`), + Do("BOUNDS", "mykey").Str("[[-130 25] [-110 38]]"), + Do("BOUNDS", "mykey", "hello").Err("wrong number of arguments for 'bounds' command"), + Do("BOUNDS", "nada").Str(""), + Do("BOUNDS", "nada").JSON().Err("key not found"), + Do("BOUNDS", "").Str(""), + Do("BOUNDS", "mykey").JSON().Str(`{"ok":true,"bounds":{"type":"Polygon","coordinates":[[[-130,25],[-110,25],[-110,38],[-130,38],[-130,25]]]}}`), ) } func keys_DEL_test(mc *mockServer) error { - return mc.DoBatch([][]interface{}{ - {"SET", "mykey", "myid", "POINT", 33, -115}, {"OK"}, - {"GET", "mykey", "myid", "POINT"}, {"[33 -115]"}, - {"DEL", "mykey", "myid"}, {"1"}, - {"GET", "mykey", "myid"}, {nil}, - }) + return mc.DoBatch( + Do("SET", "mykey", "myid", "POINT", 33, -115).OK(), + Do("GET", "mykey", "myid", "POINT").Str("[33 -115]"), + Do("DEL", "mykey", "myid2", "ERRON404").Err("id not found"), + Do("DEL", "mykey", "myid").Str("1"), + Do("DEL", "mykey", "myid").Str("0"), + Do("DEL", "mykey").Err("wrong number of arguments for 'del' command"), + Do("GET", "mykey", "myid").Str(""), + Do("DEL", "mykey", "myid", "ERRON404").Err("key not found"), + Do("DEL", "mykey", "myid", "invalid-arg").Err("invalid argument 'invalid-arg'"), + Do("SET", "mykey", "myid", "POINT", 33, -115).OK(), + Do("DEL", "mykey", "myid2", "ERRON404").JSON().Err("id not found"), + Do("DEL", "mykey", "myid").JSON().OK(), + Do("DEL", "mykey", "myid").JSON().OK(), + Do("DEL", "mykey", "myid", "ERRON404").JSON().Err("key not found"), + ) } + func keys_DROP_test(mc *mockServer) error { return mc.DoBatch([][]interface{}{ {"SET", "mykey", "myid1", "HASH", "9my5xp7"}, {"OK"}, @@ -138,14 +150,14 @@ func keys_FSET_test(mc *mockServer) error { }) } func keys_GET_test(mc *mockServer) error { - return mc.DoBatch([][]interface{}{ - {"SET", "mykey", "myid", "STRING", "value"}, {"OK"}, - {"GET", "mykey", "myid"}, {"value"}, - {"SET", "mykey", "myid", "STRING", "value2"}, {"OK"}, - {"GET", "mykey", "myid"}, {"value2"}, - {"DEL", "mykey", "myid"}, {"1"}, - {"GET", "mykey", "myid"}, {nil}, - }) + return mc.DoBatch( + Do("SET", "mykey", "myid", "STRING", "value").OK(), + Do("GET", "mykey", "myid").Str("value"), + Do("SET", "mykey", "myid", "STRING", "value2").OK(), + Do("GET", "mykey", "myid").Str("value2"), + Do("DEL", "mykey", "myid").Str("1"), + Do("GET", "mykey", "myid").Str(""), + ) } func keys_KEYS_test(mc *mockServer) error { return mc.DoBatch([][]interface{}{ @@ -314,24 +326,31 @@ func keys_FIELDS_test(mc *mockServer) error { } func keys_PDEL_test(mc *mockServer) error { - return mc.DoBatch([][]interface{}{ - {"SET", "mykey", "myid1a", "POINT", 33, -115}, {"OK"}, - {"SET", "mykey", "myid1b", "POINT", 33, -115}, {"OK"}, - {"SET", "mykey", "myid2a", "POINT", 33, -115}, {"OK"}, - {"SET", "mykey", "myid2b", "POINT", 33, -115}, {"OK"}, - {"SET", "mykey", "myid3a", "POINT", 33, -115}, {"OK"}, - {"SET", "mykey", "myid3b", "POINT", 33, -115}, {"OK"}, - {"SET", "mykey", "myid4a", "POINT", 33, -115}, {"OK"}, - {"SET", "mykey", "myid4b", "POINT", 33, -115}, {"OK"}, - {"PDEL", "mykeyNA", "*"}, {0}, - {"PDEL", "mykey", "myid1a"}, {1}, - {"PDEL", "mykey", "myid1a"}, {0}, - {"PDEL", "mykey", "myid1*"}, {1}, - {"PDEL", "mykey", "myid2*"}, {2}, - {"PDEL", "mykey", "*b"}, {2}, - {"PDEL", "mykey", "*"}, {2}, - {"PDEL", "mykey", "*"}, {0}, - }) + return mc.DoBatch( + Do("SET", "mykey", "myid1a", "POINT", 33, -115).OK(), + Do("SET", "mykey", "myid1b", "POINT", 33, -115).OK(), + Do("SET", "mykey", "myid2a", "POINT", 33, -115).OK(), + Do("SET", "mykey", "myid2b", "POINT", 33, -115).OK(), + Do("SET", "mykey", "myid3a", "POINT", 33, -115).OK(), + Do("SET", "mykey", "myid3b", "POINT", 33, -115).OK(), + Do("SET", "mykey", "myid4a", "POINT", 33, -115).OK(), + Do("SET", "mykey", "myid4b", "POINT", 33, -115).OK(), + Do("PDEL", "mykey").Err("wrong number of arguments for 'pdel' command"), + Do("PDEL", "mykeyNA", "*").Str("0"), + Do("PDEL", "mykey", "myid1a").Str("1"), + Do("PDEL", "mykey", "myid1a").Str("0"), + Do("PDEL", "mykey", "myid1*").Str("1"), + Do("PDEL", "mykey", "myid2*").Str("2"), + Do("PDEL", "mykey", "*b").Str("2"), + Do("PDEL", "mykey", "*").Str("2"), + Do("PDEL", "mykey", "*").Str("0"), + Do("SET", "mykey", "myid1a", "POINT", 33, -115).OK(), + Do("SET", "mykey", "myid1b", "POINT", 33, -115).OK(), + Do("SET", "mykey", "myid2a", "POINT", 33, -115).OK(), + Do("SET", "mykey", "myid2b", "POINT", 33, -115).OK(), + Do("SET", "mykey", "myid3a", "POINT", 33, -115).OK(), + Do("PDEL", "mykey", "*").JSON().OK(), + ) } func keys_WHEREIN_test(mc *mockServer) error { @@ -363,3 +382,14 @@ func keys_WHEREEVAL_test(mc *mockServer) error { {"WITHIN", "mykey", "WHEREEVAL", "return FIELDS.a > tonumber(ARGV[1]) and FIELDS.a ~= tonumber(ARGV[2])", 2, 0.5, 3, "BOUNDS", 32.8, -115.2, 33.2, -114.8}, {`[0 [[myid_a2 {"type":"Point","coordinates":[-115,32.99]} [a 2]] [myid_a1 {"type":"Point","coordinates":[-115,33]} [a 1]]]]`}, }) } + +func keys_TYPE_test(mc *mockServer) error { + return mc.DoBatch( + Do("SET", "mykey", "myid1", "POINT", 33, -115).OK(), + Do("TYPE", "mykey").Str("hash"), + Do("TYPE", "mykey", "hello").Err("wrong number of arguments for 'type' command"), + Do("TYPE", "mykey2").Str("none"), + Do("TYPE", "mykey2").JSON().Err("key not found"), + Do("TYPE", "mykey").JSON().Str(`{"ok":true,"type":"hash"}`), + ) +} diff --git a/tests/mock_io_test.go b/tests/mock_io_test.go index 3f07fd91..ba19ddc6 100644 --- a/tests/mock_io_test.go +++ b/tests/mock_io_test.go @@ -27,7 +27,7 @@ func (cmd *IO) JSON() *IO { cmd.json = true return cmd } -func (cmd *IO) String(s string) *IO { +func (cmd *IO) Str(s string) *IO { cmd.out = s return cmd } @@ -56,7 +56,7 @@ func (cmd *IO) OK() *IO { }) } -func (cmd *IO) Error(msg string) *IO { +func (cmd *IO) Err(msg string) *IO { return cmd.Custom(func(s string) error { if cmd.json { if gjson.Get(s, "ok").Type != gjson.False { From ede1ce026942e87f2b2b4e4a002c4def9f572b1e Mon Sep 17 00:00:00 2001 From: tidwall Date: Fri, 23 Sep 2022 10:42:43 -0700 Subject: [PATCH 04/33] Better GET/DROP tests --- internal/server/crud.go | 137 ++++++++++++++++++++----------------- internal/server/scripts.go | 4 +- internal/server/server.go | 4 +- tests/keys_test.go | 48 ++++++++++--- 4 files changed, 117 insertions(+), 76 deletions(-) diff --git a/internal/server/crud.go b/internal/server/crud.go index 65702cd3..9d07ca73 100644 --- a/internal/server/crud.go +++ b/internal/server/crud.go @@ -107,54 +107,73 @@ func (s *Server) cmdTYPE(msg *Message) (resp.Value, error) { return resp.SimpleStringValue(typ), nil } -func (s *Server) cmdGet(msg *Message) (resp.Value, error) { +// GET key id [WITHFIELDS] [OBJECT|POINT|BOUNDS|(HASH geohash)] +func (s *Server) cmdGET(msg *Message) (resp.Value, error) { start := time.Now() - vs := msg.Args[1:] - var ok bool - var key, id, typ, sprecision string - if vs, key, ok = tokenval(vs); !ok || key == "" { - return NOMessage, errInvalidNumberOfArguments - } - if vs, id, ok = tokenval(vs); !ok || id == "" { - return NOMessage, errInvalidNumberOfArguments + // >> Args + + args := msg.Args + + if len(args) < 3 { + return retrerr(errInvalidNumberOfArguments) } + key, id := args[1], args[2] withfields := false - if _, peek, ok := tokenval(vs); ok && strings.ToLower(peek) == "withfields" { - withfields = true - vs = vs[1:] + kind := "object" + var precision int64 + for i := 3; i < len(args); i++ { + switch strings.ToLower(args[i]) { + case "withfields": + withfields = true + case "object": + kind = "object" + case "point": + kind = "point" + case "bounds": + kind = "bounds" + case "hash": + kind = "hash" + i++ + if i == len(args) { + return retrerr(errInvalidNumberOfArguments) + } + var err error + precision, err = strconv.ParseInt(args[i], 10, 64) + if err != nil || precision < 1 || precision > 12 { + return retrerr(errInvalidArgument(args[i])) + } + default: + return retrerr(errInvalidNumberOfArguments) + } } + // >> Operation + col, _ := s.cols.Get(key) if col == nil { if msg.OutputType == RESP { return resp.NullValue(), nil } - return NOMessage, errKeyNotFound + return retrerr(errKeyNotFound) } o := col.Get(id) - ok = o != nil - if !ok { + if o == nil { if msg.OutputType == RESP { return resp.NullValue(), nil } - return NOMessage, errIDNotFound + return retrerr(errIDNotFound) } + // >> Response + vals := make([]resp.Value, 0, 2) var buf bytes.Buffer if msg.OutputType == JSON { buf.WriteString(`{"ok":true`) } - vs, typ, ok = tokenval(vs) - typ = strings.ToLower(typ) - if !ok { - typ = "object" - } - switch typ { - default: - return NOMessage, errInvalidArgument(typ) + switch kind { case "object": if msg.OutputType == JSON { buf.WriteString(`,"object":`) @@ -183,16 +202,9 @@ func (s *Server) cmdGet(msg *Message) (resp.Value, error) { } } case "hash": - if vs, sprecision, ok = tokenval(vs); !ok || sprecision == "" { - return NOMessage, errInvalidNumberOfArguments - } if msg.OutputType == JSON { buf.WriteString(`,"hash":`) } - precision, err := strconv.ParseInt(sprecision, 10, 64) - if err != nil || precision < 1 || precision > 12 { - return NOMessage, errInvalidArgument(sprecision) - } center := o.Geo().Center() p := geohash.EncodeWithPrecision(center.Y, center.X, uint(precision)) if msg.OutputType == JSON { @@ -219,9 +231,6 @@ func (s *Server) cmdGet(msg *Message) (resp.Value, error) { } } - if len(vs) != 0 { - return NOMessage, errInvalidNumberOfArguments - } if withfields { nfields := o.Fields().Len() if nfields > 0 { @@ -250,20 +259,17 @@ func (s *Server) cmdGet(msg *Message) (resp.Value, error) { } } } - switch msg.OutputType { - case JSON: + if msg.OutputType == JSON { buf.WriteString(`,"elapsed":"` + time.Since(start).String() + "\"}") return resp.StringValue(buf.String()), nil - case RESP: - var oval resp.Value - if withfields { - oval = resp.ArrayValue(vals) - } else { - oval = vals[0] - } - return oval, nil } - return NOMessage, nil + var oval resp.Value + if withfields { + oval = resp.ArrayValue(vals) + } else { + oval = vals[0] + } + return oval, nil } // DEL key id [ERRON404] @@ -405,27 +411,32 @@ func (s *Server) cmdPDEL(msg *Message) (resp.Value, commandDetails, error) { return res, d, nil } -func (s *Server) cmdDrop(msg *Message) (res resp.Value, d commandDetails, err error) { +// DROP key +func (s *Server) cmdDROP(msg *Message) (resp.Value, commandDetails, error) { start := time.Now() - vs := msg.Args[1:] - var ok bool - if vs, d.key, ok = tokenval(vs); !ok || d.key == "" { - err = errInvalidNumberOfArguments - return + + // >> Args + + args := msg.Args + if len(args) != 2 { + return retwerr(errInvalidNumberOfArguments) } - if len(vs) != 0 { - err = errInvalidNumberOfArguments - return - } - col, _ := s.cols.Get(d.key) + key := args[1] + + // >> Operation + + col, _ := s.cols.Get(key) if col != nil { - s.cols.Delete(d.key) - d.updated = true - } else { - d.key = "" // ignore the details - d.updated = false + s.cols.Delete(key) } - s.groupDisconnectCollection(d.key) + s.groupDisconnectCollection(key) + + // >> Response + + var res resp.Value + var d commandDetails + d.key = key + d.updated = col != nil d.command = "drop" d.timestamp = time.Now() switch msg.OutputType { @@ -438,7 +449,7 @@ func (s *Server) cmdDrop(msg *Message) (res resp.Value, d commandDetails, err er res = resp.IntegerValue(0) } } - return + return res, d, nil } func (s *Server) cmdRename(msg *Message) (res resp.Value, d commandDetails, err error) { diff --git a/internal/server/scripts.go b/internal/server/scripts.go index 1c3a5fc5..82fed1aa 100644 --- a/internal/server/scripts.go +++ b/internal/server/scripts.go @@ -600,7 +600,7 @@ func (s *Server) commandInScript(msg *Message) ( case "pdel": res, d, err = s.cmdPDEL(msg) case "drop": - res, d, err = s.cmdDrop(msg) + res, d, err = s.cmdDROP(msg) case "expire": res, d, err = s.cmdEXPIRE(msg) case "rename": @@ -626,7 +626,7 @@ func (s *Server) commandInScript(msg *Message) ( case "bounds": res, err = s.cmdBOUNDS(msg) case "get": - res, err = s.cmdGet(msg) + res, err = s.cmdGET(msg) case "jget": res, err = s.cmdJget(msg) case "jset": diff --git a/internal/server/server.go b/internal/server/server.go index b27ac477..9842791a 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -1024,7 +1024,7 @@ func (s *Server) command(msg *Message, client *Client) ( case "pdel": res, d, err = s.cmdPDEL(msg) case "drop": - res, d, err = s.cmdDrop(msg) + res, d, err = s.cmdDROP(msg) case "flushdb": res, d, err = s.cmdFLUSHDB(msg) case "rename": @@ -1098,7 +1098,7 @@ func (s *Server) command(msg *Message, client *Client) ( case "bounds": res, err = s.cmdBOUNDS(msg) case "get": - res, err = s.cmdGet(msg) + res, err = s.cmdGET(msg) case "jget": res, err = s.cmdJget(msg) case "jset": diff --git a/tests/keys_test.go b/tests/keys_test.go index ed5b3998..ee0d92b5 100644 --- a/tests/keys_test.go +++ b/tests/keys_test.go @@ -75,15 +75,20 @@ func keys_DEL_test(mc *mockServer) error { } func keys_DROP_test(mc *mockServer) error { - return mc.DoBatch([][]interface{}{ - {"SET", "mykey", "myid1", "HASH", "9my5xp7"}, {"OK"}, - {"SET", "mykey", "myid2", "HASH", "9my5xp8"}, {"OK"}, - {"SCAN", "mykey", "COUNT"}, {2}, - {"DROP", "mykey"}, {1}, - {"SCAN", "mykey", "COUNT"}, {0}, - {"DROP", "mykey"}, {0}, - {"SCAN", "mykey", "COUNT"}, {0}, - }) + return mc.DoBatch( + Do("SET", "mykey", "myid1", "HASH", "9my5xp7").OK(), + Do("SET", "mykey", "myid2", "HASH", "9my5xp8").OK(), + Do("SCAN", "mykey", "COUNT").Str("2"), + Do("DROP").Err("wrong number of arguments for 'drop' command"), + Do("DROP", "mykey", "arg3").Err("wrong number of arguments for 'drop' command"), + Do("DROP", "mykey").Str("1"), + Do("SCAN", "mykey", "COUNT").Str("0"), + Do("DROP", "mykey").Str("0"), + Do("SCAN", "mykey", "COUNT").Str("0"), + Do("SET", "mykey", "myid1", "HASH", "9my5xp7").Str("OK"), + Do("DROP", "mykey").JSON().OK(), + Do("DROP", "mykey").JSON().OK(), + ) } func keys_RENAME_test(mc *mockServer) error { return mc.DoBatch([][]interface{}{ @@ -157,6 +162,31 @@ func keys_GET_test(mc *mockServer) error { Do("GET", "mykey", "myid").Str("value2"), Do("DEL", "mykey", "myid").Str("1"), Do("GET", "mykey", "myid").Str(""), + Do("GET", "mykey").Err("wrong number of arguments for 'get' command"), + Do("GET", "mykey", "myid", "hash").Err("wrong number of arguments for 'get' command"), + Do("GET", "mykey", "myid", "hash", "0").Err("invalid argument '0'"), + Do("GET", "mykey", "myid", "hash", "-1").Err("invalid argument '-1'"), + Do("GET", "mykey", "myid", "hash", "13").Err("invalid argument '13'"), + Do("SET", "mykey", "myid", "field", "hello", "world", "field", "hiya", 55, "point", 33, -112).OK(), + Do("GET", "mykey", "myid", "hash", "1").Str("9"), + Do("GET", "mykey", "myid", "point").Str("[33 -112]"), + Do("GET", "mykey", "myid", "bounds").Str("[[33 -112] [33 -112]]"), + Do("GET", "mykey", "myid", "object").Str(`{"type":"Point","coordinates":[-112,33]}`), + Do("GET", "mykey", "myid", "object").Str(`{"type":"Point","coordinates":[-112,33]}`), + Do("GET", "mykey", "myid", "withfields", "point").Str(`[[33 -112] [hello world hiya 55]]`), + Do("GET", "mykey", "myid", "joint").Err("wrong number of arguments for 'get' command"), + Do("GET", "mykey2", "myid").Str(""), + Do("GET", "mykey2", "myid").JSON().Err("key not found"), + Do("GET", "mykey", "myid2").Str(""), + Do("GET", "mykey", "myid2").JSON().Err("id not found"), + Do("GET", "mykey", "myid", "point").JSON().Str(`{"ok":true,"point":{"lat":33,"lon":-112}}`), + Do("GET", "mykey", "myid", "object").JSON().Str(`{"ok":true,"object":{"type":"Point","coordinates":[-112,33]}}`), + Do("GET", "mykey", "myid", "hash", "1").JSON().Str(`{"ok":true,"hash":"9"}`), + Do("GET", "mykey", "myid", "bounds").JSON().Str(`{"ok":true,"bounds":{"sw":{"lat":33,"lon":-112},"ne":{"lat":33,"lon":-112}}}`), + Do("SET", "mykey", "myid2", "point", 33, -112, 55).OK(), + Do("GET", "mykey", "myid2", "point").Str("[33 -112 55]"), + Do("GET", "mykey", "myid2", "point").JSON().Str(`{"ok":true,"point":{"lat":33,"lon":-112,"z":55}}`), + Do("GET", "mykey", "myid", "withfields").JSON().Str(`{"ok":true,"object":{"type":"Point","coordinates":[-112,33]},"fields":{"hello":"world","hiya":55}}`), ) } func keys_KEYS_test(mc *mockServer) error { From 960c860b3a793da9fbe99d598809e5069a4dd673 Mon Sep 17 00:00:00 2001 From: tidwall Date: Fri, 23 Sep 2022 11:18:01 -0700 Subject: [PATCH 05/33] Better RENAME/RENAMENX tests --- internal/server/crud.go | 95 ++++++++++++++++++++++---------------- internal/server/scripts.go | 4 +- internal/server/server.go | 4 +- tests/keys_test.go | 74 ++++++++++++++++------------- 4 files changed, 103 insertions(+), 74 deletions(-) diff --git a/internal/server/crud.go b/internal/server/crud.go index 9d07ca73..651b5c06 100644 --- a/internal/server/crud.go +++ b/internal/server/crud.go @@ -244,10 +244,11 @@ func (s *Server) cmdGET(msg *Message) (resp.Value, error) { if i > 0 { buf.WriteString(`,`) } - buf.WriteString(jsonString(f.Name()) + ":" + f.Value().JSON()) + buf.WriteString(jsonString(f.Name()) + ":" + + f.Value().JSON()) } else { - fvals = append(fvals, - resp.StringValue(f.Name()), resp.StringValue(f.Value().Data())) + fvals = append(fvals, resp.StringValue(f.Name()), + resp.StringValue(f.Value().Data())) } i++ return true @@ -441,7 +442,8 @@ func (s *Server) cmdDROP(msg *Message) (resp.Value, commandDetails, error) { d.timestamp = time.Now() switch msg.OutputType { case JSON: - res = resp.StringValue(`{"ok":true,"elapsed":"` + time.Since(start).String() + "\"}") + res = resp.StringValue(`{"ok":true,"elapsed":"` + + time.Since(start).String() + "\"}") case RESP: if d.updated { res = resp.IntegerValue(1) @@ -452,54 +454,67 @@ func (s *Server) cmdDROP(msg *Message) (resp.Value, commandDetails, error) { return res, d, nil } -func (s *Server) cmdRename(msg *Message) (res resp.Value, d commandDetails, err error) { - nx := msg.Command() == "renamenx" +// RENAME key newkey +// RENAMENX key newkey +func (s *Server) cmdRENAME(msg *Message) (resp.Value, commandDetails, error) { start := time.Now() - vs := msg.Args[1:] - var ok bool - if vs, d.key, ok = tokenval(vs); !ok || d.key == "" { - err = errInvalidNumberOfArguments - return + + // >> Args + + args := msg.Args + if len(args) != 3 { + return retwerr(errInvalidNumberOfArguments) } - if vs, d.newKey, ok = tokenval(vs); !ok || d.newKey == "" { - err = errInvalidNumberOfArguments - return - } - if len(vs) != 0 { - err = errInvalidNumberOfArguments - return - } - col, _ := s.cols.Get(d.key) + nx := strings.ToLower(args[0]) == "renamenx" + key := args[1] + newKey := args[2] + + // >> Operation + + col, _ := s.cols.Get(key) if col == nil { - err = errKeyNotFound - return + return retwerr(errKeyNotFound) } + var ierr error s.hooks.Ascend(nil, func(v interface{}) bool { h := v.(*Hook) - if h.Key == d.key || h.Key == d.newKey { - err = errKeyHasHooksSet + if h.Key == key || h.Key == newKey { + ierr = errKeyHasHooksSet return false } return true }) - d.command = "rename" - newCol, _ := s.cols.Get(d.newKey) + if ierr != nil { + return retwerr(ierr) + } + var updated bool + newCol, _ := s.cols.Get(newKey) if newCol == nil { - d.updated = true - } else if nx { - d.updated = false - } else { - s.cols.Delete(d.newKey) - d.updated = true + updated = true + } else if !nx { + s.cols.Delete(newKey) + updated = true } - if d.updated { - s.cols.Delete(d.key) - s.cols.Set(d.newKey, col) + if updated { + s.cols.Delete(key) + s.cols.Set(newKey, col) } + + // >> Response + + var d commandDetails + var res resp.Value + + d.command = "rename" + d.key = key + d.newKey = newKey + d.updated = updated d.timestamp = time.Now() + switch msg.OutputType { case JSON: - res = resp.StringValue(`{"ok":true,"elapsed":"` + time.Since(start).String() + "\"}") + res = resp.StringValue(`{"ok":true,"elapsed":"` + + time.Since(start).String() + "\"}") case RESP: if !nx { res = resp.SimpleStringValue("OK") @@ -509,7 +524,7 @@ func (s *Server) cmdRename(msg *Message) (res resp.Value, d commandDetails, err res = resp.IntegerValue(0) } } - return + return res, d, nil } func (s *Server) cmdFLUSHDB(msg *Message) (res resp.Value, d commandDetails, err error) { @@ -535,7 +550,8 @@ func (s *Server) cmdFLUSHDB(msg *Message) (res resp.Value, d commandDetails, err d.timestamp = time.Now() switch msg.OutputType { case JSON: - res = resp.StringValue(`{"ok":true,"elapsed":"` + time.Since(start).String() + "\"}") + res = resp.StringValue(`{"ok":true,"elapsed":"` + + time.Since(start).String() + "\"}") case RESP: res = resp.SimpleStringValue("OK") } @@ -543,7 +559,8 @@ func (s *Server) cmdFLUSHDB(msg *Message) (res resp.Value, d commandDetails, err } // SET key id [FIELD name value ...] [EX seconds] [NX|XX] -// (OBJECT geojson)|(POINT lat lon z)|(BOUNDS minlat minlon maxlat maxlon)|(HASH geohash)|(STRING value) +// (OBJECT geojson)|(POINT lat lon z)|(BOUNDS minlat minlon maxlat maxlon)| +// (HASH geohash)|(STRING value) func (s *Server) cmdSET(msg *Message) (resp.Value, commandDetails, error) { start := time.Now() if s.config.maxMemory() > 0 && s.outOfMemory.on() { diff --git a/internal/server/scripts.go b/internal/server/scripts.go index 82fed1aa..84f9a713 100644 --- a/internal/server/scripts.go +++ b/internal/server/scripts.go @@ -604,9 +604,9 @@ func (s *Server) commandInScript(msg *Message) ( case "expire": res, d, err = s.cmdEXPIRE(msg) case "rename": - res, d, err = s.cmdRename(msg) + res, d, err = s.cmdRENAME(msg) case "renamenx": - res, d, err = s.cmdRename(msg) + res, d, err = s.cmdRENAME(msg) case "persist": res, d, err = s.cmdPERSIST(msg) case "ttl": diff --git a/internal/server/server.go b/internal/server/server.go index 9842791a..9af3f5f2 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -1028,9 +1028,9 @@ func (s *Server) command(msg *Message, client *Client) ( case "flushdb": res, d, err = s.cmdFLUSHDB(msg) case "rename": - res, d, err = s.cmdRename(msg) + res, d, err = s.cmdRENAME(msg) case "renamenx": - res, d, err = s.cmdRename(msg) + res, d, err = s.cmdRENAME(msg) case "sethook": res, d, err = s.cmdSetHook(msg) case "delhook": diff --git a/tests/keys_test.go b/tests/keys_test.go index ee0d92b5..1d3fefcf 100644 --- a/tests/keys_test.go +++ b/tests/keys_test.go @@ -85,44 +85,56 @@ func keys_DROP_test(mc *mockServer) error { Do("SCAN", "mykey", "COUNT").Str("0"), Do("DROP", "mykey").Str("0"), Do("SCAN", "mykey", "COUNT").Str("0"), - Do("SET", "mykey", "myid1", "HASH", "9my5xp7").Str("OK"), + Do("SET", "mykey", "myid1", "HASH", "9my5xp7").OK(), Do("DROP", "mykey").JSON().OK(), Do("DROP", "mykey").JSON().OK(), ) } func keys_RENAME_test(mc *mockServer) error { - return mc.DoBatch([][]interface{}{ - {"SET", "mykey", "myid1", "HASH", "9my5xp7"}, {"OK"}, - {"SET", "mykey", "myid2", "HASH", "9my5xp8"}, {"OK"}, - {"SCAN", "mykey", "COUNT"}, {2}, - {"RENAME", "mykey", "mynewkey"}, {"OK"}, - {"SCAN", "mykey", "COUNT"}, {0}, - {"SCAN", "mynewkey", "COUNT"}, {2}, - {"SET", "mykey", "myid3", "HASH", "9my5xp7"}, {"OK"}, - {"RENAME", "mykey", "mynewkey"}, {"OK"}, - {"SCAN", "mykey", "COUNT"}, {0}, - {"SCAN", "mynewkey", "COUNT"}, {1}, - {"RENAME", "foo", "mynewkey"}, {"ERR key not found"}, - {"SCAN", "mynewkey", "COUNT"}, {1}, - }) + return mc.DoBatch( + Do("SET", "mykey", "myid1", "HASH", "9my5xp7").OK(), + Do("SET", "mykey", "myid2", "HASH", "9my5xp8").OK(), + Do("SCAN", "mykey", "COUNT").Str("2"), + Do("RENAME", "foo", "mynewkey", "arg3").Err("wrong number of arguments for 'rename' command"), + Do("RENAME", "mykey", "mynewkey").OK(), + Do("SCAN", "mykey", "COUNT").Str("0"), + Do("SCAN", "mynewkey", "COUNT").Str("2"), + Do("SET", "mykey", "myid3", "HASH", "9my5xp7").OK(), + Do("RENAME", "mykey", "mynewkey").OK(), + Do("SCAN", "mykey", "COUNT").Str("0"), + Do("SCAN", "mynewkey", "COUNT").Str("1"), + Do("RENAME", "foo", "mynewkey").Err("key not found"), + Do("SCAN", "mynewkey", "COUNT").Str("1"), + Do("SETCHAN", "mychan", "INTERSECTS", "mynewkey", "BOUNDS", 10, 10, 20, 20).Str("1"), + Do("RENAME", "mynewkey", "foo2").Err("key has hooks set"), + Do("RENAMENX", "mynewkey", "foo2").Err("key has hooks set"), + Do("SET", "mykey", "myid1", "HASH", "9my5xp7").OK(), + Do("RENAME", "mykey", "foo2").OK(), + Do("RENAMENX", "foo2", "foo3").Str("1"), + Do("RENAMENX", "foo2", "foo3").Err("key not found"), + Do("RENAME", "foo2", "foo3").JSON().Err("key not found"), + Do("SET", "mykey", "myid1", "HASH", "9my5xp7").OK(), + Do("RENAMENX", "mykey", "foo3").Str("0"), + Do("RENAME", "foo3", "foo4").JSON().OK(), + ) } func keys_RENAMENX_test(mc *mockServer) error { - return mc.DoBatch([][]interface{}{ - {"SET", "mykey", "myid1", "HASH", "9my5xp7"}, {"OK"}, - {"SET", "mykey", "myid2", "HASH", "9my5xp8"}, {"OK"}, - {"SCAN", "mykey", "COUNT"}, {2}, - {"RENAMENX", "mykey", "mynewkey"}, {1}, - {"SCAN", "mykey", "COUNT"}, {0}, - {"DROP", "mykey"}, {0}, - {"SCAN", "mykey", "COUNT"}, {0}, - {"SCAN", "mynewkey", "COUNT"}, {2}, - {"SET", "mykey", "myid3", "HASH", "9my5xp7"}, {"OK"}, - {"RENAMENX", "mykey", "mynewkey"}, {0}, - {"SCAN", "mykey", "COUNT"}, {1}, - {"SCAN", "mynewkey", "COUNT"}, {2}, - {"RENAMENX", "foo", "mynewkey"}, {"ERR key not found"}, - {"SCAN", "mynewkey", "COUNT"}, {2}, - }) + return mc.DoBatch( + Do("SET", "mykey", "myid1", "HASH", "9my5xp7").OK(), + Do("SET", "mykey", "myid2", "HASH", "9my5xp8").OK(), + Do("SCAN", "mykey", "COUNT").Str("2"), + Do("RENAMENX", "mykey", "mynewkey").Str("1"), + Do("SCAN", "mykey", "COUNT").Str("0"), + Do("DROP", "mykey").Str("0"), + Do("SCAN", "mykey", "COUNT").Str("0"), + Do("SCAN", "mynewkey", "COUNT").Str("2"), + Do("SET", "mykey", "myid3", "HASH", "9my5xp7").OK(), + Do("RENAMENX", "mykey", "mynewkey").Str("0"), + Do("SCAN", "mykey", "COUNT").Str("1"), + Do("SCAN", "mynewkey", "COUNT").Str("2"), + Do("RENAMENX", "foo", "mynewkey").Str("ERR key not found"), + Do("SCAN", "mynewkey", "COUNT").Str("2"), + ) } func keys_EXPIRE_test(mc *mockServer) error { return mc.DoBatch([][]interface{}{ From d7ad01e5934b5f59b693ce8584c2b5198cebf353 Mon Sep 17 00:00:00 2001 From: tidwall Date: Fri, 23 Sep 2022 11:40:48 -0700 Subject: [PATCH 06/33] Better FLUSHDB/EXPIRES tests --- internal/server/crud.go | 45 +++++++++++++++++++++++++---------- tests/keys_test.go | 52 ++++++++++++++++++++++++++++++++++------- tests/mock_io_test.go | 17 +++++++++++--- 3 files changed, 91 insertions(+), 23 deletions(-) diff --git a/internal/server/crud.go b/internal/server/crud.go index 651b5c06..4cb297a9 100644 --- a/internal/server/crud.go +++ b/internal/server/crud.go @@ -527,14 +527,20 @@ func (s *Server) cmdRENAME(msg *Message) (resp.Value, commandDetails, error) { return res, d, nil } -func (s *Server) cmdFLUSHDB(msg *Message) (res resp.Value, d commandDetails, err error) { +// FLUSHDB +func (s *Server) cmdFLUSHDB(msg *Message) (resp.Value, commandDetails, error) { start := time.Now() - vs := msg.Args[1:] - if len(vs) != 0 { - err = errInvalidNumberOfArguments - return + + // >> Args + + args := msg.Args + + if len(args) != 1 { + return retwerr(errInvalidNumberOfArguments) } + // >> Operation + // clear the entire database s.cols.Clear() s.groupHooks.Clear() @@ -545,17 +551,21 @@ func (s *Server) cmdFLUSHDB(msg *Message) (res resp.Value, d commandDetails, err s.hookTree.Clear() s.hookCross.Clear() + // >> Response + + var d commandDetails d.command = "flushdb" d.updated = true d.timestamp = time.Now() - switch msg.OutputType { - case JSON: + + var res resp.Value + if msg.OutputType == JSON { res = resp.StringValue(`{"ok":true,"elapsed":"` + time.Since(start).String() + "\"}") - case RESP: + } else { res = resp.SimpleStringValue("OK") } - return + return res, d, nil } // SET key id [FIELD name value ...] [EX seconds] [NX|XX] @@ -857,6 +867,9 @@ func (s *Server) cmdFSET(msg *Message) (resp.Value, commandDetails, error) { // EXPIRE key id seconds func (s *Server) cmdEXPIRE(msg *Message) (resp.Value, commandDetails, error) { start := time.Now() + + // >> Args + args := msg.Args if len(args) != 4 { return retwerr(errInvalidNumberOfArguments) @@ -866,12 +879,16 @@ func (s *Server) cmdEXPIRE(msg *Message) (resp.Value, commandDetails, error) { if err != nil { return retwerr(errInvalidArgument(svalue)) } + + // >> Operation + var ok bool var obj *object.Object col, _ := s.cols.Get(key) if col != nil { - // replace the expiration by getting the old objec - ex := time.Now().Add(time.Duration(float64(time.Second) * value)).UnixNano() + // replace the expiration by getting the old object + ex := time.Now().Add( + time.Duration(float64(time.Second) * value)).UnixNano() o := col.Get(id) ok = o != nil if ok { @@ -879,6 +896,9 @@ func (s *Server) cmdEXPIRE(msg *Message) (resp.Value, commandDetails, error) { col.Set(obj) } } + + // >> Response + var d commandDetails if ok { d.key = key @@ -950,7 +970,8 @@ func (s *Server) cmdPERSIST(msg *Message) (resp.Value, commandDetails, error) { switch msg.OutputType { case JSON: - res = resp.SimpleStringValue(`{"ok":true,"elapsed":"` + time.Since(start).String() + "\"}") + res = resp.SimpleStringValue(`{"ok":true,"elapsed":"` + + time.Since(start).String() + "\"}") case RESP: if cleared { res = resp.IntegerValue(1) diff --git a/tests/keys_test.go b/tests/keys_test.go index 1d3fefcf..3d4248cd 100644 --- a/tests/keys_test.go +++ b/tests/keys_test.go @@ -31,6 +31,7 @@ func subTestKeys(t *testing.T, mc *mockServer) { runStep(t, mc, "WHEREIN", keys_WHEREIN_test) runStep(t, mc, "WHEREEVAL", keys_WHEREEVAL_test) runStep(t, mc, "TYPE", keys_TYPE_test) + runStep(t, mc, "FLUSHDB", keys_FLUSHDB_test) } func keys_BOUNDS_test(mc *mockServer) error { @@ -137,14 +138,26 @@ func keys_RENAMENX_test(mc *mockServer) error { ) } func keys_EXPIRE_test(mc *mockServer) error { - return mc.DoBatch([][]interface{}{ - {"SET", "mykey", "myid", "STRING", "value"}, {"OK"}, - {"EXPIRE", "mykey", "myid", 1}, {1}, - {time.Second / 4}, {}, // sleep - {"GET", "mykey", "myid"}, {"value"}, - {time.Second}, {}, // sleep - {"GET", "mykey", "myid"}, {nil}, - }) + return mc.DoBatch( + Do("SET", "mykey", "myid", "STRING", "value").OK(), + Do("EXPIRE", "mykey", "myid").Err("wrong number of arguments for 'expire' command"), + Do("EXPIRE", "mykey", "myid", "y").Err("invalid argument 'y'"), + Do("EXPIRE", "mykey", "myid", 1).Str("1"), + Do("EXPIRE", "mykey", "myid", 1).JSON().OK(), + Sleep(time.Second/4), + Do("GET", "mykey", "myid").Str("value"), + Sleep(time.Second), + Do("GET", "mykey", "myid").Str(""), + Do("EXPIRE", "mykey", "myid", 1).JSON().Err("key not found"), + Do("SET", "mykey", "myid1", "STRING", "value1").OK(), + Do("SET", "mykey", "myid2", "STRING", "value2").OK(), + Do("EXPIRE", "mykey", "myid1", 1).Str("1"), + Sleep(time.Second/4), + Do("GET", "mykey", "myid1").Str("value1"), + Sleep(time.Second), + Do("EXPIRE", "mykey", "myid1", 1).Str("0"), + Do("EXPIRE", "mykey", "myid1", 1).JSON().Err("id not found"), + ) } func keys_FSET_test(mc *mockServer) error { return mc.DoBatch([][]interface{}{ @@ -435,3 +448,26 @@ func keys_TYPE_test(mc *mockServer) error { Do("TYPE", "mykey").JSON().Str(`{"ok":true,"type":"hash"}`), ) } + +func keys_FLUSHDB_test(mc *mockServer) error { + return mc.DoBatch( + Do("SET", "mykey1", "myid1", "POINT", 33, -115).OK(), + Do("SET", "mykey2", "myid1", "POINT", 33, -115).OK(), + Do("SETCHAN", "mychan", "INTERSECTS", "mykey1", "BOUNDS", 10, 10, 10, 10).Str("1"), + Do("KEYS", "*").Str("[mykey1 mykey2]"), + Do("CHANS", "*").JSON().Custom(func(s string) error { + if gjson.Get(s, "chans.#").Int() != 1 { + return fmt.Errorf("expected '%d', got '%d'", 1, gjson.Get(s, "chans.#").Int()) + } + return nil + }), + Do("FLUSHDB", "arg2").Err("wrong number of arguments for 'flushdb' command"), + Do("FLUSHDB").OK(), + Do("KEYS", "*").Str("[]"), + Do("CHANS", "*").Str("[]"), + Do("SET", "mykey1", "myid1", "POINT", 33, -115).OK(), + Do("SET", "mykey2", "myid1", "POINT", 33, -115).OK(), + Do("SETCHAN", "mychan", "INTERSECTS", "mykey1", "BOUNDS", 10, 10, 10, 10).Str("1"), + Do("FLUSHDB").JSON().OK(), + ) +} diff --git a/tests/mock_io_test.go b/tests/mock_io_test.go index ba19ddc6..72a24e6b 100644 --- a/tests/mock_io_test.go +++ b/tests/mock_io_test.go @@ -10,14 +10,17 @@ import ( "path/filepath" "runtime" "strings" + "time" "github.com/tidwall/gjson" ) type IO struct { - args []any - json bool - out any + args []any + json bool + out any + sleep bool + dur time.Duration } func Do(args ...any) *IO { @@ -76,6 +79,10 @@ func (cmd *IO) Err(msg string) *IO { }) } +func Sleep(duration time.Duration) *IO { + return &IO{sleep: true, dur: duration} +} + type ioVisitor struct { fset *token.FileSet ln int @@ -222,6 +229,10 @@ func (cmd *IO) deepError(index int, err error) error { } func (mc *mockServer) doIOTest(index int, cmd *IO) error { + if cmd.sleep { + time.Sleep(cmd.dur) + return nil + } // switch json mode if desired if cmd.json { if !mc.ioJSON { From 7fa2dc4419bef0387da793a45c5b6c8037d3df78 Mon Sep 17 00:00:00 2001 From: tidwall Date: Fri, 23 Sep 2022 12:42:39 -0700 Subject: [PATCH 07/33] Better FSET tests Execute oom check immediately after setting maxmemory --- internal/server/config.go | 3 +++ internal/server/server.go | 38 +++++++++++++++++--------------- tests/keys_test.go | 46 ++++++++++++++++++++++++--------------- 3 files changed, 51 insertions(+), 36 deletions(-) diff --git a/internal/server/config.go b/internal/server/config.go index b29cfc1d..7ec0bcad 100644 --- a/internal/server/config.go +++ b/internal/server/config.go @@ -427,6 +427,9 @@ func (s *Server) cmdConfigSet(msg *Message) (res resp.Value, err error) { if err := s.config.setProperty(name, value, false); err != nil { return NOMessage, err } + if name == MaxMemory { + s.checkOutOfMemory() + } return OKMessage(msg, start), nil } func (s *Server) cmdConfigRewrite(msg *Message) (res resp.Value, err error) { diff --git a/internal/server/server.go b/internal/server/server.go index 9af3f5f2..1730362e 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -623,28 +623,30 @@ func (s *Server) watchAutoGC() { } } +func (s *Server) checkOutOfMemory() { + if s.stopServer.on() { + return + } + oom := s.outOfMemory.on() + var mem runtime.MemStats + if s.config.maxMemory() == 0 { + if oom { + s.outOfMemory.set(false) + } + return + } + if oom { + runtime.GC() + } + runtime.ReadMemStats(&mem) + s.outOfMemory.set(int(mem.HeapAlloc) > s.config.maxMemory()) +} + func (s *Server) watchOutOfMemory() { t := time.NewTicker(time.Second * 2) defer t.Stop() - var mem runtime.MemStats for range t.C { - func() { - if s.stopServer.on() { - return - } - oom := s.outOfMemory.on() - if s.config.maxMemory() == 0 { - if oom { - s.outOfMemory.set(false) - } - return - } - if oom { - runtime.GC() - } - runtime.ReadMemStats(&mem) - s.outOfMemory.set(int(mem.HeapAlloc) > s.config.maxMemory()) - }() + s.checkOutOfMemory() } } diff --git a/tests/keys_test.go b/tests/keys_test.go index 3d4248cd..b2932816 100644 --- a/tests/keys_test.go +++ b/tests/keys_test.go @@ -160,24 +160,34 @@ func keys_EXPIRE_test(mc *mockServer) error { ) } func keys_FSET_test(mc *mockServer) error { - return mc.DoBatch([][]interface{}{ - {"SET", "mykey", "myid", "HASH", "9my5xp7"}, {"OK"}, - {"GET", "mykey", "myid", "WITHFIELDS", "HASH", 7}, {"[9my5xp7]"}, - {"FSET", "mykey", "myid", "f1", 105.6}, {1}, - {"GET", "mykey", "myid", "WITHFIELDS", "HASH", 7}, {"[9my5xp7 [f1 105.6]]"}, - {"FSET", "mykey", "myid", "f1", 1.1, "f2", 2.2}, {2}, - {"GET", "mykey", "myid", "WITHFIELDS", "HASH", 7}, {"[9my5xp7 [f1 1.1 f2 2.2]]"}, - {"FSET", "mykey", "myid", "f1", 1.1, "f2", 22.22}, {1}, - {"GET", "mykey", "myid", "WITHFIELDS", "HASH", 7}, {"[9my5xp7 [f1 1.1 f2 22.22]]"}, - {"FSET", "mykey", "myid", "f1", 0}, {1}, - {"GET", "mykey", "myid", "WITHFIELDS", "HASH", 7}, {"[9my5xp7 [f2 22.22]]"}, - {"FSET", "mykey", "myid", "f2", 0}, {1}, - {"GET", "mykey", "myid", "WITHFIELDS", "HASH", 7}, {"[9my5xp7]"}, - {"FSET", "mykey", "myid2", "xx", "f1", 1.1, "f2", 2.2}, {0}, - {"GET", "mykey", "myid2"}, {nil}, - {"DEL", "mykey", "myid"}, {"1"}, - {"GET", "mykey", "myid"}, {nil}, - }) + return mc.DoBatch( + Do("SET", "mykey", "myid", "HASH", "9my5xp7").OK(), + Do("GET", "mykey", "myid", "WITHFIELDS", "HASH", 7).Str("[9my5xp7]"), + Do("FSET", "mykey", "myid", "f1", 105.6).Str("1"), + Do("GET", "mykey", "myid", "WITHFIELDS", "HASH", 7).Str("[9my5xp7 [f1 105.6]]"), + Do("FSET", "mykey", "myid", "f1", 1.1, "f2", 2.2).Str("2"), + Do("GET", "mykey", "myid", "WITHFIELDS", "HASH", 7).Str("[9my5xp7 [f1 1.1 f2 2.2]]"), + Do("FSET", "mykey", "myid", "f1", 1.1, "f2", 22.22).Str("1"), + Do("GET", "mykey", "myid", "WITHFIELDS", "HASH", 7).Str("[9my5xp7 [f1 1.1 f2 22.22]]"), + Do("FSET", "mykey", "myid", "f1", 0).Str("1"), + Do("GET", "mykey", "myid", "WITHFIELDS", "HASH", 7).Str("[9my5xp7 [f2 22.22]]"), + Do("FSET", "mykey", "myid", "f2", 0).Str("1"), + Do("GET", "mykey", "myid", "WITHFIELDS", "HASH", 7).Str("[9my5xp7]"), + Do("FSET", "mykey", "myid2", "xx", "f1", 1.1, "f2", 2.2).Str("0"), + Do("GET", "mykey", "myid2").Str(""), + Do("DEL", "mykey", "myid").Str("1"), + Do("GET", "mykey", "myid").Str(""), + Do("SET", "mykey", "myid", "HASH", "9my5xp7").OK(), + Do("CONFIG", "SET", "maxmemory", "1").OK(), + Do("FSET", "mykey", "myid", "xx", "f1", 1.1, "f2", 2.2).Err(`OOM command not allowed when used memory > 'maxmemory'`), + Do("CONFIG", "SET", "maxmemory", "0").OK(), + Do("FSET", "mykey", "myid", "xx").Err("wrong number of arguments for 'fset' command"), + Do("FSET", "mykey", "myid", "f1", "a", "f2").Err("wrong number of arguments for 'fset' command"), + Do("FSET", "mykey", "myid", "z", "a").Err("invalid argument 'z'"), + Do("FSET", "mykey2", "myid", "a", "b").Err("key not found"), + Do("FSET", "mykey", "myid2", "a", "b").Err("id not found"), + Do("FSET", "mykey", "myid", "f2", 0).JSON().OK(), + ) } func keys_GET_test(mc *mockServer) error { return mc.DoBatch( From 295a9c45a84add5fac64ae583024d4f97d7c42e6 Mon Sep 17 00:00:00 2001 From: tidwall Date: Fri, 23 Sep 2022 15:29:46 -0700 Subject: [PATCH 08/33] Better SET/PERSIST/TTL/STATS tests --- internal/server/crud.go | 100 ++++++++--------- internal/server/scripts.go | 2 +- internal/server/server.go | 2 +- internal/server/stats.go | 42 ++++---- tests/keys_test.go | 215 +++++++++++++++++++++++-------------- tests/mock_io_test.go | 6 +- 6 files changed, 206 insertions(+), 161 deletions(-) diff --git a/internal/server/crud.go b/internal/server/crud.go index 4cb297a9..ced46fa9 100644 --- a/internal/server/crud.go +++ b/internal/server/crud.go @@ -2,7 +2,7 @@ package server import ( "bytes" - "errors" + "math" "strconv" "strings" "time" @@ -706,23 +706,22 @@ func (s *Server) cmdSET(msg *Message) (resp.Value, commandDetails, error) { return retwerr(errInvalidArgument(args[i])) } } + if oobj == nil { + return retwerr(errInvalidNumberOfArguments) + } // >> Operation nada := func() (resp.Value, commandDetails, error) { // exclude operation due to 'xx' or 'nx' match - switch msg.OutputType { - default: - case JSON: + if msg.OutputType == JSON { if nx { return retwerr(errIDAlreadyExists) } else { return retwerr(errIDNotFound) } - case RESP: - return resp.NullValue(), commandDetails{}, nil } - return retwerr(errors.New("nada unknown output")) + return resp.NullValue(), commandDetails{}, nil } col, ok := s.cols.Get(key) @@ -931,11 +930,17 @@ func (s *Server) cmdEXPIRE(msg *Message) (resp.Value, commandDetails, error) { // PERSIST key id func (s *Server) cmdPERSIST(msg *Message) (resp.Value, commandDetails, error) { start := time.Now() + + // >> Args + args := msg.Args if len(args) != 3 { return retwerr(errInvalidNumberOfArguments) } key, id := args[1], args[2] + + // >> Operation + col, _ := s.cols.Get(key) if col == nil { if msg.OutputType == RESP { @@ -959,6 +964,8 @@ func (s *Server) cmdPERSIST(msg *Message) (resp.Value, commandDetails, error) { cleared = true } + // >> Response + var res resp.Value var d commandDetails @@ -985,62 +992,47 @@ func (s *Server) cmdPERSIST(msg *Message) (resp.Value, commandDetails, error) { // TTL key id func (s *Server) cmdTTL(msg *Message) (resp.Value, error) { start := time.Now() + + // >> Args + args := msg.Args if len(args) != 3 { return retrerr(errInvalidNumberOfArguments) } key, id := args[1], args[2] - var v float64 - var ok bool - var ok2 bool + + // >> Operation + col, _ := s.cols.Get(key) - if col != nil { - o := col.Get(id) - ok = o != nil - if ok { - if o.Expires() != 0 { - now := start.UnixNano() - if now > o.Expires() { - ok2 = false - } else { - v = float64(o.Expires()-now) / float64(time.Second) - if v < 0 { - v = 0 - } - ok2 = true - } - } + if col == nil { + if msg.OutputType == JSON { + return retrerr(errKeyNotFound) } + return resp.IntegerValue(-2), nil } - var res resp.Value - switch msg.OutputType { - case JSON: - if ok { - var ttl string - if ok2 { - ttl = strconv.FormatFloat(v, 'f', -1, 64) - } else { - ttl = "-1" - } - res = resp.SimpleStringValue( - `{"ok":true,"ttl":` + ttl + `,"elapsed":"` + - time.Since(start).String() + "\"}") - } else { - if col == nil { - return retrerr(errKeyNotFound) - } + + o := col.Get(id) + if o == nil { + if msg.OutputType == JSON { return retrerr(errIDNotFound) } - case RESP: - if ok { - if ok2 { - res = resp.IntegerValue(int(v)) - } else { - res = resp.IntegerValue(-1) - } - } else { - res = resp.IntegerValue(-2) - } + return resp.IntegerValue(-2), nil } - return res, nil + + var ttl float64 + if o.Expires() == 0 { + ttl = -1 + } else { + now := start.UnixNano() + ttl = math.Max(float64(o.Expires()-now)/float64(time.Second), 0) + } + + // >> Response + + if msg.OutputType == JSON { + return resp.SimpleStringValue( + `{"ok":true,"ttl":` + strconv.Itoa(int(ttl)) + `,"elapsed":"` + + time.Since(start).String() + "\"}"), nil + } + return resp.IntegerValue(int(ttl)), nil } diff --git a/internal/server/scripts.go b/internal/server/scripts.go index 84f9a713..d1948d21 100644 --- a/internal/server/scripts.go +++ b/internal/server/scripts.go @@ -612,7 +612,7 @@ func (s *Server) commandInScript(msg *Message) ( case "ttl": res, err = s.cmdTTL(msg) case "stats": - res, err = s.cmdStats(msg) + res, err = s.cmdSTATS(msg) case "scan": res, err = s.cmdScan(msg) case "nearby": diff --git a/internal/server/server.go b/internal/server/server.go index 1730362e..7b1a2781 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -1080,7 +1080,7 @@ func (s *Server) command(msg *Message, client *Client) ( case "readonly": res, err = s.cmdReadOnly(msg) case "stats": - res, err = s.cmdStats(msg) + res, err = s.cmdSTATS(msg) case "server": res, err = s.cmdServer(msg) case "healthz": diff --git a/internal/server/stats.go b/internal/server/stats.go index 87238561..4c8dcc4b 100644 --- a/internal/server/stats.go +++ b/internal/server/stats.go @@ -45,22 +45,23 @@ func readMemStats() runtime.MemStats { return ms } -func (s *Server) cmdStats(msg *Message) (res resp.Value, err error) { +// STATS key [key...] +func (s *Server) cmdSTATS(msg *Message) (resp.Value, error) { start := time.Now() - vs := msg.Args[1:] - var ms = []map[string]interface{}{} - if len(vs) == 0 { - return NOMessage, errInvalidNumberOfArguments + // >> Args + + args := msg.Args + if len(args) < 2 { + return retrerr(errInvalidNumberOfArguments) } + + // >> Operation + var vals []resp.Value - var key string - var ok bool - for { - vs, key, ok = tokenval(vs) - if !ok { - break - } + var ms = []map[string]interface{}{} + for i := 1; i < len(args); i++ { + key := args[i] col, _ := s.cols.Get(key) if col != nil { m := make(map[string]interface{}) @@ -83,18 +84,15 @@ func (s *Server) cmdStats(msg *Message) (res resp.Value, err error) { } } } - switch msg.OutputType { - case JSON: - data, err := json.Marshal(ms) - if err != nil { - return NOMessage, err - } - res = resp.StringValue(`{"ok":true,"stats":` + string(data) + `,"elapsed":"` + time.Since(start).String() + "\"}") - case RESP: - res = resp.ArrayValue(vals) + // >> Response + + if msg.OutputType == JSON { + data, _ := json.Marshal(ms) + return resp.StringValue(`{"ok":true,"stats":` + string(data) + + `,"elapsed":"` + time.Since(start).String() + "\"}"), nil } - return res, nil + return resp.ArrayValue(vals), nil } func (s *Server) cmdHealthz(msg *Message) (res resp.Value, err error) { diff --git a/tests/keys_test.go b/tests/keys_test.go index b2932816..7a698e09 100644 --- a/tests/keys_test.go +++ b/tests/keys_test.go @@ -249,94 +249,149 @@ func keys_KEYS_test(mc *mockServer) error { }) } func keys_PERSIST_test(mc *mockServer) error { - return mc.DoBatch([][]interface{}{ - {"SET", "mykey", "myid", "STRING", "value"}, {"OK"}, - {"EXPIRE", "mykey", "myid", 2}, {1}, - {"PERSIST", "mykey", "myid"}, {1}, - {"PERSIST", "mykey", "myid"}, {0}, - }) + return mc.DoBatch( + Do("SET", "mykey", "myid", "STRING", "value").OK(), + Do("EXPIRE", "mykey", "myid", 2).Str("1"), + Do("PERSIST", "mykey", "myid").Str("1"), + Do("PERSIST", "mykey", "myid").Str("0"), + Do("PERSIST", "mykey").Err("wrong number of arguments for 'persist' command"), + Do("PERSIST", "mykey2", "myid").Str("0"), + Do("PERSIST", "mykey2", "myid").JSON().Err("key not found"), + Do("PERSIST", "mykey", "myid2").Str("0"), + Do("PERSIST", "mykey", "myid2").JSON().Err("id not found"), + Do("EXPIRE", "mykey", "myid", 2).Str("1"), + Do("PERSIST", "mykey", "myid").JSON().OK(), + ) } func keys_SET_test(mc *mockServer) error { return mc.DoBatch( - "point", [][]interface{}{ - {"SET", "mykey", "myid", "POINT", 33, -115}, {"OK"}, - {"GET", "mykey", "myid", "POINT"}, {"[33 -115]"}, - {"GET", "mykey", "myid", "BOUNDS"}, {"[[33 -115] [33 -115]]"}, - {"GET", "mykey", "myid", "OBJECT"}, {`{"type":"Point","coordinates":[-115,33]}`}, - {"GET", "mykey", "myid", "HASH", 7}, {"9my5xp7"}, - {"DEL", "mykey", "myid"}, {"1"}, - {"GET", "mykey", "myid"}, {nil}, - }, - "object", [][]interface{}{ - {"SET", "mykey", "myid", "OBJECT", `{"type":"Point","coordinates":[-115,33]}`}, {"OK"}, - {"GET", "mykey", "myid", "POINT"}, {"[33 -115]"}, - {"GET", "mykey", "myid", "BOUNDS"}, {"[[33 -115] [33 -115]]"}, - {"GET", "mykey", "myid", "OBJECT"}, {`{"type":"Point","coordinates":[-115,33]}`}, - {"GET", "mykey", "myid", "HASH", 7}, {"9my5xp7"}, - {"DEL", "mykey", "myid"}, {"1"}, - {"GET", "mykey", "myid"}, {nil}, - }, - "bounds", [][]interface{}{ - {"SET", "mykey", "myid", "BOUNDS", 33, -115, 33, -115}, {"OK"}, - {"GET", "mykey", "myid", "POINT"}, {"[33 -115]"}, - {"GET", "mykey", "myid", "BOUNDS"}, {"[[33 -115] [33 -115]]"}, - {"GET", "mykey", "myid", "OBJECT"}, {`{"type":"Point","coordinates":[-115,33]}`}, - {"GET", "mykey", "myid", "HASH", 7}, {"9my5xp7"}, - {"DEL", "mykey", "myid"}, {"1"}, - {"GET", "mykey", "myid"}, {nil}, - }, - "hash", [][]interface{}{ - {"SET", "mykey", "myid", "HASH", "9my5xp7"}, {"OK"}, - {"GET", "mykey", "myid", "HASH", 7}, {"9my5xp7"}, - {"DEL", "mykey", "myid"}, {"1"}, - {"GET", "mykey", "myid"}, {nil}, - }, - "field", [][]interface{}{ - {"SET", "mykey", "myid", "FIELD", "f1", 33, "FIELD", "a2", 44.5, "HASH", "9my5xp7"}, {"OK"}, - {"GET", "mykey", "myid", "WITHFIELDS", "HASH", 7}, {"[9my5xp7 [a2 44.5 f1 33]]"}, - {"FSET", "mykey", "myid", "f1", 0}, {1}, - {"FSET", "mykey", "myid", "f1", 0}, {0}, - {"GET", "mykey", "myid", "WITHFIELDS", "HASH", 7}, {"[9my5xp7 [a2 44.5]]"}, - {"DEL", "mykey", "myid"}, {"1"}, - {"GET", "mykey", "myid"}, {nil}, - }, - "string", [][]interface{}{ - {"SET", "mykey", "myid", "STRING", "value"}, {"OK"}, - {"GET", "mykey", "myid"}, {"value"}, - {"SET", "mykey", "myid", "STRING", "value2"}, {"OK"}, - {"GET", "mykey", "myid"}, {"value2"}, - {"DEL", "mykey", "myid"}, {"1"}, - {"GET", "mykey", "myid"}, {nil}, - }, + // Section: point + Do("SET", "mykey", "myid", "POINT", 33, -115).OK(), + Do("GET", "mykey", "myid", "POINT").Str("[33 -115]"), + Do("GET", "mykey", "myid", "BOUNDS").Str("[[33 -115] [33 -115]]"), + Do("GET", "mykey", "myid", "OBJECT").Str(`{"type":"Point","coordinates":[-115,33]}`), + Do("GET", "mykey", "myid", "HASH", 7).Str("9my5xp7"), + Do("DEL", "mykey", "myid").Str("1"), + Do("GET", "mykey", "myid").Str(""), + Do("SET", "mykey", "myid", "point", "33", "-112", "99").OK(), + + // Section: object + Do("SET", "mykey", "myid", "OBJECT", `{"type":"Point","coordinates":[-115,33]}`).OK(), + Do("GET", "mykey", "myid", "POINT").Str("[33 -115]"), + Do("GET", "mykey", "myid", "BOUNDS").Str("[[33 -115] [33 -115]]"), + Do("GET", "mykey", "myid", "OBJECT").Str(`{"type":"Point","coordinates":[-115,33]}`), + Do("GET", "mykey", "myid", "HASH", 7).Str("9my5xp7"), + Do("DEL", "mykey", "myid").Str("1"), + Do("GET", "mykey", "myid").Str(""), + + // Section: bounds + Do("SET", "mykey", "myid", "BOUNDS", 33, -115, 33, -115).OK(), + Do("GET", "mykey", "myid", "POINT").Str("[33 -115]"), + Do("GET", "mykey", "myid", "BOUNDS").Str("[[33 -115] [33 -115]]"), + Do("GET", "mykey", "myid", "OBJECT").Str(`{"type":"Point","coordinates":[-115,33]}`), + Do("GET", "mykey", "myid", "HASH", 7).Str("9my5xp7"), + Do("DEL", "mykey", "myid").Str("1"), + Do("GET", "mykey", "myid").Str(""), + + // Section: hash + Do("SET", "mykey", "myid", "HASH", "9my5xp7").OK(), + Do("GET", "mykey", "myid", "HASH", 7).Str("9my5xp7"), + Do("DEL", "mykey", "myid").Str("1"), + Do("GET", "mykey", "myid").Str(""), + Do("SET", "mykey", "myid", "HASH", "9my5xp7").JSON().OK(), + + // Section: field + Do("SET", "mykey", "myid", "FIELD", "f1", 33, "FIELD", "a2", 44.5, "HASH", "9my5xp7").OK(), + Do("GET", "mykey", "myid", "WITHFIELDS", "HASH", 7).Str("[9my5xp7 [a2 44.5 f1 33]]"), + Do("FSET", "mykey", "myid", "f1", 0).Str("1"), + Do("FSET", "mykey", "myid", "f1", 0).Str("0"), + Do("GET", "mykey", "myid", "WITHFIELDS", "HASH", 7).Str("[9my5xp7 [a2 44.5]]"), + Do("DEL", "mykey", "myid").Str("1"), + Do("GET", "mykey", "myid").Str(""), + + // Section: string + Do("SET", "mykey", "myid", "STRING", "value").OK(), + Do("GET", "mykey", "myid").Str("value"), + Do("SET", "mykey", "myid", "STRING", "value2").OK(), + Do("GET", "mykey", "myid").Str("value2"), + Do("DEL", "mykey", "myid").Str("1"), + Do("GET", "mykey", "myid").Str(""), + + // Test error conditions + Do("CONFIG", "SET", "maxmemory", "1").OK(), + Do("SET", "mykey", "myid", "STRING", "value2").Err("OOM command not allowed when used memory > 'maxmemory'"), + Do("CONFIG", "SET", "maxmemory", "0").OK(), + Do("SET").Err("wrong number of arguments for 'set' command"), + Do("SET", "mykey", "myid", "FIELD", "f1").Err("wrong number of arguments for 'set' command"), + Do("SET", "mykey", "myid", "FIELD", "z", "1").Err("invalid argument 'z'"), + Do("SET", "mykey", "myid", "EX").Err("wrong number of arguments for 'set' command"), + Do("SET", "mykey", "myid", "EX", "yyy").Err("invalid argument 'yyy'"), + Do("SET", "mykey", "myid", "EX", "123").Err("wrong number of arguments for 'set' command"), + Do("SET", "mykey", "myid", "nx").Err("wrong number of arguments for 'set' command"), + Do("SET", "mykey", "myid", "nx", "xx").Err("invalid argument 'xx'"), + Do("SET", "mykey", "myid", "xx", "nx").Err("invalid argument 'nx'"), + Do("SET", "mykey", "myid", "string").Err("wrong number of arguments for 'set' command"), + Do("SET", "mykey", "myid", "point").Err("wrong number of arguments for 'set' command"), + Do("SET", "mykey", "myid", "point", "33").Err("wrong number of arguments for 'set' command"), + Do("SET", "mykey", "myid", "point", "33f", "-112").Err("invalid argument '33f'"), + Do("SET", "mykey", "myid", "point", "33", "-112f").Err("invalid argument '-112f'"), + Do("SET", "mykey", "myid", "point", "33", "-112f", "99").Err("invalid argument '-112f'"), + Do("SET", "mykey", "myid", "bounds").Err("wrong number of arguments for 'set' command"), + Do("SET", "mykey", "myid", "bounds", "fff", "1", "2", "3").Err("invalid argument 'fff'"), + Do("SET", "mykey", "myid", "hash").Err("wrong number of arguments for 'set' command"), + Do("SET", "mykey", "myid", "object").Err("wrong number of arguments for 'set' command"), + Do("SET", "mykey", "myid", "object", "asd").Err("invalid data"), + Do("SET", "mykey", "myid", "joint").Err("invalid argument 'joint'"), + Do("SET", "mykey", "myid", "XX", "HASH", "9my5xp7").Err(""), + Do("SET", "mykey", "myid", "XX", "HASH", "9my5xp7").JSON().Err("id not found"), + Do("SET", "mykey", "myid1", "HASH", "9my5xp7").OK(), + Do("SET", "mykey", "myid", "XX", "HASH", "9my5xp7").Err(""), + Do("SET", "mykey", "myid", "NX", "HASH", "9my5xp7").OK(), + Do("SET", "mykey", "myid", "XX", "HASH", "9my5xp7").OK(), + Do("SET", "mykey", "myid", "NX", "HASH", "9my5xp7").Err(""), + Do("SET", "mykey", "myid", "NX", "HASH", "9my5xp7").JSON().Err("id already exists"), ) } func keys_STATS_test(mc *mockServer) error { - return mc.DoBatch([][]interface{}{ - {"STATS", "mykey"}, {"[nil]"}, - {"SET", "mykey", "myid", "STRING", "value"}, {"OK"}, - {"STATS", "mykey"}, {"[[in_memory_size 9 num_objects 1 num_points 0 num_strings 1]]"}, - {"SET", "mykey", "myid2", "STRING", "value"}, {"OK"}, - {"STATS", "mykey"}, {"[[in_memory_size 19 num_objects 2 num_points 0 num_strings 2]]"}, - {"SET", "mykey", "myid3", "OBJECT", `{"type":"Point","coordinates":[-115,33]}`}, {"OK"}, - {"STATS", "mykey"}, {"[[in_memory_size 40 num_objects 3 num_points 1 num_strings 2]]"}, - {"DEL", "mykey", "myid"}, {1}, - {"STATS", "mykey"}, {"[[in_memory_size 31 num_objects 2 num_points 1 num_strings 1]]"}, - {"DEL", "mykey", "myid3"}, {1}, - {"STATS", "mykey"}, {"[[in_memory_size 10 num_objects 1 num_points 0 num_strings 1]]"}, - {"STATS", "mykey", "mykey2"}, {"[[in_memory_size 10 num_objects 1 num_points 0 num_strings 1] nil]"}, - {"DEL", "mykey", "myid2"}, {1}, - {"STATS", "mykey"}, {"[nil]"}, - {"STATS", "mykey", "mykey2"}, {"[nil nil]"}, - }) + return mc.DoBatch( + Do("STATS", "mykey").Str("[nil]"), + Do("SET", "mykey", "myid", "STRING", "value").OK(), + Do("STATS", "mykey").Str("[[in_memory_size 9 num_objects 1 num_points 0 num_strings 1]]"), + Do("STATS", "mykey", "hello").JSON().Str(`{"ok":true,"stats":[{"in_memory_size":9,"num_objects":1,"num_points":0,"num_strings":1},null]}`), + Do("SET", "mykey", "myid2", "STRING", "value").OK(), + Do("STATS", "mykey").Str("[[in_memory_size 19 num_objects 2 num_points 0 num_strings 2]]"), + Do("SET", "mykey", "myid3", "OBJECT", `{"type":"Point","coordinates":[-115,33]}`).OK(), + Do("STATS", "mykey").Str("[[in_memory_size 40 num_objects 3 num_points 1 num_strings 2]]"), + Do("DEL", "mykey", "myid").Str("1"), + Do("STATS", "mykey").Str("[[in_memory_size 31 num_objects 2 num_points 1 num_strings 1]]"), + Do("DEL", "mykey", "myid3").Str("1"), + Do("STATS", "mykey").Str("[[in_memory_size 10 num_objects 1 num_points 0 num_strings 1]]"), + Do("STATS", "mykey", "mykey2").Str("[[in_memory_size 10 num_objects 1 num_points 0 num_strings 1] nil]"), + Do("DEL", "mykey", "myid2").Str("1"), + Do("STATS", "mykey").Str("[nil]"), + Do("STATS", "mykey", "mykey2").Str("[nil nil]"), + ) } func keys_TTL_test(mc *mockServer) error { - return mc.DoBatch([][]interface{}{ - {"SET", "mykey", "myid", "STRING", "value"}, {"OK"}, - {"EXPIRE", "mykey", "myid", 2}, {1}, - {time.Second / 4}, {}, // sleep - {"TTL", "mykey", "myid"}, {1}, - }) + return mc.DoBatch( + Do("SET", "mykey", "myid", "STRING", "value").OK(), + Do("EXPIRE", "mykey", "myid", 2).Str("1"), + Do("EXPIRE", "mykey", "myid", 2).JSON().OK(), + Sleep(time.Millisecond*10), + Do("TTL", "mykey", "myid").Str("1"), + Do("EXPIRE", "mykey", "myid", 1).Str("1"), + Sleep(time.Millisecond*10), + Do("TTL", "mykey", "myid").Str("0"), + Do("TTL", "mykey", "myid").JSON().Str(`{"ok":true,"ttl":0}`), + Do("TTL", "mykey2", "myid").Str("-2"), + Do("TTL", "mykey", "myid2").Str("-2"), + Do("TTL", "mykey").Err("wrong number of arguments for 'ttl' command"), + Do("SET", "mykey", "myid", "STRING", "value").OK(), + Do("TTL", "mykey", "myid").Str("-1"), + Do("TTL", "mykey2", "myid").JSON().Err("key not found"), + Do("TTL", "mykey", "myid2").JSON().Err("id not found"), + ) } func keys_SET_EX_test(mc *mockServer) (err error) { @@ -465,7 +520,7 @@ func keys_FLUSHDB_test(mc *mockServer) error { Do("SET", "mykey2", "myid1", "POINT", 33, -115).OK(), Do("SETCHAN", "mychan", "INTERSECTS", "mykey1", "BOUNDS", 10, 10, 10, 10).Str("1"), Do("KEYS", "*").Str("[mykey1 mykey2]"), - Do("CHANS", "*").JSON().Custom(func(s string) error { + Do("CHANS", "*").JSON().Func(func(s string) error { if gjson.Get(s, "chans.#").Int() != 1 { return fmt.Errorf("expected '%d', got '%d'", 1, gjson.Get(s, "chans.#").Int()) } diff --git a/tests/mock_io_test.go b/tests/mock_io_test.go index 72a24e6b..90ed6960 100644 --- a/tests/mock_io_test.go +++ b/tests/mock_io_test.go @@ -34,7 +34,7 @@ func (cmd *IO) Str(s string) *IO { cmd.out = s return cmd } -func (cmd *IO) Custom(fn func(s string) error) *IO { +func (cmd *IO) Func(fn func(s string) error) *IO { cmd.out = func(s string) error { if cmd.json { if !gjson.Valid(s) { @@ -47,7 +47,7 @@ func (cmd *IO) Custom(fn func(s string) error) *IO { } func (cmd *IO) OK() *IO { - return cmd.Custom(func(s string) error { + return cmd.Func(func(s string) error { if cmd.json { if gjson.Get(s, "ok").Type != gjson.True { return errors.New("not ok") @@ -60,7 +60,7 @@ func (cmd *IO) OK() *IO { } func (cmd *IO) Err(msg string) *IO { - return cmd.Custom(func(s string) error { + return cmd.Func(func(s string) error { if cmd.json { if gjson.Get(s, "ok").Type != gjson.False { return errors.New("ok=true") From 5bcef43894b1be4179eb617be0dd4657ac44ea93 Mon Sep 17 00:00:00 2001 From: tidwall Date: Fri, 23 Sep 2022 16:12:32 -0700 Subject: [PATCH 09/33] Better KEYS tests --- internal/server/keys.go | 118 ++++++++++++++----------------------- internal/server/scripts.go | 2 +- internal/server/server.go | 2 +- tests/keys_test.go | 48 ++++++++------- 4 files changed, 73 insertions(+), 97 deletions(-) diff --git a/internal/server/keys.go b/internal/server/keys.go index 94eaf199..095a0686 100644 --- a/internal/server/keys.go +++ b/internal/server/keys.go @@ -1,8 +1,7 @@ package server import ( - "bytes" - "strings" + "encoding/json" "time" "github.com/tidwall/resp" @@ -10,86 +9,59 @@ import ( "github.com/tidwall/tile38/internal/glob" ) -func (s *Server) cmdKeys(msg *Message) (res resp.Value, err error) { +// KEYS pattern +func (s *Server) cmdKEYS(msg *Message) (resp.Value, error) { var start = time.Now() - vs := msg.Args[1:] - var pattern string - var ok bool - if vs, pattern, ok = tokenval(vs); !ok || pattern == "" { - return NOMessage, errInvalidNumberOfArguments - } - if len(vs) != 0 { - return NOMessage, errInvalidNumberOfArguments - } + // >> Args - var wr = &bytes.Buffer{} - var once bool - if msg.OutputType == JSON { - wr.WriteString(`{"ok":true,"keys":[`) + args := msg.Args + if len(args) != 2 { + return retrerr(errInvalidNumberOfArguments) } - var wild bool - if strings.Contains(pattern, "*") { - wild = true - } - var everything bool - var greater bool - var greaterPivot string - var vals []resp.Value + pattern := args[1] - iter := func(key string, col *collection.Collection) bool { - var match bool - if everything { - match = true - } else if greater { - if !strings.HasPrefix(key, greaterPivot) { - return false - } - match = true - } else { - match, _ = glob.Match(pattern, key) - } - if match { - if once { - if msg.OutputType == JSON { - wr.WriteByte(',') + // >> Operation + + keys := []string{} + g := glob.Parse(pattern, false) + everything := g.Limits[0] == "" && g.Limits[1] == "" + if everything { + s.cols.Scan( + func(key string, _ *collection.Collection) bool { + match, _ := glob.Match(pattern, key) + if match { + keys = append(keys, key) } - } else { - once = true - } - switch msg.OutputType { - case JSON: - wr.WriteString(jsonString(key)) - case RESP: - vals = append(vals, resp.StringValue(key)) - } - - // If no more than one match is expected, stop searching - if !wild { - return false - } - } - return true - } - - // TODO: This can be further optimized by using glob.Parse and limits - if pattern == "*" { - everything = true - s.cols.Scan(iter) - } else if strings.HasSuffix(pattern, "*") { - greaterPivot = pattern[:len(pattern)-1] - if glob.IsGlob(greaterPivot) { - s.cols.Scan(iter) - } else { - greater = true - s.cols.Ascend(greaterPivot, iter) - } + return true + }, + ) } else { - s.cols.Scan(iter) + s.cols.Ascend(g.Limits[0], + func(key string, _ *collection.Collection) bool { + if key > g.Limits[1] { + return false + } + match, _ := glob.Match(pattern, key) + if match { + keys = append(keys, key) + } + return true + }, + ) } + + // >> Response + if msg.OutputType == JSON { - wr.WriteString(`],"elapsed":"` + time.Since(start).String() + "\"}") - return resp.StringValue(wr.String()), nil + data, _ := json.Marshal(keys) + return resp.StringValue(`{"ok":true,"keys":` + string(data) + + `,"elapsed":"` + time.Since(start).String() + `"}`), nil + } + + var vals []resp.Value + for _, key := range keys { + vals = append(vals, resp.StringValue(key)) } return resp.ArrayValue(vals), nil } diff --git a/internal/server/scripts.go b/internal/server/scripts.go index d1948d21..acbb3310 100644 --- a/internal/server/scripts.go +++ b/internal/server/scripts.go @@ -636,7 +636,7 @@ func (s *Server) commandInScript(msg *Message) ( case "type": res, err = s.cmdTYPE(msg) case "keys": - res, err = s.cmdKeys(msg) + res, err = s.cmdKEYS(msg) case "test": res, err = s.cmdTest(msg) case "server": diff --git a/internal/server/server.go b/internal/server/server.go index 7b1a2781..0f8d7211 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -1110,7 +1110,7 @@ func (s *Server) command(msg *Message, client *Client) ( case "type": res, err = s.cmdTYPE(msg) case "keys": - res, err = s.cmdKeys(msg) + res, err = s.cmdKEYS(msg) case "output": res, err = s.cmdOutput(msg) case "aof": diff --git a/tests/keys_test.go b/tests/keys_test.go index 7a698e09..9361e8c6 100644 --- a/tests/keys_test.go +++ b/tests/keys_test.go @@ -225,28 +225,30 @@ func keys_GET_test(mc *mockServer) error { ) } func keys_KEYS_test(mc *mockServer) error { - return mc.DoBatch([][]interface{}{ - {"SET", "mykey11", "myid4", "STRING", "value"}, {"OK"}, - {"SET", "mykey22", "myid2", "HASH", "9my5xp7"}, {"OK"}, - {"SET", "mykey22", "myid1", "OBJECT", `{"type":"Point","coordinates":[-130,38,10]}`}, {"OK"}, - {"SET", "mykey11", "myid3", "OBJECT", `{"type":"Point","coordinates":[-110,25,-8]}`}, {"OK"}, - {"SET", "mykey42", "myid2", "HASH", "9my5xp7"}, {"OK"}, - {"SET", "mykey31", "myid4", "STRING", "value"}, {"OK"}, - {"SET", "mykey310", "myid5", "STRING", "value"}, {"OK"}, - {"KEYS", "*"}, {"[mykey11 mykey22 mykey31 mykey310 mykey42]"}, - {"KEYS", "*key*"}, {"[mykey11 mykey22 mykey31 mykey310 mykey42]"}, - {"KEYS", "mykey*"}, {"[mykey11 mykey22 mykey31 mykey310 mykey42]"}, - {"KEYS", "mykey4*"}, {"[mykey42]"}, - {"KEYS", "mykey*1"}, {"[mykey11 mykey31]"}, - {"KEYS", "mykey*1*"}, {"[mykey11 mykey31 mykey310]"}, - {"KEYS", "mykey*10"}, {"[mykey310]"}, - {"KEYS", "mykey*2"}, {"[mykey22 mykey42]"}, - {"KEYS", "*2"}, {"[mykey22 mykey42]"}, - {"KEYS", "*1*"}, {"[mykey11 mykey31 mykey310]"}, - {"KEYS", "mykey"}, {"[]"}, - {"KEYS", "mykey31"}, {"[mykey31]"}, - {"KEYS", "mykey[^3]*"}, {"[mykey11 mykey22 mykey42]"}, - }) + return mc.DoBatch( + Do("SET", "mykey11", "myid4", "STRING", "value").OK(), + Do("SET", "mykey22", "myid2", "HASH", "9my5xp7").OK(), + Do("SET", "mykey22", "myid1", "OBJECT", `{"type":"Point","coordinates":[-130,38,10]}`).OK(), + Do("SET", "mykey11", "myid3", "OBJECT", `{"type":"Point","coordinates":[-110,25,-8]}`).OK(), + Do("SET", "mykey42", "myid2", "HASH", "9my5xp7").OK(), + Do("SET", "mykey31", "myid4", "STRING", "value").OK(), + Do("SET", "mykey310", "myid5", "STRING", "value").OK(), + Do("KEYS", "*").Str("[mykey11 mykey22 mykey31 mykey310 mykey42]"), + Do("KEYS", "*key*").Str("[mykey11 mykey22 mykey31 mykey310 mykey42]"), + Do("KEYS", "mykey*").Str("[mykey11 mykey22 mykey31 mykey310 mykey42]"), + Do("KEYS", "mykey4*").Str("[mykey42]"), + Do("KEYS", "mykey*1").Str("[mykey11 mykey31]"), + Do("KEYS", "mykey*1*").Str("[mykey11 mykey31 mykey310]"), + Do("KEYS", "mykey*10").Str("[mykey310]"), + Do("KEYS", "mykey*2").Str("[mykey22 mykey42]"), + Do("KEYS", "*2").Str("[mykey22 mykey42]"), + Do("KEYS", "*1*").Str("[mykey11 mykey31 mykey310]"), + Do("KEYS", "mykey").Str("[]"), + Do("KEYS", "mykey31").Str("[mykey31]"), + Do("KEYS", "mykey[^3]*").Str("[mykey11 mykey22 mykey42]"), + Do("KEYS").Err("wrong number of arguments for 'keys' command"), + Do("KEYS", "*").JSON().Str(`{"ok":true,"keys":["mykey11","mykey22","mykey31","mykey310","mykey42"]}`), + ) } func keys_PERSIST_test(mc *mockServer) error { return mc.DoBatch( @@ -371,6 +373,8 @@ func keys_STATS_test(mc *mockServer) error { Do("DEL", "mykey", "myid2").Str("1"), Do("STATS", "mykey").Str("[nil]"), Do("STATS", "mykey", "mykey2").Str("[nil nil]"), + Do("STATS", "mykey").Str("[nil]"), + Do("STATS").Err(`wrong number of arguments for 'stats' command`), ) } func keys_TTL_test(mc *mockServer) error { From 5c455cbe10a40d0925c8448667a59ec2554c69d4 Mon Sep 17 00:00:00 2001 From: tidwall Date: Fri, 23 Sep 2022 17:34:09 -0700 Subject: [PATCH 10/33] Better HEALTHZ tests --- internal/server/server.go | 2 +- internal/server/stats.go | 28 ++++++++++++++++++++-------- tests/keys_search_test.go | 2 -- tests/keys_test.go | 33 +++++++++++++++++++++++++++++++++ tests/mock_test.go | 15 +++++++++------ tests/tests_test.go | 21 +++++++++++++++------ 6 files changed, 78 insertions(+), 23 deletions(-) diff --git a/internal/server/server.go b/internal/server/server.go index 0f8d7211..deb764ee 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -1084,7 +1084,7 @@ func (s *Server) command(msg *Message, client *Client) ( case "server": res, err = s.cmdServer(msg) case "healthz": - res, err = s.cmdHealthz(msg) + res, err = s.cmdHEALTHZ(msg) case "info": res, err = s.cmdInfo(msg) case "scan": diff --git a/internal/server/stats.go b/internal/server/stats.go index 4c8dcc4b..0feb361d 100644 --- a/internal/server/stats.go +++ b/internal/server/stats.go @@ -95,22 +95,34 @@ func (s *Server) cmdSTATS(msg *Message) (resp.Value, error) { return resp.ArrayValue(vals), nil } -func (s *Server) cmdHealthz(msg *Message) (res resp.Value, err error) { +// HEALTHZ +func (s *Server) cmdHEALTHZ(msg *Message) (resp.Value, error) { start := time.Now() + + // >> Args + + args := msg.Args + if len(args) != 1 { + return retrerr(errInvalidNumberOfArguments) + } + + // >> Operation + if s.config.followHost() != "" { m := make(map[string]interface{}) s.basicStats(m) if fmt.Sprintf("%v", m["caught_up"]) != "true" { - return NOMessage, errors.New("not caught up") + return retrerr(errors.New("not caught up")) } } - switch msg.OutputType { - case JSON: - res = resp.StringValue(`{"ok":true,"elapsed":"` + time.Since(start).String() + "\"}") - case RESP: - res = resp.SimpleStringValue("OK") + + // >> Response + + if msg.OutputType == JSON { + return resp.StringValue(`{"ok":true,"elapsed":"` + + time.Since(start).String() + "\"}"), nil } - return res, nil + return resp.SimpleStringValue("OK"), nil } func (s *Server) cmdServer(msg *Message) (res resp.Value, err error) { diff --git a/tests/keys_search_test.go b/tests/keys_search_test.go index e68b9236..b3f432e1 100644 --- a/tests/keys_search_test.go +++ b/tests/keys_search_test.go @@ -122,9 +122,7 @@ func keys_KNN_random_test(mc *mockServer) error { mc.Do("OUTPUT", "json") defer mc.Do("OUTPUT", "resp") - start := time.Now() res, err := redis.String(mc.Do("NEARBY", "points", "LIMIT", N, "POINT", target[1], target[0])) - println(time.Since(start).String()) if err != nil { return err } diff --git a/tests/keys_test.go b/tests/keys_test.go index 9361e8c6..0b3217a6 100644 --- a/tests/keys_test.go +++ b/tests/keys_test.go @@ -32,6 +32,8 @@ func subTestKeys(t *testing.T, mc *mockServer) { runStep(t, mc, "WHEREEVAL", keys_WHEREEVAL_test) runStep(t, mc, "TYPE", keys_TYPE_test) runStep(t, mc, "FLUSHDB", keys_FLUSHDB_test) + runStep(t, mc, "HEALTHZ", keys_HEALTHZ_test) + } func keys_BOUNDS_test(mc *mockServer) error { @@ -540,3 +542,34 @@ func keys_FLUSHDB_test(mc *mockServer) error { Do("FLUSHDB").JSON().OK(), ) } + +func keys_HEALTHZ_test(mc *mockServer) error { + + // // follow and wait + // str, err := redis.String(mc.Do("FOLLOW", "localhost", mc.alt.port)) + // if err != nil { + // return err + // } + // if str != "OK" { + // return errors.New("not ok") + // } + // start := time.Now() + // for time.Since(start) < time.Second*5 { + // str, err = redis.String(mc.Do("HEALTHZ")) + // if str == "OK" { + // err = nil + // break + // } + // time.Sleep(time.Second / 4) + // } + // if err != nil { + // return err + // } + + return mc.DoBatch( + Do("HEALTHZ").OK(), + Do("HEALTHZ").JSON().OK(), + // Do("FOLLOW", "no", "one").OK(), + Do("HEALTHZ", "arg").Err(`wrong number of arguments for 'healthz' command`), + ) +} diff --git a/tests/mock_test.go b/tests/mock_test.go index a1aec6fb..cfd9c44d 100644 --- a/tests/mock_test.go +++ b/tests/mock_test.go @@ -38,9 +38,10 @@ type mockServer struct { port int conn redis.Conn ioJSON bool + // alt *mockServer } -func mockOpenServer(silent bool) (*mockServer, error) { +func mockOpenServer(silent, metrics bool) (*mockServer, error) { rand.Seed(time.Now().UnixNano()) port := rand.Int()%20000 + 20000 dir := fmt.Sprintf("data-mock-%d", port) @@ -56,11 +57,13 @@ func mockOpenServer(silent bool) (*mockServer, error) { tlog.SetOutput(logOutput) go func() { opts := server.Options{ - Host: "localhost", - Port: port, - Dir: dir, - UseHTTP: true, - MetricsAddr: ":4321", + Host: "localhost", + Port: port, + Dir: dir, + UseHTTP: true, + } + if metrics { + opts.MetricsAddr = ":4321" } if err := server.Serve(opts); err != nil { log.Fatal(err) diff --git a/tests/tests_test.go b/tests/tests_test.go index 5f20937d..be9d3b3a 100644 --- a/tests/tests_test.go +++ b/tests/tests_test.go @@ -38,11 +38,20 @@ func TestAll(t *testing.T) { os.Exit(1) }() - mc, err := mockOpenServer(false) + mc, err := mockOpenServer(false, true) if err != nil { t.Fatal(err) } defer mc.Close() + + // mc2, err := mockOpenServer(false, false) + // if err != nil { + // t.Fatal(err) + // } + // defer mc2.Close() + // mc.alt = mc2 + // mc2.alt = mc + runSubTest(t, "keys", mc, subTestKeys) runSubTest(t, "json", mc, subTestJSON) runSubTest(t, "search", mc, subTestSearch) @@ -71,10 +80,10 @@ func runStep(t *testing.T, mc *mockServer, name string, step func(mc *mockServer mc.ResetConn() defer mc.ResetConn() // clear the database so the test is consistent - if err := mc.DoBatch([][]interface{}{ - {"OUTPUT", "resp"}, {"OK"}, - {"FLUSHDB"}, {"OK"}, - }); err != nil { + if err := mc.DoBatch( + Do("OUTPUT", "resp").OK(), + Do("FLUSHDB").OK(), + ); err != nil { return err } if err := step(mc); err != nil { @@ -102,7 +111,7 @@ func BenchmarkAll(b *testing.B) { os.Exit(1) }() - mc, err := mockOpenServer(true) + mc, err := mockOpenServer(true, true) if err != nil { b.Fatal(err) } From 9c8e7e90e1002a05626aa37d8daad629efb62191 Mon Sep 17 00:00:00 2001 From: tidwall Date: Fri, 23 Sep 2022 17:54:49 -0700 Subject: [PATCH 11/33] Clean up some tests --- internal/server/scripts.go | 2 +- internal/server/server.go | 2 +- internal/server/stats.go | 50 +++++++++++--------- tests/keys_test.go | 94 ++++++++++++++++++++++++++++---------- 4 files changed, 101 insertions(+), 47 deletions(-) diff --git a/internal/server/scripts.go b/internal/server/scripts.go index acbb3310..ed267e76 100644 --- a/internal/server/scripts.go +++ b/internal/server/scripts.go @@ -640,7 +640,7 @@ func (s *Server) commandInScript(msg *Message) ( case "test": res, err = s.cmdTest(msg) case "server": - res, err = s.cmdServer(msg) + res, err = s.cmdSERVER(msg) } s.sendMonitor(err, msg, nil, true) return diff --git a/internal/server/server.go b/internal/server/server.go index deb764ee..d7c6a399 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -1082,7 +1082,7 @@ func (s *Server) command(msg *Message, client *Client) ( case "stats": res, err = s.cmdSTATS(msg) case "server": - res, err = s.cmdServer(msg) + res, err = s.cmdSERVER(msg) case "healthz": res, err = s.cmdHEALTHZ(msg) case "info": diff --git a/internal/server/stats.go b/internal/server/stats.go index 0feb361d..12187016 100644 --- a/internal/server/stats.go +++ b/internal/server/stats.go @@ -125,35 +125,41 @@ func (s *Server) cmdHEALTHZ(msg *Message) (resp.Value, error) { return resp.SimpleStringValue("OK"), nil } -func (s *Server) cmdServer(msg *Message) (res resp.Value, err error) { +// SERVER [ext] +func (s *Server) cmdSERVER(msg *Message) (resp.Value, error) { start := time.Now() + + // >> Args + + args := msg.Args + var ext bool + for i := 1; i < len(args); i++ { + switch strings.ToLower(args[i]) { + case "ext": + ext = true + default: + return retrerr(errInvalidArgument(args[i])) + } + } + + // >> Operation + m := make(map[string]interface{}) - args := msg.Args[1:] - // Switch on the type of stats requested - switch len(args) { - case 0: + if ext { + s.extStats(m) + } else { s.basicStats(m) - case 1: - if strings.ToLower(args[0]) == "ext" { - s.extStats(m) - } - default: - return NOMessage, errInvalidNumberOfArguments } - switch msg.OutputType { - case JSON: - data, err := json.Marshal(m) - if err != nil { - return NOMessage, err - } - res = resp.StringValue(`{"ok":true,"stats":` + string(data) + `,"elapsed":"` + time.Since(start).String() + "\"}") - case RESP: - vals := respValuesSimpleMap(m) - res = resp.ArrayValue(vals) + // >> Response + + if msg.OutputType == JSON { + data, _ := json.Marshal(m) + return resp.StringValue(`{"ok":true,"stats":` + string(data) + + `,"elapsed":"` + time.Since(start).String() + "\"}"), nil } - return res, nil + return resp.ArrayValue(respValuesSimpleMap(m)), nil } // basicStats populates the passed map with basic system/go/tile38 statistics diff --git a/tests/keys_test.go b/tests/keys_test.go index 0b3217a6..2bfec979 100644 --- a/tests/keys_test.go +++ b/tests/keys_test.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "math/rand" + "strings" "testing" "time" @@ -33,6 +34,7 @@ func subTestKeys(t *testing.T, mc *mockServer) { runStep(t, mc, "TYPE", keys_TYPE_test) runStep(t, mc, "FLUSHDB", keys_FLUSHDB_test) runStep(t, mc, "HEALTHZ", keys_HEALTHZ_test) + runStep(t, mc, "SERVER", keys_SERVER_test) } @@ -480,33 +482,32 @@ func keys_PDEL_test(mc *mockServer) error { } func keys_WHEREIN_test(mc *mockServer) error { - return mc.DoBatch([][]interface{}{ - {"SET", "mykey", "myid_a1", "FIELD", "a", 1, "POINT", 33, -115}, {"OK"}, - {"WITHIN", "mykey", "WHEREIN", "a", 3, 0, 1, 2, "BOUNDS", 32.8, -115.2, 33.2, -114.8}, {`[0 [[myid_a1 {"type":"Point","coordinates":[-115,33]} [a 1]]]]`}, - {"WITHIN", "mykey", "WHEREIN", "a", "a", 0, 1, 2, "BOUNDS", 32.8, -115.2, 33.2, -114.8}, {"ERR invalid argument 'a'"}, - {"WITHIN", "mykey", "WHEREIN", "a", 1, 0, 1, 2, "BOUNDS", 32.8, -115.2, 33.2, -114.8}, {"ERR invalid argument '1'"}, - {"WITHIN", "mykey", "WHEREIN", "a", 3, 0, "a", 2, "BOUNDS", 32.8, -115.2, 33.2, -114.8}, {"[0 []]"}, - {"WITHIN", "mykey", "WHEREIN", "a", 4, 0, "a", 1, 2, "BOUNDS", 32.8, -115.2, 33.2, -114.8}, {`[0 [[myid_a1 {"type":"Point","coordinates":[-115,33]} [a 1]]]]`}, - {"SET", "mykey", "myid_a2", "FIELD", "a", 2, "POINT", 32.99, -115}, {"OK"}, - {"SET", "mykey", "myid_a3", "FIELD", "a", 3, "POINT", 33, -115.02}, {"OK"}, - {"WITHIN", "mykey", "WHEREIN", "a", 3, 0, 1, 2, "BOUNDS", 32.8, -115.2, 33.2, -114.8}, { - `[0 [[myid_a2 {"type":"Point","coordinates":[-115,32.99]} [a 2]] [myid_a1 {"type":"Point","coordinates":[-115,33]} [a 1]]]]`}, + return mc.DoBatch( + Do("SET", "mykey", "myid_a1", "FIELD", "a", 1, "POINT", 33, -115).OK(), + Do("WITHIN", "mykey", "WHEREIN", "a", 3, 0, 1, 2, "BOUNDS", 32.8, -115.2, 33.2, -114.8).Str(`[0 [[myid_a1 {"type":"Point","coordinates":[-115,33]} [a 1]]]]`), + Do("WITHIN", "mykey", "WHEREIN", "a", "a", 0, 1, 2, "BOUNDS", 32.8, -115.2, 33.2, -114.8).Err("invalid argument 'a'"), + Do("WITHIN", "mykey", "WHEREIN", "a", 1, 0, 1, 2, "BOUNDS", 32.8, -115.2, 33.2, -114.8).Err("invalid argument '1'"), + Do("WITHIN", "mykey", "WHEREIN", "a", 3, 0, "a", 2, "BOUNDS", 32.8, -115.2, 33.2, -114.8).Str("[0 []]"), + Do("WITHIN", "mykey", "WHEREIN", "a", 4, 0, "a", 1, 2, "BOUNDS", 32.8, -115.2, 33.2, -114.8).Str(`[0 [[myid_a1 {"type":"Point","coordinates":[-115,33]} [a 1]]]]`), + Do("SET", "mykey", "myid_a2", "FIELD", "a", 2, "POINT", 32.99, -115).OK(), + Do("SET", "mykey", "myid_a3", "FIELD", "a", 3, "POINT", 33, -115.02).OK(), + Do("WITHIN", "mykey", "WHEREIN", "a", 3, 0, 1, 2, "BOUNDS", 32.8, -115.2, 33.2, -114.8).Str(`[0 [[myid_a2 {"type":"Point","coordinates":[-115,32.99]} [a 2]] [myid_a1 {"type":"Point","coordinates":[-115,33]} [a 1]]]]`), // zero value should not match 1 and 2 - {"SET", "mykey", "myid_a0", "FIELD", "a", 0, "POINT", 33, -115.02}, {"OK"}, - {"WITHIN", "mykey", "WHEREIN", "a", 2, 1, 2, "BOUNDS", 32.8, -115.2, 33.2, -114.8}, {`[0 [[myid_a2 {"type":"Point","coordinates":[-115,32.99]} [a 2]] [myid_a1 {"type":"Point","coordinates":[-115,33]} [a 1]]]]`}, - }) + Do("SET", "mykey", "myid_a0", "FIELD", "a", 0, "POINT", 33, -115.02).OK(), + Do("WITHIN", "mykey", "WHEREIN", "a", 2, 1, 2, "BOUNDS", 32.8, -115.2, 33.2, -114.8).Str(`[0 [[myid_a2 {"type":"Point","coordinates":[-115,32.99]} [a 2]] [myid_a1 {"type":"Point","coordinates":[-115,33]} [a 1]]]]`), + ) } func keys_WHEREEVAL_test(mc *mockServer) error { - return mc.DoBatch([][]interface{}{ - {"SET", "mykey", "myid_a1", "FIELD", "a", 1, "POINT", 33, -115}, {"OK"}, - {"WITHIN", "mykey", "WHEREEVAL", "return FIELDS.a > tonumber(ARGV[1])", 1, 0.5, "BOUNDS", 32.8, -115.2, 33.2, -114.8}, {`[0 [[myid_a1 {"type":"Point","coordinates":[-115,33]} [a 1]]]]`}, - {"WITHIN", "mykey", "WHEREEVAL", "return FIELDS.a > tonumber(ARGV[1])", "a", 0.5, "BOUNDS", 32.8, -115.2, 33.2, -114.8}, {"ERR invalid argument 'a'"}, - {"WITHIN", "mykey", "WHEREEVAL", "return FIELDS.a > tonumber(ARGV[1])", 1, 0.5, 4, "BOUNDS", 32.8, -115.2, 33.2, -114.8}, {"ERR invalid argument '4'"}, - {"SET", "mykey", "myid_a2", "FIELD", "a", 2, "POINT", 32.99, -115}, {"OK"}, - {"SET", "mykey", "myid_a3", "FIELD", "a", 3, "POINT", 33, -115.02}, {"OK"}, - {"WITHIN", "mykey", "WHEREEVAL", "return FIELDS.a > tonumber(ARGV[1]) and FIELDS.a ~= tonumber(ARGV[2])", 2, 0.5, 3, "BOUNDS", 32.8, -115.2, 33.2, -114.8}, {`[0 [[myid_a2 {"type":"Point","coordinates":[-115,32.99]} [a 2]] [myid_a1 {"type":"Point","coordinates":[-115,33]} [a 1]]]]`}, - }) + return mc.DoBatch( + Do("SET", "mykey", "myid_a1", "FIELD", "a", 1, "POINT", 33, -115).OK(), + Do("WITHIN", "mykey", "WHEREEVAL", "return FIELDS.a > tonumber(ARGV[1])", 1, 0.5, "BOUNDS", 32.8, -115.2, 33.2, -114.8).Str(`[0 [[myid_a1 {"type":"Point","coordinates":[-115,33]} [a 1]]]]`), + Do("WITHIN", "mykey", "WHEREEVAL", "return FIELDS.a > tonumber(ARGV[1])", "a", 0.5, "BOUNDS", 32.8, -115.2, 33.2, -114.8).Err("invalid argument 'a'"), + Do("WITHIN", "mykey", "WHEREEVAL", "return FIELDS.a > tonumber(ARGV[1])", 1, 0.5, 4, "BOUNDS", 32.8, -115.2, 33.2, -114.8).Err("invalid argument '4'"), + Do("SET", "mykey", "myid_a2", "FIELD", "a", 2, "POINT", 32.99, -115).OK(), + Do("SET", "mykey", "myid_a3", "FIELD", "a", 3, "POINT", 33, -115.02).OK(), + Do("WITHIN", "mykey", "WHEREEVAL", "return FIELDS.a > tonumber(ARGV[1]) and FIELDS.a ~= tonumber(ARGV[2])", 2, 0.5, 3, "BOUNDS", 32.8, -115.2, 33.2, -114.8).Str(`[0 [[myid_a2 {"type":"Point","coordinates":[-115,32.99]} [a 2]] [myid_a1 {"type":"Point","coordinates":[-115,33]} [a 1]]]]`), + ) } func keys_TYPE_test(mc *mockServer) error { @@ -573,3 +574,50 @@ func keys_HEALTHZ_test(mc *mockServer) error { Do("HEALTHZ", "arg").Err(`wrong number of arguments for 'healthz' command`), ) } + +func keys_SERVER_test(mc *mockServer) error { + return mc.DoBatch( + Do("SERVER").Func(func(s string) error { + valid := strings.HasPrefix(s, "[") && strings.HasSuffix(s, "]") && + strings.Contains(s, "cpus") && strings.Contains(s, "mem_alloc") + if !valid { + return errors.New("looks invalid") + } + return nil + }), + Do("SERVER").JSON().Func(func(s string) error { + if !gjson.Get(s, "ok").Bool() { + return errors.New("not ok") + } + valid := gjson.Get(s, "stats.cpus").Exists() && + gjson.Get(s, "stats.mem_alloc").Exists() + if !valid { + return errors.New("looks invalid") + } + return nil + }), + Do("SERVER", "ext").Func(func(s string) error { + valid := strings.HasPrefix(s, "[") && strings.HasSuffix(s, "]") && + strings.Contains(s, "sys_cpus") && + strings.Contains(s, "tile38_connected_clients") + + if !valid { + return errors.New("looks invalid") + } + return nil + }), + Do("SERVER", "ext").JSON().Func(func(s string) error { + if !gjson.Get(s, "ok").Bool() { + return errors.New("not ok") + } + valid := gjson.Get(s, "stats.sys_cpus").Exists() && + gjson.Get(s, "stats.tile38_connected_clients").Exists() + if !valid { + return errors.New("looks invalid") + } + return nil + }), + Do("SERVER", "ett").Err(`invalid argument 'ett'`), + Do("SERVER", "ett").JSON().Err(`invalid argument 'ett'`), + ) +} From 54609980869002124609238bdd87bab9ace1a2b5 Mon Sep 17 00:00:00 2001 From: tidwall Date: Sat, 24 Sep 2022 06:22:58 -0700 Subject: [PATCH 12/33] wip - fixing the empty response error --- internal/server/client.go | 35 +++++++++++----------------- internal/server/server.go | 5 ++-- tests/client_test.go | 49 +++++++++++++++++++++++++++++---------- tests/tests_test.go | 6 ++--- 4 files changed, 56 insertions(+), 39 deletions(-) diff --git a/internal/server/client.go b/internal/server/client.go index 6af0a108..ab23550f 100644 --- a/internal/server/client.go +++ b/internal/server/client.go @@ -52,7 +52,8 @@ func (arr byID) Swap(a, b int) { arr[a], arr[b] = arr[b], arr[a] } -func (s *Server) cmdClient(msg *Message, client *Client) (resp.Value, error) { +// CLIENT (LIST | KILL | GETNAME | SETNAME) +func (s *Server) cmdCLIENT(msg *Message, client *Client) (resp.Value, error) { start := time.Now() if len(msg.Args) == 1 { @@ -89,8 +90,7 @@ func (s *Server) cmdClient(msg *Message, client *Client) (resp.Value, error) { ) client.mu.Unlock() } - switch msg.OutputType { - case JSON: + if msg.OutputType == JSON { // Create a map of all key/value info fields var cmap []map[string]interface{} clients := strings.Split(string(buf), "\n") @@ -110,32 +110,23 @@ func (s *Server) cmdClient(msg *Message, client *Client) (resp.Value, error) { } } - // Marshal the map and use the output in the JSON response - data, err := json.Marshal(cmap) - if err != nil { - return NOMessage, err - } - return resp.StringValue(`{"ok":true,"list":` + string(data) + `,"elapsed":"` + time.Since(start).String() + "\"}"), nil - case RESP: - return resp.BytesValue(buf), nil + data, _ := json.Marshal(cmap) + return resp.StringValue(`{"ok":true,"list":` + string(data) + + `,"elapsed":"` + time.Since(start).String() + "\"}"), nil } - return NOMessage, nil + return resp.BytesValue(buf), nil case "getname": if len(msg.Args) != 2 { return NOMessage, errInvalidNumberOfArguments } - name := "" - switch msg.OutputType { - case JSON: - client.mu.Lock() - name := client.name - client.mu.Unlock() - return resp.StringValue(`{"ok":true,"name":` + - jsonString(name) + + client.mu.Lock() + name := client.name + client.mu.Unlock() + if msg.OutputType == JSON { + return resp.StringValue(`{"ok":true,"name":` + jsonString(name) + `,"elapsed":"` + time.Since(start).String() + "\"}"), nil - case RESP: - return resp.StringValue(name), nil } + return resp.StringValue(name), nil case "setname": if len(msg.Args) != 3 { return NOMessage, errInvalidNumberOfArguments diff --git a/internal/server/server.go b/internal/server/server.go index d7c6a399..4365dab4 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -977,7 +977,7 @@ func (s *Server) handleInputCommand(client *Client, msg *Message) error { return err } } - if !isRespValueEmptyString(res) { + if false || !isRespValueEmptyString(res) { var resStr string resStr, err := serializeOutput(res) if err != nil { @@ -1140,7 +1140,7 @@ func (s *Server) command(msg *Message, client *Client) ( return s.command(msg, client) } case "client": - res, err = s.cmdClient(msg, client) + res, err = s.cmdCLIENT(msg, client) case "eval", "evalro", "evalna": res, err = s.cmdEvalUnified(false, msg) case "evalsha", "evalrosha", "evalnasha": @@ -1162,6 +1162,7 @@ func (s *Server) command(msg *Message, client *Client) ( case "monitor": res, err = s.cmdMonitor(msg) } + s.sendMonitor(err, msg, client, false) return } diff --git a/tests/client_test.go b/tests/client_test.go index 016b88aa..941ce6e7 100644 --- a/tests/client_test.go +++ b/tests/client_test.go @@ -3,6 +3,7 @@ package tests import ( "errors" "fmt" + "strings" "testing" "github.com/gomodule/redigo/redis" @@ -10,18 +11,19 @@ import ( ) func subTestClient(t *testing.T, mc *mockServer) { - runStep(t, mc, "valid json", client_valid_json_test) - runStep(t, mc, "valid client count", info_valid_client_count_test) + runStep(t, mc, "OUTPUT", client_OUTPUT_test) + runStep(t, mc, "CLIENT", client_CLIENT_test) } -func client_valid_json_test(mc *mockServer) error { - if err := mc.DoBatch([][]interface{}{ +func client_OUTPUT_test(mc *mockServer) error { + if err := mc.DoBatch( // tests removal of "elapsed" member. - {"OUTPUT", "json"}, {`{"ok":true}`}, - {"OUTPUT", "resp"}, {`OK`}, - }); err != nil { + Do("OUTPUT", "json").Str(`{"ok":true}`), + Do("OUTPUT", "resp").OK(), + ); err != nil { return err } + // run direct commands if _, err := mc.Do("OUTPUT", "json"); err != nil { return err @@ -45,9 +47,14 @@ func client_valid_json_test(mc *mockServer) error { return nil } -func info_valid_client_count_test(mc *mockServer) error { +func client_CLIENT_test(mc *mockServer) error { numConns := 20 var conns []redis.Conn + defer func() { + for i := range conns { + conns[i].Close() + } + }() for i := 0; i <= numConns; i++ { conn, err := redis.Dial("tcp", fmt.Sprintf(":%d", mc.port)) if err != nil { @@ -55,9 +62,6 @@ func info_valid_client_count_test(mc *mockServer) error { } conns = append(conns, conn) } - for i := range conns { - defer conns[i].Close() - } if _, err := mc.Do("OUTPUT", "JSON"); err != nil { return err } @@ -73,5 +77,26 @@ func info_valid_client_count_test(mc *mockServer) error { if len(gjson.Get(sres, "list").Array()) < numConns { return errors.New("Invalid number of connections") } - return nil + + return mc.DoBatch( + Do("CLIENT", "list").JSON().Func(func(s string) error { + if int(gjson.Get(s, "list.#").Int()) < numConns { + return errors.New("Invalid number of connections") + } + return nil + }), + Do("CLIENT", "list").Func(func(s string) error { + if len(strings.Split(strings.TrimSpace(s), "\n")) < numConns { + return errors.New("Invalid number of connections") + } + return nil + }), + Do("CLIENT").Err(`wrong number of arguments for 'client' command`), + Do("CLIENT", "hello").Err(`Syntax error, try CLIENT (LIST | KILL | GETNAME | SETNAME)`), + Do("CLIENT", "list", "arg3").Err(`wrong number of arguments for 'client' command`), + Do("CLIENT", "getname", "arg3").Err(`wrong number of arguments for 'client' command`), + Do("CLIENT", "getname").JSON().Str(`{"ok":true,"name":""}`), + Do("CLIENT", "getname").Str(``), + ) + } diff --git a/tests/tests_test.go b/tests/tests_test.go index be9d3b3a..3fb3b424 100644 --- a/tests/tests_test.go +++ b/tests/tests_test.go @@ -56,10 +56,10 @@ func TestAll(t *testing.T) { runSubTest(t, "json", mc, subTestJSON) runSubTest(t, "search", mc, subTestSearch) runSubTest(t, "testcmd", mc, subTestTestCmd) - runSubTest(t, "fence", mc, subTestFence) - runSubTest(t, "scripts", mc, subTestScripts) - runSubTest(t, "info", mc, subTestInfo) runSubTest(t, "client", mc, subTestClient) + runSubTest(t, "scripts", mc, subTestScripts) + runSubTest(t, "fence", mc, subTestFence) + runSubTest(t, "info", mc, subTestInfo) runSubTest(t, "timeouts", mc, subTestTimeout) runSubTest(t, "metrics", mc, subTestMetrics) } From e6cced4c4a249633f81809dd268aede61d856536 Mon Sep 17 00:00:00 2001 From: tidwall Date: Sat, 24 Sep 2022 07:22:39 -0700 Subject: [PATCH 13/33] Fix hang on empty RESP response --- internal/server/server.go | 22 ++++++++-------------- tests/timeout_test.go | 21 ++++++++++----------- 2 files changed, 18 insertions(+), 25 deletions(-) diff --git a/internal/server/server.go b/internal/server/server.go index 4365dab4..b1883d6c 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -949,7 +949,7 @@ func (s *Server) handleInputCommand(client *Client, msg *Message) error { } } res = NOMessage - err = writeErr("timeout") + err = errTimeout } }() } @@ -977,23 +977,17 @@ func (s *Server) handleInputCommand(client *Client, msg *Message) error { return err } } - if false || !isRespValueEmptyString(res) { - var resStr string - resStr, err := serializeOutput(res) - if err != nil { - return err - } - if err := writeOutput(resStr); err != nil { - return err - } + var resStr string + resStr, err = serializeOutput(res) + if err != nil { + return err + } + if err := writeOutput(resStr); err != nil { + return err } return nil } -func isRespValueEmptyString(val resp.Value) bool { - return !val.IsNull() && (val.Type() == resp.SimpleString || val.Type() == resp.BulkString) && len(val.Bytes()) == 0 -} - func randomKey(n int) string { b := make([]byte, n) nn, err := rand.Read(b) diff --git a/tests/timeout_test.go b/tests/timeout_test.go index 1663e47b..123971b4 100644 --- a/tests/timeout_test.go +++ b/tests/timeout_test.go @@ -54,21 +54,20 @@ func setup(mc *mockServer, count int, points bool) (err error) { return } -func timeout_spatial_test(mc *mockServer) (err error) { - err = setup(mc, 10000, true) +func timeout_spatial_test(mc *mockServer) error { + err := setup(mc, 10000, true) if err != nil { return err } + return mc.DoBatch( + Do("SCAN", "mykey", "WHERE", "foo", -1, 2, "COUNT").Str("10000"), + Do("INTERSECTS", "mykey", "WHERE", "foo", -1, 2, "COUNT", "BOUNDS", -90, -180, 90, 180).Str("10000"), + Do("WITHIN", "mykey", "WHERE", "foo", -1, 2, "COUNT", "BOUNDS", -90, -180, 90, 180).Str("10000"), - return mc.DoBatch([][]interface{}{ - {"SCAN", "mykey", "WHERE", "foo", -1, 2, "COUNT"}, {"10000"}, - {"INTERSECTS", "mykey", "WHERE", "foo", -1, 2, "COUNT", "BOUNDS", -90, -180, 90, 180}, {"10000"}, - {"WITHIN", "mykey", "WHERE", "foo", -1, 2, "COUNT", "BOUNDS", -90, -180, 90, 180}, {"10000"}, - - {"TIMEOUT", "0.000001", "SCAN", "mykey", "WHERE", "foo", -1, 2, "COUNT"}, {"ERR timeout"}, - {"TIMEOUT", "0.000001", "INTERSECTS", "mykey", "WHERE", "foo", -1, 2, "COUNT", "BOUNDS", -90, -180, 90, 180}, {"ERR timeout"}, - {"TIMEOUT", "0.000001", "WITHIN", "mykey", "WHERE", "foo", -1, 2, "COUNT", "BOUNDS", -90, -180, 90, 180}, {"ERR timeout"}, - }) + Do("TIMEOUT", "0.000001", "SCAN", "mykey", "WHERE", "foo", -1, 2, "COUNT").Err("timeout"), + Do("TIMEOUT", "0.000001", "INTERSECTS", "mykey", "WHERE", "foo", -1, 2, "COUNT", "BOUNDS", -90, -180, 90, 180).Err("timeout"), + Do("TIMEOUT", "0.000001", "WITHIN", "mykey", "WHERE", "foo", -1, 2, "COUNT", "BOUNDS", -90, -180, 90, 180).Err("timeout"), + ) } func timeout_search_test(mc *mockServer) (err error) { From d8ecbba0be8100bf47b4369deeaff5471cea3270 Mon Sep 17 00:00:00 2001 From: tidwall Date: Sat, 24 Sep 2022 13:41:36 -0700 Subject: [PATCH 14/33] Better CLIENT tests --- internal/server/client.go | 165 ++++++++++++++++++-------------------- internal/server/server.go | 1 + tests/client_test.go | 38 ++++++++- tests/mock_io_test.go | 144 +++++---------------------------- 4 files changed, 131 insertions(+), 217 deletions(-) diff --git a/internal/server/client.go b/internal/server/client.go index ab23550f..71881326 100644 --- a/internal/server/client.go +++ b/internal/server/client.go @@ -2,7 +2,6 @@ package server import ( "encoding/json" - "errors" "fmt" "io" "sort" @@ -32,6 +31,8 @@ type Client struct { name string // optional defined name opened time.Time // when the client was created/opened, unix nano last time.Time // last client request/response, unix nano + + closer io.Closer // used to close the connection } // Write ... @@ -40,33 +41,19 @@ func (client *Client) Write(b []byte) (n int, err error) { return len(b), nil } -type byID []*Client - -func (arr byID) Len() int { - return len(arr) -} -func (arr byID) Less(a, b int) bool { - return arr[a].id < arr[b].id -} -func (arr byID) Swap(a, b int) { - arr[a], arr[b] = arr[b], arr[a] -} - // CLIENT (LIST | KILL | GETNAME | SETNAME) -func (s *Server) cmdCLIENT(msg *Message, client *Client) (resp.Value, error) { +func (s *Server) cmdCLIENT(_msg *Message, client *Client) (resp.Value, error) { start := time.Now() - if len(msg.Args) == 1 { - return NOMessage, errInvalidNumberOfArguments + args := _msg.Args + if len(args) == 1 { + return retrerr(errInvalidNumberOfArguments) } - switch strings.ToLower(msg.Args[1]) { - default: - return NOMessage, clientErrorf( - "Syntax error, try CLIENT (LIST | KILL | GETNAME | SETNAME)", - ) + + switch strings.ToLower(args[1]) { case "list": - if len(msg.Args) != 2 { - return NOMessage, errInvalidNumberOfArguments + if len(args) != 2 { + return retrerr(errInvalidNumberOfArguments) } var list []*Client s.connsmu.RLock() @@ -74,7 +61,9 @@ func (s *Server) cmdCLIENT(msg *Message, client *Client) (resp.Value, error) { list = append(list, cc) } s.connsmu.RUnlock() - sort.Sort(byID(list)) + sort.Slice(list, func(i, j int) bool { + return list[i].id < list[j].id + }) now := time.Now() var buf []byte for _, client := range list { @@ -90,7 +79,7 @@ func (s *Server) cmdCLIENT(msg *Message, client *Client) (resp.Value, error) { ) client.mu.Unlock() } - if msg.OutputType == JSON { + if _msg.OutputType == JSON { // Create a map of all key/value info fields var cmap []map[string]interface{} clients := strings.Split(string(buf), "\n") @@ -116,109 +105,107 @@ func (s *Server) cmdCLIENT(msg *Message, client *Client) (resp.Value, error) { } return resp.BytesValue(buf), nil case "getname": - if len(msg.Args) != 2 { - return NOMessage, errInvalidNumberOfArguments + if len(args) != 2 { + return retrerr(errInvalidNumberOfArguments) } client.mu.Lock() name := client.name client.mu.Unlock() - if msg.OutputType == JSON { + if _msg.OutputType == JSON { return resp.StringValue(`{"ok":true,"name":` + jsonString(name) + `,"elapsed":"` + time.Since(start).String() + "\"}"), nil } return resp.StringValue(name), nil case "setname": - if len(msg.Args) != 3 { - return NOMessage, errInvalidNumberOfArguments + if len(args) != 3 { + return retrerr(errInvalidNumberOfArguments) } - name := msg.Args[2] + name := _msg.Args[2] for i := 0; i < len(name); i++ { if name[i] < '!' || name[i] > '~' { - return NOMessage, clientErrorf( + return retrerr(clientErrorf( "Client names cannot contain spaces, newlines or special characters.", - ) + )) } } client.mu.Lock() client.name = name client.mu.Unlock() - switch msg.OutputType { - case JSON: - return resp.StringValue(`{"ok":true,"elapsed":"` + time.Since(start).String() + "\"}"), nil - case RESP: - return resp.SimpleStringValue("OK"), nil + if _msg.OutputType == JSON { + return resp.StringValue(`{"ok":true,"elapsed":"` + + time.Since(start).String() + "\"}"), nil } + return resp.SimpleStringValue("OK"), nil case "kill": - if len(msg.Args) < 3 { - return NOMessage, errInvalidNumberOfArguments + if len(args) < 3 { + return retrerr(errInvalidNumberOfArguments) } var useAddr bool var addr string var useID bool var id string - for i := 2; i < len(msg.Args); i++ { - arg := msg.Args[i] + for i := 2; i < len(args); i++ { + if useAddr || useID { + return retrerr(errInvalidNumberOfArguments) + } + arg := args[i] if strings.Contains(arg, ":") { addr = arg useAddr = true - break - } - switch strings.ToLower(arg) { - default: - return NOMessage, clientErrorf("No such client") - case "addr": - i++ - if i == len(msg.Args) { - return NOMessage, errors.New("syntax error") + } else { + switch strings.ToLower(arg) { + case "addr": + i++ + if i == len(args) { + return retrerr(errInvalidNumberOfArguments) + } + addr = args[i] + useAddr = true + case "id": + i++ + if i == len(args) { + return retrerr(errInvalidNumberOfArguments) + } + id = args[i] + useID = true + default: + return retrerr(clientErrorf("No such client")) } - addr = msg.Args[i] - useAddr = true - case "id": - i++ - if i == len(msg.Args) { - return NOMessage, errors.New("syntax error") - } - id = msg.Args[i] - useID = true } } - var cclose *Client + var closing []io.Closer s.connsmu.RLock() for _, cc := range s.conns { if useID && fmt.Sprintf("%d", cc.id) == id { - cclose = cc - break - } else if useAddr && client.remoteAddr == addr { - cclose = cc - break + if cc.closer != nil { + closing = append(closing, cc.closer) + } + } else if useAddr { + if cc.remoteAddr == addr { + if cc.closer != nil { + closing = append(closing, cc.closer) + } + } } } s.connsmu.RUnlock() - if cclose == nil { - return NOMessage, clientErrorf("No such client") + if len(closing) == 0 { + return retrerr(clientErrorf("No such client")) } - - var res resp.Value - switch msg.OutputType { - case JSON: - res = resp.StringValue(`{"ok":true,"elapsed":"` + time.Since(start).String() + "\"}") - case RESP: - res = resp.SimpleStringValue("OK") + // go func() { + // close the connections behind the scene + for _, closer := range closing { + closer.Close() } - - client.conn.Close() - // closing self, return response now - // NOTE: This is the only exception where we do convert response to a string - var outBytes []byte - switch msg.OutputType { - case JSON: - outBytes = res.Bytes() - case RESP: - outBytes, _ = res.MarshalRESP() + // }() + if _msg.OutputType == JSON { + return resp.StringValue(`{"ok":true,"elapsed":"` + + time.Since(start).String() + "\"}"), nil } - cclose.conn.Write(outBytes) - cclose.conn.Close() - return res, nil + return resp.SimpleStringValue("OK"), nil + default: + return retrerr(clientErrorf( + "Syntax error, try CLIENT (LIST | KILL | GETNAME | SETNAME)", + )) } - return NOMessage, errors.New("invalid output type") } diff --git a/internal/server/server.go b/internal/server/server.go index b1883d6c..b51ded5e 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -376,6 +376,7 @@ func (s *Server) netServe() error { client.id = int(atomic.AddInt64(&clientID, 1)) client.opened = time.Now() client.remoteAddr = conn.RemoteAddr().String() + client.closer = conn // add client to server map s.connsmu.Lock() diff --git a/tests/client_test.go b/tests/client_test.go index 941ce6e7..3d75e18f 100644 --- a/tests/client_test.go +++ b/tests/client_test.go @@ -8,6 +8,7 @@ import ( "github.com/gomodule/redigo/redis" "github.com/tidwall/gjson" + "github.com/tidwall/pretty" ) func subTestClient(t *testing.T, mc *mockServer) { @@ -62,6 +63,16 @@ func client_CLIENT_test(mc *mockServer) error { } conns = append(conns, conn) } + + _, err := conns[1].Do("CLIENT", "setname", "cl1") + if err != nil { + return err + } + _, err = conns[2].Do("CLIENT", "setname", "cl2") + if err != nil { + return err + } + if _, err := mc.Do("OUTPUT", "JSON"); err != nil { return err } @@ -73,11 +84,15 @@ func client_CLIENT_test(mc *mockServer) error { if !ok { return errors.New("Failed to type assert CLIENT response") } - sres := string(bres) - if len(gjson.Get(sres, "list").Array()) < numConns { + sres := string(pretty.Pretty(bres)) + if int(gjson.Get(sres, "list.#").Int()) < numConns { return errors.New("Invalid number of connections") } + client13ID := gjson.Get(sres, "list.13.id").String() + client14Addr := gjson.Get(sres, "list.14.addr").String() + client15Addr := gjson.Get(sres, "list.15.addr").String() + return mc.DoBatch( Do("CLIENT", "list").JSON().Func(func(s string) error { if int(gjson.Get(s, "list.#").Int()) < numConns { @@ -97,6 +112,25 @@ func client_CLIENT_test(mc *mockServer) error { Do("CLIENT", "getname", "arg3").Err(`wrong number of arguments for 'client' command`), Do("CLIENT", "getname").JSON().Str(`{"ok":true,"name":""}`), Do("CLIENT", "getname").Str(``), + Do("CLIENT", "setname", "abc").OK(), + Do("CLIENT", "getname").Str(`abc`), + Do("CLIENT", "getname").JSON().Str(`{"ok":true,"name":"abc"}`), + Do("CLIENT", "setname", "abc", "efg").Err(`wrong number of arguments for 'client' command`), + Do("CLIENT", "setname", " abc ").Err(`Client names cannot contain spaces, newlines or special characters.`), + Do("CLIENT", "setname", "abcd").JSON().OK(), + Do("CLIENT", "kill", "name", "abcd").Err("No such client"), + Do("CLIENT", "getname").Str(`abcd`), + Do("CLIENT", "kill").Err(`wrong number of arguments for 'client' command`), + Do("CLIENT", "kill", "").Err(`No such client`), + Do("CLIENT", "kill", "abcd").Err(`No such client`), + Do("CLIENT", "kill", "id", client13ID).OK(), + Do("CLIENT", "kill", "id").Err("wrong number of arguments for 'client' command"), + Do("CLIENT", "kill", client14Addr).OK(), + Do("CLIENT", "kill", client14Addr, "yikes").Err("wrong number of arguments for 'client' command"), + Do("CLIENT", "kill", "addr").Err("wrong number of arguments for 'client' command"), + Do("CLIENT", "kill", "addr", client15Addr).JSON().OK(), + Do("CLIENT", "kill", "addr", client14Addr, "yikes").Err("wrong number of arguments for 'client' command"), + Do("CLIENT", "kill", "id", "1000").Err("No such client"), ) } diff --git a/tests/mock_io_test.go b/tests/mock_io_test.go index 90ed6960..003aa9d8 100644 --- a/tests/mock_io_test.go +++ b/tests/mock_io_test.go @@ -3,9 +3,6 @@ package tests import ( "errors" "fmt" - "go/ast" - "go/parser" - "go/token" "os" "path/filepath" "runtime" @@ -21,10 +18,13 @@ type IO struct { out any sleep bool dur time.Duration + cfile string + cln int } func Do(args ...any) *IO { - return &IO{args: args} + _, cfile, cln, _ := runtime.Caller(1) + return &IO{args: args, cfile: cfile, cln: cln} } func (cmd *IO) JSON() *IO { cmd.json = true @@ -83,136 +83,29 @@ func Sleep(duration time.Duration) *IO { return &IO{sleep: true, dur: duration} } -type ioVisitor struct { - fset *token.FileSet - ln int - pos int - got bool - data string - end int - done bool - index int - nidx int - frag string - fpos int -} - -func (v *ioVisitor) Visit(n ast.Node) ast.Visitor { - if n == nil || v.done { - return nil - } - - if v.got { - if int(n.Pos()) > v.end { - v.done = true - return v - } - if n, ok := n.(*ast.CallExpr); ok { - frag := strings.TrimSpace(v.data[int(n.Pos())-1 : int(n.End())]) - if _, ok := n.Fun.(*ast.Ident); ok { - if v.index == v.nidx { - frag = strings.TrimSpace(strings.TrimSuffix(frag, ".")) - idx := strings.IndexByte(frag, '(') - if idx != -1 { - frag = frag[idx:] - } - v.frag = frag - v.done = true - v.fpos = int(n.Pos()) - return v - } - v.nidx++ - } - } - return v - } - if int(n.Pos()) == v.pos { - if n, ok := n.(*ast.CallExpr); ok { - v.end = int(n.Rparen) - v.got = true - return v - } - } - return v -} - func (cmd *IO) deepError(index int, err error) error { - oerr := err - werr := func(err error) error { - return fmt.Errorf("batch[%d]: %v: %v", index, oerr, err) - } - - // analyse stack - _, file, ln, ok := runtime.Caller(3) - if !ok { - return werr(errors.New("runtime.Caller failed")) - } - - // get the character position from line - bdata, err := os.ReadFile(file) - if err != nil { - return werr(err) - } + frag := "(?)" + bdata, _ := os.ReadFile(cmd.cfile) data := string(bdata) - - var pos int - var iln int - var pln int + ln := 1 for i := 0; i < len(data); i++ { if data[i] == '\n' { - j := pln - line := data[pln:i] - pln = i + 1 - iln++ - if iln == ln { - line = strings.TrimSpace(line) - if !strings.HasPrefix(line, "return mc.DoBatch(") { - return oerr - } - for ; j < len(data); j++ { - if data[j] == 'm' { - break + ln++ + if ln == cmd.cln { + data = data[i+1:] + i = strings.IndexByte(data, '(') + if i != -1 { + j := strings.IndexByte(data[i:], ')') + if j != -1 { + frag = string(data[i : j+i+1]) } } - pos = j + 1 break } } } - if pos == 0 { - return oerr - } - - fset := token.NewFileSet() - pfile, err := parser.ParseFile(fset, file, nil, 0) - if err != nil { - return werr(err) - } - v := &ioVisitor{ - fset: fset, - ln: ln, - pos: pos, - data: string(data), - index: index, - } - ast.Walk(v, pfile) - - if v.fpos == 0 { - return oerr - } - - pln = 1 - for i := 0; i < len(data); i++ { - if data[i] == '\n' { - if i > v.fpos { - break - } - pln++ - } - } - - fsig := fmt.Sprintf("%s:%d", filepath.Base(file), pln) - emsg := oerr.Error() + fsig := fmt.Sprintf("%s:%d", filepath.Base(cmd.cfile), cmd.cln) + emsg := err.Error() if strings.HasPrefix(emsg, "expected ") && strings.Contains(emsg, ", got ") { emsg = "" + @@ -223,9 +116,8 @@ func (cmd *IO) deepError(index int, err error) error { emsg = "" + " ERROR: " + emsg } - return fmt.Errorf("\n%s: entry[%d]\n COMMAND: %s\n%s", - fsig, index+1, v.frag, emsg) + fsig, index+1, frag, emsg) } func (mc *mockServer) doIOTest(index int, cmd *IO) error { From 891fd10ef6fba42622f143f9e8090df4f8bf5fee Mon Sep 17 00:00:00 2001 From: tidwall Date: Sat, 24 Sep 2022 13:57:03 -0700 Subject: [PATCH 15/33] Added bson tests --- internal/server/bson.go | 17 +++++----------- internal/server/bson_test.go | 10 ++++++++++ internal/server/must.go | 16 +++++++++++++++ internal/server/must_test.go | 38 ++++++++++++++++++++++++++++++++++++ 4 files changed, 69 insertions(+), 12 deletions(-) create mode 100644 internal/server/bson_test.go create mode 100644 internal/server/must.go create mode 100644 internal/server/must_test.go diff --git a/internal/server/bson.go b/internal/server/bson.go index bcc9d2ea..c65adcbb 100644 --- a/internal/server/bson.go +++ b/internal/server/bson.go @@ -5,7 +5,6 @@ import ( "crypto/rand" "encoding/binary" "encoding/hex" - "io" "os" "sync/atomic" "time" @@ -23,23 +22,17 @@ func bsonID() string { var ( bsonProcess = uint16(os.Getpid()) bsonMachine = func() []byte { - host, err := os.Hostname() - if err != nil { - b := make([]byte, 3) - if _, err := io.ReadFull(rand.Reader, b); err != nil { - panic("random error: " + err.Error()) - } - return b - } + host, _ := os.Hostname() + b := make([]byte, 3) + Must(rand.Read(b)) + host = Default(host, string(b)) hw := md5.New() hw.Write([]byte(host)) return hw.Sum(nil)[:3] }() bsonCounter = func() uint32 { b := make([]byte, 4) - if _, err := io.ReadFull(rand.Reader, b); err != nil { - panic("random error: " + err.Error()) - } + Must(rand.Read(b)) return binary.BigEndian.Uint32(b) }() ) diff --git a/internal/server/bson_test.go b/internal/server/bson_test.go new file mode 100644 index 00000000..ca157446 --- /dev/null +++ b/internal/server/bson_test.go @@ -0,0 +1,10 @@ +package server + +import "testing" + +func TestBSON(t *testing.T) { + id := bsonID() + if len(id) != 24 { + t.Fail() + } +} diff --git a/internal/server/must.go b/internal/server/must.go new file mode 100644 index 00000000..b50ba653 --- /dev/null +++ b/internal/server/must.go @@ -0,0 +1,16 @@ +package server + +func Must[T any](a T, err error) T { + if err != nil { + panic(err) + } + return a +} + +func Default[T comparable](a, b T) T { + var c T + if a == c { + return b + } + return a +} diff --git a/internal/server/must_test.go b/internal/server/must_test.go new file mode 100644 index 00000000..de3bcade --- /dev/null +++ b/internal/server/must_test.go @@ -0,0 +1,38 @@ +package server + +import ( + "errors" + "testing" +) + +func TestMust(t *testing.T) { + if Must(1, nil) != 1 { + t.Fail() + } + func() { + var ended bool + defer func() { + if ended { + t.Fail() + } + err, ok := recover().(error) + if !ok { + t.Fail() + } + if err.Error() != "ok" { + t.Fail() + } + }() + Must(1, errors.New("ok")) + ended = true + }() +} + +func TestDefault(t *testing.T) { + if Default("", "2") != "2" { + t.Fail() + } + if Default("1", "2") != "1" { + t.Fail() + } +} From 1001de7311d526f0b4ef79721003dc787c55b6ed Mon Sep 17 00:00:00 2001 From: tidwall Date: Sat, 24 Sep 2022 14:01:36 -0700 Subject: [PATCH 16/33] Refactor for better coverage --- internal/server/metrics.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/internal/server/metrics.go b/internal/server/metrics.go index a5cd4623..78133ceb 100644 --- a/internal/server/metrics.go +++ b/internal/server/metrics.go @@ -92,9 +92,14 @@ func (s *Server) Collect(ch chan<- prometheus.Metric) { s.extStats(m) for metric, descr := range metricDescriptions { - if val, ok := m[metric].(int); ok { - ch <- prometheus.MustNewConstMetric(descr, prometheus.GaugeValue, float64(val)) - } else if val, ok := m[metric].(float64); ok { + val, ok := m[metric].(float64) + if !ok { + val2, ok2 := m[metric].(int) + if ok2 { + val, ok = float64(val2), true + } + } + if ok { ch <- prometheus.MustNewConstMetric(descr, prometheus.GaugeValue, val) } } From 0301545fe639abda949a1dd424246f8738b50fac Mon Sep 17 00:00:00 2001 From: tidwall Date: Sat, 24 Sep 2022 14:28:47 -0700 Subject: [PATCH 17/33] Better INFO tests --- internal/server/client.go | 14 ++++---- internal/server/server.go | 2 +- internal/server/stats.go | 68 ++++++++++++++++++++++++------------ tests/keys_test.go | 73 ++++++++++++++++++++++++++------------- 4 files changed, 102 insertions(+), 55 deletions(-) diff --git a/internal/server/client.go b/internal/server/client.go index 71881326..67fedf75 100644 --- a/internal/server/client.go +++ b/internal/server/client.go @@ -42,10 +42,10 @@ func (client *Client) Write(b []byte) (n int, err error) { } // CLIENT (LIST | KILL | GETNAME | SETNAME) -func (s *Server) cmdCLIENT(_msg *Message, client *Client) (resp.Value, error) { +func (s *Server) cmdCLIENT(msg *Message, client *Client) (resp.Value, error) { start := time.Now() - args := _msg.Args + args := msg.Args if len(args) == 1 { return retrerr(errInvalidNumberOfArguments) } @@ -79,7 +79,7 @@ func (s *Server) cmdCLIENT(_msg *Message, client *Client) (resp.Value, error) { ) client.mu.Unlock() } - if _msg.OutputType == JSON { + if msg.OutputType == JSON { // Create a map of all key/value info fields var cmap []map[string]interface{} clients := strings.Split(string(buf), "\n") @@ -111,7 +111,7 @@ func (s *Server) cmdCLIENT(_msg *Message, client *Client) (resp.Value, error) { client.mu.Lock() name := client.name client.mu.Unlock() - if _msg.OutputType == JSON { + if msg.OutputType == JSON { return resp.StringValue(`{"ok":true,"name":` + jsonString(name) + `,"elapsed":"` + time.Since(start).String() + "\"}"), nil } @@ -120,7 +120,7 @@ func (s *Server) cmdCLIENT(_msg *Message, client *Client) (resp.Value, error) { if len(args) != 3 { return retrerr(errInvalidNumberOfArguments) } - name := _msg.Args[2] + name := msg.Args[2] for i := 0; i < len(name); i++ { if name[i] < '!' || name[i] > '~' { return retrerr(clientErrorf( @@ -131,7 +131,7 @@ func (s *Server) cmdCLIENT(_msg *Message, client *Client) (resp.Value, error) { client.mu.Lock() client.name = name client.mu.Unlock() - if _msg.OutputType == JSON { + if msg.OutputType == JSON { return resp.StringValue(`{"ok":true,"elapsed":"` + time.Since(start).String() + "\"}"), nil } @@ -198,7 +198,7 @@ func (s *Server) cmdCLIENT(_msg *Message, client *Client) (resp.Value, error) { closer.Close() } // }() - if _msg.OutputType == JSON { + if msg.OutputType == JSON { return resp.StringValue(`{"ok":true,"elapsed":"` + time.Since(start).String() + "\"}"), nil } diff --git a/internal/server/server.go b/internal/server/server.go index b51ded5e..f3093320 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -1081,7 +1081,7 @@ func (s *Server) command(msg *Message, client *Client) ( case "healthz": res, err = s.cmdHEALTHZ(msg) case "info": - res, err = s.cmdInfo(msg) + res, err = s.cmdINFO(msg) case "scan": res, err = s.cmdScan(msg) case "nearby": diff --git a/internal/server/stats.go b/internal/server/stats.go index 12187016..c0ab0bc4 100644 --- a/internal/server/stats.go +++ b/internal/server/stats.go @@ -457,27 +457,52 @@ func (s *Server) writeInfoCluster(w *bytes.Buffer) { fmt.Fprintf(w, "cluster_enabled:0\r\n") } -func (s *Server) cmdInfo(msg *Message) (res resp.Value, err error) { +// INFO [section ...] +func (s *Server) cmdINFO(msg *Message) (res resp.Value, err error) { start := time.Now() - sections := []string{"server", "clients", "memory", "persistence", "stats", "replication", "cpu", "cluster", "keyspace"} - switch len(msg.Args) { - default: - return NOMessage, errInvalidNumberOfArguments - case 1: - case 2: - section := strings.ToLower(msg.Args[1]) + // >> Args + + args := msg.Args + + msects := make(map[string]bool) + allsects := []string{ + "server", "clients", "memory", "persistence", "stats", + "replication", "cpu", "cluster", "keyspace", + } + + if len(args) == 1 { + for _, s := range allsects { + msects[s] = true + } + } + for i := 1; i < len(args); i++ { + section := strings.ToLower(args[i]) switch section { + case "all", "default": + for _, s := range allsects { + msects[s] = true + } default: - sections = []string{section} - case "all": - sections = []string{"server", "clients", "memory", "persistence", "stats", "replication", "cpu", "commandstats", "cluster", "keyspace"} - case "default": + for _, s := range allsects { + if s == section { + msects[section] = true + } + } + } + } + + // >> Operation + + var sects []string + for _, s := range allsects { + if msects[s] { + sects = append(sects, s) } } w := &bytes.Buffer{} - for i, section := range sections { + for i, section := range sects { if i > 0 { w.WriteString("\r\n") } @@ -511,8 +536,9 @@ func (s *Server) cmdInfo(msg *Message) (res resp.Value, err error) { } } - switch msg.OutputType { - case JSON: + // >> Response + + if msg.OutputType == JSON { // Create a map of all key/value info fields m := make(map[string]interface{}) for _, kv := range strings.Split(w.String(), "\r\n") { @@ -525,15 +551,11 @@ func (s *Server) cmdInfo(msg *Message) (res resp.Value, err error) { } // Marshal the map and use the output in the JSON response - data, err := json.Marshal(m) - if err != nil { - return NOMessage, err - } - res = resp.StringValue(`{"ok":true,"info":` + string(data) + `,"elapsed":"` + time.Since(start).String() + "\"}") - case RESP: - res = resp.BytesValue(w.Bytes()) + data, _ := json.Marshal(m) + return resp.StringValue(`{"ok":true,"info":` + string(data) + + `,"elapsed":"` + time.Since(start).String() + "\"}"), nil } - return res, nil + return resp.BytesValue(w.Bytes()), nil } // tryParseType attempts to parse the passed string as an integer, float64 and diff --git a/tests/keys_test.go b/tests/keys_test.go index 2bfec979..5b2de5bd 100644 --- a/tests/keys_test.go +++ b/tests/keys_test.go @@ -35,7 +35,7 @@ func subTestKeys(t *testing.T, mc *mockServer) { runStep(t, mc, "FLUSHDB", keys_FLUSHDB_test) runStep(t, mc, "HEALTHZ", keys_HEALTHZ_test) runStep(t, mc, "SERVER", keys_SERVER_test) - + runStep(t, mc, "INFO", keys_INFO_test) } func keys_BOUNDS_test(mc *mockServer) error { @@ -545,32 +545,9 @@ func keys_FLUSHDB_test(mc *mockServer) error { } func keys_HEALTHZ_test(mc *mockServer) error { - - // // follow and wait - // str, err := redis.String(mc.Do("FOLLOW", "localhost", mc.alt.port)) - // if err != nil { - // return err - // } - // if str != "OK" { - // return errors.New("not ok") - // } - // start := time.Now() - // for time.Since(start) < time.Second*5 { - // str, err = redis.String(mc.Do("HEALTHZ")) - // if str == "OK" { - // err = nil - // break - // } - // time.Sleep(time.Second / 4) - // } - // if err != nil { - // return err - // } - return mc.DoBatch( Do("HEALTHZ").OK(), Do("HEALTHZ").JSON().OK(), - // Do("FOLLOW", "no", "one").OK(), Do("HEALTHZ", "arg").Err(`wrong number of arguments for 'healthz' command`), ) } @@ -621,3 +598,51 @@ func keys_SERVER_test(mc *mockServer) error { Do("SERVER", "ett").JSON().Err(`invalid argument 'ett'`), ) } + +func keys_INFO_test(mc *mockServer) error { + return mc.DoBatch( + Do("INFO").Func(func(s string) error { + if !strings.Contains(s, "# Clients") || + !strings.Contains(s, "# Stats") { + return errors.New("looks invalid") + } + return nil + }), + Do("INFO", "all").Func(func(s string) error { + if !strings.Contains(s, "# Clients") || + !strings.Contains(s, "# Stats") { + return errors.New("looks invalid") + } + return nil + }), + Do("INFO", "default").Func(func(s string) error { + if !strings.Contains(s, "# Clients") || + !strings.Contains(s, "# Stats") { + return errors.New("looks invalid") + } + return nil + }), + Do("INFO", "cpu").Func(func(s string) error { + if !strings.Contains(s, "# CPU") || + strings.Contains(s, "# Clients") || + strings.Contains(s, "# Stats") { + return errors.New("looks invalid") + } + return nil + }), + Do("INFO", "cpu", "clients").Func(func(s string) error { + if !strings.Contains(s, "# CPU") || + !strings.Contains(s, "# Clients") || + strings.Contains(s, "# Stats") { + return errors.New("looks invalid") + } + return nil + }), + Do("INFO").JSON().Func(func(s string) error { + if gjson.Get(s, "info.tile38_version").String() == "" { + return errors.New("looks invalid") + } + return nil + }), + ) +} From 13ceb7da410d62e4a35b958e212a3764ca84f553 Mon Sep 17 00:00:00 2001 From: tidwall Date: Sat, 24 Sep 2022 15:42:07 -0700 Subject: [PATCH 18/33] Removed global variables from core package The core package uses global variables that keep from having more than one Tile38 instance runnning in the same process. Move the core variables in the server.Options type which are uniquely stated per Server instance. The build variables are still present in the core package. --- cmd/tile38-server/main.go | 56 ++++++++++++++++++++++++++---------- core/options.go | 19 ------------ internal/server/aofshrink.go | 11 ++++--- internal/server/checksum.go | 5 ++-- internal/server/follow.go | 5 ++-- internal/server/server.go | 46 ++++++++++++++++++++++------- internal/server/stats.go | 4 +-- scripts/test.sh | 6 ++++ tests/aof_test.go | 35 ++++++++++++++++++++++ tests/mock_test.go | 38 ++++++++++++++++++------ tests/tests_test.go | 10 +++++-- 11 files changed, 166 insertions(+), 69 deletions(-) delete mode 100644 core/options.go create mode 100644 tests/aof_test.go diff --git a/cmd/tile38-server/main.go b/cmd/tile38-server/main.go index cbc3d241..0479b5b8 100644 --- a/cmd/tile38-server/main.go +++ b/cmd/tile38-server/main.go @@ -141,12 +141,33 @@ Developer Options: } var ( - devMode bool nohup bool showEvioDisabled bool showThreadsDisabled bool ) + var ( + // use to be in core/options.go + + // DevMode puts application in to dev mode + devMode = false + + // ShowDebugMessages allows for log.Debug to print to console. + showDebugMessages = false + + // ProtectedMode forces Tile38 to default in protected mode. + protectedMode = "no" + + // AppendOnly allows for disabling the appendonly file. + appendOnly = true + + // AppendFileName allows for custom appendonly file path + appendFileName = "" + + // QueueFileName allows for custom queue.db file path + queueFileName = "" + ) + // parse non standard args. nargs := []string{os.Args[0]} for i := 1; i < len(os.Args); i++ { @@ -163,10 +184,10 @@ Developer Options: if i < len(os.Args) { switch strings.ToLower(os.Args[i]) { case "no": - core.ProtectedMode = "no" + protectedMode = "no" continue case "yes": - core.ProtectedMode = "yes" + protectedMode = "yes" continue } } @@ -183,10 +204,10 @@ Developer Options: if i < len(os.Args) { switch strings.ToLower(os.Args[i]) { case "no": - core.AppendOnly = false + appendOnly = false continue case "yes": - core.AppendOnly = true + appendOnly = true continue } } @@ -198,14 +219,14 @@ Developer Options: fmt.Fprintf(os.Stderr, "appendfilename must have a value\n") os.Exit(1) } - core.AppendFileName = os.Args[i] + appendFileName = os.Args[i] case "--queuefilename", "-queuefilename": i++ if i == len(os.Args) || os.Args[i] == "" { fmt.Fprintf(os.Stderr, "queuefilename must have a value\n") os.Exit(1) } - core.QueueFileName = os.Args[i] + queueFileName = os.Args[i] case "--http-transport", "-http-transport": i++ if i < len(os.Args) { @@ -308,8 +329,7 @@ Developer Options: log.Level = 1 } - core.DevMode = devMode - core.ShowDebugMessages = veryVerbose + showDebugMessages = veryVerbose hostd := "" if host != "" { @@ -449,12 +469,18 @@ Developer Options: log.Warnf("thread flag is deprecated use GOMAXPROCS to set number of threads instead") } opts := server.Options{ - Host: host, - Port: port, - Dir: dir, - UseHTTP: httpTransport, - MetricsAddr: *metricsAddr, - UnixSocketPath: unixSocket, + Host: host, + Port: port, + Dir: dir, + UseHTTP: httpTransport, + MetricsAddr: *metricsAddr, + UnixSocketPath: unixSocket, + DevMode: devMode, + ShowDebugMessages: showDebugMessages, + ProtectedMode: protectedMode, + AppendOnly: appendOnly, + AppendFileName: appendFileName, + QueueFileName: queueFileName, } if err := server.Serve(opts); err != nil { log.Fatal(err) diff --git a/core/options.go b/core/options.go deleted file mode 100644 index 76e2fcfd..00000000 --- a/core/options.go +++ /dev/null @@ -1,19 +0,0 @@ -package core - -// DevMode puts application in to dev mode -var DevMode = false - -// ShowDebugMessages allows for log.Debug to print to console. -var ShowDebugMessages = false - -// ProtectedMode forces Tile38 to default in protected mode. -var ProtectedMode = "no" - -// AppendOnly allows for disabling the appendonly file. -var AppendOnly = true - -// AppendFileName allows for custom appendonly file path -var AppendFileName = "" - -// QueueFileName allows for custom queue.db file path -var QueueFileName = "" diff --git a/internal/server/aofshrink.go b/internal/server/aofshrink.go index c47693c1..efd93a3e 100644 --- a/internal/server/aofshrink.go +++ b/internal/server/aofshrink.go @@ -8,7 +8,6 @@ import ( "time" "github.com/tidwall/btree" - "github.com/tidwall/tile38/core" "github.com/tidwall/tile38/internal/collection" "github.com/tidwall/tile38/internal/field" "github.com/tidwall/tile38/internal/log" @@ -42,7 +41,7 @@ func (s *Server) aofshrink() { }() err := func() error { - f, err := os.Create(core.AppendFileName + "-shrink") + f, err := os.Create(s.opts.AppendFileName + "-shrink") if err != nil { return err } @@ -279,13 +278,13 @@ func (s *Server) aofshrink() { if err := f.Close(); err != nil { log.Fatalf("shrink new aof close fatal operation: %v", err) } - if err := os.Rename(core.AppendFileName, core.AppendFileName+"-bak"); err != nil { + if err := os.Rename(s.opts.AppendFileName, s.opts.AppendFileName+"-bak"); err != nil { log.Fatalf("shrink backup fatal operation: %v", err) } - if err := os.Rename(core.AppendFileName+"-shrink", core.AppendFileName); err != nil { + if err := os.Rename(s.opts.AppendFileName+"-shrink", s.opts.AppendFileName); err != nil { log.Fatalf("shrink rename fatal operation: %v", err) } - s.aof, err = os.OpenFile(core.AppendFileName, os.O_CREATE|os.O_RDWR, 0600) + s.aof, err = os.OpenFile(s.opts.AppendFileName, os.O_CREATE|os.O_RDWR, 0600) if err != nil { log.Fatalf("shrink openfile fatal operation: %v", err) } @@ -296,7 +295,7 @@ func (s *Server) aofshrink() { } s.aofsz = int(n) - os.Remove(core.AppendFileName + "-bak") // ignore error + os.Remove(s.opts.AppendFileName + "-bak") // ignore error return nil }() diff --git a/internal/server/checksum.go b/internal/server/checksum.go index 3538be2b..23462be6 100644 --- a/internal/server/checksum.go +++ b/internal/server/checksum.go @@ -9,7 +9,6 @@ import ( "time" "github.com/tidwall/resp" - "github.com/tidwall/tile38/core" "github.com/tidwall/tile38/internal/log" ) @@ -140,7 +139,7 @@ func getEndOfLastValuePositionInFile(fname string, startPos int64) (int64, error // We will do some various checksums on the leader until we find the correct position to start at. func (s *Server) followCheckSome(addr string, followc int, auth string, ) (pos int64, err error) { - if core.ShowDebugMessages { + if s.opts.ShowDebugMessages { log.Debug("follow:", addr, ":check some") } s.mu.Lock() @@ -211,7 +210,7 @@ func (s *Server) followCheckSome(addr string, followc int, auth string, return 0, err } if pos == fullpos { - if core.ShowDebugMessages { + if s.opts.ShowDebugMessages { log.Debug("follow: aof fully intact") } return pos, nil diff --git a/internal/server/follow.go b/internal/server/follow.go index 3b2a6772..cc8b4222 100644 --- a/internal/server/follow.go +++ b/internal/server/follow.go @@ -9,7 +9,6 @@ import ( "time" "github.com/tidwall/resp" - "github.com/tidwall/tile38/core" "github.com/tidwall/tile38/internal/log" ) @@ -240,7 +239,7 @@ func (s *Server) followStep(host string, port int, followc int) error { if v.String() != "OK" { return errors.New("invalid response to replconf request") } - if core.ShowDebugMessages { + if s.opts.ShowDebugMessages { log.Debug("follow:", addr, ":replconf") } @@ -254,7 +253,7 @@ func (s *Server) followStep(host string, port int, followc int) error { if v.String() != "OK" { return errors.New("invalid response to aof live request") } - if core.ShowDebugMessages { + if s.opts.ShowDebugMessages { log.Debug("follow:", addr, ":read aof") } diff --git a/internal/server/server.go b/internal/server/server.go index f3093320..7c19bac3 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -135,6 +135,8 @@ type Server struct { monconnsMu sync.RWMutex monconns map[net.Conn]bool // monitor connections + + opts Options } // Options for Serve() @@ -145,15 +147,36 @@ type Options struct { UseHTTP bool MetricsAddr string UnixSocketPath string // path for unix socket + + // DevMode puts application in to dev mode + DevMode bool + + // ShowDebugMessages allows for log.Debug to print to console. + ShowDebugMessages bool + + // ProtectedMode forces Tile38 to default in protected mode. + ProtectedMode string + + // AppendOnly allows for disabling the appendonly file. + AppendOnly bool + + // AppendFileName allows for custom appendonly file path + AppendFileName string + + // QueueFileName allows for custom queue.db file path + QueueFileName string } // Serve starts a new tile38 server func Serve(opts Options) error { - if core.AppendFileName == "" { - core.AppendFileName = path.Join(opts.Dir, "appendonly.aof") + if opts.AppendFileName == "" { + opts.AppendFileName = path.Join(opts.Dir, "appendonly.aof") } - if core.QueueFileName == "" { - core.QueueFileName = path.Join(opts.Dir, "queue.db") + if opts.QueueFileName == "" { + opts.QueueFileName = path.Join(opts.Dir, "queue.db") + } + if opts.ProtectedMode == "" { + opts.ProtectedMode = "no" } log.Infof("Server started, Tile38 version %s, git %s", core.Version, core.GitSHA) @@ -183,6 +206,7 @@ func Serve(opts Options) error { groupHooks: btree.NewNonConcurrent(byGroupHook), groupObjects: btree.NewNonConcurrent(byGroupObject), hookExpires: btree.NewNonConcurrent(byHookExpires), + opts: opts, } s.epc = endpoint.NewManager(s) @@ -256,7 +280,7 @@ func Serve(opts Options) error { }() // Load the queue before the aof - qdb, err := buntdb.Open(core.QueueFileName) + qdb, err := buntdb.Open(opts.QueueFileName) if err != nil { return err } @@ -284,8 +308,8 @@ func Serve(opts Options) error { if err := s.migrateAOF(); err != nil { return err } - if core.AppendOnly { - f, err := os.OpenFile(core.AppendFileName, os.O_CREATE|os.O_RDWR, 0600) + if opts.AppendOnly { + f, err := os.OpenFile(opts.AppendFileName, os.O_CREATE|os.O_RDWR, 0600) if err != nil { return err } @@ -334,7 +358,7 @@ func Serve(opts Options) error { } func (s *Server) isProtected() bool { - if core.ProtectedMode == "no" { + if s.opts.ProtectedMode == "no" { // --protected-mode no return false } @@ -1051,19 +1075,19 @@ func (s *Server) command(msg *Message, client *Client) ( case "ttl": res, err = s.cmdTTL(msg) case "shutdown": - if !core.DevMode { + if !s.opts.DevMode { err = fmt.Errorf("unknown command '%s'", msg.Args[0]) return } log.Fatal("shutdown requested by developer") case "massinsert": - if !core.DevMode { + if !s.opts.DevMode { err = fmt.Errorf("unknown command '%s'", msg.Args[0]) return } res, err = s.cmdMassInsert(msg) case "sleep": - if !core.DevMode { + if !s.opts.DevMode { err = fmt.Errorf("unknown command '%s'", msg.Args[0]) return } diff --git a/internal/server/stats.go b/internal/server/stats.go index c0ab0bc4..84d2b962 100644 --- a/internal/server/stats.go +++ b/internal/server/stats.go @@ -318,7 +318,7 @@ func (s *Server) extStats(m map[string]interface{}) { // Whether or not a cluster is enabled m["tile38_cluster_enabled"] = false // Whether or not the Tile38 AOF is enabled - m["tile38_aof_enabled"] = core.AppendOnly + m["tile38_aof_enabled"] = s.opts.AppendOnly // Whether or not an AOF shrink is currently in progress m["tile38_aof_rewrite_in_progress"] = s.shrinking // Length of time the last AOF shrink took @@ -409,7 +409,7 @@ func boolInt(t bool) int { return 0 } func (s *Server) writeInfoPersistence(w *bytes.Buffer) { - fmt.Fprintf(w, "aof_enabled:%d\r\n", boolInt(core.AppendOnly)) + fmt.Fprintf(w, "aof_enabled:%d\r\n", boolInt(s.opts.AppendOnly)) fmt.Fprintf(w, "aof_rewrite_in_progress:%d\r\n", boolInt(s.shrinking)) // Flag indicating a AOF rewrite operation is on-going fmt.Fprintf(w, "aof_last_rewrite_time_sec:%d\r\n", s.lastShrinkDuration.get()/int(time.Second)) // Duration of the last AOF rewrite operation in seconds diff --git a/scripts/test.sh b/scripts/test.sh index 36086708..93f63322 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -7,6 +7,12 @@ export CGO_ENABLED=0 cd tests go test -coverpkg=../internal/server -coverprofile=/tmp/coverage.out $GOTEST + + +# go test \ +# -coverpkg=../internal/... -coverprofile=/tmp/coverage.out \ +# -v . -v ../... $GOTEST + go tool cover -html=/tmp/coverage.out -o /tmp/coverage.html echo "details: file:///tmp/coverage.html" cd .. diff --git a/tests/aof_test.go b/tests/aof_test.go new file mode 100644 index 00000000..d9cb68b4 --- /dev/null +++ b/tests/aof_test.go @@ -0,0 +1,35 @@ +package tests + +import ( + "testing" +) + +func subTestAOF(t *testing.T, mc *mockServer) { + runStep(t, mc, "loading", aof_loading_test) +} + +func aof_loading_test(mc *mockServer) error { + + // aof, err := mc.readAOF() + // if err != nil { + // return err + // } + + // aof = append(aof, "asdfasdf\r\n"...) + // aof = nil + // mc2, err := mockOpenServer(MockServerOptions{ + // Silent: false, + // Metrics: false, + // AOFData: aof, + // }) + // if err != nil { + // return err + // } + // defer mc2.Close() + + // time.Sleep(time.Minute) + + // ` + + return nil +} diff --git a/tests/mock_test.go b/tests/mock_test.go index cfd9c44d..cbec7fe3 100644 --- a/tests/mock_test.go +++ b/tests/mock_test.go @@ -7,12 +7,12 @@ import ( "log" "math/rand" "os" + "path/filepath" "strings" "time" "github.com/gomodule/redigo/redis" "github.com/tidwall/sjson" - "github.com/tidwall/tile38/core" tlog "github.com/tidwall/tile38/internal/log" "github.com/tidwall/tile38/internal/server" ) @@ -38,34 +38,55 @@ type mockServer struct { port int conn redis.Conn ioJSON bool + dir string // alt *mockServer } -func mockOpenServer(silent, metrics bool) (*mockServer, error) { +func (mc *mockServer) readAOF() ([]byte, error) { + return os.ReadFile(filepath.Join(mc.dir, "appendonly.aof")) +} + +type MockServerOptions struct { + AOFData []byte + Silent bool + Metrics bool +} + +func mockOpenServer(opts MockServerOptions) (*mockServer, error) { rand.Seed(time.Now().UnixNano()) port := rand.Int()%20000 + 20000 dir := fmt.Sprintf("data-mock-%d", port) - if !silent { + if !opts.Silent { fmt.Printf("Starting test server at port %d\n", port) } + if len(opts.AOFData) > 0 { + if err := os.MkdirAll(dir, 0777); err != nil { + return nil, err + } + err := os.WriteFile(filepath.Join(dir, "appendonly.aof"), + opts.AOFData, 0666) + if err != nil { + return nil, err + } + } logOutput := io.Discard if os.Getenv("PRINTLOG") == "1" { logOutput = os.Stderr } - core.DevMode = true s := &mockServer{port: port} tlog.SetOutput(logOutput) go func() { - opts := server.Options{ + sopts := server.Options{ Host: "localhost", Port: port, Dir: dir, UseHTTP: true, + DevMode: true, } - if metrics { - opts.MetricsAddr = ":4321" + if opts.Metrics { + sopts.MetricsAddr = ":4321" } - if err := server.Serve(opts); err != nil { + if err := server.Serve(sopts); err != nil { log.Fatal(err) } }() @@ -73,6 +94,7 @@ func mockOpenServer(silent, metrics bool) (*mockServer, error) { s.Close() return nil, err } + s.dir = dir return s, nil } diff --git a/tests/tests_test.go b/tests/tests_test.go index 3fb3b424..415a44d0 100644 --- a/tests/tests_test.go +++ b/tests/tests_test.go @@ -38,7 +38,10 @@ func TestAll(t *testing.T) { os.Exit(1) }() - mc, err := mockOpenServer(false, true) + mc, err := mockOpenServer(MockServerOptions{ + Silent: false, + Metrics: true, + }) if err != nil { t.Fatal(err) } @@ -62,6 +65,7 @@ func TestAll(t *testing.T) { runSubTest(t, "info", mc, subTestInfo) runSubTest(t, "timeouts", mc, subTestTimeout) runSubTest(t, "metrics", mc, subTestMetrics) + runSubTest(t, "aof", mc, subTestAOF) } func runSubTest(t *testing.T, name string, mc *mockServer, test func(t *testing.T, mc *mockServer)) { @@ -111,7 +115,9 @@ func BenchmarkAll(b *testing.B) { os.Exit(1) }() - mc, err := mockOpenServer(true, true) + mc, err := mockOpenServer(MockServerOptions{ + Silent: true, Metrics: true, + }) if err != nil { b.Fatal(err) } From f2c3b3924a3a0ba98b7200576c40abc2a28459f3 Mon Sep 17 00:00:00 2001 From: tidwall Date: Sun, 25 Sep 2022 03:54:22 -0700 Subject: [PATCH 19/33] wip - aof tests --- internal/server/aof.go | 11 +----- tests/aof_test.go | 88 ++++++++++++++++++++++++++++++++++-------- tests/mock_test.go | 32 +++++++++------ tests/tests_test.go | 8 ---- 4 files changed, 93 insertions(+), 46 deletions(-) diff --git a/internal/server/aof.go b/internal/server/aof.go index 6d083e11..ffab4957 100644 --- a/internal/server/aof.go +++ b/internal/server/aof.go @@ -41,10 +41,7 @@ func (s *Server) loadAOF() (err error) { ps := float64(count) / (float64(d) / float64(time.Second)) suf := []string{"bytes/s", "KB/s", "MB/s", "GB/s", "TB/s"} bps := float64(fi.Size()) / (float64(d) / float64(time.Second)) - for i := 0; bps > 1024; i++ { - if len(suf) == 1 { - break - } + for i := 0; bps > 1024 && len(suf) > 1; i++ { bps /= 1024 suf = suf[1:] } @@ -123,11 +120,7 @@ func commandErrIsFatal(err error) bool { // FSET (and other writable commands) may return errors that we need // to ignore during the loading process. These errors may occur (though unlikely) // due to the aof rewrite operation. - switch err { - case errKeyNotFound, errIDNotFound: - return false - } - return true + return !(err == errKeyNotFound || err == errIDNotFound) } // flushAOF flushes all aof buffer data to disk. Set sync to true to sync the diff --git a/tests/aof_test.go b/tests/aof_test.go index d9cb68b4..5f74b25a 100644 --- a/tests/aof_test.go +++ b/tests/aof_test.go @@ -1,35 +1,89 @@ package tests import ( + "bytes" + "errors" + "fmt" "testing" ) func subTestAOF(t *testing.T, mc *mockServer) { runStep(t, mc, "loading", aof_loading_test) + // runStep(t, mc, "AOFMD5", aof_AOFMD5_test) +} + +func loadAOFAndClose(aof any) error { + var aofb []byte + switch aof := aof.(type) { + case []byte: + aofb = []byte(aof) + case string: + aofb = []byte(aof) + default: + return errors.New("aof is not string or bytes") + } + mc, err := mockOpenServer(MockServerOptions{ + Silent: true, + Metrics: false, + AOFData: aofb, + }) + if mc != nil { + mc.Close() + } + return err } func aof_loading_test(mc *mockServer) error { - // aof, err := mc.readAOF() - // if err != nil { - // return err - // } + var err error + // invalid command + err = loadAOFAndClose("asdfasdf\r\n") + if err == nil || err.Error() != "unknown command 'asdfasdf'" { + return fmt.Errorf("expected '%v', got '%v'", + "unknown command 'asdfasdf'", err) + } - // aof = append(aof, "asdfasdf\r\n"...) - // aof = nil - // mc2, err := mockOpenServer(MockServerOptions{ - // Silent: false, - // Metrics: false, - // AOFData: aof, - // }) - // if err != nil { - // return err - // } - // defer mc2.Close() + // incomplete command + err = loadAOFAndClose("set fleet truck point 10 10\r\nasdfasdf") + if err != nil { + return err + } - // time.Sleep(time.Minute) + // big aof file + var aof string + for i := 0; i < 10000; i++ { + aof += fmt.Sprintf("SET fleet truck%d POINT 10 10\r\n", i) + } + err = loadAOFAndClose(aof) + if err != nil { + return err + } - // ` + // extra zeros at various places + aof = "" + for i := 0; i < 1000; i++ { + if i%10 == 0 { + aof += string(bytes.Repeat([]byte{0}, 100)) + } + aof += fmt.Sprintf("SET fleet truck%d POINT 10 10\r\n", i) + } + aof += string(bytes.Repeat([]byte{0}, 5000)) + err = loadAOFAndClose(aof) + if err != nil { + return err + } + + // bad protocol + aof = "*2\r\n$1\r\nh\r\n+OK\r\n" + err = loadAOFAndClose(aof) + if fmt.Sprintf("%v", err) != "Protocol error: expected '$', got '+'" { + return fmt.Errorf("expected '%v', got '%v'", + "Protocol error: expected '$', got '+'", err) + } return nil } + +// func aof_AOFMD5_test(mc *mockServer) error { +// return nil +// } diff --git a/tests/mock_test.go b/tests/mock_test.go index cbec7fe3..a67e5f17 100644 --- a/tests/mock_test.go +++ b/tests/mock_test.go @@ -4,11 +4,11 @@ import ( "errors" "fmt" "io" - "log" "math/rand" "os" "path/filepath" "strings" + "sync/atomic" "time" "github.com/gomodule/redigo/redis" @@ -39,7 +39,6 @@ type mockServer struct { conn redis.Conn ioJSON bool dir string - // alt *mockServer } func (mc *mockServer) readAOF() ([]byte, error) { @@ -73,35 +72,41 @@ func mockOpenServer(opts MockServerOptions) (*mockServer, error) { if os.Getenv("PRINTLOG") == "1" { logOutput = os.Stderr } - s := &mockServer{port: port} + s := &mockServer{port: port, dir: dir} tlog.SetOutput(logOutput) + var ferrt int32 // atomic flag for when ferr has been set + var ferr error // ferr for when the server fails to start go func() { sopts := server.Options{ - Host: "localhost", - Port: port, - Dir: dir, - UseHTTP: true, - DevMode: true, + Host: "localhost", + Port: port, + Dir: dir, + UseHTTP: true, + DevMode: true, + AppendOnly: true, } if opts.Metrics { sopts.MetricsAddr = ":4321" } if err := server.Serve(sopts); err != nil { - log.Fatal(err) + ferr = err + atomic.StoreInt32(&ferrt, 1) } }() - if err := s.waitForStartup(); err != nil { + if err := s.waitForStartup(&ferr, &ferrt); err != nil { s.Close() return nil, err } - s.dir = dir return s, nil } -func (s *mockServer) waitForStartup() error { +func (s *mockServer) waitForStartup(ferr *error, ferrt *int32) error { var lerr error start := time.Now() for { + if atomic.LoadInt32(ferrt) != 0 { + return *ferr + } if time.Since(start) > time.Second*5 { if lerr != nil { return lerr @@ -131,6 +136,9 @@ func (mc *mockServer) Close() { if mc.conn != nil { mc.conn.Close() } + if mc.dir != "" { + os.RemoveAll(mc.dir) + } } func (mc *mockServer) ResetConn() { diff --git a/tests/tests_test.go b/tests/tests_test.go index 415a44d0..5894bb20 100644 --- a/tests/tests_test.go +++ b/tests/tests_test.go @@ -47,14 +47,6 @@ func TestAll(t *testing.T) { } defer mc.Close() - // mc2, err := mockOpenServer(false, false) - // if err != nil { - // t.Fatal(err) - // } - // defer mc2.Close() - // mc.alt = mc2 - // mc2.alt = mc - runSubTest(t, "keys", mc, subTestKeys) runSubTest(t, "json", mc, subTestJSON) runSubTest(t, "search", mc, subTestSearch) From 906824323b908790cdb7b3d26e80cd944536c5c7 Mon Sep 17 00:00:00 2001 From: tidwall Date: Sun, 25 Sep 2022 06:28:17 -0700 Subject: [PATCH 20/33] More graceful Tile38 shutdown --- internal/endpoint/amqp.go | 10 +- internal/endpoint/disque.go | 12 ++- internal/endpoint/endpoint.go | 24 ++++- internal/endpoint/eventHub.go | 4 + internal/endpoint/grpc.go | 13 ++- internal/endpoint/http.go | 4 + internal/endpoint/kafka.go | 12 ++- internal/endpoint/local.go | 4 + internal/endpoint/mqtt.go | 9 +- internal/endpoint/nats.go | 12 ++- internal/endpoint/pubsub.go | 10 +- internal/endpoint/redis.go | 12 ++- internal/endpoint/sqs.go | 10 +- internal/server/expire.go | 24 ++--- internal/server/live.go | 11 +- internal/server/server.go | 188 +++++++++++++++++++++++----------- scripts/test.sh | 5 +- tests/mock_test.go | 31 +++--- tests/tests_test.go | 14 +++ 19 files changed, 298 insertions(+), 111 deletions(-) diff --git a/internal/endpoint/amqp.go b/internal/endpoint/amqp.go index 546dee78..b69a22ef 100644 --- a/internal/endpoint/amqp.go +++ b/internal/endpoint/amqp.go @@ -27,13 +27,21 @@ func (conn *AMQPConn) Expired() bool { defer conn.mu.Unlock() if !conn.ex { if time.Since(conn.t) > amqpExpiresAfter { - conn.ex = true conn.close() + conn.ex = true } } return conn.ex } +// ExpireNow forces the connection to expire +func (conn *AMQPConn) ExpireNow() { + conn.mu.Lock() + defer conn.mu.Unlock() + conn.close() + conn.ex = true +} + func (conn *AMQPConn) close() { if conn.conn != nil { conn.conn.Close() diff --git a/internal/endpoint/disque.go b/internal/endpoint/disque.go index 35300ec6..664262f8 100644 --- a/internal/endpoint/disque.go +++ b/internal/endpoint/disque.go @@ -33,15 +33,21 @@ func (conn *DisqueConn) Expired() bool { defer conn.mu.Unlock() if !conn.ex { if time.Since(conn.t) > disqueExpiresAfter { - if conn.conn != nil { - conn.close() - } + conn.close() conn.ex = true } } return conn.ex } +// ExpireNow forces the connection to expire +func (conn *DisqueConn) ExpireNow() { + conn.mu.Lock() + defer conn.mu.Unlock() + conn.close() + conn.ex = true +} + func (conn *DisqueConn) close() { if conn.conn != nil { conn.conn.Close() diff --git a/internal/endpoint/endpoint.go b/internal/endpoint/endpoint.go index 8e5d1274..f13feeb7 100644 --- a/internal/endpoint/endpoint.go +++ b/internal/endpoint/endpoint.go @@ -6,6 +6,7 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "time" "github.com/streadway/amqp" @@ -136,6 +137,7 @@ type Endpoint struct { // Conn is an endpoint connection type Conn interface { + ExpireNow() Expired() bool Send(val string) error } @@ -145,6 +147,8 @@ type Manager struct { mu sync.RWMutex conns map[string]Conn publisher LocalPublisher + shutdown int32 // atomic bool + wg sync.WaitGroup // run wait group } // NewManager returns a new manager @@ -153,13 +157,29 @@ func NewManager(publisher LocalPublisher) *Manager { conns: make(map[string]Conn), publisher: publisher, } - go epc.Run() + epc.wg.Add(1) + go epc.run() return epc } +func (epc *Manager) Shutdown() { + defer epc.wg.Wait() + atomic.StoreInt32(&epc.shutdown, 1) + // expire the connections + epc.mu.Lock() + defer epc.mu.Unlock() + for _, conn := range epc.conns { + conn.ExpireNow() + } +} + // Run starts the managing of endpoints -func (epc *Manager) Run() { +func (epc *Manager) run() { + defer epc.wg.Done() for { + if atomic.LoadInt32(&epc.shutdown) != 0 { + return + } time.Sleep(time.Second) func() { epc.mu.Lock() diff --git a/internal/endpoint/eventHub.go b/internal/endpoint/eventHub.go index 5de50616..83ea0775 100644 --- a/internal/endpoint/eventHub.go +++ b/internal/endpoint/eventHub.go @@ -28,6 +28,10 @@ func (conn *EvenHubConn) Expired() bool { return false } +// ExpireNow forces the connection to expire +func (conn *EvenHubConn) ExpireNow() { +} + // Send sends a message func (conn *EvenHubConn) Send(msg string) error { hub, err := eventhub.NewHubFromConnectionString(conn.ep.EventHub.ConnectionString) diff --git a/internal/endpoint/grpc.go b/internal/endpoint/grpc.go index 12de3d7e..d40a53fa 100644 --- a/internal/endpoint/grpc.go +++ b/internal/endpoint/grpc.go @@ -36,14 +36,21 @@ func (conn *GRPCConn) Expired() bool { defer conn.mu.Unlock() if !conn.ex { if time.Since(conn.t) > grpcExpiresAfter { - if conn.conn != nil { - conn.close() - } + conn.close() conn.ex = true } } return conn.ex } + +// ExpireNow forces the connection to expire +func (conn *GRPCConn) ExpireNow() { + conn.mu.Lock() + defer conn.mu.Unlock() + conn.close() + conn.ex = true +} + func (conn *GRPCConn) close() { if conn.conn != nil { conn.conn.Close() diff --git a/internal/endpoint/http.go b/internal/endpoint/http.go index 52deba24..c89bb986 100644 --- a/internal/endpoint/http.go +++ b/internal/endpoint/http.go @@ -38,6 +38,10 @@ func (conn *HTTPConn) Expired() bool { return false } +// ExpireNow forces the connection to expire +func (conn *HTTPConn) ExpireNow() { +} + // Send sends a message func (conn *HTTPConn) Send(msg string) error { req, err := http.NewRequest("POST", conn.ep.Original, bytes.NewBufferString(msg)) diff --git a/internal/endpoint/kafka.go b/internal/endpoint/kafka.go index ef083380..2cf808ea 100644 --- a/internal/endpoint/kafka.go +++ b/internal/endpoint/kafka.go @@ -34,15 +34,21 @@ func (conn *KafkaConn) Expired() bool { defer conn.mu.Unlock() if !conn.ex { if time.Since(conn.t) > kafkaExpiresAfter { - if conn.conn != nil { - conn.close() - } + conn.close() conn.ex = true } } return conn.ex } +// ExpireNow forces the connection to expire +func (conn *KafkaConn) ExpireNow() { + conn.mu.Lock() + defer conn.mu.Unlock() + conn.close() + conn.ex = true +} + func (conn *KafkaConn) close() { if conn.conn != nil { conn.conn.Close() diff --git a/internal/endpoint/local.go b/internal/endpoint/local.go index 1038fd79..2c1c7d6d 100644 --- a/internal/endpoint/local.go +++ b/internal/endpoint/local.go @@ -23,6 +23,10 @@ func (conn *LocalConn) Expired() bool { return false } +// ExpireNow forces the connection to expire +func (conn *LocalConn) ExpireNow() { +} + // Send sends a message func (conn *LocalConn) Send(msg string) error { conn.publisher.Publish(conn.ep.Local.Channel, msg) diff --git a/internal/endpoint/mqtt.go b/internal/endpoint/mqtt.go index 5f80765c..d7410649 100644 --- a/internal/endpoint/mqtt.go +++ b/internal/endpoint/mqtt.go @@ -40,12 +40,19 @@ func (conn *MQTTConn) Expired() bool { return conn.ex } +// ExpireNow forces the connection to expire +func (conn *MQTTConn) ExpireNow() { + conn.mu.Lock() + defer conn.mu.Unlock() + conn.close() + conn.ex = true +} + func (conn *MQTTConn) close() { if conn.conn != nil { if conn.conn.IsConnected() { conn.conn.Disconnect(250) } - conn.conn = nil } } diff --git a/internal/endpoint/nats.go b/internal/endpoint/nats.go index 6c0c7815..ae254802 100644 --- a/internal/endpoint/nats.go +++ b/internal/endpoint/nats.go @@ -32,15 +32,21 @@ func (conn *NATSConn) Expired() bool { defer conn.mu.Unlock() if !conn.ex { if time.Since(conn.t) > natsExpiresAfter { - if conn.conn != nil { - conn.close() - } + conn.close() conn.ex = true } } return conn.ex } +// ExpireNow forces the connection to expire +func (conn *NATSConn) ExpireNow() { + conn.mu.Lock() + defer conn.mu.Unlock() + conn.close() + conn.ex = true +} + func (conn *NATSConn) close() { if conn.conn != nil { conn.conn.Close() diff --git a/internal/endpoint/pubsub.go b/internal/endpoint/pubsub.go index db6dac33..b4926474 100644 --- a/internal/endpoint/pubsub.go +++ b/internal/endpoint/pubsub.go @@ -83,13 +83,21 @@ func (conn *PubSubConn) Expired() bool { defer conn.mu.Unlock() if !conn.ex { if time.Since(conn.t) > pubsubExpiresAfter { - conn.ex = true conn.close() + conn.ex = true } } return conn.ex } +// ExpireNow forces the connection to expire +func (conn *PubSubConn) ExpireNow() { + conn.mu.Lock() + defer conn.mu.Unlock() + conn.close() + conn.ex = true +} + func newPubSubConn(ep Endpoint) *PubSubConn { return &PubSubConn{ ep: ep, diff --git a/internal/endpoint/redis.go b/internal/endpoint/redis.go index 8064d166..d5101b57 100644 --- a/internal/endpoint/redis.go +++ b/internal/endpoint/redis.go @@ -32,15 +32,21 @@ func (conn *RedisConn) Expired() bool { defer conn.mu.Unlock() if !conn.ex { if time.Since(conn.t) > redisExpiresAfter { - if conn.conn != nil { - conn.close() - } + conn.close() conn.ex = true } } return conn.ex } +// ExpireNow forces the connection to expire +func (conn *RedisConn) ExpireNow() { + conn.mu.Lock() + defer conn.mu.Unlock() + conn.close() + conn.ex = true +} + func (conn *RedisConn) close() { if conn.conn != nil { conn.conn.Close() diff --git a/internal/endpoint/sqs.go b/internal/endpoint/sqs.go index 8a1098b7..148f2a3f 100644 --- a/internal/endpoint/sqs.go +++ b/internal/endpoint/sqs.go @@ -39,13 +39,21 @@ func (conn *SQSConn) Expired() bool { defer conn.mu.Unlock() if !conn.ex { if time.Since(conn.t) > sqsExpiresAfter { - conn.ex = true conn.close() + conn.ex = true } } return conn.ex } +// ExpireNow forces the connection to expire +func (conn *SQSConn) ExpireNow() { + conn.mu.Lock() + defer conn.mu.Unlock() + conn.close() + conn.ex = true +} + func (conn *SQSConn) close() { if conn.svc != nil { conn.svc = nil diff --git a/internal/server/expire.go b/internal/server/expire.go index 39abd412..5f812913 100644 --- a/internal/server/expire.go +++ b/internal/server/expire.go @@ -1,6 +1,7 @@ package server import ( + "sync" "time" "github.com/tidwall/tile38/internal/collection" @@ -12,20 +13,15 @@ const bgExpireDelay = time.Second / 10 // backgroundExpiring deletes expired items from the database. // It's executes every 1/10 of a second. -func (s *Server) backgroundExpiring() { - for { - if s.stopServer.on() { - return - } - func() { - s.mu.Lock() - defer s.mu.Unlock() - now := time.Now() - s.backgroundExpireObjects(now) - s.backgroundExpireHooks(now) - }() - time.Sleep(bgExpireDelay) - } +func (s *Server) backgroundExpiring(wg *sync.WaitGroup) { + defer wg.Done() + s.loopUntilServerStops(bgExpireDelay, func() { + s.mu.Lock() + defer s.mu.Unlock() + now := time.Now() + s.backgroundExpireObjects(now) + s.backgroundExpireHooks(now) + }) } func (s *Server) backgroundExpireObjects(now time.Time) { diff --git a/internal/server/live.go b/internal/server/live.go index eb66ec64..338523b5 100644 --- a/internal/server/live.go +++ b/internal/server/live.go @@ -21,10 +21,16 @@ type liveBuffer struct { cond *sync.Cond } -func (s *Server) processLives() { - defer s.lwait.Done() +func (s *Server) processLives(wg *sync.WaitGroup) { + defer wg.Done() + var done abool + wg.Add(1) go func() { + defer wg.Done() for { + if done.on() { + break + } s.lcond.Broadcast() time.Sleep(time.Second / 4) } @@ -33,6 +39,7 @@ func (s *Server) processLives() { defer s.lcond.L.Unlock() for { if s.stopServer.on() { + done.set(true) return } for len(s.lstack) > 0 { diff --git a/internal/server/server.go b/internal/server/server.go index 7c19bac3..7a367592 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -79,6 +79,7 @@ type Server struct { started time.Time config *Config epc *endpoint.Manager + ln net.Listener // server listener // env opts geomParseOpts geojson.ParseOptions @@ -114,7 +115,6 @@ type Server struct { lstack []*commandDetails lives map[*liveBuffer]bool lcond *sync.Cond - lwait sync.WaitGroup fcup bool // follow caught up fcuponce bool // follow caught up once shrinking bool // aof shrinking flag @@ -165,6 +165,9 @@ type Options struct { // QueueFileName allows for custom queue.db file path QueueFileName string + + // Shutdown allows for shutting down the server. + Shutdown <-chan bool } // Serve starts a new tile38 server @@ -180,6 +183,15 @@ func Serve(opts Options) error { } log.Infof("Server started, Tile38 version %s, git %s", core.Version, core.GitSHA) + defer func() { + log.Warn("Server has shutdown, bye now") + if false { + // prints the stack, looking for running goroutines. + buf := make([]byte, 10000) + n := runtime.Stack(buf, true) + println(string(buf[:n])) + } + }() // Initialize the s s := &Server{ @@ -210,6 +222,7 @@ func Serve(opts Options) error { } s.epc = endpoint.NewManager(s) + defer s.epc.Shutdown() s.luascripts = s.newScriptMap() s.luapool = s.newPool() defer s.luapool.Shutdown() @@ -279,6 +292,13 @@ func Serve(opts Options) error { nerr <- s.netServe() }() + go func() { + <-opts.Shutdown + s.stopServer.set(true) + log.Warnf("Shutting down...") + s.ln.Close() + }() + // Load the queue before the aof qdb, err := buntdb.Open(opts.QueueFileName) if err != nil { @@ -324,32 +344,60 @@ func Serve(opts Options) error { } // Start background routines - if s.config.followHost() != "" { - go s.follow(s.config.followHost(), s.config.followPort(), - s.followc.get()) - } + var bgwg sync.WaitGroup - if opts.MetricsAddr != "" { - log.Infof("Listening for metrics at: %s", opts.MetricsAddr) + if s.config.followHost() != "" { + bgwg.Add(1) go func() { - http.HandleFunc("/", s.MetricsIndexHandler) - http.HandleFunc("/metrics", s.MetricsHandler) - log.Fatal(http.ListenAndServe(opts.MetricsAddr, nil)) + defer bgwg.Done() + s.follow(s.config.followHost(), s.config.followPort(), + s.followc.get()) }() } - s.lwait.Add(1) - go s.processLives() - go s.watchOutOfMemory() - go s.watchLuaStatePool() - go s.watchAutoGC() - go s.backgroundExpiring() - go s.backgroundSyncAOF() + var mln net.Listener + if opts.MetricsAddr != "" { + log.Infof("Listening for metrics at: %s", opts.MetricsAddr) + mln, err = net.Listen("tcp", opts.MetricsAddr) + if err != nil { + return err + } + bgwg.Add(1) + go func() { + defer bgwg.Done() + smux := http.NewServeMux() + smux.HandleFunc("/", s.MetricsIndexHandler) + smux.HandleFunc("/metrics", s.MetricsHandler) + err := http.Serve(mln, smux) + if err != nil { + if !s.stopServer.on() { + log.Fatalf("metrics server: %s", err) + } + } + }() + } + + bgwg.Add(1) + go s.processLives(&bgwg) + bgwg.Add(1) + go s.watchOutOfMemory(&bgwg) + bgwg.Add(1) + go s.watchLuaStatePool(&bgwg) + bgwg.Add(1) + go s.watchAutoGC(&bgwg) + bgwg.Add(1) + go s.backgroundExpiring(&bgwg) + bgwg.Add(1) + go s.backgroundSyncAOF(&bgwg) defer func() { + log.Debug("Stopping background routines") // Stop background routines s.followc.add(1) // this will force any follow communication to die s.stopServer.set(true) - s.lwait.Wait() + if mln != nil { + mln.Close() // Stop the metrics server + } + bgwg.Wait() }() // Server is now loaded and ready. Wait for network error messages. @@ -384,16 +432,37 @@ func (s *Server) netServe() error { if err != nil { return err } - defer ln.Close() + + var wg sync.WaitGroup + defer func() { + log.Debug("Closing client connections...") + s.connsmu.RLock() + for _, c := range s.conns { + c.closer.Close() + } + s.connsmu.RUnlock() + wg.Wait() + ln.Close() + log.Debug("Client connection closed") + }() + s.ln = ln + log.Infof("Ready to accept connections at %s", ln.Addr()) var clientID int64 for { conn, err := ln.Accept() if err != nil { - return err + if s.stopServer.on() { + return nil + } + log.Warn(err) + time.Sleep(time.Second / 5) + continue } - + wg.Add(1) go func(conn net.Conn) { + defer wg.Done() + // open connection // create the client client := new(Client) @@ -617,20 +686,16 @@ func (conn *liveConn) SetWriteDeadline(deadline time.Time) error { panic("not supported") } -func (s *Server) watchAutoGC() { - t := time.NewTicker(time.Second) - defer t.Stop() +func (s *Server) watchAutoGC(wg *sync.WaitGroup) { + defer wg.Done() start := time.Now() - for range t.C { - if s.stopServer.on() { - return - } + s.loopUntilServerStops(time.Second, func() { autoGC := s.config.autoGC() if autoGC == 0 { - continue + return } if time.Since(start) < time.Second*time.Duration(autoGC) { - continue + return } var mem1, mem2 runtime.MemStats runtime.ReadMemStats(&mem1) @@ -645,7 +710,7 @@ func (s *Server) watchAutoGC() { "alloc: %v, heap_alloc: %v, heap_released: %v", mem2.Alloc, mem2.HeapAlloc, mem2.HeapReleased) start = time.Now() - } + }) } func (s *Server) checkOutOfMemory() { @@ -667,40 +732,45 @@ func (s *Server) checkOutOfMemory() { s.outOfMemory.set(int(mem.HeapAlloc) > s.config.maxMemory()) } -func (s *Server) watchOutOfMemory() { - t := time.NewTicker(time.Second * 2) - defer t.Stop() - for range t.C { - s.checkOutOfMemory() - } -} - -func (s *Server) watchLuaStatePool() { - t := time.NewTicker(time.Second * 10) - defer t.Stop() - for range t.C { - func() { - s.luapool.Prune() - }() - } -} - -// backgroundSyncAOF ensures that the aof buffer is does not grow too big. -func (s *Server) backgroundSyncAOF() { - t := time.NewTicker(time.Second) - defer t.Stop() - for range t.C { +func (s *Server) loopUntilServerStops(dur time.Duration, op func()) { + var last time.Time + for { if s.stopServer.on() { return } - func() { - s.mu.Lock() - defer s.mu.Unlock() - s.flushAOF(true) - }() + now := time.Now() + if now.Sub(last) > dur { + op() + last = now + } + time.Sleep(time.Second / 5) } } +func (s *Server) watchOutOfMemory(wg *sync.WaitGroup) { + defer wg.Done() + s.loopUntilServerStops(time.Second*4, func() { + s.checkOutOfMemory() + }) +} + +func (s *Server) watchLuaStatePool(wg *sync.WaitGroup) { + defer wg.Done() + s.loopUntilServerStops(time.Second*10, func() { + s.luapool.Prune() + }) +} + +// backgroundSyncAOF ensures that the aof buffer is does not grow too big. +func (s *Server) backgroundSyncAOF(wg *sync.WaitGroup) { + defer wg.Done() + s.loopUntilServerStops(time.Second, func() { + s.mu.Lock() + defer s.mu.Unlock() + s.flushAOF(true) + }) +} + func isReservedFieldName(field string) bool { switch field { case "z", "lat", "lon": diff --git a/scripts/test.sh b/scripts/test.sh index 93f63322..acfc97f0 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -9,9 +9,8 @@ cd tests go test -coverpkg=../internal/server -coverprofile=/tmp/coverage.out $GOTEST -# go test \ -# -coverpkg=../internal/... -coverprofile=/tmp/coverage.out \ -# -v . -v ../... $GOTEST +# go test -coverpkg=../internal/... -coverprofile=/tmp/coverage.out \ +# -v ./... $GOTEST go tool cover -html=/tmp/coverage.out -o /tmp/coverage.html echo "details: file:///tmp/coverage.html" diff --git a/tests/mock_test.go b/tests/mock_test.go index a67e5f17..bd0597c2 100644 --- a/tests/mock_test.go +++ b/tests/mock_test.go @@ -35,10 +35,11 @@ func mockCleanup(silent bool) { } type mockServer struct { - port int - conn redis.Conn - ioJSON bool - dir string + port int + conn redis.Conn + ioJSON bool + dir string + shutdown chan bool } func (mc *mockServer) readAOF() ([]byte, error) { @@ -71,24 +72,29 @@ func mockOpenServer(opts MockServerOptions) (*mockServer, error) { logOutput := io.Discard if os.Getenv("PRINTLOG") == "1" { logOutput = os.Stderr + tlog.Level = 3 } - s := &mockServer{port: port, dir: dir} + shutdown := make(chan bool) + s := &mockServer{port: port, dir: dir, shutdown: shutdown} tlog.SetOutput(logOutput) var ferrt int32 // atomic flag for when ferr has been set var ferr error // ferr for when the server fails to start go func() { sopts := server.Options{ - Host: "localhost", - Port: port, - Dir: dir, - UseHTTP: true, - DevMode: true, - AppendOnly: true, + Host: "localhost", + Port: port, + Dir: dir, + UseHTTP: true, + DevMode: true, + AppendOnly: true, + Shutdown: shutdown, + ShowDebugMessages: true, } if opts.Metrics { sopts.MetricsAddr = ":4321" } - if err := server.Serve(sopts); err != nil { + err := server.Serve(sopts) + if err != nil { ferr = err atomic.StoreInt32(&ferrt, 1) } @@ -133,6 +139,7 @@ func (s *mockServer) waitForStartup(ferr *error, ferrt *int32) error { } func (mc *mockServer) Close() { + mc.shutdown <- true if mc.conn != nil { mc.conn.Close() } diff --git a/tests/tests_test.go b/tests/tests_test.go index 5894bb20..0ee8b4a9 100644 --- a/tests/tests_test.go +++ b/tests/tests_test.go @@ -26,6 +26,18 @@ const ( white = "\x1b[37m" ) +// type mockTest struct { +// } + +// func mockTestInit() *mockTest { +// mt := &mockTest{} +// return mt +// } + +// func (mt *mockTest) Cleanup() { + +// } + func TestAll(t *testing.T) { mockCleanup(false) defer mockCleanup(false) @@ -45,6 +57,8 @@ func TestAll(t *testing.T) { if err != nil { t.Fatal(err) } + // log.Printf("Waiting a second for everything to cleanly start...") + // time.Sleep(time.Second * 2) defer mc.Close() runSubTest(t, "keys", mc, subTestKeys) From 97da6d70c4c7f10081f673ef5d4759a35e4bbfd6 Mon Sep 17 00:00:00 2001 From: tidwall Date: Sun, 25 Sep 2022 06:34:27 -0700 Subject: [PATCH 21/33] Moved metrics into step test --- tests/metrics_test.go | 35 ++++++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/tests/metrics_test.go b/tests/metrics_test.go index 14461bb3..429369d0 100644 --- a/tests/metrics_test.go +++ b/tests/metrics_test.go @@ -1,42 +1,54 @@ package tests import ( + "fmt" "io" "net/http" "strings" "testing" ) -func downloadURLWithStatusCode(t *testing.T, u string) (int, string) { +func subTestMetrics(t *testing.T, mc *mockServer) { + runStep(t, mc, "basic", metrics_basic_test) +} + +func downloadURLWithStatusCode(u string) (int, string, error) { resp, err := http.Get(u) if err != nil { - t.Fatal(err) + return 0, "", err } defer resp.Body.Close() body, err := io.ReadAll(resp.Body) if err != nil { - t.Fatal(err) + return 0, "", err } - return resp.StatusCode, string(body) + return resp.StatusCode, string(body), nil } -func subTestMetrics(t *testing.T, mc *mockServer) { +func metrics_basic_test(mc *mockServer) error { + mc.Do("SET", "metrics_test_1", "1", "FIELD", "foo", 5.5, "POINT", 5, 5) mc.Do("SET", "metrics_test_2", "2", "FIELD", "foo", 19.19, "POINT", 19, 19) mc.Do("SET", "metrics_test_2", "3", "FIELD", "foo", 19.19, "POINT", 19, 19) mc.Do("SET", "metrics_test_2", "truck1:driver", "STRING", "John Denton") - status, index := downloadURLWithStatusCode(t, "http://127.0.0.1:4321/") + status, index, err := downloadURLWithStatusCode("http://127.0.0.1:4321/") + if err != nil { + return err + } if status != 200 { - t.Fatalf("Expected status code 200, got: %d", status) + return fmt.Errorf("Expected status code 200, got: %d", status) } if !strings.Contains(index, " Date: Mon, 26 Sep 2022 10:02:02 -0700 Subject: [PATCH 22/33] Thread safe log and support for concurrent tile38 instances --- cmd/tile38-server/main.go | 12 ++-- internal/endpoint/kafka.go | 2 +- internal/endpoint/sqs.go | 2 +- internal/log/log.go | 131 +++++++++++++++++++++++-------------- internal/log/log_test.go | 14 ++-- tests/metrics_test.go | 6 +- tests/mock_test.go | 27 +++++--- tests/tests_test.go | 61 +++++++---------- 8 files changed, 144 insertions(+), 111 deletions(-) diff --git a/cmd/tile38-server/main.go b/cmd/tile38-server/main.go index 0479b5b8..58a1fd54 100644 --- a/cmd/tile38-server/main.go +++ b/cmd/tile38-server/main.go @@ -302,7 +302,7 @@ Developer Options: flag.Parse() if logEncoding == "json" { - log.LogJSON = true + log.SetLogJSON(true) data, _ := os.ReadFile(filepath.Join(dir, "config")) if gjson.GetBytes(data, "logconfig.encoding").String() == "json" { c := gjson.GetBytes(data, "logconfig").String() @@ -320,13 +320,13 @@ Developer Options: log.SetOutput(logw) if quiet { - log.Level = 0 + log.SetLevel(0) } else if veryVerbose { - log.Level = 3 + log.SetLevel(3) } else if verbose { - log.Level = 2 + log.SetLevel(2) } else { - log.Level = 1 + log.SetLevel(1) } showDebugMessages = veryVerbose @@ -445,7 +445,7 @@ Developer Options: saddr = fmt.Sprintf("Port: %d", port) } - if log.LogJSON { + if log.LogJSON() { log.Printf(`Tile38 %s%s %d bit (%s/%s) %s%s, PID: %d. Visit tile38.com/sponsor to support the project`, core.Version, gitsha, strconv.IntSize, runtime.GOARCH, runtime.GOOS, hostd, saddr, os.Getpid()) } else { diff --git a/internal/endpoint/kafka.go b/internal/endpoint/kafka.go index 2cf808ea..4379d24d 100644 --- a/internal/endpoint/kafka.go +++ b/internal/endpoint/kafka.go @@ -68,7 +68,7 @@ func (conn *KafkaConn) Send(msg string) error { } conn.t = time.Now() - if log.Level > 2 { + if log.Level() > 2 { sarama.Logger = lg.New(log.Output(), "[sarama] ", 0) } diff --git a/internal/endpoint/sqs.go b/internal/endpoint/sqs.go index 148f2a3f..fae871e5 100644 --- a/internal/endpoint/sqs.go +++ b/internal/endpoint/sqs.go @@ -90,7 +90,7 @@ func (conn *SQSConn) Send(msg string) error { sess := session.Must(session.NewSession(&aws.Config{ Region: ®ion, Credentials: creds, - CredentialsChainVerboseErrors: aws.Bool(log.Level >= 3), + CredentialsChainVerboseErrors: aws.Bool(log.Level() >= 3), MaxRetries: aws.Int(5), })) svc := sqs.New(sess) diff --git a/internal/log/log.go b/internal/log/log.go index 4250d84c..16d8df87 100644 --- a/internal/log/log.go +++ b/internal/log/log.go @@ -6,91 +6,120 @@ import ( "io" "os" "sync" + "sync/atomic" "time" "go.uber.org/zap" "golang.org/x/term" ) -var mu sync.Mutex +var wmu sync.Mutex var wr io.Writer -var tty bool -var LogJSON = false -var logger *zap.SugaredLogger + +var zmu sync.Mutex +var zlogger *zap.SugaredLogger + +var tty atomic.Bool +var ljson atomic.Bool +var llevel atomic.Int32 + +func init() { + SetOutput(os.Stderr) + SetLevel(1) +} // Level is the log level // 0: silent - do not log // 1: normal - show everything except debug and warn // 2: verbose - show everything except debug // 3: very verbose - show everything -var Level = 1 +func SetLevel(level int) { + if level < 0 { + level = 0 + } else if level > 3 { + level = 3 + } + llevel.Store(int32(level)) +} + +// Level returns the log level +func Level() int { + return int(llevel.Load()) +} + +func SetLogJSON(logJSON bool) { + ljson.Store(logJSON) +} + +func LogJSON() bool { + return ljson.Load() +} // SetOutput sets the output of the logger func SetOutput(w io.Writer) { f, ok := w.(*os.File) - tty = ok && term.IsTerminal(int(f.Fd())) + tty.Store(ok && term.IsTerminal(int(f.Fd()))) + wmu.Lock() wr = w + wmu.Unlock() } // Build a zap logger from default or custom config func Build(c string) error { + var zcfg zap.Config if c == "" { - zcfg := zap.NewProductionConfig() + zcfg = zap.NewProductionConfig() // to be able to filter with Tile38 levels zcfg.Level.SetLevel(zap.DebugLevel) // disable caller because caller is always log.go zcfg.DisableCaller = true - core, err := zcfg.Build() - if err != nil { - return err - } - defer core.Sync() - logger = core.Sugar() } else { - var zcfg zap.Config err := json.Unmarshal([]byte(c), &zcfg) if err != nil { return err } - // to be able to filter with Tile38 levels zcfg.Level.SetLevel(zap.DebugLevel) // disable caller because caller is always log.go zcfg.DisableCaller = true - - core, err := zcfg.Build() - if err != nil { - return err - } - defer core.Sync() - logger = core.Sugar() } + core, err := zcfg.Build() + if err != nil { + return err + } + defer core.Sync() + zmu.Lock() + zlogger = core.Sugar() + zmu.Unlock() return nil } // Set a zap logger func Set(sl *zap.SugaredLogger) { - logger = sl + zmu.Lock() + zlogger = sl + zmu.Unlock() } // Get a zap logger func Get() *zap.SugaredLogger { - return logger + zmu.Lock() + sl := zlogger + zmu.Unlock() + return sl } -func init() { - SetOutput(os.Stderr) -} - -// Output retuns the output writer +// Output returns the output writer func Output() io.Writer { + wmu.Lock() + defer wmu.Unlock() return wr } func log(level int, tag, color string, formatted bool, format string, args ...interface{}) { - if Level < level { + if llevel.Load() < int32(level) { return } var msg string @@ -99,30 +128,32 @@ func log(level int, tag, color string, formatted bool, format string, args ...in } else { msg = fmt.Sprint(args...) } - if LogJSON { + if ljson.Load() { + zmu.Lock() + defer zmu.Unlock() switch tag { case "ERRO": - logger.Error(msg) + zlogger.Error(msg) case "FATA": - logger.Fatal(msg) + zlogger.Fatal(msg) case "WARN": - logger.Warn(msg) + zlogger.Warn(msg) case "DEBU": - logger.Debug(msg) + zlogger.Debug(msg) default: - logger.Info(msg) + zlogger.Info(msg) } return } s := []byte(time.Now().Format("2006/01/02 15:04:05")) s = append(s, ' ') - if tty { + if tty.Load() { s = append(s, color...) } s = append(s, '[') s = append(s, tag...) s = append(s, ']') - if tty { + if tty.Load() { s = append(s, "\x1b[0m"...) } s = append(s, ' ') @@ -130,79 +161,79 @@ func log(level int, tag, color string, formatted bool, format string, args ...in if s[len(s)-1] != '\n' { s = append(s, '\n') } - mu.Lock() + wmu.Lock() wr.Write(s) - mu.Unlock() + wmu.Unlock() } var emptyFormat string // Infof ... func Infof(format string, args ...interface{}) { - if Level >= 1 { + if llevel.Load() >= 1 { log(1, "INFO", "\x1b[36m", true, format, args...) } } // Info ... func Info(args ...interface{}) { - if Level >= 1 { + if llevel.Load() >= 1 { log(1, "INFO", "\x1b[36m", false, emptyFormat, args...) } } // HTTPf ... func HTTPf(format string, args ...interface{}) { - if Level >= 1 { + if llevel.Load() >= 1 { log(1, "HTTP", "\x1b[1m\x1b[30m", true, format, args...) } } // HTTP ... func HTTP(args ...interface{}) { - if Level >= 1 { + if llevel.Load() >= 1 { log(1, "HTTP", "\x1b[1m\x1b[30m", false, emptyFormat, args...) } } // Errorf ... func Errorf(format string, args ...interface{}) { - if Level >= 1 { + if llevel.Load() >= 1 { log(1, "ERRO", "\x1b[1m\x1b[31m", true, format, args...) } } // Error .. func Error(args ...interface{}) { - if Level >= 1 { + if llevel.Load() >= 1 { log(1, "ERRO", "\x1b[1m\x1b[31m", false, emptyFormat, args...) } } // Warnf ... func Warnf(format string, args ...interface{}) { - if Level >= 1 { + if llevel.Load() >= 1 { log(2, "WARN", "\x1b[33m", true, format, args...) } } // Warn ... func Warn(args ...interface{}) { - if Level >= 1 { + if llevel.Load() >= 1 { log(2, "WARN", "\x1b[33m", false, emptyFormat, args...) } } // Debugf ... func Debugf(format string, args ...interface{}) { - if Level >= 3 { + if llevel.Load() >= 3 { log(3, "DEBU", "\x1b[35m", true, format, args...) } } // Debug ... func Debug(args ...interface{}) { - if Level >= 3 { + if llevel.Load() >= 3 { log(3, "DEBU", "\x1b[35m", false, emptyFormat, args...) } } diff --git a/internal/log/log_test.go b/internal/log/log_test.go index 6c86128d..75f30be9 100644 --- a/internal/log/log_test.go +++ b/internal/log/log_test.go @@ -13,7 +13,7 @@ import ( func TestLog(t *testing.T) { f := &bytes.Buffer{} - LogJSON = false + SetLogJSON(false) SetOutput(f) Printf("hello %v", "everyone") if !strings.HasSuffix(f.String(), "hello everyone\n") { @@ -23,7 +23,7 @@ func TestLog(t *testing.T) { func TestLogJSON(t *testing.T) { - LogJSON = true + SetLogJSON(true) Build("") type tcase struct { @@ -40,7 +40,7 @@ func TestLogJSON(t *testing.T) { return func(t *testing.T) { observedZapCore, observedLogs := observer.New(zap.DebugLevel) Set(zap.New(observedZapCore).Sugar()) - Level = tc.level + SetLevel(tc.level) if tc.format != "" { tc.fops(tc.format, tc.args) @@ -187,8 +187,8 @@ func TestLogJSON(t *testing.T) { } func BenchmarkLogPrintf(t *testing.B) { - LogJSON = false - Level = 1 + SetLogJSON(false) + SetLevel(1) SetOutput(io.Discard) t.ResetTimer() for i := 0; i < t.N; i++ { @@ -197,8 +197,8 @@ func BenchmarkLogPrintf(t *testing.B) { } func BenchmarkLogJSONPrintf(t *testing.B) { - LogJSON = true - Level = 1 + SetLogJSON(true) + SetLevel(1) ec := zap.NewProductionEncoderConfig() ec.EncodeDuration = zapcore.NanosDurationEncoder diff --git a/tests/metrics_test.go b/tests/metrics_test.go index 429369d0..28b1b744 100644 --- a/tests/metrics_test.go +++ b/tests/metrics_test.go @@ -27,12 +27,14 @@ func downloadURLWithStatusCode(u string) (int, string, error) { func metrics_basic_test(mc *mockServer) error { + maddr := fmt.Sprintf("http://127.0.0.1:%d/", mc.metricsPort()) + mc.Do("SET", "metrics_test_1", "1", "FIELD", "foo", 5.5, "POINT", 5, 5) mc.Do("SET", "metrics_test_2", "2", "FIELD", "foo", 19.19, "POINT", 19, 19) mc.Do("SET", "metrics_test_2", "3", "FIELD", "foo", 19.19, "POINT", 19, 19) mc.Do("SET", "metrics_test_2", "truck1:driver", "STRING", "John Denton") - status, index, err := downloadURLWithStatusCode("http://127.0.0.1:4321/") + status, index, err := downloadURLWithStatusCode(maddr) if err != nil { return err } @@ -43,7 +45,7 @@ func metrics_basic_test(mc *mockServer) error { return fmt.Errorf("missing link on index page") } - status, metrics, err := downloadURLWithStatusCode("http://127.0.0.1:4321/metrics") + status, metrics, err := downloadURLWithStatusCode(maddr + "metrics") if err != nil { return err } diff --git a/tests/mock_test.go b/tests/mock_test.go index bd0597c2..d9e006d3 100644 --- a/tests/mock_test.go +++ b/tests/mock_test.go @@ -13,7 +13,7 @@ import ( "github.com/gomodule/redigo/redis" "github.com/tidwall/sjson" - tlog "github.com/tidwall/tile38/internal/log" + "github.com/tidwall/tile38/internal/log" "github.com/tidwall/tile38/internal/server" ) @@ -36,6 +36,7 @@ func mockCleanup(silent bool) { type mockServer struct { port int + mport int conn redis.Conn ioJSON bool dir string @@ -46,6 +47,10 @@ func (mc *mockServer) readAOF() ([]byte, error) { return os.ReadFile(filepath.Join(mc.dir, "appendonly.aof")) } +func (mc *mockServer) metricsPort() int { + return mc.mport +} + type MockServerOptions struct { AOFData []byte Silent bool @@ -53,6 +58,14 @@ type MockServerOptions struct { } func mockOpenServer(opts MockServerOptions) (*mockServer, error) { + + logOutput := io.Discard + if os.Getenv("PRINTLOG") == "1" { + logOutput = os.Stderr + log.SetLevel(3) + } + log.SetOutput(logOutput) + rand.Seed(time.Now().UnixNano()) port := rand.Int()%20000 + 20000 dir := fmt.Sprintf("data-mock-%d", port) @@ -69,14 +82,12 @@ func mockOpenServer(opts MockServerOptions) (*mockServer, error) { return nil, err } } - logOutput := io.Discard - if os.Getenv("PRINTLOG") == "1" { - logOutput = os.Stderr - tlog.Level = 3 - } + shutdown := make(chan bool) s := &mockServer{port: port, dir: dir, shutdown: shutdown} - tlog.SetOutput(logOutput) + if opts.Metrics { + s.mport = rand.Int()%20000 + 20000 + } var ferrt int32 // atomic flag for when ferr has been set var ferr error // ferr for when the server fails to start go func() { @@ -91,7 +102,7 @@ func mockOpenServer(opts MockServerOptions) (*mockServer, error) { ShowDebugMessages: true, } if opts.Metrics { - sopts.MetricsAddr = ":4321" + sopts.MetricsAddr = fmt.Sprintf(":%d", s.mport) } err := server.Serve(sopts) if err != nil { diff --git a/tests/tests_test.go b/tests/tests_test.go index 0ee8b4a9..b96b15bc 100644 --- a/tests/tests_test.go +++ b/tests/tests_test.go @@ -26,19 +26,8 @@ const ( white = "\x1b[37m" ) -// type mockTest struct { -// } - -// func mockTestInit() *mockTest { -// mt := &mockTest{} -// return mt -// } - -// func (mt *mockTest) Cleanup() { - -// } - func TestAll(t *testing.T) { + mockCleanup(false) defer mockCleanup(false) @@ -50,39 +39,39 @@ func TestAll(t *testing.T) { os.Exit(1) }() - mc, err := mockOpenServer(MockServerOptions{ - Silent: false, - Metrics: true, - }) - if err != nil { - t.Fatal(err) - } - // log.Printf("Waiting a second for everything to cleanly start...") - // time.Sleep(time.Second * 2) - defer mc.Close() - - runSubTest(t, "keys", mc, subTestKeys) - runSubTest(t, "json", mc, subTestJSON) - runSubTest(t, "search", mc, subTestSearch) - runSubTest(t, "testcmd", mc, subTestTestCmd) - runSubTest(t, "client", mc, subTestClient) - runSubTest(t, "scripts", mc, subTestScripts) - runSubTest(t, "fence", mc, subTestFence) - runSubTest(t, "info", mc, subTestInfo) - runSubTest(t, "timeouts", mc, subTestTimeout) - runSubTest(t, "metrics", mc, subTestMetrics) - runSubTest(t, "aof", mc, subTestAOF) + runSubTest(t, "keys", subTestKeys) + runSubTest(t, "json", subTestJSON) + runSubTest(t, "search", subTestSearch) + runSubTest(t, "testcmd", subTestTestCmd) + runSubTest(t, "client", subTestClient) + runSubTest(t, "scripts", subTestScripts) + runSubTest(t, "fence", subTestFence) + runSubTest(t, "info", subTestInfo) + runSubTest(t, "timeouts", subTestTimeout) + runSubTest(t, "metrics", subTestMetrics) + runSubTest(t, "aof", subTestAOF) } -func runSubTest(t *testing.T, name string, mc *mockServer, test func(t *testing.T, mc *mockServer)) { +func runSubTest(t *testing.T, name string, test func(t *testing.T, mc *mockServer)) { t.Run(name, func(t *testing.T) { + // t.Parallel() + t.Helper() + + mc, err := mockOpenServer(MockServerOptions{ + Silent: true, + Metrics: true, + }) + if err != nil { + t.Fatal(err) + } + defer mc.Close() + fmt.Printf(bright+"Testing %s\n"+clear, name) test(t, mc) }) } func runStep(t *testing.T, mc *mockServer, name string, step func(mc *mockServer) error) { - t.Helper() t.Run(name, func(t *testing.T) { t.Helper() if err := func() error { From c093b041e13e3bd35711f35fb7361497b83f8ee7 Mon Sep 17 00:00:00 2001 From: tidwall Date: Mon, 26 Sep 2022 13:26:46 -0700 Subject: [PATCH 23/33] Parallel integration tests --- go.mod | 3 +- go.sum | 2 + internal/server/server.go | 17 ++- tests/aof_test.go | 7 +- tests/client_test.go | 7 +- tests/fence_test.go | 19 ++-- tests/json_test.go | 10 +- tests/keys_search_test.go | 40 +++---- tests/keys_test.go | 49 +++++---- tests/metrics_test.go | 5 +- tests/mock_test.go | 25 ++++- tests/scripts_test.go | 11 +- tests/stats_test.go | 5 +- tests/testcmd_test.go | 16 ++- tests/tests_test.go | 222 +++++++++++++++++++++++++++++--------- tests/timeout_test.go | 15 ++- 16 files changed, 295 insertions(+), 158 deletions(-) diff --git a/go.mod b/go.mod index 34bc5063..cdd6d812 100644 --- a/go.mod +++ b/go.mod @@ -22,6 +22,7 @@ require ( github.com/tidwall/geojson v1.3.6 github.com/tidwall/gjson v1.14.3 github.com/tidwall/hashmap v1.6.1 + github.com/tidwall/limiter v0.4.0 github.com/tidwall/match v1.1.1 github.com/tidwall/pretty v1.2.0 github.com/tidwall/redbench v0.1.0 @@ -31,6 +32,7 @@ require ( github.com/tidwall/sjson v1.2.4 github.com/xdg/scram v1.0.5 github.com/yuin/gopher-lua v0.0.0-20200816102855-ee81675732da + go.uber.org/atomic v1.5.0 go.uber.org/zap v1.13.0 golang.org/x/net v0.0.0-20220809184613-07c6da5e1ced golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 @@ -95,7 +97,6 @@ require ( github.com/tidwall/tinyqueue v0.1.1 // indirect github.com/xdg/stringprep v1.0.3 // indirect go.opencensus.io v0.22.4 // indirect - go.uber.org/atomic v1.5.0 // indirect go.uber.org/multierr v1.3.0 // indirect go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee // indirect golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa // indirect diff --git a/go.sum b/go.sum index 8674dd65..2c0705b3 100644 --- a/go.sum +++ b/go.sum @@ -368,6 +368,8 @@ github.com/tidwall/grect v0.1.4 h1:dA3oIgNgWdSspFzn1kS4S/RDpZFLrIxAZOdJKjYapOg= github.com/tidwall/grect v0.1.4/go.mod h1:9FBsaYRaR0Tcy4UwefBX/UDcDcDy9V5jUcxHzv2jd5Q= github.com/tidwall/hashmap v1.6.1 h1:FIAHjKwcyOo1Y3/orsQO08floKhInbEX2VQv7CQRNuw= github.com/tidwall/hashmap v1.6.1/go.mod h1:hX452N3VtFD8okD3/6q/yOquJvJmYxmZ1H0nLtwkaxM= +github.com/tidwall/limiter v0.4.0 h1:nj+7mS6aMDRzp15QTVDrgkun0def5/PfB4ogs5NlIVQ= +github.com/tidwall/limiter v0.4.0/go.mod h1:n+qBGuSOgAvgcq1xUvo+mXWg8oBLQC8wkkheN9KZou0= github.com/tidwall/lotsa v1.0.2 h1:dNVBH5MErdaQ/xd9s769R31/n2dXavsQ0Yf4TMEHHw8= github.com/tidwall/lotsa v1.0.2/go.mod h1:X6NiU+4yHA3fE3Puvpnn1XMDrFZrE9JO2/w+UMuqgR8= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= diff --git a/internal/server/server.go b/internal/server/server.go index 7a367592..dd4948a7 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -79,7 +79,9 @@ type Server struct { started time.Time config *Config epc *endpoint.Manager - ln net.Listener // server listener + + lnmu sync.Mutex + ln net.Listener // server listener // env opts geomParseOpts geojson.ParseOptions @@ -296,7 +298,14 @@ func Serve(opts Options) error { <-opts.Shutdown s.stopServer.set(true) log.Warnf("Shutting down...") - s.ln.Close() + s.lnmu.Lock() + ln := s.ln + s.ln = nil + s.lnmu.Unlock() + + if ln != nil { + ln.Close() + } }() // Load the queue before the aof @@ -432,6 +441,9 @@ func (s *Server) netServe() error { if err != nil { return err } + s.lnmu.Lock() + s.ln = ln + s.lnmu.Unlock() var wg sync.WaitGroup defer func() { @@ -445,7 +457,6 @@ func (s *Server) netServe() error { ln.Close() log.Debug("Client connection closed") }() - s.ln = ln log.Infof("Ready to accept connections at %s", ln.Addr()) var clientID int64 diff --git a/tests/aof_test.go b/tests/aof_test.go index 5f74b25a..edde81d6 100644 --- a/tests/aof_test.go +++ b/tests/aof_test.go @@ -4,12 +4,11 @@ import ( "bytes" "errors" "fmt" - "testing" ) -func subTestAOF(t *testing.T, mc *mockServer) { - runStep(t, mc, "loading", aof_loading_test) - // runStep(t, mc, "AOFMD5", aof_AOFMD5_test) +func subTestAOF(g *testGroup) { + g.regSubTest("loading", aof_loading_test) + // g.regSubTest("AOFMD5", aof_AOFMD5_test) } func loadAOFAndClose(aof any) error { diff --git a/tests/client_test.go b/tests/client_test.go index 3d75e18f..d7d754fa 100644 --- a/tests/client_test.go +++ b/tests/client_test.go @@ -4,16 +4,15 @@ import ( "errors" "fmt" "strings" - "testing" "github.com/gomodule/redigo/redis" "github.com/tidwall/gjson" "github.com/tidwall/pretty" ) -func subTestClient(t *testing.T, mc *mockServer) { - runStep(t, mc, "OUTPUT", client_OUTPUT_test) - runStep(t, mc, "CLIENT", client_CLIENT_test) +func subTestClient(g *testGroup) { + g.regSubTest("OUTPUT", client_OUTPUT_test) + g.regSubTest("CLIENT", client_CLIENT_test) } func client_OUTPUT_test(mc *mockServer) error { diff --git a/tests/fence_test.go b/tests/fence_test.go index 192ead53..d668dbb7 100644 --- a/tests/fence_test.go +++ b/tests/fence_test.go @@ -12,30 +12,29 @@ import ( "strings" "sync" "sync/atomic" - "testing" "time" "github.com/gomodule/redigo/redis" "github.com/tidwall/gjson" ) -func subTestFence(t *testing.T, mc *mockServer) { +func subTestFence(g *testGroup) { // Standard - runStep(t, mc, "basic", fence_basic_test) - runStep(t, mc, "channel message order", fence_channel_message_order_test) - runStep(t, mc, "detect inside,outside", fence_detect_inside_test) + g.regSubTest("basic", fence_basic_test) + g.regSubTest("channel message order", fence_channel_message_order_test) + g.regSubTest("detect inside,outside", fence_detect_inside_test) // Roaming - runStep(t, mc, "roaming live", fence_roaming_live_test) - runStep(t, mc, "roaming channel", fence_roaming_channel_test) - runStep(t, mc, "roaming webhook", fence_roaming_webhook_test) + g.regSubTest("roaming live", fence_roaming_live_test) + g.regSubTest("roaming channel", fence_roaming_channel_test) + g.regSubTest("roaming webhook", fence_roaming_webhook_test) // channel meta - runStep(t, mc, "channel meta", fence_channel_meta_test) + g.regSubTest("channel meta", fence_channel_meta_test) // various - runStep(t, mc, "detect eecio", fence_eecio_test) + g.regSubTest("detect eecio", fence_eecio_test) } type fenceReader struct { diff --git a/tests/json_test.go b/tests/json_test.go index 863ac55d..3dda9667 100644 --- a/tests/json_test.go +++ b/tests/json_test.go @@ -1,11 +1,9 @@ package tests -import "testing" - -func subTestJSON(t *testing.T, mc *mockServer) { - runStep(t, mc, "basic", json_JSET_basic_test) - runStep(t, mc, "geojson", json_JSET_geojson_test) - runStep(t, mc, "number", json_JSET_number_test) +func subTestJSON(g *testGroup) { + g.regSubTest("basic", json_JSET_basic_test) + g.regSubTest("geojson", json_JSET_geojson_test) + g.regSubTest("number", json_JSET_number_test) } func json_JSET_basic_test(mc *mockServer) error { diff --git a/tests/keys_search_test.go b/tests/keys_search_test.go index b3f432e1..606bb0e9 100644 --- a/tests/keys_search_test.go +++ b/tests/keys_search_test.go @@ -13,26 +13,26 @@ import ( "github.com/tidwall/gjson" ) -func subTestSearch(t *testing.T, mc *mockServer) { - runStep(t, mc, "KNN_BASIC", keys_KNN_basic_test) - runStep(t, mc, "KNN_RANDOM", keys_KNN_random_test) - runStep(t, mc, "KNN_CURSOR", keys_KNN_cursor_test) - runStep(t, mc, "NEARBY_SPARSE", keys_NEARBY_SPARSE_test) - runStep(t, mc, "WITHIN_CIRCLE", keys_WITHIN_CIRCLE_test) - runStep(t, mc, "WITHIN_SECTOR", keys_WITHIN_SECTOR_test) - runStep(t, mc, "INTERSECTS_CIRCLE", keys_INTERSECTS_CIRCLE_test) - runStep(t, mc, "INTERSECTS_SECTOR", keys_INTERSECTS_SECTOR_test) - runStep(t, mc, "WITHIN", keys_WITHIN_test) - runStep(t, mc, "WITHIN_CURSOR", keys_WITHIN_CURSOR_test) - runStep(t, mc, "WITHIN_CLIPBY", keys_WITHIN_CLIPBY_test) - runStep(t, mc, "INTERSECTS", keys_INTERSECTS_test) - runStep(t, mc, "INTERSECTS_CURSOR", keys_INTERSECTS_CURSOR_test) - runStep(t, mc, "INTERSECTS_CLIPBY", keys_INTERSECTS_CLIPBY_test) - runStep(t, mc, "SCAN_CURSOR", keys_SCAN_CURSOR_test) - runStep(t, mc, "SEARCH_CURSOR", keys_SEARCH_CURSOR_test) - runStep(t, mc, "MATCH", keys_MATCH_test) - runStep(t, mc, "FIELDS", keys_FIELDS_search_test) - runStep(t, mc, "BUFFER", keys_BUFFER_search_test) +func subTestSearch(g *testGroup) { + g.regSubTest("KNN_BASIC", keys_KNN_basic_test) + g.regSubTest("KNN_RANDOM", keys_KNN_random_test) + g.regSubTest("KNN_CURSOR", keys_KNN_cursor_test) + g.regSubTest("NEARBY_SPARSE", keys_NEARBY_SPARSE_test) + g.regSubTest("WITHIN_CIRCLE", keys_WITHIN_CIRCLE_test) + g.regSubTest("WITHIN_SECTOR", keys_WITHIN_SECTOR_test) + g.regSubTest("INTERSECTS_CIRCLE", keys_INTERSECTS_CIRCLE_test) + g.regSubTest("INTERSECTS_SECTOR", keys_INTERSECTS_SECTOR_test) + g.regSubTest("WITHIN", keys_WITHIN_test) + g.regSubTest("WITHIN_CURSOR", keys_WITHIN_CURSOR_test) + g.regSubTest("WITHIN_CLIPBY", keys_WITHIN_CLIPBY_test) + g.regSubTest("INTERSECTS", keys_INTERSECTS_test) + g.regSubTest("INTERSECTS_CURSOR", keys_INTERSECTS_CURSOR_test) + g.regSubTest("INTERSECTS_CLIPBY", keys_INTERSECTS_CLIPBY_test) + g.regSubTest("SCAN_CURSOR", keys_SCAN_CURSOR_test) + g.regSubTest("SEARCH_CURSOR", keys_SEARCH_CURSOR_test) + g.regSubTest("MATCH", keys_MATCH_test) + g.regSubTest("FIELDS", keys_FIELDS_search_test) + g.regSubTest("BUFFER", keys_BUFFER_search_test) } func keys_KNN_basic_test(mc *mockServer) error { diff --git a/tests/keys_test.go b/tests/keys_test.go index 5b2de5bd..d172f501 100644 --- a/tests/keys_test.go +++ b/tests/keys_test.go @@ -5,37 +5,36 @@ import ( "fmt" "math/rand" "strings" - "testing" "time" "github.com/gomodule/redigo/redis" "github.com/tidwall/gjson" ) -func subTestKeys(t *testing.T, mc *mockServer) { - runStep(t, mc, "BOUNDS", keys_BOUNDS_test) - runStep(t, mc, "DEL", keys_DEL_test) - runStep(t, mc, "DROP", keys_DROP_test) - runStep(t, mc, "RENAME", keys_RENAME_test) - runStep(t, mc, "RENAMENX", keys_RENAMENX_test) - runStep(t, mc, "EXPIRE", keys_EXPIRE_test) - runStep(t, mc, "FSET", keys_FSET_test) - runStep(t, mc, "GET", keys_GET_test) - runStep(t, mc, "KEYS", keys_KEYS_test) - runStep(t, mc, "PERSIST", keys_PERSIST_test) - runStep(t, mc, "SET", keys_SET_test) - runStep(t, mc, "STATS", keys_STATS_test) - runStep(t, mc, "TTL", keys_TTL_test) - runStep(t, mc, "SET EX", keys_SET_EX_test) - runStep(t, mc, "PDEL", keys_PDEL_test) - runStep(t, mc, "FIELDS", keys_FIELDS_test) - runStep(t, mc, "WHEREIN", keys_WHEREIN_test) - runStep(t, mc, "WHEREEVAL", keys_WHEREEVAL_test) - runStep(t, mc, "TYPE", keys_TYPE_test) - runStep(t, mc, "FLUSHDB", keys_FLUSHDB_test) - runStep(t, mc, "HEALTHZ", keys_HEALTHZ_test) - runStep(t, mc, "SERVER", keys_SERVER_test) - runStep(t, mc, "INFO", keys_INFO_test) +func subTestKeys(g *testGroup) { + g.regSubTest("BOUNDS", keys_BOUNDS_test) + g.regSubTest("DEL", keys_DEL_test) + g.regSubTest("DROP", keys_DROP_test) + g.regSubTest("RENAME", keys_RENAME_test) + g.regSubTest("RENAMENX", keys_RENAMENX_test) + g.regSubTest("EXPIRE", keys_EXPIRE_test) + g.regSubTest("FSET", keys_FSET_test) + g.regSubTest("GET", keys_GET_test) + g.regSubTest("KEYS", keys_KEYS_test) + g.regSubTest("PERSIST", keys_PERSIST_test) + g.regSubTest("SET", keys_SET_test) + g.regSubTest("STATS", keys_STATS_test) + g.regSubTest("TTL", keys_TTL_test) + g.regSubTest("SET EX", keys_SET_EX_test) + g.regSubTest("PDEL", keys_PDEL_test) + g.regSubTest("FIELDS", keys_FIELDS_test) + g.regSubTest("WHEREIN", keys_WHEREIN_test) + g.regSubTest("WHEREEVAL", keys_WHEREEVAL_test) + g.regSubTest("TYPE", keys_TYPE_test) + g.regSubTest("FLUSHDB", keys_FLUSHDB_test) + g.regSubTest("HEALTHZ", keys_HEALTHZ_test) + g.regSubTest("SERVER", keys_SERVER_test) + g.regSubTest("INFO", keys_INFO_test) } func keys_BOUNDS_test(mc *mockServer) error { diff --git a/tests/metrics_test.go b/tests/metrics_test.go index 28b1b744..963b83d4 100644 --- a/tests/metrics_test.go +++ b/tests/metrics_test.go @@ -5,11 +5,10 @@ import ( "io" "net/http" "strings" - "testing" ) -func subTestMetrics(t *testing.T, mc *mockServer) { - runStep(t, mc, "basic", metrics_basic_test) +func subTestMetrics(g *testGroup) { + g.regSubTest("basic", metrics_basic_test) } func downloadURLWithStatusCode(u string) (int, string, error) { diff --git a/tests/mock_test.go b/tests/mock_test.go index d9e006d3..36a30814 100644 --- a/tests/mock_test.go +++ b/tests/mock_test.go @@ -5,6 +5,7 @@ import ( "fmt" "io" "math/rand" + "net" "os" "path/filepath" "strings" @@ -43,9 +44,9 @@ type mockServer struct { shutdown chan bool } -func (mc *mockServer) readAOF() ([]byte, error) { - return os.ReadFile(filepath.Join(mc.dir, "appendonly.aof")) -} +// func (mc *mockServer) readAOF() ([]byte, error) { +// return os.ReadFile(filepath.Join(mc.dir, "appendonly.aof")) +// } func (mc *mockServer) metricsPort() int { return mc.mport @@ -57,6 +58,20 @@ type MockServerOptions struct { Metrics bool } +var nextPort int32 = 10000 + +func getRandPort() int { + // choose a valid port between 10000-50000 + for { + port := int(atomic.AddInt32(&nextPort, 1)) + ln, err := net.Listen("tcp", fmt.Sprintf(":%d", port)) + if err == nil { + ln.Close() + return port + } + } +} + func mockOpenServer(opts MockServerOptions) (*mockServer, error) { logOutput := io.Discard @@ -67,7 +82,7 @@ func mockOpenServer(opts MockServerOptions) (*mockServer, error) { log.SetOutput(logOutput) rand.Seed(time.Now().UnixNano()) - port := rand.Int()%20000 + 20000 + port := getRandPort() dir := fmt.Sprintf("data-mock-%d", port) if !opts.Silent { fmt.Printf("Starting test server at port %d\n", port) @@ -86,7 +101,7 @@ func mockOpenServer(opts MockServerOptions) (*mockServer, error) { shutdown := make(chan bool) s := &mockServer{port: port, dir: dir, shutdown: shutdown} if opts.Metrics { - s.mport = rand.Int()%20000 + 20000 + s.mport = getRandPort() } var ferrt int32 // atomic flag for when ferr has been set var ferr error // ferr for when the server fails to start diff --git a/tests/scripts_test.go b/tests/scripts_test.go index e879cedf..8dbefc7f 100644 --- a/tests/scripts_test.go +++ b/tests/scripts_test.go @@ -3,14 +3,13 @@ package tests import ( "fmt" "strings" - "testing" ) -func subTestScripts(t *testing.T, mc *mockServer) { - runStep(t, mc, "BASIC", scripts_BASIC_test) - runStep(t, mc, "ATOMIC", scripts_ATOMIC_test) - runStep(t, mc, "READONLY", scripts_READONLY_test) - runStep(t, mc, "NONATOMIC", scripts_NONATOMIC_test) +func subTestScripts(g *testGroup) { + g.regSubTest("BASIC", scripts_BASIC_test) + g.regSubTest("ATOMIC", scripts_ATOMIC_test) + g.regSubTest("READONLY", scripts_READONLY_test) + g.regSubTest("NONATOMIC", scripts_NONATOMIC_test) } func scripts_BASIC_test(mc *mockServer) error { diff --git a/tests/stats_test.go b/tests/stats_test.go index 7cc466ae..30670d66 100644 --- a/tests/stats_test.go +++ b/tests/stats_test.go @@ -2,13 +2,12 @@ package tests import ( "errors" - "testing" "github.com/tidwall/gjson" ) -func subTestInfo(t *testing.T, mc *mockServer) { - runStep(t, mc, "valid json", info_valid_json_test) +func subTestInfo(g *testGroup) { + g.regSubTest("valid json", info_valid_json_test) } func info_valid_json_test(mc *mockServer) error { diff --git a/tests/testcmd_test.go b/tests/testcmd_test.go index 5556b084..7f7b0d43 100644 --- a/tests/testcmd_test.go +++ b/tests/testcmd_test.go @@ -1,15 +1,11 @@ package tests -import ( - "testing" -) - -func subTestTestCmd(t *testing.T, mc *mockServer) { - runStep(t, mc, "WITHIN", testcmd_WITHIN_test) - runStep(t, mc, "INTERSECTS", testcmd_INTERSECTS_test) - runStep(t, mc, "INTERSECTS_CLIP", testcmd_INTERSECTS_CLIP_test) - runStep(t, mc, "ExpressionErrors", testcmd_expressionErrors_test) - runStep(t, mc, "Expressions", testcmd_expression_test) +func subTestTestCmd(g *testGroup) { + g.regSubTest("WITHIN", testcmd_WITHIN_test) + g.regSubTest("INTERSECTS", testcmd_INTERSECTS_test) + g.regSubTest("INTERSECTS_CLIP", testcmd_INTERSECTS_CLIP_test) + g.regSubTest("ExpressionErrors", testcmd_expressionErrors_test) + g.regSubTest("Expressions", testcmd_expression_test) } func testcmd_WITHIN_test(mc *mockServer) error { diff --git a/tests/tests_test.go b/tests/tests_test.go index b96b15bc..b6d33eb8 100644 --- a/tests/tests_test.go +++ b/tests/tests_test.go @@ -5,11 +5,16 @@ import ( "math/rand" "os" "os/signal" + "runtime" + "strings" + "sync" "syscall" "testing" "time" "github.com/gomodule/redigo/redis" + "github.com/tidwall/limiter" + "go.uber.org/atomic" ) const ( @@ -26,10 +31,10 @@ const ( white = "\x1b[37m" ) -func TestAll(t *testing.T) { +func TestIntegration(t *testing.T) { - mockCleanup(false) - defer mockCleanup(false) + mockCleanup(true) + defer mockCleanup(true) ch := make(chan os.Signal, 1) signal.Notify(ch, os.Interrupt, syscall.SIGTERM) @@ -39,63 +44,180 @@ func TestAll(t *testing.T) { os.Exit(1) }() - runSubTest(t, "keys", subTestKeys) - runSubTest(t, "json", subTestJSON) - runSubTest(t, "search", subTestSearch) - runSubTest(t, "testcmd", subTestTestCmd) - runSubTest(t, "client", subTestClient) - runSubTest(t, "scripts", subTestScripts) - runSubTest(t, "fence", subTestFence) - runSubTest(t, "info", subTestInfo) - runSubTest(t, "timeouts", subTestTimeout) - runSubTest(t, "metrics", subTestMetrics) - runSubTest(t, "aof", subTestAOF) + regTestGroup("keys", subTestKeys) + regTestGroup("json", subTestJSON) + regTestGroup("search", subTestSearch) + regTestGroup("testcmd", subTestTestCmd) + regTestGroup("client", subTestClient) + regTestGroup("scripts", subTestScripts) + regTestGroup("fence", subTestFence) + regTestGroup("info", subTestInfo) + regTestGroup("timeouts", subTestTimeout) + regTestGroup("metrics", subTestMetrics) + regTestGroup("aof", subTestAOF) + runTestGroups(t) } -func runSubTest(t *testing.T, name string, test func(t *testing.T, mc *mockServer)) { - t.Run(name, func(t *testing.T) { - // t.Parallel() - t.Helper() +var allGroups []*testGroup +func runTestGroups(t *testing.T) { + limit := runtime.NumCPU() + if limit > 16 { + limit = 16 + } + l := limiter.New(limit) + + // Initialize all stores as "skipped", but they'll be unset if the test is + // not actually skipped. + for _, g := range allGroups { + for _, s := range g.subs { + s.skipped.Store(true) + } + } + for _, g := range allGroups { + func(g *testGroup) { + t.Run(g.name, func(t *testing.T) { + for _, s := range g.subs { + func(s *testGroupSub) { + t.Run(s.name, func(t *testing.T) { + s.skipped.Store(false) + var wg sync.WaitGroup + wg.Add(1) + var err error + go func() { + l.Begin() + defer func() { + l.End() + wg.Done() + }() + err = s.run() + }() + if false { + t.Parallel() + t.Run("bg", func(t *testing.T) { + wg.Wait() + if err != nil { + t.Fatal(err) + } + }) + } + }) + }(s) + } + }) + }(g) + } + + done := make(chan bool) + go func() { + defer func() { done <- true }() + for { + finished := true + for _, g := range allGroups { + skipped := true + for _, s := range g.subs { + if !s.skipped.Load() { + skipped = false + break + } + } + if !skipped && !g.printed.Load() { + fmt.Printf(bright+"Testing %s\n"+clear, g.name) + g.printed.Store(true) + } + for _, s := range g.subs { + if !s.skipped.Load() && !s.printedName.Load() { + fmt.Printf("[..] %s (running) ", s.name) + s.printedName.Store(true) + } + if s.done.Load() && !s.printedResult.Load() { + fmt.Printf("\r") + msg := fmt.Sprintf("[..] %s (running) ", s.name) + fmt.Print(strings.Repeat(" ", len(msg))) + fmt.Printf("\r") + if s.err != nil { + fmt.Printf("["+red+"fail"+clear+"] %s\n", s.name) + } else { + fmt.Printf("["+green+"ok"+clear+"] %s\n", s.name) + } + s.printedResult.Store(true) + } + if !s.skipped.Load() && !s.done.Load() { + finished = false + break + } + } + if !finished { + break + } + } + if finished { + break + } + time.Sleep(time.Second / 4) + } + }() + <-done + var fail bool + for _, g := range allGroups { + for _, s := range g.subs { + if s.err != nil { + t.Errorf("%s/%s/%s\n%s", t.Name(), g.name, s.name, s.err) + fail = true + } + } + } + if fail { + t.Fail() + } + +} + +type testGroup struct { + name string + subs []*testGroupSub + printed atomic.Bool +} + +type testGroupSub struct { + g *testGroup + name string + fn func(mc *mockServer) error + err error + skipped atomic.Bool + done atomic.Bool + printedName atomic.Bool + printedResult atomic.Bool +} + +func regTestGroup(name string, fn func(g *testGroup)) { + g := &testGroup{name: name} + allGroups = append(allGroups, g) + fn(g) +} + +func (g *testGroup) regSubTest(name string, fn func(mc *mockServer) error) { + s := &testGroupSub{g: g, name: name, fn: fn} + g.subs = append(g.subs, s) +} + +func (s *testGroupSub) run() (err error) { + // This all happens in a background routine. + defer func() { + s.err = err + s.done.Store(true) + }() + return func() error { mc, err := mockOpenServer(MockServerOptions{ Silent: true, Metrics: true, }) if err != nil { - t.Fatal(err) + return err } defer mc.Close() - - fmt.Printf(bright+"Testing %s\n"+clear, name) - test(t, mc) - }) -} - -func runStep(t *testing.T, mc *mockServer, name string, step func(mc *mockServer) error) { - t.Run(name, func(t *testing.T) { - t.Helper() - if err := func() error { - // reset the current server - mc.ResetConn() - defer mc.ResetConn() - // clear the database so the test is consistent - if err := mc.DoBatch( - Do("OUTPUT", "resp").OK(), - Do("FLUSHDB").OK(), - ); err != nil { - return err - } - if err := step(mc); err != nil { - return err - } - return nil - }(); err != nil { - fmt.Fprintf(os.Stderr, "["+red+"fail"+clear+"]: %s\n", name) - t.Fatal(err) - // t.Fatal(err) - } - fmt.Printf("["+green+"ok"+clear+"]: %s\n", name) - }) + return s.fn(mc) + }() } func BenchmarkAll(b *testing.B) { diff --git a/tests/timeout_test.go b/tests/timeout_test.go index 123971b4..816e820c 100644 --- a/tests/timeout_test.go +++ b/tests/timeout_test.go @@ -4,19 +4,18 @@ import ( "fmt" "math/rand" "strings" - "testing" "time" "github.com/gomodule/redigo/redis" ) -func subTestTimeout(t *testing.T, mc *mockServer) { - runStep(t, mc, "spatial", timeout_spatial_test) - runStep(t, mc, "search", timeout_search_test) - runStep(t, mc, "scripts", timeout_scripts_test) - runStep(t, mc, "no writes", timeout_no_writes_test) - runStep(t, mc, "within scripts", timeout_within_scripts_test) - runStep(t, mc, "no writes within scripts", timeout_no_writes_within_scripts_test) +func subTestTimeout(g *testGroup) { + g.regSubTest("spatial", timeout_spatial_test) + g.regSubTest("search", timeout_search_test) + g.regSubTest("scripts", timeout_scripts_test) + g.regSubTest("no writes", timeout_no_writes_test) + g.regSubTest("within scripts", timeout_within_scripts_test) + g.regSubTest("no writes within scripts", timeout_no_writes_within_scripts_test) } func setup(mc *mockServer, count int, points bool) (err error) { From ad8d40dee530909edde9f4ab25525a860e7d136e Mon Sep 17 00:00:00 2001 From: tidwall Date: Mon, 26 Sep 2022 15:43:14 -0700 Subject: [PATCH 24/33] Better AOF/AOFMD5 tests --- internal/server/aof.go | 178 ++++++++++++++------------------------ internal/server/server.go | 5 +- tests/aof_test.go | 152 +++++++++++++++++++++++++++++--- tests/mock_test.go | 6 +- tests/tests_test.go | 5 +- 5 files changed, 218 insertions(+), 128 deletions(-) diff --git a/internal/server/aof.go b/internal/server/aof.go index ffab4957..250080b1 100644 --- a/internal/server/aof.go +++ b/internal/server/aof.go @@ -399,73 +399,80 @@ func (s liveAOFSwitches) Error() string { return goingLive } -func (s *Server) cmdAOFMD5(msg *Message) (res resp.Value, err error) { +// AOFMD5 pos size +func (s *Server) cmdAOFMD5(msg *Message) (resp.Value, error) { start := time.Now() - vs := msg.Args[1:] - var ok bool - var spos, ssize string - if vs, spos, ok = tokenval(vs); !ok || spos == "" { - return NOMessage, errInvalidNumberOfArguments + // >> Args + + args := msg.Args + if len(args) != 3 { + return retrerr(errInvalidNumberOfArguments) } - if vs, ssize, ok = tokenval(vs); !ok || ssize == "" { - return NOMessage, errInvalidNumberOfArguments - } - if len(vs) != 0 { - return NOMessage, errInvalidNumberOfArguments - } - pos, err := strconv.ParseInt(spos, 10, 64) + pos, err := strconv.ParseInt(args[1], 10, 64) if err != nil || pos < 0 { - return NOMessage, errInvalidArgument(spos) + return retrerr(errInvalidArgument(args[1])) } - size, err := strconv.ParseInt(ssize, 10, 64) + size, err := strconv.ParseInt(args[2], 10, 64) if err != nil || size < 0 { - return NOMessage, errInvalidArgument(ssize) + return retrerr(errInvalidArgument(args[2])) } + + // >> Operation + sum, err := s.checksum(pos, size) if err != nil { - return NOMessage, err + return retrerr(err) } - switch msg.OutputType { - case JSON: - res = resp.StringValue( - fmt.Sprintf(`{"ok":true,"md5":"%s","elapsed":"%s"}`, sum, time.Since(start))) - case RESP: - res = resp.SimpleStringValue(sum) + + // >> Response + + if msg.OutputType == JSON { + return resp.StringValue(fmt.Sprintf( + `{"ok":true,"md5":"%s","elapsed":"%s"}`, + sum, time.Since(start))), nil } - return res, nil + return resp.SimpleStringValue(sum), nil } -func (s *Server) cmdAOF(msg *Message) (res resp.Value, err error) { +// AOF pos +func (s *Server) cmdAOF(msg *Message) (resp.Value, error) { if s.aof == nil { - return NOMessage, errors.New("aof disabled") + return retrerr(errors.New("aof disabled")) } - vs := msg.Args[1:] - var ok bool - var spos string - if vs, spos, ok = tokenval(vs); !ok || spos == "" { - return NOMessage, errInvalidNumberOfArguments + // >> Args + + args := msg.Args + if len(args) != 2 { + return retrerr(errInvalidNumberOfArguments) } - if len(vs) != 0 { - return NOMessage, errInvalidNumberOfArguments - } - pos, err := strconv.ParseInt(spos, 10, 64) + + pos, err := strconv.ParseInt(args[1], 10, 64) if err != nil || pos < 0 { - return NOMessage, errInvalidArgument(spos) + return retrerr(errInvalidArgument(args[1])) } + + // >> Operation + f, err := os.Open(s.aof.Name()) if err != nil { - return NOMessage, err + return retrerr(err) } defer f.Close() + n, err := f.Seek(0, 2) if err != nil { - return NOMessage, err + return retrerr(err) } + if n < pos { - return NOMessage, errors.New("pos is too big, must be less that the aof_size of leader") + return retrerr(errors.New( + "pos is too big, must be less that the aof_size of leader")) } + + // >> Response + var ls liveAOFSwitches ls.pos = pos return NOMessage, ls @@ -478,8 +485,6 @@ func (s *Server) liveAOF(pos int64, conn net.Conn, rd *PipelineReader, msg *Mess if err != nil { return err } - defer f.Close() - s.mu.Lock() s.aofconnM[conn] = f s.mu.Unlock() @@ -488,91 +493,42 @@ func (s *Server) liveAOF(pos int64, conn net.Conn, rd *PipelineReader, msg *Mess delete(s.aofconnM, conn) s.mu.Unlock() conn.Close() + f.Close() }() if _, err := conn.Write([]byte("+OK\r\n")); err != nil { return err } - if _, err := f.Seek(pos, 0); err != nil { return err } - cond := sync.NewCond(&sync.Mutex{}) - var mustQuit bool + var wg sync.WaitGroup + wg.Add(1) go func() { defer func() { - cond.L.Lock() - mustQuit = true - cond.Broadcast() - cond.L.Unlock() + f.Close() + conn.Close() + wg.Done() }() - for { - vs, err := rd.ReadMessages() - if err != nil { - if err != io.EOF { - log.Error(err) - } - return - } - for _, v := range vs { - switch v.Command() { - default: - log.Error("received a live command that was not QUIT") - return - case "quit", "": - return - } - } - } + // Any incoming message should end the connection + rd.ReadMessages() }() - go func() { - defer func() { - cond.L.Lock() - mustQuit = true - cond.Broadcast() - cond.L.Unlock() - }() - err := func() error { - _, err := io.Copy(conn, f) - if err != nil { + _, err = io.Copy(conn, f) + if err != nil { + return err + } + b := make([]byte, 4096*2) + for { + n, err := f.Read(b) + if n > 0 { + if _, err := conn.Write(b[:n]); err != nil { return err } - - b := make([]byte, 4096) - // The reader needs to be OK with the eof not - for { - n, err := f.Read(b) - if n > 0 { - if _, err := conn.Write(b[:n]); err != nil { - return err - } - } - if err != io.EOF { - if err != nil { - return err - } - continue - } - s.fcond.L.Lock() - s.fcond.Wait() - s.fcond.L.Unlock() - } - }() - if err != nil { - if !strings.Contains(err.Error(), "use of closed network connection") && - !strings.Contains(err.Error(), "bad file descriptor") { - log.Error(err) - } - return } - }() - for { - cond.L.Lock() - if mustQuit { - cond.L.Unlock() - return nil + if err == io.EOF { + time.Sleep(time.Second / 4) + } else if err != nil { + return err } - cond.Wait() - cond.L.Unlock() } } diff --git a/internal/server/server.go b/internal/server/server.go index dd4948a7..ea49da26 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -302,10 +302,13 @@ func Serve(opts Options) error { ln := s.ln s.ln = nil s.lnmu.Unlock() - if ln != nil { ln.Close() } + for conn, f := range s.aofconnM { + conn.Close() + f.Close() + } }() // Load the queue before the aof diff --git a/tests/aof_test.go b/tests/aof_test.go index edde81d6..6ffb0b1b 100644 --- a/tests/aof_test.go +++ b/tests/aof_test.go @@ -2,16 +2,31 @@ package tests import ( "bytes" + "crypto/md5" + "encoding/hex" "errors" "fmt" + "math/rand" + "time" + + "github.com/gomodule/redigo/redis" ) func subTestAOF(g *testGroup) { g.regSubTest("loading", aof_loading_test) - // g.regSubTest("AOFMD5", aof_AOFMD5_test) + g.regSubTest("AOF", aof_AOF_test) + g.regSubTest("AOFMD5", aof_AOFMD5_test) } func loadAOFAndClose(aof any) error { + mc, err := loadAOF(aof) + if mc != nil { + mc.Close() + } + return err +} + +func loadAOF(aof any) (*mockServer, error) { var aofb []byte switch aof := aof.(type) { case []byte: @@ -19,17 +34,13 @@ func loadAOFAndClose(aof any) error { case string: aofb = []byte(aof) default: - return errors.New("aof is not string or bytes") + return nil, errors.New("aof is not string or bytes") } - mc, err := mockOpenServer(MockServerOptions{ + return mockOpenServer(MockServerOptions{ Silent: true, Metrics: false, AOFData: aofb, }) - if mc != nil { - mc.Close() - } - return err } func aof_loading_test(mc *mockServer) error { @@ -79,10 +90,129 @@ func aof_loading_test(mc *mockServer) error { return fmt.Errorf("expected '%v', got '%v'", "Protocol error: expected '$', got '+'", err) } - return nil } -// func aof_AOFMD5_test(mc *mockServer) error { -// return nil -// } +func aof_AOFMD5_test(mc *mockServer) error { + for i := 0; i < 10000; i++ { + _, err := mc.Do("SET", "fleet", rand.Int(), + "POINT", rand.Float64()*180-90, rand.Float64()*360-180) + if err != nil { + return err + } + } + aof, err := mc.readAOF() + if err != nil { + return err + } + check := func(start, size int) func(s string) error { + return func(s string) error { + sum := md5.Sum(aof[start : start+size]) + val := hex.EncodeToString(sum[:]) + if s != val { + return fmt.Errorf("expected '%s', got '%s'", val, s) + } + return nil + } + } + return mc.DoBatch( + Do("AOFMD5").Err("wrong number of arguments for 'aofmd5' command"), + Do("AOFMD5", 0).Err("wrong number of arguments for 'aofmd5' command"), + Do("AOFMD5", 0, 0, 1).Err("wrong number of arguments for 'aofmd5' command"), + Do("AOFMD5", -1, 0).Err("invalid argument '-1'"), + Do("AOFMD5", 1, -1).Err("invalid argument '-1'"), + Do("AOFMD5", 0, 100000000000).Err("EOF"), + Do("AOFMD5", 0, 0).Str("d41d8cd98f00b204e9800998ecf8427e"), + Do("AOFMD5", 0, 0).JSON().Str(`{"ok":true,"md5":"d41d8cd98f00b204e9800998ecf8427e"}`), + Do("AOFMD5", 0, 0).Func(check(0, 0)), + Do("AOFMD5", 0, 1).Func(check(0, 1)), + Do("AOFMD5", 0, 100).Func(check(0, 100)), + Do("AOFMD5", 1002, 4321).Func(check(1002, 4321)), + ) +} + +func aof_AOF_test(mc *mockServer) error { + var argss [][]interface{} + for i := 0; i < 10000; i++ { + args := []interface{}{"SET", "fleet", fmt.Sprint(rand.Int()), + "POINT", fmt.Sprint(rand.Float64()*180 - 90), + fmt.Sprint(rand.Float64()*360 - 180)} + argss = append(argss, args) + _, err := mc.Do(fmt.Sprint(args[0]), args[1:]...) + if err != nil { + return err + } + } + readAll := func() (conn redis.Conn, err error) { + conn, err = redis.Dial("tcp", fmt.Sprintf(":%d", mc.port), + redis.DialReadTimeout(time.Second)) + if err != nil { + return nil, err + } + defer func() { + if err != nil { + conn.Close() + conn = nil + } + }() + if err := conn.Send("AOF", 0); err != nil { + return nil, err + } + if err := conn.Flush(); err != nil { + return nil, err + } + str, err := redis.String(conn.Receive()) + if err != nil { + return nil, err + } + if str != "OK" { + return nil, fmt.Errorf("expected '%s', got '%s'", "OK", str) + } + var t bool + for i := 0; i < len(argss); i++ { + args, err := redis.Values(conn.Receive()) + if err != nil { + return nil, err + } + if t || (len(args) == len(argss[0]) && + fmt.Sprintf("%s", args[2]) == fmt.Sprintf("%s", argss[0][2])) { + t = true + if fmt.Sprintf("%s", args[2]) != + fmt.Sprintf("%s", argss[i][2]) { + return nil, fmt.Errorf("expected '%s', got '%s'", + argss[i][2], args[2]) + } + } else { + i-- + } + } + return conn, nil + } + + conn, err := readAll() + if err != nil { + return err + } + defer conn.Close() + _, err = conn.Do("fancy") // non-existent error + if err == nil || err.Error() != "EOF" { + return fmt.Errorf("expected '%v', got '%v'", "EOF", err) + } + + conn, err = readAll() + if err != nil { + return err + } + defer conn.Close() + _, err = conn.Do("quit") + if err == nil || err.Error() != "EOF" { + return fmt.Errorf("expected '%v', got '%v'", "EOF", err) + } + + return mc.DoBatch( + Do("AOF").Err("wrong number of arguments for 'aof' command"), + Do("AOF", 0, 0).Err("wrong number of arguments for 'aof' command"), + Do("AOF", -1).Err("invalid argument '-1'"), + Do("AOF", 1000000000000).Err("pos is too big, must be less that the aof_size of leader"), + ) +} diff --git a/tests/mock_test.go b/tests/mock_test.go index 36a30814..c87015b1 100644 --- a/tests/mock_test.go +++ b/tests/mock_test.go @@ -44,9 +44,9 @@ type mockServer struct { shutdown chan bool } -// func (mc *mockServer) readAOF() ([]byte, error) { -// return os.ReadFile(filepath.Join(mc.dir, "appendonly.aof")) -// } +func (mc *mockServer) readAOF() ([]byte, error) { + return os.ReadFile(filepath.Join(mc.dir, "appendonly.aof")) +} func (mc *mockServer) metricsPort() int { return mc.mport diff --git a/tests/tests_test.go b/tests/tests_test.go index b6d33eb8..7c2abcd0 100644 --- a/tests/tests_test.go +++ b/tests/tests_test.go @@ -125,14 +125,15 @@ func runTestGroups(t *testing.T) { fmt.Printf(bright+"Testing %s\n"+clear, g.name) g.printed.Store(true) } + const frtmp = "[" + magenta + " " + clear + "] %s (running) " for _, s := range g.subs { if !s.skipped.Load() && !s.printedName.Load() { - fmt.Printf("[..] %s (running) ", s.name) + fmt.Printf(frtmp, s.name) s.printedName.Store(true) } if s.done.Load() && !s.printedResult.Load() { fmt.Printf("\r") - msg := fmt.Sprintf("[..] %s (running) ", s.name) + msg := fmt.Sprintf(frtmp, s.name) fmt.Print(strings.Repeat(" ", len(msg))) fmt.Printf("\r") if s.err != nil { From 588207d162f380897584dfc09f1e2acb495461c0 Mon Sep 17 00:00:00 2001 From: tidwall Date: Mon, 26 Sep 2022 16:43:55 -0700 Subject: [PATCH 25/33] Added AOFSHINK tests --- internal/server/aofshrink.go | 5 +- tests/aof_test.go | 89 +++++++++++++++++++++++++++++------- tests/mock_test.go | 6 +-- 3 files changed, 77 insertions(+), 23 deletions(-) diff --git a/internal/server/aofshrink.go b/internal/server/aofshrink.go index efd93a3e..257b102b 100644 --- a/internal/server/aofshrink.go +++ b/internal/server/aofshrink.go @@ -19,12 +19,9 @@ const maxids = 32 const maxchunk = 4 * 1024 * 1024 func (s *Server) aofshrink() { - if s.aof == nil { - return - } start := time.Now() s.mu.Lock() - if s.shrinking { + if s.aof == nil || s.shrinking { s.mu.Unlock() return } diff --git a/tests/aof_test.go b/tests/aof_test.go index 6ffb0b1b..7a68058a 100644 --- a/tests/aof_test.go +++ b/tests/aof_test.go @@ -7,6 +7,9 @@ import ( "errors" "fmt" "math/rand" + "net" + "net/http" + "sync/atomic" "time" "github.com/gomodule/redigo/redis" @@ -16,6 +19,7 @@ func subTestAOF(g *testGroup) { g.regSubTest("loading", aof_loading_test) g.regSubTest("AOF", aof_AOF_test) g.regSubTest("AOFMD5", aof_AOFMD5_test) + g.regSubTest("AOFSHRINK", aof_AOFSHRINK_test) } func loadAOFAndClose(aof any) error { @@ -131,6 +135,34 @@ func aof_AOFMD5_test(mc *mockServer) error { ) } +func openFollower(mc *mockServer) (conn redis.Conn, err error) { + conn, err = redis.Dial("tcp", fmt.Sprintf(":%d", mc.port), + redis.DialReadTimeout(time.Second)) + if err != nil { + return nil, err + } + defer func() { + if err != nil { + conn.Close() + conn = nil + } + }() + if err := conn.Send("AOF", 0); err != nil { + return nil, err + } + if err := conn.Flush(); err != nil { + return nil, err + } + str, err := redis.String(conn.Receive()) + if err != nil { + return nil, err + } + if str != "OK" { + return nil, fmt.Errorf("expected '%s', got '%s'", "OK", str) + } + return conn, nil +} + func aof_AOF_test(mc *mockServer) error { var argss [][]interface{} for i := 0; i < 10000; i++ { @@ -144,10 +176,9 @@ func aof_AOF_test(mc *mockServer) error { } } readAll := func() (conn redis.Conn, err error) { - conn, err = redis.Dial("tcp", fmt.Sprintf(":%d", mc.port), - redis.DialReadTimeout(time.Second)) + conn, err = openFollower(mc) if err != nil { - return nil, err + return } defer func() { if err != nil { @@ -155,19 +186,6 @@ func aof_AOF_test(mc *mockServer) error { conn = nil } }() - if err := conn.Send("AOF", 0); err != nil { - return nil, err - } - if err := conn.Flush(); err != nil { - return nil, err - } - str, err := redis.String(conn.Receive()) - if err != nil { - return nil, err - } - if str != "OK" { - return nil, fmt.Errorf("expected '%s', got '%s'", "OK", str) - } var t bool for i := 0; i < len(argss); i++ { args, err := redis.Values(conn.Receive()) @@ -216,3 +234,42 @@ func aof_AOF_test(mc *mockServer) error { Do("AOF", 1000000000000).Err("pos is too big, must be less that the aof_size of leader"), ) } + +func aof_AOFSHRINK_test(mc *mockServer) error { + var err error + haddr := fmt.Sprintf("localhost:%d", getNextPort()) + ln, err := net.Listen("tcp", haddr) + if err != nil { + return err + } + defer ln.Close() + var msgs atomic.Int32 + go func() { + mux := http.NewServeMux() + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + msgs.Add(1) + // println(r.URL.Path) + }) + http.Serve(ln, mux) + }() + err = mc.DoBatch( + Do("SETCHAN", "mychan", "INTERSECTS", "mi:0", "BOUNDS", -10, -10, 10, 10).Str("1"), + Do("SETHOOK", "myhook", "http://"+haddr, "INTERSECTS", "mi:0", "BOUNDS", -10, -10, 10, 10).Str("1"), + Do("MASSINSERT", 5, 10000).OK(), + ) + if err != nil { + return err + } + err = mc.DoBatch( + Do("AOFSHRINK").OK(), + Do("MASSINSERT", 5, 10000).OK(), + ) + if err != nil { + return err + } + nmsgs := msgs.Load() + if nmsgs == 0 { + return fmt.Errorf("expected > 0, got %d", nmsgs) + } + return err +} diff --git a/tests/mock_test.go b/tests/mock_test.go index c87015b1..f4354ac8 100644 --- a/tests/mock_test.go +++ b/tests/mock_test.go @@ -60,7 +60,7 @@ type MockServerOptions struct { var nextPort int32 = 10000 -func getRandPort() int { +func getNextPort() int { // choose a valid port between 10000-50000 for { port := int(atomic.AddInt32(&nextPort, 1)) @@ -82,7 +82,7 @@ func mockOpenServer(opts MockServerOptions) (*mockServer, error) { log.SetOutput(logOutput) rand.Seed(time.Now().UnixNano()) - port := getRandPort() + port := getNextPort() dir := fmt.Sprintf("data-mock-%d", port) if !opts.Silent { fmt.Printf("Starting test server at port %d\n", port) @@ -101,7 +101,7 @@ func mockOpenServer(opts MockServerOptions) (*mockServer, error) { shutdown := make(chan bool) s := &mockServer{port: port, dir: dir, shutdown: shutdown} if opts.Metrics { - s.mport = getRandPort() + s.mport = getNextPort() } var ferrt int32 // atomic flag for when ferr has been set var ferr error // ferr for when the server fails to start From 659160289cc6b7242f97de975c03207c3a152c38 Mon Sep 17 00:00:00 2001 From: tidwall Date: Mon, 26 Sep 2022 17:58:51 -0700 Subject: [PATCH 26/33] Better READONLY tests --- internal/server/readonly.go | 42 +++++++++++++++++++++---------------- internal/server/server.go | 2 +- tests/aof_test.go | 13 ++++++++++++ 3 files changed, 38 insertions(+), 19 deletions(-) diff --git a/internal/server/readonly.go b/internal/server/readonly.go index 586a9c0c..94067289 100644 --- a/internal/server/readonly.go +++ b/internal/server/readonly.go @@ -1,44 +1,50 @@ package server import ( - "strings" "time" "github.com/tidwall/resp" "github.com/tidwall/tile38/internal/log" ) -func (s *Server) cmdReadOnly(msg *Message) (res resp.Value, err error) { +// READONLY yes|no +func (s *Server) cmdREADONLY(msg *Message) (resp.Value, error) { start := time.Now() - vs := msg.Args[1:] - var arg string - var ok bool - if vs, arg, ok = tokenval(vs); !ok || arg == "" { - return NOMessage, errInvalidNumberOfArguments + // >> Args + + args := msg.Args + if len(args) != 2 { + return retrerr(errInvalidNumberOfArguments) } - if len(vs) != 0 { - return NOMessage, errInvalidNumberOfArguments - } - update := false - switch strings.ToLower(arg) { + + switch args[1] { + case "yes", "no": default: - return NOMessage, errInvalidArgument(arg) - case "yes": + return retrerr(errInvalidArgument(args[1])) + } + + // >> Operation + + var updated bool + if args[1] == "yes" { if !s.config.readOnly() { - update = true + updated = true s.config.setReadOnly(true) log.Info("read only") } - case "no": + } else { if s.config.readOnly() { - update = true + updated = true s.config.setReadOnly(false) log.Info("read write") } } - if update { + if updated { s.config.write(false) } + + // >> Response + return OKMessage(msg, start), nil } diff --git a/internal/server/server.go b/internal/server/server.go index ea49da26..9a2770a2 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -1181,7 +1181,7 @@ func (s *Server) command(msg *Message, client *Client) ( case "replconf": res, err = s.cmdReplConf(msg, client) case "readonly": - res, err = s.cmdReadOnly(msg) + res, err = s.cmdREADONLY(msg) case "stats": res, err = s.cmdSTATS(msg) case "server": diff --git a/tests/aof_test.go b/tests/aof_test.go index 7a68058a..d82667c2 100644 --- a/tests/aof_test.go +++ b/tests/aof_test.go @@ -20,6 +20,7 @@ func subTestAOF(g *testGroup) { g.regSubTest("AOF", aof_AOF_test) g.regSubTest("AOFMD5", aof_AOFMD5_test) g.regSubTest("AOFSHRINK", aof_AOFSHRINK_test) + g.regSubTest("READONLY", aof_READONLY_test) } func loadAOFAndClose(aof any) error { @@ -273,3 +274,15 @@ func aof_AOFSHRINK_test(mc *mockServer) error { } return err } + +func aof_READONLY_test(mc *mockServer) error { + return mc.DoBatch( + Do("SET", "mykey", "myid", "POINT", "10", "10").OK(), + Do("READONLY", "yes").OK(), + Do("SET", "mykey", "myid", "POINT", "10", "10").Err("read only"), + Do("READONLY", "no").OK(), + Do("SET", "mykey", "myid", "POINT", "10", "10").OK(), + Do("READONLY").Err("wrong number of arguments for 'readonly' command"), + Do("READONLY", "maybe").Err("invalid argument 'maybe'"), + ) +} From 46927b476f1fc3bb86eb3d9ac32012c85167183f Mon Sep 17 00:00:00 2001 From: tidwall Date: Tue, 27 Sep 2022 08:18:17 -0700 Subject: [PATCH 27/33] Better TEST tests --- internal/server/scripts.go | 2 +- internal/server/server.go | 2 +- internal/server/test.go | 28 ++-- tests/testcmd_test.go | 293 ++++++++++++++++++++++--------------- 4 files changed, 192 insertions(+), 133 deletions(-) diff --git a/internal/server/scripts.go b/internal/server/scripts.go index ed267e76..3aae4114 100644 --- a/internal/server/scripts.go +++ b/internal/server/scripts.go @@ -638,7 +638,7 @@ func (s *Server) commandInScript(msg *Message) ( case "keys": res, err = s.cmdKEYS(msg) case "test": - res, err = s.cmdTest(msg) + res, err = s.cmdTEST(msg) case "server": res, err = s.cmdSERVER(msg) } diff --git a/internal/server/server.go b/internal/server/server.go index 9a2770a2..c3538fbf 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -1261,7 +1261,7 @@ func (s *Server) command(msg *Message, client *Client) ( case "publish": res, err = s.cmdPublish(msg) case "test": - res, err = s.cmdTest(msg) + res, err = s.cmdTEST(msg) case "monitor": res, err = s.cmdMonitor(msg) } diff --git a/internal/server/test.go b/internal/server/test.go index 53e0d889..225f5786 100644 --- a/internal/server/test.go +++ b/internal/server/test.go @@ -50,7 +50,7 @@ func (s *Server) parseArea(ovs []string, doClip bool) (vs []string, o geojson.Ob o = geojson.NewPoint(geometry.Point{X: lon, Y: lat}) case "sector": if doClip { - err = errInvalidArgument("cannot clip with " + ltyp) + err = fmt.Errorf("invalid clip type '%s'", typ) return } var slat, slon, smeters, sb1, sb2 string @@ -288,8 +288,15 @@ func (s *Server) parseArea(ovs []string, doClip bool) (vs []string, o geojson.Ob return } -func (s *Server) cmdTest(msg *Message) (res resp.Value, err error) { +// TEST (POINT lat lon)|(GET key id)|(BOUNDS minlat minlon maxlat maxlon)| +// (OBJECT geojson)|(CIRCLE lat lon meters)|(TILE x y z)|(QUADKEY quadkey)| +// (HASH geohash) INTERSECTS|WITHIN [CLIP] (POINT lat lon)|(GET key id)| +// (BOUNDS minlat minlon maxlat maxlon)|(OBJECT geojson)| +// (CIRCLE lat lon meters)|(TILE x y z)|(QUADKEY quadkey)|(HASH geohash)| +// (SECTOR lat lon meters bearing1 bearing2) +func (s *Server) cmdTEST(msg *Message) (res resp.Value, err error) { start := time.Now() + vs := msg.Args[1:] var ok bool @@ -348,8 +355,7 @@ func (s *Server) cmdTest(msg *Message) (res resp.Value, err error) { } } } - switch msg.OutputType { - case JSON: + if msg.OutputType == JSON { var buf bytes.Buffer buf.WriteString(`{"ok":true`) if result != 0 { @@ -362,13 +368,11 @@ func (s *Server) cmdTest(msg *Message) (res resp.Value, err error) { } buf.WriteString(`,"elapsed":"` + time.Since(start).String() + "\"}") return resp.StringValue(buf.String()), nil - case RESP: - if clipped != nil { - return resp.ArrayValue([]resp.Value{ - resp.IntegerValue(result), - resp.StringValue(clipped.JSON())}), nil - } - return resp.IntegerValue(result), nil } - return NOMessage, nil + if clipped != nil { + return resp.ArrayValue([]resp.Value{ + resp.IntegerValue(result), + resp.StringValue(clipped.JSON())}), nil + } + return resp.IntegerValue(result), nil } diff --git a/tests/testcmd_test.go b/tests/testcmd_test.go index 7f7b0d43..2aff3df2 100644 --- a/tests/testcmd_test.go +++ b/tests/testcmd_test.go @@ -25,30 +25,100 @@ func testcmd_WITHIN_test(mc *mockServer) error { poly9 := `{"type":"Polygon","coordinates":[[[-122.44037926197052,37.73313523548048],[-122.44017541408539,37.73313523548048],[-122.44017541408539,37.73336857568778],[-122.44037926197052,37.73336857568778],[-122.44037926197052,37.73313523548048]]]}` poly10 := `{"type":"Polygon","coordinates":[[[-122.44040071964262,37.73359343010089],[-122.4402666091919,37.73359343010089],[-122.4402666091919,37.73373767596864],[-122.44040071964262,37.73373767596864],[-122.44040071964262,37.73359343010089]]]}` - return mc.DoBatch([][]interface{}{ - {"SET", "mykey", "point1", "POINT", 37.7335, -122.4412}, {"OK"}, - {"SET", "mykey", "point2", "POINT", 37.7335, -122.44121}, {"OK"}, - {"SET", "mykey", "line3", "OBJECT", `{"type":"LineString","coordinates":[[-122.4408378,37.7341129],[-122.4408378,37.733]]}`}, {"OK"}, - {"SET", "mykey", "poly4", "OBJECT", `{"type":"Polygon","coordinates":[[[-122.4408378,37.7341129],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.7341129],[-122.4408378,37.7341129]]]}`}, {"OK"}, - {"SET", "mykey", "multipoly5", "OBJECT", `{"type":"MultiPolygon","coordinates":[[[[-122.4408378,37.7341129],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.7341129],[-122.4408378,37.7341129]]],[[[-122.44091033935547,37.731981251280985],[-122.43994474411011,37.731981251280985],[-122.43994474411011,37.73254976045042],[-122.44091033935547,37.73254976045042],[-122.44091033935547,37.731981251280985]]]]}`}, {"OK"}, - {"SET", "mykey", "point6", "POINT", -5, 5}, {"OK"}, - {"SET", "mykey", "point7", "POINT", 33, 21}, {"OK"}, - {"SET", "mykey", "poly8", "OBJECT", poly8}, {"OK"}, + return mc.DoBatch( + Do("SET", "mykey", "point1", "POINT", 37.7335, -122.4412).OK(), + Do("SET", "mykey", "point2", "POINT", 37.7335, -122.44121).OK(), + Do("SET", "mykey", "line3", "OBJECT", `{"type":"LineString","coordinates":[[-122.4408378,37.7341129],[-122.4408378,37.733]]}`).OK(), + Do("SET", "mykey", "poly4", "OBJECT", `{"type":"Polygon","coordinates":[[[-122.4408378,37.7341129],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.7341129],[-122.4408378,37.7341129]]]}`).OK(), + Do("SET", "mykey", "multipoly5", "OBJECT", `{"type":"MultiPolygon","coordinates":[[[[-122.4408378,37.7341129],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.7341129],[-122.4408378,37.7341129]]],[[[-122.44091033935547,37.731981251280985],[-122.43994474411011,37.731981251280985],[-122.43994474411011,37.73254976045042],[-122.44091033935547,37.73254976045042],[-122.44091033935547,37.731981251280985]]]]}`).OK(), + Do("SET", "mykey", "point6", "POINT", -5, 5).OK(), + Do("SET", "mykey", "point7", "POINT", 33, 21).OK(), + Do("SET", "mykey", "poly8", "OBJECT", poly8).OK(), - {"TEST", "GET", "mykey", "point1", "WITHIN", "OBJECT", poly}, {"1"}, - {"TEST", "GET", "mykey", "point1", "WITHIN", "SECTOR", "37.72999", "-122.44760", "1000", "0", "90"}, {"1"}, - {"TEST", "GET", "mykey", "line3", "WITHIN", "OBJECT", poly}, {"1"}, - {"TEST", "GET", "mykey", "poly4", "WITHIN", "OBJECT", poly}, {"1"}, - {"TEST", "GET", "mykey", "multipoly5", "WITHIN", "OBJECT", poly}, {"1"}, - {"TEST", "GET", "mykey", "poly8", "WITHIN", "OBJECT", poly}, {"1"}, - {"TEST", "GET", "mykey", "poly8", "WITHIN", "SECTOR", "37.72999", "-122.44760", "1000", "0", "90"}, {"1"}, + Do("TEST", "GET", "mykey", "point1", "WITHIN", "OBJECT", poly).Str("1"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "SECTOR", "37.72999", "-122.44760", "1000", "0", "90").Str("1"), + Do("TEST", "GET", "mykey", "line3", "WITHIN", "OBJECT", poly).Str("1"), + Do("TEST", "GET", "mykey", "poly4", "WITHIN", "OBJECT", poly).Str("1"), + Do("TEST", "GET", "mykey", "multipoly5", "WITHIN", "OBJECT", poly).Str("1"), + Do("TEST", "GET", "mykey", "poly8", "WITHIN", "OBJECT", poly).Str("1"), + Do("TEST", "GET", "mykey", "poly8", "WITHIN", "SECTOR", "37.72999", "-122.44760", "1000", "0", "90").Str("1"), + Do("TEST", "GET", "mykey", "poly8", "WITHIN", "SECTOR", "37.72999", "-122.44760", "1000", "0", "90").JSON().Str(`{"ok":true,"result":true}`), - {"TEST", "GET", "mykey", "point6", "WITHIN", "OBJECT", poly}, {"0"}, - {"TEST", "GET", "mykey", "point7", "WITHIN", "OBJECT", poly}, {"0"}, + Do("TEST", "GET", "mykey", "point6", "WITHIN", "OBJECT", poly).Str("0"), + Do("TEST", "GET", "mykey", "point6", "WITHIN", "OBJECT", poly).JSON().Str(`{"ok":true,"result":false}`), + Do("TEST", "GET", "mykey", "point7", "WITHIN", "OBJECT", poly).Str("0"), - {"TEST", "OBJECT", poly9, "WITHIN", "OBJECT", poly8}, {"1"}, - {"TEST", "OBJECT", poly10, "WITHIN", "OBJECT", poly8}, {"0"}, - }) + Do("TEST", "OBJECT", poly9, "WITHIN", "OBJECT", poly8).Str("1"), + Do("TEST", "OBJECT", poly10, "WITHIN", "OBJECT", poly8).Str("0"), + + Do("TEST", "GET", "mykey", "point1", "WITHIN", "SECTOR", "37.72999", "-122.44760", "1000", "0", "ff").Err("invalid argument 'ff'"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "SECTOR", "37.72999", "-122.44760", "1000", "ee", "ff").Err("invalid argument 'ee'"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "SECTOR", "37.72999", "-122.44760", "dd", "ee", "ff").Err("invalid argument 'dd'"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "SECTOR", "37.72999", "cc", "dd", "ee", "ff").Err("invalid argument 'cc'"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "SECTOR", "bb", "cc", "dd", "ee", "ff").Err("invalid argument 'bb'"), + + Do("TEST", "GET", "mykey", "point1", "WITHIN", "SECTOR", "37.72999", "-122.44760", "1000", "0").Err("wrong number of arguments for 'test' command"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "SECTOR", "37.72999", "-122.44760", "1000").Err("wrong number of arguments for 'test' command"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "SECTOR", "37.72999", "-122.44760").Err("wrong number of arguments for 'test' command"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "SECTOR", "37.72999").Err("wrong number of arguments for 'test' command"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "SECTOR").Err("wrong number of arguments for 'test' command"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "SECTOR", "37.72999", "-122.44760", "1000", "1", "1").Err("equal bearings (1 == 1), use CIRCLE instead"), + + Do("TEST", "GET", "mykey", "point1", "WITHIN", "CIRCLE", "37.72999", "-122.44760", "10000").Str("1"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "CIRCLE", "37.72999", "-122.44760", "10000", "10000").Err("wrong number of arguments for 'test' command"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "CIRCLE", "37.72999", "-122.44760").Err("wrong number of arguments for 'test' command"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "CIRCLE", "37.72999").Err("wrong number of arguments for 'test' command"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "CIRCLE").Err("wrong number of arguments for 'test' command"), + + Do("TEST", "GET", "mykey", "point1", "WITHIN", "CIRCLE", "37.72999", "-122.44760", "cc").Err("invalid argument 'cc'"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "CIRCLE", "37.72999", "bb", "cc").Err("invalid argument 'bb'"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "CIRCLE", "aa", "bb", "cc").Err("invalid argument 'aa'"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "CIRCLE", "37.72999", "-122.44760", "-10000").Err("invalid argument '-10000'"), + + Do("TEST", "GET", "mykey", "point1", "WITHIN", "hash").Err("wrong number of arguments for 'test' command"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "hash", "123").Str("0"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "hash", "123", "asdf").Err("wrong number of arguments for 'test' command"), + + Do("TEST", "GET", "mykey", "point1", "WITHIN", "quadkey").Err("wrong number of arguments for 'test' command"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "quadkey", "123").Str("0"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "quadkey", "pqowie").Err("invalid argument 'pqowie'"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "quadkey", "123", "asdf").Err("wrong number of arguments for 'test' command"), + + Do("TEST", "GET", "mykey", "point1", "WITHIN", "tile").Err("wrong number of arguments for 'test' command"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "tile", "1").Err("wrong number of arguments for 'test' command"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "tile", "1", "2").Err("wrong number of arguments for 'test' command"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "tile", "1", "2", "3").Str("0"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "tile", "1", "2", "cc").Err("invalid argument 'cc'"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "tile", "1", "bb", "cc").Err("invalid argument 'bb'"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "tile", "aa", "bb", "cc").Err("invalid argument 'aa'"), + + Do("TEST", "GET", "mykey", "point1", "WITHIN", "point", "1", "2").Str("0"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "point").Err("wrong number of arguments for 'test' command"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "point", "1").Err("wrong number of arguments for 'test' command"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "point", "1", "2", "3").Err("wrong number of arguments for 'test' command"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "point", "1", "bb").Err("invalid argument 'bb'"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "point", "aa", "bb").Err("invalid argument 'aa'"), + + Do("TEST", "GET", "mykey", "point1", "WITHIN").Err("wrong number of arguments for 'test' command"), + + Do("TEST", "GET", "mykey", "point1", "WITHIN", "bounds", "1", "2", "3", "4").Str("0"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "bounds", "1", "2", "3", "4", "5").Err("wrong number of arguments for 'test' command"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "bounds", "1", "2", "3").Err("wrong number of arguments for 'test' command"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "bounds", "1", "2").Err("wrong number of arguments for 'test' command"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "bounds", "1").Err("wrong number of arguments for 'test' command"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "bounds").Err("wrong number of arguments for 'test' command"), + + Do("TEST", "GET", "mykey", "point1", "WITHIN", "bounds", "1", "2", "3", "dd").Err("invalid argument 'dd'"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "bounds", "1", "2", "cc", "dd").Err("invalid argument 'cc'"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "bounds", "1", "bb", "cc", "dd").Err("invalid argument 'bb'"), + Do("TEST", "GET", "mykey", "point1", "WITHIN", "bounds", "aa", "bb", "cc", "dd").Err("invalid argument 'aa'"), + + Do("TEST", "GET", "mykey", "point6", "WITHIN", "GET").Err("wrong number of arguments for 'test' command"), + Do("TEST", "GET", "mykey", "point6", "WITHIN", "GET", "mykey").Err("wrong number of arguments for 'test' command"), + Do("TEST", "GET", "mykey", "point6", "WITHIN", "GET", "mykey", "point6").Str("1"), + Do("TEST", "GET", "mykey", "point6", "WITHIN", "GET", "mykey__", "point6").Err("key not found"), + Do("TEST", "GET", "mykey", "point6", "WITHIN", "GET", "mykey", "point6__").Err("id not found"), + ) } func testcmd_INTERSECTS_test(mc *mockServer) error { @@ -69,30 +139,30 @@ func testcmd_INTERSECTS_test(mc *mockServer) error { poly10 := `{"type": "Polygon","coordinates": [[[-122.44040071964262,37.73359343010089],[-122.4402666091919,37.73359343010089],[-122.4402666091919,37.73373767596864],[-122.44040071964262,37.73373767596864],[-122.44040071964262,37.73359343010089]]]}` poly101 := `{"type":"Polygon","coordinates":[[[-122.44051605463028,37.73375464605226],[-122.44028002023695,37.73375464605226],[-122.44028002023695,37.733903134117966],[-122.44051605463028,37.733903134117966],[-122.44051605463028,37.73375464605226]]]}` - return mc.DoBatch([][]interface{}{ - {"SET", "mykey", "point1", "POINT", 37.7335, -122.4412}, {"OK"}, - {"SET", "mykey", "point2", "POINT", 37.7335, -122.44121}, {"OK"}, - {"SET", "mykey", "line3", "OBJECT", `{"type":"LineString","coordinates":[[-122.4408378,37.7341129],[-122.4408378,37.733]]}`}, {"OK"}, - {"SET", "mykey", "poly4", "OBJECT", `{"type":"Polygon","coordinates":[[[-122.4408378,37.7341129],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.7341129],[-122.4408378,37.7341129]]]}`}, {"OK"}, - {"SET", "mykey", "multipoly5", "OBJECT", `{"type":"MultiPolygon","coordinates":[[[[-122.4408378,37.7341129],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.7341129],[-122.4408378,37.7341129]]],[[[-122.44091033935547,37.731981251280985],[-122.43994474411011,37.731981251280985],[-122.43994474411011,37.73254976045042],[-122.44091033935547,37.73254976045042],[-122.44091033935547,37.731981251280985]]]]}`}, {"OK"}, - {"SET", "mykey", "point6", "POINT", -5, 5}, {"OK"}, - {"SET", "mykey", "point7", "POINT", 33, 21}, {"OK"}, - {"SET", "mykey", "poly8", "OBJECT", poly8}, {"OK"}, + return mc.DoBatch( + Do("SET", "mykey", "point1", "POINT", 37.7335, -122.4412).OK(), + Do("SET", "mykey", "point2", "POINT", 37.7335, -122.44121).OK(), + Do("SET", "mykey", "line3", "OBJECT", `{"type":"LineString","coordinates":[[-122.4408378,37.7341129],[-122.4408378,37.733]]}`).OK(), + Do("SET", "mykey", "poly4", "OBJECT", `{"type":"Polygon","coordinates":[[[-122.4408378,37.7341129],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.7341129],[-122.4408378,37.7341129]]]}`).OK(), + Do("SET", "mykey", "multipoly5", "OBJECT", `{"type":"MultiPolygon","coordinates":[[[[-122.4408378,37.7341129],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.7341129],[-122.4408378,37.7341129]]],[[[-122.44091033935547,37.731981251280985],[-122.43994474411011,37.731981251280985],[-122.43994474411011,37.73254976045042],[-122.44091033935547,37.73254976045042],[-122.44091033935547,37.731981251280985]]]]}`).OK(), + Do("SET", "mykey", "point6", "POINT", -5, 5).OK(), + Do("SET", "mykey", "point7", "POINT", 33, 21).OK(), + Do("SET", "mykey", "poly8", "OBJECT", poly8).OK(), - {"TEST", "GET", "mykey", "point1", "INTERSECTS", "OBJECT", poly}, {"1"}, - {"TEST", "GET", "mykey", "point2", "INTERSECTS", "OBJECT", poly}, {"1"}, - {"TEST", "GET", "mykey", "line3", "INTERSECTS", "OBJECT", poly}, {"1"}, - {"TEST", "GET", "mykey", "poly4", "INTERSECTS", "OBJECT", poly}, {"1"}, - {"TEST", "GET", "mykey", "multipoly5", "INTERSECTS", "OBJECT", poly}, {"1"}, - {"TEST", "GET", "mykey", "poly8", "INTERSECTS", "OBJECT", poly}, {"1"}, + Do("TEST", "GET", "mykey", "point1", "INTERSECTS", "OBJECT", poly).Str("1"), + Do("TEST", "GET", "mykey", "point2", "INTERSECTS", "OBJECT", poly).Str("1"), + Do("TEST", "GET", "mykey", "line3", "INTERSECTS", "OBJECT", poly).Str("1"), + Do("TEST", "GET", "mykey", "poly4", "INTERSECTS", "OBJECT", poly).Str("1"), + Do("TEST", "GET", "mykey", "multipoly5", "INTERSECTS", "OBJECT", poly).Str("1"), + Do("TEST", "GET", "mykey", "poly8", "INTERSECTS", "OBJECT", poly).Str("1"), - {"TEST", "GET", "mykey", "point6", "INTERSECTS", "OBJECT", poly}, {"0"}, - {"TEST", "GET", "mykey", "point7", "INTERSECTS", "OBJECT", poly}, {"0"}, + Do("TEST", "GET", "mykey", "point6", "INTERSECTS", "OBJECT", poly).Str("0"), + Do("TEST", "GET", "mykey", "point7", "INTERSECTS", "OBJECT", poly).Str("0"), - {"TEST", "OBJECT", poly9, "INTERSECTS", "OBJECT", poly8}, {"1"}, - {"TEST", "OBJECT", poly10, "INTERSECTS", "OBJECT", poly8}, {"1"}, - {"TEST", "OBJECT", poly101, "INTERSECTS", "OBJECT", poly8}, {"0"}, - }) + Do("TEST", "OBJECT", poly9, "INTERSECTS", "OBJECT", poly8).Str("1"), + Do("TEST", "OBJECT", poly10, "INTERSECTS", "OBJECT", poly8).Str("1"), + Do("TEST", "OBJECT", poly101, "INTERSECTS", "OBJECT", poly8).Str("0"), + ) } func testcmd_INTERSECTS_CLIP_test(mc *mockServer) error { @@ -101,53 +171,48 @@ func testcmd_INTERSECTS_CLIP_test(mc *mockServer) error { multipoly5 := `{"type":"MultiPolygon","coordinates":[[[[-122.4408378,37.7341129],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.7341129],[-122.4408378,37.7341129]]],[[[-122.44091033935547,37.731981251280985],[-122.43994474411011,37.731981251280985],[-122.43994474411011,37.73254976045042],[-122.44091033935547,37.73254976045042],[-122.44091033935547,37.731981251280985]]]]}` poly101 := `{"type":"Polygon","coordinates":[[[-122.44051605463028,37.73375464605226],[-122.44028002023695,37.73375464605226],[-122.44028002023695,37.733903134117966],[-122.44051605463028,37.733903134117966],[-122.44051605463028,37.73375464605226]]]}` - return mc.DoBatch([][]interface{}{ - {"SET", "mykey", "point1", "POINT", 37.7335, -122.4412}, {"OK"}, + return mc.DoBatch( + Do("SET", "mykey", "point1", "POINT", 37.7335, -122.4412).OK(), - {"TEST", "OBJECT", poly9, "INTERSECTS", "CLIP", "OBJECT", "{}"}, {"ERR invalid clip type 'OBJECT'"}, - {"TEST", "OBJECT", poly9, "INTERSECTS", "CLIP", "CIRCLE", "1", "2", "3"}, {"ERR invalid clip type 'CIRCLE'"}, - {"TEST", "OBJECT", poly9, "INTERSECTS", "CLIP", "GET", "mykey", "point1"}, {"ERR invalid clip type 'GET'"}, - {"TEST", "OBJECT", poly9, "WITHIN", "CLIP", "BOUNDS", 10, 10, 20, 20}, {"ERR invalid argument 'CLIP'"}, + Do("TEST", "OBJECT", poly9, "INTERSECTS", "CLIP", "OBJECT", "{}").Err("invalid clip type 'OBJECT'"), + Do("TEST", "OBJECT", poly9, "INTERSECTS", "CLIP", "CIRCLE", "1", "2", "3").Err("invalid clip type 'CIRCLE'"), + Do("TEST", "OBJECT", poly9, "INTERSECTS", "CLIP", "GET", "mykey", "point1").Err("invalid clip type 'GET'"), + Do("TEST", "OBJECT", poly9, "WITHIN", "CLIP", "BOUNDS", 10, 10, 20, 20).Err("invalid argument 'CLIP'"), - {"TEST", "OBJECT", poly9, "INTERSECTS", "CLIP", "BOUNDS", 37.732906137107, -122.44126439094543, 37.73421283683962, -122.43980526924135}, {"[1 " + poly9 + "]"}, - {"TEST", "OBJECT", poly8, "INTERSECTS", "CLIP", "BOUNDS", 37.733, -122.4408378, 37.7341129, -122.44}, {"[1 " + poly8 + "]"}, - {"TEST", "OBJECT", multipoly5, "INTERSECTS", "CLIP", "BOUNDS", 37.73227823422744, -122.44120001792908, 37.73319038868677, -122.43955314159392}, {"[1 " + `{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[-122.4408378,37.73319038868677],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.73319038868677],[-122.4408378,37.73319038868677]]]},"properties":{}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[-122.44091033935547,37.73227823422744],[-122.43994474411011,37.73227823422744],[-122.43994474411011,37.73254976045042],[-122.44091033935547,37.73254976045042],[-122.44091033935547,37.73227823422744]]]},"properties":{}}]}` + "]"}, - {"TEST", "OBJECT", poly101, "INTERSECTS", "CLIP", "BOUNDS", 37.73315644825698, -122.44054287672043, 37.73349585185455, -122.44008690118788}, {"0"}, - }) + Do("TEST", "OBJECT", poly9, "INTERSECTS", "CLIP", "SECTOR").Err("invalid clip type 'SECTOR'"), + + Do("TEST", "OBJECT", poly9, "INTERSECTS", "CLIP", "BOUNDS", 37.732906137107, -122.44126439094543, 37.73421283683962, -122.43980526924135).Str("[1 "+poly9+"]"), + Do("TEST", "OBJECT", poly9, "INTERSECTS", "CLIP", "BOUNDS", 37.732906137107, -122.44126439094543, 37.73421283683962, -122.43980526924135).JSON().Str(`{"ok":true,"result":true,"object":`+poly9+`}`), + Do("TEST", "OBJECT", poly8, "INTERSECTS", "CLIP", "BOUNDS", 37.733, -122.4408378, 37.7341129, -122.44).Str("[1 "+poly8+"]"), + Do("TEST", "OBJECT", multipoly5, "INTERSECTS", "CLIP", "BOUNDS", 37.73227823422744, -122.44120001792908, 37.73319038868677, -122.43955314159392).Str("[1 "+`{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[-122.4408378,37.73319038868677],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.73319038868677],[-122.4408378,37.73319038868677]]]},"properties":{}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[-122.44091033935547,37.73227823422744],[-122.43994474411011,37.73227823422744],[-122.43994474411011,37.73254976045042],[-122.44091033935547,37.73254976045042],[-122.44091033935547,37.73227823422744]]]},"properties":{}}]}`+"]"), + Do("TEST", "OBJECT", poly101, "INTERSECTS", "CLIP", "BOUNDS", 37.73315644825698, -122.44054287672043, 37.73349585185455, -122.44008690118788).Str("0"), + ) } func testcmd_expressionErrors_test(mc *mockServer) error { - return mc.DoBatch([][]interface{}{ - {"SET", "mykey", "foo", "OBJECT", `{"type":"LineString","coordinates":[[-122.4408378,37.7341129],[-122.4408378,37.733]]}`}, {"OK"}, - {"SET", "mykey", "bar", "OBJECT", `{"type":"LineString","coordinates":[[-122.4408378,37.7341129],[-122.4408378,37.733]]}`}, {"OK"}, - {"SET", "mykey", "baz", "OBJECT", `{"type":"LineString","coordinates":[[-122.4408378,37.7341129],[-122.4408378,37.733]]}`}, {"OK"}, + return mc.DoBatch( + Do("SET", "mykey", "foo", "OBJECT", `{"type":"LineString","coordinates":[[-122.4408378,37.7341129],[-122.4408378,37.733]]}`).OK(), + Do("SET", "mykey", "bar", "OBJECT", `{"type":"LineString","coordinates":[[-122.4408378,37.7341129],[-122.4408378,37.733]]}`).OK(), + Do("SET", "mykey", "baz", "OBJECT", `{"type":"LineString","coordinates":[[-122.4408378,37.7341129],[-122.4408378,37.733]]}`).OK(), - {"TEST", "GET", "mykey", "foo", "INTERSECTS", "(", "GET", "mykey", "bar"}, { - "ERR wrong number of arguments for 'test' command"}, - {"TEST", "GET", "mykey", "foo", "INTERSECTS", "GET", "mykey", "bar", ")"}, { - "ERR invalid argument ')'"}, + Do("TEST", "GET", "mykey", "foo", "INTERSECTS", "(", "GET", "mykey", "bar").Err("wrong number of arguments for 'test' command"), + Do("TEST", "GET", "mykey", "foo", "INTERSECTS", "GET", "mykey", "bar", ")").Err("invalid argument ')'"), - {"TEST", "GET", "mykey", "foo", "INTERSECTS", "OR", "GET", "mykey", "bar"}, { - "ERR invalid argument 'or'"}, - {"TEST", "GET", "mykey", "foo", "INTERSECTS", "AND", "GET", "mykey", "bar"}, { - "ERR invalid argument 'and'"}, - {"TEST", "GET", "mykey", "foo", "INTERSECTS", "GET", "mykey", "bar", "OR", "AND", "GET", "mykey", "baz"}, { - "ERR invalid argument 'and'"}, - {"TEST", "GET", "mykey", "foo", "INTERSECTS", "GET", "mykey", "bar", "AND", "OR", "GET", "mykey", "baz"}, { - "ERR invalid argument 'or'"}, - {"TEST", "GET", "mykey", "foo", "INTERSECTS", "GET", "mykey", "bar", "OR", "OR", "GET", "mykey", "baz"}, { - "ERR invalid argument 'or'"}, - {"TEST", "GET", "mykey", "foo", "INTERSECTS", "GET", "mykey", "bar", "AND", "AND", "GET", "mykey", "baz"}, { - "ERR invalid argument 'and'"}, - {"TEST", "GET", "mykey", "foo", "INTERSECTS", "GET", "mykey", "bar", "OR"}, { - "ERR wrong number of arguments for 'test' command"}, - {"TEST", "GET", "mykey", "foo", "INTERSECTS", "GET", "mykey", "bar", "AND"}, { - "ERR wrong number of arguments for 'test' command"}, - {"TEST", "GET", "mykey", "foo", "INTERSECTS", "GET", "mykey", "bar", "NOT"}, { - "ERR wrong number of arguments for 'test' command"}, - {"TEST", "GET", "mykey", "foo", "INTERSECTS", "GET", "mykey", "bar", "NOT", "AND", "GET", "mykey", "baz"}, { - "ERR invalid argument 'and'"}, - }) + Do("TEST", "GET", "mykey", "foo", "INTERSECTS", "OR", "GET", "mykey", "bar").Err("invalid argument 'or'"), + Do("TEST", "GET", "mykey", "foo", "INTERSECTS", "AND", "GET", "mykey", "bar").Err("invalid argument 'and'"), + Do("TEST", "GET", "mykey", "foo", "INTERSECTS", "GET", "mykey", "bar", "OR", "AND", "GET", "mykey", "baz").Err("invalid argument 'and'"), + Do("TEST", "GET", "mykey", "foo", "INTERSECTS", "GET", "mykey", "bar", "AND", "OR", "GET", "mykey", "baz").Err("invalid argument 'or'"), + Do("TEST", "GET", "mykey", "foo", "INTERSECTS", "GET", "mykey", "bar", "OR", "OR", "GET", "mykey", "baz").Err("invalid argument 'or'"), + Do("TEST", "GET", "mykey", "foo", "INTERSECTS", "GET", "mykey", "bar", "AND", "AND", "GET", "mykey", "baz").Err("invalid argument 'and'"), + Do("TEST", "GET", "mykey", "foo", "INTERSECTS", "GET", "mykey", "bar", "OR").Err("wrong number of arguments for 'test' command"), + Do("TEST", "GET", "mykey", "foo", "INTERSECTS", "GET", "mykey", "bar", "AND").Err("wrong number of arguments for 'test' command"), + Do("TEST", "GET", "mykey", "foo", "INTERSECTS", "GET", "mykey", "bar", "NOT").Err("wrong number of arguments for 'test' command"), + Do("TEST", "GET", "mykey", "foo", "INTERSECTS", "GET", "mykey", "bar", "NOT", "AND", "GET", "mykey", "baz").Err("invalid argument 'and'"), + + Do("TEST").Err("wrong number of arguments for 'test' command"), + Do("TEST", "GET", "mykey", "foo").Err("wrong number of arguments for 'test' command"), + Do("TEST", "GET", "mykey", "foo", "jello").Err("invalid argument 'jello'"), + ) } func testcmd_expression_test(mc *mockServer) error { @@ -166,46 +231,36 @@ func testcmd_expression_test(mc *mockServer) error { poly8 := `{"type":"Polygon","coordinates":[[[-122.4408378,37.7341129],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.7341129],[-122.4408378,37.7341129]],[[-122.44060993194579,37.73345766902749],[-122.44044363498686,37.73345766902749],[-122.44044363498686,37.73355524732416],[-122.44060993194579,37.73355524732416],[-122.44060993194579,37.73345766902749]],[[-122.44060724973677,37.7336888869566],[-122.4402102828026,37.7336888869566],[-122.4402102828026,37.7339752567853],[-122.44060724973677,37.7339752567853],[-122.44060724973677,37.7336888869566]]]}` poly9 := `{"type": "Polygon","coordinates": [[[-122.44037926197052,37.73313523548048],[-122.44017541408539,37.73313523548048],[-122.44017541408539,37.73336857568778],[-122.44037926197052,37.73336857568778],[-122.44037926197052,37.73313523548048]]]}` - return mc.DoBatch([][]interface{}{ - {"SET", "mykey", "line3", "OBJECT", `{"type":"LineString","coordinates":[[-122.4408378,37.7341129],[-122.4408378,37.733]]}`}, {"OK"}, - {"SET", "mykey", "poly8", "OBJECT", poly8}, {"OK"}, + return mc.DoBatch( + Do("SET", "mykey", "line3", "OBJECT", `{"type":"LineString","coordinates":[[-122.4408378,37.7341129],[-122.4408378,37.733]]}`).OK(), + Do("SET", "mykey", "poly8", "OBJECT", poly8).OK(), - {"TEST", "OBJECT", poly9, "INTERSECTS", "NOT", "OBJECT", poly}, {"0"}, - {"TEST", "OBJECT", poly9, "INTERSECTS", "NOT", "NOT", "OBJECT", poly}, {"1"}, - {"TEST", "OBJECT", poly9, "INTERSECTS", "NOT", "NOT", "NOT", "OBJECT", poly}, {"0"}, + Do("TEST", "OBJECT", poly9, "INTERSECTS", "NOT", "OBJECT", poly).Str("0"), + Do("TEST", "OBJECT", poly9, "INTERSECTS", "NOT", "NOT", "OBJECT", poly).Str("1"), + Do("TEST", "OBJECT", poly9, "INTERSECTS", "NOT", "NOT", "NOT", "OBJECT", poly).Str("0"), - {"TEST", "OBJECT", poly9, "INTERSECTS", "OBJECT", poly8, "OR", "OBJECT", poly}, {"1"}, - {"TEST", "OBJECT", poly9, "INTERSECTS", "OBJECT", poly8, "AND", "OBJECT", poly}, {"1"}, - {"TEST", "OBJECT", poly9, "INTERSECTS", "GET", "mykey", "poly8", "OR", "OBJECT", poly}, {"1"}, + Do("TEST", "OBJECT", poly9, "INTERSECTS", "OBJECT", poly8, "OR", "OBJECT", poly).Str("1"), + Do("TEST", "OBJECT", poly9, "INTERSECTS", "OBJECT", poly8, "AND", "OBJECT", poly).Str("1"), + Do("TEST", "OBJECT", poly9, "INTERSECTS", "GET", "mykey", "poly8", "OR", "OBJECT", poly).Str("1"), - {"TEST", "OBJECT", poly9, "INTERSECTS", "GET", "mykey", "line3"}, {"0"}, - {"TEST", "OBJECT", poly9, "INTERSECTS", "GET", "mykey", "poly8", "AND", - "(", "OBJECT", poly, "AND", "GET", "mykey", "line3", ")"}, {"0"}, - {"TEST", "OBJECT", poly9, "INTERSECTS", "GET", "mykey", "poly8", "AND", - "(", "OBJECT", poly, "OR", "GET", "mykey", "line3", ")"}, {"1"}, - {"TEST", "OBJECT", poly9, "INTERSECTS", "GET", "mykey", "poly8", "AND", - "(", "OBJECT", poly, "AND", "NOT", "GET", "mykey", "line3", ")"}, {"1"}, - {"TEST", "OBJECT", poly9, "INTERSECTS", "NOT", "GET", "mykey", "line3"}, {"1"}, - {"TEST", "NOT", "OBJECT", poly9, "INTERSECTS", "GET", "mykey", "line3"}, {"1"}, - {"TEST", "OBJECT", poly9, "INTERSECTS", "GET", "mykey", "line3", - "OR", "OBJECT", poly8, "AND", "OBJECT", poly}, {"1"}, - {"TEST", "OBJECT", poly9, "INTERSECTS", "OBJECT", poly8, "AND", "OBJECT", poly, - "OR", "GET", "mykey", "line3"}, {"1"}, - {"TEST", "OBJECT", poly9, "INTERSECTS", "GET", "mykey", "line3", "OR", - "(", "OBJECT", poly8, "AND", "OBJECT", poly, ")"}, {"1"}, - {"TEST", "OBJECT", poly9, "INTERSECTS", - "(", "GET", "mykey", "line3", "OR", "OBJECT", poly8, ")", "AND", "OBJECT", poly}, {"1"}, + Do("TEST", "OBJECT", poly9, "INTERSECTS", "GET", "mykey", "line3").Str("0"), + Do("TEST", "OBJECT", poly9, "INTERSECTS", "GET", "mykey", "poly8", "AND", "(", "OBJECT", poly, "AND", "GET", "mykey", "line3", ")").Str("0"), + Do("TEST", "OBJECT", poly9, "INTERSECTS", "GET", "mykey", "poly8", "AND", "(", "OBJECT", poly, "OR", "GET", "mykey", "line3", ")").Str("1"), + Do("TEST", "OBJECT", poly9, "INTERSECTS", "GET", "mykey", "poly8", "AND", "(", "OBJECT", poly, "AND", "NOT", "GET", "mykey", "line3", ")").Str("1"), + Do("TEST", "OBJECT", poly9, "INTERSECTS", "NOT", "GET", "mykey", "line3").Str("1"), + Do("TEST", "NOT", "OBJECT", poly9, "INTERSECTS", "GET", "mykey", "line3").Str("1"), + Do("TEST", "OBJECT", poly9, "INTERSECTS", "GET", "mykey", "line3", "OR", "OBJECT", poly8, "AND", "OBJECT", poly).Str("1"), + Do("TEST", "OBJECT", poly9, "INTERSECTS", "OBJECT", poly8, "AND", "OBJECT", poly, "OR", "GET", "mykey", "line3").Str("1"), + Do("TEST", "OBJECT", poly9, "INTERSECTS", "GET", "mykey", "line3", "OR", "(", "OBJECT", poly8, "AND", "OBJECT", poly, ")").Str("1"), + Do("TEST", "OBJECT", poly9, "INTERSECTS", "(", "GET", "mykey", "line3", "OR", "OBJECT", poly8, ")", "AND", "OBJECT", poly).Str("1"), - {"TEST", "OBJECT", poly9, "WITHIN", "OBJECT", poly8, "OR", "OBJECT", poly}, {"1"}, - {"TEST", "OBJECT", poly9, "WITHIN", "OBJECT", poly8, "AND", "OBJECT", poly}, {"1"}, + Do("TEST", "OBJECT", poly9, "WITHIN", "OBJECT", poly8, "OR", "OBJECT", poly).Str("1"), + Do("TEST", "OBJECT", poly9, "WITHIN", "OBJECT", poly8, "AND", "OBJECT", poly).Str("1"), - {"TEST", "OBJECT", poly9, "WITHIN", "GET", "mykey", "line3"}, {"0"}, - {"TEST", "OBJECT", poly9, "WITHIN", "GET", "mykey", "poly8", "AND", - "(", "OBJECT", poly, "AND", "GET", "mykey", "line3", ")"}, {"0"}, - {"TEST", "OBJECT", poly9, "WITHIN", "GET", "mykey", "poly8", "AND", - "(", "OBJECT", poly, "OR", "GET", "mykey", "line3", ")"}, {"1"}, - {"TEST", "OBJECT", poly9, "WITHIN", "GET", "mykey", "poly8", "AND", - "(", "OBJECT", poly, "AND", "NOT", "GET", "mykey", "line3", ")"}, {"1"}, - {"TEST", "OBJECT", poly9, "WITHIN", "NOT", "GET", "mykey", "line3"}, {"1"}, - }) + Do("TEST", "OBJECT", poly9, "WITHIN", "GET", "mykey", "line3").Str("0"), + Do("TEST", "OBJECT", poly9, "WITHIN", "GET", "mykey", "poly8", "AND", "(", "OBJECT", poly, "AND", "GET", "mykey", "line3", ")").Str("0"), + Do("TEST", "OBJECT", poly9, "WITHIN", "GET", "mykey", "poly8", "AND", "(", "OBJECT", poly, "OR", "GET", "mykey", "line3", ")").Str("1"), + Do("TEST", "OBJECT", poly9, "WITHIN", "GET", "mykey", "poly8", "AND", "(", "OBJECT", poly, "AND", "NOT", "GET", "mykey", "line3", ")").Str("1"), + Do("TEST", "OBJECT", poly9, "WITHIN", "NOT", "GET", "mykey", "line3").Str("1"), + ) } From cfc96739576e9936f75e092a09f3263854b633bf Mon Sep 17 00:00:00 2001 From: tidwall Date: Tue, 27 Sep 2022 09:47:09 -0700 Subject: [PATCH 28/33] wip - Better AOFMIGRATE tests --- tests/aof_test.go | 19 +++++++++++++++++++ tests/mock_test.go | 12 ++++++++---- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/tests/aof_test.go b/tests/aof_test.go index d82667c2..9ea140f3 100644 --- a/tests/aof_test.go +++ b/tests/aof_test.go @@ -17,6 +17,7 @@ import ( func subTestAOF(g *testGroup) { g.regSubTest("loading", aof_loading_test) + g.regSubTest("migrate", aof_migrate_test) g.regSubTest("AOF", aof_AOF_test) g.regSubTest("AOFMD5", aof_AOFMD5_test) g.regSubTest("AOFSHRINK", aof_AOFSHRINK_test) @@ -286,3 +287,21 @@ func aof_READONLY_test(mc *mockServer) error { Do("READONLY", "maybe").Err("invalid argument 'maybe'"), ) } + +func aof_migrate_test(mc *mockServer) error { + // var aof string + // aof += "set 1 2 point 10 10\r\n" + // aof += "set 2 3 point 30 30\r\n" + // mc2, err := mockOpenServer(MockServerOptions{ + // AOFFileName: "aof", + // AOFData: []byte(aof), + // Silent: true, + // Metrics: true, + // }) + // if err != nil { + // return err + // } + // defer mc2.Close() + + return nil +} diff --git a/tests/mock_test.go b/tests/mock_test.go index f4354ac8..76b15b24 100644 --- a/tests/mock_test.go +++ b/tests/mock_test.go @@ -53,9 +53,10 @@ func (mc *mockServer) metricsPort() int { } type MockServerOptions struct { - AOFData []byte - Silent bool - Metrics bool + AOFFileName string + AOFData []byte + Silent bool + Metrics bool } var nextPort int32 = 10000 @@ -88,10 +89,13 @@ func mockOpenServer(opts MockServerOptions) (*mockServer, error) { fmt.Printf("Starting test server at port %d\n", port) } if len(opts.AOFData) > 0 { + if opts.AOFFileName == "" { + opts.AOFFileName = "appendonly.aof" + } if err := os.MkdirAll(dir, 0777); err != nil { return nil, err } - err := os.WriteFile(filepath.Join(dir, "appendonly.aof"), + err := os.WriteFile(filepath.Join(dir, opts.AOFFileName), opts.AOFData, 0666) if err != nil { return nil, err From 06ebeecf4ab93726e10c2089388591692723b978 Mon Sep 17 00:00:00 2001 From: tidwall Date: Tue, 27 Sep 2022 10:06:47 -0700 Subject: [PATCH 29/33] Better AOFMIGRATE tests --- tests/aof_legacy | Bin 0 -> 56 bytes tests/aof_test.go | 66 ++++++++++++++++++++++++++++++++++++--------- tests/mock_test.go | 5 ++++ 3 files changed, 58 insertions(+), 13 deletions(-) create mode 100644 tests/aof_legacy diff --git a/tests/aof_legacy b/tests/aof_legacy new file mode 100644 index 0000000000000000000000000000000000000000..b0fd1d5e6a7d815b84e4730748cefbe030de02f0 GIT binary patch literal 56 pcmWe;U|=XtEm1I3Fj6SU&&&g|3>1tEgn_~gAOa+3q+pC82LM7F3daBd literal 0 HcmV?d00001 diff --git a/tests/aof_test.go b/tests/aof_test.go index 9ea140f3..b0734f84 100644 --- a/tests/aof_test.go +++ b/tests/aof_test.go @@ -6,6 +6,7 @@ import ( "encoding/hex" "errors" "fmt" + "io" "math/rand" "net" "net/http" @@ -13,6 +14,8 @@ import ( "time" "github.com/gomodule/redigo/redis" + + _ "embed" ) func subTestAOF(g *testGroup) { @@ -288,20 +291,57 @@ func aof_READONLY_test(mc *mockServer) error { ) } +//go:embed aof_legacy +var aofLegacy []byte + func aof_migrate_test(mc *mockServer) error { - // var aof string - // aof += "set 1 2 point 10 10\r\n" - // aof += "set 2 3 point 30 30\r\n" - // mc2, err := mockOpenServer(MockServerOptions{ - // AOFFileName: "aof", - // AOFData: []byte(aof), - // Silent: true, - // Metrics: true, - // }) - // if err != nil { - // return err - // } - // defer mc2.Close() + var aof []byte + for i := 0; i < 10000; i++ { + aof = append(aof, aofLegacy...) + } + var mc2 *mockServer + var err error + defer func() { + mc2.Close() + }() + mc2, err = mockOpenServer(MockServerOptions{ + AOFFileName: "aof", + AOFData: aof, + Silent: true, + Metrics: true, + }) + if err != nil { + return err + } + err = mc2.DoBatch( + Do("GET", "1", "2").Str(`{"type":"Point","coordinates":[20,10]}`), + ) + if err != nil { + return err + } + mc2.Close() + + mc2, err = mockOpenServer(MockServerOptions{ + AOFFileName: "aof", + AOFData: aofLegacy[:len(aofLegacy)-1], + Silent: true, + Metrics: true, + }) + if err != io.ErrUnexpectedEOF { + return fmt.Errorf("expected '%v', got '%v'", io.ErrUnexpectedEOF, err) + } + mc2.Close() + + mc2, err = mockOpenServer(MockServerOptions{ + AOFFileName: "aof", + AOFData: aofLegacy[1:], + Silent: true, + Metrics: true, + }) + if err != io.ErrUnexpectedEOF { + return fmt.Errorf("expected '%v', got '%v'", io.ErrUnexpectedEOF, err) + } + mc2.Close() return nil } diff --git a/tests/mock_test.go b/tests/mock_test.go index 76b15b24..3957dc1b 100644 --- a/tests/mock_test.go +++ b/tests/mock_test.go @@ -36,6 +36,7 @@ func mockCleanup(silent bool) { } type mockServer struct { + closed bool port int mport int conn redis.Conn @@ -169,6 +170,10 @@ func (s *mockServer) waitForStartup(ferr *error, ferrt *int32) error { } func (mc *mockServer) Close() { + if mc == nil || mc.closed { + return + } + mc.closed = true mc.shutdown <- true if mc.conn != nil { mc.conn.Close() From 8608ed0917b78274b69e825fb3832e54f5f882f7 Mon Sep 17 00:00:00 2001 From: tidwall Date: Tue, 27 Sep 2022 10:15:31 -0700 Subject: [PATCH 30/33] Replace abool/aint with new go 1.19 atomics --- internal/server/atomic.go | 32 ----------------------- internal/server/atomic_test.go | 19 -------------- internal/server/checksum.go | 2 +- internal/server/crud.go | 4 +-- internal/server/follow.go | 9 ++++--- internal/server/hooks.go | 5 ++-- internal/server/live.go | 11 ++++---- internal/server/pubsub.go | 2 +- internal/server/server.go | 48 +++++++++++++++++----------------- internal/server/stats.go | 22 ++++++++-------- 10 files changed, 53 insertions(+), 101 deletions(-) delete mode 100644 internal/server/atomic.go delete mode 100644 internal/server/atomic_test.go diff --git a/internal/server/atomic.go b/internal/server/atomic.go deleted file mode 100644 index 97e9a4cb..00000000 --- a/internal/server/atomic.go +++ /dev/null @@ -1,32 +0,0 @@ -package server - -import ( - "sync/atomic" -) - -type aint struct{ v uintptr } - -func (a *aint) add(d int) int { - if d < 0 { - return int(atomic.AddUintptr(&a.v, ^uintptr((d*-1)-1))) - } - return int(atomic.AddUintptr(&a.v, uintptr(d))) -} -func (a *aint) get() int { - return int(atomic.LoadUintptr(&a.v)) -} -func (a *aint) set(i int) int { - return int(atomic.SwapUintptr(&a.v, uintptr(i))) -} - -type abool struct{ v uint32 } - -func (a *abool) on() bool { - return atomic.LoadUint32(&a.v) != 0 -} -func (a *abool) set(t bool) bool { - if t { - return atomic.SwapUint32(&a.v, 1) != 0 - } - return atomic.SwapUint32(&a.v, 0) != 0 -} diff --git a/internal/server/atomic_test.go b/internal/server/atomic_test.go deleted file mode 100644 index 6ed7cc97..00000000 --- a/internal/server/atomic_test.go +++ /dev/null @@ -1,19 +0,0 @@ -package server - -import "testing" - -func TestAtomicInt(t *testing.T) { - var x aint - x.set(10) - if x.get() != 10 { - t.Fatalf("expected %v, got %v", 10, x.get()) - } - x.add(-9) - if x.get() != 1 { - t.Fatalf("expected %v, got %v", 1, x.get()) - } - x.add(-1) - if x.get() != 0 { - t.Fatalf("expected %v, got %v", 0, x.get()) - } -} diff --git a/internal/server/checksum.go b/internal/server/checksum.go index 23462be6..3b612923 100644 --- a/internal/server/checksum.go +++ b/internal/server/checksum.go @@ -144,7 +144,7 @@ func (s *Server) followCheckSome(addr string, followc int, auth string, } s.mu.Lock() defer s.mu.Unlock() - if s.followc.get() != followc { + if int(s.followc.Load()) != followc { return 0, errNoLongerFollowing } if s.aofsz < checksumsz { diff --git a/internal/server/crud.go b/internal/server/crud.go index ced46fa9..d5658edd 100644 --- a/internal/server/crud.go +++ b/internal/server/crud.go @@ -573,7 +573,7 @@ func (s *Server) cmdFLUSHDB(msg *Message) (resp.Value, commandDetails, error) { // (HASH geohash)|(STRING value) func (s *Server) cmdSET(msg *Message) (resp.Value, commandDetails, error) { start := time.Now() - if s.config.maxMemory() > 0 && s.outOfMemory.on() { + if s.config.maxMemory() > 0 && s.outOfMemory.Load() { return retwerr(errOOM) } @@ -780,7 +780,7 @@ func retrerr(err error) (resp.Value, error) { // FSET key id [XX] field value [field value...] func (s *Server) cmdFSET(msg *Message) (resp.Value, commandDetails, error) { start := time.Now() - if s.config.maxMemory() > 0 && s.outOfMemory.on() { + if s.config.maxMemory() > 0 && s.outOfMemory.Load() { return retwerr(errOOM) } diff --git a/internal/server/follow.go b/internal/server/follow.go index cc8b4222..21c39575 100644 --- a/internal/server/follow.go +++ b/internal/server/follow.go @@ -83,10 +83,11 @@ func (s *Server) cmdFollow(msg *Message) (res resp.Value, err error) { } s.config.write(false) if update { - s.followc.add(1) + s.followc.Add(1) if s.config.followHost() != "" { log.Infof("following new host '%s' '%s'.", host, sport) - go s.follow(s.config.followHost(), s.config.followPort(), s.followc.get()) + go s.follow(s.config.followHost(), s.config.followPort(), + int(s.followc.Load())) } else { log.Infof("following no one") } @@ -152,7 +153,7 @@ func doServer(conn *RESPConn) (map[string]string, error) { func (s *Server) followHandleCommand(args []string, followc int, w io.Writer) (int, error) { s.mu.Lock() defer s.mu.Unlock() - if s.followc.get() != followc { + if int(s.followc.Load()) != followc { return s.aofsz, errNoLongerFollowing } msg := &Message{Args: args} @@ -187,7 +188,7 @@ func (s *Server) followDoLeaderAuth(conn *RESPConn, auth string) error { } func (s *Server) followStep(host string, port int, followc int) error { - if s.followc.get() != followc { + if int(s.followc.Load()) != followc { return errNoLongerFollowing } s.mu.Lock() diff --git a/internal/server/hooks.go b/internal/server/hooks.go index bebcaef2..6d13d005 100644 --- a/internal/server/hooks.go +++ b/internal/server/hooks.go @@ -7,6 +7,7 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "time" "github.com/tidwall/buntdb" @@ -501,7 +502,7 @@ type Hook struct { query string epm *endpoint.Manager expires time.Time - counter *aint // counter that grows when a message was sent + counter *atomic.Int64 // counter that grows when a message was sent sig int } @@ -701,7 +702,7 @@ func (h *Hook) proc() (ok bool) { } log.Debugf("Endpoint send ok: %v: %v: %v", idx, endpoint, err) sent = true - h.counter.add(1) + h.counter.Add(1) break } if !sent { diff --git a/internal/server/live.go b/internal/server/live.go index 338523b5..2e694d40 100644 --- a/internal/server/live.go +++ b/internal/server/live.go @@ -11,6 +11,7 @@ import ( "github.com/tidwall/redcon" "github.com/tidwall/tile38/internal/log" + "go.uber.org/atomic" ) type liveBuffer struct { @@ -23,12 +24,12 @@ type liveBuffer struct { func (s *Server) processLives(wg *sync.WaitGroup) { defer wg.Done() - var done abool + var done atomic.Bool wg.Add(1) go func() { defer wg.Done() for { - if done.on() { + if done.Load() { break } s.lcond.Broadcast() @@ -38,8 +39,8 @@ func (s *Server) processLives(wg *sync.WaitGroup) { s.lcond.L.Lock() defer s.lcond.L.Unlock() for { - if s.stopServer.on() { - done.set(true) + if s.stopServer.Load() { + done.Store(true) return } for len(s.lstack) > 0 { @@ -211,7 +212,7 @@ func (s *Server) goLive( return nil // nil return is fine here } } - s.statsTotalMsgsSent.add(len(msgs)) + s.statsTotalMsgsSent.Add(int64(len(msgs))) lb.cond.L.Lock() } diff --git a/internal/server/pubsub.go b/internal/server/pubsub.go index 0a22831f..4016347f 100644 --- a/internal/server/pubsub.go +++ b/internal/server/pubsub.go @@ -280,7 +280,7 @@ func (s *Server) liveSubscription( write(b) } } - s.statsTotalMsgsSent.add(1) + s.statsTotalMsgsSent.Add(1) } m := [2]map[string]bool{ diff --git a/internal/server/server.go b/internal/server/server.go index c3538fbf..46865806 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -89,15 +89,15 @@ type Server struct { http500Errors bool // atomics - followc aint // counter increases when follow property changes - statsTotalConns aint // counter for total connections - statsTotalCommands aint // counter for total commands - statsTotalMsgsSent aint // counter for total sent webhook messages - statsExpired aint // item expiration counter - lastShrinkDuration aint - stopServer abool - outOfMemory abool - loadedAndReady abool // server is loaded and ready for commands + followc atomic.Int64 // counter when follow property changes + statsTotalConns atomic.Int64 // counter for total connections + statsTotalCommands atomic.Int64 // counter for total commands + statsTotalMsgsSent atomic.Int64 // counter for total sent webhook messages + statsExpired atomic.Int64 // item expiration counter + lastShrinkDuration atomic.Int64 + stopServer atomic.Bool + outOfMemory atomic.Bool + loadedAndReady atomic.Bool // server is loaded and ready for commands connsmu sync.RWMutex conns map[int]*Client @@ -296,7 +296,7 @@ func Serve(opts Options) error { go func() { <-opts.Shutdown - s.stopServer.set(true) + s.stopServer.Store(true) log.Warnf("Shutting down...") s.lnmu.Lock() ln := s.ln @@ -363,7 +363,7 @@ func Serve(opts Options) error { go func() { defer bgwg.Done() s.follow(s.config.followHost(), s.config.followPort(), - s.followc.get()) + int(s.followc.Load())) }() } @@ -382,7 +382,7 @@ func Serve(opts Options) error { smux.HandleFunc("/metrics", s.MetricsHandler) err := http.Serve(mln, smux) if err != nil { - if !s.stopServer.on() { + if !s.stopServer.Load() { log.Fatalf("metrics server: %s", err) } } @@ -404,8 +404,8 @@ func Serve(opts Options) error { defer func() { log.Debug("Stopping background routines") // Stop background routines - s.followc.add(1) // this will force any follow communication to die - s.stopServer.set(true) + s.followc.Add(1) // this will force any follow communication to die + s.stopServer.Store(true) if mln != nil { mln.Close() // Stop the metrics server } @@ -413,7 +413,7 @@ func Serve(opts Options) error { }() // Server is now loaded and ready. Wait for network error messages. - s.loadedAndReady.set(true) + s.loadedAndReady.Store(true) return <-nerr } @@ -466,7 +466,7 @@ func (s *Server) netServe() error { for { conn, err := ln.Accept() if err != nil { - if s.stopServer.on() { + if s.stopServer.Load() { return nil } log.Warn(err) @@ -489,7 +489,7 @@ func (s *Server) netServe() error { s.connsmu.Lock() s.conns[client.id] = client s.connsmu.Unlock() - s.statsTotalConns.add(1) + s.statsTotalConns.Add(1) // set the client keep-alive, if needed if s.config.keepAlive() > 0 { @@ -568,7 +568,7 @@ func (s *Server) netServe() error { client.mu.Unlock() // update total command count - s.statsTotalCommands.add(1) + s.statsTotalCommands.Add(1) // handle the command err := s.handleInputCommand(client, msg) @@ -728,14 +728,14 @@ func (s *Server) watchAutoGC(wg *sync.WaitGroup) { } func (s *Server) checkOutOfMemory() { - if s.stopServer.on() { + if s.stopServer.Load() { return } - oom := s.outOfMemory.on() + oom := s.outOfMemory.Load() var mem runtime.MemStats if s.config.maxMemory() == 0 { if oom { - s.outOfMemory.set(false) + s.outOfMemory.Store(false) } return } @@ -743,13 +743,13 @@ func (s *Server) checkOutOfMemory() { runtime.GC() } runtime.ReadMemStats(&mem) - s.outOfMemory.set(int(mem.HeapAlloc) > s.config.maxMemory()) + s.outOfMemory.Store(int(mem.HeapAlloc) > s.config.maxMemory()) } func (s *Server) loopUntilServerStops(dur time.Duration, op func()) { var last time.Time for { - if s.stopServer.on() { + if s.stopServer.Load() { return } now := time.Now() @@ -923,7 +923,7 @@ func (s *Server) handleInputCommand(client *Client, msg *Message) error { return nil } - if !s.loadedAndReady.on() { + if !s.loadedAndReady.Load() { switch msg.Command() { case "output", "ping", "echo": default: diff --git a/internal/server/stats.go b/internal/server/stats.go index 84d2b962..0d324f02 100644 --- a/internal/server/stats.go +++ b/internal/server/stats.go @@ -322,7 +322,7 @@ func (s *Server) extStats(m map[string]interface{}) { // Whether or not an AOF shrink is currently in progress m["tile38_aof_rewrite_in_progress"] = s.shrinking // Length of time the last AOF shrink took - m["tile38_aof_last_rewrite_time_sec"] = s.lastShrinkDuration.get() / int(time.Second) + m["tile38_aof_last_rewrite_time_sec"] = s.lastShrinkDuration.Load() / int64(time.Second) // Duration of the on-going AOF rewrite operation if any var currentShrinkStart time.Time if currentShrinkStart.IsZero() { @@ -335,13 +335,13 @@ func (s *Server) extStats(m map[string]interface{}) { // Whether or no the HTTP transport is being served m["tile38_http_transport"] = s.http // Number of connections accepted by the server - m["tile38_total_connections_received"] = s.statsTotalConns.get() + m["tile38_total_connections_received"] = s.statsTotalConns.Load() // Number of commands processed by the server - m["tile38_total_commands_processed"] = s.statsTotalCommands.get() + m["tile38_total_commands_processed"] = s.statsTotalCommands.Load() // Number of webhook messages sent by server - m["tile38_total_messages_sent"] = s.statsTotalMsgsSent.get() + m["tile38_total_messages_sent"] = s.statsTotalMsgsSent.Load() // Number of key expiration events - m["tile38_expired_keys"] = s.statsExpired.get() + m["tile38_expired_keys"] = s.statsExpired.Load() // Number of connected slaves m["tile38_connected_slaves"] = len(s.aofconnM) @@ -410,8 +410,8 @@ func boolInt(t bool) int { } func (s *Server) writeInfoPersistence(w *bytes.Buffer) { fmt.Fprintf(w, "aof_enabled:%d\r\n", boolInt(s.opts.AppendOnly)) - fmt.Fprintf(w, "aof_rewrite_in_progress:%d\r\n", boolInt(s.shrinking)) // Flag indicating a AOF rewrite operation is on-going - fmt.Fprintf(w, "aof_last_rewrite_time_sec:%d\r\n", s.lastShrinkDuration.get()/int(time.Second)) // Duration of the last AOF rewrite operation in seconds + fmt.Fprintf(w, "aof_rewrite_in_progress:%d\r\n", boolInt(s.shrinking)) // Flag indicating a AOF rewrite operation is on-going + fmt.Fprintf(w, "aof_last_rewrite_time_sec:%d\r\n", s.lastShrinkDuration.Load()/int64(time.Second)) // Duration of the last AOF rewrite operation in seconds var currentShrinkStart time.Time // c.currentShrinkStart.get() if currentShrinkStart.IsZero() { @@ -422,10 +422,10 @@ func (s *Server) writeInfoPersistence(w *bytes.Buffer) { } func (s *Server) writeInfoStats(w *bytes.Buffer) { - fmt.Fprintf(w, "total_connections_received:%d\r\n", s.statsTotalConns.get()) // Total number of connections accepted by the server - fmt.Fprintf(w, "total_commands_processed:%d\r\n", s.statsTotalCommands.get()) // Total number of commands processed by the server - fmt.Fprintf(w, "total_messages_sent:%d\r\n", s.statsTotalMsgsSent.get()) // Total number of commands processed by the server - fmt.Fprintf(w, "expired_keys:%d\r\n", s.statsExpired.get()) // Total number of key expiration events + fmt.Fprintf(w, "total_connections_received:%d\r\n", s.statsTotalConns.Load()) // Total number of connections accepted by the server + fmt.Fprintf(w, "total_commands_processed:%d\r\n", s.statsTotalCommands.Load()) // Total number of commands processed by the server + fmt.Fprintf(w, "total_messages_sent:%d\r\n", s.statsTotalMsgsSent.Load()) // Total number of commands processed by the server + fmt.Fprintf(w, "expired_keys:%d\r\n", s.statsExpired.Load()) // Total number of key expiration events } // writeInfoReplication writes all replication data to the 'info' response From 8e0bf1957d9ce2eee5abf0b56baf79b741394da3 Mon Sep 17 00:00:00 2001 From: tidwall Date: Tue, 27 Sep 2022 13:09:28 -0700 Subject: [PATCH 31/33] wip - follower tests --- tests/follower_test.go | 64 ++++++++++++++++++++++++++++++++++++++++++ tests/tests_test.go | 1 + 2 files changed, 65 insertions(+) create mode 100644 tests/follower_test.go diff --git a/tests/follower_test.go b/tests/follower_test.go new file mode 100644 index 00000000..e7f565b8 --- /dev/null +++ b/tests/follower_test.go @@ -0,0 +1,64 @@ +package tests + +import "time" + +func subTestFollower(g *testGroup) { + g.regSubTest("follow", follower_follow_test) +} + +func follower_follow_test(mc *mockServer) error { + mc2, err := mockOpenServer(MockServerOptions{ + Silent: true, Metrics: false, + }) + if err != nil { + return err + } + defer mc2.Close() + err = mc.DoBatch( + Do("SET", "mykey", "truck1", "POINT", 10, 10).OK(), + Do("SET", "mykey", "truck2", "POINT", 10, 10).OK(), + Do("SET", "mykey", "truck3", "POINT", 10, 10).OK(), + Do("SET", "mykey", "truck4", "POINT", 10, 10).OK(), + Do("SET", "mykey", "truck5", "POINT", 10, 10).OK(), + Do("SET", "mykey", "truck6", "POINT", 10, 10).OK(), + ) + if err != nil { + return err + } + err = mc2.DoBatch( + Do("SET", "mykey2", "truck1", "POINT", 10, 10).OK(), + Do("SET", "mykey2", "truck2", "POINT", 10, 10).OK(), + Do("GET", "mykey2", "truck1").Str(`{"type":"Point","coordinates":[10,10]}`), + Do("GET", "mykey2", "truck2").Str(`{"type":"Point","coordinates":[10,10]}`), + + Do("FOLLOW", "localhost", mc.port).OK(), + Do("GET", "mykey2", "truck1").Err("catching up to leader"), + Sleep(time.Second), + Do("GET", "mykey2", "truck1").Err(``), + Do("GET", "mykey2", "truck2").Err(``), + ) + if err != nil { + return err + } + + // err = mc.DoBatch( + // Do("SET", "mykey", "truck7", "POINT", 10, 10).OK(), + // Do("SET", "mykey", "truck8", "POINT", 10, 10).OK(), + // Do("SET", "mykey", "truck9", "POINT", 10, 10).OK(), + // ) + // if err != nil { + // return err + // } + + // err = mc2.DoBatch( + // Sleep(time.Second/2), + // Do("GET", "mykey1", "truck7").Str(`{"type":"Point","coordinates":[10,10]}`), + // Do("GET", "mykey1", "truck8").Str(`{"type":"Point","coordinates":[10,10]}`), + // Do("GET", "mykey1", "truck9").Str(`{"type":"Point","coordinates":[10,10]}`), + // ) + // if err != nil { + // return err + // } + + return nil +} diff --git a/tests/tests_test.go b/tests/tests_test.go index 7c2abcd0..0546236b 100644 --- a/tests/tests_test.go +++ b/tests/tests_test.go @@ -55,6 +55,7 @@ func TestIntegration(t *testing.T) { regTestGroup("timeouts", subTestTimeout) regTestGroup("metrics", subTestMetrics) regTestGroup("aof", subTestAOF) + regTestGroup("follower", subTestFollower) runTestGroups(t) } From 77b4efa55f2b6e7a3e3e091608d66d1acd63d6d2 Mon Sep 17 00:00:00 2001 From: tidwall Date: Tue, 27 Sep 2022 14:06:24 -0700 Subject: [PATCH 32/33] Better FOLLOW/MONITOR tests --- tests/follower_test.go | 46 +++++++++++++------------ tests/monitor_test.go | 77 ++++++++++++++++++++++++++++++++++++++++++ tests/tests_test.go | 3 +- 3 files changed, 104 insertions(+), 22 deletions(-) create mode 100644 tests/monitor_test.go diff --git a/tests/follower_test.go b/tests/follower_test.go index e7f565b8..97d44add 100644 --- a/tests/follower_test.go +++ b/tests/follower_test.go @@ -21,6 +21,8 @@ func follower_follow_test(mc *mockServer) error { Do("SET", "mykey", "truck4", "POINT", 10, 10).OK(), Do("SET", "mykey", "truck5", "POINT", 10, 10).OK(), Do("SET", "mykey", "truck6", "POINT", 10, 10).OK(), + Do("CONFIG", "SET", "requirepass", "1234").OK(), + Do("AUTH", "1234").OK(), ) if err != nil { return err @@ -31,34 +33,36 @@ func follower_follow_test(mc *mockServer) error { Do("GET", "mykey2", "truck1").Str(`{"type":"Point","coordinates":[10,10]}`), Do("GET", "mykey2", "truck2").Str(`{"type":"Point","coordinates":[10,10]}`), + Do("CONFIG", "SET", "leaderauth", "1234").OK(), Do("FOLLOW", "localhost", mc.port).OK(), - Do("GET", "mykey2", "truck1").Err("catching up to leader"), - Sleep(time.Second), - Do("GET", "mykey2", "truck1").Err(``), - Do("GET", "mykey2", "truck2").Err(``), + Do("GET", "mykey", "truck1").Err("catching up to leader"), + Sleep(time.Second/2), + + Do("GET", "mykey", "truck1").Err(`{"type":"Point","coordinates":[10,10]}`), + Do("GET", "mykey", "truck2").Err(`{"type":"Point","coordinates":[10,10]}`), ) if err != nil { return err } - // err = mc.DoBatch( - // Do("SET", "mykey", "truck7", "POINT", 10, 10).OK(), - // Do("SET", "mykey", "truck8", "POINT", 10, 10).OK(), - // Do("SET", "mykey", "truck9", "POINT", 10, 10).OK(), - // ) - // if err != nil { - // return err - // } + err = mc.DoBatch( + Do("SET", "mykey", "truck7", "POINT", 10, 10).OK(), + Do("SET", "mykey", "truck8", "POINT", 10, 10).OK(), + Do("SET", "mykey", "truck9", "POINT", 10, 10).OK(), + ) + if err != nil { + return err + } - // err = mc2.DoBatch( - // Sleep(time.Second/2), - // Do("GET", "mykey1", "truck7").Str(`{"type":"Point","coordinates":[10,10]}`), - // Do("GET", "mykey1", "truck8").Str(`{"type":"Point","coordinates":[10,10]}`), - // Do("GET", "mykey1", "truck9").Str(`{"type":"Point","coordinates":[10,10]}`), - // ) - // if err != nil { - // return err - // } + err = mc2.DoBatch( + Sleep(time.Second/2), + Do("GET", "mykey", "truck7").Str(`{"type":"Point","coordinates":[10,10]}`), + Do("GET", "mykey", "truck8").Str(`{"type":"Point","coordinates":[10,10]}`), + Do("GET", "mykey", "truck9").Str(`{"type":"Point","coordinates":[10,10]}`), + ) + if err != nil { + return err + } return nil } diff --git a/tests/monitor_test.go b/tests/monitor_test.go new file mode 100644 index 00000000..32818430 --- /dev/null +++ b/tests/monitor_test.go @@ -0,0 +1,77 @@ +package tests + +import ( + "fmt" + "strings" + "sync" + + "github.com/gomodule/redigo/redis" +) + +func subTestMonitor(g *testGroup) { + g.regSubTest("monitor", follower_monitor_test) +} + +func follower_monitor_test(mc *mockServer) error { + N := 1000 + ch := make(chan error) + var wg sync.WaitGroup + wg.Add(1) + go func() { + ch <- func() error { + conn, err := redis.Dial("tcp", fmt.Sprintf("localhost:%d", mc.port)) + if err != nil { + wg.Done() + return err + } + defer conn.Close() + s, err := redis.String(conn.Do("MONITOR")) + if err != nil { + wg.Done() + return err + } + if s != "OK" { + wg.Done() + return fmt.Errorf("expected '%s', got '%s'", "OK", s) + } + wg.Done() + + for i := 0; i < N; i++ { + s, err := redis.String(conn.Receive()) + if err != nil { + return err + } + ex := fmt.Sprintf(`"mykey" "%d"`, i) + if !strings.Contains(s, ex) { + return fmt.Errorf("expected '%s', got '%s'", ex, s) + } + } + return nil + }() + }() + + wg.Wait() + + conn, err := redis.Dial("tcp", fmt.Sprintf("localhost:%d", mc.port)) + if err != nil { + return err + } + defer conn.Close() + + for i := 0; i < N; i++ { + s, err := redis.String(conn.Do("SET", "mykey", i, "POINT", 10, 10)) + if err != nil { + return err + } + if s != "OK" { + return fmt.Errorf("expected '%s', got '%s'", "OK", s) + } + } + + err = <-ch + if err != nil { + err = fmt.Errorf("monitor client: %w", err) + } + + return err +} diff --git a/tests/tests_test.go b/tests/tests_test.go index 0546236b..7298426c 100644 --- a/tests/tests_test.go +++ b/tests/tests_test.go @@ -54,8 +54,9 @@ func TestIntegration(t *testing.T) { regTestGroup("info", subTestInfo) regTestGroup("timeouts", subTestTimeout) regTestGroup("metrics", subTestMetrics) - regTestGroup("aof", subTestAOF) regTestGroup("follower", subTestFollower) + regTestGroup("aof", subTestAOF) + regTestGroup("monitor", subTestMonitor) runTestGroups(t) } From e1df4dbf784ce0fade34d20e4a580d45e06f9c9e Mon Sep 17 00:00:00 2001 From: tidwall Date: Tue, 27 Sep 2022 14:19:57 -0700 Subject: [PATCH 33/33] Better OUTPUT tests --- internal/server/output.go | 30 +++++++++++++----------------- internal/server/server.go | 2 +- tests/client_test.go | 6 ++++++ 3 files changed, 20 insertions(+), 18 deletions(-) diff --git a/internal/server/output.go b/internal/server/output.go index 65824ada..51476a1e 100644 --- a/internal/server/output.go +++ b/internal/server/output.go @@ -7,35 +7,31 @@ import ( "github.com/tidwall/resp" ) -func (s *Server) cmdOutput(msg *Message) (res resp.Value, err error) { +// OUTPUT [resp|json] +func (s *Server) cmdOUTPUT(msg *Message) (resp.Value, error) { start := time.Now() - vs := msg.Args[1:] - var arg string - var ok bool - if len(vs) != 0 { - if _, arg, ok = tokenval(vs); !ok || arg == "" { - return NOMessage, errInvalidNumberOfArguments + args := msg.Args + switch len(args) { + case 1: + if msg.OutputType == JSON { + return resp.StringValue(`{"ok":true,"output":"json","elapsed":` + + time.Since(start).String() + `}`), nil } + return resp.StringValue("resp"), nil + case 2: // Setting the original message output type will be picked up by the // server prior to the next command being executed. - switch strings.ToLower(arg) { + switch strings.ToLower(args[1]) { default: - return NOMessage, errInvalidArgument(arg) + return retrerr(errInvalidArgument(args[1])) case "json": msg.OutputType = JSON case "resp": msg.OutputType = RESP } return OKMessage(msg, start), nil - } - // return the output - switch msg.OutputType { default: - return NOMessage, nil - case JSON: - return resp.StringValue(`{"ok":true,"output":"json","elapsed":` + time.Since(start).String() + `}`), nil - case RESP: - return resp.StringValue("resp"), nil + return retrerr(errInvalidNumberOfArguments) } } diff --git a/internal/server/server.go b/internal/server/server.go index 46865806..bf792724 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -1215,7 +1215,7 @@ func (s *Server) command(msg *Message, client *Client) ( case "keys": res, err = s.cmdKEYS(msg) case "output": - res, err = s.cmdOutput(msg) + res, err = s.cmdOUTPUT(msg) case "aof": res, err = s.cmdAOF(msg) case "aofmd5": diff --git a/tests/client_test.go b/tests/client_test.go index d7d754fa..2c8d3b7d 100644 --- a/tests/client_test.go +++ b/tests/client_test.go @@ -18,8 +18,14 @@ func subTestClient(g *testGroup) { func client_OUTPUT_test(mc *mockServer) error { if err := mc.DoBatch( // tests removal of "elapsed" member. + Do("OUTPUT", "json", "yaml").Err(`wrong number of arguments for 'output' command`), Do("OUTPUT", "json").Str(`{"ok":true}`), + Do("OUTPUT").JSON().Str(`{"ok":true,"output":"json"}`), + Do("OUTPUT").Str(`resp`), // this is due to the internal Do test Do("OUTPUT", "resp").OK(), + Do("OUTPUT", "yaml").Err(`invalid argument 'yaml'`), + Do("OUTPUT").Str(`resp`), + Do("OUTPUT").JSON().Str(`{"ok":true,"output":"json"}`), ); err != nil { return err }