From cf7f49fd9bfe84807e8e4cdbc48683cf99884de8 Mon Sep 17 00:00:00 2001 From: tidwall Date: Thu, 20 Oct 2022 11:17:01 -0700 Subject: [PATCH] Add field path queries for json and comparison operators It's now possible to query a JSON field using a GJSON path. SET fleet truck1 FIELD props '{"speed":58,"name":"Andy"}' POINT 33 -112 You can then use the GJSON type path to return the objects that match the WHERE. SCAN fleet WHERE props.speed 50 inf SCAN fleet WHERE props.name Andy Andy Included in this commit is support for '==', '<', '>', '<=', '>=', and '!='. The previous queries could be written like: SCAN fleet WHERE props.speed > 50 SCAN fleet WHERE props.name == Andy --- internal/field/field.go | 2 +- internal/field/list_binary.go | 52 +++++++++++++++++++++++++++++++---- internal/field/list_test.go | 51 ++++++++++++++++++++++++++++++++++ internal/server/token.go | 29 +++++++++++++------ tests/keys_test.go | 16 +++++++++++ 5 files changed, 136 insertions(+), 14 deletions(-) diff --git a/internal/field/field.go b/internal/field/field.go index 1e0a2789..899c447b 100644 --- a/internal/field/field.go +++ b/internal/field/field.go @@ -34,7 +34,7 @@ func (v Value) IsZero() bool { } func (v Value) Equals(b Value) bool { - return v.kind == b.kind && v.data == b.data + return !v.Less(b) && !b.Less(v) } func (v Value) Kind() Kind { diff --git a/internal/field/list_binary.go b/internal/field/list_binary.go index 2aa5ea83..bebb6f16 100644 --- a/internal/field/list_binary.go +++ b/internal/field/list_binary.go @@ -3,8 +3,11 @@ package field import ( "encoding/binary" "strconv" + "strings" "unsafe" + "github.com/tidwall/gjson" + "github.com/tidwall/pretty" "github.com/tidwall/tile38/internal/sstring" ) @@ -250,6 +253,15 @@ func putfield(b []byte, f Field, s, e int) *byte { // Get a field from the list. Or returns ZeroField if not found. func (fields List) Get(name string) Field { + var isj bool + var jname string + var jpath string + dot := strings.IndexByte(name, '.') + if dot != -1 { + isj = true + jname = name[:dot] + jpath = name[dot+1:] + } b := ptob(fields.p) var i int for { @@ -274,11 +286,23 @@ func (fields List) Get(name string) Field { data = btoa(b[i+n : i+n+x]) i += n + x } - if name < fname { - break - } - if fname == name { - return bfield(fname, kind, data) + if kind == JSON && isj { + if jname < fname { + break + } + if fname == jname { + res := gjson.Get(data, jpath) + if res.Exists() { + return bfield(name, Kind(res.Type), res.String()) + } + } + } else { + if name < fname { + break + } + if fname == name { + return bfield(name, kind, data) + } } } return ZeroField @@ -360,3 +384,21 @@ func MakeList(fields []Field) List { } return list } + +func (fields List) String() string { + var dst []byte + dst = append(dst, '{') + var i int + fields.Scan(func(f Field) bool { + if i > 0 { + dst = append(dst, ',') + } + dst = gjson.AppendJSONString(dst, f.Name()) + dst = append(dst, ':') + dst = append(dst, f.Value().JSON()...) + i++ + return true + }) + dst = append(dst, '}') + return string(pretty.UglyInPlace(dst)) +} diff --git a/internal/field/list_test.go b/internal/field/list_test.go index 749005dc..769fafe8 100644 --- a/internal/field/list_test.go +++ b/internal/field/list_test.go @@ -179,3 +179,54 @@ func TestRandom(t *testing.T) { } } + +func TestJSONGet(t *testing.T) { + + var list List + list = list.Set(Make("hello", "world")) + list = list.Set(Make("hello", `"world"`)) + list = list.Set(Make("jello", "planet")) + list = list.Set(Make("telly", `{"a":[1,2,3],"b":null,"c":true,"d":false}`)) + list = list.Set(Make("belly", `{"a":{"b":{"c":"fancy"}}}`)) + json := list.String() + exp := `{"belly":{"a":{"b":{"c":"fancy"}}},"hello":"world","jello":` + + `"planet","telly":{"a":[1,2,3],"b":null,"c":true,"d":false}}` + if json != exp { + t.Fatalf("expected '%s', got '%s'", exp, json) + } + data := list.Get("hello").Value().Data() + if data != "world" { + t.Fatalf("expected '%s', got '%s'", "world", data) + } + data = list.Get("telly").Value().Data() + if data != `{"a":[1,2,3],"b":null,"c":true,"d":false}` { + t.Fatalf("expected '%s', got '%s'", + `{"a":[1,2,3],"b":null,"c":true,"d":false}`, data) + } + data = list.Get("belly").Value().Data() + if data != `{"a":{"b":{"c":"fancy"}}}` { + t.Fatalf("expected '%s', got '%s'", + `{"a":{"b":{"c":"fancy"}}}`, data) + } + data = list.Get("belly.a").Value().Data() + if data != `{"b":{"c":"fancy"}}` { + t.Fatalf("expected '%s', got '%s'", + `{"b":{"c":"fancy"}}`, data) + } + data = list.Get("belly.a.b").Value().Data() + if data != `{"c":"fancy"}` { + t.Fatalf("expected '%s', got '%s'", + `{"c":"fancy"}`, data) + } + data = list.Get("belly.a.b.c").Value().Data() + if data != `fancy` { + t.Fatalf("expected '%s', got '%s'", + `fancy`, data) + } + // Tile38 defaults non-existent fields to zero. + data = list.Get("belly.a.b.c.d").Value().Data() + if data != `0` { + t.Fatalf("expected '%s', got '%s'", + `0`, data) + } +} diff --git a/internal/server/token.go b/internal/server/token.go index a4200a5e..253c3e8e 100644 --- a/internal/server/token.go +++ b/internal/server/token.go @@ -77,6 +77,20 @@ func mGTE(a, b field.Value) bool { return !mLT(a, b) } func mEQ(a, b field.Value) bool { return a.Equals(b) } func (where whereT) match(value field.Value) bool { + switch where.min.Data() { + case "<": + return mLT(value, where.max) + case "<=": + return mLTE(value, where.max) + case ">": + return mGT(value, where.max) + case ">=": + return mGTE(value, where.max) + case "==": + return mEQ(value, where.max) + case "!=": + return !mEQ(value, where.max) + } if !where.minx { if mLT(value, where.min) { // if value < where.min { return false @@ -276,18 +290,17 @@ func (s *Server) parseSearchScanBaseTokens( } var minx, maxx bool smin = strings.ToLower(smin) - if smin == "-inf" { - smin = "-inf" - } else { + smax = strings.ToLower(smax) + if smax == "+inf" || smax == "inf" { + smax = "inf" + } + switch smin { + case "<", "<=", ">", ">=", "==", "!=": + default: if strings.HasPrefix(smin, "(") { minx = true smin = smin[1:] } - } - smax = strings.ToLower(smax) - if smax == "+inf" || smax == "inf" { - smax = "inf" - } else { if strings.HasPrefix(smax, "(") { maxx = true smax = smax[1:] diff --git a/tests/keys_test.go b/tests/keys_test.go index 44031696..95c86d5c 100644 --- a/tests/keys_test.go +++ b/tests/keys_test.go @@ -459,6 +459,22 @@ func keys_FIELDS_test(mc *mockServer) error { Do("GET", "fleet", "truck1", "WITHFIELDS").JSON().Str(`{"ok":true,"object":{"type":"Point","coordinates":[33,-112]}}`), Do("SET", "fleet", "truck1", "FIELD", "speed", "2", "POINT", "-112", "33").JSON().OK(), Do("GET", "fleet", "truck1", "WITHFIELDS").JSON().Str(`{"ok":true,"object":{"type":"Point","coordinates":[33,-112]},"fields":{"speed":2}}`), + + // Do some GJSON queries. + Do("SET", "fleet", "truck2", "FIELD", "hello", `{"world":"tom"}`, "POINT", "-112", "33").JSON().OK(), + Do("SCAN", "fleet", "WHERE", "hello", `{"world":"tom"}`, `{"world":"tom"}`, "COUNT").JSON().Str(`{"ok":true,"count":1,"cursor":0}`), + Do("SCAN", "fleet", "WHERE", "hello.world", `tom`, `tom`, "COUNT").JSON().Str(`{"ok":true,"count":1,"cursor":0}`), + // The next scan does not match on anything, but since we're matching + // on zeros, which is the default, then all (two) objects are returned. + Do("SCAN", "fleet", "WHERE", "hello.world.1", `0`, `0`, "IDS").JSON().Str(`{"ok":true,"ids":["truck1","truck2"],"count":2,"cursor":0}`), + Do("SCAN", "fleet", "WHERE", "hello.world", ">", `tom`, "IDS").JSON().Str(`{"ok":true,"ids":[],"count":0,"cursor":0}`), + Do("SCAN", "fleet", "WHERE", "hello.world", ">=", `Tom`, "IDS").JSON().Str(`{"ok":true,"ids":["truck2"],"count":1,"cursor":0}`), + Do("SCAN", "fleet", "WHERE", "hello.world", ">=", `tom`, "IDS").JSON().Str(`{"ok":true,"ids":["truck2"],"count":1,"cursor":0}`), + Do("SCAN", "fleet", "WHERE", "hello.world", "==", `tom`, "IDS").JSON().Str(`{"ok":true,"ids":["truck2"],"count":1,"cursor":0}`), + Do("SCAN", "fleet", "WHERE", "hello.world", "<", `tom`, "IDS").JSON().Str(`{"ok":true,"ids":["truck1"],"count":1,"cursor":0}`), + Do("SCAN", "fleet", "WHERE", "hello.world", "<=", `tom`, "IDS").JSON().Str(`{"ok":true,"ids":["truck1","truck2"],"count":2,"cursor":0}`), + Do("SCAN", "fleet", "WHERE", "hello.world", "<", `uom`, "IDS").JSON().Str(`{"ok":true,"ids":["truck1","truck2"],"count":2,"cursor":0}`), + Do("SCAN", "fleet", "WHERE", "hello.world", "!=", `tom`, "IDS").JSON().Str(`{"ok":true,"ids":["truck1"],"count":1,"cursor":0}`), ) }