diff --git a/controller/collection/collection.go b/controller/collection/collection.go index 152883cb..257d2a6c 100644 --- a/controller/collection/collection.go +++ b/controller/collection/collection.go @@ -235,7 +235,7 @@ func (c *Collection) FieldArr() []string { return arr } -// Scan iterates though the collection. A cursor can be used for paging. +// Scan iterates though the collection ids. A cursor can be used for paging. func (c *Collection) Scan(cursor uint64, stype ScanType, desc bool, iterator func(id string, obj geojson.Object, fields []float64) bool, ) (ncursor uint64) { @@ -280,6 +280,50 @@ func (c *Collection) ScanRange(cursor uint64, stype ScanType, start, end string, return i } +// SearchValues iterates though the collection values. A cursor can be used for paging. +func (c *Collection) SearchValues(cursor uint64, stype ScanType, desc bool, + iterator func(id string, obj geojson.Object, fields []float64) bool, +) (ncursor uint64) { + var i uint64 + var active = true + iter := func(item btree.Item) bool { + if i >= cursor { + iitm := item.(*itemT) + active = iterator(iitm.id, iitm.object, iitm.fields) + } + i++ + return active + } + if desc { + c.values.Descend(iter) + } else { + c.values.Ascend(iter) + } + return i +} + +// SearchValuesRange iterates though the collection values. A cursor can be used for paging. +func (c *Collection) SearchValuesRange(cursor uint64, stype ScanType, start, end string, desc bool, + iterator func(id string, obj geojson.Object, fields []float64) bool, +) (ncursor uint64) { + var i uint64 + var active = true + iter := func(item btree.Item) bool { + if i >= cursor { + iitm := item.(*itemT) + active = iterator(iitm.id, iitm.object, iitm.fields) + } + i++ + return active + } + if desc { + c.values.DescendRange(&itemT{object: geojson.String(start)}, &itemT{object: geojson.String(end)}, iter) + } else { + c.values.AscendRange(&itemT{object: geojson.String(start)}, &itemT{object: geojson.String(end)}, iter) + } + return i +} + // ScanGreaterOrEqual iterates though the collection starting with specified id. A cursor can be used for paging. func (c *Collection) ScanGreaterOrEqual(id string, cursor uint64, stype ScanType, desc bool, iterator func(id string, obj geojson.Object, fields []float64) bool, @@ -448,5 +492,3 @@ func (c *Collection) Intersects(cursor uint64, sparse uint8, obj geojson.Object, return true }) } -func (c *Collection) SearchValues(pivot string, desc bool, iterator func(id string, obj geojson.Object, fields []float64) bool) { -} diff --git a/controller/controller.go b/controller/controller.go index 9c92c680..fe26142c 100644 --- a/controller/controller.go +++ b/controller/controller.go @@ -329,7 +329,7 @@ func (c *Controller) handleInputCommand(conn *server.Conn, msg *server.Message, if c.config.ReadOnly { return writeErr(errors.New("read only")) } - case "get", "keys", "scan", "nearby", "within", "intersects", "hooks": //, "search": + case "get", "keys", "scan", "nearby", "within", "intersects", "hooks", "search": // read operations c.mu.RLock() defer c.mu.RUnlock() diff --git a/controller/hooks.go b/controller/hooks.go index eb165b17..a9dd2492 100644 --- a/controller/hooks.go +++ b/controller/hooks.go @@ -231,7 +231,7 @@ func (c *Controller) cmdSetHook(msg *server.Message) (res string, d commandDetai Message: cmsg, } var wr bytes.Buffer - hook.ScanWriter, err = c.newScanWriter(&wr, cmsg, s.key, s.output, s.precision, s.glob, s.limit, s.wheres, s.nofields) + hook.ScanWriter, err = c.newScanWriter(&wr, cmsg, s.key, s.output, s.precision, s.glob, false, s.limit, s.wheres, s.nofields) if err != nil { return "", d, err } diff --git a/controller/live.go b/controller/live.go index e96bb861..102c9ac5 100644 --- a/controller/live.go +++ b/controller/live.go @@ -87,7 +87,7 @@ func (c *Controller) goLive(inerr error, conn net.Conn, rd *server.AnyReaderWrit lb.key = s.key lb.fence = &s c.mu.RLock() - sw, err = c.newScanWriter(&wr, msg, s.key, s.output, s.precision, s.glob, s.limit, s.wheres, s.nofields) + sw, err = c.newScanWriter(&wr, msg, s.key, s.output, s.precision, s.glob, false, s.limit, s.wheres, s.nofields) c.mu.RUnlock() } // everything below if for live SCAN, NEARBY, WITHIN, INTERSECTS diff --git a/controller/scan.go b/controller/scan.go index 74960cca..1a1a0599 100644 --- a/controller/scan.go +++ b/controller/scan.go @@ -31,7 +31,7 @@ func (c *Controller) cmdScan(msg *server.Message) (res string, err error) { if err != nil { return "", err } - sw, err := c.newScanWriter(wr, msg, s.key, s.output, s.precision, s.glob, s.limit, s.wheres, s.nofields) + sw, err := c.newScanWriter(wr, msg, s.key, s.output, s.precision, s.glob, false, s.limit, s.wheres, s.nofields) if err != nil { return "", err } diff --git a/controller/scanner.go b/controller/scanner.go index cff75f72..8dfc4000 100644 --- a/controller/scanner.go +++ b/controller/scanner.go @@ -51,10 +51,13 @@ type scanWriter struct { globSingle bool fullFields bool values []resp.Value + matchValues bool } func (c *Controller) newScanWriter( - wr *bytes.Buffer, msg *server.Message, key string, output outputT, precision uint64, globPattern string, limit uint64, wheres []whereT, nofields bool, + wr *bytes.Buffer, msg *server.Message, key string, output outputT, + precision uint64, globPattern string, matchValues bool, + limit uint64, wheres []whereT, nofields bool, ) ( *scanWriter, error, ) { @@ -69,15 +72,16 @@ func (c *Controller) newScanWriter( case outputIDs, outputObjects, outputCount, outputBounds, outputPoints, outputHashes: } sw := &scanWriter{ - c: c, - wr: wr, - msg: msg, - output: output, - wheres: wheres, - precision: precision, - nofields: nofields, - glob: globPattern, - limit: limit, + c: c, + wr: wr, + msg: msg, + output: output, + wheres: wheres, + precision: precision, + nofields: nofields, + glob: globPattern, + limit: limit, + matchValues: matchValues, } if globPattern == "*" || globPattern == "" { sw.globEverything = true @@ -242,7 +246,13 @@ func (sw *scanWriter) writeObject(id string, o geojson.Object, fields []float64, } keepGoing = false // return current object and stop iterating } else { - ok, _ := glob.Match(sw.glob, id) + var val string + if sw.matchValues { + val = o.String() + } else { + val = id + } + ok, _ := glob.Match(sw.glob, val) if !ok { return true } diff --git a/controller/search.go b/controller/search.go index b218c8f0..780f0687 100644 --- a/controller/search.go +++ b/controller/search.go @@ -8,6 +8,7 @@ import ( "github.com/tidwall/resp" "github.com/tidwall/tile38/controller/bing" + "github.com/tidwall/tile38/controller/collection" "github.com/tidwall/tile38/controller/glob" "github.com/tidwall/tile38/controller/server" "github.com/tidwall/tile38/geojson" @@ -264,7 +265,7 @@ func (c *Controller) cmdNearby(msg *server.Message) (res string, err error) { if s.fence { return "", s } - sw, err := c.newScanWriter(wr, msg, s.key, s.output, s.precision, s.glob, s.limit, s.wheres, s.nofields) + sw, err := c.newScanWriter(wr, msg, s.key, s.output, s.precision, s.glob, false, s.limit, s.wheres, s.nofields) if err != nil { return "", err } @@ -305,7 +306,7 @@ func (c *Controller) cmdWithinOrIntersects(cmd string, msg *server.Message) (res if s.fence { return "", s } - sw, err := c.newScanWriter(wr, msg, s.key, s.output, s.precision, s.glob, s.limit, s.wheres, s.nofields) + sw, err := c.newScanWriter(wr, msg, s.key, s.output, s.precision, s.glob, false, s.limit, s.wheres, s.nofields) if err != nil { return "", err } @@ -336,95 +337,63 @@ func (c *Controller) cmdWithinOrIntersects(cmd string, msg *server.Message) (res return string(wr.Bytes()), nil } -func (c *Controller) cmdSearch(msg *server.Message) (res string, err error) { - start := time.Now() - vs := msg.Values[1:] - var ok bool - var key string - if vs, key, ok = tokenval(vs); !ok || key == "" { +func cmdSeachValuesArgs(vs []resp.Value) (s liveFenceSwitches, err error) { + if vs, s.searchScanBaseTokens, err = parseSearchScanBaseTokens("search", vs); err != nil { + return + } + if len(vs) != 0 { err = errInvalidNumberOfArguments return } - col := c.getCol(key) - if col == nil { - err = errKeyNotFound - return - } - var tok string - var pivot string - var pivoton bool - var limiton bool - var limit int - var descon bool - var desc bool - for { - if vs, tok, ok = tokenval(vs); !ok || tok == "" { - break - } - switch strings.ToLower(tok) { - default: - err = errInvalidArgument(tok) - return - case "pivot": - if pivoton { - err = errInvalidArgument(tok) - return - } - pivoton = true - if vs, pivot, ok = tokenval(vs); !ok || pivot == "" { - err = errInvalidNumberOfArguments - return - } - case "limit": - if limiton { - err = errInvalidArgument(tok) - return - } - limiton = true - if vs, tok, ok = tokenval(vs); !ok || tok == "" { - err = errInvalidNumberOfArguments - return - } - n, err2 := strconv.ParseUint(tok, 10, 64) - if err2 != nil { - err = errInvalidArgument(tok) - return - } - limit = int(n) - case "asc", "desc": - if descon { - err = errInvalidArgument(tok) - return - } - descon = true - switch strings.ToLower(tok) { - case "asc": - desc = false - case "desc": - desc = true - } - } - } - println(pivoton, pivot) - println(limiton, limit) - println(descon, desc) + return +} + +func (c *Controller) cmdSearch(msg *server.Message) (res string, err error) { + start := time.Now() + vs := msg.Values[1:] + wr := &bytes.Buffer{} - if msg.OutputType == server.JSON { - wr.WriteString(`{"ok":true,"objects":[`) + s, err := cmdSeachValuesArgs(vs) + if err != nil { + return "", err + } + sw, err := c.newScanWriter(wr, msg, s.key, s.output, s.precision, s.glob, true, s.limit, s.wheres, s.nofields) + if err != nil { + return "", err } - n := 0 - col.SearchValues(pivot, desc, func(id string, obj geojson.Object, fields []float64) bool { - if msg.OutputType == server.JSON { - if n > 0 { - wr.WriteString(`,`) - } - wr.WriteString(`{"id":` + jsonString(id) + `,"object":` + obj.JSON() + `}`) - n++ - } - return true - }) if msg.OutputType == server.JSON { - wr.WriteString(`],"elapsed":"` + time.Now().Sub(start).String() + "\"}") + wr.WriteString(`{"ok":true`) + } + sw.writeHead() + if sw.col != nil { + stype := collection.TypeAll + if sw.output == outputCount && len(sw.wheres) == 0 && sw.globEverything == true { + count := sw.col.Count(stype) - int(s.cursor) + if count < 0 { + count = 0 + } + sw.count = uint64(count) + } else { + g := glob.Parse(sw.glob, s.desc) + if g.Limits[0] == "" && g.Limits[1] == "" { + s.cursor = sw.col.SearchValues(s.cursor, stype, s.desc, + func(id string, o geojson.Object, fields []float64) bool { + return sw.writeObject(id, o, fields, false) + }, + ) + } else { + s.cursor = sw.col.SearchValuesRange( + s.cursor, stype, g.Limits[0], g.Limits[1], s.desc, + func(id string, o geojson.Object, fields []float64) bool { + return sw.writeObject(id, o, fields, false) + }, + ) + } + } + } + sw.writeFoot(s.cursor) + if msg.OutputType == server.JSON { + wr.WriteString(`,"elapsed":"` + time.Now().Sub(start).String() + "\"}") } return string(wr.Bytes()), nil } diff --git a/controller/token.go b/controller/token.go index 0241fea8..da7deda1 100644 --- a/controller/token.go +++ b/controller/token.go @@ -309,13 +309,13 @@ func parseSearchScanBaseTokens(cmd string, vs []resp.Value) (vsout []resp.Value, } // check to make sure that there aren't any conflicts - if cmd == "scan" { + if cmd == "scan" || cmd == "search" { if ssparse != "" { - err = errors.New("SPARSE is not allowed for SCAN") + err = errors.New("SPARSE is not allowed for " + strings.ToUpper(cmd)) return } if t.fence { - err = errors.New("FENCE is not allowed for SCAN") + err = errors.New("FENCE is not allowed for " + strings.ToUpper(cmd)) return } } else { diff --git a/core/commands.json b/core/commands.json index e2fccf2b..75791672 100644 --- a/core/commands.json +++ b/core/commands.json @@ -216,6 +216,91 @@ "since": "1.0.0", "group": "keys" }, + "SEARCH": { + "summary": "Search for string values in a key", + "complexity": "O(N) where N is the number of values in the key", + "arguments":[ + { + "name": "key", + "type": "string" + }, + { + "command": "CURSOR", + "name": "start", + "type": "integer", + "optional": true + }, + { + "command": "LIMIT", + "name": "count", + "type": "integer", + "optional": true + }, + { + "command": "MATCH", + "name": "pattern", + "type": "pattern", + "optional": true + }, + { + "name": "order", + "optional": true, + "enumargs": [ + { + "name": "ASC" + }, + { + "name": "DESC" + } + ] + }, + { + "command": "WHERE", + "name": ["field","min","max"], + "type": ["string","double","double"], + "optional": true, + "multiple": true + }, + { + "command": "NOFIELDS", + "name": [], + "type": [], + "optional": true + }, + { + "name": "type", + "optional": true, + "enumargs": [ + { + "name": "COUNT" + }, + { + "name": "IDS" + }, + { + "name": "OBJECTS" + }, + { + "name": "POINTS" + }, + { + "name": "BOUNDS" + }, + { + "name": "HASHES", + "arguments": [ + { + "name": "precision", + "type": "integer" + } + ] + } + ] + } + ], + "since": "1.0.0", + "group": "search" + }, "SCAN": { "summary": "Incrementally iterate though a key", "complexity": "O(N) where N is the number of ids in the key", diff --git a/core/commands_gen.go b/core/commands_gen.go index e6e7c897..3a68e3ba 100644 --- a/core/commands_gen.go +++ b/core/commands_gen.go @@ -378,6 +378,91 @@ var commandsJSON = `{ "since": "1.0.0", "group": "keys" }, + "SEARCH": { + "summary": "Search for string values in a key", + "complexity": "O(N) where N is the number of values in the key", + "arguments":[ + { + "name": "key", + "type": "string" + }, + { + "command": "CURSOR", + "name": "start", + "type": "integer", + "optional": true + }, + { + "command": "LIMIT", + "name": "count", + "type": "integer", + "optional": true + }, + { + "command": "MATCH", + "name": "pattern", + "type": "pattern", + "optional": true + }, + { + "name": "order", + "optional": true, + "enumargs": [ + { + "name": "ASC" + }, + { + "name": "DESC" + } + ] + }, + { + "command": "WHERE", + "name": ["field","min","max"], + "type": ["string","double","double"], + "optional": true, + "multiple": true + }, + { + "command": "NOFIELDS", + "name": [], + "type": [], + "optional": true + }, + { + "name": "type", + "optional": true, + "enumargs": [ + { + "name": "COUNT" + }, + { + "name": "IDS" + }, + { + "name": "OBJECTS" + }, + { + "name": "POINTS" + }, + { + "name": "BOUNDS" + }, + { + "name": "HASHES", + "arguments": [ + { + "name": "precision", + "type": "integer" + } + ] + } + ] + } + ], + "since": "1.0.0", + "group": "search" + }, "SCAN": { "summary": "Incrementally iterate though a key", "complexity": "O(N) where N is the number of ids in the key",