diff --git a/controller/aofshrink.go b/controller/aofshrink.go index da563bf1..692696cd 100644 --- a/controller/aofshrink.go +++ b/controller/aofshrink.go @@ -150,7 +150,7 @@ func (c *Controller) aofshrink() { objs := make(map[string]objFields) c.mu.Lock() fnames := col.FieldArr() // reload an array of field names to match each object - col.ScanGreaterOrEqual(nextID, 0, collection.TypeAll, + col.ScanGreaterOrEqual(nextID, 1, collection.TypeAll, false, func(id string, obj geojson.Object, fields []float64) bool { if id != nextID { objs[id] = objFields{obj, fields} diff --git a/controller/collection/collection.go b/controller/collection/collection.go index 73a4cb7c..152883cb 100644 --- a/controller/collection/collection.go +++ b/controller/collection/collection.go @@ -236,32 +236,69 @@ func (c *Collection) FieldArr() []string { } // Scan iterates though the collection. A cursor can be used for paging. -func (c *Collection) Scan(cursor uint64, stype ScanType, iterator func(id string, obj geojson.Object, fields []float64) bool) (ncursor uint64) { +func (c *Collection) Scan(cursor uint64, stype ScanType, desc bool, + iterator func(id string, obj geojson.Object, fields []float64) bool, +) (ncursor uint64) { var i uint64 var active = true - c.items.Ascend(func(item btree.Item) bool { + iter := func(item btree.Item) bool { if i >= cursor { iitm := item.(*itemT) active = iterator(iitm.id, iitm.object, iitm.fields) } i++ return active - }) + } + if desc { + c.items.Descend(iter) + } else { + c.items.Ascend(iter) + } return i } // ScanGreaterOrEqual iterates though the collection starting with specified id. A cursor can be used for paging. -func (c *Collection) ScanGreaterOrEqual(id string, cursor uint64, stype ScanType, iterator func(id string, obj geojson.Object, fields []float64) bool) (ncursor uint64) { +func (c *Collection) ScanRange(cursor uint64, stype ScanType, start, end string, desc bool, + iterator func(id string, obj geojson.Object, fields []float64) bool, +) (ncursor uint64) { var i uint64 var active = true - c.items.AscendGreaterOrEqual(&itemT{id: id}, func(item btree.Item) bool { + iter := func(item btree.Item) bool { if i >= cursor { iitm := item.(*itemT) active = iterator(iitm.id, iitm.object, iitm.fields) } i++ return active - }) + } + + if desc { + c.items.DescendRange(&itemT{id: start}, &itemT{id: end}, iter) + } else { + c.items.AscendRange(&itemT{id: start}, &itemT{id: end}, iter) + } + return i +} + +// ScanGreaterOrEqual iterates though the collection starting with specified id. A cursor can be used for paging. +func (c *Collection) ScanGreaterOrEqual(id string, cursor uint64, stype ScanType, desc bool, + iterator func(id string, obj geojson.Object, fields []float64) bool, +) (ncursor uint64) { + var i uint64 + var active = true + iter := func(item btree.Item) bool { + if i >= cursor { + iitm := item.(*itemT) + active = iterator(iitm.id, iitm.object, iitm.fields) + } + i++ + return active + } + if desc { + c.items.DescendLessOrEqual(&itemT{id: id}, iter) + } else { + c.items.AscendGreaterOrEqual(&itemT{id: id}, iter) + } return i } diff --git a/controller/config.go b/controller/config.go index cd96e7e9..6afdb812 100644 --- a/controller/config.go +++ b/controller/config.go @@ -10,6 +10,7 @@ import ( "time" "github.com/tidwall/resp" + "github.com/tidwall/tile38/controller/glob" "github.com/tidwall/tile38/controller/server" ) @@ -147,7 +148,7 @@ func (c *Controller) setConfigProperty(name, value string, fromLoad bool) error func (c *Controller) getConfigProperties(pattern string) map[string]interface{} { m := make(map[string]interface{}) for _, name := range validProperties { - matched, _ := globMatch(pattern, name) + matched, _ := glob.Match(pattern, name) if matched { m[name] = c.getConfigProperty(name) } diff --git a/controller/fence.go b/controller/fence.go index 38390b8b..86d2d630 100644 --- a/controller/fence.go +++ b/controller/fence.go @@ -4,6 +4,7 @@ import ( "strconv" "strings" + "github.com/tidwall/tile38/controller/glob" "github.com/tidwall/tile38/controller/server" "github.com/tidwall/tile38/geojson" ) @@ -14,13 +15,13 @@ var tmfmt = "2006-01-02T15:04:05.999999999Z07:00" func FenceMatch(hookName string, sw *scanWriter, fence *liveFenceSwitches, details *commandDetailsT) []string { jshookName := jsonString(hookName) jstime := jsonString(details.timestamp.Format(tmfmt)) - glob := fence.glob + pattern := fence.glob if details.command == "drop" { return []string{`{"cmd":"drop","hook":` + jshookName + `,"time":` + jstime + `}`} } match := true - if glob != "" && glob != "*" { - match, _ = globMatch(glob, details.id) + if pattern != "" && pattern != "*" { + match, _ = glob.Match(pattern, details.id) } if !match { return nil @@ -202,7 +203,7 @@ func fenceMatchRoam(c *Controller, fence *liveFenceSwitches, tkey, tid string, o return true // skip self } if fence.roam.pattern { - match, _ = globMatch(fence.roam.id, id) + match, _ = glob.Match(fence.roam.id, id) } else { match = fence.roam.id == id } diff --git a/controller/glob.go b/controller/glob.go deleted file mode 100644 index cb4c3bba..00000000 --- a/controller/glob.go +++ /dev/null @@ -1,18 +0,0 @@ -package controller - -import "path" - -func globMatch(pattern, name string) (matched bool, err error) { - return path.Match(pattern, name) -} - -func globIsGlob(pattern string) bool { - for i := 0; i < len(pattern); i++ { - switch pattern[i] { - case '[', '*', '?': - _, err := globMatch(pattern, "whatever") - return err == nil - } - } - return false -} diff --git a/controller/hooks.go b/controller/hooks.go index fa9dc660..eb165b17 100644 --- a/controller/hooks.go +++ b/controller/hooks.go @@ -10,6 +10,7 @@ import ( "time" "github.com/tidwall/resp" + "github.com/tidwall/tile38/controller/glob" "github.com/tidwall/tile38/controller/log" "github.com/tidwall/tile38/controller/server" ) @@ -329,7 +330,7 @@ func (c *Controller) cmdHooks(msg *server.Message) (res string, err error) { var hooks []*Hook for name, hook := range c.hooks { - match, _ := globMatch(pattern, name) + match, _ := glob.Match(pattern, name) if match { hooks = append(hooks, hook) } diff --git a/controller/keys.go b/controller/keys.go index 6845f92a..a96eced6 100644 --- a/controller/keys.go +++ b/controller/keys.go @@ -7,6 +7,7 @@ import ( "github.com/tidwall/btree" "github.com/tidwall/resp" + "github.com/tidwall/tile38/controller/glob" "github.com/tidwall/tile38/controller/server" ) @@ -44,7 +45,7 @@ func (c *Controller) cmdKeys(msg *server.Message) (res string, err error) { } match = true } else { - match, _ = globMatch(pattern, key) + match, _ = glob.Match(pattern, key) } if match { if once { @@ -69,14 +70,14 @@ func (c *Controller) cmdKeys(msg *server.Message) (res string, err error) { } else { if strings.HasSuffix(pattern, "*") { greaterPivot = pattern[:len(pattern)-1] - if globIsGlob(greaterPivot) { + if glob.IsGlob(greaterPivot) { greater = false c.cols.Ascend(iterator) } else { greater = true c.cols.AscendGreaterOrEqual(&collectionT{Key: greaterPivot}, iterator) } - } else if globIsGlob(pattern) { + } else if glob.IsGlob(pattern) { greater = false c.cols.Ascend(iterator) } else { diff --git a/controller/scan.go b/controller/scan.go index 061b7a85..74960cca 100644 --- a/controller/scan.go +++ b/controller/scan.go @@ -2,11 +2,11 @@ package controller import ( "bytes" - "strings" "time" "github.com/tidwall/resp" "github.com/tidwall/tile38/controller/collection" + "github.com/tidwall/tile38/controller/glob" "github.com/tidwall/tile38/controller/server" "github.com/tidwall/tile38/geojson" ) @@ -48,23 +48,16 @@ func (c *Controller) cmdScan(msg *server.Message) (res string, err error) { } sw.count = uint64(count) } else { - if strings.HasSuffix(sw.glob, "*") { - greaterGlob := sw.glob[:len(sw.glob)-1] - if globIsGlob(greaterGlob) { - s.cursor = sw.col.Scan(s.cursor, stype, - func(id string, o geojson.Object, fields []float64) bool { - return sw.writeObject(id, o, fields, false) - }, - ) - } else { - s.cursor = sw.col.ScanGreaterOrEqual(greaterGlob, s.cursor, stype, - func(id string, o geojson.Object, fields []float64) bool { - return sw.writeObject(id, o, fields, false) - }, - ) - } + g := glob.Parse(sw.glob, s.desc) + if g.Limits[0] == "" && g.Limits[1] == "" { + s.cursor = sw.col.Scan(s.cursor, stype, s.desc, + func(id string, o geojson.Object, fields []float64) bool { + return sw.writeObject(id, o, fields, false) + }, + ) } else { - s.cursor = sw.col.Scan(s.cursor, stype, + s.cursor = sw.col.ScanRange( + s.cursor, stype, g.Limits[0], g.Limits[1], s.desc, func(id string, o geojson.Object, fields []float64) bool { return sw.writeObject(id, o, fields, false) }, diff --git a/controller/scanner.go b/controller/scanner.go index 222840ff..cff75f72 100644 --- a/controller/scanner.go +++ b/controller/scanner.go @@ -8,6 +8,7 @@ import ( "github.com/tidwall/resp" "github.com/tidwall/tile38/controller/collection" + "github.com/tidwall/tile38/controller/glob" "github.com/tidwall/tile38/controller/server" "github.com/tidwall/tile38/geojson" ) @@ -53,7 +54,7 @@ type scanWriter struct { } func (c *Controller) newScanWriter( - wr *bytes.Buffer, msg *server.Message, key string, output outputT, precision uint64, glob string, limit uint64, wheres []whereT, nofields bool, + wr *bytes.Buffer, msg *server.Message, key string, output outputT, precision uint64, globPattern string, limit uint64, wheres []whereT, nofields bool, ) ( *scanWriter, error, ) { @@ -75,13 +76,13 @@ func (c *Controller) newScanWriter( wheres: wheres, precision: precision, nofields: nofields, - glob: glob, + glob: globPattern, limit: limit, } - if glob == "*" || glob == "" { + if globPattern == "*" || globPattern == "" { sw.globEverything = true } else { - if !globIsGlob(glob) { + if !glob.IsGlob(globPattern) { sw.globSingle = true } } @@ -241,7 +242,7 @@ func (sw *scanWriter) writeObject(id string, o geojson.Object, fields []float64, } keepGoing = false // return current object and stop iterating } else { - ok, _ := globMatch(sw.glob, id) + ok, _ := glob.Match(sw.glob, id) if !ok { return true } diff --git a/controller/search.go b/controller/search.go index fa0a76fe..b218c8f0 100644 --- a/controller/search.go +++ b/controller/search.go @@ -8,6 +8,7 @@ import ( "github.com/tidwall/resp" "github.com/tidwall/tile38/controller/bing" + "github.com/tidwall/tile38/controller/glob" "github.com/tidwall/tile38/controller/server" "github.com/tidwall/tile38/geojson" "github.com/tidwall/tile38/geojson/geohash" @@ -230,7 +231,7 @@ func (c *Controller) cmdSearchArgs(cmd string, vs []resp.Value, types []string) err = errInvalidNumberOfArguments return } - s.roam.pattern = globIsGlob(s.roam.id) + s.roam.pattern = glob.IsGlob(s.roam.id) var smeters string if vs, smeters, ok = tokenval(vs); !ok || smeters == "" { err = errInvalidNumberOfArguments diff --git a/controller/token.go b/controller/token.go index e6e6fb9d..0241fea8 100644 --- a/controller/token.go +++ b/controller/token.go @@ -130,6 +130,7 @@ type searchScanBaseTokens struct { nofields bool limit uint64 sparse uint8 + desc bool } func parseSearchScanBaseTokens(cmd string, vs []resp.Value) (vsout []resp.Value, t searchScanBaseTokens, err error) { @@ -141,6 +142,7 @@ func parseSearchScanBaseTokens(cmd string, vs []resp.Value) (vsout []resp.Value, var slimit string var ssparse string var scursor string + var asc bool for { nvs, wtok, ok := tokenval(vs) if ok && len(wtok) > 0 { @@ -273,7 +275,22 @@ func parseSearchScanBaseTokens(cmd string, vs []resp.Value) (vsout []resp.Value, "cross": true, } } - + continue + } else if (wtok[0] == 'D' || wtok[0] == 'd') && strings.ToLower(wtok) == "desc" { + vs = nvs + if t.desc || asc { + err = errDuplicateArgument(strings.ToUpper(wtok)) + return + } + t.desc = true + continue + } else if (wtok[0] == 'A' || wtok[0] == 'a') && strings.ToLower(wtok) == "asc" { + vs = nvs + if t.desc || asc { + err = errDuplicateArgument(strings.ToUpper(wtok)) + return + } + asc = true continue } else if (wtok[0] == 'M' || wtok[0] == 'm') && strings.ToLower(wtok) == "match" { vs = nvs @@ -301,6 +318,15 @@ func parseSearchScanBaseTokens(cmd string, vs []resp.Value) (vsout []resp.Value, err = errors.New("FENCE is not allowed for SCAN") return } + } else { + if t.desc { + err = errors.New("DESC is not allowed for " + strings.ToUpper(cmd)) + return + } + if asc { + err = errors.New("ASC is not allowed for " + strings.ToUpper(cmd)) + return + } } if ssparse != "" && slimit != "" { err = errors.New("LIMIT is not allowed when SPARSE is specified") @@ -353,7 +379,6 @@ func parseSearchScanBaseTokens(cmd string, vs []resp.Value) (vsout []resp.Value, vs = nvs } } - if scursor != "" { if t.cursor, err = strconv.ParseUint(scursor, 10, 64); err != nil { err = errInvalidArgument(scursor) diff --git a/core/commands.json b/core/commands.json index d41b97a5..e2fccf2b 100644 --- a/core/commands.json +++ b/core/commands.json @@ -216,7 +216,6 @@ "since": "1.0.0", "group": "keys" }, - "SCAN": { "summary": "Incrementally iterate though a key", "complexity": "O(N) where N is the number of ids in the key", @@ -243,6 +242,18 @@ "type": "pattern", "optional": true }, + { + "name": "order", + "optional": true, + "enumargs": [ + { + "name": "ASC" + }, + { + "name": "DESC" + } + ] + }, { "command": "WHERE", "name": ["field","min","max"], diff --git a/core/commands_gen.go b/core/commands_gen.go index 92f04275..e6e7c897 100644 --- a/core/commands_gen.go +++ b/core/commands_gen.go @@ -378,7 +378,6 @@ var commandsJSON = `{ "since": "1.0.0", "group": "keys" }, - "SCAN": { "summary": "Incrementally iterate though a key", "complexity": "O(N) where N is the number of ids in the key", @@ -405,6 +404,18 @@ var commandsJSON = `{ "type": "pattern", "optional": true }, + { + "name": "order", + "optional": true, + "enumargs": [ + { + "name": "ASC" + }, + { + "name": "DESC" + } + ] + }, { "command": "WHERE", "name": ["field","min","max"],