mirror
/
tile38
mirror of https://github.com/tidwall/tile38.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

scan order asc/desc

This commit is contained in:
Josh Baker 2016-07-12 13:18:16 -06:00
parent 4fd7e1f821
commit 75adea6a9c
13 changed files with 126 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
controller/aofshrink.go
View File

@ -150,7 +150,7 @@ func (c *Controller) aofshrink() {
objs := make(map[string]objFields)
c.mu.Lock()
fnames := col.FieldArr() // reload an array of field names to match each object
col.ScanGreaterOrEqual(nextID, 0, collection.TypeAll,
col.ScanGreaterOrEqual(nextID, 1, collection.TypeAll, false,
func(id string, obj geojson.Object, fields []float64) bool {
if id != nextID {
objs[id] = objFields{obj, fields}

49
controller/collection/collection.go
View File

@ -236,32 +236,69 @@ func (c *Collection) FieldArr() []string {
}
// Scan iterates though the collection. A cursor can be used for paging.
func (c *Collection) Scan(cursor uint64, stype ScanType, iterator func(id string, obj geojson.Object, fields []float64) bool) (ncursor uint64) {
func (c *Collection) Scan(cursor uint64, stype ScanType, desc bool,
iterator func(id string, obj geojson.Object, fields []float64) bool,
) (ncursor uint64) {
var i uint64
var active = true
c.items.Ascend(func(item btree.Item) bool {
iter := func(item btree.Item) bool {
if i >= cursor {
iitm := item.(*itemT)
active = iterator(iitm.id, iitm.object, iitm.fields)
}
i++
return active
})
}
if desc {
c.items.Descend(iter)
} else {
c.items.Ascend(iter)
}
return i
}
// ScanGreaterOrEqual iterates though the collection starting with specified id. A cursor can be used for paging.
func (c *Collection) ScanGreaterOrEqual(id string, cursor uint64, stype ScanType, iterator func(id string, obj geojson.Object, fields []float64) bool) (ncursor uint64) {
func (c *Collection) ScanRange(cursor uint64, stype ScanType, start, end string, desc bool,
iterator func(id string, obj geojson.Object, fields []float64) bool,
) (ncursor uint64) {
var i uint64
var active = true
c.items.AscendGreaterOrEqual(&itemT{id: id}, func(item btree.Item) bool {
iter := func(item btree.Item) bool {
if i >= cursor {
iitm := item.(*itemT)
active = iterator(iitm.id, iitm.object, iitm.fields)
}
i++
return active
})
}
if desc {
c.items.DescendRange(&itemT{id: start}, &itemT{id: end}, iter)
} else {
c.items.AscendRange(&itemT{id: start}, &itemT{id: end}, iter)
}
return i
}
// ScanGreaterOrEqual iterates though the collection starting with specified id. A cursor can be used for paging.
func (c *Collection) ScanGreaterOrEqual(id string, cursor uint64, stype ScanType, desc bool,
iterator func(id string, obj geojson.Object, fields []float64) bool,
) (ncursor uint64) {
var i uint64
var active = true
iter := func(item btree.Item) bool {
if i >= cursor {
iitm := item.(*itemT)
active = iterator(iitm.id, iitm.object, iitm.fields)
}
i++
return active
}
if desc {
c.items.DescendLessOrEqual(&itemT{id: id}, iter)
} else {
c.items.AscendGreaterOrEqual(&itemT{id: id}, iter)
}
return i
}

3
controller/config.go
View File

@ -10,6 +10,7 @@ import (
"time"
"github.com/tidwall/resp"
"github.com/tidwall/tile38/controller/glob"
"github.com/tidwall/tile38/controller/server"
)
@ -147,7 +148,7 @@ func (c *Controller) setConfigProperty(name, value string, fromLoad bool) error
func (c *Controller) getConfigProperties(pattern string) map[string]interface{} {
m := make(map[string]interface{})
for _, name := range validProperties {
matched, _ := globMatch(pattern, name)
matched, _ := glob.Match(pattern, name)
if matched {
m[name] = c.getConfigProperty(name)
}

9
controller/fence.go
View File

@ -4,6 +4,7 @@ import (
"strconv"
"strings"
"github.com/tidwall/tile38/controller/glob"
"github.com/tidwall/tile38/controller/server"
"github.com/tidwall/tile38/geojson"
)
@ -14,13 +15,13 @@ var tmfmt = "2006-01-02T15:04:05.999999999Z07:00"
func FenceMatch(hookName string, sw *scanWriter, fence *liveFenceSwitches, details *commandDetailsT) []string {
jshookName := jsonString(hookName)
jstime := jsonString(details.timestamp.Format(tmfmt))
glob := fence.glob
pattern := fence.glob
if details.command == "drop" {
return []string{`{"cmd":"drop","hook":` + jshookName + `,"time":` + jstime + `}`}
}
match := true
if glob != "" && glob != "*" {
match, _ = globMatch(glob, details.id)
if pattern != "" && pattern != "*" {
match, _ = glob.Match(pattern, details.id)
}
if !match {
return nil
@ -202,7 +203,7 @@ func fenceMatchRoam(c *Controller, fence *liveFenceSwitches, tkey, tid string, o
return true // skip self
}
if fence.roam.pattern {
match, _ = globMatch(fence.roam.id, id)
match, _ = glob.Match(fence.roam.id, id)
} else {
match = fence.roam.id == id
}

18
controller/glob.go
View File

@ -1,18 +0,0 @@
package controller
import "path"
func globMatch(pattern, name string) (matched bool, err error) {
return path.Match(pattern, name)
}
func globIsGlob(pattern string) bool {
for i := 0; i < len(pattern); i++ {
switch pattern[i] {
case '[', '*', '?':
_, err := globMatch(pattern, "whatever")
return err == nil
}
}
return false
}

3
controller/hooks.go
View File

@ -10,6 +10,7 @@ import (
"time"
"github.com/tidwall/resp"
"github.com/tidwall/tile38/controller/glob"
"github.com/tidwall/tile38/controller/log"
"github.com/tidwall/tile38/controller/server"
)
@ -329,7 +330,7 @@ func (c *Controller) cmdHooks(msg *server.Message) (res string, err error) {
var hooks []*Hook
for name, hook := range c.hooks {
match, _ := globMatch(pattern, name)
match, _ := glob.Match(pattern, name)
if match {
hooks = append(hooks, hook)
}

7
controller/keys.go
View File

@ -7,6 +7,7 @@ import (
"github.com/tidwall/btree"
"github.com/tidwall/resp"
"github.com/tidwall/tile38/controller/glob"
"github.com/tidwall/tile38/controller/server"
)
@ -44,7 +45,7 @@ func (c *Controller) cmdKeys(msg *server.Message) (res string, err error) {
}
match = true
} else {
match, _ = globMatch(pattern, key)
match, _ = glob.Match(pattern, key)
}
if match {
if once {
@ -69,14 +70,14 @@ func (c *Controller) cmdKeys(msg *server.Message) (res string, err error) {
} else {
if strings.HasSuffix(pattern, "*") {
greaterPivot = pattern[:len(pattern)-1]
if globIsGlob(greaterPivot) {
if glob.IsGlob(greaterPivot) {
greater = false
c.cols.Ascend(iterator)
} else {
greater = true
c.cols.AscendGreaterOrEqual(&collectionT{Key: greaterPivot}, iterator)
}
} else if globIsGlob(pattern) {
} else if glob.IsGlob(pattern) {
greater = false
c.cols.Ascend(iterator)
} else {

27
controller/scan.go
View File

@ -2,11 +2,11 @@ package controller
import (
"bytes"
"strings"
"time"
"github.com/tidwall/resp"
"github.com/tidwall/tile38/controller/collection"
"github.com/tidwall/tile38/controller/glob"
"github.com/tidwall/tile38/controller/server"
"github.com/tidwall/tile38/geojson"
)
@ -48,23 +48,16 @@ func (c *Controller) cmdScan(msg *server.Message) (res string, err error) {
}
sw.count = uint64(count)
} else {
if strings.HasSuffix(sw.glob, "*") {
greaterGlob := sw.glob[:len(sw.glob)-1]
if globIsGlob(greaterGlob) {
s.cursor = sw.col.Scan(s.cursor, stype,
func(id string, o geojson.Object, fields []float64) bool {
return sw.writeObject(id, o, fields, false)
},
)
} else {
s.cursor = sw.col.ScanGreaterOrEqual(greaterGlob, s.cursor, stype,
func(id string, o geojson.Object, fields []float64) bool {
return sw.writeObject(id, o, fields, false)
},
)
}
g := glob.Parse(sw.glob, s.desc)
if g.Limits[0] == "" && g.Limits[1] == "" {
s.cursor = sw.col.Scan(s.cursor, stype, s.desc,
func(id string, o geojson.Object, fields []float64) bool {
return sw.writeObject(id, o, fields, false)
},
)
} else {
s.cursor = sw.col.Scan(s.cursor, stype,
s.cursor = sw.col.ScanRange(
s.cursor, stype, g.Limits[0], g.Limits[1], s.desc,
func(id string, o geojson.Object, fields []float64) bool {
return sw.writeObject(id, o, fields, false)
},

11
controller/scanner.go
View File

@ -8,6 +8,7 @@ import (
"github.com/tidwall/resp"
"github.com/tidwall/tile38/controller/collection"
"github.com/tidwall/tile38/controller/glob"
"github.com/tidwall/tile38/controller/server"
"github.com/tidwall/tile38/geojson"
)
@ -53,7 +54,7 @@ type scanWriter struct {
}
func (c *Controller) newScanWriter(
wr *bytes.Buffer, msg *server.Message, key string, output outputT, precision uint64, glob string, limit uint64, wheres []whereT, nofields bool,
wr *bytes.Buffer, msg *server.Message, key string, output outputT, precision uint64, globPattern string, limit uint64, wheres []whereT, nofields bool,
) (
*scanWriter, error,
) {
@ -75,13 +76,13 @@ func (c *Controller) newScanWriter(
wheres: wheres,
precision: precision,
nofields: nofields,
glob: glob,
glob: globPattern,
limit: limit,
}
if glob == "*" || glob == "" {
if globPattern == "*" || globPattern == "" {
sw.globEverything = true
} else {
if !globIsGlob(glob) {
if !glob.IsGlob(globPattern) {
sw.globSingle = true
}
}
@ -241,7 +242,7 @@ func (sw *scanWriter) writeObject(id string, o geojson.Object, fields []float64,
}
keepGoing = false // return current object and stop iterating
} else {
ok, _ := globMatch(sw.glob, id)
ok, _ := glob.Match(sw.glob, id)
if !ok {
return true
}

3
controller/search.go
View File

@ -8,6 +8,7 @@ import (
"github.com/tidwall/resp"
"github.com/tidwall/tile38/controller/bing"
"github.com/tidwall/tile38/controller/glob"
"github.com/tidwall/tile38/controller/server"
"github.com/tidwall/tile38/geojson"
"github.com/tidwall/tile38/geojson/geohash"
@ -230,7 +231,7 @@ func (c *Controller) cmdSearchArgs(cmd string, vs []resp.Value, types []string)
err = errInvalidNumberOfArguments
return
}
s.roam.pattern = globIsGlob(s.roam.id)
s.roam.pattern = glob.IsGlob(s.roam.id)
var smeters string
if vs, smeters, ok = tokenval(vs); !ok || smeters == "" {
err = errInvalidNumberOfArguments

29
controller/token.go
View File

@ -130,6 +130,7 @@ type searchScanBaseTokens struct {
nofields bool
limit uint64
sparse uint8
desc bool
}
func parseSearchScanBaseTokens(cmd string, vs []resp.Value) (vsout []resp.Value, t searchScanBaseTokens, err error) {
@ -141,6 +142,7 @@ func parseSearchScanBaseTokens(cmd string, vs []resp.Value) (vsout []resp.Value,
var slimit string
var ssparse string
var scursor string
var asc bool
for {
nvs, wtok, ok := tokenval(vs)
if ok && len(wtok) > 0 {
@ -273,7 +275,22 @@ func parseSearchScanBaseTokens(cmd string, vs []resp.Value) (vsout []resp.Value,
"cross": true,
}
}
continue
} else if (wtok[0] == 'D' || wtok[0] == 'd') && strings.ToLower(wtok) == "desc" {
vs = nvs
if t.desc || asc {
err = errDuplicateArgument(strings.ToUpper(wtok))
return
}
t.desc = true
continue
} else if (wtok[0] == 'A' || wtok[0] == 'a') && strings.ToLower(wtok) == "asc" {
vs = nvs
if t.desc || asc {
err = errDuplicateArgument(strings.ToUpper(wtok))
return
}
asc = true
continue
} else if (wtok[0] == 'M' || wtok[0] == 'm') && strings.ToLower(wtok) == "match" {
vs = nvs
@ -301,6 +318,15 @@ func parseSearchScanBaseTokens(cmd string, vs []resp.Value) (vsout []resp.Value,
err = errors.New("FENCE is not allowed for SCAN")
return
}
} else {
if t.desc {
err = errors.New("DESC is not allowed for " + strings.ToUpper(cmd))
return
}
if asc {
err = errors.New("ASC is not allowed for " + strings.ToUpper(cmd))
return
}
}
if ssparse != "" && slimit != "" {
err = errors.New("LIMIT is not allowed when SPARSE is specified")
@ -353,7 +379,6 @@ func parseSearchScanBaseTokens(cmd string, vs []resp.Value) (vsout []resp.Value,
vs = nvs
}
}
if scursor != "" {
if t.cursor, err = strconv.ParseUint(scursor, 10, 64); err != nil {
err = errInvalidArgument(scursor)

13
core/commands.json
View File

@ -216,7 +216,6 @@
"since": "1.0.0",
"group": "keys"
},
"SCAN": {
"summary": "Incrementally iterate though a key",
"complexity": "O(N) where N is the number of ids in the key",
@ -243,6 +242,18 @@
"type": "pattern",
"optional": true
},
{
"name": "order",
"optional": true,
"enumargs": [
{
"name": "ASC"
},
{
"name": "DESC"
}
]
},
{
"command": "WHERE",
"name": ["field","min","max"],

13
core/commands_gen.go
View File

@ -378,7 +378,6 @@ var commandsJSON = `{
"since": "1.0.0",
"group": "keys"
},
"SCAN": {
"summary": "Incrementally iterate though a key",
"complexity": "O(N) where N is the number of ids in the key",
@ -405,6 +404,18 @@ var commandsJSON = `{
"type": "pattern",
"optional": true
},
{
"name": "order",
"optional": true,
"enumargs": [
{
"name": "ASC"
},
{
"name": "DESC"
}
]
},
{
"command": "WHERE",
"name": ["field","min","max"],