mirror
/
tile38
mirror of https://github.com/tidwall/tile38.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Added Cursor interface

This commit is contained in:
tidwall 2018-11-02 06:09:56 -07:00
parent 0ea2ec216c
commit 07bae979a5
8 changed files with 151 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

124
internal/collection/collection.go
View File

@ -3,11 +3,17 @@ package collection
import (
"github.com/tidwall/boxtree/d2"
"github.com/tidwall/btree"
"github.com/tidwall/tile38/internal/ds"
"github.com/tidwall/geojson"
"github.com/tidwall/geojson/geometry"
"github.com/tidwall/tile38/internal/ds"
)
// Cursor allows for quickly paging through Scan, Within, Intersects, and Nearby
type Cursor interface {
Offset() uint64
Step(count uint64)
}
type itemT struct {
id string
obj geojson.Object
@ -308,20 +314,24 @@ func (c *Collection) FieldArr() []string {
}
// Scan iterates though the collection ids.
func (c *Collection) Scan(desc bool,
offset uint64,
inc func(n uint64),
func (c *Collection) Scan(desc bool, cursor Cursor,
iterator func(id string, obj geojson.Object, fields []float64) bool,
) bool {
var keepon = true
var count uint64
inc(offset)
var offset uint64
if cursor != nil {
offset = cursor.Offset()
cursor.Step(offset)
}
iter := func(key string, value interface{}) bool {
count++
if count <= offset {
return true
}
inc(1)
if cursor != nil {
cursor.Step(1)
}
iitm := value.(*itemT)
keepon = iterator(iitm.id, iitm.obj, c.getFieldValues(iitm.id))
return keepon
@ -335,20 +345,24 @@ func (c *Collection) Scan(desc bool,
}
// ScanRange iterates though the collection starting with specified id.
func (c *Collection) ScanRange(start, end string, desc bool,
offset uint64,
inc func(n uint64),
func (c *Collection) ScanRange(start, end string, desc bool, cursor Cursor,
iterator func(id string, obj geojson.Object, fields []float64) bool,
) bool {
var keepon = true
var count uint64
inc(offset)
var offset uint64
if cursor != nil {
offset = cursor.Offset()
cursor.Step(offset)
}
iter := func(key string, value interface{}) bool {
count++
if count <= offset {
return true
}
inc(1)
if cursor != nil {
cursor.Step(1)
}
if !desc {
if key >= end {
return false
@ -372,20 +386,24 @@ func (c *Collection) ScanRange(start, end string, desc bool,
}
// SearchValues iterates though the collection values.
func (c *Collection) SearchValues(desc bool,
offset uint64,
inc func(n uint64),
func (c *Collection) SearchValues(desc bool, cursor Cursor,
iterator func(id string, obj geojson.Object, fields []float64) bool,
) bool {
var keepon = true
var count uint64
inc(offset)
var offset uint64
if cursor != nil {
offset = cursor.Offset()
cursor.Step(offset)
}
iter := func(item btree.Item) bool {
count++
if count <= offset {
return true
}
inc(1)
if cursor != nil {
cursor.Step(1)
}
iitm := item.(*itemT)
keepon = iterator(iitm.id, iitm.obj, c.getFieldValues(iitm.id))
return keepon
@ -400,19 +418,24 @@ func (c *Collection) SearchValues(desc bool,
// SearchValuesRange iterates though the collection values.
func (c *Collection) SearchValuesRange(start, end string, desc bool,
offset uint64,
inc func(n uint64),
cursor Cursor,
iterator func(id string, obj geojson.Object, fields []float64) bool,
) bool {
var keepon = true
var count uint64
inc(offset)
var offset uint64
if cursor != nil {
offset = cursor.Offset()
cursor.Step(offset)
}
iter := func(item btree.Item) bool {
count++
if count <= offset {
return true
}
inc(1)
if cursor != nil {
cursor.Step(1)
}
iitm := item.(*itemT)
keepon = iterator(iitm.id, iitm.obj, c.getFieldValues(iitm.id))
return keepon
@ -429,10 +452,24 @@ func (c *Collection) SearchValuesRange(start, end string, desc bool,
// ScanGreaterOrEqual iterates though the collection starting with specified id.
func (c *Collection) ScanGreaterOrEqual(id string, desc bool,
cursor Cursor,
iterator func(id string, obj geojson.Object, fields []float64) bool,
) bool {
var keepon = true
var count uint64
var offset uint64
if cursor != nil {
offset = cursor.Offset()
cursor.Step(offset)
}
iter := func(key string, value interface{}) bool {
count++
if count <= offset {
return true
}
if cursor != nil {
cursor.Step(1)
}
iitm := value.(*itemT)
keepon = iterator(iitm.id, iitm.obj, c.getFieldValues(iitm.id))
return keepon
@ -534,13 +571,16 @@ func (c *Collection) geoSparseInner(
// bounding box. Set obj to nil in order to use the bounding box.
func (c *Collection) Within(
obj geojson.Object,
offset uint64,
sparse uint8,
inc func(n uint64),
cursor Cursor,
iter func(id string, obj geojson.Object, fields []float64) bool,
) bool {
var count uint64
inc(offset)
var offset uint64
if cursor != nil {
offset = cursor.Offset()
cursor.Step(offset)
}
if sparse > 0 {
return c.geoSparse(obj, sparse,
func(id string, o geojson.Object, fields []float64) (
@ -550,7 +590,9 @@ func (c *Collection) Within(
if count <= offset {
return false, true
}
inc(1)
if cursor != nil {
cursor.Step(1)
}
if match = o.Within(obj); match {
ok = iter(id, o, fields)
}
@ -564,7 +606,9 @@ func (c *Collection) Within(
if count <= offset {
return true
}
inc(1)
if cursor != nil {
cursor.Step(1)
}
if o.Within(obj) {
return iter(id, o, fields)
}
@ -577,13 +621,16 @@ func (c *Collection) Within(
// Set obj to nil in order to use the bounding box.
func (c *Collection) Intersects(
obj geojson.Object,
offset uint64,
sparse uint8,
inc func(n uint64),
cursor Cursor,
iter func(id string, obj geojson.Object, fields []float64) bool,
) bool {
var count uint64
inc(offset)
var offset uint64
if cursor != nil {
offset = cursor.Offset()
cursor.Step(offset)
}
if sparse > 0 {
return c.geoSparse(obj, sparse,
func(id string, o geojson.Object, fields []float64) (
@ -593,7 +640,9 @@ func (c *Collection) Intersects(
if count <= offset {
return false, true
}
inc(1)
if cursor != nil {
cursor.Step(1)
}
if match = o.Intersects(obj); match {
ok = iter(id, o, fields)
}
@ -607,7 +656,9 @@ func (c *Collection) Intersects(
if count <= offset {
return true
}
inc(1)
if cursor != nil {
cursor.Step(1)
}
if o.Intersects(obj) {
return iter(id, o, fields)
}
@ -619,14 +670,17 @@ func (c *Collection) Intersects(
// Nearby returns the nearest neighbors
func (c *Collection) Nearby(
target geojson.Object,
offset uint64,
inc func(n uint64),
cursor Cursor,
iter func(id string, obj geojson.Object, fields []float64) bool,
) bool {
alive := true
center := target.Center()
var count uint64
inc(offset)
var offset uint64
if cursor != nil {
offset = cursor.Offset()
cursor.Step(offset)
}
c.index.Nearby(
[]float64{center.X, center.Y},
[]float64{center.X, center.Y},
@ -635,7 +689,9 @@ func (c *Collection) Nearby(
if count <= offset {
return true
}
inc(1)
if cursor != nil {
cursor.Step(1)
}
item := itemv.(*itemT)
alive = iter(item.id, item.obj, c.getFieldValues(item.id))
return alive

48
internal/collection/collection_test.go
View File

@ -230,7 +230,7 @@ func TestCollectionScan(t *testing.T) {
}
var n int
var prevID string
c.Scan(false, 0, func(n uint64) {}, func(id string, obj geojson.Object, fields []float64) bool {
c.Scan(false, nil, func(id string, obj geojson.Object, fields []float64) bool {
if n > 0 {
expect(t, id > prevID)
}
@ -241,7 +241,7 @@ func TestCollectionScan(t *testing.T) {
})
expect(t, n == c.Count())
n = 0
c.Scan(true, 0, func(n uint64) {}, func(id string, obj geojson.Object, fields []float64) bool {
c.Scan(true, nil, func(id string, obj geojson.Object, fields []float64) bool {
if n > 0 {
expect(t, id < prevID)
}
@ -253,7 +253,7 @@ func TestCollectionScan(t *testing.T) {
expect(t, n == c.Count())
n = 0
c.ScanRange("0060", "0070", false, 0, func(n uint64) {},
c.ScanRange("0060", "0070", false, nil,
func(id string, obj geojson.Object, fields []float64) bool {
if n > 0 {
expect(t, id > prevID)
@ -266,7 +266,7 @@ func TestCollectionScan(t *testing.T) {
expect(t, n == 10)
n = 0
c.ScanRange("0070", "0060", true, 0, func(n uint64) {},
c.ScanRange("0070", "0060", true, nil,
func(id string, obj geojson.Object, fields []float64) bool {
if n > 0 {
expect(t, id < prevID)
@ -279,7 +279,7 @@ func TestCollectionScan(t *testing.T) {
expect(t, n == 10)
n = 0
c.ScanGreaterOrEqual("0070", true,
c.ScanGreaterOrEqual("0070", true, nil,
func(id string, obj geojson.Object, fields []float64) bool {
if n > 0 {
expect(t, id < prevID)
@ -292,7 +292,7 @@ func TestCollectionScan(t *testing.T) {
expect(t, n == 71)
n = 0
c.ScanGreaterOrEqual("0070", false,
c.ScanGreaterOrEqual("0070", false, nil,
func(id string, obj geojson.Object, fields []float64) bool {
if n > 0 {
expect(t, id > prevID)
@ -317,7 +317,7 @@ func TestCollectionSearch(t *testing.T) {
}
var n int
var prevValue string
c.SearchValues(false, 0, func(n uint64) {}, func(id string, obj geojson.Object, fields []float64) bool {
c.SearchValues(false, nil, func(id string, obj geojson.Object, fields []float64) bool {
if n > 0 {
expect(t, obj.String() > prevValue)
}
@ -328,7 +328,7 @@ func TestCollectionSearch(t *testing.T) {
})
expect(t, n == c.Count())
n = 0
c.SearchValues(true, 0, func(n uint64) {}, func(id string, obj geojson.Object, fields []float64) bool {
c.SearchValues(true, nil, func(id string, obj geojson.Object, fields []float64) bool {
if n > 0 {
expect(t, obj.String() < prevValue)
}
@ -340,7 +340,7 @@ func TestCollectionSearch(t *testing.T) {
expect(t, n == c.Count())
n = 0
c.SearchValuesRange("0060", "0070", false, 0, func(n uint64) {},
c.SearchValuesRange("0060", "0070", false, nil,
func(id string, obj geojson.Object, fields []float64) bool {
if n > 0 {
expect(t, obj.String() > prevValue)
@ -353,7 +353,7 @@ func TestCollectionSearch(t *testing.T) {
expect(t, n == 10)
n = 0
c.SearchValuesRange("0070", "0060", true, 0, func(n uint64) {},
c.SearchValuesRange("0070", "0060", true, nil,
func(id string, obj geojson.Object, fields []float64) bool {
if n > 0 {
expect(t, obj.String() < prevValue)
@ -436,7 +436,7 @@ func TestSpatialSearch(t *testing.T) {
var n int
n = 0
c.Within(q1, 0, 0, func(n uint64) {},
c.Within(q1, 0, nil,
func(id string, obj geojson.Object, fields []float64) bool {
n++
return true
@ -445,7 +445,7 @@ func TestSpatialSearch(t *testing.T) {
expect(t, n == 3)
n = 0
c.Within(q2, 0, 0, func(n uint64) {},
c.Within(q2, 0, nil,
func(id string, obj geojson.Object, fields []float64) bool {
n++
return true
@ -454,7 +454,7 @@ func TestSpatialSearch(t *testing.T) {
expect(t, n == 7)
n = 0
c.Within(q3, 0, 0, func(n uint64) {},
c.Within(q3, 0, nil,
func(id string, obj geojson.Object, fields []float64) bool {
n++
return true
@ -463,7 +463,7 @@ func TestSpatialSearch(t *testing.T) {
expect(t, n == 4)
n = 0
c.Intersects(q1, 0, 0, func(n uint64) {},
c.Intersects(q1, 0, nil,
func(_ string, _ geojson.Object, _ []float64) bool {
n++
return true
@ -472,7 +472,7 @@ func TestSpatialSearch(t *testing.T) {
expect(t, n == 4)
n = 0
c.Intersects(q2, 0, 0, func(n uint64) {},
c.Intersects(q2, 0, nil,
func(_ string, _ geojson.Object, _ []float64) bool {
n++
return true
@ -481,7 +481,7 @@ func TestSpatialSearch(t *testing.T) {
expect(t, n == 7)
n = 0
c.Intersects(q3, 0, 0, func(n uint64) {},
c.Intersects(q3, 0, nil,
func(_ string, _ geojson.Object, _ []float64) bool {
n++
return true
@ -490,7 +490,7 @@ func TestSpatialSearch(t *testing.T) {
expect(t, n == 5)
n = 0
c.Intersects(q3, 0, 0, func(n uint64) {},
c.Intersects(q3, 0, nil,
func(_ string, _ geojson.Object, _ []float64) bool {
n++
return n <= 1
@ -502,7 +502,7 @@ func TestSpatialSearch(t *testing.T) {
exitems := []geojson.Object{
r2, p1, p4, r1, p3, r3, p2,
}
c.Nearby(q4, 0, func(n uint64) {},
c.Nearby(q4, nil,
func(id string, obj geojson.Object, fields []float64) bool {
items = append(items, obj)
return true
@ -528,7 +528,7 @@ func TestCollectionSparse(t *testing.T) {
}
var n int
n = 0
c.Within(rect, 0,1, func(n uint64) {},
c.Within(rect, 1, nil,
func(id string, obj geojson.Object, fields []float64) bool {
n++
return true
@ -537,7 +537,7 @@ func TestCollectionSparse(t *testing.T) {
expect(t, n == 4)
n = 0
c.Within(rect, 0, 2, func(n uint64) {},
c.Within(rect, 2, nil,
func(id string, obj geojson.Object, fields []float64) bool {
n++
return true
@ -546,7 +546,7 @@ func TestCollectionSparse(t *testing.T) {
expect(t, n == 16)
n = 0
c.Within(rect, 0, 3, func(n uint64) {},
c.Within(rect, 3, nil,
func(id string, obj geojson.Object, fields []float64) bool {
n++
return true
@ -555,7 +555,7 @@ func TestCollectionSparse(t *testing.T) {
expect(t, n == 64)
n = 0
c.Within(rect, 0, 3, func(n uint64) {},
c.Within(rect, 3, nil,
func(id string, obj geojson.Object, fields []float64) bool {
n++
return n <= 30
@ -564,7 +564,7 @@ func TestCollectionSparse(t *testing.T) {
expect(t, n == 31)
n = 0
c.Intersects(rect, 0, 3, func(n uint64) {},
c.Intersects(rect, 3, nil,
func(id string, _ geojson.Object, _ []float64) bool {
n++
return true
@ -573,7 +573,7 @@ func TestCollectionSparse(t *testing.T) {
expect(t, n == 64)
n = 0
c.Intersects(rect, 0, 3, func(n uint64) {},
c.Intersects(rect, 3, nil,
func(id string, _ geojson.Object, _ []float64) bool {
n++
return n <= 30

2
internal/server/aofshrink.go
View File

@ -96,7 +96,7 @@ func (server *Server) aofshrink() {
var exm = server.expires[keys[0]] // the expiration map
var now = time.Now() // used for expiration
var count = 0 // the object count
col.ScanGreaterOrEqual(nextid, false,
col.ScanGreaterOrEqual(nextid, false, nil,
func(id string, obj geojson.Object, fields []float64) bool {
if count == maxids {
// we reached the max number of ids for one batch

4
internal/server/crud.go
View File

@ -372,9 +372,9 @@ func (server *Server) cmdPdel(msg *Message) (res resp.Value, d commandDetailsT,
if col != nil {
g := glob.Parse(d.pattern, false)
if g.Limits[0] == "" && g.Limits[1] == "" {
col.Scan(false, 0, func(n uint64) {}, iter)
col.Scan(false, nil, iter)
} else {
col.ScanRange(g.Limits[0], g.Limits[1], false, 0, func(n uint64) {}, iter)
col.ScanRange(g.Limits[0], g.Limits[1], false, nil, iter)
}
var atLeastOneNotDeleted bool
for i, dc := range d.children {

6
internal/server/fence.go
View File

@ -299,10 +299,10 @@ func extendRoamMessage(
}
g := glob.Parse(pattern, false)
if g.Limits[0] == "" && g.Limits[1] == "" {
col.Scan(false, 0, func(n uint64) {}, iterator)
col.Scan(false, nil, iterator)
} else {
col.ScanRange(g.Limits[0], g.Limits[1],
false, 0, func(n uint64) {}, iterator)
false, nil, iterator)
}
}
nmsg = append(nmsg, ']')
@ -364,7 +364,7 @@ func fenceMatchRoam(
prevNearbys := fence.roam.nearbys[tid]
var newNearbys map[string]bool
col.Intersects(obj, 0, 0, func(n uint64) {}, func(
col.Intersects(obj, 0, nil, func(
id string, obj2 geojson.Object, fields []float64,
) bool {
if c.hasExpired(fence.roam.key, id) {

21
internal/server/scan.go
View File

@ -66,7 +66,7 @@ func (c *Server) cmdScan(msg *Message) (res resp.Value, err error) {
} else {
g := glob.Parse(sw.globPattern, s.desc)
if g.Limits[0] == "" && g.Limits[1] == "" {
sw.col.Scan(s.desc, s.cursor, sw.IncCursor,
sw.col.Scan(s.desc, sw,
func(id string, o geojson.Object, fields []float64) bool {
return sw.writeObject(ScanWriterParams{
id: id,
@ -76,7 +76,7 @@ func (c *Server) cmdScan(msg *Message) (res resp.Value, err error) {
},
)
} else {
sw.col.ScanRange(g.Limits[0], g.Limits[1], s.desc, s.cursor, sw.IncCursor,
sw.col.ScanRange(g.Limits[0], g.Limits[1], s.desc, sw,
func(id string, o geojson.Object, fields []float64) bool {
return sw.writeObject(ScanWriterParams{
id: id,
@ -95,3 +95,20 @@ func (c *Server) cmdScan(msg *Message) (res resp.Value, err error) {
}
return sw.respOut, nil
}
// type tCursor struct {
// offset func() uint64
// iter func(n uint64)
// }
// func (cursor *tCursor) Offset() uint64 {
// return cursor.offset()
// }
// func (cursor *tCursor) Step(n uint64) {
// cursor.iter(n)
// }
// func newCursor(offset func() uint64, iter func(n uint64)) *tCursor {
// return &tCursor{offset, iter}
// }

8
internal/server/scanner.go
View File

@ -327,7 +327,11 @@ func (sw *scanWriter) globMatch(id string, o geojson.Object) (ok, keepGoing bool
}
// Increment cursor
func (sw *scanWriter) IncCursor(n uint64) {
func (sw *scanWriter) Offset() uint64 {
return sw.cursor
}
func (sw *scanWriter) Step(n uint64) {
sw.numberIters += n
}
@ -342,7 +346,7 @@ func (sw *scanWriter) testObject(id string, o geojson.Object, fields []float64,
}
}
nf, ok := sw.fieldMatch(fields, o)
return ok,true, nf
return ok, true, nf
}
//id string, o geojson.Object, fields []float64, noLock bool

12
internal/server/search.go
View File

@ -409,11 +409,11 @@ func (server *Server) nearestNeighbors(
maxDist := target.Haversine()
limit := int(sw.limit)
var items []iterItem
sw.col.Nearby(target, sw.cursor, sw.IncCursor, func(id string, o geojson.Object, fields []float64) bool {
sw.col.Nearby(target, sw, func(id string, o geojson.Object, fields []float64) bool {
if server.hasExpired(s.key, id) {
return true
}
ok, keepGoing, _ := sw.testObject(id, o, fields,true)
ok, keepGoing, _ := sw.testObject(id, o, fields, true)
if !ok {
return true
}
@ -480,7 +480,7 @@ func (server *Server) cmdWithinOrIntersects(cmd string, msg *Message) (res resp.
sw.writeHead()
if sw.col != nil {
if cmd == "within" {
sw.col.Within(s.obj, s.cursor, s.sparse, sw.IncCursor, func(
sw.col.Within(s.obj, s.sparse, sw, func(
id string, o geojson.Object, fields []float64,
) bool {
if server.hasExpired(s.key, id) {
@ -494,7 +494,7 @@ func (server *Server) cmdWithinOrIntersects(cmd string, msg *Message) (res resp.
})
})
} else if cmd == "intersects" {
sw.col.Intersects(s.obj, s.cursor, s.sparse, sw.IncCursor, func(
sw.col.Intersects(s.obj, s.sparse, sw, func(
id string,
o geojson.Object,
fields []float64,
@ -578,7 +578,7 @@ func (server *Server) cmdSearch(msg *Message) (res resp.Value, err error) {
} else {
g := glob.Parse(sw.globPattern, s.desc)
if g.Limits[0] == "" && g.Limits[1] == "" {
sw.col.SearchValues(s.desc, s.cursor, sw.IncCursor,
sw.col.SearchValues(s.desc, sw,
func(id string, o geojson.Object, fields []float64) bool {
return sw.writeObject(ScanWriterParams{
id: id,
@ -592,7 +592,7 @@ func (server *Server) cmdSearch(msg *Message) (res resp.Value, err error) {
// must disable globSingle for string value type matching because
// globSingle is only for ID matches, not values.
sw.globSingle = false
sw.col.SearchValuesRange(g.Limits[0], g.Limits[1], s.desc, s.cursor, sw.IncCursor,
sw.col.SearchValuesRange(g.Limits[0], g.Limits[1], s.desc, sw,
func(id string, o geojson.Object, fields []float64) bool {
return sw.writeObject(ScanWriterParams{
id: id,