From 300635727a345ca093e2ff6564d05bcf238561c9 Mon Sep 17 00:00:00 2001 From: Josh Baker Date: Mon, 24 Jul 2017 08:26:10 -0700 Subject: [PATCH] apply LIMIT after WHERE clause, fix #199 --- controller/aofshrink.go | 2 +- controller/collection/collection.go | 180 +++++++------ controller/collection/collection_test.go | 4 +- controller/crud.go | 4 +- controller/fence.go | 6 +- controller/hooks.go | 2 +- controller/live.go | 2 +- controller/scan.go | 17 +- controller/scanner.go | 8 +- controller/search.go | 25 +- index/index.go | 68 ++--- index/index_test.go | 14 +- index/rtree/knn.go | 247 ++++-------------- index/rtree/rtree_test.go | 6 +- index/rtree/rtreed.go | 4 +- vendor/github.com/tidwall/tinyqueue/LICENSE | 15 ++ vendor/github.com/tidwall/tinyqueue/README.md | 7 + .../github.com/tidwall/tinyqueue/tinyqueue.go | 86 ++++++ .../tidwall/tinyqueue/tinyqueue_test.go | 65 +++++ 19 files changed, 390 insertions(+), 372 deletions(-) create mode 100644 vendor/github.com/tidwall/tinyqueue/LICENSE create mode 100644 vendor/github.com/tidwall/tinyqueue/README.md create mode 100644 vendor/github.com/tidwall/tinyqueue/tinyqueue.go create mode 100644 vendor/github.com/tidwall/tinyqueue/tinyqueue_test.go diff --git a/controller/aofshrink.go b/controller/aofshrink.go index 668ce204..eb9add2f 100644 --- a/controller/aofshrink.go +++ b/controller/aofshrink.go @@ -93,7 +93,7 @@ func (c *Controller) aofshrink() { var exm = c.expires[keys[0]] // the expiration map var now = time.Now() // used for expiration var count = 0 // the object count - col.ScanGreaterOrEqual(nextid, 0, false, + col.ScanGreaterOrEqual(nextid, false, func(id string, obj geojson.Object, fields []float64) bool { if count == maxids { // we reached the max number of ids for one batch diff --git a/controller/collection/collection.go b/controller/collection/collection.go index 9a1ec577..cb877a3c 100644 --- a/controller/collection/collection.go +++ b/controller/collection/collection.go @@ -265,41 +265,33 @@ func (c *Collection) FieldArr() []string { return arr } -// Scan iterates though the collection ids. A cursor can be used for paging. -func (c *Collection) Scan(cursor uint64, desc bool, +// Scan iterates though the collection ids. +func (c *Collection) Scan(desc bool, iterator func(id string, obj geojson.Object, fields []float64) bool, -) (ncursor uint64) { - var i uint64 - var active = true +) bool { + var keepon = true iter := func(item btree.Item) bool { - if i >= cursor { - iitm := item.(*itemT) - active = iterator(iitm.id, iitm.object, c.getFieldValues(iitm.id)) - } - i++ - return active + iitm := item.(*itemT) + keepon = iterator(iitm.id, iitm.object, c.getFieldValues(iitm.id)) + return keepon } if desc { c.items.Descend(iter) } else { c.items.Ascend(iter) } - return i + return keepon } -// ScanGreaterOrEqual iterates though the collection starting with specified id. A cursor can be used for paging. -func (c *Collection) ScanRange(cursor uint64, start, end string, desc bool, +// ScanGreaterOrEqual iterates though the collection starting with specified id. +func (c *Collection) ScanRange(start, end string, desc bool, iterator func(id string, obj geojson.Object, fields []float64) bool, -) (ncursor uint64) { - var i uint64 - var active = true +) bool { + var keepon = true iter := func(item btree.Item) bool { - if i >= cursor { - iitm := item.(*itemT) - active = iterator(iitm.id, iitm.object, c.getFieldValues(iitm.id)) - } - i++ - return active + iitm := item.(*itemT) + keepon = iterator(iitm.id, iitm.object, c.getFieldValues(iitm.id)) + return keepon } if desc { @@ -307,77 +299,65 @@ func (c *Collection) ScanRange(cursor uint64, start, end string, desc bool, } else { c.items.AscendRange(&itemT{id: start}, &itemT{id: end}, iter) } - return i + return keepon } -// SearchValues iterates though the collection values. A cursor can be used for paging. -func (c *Collection) SearchValues(cursor uint64, desc bool, +// SearchValues iterates though the collection values. +func (c *Collection) SearchValues(desc bool, iterator func(id string, obj geojson.Object, fields []float64) bool, -) (ncursor uint64) { - var i uint64 - var active = true +) bool { + var keepon = true iter := func(item btree.Item) bool { - if i >= cursor { - iitm := item.(*itemT) - active = iterator(iitm.id, iitm.object, c.getFieldValues(iitm.id)) - } - i++ - return active + iitm := item.(*itemT) + keepon = iterator(iitm.id, iitm.object, c.getFieldValues(iitm.id)) + return keepon } if desc { c.values.Descend(iter) } else { c.values.Ascend(iter) } - return i + return keepon } -// SearchValuesRange iterates though the collection values. A cursor can be used for paging. -func (c *Collection) SearchValuesRange(cursor uint64, start, end string, desc bool, +// SearchValuesRange iterates though the collection values. +func (c *Collection) SearchValuesRange(start, end string, desc bool, iterator func(id string, obj geojson.Object, fields []float64) bool, -) (ncursor uint64) { - var i uint64 - var active = true +) bool { + var keepon = true iter := func(item btree.Item) bool { - if i >= cursor { - iitm := item.(*itemT) - active = iterator(iitm.id, iitm.object, c.getFieldValues(iitm.id)) - } - i++ - return active + iitm := item.(*itemT) + keepon = iterator(iitm.id, iitm.object, c.getFieldValues(iitm.id)) + return keepon } if desc { c.values.DescendRange(&itemT{object: geojson.String(start)}, &itemT{object: geojson.String(end)}, iter) } else { c.values.AscendRange(&itemT{object: geojson.String(start)}, &itemT{object: geojson.String(end)}, iter) } - return i + return keepon } -// ScanGreaterOrEqual iterates though the collection starting with specified id. A cursor can be used for paging. -func (c *Collection) ScanGreaterOrEqual(id string, cursor uint64, desc bool, +// ScanGreaterOrEqual iterates though the collection starting with specified id. +func (c *Collection) ScanGreaterOrEqual(id string, desc bool, iterator func(id string, obj geojson.Object, fields []float64) bool, -) (ncursor uint64) { - var i uint64 - var active = true +) bool { + var keepon = true iter := func(item btree.Item) bool { - if i >= cursor { - iitm := item.(*itemT) - active = iterator(iitm.id, iitm.object, c.getFieldValues(iitm.id)) - } - i++ - return active + iitm := item.(*itemT) + keepon = iterator(iitm.id, iitm.object, c.getFieldValues(iitm.id)) + return keepon } if desc { c.items.DescendLessOrEqual(&itemT{id: id}, iter) } else { c.items.AscendGreaterOrEqual(&itemT{id: id}, iter) } - return i + return keepon } -func (c *Collection) geoSearch(cursor uint64, bbox geojson.BBox, iterator func(id string, obj geojson.Object, fields []float64) bool) (ncursor uint64) { - return c.index.Search(cursor, bbox.Min.Y, bbox.Min.X, bbox.Max.Y, bbox.Max.X, bbox.Min.Z, bbox.Max.Z, func(item index.Item) bool { +func (c *Collection) geoSearch(bbox geojson.BBox, iterator func(id string, obj geojson.Object, fields []float64) bool) bool { + return c.index.Search(bbox.Min.Y, bbox.Min.X, bbox.Max.Y, bbox.Max.X, bbox.Min.Z, bbox.Max.Z, func(item index.Item) bool { var iitm *itemT iitm, ok := item.(*itemT) if !ok { @@ -391,14 +371,15 @@ func (c *Collection) geoSearch(cursor uint64, bbox geojson.BBox, iterator func(i } // Nearby returns all object that are nearby a point. -func (c *Collection) Nearby(cursor uint64, sparse uint8, lat, lon, meters, minZ, maxZ float64, iterator func(id string, obj geojson.Object, fields []float64) bool) (ncursor uint64) { +func (c *Collection) Nearby(sparse uint8, lat, lon, meters, minZ, maxZ float64, iterator func(id string, obj geojson.Object, fields []float64) bool) bool { + var keepon = true center := geojson.Position{X: lon, Y: lat, Z: 0} bbox := geojson.BBoxesFromCenter(lat, lon, meters) bboxes := bbox.Sparse(sparse) if sparse > 0 { for _, bbox := range bboxes { bbox.Min.Z, bbox.Max.Z = minZ, maxZ - c.geoSearch(cursor, bbox, func(id string, obj geojson.Object, fields []float64) bool { + keepon = c.geoSearch(bbox, func(id string, obj geojson.Object, fields []float64) bool { if obj.Nearby(center, meters) { if iterator(id, obj, fields) { return false @@ -406,11 +387,14 @@ func (c *Collection) Nearby(cursor uint64, sparse uint8, lat, lon, meters, minZ, } return true }) + if !keepon { + break + } } - return 0 + return keepon } bbox.Min.Z, bbox.Max.Z = minZ, maxZ - return c.geoSearch(cursor, bbox, func(id string, obj geojson.Object, fields []float64) bool { + return c.geoSearch(bbox, func(id string, obj geojson.Object, fields []float64) bool { if obj.Nearby(center, meters) { return iterator(id, obj, fields) } @@ -419,7 +403,8 @@ func (c *Collection) Nearby(cursor uint64, sparse uint8, lat, lon, meters, minZ, } // Within returns all object that are fully contained within an object or bounding box. Set obj to nil in order to use the bounding box. -func (c *Collection) Within(cursor uint64, sparse uint8, obj geojson.Object, minLat, minLon, maxLat, maxLon, minZ, maxZ float64, iterator func(id string, obj geojson.Object, fields []float64) bool) (ncursor uint64) { +func (c *Collection) Within(sparse uint8, obj geojson.Object, minLat, minLon, maxLat, maxLon, minZ, maxZ float64, iterator func(id string, obj geojson.Object, fields []float64) bool) bool { + var keepon = true var bbox geojson.BBox if obj != nil { bbox = obj.CalculatedBBox() @@ -436,7 +421,7 @@ func (c *Collection) Within(cursor uint64, sparse uint8, obj geojson.Object, min if sparse > 0 { for _, bbox := range bboxes { if obj != nil { - c.geoSearch(cursor, bbox, func(id string, o geojson.Object, fields []float64) bool { + keepon = c.geoSearch(bbox, func(id string, o geojson.Object, fields []float64) bool { if o.Within(obj) { if iterator(id, o, fields) { return false @@ -445,26 +430,31 @@ func (c *Collection) Within(cursor uint64, sparse uint8, obj geojson.Object, min return true }) } - c.geoSearch(cursor, bbox, func(id string, o geojson.Object, fields []float64) bool { - if o.WithinBBox(bbox) { - if iterator(id, o, fields) { - return false + if keepon { + keepon = c.geoSearch(bbox, func(id string, o geojson.Object, fields []float64) bool { + if o.WithinBBox(bbox) { + if iterator(id, o, fields) { + return false + } } - } - return true - }) + return true + }) + } + if !keepon { + break + } } - return 0 + return keepon } if obj != nil { - return c.geoSearch(cursor, bbox, func(id string, o geojson.Object, fields []float64) bool { + return c.geoSearch(bbox, func(id string, o geojson.Object, fields []float64) bool { if o.Within(obj) { return iterator(id, o, fields) } return true }) } - return c.geoSearch(cursor, bbox, func(id string, o geojson.Object, fields []float64) bool { + return c.geoSearch(bbox, func(id string, o geojson.Object, fields []float64) bool { if o.WithinBBox(bbox) { return iterator(id, o, fields) } @@ -473,7 +463,8 @@ func (c *Collection) Within(cursor uint64, sparse uint8, obj geojson.Object, min } // Intersects returns all object that are intersect an object or bounding box. Set obj to nil in order to use the bounding box. -func (c *Collection) Intersects(cursor uint64, sparse uint8, obj geojson.Object, minLat, minLon, maxLat, maxLon, minZ, maxZ float64, iterator func(id string, obj geojson.Object, fields []float64) bool) (ncursor uint64) { +func (c *Collection) Intersects(sparse uint8, obj geojson.Object, minLat, minLon, maxLat, maxLon, minZ, maxZ float64, iterator func(id string, obj geojson.Object, fields []float64) bool) bool { + var keepon = true var bbox geojson.BBox if obj != nil { bbox = obj.CalculatedBBox() @@ -501,7 +492,7 @@ func (c *Collection) Intersects(cursor uint64, sparse uint8, obj geojson.Object, } for _, bbox := range bboxes { if obj != nil { - c.geoSearch(cursor, bbox, func(id string, o geojson.Object, fields []float64) bool { + keepon = c.geoSearch(bbox, func(id string, o geojson.Object, fields []float64) bool { if o.Intersects(obj) { if iterator(id, o, fields) { return false @@ -510,26 +501,31 @@ func (c *Collection) Intersects(cursor uint64, sparse uint8, obj geojson.Object, return true }) } - c.geoSearch(cursor, bbox, func(id string, o geojson.Object, fields []float64) bool { - if o.IntersectsBBox(bbox) { - if iterator(id, o, fields) { - return false + if keepon { + keepon = c.geoSearch(bbox, func(id string, o geojson.Object, fields []float64) bool { + if o.IntersectsBBox(bbox) { + if iterator(id, o, fields) { + return false + } } - } - return true - }) + return true + }) + } + if !keepon { + break + } } - return 0 + return keepon } if obj != nil { - return c.geoSearch(cursor, bbox, func(id string, o geojson.Object, fields []float64) bool { + return c.geoSearch(bbox, func(id string, o geojson.Object, fields []float64) bool { if o.Intersects(obj) { return iterator(id, o, fields) } return true }) } - return c.geoSearch(cursor, bbox, func(id string, o geojson.Object, fields []float64) bool { + return c.geoSearch(bbox, func(id string, o geojson.Object, fields []float64) bool { if o.IntersectsBBox(bbox) { return iterator(id, o, fields) } @@ -537,8 +533,8 @@ func (c *Collection) Intersects(cursor uint64, sparse uint8, obj geojson.Object, }) } -func (c *Collection) NearestNeighbors(k int, lat, lon float64, iterator func(id string, obj geojson.Object, fields []float64) bool) { - c.index.NearestNeighbors(k, lat, lon, func(item index.Item) bool { +func (c *Collection) NearestNeighbors(lat, lon float64, iterator func(id string, obj geojson.Object, fields []float64) bool) bool { + return c.index.NearestNeighbors(lat, lon, func(item index.Item) bool { var iitm *itemT iitm, ok := item.(*itemT) if !ok { diff --git a/controller/collection/collection_test.go b/controller/collection/collection_test.go index 7b5df58c..ecd9ac78 100644 --- a/controller/collection/collection_test.go +++ b/controller/collection/collection_test.go @@ -32,7 +32,7 @@ func TestCollection(t *testing.T) { } count := 0 bbox := geojson.BBox{Min: geojson.Position{X: -180, Y: -90, Z: 0}, Max: geojson.Position{X: 180, Y: 90, Z: 0}} - c.geoSearch(0, bbox, func(id string, obj geojson.Object, field []float64) bool { + c.geoSearch(bbox, func(id string, obj geojson.Object, field []float64) bool { count++ return true }) @@ -84,7 +84,7 @@ func TestManyCollections(t *testing.T) { col := colsM["13"] //println(col.Count()) bbox := geojson.BBox{Min: geojson.Position{X: -180, Y: 30, Z: 0}, Max: geojson.Position{X: 34, Y: 100, Z: 0}} - col.geoSearch(0, bbox, func(id string, obj geojson.Object, fields []float64) bool { + col.geoSearch(bbox, func(id string, obj geojson.Object, fields []float64) bool { //println(id) return true }) diff --git a/controller/crud.go b/controller/crud.go index deaca8e5..e73f41b1 100644 --- a/controller/crud.go +++ b/controller/crud.go @@ -378,9 +378,9 @@ func (c *Controller) cmdPdel(msg *server.Message) (res string, d commandDetailsT if col != nil { g := glob.Parse(d.pattern, false) if g.Limits[0] == "" && g.Limits[1] == "" { - col.Scan(0, false, iter) + col.Scan(false, iter) } else { - col.ScanRange(0, g.Limits[0], g.Limits[1], false, iter) + col.ScanRange(g.Limits[0], g.Limits[1], false, iter) } var atLeastOneNotDeleted bool for i, dc := range d.children { diff --git a/controller/fence.go b/controller/fence.go index 3cf8b764..45cc4400 100644 --- a/controller/fence.go +++ b/controller/fence.go @@ -261,9 +261,9 @@ func fenceMatch(hookName string, sw *scanWriter, fence *liveFenceSwitches, metas } g := glob.Parse(pattern, false) if g.Limits[0] == "" && g.Limits[1] == "" { - col.Scan(0, false, iterator) + col.Scan(false, iterator) } else { - col.ScanRange(0, g.Limits[0], g.Limits[1], false, iterator) + col.ScanRange(g.Limits[0], g.Limits[1], false, iterator) } } }() @@ -332,7 +332,7 @@ func fenceMatchRoam(c *Controller, fence *liveFenceSwitches, tkey, tid string, o return } p := obj.CalculatedPoint() - col.Nearby(0, 0, p.Y, p.X, fence.roam.meters, math.Inf(-1), math.Inf(+1), + col.Nearby(0, p.Y, p.X, fence.roam.meters, math.Inf(-1), math.Inf(+1), func(id string, obj geojson.Object, fields []float64) bool { var match bool if id == tid { diff --git a/controller/hooks.go b/controller/hooks.go index 5d2de680..8287a89f 100644 --- a/controller/hooks.go +++ b/controller/hooks.go @@ -133,7 +133,7 @@ func (c *Controller) cmdSetHook(msg *server.Message) (res string, d commandDetai hook.cond = sync.NewCond(&hook.mu) var wr bytes.Buffer - hook.ScanWriter, err = c.newScanWriter(&wr, cmsg, s.key, s.output, s.precision, s.glob, false, s.limit, s.wheres, s.nofields) + hook.ScanWriter, err = c.newScanWriter(&wr, cmsg, s.key, s.output, s.precision, s.glob, false, s.cursor, s.limit, s.wheres, s.nofields) if err != nil { return "", d, err } diff --git a/controller/live.go b/controller/live.go index 0a53f44d..3bed9dfb 100644 --- a/controller/live.go +++ b/controller/live.go @@ -87,7 +87,7 @@ func (c *Controller) goLive(inerr error, conn net.Conn, rd *server.AnyReaderWrit lb.key = s.key lb.fence = &s c.mu.RLock() - sw, err = c.newScanWriter(&wr, msg, s.key, s.output, s.precision, s.glob, false, s.limit, s.wheres, s.nofields) + sw, err = c.newScanWriter(&wr, msg, s.key, s.output, s.precision, s.glob, false, s.cursor, s.limit, s.wheres, s.nofields) c.mu.RUnlock() } // everything below if for live SCAN, NEARBY, WITHIN, INTERSECTS diff --git a/controller/scan.go b/controller/scan.go index ddfe407e..376d2fac 100644 --- a/controller/scan.go +++ b/controller/scan.go @@ -30,7 +30,7 @@ func (c *Controller) cmdScan(msg *server.Message) (res string, err error) { if err != nil { return "", err } - sw, err := c.newScanWriter(wr, msg, s.key, s.output, s.precision, s.glob, false, s.limit, s.wheres, s.nofields) + sw, err := c.newScanWriter(wr, msg, s.key, s.output, s.precision, s.glob, false, s.cursor, s.limit, s.wheres, s.nofields) if err != nil { return "", err } @@ -48,22 +48,21 @@ func (c *Controller) cmdScan(msg *server.Message) (res string, err error) { } else { g := glob.Parse(sw.globPattern, s.desc) if g.Limits[0] == "" && g.Limits[1] == "" { - s.cursor = sw.col.Scan(s.cursor, s.desc, + sw.col.Scan(s.desc, func(id string, o geojson.Object, fields []float64) bool { return sw.writeObject(ScanWriterParams{ - id: id, - o: o, + id: id, + o: o, fields: fields, }) }, ) } else { - s.cursor = sw.col.ScanRange( - s.cursor, g.Limits[0], g.Limits[1], s.desc, + sw.col.ScanRange(g.Limits[0], g.Limits[1], s.desc, func(id string, o geojson.Object, fields []float64) bool { return sw.writeObject(ScanWriterParams{ - id: id, - o: o, + id: id, + o: o, fields: fields, }) }, @@ -71,7 +70,7 @@ func (c *Controller) cmdScan(msg *server.Message) (res string, err error) { } } } - sw.writeFoot(s.cursor) + sw.writeFoot() if msg.OutputType == server.JSON { wr.WriteString(`,"elapsed":"` + time.Now().Sub(start).String() + "\"}") } diff --git a/controller/scanner.go b/controller/scanner.go index f6172a26..d912c097 100644 --- a/controller/scanner.go +++ b/controller/scanner.go @@ -41,6 +41,7 @@ type scanWriter struct { wheres []whereT numberItems uint64 nofields bool + cursor uint64 limit uint64 hitLimit bool once bool @@ -65,7 +66,7 @@ type ScanWriterParams struct { func (c *Controller) newScanWriter( wr *bytes.Buffer, msg *server.Message, key string, output outputT, precision uint64, globPattern string, matchValues bool, - limit uint64, wheres []whereT, nofields bool, + cursor, limit uint64, wheres []whereT, nofields bool, ) ( *scanWriter, error, ) { @@ -83,6 +84,7 @@ func (c *Controller) newScanWriter( c: c, wr: wr, msg: msg, + cursor: cursor, limit: limit, wheres: wheres, output: output, @@ -149,9 +151,10 @@ func (sw *scanWriter) writeHead() { } } -func (sw *scanWriter) writeFoot(cursor uint64) { +func (sw *scanWriter) writeFoot() { sw.mu.Lock() defer sw.mu.Unlock() + cursor := sw.numberItems + sw.cursor if !sw.hitLimit { cursor = 0 } @@ -275,7 +278,6 @@ func (sw *scanWriter) writeObject(opts ScanWriterParams) bool { if sw.output == outputCount { return true } - switch sw.msg.OutputType { case server.JSON: var wr bytes.Buffer diff --git a/controller/search.go b/controller/search.go index 18902943..402de04a 100644 --- a/controller/search.go +++ b/controller/search.go @@ -296,7 +296,7 @@ func (c *Controller) cmdNearby(msg *server.Message) (res string, err error) { return "", s } minZ, maxZ := zMinMaxFromWheres(s.wheres) - sw, err := c.newScanWriter(wr, msg, s.key, s.output, s.precision, s.glob, false, s.limit, s.wheres, s.nofields) + sw, err := c.newScanWriter(wr, msg, s.key, s.output, s.precision, s.glob, false, s.cursor, s.limit, s.wheres, s.nofields) if err != nil { return "", err } @@ -322,12 +322,12 @@ func (c *Controller) cmdNearby(msg *server.Message) (res string, err error) { }) } if s.knn { - sw.col.NearestNeighbors(int(s.limit), s.lat, s.lon, iter) + sw.col.NearestNeighbors(s.lat, s.lon, iter) } else { - s.cursor = sw.col.Nearby(s.cursor, s.sparse, s.lat, s.lon, s.meters, minZ, maxZ, iter) + sw.col.Nearby(s.sparse, s.lat, s.lon, s.meters, minZ, maxZ, iter) } } - sw.writeFoot(s.cursor) + sw.writeFoot() if msg.OutputType == server.JSON { wr.WriteString(`,"elapsed":"` + time.Now().Sub(start).String() + "\"}") } @@ -355,7 +355,7 @@ func (c *Controller) cmdWithinOrIntersects(cmd string, msg *server.Message) (res if s.fence { return "", s } - sw, err := c.newScanWriter(wr, msg, s.key, s.output, s.precision, s.glob, false, s.limit, s.wheres, s.nofields) + sw, err := c.newScanWriter(wr, msg, s.key, s.output, s.precision, s.glob, false, s.cursor, s.limit, s.wheres, s.nofields) if err != nil { return "", err } @@ -368,7 +368,7 @@ func (c *Controller) cmdWithinOrIntersects(cmd string, msg *server.Message) (res sw.writeHead() minZ, maxZ := zMinMaxFromWheres(s.wheres) if cmd == "within" { - s.cursor = sw.col.Within(s.cursor, s.sparse, s.o, s.minLat, s.minLon, s.maxLat, s.maxLon, minZ, maxZ, + sw.col.Within(s.sparse, s.o, s.minLat, s.minLon, s.maxLat, s.maxLon, minZ, maxZ, func(id string, o geojson.Object, fields []float64) bool { if c.hasExpired(s.key, id) { return true @@ -381,7 +381,7 @@ func (c *Controller) cmdWithinOrIntersects(cmd string, msg *server.Message) (res }, ) } else if cmd == "intersects" { - s.cursor = sw.col.Intersects(s.cursor, s.sparse, s.o, s.minLat, s.minLon, s.maxLat, s.maxLon, minZ, maxZ, + sw.col.Intersects(s.sparse, s.o, s.minLat, s.minLon, s.maxLat, s.maxLon, minZ, maxZ, func(id string, o geojson.Object, fields []float64) bool { if c.hasExpired(s.key, id) { return true @@ -394,7 +394,7 @@ func (c *Controller) cmdWithinOrIntersects(cmd string, msg *server.Message) (res }, ) } - sw.writeFoot(s.cursor) + sw.writeFoot() if msg.OutputType == server.JSON { wr.WriteString(`,"elapsed":"` + time.Now().Sub(start).String() + "\"}") } @@ -421,7 +421,7 @@ func (c *Controller) cmdSearch(msg *server.Message) (res string, err error) { if err != nil { return "", err } - sw, err := c.newScanWriter(wr, msg, s.key, s.output, s.precision, s.glob, true, s.limit, s.wheres, s.nofields) + sw, err := c.newScanWriter(wr, msg, s.key, s.output, s.precision, s.glob, true, s.cursor, s.limit, s.wheres, s.nofields) if err != nil { return "", err } @@ -439,7 +439,7 @@ func (c *Controller) cmdSearch(msg *server.Message) (res string, err error) { } else { g := glob.Parse(sw.globPattern, s.desc) if g.Limits[0] == "" && g.Limits[1] == "" { - s.cursor = sw.col.SearchValues(s.cursor, s.desc, + sw.col.SearchValues(s.desc, func(id string, o geojson.Object, fields []float64) bool { return sw.writeObject(ScanWriterParams{ id: id, @@ -452,8 +452,7 @@ func (c *Controller) cmdSearch(msg *server.Message) (res string, err error) { // must disable globSingle for string value type matching because // globSingle is only for ID matches, not values. sw.globSingle = false - s.cursor = sw.col.SearchValuesRange( - s.cursor, g.Limits[0], g.Limits[1], s.desc, + sw.col.SearchValuesRange(g.Limits[0], g.Limits[1], s.desc, func(id string, o geojson.Object, fields []float64) bool { return sw.writeObject(ScanWriterParams{ id: id, @@ -465,7 +464,7 @@ func (c *Controller) cmdSearch(msg *server.Message) (res string, err error) { } } } - sw.writeFoot(s.cursor) + sw.writeFoot() if msg.OutputType == server.JSON { wr.WriteString(`,"elapsed":"` + time.Now().Sub(start).String() + "\"}") } diff --git a/index/index.go b/index/index.go index 80a49164..9e761bb7 100644 --- a/index/index.go +++ b/index/index.go @@ -96,7 +96,7 @@ func (ix *Index) Remove(item Item) { // Count counts all items in the index. func (ix *Index) Count() int { count := 0 - ix.Search(0, -90, -180, 90, 180, math.Inf(-1), math.Inf(+1), func(item Item) bool { + ix.Search(-90, -180, 90, 180, math.Inf(-1), math.Inf(+1), func(item Item) bool { count++ return true }) @@ -123,72 +123,62 @@ func (ix *Index) getRTreeItem(item rtree.Item) Item { return nil } -func (ix *Index) NearestNeighbors(k int, lat, lon float64, iterator func(item Item) bool) { +func (ix *Index) NearestNeighbors(lat, lon float64, iterator func(item Item) bool) bool { x, y, _ := normPoint(lat, lon) - items := ix.r.NearestNeighbors(k, x, y, 0) - for _, item := range items { + return ix.r.NearestNeighbors(x, y, 0, func(item rtree.Item, dist float64) bool { iitm := ix.getRTreeItem(item) if item == nil { - continue + return true } - if !iterator(iitm) { - break - } - } + return iterator(iitm) + }) } // Search returns all items that intersect the bounding box. -func (ix *Index) Search(cursor uint64, swLat, swLon, neLat, neLon, minZ, maxZ float64, iterator func(item Item) bool) (ncursor uint64) { - var idx uint64 - var active = true +func (ix *Index) Search(swLat, swLon, neLat, neLon, minZ, maxZ float64, iterator func(item Item) bool) bool { + var keepon = true var idm = make(map[Item]bool) mins, maxs, _ := normRect(swLat, swLon, neLat, neLon) // Points if len(mins) == 1 { // There is only one rectangle. // It's possible that a r rect may span multiple entries. Check mulm map for spanning rects. - if active { + if keepon { ix.r.Search(mins[0][0], mins[0][1], minZ, maxs[0][0], maxs[0][1], maxZ, func(item rtree.Item) bool { - if idx >= cursor { - iitm := ix.getRTreeItem(item) - if iitm != nil { - if ix.mulm[iitm] { - if !idm[iitm] { - idm[iitm] = true - active = iterator(iitm) - } - } else { - active = iterator(iitm) + iitm := ix.getRTreeItem(item) + if iitm != nil { + if ix.mulm[iitm] { + if !idm[iitm] { + idm[iitm] = true + keepon = iterator(iitm) } + } else { + keepon = iterator(iitm) } } - idx++ - return active + return keepon }) } } else { // There are multiple rectangles. Duplicates might occur. for i := range mins { - if active { + if keepon { ix.r.Search(mins[i][0], mins[i][1], minZ, maxs[i][0], maxs[i][1], maxZ, func(item rtree.Item) bool { - if idx >= cursor { - iitm := ix.getRTreeItem(item) - if iitm != nil { - if ix.mulm[iitm] { - if !idm[iitm] { - idm[iitm] = true - active = iterator(iitm) - } - } else { - active = iterator(iitm) + iitm := ix.getRTreeItem(item) + if iitm != nil { + if ix.mulm[iitm] { + if !idm[iitm] { + idm[iitm] = true + keepon = iterator(iitm) } + } else { + keepon = iterator(iitm) } } - idx++ - return active + return keepon }) } } } - return idx + return keepon } diff --git a/index/index_test.go b/index/index_test.go index 24f33d1c..d3146fa7 100644 --- a/index/index_test.go +++ b/index/index_test.go @@ -60,7 +60,7 @@ func TestRandomInserts(t *testing.T) { } count = 0 items := make([]Item, 0, l) - tr.Search(0, -90, -180, 90, 180, 0, 0, func(item Item) bool { + tr.Search(-90, -180, 90, 180, 0, 0, func(item Item) bool { count++ items = append(items, item) return true @@ -70,7 +70,7 @@ func TestRandomInserts(t *testing.T) { } start = time.Now() count1 := 0 - tr.Search(0, 33, -115, 34, -114, 0, 0, func(item Item) bool { + tr.Search(33, -115, 34, -114, 0, 0, func(item Item) bool { count1++ return true }) @@ -79,7 +79,7 @@ func TestRandomInserts(t *testing.T) { start = time.Now() count2 := 0 - tr.Search(0, 33-180, -115-360, 34-180, -114-360, 0, 0, func(item Item) bool { + tr.Search(33-180, -115-360, 34-180, -114-360, 0, 0, func(item Item) bool { count2++ return true }) @@ -87,7 +87,7 @@ func TestRandomInserts(t *testing.T) { start = time.Now() count3 := 0 - tr.Search(0, -10, 170, 20, 200, 0, 0, func(item Item) bool { + tr.Search(-10, 170, 20, 200, 0, 0, func(item Item) bool { count3++ return true }) @@ -99,7 +99,7 @@ func TestRandomInserts(t *testing.T) { fmt.Printf("Searched %d items in %s.\n", count2, searchdur2.String()) fmt.Printf("Searched %d items in %s.\n", count3, searchdur3.String()) - tr.Search(0, -10, 170, 20, 200, 0, 0, func(item Item) bool { + tr.Search(-10, 170, 20, 200, 0, 0, func(item Item) bool { lat1, lon1, _, lat2, lon2, _ := item.Rect() if lat1 == lat2 && lon1 == lon2 { return false @@ -107,7 +107,7 @@ func TestRandomInserts(t *testing.T) { return true }) - tr.Search(0, -10, 170, 20, 200, 0, 0, func(item Item) bool { + tr.Search(-10, 170, 20, 200, 0, 0, func(item Item) bool { lat1, lon1, _, lat2, lon2, _ := item.Rect() if lat1 != lat2 || lon1 != lon2 { return false @@ -173,7 +173,7 @@ func TestInsertVarious(t *testing.T) { t.Fatalf("count = %d, expect 1", count) } found := false - tr.Search(0, -90, -180, 90, 180, 0, 0, func(item2 Item) bool { + tr.Search(-90, -180, 90, 180, 0, 0, func(item2 Item) bool { if item2 == item { found = true } diff --git a/index/rtree/knn.go b/index/rtree/knn.go index 30497b6e..71310230 100644 --- a/index/rtree/knn.go +++ b/index/rtree/knn.go @@ -1,205 +1,60 @@ -// Much of the KNN code has been adapted from the -// github.com/dhconnelly/rtreego project. -// -// Copyright 2012 Daniel Connelly. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. package rtree import ( - "math" - "sort" + "github.com/tidwall/tinyqueue" ) +type queueItem struct { + node *d3nodeT + data interface{} + isItem bool + dist float64 +} + +func (item *queueItem) Less(b tinyqueue.Item) bool { + return item.dist < b.(*queueItem).dist +} +func boxDistPoint(point []float64, childBox d3rectT) float64 { + var dist float64 + for i := 0; i < len(point); i++ { + d := axisDist(point[i], float64(childBox.min[i]), float64(childBox.max[i])) + dist += d * d + } + return dist +} +func axisDist(k, min, max float64) float64 { + if k < min { + return min - k + } + if k <= max { + return 0 + } + return k - max +} + // NearestNeighbors gets the closest Spatials to the Point. -func (tr *RTree) NearestNeighbors(k int, x, y, z float64) []Item { - if tr.tr.root == nil { - return nil - } - dists := make([]float64, k) - objs := make([]Item, k) - for i := 0; i < k; i++ { - dists[i] = math.MaxFloat64 - objs[i] = nil - } - objs, _ = tr.nearestNeighbors(k, x, y, z, tr.tr.root, dists, objs) - //for i := 0; i < len(objs); i++ { - // fmt.Printf("%v\n", objs[i]) - //} - for i := 0; i < len(objs); i++ { - if objs[i] == nil { - return objs[:i] +func (tr *RTree) NearestNeighbors(x, y, z float64, iter func(item Item, dist float64) bool) bool { + knnPoint := []float64{x, y, z} + queue := tinyqueue.New(nil) + node := tr.tr.root + for node != nil { + for i := 0; i < node.count; i++ { + child := node.branch[i] + dist := boxDistPoint(knnPoint, node.branch[i].rect) + queue.Push(&queueItem{node: child.child, data: child.data, isItem: node.isLeaf(), dist: dist}) + } + for queue.Len() > 0 && queue.Peek().(*queueItem).isItem { + item := queue.Pop().(*queueItem) + if !iter(item.data.(Item), item.dist) { + return false + } + } + last := queue.Pop() + if last != nil { + node = last.(*queueItem).node + } else { + node = nil } } - return objs -} - -// minDist computes the square of the distance from a point to a rectangle. -// If the point is contained in the rectangle then the distance is zero. -// -// Implemented per Definition 2 of "Nearest Neighbor Queries" by -// N. Roussopoulos, S. Kelley and F. Vincent, ACM SIGMOD, pages 71-79, 1995. -func minDist(x, y, z float64, r d3rectT) float64 { - sum := 0.0 - p := [3]float64{x, y, z} - rp := [3]float64{ - float64(r.min[0]), float64(r.min[1]), float64(r.min[2]), - } - rq := [3]float64{ - float64(r.max[0]), float64(r.max[1]), float64(r.max[2]), - } - for i := 0; i < 3; i++ { - if p[i] < float64(rp[i]) { - d := p[i] - float64(rp[i]) - sum += d * d - } else if p[i] > float64(rq[i]) { - d := p[i] - float64(rq[i]) - sum += d * d - } - } - return sum -} - -func (tr *RTree) nearestNeighbors(k int, x, y, z float64, n *d3nodeT, dists []float64, nearest []Item) ([]Item, []float64) { - if n.isLeaf() { - for i := 0; i < n.count; i++ { - e := n.branch[i] - dist := math.Sqrt(minDist(x, y, z, e.rect)) - dists, nearest = insertNearest(k, dists, nearest, dist, e.data.(Item)) - } - } else { - branches, branchDists := sortEntries(x, y, z, n.branch[:n.count]) - branches = pruneEntries(x, y, z, branches, branchDists) - for _, e := range branches { - nearest, dists = tr.nearestNeighbors(k, x, y, z, e.child, dists, nearest) - } - } - return nearest, dists -} - -// insert obj into nearest and return the first k elements in increasing order. -func insertNearest(k int, dists []float64, nearest []Item, dist float64, obj Item) ([]float64, []Item) { - i := 0 - for i < k && dist >= dists[i] { - i++ - } - if i >= k { - return dists, nearest - } - - left, right := dists[:i], dists[i:k-1] - updatedDists := make([]float64, k) - copy(updatedDists, left) - updatedDists[i] = dist - copy(updatedDists[i+1:], right) - - leftObjs, rightObjs := nearest[:i], nearest[i:k-1] - updatedNearest := make([]Item, k) - copy(updatedNearest, leftObjs) - updatedNearest[i] = obj - copy(updatedNearest[i+1:], rightObjs) - - return updatedDists, updatedNearest -} - -type entrySlice struct { - entries []d3branchT - dists []float64 - x, y, z float64 -} - -func (s entrySlice) Len() int { return len(s.entries) } - -func (s entrySlice) Swap(i, j int) { - s.entries[i], s.entries[j] = s.entries[j], s.entries[i] - s.dists[i], s.dists[j] = s.dists[j], s.dists[i] -} -func (s entrySlice) Less(i, j int) bool { - return s.dists[i] < s.dists[j] -} - -func sortEntries(x, y, z float64, entries []d3branchT) ([]d3branchT, []float64) { - sorted := make([]d3branchT, len(entries)) - dists := make([]float64, len(entries)) - for i := 0; i < len(entries); i++ { - sorted[i] = entries[i] - dists[i] = minDist(x, y, z, entries[i].rect) - } - sort.Sort(entrySlice{sorted, dists, x, y, z}) - return sorted, dists -} - -func pruneEntries(x, y, z float64, entries []d3branchT, minDists []float64) []d3branchT { - minMinMaxDist := math.MaxFloat64 - for i := range entries { - minMaxDist := minMaxDist(x, y, z, entries[i].rect) - if minMaxDist < minMinMaxDist { - minMinMaxDist = minMaxDist - } - } - // remove all entries with minDist > minMinMaxDist - pruned := []d3branchT{} - for i := range entries { - if minDists[i] <= minMinMaxDist { - pruned = append(pruned, entries[i]) - } - } - return pruned -} - -// minMaxDist computes the minimum of the maximum distances from p to points -// on r. If r is the bounding box of some geometric objects, then there is -// at least one object contained in r within minMaxDist(p, r) of p. -// -// Implemented per Definition 4 of "Nearest Neighbor Queries" by -// N. Roussopoulos, S. Kelley and F. Vincent, ACM SIGMOD, pages 71-79, 1995. -func minMaxDist(x, y, z float64, r d3rectT) float64 { - - p := [3]float64{x, y, z} - rp := [3]float64{ - float64(r.min[0]), float64(r.min[1]), float64(r.min[2]), - } - rq := [3]float64{ - float64(r.max[0]), float64(r.max[1]), float64(r.max[2]), - } - - // by definition, MinMaxDist(p, r) = - // min{1<=k<=n}(|pk - rmk|^2 + sum{1<=i<=n, i != k}(|pi - rMi|^2)) - // where rmk and rMk are defined as follows: - - rm := func(k int) float64 { - if p[k] <= (rp[k]+rq[k])/2 { - return rp[k] - } - return rq[k] - } - - rM := func(k int) float64 { - if p[k] >= (rp[k]+rq[k])/2 { - return rp[k] - } - return rq[k] - } - - // This formula can be computed in linear time by precomputing - // S = sum{1<=i<=n}(|pi - rMi|^2). - - S := 0.0 - for i := range p { - d := p[i] - rM(i) - S += d * d - } - - // Compute MinMaxDist using the precomputed S. - min := math.MaxFloat64 - for k := range p { - d1 := p[k] - rM(k) - d2 := p[k] - rm(k) - d := S - d1*d1 + d2*d2 - if d < min { - min = d - } - } - - return min + return true } diff --git a/index/rtree/rtree_test.go b/index/rtree/rtree_test.go index 2e39a3d0..77bb7306 100644 --- a/index/rtree/rtree_test.go +++ b/index/rtree/rtree_test.go @@ -96,7 +96,11 @@ func TestKNN(t *testing.T) { tr.Insert(wpp(12, 19, 0)) tr.Insert(wpp(-5, 5, 0)) tr.Insert(wpp(33, 21, 0)) - items := tr.NearestNeighbors(10, x, y, z) + var items []Item + tr.NearestNeighbors(x, y, z, func(item Item, dist float64) bool { + items = append(items, item) + return true + }) var res string for i, item := range items { ix, iy, _, _, _, _ := item.Rect() diff --git a/index/rtree/rtreed.go b/index/rtree/rtreed.go index 0cd8c704..8ed74d06 100644 --- a/index/rtree/rtreed.go +++ b/index/rtree/rtreed.go @@ -2,9 +2,9 @@ package rtree import "math" -type float float32 +type float float64 -const d3roundValues = true // only set to true when using 32-bit floats +const d3roundValues = false // only set to true when using 32-bit floats func d3fmin(a, b float) float { if a < b { diff --git a/vendor/github.com/tidwall/tinyqueue/LICENSE b/vendor/github.com/tidwall/tinyqueue/LICENSE new file mode 100644 index 00000000..2b7cd9d3 --- /dev/null +++ b/vendor/github.com/tidwall/tinyqueue/LICENSE @@ -0,0 +1,15 @@ +ISC License + +Copyright (c) 2017, Vladimir Agafonkin + +Permission to use, copy, modify, and/or distribute this software for any purpose +with or without fee is hereby granted, provided that the above copyright notice +and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS +OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER +TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF +THIS SOFTWARE. diff --git a/vendor/github.com/tidwall/tinyqueue/README.md b/vendor/github.com/tidwall/tinyqueue/README.md new file mode 100644 index 00000000..f4edc916 --- /dev/null +++ b/vendor/github.com/tidwall/tinyqueue/README.md @@ -0,0 +1,7 @@ +# tinyqueue +GoDoc + +tinyqueue is a Go package for binary heap priority queues. +Ported from the [tinyqueue](https://github.com/mourner/tinyqueue) Javascript library. + + diff --git a/vendor/github.com/tidwall/tinyqueue/tinyqueue.go b/vendor/github.com/tidwall/tinyqueue/tinyqueue.go new file mode 100644 index 00000000..4a06258a --- /dev/null +++ b/vendor/github.com/tidwall/tinyqueue/tinyqueue.go @@ -0,0 +1,86 @@ +package tinyqueue + +type Queue struct { + length int + data []Item +} + +type Item interface { + Less(Item) bool +} + +func New(data []Item) *Queue { + q := &Queue{} + q.data = data + q.length = len(data) + if q.length > 0 { + i := q.length >> 1 + for ; i >= 0; i-- { + q.down(i) + } + } + return q +} + +func (q *Queue) Push(item Item) { + q.data = append(q.data, item) + q.length++ + q.up(q.length - 1) +} +func (q *Queue) Pop() Item { + if q.length == 0 { + return nil + } + top := q.data[0] + q.length-- + if q.length > 0 { + q.data[0] = q.data[q.length] + q.down(0) + } + q.data = q.data[:len(q.data)-1] + return top +} +func (q *Queue) Peek() Item { + if q.length == 0 { + return nil + } + return q.data[0] +} +func (q *Queue) Len() int { + return q.length +} +func (q *Queue) down(pos int) { + data := q.data + halfLength := q.length >> 1 + item := data[pos] + for pos < halfLength { + left := (pos << 1) + 1 + right := left + 1 + best := data[left] + if right < q.length && data[right].Less(best) { + left = right + best = data[right] + } + if !best.Less(item) { + break + } + data[pos] = best + pos = left + } + data[pos] = item +} + +func (q *Queue) up(pos int) { + data := q.data + item := data[pos] + for pos > 0 { + parent := (pos - 1) >> 1 + current := data[parent] + if !item.Less(current) { + break + } + data[pos] = current + pos = parent + } + data[pos] = item +} diff --git a/vendor/github.com/tidwall/tinyqueue/tinyqueue_test.go b/vendor/github.com/tidwall/tinyqueue/tinyqueue_test.go new file mode 100644 index 00000000..9d34df4f --- /dev/null +++ b/vendor/github.com/tidwall/tinyqueue/tinyqueue_test.go @@ -0,0 +1,65 @@ +package tinyqueue + +import ( + "math/rand" + "sort" + "testing" + "time" + + "github.com/json-iterator/go/assert" +) + +type floatValue float64 + +func (a floatValue) Less(b Item) bool { + return a < b.(floatValue) +} + +var data, sorted = func() ([]Item, []Item) { + rand.Seed(time.Now().UnixNano()) + var data []Item + for i := 0; i < 100; i++ { + data = append(data, floatValue(rand.Float64()*100)) + } + sorted := make([]Item, len(data)) + copy(sorted, data) + sort.Slice(sorted, func(i, j int) bool { + return sorted[i].Less(sorted[j]) + }) + return data, sorted +}() + +func TestMaintainsPriorityQueue(t *testing.T) { + q := New(nil) + for i := 0; i < len(data); i++ { + q.Push(data[i]) + } + assert.Equal(t, q.Peek(), sorted[0]) + var result []Item + for q.length > 0 { + result = append(result, q.Pop()) + } + assert.Equal(t, result, sorted) +} + +func TestAcceptsDataInConstructor(t *testing.T) { + q := New(data) + var result []Item + for q.length > 0 { + result = append(result, q.Pop()) + } + assert.Equal(t, result, sorted) +} +func TestHandlesEdgeCasesWithFewElements(t *testing.T) { + q := New(nil) + q.Push(floatValue(2)) + q.Push(floatValue(1)) + q.Pop() + q.Pop() + q.Pop() + q.Push(floatValue(2)) + q.Push(floatValue(1)) + assert.Equal(t, float64(q.Pop().(floatValue)), 1.0) + assert.Equal(t, float64(q.Pop().(floatValue)), 2.0) + assert.Equal(t, q.Pop(), nil) +}