From ba9a7679883d3d2fc861f58890618449036f319b Mon Sep 17 00:00:00 2001 From: tidwall Date: Mon, 19 Sep 2022 17:51:14 -0700 Subject: [PATCH] Changed the collection rectangle dimension type Previously used float64s, now using float32s. Saving about 15% on rectangle memory. Uses the Roundoff trick from Sqlite. --- go.mod | 2 +- go.sum | 4 +- internal/collection/collection.go | 116 ++++++++++++++++++++++-------- tests/keys_search_test.go | 2 +- 4 files changed, 92 insertions(+), 32 deletions(-) diff --git a/go.mod b/go.mod index cfc3dd8f..ed1846cd 100644 --- a/go.mod +++ b/go.mod @@ -27,7 +27,7 @@ require ( github.com/tidwall/redbench v0.1.0 github.com/tidwall/redcon v1.4.4 github.com/tidwall/resp v0.1.1 - github.com/tidwall/rtree v1.8.1 + github.com/tidwall/rtree v1.9.1 github.com/tidwall/sjson v1.2.4 github.com/xdg/scram v1.0.5 github.com/yuin/gopher-lua v0.0.0-20200816102855-ee81675732da diff --git a/go.sum b/go.sum index c8983911..f4ce2ce0 100644 --- a/go.sum +++ b/go.sum @@ -383,8 +383,8 @@ github.com/tidwall/resp v0.1.1/go.mod h1:3/FrruOBAxPTPtundW0VXgmsQ4ZBA0Aw714lVYg github.com/tidwall/rtred v0.1.2 h1:exmoQtOLvDoO8ud++6LwVsAMTu0KPzLTUrMln8u1yu8= github.com/tidwall/rtred v0.1.2/go.mod h1:hd69WNXQ5RP9vHd7dqekAz+RIdtfBogmglkZSRxCHFQ= github.com/tidwall/rtree v1.3.1/go.mod h1:S+JSsqPTI8LfWA4xHBo5eXzie8WJLVFeppAutSegl6M= -github.com/tidwall/rtree v1.8.1 h1:Hv0gvvznkDI5YwBkJp9pYh8ZEU1L2A9puLwwGPcQ9j4= -github.com/tidwall/rtree v1.8.1/go.mod h1:iDJQ9NBRtbfKkzZu02za+mIlaP+bjYPnunbSNidpbCQ= +github.com/tidwall/rtree v1.9.1 h1:UIPtvE09nLKZRnMNEwRZxu9jRAkzROAZDR+NPS/9IRs= +github.com/tidwall/rtree v1.9.1/go.mod h1:iDJQ9NBRtbfKkzZu02za+mIlaP+bjYPnunbSNidpbCQ= github.com/tidwall/sjson v1.2.4 h1:cuiLzLnaMeBhRmEv00Lpk3tkYrcxpmbU81tAY4Dw0tc= github.com/tidwall/sjson v1.2.4/go.mod h1:098SZ494YoMWPmMO6ct4dcFnqxwj9r/gF0Etp19pSNM= github.com/tidwall/tinyqueue v0.1.1 h1:SpNEvEggbpyN5DIReaJ2/1ndroY8iyEGxPYxoSaymYE= diff --git a/internal/collection/collection.go b/internal/collection/collection.go index 8eff93e2..0b7ff539 100644 --- a/internal/collection/collection.go +++ b/internal/collection/collection.go @@ -56,12 +56,19 @@ func byExpires(a, b *itemT) bool { return byID(a, b) } +func (item *itemT) Rect() geometry.Rect { + if item.obj != nil { + return item.obj.Rect() + } + return geometry.Rect{} +} + // Collection represents a collection of geojson objects. type Collection struct { - items *btree.BTreeG[*itemT] // items sorted by id - spatial *rtree.RTreeG[*itemT] // items geospatially indexed - values *btree.BTreeG[*itemT] // items sorted by value+id - expires *btree.BTreeG[*itemT] // items sorted by ex+id + items *btree.BTreeG[*itemT] // items sorted by id + spatial *rtree.RTreeGN[float32, *itemT] // items geospatially indexed + values *btree.BTreeG[*itemT] // items sorted by value+id + expires *btree.BTreeG[*itemT] // items sorted by ex+id weight int points int objects int // geometry count @@ -76,7 +83,7 @@ func New() *Collection { items: btree.NewBTreeGOptions(byID, optsNoLock), values: btree.NewBTreeGOptions(byValue, optsNoLock), expires: btree.NewBTreeGOptions(byExpires, optsNoLock), - spatial: &rtree.RTreeG[*itemT]{}, + spatial: &rtree.RTreeGN[float32, *itemT]{}, } return col } @@ -103,11 +110,15 @@ func (c *Collection) TotalWeight() int { // Bounds returns the bounds of all the items in the collection. func (c *Collection) Bounds() (minX, minY, maxX, maxY float64) { - min, max := c.spatial.Bounds() - if len(min) >= 2 && len(max) >= 2 { - return min[0], min[1], max[0], max[1] + _, _, left := c.spatial.LeftMost() + _, _, bottom := c.spatial.BottomMost() + _, _, right := c.spatial.RightMost() + _, _, top := c.spatial.TopMost() + if left == nil { + return } - return + return left.Rect().Min.X, bottom.Rect().Min.Y, + right.Rect().Max.X, top.Rect().Max.Y } func objIsSpatial(obj geojson.Object) bool { @@ -129,24 +140,57 @@ func (c *Collection) objWeight(item *itemT) int { func (c *Collection) indexDelete(item *itemT) { if !item.obj.Empty() { - rect := item.obj.Rect() - c.spatial.Delete( - [2]float64{rect.Min.X, rect.Min.Y}, - [2]float64{rect.Max.X, rect.Max.Y}, - item) + c.spatial.Delete(rtreeItem(item)) } } func (c *Collection) indexInsert(item *itemT) { if !item.obj.Empty() { - rect := item.obj.Rect() - c.spatial.Insert( - [2]float64{rect.Min.X, rect.Min.Y}, - [2]float64{rect.Max.X, rect.Max.Y}, - item) + c.spatial.Insert(rtreeItem(item)) } } +const dRNDTOWARDS = (1.0 - 1.0/8388608.0) /* Round towards zero */ +const dRNDAWAY = (1.0 + 1.0/8388608.0) /* Round away from zero */ + +func rtreeValueDown(d float64) float32 { + f := float32(d) + if float64(f) > d { + if d < 0 { + f = float32(d * dRNDAWAY) + } else { + f = float32(d * dRNDTOWARDS) + } + } + return f +} +func rtreeValueUp(d float64) float32 { + f := float32(d) + if float64(f) < d { + if d < 0 { + f = float32(d * dRNDTOWARDS) + } else { + f = float32(d * dRNDAWAY) + } + } + return f +} + +func rtreeItem(item *itemT) (min, max [2]float32, data *itemT) { + min, max = rtreeRect(item.Rect()) + return min, max, item +} + +func rtreeRect(rect geometry.Rect) (min, max [2]float32) { + return [2]float32{ + rtreeValueDown(rect.Min.X), + rtreeValueDown(rect.Min.Y), + }, [2]float32{ + rtreeValueUp(rect.Max.X), + rtreeValueUp(rect.Max.Y), + } +} + // Set adds or replaces an object in the collection and returns the fields // array. func (c *Collection) Set(id string, obj geojson.Object, fields field.List, ex int64) ( @@ -429,10 +473,10 @@ func (c *Collection) geoSearch( iter func(id string, obj geojson.Object, fields field.List) bool, ) bool { alive := true + min, max := rtreeRect(rect) c.spatial.Search( - [2]float64{rect.Min.X, rect.Min.Y}, - [2]float64{rect.Max.X, rect.Max.Y}, - func(_, _ [2]float64, item *itemT) bool { + min, max, + func(_, _ [2]float32, item *itemT) bool { alive = iter(item.id, item.obj, item.fields) return alive }, @@ -618,10 +662,19 @@ func (c *Collection) Nearby( minLat, minLon, maxLat, maxLon := geo.RectFromCenter(center.Y, center.X, meters) var exists bool + min, max := rtreeRect(geometry.Rect{ + Min: geometry.Point{ + X: minLon, + Y: minLat, + }, + Max: geometry.Point{ + X: maxLon, + Y: maxLat, + }, + }) c.spatial.Search( - [2]float64{minLon, minLat}, - [2]float64{maxLon, maxLat}, - func(_, _ [2]float64, item *itemT) bool { + min, max, + func(_, _ [2]float32, item *itemT) bool { exists = true return false }, @@ -641,15 +694,22 @@ func (c *Collection) Nearby( offset = cursor.Offset() cursor.Step(offset) } + distFn := geodeticDistAlgo[*itemT]([2]float64{center.X, center.Y}) c.spatial.Nearby( - geodeticDistAlgo[*itemT]([2]float64{center.X, center.Y}), - func(_, _ [2]float64, item *itemT, dist float64) bool { + func(min, max [2]float32, data *itemT, item bool) float32 { + return float32(distFn( + [2]float64{float64(min[0]), float64(min[1])}, + [2]float64{float64(max[0]), float64(max[1])}, + data, item, + )) + }, + func(_, _ [2]float32, item *itemT, dist float32) bool { count++ if count <= offset { return true } nextStep(count, cursor, deadline) - alive = iter(item.id, item.obj, item.fields, dist) + alive = iter(item.id, item.obj, item.fields, float64(dist)) return alive }, ) diff --git a/tests/keys_search_test.go b/tests/keys_search_test.go index e68b9236..b368a59d 100644 --- a/tests/keys_search_test.go +++ b/tests/keys_search_test.go @@ -46,7 +46,7 @@ func keys_KNN_basic_test(mc *mockServer) error { {"NEARBY", "mykey", "LIMIT", 10, "POINTS", "POINT", 20, 20}, { "[0 [[2 [19 19]] [3 [12 19]] [5 [33 21]] [1 [5 5]] [4 [-5 5]] [6 [52 13]]]]"}, {"NEARBY", "mykey", "LIMIT", 10, "IDS", "POINT", 20, 20, 4000000}, {"[0 [2 3 5 1 4 6]]"}, - {"NEARBY", "mykey", "LIMIT", 10, "DISTANCE", "IDS", "POINT", 20, 20, 1500000}, {"[0 [[2 152808.67164037024] [3 895945.1409106688] [5 1448929.5916252395]]]"}, + {"NEARBY", "mykey", "LIMIT", 10, "DISTANCE", "IDS", "POINT", 20, 20, 1500000}, {"[0 [[2 152808.671875] [3 895945.125] [5 1448929.625]]]"}, {"NEARBY", "mykey", "LIMIT", 10, "DISTANCE", "POINT", 52, 13, 100}, {`[0 [[6 {"type":"Point","coordinates":[13,52]} 0]]]`}, {"NEARBY", "mykey", "LIMIT", 10, "POINT", 52.1, 13.1, 100000}, {`[0 [[6 {"type":"Point","coordinates":[13,52]}]]]`}, {"OUTPUT", "json"}, {func(res string) bool { return gjson.Get(res, "ok").Bool() }},