From bb2bb451b2e460edc6d08f542fa1bab3bff7da78 Mon Sep 17 00:00:00 2001 From: Josh Baker Date: Mon, 24 Oct 2016 15:35:47 -0700 Subject: [PATCH] memory optimizations increased b-tree degrees from 16 to 48 increased r-tree degress from 8 to 15 changed r-tree from float64 to float32 --- controller/collection/collection.go | 9 ++- controller/stats.go | 7 +- index/rtree/rtree.go | 3 +- index/rtree/rtreed.go | 108 +++++++++++++++++++--------- 4 files changed, 88 insertions(+), 39 deletions(-) diff --git a/controller/collection/collection.go b/controller/collection/collection.go index b7018532..0b6235cc 100644 --- a/controller/collection/collection.go +++ b/controller/collection/collection.go @@ -64,8 +64,8 @@ var counter uint64 func New() *Collection { col := &Collection{ index: index.New(), - items: btree.New(16, idOrdered), - values: btree.New(16, valueOrdered), + items: btree.New(48, idOrdered), + values: btree.New(48, valueOrdered), fieldMap: make(map[string]int), } return col @@ -76,6 +76,11 @@ func (c *Collection) Count() int { return c.objects + c.nobjects } +// StringCount returns the number of string values. +func (c *Collection) StringCount() int { + return c.nobjects +} + // PointCount returns the number of points (lat/lon coordinates) in collection. func (c *Collection) PointCount() int { return c.points diff --git a/controller/stats.go b/controller/stats.go index 4de8b85a..8ed7f71a 100644 --- a/controller/stats.go +++ b/controller/stats.go @@ -33,10 +33,10 @@ func (c *Controller) cmdStats(msg *server.Message) (res string, err error) { col := c.getCol(key) if col != nil { m := make(map[string]interface{}) - points := col.PointCount() - m["num_points"] = points + m["num_points"] = col.PointCount() m["in_memory_size"] = col.TotalWeight() m["num_objects"] = col.Count() + m["num_strings"] = col.StringCount() switch msg.OutputType { case server.JSON: ms = append(ms, m) @@ -92,14 +92,17 @@ func (c *Controller) cmdServer(msg *server.Message) (res string, err error) { m["in_memory_size"] = sz points := 0 objects := 0 + strings := 0 c.cols.Ascend(func(item btree.Item) bool { col := item.(*collectionT).Collection points += col.PointCount() objects += col.Count() + strings += col.StringCount() return true }) m["num_points"] = points m["num_objects"] = objects + m["num_strings"] = strings var mem runtime.MemStats runtime.ReadMemStats(&mem) avgsz := 0 diff --git a/index/rtree/rtree.go b/index/rtree/rtree.go index 97e5ae0e..d3a7e45d 100644 --- a/index/rtree/rtree.go +++ b/index/rtree/rtree.go @@ -67,6 +67,7 @@ func (tr *RTree) Bounds() (minX, minY, minZ, maxX, maxY, maxZ float64) { } } } - minX, minY, minZ, maxX, maxY, maxZ = rect.min[0], rect.min[1], rect.min[2], rect.max[0], rect.max[1], rect.max[2] + minX, minY, minZ = float64(rect.min[0]), float64(rect.min[1]), float64(rect.min[2]) + maxX, maxY, maxZ = float64(rect.max[0]), float64(rect.max[1]), float64(rect.max[2]) return } diff --git a/index/rtree/rtreed.go b/index/rtree/rtreed.go index fb947c86..218cec49 100644 --- a/index/rtree/rtreed.go +++ b/index/rtree/rtreed.go @@ -2,13 +2,17 @@ package rtree import "math" -func d3fmin(a, b float64) float64 { +type float float32 + +const d3roundValues = true // only set to true when using 32-bit floats + +func d3fmin(a, b float) float { if a < b { return a } return b } -func d3fmax(a, b float64) float64 { +func d3fmax(a, b float) float { if a > b { return a } @@ -17,7 +21,7 @@ func d3fmax(a, b float64) float64 { const ( d3numDims = 3 - d3maxNodes = 8 + d3maxNodes = 13 d3minNodes = d3maxNodes / 2 d3useSphericalVolume = true // Better split classification, may be slower on some systems ) @@ -38,8 +42,8 @@ type d3RTree struct { /// Minimal bounding rectangle (n-dimensional) type d3rectT struct { - min [d3numDims]float64 ///< Min dimensions of bounding box - max [d3numDims]float64 ///< Max dimensions of bounding box + min [d3numDims]float ///< Min dimensions of bounding box + max [d3numDims]float ///< Max dimensions of bounding box } /// May be data or may be another subtree @@ -65,6 +69,41 @@ func (node *d3nodeT) isLeaf() bool { return (node.level == 0) // A leaf, contains data } +// Rounding constants for float32 -> float64 conversion. +const d3RNDTOWARDS = (1.0 - 1.0/8388608.0) // Round towards zero +const d3RNDAWAY = (1.0 + 1.0/8388608.0) // Round away from zero + +// Convert an sqlite3_value into an RtreeValue (presumably a float) +// while taking care to round toward negative or positive, respectively. +func d3rtreeValueDown(d float64) float { + if !d3roundValues { + return float(d) + } + f := float(d) + if float64(f) > d { + if d < 0 { + f = float(d * d3RNDAWAY) + } else { + f = float(d * d3RNDTOWARDS) + } + } + return f +} +func d3rtreeValueUp(d float64) float { + if !d3roundValues { + return float(d) + } + f := float(d) + if float64(f) < d { + if d < 0 { + f = float(d * d3RNDTOWARDS) + } else { + f = float(d * d3RNDAWAY) + } + } + return f +} + /// A link list of nodes for reinsertion after a delete operation type d3listNodeT struct { next *d3listNodeT ///< Next in list @@ -80,12 +119,12 @@ type d3partitionVarsT struct { minFill int count [2]int cover [2]d3rectT - area [2]float64 + area [2]float branchBuf [d3maxNodes + 1]d3branchT branchCount int coverSplit d3rectT - coverSplitArea float64 + coverSplitArea float } func d3New() *d3RTree { @@ -104,8 +143,8 @@ func (tr *d3RTree) Insert(min, max [d3numDims]float64, dataId interface{}) { var branch d3branchT branch.data = dataId for axis := 0; axis < d3numDims; axis++ { - branch.rect.min[axis] = min[axis] - branch.rect.max[axis] = max[axis] + branch.rect.min[axis] = d3rtreeValueDown(min[axis]) + branch.rect.max[axis] = d3rtreeValueUp(max[axis]) } d3insertRect(&branch, &tr.root, 0) } @@ -117,8 +156,8 @@ func (tr *d3RTree) Insert(min, max [d3numDims]float64, dataId interface{}) { func (tr *d3RTree) Remove(min, max [d3numDims]float64, dataId interface{}) { var rect d3rectT for axis := 0; axis < d3numDims; axis++ { - rect.min[axis] = min[axis] - rect.max[axis] = max[axis] + rect.min[axis] = d3rtreeValueDown(min[axis]) + rect.max[axis] = d3rtreeValueUp(max[axis]) } d3removeRect(&rect, dataId, &tr.root) } @@ -133,8 +172,8 @@ func (tr *d3RTree) Remove(min, max [d3numDims]float64, dataId interface{}) { func (tr *d3RTree) Search(min, max [d3numDims]float64, resultCallback func(data interface{}) bool) int { var rect d3rectT for axis := 0; axis < d3numDims; axis++ { - rect.min[axis] = min[axis] - rect.max[axis] = max[axis] + rect.min[axis] = d3rtreeValueDown(min[axis]) + rect.max[axis] = d3rtreeValueUp(max[axis]) } foundCount, _ := d3search(tr.root, rect, 0, resultCallback) return foundCount @@ -287,10 +326,10 @@ func d3disconnectBranch(node *d3nodeT, index int) { // the best resolution when searching. func d3pickBranch(rect *d3rectT, node *d3nodeT) int { var firstTime bool = true - var increase float64 - var bestIncr float64 = -1 - var area float64 - var bestArea float64 + var increase float + var bestIncr float = -1 + var area float + var bestArea float var best int var tempRect d3rectT @@ -350,8 +389,8 @@ func d3splitNode(node *d3nodeT, branch *d3branchT, newNode **d3nodeT) { } // Calculate the n-dimensional volume of a rectangle -func d3rectVolume(rect *d3rectT) float64 { - var volume float64 = 1 +func d3rectVolume(rect *d3rectT) float { + var volume float = 1 for index := 0; index < d3numDims; index++ { volume *= rect.max[index] - rect.min[index] } @@ -364,30 +403,31 @@ func d3rectSphericalVolume(rect *d3rectT) float64 { var radius float64 for index := 0; index < d3numDims; index++ { - halfExtent := (rect.max[index] - rect.min[index]) * 0.5 + halfExtent := float64(rect.max[index]-rect.min[index]) * 0.5 sumOfSquares += halfExtent * halfExtent } radius = math.Sqrt(sumOfSquares) // Pow maybe slow, so test for common dims just use x*x, x*x*x. - if d3numDims == 5 { - return (radius * radius * radius * radius * radius * d3unitSphereVolume) - } else if d3numDims == 4 { - return (radius * radius * radius * radius * d3unitSphereVolume) - } else if d3numDims == 3 { - return (radius * radius * radius * d3unitSphereVolume) - } else if d3numDims == 2 { - return (radius * radius * d3unitSphereVolume) - } else { + switch d3numDims { + default: return (math.Pow(radius, d3numDims) * d3unitSphereVolume) + case 2: + return (radius * radius * d3unitSphereVolume) + case 3: + return (radius * radius * radius * d3unitSphereVolume) + case 4: + return (radius * radius * radius * radius * d3unitSphereVolume) + case 5: + return (radius * radius * radius * radius * radius * d3unitSphereVolume) } } // Use one of the methods to calculate retangle volume -func d3calcRectVolume(rect *d3rectT) float64 { +func d3calcRectVolume(rect *d3rectT) float { if d3useSphericalVolume { - return d3rectSphericalVolume(rect) // Slower but helps certain merge cases + return float(d3rectSphericalVolume(rect)) // Slower but helps certain merge cases } else { // RTREE_USE_SPHERICAL_VOLUME return d3rectVolume(rect) // Faster but can cause poor merges } // RTREE_USE_SPHERICAL_VOLUME @@ -422,7 +462,7 @@ func d3getBranches(node *d3nodeT, branch *d3branchT, parVars *d3partitionVarsT) // fill requirement) then other group gets the rest. // These last are the ones that can go in either group most easily. func d3choosePartition(parVars *d3partitionVarsT, minFill int) { - var biggestDiff float64 + var biggestDiff float var group, chosen, betterGroup int d3initParVars(parVars, parVars.branchCount, minFill) @@ -501,8 +541,8 @@ func d3initParVars(parVars *d3partitionVarsT, maxRects, minFill int) { func d3pickSeeds(parVars *d3partitionVarsT) { var seed0, seed1 int - var worst, waste float64 - var area [d3maxNodes + 1]float64 + var worst, waste float + var area [d3maxNodes + 1]float for index := 0; index < parVars.total; index++ { area[index] = d3calcRectVolume(&parVars.branchBuf[index].rect)