mirror
/
tile38
mirror of https://github.com/tidwall/tile38.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Updated data structures to use Go generics. 

Prior to this commit all objects in the Collection data structures
were boxed in an Go interface{} which adds an extra 8 bytes per
object and requires assertion to unbox.

Go 1.18, released early 2022, introduced generics, which allows
for storing the objects without boxing. This provides a extra
boost in performance and lower in-memory footprint.
This commit is contained in:
tidwall 2022-09-12 09:12:51 -07:00
parent 498bbe23ff
commit cbfb271541
5 changed files with 67 additions and 107 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
go.mod
View File

@ -18,7 +18,6 @@ require (
github.com/streadway/amqp v1.0.0
github.com/tidwall/btree v1.4.2
github.com/tidwall/buntdb v1.2.9
github.com/tidwall/geoindex v1.6.2
github.com/tidwall/geojson v1.3.4
github.com/tidwall/gjson v1.12.1
github.com/tidwall/match v1.1.1
@ -26,7 +25,7 @@ require (
github.com/tidwall/redbench v0.1.0
github.com/tidwall/redcon v1.4.4
github.com/tidwall/resp v0.1.0
github.com/tidwall/rtree v1.7.1
github.com/tidwall/rtree v1.8.0
github.com/tidwall/sjson v1.2.4
github.com/xdg/scram v1.0.5
github.com/yuin/gopher-lua v0.0.0-20200816102855-ee81675732da
@ -88,9 +87,8 @@ require (
github.com/prometheus/common v0.32.1 // indirect
github.com/prometheus/procfs v0.7.3 // indirect
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
github.com/tidwall/cities v0.1.0 // indirect
github.com/tidwall/geoindex v1.7.0 // indirect
github.com/tidwall/grect v0.1.4 // indirect
github.com/tidwall/lotsa v1.0.2 // indirect
github.com/tidwall/rtred v0.1.2 // indirect
github.com/tidwall/tinyqueue v0.1.1 // indirect
github.com/xdg/stringprep v1.0.3 // indirect

8
go.sum
View File

@ -357,8 +357,8 @@ github.com/tidwall/buntdb v1.2.9/go.mod h1:IwyGSvvDg6hnKSIhtdZ0AqhCZGH8ukdtCAzaP
github.com/tidwall/cities v0.1.0 h1:CVNkmMf7NEC9Bvokf5GoSsArHCKRMTgLuubRTHnH0mE=
github.com/tidwall/cities v0.1.0/go.mod h1:lV/HDp2gCcRcHJWqgt6Di54GiDrTZwh1aG2ZUPNbqa4=
github.com/tidwall/geoindex v1.4.4/go.mod h1:rvVVNEFfkJVWGUdEfU8QaoOg/9zFX0h9ofWzA60mz1I=
github.com/tidwall/geoindex v1.6.2 h1:cWbqC9HFXMxc2p6KMWbs9VG6/gnrfC53EIPQEMcXO1g=
github.com/tidwall/geoindex v1.6.2/go.mod h1:rvVVNEFfkJVWGUdEfU8QaoOg/9zFX0h9ofWzA60mz1I=
github.com/tidwall/geoindex v1.7.0 h1:jtk41sfgwIt8MEDyC3xyKSj75iXXf6rjReJGDNPtR5o=
github.com/tidwall/geoindex v1.7.0/go.mod h1:rvVVNEFfkJVWGUdEfU8QaoOg/9zFX0h9ofWzA60mz1I=
github.com/tidwall/geojson v1.3.4 h1:mHB2yGK7HPgf4vFkLdPeIzguFpqkmCT2yTgGhXbrqBo=
github.com/tidwall/geojson v1.3.4/go.mod h1:1cn3UWfSYCJOq53NZoQ9rirdw89+DM0vw+ZOAVvuReg=
github.com/tidwall/gjson v1.12.1 h1:ikuZsLdhr8Ws0IdROXUS1Gi4v9Z4pGqpX/CvJkxvfpo=
@ -380,8 +380,8 @@ github.com/tidwall/resp v0.1.0/go.mod h1:18xEj855iMY2bK6tNF2A4x+nZy5gWO1iO7OOl3j
github.com/tidwall/rtred v0.1.2 h1:exmoQtOLvDoO8ud++6LwVsAMTu0KPzLTUrMln8u1yu8=
github.com/tidwall/rtred v0.1.2/go.mod h1:hd69WNXQ5RP9vHd7dqekAz+RIdtfBogmglkZSRxCHFQ=
github.com/tidwall/rtree v1.3.1/go.mod h1:S+JSsqPTI8LfWA4xHBo5eXzie8WJLVFeppAutSegl6M=
github.com/tidwall/rtree v1.7.1 h1:rv3Q8RBKH2HbJ6DsqpfrXfV9l+dCT1jupTpDiceHN3I=
github.com/tidwall/rtree v1.7.1/go.mod h1:39+jGCj9hYqhflezmsTBOlysIk09ytm+8EQsC/E/2X0=
github.com/tidwall/rtree v1.8.0 h1:nYVLh9UKJrd4CZCNawD3WbHNxmI9LYR4j3E2hqO3tjQ=
github.com/tidwall/rtree v1.8.0/go.mod h1:iDJQ9NBRtbfKkzZu02za+mIlaP+bjYPnunbSNidpbCQ=
github.com/tidwall/sjson v1.2.4 h1:cuiLzLnaMeBhRmEv00Lpk3tkYrcxpmbU81tAY4Dw0tc=
github.com/tidwall/sjson v1.2.4/go.mod h1:098SZ494YoMWPmMO6ct4dcFnqxwj9r/gF0Etp19pSNM=
github.com/tidwall/tinyqueue v0.1.1 h1:SpNEvEggbpyN5DIReaJ2/1ndroY8iyEGxPYxoSaymYE=

138
internal/collection/collection.go
View File

@ -4,7 +4,6 @@ import (
"runtime"
"github.com/tidwall/btree"
"github.com/tidwall/geoindex"
"github.com/tidwall/geojson"
"github.com/tidwall/geojson/geo"
"github.com/tidwall/geojson/geometry"
@ -28,13 +27,13 @@ type itemT struct {
fieldValuesSlot fieldValuesSlot
}
func byID(a, b interface{}) bool {
return a.(*itemT).id < b.(*itemT).id
func byID(a, b *itemT) bool {
return a.id < b.id
}
func byValue(a, b interface{}) bool {
value1 := a.(*itemT).obj.String()
value2 := b.(*itemT).obj.String()
func byValue(a, b *itemT) bool {
value1 := a.obj.String()
value2 := b.obj.String()
if value1 < value2 {
return true
}
@ -45,13 +44,11 @@ func byValue(a, b interface{}) bool {
return byID(a, b)
}
func byExpires(a, b interface{}) bool {
item1 := a.(*itemT)
item2 := b.(*itemT)
if item1.expires < item2.expires {
func byExpires(a, b *itemT) bool {
if a.expires < b.expires {
return true
}
if item1.expires > item2.expires {
if a.expires > b.expires {
return false
}
// the values match so we'll compare IDs, which are always unique.
@ -60,10 +57,10 @@ func byExpires(a, b interface{}) bool {
// Collection represents a collection of geojson objects.
type Collection struct {
items *btree.BTree // items sorted by id
index *geoindex.Index // items geospatially indexed
values *btree.BTree // items sorted by value+id
expires *btree.BTree // items sorted by ex+id
items *btree.BTreeG[*itemT] // items sorted by id
spatial *rtree.RTreeG[*itemT] // items geospatially indexed
values *btree.BTreeG[*itemT] // items sorted by value+id
expires *btree.BTreeG[*itemT] // items sorted by ex+id
fieldMap map[string]int
fieldArr []string
fieldValues *fieldValues
@ -73,13 +70,15 @@ type Collection struct {
nobjects int // non-geometry count
}
var optsNoLock = btree.Options{NoLocks: true}
// New creates an empty collection
func New() *Collection {
col := &Collection{
items: btree.NewNonConcurrent(byID),
index: geoindex.Wrap(&rtree.RTree{}),
values: btree.NewNonConcurrent(byValue),
expires: btree.NewNonConcurrent(byExpires),
items: btree.NewBTreeGOptions(byID, optsNoLock),
values: btree.NewBTreeGOptions(byValue, optsNoLock),
expires: btree.NewBTreeGOptions(byExpires, optsNoLock),
spatial: &rtree.RTreeG[*itemT]{},
fieldMap: make(map[string]int),
fieldArr: make([]string, 0),
fieldValues: &fieldValues{},
@ -109,7 +108,7 @@ func (c *Collection) TotalWeight() int {
// Bounds returns the bounds of all the items in the collection.
func (c *Collection) Bounds() (minX, minY, maxX, maxY float64) {
min, max := c.index.Bounds()
min, max := c.spatial.Bounds()
if len(min) >= 2 && len(max) >= 2 {
return min[0], min[1], max[0], max[1]
}
@ -134,7 +133,7 @@ func (c *Collection) objWeight(item *itemT) int {
func (c *Collection) indexDelete(item *itemT) {
if !item.obj.Empty() {
rect := item.obj.Rect()
c.index.Delete(
c.spatial.Delete(
[2]float64{rect.Min.X, rect.Min.Y},
[2]float64{rect.Max.X, rect.Max.Y},
item)
@ -144,7 +143,7 @@ func (c *Collection) indexDelete(item *itemT) {
func (c *Collection) indexInsert(item *itemT) {
if !item.obj.Empty() {
rect := item.obj.Rect()
c.index.Insert(
c.spatial.Insert(
[2]float64{rect.Min.X, rect.Min.Y},
[2]float64{rect.Max.X, rect.Max.Y},
item)
@ -164,9 +163,8 @@ func (c *Collection) Set(
newItem := &itemT{id: id, obj: obj, fieldValuesSlot: nilValuesSlot, expires: ex}
// add the new item to main btree and remove the old one if needed
oldItem := c.items.Set(newItem)
if oldItem != nil {
oldItem := oldItem.(*itemT)
oldItem, ok := c.items.Set(newItem)
if ok {
// the old item was removed, now let's remove it from the rtree/btree.
if objIsSpatial(oldItem.obj) {
c.indexDelete(oldItem)
@ -232,11 +230,10 @@ func (c *Collection) Set(
func (c *Collection) Delete(id string) (
obj geojson.Object, fields []float64, ok bool,
) {
v := c.items.Delete(&itemT{id: id})
if v == nil {
oldItem, ok := c.items.Delete(&itemT{id: id})
if !ok {
return nil, nil, false
}
oldItem := v.(*itemT)
if objIsSpatial(oldItem.obj) {
if !oldItem.obj.Empty() {
c.indexDelete(oldItem)
@ -263,20 +260,18 @@ func (c *Collection) Delete(id string) (
func (c *Collection) Get(id string) (
obj geojson.Object, fields []float64, ex int64, ok bool,
) {
itemV := c.items.Get(&itemT{id: id})
if itemV == nil {
item, ok := c.items.Get(&itemT{id: id})
if !ok {
return nil, nil, 0, false
}
item := itemV.(*itemT)
return item.obj, c.fieldValues.get(item.fieldValuesSlot), item.expires, true
}
func (c *Collection) SetExpires(id string, ex int64) bool {
v := c.items.Get(&itemT{id: id})
if v == nil {
item, ok := c.items.Get(&itemT{id: id})
if !ok {
return false
}
item := v.(*itemT)
if item.expires != 0 {
c.expires.Delete(item)
}
@ -292,11 +287,10 @@ func (c *Collection) SetExpires(id string, ex int64) bool {
func (c *Collection) SetField(id, field string, value float64) (
obj geojson.Object, fields []float64, updated bool, ok bool,
) {
itemV := c.items.Get(&itemT{id: id})
if itemV == nil {
item, ok := c.items.Get(&itemT{id: id})
if !ok {
return nil, nil, false, false
}
item := itemV.(*itemT)
_, updateCount, weightDelta := c.setFieldValues(item, []string{field}, []float64{value})
c.weight += weightDelta
return item.obj, c.fieldValues.get(item.fieldValuesSlot), updateCount > 0, true
@ -306,11 +300,10 @@ func (c *Collection) SetField(id, field string, value float64) (
func (c *Collection) SetFields(
id string, inFields []string, inValues []float64,
) (obj geojson.Object, fields []float64, updatedCount int, ok bool) {
itemV := c.items.Get(&itemT{id: id})
if itemV == nil {
item, ok := c.items.Get(&itemT{id: id})
if !ok {
return nil, nil, 0, false
}
item := itemV.(*itemT)
newFieldValues, updateCount, weightDelta := c.setFieldValues(item, inFields, inValues)
c.weight += weightDelta
return item.obj, newFieldValues, updateCount, true
@ -394,20 +387,19 @@ func (c *Collection) Scan(
offset = cursor.Offset()
cursor.Step(offset)
}
iter := func(item interface{}) bool {
iter := func(item *itemT) bool {
count++
if count <= offset {
return true
}
nextStep(count, cursor, deadline)
iitm := item.(*itemT)
keepon = iterator(iitm.id, iitm.obj, c.fieldValues.get(iitm.fieldValuesSlot))
keepon = iterator(item.id, item.obj, c.fieldValues.get(item.fieldValuesSlot))
return keepon
}
if desc {
c.items.Descend(nil, iter)
c.items.Reverse(iter)
} else {
c.items.Ascend(nil, iter)
c.items.Scan(iter)
}
return keepon
}
@ -427,8 +419,7 @@ func (c *Collection) ScanRange(
offset = cursor.Offset()
cursor.Step(offset)
}
iter := func(value interface{}) bool {
item := value.(*itemT)
iter := func(item *itemT) bool {
count++
if count <= offset {
return true
@ -443,8 +434,7 @@ func (c *Collection) ScanRange(
return false
}
}
iitm := value.(*itemT)
keepon = iterator(iitm.id, iitm.obj, c.fieldValues.get(iitm.fieldValuesSlot))
keepon = iterator(item.id, item.obj, c.fieldValues.get(item.fieldValuesSlot))
return keepon
}
@ -470,20 +460,19 @@ func (c *Collection) SearchValues(
offset = cursor.Offset()
cursor.Step(offset)
}
iter := func(item interface{}) bool {
iter := func(item *itemT) bool {
count++
if count <= offset {
return true
}
nextStep(count, cursor, deadline)
iitm := item.(*itemT)
keepon = iterator(iitm.id, iitm.obj, c.fieldValues.get(iitm.fieldValuesSlot))
keepon = iterator(item.id, item.obj, c.fieldValues.get(item.fieldValuesSlot))
return keepon
}
if desc {
c.values.Descend(nil, iter)
c.values.Reverse(iter)
} else {
c.values.Ascend(nil, iter)
c.values.Scan(iter)
}
return keepon
}
@ -501,33 +490,32 @@ func (c *Collection) SearchValuesRange(start, end string, desc bool,
offset = cursor.Offset()
cursor.Step(offset)
}
iter := func(item interface{}) bool {
iter := func(item *itemT) bool {
count++
if count <= offset {
return true
}
nextStep(count, cursor, deadline)
iitm := item.(*itemT)
keepon = iterator(iitm.id, iitm.obj, c.fieldValues.get(iitm.fieldValuesSlot))
keepon = iterator(item.id, item.obj, c.fieldValues.get(item.fieldValuesSlot))
return keepon
}
pstart := &itemT{obj: String(start)}
pend := &itemT{obj: String(end)}
if desc {
// descend range
c.values.Descend(pstart, func(item interface{}) bool {
c.values.Descend(pstart, func(item *itemT) bool {
return bGT(c.values, item, pend) && iter(item)
})
} else {
c.values.Ascend(pstart, func(item interface{}) bool {
c.values.Ascend(pstart, func(item *itemT) bool {
return bLT(c.values, item, pend) && iter(item)
})
}
return keepon
}
func bLT(tr *btree.BTree, a, b interface{}) bool { return tr.Less(a, b) }
func bGT(tr *btree.BTree, a, b interface{}) bool { return tr.Less(b, a) }
func bLT(tr *btree.BTreeG[*itemT], a, b *itemT) bool { return tr.Less(a, b) }
func bGT(tr *btree.BTreeG[*itemT], a, b *itemT) bool { return tr.Less(b, a) }
// ScanGreaterOrEqual iterates though the collection starting with specified id.
func (c *Collection) ScanGreaterOrEqual(id string, desc bool,
@ -542,13 +530,12 @@ func (c *Collection) ScanGreaterOrEqual(id string, desc bool,
offset = cursor.Offset()
cursor.Step(offset)
}
iter := func(v interface{}) bool {
iter := func(item *itemT) bool {
count++
if count <= offset {
return true
}
nextStep(count, cursor, deadline)
item := v.(*itemT)
keepon = iterator(item.id, item.obj, c.fieldValues.get(item.fieldValuesSlot), item.expires)
return keepon
}
@ -565,11 +552,10 @@ func (c *Collection) geoSearch(
iter func(id string, obj geojson.Object, fields []float64) bool,
) bool {
alive := true
c.index.Search(
c.spatial.Search(
[2]float64{rect.Min.X, rect.Min.Y},
[2]float64{rect.Max.X, rect.Max.Y},
func(_, _ [2]float64, itemv interface{}) bool {
item := itemv.(*itemT)
func(_, _ [2]float64, item *itemT) bool {
alive = iter(item.id, item.obj, c.fieldValues.get(item.fieldValuesSlot))
return alive
},
@ -755,10 +741,10 @@ func (c *Collection) Nearby(
minLat, minLon, maxLat, maxLon :=
geo.RectFromCenter(center.Y, center.X, meters)
var exists bool
c.index.Search(
c.spatial.Search(
[2]float64{minLon, minLat},
[2]float64{maxLon, maxLat},
func(_, _ [2]float64, itemv interface{}) bool {
func(_, _ [2]float64, item *itemT) bool {
exists = true
return false
},
@ -778,15 +764,14 @@ func (c *Collection) Nearby(
offset = cursor.Offset()
cursor.Step(offset)
}
c.index.Nearby(
geodeticDistAlgo([2]float64{center.X, center.Y}),
func(_, _ [2]float64, itemv interface{}, dist float64) bool {
c.spatial.Nearby(
geodeticDistAlgo[*itemT]([2]float64{center.X, center.Y}),
func(_, _ [2]float64, item *itemT, dist float64) bool {
count++
if count <= offset {
return true
}
nextStep(count, cursor, deadline)
item := itemv.(*itemT)
alive = iter(item.id, item.obj, c.fieldValues.get(item.fieldValuesSlot), dist)
return alive
},
@ -804,17 +789,10 @@ func nextStep(step uint64, cursor Cursor, deadline *deadline.Deadline) {
}
}
type Expired struct {
ID string
Obj geojson.Object
Fields []float64
}
// Expired returns a list of all objects that have expired.
func (c *Collection) Expired(now int64, buffer []string) (ids []string) {
ids = buffer[:0]
c.expires.Ascend(nil, func(v interface{}) bool {
item := v.(*itemT)
c.expires.Scan(func(item *itemT) bool {
if now < item.expires {
return false
}

6
internal/collection/geodesic.go
View File

@ -2,11 +2,11 @@ package collection
import "math"
func geodeticDistAlgo(center [2]float64) (
algo func(min, max [2]float64, data interface{}, item bool) (dist float64),
func geodeticDistAlgo[T any](center [2]float64) (
algo func(min, max [2]float64, data T, item bool) (dist float64),
) {
const earthRadius = 6371e3
return func(min, max [2]float64, data interface{}, item bool) (dist float64) {
return func(min, max [2]float64, data T, item bool) (dist float64) {
return earthRadius * pointRectDistGeodeticDeg(
center[1], center[0],
min[1], min[0],

16
internal/collection/string.go
View File

@ -7,83 +7,67 @@ import (
"github.com/tidwall/geojson/geometry"
)
// String ...
type String string
var _ geojson.Object = String("")
// Spatial ...
func (s String) Spatial() geojson.Spatial {
return geojson.EmptySpatial{}
}
// ForEach ...
func (s String) ForEach(iter func(geom geojson.Object) bool) bool {
return iter(s)
}
// Empty ...
func (s String) Empty() bool {
return true
}
// Valid ...
func (s String) Valid() bool {
return false
}
// Rect ...
func (s String) Rect() geometry.Rect {
return geometry.Rect{}
}
// Center ...
func (s String) Center() geometry.Point {
return geometry.Point{}
}
// AppendJSON ...
func (s String) AppendJSON(dst []byte) []byte {
data, _ := json.Marshal(string(s))
return append(dst, data...)
}
// String ...
func (s String) String() string {
return string(s)
}
// JSON ...
func (s String) JSON() string {
return string(s.AppendJSON(nil))
}
// MarshalJSON ...
func (s String) MarshalJSON() ([]byte, error) {
return s.AppendJSON(nil), nil
}
// Within ...
func (s String) Within(obj geojson.Object) bool {
return false
}
// Contains ...
func (s String) Contains(obj geojson.Object) bool {
return false
}
// Intersects ...
func (s String) Intersects(obj geojson.Object) bool {
return false
}
// NumPoints ...
func (s String) NumPoints() int {
return 0
}
// Distance ...
func (s String) Distance(obj geojson.Object) float64 {
return 0
}