mirror
/
tile38
mirror of https://github.com/tidwall/tile38.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'memoptz'

This commit is contained in:
Josh Baker 2017-01-13 10:03:02 -07:00
parent 2945479321 738f7eb43c
commit 627555cdc8
4 changed files with 237 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

195
controller/collection/collection.go
View File

@ -16,7 +16,6 @@ const (
type itemT struct {
id string
object geojson.Object
fields []float64
}
func (i *itemT) Less(item btree.Item, ctx interface{}) bool {
@ -50,14 +49,15 @@ func (i *itemT) Point() (x, y, z float64) {
// Collection represents a collection of geojson objects.
type Collection struct {
items *btree.BTree // items sorted by keys
values *btree.BTree // items sorted by value+key
index *index.Index // items geospatially indexed
fieldMap map[string]int
weight int
points int
objects int // geometry count
nobjects int // non-geometry count
items *btree.BTree // items sorted by keys
values *btree.BTree // items sorted by value+key
index *index.Index // items geospatially indexed
fieldMap map[string]int
fieldValues map[string][]float64
weight int
points int
objects int // geometry count
nobjects int // non-geometry count
}
var counter uint64
@ -66,13 +66,32 @@ var counter uint64
func New() *Collection {
col := &Collection{
index: index.New(),
items: btree.New(48, idOrdered),
values: btree.New(48, valueOrdered),
items: btree.New(128, idOrdered),
values: btree.New(128, valueOrdered),
fieldMap: make(map[string]int),
}
return col
}
func (c *Collection) setFieldValues(id string, values []float64) {
if c.fieldValues == nil {
c.fieldValues = make(map[string][]float64)
}
c.fieldValues[id] = values
}
func (c *Collection) getFieldValues(id string) (values []float64) {
if c.fieldValues == nil {
return nil
}
return c.fieldValues[id]
}
func (c *Collection) deleteFieldValues(id string) {
if c.fieldValues != nil {
delete(c.fieldValues, id)
}
}
// Count returns the number of objects in collection.
func (c *Collection) Count() int {
return c.objects + c.nobjects
@ -103,35 +122,78 @@ func (c *Collection) Bounds() (minX, minY, minZ, maxX, maxY, maxZ float64) {
// The fields argument is optional.
// The return values are the old object, the old fields, and the new fields
func (c *Collection) ReplaceOrInsert(id string, obj geojson.Object, fields []string, values []float64) (oldObject geojson.Object, oldFields []float64, newFields []float64) {
oldItem, ok := c.remove(id)
nitem := c.insert(id, obj)
if ok {
oldObject = oldItem.object
oldFields = oldItem.fields
nitem.fields = oldFields
c.weight += len(nitem.fields) * 8
}
if fields == nil && len(values) > 0 {
// directly set the field values, update weight
c.weight -= len(nitem.fields) * 8
nitem.fields = values
c.weight += len(nitem.fields) * 8
var oldItem *itemT
var newItem *itemT = &itemT{id: id, object: obj}
// add the new item to main btree and remove the old one if needed
oldItemPtr := c.items.ReplaceOrInsert(newItem)
if oldItemPtr != nil {
// the old item was removed, now let's remove from the rtree
// or strings tree.
oldItem = oldItemPtr.(*itemT)
if obj.IsGeometry() {
// geometry
c.index.Remove(oldItem)
c.objects--
} else {
// string
c.values.Delete(oldItem)
c.nobjects--
}
// decrement the point count
c.points -= oldItem.object.PositionCount()
// decrement the weights
c.weight -= len(c.getFieldValues(id)) * 8
c.weight -= oldItem.object.Weight() + len(oldItem.id)
// references
oldObject = oldItem.object
oldFields = c.getFieldValues(id)
newFields = oldFields
}
// insert the new item into the rtree or strings tree.
if obj.IsGeometry() {
c.index.Insert(newItem)
c.objects++
} else {
c.values.ReplaceOrInsert(newItem)
c.nobjects++
}
// increment the point count
c.points += obj.PositionCount()
// add the new weights
c.weight += len(newFields) * 8
c.weight += obj.Weight() + len(id)
if fields == nil {
if len(values) > 0 {
// directly set the field values, update weight
c.weight -= len(newFields) * 8
newFields = values
c.setFieldValues(id, newFields)
c.weight += len(newFields) * 8
}
} else {
//if len(fields) == 0 {
// panic("if fields is empty, make it nil")
//}
// map field name to value
for i, field := range fields {
c.setField(nitem, field, values[i])
c.setField(newItem, field, values[i])
}
newFields = c.getFieldValues(id)
}
return oldObject, oldFields, nitem.fields
return oldObject, oldFields, newFields
}
func (c *Collection) remove(id string) (item *itemT, ok bool) {
// Remove removes an object and returns it.
// If the object does not exist then the 'ok' return value will be false.
func (c *Collection) Remove(id string) (obj geojson.Object, fields []float64, ok bool) {
i := c.items.Delete(&itemT{id: id})
if i == nil {
return nil, false
return nil, nil, false
}
item = i.(*itemT)
item := i.(*itemT)
if item.object.IsGeometry() {
c.index.Remove(item)
c.objects--
@ -139,50 +201,23 @@ func (c *Collection) remove(id string) (item *itemT, ok bool) {
c.values.Delete(item)
c.nobjects--
}
c.weight -= len(item.fields) * 8
fields = c.getFieldValues(id)
c.deleteFieldValues(id)
c.weight -= len(fields) * 8
c.weight -= item.object.Weight() + len(item.id)
c.points -= item.object.PositionCount()
return item, true
}
func (c *Collection) insert(id string, obj geojson.Object) (item *itemT) {
item = &itemT{id: id, object: obj}
if obj.IsGeometry() {
c.index.Insert(item)
c.objects++
} else {
c.values.ReplaceOrInsert(item)
c.nobjects++
}
c.items.ReplaceOrInsert(item)
c.weight += obj.Weight() + len(id)
c.points += obj.PositionCount()
return item
}
// Remove removes an object and returns it.
// If the object does not exist then the 'ok' return value will be false.
func (c *Collection) Remove(id string) (obj geojson.Object, fields []float64, ok bool) {
item, ok := c.remove(id)
if !ok {
return nil, nil, false
}
return item.object, item.fields, true
}
func (c *Collection) get(id string) (obj geojson.Object, fields []float64, ok bool) {
i := c.items.Get(&itemT{id: id})
if i == nil {
return nil, nil, false
}
item := i.(*itemT)
return item.object, item.fields, true
return item.object, fields, true
}
// Get returns an object.
// If the object does not exist then the 'ok' return value will be false.
func (c *Collection) Get(id string) (obj geojson.Object, fields []float64, ok bool) {
return c.get(id)
i := c.items.Get(&itemT{id: id})
if i == nil {
return nil, nil, false
}
item := i.(*itemT)
return item.object, c.getFieldValues(id), true
}
// SetField set a field value for an object and returns that object.
@ -195,7 +230,7 @@ func (c *Collection) SetField(id, field string, value float64) (obj geojson.Obje
}
item := i.(*itemT)
updated = c.setField(item, field, value)
return item.object, item.fields, updated, true
return item.object, c.getFieldValues(id), updated, true
}
func (c *Collection) setField(item *itemT, field string, value float64) (updated bool) {
@ -204,13 +239,15 @@ func (c *Collection) setField(item *itemT, field string, value float64) (updated
idx = len(c.fieldMap)
c.fieldMap[field] = idx
}
c.weight -= len(item.fields) * 8
for idx >= len(item.fields) {
item.fields = append(item.fields, 0)
fields := c.getFieldValues(item.id)
c.weight -= len(fields) * 8
for idx >= len(fields) {
fields = append(fields, 0)
}
c.weight += len(item.fields) * 8
ovalue := item.fields[idx]
item.fields[idx] = value
c.weight += len(fields) * 8
ovalue := fields[idx]
fields[idx] = value
c.setFieldValues(item.id, fields)
return ovalue != value
}
@ -237,7 +274,7 @@ func (c *Collection) Scan(cursor uint64, desc bool,
iter := func(item btree.Item) bool {
if i >= cursor {
iitm := item.(*itemT)
active = iterator(iitm.id, iitm.object, iitm.fields)
active = iterator(iitm.id, iitm.object, c.getFieldValues(iitm.id))
}
i++
return active
@ -259,7 +296,7 @@ func (c *Collection) ScanRange(cursor uint64, start, end string, desc bool,
iter := func(item btree.Item) bool {
if i >= cursor {
iitm := item.(*itemT)
active = iterator(iitm.id, iitm.object, iitm.fields)
active = iterator(iitm.id, iitm.object, c.getFieldValues(iitm.id))
}
i++
return active
@ -282,7 +319,7 @@ func (c *Collection) SearchValues(cursor uint64, desc bool,
iter := func(item btree.Item) bool {
if i >= cursor {
iitm := item.(*itemT)
active = iterator(iitm.id, iitm.object, iitm.fields)
active = iterator(iitm.id, iitm.object, c.getFieldValues(iitm.id))
}
i++
return active
@ -304,7 +341,7 @@ func (c *Collection) SearchValuesRange(cursor uint64, start, end string, desc bo
iter := func(item btree.Item) bool {
if i >= cursor {
iitm := item.(*itemT)
active = iterator(iitm.id, iitm.object, iitm.fields)
active = iterator(iitm.id, iitm.object, c.getFieldValues(iitm.id))
}
i++
return active
@ -326,7 +363,7 @@ func (c *Collection) ScanGreaterOrEqual(id string, cursor uint64, desc bool,
iter := func(item btree.Item) bool {
if i >= cursor {
iitm := item.(*itemT)
active = iterator(iitm.id, iitm.object, iitm.fields)
active = iterator(iitm.id, iitm.object, c.getFieldValues(iitm.id))
}
i++
return active
@ -346,7 +383,7 @@ func (c *Collection) geoSearch(cursor uint64, bbox geojson.BBox, iterator func(i
if !ok {
return true // just ignore
}
if !iterator(iitm.id, iitm.object, iitm.fields) {
if !iterator(iitm.id, iitm.object, c.getFieldValues(iitm.id)) {
return false
}
return true

101
controller/collection/collection_test.go
View File

@ -1,9 +1,11 @@
package collection
import (
"fmt"
"math/rand"
"strconv"
"testing"
"time"
"github.com/tidwall/tile38/geojson"
)
@ -87,3 +89,102 @@ func TestManyCollections(t *testing.T) {
return true
})
}
type testPointItem struct {
id string
object geojson.Object
}
func BenchmarkInsert(t *testing.B) {
rand.Seed(time.Now().UnixNano())
items := make([]testPointItem, t.N)
for i := 0; i < t.N; i++ {
items[i] = testPointItem{
fmt.Sprintf("%d", i),
geojson.SimplePoint{
Y: rand.Float64()*180 - 90,
X: rand.Float64()*360 - 180,
},
}
}
col := New()
t.ResetTimer()
for i := 0; i < t.N; i++ {
col.ReplaceOrInsert(items[i].id, items[i].object, nil, nil)
}
}
func BenchmarkReplace(t *testing.B) {
rand.Seed(time.Now().UnixNano())
items := make([]testPointItem, t.N)
for i := 0; i < t.N; i++ {
items[i] = testPointItem{
fmt.Sprintf("%d", i),
geojson.SimplePoint{
Y: rand.Float64()*180 - 90,
X: rand.Float64()*360 - 180,
},
}
}
col := New()
for i := 0; i < t.N; i++ {
col.ReplaceOrInsert(items[i].id, items[i].object, nil, nil)
}
t.ResetTimer()
for _, i := range rand.Perm(t.N) {
o, _, _ := col.ReplaceOrInsert(items[i].id, items[i].object, nil, nil)
if o != items[i].object {
t.Fatal("shoot!")
}
}
}
func BenchmarkGet(t *testing.B) {
rand.Seed(time.Now().UnixNano())
items := make([]testPointItem, t.N)
for i := 0; i < t.N; i++ {
items[i] = testPointItem{
fmt.Sprintf("%d", i),
geojson.SimplePoint{
Y: rand.Float64()*180 - 90,
X: rand.Float64()*360 - 180,
},
}
}
col := New()
for i := 0; i < t.N; i++ {
col.ReplaceOrInsert(items[i].id, items[i].object, nil, nil)
}
t.ResetTimer()
for _, i := range rand.Perm(t.N) {
o, _, _ := col.Get(items[i].id)
if o != items[i].object {
t.Fatal("shoot!")
}
}
}
func BenchmarkRemove(t *testing.B) {
rand.Seed(time.Now().UnixNano())
items := make([]testPointItem, t.N)
for i := 0; i < t.N; i++ {
items[i] = testPointItem{
fmt.Sprintf("%d", i),
geojson.SimplePoint{
Y: rand.Float64()*180 - 90,
X: rand.Float64()*360 - 180,
},
}
}
col := New()
for i := 0; i < t.N; i++ {
col.ReplaceOrInsert(items[i].id, items[i].object, nil, nil)
}
t.ResetTimer()
for _, i := range rand.Perm(t.N) {
o, _, _ := col.Remove(items[i].id)
if o != items[i].object {
t.Fatal("shoot!")
}
}
}

16
tests/keys_test.go
View File

@ -29,6 +29,7 @@ func subTestKeys(t *testing.T, mc *mockServer) {
runStep(t, mc, "TTL", keys_TTL_test)
runStep(t, mc, "SET EX", keys_SET_EX_test)
runStep(t, mc, "PDEL", keys_PDEL_test)
runStep(t, mc, "FIELDS", keys_FIELDS_test)
}
func keys_BOUNDS_test(mc *mockServer) error {
@ -325,6 +326,21 @@ func keys_SET_EX_test(mc *mockServer) (err error) {
return nil
}
func keys_FIELDS_test(mc *mockServer) error {
return mc.DoBatch([][]interface{}{
{"SET", "mykey", "myid1a", "FIELD", "a", 1, "POINT", 33, -115}, {"OK"},
{"GET", "mykey", "myid1a", "WITHFIELDS"}, {`[{"type":"Point","coordinates":[-115,33]} [a 1]]`},
{"SET", "mykey", "myid1a", "FIELD", "a", "a", "POINT", 33, -115}, {"ERR invalid argument 'a'"},
{"GET", "mykey", "myid1a", "WITHFIELDS"}, {`[{"type":"Point","coordinates":[-115,33]} [a 1]]`},
{"SET", "mykey", "myid1a", "FIELD", "a", 1, "FIELD", "b", 2, "POINT", 33, -115}, {"OK"},
{"GET", "mykey", "myid1a", "WITHFIELDS"}, {`[{"type":"Point","coordinates":[-115,33]} [a 1 b 2]]`},
{"SET", "mykey", "myid1a", "FIELD", "b", 2, "POINT", 33, -115}, {"OK"},
{"GET", "mykey", "myid1a", "WITHFIELDS"}, {`[{"type":"Point","coordinates":[-115,33]} [a 1 b 2]]`},
{"SET", "mykey", "myid1a", "FIELD", "b", 2, "FIELD", "a", "1", "FIELD", "c", 3, "POINT", 33, -115}, {"OK"},
{"GET", "mykey", "myid1a", "WITHFIELDS"}, {`[{"type":"Point","coordinates":[-115,33]} [a 1 b 2 c 3]]`},
})
}
func keys_PDEL_test(mc *mockServer) error {
return mc.DoBatch([][]interface{}{
{"SET", "mykey", "myid1a", "POINT", 33, -115}, {"OK"},

5
tests/tests_test.go
View File

@ -57,7 +57,10 @@ func runStep(t *testing.T, mc *mockServer, name string, step func(mc *mockServer
mc.ResetConn()
defer mc.ResetConn()
// clear the database so the test is consistent
if err := mc.DoBatch([][]interface{}{{"FLUSHDB"}, {"OK"}}); err != nil {
if err := mc.DoBatch([][]interface{}{
{"OUTPUT", "resp"}, {"OK"},
{"FLUSHDB"}, {"OK"},
}); err != nil {
return err
}
if err := step(mc); err != nil {