Merge branch 'memoptz'

controller/collection/collection.go

@@ -16,7 +16,6 @@ const (
 type itemT struct {
 	id     string
 	object geojson.Object
-	fields []float64
 }
 
 func (i *itemT) Less(item btree.Item, ctx interface{}) bool {
@@ -50,14 +49,15 @@ func (i *itemT) Point() (x, y, z float64) {
 
 // Collection represents a collection of geojson objects.
 type Collection struct {
-	items    *btree.BTree // items sorted by keys
+	items       *btree.BTree // items sorted by keys
-	values   *btree.BTree // items sorted by value+key
+	values      *btree.BTree // items sorted by value+key
-	index    *index.Index // items geospatially indexed
+	index       *index.Index // items geospatially indexed
-	fieldMap map[string]int
+	fieldMap    map[string]int
-	weight   int
+	fieldValues map[string][]float64
-	points   int
+	weight      int
-	objects  int // geometry count
+	points      int
-	nobjects int // non-geometry count
+	objects     int // geometry count
+	nobjects    int // non-geometry count
 }
 
 var counter uint64
@@ -66,13 +66,32 @@ var counter uint64
 func New() *Collection {
 	col := &Collection{
 		index:    index.New(),
-		items:    btree.New(48, idOrdered),
+		items:    btree.New(128, idOrdered),
-		values:   btree.New(48, valueOrdered),
+		values:   btree.New(128, valueOrdered),
 		fieldMap: make(map[string]int),
 	}
 	return col
 }
 
+func (c *Collection) setFieldValues(id string, values []float64) {
+	if c.fieldValues == nil {
+		c.fieldValues = make(map[string][]float64)
+	}
+	c.fieldValues[id] = values
+}
+
+func (c *Collection) getFieldValues(id string) (values []float64) {
+	if c.fieldValues == nil {
+		return nil
+	}
+	return c.fieldValues[id]
+}
+func (c *Collection) deleteFieldValues(id string) {
+	if c.fieldValues != nil {
+		delete(c.fieldValues, id)
+	}
+}
+
 // Count returns the number of objects in collection.
 func (c *Collection) Count() int {
 	return c.objects + c.nobjects
@@ -103,35 +122,78 @@ func (c *Collection) Bounds() (minX, minY, minZ, maxX, maxY, maxZ float64) {
 // The fields argument is optional.
 // The return values are the old object, the old fields, and the new fields
 func (c *Collection) ReplaceOrInsert(id string, obj geojson.Object, fields []string, values []float64) (oldObject geojson.Object, oldFields []float64, newFields []float64) {
-	oldItem, ok := c.remove(id)
+	var oldItem *itemT
-	nitem := c.insert(id, obj)
+	var newItem *itemT = &itemT{id: id, object: obj}
-	if ok {
+	// add the new item to main btree and remove the old one if needed
-		oldObject = oldItem.object
+	oldItemPtr := c.items.ReplaceOrInsert(newItem)
-		oldFields = oldItem.fields
+	if oldItemPtr != nil {
-		nitem.fields = oldFields
+		// the old item was removed, now let's remove from the rtree
-		c.weight += len(nitem.fields) * 8
+		// or strings tree.
-	}
+		oldItem = oldItemPtr.(*itemT)
-	if fields == nil && len(values) > 0 {
+		if obj.IsGeometry() {
-		// directly set the field values, update weight
+			// geometry
-		c.weight -= len(nitem.fields) * 8
+			c.index.Remove(oldItem)
-		nitem.fields = values
+			c.objects--
-		c.weight += len(nitem.fields) * 8
+		} else {
+			// string
+			c.values.Delete(oldItem)
+			c.nobjects--
+		}
+		// decrement the point count
+		c.points -= oldItem.object.PositionCount()
 
+		// decrement the weights
+		c.weight -= len(c.getFieldValues(id)) * 8
+		c.weight -= oldItem.object.Weight() + len(oldItem.id)
+
+		// references
+		oldObject = oldItem.object
+		oldFields = c.getFieldValues(id)
+		newFields = oldFields
+	}
+	// insert the new item into the rtree or strings tree.
+	if obj.IsGeometry() {
+		c.index.Insert(newItem)
+		c.objects++
 	} else {
+		c.values.ReplaceOrInsert(newItem)
+		c.nobjects++
+	}
+	// increment the point count
+	c.points += obj.PositionCount()
+
+	// add the new weights
+	c.weight += len(newFields) * 8
+	c.weight += obj.Weight() + len(id)
+	if fields == nil {
+		if len(values) > 0 {
+			// directly set the field values, update weight
+			c.weight -= len(newFields) * 8
+			newFields = values
+			c.setFieldValues(id, newFields)
+			c.weight += len(newFields) * 8
+		}
+	} else {
+		//if len(fields) == 0 {
+		//	panic("if fields is empty, make it nil")
+		//}
 		// map field name to value
 		for i, field := range fields {
-			c.setField(nitem, field, values[i])
+			c.setField(newItem, field, values[i])
 		}
+		newFields = c.getFieldValues(id)
 	}
-	return oldObject, oldFields, nitem.fields
+	return oldObject, oldFields, newFields
 }
 
-func (c *Collection) remove(id string) (item *itemT, ok bool) {
+// Remove removes an object and returns it.
+// If the object does not exist then the 'ok' return value will be false.
+func (c *Collection) Remove(id string) (obj geojson.Object, fields []float64, ok bool) {
 	i := c.items.Delete(&itemT{id: id})
 	if i == nil {
-		return nil, false
+		return nil, nil, false
 	}
-	item = i.(*itemT)
+	item := i.(*itemT)
 	if item.object.IsGeometry() {
 		c.index.Remove(item)
 		c.objects--
@@ -139,50 +201,23 @@ func (c *Collection) remove(id string) (item *itemT, ok bool) {
 		c.values.Delete(item)
 		c.nobjects--
 	}
-	c.weight -= len(item.fields) * 8
+	fields = c.getFieldValues(id)
+	c.deleteFieldValues(id)
+	c.weight -= len(fields) * 8
 	c.weight -= item.object.Weight() + len(item.id)
 	c.points -= item.object.PositionCount()
-	return item, true
+	return item.object, fields, true
-}
-
-func (c *Collection) insert(id string, obj geojson.Object) (item *itemT) {
-	item = &itemT{id: id, object: obj}
-	if obj.IsGeometry() {
-		c.index.Insert(item)
-		c.objects++
-	} else {
-		c.values.ReplaceOrInsert(item)
-		c.nobjects++
-	}
-	c.items.ReplaceOrInsert(item)
-	c.weight += obj.Weight() + len(id)
-	c.points += obj.PositionCount()
-	return item
-}
-
-// Remove removes an object and returns it.
-// If the object does not exist then the 'ok' return value will be false.
-func (c *Collection) Remove(id string) (obj geojson.Object, fields []float64, ok bool) {
-	item, ok := c.remove(id)
-	if !ok {
-		return nil, nil, false
-	}
-	return item.object, item.fields, true
-}
-
-func (c *Collection) get(id string) (obj geojson.Object, fields []float64, ok bool) {
-	i := c.items.Get(&itemT{id: id})
-	if i == nil {
-		return nil, nil, false
-	}
-	item := i.(*itemT)
-	return item.object, item.fields, true
 }
 
 // Get returns an object.
 // If the object does not exist then the 'ok' return value will be false.
 func (c *Collection) Get(id string) (obj geojson.Object, fields []float64, ok bool) {
-	return c.get(id)
+	i := c.items.Get(&itemT{id: id})
+	if i == nil {
+		return nil, nil, false
+	}
+	item := i.(*itemT)
+	return item.object, c.getFieldValues(id), true
 }
 
 // SetField set a field value for an object and returns that object.
@@ -195,7 +230,7 @@ func (c *Collection) SetField(id, field string, value float64) (obj geojson.Obje
 	}
 	item := i.(*itemT)
 	updated = c.setField(item, field, value)
-	return item.object, item.fields, updated, true
+	return item.object, c.getFieldValues(id), updated, true
 }
 
 func (c *Collection) setField(item *itemT, field string, value float64) (updated bool) {
@@ -204,13 +239,15 @@ func (c *Collection) setField(item *itemT, field string, value float64) (updated
 		idx = len(c.fieldMap)
 		c.fieldMap[field] = idx
 	}
-	c.weight -= len(item.fields) * 8
+	fields := c.getFieldValues(item.id)
-	for idx >= len(item.fields) {
+	c.weight -= len(fields) * 8
-		item.fields = append(item.fields, 0)
+	for idx >= len(fields) {
+		fields = append(fields, 0)
 	}
-	c.weight += len(item.fields) * 8
+	c.weight += len(fields) * 8
-	ovalue := item.fields[idx]
+	ovalue := fields[idx]
-	item.fields[idx] = value
+	fields[idx] = value
+	c.setFieldValues(item.id, fields)
 	return ovalue != value
 }
 
@@ -237,7 +274,7 @@ func (c *Collection) Scan(cursor uint64, desc bool,
 	iter := func(item btree.Item) bool {
 		if i >= cursor {
 			iitm := item.(*itemT)
-			active = iterator(iitm.id, iitm.object, iitm.fields)
+			active = iterator(iitm.id, iitm.object, c.getFieldValues(iitm.id))
 		}
 		i++
 		return active
@@ -259,7 +296,7 @@ func (c *Collection) ScanRange(cursor uint64, start, end string, desc bool,
 	iter := func(item btree.Item) bool {
 		if i >= cursor {
 			iitm := item.(*itemT)
-			active = iterator(iitm.id, iitm.object, iitm.fields)
+			active = iterator(iitm.id, iitm.object, c.getFieldValues(iitm.id))
 		}
 		i++
 		return active
@@ -282,7 +319,7 @@ func (c *Collection) SearchValues(cursor uint64, desc bool,
 	iter := func(item btree.Item) bool {
 		if i >= cursor {
 			iitm := item.(*itemT)
-			active = iterator(iitm.id, iitm.object, iitm.fields)
+			active = iterator(iitm.id, iitm.object, c.getFieldValues(iitm.id))
 		}
 		i++
 		return active
@@ -304,7 +341,7 @@ func (c *Collection) SearchValuesRange(cursor uint64, start, end string, desc bo
 	iter := func(item btree.Item) bool {
 		if i >= cursor {
 			iitm := item.(*itemT)
-			active = iterator(iitm.id, iitm.object, iitm.fields)
+			active = iterator(iitm.id, iitm.object, c.getFieldValues(iitm.id))
 		}
 		i++
 		return active
@@ -326,7 +363,7 @@ func (c *Collection) ScanGreaterOrEqual(id string, cursor uint64, desc bool,
 	iter := func(item btree.Item) bool {
 		if i >= cursor {
 			iitm := item.(*itemT)
-			active = iterator(iitm.id, iitm.object, iitm.fields)
+			active = iterator(iitm.id, iitm.object, c.getFieldValues(iitm.id))
 		}
 		i++
 		return active
@@ -346,7 +383,7 @@ func (c *Collection) geoSearch(cursor uint64, bbox geojson.BBox, iterator func(i
 		if !ok {
 			return true // just ignore
 		}
-		if !iterator(iitm.id, iitm.object, iitm.fields) {
+		if !iterator(iitm.id, iitm.object, c.getFieldValues(iitm.id)) {
 			return false
 		}
 		return true

controller/collection/collection_test.go

@@ -1,9 +1,11 @@
 package collection
 
 import (
+	"fmt"
 	"math/rand"
 	"strconv"
 	"testing"
+	"time"
 
 	"github.com/tidwall/tile38/geojson"
 )
@@ -87,3 +89,102 @@ func TestManyCollections(t *testing.T) {
 		return true
 	})
 }
+
+type testPointItem struct {
+	id     string
+	object geojson.Object
+}
+
+func BenchmarkInsert(t *testing.B) {
+	rand.Seed(time.Now().UnixNano())
+	items := make([]testPointItem, t.N)
+	for i := 0; i < t.N; i++ {
+		items[i] = testPointItem{
+			fmt.Sprintf("%d", i),
+			geojson.SimplePoint{
+				Y: rand.Float64()*180 - 90,
+				X: rand.Float64()*360 - 180,
+			},
+		}
+	}
+	col := New()
+	t.ResetTimer()
+	for i := 0; i < t.N; i++ {
+		col.ReplaceOrInsert(items[i].id, items[i].object, nil, nil)
+	}
+}
+
+func BenchmarkReplace(t *testing.B) {
+	rand.Seed(time.Now().UnixNano())
+	items := make([]testPointItem, t.N)
+	for i := 0; i < t.N; i++ {
+		items[i] = testPointItem{
+			fmt.Sprintf("%d", i),
+			geojson.SimplePoint{
+				Y: rand.Float64()*180 - 90,
+				X: rand.Float64()*360 - 180,
+			},
+		}
+	}
+	col := New()
+	for i := 0; i < t.N; i++ {
+		col.ReplaceOrInsert(items[i].id, items[i].object, nil, nil)
+	}
+	t.ResetTimer()
+	for _, i := range rand.Perm(t.N) {
+		o, _, _ := col.ReplaceOrInsert(items[i].id, items[i].object, nil, nil)
+		if o != items[i].object {
+			t.Fatal("shoot!")
+		}
+	}
+}
+
+func BenchmarkGet(t *testing.B) {
+	rand.Seed(time.Now().UnixNano())
+	items := make([]testPointItem, t.N)
+	for i := 0; i < t.N; i++ {
+		items[i] = testPointItem{
+			fmt.Sprintf("%d", i),
+			geojson.SimplePoint{
+				Y: rand.Float64()*180 - 90,
+				X: rand.Float64()*360 - 180,
+			},
+		}
+	}
+	col := New()
+	for i := 0; i < t.N; i++ {
+		col.ReplaceOrInsert(items[i].id, items[i].object, nil, nil)
+	}
+	t.ResetTimer()
+	for _, i := range rand.Perm(t.N) {
+		o, _, _ := col.Get(items[i].id)
+		if o != items[i].object {
+			t.Fatal("shoot!")
+		}
+	}
+}
+
+func BenchmarkRemove(t *testing.B) {
+	rand.Seed(time.Now().UnixNano())
+	items := make([]testPointItem, t.N)
+	for i := 0; i < t.N; i++ {
+		items[i] = testPointItem{
+			fmt.Sprintf("%d", i),
+			geojson.SimplePoint{
+				Y: rand.Float64()*180 - 90,
+				X: rand.Float64()*360 - 180,
+			},
+		}
+	}
+	col := New()
+	for i := 0; i < t.N; i++ {
+		col.ReplaceOrInsert(items[i].id, items[i].object, nil, nil)
+	}
+	t.ResetTimer()
+	for _, i := range rand.Perm(t.N) {
+		o, _, _ := col.Remove(items[i].id)
+		if o != items[i].object {
+			t.Fatal("shoot!")
+		}
+	}
+}

tests/keys_test.go

@@ -29,6 +29,7 @@ func subTestKeys(t *testing.T, mc *mockServer) {
 	runStep(t, mc, "TTL", keys_TTL_test)
 	runStep(t, mc, "SET EX", keys_SET_EX_test)
 	runStep(t, mc, "PDEL", keys_PDEL_test)
+	runStep(t, mc, "FIELDS", keys_FIELDS_test)
 }
 
 func keys_BOUNDS_test(mc *mockServer) error {
@@ -325,6 +326,21 @@ func keys_SET_EX_test(mc *mockServer) (err error) {
 	return nil
 }
 
+func keys_FIELDS_test(mc *mockServer) error {
+	return mc.DoBatch([][]interface{}{
+		{"SET", "mykey", "myid1a", "FIELD", "a", 1, "POINT", 33, -115}, {"OK"},
+		{"GET", "mykey", "myid1a", "WITHFIELDS"}, {`[{"type":"Point","coordinates":[-115,33]} [a 1]]`},
+		{"SET", "mykey", "myid1a", "FIELD", "a", "a", "POINT", 33, -115}, {"ERR invalid argument 'a'"},
+		{"GET", "mykey", "myid1a", "WITHFIELDS"}, {`[{"type":"Point","coordinates":[-115,33]} [a 1]]`},
+		{"SET", "mykey", "myid1a", "FIELD", "a", 1, "FIELD", "b", 2, "POINT", 33, -115}, {"OK"},
+		{"GET", "mykey", "myid1a", "WITHFIELDS"}, {`[{"type":"Point","coordinates":[-115,33]} [a 1 b 2]]`},
+		{"SET", "mykey", "myid1a", "FIELD", "b", 2, "POINT", 33, -115}, {"OK"},
+		{"GET", "mykey", "myid1a", "WITHFIELDS"}, {`[{"type":"Point","coordinates":[-115,33]} [a 1 b 2]]`},
+		{"SET", "mykey", "myid1a", "FIELD", "b", 2, "FIELD", "a", "1", "FIELD", "c", 3, "POINT", 33, -115}, {"OK"},
+		{"GET", "mykey", "myid1a", "WITHFIELDS"}, {`[{"type":"Point","coordinates":[-115,33]} [a 1 b 2 c 3]]`},
+	})
+}
+
 func keys_PDEL_test(mc *mockServer) error {
 	return mc.DoBatch([][]interface{}{
 		{"SET", "mykey", "myid1a", "POINT", 33, -115}, {"OK"},

tests/tests_test.go

@@ -57,7 +57,10 @@ func runStep(t *testing.T, mc *mockServer, name string, step func(mc *mockServer
 			mc.ResetConn()
 			defer mc.ResetConn()
 			// clear the database so the test is consistent
-			if err := mc.DoBatch([][]interface{}{{"FLUSHDB"}, {"OK"}}); err != nil {
+			if err := mc.DoBatch([][]interface{}{
+				{"OUTPUT", "resp"}, {"OK"},
+				{"FLUSHDB"}, {"OK"},
+			}); err != nil {
 				return err
 			}
 			if err := step(mc); err != nil {