mirror
/
tile38
mirror of https://github.com/tidwall/tile38.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Isolated fields type

This commit is contained in:
tidwall 2019-02-15 08:26:55 -07:00
parent 3ac8dc2ffb
commit a60dc57598
12 changed files with 375 additions and 205 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

86
internal/collection/collection.go
View File

@ -119,39 +119,39 @@ func (c *Collection) delItem(item *item.Item) {
func (c *Collection) Set(
id string, obj geojson.Object, fields []string, values []float64,
) (
oldObj geojson.Object, oldFields []float64, newFields []float64,
oldObj geojson.Object, oldFields *Fields, newFields *Fields,
) {
// create the new item
item := item.New(id, obj)
newItem := item.New(id, obj)
// add the new item to main btree and remove the old one if needed
oldItemV, ok := c.items.Set(item)
var oldItem *item.Item
oldItemV, ok := c.items.Set(newItem)
if ok {
oldItem := oldItemV
oldItem = oldItemV
oldObj = oldItem.Obj()
// remove old item from indexes
c.delItem(oldItem)
if len(oldItem.Fields()) > 0 {
if oldItem.HasFields() {
// merge old and new fields
oldFields = oldItem.Fields()
item.CopyOverFields(oldFields)
newItem.CopyOverFields(oldItem)
}
}
if fields == nil && len(values) > 0 {
// directly set the field values, from copy
item.CopyOverFields(values)
newItem.CopyOverFields(values)
} else if len(fields) > 0 {
// add new field to new item
c.setFields(item, fields, values, false)
c.setFields(newItem, fields, values, false)
}
// add new item to indexes
c.addItem(item)
c.addItem(newItem)
// fmt.Printf("!!! %#v\n", oldObj)
return oldObj, oldFields, item.Fields()
return oldObj, itemFields(oldItem), itemFields(newItem)
}
func (c *Collection) setFields(
@ -192,7 +192,7 @@ func (c *Collection) setField(
// Delete removes an object and returns it.
// If the object does not exist then the 'ok' return value will be false.
func (c *Collection) Delete(id string) (
obj geojson.Object, fields []float64, ok bool,
obj geojson.Object, fields *Fields, ok bool,
) {
oldItemV, ok := c.items.Delete(id)
if !ok {
@ -202,13 +202,13 @@ func (c *Collection) Delete(id string) (
c.delItem(oldItem)
return oldItem.Obj(), oldItem.Fields(), true
return oldItem.Obj(), itemFields(oldItem), true
}
// Get returns an object.
// If the object does not exist then the 'ok' return value will be false.
func (c *Collection) Get(id string) (
obj geojson.Object, fields []float64, ok bool,
obj geojson.Object, fields *Fields, ok bool,
) {
itemV, ok := c.items.Get(id)
if !ok {
@ -216,13 +216,13 @@ func (c *Collection) Get(id string) (
}
item := itemV
return item.Obj(), item.Fields(), true
return item.Obj(), itemFields(item), true
}
// SetField set a field value for an object and returns that object.
// If the object does not exist then the 'ok' return value will be false.
func (c *Collection) SetField(id, fieldName string, fieldValue float64) (
obj geojson.Object, fields []float64, updated bool, ok bool,
obj geojson.Object, fields *Fields, updated bool, ok bool,
) {
itemV, ok := c.items.Get(id)
if !ok {
@ -230,13 +230,13 @@ func (c *Collection) SetField(id, fieldName string, fieldValue float64) (
}
item := itemV
updated = c.setField(item, fieldName, fieldValue, true)
return item.Obj(), item.Fields(), updated, true
return item.Obj(), itemFields(item), updated, true
}
// SetFields is similar to SetField, just setting multiple fields at once
func (c *Collection) SetFields(
id string, fieldNames []string, fieldValues []float64,
) (obj geojson.Object, fields []float64, updatedCount int, ok bool) {
) (obj geojson.Object, fields *Fields, updatedCount int, ok bool) {
itemV, ok := c.items.Get(id)
if !ok {
return nil, nil, 0, false
@ -245,7 +245,7 @@ func (c *Collection) SetFields(
updatedCount = c.setFields(item, fieldNames, fieldValues, true)
return item.Obj(), item.Fields(), updatedCount, true
return item.Obj(), itemFields(item), updatedCount, true
}
// FieldMap return a maps of the field names.
@ -264,7 +264,7 @@ func (c *Collection) FieldArr() []string {
// Scan iterates though the collection ids.
func (c *Collection) Scan(desc bool, cursor Cursor,
iterator func(id string, obj geojson.Object, fields []float64) bool,
iterator func(id string, obj geojson.Object, fields *Fields) bool,
) bool {
var keepon = true
var count uint64
@ -281,7 +281,7 @@ func (c *Collection) Scan(desc bool, cursor Cursor,
if cursor != nil {
cursor.Step(1)
}
keepon = iterator(item.ID(), item.Obj(), item.Fields())
keepon = iterator(item.ID(), item.Obj(), itemFields(item))
return keepon
}
if desc {
@ -294,7 +294,7 @@ func (c *Collection) Scan(desc bool, cursor Cursor,
// ScanRange iterates though the collection starting with specified id.
func (c *Collection) ScanRange(start, end string, desc bool, cursor Cursor,
iterator func(id string, obj geojson.Object, fields []float64) bool,
iterator func(id string, obj geojson.Object, fields *Fields) bool,
) bool {
var keepon = true
var count uint64
@ -320,7 +320,7 @@ func (c *Collection) ScanRange(start, end string, desc bool, cursor Cursor,
return false
}
}
keepon = iterator(item.ID(), item.Obj(), item.Fields())
keepon = iterator(item.ID(), item.Obj(), itemFields(item))
return keepon
}
@ -334,7 +334,7 @@ func (c *Collection) ScanRange(start, end string, desc bool, cursor Cursor,
// SearchValues iterates though the collection values.
func (c *Collection) SearchValues(desc bool, cursor Cursor,
iterator func(id string, obj geojson.Object, fields []float64) bool,
iterator func(id string, obj geojson.Object, fields *Fields) bool,
) bool {
var keepon = true
var count uint64
@ -352,7 +352,7 @@ func (c *Collection) SearchValues(desc bool, cursor Cursor,
cursor.Step(1)
}
iitm := v.(*item.Item)
keepon = iterator(iitm.ID(), iitm.Obj(), iitm.Fields())
keepon = iterator(iitm.ID(), iitm.Obj(), itemFields(iitm))
return keepon
}
if desc {
@ -366,7 +366,7 @@ func (c *Collection) SearchValues(desc bool, cursor Cursor,
// SearchValuesRange iterates though the collection values.
func (c *Collection) SearchValuesRange(start, end string, desc bool,
cursor Cursor,
iterator func(id string, obj geojson.Object, fields []float64) bool,
iterator func(id string, obj geojson.Object, fields *Fields) bool,
) bool {
var keepon = true
var count uint64
@ -384,7 +384,7 @@ func (c *Collection) SearchValuesRange(start, end string, desc bool,
cursor.Step(1)
}
iitm := v.(*item.Item)
keepon = iterator(iitm.ID(), iitm.Obj(), iitm.Fields())
keepon = iterator(iitm.ID(), iitm.Obj(), itemFields(iitm))
return keepon
}
if desc {
@ -402,7 +402,7 @@ func (c *Collection) SearchValuesRange(start, end string, desc bool,
// ScanGreaterOrEqual iterates though the collection starting with specified id.
func (c *Collection) ScanGreaterOrEqual(id string, desc bool,
cursor Cursor,
iterator func(id string, obj geojson.Object, fields []float64) bool,
iterator func(id string, obj geojson.Object, fields *Fields) bool,
) bool {
var keepon = true
var count uint64
@ -419,7 +419,7 @@ func (c *Collection) ScanGreaterOrEqual(id string, desc bool,
if cursor != nil {
cursor.Step(1)
}
keepon = iterator(item.ID(), item.Obj(), item.Fields())
keepon = iterator(item.ID(), item.Obj(), itemFields(item))
return keepon
}
if desc {
@ -432,7 +432,7 @@ func (c *Collection) ScanGreaterOrEqual(id string, desc bool,
func (c *Collection) geoSearch(
rect geometry.Rect,
iter func(id string, obj geojson.Object, fields []float64) bool,
iter func(id string, obj geojson.Object, fields *Fields) bool,
) bool {
alive := true
c.index.Search(
@ -440,7 +440,7 @@ func (c *Collection) geoSearch(
[]float64{rect.Max.X, rect.Max.Y},
func(_, _ []float64, itemv *item.Item) bool {
item := itemv
alive = iter(item.ID(), item.Obj(), item.Fields())
alive = iter(item.ID(), item.Obj(), itemFields(item))
return alive
},
)
@ -449,12 +449,12 @@ func (c *Collection) geoSearch(
func (c *Collection) geoSparse(
obj geojson.Object, sparse uint8,
iter func(id string, obj geojson.Object, fields []float64) (match, ok bool),
iter func(id string, obj geojson.Object, fields *Fields) (match, ok bool),
) bool {
matches := make(map[string]bool)
alive := true
c.geoSparseInner(obj.Rect(), sparse,
func(id string, o geojson.Object, fields []float64) (
func(id string, o geojson.Object, fields *Fields) (
match, ok bool,
) {
ok = true
@ -471,7 +471,7 @@ func (c *Collection) geoSparse(
}
func (c *Collection) geoSparseInner(
rect geometry.Rect, sparse uint8,
iter func(id string, obj geojson.Object, fields []float64) (match, ok bool),
iter func(id string, obj geojson.Object, fields *Fields) (match, ok bool),
) bool {
if sparse > 0 {
w := rect.Max.X - rect.Min.X
@ -503,7 +503,7 @@ func (c *Collection) geoSparseInner(
}
alive := true
c.geoSearch(rect,
func(id string, obj geojson.Object, fields []float64) bool {
func(id string, obj geojson.Object, fields *Fields) bool {
match, ok := iter(id, obj, fields)
if !ok {
alive = false
@ -521,7 +521,7 @@ func (c *Collection) Within(
obj geojson.Object,
sparse uint8,
cursor Cursor,
iter func(id string, obj geojson.Object, fields []float64) bool,
iter func(id string, obj geojson.Object, fields *Fields) bool,
) bool {
var count uint64
var offset uint64
@ -531,7 +531,7 @@ func (c *Collection) Within(
}
if sparse > 0 {
return c.geoSparse(obj, sparse,
func(id string, o geojson.Object, fields []float64) (
func(id string, o geojson.Object, fields *Fields) (
match, ok bool,
) {
count++
@ -549,7 +549,7 @@ func (c *Collection) Within(
)
}
return c.geoSearch(obj.Rect(),
func(id string, o geojson.Object, fields []float64) bool {
func(id string, o geojson.Object, fields *Fields) bool {
count++
if count <= offset {
return true
@ -571,7 +571,7 @@ func (c *Collection) Intersects(
obj geojson.Object,
sparse uint8,
cursor Cursor,
iter func(id string, obj geojson.Object, fields []float64) bool,
iter func(id string, obj geojson.Object, fields *Fields) bool,
) bool {
var count uint64
var offset uint64
@ -581,7 +581,7 @@ func (c *Collection) Intersects(
}
if sparse > 0 {
return c.geoSparse(obj, sparse,
func(id string, o geojson.Object, fields []float64) (
func(id string, o geojson.Object, fields *Fields) (
match, ok bool,
) {
count++
@ -599,7 +599,7 @@ func (c *Collection) Intersects(
)
}
return c.geoSearch(obj.Rect(),
func(id string, o geojson.Object, fields []float64) bool {
func(id string, o geojson.Object, fields *Fields) bool {
count++
if count <= offset {
return true
@ -619,7 +619,7 @@ func (c *Collection) Intersects(
func (c *Collection) Nearby(
target geojson.Object,
cursor Cursor,
iter func(id string, obj geojson.Object, fields []float64) bool,
iter func(id string, obj geojson.Object, fields *Fields) bool,
) bool {
// First look to see if there's at least one candidate in the circle's
// outer rectangle. This is a fast-fail operation.
@ -665,7 +665,7 @@ func (c *Collection) Nearby(
cursor.Step(1)
}
item := itemv
alive = iter(item.ID(), item.Obj(), item.Fields())
alive = iter(item.ID(), item.Obj(), itemFields(item))
return alive
},
)

141
internal/collection/collection_test.go
View File

@ -104,7 +104,7 @@ func TestCollectionNewCollection(t *testing.T) {
Min: geometry.Point{X: -180, Y: -90},
Max: geometry.Point{X: 180, Y: 90},
}
c.geoSearch(bbox, func(id string, obj geojson.Object, field []float64) bool {
c.geoSearch(bbox, func(id string, obj geojson.Object, field *Fields) bool {
count++
return true
})
@ -124,8 +124,8 @@ func TestCollectionSet(t *testing.T) {
str1 := String("hello")
oldObject, oldFields, newFields := c.Set("str", str1, nil, nil)
expect(t, oldObject == nil)
expect(t, len(oldFields) == 0)
expect(t, len(newFields) == 0)
expect(t, fieldLen(oldFields) == 0)
expect(t, fieldLen(newFields) == 0)
})
t.Run("UpdateString", func(t *testing.T) {
c := New()
@ -133,20 +133,20 @@ func TestCollectionSet(t *testing.T) {
str2 := String("world")
oldObject, oldFields, newFields := c.Set("str", str1, nil, nil)
expect(t, oldObject == nil)
expect(t, len(oldFields) == 0)
expect(t, len(newFields) == 0)
expect(t, fieldLen(oldFields) == 0)
expect(t, fieldLen(newFields) == 0)
oldObject, oldFields, newFields = c.Set("str", str2, nil, nil)
expect(t, oldObject == str1)
expect(t, len(oldFields) == 0)
expect(t, len(newFields) == 0)
expect(t, fieldLen(oldFields) == 0)
expect(t, fieldLen(newFields) == 0)
})
t.Run("AddPoint", func(t *testing.T) {
c := New()
point1 := PO(-112.1, 33.1)
oldObject, oldFields, newFields := c.Set("point", point1, nil, nil)
expect(t, oldObject == nil)
expect(t, len(oldFields) == 0)
expect(t, len(newFields) == 0)
expect(t, fieldLen(oldFields) == 0)
expect(t, fieldLen(newFields) == 0)
})
t.Run("UpdatePoint", func(t *testing.T) {
c := New()
@ -154,12 +154,12 @@ func TestCollectionSet(t *testing.T) {
point2 := PO(-112.2, 33.2)
oldObject, oldFields, newFields := c.Set("point", point1, nil, nil)
expect(t, oldObject == nil)
expect(t, len(oldFields) == 0)
expect(t, len(newFields) == 0)
expect(t, fieldLen(oldFields) == 0)
expect(t, fieldLen(newFields) == 0)
oldObject, oldFields, newFields = c.Set("point", point2, nil, nil)
expect(t, oldObject == point1)
expect(t, len(oldFields) == 0)
expect(t, len(newFields) == 0)
expect(t, fieldLen(oldFields) == 0)
expect(t, fieldLen(newFields) == 0)
})
t.Run("Fields", func(t *testing.T) {
c := New()
@ -170,23 +170,23 @@ func TestCollectionSet(t *testing.T) {
fValues := []float64{1, 2, 3}
oldObj, oldFlds, newFlds := c.Set("str", str1, fNames, fValues)
expect(t, oldObj == nil)
expect(t, len(oldFlds) == 0)
expect(t, reflect.DeepEqual(newFlds, fValues))
expect(t, fieldLen(oldFlds) == 0)
expect(t, fieldIterEquals(newFlds, fValues))
}
{
fNames := []string{"d", "e", "f"}
fValues := []float64{4, 5, 6}
oldObj, oldFlds, newFlds := c.Set("str", str2, fNames, fValues)
expect(t, oldObj == str1)
expect(t, reflect.DeepEqual(oldFlds, []float64{1, 2, 3}))
expect(t, reflect.DeepEqual(newFlds, []float64{1, 2, 3, 4, 5, 6}))
expect(t, fieldIterEquals(oldFlds, []float64{1, 2, 3}))
expect(t, fieldIterEquals(newFlds, []float64{1, 2, 3, 4, 5, 6}))
}
{
fValues := []float64{7, 8, 9, 10, 11, 12}
oldObj, oldFlds, newFlds := c.Set("str", str1, nil, fValues)
expect(t, oldObj == str2)
expect(t, reflect.DeepEqual(oldFlds, []float64{1, 2, 3, 4, 5, 6}))
expect(t, reflect.DeepEqual(newFlds, []float64{7, 8, 9, 10, 11, 12}))
expect(t, fieldIterEquals(oldFlds, []float64{1, 2, 3, 4, 5, 6}))
expect(t, fieldIterEquals(newFlds, []float64{7, 8, 9, 10, 11, 12}))
}
})
t.Run("Delete", func(t *testing.T) {
@ -204,7 +204,7 @@ func TestCollectionSet(t *testing.T) {
Max: geometry.Point{X: 1, Y: 2}})
var v geojson.Object
var ok bool
var flds []float64
var flds *Fields
var updated bool
var updateCount int
@ -226,24 +226,24 @@ func TestCollectionSet(t *testing.T) {
v, flds, updated, ok = c.SetField("3", "hello", 123)
expect(t, ok)
expect(t, reflect.DeepEqual(flds, []float64{123}))
expect(t, fieldIterEquals(flds, []float64{123}))
expect(t, updated)
expect(t, c.FieldMap()["hello"] == 0)
v, flds, updated, ok = c.SetField("3", "hello", 1234)
expect(t, ok)
expect(t, reflect.DeepEqual(flds, []float64{1234}))
expect(t, fieldIterEquals(flds, []float64{1234}))
expect(t, updated)
v, flds, updated, ok = c.SetField("3", "hello", 1234)
expect(t, ok)
expect(t, reflect.DeepEqual(flds, []float64{1234}))
expect(t, fieldIterEquals(flds, []float64{1234}))
expect(t, !updated)
v, flds, updateCount, ok = c.SetFields("3",
[]string{"planet", "world"}, []float64{55, 66})
expect(t, ok)
expect(t, reflect.DeepEqual(flds, []float64{1234, 55, 66}))
expect(t, fieldIterEquals(flds, []float64{1234, 55, 66}))
expect(t, updateCount == 2)
expect(t, c.FieldMap()["hello"] == 0)
expect(t, c.FieldMap()["planet"] == 1)
@ -277,6 +277,29 @@ func TestCollectionSet(t *testing.T) {
})
}
func fieldLen(fields *Fields) int {
var idx int
fields.ForEach(-1, func(value float64) bool {
idx++
return true
})
return idx
}
func fieldIterEquals(fields *Fields, values []float64) bool {
ok := true
var idx int
fields.ForEach(len(values), func(value float64) bool {
if value != values[idx] {
ok = false
return false
}
idx++
return true
})
return ok
}
func TestCollectionScan(t *testing.T) {
N := 256
c := New()
@ -286,22 +309,22 @@ func TestCollectionScan(t *testing.T) {
}
var n int
var prevID string
c.Scan(false, nil, func(id string, obj geojson.Object, fields []float64) bool {
c.Scan(false, nil, func(id string, obj geojson.Object, fields *Fields) bool {
if n > 0 {
expect(t, id > prevID)
}
expect(t, id == fmt.Sprintf("%04d", int(fields[0])))
expect(t, id == fmt.Sprintf("%04d", int(fields.Get(0))))
n++
prevID = id
return true
})
expect(t, n == c.Count())
n = 0
c.Scan(true, nil, func(id string, obj geojson.Object, fields []float64) bool {
c.Scan(true, nil, func(id string, obj geojson.Object, fields *Fields) bool {
if n > 0 {
expect(t, id < prevID)
}
expect(t, id == fmt.Sprintf("%04d", int(fields[0])))
expect(t, id == fmt.Sprintf("%04d", int(fields.Get(0))))
n++
prevID = id
return true
@ -310,11 +333,11 @@ func TestCollectionScan(t *testing.T) {
n = 0
c.ScanRange("0060", "0070", false, nil,
func(id string, obj geojson.Object, fields []float64) bool {
func(id string, obj geojson.Object, fields *Fields) bool {
if n > 0 {
expect(t, id > prevID)
}
expect(t, id == fmt.Sprintf("%04d", int(fields[0])))
expect(t, id == fmt.Sprintf("%04d", int(fields.Get(0))))
n++
prevID = id
return true
@ -323,11 +346,11 @@ func TestCollectionScan(t *testing.T) {
n = 0
c.ScanRange("0070", "0060", true, nil,
func(id string, obj geojson.Object, fields []float64) bool {
func(id string, obj geojson.Object, fields *Fields) bool {
if n > 0 {
expect(t, id < prevID)
}
expect(t, id == fmt.Sprintf("%04d", int(fields[0])))
expect(t, id == fmt.Sprintf("%04d", int(fields.Get(0))))
n++
prevID = id
return true
@ -336,11 +359,11 @@ func TestCollectionScan(t *testing.T) {
n = 0
c.ScanGreaterOrEqual("0070", true, nil,
func(id string, obj geojson.Object, fields []float64) bool {
func(id string, obj geojson.Object, fields *Fields) bool {
if n > 0 {
expect(t, id < prevID)
}
expect(t, id == fmt.Sprintf("%04d", int(fields[0])))
expect(t, id == fmt.Sprintf("%04d", int(fields.Get(0))))
n++
prevID = id
return true
@ -349,11 +372,11 @@ func TestCollectionScan(t *testing.T) {
n = 0
c.ScanGreaterOrEqual("0070", false, nil,
func(id string, obj geojson.Object, fields []float64) bool {
func(id string, obj geojson.Object, fields *Fields) bool {
if n > 0 {
expect(t, id > prevID)
}
expect(t, id == fmt.Sprintf("%04d", int(fields[0])))
expect(t, id == fmt.Sprintf("%04d", int(fields.Get(0))))
n++
prevID = id
return true
@ -373,22 +396,22 @@ func TestCollectionSearch(t *testing.T) {
}
var n int
var prevValue string
c.SearchValues(false, nil, func(id string, obj geojson.Object, fields []float64) bool {
c.SearchValues(false, nil, func(id string, obj geojson.Object, fields *Fields) bool {
if n > 0 {
expect(t, obj.String() > prevValue)
}
expect(t, id == fmt.Sprintf("%04d", int(fields[1])))
expect(t, id == fmt.Sprintf("%04d", int(fields.Get(1))))
n++
prevValue = obj.String()
return true
})
expect(t, n == c.Count())
n = 0
c.SearchValues(true, nil, func(id string, obj geojson.Object, fields []float64) bool {
c.SearchValues(true, nil, func(id string, obj geojson.Object, fields *Fields) bool {
if n > 0 {
expect(t, obj.String() < prevValue)
}
expect(t, id == fmt.Sprintf("%04d", int(fields[1])))
expect(t, id == fmt.Sprintf("%04d", int(fields.Get(1))))
n++
prevValue = obj.String()
return true
@ -397,11 +420,11 @@ func TestCollectionSearch(t *testing.T) {
n = 0
c.SearchValuesRange("0060", "0070", false, nil,
func(id string, obj geojson.Object, fields []float64) bool {
func(id string, obj geojson.Object, fields *Fields) bool {
if n > 0 {
expect(t, obj.String() > prevValue)
}
expect(t, id == fmt.Sprintf("%04d", int(fields[1])))
expect(t, id == fmt.Sprintf("%04d", int(fields.Get(1))))
n++
prevValue = obj.String()
return true
@ -410,11 +433,11 @@ func TestCollectionSearch(t *testing.T) {
n = 0
c.SearchValuesRange("0070", "0060", true, nil,
func(id string, obj geojson.Object, fields []float64) bool {
func(id string, obj geojson.Object, fields *Fields) bool {
if n > 0 {
expect(t, obj.String() < prevValue)
}
expect(t, id == fmt.Sprintf("%04d", int(fields[1])))
expect(t, id == fmt.Sprintf("%04d", int(fields.Get(1))))
n++
prevValue = obj.String()
return true
@ -493,7 +516,7 @@ func TestSpatialSearch(t *testing.T) {
n = 0
c.Within(q1, 0, nil,
func(id string, obj geojson.Object, fields []float64) bool {
func(id string, obj geojson.Object, fields *Fields) bool {
n++
return true
},
@ -502,7 +525,7 @@ func TestSpatialSearch(t *testing.T) {
n = 0
c.Within(q2, 0, nil,
func(id string, obj geojson.Object, fields []float64) bool {
func(id string, obj geojson.Object, fields *Fields) bool {
n++
return true
},
@ -511,7 +534,7 @@ func TestSpatialSearch(t *testing.T) {
n = 0
c.Within(q3, 0, nil,
func(id string, obj geojson.Object, fields []float64) bool {
func(id string, obj geojson.Object, fields *Fields) bool {
n++
return true
},
@ -520,7 +543,7 @@ func TestSpatialSearch(t *testing.T) {
n = 0
c.Intersects(q1, 0, nil,
func(_ string, _ geojson.Object, _ []float64) bool {
func(_ string, _ geojson.Object, _ *Fields) bool {
n++
return true
},
@ -529,7 +552,7 @@ func TestSpatialSearch(t *testing.T) {
n = 0
c.Intersects(q2, 0, nil,
func(_ string, _ geojson.Object, _ []float64) bool {
func(_ string, _ geojson.Object, _ *Fields) bool {
n++
return true
},
@ -538,7 +561,7 @@ func TestSpatialSearch(t *testing.T) {
n = 0
c.Intersects(q3, 0, nil,
func(_ string, _ geojson.Object, _ []float64) bool {
func(_ string, _ geojson.Object, _ *Fields) bool {
n++
return true
},
@ -547,7 +570,7 @@ func TestSpatialSearch(t *testing.T) {
n = 0
c.Intersects(q3, 0, nil,
func(_ string, _ geojson.Object, _ []float64) bool {
func(_ string, _ geojson.Object, _ *Fields) bool {
n++
return n <= 1
},
@ -559,7 +582,7 @@ func TestSpatialSearch(t *testing.T) {
r2, p1, p4, r1, p3, r3, p2,
}
c.Nearby(q4, nil,
func(id string, obj geojson.Object, fields []float64) bool {
func(id string, obj geojson.Object, fields *Fields) bool {
items = append(items, obj)
return true
},
@ -585,7 +608,7 @@ func TestCollectionSparse(t *testing.T) {
var n int
n = 0
c.Within(rect, 1, nil,
func(id string, obj geojson.Object, fields []float64) bool {
func(id string, obj geojson.Object, fields *Fields) bool {
n++
return true
},
@ -594,7 +617,7 @@ func TestCollectionSparse(t *testing.T) {
n = 0
c.Within(rect, 2, nil,
func(id string, obj geojson.Object, fields []float64) bool {
func(id string, obj geojson.Object, fields *Fields) bool {
n++
return true
},
@ -603,7 +626,7 @@ func TestCollectionSparse(t *testing.T) {
n = 0
c.Within(rect, 3, nil,
func(id string, obj geojson.Object, fields []float64) bool {
func(id string, obj geojson.Object, fields *Fields) bool {
n++
return true
},
@ -612,7 +635,7 @@ func TestCollectionSparse(t *testing.T) {
n = 0
c.Within(rect, 3, nil,
func(id string, obj geojson.Object, fields []float64) bool {
func(id string, obj geojson.Object, fields *Fields) bool {
n++
return n <= 30
},
@ -621,7 +644,7 @@ func TestCollectionSparse(t *testing.T) {
n = 0
c.Intersects(rect, 3, nil,
func(id string, _ geojson.Object, _ []float64) bool {
func(id string, _ geojson.Object, _ *Fields) bool {
n++
return true
},
@ -630,7 +653,7 @@ func TestCollectionSparse(t *testing.T) {
n = 0
c.Intersects(rect, 3, nil,
func(id string, _ geojson.Object, _ []float64) bool {
func(id string, _ geojson.Object, _ *Fields) bool {
n++
return n <= 30
},
@ -683,7 +706,7 @@ func TestManyCollections(t *testing.T) {
Min: geometry.Point{X: -180, Y: 30},
Max: geometry.Point{X: 34, Y: 100},
}
col.geoSearch(bbox, func(id string, obj geojson.Object, fields []float64) bool {
col.geoSearch(bbox, func(id string, obj geojson.Object, fields *Fields) bool {
//println(id)
return true
})

40
internal/collection/fields.go Normal file
View File

@ -0,0 +1,40 @@
package collection
import "github.com/tidwall/tile38/internal/collection/item"
// // FieldIter ...
// type FieldIter interface {
// ForEachField(count int, iter func(value float64) bool)
// GetField(index int) float64
// HasFields() bool
// }
// Fields ...
type Fields struct {
item *item.Item
}
// ForEach iterates over each field. The count param is the number of
// iterations. When count is less than zero, then all fields are returns.
func (fields *Fields) ForEach(count int, iter func(value float64) bool) {
if fields == nil || fields.item == nil {
return
}
fields.item.ForEachField(count, iter)
}
// Get returns the value for a field at index. If there is no field at index,
// then zero is returned.
func (fields *Fields) Get(index int) float64 {
if fields == nil || fields.item == nil {
return 0
}
return fields.item.GetField(index)
}
func itemFields(item *item.Item) *Fields {
if item == nil || !item.HasFields() {
return nil
}
return &Fields{item: item}
}

63
internal/collection/item/item.go
View File

@ -39,7 +39,7 @@ func (item *Item) ID() string {
}
// Fields returns the field values
func (item *Item) Fields() []float64 {
func (item *Item) fields() []float64 {
return *(*[]float64)((unsafe.Pointer)(&reflect.SliceHeader{
Data: uintptr(unsafe.Pointer(item.data)),
Len: int(item.fieldsLen) / 8,
@ -104,8 +104,15 @@ func (item *Item) Less(other btree.Item, ctx interface{}) bool {
return item.ID() < other.(*Item).ID()
}
// CopyOverFields overwriting previous fields
func (item *Item) CopyOverFields(values []float64) {
// CopyOverFields overwriting previous fields. Accepts an *Item or []float64
func (item *Item) CopyOverFields(from interface{}) {
var values []float64
switch from := from.(type) {
case *Item:
values = from.fields()
case []float64:
values = from
}
fieldBytes := floatsToBytes(values)
oldData := item.dataBytes()
newData := make([]byte, len(fieldBytes)+int(item.idLen))
@ -153,15 +160,6 @@ func (item *Item) SetField(index int, value float64) (updated bool) {
return true
}
// GetField returns the value for a field at index.
func (item *Item) GetField(index int) float64 {
numFields := int(item.fieldsLen / 8)
if index < numFields {
return getFieldAt(item.data, index)
}
return 0
}
func (item *Item) dataBytes() []byte {
return *(*[]byte)((unsafe.Pointer)(&reflect.SliceHeader{
Data: uintptr(unsafe.Pointer(item.data)),
@ -177,3 +175,44 @@ func floatsToBytes(f []float64) []byte {
Cap: len(f) * 8,
}))
}
// ForEachField iterates over each field. The count param is the number of
// iterations. When count is less than zero, then all fields are returns.
func (item *Item) ForEachField(count int, iter func(value float64) bool) {
if item == nil {
return
}
fields := item.fields()
var n int
if count < 0 {
n = len(fields)
} else {
n = count
}
for i := 0; i < n; i++ {
var field float64
if i < len(fields) {
field = fields[i]
}
if !iter(field) {
return
}
}
}
// GetField returns the value for a field at index.
func (item *Item) GetField(index int) float64 {
if item == nil {
return 0
}
numFields := int(item.fieldsLen / 8)
if index < numFields {
return getFieldAt(item.data, index)
}
return 0
}
// HasFields returns true when item has fields
func (item *Item) HasFields() bool {
return item != nil && item.fieldsLen > 0
}

71
internal/collection/item/item_test.go
View File

@ -19,7 +19,12 @@ func testRandItem(t *testing.T) {
for i := range values {
values[i] = rand.Float64()
}
item := New(key, geojson.NewPoint(geometry.Point{X: 1, Y: 2}))
var item *Item
if rand.Int()%2 == 0 {
item = New(key, geojson.NewSimplePoint(geometry.Point{X: 1, Y: 2}))
} else {
item = New(key, geojson.NewPoint(geometry.Point{X: 1, Y: 2}))
}
if item.ID() != key {
t.Fatalf("expected '%v', got '%v'", key, item.ID())
}
@ -38,7 +43,7 @@ func testRandItem(t *testing.T) {
t.Fatalf("expected '%v', got '%v'", values[i], item.GetField(i))
}
}
fields := item.Fields()
fields := item.fields()
for i := 0; i < len(fields); i++ {
for _, j := range setValues {
if i == j {
@ -65,6 +70,42 @@ func testRandItem(t *testing.T) {
t.Fatal("expected false")
}
}
var fvalues []float64
item.ForEachField(len(values), func(value float64) bool {
fvalues = append(fvalues, value)
return true
})
if !reflect.DeepEqual(values, fvalues) {
t.Fatalf("expected '%v', got '%v'", values, fvalues)
}
fvalues = nil
item.ForEachField(len(values), func(value float64) bool {
if len(fvalues) == 1 {
return false
}
fvalues = append(fvalues, value)
return true
})
if len(values) > 0 && len(fvalues) != 1 {
t.Fatalf("expected '%v', got '%v'", 1, len(fvalues))
}
fvalues = nil
item.ForEachField(-1, func(value float64) bool {
fvalues = append(fvalues, value)
return true
})
if !reflect.DeepEqual(values, fvalues) {
t.Fatalf("expected '%v', got '%v'", 1, len(fvalues))
}
// should not fail, must allow nil receiver
(*Item)(nil).ForEachField(1, nil)
if (*Item)(nil).GetField(1) != 0 {
t.Fatalf("expected '%v', got '%v'", 0, (*Item)(nil).GetField(1))
}
if item.ID() != key {
t.Fatalf("expected '%v', got '%v'", key, item.ID())
}
@ -79,8 +120,22 @@ func testRandItem(t *testing.T) {
if points != 1 {
t.Fatalf("expected '%v', got '%v'", 1, points)
}
if !reflect.DeepEqual(item.Fields(), values) {
t.Fatalf("expected '%v', got '%v'", values, item.Fields())
if !reflect.DeepEqual(item.fields(), values) {
t.Fatalf("expected '%v', got '%v'", values, item.fields())
}
item.CopyOverFields(item)
weight, points = item.WeightAndPoints()
if weight != len(values)*8+len(key)+points*16 {
t.Fatalf("expected '%v', got '%v'", len(values)*8+len(key)+points*16, weight)
}
if points != 1 {
t.Fatalf("expected '%v', got '%v'", 1, points)
}
if !reflect.DeepEqual(item.fields(), values) {
t.Fatalf("expected '%v', got '%v'", values, item.fields())
}
if !item.HasFields() {
t.Fatal("expected true")
}
item.CopyOverFields(nil)
@ -91,12 +146,16 @@ func testRandItem(t *testing.T) {
if points != 1 {
t.Fatalf("expected '%v', got '%v'", 1, points)
}
if len(item.Fields()) != 0 {
t.Fatalf("expected '%#v', got '%#v'", 0, len(item.Fields()))
if len(item.fields()) != 0 {
t.Fatalf("expected '%#v', got '%#v'", 0, len(item.fields()))
}
if item.ID() != key {
t.Fatalf("expected '%v', got '%v'", key, item.ID())
}
if item.HasFields() {
t.Fatal("expected false")
}
}
func TestItem(t *testing.T) {

12
internal/server/aofshrink.go
View File

@ -96,8 +96,9 @@ func (server *Server) aofshrink() {
var exm = server.expires[keys[0]] // the expiration map
var now = time.Now() // used for expiration
var count = 0 // the object count
col.ScanGreaterOrEqual(nextid, false, nil,
func(id string, obj geojson.Object, fields []float64) bool {
func(id string, obj geojson.Object, fields *collection.Fields) bool {
if count == maxids {
// we reached the max number of ids for one batch
nextid = id
@ -110,13 +111,16 @@ func (server *Server) aofshrink() {
values = append(values, "set")
values = append(values, keys[0])
values = append(values, id)
for i, fvalue := range fields {
var fidx int
fields.ForEach(len(fnames), func(fvalue float64) bool {
if fvalue != 0 {
values = append(values, "field")
values = append(values, fnames[i])
values = append(values, fnames[fidx])
values = append(values, strconv.FormatFloat(fvalue, 'f', -1, 64))
}
}
fidx++
return true
})
if exm != nil {
at, ok := exm[id]
if ok {

8
internal/server/crud.go
View File

@ -34,18 +34,16 @@ func (a byField) Swap(i, j int) {
a[i], a[j] = a[j], a[i]
}
func orderFields(fmap map[string]int, fields []float64) []fvt {
func orderFields(fmap map[string]int, fields *collection.Fields) []fvt {
var fv fvt
fvs := make([]fvt, 0, len(fmap))
for field, idx := range fmap {
if idx < len(fields) {
fv.field = field
fv.value = fields[idx]
fv.value = fields.Get(idx)
if fv.value != 0 {
fvs = append(fvs, fv)
}
}
}
sort.Sort(byField(fvs))
return fvs
}
@ -352,7 +350,7 @@ func (server *Server) cmdPdel(msg *Message) (res resp.Value, d commandDetails, e
return
}
now := time.Now()
iter := func(id string, o geojson.Object, fields []float64) bool {
iter := func(id string, o geojson.Object, fields *collection.Fields) bool {
if match, _ := glob.Match(d.pattern, id); match {
d.children = append(d.children, &commandDetails{
command: "del",

5
internal/server/fence.go
View File

@ -9,6 +9,7 @@ import (
"github.com/tidwall/geojson/geo"
"github.com/tidwall/geojson/geometry"
"github.com/tidwall/gjson"
"github.com/tidwall/tile38/internal/collection"
"github.com/tidwall/tile38/internal/glob"
)
@ -284,7 +285,7 @@ func extendRoamMessage(
}
pattern := match.id + fence.roam.scan
iterator := func(
oid string, o geojson.Object, fields []float64,
oid string, o geojson.Object, fields *collection.Fields,
) bool {
if oid == match.id {
return true
@ -371,7 +372,7 @@ func fenceMatchRoam(
Max: geometry.Point{X: maxLon, Y: maxLat},
}
col.Intersects(geojson.NewRect(rect), 0, nil, func(
id string, obj2 geojson.Object, fields []float64,
id string, obj2 geojson.Object, fields *collection.Fields,
) bool {
if c.hasExpired(fence.roam.key, id) {
return true

5
internal/server/scan.go
View File

@ -7,6 +7,7 @@ import (
"github.com/tidwall/geojson"
"github.com/tidwall/resp"
"github.com/tidwall/tile38/internal/collection"
"github.com/tidwall/tile38/internal/glob"
)
@ -67,7 +68,7 @@ func (c *Server) cmdScan(msg *Message) (res resp.Value, err error) {
g := glob.Parse(sw.globPattern, s.desc)
if g.Limits[0] == "" && g.Limits[1] == "" {
sw.col.Scan(s.desc, sw,
func(id string, o geojson.Object, fields []float64) bool {
func(id string, o geojson.Object, fields *collection.Fields) bool {
return sw.writeObject(ScanWriterParams{
id: id,
o: o,
@ -77,7 +78,7 @@ func (c *Server) cmdScan(msg *Message) (res resp.Value, err error) {
)
} else {
sw.col.ScanRange(g.Limits[0], g.Limits[1], s.desc, sw,
func(id string, o geojson.Object, fields []float64) bool {
func(id string, o geojson.Object, fields *collection.Fields) bool {
return sw.writeObject(ScanWriterParams{
id: id,
o: o,

44
internal/server/scanner.go
View File

@ -64,7 +64,7 @@ type scanWriter struct {
type ScanWriterParams struct {
id string
o geojson.Object
fields []float64
fields *collection.Fields
distance float64
noLock bool
ignoreGlobMatch bool
@ -194,7 +194,9 @@ func (sw *scanWriter) writeFoot() {
}
}
func (sw *scanWriter) fieldMatch(fields []float64, o geojson.Object) (fvals []float64, match bool) {
func (sw *scanWriter) fieldMatch(
fields *collection.Fields, o geojson.Object,
) (fvals []float64, match bool) {
var z float64
var gotz bool
fvals = sw.fvals
@ -212,9 +214,7 @@ func (sw *scanWriter) fieldMatch(fields []float64, o geojson.Object) (fvals []fl
var value float64
idx, ok := sw.fmap[where.field]
if ok {
if len(fields) > idx {
value = fields[idx]
}
value = fields.Get(idx)
}
if !where.match(value) {
return
@ -224,9 +224,7 @@ func (sw *scanWriter) fieldMatch(fields []float64, o geojson.Object) (fvals []fl
var value float64
idx, ok := sw.fmap[wherein.field]
if ok {
if len(fields) > idx {
value = fields[idx]
}
value = fields.Get(idx)
}
if !wherein.match(value) {
return
@ -235,11 +233,7 @@ func (sw *scanWriter) fieldMatch(fields []float64, o geojson.Object) (fvals []fl
for _, whereval := range sw.whereevals {
fieldsWithNames := make(map[string]float64)
for field, idx := range sw.fmap {
if idx < len(fields) {
fieldsWithNames[field] = fields[idx]
} else {
fieldsWithNames[field] = 0
}
fieldsWithNames[field] = fields.Get(idx)
}
if !whereval.match(fieldsWithNames) {
return
@ -247,11 +241,7 @@ func (sw *scanWriter) fieldMatch(fields []float64, o geojson.Object) (fvals []fl
}
} else {
for idx := range sw.farr {
var value float64
if len(fields) > idx {
value = fields[idx]
}
sw.fvals[idx] = value
sw.fvals[idx] = fields.Get(idx)
}
for _, where := range sw.wheres {
if where.field == "z" {
@ -285,11 +275,7 @@ func (sw *scanWriter) fieldMatch(fields []float64, o geojson.Object) (fvals []fl
for _, whereval := range sw.whereevals {
fieldsWithNames := make(map[string]float64)
for field, idx := range sw.fmap {
if idx < len(fields) {
fieldsWithNames[field] = fields[idx]
} else {
fieldsWithNames[field] = 0
}
fieldsWithNames[field] = fields.Get(idx)
}
if !whereval.match(fieldsWithNames) {
return
@ -333,7 +319,10 @@ func (sw *scanWriter) Step(n uint64) {
// ok is whether the object passes the test and should be written
// keepGoing is whether there could be more objects to test
func (sw *scanWriter) testObject(id string, o geojson.Object, fields []float64, ignoreGlobMatch bool) (
func (sw *scanWriter) testObject(
id string, o geojson.Object,
fields *collection.Fields, ignoreGlobMatch bool,
) (
ok, keepGoing bool, fieldVals []float64) {
if !ignoreGlobMatch {
match, kg := sw.globMatch(id, o)
@ -384,16 +373,15 @@ func (sw *scanWriter) writeObject(opts ScanWriterParams) bool {
jsfields = `,"fields":{`
var i int
for field, idx := range sw.fmap {
if len(opts.fields) > idx {
if opts.fields[idx] != 0 {
fvalue := opts.fields.Get(idx)
if fvalue != 0 {
if i > 0 {
jsfields += `,`
}
jsfields += jsonString(field) + ":" + strconv.FormatFloat(opts.fields[idx], 'f', -1, 64)
jsfields += jsonString(field) + ":" + strconv.FormatFloat(fvalue, 'f', -1, 64)
i++
}
}
}
jsfields += `}`
}

33
internal/server/search.go
View File

@ -14,6 +14,7 @@ import (
"github.com/tidwall/geojson/geometry"
"github.com/tidwall/resp"
"github.com/tidwall/tile38/internal/bing"
"github.com/tidwall/tile38/internal/collection"
"github.com/tidwall/tile38/internal/glob"
)
@ -370,7 +371,12 @@ func (server *Server) cmdNearby(msg *Message) (res resp.Value, err error) {
}
sw.writeHead()
if sw.col != nil {
iter := func(id string, o geojson.Object, fields []float64, dist float64) bool {
iter := func(
id string,
o geojson.Object,
fields *collection.Fields,
dist float64,
) bool {
meters := 0.0
if s.distance {
meters = geo.DistanceFromHaversine(dist)
@ -398,18 +404,21 @@ func (server *Server) cmdNearby(msg *Message) (res resp.Value, err error) {
type iterItem struct {
id string
o geojson.Object
fields []float64
fields *collection.Fields
dist float64
}
func (server *Server) nearestNeighbors(
s *liveFenceSwitches, sw *scanWriter, target *geojson.Circle,
iter func(id string, o geojson.Object, fields []float64, dist float64,
iter func(
id string, o geojson.Object, fields *collection.Fields, dist float64,
) bool) {
maxDist := target.Haversine()
limit := int(sw.limit)
var items []iterItem
sw.col.Nearby(target, sw, func(id string, o geojson.Object, fields []float64) bool {
sw.col.Nearby(
target, sw,
func(id string, o geojson.Object, fields *collection.Fields) bool {
if server.hasExpired(s.key, id) {
return true
}
@ -481,7 +490,7 @@ func (server *Server) cmdWithinOrIntersects(cmd string, msg *Message) (res resp.
if sw.col != nil {
if cmd == "within" {
sw.col.Within(s.obj, s.sparse, sw, func(
id string, o geojson.Object, fields []float64,
id string, o geojson.Object, fields *collection.Fields,
) bool {
if server.hasExpired(s.key, id) {
return true
@ -497,7 +506,7 @@ func (server *Server) cmdWithinOrIntersects(cmd string, msg *Message) (res resp.
sw.col.Intersects(s.obj, s.sparse, sw, func(
id string,
o geojson.Object,
fields []float64,
fields *collection.Fields,
) bool {
if server.hasExpired(s.key, id) {
return true
@ -579,7 +588,11 @@ func (server *Server) cmdSearch(msg *Message) (res resp.Value, err error) {
g := glob.Parse(sw.globPattern, s.desc)
if g.Limits[0] == "" && g.Limits[1] == "" {
sw.col.SearchValues(s.desc, sw,
func(id string, o geojson.Object, fields []float64) bool {
func(
id string,
o geojson.Object,
fields *collection.Fields,
) bool {
return sw.writeObject(ScanWriterParams{
id: id,
o: o,
@ -593,7 +606,11 @@ func (server *Server) cmdSearch(msg *Message) (res resp.Value, err error) {
// globSingle is only for ID matches, not values.
sw.globSingle = false
sw.col.SearchValuesRange(g.Limits[0], g.Limits[1], s.desc, sw,
func(id string, o geojson.Object, fields []float64) bool {
func(
id string,
o geojson.Object,
fields *collection.Fields,
) bool {
return sw.writeObject(ScanWriterParams{
id: id,
o: o,

6
internal/server/server.go
View File

@ -50,11 +50,11 @@ type commandDetails struct {
command string // client command, like "SET" or "DEL"
key, id string // collection key and object id of object
newKey string // new key, for RENAME command
fmap map[string]int // map of field names to value indexes
obj geojson.Object // new object
fields []float64 // array of field values
fmap map[string]int // map of field names to value indexes
fields *collection.Fields // array of field values
oldObj geojson.Object // previous object, if any
oldFields []float64 // previous object field values
oldFields *collection.Fields // previous object field values
updated bool // object was updated
timestamp time.Time // timestamp when the update occured
parent bool // when true, only children are forwarded