mirror
/
tile38
mirror of https://github.com/tidwall/tile38.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Reversed fields/id grouping order

This commit is contained in:
tidwall 2019-02-12 16:36:05 -07:00
parent b5dcb18c54
commit 448aa347e6
4 changed files with 82 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
internal/collection/collection.go
View File

@ -141,17 +141,16 @@ func (c *Collection) Set(
// remove old item from indexes // remove old item from indexes
c.delItem(oldItem) c.delItem(oldItem)
if len(oldItem.fields()) > 0 { if len(oldItem.fields()) > 0 {
// merge old and new fields // merge old and new fields
oldFields = oldItem.fields() oldFields = oldItem.fields()
item.directSetFields(oldFields) item.directCopyFields(oldFields)
} }
} }
if fields == nil && len(values) > 0 { if fields == nil && len(values) > 0 {
// directly set the field values, from copy // directly set the field values, from copy
item.directSetFields(values) item.directCopyFields(values)
} else if len(fields) > 0 { } else if len(fields) > 0 {
// add new field to new item // add new field to new item
c.setFields(item, fields, values, false) c.setFields(item, fields, values, false)

100
internal/collection/collection_test.go
View File

@ -38,55 +38,55 @@ func bounds(c *Collection) geometry.Rect {
} }
} }
func TestStuff(t *testing.T) { // func TestStuff(t *testing.T) {
c := New() // c := New()
key := "str" // key := "str"
str1 := String("hello") // str1 := String("hello")
str2 := String("jello") // str2 := String("jello")
{ // {
println("A") // println("A")
oldObj, _, _ := c.Set(key, str1, []string{"a", "b", "c"}, nil) // oldObj, _, _ := c.Set(key, str1, []string{"a", "b", "c"}, nil)
println("B") // println("B")
expect(t, oldObj == nil) // expect(t, oldObj == nil)
} // }
{ // {
println("C") // println("C")
oldObj, _, _ := c.Set(key, str2, nil, nil) //[]float64{4, 5, 6}) // oldObj, _, _ := c.Set(key, str2, nil, nil) //[]float64{4, 5, 6})
println("D") // println("D")
expect(t, oldObj == str1) // expect(t, oldObj == str1)
// expect(t, reflect.DeepEqual(oldFlds, nil)) //[]float64{1, 2, 3})) // // expect(t, reflect.DeepEqual(oldFlds, nil)) //[]float64{1, 2, 3}))
// expect(t, reflect.DeepEqual(newFlds, []float64{1, 2, 3, 4, 5, 6})) // // expect(t, reflect.DeepEqual(newFlds, []float64{1, 2, 3, 4, 5, 6}))
} // }
{ // {
// fValues := []float64{7, 8, 9, 10, 11, 12} // // fValues := []float64{7, 8, 9, 10, 11, 12}
println("E") // println("E")
oldObj, _, _ := c.Set(key, str1, nil, nil) // oldObj, _, _ := c.Set(key, str1, nil, nil)
println("F") // println("F")
expect(t, oldObj == str2) // expect(t, oldObj == str2)
// expect(t, reflect.DeepEqual(oldFlds, []float64{4, 5, 6})) // // expect(t, reflect.DeepEqual(oldFlds, []float64{4, 5, 6}))
// expect(t, reflect.DeepEqual(newFlds, []float64{7, 8, 9, 10, 11, 12})) // // expect(t, reflect.DeepEqual(newFlds, []float64{7, 8, 9, 10, 11, 12}))
} // }
// var old geojson.Object // // var old geojson.Object
// c := New() // // c := New()
// old, _, _ = c.Set("hello1", String("world1"), nil, nil) // // old, _, _ = c.Set("hello1", String("world1"), nil, nil)
// expect(t, old == nil) // // expect(t, old == nil)
// old, _, _ = c.Set("hello2", String("world2"), nil, nil) // // old, _, _ = c.Set("hello2", String("world2"), nil, nil)
// expect(t, old == nil) // // expect(t, old == nil)
// old, _, _ = c.Set("hello3", String("world3"), nil, nil) // // old, _, _ = c.Set("hello3", String("world3"), nil, nil)
// expect(t, old == nil) // // expect(t, old == nil)
// old, _, _ = c.Set("hello4", String("world4"), nil, nil) // // old, _, _ = c.Set("hello4", String("world4"), nil, nil)
// expect(t, old == nil) // // expect(t, old == nil)
// old, _, _ = c.Set("hello1", String("planet1"), nil, nil) // // old, _, _ = c.Set("hello1", String("planet1"), nil, nil)
// expect(t, old == String("world1")) // // expect(t, old == String("world1"))
// old, _, _ = c.Set("hello2", String("planet2"), nil, nil) // // old, _, _ = c.Set("hello2", String("planet2"), nil, nil)
// expect(t, old == String("world2")) // // expect(t, old == String("world2"))
// old, _, _ = c.Set("hello3", String("planet3"), nil, nil) // // old, _, _ = c.Set("hello3", String("planet3"), nil, nil)
// expect(t, old == String("world3")) // // expect(t, old == String("world3"))
// old, _, _ = c.Set("hello4", String("planet4"), nil, nil) // // old, _, _ = c.Set("hello4", String("planet4"), nil, nil)
// expect(t, old == String("world4")) // // expect(t, old == String("world4"))
} // }
func TestCollectionNewCollection(t *testing.T) { func TestCollectionNewCollection(t *testing.T) {
const numItems = 10000 const numItems = 10000
@ -168,9 +168,7 @@ func TestCollectionSet(t *testing.T) {
{ {
fNames := []string{"a", "b", "c"} fNames := []string{"a", "b", "c"}
fValues := []float64{1, 2, 3} fValues := []float64{1, 2, 3}
println("A")
oldObj, oldFlds, newFlds := c.Set("str", str1, fNames, fValues) oldObj, oldFlds, newFlds := c.Set("str", str1, fNames, fValues)
println("B")
expect(t, oldObj == nil) expect(t, oldObj == nil)
expect(t, len(oldFlds) == 0) expect(t, len(oldFlds) == 0)
expect(t, reflect.DeepEqual(newFlds, fValues)) expect(t, reflect.DeepEqual(newFlds, fValues))
@ -178,18 +176,14 @@ func TestCollectionSet(t *testing.T) {
{ {
fNames := []string{"d", "e", "f"} fNames := []string{"d", "e", "f"}
fValues := []float64{4, 5, 6} fValues := []float64{4, 5, 6}
println("C")
oldObj, oldFlds, newFlds := c.Set("str", str2, fNames, fValues) oldObj, oldFlds, newFlds := c.Set("str", str2, fNames, fValues)
println("D")
expect(t, oldObj == str1) expect(t, oldObj == str1)
expect(t, reflect.DeepEqual(oldFlds, []float64{1, 2, 3})) expect(t, reflect.DeepEqual(oldFlds, []float64{1, 2, 3}))
expect(t, reflect.DeepEqual(newFlds, []float64{1, 2, 3, 4, 5, 6})) expect(t, reflect.DeepEqual(newFlds, []float64{1, 2, 3, 4, 5, 6}))
} }
{ {
fValues := []float64{7, 8, 9, 10, 11, 12} fValues := []float64{7, 8, 9, 10, 11, 12}
println("E")
oldObj, oldFlds, newFlds := c.Set("str", str1, nil, fValues) oldObj, oldFlds, newFlds := c.Set("str", str1, nil, fValues)
println("F")
expect(t, oldObj == str2) expect(t, oldObj == str2)
expect(t, reflect.DeepEqual(oldFlds, []float64{1, 2, 3, 4, 5, 6})) expect(t, reflect.DeepEqual(oldFlds, []float64{1, 2, 3, 4, 5, 6}))
expect(t, reflect.DeepEqual(newFlds, []float64{7, 8, 9, 10, 11, 12})) expect(t, reflect.DeepEqual(newFlds, []float64{7, 8, 9, 10, 11, 12}))

43
internal/collection/item.go
View File

@ -10,21 +10,21 @@ import (
type itemT struct { type itemT struct {
obj geojson.Object obj geojson.Object
idLen uint32 // id block size in bytes
fieldsLen uint32 // fields block size in bytes, not num of fields fieldsLen uint32 // fields block size in bytes, not num of fields
idLen uint32 // id block size in bytes
data unsafe.Pointer data unsafe.Pointer
} }
func (item *itemT) id() string { func (item *itemT) id() string {
return *(*string)((unsafe.Pointer)(&reflect.StringHeader{ return *(*string)((unsafe.Pointer)(&reflect.StringHeader{
Data: uintptr(unsafe.Pointer(item.data)), Data: uintptr(unsafe.Pointer(item.data)) + uintptr(item.fieldsLen),
Len: int(item.idLen), Len: int(item.idLen),
})) }))
} }
func (item *itemT) fields() []float64 { func (item *itemT) fields() []float64 {
return *(*[]float64)((unsafe.Pointer)(&reflect.SliceHeader{ return *(*[]float64)((unsafe.Pointer)(&reflect.SliceHeader{
Data: uintptr(unsafe.Pointer(item.data)) + uintptr(item.idLen), Data: uintptr(unsafe.Pointer(item.data)),
Len: int(item.fieldsLen) / 8, Len: int(item.fieldsLen) / 8,
Cap: int(item.fieldsLen) / 8, Cap: int(item.fieldsLen) / 8,
})) }))
@ -74,15 +74,24 @@ func (item *itemT) Less(other btree.Item, ctx interface{}) bool {
return item.id() < other.(*itemT).id() return item.id() < other.(*itemT).id()
} }
// directSetFields copies fields, overwriting previous fields func floatsToBytes(f []float64) []byte {
func (item *itemT) directSetFields(fields []float64) { return *(*[]byte)((unsafe.Pointer)(&reflect.SliceHeader{
n := int(item.idLen) + len(fields)*8 Data: ((*reflect.SliceHeader)(unsafe.Pointer(&f))).Data,
item.fieldsLen = uint32(len(fields) * 8) Len: len(f) * 8,
if n > 0 { Cap: len(f) * 8,
newData := make([]byte, int(item.idLen)+len(fields)*8) }))
}
// directCopyFields copies fields, overwriting previous fields
func (item *itemT) directCopyFields(fields []float64) {
fieldBytes := floatsToBytes(fields)
oldData := item.dataBytes()
newData := make([]byte, len(fieldBytes)+int(item.idLen))
copy(newData, fieldBytes)
copy(newData[len(fieldBytes):], oldData[item.fieldsLen:])
item.fieldsLen = uint32(len(fieldBytes))
if len(newData) > 0 {
item.data = unsafe.Pointer(&newData[0]) item.data = unsafe.Pointer(&newData[0])
copy(newData, item.id())
copy(item.fields(), fields)
} else { } else {
item.data = nil item.data = nil
} }
@ -100,19 +109,19 @@ func (c *Collection) setField(
if idx >= len(itemFields) { if idx >= len(itemFields) {
// make room for new field // make room for new field
itemBytes := item.dataBytes()
oldLen := len(itemFields) oldLen := len(itemFields)
// print(c.weight) data := make([]byte, (idx+1)*8+int(item.idLen))
data := make([]byte, int(item.idLen)+(idx+1)*8)
copy(data, item.dataBytes()) copy(data, itemBytes[:item.fieldsLen])
copy(data[(idx+1)*8:], itemBytes[item.fieldsLen:])
item.fieldsLen = uint32((idx + 1) * 8) item.fieldsLen = uint32((idx + 1) * 8)
item.data = unsafe.Pointer(&data[0]) item.data = unsafe.Pointer(&data[0])
itemFields := item.fields() itemFields := item.fields()
if updateWeight { if updateWeight {
c.weight += (len(itemFields) - oldLen) * 8 c.weight += (len(itemFields) - oldLen) * 8
} }
// print(":")
// print(c.weight)
// println()
itemFields[idx] = fieldValue itemFields[idx] = fieldValue
updated = true updated = true
} else if itemFields[idx] != fieldValue { } else if itemFields[idx] != fieldValue {

13
internal/collection/ptrbtree/btree.go
View File

@ -15,16 +15,17 @@ type btreeItem struct {
// keyedItem must match layout of ../collection/itemT, otherwise // keyedItem must match layout of ../collection/itemT, otherwise
// there's a risk for memory corruption. // there's a risk for memory corruption.
type keyedItem struct { type keyedItem struct {
obj interface{} obj interface{}
keyLen uint32 fieldLen uint32
_ uint32 keyLen uint32
data unsafe.Pointer data unsafe.Pointer
} }
func (v btreeItem) key() string { func (v btreeItem) key() string {
return *(*string)((unsafe.Pointer)(&reflect.StringHeader{ return *(*string)((unsafe.Pointer)(&reflect.StringHeader{
Data: uintptr(unsafe.Pointer((*keyedItem)(v.ptr).data)), Data: uintptr(unsafe.Pointer((*keyedItem)(v.ptr).data)) +
Len: int((*keyedItem)(v.ptr).keyLen), uintptr((*keyedItem)(v.ptr).fieldLen),
Len: int((*keyedItem)(v.ptr).keyLen),
})) }))
} }