mirror
/
tile38
mirror of https://github.com/tidwall/tile38.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Refactor field optimizations

This commit is contained in:
tidwall 2019-02-13 12:43:38 -07:00
parent 448aa347e6
commit 9d8ce14ffd
8 changed files with 744 additions and 452 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

105
internal/collection/ptrbtree/btree.go → internal/collection/btree/btree.go
View File

@ -1,37 +1,15 @@
package ptrbtree // Package btree is designed to work specifically with
// the Tile38 collection/item.Item type.
package btree
import ( import "github.com/tidwall/tile38/internal/collection/item"
"reflect"
"unsafe"
)
const maxItems = 31 // use an odd number const maxItems = 31 // use an odd number
const minItems = maxItems * 40 / 100 const minItems = maxItems * 40 / 100
type btreeItem struct {
ptr unsafe.Pointer
}
// keyedItem must match layout of ../collection/itemT, otherwise
// there's a risk for memory corruption.
type keyedItem struct {
obj interface{}
fieldLen uint32
keyLen uint32
data unsafe.Pointer
}
func (v btreeItem) key() string {
return *(*string)((unsafe.Pointer)(&reflect.StringHeader{
Data: uintptr(unsafe.Pointer((*keyedItem)(v.ptr).data)) +
uintptr((*keyedItem)(v.ptr).fieldLen),
Len: int((*keyedItem)(v.ptr).keyLen),
}))
}
type node struct { type node struct {
numItems int numItems int
items [maxItems]btreeItem items [maxItems]*item.Item
children [maxItems + 1]*node children [maxItems + 1]*node
} }
@ -47,29 +25,28 @@ func (n *node) find(key string) (index int, found bool) {
i, j := 0, n.numItems i, j := 0, n.numItems
for i < j { for i < j {
h := i + (j-i)/2 h := i + (j-i)/2
if key >= n.items[h].key() { if key >= n.items[h].ID() {
i = h + 1 i = h + 1
} else { } else {
j = h j = h
} }
} }
if i > 0 && n.items[i-1].key() >= key { if i > 0 && n.items[i-1].ID() >= key {
return i - 1, true return i - 1, true
} }
return i, false return i, false
} }
// Set or replace a value for a key // Set or replace a value for a key
func (tr *BTree) Set(ptr unsafe.Pointer) (prev unsafe.Pointer, replaced bool) { func (tr *BTree) Set(item *item.Item) (prev *item.Item, replaced bool) {
newItem := btreeItem{ptr}
if tr.root == nil { if tr.root == nil {
tr.root = new(node) tr.root = new(node)
tr.root.items[0] = newItem tr.root.items[0] = item
tr.root.numItems = 1 tr.root.numItems = 1
tr.length = 1 tr.length = 1
return return
} }
prev, replaced = tr.root.set(newItem, tr.height) prev, replaced = tr.root.set(item, tr.height)
if replaced { if replaced {
return return
} }
@ -87,7 +64,7 @@ func (tr *BTree) Set(ptr unsafe.Pointer) (prev unsafe.Pointer, replaced bool) {
return return
} }
func (n *node) split(height int) (right *node, median btreeItem) { func (n *node) split(height int) (right *node, median *item.Item) {
right = new(node) right = new(node)
median = n.items[maxItems/2] median = n.items[maxItems/2]
copy(right.items[:maxItems/2], n.items[maxItems/2+1:]) copy(right.items[:maxItems/2], n.items[maxItems/2+1:])
@ -101,16 +78,16 @@ func (n *node) split(height int) (right *node, median btreeItem) {
} }
} }
for i := maxItems / 2; i < maxItems; i++ { for i := maxItems / 2; i < maxItems; i++ {
n.items[i] = btreeItem{} n.items[i] = nil
} }
n.numItems = maxItems / 2 n.numItems = maxItems / 2
return return
} }
func (n *node) set(newItem btreeItem, height int) (prev unsafe.Pointer, replaced bool) { func (n *node) set(newItem *item.Item, height int) (prev *item.Item, replaced bool) {
i, found := n.find(newItem.key()) i, found := n.find(newItem.ID())
if found { if found {
prev = n.items[i].ptr prev = n.items[i]
n.items[i] = newItem n.items[i] = newItem
return prev, true return prev, true
} }
@ -138,16 +115,16 @@ func (n *node) set(newItem btreeItem, height int) (prev unsafe.Pointer, replaced
} }
// Scan all items in tree // Scan all items in tree
func (tr *BTree) Scan(iter func(ptr unsafe.Pointer) bool) { func (tr *BTree) Scan(iter func(item *item.Item) bool) {
if tr.root != nil { if tr.root != nil {
tr.root.scan(iter, tr.height) tr.root.scan(iter, tr.height)
} }
} }
func (n *node) scan(iter func(ptr unsafe.Pointer) bool, height int) bool { func (n *node) scan(iter func(item *item.Item) bool, height int) bool {
if height == 0 { if height == 0 {
for i := 0; i < n.numItems; i++ { for i := 0; i < n.numItems; i++ {
if !iter(n.items[i].ptr) { if !iter(n.items[i]) {
return false return false
} }
} }
@ -157,7 +134,7 @@ func (n *node) scan(iter func(ptr unsafe.Pointer) bool, height int) bool {
if !n.children[i].scan(iter, height-1) { if !n.children[i].scan(iter, height-1) {
return false return false
} }
if !iter(n.items[i].ptr) { if !iter(n.items[i]) {
return false return false
} }
} }
@ -165,17 +142,17 @@ func (n *node) scan(iter func(ptr unsafe.Pointer) bool, height int) bool {
} }
// Get a value for key // Get a value for key
func (tr *BTree) Get(key string) (ptr unsafe.Pointer, gotten bool) { func (tr *BTree) Get(key string) (item *item.Item, gotten bool) {
if tr.root == nil { if tr.root == nil {
return return
} }
return tr.root.get(key, tr.height) return tr.root.get(key, tr.height)
} }
func (n *node) get(key string, height int) (ptr unsafe.Pointer, gotten bool) { func (n *node) get(key string, height int) (item *item.Item, gotten bool) {
i, found := n.find(key) i, found := n.find(key)
if found { if found {
return n.items[i].ptr, true return n.items[i], true
} }
if height == 0 { if height == 0 {
return nil, false return nil, false
@ -189,16 +166,16 @@ func (tr *BTree) Len() int {
} }
// Delete a value for a key // Delete a value for a key
func (tr *BTree) Delete(key string) (prev unsafe.Pointer, deleted bool) { func (tr *BTree) Delete(key string) (prev *item.Item, deleted bool) {
if tr.root == nil { if tr.root == nil {
return return
} }
var prevItem btreeItem var prevItem *item.Item
prevItem, deleted = tr.root.delete(false, key, tr.height) prevItem, deleted = tr.root.delete(false, key, tr.height)
if !deleted { if !deleted {
return return
} }
prev = prevItem.ptr prev = prevItem
if tr.root.numItems == 0 { if tr.root.numItems == 0 {
tr.root = tr.root.children[0] tr.root = tr.root.children[0]
tr.height-- tr.height--
@ -212,7 +189,7 @@ func (tr *BTree) Delete(key string) (prev unsafe.Pointer, deleted bool) {
} }
func (n *node) delete(max bool, key string, height int) ( func (n *node) delete(max bool, key string, height int) (
prev btreeItem, deleted bool, prev *item.Item, deleted bool,
) { ) {
i, found := 0, false i, found := 0, false
if max { if max {
@ -225,12 +202,12 @@ func (n *node) delete(max bool, key string, height int) (
prev = n.items[i] prev = n.items[i]
// found the items at the leaf, remove it and return. // found the items at the leaf, remove it and return.
copy(n.items[i:], n.items[i+1:n.numItems]) copy(n.items[i:], n.items[i+1:n.numItems])
n.items[n.numItems-1] = btreeItem{} n.items[n.numItems-1] = nil
n.children[n.numItems] = nil n.children[n.numItems] = nil
n.numItems-- n.numItems--
return prev, true return prev, true
} }
return btreeItem{}, false return nil, false
} }
if found { if found {
@ -254,7 +231,7 @@ func (n *node) delete(max bool, key string, height int) (
i-- i--
} }
if n.children[i].numItems+n.children[i+1].numItems+1 < maxItems { if n.children[i].numItems+n.children[i+1].numItems+1 < maxItems {
// merge left + btreeItem + right // merge left + *item.Item + right
n.children[i].items[n.children[i].numItems] = n.items[i] n.children[i].items[n.children[i].numItems] = n.items[i]
copy(n.children[i].items[n.children[i].numItems+1:], copy(n.children[i].items[n.children[i].numItems+1:],
n.children[i+1].items[:n.children[i+1].numItems]) n.children[i+1].items[:n.children[i+1].numItems])
@ -265,7 +242,7 @@ func (n *node) delete(max bool, key string, height int) (
n.children[i].numItems += n.children[i+1].numItems + 1 n.children[i].numItems += n.children[i+1].numItems + 1
copy(n.items[i:], n.items[i+1:n.numItems]) copy(n.items[i:], n.items[i+1:n.numItems])
copy(n.children[i+1:], n.children[i+2:n.numItems+1]) copy(n.children[i+1:], n.children[i+2:n.numItems+1])
n.items[n.numItems] = btreeItem{} n.items[n.numItems] = nil
n.children[n.numItems+1] = nil n.children[n.numItems+1] = nil
n.numItems-- n.numItems--
} else if n.children[i].numItems > n.children[i+1].numItems { } else if n.children[i].numItems > n.children[i+1].numItems {
@ -283,7 +260,7 @@ func (n *node) delete(max bool, key string, height int) (
} }
n.children[i+1].numItems++ n.children[i+1].numItems++
n.items[i] = n.children[i].items[n.children[i].numItems-1] n.items[i] = n.children[i].items[n.children[i].numItems-1]
n.children[i].items[n.children[i].numItems-1] = btreeItem{} n.children[i].items[n.children[i].numItems-1] = nil
if height > 1 { if height > 1 {
n.children[i].children[n.children[i].numItems] = nil n.children[i].children[n.children[i].numItems] = nil
} }
@ -310,13 +287,13 @@ func (n *node) delete(max bool, key string, height int) (
} }
// Ascend the tree within the range [pivot, last] // Ascend the tree within the range [pivot, last]
func (tr *BTree) Ascend(pivot string, iter func(ptr unsafe.Pointer) bool) { func (tr *BTree) Ascend(pivot string, iter func(item *item.Item) bool) {
if tr.root != nil { if tr.root != nil {
tr.root.ascend(pivot, iter, tr.height) tr.root.ascend(pivot, iter, tr.height)
} }
} }
func (n *node) ascend(pivot string, iter func(ptr unsafe.Pointer) bool, height int) bool { func (n *node) ascend(pivot string, iter func(item *item.Item) bool, height int) bool {
i, found := n.find(pivot) i, found := n.find(pivot)
if !found { if !found {
if height > 0 { if height > 0 {
@ -326,7 +303,7 @@ func (n *node) ascend(pivot string, iter func(ptr unsafe.Pointer) bool, height i
} }
} }
for ; i < n.numItems; i++ { for ; i < n.numItems; i++ {
if !iter(n.items[i].ptr) { if !iter(n.items[i]) {
return false return false
} }
if height > 0 { if height > 0 {
@ -339,16 +316,16 @@ func (n *node) ascend(pivot string, iter func(ptr unsafe.Pointer) bool, height i
} }
// Reverse all items in tree // Reverse all items in tree
func (tr *BTree) Reverse(iter func(ptr unsafe.Pointer) bool) { func (tr *BTree) Reverse(iter func(item *item.Item) bool) {
if tr.root != nil { if tr.root != nil {
tr.root.reverse(iter, tr.height) tr.root.reverse(iter, tr.height)
} }
} }
func (n *node) reverse(iter func(ptr unsafe.Pointer) bool, height int) bool { func (n *node) reverse(iter func(item *item.Item) bool, height int) bool {
if height == 0 { if height == 0 {
for i := n.numItems - 1; i >= 0; i-- { for i := n.numItems - 1; i >= 0; i-- {
if !iter(n.items[i].ptr) { if !iter(n.items[i]) {
return false return false
} }
} }
@ -358,7 +335,7 @@ func (n *node) reverse(iter func(ptr unsafe.Pointer) bool, height int) bool {
return false return false
} }
for i := n.numItems - 1; i >= 0; i-- { for i := n.numItems - 1; i >= 0; i-- {
if !iter(n.items[i].ptr) { if !iter(n.items[i]) {
return false return false
} }
if !n.children[i].reverse(iter, height-1) { if !n.children[i].reverse(iter, height-1) {
@ -371,14 +348,14 @@ func (n *node) reverse(iter func(ptr unsafe.Pointer) bool, height int) bool {
// Descend the tree within the range [pivot, first] // Descend the tree within the range [pivot, first]
func (tr *BTree) Descend( func (tr *BTree) Descend(
pivot string, pivot string,
iter func(ptr unsafe.Pointer) bool, iter func(item *item.Item) bool,
) { ) {
if tr.root != nil { if tr.root != nil {
tr.root.descend(pivot, iter, tr.height) tr.root.descend(pivot, iter, tr.height)
} }
} }
func (n *node) descend(pivot string, iter func(ptr unsafe.Pointer) bool, height int) bool { func (n *node) descend(pivot string, iter func(item *item.Item) bool, height int) bool {
i, found := n.find(pivot) i, found := n.find(pivot)
if !found { if !found {
if height > 0 { if height > 0 {
@ -389,7 +366,7 @@ func (n *node) descend(pivot string, iter func(ptr unsafe.Pointer) bool, height
i-- i--
} }
for ; i >= 0; i-- { for ; i >= 0; i-- {
if !iter(n.items[i].ptr) { if !iter(n.items[i]) {
return false return false
} }
if height > 0 { if height > 0 {

175
internal/collection/ptrbtree/btree_test.go → internal/collection/btree/btree_test.go
View File

@ -1,35 +1,19 @@
package ptrbtree package btree
import ( import (
"encoding/json"
"fmt" "fmt"
"math/rand" "math/rand"
"sort" "sort"
"strings" "strings"
"testing" "testing"
"time" "time"
"unsafe"
"github.com/tidwall/geojson"
"github.com/tidwall/geojson/geometry"
"github.com/tidwall/tile38/internal/collection/item"
) )
func makeItem(key string, obj interface{}) unsafe.Pointer {
item := new(keyedItem)
item.obj = obj
if len(key) > 0 {
data := make([]byte, len(key))
copy(data, key)
item.keyLen = uint32(len(key))
item.data = unsafe.Pointer(&data[0])
}
return unsafe.Pointer(item)
}
func itemKey(ptr unsafe.Pointer) string {
return (btreeItem{ptr}).key()
}
func itemValue(ptr unsafe.Pointer) interface{} {
return (*keyedItem)(ptr).obj
}
func init() { func init() {
seed := time.Now().UnixNano() seed := time.Now().UnixNano()
fmt.Printf("seed: %d\n", seed) fmt.Printf("seed: %d\n", seed)
@ -63,12 +47,12 @@ func (n *node) print(level, height int) {
n.children[i].print(level+1, height-1) n.children[i].print(level+1, height-1)
} }
if height > 0 || (height == 0 && !flatLeaf) { if height > 0 || (height == 0 && !flatLeaf) {
fmt.Printf("%s%v\n", strings.Repeat(" ", level), n.items[i].key()) fmt.Printf("%s%v\n", strings.Repeat(" ", level), n.items[i].ID())
} else { } else {
if i > 0 { if i > 0 {
fmt.Printf(",") fmt.Printf(",")
} }
fmt.Printf("%s", n.items[i].key()) fmt.Printf("%s", n.items[i].ID())
} }
} }
if height == 0 && flatLeaf { if height == 0 && flatLeaf {
@ -117,7 +101,7 @@ func stringsEquals(a, b []string) bool {
func TestDescend(t *testing.T) { func TestDescend(t *testing.T) {
var tr BTree var tr BTree
var count int var count int
tr.Descend("1", func(ptr unsafe.Pointer) bool { tr.Descend("1", func(item *item.Item) bool {
count++ count++
return true return true
}) })
@ -127,26 +111,26 @@ func TestDescend(t *testing.T) {
var keys []string var keys []string
for i := 0; i < 1000; i += 10 { for i := 0; i < 1000; i += 10 {
keys = append(keys, fmt.Sprintf("%03d", i)) keys = append(keys, fmt.Sprintf("%03d", i))
tr.Set(makeItem(keys[len(keys)-1], nil)) tr.Set(item.New(keys[len(keys)-1], nil))
} }
var exp []string var exp []string
tr.Reverse(func(ptr unsafe.Pointer) bool { tr.Reverse(func(item *item.Item) bool {
exp = append(exp, itemKey(ptr)) exp = append(exp, item.ID())
return true return true
}) })
for i := 999; i >= 0; i-- { for i := 999; i >= 0; i-- {
var key string var key string
key = fmt.Sprintf("%03d", i) key = fmt.Sprintf("%03d", i)
var all []string var all []string
tr.Descend(key, func(ptr unsafe.Pointer) bool { tr.Descend(key, func(item *item.Item) bool {
all = append(all, itemKey(ptr)) all = append(all, item.ID())
return true return true
}) })
for len(exp) > 0 && key < exp[0] { for len(exp) > 0 && key < exp[0] {
exp = exp[1:] exp = exp[1:]
} }
var count int var count int
tr.Descend(key, func(ptr unsafe.Pointer) bool { tr.Descend(key, func(item *item.Item) bool {
if count == (i+1)%maxItems { if count == (i+1)%maxItems {
return false return false
} }
@ -168,7 +152,7 @@ func TestDescend(t *testing.T) {
func TestAscend(t *testing.T) { func TestAscend(t *testing.T) {
var tr BTree var tr BTree
var count int var count int
tr.Ascend("1", func(ptr unsafe.Pointer) bool { tr.Ascend("1", func(item *item.Item) bool {
count++ count++
return true return true
}) })
@ -178,7 +162,7 @@ func TestAscend(t *testing.T) {
var keys []string var keys []string
for i := 0; i < 1000; i += 10 { for i := 0; i < 1000; i += 10 {
keys = append(keys, fmt.Sprintf("%03d", i)) keys = append(keys, fmt.Sprintf("%03d", i))
tr.Set(makeItem(keys[len(keys)-1], nil)) tr.Set(item.New(keys[len(keys)-1], nil))
} }
exp := keys exp := keys
for i := -1; i < 1000; i++ { for i := -1; i < 1000; i++ {
@ -189,8 +173,8 @@ func TestAscend(t *testing.T) {
key = fmt.Sprintf("%03d", i) key = fmt.Sprintf("%03d", i)
} }
var all []string var all []string
tr.Ascend(key, func(ptr unsafe.Pointer) bool { tr.Ascend(key, func(item *item.Item) bool {
all = append(all, itemKey(ptr)) all = append(all, item.ID())
return true return true
}) })
@ -198,7 +182,7 @@ func TestAscend(t *testing.T) {
exp = exp[1:] exp = exp[1:]
} }
var count int var count int
tr.Ascend(key, func(ptr unsafe.Pointer) bool { tr.Ascend(key, func(item *item.Item) bool {
if count == (i+1)%maxItems { if count == (i+1)%maxItems {
return false return false
} }
@ -221,7 +205,7 @@ func TestBTree(t *testing.T) {
// insert all items // insert all items
for _, key := range keys { for _, key := range keys {
value, replaced := tr.Set(makeItem(key, key)) value, replaced := tr.Set(item.New(key, testString(key)))
if replaced { if replaced {
t.Fatal("expected false") t.Fatal("expected false")
} }
@ -241,7 +225,7 @@ func TestBTree(t *testing.T) {
if !gotten { if !gotten {
t.Fatal("expected true") t.Fatal("expected true")
} }
if value == nil || itemValue(value) != key { if value == nil || value.Obj().String() != key {
t.Fatalf("expected '%v', got '%v'", key, value) t.Fatalf("expected '%v', got '%v'", key, value)
} }
} }
@ -249,15 +233,15 @@ func TestBTree(t *testing.T) {
// scan all items // scan all items
var last string var last string
all := make(map[string]interface{}) all := make(map[string]interface{})
tr.Scan(func(ptr unsafe.Pointer) bool { tr.Scan(func(item *item.Item) bool {
if itemKey(ptr) <= last { if item.ID() <= last {
t.Fatal("out of order") t.Fatal("out of order")
} }
if itemValue(ptr).(string) != itemKey(ptr) { if item.Obj().String() != item.ID() {
t.Fatalf("mismatch") t.Fatalf("mismatch")
} }
last = itemKey(ptr) last = item.ID()
all[itemKey(ptr)] = itemValue(ptr) all[item.ID()] = item.Obj().String()
return true return true
}) })
if len(all) != len(keys) { if len(all) != len(keys) {
@ -267,15 +251,15 @@ func TestBTree(t *testing.T) {
// reverse all items // reverse all items
var prev string var prev string
all = make(map[string]interface{}) all = make(map[string]interface{})
tr.Reverse(func(ptr unsafe.Pointer) bool { tr.Reverse(func(item *item.Item) bool {
if prev != "" && itemKey(ptr) >= prev { if prev != "" && item.ID() >= prev {
t.Fatal("out of order") t.Fatal("out of order")
} }
if itemValue(ptr).(string) != itemKey(ptr) { if item.Obj().String() != item.ID() {
t.Fatalf("mismatch") t.Fatalf("mismatch")
} }
prev = itemKey(ptr) prev = item.ID()
all[itemKey(ptr)] = itemValue(ptr) all[item.ID()] = item.Obj().String()
return true return true
}) })
if len(all) != len(keys) { if len(all) != len(keys) {
@ -294,7 +278,7 @@ func TestBTree(t *testing.T) {
// scan and quit at various steps // scan and quit at various steps
for i := 0; i < 100; i++ { for i := 0; i < 100; i++ {
var j int var j int
tr.Scan(func(ptr unsafe.Pointer) bool { tr.Scan(func(item *item.Item) bool {
if j == i { if j == i {
return false return false
} }
@ -306,7 +290,7 @@ func TestBTree(t *testing.T) {
// reverse and quit at various steps // reverse and quit at various steps
for i := 0; i < 100; i++ { for i := 0; i < 100; i++ {
var j int var j int
tr.Reverse(func(ptr unsafe.Pointer) bool { tr.Reverse(func(item *item.Item) bool {
if j == i { if j == i {
return false return false
} }
@ -321,7 +305,7 @@ func TestBTree(t *testing.T) {
if !deleted { if !deleted {
t.Fatal("expected true") t.Fatal("expected true")
} }
if value == nil || itemValue(value).(string) != key { if value == nil || value.Obj().String() != key {
t.Fatalf("expected '%v', got '%v'", key, value) t.Fatalf("expected '%v', got '%v'", key, value)
} }
} }
@ -361,15 +345,15 @@ func TestBTree(t *testing.T) {
// scan items // scan items
last = "" last = ""
all = make(map[string]interface{}) all = make(map[string]interface{})
tr.Scan(func(ptr unsafe.Pointer) bool { tr.Scan(func(item *item.Item) bool {
if itemKey(ptr) <= last { if item.ID() <= last {
t.Fatal("out of order") t.Fatal("out of order")
} }
if itemValue(ptr).(string) != itemKey(ptr) { if item.Obj().String() != item.ID() {
t.Fatalf("mismatch") t.Fatalf("mismatch")
} }
last = itemKey(ptr) last = item.ID()
all[itemKey(ptr)] = itemValue(ptr) all[item.ID()] = item.Obj().String()
return true return true
}) })
if len(all) != len(keys)/2 { if len(all) != len(keys)/2 {
@ -378,11 +362,11 @@ func TestBTree(t *testing.T) {
// replace second half // replace second half
for _, key := range keys[len(keys)/2:] { for _, key := range keys[len(keys)/2:] {
value, replaced := tr.Set(makeItem(key, key)) value, replaced := tr.Set(item.New(key, testString(key)))
if !replaced { if !replaced {
t.Fatal("expected true") t.Fatal("expected true")
} }
if value == nil || itemValue(value).(string) != key { if value == nil || value.Obj().String() != key {
t.Fatalf("expected '%v', got '%v'", key, value) t.Fatalf("expected '%v', got '%v'", key, value)
} }
} }
@ -393,7 +377,7 @@ func TestBTree(t *testing.T) {
if !deleted { if !deleted {
t.Fatal("expected true") t.Fatal("expected true")
} }
if value == nil || itemValue(value).(string) != key { if value == nil || value.Obj().String() != key {
t.Fatalf("expected '%v', got '%v'", key, value) t.Fatalf("expected '%v', got '%v'", key, value)
} }
} }
@ -411,11 +395,11 @@ func TestBTree(t *testing.T) {
if value != nil { if value != nil {
t.Fatal("expected nil") t.Fatal("expected nil")
} }
tr.Scan(func(ptr unsafe.Pointer) bool { tr.Scan(func(item *item.Item) bool {
t.Fatal("should not be reached") t.Fatal("should not be reached")
return true return true
}) })
tr.Reverse(func(ptr unsafe.Pointer) bool { tr.Reverse(func(item *item.Item) bool {
t.Fatal("should not be reached") t.Fatal("should not be reached")
return true return true
}) })
@ -436,7 +420,7 @@ func BenchmarkTidwallSequentialSet(b *testing.B) {
sort.Strings(keys) sort.Strings(keys)
b.ResetTimer() b.ResetTimer()
for i := 0; i < b.N; i++ { for i := 0; i < b.N; i++ {
tr.Set(makeItem(keys[i], nil)) tr.Set(item.New(keys[i], nil))
} }
} }
@ -445,7 +429,7 @@ func BenchmarkTidwallSequentialGet(b *testing.B) {
keys := randKeys(b.N) keys := randKeys(b.N)
sort.Strings(keys) sort.Strings(keys)
for i := 0; i < b.N; i++ { for i := 0; i < b.N; i++ {
tr.Set(makeItem(keys[i], nil)) tr.Set(item.New(keys[i], nil))
} }
b.ResetTimer() b.ResetTimer()
for i := 0; i < b.N; i++ { for i := 0; i < b.N; i++ {
@ -458,7 +442,7 @@ func BenchmarkTidwallRandomSet(b *testing.B) {
keys := randKeys(b.N) keys := randKeys(b.N)
b.ResetTimer() b.ResetTimer()
for i := 0; i < b.N; i++ { for i := 0; i < b.N; i++ {
tr.Set(makeItem(keys[i], nil)) tr.Set(item.New(keys[i], nil))
} }
} }
@ -466,7 +450,7 @@ func BenchmarkTidwallRandomGet(b *testing.B) {
var tr BTree var tr BTree
keys := randKeys(b.N) keys := randKeys(b.N)
for i := 0; i < b.N; i++ { for i := 0; i < b.N; i++ {
tr.Set(makeItem(keys[i], nil)) tr.Set(item.New(keys[i], nil))
} }
b.ResetTimer() b.ResetTimer()
for i := 0; i < b.N; i++ { for i := 0; i < b.N; i++ {
@ -544,11 +528,11 @@ func BenchmarkTidwallRandomGet(b *testing.B) {
func TestBTreeOne(t *testing.T) { func TestBTreeOne(t *testing.T) {
var tr BTree var tr BTree
tr.Set(makeItem("1", "1")) tr.Set(item.New("1", testString("1")))
tr.Delete("1") tr.Delete("1")
tr.Set(makeItem("1", "1")) tr.Set(item.New("1", testString("1")))
tr.Delete("1") tr.Delete("1")
tr.Set(makeItem("1", "1")) tr.Set(item.New("1", testString("1")))
tr.Delete("1") tr.Delete("1")
} }
@ -557,7 +541,7 @@ func TestBTree256(t *testing.T) {
var n int var n int
for j := 0; j < 2; j++ { for j := 0; j < 2; j++ {
for _, i := range rand.Perm(256) { for _, i := range rand.Perm(256) {
tr.Set(makeItem(fmt.Sprintf("%d", i), i)) tr.Set(item.New(fmt.Sprintf("%d", i), testString(fmt.Sprintf("%d", i))))
n++ n++
if tr.Len() != n { if tr.Len() != n {
t.Fatalf("expected 256, got %d", n) t.Fatalf("expected 256, got %d", n)
@ -568,8 +552,8 @@ func TestBTree256(t *testing.T) {
if !ok { if !ok {
t.Fatal("expected true") t.Fatal("expected true")
} }
if itemValue(v).(int) != i { if v.Obj().String() != fmt.Sprintf("%d", i) {
t.Fatalf("expected %d, got %d", i, itemValue(v).(int)) t.Fatalf("expected %d, got %s", i, v.Obj().String())
} }
} }
for _, i := range rand.Perm(256) { for _, i := range rand.Perm(256) {
@ -587,3 +571,52 @@ func TestBTree256(t *testing.T) {
} }
} }
} }
type testString string
func (s testString) Spatial() geojson.Spatial {
return geojson.EmptySpatial{}
}
func (s testString) ForEach(iter func(geom geojson.Object) bool) bool {
return iter(s)
}
func (s testString) Empty() bool {
return true
}
func (s testString) Valid() bool {
return false
}
func (s testString) Rect() geometry.Rect {
return geometry.Rect{}
}
func (s testString) Center() geometry.Point {
return geometry.Point{}
}
func (s testString) AppendJSON(dst []byte) []byte {
data, _ := json.Marshal(string(s))
return append(dst, data...)
}
func (s testString) String() string {
return string(s)
}
func (s testString) JSON() string {
return string(s.AppendJSON(nil))
}
func (s testString) MarshalJSON() ([]byte, error) {
return s.AppendJSON(nil), nil
}
func (s testString) Within(obj geojson.Object) bool {
return false
}
func (s testString) Contains(obj geojson.Object) bool {
return false
}
func (s testString) Intersects(obj geojson.Object) bool {
return false
}
func (s testString) NumPoints() int {
return 0
}
func (s testString) Distance(obj geojson.Object) float64 {
return 0
}

183
internal/collection/collection.go
View File

@ -1,14 +1,13 @@
package collection package collection
import ( import (
"unsafe" ifbtree "github.com/tidwall/btree"
"github.com/tidwall/btree"
"github.com/tidwall/geojson" "github.com/tidwall/geojson"
"github.com/tidwall/geojson/geo" "github.com/tidwall/geojson/geo"
"github.com/tidwall/geojson/geometry" "github.com/tidwall/geojson/geometry"
"github.com/tidwall/tile38/internal/collection/ptrbtree" "github.com/tidwall/tile38/internal/collection/btree"
"github.com/tidwall/tile38/internal/collection/ptrrtree" "github.com/tidwall/tile38/internal/collection/item"
"github.com/tidwall/tile38/internal/collection/rtree"
) )
// Cursor allows for quickly paging through Scan, Within, Intersects, and Nearby // Cursor allows for quickly paging through Scan, Within, Intersects, and Nearby
@ -19,9 +18,9 @@ type Cursor interface {
// Collection represents a collection of geojson objects. // Collection represents a collection of geojson objects.
type Collection struct { type Collection struct {
items ptrbtree.BTree // items sorted by keys items btree.BTree // items sorted by keys
index ptrrtree.BoxTree // items geospatially indexed index rtree.BoxTree // items geospatially indexed
values *btree.BTree // items sorted by value+key values *ifbtree.BTree // items sorted by value+key
fieldMap map[string]int fieldMap map[string]int
weight int weight int
points int points int
@ -34,7 +33,7 @@ var counter uint64
// New creates an empty collection // New creates an empty collection
func New() *Collection { func New() *Collection {
col := &Collection{ col := &Collection{
values: btree.New(16, nil), values: ifbtree.New(16, nil),
fieldMap: make(map[string]int), fieldMap: make(map[string]int),
} }
return col return col
@ -74,48 +73,40 @@ func objIsSpatial(obj geojson.Object) bool {
return ok return ok
} }
func (c *Collection) indexDelete(item *itemT) { func (c *Collection) addItem(item *item.Item) {
if !item.obj.Empty() { if objIsSpatial(item.Obj()) {
rect := item.obj.Rect() if !item.Obj().Empty() {
c.index.Delete( rect := item.Obj().Rect()
[]float64{rect.Min.X, rect.Min.Y}, c.index.Insert(
[]float64{rect.Max.X, rect.Max.Y}, []float64{rect.Min.X, rect.Min.Y},
unsafe.Pointer(item)) []float64{rect.Max.X, rect.Max.Y},
} item)
} }
func (c *Collection) indexInsert(item *itemT) {
if !item.obj.Empty() {
rect := item.obj.Rect()
c.index.Insert(
[]float64{rect.Min.X, rect.Min.Y},
[]float64{rect.Max.X, rect.Max.Y},
unsafe.Pointer(item))
}
}
func (c *Collection) addItem(item *itemT) {
if objIsSpatial(item.obj) {
c.indexInsert(item)
c.objects++ c.objects++
} else { } else {
c.values.ReplaceOrInsert(item) c.values.ReplaceOrInsert(item)
c.nobjects++ c.nobjects++
} }
weight, points := item.weightAndPoints() weight, points := item.WeightAndPoints()
c.weight += weight c.weight += weight
c.points += points c.points += points
} }
func (c *Collection) delItem(item *itemT) { func (c *Collection) delItem(item *item.Item) {
if objIsSpatial(item.obj) { if objIsSpatial(item.Obj()) {
c.indexDelete(item) if !item.Obj().Empty() {
rect := item.Obj().Rect()
c.index.Delete(
[]float64{rect.Min.X, rect.Min.Y},
[]float64{rect.Max.X, rect.Max.Y},
item)
}
c.objects-- c.objects--
} else { } else {
c.values.Delete(item) c.values.Delete(item)
c.nobjects-- c.nobjects--
} }
weight, points := item.weightAndPoints() weight, points := item.WeightAndPoints()
c.weight -= weight c.weight -= weight
c.points -= points c.points -= points
} }
@ -131,26 +122,26 @@ func (c *Collection) Set(
oldObj geojson.Object, oldFields []float64, newFields []float64, oldObj geojson.Object, oldFields []float64, newFields []float64,
) { ) {
// create the new item // create the new item
item := newItem(id, obj) item := item.New(id, obj)
// add the new item to main btree and remove the old one if needed // add the new item to main btree and remove the old one if needed
oldItemV, ok := c.items.Set(unsafe.Pointer(item)) oldItemV, ok := c.items.Set(item)
if ok { if ok {
oldItem := (*itemT)(oldItemV) oldItem := oldItemV
oldObj = oldItem.obj oldObj = oldItem.Obj()
// remove old item from indexes // remove old item from indexes
c.delItem(oldItem) c.delItem(oldItem)
if len(oldItem.fields()) > 0 { if len(oldItem.Fields()) > 0 {
// merge old and new fields // merge old and new fields
oldFields = oldItem.fields() oldFields = oldItem.Fields()
item.directCopyFields(oldFields) item.CopyOverFields(oldFields)
} }
} }
if fields == nil && len(values) > 0 { if fields == nil && len(values) > 0 {
// directly set the field values, from copy // directly set the field values, from copy
item.directCopyFields(values) item.CopyOverFields(values)
} else if len(fields) > 0 { } else if len(fields) > 0 {
// add new field to new item // add new field to new item
c.setFields(item, fields, values, false) c.setFields(item, fields, values, false)
@ -160,7 +151,42 @@ func (c *Collection) Set(
c.addItem(item) c.addItem(item)
// fmt.Printf("!!! %#v\n", oldObj) // fmt.Printf("!!! %#v\n", oldObj)
return oldObj, oldFields, item.fields() return oldObj, oldFields, item.Fields()
}
func (c *Collection) setFields(
item *item.Item, fieldNames []string, fieldValues []float64, updateWeight bool,
) (updatedCount int) {
for i, fieldName := range fieldNames {
var fieldValue float64
if i < len(fieldValues) {
fieldValue = fieldValues[i]
}
if c.setField(item, fieldName, fieldValue, updateWeight) {
updatedCount++
}
}
return updatedCount
}
func (c *Collection) setField(
item *item.Item, fieldName string, fieldValue float64, updateWeight bool,
) (updated bool) {
idx, ok := c.fieldMap[fieldName]
if !ok {
idx = len(c.fieldMap)
c.fieldMap[fieldName] = idx
}
var pweight int
if updateWeight {
pweight, _ = item.WeightAndPoints()
}
updated = item.SetField(idx, fieldValue)
if updateWeight && updated {
nweight, _ := item.WeightAndPoints()
c.weight = c.weight - pweight + nweight
}
return updated
} }
// Delete removes an object and returns it. // Delete removes an object and returns it.
@ -172,11 +198,11 @@ func (c *Collection) Delete(id string) (
if !ok { if !ok {
return nil, nil, false return nil, nil, false
} }
oldItem := (*itemT)(oldItemV) oldItem := oldItemV
c.delItem(oldItem) c.delItem(oldItem)
return oldItem.obj, oldItem.fields(), true return oldItem.Obj(), oldItem.Fields(), true
} }
// Get returns an object. // Get returns an object.
@ -188,9 +214,9 @@ func (c *Collection) Get(id string) (
if !ok { if !ok {
return nil, nil, false return nil, nil, false
} }
item := (*itemT)(itemV) item := itemV
return item.obj, item.fields(), true return item.Obj(), item.Fields(), true
} }
// SetField set a field value for an object and returns that object. // SetField set a field value for an object and returns that object.
@ -202,9 +228,9 @@ func (c *Collection) SetField(id, fieldName string, fieldValue float64) (
if !ok { if !ok {
return nil, nil, false, false return nil, nil, false, false
} }
item := (*itemT)(itemV) item := itemV
updated = c.setField(item, fieldName, fieldValue, true) updated = c.setField(item, fieldName, fieldValue, true)
return item.obj, item.fields(), updated, true return item.Obj(), item.Fields(), updated, true
} }
// SetFields is similar to SetField, just setting multiple fields at once // SetFields is similar to SetField, just setting multiple fields at once
@ -215,11 +241,11 @@ func (c *Collection) SetFields(
if !ok { if !ok {
return nil, nil, 0, false return nil, nil, 0, false
} }
item := (*itemT)(itemV) item := itemV
updatedCount = c.setFields(item, fieldNames, fieldValues, true) updatedCount = c.setFields(item, fieldNames, fieldValues, true)
return item.obj, item.fields(), updatedCount, true return item.Obj(), item.Fields(), updatedCount, true
} }
// FieldMap return a maps of the field names. // FieldMap return a maps of the field names.
@ -247,7 +273,7 @@ func (c *Collection) Scan(desc bool, cursor Cursor,
offset = cursor.Offset() offset = cursor.Offset()
cursor.Step(offset) cursor.Step(offset)
} }
iter := func(ptr unsafe.Pointer) bool { iter := func(item *item.Item) bool {
count++ count++
if count <= offset { if count <= offset {
return true return true
@ -255,8 +281,7 @@ func (c *Collection) Scan(desc bool, cursor Cursor,
if cursor != nil { if cursor != nil {
cursor.Step(1) cursor.Step(1)
} }
iitm := (*itemT)(ptr) keepon = iterator(item.ID(), item.Obj(), item.Fields())
keepon = iterator(iitm.id(), iitm.obj, iitm.fields())
return keepon return keepon
} }
if desc { if desc {
@ -278,7 +303,7 @@ func (c *Collection) ScanRange(start, end string, desc bool, cursor Cursor,
offset = cursor.Offset() offset = cursor.Offset()
cursor.Step(offset) cursor.Step(offset)
} }
iter := func(ptr unsafe.Pointer) bool { iter := func(item *item.Item) bool {
count++ count++
if count <= offset { if count <= offset {
return true return true
@ -286,17 +311,16 @@ func (c *Collection) ScanRange(start, end string, desc bool, cursor Cursor,
if cursor != nil { if cursor != nil {
cursor.Step(1) cursor.Step(1)
} }
iitm := (*itemT)(ptr)
if !desc { if !desc {
if iitm.id() >= end { if item.ID() >= end {
return false return false
} }
} else { } else {
if iitm.id() <= end { if item.ID() <= end {
return false return false
} }
} }
keepon = iterator(iitm.id(), iitm.obj, iitm.fields()) keepon = iterator(item.ID(), item.Obj(), item.Fields())
return keepon return keepon
} }
@ -319,7 +343,7 @@ func (c *Collection) SearchValues(desc bool, cursor Cursor,
offset = cursor.Offset() offset = cursor.Offset()
cursor.Step(offset) cursor.Step(offset)
} }
iter := func(item btree.Item) bool { iter := func(v ifbtree.Item) bool {
count++ count++
if count <= offset { if count <= offset {
return true return true
@ -327,8 +351,8 @@ func (c *Collection) SearchValues(desc bool, cursor Cursor,
if cursor != nil { if cursor != nil {
cursor.Step(1) cursor.Step(1)
} }
iitm := item.(*itemT) iitm := v.(*item.Item)
keepon = iterator(iitm.id(), iitm.obj, iitm.fields()) keepon = iterator(iitm.ID(), iitm.Obj(), iitm.Fields())
return keepon return keepon
} }
if desc { if desc {
@ -351,7 +375,7 @@ func (c *Collection) SearchValuesRange(start, end string, desc bool,
offset = cursor.Offset() offset = cursor.Offset()
cursor.Step(offset) cursor.Step(offset)
} }
iter := func(item btree.Item) bool { iter := func(v ifbtree.Item) bool {
count++ count++
if count <= offset { if count <= offset {
return true return true
@ -359,17 +383,17 @@ func (c *Collection) SearchValuesRange(start, end string, desc bool,
if cursor != nil { if cursor != nil {
cursor.Step(1) cursor.Step(1)
} }
iitm := item.(*itemT) iitm := v.(*item.Item)
keepon = iterator(iitm.id(), iitm.obj, iitm.fields()) keepon = iterator(iitm.ID(), iitm.Obj(), iitm.Fields())
return keepon return keepon
} }
if desc { if desc {
c.values.DescendRange( c.values.DescendRange(
newItem("", String(start)), newItem("", String(end)), iter, item.New("", String(start)), item.New("", String(end)), iter,
) )
} else { } else {
c.values.AscendRange( c.values.AscendRange(
newItem("", String(start)), newItem("", String(end)), iter, item.New("", String(start)), item.New("", String(end)), iter,
) )
} }
return keepon return keepon
@ -387,7 +411,7 @@ func (c *Collection) ScanGreaterOrEqual(id string, desc bool,
offset = cursor.Offset() offset = cursor.Offset()
cursor.Step(offset) cursor.Step(offset)
} }
iter := func(ptr unsafe.Pointer) bool { iter := func(item *item.Item) bool {
count++ count++
if count <= offset { if count <= offset {
return true return true
@ -395,8 +419,7 @@ func (c *Collection) ScanGreaterOrEqual(id string, desc bool,
if cursor != nil { if cursor != nil {
cursor.Step(1) cursor.Step(1)
} }
iitm := (*itemT)(ptr) keepon = iterator(item.ID(), item.Obj(), item.Fields())
keepon = iterator(iitm.id(), iitm.obj, iitm.fields())
return keepon return keepon
} }
if desc { if desc {
@ -415,9 +438,9 @@ func (c *Collection) geoSearch(
c.index.Search( c.index.Search(
[]float64{rect.Min.X, rect.Min.Y}, []float64{rect.Min.X, rect.Min.Y},
[]float64{rect.Max.X, rect.Max.Y}, []float64{rect.Max.X, rect.Max.Y},
func(_, _ []float64, itemv unsafe.Pointer) bool { func(_, _ []float64, itemv *item.Item) bool {
item := (*itemT)(itemv) item := itemv
alive = iter(item.id(), item.obj, item.fields()) alive = iter(item.ID(), item.Obj(), item.Fields())
return alive return alive
}, },
) )
@ -610,7 +633,7 @@ func (c *Collection) Nearby(
c.index.Search( c.index.Search(
[]float64{minLon, minLat}, []float64{minLon, minLat},
[]float64{maxLon, maxLat}, []float64{maxLon, maxLat},
func(_, _ []float64, itemv unsafe.Pointer) bool { func(_, _ []float64, itemv *item.Item) bool {
exists = true exists = true
return false return false
}, },
@ -633,7 +656,7 @@ func (c *Collection) Nearby(
c.index.Nearby( c.index.Nearby(
[]float64{center.X, center.Y}, []float64{center.X, center.Y},
[]float64{center.X, center.Y}, []float64{center.X, center.Y},
func(_, _ []float64, itemv unsafe.Pointer) bool { func(_, _ []float64, itemv *item.Item) bool {
count++ count++
if count <= offset { if count <= offset {
return true return true
@ -641,8 +664,8 @@ func (c *Collection) Nearby(
if cursor != nil { if cursor != nil {
cursor.Step(1) cursor.Step(1)
} }
item := (*itemT)(itemv) item := itemv
alive = iter(item.id(), item.obj, item.fields()) alive = iter(item.ID(), item.Obj(), item.Fields())
return alive return alive
}, },
) )

149
internal/collection/item.go
View File

@ -1,149 +0,0 @@
package collection
import (
"reflect"
"unsafe"
"github.com/tidwall/btree"
"github.com/tidwall/geojson"
)
type itemT struct {
obj geojson.Object
fieldsLen uint32 // fields block size in bytes, not num of fields
idLen uint32 // id block size in bytes
data unsafe.Pointer
}
func (item *itemT) id() string {
return *(*string)((unsafe.Pointer)(&reflect.StringHeader{
Data: uintptr(unsafe.Pointer(item.data)) + uintptr(item.fieldsLen),
Len: int(item.idLen),
}))
}
func (item *itemT) fields() []float64 {
return *(*[]float64)((unsafe.Pointer)(&reflect.SliceHeader{
Data: uintptr(unsafe.Pointer(item.data)),
Len: int(item.fieldsLen) / 8,
Cap: int(item.fieldsLen) / 8,
}))
}
func (item *itemT) dataBytes() []byte {
return *(*[]byte)((unsafe.Pointer)(&reflect.SliceHeader{
Data: uintptr(unsafe.Pointer(item.data)),
Len: int(item.fieldsLen) + int(item.idLen),
Cap: int(item.fieldsLen) + int(item.idLen),
}))
}
func newItem(id string, obj geojson.Object) *itemT {
item := new(itemT)
item.obj = obj
item.idLen = uint32(len(id))
if len(id) > 0 {
data := make([]byte, len(id))
copy(data, id)
item.data = unsafe.Pointer(&data[0])
}
return item
}
func (item *itemT) weightAndPoints() (weight, points int) {
if objIsSpatial(item.obj) {
points = item.obj.NumPoints()
weight = points * 16
} else {
weight = len(item.obj.String())
}
weight += int(item.fieldsLen + item.idLen)
return weight, points
}
func (item *itemT) Less(other btree.Item, ctx interface{}) bool {
value1 := item.obj.String()
value2 := other.(*itemT).obj.String()
if value1 < value2 {
return true
}
if value1 > value2 {
return false
}
// the values match so we'll compare IDs, which are always unique.
return item.id() < other.(*itemT).id()
}
func floatsToBytes(f []float64) []byte {
return *(*[]byte)((unsafe.Pointer)(&reflect.SliceHeader{
Data: ((*reflect.SliceHeader)(unsafe.Pointer(&f))).Data,
Len: len(f) * 8,
Cap: len(f) * 8,
}))
}
// directCopyFields copies fields, overwriting previous fields
func (item *itemT) directCopyFields(fields []float64) {
fieldBytes := floatsToBytes(fields)
oldData := item.dataBytes()
newData := make([]byte, len(fieldBytes)+int(item.idLen))
copy(newData, fieldBytes)
copy(newData[len(fieldBytes):], oldData[item.fieldsLen:])
item.fieldsLen = uint32(len(fieldBytes))
if len(newData) > 0 {
item.data = unsafe.Pointer(&newData[0])
} else {
item.data = nil
}
}
func (c *Collection) setField(
item *itemT, fieldName string, fieldValue float64, updateWeight bool,
) (updated bool) {
idx, ok := c.fieldMap[fieldName]
if !ok {
idx = len(c.fieldMap)
c.fieldMap[fieldName] = idx
}
itemFields := item.fields()
if idx >= len(itemFields) {
// make room for new field
itemBytes := item.dataBytes()
oldLen := len(itemFields)
data := make([]byte, (idx+1)*8+int(item.idLen))
copy(data, itemBytes[:item.fieldsLen])
copy(data[(idx+1)*8:], itemBytes[item.fieldsLen:])
item.fieldsLen = uint32((idx + 1) * 8)
item.data = unsafe.Pointer(&data[0])
itemFields := item.fields()
if updateWeight {
c.weight += (len(itemFields) - oldLen) * 8
}
itemFields[idx] = fieldValue
updated = true
} else if itemFields[idx] != fieldValue {
// existing field needs updating
itemFields[idx] = fieldValue
updated = true
}
return updated
}
func (c *Collection) setFields(
item *itemT, fieldNames []string, fieldValues []float64, updateWeight bool,
) (updatedCount int) {
// TODO: optimize to predict the item data growth.
// TODO: do all sets here, instead of calling setFields in a loop
for i, fieldName := range fieldNames {
var fieldValue float64
if i < len(fieldValues) {
fieldValue = fieldValues[i]
}
if c.setField(item, fieldName, fieldValue, updateWeight) {
updatedCount++
}
}
return updatedCount
}

153
internal/collection/item/item.go Normal file
View File

@ -0,0 +1,153 @@
package item
import (
"reflect"
"unsafe"
"github.com/tidwall/btree"
"github.com/tidwall/geojson"
)
// Item is a item for Tile38 collections
type Item struct {
obj geojson.Object // geojson or string
fieldsLen uint32 // fields block size in bytes, not num of fields
idLen uint32 // id block size in bytes
data unsafe.Pointer // pointer to raw block of bytes
}
// ID returns the items ID as a string
func (item *Item) ID() string {
return *(*string)((unsafe.Pointer)(&reflect.StringHeader{
Data: uintptr(unsafe.Pointer(item.data)) + uintptr(item.fieldsLen),
Len: int(item.idLen),
}))
}
// Fields returns the field values
func (item *Item) Fields() []float64 {
return *(*[]float64)((unsafe.Pointer)(&reflect.SliceHeader{
Data: uintptr(unsafe.Pointer(item.data)),
Len: int(item.fieldsLen) / 8,
Cap: int(item.fieldsLen) / 8,
}))
}
// Obj returns the geojson object
func (item *Item) Obj() geojson.Object {
return item.obj
}
// New returns a newly allocated Item
func New(id string, obj geojson.Object) *Item {
item := new(Item)
item.obj = obj
item.idLen = uint32(len(id))
if len(id) > 0 {
data := make([]byte, len(id))
copy(data, id)
item.data = unsafe.Pointer(&data[0])
}
return item
}
// WeightAndPoints returns the memory weight and number of points for Item.
func (item *Item) WeightAndPoints() (weight, points int) {
_, objIsSpatial := item.obj.(geojson.Spatial)
if objIsSpatial {
points = item.obj.NumPoints()
weight = points * 16
} else if item.obj != nil {
weight = len(item.obj.String())
}
weight += int(item.fieldsLen + item.idLen)
return weight, points
}
// Less is a btree interface that compares if item is less than other item.
func (item *Item) Less(other btree.Item, ctx interface{}) bool {
value1 := item.obj.String()
value2 := other.(*Item).obj.String()
if value1 < value2 {
return true
}
if value1 > value2 {
return false
}
// the values match so we'll compare IDs, which are always unique.
return item.ID() < other.(*Item).ID()
}
// CopyOverFields overwriting previous fields
func (item *Item) CopyOverFields(values []float64) {
fieldBytes := floatsToBytes(values)
oldData := item.dataBytes()
newData := make([]byte, len(fieldBytes)+int(item.idLen))
copy(newData, fieldBytes)
copy(newData[len(fieldBytes):], oldData[item.fieldsLen:])
item.fieldsLen = uint32(len(fieldBytes))
if len(newData) > 0 {
item.data = unsafe.Pointer(&newData[0])
} else {
item.data = nil
}
}
func getFieldAt(data unsafe.Pointer, index int) float64 {
return *(*float64)(unsafe.Pointer(uintptr(data) + uintptr(index*8)))
}
func setFieldAt(data unsafe.Pointer, index int, value float64) {
*(*float64)(unsafe.Pointer(uintptr(data) + uintptr(index*8))) = value
}
// SetField set a field value at specified index.
func (item *Item) SetField(index int, value float64) (updated bool) {
numFields := int(item.fieldsLen / 8)
if index < numFields {
// field exists
if getFieldAt(item.data, index) == value {
return false
}
} else {
// make room for new field
oldBytes := item.dataBytes()
newData := make([]byte, (index+1)*8+int(item.idLen))
// copy the existing fields
copy(newData, oldBytes[:item.fieldsLen])
// copy the id
copy(newData[(index+1)*8:], oldBytes[item.fieldsLen:])
// update the fields length
item.fieldsLen = uint32((index + 1) * 8)
// update the raw data
item.data = unsafe.Pointer(&newData[0])
}
// set the new field
setFieldAt(item.data, index, value)
return true
}
// GetField returns the value for a field at index.
func (item *Item) GetField(index int) float64 {
numFields := int(item.fieldsLen / 8)
if index < numFields {
return getFieldAt(item.data, index)
}
return 0
}
func (item *Item) dataBytes() []byte {
return *(*[]byte)((unsafe.Pointer)(&reflect.SliceHeader{
Data: uintptr(unsafe.Pointer(item.data)),
Len: int(item.fieldsLen) + int(item.idLen),
Cap: int(item.fieldsLen) + int(item.idLen),
}))
}
func floatsToBytes(f []float64) []byte {
return *(*[]byte)((unsafe.Pointer)(&reflect.SliceHeader{
Data: ((*reflect.SliceHeader)(unsafe.Pointer(&f))).Data,
Len: len(f) * 8,
Cap: len(f) * 8,
}))
}

189
internal/collection/item/item_test.go Normal file
View File

@ -0,0 +1,189 @@
package item
import (
"encoding/json"
"math/rand"
"reflect"
"testing"
"time"
"github.com/tidwall/geojson"
"github.com/tidwall/geojson/geometry"
)
func testRandItem(t *testing.T) {
keyb := make([]byte, rand.Int()%16)
rand.Read(keyb)
key := string(keyb)
values := make([]float64, rand.Int()%1024)
for i := range values {
values[i] = rand.Float64()
}
item := New(key, geojson.NewPoint(geometry.Point{X: 1, Y: 2}))
if item.ID() != key {
t.Fatalf("expected '%v', got '%v'", key, item.ID())
}
var setValues []int
for _, i := range rand.Perm(len(values)) {
if !item.SetField(i, values[i]) {
t.Fatal("expected true")
}
setValues = append(setValues, i)
if item.ID() != key {
t.Fatalf("expected '%v', got '%v'", key, item.ID())
}
for _, i := range setValues {
if item.GetField(i) != values[i] {
t.Fatalf("expected '%v', got '%v'", values[i], item.GetField(i))
}
}
fields := item.Fields()
for i := 0; i < len(fields); i++ {
for _, j := range setValues {
if i == j {
if fields[i] != values[i] {
t.Fatalf("expected '%v', got '%v'", values[i], fields[i])
}
break
}
}
}
weight, points := item.WeightAndPoints()
if weight != len(fields)*8+len(key)+points*16 {
t.Fatalf("expected '%v', got '%v'", len(fields)*8+len(key)+points*16, weight)
}
if points != 1 {
t.Fatalf("expected '%v', got '%v'", 1, points)
}
}
if item.GetField(len(values)) != 0 {
t.Fatalf("expected '%v', got '%v'", 0, item.GetField(len(values)))
}
for _, i := range rand.Perm(len(values)) {
if item.SetField(i, values[i]) {
t.Fatal("expected false")
}
}
if item.ID() != key {
t.Fatalf("expected '%v', got '%v'", key, item.ID())
}
if item.Obj().NumPoints() != 1 {
t.Fatalf("expected '%v', got '%v'", 1, item.Obj().NumPoints())
}
item.CopyOverFields(values)
weight, points := item.WeightAndPoints()
if weight != len(values)*8+len(key)+points*16 {
t.Fatalf("expected '%v', got '%v'", len(values)*8+len(key)+points*16, weight)
}
if points != 1 {
t.Fatalf("expected '%v', got '%v'", 1, points)
}
if !reflect.DeepEqual(item.Fields(), values) {
t.Fatalf("expected '%v', got '%v'", values, item.Fields())
}
item.CopyOverFields(nil)
weight, points = item.WeightAndPoints()
if weight != len(key)+points*16 {
t.Fatalf("expected '%v', got '%v'", len(key)+points*16, weight)
}
if points != 1 {
t.Fatalf("expected '%v', got '%v'", 1, points)
}
if len(item.Fields()) != 0 {
t.Fatalf("expected '%#v', got '%#v'", 0, len(item.Fields()))
}
if item.ID() != key {
t.Fatalf("expected '%v', got '%v'", key, item.ID())
}
}
func TestItem(t *testing.T) {
rand.Seed(time.Now().UnixNano())
start := time.Now()
for time.Since(start) < time.Second {
testRandItem(t)
}
}
func TestItemLess(t *testing.T) {
item0 := New("0", testString("0"))
item1 := New("1", testString("1"))
item2 := New("1", testString("2"))
item3 := New("3", testString("2"))
if !item0.Less(item1, nil) {
t.Fatal("expected true")
}
if item1.Less(item0, nil) {
t.Fatal("expected false")
}
if !item1.Less(item2, nil) {
t.Fatal("expected true")
}
if item2.Less(item1, nil) {
t.Fatal("expected false")
}
if !item2.Less(item3, nil) {
t.Fatal("expected true")
}
if item3.Less(item2, nil) {
t.Fatal("expected false")
}
weight, points := item0.WeightAndPoints()
if weight != 2 {
t.Fatalf("expected '%v', got '%v'", 2, weight)
}
if points != 0 {
t.Fatalf("expected '%v', got '%v'", 0, points)
}
}
type testString string
func (s testString) Spatial() geojson.Spatial {
return geojson.EmptySpatial{}
}
func (s testString) ForEach(iter func(geom geojson.Object) bool) bool {
return iter(s)
}
func (s testString) Empty() bool {
return true
}
func (s testString) Valid() bool {
return false
}
func (s testString) Rect() geometry.Rect {
return geometry.Rect{}
}
func (s testString) Center() geometry.Point {
return geometry.Point{}
}
func (s testString) AppendJSON(dst []byte) []byte {
data, _ := json.Marshal(string(s))
return append(dst, data...)
}
func (s testString) String() string {
return string(s)
}
func (s testString) JSON() string {
return string(s.AppendJSON(nil))
}
func (s testString) MarshalJSON() ([]byte, error) {
return s.AppendJSON(nil), nil
}
func (s testString) Within(obj geojson.Object) bool {
return false
}
func (s testString) Contains(obj geojson.Object) bool {
return false
}
func (s testString) Intersects(obj geojson.Object) bool {
return false
}
func (s testString) NumPoints() int {
return 0
}
func (s testString) Distance(obj geojson.Object) float64 {
return 0
}

55
internal/collection/ptrrtree/rtree.go → internal/collection/rtree/rtree.go
View File

@ -1,6 +1,10 @@
package ptrrtree package rtree
import "unsafe" import (
"unsafe"
"github.com/tidwall/tile38/internal/collection/item"
)
const dims = 2 const dims = 2
@ -90,10 +94,10 @@ func (r *box) enlargedArea(b *box) float64 {
} }
// Insert inserts an item into the RTree // Insert inserts an item into the RTree
func (tr *BoxTree) Insert(min, max []float64, value unsafe.Pointer) { func (tr *BoxTree) Insert(min, max []float64, item *item.Item) {
var item box var box box
fit(min, max, value, &item) fit(min, max, unsafe.Pointer(item), &box)
tr.insert(&item) tr.insert(&box)
} }
func (tr *BoxTree) insert(item *box) { func (tr *BoxTree) insert(item *box) {
@ -317,14 +321,14 @@ func (r *box) intersects(b *box) bool {
func (r *box) search( func (r *box) search(
target *box, height int, target *box, height int,
iter func(min, max []float64, value unsafe.Pointer) bool, iter func(min, max []float64, item *item.Item) bool,
) bool { ) bool {
n := (*node)(r.data) n := (*node)(r.data)
if height == 0 { if height == 0 {
for i := 0; i < n.count; i++ { for i := 0; i < n.count; i++ {
if target.intersects(&n.boxes[i]) { if target.intersects(&n.boxes[i]) {
if !iter(n.boxes[i].min[:], n.boxes[i].max[:], if !iter(n.boxes[i].min[:], n.boxes[i].max[:],
n.boxes[i].data) { (*item.Item)(n.boxes[i].data)) {
return false return false
} }
} }
@ -348,7 +352,7 @@ func (r *box) search(
func (tr *BoxTree) search( func (tr *BoxTree) search(
target *box, target *box,
iter func(min, max []float64, value unsafe.Pointer) bool, iter func(min, max []float64, item *item.Item) bool,
) { ) {
if tr.root.data == nil { if tr.root.data == nil {
return return
@ -363,7 +367,7 @@ func (tr *BoxTree) search(
// Search ... // Search ...
func (tr *BoxTree) Search(min, max []float64, func (tr *BoxTree) Search(min, max []float64,
iter func(min, max []float64, value unsafe.Pointer) bool, iter func(min, max []float64, item *item.Item) bool,
) { ) {
var target box var target box
fit(min, max, nil, &target) fit(min, max, nil, &target)
@ -381,7 +385,7 @@ const (
// Traverse iterates through all items and container boxes in tree. // Traverse iterates through all items and container boxes in tree.
func (tr *BoxTree) Traverse( func (tr *BoxTree) Traverse(
iter func(min, max []float64, height, level int, value unsafe.Pointer) int, iter func(min, max []float64, height, level int, item *item.Item) int,
) { ) {
if tr.root.data == nil { if tr.root.data == nil {
return return
@ -393,13 +397,13 @@ func (tr *BoxTree) Traverse(
func (r *box) traverse( func (r *box) traverse(
height, level int, height, level int,
iter func(min, max []float64, height, level int, value unsafe.Pointer) int, iter func(min, max []float64, height, level int, value *item.Item) int,
) int { ) int {
n := (*node)(r.data) n := (*node)(r.data)
if height == 0 { if height == 0 {
for i := 0; i < n.count; i++ { for i := 0; i < n.count; i++ {
action := iter(n.boxes[i].min[:], n.boxes[i].max[:], height, level, action := iter(n.boxes[i].min[:], n.boxes[i].max[:], height, level,
n.boxes[i].data) (*item.Item)(n.boxes[i].data))
if action == Stop { if action == Stop {
return Stop return Stop
} }
@ -407,7 +411,7 @@ func (r *box) traverse(
} else { } else {
for i := 0; i < n.count; i++ { for i := 0; i < n.count; i++ {
switch iter(n.boxes[i].min[:], n.boxes[i].max[:], height, level, switch iter(n.boxes[i].min[:], n.boxes[i].max[:], height, level,
n.boxes[i].data) { (*item.Item)(n.boxes[i].data)) {
case Ignore: case Ignore:
case Continue: case Continue:
if n.boxes[i].traverse(height-1, level+1, iter) == Stop { if n.boxes[i].traverse(height-1, level+1, iter) == Stop {
@ -422,12 +426,13 @@ func (r *box) traverse(
} }
func (r *box) scan( func (r *box) scan(
height int, iter func(min, max []float64, value unsafe.Pointer) bool, height int, iter func(min, max []float64, value *item.Item) bool,
) bool { ) bool {
n := (*node)(r.data) n := (*node)(r.data)
if height == 0 { if height == 0 {
for i := 0; i < n.count; i++ { for i := 0; i < n.count; i++ {
if !iter(n.boxes[i].min[:], n.boxes[i].max[:], n.boxes[i].data) { if !iter(n.boxes[i].min[:], n.boxes[i].max[:],
(*item.Item)(n.boxes[i].data)) {
return false return false
} }
} }
@ -442,7 +447,7 @@ func (r *box) scan(
} }
// Scan iterates through all items in tree. // Scan iterates through all items in tree.
func (tr *BoxTree) Scan(iter func(min, max []float64, value unsafe.Pointer) bool) { func (tr *BoxTree) Scan(iter func(min, max []float64, item *item.Item) bool) {
if tr.root.data == nil { if tr.root.data == nil {
return return
} }
@ -450,15 +455,15 @@ func (tr *BoxTree) Scan(iter func(min, max []float64, value unsafe.Pointer) bool
} }
// Delete ... // Delete ...
func (tr *BoxTree) Delete(min, max []float64, value unsafe.Pointer) { func (tr *BoxTree) Delete(min, max []float64, item *item.Item) {
var item box var target box
fit(min, max, value, &item) fit(min, max, unsafe.Pointer(item), &target)
if tr.root.data == nil || !tr.root.contains(&item) { if tr.root.data == nil || !tr.root.contains(&target) {
return return
} }
var removed, recalced bool var removed, recalced bool
removed, recalced, tr.reinsert = removed, recalced, tr.reinsert =
tr.root.delete(&item, tr.height, tr.reinsert[:0]) tr.root.delete(&target, tr.height, tr.reinsert[:0])
if !removed { if !removed {
return return
} }
@ -647,7 +652,7 @@ func (q *queue) pop() qnode {
// Nearby returns items nearest to farthest. // Nearby returns items nearest to farthest.
// The dist param is the "box distance". // The dist param is the "box distance".
func (tr *BoxTree) Nearby(min, max []float64, func (tr *BoxTree) Nearby(min, max []float64,
iter func(min, max []float64, item unsafe.Pointer) bool) { iter func(min, max []float64, item *item.Item) bool) {
if tr.root.data == nil { if tr.root.data == nil {
return return
} }
@ -666,8 +671,8 @@ func (tr *BoxTree) Nearby(min, max []float64,
if q.peek().height > -1 { if q.peek().height > -1 {
break break
} }
item := q.pop() v := q.pop()
if !iter(item.box.min[:], item.box.max[:], item.box.data) { if !iter(v.box.min[:], v.box.max[:], (*item.Item)(v.box.data)) {
return return
} }
} }

187
internal/collection/ptrrtree/rtree_test.go → internal/collection/rtree/rtree_test.go
View File

@ -1,4 +1,4 @@
package ptrrtree package rtree
import ( import (
"fmt" "fmt"
@ -8,7 +8,10 @@ import (
"strings" "strings"
"testing" "testing"
"time" "time"
"unsafe"
"github.com/tidwall/geojson"
"github.com/tidwall/geojson/geometry"
"github.com/tidwall/tile38/internal/collection/item"
) )
type tBox struct { type tBox struct {
@ -16,8 +19,8 @@ type tBox struct {
max [dims]float64 max [dims]float64
} }
var boxes []*tBox var boxes []*item.Item
var points []*tBox var points []*item.Item
func init() { func init() {
seed := time.Now().UnixNano() seed := time.Now().UnixNano()
@ -26,54 +29,65 @@ func init() {
rand.Seed(seed) rand.Seed(seed)
} }
func randPoints(N int) []*tBox { func boxMin(box *item.Item) []float64 {
boxes := make([]*tBox, N) return box.Obj().(*tBox).min[:]
}
func boxMax(box *item.Item) []float64 {
return box.Obj().(*tBox).max[:]
}
func randPoints(N int) []*item.Item {
boxes := make([]*item.Item, N)
for i := 0; i < N; i++ { for i := 0; i < N; i++ {
boxes[i] = new(tBox) box := new(tBox)
boxes[i].min[0] = rand.Float64()*360 - 180 box.min[0] = rand.Float64()*360 - 180
boxes[i].min[1] = rand.Float64()*180 - 90 box.min[1] = rand.Float64()*180 - 90
for j := 2; j < dims; j++ { for j := 2; j < dims; j++ {
boxes[i].min[j] = rand.Float64() box.min[j] = rand.Float64()
} }
boxes[i].max = boxes[i].min box.max = box.min
boxes[i] = item.New(fmt.Sprintf("%d", i), box)
} }
return boxes return boxes
} }
func randBoxes(N int) []*tBox { func randBoxes(N int) []*item.Item {
boxes := make([]*tBox, N) boxes := make([]*item.Item, N)
for i := 0; i < N; i++ { for i := 0; i < N; i++ {
boxes[i] = new(tBox) box := new(tBox)
boxes[i].min[0] = rand.Float64()*360 - 180 box.min[0] = rand.Float64()*360 - 180
boxes[i].min[1] = rand.Float64()*180 - 90 box.min[1] = rand.Float64()*180 - 90
for j := 2; j < dims; j++ { for j := 2; j < dims; j++ {
boxes[i].min[j] = rand.Float64() * 100 box.min[j] = rand.Float64() * 100
} }
boxes[i].max[0] = boxes[i].min[0] + rand.Float64() box.max[0] = box.min[0] + rand.Float64()
boxes[i].max[1] = boxes[i].min[1] + rand.Float64() box.max[1] = box.min[1] + rand.Float64()
for j := 2; j < dims; j++ { for j := 2; j < dims; j++ {
boxes[i].max[j] = boxes[i].min[j] + rand.Float64() box.max[j] = box.min[j] + rand.Float64()
} }
if boxes[i].max[0] > 180 || boxes[i].max[1] > 90 { if box.max[0] > 180 || box.max[1] > 90 {
i-- i--
} }
boxes[i] = item.New(fmt.Sprintf("%d", i), box)
} }
return boxes return boxes
} }
func sortBoxes(boxes []*tBox) { func sortBoxes(boxes []*item.Item) {
sort.Slice(boxes, func(i, j int) bool { sort.Slice(boxes, func(i, j int) bool {
for k := 0; k < len(boxes[i].min); k++ { iMin, iMax := boxMin(boxes[i]), boxMax(boxes[i])
if boxes[i].min[k] < boxes[j].min[k] { jMin, jMax := boxMin(boxes[j]), boxMax(boxes[j])
for k := 0; k < len(iMin); k++ {
if iMin[k] < jMin[k] {
return true return true
} }
if boxes[i].min[k] > boxes[j].min[k] { if iMin[k] > jMin[k] {
return false return false
} }
if boxes[i].max[k] < boxes[j].max[k] { if iMax[k] < jMax[k] {
return true return true
} }
if boxes[i].max[k] > boxes[j].max[k] { if iMax[k] > jMax[k] {
return false return false
} }
} }
@ -81,10 +95,10 @@ func sortBoxes(boxes []*tBox) {
}) })
} }
func sortBoxesNearby(boxes []tBox, min, max []float64) { func sortBoxesNearby(boxes []*item.Item, min, max []float64) {
sort.Slice(boxes, func(i, j int) bool { sort.Slice(boxes, func(i, j int) bool {
return testBoxDist(boxes[i].min[:], boxes[i].max[:], min, max) < return testBoxDist(boxMin(boxes[i]), boxMax(boxes[i]), min, max) <
testBoxDist(boxes[j].min[:], boxes[j].max[:], min, max) testBoxDist(boxMin(boxes[j]), boxMax(boxes[j]), min, max)
}) })
} }
@ -110,7 +124,7 @@ func testBoxDist(amin, amax, bmin, bmax []float64) float64 {
return dist return dist
} }
func testBoxesVarious(t *testing.T, boxes []*tBox, label string) { func testBoxesVarious(t *testing.T, items []*item.Item, label string) {
N := len(boxes) N := len(boxes)
var tr BoxTree var tr BoxTree
@ -122,7 +136,7 @@ func testBoxesVarious(t *testing.T, boxes []*tBox, label string) {
// insert // insert
///////////////////////////////////////// /////////////////////////////////////////
for i := 0; i < N; i++ { for i := 0; i < N; i++ {
tr.Insert(boxes[i].min[:], boxes[i].max[:], unsafe.Pointer(boxes[i])) tr.Insert(boxMin(boxes[i]), boxMax(boxes[i]), boxes[i])
} }
if tr.Count() != N { if tr.Count() != N {
t.Fatalf("expected %d, got %d", N, tr.Count()) t.Fatalf("expected %d, got %d", N, tr.Count())
@ -136,7 +150,7 @@ func testBoxesVarious(t *testing.T, boxes []*tBox, label string) {
// scan all items and count one-by-one // scan all items and count one-by-one
///////////////////////////////////////// /////////////////////////////////////////
var count int var count int
tr.Scan(func(min, max []float64, value unsafe.Pointer) bool { tr.Scan(func(min, max []float64, _ *item.Item) bool {
count++ count++
return true return true
}) })
@ -147,12 +161,12 @@ func testBoxesVarious(t *testing.T, boxes []*tBox, label string) {
///////////////////////////////////////// /////////////////////////////////////////
// check every point for correctness // check every point for correctness
///////////////////////////////////////// /////////////////////////////////////////
var tboxes1 []*tBox var tboxes1 []*item.Item
tr.Scan(func(min, max []float64, value unsafe.Pointer) bool { tr.Scan(func(min, max []float64, item *item.Item) bool {
tboxes1 = append(tboxes1, (*tBox)(value)) tboxes1 = append(tboxes1, item)
return true return true
}) })
tboxes2 := make([]*tBox, len(boxes)) tboxes2 := make([]*item.Item, len(boxes))
copy(tboxes2, boxes) copy(tboxes2, boxes)
sortBoxes(tboxes1) sortBoxes(tboxes1)
sortBoxes(tboxes2) sortBoxes(tboxes2)
@ -167,9 +181,9 @@ func testBoxesVarious(t *testing.T, boxes []*tBox, label string) {
///////////////////////////////////////// /////////////////////////////////////////
for i := 0; i < N; i++ { for i := 0; i < N; i++ {
var found bool var found bool
tr.Search(boxes[i].min[:], boxes[i].max[:], tr.Search(boxMin(boxes[i]), boxMax(boxes[i]),
func(min, max []float64, value unsafe.Pointer) bool { func(min, max []float64, v *item.Item) bool {
if value == unsafe.Pointer(boxes[i]) { if v == boxes[i] {
found = true found = true
return false return false
} }
@ -192,7 +206,7 @@ func testBoxesVarious(t *testing.T, boxes []*tBox, label string) {
for i := 0; i < N/5; i++ { for i := 0; i < N/5; i++ {
var count int var count int
tr.Search(centerMin, centerMax, tr.Search(centerMin, centerMax,
func(min, max []float64, value unsafe.Pointer) bool { func(min, max []float64, _ *item.Item) bool {
count++ count++
return true return true
}, },
@ -204,14 +218,14 @@ func testBoxesVarious(t *testing.T, boxes []*tBox, label string) {
///////////////////////////////////////// /////////////////////////////////////////
for i := 0; i < N/2; i++ { for i := 0; i < N/2; i++ {
j := i * 2 j := i * 2
tr.Delete(boxes[j].min[:], boxes[j].max[:], unsafe.Pointer(boxes[j])) tr.Delete(boxMin(boxes[j]), boxMax(boxes[j]), boxes[j])
} }
///////////////////////////////////////// /////////////////////////////////////////
// count all items. should be half of N // count all items. should be half of N
///////////////////////////////////////// /////////////////////////////////////////
count = 0 count = 0
tr.Scan(func(min, max []float64, value unsafe.Pointer) bool { tr.Scan(func(min, max []float64, _ *item.Item) bool {
count++ count++
return true return true
}) })
@ -232,28 +246,29 @@ func testBoxesVarious(t *testing.T, boxes []*tBox, label string) {
}) })
for i := 0; i < N/2; i++ { for i := 0; i < N/2; i++ {
j := ij[i] j := ij[i]
tr.Insert(boxes[j].min[:], boxes[j].max[:], unsafe.Pointer(boxes[j])) tr.Insert(boxMin(boxes[j]), boxMax(boxes[j]), boxes[j])
} }
////////////////////////////////////////////////////// //////////////////////////////////////////////////////
// replace each item with an item that is very close // replace each item with an item that is very close
////////////////////////////////////////////////////// //////////////////////////////////////////////////////
var nboxes = make([]*tBox, N) var nboxes = make([]*item.Item, N)
for i := 0; i < N; i++ { for i := 0; i < N; i++ {
nboxes[i] = new(tBox) box := boxes[i].Obj().(*tBox)
for j := 0; j < len(boxes[i].min); j++ { nbox := new(tBox)
nboxes[i].min[j] = boxes[i].min[j] + (rand.Float64() - 0.5) for j := 0; j < len(box.min); j++ {
if boxes[i].min == boxes[i].max { nbox.min[j] = box.min[j] + (rand.Float64() - 0.5)
nboxes[i].max[j] = nboxes[i].min[j] if box.min == box.max {
nbox.max[j] = nbox.min[j]
} else { } else {
nboxes[i].max[j] = boxes[i].max[j] + (rand.Float64() - 0.5) nbox.max[j] = box.max[j] + (rand.Float64() - 0.5)
} }
} }
nboxes[i] = item.New(fmt.Sprintf("%d", i), nbox)
} }
for i := 0; i < N; i++ { for i := 0; i < N; i++ {
tr.Insert(nboxes[i].min[:], nboxes[i].max[:], unsafe.Pointer(nboxes[i])) tr.Insert(boxMin(nboxes[i]), boxMax(nboxes[i]), nboxes[i])
tr.Delete(boxes[i].min[:], boxes[i].max[:], unsafe.Pointer(boxes[i])) tr.Delete(boxMin(boxes[i]), boxMax(boxes[i]), boxes[i])
} }
if tr.Count() != N { if tr.Count() != N {
t.Fatalf("expected %d, got %d", N, tr.Count()) t.Fatalf("expected %d, got %d", N, tr.Count())
@ -265,11 +280,11 @@ func testBoxesVarious(t *testing.T, boxes []*tBox, label string) {
// check every point for correctness // check every point for correctness
///////////////////////////////////////// /////////////////////////////////////////
tboxes1 = nil tboxes1 = nil
tr.Scan(func(min, max []float64, value unsafe.Pointer) bool { tr.Scan(func(min, max []float64, value *item.Item) bool {
tboxes1 = append(tboxes1, (*tBox)(value)) tboxes1 = append(tboxes1, value)
return true return true
}) })
tboxes2 = make([]*tBox, len(nboxes)) tboxes2 = make([]*item.Item, len(nboxes))
copy(tboxes2, nboxes) copy(tboxes2, nboxes)
sortBoxes(tboxes1) sortBoxes(tboxes1)
sortBoxes(tboxes2) sortBoxes(tboxes2)
@ -285,17 +300,17 @@ func testBoxesVarious(t *testing.T, boxes []*tBox, label string) {
for i := 0; i < N/5; i++ { for i := 0; i < N/5; i++ {
var count int var count int
tr.Search(centerMin, centerMax, tr.Search(centerMin, centerMax,
func(min, max []float64, value unsafe.Pointer) bool { func(min, max []float64, value *item.Item) bool {
count++ count++
return true return true
}, },
) )
} }
var boxes3 []*tBox var boxes3 []*item.Item
tr.Nearby(centerMin, centerMax, tr.Nearby(centerMin, centerMax,
func(min, max []float64, value unsafe.Pointer) bool { func(min, max []float64, value *item.Item) bool {
boxes3 = append(boxes3, (*tBox)(value)) boxes3 = append(boxes3, value)
return true return true
}, },
) )
@ -307,7 +322,7 @@ func testBoxesVarious(t *testing.T, boxes []*tBox, label string) {
} }
var ldist float64 var ldist float64
for i, box := range boxes3 { for i, box := range boxes3 {
dist := testBoxDist(box.min[:], box.max[:], centerMin, centerMax) dist := testBoxDist(boxMin(box), boxMax(box), centerMin, centerMax)
if i > 0 && dist < ldist { if i > 0 && dist < ldist {
t.Fatalf("out of order") t.Fatalf("out of order")
} }
@ -378,6 +393,52 @@ func BenchmarkRandomInsert(b *testing.B) {
boxes := randBoxes(b.N) boxes := randBoxes(b.N)
b.ResetTimer() b.ResetTimer()
for i := 0; i < b.N; i++ { for i := 0; i < b.N; i++ {
tr.Insert(boxes[i].min[:], boxes[i].max[:], nil) tr.Insert(boxMin(boxes[i]), boxMax(boxes[i]), nil)
} }
} }
func (s *tBox) Spatial() geojson.Spatial {
return geojson.EmptySpatial{}
}
func (s *tBox) ForEach(iter func(geom geojson.Object) bool) bool {
return iter(s)
}
func (s *tBox) Empty() bool {
return true
}
func (s *tBox) Valid() bool {
return false
}
func (s *tBox) Rect() geometry.Rect {
return geometry.Rect{}
}
func (s *tBox) Center() geometry.Point {
return geometry.Point{}
}
func (s *tBox) AppendJSON(dst []byte) []byte {
return nil
}
func (s *tBox) String() string {
return ""
}
func (s *tBox) JSON() string {
return string(s.AppendJSON(nil))
}
func (s *tBox) MarshalJSON() ([]byte, error) {
return s.AppendJSON(nil), nil
}
func (s *tBox) Within(obj geojson.Object) bool {
return false
}
func (s *tBox) Contains(obj geojson.Object) bool {
return false
}
func (s *tBox) Intersects(obj geojson.Object) bool {
return false
}
func (s *tBox) NumPoints() int {
return 0
}
func (s *tBox) Distance(obj geojson.Object) float64 {
return 0
}