From 9d8ce14ffd4927dffc324e164043db775d79d96b Mon Sep 17 00:00:00 2001 From: tidwall Date: Wed, 13 Feb 2019 12:43:38 -0700 Subject: [PATCH] Refactor field optimizations --- .../collection/{ptrbtree => btree}/btree.go | 105 ++++------ .../{ptrbtree => btree}/btree_test.go | 175 +++++++++------- internal/collection/collection.go | 183 +++++++++-------- internal/collection/item.go | 149 -------------- internal/collection/item/item.go | 153 ++++++++++++++ internal/collection/item/item_test.go | 189 ++++++++++++++++++ .../collection/{ptrrtree => rtree}/rtree.go | 55 ++--- .../{ptrrtree => rtree}/rtree_test.go | 187 +++++++++++------ 8 files changed, 744 insertions(+), 452 deletions(-) rename internal/collection/{ptrbtree => btree}/btree.go (74%) rename internal/collection/{ptrbtree => btree}/btree_test.go (75%) delete mode 100644 internal/collection/item.go create mode 100644 internal/collection/item/item.go create mode 100644 internal/collection/item/item_test.go rename internal/collection/{ptrrtree => rtree}/rtree.go (91%) rename internal/collection/{ptrrtree => rtree}/rtree_test.go (60%) diff --git a/internal/collection/ptrbtree/btree.go b/internal/collection/btree/btree.go similarity index 74% rename from internal/collection/ptrbtree/btree.go rename to internal/collection/btree/btree.go index bc1ef8ab..484fc512 100644 --- a/internal/collection/ptrbtree/btree.go +++ b/internal/collection/btree/btree.go @@ -1,37 +1,15 @@ -package ptrbtree +// Package btree is designed to work specifically with +// the Tile38 collection/item.Item type. +package btree -import ( - "reflect" - "unsafe" -) +import "github.com/tidwall/tile38/internal/collection/item" const maxItems = 31 // use an odd number const minItems = maxItems * 40 / 100 -type btreeItem struct { - ptr unsafe.Pointer -} - -// keyedItem must match layout of ../collection/itemT, otherwise -// there's a risk for memory corruption. -type keyedItem struct { - obj interface{} - fieldLen uint32 - keyLen uint32 - data unsafe.Pointer -} - -func (v btreeItem) key() string { - return *(*string)((unsafe.Pointer)(&reflect.StringHeader{ - Data: uintptr(unsafe.Pointer((*keyedItem)(v.ptr).data)) + - uintptr((*keyedItem)(v.ptr).fieldLen), - Len: int((*keyedItem)(v.ptr).keyLen), - })) -} - type node struct { numItems int - items [maxItems]btreeItem + items [maxItems]*item.Item children [maxItems + 1]*node } @@ -47,29 +25,28 @@ func (n *node) find(key string) (index int, found bool) { i, j := 0, n.numItems for i < j { h := i + (j-i)/2 - if key >= n.items[h].key() { + if key >= n.items[h].ID() { i = h + 1 } else { j = h } } - if i > 0 && n.items[i-1].key() >= key { + if i > 0 && n.items[i-1].ID() >= key { return i - 1, true } return i, false } // Set or replace a value for a key -func (tr *BTree) Set(ptr unsafe.Pointer) (prev unsafe.Pointer, replaced bool) { - newItem := btreeItem{ptr} +func (tr *BTree) Set(item *item.Item) (prev *item.Item, replaced bool) { if tr.root == nil { tr.root = new(node) - tr.root.items[0] = newItem + tr.root.items[0] = item tr.root.numItems = 1 tr.length = 1 return } - prev, replaced = tr.root.set(newItem, tr.height) + prev, replaced = tr.root.set(item, tr.height) if replaced { return } @@ -87,7 +64,7 @@ func (tr *BTree) Set(ptr unsafe.Pointer) (prev unsafe.Pointer, replaced bool) { return } -func (n *node) split(height int) (right *node, median btreeItem) { +func (n *node) split(height int) (right *node, median *item.Item) { right = new(node) median = n.items[maxItems/2] copy(right.items[:maxItems/2], n.items[maxItems/2+1:]) @@ -101,16 +78,16 @@ func (n *node) split(height int) (right *node, median btreeItem) { } } for i := maxItems / 2; i < maxItems; i++ { - n.items[i] = btreeItem{} + n.items[i] = nil } n.numItems = maxItems / 2 return } -func (n *node) set(newItem btreeItem, height int) (prev unsafe.Pointer, replaced bool) { - i, found := n.find(newItem.key()) +func (n *node) set(newItem *item.Item, height int) (prev *item.Item, replaced bool) { + i, found := n.find(newItem.ID()) if found { - prev = n.items[i].ptr + prev = n.items[i] n.items[i] = newItem return prev, true } @@ -138,16 +115,16 @@ func (n *node) set(newItem btreeItem, height int) (prev unsafe.Pointer, replaced } // Scan all items in tree -func (tr *BTree) Scan(iter func(ptr unsafe.Pointer) bool) { +func (tr *BTree) Scan(iter func(item *item.Item) bool) { if tr.root != nil { tr.root.scan(iter, tr.height) } } -func (n *node) scan(iter func(ptr unsafe.Pointer) bool, height int) bool { +func (n *node) scan(iter func(item *item.Item) bool, height int) bool { if height == 0 { for i := 0; i < n.numItems; i++ { - if !iter(n.items[i].ptr) { + if !iter(n.items[i]) { return false } } @@ -157,7 +134,7 @@ func (n *node) scan(iter func(ptr unsafe.Pointer) bool, height int) bool { if !n.children[i].scan(iter, height-1) { return false } - if !iter(n.items[i].ptr) { + if !iter(n.items[i]) { return false } } @@ -165,17 +142,17 @@ func (n *node) scan(iter func(ptr unsafe.Pointer) bool, height int) bool { } // Get a value for key -func (tr *BTree) Get(key string) (ptr unsafe.Pointer, gotten bool) { +func (tr *BTree) Get(key string) (item *item.Item, gotten bool) { if tr.root == nil { return } return tr.root.get(key, tr.height) } -func (n *node) get(key string, height int) (ptr unsafe.Pointer, gotten bool) { +func (n *node) get(key string, height int) (item *item.Item, gotten bool) { i, found := n.find(key) if found { - return n.items[i].ptr, true + return n.items[i], true } if height == 0 { return nil, false @@ -189,16 +166,16 @@ func (tr *BTree) Len() int { } // Delete a value for a key -func (tr *BTree) Delete(key string) (prev unsafe.Pointer, deleted bool) { +func (tr *BTree) Delete(key string) (prev *item.Item, deleted bool) { if tr.root == nil { return } - var prevItem btreeItem + var prevItem *item.Item prevItem, deleted = tr.root.delete(false, key, tr.height) if !deleted { return } - prev = prevItem.ptr + prev = prevItem if tr.root.numItems == 0 { tr.root = tr.root.children[0] tr.height-- @@ -212,7 +189,7 @@ func (tr *BTree) Delete(key string) (prev unsafe.Pointer, deleted bool) { } func (n *node) delete(max bool, key string, height int) ( - prev btreeItem, deleted bool, + prev *item.Item, deleted bool, ) { i, found := 0, false if max { @@ -225,12 +202,12 @@ func (n *node) delete(max bool, key string, height int) ( prev = n.items[i] // found the items at the leaf, remove it and return. copy(n.items[i:], n.items[i+1:n.numItems]) - n.items[n.numItems-1] = btreeItem{} + n.items[n.numItems-1] = nil n.children[n.numItems] = nil n.numItems-- return prev, true } - return btreeItem{}, false + return nil, false } if found { @@ -254,7 +231,7 @@ func (n *node) delete(max bool, key string, height int) ( i-- } if n.children[i].numItems+n.children[i+1].numItems+1 < maxItems { - // merge left + btreeItem + right + // merge left + *item.Item + right n.children[i].items[n.children[i].numItems] = n.items[i] copy(n.children[i].items[n.children[i].numItems+1:], n.children[i+1].items[:n.children[i+1].numItems]) @@ -265,7 +242,7 @@ func (n *node) delete(max bool, key string, height int) ( n.children[i].numItems += n.children[i+1].numItems + 1 copy(n.items[i:], n.items[i+1:n.numItems]) copy(n.children[i+1:], n.children[i+2:n.numItems+1]) - n.items[n.numItems] = btreeItem{} + n.items[n.numItems] = nil n.children[n.numItems+1] = nil n.numItems-- } else if n.children[i].numItems > n.children[i+1].numItems { @@ -283,7 +260,7 @@ func (n *node) delete(max bool, key string, height int) ( } n.children[i+1].numItems++ n.items[i] = n.children[i].items[n.children[i].numItems-1] - n.children[i].items[n.children[i].numItems-1] = btreeItem{} + n.children[i].items[n.children[i].numItems-1] = nil if height > 1 { n.children[i].children[n.children[i].numItems] = nil } @@ -310,13 +287,13 @@ func (n *node) delete(max bool, key string, height int) ( } // Ascend the tree within the range [pivot, last] -func (tr *BTree) Ascend(pivot string, iter func(ptr unsafe.Pointer) bool) { +func (tr *BTree) Ascend(pivot string, iter func(item *item.Item) bool) { if tr.root != nil { tr.root.ascend(pivot, iter, tr.height) } } -func (n *node) ascend(pivot string, iter func(ptr unsafe.Pointer) bool, height int) bool { +func (n *node) ascend(pivot string, iter func(item *item.Item) bool, height int) bool { i, found := n.find(pivot) if !found { if height > 0 { @@ -326,7 +303,7 @@ func (n *node) ascend(pivot string, iter func(ptr unsafe.Pointer) bool, height i } } for ; i < n.numItems; i++ { - if !iter(n.items[i].ptr) { + if !iter(n.items[i]) { return false } if height > 0 { @@ -339,16 +316,16 @@ func (n *node) ascend(pivot string, iter func(ptr unsafe.Pointer) bool, height i } // Reverse all items in tree -func (tr *BTree) Reverse(iter func(ptr unsafe.Pointer) bool) { +func (tr *BTree) Reverse(iter func(item *item.Item) bool) { if tr.root != nil { tr.root.reverse(iter, tr.height) } } -func (n *node) reverse(iter func(ptr unsafe.Pointer) bool, height int) bool { +func (n *node) reverse(iter func(item *item.Item) bool, height int) bool { if height == 0 { for i := n.numItems - 1; i >= 0; i-- { - if !iter(n.items[i].ptr) { + if !iter(n.items[i]) { return false } } @@ -358,7 +335,7 @@ func (n *node) reverse(iter func(ptr unsafe.Pointer) bool, height int) bool { return false } for i := n.numItems - 1; i >= 0; i-- { - if !iter(n.items[i].ptr) { + if !iter(n.items[i]) { return false } if !n.children[i].reverse(iter, height-1) { @@ -371,14 +348,14 @@ func (n *node) reverse(iter func(ptr unsafe.Pointer) bool, height int) bool { // Descend the tree within the range [pivot, first] func (tr *BTree) Descend( pivot string, - iter func(ptr unsafe.Pointer) bool, + iter func(item *item.Item) bool, ) { if tr.root != nil { tr.root.descend(pivot, iter, tr.height) } } -func (n *node) descend(pivot string, iter func(ptr unsafe.Pointer) bool, height int) bool { +func (n *node) descend(pivot string, iter func(item *item.Item) bool, height int) bool { i, found := n.find(pivot) if !found { if height > 0 { @@ -389,7 +366,7 @@ func (n *node) descend(pivot string, iter func(ptr unsafe.Pointer) bool, height i-- } for ; i >= 0; i-- { - if !iter(n.items[i].ptr) { + if !iter(n.items[i]) { return false } if height > 0 { diff --git a/internal/collection/ptrbtree/btree_test.go b/internal/collection/btree/btree_test.go similarity index 75% rename from internal/collection/ptrbtree/btree_test.go rename to internal/collection/btree/btree_test.go index a12aa463..5f1a889d 100644 --- a/internal/collection/ptrbtree/btree_test.go +++ b/internal/collection/btree/btree_test.go @@ -1,35 +1,19 @@ -package ptrbtree +package btree import ( + "encoding/json" "fmt" "math/rand" "sort" "strings" "testing" "time" - "unsafe" + + "github.com/tidwall/geojson" + "github.com/tidwall/geojson/geometry" + "github.com/tidwall/tile38/internal/collection/item" ) -func makeItem(key string, obj interface{}) unsafe.Pointer { - item := new(keyedItem) - item.obj = obj - if len(key) > 0 { - data := make([]byte, len(key)) - copy(data, key) - item.keyLen = uint32(len(key)) - item.data = unsafe.Pointer(&data[0]) - } - return unsafe.Pointer(item) -} - -func itemKey(ptr unsafe.Pointer) string { - return (btreeItem{ptr}).key() -} - -func itemValue(ptr unsafe.Pointer) interface{} { - return (*keyedItem)(ptr).obj -} - func init() { seed := time.Now().UnixNano() fmt.Printf("seed: %d\n", seed) @@ -63,12 +47,12 @@ func (n *node) print(level, height int) { n.children[i].print(level+1, height-1) } if height > 0 || (height == 0 && !flatLeaf) { - fmt.Printf("%s%v\n", strings.Repeat(" ", level), n.items[i].key()) + fmt.Printf("%s%v\n", strings.Repeat(" ", level), n.items[i].ID()) } else { if i > 0 { fmt.Printf(",") } - fmt.Printf("%s", n.items[i].key()) + fmt.Printf("%s", n.items[i].ID()) } } if height == 0 && flatLeaf { @@ -117,7 +101,7 @@ func stringsEquals(a, b []string) bool { func TestDescend(t *testing.T) { var tr BTree var count int - tr.Descend("1", func(ptr unsafe.Pointer) bool { + tr.Descend("1", func(item *item.Item) bool { count++ return true }) @@ -127,26 +111,26 @@ func TestDescend(t *testing.T) { var keys []string for i := 0; i < 1000; i += 10 { keys = append(keys, fmt.Sprintf("%03d", i)) - tr.Set(makeItem(keys[len(keys)-1], nil)) + tr.Set(item.New(keys[len(keys)-1], nil)) } var exp []string - tr.Reverse(func(ptr unsafe.Pointer) bool { - exp = append(exp, itemKey(ptr)) + tr.Reverse(func(item *item.Item) bool { + exp = append(exp, item.ID()) return true }) for i := 999; i >= 0; i-- { var key string key = fmt.Sprintf("%03d", i) var all []string - tr.Descend(key, func(ptr unsafe.Pointer) bool { - all = append(all, itemKey(ptr)) + tr.Descend(key, func(item *item.Item) bool { + all = append(all, item.ID()) return true }) for len(exp) > 0 && key < exp[0] { exp = exp[1:] } var count int - tr.Descend(key, func(ptr unsafe.Pointer) bool { + tr.Descend(key, func(item *item.Item) bool { if count == (i+1)%maxItems { return false } @@ -168,7 +152,7 @@ func TestDescend(t *testing.T) { func TestAscend(t *testing.T) { var tr BTree var count int - tr.Ascend("1", func(ptr unsafe.Pointer) bool { + tr.Ascend("1", func(item *item.Item) bool { count++ return true }) @@ -178,7 +162,7 @@ func TestAscend(t *testing.T) { var keys []string for i := 0; i < 1000; i += 10 { keys = append(keys, fmt.Sprintf("%03d", i)) - tr.Set(makeItem(keys[len(keys)-1], nil)) + tr.Set(item.New(keys[len(keys)-1], nil)) } exp := keys for i := -1; i < 1000; i++ { @@ -189,8 +173,8 @@ func TestAscend(t *testing.T) { key = fmt.Sprintf("%03d", i) } var all []string - tr.Ascend(key, func(ptr unsafe.Pointer) bool { - all = append(all, itemKey(ptr)) + tr.Ascend(key, func(item *item.Item) bool { + all = append(all, item.ID()) return true }) @@ -198,7 +182,7 @@ func TestAscend(t *testing.T) { exp = exp[1:] } var count int - tr.Ascend(key, func(ptr unsafe.Pointer) bool { + tr.Ascend(key, func(item *item.Item) bool { if count == (i+1)%maxItems { return false } @@ -221,7 +205,7 @@ func TestBTree(t *testing.T) { // insert all items for _, key := range keys { - value, replaced := tr.Set(makeItem(key, key)) + value, replaced := tr.Set(item.New(key, testString(key))) if replaced { t.Fatal("expected false") } @@ -241,7 +225,7 @@ func TestBTree(t *testing.T) { if !gotten { t.Fatal("expected true") } - if value == nil || itemValue(value) != key { + if value == nil || value.Obj().String() != key { t.Fatalf("expected '%v', got '%v'", key, value) } } @@ -249,15 +233,15 @@ func TestBTree(t *testing.T) { // scan all items var last string all := make(map[string]interface{}) - tr.Scan(func(ptr unsafe.Pointer) bool { - if itemKey(ptr) <= last { + tr.Scan(func(item *item.Item) bool { + if item.ID() <= last { t.Fatal("out of order") } - if itemValue(ptr).(string) != itemKey(ptr) { + if item.Obj().String() != item.ID() { t.Fatalf("mismatch") } - last = itemKey(ptr) - all[itemKey(ptr)] = itemValue(ptr) + last = item.ID() + all[item.ID()] = item.Obj().String() return true }) if len(all) != len(keys) { @@ -267,15 +251,15 @@ func TestBTree(t *testing.T) { // reverse all items var prev string all = make(map[string]interface{}) - tr.Reverse(func(ptr unsafe.Pointer) bool { - if prev != "" && itemKey(ptr) >= prev { + tr.Reverse(func(item *item.Item) bool { + if prev != "" && item.ID() >= prev { t.Fatal("out of order") } - if itemValue(ptr).(string) != itemKey(ptr) { + if item.Obj().String() != item.ID() { t.Fatalf("mismatch") } - prev = itemKey(ptr) - all[itemKey(ptr)] = itemValue(ptr) + prev = item.ID() + all[item.ID()] = item.Obj().String() return true }) if len(all) != len(keys) { @@ -294,7 +278,7 @@ func TestBTree(t *testing.T) { // scan and quit at various steps for i := 0; i < 100; i++ { var j int - tr.Scan(func(ptr unsafe.Pointer) bool { + tr.Scan(func(item *item.Item) bool { if j == i { return false } @@ -306,7 +290,7 @@ func TestBTree(t *testing.T) { // reverse and quit at various steps for i := 0; i < 100; i++ { var j int - tr.Reverse(func(ptr unsafe.Pointer) bool { + tr.Reverse(func(item *item.Item) bool { if j == i { return false } @@ -321,7 +305,7 @@ func TestBTree(t *testing.T) { if !deleted { t.Fatal("expected true") } - if value == nil || itemValue(value).(string) != key { + if value == nil || value.Obj().String() != key { t.Fatalf("expected '%v', got '%v'", key, value) } } @@ -361,15 +345,15 @@ func TestBTree(t *testing.T) { // scan items last = "" all = make(map[string]interface{}) - tr.Scan(func(ptr unsafe.Pointer) bool { - if itemKey(ptr) <= last { + tr.Scan(func(item *item.Item) bool { + if item.ID() <= last { t.Fatal("out of order") } - if itemValue(ptr).(string) != itemKey(ptr) { + if item.Obj().String() != item.ID() { t.Fatalf("mismatch") } - last = itemKey(ptr) - all[itemKey(ptr)] = itemValue(ptr) + last = item.ID() + all[item.ID()] = item.Obj().String() return true }) if len(all) != len(keys)/2 { @@ -378,11 +362,11 @@ func TestBTree(t *testing.T) { // replace second half for _, key := range keys[len(keys)/2:] { - value, replaced := tr.Set(makeItem(key, key)) + value, replaced := tr.Set(item.New(key, testString(key))) if !replaced { t.Fatal("expected true") } - if value == nil || itemValue(value).(string) != key { + if value == nil || value.Obj().String() != key { t.Fatalf("expected '%v', got '%v'", key, value) } } @@ -393,7 +377,7 @@ func TestBTree(t *testing.T) { if !deleted { t.Fatal("expected true") } - if value == nil || itemValue(value).(string) != key { + if value == nil || value.Obj().String() != key { t.Fatalf("expected '%v', got '%v'", key, value) } } @@ -411,11 +395,11 @@ func TestBTree(t *testing.T) { if value != nil { t.Fatal("expected nil") } - tr.Scan(func(ptr unsafe.Pointer) bool { + tr.Scan(func(item *item.Item) bool { t.Fatal("should not be reached") return true }) - tr.Reverse(func(ptr unsafe.Pointer) bool { + tr.Reverse(func(item *item.Item) bool { t.Fatal("should not be reached") return true }) @@ -436,7 +420,7 @@ func BenchmarkTidwallSequentialSet(b *testing.B) { sort.Strings(keys) b.ResetTimer() for i := 0; i < b.N; i++ { - tr.Set(makeItem(keys[i], nil)) + tr.Set(item.New(keys[i], nil)) } } @@ -445,7 +429,7 @@ func BenchmarkTidwallSequentialGet(b *testing.B) { keys := randKeys(b.N) sort.Strings(keys) for i := 0; i < b.N; i++ { - tr.Set(makeItem(keys[i], nil)) + tr.Set(item.New(keys[i], nil)) } b.ResetTimer() for i := 0; i < b.N; i++ { @@ -458,7 +442,7 @@ func BenchmarkTidwallRandomSet(b *testing.B) { keys := randKeys(b.N) b.ResetTimer() for i := 0; i < b.N; i++ { - tr.Set(makeItem(keys[i], nil)) + tr.Set(item.New(keys[i], nil)) } } @@ -466,7 +450,7 @@ func BenchmarkTidwallRandomGet(b *testing.B) { var tr BTree keys := randKeys(b.N) for i := 0; i < b.N; i++ { - tr.Set(makeItem(keys[i], nil)) + tr.Set(item.New(keys[i], nil)) } b.ResetTimer() for i := 0; i < b.N; i++ { @@ -544,11 +528,11 @@ func BenchmarkTidwallRandomGet(b *testing.B) { func TestBTreeOne(t *testing.T) { var tr BTree - tr.Set(makeItem("1", "1")) + tr.Set(item.New("1", testString("1"))) tr.Delete("1") - tr.Set(makeItem("1", "1")) + tr.Set(item.New("1", testString("1"))) tr.Delete("1") - tr.Set(makeItem("1", "1")) + tr.Set(item.New("1", testString("1"))) tr.Delete("1") } @@ -557,7 +541,7 @@ func TestBTree256(t *testing.T) { var n int for j := 0; j < 2; j++ { for _, i := range rand.Perm(256) { - tr.Set(makeItem(fmt.Sprintf("%d", i), i)) + tr.Set(item.New(fmt.Sprintf("%d", i), testString(fmt.Sprintf("%d", i)))) n++ if tr.Len() != n { t.Fatalf("expected 256, got %d", n) @@ -568,8 +552,8 @@ func TestBTree256(t *testing.T) { if !ok { t.Fatal("expected true") } - if itemValue(v).(int) != i { - t.Fatalf("expected %d, got %d", i, itemValue(v).(int)) + if v.Obj().String() != fmt.Sprintf("%d", i) { + t.Fatalf("expected %d, got %s", i, v.Obj().String()) } } for _, i := range rand.Perm(256) { @@ -587,3 +571,52 @@ func TestBTree256(t *testing.T) { } } } + +type testString string + +func (s testString) Spatial() geojson.Spatial { + return geojson.EmptySpatial{} +} +func (s testString) ForEach(iter func(geom geojson.Object) bool) bool { + return iter(s) +} +func (s testString) Empty() bool { + return true +} +func (s testString) Valid() bool { + return false +} +func (s testString) Rect() geometry.Rect { + return geometry.Rect{} +} +func (s testString) Center() geometry.Point { + return geometry.Point{} +} +func (s testString) AppendJSON(dst []byte) []byte { + data, _ := json.Marshal(string(s)) + return append(dst, data...) +} +func (s testString) String() string { + return string(s) +} +func (s testString) JSON() string { + return string(s.AppendJSON(nil)) +} +func (s testString) MarshalJSON() ([]byte, error) { + return s.AppendJSON(nil), nil +} +func (s testString) Within(obj geojson.Object) bool { + return false +} +func (s testString) Contains(obj geojson.Object) bool { + return false +} +func (s testString) Intersects(obj geojson.Object) bool { + return false +} +func (s testString) NumPoints() int { + return 0 +} +func (s testString) Distance(obj geojson.Object) float64 { + return 0 +} diff --git a/internal/collection/collection.go b/internal/collection/collection.go index e0747c47..5167e70f 100644 --- a/internal/collection/collection.go +++ b/internal/collection/collection.go @@ -1,14 +1,13 @@ package collection import ( - "unsafe" - - "github.com/tidwall/btree" + ifbtree "github.com/tidwall/btree" "github.com/tidwall/geojson" "github.com/tidwall/geojson/geo" "github.com/tidwall/geojson/geometry" - "github.com/tidwall/tile38/internal/collection/ptrbtree" - "github.com/tidwall/tile38/internal/collection/ptrrtree" + "github.com/tidwall/tile38/internal/collection/btree" + "github.com/tidwall/tile38/internal/collection/item" + "github.com/tidwall/tile38/internal/collection/rtree" ) // Cursor allows for quickly paging through Scan, Within, Intersects, and Nearby @@ -19,9 +18,9 @@ type Cursor interface { // Collection represents a collection of geojson objects. type Collection struct { - items ptrbtree.BTree // items sorted by keys - index ptrrtree.BoxTree // items geospatially indexed - values *btree.BTree // items sorted by value+key + items btree.BTree // items sorted by keys + index rtree.BoxTree // items geospatially indexed + values *ifbtree.BTree // items sorted by value+key fieldMap map[string]int weight int points int @@ -34,7 +33,7 @@ var counter uint64 // New creates an empty collection func New() *Collection { col := &Collection{ - values: btree.New(16, nil), + values: ifbtree.New(16, nil), fieldMap: make(map[string]int), } return col @@ -74,48 +73,40 @@ func objIsSpatial(obj geojson.Object) bool { return ok } -func (c *Collection) indexDelete(item *itemT) { - if !item.obj.Empty() { - rect := item.obj.Rect() - c.index.Delete( - []float64{rect.Min.X, rect.Min.Y}, - []float64{rect.Max.X, rect.Max.Y}, - unsafe.Pointer(item)) - } -} - -func (c *Collection) indexInsert(item *itemT) { - if !item.obj.Empty() { - rect := item.obj.Rect() - c.index.Insert( - []float64{rect.Min.X, rect.Min.Y}, - []float64{rect.Max.X, rect.Max.Y}, - unsafe.Pointer(item)) - } -} - -func (c *Collection) addItem(item *itemT) { - if objIsSpatial(item.obj) { - c.indexInsert(item) +func (c *Collection) addItem(item *item.Item) { + if objIsSpatial(item.Obj()) { + if !item.Obj().Empty() { + rect := item.Obj().Rect() + c.index.Insert( + []float64{rect.Min.X, rect.Min.Y}, + []float64{rect.Max.X, rect.Max.Y}, + item) + } c.objects++ } else { c.values.ReplaceOrInsert(item) c.nobjects++ } - weight, points := item.weightAndPoints() + weight, points := item.WeightAndPoints() c.weight += weight c.points += points } -func (c *Collection) delItem(item *itemT) { - if objIsSpatial(item.obj) { - c.indexDelete(item) +func (c *Collection) delItem(item *item.Item) { + if objIsSpatial(item.Obj()) { + if !item.Obj().Empty() { + rect := item.Obj().Rect() + c.index.Delete( + []float64{rect.Min.X, rect.Min.Y}, + []float64{rect.Max.X, rect.Max.Y}, + item) + } c.objects-- } else { c.values.Delete(item) c.nobjects-- } - weight, points := item.weightAndPoints() + weight, points := item.WeightAndPoints() c.weight -= weight c.points -= points } @@ -131,26 +122,26 @@ func (c *Collection) Set( oldObj geojson.Object, oldFields []float64, newFields []float64, ) { // create the new item - item := newItem(id, obj) + item := item.New(id, obj) // add the new item to main btree and remove the old one if needed - oldItemV, ok := c.items.Set(unsafe.Pointer(item)) + oldItemV, ok := c.items.Set(item) if ok { - oldItem := (*itemT)(oldItemV) - oldObj = oldItem.obj + oldItem := oldItemV + oldObj = oldItem.Obj() // remove old item from indexes c.delItem(oldItem) - if len(oldItem.fields()) > 0 { + if len(oldItem.Fields()) > 0 { // merge old and new fields - oldFields = oldItem.fields() - item.directCopyFields(oldFields) + oldFields = oldItem.Fields() + item.CopyOverFields(oldFields) } } if fields == nil && len(values) > 0 { // directly set the field values, from copy - item.directCopyFields(values) + item.CopyOverFields(values) } else if len(fields) > 0 { // add new field to new item c.setFields(item, fields, values, false) @@ -160,7 +151,42 @@ func (c *Collection) Set( c.addItem(item) // fmt.Printf("!!! %#v\n", oldObj) - return oldObj, oldFields, item.fields() + return oldObj, oldFields, item.Fields() +} + +func (c *Collection) setFields( + item *item.Item, fieldNames []string, fieldValues []float64, updateWeight bool, +) (updatedCount int) { + for i, fieldName := range fieldNames { + var fieldValue float64 + if i < len(fieldValues) { + fieldValue = fieldValues[i] + } + if c.setField(item, fieldName, fieldValue, updateWeight) { + updatedCount++ + } + } + return updatedCount +} + +func (c *Collection) setField( + item *item.Item, fieldName string, fieldValue float64, updateWeight bool, +) (updated bool) { + idx, ok := c.fieldMap[fieldName] + if !ok { + idx = len(c.fieldMap) + c.fieldMap[fieldName] = idx + } + var pweight int + if updateWeight { + pweight, _ = item.WeightAndPoints() + } + updated = item.SetField(idx, fieldValue) + if updateWeight && updated { + nweight, _ := item.WeightAndPoints() + c.weight = c.weight - pweight + nweight + } + return updated } // Delete removes an object and returns it. @@ -172,11 +198,11 @@ func (c *Collection) Delete(id string) ( if !ok { return nil, nil, false } - oldItem := (*itemT)(oldItemV) + oldItem := oldItemV c.delItem(oldItem) - return oldItem.obj, oldItem.fields(), true + return oldItem.Obj(), oldItem.Fields(), true } // Get returns an object. @@ -188,9 +214,9 @@ func (c *Collection) Get(id string) ( if !ok { return nil, nil, false } - item := (*itemT)(itemV) + item := itemV - return item.obj, item.fields(), true + return item.Obj(), item.Fields(), true } // SetField set a field value for an object and returns that object. @@ -202,9 +228,9 @@ func (c *Collection) SetField(id, fieldName string, fieldValue float64) ( if !ok { return nil, nil, false, false } - item := (*itemT)(itemV) + item := itemV updated = c.setField(item, fieldName, fieldValue, true) - return item.obj, item.fields(), updated, true + return item.Obj(), item.Fields(), updated, true } // SetFields is similar to SetField, just setting multiple fields at once @@ -215,11 +241,11 @@ func (c *Collection) SetFields( if !ok { return nil, nil, 0, false } - item := (*itemT)(itemV) + item := itemV updatedCount = c.setFields(item, fieldNames, fieldValues, true) - return item.obj, item.fields(), updatedCount, true + return item.Obj(), item.Fields(), updatedCount, true } // FieldMap return a maps of the field names. @@ -247,7 +273,7 @@ func (c *Collection) Scan(desc bool, cursor Cursor, offset = cursor.Offset() cursor.Step(offset) } - iter := func(ptr unsafe.Pointer) bool { + iter := func(item *item.Item) bool { count++ if count <= offset { return true @@ -255,8 +281,7 @@ func (c *Collection) Scan(desc bool, cursor Cursor, if cursor != nil { cursor.Step(1) } - iitm := (*itemT)(ptr) - keepon = iterator(iitm.id(), iitm.obj, iitm.fields()) + keepon = iterator(item.ID(), item.Obj(), item.Fields()) return keepon } if desc { @@ -278,7 +303,7 @@ func (c *Collection) ScanRange(start, end string, desc bool, cursor Cursor, offset = cursor.Offset() cursor.Step(offset) } - iter := func(ptr unsafe.Pointer) bool { + iter := func(item *item.Item) bool { count++ if count <= offset { return true @@ -286,17 +311,16 @@ func (c *Collection) ScanRange(start, end string, desc bool, cursor Cursor, if cursor != nil { cursor.Step(1) } - iitm := (*itemT)(ptr) if !desc { - if iitm.id() >= end { + if item.ID() >= end { return false } } else { - if iitm.id() <= end { + if item.ID() <= end { return false } } - keepon = iterator(iitm.id(), iitm.obj, iitm.fields()) + keepon = iterator(item.ID(), item.Obj(), item.Fields()) return keepon } @@ -319,7 +343,7 @@ func (c *Collection) SearchValues(desc bool, cursor Cursor, offset = cursor.Offset() cursor.Step(offset) } - iter := func(item btree.Item) bool { + iter := func(v ifbtree.Item) bool { count++ if count <= offset { return true @@ -327,8 +351,8 @@ func (c *Collection) SearchValues(desc bool, cursor Cursor, if cursor != nil { cursor.Step(1) } - iitm := item.(*itemT) - keepon = iterator(iitm.id(), iitm.obj, iitm.fields()) + iitm := v.(*item.Item) + keepon = iterator(iitm.ID(), iitm.Obj(), iitm.Fields()) return keepon } if desc { @@ -351,7 +375,7 @@ func (c *Collection) SearchValuesRange(start, end string, desc bool, offset = cursor.Offset() cursor.Step(offset) } - iter := func(item btree.Item) bool { + iter := func(v ifbtree.Item) bool { count++ if count <= offset { return true @@ -359,17 +383,17 @@ func (c *Collection) SearchValuesRange(start, end string, desc bool, if cursor != nil { cursor.Step(1) } - iitm := item.(*itemT) - keepon = iterator(iitm.id(), iitm.obj, iitm.fields()) + iitm := v.(*item.Item) + keepon = iterator(iitm.ID(), iitm.Obj(), iitm.Fields()) return keepon } if desc { c.values.DescendRange( - newItem("", String(start)), newItem("", String(end)), iter, + item.New("", String(start)), item.New("", String(end)), iter, ) } else { c.values.AscendRange( - newItem("", String(start)), newItem("", String(end)), iter, + item.New("", String(start)), item.New("", String(end)), iter, ) } return keepon @@ -387,7 +411,7 @@ func (c *Collection) ScanGreaterOrEqual(id string, desc bool, offset = cursor.Offset() cursor.Step(offset) } - iter := func(ptr unsafe.Pointer) bool { + iter := func(item *item.Item) bool { count++ if count <= offset { return true @@ -395,8 +419,7 @@ func (c *Collection) ScanGreaterOrEqual(id string, desc bool, if cursor != nil { cursor.Step(1) } - iitm := (*itemT)(ptr) - keepon = iterator(iitm.id(), iitm.obj, iitm.fields()) + keepon = iterator(item.ID(), item.Obj(), item.Fields()) return keepon } if desc { @@ -415,9 +438,9 @@ func (c *Collection) geoSearch( c.index.Search( []float64{rect.Min.X, rect.Min.Y}, []float64{rect.Max.X, rect.Max.Y}, - func(_, _ []float64, itemv unsafe.Pointer) bool { - item := (*itemT)(itemv) - alive = iter(item.id(), item.obj, item.fields()) + func(_, _ []float64, itemv *item.Item) bool { + item := itemv + alive = iter(item.ID(), item.Obj(), item.Fields()) return alive }, ) @@ -610,7 +633,7 @@ func (c *Collection) Nearby( c.index.Search( []float64{minLon, minLat}, []float64{maxLon, maxLat}, - func(_, _ []float64, itemv unsafe.Pointer) bool { + func(_, _ []float64, itemv *item.Item) bool { exists = true return false }, @@ -633,7 +656,7 @@ func (c *Collection) Nearby( c.index.Nearby( []float64{center.X, center.Y}, []float64{center.X, center.Y}, - func(_, _ []float64, itemv unsafe.Pointer) bool { + func(_, _ []float64, itemv *item.Item) bool { count++ if count <= offset { return true @@ -641,8 +664,8 @@ func (c *Collection) Nearby( if cursor != nil { cursor.Step(1) } - item := (*itemT)(itemv) - alive = iter(item.id(), item.obj, item.fields()) + item := itemv + alive = iter(item.ID(), item.Obj(), item.Fields()) return alive }, ) diff --git a/internal/collection/item.go b/internal/collection/item.go deleted file mode 100644 index 37abe630..00000000 --- a/internal/collection/item.go +++ /dev/null @@ -1,149 +0,0 @@ -package collection - -import ( - "reflect" - "unsafe" - - "github.com/tidwall/btree" - "github.com/tidwall/geojson" -) - -type itemT struct { - obj geojson.Object - fieldsLen uint32 // fields block size in bytes, not num of fields - idLen uint32 // id block size in bytes - data unsafe.Pointer -} - -func (item *itemT) id() string { - return *(*string)((unsafe.Pointer)(&reflect.StringHeader{ - Data: uintptr(unsafe.Pointer(item.data)) + uintptr(item.fieldsLen), - Len: int(item.idLen), - })) -} - -func (item *itemT) fields() []float64 { - return *(*[]float64)((unsafe.Pointer)(&reflect.SliceHeader{ - Data: uintptr(unsafe.Pointer(item.data)), - Len: int(item.fieldsLen) / 8, - Cap: int(item.fieldsLen) / 8, - })) -} - -func (item *itemT) dataBytes() []byte { - return *(*[]byte)((unsafe.Pointer)(&reflect.SliceHeader{ - Data: uintptr(unsafe.Pointer(item.data)), - Len: int(item.fieldsLen) + int(item.idLen), - Cap: int(item.fieldsLen) + int(item.idLen), - })) -} - -func newItem(id string, obj geojson.Object) *itemT { - item := new(itemT) - item.obj = obj - item.idLen = uint32(len(id)) - if len(id) > 0 { - data := make([]byte, len(id)) - copy(data, id) - item.data = unsafe.Pointer(&data[0]) - } - return item -} - -func (item *itemT) weightAndPoints() (weight, points int) { - if objIsSpatial(item.obj) { - points = item.obj.NumPoints() - weight = points * 16 - } else { - weight = len(item.obj.String()) - } - weight += int(item.fieldsLen + item.idLen) - return weight, points -} - -func (item *itemT) Less(other btree.Item, ctx interface{}) bool { - value1 := item.obj.String() - value2 := other.(*itemT).obj.String() - if value1 < value2 { - return true - } - if value1 > value2 { - return false - } - // the values match so we'll compare IDs, which are always unique. - return item.id() < other.(*itemT).id() -} - -func floatsToBytes(f []float64) []byte { - return *(*[]byte)((unsafe.Pointer)(&reflect.SliceHeader{ - Data: ((*reflect.SliceHeader)(unsafe.Pointer(&f))).Data, - Len: len(f) * 8, - Cap: len(f) * 8, - })) -} - -// directCopyFields copies fields, overwriting previous fields -func (item *itemT) directCopyFields(fields []float64) { - fieldBytes := floatsToBytes(fields) - oldData := item.dataBytes() - newData := make([]byte, len(fieldBytes)+int(item.idLen)) - copy(newData, fieldBytes) - copy(newData[len(fieldBytes):], oldData[item.fieldsLen:]) - item.fieldsLen = uint32(len(fieldBytes)) - if len(newData) > 0 { - item.data = unsafe.Pointer(&newData[0]) - } else { - item.data = nil - } -} - -func (c *Collection) setField( - item *itemT, fieldName string, fieldValue float64, updateWeight bool, -) (updated bool) { - idx, ok := c.fieldMap[fieldName] - if !ok { - idx = len(c.fieldMap) - c.fieldMap[fieldName] = idx - } - itemFields := item.fields() - if idx >= len(itemFields) { - // make room for new field - - itemBytes := item.dataBytes() - oldLen := len(itemFields) - data := make([]byte, (idx+1)*8+int(item.idLen)) - - copy(data, itemBytes[:item.fieldsLen]) - copy(data[(idx+1)*8:], itemBytes[item.fieldsLen:]) - item.fieldsLen = uint32((idx + 1) * 8) - item.data = unsafe.Pointer(&data[0]) - - itemFields := item.fields() - if updateWeight { - c.weight += (len(itemFields) - oldLen) * 8 - } - itemFields[idx] = fieldValue - updated = true - } else if itemFields[idx] != fieldValue { - // existing field needs updating - itemFields[idx] = fieldValue - updated = true - } - return updated -} -func (c *Collection) setFields( - item *itemT, fieldNames []string, fieldValues []float64, updateWeight bool, -) (updatedCount int) { - // TODO: optimize to predict the item data growth. - // TODO: do all sets here, instead of calling setFields in a loop - for i, fieldName := range fieldNames { - var fieldValue float64 - if i < len(fieldValues) { - fieldValue = fieldValues[i] - } - if c.setField(item, fieldName, fieldValue, updateWeight) { - updatedCount++ - } - } - return updatedCount -} diff --git a/internal/collection/item/item.go b/internal/collection/item/item.go new file mode 100644 index 00000000..ed6b7bab --- /dev/null +++ b/internal/collection/item/item.go @@ -0,0 +1,153 @@ +package item + +import ( + "reflect" + "unsafe" + + "github.com/tidwall/btree" + "github.com/tidwall/geojson" +) + +// Item is a item for Tile38 collections +type Item struct { + obj geojson.Object // geojson or string + fieldsLen uint32 // fields block size in bytes, not num of fields + idLen uint32 // id block size in bytes + data unsafe.Pointer // pointer to raw block of bytes +} + +// ID returns the items ID as a string +func (item *Item) ID() string { + return *(*string)((unsafe.Pointer)(&reflect.StringHeader{ + Data: uintptr(unsafe.Pointer(item.data)) + uintptr(item.fieldsLen), + Len: int(item.idLen), + })) +} + +// Fields returns the field values +func (item *Item) Fields() []float64 { + return *(*[]float64)((unsafe.Pointer)(&reflect.SliceHeader{ + Data: uintptr(unsafe.Pointer(item.data)), + Len: int(item.fieldsLen) / 8, + Cap: int(item.fieldsLen) / 8, + })) +} + +// Obj returns the geojson object +func (item *Item) Obj() geojson.Object { + return item.obj +} + +// New returns a newly allocated Item +func New(id string, obj geojson.Object) *Item { + item := new(Item) + item.obj = obj + item.idLen = uint32(len(id)) + if len(id) > 0 { + data := make([]byte, len(id)) + copy(data, id) + item.data = unsafe.Pointer(&data[0]) + } + return item +} + +// WeightAndPoints returns the memory weight and number of points for Item. +func (item *Item) WeightAndPoints() (weight, points int) { + _, objIsSpatial := item.obj.(geojson.Spatial) + if objIsSpatial { + points = item.obj.NumPoints() + weight = points * 16 + } else if item.obj != nil { + weight = len(item.obj.String()) + } + weight += int(item.fieldsLen + item.idLen) + return weight, points +} + +// Less is a btree interface that compares if item is less than other item. +func (item *Item) Less(other btree.Item, ctx interface{}) bool { + value1 := item.obj.String() + value2 := other.(*Item).obj.String() + if value1 < value2 { + return true + } + if value1 > value2 { + return false + } + // the values match so we'll compare IDs, which are always unique. + return item.ID() < other.(*Item).ID() +} + +// CopyOverFields overwriting previous fields +func (item *Item) CopyOverFields(values []float64) { + fieldBytes := floatsToBytes(values) + oldData := item.dataBytes() + newData := make([]byte, len(fieldBytes)+int(item.idLen)) + copy(newData, fieldBytes) + copy(newData[len(fieldBytes):], oldData[item.fieldsLen:]) + item.fieldsLen = uint32(len(fieldBytes)) + if len(newData) > 0 { + item.data = unsafe.Pointer(&newData[0]) + } else { + item.data = nil + } +} + +func getFieldAt(data unsafe.Pointer, index int) float64 { + return *(*float64)(unsafe.Pointer(uintptr(data) + uintptr(index*8))) +} + +func setFieldAt(data unsafe.Pointer, index int, value float64) { + *(*float64)(unsafe.Pointer(uintptr(data) + uintptr(index*8))) = value +} + +// SetField set a field value at specified index. +func (item *Item) SetField(index int, value float64) (updated bool) { + numFields := int(item.fieldsLen / 8) + if index < numFields { + // field exists + if getFieldAt(item.data, index) == value { + return false + } + } else { + // make room for new field + oldBytes := item.dataBytes() + newData := make([]byte, (index+1)*8+int(item.idLen)) + // copy the existing fields + copy(newData, oldBytes[:item.fieldsLen]) + // copy the id + copy(newData[(index+1)*8:], oldBytes[item.fieldsLen:]) + // update the fields length + item.fieldsLen = uint32((index + 1) * 8) + // update the raw data + item.data = unsafe.Pointer(&newData[0]) + } + // set the new field + setFieldAt(item.data, index, value) + return true +} + +// GetField returns the value for a field at index. +func (item *Item) GetField(index int) float64 { + numFields := int(item.fieldsLen / 8) + if index < numFields { + return getFieldAt(item.data, index) + } + return 0 +} + +func (item *Item) dataBytes() []byte { + return *(*[]byte)((unsafe.Pointer)(&reflect.SliceHeader{ + Data: uintptr(unsafe.Pointer(item.data)), + Len: int(item.fieldsLen) + int(item.idLen), + Cap: int(item.fieldsLen) + int(item.idLen), + })) +} + +func floatsToBytes(f []float64) []byte { + return *(*[]byte)((unsafe.Pointer)(&reflect.SliceHeader{ + Data: ((*reflect.SliceHeader)(unsafe.Pointer(&f))).Data, + Len: len(f) * 8, + Cap: len(f) * 8, + })) +} diff --git a/internal/collection/item/item_test.go b/internal/collection/item/item_test.go new file mode 100644 index 00000000..6160dde6 --- /dev/null +++ b/internal/collection/item/item_test.go @@ -0,0 +1,189 @@ +package item + +import ( + "encoding/json" + "math/rand" + "reflect" + "testing" + "time" + + "github.com/tidwall/geojson" + "github.com/tidwall/geojson/geometry" +) + +func testRandItem(t *testing.T) { + keyb := make([]byte, rand.Int()%16) + rand.Read(keyb) + key := string(keyb) + values := make([]float64, rand.Int()%1024) + for i := range values { + values[i] = rand.Float64() + } + item := New(key, geojson.NewPoint(geometry.Point{X: 1, Y: 2})) + if item.ID() != key { + t.Fatalf("expected '%v', got '%v'", key, item.ID()) + } + + var setValues []int + for _, i := range rand.Perm(len(values)) { + if !item.SetField(i, values[i]) { + t.Fatal("expected true") + } + setValues = append(setValues, i) + if item.ID() != key { + t.Fatalf("expected '%v', got '%v'", key, item.ID()) + } + for _, i := range setValues { + if item.GetField(i) != values[i] { + t.Fatalf("expected '%v', got '%v'", values[i], item.GetField(i)) + } + } + fields := item.Fields() + for i := 0; i < len(fields); i++ { + for _, j := range setValues { + if i == j { + if fields[i] != values[i] { + t.Fatalf("expected '%v', got '%v'", values[i], fields[i]) + } + break + } + } + } + weight, points := item.WeightAndPoints() + if weight != len(fields)*8+len(key)+points*16 { + t.Fatalf("expected '%v', got '%v'", len(fields)*8+len(key)+points*16, weight) + } + if points != 1 { + t.Fatalf("expected '%v', got '%v'", 1, points) + } + } + if item.GetField(len(values)) != 0 { + t.Fatalf("expected '%v', got '%v'", 0, item.GetField(len(values))) + } + for _, i := range rand.Perm(len(values)) { + if item.SetField(i, values[i]) { + t.Fatal("expected false") + } + } + if item.ID() != key { + t.Fatalf("expected '%v', got '%v'", key, item.ID()) + } + if item.Obj().NumPoints() != 1 { + t.Fatalf("expected '%v', got '%v'", 1, item.Obj().NumPoints()) + } + item.CopyOverFields(values) + weight, points := item.WeightAndPoints() + if weight != len(values)*8+len(key)+points*16 { + t.Fatalf("expected '%v', got '%v'", len(values)*8+len(key)+points*16, weight) + } + if points != 1 { + t.Fatalf("expected '%v', got '%v'", 1, points) + } + if !reflect.DeepEqual(item.Fields(), values) { + t.Fatalf("expected '%v', got '%v'", values, item.Fields()) + } + + item.CopyOverFields(nil) + weight, points = item.WeightAndPoints() + if weight != len(key)+points*16 { + t.Fatalf("expected '%v', got '%v'", len(key)+points*16, weight) + } + if points != 1 { + t.Fatalf("expected '%v', got '%v'", 1, points) + } + if len(item.Fields()) != 0 { + t.Fatalf("expected '%#v', got '%#v'", 0, len(item.Fields())) + } + if item.ID() != key { + t.Fatalf("expected '%v', got '%v'", key, item.ID()) + } +} + +func TestItem(t *testing.T) { + rand.Seed(time.Now().UnixNano()) + start := time.Now() + for time.Since(start) < time.Second { + testRandItem(t) + } +} + +func TestItemLess(t *testing.T) { + item0 := New("0", testString("0")) + item1 := New("1", testString("1")) + item2 := New("1", testString("2")) + item3 := New("3", testString("2")) + if !item0.Less(item1, nil) { + t.Fatal("expected true") + } + if item1.Less(item0, nil) { + t.Fatal("expected false") + } + if !item1.Less(item2, nil) { + t.Fatal("expected true") + } + if item2.Less(item1, nil) { + t.Fatal("expected false") + } + if !item2.Less(item3, nil) { + t.Fatal("expected true") + } + if item3.Less(item2, nil) { + t.Fatal("expected false") + } + weight, points := item0.WeightAndPoints() + if weight != 2 { + t.Fatalf("expected '%v', got '%v'", 2, weight) + } + if points != 0 { + t.Fatalf("expected '%v', got '%v'", 0, points) + } +} + +type testString string + +func (s testString) Spatial() geojson.Spatial { + return geojson.EmptySpatial{} +} +func (s testString) ForEach(iter func(geom geojson.Object) bool) bool { + return iter(s) +} +func (s testString) Empty() bool { + return true +} +func (s testString) Valid() bool { + return false +} +func (s testString) Rect() geometry.Rect { + return geometry.Rect{} +} +func (s testString) Center() geometry.Point { + return geometry.Point{} +} +func (s testString) AppendJSON(dst []byte) []byte { + data, _ := json.Marshal(string(s)) + return append(dst, data...) +} +func (s testString) String() string { + return string(s) +} +func (s testString) JSON() string { + return string(s.AppendJSON(nil)) +} +func (s testString) MarshalJSON() ([]byte, error) { + return s.AppendJSON(nil), nil +} +func (s testString) Within(obj geojson.Object) bool { + return false +} +func (s testString) Contains(obj geojson.Object) bool { + return false +} +func (s testString) Intersects(obj geojson.Object) bool { + return false +} +func (s testString) NumPoints() int { + return 0 +} +func (s testString) Distance(obj geojson.Object) float64 { + return 0 +} diff --git a/internal/collection/ptrrtree/rtree.go b/internal/collection/rtree/rtree.go similarity index 91% rename from internal/collection/ptrrtree/rtree.go rename to internal/collection/rtree/rtree.go index cd86e05f..e153b55c 100644 --- a/internal/collection/ptrrtree/rtree.go +++ b/internal/collection/rtree/rtree.go @@ -1,6 +1,10 @@ -package ptrrtree +package rtree -import "unsafe" +import ( + "unsafe" + + "github.com/tidwall/tile38/internal/collection/item" +) const dims = 2 @@ -90,10 +94,10 @@ func (r *box) enlargedArea(b *box) float64 { } // Insert inserts an item into the RTree -func (tr *BoxTree) Insert(min, max []float64, value unsafe.Pointer) { - var item box - fit(min, max, value, &item) - tr.insert(&item) +func (tr *BoxTree) Insert(min, max []float64, item *item.Item) { + var box box + fit(min, max, unsafe.Pointer(item), &box) + tr.insert(&box) } func (tr *BoxTree) insert(item *box) { @@ -317,14 +321,14 @@ func (r *box) intersects(b *box) bool { func (r *box) search( target *box, height int, - iter func(min, max []float64, value unsafe.Pointer) bool, + iter func(min, max []float64, item *item.Item) bool, ) bool { n := (*node)(r.data) if height == 0 { for i := 0; i < n.count; i++ { if target.intersects(&n.boxes[i]) { if !iter(n.boxes[i].min[:], n.boxes[i].max[:], - n.boxes[i].data) { + (*item.Item)(n.boxes[i].data)) { return false } } @@ -348,7 +352,7 @@ func (r *box) search( func (tr *BoxTree) search( target *box, - iter func(min, max []float64, value unsafe.Pointer) bool, + iter func(min, max []float64, item *item.Item) bool, ) { if tr.root.data == nil { return @@ -363,7 +367,7 @@ func (tr *BoxTree) search( // Search ... func (tr *BoxTree) Search(min, max []float64, - iter func(min, max []float64, value unsafe.Pointer) bool, + iter func(min, max []float64, item *item.Item) bool, ) { var target box fit(min, max, nil, &target) @@ -381,7 +385,7 @@ const ( // Traverse iterates through all items and container boxes in tree. func (tr *BoxTree) Traverse( - iter func(min, max []float64, height, level int, value unsafe.Pointer) int, + iter func(min, max []float64, height, level int, item *item.Item) int, ) { if tr.root.data == nil { return @@ -393,13 +397,13 @@ func (tr *BoxTree) Traverse( func (r *box) traverse( height, level int, - iter func(min, max []float64, height, level int, value unsafe.Pointer) int, + iter func(min, max []float64, height, level int, value *item.Item) int, ) int { n := (*node)(r.data) if height == 0 { for i := 0; i < n.count; i++ { action := iter(n.boxes[i].min[:], n.boxes[i].max[:], height, level, - n.boxes[i].data) + (*item.Item)(n.boxes[i].data)) if action == Stop { return Stop } @@ -407,7 +411,7 @@ func (r *box) traverse( } else { for i := 0; i < n.count; i++ { switch iter(n.boxes[i].min[:], n.boxes[i].max[:], height, level, - n.boxes[i].data) { + (*item.Item)(n.boxes[i].data)) { case Ignore: case Continue: if n.boxes[i].traverse(height-1, level+1, iter) == Stop { @@ -422,12 +426,13 @@ func (r *box) traverse( } func (r *box) scan( - height int, iter func(min, max []float64, value unsafe.Pointer) bool, + height int, iter func(min, max []float64, value *item.Item) bool, ) bool { n := (*node)(r.data) if height == 0 { for i := 0; i < n.count; i++ { - if !iter(n.boxes[i].min[:], n.boxes[i].max[:], n.boxes[i].data) { + if !iter(n.boxes[i].min[:], n.boxes[i].max[:], + (*item.Item)(n.boxes[i].data)) { return false } } @@ -442,7 +447,7 @@ func (r *box) scan( } // Scan iterates through all items in tree. -func (tr *BoxTree) Scan(iter func(min, max []float64, value unsafe.Pointer) bool) { +func (tr *BoxTree) Scan(iter func(min, max []float64, item *item.Item) bool) { if tr.root.data == nil { return } @@ -450,15 +455,15 @@ func (tr *BoxTree) Scan(iter func(min, max []float64, value unsafe.Pointer) bool } // Delete ... -func (tr *BoxTree) Delete(min, max []float64, value unsafe.Pointer) { - var item box - fit(min, max, value, &item) - if tr.root.data == nil || !tr.root.contains(&item) { +func (tr *BoxTree) Delete(min, max []float64, item *item.Item) { + var target box + fit(min, max, unsafe.Pointer(item), &target) + if tr.root.data == nil || !tr.root.contains(&target) { return } var removed, recalced bool removed, recalced, tr.reinsert = - tr.root.delete(&item, tr.height, tr.reinsert[:0]) + tr.root.delete(&target, tr.height, tr.reinsert[:0]) if !removed { return } @@ -647,7 +652,7 @@ func (q *queue) pop() qnode { // Nearby returns items nearest to farthest. // The dist param is the "box distance". func (tr *BoxTree) Nearby(min, max []float64, - iter func(min, max []float64, item unsafe.Pointer) bool) { + iter func(min, max []float64, item *item.Item) bool) { if tr.root.data == nil { return } @@ -666,8 +671,8 @@ func (tr *BoxTree) Nearby(min, max []float64, if q.peek().height > -1 { break } - item := q.pop() - if !iter(item.box.min[:], item.box.max[:], item.box.data) { + v := q.pop() + if !iter(v.box.min[:], v.box.max[:], (*item.Item)(v.box.data)) { return } } diff --git a/internal/collection/ptrrtree/rtree_test.go b/internal/collection/rtree/rtree_test.go similarity index 60% rename from internal/collection/ptrrtree/rtree_test.go rename to internal/collection/rtree/rtree_test.go index c7db757e..938a4260 100644 --- a/internal/collection/ptrrtree/rtree_test.go +++ b/internal/collection/rtree/rtree_test.go @@ -1,4 +1,4 @@ -package ptrrtree +package rtree import ( "fmt" @@ -8,7 +8,10 @@ import ( "strings" "testing" "time" - "unsafe" + + "github.com/tidwall/geojson" + "github.com/tidwall/geojson/geometry" + "github.com/tidwall/tile38/internal/collection/item" ) type tBox struct { @@ -16,8 +19,8 @@ type tBox struct { max [dims]float64 } -var boxes []*tBox -var points []*tBox +var boxes []*item.Item +var points []*item.Item func init() { seed := time.Now().UnixNano() @@ -26,54 +29,65 @@ func init() { rand.Seed(seed) } -func randPoints(N int) []*tBox { - boxes := make([]*tBox, N) +func boxMin(box *item.Item) []float64 { + return box.Obj().(*tBox).min[:] +} +func boxMax(box *item.Item) []float64 { + return box.Obj().(*tBox).max[:] +} + +func randPoints(N int) []*item.Item { + boxes := make([]*item.Item, N) for i := 0; i < N; i++ { - boxes[i] = new(tBox) - boxes[i].min[0] = rand.Float64()*360 - 180 - boxes[i].min[1] = rand.Float64()*180 - 90 + box := new(tBox) + box.min[0] = rand.Float64()*360 - 180 + box.min[1] = rand.Float64()*180 - 90 for j := 2; j < dims; j++ { - boxes[i].min[j] = rand.Float64() + box.min[j] = rand.Float64() } - boxes[i].max = boxes[i].min + box.max = box.min + boxes[i] = item.New(fmt.Sprintf("%d", i), box) } return boxes } -func randBoxes(N int) []*tBox { - boxes := make([]*tBox, N) +func randBoxes(N int) []*item.Item { + boxes := make([]*item.Item, N) for i := 0; i < N; i++ { - boxes[i] = new(tBox) - boxes[i].min[0] = rand.Float64()*360 - 180 - boxes[i].min[1] = rand.Float64()*180 - 90 + box := new(tBox) + box.min[0] = rand.Float64()*360 - 180 + box.min[1] = rand.Float64()*180 - 90 for j := 2; j < dims; j++ { - boxes[i].min[j] = rand.Float64() * 100 + box.min[j] = rand.Float64() * 100 } - boxes[i].max[0] = boxes[i].min[0] + rand.Float64() - boxes[i].max[1] = boxes[i].min[1] + rand.Float64() + box.max[0] = box.min[0] + rand.Float64() + box.max[1] = box.min[1] + rand.Float64() for j := 2; j < dims; j++ { - boxes[i].max[j] = boxes[i].min[j] + rand.Float64() + box.max[j] = box.min[j] + rand.Float64() } - if boxes[i].max[0] > 180 || boxes[i].max[1] > 90 { + if box.max[0] > 180 || box.max[1] > 90 { i-- } + boxes[i] = item.New(fmt.Sprintf("%d", i), box) } return boxes } -func sortBoxes(boxes []*tBox) { +func sortBoxes(boxes []*item.Item) { sort.Slice(boxes, func(i, j int) bool { - for k := 0; k < len(boxes[i].min); k++ { - if boxes[i].min[k] < boxes[j].min[k] { + iMin, iMax := boxMin(boxes[i]), boxMax(boxes[i]) + jMin, jMax := boxMin(boxes[j]), boxMax(boxes[j]) + for k := 0; k < len(iMin); k++ { + if iMin[k] < jMin[k] { return true } - if boxes[i].min[k] > boxes[j].min[k] { + if iMin[k] > jMin[k] { return false } - if boxes[i].max[k] < boxes[j].max[k] { + if iMax[k] < jMax[k] { return true } - if boxes[i].max[k] > boxes[j].max[k] { + if iMax[k] > jMax[k] { return false } } @@ -81,10 +95,10 @@ func sortBoxes(boxes []*tBox) { }) } -func sortBoxesNearby(boxes []tBox, min, max []float64) { +func sortBoxesNearby(boxes []*item.Item, min, max []float64) { sort.Slice(boxes, func(i, j int) bool { - return testBoxDist(boxes[i].min[:], boxes[i].max[:], min, max) < - testBoxDist(boxes[j].min[:], boxes[j].max[:], min, max) + return testBoxDist(boxMin(boxes[i]), boxMax(boxes[i]), min, max) < + testBoxDist(boxMin(boxes[j]), boxMax(boxes[j]), min, max) }) } @@ -110,7 +124,7 @@ func testBoxDist(amin, amax, bmin, bmax []float64) float64 { return dist } -func testBoxesVarious(t *testing.T, boxes []*tBox, label string) { +func testBoxesVarious(t *testing.T, items []*item.Item, label string) { N := len(boxes) var tr BoxTree @@ -122,7 +136,7 @@ func testBoxesVarious(t *testing.T, boxes []*tBox, label string) { // insert ///////////////////////////////////////// for i := 0; i < N; i++ { - tr.Insert(boxes[i].min[:], boxes[i].max[:], unsafe.Pointer(boxes[i])) + tr.Insert(boxMin(boxes[i]), boxMax(boxes[i]), boxes[i]) } if tr.Count() != N { t.Fatalf("expected %d, got %d", N, tr.Count()) @@ -136,7 +150,7 @@ func testBoxesVarious(t *testing.T, boxes []*tBox, label string) { // scan all items and count one-by-one ///////////////////////////////////////// var count int - tr.Scan(func(min, max []float64, value unsafe.Pointer) bool { + tr.Scan(func(min, max []float64, _ *item.Item) bool { count++ return true }) @@ -147,12 +161,12 @@ func testBoxesVarious(t *testing.T, boxes []*tBox, label string) { ///////////////////////////////////////// // check every point for correctness ///////////////////////////////////////// - var tboxes1 []*tBox - tr.Scan(func(min, max []float64, value unsafe.Pointer) bool { - tboxes1 = append(tboxes1, (*tBox)(value)) + var tboxes1 []*item.Item + tr.Scan(func(min, max []float64, item *item.Item) bool { + tboxes1 = append(tboxes1, item) return true }) - tboxes2 := make([]*tBox, len(boxes)) + tboxes2 := make([]*item.Item, len(boxes)) copy(tboxes2, boxes) sortBoxes(tboxes1) sortBoxes(tboxes2) @@ -167,9 +181,9 @@ func testBoxesVarious(t *testing.T, boxes []*tBox, label string) { ///////////////////////////////////////// for i := 0; i < N; i++ { var found bool - tr.Search(boxes[i].min[:], boxes[i].max[:], - func(min, max []float64, value unsafe.Pointer) bool { - if value == unsafe.Pointer(boxes[i]) { + tr.Search(boxMin(boxes[i]), boxMax(boxes[i]), + func(min, max []float64, v *item.Item) bool { + if v == boxes[i] { found = true return false } @@ -192,7 +206,7 @@ func testBoxesVarious(t *testing.T, boxes []*tBox, label string) { for i := 0; i < N/5; i++ { var count int tr.Search(centerMin, centerMax, - func(min, max []float64, value unsafe.Pointer) bool { + func(min, max []float64, _ *item.Item) bool { count++ return true }, @@ -204,14 +218,14 @@ func testBoxesVarious(t *testing.T, boxes []*tBox, label string) { ///////////////////////////////////////// for i := 0; i < N/2; i++ { j := i * 2 - tr.Delete(boxes[j].min[:], boxes[j].max[:], unsafe.Pointer(boxes[j])) + tr.Delete(boxMin(boxes[j]), boxMax(boxes[j]), boxes[j]) } ///////////////////////////////////////// // count all items. should be half of N ///////////////////////////////////////// count = 0 - tr.Scan(func(min, max []float64, value unsafe.Pointer) bool { + tr.Scan(func(min, max []float64, _ *item.Item) bool { count++ return true }) @@ -232,28 +246,29 @@ func testBoxesVarious(t *testing.T, boxes []*tBox, label string) { }) for i := 0; i < N/2; i++ { j := ij[i] - tr.Insert(boxes[j].min[:], boxes[j].max[:], unsafe.Pointer(boxes[j])) + tr.Insert(boxMin(boxes[j]), boxMax(boxes[j]), boxes[j]) } ////////////////////////////////////////////////////// // replace each item with an item that is very close ////////////////////////////////////////////////////// - var nboxes = make([]*tBox, N) + var nboxes = make([]*item.Item, N) for i := 0; i < N; i++ { - nboxes[i] = new(tBox) - for j := 0; j < len(boxes[i].min); j++ { - nboxes[i].min[j] = boxes[i].min[j] + (rand.Float64() - 0.5) - if boxes[i].min == boxes[i].max { - nboxes[i].max[j] = nboxes[i].min[j] + box := boxes[i].Obj().(*tBox) + nbox := new(tBox) + for j := 0; j < len(box.min); j++ { + nbox.min[j] = box.min[j] + (rand.Float64() - 0.5) + if box.min == box.max { + nbox.max[j] = nbox.min[j] } else { - nboxes[i].max[j] = boxes[i].max[j] + (rand.Float64() - 0.5) + nbox.max[j] = box.max[j] + (rand.Float64() - 0.5) } } - + nboxes[i] = item.New(fmt.Sprintf("%d", i), nbox) } for i := 0; i < N; i++ { - tr.Insert(nboxes[i].min[:], nboxes[i].max[:], unsafe.Pointer(nboxes[i])) - tr.Delete(boxes[i].min[:], boxes[i].max[:], unsafe.Pointer(boxes[i])) + tr.Insert(boxMin(nboxes[i]), boxMax(nboxes[i]), nboxes[i]) + tr.Delete(boxMin(boxes[i]), boxMax(boxes[i]), boxes[i]) } if tr.Count() != N { t.Fatalf("expected %d, got %d", N, tr.Count()) @@ -265,11 +280,11 @@ func testBoxesVarious(t *testing.T, boxes []*tBox, label string) { // check every point for correctness ///////////////////////////////////////// tboxes1 = nil - tr.Scan(func(min, max []float64, value unsafe.Pointer) bool { - tboxes1 = append(tboxes1, (*tBox)(value)) + tr.Scan(func(min, max []float64, value *item.Item) bool { + tboxes1 = append(tboxes1, value) return true }) - tboxes2 = make([]*tBox, len(nboxes)) + tboxes2 = make([]*item.Item, len(nboxes)) copy(tboxes2, nboxes) sortBoxes(tboxes1) sortBoxes(tboxes2) @@ -285,17 +300,17 @@ func testBoxesVarious(t *testing.T, boxes []*tBox, label string) { for i := 0; i < N/5; i++ { var count int tr.Search(centerMin, centerMax, - func(min, max []float64, value unsafe.Pointer) bool { + func(min, max []float64, value *item.Item) bool { count++ return true }, ) } - var boxes3 []*tBox + var boxes3 []*item.Item tr.Nearby(centerMin, centerMax, - func(min, max []float64, value unsafe.Pointer) bool { - boxes3 = append(boxes3, (*tBox)(value)) + func(min, max []float64, value *item.Item) bool { + boxes3 = append(boxes3, value) return true }, ) @@ -307,7 +322,7 @@ func testBoxesVarious(t *testing.T, boxes []*tBox, label string) { } var ldist float64 for i, box := range boxes3 { - dist := testBoxDist(box.min[:], box.max[:], centerMin, centerMax) + dist := testBoxDist(boxMin(box), boxMax(box), centerMin, centerMax) if i > 0 && dist < ldist { t.Fatalf("out of order") } @@ -378,6 +393,52 @@ func BenchmarkRandomInsert(b *testing.B) { boxes := randBoxes(b.N) b.ResetTimer() for i := 0; i < b.N; i++ { - tr.Insert(boxes[i].min[:], boxes[i].max[:], nil) + tr.Insert(boxMin(boxes[i]), boxMax(boxes[i]), nil) } } + +func (s *tBox) Spatial() geojson.Spatial { + return geojson.EmptySpatial{} +} +func (s *tBox) ForEach(iter func(geom geojson.Object) bool) bool { + return iter(s) +} +func (s *tBox) Empty() bool { + return true +} +func (s *tBox) Valid() bool { + return false +} +func (s *tBox) Rect() geometry.Rect { + return geometry.Rect{} +} +func (s *tBox) Center() geometry.Point { + return geometry.Point{} +} +func (s *tBox) AppendJSON(dst []byte) []byte { + return nil +} +func (s *tBox) String() string { + return "" +} +func (s *tBox) JSON() string { + return string(s.AppendJSON(nil)) +} +func (s *tBox) MarshalJSON() ([]byte, error) { + return s.AppendJSON(nil), nil +} +func (s *tBox) Within(obj geojson.Object) bool { + return false +} +func (s *tBox) Contains(obj geojson.Object) bool { + return false +} +func (s *tBox) Intersects(obj geojson.Object) bool { + return false +} +func (s *tBox) NumPoints() int { + return 0 +} +func (s *tBox) Distance(obj geojson.Object) float64 { + return 0 +}