mirror
/
tile38
mirror of https://github.com/tidwall/tile38.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Replace tinybtree

This commit is contained in:
tidwall 2021-07-08 06:35:15 -07:00
parent 20ee5e3396
commit f44bae43ca
10 changed files with 58 additions and 508 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
go.mod
View File

@ -33,6 +33,7 @@ require (
github.com/tidwall/rhh v1.1.0
github.com/tidwall/sjson v1.1.1
github.com/tidwall/tinybtree v1.0.1
github.com/tidwall/tinyqueue v0.0.0-20180302190814-1e39f5511563 // indirect
github.com/yuin/gopher-lua v0.0.0-20200816102855-ee81675732da
golang.org/x/crypto v0.0.0-20200204104054-c9f3fb736b72
golang.org/x/net v0.0.0-20200301022130-244492dfa37a

3
go.sum
View File

@ -203,8 +203,11 @@ github.com/tidwall/rtree v0.0.0-20201027154624-32188eeb08a8 h1:BsKSRhu0TDB6Snq8S
github.com/tidwall/rtree v0.0.0-20201027154624-32188eeb08a8/go.mod h1:/h+UnNGt0IhNNJLkGikcdcJqm66zGD/uJGMRxK/9+Ao=
github.com/tidwall/sjson v1.1.1 h1:7h1vk049Jnd5EH9NyzNiEuwYW4b5qgreBbqRC19AS3U=
github.com/tidwall/sjson v1.1.1/go.mod h1:yvVuSnpEQv5cYIrO+AT6kw4QVfd5SDZoGIS7/5+fZFs=
<<<<<<< HEAD
github.com/tidwall/tinybtree v1.0.1 h1:g1kLLw/dCJgtH14AFqUoob0MtSfThw4xQILCGMQd8J8=
github.com/tidwall/tinybtree v1.0.1/go.mod h1:0aFQG6KLQz3j57CeVgXlmKO3RSQ3myhJn2H+r84IgSY=
=======
>>>>>>> 016f3971 (Replace tinybtree)
github.com/tidwall/tinyqueue v0.0.0-20180302190814-1e39f5511563 h1:Otn9S136ELckZ3KKDyCkxapfufrqDqwmGjcHfAyXRrE=
github.com/tidwall/tinyqueue v0.0.0-20180302190814-1e39f5511563/go.mod h1:mLqSmt7Dv/CNneF2wfcChfN1rvapyQr01LGKnKex0DQ=
github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=

4
internal/server/crud.go
View File

@ -7,6 +7,7 @@ import (
"time"
"github.com/mmcloughlin/geohash"
"github.com/tidwall/btree"
"github.com/tidwall/geojson"
"github.com/tidwall/geojson/geometry"
"github.com/tidwall/rbang"
@ -14,7 +15,6 @@ import (
"github.com/tidwall/rhh"
"github.com/tidwall/tile38/internal/collection"
"github.com/tidwall/tile38/internal/glob"
"github.com/tidwall/tinybtree"
)
type fvt struct {
@ -509,7 +509,7 @@ func (server *Server) cmdFlushDB(msg *Message) (res resp.Value, d commandDetails
err = errInvalidNumberOfArguments
return
}
server.cols = tinybtree.BTree{}
server.cols = btree.New(byCollectionKey)
server.expires = rhh.New(0)
server.hooks = make(map[string]*Hook)
server.hooksOut = make(map[string]*Hook)

19
internal/server/keys.go
View File

@ -36,17 +36,18 @@ func (s *Server) cmdKeys(msg *Message) (res resp.Value, err error) {
var greaterPivot string
var vals []resp.Value
iterator := func(key string, value interface{}) bool {
iterator := func(v interface{}) bool {
vcol := v.(*collectionKeyContainer)
var match bool
if everything {
match = true
} else if greater {
if !strings.HasPrefix(key, greaterPivot) {
if !strings.HasPrefix(vcol.key, greaterPivot) {
return false
}
match = true
} else {
match, _ = glob.Match(pattern, key)
match, _ = glob.Match(pattern, vcol.key)
}
if match {
if once {
@ -58,9 +59,9 @@ func (s *Server) cmdKeys(msg *Message) (res resp.Value, err error) {
}
switch msg.OutputType {
case JSON:
wr.WriteString(jsonString(key))
wr.WriteString(jsonString(vcol.key))
case RESP:
vals = append(vals, resp.StringValue(key))
vals = append(vals, resp.StringValue(vcol.key))
}
// If no more than one match is expected, stop searching
@ -74,17 +75,17 @@ func (s *Server) cmdKeys(msg *Message) (res resp.Value, err error) {
// TODO: This can be further optimized by using glob.Parse and limits
if pattern == "*" {
everything = true
s.cols.Scan(iterator)
s.cols.Ascend(nil, iterator)
} else if strings.HasSuffix(pattern, "*") {
greaterPivot = pattern[:len(pattern)-1]
if glob.IsGlob(greaterPivot) {
s.cols.Scan(iterator)
s.cols.Ascend(nil, iterator)
} else {
greater = true
s.cols.Ascend(greaterPivot, iterator)
s.cols.Ascend(&collectionKeyContainer{key: greaterPivot}, iterator)
}
} else {
s.cols.Scan(iterator)
s.cols.Ascend(nil, iterator)
}
if msg.OutputType == JSON {
wr.WriteString(`],"elapsed":"` + time.Now().Sub(start).String() + "\"}")

52
internal/server/server.go
View File

@ -22,6 +22,7 @@ import (
"sync/atomic"
"time"
"github.com/tidwall/btree"
"github.com/tidwall/buntdb"
"github.com/tidwall/geojson"
"github.com/tidwall/geojson/geometry"
@ -36,7 +37,6 @@ import (
"github.com/tidwall/tile38/internal/endpoint"
"github.com/tidwall/tile38/internal/expire"
"github.com/tidwall/tile38/internal/log"
"github.com/tidwall/tinybtree"
)
var errOOM = errors.New("OOM command not allowed when used memory > 'maxmemory'")
@ -98,14 +98,14 @@ type Server struct {
conns map[int]*Client
mu sync.RWMutex
aof *os.File // active aof file
aofdirty int32 // mark the aofbuf as having data
aofbuf []byte // prewrite buffer
aofsz int // active size of the aof file
qdb *buntdb.DB // hook queue log
qidx uint64 // hook queue log last idx
cols tinybtree.BTree // data collections
expires *rhh.Map // map[string]map[string]time.Time
aof *os.File // active aof file
aofdirty int32 // mark the aofbuf as having data
aofbuf []byte // prewrite buffer
aofsz int // active size of the aof file
qdb *buntdb.DB // hook queue log
qidx uint64 // hook queue log last idx
cols *btree.BTree // data collections
expires *rhh.Map // map[string]map[string]time.Time
follows map[*bytes.Buffer]bool
fcond *sync.Cond
@ -159,6 +159,7 @@ func Serve(host string, port int, dir string, http bool) error {
http: http,
pubsub: newPubsub(),
monconns: make(map[net.Conn]bool),
cols: btree.New(byCollectionKey),
}
server.hookex.Expired = func(item expire.Item) {
@ -636,13 +637,25 @@ func (server *Server) backgroundSyncAOF() {
}
}
// collectionKeyContainer is a wrapper object around a collection that includes
// the collection and the key. It's needed for support with the btree package,
// which requires a comparator less function.
type collectionKeyContainer struct {
key string
col *collection.Collection
}
func byCollectionKey(a, b interface{}) bool {
return a.(*collectionKeyContainer).key < b.(*collectionKeyContainer).key
}
func (server *Server) setCol(key string, col *collection.Collection) {
server.cols.Set(key, col)
server.cols.Set(&collectionKeyContainer{key, col})
}
func (server *Server) getCol(key string) *collection.Collection {
if value, ok := server.cols.Get(key); ok {
return value.(*collection.Collection)
if v := server.cols.Get(&collectionKeyContainer{key: key}); v != nil {
return v.(*collectionKeyContainer).col
}
return nil
}
@ -650,14 +663,17 @@ func (server *Server) getCol(key string) *collection.Collection {
func (server *Server) scanGreaterOrEqual(
key string, iterator func(key string, col *collection.Collection) bool,
) {
server.cols.Ascend(key, func(ikey string, ivalue interface{}) bool {
return iterator(ikey, ivalue.(*collection.Collection))
})
server.cols.Ascend(&collectionKeyContainer{key: key},
func(v interface{}) bool {
vcol := v.(*collectionKeyContainer)
return iterator(vcol.key, vcol.col)
},
)
}
func (server *Server) deleteCol(key string) *collection.Collection {
if prev, ok := server.cols.Delete(key); ok {
return prev.(*collection.Collection)
if v := server.cols.Delete(&collectionKeyContainer{key: key}); v != nil {
return v.(*collectionKeyContainer).col
}
return nil
}
@ -956,7 +972,7 @@ func randomKey(n int) string {
func (server *Server) reset() {
server.aofsz = 0
server.cols = tinybtree.BTree{}
server.cols = btree.New(byCollectionKey)
server.expires = rhh.New(0)
}

17
internal/server/stats.go
View File

@ -14,7 +14,6 @@ import (
"github.com/tidwall/resp"
"github.com/tidwall/tile38/core"
"github.com/tidwall/tile38/internal/collection"
)
var memStats runtime.MemStats
@ -141,8 +140,8 @@ func (s *Server) basicStats(m map[string]interface{}) {
m["num_collections"] = s.cols.Len()
m["num_hooks"] = len(s.hooks)
sz := 0
s.cols.Scan(func(key string, value interface{}) bool {
col := value.(*collection.Collection)
s.cols.Ascend(nil, func(v interface{}) bool {
col := v.(*collectionKeyContainer).col
sz += col.TotalWeight()
return true
})
@ -150,8 +149,8 @@ func (s *Server) basicStats(m map[string]interface{}) {
points := 0
objects := 0
strings := 0
s.cols.Scan(func(key string, value interface{}) bool {
col := value.(*collection.Collection)
s.cols.Ascend(nil, func(v interface{}) bool {
col := v.(*collectionKeyContainer).col
points += col.PointCount()
objects += col.Count()
strings += col.StringCount()
@ -302,8 +301,8 @@ func (s *Server) extStats(m map[string]interface{}) {
points := 0
objects := 0
strings := 0
s.cols.Scan(func(key string, value interface{}) bool {
col := value.(*collection.Collection)
s.cols.Ascend(nil, func(v interface{}) bool {
col := v.(*collectionKeyContainer).col
points += col.PointCount()
objects += col.Count()
strings += col.StringCount()
@ -330,8 +329,8 @@ func (s *Server) extStats(m map[string]interface{}) {
m["tile38_avg_point_size"] = avgsz
sz := 0
s.cols.Scan(func(key string, value interface{}) bool {
col := value.(*collection.Collection)
s.cols.Ascend(nil, func(v interface{}) bool {
col := v.(*collectionKeyContainer).col
sz += col.TotalWeight()
return true
})

20
vendor/github.com/tidwall/tinybtree/LICENSE generated vendored
View File

@ -1,20 +0,0 @@
The MIT License (MIT)
Copyright (c) 2017 Joshua J Baker
Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
the Software, and to permit persons to whom the Software is furnished to do so,
subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

44
vendor/github.com/tidwall/tinybtree/README.md generated vendored
View File

@ -1,44 +0,0 @@
# `tinybtree`
[![GoDoc](https://godoc.org/github.com/tidwall/tinybtree?status.svg)](https://godoc.org/github.com/tidwall/tinybtree)
Just an itsy bitsy b-tree.
## Usage
Keys are strings, values are interfaces.
### Functions
```
Get(key string) (value interface{}, gotten bool)
Set(key string, value interface{}) (prev interface{}, replaced bool)
Delete(key string) (prev interface{}, deleted bool)
Scan(iter func(key string, value interface{}) bool)
Ascend(pivot string, iter func(key string, value interface{}) bool)
Descend(pivot string, iter func(key string, value interface{}) bool)
```
### Example
```go
// Create a btree
var tr tinybtree.BTree
// Set a key. Returns the previous value and ok a previous value exists.
prev, ok := tr.Set("hello", "world")
// Get a key. Returns the value and ok if the value exists.
value, ok := tr.Get("hello")
// Delete a key. Returns the deleted value and ok if the previous value exists.
prev, ok := tr.Delete("hello")
```
## Contact
Josh Baker [@tidwall](http://twitter.com/tidwall)
## License
`tinybtree` source code is available under the MIT [License](/LICENSE).

404
vendor/github.com/tidwall/tinybtree/btree.go generated vendored
View File

@ -1,404 +0,0 @@
package tinybtree
const maxItems = 255
const minItems = maxItems * 40 / 100
type item struct {
key string
value interface{}
}
type node struct {
numItems int
items [maxItems]item
children [maxItems + 1]*node
}
// BTree is an ordered set of key/value pairs where the key is a string
// and the value is an interface{}
type BTree struct {
height int
root *node
length int
}
func (n *node) find(key string) (index int, found bool) {
low := 0
high := n.numItems - 1
for low <= high {
mid := low + ((high+1)-low)/2
if key >= n.items[mid].key {
low = mid + 1
} else {
high = mid - 1
}
}
if low > 0 && n.items[low-1].key == key {
index = low - 1
found = true
} else {
index = low
found = false
}
return index, found
}
// Set or replace a value for a key
func (tr *BTree) Set(key string, value interface{}) (
prev interface{}, replaced bool,
) {
if tr.root == nil {
tr.root = new(node)
tr.root.items[0] = item{key, value}
tr.root.numItems = 1
tr.length = 1
return
}
prev, replaced = tr.root.set(key, value, tr.height)
if replaced {
return
}
if tr.root.numItems == maxItems {
n := tr.root
right, median := n.split(tr.height)
tr.root = new(node)
tr.root.children[0] = n
tr.root.items[0] = median
tr.root.children[1] = right
tr.root.numItems = 1
tr.height++
}
tr.length++
return
}
func (n *node) split(height int) (right *node, median item) {
right = new(node)
median = n.items[maxItems/2]
copy(right.items[:maxItems/2], n.items[maxItems/2+1:])
if height > 0 {
copy(right.children[:maxItems/2+1], n.children[maxItems/2+1:])
}
right.numItems = maxItems / 2
if height > 0 {
for i := maxItems/2 + 1; i < maxItems+1; i++ {
n.children[i] = nil
}
}
for i := maxItems / 2; i < maxItems; i++ {
n.items[i] = item{}
}
n.numItems = maxItems / 2
return
}
func (n *node) set(key string, value interface{}, height int) (
prev interface{}, replaced bool,
) {
i, found := n.find(key)
if found {
prev = n.items[i].value
n.items[i].value = value
return prev, true
}
if height == 0 {
for j := n.numItems; j > i; j-- {
n.items[j] = n.items[j-1]
}
n.items[i] = item{key, value}
n.numItems++
return nil, false
}
prev, replaced = n.children[i].set(key, value, height-1)
if replaced {
return
}
if n.children[i].numItems == maxItems {
right, median := n.children[i].split(height - 1)
copy(n.children[i+1:], n.children[i:])
copy(n.items[i+1:], n.items[i:])
n.items[i] = median
n.children[i+1] = right
n.numItems++
}
return
}
// Scan all items in tree
func (tr *BTree) Scan(iter func(key string, value interface{}) bool) {
if tr.root != nil {
tr.root.scan(iter, tr.height)
}
}
func (n *node) scan(
iter func(key string, value interface{}) bool, height int,
) bool {
if height == 0 {
for i := 0; i < n.numItems; i++ {
if !iter(n.items[i].key, n.items[i].value) {
return false
}
}
return true
}
for i := 0; i < n.numItems; i++ {
if !n.children[i].scan(iter, height-1) {
return false
}
if !iter(n.items[i].key, n.items[i].value) {
return false
}
}
return n.children[n.numItems].scan(iter, height-1)
}
// Get a value for key
func (tr *BTree) Get(key string) (value interface{}, gotten bool) {
if tr.root == nil {
return
}
return tr.root.get(key, tr.height)
}
func (n *node) get(key string, height int) (value interface{}, gotten bool) {
i, found := n.find(key)
if found {
return n.items[i].value, true
}
if height == 0 {
return nil, false
}
return n.children[i].get(key, height-1)
}
// Len returns the number of items in the tree
func (tr *BTree) Len() int {
return tr.length
}
// Delete a value for a key
func (tr *BTree) Delete(key string) (prev interface{}, deleted bool) {
if tr.root == nil {
return
}
var prevItem item
prevItem, deleted = tr.root.delete(false, key, tr.height)
if !deleted {
return
}
prev = prevItem.value
if tr.root.numItems == 0 {
tr.root = tr.root.children[0]
tr.height--
}
tr.length--
if tr.length == 0 {
tr.root = nil
tr.height = 0
}
return
}
func (n *node) delete(max bool, key string, height int) (
prev item, deleted bool,
) {
i, found := 0, false
if max {
i, found = n.numItems-1, true
} else {
i, found = n.find(key)
}
if height == 0 {
if found {
prev = n.items[i]
// found the items at the leaf, remove it and return.
copy(n.items[i:], n.items[i+1:n.numItems])
n.items[n.numItems-1] = item{}
n.children[n.numItems] = nil
n.numItems--
return prev, true
}
return item{}, false
}
if found {
if max {
i++
prev, deleted = n.children[i].delete(true, "", height-1)
} else {
prev = n.items[i]
maxItem, _ := n.children[i].delete(true, "", height-1)
n.items[i] = maxItem
deleted = true
}
} else {
prev, deleted = n.children[i].delete(max, key, height-1)
}
if !deleted {
return
}
if n.children[i].numItems < minItems {
if i == n.numItems {
i--
}
if n.children[i].numItems+n.children[i+1].numItems+1 < maxItems {
// merge left + item + right
n.children[i].items[n.children[i].numItems] = n.items[i]
copy(n.children[i].items[n.children[i].numItems+1:],
n.children[i+1].items[:n.children[i+1].numItems])
if height > 1 {
copy(n.children[i].children[n.children[i].numItems+1:],
n.children[i+1].children[:n.children[i+1].numItems+1])
}
n.children[i].numItems += n.children[i+1].numItems + 1
copy(n.items[i:], n.items[i+1:n.numItems])
copy(n.children[i+1:], n.children[i+2:n.numItems+1])
n.items[n.numItems] = item{}
n.children[n.numItems+1] = nil
n.numItems--
} else if n.children[i].numItems > n.children[i+1].numItems {
// move left -> right
copy(n.children[i+1].items[1:],
n.children[i+1].items[:n.children[i+1].numItems])
if height > 1 {
copy(n.children[i+1].children[1:],
n.children[i+1].children[:n.children[i+1].numItems+1])
}
n.children[i+1].items[0] = n.items[i]
if height > 1 {
n.children[i+1].children[0] =
n.children[i].children[n.children[i].numItems]
}
n.children[i+1].numItems++
n.items[i] = n.children[i].items[n.children[i].numItems-1]
n.children[i].items[n.children[i].numItems-1] = item{}
if height > 1 {
n.children[i].children[n.children[i].numItems] = nil
}
n.children[i].numItems--
} else {
// move right -> left
n.children[i].items[n.children[i].numItems] = n.items[i]
if height > 1 {
n.children[i].children[n.children[i].numItems+1] =
n.children[i+1].children[0]
}
n.children[i].numItems++
n.items[i] = n.children[i+1].items[0]
copy(n.children[i+1].items[:],
n.children[i+1].items[1:n.children[i+1].numItems])
if height > 1 {
copy(n.children[i+1].children[:],
n.children[i+1].children[1:n.children[i+1].numItems+1])
}
n.children[i+1].numItems--
}
}
return
}
// Ascend the tree within the range [pivot, last]
func (tr *BTree) Ascend(
pivot string,
iter func(key string, value interface{}) bool,
) {
if tr.root != nil {
tr.root.ascend(pivot, iter, tr.height)
}
}
func (n *node) ascend(
pivot string,
iter func(key string, value interface{}) bool,
height int,
) bool {
i, found := n.find(pivot)
if !found {
if height > 0 {
if !n.children[i].ascend(pivot, iter, height-1) {
return false
}
}
}
for ; i < n.numItems; i++ {
if !iter(n.items[i].key, n.items[i].value) {
return false
}
if height > 0 {
if !n.children[i+1].scan(iter, height-1) {
return false
}
}
}
return true
}
// Reverse all items in tree
func (tr *BTree) Reverse(iter func(key string, value interface{}) bool) {
if tr.root != nil {
tr.root.reverse(iter, tr.height)
}
}
func (n *node) reverse(
iter func(key string, value interface{}) bool, height int,
) bool {
if height == 0 {
for i := n.numItems - 1; i >= 0; i-- {
if !iter(n.items[i].key, n.items[i].value) {
return false
}
}
return true
}
if !n.children[n.numItems].reverse(iter, height-1) {
return false
}
for i := n.numItems - 1; i >= 0; i-- {
if !iter(n.items[i].key, n.items[i].value) {
return false
}
if !n.children[i].reverse(iter, height-1) {
return false
}
}
return true
}
// Descend the tree within the range [pivot, first]
func (tr *BTree) Descend(
pivot string,
iter func(key string, value interface{}) bool,
) {
if tr.root != nil {
tr.root.descend(pivot, iter, tr.height)
}
}
func (n *node) descend(
pivot string,
iter func(key string, value interface{}) bool,
height int,
) bool {
i, found := n.find(pivot)
if !found {
if height > 0 {
if !n.children[i].descend(pivot, iter, height-1) {
return false
}
}
i--
}
for ; i >= 0; i-- {
if !iter(n.items[i].key, n.items[i].value) {
return false
}
if height > 0 {
if !n.children[i].reverse(iter, height-1) {
return false
}
}
}
return true
}

2
vendor/modules.txt vendored
View File

@ -126,8 +126,6 @@ github.com/tidwall/rtree
github.com/tidwall/rtree/base
# github.com/tidwall/sjson v1.1.1
github.com/tidwall/sjson
# github.com/tidwall/tinybtree v1.0.1
github.com/tidwall/tinybtree
# github.com/tidwall/tinyqueue v0.0.0-20180302190814-1e39f5511563
github.com/tidwall/tinyqueue
# github.com/yuin/gopher-lua v0.0.0-20200816102855-ee81675732da