mirror
/
tile38
mirror of https://github.com/tidwall/tile38.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Dep ensure

This commit is contained in:
tidwall 2019-03-14 09:13:18 -07:00
parent 4d5b6571da
commit d634b2e302
4 changed files with 311 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
Gopkg.lock generated
View File

@ -250,11 +250,7 @@
version = "v1.0.2"
[[projects]]
<<<<<<< HEAD
digest = "1:ab5e0d19c706286deed5e6ec63a35ee0f2b92d7b9e97083eb67e5d2d76b4bfdb"
=======
digest = "1:cdab3bce90a53a124ac3982719abde77d779e961d9c180e55c23fb74fc62563a"
>>>>>>> master
name = "github.com/tidwall/geojson"
packages = [
".",
@ -262,13 +258,8 @@
"geometry",
]
pruneopts = ""
<<<<<<< HEAD
revision = "f9500c7d3da6ce149bf80530c36b1a784dcd0f2b"
version = "v1.1.1"
=======
revision = "eaf6e0a55a79c1e879bbbcc879a3176c720d99cd"
version = "v1.1.3"
>>>>>>> master
[[projects]]
digest = "1:eade4ea6782f5eed4a6b3138a648f9a332900650804fd206e5daaf99cc5613ea"

170
vendor/github.com/tidwall/buntdb/README.md generated vendored
View File

@ -1,6 +1,6 @@
<p align="center">
<img
src="logo.png"
<img
src="logo.png"
width="307" height="150" border="0" alt="BuntDB">
<br>
<a href="https://travis-ci.org/tidwall/buntdb"><img src="https://img.shields.io/travis/tidwall/buntdb.svg?style=flat-square" alt="Build Status"></a>
@ -9,15 +9,12 @@
<a href="https://godoc.org/github.com/tidwall/buntdb"><img src="https://img.shields.io/badge/api-reference-blue.svg?style=flat-square" alt="GoDoc"></a>
</p>
BuntDB is a low-level, in-memory, key/value store in pure Go.
BuntDB is a low-level, in-memory, key/value store in pure Go.
It persists to disk, is ACID compliant, and uses locking for multiple
readers and a single writer. It supports custom indexes and geospatial
data. It's ideal for projects that need a dependable database and favor
readers and a single writer. It supports custom indexes and geospatial
data. It's ideal for projects that need a dependable database and favor
speed over data size.
The desire to create BuntDB stems from the need for a new embeddable
database for [Tile38](https://github.com/tidwall/tile38) and [SummitDB](https://github.com/tidwall/summitdb).
Features
========
@ -52,7 +49,7 @@ This will retrieve the library.
## Opening a database
The primary object in BuntDB is a `DB`. To open or create your
The primary object in BuntDB is a `DB`. To open or create your
database, use the `buntdb.Open()` function:
```go
@ -71,8 +68,8 @@ func main() {
log.Fatal(err)
}
defer db.Close()
...
...
}
```
@ -94,18 +91,18 @@ A read-only transaction should be used when you don't need to make changes to th
```go
err := db.View(func(tx *buntdb.Tx) error {
...
return nil
...
return nil
})
```
### Read/write Transactions
A read/write transaction is used when you need to make changes to your data. There can only be one read/write transaction running at a time. So make sure you close it as soon as you are done with it.
A read/write transaction is used when you need to make changes to your data. There can only be one read/write transaction running at a time. So make sure you close it as soon as you are done with it.
```go
err := db.Update(func(tx *buntdb.Tx) error {
...
return nil
...
return nil
})
```
@ -115,8 +112,8 @@ To set a value you must open a read/write transaction:
```go
err := db.Update(func(tx *buntdb.Tx) error {
_, _, err := tx.Set("mykey", "myvalue", nil)
return err
_, _, err := tx.Set("mykey", "myvalue", nil)
return err
})
```
@ -125,34 +122,32 @@ To get the value:
```go
err := db.View(func(tx *buntdb.Tx) error {
val, err := tx.Get("mykey")
if err != nil{
return err
}
fmt.Printf("value is %s\n", val)
return nil
val, err := tx.Get("mykey")
if err != nil{
return err
}
fmt.Printf("value is %s\n", val)
return nil
})
```
Getting non-existent values will case an `ErrNotFound` error.
Getting non-existent values will cause an `ErrNotFound` error.
### Iterating
All keys/value pairs are ordered in the database by the key. To iterate over the keys:
```go
err := db.View(func(tx *buntdb.Tx) error {
err := tx.Ascend("", func(key, value string) bool{
fmt.Printf("key: %s, value: %s\n", key, value)
})
return err
err := tx.Ascend("", func(key, value string) bool {
fmt.Printf("key: %s, value: %s\n", key, value)
})
return err
})
```
There is also `AscendGreaterOrEqual`, `AscendLessThan`, `AscendRange`, `AscendEqual`, `Descend`, `DescendLessOrEqual`, `DescendGreaterThan`, `DescendRange`, and `DescendEqual`. Please see the [documentation](https://godoc.org/github.com/tidwall/buntdb) for more information on these functions.
## Custom Indexes
Initially all data is stored in a single [B-tree](https://en.wikipedia.org/wiki/B-tree) with each item having one key and one value. All of these items are ordered by the key. This is great for quickly getting a value from a key or [iterating](#iterating) over the keys. Feel free to peruse the [B-tree implementation](https://github.com/tidwall/btree).
@ -170,7 +165,7 @@ Now you can add various names:
```go
db.Update(func(tx *buntdb.Tx) error {
tx.Set("user:0:name", "tom", nil)
tx.Set("user:0:name", "tom", nil)
tx.Set("user:1:name", "Randi", nil)
tx.Set("user:2:name", "jane", nil)
tx.Set("user:4:name", "Janet", nil)
@ -186,10 +181,10 @@ Finally you can iterate over the index:
```go
db.View(func(tx *buntdb.Tx) error {
tx.Ascend("names", func(key, val string) bool {
fmt.Printf(buf, "%s %s\n", key, val)
fmt.Printf(buf, "%s %s\n", key, val)
return true
})
return nil
return nil
})
```
The output should be:
@ -213,7 +208,7 @@ Now only items with keys that have the prefix `user:` will be added to the `name
### Built-in types
Along with `IndexString`, there is also `IndexInt`, `IndexUint`, and `IndexFloat`.
Along with `IndexString`, there is also `IndexInt`, `IndexUint`, and `IndexFloat`.
These are built-in types for indexing. You can choose to use these or create your own.
So to create an index that is numerically ordered on an age key, we could use:
@ -226,7 +221,7 @@ And then add values:
```go
db.Update(func(tx *buntdb.Tx) error {
tx.Set("user:0:age", "35", nil)
tx.Set("user:0:age", "35", nil)
tx.Set("user:1:age", "49", nil)
tx.Set("user:2:age", "13", nil)
tx.Set("user:4:age", "63", nil)
@ -240,22 +235,22 @@ db.Update(func(tx *buntdb.Tx) error {
```go
db.View(func(tx *buntdb.Tx) error {
tx.Ascend("ages", func(key, val string) bool {
fmt.Printf(buf, "%s %s\n", key, val)
fmt.Printf(buf, "%s %s\n", key, val)
return true
})
return nil
return nil
})
```
The output should be:
```
user:6:name 3
user:5:name 8
user:2:name 13
user:7:name 16
user:0:name 35
user:1:name 49
user:4:name 63
user:6:age 3
user:5:age 8
user:2:age 13
user:7:age 16
user:0:age 35
user:1:age 49
user:4:age 63
```
## Spatial Indexes
@ -273,7 +268,7 @@ To add some lon,lat points to the `fleet` index:
```go
db.Update(func(tx *buntdb.Tx) error {
tx.Set("fleet:0:pos", "[-115.567 33.532]", nil)
tx.Set("fleet:0:pos", "[-115.567 33.532]", nil)
tx.Set("fleet:1:pos", "[-116.671 35.735]", nil)
tx.Set("fleet:2:pos", "[-113.902 31.234]", nil)
return nil
@ -284,38 +279,52 @@ And then you can run the `Intersects` function on the index:
```go
db.View(func(tx *buntdb.Tx) error {
tx.Intersects("fleet", "[-117 30],[-112 36]", func(key, val string) bool {
...
return true
})
return nil
tx.Intersects("fleet", "[-117 30],[-112 36]", func(key, val string) bool {
...
return true
})
return nil
})
```
This will get all three positions.
### k-Nearest Neighbors
Use the `Nearby` function to get all the positions in order of nearest to farthest :
```go
db.View(func(tx *buntdb.Tx) error {
tx.Nearby("fleet", "[-113 33]", func(key, val string, dist float64) bool {
...
return true
})
return nil
})
```
### Spatial bracket syntax
The bracket syntax `[-117 30],[-112 36]` is unique to BuntDB, and it's how the built-in rectangles are processed. But, you are not limited to this syntax. Whatever Rect function you choose to use during `CreateSpatialIndex` will be used to process the parameter, in this case it's `IndexRect`.
- **2D rectangle:** `[10 15],[20 25]`
- **2D rectangle:** `[10 15],[20 25]`
*Min XY: "10x15", Max XY: "20x25"*
- **3D rectangle:** `[10 15 12],[20 25 18]`
- **3D rectangle:** `[10 15 12],[20 25 18]`
*Min XYZ: "10x15x12", Max XYZ: "20x25x18"*
- **2D point:** `[10 15]`
- **2D point:** `[10 15]`
*XY: "10x15"*
- **LatLon point:** `[-112.2693 33.5123]`
- **LonLat point:** `[-112.2693 33.5123]`
*LatLon: "33.5123 -112.2693"*
- **LatLon bounding box:** `[-112.26 33.51],[-112.18 33.67]`
- **LonLat bounding box:** `[-112.26 33.51],[-112.18 33.67]`
*Min LatLon: "33.51 -112.26", Max LatLon: "33.67 -112.18"*
**Notice:** The longitude is the Y axis and is on the left, and latitude is the X axis and is on the right.
You can also represent `Infinity` by using `-inf` and `+inf`.
You can also represent `Infinity` by using `-inf` and `+inf`.
For example, you might have the following points (`[X Y M]` where XY is a point and M is a timestamp):
```
[3 9 1]
@ -330,8 +339,8 @@ You can then do a search for all points with `M` between 2-4 by calling `Interse
```go
tx.Intersects("points", "[-inf -inf 2],[+inf +inf 4]", func(key, val string) bool {
println(val)
return true
println(val)
return true
})
```
@ -410,7 +419,7 @@ Order by age range 30-50
```
## Multi Value Index
With BuntDB it's possible to join multiple values on a single index.
With BuntDB it's possible to join multiple values on a single index.
This is similar to a [multi column index](http://dev.mysql.com/doc/refman/5.7/en/multiple-column-indexes.html) in a traditional SQL database.
In this example we are creating a multi value index on "name.last" and "age":
@ -448,9 +457,9 @@ db.View(func(tx *buntdb.Tx) error {
Any index can be put in descending order by wrapping it's less function with `buntdb.Desc`.
```go
db.CreateIndex("last_name_age", "*",
buntdb.IndexJSON("name.last"),
buntdb.Desc(buntdb.IndexJSON("age")))
db.CreateIndex("last_name_age", "*",
buntdb.IndexJSON("name.last"),
buntdb.Desc(buntdb.IndexJSON("age")))
```
This will create a multi value index where the last name is ascending and the age is descending.
@ -474,9 +483,9 @@ import "github.com/tidwall/collate"
// To sort case-insensitive in French.
db.CreateIndex("name", "*", collate.IndexString("FRENCH_CI"))
// To specify that numbers should sort numerically ("2" < "12")
// To specify that numbers should sort numerically ("2" < "12")
// and use a comma to represent a decimal point.
db.CreateIndex("amount", "*", collate.IndexString("FRENCH_NUM"))
db.CreateIndex("amount", "*", collate.IndexString("FRENCH_NUM"))
```
There's also support for Collation on JSON indexes:
@ -492,16 +501,35 @@ Items can be automatically evicted by using the `SetOptions` object in the `Set`
```go
db.Update(func(tx *buntdb.Tx) error {
tx.Set("mykey", "myval", &buntdb.SetOptions{Expires:true, TTL:time.Second})
tx.Set("mykey", "myval", &buntdb.SetOptions{Expires:true, TTL:time.Second})
return nil
})
```
Now `mykey` will automatically be deleted after one second. You can remove the TTL by setting the value again with the same key/value, but with the options parameter set to nil.
## Delete while iterating
BuntDB does not currently support deleting a key while in the process of iterating.
As a workaround you'll need to delete keys following the completion of the iterator.
```go
var delkeys []string
tx.AscendKeys("object:*", func(k, v string) bool {
if someCondition(k) == true {
delkeys = append(delkeys, k)
}
return true // continue
})
for _, k := range delkeys {
if _, err = tx.Delete(k); err != nil {
return err
}
}
```
## Append-only File
BuntDB uses an AOF (append-only file) which is a log of all database changes that occur from operations like `Set()` and `Delete()`.
BuntDB uses an AOF (append-only file) which is a log of all database changes that occur from operations like `Set()` and `Delete()`.
The format of this file looks like:
```
@ -531,7 +559,7 @@ The `Config.SyncPolicy` has the following options:
- `EverySecond` - fsync every second, fast and safer, this is the default
- `Always` - fsync after every write, very durable, slower
## Config
## Config
Here are some configuration options that can be use to change various behaviors of the database.
@ -546,10 +574,10 @@ To update the configuration you should call `ReadConfig` followed by `SetConfig`
var config buntdb.Config
if err := db.ReadConfig(&config); err != nil{
log.Fatal(err)
log.Fatal(err)
}
if err := db.WriteConfig(config); err != nil{
log.Fatal(err)
log.Fatal(err)
}
```
@ -557,7 +585,7 @@ if err := db.WriteConfig(config); err != nil{
How fast is BuntDB?
Here are some example [benchmarks](https://github.com/tidwall/raft-buntdb#raftstore-performance-comparison) when using BuntDB in a Raft Store implementation.
Here are some example [benchmarks](https://github.com/tidwall/raft-buntdb#raftstore-performance-comparison) when using BuntDB in a Raft Store implementation.
You can also run the standard Go benchmark tool from the project root directory:

87
vendor/github.com/tidwall/buntdb/buntdb.go generated vendored
View File

@ -121,6 +121,12 @@ type Config struct {
// OnExpired is used to custom handle the deletion option when a key
// has been expired.
OnExpired func(keys []string)
// OnExpiredSync will be called inside the same transaction that is performing
// the deletion of expired items. If OnExpired is present then this callback
// will not be called. If this callback is present, then the deletion of the
// timeed-out item is the explicit responsibility of this callback.
OnExpiredSync func(key, value string, tx *Tx) error
}
// exctx is a simple b-tree context for ordering by expiration.
@ -544,9 +550,13 @@ func (db *DB) backgroundManager() {
// Open a standard view. This will take a full lock of the
// database thus allowing for access to anything we need.
var onExpired func([]string)
var expired []string
var expired []*dbItem
var onExpiredSync func(key, value string, tx *Tx) error
err := db.Update(func(tx *Tx) error {
onExpired = db.config.OnExpired
if onExpired == nil {
onExpiredSync = db.config.OnExpiredSync
}
if db.persist && !db.config.AutoShrinkDisabled {
pos, err := db.file.Seek(0, 1)
if err != nil {
@ -562,12 +572,12 @@ func (db *DB) backgroundManager() {
db.exps.AscendLessThan(&dbItem{
opts: &dbItemOpts{ex: true, exat: time.Now()},
}, func(item btree.Item) bool {
expired = append(expired, item.(*dbItem).key)
expired = append(expired, item.(*dbItem))
return true
})
if onExpired == nil {
for _, key := range expired {
if _, err := tx.Delete(key); err != nil {
if onExpired == nil && onExpiredSync == nil {
for _, itm := range expired {
if _, err := tx.Delete(itm.key); err != nil {
// it's ok to get a "not found" because the
// 'Delete' method reports "not found" for
// expired items.
@ -576,6 +586,12 @@ func (db *DB) backgroundManager() {
}
}
}
} else if onExpiredSync != nil {
for _, itm := range expired {
if err := onExpiredSync(itm.key, itm.val, tx); err != nil {
return err
}
}
}
return nil
})
@ -585,7 +601,11 @@ func (db *DB) backgroundManager() {
// send expired event, if needed
if onExpired != nil && len(expired) > 0 {
onExpired(expired)
keys := make([]string, 0, 32)
for _, itm := range expired {
keys = append(keys, itm.key)
}
onExpired(keys)
}
// execute a disk sync, if needed
@ -1399,13 +1419,18 @@ func (tx *Tx) Set(key, value string, opts *SetOptions) (previousValue string,
}
// Get returns a value for a key. If the item does not exist or if the item
// has expired then ErrNotFound is returned.
func (tx *Tx) Get(key string) (val string, err error) {
// has expired then ErrNotFound is returned. If ignoreExpired is true, then
// the found value will be returned even if it is expired.
func (tx *Tx) Get(key string, ignoreExpired ...bool) (val string, err error) {
if tx.db == nil {
return "", ErrTxClosed
}
var ignore bool
if len(ignoreExpired) != 0 {
ignore = ignoreExpired[0]
}
item := tx.db.get(key)
if item == nil || item.expired() {
if item == nil || (item.expired() && !ignore) {
// The item does not exists or has expired. Let's assume that
// the caller is only interested in items that have not expired.
return "", ErrNotFound
@ -1775,7 +1800,7 @@ func (tx *Tx) DescendEqual(index, pivot string,
})
}
// rect is used by Intersects
// rect is used by Intersects and Nearby
type rect struct {
min, max []float64
}
@ -1784,6 +1809,48 @@ func (r *rect) Rect(ctx interface{}) (min, max []float64) {
return r.min, r.max
}
// Nearby searches for rectangle items that are nearby a target rect.
// All items belonging to the specified index will be returned in order of
// nearest to farthest.
// The specified index must have been created by AddIndex() and the target
// is represented by the rect string. This string will be processed by the
// same bounds function that was passed to the CreateSpatialIndex() function.
// An invalid index will return an error.
// The dist param is the distance of the bounding boxes. In the case of
// simple 2D points, it's the distance of the two 2D points squared.
func (tx *Tx) Nearby(index, bounds string,
iterator func(key, value string, dist float64) bool) error {
if tx.db == nil {
return ErrTxClosed
}
if index == "" {
// cannot search on keys tree. just return nil.
return nil
}
// // wrap a rtree specific iterator around the user-defined iterator.
iter := func(item rtree.Item, dist float64) bool {
dbi := item.(*dbItem)
return iterator(dbi.key, dbi.val, dist)
}
idx := tx.db.idxs[index]
if idx == nil {
// index was not found. return error
return ErrNotFound
}
if idx.rtr == nil {
// not an r-tree index. just return nil
return nil
}
// execute the nearby search
var min, max []float64
if idx.rect != nil {
min, max = idx.rect(bounds)
}
// set the center param to false, which uses the box dist calc.
idx.rtr.KNN(&rect{min, max}, false, iter)
return nil
}
// Intersects searches for rectangle items that intersect a target rect.
// The specified index must have been created by AddIndex() and the target
// is represented by the rect string. This string will be processed by the

135
vendor/github.com/tidwall/buntdb/buntdb_test.go generated vendored
View File

@ -1021,6 +1021,46 @@ func TestVariousTx(t *testing.T) {
}
}
func TestNearby(t *testing.T) {
rand.Seed(time.Now().UnixNano())
N := 100000
db, _ := Open(":memory:")
db.CreateSpatialIndex("points", "*", IndexRect)
db.Update(func(tx *Tx) error {
for i := 0; i < N; i++ {
p := Point(
rand.Float64()*100,
rand.Float64()*100,
rand.Float64()*100,
rand.Float64()*100,
)
tx.Set(fmt.Sprintf("p:%d", i), p, nil)
}
return nil
})
var keys, values []string
var dists []float64
var pdist float64
var i int
db.View(func(tx *Tx) error {
tx.Nearby("points", Point(0, 0, 0, 0), func(key, value string, dist float64) bool {
if i != 0 && dist < pdist {
t.Fatal("out of order")
}
keys = append(keys, key)
values = append(values, value)
dists = append(dists, dist)
pdist = dist
i++
return true
})
return nil
})
if len(keys) != N {
t.Fatalf("expected '%v', got '%v'", N, len(keys))
}
}
func Example_descKeys() {
db, _ := Open(":memory:")
db.CreateIndex("name", "*", IndexString)
@ -2511,3 +2551,98 @@ func TestJSONIndex(t *testing.T) {
t.Fatalf("expected %v, got %v", expect, strings.Join(keys, ","))
}
}
func TestOnExpiredSync(t *testing.T) {
db := testOpen(t)
defer testClose(db)
var config Config
if err := db.ReadConfig(&config); err != nil {
t.Fatal(err)
}
hits := make(chan int, 3)
config.OnExpiredSync = func(key, value string, tx *Tx) error {
n, err := strconv.Atoi(value)
if err != nil {
return err
}
defer func() { hits <- n }()
if n >= 2 {
_, err = tx.Delete(key)
if err != ErrNotFound {
return err
}
return nil
}
n++
_, _, err = tx.Set(key, strconv.Itoa(n), &SetOptions{Expires: true, TTL: time.Millisecond * 100})
return err
}
if err := db.SetConfig(config); err != nil {
t.Fatal(err)
}
err := db.Update(func(tx *Tx) error {
_, _, err := tx.Set("K", "0", &SetOptions{Expires: true, TTL: time.Millisecond * 100})
return err
})
if err != nil {
t.Fail()
}
done := make(chan struct{})
go func() {
ticks := time.NewTicker(time.Millisecond * 50)
defer ticks.Stop()
for {
select {
case <-done:
return
case <-ticks.C:
err := db.View(func(tx *Tx) error {
v, err := tx.Get("K", true)
if err != nil {
return err
}
n, err := strconv.Atoi(v)
if err != nil {
return err
}
if n < 0 || n > 2 {
t.Fail()
}
return nil
})
if err != nil {
t.Fail()
}
}
}
}()
OUTER1:
for {
select {
case <-time.After(time.Second * 2):
t.Fail()
case v := <-hits:
if v >= 2 {
break OUTER1
}
}
}
err = db.View(func(tx *Tx) error {
defer close(done)
v, err := tx.Get("K")
if err != nil {
t.Fail()
return err
}
if v != "2" {
t.Fail()
}
return nil
})
if err != nil {
t.Fail()
}
}