From d634b2e302ab0383a088045834a3669ca12d6a05 Mon Sep 17 00:00:00 2001 From: tidwall Date: Thu, 14 Mar 2019 09:13:18 -0700 Subject: [PATCH] Dep ensure --- Gopkg.lock | 9 - vendor/github.com/tidwall/buntdb/README.md | 170 ++++++++++-------- vendor/github.com/tidwall/buntdb/buntdb.go | 87 +++++++-- .../github.com/tidwall/buntdb/buntdb_test.go | 135 ++++++++++++++ 4 files changed, 311 insertions(+), 90 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index 9b61cc43..3ca15cdb 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -250,11 +250,7 @@ version = "v1.0.2" [[projects]] -<<<<<<< HEAD - digest = "1:ab5e0d19c706286deed5e6ec63a35ee0f2b92d7b9e97083eb67e5d2d76b4bfdb" -======= digest = "1:cdab3bce90a53a124ac3982719abde77d779e961d9c180e55c23fb74fc62563a" ->>>>>>> master name = "github.com/tidwall/geojson" packages = [ ".", @@ -262,13 +258,8 @@ "geometry", ] pruneopts = "" -<<<<<<< HEAD - revision = "f9500c7d3da6ce149bf80530c36b1a784dcd0f2b" - version = "v1.1.1" -======= revision = "eaf6e0a55a79c1e879bbbcc879a3176c720d99cd" version = "v1.1.3" ->>>>>>> master [[projects]] digest = "1:eade4ea6782f5eed4a6b3138a648f9a332900650804fd206e5daaf99cc5613ea" diff --git a/vendor/github.com/tidwall/buntdb/README.md b/vendor/github.com/tidwall/buntdb/README.md index 66562322..31798093 100644 --- a/vendor/github.com/tidwall/buntdb/README.md +++ b/vendor/github.com/tidwall/buntdb/README.md @@ -1,6 +1,6 @@

-BuntDB
Build Status @@ -9,15 +9,12 @@ GoDoc

-BuntDB is a low-level, in-memory, key/value store in pure Go. +BuntDB is a low-level, in-memory, key/value store in pure Go. It persists to disk, is ACID compliant, and uses locking for multiple -readers and a single writer. It supports custom indexes and geospatial -data. It's ideal for projects that need a dependable database and favor +readers and a single writer. It supports custom indexes and geospatial +data. It's ideal for projects that need a dependable database and favor speed over data size. -The desire to create BuntDB stems from the need for a new embeddable -database for [Tile38](https://github.com/tidwall/tile38) and [SummitDB](https://github.com/tidwall/summitdb). - Features ======== @@ -52,7 +49,7 @@ This will retrieve the library. ## Opening a database -The primary object in BuntDB is a `DB`. To open or create your +The primary object in BuntDB is a `DB`. To open or create your database, use the `buntdb.Open()` function: ```go @@ -71,8 +68,8 @@ func main() { log.Fatal(err) } defer db.Close() - - ... + + ... } ``` @@ -94,18 +91,18 @@ A read-only transaction should be used when you don't need to make changes to th ```go err := db.View(func(tx *buntdb.Tx) error { - ... - return nil + ... + return nil }) ``` ### Read/write Transactions -A read/write transaction is used when you need to make changes to your data. There can only be one read/write transaction running at a time. So make sure you close it as soon as you are done with it. +A read/write transaction is used when you need to make changes to your data. There can only be one read/write transaction running at a time. So make sure you close it as soon as you are done with it. ```go err := db.Update(func(tx *buntdb.Tx) error { - ... - return nil + ... + return nil }) ``` @@ -115,8 +112,8 @@ To set a value you must open a read/write transaction: ```go err := db.Update(func(tx *buntdb.Tx) error { - _, _, err := tx.Set("mykey", "myvalue", nil) - return err + _, _, err := tx.Set("mykey", "myvalue", nil) + return err }) ``` @@ -125,34 +122,32 @@ To get the value: ```go err := db.View(func(tx *buntdb.Tx) error { - val, err := tx.Get("mykey") - if err != nil{ - return err - } - fmt.Printf("value is %s\n", val) - return nil + val, err := tx.Get("mykey") + if err != nil{ + return err + } + fmt.Printf("value is %s\n", val) + return nil }) ``` -Getting non-existent values will case an `ErrNotFound` error. +Getting non-existent values will cause an `ErrNotFound` error. ### Iterating All keys/value pairs are ordered in the database by the key. To iterate over the keys: ```go err := db.View(func(tx *buntdb.Tx) error { -err := tx.Ascend("", func(key, value string) bool{ - fmt.Printf("key: %s, value: %s\n", key, value) - }) - return err + err := tx.Ascend("", func(key, value string) bool { + fmt.Printf("key: %s, value: %s\n", key, value) + }) + return err }) ``` There is also `AscendGreaterOrEqual`, `AscendLessThan`, `AscendRange`, `AscendEqual`, `Descend`, `DescendLessOrEqual`, `DescendGreaterThan`, `DescendRange`, and `DescendEqual`. Please see the [documentation](https://godoc.org/github.com/tidwall/buntdb) for more information on these functions. - - ## Custom Indexes Initially all data is stored in a single [B-tree](https://en.wikipedia.org/wiki/B-tree) with each item having one key and one value. All of these items are ordered by the key. This is great for quickly getting a value from a key or [iterating](#iterating) over the keys. Feel free to peruse the [B-tree implementation](https://github.com/tidwall/btree). @@ -170,7 +165,7 @@ Now you can add various names: ```go db.Update(func(tx *buntdb.Tx) error { - tx.Set("user:0:name", "tom", nil) + tx.Set("user:0:name", "tom", nil) tx.Set("user:1:name", "Randi", nil) tx.Set("user:2:name", "jane", nil) tx.Set("user:4:name", "Janet", nil) @@ -186,10 +181,10 @@ Finally you can iterate over the index: ```go db.View(func(tx *buntdb.Tx) error { tx.Ascend("names", func(key, val string) bool { - fmt.Printf(buf, "%s %s\n", key, val) + fmt.Printf(buf, "%s %s\n", key, val) return true }) - return nil + return nil }) ``` The output should be: @@ -213,7 +208,7 @@ Now only items with keys that have the prefix `user:` will be added to the `name ### Built-in types -Along with `IndexString`, there is also `IndexInt`, `IndexUint`, and `IndexFloat`. +Along with `IndexString`, there is also `IndexInt`, `IndexUint`, and `IndexFloat`. These are built-in types for indexing. You can choose to use these or create your own. So to create an index that is numerically ordered on an age key, we could use: @@ -226,7 +221,7 @@ And then add values: ```go db.Update(func(tx *buntdb.Tx) error { - tx.Set("user:0:age", "35", nil) + tx.Set("user:0:age", "35", nil) tx.Set("user:1:age", "49", nil) tx.Set("user:2:age", "13", nil) tx.Set("user:4:age", "63", nil) @@ -240,22 +235,22 @@ db.Update(func(tx *buntdb.Tx) error { ```go db.View(func(tx *buntdb.Tx) error { tx.Ascend("ages", func(key, val string) bool { - fmt.Printf(buf, "%s %s\n", key, val) + fmt.Printf(buf, "%s %s\n", key, val) return true }) - return nil + return nil }) ``` The output should be: ``` -user:6:name 3 -user:5:name 8 -user:2:name 13 -user:7:name 16 -user:0:name 35 -user:1:name 49 -user:4:name 63 +user:6:age 3 +user:5:age 8 +user:2:age 13 +user:7:age 16 +user:0:age 35 +user:1:age 49 +user:4:age 63 ``` ## Spatial Indexes @@ -273,7 +268,7 @@ To add some lon,lat points to the `fleet` index: ```go db.Update(func(tx *buntdb.Tx) error { - tx.Set("fleet:0:pos", "[-115.567 33.532]", nil) + tx.Set("fleet:0:pos", "[-115.567 33.532]", nil) tx.Set("fleet:1:pos", "[-116.671 35.735]", nil) tx.Set("fleet:2:pos", "[-113.902 31.234]", nil) return nil @@ -284,38 +279,52 @@ And then you can run the `Intersects` function on the index: ```go db.View(func(tx *buntdb.Tx) error { - tx.Intersects("fleet", "[-117 30],[-112 36]", func(key, val string) bool { - ... - return true - }) - return nil + tx.Intersects("fleet", "[-117 30],[-112 36]", func(key, val string) bool { + ... + return true + }) + return nil }) ``` This will get all three positions. +### k-Nearest Neighbors + +Use the `Nearby` function to get all the positions in order of nearest to farthest : + +```go +db.View(func(tx *buntdb.Tx) error { + tx.Nearby("fleet", "[-113 33]", func(key, val string, dist float64) bool { + ... + return true + }) + return nil +}) +``` + ### Spatial bracket syntax The bracket syntax `[-117 30],[-112 36]` is unique to BuntDB, and it's how the built-in rectangles are processed. But, you are not limited to this syntax. Whatever Rect function you choose to use during `CreateSpatialIndex` will be used to process the parameter, in this case it's `IndexRect`. -- **2D rectangle:** `[10 15],[20 25]` +- **2D rectangle:** `[10 15],[20 25]` *Min XY: "10x15", Max XY: "20x25"* -- **3D rectangle:** `[10 15 12],[20 25 18]` +- **3D rectangle:** `[10 15 12],[20 25 18]` *Min XYZ: "10x15x12", Max XYZ: "20x25x18"* -- **2D point:** `[10 15]` +- **2D point:** `[10 15]` *XY: "10x15"* -- **LatLon point:** `[-112.2693 33.5123]` +- **LonLat point:** `[-112.2693 33.5123]` *LatLon: "33.5123 -112.2693"* -- **LatLon bounding box:** `[-112.26 33.51],[-112.18 33.67]` +- **LonLat bounding box:** `[-112.26 33.51],[-112.18 33.67]` *Min LatLon: "33.51 -112.26", Max LatLon: "33.67 -112.18"* **Notice:** The longitude is the Y axis and is on the left, and latitude is the X axis and is on the right. -You can also represent `Infinity` by using `-inf` and `+inf`. +You can also represent `Infinity` by using `-inf` and `+inf`. For example, you might have the following points (`[X Y M]` where XY is a point and M is a timestamp): ``` [3 9 1] @@ -330,8 +339,8 @@ You can then do a search for all points with `M` between 2-4 by calling `Interse ```go tx.Intersects("points", "[-inf -inf 2],[+inf +inf 4]", func(key, val string) bool { - println(val) - return true + println(val) + return true }) ``` @@ -410,7 +419,7 @@ Order by age range 30-50 ``` ## Multi Value Index -With BuntDB it's possible to join multiple values on a single index. +With BuntDB it's possible to join multiple values on a single index. This is similar to a [multi column index](http://dev.mysql.com/doc/refman/5.7/en/multiple-column-indexes.html) in a traditional SQL database. In this example we are creating a multi value index on "name.last" and "age": @@ -448,9 +457,9 @@ db.View(func(tx *buntdb.Tx) error { Any index can be put in descending order by wrapping it's less function with `buntdb.Desc`. ```go -db.CreateIndex("last_name_age", "*", - buntdb.IndexJSON("name.last"), - buntdb.Desc(buntdb.IndexJSON("age"))) +db.CreateIndex("last_name_age", "*", +buntdb.IndexJSON("name.last"), +buntdb.Desc(buntdb.IndexJSON("age"))) ``` This will create a multi value index where the last name is ascending and the age is descending. @@ -474,9 +483,9 @@ import "github.com/tidwall/collate" // To sort case-insensitive in French. db.CreateIndex("name", "*", collate.IndexString("FRENCH_CI")) -// To specify that numbers should sort numerically ("2" < "12") +// To specify that numbers should sort numerically ("2" < "12") // and use a comma to represent a decimal point. -db.CreateIndex("amount", "*", collate.IndexString("FRENCH_NUM")) +db.CreateIndex("amount", "*", collate.IndexString("FRENCH_NUM")) ``` There's also support for Collation on JSON indexes: @@ -492,16 +501,35 @@ Items can be automatically evicted by using the `SetOptions` object in the `Set` ```go db.Update(func(tx *buntdb.Tx) error { - tx.Set("mykey", "myval", &buntdb.SetOptions{Expires:true, TTL:time.Second}) + tx.Set("mykey", "myval", &buntdb.SetOptions{Expires:true, TTL:time.Second}) return nil }) ``` Now `mykey` will automatically be deleted after one second. You can remove the TTL by setting the value again with the same key/value, but with the options parameter set to nil. +## Delete while iterating +BuntDB does not currently support deleting a key while in the process of iterating. +As a workaround you'll need to delete keys following the completion of the iterator. + +```go +var delkeys []string +tx.AscendKeys("object:*", func(k, v string) bool { + if someCondition(k) == true { + delkeys = append(delkeys, k) + } + return true // continue +}) +for _, k := range delkeys { + if _, err = tx.Delete(k); err != nil { + return err + } +} +``` + ## Append-only File -BuntDB uses an AOF (append-only file) which is a log of all database changes that occur from operations like `Set()` and `Delete()`. +BuntDB uses an AOF (append-only file) which is a log of all database changes that occur from operations like `Set()` and `Delete()`. The format of this file looks like: ``` @@ -531,7 +559,7 @@ The `Config.SyncPolicy` has the following options: - `EverySecond` - fsync every second, fast and safer, this is the default - `Always` - fsync after every write, very durable, slower -## Config +## Config Here are some configuration options that can be use to change various behaviors of the database. @@ -546,10 +574,10 @@ To update the configuration you should call `ReadConfig` followed by `SetConfig` var config buntdb.Config if err := db.ReadConfig(&config); err != nil{ - log.Fatal(err) + log.Fatal(err) } if err := db.WriteConfig(config); err != nil{ - log.Fatal(err) + log.Fatal(err) } ``` @@ -557,7 +585,7 @@ if err := db.WriteConfig(config); err != nil{ How fast is BuntDB? -Here are some example [benchmarks](https://github.com/tidwall/raft-buntdb#raftstore-performance-comparison) when using BuntDB in a Raft Store implementation. +Here are some example [benchmarks](https://github.com/tidwall/raft-buntdb#raftstore-performance-comparison) when using BuntDB in a Raft Store implementation. You can also run the standard Go benchmark tool from the project root directory: diff --git a/vendor/github.com/tidwall/buntdb/buntdb.go b/vendor/github.com/tidwall/buntdb/buntdb.go index 9fcaf31b..35f85200 100644 --- a/vendor/github.com/tidwall/buntdb/buntdb.go +++ b/vendor/github.com/tidwall/buntdb/buntdb.go @@ -121,6 +121,12 @@ type Config struct { // OnExpired is used to custom handle the deletion option when a key // has been expired. OnExpired func(keys []string) + + // OnExpiredSync will be called inside the same transaction that is performing + // the deletion of expired items. If OnExpired is present then this callback + // will not be called. If this callback is present, then the deletion of the + // timeed-out item is the explicit responsibility of this callback. + OnExpiredSync func(key, value string, tx *Tx) error } // exctx is a simple b-tree context for ordering by expiration. @@ -544,9 +550,13 @@ func (db *DB) backgroundManager() { // Open a standard view. This will take a full lock of the // database thus allowing for access to anything we need. var onExpired func([]string) - var expired []string + var expired []*dbItem + var onExpiredSync func(key, value string, tx *Tx) error err := db.Update(func(tx *Tx) error { onExpired = db.config.OnExpired + if onExpired == nil { + onExpiredSync = db.config.OnExpiredSync + } if db.persist && !db.config.AutoShrinkDisabled { pos, err := db.file.Seek(0, 1) if err != nil { @@ -562,12 +572,12 @@ func (db *DB) backgroundManager() { db.exps.AscendLessThan(&dbItem{ opts: &dbItemOpts{ex: true, exat: time.Now()}, }, func(item btree.Item) bool { - expired = append(expired, item.(*dbItem).key) + expired = append(expired, item.(*dbItem)) return true }) - if onExpired == nil { - for _, key := range expired { - if _, err := tx.Delete(key); err != nil { + if onExpired == nil && onExpiredSync == nil { + for _, itm := range expired { + if _, err := tx.Delete(itm.key); err != nil { // it's ok to get a "not found" because the // 'Delete' method reports "not found" for // expired items. @@ -576,6 +586,12 @@ func (db *DB) backgroundManager() { } } } + } else if onExpiredSync != nil { + for _, itm := range expired { + if err := onExpiredSync(itm.key, itm.val, tx); err != nil { + return err + } + } } return nil }) @@ -585,7 +601,11 @@ func (db *DB) backgroundManager() { // send expired event, if needed if onExpired != nil && len(expired) > 0 { - onExpired(expired) + keys := make([]string, 0, 32) + for _, itm := range expired { + keys = append(keys, itm.key) + } + onExpired(keys) } // execute a disk sync, if needed @@ -1399,13 +1419,18 @@ func (tx *Tx) Set(key, value string, opts *SetOptions) (previousValue string, } // Get returns a value for a key. If the item does not exist or if the item -// has expired then ErrNotFound is returned. -func (tx *Tx) Get(key string) (val string, err error) { +// has expired then ErrNotFound is returned. If ignoreExpired is true, then +// the found value will be returned even if it is expired. +func (tx *Tx) Get(key string, ignoreExpired ...bool) (val string, err error) { if tx.db == nil { return "", ErrTxClosed } + var ignore bool + if len(ignoreExpired) != 0 { + ignore = ignoreExpired[0] + } item := tx.db.get(key) - if item == nil || item.expired() { + if item == nil || (item.expired() && !ignore) { // The item does not exists or has expired. Let's assume that // the caller is only interested in items that have not expired. return "", ErrNotFound @@ -1775,7 +1800,7 @@ func (tx *Tx) DescendEqual(index, pivot string, }) } -// rect is used by Intersects +// rect is used by Intersects and Nearby type rect struct { min, max []float64 } @@ -1784,6 +1809,48 @@ func (r *rect) Rect(ctx interface{}) (min, max []float64) { return r.min, r.max } +// Nearby searches for rectangle items that are nearby a target rect. +// All items belonging to the specified index will be returned in order of +// nearest to farthest. +// The specified index must have been created by AddIndex() and the target +// is represented by the rect string. This string will be processed by the +// same bounds function that was passed to the CreateSpatialIndex() function. +// An invalid index will return an error. +// The dist param is the distance of the bounding boxes. In the case of +// simple 2D points, it's the distance of the two 2D points squared. +func (tx *Tx) Nearby(index, bounds string, + iterator func(key, value string, dist float64) bool) error { + if tx.db == nil { + return ErrTxClosed + } + if index == "" { + // cannot search on keys tree. just return nil. + return nil + } + // // wrap a rtree specific iterator around the user-defined iterator. + iter := func(item rtree.Item, dist float64) bool { + dbi := item.(*dbItem) + return iterator(dbi.key, dbi.val, dist) + } + idx := tx.db.idxs[index] + if idx == nil { + // index was not found. return error + return ErrNotFound + } + if idx.rtr == nil { + // not an r-tree index. just return nil + return nil + } + // execute the nearby search + var min, max []float64 + if idx.rect != nil { + min, max = idx.rect(bounds) + } + // set the center param to false, which uses the box dist calc. + idx.rtr.KNN(&rect{min, max}, false, iter) + return nil +} + // Intersects searches for rectangle items that intersect a target rect. // The specified index must have been created by AddIndex() and the target // is represented by the rect string. This string will be processed by the diff --git a/vendor/github.com/tidwall/buntdb/buntdb_test.go b/vendor/github.com/tidwall/buntdb/buntdb_test.go index 9668f2a5..01dd712d 100644 --- a/vendor/github.com/tidwall/buntdb/buntdb_test.go +++ b/vendor/github.com/tidwall/buntdb/buntdb_test.go @@ -1021,6 +1021,46 @@ func TestVariousTx(t *testing.T) { } } +func TestNearby(t *testing.T) { + rand.Seed(time.Now().UnixNano()) + N := 100000 + db, _ := Open(":memory:") + db.CreateSpatialIndex("points", "*", IndexRect) + db.Update(func(tx *Tx) error { + for i := 0; i < N; i++ { + p := Point( + rand.Float64()*100, + rand.Float64()*100, + rand.Float64()*100, + rand.Float64()*100, + ) + tx.Set(fmt.Sprintf("p:%d", i), p, nil) + } + return nil + }) + var keys, values []string + var dists []float64 + var pdist float64 + var i int + db.View(func(tx *Tx) error { + tx.Nearby("points", Point(0, 0, 0, 0), func(key, value string, dist float64) bool { + if i != 0 && dist < pdist { + t.Fatal("out of order") + } + keys = append(keys, key) + values = append(values, value) + dists = append(dists, dist) + pdist = dist + i++ + return true + }) + return nil + }) + if len(keys) != N { + t.Fatalf("expected '%v', got '%v'", N, len(keys)) + } +} + func Example_descKeys() { db, _ := Open(":memory:") db.CreateIndex("name", "*", IndexString) @@ -2511,3 +2551,98 @@ func TestJSONIndex(t *testing.T) { t.Fatalf("expected %v, got %v", expect, strings.Join(keys, ",")) } } + +func TestOnExpiredSync(t *testing.T) { + db := testOpen(t) + defer testClose(db) + + var config Config + if err := db.ReadConfig(&config); err != nil { + t.Fatal(err) + } + hits := make(chan int, 3) + config.OnExpiredSync = func(key, value string, tx *Tx) error { + n, err := strconv.Atoi(value) + if err != nil { + return err + } + defer func() { hits <- n }() + if n >= 2 { + _, err = tx.Delete(key) + if err != ErrNotFound { + return err + } + return nil + } + n++ + _, _, err = tx.Set(key, strconv.Itoa(n), &SetOptions{Expires: true, TTL: time.Millisecond * 100}) + return err + } + if err := db.SetConfig(config); err != nil { + t.Fatal(err) + } + err := db.Update(func(tx *Tx) error { + _, _, err := tx.Set("K", "0", &SetOptions{Expires: true, TTL: time.Millisecond * 100}) + return err + }) + if err != nil { + t.Fail() + } + + done := make(chan struct{}) + go func() { + ticks := time.NewTicker(time.Millisecond * 50) + defer ticks.Stop() + for { + select { + case <-done: + return + case <-ticks.C: + err := db.View(func(tx *Tx) error { + v, err := tx.Get("K", true) + if err != nil { + return err + } + n, err := strconv.Atoi(v) + if err != nil { + return err + } + if n < 0 || n > 2 { + t.Fail() + } + return nil + }) + if err != nil { + t.Fail() + } + } + } + }() + +OUTER1: + for { + select { + case <-time.After(time.Second * 2): + t.Fail() + case v := <-hits: + if v >= 2 { + break OUTER1 + } + } + } + err = db.View(func(tx *Tx) error { + defer close(done) + v, err := tx.Get("K") + if err != nil { + t.Fail() + return err + } + if v != "2" { + t.Fail() + } + return nil + }) + if err != nil { + t.Fail() + } +}