Dep ensure

Gopkg.lock

@@ -250,11 +250,7 @@
   version = "v1.0.2"
 
 [[projects]]
-<<<<<<< HEAD
-  digest = "1:ab5e0d19c706286deed5e6ec63a35ee0f2b92d7b9e97083eb67e5d2d76b4bfdb"
-=======
   digest = "1:cdab3bce90a53a124ac3982719abde77d779e961d9c180e55c23fb74fc62563a"
->>>>>>> master
   name = "github.com/tidwall/geojson"
   packages = [
     ".",
@@ -262,13 +258,8 @@
     "geometry",
   ]
   pruneopts = ""
-<<<<<<< HEAD
-  revision = "f9500c7d3da6ce149bf80530c36b1a784dcd0f2b"
-  version = "v1.1.1"
-=======
   revision = "eaf6e0a55a79c1e879bbbcc879a3176c720d99cd"
   version = "v1.1.3"
->>>>>>> master
 
 [[projects]]
   digest = "1:eade4ea6782f5eed4a6b3138a648f9a332900650804fd206e5daaf99cc5613ea"

vendor/github.com/tidwall/buntdb/README.md

@@ -15,9 +15,6 @@ readers and a single writer. It supports custom indexes and geospatial
 data. It's ideal for projects that need a dependable database and favor
 speed over data size.
 
-The desire to create BuntDB stems from the need for a new embeddable
-database for [Tile38](https://github.com/tidwall/tile38) and [SummitDB](https://github.com/tidwall/summitdb).
-
 Features
 ========
 
@@ -94,8 +91,8 @@ A read-only transaction should be used when you don't need to make changes to th
 
 ```go
 err := db.View(func(tx *buntdb.Tx) error {
-    ...
+	...
-    return nil
+	return nil
 })
 ```
 
@@ -104,8 +101,8 @@ A read/write transaction is used when you need to make changes to your data. The
 
 ```go
 err := db.Update(func(tx *buntdb.Tx) error {
-    ...
+	...
-    return nil
+	return nil
 })
 ```
 
@@ -115,8 +112,8 @@ To set a value you must open a read/write transaction:
 
 ```go
 err := db.Update(func(tx *buntdb.Tx) error {
-    _, _, err := tx.Set("mykey", "myvalue", nil)
+	_, _, err := tx.Set("mykey", "myvalue", nil)
-    return err
+	return err
 })
 ```
 
@@ -125,34 +122,32 @@ To get the value:
 
 ```go
 err := db.View(func(tx *buntdb.Tx) error {
-    val, err := tx.Get("mykey") 
+	val, err := tx.Get("mykey")
-    if err != nil{
+	if err != nil{
-    	return err
+		return err
-    }
+	}
-    fmt.Printf("value is %s\n", val) 
+	fmt.Printf("value is %s\n", val)
-    return nil
+	return nil
 })
 ```
 
-Getting non-existent values will case an `ErrNotFound` error.
+Getting non-existent values will cause an `ErrNotFound` error.
 
 ### Iterating
 All keys/value pairs are ordered in the database by the key. To iterate over the keys:
 
 ```go
 err := db.View(func(tx *buntdb.Tx) error {
-err := tx.Ascend("", func(key, value string) bool{
+	err := tx.Ascend("", func(key, value string) bool {
-    	fmt.Printf("key: %s, value: %s\n", key, value)
+		fmt.Printf("key: %s, value: %s\n", key, value)
-    })
+	})
-    return err
+	return err
 })
 ```
 
 There is also `AscendGreaterOrEqual`, `AscendLessThan`, `AscendRange`, `AscendEqual`, `Descend`, `DescendLessOrEqual`, `DescendGreaterThan`, `DescendRange`, and `DescendEqual`. Please see the [documentation](https://godoc.org/github.com/tidwall/buntdb) for more information on these functions.
 
 
-
-
 ## Custom Indexes
 Initially all data is stored in a single [B-tree](https://en.wikipedia.org/wiki/B-tree) with each item having one key and one value. All of these items are ordered by the key. This is great for quickly getting a value from a key or [iterating](#iterating) over the keys. Feel free to peruse the [B-tree implementation](https://github.com/tidwall/btree).
 
@@ -170,7 +165,7 @@ Now you can add various names:
 
 ```go
 db.Update(func(tx *buntdb.Tx) error {
-    tx.Set("user:0:name", "tom", nil)
+	tx.Set("user:0:name", "tom", nil)
 	tx.Set("user:1:name", "Randi", nil)
 	tx.Set("user:2:name", "jane", nil)
 	tx.Set("user:4:name", "Janet", nil)
@@ -186,10 +181,10 @@ Finally you can iterate over the index:
 ```go
 db.View(func(tx *buntdb.Tx) error {
 	tx.Ascend("names", func(key, val string) bool {
-		fmt.Printf(buf, "%s %s\n", key, val)
+	fmt.Printf(buf, "%s %s\n", key, val)
 		return true
 	})
-    return nil
+	return nil
 })
 ```
 The output should be:
@@ -226,7 +221,7 @@ And then add values:
 
 ```go
 db.Update(func(tx *buntdb.Tx) error {
-    tx.Set("user:0:age", "35", nil)
+	tx.Set("user:0:age", "35", nil)
 	tx.Set("user:1:age", "49", nil)
 	tx.Set("user:2:age", "13", nil)
 	tx.Set("user:4:age", "63", nil)
@@ -240,22 +235,22 @@ db.Update(func(tx *buntdb.Tx) error {
 ```go
 db.View(func(tx *buntdb.Tx) error {
 	tx.Ascend("ages", func(key, val string) bool {
-		fmt.Printf(buf, "%s %s\n", key, val)
+	fmt.Printf(buf, "%s %s\n", key, val)
 		return true
 	})
-    return nil
+	return nil
 })
 ```
 
 The output should be:
 ```
-user:6:name 3
+user:6:age 3
-user:5:name 8
+user:5:age 8
-user:2:name 13
+user:2:age 13
-user:7:name 16
+user:7:age 16
-user:0:name 35
+user:0:age 35
-user:1:name 49
+user:1:age 49
-user:4:name 63
+user:4:age 63
 ```
 
 ## Spatial Indexes
@@ -273,7 +268,7 @@ To add some lon,lat points to the `fleet` index:
 
 ```go
 db.Update(func(tx *buntdb.Tx) error {
-    tx.Set("fleet:0:pos", "[-115.567 33.532]", nil)
+	tx.Set("fleet:0:pos", "[-115.567 33.532]", nil)
 	tx.Set("fleet:1:pos", "[-116.671 35.735]", nil)
 	tx.Set("fleet:2:pos", "[-113.902 31.234]", nil)
 	return nil
@@ -284,16 +279,30 @@ And then you can run the `Intersects` function on the index:
 
 ```go
 db.View(func(tx *buntdb.Tx) error {
-    tx.Intersects("fleet", "[-117 30],[-112 36]", func(key, val string) bool {
+	tx.Intersects("fleet", "[-117 30],[-112 36]", func(key, val string) bool {
-    	...
+		...
-    	return true
+		return true
-    })
+	})
-    return nil
+	return nil
 })
 ```
 
 This will get all three positions.
 
+### k-Nearest Neighbors
+
+Use the `Nearby` function to get all the positions in order of nearest to farthest :
+
+```go
+db.View(func(tx *buntdb.Tx) error {
+	tx.Nearby("fleet", "[-113 33]", func(key, val string, dist float64) bool {
+		...
+		return true
+	})
+	return nil
+})
+```
+
 ### Spatial bracket syntax
 
 The bracket syntax `[-117 30],[-112 36]` is unique to BuntDB, and it's how the built-in rectangles are processed. But, you are not limited to this syntax. Whatever Rect function you choose to use during `CreateSpatialIndex` will be used to process the parameter, in this case it's `IndexRect`.
@@ -307,10 +316,10 @@ The bracket syntax `[-117 30],[-112 36]` is unique to BuntDB, and it's how the b
 - **2D point:** `[10 15]`
 *XY: "10x15"*
 
-- **LatLon point:** `[-112.2693 33.5123]`  
+- **LonLat point:** `[-112.2693 33.5123]`
 *LatLon: "33.5123 -112.2693"*
 
-- **LatLon bounding box:** `[-112.26 33.51],[-112.18 33.67]`  
+- **LonLat bounding box:** `[-112.26 33.51],[-112.18 33.67]`
 *Min LatLon: "33.51 -112.26", Max LatLon: "33.67 -112.18"*
 
 **Notice:** The longitude is the Y axis and is on the left, and latitude is the X axis and is on the right.
@@ -330,8 +339,8 @@ You can then do a search for all points with `M` between 2-4 by calling `Interse
 
 ```go
 tx.Intersects("points", "[-inf -inf 2],[+inf +inf 4]", func(key, val string) bool {
-    println(val)
+	println(val)
-    return true
+	return true
 })
 ```
 
@@ -449,8 +458,8 @@ Any index can be put in descending order by wrapping it's less function with `bu
 
 ```go
 db.CreateIndex("last_name_age", "*",
-	buntdb.IndexJSON("name.last"), 
+buntdb.IndexJSON("name.last"),
-	buntdb.Desc(buntdb.IndexJSON("age")))
+buntdb.Desc(buntdb.IndexJSON("age")))
 ```
 
 This will create a multi value index where the last name is ascending and the age is descending.
@@ -492,13 +501,32 @@ Items can be automatically evicted by using the `SetOptions` object in the `Set`
 
 ```go
 db.Update(func(tx *buntdb.Tx) error {
-    tx.Set("mykey", "myval", &buntdb.SetOptions{Expires:true, TTL:time.Second})
+	tx.Set("mykey", "myval", &buntdb.SetOptions{Expires:true, TTL:time.Second})
 	return nil
 })
 ```
 
 Now `mykey` will automatically be deleted after one second. You can remove the TTL by setting the value again with the same key/value, but with the options parameter set to nil.
 
+## Delete while iterating
+BuntDB does not currently support deleting a key while in the process of iterating.
+As a workaround you'll need to delete keys following the completion of the iterator.
+
+```go
+var delkeys []string
+tx.AscendKeys("object:*", func(k, v string) bool {
+	if someCondition(k) == true {
+		delkeys = append(delkeys, k)
+	}
+	return true // continue
+})
+for _, k := range delkeys {
+	if _, err = tx.Delete(k); err != nil {
+		return err
+	}
+}
+```
+
 ## Append-only File
 
 BuntDB uses an AOF (append-only file) which is a log of all database changes that occur from operations like `Set()` and `Delete()`.
@@ -546,10 +574,10 @@ To update the configuration you should call `ReadConfig` followed by `SetConfig`
 
 var config buntdb.Config
 if err := db.ReadConfig(&config); err != nil{
-    log.Fatal(err)
+	log.Fatal(err)
 }
 if err := db.WriteConfig(config); err != nil{
-    log.Fatal(err)
+	log.Fatal(err)
 }
 ```
 

vendor/github.com/tidwall/buntdb/buntdb.go

@@ -121,6 +121,12 @@ type Config struct {
 	// OnExpired is used to custom handle the deletion option when a key
 	// has been expired.
 	OnExpired func(keys []string)
+
+	// OnExpiredSync will be called inside the same transaction that is performing
+	// the deletion of expired items. If OnExpired is present then this callback
+	// will not be called. If this callback is present, then the deletion of the
+	// timeed-out item is the explicit responsibility of this callback.
+	OnExpiredSync func(key, value string, tx *Tx) error
 }
 
 // exctx is a simple b-tree context for ordering by expiration.
@@ -544,9 +550,13 @@ func (db *DB) backgroundManager() {
 		// Open a standard view. This will take a full lock of the
 		// database thus allowing for access to anything we need.
 		var onExpired func([]string)
-		var expired []string
+		var expired []*dbItem
+		var onExpiredSync func(key, value string, tx *Tx) error
 		err := db.Update(func(tx *Tx) error {
 			onExpired = db.config.OnExpired
+			if onExpired == nil {
+				onExpiredSync = db.config.OnExpiredSync
+			}
 			if db.persist && !db.config.AutoShrinkDisabled {
 				pos, err := db.file.Seek(0, 1)
 				if err != nil {
@@ -562,12 +572,12 @@ func (db *DB) backgroundManager() {
 			db.exps.AscendLessThan(&dbItem{
 				opts: &dbItemOpts{ex: true, exat: time.Now()},
 			}, func(item btree.Item) bool {
-				expired = append(expired, item.(*dbItem).key)
+				expired = append(expired, item.(*dbItem))
 				return true
 			})
-			if onExpired == nil {
+			if onExpired == nil && onExpiredSync == nil {
-				for _, key := range expired {
+				for _, itm := range expired {
-					if _, err := tx.Delete(key); err != nil {
+					if _, err := tx.Delete(itm.key); err != nil {
 						// it's ok to get a "not found" because the
 						// 'Delete' method reports "not found" for
 						// expired items.
@@ -576,6 +586,12 @@ func (db *DB) backgroundManager() {
 						}
 					}
 				}
+			} else if onExpiredSync != nil {
+				for _, itm := range expired {
+					if err := onExpiredSync(itm.key, itm.val, tx); err != nil {
+						return err
+					}
+				}
 			}
 			return nil
 		})
@@ -585,7 +601,11 @@ func (db *DB) backgroundManager() {
 
 		// send expired event, if needed
 		if onExpired != nil && len(expired) > 0 {
-			onExpired(expired)
+			keys := make([]string, 0, 32)
+			for _, itm := range expired {
+				keys = append(keys, itm.key)
+			}
+			onExpired(keys)
 		}
 
 		// execute a disk sync, if needed
@@ -1399,13 +1419,18 @@ func (tx *Tx) Set(key, value string, opts *SetOptions) (previousValue string,
 }
 
 // Get returns a value for a key. If the item does not exist or if the item
-// has expired then ErrNotFound is returned.
+// has expired then ErrNotFound is returned. If ignoreExpired is true, then
-func (tx *Tx) Get(key string) (val string, err error) {
+// the found value will be returned even if it is expired.
+func (tx *Tx) Get(key string, ignoreExpired ...bool) (val string, err error) {
 	if tx.db == nil {
 		return "", ErrTxClosed
 	}
+	var ignore bool
+	if len(ignoreExpired) != 0 {
+		ignore = ignoreExpired[0]
+	}
 	item := tx.db.get(key)
-	if item == nil || item.expired() {
+	if item == nil || (item.expired() && !ignore) {
 		// The item does not exists or has expired. Let's assume that
 		// the caller is only interested in items that have not expired.
 		return "", ErrNotFound
@@ -1775,7 +1800,7 @@ func (tx *Tx) DescendEqual(index, pivot string,
 	})
 }
 
-// rect is used by Intersects
+// rect is used by Intersects and Nearby
 type rect struct {
 	min, max []float64
 }
@@ -1784,6 +1809,48 @@ func (r *rect) Rect(ctx interface{}) (min, max []float64) {
 	return r.min, r.max
 }
 
+// Nearby searches for rectangle items that are nearby a target rect.
+// All items belonging to the specified index will be returned in order of
+// nearest to farthest.
+// The specified index must have been created by AddIndex() and the target
+// is represented by the rect string. This string will be processed by the
+// same bounds function that was passed to the CreateSpatialIndex() function.
+// An invalid index will return an error.
+// The dist param is the distance of the bounding boxes. In the case of
+// simple 2D points, it's the distance of the two 2D points squared.
+func (tx *Tx) Nearby(index, bounds string,
+	iterator func(key, value string, dist float64) bool) error {
+	if tx.db == nil {
+		return ErrTxClosed
+	}
+	if index == "" {
+		// cannot search on keys tree. just return nil.
+		return nil
+	}
+	// // wrap a rtree specific iterator around the user-defined iterator.
+	iter := func(item rtree.Item, dist float64) bool {
+		dbi := item.(*dbItem)
+		return iterator(dbi.key, dbi.val, dist)
+	}
+	idx := tx.db.idxs[index]
+	if idx == nil {
+		// index was not found. return error
+		return ErrNotFound
+	}
+	if idx.rtr == nil {
+		// not an r-tree index. just return nil
+		return nil
+	}
+	// execute the nearby search
+	var min, max []float64
+	if idx.rect != nil {
+		min, max = idx.rect(bounds)
+	}
+	// set the center param to false, which uses the box dist calc.
+	idx.rtr.KNN(&rect{min, max}, false, iter)
+	return nil
+}
+
 // Intersects searches for rectangle items that intersect a target rect.
 // The specified index must have been created by AddIndex() and the target
 // is represented by the rect string. This string will be processed by the

vendor/github.com/tidwall/buntdb/buntdb_test.go

@@ -1021,6 +1021,46 @@ func TestVariousTx(t *testing.T) {
 	}
 }
 
+func TestNearby(t *testing.T) {
+	rand.Seed(time.Now().UnixNano())
+	N := 100000
+	db, _ := Open(":memory:")
+	db.CreateSpatialIndex("points", "*", IndexRect)
+	db.Update(func(tx *Tx) error {
+		for i := 0; i < N; i++ {
+			p := Point(
+				rand.Float64()*100,
+				rand.Float64()*100,
+				rand.Float64()*100,
+				rand.Float64()*100,
+			)
+			tx.Set(fmt.Sprintf("p:%d", i), p, nil)
+		}
+		return nil
+	})
+	var keys, values []string
+	var dists []float64
+	var pdist float64
+	var i int
+	db.View(func(tx *Tx) error {
+		tx.Nearby("points", Point(0, 0, 0, 0), func(key, value string, dist float64) bool {
+			if i != 0 && dist < pdist {
+				t.Fatal("out of order")
+			}
+			keys = append(keys, key)
+			values = append(values, value)
+			dists = append(dists, dist)
+			pdist = dist
+			i++
+			return true
+		})
+		return nil
+	})
+	if len(keys) != N {
+		t.Fatalf("expected '%v', got '%v'", N, len(keys))
+	}
+}
+
 func Example_descKeys() {
 	db, _ := Open(":memory:")
 	db.CreateIndex("name", "*", IndexString)
@@ -2511,3 +2551,98 @@ func TestJSONIndex(t *testing.T) {
 		t.Fatalf("expected %v, got %v", expect, strings.Join(keys, ","))
 	}
 }
+
+func TestOnExpiredSync(t *testing.T) {
+	db := testOpen(t)
+	defer testClose(db)
+
+	var config Config
+	if err := db.ReadConfig(&config); err != nil {
+		t.Fatal(err)
+	}
+	hits := make(chan int, 3)
+	config.OnExpiredSync = func(key, value string, tx *Tx) error {
+		n, err := strconv.Atoi(value)
+		if err != nil {
+			return err
+		}
+		defer func() { hits <- n }()
+		if n >= 2 {
+			_, err = tx.Delete(key)
+			if err != ErrNotFound {
+				return err
+			}
+			return nil
+		}
+		n++
+		_, _, err = tx.Set(key, strconv.Itoa(n), &SetOptions{Expires: true, TTL: time.Millisecond * 100})
+		return err
+	}
+	if err := db.SetConfig(config); err != nil {
+		t.Fatal(err)
+	}
+	err := db.Update(func(tx *Tx) error {
+		_, _, err := tx.Set("K", "0", &SetOptions{Expires: true, TTL: time.Millisecond * 100})
+		return err
+	})
+	if err != nil {
+		t.Fail()
+	}
+
+	done := make(chan struct{})
+	go func() {
+		ticks := time.NewTicker(time.Millisecond * 50)
+		defer ticks.Stop()
+		for {
+			select {
+			case <-done:
+				return
+			case <-ticks.C:
+				err := db.View(func(tx *Tx) error {
+					v, err := tx.Get("K", true)
+					if err != nil {
+						return err
+					}
+					n, err := strconv.Atoi(v)
+					if err != nil {
+						return err
+					}
+					if n < 0 || n > 2 {
+						t.Fail()
+					}
+					return nil
+				})
+				if err != nil {
+					t.Fail()
+				}
+			}
+		}
+	}()
+
+OUTER1:
+	for {
+		select {
+		case <-time.After(time.Second * 2):
+			t.Fail()
+		case v := <-hits:
+			if v >= 2 {
+				break OUTER1
+			}
+		}
+	}
+	err = db.View(func(tx *Tx) error {
+		defer close(done)
+		v, err := tx.Get("K")
+		if err != nil {
+			t.Fail()
+			return err
+		}
+		if v != "2" {
+			t.Fail()
+		}
+		return nil
+	})
+	if err != nil {
+		t.Fail()
+	}
+}