From fd20190bff26b9b0e9acf30f3df7dcb977545ee6 Mon Sep 17 00:00:00 2001 From: Steven Wolfe Date: Fri, 22 Feb 2019 15:58:13 -0700 Subject: [PATCH 1/5] Verify hook names match for processing --- internal/server/hooks.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/internal/server/hooks.go b/internal/server/hooks.go index 0314ea8e..cd37843e 100644 --- a/internal/server/hooks.go +++ b/internal/server/hooks.go @@ -10,6 +10,7 @@ import ( "time" "github.com/tidwall/buntdb" + "github.com/tidwall/gjson" "github.com/tidwall/resp" "github.com/tidwall/tile38/internal/endpoint" "github.com/tidwall/tile38/internal/glob" @@ -597,8 +598,11 @@ func (h *Hook) proc() (ok bool) { err := tx.AscendGreaterOrEqual("hooks", h.query, func(key, val string) bool { if strings.HasPrefix(key, hookLogPrefix) { - keys = append(keys, key) - vals = append(vals, val) + // Verify this hooks name matches the one in the notif + if h.Name == gjson.Get(val, "hook").String() { + keys = append(keys, key) + vals = append(vals, val) + } } return true }, From c9904fc3c78f47dafb7a6c5c70a8f532c6e25b19 Mon Sep 17 00:00:00 2001 From: Josh Baker Date: Sun, 24 Feb 2019 12:41:48 -0700 Subject: [PATCH 2/5] Update https links --- README.md | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 2fcc7eb1..8e27681a 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@

- Tile38

@@ -13,22 +13,22 @@ Tile38 is an open source (MIT licensed), in-memory geolocation data store, spatial index, and realtime geofence. It supports a variety of object types including lat/lon points, bounding boxes, XYZ tiles, Geohashes, and GeoJSON.

-This README is quick start document. You can find detailed documentation at http://tile38.com.

+This README is quick start document. You can find detailed documentation at https://tile38.com.

Nearby Within Intersects -Geofencing -Roaming Geofences +Geofencing +Roaming Geofences

## Features - Spatial index with [search](#searching) methods such as Nearby, Within, and Intersects. -- Realtime [geofencing](#geofencing) through [webhooks](http://tile38.com/commands/sethook) or [pub/sub channels](#pubsub-channels). +- Realtime [geofencing](#geofencing) through [webhooks](https://tile38.com/commands/sethook) or [pub/sub channels](#pubsub-channels). - Object types of [lat/lon](#latlon-point), [bbox](#bounding-box), [Geohash](#geohash), [GeoJSON](#geojson), [QuadKey](#quadkey), and [XYZ tile](#xyz-tile). - Support for lots of [Clients Libraries](#client-libraries) written in many different languages. -- Variety of protocols, including [http](#http) (curl), [websockets](#websockets), [telnet](#telnet), and the [Redis RESP](http://redis.io/topics/protocol). -- Server responses are [RESP](http://redis.io/topics/protocol) or [JSON](http://www.json.org). +- Variety of protocols, including [http](#http) (curl), [websockets](#websockets), [telnet](#telnet), and the [Redis RESP](https://redis.io/topics/protocol). +- Server responses are [RESP](https://redis.io/topics/protocol) or [JSON](https://www.json.org). - Full [command line interface](#cli). - Leader / follower [replication](#replication). - In-memory database that persists on disk. @@ -57,7 +57,7 @@ Visit the [Tile38 hub page](https://hub.docker.com/r/tile38/tile38/) for more in ### Homebrew (macOS) -Install Tile38 using [Homebrew](http://brew.sh/) +Install Tile38 using [Homebrew](https://brew.sh/) ```sh brew install tile38 @@ -114,7 +114,7 @@ $ ./tile38-cli > drop fleet # removes all ``` -Tile38 has a ton of [great commands](http://tile38.com/commands). +Tile38 has a ton of [great commands](https://tile38.com/commands). ## Fields Fields are extra data that belongs to an object. A field is always a double precision floating point. There is no limit to the number of fields that an object can have. @@ -183,7 +183,7 @@ You can choose a value between 1 and 8. The value 1 will result in no more than Geofence animation A geofence is a virtual boundary that can detect when an object enters or exits the area. This boundary can be a radius, bounding box, or a polygon. Tile38 can turn any standard search into a geofence monitor by adding the FENCE keyword to the search. -*Tile38 also allows for [Webhooks](http://tile38.com/commands/sethook) to be assigned to Geofences.* +*Tile38 also allows for [Webhooks](https://tile38.com/commands/sethook) to be assigned to Geofences.*
@@ -260,7 +260,7 @@ set fleet truck1 hash 9tbnthxzr # this would be equivlent to 'point 33.5123 -112 ``` #### GeoJSON -[GeoJSON](http://geojson.org/) is an industry standard format for representing a variety of object types including a point, multipoint, linestring, multilinestring, polygon, multipolygon, geometrycollection, feature, and featurecollection. +[GeoJSON](https://tools.ietf.org/html/rfc7946) is an industry standard format for representing a variety of object types including a point, multipoint, linestring, multilinestring, polygon, multipolygon, geometrycollection, feature, and featurecollection. * All ignored members will not persist. @@ -272,7 +272,7 @@ set city tempe object {"type":"Polygon","coordinates":[[[0,0],[10,10],[10,0],[0, #### XYZ Tile An XYZ tile is rectangle bounding area on earth that is represented by an X, Y coordinate and a Z (zoom) level. -Check out [maptiler.org](http://www.maptiler.org/google-maps-coordinates-tile-bounds-projection/) for an interactive example. +Check out [maptiler.org](https://www.maptiler.org/google-maps-coordinates-tile-bounds-projection/) for an interactive example. #### QuadKey A QuadKey used the same coordinate system as an XYZ tile except that the string representation is a string characters composed of 0, 1, 2, or 3. For a detailed explanation checkout [The Bing Maps Tile System](https://msdn.microsoft.com/en-us/library/bb259689.aspx). @@ -297,7 +297,7 @@ curl localhost:9851/set+fleet+truck3+point+33.4762+-112.10923 Websockets can be used when you need to Geofence and keep the connection alive. It works just like the HTTP example above, with the exception that the connection stays alive and the data is sent from the server as text websocket messages. #### Telnet -There is the option to use a plain telnet connection. The default output through telnet is [RESP](http://redis.io/topics/protocol). +There is the option to use a plain telnet connection. The default output through telnet is [RESP](https://redis.io/topics/protocol). ``` telnet localhost 9851 @@ -306,14 +306,14 @@ set fleet truck3 point 33.4762 -112.10923 ``` -The server will respond in [JSON](http://json.org) or [RESP](http://redis.io/topics/protocol) depending on which protocol is used when initiating the first command. +The server will respond in [JSON](https://json.org) or [RESP](https://redis.io/topics/protocol) depending on which protocol is used when initiating the first command. - HTTP and Websockets use JSON. - Telnet and RESP clients use RESP. ## Client Libraries -Tile38 uses the [Redis RESP](http://redis.io/topics/protocol) protocol natively. Therefore most clients that support basic Redis commands will in turn support Tile38. Below are a few of the popular clients. +Tile38 uses the [Redis RESP](https://redis.io/topics/protocol) protocol natively. Therefore most clients that support basic Redis commands will in turn support Tile38. Below are a few of the popular clients. - C: [hiredis](https://github.com/redis/hiredis) - C#: [StackExchange.Redis](https://github.com/StackExchange/StackExchange.Redis) @@ -339,7 +339,7 @@ Tile38 uses the [Redis RESP](http://redis.io/topics/protocol) protocol natively. ## Contact -Josh Baker [@tidwall](http://twitter.com/tidwall) +Josh Baker [@tidwall](https://twitter.com/tidwall) ## License From 714cca6d6cabdadcb8d01f84ce9e3eccd5e8ad1a Mon Sep 17 00:00:00 2001 From: tidwall Date: Mon, 25 Feb 2019 15:39:58 -0700 Subject: [PATCH 3/5] 1.16.0 --- CHANGELOG.md | 13 +++++++++++++ README.md | 2 +- build.sh | 2 +- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a042f0fa..5f11827c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,19 @@ All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](http://semver.org/). +## [1.16.0] - 2019-02-25 +### Fixes +- #415: Fix overlapping geofences sending notifcation to wrong endpoint. (@belek, @s32x) +- #412: Allow SERVER command for Lua scripts (@1995parham) +- #410: Allow slashes in MQTT Topics (@pstuifzand) +- #409: Fixed bug in polygon clipping. (@rshura) +- 30f903b: Require properties member for geojson features. (@rshura) + +### Added +- #409: Added TEST command for executing WITHIN and INTERSECTS on two objects. (@rshura) +- #407: Allow 201 & 202 status code on webhooks. (@s32x) +- #404: Adding more replication data to INFO response. (@s32x) + ## [1.15.0] - 2019-01-16 ### Fixes - #403: JSON Output for INFO and CLIENT (@s32x) diff --git a/README.md b/README.md index 8e27681a..1180edd4 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@

Slack Channel -Version +Version Build Status Docker Ready

diff --git a/build.sh b/build.sh index 573f5690..dedf919b 100755 --- a/build.sh +++ b/build.sh @@ -4,7 +4,7 @@ set -e cd $(dirname "${BASH_SOURCE[0]}") OD="$(pwd)" -VERSION=1.15.0 +VERSION=1.16.0 PROTECTED_MODE="no" # Hardcode some values to the core package From fc3e8b4359dfd929d077101ac3db9e94ce6028b8 Mon Sep 17 00:00:00 2001 From: tidwall Date: Fri, 1 Mar 2019 06:55:26 -0700 Subject: [PATCH 4/5] Fix nearby with match query invalid results closes #421 --- internal/server/scanner.go | 2 +- internal/server/search.go | 2 +- tests/README.md | 33 +++++++++++++++++++ tests/keys_search_test.go | 67 ++++++++++++++++++++++++++++++++++++-- tests/mock_test.go | 22 ++++++++++--- tests/tests.go | 1 - 6 files changed, 117 insertions(+), 10 deletions(-) create mode 100644 tests/README.md delete mode 100644 tests/tests.go diff --git a/internal/server/scanner.go b/internal/server/scanner.go index c1580440..e3cae7b1 100644 --- a/internal/server/scanner.go +++ b/internal/server/scanner.go @@ -342,7 +342,7 @@ func (sw *scanWriter) testObject(id string, o geojson.Object, fields []float64, if !ignoreGlobMatch { match, kg := sw.globMatch(id, o) if !match { - return true, kg, fieldVals + return false, kg, fieldVals } } nf, ok := sw.fieldMatch(fields, o) diff --git a/internal/server/search.go b/internal/server/search.go index 9315d77a..9cc12615 100644 --- a/internal/server/search.go +++ b/internal/server/search.go @@ -413,7 +413,7 @@ func (server *Server) nearestNeighbors( if server.hasExpired(s.key, id) { return true } - ok, keepGoing, _ := sw.testObject(id, o, fields, true) + ok, keepGoing, _ := sw.testObject(id, o, fields, false) if !ok { return true } diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 00000000..7640e54a --- /dev/null +++ b/tests/README.md @@ -0,0 +1,33 @@ +## Tile38 Integation Testing + +- Uses Redis protocol +- The Tile38 data is flushed before every `DoBatch` + +A basic test operation looks something like: + +```go +func keys_SET_test(mc *mockServer) error { + return mc.DoBatch([][]interface{}{ + {"SET", "fleet", "truck1", "POINT", 33.0001, -112.0001}, {"OK"}, + {"GET", "fleet", "truck1", "POINT"}, {"[33.0001 -112.0001]"}, + } +} +``` + +Using a custom function: + +```go +func keys_MATCH_test(mc *mockServer) error { + return mc.DoBatch([][]interface{}{ + {"SET", "fleet", "truck1", "POINT", 33.0001, -112.0001}, { + func(v interface{}) (resp, expect interface{}) { + // v is the value as strings or slices of strings + // test will pass as long as `resp` and `expect` are the same. + return v, "OK" + }, + }, + } +} +``` + + diff --git a/tests/keys_search_test.go b/tests/keys_search_test.go index f0767656..2552f387 100644 --- a/tests/keys_search_test.go +++ b/tests/keys_search_test.go @@ -1,6 +1,8 @@ package tests import ( + "fmt" + "sort" "testing" ) @@ -15,6 +17,7 @@ func subTestSearch(t *testing.T, mc *mockServer) { runStep(t, mc, "INTERSECTS_CURSOR", keys_INTERSECTS_CURSOR_test) runStep(t, mc, "SCAN_CURSOR", keys_SCAN_CURSOR_test) runStep(t, mc, "SEARCH_CURSOR", keys_SEARCH_CURSOR_test) + runStep(t, mc, "MATCH", keys_MATCH_test) } func keys_KNN_test(mc *mockServer) error { @@ -34,7 +37,7 @@ func keys_KNN_test(mc *mockServer) error { func keys_KNN_cursor_test(mc *mockServer) error { return mc.DoBatch([][]interface{}{ {"SET", "mykey", "1", "FIELD", "foo", 5.5, "POINT", 5, 5}, {"OK"}, - {"SET", "mykey", "2", "FIELD", "foo", 19.19, "POINT",19, 19}, {"OK"}, + {"SET", "mykey", "2", "FIELD", "foo", 19.19, "POINT", 19, 19}, {"OK"}, {"SET", "mykey", "3", "FIELD", "foo", 12.19, "POINT", 12, 19}, {"OK"}, {"SET", "mykey", "4", "FIELD", "foo", -5.5, "POINT", -5, 5}, {"OK"}, {"SET", "mykey", "5", "FIELD", "foo", 13.21, "POINT", 33, 21}, {"OK"}, @@ -118,7 +121,7 @@ func keys_WITHIN_CURSOR_test(mc *mockServer) error { {"SET", "mykey", "point2", "FIELD", "foo", 2, "POINT", 37.7335, -122.44121}, {"OK"}, {"SET", "mykey", "line3", "FIELD", "foo", 3, "OBJECT", `{"type":"LineString","coordinates":[[-122.4408378,37.7341129],[-122.4408378,37.733]]}`}, {"OK"}, {"SET", "mykey", "poly4", "FIELD", "foo", 4, "OBJECT", `{"type":"Polygon","coordinates":[[[-122.4408378,37.7341129],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.7341129],[-122.4408378,37.7341129]]]}`}, {"OK"}, - {"SET", "mykey", "multipoly5","FIELD", "foo", 5, "OBJECT", `{"type":"MultiPolygon","coordinates":[[[[-122.4408378,37.7341129],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.7341129],[-122.4408378,37.7341129]]],[[[-122.44091033935547,37.731981251280985],[-122.43994474411011,37.731981251280985],[-122.43994474411011,37.73254976045042],[-122.44091033935547,37.73254976045042],[-122.44091033935547,37.731981251280985]]]]}`}, {"OK"}, + {"SET", "mykey", "multipoly5", "FIELD", "foo", 5, "OBJECT", `{"type":"MultiPolygon","coordinates":[[[[-122.4408378,37.7341129],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.7341129],[-122.4408378,37.7341129]]],[[[-122.44091033935547,37.731981251280985],[-122.43994474411011,37.731981251280985],[-122.43994474411011,37.73254976045042],[-122.44091033935547,37.73254976045042],[-122.44091033935547,37.731981251280985]]]]}`}, {"OK"}, {"SET", "mykey", "point6", "FIELD", "foo", 6, "POINT", -5, 5}, {"OK"}, {"SET", "mykey", "point7", "FIELD", "foo", 7, "POINT", 33, 21}, {"OK"}, {"SET", "mykey", "poly8", "FIELD", "foo", 8, "OBJECT", `{"type":"Polygon","coordinates":[[[-122.4408378,37.7341129],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.7341129],[-122.4408378,37.7341129]],[[-122.44060993194579,37.73345766902749],[-122.44044363498686,37.73345766902749],[-122.44044363498686,37.73355524732416],[-122.44060993194579,37.73355524732416],[-122.44060993194579,37.73345766902749]],[[-122.44060724973677,37.7336888869566],[-122.4402102828026,37.7336888869566],[-122.4402102828026,37.7339752567853],[-122.44060724973677,37.7339752567853],[-122.44060724973677,37.7336888869566]]]}`}, {"OK"}, @@ -302,3 +305,63 @@ func keys_SEARCH_CURSOR_test(mc *mockServer) error { {"SEARCH", "mykey", "LIMIT", 3, "DESC", "IDS"}, {"[3 [id9 id8 id7]]"}, }) } + +func keys_MATCH_test(mc *mockServer) error { + return mc.DoBatch([][]interface{}{ + {"SET", "fleet", "truck1", "POINT", "33.0001", "-112.0001"}, {"OK"}, + {"SET", "fleet", "truck2", "POINT", "33.0002", "-112.0002"}, {"OK"}, + {"SET", "fleet", "luck1", "POINT", "33.0003", "-112.0003"}, {"OK"}, + {"SET", "fleet", "luck2", "POINT", "33.0004", "-112.0004"}, {"OK"}, + + {"SCAN", "fleet", "IDS"}, {"[0 [luck1 luck2 truck1 truck2]]"}, + {"SCAN", "fleet", "MATCH", "*", "IDS"}, {"[0 [luck1 luck2 truck1 truck2]]"}, + {"SCAN", "fleet", "MATCH", "truck*", "IDS"}, {"[0 [truck1 truck2]]"}, + {"SCAN", "fleet", "MATCH", "luck*", "IDS"}, {"[0 [luck1 luck2]]"}, + {"SCAN", "fleet", "MATCH", "*2", "IDS"}, {"[0 [luck2 truck2]]"}, + {"SCAN", "fleet", "MATCH", "*2*", "IDS"}, {"[0 [luck2 truck2]]"}, + {"SCAN", "fleet", "MATCH", "*u*", "IDS"}, {"[0 [luck1 luck2 truck1 truck2]]"}, + + {"NEARBY", "fleet", "IDS", "POINT", 33.00005, -112.00005, 100000}, { + match("[0 [luck1 luck2 truck1 truck2]]"), + }, + {"NEARBY", "fleet", "MATCH", "*", "IDS", "POINT", 33.00005, -112.00005, 100000}, { + match("[0 [luck1 luck2 truck1 truck2]]"), + }, + {"NEARBY", "fleet", "MATCH", "t*", "IDS", "POINT", 33.00005, -112.00005, 100000}, { + match("[0 [truck1 truck2]]"), + }, + {"NEARBY", "fleet", "MATCH", "t*2", "IDS", "POINT", 33.00005, -112.00005, 100000}, { + match("[0 [truck2]]"), + }, + {"NEARBY", "fleet", "MATCH", "*2", "IDS", "POINT", 33.00005, -112.00005, 100000}, { + match("[0 [luck2 truck2]]"), + }, + + {"INTERSECTS", "fleet", "IDS", "BOUNDS", 33, -113, 34, -112}, { + match("[0 [luck1 luck2 truck1 truck2]]"), + }, + {"INTERSECTS", "fleet", "MATCH", "*", "IDS", "BOUNDS", 33, -113, 34, -112}, { + match("[0 [luck1 luck2 truck1 truck2]]"), + }, + {"INTERSECTS", "fleet", "MATCH", "t*", "IDS", "BOUNDS", 33, -113, 34, -112}, { + match("[0 [truck1 truck2]]"), + }, + {"INTERSECTS", "fleet", "MATCH", "t*2", "IDS", "BOUNDS", 33, -113, 34, -112}, { + match("[0 [truck2]]"), + }, + {"INTERSECTS", "fleet", "MATCH", "*2", "IDS", "BOUNDS", 33, -113, 34, -112}, { + match("[0 [luck2 truck2]]"), + }, + }) +} + +// match sorts the response and compares to the expected input +func match(expectIn string) func(org, v interface{}) (resp, expect interface{}) { + return func(v, org interface{}) (resp, expect interface{}) { + sort.Slice(org.([]interface{})[1], func(i, j int) bool { + return org.([]interface{})[1].([]interface{})[i].(string) < + org.([]interface{})[1].([]interface{})[j].(string) + }) + return fmt.Sprintf("%v", org), expectIn + } +} diff --git a/tests/mock_test.go b/tests/mock_test.go index 53929d78..f4f5f2d6 100644 --- a/tests/mock_test.go +++ b/tests/mock_test.go @@ -226,11 +226,23 @@ func (mc *mockServer) DoExpect(expect interface{}, commandName string, args ...i resp = string([]byte(b)) } } - if fn, ok := expect.(func(v, org interface{}) (resp, expect interface{})); ok { - resp, expect = fn(resp, oresp) - } - if fn, ok := expect.(func(v interface{}) (resp, expect interface{})); ok { - resp, expect = fn(resp) + err = func() (err error) { + defer func() { + v := recover() + if v != nil { + err = fmt.Errorf("panic '%v'", v) + } + }() + if fn, ok := expect.(func(v, org interface{}) (resp, expect interface{})); ok { + resp, expect = fn(resp, oresp) + } + if fn, ok := expect.(func(v interface{}) (resp, expect interface{})); ok { + resp, expect = fn(resp) + } + return nil + }() + if err != nil { + return err } if fmt.Sprintf("%v", resp) != fmt.Sprintf("%v", expect) { return fmt.Errorf("expected '%v', got '%v'", expect, resp) diff --git a/tests/tests.go b/tests/tests.go deleted file mode 100644 index ca8701d2..00000000 --- a/tests/tests.go +++ /dev/null @@ -1 +0,0 @@ -package tests From 2bf69c5a051814167ff1565ac3a303554235a2a9 Mon Sep 17 00:00:00 2001 From: tidwall Date: Fri, 1 Mar 2019 06:56:29 -0700 Subject: [PATCH 5/5] 1.16.1 --- CHANGELOG.md | 4 ++++ README.md | 2 +- build.sh | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5f11827c..b2116d17 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](http://semver.org/). +## [1.16.1] - 2019-03-01 +### Fixes +- #421: Nearby with MATCH is returning invalid results (@nithinkota) + ## [1.16.0] - 2019-02-25 ### Fixes - #415: Fix overlapping geofences sending notifcation to wrong endpoint. (@belek, @s32x) diff --git a/README.md b/README.md index 1180edd4..13a48c37 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@

Slack Channel -Version +Version Build Status Docker Ready

diff --git a/build.sh b/build.sh index dedf919b..c0e46db7 100755 --- a/build.sh +++ b/build.sh @@ -4,7 +4,7 @@ set -e cd $(dirname "${BASH_SOURCE[0]}") OD="$(pwd)" -VERSION=1.16.0 +VERSION=1.16.1 PROTECTED_MODE="no" # Hardcode some values to the core package