From bc62edb69215ac0546936d3e5db211d27692447f Mon Sep 17 00:00:00 2001 From: Benjamin Ramser Date: Sun, 5 Sep 2021 11:48:34 +0200 Subject: [PATCH 01/15] feat: add sector --- core/commands.json | 290 ++++++++++-------- core/commands_gen.go | 290 ++++++++++-------- go.mod | 1 + go.sum | 2 + internal/server/search.go | 66 +++- internal/server/test.go | 62 ++++ internal/server/token.go | 2 +- tests/testcmd_test.go | 2 + vendor/github.com/iwpnd/sectr/.gitignore | 19 ++ vendor/github.com/iwpnd/sectr/.gitlint | 119 +++++++ .../iwpnd/sectr/.pre-commit-config.yaml | 13 + vendor/github.com/iwpnd/sectr/go.mod | 3 + vendor/github.com/iwpnd/sectr/readme.md | 71 +++++ vendor/github.com/iwpnd/sectr/sectr.go | 160 ++++++++++ vendor/modules.txt | 3 + 15 files changed, 859 insertions(+), 244 deletions(-) create mode 100644 vendor/github.com/iwpnd/sectr/.gitignore create mode 100644 vendor/github.com/iwpnd/sectr/.gitlint create mode 100644 vendor/github.com/iwpnd/sectr/.pre-commit-config.yaml create mode 100644 vendor/github.com/iwpnd/sectr/go.mod create mode 100644 vendor/github.com/iwpnd/sectr/readme.md create mode 100644 vendor/github.com/iwpnd/sectr/sectr.go diff --git a/core/commands.json b/core/commands.json index 93372653..26c9ed69 100644 --- a/core/commands.json +++ b/core/commands.json @@ -1,5 +1,5 @@ { - "SET":{ + "SET": { "summary": "Sets the value of an id", "complexity": "O(1)", "arguments": [ @@ -35,14 +35,14 @@ { "name": "XX" } - ] + ] }, - { - "name": "value", + { + "name": "value", "enumargs": [ { "name": "OBJECT", - "arguments":[ + "arguments": [ { "name": "geojson", "type": "geojson" @@ -51,7 +51,7 @@ }, { "name": "POINT", - "arguments":[ + "arguments": [ { "name": "lat", "type": "double" @@ -69,7 +69,7 @@ }, { "name": "BOUNDS", - "arguments":[ + "arguments": [ { "name": "minlat", "type": "double" @@ -90,7 +90,7 @@ }, { "name": "HASH", - "arguments":[ + "arguments": [ { "name": "geohash", "type": "geohash" @@ -99,7 +99,7 @@ }, { "name": "STRING", - "arguments":[ + "arguments": [ { "name": "value", "type": "string" @@ -115,7 +115,7 @@ "EXPIRE": { "summary": "Set a timeout on an id", "complexity": "O(1)", - "arguments":[ + "arguments": [ { "name": "key", "type": "string" @@ -135,7 +135,7 @@ "TTL": { "summary": "Get a timeout on an id", "complexity": "O(1)", - "arguments":[ + "arguments": [ { "name": "key", "type": "string" @@ -151,7 +151,7 @@ "PERSIST": { "summary": "Remove the existing timeout on an id", "complexity": "O(1)", - "arguments":[ + "arguments": [ { "name": "key", "type": "string" @@ -167,7 +167,7 @@ "FSET": { "summary": "Set the value for one or more fields of an id", "complexity": "O(1)", - "arguments":[ + "arguments": [ { "name": "key", "type": "string" @@ -183,12 +183,12 @@ "optional": true }, { - "name": ["field","value"], - "type": ["string","double"] + "name": ["field", "value"], + "type": ["string", "double"] }, { - "name": ["field","value"], - "type": ["string","double"], + "name": ["field", "value"], + "type": ["string", "double"], "multiple": true, "optional": true } @@ -199,7 +199,7 @@ "BOUNDS": { "summary": "Get the combined bounds of all the objects in a key", "complexity": "O(1)", - "arguments":[ + "arguments": [ { "name": "key", "type": "string" @@ -211,7 +211,7 @@ "GET": { "summary": "Get the object of an id", "complexity": "O(1)", - "arguments":[ + "arguments": [ { "name": "key", "type": "string" @@ -257,7 +257,7 @@ "DEL": { "summary": "Delete an id from a key", "complexity": "O(1)", - "arguments":[ + "arguments": [ { "name": "key", "type": "string" @@ -273,7 +273,7 @@ "DROP": { "summary": "Remove a key from the database", "complexity": "O(1)", - "arguments":[ + "arguments": [ { "name": "key", "type": "string" @@ -317,7 +317,7 @@ "KEYS": { "summary": "Finds all keys matching the given pattern", "complexity": "O(N) where N is the number of keys in the database", - "arguments":[ + "arguments": [ { "name": "pattern", "type": "pattern" @@ -329,7 +329,7 @@ "STATS": { "summary": "Show stats for one or more keys", "complexity": "O(N) where N is the number of keys being requested", - "arguments":[ + "arguments": [ { "name": "key", "type": "string", @@ -342,7 +342,7 @@ "SEARCH": { "summary": "Search for string values in a key", "complexity": "O(N) where N is the number of values in the key", - "arguments":[ + "arguments": [ { "name": "key", "type": "string" @@ -375,35 +375,35 @@ { "name": "DESC" } - ] - }, + ] + }, { "command": "WHERE", - "name": ["field","min","max"], - "type": ["string","double","double"], + "name": ["field", "min", "max"], + "type": ["string", "double", "double"], "optional": true, "multiple": true }, { "command": "WHEREIN", - "name": ["field","count","value"], - "type": ["string","integer","double"], + "name": ["field", "count", "value"], + "type": ["string", "integer", "double"], "optional": true, "multiple": true, "variadic": true }, { "command": "WHEREEVAL", - "name": ["script","numargs","arg"], - "type": ["string","integer","string"], + "name": ["script", "numargs", "arg"], + "type": ["string", "integer", "string"], "optional": true, "multiple": true, "variadic": true }, { "command": "WHEREEVALSHA", - "name": ["sha1","numargs","arg"], - "type": ["string","integer","string"], + "name": ["sha1", "numargs", "arg"], + "type": ["string", "integer", "string"], "optional": true, "multiple": true, "variadic": true @@ -433,7 +433,7 @@ "SCAN": { "summary": "Incrementally iterate though a key", "complexity": "O(N) where N is the number of ids in the key", - "arguments":[ + "arguments": [ { "name": "key", "type": "string" @@ -466,35 +466,35 @@ { "name": "DESC" } - ] - }, + ] + }, { "command": "WHERE", - "name": ["field","min","max"], - "type": ["string","double","double"], + "name": ["field", "min", "max"], + "type": ["string", "double", "double"], "optional": true, "multiple": true }, { "command": "WHEREIN", - "name": ["field","count","value"], - "type": ["string","integer","double"], + "name": ["field", "count", "value"], + "type": ["string", "integer", "double"], "optional": true, "multiple": true, "variadic": true }, { "command": "WHEREEVAL", - "name": ["script","numargs","arg"], - "type": ["string","integer","string"], + "name": ["script", "numargs", "arg"], + "type": ["string", "integer", "string"], "optional": true, "multiple": true, "variadic": true }, { "command": "WHEREEVALSHA", - "name": ["sha1","numargs","arg"], - "type": ["string","integer","string"], + "name": ["sha1", "numargs", "arg"], + "type": ["string", "integer", "string"], "optional": true, "multiple": true, "variadic": true @@ -542,7 +542,7 @@ "NEARBY": { "summary": "Searches for ids that are nearby a point", "complexity": "O(log(N)) where N is the number of ids in the area", - "arguments":[ + "arguments": [ { "name": "key", "type": "string" @@ -579,31 +579,31 @@ }, { "command": "WHERE", - "name": ["field","min","max"], - "type": ["string","double","double"], + "name": ["field", "min", "max"], + "type": ["string", "double", "double"], "optional": true, "multiple": true }, { "command": "WHEREIN", - "name": ["field","count","value"], - "type": ["string","integer","double"], + "name": ["field", "count", "value"], + "type": ["string", "integer", "double"], "optional": true, "multiple": true, "variadic": true }, { "command": "WHEREEVAL", - "name": ["script","numargs","arg"], - "type": ["string","integer","string"], + "name": ["script", "numargs", "arg"], + "type": ["string", "integer", "string"], "optional": true, "multiple": true, "variadic": true }, { "command": "WHEREEVALSHA", - "name": ["sha1","numargs","arg"], - "type": ["string","integer","string"], + "name": ["sha1", "numargs", "arg"], + "type": ["string", "integer", "string"], "optional": true, "multiple": true, "variadic": true @@ -684,7 +684,7 @@ }, { "name": "ROAM", - "arguments":[ + "arguments": [ { "name": "key", "type": "string" @@ -708,7 +708,7 @@ "WITHIN": { "summary": "Searches for ids that completely within the area", "complexity": "O(log(N)) where N is the number of ids in the area", - "arguments":[ + "arguments": [ { "name": "key", "type": "string" @@ -739,31 +739,31 @@ }, { "command": "WHERE", - "name": ["field","min","max"], - "type": ["string","double","double"], + "name": ["field", "min", "max"], + "type": ["string", "double", "double"], "optional": true, "multiple": true }, { "command": "WHEREIN", - "name": ["field","count","value"], - "type": ["string","integer","double"], + "name": ["field", "count", "value"], + "type": ["string", "integer", "double"], "optional": true, "multiple": true, "variadic": true }, { "command": "WHEREEVAL", - "name": ["script","numargs","arg"], - "type": ["string","integer","string"], + "name": ["script", "numargs", "arg"], + "type": ["string", "integer", "string"], "optional": true, "multiple": true, "variadic": true }, { "command": "WHEREEVALSHA", - "name": ["sha1","numargs","arg"], - "type": ["string","integer","string"], + "name": ["sha1", "numargs", "arg"], + "type": ["string", "integer", "string"], "optional": true, "multiple": true, "variadic": true @@ -840,7 +840,7 @@ }, { "name": "BOUNDS", - "arguments":[ + "arguments": [ { "name": "minlat", "type": "double" @@ -861,7 +861,7 @@ }, { "name": "OBJECT", - "arguments":[ + "arguments": [ { "name": "geojson", "type": "geojson" @@ -887,7 +887,7 @@ }, { "name": "TILE", - "arguments":[ + "arguments": [ { "name": "x", "type": "double" @@ -904,7 +904,7 @@ }, { "name": "QUADKEY", - "arguments":[ + "arguments": [ { "name": "quadkey", "type": "string" @@ -919,6 +919,31 @@ "type": "geohash" } ] + }, + { + "name": "SECTOR", + "arguments": [ + { + "name": "lat", + "type": "double" + }, + { + "name": "lon", + "type": "double" + }, + { + "name": "radius", + "type": "double" + }, + { + "name": "startBearing", + "type": "double" + }, + { + "name": "endBearing", + "type": "double" + } + ] } ] } @@ -929,7 +954,7 @@ "INTERSECTS": { "summary": "Searches for ids that intersect an area", "complexity": "O(log(N)) where N is the number of ids in the area", - "arguments":[ + "arguments": [ { "name": "key", "type": "string" @@ -960,31 +985,31 @@ }, { "command": "WHERE", - "name": ["field","min","max"], - "type": ["string","double","double"], + "name": ["field", "min", "max"], + "type": ["string", "double", "double"], "optional": true, "multiple": true }, { "command": "WHEREIN", - "name": ["field","count","value"], - "type": ["string","integer","double"], + "name": ["field", "count", "value"], + "type": ["string", "integer", "double"], "optional": true, "multiple": true, "variadic": true }, { "command": "WHEREEVAL", - "name": ["script","numargs","arg"], - "type": ["string","integer","string"], + "name": ["script", "numargs", "arg"], + "type": ["string", "integer", "string"], "optional": true, "multiple": true, "variadic": true }, { "command": "WHEREEVALSHA", - "name": ["sha1","numargs","arg"], - "type": ["string","integer","string"], + "name": ["sha1", "numargs", "arg"], + "type": ["string", "integer", "string"], "optional": true, "multiple": true, "variadic": true @@ -1067,7 +1092,7 @@ }, { "name": "BOUNDS", - "arguments":[ + "arguments": [ { "name": "minlat", "type": "double" @@ -1088,7 +1113,7 @@ }, { "name": "OBJECT", - "arguments":[ + "arguments": [ { "name": "geojson", "type": "geojson" @@ -1114,7 +1139,7 @@ }, { "name": "TILE", - "arguments":[ + "arguments": [ { "name": "x", "type": "double" @@ -1131,7 +1156,7 @@ }, { "name": "QUADKEY", - "arguments":[ + "arguments": [ { "name": "quadkey", "type": "string" @@ -1146,6 +1171,31 @@ "type": "geohash" } ] + }, + { + "name": "SECTOR", + "arguments": [ + { + "name": "lat", + "type": "double" + }, + { + "name": "lon", + "type": "double" + }, + { + "name": "radius", + "type": "double" + }, + { + "name": "startBearing", + "type": "double" + }, + { + "name": "endBearing", + "type": "double" + } + ] } ] } @@ -1155,7 +1205,7 @@ }, "CONFIG GET": { "summary": "Get the value of a configuration parameter", - "arguments":[ + "arguments": [ { "name": "parameter", "type": "string" @@ -1165,7 +1215,7 @@ }, "CONFIG SET": { "summary": "Set a configuration parameter to the given value", - "arguments":[ + "arguments": [ { "name": "parameter", "type": "string" @@ -1180,18 +1230,18 @@ }, "CONFIG REWRITE": { "summary": "Rewrite the configuration file with the in memory configuration", - "arguments":[], + "arguments": [], "group": "server" }, "SERVER": { - "summary":"Show server stats and details", + "summary": "Show server stats and details", "complexity": "O(1)", "arguments": [], "since": "1.0.0", "group": "server" }, "GC": { - "summary":"Forces a garbage collection", + "summary": "Forces a garbage collection", "complexity": "O(1)", "arguments": [], "since": "1.0.0", @@ -1202,14 +1252,14 @@ "complexity": "O(1)", "arguments": [ { - "enum": ["yes","no"] + "enum": ["yes", "no"] } ], "since": "1.0.0", "group": "server" }, "FLUSHDB": { - "summary":"Removes all keys", + "summary": "Removes all keys", "complexity": "O(1)", "arguments": [], "since": "1.0.0", @@ -1335,7 +1385,7 @@ "name": ["name", "value"], "type": ["string", "string"], "optional": true, - "multiple": true + "multiple": true }, { "command": "EX", @@ -1373,7 +1423,6 @@ "type": "string", "variadic": true } - ], "group": "webhook" }, @@ -1389,7 +1438,7 @@ }, "HOOKS": { "summary": "Finds all hooks matching a pattern", - "arguments":[ + "arguments": [ { "name": "pattern", "type": "pattern" @@ -1399,7 +1448,7 @@ }, "PDELHOOK": { "summary": "Removes all hooks matching a pattern", - "arguments":[ + "arguments": [ { "name": "pattern", "type": "pattern" @@ -1420,7 +1469,7 @@ "name": ["name", "value"], "type": ["string", "string"], "optional": true, - "multiple": true + "multiple": true }, { "command": "EX", @@ -1458,7 +1507,6 @@ "type": "string", "variadic": true } - ], "group": "pubsub" }, @@ -1474,7 +1522,7 @@ }, "CHANS": { "summary": "Finds all channels matching a pattern", - "arguments":[ + "arguments": [ { "name": "pattern", "type": "pattern" @@ -1484,7 +1532,7 @@ }, "PDELCHAN": { "summary": "Removes all channels matching a pattern", - "arguments":[ + "arguments": [ { "name": "pattern", "type": "pattern" @@ -1494,7 +1542,7 @@ }, "SUBSCRIBE": { "summary": "Subscribe to a geofence channel", - "arguments":[ + "arguments": [ { "name": "channel", "type": "string", @@ -1505,7 +1553,7 @@ }, "PSUBSCRIBE": { "summary": "Subscribes the client to the given patterns", - "arguments":[ + "arguments": [ { "name": "pattern", "type": "pattern", @@ -1516,12 +1564,12 @@ }, "PDEL": { "summary": "Removes all objects matching a pattern", - "arguments":[ + "arguments": [ { "name": "key", "type": "string" }, - { + { "name": "pattern", "type": "pattern" } @@ -1531,7 +1579,7 @@ "JGET": { "summary": "Get a value from a JSON document", "complexity": "O(1)", - "arguments":[ + "arguments": [ { "name": "key", "type": "string" @@ -1556,7 +1604,7 @@ "JSET": { "summary": "Set a value in a JSON document", "complexity": "O(1)", - "arguments":[ + "arguments": [ { "name": "key", "type": "string" @@ -1591,7 +1639,7 @@ "JDEL": { "summary": "Delete a value from a JSON document", "complexity": "O(1)", - "arguments":[ + "arguments": [ { "name": "key", "type": "string" @@ -1607,7 +1655,7 @@ ], "group": "keys" }, - "EVAL":{ + "EVAL": { "summary": "Evaluates a Lua script", "complexity": "Depends on the evaluated script", "arguments": [ @@ -1635,7 +1683,7 @@ "since": "1.10.0", "group": "scripting" }, - "EVALSHA":{ + "EVALSHA": { "summary": "Evaluates a Lua script cached on the server by its SHA1 digest", "complexity": "Depends on the evaluated script", "arguments": [ @@ -1663,7 +1711,7 @@ "since": "1.10.0", "group": "scripting" }, - "EVALRO":{ + "EVALRO": { "summary": "Evaluates a read-only Lua script", "complexity": "Depends on the evaluated script", "arguments": [ @@ -1691,7 +1739,7 @@ "since": "1.10.0", "group": "scripting" }, - "EVALROSHA":{ + "EVALROSHA": { "summary": "Evaluates a read-only Lua script cached on the server by its SHA1 digest", "complexity": "Depends on the evaluated script", "arguments": [ @@ -1719,7 +1767,7 @@ "since": "1.10.0", "group": "scripting" }, - "EVALNA":{ + "EVALNA": { "summary": "Evaluates a Lua script in a non-atomic fashion", "complexity": "Depends on the evaluated script", "arguments": [ @@ -1747,7 +1795,7 @@ "since": "1.10.0", "group": "scripting" }, - "EVALNASHA":{ + "EVALNASHA": { "summary": "Evaluates, in a non-atomic fashion, a Lua script cached on the server by its SHA1 digest", "complexity": "Depends on the evaluated script", "arguments": [ @@ -1775,7 +1823,7 @@ "since": "1.10.0", "group": "scripting" }, - "SCRIPT EXISTS":{ + "SCRIPT EXISTS": { "summary": "Returns information about the existence of the scripts in server cache", "complexity": "O(N) where N is the number of provided sha1 arguments", "arguments": [ @@ -1788,7 +1836,7 @@ "since": "1.10.0", "group": "scripting" }, - "SCRIPT LOAD":{ + "SCRIPT LOAD": { "summary": "Loads the compiled version of a script into the server cache, without executing", "complexity": "O(N) where N is the number of bytes in the script", "arguments": [ @@ -1800,13 +1848,13 @@ "since": "1.10.0", "group": "scripting" }, - "SCRIPT FLUSH":{ + "SCRIPT FLUSH": { "summary": "Flushes the server cache of Lua scripts", "complexity": "O(1)", "since": "1.10.0", "group": "scripting" }, - "TEST":{ + "TEST": { "summary": "Performs spatial test", "complexity": "One test per command, complexity depends on the test", "arguments": [ @@ -1841,7 +1889,7 @@ }, { "name": "BOUNDS", - "arguments":[ + "arguments": [ { "name": "minlat", "type": "double" @@ -1862,7 +1910,7 @@ }, { "name": "OBJECT", - "arguments":[ + "arguments": [ { "name": "geojson", "type": "geojson" @@ -1888,7 +1936,7 @@ }, { "name": "TILE", - "arguments":[ + "arguments": [ { "name": "x", "type": "double" @@ -1905,7 +1953,7 @@ }, { "name": "QUADKEY", - "arguments":[ + "arguments": [ { "name": "quadkey", "type": "string" @@ -1971,7 +2019,7 @@ }, { "name": "BOUNDS", - "arguments":[ + "arguments": [ { "name": "minlat", "type": "double" @@ -1992,7 +2040,7 @@ }, { "name": "OBJECT", - "arguments":[ + "arguments": [ { "name": "geojson", "type": "geojson" @@ -2018,7 +2066,7 @@ }, { "name": "TILE", - "arguments":[ + "arguments": [ { "name": "x", "type": "double" @@ -2035,7 +2083,7 @@ }, { "name": "QUADKEY", - "arguments":[ + "arguments": [ { "name": "quadkey", "type": "string" diff --git a/core/commands_gen.go b/core/commands_gen.go index e4a4a884..4e40941a 100644 --- a/core/commands_gen.go +++ b/core/commands_gen.go @@ -165,7 +165,7 @@ var Commands = func() map[string]Command { }() var commandsJSON = `{ - "SET":{ + "SET": { "summary": "Sets the value of an id", "complexity": "O(1)", "arguments": [ @@ -201,14 +201,14 @@ var commandsJSON = `{ { "name": "XX" } - ] + ] }, - { - "name": "value", + { + "name": "value", "enumargs": [ { "name": "OBJECT", - "arguments":[ + "arguments": [ { "name": "geojson", "type": "geojson" @@ -217,7 +217,7 @@ var commandsJSON = `{ }, { "name": "POINT", - "arguments":[ + "arguments": [ { "name": "lat", "type": "double" @@ -235,7 +235,7 @@ var commandsJSON = `{ }, { "name": "BOUNDS", - "arguments":[ + "arguments": [ { "name": "minlat", "type": "double" @@ -256,7 +256,7 @@ var commandsJSON = `{ }, { "name": "HASH", - "arguments":[ + "arguments": [ { "name": "geohash", "type": "geohash" @@ -265,7 +265,7 @@ var commandsJSON = `{ }, { "name": "STRING", - "arguments":[ + "arguments": [ { "name": "value", "type": "string" @@ -281,7 +281,7 @@ var commandsJSON = `{ "EXPIRE": { "summary": "Set a timeout on an id", "complexity": "O(1)", - "arguments":[ + "arguments": [ { "name": "key", "type": "string" @@ -301,7 +301,7 @@ var commandsJSON = `{ "TTL": { "summary": "Get a timeout on an id", "complexity": "O(1)", - "arguments":[ + "arguments": [ { "name": "key", "type": "string" @@ -317,7 +317,7 @@ var commandsJSON = `{ "PERSIST": { "summary": "Remove the existing timeout on an id", "complexity": "O(1)", - "arguments":[ + "arguments": [ { "name": "key", "type": "string" @@ -333,7 +333,7 @@ var commandsJSON = `{ "FSET": { "summary": "Set the value for one or more fields of an id", "complexity": "O(1)", - "arguments":[ + "arguments": [ { "name": "key", "type": "string" @@ -349,12 +349,12 @@ var commandsJSON = `{ "optional": true }, { - "name": ["field","value"], - "type": ["string","double"] + "name": ["field", "value"], + "type": ["string", "double"] }, { - "name": ["field","value"], - "type": ["string","double"], + "name": ["field", "value"], + "type": ["string", "double"], "multiple": true, "optional": true } @@ -365,7 +365,7 @@ var commandsJSON = `{ "BOUNDS": { "summary": "Get the combined bounds of all the objects in a key", "complexity": "O(1)", - "arguments":[ + "arguments": [ { "name": "key", "type": "string" @@ -377,7 +377,7 @@ var commandsJSON = `{ "GET": { "summary": "Get the object of an id", "complexity": "O(1)", - "arguments":[ + "arguments": [ { "name": "key", "type": "string" @@ -423,7 +423,7 @@ var commandsJSON = `{ "DEL": { "summary": "Delete an id from a key", "complexity": "O(1)", - "arguments":[ + "arguments": [ { "name": "key", "type": "string" @@ -439,7 +439,7 @@ var commandsJSON = `{ "DROP": { "summary": "Remove a key from the database", "complexity": "O(1)", - "arguments":[ + "arguments": [ { "name": "key", "type": "string" @@ -483,7 +483,7 @@ var commandsJSON = `{ "KEYS": { "summary": "Finds all keys matching the given pattern", "complexity": "O(N) where N is the number of keys in the database", - "arguments":[ + "arguments": [ { "name": "pattern", "type": "pattern" @@ -495,7 +495,7 @@ var commandsJSON = `{ "STATS": { "summary": "Show stats for one or more keys", "complexity": "O(N) where N is the number of keys being requested", - "arguments":[ + "arguments": [ { "name": "key", "type": "string", @@ -508,7 +508,7 @@ var commandsJSON = `{ "SEARCH": { "summary": "Search for string values in a key", "complexity": "O(N) where N is the number of values in the key", - "arguments":[ + "arguments": [ { "name": "key", "type": "string" @@ -541,35 +541,35 @@ var commandsJSON = `{ { "name": "DESC" } - ] - }, + ] + }, { "command": "WHERE", - "name": ["field","min","max"], - "type": ["string","double","double"], + "name": ["field", "min", "max"], + "type": ["string", "double", "double"], "optional": true, "multiple": true }, { "command": "WHEREIN", - "name": ["field","count","value"], - "type": ["string","integer","double"], + "name": ["field", "count", "value"], + "type": ["string", "integer", "double"], "optional": true, "multiple": true, "variadic": true }, { "command": "WHEREEVAL", - "name": ["script","numargs","arg"], - "type": ["string","integer","string"], + "name": ["script", "numargs", "arg"], + "type": ["string", "integer", "string"], "optional": true, "multiple": true, "variadic": true }, { "command": "WHEREEVALSHA", - "name": ["sha1","numargs","arg"], - "type": ["string","integer","string"], + "name": ["sha1", "numargs", "arg"], + "type": ["string", "integer", "string"], "optional": true, "multiple": true, "variadic": true @@ -599,7 +599,7 @@ var commandsJSON = `{ "SCAN": { "summary": "Incrementally iterate though a key", "complexity": "O(N) where N is the number of ids in the key", - "arguments":[ + "arguments": [ { "name": "key", "type": "string" @@ -632,35 +632,35 @@ var commandsJSON = `{ { "name": "DESC" } - ] - }, + ] + }, { "command": "WHERE", - "name": ["field","min","max"], - "type": ["string","double","double"], + "name": ["field", "min", "max"], + "type": ["string", "double", "double"], "optional": true, "multiple": true }, { "command": "WHEREIN", - "name": ["field","count","value"], - "type": ["string","integer","double"], + "name": ["field", "count", "value"], + "type": ["string", "integer", "double"], "optional": true, "multiple": true, "variadic": true }, { "command": "WHEREEVAL", - "name": ["script","numargs","arg"], - "type": ["string","integer","string"], + "name": ["script", "numargs", "arg"], + "type": ["string", "integer", "string"], "optional": true, "multiple": true, "variadic": true }, { "command": "WHEREEVALSHA", - "name": ["sha1","numargs","arg"], - "type": ["string","integer","string"], + "name": ["sha1", "numargs", "arg"], + "type": ["string", "integer", "string"], "optional": true, "multiple": true, "variadic": true @@ -708,7 +708,7 @@ var commandsJSON = `{ "NEARBY": { "summary": "Searches for ids that are nearby a point", "complexity": "O(log(N)) where N is the number of ids in the area", - "arguments":[ + "arguments": [ { "name": "key", "type": "string" @@ -745,31 +745,31 @@ var commandsJSON = `{ }, { "command": "WHERE", - "name": ["field","min","max"], - "type": ["string","double","double"], + "name": ["field", "min", "max"], + "type": ["string", "double", "double"], "optional": true, "multiple": true }, { "command": "WHEREIN", - "name": ["field","count","value"], - "type": ["string","integer","double"], + "name": ["field", "count", "value"], + "type": ["string", "integer", "double"], "optional": true, "multiple": true, "variadic": true }, { "command": "WHEREEVAL", - "name": ["script","numargs","arg"], - "type": ["string","integer","string"], + "name": ["script", "numargs", "arg"], + "type": ["string", "integer", "string"], "optional": true, "multiple": true, "variadic": true }, { "command": "WHEREEVALSHA", - "name": ["sha1","numargs","arg"], - "type": ["string","integer","string"], + "name": ["sha1", "numargs", "arg"], + "type": ["string", "integer", "string"], "optional": true, "multiple": true, "variadic": true @@ -850,7 +850,7 @@ var commandsJSON = `{ }, { "name": "ROAM", - "arguments":[ + "arguments": [ { "name": "key", "type": "string" @@ -874,7 +874,7 @@ var commandsJSON = `{ "WITHIN": { "summary": "Searches for ids that completely within the area", "complexity": "O(log(N)) where N is the number of ids in the area", - "arguments":[ + "arguments": [ { "name": "key", "type": "string" @@ -905,31 +905,31 @@ var commandsJSON = `{ }, { "command": "WHERE", - "name": ["field","min","max"], - "type": ["string","double","double"], + "name": ["field", "min", "max"], + "type": ["string", "double", "double"], "optional": true, "multiple": true }, { "command": "WHEREIN", - "name": ["field","count","value"], - "type": ["string","integer","double"], + "name": ["field", "count", "value"], + "type": ["string", "integer", "double"], "optional": true, "multiple": true, "variadic": true }, { "command": "WHEREEVAL", - "name": ["script","numargs","arg"], - "type": ["string","integer","string"], + "name": ["script", "numargs", "arg"], + "type": ["string", "integer", "string"], "optional": true, "multiple": true, "variadic": true }, { "command": "WHEREEVALSHA", - "name": ["sha1","numargs","arg"], - "type": ["string","integer","string"], + "name": ["sha1", "numargs", "arg"], + "type": ["string", "integer", "string"], "optional": true, "multiple": true, "variadic": true @@ -1006,7 +1006,7 @@ var commandsJSON = `{ }, { "name": "BOUNDS", - "arguments":[ + "arguments": [ { "name": "minlat", "type": "double" @@ -1027,7 +1027,7 @@ var commandsJSON = `{ }, { "name": "OBJECT", - "arguments":[ + "arguments": [ { "name": "geojson", "type": "geojson" @@ -1053,7 +1053,7 @@ var commandsJSON = `{ }, { "name": "TILE", - "arguments":[ + "arguments": [ { "name": "x", "type": "double" @@ -1070,7 +1070,7 @@ var commandsJSON = `{ }, { "name": "QUADKEY", - "arguments":[ + "arguments": [ { "name": "quadkey", "type": "string" @@ -1085,6 +1085,31 @@ var commandsJSON = `{ "type": "geohash" } ] + }, + { + "name": "SECTOR", + "arguments": [ + { + "name": "lat", + "type": "double" + }, + { + "name": "lon", + "type": "double" + }, + { + "name": "radius", + "type": "double" + }, + { + "name": "startBearing", + "type": "double" + }, + { + "name": "endBearing", + "type": "double" + } + ] } ] } @@ -1095,7 +1120,7 @@ var commandsJSON = `{ "INTERSECTS": { "summary": "Searches for ids that intersect an area", "complexity": "O(log(N)) where N is the number of ids in the area", - "arguments":[ + "arguments": [ { "name": "key", "type": "string" @@ -1126,31 +1151,31 @@ var commandsJSON = `{ }, { "command": "WHERE", - "name": ["field","min","max"], - "type": ["string","double","double"], + "name": ["field", "min", "max"], + "type": ["string", "double", "double"], "optional": true, "multiple": true }, { "command": "WHEREIN", - "name": ["field","count","value"], - "type": ["string","integer","double"], + "name": ["field", "count", "value"], + "type": ["string", "integer", "double"], "optional": true, "multiple": true, "variadic": true }, { "command": "WHEREEVAL", - "name": ["script","numargs","arg"], - "type": ["string","integer","string"], + "name": ["script", "numargs", "arg"], + "type": ["string", "integer", "string"], "optional": true, "multiple": true, "variadic": true }, { "command": "WHEREEVALSHA", - "name": ["sha1","numargs","arg"], - "type": ["string","integer","string"], + "name": ["sha1", "numargs", "arg"], + "type": ["string", "integer", "string"], "optional": true, "multiple": true, "variadic": true @@ -1233,7 +1258,7 @@ var commandsJSON = `{ }, { "name": "BOUNDS", - "arguments":[ + "arguments": [ { "name": "minlat", "type": "double" @@ -1254,7 +1279,7 @@ var commandsJSON = `{ }, { "name": "OBJECT", - "arguments":[ + "arguments": [ { "name": "geojson", "type": "geojson" @@ -1280,7 +1305,7 @@ var commandsJSON = `{ }, { "name": "TILE", - "arguments":[ + "arguments": [ { "name": "x", "type": "double" @@ -1297,7 +1322,7 @@ var commandsJSON = `{ }, { "name": "QUADKEY", - "arguments":[ + "arguments": [ { "name": "quadkey", "type": "string" @@ -1312,6 +1337,31 @@ var commandsJSON = `{ "type": "geohash" } ] + }, + { + "name": "SECTOR", + "arguments": [ + { + "name": "lat", + "type": "double" + }, + { + "name": "lon", + "type": "double" + }, + { + "name": "radius", + "type": "double" + }, + { + "name": "startBearing", + "type": "double" + }, + { + "name": "endBearing", + "type": "double" + } + ] } ] } @@ -1321,7 +1371,7 @@ var commandsJSON = `{ }, "CONFIG GET": { "summary": "Get the value of a configuration parameter", - "arguments":[ + "arguments": [ { "name": "parameter", "type": "string" @@ -1331,7 +1381,7 @@ var commandsJSON = `{ }, "CONFIG SET": { "summary": "Set a configuration parameter to the given value", - "arguments":[ + "arguments": [ { "name": "parameter", "type": "string" @@ -1346,18 +1396,18 @@ var commandsJSON = `{ }, "CONFIG REWRITE": { "summary": "Rewrite the configuration file with the in memory configuration", - "arguments":[], + "arguments": [], "group": "server" }, "SERVER": { - "summary":"Show server stats and details", + "summary": "Show server stats and details", "complexity": "O(1)", "arguments": [], "since": "1.0.0", "group": "server" }, "GC": { - "summary":"Forces a garbage collection", + "summary": "Forces a garbage collection", "complexity": "O(1)", "arguments": [], "since": "1.0.0", @@ -1368,14 +1418,14 @@ var commandsJSON = `{ "complexity": "O(1)", "arguments": [ { - "enum": ["yes","no"] + "enum": ["yes", "no"] } ], "since": "1.0.0", "group": "server" }, "FLUSHDB": { - "summary":"Removes all keys", + "summary": "Removes all keys", "complexity": "O(1)", "arguments": [], "since": "1.0.0", @@ -1501,7 +1551,7 @@ var commandsJSON = `{ "name": ["name", "value"], "type": ["string", "string"], "optional": true, - "multiple": true + "multiple": true }, { "command": "EX", @@ -1539,7 +1589,6 @@ var commandsJSON = `{ "type": "string", "variadic": true } - ], "group": "webhook" }, @@ -1555,7 +1604,7 @@ var commandsJSON = `{ }, "HOOKS": { "summary": "Finds all hooks matching a pattern", - "arguments":[ + "arguments": [ { "name": "pattern", "type": "pattern" @@ -1565,7 +1614,7 @@ var commandsJSON = `{ }, "PDELHOOK": { "summary": "Removes all hooks matching a pattern", - "arguments":[ + "arguments": [ { "name": "pattern", "type": "pattern" @@ -1586,7 +1635,7 @@ var commandsJSON = `{ "name": ["name", "value"], "type": ["string", "string"], "optional": true, - "multiple": true + "multiple": true }, { "command": "EX", @@ -1624,7 +1673,6 @@ var commandsJSON = `{ "type": "string", "variadic": true } - ], "group": "pubsub" }, @@ -1640,7 +1688,7 @@ var commandsJSON = `{ }, "CHANS": { "summary": "Finds all channels matching a pattern", - "arguments":[ + "arguments": [ { "name": "pattern", "type": "pattern" @@ -1650,7 +1698,7 @@ var commandsJSON = `{ }, "PDELCHAN": { "summary": "Removes all channels matching a pattern", - "arguments":[ + "arguments": [ { "name": "pattern", "type": "pattern" @@ -1660,7 +1708,7 @@ var commandsJSON = `{ }, "SUBSCRIBE": { "summary": "Subscribe to a geofence channel", - "arguments":[ + "arguments": [ { "name": "channel", "type": "string", @@ -1671,7 +1719,7 @@ var commandsJSON = `{ }, "PSUBSCRIBE": { "summary": "Subscribes the client to the given patterns", - "arguments":[ + "arguments": [ { "name": "pattern", "type": "pattern", @@ -1682,12 +1730,12 @@ var commandsJSON = `{ }, "PDEL": { "summary": "Removes all objects matching a pattern", - "arguments":[ + "arguments": [ { "name": "key", "type": "string" }, - { + { "name": "pattern", "type": "pattern" } @@ -1697,7 +1745,7 @@ var commandsJSON = `{ "JGET": { "summary": "Get a value from a JSON document", "complexity": "O(1)", - "arguments":[ + "arguments": [ { "name": "key", "type": "string" @@ -1722,7 +1770,7 @@ var commandsJSON = `{ "JSET": { "summary": "Set a value in a JSON document", "complexity": "O(1)", - "arguments":[ + "arguments": [ { "name": "key", "type": "string" @@ -1757,7 +1805,7 @@ var commandsJSON = `{ "JDEL": { "summary": "Delete a value from a JSON document", "complexity": "O(1)", - "arguments":[ + "arguments": [ { "name": "key", "type": "string" @@ -1773,7 +1821,7 @@ var commandsJSON = `{ ], "group": "keys" }, - "EVAL":{ + "EVAL": { "summary": "Evaluates a Lua script", "complexity": "Depends on the evaluated script", "arguments": [ @@ -1801,7 +1849,7 @@ var commandsJSON = `{ "since": "1.10.0", "group": "scripting" }, - "EVALSHA":{ + "EVALSHA": { "summary": "Evaluates a Lua script cached on the server by its SHA1 digest", "complexity": "Depends on the evaluated script", "arguments": [ @@ -1829,7 +1877,7 @@ var commandsJSON = `{ "since": "1.10.0", "group": "scripting" }, - "EVALRO":{ + "EVALRO": { "summary": "Evaluates a read-only Lua script", "complexity": "Depends on the evaluated script", "arguments": [ @@ -1857,7 +1905,7 @@ var commandsJSON = `{ "since": "1.10.0", "group": "scripting" }, - "EVALROSHA":{ + "EVALROSHA": { "summary": "Evaluates a read-only Lua script cached on the server by its SHA1 digest", "complexity": "Depends on the evaluated script", "arguments": [ @@ -1885,7 +1933,7 @@ var commandsJSON = `{ "since": "1.10.0", "group": "scripting" }, - "EVALNA":{ + "EVALNA": { "summary": "Evaluates a Lua script in a non-atomic fashion", "complexity": "Depends on the evaluated script", "arguments": [ @@ -1913,7 +1961,7 @@ var commandsJSON = `{ "since": "1.10.0", "group": "scripting" }, - "EVALNASHA":{ + "EVALNASHA": { "summary": "Evaluates, in a non-atomic fashion, a Lua script cached on the server by its SHA1 digest", "complexity": "Depends on the evaluated script", "arguments": [ @@ -1941,7 +1989,7 @@ var commandsJSON = `{ "since": "1.10.0", "group": "scripting" }, - "SCRIPT EXISTS":{ + "SCRIPT EXISTS": { "summary": "Returns information about the existence of the scripts in server cache", "complexity": "O(N) where N is the number of provided sha1 arguments", "arguments": [ @@ -1954,7 +2002,7 @@ var commandsJSON = `{ "since": "1.10.0", "group": "scripting" }, - "SCRIPT LOAD":{ + "SCRIPT LOAD": { "summary": "Loads the compiled version of a script into the server cache, without executing", "complexity": "O(N) where N is the number of bytes in the script", "arguments": [ @@ -1966,13 +2014,13 @@ var commandsJSON = `{ "since": "1.10.0", "group": "scripting" }, - "SCRIPT FLUSH":{ + "SCRIPT FLUSH": { "summary": "Flushes the server cache of Lua scripts", "complexity": "O(1)", "since": "1.10.0", "group": "scripting" }, - "TEST":{ + "TEST": { "summary": "Performs spatial test", "complexity": "One test per command, complexity depends on the test", "arguments": [ @@ -2007,7 +2055,7 @@ var commandsJSON = `{ }, { "name": "BOUNDS", - "arguments":[ + "arguments": [ { "name": "minlat", "type": "double" @@ -2028,7 +2076,7 @@ var commandsJSON = `{ }, { "name": "OBJECT", - "arguments":[ + "arguments": [ { "name": "geojson", "type": "geojson" @@ -2054,7 +2102,7 @@ var commandsJSON = `{ }, { "name": "TILE", - "arguments":[ + "arguments": [ { "name": "x", "type": "double" @@ -2071,7 +2119,7 @@ var commandsJSON = `{ }, { "name": "QUADKEY", - "arguments":[ + "arguments": [ { "name": "quadkey", "type": "string" @@ -2137,7 +2185,7 @@ var commandsJSON = `{ }, { "name": "BOUNDS", - "arguments":[ + "arguments": [ { "name": "minlat", "type": "double" @@ -2158,7 +2206,7 @@ var commandsJSON = `{ }, { "name": "OBJECT", - "arguments":[ + "arguments": [ { "name": "geojson", "type": "geojson" @@ -2184,7 +2232,7 @@ var commandsJSON = `{ }, { "name": "TILE", - "arguments":[ + "arguments": [ { "name": "x", "type": "double" @@ -2201,7 +2249,7 @@ var commandsJSON = `{ }, { "name": "QUADKEY", - "arguments":[ + "arguments": [ { "name": "quadkey", "type": "string" diff --git a/go.mod b/go.mod index ff78dd07..82383ee6 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/eclipse/paho.mqtt.golang v1.3.1 github.com/golang/protobuf v1.4.3 github.com/gomodule/redigo v1.8.3 + github.com/iwpnd/sectr v0.1.2 github.com/mmcloughlin/geohash v0.10.0 github.com/nats-io/nats-server/v2 v2.2.0 // indirect github.com/nats-io/nats.go v1.10.1-0.20210228004050-ed743748acac diff --git a/go.sum b/go.sum index 6afff4ac..1c8bfaff 100644 --- a/go.sum +++ b/go.sum @@ -214,6 +214,8 @@ github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmK github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= +github.com/iwpnd/sectr v0.1.2 h1:FauaPRn5C2tC42HTF7gM3FJZXvGXWc6jabBbIxzTMag= +github.com/iwpnd/sectr v0.1.2/go.mod h1:Dm6YXDJCRx1NTfMX/1RMIkGfVp2ORjCY/cQGfbknz4c= github.com/jcmturner/gofork v1.0.0 h1:J7uCkflzTEhUZ64xqKnkDxq3kzc96ajM1Gli5ktUem8= github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= diff --git a/internal/server/search.go b/internal/server/search.go index c662915d..c37cc975 100644 --- a/internal/server/search.go +++ b/internal/server/search.go @@ -3,10 +3,12 @@ package server import ( "bytes" "errors" + "fmt" "strconv" "strings" "time" + "github.com/iwpnd/sectr" "github.com/mmcloughlin/geohash" "github.com/tidwall/geojson" "github.com/tidwall/geojson/geometry" @@ -214,6 +216,67 @@ func (server *Server) cmdSearchArgs( switch ltyp { case "point": fallthrough + case "sector": + if s.clip { + err = errInvalidArgument("cannot clip with " + ltyp) + return + } + var slat, slon, smeters, sb1, sb2 string + if vs, slat, ok = tokenval(vs); !ok || slat == "" { + err = errInvalidNumberOfArguments + return + } + if vs, slon, ok = tokenval(vs); !ok || slon == "" { + err = errInvalidNumberOfArguments + return + } + if vs, smeters, ok = tokenval(vs); !ok || smeters == "" { + err = errInvalidNumberOfArguments + return + } + if vs, sb1, ok = tokenval(vs); !ok || sb1 == "" { + err = errInvalidNumberOfArguments + return + } + if vs, sb2, ok = tokenval(vs); !ok || sb2 == "" { + err = errInvalidNumberOfArguments + return + } + var lat, lon, meters, b1, b2 float64 + if lat, err = strconv.ParseFloat(slat, 64); err != nil { + err = errInvalidArgument(slat) + return + } + if lon, err = strconv.ParseFloat(slon, 64); err != nil { + err = errInvalidArgument(slon) + return + } + if meters, err = strconv.ParseFloat(smeters, 64); err != nil { + err = errInvalidArgument(smeters) + return + } + if b1, err = strconv.ParseFloat(sb1, 64); err != nil { + err = errInvalidArgument(sb1) + return + } + if b2, err = strconv.ParseFloat(sb2, 64); err != nil { + err = errInvalidArgument(sb2) + return + } + + if b1 == b2 { + err = fmt.Errorf("equal bearings (%s == %s), use CIRCLE instead", sb1, sb2) + return + } + + origin := sectr.Point{Lng: lon, Lat: lat} + sector := sectr.NewSector(origin, meters, b1, b2) + + s.obj, err = geojson.Parse(string(sector.JSON()), &server.geomParseOpts) + if err != nil { + return + } + case "circle": if s.clip { err = errInvalidArgument("cannot clip with " + ltyp) @@ -379,7 +442,8 @@ func (server *Server) cmdSearchArgs( var nearbyTypes = []string{"point"} var withinOrIntersectsTypes = []string{ - "geo", "bounds", "hash", "tile", "quadkey", "get", "object", "circle"} + "geo", "bounds", "hash", "tile", "quadkey", "get", "object", "circle", "sector", +} func (server *Server) cmdNearby(msg *Message) (res resp.Value, err error) { start := time.Now() diff --git a/internal/server/test.go b/internal/server/test.go index 520babd6..01c7a604 100644 --- a/internal/server/test.go +++ b/internal/server/test.go @@ -9,6 +9,7 @@ import ( "strings" "time" + "github.com/iwpnd/sectr" "github.com/mmcloughlin/geohash" "github.com/tidwall/geojson" "github.com/tidwall/geojson/geometry" @@ -47,6 +48,67 @@ func (s *Server) parseArea(ovs []string, doClip bool) (vs []string, o geojson.Ob return } o = geojson.NewPoint(geometry.Point{X: lon, Y: lat}) + case "sector": + if doClip { + err = errInvalidArgument("cannot clip with " + ltyp) + return + } + var slat, slon, smeters, sb1, sb2 string + if vs, slat, ok = tokenval(vs); !ok || slat == "" { + err = errInvalidNumberOfArguments + return + } + if vs, slon, ok = tokenval(vs); !ok || slon == "" { + err = errInvalidNumberOfArguments + return + } + if vs, smeters, ok = tokenval(vs); !ok || smeters == "" { + err = errInvalidNumberOfArguments + return + } + if vs, sb1, ok = tokenval(vs); !ok || sb1 == "" { + err = errInvalidNumberOfArguments + return + } + if vs, sb2, ok = tokenval(vs); !ok || sb2 == "" { + err = errInvalidNumberOfArguments + return + } + var lat, lon, meters, b1, b2 float64 + if lat, err = strconv.ParseFloat(slat, 64); err != nil { + err = errInvalidArgument(slat) + return + } + if lon, err = strconv.ParseFloat(slon, 64); err != nil { + err = errInvalidArgument(slon) + return + } + if meters, err = strconv.ParseFloat(smeters, 64); err != nil { + err = errInvalidArgument(smeters) + return + } + if b1, err = strconv.ParseFloat(sb1, 64); err != nil { + err = errInvalidArgument(sb1) + return + } + if b2, err = strconv.ParseFloat(sb2, 64); err != nil { + err = errInvalidArgument(sb2) + return + } + + if b1 == b2 { + err = fmt.Errorf("equal bearings (%s == %s), use CIRCLE instead", sb1, sb2) + return + } + + origin := sectr.Point{Lng: lon, Lat: lat} + sector := sectr.NewSector(origin, meters, b1, b2) + + o, err = geojson.Parse(string(sector.JSON()), &s.geomParseOpts) + if err != nil { + return + } + case "circle": if doClip { err = fmt.Errorf("invalid clip type '%s'", typ) diff --git a/internal/server/token.go b/internal/server/token.go index a0207906..30d25bae 100644 --- a/internal/server/token.go +++ b/internal/server/token.go @@ -763,7 +763,7 @@ loop: ae = &areaExpression{op: OR, children: []*areaExpression{ae}} } vsout = nvs - case "point", "circle", "object", "bounds", "hash", "quadkey", "tile", "get": + case "point", "circle", "object", "bounds", "hash", "quadkey", "tile", "get", "sector": parsedVs, parsedObj, areaErr := s.parseArea(vsout, doClip) if areaErr != nil { err = areaErr diff --git a/tests/testcmd_test.go b/tests/testcmd_test.go index 761c3923..5556b084 100644 --- a/tests/testcmd_test.go +++ b/tests/testcmd_test.go @@ -40,10 +40,12 @@ func testcmd_WITHIN_test(mc *mockServer) error { {"SET", "mykey", "poly8", "OBJECT", poly8}, {"OK"}, {"TEST", "GET", "mykey", "point1", "WITHIN", "OBJECT", poly}, {"1"}, + {"TEST", "GET", "mykey", "point1", "WITHIN", "SECTOR", "37.72999", "-122.44760", "1000", "0", "90"}, {"1"}, {"TEST", "GET", "mykey", "line3", "WITHIN", "OBJECT", poly}, {"1"}, {"TEST", "GET", "mykey", "poly4", "WITHIN", "OBJECT", poly}, {"1"}, {"TEST", "GET", "mykey", "multipoly5", "WITHIN", "OBJECT", poly}, {"1"}, {"TEST", "GET", "mykey", "poly8", "WITHIN", "OBJECT", poly}, {"1"}, + {"TEST", "GET", "mykey", "poly8", "WITHIN", "SECTOR", "37.72999", "-122.44760", "1000", "0", "90"}, {"1"}, {"TEST", "GET", "mykey", "point6", "WITHIN", "OBJECT", poly}, {"0"}, {"TEST", "GET", "mykey", "point7", "WITHIN", "OBJECT", poly}, {"0"}, diff --git a/vendor/github.com/iwpnd/sectr/.gitignore b/vendor/github.com/iwpnd/sectr/.gitignore new file mode 100644 index 00000000..398949e8 --- /dev/null +++ b/vendor/github.com/iwpnd/sectr/.gitignore @@ -0,0 +1,19 @@ +.ipynb_checkpoints/ +/tmp/ + +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib +.DS_Store + +# Test binary, built with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Dependency directories (remove the comment below to include it) +# vendor/ diff --git a/vendor/github.com/iwpnd/sectr/.gitlint b/vendor/github.com/iwpnd/sectr/.gitlint new file mode 100644 index 00000000..47c85054 --- /dev/null +++ b/vendor/github.com/iwpnd/sectr/.gitlint @@ -0,0 +1,119 @@ +# Edit this file as you like. +# +# All these sections are optional. Each section with the exception of [general] represents +# one rule and each key in it is an option for that specific rule. +# +# Rules and sections can be referenced by their full name or by id. For example +# section "[body-max-line-length]" could also be written as "[B1]". Full section names are +# used in here for clarity. +# +[general] +# Ignore certain rules, this example uses both full name and id +# ignore=title-trailing-punctuation, T3 + +# verbosity should be a value between 1 and 3, the commandline -v flags take precedence over this +# verbosity = 2 + +# By default gitlint will ignore merge, revert, fixup and squash commits. +# ignore-merge-commits=true +# ignore-revert-commits=true +# ignore-fixup-commits=true +# ignore-squash-commits=true + +# Ignore any data send to gitlint via stdin +# ignore-stdin=true + +# Fetch additional meta-data from the local repository when manually passing a +# commit message to gitlint via stdin or --commit-msg. Disabled by default. +# staged=true + +# Enable debug mode (prints more output). Disabled by default. +# debug=true + +# Enable community contributed rules +# See http://jorisroovers.github.io/gitlint/contrib_rules for details +contrib=contrib-title-conventional-commits + +# Set the extra-path where gitlint will search for user defined rules +# See http://jorisroovers.github.io/gitlint/user_defined_rules for details +# extra-path=examples/ + +# This is an example of how to configure the "title-max-length" rule and +# set the line-length it enforces to 80 +# [title-max-length] +# line-length=50 + +# Conversely, you can also enforce minimal length of a title with the +# "title-min-length" rule: +# [title-min-length] +# min-length=5 + +# [title-must-not-contain-word] +# Comma-separated list of words that should not occur in the title. Matching is case +# insensitive. It's fine if the keyword occurs as part of a larger word (so "WIPING" +# will not cause a violation, but "WIP: my title" will. +# words=wip + +# [title-match-regex] +# python-style regex that the commit-msg title must match +# Note that the regex can contradict with other rules if not used correctly +# (e.g. title-must-not-contain-word). +# regex=^US[0-9]* + +# [body-max-line-length] +# line-length=72 + +# [body-min-length] +# min-length=5 + +#[body-is-missing] +# Whether to ignore this rule on merge commits (which typically only have a title) +# default = True +#ignore-merge-commits=false + +# [body-changed-file-mention] +# List of files that need to be explicitly mentioned in the body when they are changed +# This is useful for when developers often erroneously edit certain files or git submodules. +# By specifying this rule, developers can only change the file when they explicitly reference +# it in the commit message. +# files=gitlint/rules.py,README.md + +# [body-match-regex] +# python-style regex that the commit-msg body must match. +# E.g. body must end in My-Commit-Tag: foo +# regex=My-Commit-Tag: foo$ + +# [author-valid-email] +# python-style regex that the commit author email address must match. +# For example, use the following regex if you only want to allow email addresses from foo.com +# regex=[^@]+@foo.com + +# [ignore-by-title] +# Ignore certain rules for commits of which the title matches a regex +# E.g. Match commit titles that start with "Release" +# regex=^Release(.*) + +# Ignore certain rules, you can reference them by their id or by their full name +# Use 'all' to ignore all rules +ignore=B6,CC1 + +# [ignore-by-body] +# Ignore certain rules for commits of which the body has a line that matches a regex +# E.g. Match bodies that have a line that that contain "release" +# regex=(.*)release(.*) +# +# Ignore certain rules, you can reference them by their id or by their full name +# Use 'all' to ignore all rules +# ignore=T1,body-min-length + +# [ignore-body-lines] +# Ignore certain lines in a commit body that match a regex. +# E.g. Ignore all lines that start with 'Co-Authored-By' +# regex=^Co-Authored-By + +# This is a contrib rule - a community contributed rule. These are disabled by default. +# You need to explicitly enable them one-by-one by adding them to the "contrib" option +# under [general] section above. +# [contrib-title-conventional-commits] +# Specify allowed commit types. For details see: https://www.conventionalcommits.org/ + #types = feat,fix,refactor,docs,ci,chore diff --git a/vendor/github.com/iwpnd/sectr/.pre-commit-config.yaml b/vendor/github.com/iwpnd/sectr/.pre-commit-config.yaml new file mode 100644 index 00000000..2191cec1 --- /dev/null +++ b/vendor/github.com/iwpnd/sectr/.pre-commit-config.yaml @@ -0,0 +1,13 @@ +repos: +- repo: git://github.com/dnephin/pre-commit-golang + rev: v0.4.0 + hooks: + - id: go-fmt + - id: go-vet + - id: go-lint + - id: go-imports + - id: go-mod-tidy +- repo: https://github.com/jorisroovers/gitlint + rev: v0.15.1 + hooks: + - id: gitlint diff --git a/vendor/github.com/iwpnd/sectr/go.mod b/vendor/github.com/iwpnd/sectr/go.mod new file mode 100644 index 00000000..2678b25e --- /dev/null +++ b/vendor/github.com/iwpnd/sectr/go.mod @@ -0,0 +1,3 @@ +module github.com/iwpnd/sectr + +go 1.16 diff --git a/vendor/github.com/iwpnd/sectr/readme.md b/vendor/github.com/iwpnd/sectr/readme.md new file mode 100644 index 00000000..fa662df1 --- /dev/null +++ b/vendor/github.com/iwpnd/sectr/readme.md @@ -0,0 +1,71 @@ +# sectr 🍕 + +Build a circular sector (pizza piece 😅 ) spanning the angle between two given bearings, a radius and a center point. + +## installation + +``` +go get -u github.com/iwpnd/sectr +``` + +## usage + +Logo + +```go +package main + +import ( + "fmt" + + "github.com/iwpnd/sectr" + ) + +func main() { + p := sectr.Point{Lat: 52.25, Lng: 13.37} + sector := sectr.NewSector(p, 100, 0, 90) + + fmt.Printf("%s", sector.JSON()) +} + +>> { + "type": "Polygon", + "coordinates": [ + [ + [13.37,52.25], + [13.37,52.25089932], + [13.37012803,52.2508959], + [13.3702803,52.2508828], + [13.37040491,52.25086448], + [13.37055029,52.25083383], + [13.37068965,52.25079405], + [13.37080006,52.25075423], + [13.37092446,52.2506989], + [13.37103872,52.25063591], + [13.3711253,52.25057807], + [13.37121783,52.25050289], + [13.37128479,52.25043599], + [13.37135219,52.25035138], + [13.37140478,52.25026293], + [13.37143686,52.25018697], + [13.37146091,52.250094], + [13.37146896,52.24999999], + [13.37,52.25] + ] + ] +} +``` + +## License + +MIT + +## Acknowledgement + +[Chris Veness](https://github.com/chrisveness) for refreshing my university left-overs with this blog [moveable-type](https://www.movable-type.co.uk/scripts/latlong.html) + +## Maintainer + +Benjamin Ramser - [@iwpnd](https://github.com/iwpnd) + +Project Link: [https://github.com/iwpnd/sectr](https://github.com/iwpnd/sectr) diff --git a/vendor/github.com/iwpnd/sectr/sectr.go b/vendor/github.com/iwpnd/sectr/sectr.go new file mode 100644 index 00000000..44658259 --- /dev/null +++ b/vendor/github.com/iwpnd/sectr/sectr.go @@ -0,0 +1,160 @@ +package sectr + +import ( + "encoding/json" + "math" +) + +const earthRadius = 6371008.8 // earth radius + +// Point ... +type Point struct { + Lng, Lat float64 +} + +// Sector ... +type Sector struct { + coordinates [][][]float64 + origin Point + radius float64 + bearing1 float64 + bearing2 float64 +} + +// SectorGeometry ... +type SectorGeometry struct { + Type string `json:"type"` + Coordinates [][][]float64 `json:"coordinates"` +} + +func radToDegree(rad float64) float64 { + return rad * 180 / math.Pi +} + +func degreeToRad(degree float64) float64 { + return degree * math.Pi / 180 +} + +func distanceToRadians(distance float64) float64 { + const r = earthRadius + + return distance / r +} + +// terminal calculates the terminal position travelling a distance +// from a given origin +// see https://www.movable-type.co.uk/scripts/latlong.html +func terminal(start Point, distance, bearing float64) Point { + φ1 := degreeToRad(start.Lat) + λ1 := degreeToRad(start.Lng) + bearingRad := degreeToRad(bearing) + distanceRad := distanceToRadians(distance) + + φ2 := math.Asin( + math.Sin(φ1)* + math.Cos(distanceRad) + + math.Cos(φ1)* + math.Sin(distanceRad)* + math.Cos(bearingRad)) + + λ2 := λ1 + math.Atan2( + math.Sin(bearingRad)* + math.Sin(distanceRad)* + math.Cos(φ1), + math.Cos(distanceRad)- + math.Sin(φ1)* + math.Sin(φ2)) + + // cap decimals at .00000001 degree ~= 1.11mm + lng := math.Round(radToDegree(λ2)*100000000) / 100000000 + lat := math.Round(radToDegree(φ2)*100000000) / 100000000 + + return Point{Lng: lng, Lat: lat} +} + +func bearingToAngle(bearing float64) float64 { + angle := math.Mod(bearing, 360) + + if angle < 0 { + angle = angle + 360 + } + + return angle +} + +// NewSector creates a sector from a given origin point, a radius and two bearings +func NewSector(origin Point, radius, bearing1, bearing2 float64) *Sector { + + // to cap the maximum positions in a sector/circle to 64 + // the higher the smoother, yet the bigger the coordinate array + const steps = 64 + + s := &Sector{ + origin: origin, + bearing1: bearing1, + bearing2: bearing2, + radius: radius, + } + + angle1 := bearingToAngle(bearing1) + angle2 := bearingToAngle(bearing2) + + // if angle1 == angle2 return circle + if angle1 == angle2 { + for i := 1; i < steps; i++ { + α := float64(i * -360 / steps) + t := terminal(origin, radius, α) + s.addPoint(t) + } + + s.coordinates[0] = append(s.coordinates[0], s.coordinates[0][0]) + + return s + } + + var endDegree float64 + startDegree := angle1 + + if angle1 < angle2 { + endDegree = angle2 + } else { + endDegree = angle2 + 360 + } + + α := startDegree + + s.addPoint(origin) + + for i := 1; ; i++ { + if α < endDegree { + t := terminal(origin, radius, α) + s.addPoint(t) + α = startDegree + float64((i*360)/steps) + } + + if α >= endDegree { + t := terminal(origin, radius, endDegree) + s.addPoint(t) + s.addPoint(origin) + + return s + } + } +} + +func (s *Sector) addPoint(p Point) { + if len(s.coordinates) == 0 { + s.coordinates = append(s.coordinates, [][]float64{{p.Lng, p.Lat}}) + return + } + + s.coordinates[0] = append(s.coordinates[0], []float64{p.Lng, p.Lat}) +} + +// JSON exports the Sector as json +func (s Sector) JSON() []byte { + f := SectorGeometry{Type: "Polygon", Coordinates: s.coordinates} + j, _ := json.Marshal(f) + + return j +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 2572508a..274d9cd2 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -103,6 +103,9 @@ github.com/googleapis/gax-go/v2 github.com/gorilla/websocket # github.com/hashicorp/go-uuid v1.0.2 github.com/hashicorp/go-uuid +# github.com/iwpnd/sectr v0.1.2 +## explicit +github.com/iwpnd/sectr # github.com/jcmturner/gofork v1.0.0 github.com/jcmturner/gofork/encoding/asn1 github.com/jcmturner/gofork/x/crypto/pbkdf2 From 6eb1cca78c553bb7ba702e6d5b3cf446047fc380 Mon Sep 17 00:00:00 2001 From: Benjamin Ramser Date: Sun, 5 Sep 2021 12:05:15 +0200 Subject: [PATCH 02/15] fix: point -> circle fallthrough --- internal/server/search.go | 124 +++++++++++++++++++------------------- tests/keys_search_test.go | 1 + 2 files changed, 62 insertions(+), 63 deletions(-) diff --git a/internal/server/search.go b/internal/server/search.go index c37cc975..a62f89ca 100644 --- a/internal/server/search.go +++ b/internal/server/search.go @@ -216,67 +216,6 @@ func (server *Server) cmdSearchArgs( switch ltyp { case "point": fallthrough - case "sector": - if s.clip { - err = errInvalidArgument("cannot clip with " + ltyp) - return - } - var slat, slon, smeters, sb1, sb2 string - if vs, slat, ok = tokenval(vs); !ok || slat == "" { - err = errInvalidNumberOfArguments - return - } - if vs, slon, ok = tokenval(vs); !ok || slon == "" { - err = errInvalidNumberOfArguments - return - } - if vs, smeters, ok = tokenval(vs); !ok || smeters == "" { - err = errInvalidNumberOfArguments - return - } - if vs, sb1, ok = tokenval(vs); !ok || sb1 == "" { - err = errInvalidNumberOfArguments - return - } - if vs, sb2, ok = tokenval(vs); !ok || sb2 == "" { - err = errInvalidNumberOfArguments - return - } - var lat, lon, meters, b1, b2 float64 - if lat, err = strconv.ParseFloat(slat, 64); err != nil { - err = errInvalidArgument(slat) - return - } - if lon, err = strconv.ParseFloat(slon, 64); err != nil { - err = errInvalidArgument(slon) - return - } - if meters, err = strconv.ParseFloat(smeters, 64); err != nil { - err = errInvalidArgument(smeters) - return - } - if b1, err = strconv.ParseFloat(sb1, 64); err != nil { - err = errInvalidArgument(sb1) - return - } - if b2, err = strconv.ParseFloat(sb2, 64); err != nil { - err = errInvalidArgument(sb2) - return - } - - if b1 == b2 { - err = fmt.Errorf("equal bearings (%s == %s), use CIRCLE instead", sb1, sb2) - return - } - - origin := sectr.Point{Lng: lon, Lat: lat} - sector := sectr.NewSector(origin, meters, b1, b2) - - s.obj, err = geojson.Parse(string(sector.JSON()), &server.geomParseOpts) - if err != nil { - return - } - case "circle": if s.clip { err = errInvalidArgument("cannot clip with " + ltyp) @@ -343,6 +282,66 @@ func (server *Server) cmdSearchArgs( if err != nil { return } + case "sector": + if s.clip { + err = errInvalidArgument("cannot clip with " + ltyp) + return + } + var slat, slon, smeters, sb1, sb2 string + if vs, slat, ok = tokenval(vs); !ok || slat == "" { + err = errInvalidNumberOfArguments + return + } + if vs, slon, ok = tokenval(vs); !ok || slon == "" { + err = errInvalidNumberOfArguments + return + } + if vs, smeters, ok = tokenval(vs); !ok || smeters == "" { + err = errInvalidNumberOfArguments + return + } + if vs, sb1, ok = tokenval(vs); !ok || sb1 == "" { + err = errInvalidNumberOfArguments + return + } + if vs, sb2, ok = tokenval(vs); !ok || sb2 == "" { + err = errInvalidNumberOfArguments + return + } + var lat, lon, meters, b1, b2 float64 + if lat, err = strconv.ParseFloat(slat, 64); err != nil { + err = errInvalidArgument(slat) + return + } + if lon, err = strconv.ParseFloat(slon, 64); err != nil { + err = errInvalidArgument(slon) + return + } + if meters, err = strconv.ParseFloat(smeters, 64); err != nil { + err = errInvalidArgument(smeters) + return + } + if b1, err = strconv.ParseFloat(sb1, 64); err != nil { + err = errInvalidArgument(sb1) + return + } + if b2, err = strconv.ParseFloat(sb2, 64); err != nil { + err = errInvalidArgument(sb2) + return + } + + if b1 == b2 { + err = fmt.Errorf("equal bearings (%s == %s), use CIRCLE instead", sb1, sb2) + return + } + + origin := sectr.Point{Lng: lon, Lat: lat} + sector := sectr.NewSector(origin, meters, b1, b2) + + s.obj, err = geojson.Parse(string(sector.JSON()), &server.geomParseOpts) + if err != nil { + return + } case "bounds", "hash", "tile", "quadkey": vs, s.obj, err = parseRectArea(ltyp, vs) if err != nil { @@ -442,8 +441,7 @@ func (server *Server) cmdSearchArgs( var nearbyTypes = []string{"point"} var withinOrIntersectsTypes = []string{ - "geo", "bounds", "hash", "tile", "quadkey", "get", "object", "circle", "sector", -} + "geo", "bounds", "hash", "tile", "quadkey", "get", "object", "circle", "sector"} func (server *Server) cmdNearby(msg *Message) (res resp.Value, err error) { start := time.Now() diff --git a/tests/keys_search_test.go b/tests/keys_search_test.go index 7853bc9c..f1c680aa 100644 --- a/tests/keys_search_test.go +++ b/tests/keys_search_test.go @@ -131,6 +131,7 @@ func keys_WITHIN_test(mc *mockServer) error { {"SET", "mykey", "point6", "POINT", -5, 5}, {"OK"}, {"SET", "mykey", "point7", "POINT", 33, 21}, {"OK"}, {"SET", "mykey", "poly8", "OBJECT", `{"type":"Polygon","coordinates":[[[-122.4408378,37.7341129],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.7341129],[-122.4408378,37.7341129]],[[-122.44060993194579,37.73345766902749],[-122.44044363498686,37.73345766902749],[-122.44044363498686,37.73355524732416],[-122.44060993194579,37.73355524732416],[-122.44060993194579,37.73345766902749]],[[-122.44060724973677,37.7336888869566],[-122.4402102828026,37.7336888869566],[-122.4402102828026,37.7339752567853],[-122.44060724973677,37.7339752567853],[-122.44060724973677,37.7336888869566]]]}`}, {"OK"}, + {"WITHIN", "mykey", "IDS", "SECTOR", "37.72999", "-122.44760", "1000", "0", "90"}, {"[0 [point1 point2 line3 poly4 multipoly5 poly8]]"}, {"WITHIN", "mykey", "IDS", "OBJECT", `{ "type": "Polygon", From c87fd4836ecf626e457499ba792fe0e5feddf2b9 Mon Sep 17 00:00:00 2001 From: Benjamin Ramser Date: Sun, 5 Sep 2021 12:56:45 +0200 Subject: [PATCH 03/15] test: add more tests --- tests/keys_search_test.go | 43 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/tests/keys_search_test.go b/tests/keys_search_test.go index f1c680aa..11096ea4 100644 --- a/tests/keys_search_test.go +++ b/tests/keys_search_test.go @@ -19,7 +19,9 @@ func subTestSearch(t *testing.T, mc *mockServer) { runStep(t, mc, "KNN_CURSOR", keys_KNN_cursor_test) runStep(t, mc, "NEARBY_SPARSE", keys_NEARBY_SPARSE_test) runStep(t, mc, "WITHIN_CIRCLE", keys_WITHIN_CIRCLE_test) + runStep(t, mc, "WITHIN_SECTOR", keys_WITHIN_SECTOR_test) runStep(t, mc, "INTERSECTS_CIRCLE", keys_INTERSECTS_CIRCLE_test) + runStep(t, mc, "INTERSECTS_SECTOR", keys_INTERSECTS_SECTOR_test) runStep(t, mc, "WITHIN", keys_WITHIN_test) runStep(t, mc, "WITHIN_CURSOR", keys_WITHIN_CURSOR_test) runStep(t, mc, "WITHIN_CLIPBY", keys_WITHIN_CLIPBY_test) @@ -131,7 +133,6 @@ func keys_WITHIN_test(mc *mockServer) error { {"SET", "mykey", "point6", "POINT", -5, 5}, {"OK"}, {"SET", "mykey", "point7", "POINT", 33, 21}, {"OK"}, {"SET", "mykey", "poly8", "OBJECT", `{"type":"Polygon","coordinates":[[[-122.4408378,37.7341129],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.7341129],[-122.4408378,37.7341129]],[[-122.44060993194579,37.73345766902749],[-122.44044363498686,37.73345766902749],[-122.44044363498686,37.73355524732416],[-122.44060993194579,37.73355524732416],[-122.44060993194579,37.73345766902749]],[[-122.44060724973677,37.7336888869566],[-122.4402102828026,37.7336888869566],[-122.4402102828026,37.7339752567853],[-122.44060724973677,37.7339752567853],[-122.44060724973677,37.7336888869566]]]}`}, {"OK"}, - {"WITHIN", "mykey", "IDS", "SECTOR", "37.72999", "-122.44760", "1000", "0", "90"}, {"[0 [point1 point2 line3 poly4 multipoly5 poly8]]"}, {"WITHIN", "mykey", "IDS", "OBJECT", `{ "type": "Polygon", @@ -145,6 +146,8 @@ func keys_WITHIN_test(mc *mockServer) error { ] ] }`}, {"[0 [point1 point2 line3 poly4 multipoly5 poly8]]"}, + {"WITHIN", "mykey", "IDS", "SECTOR", "37.72999", "-122.44760", "1000", "0", "90"}, {"[0 [point1 point2 line3 poly4 multipoly5 poly8]]"}, + {"WITHIN", "mykey", "IDS", "SECTOR", "37.72999", "-122.44760", "1000", "0", "0"}, {"ERR equal bearings (0 == 0), use CIRCLE instead"}, {"SET", "key2", "poly9", "OBJECT", `{"type":"Polygon","coordinates":[[[-122.44037926197052,37.73313523548048],[-122.44017541408539,37.73313523548048],[-122.44017541408539,37.73336857568778],[-122.44037926197052,37.73336857568778],[-122.44037926197052,37.73313523548048]]]}`}, {"OK"}, {"SET", "key2", "poly10", "OBJECT", `{"type":"Polygon","coordinates":[[[-122.44040071964262,37.73359343010089],[-122.4402666091919,37.73359343010089],[-122.4402666091919,37.73373767596864],[-122.44040071964262,37.73373767596864],[-122.44040071964262,37.73359343010089]]]}`}, {"OK"}, @@ -204,6 +207,8 @@ func keys_WITHIN_CURSOR_test(mc *mockServer) error { "[6 [poly8]]"}, {"WITHIN", "mykey", "CURSOR", 6, "LIMIT", 1, "WHERE", "foo", 8, 9, "IDS", "OBJECT", testArea}, { "[7 [point9]]"}, + {"WITHIN", "mykey", "CURSOR", 6, "LIMIT", 1, "WHERE", "foo", 8, 9, "IDS", "SECTOR", "37.72999", "-122.44760", "1000", "0", "90"}, { + "[7 [point9]]"}, }) } @@ -274,6 +279,8 @@ func keys_INTERSECTS_test(mc *mockServer) error { ] ] }`}, {"[0 [point1 point2 line3 poly4 multipoly5 poly8]]"}, + {"INTERSECTS", "mykey", "IDS", "SECTOR", "37.72999", "-122.44760", "1000", "0", "90"}, {"[0 [point1 point2 line3 poly4 multipoly5 poly8]]"}, + {"INTERSECTS", "mykey", "IDS", "SECTOR", "37.72999", "-122.44760", "1000", "0", "0"}, {"ERR equal bearings (0 == 0), use CIRCLE instead"}, {"SET", "key2", "poly9", "OBJECT", `{"type": "Polygon","coordinates": [[[-122.44037926197052,37.73313523548048],[-122.44017541408539,37.73313523548048],[-122.44017541408539,37.73336857568778],[-122.44037926197052,37.73336857568778],[-122.44037926197052,37.73313523548048]]]}`}, {"OK"}, {"SET", "key2", "poly10", "OBJECT", `{"type": "Polygon","coordinates": [[[-122.44040071964262,37.73359343010089],[-122.4402666091919,37.73359343010089],[-122.4402666091919,37.73373767596864],[-122.44040071964262,37.73373767596864],[-122.44040071964262,37.73359343010089]]]}`}, {"OK"}, @@ -380,6 +387,8 @@ func keys_INTERSECTS_CURSOR_test(mc *mockServer) error { "[6 [poly8]]"}, {"INTERSECTS", "mykey", "CURSOR", 6, "LIMIT", 1, "WHERE", "foo", 8, 9, "IDS", "OBJECT", testArea}, { "[7 [point9]]"}, + {"INTERSECTS", "mykey", "CURSOR", 6, "LIMIT", 1, "WHERE", "foo", 8, 9, "IDS", "SECTOR", "37.72999", "-122.44760", "1000", "0", "90"}, { + "[7 [point9]]"}, }) } @@ -399,6 +408,22 @@ func keys_WITHIN_CIRCLE_test(mc *mockServer) error { }) } +func keys_WITHIN_SECTOR_test(mc *mockServer) error { + return mc.DoBatch([][]interface{}{ + {"SET", "mykey", "1", "POINT", 37.7324, -122.4424}, {"OK"}, + {"SET", "mykey", "2", "POINT", 37.73241, -122.44241}, {"OK"}, + {"SET", "mykey", "3", "OBJECT", `{"type":"LineString","coordinates":[[-122.4408378,37.7341129],[-122.4408378,37.733]]}`}, {"OK"}, + {"SET", "mykey", "4", "OBJECT", `{"type":"Polygon","coordinates":[[[-122.4408378,37.7341129],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.7341129],[-122.4408378,37.7341129]]]}`}, {"OK"}, + {"SET", "mykey", "5", "OBJECT", `{"type":"MultiPolygon","coordinates":[[[[-122.4408378,37.7341129],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.7341129],[-122.4408378,37.7341129]]]]}`}, {"OK"}, + {"SET", "mykey", "6", "POINT", -5, 5}, {"OK"}, + {"SET", "mykey", "7", "POINT", 33, 21}, {"OK"}, + {"WITHIN", "mykey", "IDS", "SECTOR", 37.731930, -122.443270, 1000, 0, 90}, { + "[0 [1 2 3 4 5]]"}, + {"WITHIN", "mykey", "IDS", "SECTOR", 37.731930, -122.443270, 100, 0, 90}, { + "[0 [1 2]]"}, + }) +} + func keys_NEARBY_SPARSE_test(mc *mockServer) error { // https://github.com/tidwall/tile38/issues/618 return mc.DoBatch([][]interface{}{ @@ -498,6 +523,22 @@ func keys_INTERSECTS_CIRCLE_test(mc *mockServer) error { }) } +func keys_INTERSECTS_SECTOR_test(mc *mockServer) error { + return mc.DoBatch([][]interface{}{ + {"SET", "mykey", "1", "POINT", 37.7324, -122.4424}, {"OK"}, + {"SET", "mykey", "2", "POINT", 37.73241, -122.44241}, {"OK"}, + {"SET", "mykey", "3", "OBJECT", `{"type":"LineString","coordinates":[[-122.4408378,37.7341129],[-122.4408378,37.733]]}`}, {"OK"}, + {"SET", "mykey", "4", "OBJECT", `{"type":"Polygon","coordinates":[[[-122.4408378,37.7341129],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.7341129],[-122.4408378,37.7341129]]]}`}, {"OK"}, + {"SET", "mykey", "5", "OBJECT", `{"type":"MultiPolygon","coordinates":[[[[-122.4408378,37.7341129],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.7341129],[-122.4408378,37.7341129]]]]}`}, {"OK"}, + {"SET", "mykey", "6", "POINT", -5, 5}, {"OK"}, + {"SET", "mykey", "7", "POINT", 33, 21}, {"OK"}, + {"INTERSECTS", "mykey", "IDS", "SECTOR", 37.731930, -122.443270, 1000, 0, 90}, { + "[0 [1 2 3 4 5]]"}, + {"INTERSECTS", "mykey", "IDS", "SECTOR", 37.731930, -122.443270, 100, 0, 90}, { + "[0 [1 2]]"}, + }) +} + func keys_SCAN_CURSOR_test(mc *mockServer) error { return mc.DoBatch([][]interface{}{ {"SET", "mykey", "id1", "FIELD", "foo", 1, "STRING", "bar1"}, {"OK"}, From f9e6f74753a2157572c9f2d6c60510e529ef2751 Mon Sep 17 00:00:00 2001 From: tidwall Date: Mon, 6 Sep 2021 08:55:13 -0700 Subject: [PATCH 04/15] Use struct for server options --- cmd/tile38-server/main.go | 9 ++++++++- internal/server/server.go | 32 ++++++++++++++++++++------------ tests/107/main.go | 9 ++++++++- tests/mock_test.go | 9 ++++++++- 4 files changed, 44 insertions(+), 15 deletions(-) diff --git a/cmd/tile38-server/main.go b/cmd/tile38-server/main.go index 4b2e14f4..04beab45 100644 --- a/cmd/tile38-server/main.go +++ b/cmd/tile38-server/main.go @@ -419,7 +419,14 @@ Developer Options: if showThreadsDisabled { log.Warnf("thread flag is deprecated use GOMAXPROCS to set number of threads instead") } - if err := server.Serve(host, port, dir, httpTransport, *metricsAddr); err != nil { + opts := server.Options{ + Host: host, + Port: port, + Dir: dir, + UseHTTP: httpTransport, + MetricsAddr: *metricsAddr, + } + if err := server.Serve(opts); err != nil { log.Fatal(err) } } diff --git a/internal/server/server.go b/internal/server/server.go index d4e44c4f..922e52c4 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -134,21 +134,29 @@ type Server struct { monconns map[net.Conn]bool // monitor connections } +type Options struct { + Host string + Port int + Dir string + UseHTTP bool + MetricsAddr string +} + // Serve starts a new tile38 server -func Serve(host string, port int, dir string, useHTTP bool, metricsAddr string) error { +func Serve(opts Options) error { if core.AppendFileName == "" { - core.AppendFileName = path.Join(dir, "appendonly.aof") + core.AppendFileName = path.Join(opts.Dir, "appendonly.aof") } if core.QueueFileName == "" { - core.QueueFileName = path.Join(dir, "queue.db") + core.QueueFileName = path.Join(opts.Dir, "queue.db") } log.Infof("Server started, Tile38 version %s, git %s", core.Version, core.GitSHA) // Initialize the server server := &Server{ - host: host, - port: port, - dir: dir, + host: opts.Host, + port: opts.Port, + dir: opts.Dir, follows: make(map[*bytes.Buffer]bool), fcond: sync.NewCond(&sync.Mutex{}), lives: make(map[*liveBuffer]bool), @@ -160,7 +168,7 @@ func Serve(host string, port int, dir string, useHTTP bool, metricsAddr string) aofconnM: make(map[net.Conn]io.Closer), started: time.Now(), conns: make(map[int]*Client), - http: useHTTP, + http: opts.UseHTTP, pubsub: newPubsub(), monconns: make(map[net.Conn]bool), cols: btree.NewNonConcurrent(byCollectionKey), @@ -180,11 +188,11 @@ func Serve(host string, port int, dir string, useHTTP bool, metricsAddr string) server.luapool = server.newPool() defer server.luapool.Shutdown() - if err := os.MkdirAll(dir, 0700); err != nil { + if err := os.MkdirAll(opts.Dir, 0700); err != nil { return err } var err error - server.config, err = loadConfig(filepath.Join(dir, "config")) + server.config, err = loadConfig(filepath.Join(opts.Dir, "config")) if err != nil { return err } @@ -290,12 +298,12 @@ func Serve(host string, port int, dir string, useHTTP bool, metricsAddr string) server.followc.get()) } - if metricsAddr != "" { - log.Infof("Listening for metrics at: %s", metricsAddr) + if opts.MetricsAddr != "" { + log.Infof("Listening for metrics at: %s", opts.MetricsAddr) go func() { http.HandleFunc("/", server.MetricsIndexHandler) http.HandleFunc("/metrics", server.MetricsHandler) - log.Fatal(http.ListenAndServe(metricsAddr, nil)) + log.Fatal(http.ListenAndServe(opts.MetricsAddr, nil)) }() } diff --git a/tests/107/main.go b/tests/107/main.go index 23df4ce1..202db9f0 100644 --- a/tests/107/main.go +++ b/tests/107/main.go @@ -100,7 +100,14 @@ func main() { func startTile38Server() { log.Println("start tile38 server") - err := server.Serve("localhost", tile38Port, "data", false, "") + opts := server.Options{ + Host: "localhost", + Port: tile38Port, + Dir: "data", + UseHTTP: false, + MetricsAddr: "", + } + err := server.Serve(opts) if err != nil { log.Fatal(err) } diff --git a/tests/mock_test.go b/tests/mock_test.go index c589e6a9..f3a07cf2 100644 --- a/tests/mock_test.go +++ b/tests/mock_test.go @@ -58,7 +58,14 @@ func mockOpenServer(silent bool) (*mockServer, error) { s := &mockServer{port: port} tlog.SetOutput(logOutput) go func() { - if err := server.Serve("localhost", port, dir, true, ":4321"); err != nil { + opts := server.Options{ + Host: "localhost", + Port: port, + Dir: dir, + UseHTTP: true, + MetricsAddr: ":4321", + } + if err := server.Serve(opts); err != nil { log.Fatal(err) } }() From 43f0fbe3d5f6e7cd345d4f957d4a00be61384d36 Mon Sep 17 00:00:00 2001 From: Josh Baker Date: Mon, 6 Sep 2021 19:25:34 -0700 Subject: [PATCH 05/15] Create FUNDING.yml --- .github/FUNDING.yml | 1 + 1 file changed, 1 insertion(+) create mode 100644 .github/FUNDING.yml diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 00000000..cb89a533 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1 @@ +github: tidwall From 9b760ffdd514a0a8a9e24192b0200707650cc4ff Mon Sep 17 00:00:00 2001 From: tidwall Date: Tue, 7 Sep 2021 05:51:15 -0700 Subject: [PATCH 06/15] Add unix socket support --- cmd/tile38-server/main.go | 65 +++++++++++++++++++++++---------------- internal/server/server.go | 24 +++++++++++---- 2 files changed, 56 insertions(+), 33 deletions(-) diff --git a/cmd/tile38-server/main.go b/cmd/tile38-server/main.go index 04beab45..1fc2a9bd 100644 --- a/cmd/tile38-server/main.go +++ b/cmd/tile38-server/main.go @@ -252,6 +252,7 @@ Developer Options: dir string port int host string + unixSocket string verbose bool veryVerbose bool quiet bool @@ -261,13 +262,14 @@ Developer Options: pprofport int ) - flag.IntVar(&port, "p", 9851, "The listening port.") + flag.IntVar(&port, "p", 9851, "The listening port") flag.StringVar(&pidfile, "pidfile", "", "A file that contains the pid") - flag.StringVar(&host, "h", "", "The listening host.") - flag.StringVar(&dir, "d", "data", "The data directory.") - flag.BoolVar(&verbose, "v", false, "Enable verbose logging.") - flag.BoolVar(&quiet, "q", false, "Quiet logging. Totally silent.") - flag.BoolVar(&veryVerbose, "vv", false, "Enable very verbose logging.") + flag.StringVar(&host, "h", "", "The listening host") + flag.StringVar(&unixSocket, "s", "", "Listen on a unix socket") + flag.StringVar(&dir, "d", "data", "The data directory") + flag.BoolVar(&verbose, "v", false, "Enable verbose logging") + flag.BoolVar(&quiet, "q", false, "Quiet logging. Totally silent") + flag.BoolVar(&veryVerbose, "vv", false, "Enable very verbose logging") flag.IntVar(&pprofport, "pprofport", 0, "pprofport http at port") flag.StringVar(&cpuprofile, "cpuprofile", "", "write cpu profile to `file`") flag.StringVar(&memprofile, "memprofile", "", "write memory profile to `file`") @@ -277,7 +279,9 @@ Developer Options: if quiet { logw = ioutil.Discard } + log.SetOutput(logw) + if quiet { log.Level = 0 } else if veryVerbose { @@ -344,6 +348,10 @@ Developer Options: }() } + if unixSocket != "" { + port = 0 + } + // pid file var pidferr error var pidcleanedup bool @@ -364,9 +372,7 @@ Developer Options: } defer pidcleanup() if pidfile != "" { - pidferr := ioutil.WriteFile(pidfile, []byte(fmt.Sprintf("%d\n", os.Getpid())), 0666) - if pidferr == nil { - } + ioutil.WriteFile(pidfile, []byte(fmt.Sprintf("%d\n", os.Getpid())), 0666) } c := make(chan os.Signal, 1) @@ -395,36 +401,41 @@ Developer Options: } }() - // _____ _ _ ___ ___ - // |_ _|_| |___|_ | . | - // | | | | | -_|_ | . | - // |_| |_|_|___|___|___| + var saddr string + if unixSocket != "" { + saddr = fmt.Sprintf("Socket: %s", unixSocket) + } else { + saddr = fmt.Sprintf("Port: %d", port) + } + fmt.Fprintf(logw, ` - _______ _______ - | | | - |____ | _ | Tile38 %s%s %d bit (%s/%s) - | | | %sPort: %d, PID: %d - |____ | _ | - | | | tile38.com - |_______|_______| -`+"\n", core.Version, gitsha, strconv.IntSize, runtime.GOARCH, runtime.GOOS, hostd, port, os.Getpid()) + _____ _ _ ___ ___ + |_ _|_| |___|_ | . | Tile38 %s%s %d bit (%s/%s) + | | | | | -_|_ | . | %s%s, PID: %d + |_| |_|_|___|___|___| tile38.com + +Please consider sponsoring Tile38 development, especially if your company +benefits from this software. Visit tile38.com/sponsor today to learn more. + +`, core.Version, gitsha, strconv.IntSize, runtime.GOARCH, runtime.GOOS, hostd, + saddr, os.Getpid()) if pidferr != nil { log.Warnf("pidfile: %v", pidferr) } if showEvioDisabled { - // we don't currently support evio in Tile38 log.Warnf("evio is not currently supported") } if showThreadsDisabled { log.Warnf("thread flag is deprecated use GOMAXPROCS to set number of threads instead") } opts := server.Options{ - Host: host, - Port: port, - Dir: dir, - UseHTTP: httpTransport, - MetricsAddr: *metricsAddr, + Host: host, + Port: port, + Dir: dir, + UseHTTP: httpTransport, + MetricsAddr: *metricsAddr, + UnixSocketPath: unixSocket, } if err := server.Serve(opts); err != nil { log.Fatal(err) diff --git a/internal/server/server.go b/internal/server/server.go index 922e52c4..98df3f84 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -72,6 +72,7 @@ type commandDetails struct { // Server is a tile38 controller type Server struct { // static values + unix string host string port int http bool @@ -134,12 +135,14 @@ type Server struct { monconns map[net.Conn]bool // monitor connections } +// Options for Serve() type Options struct { - Host string - Port int - Dir string - UseHTTP bool - MetricsAddr string + Host string + Port int + Dir string + UseHTTP bool + MetricsAddr string + UnixSocketPath string // path for unix socket } // Serve starts a new tile38 server @@ -154,6 +157,7 @@ func Serve(opts Options) error { // Initialize the server server := &Server{ + unix: opts.UnixSocketPath, host: opts.Host, port: opts.Port, dir: opts.Dir, @@ -343,7 +347,15 @@ func (server *Server) isProtected() bool { } func (server *Server) netServe() error { - ln, err := net.Listen("tcp", fmt.Sprintf("%s:%d", server.host, server.port)) + var ln net.Listener + var err error + if server.unix != "" { + os.RemoveAll(server.unix) + ln, err = net.Listen("unix", server.unix) + } else { + tcpAddr := fmt.Sprintf("%s:%d", server.host, server.port) + ln, err = net.Listen("tcp", tcpAddr) + } if err != nil { return err } From 81ed3e05ed1c414869cc665d8e3dbed7163e1977 Mon Sep 17 00:00:00 2001 From: tidwall Date: Tue, 7 Sep 2021 05:53:01 -0700 Subject: [PATCH 07/15] Add unix socket flag to usage --- cmd/tile38-server/main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/tile38-server/main.go b/cmd/tile38-server/main.go index 1fc2a9bd..f7bc9503 100644 --- a/cmd/tile38-server/main.go +++ b/cmd/tile38-server/main.go @@ -75,6 +75,7 @@ Basic Options: -h hostname : listening host -p port : listening port (default: 9851) -d path : data directory (default: data) + -s socket : listen on unix socket file -q : no logging. totally silent output -v : enable verbose logging -vv : enable very verbose logging From d38aed4d2b18bba555a2dd8fa17ebf37b8e8fbde Mon Sep 17 00:00:00 2001 From: tidwall Date: Wed, 8 Sep 2021 11:07:07 -0700 Subject: [PATCH 08/15] Increase the precision of TIMEOUT This commit ensures that the TIMEOUT is always checked prior to returning data to the client, and that the elapsed command time cannot be greater than the timeout value. --- internal/server/server.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/internal/server/server.go b/internal/server/server.go index 98df3f84..56f84a61 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -965,7 +965,11 @@ func (server *Server) handleInputCommand(client *Client, msg *Message) error { } }() } - return server.command(msg, client) + res, d, err = server.command(msg, client) + if msg.Deadline != nil { + msg.Deadline.Check() + } + return res, d, err }() if res.Type() == resp.Error { return writeErr(res.String()) From 7e10a80319ddbcc0c04c45aee5efee55a5097468 Mon Sep 17 00:00:00 2001 From: tidwall Date: Sun, 12 Sep 2021 09:03:05 -0700 Subject: [PATCH 09/15] Return hook ttl with HOOKS request --- internal/server/hooks.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/internal/server/hooks.go b/internal/server/hooks.go index 20e4ded2..11d18db9 100644 --- a/internal/server/hooks.go +++ b/internal/server/hooks.go @@ -411,12 +411,20 @@ func (s *Server) cmdHooks(msg *Message, channel bool) ( buf.WriteString(`"hooks":[`) } for i, hook := range hooks { + var ttl = -1 + if !hook.expires.IsZero() { + ttl = int(hook.expires.Sub(start).Seconds()) + if ttl < 0 { + ttl = 0 + } + } if i > 0 { buf.WriteByte(',') } buf.WriteString(`{`) buf.WriteString(`"name":` + jsonString(hook.Name)) buf.WriteString(`,"key":` + jsonString(hook.Key)) + buf.WriteString(`,"ttl":` + strconv.Itoa(ttl)) if !channel { buf.WriteString(`,"endpoints":[`) for i, endpoint := range hook.Endpoints { From decafae2d7b9787214647cbc65829e36ceafa1bf Mon Sep 17 00:00:00 2001 From: tidwall Date: Sun, 12 Sep 2021 09:55:58 -0700 Subject: [PATCH 10/15] Update hook expiration logic --- internal/expire/expire.go | 114 --------------------------------- internal/expire/expire_test.go | 88 ------------------------- internal/server/crud.go | 1 + internal/server/expire.go | 88 +++++++++++++++++-------- internal/server/hooks.go | 54 +++++++--------- internal/server/server.go | 10 +-- 6 files changed, 89 insertions(+), 266 deletions(-) delete mode 100644 internal/expire/expire.go delete mode 100644 internal/expire/expire_test.go diff --git a/internal/expire/expire.go b/internal/expire/expire.go deleted file mode 100644 index a501f28a..00000000 --- a/internal/expire/expire.go +++ /dev/null @@ -1,114 +0,0 @@ -package expire - -import ( - "sync" - "time" -) - -// Item is a something that can expire -type Item interface { - Expires() time.Time -} - -// List of expireable items -type List struct { - mu sync.Mutex - queue queue - bgrun bool - Expired func(item Item) -} - -// Push an item onto the queue -func (list *List) Push(item Item) { - unix := item.Expires().UnixNano() - list.mu.Lock() - if !list.bgrun { - list.bgrun = true - go list.bg() - } - list.queue.push(unix, item) - list.mu.Unlock() -} - -func (list *List) bg() { - now := time.Now().UnixNano() - for { - list.mu.Lock() - if list.queue.len == 0 { - list.bgrun = false - list.mu.Unlock() - break - } - if now > list.queue.peek().unix { // now.After(list.queue.peek().unix) - n := list.queue.pop() - exfn := list.Expired - list.mu.Unlock() - if exfn != nil { - exfn(n.item) - } - } else { - list.mu.Unlock() - time.Sleep(time.Second / 10) - now = time.Now().UnixNano() - } - } -} - -type qnode struct { - unix int64 - item Item -} - -type queue struct { - nodes []qnode - len int -} - -func (q *queue) push(unix int64, item Item) { - if q.nodes == nil { - q.nodes = make([]qnode, 2) - } else { - q.nodes = append(q.nodes, qnode{}) - } - i := q.len + 1 - j := i / 2 - for i > 1 && q.nodes[j].unix > unix { - q.nodes[i] = q.nodes[j] - i = j - j = j / 2 - } - q.nodes[i].unix = unix - q.nodes[i].item = item - q.len++ -} - -func (q *queue) peek() qnode { - if q.len == 0 { - return qnode{} - } - return q.nodes[1] -} - -func (q *queue) pop() qnode { - if q.len == 0 { - return qnode{} - } - n := q.nodes[1] - q.nodes[1] = q.nodes[q.len] - q.len-- - var j, k int - i := 1 - for i != q.len+1 { - k = q.len + 1 - j = 2 * i - if j <= q.len && q.nodes[j].unix < q.nodes[k].unix { - k = j - } - if j+1 <= q.len && q.nodes[j+1].unix < q.nodes[k].unix { - k = j + 1 - } - q.nodes[i] = q.nodes[k] - i = k - } - return n -} diff --git a/internal/expire/expire_test.go b/internal/expire/expire_test.go deleted file mode 100644 index 1707b45b..00000000 --- a/internal/expire/expire_test.go +++ /dev/null @@ -1,88 +0,0 @@ -package expire - -import ( - "fmt" - "math/rand" - "sort" - "sync" - "testing" - "time" -) - -type testItem struct { - str string - exp time.Time -} - -func (item *testItem) Expires() time.Time { - return item.exp -} - -func TestBasic(t *testing.T) { - var list List - now := time.Now() - list.Push(&testItem{"13", now.Add(13)}) - list.Push(&testItem{"11", now.Add(11)}) - list.Push(&testItem{"14", now.Add(14)}) - list.Push(&testItem{"10", now.Add(10)}) - list.Push(&testItem{"15", now.Add(15)}) - list.Push(&testItem{"12", now.Add(12)}) - - var lunix int64 - for list.queue.len > 0 { - n2 := list.queue.pop() - if n2.unix < lunix { - t.Fatal("out of order") - } - } -} - -func TestRandomQueue(t *testing.T) { - N := 1000 - now := time.Now() - var list List - for i := 0; i < N; i++ { - list.Push(&testItem{fmt.Sprintf("%d", i), - now.Add(time.Duration(rand.Float64() * float64(time.Second)))}) - } - var items []Item - for list.queue.len > 0 { - n1 := list.queue.peek() - n2 := list.queue.pop() - if n1 != n2 { - t.Fatal("mismatch") - } - if n1.unix > n2.unix { - t.Fatal("out of order") - } - items = append(items, n2.item) - } - - if !sort.SliceIsSorted(items, func(i, j int) bool { - return items[i].Expires().Before(items[j].Expires()) - }) { - t.Fatal("out of order") - } - -} - -func TestExpires(t *testing.T) { - N := 1000 - now := time.Now() - var list List - for i := 0; i < N; i++ { - list.Push(&testItem{fmt.Sprintf("%d", i), - now.Add(time.Duration(rand.Float64() * float64(time.Second)))}) - } - var wg sync.WaitGroup - wg.Add(N) - var items []Item - list.Expired = func(item Item) { - items = append(items, item) - wg.Done() - } - wg.Wait() - if len(items) != N { - t.Fatal("wrong result") - } -} diff --git a/internal/server/crud.go b/internal/server/crud.go index 5578b316..10be2793 100644 --- a/internal/server/crud.go +++ b/internal/server/crud.go @@ -508,6 +508,7 @@ func (server *Server) cmdFlushDB(msg *Message) (res resp.Value, d commandDetails server.cols = btree.NewNonConcurrent(byCollectionKey) server.groupHooks = btree.NewNonConcurrent(byGroupHook) server.groupObjects = btree.NewNonConcurrent(byGroupObject) + server.hookExpires = btree.NewNonConcurrent(byHookExpires) server.hooks = make(map[string]*Hook) server.hooksOut = make(map[string]*Hook) server.hookTree = &rtree.RTree{} diff --git a/internal/server/expire.go b/internal/server/expire.go index 216cece3..16030c65 100644 --- a/internal/server/expire.go +++ b/internal/server/expire.go @@ -18,32 +18,70 @@ func (s *Server) backgroundExpiring() { func() { s.mu.Lock() defer s.mu.Unlock() - now := time.Now().UnixNano() - var ids []string - var msgs []*Message - s.cols.Ascend(nil, func(v interface{}) bool { - col := v.(*collectionKeyContainer) - ids = col.col.Expired(now, ids[:0]) - for _, id := range ids { - msgs = append(msgs, &Message{ - Args: []string{"del", col.key, id}, - }) - } - return true - }) - for _, msg := range msgs { - _, d, err := s.cmdDel(msg) - if err != nil { - log.Fatal(err) - } - if err := s.writeAOF(msg.Args, &d); err != nil { - log.Fatal(err) - } - } - if len(msgs) > 0 { - log.Debugf("Expired %d items\n", len(msgs)) - } + now := time.Now() + s.backgroundExpireObjects(now) + s.backgroundExpireHooks(now) }() time.Sleep(bgExpireDelay) } } + +func (s *Server) backgroundExpireObjects(now time.Time) { + nano := now.UnixNano() + var ids []string + var msgs []*Message + s.cols.Ascend(nil, func(v interface{}) bool { + col := v.(*collectionKeyContainer) + ids = col.col.Expired(nano, ids[:0]) + for _, id := range ids { + msgs = append(msgs, &Message{ + Args: []string{"del", col.key, id}, + }) + } + return true + }) + for _, msg := range msgs { + _, d, err := s.cmdDel(msg) + if err != nil { + log.Fatal(err) + } + if err := s.writeAOF(msg.Args, &d); err != nil { + log.Fatal(err) + } + } + if len(msgs) > 0 { + log.Debugf("Expired %d objects\n", len(msgs)) + } + +} + +func (s *Server) backgroundExpireHooks(now time.Time) { + var msgs []*Message + s.hookExpires.Ascend(nil, func(v interface{}) bool { + h := v.(*Hook) + if h.expires.After(now) { + return false + } + msg := &Message{} + if h.channel { + msg.Args = []string{"delchan", h.Name} + } else { + msg.Args = []string{"delhook", h.Name} + } + msgs = append(msgs, msg) + return true + }) + + for _, msg := range msgs { + _, d, err := s.cmdDelHook(msg, msg.Args[0] == "delchan") + if err != nil { + log.Fatal(err) + } + if err := s.writeAOF(msg.Args, &d); err != nil { + log.Fatal(err) + } + } + if len(msgs) > 0 { + log.Debugf("Expired %d hooks\n", len(msgs)) + } +} diff --git a/internal/server/hooks.go b/internal/server/hooks.go index 11d18db9..0a4e63d3 100644 --- a/internal/server/hooks.go +++ b/internal/server/hooks.go @@ -170,7 +170,7 @@ func (s *Server) cmdSetHook(msg *Message, chanCmd bool) ( // for good measure. prevHook.Signal() if !hook.expires.IsZero() { - s.hookex.Push(hook) + s.hookExpires.Set(hook) } switch msg.OutputType { case JSON: @@ -182,6 +182,9 @@ func (s *Server) cmdSetHook(msg *Message, chanCmd bool) ( prevHook.Close() delete(s.hooks, name) delete(s.hooksOut, name) + if !prevHook.expires.IsZero() { + s.hookExpires.Delete(prevHook) + } s.groupDisconnectHook(name) } @@ -224,7 +227,7 @@ func (s *Server) cmdSetHook(msg *Message, chanCmd bool) ( hook.Open() // Opens a goroutine to notify the hook if !hook.expires.IsZero() { - s.hookex.Push(hook) + s.hookExpires.Set(hook) } switch msg.OutputType { case JSON: @@ -235,6 +238,18 @@ func (s *Server) cmdSetHook(msg *Message, chanCmd bool) ( return NOMessage, d, nil } +func byHookExpires(a, b interface{}) bool { + ha := a.(*Hook) + hb := b.(*Hook) + if ha.expires.Before(hb.expires) { + return true + } + if ha.expires.After(hb.expires) { + return false + } + return ha.Name < hb.Name +} + func (s *Server) cmdDelHook(msg *Message, chanCmd bool) ( res resp.Value, d commandDetails, err error, ) { @@ -254,6 +269,9 @@ func (s *Server) cmdDelHook(msg *Message, chanCmd bool) ( // remove hook from maps delete(s.hooks, hook.Name) delete(s.hooksOut, hook.Name) + if !hook.expires.IsZero() { + s.hookExpires.Delete(hook) + } // remove any hook / object connections s.groupDisconnectHook(hook.Name) // remove hook from spatial index @@ -314,6 +332,9 @@ func (s *Server) cmdPDelHook(msg *Message, channel bool) ( // remove hook from maps delete(s.hooks, hook.Name) delete(s.hooksOut, hook.Name) + if !hook.expires.IsZero() { + s.hookExpires.Delete(hook) + } // remove any hook / object connections s.groupDisconnectHook(hook.Name) // remove hook from spatial index @@ -344,35 +365,6 @@ func (s *Server) cmdPDelHook(msg *Message, channel bool) ( return } -// possiblyExpireHook will evaluate a hook by it's name for expiration and -// purge it from the database if needed. This operation is called from an -// independent goroutine -func (s *Server) possiblyExpireHook(name string) { - s.mu.Lock() - if h, ok := s.hooks[name]; ok { - if !h.expires.IsZero() && time.Now().After(h.expires) { - // purge from database - msg := &Message{} - if h.channel { - msg.Args = []string{"delchan", h.Name} - } else { - msg.Args = []string{"delhook", h.Name} - } - _, d, err := s.cmdDelHook(msg, h.channel) - if err != nil { - s.mu.Unlock() - panic(err) - } - if err := s.writeAOF(msg.Args, &d); err != nil { - s.mu.Unlock() - panic(err) - } - log.Debugf("purged hook %v", h.Name) - } - } - s.mu.Unlock() -} - func (s *Server) cmdHooks(msg *Message, channel bool) ( res resp.Value, err error, ) { diff --git a/internal/server/server.go b/internal/server/server.go index 56f84a61..659157ef 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -36,7 +36,6 @@ import ( "github.com/tidwall/tile38/internal/collection" "github.com/tidwall/tile38/internal/deadline" "github.com/tidwall/tile38/internal/endpoint" - "github.com/tidwall/tile38/internal/expire" "github.com/tidwall/tile38/internal/log" ) @@ -123,13 +122,13 @@ type Server struct { hooksOut map[string]*Hook // hooks with "outside" detection groupHooks *btree.BTree // hooks that are connected to objects groupObjects *btree.BTree // objects that are connected to hooks + hookExpires *btree.BTree // queue of all hooks marked for expiration aofconnM map[net.Conn]io.Closer luascripts *lScriptMap luapool *lStatePool pubsub *pubsub - hookex expire.List monconnsMu sync.RWMutex monconns map[net.Conn]bool // monitor connections @@ -179,14 +178,9 @@ func Serve(opts Options) error { groupHooks: btree.NewNonConcurrent(byGroupHook), groupObjects: btree.NewNonConcurrent(byGroupObject), + hookExpires: btree.NewNonConcurrent(byHookExpires), } - server.hookex.Expired = func(item expire.Item) { - switch v := item.(type) { - case *Hook: - server.possiblyExpireHook(v.Name) - } - } server.epc = endpoint.NewManager(server) server.luascripts = server.newScriptMap() server.luapool = server.newPool() From 7ff0d188683c8c20879a14d8b9c29ed43b3b5b8e Mon Sep 17 00:00:00 2001 From: tidwall Date: Mon, 13 Sep 2021 10:02:36 -0700 Subject: [PATCH 11/15] Change hooks collection type from hashmap to btree This commit changes the collection type that holds all of the hooks from a hashmap to a btree. This allows for better flexibility for operations that need to perform range searches and scanning of the collection. --- internal/server/aof.go | 6 ++- internal/server/aofshrink.go | 16 ++++--- internal/server/crud.go | 15 ++++-- internal/server/hooks.go | 91 +++++++++++++++++++----------------- internal/server/server.go | 26 +++++------ internal/server/stats.go | 4 +- 6 files changed, 86 insertions(+), 72 deletions(-) diff --git a/internal/server/aof.go b/internal/server/aof.go index 73cdfcf3..6b440e21 100644 --- a/internal/server/aof.go +++ b/internal/server/aof.go @@ -224,11 +224,13 @@ func (s *Server) writeAOF(args []string, d *commandDetails) error { func (s *Server) getQueueCandidates(d *commandDetails) []*Hook { candidates := make(map[*Hook]bool) // add the hooks with "outside" detection - for _, hook := range s.hooksOut { + s.hooksOut.Ascend(nil, func(v interface{}) bool { + hook := v.(*Hook) if hook.Key == d.key { candidates[hook] = true } - } + return true + }) // look for candidates that might "cross" geofences if d.oldObj != nil && d.obj != nil && s.hookCross.Len() > 0 { r1, r2 := d.oldObj.Rect(), d.obj.Rect() diff --git a/internal/server/aofshrink.go b/internal/server/aofshrink.go index f53db56e..bd1c5f08 100644 --- a/internal/server/aofshrink.go +++ b/internal/server/aofshrink.go @@ -3,11 +3,11 @@ package server import ( "math" "os" - "sort" "strconv" "strings" "time" + "github.com/tidwall/btree" "github.com/tidwall/geojson" "github.com/tidwall/tile38/core" "github.com/tidwall/tile38/internal/collection" @@ -169,17 +169,19 @@ func (server *Server) aofshrink() { func() { server.mu.Lock() defer server.mu.Unlock() - for name := range server.hooks { - hnames = append(hnames, name) - } + hnames = make([]string, 0, server.hooks.Len()) + server.hooks.Walk(func(v []interface{}) { + for _, v := range v { + hnames = append(hnames, v.(*Hook).Name) + } + }) }() - // sort the names for consistency - sort.Strings(hnames) + var hookHint btree.PathHint for _, name := range hnames { func() { server.mu.Lock() defer server.mu.Unlock() - hook := server.hooks[name] + hook, _ := server.hooks.GetHint(name, &hookHint).(*Hook) if hook == nil { return } diff --git a/internal/server/crud.go b/internal/server/crud.go index 10be2793..5c0e1c03 100644 --- a/internal/server/crud.go +++ b/internal/server/crud.go @@ -462,12 +462,14 @@ func (server *Server) cmdRename(msg *Message, nx bool) (res resp.Value, d comman err = errKeyNotFound return } - for _, h := range server.hooks { + server.hooks.Ascend(nil, func(v interface{}) bool { + h := v.(*Hook) if h.Key == d.key || h.Key == d.newKey { err = errKeyHasHooksSet - return + return false } - } + return true + }) d.command = "rename" newCol := server.getCol(d.newKey) if newCol == nil { @@ -505,14 +507,17 @@ func (server *Server) cmdFlushDB(msg *Message) (res resp.Value, d commandDetails err = errInvalidNumberOfArguments return } + + // clear the entire database server.cols = btree.NewNonConcurrent(byCollectionKey) server.groupHooks = btree.NewNonConcurrent(byGroupHook) server.groupObjects = btree.NewNonConcurrent(byGroupObject) server.hookExpires = btree.NewNonConcurrent(byHookExpires) - server.hooks = make(map[string]*Hook) - server.hooksOut = make(map[string]*Hook) + server.hooks = btree.NewNonConcurrent(byHookName) + server.hooksOut = btree.NewNonConcurrent(byHookName) server.hookTree = &rtree.RTree{} server.hookCross = &rtree.RTree{} + d.command = "flushdb" d.updated = true d.timestamp = time.Now() diff --git a/internal/server/hooks.go b/internal/server/hooks.go index 0a4e63d3..71e5dd58 100644 --- a/internal/server/hooks.go +++ b/internal/server/hooks.go @@ -22,18 +22,8 @@ var hookLogSetDefaults = &buntdb.SetOptions{ TTL: time.Second * 30, } -type hooksByName []*Hook - -func (a hooksByName) Len() int { - return len(a) -} - -func (a hooksByName) Less(i, j int) bool { - return a[i].Name < a[j].Name -} - -func (a hooksByName) Swap(i, j int) { - a[i], a[j] = a[j], a[i] +func byHookName(a, b interface{}) bool { + return a.(*Hook).Name < b.(*Hook).Name } func (s *Server) cmdSetHook(msg *Message, chanCmd bool) ( @@ -159,7 +149,7 @@ func (s *Server) cmdSetHook(msg *Message, chanCmd bool) ( return NOMessage, d, err } - prevHook := s.hooks[name] + prevHook, _ := s.hooks.Get(&Hook{Name: name}).(*Hook) if prevHook != nil { if prevHook.channel != chanCmd { return NOMessage, d, @@ -180,8 +170,8 @@ func (s *Server) cmdSetHook(msg *Message, chanCmd bool) ( } } prevHook.Close() - delete(s.hooks, name) - delete(s.hooksOut, name) + s.hooks.Delete(prevHook) + s.hooksOut.Delete(prevHook) if !prevHook.expires.IsZero() { s.hookExpires.Delete(prevHook) } @@ -191,9 +181,9 @@ func (s *Server) cmdSetHook(msg *Message, chanCmd bool) ( d.updated = true d.timestamp = time.Now() - s.hooks[name] = hook + s.hooks.Set(hook) if hook.Fence.detect == nil || hook.Fence.detect["outside"] { - s.hooksOut[name] = hook + s.hooksOut.Set(hook) } // remove previous hook from spatial index @@ -264,11 +254,12 @@ func (s *Server) cmdDelHook(msg *Message, chanCmd bool) ( if len(vs) != 0 { return NOMessage, d, errInvalidNumberOfArguments } - if hook, ok := s.hooks[name]; ok && hook.channel == chanCmd { + hook, _ := s.hooks.Get(&Hook{Name: name}).(*Hook) + if hook != nil && hook.channel == chanCmd { hook.Close() // remove hook from maps - delete(s.hooks, hook.Name) - delete(s.hooksOut, hook.Name) + s.hooks.Delete(hook) + s.hooksOut.Delete(hook) if !hook.expires.IsZero() { s.hookExpires.Delete(hook) } @@ -320,18 +311,20 @@ func (s *Server) cmdPDelHook(msg *Message, channel bool) ( } count := 0 - for name, hook := range s.hooks { + var hooks []*Hook + s.forEachHookByPattern(pattern, channel, func(hook *Hook) bool { + hooks = append(hooks, hook) + return true + }) + + for _, hook := range hooks { if hook.channel != channel { continue } - match, _ := glob.Match(pattern, name) - if !match { - continue - } hook.Close() // remove hook from maps - delete(s.hooks, hook.Name) - delete(s.hooksOut, hook.Name) + s.hooks.Delete(hook) + s.hooksOut.Delete(hook) if !hook.expires.IsZero() { s.hookExpires.Delete(hook) } @@ -365,6 +358,26 @@ func (s *Server) cmdPDelHook(msg *Message, channel bool) ( return } +func (s *Server) forEachHookByPattern( + pattern string, channel bool, iter func(hook *Hook) bool, +) { + g := glob.Parse(pattern, false) + hasUpperLimit := g.Limits[1] != "" + s.hooks.Ascend(&Hook{Name: g.Limits[0]}, func(v interface{}) bool { + hook := v.(*Hook) + if hasUpperLimit && hook.Name > g.Limits[1] { + return false + } + if hook.channel == channel { + match, _ := glob.Match(pattern, hook.Name) + if match { + return iter(hook) + } + } + return true + }) +} + func (s *Server) cmdHooks(msg *Message, channel bool) ( res resp.Value, err error, ) { @@ -381,18 +394,6 @@ func (s *Server) cmdHooks(msg *Message, channel bool) ( return NOMessage, errInvalidNumberOfArguments } - var hooks []*Hook - for name, hook := range s.hooks { - if hook.channel != channel { - continue - } - match, _ := glob.Match(pattern, name) - if match { - hooks = append(hooks, hook) - } - } - sort.Sort(hooksByName(hooks)) - switch msg.OutputType { case JSON: buf := &bytes.Buffer{} @@ -402,7 +403,8 @@ func (s *Server) cmdHooks(msg *Message, channel bool) ( } else { buf.WriteString(`"hooks":[`) } - for i, hook := range hooks { + var i int + s.forEachHookByPattern(pattern, channel, func(hook *Hook) bool { var ttl = -1 if !hook.expires.IsZero() { ttl = int(hook.expires.Sub(start).Seconds()) @@ -444,13 +446,15 @@ func (s *Server) cmdHooks(msg *Message, channel bool) ( buf.WriteString(jsonString(meta.Value)) } buf.WriteString(`}}`) - } + i++ + return true + }) buf.WriteString(`],"elapsed":"` + time.Since(start).String() + "\"}") return resp.StringValue(buf.String()), nil case RESP: var vals []resp.Value - for _, hook := range hooks { + s.forEachHookByPattern(pattern, channel, func(hook *Hook) bool { var hvals []resp.Value hvals = append(hvals, resp.StringValue(hook.Name)) hvals = append(hvals, resp.StringValue(hook.Key)) @@ -471,7 +475,8 @@ func (s *Server) cmdHooks(msg *Message, channel bool) ( } hvals = append(hvals, resp.ArrayValue(metas)) vals = append(vals, resp.ArrayValue(hvals)) - } + return true + }) return resp.ArrayValue(vals), nil } return resp.SimpleStringValue(""), nil diff --git a/internal/server/server.go b/internal/server/server.go index 659157ef..831589ce 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -112,17 +112,17 @@ type Server struct { lstack []*commandDetails lives map[*liveBuffer]bool lcond *sync.Cond - fcup bool // follow caught up - fcuponce bool // follow caught up once - shrinking bool // aof shrinking flag - shrinklog [][]string // aof shrinking log - hooks map[string]*Hook // hook name - hookCross *rtree.RTree // hook spatial tree for "cross" geofences - hookTree *rtree.RTree // hook spatial tree for all - hooksOut map[string]*Hook // hooks with "outside" detection - groupHooks *btree.BTree // hooks that are connected to objects - groupObjects *btree.BTree // objects that are connected to hooks - hookExpires *btree.BTree // queue of all hooks marked for expiration + fcup bool // follow caught up + fcuponce bool // follow caught up once + shrinking bool // aof shrinking flag + shrinklog [][]string // aof shrinking log + hooks *btree.BTree // hook name -- [string]*Hook + hookCross *rtree.RTree // hook spatial tree for "cross" geofences + hookTree *rtree.RTree // hook spatial tree for all + hooksOut *btree.BTree // hooks with "outside" detection -- [string]*Hook + groupHooks *btree.BTree // hooks that are connected to objects + groupObjects *btree.BTree // objects that are connected to hooks + hookExpires *btree.BTree // queue of all hooks marked for expiration aofconnM map[net.Conn]io.Closer luascripts *lScriptMap @@ -164,8 +164,8 @@ func Serve(opts Options) error { fcond: sync.NewCond(&sync.Mutex{}), lives: make(map[*liveBuffer]bool), lcond: sync.NewCond(&sync.Mutex{}), - hooks: make(map[string]*Hook), - hooksOut: make(map[string]*Hook), + hooks: btree.NewNonConcurrent(byHookName), + hooksOut: btree.NewNonConcurrent(byHookName), hookCross: &rtree.RTree{}, hookTree: &rtree.RTree{}, aofconnM: make(map[net.Conn]io.Closer), diff --git a/internal/server/stats.go b/internal/server/stats.go index 0e0ca699..503beafe 100644 --- a/internal/server/stats.go +++ b/internal/server/stats.go @@ -157,7 +157,7 @@ func (s *Server) basicStats(m map[string]interface{}) { m["pid"] = os.Getpid() m["aof_size"] = s.aofsz m["num_collections"] = s.cols.Len() - m["num_hooks"] = len(s.hooks) + m["num_hooks"] = s.hooks.Len() sz := 0 s.cols.Ascend(nil, func(v interface{}) bool { col := v.(*collectionKeyContainer).col @@ -337,7 +337,7 @@ func (s *Server) extStats(m map[string]interface{}) { // Number of collections in the database m["tile38_num_collections"] = s.cols.Len() // Number of hooks in the database - m["tile38_num_hooks"] = len(s.hooks) + m["tile38_num_hooks"] = s.hooks.Len() // Number of hook groups in the database m["tile38_num_hook_groups"] = s.groupHooks.Len() // Number of object groups in the database From 19deea10c0ac6e42cd06786bb8fd74dc02075fce Mon Sep 17 00:00:00 2001 From: tidwall Date: Mon, 13 Sep 2021 10:52:20 -0700 Subject: [PATCH 12/15] Use same input args for command functions --- internal/server/crud.go | 3 ++- internal/server/expire.go | 2 +- internal/server/hooks.go | 22 +++++++++++++--------- internal/server/scripts.go | 4 ++-- internal/server/server.go | 20 ++++++++++---------- 5 files changed, 28 insertions(+), 23 deletions(-) diff --git a/internal/server/crud.go b/internal/server/crud.go index 5c0e1c03..08142c4f 100644 --- a/internal/server/crud.go +++ b/internal/server/crud.go @@ -441,7 +441,8 @@ func (server *Server) cmdDrop(msg *Message) (res resp.Value, d commandDetails, e return } -func (server *Server) cmdRename(msg *Message, nx bool) (res resp.Value, d commandDetails, err error) { +func (server *Server) cmdRename(msg *Message) (res resp.Value, d commandDetails, err error) { + nx := msg.Command() == "renamenx" start := time.Now() vs := msg.Args[1:] var ok bool diff --git a/internal/server/expire.go b/internal/server/expire.go index 16030c65..81b88d8a 100644 --- a/internal/server/expire.go +++ b/internal/server/expire.go @@ -73,7 +73,7 @@ func (s *Server) backgroundExpireHooks(now time.Time) { }) for _, msg := range msgs { - _, d, err := s.cmdDelHook(msg, msg.Args[0] == "delchan") + _, d, err := s.cmdDelHook(msg) if err != nil { log.Fatal(err) } diff --git a/internal/server/hooks.go b/internal/server/hooks.go index 71e5dd58..c6697d8a 100644 --- a/internal/server/hooks.go +++ b/internal/server/hooks.go @@ -26,9 +26,10 @@ func byHookName(a, b interface{}) bool { return a.(*Hook).Name < b.(*Hook).Name } -func (s *Server) cmdSetHook(msg *Message, chanCmd bool) ( +func (s *Server) cmdSetHook(msg *Message) ( res resp.Value, d commandDetails, err error, ) { + channel := msg.Command() == "setchan" start := time.Now() vs := msg.Args[1:] var name, urls, cmd string @@ -37,7 +38,7 @@ func (s *Server) cmdSetHook(msg *Message, chanCmd bool) ( return NOMessage, d, errInvalidNumberOfArguments } var endpoints []string - if chanCmd { + if channel { endpoints = []string{"local://" + name} } else { if vs, urls, ok = tokenval(vs); !ok || urls == "" { @@ -129,7 +130,7 @@ func (s *Server) cmdSetHook(msg *Message, chanCmd bool) ( Message: cmsg, epm: s.epc, Metas: metas, - channel: chanCmd, + channel: channel, cond: sync.NewCond(&sync.Mutex{}), counter: &s.statsTotalMsgsSent, } @@ -137,7 +138,7 @@ func (s *Server) cmdSetHook(msg *Message, chanCmd bool) ( hook.expires = time.Now().Add(time.Duration(expires * float64(time.Second))) } - if !chanCmd { + if !channel { hook.db = s.qdb } var wr bytes.Buffer @@ -151,7 +152,7 @@ func (s *Server) cmdSetHook(msg *Message, chanCmd bool) ( } prevHook, _ := s.hooks.Get(&Hook{Name: name}).(*Hook) if prevHook != nil { - if prevHook.channel != chanCmd { + if prevHook.channel != channel { return NOMessage, d, errors.New("hooks and channels cannot share the same name") } @@ -240,9 +241,10 @@ func byHookExpires(a, b interface{}) bool { return ha.Name < hb.Name } -func (s *Server) cmdDelHook(msg *Message, chanCmd bool) ( +func (s *Server) cmdDelHook(msg *Message) ( res resp.Value, d commandDetails, err error, ) { + channel := msg.Command() == "delchan" start := time.Now() vs := msg.Args[1:] @@ -255,7 +257,7 @@ func (s *Server) cmdDelHook(msg *Message, chanCmd bool) ( return NOMessage, d, errInvalidNumberOfArguments } hook, _ := s.hooks.Get(&Hook{Name: name}).(*Hook) - if hook != nil && hook.channel == chanCmd { + if hook != nil && hook.channel == channel { hook.Close() // remove hook from maps s.hooks.Delete(hook) @@ -295,9 +297,10 @@ func (s *Server) cmdDelHook(msg *Message, chanCmd bool) ( return } -func (s *Server) cmdPDelHook(msg *Message, channel bool) ( +func (s *Server) cmdPDelHook(msg *Message) ( res resp.Value, d commandDetails, err error, ) { + channel := msg.Command() == "pdelchan" start := time.Now() vs := msg.Args[1:] @@ -378,9 +381,10 @@ func (s *Server) forEachHookByPattern( }) } -func (s *Server) cmdHooks(msg *Message, channel bool) ( +func (s *Server) cmdHooks(msg *Message) ( res resp.Value, err error, ) { + channel := msg.Command() == "chans" start := time.Now() vs := msg.Args[1:] diff --git a/internal/server/scripts.go b/internal/server/scripts.go index 98e16d94..7e0b739e 100644 --- a/internal/server/scripts.go +++ b/internal/server/scripts.go @@ -604,9 +604,9 @@ func (s *Server) commandInScript(msg *Message) ( case "expire": res, d, err = s.cmdExpire(msg) case "rename": - res, d, err = s.cmdRename(msg, false) + res, d, err = s.cmdRename(msg) case "renamenx": - res, d, err = s.cmdRename(msg, true) + res, d, err = s.cmdRename(msg) case "persist": res, d, err = s.cmdPersist(msg) case "ttl": diff --git a/internal/server/server.go b/internal/server/server.go index 831589ce..0dee910a 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -1036,25 +1036,25 @@ func (server *Server) command(msg *Message, client *Client) ( case "flushdb": res, d, err = server.cmdFlushDB(msg) case "rename": - res, d, err = server.cmdRename(msg, false) + res, d, err = server.cmdRename(msg) case "renamenx": - res, d, err = server.cmdRename(msg, true) + res, d, err = server.cmdRename(msg) case "sethook": - res, d, err = server.cmdSetHook(msg, false) + res, d, err = server.cmdSetHook(msg) case "delhook": - res, d, err = server.cmdDelHook(msg, false) + res, d, err = server.cmdDelHook(msg) case "pdelhook": - res, d, err = server.cmdPDelHook(msg, false) + res, d, err = server.cmdPDelHook(msg) case "hooks": - res, err = server.cmdHooks(msg, false) + res, err = server.cmdHooks(msg) case "setchan": - res, d, err = server.cmdSetHook(msg, true) + res, d, err = server.cmdSetHook(msg) case "delchan": - res, d, err = server.cmdDelHook(msg, true) + res, d, err = server.cmdDelHook(msg) case "pdelchan": - res, d, err = server.cmdPDelHook(msg, true) + res, d, err = server.cmdPDelHook(msg) case "chans": - res, err = server.cmdHooks(msg, true) + res, err = server.cmdHooks(msg) case "expire": res, d, err = server.cmdExpire(msg) case "persist": From 4f4e168445f9d2f4ad24e52cfdebf93b737aa63b Mon Sep 17 00:00:00 2001 From: tidwall Date: Tue, 14 Sep 2021 12:25:23 -0700 Subject: [PATCH 13/15] 1.25.4 --- CHANGELOG.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a25692f5..50a3a007 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,17 @@ All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](http://semver.org/). +## [1.25.4] = 2021-09-14 +### Added +- a737a78: Add unix socket support + +### Updated +- 8829b8f: Change hooks collection type from hashmap to btree +- 83094b2: Update hook expiration logic +- c686b87: Return hook ttl with HOOKS request +- 06a92d8: Increase the precision of TIMEOUT +- Upgrade to Go 1.17.1 + ## [1.25.3] = 2021-08-23 ### Fixed - #621: Fixed a memory leak (@Morgiflute) @@ -10,7 +21,6 @@ This project adheres to [Semantic Versioning](http://semver.org/). - Update B-tree library - Upgrade to Go 1.17 - ## [1.25.2] = 2021-08-10 ### Fixed - #620: Fixed kafka authentication methods From de59d23ac41ae5e6eeb87fa776b166de20880100 Mon Sep 17 00:00:00 2001 From: tidwall Date: Sun, 26 Sep 2021 06:09:43 -0700 Subject: [PATCH 14/15] Fixed Z not matching on where clause Feature points. This issues fixes an issue where a search command with a where clause using the "z" field would not match correctly for point that where contained inside a GeoJSON Feature type. Tile38 now extracts the Z coordinate from Point and Feature/Point types. fixes #622 --- internal/server/crud.go | 5 +---- internal/server/json.go | 5 +---- internal/server/scanner.go | 26 ++++++++++++++++---------- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/internal/server/crud.go b/internal/server/crud.go index 08142c4f..4d90a700 100644 --- a/internal/server/crud.go +++ b/internal/server/crud.go @@ -183,10 +183,7 @@ func (server *Server) cmdGet(msg *Message) (resp.Value, error) { buf.Write(appendJSONSimplePoint(nil, o)) } else { point := o.Center() - var z float64 - if gPoint, ok := o.(*geojson.Point); ok { - z = gPoint.Z() - } + z := extractZCoordinate(o) if z != 0 { vals = append(vals, resp.ArrayValue([]resp.Value{ resp.StringValue(strconv.FormatFloat(point.Y, 'f', -1, 64)), diff --git a/internal/server/json.go b/internal/server/json.go index ab7fb1be..4dad895d 100644 --- a/internal/server/json.go +++ b/internal/server/json.go @@ -133,10 +133,7 @@ func appendJSONSimpleBounds(dst []byte, o geojson.Object) []byte { func appendJSONSimplePoint(dst []byte, o geojson.Object) []byte { point := o.Center() - var z float64 - if gPoint, ok := o.(*geojson.Point); ok { - z = gPoint.Z() - } + z := extractZCoordinate(o) dst = append(dst, `{"lat":`...) dst = strconv.AppendFloat(dst, point.Y, 'f', -1, 64) dst = append(dst, `,"lon":`...) diff --git a/internal/server/scanner.go b/internal/server/scanner.go index e11c14ad..d6b7955b 100644 --- a/internal/server/scanner.go +++ b/internal/server/scanner.go @@ -214,6 +214,19 @@ func (sw *scanWriter) writeFoot() { } } +func extractZCoordinate(o geojson.Object) float64 { + for { + switch g := o.(type) { + case *geojson.Point: + return g.Z() + case *geojson.Feature: + o = g.Base() + default: + return 0 + } + } +} + func (sw *scanWriter) fieldMatch(fields []float64, o geojson.Object) (fvals []float64, match bool) { var z float64 var gotz bool @@ -222,9 +235,7 @@ func (sw *scanWriter) fieldMatch(fields []float64, o geojson.Object) (fvals []fl for _, where := range sw.wheres { if where.field == "z" { if !gotz { - if point, ok := o.(*geojson.Point); ok { - z = point.Z() - } + z = extractZCoordinate(o) } if !where.match(z) { return @@ -270,9 +281,7 @@ func (sw *scanWriter) fieldMatch(fields []float64, o geojson.Object) (fvals []fl for _, where := range sw.wheres { if where.field == "z" { if !gotz { - if point, ok := o.(*geojson.Point); ok { - z = point.Z() - } + z = extractZCoordinate(o) } if !where.match(z) { return @@ -455,10 +464,7 @@ func (sw *scanWriter) writeObject(opts ScanWriterParams) bool { vals = append(vals, resp.StringValue(opts.o.String())) case outputPoints: point := opts.o.Center() - var z float64 - if point, ok := opts.o.(*geojson.Point); ok { - z = point.Z() - } + z := extractZCoordinate(opts.o) if z != 0 { vals = append(vals, resp.ArrayValue([]resp.Value{ resp.FloatValue(point.Y), From 03e596db8aad3bae47c3b3d584ee2013ab057d6f Mon Sep 17 00:00:00 2001 From: tidwall Date: Sun, 26 Sep 2021 06:18:48 -0700 Subject: [PATCH 15/15] 1.25.5 --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 50a3a007..dd73fad3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](http://semver.org/). +## [1.25.5] = 2021-09-26 +### Fixed +- 8ebcbeb: Fixed Z not matching on where clause for Feature/Point. (@tomquas) + ## [1.25.4] = 2021-09-14 ### Added - a737a78: Add unix socket support