From 2be07e4762c0edea518ffa7df1dc973d8b1d1b0e Mon Sep 17 00:00:00 2001 From: Erik Serating Date: Mon, 20 Jun 2022 09:50:40 -0400 Subject: [PATCH 01/13] Issue #642 - Expose config and INFO response for slave_priority --- internal/server/config.go | 32 ++++++++++++++++++++++++++------ internal/server/stats.go | 3 +++ 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/internal/server/config.go b/internal/server/config.go index b7355900..e17aa438 100644 --- a/internal/server/config.go +++ b/internal/server/config.go @@ -25,6 +25,7 @@ const ( FollowPort = "follow_port" FollowID = "follow_id" FollowPos = "follow_pos" + SlavePriority = "slave_priority" ServerID = "server_id" ReadOnly = "read_only" RequirePass = "requirepass" @@ -44,12 +45,13 @@ type Config struct { mu sync.RWMutex - _followHost string - _followPort int64 - _followID string - _followPos int64 - _serverID string - _readOnly bool + _followHost string + _followPort int64 + _followID string + _followPos int64 + _slavePriority int64 + _serverID string + _readOnly bool _requirePassP string _requirePass string @@ -99,6 +101,15 @@ func loadConfig(path string) (*Config, error) { config._serverID = randomKey(16) } + // Need to be sure we look for existence vs not zero because zero is an intentional setting + // anything less than zero will be considered default and will result in no slave_priority + // being output when INFO is called. + if gjson.Get(json, SlavePriority).Exists() { + config._slavePriority = gjson.Get(json, SlavePriority).Int() + } else { + config._slavePriority = -1 + } + // load properties if err := config.setProperty(RequirePass, config._requirePassP, true); err != nil { return nil, err @@ -167,6 +178,9 @@ func (config *Config) write(writeProperties bool) { if config._followPos != 0 { m[FollowPos] = config._followPos } + if config._slavePriority >= 0 { + m[SlavePriority] = config._slavePriority + } if config._serverID != "" { m[ServerID] = config._serverID } @@ -426,6 +440,12 @@ func (config *Config) followPort() int { config.mu.RUnlock() return int(v) } +func (config *Config) slavePriority() int { + config.mu.RLock() + v := config._slavePriority + config.mu.RUnlock() + return int(v) +} func (config *Config) serverID() string { config.mu.RLock() v := config._serverID diff --git a/internal/server/stats.go b/internal/server/stats.go index 503beafe..975665c2 100644 --- a/internal/server/stats.go +++ b/internal/server/stats.go @@ -408,6 +408,9 @@ func (s *Server) writeInfoReplication(w *bytes.Buffer) { fmt.Fprintf(w, "role:slave\r\n") fmt.Fprintf(w, "master_host:%s\r\n", s.config.followHost()) fmt.Fprintf(w, "master_port:%v\r\n", s.config.followPort()) + if s.config.slavePriority() >= 0 { + fmt.Fprintf(w, "slave_priority:%v\r\n", s.config.slavePriority()) + } } else { fmt.Fprintf(w, "role:master\r\n") var i int From 92ebaf2dece0ee1e521a7d13a09ceebe3b97d532 Mon Sep 17 00:00:00 2001 From: Josh Baker Date: Mon, 20 Jun 2022 13:29:35 -0700 Subject: [PATCH 02/13] Add tile38-ts client to README --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 770d0661..5ae9cb1a 100644 --- a/README.md +++ b/README.md @@ -318,6 +318,7 @@ Clients that support most Tile38 features are marked with a ⭐️. - ⭐️ Go: [xjem/t38c](https://github.com/xjem/t38c) - ⭐️ Node.js: [node-tile38](https://github.com/phulst/node-tile38) ([example code](https://github.com/tidwall/tile38/wiki/Node.js-example-(node-tile38))) - ⭐️ Python: [pyle38](https://github.com/iwpnd/pyle38) +- ⭐️ TypeScript: [tile38-ts](https://github.com/tiermobility/tile38-ts) - Go: [cjkreklow/t38c](https://github.com/cjkreklow/t38c) - Python: [pytile38](https://github.com/mitghi/pytile38) - Rust: [nazar](https://github.com/younisshah/nazar) From b256d4752bd1a04ea60c224e86b291cb2f94ae04 Mon Sep 17 00:00:00 2001 From: Erik Serating Date: Wed, 22 Jun 2022 12:50:43 -0400 Subject: [PATCH 03/13] Issue #642 - Renamed config property to replica-priority and added config set/get functionality --- internal/server/config.go | 71 +++++++++++++++++++++++---------------- internal/server/stats.go | 4 +-- 2 files changed, 44 insertions(+), 31 deletions(-) diff --git a/internal/server/config.go b/internal/server/config.go index e17aa438..31aca1d0 100644 --- a/internal/server/config.go +++ b/internal/server/config.go @@ -21,23 +21,23 @@ const ( // Config keys const ( - FollowHost = "follow_host" - FollowPort = "follow_port" - FollowID = "follow_id" - FollowPos = "follow_pos" - SlavePriority = "slave_priority" - ServerID = "server_id" - ReadOnly = "read_only" - RequirePass = "requirepass" - LeaderAuth = "leaderauth" - ProtectedMode = "protected-mode" - MaxMemory = "maxmemory" - AutoGC = "autogc" - KeepAlive = "keepalive" - LogConfig = "logconfig" + FollowHost = "follow_host" + FollowPort = "follow_port" + FollowID = "follow_id" + FollowPos = "follow_pos" + ReplicaPriority = "replica-priority" + ServerID = "server_id" + ReadOnly = "read_only" + RequirePass = "requirepass" + LeaderAuth = "leaderauth" + ProtectedMode = "protected-mode" + MaxMemory = "maxmemory" + AutoGC = "autogc" + KeepAlive = "keepalive" + LogConfig = "logconfig" ) -var validProperties = []string{RequirePass, LeaderAuth, ProtectedMode, MaxMemory, AutoGC, KeepAlive, LogConfig} +var validProperties = []string{RequirePass, LeaderAuth, ProtectedMode, MaxMemory, AutoGC, KeepAlive, LogConfig, ReplicaPriority} // Config is a tile38 config type Config struct { @@ -45,13 +45,13 @@ type Config struct { mu sync.RWMutex - _followHost string - _followPort int64 - _followID string - _followPos int64 - _slavePriority int64 - _serverID string - _readOnly bool + _followHost string + _followPort int64 + _followID string + _followPos int64 + _replicaPriority int64 + _serverID string + _readOnly bool _requirePassP string _requirePass string @@ -104,10 +104,10 @@ func loadConfig(path string) (*Config, error) { // Need to be sure we look for existence vs not zero because zero is an intentional setting // anything less than zero will be considered default and will result in no slave_priority // being output when INFO is called. - if gjson.Get(json, SlavePriority).Exists() { - config._slavePriority = gjson.Get(json, SlavePriority).Int() + if gjson.Get(json, ReplicaPriority).Exists() { + config._replicaPriority = gjson.Get(json, ReplicaPriority).Int() } else { - config._slavePriority = -1 + config._replicaPriority = -1 } // load properties @@ -178,8 +178,8 @@ func (config *Config) write(writeProperties bool) { if config._followPos != 0 { m[FollowPos] = config._followPos } - if config._slavePriority >= 0 { - m[SlavePriority] = config._slavePriority + if config._replicaPriority >= 0 { + m[ReplicaPriority] = config._replicaPriority } if config._serverID != "" { m[ServerID] = config._serverID @@ -326,6 +326,13 @@ func (config *Config) setProperty(name, value string, fromLoad bool) error { } else { config._logConfig = value } + case ReplicaPriority: + replicaPriority, err := strconv.ParseUint(value, 10, 64) + if err != nil || replicaPriority < 0 { + invalid = true + } else { + config._replicaPriority = int64(replicaPriority) + } } if invalid { @@ -365,6 +372,12 @@ func (config *Config) getProperty(name string) string { return strconv.FormatUint(uint64(config._keepAlive), 10) case LogConfig: return config._logConfig + case ReplicaPriority: + if config._replicaPriority < 0 { + return "" + } else { + return strconv.FormatUint(uint64(config._replicaPriority), 10) + } } } @@ -440,9 +453,9 @@ func (config *Config) followPort() int { config.mu.RUnlock() return int(v) } -func (config *Config) slavePriority() int { +func (config *Config) replicaPriority() int { config.mu.RLock() - v := config._slavePriority + v := config._replicaPriority config.mu.RUnlock() return int(v) } diff --git a/internal/server/stats.go b/internal/server/stats.go index 975665c2..b89c274e 100644 --- a/internal/server/stats.go +++ b/internal/server/stats.go @@ -408,8 +408,8 @@ func (s *Server) writeInfoReplication(w *bytes.Buffer) { fmt.Fprintf(w, "role:slave\r\n") fmt.Fprintf(w, "master_host:%s\r\n", s.config.followHost()) fmt.Fprintf(w, "master_port:%v\r\n", s.config.followPort()) - if s.config.slavePriority() >= 0 { - fmt.Fprintf(w, "slave_priority:%v\r\n", s.config.slavePriority()) + if s.config.replicaPriority() >= 0 { + fmt.Fprintf(w, "slave_priority:%v\r\n", s.config.replicaPriority()) } } else { fmt.Fprintf(w, "role:master\r\n") From b883f358d5576636dbd0c69cf10fa7e76d157235 Mon Sep 17 00:00:00 2001 From: tidwall Date: Mon, 11 Jul 2022 08:39:40 -0700 Subject: [PATCH 04/13] Add pending_events stat --- internal/server/stats.go | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/internal/server/stats.go b/internal/server/stats.go index b89c274e..93f6107e 100644 --- a/internal/server/stats.go +++ b/internal/server/stats.go @@ -13,6 +13,7 @@ import ( "sync" "time" + "github.com/tidwall/buntdb" "github.com/tidwall/resp" "github.com/tidwall/tile38/core" ) @@ -167,17 +168,17 @@ func (s *Server) basicStats(m map[string]interface{}) { m["in_memory_size"] = sz points := 0 objects := 0 - strings := 0 + nstrings := 0 s.cols.Ascend(nil, func(v interface{}) bool { col := v.(*collectionKeyContainer).col points += col.PointCount() objects += col.Count() - strings += col.StringCount() + nstrings += col.StringCount() return true }) m["num_points"] = points m["num_objects"] = objects - m["num_strings"] = strings + m["num_strings"] = nstrings mem := readMemStats() avgsz := 0 if points != 0 { @@ -194,6 +195,18 @@ func (s *Server) basicStats(m map[string]interface{}) { m["cpus"] = runtime.NumCPU() n, _ := runtime.ThreadCreateProfile(nil) m["threads"] = float64(n) + var nevents int + s.qdb.View(func(tx *buntdb.Tx) error { + // All entries in the buntdb log are events, except for one, which + // is "hook:idx". + nevents, _ = tx.Len() + nevents -= 1 // Ignore the "hook:idx" + if nevents < 0 { + nevents = 0 + } + return nil + }) + m["pending_events"] = nevents } // extStats populates the passed map with extended system/go/tile38 statistics From c3addbe37ed253860a49d52d0f5a2d149511de30 Mon Sep 17 00:00:00 2001 From: tidwall Date: Thu, 14 Jul 2022 04:57:51 -0700 Subject: [PATCH 05/13] Update header --- cmd/tile38-server/main.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/cmd/tile38-server/main.go b/cmd/tile38-server/main.go index 03dec935..5a583767 100644 --- a/cmd/tile38-server/main.go +++ b/cmd/tile38-server/main.go @@ -436,9 +436,6 @@ Developer Options: | | | | | -_|_ | . | %s%s, PID: %d |_| |_|_|___|___|___| tile38.com -Please consider sponsoring Tile38 development, especially if your company -benefits from this software. Visit tile38.com/sponsor today to learn more. - `, core.Version, gitsha, strconv.IntSize, runtime.GOARCH, runtime.GOOS, hostd, saddr, os.Getpid()) } From 757db4d50904c979a928fcea51166fe9d287a595 Mon Sep 17 00:00:00 2001 From: tidwall Date: Thu, 14 Jul 2022 04:58:10 -0700 Subject: [PATCH 06/13] 1.29.0 --- CHANGELOG.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 43200e37..de0f2ddf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,15 @@ All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](http://semver.org/). +## [1.29.0] = 2022-07-14 + +### Added +- b883f35: Add pending_events stat +- #643: Expose config and INFO response for replia-priorty (@rave-eserating) + +### Fixed +- 8e61f81: Fixed test on Apple silicon + ## [1.28.0] = 2022-04-12 ### Added - 10f8564: Added option to "not found" for DEL From cc9320e246afb683bcbe96c7d11273f5a64936dd Mon Sep 17 00:00:00 2001 From: tidwall Date: Fri, 26 Aug 2022 16:23:28 -0700 Subject: [PATCH 07/13] Allow for WHERE for on geofence detection --- internal/collection/collection.go | 4 +++- internal/server/fence.go | 12 +++++++++++- internal/server/scanner.go | 12 +++++++++--- 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/internal/collection/collection.go b/internal/collection/collection.go index 2bce6966..63c54713 100644 --- a/internal/collection/collection.go +++ b/internal/collection/collection.go @@ -191,8 +191,10 @@ func (c *Collection) Set( oldFieldValues = c.fieldValues.get(oldItem.fieldValuesSlot) newFieldValues = oldFieldValues newItem.fieldValuesSlot = oldItem.fieldValuesSlot + if len(oldFieldValues) > 0 { + oldFieldValues = append([]float64{}, oldFieldValues...) + } } - if fields == nil { if len(values) > 0 { newFieldValues = values diff --git a/internal/server/fence.go b/internal/server/fence.go index eb0de389..94bc6de5 100644 --- a/internal/server/fence.go +++ b/internal/server/fence.go @@ -105,9 +105,18 @@ func fenceMatch( } detect = "roam" } else { + var nocross bool // not using roaming match1 := fenceMatchObject(fence, details.oldObj) + if match1 { + match1, _, _ = sw.testObject(details.id, details.oldObj, details.oldFields) + nocross = !match1 + } match2 := fenceMatchObject(fence, details.obj) + if match2 { + match2, _, _ = sw.testObject(details.id, details.obj, details.fields) + nocross = !match2 + } if match1 && match2 { detect = "inside" } else if match1 && !match2 { @@ -121,7 +130,7 @@ func fenceMatch( if details.command != "fset" { // Maybe the old object and new object create a line that crosses the fence. // Must detect for that possibility. - if details.oldObj != nil { + if !nocross && details.oldObj != nil { ls := geojson.NewLineString(geometry.NewLine( []geometry.Point{ details.oldObj.Center(), @@ -176,6 +185,7 @@ func fenceMatch( o: details.obj, fields: details.fields, noLock: true, + noTest: true, distance: distance, distOutput: fence.distance, }) diff --git a/internal/server/scanner.go b/internal/server/scanner.go index 7f73a649..839870d8 100644 --- a/internal/server/scanner.go +++ b/internal/server/scanner.go @@ -68,6 +68,7 @@ type ScanWriterParams struct { distance float64 distOutput bool // query or fence requested distance output noLock bool + noTest bool ignoreGlobMatch bool clip geojson.Object skipTesting bool @@ -373,9 +374,14 @@ func (sw *scanWriter) writeObject(opts ScanWriterParams) bool { sw.mu.Lock() defer sw.mu.Unlock() } - ok, keepGoing, _ := sw.testObject(opts.id, opts.o, opts.fields) - if !ok { - return keepGoing + + keepGoing := true + if !opts.noTest { + var ok bool + ok, keepGoing, _ = sw.testObject(opts.id, opts.o, opts.fields) + if !ok { + return keepGoing + } } sw.count++ if sw.output == outputCount { From 67916f38f8aee0abacdd94032deba2102ed807fe Mon Sep 17 00:00:00 2001 From: tidwall Date: Tue, 30 Aug 2022 16:50:19 -0700 Subject: [PATCH 08/13] Reset wheres while geofencing --- internal/server/aof.go | 1 + internal/server/scanner.go | 36 +++++++++++++++++++++++++++--------- 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/internal/server/aof.go b/internal/server/aof.go index f90ab6e6..47e92f40 100644 --- a/internal/server/aof.go +++ b/internal/server/aof.go @@ -294,6 +294,7 @@ func (s *Server) queueHooks(d *commandDetails) error { for _, hook := range candidates { // Calculate all matching fence messages for all candidates and append // them to the appropriate message slice + hook.ScanWriter.loadWheres() msgs := FenceMatch(hook.Name, hook.ScanWriter, hook.Fence, hook.Metas, d) if len(msgs) > 0 { if hook.channel { diff --git a/internal/server/scanner.go b/internal/server/scanner.go index 839870d8..b189cd56 100644 --- a/internal/server/scanner.go +++ b/internal/server/scanner.go @@ -33,6 +33,7 @@ type scanWriter struct { mu sync.Mutex s *Server wr *bytes.Buffer + key string msg *Message col *collection.Collection fmap map[string]int @@ -58,6 +59,8 @@ type scanWriter struct { values []resp.Value matchValues bool respOut resp.Value + orgWheres []whereT + orgWhereins []whereinT } // ScanWriterParams ... @@ -97,6 +100,7 @@ func (s *Server) newScanWriter( sw := &scanWriter{ s: s, wr: wr, + key: key, msg: msg, limit: limit, cursor: cursor, @@ -114,34 +118,48 @@ func (s *Server) newScanWriter( sw.globSingle = true } } - sw.col = s.getCol(key) + sw.orgWheres = wheres + sw.orgWhereins = whereins + sw.loadWheres() + return sw, nil +} + +func (sw *scanWriter) loadWheres() { + sw.fmap = nil + sw.farr = nil + sw.wheres = nil + sw.whereins = nil + sw.fvals = nil + sw.col = sw.s.getCol(sw.key) if sw.col != nil { sw.fmap = sw.col.FieldMap() sw.farr = sw.col.FieldArr() // This fills index value in wheres/whereins // so we don't have to map string field names for each tested object var ok bool - if len(wheres) > 0 { - sw.wheres = make([]whereT, len(wheres)) - for i, where := range wheres { + if len(sw.orgWheres) > 0 { + sw.wheres = make([]whereT, len(sw.orgWheres)) + for i, where := range sw.orgWheres { if where.index, ok = sw.fmap[where.field]; !ok { where.index = math.MaxInt32 } sw.wheres[i] = where } } - if len(whereins) > 0 { - sw.whereins = make([]whereinT, len(whereins)) - for i, wherein := range whereins { + if len(sw.orgWhereins) > 0 { + sw.whereins = make([]whereinT, len(sw.orgWhereins)) + for i, wherein := range sw.orgWhereins { if wherein.index, ok = sw.fmap[wherein.field]; !ok { wherein.index = math.MaxInt32 } sw.whereins[i] = wherein } } + if len(sw.farr) > 0 { + sw.fvals = make([]float64, len(sw.farr)) + } } - sw.fvals = make([]float64, len(sw.farr)) - return sw, nil + } func (sw *scanWriter) hasFieldsOutput() bool { From f24c251ee61e9c7a3cea50df82f1ebdc7be2bb64 Mon Sep 17 00:00:00 2001 From: tidwall Date: Thu, 1 Sep 2022 19:43:30 -0700 Subject: [PATCH 09/13] Allow for multiple MATCH patterns Each MATCH is inclusive OR, thus WITHIN fleet MATCH train* truck* BOUNDS 33 -112 34 -113 will find all trains and trucks that within the provides bounds. --- internal/server/fence.go | 21 ++++++++++++----- internal/server/hooks.go | 2 +- internal/server/live.go | 6 ++--- internal/server/scan.go | 9 ++++---- internal/server/scanner.go | 46 +++++++++++++++++--------------------- internal/server/search.go | 42 ++++++++++++++++++++++++++++------ internal/server/token.go | 10 ++++----- tests/keys_search_test.go | 21 ++++++++++------- 8 files changed, 96 insertions(+), 61 deletions(-) diff --git a/internal/server/fence.go b/internal/server/fence.go index 94bc6de5..2829fef0 100644 --- a/internal/server/fence.go +++ b/internal/server/fence.go @@ -55,6 +55,20 @@ func objIsSpatial(obj geojson.Object) bool { func hookJSONString(hookName string, metas []FenceMeta) string { return string(appendHookDetails(nil, hookName, metas)) } + +func multiGlobMatch(globs []string, s string) bool { + if len(globs) == 0 || (len(globs) == 1 && globs[0] == "*") { + return true + } + for _, pattern := range globs { + match, _ := glob.Match(pattern, s) + if match { + return true + } + } + return false +} + func fenceMatch( hookName string, sw *scanWriter, fence *liveFenceSwitches, metas []FenceMeta, details *commandDetails, @@ -66,11 +80,8 @@ func fenceMatch( `,"time":` + jsonTimeFormat(details.timestamp) + `}`, } } - if len(fence.glob) > 0 && !(len(fence.glob) == 1 && fence.glob[0] == '*') { - match, _ := glob.Match(fence.glob, details.id) - if !match { - return nil - } + if !multiGlobMatch(fence.globs, details.id) { + return nil } if details.obj == nil || !objIsSpatial(details.obj) { return nil diff --git a/internal/server/hooks.go b/internal/server/hooks.go index e2c22505..4e490d77 100644 --- a/internal/server/hooks.go +++ b/internal/server/hooks.go @@ -143,7 +143,7 @@ func (s *Server) cmdSetHook(msg *Message) ( } var wr bytes.Buffer hook.ScanWriter, err = s.newScanWriter( - &wr, cmsg, args.key, args.output, args.precision, args.glob, false, + &wr, cmsg, args.key, args.output, args.precision, args.globs, false, args.cursor, args.limit, args.wheres, args.whereins, args.whereevals, args.nofields) if err != nil { diff --git a/internal/server/live.go b/internal/server/live.go index 90d0a116..1f595327 100644 --- a/internal/server/live.go +++ b/internal/server/live.go @@ -14,7 +14,7 @@ import ( type liveBuffer struct { key string - glob string + globs []string fence *liveFenceSwitches details []*commandDetails cond *sync.Cond @@ -101,12 +101,12 @@ func (s *Server) goLive( var sw *scanWriter var wr bytes.Buffer lfs := inerr.(liveFenceSwitches) - lb.glob = lfs.glob + lb.globs = lfs.globs lb.key = lfs.key lb.fence = &lfs s.mu.RLock() sw, err = s.newScanWriter( - &wr, msg, lfs.key, lfs.output, lfs.precision, lfs.glob, false, + &wr, msg, lfs.key, lfs.output, lfs.precision, lfs.globs, false, lfs.cursor, lfs.limit, lfs.wheres, lfs.whereins, lfs.whereevals, lfs.nofields) s.mu.RUnlock() diff --git a/internal/server/scan.go b/internal/server/scan.go index ad595a12..0151ae90 100644 --- a/internal/server/scan.go +++ b/internal/server/scan.go @@ -7,7 +7,6 @@ import ( "github.com/tidwall/geojson" "github.com/tidwall/resp" - "github.com/tidwall/tile38/internal/glob" ) func (s *Server) cmdScanArgs(vs []string) ( @@ -46,7 +45,7 @@ func (s *Server) cmdScan(msg *Message) (res resp.Value, err error) { } wr := &bytes.Buffer{} sw, err := s.newScanWriter( - wr, msg, args.key, args.output, args.precision, args.glob, false, + wr, msg, args.key, args.output, args.precision, args.globs, false, args.cursor, args.limit, args.wheres, args.whereins, args.whereevals, args.nofields) if err != nil { @@ -65,8 +64,8 @@ func (s *Server) cmdScan(msg *Message) (res resp.Value, err error) { } sw.count = uint64(count) } else { - g := glob.Parse(sw.globPattern, args.desc) - if g.Limits[0] == "" && g.Limits[1] == "" { + limits := multiGlobParse(sw.globs, args.desc) + if limits[0] == "" && limits[1] == "" { sw.col.Scan(args.desc, sw, msg.Deadline, func(id string, o geojson.Object, fields []float64) bool { @@ -78,7 +77,7 @@ func (s *Server) cmdScan(msg *Message) (res resp.Value, err error) { }, ) } else { - sw.col.ScanRange(g.Limits[0], g.Limits[1], args.desc, sw, + sw.col.ScanRange(limits[0], limits[1], args.desc, sw, msg.Deadline, func(id string, o geojson.Object, fields []float64) bool { return sw.writeObject(ScanWriterParams{ diff --git a/internal/server/scanner.go b/internal/server/scanner.go index b189cd56..3053ca93 100644 --- a/internal/server/scanner.go +++ b/internal/server/scanner.go @@ -52,9 +52,8 @@ type scanWriter struct { once bool count uint64 precision uint64 - globPattern string + globs []string globEverything bool - globSingle bool fullFields bool values []resp.Value matchValues bool @@ -79,7 +78,7 @@ type ScanWriterParams struct { func (s *Server) newScanWriter( wr *bytes.Buffer, msg *Message, key string, output outputT, - precision uint64, globPattern string, matchValues bool, + precision uint64, globs []string, matchValues bool, cursor, limit uint64, wheres []whereT, whereins []whereinT, whereevals []whereevalT, nofields bool, ) ( @@ -102,21 +101,18 @@ func (s *Server) newScanWriter( wr: wr, key: key, msg: msg, + globs: globs, limit: limit, cursor: cursor, output: output, nofields: nofields, precision: precision, whereevals: whereevals, - globPattern: globPattern, matchValues: matchValues, } - if globPattern == "*" || globPattern == "" { + + if len(globs) == 0 || (len(globs) == 1 && globs[0] == "*") { sw.globEverything = true - } else { - if !glob.IsGlob(globPattern) { - sw.globSingle = true - } } sw.orgWheres = wheres sw.orgWhereins = whereins @@ -344,25 +340,23 @@ func (sw *scanWriter) fieldMatch(fields []float64, o geojson.Object) (fvals []fl } func (sw *scanWriter) globMatch(id string, o geojson.Object) (ok, keepGoing bool) { - if !sw.globEverything { - if sw.globSingle { - if sw.globPattern != id { - return false, true - } - return true, false - } - var val string - if sw.matchValues { - val = o.String() - } else { - val = id - } - ok, _ := glob.Match(sw.globPattern, val) - if !ok { - return false, true + if sw.globEverything { + return true, true + } + var val string + if sw.matchValues { + val = o.String() + } else { + val = id + } + for _, pattern := range sw.globs { + ok, _ := glob.Match(pattern, val) + if ok { + return true, true } } - return true, true + return false, true + } // Increment cursor diff --git a/internal/server/search.go b/internal/server/search.go index 925683a5..57c744f5 100644 --- a/internal/server/search.go +++ b/internal/server/search.go @@ -488,7 +488,7 @@ func (s *Server) cmdNearby(msg *Message) (res resp.Value, err error) { return NOMessage, sargs } sw, err := s.newScanWriter( - wr, msg, sargs.key, sargs.output, sargs.precision, sargs.glob, false, + wr, msg, sargs.key, sargs.output, sargs.precision, sargs.globs, false, sargs.cursor, sargs.limit, sargs.wheres, sargs.whereins, sargs.whereevals, sargs.nofields) if err != nil { return NOMessage, err @@ -580,7 +580,7 @@ func (s *Server) cmdWithinOrIntersects(cmd string, msg *Message) (res resp.Value return NOMessage, sargs } sw, err := s.newScanWriter( - wr, msg, sargs.key, sargs.output, sargs.precision, sargs.glob, false, + wr, msg, sargs.key, sargs.output, sargs.precision, sargs.globs, false, sargs.cursor, sargs.limit, sargs.wheres, sargs.whereins, sargs.whereevals, sargs.nofields) if err != nil { return NOMessage, err @@ -644,6 +644,35 @@ func (s *Server) cmdSeachValuesArgs(vs []string) ( return } +func multiGlobParse(globs []string, desc bool) [2]string { + var limits [2]string + for i, pattern := range globs { + g := glob.Parse(pattern, desc) + if g.Limits[0] == "" && g.Limits[1] == "" { + limits[0], limits[1] = "", "" + break + } + if i == 0 { + limits[0], limits[1] = g.Limits[0], g.Limits[1] + } else if desc { + if g.Limits[0] > limits[0] { + limits[0] = g.Limits[0] + } + if g.Limits[1] < limits[1] { + limits[1] = g.Limits[1] + } + } else { + if g.Limits[0] < limits[0] { + limits[0] = g.Limits[0] + } + if g.Limits[1] > limits[1] { + limits[1] = g.Limits[1] + } + } + } + return limits +} + func (s *Server) cmdSearch(msg *Message) (res resp.Value, err error) { start := time.Now() vs := msg.Args[1:] @@ -664,7 +693,7 @@ func (s *Server) cmdSearch(msg *Message) (res resp.Value, err error) { return NOMessage, err } sw, err := s.newScanWriter( - wr, msg, sargs.key, sargs.output, sargs.precision, sargs.glob, true, + wr, msg, sargs.key, sargs.output, sargs.precision, sargs.globs, true, sargs.cursor, sargs.limit, sargs.wheres, sargs.whereins, sargs.whereevals, sargs.nofields) if err != nil { return NOMessage, err @@ -681,8 +710,8 @@ func (s *Server) cmdSearch(msg *Message) (res resp.Value, err error) { } sw.count = uint64(count) } else { - g := glob.Parse(sw.globPattern, sargs.desc) - if g.Limits[0] == "" && g.Limits[1] == "" { + limits := multiGlobParse(sw.globs, sargs.desc) + if limits[0] == "" && limits[1] == "" { sw.col.SearchValues(sargs.desc, sw, msg.Deadline, func(id string, o geojson.Object, fields []float64) bool { return sw.writeObject(ScanWriterParams{ @@ -696,8 +725,7 @@ func (s *Server) cmdSearch(msg *Message) (res resp.Value, err error) { } else { // must disable globSingle for string value type matching because // globSingle is only for ID matches, not values. - sw.globSingle = false - sw.col.SearchValuesRange(g.Limits[0], g.Limits[1], sargs.desc, sw, + sw.col.SearchValuesRange(limits[0], limits[1], sargs.desc, sw, msg.Deadline, func(id string, o geojson.Object, fields []float64) bool { return sw.writeObject(ScanWriterParams{ diff --git a/internal/server/token.go b/internal/server/token.go index 50925626..178d9a1f 100644 --- a/internal/server/token.go +++ b/internal/server/token.go @@ -200,7 +200,7 @@ type searchScanBaseTokens struct { nodwell bool detect map[string]bool accept map[string]bool - glob string + globs []string wheres []whereT whereins []whereinT whereevals []whereevalT @@ -543,14 +543,12 @@ func (s *Server) parseSearchScanBaseTokens( continue case "match": vs = nvs - if t.glob != "" { - err = errDuplicateArgument(strings.ToUpper(wtok)) - return - } - if vs, t.glob, ok = tokenval(vs); !ok || t.glob == "" { + var glob string + if vs, glob, ok = tokenval(vs); !ok || glob == "" { err = errInvalidNumberOfArguments return } + t.globs = append(t.globs, glob) continue case "clip": vs = nvs diff --git a/tests/keys_search_test.go b/tests/keys_search_test.go index 7f3ee971..1f8ebb12 100644 --- a/tests/keys_search_test.go +++ b/tests/keys_search_test.go @@ -591,23 +591,28 @@ func keys_MATCH_test(mc *mockServer) error { {"SET", "fleet", "truck2", "POINT", "33.0002", "-112.0002"}, {"OK"}, {"SET", "fleet", "luck1", "POINT", "33.0003", "-112.0003"}, {"OK"}, {"SET", "fleet", "luck2", "POINT", "33.0004", "-112.0004"}, {"OK"}, + {"SET", "fleet", "train1", "POINT", "33.0005", "-112.0005"}, {"OK"}, - {"SCAN", "fleet", "IDS"}, {"[0 [luck1 luck2 truck1 truck2]]"}, - {"SCAN", "fleet", "MATCH", "*", "IDS"}, {"[0 [luck1 luck2 truck1 truck2]]"}, + {"SCAN", "fleet", "IDS"}, {"[0 [luck1 luck2 train1 truck1 truck2]]"}, + {"SCAN", "fleet", "MATCH", "*", "IDS"}, {"[0 [luck1 luck2 train1 truck1 truck2]]"}, {"SCAN", "fleet", "MATCH", "truck*", "IDS"}, {"[0 [truck1 truck2]]"}, {"SCAN", "fleet", "MATCH", "luck*", "IDS"}, {"[0 [luck1 luck2]]"}, {"SCAN", "fleet", "MATCH", "*2", "IDS"}, {"[0 [luck2 truck2]]"}, {"SCAN", "fleet", "MATCH", "*2*", "IDS"}, {"[0 [luck2 truck2]]"}, {"SCAN", "fleet", "MATCH", "*u*", "IDS"}, {"[0 [luck1 luck2 truck1 truck2]]"}, + {"SCAN", "fleet", "MATCH", "*u*", "MATCH", "*u*", "IDS"}, {"[0 [luck1 luck2 truck1 truck2]]"}, + {"SCAN", "fleet", "MATCH", "*u*", "MATCH", "*a*", "IDS"}, {"[0 [luck1 luck2 train1 truck1 truck2]]"}, + {"SCAN", "fleet", "MATCH", "train*", "MATCH", "truck*", "IDS"}, {"[0 [train1 truck1 truck2]]"}, + {"SCAN", "fleet", "MATCH", "train*", "MATCH", "truck*", "MATCH", "luck1", "IDS"}, {"[0 [luck1 train1 truck1 truck2]]"}, {"NEARBY", "fleet", "IDS", "POINT", 33.00005, -112.00005, 100000}, { - match("[0 [luck1 luck2 truck1 truck2]]"), + match("[0 [luck1 luck2 train1 truck1 truck2]]"), }, {"NEARBY", "fleet", "MATCH", "*", "IDS", "POINT", 33.00005, -112.00005, 100000}, { - match("[0 [luck1 luck2 truck1 truck2]]"), + match("[0 [luck1 luck2 train1 truck1 truck2]]"), }, {"NEARBY", "fleet", "MATCH", "t*", "IDS", "POINT", 33.00005, -112.00005, 100000}, { - match("[0 [truck1 truck2]]"), + match("[0 [train1 truck1 truck2]]"), }, {"NEARBY", "fleet", "MATCH", "t*2", "IDS", "POINT", 33.00005, -112.00005, 100000}, { match("[0 [truck2]]"), @@ -617,13 +622,13 @@ func keys_MATCH_test(mc *mockServer) error { }, {"INTERSECTS", "fleet", "IDS", "BOUNDS", 33, -113, 34, -112}, { - match("[0 [luck1 luck2 truck1 truck2]]"), + match("[0 [luck1 luck2 train1 truck1 truck2]]"), }, {"INTERSECTS", "fleet", "MATCH", "*", "IDS", "BOUNDS", 33, -113, 34, -112}, { - match("[0 [luck1 luck2 truck1 truck2]]"), + match("[0 [luck1 luck2 train1 truck1 truck2]]"), }, {"INTERSECTS", "fleet", "MATCH", "t*", "IDS", "BOUNDS", 33, -113, 34, -112}, { - match("[0 [truck1 truck2]]"), + match("[0 [train1 truck1 truck2]]"), }, {"INTERSECTS", "fleet", "MATCH", "t*2", "IDS", "BOUNDS", 33, -113, 34, -112}, { match("[0 [truck2]]"), From 09ca96717bbe6c596cc94cc40d75621d1553d9fa Mon Sep 17 00:00:00 2001 From: Benjamin Ramser Date: Wed, 7 Sep 2022 10:34:24 +0200 Subject: [PATCH 10/13] fix: allow host ca sets for SASL and TLS connections ?ssl=true previously would require the user to provide a cacertfile stripping the option to use the hosts ca set. bumping sarama to version 1.36.0 bumping alpine to 3.16.2 fix: tls path --- Dockerfile | 2 +- go.mod | 36 +++++------ go.sum | 119 +++++++++++++++++++++---------------- internal/endpoint/kafka.go | 39 ++++++------ 4 files changed, 111 insertions(+), 85 deletions(-) diff --git a/Dockerfile b/Dockerfile index f466be99..b9cf53c4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM alpine:3.15 +FROM alpine:3.16.2 RUN apk add --no-cache ca-certificates ADD tile38-server /usr/local/bin diff --git a/go.mod b/go.mod index c956d681..deda3b75 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.18 require ( cloud.google.com/go/pubsub v1.3.1 github.com/Azure/azure-event-hubs-go/v3 v3.3.16 - github.com/Shopify/sarama v1.27.2 + github.com/Shopify/sarama v1.36.0 github.com/aws/aws-sdk-go v1.37.3 github.com/eclipse/paho.mqtt.golang v1.3.1 github.com/golang/protobuf v1.5.2 @@ -31,8 +31,8 @@ require ( github.com/xdg/scram v1.0.5 github.com/yuin/gopher-lua v0.0.0-20200816102855-ee81675732da go.uber.org/zap v1.13.0 - golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 - golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 + golang.org/x/net v0.0.0-20220809184613-07c6da5e1ced + golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 google.golang.org/api v0.30.0 google.golang.org/grpc v1.35.0 layeh.com/gopher-json v0.0.0-20201124131017-552bb3c4c3bf @@ -56,32 +56,38 @@ require ( github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/devigned/tab v0.1.1 // indirect - github.com/eapache/go-resiliency v1.2.0 // indirect + github.com/eapache/go-resiliency v1.3.0 // indirect github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 // indirect github.com/eapache/queue v1.1.0 // indirect github.com/form3tech-oss/jwt-go v3.2.2+incompatible // indirect github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect - github.com/golang/snappy v0.0.1 // indirect + github.com/golang/snappy v0.0.4 // indirect github.com/google/go-cmp v0.5.5 // indirect github.com/googleapis/gax-go/v2 v2.0.5 // indirect github.com/gorilla/websocket v1.4.2 // indirect - github.com/hashicorp/go-uuid v1.0.2 // indirect - github.com/jcmturner/gofork v1.0.0 // indirect + github.com/hashicorp/errwrap v1.0.0 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect + github.com/hashicorp/go-uuid v1.0.3 // indirect + github.com/jcmturner/aescts/v2 v2.0.0 // indirect + github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect + github.com/jcmturner/gofork v1.7.6 // indirect + github.com/jcmturner/gokrb5/v8 v8.4.3 // indirect + github.com/jcmturner/rpc/v2 v2.0.3 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/jpillora/backoff v1.0.0 // indirect github.com/jstemmer/go-junit-report v0.9.1 // indirect - github.com/klauspost/compress v1.14.4 // indirect + github.com/klauspost/compress v1.15.9 // indirect github.com/mattn/go-runewidth v0.0.3 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/mitchellh/mapstructure v1.1.2 // indirect github.com/nats-io/nats-server/v2 v2.7.4 // indirect github.com/nats-io/nkeys v0.3.0 // indirect github.com/nats-io/nuid v1.0.1 // indirect - github.com/pierrec/lz4 v2.5.2+incompatible // indirect + github.com/pierrec/lz4/v4 v4.1.15 // indirect github.com/prometheus/client_model v0.2.0 // indirect github.com/prometheus/common v0.32.1 // indirect github.com/prometheus/procfs v0.7.3 // indirect - github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 // indirect + github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect github.com/tidwall/cities v0.1.0 // indirect github.com/tidwall/grect v0.1.4 // indirect github.com/tidwall/lotsa v1.0.2 // indirect @@ -92,22 +98,18 @@ require ( go.uber.org/atomic v1.5.0 // indirect go.uber.org/multierr v1.3.0 // indirect go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee // indirect - golang.org/x/crypto v0.0.0-20220112180741-5e0467b6c7ce // indirect + golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa // indirect golang.org/x/lint v0.0.0-20200302205851-738671d3881b // indirect golang.org/x/mod v0.3.0 // indirect golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c // indirect - golang.org/x/sync v0.0.0-20201207232520-09787c993a3a // indirect - golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 // indirect + golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect + golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 // indirect golang.org/x/text v0.3.7 // indirect golang.org/x/tools v0.0.0-20200825202427-b303f430e36d // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect google.golang.org/appengine v1.6.6 // indirect google.golang.org/genproto v0.0.0-20200825200019-8632dd797987 // indirect google.golang.org/protobuf v1.26.0 // indirect - gopkg.in/jcmturner/aescts.v1 v1.0.1 // indirect - gopkg.in/jcmturner/dnsutils.v1 v1.0.1 // indirect - gopkg.in/jcmturner/gokrb5.v7 v7.5.0 // indirect - gopkg.in/jcmturner/rpc.v1 v1.1.0 // indirect honnef.co/go/tools v0.0.1-2020.1.4 // indirect ) diff --git a/go.sum b/go.sum index ec250d65..2d31dbd7 100644 --- a/go.sum +++ b/go.sum @@ -81,10 +81,10 @@ github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBp github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/Shopify/sarama v1.27.2 h1:1EyY1dsxNDUQEv0O/4TsjosHI2CgB1uo9H/v56xzTxc= -github.com/Shopify/sarama v1.27.2/go.mod h1:g5s5osgELxgM+Md9Qni9rzo7Rbt+vvFQI4bt/Mc93II= -github.com/Shopify/toxiproxy v2.1.4+incompatible h1:TKdv8HiTLgE5wdJuEML90aBgNWsokNbMijUGhmcoBJc= -github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= +github.com/Shopify/sarama v1.36.0 h1:0OJs3eCcnezkWniVjwBbCJVaa0B1k7ImCRS3WN6NsSk= +github.com/Shopify/sarama v1.36.0/go.mod h1:9glG3eX83tgVYJ5aVtrjVUnEsOPqQIBGx1BWfN+X51I= +github.com/Shopify/toxiproxy/v2 v2.4.0 h1:O1e4Jfvr/hefNTNu+8VtdEG5lSeamJRo4aKhMOKNM64= +github.com/Shopify/toxiproxy/v2 v2.4.0/go.mod h1:3ilnjng821bkozDRxNoo64oI/DKqM+rOyJzb564+bvg= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -106,6 +106,7 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -115,8 +116,8 @@ github.com/devigned/tab v0.1.1/go.mod h1:XG9mPq0dFghrYvoBF3xdRrJzSTX1b7IQrvaL9mz github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dimchansky/utfbom v1.1.0 h1:FcM3g+nofKgUteL8dm/UpdRXNC9KmADgTpLKsu0TRo4= github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8= -github.com/eapache/go-resiliency v1.2.0 h1:v7g92e/KSN71Rq7vSThKaWIq68fL4YHvWyiUKorFR1Q= -github.com/eapache/go-resiliency v1.2.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= +github.com/eapache/go-resiliency v1.3.0 h1:RRL0nge+cWGlxXbUzJ7yMcq6w2XBEr19dCN6HECGaT0= +github.com/eapache/go-resiliency v1.3.0/go.mod h1:5yPzW0MIvSe0JDsv0v+DvcjEv2FyD6iZYSs1ZI+iQho= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= @@ -132,8 +133,6 @@ github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTg github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= -github.com/frankban/quicktest v1.10.2 h1:19ARM85nVi4xH7xPXuc5eM/udya5ieh7b/Sv+d844Tk= -github.com/frankban/quicktest v1.10.2/go.mod h1:K+q6oSqb0W0Ininfk863uOk1lMy69l/P6txr3mVT54s= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -174,8 +173,8 @@ github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= -github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/gomodule/redigo v1.8.3 h1:HR0kYDX2RJZvAup8CsiJwxB4dTCSC0AaUq6S4SiLwUc= github.com/gomodule/redigo v1.8.3/go.mod h1:P9dn9mFrCBvWhGE1wpxx6fgq7BAeLBk+UUUzlpkBYO0= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -187,7 +186,6 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= @@ -207,17 +205,35 @@ github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= +github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2IGE= +github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= +github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/iwpnd/sectr v0.1.2 h1:FauaPRn5C2tC42HTF7gM3FJZXvGXWc6jabBbIxzTMag= github.com/iwpnd/sectr v0.1.2/go.mod h1:Dm6YXDJCRx1NTfMX/1RMIkGfVp2ORjCY/cQGfbknz4c= -github.com/jcmturner/gofork v1.0.0 h1:J7uCkflzTEhUZ64xqKnkDxq3kzc96ajM1Gli5ktUem8= -github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= +github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8= +github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs= +github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo= +github.com/jcmturner/dnsutils/v2 v2.0.0/go.mod h1:b0TnjGOvI/n42bZa+hmXL+kFJZsFT7G4t3HTlQ184QM= +github.com/jcmturner/gofork v1.7.6 h1:QH0l3hzAU1tfT3rZCnW5zXl+orbkNMMRGJfdJjHVETg= +github.com/jcmturner/gofork v1.7.6/go.mod h1:1622LH6i/EZqLloHfE7IeZ0uEJwMSUyQ/nDd82IeqRo= +github.com/jcmturner/goidentity/v6 v6.0.1 h1:VKnZd2oEIMorCTsFBnJWbExfNN7yZr3EhJAxwOkZg6o= +github.com/jcmturner/goidentity/v6 v6.0.1/go.mod h1:X1YW3bgtvwAXju7V3LCIMpY0Gbxyjn/mY9zx4tFonSg= +github.com/jcmturner/gokrb5/v8 v8.4.3 h1:iTonLeSJOn7MVUtyMT+arAn5AKAPrkilzhGw8wE/Tq8= +github.com/jcmturner/gokrb5/v8 v8.4.3/go.mod h1:dqRwJGXznQrzw6cWmyo6kH+E7jksEQG/CyVWsJEsJO0= +github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY= +github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= @@ -237,19 +253,16 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.11.0/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/compress v1.14.4 h1:eijASRJcobkVtSt81Olfh7JX43osYLwy5krOJo6YEu4= -github.com/klauspost/compress v1.14.4/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY= +github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/mattn/go-runewidth v0.0.3 h1:a+kO+98RDGEfo6asOGMmpodZq4FNtnGP54yps8BzLR4= github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= @@ -278,11 +291,10 @@ github.com/nats-io/nkeys v0.3.0 h1:cgM5tL53EvYRU+2YLXIK0G2mJtK12Ft9oeooSZMA2G8= github.com/nats-io/nkeys v0.3.0/go.mod h1:gvUNGjVcM2IPr5rCsRsC6Wb3Hr2CQAm08dsxtV6A5y4= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/peterh/liner v1.2.1 h1:O4BlKaq/LWu6VRWmol4ByWfzx6MfXc5Op5HETyIy5yg= github.com/peterh/liner v1.2.1/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0= -github.com/pierrec/lz4 v2.5.2+incompatible h1:WCjObylUIOlKy/+7Abdn34TLIkXiA4UWUMhxq9m9ZXI= -github.com/pierrec/lz4 v2.5.2+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0= +github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -311,23 +323,30 @@ github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 h1:MkV+77GLUNo5oJ0jf870itWm3D0Sjh7+Za9gazKc5LQ= -github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= +github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/streadway/amqp v1.0.0 h1:kuuDrUJFZL1QYL9hUNuCxNObNzB0bV/ZG5jV3RWAQgo= github.com/streadway/amqp v1.0.0/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/tidwall/assert v0.1.0 h1:aWcKyRBUAdLoVebxo95N7+YZVTFF/ASTr7BN4sLP6XI= github.com/tidwall/assert v0.1.0/go.mod h1:QLYtGyeqse53vuELQheYl9dngGCJQ+mTtlxcktb+Kj8= github.com/tidwall/btree v1.1.0 h1:5P+9WU8ui5uhmcg3SoPyTwoI0mVyZ1nps7YQzTZFkYM= @@ -364,10 +383,12 @@ github.com/tidwall/sjson v1.2.4 h1:cuiLzLnaMeBhRmEv00Lpk3tkYrcxpmbU81tAY4Dw0tc= github.com/tidwall/sjson v1.2.4/go.mod h1:098SZ494YoMWPmMO6ct4dcFnqxwj9r/gF0Etp19pSNM= github.com/tidwall/tinyqueue v0.1.1 h1:SpNEvEggbpyN5DIReaJ2/1ndroY8iyEGxPYxoSaymYE= github.com/tidwall/tinyqueue v0.1.1/go.mod h1:O/QNHwrnjqr6IHItYrzoHAKYhBkLI67Q096fQP5zMYw= -github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= +github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g= +github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8= github.com/xdg/scram v1.0.5 h1:TuS0RFmt5Is5qm9Tm2SoD89OPqe4IRiFtyFY4iwWXsw= github.com/xdg/scram v1.0.5/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= -github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/xdg/stringprep v1.0.3 h1:cmL5Enob4W83ti/ZHuZLuKD/xqJfus4fVPwE+/BDm+4= github.com/xdg/stringprep v1.0.3/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -396,11 +417,10 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210314154223-e6e6c4f2bb5b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= -golang.org/x/crypto v0.0.0-20220112180741-5e0467b6c7ce h1:Roh6XWxHFKrPgC/EQhVubSAGQ6Ozk6IdxHSzt1mR0EI= -golang.org/x/crypto v0.0.0-20220112180741-5e0467b6c7ce/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa h1:zuSxTR4o9y82ebqCUJYNGJbGPo6sKVl54f/TVDObg1c= +golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -462,12 +482,13 @@ golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 h1:CIJ76btIcR3eFI5EgSo6k1qKw9KJexJuRLI9G7Hp5wE= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220725212005-46097bf591d3/go.mod h1:AaygXjzTFtRAg2ttMY5RMuhpJ3cNnI0XpyFJD1iQRSM= +golang.org/x/net v0.0.0-20220809184613-07c6da5e1ced h1:3dYNDff0VT5xj+mbj2XucFst9WKk6PdGOrb9n+SbIvw= +golang.org/x/net v0.0.0-20220809184613-07c6da5e1ced/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -483,8 +504,9 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201207232520-09787c993a3a h1:DcqTD9SDLc+1P/r1EmRBwnVsrOwW+kk2vWf9n+1sGhs= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -499,6 +521,7 @@ golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -522,10 +545,14 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 h1:XfKQ4OlFl8okEOr5UvAqFRVj8pY/4yfcXrddB8qAbU0= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 h1:WIoqL4EROvwiPdUtaip4VcDdpZ4kha7wBWZrbVKCIZg= +golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -666,20 +693,12 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/jcmturner/aescts.v1 v1.0.1 h1:cVVZBK2b1zY26haWB4vbBiZrfFQnfbTVrE3xZq6hrEw= -gopkg.in/jcmturner/aescts.v1 v1.0.1/go.mod h1:nsR8qBOg+OucoIW+WMhB3GspUQXq9XorLnQb9XtvcOo= -gopkg.in/jcmturner/dnsutils.v1 v1.0.1 h1:cIuC1OLRGZrld+16ZJvvZxVJeKPsvd5eUIvxfoN5hSM= -gopkg.in/jcmturner/dnsutils.v1 v1.0.1/go.mod h1:m3v+5svpVOhtFAP/wSz+yzh4Mc0Fg7eRhxkJMWSIz9Q= -gopkg.in/jcmturner/goidentity.v3 v3.0.0 h1:1duIyWiTaYvVx3YX2CYtpJbUFd7/UuPYCfgXtQ3VTbI= -gopkg.in/jcmturner/goidentity.v3 v3.0.0/go.mod h1:oG2kH0IvSYNIu80dVAyu/yoefjq1mNfM5bm88whjWx4= -gopkg.in/jcmturner/gokrb5.v7 v7.5.0 h1:a9tsXlIDD9SKxotJMK3niV7rPZAJeX2aD/0yg3qlIrg= -gopkg.in/jcmturner/gokrb5.v7 v7.5.0/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM= -gopkg.in/jcmturner/rpc.v1 v1.1.0 h1:QHIUxTX1ISuAv9dD2wJ9HWQVuWDX/Zc0PfeC2tjc4rU= -gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLvuNnlv8= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -687,8 +706,8 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ= -gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/internal/endpoint/kafka.go b/internal/endpoint/kafka.go index 86ed770b..2793e0da 100644 --- a/internal/endpoint/kafka.go +++ b/internal/endpoint/kafka.go @@ -80,26 +80,30 @@ func (conn *KafkaConn) Send(msg string) error { switch conn.ep.Kafka.Auth { case "sasl": - // Other than TLS authentication, SASL does not require SSL + // This path allows to either provide a custom ca certificate + // or, because RootCAs is nil, is using the hosts ca set + // to verify the server certificate if conn.ep.Kafka.SSL { tlsConfig := tls.Config{} - log.Debugf("building kafka tls root config") - caCertPool, err := loadRootTLSCert(conn.ep.Kafka.CACertFile) - if err != nil { - return err + if conn.ep.Kafka.CACertFile != "" { + caCertPool, err := loadRootTLSCert(conn.ep.Kafka.CACertFile) + if err != nil { + return err + } + tlsConfig.RootCAs = &caCertPool } - tlsConfig.RootCAs = &caCertPool cfg.Net.TLS.Enable = true cfg.Net.TLS.Config = &tlsConfig } - log.Debugf("building kafka sasl config") cfg.Net.SASL.Enable = true cfg.Net.SASL.User = os.Getenv("KAFKA_USERNAME") cfg.Net.SASL.Password = os.Getenv("KAFKA_PASSWORD") cfg.Net.SASL.Handshake = true + cfg.Net.SASL.Mechanism = sarama.SASLTypePlaintext + if conn.ep.Kafka.SASLSHA256 { cfg.Net.SASL.SCRAMClientGeneratorFunc = func() sarama.SCRAMClient { return &XDGSCRAMClient{HashGeneratorFcn: SHA256} } cfg.Net.SASL.Mechanism = sarama.SASLTypeSCRAMSHA256 @@ -110,25 +114,26 @@ func (conn *KafkaConn) Send(msg string) error { } case "tls": tlsConfig := tls.Config{} + cfg.Net.TLS.Enable = true - log.Debugf("building kafka tls client config") certificates, err := loadClientTLSCert(conn.ep.Kafka.KeyFile, conn.ep.Kafka.CertFile) if err != nil { cfg.MetricRegistry.UnregisterAll() return err } + tlsConfig.Certificates = certificates - // TLS authentication requires SSL - // and Tile38 requires certificates to be validated - caCertPool, err := loadRootTLSCert(conn.ep.Kafka.CACertFile) - if err != nil { - return err + // This path allows to either provide a custom ca certificate + // or, because RootCAs is nil, is using the hosts ca set + // to verify server certificate + if conn.ep.Kafka.CACertFile != "" { + caCertPool, err := loadRootTLSCert(conn.ep.Kafka.CACertFile) + if err != nil { + return err + } + tlsConfig.RootCAs = &caCertPool } - cfg.Net.TLS.Enable = true - - tlsConfig.Certificates = certificates - tlsConfig.RootCAs = &caCertPool cfg.Net.TLS.Config = &tlsConfig } From 7f2ce239494c228b87a84b543c8d165107e3d4e1 Mon Sep 17 00:00:00 2001 From: tidwall Date: Sun, 11 Sep 2022 12:39:23 -0700 Subject: [PATCH 11/13] Upgrade to Go 1.19 --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 37587058..0fdaea26 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -16,7 +16,7 @@ jobs: - name: Set up Go 1.x uses: actions/setup-go@v2 with: - go-version: ^1.18 + go-version: ^1.19 - name: Check out code uses: actions/checkout@v2 From 498bbe23ff0010efcabd1b36c74dd000794840b1 Mon Sep 17 00:00:00 2001 From: tidwall Date: Sun, 11 Sep 2022 14:31:00 -0700 Subject: [PATCH 12/13] Updated btree and rtree This commit updates to the latest btree and rtree. The rtree algorithm has been modified in `tidwall/rtree@v1.7` which now keeps internal and leaf rect sorted by the min-x coordinate. This make for much faster (up to 50%) faster searches and replacements, but slightly slower inserts. Because of the R-tree update, the tests needed to be updated to account for the change in order for undeterministic WITHIN and INTERSECTS commands. --- go.mod | 6 +- go.sum | 9 ++- tests/keys_search_test.go | 115 ++++++++++++++++++++++---------------- tests/keys_test.go | 7 ++- 4 files changed, 81 insertions(+), 56 deletions(-) diff --git a/go.mod b/go.mod index deda3b75..8eefc51c 100644 --- a/go.mod +++ b/go.mod @@ -16,9 +16,9 @@ require ( github.com/peterh/liner v1.2.1 github.com/prometheus/client_golang v1.12.1 github.com/streadway/amqp v1.0.0 - github.com/tidwall/btree v1.1.0 + github.com/tidwall/btree v1.4.2 github.com/tidwall/buntdb v1.2.9 - github.com/tidwall/geoindex v1.4.4 + github.com/tidwall/geoindex v1.6.2 github.com/tidwall/geojson v1.3.4 github.com/tidwall/gjson v1.12.1 github.com/tidwall/match v1.1.1 @@ -26,7 +26,7 @@ require ( github.com/tidwall/redbench v0.1.0 github.com/tidwall/redcon v1.4.4 github.com/tidwall/resp v0.1.0 - github.com/tidwall/rtree v1.3.1 + github.com/tidwall/rtree v1.7.1 github.com/tidwall/sjson v1.2.4 github.com/xdg/scram v1.0.5 github.com/yuin/gopher-lua v0.0.0-20200816102855-ee81675732da diff --git a/go.sum b/go.sum index 2d31dbd7..b4416271 100644 --- a/go.sum +++ b/go.sum @@ -349,14 +349,16 @@ github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PK github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/tidwall/assert v0.1.0 h1:aWcKyRBUAdLoVebxo95N7+YZVTFF/ASTr7BN4sLP6XI= github.com/tidwall/assert v0.1.0/go.mod h1:QLYtGyeqse53vuELQheYl9dngGCJQ+mTtlxcktb+Kj8= -github.com/tidwall/btree v1.1.0 h1:5P+9WU8ui5uhmcg3SoPyTwoI0mVyZ1nps7YQzTZFkYM= github.com/tidwall/btree v1.1.0/go.mod h1:TzIRzen6yHbibdSfK6t8QimqbUnoxUSrZfeW7Uob0q4= +github.com/tidwall/btree v1.4.2 h1:PpkaieETJMUxYNADsjgtNRcERX7mGc/GP2zp/r5FM3g= +github.com/tidwall/btree v1.4.2/go.mod h1:LGm8L/DZjPLmeWGjv5kFrY8dL4uVhMmzmmLYmsObdKE= github.com/tidwall/buntdb v1.2.9 h1:XVz684P7X6HCTrdr385yDZWB1zt/n20ZNG3M1iGyFm4= github.com/tidwall/buntdb v1.2.9/go.mod h1:IwyGSvvDg6hnKSIhtdZ0AqhCZGH8ukdtCAzaP8fI1X4= github.com/tidwall/cities v0.1.0 h1:CVNkmMf7NEC9Bvokf5GoSsArHCKRMTgLuubRTHnH0mE= github.com/tidwall/cities v0.1.0/go.mod h1:lV/HDp2gCcRcHJWqgt6Di54GiDrTZwh1aG2ZUPNbqa4= -github.com/tidwall/geoindex v1.4.4 h1:hdwzy5qNtK75i7nus59Ibr+SwcH4F2v65bw4txrLJ9M= github.com/tidwall/geoindex v1.4.4/go.mod h1:rvVVNEFfkJVWGUdEfU8QaoOg/9zFX0h9ofWzA60mz1I= +github.com/tidwall/geoindex v1.6.2 h1:cWbqC9HFXMxc2p6KMWbs9VG6/gnrfC53EIPQEMcXO1g= +github.com/tidwall/geoindex v1.6.2/go.mod h1:rvVVNEFfkJVWGUdEfU8QaoOg/9zFX0h9ofWzA60mz1I= github.com/tidwall/geojson v1.3.4 h1:mHB2yGK7HPgf4vFkLdPeIzguFpqkmCT2yTgGhXbrqBo= github.com/tidwall/geojson v1.3.4/go.mod h1:1cn3UWfSYCJOq53NZoQ9rirdw89+DM0vw+ZOAVvuReg= github.com/tidwall/gjson v1.12.1 h1:ikuZsLdhr8Ws0IdROXUS1Gi4v9Z4pGqpX/CvJkxvfpo= @@ -377,8 +379,9 @@ github.com/tidwall/resp v0.1.0 h1:zZ6Hq+2cY4QqhZ4LqrV05T5yLOSPspj+l+DgAoJ25Ak= github.com/tidwall/resp v0.1.0/go.mod h1:18xEj855iMY2bK6tNF2A4x+nZy5gWO1iO7OOl3jETKw= github.com/tidwall/rtred v0.1.2 h1:exmoQtOLvDoO8ud++6LwVsAMTu0KPzLTUrMln8u1yu8= github.com/tidwall/rtred v0.1.2/go.mod h1:hd69WNXQ5RP9vHd7dqekAz+RIdtfBogmglkZSRxCHFQ= -github.com/tidwall/rtree v1.3.1 h1:xu3vJPKJrmGce7YJcFUCoqLrp9DTUEJBnVgdPSXHgHs= github.com/tidwall/rtree v1.3.1/go.mod h1:S+JSsqPTI8LfWA4xHBo5eXzie8WJLVFeppAutSegl6M= +github.com/tidwall/rtree v1.7.1 h1:rv3Q8RBKH2HbJ6DsqpfrXfV9l+dCT1jupTpDiceHN3I= +github.com/tidwall/rtree v1.7.1/go.mod h1:39+jGCj9hYqhflezmsTBOlysIk09ytm+8EQsC/E/2X0= github.com/tidwall/sjson v1.2.4 h1:cuiLzLnaMeBhRmEv00Lpk3tkYrcxpmbU81tAY4Dw0tc= github.com/tidwall/sjson v1.2.4/go.mod h1:098SZ494YoMWPmMO6ct4dcFnqxwj9r/gF0Etp19pSNM= github.com/tidwall/tinyqueue v0.1.1 h1:SpNEvEggbpyN5DIReaJ2/1ndroY8iyEGxPYxoSaymYE= diff --git a/tests/keys_search_test.go b/tests/keys_search_test.go index 1f8ebb12..0415fcf1 100644 --- a/tests/keys_search_test.go +++ b/tests/keys_search_test.go @@ -146,8 +146,9 @@ func keys_WITHIN_test(mc *mockServer) error { [-122.44126439094543,37.72906137107] ] ] - }`}, {"[0 [point1 point2 line3 poly4 multipoly5 poly8]]"}, - {"WITHIN", "mykey", "IDS", "SECTOR", "37.72999", "-122.44760", "1000", "0", "90"}, {"[0 [point1 point2 line3 poly4 multipoly5 poly8]]"}, + }`}, {"[0 [point2 point1 multipoly5 poly8 poly4 line3]]"}, + {"WITHIN", "mykey", "IDS", "SECTOR", "37.72999", "-122.44760", "1000", "0", "90"}, { + "[0 [point2 point1 multipoly5 poly8 poly4 line3]]"}, {"WITHIN", "mykey", "IDS", "SECTOR", "37.72999", "-122.44760", "1000", "0", "0"}, {"ERR equal bearings (0 == 0), use CIRCLE instead"}, {"SET", "key2", "poly9", "OBJECT", `{"type":"Polygon","coordinates":[[[-122.44037926197052,37.73313523548048],[-122.44017541408539,37.73313523548048],[-122.44017541408539,37.73336857568778],[-122.44037926197052,37.73336857568778],[-122.44037926197052,37.73313523548048]]]}`}, {"OK"}, @@ -197,19 +198,25 @@ func keys_WITHIN_CURSOR_test(mc *mockServer) error { {"SET", "mykey", "poly8", "FIELD", "foo", 8, "OBJECT", `{"type":"Polygon","coordinates":[[[-122.4408378,37.7341129],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.7341129],[-122.4408378,37.7341129]],[[-122.44060993194579,37.73345766902749],[-122.44044363498686,37.73345766902749],[-122.44044363498686,37.73355524732416],[-122.44060993194579,37.73355524732416],[-122.44060993194579,37.73345766902749]],[[-122.44060724973677,37.7336888869566],[-122.4402102828026,37.7336888869566],[-122.4402102828026,37.7339752567853],[-122.44060724973677,37.7339752567853],[-122.44060724973677,37.7336888869566]]]}`}, {"OK"}, {"SET", "mykey", "point9", "FIELD", "foo", 9, "POINT", 37.7335, -122.4412}, {"OK"}, {"WITHIN", "mykey", "LIMIT", 3, "IDS", "OBJECT", testArea}, { - "[3 [point1 point2 line3]]"}, + "[3 [point2 point9 point1]]"}, {"WITHIN", "mykey", "CURSOR", 3, "LIMIT", 3, "IDS", "OBJECT", testArea}, { - "[6 [poly4 multipoly5 poly8]]"}, + "[6 [multipoly5 poly8 poly4]]"}, {"WITHIN", "mykey", "WHERE", "foo", 3, 5, "IDS", "OBJECT", testArea}, { - "[0 [line3 poly4 multipoly5]]"}, + "[0 [multipoly5 poly4 line3]]"}, {"WITHIN", "mykey", "LIMIT", 1, "WHERE", "foo", 3, 5, "IDS", "OBJECT", testArea}, { - "[3 [line3]]"}, + "[4 [multipoly5]]"}, + {"WITHIN", "mykey", "CURSOR", 0, "LIMIT", 1, "WHERE", "foo", 8, 9, "IDS", "OBJECT", testArea}, { + "[2 [point9]]"}, + {"WITHIN", "mykey", "CURSOR", 1, "LIMIT", 1, "WHERE", "foo", 8, 9, "IDS", "OBJECT", testArea}, { + "[2 [point9]]"}, + {"WITHIN", "mykey", "CURSOR", 2, "LIMIT", 1, "WHERE", "foo", 8, 9, "IDS", "OBJECT", testArea}, { + "[5 [poly8]]"}, {"WITHIN", "mykey", "CURSOR", 3, "LIMIT", 1, "WHERE", "foo", 8, 9, "IDS", "OBJECT", testArea}, { - "[6 [poly8]]"}, - {"WITHIN", "mykey", "CURSOR", 6, "LIMIT", 1, "WHERE", "foo", 8, 9, "IDS", "OBJECT", testArea}, { - "[7 [point9]]"}, - {"WITHIN", "mykey", "CURSOR", 6, "LIMIT", 1, "WHERE", "foo", 8, 9, "IDS", "SECTOR", "37.72999", "-122.44760", "1000", "0", "90"}, { - "[7 [point9]]"}, + "[5 [poly8]]"}, + {"WITHIN", "mykey", "CURSOR", 4, "LIMIT", 1, "WHERE", "foo", 8, 9, "IDS", "OBJECT", testArea}, { + "[5 [poly8]]"}, + {"WITHIN", "mykey", "CURSOR", 5, "LIMIT", 1, "WHERE", "foo", 8, 9, "IDS", "OBJECT", testArea}, { + "[0 []]"}, }) } @@ -244,7 +251,7 @@ func keys_WITHIN_CLIPBY_test(mc *mockServer) error { }, {"[0 [point4]]"}, {"WITHIN", "mykey", "IDS", "BOUNDS", 37.74411415606583, -122.48034954071045, 37.7536833241461, -122.47163772583008, - }, {"[0 [point2 point3 point4]]"}, + }, {"[0 [point3 point4 point2]]"}, {"WITHIN", "mykey", "IDS", "GET", "other", "jagged", "CLIPBY", "BOUNDS", 37.74411415606583, -122.48034954071045, 37.7536833241461, -122.47163772583008, }, {"[0 [point4]]"}, @@ -279,33 +286,39 @@ func keys_INTERSECTS_test(mc *mockServer) error { [-122.44126439094543,37.732906137107] ] ] - }`}, {"[0 [point1 point2 line3 poly4 multipoly5 poly8]]"}, - {"INTERSECTS", "mykey", "IDS", "SECTOR", "37.72999", "-122.44760", "1000", "0", "90"}, {"[0 [point1 point2 line3 poly4 multipoly5 poly8]]"}, + }`}, {"[0 [point2 point1 multipoly5 poly8 poly4 line3]]"}, + {"INTERSECTS", "mykey", "IDS", "SECTOR", "37.72999", "-122.44760", "1000", "0", "90"}, { + "[0 [point2 point1 multipoly5 poly8 poly4 line3]]"}, {"INTERSECTS", "mykey", "IDS", "SECTOR", "37.72999", "-122.44760", "1000", "0", "0"}, {"ERR equal bearings (0 == 0), use CIRCLE instead"}, {"SET", "key2", "poly9", "OBJECT", `{"type": "Polygon","coordinates": [[[-122.44037926197052,37.73313523548048],[-122.44017541408539,37.73313523548048],[-122.44017541408539,37.73336857568778],[-122.44037926197052,37.73336857568778],[-122.44037926197052,37.73313523548048]]]}`}, {"OK"}, {"SET", "key2", "poly10", "OBJECT", `{"type": "Polygon","coordinates": [[[-122.44040071964262,37.73359343010089],[-122.4402666091919,37.73359343010089],[-122.4402666091919,37.73373767596864],[-122.44040071964262,37.73373767596864],[-122.44040071964262,37.73359343010089]]]}`}, {"OK"}, {"SET", "key2", "poly10.1", "OBJECT", `{"type":"Polygon","coordinates":[[[-122.44051605463028,37.73375464605226],[-122.44028002023695,37.73375464605226],[-122.44028002023695,37.733903134117966],[-122.44051605463028,37.733903134117966],[-122.44051605463028,37.73375464605226]]]}`}, {"OK"}, - {"INTERSECTS", "key2", "IDS", "GET", "mykey", "poly8"}, {"[0 [poly9 poly10]]"}, + {"INTERSECTS", "key2", "IDS", "GET", "mykey", "poly8"}, { + "[0 [poly10 poly9]]"}, {"SET", "key3", "poly11", "OBJECT", `{"type":"Polygon","coordinates":[[[-122.44059920310973,37.733279482240874],[-122.4402344226837,37.733279482240874],[-122.4402344226837,37.73375464605226],[-122.44059920310973,37.73375464605226],[-122.44059920310973,37.733279482240874]]]}`}, {"OK"}, {"SET", "key3", "poly12", "OBJECT", `{"type":"Polygon","coordinates":[[[-122.44060993194579,37.73238005667773],[-122.44013786315918,37.73238005667773],[-122.44013786315918,37.73316917591997],[-122.44060993194579,37.73316917591997],[-122.44060993194579,37.73238005667773]]]}`}, {"OK"}, {"SET", "key3", "poly12.1", "OBJECT", `{"type":"Polygon","coordinates":[[[-122.44074940681458,37.73270249351328],[-122.44008421897887,37.73270249351328],[-122.44008421897887,37.732872196546936],[-122.44074940681458,37.732872196546936],[-122.44074940681458,37.73270249351328]]]}`}, {"OK"}, - {"INTERSECTS", "key3", "IDS", "GET", "mykey", "multipoly5"}, {"[0 [poly11 poly12]]"}, + {"INTERSECTS", "key3", "IDS", "GET", "mykey", "multipoly5"}, { + "[0 [poly12 poly11]]"}, {"SET", "key5", "poly13", "OBJECT", `{"type":"Polygon","coordinates":[[[-122.44073867797852,37.733211601447465],[-122.44011640548705,37.733211601447465],[-122.44011640548705,37.7340516218859],[-122.44073867797852,37.7340516218859],[-122.44073867797852,37.733211601447465]],[[-122.44060993194579,37.73345766902749],[-122.44060993194579,37.73355524732416],[-122.44044363498686,37.73355524732416],[-122.44044363498686,37.73345766902749],[-122.44060993194579,37.73345766902749]],[[-122.44060724973677,37.7336888869566],[-122.44060724973677,37.7339752567853],[-122.4402102828026,37.7339752567853],[-122.4402102828026,37.7336888869566],[-122.44060724973677,37.7336888869566]]]}`}, {"OK"}, {"SET", "key5", "poly14", "OBJECT", `{"type":"Polygon","coordinates":[[[-122.44154334068298,37.73179457567642],[-122.43935465812682,37.73179457567642],[-122.43935465812682,37.7343740514423],[-122.44154334068298,37.7343740514423],[-122.44154334068298,37.73179457567642]],[[-122.44104981422423,37.73286371140448],[-122.44104981422423,37.73424677678513],[-122.43990182876587,37.73424677678513],[-122.43990182876587,37.73286371140448],[-122.44104981422423,37.73286371140448]],[[-122.44109272956847,37.731870943026074],[-122.43976235389708,37.731870943026074],[-122.43976235389708,37.7326855231885],[-122.44109272956847,37.7326855231885],[-122.44109272956847,37.731870943026074]]]}`}, {"OK"}, - {"INTERSECTS", "key5", "IDS", "GET", "mykey", "multipoly5"}, {"[0 [poly13]]"}, + {"INTERSECTS", "key5", "IDS", "GET", "mykey", "multipoly5"}, { + "[0 [poly13]]"}, {"SET", "key6", "multipoly5", "OBJECT", `{"type":"MultiPolygon","coordinates":[[[[-122.4408378,37.7341129],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.7341129],[-122.4408378,37.7341129]]],[[[-122.44091033935547,37.731981251280985],[-122.43994474411011,37.731981251280985],[-122.43994474411011,37.73254976045042],[-122.44091033935547,37.73254976045042],[-122.44091033935547,37.731981251280985]]]]}`}, {"OK"}, {"SET", "key6", "poly13", "OBJECT", `{"type":"Polygon","coordinates":[[[-122.44073867797852,37.733211601447465],[-122.44011640548705,37.733211601447465],[-122.44011640548705,37.7340516218859],[-122.44073867797852,37.7340516218859],[-122.44073867797852,37.733211601447465]],[[-122.44060993194579,37.73345766902749],[-122.44060993194579,37.73355524732416],[-122.44044363498686,37.73355524732416],[-122.44044363498686,37.73345766902749],[-122.44060993194579,37.73345766902749]],[[-122.44060724973677,37.7336888869566],[-122.44060724973677,37.7339752567853],[-122.4402102828026,37.7339752567853],[-122.4402102828026,37.7336888869566],[-122.44060724973677,37.7336888869566]]]}`}, {"OK"}, - {"INTERSECTS", "key6", "IDS", "GET", "key5", "poly14"}, {"[0 []]"}, + {"INTERSECTS", "key6", "IDS", "GET", "key5", "poly14"}, { + "[0 []]"}, {"SET", "key7", "multipoly15", "OBJECT", `{"type":"MultiPolygon","coordinates":[[[[-122.44056701660155,37.7332964524295],[-122.44029879570007,37.7332964524295],[-122.44029879570007,37.73375464605226],[-122.44056701660155,37.73375464605226],[-122.44056701660155,37.7332964524295]]],[[[-122.44067430496216,37.73217641163713],[-122.44034171104431,37.73217641163713],[-122.44034171104431,37.732430967850384],[-122.44067430496216,37.732430967850384],[-122.44067430496216,37.73217641163713]]]]}`}, {"OK"}, {"SET", "key7", "multipoly16", "OBJECT", `{"type":"MultiPolygon","coordinates":[[[[-122.44056701660155,37.7332964524295],[-122.44029879570007,37.7332964524295],[-122.44029879570007,37.73375464605226],[-122.44056701660155,37.73375464605226],[-122.44056701660155,37.7332964524295]]],[[[-122.4402666091919,37.733109780140644],[-122.4401271343231,37.733109780140644],[-122.4401271343231,37.73323705675229],[-122.4402666091919,37.73323705675229],[-122.4402666091919,37.733109780140644]]]]}`}, {"OK"}, {"SET", "key7", "multipoly17", "OBJECT", `{"type":"MultiPolygon","coordinates":[[[[-122.44056701660155,37.7332964524295],[-122.44029879570007,37.7332964524295],[-122.44029879570007,37.73375464605226],[-122.44056701660155,37.73375464605226],[-122.44056701660155,37.7332964524295]]],[[[-122.44032025337218,37.73267703802467],[-122.44013786315918,37.73267703802467],[-122.44013786315918,37.732838255971316],[-122.44032025337218,37.732838255971316],[-122.44032025337218,37.73267703802467]]]]}`}, {"OK"}, {"SET", "key7", "multipoly17.1", "OBJECT", `{"type":"MultiPolygon","coordinates":[[[[-122.4407172203064,37.73270249351328],[-122.44049191474916,37.73270249351328],[-122.44049191474916,37.73286371140448],[-122.4407172203064,37.73286371140448],[-122.4407172203064,37.73270249351328]]],[[[-122.44032025337218,37.73267703802467],[-122.44013786315918,37.73267703802467],[-122.44013786315918,37.732838255971316],[-122.44032025337218,37.732838255971316],[-122.44032025337218,37.73267703802467]]]]}`}, {"OK"}, - {"INTERSECTS", "key7", "IDS", "GET", "mykey", "multipoly5"}, {"[0 [multipoly15 multipoly16 multipoly17]]"}, + {"INTERSECTS", "key7", "IDS", "GET", "mykey", "multipoly5"}, { + "[0 [multipoly15 multipoly17 multipoly16]]"}, }) } @@ -340,7 +353,7 @@ func keys_INTERSECTS_CLIPBY_test(mc *mockServer) error { }, {"[0 [point4]]"}, {"INTERSECTS", "mykey", "IDS", "BOUNDS", 37.74411415606583, -122.48034954071045, 37.7536833241461, -122.47163772583008, - }, {"[0 [point2 point3 point4]]"}, + }, {"[0 [point3 point4 point2]]"}, {"INTERSECTS", "mykey", "IDS", "GET", "other", "jagged", "CLIPBY", "BOUNDS", 37.74411415606583, -122.48034954071045, 37.7536833241461, -122.47163772583008, }, {"[0 [point4]]"}, @@ -377,19 +390,25 @@ func keys_INTERSECTS_CURSOR_test(mc *mockServer) error { {"SET", "mykey", "poly8", "FIELD", "foo", 8, "OBJECT", `{"type":"Polygon","coordinates":[[[-122.4408378,37.7341129],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.7341129],[-122.4408378,37.7341129]],[[-122.44060993194579,37.73345766902749],[-122.44044363498686,37.73345766902749],[-122.44044363498686,37.73355524732416],[-122.44060993194579,37.73355524732416],[-122.44060993194579,37.73345766902749]],[[-122.44060724973677,37.7336888869566],[-122.4402102828026,37.7336888869566],[-122.4402102828026,37.7339752567853],[-122.44060724973677,37.7339752567853],[-122.44060724973677,37.7336888869566]]]}`}, {"OK"}, {"SET", "mykey", "point9", "FIELD", "foo", 9, "POINT", 37.7335, -122.4412}, {"OK"}, {"INTERSECTS", "mykey", "LIMIT", 3, "IDS", "OBJECT", testArea}, { - "[3 [point1 point2 line3]]"}, + "[3 [point2 point9 point1]]"}, {"INTERSECTS", "mykey", "CURSOR", 3, "LIMIT", 3, "IDS", "OBJECT", testArea}, { - "[6 [poly4 multipoly5 poly8]]"}, + "[6 [multipoly5 poly8 poly4]]"}, {"INTERSECTS", "mykey", "WHERE", "foo", 3, 5, "IDS", "OBJECT", testArea}, { - "[0 [line3 poly4 multipoly5]]"}, + "[0 [multipoly5 poly4 line3]]"}, {"INTERSECTS", "mykey", "LIMIT", 1, "WHERE", "foo", 3, 5, "IDS", "OBJECT", testArea}, { - "[3 [line3]]"}, + "[4 [multipoly5]]"}, + {"INTERSECTS", "mykey", "CURSOR", 1, "LIMIT", 1, "WHERE", "foo", 8, 9, "IDS", "OBJECT", testArea}, { + "[2 [point9]]"}, + {"INTERSECTS", "mykey", "CURSOR", 2, "LIMIT", 1, "WHERE", "foo", 8, 9, "IDS", "OBJECT", testArea}, { + "[5 [poly8]]"}, {"INTERSECTS", "mykey", "CURSOR", 3, "LIMIT", 1, "WHERE", "foo", 8, 9, "IDS", "OBJECT", testArea}, { - "[6 [poly8]]"}, + "[5 [poly8]]"}, + {"INTERSECTS", "mykey", "CURSOR", 4, "LIMIT", 1, "WHERE", "foo", 8, 9, "IDS", "OBJECT", testArea}, { + "[5 [poly8]]"}, + {"INTERSECTS", "mykey", "CURSOR", 5, "LIMIT", 1, "WHERE", "foo", 8, 9, "IDS", "OBJECT", testArea}, { + "[0 []]"}, {"INTERSECTS", "mykey", "CURSOR", 6, "LIMIT", 1, "WHERE", "foo", 8, 9, "IDS", "OBJECT", testArea}, { - "[7 [point9]]"}, - {"INTERSECTS", "mykey", "CURSOR", 6, "LIMIT", 1, "WHERE", "foo", 8, 9, "IDS", "SECTOR", "37.72999", "-122.44760", "1000", "0", "90"}, { - "[7 [point9]]"}, + "[0 []]"}, }) } @@ -403,9 +422,9 @@ func keys_WITHIN_CIRCLE_test(mc *mockServer) error { {"SET", "mykey", "6", "POINT", -5, 5}, {"OK"}, {"SET", "mykey", "7", "POINT", 33, 21}, {"OK"}, {"WITHIN", "mykey", "IDS", "CIRCLE", 37.7335, -122.4412, 1000}, { - "[0 [1 2 3 4 5]]"}, + "[0 [2 1 5 4 3]]"}, {"WITHIN", "mykey", "IDS", "CIRCLE", 37.7335, -122.4412, 10}, { - "[0 [1 2]]"}, + "[0 [2 1]]"}, }) } @@ -419,9 +438,9 @@ func keys_WITHIN_SECTOR_test(mc *mockServer) error { {"SET", "mykey", "6", "POINT", -5, 5}, {"OK"}, {"SET", "mykey", "7", "POINT", 33, 21}, {"OK"}, {"WITHIN", "mykey", "IDS", "SECTOR", 37.731930, -122.443270, 1000, 0, 90}, { - "[0 [1 2 3 4 5]]"}, + "[0 [2 1 5 4 3]]"}, {"WITHIN", "mykey", "IDS", "SECTOR", 37.731930, -122.443270, 100, 0, 90}, { - "[0 [1 2]]"}, + "[0 [2 1]]"}, }) } @@ -518,9 +537,9 @@ func keys_INTERSECTS_CIRCLE_test(mc *mockServer) error { {"SET", "mykey", "6", "POINT", -5, 5}, {"OK"}, {"SET", "mykey", "7", "POINT", 33, 21}, {"OK"}, {"INTERSECTS", "mykey", "IDS", "CIRCLE", 37.7335, -122.4412, 70}, { - "[0 [1 2 3 4 5]]"}, + "[0 [2 1 5 4 3]]"}, {"INTERSECTS", "mykey", "IDS", "CIRCLE", 37.7335, -122.4412, 10}, { - "[0 [1 2]]"}, + "[0 [2 1]]"}, }) } @@ -534,9 +553,9 @@ func keys_INTERSECTS_SECTOR_test(mc *mockServer) error { {"SET", "mykey", "6", "POINT", -5, 5}, {"OK"}, {"SET", "mykey", "7", "POINT", 33, 21}, {"OK"}, {"INTERSECTS", "mykey", "IDS", "SECTOR", 37.731930, -122.443270, 1000, 0, 90}, { - "[0 [1 2 3 4 5]]"}, + "[0 [2 1 5 4 3]]"}, {"INTERSECTS", "mykey", "IDS", "SECTOR", 37.731930, -122.443270, 100, 0, 90}, { - "[0 [1 2]]"}, + "[0 [2 1]]"}, }) } @@ -655,7 +674,8 @@ func keys_FIELDS_search_test(mc *mockServer) error { `[1 {"type":"Point","coordinates":[-112.2791,33.522]} [field1 10 field2 11]] ` + `[3 {"type":"Point","coordinates":[-112.2795,33.5224]} [field1 30 field2 13]] ` + `[4 {"type":"Point","coordinates":[-112.2797,33.5226]} [field1 40 field2 14]] ` + - `[5 {"type":"Point","coordinates":[-112.2799,33.5228]} [field2 15 field3 28]]]]`}, + `[5 {"type":"Point","coordinates":[-112.2799,33.5228]} [field2 15 field3 28]]]]`, + }, {"NEARBY", "mykey", "WHERE", "field2", 0, 2, "POINT", 33.462, -112.268, 60000}, { `[0 [` + `[6 {"type":"Point","coordinates":[-112.2801,33.523]} [field3 29]] ` + @@ -663,14 +683,15 @@ func keys_FIELDS_search_test(mc *mockServer) error { {"WITHIN", "mykey", "WHERE", "field2", 11, "+inf", "CIRCLE", 33.462, -112.268, 60000}, { `[0 [` + - `[1 {"type":"Point","coordinates":[-112.2791,33.522]} [field1 10 field2 11]] ` + - `[3 {"type":"Point","coordinates":[-112.2795,33.5224]} [field1 30 field2 13]] ` + + `[5 {"type":"Point","coordinates":[-112.2799,33.5228]} [field2 15 field3 28]] ` + `[4 {"type":"Point","coordinates":[-112.2797,33.5226]} [field1 40 field2 14]] ` + - `[5 {"type":"Point","coordinates":[-112.2799,33.5228]} [field2 15 field3 28]]]]`}, + `[3 {"type":"Point","coordinates":[-112.2795,33.5224]} [field1 30 field2 13]] ` + + `[1 {"type":"Point","coordinates":[-112.2791,33.522]} [field1 10 field2 11]]]]`, + }, {"WITHIN", "mykey", "WHERE", "field2", 0, 2, "CIRCLE", 33.462, -112.268, 60000}, { `[0 [` + - `[6 {"type":"Point","coordinates":[-112.2801,33.523]} [field3 29]] ` + - `[7 {"type":"Point","coordinates":[-112.2803,33.5232]}]]]`}, + `[7 {"type":"Point","coordinates":[-112.2803,33.5232]}] ` + + `[6 {"type":"Point","coordinates":[-112.2801,33.523]} [field3 29]]]]`}, // test JSON output {"OUTPUT", "json"}, {`{"ok":true}`}, @@ -689,15 +710,15 @@ func keys_FIELDS_search_test(mc *mockServer) error { {"WITHIN", "mykey", "WHERE", "field2", 11, "+inf", "CIRCLE", 33.462, -112.268, 60000}, { `{"ok":true,"fields":["field1","field2","field3"],"objects":[` + - `{"id":"1","object":{"type":"Point","coordinates":[-112.2791,33.522]},"fields":[10,11,0]},` + - `{"id":"3","object":{"type":"Point","coordinates":[-112.2795,33.5224]},"fields":[30,13,0]},` + + `{"id":"5","object":{"type":"Point","coordinates":[-112.2799,33.5228]},"fields":[0,15,28]},` + `{"id":"4","object":{"type":"Point","coordinates":[-112.2797,33.5226]},"fields":[40,14,0]},` + - `{"id":"5","object":{"type":"Point","coordinates":[-112.2799,33.5228]},"fields":[0,15,28]}` + + `{"id":"3","object":{"type":"Point","coordinates":[-112.2795,33.5224]},"fields":[30,13,0]},` + + `{"id":"1","object":{"type":"Point","coordinates":[-112.2791,33.522]},"fields":[10,11,0]}` + `],"count":4,"cursor":0}`}, {"WITHIN", "mykey", "WHERE", "field2", 0, 2, "CIRCLE", 33.462, -112.268, 60000}, { `{"ok":true,"fields":["field1","field2","field3"],"objects":[` + - `{"id":"6","object":{"type":"Point","coordinates":[-112.2801,33.523]},"fields":[0,0,29]},` + - `{"id":"7","object":{"type":"Point","coordinates":[-112.2803,33.5232]},"fields":[0,0,0]}` + + `{"id":"7","object":{"type":"Point","coordinates":[-112.2803,33.5232]},"fields":[0,0,0]},` + + `{"id":"6","object":{"type":"Point","coordinates":[-112.2801,33.523]},"fields":[0,0,29]}` + `],"count":2,"cursor":0}`}, }) } diff --git a/tests/keys_test.go b/tests/keys_test.go index 0170ca27..5e5e8254 100644 --- a/tests/keys_test.go +++ b/tests/keys_test.go @@ -393,10 +393,11 @@ func keys_WHEREIN_test(mc *mockServer) error { {"WITHIN", "mykey", "WHEREIN", "a", 3, 0, "a", 2, "BOUNDS", 32.8, -115.2, 33.2, -114.8}, {"ERR invalid argument 'a'"}, {"SET", "mykey", "myid_a2", "FIELD", "a", 2, "POINT", 32.99, -115}, {"OK"}, {"SET", "mykey", "myid_a3", "FIELD", "a", 3, "POINT", 33, -115.02}, {"OK"}, - {"WITHIN", "mykey", "WHEREIN", "a", 3, 0, 1, 2, "BOUNDS", 32.8, -115.2, 33.2, -114.8}, {`[0 [[myid_a1 {"type":"Point","coordinates":[-115,33]} [a 1]] [myid_a2 {"type":"Point","coordinates":[-115,32.99]} [a 2]]]]`}, + {"WITHIN", "mykey", "WHEREIN", "a", 3, 0, 1, 2, "BOUNDS", 32.8, -115.2, 33.2, -114.8}, { + `[0 [[myid_a2 {"type":"Point","coordinates":[-115,32.99]} [a 2]] [myid_a1 {"type":"Point","coordinates":[-115,33]} [a 1]]]]`}, // zero value should not match 1 and 2 {"SET", "mykey", "myid_a0", "FIELD", "a", 0, "POINT", 33, -115.02}, {"OK"}, - {"WITHIN", "mykey", "WHEREIN", "a", 2, 1, 2, "BOUNDS", 32.8, -115.2, 33.2, -114.8}, {`[0 [[myid_a1 {"type":"Point","coordinates":[-115,33]} [a 1]] [myid_a2 {"type":"Point","coordinates":[-115,32.99]} [a 2]]]]`}, + {"WITHIN", "mykey", "WHEREIN", "a", 2, 1, 2, "BOUNDS", 32.8, -115.2, 33.2, -114.8}, {`[0 [[myid_a2 {"type":"Point","coordinates":[-115,32.99]} [a 2]] [myid_a1 {"type":"Point","coordinates":[-115,33]} [a 1]]]]`}, }) } @@ -408,6 +409,6 @@ func keys_WHEREEVAL_test(mc *mockServer) error { {"WITHIN", "mykey", "WHEREEVAL", "return FIELDS.a > tonumber(ARGV[1])", 1, 0.5, 4, "BOUNDS", 32.8, -115.2, 33.2, -114.8}, {"ERR invalid argument '4'"}, {"SET", "mykey", "myid_a2", "FIELD", "a", 2, "POINT", 32.99, -115}, {"OK"}, {"SET", "mykey", "myid_a3", "FIELD", "a", 3, "POINT", 33, -115.02}, {"OK"}, - {"WITHIN", "mykey", "WHEREEVAL", "return FIELDS.a > tonumber(ARGV[1]) and FIELDS.a ~= tonumber(ARGV[2])", 2, 0.5, 3, "BOUNDS", 32.8, -115.2, 33.2, -114.8}, {`[0 [[myid_a1 {"type":"Point","coordinates":[-115,33]} [a 1]] [myid_a2 {"type":"Point","coordinates":[-115,32.99]} [a 2]]]]`}, + {"WITHIN", "mykey", "WHEREEVAL", "return FIELDS.a > tonumber(ARGV[1]) and FIELDS.a ~= tonumber(ARGV[2])", 2, 0.5, 3, "BOUNDS", 32.8, -115.2, 33.2, -114.8}, {`[0 [[myid_a2 {"type":"Point","coordinates":[-115,32.99]} [a 2]] [myid_a1 {"type":"Point","coordinates":[-115,33]} [a 1]]]]`}, }) } From cbfb271541bb7ef1d980bc41c4ebdec680187c05 Mon Sep 17 00:00:00 2001 From: tidwall Date: Mon, 12 Sep 2022 09:12:51 -0700 Subject: [PATCH 13/13] Updated data structures to use Go generics. Prior to this commit all objects in the Collection data structures were boxed in an Go interface{} which adds an extra 8 bytes per object and requires assertion to unbox. Go 1.18, released early 2022, introduced generics, which allows for storing the objects without boxing. This provides a extra boost in performance and lower in-memory footprint. --- go.mod | 6 +- go.sum | 8 +- internal/collection/collection.go | 138 +++++++++++++----------------- internal/collection/geodesic.go | 6 +- internal/collection/string.go | 16 ---- 5 files changed, 67 insertions(+), 107 deletions(-) diff --git a/go.mod b/go.mod index 8eefc51c..81c01ae4 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,6 @@ require ( github.com/streadway/amqp v1.0.0 github.com/tidwall/btree v1.4.2 github.com/tidwall/buntdb v1.2.9 - github.com/tidwall/geoindex v1.6.2 github.com/tidwall/geojson v1.3.4 github.com/tidwall/gjson v1.12.1 github.com/tidwall/match v1.1.1 @@ -26,7 +25,7 @@ require ( github.com/tidwall/redbench v0.1.0 github.com/tidwall/redcon v1.4.4 github.com/tidwall/resp v0.1.0 - github.com/tidwall/rtree v1.7.1 + github.com/tidwall/rtree v1.8.0 github.com/tidwall/sjson v1.2.4 github.com/xdg/scram v1.0.5 github.com/yuin/gopher-lua v0.0.0-20200816102855-ee81675732da @@ -88,9 +87,8 @@ require ( github.com/prometheus/common v0.32.1 // indirect github.com/prometheus/procfs v0.7.3 // indirect github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect - github.com/tidwall/cities v0.1.0 // indirect + github.com/tidwall/geoindex v1.7.0 // indirect github.com/tidwall/grect v0.1.4 // indirect - github.com/tidwall/lotsa v1.0.2 // indirect github.com/tidwall/rtred v0.1.2 // indirect github.com/tidwall/tinyqueue v0.1.1 // indirect github.com/xdg/stringprep v1.0.3 // indirect diff --git a/go.sum b/go.sum index b4416271..d7dff071 100644 --- a/go.sum +++ b/go.sum @@ -357,8 +357,8 @@ github.com/tidwall/buntdb v1.2.9/go.mod h1:IwyGSvvDg6hnKSIhtdZ0AqhCZGH8ukdtCAzaP github.com/tidwall/cities v0.1.0 h1:CVNkmMf7NEC9Bvokf5GoSsArHCKRMTgLuubRTHnH0mE= github.com/tidwall/cities v0.1.0/go.mod h1:lV/HDp2gCcRcHJWqgt6Di54GiDrTZwh1aG2ZUPNbqa4= github.com/tidwall/geoindex v1.4.4/go.mod h1:rvVVNEFfkJVWGUdEfU8QaoOg/9zFX0h9ofWzA60mz1I= -github.com/tidwall/geoindex v1.6.2 h1:cWbqC9HFXMxc2p6KMWbs9VG6/gnrfC53EIPQEMcXO1g= -github.com/tidwall/geoindex v1.6.2/go.mod h1:rvVVNEFfkJVWGUdEfU8QaoOg/9zFX0h9ofWzA60mz1I= +github.com/tidwall/geoindex v1.7.0 h1:jtk41sfgwIt8MEDyC3xyKSj75iXXf6rjReJGDNPtR5o= +github.com/tidwall/geoindex v1.7.0/go.mod h1:rvVVNEFfkJVWGUdEfU8QaoOg/9zFX0h9ofWzA60mz1I= github.com/tidwall/geojson v1.3.4 h1:mHB2yGK7HPgf4vFkLdPeIzguFpqkmCT2yTgGhXbrqBo= github.com/tidwall/geojson v1.3.4/go.mod h1:1cn3UWfSYCJOq53NZoQ9rirdw89+DM0vw+ZOAVvuReg= github.com/tidwall/gjson v1.12.1 h1:ikuZsLdhr8Ws0IdROXUS1Gi4v9Z4pGqpX/CvJkxvfpo= @@ -380,8 +380,8 @@ github.com/tidwall/resp v0.1.0/go.mod h1:18xEj855iMY2bK6tNF2A4x+nZy5gWO1iO7OOl3j github.com/tidwall/rtred v0.1.2 h1:exmoQtOLvDoO8ud++6LwVsAMTu0KPzLTUrMln8u1yu8= github.com/tidwall/rtred v0.1.2/go.mod h1:hd69WNXQ5RP9vHd7dqekAz+RIdtfBogmglkZSRxCHFQ= github.com/tidwall/rtree v1.3.1/go.mod h1:S+JSsqPTI8LfWA4xHBo5eXzie8WJLVFeppAutSegl6M= -github.com/tidwall/rtree v1.7.1 h1:rv3Q8RBKH2HbJ6DsqpfrXfV9l+dCT1jupTpDiceHN3I= -github.com/tidwall/rtree v1.7.1/go.mod h1:39+jGCj9hYqhflezmsTBOlysIk09ytm+8EQsC/E/2X0= +github.com/tidwall/rtree v1.8.0 h1:nYVLh9UKJrd4CZCNawD3WbHNxmI9LYR4j3E2hqO3tjQ= +github.com/tidwall/rtree v1.8.0/go.mod h1:iDJQ9NBRtbfKkzZu02za+mIlaP+bjYPnunbSNidpbCQ= github.com/tidwall/sjson v1.2.4 h1:cuiLzLnaMeBhRmEv00Lpk3tkYrcxpmbU81tAY4Dw0tc= github.com/tidwall/sjson v1.2.4/go.mod h1:098SZ494YoMWPmMO6ct4dcFnqxwj9r/gF0Etp19pSNM= github.com/tidwall/tinyqueue v0.1.1 h1:SpNEvEggbpyN5DIReaJ2/1ndroY8iyEGxPYxoSaymYE= diff --git a/internal/collection/collection.go b/internal/collection/collection.go index 63c54713..b28550be 100644 --- a/internal/collection/collection.go +++ b/internal/collection/collection.go @@ -4,7 +4,6 @@ import ( "runtime" "github.com/tidwall/btree" - "github.com/tidwall/geoindex" "github.com/tidwall/geojson" "github.com/tidwall/geojson/geo" "github.com/tidwall/geojson/geometry" @@ -28,13 +27,13 @@ type itemT struct { fieldValuesSlot fieldValuesSlot } -func byID(a, b interface{}) bool { - return a.(*itemT).id < b.(*itemT).id +func byID(a, b *itemT) bool { + return a.id < b.id } -func byValue(a, b interface{}) bool { - value1 := a.(*itemT).obj.String() - value2 := b.(*itemT).obj.String() +func byValue(a, b *itemT) bool { + value1 := a.obj.String() + value2 := b.obj.String() if value1 < value2 { return true } @@ -45,13 +44,11 @@ func byValue(a, b interface{}) bool { return byID(a, b) } -func byExpires(a, b interface{}) bool { - item1 := a.(*itemT) - item2 := b.(*itemT) - if item1.expires < item2.expires { +func byExpires(a, b *itemT) bool { + if a.expires < b.expires { return true } - if item1.expires > item2.expires { + if a.expires > b.expires { return false } // the values match so we'll compare IDs, which are always unique. @@ -60,10 +57,10 @@ func byExpires(a, b interface{}) bool { // Collection represents a collection of geojson objects. type Collection struct { - items *btree.BTree // items sorted by id - index *geoindex.Index // items geospatially indexed - values *btree.BTree // items sorted by value+id - expires *btree.BTree // items sorted by ex+id + items *btree.BTreeG[*itemT] // items sorted by id + spatial *rtree.RTreeG[*itemT] // items geospatially indexed + values *btree.BTreeG[*itemT] // items sorted by value+id + expires *btree.BTreeG[*itemT] // items sorted by ex+id fieldMap map[string]int fieldArr []string fieldValues *fieldValues @@ -73,13 +70,15 @@ type Collection struct { nobjects int // non-geometry count } +var optsNoLock = btree.Options{NoLocks: true} + // New creates an empty collection func New() *Collection { col := &Collection{ - items: btree.NewNonConcurrent(byID), - index: geoindex.Wrap(&rtree.RTree{}), - values: btree.NewNonConcurrent(byValue), - expires: btree.NewNonConcurrent(byExpires), + items: btree.NewBTreeGOptions(byID, optsNoLock), + values: btree.NewBTreeGOptions(byValue, optsNoLock), + expires: btree.NewBTreeGOptions(byExpires, optsNoLock), + spatial: &rtree.RTreeG[*itemT]{}, fieldMap: make(map[string]int), fieldArr: make([]string, 0), fieldValues: &fieldValues{}, @@ -109,7 +108,7 @@ func (c *Collection) TotalWeight() int { // Bounds returns the bounds of all the items in the collection. func (c *Collection) Bounds() (minX, minY, maxX, maxY float64) { - min, max := c.index.Bounds() + min, max := c.spatial.Bounds() if len(min) >= 2 && len(max) >= 2 { return min[0], min[1], max[0], max[1] } @@ -134,7 +133,7 @@ func (c *Collection) objWeight(item *itemT) int { func (c *Collection) indexDelete(item *itemT) { if !item.obj.Empty() { rect := item.obj.Rect() - c.index.Delete( + c.spatial.Delete( [2]float64{rect.Min.X, rect.Min.Y}, [2]float64{rect.Max.X, rect.Max.Y}, item) @@ -144,7 +143,7 @@ func (c *Collection) indexDelete(item *itemT) { func (c *Collection) indexInsert(item *itemT) { if !item.obj.Empty() { rect := item.obj.Rect() - c.index.Insert( + c.spatial.Insert( [2]float64{rect.Min.X, rect.Min.Y}, [2]float64{rect.Max.X, rect.Max.Y}, item) @@ -164,9 +163,8 @@ func (c *Collection) Set( newItem := &itemT{id: id, obj: obj, fieldValuesSlot: nilValuesSlot, expires: ex} // add the new item to main btree and remove the old one if needed - oldItem := c.items.Set(newItem) - if oldItem != nil { - oldItem := oldItem.(*itemT) + oldItem, ok := c.items.Set(newItem) + if ok { // the old item was removed, now let's remove it from the rtree/btree. if objIsSpatial(oldItem.obj) { c.indexDelete(oldItem) @@ -232,11 +230,10 @@ func (c *Collection) Set( func (c *Collection) Delete(id string) ( obj geojson.Object, fields []float64, ok bool, ) { - v := c.items.Delete(&itemT{id: id}) - if v == nil { + oldItem, ok := c.items.Delete(&itemT{id: id}) + if !ok { return nil, nil, false } - oldItem := v.(*itemT) if objIsSpatial(oldItem.obj) { if !oldItem.obj.Empty() { c.indexDelete(oldItem) @@ -263,20 +260,18 @@ func (c *Collection) Delete(id string) ( func (c *Collection) Get(id string) ( obj geojson.Object, fields []float64, ex int64, ok bool, ) { - itemV := c.items.Get(&itemT{id: id}) - if itemV == nil { + item, ok := c.items.Get(&itemT{id: id}) + if !ok { return nil, nil, 0, false } - item := itemV.(*itemT) return item.obj, c.fieldValues.get(item.fieldValuesSlot), item.expires, true } func (c *Collection) SetExpires(id string, ex int64) bool { - v := c.items.Get(&itemT{id: id}) - if v == nil { + item, ok := c.items.Get(&itemT{id: id}) + if !ok { return false } - item := v.(*itemT) if item.expires != 0 { c.expires.Delete(item) } @@ -292,11 +287,10 @@ func (c *Collection) SetExpires(id string, ex int64) bool { func (c *Collection) SetField(id, field string, value float64) ( obj geojson.Object, fields []float64, updated bool, ok bool, ) { - itemV := c.items.Get(&itemT{id: id}) - if itemV == nil { + item, ok := c.items.Get(&itemT{id: id}) + if !ok { return nil, nil, false, false } - item := itemV.(*itemT) _, updateCount, weightDelta := c.setFieldValues(item, []string{field}, []float64{value}) c.weight += weightDelta return item.obj, c.fieldValues.get(item.fieldValuesSlot), updateCount > 0, true @@ -306,11 +300,10 @@ func (c *Collection) SetField(id, field string, value float64) ( func (c *Collection) SetFields( id string, inFields []string, inValues []float64, ) (obj geojson.Object, fields []float64, updatedCount int, ok bool) { - itemV := c.items.Get(&itemT{id: id}) - if itemV == nil { + item, ok := c.items.Get(&itemT{id: id}) + if !ok { return nil, nil, 0, false } - item := itemV.(*itemT) newFieldValues, updateCount, weightDelta := c.setFieldValues(item, inFields, inValues) c.weight += weightDelta return item.obj, newFieldValues, updateCount, true @@ -394,20 +387,19 @@ func (c *Collection) Scan( offset = cursor.Offset() cursor.Step(offset) } - iter := func(item interface{}) bool { + iter := func(item *itemT) bool { count++ if count <= offset { return true } nextStep(count, cursor, deadline) - iitm := item.(*itemT) - keepon = iterator(iitm.id, iitm.obj, c.fieldValues.get(iitm.fieldValuesSlot)) + keepon = iterator(item.id, item.obj, c.fieldValues.get(item.fieldValuesSlot)) return keepon } if desc { - c.items.Descend(nil, iter) + c.items.Reverse(iter) } else { - c.items.Ascend(nil, iter) + c.items.Scan(iter) } return keepon } @@ -427,8 +419,7 @@ func (c *Collection) ScanRange( offset = cursor.Offset() cursor.Step(offset) } - iter := func(value interface{}) bool { - item := value.(*itemT) + iter := func(item *itemT) bool { count++ if count <= offset { return true @@ -443,8 +434,7 @@ func (c *Collection) ScanRange( return false } } - iitm := value.(*itemT) - keepon = iterator(iitm.id, iitm.obj, c.fieldValues.get(iitm.fieldValuesSlot)) + keepon = iterator(item.id, item.obj, c.fieldValues.get(item.fieldValuesSlot)) return keepon } @@ -470,20 +460,19 @@ func (c *Collection) SearchValues( offset = cursor.Offset() cursor.Step(offset) } - iter := func(item interface{}) bool { + iter := func(item *itemT) bool { count++ if count <= offset { return true } nextStep(count, cursor, deadline) - iitm := item.(*itemT) - keepon = iterator(iitm.id, iitm.obj, c.fieldValues.get(iitm.fieldValuesSlot)) + keepon = iterator(item.id, item.obj, c.fieldValues.get(item.fieldValuesSlot)) return keepon } if desc { - c.values.Descend(nil, iter) + c.values.Reverse(iter) } else { - c.values.Ascend(nil, iter) + c.values.Scan(iter) } return keepon } @@ -501,33 +490,32 @@ func (c *Collection) SearchValuesRange(start, end string, desc bool, offset = cursor.Offset() cursor.Step(offset) } - iter := func(item interface{}) bool { + iter := func(item *itemT) bool { count++ if count <= offset { return true } nextStep(count, cursor, deadline) - iitm := item.(*itemT) - keepon = iterator(iitm.id, iitm.obj, c.fieldValues.get(iitm.fieldValuesSlot)) + keepon = iterator(item.id, item.obj, c.fieldValues.get(item.fieldValuesSlot)) return keepon } pstart := &itemT{obj: String(start)} pend := &itemT{obj: String(end)} if desc { // descend range - c.values.Descend(pstart, func(item interface{}) bool { + c.values.Descend(pstart, func(item *itemT) bool { return bGT(c.values, item, pend) && iter(item) }) } else { - c.values.Ascend(pstart, func(item interface{}) bool { + c.values.Ascend(pstart, func(item *itemT) bool { return bLT(c.values, item, pend) && iter(item) }) } return keepon } -func bLT(tr *btree.BTree, a, b interface{}) bool { return tr.Less(a, b) } -func bGT(tr *btree.BTree, a, b interface{}) bool { return tr.Less(b, a) } +func bLT(tr *btree.BTreeG[*itemT], a, b *itemT) bool { return tr.Less(a, b) } +func bGT(tr *btree.BTreeG[*itemT], a, b *itemT) bool { return tr.Less(b, a) } // ScanGreaterOrEqual iterates though the collection starting with specified id. func (c *Collection) ScanGreaterOrEqual(id string, desc bool, @@ -542,13 +530,12 @@ func (c *Collection) ScanGreaterOrEqual(id string, desc bool, offset = cursor.Offset() cursor.Step(offset) } - iter := func(v interface{}) bool { + iter := func(item *itemT) bool { count++ if count <= offset { return true } nextStep(count, cursor, deadline) - item := v.(*itemT) keepon = iterator(item.id, item.obj, c.fieldValues.get(item.fieldValuesSlot), item.expires) return keepon } @@ -565,11 +552,10 @@ func (c *Collection) geoSearch( iter func(id string, obj geojson.Object, fields []float64) bool, ) bool { alive := true - c.index.Search( + c.spatial.Search( [2]float64{rect.Min.X, rect.Min.Y}, [2]float64{rect.Max.X, rect.Max.Y}, - func(_, _ [2]float64, itemv interface{}) bool { - item := itemv.(*itemT) + func(_, _ [2]float64, item *itemT) bool { alive = iter(item.id, item.obj, c.fieldValues.get(item.fieldValuesSlot)) return alive }, @@ -755,10 +741,10 @@ func (c *Collection) Nearby( minLat, minLon, maxLat, maxLon := geo.RectFromCenter(center.Y, center.X, meters) var exists bool - c.index.Search( + c.spatial.Search( [2]float64{minLon, minLat}, [2]float64{maxLon, maxLat}, - func(_, _ [2]float64, itemv interface{}) bool { + func(_, _ [2]float64, item *itemT) bool { exists = true return false }, @@ -778,15 +764,14 @@ func (c *Collection) Nearby( offset = cursor.Offset() cursor.Step(offset) } - c.index.Nearby( - geodeticDistAlgo([2]float64{center.X, center.Y}), - func(_, _ [2]float64, itemv interface{}, dist float64) bool { + c.spatial.Nearby( + geodeticDistAlgo[*itemT]([2]float64{center.X, center.Y}), + func(_, _ [2]float64, item *itemT, dist float64) bool { count++ if count <= offset { return true } nextStep(count, cursor, deadline) - item := itemv.(*itemT) alive = iter(item.id, item.obj, c.fieldValues.get(item.fieldValuesSlot), dist) return alive }, @@ -804,17 +789,10 @@ func nextStep(step uint64, cursor Cursor, deadline *deadline.Deadline) { } } -type Expired struct { - ID string - Obj geojson.Object - Fields []float64 -} - // Expired returns a list of all objects that have expired. func (c *Collection) Expired(now int64, buffer []string) (ids []string) { ids = buffer[:0] - c.expires.Ascend(nil, func(v interface{}) bool { - item := v.(*itemT) + c.expires.Scan(func(item *itemT) bool { if now < item.expires { return false } diff --git a/internal/collection/geodesic.go b/internal/collection/geodesic.go index d531b01c..b6330bcf 100644 --- a/internal/collection/geodesic.go +++ b/internal/collection/geodesic.go @@ -2,11 +2,11 @@ package collection import "math" -func geodeticDistAlgo(center [2]float64) ( - algo func(min, max [2]float64, data interface{}, item bool) (dist float64), +func geodeticDistAlgo[T any](center [2]float64) ( + algo func(min, max [2]float64, data T, item bool) (dist float64), ) { const earthRadius = 6371e3 - return func(min, max [2]float64, data interface{}, item bool) (dist float64) { + return func(min, max [2]float64, data T, item bool) (dist float64) { return earthRadius * pointRectDistGeodeticDeg( center[1], center[0], min[1], min[0], diff --git a/internal/collection/string.go b/internal/collection/string.go index ad7dc042..672ba079 100644 --- a/internal/collection/string.go +++ b/internal/collection/string.go @@ -7,83 +7,67 @@ import ( "github.com/tidwall/geojson/geometry" ) -// String ... type String string var _ geojson.Object = String("") -// Spatial ... func (s String) Spatial() geojson.Spatial { return geojson.EmptySpatial{} } -// ForEach ... func (s String) ForEach(iter func(geom geojson.Object) bool) bool { return iter(s) } -// Empty ... func (s String) Empty() bool { return true } -// Valid ... func (s String) Valid() bool { return false } -// Rect ... func (s String) Rect() geometry.Rect { return geometry.Rect{} } -// Center ... func (s String) Center() geometry.Point { return geometry.Point{} } -// AppendJSON ... func (s String) AppendJSON(dst []byte) []byte { data, _ := json.Marshal(string(s)) return append(dst, data...) } -// String ... func (s String) String() string { return string(s) } -// JSON ... func (s String) JSON() string { return string(s.AppendJSON(nil)) } -// MarshalJSON ... func (s String) MarshalJSON() ([]byte, error) { return s.AppendJSON(nil), nil } -// Within ... func (s String) Within(obj geojson.Object) bool { return false } -// Contains ... func (s String) Contains(obj geojson.Object) bool { return false } -// Intersects ... func (s String) Intersects(obj geojson.Object) bool { return false } -// NumPoints ... func (s String) NumPoints() int { return 0 } -// Distance ... func (s String) Distance(obj geojson.Object) float64 { return 0 }