From f24c251ee61e9c7a3cea50df82f1ebdc7be2bb64 Mon Sep 17 00:00:00 2001 From: tidwall Date: Thu, 1 Sep 2022 19:43:30 -0700 Subject: [PATCH] Allow for multiple MATCH patterns Each MATCH is inclusive OR, thus WITHIN fleet MATCH train* truck* BOUNDS 33 -112 34 -113 will find all trains and trucks that within the provides bounds. --- internal/server/fence.go | 21 ++++++++++++----- internal/server/hooks.go | 2 +- internal/server/live.go | 6 ++--- internal/server/scan.go | 9 ++++---- internal/server/scanner.go | 46 +++++++++++++++++--------------------- internal/server/search.go | 42 ++++++++++++++++++++++++++++------ internal/server/token.go | 10 ++++----- tests/keys_search_test.go | 21 ++++++++++------- 8 files changed, 96 insertions(+), 61 deletions(-) diff --git a/internal/server/fence.go b/internal/server/fence.go index 94bc6de5..2829fef0 100644 --- a/internal/server/fence.go +++ b/internal/server/fence.go @@ -55,6 +55,20 @@ func objIsSpatial(obj geojson.Object) bool { func hookJSONString(hookName string, metas []FenceMeta) string { return string(appendHookDetails(nil, hookName, metas)) } + +func multiGlobMatch(globs []string, s string) bool { + if len(globs) == 0 || (len(globs) == 1 && globs[0] == "*") { + return true + } + for _, pattern := range globs { + match, _ := glob.Match(pattern, s) + if match { + return true + } + } + return false +} + func fenceMatch( hookName string, sw *scanWriter, fence *liveFenceSwitches, metas []FenceMeta, details *commandDetails, @@ -66,11 +80,8 @@ func fenceMatch( `,"time":` + jsonTimeFormat(details.timestamp) + `}`, } } - if len(fence.glob) > 0 && !(len(fence.glob) == 1 && fence.glob[0] == '*') { - match, _ := glob.Match(fence.glob, details.id) - if !match { - return nil - } + if !multiGlobMatch(fence.globs, details.id) { + return nil } if details.obj == nil || !objIsSpatial(details.obj) { return nil diff --git a/internal/server/hooks.go b/internal/server/hooks.go index e2c22505..4e490d77 100644 --- a/internal/server/hooks.go +++ b/internal/server/hooks.go @@ -143,7 +143,7 @@ func (s *Server) cmdSetHook(msg *Message) ( } var wr bytes.Buffer hook.ScanWriter, err = s.newScanWriter( - &wr, cmsg, args.key, args.output, args.precision, args.glob, false, + &wr, cmsg, args.key, args.output, args.precision, args.globs, false, args.cursor, args.limit, args.wheres, args.whereins, args.whereevals, args.nofields) if err != nil { diff --git a/internal/server/live.go b/internal/server/live.go index 90d0a116..1f595327 100644 --- a/internal/server/live.go +++ b/internal/server/live.go @@ -14,7 +14,7 @@ import ( type liveBuffer struct { key string - glob string + globs []string fence *liveFenceSwitches details []*commandDetails cond *sync.Cond @@ -101,12 +101,12 @@ func (s *Server) goLive( var sw *scanWriter var wr bytes.Buffer lfs := inerr.(liveFenceSwitches) - lb.glob = lfs.glob + lb.globs = lfs.globs lb.key = lfs.key lb.fence = &lfs s.mu.RLock() sw, err = s.newScanWriter( - &wr, msg, lfs.key, lfs.output, lfs.precision, lfs.glob, false, + &wr, msg, lfs.key, lfs.output, lfs.precision, lfs.globs, false, lfs.cursor, lfs.limit, lfs.wheres, lfs.whereins, lfs.whereevals, lfs.nofields) s.mu.RUnlock() diff --git a/internal/server/scan.go b/internal/server/scan.go index ad595a12..0151ae90 100644 --- a/internal/server/scan.go +++ b/internal/server/scan.go @@ -7,7 +7,6 @@ import ( "github.com/tidwall/geojson" "github.com/tidwall/resp" - "github.com/tidwall/tile38/internal/glob" ) func (s *Server) cmdScanArgs(vs []string) ( @@ -46,7 +45,7 @@ func (s *Server) cmdScan(msg *Message) (res resp.Value, err error) { } wr := &bytes.Buffer{} sw, err := s.newScanWriter( - wr, msg, args.key, args.output, args.precision, args.glob, false, + wr, msg, args.key, args.output, args.precision, args.globs, false, args.cursor, args.limit, args.wheres, args.whereins, args.whereevals, args.nofields) if err != nil { @@ -65,8 +64,8 @@ func (s *Server) cmdScan(msg *Message) (res resp.Value, err error) { } sw.count = uint64(count) } else { - g := glob.Parse(sw.globPattern, args.desc) - if g.Limits[0] == "" && g.Limits[1] == "" { + limits := multiGlobParse(sw.globs, args.desc) + if limits[0] == "" && limits[1] == "" { sw.col.Scan(args.desc, sw, msg.Deadline, func(id string, o geojson.Object, fields []float64) bool { @@ -78,7 +77,7 @@ func (s *Server) cmdScan(msg *Message) (res resp.Value, err error) { }, ) } else { - sw.col.ScanRange(g.Limits[0], g.Limits[1], args.desc, sw, + sw.col.ScanRange(limits[0], limits[1], args.desc, sw, msg.Deadline, func(id string, o geojson.Object, fields []float64) bool { return sw.writeObject(ScanWriterParams{ diff --git a/internal/server/scanner.go b/internal/server/scanner.go index b189cd56..3053ca93 100644 --- a/internal/server/scanner.go +++ b/internal/server/scanner.go @@ -52,9 +52,8 @@ type scanWriter struct { once bool count uint64 precision uint64 - globPattern string + globs []string globEverything bool - globSingle bool fullFields bool values []resp.Value matchValues bool @@ -79,7 +78,7 @@ type ScanWriterParams struct { func (s *Server) newScanWriter( wr *bytes.Buffer, msg *Message, key string, output outputT, - precision uint64, globPattern string, matchValues bool, + precision uint64, globs []string, matchValues bool, cursor, limit uint64, wheres []whereT, whereins []whereinT, whereevals []whereevalT, nofields bool, ) ( @@ -102,21 +101,18 @@ func (s *Server) newScanWriter( wr: wr, key: key, msg: msg, + globs: globs, limit: limit, cursor: cursor, output: output, nofields: nofields, precision: precision, whereevals: whereevals, - globPattern: globPattern, matchValues: matchValues, } - if globPattern == "*" || globPattern == "" { + + if len(globs) == 0 || (len(globs) == 1 && globs[0] == "*") { sw.globEverything = true - } else { - if !glob.IsGlob(globPattern) { - sw.globSingle = true - } } sw.orgWheres = wheres sw.orgWhereins = whereins @@ -344,25 +340,23 @@ func (sw *scanWriter) fieldMatch(fields []float64, o geojson.Object) (fvals []fl } func (sw *scanWriter) globMatch(id string, o geojson.Object) (ok, keepGoing bool) { - if !sw.globEverything { - if sw.globSingle { - if sw.globPattern != id { - return false, true - } - return true, false - } - var val string - if sw.matchValues { - val = o.String() - } else { - val = id - } - ok, _ := glob.Match(sw.globPattern, val) - if !ok { - return false, true + if sw.globEverything { + return true, true + } + var val string + if sw.matchValues { + val = o.String() + } else { + val = id + } + for _, pattern := range sw.globs { + ok, _ := glob.Match(pattern, val) + if ok { + return true, true } } - return true, true + return false, true + } // Increment cursor diff --git a/internal/server/search.go b/internal/server/search.go index 925683a5..57c744f5 100644 --- a/internal/server/search.go +++ b/internal/server/search.go @@ -488,7 +488,7 @@ func (s *Server) cmdNearby(msg *Message) (res resp.Value, err error) { return NOMessage, sargs } sw, err := s.newScanWriter( - wr, msg, sargs.key, sargs.output, sargs.precision, sargs.glob, false, + wr, msg, sargs.key, sargs.output, sargs.precision, sargs.globs, false, sargs.cursor, sargs.limit, sargs.wheres, sargs.whereins, sargs.whereevals, sargs.nofields) if err != nil { return NOMessage, err @@ -580,7 +580,7 @@ func (s *Server) cmdWithinOrIntersects(cmd string, msg *Message) (res resp.Value return NOMessage, sargs } sw, err := s.newScanWriter( - wr, msg, sargs.key, sargs.output, sargs.precision, sargs.glob, false, + wr, msg, sargs.key, sargs.output, sargs.precision, sargs.globs, false, sargs.cursor, sargs.limit, sargs.wheres, sargs.whereins, sargs.whereevals, sargs.nofields) if err != nil { return NOMessage, err @@ -644,6 +644,35 @@ func (s *Server) cmdSeachValuesArgs(vs []string) ( return } +func multiGlobParse(globs []string, desc bool) [2]string { + var limits [2]string + for i, pattern := range globs { + g := glob.Parse(pattern, desc) + if g.Limits[0] == "" && g.Limits[1] == "" { + limits[0], limits[1] = "", "" + break + } + if i == 0 { + limits[0], limits[1] = g.Limits[0], g.Limits[1] + } else if desc { + if g.Limits[0] > limits[0] { + limits[0] = g.Limits[0] + } + if g.Limits[1] < limits[1] { + limits[1] = g.Limits[1] + } + } else { + if g.Limits[0] < limits[0] { + limits[0] = g.Limits[0] + } + if g.Limits[1] > limits[1] { + limits[1] = g.Limits[1] + } + } + } + return limits +} + func (s *Server) cmdSearch(msg *Message) (res resp.Value, err error) { start := time.Now() vs := msg.Args[1:] @@ -664,7 +693,7 @@ func (s *Server) cmdSearch(msg *Message) (res resp.Value, err error) { return NOMessage, err } sw, err := s.newScanWriter( - wr, msg, sargs.key, sargs.output, sargs.precision, sargs.glob, true, + wr, msg, sargs.key, sargs.output, sargs.precision, sargs.globs, true, sargs.cursor, sargs.limit, sargs.wheres, sargs.whereins, sargs.whereevals, sargs.nofields) if err != nil { return NOMessage, err @@ -681,8 +710,8 @@ func (s *Server) cmdSearch(msg *Message) (res resp.Value, err error) { } sw.count = uint64(count) } else { - g := glob.Parse(sw.globPattern, sargs.desc) - if g.Limits[0] == "" && g.Limits[1] == "" { + limits := multiGlobParse(sw.globs, sargs.desc) + if limits[0] == "" && limits[1] == "" { sw.col.SearchValues(sargs.desc, sw, msg.Deadline, func(id string, o geojson.Object, fields []float64) bool { return sw.writeObject(ScanWriterParams{ @@ -696,8 +725,7 @@ func (s *Server) cmdSearch(msg *Message) (res resp.Value, err error) { } else { // must disable globSingle for string value type matching because // globSingle is only for ID matches, not values. - sw.globSingle = false - sw.col.SearchValuesRange(g.Limits[0], g.Limits[1], sargs.desc, sw, + sw.col.SearchValuesRange(limits[0], limits[1], sargs.desc, sw, msg.Deadline, func(id string, o geojson.Object, fields []float64) bool { return sw.writeObject(ScanWriterParams{ diff --git a/internal/server/token.go b/internal/server/token.go index 50925626..178d9a1f 100644 --- a/internal/server/token.go +++ b/internal/server/token.go @@ -200,7 +200,7 @@ type searchScanBaseTokens struct { nodwell bool detect map[string]bool accept map[string]bool - glob string + globs []string wheres []whereT whereins []whereinT whereevals []whereevalT @@ -543,14 +543,12 @@ func (s *Server) parseSearchScanBaseTokens( continue case "match": vs = nvs - if t.glob != "" { - err = errDuplicateArgument(strings.ToUpper(wtok)) - return - } - if vs, t.glob, ok = tokenval(vs); !ok || t.glob == "" { + var glob string + if vs, glob, ok = tokenval(vs); !ok || glob == "" { err = errInvalidNumberOfArguments return } + t.globs = append(t.globs, glob) continue case "clip": vs = nvs diff --git a/tests/keys_search_test.go b/tests/keys_search_test.go index 7f3ee971..1f8ebb12 100644 --- a/tests/keys_search_test.go +++ b/tests/keys_search_test.go @@ -591,23 +591,28 @@ func keys_MATCH_test(mc *mockServer) error { {"SET", "fleet", "truck2", "POINT", "33.0002", "-112.0002"}, {"OK"}, {"SET", "fleet", "luck1", "POINT", "33.0003", "-112.0003"}, {"OK"}, {"SET", "fleet", "luck2", "POINT", "33.0004", "-112.0004"}, {"OK"}, + {"SET", "fleet", "train1", "POINT", "33.0005", "-112.0005"}, {"OK"}, - {"SCAN", "fleet", "IDS"}, {"[0 [luck1 luck2 truck1 truck2]]"}, - {"SCAN", "fleet", "MATCH", "*", "IDS"}, {"[0 [luck1 luck2 truck1 truck2]]"}, + {"SCAN", "fleet", "IDS"}, {"[0 [luck1 luck2 train1 truck1 truck2]]"}, + {"SCAN", "fleet", "MATCH", "*", "IDS"}, {"[0 [luck1 luck2 train1 truck1 truck2]]"}, {"SCAN", "fleet", "MATCH", "truck*", "IDS"}, {"[0 [truck1 truck2]]"}, {"SCAN", "fleet", "MATCH", "luck*", "IDS"}, {"[0 [luck1 luck2]]"}, {"SCAN", "fleet", "MATCH", "*2", "IDS"}, {"[0 [luck2 truck2]]"}, {"SCAN", "fleet", "MATCH", "*2*", "IDS"}, {"[0 [luck2 truck2]]"}, {"SCAN", "fleet", "MATCH", "*u*", "IDS"}, {"[0 [luck1 luck2 truck1 truck2]]"}, + {"SCAN", "fleet", "MATCH", "*u*", "MATCH", "*u*", "IDS"}, {"[0 [luck1 luck2 truck1 truck2]]"}, + {"SCAN", "fleet", "MATCH", "*u*", "MATCH", "*a*", "IDS"}, {"[0 [luck1 luck2 train1 truck1 truck2]]"}, + {"SCAN", "fleet", "MATCH", "train*", "MATCH", "truck*", "IDS"}, {"[0 [train1 truck1 truck2]]"}, + {"SCAN", "fleet", "MATCH", "train*", "MATCH", "truck*", "MATCH", "luck1", "IDS"}, {"[0 [luck1 train1 truck1 truck2]]"}, {"NEARBY", "fleet", "IDS", "POINT", 33.00005, -112.00005, 100000}, { - match("[0 [luck1 luck2 truck1 truck2]]"), + match("[0 [luck1 luck2 train1 truck1 truck2]]"), }, {"NEARBY", "fleet", "MATCH", "*", "IDS", "POINT", 33.00005, -112.00005, 100000}, { - match("[0 [luck1 luck2 truck1 truck2]]"), + match("[0 [luck1 luck2 train1 truck1 truck2]]"), }, {"NEARBY", "fleet", "MATCH", "t*", "IDS", "POINT", 33.00005, -112.00005, 100000}, { - match("[0 [truck1 truck2]]"), + match("[0 [train1 truck1 truck2]]"), }, {"NEARBY", "fleet", "MATCH", "t*2", "IDS", "POINT", 33.00005, -112.00005, 100000}, { match("[0 [truck2]]"), @@ -617,13 +622,13 @@ func keys_MATCH_test(mc *mockServer) error { }, {"INTERSECTS", "fleet", "IDS", "BOUNDS", 33, -113, 34, -112}, { - match("[0 [luck1 luck2 truck1 truck2]]"), + match("[0 [luck1 luck2 train1 truck1 truck2]]"), }, {"INTERSECTS", "fleet", "MATCH", "*", "IDS", "BOUNDS", 33, -113, 34, -112}, { - match("[0 [luck1 luck2 truck1 truck2]]"), + match("[0 [luck1 luck2 train1 truck1 truck2]]"), }, {"INTERSECTS", "fleet", "MATCH", "t*", "IDS", "BOUNDS", 33, -113, 34, -112}, { - match("[0 [truck1 truck2]]"), + match("[0 [train1 truck1 truck2]]"), }, {"INTERSECTS", "fleet", "MATCH", "t*2", "IDS", "BOUNDS", 33, -113, 34, -112}, { match("[0 [truck2]]"),