mirror
/
tile38
mirror of https://github.com/tidwall/tile38.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'rshura-optimize-field-match'

This commit is contained in:
tidwall 2021-07-10 17:59:22 -07:00
parent 1467cba769 25579a052c
commit 0d83b1ca53
4 changed files with 101 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

57
internal/server/scanner.go
View File

@ -98,8 +98,6 @@ func (s *Server) newScanWriter(
msg: msg,
cursor: cursor,
limit: limit,
wheres: wheres,
whereins: whereins,
whereevals: whereevals,
output: output,
nofields: nofields,
@ -118,6 +116,27 @@ func (s *Server) newScanWriter(
if sw.col != nil {
sw.fmap = sw.col.FieldMap()
sw.farr = sw.col.FieldArr()
// This fills index value in wheres/whereins
// so we don't have to map string field names for each tested object
var ok bool
if len(wheres) > 0 {
sw.wheres = make([]whereT, len(wheres))
for i, where := range wheres {
if where.index, ok = sw.fmap[where.field]; !ok {
where.index = math.MaxInt32
}
sw.wheres[i] = where
}
}
if len(whereins) > 0 {
sw.whereins = make([]whereinT, len(whereins))
for i, wherein := range whereins {
if wherein.index, ok = sw.fmap[wherein.field]; !ok {
wherein.index = math.MaxInt32
}
sw.whereins[i] = wherein
}
}
}
sw.fvals = make([]float64, len(sw.farr))
return sw, nil
@ -213,11 +232,8 @@ func (sw *scanWriter) fieldMatch(fields []float64, o geojson.Object) (fvals []fl
continue
}
var value float64
idx, ok := sw.fmap[where.field]
if ok {
if len(fields) > idx {
value = fields[idx]
}
if where.index < len(fields) {
value = fields[where.index]
}
if !where.match(value) {
return
@ -225,11 +241,8 @@ func (sw *scanWriter) fieldMatch(fields []float64, o geojson.Object) (fvals []fl
}
for _, wherein := range sw.whereins {
var value float64
idx, ok := sw.fmap[wherein.field]
if ok {
if len(fields) > idx {
value = fields[idx]
}
if wherein.index < len(fields) {
value = fields[wherein.index]
}
if !wherein.match(value) {
return
@ -249,12 +262,10 @@ func (sw *scanWriter) fieldMatch(fields []float64, o geojson.Object) (fvals []fl
}
}
} else {
for idx := range sw.farr {
var value float64
if len(fields) > idx {
value = fields[idx]
}
sw.fvals[idx] = value
copy(sw.fvals, fields)
// fields might be shorter for this item, need to pad sw.fvals with zeros
for i := len(fields); i < len(sw.fvals); i++ {
sw.fvals[i] = 0
}
for _, where := range sw.wheres {
if where.field == "z" {
@ -269,9 +280,8 @@ func (sw *scanWriter) fieldMatch(fields []float64, o geojson.Object) (fvals []fl
continue
}
var value float64
idx, ok := sw.fmap[where.field]
if ok {
value = sw.fvals[idx]
if where.index < len(sw.fvals) {
value = sw.fvals[where.index]
}
if !where.match(value) {
return
@ -279,9 +289,8 @@ func (sw *scanWriter) fieldMatch(fields []float64, o geojson.Object) (fvals []fl
}
for _, wherein := range sw.whereins {
var value float64
idx, ok := sw.fmap[wherein.field]
if ok {
value = sw.fvals[idx]
if wherein.index < len(sw.fvals) {
value = sw.fvals[wherein.index]
}
if !wherein.match(value) {
return

52
internal/server/scanner_test.go Normal file
View File

@ -0,0 +1,52 @@
package server
import (
"math"
"math/rand"
"testing"
"time"
"github.com/tidwall/geojson"
"github.com/tidwall/geojson/geometry"
)
type testPointItem struct {
object geojson.Object
fields []float64
}
func PO(x, y float64) *geojson.Point {
return geojson.NewPoint(geometry.Point{X: x, Y: y})
}
func BenchmarkFieldMatch(t *testing.B) {
rand.Seed(time.Now().UnixNano())
items := make([]testPointItem, t.N)
for i := 0; i < t.N; i++ {
items[i] = testPointItem{
PO(rand.Float64()*360-180, rand.Float64()*180-90),
[]float64{rand.Float64()*9+1, math.Round(rand.Float64()*30) + 1},
}
}
sw := &scanWriter{
wheres: []whereT{
{"foo", 0, false, 1, false, 3},
{"bar", 1, false, 10, false, 30},
},
whereins: []whereinT{
{"foo", 0, []float64{1, 2}},
{"bar", 1, []float64{11, 25}},
},
fmap: map[string]int{"foo": 0, "bar": 1},
farr: []string{"bar", "foo"},
}
sw.fvals = make([]float64, len(sw.farr))
t.ResetTimer()
for i := 0; i < t.N; i++ {
// one call is super fast, measurements are not reliable, let's do 100
for ix := 0; ix < 100; ix++ {
sw.fieldMatch(items[i].fields, items[i].object)
}
}
}

21
internal/server/token.go
View File

@ -88,6 +88,7 @@ func lc(s1, s2 string) bool {
type whereT struct {
field string
index int
minx bool
min float64
maxx bool
@ -118,12 +119,17 @@ func (where whereT) match(value float64) bool {
type whereinT struct {
field string
valMap map[float64]struct{}
index int
valArr []float64
}
func (wherein whereinT) match(value float64) bool {
_, ok := wherein.valMap[value]
return ok
for _, val := range wherein.valArr {
if val == value {
return true
}
}
return false
}
type whereevalT struct {
@ -282,7 +288,7 @@ func (s *Server) parseSearchScanBaseTokens(
return
}
}
t.wheres = append(t.wheres, whereT{field, minx, min, maxx, max})
t.wheres = append(t.wheres, whereT{field, -1, minx, min, maxx, max})
continue
case "wherein":
vs = nvs
@ -300,9 +306,8 @@ func (s *Server) parseSearchScanBaseTokens(
err = errInvalidArgument(nvalsStr)
return
}
valMap := make(map[float64]struct{})
valArr := make([]float64, nvals)
var val float64
var empty struct{}
for i = 0; i < nvals; i++ {
if vs, valStr, ok = tokenval(vs); !ok || valStr == "" {
err = errInvalidNumberOfArguments
@ -312,9 +317,9 @@ func (s *Server) parseSearchScanBaseTokens(
err = errInvalidArgument(valStr)
return
}
valMap[val] = empty
valArr[i] = val
}
t.whereins = append(t.whereins, whereinT{field, valMap})
t.whereins = append(t.whereins, whereinT{field, -1, valArr})
continue
case "whereevalsha":
fallthrough

3
tests/keys_test.go
View File

@ -394,6 +394,9 @@ func keys_WHEREIN_test(mc *mockServer) error {
{"SET", "mykey", "myid_a2", "FIELD", "a", 2, "POINT", 32.99, -115}, {"OK"},
{"SET", "mykey", "myid_a3", "FIELD", "a", 3, "POINT", 33, -115.02}, {"OK"},
{"WITHIN", "mykey", "WHEREIN", "a", 3, 0, 1, 2, "BOUNDS", 32.8, -115.2, 33.2, -114.8}, {`[0 [[myid_a1 {"type":"Point","coordinates":[-115,33]} [a 1]] [myid_a2 {"type":"Point","coordinates":[-115,32.99]} [a 2]]]]`},
// zero value should not match 1 and 2
{"SET", "mykey", "myid_a0", "FIELD", "a", 0, "POINT", 33, -115.02}, {"OK"},
{"WITHIN", "mykey", "WHEREIN", "a", 2, 1, 2, "BOUNDS", 32.8, -115.2, 33.2, -114.8}, {`[0 [[myid_a1 {"type":"Point","coordinates":[-115,33]} [a 1]] [myid_a2 {"type":"Point","coordinates":[-115,32.99]} [a 2]]]]`},
})
}