Field overhaul

go.mod

@@ -16,15 +16,17 @@ require (
 	github.com/peterh/liner v1.2.1
 	github.com/prometheus/client_golang v1.12.1
 	github.com/streadway/amqp v1.0.0
+	github.com/tidwall/assert v0.1.0
 	github.com/tidwall/btree v1.4.3
 	github.com/tidwall/buntdb v1.2.9
 	github.com/tidwall/geojson v1.3.6
-	github.com/tidwall/gjson v1.12.1
+	github.com/tidwall/gjson v1.14.3
+	github.com/tidwall/hashmap v1.6.1
 	github.com/tidwall/match v1.1.1
 	github.com/tidwall/pretty v1.2.0
 	github.com/tidwall/redbench v0.1.0
 	github.com/tidwall/redcon v1.4.4
-	github.com/tidwall/resp v0.1.0
+	github.com/tidwall/resp v0.1.1
 	github.com/tidwall/rtree v1.8.1
 	github.com/tidwall/sjson v1.2.4
 	github.com/xdg/scram v1.0.5
@@ -101,7 +103,7 @@ require (
 	golang.org/x/mod v0.3.0 // indirect
 	golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c // indirect
 	golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect
-	golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 // indirect
+	golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64 // indirect
 	golang.org/x/text v0.3.7 // indirect
 	golang.org/x/tools v0.0.0-20200825202427-b303f430e36d // indirect
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect

go.sum

@@ -361,10 +361,13 @@ github.com/tidwall/geoindex v1.7.0 h1:jtk41sfgwIt8MEDyC3xyKSj75iXXf6rjReJGDNPtR5
 github.com/tidwall/geoindex v1.7.0/go.mod h1:rvVVNEFfkJVWGUdEfU8QaoOg/9zFX0h9ofWzA60mz1I=
 github.com/tidwall/geojson v1.3.6 h1:ZbpDNwdhXyDe8XGTplGVaGrcS2ViFaSoo3QBNXe1uhM=
 github.com/tidwall/geojson v1.3.6/go.mod h1:1cn3UWfSYCJOq53NZoQ9rirdw89+DM0vw+ZOAVvuReg=
-github.com/tidwall/gjson v1.12.1 h1:ikuZsLdhr8Ws0IdROXUS1Gi4v9Z4pGqpX/CvJkxvfpo=
 github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/gjson v1.14.3 h1:9jvXn7olKEHU1S9vwoMGliaT8jq1vJ7IH/n9zD9Dnlw=
+github.com/tidwall/gjson v1.14.3/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
 github.com/tidwall/grect v0.1.4 h1:dA3oIgNgWdSspFzn1kS4S/RDpZFLrIxAZOdJKjYapOg=
 github.com/tidwall/grect v0.1.4/go.mod h1:9FBsaYRaR0Tcy4UwefBX/UDcDcDy9V5jUcxHzv2jd5Q=
+github.com/tidwall/hashmap v1.6.1 h1:FIAHjKwcyOo1Y3/orsQO08floKhInbEX2VQv7CQRNuw=
+github.com/tidwall/hashmap v1.6.1/go.mod h1:hX452N3VtFD8okD3/6q/yOquJvJmYxmZ1H0nLtwkaxM=
 github.com/tidwall/lotsa v1.0.2 h1:dNVBH5MErdaQ/xd9s769R31/n2dXavsQ0Yf4TMEHHw8=
 github.com/tidwall/lotsa v1.0.2/go.mod h1:X6NiU+4yHA3fE3Puvpnn1XMDrFZrE9JO2/w+UMuqgR8=
 github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
@@ -375,8 +378,8 @@ github.com/tidwall/redbench v0.1.0 h1:UZYUMhwMMObQRq5xU4SA3lmlJRztXzqtushDii+AmP
 github.com/tidwall/redbench v0.1.0/go.mod h1:zxcRGCq/JcqV48YjK9WxBNJL7JSpMzbLXaHvMcnanKQ=
 github.com/tidwall/redcon v1.4.4 h1:N3ZwZx6n5dqNxB3cfmj9D/8zNboFia5FAv1wt+azwyU=
 github.com/tidwall/redcon v1.4.4/go.mod h1:p5Wbsgeyi2VSTBWOcA5vRXrOb9arFTcU2+ZzFjqV75Y=
-github.com/tidwall/resp v0.1.0 h1:zZ6Hq+2cY4QqhZ4LqrV05T5yLOSPspj+l+DgAoJ25Ak=
+github.com/tidwall/resp v0.1.1 h1:Ly20wkhqKTmDUPlyM1S7pWo5kk0tDu8OoC/vFArXmwE=
-github.com/tidwall/resp v0.1.0/go.mod h1:18xEj855iMY2bK6tNF2A4x+nZy5gWO1iO7OOl3jETKw=
+github.com/tidwall/resp v0.1.1/go.mod h1:3/FrruOBAxPTPtundW0VXgmsQ4ZBA0Aw714lVYgwFa0=
 github.com/tidwall/rtred v0.1.2 h1:exmoQtOLvDoO8ud++6LwVsAMTu0KPzLTUrMln8u1yu8=
 github.com/tidwall/rtred v0.1.2/go.mod h1:hd69WNXQ5RP9vHd7dqekAz+RIdtfBogmglkZSRxCHFQ=
 github.com/tidwall/rtree v1.3.1/go.mod h1:S+JSsqPTI8LfWA4xHBo5eXzie8WJLVFeppAutSegl6M=
@@ -551,8 +554,9 @@ golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 h1:WIoqL4EROvwiPdUtaip4VcDdpZ4kha7wBWZrbVKCIZg=
 golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64 h1:UiNENfZ8gDvpiWw7IpOMQ27spWmThO1RwwdQVbJahJM=
+golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=

internal/collection/collection.go

@@ -9,6 +9,7 @@ import (
 	"github.com/tidwall/geojson/geometry"
 	"github.com/tidwall/rtree"
 	"github.com/tidwall/tile38/internal/deadline"
+	"github.com/tidwall/tile38/internal/field"
 )
 
 // yieldStep forces the iterator to yield goroutine every 256 steps.
@@ -24,7 +25,7 @@ type itemT struct {
 	id      string
 	obj     geojson.Object
 	expires int64 // unix nano expiration
-	fieldValuesSlot fieldValuesSlot
+	fields  field.List
 }
 
 func byID(a, b *itemT) bool {
@@ -61,9 +62,6 @@ type Collection struct {
 	spatial  *rtree.RTreeG[*itemT] // items geospatially indexed
 	values   *btree.BTreeG[*itemT] // items sorted by value+id
 	expires  *btree.BTreeG[*itemT] // items sorted by ex+id
-	fieldMap    map[string]int
-	fieldArr    []string
-	fieldValues *fieldValues
 	weight   int
 	points   int
 	objects  int // geometry count
@@ -79,9 +77,6 @@ func New() *Collection {
 		values:  btree.NewBTreeGOptions(byValue, optsNoLock),
 		expires: btree.NewBTreeGOptions(byExpires, optsNoLock),
 		spatial: &rtree.RTreeG[*itemT]{},
-		fieldMap:    make(map[string]int),
-		fieldArr:    make([]string, 0),
-		fieldValues: &fieldValues{},
 	}
 	return col
 }
@@ -122,12 +117,14 @@ func objIsSpatial(obj geojson.Object) bool {
 
 func (c *Collection) objWeight(item *itemT) int {
 	var weight int
+	weight += len(item.id)
 	if objIsSpatial(item.obj) {
-		weight = item.obj.NumPoints() * 16
+		weight += item.obj.NumPoints() * 16
 	} else {
-		weight = len(item.obj.String())
+		weight += len(item.obj.String())
 	}
-	return weight + len(c.fieldValues.get(item.fieldValuesSlot))*8 + len(item.id)
+	weight += item.fields.Weight()
+	return weight
 }
 
 func (c *Collection) indexDelete(item *itemT) {
@@ -151,16 +148,16 @@ func (c *Collection) indexInsert(item *itemT) {
 }
 
 // Set adds or replaces an object in the collection and returns the fields
-// array. If an item with the same id is already in the collection then the
+// array.
-// new item will adopt the old item's fields.
+func (c *Collection) Set(id string, obj geojson.Object, fields field.List, ex int64) (
-// The fields argument is optional.
+	oldObject geojson.Object, oldFields, newFields field.List,
-// The return values are the old object, the old fields, and the new fields
-func (c *Collection) Set(
-	id string, obj geojson.Object, fields []string, values []float64, ex int64,
-) (
-	oldObject geojson.Object, oldFieldValues []float64, newFieldValues []float64,
 ) {
-	newItem := &itemT{id: id, obj: obj, fieldValuesSlot: nilValuesSlot, expires: ex}
+	newItem := &itemT{
+		id:      id,
+		obj:     obj,
+		expires: ex,
+		fields:  fields,
+	}
 
 	// add the new item to main btree and remove the old one if needed
 	oldItem, ok := c.items.Set(newItem)
@@ -183,24 +180,6 @@ func (c *Collection) Set(
 
 		// decrement the weights
 		c.weight -= c.objWeight(oldItem)
-
-		// references
-		oldObject = oldItem.obj
-		oldFieldValues = c.fieldValues.get(oldItem.fieldValuesSlot)
-		newFieldValues = oldFieldValues
-		newItem.fieldValuesSlot = oldItem.fieldValuesSlot
-		if len(oldFieldValues) > 0 {
-			oldFieldValues = append([]float64{}, oldFieldValues...)
-		}
-	}
-	if fields == nil {
-		if len(values) > 0 {
-			newFieldValues = values
-			newFieldValuesSlot := c.fieldValues.set(newItem.fieldValuesSlot, newFieldValues)
-			newItem.fieldValuesSlot = newFieldValuesSlot
-		}
-	} else {
-		newFieldValues, _, _ = c.setFieldValues(newItem, fields, values)
 	}
 
 	// insert the new item into the rtree or strings tree.
@@ -222,17 +201,20 @@ func (c *Collection) Set(
 	// add the new weights
 	c.weight += c.objWeight(newItem)
 
-	return oldObject, oldFieldValues, newFieldValues
+	if oldItem != nil {
+		return oldItem.obj, oldItem.fields, newItem.fields
+	}
+	return nil, field.List{}, newItem.fields
 }
 
 // Delete removes an object and returns it.
 // If the object does not exist then the 'ok' return value will be false.
 func (c *Collection) Delete(id string) (
-	obj geojson.Object, fields []float64, ok bool,
+	obj geojson.Object, fields field.List, ok bool,
 ) {
 	oldItem, ok := c.items.Delete(&itemT{id: id})
 	if !ok {
-		return nil, nil, false
+		return nil, field.List{}, false
 	}
 	if objIsSpatial(oldItem.obj) {
 		if !oldItem.obj.Empty() {
@@ -250,127 +232,22 @@ func (c *Collection) Delete(id string) (
 	c.weight -= c.objWeight(oldItem)
 	c.points -= oldItem.obj.NumPoints()
 
-	fields = c.fieldValues.get(oldItem.fieldValuesSlot)
+	return oldItem.obj, oldItem.fields, true
-	c.fieldValues.remove(oldItem.fieldValuesSlot)
-	return oldItem.obj, fields, true
 }
 
 // Get returns an object.
 // If the object does not exist then the 'ok' return value will be false.
 func (c *Collection) Get(id string) (
-	obj geojson.Object, fields []float64, ex int64, ok bool,
+	obj geojson.Object,
+	fields field.List,
+	ex int64,
+	ok bool,
 ) {
 	item, ok := c.items.Get(&itemT{id: id})
 	if !ok {
-		return nil, nil, 0, false
+		return nil, field.List{}, 0, false
-	}
-	return item.obj, c.fieldValues.get(item.fieldValuesSlot), item.expires, true
-}
-
-func (c *Collection) SetExpires(id string, ex int64) bool {
-	item, ok := c.items.Get(&itemT{id: id})
-	if !ok {
-		return false
-	}
-	if item.expires != 0 {
-		c.expires.Delete(item)
-	}
-	item.expires = ex
-	if item.expires != 0 {
-		c.expires.Set(item)
-	}
-	return true
-}
-
-// SetField set a field value for an object and returns that object.
-// If the object does not exist then the 'ok' return value will be false.
-func (c *Collection) SetField(id, field string, value float64) (
-	obj geojson.Object, fields []float64, updated bool, ok bool,
-) {
-	item, ok := c.items.Get(&itemT{id: id})
-	if !ok {
-		return nil, nil, false, false
-	}
-	_, updateCount, weightDelta := c.setFieldValues(item, []string{field}, []float64{value})
-	c.weight += weightDelta
-	return item.obj, c.fieldValues.get(item.fieldValuesSlot), updateCount > 0, true
-}
-
-// SetFields is similar to SetField, just setting multiple fields at once
-func (c *Collection) SetFields(
-	id string, inFields []string, inValues []float64,
-) (obj geojson.Object, fields []float64, updatedCount int, ok bool) {
-	item, ok := c.items.Get(&itemT{id: id})
-	if !ok {
-		return nil, nil, 0, false
-	}
-	newFieldValues, updateCount, weightDelta := c.setFieldValues(item, inFields, inValues)
-	c.weight += weightDelta
-	return item.obj, newFieldValues, updateCount, true
-}
-
-func (c *Collection) setFieldValues(item *itemT, fields []string, updateValues []float64) (
-	newValues []float64,
-	updated int,
-	weightDelta int,
-) {
-	newValues = c.fieldValues.get(item.fieldValuesSlot)
-	for i, field := range fields {
-		fieldIdx, ok := c.fieldMap[field]
-		if !ok {
-			fieldIdx = len(c.fieldMap)
-			c.fieldMap[field] = fieldIdx
-			c.addToFieldArr(field)
-		}
-		for fieldIdx >= len(newValues) {
-			newValues = append(newValues, 0)
-			weightDelta += 8
-		}
-		ovalue := newValues[fieldIdx]
-		nvalue := updateValues[i]
-		newValues[fieldIdx] = nvalue
-		if ovalue != nvalue {
-			updated++
-		}
-	}
-	newSlot := c.fieldValues.set(item.fieldValuesSlot, newValues)
-	item.fieldValuesSlot = newSlot
-	return newValues, updated, weightDelta
-}
-
-// FieldMap return a maps of the field names.
-func (c *Collection) FieldMap() map[string]int {
-	return c.fieldMap
-}
-
-// FieldArr return an array representation of the field names.
-func (c *Collection) FieldArr() []string {
-	return c.fieldArr
-}
-
-// bsearch searches array for value.
-func bsearch(arr []string, val string) (index int, found bool) {
-	i, j := 0, len(arr)
-	for i < j {
-		h := i + (j-i)/2
-		if val >= arr[h] {
-			i = h + 1
-		} else {
-			j = h
-		}
-	}
-	if i > 0 && arr[i-1] >= val {
-		return i - 1, true
-	}
-	return i, false
-}
-
-func (c *Collection) addToFieldArr(field string) {
-	if index, found := bsearch(c.fieldArr, field); !found {
-		c.fieldArr = append(c.fieldArr, "")
-		copy(c.fieldArr[index+1:], c.fieldArr[index:len(c.fieldArr)-1])
-		c.fieldArr[index] = field
 	}
+	return item.obj, item.fields, item.expires, true
 }
 
 // Scan iterates though the collection ids.
@@ -378,7 +255,7 @@ func (c *Collection) Scan(
 	desc bool,
 	cursor Cursor,
 	deadline *deadline.Deadline,
-	iterator func(id string, obj geojson.Object, fields []float64) bool,
+	iterator func(id string, obj geojson.Object, fields field.List) bool,
 ) bool {
 	var keepon = true
 	var count uint64
@@ -393,7 +270,7 @@ func (c *Collection) Scan(
 			return true
 		}
 		nextStep(count, cursor, deadline)
-		keepon = iterator(item.id, item.obj, c.fieldValues.get(item.fieldValuesSlot))
+		keepon = iterator(item.id, item.obj, item.fields)
 		return keepon
 	}
 	if desc {
@@ -410,7 +287,7 @@ func (c *Collection) ScanRange(
 	desc bool,
 	cursor Cursor,
 	deadline *deadline.Deadline,
-	iterator func(id string, obj geojson.Object, fields []float64) bool,
+	iterator func(id string, obj geojson.Object, fields field.List) bool,
 ) bool {
 	var keepon = true
 	var count uint64
@@ -434,7 +311,7 @@ func (c *Collection) ScanRange(
 				return false
 			}
 		}
-		keepon = iterator(item.id, item.obj, c.fieldValues.get(item.fieldValuesSlot))
+		keepon = iterator(item.id, item.obj, item.fields)
 		return keepon
 	}
 
@@ -451,7 +328,7 @@ func (c *Collection) SearchValues(
 	desc bool,
 	cursor Cursor,
 	deadline *deadline.Deadline,
-	iterator func(id string, obj geojson.Object, fields []float64) bool,
+	iterator func(id string, obj geojson.Object, fields field.List) bool,
 ) bool {
 	var keepon = true
 	var count uint64
@@ -466,7 +343,7 @@ func (c *Collection) SearchValues(
 			return true
 		}
 		nextStep(count, cursor, deadline)
-		keepon = iterator(item.id, item.obj, c.fieldValues.get(item.fieldValuesSlot))
+		keepon = iterator(item.id, item.obj, item.fields)
 		return keepon
 	}
 	if desc {
@@ -481,7 +358,7 @@ func (c *Collection) SearchValues(
 func (c *Collection) SearchValuesRange(start, end string, desc bool,
 	cursor Cursor,
 	deadline *deadline.Deadline,
-	iterator func(id string, obj geojson.Object, fields []float64) bool,
+	iterator func(id string, obj geojson.Object, fields field.List) bool,
 ) bool {
 	var keepon = true
 	var count uint64
@@ -496,7 +373,7 @@ func (c *Collection) SearchValuesRange(start, end string, desc bool,
 			return true
 		}
 		nextStep(count, cursor, deadline)
-		keepon = iterator(item.id, item.obj, c.fieldValues.get(item.fieldValuesSlot))
+		keepon = iterator(item.id, item.obj, item.fields)
 		return keepon
 	}
 	pstart := &itemT{obj: String(start)}
@@ -521,7 +398,7 @@ func bGT(tr *btree.BTreeG[*itemT], a, b *itemT) bool { return tr.Less(b, a) }
 func (c *Collection) ScanGreaterOrEqual(id string, desc bool,
 	cursor Cursor,
 	deadline *deadline.Deadline,
-	iterator func(id string, obj geojson.Object, fields []float64, ex int64) bool,
+	iterator func(id string, obj geojson.Object, fields field.List, ex int64) bool,
 ) bool {
 	var keepon = true
 	var count uint64
@@ -536,7 +413,7 @@ func (c *Collection) ScanGreaterOrEqual(id string, desc bool,
 			return true
 		}
 		nextStep(count, cursor, deadline)
-		keepon = iterator(item.id, item.obj, c.fieldValues.get(item.fieldValuesSlot), item.expires)
+		keepon = iterator(item.id, item.obj, item.fields, item.expires)
 		return keepon
 	}
 	if desc {
@@ -549,14 +426,14 @@ func (c *Collection) ScanGreaterOrEqual(id string, desc bool,
 
 func (c *Collection) geoSearch(
 	rect geometry.Rect,
-	iter func(id string, obj geojson.Object, fields []float64) bool,
+	iter func(id string, obj geojson.Object, fields field.List) bool,
 ) bool {
 	alive := true
 	c.spatial.Search(
 		[2]float64{rect.Min.X, rect.Min.Y},
 		[2]float64{rect.Max.X, rect.Max.Y},
 		func(_, _ [2]float64, item *itemT) bool {
-			alive = iter(item.id, item.obj, c.fieldValues.get(item.fieldValuesSlot))
+			alive = iter(item.id, item.obj, item.fields)
 			return alive
 		},
 	)
@@ -565,12 +442,12 @@ func (c *Collection) geoSearch(
 
 func (c *Collection) geoSparse(
 	obj geojson.Object, sparse uint8,
-	iter func(id string, obj geojson.Object, fields []float64) (match, ok bool),
+	iter func(id string, obj geojson.Object, fields field.List) (match, ok bool),
 ) bool {
 	matches := make(map[string]bool)
 	alive := true
 	c.geoSparseInner(obj.Rect(), sparse,
-		func(id string, o geojson.Object, fields []float64) (
+		func(id string, o geojson.Object, fields field.List) (
 			match, ok bool,
 		) {
 			ok = true
@@ -587,7 +464,7 @@ func (c *Collection) geoSparse(
 }
 func (c *Collection) geoSparseInner(
 	rect geometry.Rect, sparse uint8,
-	iter func(id string, obj geojson.Object, fields []float64) (match, ok bool),
+	iter func(id string, obj geojson.Object, fields field.List) (match, ok bool),
 ) bool {
 	if sparse > 0 {
 		w := rect.Max.X - rect.Min.X
@@ -619,7 +496,7 @@ func (c *Collection) geoSparseInner(
 	}
 	alive := true
 	c.geoSearch(rect,
-		func(id string, obj geojson.Object, fields []float64) bool {
+		func(id string, obj geojson.Object, fields field.List) bool {
 			match, ok := iter(id, obj, fields)
 			if !ok {
 				alive = false
@@ -638,7 +515,7 @@ func (c *Collection) Within(
 	sparse uint8,
 	cursor Cursor,
 	deadline *deadline.Deadline,
-	iter func(id string, obj geojson.Object, fields []float64) bool,
+	iter func(id string, obj geojson.Object, fields field.List) bool,
 ) bool {
 	var count uint64
 	var offset uint64
@@ -648,7 +525,7 @@ func (c *Collection) Within(
 	}
 	if sparse > 0 {
 		return c.geoSparse(obj, sparse,
-			func(id string, o geojson.Object, fields []float64) (
+			func(id string, o geojson.Object, fields field.List) (
 				match, ok bool,
 			) {
 				count++
@@ -664,7 +541,7 @@ func (c *Collection) Within(
 		)
 	}
 	return c.geoSearch(obj.Rect(),
-		func(id string, o geojson.Object, fields []float64) bool {
+		func(id string, o geojson.Object, fields field.List) bool {
 			count++
 			if count <= offset {
 				return true
@@ -685,7 +562,7 @@ func (c *Collection) Intersects(
 	sparse uint8,
 	cursor Cursor,
 	deadline *deadline.Deadline,
-	iter func(id string, obj geojson.Object, fields []float64) bool,
+	iter func(id string, obj geojson.Object, fields field.List) bool,
 ) bool {
 	var count uint64
 	var offset uint64
@@ -695,7 +572,7 @@ func (c *Collection) Intersects(
 	}
 	if sparse > 0 {
 		return c.geoSparse(obj, sparse,
-			func(id string, o geojson.Object, fields []float64) (
+			func(id string, o geojson.Object, fields field.List) (
 				match, ok bool,
 			) {
 				count++
@@ -711,7 +588,7 @@ func (c *Collection) Intersects(
 		)
 	}
 	return c.geoSearch(obj.Rect(),
-		func(id string, o geojson.Object, fields []float64) bool {
+		func(id string, o geojson.Object, fields field.List) bool {
 			count++
 			if count <= offset {
 				return true
@@ -730,7 +607,7 @@ func (c *Collection) Nearby(
 	target geojson.Object,
 	cursor Cursor,
 	deadline *deadline.Deadline,
-	iter func(id string, obj geojson.Object, fields []float64, dist float64) bool,
+	iter func(id string, obj geojson.Object, fields field.List, dist float64) bool,
 ) bool {
 	// First look to see if there's at least one candidate in the circle's
 	// outer rectangle. This is a fast-fail operation.
@@ -772,7 +649,7 @@ func (c *Collection) Nearby(
 				return true
 			}
 			nextStep(count, cursor, deadline)
-			alive = iter(item.id, item.obj, c.fieldValues.get(item.fieldValuesSlot), dist)
+			alive = iter(item.id, item.obj, item.fields, dist)
 			return alive
 		},
 	)

internal/collection/collection_test.go

@@ -11,6 +11,7 @@ import (
 	"github.com/tidwall/geojson"
 	"github.com/tidwall/geojson/geometry"
 	"github.com/tidwall/gjson"
+	"github.com/tidwall/tile38/internal/field"
 )
 
 func PO(x, y float64) *geojson.Point {
@@ -46,14 +47,14 @@ func TestCollectionNewCollection(t *testing.T) {
 		id := strconv.FormatInt(int64(i), 10)
 		obj := PO(rand.Float64()*360-180, rand.Float64()*180-90)
 		objs[id] = obj
-		c.Set(id, obj, nil, nil, 0)
+		c.Set(id, obj, field.List{}, 0)
 	}
 	count := 0
 	bbox := geometry.Rect{
 		Min: geometry.Point{X: -180, Y: -90},
 		Max: geometry.Point{X: 180, Y: 90},
 	}
-	c.geoSearch(bbox, func(id string, obj geojson.Object, field []float64) bool {
+	c.geoSearch(bbox, func(id string, obj geojson.Object, _ field.List) bool {
 		count++
 		return true
 	})
@@ -67,77 +68,95 @@ func TestCollectionNewCollection(t *testing.T) {
 	testCollectionVerifyContents(t, c, objs)
 }
 
+func toFields(fNames, fValues []string) field.List {
+	var fields field.List
+	for i := 0; i < len(fNames); i++ {
+		fields = fields.Set(field.Make(fNames[i], fValues[i]))
+	}
+	return fields
+}
+
 func TestCollectionSet(t *testing.T) {
 	t.Run("AddString", func(t *testing.T) {
 		c := New()
 		str1 := String("hello")
-		oldObject, oldFields, newFields := c.Set("str", str1, nil, nil, 0)
+		oldObject, oldFields, newFields := c.Set("str", str1, field.List{}, 0)
 		expect(t, oldObject == nil)
-		expect(t, len(oldFields) == 0)
+		expect(t, oldFields.Len() == 0)
-		expect(t, len(newFields) == 0)
+		expect(t, newFields.Len() == 0)
 	})
 	t.Run("UpdateString", func(t *testing.T) {
 		c := New()
 		str1 := String("hello")
 		str2 := String("world")
-		oldObject, oldFields, newFields := c.Set("str", str1, nil, nil, 0)
+		oldObject, oldFields, newFields := c.Set("str", str1, field.List{}, 0)
 		expect(t, oldObject == nil)
-		expect(t, len(oldFields) == 0)
+		expect(t, oldFields.Len() == 0)
-		expect(t, len(newFields) == 0)
+		expect(t, newFields.Len() == 0)
-		oldObject, oldFields, newFields = c.Set("str", str2, nil, nil, 0)
+		oldObject, oldFields, newFields = c.Set("str", str2, field.List{}, 0)
 		expect(t, oldObject == str1)
-		expect(t, len(oldFields) == 0)
+		expect(t, oldFields.Len() == 0)
-		expect(t, len(newFields) == 0)
+		expect(t, newFields.Len() == 0)
 	})
 	t.Run("AddPoint", func(t *testing.T) {
 		c := New()
 		point1 := PO(-112.1, 33.1)
-		oldObject, oldFields, newFields := c.Set("point", point1, nil, nil, 0)
+		oldObject, oldFields, newFields := c.Set("point", point1, field.List{}, 0)
 		expect(t, oldObject == nil)
-		expect(t, len(oldFields) == 0)
+		expect(t, oldFields.Len() == 0)
-		expect(t, len(newFields) == 0)
+		expect(t, newFields.Len() == 0)
 	})
 	t.Run("UpdatePoint", func(t *testing.T) {
 		c := New()
 		point1 := PO(-112.1, 33.1)
 		point2 := PO(-112.2, 33.2)
-		oldObject, oldFields, newFields := c.Set("point", point1, nil, nil, 0)
+		oldObject, oldFields, newFields := c.Set("point", point1, field.List{}, 0)
 		expect(t, oldObject == nil)
-		expect(t, len(oldFields) == 0)
+		expect(t, oldFields.Len() == 0)
-		expect(t, len(newFields) == 0)
+		expect(t, newFields.Len() == 0)
-		oldObject, oldFields, newFields = c.Set("point", point2, nil, nil, 0)
+		oldObject, oldFields, newFields = c.Set("point", point2, field.List{}, 0)
 		expect(t, oldObject == point1)
-		expect(t, len(oldFields) == 0)
+		expect(t, oldFields.Len() == 0)
-		expect(t, len(newFields) == 0)
+		expect(t, newFields.Len() == 0)
 	})
 	t.Run("Fields", func(t *testing.T) {
 		c := New()
 		str1 := String("hello")
+
 		fNames := []string{"a", "b", "c"}
-		fValues := []float64{1, 2, 3}
+		fValues := []string{"1", "2", "3"}
-		oldObj, oldFlds, newFlds := c.Set("str", str1, fNames, fValues, 0)
+		fields1 := toFields(fNames, fValues)
+		oldObj, oldFlds, newFlds := c.Set("str", str1, fields1, 0)
+
 		expect(t, oldObj == nil)
-		expect(t, len(oldFlds) == 0)
+		expect(t, oldFlds.Len() == 0)
-		expect(t, reflect.DeepEqual(newFlds, fValues))
+		expect(t, reflect.DeepEqual(newFlds, fields1))
+
 		str2 := String("hello")
+
 		fNames = []string{"d", "e", "f"}
-		fValues = []float64{4, 5, 6}
+		fValues = []string{"4", "5", "6"}
-		oldObj, oldFlds, newFlds = c.Set("str", str2, fNames, fValues, 0)
+		fields2 := toFields(fNames, fValues)
+
+		oldObj, oldFlds, newFlds = c.Set("str", str2, fields2, 0)
 		expect(t, oldObj == str1)
-		expect(t, reflect.DeepEqual(oldFlds, []float64{1, 2, 3}))
+		expect(t, reflect.DeepEqual(oldFlds, fields1))
-		expect(t, reflect.DeepEqual(newFlds, []float64{1, 2, 3, 4, 5, 6}))
+		expect(t, reflect.DeepEqual(newFlds, fields2))
-		fValues = []float64{7, 8, 9, 10, 11, 12}
+
-		oldObj, oldFlds, newFlds = c.Set("str", str1, nil, fValues, 0)
+		fNames = []string{"a", "b", "c", "d", "e", "f"}
+		fValues = []string{"7", "8", "9", "10", "11", "12"}
+		fields3 := toFields(fNames, fValues)
+		oldObj, oldFlds, newFlds = c.Set("str", str1, fields3, 0)
 		expect(t, oldObj == str2)
-		expect(t, reflect.DeepEqual(oldFlds, []float64{1, 2, 3, 4, 5, 6}))
+		expect(t, reflect.DeepEqual(oldFlds, fields2))
-		expect(t, reflect.DeepEqual(newFlds, []float64{7, 8, 9, 10, 11, 12}))
+		expect(t, reflect.DeepEqual(newFlds, fields3))
 	})
 	t.Run("Delete", func(t *testing.T) {
 		c := New()
 
-		c.Set("1", String("1"), nil, nil, 0)
+		c.Set("1", String("1"), field.List{}, 0)
-		c.Set("2", String("2"), nil, nil, 0)
+		c.Set("2", String("2"), field.List{}, 0)
-		c.Set("3", PO(1, 2), nil, nil, 0)
+		c.Set("3", PO(1, 2), field.List{}, 0)
 
 		expect(t, c.Count() == 3)
 		expect(t, c.StringCount() == 2)
@@ -147,9 +166,9 @@ func TestCollectionSet(t *testing.T) {
 			Max: geometry.Point{X: 1, Y: 2}})
 		var v geojson.Object
 		var ok bool
-		var flds []float64
+		// var flds []float64
-		var updated bool
+		// var updated bool
-		var updateCount int
+		// var updateCount int
 
 		v, _, ok = c.Delete("2")
 		expect(t, v.String() == "2")
@@ -165,32 +184,32 @@ func TestCollectionSet(t *testing.T) {
 		expect(t, c.StringCount() == 0)
 		expect(t, c.PointCount() == 1)
 
-		expect(t, len(c.FieldMap()) == 0)
+		// expect(t, len(c.FieldMap()) == 0)
 
-		_, flds, updated, ok = c.SetField("3", "hello", 123)
+		// _, flds, updated, ok = c.SetField("3", "hello", 123)
-		expect(t, ok)
+		// expect(t, ok)
-		expect(t, reflect.DeepEqual(flds, []float64{123}))
+		// expect(t, reflect.DeepEqual(flds, []float64{123}))
-		expect(t, updated)
+		// expect(t, updated)
-		expect(t, c.FieldMap()["hello"] == 0)
+		// expect(t, c.FieldMap()["hello"] == 0)
 
-		_, flds, updated, ok = c.SetField("3", "hello", 1234)
+		// _, flds, updated, ok = c.SetField("3", "hello", 1234)
-		expect(t, ok)
+		// expect(t, ok)
-		expect(t, reflect.DeepEqual(flds, []float64{1234}))
+		// expect(t, reflect.DeepEqual(flds, []float64{1234}))
-		expect(t, updated)
+		// expect(t, updated)
 
-		_, flds, updated, ok = c.SetField("3", "hello", 1234)
+		// _, flds, updated, ok = c.SetField("3", "hello", 1234)
-		expect(t, ok)
+		// expect(t, ok)
-		expect(t, reflect.DeepEqual(flds, []float64{1234}))
+		// expect(t, reflect.DeepEqual(flds, []float64{1234}))
-		expect(t, !updated)
+		// expect(t, !updated)
 
-		_, flds, updateCount, ok = c.SetFields("3",
+		// _, flds, updateCount, ok = c.SetFields("3",
-			[]string{"planet", "world"}, []float64{55, 66})
+		// 	[]string{"planet", "world"}, []float64{55, 66})
-		expect(t, ok)
+		// expect(t, ok)
-		expect(t, reflect.DeepEqual(flds, []float64{1234, 55, 66}))
+		// expect(t, reflect.DeepEqual(flds, []float64{1234, 55, 66}))
-		expect(t, updateCount == 2)
+		// expect(t, updateCount == 2)
-		expect(t, c.FieldMap()["hello"] == 0)
+		// expect(t, c.FieldMap()["hello"] == 0)
-		expect(t, c.FieldMap()["planet"] == 1)
+		// expect(t, c.FieldMap()["planet"] == 1)
-		expect(t, c.FieldMap()["world"] == 2)
+		// expect(t, c.FieldMap()["world"] == 2)
 
 		v, _, ok = c.Delete("3")
 		expect(t, v.String() == `{"type":"Point","coordinates":[1,2]}`)
@@ -206,45 +225,63 @@ func TestCollectionSet(t *testing.T) {
 		v, _, _, ok = c.Get("3")
 		expect(t, v == nil)
 		expect(t, !ok)
-		_, _, _, ok = c.SetField("3", "hello", 123)
+		// _, _, _, ok = c.SetField("3", "hello", 123)
-		expect(t, !ok)
+		// expect(t, !ok)
-		_, _, _, ok = c.SetFields("3", []string{"hello"}, []float64{123})
+		// _, _, _, ok = c.SetFields("3", []string{"hello"}, []float64{123})
-		expect(t, !ok)
+		// expect(t, !ok)
-		expect(t, c.TotalWeight() == 0)
+		// expect(t, c.TotalWeight() == 0)
-		expect(t, c.FieldMap()["hello"] == 0)
+		// expect(t, c.FieldMap()["hello"] == 0)
-		expect(t, c.FieldMap()["planet"] == 1)
+		// expect(t, c.FieldMap()["planet"] == 1)
-		expect(t, c.FieldMap()["world"] == 2)
+		// expect(t, c.FieldMap()["world"] == 2)
-		expect(t, reflect.DeepEqual(
+		// expect(t, reflect.DeepEqual(
-			c.FieldArr(), []string{"hello", "planet", "world"}),
+		// 	c.FieldArr(), []string{"hello", "planet", "world"}),
-		)
+		// )
 	})
 }
 
+func fieldValueAt(fields field.List, index int) string {
+	if index < 0 || index >= fields.Len() {
+		panic("out of bounds")
+	}
+	var retval string
+	var i int
+	fields.Scan(func(f field.Field) bool {
+		if i == index {
+			retval = f.Value().Data()
+		}
+		i++
+		return true
+	})
+	return retval
+}
+
 func TestCollectionScan(t *testing.T) {
 	N := 256
 	c := New()
 	for _, i := range rand.Perm(N) {
 		id := fmt.Sprintf("%04d", i)
-		c.Set(id, String(id), []string{"ex"}, []float64{float64(i)}, 0)
+		c.Set(id, String(id), makeFields(
+			field.Make("ex", id),
+		), 0)
 	}
 	var n int
 	var prevID string
-	c.Scan(false, nil, nil, func(id string, obj geojson.Object, fields []float64) bool {
+	c.Scan(false, nil, nil, func(id string, obj geojson.Object, fields field.List) bool {
 		if n > 0 {
 			expect(t, id > prevID)
 		}
-		expect(t, id == fmt.Sprintf("%04d", int(fields[0])))
+		expect(t, id == fieldValueAt(fields, 0))
 		n++
 		prevID = id
 		return true
 	})
 	expect(t, n == c.Count())
 	n = 0
-	c.Scan(true, nil, nil, func(id string, obj geojson.Object, fields []float64) bool {
+	c.Scan(true, nil, nil, func(id string, obj geojson.Object, fields field.List) bool {
 		if n > 0 {
 			expect(t, id < prevID)
 		}
-		expect(t, id == fmt.Sprintf("%04d", int(fields[0])))
+		expect(t, id == fieldValueAt(fields, 0))
 		n++
 		prevID = id
 		return true
@@ -253,11 +290,11 @@ func TestCollectionScan(t *testing.T) {
 
 	n = 0
 	c.ScanRange("0060", "0070", false, nil, nil,
-		func(id string, obj geojson.Object, fields []float64) bool {
+		func(id string, obj geojson.Object, fields field.List) bool {
 			if n > 0 {
 				expect(t, id > prevID)
 			}
-			expect(t, id == fmt.Sprintf("%04d", int(fields[0])))
+			expect(t, id == fieldValueAt(fields, 0))
 			n++
 			prevID = id
 			return true
@@ -266,11 +303,11 @@ func TestCollectionScan(t *testing.T) {
 
 	n = 0
 	c.ScanRange("0070", "0060", true, nil, nil,
-		func(id string, obj geojson.Object, fields []float64) bool {
+		func(id string, obj geojson.Object, fields field.List) bool {
 			if n > 0 {
 				expect(t, id < prevID)
 			}
-			expect(t, id == fmt.Sprintf("%04d", int(fields[0])))
+			expect(t, id == fieldValueAt(fields, 0))
 			n++
 			prevID = id
 			return true
@@ -279,11 +316,11 @@ func TestCollectionScan(t *testing.T) {
 
 	n = 0
 	c.ScanGreaterOrEqual("0070", true, nil, nil,
-		func(id string, obj geojson.Object, fields []float64, ex int64) bool {
+		func(id string, obj geojson.Object, fields field.List, ex int64) bool {
 			if n > 0 {
 				expect(t, id < prevID)
 			}
-			expect(t, id == fmt.Sprintf("%04d", int(fields[0])))
+			expect(t, id == fieldValueAt(fields, 0))
 			n++
 			prevID = id
 			return true
@@ -292,11 +329,11 @@ func TestCollectionScan(t *testing.T) {
 
 	n = 0
 	c.ScanGreaterOrEqual("0070", false, nil, nil,
-		func(id string, obj geojson.Object, fields []float64, ex int64) bool {
+		func(id string, obj geojson.Object, fields field.List, ex int64) bool {
 			if n > 0 {
 				expect(t, id > prevID)
 			}
-			expect(t, id == fmt.Sprintf("%04d", int(fields[0])))
+			expect(t, id == fieldValueAt(fields, 0))
 			n++
 			prevID = id
 			return true
@@ -305,33 +342,44 @@ func TestCollectionScan(t *testing.T) {
 
 }
 
+func makeFields(entries ...field.Field) field.List {
+	var fields field.List
+	for _, f := range entries {
+		fields = fields.Set(f)
+	}
+	return fields
+}
+
 func TestCollectionSearch(t *testing.T) {
 	N := 256
 	c := New()
 	for i, j := range rand.Perm(N) {
 		id := fmt.Sprintf("%04d", j)
 		ex := fmt.Sprintf("%04d", i)
-		c.Set(id, String(ex), []string{"i", "j"},
+		c.Set(id, String(ex),
-			[]float64{float64(i), float64(j)}, 0)
+			makeFields(
+				field.Make("i", ex),
+				field.Make("j", id),
+			), 0)
 	}
 	var n int
 	var prevValue string
-	c.SearchValues(false, nil, nil, func(id string, obj geojson.Object, fields []float64) bool {
+	c.SearchValues(false, nil, nil, func(id string, obj geojson.Object, fields field.List) bool {
 		if n > 0 {
 			expect(t, obj.String() > prevValue)
 		}
-		expect(t, id == fmt.Sprintf("%04d", int(fields[1])))
+		expect(t, id == fieldValueAt(fields, 1))
 		n++
 		prevValue = obj.String()
 		return true
 	})
 	expect(t, n == c.Count())
 	n = 0
-	c.SearchValues(true, nil, nil, func(id string, obj geojson.Object, fields []float64) bool {
+	c.SearchValues(true, nil, nil, func(id string, obj geojson.Object, fields field.List) bool {
 		if n > 0 {
 			expect(t, obj.String() < prevValue)
 		}
-		expect(t, id == fmt.Sprintf("%04d", int(fields[1])))
+		expect(t, id == fieldValueAt(fields, 1))
 		n++
 		prevValue = obj.String()
 		return true
@@ -340,11 +388,11 @@ func TestCollectionSearch(t *testing.T) {
 
 	n = 0
 	c.SearchValuesRange("0060", "0070", false, nil, nil,
-		func(id string, obj geojson.Object, fields []float64) bool {
+		func(id string, obj geojson.Object, fields field.List) bool {
 			if n > 0 {
 				expect(t, obj.String() > prevValue)
 			}
-			expect(t, id == fmt.Sprintf("%04d", int(fields[1])))
+			expect(t, id == fieldValueAt(fields, 1))
 			n++
 			prevValue = obj.String()
 			return true
@@ -353,11 +401,11 @@ func TestCollectionSearch(t *testing.T) {
 
 	n = 0
 	c.SearchValuesRange("0070", "0060", true, nil, nil,
-		func(id string, obj geojson.Object, fields []float64) bool {
+		func(id string, obj geojson.Object, fields field.List) bool {
 			if n > 0 {
 				expect(t, obj.String() < prevValue)
 			}
-			expect(t, id == fmt.Sprintf("%04d", int(fields[1])))
+			expect(t, id == fieldValueAt(fields, 1))
 			n++
 			prevValue = obj.String()
 			return true
@@ -367,31 +415,39 @@ func TestCollectionSearch(t *testing.T) {
 
 func TestCollectionWeight(t *testing.T) {
 	c := New()
-	c.Set("1", String("1"), nil, nil, 0)
+	c.Set("1", String("1"), field.List{}, 0)
 	expect(t, c.TotalWeight() > 0)
 	c.Delete("1")
 	expect(t, c.TotalWeight() == 0)
 	c.Set("1", String("1"),
+		toFields(
 			[]string{"a", "b", "c"},
-		[]float64{1, 2, 3},
+			[]string{"1", "2", "3"},
+		),
 		0,
 	)
 	expect(t, c.TotalWeight() > 0)
 	c.Delete("1")
 	expect(t, c.TotalWeight() == 0)
 	c.Set("1", String("1"),
+		toFields(
 			[]string{"a", "b", "c"},
-		[]float64{1, 2, 3},
+			[]string{"1", "2", "3"},
+		),
 		0,
 	)
 	c.Set("2", String("2"),
+		toFields(
 			[]string{"d", "e", "f"},
-		[]float64{4, 5, 6},
+			[]string{"4", "5", "6"},
+		),
 		0,
 	)
 	c.Set("1", String("1"),
+		toFields(
 			[]string{"d", "e", "f"},
-		[]float64{4, 5, 6},
+			[]string{"4", "5", "6"},
+		),
 		0,
 	)
 	c.Delete("1")
@@ -428,19 +484,19 @@ func TestSpatialSearch(t *testing.T) {
 	q4, _ := geojson.Parse(gjson.Get(json, `features.#[id=="q4"]`).Raw, nil)
 
 	c := New()
-	c.Set("p1", p1, nil, nil, 0)
+	c.Set("p1", p1, field.List{}, 0)
-	c.Set("p2", p2, nil, nil, 0)
+	c.Set("p2", p2, field.List{}, 0)
-	c.Set("p3", p3, nil, nil, 0)
+	c.Set("p3", p3, field.List{}, 0)
-	c.Set("p4", p4, nil, nil, 0)
+	c.Set("p4", p4, field.List{}, 0)
-	c.Set("r1", r1, nil, nil, 0)
+	c.Set("r1", r1, field.List{}, 0)
-	c.Set("r2", r2, nil, nil, 0)
+	c.Set("r2", r2, field.List{}, 0)
-	c.Set("r3", r3, nil, nil, 0)
+	c.Set("r3", r3, field.List{}, 0)
 
 	var n int
 
 	n = 0
 	c.Within(q1, 0, nil, nil,
-		func(id string, obj geojson.Object, fields []float64) bool {
+		func(id string, obj geojson.Object, _ field.List) bool {
 			n++
 			return true
 		},
@@ -449,7 +505,7 @@ func TestSpatialSearch(t *testing.T) {
 
 	n = 0
 	c.Within(q2, 0, nil, nil,
-		func(id string, obj geojson.Object, fields []float64) bool {
+		func(id string, obj geojson.Object, _ field.List) bool {
 			n++
 			return true
 		},
@@ -458,7 +514,7 @@ func TestSpatialSearch(t *testing.T) {
 
 	n = 0
 	c.Within(q3, 0, nil, nil,
-		func(id string, obj geojson.Object, fields []float64) bool {
+		func(id string, obj geojson.Object, _ field.List) bool {
 			n++
 			return true
 		},
@@ -467,7 +523,7 @@ func TestSpatialSearch(t *testing.T) {
 
 	n = 0
 	c.Intersects(q1, 0, nil, nil,
-		func(_ string, _ geojson.Object, _ []float64) bool {
+		func(_ string, _ geojson.Object, _ field.List) bool {
 			n++
 			return true
 		},
@@ -476,7 +532,7 @@ func TestSpatialSearch(t *testing.T) {
 
 	n = 0
 	c.Intersects(q2, 0, nil, nil,
-		func(_ string, _ geojson.Object, _ []float64) bool {
+		func(_ string, _ geojson.Object, _ field.List) bool {
 			n++
 			return true
 		},
@@ -485,7 +541,7 @@ func TestSpatialSearch(t *testing.T) {
 
 	n = 0
 	c.Intersects(q3, 0, nil, nil,
-		func(_ string, _ geojson.Object, _ []float64) bool {
+		func(_ string, _ geojson.Object, _ field.List) bool {
 			n++
 			return true
 		},
@@ -494,7 +550,7 @@ func TestSpatialSearch(t *testing.T) {
 
 	n = 0
 	c.Intersects(q3, 0, nil, nil,
-		func(_ string, _ geojson.Object, _ []float64) bool {
+		func(_ string, _ geojson.Object, _ field.List) bool {
 			n++
 			return n <= 1
 		},
@@ -509,7 +565,7 @@ func TestSpatialSearch(t *testing.T) {
 	lastDist := float64(-1)
 	distsMonotonic := true
 	c.Nearby(q4, nil, nil,
-		func(id string, obj geojson.Object, fields []float64, dist float64) bool {
+		func(id string, obj geojson.Object, fields field.List, dist float64) bool {
 			if dist < lastDist {
 				distsMonotonic = false
 			}
@@ -534,12 +590,12 @@ func TestCollectionSparse(t *testing.T) {
 		x := (r.Max.X-r.Min.X)*rand.Float64() + r.Min.X
 		y := (r.Max.Y-r.Min.Y)*rand.Float64() + r.Min.Y
 		point := PO(x, y)
-		c.Set(fmt.Sprintf("%d", i), point, nil, nil, 0)
+		c.Set(fmt.Sprintf("%d", i), point, field.List{}, 0)
 	}
 	var n int
 	n = 0
 	c.Within(rect, 1, nil, nil,
-		func(id string, obj geojson.Object, fields []float64) bool {
+		func(id string, obj geojson.Object, fields field.List) bool {
 			n++
 			return true
 		},
@@ -548,7 +604,7 @@ func TestCollectionSparse(t *testing.T) {
 
 	n = 0
 	c.Within(rect, 2, nil, nil,
-		func(id string, obj geojson.Object, fields []float64) bool {
+		func(id string, obj geojson.Object, fields field.List) bool {
 			n++
 			return true
 		},
@@ -557,7 +613,7 @@ func TestCollectionSparse(t *testing.T) {
 
 	n = 0
 	c.Within(rect, 3, nil, nil,
-		func(id string, obj geojson.Object, fields []float64) bool {
+		func(id string, obj geojson.Object, fields field.List) bool {
 			n++
 			return true
 		},
@@ -566,7 +622,7 @@ func TestCollectionSparse(t *testing.T) {
 
 	n = 0
 	c.Within(rect, 3, nil, nil,
-		func(id string, obj geojson.Object, fields []float64) bool {
+		func(id string, obj geojson.Object, fields field.List) bool {
 			n++
 			return n <= 30
 		},
@@ -575,7 +631,7 @@ func TestCollectionSparse(t *testing.T) {
 
 	n = 0
 	c.Intersects(rect, 3, nil, nil,
-		func(id string, _ geojson.Object, _ []float64) bool {
+		func(id string, _ geojson.Object, _ field.List) bool {
 			n++
 			return true
 		},
@@ -584,7 +640,7 @@ func TestCollectionSparse(t *testing.T) {
 
 	n = 0
 	c.Intersects(rect, 3, nil, nil,
-		func(id string, _ geojson.Object, _ []float64) bool {
+		func(id string, _ geojson.Object, _ field.List) bool {
 			n++
 			return n <= 30
 		},
@@ -626,7 +682,7 @@ func TestManyCollections(t *testing.T) {
 				col = New()
 				colsM[key] = col
 			}
-			col.Set(id, obj, nil, nil, 0)
+			col.Set(id, obj, field.List{}, 0)
 			k++
 		}
 	}
@@ -637,7 +693,7 @@ func TestManyCollections(t *testing.T) {
 		Min: geometry.Point{X: -180, Y: 30},
 		Max: geometry.Point{X: 34, Y: 100},
 	}
-	col.geoSearch(bbox, func(id string, obj geojson.Object, fields []float64) bool {
+	col.geoSearch(bbox, func(id string, obj geojson.Object, fields field.List) bool {
 		//println(id)
 		return true
 	})
@@ -646,15 +702,17 @@ func TestManyCollections(t *testing.T) {
 type testPointItem struct {
 	id     string
 	object geojson.Object
-	fields []float64
+	fields field.List
 }
 
-func makeBenchFields(nFields int) []float64 {
+func makeBenchFields(nFields int) field.List {
-	if nFields == 0 {
+	var fields field.List
-		return nil
+	for i := 0; i < nFields; i++ {
+		key := fmt.Sprintf("%d", i)
+		val := key
+		fields = fields.Set(field.Make(key, val))
 	}
-
+	return fields
-	return make([]float64, nFields)
 }
 
 func BenchmarkInsert_Fields(t *testing.B) {
@@ -678,7 +736,7 @@ func benchmarkInsert(t *testing.B, nFields int) {
 	col := New()
 	t.ResetTimer()
 	for i := 0; i < t.N; i++ {
-		col.Set(items[i].id, items[i].object, nil, items[i].fields, 0)
+		col.Set(items[i].id, items[i].object, items[i].fields, 0)
 	}
 }
 
@@ -702,11 +760,11 @@ func benchmarkReplace(t *testing.B, nFields int) {
 	}
 	col := New()
 	for i := 0; i < t.N; i++ {
-		col.Set(items[i].id, items[i].object, nil, items[i].fields, 0)
+		col.Set(items[i].id, items[i].object, items[i].fields, 0)
 	}
 	t.ResetTimer()
 	for _, i := range rand.Perm(t.N) {
-		o, _, _ := col.Set(items[i].id, items[i].object, nil, nil, 0)
+		o, _, _ := col.Set(items[i].id, items[i].object, field.List{}, 0)
 		if o != items[i].object {
 			t.Fatal("shoot!")
 		}
@@ -733,7 +791,7 @@ func benchmarkGet(t *testing.B, nFields int) {
 	}
 	col := New()
 	for i := 0; i < t.N; i++ {
-		col.Set(items[i].id, items[i].object, nil, items[i].fields, 0)
+		col.Set(items[i].id, items[i].object, items[i].fields, 0)
 	}
 	t.ResetTimer()
 	for _, i := range rand.Perm(t.N) {
@@ -764,7 +822,7 @@ func benchmarkRemove(t *testing.B, nFields int) {
 	}
 	col := New()
 	for i := 0; i < t.N; i++ {
-		col.Set(items[i].id, items[i].object, nil, items[i].fields, 0)
+		col.Set(items[i].id, items[i].object, items[i].fields, 0)
 	}
 	t.ResetTimer()
 	for _, i := range rand.Perm(t.N) {
@@ -795,12 +853,12 @@ func benchmarkScan(t *testing.B, nFields int) {
 	}
 	col := New()
 	for i := 0; i < t.N; i++ {
-		col.Set(items[i].id, items[i].object, nil, items[i].fields, 0)
+		col.Set(items[i].id, items[i].object, items[i].fields, 0)
 	}
 	t.ResetTimer()
 	for i := 0; i < t.N; i++ {
 		var scanIteration int
-		col.Scan(true, nil, nil, func(id string, obj geojson.Object, fields []float64) bool {
+		col.Scan(true, nil, nil, func(id string, obj geojson.Object, fields field.List) bool {
 			scanIteration++
 			return scanIteration <= 500
 		})

internal/collection/fieldvalues.go

@@ -1,53 +0,0 @@
-package collection
-
-type fieldValues struct {
-	freelist []fieldValuesSlot
-	data     [][]float64
-}
-
-type fieldValuesSlot int
-
-const nilValuesSlot fieldValuesSlot = -1
-
-func (f *fieldValues) get(k fieldValuesSlot) []float64 {
-	if k == nilValuesSlot {
-		return nil
-	}
-	return f.data[int(k)]
-}
-
-func (f *fieldValues) set(k fieldValuesSlot, itemData []float64) fieldValuesSlot {
-	// if we're asked to store into the nil values slot, it means one of two things:
-	//   - we are doing a replace on an item that previously had nil fields
-	//   - we are inserting a new item
-	// in either case, check if the new values are not nil, and if so allocate a
-	// new slot
-	if k == nilValuesSlot {
-		if itemData == nil {
-			return nilValuesSlot
-		}
-
-		// first check if there is a slot on the freelist to reuse
-		if len(f.freelist) > 0 {
-			var slot fieldValuesSlot
-			slot, f.freelist = f.freelist[len(f.freelist)-1], f.freelist[:len(f.freelist)-1]
-			f.data[slot] = itemData
-			return slot
-		}
-
-		// no reusable slot, append
-		f.data = append(f.data, itemData)
-		return fieldValuesSlot(len(f.data) - 1)
-
-	}
-	f.data[int(k)] = itemData
-	return k
-}
-
-func (f *fieldValues) remove(k fieldValuesSlot) {
-	if k == nilValuesSlot {
-		return
-	}
-	f.data[int(k)] = nil
-	f.freelist = append(f.freelist, k)
-}

internal/field/field.go

@@ -0,0 +1,232 @@
+package field
+
+import (
+	"math"
+	"strconv"
+	"strings"
+
+	"github.com/tidwall/gjson"
+	"github.com/tidwall/pretty"
+)
+
+var ZeroValue = Value{kind: Number, data: "0", num: 0}
+var ZeroField = Field{name: "", value: ZeroValue}
+
+type Kind byte
+
+const (
+	Null   = Kind(gjson.Null)
+	False  = Kind(gjson.False)
+	Number = Kind(gjson.Number)
+	String = Kind(gjson.String)
+	True   = Kind(gjson.True)
+	JSON   = Kind(gjson.JSON)
+)
+
+type Value struct {
+	kind Kind
+	data string
+	num  float64
+}
+
+func (v Value) IsZero() bool {
+	return (v.kind == Number && v.data == "0" && v.num == 0) || v == (Value{})
+}
+
+func (v Value) Equals(b Value) bool {
+	return v.kind == b.kind && v.data == b.data
+}
+
+func (v Value) Kind() Kind {
+	return v.kind
+}
+
+func (v Value) Data() string {
+	return v.data
+}
+
+func (v Value) Num() float64 {
+	return v.num
+}
+
+func (v Value) JSON() string {
+	switch v.Kind() {
+	case Number:
+		switch v.Data() {
+		case "NaN":
+			return `"NaN"`
+		case "+Inf":
+			return `"+Inf"`
+		case "-Inf":
+			return `"-Inf"`
+		default:
+			return v.Data()
+		}
+	case String:
+		return string(gjson.AppendJSONString(nil, v.Data()))
+	case True:
+		return "true"
+	case False:
+		return "false"
+	case Null:
+		if v != (Value{}) {
+			return "null"
+		}
+	case JSON:
+		return v.Data()
+	}
+	return "0"
+}
+
+func stringLessInsensitive(a, b string) bool {
+	for i := 0; i < len(a) && i < len(b); i++ {
+		if a[i] >= 'A' && a[i] <= 'Z' {
+			if b[i] >= 'A' && b[i] <= 'Z' {
+				// both are uppercase, do nothing
+				if a[i] < b[i] {
+					return true
+				} else if a[i] > b[i] {
+					return false
+				}
+			} else {
+				// a is uppercase, convert a to lowercase
+				if a[i]+32 < b[i] {
+					return true
+				} else if a[i]+32 > b[i] {
+					return false
+				}
+			}
+		} else if b[i] >= 'A' && b[i] <= 'Z' {
+			// b is uppercase, convert b to lowercase
+			if a[i] < b[i]+32 {
+				return true
+			} else if a[i] > b[i]+32 {
+				return false
+			}
+		} else {
+			// neither are uppercase
+			if a[i] < b[i] {
+				return true
+			} else if a[i] > b[i] {
+				return false
+			}
+		}
+	}
+	return len(a) < len(b)
+}
+
+// Less return true if a value is less than another value.
+// The caseSensitive paramater is used when the value are Strings.
+// The order when comparing two different kinds is:
+//
+//	Null < False < Number < String < True < JSON
+//
+// Pulled from github.com/tidwall/gjson
+func (v Value) LessCase(b Value, caseSensitive bool) bool {
+	if v.kind < b.kind {
+		return true
+	}
+	if v.kind > b.kind {
+		return false
+	}
+	if v.kind == Number {
+		return v.num < b.num
+	}
+	if v.kind == String {
+		if caseSensitive {
+			return v.data < b.data
+		}
+		return stringLessInsensitive(v.data, b.data)
+	}
+	return v.data < b.data
+}
+
+// Less return true if a value is less than another value.
+//
+//	Null < False < Number < String < True < JSON
+//
+// Pulled from github.com/tidwall/gjson
+func (v Value) Less(b Value) bool {
+	return v.LessCase(b, false)
+}
+
+type Field struct {
+	name  string
+	value Value
+}
+
+func (f Field) Name() string {
+	return f.name
+}
+
+func (f Field) Value() Value {
+	return f.value
+}
+
+func (f Field) Weight() int {
+	return len(f.name) + 8 + len(f.value.data)
+}
+
+var nan = math.NaN()
+var pinf = math.Inf(+1)
+var ninf = math.Inf(-1)
+
+func ValueOf(data string) Value {
+	data = strings.TrimSpace(data)
+	num, err := strconv.ParseFloat(data, 64)
+	if err == nil {
+		if math.IsInf(num, 0) {
+			if math.IsInf(num, +1) {
+				return Value{kind: Number, data: "+Inf", num: pinf}
+			} else {
+				return Value{kind: Number, data: "-Inf", num: ninf}
+			}
+		} else if math.IsNaN(num) {
+			return Value{kind: Number, data: "NaN", num: nan}
+		}
+		return Value{kind: Number, data: data, num: num}
+	}
+	if gjson.Valid(data) {
+		data = strings.TrimSpace(data)
+		r := gjson.Parse(data)
+		switch r.Type {
+		case gjson.Null:
+			return Value{kind: Null, data: "null"}
+		case gjson.JSON:
+			return Value{kind: JSON, data: string(pretty.Ugly([]byte(data)))}
+		case gjson.True:
+			return Value{kind: True, data: "true"}
+		case gjson.False:
+			return Value{kind: False, data: "false"}
+		case gjson.Number:
+			// Ignore. Numbers will always be picked up by the ParseFloat above.
+		case gjson.String:
+			// Ignore. Strings fallthrough by default
+		}
+		// Extract String from JSON
+		data = r.String()
+	}
+	// Check if string is NaN, Inf(inity), +Inf(inity), -Inf(inity)
+	if len(data) >= 3 && len(data) <= 9 {
+		switch data[0] {
+		case '-', '+', 'I', 'i', 'N', 'n':
+			switch strings.ToLower(data) {
+			case "nan":
+				return Value{kind: Number, data: "NaN", num: nan}
+			case "inf", "+inf", "infinity", "+infinity":
+				return Value{kind: Number, data: "+Inf", num: pinf}
+			case "-inf", "-infinity":
+				return Value{kind: Number, data: "-Inf", num: ninf}
+			}
+		}
+	}
+
+	return Value{kind: String, data: data}
+}
+
+func Make(name, data string) Field {
+	return Field{
+		strings.ToLower(strings.TrimSpace(name)),
+		ValueOf(data),
+	}
+}

internal/field/field_test.go

@@ -0,0 +1,134 @@
+package field
+
+import (
+	"testing"
+
+	"github.com/tidwall/assert"
+)
+
+func mLT(a, b Value) bool  { return a.Less(b) }
+func mLTE(a, b Value) bool { return !mLT(b, a) }
+func mGT(a, b Value) bool  { return mLT(b, a) }
+func mGTE(a, b Value) bool { return !mLT(a, b) }
+func mEQ(a, b Value) bool  { return !mLT(a, b) && !mLT(b, a) }
+
+func TestOrder(t *testing.T) {
+	assert.Assert(mLT(ValueOf("hello"), ValueOf("jello")))
+	assert.Assert(mLT(ValueOf("hello"), ValueOf("JELLO")))
+	assert.Assert(mLT(ValueOf("HELLO"), ValueOf("JELLO")))
+	assert.Assert(mLT(ValueOf("HELLO"), ValueOf("jello")))
+	assert.Assert(!mLT(ValueOf("hello"), ValueOf("hello")))
+	assert.Assert(!mLT(ValueOf("jello"), ValueOf("hello")))
+	assert.Assert(!mLT(ValueOf("Jello"), ValueOf("Hello")))
+	assert.Assert(!mLT(ValueOf("Jello"), ValueOf("hello")))
+	assert.Assert(!mLT(ValueOf("jello"), ValueOf("Hello")))
+	assert.Assert(mGT(ValueOf("jello"), ValueOf("hello")))
+	assert.Assert(!mGT(ValueOf("jello"), ValueOf("jello")))
+	assert.Assert(!mGT(ValueOf("hello"), ValueOf("jello")))
+	assert.Assert(mLTE(ValueOf("hello"), ValueOf("jello")))
+	assert.Assert(mLTE(ValueOf("hello"), ValueOf("hello")))
+	assert.Assert(mLTE(ValueOf("hello"), ValueOf("HELLO")))
+	assert.Assert(!mLTE(ValueOf("jello"), ValueOf("hello")))
+	assert.Assert(mGTE(ValueOf("jello"), ValueOf("jello")))
+	assert.Assert(mGTE(ValueOf("jello"), ValueOf("hello")))
+	assert.Assert(mGTE(ValueOf("jello"), ValueOf("JELLO")))
+	assert.Assert(!mGTE(ValueOf("hello"), ValueOf("jello")))
+	assert.Assert(mEQ(ValueOf("jello"), ValueOf("jello")))
+	assert.Assert(mEQ(ValueOf("jello"), ValueOf("JELLO")))
+	assert.Assert(!mEQ(ValueOf("jello"), ValueOf("hello")))
+}
+
+func TestLess(t *testing.T) {
+	assert.Assert(mLT(ValueOf("null"), ValueOf("false")))
+	assert.Assert(mLT(ValueOf("null"), ValueOf("123")))
+	assert.Assert(mLT(ValueOf("null"), ValueOf("hello")))
+	assert.Assert(mLT(ValueOf("null"), ValueOf("true")))
+	assert.Assert(mLT(ValueOf("null"), ValueOf("[]")))
+	assert.Assert(mLT(ValueOf("false"), ValueOf("123")))
+	assert.Assert(mLT(ValueOf("false"), ValueOf("hello")))
+	assert.Assert(mLT(ValueOf("false"), ValueOf("true")))
+	assert.Assert(mLT(ValueOf("false"), ValueOf("[]")))
+	assert.Assert(mLT(ValueOf("123"), ValueOf("hello")))
+	assert.Assert(mLT(ValueOf("123"), ValueOf("true")))
+	assert.Assert(mLT(ValueOf("123"), ValueOf("[]")))
+	assert.Assert(mLT(ValueOf("hello"), ValueOf("true")))
+	assert.Assert(mLT(ValueOf("hello"), ValueOf("[]")))
+	assert.Assert(mLT(ValueOf("true"), ValueOf("[]")))
+	assert.Assert(!mLT(ValueOf("false"), ValueOf("null")))
+	assert.Assert(!mLT(ValueOf("123"), ValueOf("null")))
+	assert.Assert(!mLT(ValueOf("hello"), ValueOf("null")))
+	assert.Assert(!mLT(ValueOf("true"), ValueOf("null")))
+	assert.Assert(!mLT(ValueOf("[]"), ValueOf("null")))
+	assert.Assert(!mLT(ValueOf("123"), ValueOf("false")))
+	assert.Assert(!mLT(ValueOf("hello"), ValueOf("false")))
+	assert.Assert(!mLT(ValueOf("true"), ValueOf("false")))
+	assert.Assert(!mLT(ValueOf("[]"), ValueOf("false")))
+	assert.Assert(!mLT(ValueOf("hello"), ValueOf("123")))
+	assert.Assert(!mLT(ValueOf("true"), ValueOf("123")))
+	assert.Assert(!mLT(ValueOf("[]"), ValueOf("123")))
+	assert.Assert(!mLT(ValueOf("true"), ValueOf("hello")))
+	assert.Assert(!mLT(ValueOf("[]"), ValueOf("hello")))
+	assert.Assert(!mLT(ValueOf("[]"), ValueOf("true")))
+	assert.Assert(mLT(ValueOf("123"), ValueOf("456")))
+	assert.Assert(mLT(ValueOf("[1]"), ValueOf("[2]")))
+}
+
+func TestLessCase(t *testing.T) {
+	assert.Assert(ValueOf("A").LessCase(ValueOf("B"), true))
+	assert.Assert(!ValueOf("A").LessCase(ValueOf("A"), true))
+	assert.Assert(!ValueOf("B").LessCase(ValueOf("A"), true))
+}
+
+func TestVarious(t *testing.T) {
+	assert.Assert(!ValueOf("A").IsZero())
+	assert.Assert(ValueOf("0").IsZero())
+	assert.Assert(Value{}.IsZero())
+	assert.Assert(ZeroValue.IsZero())
+	assert.Assert(ZeroValue.Equals(ZeroValue))
+	assert.Assert(ZeroValue.Kind() == Number)
+	assert.Assert(ValueOf("0").Kind() == Number)
+	assert.Assert(ValueOf("hello").Kind() == String)
+	assert.Assert(ValueOf(`"hello"`).Kind() == String)
+	assert.Assert(ValueOf(`"123"`).Kind() == String)
+	assert.Assert(ValueOf(`"123"`).Data() == `123`)
+	assert.Assert(ValueOf(`"123"`).Num() == 0)
+}
+
+func TestJSON(t *testing.T) {
+	assert.Assert(ValueOf(`A`).JSON() == `"A"`)
+	assert.Assert(ValueOf(`"A"`).JSON() == `"A"`)
+	assert.Assert(ValueOf(`123`).JSON() == `123`)
+	assert.Assert(ValueOf(`{}`).JSON() == `{}`)
+	assert.Assert(ValueOf(`{  }`).JSON() == `{}`)
+	assert.Assert(ValueOf(` -Inf `).JSON() == `"-Inf"`)
+	assert.Assert(ValueOf(` "-Inf" `).JSON() == `"-Inf"`)
+	assert.Assert(ValueOf(`+Inf`).JSON() == `"+Inf"`)
+	assert.Assert(ValueOf(`"+Inf"`).JSON() == `"+Inf"`)
+	assert.Assert(ValueOf(`Inf`).JSON() == `"+Inf"`)
+	assert.Assert(ValueOf(`"Inf"`).JSON() == `"+Inf"`)
+	assert.Assert(ValueOf(`NaN`).JSON() == `"NaN"`)
+	assert.Assert(ValueOf(`"NaN"`).JSON() == `"NaN"`)
+	assert.Assert(ValueOf(`nan`).JSON() == `"NaN"`)
+	assert.Assert(ValueOf(`infinity`).JSON() == `"+Inf"`)
+	assert.Assert(ValueOf(` true `).JSON() == `true`)
+	assert.Assert(ValueOf(` false `).JSON() == `false`)
+	assert.Assert(ValueOf(` null `).JSON() == `null`)
+	assert.Assert(Value{}.JSON() == `0`)
+	assert.Assert(Value{}.JSON() == `0`)
+}
+
+func TestField(t *testing.T) {
+	assert.Assert(Make("hello", "123").Name() == "hello")
+	assert.Assert(Make("HELLO", "123").Name() == "hello")
+	assert.Assert(Make("HELLO", "123").Value().Num() == 123)
+	assert.Assert(Make("HELLO", "123").Value().JSON() == "123")
+	assert.Assert(Make("HELLO", "123").Value().Num() == 123)
+}
+
+func TestWeight(t *testing.T) {
+	assert.Assert(Make("hello", "123").Weight() == 16)
+}
+
+func TestNumber(t *testing.T) {
+	assert.Assert(ValueOf("012").Num() == 12)
+}

internal/field/list_array.go

@@ -0,0 +1,84 @@
+//go:build exclude
+
+package field
+
+type List struct {
+	entries []Field
+}
+
+// bsearch searches array for value.
+func (fields List) bsearch(name string) (index int, found bool) {
+	i, j := 0, len(fields.entries)
+	for i < j {
+		h := i + (j-i)/2
+		if name >= fields.entries[h].name {
+			i = h + 1
+		} else {
+			j = h
+		}
+	}
+	if i > 0 && fields.entries[i-1].name >= name {
+		return i - 1, true
+	}
+	return i, false
+}
+
+func (fields List) Set(field Field) List {
+	var updated List
+	index, found := fields.bsearch(field.name)
+	if found {
+		if field.value.IsZero() {
+			// delete
+			if len(fields.entries) > 1 {
+				updated.entries = make([]Field, len(fields.entries)-1)
+				copy(updated.entries, fields.entries[:index])
+				copy(updated.entries[index:], fields.entries[index+1:])
+			}
+		} else if !fields.entries[index].value.Equals(field.value) {
+			// update
+			updated.entries = make([]Field, len(fields.entries))
+			copy(updated.entries, fields.entries)
+			updated.entries[index].value = field.value
+		} else {
+			// nothing changes
+			updated = fields
+		}
+		return updated
+	}
+	if field.Value().IsZero() {
+		return fields
+	}
+	updated.entries = make([]Field, len(fields.entries)+1)
+	copy(updated.entries, fields.entries[:index])
+	copy(updated.entries[index+1:], fields.entries[index:])
+	updated.entries[index] = field
+	return updated
+}
+
+func (fields List) Get(name string) Field {
+	index, found := fields.bsearch(name)
+	if !found {
+		return ZeroField
+	}
+	return fields.entries[index]
+}
+
+func (fields List) Scan(iter func(field Field) bool) {
+	for _, f := range fields.entries {
+		if !iter(f) {
+			return
+		}
+	}
+}
+
+func (fields List) Len() int {
+	return len(fields.entries)
+}
+
+func (fields List) Weight() int {
+	var weight int
+	for _, f := range fields.entries {
+		weight += f.Weight()
+	}
+	return weight
+}

internal/field/list_binary.go

@@ -0,0 +1,349 @@
+package field
+
+import (
+	"encoding/binary"
+	"strconv"
+	"unsafe"
+
+	"github.com/tidwall/tile38/internal/sstring"
+)
+
+// binary format
+//   (size,entry,[entry...])
+//   size: uvarint            -- size of the full byte slice, excluding itself.
+//   entry: (name,value)      -- one field entry
+//   name: shared string num  -- field name, string data, uses the shared library
+//   size: uvarint            -- number of bytes in data
+//   value: (kind,vdata)      -- field value
+//   kind: byte               -- value kind
+//   vdata: (size,data)       -- value data, string data
+
+// useSharedNames will results in smaller memory usage by sharing the names
+// of fields using the sstring package. Otherwise the names are embeded with
+// the list.
+const useSharedNames = true
+
+// List of fields, ordered by Name.
+type List struct {
+	p *byte
+}
+
+type bytes struct {
+	p *byte
+	l int
+	c int
+}
+
+func ptob(p *byte) []byte {
+	if p == nil {
+		return nil
+	}
+	// Get the size of the bytes (excluding the header)
+	x, n := uvarint(*(*[]byte)(unsafe.Pointer(&bytes{p, 10, 10})))
+	// Return the byte slice (excluding the header)
+	return (*(*[]byte)(unsafe.Pointer(&bytes{p, n + x, n + x})))[n:]
+}
+
+func btoa(b []byte) string {
+	return *(*string)(unsafe.Pointer(&b))
+}
+
+// uvarint is a slightly modified version of binary.Uvarint, and it's a little
+// faster. But it lacks overflow checks which are not needed for our use.
+func uvarint(buf []byte) (int, int) {
+	var x uint64
+	for i := 0; i < len(buf); i++ {
+		b := buf[i]
+		if b < 0x80 {
+			return int(x | uint64(b)<<(i*7)), i + 1
+		}
+		x |= uint64(b&0x7f) << (i * 7)
+	}
+	return 0, 0
+}
+
+func datakind(kind Kind) bool {
+	switch kind {
+	case Number, String, JSON:
+		return true
+	}
+	return false
+}
+
+func bfield(name string, kind Kind, data string) Field {
+	var num float64
+	switch kind {
+	case Number:
+		num, _ = strconv.ParseFloat(data, 64)
+	case Null:
+		data = "null"
+	case False:
+		data = "false"
+	case True:
+		data = "true"
+	}
+	return Field{
+		name: name,
+		value: Value{
+			kind: Kind(kind),
+			data: data,
+			num:  num,
+		},
+	}
+}
+
+// Set a field in the list.
+// If the input field value is zero `f.Value().IsZero()` then the field is
+// deleted or removed from the list since lists cannot have Zero values.
+// Returns a newly allocated list the updated field.
+// The original (receiver) list is not modified.
+func (fields List) Set(field Field) List {
+	b := ptob(fields.p)
+	var i int
+	for {
+		s := i
+		// read the name
+		var name string
+		x, n := uvarint(b[i:])
+		if n == 0 {
+			break
+		}
+		if useSharedNames {
+			name = sstring.Load(x)
+			i += n
+		} else {
+			name = btoa(b[i+n : i+n+x])
+			i += n + x
+		}
+		kind := Kind(b[i])
+		i++
+		var data string
+		if datakind(kind) {
+			x, n = uvarint(b[i:])
+			data = btoa(b[i+n : i+n+x])
+			i += n + x
+		}
+		if field.name < name {
+			// insert before
+			i = s
+			break
+		}
+		if name == field.name {
+			if field.Value().IsZero() {
+				// delete
+				return List{delfield(b, s, i)}
+			}
+			prev := bfield(name, kind, data)
+			if prev.Value().Equals(field.Value()) {
+				// no change
+				return fields
+			}
+			// replace
+			return List{putfield(b, field, s, i)}
+		}
+	}
+	if field.Value().IsZero() {
+		return fields
+	}
+	// insert after
+	return List{putfield(b, field, i, i)}
+}
+
+func delfield(b []byte, s, e int) *byte {
+	totallen := s + (len(b) - e)
+	var psz [10]byte
+	pn := binary.PutUvarint(psz[:], uint64(totallen))
+	plen := pn + totallen
+	p := make([]byte, plen)
+	// copy each component
+	i := 0
+
+	// -- header size
+	copy(p[i:], psz[:pn])
+	i += pn
+
+	// -- head entries
+	copy(p[i:], b[:s])
+	i += s
+
+	// -- tail entries
+	copy(p[i:], b[e:])
+
+	return &p[0]
+}
+
+func putfield(b []byte, f Field, s, e int) *byte {
+	name := f.Name()
+	var namesz [10]byte
+	var namen int
+	if useSharedNames {
+		num := sstring.Store(name)
+		namen = binary.PutUvarint(namesz[:], uint64(num))
+	} else {
+		namen = binary.PutUvarint(namesz[:], uint64(len(name)))
+	}
+	value := f.Value()
+	kind := value.Kind()
+	isdatakind := datakind(kind)
+	var data string
+	var datasz [10]byte
+	var datan int
+	if isdatakind {
+		data = value.Data()
+		datan = binary.PutUvarint(datasz[:], uint64(len(data)))
+	}
+	var totallen int
+	if useSharedNames {
+		totallen = s + namen + 1 + (len(b) - e)
+	} else {
+		totallen = s + namen + len(name) + 1 + +(len(b) - e)
+	}
+	if isdatakind {
+		totallen += datan + len(data)
+	}
+	var psz [10]byte
+	pn := binary.PutUvarint(psz[:], uint64(totallen))
+	plen := pn + totallen
+	p := make([]byte, plen)
+
+	// copy each component
+	i := 0
+
+	// -- header size
+	copy(p[i:], psz[:pn])
+	i += pn
+
+	// -- head entries
+	copy(p[i:], b[:s])
+	i += s
+
+	// -- name
+	copy(p[i:], namesz[:namen])
+	i += namen
+
+	if !useSharedNames {
+		copy(p[i:], name)
+		i += len(name)
+	}
+
+	// -- kind
+	p[i] = byte(kind)
+	i++
+
+	if isdatakind {
+		// -- data
+		copy(p[i:], datasz[:datan])
+		i += datan
+
+		copy(p[i:], data)
+		i += len(data)
+	}
+
+	// -- tail entries
+	copy(p[i:], b[e:])
+
+	return &p[0]
+}
+
+// Get a field from the list. Or returns ZeroField if not found.
+func (fields List) Get(name string) Field {
+	b := ptob(fields.p)
+	var i int
+	for {
+		// read the fname
+		var fname string
+		x, n := uvarint(b[i:])
+		if n == 0 {
+			break
+		}
+		if useSharedNames {
+			fname = sstring.Load(x)
+			i += n
+		} else {
+			fname = btoa(b[i+n : i+n+x])
+			i += n + x
+		}
+		kind := Kind(b[i])
+		i++
+		var data string
+		if datakind(kind) {
+			x, n = uvarint(b[i:])
+			data = btoa(b[i+n : i+n+x])
+			i += n + x
+		}
+		if name < fname {
+			break
+		}
+		if fname == name {
+			return bfield(fname, kind, data)
+		}
+	}
+	return ZeroField
+}
+
+// Scan each field in list
+func (fields List) Scan(iter func(field Field) bool) {
+	b := ptob(fields.p)
+	var i int
+	for {
+		// read the fname
+		var fname string
+		x, n := uvarint(b[i:])
+		if n == 0 {
+			break
+		}
+		if useSharedNames {
+			fname = sstring.Load(x)
+			i += n
+		} else {
+			fname = btoa(b[i+n : i+n+x])
+			i += n + x
+		}
+		kind := Kind(b[i])
+		i++
+		var data string
+		if datakind(kind) {
+			x, n = uvarint(b[i:])
+			data = btoa(b[i+n : i+n+x])
+			i += n + x
+		}
+		if !iter(bfield(fname, kind, data)) {
+			return
+		}
+	}
+}
+
+// Len return the number of fields in list.
+func (fields List) Len() int {
+	var count int
+	b := ptob(fields.p)
+	var i int
+	for {
+		x, n := uvarint(b[i:])
+		if n == 0 {
+			break
+		}
+		if useSharedNames {
+			i += n
+		} else {
+			i += n + x
+		}
+		isdatakind := datakind(Kind(b[i]))
+		i++
+		if isdatakind {
+			x, n = uvarint(b[i:])
+			i += n + x
+		}
+		count++
+	}
+	return count
+}
+
+// Weight is the number of bytes of the list.
+func (fields List) Weight() int {
+	if fields.p == nil {
+		return 0
+	}
+	x, n := uvarint(*(*[]byte)(unsafe.Pointer(&bytes{fields.p, 10, 10})))
+	return x + n
+}

internal/field/list_test.go

@@ -0,0 +1,181 @@
+package field
+
+import (
+	"fmt"
+	"math/rand"
+	"testing"
+	"time"
+
+	"github.com/tidwall/assert"
+	"github.com/tidwall/btree"
+)
+
+func TestList(t *testing.T) {
+	var fields List
+
+	fields = fields.Set(Make("hello", "123"))
+	assert.Assert(fields.Len() == 1)
+	// println(fields.Weight())
+	// assert.Assert(fields.Weight() == 16)
+
+	fields = fields.Set(Make("jello", "456"))
+	assert.Assert(fields.Len() == 2)
+	// assert.Assert(fields.Weight() == 32)
+
+	value := fields.Get("jello")
+	assert.Assert(value.Value().Data() == "456")
+	assert.Assert(value.Value().JSON() == "456")
+	assert.Assert(value.Value().Num() == 456)
+
+	value = fields.Get("nello")
+	assert.Assert(value.Name() == "")
+	assert.Assert(value.Value().IsZero())
+
+	fields = fields.Set(Make("jello", "789"))
+	assert.Assert(fields.Len() == 2)
+	// assert.Assert(fields.Weight() == 32)
+
+	fields = fields.Set(Make("nello", "0"))
+	assert.Assert(fields.Len() == 2)
+	// assert.Assert(fields.Weight() == 32)
+
+	fields = fields.Set(Make("jello", "789"))
+	assert.Assert(fields.Len() == 2)
+	// assert.Assert(fields.Weight() == 32)
+
+	fields = fields.Set(Make("jello", "0"))
+	assert.Assert(fields.Len() == 1)
+	// assert.Assert(fields.Weight() == 16)
+
+	fields = fields.Set(Make("nello", "012"))
+	fields = fields.Set(Make("hello", "456"))
+	fields = fields.Set(Make("fello", "123"))
+	fields = fields.Set(Make("jello", "789"))
+
+	var names string
+	var datas string
+	var nums float64
+	fields.Scan(func(f Field) bool {
+		names += f.Name()
+		datas += f.Value().Data()
+		nums += f.Value().Num()
+		return true
+	})
+	assert.Assert(names == "fellohellojellonello")
+	assert.Assert(datas == "123456789012")
+	assert.Assert(nums == 1380)
+
+	names = ""
+	datas = ""
+	nums = 0
+	fields.Scan(func(f Field) bool {
+		names += f.Name()
+		datas += f.Value().Data()
+		nums += f.Value().Num()
+		return false
+	})
+	assert.Assert(names == "fello")
+	assert.Assert(datas == "123")
+	assert.Assert(nums == 123)
+
+}
+
+func randStr(n int) string {
+	b := make([]byte, n)
+	rand.Read(b)
+	for i := 0; i < n; i++ {
+		b[i] = 'a' + b[i]%26
+	}
+	return string(b)
+}
+
+func randVal(n int) string {
+	switch rand.Intn(10) {
+	case 0:
+		return "null"
+	case 1:
+		return "true"
+	case 2:
+		return "false"
+	case 3:
+		return `{"a":"` + randStr(n) + `"}`
+	case 4:
+		return `["` + randStr(n) + `"]`
+	case 5:
+		return `"` + randStr(n) + `"`
+	case 6:
+		return randStr(n)
+	default:
+		return fmt.Sprintf("%f", rand.Float64()*360)
+	}
+}
+
+func TestRandom(t *testing.T) {
+	seed := time.Now().UnixNano()
+	// seed = 1663607868546669000
+	rand.Seed(seed)
+	start := time.Now()
+	var total int
+	for time.Since(start) < time.Second*2 {
+		N := rand.Intn(500)
+		var org []Field
+		var tr btree.Map[string, Field]
+		var fields List
+		for i := 0; i < N; i++ {
+			name := randStr(rand.Intn(10))
+			value := randVal(rand.Intn(10))
+			field := Make(name, value)
+			org = append(org, field)
+			fields = fields.Set(field)
+			v := fields.Get(name)
+			// println(name, v.Value().Data(), field.Value().Data())
+			if !v.Value().Equals(field.Value()) {
+				t.Fatalf("seed: %d, expected true", seed)
+			}
+			tr.Set(name, field)
+			if fields.Len() != tr.Len() {
+				t.Fatalf("seed: %d, expected %d, got %d",
+					seed, tr.Len(), fields.Len())
+			}
+		}
+		comp := func() {
+			var all []Field
+			fields.Scan(func(f Field) bool {
+				all = append(all, f)
+				return true
+			})
+			if len(all) != fields.Len() {
+				t.Fatalf("seed: %d, expected %d, got %d",
+					seed, fields.Len(), len(all))
+			}
+			if fields.Len() != tr.Len() {
+				t.Fatalf("seed: %d, expected %d, got %d",
+					seed, tr.Len(), fields.Len())
+			}
+			var i int
+			tr.Scan(func(name string, f Field) bool {
+				if name != f.Name() || all[i].Name() != f.Name() {
+					t.Fatalf("seed: %d, out of order", seed)
+				}
+				i++
+				return true
+			})
+		}
+		comp()
+		rand.Shuffle(len(org), func(i, j int) {
+			org[i], org[j] = org[j], org[i]
+		})
+		for _, f := range org {
+			comp()
+			tr.Delete(f.Name())
+			fields = fields.Set(Make(f.Name(), "0"))
+			if fields.Len() != tr.Len() {
+				t.Fatalf("seed: %d, expected %d, got %d",
+					seed, tr.Len(), fields.Len())
+			}
+			comp()
+		}
+		total++
+	}
+
+}

internal/server/aof.go

@@ -294,7 +294,6 @@ func (s *Server) queueHooks(d *commandDetails) error {
 	for _, hook := range candidates {
 		// Calculate all matching fence messages for all candidates and append
 		// them to the appropriate message slice
-		hook.ScanWriter.loadWheres()
 		msgs := FenceMatch(hook.Name, hook.ScanWriter, hook.Fence, hook.Metas, d)
 		if len(msgs) > 0 {
 			if hook.channel {

internal/server/aofshrink.go

@@ -11,6 +11,7 @@ import (
 	"github.com/tidwall/geojson"
 	"github.com/tidwall/tile38/core"
 	"github.com/tidwall/tile38/internal/collection"
+	"github.com/tidwall/tile38/internal/field"
 	"github.com/tidwall/tile38/internal/log"
 )
 
@@ -93,12 +94,10 @@ func (s *Server) aofshrink() {
 					if !ok {
 						return
 					}
-					var fnames = col.FieldArr()     // reload an array of field names to match each object
-					var fmap = col.FieldMap()       //
 					var now = time.Now().UnixNano() // used for expiration
 					var count = 0                   // the object count
 					col.ScanGreaterOrEqual(nextid, false, nil, nil,
-						func(id string, obj geojson.Object, fields []float64, ex int64) bool {
+						func(id string, obj geojson.Object, fields field.List, ex int64) bool {
 							if count == maxids {
 								// we reached the max number of ids for one batch
 								nextid = id
@@ -110,16 +109,14 @@ func (s *Server) aofshrink() {
 							values = append(values, "set")
 							values = append(values, keys[0])
 							values = append(values, id)
-							if len(fields) > 0 {
+							fields.Scan(func(f field.Field) bool {
-								fvs := orderFields(fmap, fnames, fields)
+								if !f.Value().IsZero() {
-								for _, fv := range fvs {
-									if fv.value != 0 {
 									values = append(values, "field")
-										values = append(values, fv.field)
+									values = append(values, f.Name())
-										values = append(values, strconv.FormatFloat(fv.value, 'f', -1, 64))
+									values = append(values, f.Value().JSON())
-									}
-								}
 								}
+								return true
+							})
 							if ex != 0 {
 								ttl := math.Floor(float64(ex-now)/float64(time.Second)*10) / 10
 								if ttl < 0.1 {

internal/server/crud.go

@@ -2,6 +2,7 @@ package server
 
 import (
 	"bytes"
+	"errors"
 	"strconv"
 	"strings"
 	"time"
@@ -11,30 +12,32 @@ import (
 	"github.com/tidwall/geojson/geometry"
 	"github.com/tidwall/resp"
 	"github.com/tidwall/tile38/internal/collection"
+	"github.com/tidwall/tile38/internal/field"
 	"github.com/tidwall/tile38/internal/glob"
 )
 
-type fvt struct {
+// type fvt struct {
-	field string
+// 	field string
-	value float64
+// 	value float64
-}
+// }
+
+// func orderFields(fmap map[string]int, farr []string, fields []float64) []fvt {
+// 	var fv fvt
+// 	var idx int
+// 	fvs := make([]fvt, 0, len(fmap))
+// 	for _, field := range farr {
+// 		idx = fmap[field]
+// 		if idx < len(fields) {
+// 			fv.field = field
+// 			fv.value = fields[idx]
+// 			if fv.value != 0 {
+// 				fvs = append(fvs, fv)
+// 			}
+// 		}
+// 	}
+// 	return fvs
+// }
 
-func orderFields(fmap map[string]int, farr []string, fields []float64) []fvt {
-	var fv fvt
-	var idx int
-	fvs := make([]fvt, 0, len(fmap))
-	for _, field := range farr {
-		idx = fmap[field]
-		if idx < len(fields) {
-			fv.field = field
-			fv.value = fields[idx]
-			if fv.value != 0 {
-				fvs = append(fvs, fv)
-			}
-		}
-	}
-	return fvs
-}
 func (s *Server) cmdBounds(msg *Message) (resp.Value, error) {
 	start := time.Now()
 	vs := msg.Args[1:]
@@ -236,23 +239,26 @@ func (s *Server) cmdGet(msg *Message) (resp.Value, error) {
 		return NOMessage, errInvalidNumberOfArguments
 	}
 	if withfields {
-		fvs := orderFields(col.FieldMap(), col.FieldArr(), fields)
+		nfields := fields.Len()
-		if len(fvs) > 0 {
+		if nfields > 0 {
-			fvals := make([]resp.Value, 0, len(fvs)*2)
+			fvals := make([]resp.Value, 0, nfields*2)
 			if msg.OutputType == JSON {
 				buf.WriteString(`,"fields":{`)
 			}
-			for i, fv := range fvs {
+			var i int
+			fields.Scan(func(f field.Field) bool {
 				if msg.OutputType == JSON {
 					if i > 0 {
 						buf.WriteString(`,`)
 					}
-					buf.WriteString(jsonString(fv.field) + ":" + strconv.FormatFloat(fv.value, 'f', -1, 64))
+					buf.WriteString(jsonString(f.Name()) + ":" + f.Value().JSON())
 				} else {
-					fvals = append(fvals, resp.StringValue(fv.field), resp.StringValue(strconv.FormatFloat(fv.value, 'f', -1, 64)))
+					fvals = append(fvals,
+						resp.StringValue(f.Name()), resp.StringValue(f.Value().Data()))
 				}
 				i++
-			}
+				return true
+			})
 			if msg.OutputType == JSON {
 				buf.WriteString(`}`)
 			} else {
@@ -354,7 +360,7 @@ func (s *Server) cmdPdel(msg *Message) (res resp.Value, d commandDetails, err er
 		return
 	}
 	now := time.Now()
-	iter := func(id string, o geojson.Object, fields []float64) bool {
+	iter := func(id string, o geojson.Object, fields field.List) bool {
 		if match, _ := glob.Match(d.pattern, id); match {
 			d.children = append(d.children, &commandDetails{
 				command:   "del",
@@ -513,7 +519,7 @@ func (s *Server) cmdRename(msg *Message) (res resp.Value, d commandDetails, err
 	return
 }
 
-func (s *Server) cmdFlushDB(msg *Message) (res resp.Value, d commandDetails, err error) {
+func (s *Server) cmdFLUSHDB(msg *Message) (res resp.Value, d commandDetails, err error) {
 	start := time.Now()
 	vs := msg.Args[1:]
 	if len(vs) != 0 {
@@ -543,424 +549,347 @@ func (s *Server) cmdFlushDB(msg *Message) (res resp.Value, d commandDetails, err
 	return
 }
 
-func (s *Server) parseSetArgs(vs []string) (
+// SET key id [FIELD name value ...] [EX seconds] [NX|XX]
-	d commandDetails, fields []string, values []float64,
+// (OBJECT geojson)|(POINT lat lon z)|(BOUNDS minlat minlon maxlat maxlon)|(HASH geohash)|(STRING value)
-	xx, nx bool,
+func (s *Server) cmdSET(msg *Message) (resp.Value, commandDetails, error) {
-	ex int64, etype []byte, evs []string, err error,
+	start := time.Now()
-) {
+	if s.config.maxMemory() > 0 && s.outOfMemory.on() {
-	var ok bool
+		return retwerr(errOOM)
-	var typ []byte
-	if vs, d.key, ok = tokenval(vs); !ok || d.key == "" {
-		err = errInvalidNumberOfArguments
-		return
 	}
-	if vs, d.id, ok = tokenval(vs); !ok || d.id == "" {
+
-		err = errInvalidNumberOfArguments
+	// >> Args
-		return
+
+	var key string
+	var id string
+	var fields []field.Field
+	var ex int64
+	var xx bool
+	var nx bool
+	var obj geojson.Object
+
+	args := msg.Args
+	if len(args) < 3 {
+		return retwerr(errInvalidNumberOfArguments)
 	}
-	var arg []byte
+
-	var nvs []string
+	key, id = args[1], args[2]
-	for {
+
-		if nvs, arg, ok = tokenvalbytes(vs); !ok || len(arg) == 0 {
+	for i := 3; i < len(args); i++ {
-			err = errInvalidNumberOfArguments
+		switch strings.ToLower(args[i]) {
-			return
+		case "field":
+			if i+2 >= len(args) {
+				return retwerr(errInvalidNumberOfArguments)
 			}
-		if lcb(arg, "field") {
+			fkey := strings.ToLower(args[i+1])
-			vs = nvs
+			fval := args[i+2]
-			var name string
+			i += 2
-			var svalue string
+			if isReservedFieldName(fkey) {
-			var value float64
+				return retwerr(errInvalidArgument(fkey))
-			if vs, name, ok = tokenval(vs); !ok || name == "" {
-				err = errInvalidNumberOfArguments
-				return
 			}
-			if isReservedFieldName(name) {
+			fields = append(fields, field.Make(fkey, fval))
-				err = errInvalidArgument(name)
+		case "ex":
-				return
+			if i+1 >= len(args) {
+				return retwerr(errInvalidNumberOfArguments)
 			}
-			if vs, svalue, ok = tokenval(vs); !ok || svalue == "" {
+			exval := args[i+1]
-				err = errInvalidNumberOfArguments
+			i += 1
-				return
+			x, err := strconv.ParseFloat(exval, 64)
-			}
-			value, err = strconv.ParseFloat(svalue, 64)
 			if err != nil {
-				err = errInvalidArgument(svalue)
+				return retwerr(errInvalidArgument(exval))
-				return
 			}
-			fields = append(fields, name)
+			ex = time.Now().UnixNano() + int64(float64(time.Second)*x)
-			values = append(values, value)
+		case "nx":
-			continue
-		}
-		if lcb(arg, "ex") {
-			vs = nvs
-			if ex != 0 {
-				err = errInvalidArgument(string(arg))
-				return
-			}
-			var s string
-			var v float64
-			if vs, s, ok = tokenval(vs); !ok || s == "" {
-				err = errInvalidNumberOfArguments
-				return
-			}
-			v, err = strconv.ParseFloat(s, 64)
-			if err != nil {
-				err = errInvalidArgument(s)
-				return
-			}
-			ex = time.Now().UnixNano() + int64(float64(time.Second)*v)
-			continue
-		}
-		if lcb(arg, "xx") {
-			vs = nvs
-			if nx {
-				err = errInvalidArgument(string(arg))
-				return
-			}
-			xx = true
-			continue
-		}
-		if lcb(arg, "nx") {
-			vs = nvs
 			if xx {
-				err = errInvalidArgument(string(arg))
+				return retwerr(errInvalidArgument(args[i]))
-				return
 			}
 			nx = true
-			continue
+		case "xx":
+			if nx {
+				return retwerr(errInvalidArgument(args[i]))
 			}
-		break
+			xx = true
+		case "string":
+			if i+1 >= len(args) {
+				return retwerr(errInvalidNumberOfArguments)
 			}
-	if vs, typ, ok = tokenvalbytes(vs); !ok || len(typ) == 0 {
+			str := args[i+1]
-		err = errInvalidNumberOfArguments
+			i += 1
-		return
+			obj = collection.String(str)
+		case "point":
+			if i+2 >= len(args) {
+				return retwerr(errInvalidNumberOfArguments)
 			}
-	if len(vs) == 0 {
+			slat := args[i+1]
-		err = errInvalidNumberOfArguments
+			slon := args[i+2]
-		return
+			i += 2
+			var z float64
+			var hasZ bool
+			if i+1 < len(args) {
+				// probe for possible z coordinate
+				var err error
+				z, err = strconv.ParseFloat(args[i+1], 64)
+				if err == nil {
+					hasZ = true
+					i++
 				}
-	etype = typ
-	evs = vs
-	switch {
-	default:
-		err = errInvalidArgument(string(typ))
-		return
-	case lcb(typ, "string"):
-		var str string
-		if vs, str, ok = tokenval(vs); !ok {
-			err = errInvalidNumberOfArguments
-			return
 			}
-		d.obj = collection.String(str)
+			y, err := strconv.ParseFloat(slat, 64)
-	case lcb(typ, "point"):
-		var slat, slon, sz string
-		if vs, slat, ok = tokenval(vs); !ok || slat == "" {
-			err = errInvalidNumberOfArguments
-			return
-		}
-		if vs, slon, ok = tokenval(vs); !ok || slon == "" {
-			err = errInvalidNumberOfArguments
-			return
-		}
-		vs, sz, ok = tokenval(vs)
-		if !ok || sz == "" {
-			var x, y float64
-			y, err = strconv.ParseFloat(slat, 64)
 			if err != nil {
-				err = errInvalidArgument(slat)
+				return retwerr(errInvalidArgument(slat))
-				return
 			}
-			x, err = strconv.ParseFloat(slon, 64)
+			x, err := strconv.ParseFloat(slon, 64)
 			if err != nil {
-				err = errInvalidArgument(slon)
+				return retwerr(errInvalidArgument(slon))
-				return
 			}
-			d.obj = geojson.NewPoint(geometry.Point{X: x, Y: y})
+			if !hasZ {
+				obj = geojson.NewPoint(geometry.Point{X: x, Y: y})
 			} else {
-			var x, y, z float64
+				obj = geojson.NewPointZ(geometry.Point{X: x, Y: y}, z)
-			y, err = strconv.ParseFloat(slat, 64)
+			}
+		case "bounds":
+			if i+4 >= len(args) {
+				return retwerr(errInvalidNumberOfArguments)
+			}
+			var vals [4]float64
+			for j := 0; j < 4; j++ {
+				var err error
+				vals[j], err = strconv.ParseFloat(args[i+1+j], 64)
 				if err != nil {
-				err = errInvalidArgument(slat)
+					return retwerr(errInvalidArgument(args[i+1+j]))
-				return
 				}
-			x, err = strconv.ParseFloat(slon, 64)
-			if err != nil {
-				err = errInvalidArgument(slon)
-				return
 			}
-			z, err = strconv.ParseFloat(sz, 64)
+			i += 4
-			if err != nil {
+			obj = geojson.NewRect(geometry.Rect{
-				err = errInvalidArgument(sz)
+				Min: geometry.Point{X: vals[1], Y: vals[0]},
-				return
+				Max: geometry.Point{X: vals[3], Y: vals[2]},
-			}
-			d.obj = geojson.NewPointZ(geometry.Point{X: x, Y: y}, z)
-		}
-	case lcb(typ, "bounds"):
-		var sminlat, sminlon, smaxlat, smaxlon string
-		if vs, sminlat, ok = tokenval(vs); !ok || sminlat == "" {
-			err = errInvalidNumberOfArguments
-			return
-		}
-		if vs, sminlon, ok = tokenval(vs); !ok || sminlon == "" {
-			err = errInvalidNumberOfArguments
-			return
-		}
-		if vs, smaxlat, ok = tokenval(vs); !ok || smaxlat == "" {
-			err = errInvalidNumberOfArguments
-			return
-		}
-		if vs, smaxlon, ok = tokenval(vs); !ok || smaxlon == "" {
-			err = errInvalidNumberOfArguments
-			return
-		}
-		var minlat, minlon, maxlat, maxlon float64
-		minlat, err = strconv.ParseFloat(sminlat, 64)
-		if err != nil {
-			err = errInvalidArgument(sminlat)
-			return
-		}
-		minlon, err = strconv.ParseFloat(sminlon, 64)
-		if err != nil {
-			err = errInvalidArgument(sminlon)
-			return
-		}
-		maxlat, err = strconv.ParseFloat(smaxlat, 64)
-		if err != nil {
-			err = errInvalidArgument(smaxlat)
-			return
-		}
-		maxlon, err = strconv.ParseFloat(smaxlon, 64)
-		if err != nil {
-			err = errInvalidArgument(smaxlon)
-			return
-		}
-		d.obj = geojson.NewRect(geometry.Rect{
-			Min: geometry.Point{X: minlon, Y: minlat},
-			Max: geometry.Point{X: maxlon, Y: maxlat},
 			})
-	case lcb(typ, "hash"):
+		case "hash":
-		var shash string
+			if i+1 >= len(args) {
-		if vs, shash, ok = tokenval(vs); !ok || shash == "" {
+				return retwerr(errInvalidNumberOfArguments)
-			err = errInvalidNumberOfArguments
-			return
 			}
+			shash := args[i+1]
+			i += 1
 			lat, lon := geohash.Decode(shash)
-		d.obj = geojson.NewPoint(geometry.Point{X: lon, Y: lat})
+			obj = geojson.NewPoint(geometry.Point{X: lon, Y: lat})
-	case lcb(typ, "object"):
+		case "object":
-		var object string
+			if i+1 >= len(args) {
-		if vs, object, ok = tokenval(vs); !ok || object == "" {
+				return retwerr(errInvalidNumberOfArguments)
-			err = errInvalidNumberOfArguments
-			return
 			}
-		d.obj, err = geojson.Parse(object, &s.geomParseOpts)
+			json := args[i+1]
+			i += 1
+			var err error
+			obj, err = geojson.Parse(json, &s.geomParseOpts)
 			if err != nil {
-			return
+				return retwerr(err)
+			}
+		default:
+			return retwerr(errInvalidArgument(args[i]))
 		}
 	}
-	if len(vs) != 0 {
-		err = errInvalidNumberOfArguments
-	}
-	return
-}
 
-func (s *Server) cmdSet(msg *Message) (res resp.Value, d commandDetails, err error) {
+	// >> Operation
-	if s.config.maxMemory() > 0 && s.outOfMemory.on() {
+
-		err = errOOM
+	var nada bool
-		return
+	col, ok := s.cols.Get(key)
-	}
+	if !ok {
-	start := time.Now()
-	vs := msg.Args[1:]
-	var fmap map[string]int
-	var fields []string
-	var values []float64
-	var xx, nx bool
-	var ex int64
-	d, fields, values, xx, nx, ex, _, _, err = s.parseSetArgs(vs)
-	if err != nil {
-		return
-	}
-	col, _ := s.cols.Get(d.key)
-	if col == nil {
 		if xx {
-			goto notok
+			nada = true
-		}
+		} else {
 			col = collection.New()
-		s.cols.Set(d.key, col)
+			s.cols.Set(key, col)
 		}
+	}
+
+	var ofields field.List
+	if !nada {
+		_, ofields, _, ok = col.Get(id)
 		if xx || nx {
-		_, _, _, ok := col.Get(d.id)
 			if (nx && ok) || (xx && !ok) {
-			goto notok
+				nada = true
 			}
 		}
-	d.oldObj, d.oldFields, d.fields = col.Set(d.id, d.obj, fields, values, ex)
-	d.command = "set"
-	d.updated = true // perhaps we should do a diff on the previous object?
-	d.timestamp = time.Now()
-	if msg.ConnType != Null || msg.OutputType != Null {
-		// likely loaded from aof at server startup, ignore field remapping.
-		fmap = col.FieldMap()
-		d.fmap = make(map[string]int)
-		for key, idx := range fmap {
-			d.fmap[key] = idx
 	}
-	}
+
-	// if ex != nil {
+	if nada {
-	// 	server.expireAt(d.key, d.id, d.timestamp.Add(time.Duration(float64(time.Second)*(*ex))))
+		// exclude operation due to 'xx' or 'nx' match
-	// }
 		switch msg.OutputType {
 		default:
 		case JSON:
-		res = resp.StringValue(`{"ok":true,"elapsed":"` + time.Since(start).String() + "\"}")
+			if nx {
+				return retwerr(errIDAlreadyExists)
+			} else {
+				return retwerr(errIDNotFound)
+			}
+		case RESP:
+			return resp.NullValue(), commandDetails{}, nil
+		}
+		return retwerr(errors.New("nada unknown output"))
+	}
+
+	for _, f := range fields {
+		ofields = ofields.Set(f)
+	}
+
+	oldObj, oldFields, newFields := col.Set(id, obj, ofields, ex)
+
+	// >> Response
+
+	var d commandDetails
+	d.command = "set"
+	d.key = key
+	d.id = id
+	d.obj = obj
+	d.oldObj = oldObj
+	d.oldFields = oldFields
+	d.fields = newFields
+	d.updated = true // perhaps we should do a diff on the previous object?
+	d.timestamp = time.Now()
+
+	var res resp.Value
+	switch msg.OutputType {
+	default:
+	case JSON:
+		res = resp.StringValue(`{"ok":true,"elapsed":"` +
+			time.Since(start).String() + "\"}")
 	case RESP:
 		res = resp.SimpleStringValue("OK")
 	}
-	return
+	return res, d, nil
-notok:
-	switch msg.OutputType {
-	default:
-	case JSON:
-		if nx {
-			err = errIDAlreadyExists
-		} else {
-			err = errIDNotFound
-		}
-		return
-	case RESP:
-		res = resp.NullValue()
-	}
-	return
 }
 
-func (s *Server) parseFSetArgs(vs []string) (
+func retwerr(err error) (resp.Value, commandDetails, error) {
-	d commandDetails, fields []string, values []float64, xx bool, err error,
+	return resp.Value{}, commandDetails{}, err
-) {
+}
-	var ok bool
+func retrerr(err error) (resp.Value, error) {
-	if vs, d.key, ok = tokenval(vs); !ok || d.key == "" {
+	return resp.Value{}, err
-		err = errInvalidNumberOfArguments
-		return
-	}
-	if vs, d.id, ok = tokenval(vs); !ok || d.id == "" {
-		err = errInvalidNumberOfArguments
-		return
-	}
-	for len(vs) > 0 {
-		var name string
-		if vs, name, ok = tokenval(vs); !ok || name == "" {
-			err = errInvalidNumberOfArguments
-			return
-		}
-		if lc(name, "xx") {
-			xx = true
-			continue
-		}
-		if isReservedFieldName(name) {
-			err = errInvalidArgument(name)
-			return
-		}
-		var svalue string
-		var value float64
-		if vs, svalue, ok = tokenval(vs); !ok || svalue == "" {
-			err = errInvalidNumberOfArguments
-			return
-		}
-		value, err = strconv.ParseFloat(svalue, 64)
-		if err != nil {
-			err = errInvalidArgument(svalue)
-			return
-		}
-		fields = append(fields, name)
-		values = append(values, value)
-	}
-	return
 }
 
-func (s *Server) cmdFset(msg *Message) (res resp.Value, d commandDetails, err error) {
+// FSET key id [XX] field value [field value...]
-	if s.config.maxMemory() > 0 && s.outOfMemory.on() {
+func (s *Server) cmdFSET(msg *Message) (resp.Value, commandDetails, error) {
-		err = errOOM
-		return
-	}
 	start := time.Now()
-	vs := msg.Args[1:]
+	if s.config.maxMemory() > 0 && s.outOfMemory.on() {
-	var fields []string
+		return retwerr(errOOM)
-	var values []float64
+	}
-	var xx bool
-	var updateCount int
-	d, fields, values, xx, err = s.parseFSetArgs(vs)
 
-	col, _ := s.cols.Get(d.key)
+	// >> Args
-	if col == nil {
+
-		err = errKeyNotFound
+	var id string
-		return
+	var key string
+	var xx bool
+	var fields []field.Field // raw fields
+
+	args := msg.Args
+	if len(args) < 5 {
+		return retwerr(errInvalidNumberOfArguments)
 	}
-	var ok bool
+	key, id = args[1], args[2]
-	d.obj, d.fields, updateCount, ok = col.SetFields(d.id, fields, values)
+	for i := 3; i < len(args); i++ {
+		arg := strings.ToLower(args[i])
+		switch arg {
+		case "xx":
+			xx = true
+		default:
+			fkey := arg
+			i++
+			if i == len(args) {
+				return retwerr(errInvalidNumberOfArguments)
+			}
+			if isReservedFieldName(fkey) {
+				return retwerr(errInvalidArgument(fkey))
+			}
+			fval := args[i]
+			fields = append(fields, field.Make(fkey, fval))
+		}
+	}
+
+	// >> Operation
+
+	var d commandDetails
+	var updateCount int
+
+	col, ok := s.cols.Get(key)
+	if !ok {
+		return retwerr(errKeyNotFound)
+	}
+	obj, ofields, ex, ok := col.Get(id)
 	if !(ok || xx) {
-		err = errIDNotFound
+		return retwerr(errIDNotFound)
-		return
 	}
+
 	if ok {
+		for _, f := range fields {
+			prev := ofields.Get(f.Name())
+			if !prev.Value().Equals(f.Value()) {
+				ofields = ofields.Set(f)
+				updateCount++
+			}
+		}
+		col.Set(id, obj, ofields, ex)
+		d.obj = obj
 		d.command = "fset"
+		d.key = key
+		d.id = id
 		d.timestamp = time.Now()
 		d.updated = updateCount > 0
-		fmap := col.FieldMap()
-		d.fmap = make(map[string]int)
-		for key, idx := range fmap {
-			d.fmap[key] = idx
-		}
 	}
 
+	// >> Response
+
+	var res resp.Value
+
 	switch msg.OutputType {
 	case JSON:
-		res = resp.StringValue(`{"ok":true,"elapsed":"` + time.Since(start).String() + "\"}")
+		res = resp.StringValue(`{"ok":true,"elapsed":"` +
+			time.Since(start).String() + "\"}")
 	case RESP:
 		res = resp.IntegerValue(updateCount)
 	}
-	return
+
+	return res, d, nil
 }
 
-func (s *Server) cmdExpire(msg *Message) (res resp.Value, d commandDetails, err error) {
+// EXPIRE key id seconds
+func (s *Server) cmdEXPIRE(msg *Message) (resp.Value, commandDetails, error) {
 	start := time.Now()
-	vs := msg.Args[1:]
+	args := msg.Args
-	var key, id, svalue string
+	if len(args) != 4 {
-	var ok bool
+		return retwerr(errInvalidNumberOfArguments)
-	if vs, key, ok = tokenval(vs); !ok || key == "" {
-		err = errInvalidNumberOfArguments
-		return
 	}
-	if vs, id, ok = tokenval(vs); !ok || id == "" {
+	key, id, svalue := args[1], args[2], args[3]
-		err = errInvalidNumberOfArguments
+	value, err := strconv.ParseFloat(svalue, 64)
-		return
-	}
-	if vs, svalue, ok = tokenval(vs); !ok || svalue == "" {
-		err = errInvalidNumberOfArguments
-		return
-	}
-	if len(vs) != 0 {
-		err = errInvalidNumberOfArguments
-		return
-	}
-	var value float64
-	value, err = strconv.ParseFloat(svalue, 64)
 	if err != nil {
-		err = errInvalidArgument(svalue)
+		return retwerr(errInvalidArgument(svalue))
-		return
 	}
-	ok = false
+	var ok bool
 	col, _ := s.cols.Get(key)
 	if col != nil {
+		// replace the expiration by getting the old objec
 		ex := time.Now().Add(time.Duration(float64(time.Second) * value)).UnixNano()
-		ok = col.SetExpires(id, ex)
+		var obj geojson.Object
-	}
+		var fields field.List
+		obj, fields, _, ok = col.Get(id)
 		if ok {
-		d.updated = true
+			col.Set(id, obj, fields, ex)
 		}
+	}
+	var d commandDetails
+	if ok {
+		d.key = key
+		d.id = id
+		d.command = "expire"
+		d.updated = true
+		d.timestamp = time.Now()
+	}
+	var res resp.Value
 	switch msg.OutputType {
 	case JSON:
 		if ok {
-			res = resp.StringValue(`{"ok":true,"elapsed":"` + time.Since(start).String() + "\"}")
+			res = resp.StringValue(`{"ok":true,"elapsed":"` +
+				time.Since(start).String() + "\"}")
+		} else if col == nil {
+			return retwerr(errKeyNotFound)
 		} else {
-			return resp.SimpleStringValue(""), d, errIDNotFound
+			return retwerr(errIDNotFound)
 		}
 	case RESP:
 		if ok {
@@ -969,48 +898,55 @@ func (s *Server) cmdExpire(msg *Message) (res resp.Value, d commandDetails, err
 			res = resp.IntegerValue(0)
 		}
 	}
-	return
+	return res, d, nil
 }
 
-func (s *Server) cmdPersist(msg *Message) (res resp.Value, d commandDetails, err error) {
+// PERSIST key id
+func (s *Server) cmdPERSIST(msg *Message) (resp.Value, commandDetails, error) {
 	start := time.Now()
-	vs := msg.Args[1:]
+	args := msg.Args
-	var key, id string
+	if len(args) != 3 {
-	var ok bool
+		return retwerr(errInvalidNumberOfArguments)
-	if vs, key, ok = tokenval(vs); !ok || key == "" {
-		err = errInvalidNumberOfArguments
-		return
-	}
-	if vs, id, ok = tokenval(vs); !ok || id == "" {
-		err = errInvalidNumberOfArguments
-		return
-	}
-	if len(vs) != 0 {
-		err = errInvalidNumberOfArguments
-		return
 	}
+	key, id := args[1], args[2]
 	var cleared bool
-	ok = false
+	var ok bool
 	col, _ := s.cols.Get(key)
 	if col != nil {
 		var ex int64
 		_, _, ex, ok = col.Get(id)
 		if ok && ex != 0 {
-			ok = col.SetExpires(id, 0)
+			var obj geojson.Object
+			var fields field.List
+			obj, fields, _, ok = col.Get(id)
+			if ok {
+				col.Set(id, obj, fields, 0)
+			}
 			if ok {
 				cleared = true
 			}
 		}
 	}
+
 	if !ok {
 		if msg.OutputType == RESP {
-			return resp.IntegerValue(0), d, nil
+			return resp.IntegerValue(0), commandDetails{}, nil
 		}
-		return resp.SimpleStringValue(""), d, errIDNotFound
+		if col == nil {
+			return retwerr(errKeyNotFound)
 		}
+		return retwerr(errIDNotFound)
+	}
+
+	var res resp.Value
+
+	var d commandDetails
+	d.key = key
+	d.id = id
 	d.command = "persist"
 	d.updated = cleared
 	d.timestamp = time.Now()
+
 	switch msg.OutputType {
 	case JSON:
 		res = resp.SimpleStringValue(`{"ok":true,"elapsed":"` + time.Since(start).String() + "\"}")
@@ -1021,28 +957,19 @@ func (s *Server) cmdPersist(msg *Message) (res resp.Value, d commandDetails, err
 			res = resp.IntegerValue(0)
 		}
 	}
-	return
+	return res, d, nil
 }
 
-func (s *Server) cmdTTL(msg *Message) (res resp.Value, err error) {
+// TTL key id
+func (s *Server) cmdTTL(msg *Message) (resp.Value, error) {
 	start := time.Now()
-	vs := msg.Args[1:]
+	args := msg.Args
-	var key, id string
+	if len(args) != 3 {
-	var ok bool
+		return retrerr(errInvalidNumberOfArguments)
-	if vs, key, ok = tokenval(vs); !ok || key == "" {
-		err = errInvalidNumberOfArguments
-		return
-	}
-	if vs, id, ok = tokenval(vs); !ok || id == "" {
-		err = errInvalidNumberOfArguments
-		return
-	}
-	if len(vs) != 0 {
-		err = errInvalidNumberOfArguments
-		return
 	}
+	key, id := args[1], args[2]
 	var v float64
-	ok = false
+	var ok bool
 	var ok2 bool
 	col, _ := s.cols.Get(key)
 	if col != nil {
@@ -1063,6 +990,7 @@ func (s *Server) cmdTTL(msg *Message) (res resp.Value, err error) {
 			}
 		}
 	}
+	var res resp.Value
 	switch msg.OutputType {
 	case JSON:
 		if ok {
@@ -1073,9 +1001,13 @@ func (s *Server) cmdTTL(msg *Message) (res resp.Value, err error) {
 				ttl = "-1"
 			}
 			res = resp.SimpleStringValue(
-				`{"ok":true,"ttl":` + ttl + `,"elapsed":"` + time.Since(start).String() + "\"}")
+				`{"ok":true,"ttl":` + ttl + `,"elapsed":"` +
+					time.Since(start).String() + "\"}")
 		} else {
-			return resp.SimpleStringValue(""), errIDNotFound
+			if col == nil {
+				return retrerr(errKeyNotFound)
+			}
+			return retrerr(errIDNotFound)
 		}
 	case RESP:
 		if ok {
@@ -1088,5 +1020,5 @@ func (s *Server) cmdTTL(msg *Message) (res resp.Value, err error) {
 			res = resp.IntegerValue(-2)
 		}
 	}
-	return
+	return res, nil
 }

internal/server/fence.go

@@ -10,6 +10,7 @@ import (
 	"github.com/tidwall/geojson/geo"
 	"github.com/tidwall/geojson/geometry"
 	"github.com/tidwall/gjson"
+	"github.com/tidwall/tile38/internal/field"
 	"github.com/tidwall/tile38/internal/glob"
 )
 
@@ -87,9 +88,7 @@ func fenceMatch(
 		return nil
 	}
 	if details.command == "fset" {
-		sw.mu.Lock()
 		nofields := sw.nofields
-		sw.mu.Unlock()
 		if nofields {
 			return nil
 		}
@@ -166,9 +165,10 @@ func fenceMatch(
 		}
 	}
 
-	if details.fmap == nil {
+	// TODO: fields
-		return nil
+	// if details.fmap == nil {
-	}
+	// 	return nil
+	// }
 	for {
 		if fence.detect != nil && !fence.detect[detect] {
 			if detect == "enter" {
@@ -183,26 +183,24 @@ func fenceMatch(
 		}
 		break
 	}
-	sw.mu.Lock()
 	var distance float64
 	if fence.distance && fence.obj != nil {
 		distance = details.obj.Distance(fence.obj)
 	}
-	sw.fmap = details.fmap
+	// TODO: fields
+	// sw.fmap = details.fmap
 	sw.fullFields = true
 	sw.msg.OutputType = JSON
 	sw.writeObject(ScanWriterParams{
 		id:         details.id,
 		o:          details.obj,
 		fields:     details.fields,
-		noLock:     true,
 		noTest:     true,
 		distance:   distance,
 		distOutput: fence.distance,
 	})
 
 	if sw.wr.Len() == 0 {
-		sw.mu.Unlock()
 		return nil
 	}
 
@@ -214,7 +212,6 @@ func fenceMatch(
 	if sw.output == outputIDs {
 		res = `{"id":` + string(res) + `}`
 	}
-	sw.mu.Unlock()
 
 	var group string
 	if detect == "enter" {
@@ -300,7 +297,7 @@ func extendRoamMessage(
 			}
 			pattern := match.id + fence.roam.scan
 			iterator := func(
-				oid string, o geojson.Object, fields []float64,
+				oid string, o geojson.Object, fields field.List,
 			) bool {
 				if oid == match.id {
 					return true
@@ -387,7 +384,7 @@ func fenceMatchNearbys(
 		Max: geometry.Point{X: maxLon, Y: maxLat},
 	}
 	col.Intersects(geojson.NewRect(rect), 0, nil, nil, func(
-		id2 string, obj2 geojson.Object, fields []float64,
+		id2 string, obj2 geojson.Object, fields field.List,
 	) bool {
 		var idMatch bool
 		if id2 == id {

internal/server/json.go

@@ -270,7 +270,7 @@ func (s *Server) cmdJset(msg *Message) (res resp.Value, d commandDetails, err er
 	}
 	var json string
 	var geoobj bool
-	o, _, _, ok := col.Get(id)
+	o, fields, _, ok := col.Get(id)
 	if ok {
 		geoobj = objIsSpatial(o)
 		json = o.String()
@@ -290,7 +290,7 @@ func (s *Server) cmdJset(msg *Message) (res resp.Value, d commandDetails, err er
 		nmsg := *msg
 		nmsg.Args = []string{"SET", key, id, "OBJECT", json}
 		// SET key id OBJECT json
-		return s.cmdSet(&nmsg)
+		return s.cmdSET(&nmsg)
 	}
 	if createcol {
 		s.cols.Set(key, col)
@@ -302,7 +302,7 @@ func (s *Server) cmdJset(msg *Message) (res resp.Value, d commandDetails, err er
 	d.timestamp = time.Now()
 	d.updated = true
 
-	col.Set(d.id, d.obj, nil, nil, 0)
+	col.Set(d.id, d.obj, fields, 0)
 	switch msg.OutputType {
 	case JSON:
 		var buf bytes.Buffer
@@ -335,7 +335,7 @@ func (s *Server) cmdJdel(msg *Message) (res resp.Value, d commandDetails, err er
 
 	var json string
 	var geoobj bool
-	o, _, _, ok := col.Get(id)
+	o, fields, _, ok := col.Get(id)
 	if ok {
 		geoobj = objIsSpatial(o)
 		json = o.String()
@@ -358,7 +358,7 @@ func (s *Server) cmdJdel(msg *Message) (res resp.Value, d commandDetails, err er
 		nmsg := *msg
 		nmsg.Args = []string{"SET", key, id, "OBJECT", json}
 		// SET key id OBJECT json
-		return s.cmdSet(&nmsg)
+		return s.cmdSET(&nmsg)
 	}
 
 	d.key = key
@@ -366,8 +366,7 @@ func (s *Server) cmdJdel(msg *Message) (res resp.Value, d commandDetails, err er
 	d.obj = collection.String(json)
 	d.timestamp = time.Now()
 	d.updated = true
-
+	col.Set(d.id, d.obj, fields, 0)
-	col.Set(d.id, d.obj, nil, nil, 0)
 	switch msg.OutputType {
 	case JSON:
 		var buf bytes.Buffer

internal/server/scan.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/tidwall/geojson"
 	"github.com/tidwall/resp"
+	"github.com/tidwall/tile38/internal/field"
 )
 
 func (s *Server) cmdScanArgs(vs []string) (
@@ -54,10 +55,11 @@ func (s *Server) cmdScan(msg *Message) (res resp.Value, err error) {
 	if msg.OutputType == JSON {
 		wr.WriteString(`{"ok":true`)
 	}
-	sw.writeHead()
+	var ierr error
 	if sw.col != nil {
 		if sw.output == outputCount && len(sw.wheres) == 0 &&
-			len(sw.whereins) == 0 && sw.globEverything {
+			len(sw.whereins) == 0 && len(sw.whereevals) == 0 &&
+			sw.globEverything {
 			count := sw.col.Count() - int(args.cursor)
 			if count < 0 {
 				count = 0
@@ -68,28 +70,41 @@ func (s *Server) cmdScan(msg *Message) (res resp.Value, err error) {
 			if limits[0] == "" && limits[1] == "" {
 				sw.col.Scan(args.desc, sw,
 					msg.Deadline,
-					func(id string, o geojson.Object, fields []float64) bool {
+					func(id string, o geojson.Object, fields field.List) bool {
-						return sw.writeObject(ScanWriterParams{
+						keepGoing, err := sw.pushObject(ScanWriterParams{
 							id:     id,
 							o:      o,
 							fields: fields,
 						})
+						if err != nil {
+							ierr = err
+							return false
+						}
+						return keepGoing
 					},
 				)
 			} else {
 				sw.col.ScanRange(limits[0], limits[1], args.desc, sw,
 					msg.Deadline,
-					func(id string, o geojson.Object, fields []float64) bool {
+					func(id string, o geojson.Object, fields field.List) bool {
-						return sw.writeObject(ScanWriterParams{
+						keepGoing, err := sw.pushObject(ScanWriterParams{
 							id:     id,
 							o:      o,
 							fields: fields,
 						})
+						if err != nil {
+							ierr = err
+							return false
+						}
+						return keepGoing
 					},
 				)
 			}
 		}
 	}
+	if ierr != nil {
+		return retrerr(ierr)
+	}
 	sw.writeFoot()
 	if msg.OutputType == JSON {
 		wr.WriteString(`,"elapsed":"` + time.Since(start).String() + "\"}")

internal/server/scanner.go

@@ -5,13 +5,14 @@ import (
 	"errors"
 	"math"
 	"strconv"
-	"sync"
 
 	"github.com/mmcloughlin/geohash"
+	"github.com/tidwall/btree"
 	"github.com/tidwall/geojson"
 	"github.com/tidwall/resp"
 	"github.com/tidwall/tile38/internal/clip"
 	"github.com/tidwall/tile38/internal/collection"
+	"github.com/tidwall/tile38/internal/field"
 	"github.com/tidwall/tile38/internal/glob"
 )
 
@@ -30,15 +31,12 @@ const (
 )
 
 type scanWriter struct {
-	mu             sync.Mutex
 	s              *Server
 	wr             *bytes.Buffer
-	key            string
+	name           string
 	msg            *Message
 	col            *collection.Collection
-	fmap           map[string]int
+	fkeys          btree.Set[string]
-	farr           []string
-	fvals          []float64
 	output         outputT
 	wheres         []whereT
 	whereins       []whereinT
@@ -58,18 +56,15 @@ type scanWriter struct {
 	values         []resp.Value
 	matchValues    bool
 	respOut        resp.Value
-	orgWheres      []whereT
+	filled         []ScanWriterParams
-	orgWhereins    []whereinT
 }
 
-// ScanWriterParams ...
 type ScanWriterParams struct {
 	id              string
 	o               geojson.Object
-	fields          []float64
+	fields          field.List
 	distance        float64
 	distOutput      bool // query or fence requested distance output
-	noLock          bool
 	noTest          bool
 	ignoreGlobMatch bool
 	clip            geojson.Object
@@ -77,7 +72,7 @@ type ScanWriterParams struct {
 }
 
 func (s *Server) newScanWriter(
-	wr *bytes.Buffer, msg *Message, key string, output outputT,
+	wr *bytes.Buffer, msg *Message, name string, output outputT,
 	precision uint64, globs []string, matchValues bool,
 	cursor, limit uint64, wheres []whereT, whereins []whereinT,
 	whereevals []whereevalT, nofields bool,
@@ -99,7 +94,7 @@ func (s *Server) newScanWriter(
 	sw := &scanWriter{
 		s:           s,
 		wr:          wr,
-		key:         key,
+		name:        name,
 		msg:         msg,
 		globs:       globs,
 		limit:       limit,
@@ -114,50 +109,12 @@ func (s *Server) newScanWriter(
 	if len(globs) == 0 || (len(globs) == 1 && globs[0] == "*") {
 		sw.globEverything = true
 	}
-	sw.orgWheres = wheres
+	sw.wheres = wheres
-	sw.orgWhereins = whereins
+	sw.whereins = whereins
-	sw.loadWheres()
+	sw.col, _ = sw.s.cols.Get(sw.name)
 	return sw, nil
 }
 
-func (sw *scanWriter) loadWheres() {
-	sw.fmap = nil
-	sw.farr = nil
-	sw.wheres = nil
-	sw.whereins = nil
-	sw.fvals = nil
-	sw.col, _ = sw.s.cols.Get(sw.key)
-	if sw.col != nil {
-		sw.fmap = sw.col.FieldMap()
-		sw.farr = sw.col.FieldArr()
-		// This fills index value in wheres/whereins
-		// so we don't have to map string field names for each tested object
-		var ok bool
-		if len(sw.orgWheres) > 0 {
-			sw.wheres = make([]whereT, len(sw.orgWheres))
-			for i, where := range sw.orgWheres {
-				if where.index, ok = sw.fmap[where.field]; !ok {
-					where.index = math.MaxInt32
-				}
-				sw.wheres[i] = where
-			}
-		}
-		if len(sw.orgWhereins) > 0 {
-			sw.whereins = make([]whereinT, len(sw.orgWhereins))
-			for i, wherein := range sw.orgWhereins {
-				if wherein.index, ok = sw.fmap[wherein.field]; !ok {
-					wherein.index = math.MaxInt32
-				}
-				sw.whereins[i] = wherein
-			}
-		}
-		if len(sw.farr) > 0 {
-			sw.fvals = make([]float64, len(sw.farr))
-		}
-	}
-
-}
-
 func (sw *scanWriter) hasFieldsOutput() bool {
 	switch sw.output {
 	default:
@@ -167,19 +124,20 @@ func (sw *scanWriter) hasFieldsOutput() bool {
 	}
 }
 
-func (sw *scanWriter) writeHead() {
+func (sw *scanWriter) writeFoot() {
-	sw.mu.Lock()
-	defer sw.mu.Unlock()
 	switch sw.msg.OutputType {
 	case JSON:
-		if len(sw.farr) > 0 && sw.hasFieldsOutput() {
+		if sw.fkeys.Len() > 0 && sw.hasFieldsOutput() {
 			sw.wr.WriteString(`,"fields":[`)
-			for i, field := range sw.farr {
+			var i int
+			sw.fkeys.Scan(func(name string) bool {
 				if i > 0 {
 					sw.wr.WriteByte(',')
 				}
-				sw.wr.WriteString(jsonString(field))
+				sw.wr.WriteString(jsonString(name))
-			}
+				i++
+				return true
+			})
 			sw.wr.WriteByte(']')
 		}
 		switch sw.output {
@@ -198,11 +156,11 @@ func (sw *scanWriter) writeHead() {
 		}
 	case RESP:
 	}
-}
 
-func (sw *scanWriter) writeFoot() {
+	for _, opts := range sw.filled {
-	sw.mu.Lock()
+		sw.writeFilled(opts)
-	defer sw.mu.Unlock()
+	}
+
 	cursor := sw.numberIters
 	if !sw.hitLimit {
 		cursor = 0
@@ -243,100 +201,43 @@ func extractZCoordinate(o geojson.Object) float64 {
 	}
 }
 
-func (sw *scanWriter) fieldMatch(fields []float64, o geojson.Object) (fvals []float64, match bool) {
+func getFieldValue(o geojson.Object, fields field.List, name string) field.Value {
-	var z float64
+	if name == "z" {
-	var gotz bool
+		return field.ValueOf(strconv.FormatFloat(extractZCoordinate(o), 'f', -1, 64))
-	fvals = sw.fvals
+	}
-	if !sw.hasFieldsOutput() || sw.fullFields {
+	f := fields.Get(name)
+	return f.Value()
+}
+
+func (sw *scanWriter) fieldMatch(o geojson.Object, fields field.List) (bool, error) {
 	for _, where := range sw.wheres {
-			if where.field == "z" {
+		if !where.match(getFieldValue(o, fields, where.name)) {
-				if !gotz {
+			return false, nil
-					z = extractZCoordinate(o)
-				}
-				if !where.match(z) {
-					return
-				}
-				continue
-			}
-			var value float64
-			if where.index < len(fields) {
-				value = fields[where.index]
-			}
-			if !where.match(value) {
-				return
 		}
 	}
 	for _, wherein := range sw.whereins {
-			var value float64
+		if !wherein.match(getFieldValue(o, fields, wherein.name)) {
-			if wherein.index < len(fields) {
+			return false, nil
-				value = fields[wherein.index]
-			}
-			if !wherein.match(value) {
-				return
 		}
 	}
+	if len(sw.whereevals) > 0 {
+		fieldsWithNames := make(map[string]field.Value)
+		fieldsWithNames["z"] = field.ValueOf(strconv.FormatFloat(extractZCoordinate(o), 'f', -1, 64))
+		fields.Scan(func(f field.Field) bool {
+			fieldsWithNames[f.Name()] = f.Value()
+			return true
+		})
 		for _, whereval := range sw.whereevals {
-			fieldsWithNames := make(map[string]float64)
+			match, err := whereval.match(fieldsWithNames)
-			for field, idx := range sw.fmap {
+			if err != nil {
-				if idx < len(fields) {
+				return false, err
-					fieldsWithNames[field] = fields[idx]
-				} else {
-					fieldsWithNames[field] = 0
 			}
-			}
+			if !match {
-			if !whereval.match(fieldsWithNames) {
+				return false, nil
-				return
-			}
-		}
-	} else {
-		copy(sw.fvals, fields)
-		// fields might be shorter for this item, need to pad sw.fvals with zeros
-		for i := len(fields); i < len(sw.fvals); i++ {
-			sw.fvals[i] = 0
-		}
-		for _, where := range sw.wheres {
-			if where.field == "z" {
-				if !gotz {
-					z = extractZCoordinate(o)
-				}
-				if !where.match(z) {
-					return
-				}
-				continue
-			}
-			var value float64
-			if where.index < len(sw.fvals) {
-				value = sw.fvals[where.index]
-			}
-			if !where.match(value) {
-				return
-			}
-		}
-		for _, wherein := range sw.whereins {
-			var value float64
-			if wherein.index < len(sw.fvals) {
-				value = sw.fvals[wherein.index]
-			}
-			if !wherein.match(value) {
-				return
-			}
-		}
-		for _, whereval := range sw.whereevals {
-			fieldsWithNames := make(map[string]float64)
-			for field, idx := range sw.fmap {
-				if idx < len(fields) {
-					fieldsWithNames[field] = fields[idx]
-				} else {
-					fieldsWithNames[field] = 0
-				}
-			}
-			if !whereval.match(fieldsWithNames) {
-				return
 			}
 		}
 	}
-	match = true
+	return true, nil
-	return
 }
 
 func (sw *scanWriter) globMatch(id string, o geojson.Object) (ok, keepGoing bool) {
@@ -356,7 +257,6 @@ func (sw *scanWriter) globMatch(id string, o geojson.Object) (ok, keepGoing bool
 		}
 	}
 	return false, true
-
 }
 
 // Increment cursor
@@ -370,38 +270,64 @@ func (sw *scanWriter) Step(n uint64) {
 
 // ok is whether the object passes the test and should be written
 // keepGoing is whether there could be more objects to test
-func (sw *scanWriter) testObject(id string, o geojson.Object, fields []float64) (
+func (sw *scanWriter) testObject(id string, o geojson.Object, fields field.List,
-	ok, keepGoing bool, fieldVals []float64) {
+) (ok, keepGoing bool, err error) {
 	match, kg := sw.globMatch(id, o)
 	if !match {
-		return false, kg, fieldVals
+		return false, kg, nil
 	}
-	nf, ok := sw.fieldMatch(fields, o)
+	ok, err = sw.fieldMatch(o, fields)
-	return ok, true, nf
+	if err != nil {
+		return false, false, err
+	}
+	return ok, true, nil
 }
 
-// id string, o geojson.Object, fields []float64, noLock bool
+func (sw *scanWriter) pushObject(opts ScanWriterParams) (keepGoing bool, err error) {
-func (sw *scanWriter) writeObject(opts ScanWriterParams) bool {
+	keepGoing = true
-	if !opts.noLock {
-		sw.mu.Lock()
-		defer sw.mu.Unlock()
-	}
-
-	keepGoing := true
 	if !opts.noTest {
 		var ok bool
-		ok, keepGoing, _ = sw.testObject(opts.id, opts.o, opts.fields)
+		var err error
+		ok, keepGoing, err = sw.testObject(opts.id, opts.o, opts.fields)
+		if err != nil {
+			return false, err
+		}
 		if !ok {
-			return keepGoing
+			return keepGoing, nil
 		}
 	}
 	sw.count++
 	if sw.output == outputCount {
-		return sw.count < sw.limit
+		return sw.count < sw.limit, nil
 	}
 	if opts.clip != nil {
 		opts.o = clip.Clip(opts.o, opts.clip, &sw.s.geomIndexOpts)
 	}
+	if !sw.fullFields {
+		opts.fields.Scan(func(f field.Field) bool {
+			sw.fkeys.Insert(f.Name())
+			return true
+		})
+	}
+	sw.filled = append(sw.filled, opts)
+	sw.numberItems++
+	if sw.numberItems == sw.limit {
+		sw.hitLimit = true
+		return false, nil
+	}
+	return keepGoing, nil
+}
+
+func (sw *scanWriter) writeObject(opts ScanWriterParams) {
+	n := len(sw.filled)
+	sw.pushObject(opts)
+	if len(sw.filled) > n {
+		sw.writeFilled(sw.filled[len(sw.filled)-1])
+		sw.filled = sw.filled[:n]
+	}
+}
+
+func (sw *scanWriter) writeFilled(opts ScanWriterParams) {
 	switch sw.msg.OutputType {
 	case JSON:
 		var wr bytes.Buffer
@@ -411,41 +337,37 @@ func (sw *scanWriter) writeObject(opts ScanWriterParams) bool {
 		} else {
 			sw.once = true
 		}
-		if sw.hasFieldsOutput() {
+		fieldsOutput := sw.hasFieldsOutput()
-			if sw.fullFields {
+		if fieldsOutput && sw.fullFields {
-				if len(sw.fmap) > 0 {
+			if opts.fields.Len() > 0 {
 				jsfields = `,"fields":{`
 				var i int
-					for field, idx := range sw.fmap {
+				opts.fields.Scan(func(f field.Field) bool {
-						if len(opts.fields) > idx {
+					if !f.Value().IsZero() {
-							if opts.fields[idx] != 0 {
 						if i > 0 {
 							jsfields += `,`
 						}
-								jsfields += jsonString(field) + ":" + strconv.FormatFloat(opts.fields[idx], 'f', -1, 64)
+						jsfields += jsonString(f.Name()) + ":" + f.Value().JSON()
 						i++
 					}
-						}
+					return true
-					}
+				})
 				jsfields += `}`
 			}
-
+		} else if fieldsOutput && sw.fkeys.Len() > 0 && !sw.fullFields {
-			} else if len(sw.farr) > 0 {
 			jsfields = `,"fields":[`
-				for i, name := range sw.farr {
+			var i int
+			sw.fkeys.Scan(func(name string) bool {
 				if i > 0 {
 					jsfields += `,`
 				}
-					j := sw.fmap[name]
+				f := opts.fields.Get(name)
-					if j < len(opts.fields) {
+				jsfields += f.Value().JSON()
-						jsfields += strconv.FormatFloat(opts.fields[j], 'f', -1, 64)
+				i++
-					} else {
+				return true
-						jsfields += "0"
+			})
-					}
-				}
 			jsfields += `]`
 		}
-		}
 		if sw.output == outputIDs {
 			if opts.distOutput || opts.distance > 0 {
 				wr.WriteString(`{"id":` + jsonString(opts.id) +
@@ -467,9 +389,7 @@ func (sw *scanWriter) writeObject(opts ScanWriterParams) bool {
 			case outputBounds:
 				wr.WriteString(`,"bounds":` + string(appendJSONSimpleBounds(nil, opts.o)))
 			}
-
 			wr.WriteString(jsfields)
-
 			if opts.distOutput || opts.distance > 0 {
 				wr.WriteString(`,"distance":` + strconv.FormatFloat(opts.distance, 'f', -1, 64))
 			}
@@ -523,15 +443,17 @@ func (sw *scanWriter) writeObject(opts ScanWriterParams) bool {
 					}),
 				}))
 			}
-
 			if sw.hasFieldsOutput() {
-				fvs := orderFields(sw.fmap, sw.farr, opts.fields)
+				if opts.fields.Len() > 0 {
-				if len(fvs) > 0 {
+					var fvals []resp.Value
-					fvals := make([]resp.Value, 0, len(fvs)*2)
+					var i int
-					for i, fv := range fvs {
+					opts.fields.Scan(func(f field.Field) bool {
-						fvals = append(fvals, resp.StringValue(fv.field), resp.StringValue(strconv.FormatFloat(fv.value, 'f', -1, 64)))
+						if !f.Value().IsZero() {
+							fvals = append(fvals, resp.StringValue(f.Name()), resp.StringValue(f.Value().Data()))
 							i++
 						}
+						return true
+					})
 					vals = append(vals, resp.ArrayValue(fvals))
 				}
 			}
@@ -542,10 +464,4 @@ func (sw *scanWriter) writeObject(opts ScanWriterParams) bool {
 			sw.values = append(sw.values, resp.ArrayValue(vals))
 		}
 	}
-	sw.numberItems++
-	if sw.numberItems == sw.limit {
-		sw.hitLimit = true
-		return false
-	}
-	return keepGoing
 }

internal/server/scanner_test.go

@@ -1,6 +1,7 @@
 package server
 
 import (
+	"fmt"
 	"math"
 	"math/rand"
 	"testing"
@@ -8,11 +9,12 @@ import (
 
 	"github.com/tidwall/geojson"
 	"github.com/tidwall/geojson/geometry"
+	"github.com/tidwall/tile38/internal/field"
 )
 
 type testPointItem struct {
 	object geojson.Object
-	fields []float64
+	fields field.List
 }
 
 func PO(x, y float64) *geojson.Point {
@@ -23,29 +25,29 @@ func BenchmarkFieldMatch(t *testing.B) {
 	rand.Seed(time.Now().UnixNano())
 	items := make([]testPointItem, t.N)
 	for i := 0; i < t.N; i++ {
+		var fields field.List
+		fields = fields.Set(field.Make("foo", fmt.Sprintf("%f", rand.Float64()*9+1)))
+		fields = fields.Set(field.Make("bar", fmt.Sprintf("%f", math.Round(rand.Float64()*30)+1)))
 		items[i] = testPointItem{
 			PO(rand.Float64()*360-180, rand.Float64()*180-90),
-			[]float64{rand.Float64()*9 + 1, math.Round(rand.Float64()*30) + 1},
+			fields,
 		}
 	}
 	sw := &scanWriter{
 		wheres: []whereT{
-			{"foo", 0, false, 1, false, 3},
+			{"foo", false, field.ValueOf("1"), false, field.ValueOf("3")},
-			{"bar", 1, false, 10, false, 30},
+			{"bar", false, field.ValueOf("10"), false, field.ValueOf("30")},
 		},
 		whereins: []whereinT{
-			{"foo", 0, []float64{1, 2}},
+			{"foo", []field.Value{field.ValueOf("1"), field.ValueOf("2")}},
-			{"bar", 1, []float64{11, 25}},
+			{"bar", []field.Value{field.ValueOf("11"), field.ValueOf("25")}},
 		},
-		fmap: map[string]int{"foo": 0, "bar": 1},
-		farr: []string{"bar", "foo"},
 	}
-	sw.fvals = make([]float64, len(sw.farr))
 	t.ResetTimer()
 	for i := 0; i < t.N; i++ {
 		// one call is super fast, measurements are not reliable, let's do 100
 		for ix := 0; ix < 100; ix++ {
-			sw.fieldMatch(items[i].fields, items[i].object)
+			sw.fieldMatch(items[i].object, items[i].fields)
 		}
 	}
 }

internal/server/scripts.go

@@ -592,9 +592,9 @@ func (s *Server) commandInScript(msg *Message) (
 	default:
 		err = fmt.Errorf("unknown command '%s'", msg.Args[0])
 	case "set":
-		res, d, err = s.cmdSet(msg)
+		res, d, err = s.cmdSET(msg)
 	case "fset":
-		res, d, err = s.cmdFset(msg)
+		res, d, err = s.cmdFSET(msg)
 	case "del":
 		res, d, err = s.cmdDel(msg)
 	case "pdel":
@@ -602,13 +602,13 @@ func (s *Server) commandInScript(msg *Message) (
 	case "drop":
 		res, d, err = s.cmdDrop(msg)
 	case "expire":
-		res, d, err = s.cmdExpire(msg)
+		res, d, err = s.cmdEXPIRE(msg)
 	case "rename":
 		res, d, err = s.cmdRename(msg)
 	case "renamenx":
 		res, d, err = s.cmdRename(msg)
 	case "persist":
-		res, d, err = s.cmdPersist(msg)
+		res, d, err = s.cmdPERSIST(msg)
 	case "ttl":
 		res, err = s.cmdTTL(msg)
 	case "stats":
@@ -618,9 +618,9 @@ func (s *Server) commandInScript(msg *Message) (
 	case "nearby":
 		res, err = s.cmdNearby(msg)
 	case "within":
-		res, err = s.cmdWithin(msg)
+		res, err = s.cmdWITHIN(msg)
 	case "intersects":
-		res, err = s.cmdIntersects(msg)
+		res, err = s.cmdINTERSECTS(msg)
 	case "search":
 		res, err = s.cmdSearch(msg)
 	case "bounds":

internal/server/search.go

@@ -16,6 +16,7 @@ import (
 	"github.com/tidwall/tile38/internal/bing"
 	"github.com/tidwall/tile38/internal/buffer"
 	"github.com/tidwall/tile38/internal/clip"
+	"github.com/tidwall/tile38/internal/field"
 	"github.com/tidwall/tile38/internal/glob"
 )
 
@@ -496,19 +497,23 @@ func (s *Server) cmdNearby(msg *Message) (res resp.Value, err error) {
 	if msg.OutputType == JSON {
 		wr.WriteString(`{"ok":true`)
 	}
-	sw.writeHead()
+	var ierr error
 	if sw.col != nil {
-		iterStep := func(id string, o geojson.Object, fields []float64, meters float64) bool {
+		iterStep := func(id string, o geojson.Object, fields field.List, meters float64) bool {
-			return sw.writeObject(ScanWriterParams{
+			keepGoing, err := sw.pushObject(ScanWriterParams{
 				id:              id,
 				o:               o,
 				fields:          fields,
 				distance:        meters,
 				distOutput:      sargs.distance,
-				noLock:          true,
 				ignoreGlobMatch: true,
 				skipTesting:     true,
 			})
+			if err != nil {
+				ierr = err
+				return false
+			}
+			return keepGoing
 		}
 		maxDist := sargs.obj.(*geojson.Circle).Meters()
 		if sargs.sparse > 0 {
@@ -518,7 +523,7 @@ func (s *Server) cmdNearby(msg *Message) (res resp.Value, err error) {
 					errors.New("cannot use SPARSE without a point distance")
 			}
 			// An intersects operation is required for SPARSE
-			iter := func(id string, o geojson.Object, fields []float64) bool {
+			iter := func(id string, o geojson.Object, fields field.List) bool {
 				var meters float64
 				if sargs.distance {
 					meters = o.Distance(sargs.obj)
@@ -527,7 +532,7 @@ func (s *Server) cmdNearby(msg *Message) (res resp.Value, err error) {
 			}
 			sw.col.Intersects(sargs.obj, sargs.sparse, sw, msg.Deadline, iter)
 		} else {
-			iter := func(id string, o geojson.Object, fields []float64, dist float64) bool {
+			iter := func(id string, o geojson.Object, fields field.List, dist float64) bool {
 				if maxDist > 0 && dist > maxDist {
 					return false
 				}
@@ -540,6 +545,9 @@ func (s *Server) cmdNearby(msg *Message) (res resp.Value, err error) {
 			sw.col.Nearby(sargs.obj, sw, msg.Deadline, iter)
 		}
 	}
+	if ierr != nil {
+		return retrerr(ierr)
+	}
 	sw.writeFoot()
 	if msg.OutputType == JSON {
 		wr.WriteString(`,"elapsed":"` + time.Since(start).String() + "\"}")
@@ -548,15 +556,15 @@ func (s *Server) cmdNearby(msg *Message) (res resp.Value, err error) {
 	return sw.respOut, nil
 }
 
-func (s *Server) cmdWithin(msg *Message) (res resp.Value, err error) {
+func (s *Server) cmdWITHIN(msg *Message) (res resp.Value, err error) {
-	return s.cmdWithinOrIntersects("within", msg)
+	return s.cmdWITHINorINTERSECTS("within", msg)
 }
 
-func (s *Server) cmdIntersects(msg *Message) (res resp.Value, err error) {
+func (s *Server) cmdINTERSECTS(msg *Message) (res resp.Value, err error) {
-	return s.cmdWithinOrIntersects("intersects", msg)
+	return s.cmdWITHINorINTERSECTS("intersects", msg)
 }
 
-func (s *Server) cmdWithinOrIntersects(cmd string, msg *Message) (res resp.Value, err error) {
+func (s *Server) cmdWITHINorINTERSECTS(cmd string, msg *Message) (res resp.Value, err error) {
 	start := time.Now()
 	vs := msg.Args[1:]
 
@@ -588,38 +596,49 @@ func (s *Server) cmdWithinOrIntersects(cmd string, msg *Message) (res resp.Value
 	if msg.OutputType == JSON {
 		wr.WriteString(`{"ok":true`)
 	}
-	sw.writeHead()
+	var ierr error
 	if sw.col != nil {
 		if cmd == "within" {
 			sw.col.Within(sargs.obj, sargs.sparse, sw, msg.Deadline, func(
-				id string, o geojson.Object, fields []float64,
+				id string, o geojson.Object, fields field.List,
 			) bool {
-				return sw.writeObject(ScanWriterParams{
+				keepGoing, err := sw.pushObject(ScanWriterParams{
 					id:     id,
 					o:      o,
 					fields: fields,
-					noLock: true,
 				})
+				if err != nil {
+					ierr = err
+					return false
+				}
+				return keepGoing
 			})
 		} else if cmd == "intersects" {
 			sw.col.Intersects(sargs.obj, sargs.sparse, sw, msg.Deadline, func(
 				id string,
 				o geojson.Object,
-				fields []float64,
+				fields field.List,
 			) bool {
 				params := ScanWriterParams{
 					id:     id,
 					o:      o,
 					fields: fields,
-					noLock: true,
 				}
 				if sargs.clip {
 					params.clip = sargs.obj
 				}
-				return sw.writeObject(params)
+				keepGoing, err := sw.pushObject(params)
+				if err != nil {
+					ierr = err
+					return false
+				}
+				return keepGoing
 			})
 		}
 	}
+	if ierr != nil {
+		return retrerr(ierr)
+	}
 	sw.writeFoot()
 	if msg.OutputType == JSON {
 		wr.WriteString(`,"elapsed":"` + time.Since(start).String() + "\"}")
@@ -701,7 +720,7 @@ func (s *Server) cmdSearch(msg *Message) (res resp.Value, err error) {
 	if msg.OutputType == JSON {
 		wr.WriteString(`{"ok":true`)
 	}
-	sw.writeHead()
+	var ierr error
 	if sw.col != nil {
 		if sw.output == outputCount && len(sw.wheres) == 0 && sw.globEverything {
 			count := sw.col.Count() - int(sargs.cursor)
@@ -713,13 +732,17 @@ func (s *Server) cmdSearch(msg *Message) (res resp.Value, err error) {
 			limits := multiGlobParse(sw.globs, sargs.desc)
 			if limits[0] == "" && limits[1] == "" {
 				sw.col.SearchValues(sargs.desc, sw, msg.Deadline,
-					func(id string, o geojson.Object, fields []float64) bool {
+					func(id string, o geojson.Object, fields field.List) bool {
-						return sw.writeObject(ScanWriterParams{
+						keepGoing, err := sw.pushObject(ScanWriterParams{
 							id:     id,
 							o:      o,
 							fields: fields,
-							noLock: true,
 						})
+						if err != nil {
+							ierr = err
+							return false
+						}
+						return keepGoing
 					},
 				)
 			} else {
@@ -727,18 +750,25 @@ func (s *Server) cmdSearch(msg *Message) (res resp.Value, err error) {
 				// globSingle is only for ID matches, not values.
 				sw.col.SearchValuesRange(limits[0], limits[1], sargs.desc, sw,
 					msg.Deadline,
-					func(id string, o geojson.Object, fields []float64) bool {
+					func(id string, o geojson.Object, fields field.List) bool {
-						return sw.writeObject(ScanWriterParams{
+						keepGoing, err := sw.pushObject(ScanWriterParams{
 							id:     id,
 							o:      o,
 							fields: fields,
-							noLock: true,
 						})
+						if err != nil {
+							ierr = err
+							return false
+						}
+						return keepGoing
 					},
 				)
 			}
 		}
 	}
+	if ierr != nil {
+		return retrerr(ierr)
+	}
 	sw.writeFoot()
 	if msg.OutputType == JSON {
 		wr.WriteString(`,"elapsed":"` + time.Since(start).String() + "\"}")

internal/server/server.go

@@ -36,6 +36,7 @@ import (
 	"github.com/tidwall/tile38/internal/collection"
 	"github.com/tidwall/tile38/internal/deadline"
 	"github.com/tidwall/tile38/internal/endpoint"
+	"github.com/tidwall/tile38/internal/field"
 	"github.com/tidwall/tile38/internal/log"
 )
 
@@ -56,11 +57,13 @@ type commandDetails struct {
 	command string // client command, like "SET" or "DEL"
 	key, id string // collection key and object id of object
 	newKey  string // new key, for RENAME command
-	fmap      map[string]int    // map of field names to value indexes
+
 	obj    geojson.Object // new object
-	fields    []float64         // array of field values
+	fields field.List     // array of field values
+
 	oldObj    geojson.Object // previous object, if any
-	oldFields []float64         // previous object field values
+	oldFields field.List     // previous object field values
+
 	updated   bool              // object was updated
 	timestamp time.Time         // timestamp when the update occured
 	parent    bool              // when true, only children are forwarded
@@ -1016,9 +1019,9 @@ func (s *Server) command(msg *Message, client *Client) (
 	default:
 		err = fmt.Errorf("unknown command '%s'", msg.Args[0])
 	case "set":
-		res, d, err = s.cmdSet(msg)
+		res, d, err = s.cmdSET(msg)
 	case "fset":
-		res, d, err = s.cmdFset(msg)
+		res, d, err = s.cmdFSET(msg)
 	case "del":
 		res, d, err = s.cmdDel(msg)
 	case "pdel":
@@ -1026,7 +1029,7 @@ func (s *Server) command(msg *Message, client *Client) (
 	case "drop":
 		res, d, err = s.cmdDrop(msg)
 	case "flushdb":
-		res, d, err = s.cmdFlushDB(msg)
+		res, d, err = s.cmdFLUSHDB(msg)
 	case "rename":
 		res, d, err = s.cmdRename(msg)
 	case "renamenx":
@@ -1048,9 +1051,9 @@ func (s *Server) command(msg *Message, client *Client) (
 	case "chans":
 		res, err = s.cmdHooks(msg)
 	case "expire":
-		res, d, err = s.cmdExpire(msg)
+		res, d, err = s.cmdEXPIRE(msg)
 	case "persist":
-		res, d, err = s.cmdPersist(msg)
+		res, d, err = s.cmdPERSIST(msg)
 	case "ttl":
 		res, err = s.cmdTTL(msg)
 	case "shutdown":
@@ -1090,9 +1093,9 @@ func (s *Server) command(msg *Message, client *Client) (
 	case "nearby":
 		res, err = s.cmdNearby(msg)
 	case "within":
-		res, err = s.cmdWithin(msg)
+		res, err = s.cmdWITHIN(msg)
 	case "intersects":
-		res, err = s.cmdIntersects(msg)
+		res, err = s.cmdINTERSECTS(msg)
 	case "search":
 		res, err = s.cmdSearch(msg)
 	case "bounds":

internal/server/token.go

@@ -7,6 +7,7 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/tidwall/tile38/internal/field"
 	lua "github.com/yuin/gopher-lua"
 )
 
@@ -44,31 +45,6 @@ func tokenval(vs []string) (nvs []string, token string, ok bool) {
 	return
 }
 
-func tokenvalbytes(vs []string) (nvs []string, token []byte, ok bool) {
-	if len(vs) > 0 {
-		token = []byte(vs[0])
-		nvs = vs[1:]
-		ok = true
-	}
-	return
-}
-
-func lcb(s1 []byte, s2 string) bool {
-	if len(s1) != len(s2) {
-		return false
-	}
-	for i := 0; i < len(s1); i++ {
-		ch := s1[i]
-		if ch >= 'A' && ch <= 'Z' {
-			if ch+32 != s2[i] {
-				return false
-			}
-		} else if ch != s2[i] {
-			return false
-		}
-	}
-	return true
-}
 func lc(s1, s2 string) bool {
 	if len(s1) != len(s2) {
 		return false
@@ -87,30 +63,35 @@ func lc(s1, s2 string) bool {
 }
 
 type whereT struct {
-	field string
+	name string
-	index int
 	minx bool
-	min   float64
+	min  field.Value
 	maxx bool
-	max   float64
+	max  field.Value
 }
 
-func (where whereT) match(value float64) bool {
+func mLT(a, b field.Value) bool  { return a.Less(b) }
+func mLTE(a, b field.Value) bool { return !mLT(b, a) }
+func mGT(a, b field.Value) bool  { return mLT(b, a) }
+func mGTE(a, b field.Value) bool { return !mLT(a, b) }
+func mEQ(a, b field.Value) bool  { return a.Equals(b) }
+
+func (where whereT) match(value field.Value) bool {
 	if !where.minx {
-		if value < where.min {
+		if mLT(value, where.min) { // if value < where.min {
 			return false
 		}
 	} else {
-		if value <= where.min {
+		if mLTE(value, where.min) { // if value <= where.min {
 			return false
 		}
 	}
 	if !where.maxx {
-		if value > where.max {
+		if mGT(value, where.max) { // if value > where.max {
 			return false
 		}
 	} else {
-		if value >= where.max {
+		if mGTE(value, where.max) { // if value >= where.max {
 			return false
 		}
 	}
@@ -118,14 +99,13 @@ func (where whereT) match(value float64) bool {
 }
 
 type whereinT struct {
-	field  string
+	name   string
-	index  int
+	valArr []field.Value
-	valArr []float64
 }
 
-func (wherein whereinT) match(value float64) bool {
+func (wherein whereinT) match(value field.Value) bool {
 	for _, val := range wherein.valArr {
-		if val == value {
+		if mEQ(val, value) {
 			return true
 		}
 	}
@@ -146,12 +126,28 @@ func (whereeval whereevalT) Close() {
 	whereeval.c.luapool.Put(whereeval.luaState)
 }
 
-func (whereeval whereevalT) match(fieldsWithNames map[string]float64) bool {
+func luaSetField(tbl *lua.LTable, name string, val field.Value) {
-	fieldsTbl := whereeval.luaState.CreateTable(0, len(fieldsWithNames))
+	var lval lua.LValue
-	for field, val := range fieldsWithNames {
+	switch val.Kind() {
-		fieldsTbl.RawSetString(field, lua.LNumber(val))
+	case field.Null:
+		lval = lua.LNil
+	case field.False:
+		lval = lua.LFalse
+	case field.True:
+		lval = lua.LTrue
+	case field.Number:
+		lval = lua.LNumber(val.Num())
+	default:
+		lval = lua.LString(val.Data())
 	}
+	tbl.RawSetString(name, lval)
+}
 
+func (whereeval whereevalT) match(fieldsWithNames map[string]field.Value) (bool, error) {
+	fieldsTbl := whereeval.luaState.CreateTable(0, len(fieldsWithNames))
+	for name, val := range fieldsWithNames {
+		luaSetField(fieldsTbl, name, val)
+	}
 	luaSetRawGlobals(
 		whereeval.luaState, map[string]lua.LValue{
 			"FIELDS": fieldsTbl,
@@ -163,7 +159,7 @@ func (whereeval whereevalT) match(fieldsWithNames map[string]float64) bool {
 
 	whereeval.luaState.Push(whereeval.fn)
 	if err := whereeval.luaState.PCall(0, 1, nil); err != nil {
-		panic(err.Error())
+		return false, err
 	}
 	ret := whereeval.luaState.Get(-1)
 	whereeval.luaState.Pop(1)
@@ -171,23 +167,23 @@ func (whereeval whereevalT) match(fieldsWithNames map[string]float64) bool {
 	// Make bool out of returned lua value
 	switch ret.Type() {
 	case lua.LTNil:
-		return false
+		return false, nil
 	case lua.LTBool:
-		return ret == lua.LTrue
+		return ret == lua.LTrue, nil
 	case lua.LTNumber:
-		return float64(ret.(lua.LNumber)) != 0
+		return float64(ret.(lua.LNumber)) != 0, nil
 	case lua.LTString:
-		return ret.String() != ""
+		return ret.String() != "", nil
 	case lua.LTTable:
 		tbl := ret.(*lua.LTable)
 		if tbl.Len() != 0 {
-			return true
+			return true, nil
 		}
 		var match bool
 		tbl.ForEach(func(lk lua.LValue, lv lua.LValue) { match = true })
-		return match
+		return match, nil
 	}
-	panic(fmt.Sprintf("Script returned value of type %s", ret.Type()))
+	return false, fmt.Errorf("script returned value of type %s", ret.Type())
 }
 
 type searchScanBaseTokens struct {
@@ -265,57 +261,54 @@ func (s *Server) parseSearchScanBaseTokens(
 				continue
 			case "where":
 				vs = nvs
-				var field, smin, smax string
+				var name, smin, smax string
-				if vs, field, ok = tokenval(vs); !ok || field == "" {
+				if vs, name, ok = tokenval(vs); !ok {
 					err = errInvalidNumberOfArguments
 					return
 				}
-				if vs, smin, ok = tokenval(vs); !ok || smin == "" {
+				if vs, smin, ok = tokenval(vs); !ok {
 					err = errInvalidNumberOfArguments
 					return
 				}
-				if vs, smax, ok = tokenval(vs); !ok || smax == "" {
+				if vs, smax, ok = tokenval(vs); !ok {
 					err = errInvalidNumberOfArguments
 					return
 				}
 				var minx, maxx bool
-				var min, max float64
+				smin = strings.ToLower(smin)
-				if strings.ToLower(smin) == "-inf" {
+				if smin == "-inf" {
-					min = math.Inf(-1)
+					smin = "-inf"
 				} else {
 					if strings.HasPrefix(smin, "(") {
 						minx = true
 						smin = smin[1:]
 					}
-					min, err = strconv.ParseFloat(smin, 64)
-					if err != nil {
-						err = errInvalidArgument(smin)
-						return
 				}
-				}
+				smax = strings.ToLower(smax)
-				if strings.ToLower(smax) == "+inf" {
+				if smax == "+inf" || smax == "inf" {
-					max = math.Inf(+1)
+					smax = "inf"
 				} else {
 					if strings.HasPrefix(smax, "(") {
 						maxx = true
 						smax = smax[1:]
 					}
-					max, err = strconv.ParseFloat(smax, 64)
-					if err != nil {
-						err = errInvalidArgument(smax)
-						return
 				}
-				}
+				t.wheres = append(t.wheres, whereT{
-				t.wheres = append(t.wheres, whereT{field, -1, minx, min, maxx, max})
+					name: strings.ToLower(name),
+					minx: minx,
+					min:  field.ValueOf(smin),
+					maxx: maxx,
+					max:  field.ValueOf(smax),
+				})
 				continue
 			case "wherein":
 				vs = nvs
-				var field, nvalsStr, valStr string
+				var name, nvalsStr, valStr string
-				if vs, field, ok = tokenval(vs); !ok || field == "" {
+				if vs, name, ok = tokenval(vs); !ok {
 					err = errInvalidNumberOfArguments
 					return
 				}
-				if vs, nvalsStr, ok = tokenval(vs); !ok || nvalsStr == "" {
+				if vs, nvalsStr, ok = tokenval(vs); !ok {
 					err = errInvalidNumberOfArguments
 					return
 				}
@@ -324,20 +317,18 @@ func (s *Server) parseSearchScanBaseTokens(
 					err = errInvalidArgument(nvalsStr)
 					return
 				}
-				valArr := make([]float64, nvals)
+				valArr := make([]field.Value, nvals)
-				var val float64
 				for i = 0; i < nvals; i++ {
-					if vs, valStr, ok = tokenval(vs); !ok || valStr == "" {
+					if vs, valStr, ok = tokenval(vs); !ok {
 						err = errInvalidNumberOfArguments
 						return
 					}
-					if val, err = strconv.ParseFloat(valStr, 64); err != nil {
+					valArr[i] = field.ValueOf(valStr)
-						err = errInvalidArgument(valStr)
-						return
 				}
-					valArr[i] = val
+				t.whereins = append(t.whereins, whereinT{
-				}
+					name:   strings.ToLower(name),
-				t.whereins = append(t.whereins, whereinT{field, -1, valArr})
+					valArr: valArr,
+				})
 				continue
 			case "whereevalsha":
 				fallthrough
@@ -409,7 +400,9 @@ func (s *Server) parseSearchScanBaseTokens(
 					}
 					s.luascripts.Put(shaSum, fn.Proto)
 				}
-				t.whereevals = append(t.whereevals, whereevalT{s, luaState, fn})
+				t.whereevals = append(t.whereevals, whereevalT{
+					c: s, luaState: luaState, fn: fn,
+				})
 				continue
 			case "nofields":
 				vs = nvs

internal/sstring/sstring.go

@@ -0,0 +1,54 @@
+// Package shared allows for
+package sstring
+
+import (
+	"sync"
+	"unsafe"
+
+	"github.com/tidwall/hashmap"
+)
+
+var mu sync.Mutex
+var nums hashmap.Map[string, int]
+var strs []string
+
+// Load a shared string from its number.
+// Panics when there is no string assigned with that number.
+func Load(num int) (str string) {
+	mu.Lock()
+	if num >= 0 && num < len(strs) {
+		str = strs[num]
+		mu.Unlock()
+		return str
+	}
+	mu.Unlock()
+	panic("string not found")
+}
+
+// Store a shared string.
+// Returns a unique number that can be used to load the string later.
+// The number is al
+func Store(str string) (num int) {
+	mu.Lock()
+	var ok bool
+	num, ok = nums.Get(str)
+	if !ok {
+		// Make a copy of the string to ensure we don't take in slices.
+		b := make([]byte, len(str))
+		copy(b, str)
+		str = *(*string)(unsafe.Pointer(&b))
+		num = len(strs)
+		strs = append(strs, str)
+		nums.Set(str, num)
+	}
+	mu.Unlock()
+	return num
+}
+
+// Len returns the number of shared strings
+func Len() int {
+	mu.Lock()
+	n := len(strs)
+	mu.Unlock()
+	return n
+}

internal/sstring/sstring_test.go

@@ -0,0 +1,85 @@
+package sstring
+
+import (
+	"math/rand"
+	"testing"
+	"time"
+
+	"github.com/tidwall/assert"
+)
+
+func TestShared(t *testing.T) {
+	for i := -1; i < 10; i++ {
+		var str string
+		func() {
+			defer func() {
+				assert.Assert(recover().(string) == "string not found")
+			}()
+			str = Load(i)
+		}()
+		assert.Assert(str == "")
+	}
+	assert.Assert(Store("hello") == 0)
+	assert.Assert(Store("") == 1)
+	assert.Assert(Store("jello") == 2)
+	assert.Assert(Store("hello") == 0)
+	assert.Assert(Store("") == 1)
+	assert.Assert(Store("jello") == 2)
+	str := Load(0)
+	assert.Assert(str == "hello")
+	str = Load(1)
+	assert.Assert(str == "")
+	str = Load(2)
+	assert.Assert(str == "jello")
+
+	assert.Assert(Len() == 3)
+
+}
+
+func randStr(n int) string {
+	b := make([]byte, n)
+	rand.Read(b)
+	for i := 0; i < n; i++ {
+		b[i] = 'a' + b[i]%26
+	}
+	return string(b)
+}
+
+func BenchmarkStore(b *testing.B) {
+	rand.Seed(time.Now().UnixNano())
+	wmap := make(map[string]bool, b.N)
+	for len(wmap) < b.N {
+		wmap[randStr(10)] = true
+	}
+	words := make([]string, 0, b.N)
+	for word := range wmap {
+		words = append(words, word)
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		Store(words[i])
+	}
+}
+
+func BenchmarkLoad(b *testing.B) {
+	rand.Seed(time.Now().UnixNano())
+	wmap := make(map[string]bool, b.N)
+	for len(wmap) < b.N {
+		wmap[randStr(10)] = true
+	}
+	words := make([]string, 0, b.N)
+	for word := range wmap {
+		words = append(words, word)
+	}
+	var nums []int
+	for i := 0; i < b.N; i++ {
+		nums = append(nums, Store(words[i]))
+	}
+	rand.Shuffle(len(nums), func(i, j int) {
+		nums[i], nums[j] = nums[j], nums[i]
+	})
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		Load(nums[i])
+	}
+}

tests/keys_search_test.go

@@ -740,9 +740,9 @@ func keys_FIELDS_search_test(mc *mockServer) error {
 				`{"id":"5","object":{"type":"Point","coordinates":[-112.2799,33.5228]},"fields":[0,15,28]}` +
 				`],"count":4,"cursor":0}`},
 		{"NEARBY", "mykey", "WHERE", "field2", 0, 2, "POINT", 33.462, -112.268, 60000}, {
-			`{"ok":true,"fields":["field1","field2","field3"],"objects":[` +
+			`{"ok":true,"fields":["field3"],"objects":[` +
-				`{"id":"6","object":{"type":"Point","coordinates":[-112.2801,33.523]},"fields":[0,0,29]},` +
+				`{"id":"6","object":{"type":"Point","coordinates":[-112.2801,33.523]},"fields":[29]},` +
-				`{"id":"7","object":{"type":"Point","coordinates":[-112.2803,33.5232]},"fields":[0,0,0]}` +
+				`{"id":"7","object":{"type":"Point","coordinates":[-112.2803,33.5232]},"fields":[0]}` +
 				`],"count":2,"cursor":0}`},
 
 		{"WITHIN", "mykey", "WHERE", "field2", 11, "+inf", "CIRCLE", 33.462, -112.268, 60000}, {
@@ -753,9 +753,9 @@ func keys_FIELDS_search_test(mc *mockServer) error {
 				`{"id":"1","object":{"type":"Point","coordinates":[-112.2791,33.522]},"fields":[10,11,0]}` +
 				`],"count":4,"cursor":0}`},
 		{"WITHIN", "mykey", "WHERE", "field2", 0, 2, "CIRCLE", 33.462, -112.268, 60000}, {
-			`{"ok":true,"fields":["field1","field2","field3"],"objects":[` +
+			`{"ok":true,"fields":["field3"],"objects":[` +
-				`{"id":"7","object":{"type":"Point","coordinates":[-112.2803,33.5232]},"fields":[0,0,0]},` +
+				`{"id":"7","object":{"type":"Point","coordinates":[-112.2803,33.5232]},"fields":[0]},` +
-				`{"id":"6","object":{"type":"Point","coordinates":[-112.2801,33.523]},"fields":[0,0,29]}` +
+				`{"id":"6","object":{"type":"Point","coordinates":[-112.2801,33.523]},"fields":[29]}` +
 				`],"count":2,"cursor":0}`},
 	})
 }

tests/keys_test.go

@@ -352,8 +352,8 @@ func keys_FIELDS_test(mc *mockServer) error {
 	return mc.DoBatch([][]interface{}{
 		{"SET", "mykey", "myid1a", "FIELD", "a", 1, "POINT", 33, -115}, {"OK"},
 		{"GET", "mykey", "myid1a", "WITHFIELDS"}, {`[{"type":"Point","coordinates":[-115,33]} [a 1]]`},
-		{"SET", "mykey", "myid1a", "FIELD", "a", "a", "POINT", 33, -115}, {"ERR invalid argument 'a'"},
+		{"SET", "mykey", "myid1a", "FIELD", "a", "a", "POINT", 33, -115}, {"OK"},
-		{"GET", "mykey", "myid1a", "WITHFIELDS"}, {`[{"type":"Point","coordinates":[-115,33]} [a 1]]`},
+		{"GET", "mykey", "myid1a", "WITHFIELDS"}, {`[{"type":"Point","coordinates":[-115,33]} [a a]]`},
 		{"SET", "mykey", "myid1a", "FIELD", "a", 1, "FIELD", "b", 2, "POINT", 33, -115}, {"OK"},
 		{"GET", "mykey", "myid1a", "WITHFIELDS"}, {`[{"type":"Point","coordinates":[-115,33]} [a 1 b 2]]`},
 		{"SET", "mykey", "myid1a", "FIELD", "b", 2, "POINT", 33, -115}, {"OK"},
@@ -390,7 +390,8 @@ func keys_WHEREIN_test(mc *mockServer) error {
 		{"WITHIN", "mykey", "WHEREIN", "a", 3, 0, 1, 2, "BOUNDS", 32.8, -115.2, 33.2, -114.8}, {`[0 [[myid_a1 {"type":"Point","coordinates":[-115,33]} [a 1]]]]`},
 		{"WITHIN", "mykey", "WHEREIN", "a", "a", 0, 1, 2, "BOUNDS", 32.8, -115.2, 33.2, -114.8}, {"ERR invalid argument 'a'"},
 		{"WITHIN", "mykey", "WHEREIN", "a", 1, 0, 1, 2, "BOUNDS", 32.8, -115.2, 33.2, -114.8}, {"ERR invalid argument '1'"},
-		{"WITHIN", "mykey", "WHEREIN", "a", 3, 0, "a", 2, "BOUNDS", 32.8, -115.2, 33.2, -114.8}, {"ERR invalid argument 'a'"},
+		{"WITHIN", "mykey", "WHEREIN", "a", 3, 0, "a", 2, "BOUNDS", 32.8, -115.2, 33.2, -114.8}, {"[0 []]"},
+		{"WITHIN", "mykey", "WHEREIN", "a", 4, 0, "a", 1, 2, "BOUNDS", 32.8, -115.2, 33.2, -114.8}, {`[0 [[myid_a1 {"type":"Point","coordinates":[-115,33]} [a 1]]]]`},
 		{"SET", "mykey", "myid_a2", "FIELD", "a", 2, "POINT", 32.99, -115}, {"OK"},
 		{"SET", "mykey", "myid_a3", "FIELD", "a", 3, "POINT", 33, -115.02}, {"OK"},
 		{"WITHIN", "mykey", "WHEREIN", "a", 3, 0, 1, 2, "BOUNDS", 32.8, -115.2, 33.2, -114.8}, {