tile38/internal/server/crud.go

1089 lines
24 KiB
Go
Raw Permalink Normal View History

package server
2016-03-05 02:08:16 +03:00
import (
"bytes"
2016-03-28 18:57:41 +03:00
"sort"
2016-03-05 02:08:16 +03:00
"strconv"
"strings"
"time"
"github.com/mmcloughlin/geohash"
"github.com/tidwall/geojson"
"github.com/tidwall/geojson/geometry"
2019-09-13 04:42:53 +03:00
"github.com/tidwall/rbang"
2016-03-28 18:57:41 +03:00
"github.com/tidwall/resp"
Fix excessive memory usage for objects with TTLs This commit fixes an issue where Tile38 was using lots of extra memory to track objects that are marked to expire. This was creating problems with applications that set big TTLs. How it worked before: Every collection had a unique hashmap that stores expiration timestamps for every object in that collection. Along with the hashmaps, there's also one big server-wide list that gets appended every time a new SET+EX is performed. From a background routine, this list is looped over at least 10 times per second and is randomly searched for potential candidates that might need expiring. The routine then removes those entries from the list and tests if the objects matching the entries have actually expired. If so, these objects are deleted them from the database. When at least 25% of the 20 candidates are deleted the loop is immediately continued, otherwise the loop backs off with a 100ms pause. Why this was a problem. The list grows one entry for every SET+EX. When TTLs are long, like 24-hours or more, it would take at least that much time before the entry is removed. So for databased that have objects that use TTLs and are updated often this could lead to a very large list. How it was fixed. The list was removed and the hashmap is now search randomly. This required a new hashmap implementation, as the built-in Go map does not provide an operation for randomly geting entries. The chosen implementation is a robinhood-hash because it provides open-addressing, which makes for simple random bucket selections. Issue #502
2019-10-29 21:04:07 +03:00
"github.com/tidwall/rhh"
"github.com/tidwall/tile38/internal/collection"
"github.com/tidwall/tile38/internal/glob"
2019-02-11 23:39:29 +03:00
"github.com/tidwall/tinybtree"
2016-03-05 02:08:16 +03:00
)
2016-03-28 18:57:41 +03:00
type fvt struct {
field string
value float64
}
type byField []fvt
func (a byField) Len() int {
return len(a)
}
func (a byField) Less(i, j int) bool {
return a[i].field < a[j].field
}
func (a byField) Swap(i, j int) {
a[i], a[j] = a[j], a[i]
}
func orderFields(fmap map[string]int, fields []float64) []fvt {
var fv fvt
fvs := make([]fvt, 0, len(fmap))
for field, idx := range fmap {
if idx < len(fields) {
fv.field = field
fv.value = fields[idx]
2016-03-30 19:32:38 +03:00
if fv.value != 0 {
2016-03-28 18:57:41 +03:00
fvs = append(fvs, fv)
}
}
}
sort.Sort(byField(fvs))
return fvs
}
func (server *Server) cmdBounds(msg *Message) (resp.Value, error) {
start := time.Now()
vs := msg.Args[1:]
var ok bool
var key string
if vs, key, ok = tokenval(vs); !ok || key == "" {
return NOMessage, errInvalidNumberOfArguments
}
if len(vs) != 0 {
return NOMessage, errInvalidNumberOfArguments
}
col := server.getCol(key)
if col == nil {
if msg.OutputType == RESP {
2017-10-05 18:20:40 +03:00
return resp.NullValue(), nil
}
return NOMessage, errKeyNotFound
}
vals := make([]resp.Value, 0, 2)
var buf bytes.Buffer
if msg.OutputType == JSON {
buf.WriteString(`{"ok":true`)
}
2017-08-11 03:32:40 +03:00
minX, minY, maxX, maxY := col.Bounds()
2016-10-03 21:37:16 +03:00
bbox := geojson.NewRect(geometry.Rect{
Min: geometry.Point{X: minX, Y: minY},
Max: geometry.Point{X: maxX, Y: maxY},
})
if msg.OutputType == JSON {
buf.WriteString(`,"bounds":`)
buf.WriteString(string(bbox.AppendJSON(nil)))
} else {
vals = append(vals, resp.ArrayValue([]resp.Value{
resp.ArrayValue([]resp.Value{
2016-10-03 21:37:16 +03:00
resp.FloatValue(minX),
resp.FloatValue(minY),
}),
resp.ArrayValue([]resp.Value{
2016-10-03 21:37:16 +03:00
resp.FloatValue(maxX),
resp.FloatValue(maxY),
}),
}))
}
switch msg.OutputType {
case JSON:
buf.WriteString(`,"elapsed":"` + time.Now().Sub(start).String() + "\"}")
2017-10-05 18:20:40 +03:00
return resp.StringValue(buf.String()), nil
case RESP:
2017-10-05 18:20:40 +03:00
return vals[0], nil
}
return NOMessage, nil
}
2017-10-05 18:20:40 +03:00
func (server *Server) cmdType(msg *Message) (resp.Value, error) {
2016-08-26 23:42:52 +03:00
start := time.Now()
vs := msg.Args[1:]
2016-08-26 23:42:52 +03:00
var ok bool
var key string
if vs, key, ok = tokenval(vs); !ok || key == "" {
return NOMessage, errInvalidNumberOfArguments
2016-08-26 23:42:52 +03:00
}
col := server.getCol(key)
2016-08-26 23:42:52 +03:00
if col == nil {
if msg.OutputType == RESP {
2017-10-05 18:20:40 +03:00
return resp.SimpleStringValue("none"), nil
2016-08-26 23:42:52 +03:00
}
return NOMessage, errKeyNotFound
2016-08-26 23:42:52 +03:00
}
typ := "hash"
switch msg.OutputType {
case JSON:
2017-10-05 18:20:40 +03:00
return resp.StringValue(`{"ok":true,"type":` + string(typ) + `,"elapsed":"` + time.Now().Sub(start).String() + "\"}"), nil
case RESP:
2017-10-05 18:20:40 +03:00
return resp.SimpleStringValue(typ), nil
2016-08-26 23:42:52 +03:00
}
return NOMessage, nil
2016-08-26 23:42:52 +03:00
}
2016-03-28 18:57:41 +03:00
func (server *Server) cmdGet(msg *Message) (resp.Value, error) {
2016-03-05 02:08:16 +03:00
start := time.Now()
vs := msg.Args[1:]
2016-03-28 18:57:41 +03:00
var ok bool
2016-03-05 02:08:16 +03:00
var key, id, typ, sprecision string
2016-03-28 18:57:41 +03:00
if vs, key, ok = tokenval(vs); !ok || key == "" {
return NOMessage, errInvalidNumberOfArguments
2016-03-05 02:08:16 +03:00
}
2016-03-28 18:57:41 +03:00
if vs, id, ok = tokenval(vs); !ok || id == "" {
return NOMessage, errInvalidNumberOfArguments
2016-03-05 02:08:16 +03:00
}
2016-04-01 22:46:39 +03:00
withfields := false
if _, peek, ok := tokenval(vs); ok && strings.ToLower(peek) == "withfields" {
withfields = true
vs = vs[1:]
}
col := server.getCol(key)
2016-03-05 02:08:16 +03:00
if col == nil {
if msg.OutputType == RESP {
2017-10-05 18:20:40 +03:00
return resp.NullValue(), nil
2016-03-28 18:57:41 +03:00
}
return NOMessage, errKeyNotFound
2016-03-05 02:08:16 +03:00
}
o, fields, ok := col.Get(id)
ok = ok && !server.hasExpired(key, id)
2016-03-05 02:08:16 +03:00
if !ok {
if msg.OutputType == RESP {
2017-10-05 18:20:40 +03:00
return resp.NullValue(), nil
2016-03-28 18:57:41 +03:00
}
return NOMessage, errIDNotFound
2016-03-05 02:08:16 +03:00
}
2016-03-28 18:57:41 +03:00
vals := make([]resp.Value, 0, 2)
2016-03-05 02:08:16 +03:00
var buf bytes.Buffer
if msg.OutputType == JSON {
2016-03-28 18:57:41 +03:00
buf.WriteString(`{"ok":true`)
}
2016-07-10 23:23:50 +03:00
vs, typ, ok = tokenval(vs)
typ = strings.ToLower(typ)
if !ok {
typ = "object"
}
switch typ {
default:
return NOMessage, errInvalidArgument(typ)
2016-07-10 23:23:50 +03:00
case "object":
if msg.OutputType == JSON {
2016-03-28 18:57:41 +03:00
buf.WriteString(`,"object":`)
buf.WriteString(string(o.AppendJSON(nil)))
2016-03-28 18:57:41 +03:00
} else {
2016-07-10 05:44:28 +03:00
vals = append(vals, resp.StringValue(o.String()))
2016-03-28 18:57:41 +03:00
}
2016-07-10 23:23:50 +03:00
case "point":
if msg.OutputType == JSON {
2016-07-10 23:23:50 +03:00
buf.WriteString(`,"point":`)
buf.Write(appendJSONSimplePoint(nil, o))
2016-07-10 23:23:50 +03:00
} else {
point := o.Center()
var z float64
if gPoint, ok := o.(*geojson.Point); ok {
z = gPoint.Z()
}
if z != 0 {
2016-07-10 23:23:50 +03:00
vals = append(vals, resp.ArrayValue([]resp.Value{
resp.StringValue(strconv.FormatFloat(point.Y, 'f', -1, 64)),
resp.StringValue(strconv.FormatFloat(point.X, 'f', -1, 64)),
resp.StringValue(strconv.FormatFloat(z, 'f', -1, 64)),
2016-07-10 23:23:50 +03:00
}))
2016-03-28 18:57:41 +03:00
} else {
vals = append(vals, resp.ArrayValue([]resp.Value{
2016-07-10 23:23:50 +03:00
resp.StringValue(strconv.FormatFloat(point.Y, 'f', -1, 64)),
resp.StringValue(strconv.FormatFloat(point.X, 'f', -1, 64)),
2016-03-28 18:57:41 +03:00
}))
}
2016-03-05 02:08:16 +03:00
}
2016-07-10 23:23:50 +03:00
case "hash":
if vs, sprecision, ok = tokenval(vs); !ok || sprecision == "" {
return NOMessage, errInvalidNumberOfArguments
2016-07-10 23:23:50 +03:00
}
if msg.OutputType == JSON {
2016-07-10 23:23:50 +03:00
buf.WriteString(`,"hash":`)
}
precision, err := strconv.ParseInt(sprecision, 10, 64)
if err != nil || precision < 1 || precision > 64 {
return NOMessage, errInvalidArgument(sprecision)
2016-07-10 23:23:50 +03:00
}
center := o.Center()
p := geohash.EncodeWithPrecision(center.Y, center.X, uint(precision))
if msg.OutputType == JSON {
2016-07-10 23:23:50 +03:00
buf.WriteString(`"` + p + `"`)
} else {
vals = append(vals, resp.StringValue(p))
}
case "bounds":
if msg.OutputType == JSON {
2016-07-10 23:23:50 +03:00
buf.WriteString(`,"bounds":`)
buf.Write(appendJSONSimpleBounds(nil, o))
2016-07-10 23:23:50 +03:00
} else {
bbox := o.Rect()
2016-07-10 23:23:50 +03:00
vals = append(vals, resp.ArrayValue([]resp.Value{
resp.ArrayValue([]resp.Value{
resp.FloatValue(bbox.Min.Y),
resp.FloatValue(bbox.Min.X),
}),
resp.ArrayValue([]resp.Value{
resp.FloatValue(bbox.Max.Y),
resp.FloatValue(bbox.Max.X),
}),
}))
}
2016-03-05 02:08:16 +03:00
}
2016-07-10 23:23:50 +03:00
2016-03-28 18:57:41 +03:00
if len(vs) != 0 {
return NOMessage, errInvalidNumberOfArguments
2016-03-05 02:08:16 +03:00
}
2016-04-01 22:46:39 +03:00
if withfields {
fvs := orderFields(col.FieldMap(), fields)
if len(fvs) > 0 {
fvals := make([]resp.Value, 0, len(fvs)*2)
if msg.OutputType == JSON {
2016-04-01 22:46:39 +03:00
buf.WriteString(`,"fields":{`)
}
for i, fv := range fvs {
if msg.OutputType == JSON {
2016-04-01 22:46:39 +03:00
if i > 0 {
buf.WriteString(`,`)
}
buf.WriteString(jsonString(fv.field) + ":" + strconv.FormatFloat(fv.value, 'f', -1, 64))
} else {
fvals = append(fvals, resp.StringValue(fv.field), resp.StringValue(strconv.FormatFloat(fv.value, 'f', -1, 64)))
2016-03-05 02:08:16 +03:00
}
2016-04-01 22:46:39 +03:00
i++
}
if msg.OutputType == JSON {
2016-04-01 22:46:39 +03:00
buf.WriteString(`}`)
2016-03-28 18:57:41 +03:00
} else {
2016-04-01 22:46:39 +03:00
vals = append(vals, resp.ArrayValue(fvals))
2016-03-05 02:08:16 +03:00
}
2016-03-28 18:57:41 +03:00
}
}
2016-04-01 22:46:39 +03:00
switch msg.OutputType {
case JSON:
2016-03-28 18:57:41 +03:00
buf.WriteString(`,"elapsed":"` + time.Now().Sub(start).String() + "\"}")
2017-10-05 18:20:40 +03:00
return resp.StringValue(buf.String()), nil
case RESP:
2016-04-01 22:46:39 +03:00
var oval resp.Value
if withfields {
oval = resp.ArrayValue(vals)
} else {
oval = vals[0]
}
2017-10-05 18:20:40 +03:00
return oval, nil
2016-03-05 02:08:16 +03:00
}
return NOMessage, nil
2016-03-05 02:08:16 +03:00
}
2018-11-24 01:53:33 +03:00
func (server *Server) cmdDel(msg *Message) (res resp.Value, d commandDetails, err error) {
2016-03-28 18:57:41 +03:00
start := time.Now()
vs := msg.Args[1:]
2016-03-28 18:57:41 +03:00
var ok bool
if vs, d.key, ok = tokenval(vs); !ok || d.key == "" {
2016-03-05 02:08:16 +03:00
err = errInvalidNumberOfArguments
return
}
2016-03-28 18:57:41 +03:00
if vs, d.id, ok = tokenval(vs); !ok || d.id == "" {
2016-03-05 02:08:16 +03:00
err = errInvalidNumberOfArguments
return
}
2016-03-28 18:57:41 +03:00
if len(vs) != 0 {
2016-03-05 02:08:16 +03:00
err = errInvalidNumberOfArguments
return
}
2016-03-28 18:57:41 +03:00
found := false
col := server.getCol(d.key)
2016-03-05 02:08:16 +03:00
if col != nil {
2018-08-16 23:07:55 +03:00
d.obj, d.fields, ok = col.Delete(d.id)
2016-03-28 18:57:41 +03:00
if ok {
2016-07-13 07:59:36 +03:00
if col.Count() == 0 {
server.deleteCol(d.key)
2016-03-28 18:57:41 +03:00
}
found = true
2016-03-05 02:08:16 +03:00
}
}
server.clearIDExpires(d.key, d.id)
2016-03-05 02:08:16 +03:00
d.command = "del"
2016-03-28 18:57:41 +03:00
d.updated = found
2016-04-02 17:20:30 +03:00
d.timestamp = time.Now()
2016-03-28 18:57:41 +03:00
switch msg.OutputType {
case JSON:
2017-10-05 18:20:40 +03:00
res = resp.StringValue(`{"ok":true,"elapsed":"` + time.Now().Sub(start).String() + "\"}")
case RESP:
2016-03-28 18:57:41 +03:00
if d.updated {
2017-10-05 18:20:40 +03:00
res = resp.IntegerValue(1)
2016-03-28 18:57:41 +03:00
} else {
2017-10-05 18:20:40 +03:00
res = resp.IntegerValue(0)
2016-03-28 18:57:41 +03:00
}
}
2016-03-05 02:08:16 +03:00
return
}
2018-11-24 01:53:33 +03:00
func (server *Server) cmdPdel(msg *Message) (res resp.Value, d commandDetails, err error) {
start := time.Now()
vs := msg.Args[1:]
var ok bool
if vs, d.key, ok = tokenval(vs); !ok || d.key == "" {
err = errInvalidNumberOfArguments
return
}
if vs, d.pattern, ok = tokenval(vs); !ok || d.pattern == "" {
err = errInvalidNumberOfArguments
return
}
if len(vs) != 0 {
err = errInvalidNumberOfArguments
return
}
now := time.Now()
iter := func(id string, o geojson.Object, fields []float64) bool {
if match, _ := glob.Match(d.pattern, id); match {
2018-11-24 01:53:33 +03:00
d.children = append(d.children, &commandDetails{
command: "del",
updated: true,
timestamp: now,
key: d.key,
id: id,
})
}
return true
}
var expired int
col := server.getCol(d.key)
if col != nil {
g := glob.Parse(d.pattern, false)
if g.Limits[0] == "" && g.Limits[1] == "" {
2019-04-24 15:09:41 +03:00
col.Scan(false, nil, msg.Deadline, iter)
} else {
2019-04-24 15:09:41 +03:00
col.ScanRange(g.Limits[0], g.Limits[1], false, nil, msg.Deadline, iter)
}
var atLeastOneNotDeleted bool
for i, dc := range d.children {
2018-08-16 23:07:55 +03:00
dc.obj, dc.fields, ok = col.Delete(dc.id)
if !ok {
d.children[i].command = "?"
atLeastOneNotDeleted = true
} else {
d.children[i] = dc
}
server.clearIDExpires(d.key, dc.id)
}
if atLeastOneNotDeleted {
2018-11-24 01:53:33 +03:00
var nchildren []*commandDetails
for _, dc := range d.children {
if dc.command == "del" {
nchildren = append(nchildren, dc)
}
}
d.children = nchildren
}
if col.Count() == 0 {
server.deleteCol(d.key)
}
}
d.command = "pdel"
d.updated = len(d.children) > 0
d.timestamp = now
d.parent = true
switch msg.OutputType {
case JSON:
2017-10-05 18:20:40 +03:00
res = resp.StringValue(`{"ok":true,"elapsed":"` + time.Now().Sub(start).String() + "\"}")
case RESP:
total := len(d.children) - expired
if total < 0 {
total = 0
}
2017-10-05 18:20:40 +03:00
res = resp.IntegerValue(total)
}
return
}
2018-11-24 01:53:33 +03:00
func (server *Server) cmdDrop(msg *Message) (res resp.Value, d commandDetails, err error) {
2016-03-28 18:57:41 +03:00
start := time.Now()
vs := msg.Args[1:]
2016-03-28 18:57:41 +03:00
var ok bool
if vs, d.key, ok = tokenval(vs); !ok || d.key == "" {
2016-03-05 02:08:16 +03:00
err = errInvalidNumberOfArguments
return
}
2016-03-28 18:57:41 +03:00
if len(vs) != 0 {
2016-03-05 02:08:16 +03:00
err = errInvalidNumberOfArguments
return
}
col := server.getCol(d.key)
2016-03-05 02:08:16 +03:00
if col != nil {
server.deleteCol(d.key)
2016-03-28 18:57:41 +03:00
d.updated = true
2016-03-05 02:08:16 +03:00
} else {
d.key = "" // ignore the details
2016-03-28 18:57:41 +03:00
d.updated = false
2016-03-05 02:08:16 +03:00
}
d.command = "drop"
2016-04-02 17:20:30 +03:00
d.timestamp = time.Now()
server.clearKeyExpires(d.key)
2016-03-28 18:57:41 +03:00
switch msg.OutputType {
case JSON:
2017-10-05 18:20:40 +03:00
res = resp.StringValue(`{"ok":true,"elapsed":"` + time.Now().Sub(start).String() + "\"}")
case RESP:
2016-03-28 18:57:41 +03:00
if d.updated {
2017-10-05 18:20:40 +03:00
res = resp.IntegerValue(1)
2016-03-28 18:57:41 +03:00
} else {
2017-10-05 18:20:40 +03:00
res = resp.IntegerValue(0)
2016-03-28 18:57:41 +03:00
}
}
2016-03-05 02:08:16 +03:00
return
}
2018-12-28 04:15:53 +03:00
func (server *Server) cmdRename(msg *Message, nx bool) (res resp.Value, d commandDetails, err error) {
start := time.Now()
vs := msg.Args[1:]
var ok bool
if vs, d.key, ok = tokenval(vs); !ok || d.key == "" {
err = errInvalidNumberOfArguments
return
}
if vs, d.newKey, ok = tokenval(vs); !ok || d.newKey == "" {
err = errInvalidNumberOfArguments
return
}
if len(vs) != 0 {
err = errInvalidNumberOfArguments
return
}
col := server.getCol(d.key)
if col == nil {
err = errKeyNotFound
return
}
2018-12-28 04:15:53 +03:00
for _, h := range server.hooks {
if h.Key == d.key || h.Key == d.newKey {
err = errKeyHasHooksSet
return
}
}
d.command = "rename"
newCol := server.getCol(d.newKey)
if newCol == nil {
d.updated = true
} else if nx {
d.updated = false
} else {
server.deleteCol(d.newKey)
server.clearKeyExpires(d.newKey)
d.updated = true
2018-12-28 04:15:53 +03:00
}
if d.updated {
server.deleteCol(d.key)
server.setCol(d.newKey, col)
server.moveKeyExpires(d.key, d.newKey)
}
d.timestamp = time.Now()
switch msg.OutputType {
case JSON:
res = resp.StringValue(`{"ok":true,"elapsed":"` + time.Now().Sub(start).String() + "\"}")
case RESP:
if !nx {
res = resp.SimpleStringValue("OK")
} else if d.updated {
2018-12-28 04:15:53 +03:00
res = resp.IntegerValue(1)
} else {
res = resp.IntegerValue(0)
}
}
return
}
2018-11-24 01:53:33 +03:00
func (server *Server) cmdFlushDB(msg *Message) (res resp.Value, d commandDetails, err error) {
2016-03-28 18:57:41 +03:00
start := time.Now()
vs := msg.Args[1:]
2016-03-28 18:57:41 +03:00
if len(vs) != 0 {
2016-03-05 02:08:16 +03:00
err = errInvalidNumberOfArguments
return
}
2019-02-11 23:39:29 +03:00
server.cols = tinybtree.BTree{}
Fix excessive memory usage for objects with TTLs This commit fixes an issue where Tile38 was using lots of extra memory to track objects that are marked to expire. This was creating problems with applications that set big TTLs. How it worked before: Every collection had a unique hashmap that stores expiration timestamps for every object in that collection. Along with the hashmaps, there's also one big server-wide list that gets appended every time a new SET+EX is performed. From a background routine, this list is looped over at least 10 times per second and is randomly searched for potential candidates that might need expiring. The routine then removes those entries from the list and tests if the objects matching the entries have actually expired. If so, these objects are deleted them from the database. When at least 25% of the 20 candidates are deleted the loop is immediately continued, otherwise the loop backs off with a 100ms pause. Why this was a problem. The list grows one entry for every SET+EX. When TTLs are long, like 24-hours or more, it would take at least that much time before the entry is removed. So for databased that have objects that use TTLs and are updated often this could lead to a very large list. How it was fixed. The list was removed and the hashmap is now search randomly. This required a new hashmap implementation, as the built-in Go map does not provide an operation for randomly geting entries. The chosen implementation is a robinhood-hash because it provides open-addressing, which makes for simple random bucket selections. Issue #502
2019-10-29 21:04:07 +03:00
server.expires = rhh.New(0)
server.hooks = make(map[string]*Hook)
2018-11-24 04:15:14 +03:00
server.hooksOut = make(map[string]*Hook)
2019-09-13 04:42:53 +03:00
server.hookTree = rbang.RTree{}
2016-03-05 02:08:16 +03:00
d.command = "flushdb"
2016-03-28 18:57:41 +03:00
d.updated = true
2016-04-02 17:20:30 +03:00
d.timestamp = time.Now()
2016-03-28 18:57:41 +03:00
switch msg.OutputType {
case JSON:
2017-10-05 18:20:40 +03:00
res = resp.StringValue(`{"ok":true,"elapsed":"` + time.Now().Sub(start).String() + "\"}")
case RESP:
2017-10-05 18:20:40 +03:00
res = resp.SimpleStringValue("OK")
2016-03-28 18:57:41 +03:00
}
2016-03-05 02:08:16 +03:00
return
}
func (server *Server) parseSetArgs(vs []string) (
2018-11-24 01:53:33 +03:00
d commandDetails, fields []string, values []float64,
2016-10-03 18:31:13 +03:00
xx, nx bool,
expires *float64, etype []byte, evs []string, err error,
2016-07-15 22:22:48 +03:00
) {
2016-03-28 18:57:41 +03:00
var ok bool
var typ []byte
2016-03-28 18:57:41 +03:00
if vs, d.key, ok = tokenval(vs); !ok || d.key == "" {
2016-03-05 02:08:16 +03:00
err = errInvalidNumberOfArguments
return
}
2016-03-28 18:57:41 +03:00
if vs, d.id, ok = tokenval(vs); !ok || d.id == "" {
2016-03-05 02:08:16 +03:00
err = errInvalidNumberOfArguments
return
}
var arg []byte
var nvs []string
2016-03-05 02:08:16 +03:00
for {
if nvs, arg, ok = tokenvalbytes(vs); !ok || len(arg) == 0 {
2016-03-05 02:08:16 +03:00
err = errInvalidNumberOfArguments
return
}
if lcb(arg, "field") {
2016-03-28 18:57:41 +03:00
vs = nvs
2016-03-05 02:08:16 +03:00
var name string
var svalue string
var value float64
2016-03-28 18:57:41 +03:00
if vs, name, ok = tokenval(vs); !ok || name == "" {
2016-03-05 02:08:16 +03:00
err = errInvalidNumberOfArguments
return
}
if isReservedFieldName(name) {
err = errInvalidArgument(name)
return
}
2016-03-28 18:57:41 +03:00
if vs, svalue, ok = tokenval(vs); !ok || svalue == "" {
2016-03-05 02:08:16 +03:00
err = errInvalidNumberOfArguments
return
}
value, err = strconv.ParseFloat(svalue, 64)
if err != nil {
err = errInvalidArgument(svalue)
return
}
fields = append(fields, name)
values = append(values, value)
continue
}
if lcb(arg, "ex") {
2016-07-15 22:22:48 +03:00
vs = nvs
if expires != nil {
err = errInvalidArgument(string(arg))
2016-07-15 22:22:48 +03:00
return
}
var s string
var v float64
if vs, s, ok = tokenval(vs); !ok || s == "" {
err = errInvalidNumberOfArguments
return
}
v, err = strconv.ParseFloat(s, 64)
if err != nil {
err = errInvalidArgument(s)
return
}
expires = &v
continue
}
if lcb(arg, "xx") {
2016-10-03 18:31:13 +03:00
vs = nvs
if nx {
err = errInvalidArgument(string(arg))
2016-10-03 18:31:13 +03:00
return
}
xx = true
continue
}
if lcb(arg, "nx") {
2016-10-03 18:31:13 +03:00
vs = nvs
if xx {
err = errInvalidArgument(string(arg))
2016-10-03 18:31:13 +03:00
return
}
nx = true
continue
}
2016-03-05 02:08:16 +03:00
break
}
if vs, typ, ok = tokenvalbytes(vs); !ok || len(typ) == 0 {
2016-03-05 02:08:16 +03:00
err = errInvalidNumberOfArguments
return
}
2016-03-28 18:57:41 +03:00
if len(vs) == 0 {
2016-03-05 02:08:16 +03:00
err = errInvalidNumberOfArguments
return
}
etype = typ
2016-03-28 18:57:41 +03:00
evs = vs
2016-03-05 02:08:16 +03:00
switch {
default:
err = errInvalidArgument(string(typ))
2016-03-05 02:08:16 +03:00
return
case lcb(typ, "string"):
2016-07-10 05:44:28 +03:00
var str string
if vs, str, ok = tokenval(vs); !ok {
err = errInvalidNumberOfArguments
return
}
d.obj = collection.String(str)
case lcb(typ, "point"):
2016-03-05 02:08:16 +03:00
var slat, slon, sz string
2016-03-28 18:57:41 +03:00
if vs, slat, ok = tokenval(vs); !ok || slat == "" {
2016-03-05 02:08:16 +03:00
err = errInvalidNumberOfArguments
return
}
2016-03-28 18:57:41 +03:00
if vs, slon, ok = tokenval(vs); !ok || slon == "" {
2016-03-05 02:08:16 +03:00
err = errInvalidNumberOfArguments
return
}
2016-03-28 18:57:41 +03:00
vs, sz, ok = tokenval(vs)
if !ok || sz == "" {
var x, y float64
y, err = strconv.ParseFloat(slat, 64)
2016-03-05 02:08:16 +03:00
if err != nil {
err = errInvalidArgument(slat)
return
}
x, err = strconv.ParseFloat(slon, 64)
2016-03-05 02:08:16 +03:00
if err != nil {
err = errInvalidArgument(slon)
return
}
d.obj = geojson.NewPoint(geometry.Point{X: x, Y: y})
2016-03-05 02:08:16 +03:00
} else {
var x, y, z float64
y, err = strconv.ParseFloat(slat, 64)
2016-03-05 02:08:16 +03:00
if err != nil {
err = errInvalidArgument(slat)
return
}
x, err = strconv.ParseFloat(slon, 64)
2016-03-05 02:08:16 +03:00
if err != nil {
err = errInvalidArgument(slon)
return
}
z, err = strconv.ParseFloat(sz, 64)
2016-03-05 02:08:16 +03:00
if err != nil {
err = errInvalidArgument(sz)
return
}
d.obj = geojson.NewPointZ(geometry.Point{X: x, Y: y}, z)
2016-03-05 02:08:16 +03:00
}
case lcb(typ, "bounds"):
2016-03-05 02:08:16 +03:00
var sminlat, sminlon, smaxlat, smaxlon string
2016-03-28 18:57:41 +03:00
if vs, sminlat, ok = tokenval(vs); !ok || sminlat == "" {
2016-03-05 02:08:16 +03:00
err = errInvalidNumberOfArguments
return
}
2016-03-28 18:57:41 +03:00
if vs, sminlon, ok = tokenval(vs); !ok || sminlon == "" {
2016-03-05 02:08:16 +03:00
err = errInvalidNumberOfArguments
return
}
2016-03-28 18:57:41 +03:00
if vs, smaxlat, ok = tokenval(vs); !ok || smaxlat == "" {
2016-03-05 02:08:16 +03:00
err = errInvalidNumberOfArguments
return
}
2016-03-28 18:57:41 +03:00
if vs, smaxlon, ok = tokenval(vs); !ok || smaxlon == "" {
2016-03-05 02:08:16 +03:00
err = errInvalidNumberOfArguments
return
}
var minlat, minlon, maxlat, maxlon float64
minlat, err = strconv.ParseFloat(sminlat, 64)
if err != nil {
err = errInvalidArgument(sminlat)
return
}
minlon, err = strconv.ParseFloat(sminlon, 64)
if err != nil {
err = errInvalidArgument(sminlon)
return
}
maxlat, err = strconv.ParseFloat(smaxlat, 64)
if err != nil {
err = errInvalidArgument(smaxlat)
return
}
maxlon, err = strconv.ParseFloat(smaxlon, 64)
if err != nil {
err = errInvalidArgument(smaxlon)
return
}
d.obj = geojson.NewRect(geometry.Rect{
Min: geometry.Point{X: minlon, Y: minlat},
Max: geometry.Point{X: maxlon, Y: maxlat},
})
case lcb(typ, "hash"):
2016-03-05 02:08:16 +03:00
var shash string
2016-03-28 18:57:41 +03:00
if vs, shash, ok = tokenval(vs); !ok || shash == "" {
2016-03-05 02:08:16 +03:00
err = errInvalidNumberOfArguments
return
}
lat, lon := geohash.Decode(shash)
d.obj = geojson.NewPoint(geometry.Point{X: lon, Y: lat})
case lcb(typ, "object"):
2016-03-28 18:57:41 +03:00
var object string
if vs, object, ok = tokenval(vs); !ok || object == "" {
err = errInvalidNumberOfArguments
return
}
d.obj, err = geojson.Parse(object, &server.geomParseOpts)
2016-03-05 02:08:16 +03:00
if err != nil {
return
}
}
2016-03-28 18:57:41 +03:00
if len(vs) != 0 {
err = errInvalidNumberOfArguments
}
2016-03-05 02:08:16 +03:00
return
}
2019-10-08 19:45:46 +03:00
func (server *Server) cmdSet(msg *Message, resetExpires bool) (res resp.Value, d commandDetails, err error) {
if server.config.maxMemory() > 0 && server.outOfMemory.on() {
2016-05-24 05:44:25 +03:00
err = errOOM
return
}
2016-03-28 18:57:41 +03:00
start := time.Now()
vs := msg.Args[1:]
2016-10-03 18:31:13 +03:00
var fmap map[string]int
2016-03-05 02:08:16 +03:00
var fields []string
var values []float64
2016-10-03 18:31:13 +03:00
var xx, nx bool
2016-07-15 22:22:48 +03:00
var ex *float64
d, fields, values, xx, nx, ex, _, _, err = server.parseSetArgs(vs)
2016-03-05 02:08:16 +03:00
if err != nil {
return
}
col := server.getCol(d.key)
2016-03-05 02:08:16 +03:00
if col == nil {
2016-10-03 18:31:13 +03:00
if xx {
goto notok
}
2016-03-05 02:08:16 +03:00
col = collection.New()
server.setCol(d.key, col)
2016-03-05 02:08:16 +03:00
}
2016-10-31 20:17:30 +03:00
if xx || nx {
2016-10-03 18:31:13 +03:00
_, _, ok := col.Get(d.id)
2016-10-31 20:17:30 +03:00
if (nx && ok) || (xx && !ok) {
2016-10-03 18:31:13 +03:00
goto notok
}
}
2019-10-08 19:45:46 +03:00
if resetExpires {
server.clearIDExpires(d.key, d.id)
}
2018-08-16 23:07:55 +03:00
d.oldObj, d.oldFields, d.fields = col.Set(d.id, d.obj, fields, values)
2016-03-05 02:08:16 +03:00
d.command = "set"
2016-03-28 18:57:41 +03:00
d.updated = true // perhaps we should do a diff on the previous object?
2016-04-02 17:20:30 +03:00
d.timestamp = time.Now()
if msg.ConnType != Null || msg.OutputType != Null {
// likely loaded from aof at server startup, ignore field remapping.
fmap = col.FieldMap()
d.fmap = make(map[string]int)
for key, idx := range fmap {
d.fmap[key] = idx
}
}
2016-07-15 22:22:48 +03:00
if ex != nil {
server.expireAt(d.key, d.id, d.timestamp.Add(time.Duration(float64(time.Second)*(*ex))))
2016-07-15 22:22:48 +03:00
}
2016-03-28 18:57:41 +03:00
switch msg.OutputType {
default:
case JSON:
2017-10-05 18:20:40 +03:00
res = resp.StringValue(`{"ok":true,"elapsed":"` + time.Now().Sub(start).String() + "\"}")
case RESP:
2017-10-05 18:20:40 +03:00
res = resp.SimpleStringValue("OK")
2016-03-28 18:57:41 +03:00
}
2016-03-05 02:08:16 +03:00
return
2016-10-03 18:31:13 +03:00
notok:
switch msg.OutputType {
default:
case JSON:
2016-10-31 20:17:30 +03:00
if nx {
err = errIDAlreadyExists
} else {
err = errIDNotFound
}
return
case RESP:
2017-10-05 18:20:40 +03:00
res = resp.NullValue()
2016-10-03 18:31:13 +03:00
}
return
2016-03-05 02:08:16 +03:00
}
func (server *Server) parseFSetArgs(vs []string) (
2018-11-24 01:53:33 +03:00
d commandDetails, fields []string, values []float64, xx bool, err error,
) {
2016-03-28 18:57:41 +03:00
var ok bool
if vs, d.key, ok = tokenval(vs); !ok || d.key == "" {
2016-03-05 02:08:16 +03:00
err = errInvalidNumberOfArguments
return
}
2016-03-28 18:57:41 +03:00
if vs, d.id, ok = tokenval(vs); !ok || d.id == "" {
2016-03-05 02:08:16 +03:00
err = errInvalidNumberOfArguments
return
}
for len(vs) > 0 {
var name string
if vs, name, ok = tokenval(vs); !ok || name == "" {
err = errInvalidNumberOfArguments
return
}
if lc(name, "xx") {
xx = true
continue
}
if isReservedFieldName(name) {
err = errInvalidArgument(name)
return
}
var svalue string
var value float64
if vs, svalue, ok = tokenval(vs); !ok || svalue == "" {
err = errInvalidNumberOfArguments
return
}
value, err = strconv.ParseFloat(svalue, 64)
if err != nil {
err = errInvalidArgument(svalue)
return
}
fields = append(fields, name)
values = append(values, value)
2016-03-05 02:08:16 +03:00
}
return
}
2018-11-24 01:53:33 +03:00
func (server *Server) cmdFset(msg *Message) (res resp.Value, d commandDetails, err error) {
if server.config.maxMemory() > 0 && server.outOfMemory.on() {
2018-01-24 02:54:10 +03:00
err = errOOM
return
}
2016-03-28 18:57:41 +03:00
start := time.Now()
vs := msg.Args[1:]
var fields []string
var values []float64
var xx bool
var updateCount int
d, fields, values, xx, err = server.parseFSetArgs(vs)
col := server.getCol(d.key)
2016-03-05 02:08:16 +03:00
if col == nil {
err = errKeyNotFound
return
}
var ok bool
d.obj, d.fields, updateCount, ok = col.SetFields(d.id, fields, values)
if !(ok || xx) {
2016-03-05 02:08:16 +03:00
err = errIDNotFound
return
}
if ok {
d.command = "fset"
d.timestamp = time.Now()
d.updated = updateCount > 0
fmap := col.FieldMap()
d.fmap = make(map[string]int)
for key, idx := range fmap {
d.fmap[key] = idx
}
2016-04-02 17:20:30 +03:00
}
2016-03-28 18:57:41 +03:00
switch msg.OutputType {
case JSON:
2017-10-05 18:20:40 +03:00
res = resp.StringValue(`{"ok":true,"elapsed":"` + time.Now().Sub(start).String() + "\"}")
case RESP:
res = resp.IntegerValue(updateCount)
2016-03-28 18:57:41 +03:00
}
2016-03-05 02:08:16 +03:00
return
}
2016-07-15 22:22:48 +03:00
2018-11-24 01:53:33 +03:00
func (server *Server) cmdExpire(msg *Message) (res resp.Value, d commandDetails, err error) {
2016-07-15 22:22:48 +03:00
start := time.Now()
vs := msg.Args[1:]
2016-07-15 22:22:48 +03:00
var key, id, svalue string
var ok bool
if vs, key, ok = tokenval(vs); !ok || key == "" {
err = errInvalidNumberOfArguments
return
}
if vs, id, ok = tokenval(vs); !ok || id == "" {
err = errInvalidNumberOfArguments
return
}
if vs, svalue, ok = tokenval(vs); !ok || svalue == "" {
err = errInvalidNumberOfArguments
return
}
if len(vs) != 0 {
err = errInvalidNumberOfArguments
return
}
var value float64
value, err = strconv.ParseFloat(svalue, 64)
if err != nil {
err = errInvalidArgument(svalue)
return
}
ok = false
col := server.getCol(key)
2016-07-15 22:22:48 +03:00
if col != nil {
_, _, ok = col.Get(id)
ok = ok && !server.hasExpired(key, id)
}
if ok {
server.expireAt(key, id, time.Now().Add(time.Duration(float64(time.Second)*value)))
d.updated = true
2016-07-15 22:22:48 +03:00
}
switch msg.OutputType {
case JSON:
if ok {
2017-10-05 18:20:40 +03:00
res = resp.StringValue(`{"ok":true,"elapsed":"` + time.Now().Sub(start).String() + "\"}")
} else {
2017-10-05 18:20:40 +03:00
return resp.SimpleStringValue(""), d, errIDNotFound
}
case RESP:
2016-07-15 22:22:48 +03:00
if ok {
2017-10-05 18:20:40 +03:00
res = resp.IntegerValue(1)
2016-07-15 22:22:48 +03:00
} else {
2017-10-05 18:20:40 +03:00
res = resp.IntegerValue(0)
2016-07-15 22:22:48 +03:00
}
}
return
}
2018-11-24 01:53:33 +03:00
func (server *Server) cmdPersist(msg *Message) (res resp.Value, d commandDetails, err error) {
2016-07-15 22:22:48 +03:00
start := time.Now()
vs := msg.Args[1:]
2016-07-15 22:22:48 +03:00
var key, id string
var ok bool
if vs, key, ok = tokenval(vs); !ok || key == "" {
err = errInvalidNumberOfArguments
return
}
if vs, id, ok = tokenval(vs); !ok || id == "" {
err = errInvalidNumberOfArguments
return
}
if len(vs) != 0 {
err = errInvalidNumberOfArguments
return
}
var cleared bool
2016-07-15 22:22:48 +03:00
ok = false
col := server.getCol(key)
2016-07-15 22:22:48 +03:00
if col != nil {
_, _, ok = col.Get(id)
ok = ok && !server.hasExpired(key, id)
2016-07-15 22:22:48 +03:00
if ok {
cleared = server.clearIDExpires(key, id)
}
}
if !ok {
if msg.OutputType == RESP {
2017-10-05 18:20:40 +03:00
return resp.IntegerValue(0), d, nil
2016-07-15 22:22:48 +03:00
}
2017-10-05 18:20:40 +03:00
return resp.SimpleStringValue(""), d, errIDNotFound
2016-07-15 22:22:48 +03:00
}
d.command = "persist"
d.updated = cleared
d.timestamp = time.Now()
2016-07-15 22:22:48 +03:00
switch msg.OutputType {
case JSON:
2017-10-05 18:20:40 +03:00
res = resp.SimpleStringValue(`{"ok":true,"elapsed":"` + time.Now().Sub(start).String() + "\"}")
case RESP:
if cleared {
2017-10-05 18:20:40 +03:00
res = resp.IntegerValue(1)
2016-07-15 22:22:48 +03:00
} else {
2017-10-05 18:20:40 +03:00
res = resp.IntegerValue(0)
2016-07-15 22:22:48 +03:00
}
}
return
}
func (server *Server) cmdTTL(msg *Message) (res resp.Value, err error) {
2016-07-15 22:22:48 +03:00
start := time.Now()
vs := msg.Args[1:]
2016-07-15 22:22:48 +03:00
var key, id string
var ok bool
if vs, key, ok = tokenval(vs); !ok || key == "" {
err = errInvalidNumberOfArguments
return
}
if vs, id, ok = tokenval(vs); !ok || id == "" {
err = errInvalidNumberOfArguments
return
}
if len(vs) != 0 {
err = errInvalidNumberOfArguments
return
}
var v float64
ok = false
var ok2 bool
col := server.getCol(key)
2016-07-15 22:22:48 +03:00
if col != nil {
_, _, ok = col.Get(id)
ok = ok && !server.hasExpired(key, id)
2016-07-15 22:22:48 +03:00
if ok {
var at time.Time
at, ok2 = server.getExpires(key, id)
2016-07-15 22:22:48 +03:00
if ok2 {
if time.Now().After(at) {
ok2 = false
} else {
v = float64(at.Sub(time.Now())) / float64(time.Second)
if v < 0 {
v = 0
}
2016-07-15 22:22:48 +03:00
}
}
}
}
switch msg.OutputType {
case JSON:
if ok {
var ttl string
if ok2 {
ttl = strconv.FormatFloat(v, 'f', -1, 64)
} else {
ttl = "-1"
}
2017-10-05 18:20:40 +03:00
res = resp.SimpleStringValue(
`{"ok":true,"ttl":` + ttl + `,"elapsed":"` + time.Now().Sub(start).String() + "\"}")
} else {
2017-10-05 18:20:40 +03:00
return resp.SimpleStringValue(""), errIDNotFound
}
case RESP:
2016-07-15 22:22:48 +03:00
if ok {
if ok2 {
2017-10-05 18:20:40 +03:00
res = resp.IntegerValue(int(v))
2016-07-15 22:22:48 +03:00
} else {
2017-10-05 18:20:40 +03:00
res = resp.IntegerValue(-1)
2016-07-15 22:22:48 +03:00
}
} else {
2017-10-05 18:20:40 +03:00
res = resp.IntegerValue(-2)
2016-07-15 22:22:48 +03:00
}
}
return
}