From fbe19564b931c637d51b6a896824393c4a241e85 Mon Sep 17 00:00:00 2001 From: Josh Baker Date: Sun, 10 Jul 2016 13:23:50 -0700 Subject: [PATCH] added string type --- controller/aofshrink.go | 22 +++-- controller/collection/collection.go | 66 +++++++++---- controller/collection/collection_test.go | 6 +- controller/crud.go | 120 ++++++++++++----------- controller/fence.go | 2 +- controller/scan.go | 37 ++++--- controller/stats.go | 7 +- core/commands.json | 11 ++- core/commands_gen.go | 9 ++ 9 files changed, 166 insertions(+), 114 deletions(-) diff --git a/controller/aofshrink.go b/controller/aofshrink.go index 07ee3ebb..da563bf1 100644 --- a/controller/aofshrink.go +++ b/controller/aofshrink.go @@ -150,13 +150,15 @@ func (c *Controller) aofshrink() { objs := make(map[string]objFields) c.mu.Lock() fnames := col.FieldArr() // reload an array of field names to match each object - col.ScanGreaterOrEqual(nextID, 0, func(id string, obj geojson.Object, fields []float64) bool { - if id != nextID { - objs[id] = objFields{obj, fields} - nextID = id - } - return len(objs) < maxIDGroup - }) + col.ScanGreaterOrEqual(nextID, 0, collection.TypeAll, + func(id string, obj geojson.Object, fields []float64) bool { + if id != nextID { + objs[id] = objFields{obj, fields} + nextID = id + } + return len(objs) < maxIDGroup + }, + ) c.mu.Unlock() ids := make([]string, 0, maxIDGroup) @@ -177,7 +179,11 @@ func (c *Controller) aofshrink() { } switch obj := obj.obj.(type) { default: - values = append(values, resp.StringValue("object"), resp.StringValue(obj.JSON())) + if obj.IsGeometry() { + values = append(values, resp.StringValue("object"), resp.StringValue(obj.JSON())) + } else { + values = append(values, resp.StringValue("string"), resp.StringValue(obj.String())) + } case geojson.SimplePoint: values = append(values, resp.StringValue("point"), resp.FloatValue(obj.Y), resp.FloatValue(obj.X)) case geojson.Point: diff --git a/controller/collection/collection.go b/controller/collection/collection.go index 61c5695c..fbf2fbdc 100644 --- a/controller/collection/collection.go +++ b/controller/collection/collection.go @@ -6,9 +6,21 @@ import ( "github.com/tidwall/tile38/index" ) +// ScanType is the classification of objects that are returned from Scan() +type ScanType int + +const ( + // TypeAll means to return all type during a Scan() + TypeAll = ScanType(0) + // TypeGeometry means to return only geometries + TypeGeometry = ScanType(1) + // TypeNonGeometry means to return non-geometries + TypeNonGeometry = ScanType(2) +) + const ( idOrdered = 0 - valueOrdered = 2 + valueOrdered = 1 ) type itemT struct { @@ -24,12 +36,14 @@ func (i *itemT) Less(item btree.Item, ctx int) bool { case idOrdered: return i.id < item.(*itemT).id case valueOrdered: - if i.object.String() < item.(*itemT).object.String() { + i1, i2 := i.object.String(), item.(*itemT).object.String() + if i1 < i2 { return true } - if i.object.String() > item.(*itemT).object.String() { + if i1 > i2 { return false } + // the values match so we will compare the ids, which are always unique. return i.id < item.(*itemT).id } @@ -53,7 +67,8 @@ type Collection struct { fieldMap map[string]int weight int points int - objects int + objects int // geometry count + nobjects int // non-geometry count } var counter uint64 @@ -70,8 +85,15 @@ func New() *Collection { } // Count returns the number of objects in collection. -func (c *Collection) Count() int { - return c.objects +func (c *Collection) Count(stype ScanType) int { + switch stype { + default: + return c.objects + c.nobjects + case TypeGeometry: + return c.objects + case TypeNonGeometry: + return c.nobjects + } } // PointCount returns the number of points (lat/lon coordinates) in collection. @@ -120,13 +142,14 @@ func (c *Collection) remove(id string) (item *itemT, ok bool) { item = i.(*itemT) if item.object.IsGeometry() { c.index.Remove(item) + c.objects-- } else { c.values.Delete(item) + c.nobjects-- } c.weight -= len(item.fields) * 8 c.weight -= item.object.Weight() + len(item.id) c.points -= item.object.PositionCount() - c.objects-- return item, true } @@ -134,13 +157,14 @@ func (c *Collection) insert(id string, obj geojson.Object) (item *itemT) { item = &itemT{id: id, object: obj} if obj.IsGeometry() { c.index.Insert(item) + c.objects++ } else { c.values.ReplaceOrInsert(item) + c.nobjects++ } c.items.ReplaceOrInsert(item) c.weight += obj.Weight() + len(id) c.points += obj.PositionCount() - c.objects++ return item } @@ -213,7 +237,7 @@ func (c *Collection) FieldArr() []string { } // Scan iterates though the collection. A cursor can be used for paging. -func (c *Collection) Scan(cursor uint64, iterator func(id string, obj geojson.Object, fields []float64) bool) (ncursor uint64) { +func (c *Collection) Scan(cursor uint64, stype ScanType, iterator func(id string, obj geojson.Object, fields []float64) bool) (ncursor uint64) { var i uint64 var active = true c.items.Ascend(func(item btree.Item) bool { @@ -228,7 +252,7 @@ func (c *Collection) Scan(cursor uint64, iterator func(id string, obj geojson.Ob } // ScanGreaterOrEqual iterates though the collection starting with specified id. A cursor can be used for paging. -func (c *Collection) ScanGreaterOrEqual(id string, cursor uint64, iterator func(id string, obj geojson.Object, fields []float64) bool) (ncursor uint64) { +func (c *Collection) ScanGreaterOrEqual(id string, cursor uint64, stype ScanType, iterator func(id string, obj geojson.Object, fields []float64) bool) (ncursor uint64) { var i uint64 var active = true c.items.AscendGreaterOrEqual(&itemT{id: id}, func(item btree.Item) bool { @@ -242,7 +266,7 @@ func (c *Collection) ScanGreaterOrEqual(id string, cursor uint64, iterator func( return i } -func (c *Collection) search(cursor uint64, bbox geojson.BBox, iterator func(id string, obj geojson.Object, fields []float64) bool) (ncursor uint64) { +func (c *Collection) geoSearch(cursor uint64, bbox geojson.BBox, iterator func(id string, obj geojson.Object, fields []float64) bool) (ncursor uint64) { return c.index.Search(cursor, bbox.Min.Y, bbox.Min.X, bbox.Max.Y, bbox.Max.X, func(item index.Item) bool { var iitm *itemT iitm, ok := item.(*itemT) @@ -263,7 +287,7 @@ func (c *Collection) Nearby(cursor uint64, sparse uint8, lat, lon, meters float6 bboxes := bbox.Sparse(sparse) if sparse > 0 { for _, bbox := range bboxes { - c.search(cursor, bbox, func(id string, obj geojson.Object, fields []float64) bool { + c.geoSearch(cursor, bbox, func(id string, obj geojson.Object, fields []float64) bool { if obj.Nearby(center, meters) { if iterator(id, obj, fields) { return false @@ -274,7 +298,7 @@ func (c *Collection) Nearby(cursor uint64, sparse uint8, lat, lon, meters float6 } return 0 } - return c.search(cursor, bbox, func(id string, obj geojson.Object, fields []float64) bool { + return c.geoSearch(cursor, bbox, func(id string, obj geojson.Object, fields []float64) bool { if obj.Nearby(center, meters) { return iterator(id, obj, fields) } @@ -294,7 +318,7 @@ func (c *Collection) Within(cursor uint64, sparse uint8, obj geojson.Object, min if sparse > 0 { for _, bbox := range bboxes { if obj != nil { - c.search(cursor, bbox, func(id string, o geojson.Object, fields []float64) bool { + c.geoSearch(cursor, bbox, func(id string, o geojson.Object, fields []float64) bool { if o.Within(obj) { if iterator(id, o, fields) { return false @@ -303,7 +327,7 @@ func (c *Collection) Within(cursor uint64, sparse uint8, obj geojson.Object, min return true }) } - c.search(cursor, bbox, func(id string, o geojson.Object, fields []float64) bool { + c.geoSearch(cursor, bbox, func(id string, o geojson.Object, fields []float64) bool { if o.WithinBBox(bbox) { if iterator(id, o, fields) { return false @@ -315,14 +339,14 @@ func (c *Collection) Within(cursor uint64, sparse uint8, obj geojson.Object, min return 0 } if obj != nil { - return c.search(cursor, bbox, func(id string, o geojson.Object, fields []float64) bool { + return c.geoSearch(cursor, bbox, func(id string, o geojson.Object, fields []float64) bool { if o.Within(obj) { return iterator(id, o, fields) } return true }) } - return c.search(cursor, bbox, func(id string, o geojson.Object, fields []float64) bool { + return c.geoSearch(cursor, bbox, func(id string, o geojson.Object, fields []float64) bool { if o.WithinBBox(bbox) { return iterator(id, o, fields) } @@ -353,7 +377,7 @@ func (c *Collection) Intersects(cursor uint64, sparse uint8, obj geojson.Object, } for _, bbox := range bboxes { if obj != nil { - c.search(cursor, bbox, func(id string, o geojson.Object, fields []float64) bool { + c.geoSearch(cursor, bbox, func(id string, o geojson.Object, fields []float64) bool { if o.Intersects(obj) { if iterator(id, o, fields) { return false @@ -362,7 +386,7 @@ func (c *Collection) Intersects(cursor uint64, sparse uint8, obj geojson.Object, return true }) } - c.search(cursor, bbox, func(id string, o geojson.Object, fields []float64) bool { + c.geoSearch(cursor, bbox, func(id string, o geojson.Object, fields []float64) bool { if o.IntersectsBBox(bbox) { if iterator(id, o, fields) { return false @@ -374,14 +398,14 @@ func (c *Collection) Intersects(cursor uint64, sparse uint8, obj geojson.Object, return 0 } if obj != nil { - return c.search(cursor, bbox, func(id string, o geojson.Object, fields []float64) bool { + return c.geoSearch(cursor, bbox, func(id string, o geojson.Object, fields []float64) bool { if o.Intersects(obj) { return iterator(id, o, fields) } return true }) } - return c.search(cursor, bbox, func(id string, o geojson.Object, fields []float64) bool { + return c.geoSearch(cursor, bbox, func(id string, o geojson.Object, fields []float64) bool { if o.IntersectsBBox(bbox) { return iterator(id, o, fields) } diff --git a/controller/collection/collection_test.go b/controller/collection/collection_test.go index 29cc0883..6eb1cbd5 100644 --- a/controller/collection/collection_test.go +++ b/controller/collection/collection_test.go @@ -31,14 +31,14 @@ func TestCollection(t *testing.T) { } count := 0 bbox := geojson.BBox{Min: geojson.Position{X: -180, Y: -90, Z: 0}, Max: geojson.Position{X: 180, Y: 90, Z: 0}} - c.search(0, bbox, func(id string, obj geojson.Object, field []float64) bool { + c.geoSearch(0, bbox, func(id string, obj geojson.Object, field []float64) bool { count++ return true }) if count != len(objs) { t.Fatalf("count = %d, expect %d", count, len(objs)) } - count = c.Count() + count = c.Count(TypeAll) if count != len(objs) { t.Fatalf("c.Count() = %d, expect %d", count, len(objs)) } @@ -89,7 +89,7 @@ func TestManyCollections(t *testing.T) { col := colsM["13"] //println(col.Count()) bbox := geojson.BBox{Min: geojson.Position{X: -180, Y: 30, Z: 0}, Max: geojson.Position{X: 34, Y: 100, Z: 0}} - col.search(0, bbox, func(id string, obj geojson.Object, fields []float64) bool { + col.geoSearch(0, bbox, func(id string, obj geojson.Object, fields []float64) bool { //println(id) return true }) diff --git a/controller/crud.go b/controller/crud.go index ba7c5bf1..e57a96ed 100644 --- a/controller/crud.go +++ b/controller/crud.go @@ -87,75 +87,79 @@ func (c *Controller) cmdGet(msg *server.Message) (string, error) { if msg.OutputType == server.JSON { buf.WriteString(`{"ok":true`) } - if vs, typ, ok = tokenval(vs); !ok || strings.ToLower(typ) == "object" { + vs, typ, ok = tokenval(vs) + typ = strings.ToLower(typ) + if !ok { + typ = "object" + } + switch typ { + default: + return "", errInvalidArgument(typ) + case "object": if msg.OutputType == server.JSON { buf.WriteString(`,"object":`) buf.WriteString(o.JSON()) } else { vals = append(vals, resp.StringValue(o.String())) } - } else { - switch strings.ToLower(typ) { - default: - return "", errInvalidArgument(typ) - case "point": - point := o.CalculatedPoint() - if msg.OutputType == server.JSON { - buf.WriteString(`,"point":`) - buf.WriteString(point.ExternalJSON()) - } else { - if point.Z != 0 { - vals = append(vals, resp.ArrayValue([]resp.Value{ - resp.StringValue(strconv.FormatFloat(point.Y, 'f', -1, 64)), - resp.StringValue(strconv.FormatFloat(point.X, 'f', -1, 64)), - resp.StringValue(strconv.FormatFloat(point.Z, 'f', -1, 64)), - })) - } else { - vals = append(vals, resp.ArrayValue([]resp.Value{ - resp.StringValue(strconv.FormatFloat(point.Y, 'f', -1, 64)), - resp.StringValue(strconv.FormatFloat(point.X, 'f', -1, 64)), - })) - } - } - case "hash": - if vs, sprecision, ok = tokenval(vs); !ok || sprecision == "" { - return "", errInvalidNumberOfArguments - } - if msg.OutputType == server.JSON { - buf.WriteString(`,"hash":`) - } - precision, err := strconv.ParseInt(sprecision, 10, 64) - if err != nil || precision < 1 || precision > 64 { - return "", errInvalidArgument(sprecision) - } - p, err := o.Geohash(int(precision)) - if err != nil { - return "", err - } - if msg.OutputType == server.JSON { - buf.WriteString(`"` + p + `"`) - } else { - vals = append(vals, resp.StringValue(p)) - } - case "bounds": - bbox := o.CalculatedBBox() - if msg.OutputType == server.JSON { - buf.WriteString(`,"bounds":`) - buf.WriteString(bbox.ExternalJSON()) + case "point": + point := o.CalculatedPoint() + if msg.OutputType == server.JSON { + buf.WriteString(`,"point":`) + buf.WriteString(point.ExternalJSON()) + } else { + if point.Z != 0 { + vals = append(vals, resp.ArrayValue([]resp.Value{ + resp.StringValue(strconv.FormatFloat(point.Y, 'f', -1, 64)), + resp.StringValue(strconv.FormatFloat(point.X, 'f', -1, 64)), + resp.StringValue(strconv.FormatFloat(point.Z, 'f', -1, 64)), + })) } else { vals = append(vals, resp.ArrayValue([]resp.Value{ - resp.ArrayValue([]resp.Value{ - resp.FloatValue(bbox.Min.Y), - resp.FloatValue(bbox.Min.X), - }), - resp.ArrayValue([]resp.Value{ - resp.FloatValue(bbox.Max.Y), - resp.FloatValue(bbox.Max.X), - }), + resp.StringValue(strconv.FormatFloat(point.Y, 'f', -1, 64)), + resp.StringValue(strconv.FormatFloat(point.X, 'f', -1, 64)), })) } } + case "hash": + if vs, sprecision, ok = tokenval(vs); !ok || sprecision == "" { + return "", errInvalidNumberOfArguments + } + if msg.OutputType == server.JSON { + buf.WriteString(`,"hash":`) + } + precision, err := strconv.ParseInt(sprecision, 10, 64) + if err != nil || precision < 1 || precision > 64 { + return "", errInvalidArgument(sprecision) + } + p, err := o.Geohash(int(precision)) + if err != nil { + return "", err + } + if msg.OutputType == server.JSON { + buf.WriteString(`"` + p + `"`) + } else { + vals = append(vals, resp.StringValue(p)) + } + case "bounds": + bbox := o.CalculatedBBox() + if msg.OutputType == server.JSON { + buf.WriteString(`,"bounds":`) + buf.WriteString(bbox.ExternalJSON()) + } else { + vals = append(vals, resp.ArrayValue([]resp.Value{ + resp.ArrayValue([]resp.Value{ + resp.FloatValue(bbox.Min.Y), + resp.FloatValue(bbox.Min.X), + }), + resp.ArrayValue([]resp.Value{ + resp.FloatValue(bbox.Max.Y), + resp.FloatValue(bbox.Max.X), + }), + })) + } } + if len(vs) != 0 { return "", errInvalidNumberOfArguments } @@ -225,7 +229,7 @@ func (c *Controller) cmdDel(msg *server.Message) (res string, d commandDetailsT, if col != nil { d.obj, d.fields, ok = col.Remove(d.id) if ok { - if col.Count() == 0 { + if col.Count(collection.TypeAll) == 0 { c.deleteCol(d.key) d.revert = func() { c.setCol(d.key, col) diff --git a/controller/fence.go b/controller/fence.go index d9be0f52..38390b8b 100644 --- a/controller/fence.go +++ b/controller/fence.go @@ -30,7 +30,7 @@ func FenceMatch(hookName string, sw *scanWriter, fence *liveFenceSwitches, detai nofields := sw.nofields sw.mu.Unlock() - if details.obj == nil || (details.command == "fset" && nofields) { + if details.obj == nil || !details.obj.IsGeometry() || (details.command == "fset" && nofields) { return nil } match = false diff --git a/controller/scan.go b/controller/scan.go index 60e229cc..061b7a85 100644 --- a/controller/scan.go +++ b/controller/scan.go @@ -6,6 +6,7 @@ import ( "time" "github.com/tidwall/resp" + "github.com/tidwall/tile38/controller/collection" "github.com/tidwall/tile38/controller/server" "github.com/tidwall/tile38/geojson" ) @@ -34,22 +35,14 @@ func (c *Controller) cmdScan(msg *server.Message) (res string, err error) { if err != nil { return "", err } - if s.sparse > 0 && sw.col != nil { - msg.Values = append(msg.Values, - resp.StringValue("BOUNDS"), - resp.StringValue("-90"), - resp.StringValue("-180"), - resp.StringValue("180"), - ) - return c.cmdWithinOrIntersects("within", msg) - } if msg.OutputType == server.JSON { wr.WriteString(`{"ok":true`) } sw.writeHead() if sw.col != nil { + stype := collection.TypeAll if sw.output == outputCount && len(sw.wheres) == 0 && sw.globEverything == true { - count := sw.col.Count() - int(s.cursor) + count := sw.col.Count(stype) - int(s.cursor) if count < 0 { count = 0 } @@ -58,18 +51,24 @@ func (c *Controller) cmdScan(msg *server.Message) (res string, err error) { if strings.HasSuffix(sw.glob, "*") { greaterGlob := sw.glob[:len(sw.glob)-1] if globIsGlob(greaterGlob) { - s.cursor = sw.col.Scan(s.cursor, func(id string, o geojson.Object, fields []float64) bool { - return sw.writeObject(id, o, fields, false) - }) + s.cursor = sw.col.Scan(s.cursor, stype, + func(id string, o geojson.Object, fields []float64) bool { + return sw.writeObject(id, o, fields, false) + }, + ) } else { - s.cursor = sw.col.ScanGreaterOrEqual(greaterGlob, s.cursor, func(id string, o geojson.Object, fields []float64) bool { - return sw.writeObject(id, o, fields, false) - }) + s.cursor = sw.col.ScanGreaterOrEqual(greaterGlob, s.cursor, stype, + func(id string, o geojson.Object, fields []float64) bool { + return sw.writeObject(id, o, fields, false) + }, + ) } } else { - s.cursor = sw.col.Scan(s.cursor, func(id string, o geojson.Object, fields []float64) bool { - return sw.writeObject(id, o, fields, false) - }) + s.cursor = sw.col.Scan(s.cursor, stype, + func(id string, o geojson.Object, fields []float64) bool { + return sw.writeObject(id, o, fields, false) + }, + ) } } } diff --git a/controller/stats.go b/controller/stats.go index c2d8f33c..5b34ceba 100644 --- a/controller/stats.go +++ b/controller/stats.go @@ -9,6 +9,7 @@ import ( "github.com/tidwall/btree" "github.com/tidwall/resp" + "github.com/tidwall/tile38/controller/collection" "github.com/tidwall/tile38/controller/server" ) @@ -33,7 +34,7 @@ func (c *Controller) cmdStats(msg *server.Message) (res string, err error) { points := col.PointCount() m["num_points"] = points m["in_memory_size"] = col.TotalWeight() - m["num_objects"] = col.Count() + m["num_objects"] = col.Count(collection.TypeAll) switch msg.OutputType { case server.JSON: ms = append(ms, m) @@ -92,7 +93,7 @@ func (c *Controller) cmdServer(msg *server.Message) (res string, err error) { c.cols.Ascend(func(item btree.Item) bool { col := item.(*collectionT).Collection points += col.PointCount() - objects += col.Count() + objects += col.Count(collection.TypeAll) return true }) m["num_points"] = points @@ -155,7 +156,7 @@ func (c *Controller) statsCollections(line string) (string, error) { points := col.PointCount() m["num_points"] = points m["in_memory_size"] = col.TotalWeight() - m["num_objects"] = col.Count() + m["num_objects"] = col.Count(collection.TypeAll) ms = append(ms, m) } else { ms = append(ms, nil) diff --git a/core/commands.json b/core/commands.json index a84e4f74..d41b97a5 100644 --- a/core/commands.json +++ b/core/commands.json @@ -77,6 +77,15 @@ "type": "geohash" } ] + }, + { + "name": "STRING", + "arguments":[ + { + "name": "value", + "type": "string" + } + ] } ] } @@ -963,4 +972,4 @@ ], "group": "webhook" } -} \ No newline at end of file +} diff --git a/core/commands_gen.go b/core/commands_gen.go index e4b41ffc..92f04275 100644 --- a/core/commands_gen.go +++ b/core/commands_gen.go @@ -239,6 +239,15 @@ var commandsJSON = `{ "type": "geohash" } ] + }, + { + "name": "STRING", + "arguments":[ + { + "name": "value", + "type": "string" + } + ] } ] }