mirror
/
tile38
mirror of https://github.com/tidwall/tile38.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

added string type

This commit is contained in:
Josh Baker 2016-07-10 13:23:50 -07:00
parent 8d89198eaf
commit fbe19564b9
9 changed files with 166 additions and 114 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
controller/aofshrink.go
View File

@ -150,13 +150,15 @@ func (c *Controller) aofshrink() {
objs := make(map[string]objFields)
c.mu.Lock()
fnames := col.FieldArr() // reload an array of field names to match each object
col.ScanGreaterOrEqual(nextID, 0, func(id string, obj geojson.Object, fields []float64) bool {
if id != nextID {
objs[id] = objFields{obj, fields}
nextID = id
}
return len(objs) < maxIDGroup
})
col.ScanGreaterOrEqual(nextID, 0, collection.TypeAll,
func(id string, obj geojson.Object, fields []float64) bool {
if id != nextID {
objs[id] = objFields{obj, fields}
nextID = id
}
return len(objs) < maxIDGroup
},
)
c.mu.Unlock()
ids := make([]string, 0, maxIDGroup)
@ -177,7 +179,11 @@ func (c *Controller) aofshrink() {
}
switch obj := obj.obj.(type) {
default:
values = append(values, resp.StringValue("object"), resp.StringValue(obj.JSON()))
if obj.IsGeometry() {
values = append(values, resp.StringValue("object"), resp.StringValue(obj.JSON()))
} else {
values = append(values, resp.StringValue("string"), resp.StringValue(obj.String()))
}
case geojson.SimplePoint:
values = append(values, resp.StringValue("point"), resp.FloatValue(obj.Y), resp.FloatValue(obj.X))
case geojson.Point:

66
controller/collection/collection.go
View File

@ -6,9 +6,21 @@ import (
"github.com/tidwall/tile38/index"
)
// ScanType is the classification of objects that are returned from Scan()
type ScanType int
const (
// TypeAll means to return all type during a Scan()
TypeAll = ScanType(0)
// TypeGeometry means to return only geometries
TypeGeometry = ScanType(1)
// TypeNonGeometry means to return non-geometries
TypeNonGeometry = ScanType(2)
)
const (
idOrdered = 0
valueOrdered = 2
valueOrdered = 1
)
type itemT struct {
@ -24,12 +36,14 @@ func (i *itemT) Less(item btree.Item, ctx int) bool {
case idOrdered:
return i.id < item.(*itemT).id
case valueOrdered:
if i.object.String() < item.(*itemT).object.String() {
i1, i2 := i.object.String(), item.(*itemT).object.String()
if i1 < i2 {
return true
}
if i.object.String() > item.(*itemT).object.String() {
if i1 > i2 {
return false
}
// the values match so we will compare the ids, which are always unique.
return i.id < item.(*itemT).id
}
@ -53,7 +67,8 @@ type Collection struct {
fieldMap map[string]int
weight int
points int
objects int
objects int // geometry count
nobjects int // non-geometry count
}
var counter uint64
@ -70,8 +85,15 @@ func New() *Collection {
}
// Count returns the number of objects in collection.
func (c *Collection) Count() int {
return c.objects
func (c *Collection) Count(stype ScanType) int {
switch stype {
default:
return c.objects + c.nobjects
case TypeGeometry:
return c.objects
case TypeNonGeometry:
return c.nobjects
}
}
// PointCount returns the number of points (lat/lon coordinates) in collection.
@ -120,13 +142,14 @@ func (c *Collection) remove(id string) (item *itemT, ok bool) {
item = i.(*itemT)
if item.object.IsGeometry() {
c.index.Remove(item)
c.objects--
} else {
c.values.Delete(item)
c.nobjects--
}
c.weight -= len(item.fields) * 8
c.weight -= item.object.Weight() + len(item.id)
c.points -= item.object.PositionCount()
c.objects--
return item, true
}
@ -134,13 +157,14 @@ func (c *Collection) insert(id string, obj geojson.Object) (item *itemT) {
item = &itemT{id: id, object: obj}
if obj.IsGeometry() {
c.index.Insert(item)
c.objects++
} else {
c.values.ReplaceOrInsert(item)
c.nobjects++
}
c.items.ReplaceOrInsert(item)
c.weight += obj.Weight() + len(id)
c.points += obj.PositionCount()
c.objects++
return item
}
@ -213,7 +237,7 @@ func (c *Collection) FieldArr() []string {
}
// Scan iterates though the collection. A cursor can be used for paging.
func (c *Collection) Scan(cursor uint64, iterator func(id string, obj geojson.Object, fields []float64) bool) (ncursor uint64) {
func (c *Collection) Scan(cursor uint64, stype ScanType, iterator func(id string, obj geojson.Object, fields []float64) bool) (ncursor uint64) {
var i uint64
var active = true
c.items.Ascend(func(item btree.Item) bool {
@ -228,7 +252,7 @@ func (c *Collection) Scan(cursor uint64, iterator func(id string, obj geojson.Ob
}
// ScanGreaterOrEqual iterates though the collection starting with specified id. A cursor can be used for paging.
func (c *Collection) ScanGreaterOrEqual(id string, cursor uint64, iterator func(id string, obj geojson.Object, fields []float64) bool) (ncursor uint64) {
func (c *Collection) ScanGreaterOrEqual(id string, cursor uint64, stype ScanType, iterator func(id string, obj geojson.Object, fields []float64) bool) (ncursor uint64) {
var i uint64
var active = true
c.items.AscendGreaterOrEqual(&itemT{id: id}, func(item btree.Item) bool {
@ -242,7 +266,7 @@ func (c *Collection) ScanGreaterOrEqual(id string, cursor uint64, iterator func(
return i
}
func (c *Collection) search(cursor uint64, bbox geojson.BBox, iterator func(id string, obj geojson.Object, fields []float64) bool) (ncursor uint64) {
func (c *Collection) geoSearch(cursor uint64, bbox geojson.BBox, iterator func(id string, obj geojson.Object, fields []float64) bool) (ncursor uint64) {
return c.index.Search(cursor, bbox.Min.Y, bbox.Min.X, bbox.Max.Y, bbox.Max.X, func(item index.Item) bool {
var iitm *itemT
iitm, ok := item.(*itemT)
@ -263,7 +287,7 @@ func (c *Collection) Nearby(cursor uint64, sparse uint8, lat, lon, meters float6
bboxes := bbox.Sparse(sparse)
if sparse > 0 {
for _, bbox := range bboxes {
c.search(cursor, bbox, func(id string, obj geojson.Object, fields []float64) bool {
c.geoSearch(cursor, bbox, func(id string, obj geojson.Object, fields []float64) bool {
if obj.Nearby(center, meters) {
if iterator(id, obj, fields) {
return false
@ -274,7 +298,7 @@ func (c *Collection) Nearby(cursor uint64, sparse uint8, lat, lon, meters float6
}
return 0
}
return c.search(cursor, bbox, func(id string, obj geojson.Object, fields []float64) bool {
return c.geoSearch(cursor, bbox, func(id string, obj geojson.Object, fields []float64) bool {
if obj.Nearby(center, meters) {
return iterator(id, obj, fields)
}
@ -294,7 +318,7 @@ func (c *Collection) Within(cursor uint64, sparse uint8, obj geojson.Object, min
if sparse > 0 {
for _, bbox := range bboxes {
if obj != nil {
c.search(cursor, bbox, func(id string, o geojson.Object, fields []float64) bool {
c.geoSearch(cursor, bbox, func(id string, o geojson.Object, fields []float64) bool {
if o.Within(obj) {
if iterator(id, o, fields) {
return false
@ -303,7 +327,7 @@ func (c *Collection) Within(cursor uint64, sparse uint8, obj geojson.Object, min
return true
})
}
c.search(cursor, bbox, func(id string, o geojson.Object, fields []float64) bool {
c.geoSearch(cursor, bbox, func(id string, o geojson.Object, fields []float64) bool {
if o.WithinBBox(bbox) {
if iterator(id, o, fields) {
return false
@ -315,14 +339,14 @@ func (c *Collection) Within(cursor uint64, sparse uint8, obj geojson.Object, min
return 0
}
if obj != nil {
return c.search(cursor, bbox, func(id string, o geojson.Object, fields []float64) bool {
return c.geoSearch(cursor, bbox, func(id string, o geojson.Object, fields []float64) bool {
if o.Within(obj) {
return iterator(id, o, fields)
}
return true
})
}
return c.search(cursor, bbox, func(id string, o geojson.Object, fields []float64) bool {
return c.geoSearch(cursor, bbox, func(id string, o geojson.Object, fields []float64) bool {
if o.WithinBBox(bbox) {
return iterator(id, o, fields)
}
@ -353,7 +377,7 @@ func (c *Collection) Intersects(cursor uint64, sparse uint8, obj geojson.Object,
}
for _, bbox := range bboxes {
if obj != nil {
c.search(cursor, bbox, func(id string, o geojson.Object, fields []float64) bool {
c.geoSearch(cursor, bbox, func(id string, o geojson.Object, fields []float64) bool {
if o.Intersects(obj) {
if iterator(id, o, fields) {
return false
@ -362,7 +386,7 @@ func (c *Collection) Intersects(cursor uint64, sparse uint8, obj geojson.Object,
return true
})
}
c.search(cursor, bbox, func(id string, o geojson.Object, fields []float64) bool {
c.geoSearch(cursor, bbox, func(id string, o geojson.Object, fields []float64) bool {
if o.IntersectsBBox(bbox) {
if iterator(id, o, fields) {
return false
@ -374,14 +398,14 @@ func (c *Collection) Intersects(cursor uint64, sparse uint8, obj geojson.Object,
return 0
}
if obj != nil {
return c.search(cursor, bbox, func(id string, o geojson.Object, fields []float64) bool {
return c.geoSearch(cursor, bbox, func(id string, o geojson.Object, fields []float64) bool {
if o.Intersects(obj) {
return iterator(id, o, fields)
}
return true
})
}
return c.search(cursor, bbox, func(id string, o geojson.Object, fields []float64) bool {
return c.geoSearch(cursor, bbox, func(id string, o geojson.Object, fields []float64) bool {
if o.IntersectsBBox(bbox) {
return iterator(id, o, fields)
}

6
controller/collection/collection_test.go
View File

@ -31,14 +31,14 @@ func TestCollection(t *testing.T) {
}
count := 0
bbox := geojson.BBox{Min: geojson.Position{X: -180, Y: -90, Z: 0}, Max: geojson.Position{X: 180, Y: 90, Z: 0}}
c.search(0, bbox, func(id string, obj geojson.Object, field []float64) bool {
c.geoSearch(0, bbox, func(id string, obj geojson.Object, field []float64) bool {
count++
return true
})
if count != len(objs) {
t.Fatalf("count = %d, expect %d", count, len(objs))
}
count = c.Count()
count = c.Count(TypeAll)
if count != len(objs) {
t.Fatalf("c.Count() = %d, expect %d", count, len(objs))
}
@ -89,7 +89,7 @@ func TestManyCollections(t *testing.T) {
col := colsM["13"]
//println(col.Count())
bbox := geojson.BBox{Min: geojson.Position{X: -180, Y: 30, Z: 0}, Max: geojson.Position{X: 34, Y: 100, Z: 0}}
col.search(0, bbox, func(id string, obj geojson.Object, fields []float64) bool {
col.geoSearch(0, bbox, func(id string, obj geojson.Object, fields []float64) bool {
//println(id)
return true
})

120
controller/crud.go
View File

@ -87,75 +87,79 @@ func (c *Controller) cmdGet(msg *server.Message) (string, error) {
if msg.OutputType == server.JSON {
buf.WriteString(`{"ok":true`)
}
if vs, typ, ok = tokenval(vs); !ok || strings.ToLower(typ) == "object" {
vs, typ, ok = tokenval(vs)
typ = strings.ToLower(typ)
if !ok {
typ = "object"
}
switch typ {
default:
return "", errInvalidArgument(typ)
case "object":
if msg.OutputType == server.JSON {
buf.WriteString(`,"object":`)
buf.WriteString(o.JSON())
} else {
vals = append(vals, resp.StringValue(o.String()))
}
} else {
switch strings.ToLower(typ) {
default:
return "", errInvalidArgument(typ)
case "point":
point := o.CalculatedPoint()
if msg.OutputType == server.JSON {
buf.WriteString(`,"point":`)
buf.WriteString(point.ExternalJSON())
} else {
if point.Z != 0 {
vals = append(vals, resp.ArrayValue([]resp.Value{
resp.StringValue(strconv.FormatFloat(point.Y, 'f', -1, 64)),
resp.StringValue(strconv.FormatFloat(point.X, 'f', -1, 64)),
resp.StringValue(strconv.FormatFloat(point.Z, 'f', -1, 64)),
}))
} else {
vals = append(vals, resp.ArrayValue([]resp.Value{
resp.StringValue(strconv.FormatFloat(point.Y, 'f', -1, 64)),
resp.StringValue(strconv.FormatFloat(point.X, 'f', -1, 64)),
}))
}
}
case "hash":
if vs, sprecision, ok = tokenval(vs); !ok || sprecision == "" {
return "", errInvalidNumberOfArguments
}
if msg.OutputType == server.JSON {
buf.WriteString(`,"hash":`)
}
precision, err := strconv.ParseInt(sprecision, 10, 64)
if err != nil || precision < 1 || precision > 64 {
return "", errInvalidArgument(sprecision)
}
p, err := o.Geohash(int(precision))
if err != nil {
return "", err
}
if msg.OutputType == server.JSON {
buf.WriteString(`"` + p + `"`)
} else {
vals = append(vals, resp.StringValue(p))
}
case "bounds":
bbox := o.CalculatedBBox()
if msg.OutputType == server.JSON {
buf.WriteString(`,"bounds":`)
buf.WriteString(bbox.ExternalJSON())
case "point":
point := o.CalculatedPoint()
if msg.OutputType == server.JSON {
buf.WriteString(`,"point":`)
buf.WriteString(point.ExternalJSON())
} else {
if point.Z != 0 {
vals = append(vals, resp.ArrayValue([]resp.Value{
resp.StringValue(strconv.FormatFloat(point.Y, 'f', -1, 64)),
resp.StringValue(strconv.FormatFloat(point.X, 'f', -1, 64)),
resp.StringValue(strconv.FormatFloat(point.Z, 'f', -1, 64)),
}))
} else {
vals = append(vals, resp.ArrayValue([]resp.Value{
resp.ArrayValue([]resp.Value{
resp.FloatValue(bbox.Min.Y),
resp.FloatValue(bbox.Min.X),
}),
resp.ArrayValue([]resp.Value{
resp.FloatValue(bbox.Max.Y),
resp.FloatValue(bbox.Max.X),
}),
resp.StringValue(strconv.FormatFloat(point.Y, 'f', -1, 64)),
resp.StringValue(strconv.FormatFloat(point.X, 'f', -1, 64)),
}))
}
}
case "hash":
if vs, sprecision, ok = tokenval(vs); !ok || sprecision == "" {
return "", errInvalidNumberOfArguments
}
if msg.OutputType == server.JSON {
buf.WriteString(`,"hash":`)
}
precision, err := strconv.ParseInt(sprecision, 10, 64)
if err != nil || precision < 1 || precision > 64 {
return "", errInvalidArgument(sprecision)
}
p, err := o.Geohash(int(precision))
if err != nil {
return "", err
}
if msg.OutputType == server.JSON {
buf.WriteString(`"` + p + `"`)
} else {
vals = append(vals, resp.StringValue(p))
}
case "bounds":
bbox := o.CalculatedBBox()
if msg.OutputType == server.JSON {
buf.WriteString(`,"bounds":`)
buf.WriteString(bbox.ExternalJSON())
} else {
vals = append(vals, resp.ArrayValue([]resp.Value{
resp.ArrayValue([]resp.Value{
resp.FloatValue(bbox.Min.Y),
resp.FloatValue(bbox.Min.X),
}),
resp.ArrayValue([]resp.Value{
resp.FloatValue(bbox.Max.Y),
resp.FloatValue(bbox.Max.X),
}),
}))
}
}
if len(vs) != 0 {
return "", errInvalidNumberOfArguments
}
@ -225,7 +229,7 @@ func (c *Controller) cmdDel(msg *server.Message) (res string, d commandDetailsT,
if col != nil {
d.obj, d.fields, ok = col.Remove(d.id)
if ok {
if col.Count() == 0 {
if col.Count(collection.TypeAll) == 0 {
c.deleteCol(d.key)
d.revert = func() {
c.setCol(d.key, col)

2
controller/fence.go
View File

@ -30,7 +30,7 @@ func FenceMatch(hookName string, sw *scanWriter, fence *liveFenceSwitches, detai
nofields := sw.nofields
sw.mu.Unlock()
if details.obj == nil || (details.command == "fset" && nofields) {
if details.obj == nil || !details.obj.IsGeometry() || (details.command == "fset" && nofields) {
return nil
}
match = false

37
controller/scan.go
View File

@ -6,6 +6,7 @@ import (
"time"
"github.com/tidwall/resp"
"github.com/tidwall/tile38/controller/collection"
"github.com/tidwall/tile38/controller/server"
"github.com/tidwall/tile38/geojson"
)
@ -34,22 +35,14 @@ func (c *Controller) cmdScan(msg *server.Message) (res string, err error) {
if err != nil {
return "", err
}
if s.sparse > 0 && sw.col != nil {
msg.Values = append(msg.Values,
resp.StringValue("BOUNDS"),
resp.StringValue("-90"),
resp.StringValue("-180"),
resp.StringValue("180"),
)
return c.cmdWithinOrIntersects("within", msg)
}
if msg.OutputType == server.JSON {
wr.WriteString(`{"ok":true`)
}
sw.writeHead()
if sw.col != nil {
stype := collection.TypeAll
if sw.output == outputCount && len(sw.wheres) == 0 && sw.globEverything == true {
count := sw.col.Count() - int(s.cursor)
count := sw.col.Count(stype) - int(s.cursor)
if count < 0 {
count = 0
}
@ -58,18 +51,24 @@ func (c *Controller) cmdScan(msg *server.Message) (res string, err error) {
if strings.HasSuffix(sw.glob, "*") {
greaterGlob := sw.glob[:len(sw.glob)-1]
if globIsGlob(greaterGlob) {
s.cursor = sw.col.Scan(s.cursor, func(id string, o geojson.Object, fields []float64) bool {
return sw.writeObject(id, o, fields, false)
})
s.cursor = sw.col.Scan(s.cursor, stype,
func(id string, o geojson.Object, fields []float64) bool {
return sw.writeObject(id, o, fields, false)
},
)
} else {
s.cursor = sw.col.ScanGreaterOrEqual(greaterGlob, s.cursor, func(id string, o geojson.Object, fields []float64) bool {
return sw.writeObject(id, o, fields, false)
})
s.cursor = sw.col.ScanGreaterOrEqual(greaterGlob, s.cursor, stype,
func(id string, o geojson.Object, fields []float64) bool {
return sw.writeObject(id, o, fields, false)
},
)
}
} else {
s.cursor = sw.col.Scan(s.cursor, func(id string, o geojson.Object, fields []float64) bool {
return sw.writeObject(id, o, fields, false)
})
s.cursor = sw.col.Scan(s.cursor, stype,
func(id string, o geojson.Object, fields []float64) bool {
return sw.writeObject(id, o, fields, false)
},
)
}
}
}

7
controller/stats.go
View File

@ -9,6 +9,7 @@ import (
"github.com/tidwall/btree"
"github.com/tidwall/resp"
"github.com/tidwall/tile38/controller/collection"
"github.com/tidwall/tile38/controller/server"
)
@ -33,7 +34,7 @@ func (c *Controller) cmdStats(msg *server.Message) (res string, err error) {
points := col.PointCount()
m["num_points"] = points
m["in_memory_size"] = col.TotalWeight()
m["num_objects"] = col.Count()
m["num_objects"] = col.Count(collection.TypeAll)
switch msg.OutputType {
case server.JSON:
ms = append(ms, m)
@ -92,7 +93,7 @@ func (c *Controller) cmdServer(msg *server.Message) (res string, err error) {
c.cols.Ascend(func(item btree.Item) bool {
col := item.(*collectionT).Collection
points += col.PointCount()
objects += col.Count()
objects += col.Count(collection.TypeAll)
return true
})
m["num_points"] = points
@ -155,7 +156,7 @@ func (c *Controller) statsCollections(line string) (string, error) {
points := col.PointCount()
m["num_points"] = points
m["in_memory_size"] = col.TotalWeight()
m["num_objects"] = col.Count()
m["num_objects"] = col.Count(collection.TypeAll)
ms = append(ms, m)
} else {
ms = append(ms, nil)

11
core/commands.json
View File

@ -77,6 +77,15 @@
"type": "geohash"
}
]
},
{
"name": "STRING",
"arguments":[
{
"name": "value",
"type": "string"
}
]
}
]
}
@ -963,4 +972,4 @@
],
"group": "webhook"
}
}
}

9
core/commands_gen.go
View File

@ -239,6 +239,15 @@ var commandsJSON = `{
"type": "geohash"
}
]
},
{
"name": "STRING",
"arguments":[
{
"name": "value",
"type": "string"
}
]
}
]
}