mirror
/
tile38
mirror of https://github.com/tidwall/tile38.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

added string type

This commit is contained in:
Josh Baker 2016-07-10 13:23:50 -07:00
parent 8d89198eaf
commit fbe19564b9
9 changed files with 166 additions and 114 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
controller/aofshrink.go
View File

@ -150,13 +150,15 @@ func (c *Controller) aofshrink() {
objs := make(map[string]objFields) objs := make(map[string]objFields)
c.mu.Lock() c.mu.Lock()
fnames := col.FieldArr() // reload an array of field names to match each object fnames := col.FieldArr() // reload an array of field names to match each object
col.ScanGreaterOrEqual(nextID, 0, func(id string, obj geojson.Object, fields []float64) bool { col.ScanGreaterOrEqual(nextID, 0, collection.TypeAll,
func(id string, obj geojson.Object, fields []float64) bool {
if id != nextID { if id != nextID {
objs[id] = objFields{obj, fields} objs[id] = objFields{obj, fields}
nextID = id nextID = id
} }
return len(objs) < maxIDGroup return len(objs) < maxIDGroup
}) },
)
c.mu.Unlock() c.mu.Unlock()
ids := make([]string, 0, maxIDGroup) ids := make([]string, 0, maxIDGroup)
@ -177,7 +179,11 @@ func (c *Controller) aofshrink() {
} }
switch obj := obj.obj.(type) { switch obj := obj.obj.(type) {
default: default:
if obj.IsGeometry() {
values = append(values, resp.StringValue("object"), resp.StringValue(obj.JSON())) values = append(values, resp.StringValue("object"), resp.StringValue(obj.JSON()))
} else {
values = append(values, resp.StringValue("string"), resp.StringValue(obj.String()))
}
case geojson.SimplePoint: case geojson.SimplePoint:
values = append(values, resp.StringValue("point"), resp.FloatValue(obj.Y), resp.FloatValue(obj.X)) values = append(values, resp.StringValue("point"), resp.FloatValue(obj.Y), resp.FloatValue(obj.X))
case geojson.Point: case geojson.Point:

64
controller/collection/collection.go
View File

@ -6,9 +6,21 @@ import (
"github.com/tidwall/tile38/index" "github.com/tidwall/tile38/index"
) )
// ScanType is the classification of objects that are returned from Scan()
type ScanType int
const (
// TypeAll means to return all type during a Scan()
TypeAll = ScanType(0)
// TypeGeometry means to return only geometries
TypeGeometry = ScanType(1)
// TypeNonGeometry means to return non-geometries
TypeNonGeometry = ScanType(2)
)
const ( const (
idOrdered = 0 idOrdered = 0
valueOrdered = 2 valueOrdered = 1
) )
type itemT struct { type itemT struct {
@ -24,12 +36,14 @@ func (i *itemT) Less(item btree.Item, ctx int) bool {
case idOrdered: case idOrdered:
return i.id < item.(*itemT).id return i.id < item.(*itemT).id
case valueOrdered: case valueOrdered:
if i.object.String() < item.(*itemT).object.String() { i1, i2 := i.object.String(), item.(*itemT).object.String()
if i1 < i2 {
return true return true
} }
if i.object.String() > item.(*itemT).object.String() { if i1 > i2 {
return false return false
} }
// the values match so we will compare the ids, which are always unique.
return i.id < item.(*itemT).id return i.id < item.(*itemT).id
} }
@ -53,7 +67,8 @@ type Collection struct {
fieldMap map[string]int fieldMap map[string]int
weight int weight int
points int points int
objects int objects int // geometry count
nobjects int // non-geometry count
} }
var counter uint64 var counter uint64
@ -70,8 +85,15 @@ func New() *Collection {
} }
// Count returns the number of objects in collection. // Count returns the number of objects in collection.
func (c *Collection) Count() int { func (c *Collection) Count(stype ScanType) int {
switch stype {
default:
return c.objects + c.nobjects
case TypeGeometry:
return c.objects return c.objects
case TypeNonGeometry:
return c.nobjects
}
} }
// PointCount returns the number of points (lat/lon coordinates) in collection. // PointCount returns the number of points (lat/lon coordinates) in collection.
@ -120,13 +142,14 @@ func (c *Collection) remove(id string) (item *itemT, ok bool) {
item = i.(*itemT) item = i.(*itemT)
if item.object.IsGeometry() { if item.object.IsGeometry() {
c.index.Remove(item) c.index.Remove(item)
c.objects--
} else { } else {
c.values.Delete(item) c.values.Delete(item)
c.nobjects--
} }
c.weight -= len(item.fields) * 8 c.weight -= len(item.fields) * 8
c.weight -= item.object.Weight() + len(item.id) c.weight -= item.object.Weight() + len(item.id)
c.points -= item.object.PositionCount() c.points -= item.object.PositionCount()
c.objects--
return item, true return item, true
} }
@ -134,13 +157,14 @@ func (c *Collection) insert(id string, obj geojson.Object) (item *itemT) {
item = &itemT{id: id, object: obj} item = &itemT{id: id, object: obj}
if obj.IsGeometry() { if obj.IsGeometry() {
c.index.Insert(item) c.index.Insert(item)
c.objects++
} else { } else {
c.values.ReplaceOrInsert(item) c.values.ReplaceOrInsert(item)
c.nobjects++
} }
c.items.ReplaceOrInsert(item) c.items.ReplaceOrInsert(item)
c.weight += obj.Weight() + len(id) c.weight += obj.Weight() + len(id)
c.points += obj.PositionCount() c.points += obj.PositionCount()
c.objects++
return item return item
} }
@ -213,7 +237,7 @@ func (c *Collection) FieldArr() []string {
} }
// Scan iterates though the collection. A cursor can be used for paging. // Scan iterates though the collection. A cursor can be used for paging.
func (c *Collection) Scan(cursor uint64, iterator func(id string, obj geojson.Object, fields []float64) bool) (ncursor uint64) { func (c *Collection) Scan(cursor uint64, stype ScanType, iterator func(id string, obj geojson.Object, fields []float64) bool) (ncursor uint64) {
var i uint64 var i uint64
var active = true var active = true
c.items.Ascend(func(item btree.Item) bool { c.items.Ascend(func(item btree.Item) bool {
@ -228,7 +252,7 @@ func (c *Collection) Scan(cursor uint64, iterator func(id string, obj geojson.Ob
} }
// ScanGreaterOrEqual iterates though the collection starting with specified id. A cursor can be used for paging. // ScanGreaterOrEqual iterates though the collection starting with specified id. A cursor can be used for paging.
func (c *Collection) ScanGreaterOrEqual(id string, cursor uint64, iterator func(id string, obj geojson.Object, fields []float64) bool) (ncursor uint64) { func (c *Collection) ScanGreaterOrEqual(id string, cursor uint64, stype ScanType, iterator func(id string, obj geojson.Object, fields []float64) bool) (ncursor uint64) {
var i uint64 var i uint64
var active = true var active = true
c.items.AscendGreaterOrEqual(&itemT{id: id}, func(item btree.Item) bool { c.items.AscendGreaterOrEqual(&itemT{id: id}, func(item btree.Item) bool {
@ -242,7 +266,7 @@ func (c *Collection) ScanGreaterOrEqual(id string, cursor uint64, iterator func(
return i return i
} }
func (c *Collection) search(cursor uint64, bbox geojson.BBox, iterator func(id string, obj geojson.Object, fields []float64) bool) (ncursor uint64) { func (c *Collection) geoSearch(cursor uint64, bbox geojson.BBox, iterator func(id string, obj geojson.Object, fields []float64) bool) (ncursor uint64) {
return c.index.Search(cursor, bbox.Min.Y, bbox.Min.X, bbox.Max.Y, bbox.Max.X, func(item index.Item) bool { return c.index.Search(cursor, bbox.Min.Y, bbox.Min.X, bbox.Max.Y, bbox.Max.X, func(item index.Item) bool {
var iitm *itemT var iitm *itemT
iitm, ok := item.(*itemT) iitm, ok := item.(*itemT)
@ -263,7 +287,7 @@ func (c *Collection) Nearby(cursor uint64, sparse uint8, lat, lon, meters float6
bboxes := bbox.Sparse(sparse) bboxes := bbox.Sparse(sparse)
if sparse > 0 { if sparse > 0 {
for _, bbox := range bboxes { for _, bbox := range bboxes {
c.search(cursor, bbox, func(id string, obj geojson.Object, fields []float64) bool { c.geoSearch(cursor, bbox, func(id string, obj geojson.Object, fields []float64) bool {
if obj.Nearby(center, meters) { if obj.Nearby(center, meters) {
if iterator(id, obj, fields) { if iterator(id, obj, fields) {
return false return false
@ -274,7 +298,7 @@ func (c *Collection) Nearby(cursor uint64, sparse uint8, lat, lon, meters float6
} }
return 0 return 0
} }
return c.search(cursor, bbox, func(id string, obj geojson.Object, fields []float64) bool { return c.geoSearch(cursor, bbox, func(id string, obj geojson.Object, fields []float64) bool {
if obj.Nearby(center, meters) { if obj.Nearby(center, meters) {
return iterator(id, obj, fields) return iterator(id, obj, fields)
} }
@ -294,7 +318,7 @@ func (c *Collection) Within(cursor uint64, sparse uint8, obj geojson.Object, min
if sparse > 0 { if sparse > 0 {
for _, bbox := range bboxes { for _, bbox := range bboxes {
if obj != nil { if obj != nil {
c.search(cursor, bbox, func(id string, o geojson.Object, fields []float64) bool { c.geoSearch(cursor, bbox, func(id string, o geojson.Object, fields []float64) bool {
if o.Within(obj) { if o.Within(obj) {
if iterator(id, o, fields) { if iterator(id, o, fields) {
return false return false
@ -303,7 +327,7 @@ func (c *Collection) Within(cursor uint64, sparse uint8, obj geojson.Object, min
return true return true
}) })
} }
c.search(cursor, bbox, func(id string, o geojson.Object, fields []float64) bool { c.geoSearch(cursor, bbox, func(id string, o geojson.Object, fields []float64) bool {
if o.WithinBBox(bbox) { if o.WithinBBox(bbox) {
if iterator(id, o, fields) { if iterator(id, o, fields) {
return false return false
@ -315,14 +339,14 @@ func (c *Collection) Within(cursor uint64, sparse uint8, obj geojson.Object, min
return 0 return 0
} }
if obj != nil { if obj != nil {
return c.search(cursor, bbox, func(id string, o geojson.Object, fields []float64) bool { return c.geoSearch(cursor, bbox, func(id string, o geojson.Object, fields []float64) bool {
if o.Within(obj) { if o.Within(obj) {
return iterator(id, o, fields) return iterator(id, o, fields)
} }
return true return true
}) })
} }
return c.search(cursor, bbox, func(id string, o geojson.Object, fields []float64) bool { return c.geoSearch(cursor, bbox, func(id string, o geojson.Object, fields []float64) bool {
if o.WithinBBox(bbox) { if o.WithinBBox(bbox) {
return iterator(id, o, fields) return iterator(id, o, fields)
} }
@ -353,7 +377,7 @@ func (c *Collection) Intersects(cursor uint64, sparse uint8, obj geojson.Object,
} }
for _, bbox := range bboxes { for _, bbox := range bboxes {
if obj != nil { if obj != nil {
c.search(cursor, bbox, func(id string, o geojson.Object, fields []float64) bool { c.geoSearch(cursor, bbox, func(id string, o geojson.Object, fields []float64) bool {
if o.Intersects(obj) { if o.Intersects(obj) {
if iterator(id, o, fields) { if iterator(id, o, fields) {
return false return false
@ -362,7 +386,7 @@ func (c *Collection) Intersects(cursor uint64, sparse uint8, obj geojson.Object,
return true return true
}) })
} }
c.search(cursor, bbox, func(id string, o geojson.Object, fields []float64) bool { c.geoSearch(cursor, bbox, func(id string, o geojson.Object, fields []float64) bool {
if o.IntersectsBBox(bbox) { if o.IntersectsBBox(bbox) {
if iterator(id, o, fields) { if iterator(id, o, fields) {
return false return false
@ -374,14 +398,14 @@ func (c *Collection) Intersects(cursor uint64, sparse uint8, obj geojson.Object,
return 0 return 0
} }
if obj != nil { if obj != nil {
return c.search(cursor, bbox, func(id string, o geojson.Object, fields []float64) bool { return c.geoSearch(cursor, bbox, func(id string, o geojson.Object, fields []float64) bool {
if o.Intersects(obj) { if o.Intersects(obj) {
return iterator(id, o, fields) return iterator(id, o, fields)
} }
return true return true
}) })
} }
return c.search(cursor, bbox, func(id string, o geojson.Object, fields []float64) bool { return c.geoSearch(cursor, bbox, func(id string, o geojson.Object, fields []float64) bool {
if o.IntersectsBBox(bbox) { if o.IntersectsBBox(bbox) {
return iterator(id, o, fields) return iterator(id, o, fields)
} }

6
controller/collection/collection_test.go
View File

@ -31,14 +31,14 @@ func TestCollection(t *testing.T) {
} }
count := 0 count := 0
bbox := geojson.BBox{Min: geojson.Position{X: -180, Y: -90, Z: 0}, Max: geojson.Position{X: 180, Y: 90, Z: 0}} bbox := geojson.BBox{Min: geojson.Position{X: -180, Y: -90, Z: 0}, Max: geojson.Position{X: 180, Y: 90, Z: 0}}
c.search(0, bbox, func(id string, obj geojson.Object, field []float64) bool { c.geoSearch(0, bbox, func(id string, obj geojson.Object, field []float64) bool {
count++ count++
return true return true
}) })
if count != len(objs) { if count != len(objs) {
t.Fatalf("count = %d, expect %d", count, len(objs)) t.Fatalf("count = %d, expect %d", count, len(objs))
} }
count = c.Count() count = c.Count(TypeAll)
if count != len(objs) { if count != len(objs) {
t.Fatalf("c.Count() = %d, expect %d", count, len(objs)) t.Fatalf("c.Count() = %d, expect %d", count, len(objs))
} }
@ -89,7 +89,7 @@ func TestManyCollections(t *testing.T) {
col := colsM["13"] col := colsM["13"]
//println(col.Count()) //println(col.Count())
bbox := geojson.BBox{Min: geojson.Position{X: -180, Y: 30, Z: 0}, Max: geojson.Position{X: 34, Y: 100, Z: 0}} bbox := geojson.BBox{Min: geojson.Position{X: -180, Y: 30, Z: 0}, Max: geojson.Position{X: 34, Y: 100, Z: 0}}
col.search(0, bbox, func(id string, obj geojson.Object, fields []float64) bool { col.geoSearch(0, bbox, func(id string, obj geojson.Object, fields []float64) bool {
//println(id) //println(id)
return true return true
}) })

18
controller/crud.go
View File

@ -87,17 +87,21 @@ func (c *Controller) cmdGet(msg *server.Message) (string, error) {
if msg.OutputType == server.JSON { if msg.OutputType == server.JSON {
buf.WriteString(`{"ok":true`) buf.WriteString(`{"ok":true`)
} }
if vs, typ, ok = tokenval(vs); !ok || strings.ToLower(typ) == "object" { vs, typ, ok = tokenval(vs)
typ = strings.ToLower(typ)
if !ok {
typ = "object"
}
switch typ {
default:
return "", errInvalidArgument(typ)
case "object":
if msg.OutputType == server.JSON { if msg.OutputType == server.JSON {
buf.WriteString(`,"object":`) buf.WriteString(`,"object":`)
buf.WriteString(o.JSON()) buf.WriteString(o.JSON())
} else { } else {
vals = append(vals, resp.StringValue(o.String())) vals = append(vals, resp.StringValue(o.String()))
} }
} else {
switch strings.ToLower(typ) {
default:
return "", errInvalidArgument(typ)
case "point": case "point":
point := o.CalculatedPoint() point := o.CalculatedPoint()
if msg.OutputType == server.JSON { if msg.OutputType == server.JSON {
@ -155,7 +159,7 @@ func (c *Controller) cmdGet(msg *server.Message) (string, error) {
})) }))
} }
} }
}
if len(vs) != 0 { if len(vs) != 0 {
return "", errInvalidNumberOfArguments return "", errInvalidNumberOfArguments
} }
@ -225,7 +229,7 @@ func (c *Controller) cmdDel(msg *server.Message) (res string, d commandDetailsT,
if col != nil { if col != nil {
d.obj, d.fields, ok = col.Remove(d.id) d.obj, d.fields, ok = col.Remove(d.id)
if ok { if ok {
if col.Count() == 0 { if col.Count(collection.TypeAll) == 0 {
c.deleteCol(d.key) c.deleteCol(d.key)
d.revert = func() { d.revert = func() {
c.setCol(d.key, col) c.setCol(d.key, col)

2
controller/fence.go
View File

@ -30,7 +30,7 @@ func FenceMatch(hookName string, sw *scanWriter, fence *liveFenceSwitches, detai
nofields := sw.nofields nofields := sw.nofields
sw.mu.Unlock() sw.mu.Unlock()
if details.obj == nil || (details.command == "fset" && nofields) { if details.obj == nil || !details.obj.IsGeometry() || (details.command == "fset" && nofields) {
return nil return nil
} }
match = false match = false

31
controller/scan.go
View File

@ -6,6 +6,7 @@ import (
"time" "time"
"github.com/tidwall/resp" "github.com/tidwall/resp"
"github.com/tidwall/tile38/controller/collection"
"github.com/tidwall/tile38/controller/server" "github.com/tidwall/tile38/controller/server"
"github.com/tidwall/tile38/geojson" "github.com/tidwall/tile38/geojson"
) )
@ -34,22 +35,14 @@ func (c *Controller) cmdScan(msg *server.Message) (res string, err error) {
if err != nil { if err != nil {
return "", err return "", err
} }
if s.sparse > 0 && sw.col != nil {
msg.Values = append(msg.Values,
resp.StringValue("BOUNDS"),
resp.StringValue("-90"),
resp.StringValue("-180"),
resp.StringValue("180"),
)
return c.cmdWithinOrIntersects("within", msg)
}
if msg.OutputType == server.JSON { if msg.OutputType == server.JSON {
wr.WriteString(`{"ok":true`) wr.WriteString(`{"ok":true`)
} }
sw.writeHead() sw.writeHead()
if sw.col != nil { if sw.col != nil {
stype := collection.TypeAll
if sw.output == outputCount && len(sw.wheres) == 0 && sw.globEverything == true { if sw.output == outputCount && len(sw.wheres) == 0 && sw.globEverything == true {
count := sw.col.Count() - int(s.cursor) count := sw.col.Count(stype) - int(s.cursor)
if count < 0 { if count < 0 {
count = 0 count = 0
} }
@ -58,18 +51,24 @@ func (c *Controller) cmdScan(msg *server.Message) (res string, err error) {
if strings.HasSuffix(sw.glob, "*") { if strings.HasSuffix(sw.glob, "*") {
greaterGlob := sw.glob[:len(sw.glob)-1] greaterGlob := sw.glob[:len(sw.glob)-1]
if globIsGlob(greaterGlob) { if globIsGlob(greaterGlob) {
s.cursor = sw.col.Scan(s.cursor, func(id string, o geojson.Object, fields []float64) bool { s.cursor = sw.col.Scan(s.cursor, stype,
func(id string, o geojson.Object, fields []float64) bool {
return sw.writeObject(id, o, fields, false) return sw.writeObject(id, o, fields, false)
}) },
)
} else { } else {
s.cursor = sw.col.ScanGreaterOrEqual(greaterGlob, s.cursor, func(id string, o geojson.Object, fields []float64) bool { s.cursor = sw.col.ScanGreaterOrEqual(greaterGlob, s.cursor, stype,
func(id string, o geojson.Object, fields []float64) bool {
return sw.writeObject(id, o, fields, false) return sw.writeObject(id, o, fields, false)
}) },
)
} }
} else { } else {
s.cursor = sw.col.Scan(s.cursor, func(id string, o geojson.Object, fields []float64) bool { s.cursor = sw.col.Scan(s.cursor, stype,
func(id string, o geojson.Object, fields []float64) bool {
return sw.writeObject(id, o, fields, false) return sw.writeObject(id, o, fields, false)
}) },
)
} }
} }
} }

7
controller/stats.go
View File

@ -9,6 +9,7 @@ import (
"github.com/tidwall/btree" "github.com/tidwall/btree"
"github.com/tidwall/resp" "github.com/tidwall/resp"
"github.com/tidwall/tile38/controller/collection"
"github.com/tidwall/tile38/controller/server" "github.com/tidwall/tile38/controller/server"
) )
@ -33,7 +34,7 @@ func (c *Controller) cmdStats(msg *server.Message) (res string, err error) {
points := col.PointCount() points := col.PointCount()
m["num_points"] = points m["num_points"] = points
m["in_memory_size"] = col.TotalWeight() m["in_memory_size"] = col.TotalWeight()
m["num_objects"] = col.Count() m["num_objects"] = col.Count(collection.TypeAll)
switch msg.OutputType { switch msg.OutputType {
case server.JSON: case server.JSON:
ms = append(ms, m) ms = append(ms, m)
@ -92,7 +93,7 @@ func (c *Controller) cmdServer(msg *server.Message) (res string, err error) {
c.cols.Ascend(func(item btree.Item) bool { c.cols.Ascend(func(item btree.Item) bool {
col := item.(*collectionT).Collection col := item.(*collectionT).Collection
points += col.PointCount() points += col.PointCount()
objects += col.Count() objects += col.Count(collection.TypeAll)
return true return true
}) })
m["num_points"] = points m["num_points"] = points
@ -155,7 +156,7 @@ func (c *Controller) statsCollections(line string) (string, error) {
points := col.PointCount() points := col.PointCount()
m["num_points"] = points m["num_points"] = points
m["in_memory_size"] = col.TotalWeight() m["in_memory_size"] = col.TotalWeight()
m["num_objects"] = col.Count() m["num_objects"] = col.Count(collection.TypeAll)
ms = append(ms, m) ms = append(ms, m)
} else { } else {
ms = append(ms, nil) ms = append(ms, nil)

9
core/commands.json
View File

@ -77,6 +77,15 @@
"type": "geohash" "type": "geohash"
} }
] ]
},
{
"name": "STRING",
"arguments":[
{
"name": "value",
"type": "string"
}
]
} }
] ]
} }

9
core/commands_gen.go
View File

@ -239,6 +239,15 @@ var commandsJSON = `{
"type": "geohash" "type": "geohash"
} }
] ]
},
{
"name": "STRING",
"arguments":[
{
"name": "value",
"type": "string"
}
]
} }
] ]
} }