From 8d89198eaf5c9655db91f4a5103479a13564cb86 Mon Sep 17 00:00:00 2001 From: Josh Baker Date: Sat, 9 Jul 2016 19:44:28 -0700 Subject: [PATCH] wip. adding string value type. --- controller/collection/collection.go | 63 +++++++++++-------- controller/controller.go | 8 +-- controller/crud.go | 13 +++- controller/keys.go | 2 +- controller/scanner.go | 2 +- controller/stats.go | 2 +- geojson/feature.go | 10 +++ geojson/featurecollection.go | 10 +++ geojson/geometrycollection.go | 10 +++ geojson/linestring.go | 10 +++ geojson/multilinestring.go | 10 +++ geojson/multipoint.go | 10 +++ geojson/multipolygon.go | 10 +++ geojson/object.go | 4 ++ geojson/point.go | 10 +++ geojson/polygon.go | 10 +++ geojson/simplepoint.go | 10 +++ geojson/string.go | 94 +++++++++++++++++++++++++++++ 18 files changed, 253 insertions(+), 35 deletions(-) create mode 100644 geojson/string.go diff --git a/controller/collection/collection.go b/controller/collection/collection.go index 6f49406d..61c5695c 100644 --- a/controller/collection/collection.go +++ b/controller/collection/collection.go @@ -1,19 +1,38 @@ package collection import ( - "github.com/google/btree" + "github.com/tidwall/btree" "github.com/tidwall/tile38/geojson" "github.com/tidwall/tile38/index" ) +const ( + idOrdered = 0 + valueOrdered = 2 +) + type itemT struct { id string object geojson.Object fields []float64 } -func (i *itemT) Less(item btree.Item) bool { - return i.id < item.(*itemT).id +func (i *itemT) Less(item btree.Item, ctx int) bool { + switch ctx { + default: + return false + case idOrdered: + return i.id < item.(*itemT).id + case valueOrdered: + if i.object.String() < item.(*itemT).object.String() { + return true + } + if i.object.String() > item.(*itemT).object.String() { + return false + } + return i.id < item.(*itemT).id + } + } func (i *itemT) Rect() (minX, minY, maxX, maxY float64) { @@ -28,8 +47,9 @@ func (i *itemT) Point() (x, y float64) { // Collection represents a collection of geojson objects. type Collection struct { - items *btree.BTree - index *index.Index + items *btree.BTree // items sorted by keys + values *btree.BTree // items sorted by value+key + index *index.Index // items geospatially indexed fieldMap map[string]int weight int points int @@ -42,7 +62,8 @@ var counter uint64 func New() *Collection { col := &Collection{ index: index.New(), - items: btree.New(16), + items: btree.New(16, idOrdered), + values: btree.New(16, valueOrdered), fieldMap: make(map[string]int), } return col @@ -60,23 +81,7 @@ func (c *Collection) PointCount() int { // TotalWeight calculates the in-memory cost of the collection in bytes. func (c *Collection) TotalWeight() int { - return c.weight + c.overheadWeight() -} - -func (c *Collection) overheadWeight() int { - // the field map. - mapweight := 0 - for field := range c.fieldMap { - mapweight += len(field) + 8 // key + value - } - mapweight = int((float64(mapweight) * 1.05) + 28.0) // about an 8% pad plus golang 28 byte map overhead. - // the btree. each object takes up 64bits for the interface head for each item. - btreeweight := (c.objects * 8) - // plus roughly one pointer for every item - btreeweight += (c.objects * 8) - // also the btree header weight - btreeweight += 24 - return mapweight + btreeweight + return c.weight } // ReplaceOrInsert adds or replaces an object in the collection and returns the fields array. @@ -113,7 +118,11 @@ func (c *Collection) remove(id string) (item *itemT, ok bool) { return nil, false } item = i.(*itemT) - c.index.Remove(item) + if item.object.IsGeometry() { + c.index.Remove(item) + } else { + c.values.Delete(item) + } c.weight -= len(item.fields) * 8 c.weight -= item.object.Weight() + len(item.id) c.points -= item.object.PositionCount() @@ -123,7 +132,11 @@ func (c *Collection) remove(id string) (item *itemT, ok bool) { func (c *Collection) insert(id string, obj geojson.Object) (item *itemT) { item = &itemT{id: id, object: obj} - c.index.Insert(item) + if obj.IsGeometry() { + c.index.Insert(item) + } else { + c.values.ReplaceOrInsert(item) + } c.items.ReplaceOrInsert(item) c.weight += obj.Weight() + len(id) c.points += obj.PositionCount() diff --git a/controller/controller.go b/controller/controller.go index a9a95cf4..9a78c2e0 100644 --- a/controller/controller.go +++ b/controller/controller.go @@ -14,7 +14,7 @@ import ( "sync" "time" - "github.com/google/btree" + "github.com/tidwall/btree" "github.com/tidwall/resp" "github.com/tidwall/tile38/controller/collection" "github.com/tidwall/tile38/controller/log" @@ -45,7 +45,7 @@ type commandDetailsT struct { timestamp time.Time } -func (col *collectionT) Less(item btree.Item) bool { +func (col *collectionT) Less(item btree.Item, ctx int) bool { return col.Key < item.(*collectionT).Key } @@ -82,7 +82,7 @@ func ListenAndServe(host string, port int, dir string) error { host: host, port: port, dir: dir, - cols: btree.New(16), + cols: btree.New(16, 0), follows: make(map[*bytes.Buffer]bool), fcond: sync.NewCond(&sync.Mutex{}), lives: make(map[*liveBuffer]bool), @@ -387,7 +387,7 @@ func randomKey(n int) string { func (c *Controller) reset() { c.aofsz = 0 - c.cols = btree.New(16) + c.cols = btree.New(16, 0) } func (c *Controller) command(msg *server.Message, w io.Writer) (res string, d commandDetailsT, err error) { diff --git a/controller/crud.go b/controller/crud.go index 98a3bdc0..ba7c5bf1 100644 --- a/controller/crud.go +++ b/controller/crud.go @@ -7,7 +7,7 @@ import ( "strings" "time" - "github.com/google/btree" + "github.com/tidwall/btree" "github.com/tidwall/resp" "github.com/tidwall/tile38/controller/collection" "github.com/tidwall/tile38/controller/server" @@ -92,7 +92,7 @@ func (c *Controller) cmdGet(msg *server.Message) (string, error) { buf.WriteString(`,"object":`) buf.WriteString(o.JSON()) } else { - vals = append(vals, resp.StringValue(o.JSON())) + vals = append(vals, resp.StringValue(o.String())) } } else { switch strings.ToLower(typ) { @@ -300,7 +300,7 @@ func (c *Controller) cmdFlushDB(msg *server.Message) (res string, d commandDetai err = errInvalidNumberOfArguments return } - c.cols = btree.New(16) + c.cols = btree.New(16, 0) c.hooks = make(map[string]*Hook) c.hookcols = make(map[string]map[string]*Hook) d.command = "flushdb" @@ -379,6 +379,13 @@ func (c *Controller) parseSetArgs(vs []resp.Value) (d commandDetailsT, fields [] default: err = errInvalidArgument(typ) return + case lc(typ, "string"): + var str string + if vs, str, ok = tokenval(vs); !ok { + err = errInvalidNumberOfArguments + return + } + d.obj = geojson.String(str) case lc(typ, "point"): var slat, slon, sz string if vs, slat, ok = tokenval(vs); !ok || slat == "" { diff --git a/controller/keys.go b/controller/keys.go index bbddeb10..6845f92a 100644 --- a/controller/keys.go +++ b/controller/keys.go @@ -5,7 +5,7 @@ import ( "strings" "time" - "github.com/google/btree" + "github.com/tidwall/btree" "github.com/tidwall/resp" "github.com/tidwall/tile38/controller/server" ) diff --git a/controller/scanner.go b/controller/scanner.go index 9c742d59..222840ff 100644 --- a/controller/scanner.go +++ b/controller/scanner.go @@ -325,7 +325,7 @@ func (sw *scanWriter) writeObject(id string, o geojson.Object, fields []float64, } else { switch sw.output { case outputObjects: - vals = append(vals, resp.StringValue(o.JSON())) + vals = append(vals, resp.StringValue(o.String())) case outputPoints: point := o.CalculatedPoint() if point.Z != 0 { diff --git a/controller/stats.go b/controller/stats.go index ea0240b0..c2d8f33c 100644 --- a/controller/stats.go +++ b/controller/stats.go @@ -7,7 +7,7 @@ import ( "sort" "time" - "github.com/google/btree" + "github.com/tidwall/btree" "github.com/tidwall/resp" "github.com/tidwall/tile38/controller/server" ) diff --git a/geojson/feature.go b/geojson/feature.go index 47f92698..16d57561 100644 --- a/geojson/feature.go +++ b/geojson/feature.go @@ -150,6 +150,11 @@ func (g Feature) JSON() string { return buf.String() } +// String returns a string representation of the object. This might be JSON or something else. +func (g Feature) String() string { + return g.JSON() +} + // Bytes is the bytes representation of the object. func (g Feature) Bytes() []byte { var buf bytes.Buffer @@ -226,3 +231,8 @@ func (g Feature) Nearby(center Position, meters float64) bool { func (g Feature) IsBBoxDefined() bool { return g.BBox != nil } + +// IsGeometry return true if the object is a geojson geometry object. false if it something else. +func (g Feature) IsGeometry() bool { + return true +} diff --git a/geojson/featurecollection.go b/geojson/featurecollection.go index 199afc46..f96076d5 100644 --- a/geojson/featurecollection.go +++ b/geojson/featurecollection.go @@ -126,6 +126,11 @@ func (g FeatureCollection) JSON() string { return buf.String() } +// String returns a string representation of the object. This might be JSON or something else. +func (g FeatureCollection) String() string { + return g.JSON() +} + // Bytes is the bytes representation of the object. func (g FeatureCollection) Bytes() []byte { var buf bytes.Buffer @@ -248,3 +253,8 @@ func (g FeatureCollection) Nearby(center Position, meters float64) bool { func (g FeatureCollection) IsBBoxDefined() bool { return g.BBox != nil } + +// IsGeometry return true if the object is a geojson geometry object. false if it something else. +func (g FeatureCollection) IsGeometry() bool { + return true +} diff --git a/geojson/geometrycollection.go b/geojson/geometrycollection.go index 13fe100a..7556f189 100644 --- a/geojson/geometrycollection.go +++ b/geojson/geometrycollection.go @@ -126,6 +126,11 @@ func (g GeometryCollection) JSON() string { return buf.String() } +// String returns a string representation of the object. This might be JSON or something else. +func (g GeometryCollection) String() string { + return g.JSON() +} + // Bytes is the bytes representation of the object. func (g GeometryCollection) Bytes() []byte { var buf bytes.Buffer @@ -248,3 +253,8 @@ func (g GeometryCollection) Nearby(center Position, meters float64) bool { func (g GeometryCollection) IsBBoxDefined() bool { return g.BBox != nil } + +// IsGeometry return true if the object is a geojson geometry object. false if it something else. +func (g GeometryCollection) IsGeometry() bool { + return true +} diff --git a/geojson/linestring.go b/geojson/linestring.go index 5fecedc6..074128bd 100644 --- a/geojson/linestring.go +++ b/geojson/linestring.go @@ -56,6 +56,11 @@ func (g LineString) JSON() string { return level2JSON("LineString", g.Coordinates, g.BBox) } +// String returns a string representation of the object. This might be JSON or something else. +func (g LineString) String() string { + return g.JSON() +} + // Bytes is the bytes representation of the object. func (g LineString) Bytes() []byte { return level2Bytes(lineString, g.Coordinates, g.BBox) @@ -126,3 +131,8 @@ func (g LineString) Nearby(center Position, meters float64) bool { func (g LineString) IsBBoxDefined() bool { return g.BBox != nil } + +// IsGeometry return true if the object is a geojson geometry object. false if it something else. +func (g LineString) IsGeometry() bool { + return true +} diff --git a/geojson/multilinestring.go b/geojson/multilinestring.go index e7b03153..98be4c31 100644 --- a/geojson/multilinestring.go +++ b/geojson/multilinestring.go @@ -62,6 +62,11 @@ func (g MultiLineString) JSON() string { return level3JSON("MultiLineString", g.Coordinates, g.BBox) } +// String returns a string representation of the object. This might be JSON or something else. +func (g MultiLineString) String() string { + return g.JSON() +} + // Bytes is the bytes representation of the object. func (g MultiLineString) Bytes() []byte { return level3Bytes(multiLineString, g.Coordinates, g.BBox) @@ -184,3 +189,8 @@ func (g MultiLineString) Nearby(center Position, meters float64) bool { func (g MultiLineString) IsBBoxDefined() bool { return g.BBox != nil } + +// IsGeometry return true if the object is a geojson geometry object. false if it something else. +func (g MultiLineString) IsGeometry() bool { + return true +} diff --git a/geojson/multipoint.go b/geojson/multipoint.go index 60072c1f..47598104 100644 --- a/geojson/multipoint.go +++ b/geojson/multipoint.go @@ -54,6 +54,11 @@ func (g MultiPoint) JSON() string { return level2JSON("MultiPoint", g.Coordinates, g.BBox) } +// String returns a string representation of the object. This might be JSON or something else. +func (g MultiPoint) String() string { + return g.JSON() +} + // Bytes is the bytes representation of the object. func (g MultiPoint) Bytes() []byte { return level2Bytes(multiPoint, g.Coordinates, g.BBox) @@ -163,3 +168,8 @@ func (g MultiPoint) Nearby(center Position, meters float64) bool { func (g MultiPoint) IsBBoxDefined() bool { return g.BBox != nil } + +// IsGeometry return true if the object is a geojson geometry object. false if it something else. +func (g MultiPoint) IsGeometry() bool { + return true +} diff --git a/geojson/multipolygon.go b/geojson/multipolygon.go index f2c8ae99..20b4e816 100644 --- a/geojson/multipolygon.go +++ b/geojson/multipolygon.go @@ -66,6 +66,11 @@ func (g MultiPolygon) JSON() string { return level4JSON("MultiPolygon", g.Coordinates, g.BBox) } +// String returns a string representation of the object. This might be JSON or something else. +func (g MultiPolygon) String() string { + return g.JSON() +} + // Bytes is the bytes representation of the object. func (g MultiPolygon) Bytes() []byte { return level4Bytes(multiPolygon, g.Coordinates, g.BBox) @@ -193,3 +198,8 @@ func (g MultiPolygon) Nearby(center Position, meters float64) bool { func (g MultiPolygon) IsBBoxDefined() bool { return g.BBox != nil } + +// IsGeometry return true if the object is a geojson geometry object. false if it something else. +func (g MultiPolygon) IsGeometry() bool { + return true +} diff --git a/geojson/object.go b/geojson/object.go index 608f9fd5..dcf0545f 100644 --- a/geojson/object.go +++ b/geojson/object.go @@ -76,6 +76,8 @@ type Object interface { CalculatedPoint() Position // JSON is the json representation of the object. This might not be exactly the same as the original. JSON() string + // String returns a string represenation of the object. This may be JSON or something else. + String() string // Bytes is the bytes representation of the object. Bytes() []byte // PositionCount return the number of coordinates. @@ -88,6 +90,8 @@ type Object interface { Geohash(precision int) (string, error) // IsBBoxDefined returns true if the object has a defined bbox. IsBBoxDefined() bool + // IsGeometry return true if the object is a geojson geometry object. false if it something else. + IsGeometry() bool } func writeHeader(buf *bytes.Buffer, objType byte, bbox *BBox, isCordZ bool) { diff --git a/geojson/point.go b/geojson/point.go index e3ff97de..e7fe9e74 100644 --- a/geojson/point.go +++ b/geojson/point.go @@ -55,6 +55,11 @@ func (g Point) JSON() string { return level1JSON("Point", g.Coordinates, g.BBox) } +// String returns a string representation of the object. This might be JSON or something else. +func (g Point) String() string { + return g.JSON() +} + // Bytes is the bytes representation of the object. func (g Point) Bytes() []byte { return level1Bytes(point, g.Coordinates, g.BBox) @@ -135,3 +140,8 @@ func (g Point) Nearby(center Position, meters float64) bool { func (g Point) IsBBoxDefined() bool { return g.BBox != nil } + +// IsGeometry return true if the object is a geojson geometry object. false if it something else. +func (g Point) IsGeometry() bool { + return true +} diff --git a/geojson/polygon.go b/geojson/polygon.go index 7247b596..c33bf587 100644 --- a/geojson/polygon.go +++ b/geojson/polygon.go @@ -69,6 +69,11 @@ func (g Polygon) JSON() string { return level3JSON("Polygon", g.Coordinates, g.BBox) } +// String returns a string representation of the object. This might be JSON or something else. +func (g Polygon) String() string { + return g.JSON() +} + // Bytes is the bytes representation of the object. func (g Polygon) Bytes() []byte { return level3Bytes(polygon, g.Coordinates, g.BBox) @@ -202,3 +207,8 @@ func (g Polygon) KML() string { func (g Polygon) IsBBoxDefined() bool { return g.BBox != nil } + +// IsGeometry return true if the object is a geojson geometry object. false if it something else. +func (g Polygon) IsGeometry() bool { + return true +} diff --git a/geojson/simplepoint.go b/geojson/simplepoint.go index c1d610e3..9e385add 100644 --- a/geojson/simplepoint.go +++ b/geojson/simplepoint.go @@ -59,6 +59,11 @@ func (g SimplePoint) JSON() string { return level1JSON("Point", Position{X: g.X, Y: g.Y, Z: 0}, nil) } +// String returns a string representation of the object. This might be JSON or something else. +func (g SimplePoint) String() string { + return g.JSON() +} + // Bytes is the bytes representation of the object. func (g SimplePoint) Bytes() []byte { return level1Bytes(point, Position{X: g.X, Y: g.Y, Z: 0}, nil) @@ -123,3 +128,8 @@ func (g SimplePoint) Nearby(center Position, meters float64) bool { func (g SimplePoint) IsBBoxDefined() bool { return false } + +// IsGeometry return true if the object is a geojson geometry object. false if it something else. +func (g SimplePoint) IsGeometry() bool { + return true +} diff --git a/geojson/string.go b/geojson/string.go new file mode 100644 index 00000000..2c99db22 --- /dev/null +++ b/geojson/string.go @@ -0,0 +1,94 @@ +package geojson + +import "encoding/json" + +// String is a not a geojson object, but just a string +type String string + +func (s String) bboxPtr() *BBox { + return nil +} +func (s String) hasPositions() bool { + return false +} + +// WithinBBox detects if the object is fully contained inside a bbox. +func (s String) WithinBBox(bbox BBox) bool { + return false +} + +// IntersectsBBox detects if the object intersects a bbox. +func (s String) IntersectsBBox(bbox BBox) bool { + return false +} + +// Within detects if the object is fully contained inside another object. +func (s String) Within(o Object) bool { + return false +} + +// Intersects detects if the object intersects another object. +func (s String) Intersects(o Object) bool { + return false +} + +// Nearby detects if the object is nearby a position. +func (s String) Nearby(center Position, meters float64) bool { + return false +} + +// CalculatedBBox is exterior bbox containing the object. +func (s String) CalculatedBBox() BBox { + return BBox{} +} + +// CalculatedPoint is a point representation of the object. +func (s String) CalculatedPoint() Position { + return Position{} +} + +// JSON is the json representation of the object. This might not be exactly the same as the original. +func (s String) JSON() string { + b, _ := s.MarshalJSON() + return string(b) +} + +// String returns a string representation of the object. This might be JSON or something else. +func (s String) String() string { + return string(s) +} + +// IsGeometry return true if the object is a geojson geometry object. false if it something else. +func (g String) IsGeometry() bool { + return false +} + +// Bytes is the bytes representation of the object. +func (s String) Bytes() []byte { + return []byte(s.String()) +} + +// PositionCount return the number of coordinates. +func (s String) PositionCount() int { + return 0 +} + +// Weight returns the in-memory size of the object. +func (s String) Weight() int { + return len(s) +} + +// MarshalJSON allows the object to be encoded in json.Marshal calls. +func (s String) MarshalJSON() ([]byte, error) { + return json.Marshal(string(s)) +} + +// Geohash converts the object to a geohash value. +func (s String) Geohash(precision int) (string, error) { + return "", nil +} + +// IsBBoxDefined returns true if the object has a defined bbox. +func (s String) IsBBoxDefined() bool { + return false +}