From 3ac8dc2ffbc46837dc03226dfeb49e5a3661cd59 Mon Sep 17 00:00:00 2001 From: tidwall Date: Thu, 14 Feb 2019 15:53:46 -0700 Subject: [PATCH] Optimized simple points for smaller memory --- Gopkg.lock | 6 +- Gopkg.toml | 2 +- internal/clip/clip.go | 2 + internal/clip/point.go | 6 + internal/collection/item/item.go | 56 +++++-- internal/server/crud.go | 17 +- internal/server/json.go | 5 +- internal/server/scanner.go | 13 +- internal/server/server.go | 4 +- vendor/github.com/tidwall/geojson/object.go | 17 +- vendor/github.com/tidwall/geojson/point.go | 32 +++- .../github.com/tidwall/geojson/simplepoint.go | 156 ++++++++++++++++++ .../tidwall/geojson/simplepoint_test.go | 61 +++++++ 13 files changed, 322 insertions(+), 55 deletions(-) create mode 100644 vendor/github.com/tidwall/geojson/simplepoint.go create mode 100644 vendor/github.com/tidwall/geojson/simplepoint_test.go diff --git a/Gopkg.lock b/Gopkg.lock index bab07235..eb0ca46c 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -242,7 +242,7 @@ version = "v1.0.2" [[projects]] - digest = "1:fc81262a6ad5aeec27e1bd15356f790e6b2d8fd14acb6bd5ff3f0f51bf67417f" + digest = "1:ab5e0d19c706286deed5e6ec63a35ee0f2b92d7b9e97083eb67e5d2d76b4bfdb" name = "github.com/tidwall/geojson" packages = [ ".", @@ -250,8 +250,8 @@ "geometry", ] pruneopts = "" - revision = "6baab67ab6a9bac4abf153ab779c736254a37fd1" - version = "v1.1.0" + revision = "f9500c7d3da6ce149bf80530c36b1a784dcd0f2b" + version = "v1.1.1" [[projects]] digest = "1:3ddca2bd5496c6922a2a9e636530e178a43c2a534ea6634211acdc7d10222794" diff --git a/Gopkg.toml b/Gopkg.toml index fbf4b850..137abfd2 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -36,7 +36,7 @@ required = [ [[constraint]] name = "github.com/tidwall/geojson" - version = "1.1.0" + version = "1.1.1" [[constraint]] name = "github.com/Shopify/sarama" diff --git a/internal/clip/clip.go b/internal/clip/clip.go index 92a88a46..095e9e81 100644 --- a/internal/clip/clip.go +++ b/internal/clip/clip.go @@ -10,6 +10,8 @@ func Clip(obj geojson.Object, clipper geojson.Object) (clipped geojson.Object) { switch obj := obj.(type) { case *geojson.Point: return clipPoint(obj, clipper) + case *geojson.SimplePoint: + return clipSimplePoint(obj, clipper) case *geojson.Rect: return clipRect(obj, clipper) case *geojson.LineString: diff --git a/internal/clip/point.go b/internal/clip/point.go index cf6de5c0..5760a834 100644 --- a/internal/clip/point.go +++ b/internal/clip/point.go @@ -8,3 +8,9 @@ func clipPoint(point *geojson.Point, clipper geojson.Object) geojson.Object { } return geojson.NewMultiPoint(nil) } +func clipSimplePoint(point *geojson.SimplePoint, clipper geojson.Object) geojson.Object { + if point.IntersectsRect(clipper.Rect()) { + return point + } + return geojson.NewMultiPoint(nil) +} diff --git a/internal/collection/item/item.go b/internal/collection/item/item.go index 131da220..8177da46 100644 --- a/internal/collection/item/item.go +++ b/internal/collection/item/item.go @@ -10,10 +10,24 @@ import ( // Item is a item for Tile38 collections type Item struct { - fieldsLen uint32 // fields block size in bytes, not num of fields + point bool // true: Item is pointItem, false: Item is objItem + fieldsLen uint16 // fields block size in bytes, not num of fields idLen uint32 // id block size in bytes - data unsafe.Pointer // pointer to raw block of bytes - obj geojson.Object // geojson or string + data unsafe.Pointer // pointer to raw block of bytes, fields+id +} +type objItem struct { + _ bool + _ uint16 + _ uint32 + _ unsafe.Pointer + obj geojson.Object +} +type pointItem struct { + _ bool + _ uint16 + _ uint32 + _ unsafe.Pointer + pt geojson.SimplePoint } // ID returns the items ID as a string @@ -35,13 +49,25 @@ func (item *Item) Fields() []float64 { // Obj returns the geojson object func (item *Item) Obj() geojson.Object { - return item.obj + if item.point { + return &(*pointItem)(unsafe.Pointer(item)).pt + } + return (*objItem)(unsafe.Pointer(item)).obj } // New returns a newly allocated Item func New(id string, obj geojson.Object) *Item { - item := new(Item) - item.obj = obj + var item *Item + if pt, ok := obj.(*geojson.SimplePoint); ok { + pitem := new(pointItem) + pitem.pt = *pt + item = (*Item)(unsafe.Pointer(pitem)) + item.point = true + } else { + oitem := new(objItem) + oitem.obj = obj + item = (*Item)(unsafe.Pointer(oitem)) + } item.idLen = uint32(len(id)) if len(id) > 0 { data := make([]byte, len(id)) @@ -53,21 +79,21 @@ func New(id string, obj geojson.Object) *Item { // WeightAndPoints returns the memory weight and number of points for Item. func (item *Item) WeightAndPoints() (weight, points int) { - _, objIsSpatial := item.obj.(geojson.Spatial) + _, objIsSpatial := item.Obj().(geojson.Spatial) if objIsSpatial { - points = item.obj.NumPoints() + points = item.Obj().NumPoints() weight = points * 16 - } else if item.obj != nil { - weight = len(item.obj.String()) + } else if item.Obj() != nil { + weight = len(item.Obj().String()) } - weight += int(item.fieldsLen + item.idLen) + weight += int(item.fieldsLen) + int(item.idLen) return weight, points } // Less is a btree interface that compares if item is less than other item. func (item *Item) Less(other btree.Item, ctx interface{}) bool { - value1 := item.obj.String() - value2 := other.(*Item).obj.String() + value1 := item.Obj().String() + value2 := other.(*Item).Obj().String() if value1 < value2 { return true } @@ -85,7 +111,7 @@ func (item *Item) CopyOverFields(values []float64) { newData := make([]byte, len(fieldBytes)+int(item.idLen)) copy(newData, fieldBytes) copy(newData[len(fieldBytes):], oldData[item.fieldsLen:]) - item.fieldsLen = uint32(len(fieldBytes)) + item.fieldsLen = uint16(len(fieldBytes)) if len(newData) > 0 { item.data = unsafe.Pointer(&newData[0]) } else { @@ -118,7 +144,7 @@ func (item *Item) SetField(index int, value float64) (updated bool) { // copy the id copy(newData[(index+1)*8:], oldBytes[item.fieldsLen:]) // update the fields length - item.fieldsLen = uint32((index + 1) * 8) + item.fieldsLen = uint16((index + 1) * 8) // update the raw data item.data = unsafe.Pointer(&newData[0]) } diff --git a/internal/server/crud.go b/internal/server/crud.go index 664a9ed3..e7aa5b03 100644 --- a/internal/server/crud.go +++ b/internal/server/crud.go @@ -196,10 +196,7 @@ func (server *Server) cmdGet(msg *Message) (resp.Value, error) { buf.Write(appendJSONSimplePoint(nil, o)) } else { point := o.Center() - var z float64 - if gPoint, ok := o.(*geojson.Point); ok { - z = gPoint.Z() - } + z, _ := geojson.IsPoint(o) if z != 0 { vals = append(vals, resp.ArrayValue([]resp.Value{ resp.StringValue(strconv.FormatFloat(point.Y, 'f', -1, 64)), @@ -672,7 +669,11 @@ func (server *Server) parseSetArgs(vs []string) ( err = errInvalidArgument(slon) return } - d.obj = geojson.NewPoint(geometry.Point{X: x, Y: y}) + if server.geomParseOpts.AllowSimplePoints { + d.obj = geojson.NewSimplePoint(geometry.Point{X: x, Y: y}) + } else { + d.obj = geojson.NewPoint(geometry.Point{X: x, Y: y}) + } } else { var x, y, z float64 y, err = strconv.ParseFloat(slat, 64) @@ -742,7 +743,11 @@ func (server *Server) parseSetArgs(vs []string) ( return } lat, lon := geohash.Decode(shash) - d.obj = geojson.NewPoint(geometry.Point{X: lon, Y: lat}) + if server.geomParseOpts.AllowSimplePoints { + d.obj = geojson.NewSimplePoint(geometry.Point{X: lon, Y: lat}) + } else { + d.obj = geojson.NewPoint(geometry.Point{X: lon, Y: lat}) + } case lcb(typ, "object"): var object string if vs, object, ok = tokenval(vs); !ok || object == "" { diff --git a/internal/server/json.go b/internal/server/json.go index 8706dbb8..c2128cea 100644 --- a/internal/server/json.go +++ b/internal/server/json.go @@ -56,10 +56,7 @@ func appendJSONSimpleBounds(dst []byte, o geojson.Object) []byte { func appendJSONSimplePoint(dst []byte, o geojson.Object) []byte { point := o.Center() - var z float64 - if gPoint, ok := o.(*geojson.Point); ok { - z = gPoint.Z() - } + z, _ := geojson.IsPoint(o) dst = append(dst, `{"lat":`...) dst = strconv.AppendFloat(dst, point.Y, 'f', -1, 64) dst = append(dst, `,"lon":`...) diff --git a/internal/server/scanner.go b/internal/server/scanner.go index c1580440..7aa4434e 100644 --- a/internal/server/scanner.go +++ b/internal/server/scanner.go @@ -202,9 +202,7 @@ func (sw *scanWriter) fieldMatch(fields []float64, o geojson.Object) (fvals []fl for _, where := range sw.wheres { if where.field == "z" { if !gotz { - if point, ok := o.(*geojson.Point); ok { - z = point.Z() - } + z, _ = geojson.IsPoint(o) } if !where.match(z) { return @@ -258,9 +256,7 @@ func (sw *scanWriter) fieldMatch(fields []float64, o geojson.Object) (fvals []fl for _, where := range sw.wheres { if where.field == "z" { if !gotz { - if point, ok := o.(*geojson.Point); ok { - z = point.Z() - } + z, _ = geojson.IsPoint(o) } if !where.match(z) { return @@ -449,10 +445,7 @@ func (sw *scanWriter) writeObject(opts ScanWriterParams) bool { vals = append(vals, resp.StringValue(opts.o.String())) case outputPoints: point := opts.o.Center() - var z float64 - if point, ok := opts.o.(*geojson.Point); ok { - z = point.Z() - } + z, _ := geojson.IsPoint(opts.o) if z != 0 { vals = append(vals, resp.ArrayValue([]resp.Value{ resp.FloatValue(point.Y), diff --git a/internal/server/server.go b/internal/server/server.go index 94f38922..a31140a9 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -170,11 +170,13 @@ func Serve(host string, port int, dir string, http bool) error { return err } + server.geomParseOpts = *geojson.DefaultParseOptions + server.geomParseOpts.AllowSimplePoints = true + // Allow for geometry indexing options through environment variables: // T38IDXGEOMKIND -- None, RTree, QuadTree // T38IDXGEOM -- Min number of points in a geometry for indexing. // T38IDXMULTI -- Min number of object in a Multi/Collection for indexing. - server.geomParseOpts = *geojson.DefaultParseOptions n, err := strconv.ParseUint(os.Getenv("T38IDXGEOM"), 10, 32) if err == nil { server.geomParseOpts.IndexGeometry = int(n) diff --git a/vendor/github.com/tidwall/geojson/object.go b/vendor/github.com/tidwall/geojson/object.go index fe023f83..05db9261 100644 --- a/vendor/github.com/tidwall/geojson/object.go +++ b/vendor/github.com/tidwall/geojson/object.go @@ -50,7 +50,7 @@ var _ = []Object{ &Point{}, &LineString{}, &Polygon{}, &Feature{}, &MultiPoint{}, &MultiLineString{}, &MultiPolygon{}, &GeometryCollection{}, &FeatureCollection{}, - &Rect{}, &Circle{}, + &Rect{}, &Circle{}, &SimplePoint{}, } // Collection is a searchable collection type. @@ -89,7 +89,12 @@ type ParseOptions struct { // IndexGeometryKind is the kind of index implementation. // Default is QuadTreeCompressed IndexGeometryKind geometry.IndexKind - RequireValid bool + // RequireValid option cause parse to fail when a geojson object is invalid. + RequireValid bool + // AllowSimplePoints options will force to parse to return the SimplePoint + // type when a geojson point only consists of an 2D x/y coord and no extra + // json members. + AllowSimplePoints bool } // DefaultParseOptions ... @@ -98,6 +103,7 @@ var DefaultParseOptions = &ParseOptions{ IndexGeometry: 64, IndexGeometryKind: geometry.QuadTree, RequireValid: false, + AllowSimplePoints: false, } // Parse a GeoJSON object @@ -296,10 +302,3 @@ func unionRects(a, b geometry.Rect) geometry.Rect { func geoDistancePoints(a, b geometry.Point) float64 { return geo.DistanceTo(a.Y, a.X, b.Y, b.X) } - -// func geoDistanceCenterToPoint(obj Object, point geometry.Point) float64 { -// if obj.Empty() { -// return 0 -// } -// return geoDistancePointsA(obj.Center(), point) -// } diff --git a/vendor/github.com/tidwall/geojson/point.go b/vendor/github.com/tidwall/geojson/point.go index fa955404..19be1d52 100644 --- a/vendor/github.com/tidwall/geojson/point.go +++ b/vendor/github.com/tidwall/geojson/point.go @@ -152,21 +152,30 @@ func (g *Point) Z() float64 { } func parseJSONPoint(keys *parseKeys, opts *ParseOptions) (Object, error) { - var g Point - var err error - g.base, g.extra, err = parseJSONPointCoords(keys, gjson.Result{}, opts) + var o Object + base, extra, err := parseJSONPointCoords(keys, gjson.Result{}, opts) if err != nil { return nil, err } - if err := parseBBoxAndExtras(&g.extra, keys, opts); err != nil { + if err := parseBBoxAndExtras(&extra, keys, opts); err != nil { return nil, err } + if extra == nil && opts.AllowSimplePoints { + var g SimplePoint + g.base = base + o = &g + } else { + var g Point + g.base = base + g.extra = extra + o = &g + } if opts.RequireValid { - if !g.Valid() { + if !o.Valid() { return nil, errCoordinatesInvalid } } - return &g, nil + return o, nil } func parseJSONPointCoords( @@ -244,3 +253,14 @@ func (g *Point) DistanceLine(line *geometry.Line) float64 { func (g *Point) DistancePoly(poly *geometry.Poly) float64 { return geoDistancePoints(g.Center(), poly.Rect().Center()) } + +// IsPoint returns true if the object is a {"type":"Point"} +func IsPoint(obj Object) (z float64, ok bool) { + switch pt := obj.(type) { + case *SimplePoint: + return 0, true + case *Point: + return pt.Z(), true + } + return 0, false +} diff --git a/vendor/github.com/tidwall/geojson/simplepoint.go b/vendor/github.com/tidwall/geojson/simplepoint.go new file mode 100644 index 00000000..8922c90e --- /dev/null +++ b/vendor/github.com/tidwall/geojson/simplepoint.go @@ -0,0 +1,156 @@ +package geojson + +import "github.com/tidwall/geojson/geometry" + +// SimplePoint ... +type SimplePoint struct { + base geometry.Point +} + +// NewSimplePoint ... +func NewSimplePoint(point geometry.Point) *SimplePoint { + return &SimplePoint{base: point} +} + +// ForEach ... +func (g *SimplePoint) ForEach(iter func(geom Object) bool) bool { + return iter(g) +} + +// Empty ... +func (g *SimplePoint) Empty() bool { + return g.base.Empty() +} + +// Valid ... +func (g *SimplePoint) Valid() bool { + return g.base.Valid() +} + +// Rect ... +func (g *SimplePoint) Rect() geometry.Rect { + return g.base.Rect() +} + +// Spatial ... +func (g *SimplePoint) Spatial() Spatial { + return g +} + +// Center ... +func (g *SimplePoint) Center() geometry.Point { + return g.base +} + +// Base ... +func (g *SimplePoint) Base() geometry.Point { + return g.base +} + +// AppendJSON ... +func (g *SimplePoint) AppendJSON(dst []byte) []byte { + dst = append(dst, `{"type":"Point","coordinates":`...) + dst = appendJSONPoint(dst, g.base, nil, 0) + dst = append(dst, '}') + return dst +} + +// JSON ... +func (g *SimplePoint) JSON() string { + return string(g.AppendJSON(nil)) +} + +// MarshalJSON ... +func (g *SimplePoint) MarshalJSON() ([]byte, error) { + return g.AppendJSON(nil), nil +} + +// String ... +func (g *SimplePoint) String() string { + return string(g.AppendJSON(nil)) +} + +// Within ... +func (g *SimplePoint) Within(obj Object) bool { + return obj.Contains(g) +} + +// Contains ... +func (g *SimplePoint) Contains(obj Object) bool { + return obj.Spatial().WithinPoint(g.base) +} + +// Intersects ... +func (g *SimplePoint) Intersects(obj Object) bool { + return obj.Spatial().IntersectsPoint(g.base) +} + +// WithinRect ... +func (g *SimplePoint) WithinRect(rect geometry.Rect) bool { + return rect.ContainsPoint(g.base) +} + +// WithinPoint ... +func (g *SimplePoint) WithinPoint(point geometry.Point) bool { + return point.ContainsPoint(g.base) +} + +// WithinLine ... +func (g *SimplePoint) WithinLine(line *geometry.Line) bool { + return line.ContainsPoint(g.base) +} + +// WithinPoly ... +func (g *SimplePoint) WithinPoly(poly *geometry.Poly) bool { + return poly.ContainsPoint(g.base) +} + +// IntersectsPoint ... +func (g *SimplePoint) IntersectsPoint(point geometry.Point) bool { + return g.base.IntersectsPoint(point) +} + +// IntersectsRect ... +func (g *SimplePoint) IntersectsRect(rect geometry.Rect) bool { + return g.base.IntersectsRect(rect) +} + +// IntersectsLine ... +func (g *SimplePoint) IntersectsLine(line *geometry.Line) bool { + return g.base.IntersectsLine(line) +} + +// IntersectsPoly ... +func (g *SimplePoint) IntersectsPoly(poly *geometry.Poly) bool { + return g.base.IntersectsPoly(poly) +} + +// NumPoints ... +func (g *SimplePoint) NumPoints() int { + return 1 +} + +// Distance ... +func (g *SimplePoint) Distance(obj Object) float64 { + return obj.Spatial().DistancePoint(g.base) +} + +// DistancePoint ... +func (g *SimplePoint) DistancePoint(point geometry.Point) float64 { + return geoDistancePoints(g.Center(), point) +} + +// DistanceRect ... +func (g *SimplePoint) DistanceRect(rect geometry.Rect) float64 { + return geoDistancePoints(g.Center(), rect.Center()) +} + +// DistanceLine ... +func (g *SimplePoint) DistanceLine(line *geometry.Line) float64 { + return geoDistancePoints(g.Center(), line.Rect().Center()) +} + +// DistancePoly ... +func (g *SimplePoint) DistancePoly(poly *geometry.Poly) float64 { + return geoDistancePoints(g.Center(), poly.Rect().Center()) +} diff --git a/vendor/github.com/tidwall/geojson/simplepoint_test.go b/vendor/github.com/tidwall/geojson/simplepoint_test.go new file mode 100644 index 00000000..acea3a97 --- /dev/null +++ b/vendor/github.com/tidwall/geojson/simplepoint_test.go @@ -0,0 +1,61 @@ +package geojson + +import "testing" + +func TestSimplePointNotSimple(t *testing.T) { + p := expectJSONOpts(t, `{"type":"Point","coordinates":[1,2,3]}`, nil, &ParseOptions{AllowSimplePoints: true}) + expect(t, p.Center() == P(1, 2)) + expectJSONOpts(t, `{"type":"Point","coordinates":[1,null]}`, errCoordinatesInvalid, &ParseOptions{AllowSimplePoints: true}) + expectJSONOpts(t, `{"type":"Point","coordinates":[1,2],"bbox":null}`, nil, &ParseOptions{AllowSimplePoints: true}) + expectJSONOpts(t, `{"type":"Point"}`, errCoordinatesMissing, &ParseOptions{AllowSimplePoints: true}) + expectJSONOpts(t, `{"type":"Point","coordinates":null}`, errCoordinatesInvalid, &ParseOptions{AllowSimplePoints: true}) + expectJSONOpts(t, `{"type":"Point","coordinates":[1,2,3,4,5]}`, `{"type":"Point","coordinates":[1,2,3,4]}`, &ParseOptions{AllowSimplePoints: true}) + expectJSONOpts(t, `{"type":"Point","coordinates":[1]}`, errCoordinatesInvalid, &ParseOptions{AllowSimplePoints: true}) + expectJSONOpts(t, `{"type":"Point","coordinates":[1,2,3],"bbox":[1,2,3,4]}`, `{"type":"Point","coordinates":[1,2,3],"bbox":[1,2,3,4]}`, &ParseOptions{AllowSimplePoints: true}) +} + +func TestSimplePointParseValid(t *testing.T) { + json := `{"type":"Point","coordinates":[190,90]}` + p := expectJSONOpts(t, json, nil, &ParseOptions{AllowSimplePoints: true}) + expect(t, !p.(*SimplePoint).Empty()) + p = expectJSONOpts(t, json, nil, &ParseOptions{AllowSimplePoints: false}) + expect(t, !p.(*Point).Empty()) + p = expectJSONOpts(t, json, errCoordinatesInvalid, &ParseOptions{RequireValid: true, AllowSimplePoints: true}) + expect(t, p == nil) +} + +func TestSimplePointVarious(t *testing.T) { + var g Object = PO(10, 20) + expect(t, string(g.AppendJSON(nil)) == `{"type":"Point","coordinates":[10,20]}`) + expect(t, g.Rect() == R(10, 20, 10, 20)) + expect(t, g.Center() == P(10, 20)) + expect(t, !g.Empty()) +} + +func TestSimplePointValid(t *testing.T) { + var g Object = PO(0, 20) + expect(t, g.Valid()) + + var g1 Object = PO(10, 20) + expect(t, g1.Valid()) +} + +func TestSimplePointInvalidLargeX(t *testing.T) { + var g Object = PO(10, 91) + expect(t, !g.Valid()) +} + +func TestSimplePointInvalidLargeY(t *testing.T) { + var g Object = PO(181, 20) + expect(t, !g.Valid()) +} + +func TestSimplePointValidLargeX(t *testing.T) { + var g Object = PO(180, 20) + expect(t, g.Valid()) +} + +func TestSimplePointValidLargeY(t *testing.T) { + var g Object = PO(180, 90) + expect(t, g.Valid()) +}