mirror
/
tile38
mirror of https://github.com/tidwall/tile38.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Optimized simple points for smaller memory

This commit is contained in:
tidwall 2019-02-14 15:53:46 -07:00
parent 8a5b842e3c
commit 3ac8dc2ffb
13 changed files with 322 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
Gopkg.lock generated
View File

@ -242,7 +242,7 @@
version = "v1.0.2"
[[projects]]
digest = "1:fc81262a6ad5aeec27e1bd15356f790e6b2d8fd14acb6bd5ff3f0f51bf67417f"
digest = "1:ab5e0d19c706286deed5e6ec63a35ee0f2b92d7b9e97083eb67e5d2d76b4bfdb"
name = "github.com/tidwall/geojson"
packages = [
".",
@ -250,8 +250,8 @@
"geometry",
]
pruneopts = ""
revision = "6baab67ab6a9bac4abf153ab779c736254a37fd1"
version = "v1.1.0"
revision = "f9500c7d3da6ce149bf80530c36b1a784dcd0f2b"
version = "v1.1.1"
[[projects]]
digest = "1:3ddca2bd5496c6922a2a9e636530e178a43c2a534ea6634211acdc7d10222794"

2
Gopkg.toml
View File

@ -36,7 +36,7 @@ required = [
[[constraint]]
name = "github.com/tidwall/geojson"
version = "1.1.0"
version = "1.1.1"
[[constraint]]
name = "github.com/Shopify/sarama"

2
internal/clip/clip.go
View File

@ -10,6 +10,8 @@ func Clip(obj geojson.Object, clipper geojson.Object) (clipped geojson.Object) {
switch obj := obj.(type) {
case *geojson.Point:
return clipPoint(obj, clipper)
case *geojson.SimplePoint:
return clipSimplePoint(obj, clipper)
case *geojson.Rect:
return clipRect(obj, clipper)
case *geojson.LineString:

6
internal/clip/point.go
View File

@ -8,3 +8,9 @@ func clipPoint(point *geojson.Point, clipper geojson.Object) geojson.Object {
}
return geojson.NewMultiPoint(nil)
}
func clipSimplePoint(point *geojson.SimplePoint, clipper geojson.Object) geojson.Object {
if point.IntersectsRect(clipper.Rect()) {
return point
}
return geojson.NewMultiPoint(nil)
}

56
internal/collection/item/item.go
View File

@ -10,10 +10,24 @@ import (
// Item is a item for Tile38 collections
type Item struct {
fieldsLen uint32 // fields block size in bytes, not num of fields
point bool // true: Item is pointItem, false: Item is objItem
fieldsLen uint16 // fields block size in bytes, not num of fields
idLen uint32 // id block size in bytes
data unsafe.Pointer // pointer to raw block of bytes
obj geojson.Object // geojson or string
data unsafe.Pointer // pointer to raw block of bytes, fields+id
}
type objItem struct {
_ bool
_ uint16
_ uint32
_ unsafe.Pointer
obj geojson.Object
}
type pointItem struct {
_ bool
_ uint16
_ uint32
_ unsafe.Pointer
pt geojson.SimplePoint
}
// ID returns the items ID as a string
@ -35,13 +49,25 @@ func (item *Item) Fields() []float64 {
// Obj returns the geojson object
func (item *Item) Obj() geojson.Object {
return item.obj
if item.point {
return &(*pointItem)(unsafe.Pointer(item)).pt
}
return (*objItem)(unsafe.Pointer(item)).obj
}
// New returns a newly allocated Item
func New(id string, obj geojson.Object) *Item {
item := new(Item)
item.obj = obj
var item *Item
if pt, ok := obj.(*geojson.SimplePoint); ok {
pitem := new(pointItem)
pitem.pt = *pt
item = (*Item)(unsafe.Pointer(pitem))
item.point = true
} else {
oitem := new(objItem)
oitem.obj = obj
item = (*Item)(unsafe.Pointer(oitem))
}
item.idLen = uint32(len(id))
if len(id) > 0 {
data := make([]byte, len(id))
@ -53,21 +79,21 @@ func New(id string, obj geojson.Object) *Item {
// WeightAndPoints returns the memory weight and number of points for Item.
func (item *Item) WeightAndPoints() (weight, points int) {
_, objIsSpatial := item.obj.(geojson.Spatial)
_, objIsSpatial := item.Obj().(geojson.Spatial)
if objIsSpatial {
points = item.obj.NumPoints()
points = item.Obj().NumPoints()
weight = points * 16
} else if item.obj != nil {
weight = len(item.obj.String())
} else if item.Obj() != nil {
weight = len(item.Obj().String())
}
weight += int(item.fieldsLen + item.idLen)
weight += int(item.fieldsLen) + int(item.idLen)
return weight, points
}
// Less is a btree interface that compares if item is less than other item.
func (item *Item) Less(other btree.Item, ctx interface{}) bool {
value1 := item.obj.String()
value2 := other.(*Item).obj.String()
value1 := item.Obj().String()
value2 := other.(*Item).Obj().String()
if value1 < value2 {
return true
}
@ -85,7 +111,7 @@ func (item *Item) CopyOverFields(values []float64) {
newData := make([]byte, len(fieldBytes)+int(item.idLen))
copy(newData, fieldBytes)
copy(newData[len(fieldBytes):], oldData[item.fieldsLen:])
item.fieldsLen = uint32(len(fieldBytes))
item.fieldsLen = uint16(len(fieldBytes))
if len(newData) > 0 {
item.data = unsafe.Pointer(&newData[0])
} else {
@ -118,7 +144,7 @@ func (item *Item) SetField(index int, value float64) (updated bool) {
// copy the id
copy(newData[(index+1)*8:], oldBytes[item.fieldsLen:])
// update the fields length
item.fieldsLen = uint32((index + 1) * 8)
item.fieldsLen = uint16((index + 1) * 8)
// update the raw data
item.data = unsafe.Pointer(&newData[0])
}

17
internal/server/crud.go
View File

@ -196,10 +196,7 @@ func (server *Server) cmdGet(msg *Message) (resp.Value, error) {
buf.Write(appendJSONSimplePoint(nil, o))
} else {
point := o.Center()
var z float64
if gPoint, ok := o.(*geojson.Point); ok {
z = gPoint.Z()
}
z, _ := geojson.IsPoint(o)
if z != 0 {
vals = append(vals, resp.ArrayValue([]resp.Value{
resp.StringValue(strconv.FormatFloat(point.Y, 'f', -1, 64)),
@ -672,7 +669,11 @@ func (server *Server) parseSetArgs(vs []string) (
err = errInvalidArgument(slon)
return
}
d.obj = geojson.NewPoint(geometry.Point{X: x, Y: y})
if server.geomParseOpts.AllowSimplePoints {
d.obj = geojson.NewSimplePoint(geometry.Point{X: x, Y: y})
} else {
d.obj = geojson.NewPoint(geometry.Point{X: x, Y: y})
}
} else {
var x, y, z float64
y, err = strconv.ParseFloat(slat, 64)
@ -742,7 +743,11 @@ func (server *Server) parseSetArgs(vs []string) (
return
}
lat, lon := geohash.Decode(shash)
d.obj = geojson.NewPoint(geometry.Point{X: lon, Y: lat})
if server.geomParseOpts.AllowSimplePoints {
d.obj = geojson.NewSimplePoint(geometry.Point{X: lon, Y: lat})
} else {
d.obj = geojson.NewPoint(geometry.Point{X: lon, Y: lat})
}
case lcb(typ, "object"):
var object string
if vs, object, ok = tokenval(vs); !ok || object == "" {

5
internal/server/json.go
View File

@ -56,10 +56,7 @@ func appendJSONSimpleBounds(dst []byte, o geojson.Object) []byte {
func appendJSONSimplePoint(dst []byte, o geojson.Object) []byte {
point := o.Center()
var z float64
if gPoint, ok := o.(*geojson.Point); ok {
z = gPoint.Z()
}
z, _ := geojson.IsPoint(o)
dst = append(dst, `{"lat":`...)
dst = strconv.AppendFloat(dst, point.Y, 'f', -1, 64)
dst = append(dst, `,"lon":`...)

13
internal/server/scanner.go
View File

@ -202,9 +202,7 @@ func (sw *scanWriter) fieldMatch(fields []float64, o geojson.Object) (fvals []fl
for _, where := range sw.wheres {
if where.field == "z" {
if !gotz {
if point, ok := o.(*geojson.Point); ok {
z = point.Z()
}
z, _ = geojson.IsPoint(o)
}
if !where.match(z) {
return
@ -258,9 +256,7 @@ func (sw *scanWriter) fieldMatch(fields []float64, o geojson.Object) (fvals []fl
for _, where := range sw.wheres {
if where.field == "z" {
if !gotz {
if point, ok := o.(*geojson.Point); ok {
z = point.Z()
}
z, _ = geojson.IsPoint(o)
}
if !where.match(z) {
return
@ -449,10 +445,7 @@ func (sw *scanWriter) writeObject(opts ScanWriterParams) bool {
vals = append(vals, resp.StringValue(opts.o.String()))
case outputPoints:
point := opts.o.Center()
var z float64
if point, ok := opts.o.(*geojson.Point); ok {
z = point.Z()
}
z, _ := geojson.IsPoint(opts.o)
if z != 0 {
vals = append(vals, resp.ArrayValue([]resp.Value{
resp.FloatValue(point.Y),

4
internal/server/server.go
View File

@ -170,11 +170,13 @@ func Serve(host string, port int, dir string, http bool) error {
return err
}
server.geomParseOpts = *geojson.DefaultParseOptions
server.geomParseOpts.AllowSimplePoints = true
// Allow for geometry indexing options through environment variables:
// T38IDXGEOMKIND -- None, RTree, QuadTree
// T38IDXGEOM -- Min number of points in a geometry for indexing.
// T38IDXMULTI -- Min number of object in a Multi/Collection for indexing.
server.geomParseOpts = *geojson.DefaultParseOptions
n, err := strconv.ParseUint(os.Getenv("T38IDXGEOM"), 10, 32)
if err == nil {
server.geomParseOpts.IndexGeometry = int(n)

17
vendor/github.com/tidwall/geojson/object.go generated vendored
View File

@ -50,7 +50,7 @@ var _ = []Object{
&Point{}, &LineString{}, &Polygon{}, &Feature{},
&MultiPoint{}, &MultiLineString{}, &MultiPolygon{},
&GeometryCollection{}, &FeatureCollection{},
&Rect{}, &Circle{},
&Rect{}, &Circle{}, &SimplePoint{},
}
// Collection is a searchable collection type.
@ -89,7 +89,12 @@ type ParseOptions struct {
// IndexGeometryKind is the kind of index implementation.
// Default is QuadTreeCompressed
IndexGeometryKind geometry.IndexKind
RequireValid bool
// RequireValid option cause parse to fail when a geojson object is invalid.
RequireValid bool
// AllowSimplePoints options will force to parse to return the SimplePoint
// type when a geojson point only consists of an 2D x/y coord and no extra
// json members.
AllowSimplePoints bool
}
// DefaultParseOptions ...
@ -98,6 +103,7 @@ var DefaultParseOptions = &ParseOptions{
IndexGeometry: 64,
IndexGeometryKind: geometry.QuadTree,
RequireValid: false,
AllowSimplePoints: false,
}
// Parse a GeoJSON object
@ -296,10 +302,3 @@ func unionRects(a, b geometry.Rect) geometry.Rect {
func geoDistancePoints(a, b geometry.Point) float64 {
return geo.DistanceTo(a.Y, a.X, b.Y, b.X)
}
// func geoDistanceCenterToPoint(obj Object, point geometry.Point) float64 {
// if obj.Empty() {
// return 0
// }
// return geoDistancePointsA(obj.Center(), point)
// }

32
vendor/github.com/tidwall/geojson/point.go generated vendored
View File

@ -152,21 +152,30 @@ func (g *Point) Z() float64 {
}
func parseJSONPoint(keys *parseKeys, opts *ParseOptions) (Object, error) {
var g Point
var err error
g.base, g.extra, err = parseJSONPointCoords(keys, gjson.Result{}, opts)
var o Object
base, extra, err := parseJSONPointCoords(keys, gjson.Result{}, opts)
if err != nil {
return nil, err
}
if err := parseBBoxAndExtras(&g.extra, keys, opts); err != nil {
if err := parseBBoxAndExtras(&extra, keys, opts); err != nil {
return nil, err
}
if extra == nil && opts.AllowSimplePoints {
var g SimplePoint
g.base = base
o = &g
} else {
var g Point
g.base = base
g.extra = extra
o = &g
}
if opts.RequireValid {
if !g.Valid() {
if !o.Valid() {
return nil, errCoordinatesInvalid
}
}
return &g, nil
return o, nil
}
func parseJSONPointCoords(
@ -244,3 +253,14 @@ func (g *Point) DistanceLine(line *geometry.Line) float64 {
func (g *Point) DistancePoly(poly *geometry.Poly) float64 {
return geoDistancePoints(g.Center(), poly.Rect().Center())
}
// IsPoint returns true if the object is a {"type":"Point"}
func IsPoint(obj Object) (z float64, ok bool) {
switch pt := obj.(type) {
case *SimplePoint:
return 0, true
case *Point:
return pt.Z(), true
}
return 0, false
}

156
vendor/github.com/tidwall/geojson/simplepoint.go generated vendored Normal file
View File

@ -0,0 +1,156 @@
package geojson
import "github.com/tidwall/geojson/geometry"
// SimplePoint ...
type SimplePoint struct {
base geometry.Point
}
// NewSimplePoint ...
func NewSimplePoint(point geometry.Point) *SimplePoint {
return &SimplePoint{base: point}
}
// ForEach ...
func (g *SimplePoint) ForEach(iter func(geom Object) bool) bool {
return iter(g)
}
// Empty ...
func (g *SimplePoint) Empty() bool {
return g.base.Empty()
}
// Valid ...
func (g *SimplePoint) Valid() bool {
return g.base.Valid()
}
// Rect ...
func (g *SimplePoint) Rect() geometry.Rect {
return g.base.Rect()
}
// Spatial ...
func (g *SimplePoint) Spatial() Spatial {
return g
}
// Center ...
func (g *SimplePoint) Center() geometry.Point {
return g.base
}
// Base ...
func (g *SimplePoint) Base() geometry.Point {
return g.base
}
// AppendJSON ...
func (g *SimplePoint) AppendJSON(dst []byte) []byte {
dst = append(dst, `{"type":"Point","coordinates":`...)
dst = appendJSONPoint(dst, g.base, nil, 0)
dst = append(dst, '}')
return dst
}
// JSON ...
func (g *SimplePoint) JSON() string {
return string(g.AppendJSON(nil))
}
// MarshalJSON ...
func (g *SimplePoint) MarshalJSON() ([]byte, error) {
return g.AppendJSON(nil), nil
}
// String ...
func (g *SimplePoint) String() string {
return string(g.AppendJSON(nil))
}
// Within ...
func (g *SimplePoint) Within(obj Object) bool {
return obj.Contains(g)
}
// Contains ...
func (g *SimplePoint) Contains(obj Object) bool {
return obj.Spatial().WithinPoint(g.base)
}
// Intersects ...
func (g *SimplePoint) Intersects(obj Object) bool {
return obj.Spatial().IntersectsPoint(g.base)
}
// WithinRect ...
func (g *SimplePoint) WithinRect(rect geometry.Rect) bool {
return rect.ContainsPoint(g.base)
}
// WithinPoint ...
func (g *SimplePoint) WithinPoint(point geometry.Point) bool {
return point.ContainsPoint(g.base)
}
// WithinLine ...
func (g *SimplePoint) WithinLine(line *geometry.Line) bool {
return line.ContainsPoint(g.base)
}
// WithinPoly ...
func (g *SimplePoint) WithinPoly(poly *geometry.Poly) bool {
return poly.ContainsPoint(g.base)
}
// IntersectsPoint ...
func (g *SimplePoint) IntersectsPoint(point geometry.Point) bool {
return g.base.IntersectsPoint(point)
}
// IntersectsRect ...
func (g *SimplePoint) IntersectsRect(rect geometry.Rect) bool {
return g.base.IntersectsRect(rect)
}
// IntersectsLine ...
func (g *SimplePoint) IntersectsLine(line *geometry.Line) bool {
return g.base.IntersectsLine(line)
}
// IntersectsPoly ...
func (g *SimplePoint) IntersectsPoly(poly *geometry.Poly) bool {
return g.base.IntersectsPoly(poly)
}
// NumPoints ...
func (g *SimplePoint) NumPoints() int {
return 1
}
// Distance ...
func (g *SimplePoint) Distance(obj Object) float64 {
return obj.Spatial().DistancePoint(g.base)
}
// DistancePoint ...
func (g *SimplePoint) DistancePoint(point geometry.Point) float64 {
return geoDistancePoints(g.Center(), point)
}
// DistanceRect ...
func (g *SimplePoint) DistanceRect(rect geometry.Rect) float64 {
return geoDistancePoints(g.Center(), rect.Center())
}
// DistanceLine ...
func (g *SimplePoint) DistanceLine(line *geometry.Line) float64 {
return geoDistancePoints(g.Center(), line.Rect().Center())
}
// DistancePoly ...
func (g *SimplePoint) DistancePoly(poly *geometry.Poly) float64 {
return geoDistancePoints(g.Center(), poly.Rect().Center())
}

61
vendor/github.com/tidwall/geojson/simplepoint_test.go generated vendored Normal file
View File

@ -0,0 +1,61 @@
package geojson
import "testing"
func TestSimplePointNotSimple(t *testing.T) {
p := expectJSONOpts(t, `{"type":"Point","coordinates":[1,2,3]}`, nil, &ParseOptions{AllowSimplePoints: true})
expect(t, p.Center() == P(1, 2))
expectJSONOpts(t, `{"type":"Point","coordinates":[1,null]}`, errCoordinatesInvalid, &ParseOptions{AllowSimplePoints: true})
expectJSONOpts(t, `{"type":"Point","coordinates":[1,2],"bbox":null}`, nil, &ParseOptions{AllowSimplePoints: true})
expectJSONOpts(t, `{"type":"Point"}`, errCoordinatesMissing, &ParseOptions{AllowSimplePoints: true})
expectJSONOpts(t, `{"type":"Point","coordinates":null}`, errCoordinatesInvalid, &ParseOptions{AllowSimplePoints: true})
expectJSONOpts(t, `{"type":"Point","coordinates":[1,2,3,4,5]}`, `{"type":"Point","coordinates":[1,2,3,4]}`, &ParseOptions{AllowSimplePoints: true})
expectJSONOpts(t, `{"type":"Point","coordinates":[1]}`, errCoordinatesInvalid, &ParseOptions{AllowSimplePoints: true})
expectJSONOpts(t, `{"type":"Point","coordinates":[1,2,3],"bbox":[1,2,3,4]}`, `{"type":"Point","coordinates":[1,2,3],"bbox":[1,2,3,4]}`, &ParseOptions{AllowSimplePoints: true})
}
func TestSimplePointParseValid(t *testing.T) {
json := `{"type":"Point","coordinates":[190,90]}`
p := expectJSONOpts(t, json, nil, &ParseOptions{AllowSimplePoints: true})
expect(t, !p.(*SimplePoint).Empty())
p = expectJSONOpts(t, json, nil, &ParseOptions{AllowSimplePoints: false})
expect(t, !p.(*Point).Empty())
p = expectJSONOpts(t, json, errCoordinatesInvalid, &ParseOptions{RequireValid: true, AllowSimplePoints: true})
expect(t, p == nil)
}
func TestSimplePointVarious(t *testing.T) {
var g Object = PO(10, 20)
expect(t, string(g.AppendJSON(nil)) == `{"type":"Point","coordinates":[10,20]}`)
expect(t, g.Rect() == R(10, 20, 10, 20))
expect(t, g.Center() == P(10, 20))
expect(t, !g.Empty())
}
func TestSimplePointValid(t *testing.T) {
var g Object = PO(0, 20)
expect(t, g.Valid())
var g1 Object = PO(10, 20)
expect(t, g1.Valid())
}
func TestSimplePointInvalidLargeX(t *testing.T) {
var g Object = PO(10, 91)
expect(t, !g.Valid())
}
func TestSimplePointInvalidLargeY(t *testing.T) {
var g Object = PO(181, 20)
expect(t, !g.Valid())
}
func TestSimplePointValidLargeX(t *testing.T) {
var g Object = PO(180, 20)
expect(t, g.Valid())
}
func TestSimplePointValidLargeY(t *testing.T) {
var g Object = PO(180, 90)
expect(t, g.Valid())
}