From fd29f8872f22521f7fceef8823a43fc8e90f7a6d Mon Sep 17 00:00:00 2001 From: Josh Baker Date: Sat, 17 Dec 2016 12:20:55 -0700 Subject: [PATCH] use gzip for many properties --- geojson/feature.go | 46 ++++++++++++++------- geojson/feature_test.go | 91 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 121 insertions(+), 16 deletions(-) diff --git a/geojson/feature.go b/geojson/feature.go index a74f80ce..e0a0bd8a 100644 --- a/geojson/feature.go +++ b/geojson/feature.go @@ -2,7 +2,9 @@ package geojson import ( "bytes" + "compress/gzip" "encoding/binary" + "io/ioutil" "github.com/tidwall/gjson" "github.com/tidwall/tile38/geojson/geohash" @@ -12,7 +14,7 @@ import ( type Feature struct { Geometry Object BBox *BBox - idprops string // raw id and properties seperated by a '\0' + idprops []byte // raw id and properties combined } func fillFeatureMap(json string) (Feature, error) { @@ -92,25 +94,30 @@ func (g Feature) MarshalJSON() ([]byte, error) { return []byte(g.JSON()), nil } -func (g Feature) getRaw() (id, props string) { +func (g Feature) getRaw() (id, props []byte) { if len(g.idprops) == 0 { - return "", "" + return nil, nil } - switch g.idprops[0] { + buf := g.idprops + rd, err := gzip.NewReader(bytes.NewReader(buf)) + if err == nil { + buf, _ = ioutil.ReadAll(rd) + } + switch buf[0] { default: - lnp := int(g.idprops[0]) + 1 - return g.idprops[1:lnp], g.idprops[lnp:] + lnp := int(buf[0]) + 1 + return buf[1:lnp], buf[lnp:] case 255: - lnp := int(binary.LittleEndian.Uint64([]byte(g.idprops[1:9]))) + 9 - return g.idprops[9:lnp], g.idprops[lnp:] + lnp := int(binary.LittleEndian.Uint64([]byte(buf[1:9]))) + 9 + return buf[9:lnp], buf[lnp:] } } -func makeCompositeRaw(idRaw, propsRaw string) string { +func makeCompositeRaw(idRaw, propsRaw string) []byte { idRaw = stripWhitespace(idRaw) propsRaw = stripWhitespace(propsRaw) if len(idRaw) == 0 && len(propsRaw) == 0 { - return "" + return nil } var raw []byte if len(idRaw) > 0xFF-1 { @@ -125,7 +132,16 @@ func makeCompositeRaw(idRaw, propsRaw string) string { copy(raw[1:], idRaw) copy(raw[len(idRaw)+1:], propsRaw) } - return string(raw) + if len(raw) < 256 { + return raw + } + var buf bytes.Buffer + gbuf := gzip.NewWriter(&buf) + gbuf.Write(raw) + gbuf.Close() + tight := make([]byte, len(buf.Bytes())) + copy(tight, buf.Bytes()) + return tight } // JSON is the json representation of the object. This might not be exactly the same as the original. @@ -135,13 +151,13 @@ func (g Feature) JSON() string { buf.WriteString(g.Geometry.JSON()) g.BBox.write(&buf) idRaw, propsRaw := g.getRaw() - if propsRaw != "" { + if len(propsRaw) != 0 { buf.WriteString(`,"properties":`) - buf.WriteString(propsRaw) + buf.Write(propsRaw) } - if idRaw != "" { + if len(idRaw) != 0 { buf.WriteString(`,"id":`) - buf.WriteString(idRaw) + buf.Write(idRaw) } buf.WriteByte('}') return buf.String() diff --git a/geojson/feature_test.go b/geojson/feature_test.go index 46c7a67d..d475f997 100644 --- a/geojson/feature_test.go +++ b/geojson/feature_test.go @@ -1,6 +1,9 @@ package geojson -import "testing" +import ( + "fmt" + "testing" +) func TestFeature(t *testing.T) { testJSON(t, `{ @@ -32,3 +35,89 @@ func TestFeature(t *testing.T) { } }`) } + +var complexFeature = `{ + "id": 202418985, + "type": "Feature", + "properties": { + "addr:full":"5607 McKinley Ave Los Angeles CA 90011", + "addr:housenumber":"5607", + "addr:postcode":"90011", + "addr:street":"Mckinley Ave", + "edtf:cessation":"uuuu", + "edtf:inception":"uuuu", + "geom:area":0.0, + "geom:bbox":"-118.26089,33.99073,-118.26089,33.99073", + "geom:latitude":33.99073, + "geom:longitude":-118.26089, + "iso:country":"US", + "mz:hierarchy_label":1, + "sg:address":"5607 McKinley Ave", + "sg:city":"Los Angeles", + "sg:classifiers":[ + { + "category":"Wholesale", + "subcategory":"Toys & Hobbies", + "type":"Manufacturing & Wholesale Goods" + } + ], + "sg:owner":"simplegeo", + "sg:phone":"+1 323 231 0540", + "sg:postcode":"90011", + "sg:province":"CA", + "sg:tags":[ + "wholesaler" + ], + "src:geom":"simplegeo", + "wof:belongsto":[ + 85633793, + 85688637 + ], + "wof:breaches":[], + "wof:concordances":{ + "sg:id":"SG_0i3ZtGVxBmvnGGcg7wZrlY_33.990730_-118.260890@1294081369" + }, + "wof:country":"US", + "wof:geomhash":"fa3426d7a9b6c92b5e2857b4daef560f", + "wof:hierarchy":[ + { + "continent_id":-1, + "country_id":85633793, + "locality_id":-1, + "neighbourhood_id":-1, + "region_id":85688637, + "venue_id":202418985 + } + ], + "wof:id":202418985, + "wof:lastmodified":1472331065, + "wof:name":"Hotfix Wholesale Inc", + "wof:parent_id":-1, + "wof:placetype":"venue", + "wof:repo":"whosonfirst-data-venue-us-ca", + "wof:superseded_by":[], + "wof:supersedes":[], + "wof:tags":[ + "wholesaler" + ], + "added:by:tidwall": "\n\"\\\\15\u00f8C 3\u0111" +}, + "bbox": [ + -118.26089, + 33.99073, + -118.26089, + 33.99073 +], + "geometry": {"coordinates":[-118.26089,33.99073],"type":"Point"} +}` + +func TestComplexFeature(t *testing.T) { + testJSON(t, complexFeature) + o, err := ObjectJSON(complexFeature) + if err != nil { + t.Fatal(err) + } + return + println(len(o.(Feature).idprops), cap(o.(Feature).idprops)) + fmt.Printf("%v\n", o.JSON()) +}