use gzip for many properties

geojson/feature.go

@@ -2,7 +2,9 @@ package geojson
 
 import (
 	"bytes"
+	"compress/gzip"
 	"encoding/binary"
+	"io/ioutil"
 
 	"github.com/tidwall/gjson"
 	"github.com/tidwall/tile38/geojson/geohash"
@@ -12,7 +14,7 @@ import (
 type Feature struct {
 	Geometry Object
 	BBox     *BBox
-	idprops  string // raw id and properties seperated by a '\0'
+	idprops  []byte // raw id and properties combined
 }
 
 func fillFeatureMap(json string) (Feature, error) {
@@ -92,25 +94,30 @@ func (g Feature) MarshalJSON() ([]byte, error) {
 	return []byte(g.JSON()), nil
 }
 
-func (g Feature) getRaw() (id, props string) {
+func (g Feature) getRaw() (id, props []byte) {
 	if len(g.idprops) == 0 {
-		return "", ""
+		return nil, nil
 	}
-	switch g.idprops[0] {
+	buf := g.idprops
+	rd, err := gzip.NewReader(bytes.NewReader(buf))
+	if err == nil {
+		buf, _ = ioutil.ReadAll(rd)
+	}
+	switch buf[0] {
 	default:
-		lnp := int(g.idprops[0]) + 1
+		lnp := int(buf[0]) + 1
-		return g.idprops[1:lnp], g.idprops[lnp:]
+		return buf[1:lnp], buf[lnp:]
 	case 255:
-		lnp := int(binary.LittleEndian.Uint64([]byte(g.idprops[1:9]))) + 9
+		lnp := int(binary.LittleEndian.Uint64([]byte(buf[1:9]))) + 9
-		return g.idprops[9:lnp], g.idprops[lnp:]
+		return buf[9:lnp], buf[lnp:]
 	}
 }
 
-func makeCompositeRaw(idRaw, propsRaw string) string {
+func makeCompositeRaw(idRaw, propsRaw string) []byte {
 	idRaw = stripWhitespace(idRaw)
 	propsRaw = stripWhitespace(propsRaw)
 	if len(idRaw) == 0 && len(propsRaw) == 0 {
-		return ""
+		return nil
 	}
 	var raw []byte
 	if len(idRaw) > 0xFF-1 {
@@ -125,7 +132,16 @@ func makeCompositeRaw(idRaw, propsRaw string) string {
 		copy(raw[1:], idRaw)
 		copy(raw[len(idRaw)+1:], propsRaw)
 	}
-	return string(raw)
+	if len(raw) < 256 {
+		return raw
+	}
+	var buf bytes.Buffer
+	gbuf := gzip.NewWriter(&buf)
+	gbuf.Write(raw)
+	gbuf.Close()
+	tight := make([]byte, len(buf.Bytes()))
+	copy(tight, buf.Bytes())
+	return tight
 }
 
 // JSON is the json representation of the object. This might not be exactly the same as the original.
@@ -135,13 +151,13 @@ func (g Feature) JSON() string {
 	buf.WriteString(g.Geometry.JSON())
 	g.BBox.write(&buf)
 	idRaw, propsRaw := g.getRaw()
-	if propsRaw != "" {
+	if len(propsRaw) != 0 {
 		buf.WriteString(`,"properties":`)
-		buf.WriteString(propsRaw)
+		buf.Write(propsRaw)
 	}
-	if idRaw != "" {
+	if len(idRaw) != 0 {
 		buf.WriteString(`,"id":`)
-		buf.WriteString(idRaw)
+		buf.Write(idRaw)
 	}
 	buf.WriteByte('}')
 	return buf.String()

geojson/feature_test.go

@@ -1,6 +1,9 @@
 package geojson
 
-import "testing"
+import (
+	"fmt"
+	"testing"
+)
 
 func TestFeature(t *testing.T) {
 	testJSON(t, `{
@@ -32,3 +35,89 @@ func TestFeature(t *testing.T) {
     }
 }`)
 }
+
+var complexFeature = `{
+  "id": 202418985,
+  "type": "Feature",
+  "properties": {
+    "addr:full":"5607 McKinley Ave Los Angeles CA 90011",
+    "addr:housenumber":"5607",
+    "addr:postcode":"90011",
+    "addr:street":"Mckinley Ave",
+    "edtf:cessation":"uuuu",
+    "edtf:inception":"uuuu",
+    "geom:area":0.0,
+    "geom:bbox":"-118.26089,33.99073,-118.26089,33.99073",
+    "geom:latitude":33.99073,
+    "geom:longitude":-118.26089,
+    "iso:country":"US",
+    "mz:hierarchy_label":1,
+    "sg:address":"5607 McKinley Ave",
+    "sg:city":"Los Angeles",
+    "sg:classifiers":[
+        {
+            "category":"Wholesale",
+            "subcategory":"Toys & Hobbies",
+            "type":"Manufacturing & Wholesale Goods"
+        }
+    ],
+    "sg:owner":"simplegeo",
+    "sg:phone":"+1 323 231 0540",
+    "sg:postcode":"90011",
+    "sg:province":"CA",
+    "sg:tags":[
+        "wholesaler"
+    ],
+    "src:geom":"simplegeo",
+    "wof:belongsto":[
+        85633793,
+        85688637
+    ],
+    "wof:breaches":[],
+    "wof:concordances":{
+        "sg:id":"SG_0i3ZtGVxBmvnGGcg7wZrlY_33.990730_-118.260890@1294081369"
+    },
+    "wof:country":"US",
+    "wof:geomhash":"fa3426d7a9b6c92b5e2857b4daef560f",
+    "wof:hierarchy":[
+        {
+            "continent_id":-1,
+            "country_id":85633793,
+            "locality_id":-1,
+            "neighbourhood_id":-1,
+            "region_id":85688637,
+            "venue_id":202418985
+        }
+    ],
+    "wof:id":202418985,
+    "wof:lastmodified":1472331065,
+    "wof:name":"Hotfix Wholesale Inc",
+    "wof:parent_id":-1,
+    "wof:placetype":"venue",
+    "wof:repo":"whosonfirst-data-venue-us-ca",
+    "wof:superseded_by":[],
+    "wof:supersedes":[],
+    "wof:tags":[
+        "wholesaler"
+    ],
+	"added:by:tidwall": "\n\"\\\\15\u00f8C 3\u0111"
+},
+  "bbox": [
+    -118.26089,
+    33.99073,
+    -118.26089,
+    33.99073
+],
+  "geometry": {"coordinates":[-118.26089,33.99073],"type":"Point"}
+}`
+
+func TestComplexFeature(t *testing.T) {
+	testJSON(t, complexFeature)
+	o, err := ObjectJSON(complexFeature)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return
+	println(len(o.(Feature).idprops), cap(o.(Feature).idprops))
+	fmt.Printf("%v\n", o.JSON())
+}