mirror of https://github.com/tidwall/tile38.git
use gzip for many properties
This commit is contained in:
Josh Baker
parent
19da27b562
commit
fd29f8872f
|
|
@ -2,7 +2,9 @@ package geojson
|
|||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"encoding/binary"
|
||||
"io/ioutil"
|
||||
|
||||
"github.com/tidwall/gjson"
|
||||
"github.com/tidwall/tile38/geojson/geohash"
|
||||
|
|
@ -12,7 +14,7 @@ import (
|
|||
type Feature struct {
|
||||
Geometry Object
|
||||
BBox *BBox
|
||||
idprops string // raw id and properties seperated by a '\0'
|
||||
idprops []byte // raw id and properties combined
|
||||
}
|
||||
|
||||
func fillFeatureMap(json string) (Feature, error) {
|
||||
|
|
@ -92,25 +94,30 @@ func (g Feature) MarshalJSON() ([]byte, error) {
|
|||
return []byte(g.JSON()), nil
|
||||
}
|
||||
|
||||
func (g Feature) getRaw() (id, props string) {
|
||||
func (g Feature) getRaw() (id, props []byte) {
|
||||
if len(g.idprops) == 0 {
|
||||
return "", ""
|
||||
return nil, nil
|
||||
}
|
||||
switch g.idprops[0] {
|
||||
buf := g.idprops
|
||||
rd, err := gzip.NewReader(bytes.NewReader(buf))
|
||||
if err == nil {
|
||||
buf, _ = ioutil.ReadAll(rd)
|
||||
}
|
||||
switch buf[0] {
|
||||
default:
|
||||
lnp := int(g.idprops[0]) + 1
|
||||
return g.idprops[1:lnp], g.idprops[lnp:]
|
||||
lnp := int(buf[0]) + 1
|
||||
return buf[1:lnp], buf[lnp:]
|
||||
case 255:
|
||||
lnp := int(binary.LittleEndian.Uint64([]byte(g.idprops[1:9]))) + 9
|
||||
return g.idprops[9:lnp], g.idprops[lnp:]
|
||||
lnp := int(binary.LittleEndian.Uint64([]byte(buf[1:9]))) + 9
|
||||
return buf[9:lnp], buf[lnp:]
|
||||
}
|
||||
}
|
||||
|
||||
func makeCompositeRaw(idRaw, propsRaw string) string {
|
||||
func makeCompositeRaw(idRaw, propsRaw string) []byte {
|
||||
idRaw = stripWhitespace(idRaw)
|
||||
propsRaw = stripWhitespace(propsRaw)
|
||||
if len(idRaw) == 0 && len(propsRaw) == 0 {
|
||||
return ""
|
||||
return nil
|
||||
}
|
||||
var raw []byte
|
||||
if len(idRaw) > 0xFF-1 {
|
||||
|
|
@ -125,7 +132,16 @@ func makeCompositeRaw(idRaw, propsRaw string) string {
|
|||
copy(raw[1:], idRaw)
|
||||
copy(raw[len(idRaw)+1:], propsRaw)
|
||||
}
|
||||
return string(raw)
|
||||
if len(raw) < 256 {
|
||||
return raw
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
gbuf := gzip.NewWriter(&buf)
|
||||
gbuf.Write(raw)
|
||||
gbuf.Close()
|
||||
tight := make([]byte, len(buf.Bytes()))
|
||||
copy(tight, buf.Bytes())
|
||||
return tight
|
||||
}
|
||||
|
||||
// JSON is the json representation of the object. This might not be exactly the same as the original.
|
||||
|
|
@ -135,13 +151,13 @@ func (g Feature) JSON() string {
|
|||
buf.WriteString(g.Geometry.JSON())
|
||||
g.BBox.write(&buf)
|
||||
idRaw, propsRaw := g.getRaw()
|
||||
if propsRaw != "" {
|
||||
if len(propsRaw) != 0 {
|
||||
buf.WriteString(`,"properties":`)
|
||||
buf.WriteString(propsRaw)
|
||||
buf.Write(propsRaw)
|
||||
}
|
||||
if idRaw != "" {
|
||||
if len(idRaw) != 0 {
|
||||
buf.WriteString(`,"id":`)
|
||||
buf.WriteString(idRaw)
|
||||
buf.Write(idRaw)
|
||||
}
|
||||
buf.WriteByte('}')
|
||||
return buf.String()
|
||||
|
|
|
|||
|
|
@ -1,6 +1,9 @@
|
|||
package geojson
|
||||
|
||||
import "testing"
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestFeature(t *testing.T) {
|
||||
testJSON(t, `{
|
||||
|
|
@ -32,3 +35,89 @@ func TestFeature(t *testing.T) {
|
|||
}
|
||||
}`)
|
||||
}
|
||||
|
||||
var complexFeature = `{
|
||||
"id": 202418985,
|
||||
"type": "Feature",
|
||||
"properties": {
|
||||
"addr:full":"5607 McKinley Ave Los Angeles CA 90011",
|
||||
"addr:housenumber":"5607",
|
||||
"addr:postcode":"90011",
|
||||
"addr:street":"Mckinley Ave",
|
||||
"edtf:cessation":"uuuu",
|
||||
"edtf:inception":"uuuu",
|
||||
"geom:area":0.0,
|
||||
"geom:bbox":"-118.26089,33.99073,-118.26089,33.99073",
|
||||
"geom:latitude":33.99073,
|
||||
"geom:longitude":-118.26089,
|
||||
"iso:country":"US",
|
||||
"mz:hierarchy_label":1,
|
||||
"sg:address":"5607 McKinley Ave",
|
||||
"sg:city":"Los Angeles",
|
||||
"sg:classifiers":[
|
||||
{
|
||||
"category":"Wholesale",
|
||||
"subcategory":"Toys & Hobbies",
|
||||
"type":"Manufacturing & Wholesale Goods"
|
||||
}
|
||||
],
|
||||
"sg:owner":"simplegeo",
|
||||
"sg:phone":"+1 323 231 0540",
|
||||
"sg:postcode":"90011",
|
||||
"sg:province":"CA",
|
||||
"sg:tags":[
|
||||
"wholesaler"
|
||||
],
|
||||
"src:geom":"simplegeo",
|
||||
"wof:belongsto":[
|
||||
85633793,
|
||||
85688637
|
||||
],
|
||||
"wof:breaches":[],
|
||||
"wof:concordances":{
|
||||
"sg:id":"SG_0i3ZtGVxBmvnGGcg7wZrlY_33.990730_-118.260890@1294081369"
|
||||
},
|
||||
"wof:country":"US",
|
||||
"wof:geomhash":"fa3426d7a9b6c92b5e2857b4daef560f",
|
||||
"wof:hierarchy":[
|
||||
{
|
||||
"continent_id":-1,
|
||||
"country_id":85633793,
|
||||
"locality_id":-1,
|
||||
"neighbourhood_id":-1,
|
||||
"region_id":85688637,
|
||||
"venue_id":202418985
|
||||
}
|
||||
],
|
||||
"wof:id":202418985,
|
||||
"wof:lastmodified":1472331065,
|
||||
"wof:name":"Hotfix Wholesale Inc",
|
||||
"wof:parent_id":-1,
|
||||
"wof:placetype":"venue",
|
||||
"wof:repo":"whosonfirst-data-venue-us-ca",
|
||||
"wof:superseded_by":[],
|
||||
"wof:supersedes":[],
|
||||
"wof:tags":[
|
||||
"wholesaler"
|
||||
],
|
||||
"added:by:tidwall": "\n\"\\\\15\u00f8C 3\u0111"
|
||||
},
|
||||
"bbox": [
|
||||
-118.26089,
|
||||
33.99073,
|
||||
-118.26089,
|
||||
33.99073
|
||||
],
|
||||
"geometry": {"coordinates":[-118.26089,33.99073],"type":"Point"}
|
||||
}`
|
||||
|
||||
func TestComplexFeature(t *testing.T) {
|
||||
testJSON(t, complexFeature)
|
||||
o, err := ObjectJSON(complexFeature)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return
|
||||
println(len(o.(Feature).idprops), cap(o.(Feature).idprops))
|
||||
fmt.Printf("%v\n", o.JSON())
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue