From 1ac6ad9ebd55ee1e5244813cb900e2403046091a Mon Sep 17 00:00:00 2001 From: Josh Baker Date: Mon, 7 Nov 2016 13:04:21 -0700 Subject: [PATCH] optimized idprops field for #71 --- controller/json.go | 10 +- geojson/feature.go | 45 ++++++-- geojson/object.go | 14 ++- geojson/string.go | 4 +- vendor/github.com/tidwall/cast/LICENSE | 20 ---- vendor/github.com/tidwall/cast/README.md | 42 -------- vendor/github.com/tidwall/cast/cast.go | 27 ----- vendor/github.com/tidwall/cast/cast_test.go | 107 -------------------- 8 files changed, 48 insertions(+), 221 deletions(-) delete mode 100644 vendor/github.com/tidwall/cast/LICENSE delete mode 100644 vendor/github.com/tidwall/cast/README.md delete mode 100644 vendor/github.com/tidwall/cast/cast.go delete mode 100644 vendor/github.com/tidwall/cast/cast_test.go diff --git a/controller/json.go b/controller/json.go index 0c8fc35a..3cf765d2 100644 --- a/controller/json.go +++ b/controller/json.go @@ -1,10 +1,6 @@ package controller -import ( - "encoding/json" - - "github.com/tidwall/cast" -) +import "encoding/json" func jsonString(s string) string { for i := 0; i < len(s); i++ { @@ -15,7 +11,7 @@ func jsonString(s string) string { } b := make([]byte, len(s)+2) b[0] = '"' - copy(b[1:], cast.ToBytes(s)) + copy(b[1:], s) b[len(b)-1] = '"' - return cast.ToString(b) + return string(b) } diff --git a/geojson/feature.go b/geojson/feature.go index cb209f40..bd7a5576 100644 --- a/geojson/feature.go +++ b/geojson/feature.go @@ -2,6 +2,7 @@ package geojson import ( "bytes" + "encoding/binary" "github.com/tidwall/gjson" "github.com/tidwall/tile38/geojson/geohash" @@ -46,12 +47,7 @@ func fillFeatureMap(json string) (Feature, []byte, error) { } id := gjson.Get(json, "id") if id.Exists() || propsExists { - idRaw := stripWhitespace(id.Raw) - propsRaw := stripWhitespace(props.Raw) - raw := make([]byte, len(idRaw)+len(propsRaw)+1) - copy(raw, idRaw) - copy(raw[len(idRaw)+1:], propsRaw) - g.idprops = string(raw) + g.idprops = makeCompositeRaw(id.Raw, props.Raw) } return g, nil, err } @@ -97,12 +93,39 @@ func (g Feature) MarshalJSON() ([]byte, error) { } func (g Feature) getRaw() (id, props string) { - for i := 0; i < len(g.idprops); i++ { - if g.idprops[i] == 0 { - return g.idprops[:i], g.idprops[i+1:] - } + if len(g.idprops) == 0 { + return "", "" } - return "", "" + switch g.idprops[0] { + default: + lnp := int(g.idprops[0]) + 1 + return g.idprops[1:lnp], g.idprops[lnp:] + case 255: + lnp := int(binary.LittleEndian.Uint64([]byte(g.idprops[1:9]))) + 9 + return g.idprops[9:lnp], g.idprops[lnp:] + } +} + +func makeCompositeRaw(idRaw, propsRaw string) string { + idRaw = stripWhitespace(idRaw) + propsRaw = stripWhitespace(propsRaw) + if len(idRaw) == 0 && len(propsRaw) == 0 { + return "" + } + var raw []byte + if len(idRaw) > 0xFF-1 { + raw = make([]byte, len(idRaw)+len(propsRaw)+9) + raw[0] = 0xFF + binary.LittleEndian.PutUint64(raw[1:9], uint64(len(idRaw))) + copy(raw[9:], idRaw) + copy(raw[len(idRaw)+9:], propsRaw) + } else { + raw = make([]byte, len(idRaw)+len(propsRaw)+1) + raw[0] = byte(len(idRaw)) + copy(raw[1:], idRaw) + copy(raw[len(idRaw)+1:], propsRaw) + } + return string(raw) } // JSON is the json representation of the object. This might not be exactly the same as the original. diff --git a/geojson/object.go b/geojson/object.go index 3031daa3..5f44c4b3 100644 --- a/geojson/object.go +++ b/geojson/object.go @@ -3,6 +3,7 @@ package geojson import ( "bytes" "encoding/binary" + "encoding/json" "errors" "fmt" "math" @@ -336,13 +337,18 @@ func nearbyObjectShared(g Object, x, y float64, meters float64) bool { return g.Intersects(circlePoly) } -func mustMarshalString(s string) bool { +func jsonMarshalString(s string) []byte { for i := 0; i < len(s); i++ { - if s[i] < ' ' || s[i] > 0x7f || s[i] == '"' { - return true + if s[i] < ' ' || s[i] == '\\' || s[i] == '"' || s[i] > 126 { + b, _ := json.Marshal(s) + return b } } - return false + b := make([]byte, len(s)+2) + b[0] = '"' + copy(b[1:], s) + b[len(b)-1] = '"' + return b } func stripWhitespace(s string) string { diff --git a/geojson/string.go b/geojson/string.go index 2c99db22..716f43e6 100644 --- a/geojson/string.go +++ b/geojson/string.go @@ -1,7 +1,5 @@ package geojson -import "encoding/json" - // String is a not a geojson object, but just a string type String string @@ -80,7 +78,7 @@ func (s String) Weight() int { // MarshalJSON allows the object to be encoded in json.Marshal calls. func (s String) MarshalJSON() ([]byte, error) { - return json.Marshal(string(s)) + return jsonMarshalString(string(s)), nil } // Geohash converts the object to a geohash value. diff --git a/vendor/github.com/tidwall/cast/LICENSE b/vendor/github.com/tidwall/cast/LICENSE deleted file mode 100644 index 58f5819a..00000000 --- a/vendor/github.com/tidwall/cast/LICENSE +++ /dev/null @@ -1,20 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2016 Josh Baker - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of -the Software, and to permit persons to whom the Software is furnished to do so, -subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS -FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR -COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER -IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/tidwall/cast/README.md b/vendor/github.com/tidwall/cast/README.md deleted file mode 100644 index c518d9eb..00000000 --- a/vendor/github.com/tidwall/cast/README.md +++ /dev/null @@ -1,42 +0,0 @@ -CAST -==== -![Travis CI Build Status](https://api.travis-ci.org/tidwall/cast.svg?branch=master) -[![GoDoc](https://godoc.org/github.com/tidwall/cast?status.svg)](https://godoc.org/github.com/tidwall/cast) - -Quickly convert string <-> []byte without memory reallocations and create mutable string or immutable []byte. - -This package is a **danger zone** and should not be entered without understanding the ground rules. - -1. Converting a string -> []byte will result in an immutable byte slice. Editing will cause a panic. -2. Converting a []byte -> string will result in a mutable string. Editing the originial bytes will change the string too. - - -Create immutable []byte: - -```go -var s = "Hello Planet" -b := cast.ToBytes(s) -fmt.Printf("%s\n", "J"+string(b[1:])) - -// Output: -// Jello Planet -``` - -Create mutable string: - -```go -var b = []byte("Hello Planet") -s := cast.ToString(b) -b[0] = 'J' -fmt.Printf("%s\n", s) - -// Output: -// Jello Planet -``` - -## Contact -Josh Baker [@tidwall](http://twitter.com/tidwall) - -## License - -CAST source code is available under the MIT [License](/LICENSE). diff --git a/vendor/github.com/tidwall/cast/cast.go b/vendor/github.com/tidwall/cast/cast.go deleted file mode 100644 index 043752dc..00000000 --- a/vendor/github.com/tidwall/cast/cast.go +++ /dev/null @@ -1,27 +0,0 @@ -package cast - -import ( - "reflect" - "unsafe" -) - -// ToString will quickly return a string representation of a []byte without -// allocating new memory or having to copy data. Converting a []byte -> string -// will result in a mutable string. Editing the originial bytes will change the -// string too. -func ToString(b []byte) string { - // A string is 16 bytes and []byte is 24 bytes, therefore []byte can be - // directly assigned to a string. - return *(*string)(unsafe.Pointer(&b)) -} - -// ToBytes will quickly return a []byte representation of a string without -// allocating new memory or having to copy data. Converting a string -> []byte -// will result in an immutable byte slice. Editing will cause a panic. -func ToBytes(s string) []byte { - // A string is smaller than a []byte so we need to create a new byte - // header and fill in the fields. - sh := *(*reflect.StringHeader)(unsafe.Pointer(&s)) - bh := reflect.SliceHeader{sh.Data, sh.Len, sh.Len} - return *(*[]byte)(unsafe.Pointer(&bh)) -} diff --git a/vendor/github.com/tidwall/cast/cast_test.go b/vendor/github.com/tidwall/cast/cast_test.go deleted file mode 100644 index 5940b4c1..00000000 --- a/vendor/github.com/tidwall/cast/cast_test.go +++ /dev/null @@ -1,107 +0,0 @@ -package cast - -import ( - "crypto/rand" - "fmt" - "runtime" - "testing" - "time" -) - -// getHeap returns the size of the heap -func getHeap(seed int) int { - runtime.GC() - var ms runtime.MemStats - runtime.ReadMemStats(&ms) - return int(ms.HeapAlloc) - seed -} - -// TestCast will create a 100KB []byte and assign the first 50KB -// to a string using ToString() and the copy the second 50KB to a -// string using string(). Both will be assigned to a map[string]bool -// and should result in around 150KB of heap. 100KB for the original -// []byte and 50 for the string() copy. -// We will test read the heap alloc to see if it's around 150KB. -// Then we'll nil the map. -// Then wait up to 10 seconds for the memory to get near zero. -func TestToString(t *testing.T) { - const sz = 1024 * 500 - var m [2]string - ch := make(chan bool) - var start time.Time - seed := getHeap(0) - go func() { - b := make([]byte, sz) - rand.Read(b) - m[0] = ToString(b[len(b)/2:]) - m[1] = string(b[:len(b)/2]) - ch <- true - }() - <-ch - if 1.0-float64(getHeap(seed))/(sz+sz/2.0) > 0.05 { - t.Fatal("failed") - } - m[0], m[1] = "", "" - start = time.Now() - for { - if time.Now().Sub(start) > time.Second*10 { - t.Fatal("failed") - } - per := 1.0 - float64(getHeap(seed))/(sz+sz/2.0) - if per > 0.95 { - break - } - } -} - -// TestToBytes is the same as TestToString, but the other way around. -func TestToBytes(t *testing.T) { - const sz = 1024 * 500 - var m [2][]byte - ch := make(chan bool) - var start time.Time - seed := getHeap(0) - go func() { - b := make([]byte, sz) - rand.Read(b) - s := string(b) - b = nil - m[0] = ToBytes(s[len(s)/2:]) - m[1] = []byte(s[:len(s)/2]) - ch <- true - }() - <-ch - if 1.0-float64(getHeap(seed))/(sz+sz/2.0) > 0.05 { - t.Fatal("failed") - } - m[0], m[1] = nil, nil - start = time.Now() - for { - if time.Now().Sub(start) > time.Second*10 { - t.Fatal("failed") - } - per := 1.0 - float64(getHeap(seed))/(sz+sz/2.0) - if per > 0.95 { - break - } - } -} - -func ExampleToBytes() { - var s = "Hello Planet" - b := ToBytes(s) - fmt.Printf("%s\n", "J"+string(b[1:])) - - // Output: - // Jello Planet -} - -func ExampleToString() { - var b = []byte("Hello Planet") - s := ToString(b) - b[0] = 'J' - fmt.Printf("%s\n", s) - - // Output: - // Jello Planet -}