optimized idprops field for #71

controller/json.go

@@ -1,10 +1,6 @@
 package controller
 
-import (
+import "encoding/json"
-	"encoding/json"
-
-	"github.com/tidwall/cast"
-)
 
 func jsonString(s string) string {
 	for i := 0; i < len(s); i++ {
@@ -15,7 +11,7 @@ func jsonString(s string) string {
 	}
 	b := make([]byte, len(s)+2)
 	b[0] = '"'
-	copy(b[1:], cast.ToBytes(s))
+	copy(b[1:], s)
 	b[len(b)-1] = '"'
-	return cast.ToString(b)
+	return string(b)
 }

geojson/feature.go

@@ -2,6 +2,7 @@ package geojson
 
 import (
 	"bytes"
+	"encoding/binary"
 
 	"github.com/tidwall/gjson"
 	"github.com/tidwall/tile38/geojson/geohash"
@@ -46,12 +47,7 @@ func fillFeatureMap(json string) (Feature, []byte, error) {
 	}
 	id := gjson.Get(json, "id")
 	if id.Exists() || propsExists {
-		idRaw := stripWhitespace(id.Raw)
+		g.idprops = makeCompositeRaw(id.Raw, props.Raw)
-		propsRaw := stripWhitespace(props.Raw)
-		raw := make([]byte, len(idRaw)+len(propsRaw)+1)
-		copy(raw, idRaw)
-		copy(raw[len(idRaw)+1:], propsRaw)
-		g.idprops = string(raw)
 	}
 	return g, nil, err
 }
@@ -97,12 +93,39 @@ func (g Feature) MarshalJSON() ([]byte, error) {
 }
 
 func (g Feature) getRaw() (id, props string) {
-	for i := 0; i < len(g.idprops); i++ {
+	if len(g.idprops) == 0 {
-		if g.idprops[i] == 0 {
+		return "", ""
-			return g.idprops[:i], g.idprops[i+1:]
-		}
 	}
-	return "", ""
+	switch g.idprops[0] {
+	default:
+		lnp := int(g.idprops[0]) + 1
+		return g.idprops[1:lnp], g.idprops[lnp:]
+	case 255:
+		lnp := int(binary.LittleEndian.Uint64([]byte(g.idprops[1:9]))) + 9
+		return g.idprops[9:lnp], g.idprops[lnp:]
+	}
+}
+
+func makeCompositeRaw(idRaw, propsRaw string) string {
+	idRaw = stripWhitespace(idRaw)
+	propsRaw = stripWhitespace(propsRaw)
+	if len(idRaw) == 0 && len(propsRaw) == 0 {
+		return ""
+	}
+	var raw []byte
+	if len(idRaw) > 0xFF-1 {
+		raw = make([]byte, len(idRaw)+len(propsRaw)+9)
+		raw[0] = 0xFF
+		binary.LittleEndian.PutUint64(raw[1:9], uint64(len(idRaw)))
+		copy(raw[9:], idRaw)
+		copy(raw[len(idRaw)+9:], propsRaw)
+	} else {
+		raw = make([]byte, len(idRaw)+len(propsRaw)+1)
+		raw[0] = byte(len(idRaw))
+		copy(raw[1:], idRaw)
+		copy(raw[len(idRaw)+1:], propsRaw)
+	}
+	return string(raw)
 }
 
 // JSON is the json representation of the object. This might not be exactly the same as the original.

geojson/object.go

@@ -3,6 +3,7 @@ package geojson
 import (
 	"bytes"
 	"encoding/binary"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"math"
@@ -336,13 +337,18 @@ func nearbyObjectShared(g Object, x, y float64, meters float64) bool {
 	return g.Intersects(circlePoly)
 }
 
-func mustMarshalString(s string) bool {
+func jsonMarshalString(s string) []byte {
 	for i := 0; i < len(s); i++ {
-		if s[i] < ' ' || s[i] > 0x7f || s[i] == '"' {
+		if s[i] < ' ' || s[i] == '\\' || s[i] == '"' || s[i] > 126 {
-			return true
+			b, _ := json.Marshal(s)
+			return b
 		}
 	}
-	return false
+	b := make([]byte, len(s)+2)
+	b[0] = '"'
+	copy(b[1:], s)
+	b[len(b)-1] = '"'
+	return b
 }
 
 func stripWhitespace(s string) string {

geojson/string.go

@@ -1,7 +1,5 @@
 package geojson
 
-import "encoding/json"
-
 // String is a not a geojson object, but just a string
 type String string
 
@@ -80,7 +78,7 @@ func (s String) Weight() int {
 
 // MarshalJSON allows the object to be encoded in json.Marshal calls.
 func (s String) MarshalJSON() ([]byte, error) {
-	return json.Marshal(string(s))
+	return jsonMarshalString(string(s)), nil
 }
 
 // Geohash converts the object to a geohash value.

vendor/github.com/tidwall/cast/LICENSE

@@ -1,20 +0,0 @@
-The MIT License (MIT)
-
-Copyright (c) 2016 Josh Baker
-
-Permission is hereby granted, free of charge, to any person obtaining a copy of
-this software and associated documentation files (the "Software"), to deal in
-the Software without restriction, including without limitation the rights to
-use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
-the Software, and to permit persons to whom the Software is furnished to do so,
-subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
-FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
-COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
-IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

vendor/github.com/tidwall/cast/README.md

@@ -1,42 +0,0 @@
-CAST
-====
-![Travis CI Build Status](https://api.travis-ci.org/tidwall/cast.svg?branch=master)
-[![GoDoc](https://godoc.org/github.com/tidwall/cast?status.svg)](https://godoc.org/github.com/tidwall/cast)
-
-Quickly convert string <-> []byte without memory reallocations and create mutable string or immutable []byte.
-
-This package is a **danger zone** and should not be entered without understanding the ground rules.
-
-1. Converting a string -> []byte will result in an immutable byte slice. Editing will cause a panic.
-2. Converting a []byte -> string will result in a mutable string. Editing the originial bytes will change the string too.
-
-
-Create immutable []byte:
-
-```go
-var s = "Hello Planet"
-b := cast.ToBytes(s)
-fmt.Printf("%s\n", "J"+string(b[1:]))
-
-// Output:
-// Jello Planet
-```
-
-Create mutable string:
-
-```go
-var b = []byte("Hello Planet")
-s := cast.ToString(b)
-b[0] = 'J'
-fmt.Printf("%s\n", s)
-
-// Output:
-// Jello Planet
-```
-
-## Contact
-Josh Baker [@tidwall](http://twitter.com/tidwall)
-
-## License
-
-CAST source code is available under the MIT [License](/LICENSE).

vendor/github.com/tidwall/cast/cast.go

@@ -1,27 +0,0 @@
-package cast
-
-import (
-	"reflect"
-	"unsafe"
-)
-
-// ToString will quickly return a string representation of a []byte without
-// allocating new memory or having to copy data. Converting a []byte -> string
-// will result in a mutable string. Editing the originial bytes will change the
-// string too.
-func ToString(b []byte) string {
-	// A string is 16 bytes and []byte is 24 bytes, therefore []byte can be
-	// directly assigned to a string.
-	return *(*string)(unsafe.Pointer(&b))
-}
-
-// ToBytes will quickly return a []byte representation of a string without
-// allocating new memory or having to copy data. Converting a string -> []byte
-// will result in an immutable byte slice. Editing will cause a panic.
-func ToBytes(s string) []byte {
-	// A string is smaller than a []byte so we need to create a new byte
-	// header and fill in the fields.
-	sh := *(*reflect.StringHeader)(unsafe.Pointer(&s))
-	bh := reflect.SliceHeader{sh.Data, sh.Len, sh.Len}
-	return *(*[]byte)(unsafe.Pointer(&bh))
-}

vendor/github.com/tidwall/cast/cast_test.go

@@ -1,107 +0,0 @@
-package cast
-
-import (
-	"crypto/rand"
-	"fmt"
-	"runtime"
-	"testing"
-	"time"
-)
-
-// getHeap returns the size of the heap
-func getHeap(seed int) int {
-	runtime.GC()
-	var ms runtime.MemStats
-	runtime.ReadMemStats(&ms)
-	return int(ms.HeapAlloc) - seed
-}
-
-// TestCast will create a 100KB []byte and assign the first 50KB
-// to a string using ToString() and the copy the second 50KB to a
-// string using string(). Both will be assigned to a map[string]bool
-// and should result in around 150KB of heap. 100KB for the original
-// []byte and 50 for the string() copy.
-// We will test read the heap alloc to see if it's around 150KB.
-// Then we'll nil the map.
-// Then wait up to 10 seconds for the memory to get near zero.
-func TestToString(t *testing.T) {
-	const sz = 1024 * 500
-	var m [2]string
-	ch := make(chan bool)
-	var start time.Time
-	seed := getHeap(0)
-	go func() {
-		b := make([]byte, sz)
-		rand.Read(b)
-		m[0] = ToString(b[len(b)/2:])
-		m[1] = string(b[:len(b)/2])
-		ch <- true
-	}()
-	<-ch
-	if 1.0-float64(getHeap(seed))/(sz+sz/2.0) > 0.05 {
-		t.Fatal("failed")
-	}
-	m[0], m[1] = "", ""
-	start = time.Now()
-	for {
-		if time.Now().Sub(start) > time.Second*10 {
-			t.Fatal("failed")
-		}
-		per := 1.0 - float64(getHeap(seed))/(sz+sz/2.0)
-		if per > 0.95 {
-			break
-		}
-	}
-}
-
-// TestToBytes is the same as TestToString, but the other way around.
-func TestToBytes(t *testing.T) {
-	const sz = 1024 * 500
-	var m [2][]byte
-	ch := make(chan bool)
-	var start time.Time
-	seed := getHeap(0)
-	go func() {
-		b := make([]byte, sz)
-		rand.Read(b)
-		s := string(b)
-		b = nil
-		m[0] = ToBytes(s[len(s)/2:])
-		m[1] = []byte(s[:len(s)/2])
-		ch <- true
-	}()
-	<-ch
-	if 1.0-float64(getHeap(seed))/(sz+sz/2.0) > 0.05 {
-		t.Fatal("failed")
-	}
-	m[0], m[1] = nil, nil
-	start = time.Now()
-	for {
-		if time.Now().Sub(start) > time.Second*10 {
-			t.Fatal("failed")
-		}
-		per := 1.0 - float64(getHeap(seed))/(sz+sz/2.0)
-		if per > 0.95 {
-			break
-		}
-	}
-}
-
-func ExampleToBytes() {
-	var s = "Hello Planet"
-	b := ToBytes(s)
-	fmt.Printf("%s\n", "J"+string(b[1:]))
-
-	// Output:
-	// Jello Planet
-}
-
-func ExampleToString() {
-	var b = []byte("Hello Planet")
-	s := ToString(b)
-	b[0] = 'J'
-	fmt.Printf("%s\n", s)
-
-	// Output:
-	// Jello Planet
-}