mirror of https://github.com/tidwall/tile38.git
optimized idprops field for #71
This commit is contained in:
Josh Baker
parent
bdcbc9c7cc
commit
1ac6ad9ebd
|
|
@ -1,10 +1,6 @@
|
|||
package controller
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
|
||||
"github.com/tidwall/cast"
|
||||
)
|
||||
import "encoding/json"
|
||||
|
||||
func jsonString(s string) string {
|
||||
for i := 0; i < len(s); i++ {
|
||||
|
|
@ -15,7 +11,7 @@ func jsonString(s string) string {
|
|||
}
|
||||
b := make([]byte, len(s)+2)
|
||||
b[0] = '"'
|
||||
copy(b[1:], cast.ToBytes(s))
|
||||
copy(b[1:], s)
|
||||
b[len(b)-1] = '"'
|
||||
return cast.ToString(b)
|
||||
return string(b)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ package geojson
|
|||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
|
||||
"github.com/tidwall/gjson"
|
||||
"github.com/tidwall/tile38/geojson/geohash"
|
||||
|
|
@ -46,12 +47,7 @@ func fillFeatureMap(json string) (Feature, []byte, error) {
|
|||
}
|
||||
id := gjson.Get(json, "id")
|
||||
if id.Exists() || propsExists {
|
||||
idRaw := stripWhitespace(id.Raw)
|
||||
propsRaw := stripWhitespace(props.Raw)
|
||||
raw := make([]byte, len(idRaw)+len(propsRaw)+1)
|
||||
copy(raw, idRaw)
|
||||
copy(raw[len(idRaw)+1:], propsRaw)
|
||||
g.idprops = string(raw)
|
||||
g.idprops = makeCompositeRaw(id.Raw, props.Raw)
|
||||
}
|
||||
return g, nil, err
|
||||
}
|
||||
|
|
@ -97,12 +93,39 @@ func (g Feature) MarshalJSON() ([]byte, error) {
|
|||
}
|
||||
|
||||
func (g Feature) getRaw() (id, props string) {
|
||||
for i := 0; i < len(g.idprops); i++ {
|
||||
if g.idprops[i] == 0 {
|
||||
return g.idprops[:i], g.idprops[i+1:]
|
||||
}
|
||||
}
|
||||
if len(g.idprops) == 0 {
|
||||
return "", ""
|
||||
}
|
||||
switch g.idprops[0] {
|
||||
default:
|
||||
lnp := int(g.idprops[0]) + 1
|
||||
return g.idprops[1:lnp], g.idprops[lnp:]
|
||||
case 255:
|
||||
lnp := int(binary.LittleEndian.Uint64([]byte(g.idprops[1:9]))) + 9
|
||||
return g.idprops[9:lnp], g.idprops[lnp:]
|
||||
}
|
||||
}
|
||||
|
||||
func makeCompositeRaw(idRaw, propsRaw string) string {
|
||||
idRaw = stripWhitespace(idRaw)
|
||||
propsRaw = stripWhitespace(propsRaw)
|
||||
if len(idRaw) == 0 && len(propsRaw) == 0 {
|
||||
return ""
|
||||
}
|
||||
var raw []byte
|
||||
if len(idRaw) > 0xFF-1 {
|
||||
raw = make([]byte, len(idRaw)+len(propsRaw)+9)
|
||||
raw[0] = 0xFF
|
||||
binary.LittleEndian.PutUint64(raw[1:9], uint64(len(idRaw)))
|
||||
copy(raw[9:], idRaw)
|
||||
copy(raw[len(idRaw)+9:], propsRaw)
|
||||
} else {
|
||||
raw = make([]byte, len(idRaw)+len(propsRaw)+1)
|
||||
raw[0] = byte(len(idRaw))
|
||||
copy(raw[1:], idRaw)
|
||||
copy(raw[len(idRaw)+1:], propsRaw)
|
||||
}
|
||||
return string(raw)
|
||||
}
|
||||
|
||||
// JSON is the json representation of the object. This might not be exactly the same as the original.
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ package geojson
|
|||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
|
|
@ -336,13 +337,18 @@ func nearbyObjectShared(g Object, x, y float64, meters float64) bool {
|
|||
return g.Intersects(circlePoly)
|
||||
}
|
||||
|
||||
func mustMarshalString(s string) bool {
|
||||
func jsonMarshalString(s string) []byte {
|
||||
for i := 0; i < len(s); i++ {
|
||||
if s[i] < ' ' || s[i] > 0x7f || s[i] == '"' {
|
||||
return true
|
||||
if s[i] < ' ' || s[i] == '\\' || s[i] == '"' || s[i] > 126 {
|
||||
b, _ := json.Marshal(s)
|
||||
return b
|
||||
}
|
||||
}
|
||||
return false
|
||||
b := make([]byte, len(s)+2)
|
||||
b[0] = '"'
|
||||
copy(b[1:], s)
|
||||
b[len(b)-1] = '"'
|
||||
return b
|
||||
}
|
||||
|
||||
func stripWhitespace(s string) string {
|
||||
|
|
|
|||
|
|
@ -1,7 +1,5 @@
|
|||
package geojson
|
||||
|
||||
import "encoding/json"
|
||||
|
||||
// String is a not a geojson object, but just a string
|
||||
type String string
|
||||
|
||||
|
|
@ -80,7 +78,7 @@ func (s String) Weight() int {
|
|||
|
||||
// MarshalJSON allows the object to be encoded in json.Marshal calls.
|
||||
func (s String) MarshalJSON() ([]byte, error) {
|
||||
return json.Marshal(string(s))
|
||||
return jsonMarshalString(string(s)), nil
|
||||
}
|
||||
|
||||
// Geohash converts the object to a geohash value.
|
||||
|
|
|
|||
|
|
@ -1,20 +0,0 @@
|
|||
The MIT License (MIT)
|
||||
|
||||
Copyright (c) 2016 Josh Baker
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of
|
||||
this software and associated documentation files (the "Software"), to deal in
|
||||
the Software without restriction, including without limitation the rights to
|
||||
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
|
||||
the Software, and to permit persons to whom the Software is furnished to do so,
|
||||
subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
||||
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
||||
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
|
||||
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
|
||||
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
|
|
@ -1,42 +0,0 @@
|
|||
CAST
|
||||
====
|
||||
|
||||
[](https://godoc.org/github.com/tidwall/cast)
|
||||
|
||||
Quickly convert string <-> []byte without memory reallocations and create mutable string or immutable []byte.
|
||||
|
||||
This package is a **danger zone** and should not be entered without understanding the ground rules.
|
||||
|
||||
1. Converting a string -> []byte will result in an immutable byte slice. Editing will cause a panic.
|
||||
2. Converting a []byte -> string will result in a mutable string. Editing the originial bytes will change the string too.
|
||||
|
||||
|
||||
Create immutable []byte:
|
||||
|
||||
```go
|
||||
var s = "Hello Planet"
|
||||
b := cast.ToBytes(s)
|
||||
fmt.Printf("%s\n", "J"+string(b[1:]))
|
||||
|
||||
// Output:
|
||||
// Jello Planet
|
||||
```
|
||||
|
||||
Create mutable string:
|
||||
|
||||
```go
|
||||
var b = []byte("Hello Planet")
|
||||
s := cast.ToString(b)
|
||||
b[0] = 'J'
|
||||
fmt.Printf("%s\n", s)
|
||||
|
||||
// Output:
|
||||
// Jello Planet
|
||||
```
|
||||
|
||||
## Contact
|
||||
Josh Baker [@tidwall](http://twitter.com/tidwall)
|
||||
|
||||
## License
|
||||
|
||||
CAST source code is available under the MIT [License](/LICENSE).
|
||||
|
|
@ -1,27 +0,0 @@
|
|||
package cast
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
// ToString will quickly return a string representation of a []byte without
|
||||
// allocating new memory or having to copy data. Converting a []byte -> string
|
||||
// will result in a mutable string. Editing the originial bytes will change the
|
||||
// string too.
|
||||
func ToString(b []byte) string {
|
||||
// A string is 16 bytes and []byte is 24 bytes, therefore []byte can be
|
||||
// directly assigned to a string.
|
||||
return *(*string)(unsafe.Pointer(&b))
|
||||
}
|
||||
|
||||
// ToBytes will quickly return a []byte representation of a string without
|
||||
// allocating new memory or having to copy data. Converting a string -> []byte
|
||||
// will result in an immutable byte slice. Editing will cause a panic.
|
||||
func ToBytes(s string) []byte {
|
||||
// A string is smaller than a []byte so we need to create a new byte
|
||||
// header and fill in the fields.
|
||||
sh := *(*reflect.StringHeader)(unsafe.Pointer(&s))
|
||||
bh := reflect.SliceHeader{sh.Data, sh.Len, sh.Len}
|
||||
return *(*[]byte)(unsafe.Pointer(&bh))
|
||||
}
|
||||
|
|
@ -1,107 +0,0 @@
|
|||
package cast
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"fmt"
|
||||
"runtime"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// getHeap returns the size of the heap
|
||||
func getHeap(seed int) int {
|
||||
runtime.GC()
|
||||
var ms runtime.MemStats
|
||||
runtime.ReadMemStats(&ms)
|
||||
return int(ms.HeapAlloc) - seed
|
||||
}
|
||||
|
||||
// TestCast will create a 100KB []byte and assign the first 50KB
|
||||
// to a string using ToString() and the copy the second 50KB to a
|
||||
// string using string(). Both will be assigned to a map[string]bool
|
||||
// and should result in around 150KB of heap. 100KB for the original
|
||||
// []byte and 50 for the string() copy.
|
||||
// We will test read the heap alloc to see if it's around 150KB.
|
||||
// Then we'll nil the map.
|
||||
// Then wait up to 10 seconds for the memory to get near zero.
|
||||
func TestToString(t *testing.T) {
|
||||
const sz = 1024 * 500
|
||||
var m [2]string
|
||||
ch := make(chan bool)
|
||||
var start time.Time
|
||||
seed := getHeap(0)
|
||||
go func() {
|
||||
b := make([]byte, sz)
|
||||
rand.Read(b)
|
||||
m[0] = ToString(b[len(b)/2:])
|
||||
m[1] = string(b[:len(b)/2])
|
||||
ch <- true
|
||||
}()
|
||||
<-ch
|
||||
if 1.0-float64(getHeap(seed))/(sz+sz/2.0) > 0.05 {
|
||||
t.Fatal("failed")
|
||||
}
|
||||
m[0], m[1] = "", ""
|
||||
start = time.Now()
|
||||
for {
|
||||
if time.Now().Sub(start) > time.Second*10 {
|
||||
t.Fatal("failed")
|
||||
}
|
||||
per := 1.0 - float64(getHeap(seed))/(sz+sz/2.0)
|
||||
if per > 0.95 {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestToBytes is the same as TestToString, but the other way around.
|
||||
func TestToBytes(t *testing.T) {
|
||||
const sz = 1024 * 500
|
||||
var m [2][]byte
|
||||
ch := make(chan bool)
|
||||
var start time.Time
|
||||
seed := getHeap(0)
|
||||
go func() {
|
||||
b := make([]byte, sz)
|
||||
rand.Read(b)
|
||||
s := string(b)
|
||||
b = nil
|
||||
m[0] = ToBytes(s[len(s)/2:])
|
||||
m[1] = []byte(s[:len(s)/2])
|
||||
ch <- true
|
||||
}()
|
||||
<-ch
|
||||
if 1.0-float64(getHeap(seed))/(sz+sz/2.0) > 0.05 {
|
||||
t.Fatal("failed")
|
||||
}
|
||||
m[0], m[1] = nil, nil
|
||||
start = time.Now()
|
||||
for {
|
||||
if time.Now().Sub(start) > time.Second*10 {
|
||||
t.Fatal("failed")
|
||||
}
|
||||
per := 1.0 - float64(getHeap(seed))/(sz+sz/2.0)
|
||||
if per > 0.95 {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func ExampleToBytes() {
|
||||
var s = "Hello Planet"
|
||||
b := ToBytes(s)
|
||||
fmt.Printf("%s\n", "J"+string(b[1:]))
|
||||
|
||||
// Output:
|
||||
// Jello Planet
|
||||
}
|
||||
|
||||
func ExampleToString() {
|
||||
var b = []byte("Hello Planet")
|
||||
s := ToString(b)
|
||||
b[0] = 'J'
|
||||
fmt.Printf("%s\n", s)
|
||||
|
||||
// Output:
|
||||
// Jello Planet
|
||||
}
|
||||
Loading…
Reference in New Issue