From 30d31d0926f99f83eb48fb8d20fea005af7d9cc8 Mon Sep 17 00:00:00 2001 From: tidwall Date: Sun, 17 Feb 2019 13:10:02 -0700 Subject: [PATCH] Packed fields option --- Gopkg.lock | 19 +- Gopkg.toml | 5 +- internal/collection/btree/btree_test.go | 24 +- internal/collection/item/item.go | 166 ++++------ internal/collection/item/item_test.go | 74 +++-- internal/collection/item/packed.go | 284 ++++++++++++++++++ internal/collection/item/packed_test.go | 117 ++++++++ internal/collection/item/unpacked.go | 95 ++++++ internal/collection/rtree/rtree_test.go | 6 +- vendor/github.com/h2so5/half/.gitignore | 22 ++ vendor/github.com/h2so5/half/LICENSE.txt | 121 ++++++++ vendor/github.com/h2so5/half/README.md | 6 + vendor/github.com/h2so5/half/float16.go | 59 ++++ vendor/github.com/h2so5/half/float16_test.go | 55 ++++ vendor/github.com/tidwall/gjson/README.md | 141 +++++++-- vendor/github.com/tidwall/gjson/gjson.go | 218 +++++++++++++- vendor/github.com/tidwall/gjson/gjson_gae.go | 10 +- vendor/github.com/tidwall/gjson/gjson_ngae.go | 78 ++--- vendor/github.com/tidwall/gjson/gjson_test.go | 83 ++++- 19 files changed, 1348 insertions(+), 235 deletions(-) create mode 100644 internal/collection/item/packed.go create mode 100644 internal/collection/item/packed_test.go create mode 100644 internal/collection/item/unpacked.go create mode 100644 vendor/github.com/h2so5/half/.gitignore create mode 100644 vendor/github.com/h2so5/half/LICENSE.txt create mode 100644 vendor/github.com/h2so5/half/README.md create mode 100644 vendor/github.com/h2so5/half/float16.go create mode 100644 vendor/github.com/h2so5/half/float16_test.go diff --git a/Gopkg.lock b/Gopkg.lock index eb0ca46c..35f8807f 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -123,6 +123,14 @@ pruneopts = "" revision = "e8fc0692a7e26a05b06517348ed466349062eb47" +[[projects]] + branch = "master" + digest = "1:de10194e2fb3787b2efdea394a7f710f49bfdcaae78f715b38a9057c602a4998" + name = "github.com/h2so5/half" + packages = ["."] + pruneopts = "" + revision = "c705bde19bd0aa32d718d167c961fe681ee91473" + [[projects]] digest = "1:6f49eae0c1e5dab1dafafee34b207aeb7a42303105960944828c2079b92fc88e" name = "github.com/jmespath/go-jmespath" @@ -224,11 +232,11 @@ [[projects]] branch = "master" - digest = "1:2fef6390e8d9118debd4937a699afad9e1629f2d5d3c965a58fc33afe04f9b46" + digest = "1:4d2ec831fbaaf74fd75d2d9fe107e605c92489ec6cef6d36e1f23b678e9f2bd4" name = "github.com/tidwall/buntdb" packages = ["."] pruneopts = "" - revision = "b67b1b8c1658cb01502801c14e33c61e6c4cbb95" + revision = "6249481c29c2cd96f53b691b74ac1893f72774c2" [[projects]] digest = "1:91acf4d86b348c1f1832336836035373b047ffcb16a0fde066bd531bbe3452b2" @@ -254,12 +262,12 @@ version = "v1.1.1" [[projects]] - digest = "1:3ddca2bd5496c6922a2a9e636530e178a43c2a534ea6634211acdc7d10222794" + digest = "1:eade4ea6782f5eed4a6b3138a648f9a332900650804fd206e5daaf99cc5613ea" name = "github.com/tidwall/gjson" packages = ["."] pruneopts = "" - revision = "1e3f6aeaa5bad08d777ea7807b279a07885dd8b2" - version = "v1.1.3" + revision = "eee0b6226f0d1db2675a176fdfaa8419bcad4ca8" + version = "v1.2.1" [[projects]] branch = "master" @@ -466,6 +474,7 @@ "github.com/eclipse/paho.mqtt.golang", "github.com/golang/protobuf/proto", "github.com/gomodule/redigo/redis", + "github.com/h2so5/half", "github.com/mmcloughlin/geohash", "github.com/nats-io/go-nats", "github.com/peterh/liner", diff --git a/Gopkg.toml b/Gopkg.toml index 137abfd2..f5eaa215 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -23,7 +23,8 @@ required = [ "github.com/tidwall/lotsa", "github.com/mmcloughlin/geohash", - "github.com/tidwall/evio" + "github.com/tidwall/evio", + "github.com/h2so5/half" ] [[constraint]] @@ -72,7 +73,7 @@ required = [ [[constraint]] name = "github.com/tidwall/gjson" - version = "1.0.1" + version = "1.2.1" [[constraint]] branch = "master" diff --git a/internal/collection/btree/btree_test.go b/internal/collection/btree/btree_test.go index 5f1a889d..c48f639a 100644 --- a/internal/collection/btree/btree_test.go +++ b/internal/collection/btree/btree_test.go @@ -111,7 +111,7 @@ func TestDescend(t *testing.T) { var keys []string for i := 0; i < 1000; i += 10 { keys = append(keys, fmt.Sprintf("%03d", i)) - tr.Set(item.New(keys[len(keys)-1], nil)) + tr.Set(item.New(keys[len(keys)-1], nil, false)) } var exp []string tr.Reverse(func(item *item.Item) bool { @@ -162,7 +162,7 @@ func TestAscend(t *testing.T) { var keys []string for i := 0; i < 1000; i += 10 { keys = append(keys, fmt.Sprintf("%03d", i)) - tr.Set(item.New(keys[len(keys)-1], nil)) + tr.Set(item.New(keys[len(keys)-1], nil, false)) } exp := keys for i := -1; i < 1000; i++ { @@ -205,7 +205,7 @@ func TestBTree(t *testing.T) { // insert all items for _, key := range keys { - value, replaced := tr.Set(item.New(key, testString(key))) + value, replaced := tr.Set(item.New(key, testString(key), false)) if replaced { t.Fatal("expected false") } @@ -362,7 +362,7 @@ func TestBTree(t *testing.T) { // replace second half for _, key := range keys[len(keys)/2:] { - value, replaced := tr.Set(item.New(key, testString(key))) + value, replaced := tr.Set(item.New(key, testString(key), false)) if !replaced { t.Fatal("expected true") } @@ -420,7 +420,7 @@ func BenchmarkTidwallSequentialSet(b *testing.B) { sort.Strings(keys) b.ResetTimer() for i := 0; i < b.N; i++ { - tr.Set(item.New(keys[i], nil)) + tr.Set(item.New(keys[i], nil, false)) } } @@ -429,7 +429,7 @@ func BenchmarkTidwallSequentialGet(b *testing.B) { keys := randKeys(b.N) sort.Strings(keys) for i := 0; i < b.N; i++ { - tr.Set(item.New(keys[i], nil)) + tr.Set(item.New(keys[i], nil, false)) } b.ResetTimer() for i := 0; i < b.N; i++ { @@ -442,7 +442,7 @@ func BenchmarkTidwallRandomSet(b *testing.B) { keys := randKeys(b.N) b.ResetTimer() for i := 0; i < b.N; i++ { - tr.Set(item.New(keys[i], nil)) + tr.Set(item.New(keys[i], nil, false)) } } @@ -450,7 +450,7 @@ func BenchmarkTidwallRandomGet(b *testing.B) { var tr BTree keys := randKeys(b.N) for i := 0; i < b.N; i++ { - tr.Set(item.New(keys[i], nil)) + tr.Set(item.New(keys[i], nil, false)) } b.ResetTimer() for i := 0; i < b.N; i++ { @@ -528,11 +528,11 @@ func BenchmarkTidwallRandomGet(b *testing.B) { func TestBTreeOne(t *testing.T) { var tr BTree - tr.Set(item.New("1", testString("1"))) + tr.Set(item.New("1", testString("1"), false)) tr.Delete("1") - tr.Set(item.New("1", testString("1"))) + tr.Set(item.New("1", testString("1"), false)) tr.Delete("1") - tr.Set(item.New("1", testString("1"))) + tr.Set(item.New("1", testString("1"), false)) tr.Delete("1") } @@ -541,7 +541,7 @@ func TestBTree256(t *testing.T) { var n int for j := 0; j < 2; j++ { for _, i := range rand.Perm(256) { - tr.Set(item.New(fmt.Sprintf("%d", i), testString(fmt.Sprintf("%d", i)))) + tr.Set(item.New(fmt.Sprintf("%d", i), testString(fmt.Sprintf("%d", i)), false)) n++ if tr.Len() != n { t.Fatalf("expected 256, got %d", n) diff --git a/internal/collection/item/item.go b/internal/collection/item/item.go index 637b244e..d040c8fa 100644 --- a/internal/collection/item/item.go +++ b/internal/collection/item/item.go @@ -57,36 +57,28 @@ func (item *Item) setIsPacked(isPacked bool) { } } -func (item *Item) fieldsLen() int { +func (item *Item) fieldsDataSize() int { return int(item.head[0] & 0x3FFFFFFF) } -func (item *Item) setFieldsLen(len int) { +func (item *Item) setFieldsDataSize(len int) { item.head[0] = item.head[0]>>30<<30 | uint32(len) } -func (item *Item) idLen() int { +func (item *Item) idDataSize() int { return int(item.head[1]) } -func (item *Item) setIDLen(len int) { +func (item *Item) setIDDataSize(len int) { item.head[1] = uint32(len) } // ID returns the items ID as a string func (item *Item) ID() string { return *(*string)((unsafe.Pointer)(&reflect.StringHeader{ - Data: uintptr(unsafe.Pointer(item.data)) + uintptr(item.fieldsLen()), - Len: item.idLen(), - })) -} - -// Fields returns the field values -func (item *Item) fields() []float64 { - return *(*[]float64)((unsafe.Pointer)(&reflect.SliceHeader{ - Data: uintptr(unsafe.Pointer(item.data)), - Len: item.fieldsLen() / 8, - Cap: item.fieldsLen() / 8, + Data: uintptr(unsafe.Pointer(item.data)) + + uintptr(item.fieldsDataSize()), + Len: item.idDataSize(), })) } @@ -112,7 +104,7 @@ func New(id string, obj geojson.Object, packed bool) *Item { item = (*Item)(unsafe.Pointer(oitem)) } item.setIsPacked(packed) - item.setIDLen(len(id)) + item.setIDDataSize(len(id)) item.data = unsafe.Pointer((*reflect.SliceHeader)(unsafe.Pointer(&id)).Data) return item } @@ -126,7 +118,7 @@ func (item *Item) WeightAndPoints() (weight, points int) { } else if item.Obj() != nil { weight = len(item.Obj().String()) } - weight += item.fieldsLen() + item.idLen() + weight += item.fieldsDataSize() + item.idDataSize() return weight, points } @@ -144,21 +136,56 @@ func (item *Item) Less(other btree.Item, ctx interface{}) bool { return item.ID() < other.(*Item).ID() } +// fieldBytes returns the raw fields data section +func (item *Item) fieldsBytes() []byte { + return *(*[]byte)((unsafe.Pointer)(&reflect.SliceHeader{ + Data: uintptr(unsafe.Pointer(item.data)), + Len: item.fieldsDataSize(), + Cap: item.fieldsDataSize(), + })) +} + +// Packed returns true when the item's fields are packed +func (item *Item) Packed() bool { + return item == nil || item.isPacked() +} + // CopyOverFields overwriting previous fields. Accepts an *Item or []float64 func (item *Item) CopyOverFields(from interface{}) { + if item == nil { + return + } var values []float64 + var fieldBytes []byte + var directCopy bool switch from := from.(type) { case *Item: - values = from.fields() + if item.Packed() == from.Packed() { + // direct copy the bytes + fieldBytes = from.fieldsBytes() + directCopy = true + } else { + // get the values through iteration + item.ForEachField(-1, func(value float64) bool { + values = append(values, value) + return true + }) + } case []float64: values = from } - fieldBytes := floatsToBytes(values) - oldData := item.dataBytes() - newData := make([]byte, len(fieldBytes)+item.idLen()) + if !directCopy { + if item.Packed() { + fieldBytes = item.packedGenerateFieldBytes(values) + } else { + fieldBytes = item.unpackedGenerateFieldBytes(values) + } + } + id := item.ID() + newData := make([]byte, len(fieldBytes)+len(id)) copy(newData, fieldBytes) - copy(newData[len(fieldBytes):], oldData[item.fieldsLen():]) - item.setFieldsLen(len(fieldBytes)) + copy(newData[len(fieldBytes):], id) + item.setFieldsDataSize(len(fieldBytes)) if len(newData) > 0 { item.data = unsafe.Pointer(&newData[0]) } else { @@ -166,54 +193,15 @@ func (item *Item) CopyOverFields(from interface{}) { } } -func getFieldAt(data unsafe.Pointer, index int) float64 { - return *(*float64)(unsafe.Pointer(uintptr(data) + uintptr(index*8))) -} - -func setFieldAt(data unsafe.Pointer, index int, value float64) { - *(*float64)(unsafe.Pointer(uintptr(data) + uintptr(index*8))) = value -} - // SetField set a field value at specified index. func (item *Item) SetField(index int, value float64) (updated bool) { - numFields := item.fieldsLen() / 8 - if index < numFields { - // field exists - if getFieldAt(item.data, index) == value { - return false - } - } else { - // make room for new field - oldBytes := item.dataBytes() - newData := make([]byte, (index+1)*8+item.idLen()) - // copy the existing fields - copy(newData, oldBytes[:item.fieldsLen()]) - // copy the id - copy(newData[(index+1)*8:], oldBytes[item.fieldsLen():]) - // update the fields length - item.setFieldsLen((index + 1) * 8) - // update the raw data - item.data = unsafe.Pointer(&newData[0]) + if item == nil { + return false } - // set the new field - setFieldAt(item.data, index, value) - return true -} - -func (item *Item) dataBytes() []byte { - return *(*[]byte)((unsafe.Pointer)(&reflect.SliceHeader{ - Data: uintptr(unsafe.Pointer(item.data)), - Len: item.fieldsLen() + item.idLen(), - Cap: item.fieldsLen() + item.idLen(), - })) -} - -func floatsToBytes(f []float64) []byte { - return *(*[]byte)((unsafe.Pointer)(&reflect.SliceHeader{ - Data: ((*reflect.SliceHeader)(unsafe.Pointer(&f))).Data, - Len: len(f) * 8, - Cap: len(f) * 8, - })) + if item.Packed() { + return item.packedSetField(index, value) + } + return item.unpackedSetField(index, value) } // ForEachField iterates over each field. The count param is the number of @@ -222,27 +210,11 @@ func (item *Item) ForEachField(count int, iter func(value float64) bool) { if item == nil { return } - fields := item.fields() - var n int - if count < 0 { - n = len(fields) + if item.Packed() { + item.packedForEachField(count, iter) } else { - n = count + item.unpackedForEachField(count, iter) } - for i := 0; i < n; i++ { - var field float64 - if i < len(fields) { - field = fields[i] - } - if !iter(field) { - return - } - } -} - -// Packed returns true when the item's fields are packed -func (item *Item) Packed() bool { - return item == nil || item.isPacked() } // GetField returns the value for a field at index. @@ -254,26 +226,12 @@ func (item *Item) GetField(index int) float64 { return 0 } if item.Packed() { - var fvalue float64 - var idx int - item.ForEachField(-1, func(value float64) bool { - if idx == index { - fvalue = value - return false - } - idx++ - return true - }) - return fvalue + return item.packedGetField(index) } - numFields := item.fieldsLen() / 8 - if index < numFields { - return getFieldAt(item.data, index) - } - return 0 + return item.unpackedGetField(index) } // HasFields returns true when item has fields func (item *Item) HasFields() bool { - return item != nil && item.fieldsLen() > 0 + return item != nil && item.fieldsDataSize() > 0 } diff --git a/internal/collection/item/item_test.go b/internal/collection/item/item_test.go index 9fc83d31..a8f4eb60 100644 --- a/internal/collection/item/item_test.go +++ b/internal/collection/item/item_test.go @@ -10,23 +10,17 @@ import ( "github.com/tidwall/geojson/geometry" ) -func init() { - seed := time.Now().UnixNano() - println(seed) - rand.Seed(seed) -} - func testRandItemHead(t *testing.T, idx int, item *Item) { t.Helper() if idx == 0 { if item.isPoint() { t.Fatalf("expected false") } - if item.fieldsLen() != 0 { - t.Fatalf("expected '%v', got '%v'", 0, item.fieldsLen()) + if item.fieldsDataSize() != 0 { + t.Fatalf("expected '%v', got '%v'", 0, item.fieldsDataSize()) } - if item.idLen() != 0 { - t.Fatalf("expected '%v', got '%v'", 0, item.idLen()) + if item.idDataSize() != 0 { + t.Fatalf("expected '%v', got '%v'", 0, item.idDataSize()) } } isPoint := rand.Int()%2 == 0 @@ -34,17 +28,17 @@ func testRandItemHead(t *testing.T, idx int, item *Item) { idLen := int(rand.Uint32()) item.setIsPoint(isPoint) - item.setFieldsLen(fieldsLen) - item.setIDLen(idLen) + item.setFieldsDataSize(fieldsLen) + item.setIDDataSize(idLen) if item.isPoint() != isPoint { t.Fatalf("isPoint: expected '%v', got '%v'", isPoint, item.isPoint()) } - if item.fieldsLen() != fieldsLen { - t.Fatalf("fieldsLen: expected '%v', got '%v'", fieldsLen, item.fieldsLen()) + if item.fieldsDataSize() != fieldsLen { + t.Fatalf("fieldsLen: expected '%v', got '%v'", fieldsLen, item.fieldsDataSize()) } - if item.idLen() != idLen { - t.Fatalf("idLen: expected '%v', got '%v'", idLen, item.idLen()) + if item.idDataSize() != idLen { + t.Fatalf("idLen: expected '%v', got '%v'", idLen, item.idDataSize()) } } @@ -62,7 +56,6 @@ func testRandItem(t *testing.T) { keyb := make([]byte, rand.Int()%16) rand.Read(keyb) key := string(keyb) - packed := rand.Int()%2 == 0 values := make([]float64, rand.Int()%64) for i := range values { @@ -111,10 +104,10 @@ func testRandItem(t *testing.T) { } for _, i := range setValues { if item.GetField(i) != values[i] { - t.Fatalf("expected '%v', got '%v'", values[i], item.GetField(i)) + t.Fatalf("expected '%v', got '%v' for index %d", values[i], item.GetField(i), i) } } - fields := item.fields() + fields := itemFields(item) for i := 0; i < len(fields); i++ { for _, j := range setValues { if i == j { @@ -126,8 +119,9 @@ func testRandItem(t *testing.T) { } } weight, points := item.WeightAndPoints() - if weight != len(fields)*8+len(key)+points*16 { - t.Fatalf("expected '%v', got '%v'", len(fields)*8+len(key)+points*16, weight) + if weight != item.fieldsDataSize()+len(key)+points*16 { + t.Fatalf("expected '%v', got '%v'", + item.fieldsDataSize()+len(key)+points*16, weight) } if points != 1 { t.Fatalf("expected '%v', got '%v'", 1, points) @@ -167,8 +161,11 @@ func testRandItem(t *testing.T) { fvalues = append(fvalues, value) return true }) + for len(fvalues) < len(values) { + fvalues = append(fvalues, 0) + } if !floatsEquals(values, fvalues) { - t.Fatalf("expected '%v', got '%v'", 1, len(fvalues)) + t.Fatalf("expected true") } // should not fail, must allow nil receiver @@ -185,25 +182,25 @@ func testRandItem(t *testing.T) { } item.CopyOverFields(values) weight, points := item.WeightAndPoints() - if weight != len(values)*8+len(key)+points*16 { - t.Fatalf("expected '%v', got '%v'", len(values)*8+len(key)+points*16, weight) + if weight != item.fieldsDataSize()+len(key)+points*16 { + t.Fatalf("expected '%v', got '%v'", item.fieldsDataSize()+len(key)+points*16, weight) } if points != 1 { t.Fatalf("expected '%v', got '%v'", 1, points) } - if !floatsEquals(item.fields(), values) { - t.Fatalf("expected '%v', got '%v'", values, item.fields()) + if !floatsEquals(itemFields(item), values) { + t.Fatalf("expected '%v', got '%v'", values, itemFields(item)) } item.CopyOverFields(item) weight, points = item.WeightAndPoints() - if weight != len(values)*8+len(key)+points*16 { - t.Fatalf("expected '%v', got '%v'", len(values)*8+len(key)+points*16, weight) + if weight != item.fieldsDataSize()+len(key)+points*16 { + t.Fatalf("expected '%v', got '%v'", item.fieldsDataSize()+len(key)+points*16, weight) } if points != 1 { t.Fatalf("expected '%v', got '%v'", 1, points) } - if !floatsEquals(item.fields(), values) { - t.Fatalf("expected '%v', got '%v'", values, item.fields()) + if !floatsEquals(itemFields(item), values) { + t.Fatalf("expected '%v', got '%v'", values, itemFields(item)) } if len(values) > 0 && !item.HasFields() { t.Fatal("expected true") @@ -217,8 +214,8 @@ func testRandItem(t *testing.T) { if points != 1 { t.Fatalf("expected '%v', got '%v'", 1, points) } - if len(item.fields()) != 0 { - t.Fatalf("expected '%#v', got '%#v'", 0, len(item.fields())) + if len(itemFields(item)) != 0 { + t.Fatalf("expected '%#v', got '%#v'", 0, len(itemFields(item))) } if item.ID() != key { t.Fatalf("expected '%v', got '%v'", key, item.ID()) @@ -229,7 +226,20 @@ func testRandItem(t *testing.T) { } +func itemFields(item *Item) []float64 { + var values []float64 + item.ForEachField(-1, func(value float64) bool { + values = append(values, value) + return true + }) + return values +} + func TestItem(t *testing.T) { + seed := time.Now().UnixNano() + seed = 1550371581595971000 + println("TestItem seed", seed) + rand.Seed(seed) start := time.Now() for time.Since(start) < time.Second/2 { testRandItem(t) diff --git a/internal/collection/item/packed.go b/internal/collection/item/packed.go new file mode 100644 index 00000000..2c0109bc --- /dev/null +++ b/internal/collection/item/packed.go @@ -0,0 +1,284 @@ +package item + +import ( + "fmt" + "unsafe" + + "github.com/h2so5/half" +) + +// kind bits bytes values min max +// -------------------------------------------------------------------- +// 0 5 1 32 -16 15 +// 1 13 2 16384 -4095 4095 +// 2 21 3 2097152 -1048576 1048575 +// 3 29 4 536870912 -268435456 268435455 +// 4 16 3 -- standard 16-bit floating point -- +// 5 32 5 -- standard 32-bit floating point -- +// 6 64 9 -- standard 64-bit floating point -- + +const maxFieldBytes = 9 + +const ( + maxInt5 = 15 + maxInt13 = 4095 + maxInt21 = 1048575 + maxInt29 = 268435455 +) + +func appendPacked(dst []byte, f64 float64) []byte { + if f64 == 0 { + return append(dst, 0) + } + i64 := int64(f64) + if f64 == float64(i64) { + // whole number + var signed byte + if i64 < 0 { + i64 *= -1 + signed = 16 + } + if i64 <= maxInt5 { + return append(dst, 0<<5|signed| + byte(i64)) + } + if i64 <= maxInt13 { + return append(dst, 1<<5|signed| + byte(i64>>8), byte(i64)) + } + if i64 <= maxInt21 { + return append(dst, 2<<5|signed| + byte(i64>>16), byte(i64>>8), byte(i64)) + } + if i64 <= maxInt29 { + return append(dst, 3<<5|signed| + byte(i64>>24), byte(i64>>16), byte(i64>>8), byte(i64)) + } + // fallthrough + } + f32 := float32(f64) + if f64 == float64(f32) { + f16 := half.NewFloat16(f32) + if f32 == f16.Float32() { + dst = append(dst, 4<<5, 0, 0) + *(*half.Float16)(unsafe.Pointer(&dst[len(dst)-2])) = f16 + return dst + } + dst = append(dst, 5<<5, 0, 0, 0, 0) + *(*float32)(unsafe.Pointer(&dst[len(dst)-4])) = f32 + return dst + } + dst = append(dst, 6<<5, 0, 0, 0, 0, 0, 0, 0, 0) + *(*float64)(unsafe.Pointer(&dst[len(dst)-8])) = f64 + return dst +} + +func skipPacked(data []byte, count int) (out []byte, read int) { + var i int + for i < len(data) { + if read >= count { + return data[i:], read + } + kind := data[i] >> 5 + if kind < 4 { + i += int(kind) + 1 + } else if kind == 4 { + i += 3 + } else if kind == 5 { + i += 5 + } else { + i += 9 + } + read++ + } + return nil, read +} + +func readPacked(data []byte) ([]byte, float64) { + if len(data) == 0 { + return nil, 0 + } + if data[0] == 0 { + return data[1:], 0 + } + kind := data[0] >> 5 + switch kind { + case 0, 1, 2, 3: + // whole number + var value float64 + if kind == 0 { + value = float64( + uint32(data[0] & 0xF), + ) + } else if kind == 1 { + value = float64( + uint32(data[0]&0xF)<<8 | uint32(data[1]), + ) + } else if kind == 2 { + value = float64( + uint32(data[0]&0xF)<<16 | uint32(data[1])<<8 | + uint32(data[2]), + ) + } else { + value = float64( + uint32(data[0]&0xF)<<24 | uint32(data[1])<<16 | + uint32(data[2])<<8 | uint32(data[3]), + ) + } + if data[0]&0x10 != 0 { + value *= -1 + } + return data[kind+1:], value + case 4: + // 16-bit float + return data[3:], + float64((*half.Float16)(unsafe.Pointer(&data[1])).Float32()) + case 5: + // 32-bit float + return data[5:], + float64(*(*float32)(unsafe.Pointer(&data[1]))) + case 6: + // 64-bit float + return data[9:], *(*float64)(unsafe.Pointer(&data[1])) + } + panic("invalid data") +} + +func (item *Item) packedGenerateFieldBytes(values []float64) []byte { + var dst []byte + for i := 0; i < len(values); i++ { + dst = appendPacked(dst, values[i]) + } + return dst +} + +func (item *Item) packedSetField(index int, value float64) (updated bool) { + if false { + func() { + data := item.fieldsBytes() + fmt.Printf("%v >> [%x]", value, data) + defer func() { + data := item.fieldsBytes() + fmt.Printf(" >> [%x]\n", data) + }() + }() + } + ///////////////////////////////////////////////////////////////// + + // original field bytes + headBytes := item.fieldsBytes() + + // quickly skip over head fields. + // returns the start of the field at index, and the number of valid + // fields that were read. + fieldBytes, read := skipPacked(headBytes, index) + + // number of empty/blank bytes that need to be added between the + // head bytes and the new field bytes. + var blankSpace int + + // data a that follows the new field bytes + var tailBytes []byte + + if len(fieldBytes) == 0 { + // field at index was not found. + if value == 0 { + // zero value is the default, so we can assume that the fields was + // not updated. + return false + } + // set the blank space + blankSpace = index - read + fieldBytes = nil + } else { + // field at index was found. + + // truncate the head bytes to reflect only the bytes up to + // the current field. + headBytes = headBytes[:len(headBytes)-len(fieldBytes)] + + // read the current value and get the tail data following the + // current field. + var cvalue float64 + tailBytes, cvalue = readPacked(fieldBytes) + if cvalue == value { + // no change to value + return false + } + + // truncate the field bytes to exactly match current field. + fieldBytes = fieldBytes[:len(fieldBytes)-len(tailBytes)] + } + + // create the new field bytes + { + var buf [maxFieldBytes]byte + newFieldBytes := appendPacked(buf[:0], value) + if len(newFieldBytes) == len(fieldBytes) { + // no change in data size, update in place + copy(fieldBytes, newFieldBytes) + return true + } + // reassign the field bytes + fieldBytes = newFieldBytes + } + + // hang on to the item id + id := item.ID() + + // create a new byte slice + // head+blank+field+tail+id + nbytes := make([]byte, + len(headBytes)+blankSpace+len(fieldBytes)+len(tailBytes)+len(id)) + + // fill the data + copy(nbytes, headBytes) + copy(nbytes[len(headBytes)+blankSpace:], fieldBytes) + copy(nbytes[len(headBytes)+blankSpace+len(fieldBytes):], tailBytes) + copy(nbytes[len(headBytes)+blankSpace+len(fieldBytes)+len(tailBytes):], id) + + // update the field size + item.setFieldsDataSize(len(nbytes) - len(id)) + + // update the data pointer + item.data = unsafe.Pointer(&nbytes[0]) + + return true +} + +func (item *Item) packedForEachField(count int, iter func(value float64) bool) { + data := item.fieldsBytes() + if count < 0 { + // iterate over of the known the values + for len(data) > 0 { + var value float64 + data, value = readPacked(data) + if !iter(value) { + return + } + } + } else { + for i := 0; i < count; i++ { + var value float64 + data, value = readPacked(data) + if !iter(value) { + return + } + } + } +} + +func (item *Item) packedGetField(index int) float64 { + + var idx int + var fvalue float64 + item.packedForEachField(-1, func(value float64) bool { + if idx == index { + fvalue = value + return false + } + idx++ + return true + }) + return fvalue +} diff --git a/internal/collection/item/packed_test.go b/internal/collection/item/packed_test.go new file mode 100644 index 00000000..9d2556ca --- /dev/null +++ b/internal/collection/item/packed_test.go @@ -0,0 +1,117 @@ +package item + +import ( + "math/rand" + "testing" + "time" +) + +func TestPacked(t *testing.T) { + start := time.Now() + for time.Since(start) < time.Second/2 { + testPacked(t) + } +} +func testPacked(t *testing.T) { + n := rand.Int() % 1024 + if n%2 == 1 { + n++ + } + values := make([]float64, n) + for i := 0; i < len(values); i++ { + switch rand.Int() % 9 { + case 0: + values[i] = 0 + case 1: + values[i] = float64((rand.Int() % 32) - 32/2) + case 2: + values[i] = float64((rand.Int() % 128)) + case 3: + values[i] = float64((rand.Int() % 8191) - 8191/2) + case 4: + values[i] = float64((rand.Int() % 2097152) - 2097152/2) + case 5: + values[i] = float64((rand.Int() % 536870912) - 536870912/2) + case 6: + values[i] = float64(rand.Int() % 500) + switch rand.Int() % 4 { + case 1: + values[i] = 0.25 + case 2: + values[i] = 0.50 + case 3: + values[i] = 0.75 + } + case 7: + values[i] = float64(rand.Float32()) + case 8: + values[i] = rand.Float64() + } + } + var dst []byte + for i := 0; i < len(values); i++ { + dst = appendPacked(dst, values[i]) + } + data := dst + var pvalues []float64 + for { + var value float64 + data, value = readPacked(data) + if data == nil { + break + } + pvalues = append(pvalues, value) + } + if !floatsEquals(values, pvalues) { + if len(values) != len(pvalues) { + t.Fatalf("sizes not equal") + } + for i := 0; i < len(values); i++ { + if values[i] != pvalues[i] { + t.Fatalf("expected '%v', got '%v'", values[i], pvalues[i]) + } + } + } + data = dst + var read int + + data, read = skipPacked(data, len(values)/2) + if read != len(values)/2 { + t.Fatalf("expected '%v', got '%v'", len(values)/2, read) + } + data, read = skipPacked(data, len(values)/2) + if read != len(values)/2 { + t.Fatalf("expected '%v', got '%v'", len(values)/2, read) + } + if len(data) != 0 { + t.Fatalf("expected '%v', got '%v'", 0, len(data)) + } + +} + +// func TestPackedItem(t *testing.T) { +// item := New("hello", nil, true) +// values := []float64{0, 1, 1, 0, 0, 1, 1, 0, 1} //, 1} //, 1, 0, 1, 0, 0, 1} +// fmt.Println(values) +// for i := 0; i < len(values); i++ { +// item.SetField(i, values[i]) +// } +// fmt.Print("[") +// for j := 0; j < len(values); j++ { +// if j > 0 { +// print(" ") +// } +// fmt.Print(item.GetField(j)) +// } +// print("]") +// println(item.ID()) + +// // for i := 0; i < len(values); i++ { + +// // fmt.Println(values[i], item.GetField(i)) +// // } + +// // fmt.Println(item.GetField(0)) +// // println(">>", item.ID()) + +// } diff --git a/internal/collection/item/unpacked.go b/internal/collection/item/unpacked.go new file mode 100644 index 00000000..6450ca08 --- /dev/null +++ b/internal/collection/item/unpacked.go @@ -0,0 +1,95 @@ +package item + +import ( + "reflect" + "unsafe" +) + +func getFieldAt(data unsafe.Pointer, index int) float64 { + return *(*float64)(unsafe.Pointer(uintptr(data) + uintptr(index*8))) +} + +func setFieldAt(data unsafe.Pointer, index int, value float64) { + *(*float64)(unsafe.Pointer(uintptr(data) + uintptr(index*8))) = value +} + +func (item *Item) dataBytes() []byte { + return *(*[]byte)((unsafe.Pointer)(&reflect.SliceHeader{ + Data: uintptr(unsafe.Pointer(item.data)), + Len: item.fieldsDataSize() + item.idDataSize(), + Cap: item.fieldsDataSize() + item.idDataSize(), + })) +} + +func bytesToFloats(f []byte) []float64 { + return *(*[]float64)((unsafe.Pointer)(&reflect.SliceHeader{ + Data: ((*reflect.SliceHeader)(unsafe.Pointer(&f))).Data, + Len: len(f) / 8, + Cap: len(f) / 8, + })) +} + +func (item *Item) unpackedGenerateFieldBytes(values []float64) []byte { + return *(*[]byte)((unsafe.Pointer)(&reflect.SliceHeader{ + Data: ((*reflect.SliceHeader)(unsafe.Pointer(&values))).Data, + Len: len(values) * 8, + Cap: len(values) * 8, + })) +} + +func (item *Item) unpackedSetField(index int, value float64) (updated bool) { + numFields := item.fieldsDataSize() / 8 + if index < numFields { + // field exists + if getFieldAt(item.data, index) == value { + return false + } + } else if value == 0 { + return false + } else { + + // make room for new field + oldBytes := item.dataBytes() + newData := make([]byte, (index+1)*8+item.idDataSize()) + // copy the existing fields + copy(newData, oldBytes[:item.fieldsDataSize()]) + // copy the id + copy(newData[(index+1)*8:], oldBytes[item.fieldsDataSize():]) + // update the fields length + item.setFieldsDataSize((index + 1) * 8) + // update the raw data + item.data = unsafe.Pointer(&newData[0]) + } + // set the new field + setFieldAt(item.data, index, value) + return true +} + +func (item *Item) unpackedForEachField( + count int, iter func(value float64) bool, +) { + fields := bytesToFloats(item.fieldsBytes()) + var n int + if count < 0 { + n = len(fields) + } else { + n = count + } + for i := 0; i < n; i++ { + var field float64 + if i < len(fields) { + field = fields[i] + } + if !iter(field) { + return + } + } +} + +func (item *Item) unpackedGetField(index int) float64 { + numFields := item.fieldsDataSize() / 8 + if index < numFields { + return getFieldAt(item.data, index) + } + return 0 +} diff --git a/internal/collection/rtree/rtree_test.go b/internal/collection/rtree/rtree_test.go index 938a4260..e7ad225a 100644 --- a/internal/collection/rtree/rtree_test.go +++ b/internal/collection/rtree/rtree_test.go @@ -46,7 +46,7 @@ func randPoints(N int) []*item.Item { box.min[j] = rand.Float64() } box.max = box.min - boxes[i] = item.New(fmt.Sprintf("%d", i), box) + boxes[i] = item.New(fmt.Sprintf("%d", i), box, false) } return boxes } @@ -68,7 +68,7 @@ func randBoxes(N int) []*item.Item { if box.max[0] > 180 || box.max[1] > 90 { i-- } - boxes[i] = item.New(fmt.Sprintf("%d", i), box) + boxes[i] = item.New(fmt.Sprintf("%d", i), box, false) } return boxes } @@ -264,7 +264,7 @@ func testBoxesVarious(t *testing.T, items []*item.Item, label string) { nbox.max[j] = box.max[j] + (rand.Float64() - 0.5) } } - nboxes[i] = item.New(fmt.Sprintf("%d", i), nbox) + nboxes[i] = item.New(fmt.Sprintf("%d", i), nbox, false) } for i := 0; i < N; i++ { tr.Insert(boxMin(nboxes[i]), boxMax(nboxes[i]), nboxes[i]) diff --git a/vendor/github.com/h2so5/half/.gitignore b/vendor/github.com/h2so5/half/.gitignore new file mode 100644 index 00000000..00268614 --- /dev/null +++ b/vendor/github.com/h2so5/half/.gitignore @@ -0,0 +1,22 @@ +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so + +# Folders +_obj +_test + +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out + +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* + +_testmain.go + +*.exe diff --git a/vendor/github.com/h2so5/half/LICENSE.txt b/vendor/github.com/h2so5/half/LICENSE.txt new file mode 100644 index 00000000..0e259d42 --- /dev/null +++ b/vendor/github.com/h2so5/half/LICENSE.txt @@ -0,0 +1,121 @@ +Creative Commons Legal Code + +CC0 1.0 Universal + + CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE + LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN + ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS + INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES + REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS + PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM + THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED + HEREUNDER. + +Statement of Purpose + +The laws of most jurisdictions throughout the world automatically confer +exclusive Copyright and Related Rights (defined below) upon the creator +and subsequent owner(s) (each and all, an "owner") of an original work of +authorship and/or a database (each, a "Work"). + +Certain owners wish to permanently relinquish those rights to a Work for +the purpose of contributing to a commons of creative, cultural and +scientific works ("Commons") that the public can reliably and without fear +of later claims of infringement build upon, modify, incorporate in other +works, reuse and redistribute as freely as possible in any form whatsoever +and for any purposes, including without limitation commercial purposes. +These owners may contribute to the Commons to promote the ideal of a free +culture and the further production of creative, cultural and scientific +works, or to gain reputation or greater distribution for their Work in +part through the use and efforts of others. + +For these and/or other purposes and motivations, and without any +expectation of additional consideration or compensation, the person +associating CC0 with a Work (the "Affirmer"), to the extent that he or she +is an owner of Copyright and Related Rights in the Work, voluntarily +elects to apply CC0 to the Work and publicly distribute the Work under its +terms, with knowledge of his or her Copyright and Related Rights in the +Work and the meaning and intended legal effect of CC0 on those rights. + +1. Copyright and Related Rights. A Work made available under CC0 may be +protected by copyright and related or neighboring rights ("Copyright and +Related Rights"). Copyright and Related Rights include, but are not +limited to, the following: + + i. the right to reproduce, adapt, distribute, perform, display, + communicate, and translate a Work; + ii. moral rights retained by the original author(s) and/or performer(s); +iii. publicity and privacy rights pertaining to a person's image or + likeness depicted in a Work; + iv. rights protecting against unfair competition in regards to a Work, + subject to the limitations in paragraph 4(a), below; + v. rights protecting the extraction, dissemination, use and reuse of data + in a Work; + vi. database rights (such as those arising under Directive 96/9/EC of the + European Parliament and of the Council of 11 March 1996 on the legal + protection of databases, and under any national implementation + thereof, including any amended or successor version of such + directive); and +vii. other similar, equivalent or corresponding rights throughout the + world based on applicable law or treaty, and any national + implementations thereof. + +2. Waiver. To the greatest extent permitted by, but not in contravention +of, applicable law, Affirmer hereby overtly, fully, permanently, +irrevocably and unconditionally waives, abandons, and surrenders all of +Affirmer's Copyright and Related Rights and associated claims and causes +of action, whether now known or unknown (including existing as well as +future claims and causes of action), in the Work (i) in all territories +worldwide, (ii) for the maximum duration provided by applicable law or +treaty (including future time extensions), (iii) in any current or future +medium and for any number of copies, and (iv) for any purpose whatsoever, +including without limitation commercial, advertising or promotional +purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each +member of the public at large and to the detriment of Affirmer's heirs and +successors, fully intending that such Waiver shall not be subject to +revocation, rescission, cancellation, termination, or any other legal or +equitable action to disrupt the quiet enjoyment of the Work by the public +as contemplated by Affirmer's express Statement of Purpose. + +3. Public License Fallback. Should any part of the Waiver for any reason +be judged legally invalid or ineffective under applicable law, then the +Waiver shall be preserved to the maximum extent permitted taking into +account Affirmer's express Statement of Purpose. In addition, to the +extent the Waiver is so judged Affirmer hereby grants to each affected +person a royalty-free, non transferable, non sublicensable, non exclusive, +irrevocable and unconditional license to exercise Affirmer's Copyright and +Related Rights in the Work (i) in all territories worldwide, (ii) for the +maximum duration provided by applicable law or treaty (including future +time extensions), (iii) in any current or future medium and for any number +of copies, and (iv) for any purpose whatsoever, including without +limitation commercial, advertising or promotional purposes (the +"License"). The License shall be deemed effective as of the date CC0 was +applied by Affirmer to the Work. Should any part of the License for any +reason be judged legally invalid or ineffective under applicable law, such +partial invalidity or ineffectiveness shall not invalidate the remainder +of the License, and in such case Affirmer hereby affirms that he or she +will not (i) exercise any of his or her remaining Copyright and Related +Rights in the Work or (ii) assert any associated claims and causes of +action with respect to the Work, in either case contrary to Affirmer's +express Statement of Purpose. + +4. Limitations and Disclaimers. + + a. No trademark or patent rights held by Affirmer are waived, abandoned, + surrendered, licensed or otherwise affected by this document. + b. Affirmer offers the Work as-is and makes no representations or + warranties of any kind concerning the Work, express, implied, + statutory or otherwise, including without limitation warranties of + title, merchantability, fitness for a particular purpose, non + infringement, or the absence of latent or other defects, accuracy, or + the present or absence of errors, whether or not discoverable, all to + the greatest extent permissible under applicable law. + c. Affirmer disclaims responsibility for clearing rights of other persons + that may apply to the Work or any use thereof, including without + limitation any person's Copyright and Related Rights in the Work. + Further, Affirmer disclaims responsibility for obtaining any necessary + consents, permissions or other rights required for any use of the + Work. + d. Affirmer understands and acknowledges that Creative Commons is not a + party to this document and has no duty or obligation with respect to + this CC0 or use of the Work. diff --git a/vendor/github.com/h2so5/half/README.md b/vendor/github.com/h2so5/half/README.md new file mode 100644 index 00000000..75947a52 --- /dev/null +++ b/vendor/github.com/h2so5/half/README.md @@ -0,0 +1,6 @@ +half +========== + +IEEE 754 binary16 half precision format for Go + +[![GoDoc](https://godoc.org/github.com/h2so5/half?status.svg)](https://godoc.org/github.com/h2so5/half) diff --git a/vendor/github.com/h2so5/half/float16.go b/vendor/github.com/h2so5/half/float16.go new file mode 100644 index 00000000..96b94b94 --- /dev/null +++ b/vendor/github.com/h2so5/half/float16.go @@ -0,0 +1,59 @@ +/* + + go-float16 - IEEE 754 binary16 half precision format + Written in 2013 by h2so5 + + To the extent possible under law, the author(s) have dedicated all copyright and + related and neighboring rights to this software to the public domain worldwide. + This software is distributed without any warranty. + You should have received a copy of the CC0 Public Domain Dedication along with this software. + If not, see . + +*/ + +// Package half is an IEEE 754 binary16 half precision format. +package half + +import "math" + +// A Float16 represents a 16-bit floating point number. +type Float16 uint16 + +// NewFloat16 allocates and returns a new Float16 set to f. +func NewFloat16(f float32) Float16 { + i := math.Float32bits(f) + sign := uint16((i >> 31) & 0x1) + exp := (i >> 23) & 0xff + exp16 := int16(exp) - 127 + 15 + frac := uint16(i>>13) & 0x3ff + if exp == 0 { + exp16 = 0 + } else if exp == 0xff { + exp16 = 0x1f + } else { + if exp16 > 0x1e { + exp16 = 0x1f + frac = 0 + } else if exp16 < 0x01 { + exp16 = 0 + frac = 0 + } + } + f16 := (sign << 15) | uint16(exp16<<10) | frac + return Float16(f16) +} + +// Float32 returns the float32 representation of f. +func (f Float16) Float32() float32 { + sign := uint32((f >> 15) & 0x1) + exp := (f >> 10) & 0x1f + exp32 := uint32(exp) + 127 - 15 + if exp == 0 { + exp32 = 0 + } else if exp == 0x1f { + exp32 = 0xff + } + frac := uint32(f & 0x3ff) + i := (sign << 31) | (exp32 << 23) | (frac << 13) + return math.Float32frombits(i) +} diff --git a/vendor/github.com/h2so5/half/float16_test.go b/vendor/github.com/h2so5/half/float16_test.go new file mode 100644 index 00000000..0993a760 --- /dev/null +++ b/vendor/github.com/h2so5/half/float16_test.go @@ -0,0 +1,55 @@ +/* + + go-float16 - IEEE 754 binary16 half precision format + Written in 2013 by h2so5 + + To the extent possible under law, the author(s) have dedicated all copyright and + related and neighboring rights to this software to the public domain worldwide. + This software is distributed without any warranty. + You should have received a copy of the CC0 Public Domain Dedication along with this software. + If not, see . + +*/ + +package half + +import ( + "math" + "testing" +) + +func getFloatTable() map[Float16]float32 { + table := map[Float16]float32{ + 0x3c00: 1, + 0x4000: 2, + 0xc000: -2, + 0x7bfe: 65472, + 0x7bff: 65504, + 0xfbff: -65504, + 0x0000: 0, + 0x8000: float32(math.Copysign(0, -1)), + 0x7c00: float32(math.Inf(1)), + 0xfc00: float32(math.Inf(-1)), + 0x5b8f: 241.875, + 0x48c8: 9.5625, + } + return table +} + +func TestFloat32(t *testing.T) { + for k, v := range getFloatTable() { + f := k.Float32() + if f != v { + t.Errorf("ToFloat32(%d) = %f, want %f.", k, f, v) + } + } +} + +func TestNewFloat16(t *testing.T) { + for k, v := range getFloatTable() { + i := NewFloat16(v) + if i != k { + t.Errorf("FromFloat32(%f) = %d, want %d.", v, i, k) + } + } +} diff --git a/vendor/github.com/tidwall/gjson/README.md b/vendor/github.com/tidwall/gjson/README.md index 70240d99..1e38644d 100644 --- a/vendor/github.com/tidwall/gjson/README.md +++ b/vendor/github.com/tidwall/gjson/README.md @@ -88,43 +88,14 @@ The dot and wildcard characters can be escaped with '\\'. ``` You can also query an array for the first match by using `#[...]`, or find all matches with `#[...]#`. -Queries support the `==`, `!=`, `<`, `<=`, `>`, `>=` comparison operators and the simple pattern matching `%` operator. +Queries support the `==`, `!=`, `<`, `<=`, `>`, `>=` comparison operators and the simple pattern matching `%` (like) and `!%` (not like) operators. ``` friends.#[last=="Murphy"].first >> "Dale" friends.#[last=="Murphy"]#.first >> ["Dale","Jane"] friends.#[age>45]#.last >> ["Craig","Murphy"] friends.#[first%"D*"].last >> "Murphy" -``` - -## JSON Lines - -There's support for [JSON Lines](http://jsonlines.org/) using the `..` prefix, which treats a multilined document as an array. - -For example: - -``` -{"name": "Gilbert", "age": 61} -{"name": "Alexa", "age": 34} -{"name": "May", "age": 57} -{"name": "Deloise", "age": 44} -``` - -``` -..# >> 4 -..1 >> {"name": "Alexa", "age": 34} -..3 >> {"name": "Deloise", "age": 44} -..#.name >> ["Gilbert","Alexa","May","Deloise"] -..#[name="May"].age >> 57 -``` - -The `ForEachLines` function will iterate through JSON lines. - -```go -gjson.ForEachLine(json, func(line gjson.Result) bool{ - println(line.String()) - return true -}) +friends.#[first!%"D*"].last >> "Craig" ``` ## Result Type @@ -193,6 +164,114 @@ result.Int() int64 // -9223372036854775808 to 9223372036854775807 result.Uint() int64 // 0 to 18446744073709551615 ``` +## Modifiers and path chaining + +New in version 1.2 is support for modifier functions and path chaining. + +A modifier is a path component that performs custom processing on the +json. + +Multiple paths can be "chained" together using the pipe character. +This is useful for getting results from a modified query. + +For example, using the built-in `@reverse` modifier on the above json document, +we'll get `children` array and reverse the order: + +``` +"children|@reverse" >> ["Jack","Alex","Sara"] +"children|@reverse|#" >> "Jack" +``` + +There are currently three built-in modifiers: + +- `@reverse`: Reverse an array or the members of an object. +- `@ugly`: Remove all whitespace from a json document. +- `@pretty`: Make the json document more human readable. + +### Modifier arguments + +A modifier may accept an optional argument. The argument can be a valid JSON +document or just characters. + +For example, the `@pretty` modifier takes a json object as its argument. + +``` +@pretty:{"sortKeys":true} +``` + +Which makes the json pretty and orders all of its keys. + +```json +{ + "age":37, + "children": ["Sara","Alex","Jack"], + "fav.movie": "Deer Hunter", + "friends": [ + {"age": 44, "first": "Dale", "last": "Murphy"}, + {"age": 68, "first": "Roger", "last": "Craig"}, + {"age": 47, "first": "Jane", "last": "Murphy"} + ], + "name": {"first": "Tom", "last": "Anderson"} +} +``` + +*The full list of `@pretty` options are `sortKeys`, `indent`, `prefix`, and `width`. +Please see [Pretty Options](https://github.com/tidwall/pretty#customized-output) for more information.* + +### Custom modifiers + +You can also add custom modifiers. + +For example, here we create a modifier that makes the entire json document upper +or lower case. + +```go +gjson.AddModifier("case", func(json, arg string) string { + if arg == "upper" { + return strings.ToUpper(json) + } + if arg == "lower" { + return strings.ToLower(json) + } + return json +}) +``` + +``` +"children|@case:upper" >> ["SARA","ALEX","JACK"] +"children|@case:lower|@reverse" >> ["jack","alex","sara"] +``` + +## JSON Lines + +There's support for [JSON Lines](http://jsonlines.org/) using the `..` prefix, which treats a multilined document as an array. + +For example: + +``` +{"name": "Gilbert", "age": 61} +{"name": "Alexa", "age": 34} +{"name": "May", "age": 57} +{"name": "Deloise", "age": 44} +``` + +``` +..# >> 4 +..1 >> {"name": "Alexa", "age": 34} +..3 >> {"name": "Deloise", "age": 44} +..#.name >> ["Gilbert","Alexa","May","Deloise"] +..#[name="May"].age >> 57 +``` + +The `ForEachLines` function will iterate through JSON lines. + +```go +gjson.ForEachLine(json, func(line gjson.Result) bool{ + println(line.String()) + return true +}) +``` + ## Get nested array values Suppose you want all the last names from the following json: diff --git a/vendor/github.com/tidwall/gjson/gjson.go b/vendor/github.com/tidwall/gjson/gjson.go index 74515ed5..a344b2d9 100644 --- a/vendor/github.com/tidwall/gjson/gjson.go +++ b/vendor/github.com/tidwall/gjson/gjson.go @@ -15,6 +15,7 @@ import ( "unicode/utf8" "github.com/tidwall/match" + "github.com/tidwall/pretty" ) // Type is Result type @@ -695,6 +696,8 @@ func parseLiteral(json string, i int) (int, string) { type arrayPathResult struct { part string path string + pipe string + piped bool more bool alogok bool arrch bool @@ -710,6 +713,14 @@ type arrayPathResult struct { func parseArrayPath(path string) (r arrayPathResult) { for i := 0; i < len(path); i++ { + if !DisableChaining { + if path[i] == '|' { + r.part = path[:i] + r.pipe = path[i+1:] + r.piped = true + return + } + } if path[i] == '.' { r.part = path[:i] r.path = path[i+1:] @@ -755,7 +766,7 @@ func parseArrayPath(path string) (r arrayPathResult) { if i < len(path) { s = i if path[i] == '!' { - if i < len(path)-1 && path[i+1] == '=' { + if i < len(path)-1 && (path[i+1] == '=' || path[i+1] == '%') { i++ } } else if path[i] == '<' || path[i] == '>' { @@ -828,15 +839,28 @@ func parseArrayPath(path string) (r arrayPathResult) { return } +// DisableChaining will disable the chaining (pipe) syntax +var DisableChaining = false + type objectPathResult struct { - part string - path string - wild bool - more bool + part string + path string + pipe string + piped bool + wild bool + more bool } func parseObjectPath(path string) (r objectPathResult) { for i := 0; i < len(path); i++ { + if !DisableChaining { + if path[i] == '|' { + r.part = path[:i] + r.pipe = path[i+1:] + r.piped = true + return + } + } if path[i] == '.' { r.part = path[:i] r.path = path[i+1:] @@ -934,6 +958,10 @@ func parseObject(c *parseContext, i int, path string) (int, bool) { var pmatch, kesc, vesc, ok, hit bool var key, val string rp := parseObjectPath(path) + if !rp.more && rp.piped { + c.pipe = rp.pipe + c.piped = true + } for i < len(c.json) { for ; i < len(c.json); i++ { if c.json[i] == '"' { @@ -1099,6 +1127,8 @@ func queryMatches(rp *arrayPathResult, value Result) bool { return value.Str >= rpv case "%": return match.Match(value.Str, rpv) + case "!%": + return !match.Match(value.Str, rpv) } case Number: rpvn, _ := strconv.ParseFloat(rpv, 64) @@ -1157,6 +1187,10 @@ func parseArray(c *parseContext, i int, path string) (int, bool) { partidx = int(n) } } + if !rp.more && rp.piped { + c.pipe = rp.pipe + c.piped = true + } for i < len(c.json)+1 { if !rp.arrch { pmatch = partidx == h @@ -1351,6 +1385,8 @@ func ForEachLine(json string, iterator func(line Result) bool) { type parseContext struct { json string value Result + pipe string + piped bool calcd bool lines bool } @@ -1388,6 +1424,22 @@ type parseContext struct { // If you are consuming JSON from an unpredictable source then you may want to // use the Valid function first. func Get(json, path string) Result { + if !DisableModifiers { + if len(path) > 1 && path[0] == '@' { + // possible modifier + var ok bool + var rjson string + path, rjson, ok = execModifier(json, path) + if ok { + if len(path) > 0 && path[0] == '|' { + res := Get(rjson, path[1:]) + res.Index = 0 + return res + } + return Parse(rjson) + } + } + } var i int var c = &parseContext{json: json} if len(path) >= 2 && path[0] == '.' && path[1] == '.' { @@ -1407,6 +1459,11 @@ func Get(json, path string) Result { } } } + if c.piped { + res := c.value.Get(c.pipe) + res.Index = 0 + return res + } fillIndex(json, c) return c.value } @@ -1626,7 +1683,11 @@ func GetMany(json string, path ...string) []Result { // The return value is a Result array where the number of items // will be equal to the number of input paths. func GetManyBytes(json []byte, path ...string) []Result { - return GetMany(string(json), path...) + res := make([]Result, len(path)) + for i, path := range path { + res[i] = GetBytes(json, path) + } + return res } var fieldsmu sync.RWMutex @@ -2032,7 +2093,7 @@ func validnull(data []byte, i int) (outi int, ok bool) { // value := gjson.Get(json, "name.last") // func Valid(json string) bool { - _, ok := validpayload([]byte(json), 0) + _, ok := validpayload(stringBytes(json), 0) return ok } @@ -2108,3 +2169,146 @@ func floatToInt(f float64) (n int64, ok bool) { } return 0, false } + +// execModifier parses the path to find a matching modifier function. +// then input expects that the path already starts with a '@' +func execModifier(json, path string) (pathOut, res string, ok bool) { + name := path[1:] + var hasArgs bool + for i := 1; i < len(path); i++ { + if path[i] == ':' { + pathOut = path[i+1:] + name = path[1:i] + hasArgs = len(pathOut) > 0 + break + } + if !DisableChaining { + if path[i] == '|' { + pathOut = path[i:] + name = path[1:i] + break + } + } + } + if fn, ok := modifiers[name]; ok { + var args string + if hasArgs { + var parsedArgs bool + switch pathOut[0] { + case '{', '[', '"': + res := Parse(pathOut) + if res.Exists() { + _, args = parseSquash(pathOut, 0) + pathOut = pathOut[len(args):] + parsedArgs = true + } + } + if !parsedArgs { + idx := -1 + if !DisableChaining { + idx = strings.IndexByte(pathOut, '|') + } + if idx == -1 { + args = pathOut + pathOut = "" + } else { + args = pathOut[:idx] + pathOut = pathOut[idx:] + } + } + } + return pathOut, fn(json, args), true + } + return pathOut, res, false +} + +// DisableModifiers will disable the modifier syntax +var DisableModifiers = false + +var modifiers = map[string]func(json, arg string) string{ + "pretty": modPretty, + "ugly": modUgly, + "reverse": modReverse, +} + +// AddModifier binds a custom modifier command to the GJSON syntax. +// This operation is not thread safe and should be executed prior to +// using all other gjson function. +func AddModifier(name string, fn func(json, arg string) string) { + modifiers[name] = fn +} + +// ModifierExists returns true when the specified modifier exists. +func ModifierExists(name string, fn func(json, arg string) string) bool { + _, ok := modifiers[name] + return ok +} + +// @pretty modifier makes the json look nice. +func modPretty(json, arg string) string { + if len(arg) > 0 { + opts := *pretty.DefaultOptions + Parse(arg).ForEach(func(key, value Result) bool { + switch key.String() { + case "sortKeys": + opts.SortKeys = value.Bool() + case "indent": + opts.Indent = value.String() + case "prefix": + opts.Prefix = value.String() + case "width": + opts.Width = int(value.Int()) + } + return true + }) + return bytesString(pretty.PrettyOptions(stringBytes(json), &opts)) + } + return bytesString(pretty.Pretty(stringBytes(json))) +} + +// @ugly modifier removes all whitespace. +func modUgly(json, arg string) string { + return bytesString(pretty.Ugly(stringBytes(json))) +} + +// @reverse reverses array elements or root object members. +func modReverse(json, arg string) string { + res := Parse(json) + if res.IsArray() { + var values []Result + res.ForEach(func(_, value Result) bool { + values = append(values, value) + return true + }) + out := make([]byte, 0, len(json)) + out = append(out, '[') + for i, j := len(values)-1, 0; i >= 0; i, j = i-1, j+1 { + if j > 0 { + out = append(out, ',') + } + out = append(out, values[i].Raw...) + } + out = append(out, ']') + return bytesString(out) + } + if res.IsObject() { + var keyValues []Result + res.ForEach(func(key, value Result) bool { + keyValues = append(keyValues, key, value) + return true + }) + out := make([]byte, 0, len(json)) + out = append(out, '{') + for i, j := len(keyValues)-2, 0; i >= 0; i, j = i-2, j+1 { + if j > 0 { + out = append(out, ',') + } + out = append(out, keyValues[i+0].Raw...) + out = append(out, ':') + out = append(out, keyValues[i+1].Raw...) + } + out = append(out, '}') + return bytesString(out) + } + return json +} diff --git a/vendor/github.com/tidwall/gjson/gjson_gae.go b/vendor/github.com/tidwall/gjson/gjson_gae.go index cbe2ab42..95869039 100644 --- a/vendor/github.com/tidwall/gjson/gjson_gae.go +++ b/vendor/github.com/tidwall/gjson/gjson_gae.go @@ -1,4 +1,4 @@ -//+build appengine +//+build appengine js package gjson @@ -8,3 +8,11 @@ func getBytes(json []byte, path string) Result { func fillIndex(json string, c *parseContext) { // noop. Use zero for the Index value. } + +func stringBytes(s string) []byte { + return []byte(s) +} + +func bytesString(b []byte) string { + return string(b) +} diff --git a/vendor/github.com/tidwall/gjson/gjson_ngae.go b/vendor/github.com/tidwall/gjson/gjson_ngae.go index ff313a78..bc608b53 100644 --- a/vendor/github.com/tidwall/gjson/gjson_ngae.go +++ b/vendor/github.com/tidwall/gjson/gjson_ngae.go @@ -1,4 +1,5 @@ //+build !appengine +//+build !js package gjson @@ -15,45 +16,40 @@ func getBytes(json []byte, path string) Result { if json != nil { // unsafe cast to string result = Get(*(*string)(unsafe.Pointer(&json)), path) - result = fromBytesGet(result) - } - return result -} - -func fromBytesGet(result Result) Result { - // safely get the string headers - rawhi := *(*reflect.StringHeader)(unsafe.Pointer(&result.Raw)) - strhi := *(*reflect.StringHeader)(unsafe.Pointer(&result.Str)) - // create byte slice headers - rawh := reflect.SliceHeader{Data: rawhi.Data, Len: rawhi.Len} - strh := reflect.SliceHeader{Data: strhi.Data, Len: strhi.Len} - if strh.Data == 0 { - // str is nil - if rawh.Data == 0 { + // safely get the string headers + rawhi := *(*reflect.StringHeader)(unsafe.Pointer(&result.Raw)) + strhi := *(*reflect.StringHeader)(unsafe.Pointer(&result.Str)) + // create byte slice headers + rawh := reflect.SliceHeader{Data: rawhi.Data, Len: rawhi.Len} + strh := reflect.SliceHeader{Data: strhi.Data, Len: strhi.Len} + if strh.Data == 0 { + // str is nil + if rawh.Data == 0 { + // raw is nil + result.Raw = "" + } else { + // raw has data, safely copy the slice header to a string + result.Raw = string(*(*[]byte)(unsafe.Pointer(&rawh))) + } + result.Str = "" + } else if rawh.Data == 0 { // raw is nil result.Raw = "" - } else { - // raw has data, safely copy the slice header to a string + // str has data, safely copy the slice header to a string + result.Str = string(*(*[]byte)(unsafe.Pointer(&strh))) + } else if strh.Data >= rawh.Data && + int(strh.Data)+strh.Len <= int(rawh.Data)+rawh.Len { + // Str is a substring of Raw. + start := int(strh.Data - rawh.Data) + // safely copy the raw slice header result.Raw = string(*(*[]byte)(unsafe.Pointer(&rawh))) + // substring the raw + result.Str = result.Raw[start : start+strh.Len] + } else { + // safely copy both the raw and str slice headers to strings + result.Raw = string(*(*[]byte)(unsafe.Pointer(&rawh))) + result.Str = string(*(*[]byte)(unsafe.Pointer(&strh))) } - result.Str = "" - } else if rawh.Data == 0 { - // raw is nil - result.Raw = "" - // str has data, safely copy the slice header to a string - result.Str = string(*(*[]byte)(unsafe.Pointer(&strh))) - } else if strh.Data >= rawh.Data && - int(strh.Data)+strh.Len <= int(rawh.Data)+rawh.Len { - // Str is a substring of Raw. - start := int(strh.Data - rawh.Data) - // safely copy the raw slice header - result.Raw = string(*(*[]byte)(unsafe.Pointer(&rawh))) - // substring the raw - result.Str = result.Raw[start : start+strh.Len] - } else { - // safely copy both the raw and str slice headers to strings - result.Raw = string(*(*[]byte)(unsafe.Pointer(&rawh))) - result.Str = string(*(*[]byte)(unsafe.Pointer(&strh))) } return result } @@ -71,3 +67,15 @@ func fillIndex(json string, c *parseContext) { } } } + +func stringBytes(s string) []byte { + return *(*[]byte)(unsafe.Pointer(&reflect.SliceHeader{ + Data: (*reflect.StringHeader)(unsafe.Pointer(&s)).Data, + Len: len(s), + Cap: len(s), + })) +} + +func bytesString(b []byte) string { + return *(*string)(unsafe.Pointer(&b)) +} diff --git a/vendor/github.com/tidwall/gjson/gjson_test.go b/vendor/github.com/tidwall/gjson/gjson_test.go index 6333be7d..41902797 100644 --- a/vendor/github.com/tidwall/gjson/gjson_test.go +++ b/vendor/github.com/tidwall/gjson/gjson_test.go @@ -11,6 +11,8 @@ import ( "strings" "testing" "time" + + "github.com/tidwall/pretty" ) // TestRandomData is a fuzzing test that throws random data at the Parse @@ -403,6 +405,10 @@ func TestBasic2(t *testing.T) { if mtok.String() != "aaaa" { t.Fatalf("expected %v, got %v", "aaaa", mtok.String()) } + mtok = get(basicJSON, `loggy.programmers.#[firstName !% "Bre*"].email`) + if mtok.String() != "bbbb" { + t.Fatalf("expected %v, got %v", "bbbb", mtok.String()) + } mtok = get(basicJSON, `loggy.programmers.#[firstName == "Brett"].email`) if mtok.String() != "aaaa" { t.Fatalf("expected %v, got %v", "aaaa", mtok.String()) @@ -1357,7 +1363,7 @@ null } func TestNumUint64String(t *testing.T) { - i := 9007199254740993 //2^53 + 1 + var i int64 = 9007199254740993 //2^53 + 1 j := fmt.Sprintf(`{"data": [ %d, "hello" ] }`, i) res := Get(j, "data.0") if res.String() != "9007199254740993" { @@ -1366,7 +1372,7 @@ func TestNumUint64String(t *testing.T) { } func TestNumInt64String(t *testing.T) { - i := -9007199254740993 + var i int64 = -9007199254740993 j := fmt.Sprintf(`{"data":[ "hello", %d ]}`, i) res := Get(j, "data.1") if res.String() != "-9007199254740993" { @@ -1384,7 +1390,7 @@ func TestNumBigString(t *testing.T) { } func TestNumFloatString(t *testing.T) { - i := -9007199254740993 + var i int64 = -9007199254740993 j := fmt.Sprintf(`{"data":[ "hello", %d ]}`, i) //No quotes around value!! res := Get(j, "data.1") if res.String() != "-9007199254740993" { @@ -1427,3 +1433,74 @@ func TestArrayValues(t *testing.T) { } } + +func BenchmarkValid(b *testing.B) { + for i := 0; i < b.N; i++ { + Valid(complicatedJSON) + } +} + +func BenchmarkValidBytes(b *testing.B) { + complicatedJSON := []byte(complicatedJSON) + for i := 0; i < b.N; i++ { + ValidBytes(complicatedJSON) + } +} + +func BenchmarkGoStdlibValidBytes(b *testing.B) { + complicatedJSON := []byte(complicatedJSON) + for i := 0; i < b.N; i++ { + json.Valid(complicatedJSON) + } +} + +func TestModifier(t *testing.T) { + json := `{"other":{"hello":"world"},"arr":[1,2,3,4,5,6]}` + opts := *pretty.DefaultOptions + opts.SortKeys = true + exp := string(pretty.PrettyOptions([]byte(json), &opts)) + res := Get(json, `@pretty:{"sortKeys":true}`).String() + if res != exp { + t.Fatalf("expected '%v', got '%v'", exp, res) + } + res = Get(res, "@pretty|@reverse|@ugly").String() + if res != json { + t.Fatalf("expected '%v', got '%v'", json, res) + } + res = Get(res, "@pretty|@reverse|arr|@reverse|2").String() + if res != "4" { + t.Fatalf("expected '%v', got '%v'", "4", res) + } + AddModifier("case", func(json, arg string) string { + if arg == "upper" { + return strings.ToUpper(json) + } + if arg == "lower" { + return strings.ToLower(json) + } + return json + }) + res = Get(json, "other|@case:upper").String() + if res != `{"HELLO":"WORLD"}` { + t.Fatalf("expected '%v', got '%v'", `{"HELLO":"WORLD"}`, res) + } +} + +func TestChaining(t *testing.T) { + json := `{ + "friends": [ + {"first": "Dale", "last": "Murphy", "age": 44}, + {"first": "Roger", "last": "Craig", "age": 68}, + {"first": "Jane", "last": "Murphy", "age": 47} + ] + }` + res := Get(json, "friends|0|first").String() + if res != "Dale" { + t.Fatalf("expected '%v', got '%v'", "Dale", res) + } + res = Get(json, "friends|@reverse|0|age").String() + if res != "47" { + t.Fatalf("expected '%v', got '%v'", "47", res) + } + +}