diff --git a/controller/controller.go b/controller/controller.go index b700f974..3029102d 100644 --- a/controller/controller.go +++ b/controller/controller.go @@ -390,7 +390,8 @@ func (c *Controller) handleInputCommand(conn *server.Conn, msg *server.Message, default: c.mu.RLock() defer c.mu.RUnlock() - case "set", "del", "drop", "fset", "flushdb", "sethook", "pdelhook", "delhook", "expire", "persist": + case "set", "del", "drop", "fset", "flushdb", "sethook", "pdelhook", "delhook", + "expire", "persist", "jset": // write operations write = true c.mu.Lock() @@ -401,7 +402,8 @@ func (c *Controller) handleInputCommand(conn *server.Conn, msg *server.Message, if c.config.ReadOnly { return writeErr(errors.New("read only")) } - case "get", "keys", "scan", "nearby", "within", "intersects", "hooks", "search", "ttl", "bounds", "server", "info", "type": + case "get", "keys", "scan", "nearby", "within", "intersects", "hooks", "search", + "ttl", "bounds", "server", "info", "type", "jget": // read operations c.mu.RLock() defer c.mu.RUnlock() @@ -533,6 +535,12 @@ func (c *Controller) command(msg *server.Message, w io.Writer) (res string, d co res, err = c.cmdBounds(msg) case "get": res, err = c.cmdGet(msg) + case "jget": + res, err = c.cmdJget(msg) + case "jset": + res, d, err = c.cmdJset(msg) + case "jdel": + res, d, err = c.cmdJdel(msg) case "type": res, err = c.cmdType(msg) case "keys": diff --git a/controller/json.go b/controller/json.go index 3cf765d2..30ed6ef2 100644 --- a/controller/json.go +++ b/controller/json.go @@ -1,6 +1,19 @@ package controller -import "encoding/json" +import ( + "bytes" + "encoding/json" + "strconv" + "strings" + "time" + + "github.com/tidwall/gjson" + "github.com/tidwall/resp" + "github.com/tidwall/sjson" + "github.com/tidwall/tile38/controller/collection" + "github.com/tidwall/tile38/controller/server" + "github.com/tidwall/tile38/geojson" +) func jsonString(s string) string { for i := 0; i < len(s); i++ { @@ -15,3 +28,232 @@ func jsonString(s string) string { b[len(b)-1] = '"' return string(b) } + +func (c *Controller) cmdJget(msg *server.Message) (string, error) { + start := time.Now() + if len(msg.Values) < 3 { + return "", errInvalidNumberOfArguments + } + if len(msg.Values) > 5 { + return "", errInvalidNumberOfArguments + } + key := msg.Values[1].String() + id := msg.Values[2].String() + var doget bool + var path string + var raw bool + if len(msg.Values) > 3 { + doget = true + path = msg.Values[3].String() + if len(msg.Values) == 5 { + if strings.ToLower(msg.Values[4].String()) == "raw" { + raw = true + } else { + return "", errInvalidArgument(msg.Values[4].String()) + } + } + } + col := c.getCol(key) + if col == nil { + if msg.OutputType == server.RESP { + return "$-1\r\n", nil + } + return "", errKeyNotFound + } + o, _, ok := col.Get(id) + if !ok { + if msg.OutputType == server.RESP { + return "$-1\r\n", nil + } + return "", errIDNotFound + } + var res gjson.Result + if doget { + res = gjson.Get(o.String(), path) + } else { + res = gjson.Parse(o.String()) + } + var val string + if raw { + val = res.Raw + } else { + val = res.String() + } + var buf bytes.Buffer + if msg.OutputType == server.JSON { + buf.WriteString(`{"ok":true`) + } + switch msg.OutputType { + case server.JSON: + if res.Exists() { + var val string + buf.WriteString(`,"value":` + jsonString(val)) + } + buf.WriteString(`,"elapsed":"` + time.Now().Sub(start).String() + "\"}") + return buf.String(), nil + case server.RESP: + if !res.Exists() { + return "$-1\r\n", nil + } + return "$" + strconv.FormatInt(int64(len(val)), 10) + "\r\n" + val + "\r\n", nil + } + return "", nil +} + +func (c *Controller) cmdJset(msg *server.Message) (res string, d commandDetailsT, err error) { + // JSET key path value [RAW] + start := time.Now() + var raw, str bool + switch len(msg.Values) { + default: + return "", d, errInvalidNumberOfArguments + case 5: + case 6: + switch strings.ToLower(msg.Values[5].String()) { + default: + return "", d, errInvalidArgument(msg.Values[5].String()) + case "raw": + raw = true + case "str": + str = true + } + } + + key := msg.Values[1].String() + id := msg.Values[2].String() + path := msg.Values[3].String() + val := msg.Values[4].String() + if !str && !raw { + switch val { + default: + if len(val) > 0 { + if (val[0] >= '0' && val[0] <= '9') || val[0] == '-' { + if _, err := strconv.ParseFloat(val, 64); err == nil { + raw = true + } + } + } + case "true", "false", "null": + raw = true + } + } + col := c.getCol(key) + var createcol bool + if col == nil { + col = collection.New() + createcol = true + } + var json string + var geoobj bool + o, _, ok := col.Get(id) + if ok { + if _, ok := o.(geojson.String); !ok { + geoobj = true + } + json = o.String() + } + if raw { + // set as raw block + json, err = sjson.SetRaw(json, path, val) + } else { + // set as a string + json, err = sjson.Set(json, path, val) + } + if err != nil { + return "", d, err + } + + if geoobj { + nmsg := *msg + nmsg.Values = []resp.Value{ + resp.StringValue("SET"), + resp.StringValue(key), + resp.StringValue(id), + resp.StringValue("OBJECT"), + resp.StringValue(json), + } + // SET key id OBJECT json + return c.cmdSet(&nmsg) + } + if createcol { + c.setCol(key, col) + } + c.clearIDExpires(key, id) + col.ReplaceOrInsert(id, geojson.String(json), nil, nil) + switch msg.OutputType { + case server.JSON: + var buf bytes.Buffer + buf.WriteString(`{"ok":true`) + buf.WriteString(`,"elapsed":"` + time.Now().Sub(start).String() + "\"}") + return buf.String(), d, nil + case server.RESP: + return "+OK\r\n", d, nil + } + return "", d, nil +} + +func (c *Controller) cmdJdel(msg *server.Message) (res string, d commandDetailsT, err error) { + start := time.Now() + if len(msg.Values) != 4 { + return "", d, errInvalidNumberOfArguments + } + key := msg.Values[1].String() + id := msg.Values[2].String() + path := msg.Values[3].String() + + col := c.getCol(key) + if col == nil { + if msg.OutputType == server.RESP { + return ":0\r\n", d, nil + } + return "", d, errKeyNotFound + } + + var json string + var geoobj bool + o, _, ok := col.Get(id) + if ok { + if _, ok := o.(geojson.String); !ok { + geoobj = true + } + json = o.String() + } + njson, err := sjson.Delete(json, path) + if err != nil { + return "", d, err + } + if njson == json { + switch msg.OutputType { + case server.JSON: + return "", d, errPathNotFound + case server.RESP: + return ":0\r\n", d, nil + } + return "", d, nil + } + json = njson + if geoobj { + nmsg := *msg + nmsg.Values = []resp.Value{ + resp.StringValue("SET"), + resp.StringValue(key), + resp.StringValue(id), + resp.StringValue("OBJECT"), + resp.StringValue(json), + } + // SET key id OBJECT json + return c.cmdSet(&nmsg) + } + c.clearIDExpires(key, id) + col.ReplaceOrInsert(id, geojson.String(json), nil, nil) + switch msg.OutputType { + case server.JSON: + var buf bytes.Buffer + buf.WriteString(`{"ok":true`) + buf.WriteString(`,"elapsed":"` + time.Now().Sub(start).String() + "\"}") + return buf.String(), d, nil + case server.RESP: + return ":1\r\n", d, nil + } + return "", d, nil +} diff --git a/controller/token.go b/controller/token.go index 5f0158dd..eb669520 100644 --- a/controller/token.go +++ b/controller/token.go @@ -16,6 +16,7 @@ var errInvalidNumberOfArguments = errors.New("invalid number of arguments") var errKeyNotFound = errors.New("key not found") var errIDNotFound = errors.New("id not found") var errIDAlreadyExists = errors.New("id already exists") +var errPathNotFound = errors.New("path not found") func errInvalidArgument(arg string) error { return fmt.Errorf("invalid argument '%s'", arg) diff --git a/tests/json_tests.go b/tests/json_tests.go new file mode 100644 index 00000000..d632f0ae --- /dev/null +++ b/tests/json_tests.go @@ -0,0 +1,40 @@ +package tests + +import "testing" + +func subTestJSON(t *testing.T, mc *mockServer) { + runStep(t, mc, "basic", json_JSET_basic_test) + runStep(t, mc, "geojson", json_JSET_geojson_test) +} +func json_JSET_basic_test(mc *mockServer) error { + return mc.DoBatch([][]interface{}{ + {"JSET", "mykey", "myid1", "hello", "world"}, {"OK"}, + {"JGET", "mykey", "myid1"}, {`{"hello":"world"}`}, + {"JSET", "mykey", "myid1", "hello", "planet"}, {"OK"}, + {"JGET", "mykey", "myid1"}, {`{"hello":"planet"}`}, + {"JSET", "mykey", "myid1", "user.name.last", "tom"}, {"OK"}, + {"JSET", "mykey", "myid1", "user.name.first", "andrew"}, {"OK"}, + {"JGET", "mykey", "myid1"}, {`{"user":{"name":{"first":"andrew","last":"tom"}},"hello":"planet"}`}, + {"JDEL", "mykey", "myid1", "user.name.last"}, {1}, + {"JGET", "mykey", "myid1"}, {`{"user":{"name":{"first":"andrew"}},"hello":"planet"}`}, + {"JDEL", "mykey", "myid1", "user.name.last"}, {0}, + {"JGET", "mykey", "myid1"}, {`{"user":{"name":{"first":"andrew"}},"hello":"planet"}`}, + {"JDEL", "mykey2", "myid1", "user.name.last"}, {0}, + }) +} +func json_JSET_geojson_test(mc *mockServer) error { + return mc.DoBatch([][]interface{}{ + {"SET", "mykey", "myid1", "POINT", 33, -115}, {"OK"}, + {"JGET", "mykey", "myid1"}, {`{"type":"Point","coordinates":[-115,33]}`}, + {"JSET", "mykey", "myid1", "coordinates.1", 44}, {"OK"}, + {"JGET", "mykey", "myid1"}, {`{"type":"Point","coordinates":[-115,44]}`}, + {"SET", "mykey", "myid1", "OBJECT", `{"type":"Feature","geometry":{"type":"Point","coordinates":[-115,44]}}`}, {"OK"}, + {"JGET", "mykey", "myid1"}, {`{"type":"Feature","geometry":{"type":"Point","coordinates":[-115,44]}}`}, + {"JGET", "mykey", "myid1", "geometry.type"}, {"Point"}, + {"JSET", "mykey", "myid1", "properties.tags.-1", "southwest"}, {"OK"}, + {"JSET", "mykey", "myid1", "properties.tags.-1", "united states"}, {"OK"}, + {"JSET", "mykey", "myid1", "properties.tags.-1", "hot"}, {"OK"}, + {"JGET", "mykey", "myid1"}, {`{"type":"Feature","geometry":{"type":"Point","coordinates":[-115,44]},"properties":{"tags":["southwest","united states","hot"]}}`}, + {"JDEL", "mykey", "myid1", "type"}, {"ERR Type member is invalid. Expecting a string"}, + }) +} diff --git a/tests/tests_test.go b/tests/tests_test.go index 926495d0..266dfdb9 100644 --- a/tests/tests_test.go +++ b/tests/tests_test.go @@ -40,6 +40,7 @@ func TestAll(t *testing.T) { } defer mc.Close() runSubTest(t, "keys", mc, subTestKeys) + runSubTest(t, "json", mc, subTestJSON) } func runSubTest(t *testing.T, name string, mc *mockServer, test func(t *testing.T, mc *mockServer)) { diff --git a/vendor/github.com/tidwall/gjson/README.md b/vendor/github.com/tidwall/gjson/README.md index 1ee5ae3d..2733e586 100644 --- a/vendor/github.com/tidwall/gjson/README.md +++ b/vendor/github.com/tidwall/gjson/README.md @@ -11,7 +11,9 @@

get a json value quickly

-GJSON is a Go package the provides a [very fast](#performance) and simple way to get a value from a json document. The reason for this library it to give efficient json indexing for the [BuntDB](https://github.com/tidwall/buntdb) project. +GJSON is a Go package that provides a [very fast](#performance) and simple way to get a value from a json document. The purpose for this library it to give efficient json indexing for the [BuntDB](https://github.com/tidwall/buntdb) project. + +For a command line interface check out [JSONed](https://github.com/tidwall/jsoned). Getting Started =============== @@ -27,7 +29,7 @@ $ go get -u github.com/tidwall/gjson This will retrieve the library. ## Get a value -Get searches json for the specified path. A path is in dot syntax, such as "name.last" or "age". This function expects that the json is well-formed and validates. Invalid json will not panic, but it may return back unexpected results. When the value is found it's returned immediately. +Get searches json for the specified path. A path is in dot syntax, such as "name.last" or "age". This function expects that the json is well-formed and validates. Invalid json will not panic, but it may return back unexpected results. When the value is found it's returned immediately. ```go package main @@ -47,6 +49,7 @@ This will print: ``` Prichard ``` +*There's also the [GetMany](#get-multiple-values-at-once) function to get multiple values at once, and [GetBytes](#working-with-bytes) for working with JSON byte slices.* ## Path Syntax @@ -63,25 +66,33 @@ The dot and wildcard characters can be escaped with '\'. "children": ["Sara","Alex","Jack"], "fav.movie": "Deer Hunter", "friends": [ - {"first": "James", "last": "Murphy"}, - {"first": "Roger", "last": "Craig"} + {"first": "Dale", "last": "Murphy", "age": 44}, + {"first": "Roger", "last": "Craig", "age": 68}, + {"first": "Jane", "last": "Murphy", "age": 47} ] } ``` ``` "name.last" >> "Anderson" "age" >> 37 +"children" >> ["Sara","Alex","Jack"] "children.#" >> 3 "children.1" >> "Alex" "child*.2" >> "Jack" "c?ildren.0" >> "Sara" "fav\.movie" >> "Deer Hunter" -"friends.#.first" >> [ "James", "Roger" ] +"friends.#.first" >> ["Dale","Roger","Jane"] "friends.1.last" >> "Craig" ``` -To query an array: + +You can also query an array for the first match by using `#[...]`, or find all matches with `#[...]#`. +Queries support the `==`, `!=`, `<`, `<=`, `>`, `>=` comparison operators and the simple pattern matching `%` operator. + ``` -`friends.#[last="Murphy"].first` >> "James" +friends.#[last=="Murphy"].first >> "Dale" +friends.#[last=="Murphy"]#.first >> ["Dale","Jane"] +friends.#[age>45]#.last >> ["Craig","Murphy"] +friends.#[first%"D*"].last >> "Murphy" ``` ## Result Type @@ -105,7 +116,7 @@ result.Type // can be String, Number, True, False, Null, or JSON result.Str // holds the string result.Num // holds the float64 number result.Raw // holds the raw json -result.Multi // holds nested array values +result.Index // index of raw value in original json, zero means index unknown ``` There are a variety of handy functions that work on a result: @@ -113,16 +124,25 @@ There are a variety of handy functions that work on a result: ```go result.Value() interface{} result.Int() int64 +result.Uint() uint64 result.Float() float64 result.String() string result.Bool() bool result.Array() []gjson.Result result.Map() map[string]gjson.Result result.Get(path string) Result +result.ForEach(iterator func(key, value Result) bool) +result.Less(token Result, caseSensitive bool) bool ``` The `result.Value()` function returns an `interface{}` which requires type assertion and is one of the following Go types: + + +The `result.Array()` function returns back an array of values. +If the result represents a non-existent value, then an empty array will be returned. +If the result is not a JSON array, the return value will be an array containing one result. + ```go boolean >> bool number >> float64 @@ -169,6 +189,20 @@ name := gjson.Get(json, `programmers.#[lastName="Hunter"].firstName`) println(name.String()) // prints "Elliotte" ``` +## Iterate through an object or array + +The `ForEach` function allows for quickly iterating through an object or array. +The key and value are passed to the iterator function for objects. +Only the value is passed for arrays. +Returning `false` from an iterator will stop iteration. + +```go +result := gjson.Get(json, "programmers") +result.ForEach(func(key, value Result) bool{ + println(value.String()) + return true // keep iterating +}) +``` ## Simple Parse and Get @@ -184,7 +218,7 @@ gjson.Get(json, "name.last") ## Check for the existence of a value -Sometimes you just want to know you if a value exists. +Sometimes you just want to know if a value exists. ```go value := gjson.Get(json, "name.last") @@ -211,6 +245,40 @@ if !ok{ } ``` +## Working with Bytes + +If your JSON is contained in a `[]byte` slice, there's the [GetBytes](https://godoc.org/github.com/tidwall/gjson#GetBytes) function. This is preferred over `Get(string(data), path)`. + +```go +var json []byte = ... +result := gjson.GetBytes(json, path) +``` + +If you are using the `gjson.GetBytes(json, path)` function and you want to avoid converting `result.Raw` to a `[]byte`, then you can use this pattern: + +```go +var json []byte = ... +result := gjson.GetBytes(json, path) +var raw []byte +if result.Index > 0 { + raw = json[result.Index:result.Index+len(result.Raw)] +} else { + raw = []byte(result.Raw) +} +``` + +This is a best-effort no allocation sub slice of the original json. This method utilizes the `result.Index` field, which is the position of the raw data in the original json. It's possible that the value of `result.Index` equals zero, in which case the `result.Raw` is converted to a `[]byte`. + +## Get multiple values at once + +The `GetMany` function can be used to get multiple values at the same time, and is optimized to scan over a JSON payload once. + +```go +results := gjson.GetMany(json, "name.first", "name.last", "age") +``` + +The return value is a `[]Result`, which will always contain exactly the same number of items as the input paths. + ## Performance Benchmarks of GJSON alongside [encoding/json](https://golang.org/pkg/encoding/json/), @@ -229,6 +297,17 @@ BenchmarkEasyJSONLexer-8 3000000 938 ns/op 613 B/op BenchmarkJSONParserGet-8 3000000 442 ns/op 21 B/op 0 allocs/op ``` +Benchmarks for the `GetMany` function: + +``` +BenchmarkGJSONGetMany4Paths-8 4000000 319 ns/op 112 B/op 0 allocs/op +BenchmarkGJSONGetMany8Paths-8 8000000 218 ns/op 56 B/op 0 allocs/op +BenchmarkGJSONGetMany16Paths-8 16000000 160 ns/op 56 B/op 0 allocs/op +BenchmarkGJSONGetMany32Paths-8 32000000 130 ns/op 64 B/op 0 allocs/op +BenchmarkGJSONGetMany64Paths-8 64000000 117 ns/op 64 B/op 0 allocs/op +BenchmarkGJSONGetMany128Paths-8 128000000 109 ns/op 64 B/op 0 allocs/op +``` + JSON document used: ```json @@ -267,6 +346,20 @@ widget.image.hOffset widget.text.onMouseUp ``` +For the `GetMany` benchmarks these paths are used: + +``` +widget.window.name +widget.image.hOffset +widget.text.onMouseUp +widget.window.title +widget.image.alignment +widget.text.style +widget.window.height +widget.image.src +widget.text.data +widget.text.size +``` *These benchmarks were run on a MacBook Pro 15" 2.8 GHz Intel Core i7 using Go 1.7.* diff --git a/vendor/github.com/tidwall/gjson/gjson.go b/vendor/github.com/tidwall/gjson/gjson.go index 6ea5ebef..1ee26c92 100644 --- a/vendor/github.com/tidwall/gjson/gjson.go +++ b/vendor/github.com/tidwall/gjson/gjson.go @@ -27,6 +27,26 @@ const ( JSON ) +// String returns a string representation of the type. +func (t Type) String() string { + switch t { + default: + return "" + case Null: + return "Null" + case False: + return "False" + case Number: + return "Number" + case String: + return "String" + case True: + return "True" + case JSON: + return "JSON" + } +} + // Result represents a json value that is returned from Get(). type Result struct { // Type is the json type @@ -37,6 +57,8 @@ type Result struct { Str string // Num is the json number Num float64 + // Index of raw value in original json, zero means index unknown + Index int } // String returns a string representation of the value. @@ -86,6 +108,21 @@ func (t Result) Int() int64 { } } +// Uint returns an unsigned integer representation. +func (t Result) Uint() uint64 { + switch t.Type { + default: + return 0 + case True: + return 1 + case String: + n, _ := strconv.ParseUint(t.Str, 10, 64) + return n + case Number: + return uint64(t.Num) + } +} + // Float returns an float64 representation. func (t Result) Float() float64 { switch t.Type { @@ -101,16 +138,91 @@ func (t Result) Float() float64 { } } -// Array returns back an array of children. The result must be a JSON array. +// Array returns back an array of values. +// If the result represents a non-existent value, then an empty array will be returned. +// If the result is not a JSON array, the return value will be an array containing one result. func (t Result) Array() []Result { - if t.Type != JSON { + if !t.Exists() { return nil } + if t.Type != JSON { + return []Result{t} + } r := t.arrayOrMap('[', false) return r.a } -// Map returns back an map of children. The result should be a JSON array. +// ForEach iterates through values. +// If the result represents a non-existent value, then no values will be iterated. +// If the result is an Object, the iterator will pass the key and value of each item. +// If the result is an Array, the iterator will only pass the value of each item. +// If the result is not a JSON array or object, the iterator will pass back one value equal to the result. +func (t Result) ForEach(iterator func(key, value Result) bool) { + if !t.Exists() { + return + } + if t.Type != JSON { + iterator(Result{}, t) + return + } + json := t.Raw + var keys bool + var i int + var key, value Result + for ; i < len(json); i++ { + if json[i] == '{' { + i++ + key.Type = String + keys = true + break + } else if json[i] == '[' { + i++ + break + } + if json[i] > ' ' { + return + } + } + var str string + var vesc bool + var ok bool + for ; i < len(json); i++ { + if keys { + if json[i] != '"' { + continue + } + s := i + i, str, vesc, ok = parseString(json, i+1) + if !ok { + return + } + if vesc { + key.Str = unescape(str[1 : len(str)-1]) + } else { + key.Str = str[1 : len(str)-1] + } + key.Raw = str + key.Index = s + } + for ; i < len(json); i++ { + if json[i] <= ' ' || json[i] == ',' || json[i] == ':' { + continue + } + break + } + s := i + i, value, ok = parseAny(json, i, true) + if !ok { + return + } + value.Index = s + if !iterator(key, value) { + return + } + } +} + +// Map returns back an map of values. The result should be a JSON array. func (t Result) Map() map[string]Result { if t.Type != JSON { return map[string]Result{} @@ -232,7 +344,7 @@ end: return } -// Parse parses the json and returns a result +// Parse parses the json and returns a result. func Parse(json string) Result { var value Result for i := 0; i < len(json); i++ { @@ -270,6 +382,12 @@ func Parse(json string) Result { return value } +// ParseBytes parses the json and returns a result. +// If working with bytes, this method preferred over Parse(string(data)) +func ParseBytes(json []byte) Result { + return Parse(string(json)) +} + func squash(json string) string { // expects that the lead character is a '[' or '{' // squash the value, ignoring all nested arrays and objects. @@ -387,7 +505,13 @@ func tostr(json string) (raw string, str string) { break } } - return json[:i+1], unescape(json[1:i]) + var ret string + if i+1 < len(json) { + ret = json[:i+1] + } else { + ret = json[:i] + } + return ret, unescape(json[1:i]) } } return json, json[1:] @@ -506,6 +630,7 @@ type arrayPathResult struct { path string op string value string + all bool } } @@ -536,8 +661,12 @@ func parseArrayPath(path string) (r arrayPathResult) { } s := i for ; i < len(path); i++ { - if path[i] <= ' ' || path[i] == '=' || - path[i] == '<' || path[i] == '>' || + if path[i] <= ' ' || + path[i] == '!' || + path[i] == '=' || + path[i] == '<' || + path[i] == '>' || + path[i] == '%' || path[i] == ']' { break } @@ -551,7 +680,11 @@ func parseArrayPath(path string) (r arrayPathResult) { } if i < len(path) { s = i - if path[i] == '<' || path[i] == '>' { + if path[i] == '!' { + if i < len(path)-1 && path[i+1] == '=' { + i++ + } + } else if path[i] == '<' || path[i] == '>' { if i < len(path)-1 && path[i+1] == '=' { i++ } @@ -596,6 +729,9 @@ func parseArrayPath(path string) (r arrayPathResult) { } } } else if path[i] == ']' { + if i+1 < len(path) && path[i+1] == '#' { + r.query.all = true + } break } } @@ -877,6 +1013,8 @@ func queryMatches(rp *arrayPathResult, value Result) bool { switch rp.query.op { case "=": return value.Str == rpv + case "!=": + return value.Str != rpv case "<": return value.Str < rpv case "<=": @@ -885,12 +1023,16 @@ func queryMatches(rp *arrayPathResult, value Result) bool { return value.Str > rpv case ">=": return value.Str >= rpv + case "%": + return match.Match(value.Str, rpv) } case Number: rpvn, _ := strconv.ParseFloat(rpv, 64) switch rp.query.op { case "=": return value.Num == rpvn + case "!=": + return value.Num == rpvn case "<": return value.Num < rpvn case "<=": @@ -904,6 +1046,8 @@ func queryMatches(rp *arrayPathResult, value Result) bool { switch rp.query.op { case "=": return rpv == "true" + case "!=": + return rpv != "true" case ">": return rpv == "false" case ">=": @@ -913,6 +1057,8 @@ func queryMatches(rp *arrayPathResult, value Result) bool { switch rp.query.op { case "=": return rpv == "false" + case "!=": + return rpv != "false" case "<": return rpv == "true" case "<=": @@ -927,6 +1073,7 @@ func parseArray(c *parseContext, i int, path string) (int, bool) { var h int var alog []int var partidx int + var multires []byte rp := parseArrayPath(path) if !rp.arrch { n, err := strconv.ParseUint(rp.part, 10, 64) @@ -983,12 +1130,21 @@ func parseArray(c *parseContext, i int, path string) (int, bool) { res := Get(val, rp.query.path) if queryMatches(&rp, res) { if rp.more { - c.value = Get(val, rp.path) + res = Get(val, rp.path) } else { - c.value.Raw = val - c.value.Type = JSON + res = Result{Raw: val, Type: JSON} + } + if rp.query.all { + if len(multires) == 0 { + multires = append(multires, '[') + } else { + multires = append(multires, ',') + } + multires = append(multires, res.Raw...) + } else { + c.value = res + return i, true } - return i, true } } else if hit { if rp.alogok { @@ -1051,13 +1207,14 @@ func parseArray(c *parseContext, i int, path string) (int, bool) { if rp.alogok { var jsons = make([]byte, 0, 64) jsons = append(jsons, '[') - for j := 0; j < len(alog); j++ { + for j, k := 0, 0; j < len(alog); j++ { res := Get(c.json[alog[j]:], rp.alogkey) if res.Exists() { - if j > 0 { + if k > 0 { jsons = append(jsons, ',') } jsons = append(jsons, []byte(res.Raw)...) + k++ } } jsons = append(jsons, ']') @@ -1071,9 +1228,16 @@ func parseArray(c *parseContext, i int, path string) (int, bool) { c.value.Raw = val c.value.Type = Number c.value.Num = float64(h - 1) + c.calcd = true return i + 1, true } } + if len(multires) > 0 && !c.value.Exists() { + c.value = Result{ + Raw: string(append(multires, ']')), + Type: JSON, + } + } return i + 1, false } break @@ -1085,6 +1249,7 @@ func parseArray(c *parseContext, i int, path string) (int, bool) { type parseContext struct { json string value Result + calcd bool } // Get searches json for the specified path. @@ -1093,7 +1258,7 @@ type parseContext struct { // Invalid json will not panic, but it may return back unexpected results. // When the value is found it's returned immediately. // -// A path is a series of keys seperated by a dot. +// A path is a series of keys searated by a dot. // A key may contain special wildcard characters '*' and '?'. // To access an array value use the index as the key. // To get the number of elements in an array or to access a child path, use the '#' character. @@ -1110,11 +1275,12 @@ type parseContext struct { // } // "name.last" >> "Anderson" // "age" >> 37 +// "children" >> ["Sara","Alex","Jack"] // "children.#" >> 3 // "children.1" >> "Alex" // "child*.2" >> "Jack" // "c?ildren.0" >> "Sara" -// "friends.#.first" >> [ "James", "Roger" ] +// "friends.#.first" >> ["James","Roger"] // func Get(json, path string) Result { var i int @@ -1131,8 +1297,53 @@ func Get(json, path string) Result { break } } + if len(c.value.Raw) > 0 && !c.calcd { + jhdr := *(*reflect.StringHeader)(unsafe.Pointer(&json)) + rhdr := *(*reflect.StringHeader)(unsafe.Pointer(&(c.value.Raw))) + c.value.Index = int(rhdr.Data - jhdr.Data) + if c.value.Index < 0 || c.value.Index >= len(json) { + c.value.Index = 0 + } + } return c.value } +func fromBytesGet(result Result) Result { + // safely get the string headers + rawhi := *(*reflect.StringHeader)(unsafe.Pointer(&result.Raw)) + strhi := *(*reflect.StringHeader)(unsafe.Pointer(&result.Str)) + // create byte slice headers + rawh := reflect.SliceHeader{Data: rawhi.Data, Len: rawhi.Len} + strh := reflect.SliceHeader{Data: strhi.Data, Len: strhi.Len} + if strh.Data == 0 { + // str is nil + if rawh.Data == 0 { + // raw is nil + result.Raw = "" + } else { + // raw has data, safely copy the slice header to a string + result.Raw = string(*(*[]byte)(unsafe.Pointer(&rawh))) + } + result.Str = "" + } else if rawh.Data == 0 { + // raw is nil + result.Raw = "" + // str has data, safely copy the slice header to a string + result.Str = string(*(*[]byte)(unsafe.Pointer(&strh))) + } else if strh.Data >= rawh.Data && + int(strh.Data)+strh.Len <= int(rawh.Data)+rawh.Len { + // Str is a substring of Raw. + start := int(strh.Data - rawh.Data) + // safely copy the raw slice header + result.Raw = string(*(*[]byte)(unsafe.Pointer(&rawh))) + // substring the raw + result.Str = result.Raw[start : start+strh.Len] + } else { + // safely copy both the raw and str slice headers to strings + result.Raw = string(*(*[]byte)(unsafe.Pointer(&rawh))) + result.Str = string(*(*[]byte)(unsafe.Pointer(&strh))) + } + return result +} // GetBytes searches json for the specified path. // If working with bytes, this method preferred over Get(string(data), path) @@ -1141,40 +1352,7 @@ func GetBytes(json []byte, path string) Result { if json != nil { // unsafe cast to string result = Get(*(*string)(unsafe.Pointer(&json)), path) - // safely get the string headers - rawhi := *(*reflect.StringHeader)(unsafe.Pointer(&result.Raw)) - strhi := *(*reflect.StringHeader)(unsafe.Pointer(&result.Str)) - // create byte slice headers - rawh := reflect.SliceHeader{Data: rawhi.Data, Len: rawhi.Len} - strh := reflect.SliceHeader{Data: strhi.Data, Len: strhi.Len} - if strh.Data == 0 { - // str is nil - if rawh.Data == 0 { - // raw is nil - result.Raw = "" - } else { - // raw has data, safely copy the slice header to a string - result.Raw = string(*(*[]byte)(unsafe.Pointer(&rawh))) - } - result.Str = "" - } else if rawh.Data == 0 { - // raw is nil - result.Raw = "" - // str has data, safely copy the slice header to a string - result.Str = string(*(*[]byte)(unsafe.Pointer(&strh))) - } else if strh.Data >= rawh.Data && - int(strh.Data)+strh.Len <= int(rawh.Data)+rawh.Len { - // Str is a substring of Raw. - start := int(strh.Data - rawh.Data) - // safely copy the raw slice header - result.Raw = string(*(*[]byte)(unsafe.Pointer(&rawh))) - // substring the raw - result.Str = result.Raw[start : start+strh.Len] - } else { - // safely copy both the raw and str slice headers to strings - result.Raw = string(*(*[]byte)(unsafe.Pointer(&rawh))) - result.Str = string(*(*[]byte)(unsafe.Pointer(&strh))) - } + result = fromBytesGet(result) } return result } @@ -1300,3 +1478,465 @@ func stringLessInsensitive(a, b string) bool { } return len(a) < len(b) } + +// parseAny parses the next value from a json string. +// A Result is returned when the hit param is set. +// The return values are (i int, res Result, ok bool) +func parseAny(json string, i int, hit bool) (int, Result, bool) { + var res Result + var val string + for ; i < len(json); i++ { + if json[i] == '{' || json[i] == '[' { + i, val = parseSquash(json, i) + if hit { + res.Raw = val + res.Type = JSON + } + return i, res, true + } + if json[i] <= ' ' { + continue + } + switch json[i] { + case '"': + i++ + var vesc bool + var ok bool + i, val, vesc, ok = parseString(json, i) + if !ok { + return i, res, false + } + if hit { + res.Type = String + res.Raw = val + if vesc { + res.Str = unescape(val[1 : len(val)-1]) + } else { + res.Str = val[1 : len(val)-1] + } + } + return i, res, true + case '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': + i, val = parseNumber(json, i) + if hit { + res.Raw = val + res.Type = Number + res.Num, _ = strconv.ParseFloat(val, 64) + } + return i, res, true + case 't', 'f', 'n': + vc := json[i] + i, val = parseLiteral(json, i) + if hit { + res.Raw = val + switch vc { + case 't': + res.Type = True + case 'f': + res.Type = False + } + return i, res, true + } + } + } + return i, res, false +} + +var ( // used for testing + testWatchForFallback bool + testLastWasFallback bool +) + +// areSimplePaths returns true if all the paths are simple enough +// to parse quickly for GetMany(). Allows alpha-numeric, dots, +// underscores, and the dollar sign. It does not allow non-alnum, +// escape characters, or keys which start with a numbers. +// For example: +// "name.last" == OK +// "user.id0" == OK +// "user.ID" == OK +// "user.first_name" == OK +// "user.firstName" == OK +// "user.0item" == BAD +// "user.#id" == BAD +// "user\.name" == BAD +func areSimplePaths(paths []string) bool { + for _, path := range paths { + var fi int // first key index, for keys with numeric prefix + for i := 0; i < len(path); i++ { + if path[i] >= 'a' && path[i] <= 'z' { + // a-z is likely to be the highest frequency charater. + continue + } + if path[i] == '.' { + fi = i + 1 + continue + } + if path[i] >= 'A' && path[i] <= 'Z' { + continue + } + if path[i] == '_' || path[i] == '$' { + continue + } + if i > fi && path[i] >= '0' && path[i] <= '9' { + continue + } + return false + } + } + return true +} + +// GetMany searches json for the multiple paths. +// The return value is a Result array where the number of items +// will be equal to the number of input paths. +func GetMany(json string, paths ...string) []Result { + if len(paths) < 4 { + if testWatchForFallback { + testLastWasFallback = false + } + switch len(paths) { + case 0: + // return nil when no paths are specified. + return nil + case 1: + return []Result{Get(json, paths[0])} + case 2: + return []Result{Get(json, paths[0]), Get(json, paths[1])} + case 3: + return []Result{Get(json, paths[0]), Get(json, paths[1]), Get(json, paths[2])} + } + } + var results []Result + var ok bool + var i int + if len(paths) > 512 { + // we can only support up to 512 paths. Is that too many? + goto fallback + } + if !areSimplePaths(paths) { + // If there is even one path that is not considered "simple" then + // we need to use the fallback method. + goto fallback + } + // locate the object token. + for ; i < len(json); i++ { + if json[i] == '{' { + i++ + break + } + if json[i] <= ' ' { + continue + } + goto fallback + } + // use the call function table. + if len(paths) <= 8 { + results, ok = getMany8(json, i, paths) + } else if len(paths) <= 16 { + results, ok = getMany16(json, i, paths) + } else if len(paths) <= 32 { + results, ok = getMany32(json, i, paths) + } else if len(paths) <= 64 { + results, ok = getMany64(json, i, paths) + } else if len(paths) <= 128 { + results, ok = getMany128(json, i, paths) + } else if len(paths) <= 256 { + results, ok = getMany256(json, i, paths) + } else if len(paths) <= 512 { + results, ok = getMany512(json, i, paths) + } + if !ok { + // there was some fault while parsing. we should try the + // fallback method. This could result in performance + // degregation in some cases. + goto fallback + } + if testWatchForFallback { + testLastWasFallback = false + } + return results +fallback: + results = results[:0] + for i := 0; i < len(paths); i++ { + results = append(results, Get(json, paths[i])) + } + if testWatchForFallback { + testLastWasFallback = true + } + return results +} + +// GetManyBytes searches json for the specified path. +// If working with bytes, this method preferred over +// GetMany(string(data), paths...) +func GetManyBytes(json []byte, paths ...string) []Result { + if json == nil { + return GetMany("", paths...) + } + results := GetMany(*(*string)(unsafe.Pointer(&json)), paths...) + for i := range results { + results[i] = fromBytesGet(results[i]) + } + return results +} + +// parseGetMany parses a json object for keys that match against the callers +// paths. It's a best-effort attempt and quickly locating and assigning the +// values to the []Result array. If there are failures such as bad json, or +// invalid input paths, or too much recursion, the function will exit with a +// return value of 'false'. +func parseGetMany( + json string, i int, + level uint, kplen int, + paths []string, completed []bool, matches []uint64, results []Result, +) (int, bool) { + if level > 62 { + // The recursion level is limited because the matches []uint64 + // array cannot handle more the 64-bits. + return i, false + } + // At this point the last character read was a '{'. + // Read all object keys and try to match against the paths. + var key string + var val string + var vesc, ok bool +next_key: + for ; i < len(json); i++ { + if json[i] == '"' { + // read the key + i, val, vesc, ok = parseString(json, i+1) + if !ok { + return i, false + } + if vesc { + // the value is escaped + key = unescape(val[1 : len(val)-1]) + } else { + // just a plain old ascii key + key = val[1 : len(val)-1] + } + var hasMatch bool + var parsedVal bool + var valOrgIndex int + var valPathIndex int + for j := 0; j < len(key); j++ { + if key[j] == '.' { + // we need to look for keys with dot and ignore them. + if i, _, ok = parseAny(json, i, false); !ok { + return i, false + } + continue next_key + } + } + var usedPaths int + // loop through paths and look for matches + for j := 0; j < len(paths); j++ { + if completed[j] { + usedPaths++ + // ignore completed paths + continue + } + if level > 0 && (matches[j]>>(level-1))&1 == 0 { + // ignore unmatched paths + usedPaths++ + continue + } + + // try to match the key to the path + // this is spaghetti code but the idea is to minimize + // calls and variable assignments when comparing the + // key to paths + if len(paths[j])-kplen >= len(key) { + i, k := kplen, 0 + for ; k < len(key); k, i = k+1, i+1 { + if key[k] != paths[j][i] { + // no match + goto nomatch + } + } + if i < len(paths[j]) { + if paths[j][i] == '.' { + // matched, but there still more keys in the path + goto match_not_atend + } + } + // matched and at the end of the path + goto match_atend + } + // no match, jump to the nomatch label + goto nomatch + match_atend: + // found a match + // at the end of the path. we must take the value. + usedPaths++ + if !parsedVal { + // the value has not been parsed yet. let's do so. + valOrgIndex = i // keep track of the current position. + i, results[j], ok = parseAny(json, i, true) + if !ok { + return i, false + } + parsedVal = true + valPathIndex = j + } else { + results[j] = results[valPathIndex] + } + // mark as complete + completed[j] = true + // jump over the match_not_atend label + goto nomatch + match_not_atend: + // found a match + // still in the middle of the path. + usedPaths++ + // mark the path as matched + matches[j] |= 1 << level + if !hasMatch { + hasMatch = true + } + nomatch: // noop label + } + + if !parsedVal { + if hasMatch { + // we found a match and the value has not been parsed yet. + // let's find out if the next value type is an object. + for ; i < len(json); i++ { + if json[i] <= ' ' || json[i] == ':' { + continue + } + break + } + if i < len(json) { + if json[i] == '{' { + // it's an object. let's go deeper + i, ok = parseGetMany(json, i+1, level+1, kplen+len(key)+1, paths, completed, matches, results) + if !ok { + return i, false + } + } else { + // not an object. just parse and ignore. + if i, _, ok = parseAny(json, i, false); !ok { + return i, false + } + } + } + } else { + // Since there was no matches we can just parse the value and + // ignore the result. + if i, _, ok = parseAny(json, i, false); !ok { + return i, false + } + } + } else if hasMatch && len(results[valPathIndex].Raw) > 0 && results[valPathIndex].Raw[0] == '{' { + // The value was already parsed and the value type is an object. + // Rewind the json index and let's parse deeper. + i = valOrgIndex + for ; i < len(json); i++ { + if json[i] == '{' { + break + } + } + i, ok = parseGetMany(json, i+1, level+1, kplen+len(key)+1, paths, completed, matches, results) + if !ok { + return i, false + } + } + if usedPaths == len(paths) { + // all paths have been used, either completed or matched. + // we should stop parsing this object to save CPU cycles. + if level > 0 && i < len(json) { + i, _ = parseSquash(json, i) + } + return i, true + } + } else if json[i] == '}' { + // reached the end of the object. end it here. + return i + 1, true + } + } + return i, true +} + +// Call table for GetMany. Using an isolated function allows for allocating +// arrays with know capacities on the stack, as opposed to dynamically +// allocating on the heap. This can provide a tremendous performance boost +// by avoiding the GC. +func getMany8(json string, i int, paths []string) ([]Result, bool) { + const max = 8 + var completed = make([]bool, 0, max) + var matches = make([]uint64, 0, max) + var results = make([]Result, 0, max) + completed = completed[0:len(paths):max] + matches = matches[0:len(paths):max] + results = results[0:len(paths):max] + _, ok := parseGetMany(json, i, 0, 0, paths, completed, matches, results) + return results, ok +} +func getMany16(json string, i int, paths []string) ([]Result, bool) { + const max = 16 + var completed = make([]bool, 0, max) + var matches = make([]uint64, 0, max) + var results = make([]Result, 0, max) + completed = completed[0:len(paths):max] + matches = matches[0:len(paths):max] + results = results[0:len(paths):max] + _, ok := parseGetMany(json, i, 0, 0, paths, completed, matches, results) + return results, ok +} +func getMany32(json string, i int, paths []string) ([]Result, bool) { + const max = 32 + var completed = make([]bool, 0, max) + var matches = make([]uint64, 0, max) + var results = make([]Result, 0, max) + completed = completed[0:len(paths):max] + matches = matches[0:len(paths):max] + results = results[0:len(paths):max] + _, ok := parseGetMany(json, i, 0, 0, paths, completed, matches, results) + return results, ok +} +func getMany64(json string, i int, paths []string) ([]Result, bool) { + const max = 64 + var completed = make([]bool, 0, max) + var matches = make([]uint64, 0, max) + var results = make([]Result, 0, max) + completed = completed[0:len(paths):max] + matches = matches[0:len(paths):max] + results = results[0:len(paths):max] + _, ok := parseGetMany(json, i, 0, 0, paths, completed, matches, results) + return results, ok +} +func getMany128(json string, i int, paths []string) ([]Result, bool) { + const max = 128 + var completed = make([]bool, 0, max) + var matches = make([]uint64, 0, max) + var results = make([]Result, 0, max) + completed = completed[0:len(paths):max] + matches = matches[0:len(paths):max] + results = results[0:len(paths):max] + _, ok := parseGetMany(json, i, 0, 0, paths, completed, matches, results) + return results, ok +} +func getMany256(json string, i int, paths []string) ([]Result, bool) { + const max = 256 + var completed = make([]bool, 0, max) + var matches = make([]uint64, 0, max) + var results = make([]Result, 0, max) + completed = completed[0:len(paths):max] + matches = matches[0:len(paths):max] + results = results[0:len(paths):max] + _, ok := parseGetMany(json, i, 0, 0, paths, completed, matches, results) + return results, ok +} +func getMany512(json string, i int, paths []string) ([]Result, bool) { + const max = 512 + var completed = make([]bool, 0, max) + var matches = make([]uint64, 0, max) + var results = make([]Result, 0, max) + completed = completed[0:len(paths):max] + matches = matches[0:len(paths):max] + results = results[0:len(paths):max] + _, ok := parseGetMany(json, i, 0, 0, paths, completed, matches, results) + return results, ok +} diff --git a/vendor/github.com/tidwall/gjson/gjson_test.go b/vendor/github.com/tidwall/gjson/gjson_test.go index b92db2a7..99b5048f 100644 --- a/vendor/github.com/tidwall/gjson/gjson_test.go +++ b/vendor/github.com/tidwall/gjson/gjson_test.go @@ -36,6 +36,7 @@ func TestRandomData(t *testing.T) { } lstr = string(b[:n]) GetBytes([]byte(lstr), "zzzz") + Parse(lstr) } } @@ -104,19 +105,22 @@ var basicJSON = `{"age":100, "name":{"here":"B\\\"R"}, "loggy":{ "programmers": [ { - "firstName": "Brett", - "lastName": "McLaughlin", - "email": "aaaa" - }, + "firstName": "Brett", + "lastName": "McLaughlin", + "email": "aaaa", + "tag": "good" + }, { - "firstName": "Jason", - "lastName": "Hunter", - "email": "bbbb" - }, + "firstName": "Jason", + "lastName": "Hunter", + "email": "bbbb", + "tag": "bad" + }, { - "firstName": "Elliotte", - "lastName": "Harold", - "email": "cccc" + "firstName": "Elliotte", + "lastName": "Harold", + "email": "cccc", + "tag":, "good" }, { "firstName": 1002.3, @@ -151,12 +155,63 @@ func get(json, path string) Result { func TestBasic(t *testing.T) { var mtok Result - - mtok = get(basicJSON, `loggy.programmers.#[age=101].firstName`) - if mtok.String() != "1002.3" { - t.Fatalf("expected %v, got %v", "1002,3", mtok.String()) + mtok = get(basicJSON, `loggy.programmers.#[tag="good"].firstName`) + if mtok.String() != "Brett" { + t.Fatalf("expected %v, got %v", "Brett", mtok.String()) + } + mtok = get(basicJSON, `loggy.programmers.#[tag="good"]#.firstName`) + if mtok.String() != `["Brett","Elliotte"]` { + t.Fatalf("expected %v, got %v", `["Brett","Elliotte"]`, mtok.String()) } + mtok = get(basicJSON, `loggy.programmers`) + var count int + mtok.ForEach(func(key, value Result) bool { + if key.Exists() { + t.Fatalf("expected %v, got %v", false, key.Exists()) + } + count++ + if count == 3 { + return false + } + if count == 1 { + i := 0 + value.ForEach(func(key, value Result) bool { + switch i { + case 0: + if key.String() != "firstName" || value.String() != "Brett" { + t.Fatalf("expected %v/%v got %v/%v", "firstName", "Brett", key.String(), value.String()) + } + case 1: + if key.String() != "lastName" || value.String() != "McLaughlin" { + t.Fatalf("expected %v/%v got %v/%v", "lastName", "McLaughlin", key.String(), value.String()) + } + case 2: + if key.String() != "email" || value.String() != "aaaa" { + t.Fatalf("expected %v/%v got %v/%v", "email", "aaaa", key.String(), value.String()) + } + } + i++ + return true + }) + } + return true + }) + if count != 3 { + t.Fatalf("expected %v, got %v", 3, count) + } + mtok = get(basicJSON, `loggy.programmers.#[age=101].firstName`) + if mtok.String() != "1002.3" { + t.Fatalf("expected %v, got %v", "1002.3", mtok.String()) + } + mtok = get(basicJSON, `loggy.programmers.#[firstName != "Brett"].firstName`) + if mtok.String() != "Jason" { + t.Fatalf("expected %v, got %v", "Jason", mtok.String()) + } + mtok = get(basicJSON, `loggy.programmers.#[firstName % "Bre*"].email`) + if mtok.String() != "aaaa" { + t.Fatalf("expected %v, got %v", "aaaa", mtok.String()) + } mtok = get(basicJSON, `loggy.programmers.#[firstName == "Brett"].email`) if mtok.String() != "aaaa" { t.Fatalf("expected %v, got %v", "aaaa", mtok.String()) @@ -444,6 +499,119 @@ func TestUnmarshalMap(t *testing.T) { } } +func TestSingleArrayValue(t *testing.T) { + var json = `{"key": "value","key2":[1,2,3,4,"A"]}` + var result = Get(json, "key") + var array = result.Array() + if len(array) != 1 { + t.Fatal("array is empty") + } + if array[0].String() != "value" { + t.Fatal("got %s, should be %s", array[0].String(), "value") + } + + array = Get(json, "key2.#").Array() + if len(array) != 1 { + t.Fatal("got '%v', expected '%v'", len(array), 1) + } + + array = Get(json, "key3").Array() + if len(array) != 0 { + t.Fatal("got '%v', expected '%v'", len(array), 0) + } + +} + +var manyJSON = ` { + "a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{ + "a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{ + "a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{ + "a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{ + "a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{ + "a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{ + "a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"hello":"world" + }}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}} + "position":{"type":"Point","coordinates":[-115.24,33.09]}, + "loves":["world peace"], + "name":{"last":"Anderson","first":"Nancy"}, + "age":31 + "":{"a":"emptya","b":"emptyb"}, + "name.last":"Yellow", + "name.first":"Cat", +}` + +func combine(results []Result) string { + return fmt.Sprintf("%v", results) +} +func TestManyBasic(t *testing.T) { + testWatchForFallback = true + defer func() { + testWatchForFallback = false + }() + testMany := func(shouldFallback bool, expect string, paths ...string) { + results := GetMany( + manyJSON, + paths..., + ) + if len(results) != len(paths) { + t.Fatalf("expected %v, got %v", len(paths), len(results)) + } + if fmt.Sprintf("%v", results) != expect { + t.Fatalf("expected %v, got %v", expect, results) + } + return + if testLastWasFallback != shouldFallback { + t.Fatalf("expected %v, got %v", shouldFallback, testLastWasFallback) + } + } + testMany(false, "[Point]", "position.type") + testMany(false, `[emptya ["world peace"] 31]`, ".a", "loves", "age") + testMany(false, `[["world peace"]]`, "loves") + testMany(false, `[{"last":"Anderson","first":"Nancy"} Nancy]`, "name", "name.first") + testMany(true, `[null]`, strings.Repeat("a.", 40)+"hello") + res := Get(manyJSON, strings.Repeat("a.", 48)+"a") + testMany(true, `[`+res.String()+`]`, strings.Repeat("a.", 48)+"a") + // these should fallback + testMany(true, `[Cat Nancy]`, "name\\.first", "name.first") + testMany(true, `[world]`, strings.Repeat("a.", 70)+"hello") +} + +func TestRandomMany(t *testing.T) { + var lstr string + defer func() { + if v := recover(); v != nil { + println("'" + hex.EncodeToString([]byte(lstr)) + "'") + println("'" + lstr + "'") + panic(v) + } + }() + rand.Seed(time.Now().UnixNano()) + b := make([]byte, 512) + for i := 0; i < 50000; i++ { + n, err := rand.Read(b[:rand.Int()%len(b)]) + if err != nil { + t.Fatal(err) + } + lstr = string(b[:n]) + paths := make([]string, rand.Int()%64) + for i := range paths { + var b []byte + n := rand.Int() % 5 + for j := 0; j < n; j++ { + if j > 0 { + b = append(b, '.') + } + nn := rand.Int() % 10 + for k := 0; k < nn; k++ { + b = append(b, 'a'+byte(rand.Int()%26)) + } + } + paths[i] = string(b) + } + GetMany(lstr, paths...) + } +} + type BenchStruct struct { Widget struct { Window struct { @@ -464,6 +632,19 @@ var benchPaths = []string{ "widget.text.onMouseUp", } +var benchManyPaths = []string{ + "widget.window.name", + "widget.image.hOffset", + "widget.text.onMouseUp", + "widget.window.title", + "widget.image.alignment", + "widget.text.style", + "widget.window.height", + "widget.image.src", + "widget.text.data", + "widget.text.size", +} + func BenchmarkGJSONGet(t *testing.B) { t.ReportAllocs() t.ResetTimer() @@ -476,6 +657,51 @@ func BenchmarkGJSONGet(t *testing.B) { } t.N *= len(benchPaths) // because we are running against 3 paths } +func BenchmarkGJSONGetMany4Paths(t *testing.B) { + benchmarkGJSONGetManyN(t, 4) +} +func BenchmarkGJSONGetMany8Paths(t *testing.B) { + benchmarkGJSONGetManyN(t, 8) +} +func BenchmarkGJSONGetMany16Paths(t *testing.B) { + benchmarkGJSONGetManyN(t, 16) +} +func BenchmarkGJSONGetMany32Paths(t *testing.B) { + benchmarkGJSONGetManyN(t, 32) +} +func BenchmarkGJSONGetMany64Paths(t *testing.B) { + benchmarkGJSONGetManyN(t, 64) +} +func BenchmarkGJSONGetMany128Paths(t *testing.B) { + benchmarkGJSONGetManyN(t, 128) +} +func BenchmarkGJSONGetMany256Paths(t *testing.B) { + benchmarkGJSONGetManyN(t, 256) +} +func BenchmarkGJSONGetMany512Paths(t *testing.B) { + benchmarkGJSONGetManyN(t, 512) +} +func benchmarkGJSONGetManyN(t *testing.B, n int) { + var paths []string + for len(paths) < n { + paths = append(paths, benchManyPaths...) + } + paths = paths[:n] + t.ReportAllocs() + t.ResetTimer() + for i := 0; i < t.N; i++ { + results := GetMany(exampleJSON, paths...) + if len(results) == 0 { + t.Fatal("did not find the value") + } + for j := 0; j < len(results); j++ { + if results[j].Type == Null { + t.Fatal("did not find the value") + } + } + } + t.N *= len(paths) // because we are running against 3 paths +} func BenchmarkGJSONUnmarshalMap(t *testing.B) { t.ReportAllocs() diff --git a/vendor/github.com/tidwall/sjson/LICENSE b/vendor/github.com/tidwall/sjson/LICENSE new file mode 100644 index 00000000..89593c7c --- /dev/null +++ b/vendor/github.com/tidwall/sjson/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2016 Josh Baker + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + diff --git a/vendor/github.com/tidwall/sjson/README.md b/vendor/github.com/tidwall/sjson/README.md new file mode 100644 index 00000000..1a7c5c42 --- /dev/null +++ b/vendor/github.com/tidwall/sjson/README.md @@ -0,0 +1,278 @@ +

+SJSON +
+Build Status +GoDoc +

+ +

set a json value quickly

+ +SJSON is a Go package that provides a [very fast](#performance) and simple way to set a value in a json document. The purpose for this library is to provide efficient json updating for the [SummitDB](https://github.com/tidwall/summitdb) project. +For quickly retrieving json values check out [GJSON](https://github.com/tidwall/gjson). + +For a command line interface check out [JSONed](https://github.com/tidwall/jsoned). + +Getting Started +=============== + +Installing +---------- + +To start using SJSON, install Go and run `go get`: + +```sh +$ go get -u github.com/tidwall/sjson +``` + +This will retrieve the library. + +Set a value +----------- +Set sets the value for the specified path. +A path is in dot syntax, such as "name.last" or "age". +This function expects that the json is well-formed and validated. +Invalid json will not panic, but it may return back unexpected results. +Invalid paths may return an error. + +```go +package main + +import "github.com/tidwall/sjson" + +const json = `{"name":{"first":"Janet","last":"Prichard"},"age":47}` + +func main() { + value, _ := sjson.Set(json, "name.last", "Anderson") + println(value) +} +``` + +This will print: + +```json +{"name":{"first":"Janet","last":"Anderson"},"age":47} +``` + +Path syntax +----------- + +A path is a series of keys separated by a dot. +The dot and colon characters can be escaped with '\'. + +```json +{ + "name": {"first": "Tom", "last": "Anderson"}, + "age":37, + "children": ["Sara","Alex","Jack"], + "fav.movie": "Deer Hunter", + "friends": [ + {"first": "James", "last": "Murphy"}, + {"first": "Roger", "last": "Craig"} + ] +} +``` +``` +"name.last" >> "Anderson" +"age" >> 37 +"children.1" >> "Alex" +"friends.1.last" >> "Craig" +``` + +The `-1` key can be used to append a value to an existing array: + +``` +"children.-1" >> appends a new value to the end of the children array +``` + +Normally number keys are used to modify arrays, but it's possible to force a numeric object key by using the colon character: + +```json +{ + "users":{ + "2313":{"name":"Sara"}, + "7839":{"name":"Andy"} + } +} +``` + +A colon path would look like: + +``` +"users.:2313.name" >> "Sara" +``` + +Supported types +--------------- + +Pretty much any type is supported: + +```go +sjson.Set(`{"key":true}`, "key", nil) +sjson.Set(`{"key":true}`, "key", false) +sjson.Set(`{"key":true}`, "key", 1) +sjson.Set(`{"key":true}`, "key", 10.5) +sjson.Set(`{"key":true}`, "key", "hello") +sjson.Set(`{"key":true}`, "key", map[string]interface{}{"hello":"world"}) +``` + +When a type is not recognized, SJSON will fallback to the `encoding/json` Marshaller. + + +Examples +-------- + +Set a value from empty document: +```go +value, _ := sjson.Set("", "name", "Tom") +println(value) + +// Output: +// {"name":"Tom"} +``` + +Set a nested value from empty document: +```go +value, _ := sjson.Set("", "name.last", "Anderson") +println(value) + +// Output: +// {"name":{"last":"Anderson"}} +``` + +Set a new value: +```go +value, _ := sjson.Set(`{"name":{"last":"Anderson"}}`, "name.first", "Sara") +println(value) + +// Output: +// {"name":{"first":"Sara","last":"Anderson"}} +``` + +Update an existing value: +```go +value, _ := sjson.Set(`{"name":{"last":"Anderson"}}`, "name.last", "Smith") +println(value) + +// Output: +// {"name":{"last":"Smith"}} +``` + +Set a new array value: +```go +value, _ := sjson.Set(`{"friends":["Andy","Carol"]}`, "friends.2", "Sara") +println(value) + +// Output: +// {"friends":["Andy","Carol","Sara"] +``` + +Append an array value by using the `-1` key in a path: +```go +value, _ := sjson.Set(`{"friends":["Andy","Carol"]}`, "friends.-1", "Sara") +println(value) + +// Output: +// {"friends":["Andy","Carol","Sara"] +``` + +Append an array value that is past the end: +```go +value, _ := sjson.Set(`{"friends":["Andy","Carol"]}`, "friends.4", "Sara") +println(value) + +// Output: +// {"friends":["Andy","Carol",null,null,"Sara"] +``` + +Delete a value: +```go +value, _ := sjson.Delete(`{"name":{"first":"Sara","last":"Anderson"}}`, "name.first") +println(value) + +// Output: +// {"name":{"last":"Anderson"}} +``` + +Delete an array value: +```go +value, _ := sjson.Delete(`{"friends":["Andy","Carol"]}`, "friends.1") +println(value) + +// Output: +// {"friends":["Andy"]} +``` + +Delete the last array value: +```go +value, _ := sjson.Delete(`{"friends":["Andy","Carol"]}`, "friends.-1") +println(value) + +// Output: +// {"friends":["Andy"]} +``` + +## Performance + +Benchmarks of SJSON alongside [encoding/json](https://golang.org/pkg/encoding/json/), +[ffjson](https://github.com/pquerna/ffjson), +[EasyJSON](https://github.com/mailru/easyjson), +and [Gabs](https://github.com/Jeffail/gabs) + +``` +Benchmark_SJSON-8 3000000 805 ns/op 1077 B/op 3 allocs/op +Benchmark_SJSON_ReplaceInPlace-8 3000000 449 ns/op 0 B/op 0 allocs/op +Benchmark_JSON_Map-8 300000 21236 ns/op 6392 B/op 150 allocs/op +Benchmark_JSON_Struct-8 300000 14691 ns/op 1789 B/op 24 allocs/op +Benchmark_Gabs-8 300000 21311 ns/op 6752 B/op 150 allocs/op +Benchmark_FFJSON-8 300000 17673 ns/op 3589 B/op 47 allocs/op +Benchmark_EasyJSON-8 1500000 3119 ns/op 1061 B/op 13 allocs/op +``` + +JSON document used: + +```json +{ + "widget": { + "debug": "on", + "window": { + "title": "Sample Konfabulator Widget", + "name": "main_window", + "width": 500, + "height": 500 + }, + "image": { + "src": "Images/Sun.png", + "hOffset": 250, + "vOffset": 250, + "alignment": "center" + }, + "text": { + "data": "Click Here", + "size": 36, + "style": "bold", + "vOffset": 100, + "alignment": "center", + "onMouseUp": "sun1.opacity = (sun1.opacity / 100) * 90;" + } + } +} +``` + +Each operation was rotated though one of the following search paths: + +``` +widget.window.name +widget.image.hOffset +widget.text.onMouseUp +``` + +*These benchmarks were run on a MacBook Pro 15" 2.8 GHz Intel Core i7 using Go 1.7.* + +## Contact +Josh Baker [@tidwall](http://twitter.com/tidwall) + +## License + +SJSON source code is available under the MIT [License](/LICENSE). diff --git a/vendor/github.com/tidwall/sjson/logo.png b/vendor/github.com/tidwall/sjson/logo.png new file mode 100644 index 00000000..b5aa257b Binary files /dev/null and b/vendor/github.com/tidwall/sjson/logo.png differ diff --git a/vendor/github.com/tidwall/sjson/sjson.go b/vendor/github.com/tidwall/sjson/sjson.go new file mode 100644 index 00000000..7f1d3588 --- /dev/null +++ b/vendor/github.com/tidwall/sjson/sjson.go @@ -0,0 +1,653 @@ +// Package sjson provides setting json values. +package sjson + +import ( + jsongo "encoding/json" + "reflect" + "strconv" + "unsafe" + + "github.com/tidwall/gjson" +) + +type errorType struct { + msg string +} + +func (err *errorType) Error() string { + return err.msg +} + +// Options represents additional options for the Set and Delete functions. +type Options struct { + // Optimistic is a hint that the value likely exists which + // allows for the sjson to perform a fast-track search and replace. + Optimistic bool + // ReplaceInPlace is a hint to replace the input json rather than + // allocate a new json byte slice. When this field is specified + // the input json will not longer be valid and it should not be used + // In the case when the destination slice doesn't have enough free + // bytes to replace the data in place, a new bytes slice will be + // created under the hood. + // The Optimistic flag must be set to true and the input must be a + // byte slice in order to use this field. + ReplaceInPlace bool +} + +type pathResult struct { + part string // current key part + path string // remaining path + force bool // force a string key + more bool // there is more path to parse +} + +func parsePath(path string) (pathResult, error) { + var r pathResult + if len(path) > 0 && path[0] == ':' { + r.force = true + path = path[1:] + } + for i := 0; i < len(path); i++ { + if path[i] == '.' { + r.part = path[:i] + r.path = path[i+1:] + r.more = true + return r, nil + } + if path[i] == '*' || path[i] == '?' { + return r, &errorType{"wildcard characters not allowed in path"} + } else if path[i] == '#' { + return r, &errorType{"array access character not allowed in path"} + } + if path[i] == '\\' { + // go into escape mode. this is a slower path that + // strips off the escape character from the part. + epart := []byte(path[:i]) + i++ + if i < len(path) { + epart = append(epart, path[i]) + i++ + for ; i < len(path); i++ { + if path[i] == '\\' { + i++ + if i < len(path) { + epart = append(epart, path[i]) + } + continue + } else if path[i] == '.' { + r.part = string(epart) + r.path = path[i+1:] + r.more = true + return r, nil + } else if path[i] == '*' || path[i] == '?' { + return r, &errorType{ + "wildcard characters not allowed in path"} + } else if path[i] == '#' { + return r, &errorType{ + "array access character not allowed in path"} + } + epart = append(epart, path[i]) + } + } + // append the last part + r.part = string(epart) + return r, nil + } + } + r.part = path + return r, nil +} + +func mustMarshalString(s string) bool { + for i := 0; i < len(s); i++ { + if s[i] < ' ' || s[i] > 0x7f || s[i] == '"' { + return true + } + } + return false +} + +// appendStringify makes a json string and appends to buf. +func appendStringify(buf []byte, s string) []byte { + if mustMarshalString(s) { + b, _ := jsongo.Marshal(s) + return append(buf, b...) + } + buf = append(buf, '"') + buf = append(buf, s...) + buf = append(buf, '"') + return buf +} + +// appendBuild builds a json block from a json path. +func appendBuild(buf []byte, array bool, paths []pathResult, raw string, + stringify bool) []byte { + if !array { + buf = appendStringify(buf, paths[0].part) + buf = append(buf, ':') + } + if len(paths) > 1 { + n, numeric := atoui(paths[1]) + if numeric || (!paths[1].force && paths[1].part == "-1") { + buf = append(buf, '[') + buf = appendRepeat(buf, "null,", n) + buf = appendBuild(buf, true, paths[1:], raw, stringify) + buf = append(buf, ']') + } else { + buf = append(buf, '{') + buf = appendBuild(buf, false, paths[1:], raw, stringify) + buf = append(buf, '}') + } + } else { + if stringify { + buf = appendStringify(buf, raw) + } else { + buf = append(buf, raw...) + } + } + return buf +} + +// atoui does a rip conversion of string -> unigned int. +func atoui(r pathResult) (n int, ok bool) { + if r.force { + return 0, false + } + for i := 0; i < len(r.part); i++ { + if r.part[i] < '0' || r.part[i] > '9' { + return 0, false + } + n = n*10 + int(r.part[i]-'0') + } + return n, true +} + +// appendRepeat repeats string "n" times and appends to buf. +func appendRepeat(buf []byte, s string, n int) []byte { + for i := 0; i < n; i++ { + buf = append(buf, s...) + } + return buf +} + +// trim does a rip trim +func trim(s string) string { + for len(s) > 0 { + if s[0] <= ' ' { + s = s[1:] + continue + } + break + } + for len(s) > 0 { + if s[len(s)-1] <= ' ' { + s = s[:len(s)-1] + continue + } + break + } + return s +} + +// deleteTailItem deletes the previous key or comma. +func deleteTailItem(buf []byte) ([]byte, bool) { +loop: + for i := len(buf) - 1; i >= 0; i-- { + // look for either a ',',':','[' + switch buf[i] { + case '[': + return buf, true + case ',': + return buf[:i], false + case ':': + // delete tail string + i-- + for ; i >= 0; i-- { + if buf[i] == '"' { + i-- + for ; i >= 0; i-- { + if buf[i] == '"' { + i-- + if i >= 0 && i == '\\' { + i-- + continue + } + for ; i >= 0; i-- { + // look for either a ',','{' + switch buf[i] { + case '{': + return buf[:i+1], true + case ',': + return buf[:i], false + } + } + } + } + break + } + } + break loop + } + } + return buf, false +} + +var errNoChange = &errorType{"no change"} + +func appendRawPaths(buf []byte, jstr string, paths []pathResult, raw string, + stringify, del bool) ([]byte, error) { + var err error + var res gjson.Result + var found bool + if del { + if paths[0].part == "-1" && !paths[0].force { + res = gjson.Get(jstr, "#") + if res.Int() > 0 { + res = gjson.Get(jstr, strconv.FormatInt(int64(res.Int()-1), 10)) + found = true + } + } + } + if !found { + res = gjson.Get(jstr, paths[0].part) + } + if res.Index > 0 { + if len(paths) > 1 { + buf = append(buf, jstr[:res.Index]...) + buf, err = appendRawPaths(buf, res.Raw, paths[1:], raw, + stringify, del) + if err != nil { + return nil, err + } + buf = append(buf, jstr[res.Index+len(res.Raw):]...) + return buf, nil + } + buf = append(buf, jstr[:res.Index]...) + var exidx int // additional forward stripping + if del { + var delNextComma bool + buf, delNextComma = deleteTailItem(buf) + if delNextComma { + i, j := res.Index+len(res.Raw), 0 + for ; i < len(jstr); i, j = i+1, j+1 { + if jstr[i] <= ' ' { + continue + } + if jstr[i] == ',' { + exidx = j + 1 + } + break + } + } + } else { + if stringify { + buf = appendStringify(buf, raw) + } else { + buf = append(buf, raw...) + } + } + buf = append(buf, jstr[res.Index+len(res.Raw)+exidx:]...) + return buf, nil + } + if del { + return nil, errNoChange + } + n, numeric := atoui(paths[0]) + isempty := true + for i := 0; i < len(jstr); i++ { + if jstr[i] > ' ' { + isempty = false + break + } + } + if isempty { + if numeric { + jstr = "[]" + } else { + jstr = "{}" + } + } + jsres := gjson.Parse(jstr) + if jsres.Type != gjson.JSON { + if numeric { + jstr = "[]" + } else { + jstr = "{}" + } + jsres = gjson.Parse(jstr) + } + var comma bool + for i := 1; i < len(jsres.Raw); i++ { + if jsres.Raw[i] <= ' ' { + continue + } + if jsres.Raw[i] == '}' || jsres.Raw[i] == ']' { + break + } + comma = true + break + } + switch jsres.Raw[0] { + default: + return nil, &errorType{"json must be an object or array"} + case '{': + buf = append(buf, '{') + buf = appendBuild(buf, false, paths, raw, stringify) + if comma { + buf = append(buf, ',') + } + buf = append(buf, jsres.Raw[1:]...) + return buf, nil + case '[': + var appendit bool + if !numeric { + if paths[0].part == "-1" && !paths[0].force { + appendit = true + } else { + return nil, &errorType{ + "cannot set array element for non-numeric key '" + + paths[0].part + "'"} + } + } + if appendit { + njson := trim(jsres.Raw) + if njson[len(njson)-1] == ']' { + njson = njson[:len(njson)-1] + } + buf = append(buf, njson...) + if comma { + buf = append(buf, ',') + } + + buf = appendBuild(buf, true, paths, raw, stringify) + buf = append(buf, ']') + return buf, nil + } + buf = append(buf, '[') + ress := jsres.Array() + for i := 0; i < len(ress); i++ { + if i > 0 { + buf = append(buf, ',') + } + buf = append(buf, ress[i].Raw...) + } + if len(ress) == 0 { + buf = appendRepeat(buf, "null,", n-len(ress)) + } else { + buf = appendRepeat(buf, ",null", n-len(ress)) + if comma { + buf = append(buf, ',') + } + } + buf = appendBuild(buf, true, paths, raw, stringify) + buf = append(buf, ']') + return buf, nil + } +} + +func isOptimisticPath(path string) bool { + for i := 0; i < len(path); i++ { + if path[i] < '.' || path[i] > 'z' { + return false + } + if path[i] > '9' && path[i] < 'A' { + return false + } + if path[i] > 'z' { + return false + } + } + return true +} + +func set(jstr, path, raw string, + stringify, del, optimistic, inplace bool) ([]byte, error) { + if path == "" { + return nil, &errorType{"path cannot be empty"} + } + if !del && optimistic && isOptimisticPath(path) { + res := gjson.Get(jstr, path) + if res.Exists() && res.Index > 0 { + sz := len(jstr) - len(res.Raw) + len(raw) + if stringify { + sz += 2 + } + if inplace && sz <= len(jstr) { + if !stringify || !mustMarshalString(raw) { + jsonh := *(*reflect.StringHeader)(unsafe.Pointer(&jstr)) + jsonbh := reflect.SliceHeader{ + Data: jsonh.Data, Len: jsonh.Len, Cap: jsonh.Len} + jbytes := *(*[]byte)(unsafe.Pointer(&jsonbh)) + if stringify { + jbytes[res.Index] = '"' + copy(jbytes[res.Index+1:], []byte(raw)) + jbytes[res.Index+1+len(raw)] = '"' + copy(jbytes[res.Index+1+len(raw)+1:], + jbytes[res.Index+len(res.Raw):]) + } else { + copy(jbytes[res.Index:], []byte(raw)) + copy(jbytes[res.Index+len(raw):], + jbytes[res.Index+len(res.Raw):]) + } + return jbytes[:sz], nil + } + return nil, nil + } + buf := make([]byte, 0, sz) + buf = append(buf, jstr[:res.Index]...) + if stringify { + buf = appendStringify(buf, raw) + } else { + buf = append(buf, raw...) + } + buf = append(buf, jstr[res.Index+len(res.Raw):]...) + return buf, nil + } + } + // parse the path, make sure that it does not contain invalid characters + // such as '#', '?', '*' + paths := make([]pathResult, 0, 4) + r, err := parsePath(path) + if err != nil { + return nil, err + } + paths = append(paths, r) + for r.more { + if r, err = parsePath(r.path); err != nil { + return nil, err + } + paths = append(paths, r) + } + + njson, err := appendRawPaths(nil, jstr, paths, raw, stringify, del) + if err != nil { + return nil, err + } + return njson, nil +} + +// Set sets a json value for the specified path. +// A path is in dot syntax, such as "name.last" or "age". +// This function expects that the json is well-formed, and does not validate. +// Invalid json will not panic, but it may return back unexpected results. +// An error is returned if the path is not valid. +// +// A path is a series of keys separated by a dot. +// +// { +// "name": {"first": "Tom", "last": "Anderson"}, +// "age":37, +// "children": ["Sara","Alex","Jack"], +// "friends": [ +// {"first": "James", "last": "Murphy"}, +// {"first": "Roger", "last": "Craig"} +// ] +// } +// "name.last" >> "Anderson" +// "age" >> 37 +// "children.1" >> "Alex" +// +func Set(json, path string, value interface{}) (string, error) { + return SetOptions(json, path, value, nil) +} + +// SetOptions sets a json value for the specified path with options. +// A path is in dot syntax, such as "name.last" or "age". +// This function expects that the json is well-formed, and does not validate. +// Invalid json will not panic, but it may return back unexpected results. +// An error is returned if the path is not valid. +func SetOptions(json, path string, value interface{}, + opts *Options) (string, error) { + if opts != nil { + if opts.ReplaceInPlace { + // it's not safe to replace bytes in-place for strings + // copy the Options and set options.ReplaceInPlace to false. + nopts := *opts + opts = &nopts + opts.ReplaceInPlace = false + } + } + jsonh := *(*reflect.StringHeader)(unsafe.Pointer(&json)) + jsonbh := reflect.SliceHeader{Data: jsonh.Data, Len: jsonh.Len} + jsonb := *(*[]byte)(unsafe.Pointer(&jsonbh)) + res, err := SetBytesOptions(jsonb, path, value, opts) + return string(res), err +} + +// SetBytes sets a json value for the specified path. +// If working with bytes, this method preferred over +// Set(string(data), path, value) +func SetBytes(json []byte, path string, value interface{}) ([]byte, error) { + return SetBytesOptions(json, path, value, nil) +} + +// SetBytesOptions sets a json value for the specified path with options. +// If working with bytes, this method preferred over +// SetOptions(string(data), path, value) +func SetBytesOptions(json []byte, path string, value interface{}, + opts *Options) ([]byte, error) { + var optimistic, inplace bool + if opts != nil { + optimistic = opts.Optimistic + inplace = opts.ReplaceInPlace + } + jstr := *(*string)(unsafe.Pointer(&json)) + var res []byte + var err error + switch v := value.(type) { + default: + b, err := jsongo.Marshal(value) + if err != nil { + return nil, err + } + raw := *(*string)(unsafe.Pointer(&b)) + res, err = set(jstr, path, raw, false, false, optimistic, inplace) + case dtype: + res, err = set(jstr, path, "", false, true, optimistic, inplace) + case string: + res, err = set(jstr, path, v, true, false, optimistic, inplace) + case []byte: + raw := *(*string)(unsafe.Pointer(&v)) + res, err = set(jstr, path, raw, true, false, optimistic, inplace) + case bool: + if v { + res, err = set(jstr, path, "true", false, false, optimistic, inplace) + } else { + res, err = set(jstr, path, "false", false, false, optimistic, inplace) + } + case int8: + res, err = set(jstr, path, strconv.FormatInt(int64(v), 10), + false, false, optimistic, inplace) + case int16: + res, err = set(jstr, path, strconv.FormatInt(int64(v), 10), + false, false, optimistic, inplace) + case int32: + res, err = set(jstr, path, strconv.FormatInt(int64(v), 10), + false, false, optimistic, inplace) + case int64: + res, err = set(jstr, path, strconv.FormatInt(int64(v), 10), + false, false, optimistic, inplace) + case uint8: + res, err = set(jstr, path, strconv.FormatUint(uint64(v), 10), + false, false, optimistic, inplace) + case uint16: + res, err = set(jstr, path, strconv.FormatUint(uint64(v), 10), + false, false, optimistic, inplace) + case uint32: + res, err = set(jstr, path, strconv.FormatUint(uint64(v), 10), + false, false, optimistic, inplace) + case uint64: + res, err = set(jstr, path, strconv.FormatUint(uint64(v), 10), + false, false, optimistic, inplace) + case float32: + res, err = set(jstr, path, strconv.FormatFloat(float64(v), 'f', -1, 64), + false, false, optimistic, inplace) + case float64: + res, err = set(jstr, path, strconv.FormatFloat(float64(v), 'f', -1, 64), + false, false, optimistic, inplace) + } + if err == errNoChange { + return json, nil + } + return res, err +} + +// SetRaw sets a raw json value for the specified path. +// This function works the same as Set except that the value is set as a +// raw block of json. This allows for setting premarshalled json objects. +func SetRaw(json, path, value string) (string, error) { + return SetRawOptions(json, path, value, nil) +} + +// SetRawOptions sets a raw json value for the specified path with options. +// This furnction works the same as SetOptions except that the value is set +// as a raw block of json. This allows for setting premarshalled json objects. +func SetRawOptions(json, path, value string, opts *Options) (string, error) { + var optimistic bool + if opts != nil { + optimistic = opts.Optimistic + } + res, err := set(json, path, value, false, false, optimistic, false) + if err == errNoChange { + return json, nil + } + return string(res), err +} + +// SetRawBytes sets a raw json value for the specified path. +// If working with bytes, this method preferred over +// SetRaw(string(data), path, value) +func SetRawBytes(json []byte, path string, value []byte) ([]byte, error) { + return SetRawBytesOptions(json, path, value, nil) +} + +// SetRawBytesOptions sets a raw json value for the specified path with options. +// If working with bytes, this method preferred over +// SetRawOptions(string(data), path, value, opts) +func SetRawBytesOptions(json []byte, path string, value []byte, + opts *Options) ([]byte, error) { + jstr := *(*string)(unsafe.Pointer(&json)) + vstr := *(*string)(unsafe.Pointer(&value)) + var optimistic, inplace bool + if opts != nil { + optimistic = opts.Optimistic + inplace = opts.ReplaceInPlace + } + res, err := set(jstr, path, vstr, false, false, optimistic, inplace) + if err == errNoChange { + return json, nil + } + return res, err +} + +type dtype struct{} + +// Delete deletes a value from json for the specified path. +func Delete(json, path string) (string, error) { + return Set(json, path, dtype{}) +} + +// DeleteBytes deletes a value from json for the specified path. +func DeleteBytes(json []byte, path string) ([]byte, error) { + return SetBytes(json, path, dtype{}) +} diff --git a/vendor/github.com/tidwall/sjson/sjson_test.go b/vendor/github.com/tidwall/sjson/sjson_test.go new file mode 100644 index 00000000..a7a8f7e1 --- /dev/null +++ b/vendor/github.com/tidwall/sjson/sjson_test.go @@ -0,0 +1,1239 @@ +package sjson + +import ( + "bytes" + "encoding/hex" + gojson "encoding/json" + "fmt" + "math/rand" + "strings" + "testing" + "time" + + "github.com/Jeffail/gabs" + + jlexer "github.com/mailru/easyjson/jlexer" + jwriter "github.com/mailru/easyjson/jwriter" + fflib "github.com/pquerna/ffjson/fflib/v1" +) + +func TestInvalidPaths(t *testing.T) { + var err error + _, err = SetRaw(`{"hello":"world"}`, "", `"planet"`) + if err == nil || err.Error() != "path cannot be empty" { + t.Fatalf("expecting '%v', got '%v'", "path cannot be empty", err) + } + _, err = SetRaw("", "name.last.#", "") + if err == nil || err.Error() != "array access character not allowed in path" { + t.Fatalf("expecting '%v', got '%v'", "array access character not allowed in path", err) + } + _, err = SetRaw("", "name.last.\\1#", "") + if err == nil || err.Error() != "array access character not allowed in path" { + t.Fatalf("expecting '%v', got '%v'", "array access character not allowed in path", err) + } + _, err = SetRaw("", "name.las?t", "") + if err == nil || err.Error() != "wildcard characters not allowed in path" { + t.Fatalf("expecting '%v', got '%v'", "wildcard characters not allowed in path", err) + } + _, err = SetRaw("", "name.la\\s?t", "") + if err == nil || err.Error() != "wildcard characters not allowed in path" { + t.Fatalf("expecting '%v', got '%v'", "wildcard characters not allowed in path", err) + } + _, err = SetRaw("", "name.las*t", "") + if err == nil || err.Error() != "wildcard characters not allowed in path" { + t.Fatalf("expecting '%v', got '%v'", "wildcard characters not allowed in path", err) + } + _, err = SetRaw("", "name.las\\a*t", "") + if err == nil || err.Error() != "wildcard characters not allowed in path" { + t.Fatalf("expecting '%v', got '%v'", "wildcard characters not allowed in path", err) + } +} + +const ( + setRaw = 1 + setBool = 2 + setInt = 3 + setFloat = 4 + setString = 5 + setDelete = 6 +) + +func testRaw(t *testing.T, kind int, expect, json, path string, value interface{}) { + var json2 string + var err error + switch kind { + default: + json2, err = Set(json, path, value) + case setRaw: + json2, err = SetRaw(json, path, value.(string)) + case setDelete: + json2, err = Delete(json, path) + } + if err != nil { + t.Fatal(err) + } else if json2 != expect { + t.Fatalf("expected '%v', got '%v'", expect, json2) + } + + var json3 []byte + switch kind { + default: + json3, err = SetBytes([]byte(json), path, value) + case setRaw: + json3, err = SetRawBytes([]byte(json), path, []byte(value.(string))) + case setDelete: + json3, err = DeleteBytes([]byte(json), path) + } + if err != nil { + t.Fatal(err) + } else if string(json3) != expect { + t.Fatalf("expected '%v', got '%v'", expect, string(json3)) + } +} +func TestBasic(t *testing.T) { + testRaw(t, setRaw, `[{"hiw":"planet","hi":"world"}]`, `[{"hi":"world"}]`, "0.hiw", `"planet"`) + testRaw(t, setRaw, `[true]`, ``, "0", `true`) + testRaw(t, setRaw, `[null,true]`, ``, "1", `true`) + testRaw(t, setRaw, `[1,null,true]`, `[1]`, "2", `true`) + testRaw(t, setRaw, `[1,true,false]`, `[1,null,false]`, "1", `true`) + testRaw(t, setRaw, + `[1,{"hello":"when","this":[0,null,2]},false]`, + `[1,{"hello":"when","this":[0,1,2]},false]`, + "1.this.1", `null`) + testRaw(t, setRaw, + `{"a":1,"b":{"hello":"when","this":[0,null,2]},"c":false}`, + `{"a":1,"b":{"hello":"when","this":[0,1,2]},"c":false}`, + "b.this.1", `null`) + testRaw(t, setRaw, + `{"a":1,"b":{"hello":"when","this":[0,null,2,null,4]},"c":false}`, + `{"a":1,"b":{"hello":"when","this":[0,null,2]},"c":false}`, + "b.this.4", `4`) + testRaw(t, setRaw, + `{"b":{"this":[null,null,null,null,4]}}`, + ``, + "b.this.4", `4`) + testRaw(t, setRaw, + `[null,{"this":[null,null,null,null,4]}]`, + ``, + "1.this.4", `4`) + testRaw(t, setRaw, + `{"1":{"this":[null,null,null,null,4]}}`, + ``, + ":1.this.4", `4`) + testRaw(t, setRaw, + `{":1":{"this":[null,null,null,null,4]}}`, + ``, + "\\:1.this.4", `4`) + testRaw(t, setRaw, + `{":\1":{"this":[null,null,null,null,{".HI":4}]}}`, + ``, + "\\:\\\\1.this.4.\\.HI", `4`) + testRaw(t, setRaw, + `{"b":{"this":{"😇":""}}}`, + ``, + "b.this.😇", `""`) + testRaw(t, setRaw, + `[ 1,2 ,3]`, + ` [ 1,2 ] `, + "-1", `3`) + testRaw(t, setInt, `[1234]`, ``, `0`, int64(1234)) + testRaw(t, setFloat, `[1234.5]`, ``, `0`, float64(1234.5)) + testRaw(t, setString, `["1234.5"]`, ``, `0`, "1234.5") + testRaw(t, setBool, `[true]`, ``, `0`, true) + testRaw(t, setBool, `[null]`, ``, `0`, nil) + testRaw(t, setString, `{"arr":[1]}`, ``, `arr.-1`, 1) +} + +func TestDelete(t *testing.T) { + testRaw(t, setDelete, `[456]`, `[123,456]`, `0`, nil) + testRaw(t, setDelete, `[123,789]`, `[123,456,789]`, `1`, nil) + testRaw(t, setDelete, `[123,456]`, `[123,456,789]`, `-1`, nil) + testRaw(t, setDelete, `{"a":[123,456]}`, `{"a":[123,456,789]}`, `a.-1`, nil) + testRaw(t, setDelete, `{"and":"another"}`, `{"this":"that","and":"another"}`, `this`, nil) + testRaw(t, setDelete, `{"this":"that"}`, `{"this":"that","and":"another"}`, `and`, nil) + testRaw(t, setDelete, `{}`, `{"and":"another"}`, `and`, nil) + testRaw(t, setDelete, `{"1":"2"}`, `{"1":"2"}`, `3`, nil) +} + +// TestRandomData is a fuzzing test that throws random data at SetRaw +// function looking for panics. +func TestRandomData(t *testing.T) { + var lstr string + defer func() { + if v := recover(); v != nil { + println("'" + hex.EncodeToString([]byte(lstr)) + "'") + println("'" + lstr + "'") + panic(v) + } + }() + rand.Seed(time.Now().UnixNano()) + b := make([]byte, 200) + for i := 0; i < 2000000; i++ { + n, err := rand.Read(b[:rand.Int()%len(b)]) + if err != nil { + t.Fatal(err) + } + lstr = string(b[:n]) + SetRaw(lstr, "zzzz.zzzz.zzzz", "123") + } +} + +var exampleJSON = ` +{ + "sha": "d25341478381063d1c76e81b3a52e0592a7c997f", + "commit": { + "author": { + "name": "Tom Tom Anderson", + "email": "tomtom@anderson.edu", + "date": "2013-06-22T16:30:59Z" + }, + "committer": { + "name": "Tom Tom Anderson", + "email": "jeffditto@anderson.edu", + "date": "2013-06-22T16:30:59Z" + }, + "message": "Merge pull request #162 from stedolan/utf8-fixes\n\nUtf8 fixes. Closes #161", + "tree": { + "sha": "6ab697a8dfb5a96e124666bf6d6213822599fb40", + "url": "https://api.github.com/repos/stedolan/jq/git/trees/6ab697a8dfb5a96e124666bf6d6213822599fb40" + }, + "url": "https://api.github.com/repos/stedolan/jq/git/commits/d25341478381063d1c76e81b3a52e0592a7c997f", + "comment_count": 0 + } +} +` +var path = "commit.committer.email" +var value = "tomtom@anderson.com" +var rawValue = `"tomtom@anderson.com"` +var rawValueBytes = []byte(rawValue) +var expect = strings.Replace(exampleJSON, "jeffditto@anderson.edu", "tomtom@anderson.com", 1) +var jsonBytes = []byte(exampleJSON) +var jsonBytes2 = []byte(exampleJSON) +var expectBytes = []byte(expect) +var opts = &Options{Optimistic: true} +var optsInPlace = &Options{Optimistic: true, ReplaceInPlace: true} + +func BenchmarkSet(t *testing.B) { + t.ReportAllocs() + for i := 0; i < t.N; i++ { + res, err := Set(exampleJSON, path, value) + if err != nil { + t.Fatal(err) + } + if res != expect { + t.Fatal("expected '%v', got '%v'", expect, res) + } + } +} + +func BenchmarkSetRaw(t *testing.B) { + t.ReportAllocs() + for i := 0; i < t.N; i++ { + res, err := SetRaw(exampleJSON, path, rawValue) + if err != nil { + t.Fatal(err) + } + if res != expect { + t.Fatal("expected '%v', got '%v'", expect, res) + } + } +} + +func BenchmarkSetBytes(t *testing.B) { + t.ReportAllocs() + for i := 0; i < t.N; i++ { + res, err := SetBytes(jsonBytes, path, value) + if err != nil { + t.Fatal(err) + } + if bytes.Compare(res, expectBytes) != 0 { + t.Fatal("expected '%v', got '%v'", expect, res) + } + } +} + +func BenchmarkSetRawBytes(t *testing.B) { + t.ReportAllocs() + for i := 0; i < t.N; i++ { + res, err := SetRawBytes(jsonBytes, path, rawValueBytes) + if err != nil { + t.Fatal(err) + } + if bytes.Compare(res, expectBytes) != 0 { + t.Fatal("expected '%v', got '%v'", expect, res) + } + } +} + +func BenchmarkSetOptimistic(t *testing.B) { + t.ReportAllocs() + for i := 0; i < t.N; i++ { + res, err := SetOptions(exampleJSON, path, value, opts) + if err != nil { + t.Fatal(err) + } + if res != expect { + t.Fatal("expected '%v', got '%v'", expect, res) + } + } +} + +func BenchmarkSetInPlace(t *testing.B) { + t.ReportAllocs() + for i := 0; i < t.N; i++ { + res, err := SetOptions(exampleJSON, path, value, optsInPlace) + if err != nil { + t.Fatal(err) + } + if res != expect { + t.Fatal("expected '%v', got '%v'", expect, res) + } + } +} + +func BenchmarkSetRawOptimistic(t *testing.B) { + t.ReportAllocs() + for i := 0; i < t.N; i++ { + res, err := SetRawOptions(exampleJSON, path, rawValue, opts) + if err != nil { + t.Fatal(err) + } + if res != expect { + t.Fatal("expected '%v', got '%v'", expect, res) + } + } +} + +func BenchmarkSetRawInPlace(t *testing.B) { + t.ReportAllocs() + for i := 0; i < t.N; i++ { + res, err := SetRawOptions(exampleJSON, path, rawValue, optsInPlace) + if err != nil { + t.Fatal(err) + } + if res != expect { + t.Fatal("expected '%v', got '%v'", expect, res) + } + } +} + +func BenchmarkSetBytesOptimistic(t *testing.B) { + t.ReportAllocs() + for i := 0; i < t.N; i++ { + res, err := SetBytesOptions(jsonBytes, path, value, opts) + if err != nil { + t.Fatal(err) + } + if bytes.Compare(res, expectBytes) != 0 { + t.Fatal("expected '%v', got '%v'", string(expectBytes), string(res)) + } + } +} + +func BenchmarkSetBytesInPlace(t *testing.B) { + t.ReportAllocs() + for i := 0; i < t.N; i++ { + copy(jsonBytes2, jsonBytes) + res, err := SetBytesOptions(jsonBytes2, path, value, optsInPlace) + if err != nil { + t.Fatal(err) + } + if bytes.Compare(res, expectBytes) != 0 { + t.Fatal("expected '%v', got '%v'", string(expectBytes), string(res)) + } + } +} + +func BenchmarkSetRawBytesOptimistic(t *testing.B) { + t.ReportAllocs() + for i := 0; i < t.N; i++ { + res, err := SetRawBytesOptions(jsonBytes, path, rawValueBytes, opts) + if err != nil { + t.Fatal(err) + } + if bytes.Compare(res, expectBytes) != 0 { + t.Fatal("expected '%v', got '%v'", string(expectBytes), string(res)) + } + } +} + +func BenchmarkSetRawBytesInPlace(t *testing.B) { + t.ReportAllocs() + for i := 0; i < t.N; i++ { + copy(jsonBytes2, jsonBytes) + res, err := SetRawBytesOptions(jsonBytes2, path, rawValueBytes, optsInPlace) + if err != nil { + t.Fatal(err) + } + if bytes.Compare(res, expectBytes) != 0 { + t.Fatal("expected '%v', got '%v'", string(expectBytes), string(res)) + } + } +} + +const benchJSON = ` +{ + "widget": { + "debug": "on", + "window": { + "title": "Sample Konfabulator Widget", + "name": "main_window", + "width": 500, + "height": 500 + }, + "image": { + "src": "Images/Sun.png", + "hOffset": 250, + "vOffset": 250, + "alignment": "center" + }, + "text": { + "data": "Click Here", + "size": 36, + "style": "bold", + "vOffset": 100, + "alignment": "center", + "onMouseUp": "sun1.opacity = (sun1.opacity / 100) * 90;" + } + } +} +` + +type BenchStruct struct { + Widget struct { + Debug string `json:"debug"` + Window struct { + Title string `json:"title"` + Name string `json:"name"` + Width int `json:"width"` + Height int `json:"height"` + } `json:"window"` + Image struct { + Src string `json:"src"` + HOffset int `json:"hOffset"` + VOffset int `json:"vOffset"` + Alignment string `json:"alignment"` + } `json:"image"` + Text struct { + Data string `json:"data"` + Size int `json:"size"` + Style string `json:"style"` + VOffset int `json:"vOffset"` + Alignment string `json:"alignment"` + OnMouseUp string `json:"onMouseUp"` + } `json:"text"` + } `json:"widget"` +} + +var benchPaths = []string{ + "widget.window.name", + "widget.image.hOffset", + "widget.text.onMouseUp", +} + +func Benchmark_SJSON(t *testing.B) { + opts := Options{Optimistic: true} + t.ReportAllocs() + t.ResetTimer() + for i := 0; i < t.N; i++ { + for _, path := range benchPaths { + var err error + switch path { + case "widget.window.name": + _, err = SetOptions(benchJSON, path, "1", &opts) + case "widget.image.hOffset": + _, err = SetOptions(benchJSON, path, 1, &opts) + case "widget.text.onMouseUp": + _, err = SetOptions(benchJSON, path, "1", &opts) + } + if err != nil { + t.Fatal(err) + } + + } + } + t.N *= len(benchPaths) +} + +func Benchmark_SJSON_ReplaceInPlace(t *testing.B) { + data := []byte(benchJSON) + opts := Options{ + Optimistic: true, + ReplaceInPlace: true, + } + v1, v2 := []byte(`"1"`), []byte("1") + t.ReportAllocs() + t.ResetTimer() + for i := 0; i < t.N; i++ { + for _, path := range benchPaths { + var err error + switch path { + case "widget.window.name": + _, err = SetRawBytesOptions(data, path, v1, &opts) + case "widget.image.hOffset": + _, err = SetRawBytesOptions(data, path, v2, &opts) + case "widget.text.onMouseUp": + _, err = SetRawBytesOptions(data, path, v1, &opts) + } + if err != nil { + t.Fatal(err) + } + + } + } + t.N *= len(benchPaths) +} + +func Benchmark_Encoding_JSON_Map(t *testing.B) { + data := []byte(benchJSON) + t.ReportAllocs() + t.ResetTimer() + for i := 0; i < t.N; i++ { + for _, path := range benchPaths { + var m map[string]interface{} + if err := gojson.Unmarshal(data, &m); err != nil { + t.Fatal(err) + } + switch path { + case "widget.window.name": + m["widget"].(map[string]interface{})["window"].(map[string]interface{})["name"] = "1" + case "widget.image.hOffset": + m["widget"].(map[string]interface{})["image"].(map[string]interface{})["hOffset"] = 1 + case "widget.text.onMouseUp": + m["widget"].(map[string]interface{})["text"].(map[string]interface{})["onMouseUp"] = "1" + } + _, err := gojson.Marshal(&m) + if err != nil { + t.Fatal(err) + } + } + } + t.N *= len(benchPaths) +} + +func Benchmark_Encoding_JSON_Struct(t *testing.B) { + data := []byte(benchJSON) + t.ReportAllocs() + t.ResetTimer() + for i := 0; i < t.N; i++ { + for _, path := range benchPaths { + var v BenchStruct + if err := gojson.Unmarshal(data, &v); err != nil { + t.Fatal(err) + } + switch path { + case "widget.window.name": + v.Widget.Window.Name = "1" + case "widget.image.hOffset": + v.Widget.Image.HOffset = 1 + case "widget.text.onMouseUp": + v.Widget.Text.OnMouseUp = "1" + } + _, err := gojson.Marshal(&v) + if err != nil { + t.Fatal(err) + } + } + } + t.N *= len(benchPaths) +} + +func Benchmark_Gabs(t *testing.B) { + data := []byte(benchJSON) + t.ReportAllocs() + t.ResetTimer() + for i := 0; i < t.N; i++ { + for _, path := range benchPaths { + jsonParsed, err := gabs.ParseJSON(data) + if err != nil { + t.Fatal(err) + } + switch path { + case "widget.window.name": + jsonParsed.SetP("1", path) + case "widget.image.hOffset": + jsonParsed.SetP(1, path) + case "widget.text.onMouseUp": + jsonParsed.SetP("1", path) + } + jsonParsed.String() + } + } + t.N *= len(benchPaths) +} + +func Benchmark_FFJSON(t *testing.B) { + data := []byte(benchJSON) + t.ReportAllocs() + t.ResetTimer() + for i := 0; i < t.N; i++ { + for _, path := range benchPaths { + var v BenchStruct + if err := v.UnmarshalFFJSONFromData(data); err != nil { + t.Fatal(err) + } + switch path { + case "widget.window.name": + v.Widget.Window.Name = "1" + case "widget.image.hOffset": + v.Widget.Image.HOffset = 1 + case "widget.text.onMouseUp": + v.Widget.Text.OnMouseUp = "1" + } + _, err := v.MarshalFFJSONFromData() + if err != nil { + t.Fatal(err) + } + } + } + t.N *= len(benchPaths) +} + +func Benchmark_EasyJSON(t *testing.B) { + data := []byte(benchJSON) + t.ReportAllocs() + t.ResetTimer() + for i := 0; i < t.N; i++ { + for _, path := range benchPaths { + var v BenchStruct + if err := v.UnmarshalEasyJSONFromData(data); err != nil { + t.Fatal(err) + } + switch path { + case "widget.window.name": + v.Widget.Window.Name = "1" + case "widget.image.hOffset": + v.Widget.Image.HOffset = 1 + case "widget.text.onMouseUp": + v.Widget.Text.OnMouseUp = "1" + } + _, err := v.MarshalEasyJSONFromData() + if err != nil { + t.Fatal(err) + } + } + } + t.N *= len(benchPaths) +} + +////////////////////////////////////////////////////////////// +// EVERYTHING BELOW IS AUTOGENERATED + +// suppress unused package warning +var ( + _ = gojson.RawMessage{} + _ = jlexer.Lexer{} + _ = jwriter.Writer{} +) + +func easyjsonDbb23193DecodeGithubComTidwallSjson(in *jlexer.Lexer, out *BenchStruct) { + if in.IsNull() { + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeString() + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "widget": + easyjsonDbb23193Decode(in, &out.Widget) + default: + in.SkipRecursive() + } + in.WantComma() + } + in.Delim('}') +} +func easyjsonDbb23193EncodeGithubComTidwallSjson(out *jwriter.Writer, in BenchStruct) { + out.RawByte('{') + first := true + _ = first + if !first { + out.RawByte(',') + } + first = false + out.RawString("\"widget\":") + easyjsonDbb23193Encode(out, in.Widget) + out.RawByte('}') +} + +// MarshalJSON supports json.Marshaler interface +func (v BenchStruct) MarshalEasyJSONFromData() ([]byte, error) { + w := jwriter.Writer{} + easyjsonDbb23193EncodeGithubComTidwallSjson(&w, v) + return w.Buffer.BuildBytes(), w.Error +} + +// MarshalEasyJSON supports easyjson.Marshaler interface +func (v BenchStruct) MarshalEasyJSON(w *jwriter.Writer) { + easyjsonDbb23193EncodeGithubComTidwallSjson(w, v) +} + +// UnmarshalJSON supports json.Unmarshaler interface +func (v *BenchStruct) UnmarshalEasyJSONFromData(data []byte) error { + r := jlexer.Lexer{Data: data} + easyjsonDbb23193DecodeGithubComTidwallSjson(&r, v) + return r.Error() +} + +// UnmarshalEasyJSON supports easyjson.Unmarshaler interface +func (v *BenchStruct) UnmarshalEasyJSON(l *jlexer.Lexer) { + easyjsonDbb23193DecodeGithubComTidwallSjson(l, v) +} +func easyjsonDbb23193Decode(in *jlexer.Lexer, out *struct { + Debug string "json:\"debug\"" + Window struct { + Title string "json:\"title\"" + Name string "json:\"name\"" + Width int "json:\"width\"" + Height int "json:\"height\"" + } "json:\"window\"" + Image struct { + Src string "json:\"src\"" + HOffset int "json:\"hOffset\"" + VOffset int "json:\"vOffset\"" + Alignment string "json:\"alignment\"" + } "json:\"image\"" + Text struct { + Data string "json:\"data\"" + Size int "json:\"size\"" + Style string "json:\"style\"" + VOffset int "json:\"vOffset\"" + Alignment string "json:\"alignment\"" + OnMouseUp string "json:\"onMouseUp\"" + } "json:\"text\"" +}) { + if in.IsNull() { + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeString() + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "debug": + out.Debug = string(in.String()) + case "window": + easyjsonDbb23193Decode1(in, &out.Window) + case "image": + easyjsonDbb23193Decode2(in, &out.Image) + case "text": + easyjsonDbb23193Decode3(in, &out.Text) + default: + in.SkipRecursive() + } + in.WantComma() + } + in.Delim('}') +} +func easyjsonDbb23193Encode(out *jwriter.Writer, in struct { + Debug string "json:\"debug\"" + Window struct { + Title string "json:\"title\"" + Name string "json:\"name\"" + Width int "json:\"width\"" + Height int "json:\"height\"" + } "json:\"window\"" + Image struct { + Src string "json:\"src\"" + HOffset int "json:\"hOffset\"" + VOffset int "json:\"vOffset\"" + Alignment string "json:\"alignment\"" + } "json:\"image\"" + Text struct { + Data string "json:\"data\"" + Size int "json:\"size\"" + Style string "json:\"style\"" + VOffset int "json:\"vOffset\"" + Alignment string "json:\"alignment\"" + OnMouseUp string "json:\"onMouseUp\"" + } "json:\"text\"" +}) { + out.RawByte('{') + first := true + _ = first + if !first { + out.RawByte(',') + } + first = false + out.RawString("\"debug\":") + out.String(string(in.Debug)) + if !first { + out.RawByte(',') + } + first = false + out.RawString("\"window\":") + easyjsonDbb23193Encode1(out, in.Window) + if !first { + out.RawByte(',') + } + first = false + out.RawString("\"image\":") + easyjsonDbb23193Encode2(out, in.Image) + if !first { + out.RawByte(',') + } + first = false + out.RawString("\"text\":") + easyjsonDbb23193Encode3(out, in.Text) + out.RawByte('}') +} +func easyjsonDbb23193Decode3(in *jlexer.Lexer, out *struct { + Data string "json:\"data\"" + Size int "json:\"size\"" + Style string "json:\"style\"" + VOffset int "json:\"vOffset\"" + Alignment string "json:\"alignment\"" + OnMouseUp string "json:\"onMouseUp\"" +}) { + if in.IsNull() { + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeString() + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "data": + out.Data = string(in.String()) + case "size": + out.Size = int(in.Int()) + case "style": + out.Style = string(in.String()) + case "vOffset": + out.VOffset = int(in.Int()) + case "alignment": + out.Alignment = string(in.String()) + case "onMouseUp": + out.OnMouseUp = string(in.String()) + default: + in.SkipRecursive() + } + in.WantComma() + } + in.Delim('}') +} +func easyjsonDbb23193Encode3(out *jwriter.Writer, in struct { + Data string "json:\"data\"" + Size int "json:\"size\"" + Style string "json:\"style\"" + VOffset int "json:\"vOffset\"" + Alignment string "json:\"alignment\"" + OnMouseUp string "json:\"onMouseUp\"" +}) { + out.RawByte('{') + first := true + _ = first + if !first { + out.RawByte(',') + } + first = false + out.RawString("\"data\":") + out.String(string(in.Data)) + if !first { + out.RawByte(',') + } + first = false + out.RawString("\"size\":") + out.Int(int(in.Size)) + if !first { + out.RawByte(',') + } + first = false + out.RawString("\"style\":") + out.String(string(in.Style)) + if !first { + out.RawByte(',') + } + first = false + out.RawString("\"vOffset\":") + out.Int(int(in.VOffset)) + if !first { + out.RawByte(',') + } + first = false + out.RawString("\"alignment\":") + out.String(string(in.Alignment)) + if !first { + out.RawByte(',') + } + first = false + out.RawString("\"onMouseUp\":") + out.String(string(in.OnMouseUp)) + out.RawByte('}') +} +func easyjsonDbb23193Decode2(in *jlexer.Lexer, out *struct { + Src string "json:\"src\"" + HOffset int "json:\"hOffset\"" + VOffset int "json:\"vOffset\"" + Alignment string "json:\"alignment\"" +}) { + if in.IsNull() { + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeString() + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "src": + out.Src = string(in.String()) + case "hOffset": + out.HOffset = int(in.Int()) + case "vOffset": + out.VOffset = int(in.Int()) + case "alignment": + out.Alignment = string(in.String()) + default: + in.SkipRecursive() + } + in.WantComma() + } + in.Delim('}') +} +func easyjsonDbb23193Encode2(out *jwriter.Writer, in struct { + Src string "json:\"src\"" + HOffset int "json:\"hOffset\"" + VOffset int "json:\"vOffset\"" + Alignment string "json:\"alignment\"" +}) { + out.RawByte('{') + first := true + _ = first + if !first { + out.RawByte(',') + } + first = false + out.RawString("\"src\":") + out.String(string(in.Src)) + if !first { + out.RawByte(',') + } + first = false + out.RawString("\"hOffset\":") + out.Int(int(in.HOffset)) + if !first { + out.RawByte(',') + } + first = false + out.RawString("\"vOffset\":") + out.Int(int(in.VOffset)) + if !first { + out.RawByte(',') + } + first = false + out.RawString("\"alignment\":") + out.String(string(in.Alignment)) + out.RawByte('}') +} +func easyjsonDbb23193Decode1(in *jlexer.Lexer, out *struct { + Title string "json:\"title\"" + Name string "json:\"name\"" + Width int "json:\"width\"" + Height int "json:\"height\"" +}) { + if in.IsNull() { + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeString() + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "title": + out.Title = string(in.String()) + case "name": + out.Name = string(in.String()) + case "width": + out.Width = int(in.Int()) + case "height": + out.Height = int(in.Int()) + default: + in.SkipRecursive() + } + in.WantComma() + } + in.Delim('}') +} +func easyjsonDbb23193Encode1(out *jwriter.Writer, in struct { + Title string "json:\"title\"" + Name string "json:\"name\"" + Width int "json:\"width\"" + Height int "json:\"height\"" +}) { + out.RawByte('{') + first := true + _ = first + if !first { + out.RawByte(',') + } + first = false + out.RawString("\"title\":") + out.String(string(in.Title)) + if !first { + out.RawByte(',') + } + first = false + out.RawString("\"name\":") + out.String(string(in.Name)) + if !first { + out.RawByte(',') + } + first = false + out.RawString("\"width\":") + out.Int(int(in.Width)) + if !first { + out.RawByte(',') + } + first = false + out.RawString("\"height\":") + out.Int(int(in.Height)) + out.RawByte('}') +} +func (mj *BenchStruct) MarshalFFJSONFromData() ([]byte, error) { + var buf fflib.Buffer + if mj == nil { + buf.WriteString("null") + return buf.Bytes(), nil + } + err := mj.MarshalJSONBufFFJSON(&buf) + if err != nil { + return nil, err + } + return buf.Bytes(), nil +} +func (mj *BenchStruct) MarshalJSONBufFFJSON(buf fflib.EncodingBuffer) error { + if mj == nil { + buf.WriteString("null") + return nil + } + var err error + var obj []byte + _ = obj + _ = err + /* Inline struct. type=struct { Debug string "json:\"debug\""; Window struct { Title string "json:\"title\""; Name string "json:\"name\""; Width int "json:\"width\""; Height int "json:\"height\"" } "json:\"window\""; Image struct { Src string "json:\"src\""; HOffset int "json:\"hOffset\""; VOffset int "json:\"vOffset\""; Alignment string "json:\"alignment\"" } "json:\"image\""; Text struct { Data string "json:\"data\""; Size int "json:\"size\""; Style string "json:\"style\""; VOffset int "json:\"vOffset\""; Alignment string "json:\"alignment\""; OnMouseUp string "json:\"onMouseUp\"" } "json:\"text\"" } kind=struct */ + buf.WriteString(`{"widget":{ "debug":`) + fflib.WriteJsonString(buf, string(mj.Widget.Debug)) + /* Inline struct. type=struct { Title string "json:\"title\""; Name string "json:\"name\""; Width int "json:\"width\""; Height int "json:\"height\"" } kind=struct */ + buf.WriteString(`,"window":{ "title":`) + fflib.WriteJsonString(buf, string(mj.Widget.Window.Title)) + buf.WriteString(`,"name":`) + fflib.WriteJsonString(buf, string(mj.Widget.Window.Name)) + buf.WriteString(`,"width":`) + fflib.FormatBits2(buf, uint64(mj.Widget.Window.Width), 10, mj.Widget.Window.Width < 0) + buf.WriteString(`,"height":`) + fflib.FormatBits2(buf, uint64(mj.Widget.Window.Height), 10, mj.Widget.Window.Height < 0) + buf.WriteByte('}') + /* Inline struct. type=struct { Src string "json:\"src\""; HOffset int "json:\"hOffset\""; VOffset int "json:\"vOffset\""; Alignment string "json:\"alignment\"" } kind=struct */ + buf.WriteString(`,"image":{ "src":`) + fflib.WriteJsonString(buf, string(mj.Widget.Image.Src)) + buf.WriteString(`,"hOffset":`) + fflib.FormatBits2(buf, uint64(mj.Widget.Image.HOffset), 10, mj.Widget.Image.HOffset < 0) + buf.WriteString(`,"vOffset":`) + fflib.FormatBits2(buf, uint64(mj.Widget.Image.VOffset), 10, mj.Widget.Image.VOffset < 0) + buf.WriteString(`,"alignment":`) + fflib.WriteJsonString(buf, string(mj.Widget.Image.Alignment)) + buf.WriteByte('}') + /* Inline struct. type=struct { Data string "json:\"data\""; Size int "json:\"size\""; Style string "json:\"style\""; VOffset int "json:\"vOffset\""; Alignment string "json:\"alignment\""; OnMouseUp string "json:\"onMouseUp\"" } kind=struct */ + buf.WriteString(`,"text":{ "data":`) + fflib.WriteJsonString(buf, string(mj.Widget.Text.Data)) + buf.WriteString(`,"size":`) + fflib.FormatBits2(buf, uint64(mj.Widget.Text.Size), 10, mj.Widget.Text.Size < 0) + buf.WriteString(`,"style":`) + fflib.WriteJsonString(buf, string(mj.Widget.Text.Style)) + buf.WriteString(`,"vOffset":`) + fflib.FormatBits2(buf, uint64(mj.Widget.Text.VOffset), 10, mj.Widget.Text.VOffset < 0) + buf.WriteString(`,"alignment":`) + fflib.WriteJsonString(buf, string(mj.Widget.Text.Alignment)) + buf.WriteString(`,"onMouseUp":`) + fflib.WriteJsonString(buf, string(mj.Widget.Text.OnMouseUp)) + buf.WriteByte('}') + buf.WriteByte('}') + buf.WriteByte('}') + return nil +} + +const ( + ffj_t_BenchStructbase = iota + ffj_t_BenchStructno_such_key + + ffj_t_BenchStruct_Widget +) + +var ffj_key_BenchStruct_Widget = []byte("widget") + +func (uj *BenchStruct) UnmarshalFFJSONFromData(input []byte) error { + fs := fflib.NewFFLexer(input) + return uj.UnmarshalJSONFFLexer(fs, fflib.FFParse_map_start) +} + +func (uj *BenchStruct) UnmarshalJSONFFLexer(fs *fflib.FFLexer, state fflib.FFParseState) error { + var err error = nil + currentKey := ffj_t_BenchStructbase + _ = currentKey + tok := fflib.FFTok_init + wantedTok := fflib.FFTok_init + +mainparse: + for { + tok = fs.Scan() + // println(fmt.Sprintf("debug: tok: %v state: %v", tok, state)) + if tok == fflib.FFTok_error { + goto tokerror + } + + switch state { + + case fflib.FFParse_map_start: + if tok != fflib.FFTok_left_bracket { + wantedTok = fflib.FFTok_left_bracket + goto wrongtokenerror + } + state = fflib.FFParse_want_key + continue + + case fflib.FFParse_after_value: + if tok == fflib.FFTok_comma { + state = fflib.FFParse_want_key + } else if tok == fflib.FFTok_right_bracket { + goto done + } else { + wantedTok = fflib.FFTok_comma + goto wrongtokenerror + } + + case fflib.FFParse_want_key: + // json {} ended. goto exit. woo. + if tok == fflib.FFTok_right_bracket { + goto done + } + if tok != fflib.FFTok_string { + wantedTok = fflib.FFTok_string + goto wrongtokenerror + } + + kn := fs.Output.Bytes() + if len(kn) <= 0 { + // "" case. hrm. + currentKey = ffj_t_BenchStructno_such_key + state = fflib.FFParse_want_colon + goto mainparse + } else { + switch kn[0] { + + case 'w': + + if bytes.Equal(ffj_key_BenchStruct_Widget, kn) { + currentKey = ffj_t_BenchStruct_Widget + state = fflib.FFParse_want_colon + goto mainparse + } + + } + + if fflib.SimpleLetterEqualFold(ffj_key_BenchStruct_Widget, kn) { + currentKey = ffj_t_BenchStruct_Widget + state = fflib.FFParse_want_colon + goto mainparse + } + + currentKey = ffj_t_BenchStructno_such_key + state = fflib.FFParse_want_colon + goto mainparse + } + + case fflib.FFParse_want_colon: + if tok != fflib.FFTok_colon { + wantedTok = fflib.FFTok_colon + goto wrongtokenerror + } + state = fflib.FFParse_want_value + continue + case fflib.FFParse_want_value: + + if tok == fflib.FFTok_left_brace || tok == fflib.FFTok_left_bracket || tok == fflib.FFTok_integer || tok == fflib.FFTok_double || tok == fflib.FFTok_string || tok == fflib.FFTok_bool || tok == fflib.FFTok_null { + switch currentKey { + + case ffj_t_BenchStruct_Widget: + goto handle_Widget + + case ffj_t_BenchStructno_such_key: + err = fs.SkipField(tok) + if err != nil { + return fs.WrapErr(err) + } + state = fflib.FFParse_after_value + goto mainparse + } + } else { + goto wantedvalue + } + } + } + +handle_Widget: + + /* handler: uj.Widget type=struct { Debug string "json:\"debug\""; Window struct { Title string "json:\"title\""; Name string "json:\"name\""; Width int "json:\"width\""; Height int "json:\"height\"" } "json:\"window\""; Image struct { Src string "json:\"src\""; HOffset int "json:\"hOffset\""; VOffset int "json:\"vOffset\""; Alignment string "json:\"alignment\"" } "json:\"image\""; Text struct { Data string "json:\"data\""; Size int "json:\"size\""; Style string "json:\"style\""; VOffset int "json:\"vOffset\""; Alignment string "json:\"alignment\""; OnMouseUp string "json:\"onMouseUp\"" } "json:\"text\"" } kind=struct quoted=false*/ + + { + /* Falling back. type=struct { Debug string "json:\"debug\""; Window struct { Title string "json:\"title\""; Name string "json:\"name\""; Width int "json:\"width\""; Height int "json:\"height\"" } "json:\"window\""; Image struct { Src string "json:\"src\""; HOffset int "json:\"hOffset\""; VOffset int "json:\"vOffset\""; Alignment string "json:\"alignment\"" } "json:\"image\""; Text struct { Data string "json:\"data\""; Size int "json:\"size\""; Style string "json:\"style\""; VOffset int "json:\"vOffset\""; Alignment string "json:\"alignment\""; OnMouseUp string "json:\"onMouseUp\"" } "json:\"text\"" } kind=struct */ + tbuf, err := fs.CaptureField(tok) + if err != nil { + return fs.WrapErr(err) + } + + err = gojson.Unmarshal(tbuf, &uj.Widget) + if err != nil { + return fs.WrapErr(err) + } + } + + state = fflib.FFParse_after_value + goto mainparse + +wantedvalue: + return fs.WrapErr(fmt.Errorf("wanted value token, but got token: %v", tok)) +wrongtokenerror: + return fs.WrapErr(fmt.Errorf("ffjson: wanted token: %v, but got token: %v output=%s", wantedTok, tok, fs.Output.String())) +tokerror: + if fs.BigError != nil { + return fs.WrapErr(fs.BigError) + } + err = fs.Error.ToError() + if err != nil { + return fs.WrapErr(err) + } + panic("ffjson-generated: unreachable, please report bug.") +done: + return nil +}