From 3e41a2ecce19d55d7bb2486a19bde38f5c6606a2 Mon Sep 17 00:00:00 2001 From: tidwall Date: Thu, 18 Oct 2018 06:30:41 -0700 Subject: [PATCH] Update gjson/sjson --- Gopkg.lock | 12 +- vendor/github.com/tidwall/gjson/README.md | 124 +- vendor/github.com/tidwall/gjson/gjson.go | 621 ++------- vendor/github.com/tidwall/gjson/gjson_gae.go | 10 + vendor/github.com/tidwall/gjson/gjson_ngae.go | 73 + vendor/github.com/tidwall/gjson/gjson_test.go | 466 ++++++- vendor/github.com/tidwall/sjson/README.md | 8 +- vendor/github.com/tidwall/sjson/sjson.go | 197 +-- vendor/github.com/tidwall/sjson/sjson_gae.go | 196 +++ vendor/github.com/tidwall/sjson/sjson_ngae.go | 191 +++ vendor/github.com/tidwall/sjson/sjson_test.go | 1186 ++--------------- 11 files changed, 1206 insertions(+), 1878 deletions(-) create mode 100644 vendor/github.com/tidwall/gjson/gjson_gae.go create mode 100644 vendor/github.com/tidwall/gjson/gjson_ngae.go create mode 100644 vendor/github.com/tidwall/sjson/sjson_gae.go create mode 100644 vendor/github.com/tidwall/sjson/sjson_ngae.go diff --git a/Gopkg.lock b/Gopkg.lock index 6bfe5f20..adf5e1ed 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -238,12 +238,12 @@ revision = "5302514a34feb71743bf597938742b51831ba289" [[projects]] - digest = "1:211773b67c5594aa92b1e8389c59558fa4927614507ea38237265e00c0ba6b81" + digest = "1:3ddca2bd5496c6922a2a9e636530e178a43c2a534ea6634211acdc7d10222794" name = "github.com/tidwall/gjson" packages = ["."] pruneopts = "" - revision = "5a69e67cfd8f6f9b0044ed49f5079d0eeed28653" - version = "v1.0.1" + revision = "1e3f6aeaa5bad08d777ea7807b279a07885dd8b2" + version = "v1.1.3" [[projects]] branch = "master" @@ -310,12 +310,12 @@ revision = "d4a8a3d30d5729f85edfba1745241f3a621d0359" [[projects]] - digest = "1:ed6a1c415a0bd35c9c18eec74bfd460a57ba21fb3bc0da629afc275096edffa4" + digest = "1:ca969d3e75ed5b3003f4f5864bb5c13d99471ef57f9049bf78562d7ee1ac019c" name = "github.com/tidwall/sjson" packages = ["."] pruneopts = "" - revision = "6a22caf2fd45d5e2119bfc3717e984f15a7eb7ee" - version = "v1.0.0" + revision = "48d34adceb39a5bd6ed7c12f38c78cd425436442" + version = "v1.0.2" [[projects]] branch = "master" diff --git a/vendor/github.com/tidwall/gjson/README.md b/vendor/github.com/tidwall/gjson/README.md index 5dbfed3c..70240d99 100644 --- a/vendor/github.com/tidwall/gjson/README.md +++ b/vendor/github.com/tidwall/gjson/README.md @@ -5,15 +5,17 @@
Build Status GoDoc -GJSON Playground +GJSON Playground

-

get a json value quickly

+

get json values quickly

GJSON is a Go package that provides a [fast](#performance) and [simple](#get-a-value) way to get values from a json document. -It has features such as [one line retrieval](#get-a-value), [dot notation paths](#path-syntax), [iteration](#iterate-through-an-object-or-array). +It has features such as [one line retrieval](#get-a-value), [dot notation paths](#path-syntax), [iteration](#iterate-through-an-object-or-array), and [parsing json lines](#json-lines). + +Also check out [SJSON](https://github.com/tidwall/sjson) for modifying json, and the [JJ](https://github.com/tidwall/jj) command line tool. Getting Started =============== @@ -29,7 +31,7 @@ $ go get -u github.com/tidwall/gjson This will retrieve the library. ## Get a value -Get searches json for the specified path. A path is in dot syntax, such as "name.last" or "age". This function expects that the json is well-formed. Bad json will not panic, but it may return back unexpected results. When the value is found it's returned immediately. +Get searches json for the specified path. A path is in dot syntax, such as "name.last" or "age". When the value is found it's returned immediately. ```go package main @@ -95,6 +97,36 @@ friends.#[age>45]#.last >> ["Craig","Murphy"] friends.#[first%"D*"].last >> "Murphy" ``` +## JSON Lines + +There's support for [JSON Lines](http://jsonlines.org/) using the `..` prefix, which treats a multilined document as an array. + +For example: + +``` +{"name": "Gilbert", "age": 61} +{"name": "Alexa", "age": 34} +{"name": "May", "age": 57} +{"name": "Deloise", "age": 44} +``` + +``` +..# >> 4 +..1 >> {"name": "Alexa", "age": 34} +..3 >> {"name": "Deloise", "age": 44} +..#.name >> ["Gilbert","Alexa","May","Deloise"] +..#[name="May"].age >> 57 +``` + +The `ForEachLines` function will iterate through JSON lines. + +```go +gjson.ForEachLine(json, func(line gjson.Result) bool{ + println(line.String()) + return true +}) +``` + ## Result Type GJSON supports the json types `string`, `number`, `bool`, and `null`. @@ -152,6 +184,15 @@ array >> []interface{} object >> map[string]interface{} ``` +### 64-bit integers + +The `result.Int()` and `result.Uint()` calls are capable of reading all 64 bits, allowing for large JSON integers. + +```go +result.Int() int64 // -9223372036854775808 to 9223372036854775807 +result.Uint() int64 // 0 to 18446744073709551615 +``` + ## Get nested array values Suppose you want all the last names from the following json: @@ -234,51 +275,17 @@ if gjson.Get(json, "name.last").Exists() { } ``` -## Unmarshalling +## Validate JSON -There's a `gjson.Unmarshal` function which loads json data into a value. -It's a general replacement for `json.Unmarshal` and you can typically -see a 2-3x boost in performance without the need for external generators. - -This function works almost identically to `json.Unmarshal` except that -`gjson.Unmarshal` will automatically attempt to convert JSON values to any -Go type. For example, the JSON string "100" or the JSON number 100 can be -equally assigned to Go string, int, byte, uint64, etc. This rule applies to -all types. +The `Get*` and `Parse*` functions expects that the json is well-formed. Bad json will not panic, but it may return back unexpected results. +If you are consuming JSON from an unpredictable source then you may want to validate prior to using GJSON. ```go -package main - -import ( - "fmt" - - "github.com/tidwall/gjson" -) - -type Animal struct { - Type string `json:"type"` - Sound string `json:"sound"` - Age int `json:"age"` +if !gjson.Valid(json) { + return errors.New("invalid json") } - -var json = `{ - "type": "Dog", - "Sound": "Bark", - "Age": "11" -}` - -func main() { - var dog Animal - gjson.Unmarshal([]byte(json), &dog) - fmt.Printf("type: %s, sound: %s, age: %d\n", dog.Type, dog.Sound, dog.Age) -} -``` - -This will print: - -``` -type: Dog, sound: Bark, age: 11 +value := gjson.Get(json, "name.last") ``` ## Unmarshal to a map @@ -318,7 +325,7 @@ This is a best-effort no allocation sub slice of the original json. This method ## Get multiple values at once -The `GetMany` function can be used to get multiple values at the same time, and is optimized to scan over a JSON payload once. +The `GetMany` function can be used to get multiple values at the same time. ```go results := gjson.GetMany(json, "name.first", "name.last", "age") @@ -338,7 +345,6 @@ and [json-iterator](https://github.com/json-iterator/go) BenchmarkGJSONGet-8 3000000 372 ns/op 0 B/op 0 allocs/op BenchmarkGJSONUnmarshalMap-8 900000 4154 ns/op 1920 B/op 26 allocs/op BenchmarkJSONUnmarshalMap-8 600000 9019 ns/op 3048 B/op 69 allocs/op -BenchmarkJSONUnmarshalStruct-8 600000 9268 ns/op 1832 B/op 69 allocs/op BenchmarkJSONDecoder-8 300000 14120 ns/op 4224 B/op 184 allocs/op BenchmarkFFJSONLexer-8 1500000 3111 ns/op 896 B/op 8 allocs/op BenchmarkEasyJSONLexer-8 3000000 887 ns/op 613 B/op 6 allocs/op @@ -346,17 +352,6 @@ BenchmarkJSONParserGet-8 3000000 499 ns/op 21 B/op BenchmarkJSONIterator-8 3000000 812 ns/op 544 B/op 9 allocs/op ``` -Benchmarks for the `GetMany` function: - -``` -BenchmarkGJSONGetMany4Paths-8 4000000 303 ns/op 112 B/op 0 allocs/op -BenchmarkGJSONGetMany8Paths-8 8000000 208 ns/op 56 B/op 0 allocs/op -BenchmarkGJSONGetMany16Paths-8 16000000 156 ns/op 56 B/op 0 allocs/op -BenchmarkGJSONGetMany32Paths-8 32000000 127 ns/op 64 B/op 0 allocs/op -BenchmarkGJSONGetMany64Paths-8 64000000 117 ns/op 64 B/op 0 allocs/op -BenchmarkGJSONGetMany128Paths-8 128000000 109 ns/op 64 B/op 0 allocs/op -``` - JSON document used: ```json @@ -395,21 +390,6 @@ widget.image.hOffset widget.text.onMouseUp ``` -For the `GetMany` benchmarks these paths are used: - -``` -widget.window.name -widget.image.hOffset -widget.text.onMouseUp -widget.window.title -widget.image.alignment -widget.text.style -widget.window.height -widget.image.src -widget.text.data -widget.text.size -``` - *These benchmarks were run on a MacBook Pro 15" 2.8 GHz Intel Core i7 using Go 1.8 and can be be found [here](https://github.com/tidwall/gjson-benchmarks).* diff --git a/vendor/github.com/tidwall/gjson/gjson.go b/vendor/github.com/tidwall/gjson/gjson.go index fe4143c9..74515ed5 100644 --- a/vendor/github.com/tidwall/gjson/gjson.go +++ b/vendor/github.com/tidwall/gjson/gjson.go @@ -13,7 +13,6 @@ import ( "time" "unicode/utf16" "unicode/utf8" - "unsafe" "github.com/tidwall/match" ) @@ -78,7 +77,20 @@ func (t Result) String() string { case False: return "false" case Number: - return strconv.FormatFloat(t.Num, 'f', -1, 64) + if len(t.Raw) == 0 { + // calculated result + return strconv.FormatFloat(t.Num, 'f', -1, 64) + } + var i int + if t.Raw[0] == '-' { + i++ + } + for ; i < len(t.Raw); i++ { + if t.Raw[i] < '0' || t.Raw[i] > '9' { + return strconv.FormatFloat(t.Num, 'f', -1, 64) + } + } + return t.Raw case String: return t.Str case JSON: @@ -96,7 +108,7 @@ func (t Result) Bool() bool { case True: return true case String: - return t.Str != "" && t.Str != "0" + return t.Str != "" && t.Str != "0" && t.Str != "false" case Number: return t.Num != 0 } @@ -177,8 +189,8 @@ func (t Result) Time() time.Time { // If the result represents a non-existent value, then an empty array will be returned. // If the result is not a JSON array, the return value will be an array containing one result. func (t Result) Array() []Result { - if !t.Exists() { - return nil + if t.Type == Null { + return []Result{} } if t.Type != JSON { return []Result{t} @@ -192,7 +204,7 @@ func (t Result) IsObject() bool { return t.Type == JSON && len(t.Raw) > 0 && t.Raw[0] == '{' } -// IsObject returns true if the result value is a JSON array. +// IsArray returns true if the result value is a JSON array. func (t Result) IsArray() bool { return t.Type == JSON && len(t.Raw) > 0 && t.Raw[0] == '[' } @@ -345,24 +357,30 @@ func (t Result) arrayOrMap(vc byte, valueize bool) (r arrayOrMapResult) { if (json[i] >= '0' && json[i] <= '9') || json[i] == '-' { value.Type = Number value.Raw, value.Num = tonum(json[i:]) + value.Str = "" } else { continue } case '{', '[': value.Type = JSON value.Raw = squash(json[i:]) + value.Str, value.Num = "", 0 case 'n': value.Type = Null value.Raw = tolit(json[i:]) + value.Str, value.Num = "", 0 case 't': value.Type = True value.Raw = tolit(json[i:]) + value.Str, value.Num = "", 0 case 'f': value.Type = False value.Raw = tolit(json[i:]) + value.Str, value.Num = "", 0 case '"': value.Type = String value.Raw, value.Str = tostr(json[i:]) + value.Num = 0 } i += len(value.Raw) - 1 @@ -371,9 +389,13 @@ func (t Result) arrayOrMap(vc byte, valueize bool) (r arrayOrMapResult) { key = value } else { if valueize { - r.oi[key.Str] = value.Value() + if _, ok := r.oi[key.Str]; !ok { + r.oi[key.Str] = value.Value() + } } else { - r.o[key.Str] = value + if _, ok := r.o[key.Str]; !ok { + r.o[key.Str] = value + } } } count++ @@ -390,6 +412,11 @@ end: } // Parse parses the json and returns a result. +// +// This function expects that the json is well-formed, and does not validate. +// Invalid json will not panic, but it may return back unexpected results. +// If you are consuming JSON from an unpredictable source then you may want to +// use the Valid function first. func Parse(json string) Result { var value Result for i := 0; i < len(json); i++ { @@ -511,7 +538,7 @@ func tonum(json string) (raw string, num float64) { func tolit(json string) (raw string) { for i := 1; i < len(json); i++ { - if json[i] <= 'a' || json[i] >= 'z' { + if json[i] < 'a' || json[i] > 'z' { return json[:i] } } @@ -578,6 +605,8 @@ func (t Result) Exists() bool { // Number, for JSON numbers // string, for JSON string literals // nil, for JSON null +// map[string]interface{}, for JSON objects +// []interface{}, for JSON arrays // func (t Result) Value() interface{} { if t.Type == String { @@ -1077,7 +1106,7 @@ func queryMatches(rp *arrayPathResult, value Result) bool { case "=": return value.Num == rpvn case "!=": - return value.Num == rpvn + return value.Num != rpvn case "<": return value.Num < rpvn case "<=": @@ -1128,7 +1157,7 @@ func parseArray(c *parseContext, i int, path string) (int, bool) { partidx = int(n) } } - for i < len(c.json) { + for i < len(c.json)+1 { if !rp.arrch { pmatch = partidx == h hit = pmatch && !rp.more @@ -1137,8 +1166,16 @@ func parseArray(c *parseContext, i int, path string) (int, bool) { if rp.alogok { alog = append(alog, i) } - for ; i < len(c.json); i++ { - switch c.json[i] { + for ; ; i++ { + var ch byte + if i > len(c.json) { + break + } else if i == len(c.json) { + ch = ']' + } else { + ch = c.json[i] + } + switch ch { default: continue case '"': @@ -1252,14 +1289,18 @@ func parseArray(c *parseContext, i int, path string) (int, bool) { if rp.alogok { var jsons = make([]byte, 0, 64) jsons = append(jsons, '[') + for j, k := 0, 0; j < len(alog); j++ { - res := Get(c.json[alog[j]:], rp.alogkey) - if res.Exists() { - if k > 0 { - jsons = append(jsons, ',') + _, res, ok := parseAny(c.json, alog[j], true) + if ok { + res := res.Get(rp.alogkey) + if res.Exists() { + if k > 0 { + jsons = append(jsons, ',') + } + jsons = append(jsons, []byte(res.Raw)...) + k++ } - jsons = append(jsons, []byte(res.Raw)...) - k++ } } jsons = append(jsons, ']') @@ -1270,7 +1311,7 @@ func parseArray(c *parseContext, i int, path string) (int, bool) { if rp.alogok { break } - c.value.Raw = val + c.value.Raw = "" c.value.Type = Number c.value.Num = float64(h - 1) c.calcd = true @@ -1290,16 +1331,32 @@ func parseArray(c *parseContext, i int, path string) (int, bool) { return i, false } +// ForEachLine iterates through lines of JSON as specified by the JSON Lines +// format (http://jsonlines.org/). +// Each line is returned as a GJSON Result. +func ForEachLine(json string, iterator func(line Result) bool) { + var res Result + var i int + for { + i, res, _ = parseAny(json, i, true) + if !res.Exists() { + break + } + if !iterator(res) { + return + } + } +} + type parseContext struct { json string value Result calcd bool + lines bool } // Get searches json for the specified path. // A path is in dot syntax, such as "name.last" or "age". -// This function expects that the json is well-formed, and does not validate. -// Invalid json will not panic, but it may return back unexpected results. // When the value is found it's returned immediately. // // A path is a series of keys searated by a dot. @@ -1326,79 +1383,38 @@ type parseContext struct { // "c?ildren.0" >> "Sara" // "friends.#.first" >> ["James","Roger"] // +// This function expects that the json is well-formed, and does not validate. +// Invalid json will not panic, but it may return back unexpected results. +// If you are consuming JSON from an unpredictable source then you may want to +// use the Valid function first. func Get(json, path string) Result { var i int var c = &parseContext{json: json} - for ; i < len(c.json); i++ { - if c.json[i] == '{' { - i++ - parseObject(c, i, path) - break - } - if c.json[i] == '[' { - i++ - parseArray(c, i, path) - break - } - } - if len(c.value.Raw) > 0 && !c.calcd { - jhdr := *(*reflect.StringHeader)(unsafe.Pointer(&json)) - rhdr := *(*reflect.StringHeader)(unsafe.Pointer(&(c.value.Raw))) - c.value.Index = int(rhdr.Data - jhdr.Data) - if c.value.Index < 0 || c.value.Index >= len(json) { - c.value.Index = 0 - } - } - return c.value -} -func fromBytesGet(result Result) Result { - // safely get the string headers - rawhi := *(*reflect.StringHeader)(unsafe.Pointer(&result.Raw)) - strhi := *(*reflect.StringHeader)(unsafe.Pointer(&result.Str)) - // create byte slice headers - rawh := reflect.SliceHeader{Data: rawhi.Data, Len: rawhi.Len} - strh := reflect.SliceHeader{Data: strhi.Data, Len: strhi.Len} - if strh.Data == 0 { - // str is nil - if rawh.Data == 0 { - // raw is nil - result.Raw = "" - } else { - // raw has data, safely copy the slice header to a string - result.Raw = string(*(*[]byte)(unsafe.Pointer(&rawh))) - } - result.Str = "" - } else if rawh.Data == 0 { - // raw is nil - result.Raw = "" - // str has data, safely copy the slice header to a string - result.Str = string(*(*[]byte)(unsafe.Pointer(&strh))) - } else if strh.Data >= rawh.Data && - int(strh.Data)+strh.Len <= int(rawh.Data)+rawh.Len { - // Str is a substring of Raw. - start := int(strh.Data - rawh.Data) - // safely copy the raw slice header - result.Raw = string(*(*[]byte)(unsafe.Pointer(&rawh))) - // substring the raw - result.Str = result.Raw[start : start+strh.Len] + if len(path) >= 2 && path[0] == '.' && path[1] == '.' { + c.lines = true + parseArray(c, 0, path[2:]) } else { - // safely copy both the raw and str slice headers to strings - result.Raw = string(*(*[]byte)(unsafe.Pointer(&rawh))) - result.Str = string(*(*[]byte)(unsafe.Pointer(&strh))) + for ; i < len(c.json); i++ { + if c.json[i] == '{' { + i++ + parseObject(c, i, path) + break + } + if c.json[i] == '[' { + i++ + parseArray(c, i, path) + break + } + } } - return result + fillIndex(json, c) + return c.value } // GetBytes searches json for the specified path. // If working with bytes, this method preferred over Get(string(data), path) func GetBytes(json []byte, path string) Result { - var result Result - if json != nil { - // unsafe cast to string - result = Get(*(*string)(unsafe.Pointer(&json)), path) - result = fromBytesGet(result) - } - return result + return getBytes(json, path) } // runeit returns the rune from the the \uXXXX @@ -1595,405 +1611,22 @@ var ( // used for testing testLastWasFallback bool ) -// areSimplePaths returns true if all the paths are simple enough -// to parse quickly for GetMany(). Allows alpha-numeric, dots, -// underscores, and the dollar sign. It does not allow non-alnum, -// escape characters, or keys which start with a numbers. -// For example: -// "name.last" == OK -// "user.id0" == OK -// "user.ID" == OK -// "user.first_name" == OK -// "user.firstName" == OK -// "user.0item" == BAD -// "user.#id" == BAD -// "user\.name" == BAD -func areSimplePaths(paths []string) bool { - for _, path := range paths { - var fi int // first key index, for keys with numeric prefix - for i := 0; i < len(path); i++ { - if path[i] >= 'a' && path[i] <= 'z' { - // a-z is likely to be the highest frequency charater. - continue - } - if path[i] == '.' { - fi = i + 1 - continue - } - if path[i] >= 'A' && path[i] <= 'Z' { - continue - } - if path[i] == '_' || path[i] == '$' { - continue - } - if i > fi && path[i] >= '0' && path[i] <= '9' { - continue - } - return false - } - } - return true -} - // GetMany searches json for the multiple paths. // The return value is a Result array where the number of items // will be equal to the number of input paths. -func GetMany(json string, paths ...string) []Result { - if len(paths) < 4 { - if testWatchForFallback { - testLastWasFallback = false - } - switch len(paths) { - case 0: - // return nil when no paths are specified. - return nil - case 1: - return []Result{Get(json, paths[0])} - case 2: - return []Result{Get(json, paths[0]), Get(json, paths[1])} - case 3: - return []Result{Get(json, paths[0]), Get(json, paths[1]), Get(json, paths[2])} - } +func GetMany(json string, path ...string) []Result { + res := make([]Result, len(path)) + for i, path := range path { + res[i] = Get(json, path) } - var results []Result - var ok bool - var i int - if len(paths) > 512 { - // we can only support up to 512 paths. Is that too many? - goto fallback - } - if !areSimplePaths(paths) { - // If there is even one path that is not considered "simple" then - // we need to use the fallback method. - goto fallback - } - // locate the object token. - for ; i < len(json); i++ { - if json[i] == '{' { - i++ - break - } - if json[i] <= ' ' { - continue - } - goto fallback - } - // use the call function table. - if len(paths) <= 8 { - results, ok = getMany8(json, i, paths) - } else if len(paths) <= 16 { - results, ok = getMany16(json, i, paths) - } else if len(paths) <= 32 { - results, ok = getMany32(json, i, paths) - } else if len(paths) <= 64 { - results, ok = getMany64(json, i, paths) - } else if len(paths) <= 128 { - results, ok = getMany128(json, i, paths) - } else if len(paths) <= 256 { - results, ok = getMany256(json, i, paths) - } else if len(paths) <= 512 { - results, ok = getMany512(json, i, paths) - } - if !ok { - // there was some fault while parsing. we should try the - // fallback method. This could result in performance - // degregation in some cases. - goto fallback - } - if testWatchForFallback { - testLastWasFallback = false - } - return results -fallback: - results = results[:0] - for i := 0; i < len(paths); i++ { - results = append(results, Get(json, paths[i])) - } - if testWatchForFallback { - testLastWasFallback = true - } - return results + return res } -// GetManyBytes searches json for the specified path. -// If working with bytes, this method preferred over -// GetMany(string(data), paths...) -func GetManyBytes(json []byte, paths ...string) []Result { - if json == nil { - return GetMany("", paths...) - } - results := GetMany(*(*string)(unsafe.Pointer(&json)), paths...) - for i := range results { - results[i] = fromBytesGet(results[i]) - } - return results -} - -// parseGetMany parses a json object for keys that match against the callers -// paths. It's a best-effort attempt and quickly locating and assigning the -// values to the []Result array. If there are failures such as bad json, or -// invalid input paths, or too much recursion, the function will exit with a -// return value of 'false'. -func parseGetMany( - json string, i int, - level uint, kplen int, - paths []string, completed []bool, matches []uint64, results []Result, -) (int, bool) { - if level > 62 { - // The recursion level is limited because the matches []uint64 - // array cannot handle more the 64-bits. - return i, false - } - // At this point the last character read was a '{'. - // Read all object keys and try to match against the paths. - var key string - var val string - var vesc, ok bool -next_key: - for ; i < len(json); i++ { - if json[i] == '"' { - // read the key - i, val, vesc, ok = parseString(json, i+1) - if !ok { - return i, false - } - if vesc { - // the value is escaped - key = unescape(val[1 : len(val)-1]) - } else { - // just a plain old ascii key - key = val[1 : len(val)-1] - } - var hasMatch bool - var parsedVal bool - var valOrgIndex int - var valPathIndex int - for j := 0; j < len(key); j++ { - if key[j] == '.' { - // we need to look for keys with dot and ignore them. - if i, _, ok = parseAny(json, i, false); !ok { - return i, false - } - continue next_key - } - } - var usedPaths int - // loop through paths and look for matches - for j := 0; j < len(paths); j++ { - if completed[j] { - usedPaths++ - // ignore completed paths - continue - } - if level > 0 && (matches[j]>>(level-1))&1 == 0 { - // ignore unmatched paths - usedPaths++ - continue - } - // try to match the key to the path - // this is spaghetti code but the idea is to minimize - // calls and variable assignments when comparing the - // key to paths - if len(paths[j])-kplen >= len(key) { - i, k := kplen, 0 - for ; k < len(key); k, i = k+1, i+1 { - if key[k] != paths[j][i] { - // no match - goto nomatch - } - } - if i < len(paths[j]) { - if paths[j][i] == '.' { - // matched, but there are still more keys in path - goto match_not_atend - } - } - if len(paths[j]) <= len(key) || kplen != 0 { - if len(paths[j]) != i { - goto nomatch - } - // matched and at the end of the path - goto match_atend - } - } - // no match, jump to the nomatch label - goto nomatch - match_atend: - // found a match - // at the end of the path. we must take the value. - usedPaths++ - if !parsedVal { - // the value has not been parsed yet. let's do so. - valOrgIndex = i // keep track of the current position. - i, results[j], ok = parseAny(json, i, true) - if !ok { - return i, false - } - parsedVal = true - valPathIndex = j - } else { - results[j] = results[valPathIndex] - } - // mark as complete - completed[j] = true - // jump over the match_not_atend label - goto nomatch - match_not_atend: - // found a match - // still in the middle of the path. - usedPaths++ - // mark the path as matched - matches[j] |= 1 << level - if !hasMatch { - hasMatch = true - } - nomatch: // noop label - } - - if !hasMatch && i < len(json) && json[i] == '}' { - return i + 1, true - } - if !parsedVal { - if hasMatch { - // we found a match and the value has not been parsed yet. - // let's find out if the next value type is an object. - for ; i < len(json); i++ { - if json[i] <= ' ' || json[i] == ':' { - continue - } - break - } - if i < len(json) { - if json[i] == '{' { - // it's an object. let's go deeper - i, ok = parseGetMany(json, i+1, level+1, kplen+len(key)+1, paths, completed, matches, results) - if !ok { - return i, false - } - } else { - // not an object. just parse and ignore. - if i, _, ok = parseAny(json, i, false); !ok { - return i, false - } - } - } - } else { - // Since there was no matches we can just parse the value and - // ignore the result. - if i, _, ok = parseAny(json, i, false); !ok { - return i, false - } - } - } else if hasMatch && len(results[valPathIndex].Raw) > 0 && results[valPathIndex].Raw[0] == '{' { - // The value was already parsed and the value type is an object. - // Rewind the json index and let's parse deeper. - i = valOrgIndex - for ; i < len(json); i++ { - if json[i] == '{' { - break - } - } - i, ok = parseGetMany(json, i+1, level+1, kplen+len(key)+1, paths, completed, matches, results) - if !ok { - return i, false - } - } - if usedPaths == len(paths) { - // all paths have been used, either completed or matched. - // we should stop parsing this object to save CPU cycles. - if level > 0 && i < len(json) { - i, _ = parseSquash(json, i) - } - return i, true - } - } else if json[i] == '}' { - // reached the end of the object. end it here. - return i + 1, true - } - } - return i, true -} - -// Call table for GetMany. Using an isolated function allows for allocating -// arrays with know capacities on the stack, as opposed to dynamically -// allocating on the heap. This can provide a tremendous performance boost -// by avoiding the GC. -func getMany8(json string, i int, paths []string) ([]Result, bool) { - const max = 8 - var completed = make([]bool, 0, max) - var matches = make([]uint64, 0, max) - var results = make([]Result, 0, max) - completed = completed[0:len(paths):max] - matches = matches[0:len(paths):max] - results = results[0:len(paths):max] - _, ok := parseGetMany(json, i, 0, 0, paths, completed, matches, results) - return results, ok -} -func getMany16(json string, i int, paths []string) ([]Result, bool) { - const max = 16 - var completed = make([]bool, 0, max) - var matches = make([]uint64, 0, max) - var results = make([]Result, 0, max) - completed = completed[0:len(paths):max] - matches = matches[0:len(paths):max] - results = results[0:len(paths):max] - _, ok := parseGetMany(json, i, 0, 0, paths, completed, matches, results) - return results, ok -} -func getMany32(json string, i int, paths []string) ([]Result, bool) { - const max = 32 - var completed = make([]bool, 0, max) - var matches = make([]uint64, 0, max) - var results = make([]Result, 0, max) - completed = completed[0:len(paths):max] - matches = matches[0:len(paths):max] - results = results[0:len(paths):max] - _, ok := parseGetMany(json, i, 0, 0, paths, completed, matches, results) - return results, ok -} -func getMany64(json string, i int, paths []string) ([]Result, bool) { - const max = 64 - var completed = make([]bool, 0, max) - var matches = make([]uint64, 0, max) - var results = make([]Result, 0, max) - completed = completed[0:len(paths):max] - matches = matches[0:len(paths):max] - results = results[0:len(paths):max] - _, ok := parseGetMany(json, i, 0, 0, paths, completed, matches, results) - return results, ok -} -func getMany128(json string, i int, paths []string) ([]Result, bool) { - const max = 128 - var completed = make([]bool, 0, max) - var matches = make([]uint64, 0, max) - var results = make([]Result, 0, max) - completed = completed[0:len(paths):max] - matches = matches[0:len(paths):max] - results = results[0:len(paths):max] - _, ok := parseGetMany(json, i, 0, 0, paths, completed, matches, results) - return results, ok -} -func getMany256(json string, i int, paths []string) ([]Result, bool) { - const max = 256 - var completed = make([]bool, 0, max) - var matches = make([]uint64, 0, max) - var results = make([]Result, 0, max) - completed = completed[0:len(paths):max] - matches = matches[0:len(paths):max] - results = results[0:len(paths):max] - _, ok := parseGetMany(json, i, 0, 0, paths, completed, matches, results) - return results, ok -} -func getMany512(json string, i int, paths []string) ([]Result, bool) { - const max = 512 - var completed = make([]bool, 0, max) - var matches = make([]uint64, 0, max) - var results = make([]Result, 0, max) - completed = completed[0:len(paths):max] - matches = matches[0:len(paths):max] - results = results[0:len(paths):max] - _, ok := parseGetMany(json, i, 0, 0, paths, completed, matches, results) - return results, ok +// GetManyBytes searches json for the multiple paths. +// The return value is a Result array where the number of items +// will be equal to the number of input paths. +func GetManyBytes(json []byte, path ...string) []Result { + return GetMany(string(json), path...) } var fieldsmu sync.RWMutex @@ -2099,6 +1732,8 @@ var validate uintptr = 1 // UnmarshalValidationEnabled provides the option to disable JSON validation // during the Unmarshal routine. Validation is enabled by default. +// +// Deprecated: Use encoder/json.Unmarshal instead func UnmarshalValidationEnabled(enabled bool) { if enabled { atomic.StoreUintptr(&validate, 1) @@ -2113,6 +1748,8 @@ func UnmarshalValidationEnabled(enabled bool) { // gjson.Unmarshal will automatically attempt to convert JSON values to any Go // type. For example, the JSON string "100" or the JSON number 100 can be equally // assigned to Go string, int, byte, uint64, etc. This rule applies to all types. +// +// Deprecated: Use encoder/json.Unmarshal instead func Unmarshal(data []byte, v interface{}) error { if atomic.LoadUintptr(&validate) == 1 { _, ok := validpayload(data, 0) @@ -2200,8 +1837,14 @@ func validobject(data []byte, i int) (outi int, ok bool) { if data[i] == '}' { return i + 1, true } + i++ for ; i < len(data); i++ { - if data[i] == '"' { + switch data[i] { + default: + return i, false + case ' ', '\t', '\n', '\r': + continue + case '"': goto key } } @@ -2382,11 +2025,31 @@ func validnull(data []byte, i int) (outi int, ok bool) { } // Valid returns true if the input is valid json. +// +// if !gjson.Valid(json) { +// return errors.New("invalid json") +// } +// value := gjson.Get(json, "name.last") +// func Valid(json string) bool { _, ok := validpayload([]byte(json), 0) return ok } +// ValidBytes returns true if the input is valid json. +// +// if !gjson.Valid(json) { +// return errors.New("invalid json") +// } +// value := gjson.Get(json, "name.last") +// +// If working with bytes, this method preferred over Valid(string(data)) +// +func ValidBytes(json []byte) bool { + _, ok := validpayload(json, 0) + return ok +} + func parseUint(s string) (n uint64, ok bool) { var i int if i == len(s) { diff --git a/vendor/github.com/tidwall/gjson/gjson_gae.go b/vendor/github.com/tidwall/gjson/gjson_gae.go new file mode 100644 index 00000000..cbe2ab42 --- /dev/null +++ b/vendor/github.com/tidwall/gjson/gjson_gae.go @@ -0,0 +1,10 @@ +//+build appengine + +package gjson + +func getBytes(json []byte, path string) Result { + return Get(string(json), path) +} +func fillIndex(json string, c *parseContext) { + // noop. Use zero for the Index value. +} diff --git a/vendor/github.com/tidwall/gjson/gjson_ngae.go b/vendor/github.com/tidwall/gjson/gjson_ngae.go new file mode 100644 index 00000000..ff313a78 --- /dev/null +++ b/vendor/github.com/tidwall/gjson/gjson_ngae.go @@ -0,0 +1,73 @@ +//+build !appengine + +package gjson + +import ( + "reflect" + "unsafe" +) + +// getBytes casts the input json bytes to a string and safely returns the +// results as uniquely allocated data. This operation is intended to minimize +// copies and allocations for the large json string->[]byte. +func getBytes(json []byte, path string) Result { + var result Result + if json != nil { + // unsafe cast to string + result = Get(*(*string)(unsafe.Pointer(&json)), path) + result = fromBytesGet(result) + } + return result +} + +func fromBytesGet(result Result) Result { + // safely get the string headers + rawhi := *(*reflect.StringHeader)(unsafe.Pointer(&result.Raw)) + strhi := *(*reflect.StringHeader)(unsafe.Pointer(&result.Str)) + // create byte slice headers + rawh := reflect.SliceHeader{Data: rawhi.Data, Len: rawhi.Len} + strh := reflect.SliceHeader{Data: strhi.Data, Len: strhi.Len} + if strh.Data == 0 { + // str is nil + if rawh.Data == 0 { + // raw is nil + result.Raw = "" + } else { + // raw has data, safely copy the slice header to a string + result.Raw = string(*(*[]byte)(unsafe.Pointer(&rawh))) + } + result.Str = "" + } else if rawh.Data == 0 { + // raw is nil + result.Raw = "" + // str has data, safely copy the slice header to a string + result.Str = string(*(*[]byte)(unsafe.Pointer(&strh))) + } else if strh.Data >= rawh.Data && + int(strh.Data)+strh.Len <= int(rawh.Data)+rawh.Len { + // Str is a substring of Raw. + start := int(strh.Data - rawh.Data) + // safely copy the raw slice header + result.Raw = string(*(*[]byte)(unsafe.Pointer(&rawh))) + // substring the raw + result.Str = result.Raw[start : start+strh.Len] + } else { + // safely copy both the raw and str slice headers to strings + result.Raw = string(*(*[]byte)(unsafe.Pointer(&rawh))) + result.Str = string(*(*[]byte)(unsafe.Pointer(&strh))) + } + return result +} + +// fillIndex finds the position of Raw data and assigns it to the Index field +// of the resulting value. If the position cannot be found then Index zero is +// used instead. +func fillIndex(json string, c *parseContext) { + if len(c.value.Raw) > 0 && !c.calcd { + jhdr := *(*reflect.StringHeader)(unsafe.Pointer(&json)) + rhdr := *(*reflect.StringHeader)(unsafe.Pointer(&(c.value.Raw))) + c.value.Index = int(rhdr.Data - jhdr.Data) + if c.value.Index < 0 || c.value.Index >= len(json) { + c.value.Index = 0 + } + } +} diff --git a/vendor/github.com/tidwall/gjson/gjson_test.go b/vendor/github.com/tidwall/gjson/gjson_test.go index 068e7426..6333be7d 100644 --- a/vendor/github.com/tidwall/gjson/gjson_test.go +++ b/vendor/github.com/tidwall/gjson/gjson_test.go @@ -7,6 +7,7 @@ import ( "fmt" "math/rand" "reflect" + "strconv" "strings" "testing" "time" @@ -147,7 +148,7 @@ func TestTimeResult(t *testing.T) { func TestParseAny(t *testing.T) { assert(t, Parse("100").Float() == 100) assert(t, Parse("true").Bool()) - assert(t, Parse("valse").Bool() == false) + assert(t, Parse("false").Bool() == false) } func TestManyVariousPathCounts(t *testing.T) { @@ -478,7 +479,8 @@ func TestBasic4(t *testing.T) { } token = get(basicJSON, "arr.#") if token.String() != "6" { - t.Fatal("expecting '6'", "got", token.String()) + fmt.Printf("%#v\n", token) + t.Fatal("expecting 6", "got", token.String()) } token = get(basicJSON, "arr.3.hello") if token.String() != "world" { @@ -969,80 +971,82 @@ func TestUnmarshal(t *testing.T) { assert(t, str == Get(complicatedJSON, "LeftOut").String()) } -func testvalid(json string, expect bool) { +func testvalid(t *testing.T, json string, expect bool) { + t.Helper() _, ok := validpayload([]byte(json), 0) if ok != expect { - panic("mismatch") + t.Fatal("mismatch") } } func TestValidBasic(t *testing.T) { - testvalid("0", true) - testvalid("00", false) - testvalid("-00", false) - testvalid("-.", false) - testvalid("0.0", true) - testvalid("10.0", true) - testvalid("10e1", true) - testvalid("10EE", false) - testvalid("10E-", false) - testvalid("10E+", false) - testvalid("10E123", true) - testvalid("10E-123", true) - testvalid("10E-0123", true) - testvalid("", false) - testvalid(" ", false) - testvalid("{}", true) - testvalid("{", false) - testvalid("-", false) - testvalid("-1", true) - testvalid("-1.", false) - testvalid("-1.0", true) - testvalid(" -1.0", true) - testvalid(" -1.0 ", true) - testvalid("-1.0 ", true) - testvalid("-1.0 i", false) - testvalid("-1.0 i", false) - testvalid("true", true) - testvalid(" true", true) - testvalid(" true ", true) - testvalid(" True ", false) - testvalid(" tru", false) - testvalid("false", true) - testvalid(" false", true) - testvalid(" false ", true) - testvalid(" False ", false) - testvalid(" fals", false) - testvalid("null", true) - testvalid(" null", true) - testvalid(" null ", true) - testvalid(" Null ", false) - testvalid(" nul", false) - testvalid(" []", true) - testvalid(" [true]", true) - testvalid(" [ true, null ]", true) - testvalid(" [ true,]", false) - testvalid(`{"hello":"world"}`, true) - testvalid(`{ "hello": "world" }`, true) - testvalid(`{ "hello": "world", }`, false) - testvalid(`{"a":"b",}`, false) - testvalid(`{"a":"b","a"}`, false) - testvalid(`{"a":"b","a":}`, false) - testvalid(`{"a":"b","a":1}`, true) - testvalid(`{"a":"b","a": 1, "c":{"hi":"there"} }`, true) - testvalid(`{"a":"b","a": 1, "c":{"hi":"there", "easy":["going",{"mixed":"bag"}]} }`, true) - testvalid(`""`, true) - testvalid(`"`, false) - testvalid(`"\n"`, true) - testvalid(`"\"`, false) - testvalid(`"\\"`, true) - testvalid(`"a\\b"`, true) - testvalid(`"a\\b\\\"a"`, true) - testvalid(`"a\\b\\\uFFAAa"`, true) - testvalid(`"a\\b\\\uFFAZa"`, false) - testvalid(`"a\\b\\\uFFA"`, false) - testvalid(string(complicatedJSON), true) - testvalid(string(exampleJSON), true) + testvalid(t, "0", true) + testvalid(t, "00", false) + testvalid(t, "-00", false) + testvalid(t, "-.", false) + testvalid(t, "0.0", true) + testvalid(t, "10.0", true) + testvalid(t, "10e1", true) + testvalid(t, "10EE", false) + testvalid(t, "10E-", false) + testvalid(t, "10E+", false) + testvalid(t, "10E123", true) + testvalid(t, "10E-123", true) + testvalid(t, "10E-0123", true) + testvalid(t, "", false) + testvalid(t, " ", false) + testvalid(t, "{}", true) + testvalid(t, "{", false) + testvalid(t, "-", false) + testvalid(t, "-1", true) + testvalid(t, "-1.", false) + testvalid(t, "-1.0", true) + testvalid(t, " -1.0", true) + testvalid(t, " -1.0 ", true) + testvalid(t, "-1.0 ", true) + testvalid(t, "-1.0 i", false) + testvalid(t, "-1.0 i", false) + testvalid(t, "true", true) + testvalid(t, " true", true) + testvalid(t, " true ", true) + testvalid(t, " True ", false) + testvalid(t, " tru", false) + testvalid(t, "false", true) + testvalid(t, " false", true) + testvalid(t, " false ", true) + testvalid(t, " False ", false) + testvalid(t, " fals", false) + testvalid(t, "null", true) + testvalid(t, " null", true) + testvalid(t, " null ", true) + testvalid(t, " Null ", false) + testvalid(t, " nul", false) + testvalid(t, " []", true) + testvalid(t, " [true]", true) + testvalid(t, " [ true, null ]", true) + testvalid(t, " [ true,]", false) + testvalid(t, `{"hello":"world"}`, true) + testvalid(t, `{ "hello": "world" }`, true) + testvalid(t, `{ "hello": "world", }`, false) + testvalid(t, `{"a":"b",}`, false) + testvalid(t, `{"a":"b","a"}`, false) + testvalid(t, `{"a":"b","a":}`, false) + testvalid(t, `{"a":"b","a":1}`, true) + testvalid(t, `{"a":"b",2"1":2}`, false) + testvalid(t, `{"a":"b","a": 1, "c":{"hi":"there"} }`, true) + testvalid(t, `{"a":"b","a": 1, "c":{"hi":"there", "easy":["going",{"mixed":"bag"}]} }`, true) + testvalid(t, `""`, true) + testvalid(t, `"`, false) + testvalid(t, `"\n"`, true) + testvalid(t, `"\"`, false) + testvalid(t, `"\\"`, true) + testvalid(t, `"a\\b"`, true) + testvalid(t, `"a\\b\\\"a"`, true) + testvalid(t, `"a\\b\\\uFFAAa"`, true) + testvalid(t, `"a\\b\\\uFFAZa"`, false) + testvalid(t, `"a\\b\\\uFFA"`, false) + testvalid(t, string(complicatedJSON), true) + testvalid(t, string(exampleJSON), true) } var jsonchars = []string{"{", "[", ",", ":", "}", "]", "1", "0", "true", "false", "null", `""`, `"\""`, `"a"`} @@ -1101,3 +1105,325 @@ func TestGetMany48(t *testing.T) { } } } + +func TestResultRawForLiteral(t *testing.T) { + for _, lit := range []string{"null", "true", "false"} { + result := Parse(lit) + if result.Raw != lit { + t.Fatalf("expected '%v', got '%v'", lit, result.Raw) + } + } +} + +func TestNullArray(t *testing.T) { + n := len(Get(`{"data":null}`, "data").Array()) + if n != 0 { + t.Fatalf("expected '%v', got '%v'", 0, n) + } + n = len(Get(`{}`, "data").Array()) + if n != 0 { + t.Fatalf("expected '%v', got '%v'", 0, n) + } + n = len(Get(`{"data":[]}`, "data").Array()) + if n != 0 { + t.Fatalf("expected '%v', got '%v'", 0, n) + } + n = len(Get(`{"data":[null]}`, "data").Array()) + if n != 1 { + t.Fatalf("expected '%v', got '%v'", 1, n) + } +} + +func TestRandomGetMany(t *testing.T) { + start := time.Now() + for time.Since(start) < time.Second*3 { + testRandomGetMany(t) + } +} +func testRandomGetMany(t *testing.T) { + rand.Seed(time.Now().UnixNano()) + json, keys := randomJSON() + for _, key := range keys { + r := Get(json, key) + if !r.Exists() { + t.Fatal("should exist") + } + } + rkeysi := rand.Perm(len(keys)) + rkeysn := 1 + rand.Int()%32 + if len(rkeysi) > rkeysn { + rkeysi = rkeysi[:rkeysn] + } + var rkeys []string + for i := 0; i < len(rkeysi); i++ { + rkeys = append(rkeys, keys[rkeysi[i]]) + } + mres1 := GetMany(json, rkeys...) + var mres2 []Result + for _, rkey := range rkeys { + mres2 = append(mres2, Get(json, rkey)) + } + if len(mres1) != len(mres2) { + t.Fatalf("expected %d, got %d", len(mres2), len(mres1)) + } + for i := 0; i < len(mres1); i++ { + mres1[i].Index = 0 + mres2[i].Index = 0 + v1 := fmt.Sprintf("%#v", mres1[i]) + v2 := fmt.Sprintf("%#v", mres2[i]) + if v1 != v2 { + t.Fatalf("\nexpected %s\n"+ + " got %s", v2, v1) + } + } +} + +func TestIssue54(t *testing.T) { + var r []Result + json := `{"MarketName":null,"Nounce":6115}` + r = GetMany(json, "Nounce", "Buys", "Sells", "Fills") + if strings.Replace(fmt.Sprintf("%v", r), " ", "", -1) != "[6115]" { + t.Fatalf("expected '%v', got '%v'", "[6115]", strings.Replace(fmt.Sprintf("%v", r), " ", "", -1)) + } + r = GetMany(json, "Nounce", "Buys", "Sells") + if strings.Replace(fmt.Sprintf("%v", r), " ", "", -1) != "[6115]" { + t.Fatalf("expected '%v', got '%v'", "[6115]", strings.Replace(fmt.Sprintf("%v", r), " ", "", -1)) + } + r = GetMany(json, "Nounce") + if strings.Replace(fmt.Sprintf("%v", r), " ", "", -1) != "[6115]" { + t.Fatalf("expected '%v', got '%v'", "[6115]", strings.Replace(fmt.Sprintf("%v", r), " ", "", -1)) + } +} + +func randomString() string { + var key string + N := 1 + rand.Int()%16 + for i := 0; i < N; i++ { + r := rand.Int() % 62 + if r < 10 { + key += string(byte('0' + r)) + } else if r-10 < 26 { + key += string(byte('a' + r - 10)) + } else { + key += string(byte('A' + r - 10 - 26)) + } + } + return `"` + key + `"` +} +func randomBool() string { + switch rand.Int() % 2 { + default: + return "false" + case 1: + return "true" + } +} +func randomNumber() string { + return strconv.FormatInt(int64(rand.Int()%1000000), 10) +} + +func randomObjectOrArray(keys []string, prefix string, array bool, depth int) (string, []string) { + N := 5 + rand.Int()%5 + var json string + if array { + json = "[" + } else { + json = "{" + } + for i := 0; i < N; i++ { + if i > 0 { + json += "," + } + var pkey string + if array { + pkey = prefix + "." + strconv.FormatInt(int64(i), 10) + } else { + key := randomString() + pkey = prefix + "." + key[1:len(key)-1] + json += key + `:` + } + keys = append(keys, pkey[1:]) + var kind int + if depth == 5 { + kind = rand.Int() % 4 + } else { + kind = rand.Int() % 6 + } + switch kind { + case 0: + json += randomString() + case 1: + json += randomBool() + case 2: + json += "null" + case 3: + json += randomNumber() + case 4: + var njson string + njson, keys = randomObjectOrArray(keys, pkey, true, depth+1) + json += njson + case 5: + var njson string + njson, keys = randomObjectOrArray(keys, pkey, false, depth+1) + json += njson + } + + } + if array { + json += "]" + } else { + json += "}" + } + return json, keys +} + +func randomJSON() (json string, keys []string) { + return randomObjectOrArray(nil, "", false, 0) +} + +func TestIssue55(t *testing.T) { + json := `{"one": {"two": 2, "three": 3}, "four": 4, "five": 5}` + results := GetMany(json, "four", "five", "one.two", "one.six") + expected := []string{"4", "5", "2", ""} + for i, r := range results { + if r.String() != expected[i] { + t.Fatalf("expected %v, got %v", expected[i], r.String()) + } + } +} +func TestIssue58(t *testing.T) { + json := `{"data":[{"uid": 1},{"uid": 2}]}` + res := Get(json, `data.#[uid!=1]`).Raw + if res != `{"uid": 2}` { + t.Fatalf("expected '%v', got '%v'", `{"uid": 1}`, res) + } +} + +func TestObjectGrouping(t *testing.T) { + json := ` +[ + true, + {"name":"tom"}, + false, + {"name":"janet"}, + null +] +` + res := Get(json, "#.name") + if res.String() != `["tom","janet"]` { + t.Fatalf("expected '%v', got '%v'", `["tom","janet"]`, res.String()) + } +} + +func TestJSONLines(t *testing.T) { + json := ` +true +false +{"name":"tom"} +[1,2,3,4,5] +{"name":"janet"} +null +12930.1203 + ` + paths := []string{"..#", "..0", "..2.name", "..#.name", "..6", "..7"} + ress := []string{"7", "true", "tom", `["tom","janet"]`, "12930.1203", ""} + for i, path := range paths { + res := Get(json, path) + if res.String() != ress[i] { + t.Fatalf("expected '%v', got '%v'", ress[i], res.String()) + } + } + + json = ` +{"name": "Gilbert", "wins": [["straight", "7♣"], ["one pair", "10♥"]]} +{"name": "Alexa", "wins": [["two pair", "4♠"], ["two pair", "9♠"]]} +{"name": "May", "wins": []} +{"name": "Deloise", "wins": [["three of a kind", "5♣"]]} +` + + var i int + lines := strings.Split(strings.TrimSpace(json), "\n") + ForEachLine(json, func(line Result) bool { + if line.Raw != lines[i] { + t.Fatalf("expected '%v', got '%v'", lines[i], line.Raw) + } + i++ + return true + }) + if i != 4 { + t.Fatalf("expected '%v', got '%v'", 4, i) + } + +} + +func TestNumUint64String(t *testing.T) { + i := 9007199254740993 //2^53 + 1 + j := fmt.Sprintf(`{"data": [ %d, "hello" ] }`, i) + res := Get(j, "data.0") + if res.String() != "9007199254740993" { + t.Fatalf("expected '%v', got '%v'", "9007199254740993", res.String()) + } +} + +func TestNumInt64String(t *testing.T) { + i := -9007199254740993 + j := fmt.Sprintf(`{"data":[ "hello", %d ]}`, i) + res := Get(j, "data.1") + if res.String() != "-9007199254740993" { + t.Fatalf("expected '%v', got '%v'", "-9007199254740993", res.String()) + } +} + +func TestNumBigString(t *testing.T) { + i := "900719925474099301239109123101" // very big + j := fmt.Sprintf(`{"data":[ "hello", "%s" ]}`, i) + res := Get(j, "data.1") + if res.String() != "900719925474099301239109123101" { + t.Fatalf("expected '%v', got '%v'", "900719925474099301239109123101", res.String()) + } +} + +func TestNumFloatString(t *testing.T) { + i := -9007199254740993 + j := fmt.Sprintf(`{"data":[ "hello", %d ]}`, i) //No quotes around value!! + res := Get(j, "data.1") + if res.String() != "-9007199254740993" { + t.Fatalf("expected '%v', got '%v'", "-9007199254740993", res.String()) + } +} + +func TestDuplicateKeys(t *testing.T) { + // this is vaild json according to the JSON spec + var json = `{"name": "Alex","name": "Peter"}` + if Parse(json).Get("name").String() != + Parse(json).Map()["name"].String() { + t.Fatalf("expected '%v', got '%v'", + Parse(json).Get("name").String(), + Parse(json).Map()["name"].String(), + ) + } + if !Valid(json) { + t.Fatal("should be valid") + } +} + +func TestArrayValues(t *testing.T) { + var json = `{"array": ["PERSON1","PERSON2",0],}` + values := Get(json, "array").Array() + var output string + for i, val := range values { + if i > 0 { + output += "\n" + } + output += fmt.Sprintf("%#v", val) + } + expect := strings.Join([]string{ + `gjson.Result{Type:3, Raw:"\"PERSON1\"", Str:"PERSON1", Num:0, Index:0}`, + `gjson.Result{Type:3, Raw:"\"PERSON2\"", Str:"PERSON2", Num:0, Index:0}`, + `gjson.Result{Type:2, Raw:"0", Str:"", Num:0, Index:0}`, + }, "\n") + if output != expect { + t.Fatalf("expected '%v', got '%v'", expect, output) + } + +} diff --git a/vendor/github.com/tidwall/sjson/README.md b/vendor/github.com/tidwall/sjson/README.md index 1a7c5c42..aeb940e9 100644 --- a/vendor/github.com/tidwall/sjson/README.md +++ b/vendor/github.com/tidwall/sjson/README.md @@ -7,12 +7,12 @@ GoDoc

-

set a json value quickly

+

set a json value quickly

SJSON is a Go package that provides a [very fast](#performance) and simple way to set a value in a json document. The purpose for this library is to provide efficient json updating for the [SummitDB](https://github.com/tidwall/summitdb) project. For quickly retrieving json values check out [GJSON](https://github.com/tidwall/gjson). -For a command line interface check out [JSONed](https://github.com/tidwall/jsoned). +For a command line interface check out [JJ](https://github.com/tidwall/jj). Getting Started =============== @@ -59,7 +59,7 @@ Path syntax ----------- A path is a series of keys separated by a dot. -The dot and colon characters can be escaped with '\'. +The dot and colon characters can be escaped with ``\``. ```json { @@ -268,7 +268,7 @@ widget.image.hOffset widget.text.onMouseUp ``` -*These benchmarks were run on a MacBook Pro 15" 2.8 GHz Intel Core i7 using Go 1.7.* +*These benchmarks were run on a MacBook Pro 15" 2.8 GHz Intel Core i7 using Go 1.7 and can be be found [here](https://github.com/tidwall/sjson-benchmarks)*. ## Contact Josh Baker [@tidwall](http://twitter.com/tidwall) diff --git a/vendor/github.com/tidwall/sjson/sjson.go b/vendor/github.com/tidwall/sjson/sjson.go index 7f1d3588..245194ed 100644 --- a/vendor/github.com/tidwall/sjson/sjson.go +++ b/vendor/github.com/tidwall/sjson/sjson.go @@ -3,9 +3,7 @@ package sjson import ( jsongo "encoding/json" - "reflect" "strconv" - "unsafe" "github.com/tidwall/gjson" ) @@ -36,6 +34,7 @@ type Options struct { type pathResult struct { part string // current key part + gpart string // gjson get part path string // remaining path force bool // force a string key more bool // there is more path to parse @@ -50,6 +49,7 @@ func parsePath(path string) (pathResult, error) { for i := 0; i < len(path); i++ { if path[i] == '.' { r.part = path[:i] + r.gpart = path[:i] r.path = path[i+1:] r.more = true return r, nil @@ -63,19 +63,24 @@ func parsePath(path string) (pathResult, error) { // go into escape mode. this is a slower path that // strips off the escape character from the part. epart := []byte(path[:i]) + gpart := []byte(path[:i+1]) i++ if i < len(path) { epart = append(epart, path[i]) + gpart = append(gpart, path[i]) i++ for ; i < len(path); i++ { if path[i] == '\\' { + gpart = append(gpart, '\\') i++ if i < len(path) { epart = append(epart, path[i]) + gpart = append(gpart, path[i]) } continue } else if path[i] == '.' { r.part = string(epart) + r.gpart = string(gpart) r.path = path[i+1:] r.more = true return r, nil @@ -87,20 +92,23 @@ func parsePath(path string) (pathResult, error) { "array access character not allowed in path"} } epart = append(epart, path[i]) + gpart = append(gpart, path[i]) } } // append the last part r.part = string(epart) + r.gpart = string(gpart) return r, nil } } r.part = path + r.gpart = path return r, nil } func mustMarshalString(s string) bool { for i := 0; i < len(s); i++ { - if s[i] < ' ' || s[i] > 0x7f || s[i] == '"' { + if s[i] < ' ' || s[i] > 0x7f || s[i] == '"' || (s[i] == '\\' && i == len(s)-1) { return true } } @@ -208,7 +216,7 @@ loop: for ; i >= 0; i-- { if buf[i] == '"' { i-- - if i >= 0 && i == '\\' { + if i >= 0 && buf[i] == '\\' { i-- continue } @@ -249,7 +257,7 @@ func appendRawPaths(buf []byte, jstr string, paths []pathResult, raw string, } } if !found { - res = gjson.Get(jstr, paths[0].part) + res = gjson.Get(jstr, paths[0].gpart) } if res.Index > 0 { if len(paths) > 1 { @@ -400,72 +408,6 @@ func isOptimisticPath(path string) bool { return true } -func set(jstr, path, raw string, - stringify, del, optimistic, inplace bool) ([]byte, error) { - if path == "" { - return nil, &errorType{"path cannot be empty"} - } - if !del && optimistic && isOptimisticPath(path) { - res := gjson.Get(jstr, path) - if res.Exists() && res.Index > 0 { - sz := len(jstr) - len(res.Raw) + len(raw) - if stringify { - sz += 2 - } - if inplace && sz <= len(jstr) { - if !stringify || !mustMarshalString(raw) { - jsonh := *(*reflect.StringHeader)(unsafe.Pointer(&jstr)) - jsonbh := reflect.SliceHeader{ - Data: jsonh.Data, Len: jsonh.Len, Cap: jsonh.Len} - jbytes := *(*[]byte)(unsafe.Pointer(&jsonbh)) - if stringify { - jbytes[res.Index] = '"' - copy(jbytes[res.Index+1:], []byte(raw)) - jbytes[res.Index+1+len(raw)] = '"' - copy(jbytes[res.Index+1+len(raw)+1:], - jbytes[res.Index+len(res.Raw):]) - } else { - copy(jbytes[res.Index:], []byte(raw)) - copy(jbytes[res.Index+len(raw):], - jbytes[res.Index+len(res.Raw):]) - } - return jbytes[:sz], nil - } - return nil, nil - } - buf := make([]byte, 0, sz) - buf = append(buf, jstr[:res.Index]...) - if stringify { - buf = appendStringify(buf, raw) - } else { - buf = append(buf, raw...) - } - buf = append(buf, jstr[res.Index+len(res.Raw):]...) - return buf, nil - } - } - // parse the path, make sure that it does not contain invalid characters - // such as '#', '?', '*' - paths := make([]pathResult, 0, 4) - r, err := parsePath(path) - if err != nil { - return nil, err - } - paths = append(paths, r) - for r.more { - if r, err = parsePath(r.path); err != nil { - return nil, err - } - paths = append(paths, r) - } - - njson, err := appendRawPaths(nil, jstr, paths, raw, stringify, del) - if err != nil { - return nil, err - } - return njson, nil -} - // Set sets a json value for the specified path. // A path is in dot syntax, such as "name.last" or "age". // This function expects that the json is well-formed, and does not validate. @@ -491,29 +433,6 @@ func Set(json, path string, value interface{}) (string, error) { return SetOptions(json, path, value, nil) } -// SetOptions sets a json value for the specified path with options. -// A path is in dot syntax, such as "name.last" or "age". -// This function expects that the json is well-formed, and does not validate. -// Invalid json will not panic, but it may return back unexpected results. -// An error is returned if the path is not valid. -func SetOptions(json, path string, value interface{}, - opts *Options) (string, error) { - if opts != nil { - if opts.ReplaceInPlace { - // it's not safe to replace bytes in-place for strings - // copy the Options and set options.ReplaceInPlace to false. - nopts := *opts - opts = &nopts - opts.ReplaceInPlace = false - } - } - jsonh := *(*reflect.StringHeader)(unsafe.Pointer(&json)) - jsonbh := reflect.SliceHeader{Data: jsonh.Data, Len: jsonh.Len} - jsonb := *(*[]byte)(unsafe.Pointer(&jsonbh)) - res, err := SetBytesOptions(jsonb, path, value, opts) - return string(res), err -} - // SetBytes sets a json value for the specified path. // If working with bytes, this method preferred over // Set(string(data), path, value) @@ -521,77 +440,6 @@ func SetBytes(json []byte, path string, value interface{}) ([]byte, error) { return SetBytesOptions(json, path, value, nil) } -// SetBytesOptions sets a json value for the specified path with options. -// If working with bytes, this method preferred over -// SetOptions(string(data), path, value) -func SetBytesOptions(json []byte, path string, value interface{}, - opts *Options) ([]byte, error) { - var optimistic, inplace bool - if opts != nil { - optimistic = opts.Optimistic - inplace = opts.ReplaceInPlace - } - jstr := *(*string)(unsafe.Pointer(&json)) - var res []byte - var err error - switch v := value.(type) { - default: - b, err := jsongo.Marshal(value) - if err != nil { - return nil, err - } - raw := *(*string)(unsafe.Pointer(&b)) - res, err = set(jstr, path, raw, false, false, optimistic, inplace) - case dtype: - res, err = set(jstr, path, "", false, true, optimistic, inplace) - case string: - res, err = set(jstr, path, v, true, false, optimistic, inplace) - case []byte: - raw := *(*string)(unsafe.Pointer(&v)) - res, err = set(jstr, path, raw, true, false, optimistic, inplace) - case bool: - if v { - res, err = set(jstr, path, "true", false, false, optimistic, inplace) - } else { - res, err = set(jstr, path, "false", false, false, optimistic, inplace) - } - case int8: - res, err = set(jstr, path, strconv.FormatInt(int64(v), 10), - false, false, optimistic, inplace) - case int16: - res, err = set(jstr, path, strconv.FormatInt(int64(v), 10), - false, false, optimistic, inplace) - case int32: - res, err = set(jstr, path, strconv.FormatInt(int64(v), 10), - false, false, optimistic, inplace) - case int64: - res, err = set(jstr, path, strconv.FormatInt(int64(v), 10), - false, false, optimistic, inplace) - case uint8: - res, err = set(jstr, path, strconv.FormatUint(uint64(v), 10), - false, false, optimistic, inplace) - case uint16: - res, err = set(jstr, path, strconv.FormatUint(uint64(v), 10), - false, false, optimistic, inplace) - case uint32: - res, err = set(jstr, path, strconv.FormatUint(uint64(v), 10), - false, false, optimistic, inplace) - case uint64: - res, err = set(jstr, path, strconv.FormatUint(uint64(v), 10), - false, false, optimistic, inplace) - case float32: - res, err = set(jstr, path, strconv.FormatFloat(float64(v), 'f', -1, 64), - false, false, optimistic, inplace) - case float64: - res, err = set(jstr, path, strconv.FormatFloat(float64(v), 'f', -1, 64), - false, false, optimistic, inplace) - } - if err == errNoChange { - return json, nil - } - return res, err -} - // SetRaw sets a raw json value for the specified path. // This function works the same as Set except that the value is set as a // raw block of json. This allows for setting premarshalled json objects. @@ -621,25 +469,6 @@ func SetRawBytes(json []byte, path string, value []byte) ([]byte, error) { return SetRawBytesOptions(json, path, value, nil) } -// SetRawBytesOptions sets a raw json value for the specified path with options. -// If working with bytes, this method preferred over -// SetRawOptions(string(data), path, value, opts) -func SetRawBytesOptions(json []byte, path string, value []byte, - opts *Options) ([]byte, error) { - jstr := *(*string)(unsafe.Pointer(&json)) - vstr := *(*string)(unsafe.Pointer(&value)) - var optimistic, inplace bool - if opts != nil { - optimistic = opts.Optimistic - inplace = opts.ReplaceInPlace - } - res, err := set(jstr, path, vstr, false, false, optimistic, inplace) - if err == errNoChange { - return json, nil - } - return res, err -} - type dtype struct{} // Delete deletes a value from json for the specified path. diff --git a/vendor/github.com/tidwall/sjson/sjson_gae.go b/vendor/github.com/tidwall/sjson/sjson_gae.go new file mode 100644 index 00000000..4b83a17c --- /dev/null +++ b/vendor/github.com/tidwall/sjson/sjson_gae.go @@ -0,0 +1,196 @@ +//+build appengine + +package sjson + +import ( + jsongo "encoding/json" + "strconv" + + "github.com/tidwall/gjson" +) + +func set(jstr, path, raw string, + stringify, del, optimistic, inplace bool) ([]byte, error) { + if path == "" { + return nil, &errorType{"path cannot be empty"} + } + if !del && optimistic && isOptimisticPath(path) { + res := gjson.Get(jstr, path) + if res.Exists() && res.Index > 0 { + sz := len(jstr) - len(res.Raw) + len(raw) + if stringify { + sz += 2 + } + if inplace && sz <= len(jstr) { + if !stringify || !mustMarshalString(raw) { + // jsonh := *(*reflect.StringHeader)(unsafe.Pointer(&jstr)) + // jsonbh := reflect.SliceHeader{ + // Data: jsonh.Data, Len: jsonh.Len, Cap: jsonh.Len} + // jbytes := *(*[]byte)(unsafe.Pointer(&jsonbh)) + jbytes := []byte(jstr) + if stringify { + jbytes[res.Index] = '"' + copy(jbytes[res.Index+1:], []byte(raw)) + jbytes[res.Index+1+len(raw)] = '"' + copy(jbytes[res.Index+1+len(raw)+1:], + jbytes[res.Index+len(res.Raw):]) + } else { + copy(jbytes[res.Index:], []byte(raw)) + copy(jbytes[res.Index+len(raw):], + jbytes[res.Index+len(res.Raw):]) + } + return jbytes[:sz], nil + } + return nil, nil + } + buf := make([]byte, 0, sz) + buf = append(buf, jstr[:res.Index]...) + if stringify { + buf = appendStringify(buf, raw) + } else { + buf = append(buf, raw...) + } + buf = append(buf, jstr[res.Index+len(res.Raw):]...) + return buf, nil + } + } + // parse the path, make sure that it does not contain invalid characters + // such as '#', '?', '*' + paths := make([]pathResult, 0, 4) + r, err := parsePath(path) + if err != nil { + return nil, err + } + paths = append(paths, r) + for r.more { + if r, err = parsePath(r.path); err != nil { + return nil, err + } + paths = append(paths, r) + } + + njson, err := appendRawPaths(nil, jstr, paths, raw, stringify, del) + if err != nil { + return nil, err + } + return njson, nil +} + +// SetOptions sets a json value for the specified path with options. +// A path is in dot syntax, such as "name.last" or "age". +// This function expects that the json is well-formed, and does not validate. +// Invalid json will not panic, but it may return back unexpected results. +// An error is returned if the path is not valid. +func SetOptions(json, path string, value interface{}, + opts *Options) (string, error) { + if opts != nil { + if opts.ReplaceInPlace { + // it's not safe to replace bytes in-place for strings + // copy the Options and set options.ReplaceInPlace to false. + nopts := *opts + opts = &nopts + opts.ReplaceInPlace = false + } + } + // jsonh := *(*reflect.StringHeader)(unsafe.Pointer(&json)) + // jsonbh := reflect.SliceHeader{Data: jsonh.Data, Len: jsonh.Len} + // jsonb := *(*[]byte)(unsafe.Pointer(&jsonbh)) + jsonb := []byte(json) + res, err := SetBytesOptions(jsonb, path, value, opts) + return string(res), err +} + +// SetBytesOptions sets a json value for the specified path with options. +// If working with bytes, this method preferred over +// SetOptions(string(data), path, value) +func SetBytesOptions(json []byte, path string, value interface{}, + opts *Options) ([]byte, error) { + var optimistic, inplace bool + if opts != nil { + optimistic = opts.Optimistic + inplace = opts.ReplaceInPlace + } + // jstr := *(*string)(unsafe.Pointer(&json)) + jstr := string(json) + var res []byte + var err error + switch v := value.(type) { + default: + b, err := jsongo.Marshal(value) + if err != nil { + return nil, err + } + // raw := *(*string)(unsafe.Pointer(&b)) + raw := string(b) + res, err = set(jstr, path, raw, false, false, optimistic, inplace) + case dtype: + res, err = set(jstr, path, "", false, true, optimistic, inplace) + case string: + res, err = set(jstr, path, v, true, false, optimistic, inplace) + case []byte: + // raw := *(*string)(unsafe.Pointer(&v)) + raw := string(v) + res, err = set(jstr, path, raw, true, false, optimistic, inplace) + case bool: + if v { + res, err = set(jstr, path, "true", false, false, optimistic, inplace) + } else { + res, err = set(jstr, path, "false", false, false, optimistic, inplace) + } + case int8: + res, err = set(jstr, path, strconv.FormatInt(int64(v), 10), + false, false, optimistic, inplace) + case int16: + res, err = set(jstr, path, strconv.FormatInt(int64(v), 10), + false, false, optimistic, inplace) + case int32: + res, err = set(jstr, path, strconv.FormatInt(int64(v), 10), + false, false, optimistic, inplace) + case int64: + res, err = set(jstr, path, strconv.FormatInt(int64(v), 10), + false, false, optimistic, inplace) + case uint8: + res, err = set(jstr, path, strconv.FormatUint(uint64(v), 10), + false, false, optimistic, inplace) + case uint16: + res, err = set(jstr, path, strconv.FormatUint(uint64(v), 10), + false, false, optimistic, inplace) + case uint32: + res, err = set(jstr, path, strconv.FormatUint(uint64(v), 10), + false, false, optimistic, inplace) + case uint64: + res, err = set(jstr, path, strconv.FormatUint(uint64(v), 10), + false, false, optimistic, inplace) + case float32: + res, err = set(jstr, path, strconv.FormatFloat(float64(v), 'f', -1, 64), + false, false, optimistic, inplace) + case float64: + res, err = set(jstr, path, strconv.FormatFloat(float64(v), 'f', -1, 64), + false, false, optimistic, inplace) + } + if err == errNoChange { + return json, nil + } + return res, err +} + +// SetRawBytesOptions sets a raw json value for the specified path with options. +// If working with bytes, this method preferred over +// SetRawOptions(string(data), path, value, opts) +func SetRawBytesOptions(json []byte, path string, value []byte, + opts *Options) ([]byte, error) { + // jstr := *(*string)(unsafe.Pointer(&json)) + // vstr := *(*string)(unsafe.Pointer(&value)) + jstr := string(json) + vstr := string(value) + var optimistic, inplace bool + if opts != nil { + optimistic = opts.Optimistic + inplace = opts.ReplaceInPlace + } + res, err := set(jstr, path, vstr, false, false, optimistic, inplace) + if err == errNoChange { + return json, nil + } + return res, err +} diff --git a/vendor/github.com/tidwall/sjson/sjson_ngae.go b/vendor/github.com/tidwall/sjson/sjson_ngae.go new file mode 100644 index 00000000..f3624f43 --- /dev/null +++ b/vendor/github.com/tidwall/sjson/sjson_ngae.go @@ -0,0 +1,191 @@ +//+build !appengine + +package sjson + +import ( + jsongo "encoding/json" + "reflect" + "strconv" + "unsafe" + + "github.com/tidwall/gjson" +) + +func set(jstr, path, raw string, + stringify, del, optimistic, inplace bool) ([]byte, error) { + if path == "" { + return nil, &errorType{"path cannot be empty"} + } + if !del && optimistic && isOptimisticPath(path) { + res := gjson.Get(jstr, path) + if res.Exists() && res.Index > 0 { + sz := len(jstr) - len(res.Raw) + len(raw) + if stringify { + sz += 2 + } + if inplace && sz <= len(jstr) { + if !stringify || !mustMarshalString(raw) { + jsonh := *(*reflect.StringHeader)(unsafe.Pointer(&jstr)) + jsonbh := reflect.SliceHeader{ + Data: jsonh.Data, Len: jsonh.Len, Cap: jsonh.Len} + jbytes := *(*[]byte)(unsafe.Pointer(&jsonbh)) + if stringify { + jbytes[res.Index] = '"' + copy(jbytes[res.Index+1:], []byte(raw)) + jbytes[res.Index+1+len(raw)] = '"' + copy(jbytes[res.Index+1+len(raw)+1:], + jbytes[res.Index+len(res.Raw):]) + } else { + copy(jbytes[res.Index:], []byte(raw)) + copy(jbytes[res.Index+len(raw):], + jbytes[res.Index+len(res.Raw):]) + } + return jbytes[:sz], nil + } + return nil, nil + } + buf := make([]byte, 0, sz) + buf = append(buf, jstr[:res.Index]...) + if stringify { + buf = appendStringify(buf, raw) + } else { + buf = append(buf, raw...) + } + buf = append(buf, jstr[res.Index+len(res.Raw):]...) + return buf, nil + } + } + // parse the path, make sure that it does not contain invalid characters + // such as '#', '?', '*' + paths := make([]pathResult, 0, 4) + r, err := parsePath(path) + if err != nil { + return nil, err + } + paths = append(paths, r) + for r.more { + if r, err = parsePath(r.path); err != nil { + return nil, err + } + paths = append(paths, r) + } + + njson, err := appendRawPaths(nil, jstr, paths, raw, stringify, del) + if err != nil { + return nil, err + } + return njson, nil +} + +// SetOptions sets a json value for the specified path with options. +// A path is in dot syntax, such as "name.last" or "age". +// This function expects that the json is well-formed, and does not validate. +// Invalid json will not panic, but it may return back unexpected results. +// An error is returned if the path is not valid. +func SetOptions(json, path string, value interface{}, + opts *Options) (string, error) { + if opts != nil { + if opts.ReplaceInPlace { + // it's not safe to replace bytes in-place for strings + // copy the Options and set options.ReplaceInPlace to false. + nopts := *opts + opts = &nopts + opts.ReplaceInPlace = false + } + } + jsonh := *(*reflect.StringHeader)(unsafe.Pointer(&json)) + jsonbh := reflect.SliceHeader{Data: jsonh.Data, Len: jsonh.Len} + jsonb := *(*[]byte)(unsafe.Pointer(&jsonbh)) + res, err := SetBytesOptions(jsonb, path, value, opts) + return string(res), err +} + +// SetBytesOptions sets a json value for the specified path with options. +// If working with bytes, this method preferred over +// SetOptions(string(data), path, value) +func SetBytesOptions(json []byte, path string, value interface{}, + opts *Options) ([]byte, error) { + var optimistic, inplace bool + if opts != nil { + optimistic = opts.Optimistic + inplace = opts.ReplaceInPlace + } + jstr := *(*string)(unsafe.Pointer(&json)) + var res []byte + var err error + switch v := value.(type) { + default: + b, err := jsongo.Marshal(value) + if err != nil { + return nil, err + } + raw := *(*string)(unsafe.Pointer(&b)) + res, err = set(jstr, path, raw, false, false, optimistic, inplace) + case dtype: + res, err = set(jstr, path, "", false, true, optimistic, inplace) + case string: + res, err = set(jstr, path, v, true, false, optimistic, inplace) + case []byte: + raw := *(*string)(unsafe.Pointer(&v)) + res, err = set(jstr, path, raw, true, false, optimistic, inplace) + case bool: + if v { + res, err = set(jstr, path, "true", false, false, optimistic, inplace) + } else { + res, err = set(jstr, path, "false", false, false, optimistic, inplace) + } + case int8: + res, err = set(jstr, path, strconv.FormatInt(int64(v), 10), + false, false, optimistic, inplace) + case int16: + res, err = set(jstr, path, strconv.FormatInt(int64(v), 10), + false, false, optimistic, inplace) + case int32: + res, err = set(jstr, path, strconv.FormatInt(int64(v), 10), + false, false, optimistic, inplace) + case int64: + res, err = set(jstr, path, strconv.FormatInt(int64(v), 10), + false, false, optimistic, inplace) + case uint8: + res, err = set(jstr, path, strconv.FormatUint(uint64(v), 10), + false, false, optimistic, inplace) + case uint16: + res, err = set(jstr, path, strconv.FormatUint(uint64(v), 10), + false, false, optimistic, inplace) + case uint32: + res, err = set(jstr, path, strconv.FormatUint(uint64(v), 10), + false, false, optimistic, inplace) + case uint64: + res, err = set(jstr, path, strconv.FormatUint(uint64(v), 10), + false, false, optimistic, inplace) + case float32: + res, err = set(jstr, path, strconv.FormatFloat(float64(v), 'f', -1, 64), + false, false, optimistic, inplace) + case float64: + res, err = set(jstr, path, strconv.FormatFloat(float64(v), 'f', -1, 64), + false, false, optimistic, inplace) + } + if err == errNoChange { + return json, nil + } + return res, err +} + +// SetRawBytesOptions sets a raw json value for the specified path with options. +// If working with bytes, this method preferred over +// SetRawOptions(string(data), path, value, opts) +func SetRawBytesOptions(json []byte, path string, value []byte, + opts *Options) ([]byte, error) { + jstr := *(*string)(unsafe.Pointer(&json)) + vstr := *(*string)(unsafe.Pointer(&value)) + var optimistic, inplace bool + if opts != nil { + optimistic = opts.Optimistic + inplace = opts.ReplaceInPlace + } + res, err := set(jstr, path, vstr, false, false, optimistic, inplace) + if err == errNoChange { + return json, nil + } + return res, err +} diff --git a/vendor/github.com/tidwall/sjson/sjson_test.go b/vendor/github.com/tidwall/sjson/sjson_test.go index a7a8f7e1..75f1c4d3 100644 --- a/vendor/github.com/tidwall/sjson/sjson_test.go +++ b/vendor/github.com/tidwall/sjson/sjson_test.go @@ -1,20 +1,13 @@ package sjson import ( - "bytes" "encoding/hex" - gojson "encoding/json" "fmt" "math/rand" - "strings" "testing" "time" - "github.com/Jeffail/gabs" - - jlexer "github.com/mailru/easyjson/jlexer" - jwriter "github.com/mailru/easyjson/jwriter" - fflib "github.com/pquerna/ffjson/fflib/v1" + "github.com/tidwall/gjson" ) func TestInvalidPaths(t *testing.T) { @@ -128,6 +121,10 @@ func TestBasic(t *testing.T) { `{":\1":{"this":[null,null,null,null,{".HI":4}]}}`, ``, "\\:\\\\1.this.4.\\.HI", `4`) + testRaw(t, setRaw, + `{"app.token":"cde"}`, + `{"app.token":"abc"}`, + "app\\.token", `"cde"`) testRaw(t, setRaw, `{"b":{"this":{"😇":""}}}`, ``, @@ -142,6 +139,7 @@ func TestBasic(t *testing.T) { testRaw(t, setBool, `[true]`, ``, `0`, true) testRaw(t, setBool, `[null]`, ``, `0`, nil) testRaw(t, setString, `{"arr":[1]}`, ``, `arr.-1`, 1) + testRaw(t, setString, `{"a":"\\"}`, ``, `a`, "\\") } func TestDelete(t *testing.T) { @@ -178,1062 +176,124 @@ func TestRandomData(t *testing.T) { } } -var exampleJSON = ` -{ - "sha": "d25341478381063d1c76e81b3a52e0592a7c997f", - "commit": { - "author": { - "name": "Tom Tom Anderson", - "email": "tomtom@anderson.edu", - "date": "2013-06-22T16:30:59Z" - }, - "committer": { - "name": "Tom Tom Anderson", - "email": "jeffditto@anderson.edu", - "date": "2013-06-22T16:30:59Z" - }, - "message": "Merge pull request #162 from stedolan/utf8-fixes\n\nUtf8 fixes. Closes #161", - "tree": { - "sha": "6ab697a8dfb5a96e124666bf6d6213822599fb40", - "url": "https://api.github.com/repos/stedolan/jq/git/trees/6ab697a8dfb5a96e124666bf6d6213822599fb40" - }, - "url": "https://api.github.com/repos/stedolan/jq/git/commits/d25341478381063d1c76e81b3a52e0592a7c997f", - "comment_count": 0 - } -} -` -var path = "commit.committer.email" -var value = "tomtom@anderson.com" -var rawValue = `"tomtom@anderson.com"` -var rawValueBytes = []byte(rawValue) -var expect = strings.Replace(exampleJSON, "jeffditto@anderson.edu", "tomtom@anderson.com", 1) -var jsonBytes = []byte(exampleJSON) -var jsonBytes2 = []byte(exampleJSON) -var expectBytes = []byte(expect) -var opts = &Options{Optimistic: true} -var optsInPlace = &Options{Optimistic: true, ReplaceInPlace: true} - -func BenchmarkSet(t *testing.B) { - t.ReportAllocs() - for i := 0; i < t.N; i++ { - res, err := Set(exampleJSON, path, value) - if err != nil { - t.Fatal(err) - } - if res != expect { - t.Fatal("expected '%v', got '%v'", expect, res) - } - } -} - -func BenchmarkSetRaw(t *testing.B) { - t.ReportAllocs() - for i := 0; i < t.N; i++ { - res, err := SetRaw(exampleJSON, path, rawValue) - if err != nil { - t.Fatal(err) - } - if res != expect { - t.Fatal("expected '%v', got '%v'", expect, res) - } - } -} - -func BenchmarkSetBytes(t *testing.B) { - t.ReportAllocs() - for i := 0; i < t.N; i++ { - res, err := SetBytes(jsonBytes, path, value) - if err != nil { - t.Fatal(err) - } - if bytes.Compare(res, expectBytes) != 0 { - t.Fatal("expected '%v', got '%v'", expect, res) - } - } -} - -func BenchmarkSetRawBytes(t *testing.B) { - t.ReportAllocs() - for i := 0; i < t.N; i++ { - res, err := SetRawBytes(jsonBytes, path, rawValueBytes) - if err != nil { - t.Fatal(err) - } - if bytes.Compare(res, expectBytes) != 0 { - t.Fatal("expected '%v', got '%v'", expect, res) - } - } -} - -func BenchmarkSetOptimistic(t *testing.B) { - t.ReportAllocs() - for i := 0; i < t.N; i++ { - res, err := SetOptions(exampleJSON, path, value, opts) - if err != nil { - t.Fatal(err) - } - if res != expect { - t.Fatal("expected '%v', got '%v'", expect, res) - } - } -} - -func BenchmarkSetInPlace(t *testing.B) { - t.ReportAllocs() - for i := 0; i < t.N; i++ { - res, err := SetOptions(exampleJSON, path, value, optsInPlace) - if err != nil { - t.Fatal(err) - } - if res != expect { - t.Fatal("expected '%v', got '%v'", expect, res) - } - } -} - -func BenchmarkSetRawOptimistic(t *testing.B) { - t.ReportAllocs() - for i := 0; i < t.N; i++ { - res, err := SetRawOptions(exampleJSON, path, rawValue, opts) - if err != nil { - t.Fatal(err) - } - if res != expect { - t.Fatal("expected '%v', got '%v'", expect, res) - } - } -} - -func BenchmarkSetRawInPlace(t *testing.B) { - t.ReportAllocs() - for i := 0; i < t.N; i++ { - res, err := SetRawOptions(exampleJSON, path, rawValue, optsInPlace) - if err != nil { - t.Fatal(err) - } - if res != expect { - t.Fatal("expected '%v', got '%v'", expect, res) - } - } -} - -func BenchmarkSetBytesOptimistic(t *testing.B) { - t.ReportAllocs() - for i := 0; i < t.N; i++ { - res, err := SetBytesOptions(jsonBytes, path, value, opts) - if err != nil { - t.Fatal(err) - } - if bytes.Compare(res, expectBytes) != 0 { - t.Fatal("expected '%v', got '%v'", string(expectBytes), string(res)) - } - } -} - -func BenchmarkSetBytesInPlace(t *testing.B) { - t.ReportAllocs() - for i := 0; i < t.N; i++ { - copy(jsonBytes2, jsonBytes) - res, err := SetBytesOptions(jsonBytes2, path, value, optsInPlace) - if err != nil { - t.Fatal(err) - } - if bytes.Compare(res, expectBytes) != 0 { - t.Fatal("expected '%v', got '%v'", string(expectBytes), string(res)) - } - } -} - -func BenchmarkSetRawBytesOptimistic(t *testing.B) { - t.ReportAllocs() - for i := 0; i < t.N; i++ { - res, err := SetRawBytesOptions(jsonBytes, path, rawValueBytes, opts) - if err != nil { - t.Fatal(err) - } - if bytes.Compare(res, expectBytes) != 0 { - t.Fatal("expected '%v', got '%v'", string(expectBytes), string(res)) - } - } -} - -func BenchmarkSetRawBytesInPlace(t *testing.B) { - t.ReportAllocs() - for i := 0; i < t.N; i++ { - copy(jsonBytes2, jsonBytes) - res, err := SetRawBytesOptions(jsonBytes2, path, rawValueBytes, optsInPlace) - if err != nil { - t.Fatal(err) - } - if bytes.Compare(res, expectBytes) != 0 { - t.Fatal("expected '%v', got '%v'", string(expectBytes), string(res)) - } - } -} - -const benchJSON = ` -{ - "widget": { - "debug": "on", - "window": { - "title": "Sample Konfabulator Widget", - "name": "main_window", - "width": 500, - "height": 500 - }, - "image": { - "src": "Images/Sun.png", - "hOffset": 250, - "vOffset": 250, - "alignment": "center" - }, - "text": { - "data": "Click Here", - "size": 36, - "style": "bold", - "vOffset": 100, - "alignment": "center", - "onMouseUp": "sun1.opacity = (sun1.opacity / 100) * 90;" - } - } -} -` - -type BenchStruct struct { - Widget struct { - Debug string `json:"debug"` - Window struct { - Title string `json:"title"` - Name string `json:"name"` - Width int `json:"width"` - Height int `json:"height"` - } `json:"window"` - Image struct { - Src string `json:"src"` - HOffset int `json:"hOffset"` - VOffset int `json:"vOffset"` - Alignment string `json:"alignment"` - } `json:"image"` - Text struct { - Data string `json:"data"` - Size int `json:"size"` - Style string `json:"style"` - VOffset int `json:"vOffset"` - Alignment string `json:"alignment"` - OnMouseUp string `json:"onMouseUp"` - } `json:"text"` - } `json:"widget"` -} - -var benchPaths = []string{ - "widget.window.name", - "widget.image.hOffset", - "widget.text.onMouseUp", -} - -func Benchmark_SJSON(t *testing.B) { - opts := Options{Optimistic: true} - t.ReportAllocs() - t.ResetTimer() - for i := 0; i < t.N; i++ { - for _, path := range benchPaths { - var err error - switch path { - case "widget.window.name": - _, err = SetOptions(benchJSON, path, "1", &opts) - case "widget.image.hOffset": - _, err = SetOptions(benchJSON, path, 1, &opts) - case "widget.text.onMouseUp": - _, err = SetOptions(benchJSON, path, "1", &opts) - } - if err != nil { - t.Fatal(err) - } - - } - } - t.N *= len(benchPaths) -} - -func Benchmark_SJSON_ReplaceInPlace(t *testing.B) { - data := []byte(benchJSON) - opts := Options{ - Optimistic: true, - ReplaceInPlace: true, - } - v1, v2 := []byte(`"1"`), []byte("1") - t.ReportAllocs() - t.ResetTimer() - for i := 0; i < t.N; i++ { - for _, path := range benchPaths { - var err error - switch path { - case "widget.window.name": - _, err = SetRawBytesOptions(data, path, v1, &opts) - case "widget.image.hOffset": - _, err = SetRawBytesOptions(data, path, v2, &opts) - case "widget.text.onMouseUp": - _, err = SetRawBytesOptions(data, path, v1, &opts) - } - if err != nil { - t.Fatal(err) - } - - } - } - t.N *= len(benchPaths) -} - -func Benchmark_Encoding_JSON_Map(t *testing.B) { - data := []byte(benchJSON) - t.ReportAllocs() - t.ResetTimer() - for i := 0; i < t.N; i++ { - for _, path := range benchPaths { - var m map[string]interface{} - if err := gojson.Unmarshal(data, &m); err != nil { - t.Fatal(err) - } - switch path { - case "widget.window.name": - m["widget"].(map[string]interface{})["window"].(map[string]interface{})["name"] = "1" - case "widget.image.hOffset": - m["widget"].(map[string]interface{})["image"].(map[string]interface{})["hOffset"] = 1 - case "widget.text.onMouseUp": - m["widget"].(map[string]interface{})["text"].(map[string]interface{})["onMouseUp"] = "1" - } - _, err := gojson.Marshal(&m) - if err != nil { - t.Fatal(err) - } - } - } - t.N *= len(benchPaths) -} - -func Benchmark_Encoding_JSON_Struct(t *testing.B) { - data := []byte(benchJSON) - t.ReportAllocs() - t.ResetTimer() - for i := 0; i < t.N; i++ { - for _, path := range benchPaths { - var v BenchStruct - if err := gojson.Unmarshal(data, &v); err != nil { - t.Fatal(err) - } - switch path { - case "widget.window.name": - v.Widget.Window.Name = "1" - case "widget.image.hOffset": - v.Widget.Image.HOffset = 1 - case "widget.text.onMouseUp": - v.Widget.Text.OnMouseUp = "1" - } - _, err := gojson.Marshal(&v) - if err != nil { - t.Fatal(err) - } - } - } - t.N *= len(benchPaths) -} - -func Benchmark_Gabs(t *testing.B) { - data := []byte(benchJSON) - t.ReportAllocs() - t.ResetTimer() - for i := 0; i < t.N; i++ { - for _, path := range benchPaths { - jsonParsed, err := gabs.ParseJSON(data) - if err != nil { - t.Fatal(err) - } - switch path { - case "widget.window.name": - jsonParsed.SetP("1", path) - case "widget.image.hOffset": - jsonParsed.SetP(1, path) - case "widget.text.onMouseUp": - jsonParsed.SetP("1", path) - } - jsonParsed.String() - } - } - t.N *= len(benchPaths) -} - -func Benchmark_FFJSON(t *testing.B) { - data := []byte(benchJSON) - t.ReportAllocs() - t.ResetTimer() - for i := 0; i < t.N; i++ { - for _, path := range benchPaths { - var v BenchStruct - if err := v.UnmarshalFFJSONFromData(data); err != nil { - t.Fatal(err) - } - switch path { - case "widget.window.name": - v.Widget.Window.Name = "1" - case "widget.image.hOffset": - v.Widget.Image.HOffset = 1 - case "widget.text.onMouseUp": - v.Widget.Text.OnMouseUp = "1" - } - _, err := v.MarshalFFJSONFromData() - if err != nil { - t.Fatal(err) - } - } - } - t.N *= len(benchPaths) -} - -func Benchmark_EasyJSON(t *testing.B) { - data := []byte(benchJSON) - t.ReportAllocs() - t.ResetTimer() - for i := 0; i < t.N; i++ { - for _, path := range benchPaths { - var v BenchStruct - if err := v.UnmarshalEasyJSONFromData(data); err != nil { - t.Fatal(err) - } - switch path { - case "widget.window.name": - v.Widget.Window.Name = "1" - case "widget.image.hOffset": - v.Widget.Image.HOffset = 1 - case "widget.text.onMouseUp": - v.Widget.Text.OnMouseUp = "1" - } - _, err := v.MarshalEasyJSONFromData() - if err != nil { - t.Fatal(err) - } - } - } - t.N *= len(benchPaths) -} - -////////////////////////////////////////////////////////////// -// EVERYTHING BELOW IS AUTOGENERATED - -// suppress unused package warning -var ( - _ = gojson.RawMessage{} - _ = jlexer.Lexer{} - _ = jwriter.Writer{} -) - -func easyjsonDbb23193DecodeGithubComTidwallSjson(in *jlexer.Lexer, out *BenchStruct) { - if in.IsNull() { - in.Skip() - return - } - in.Delim('{') - for !in.IsDelim('}') { - key := in.UnsafeString() - in.WantColon() - if in.IsNull() { - in.Skip() - in.WantComma() - continue - } - switch key { - case "widget": - easyjsonDbb23193Decode(in, &out.Widget) - default: - in.SkipRecursive() - } - in.WantComma() - } - in.Delim('}') -} -func easyjsonDbb23193EncodeGithubComTidwallSjson(out *jwriter.Writer, in BenchStruct) { - out.RawByte('{') - first := true - _ = first - if !first { - out.RawByte(',') - } - first = false - out.RawString("\"widget\":") - easyjsonDbb23193Encode(out, in.Widget) - out.RawByte('}') -} - -// MarshalJSON supports json.Marshaler interface -func (v BenchStruct) MarshalEasyJSONFromData() ([]byte, error) { - w := jwriter.Writer{} - easyjsonDbb23193EncodeGithubComTidwallSjson(&w, v) - return w.Buffer.BuildBytes(), w.Error -} - -// MarshalEasyJSON supports easyjson.Marshaler interface -func (v BenchStruct) MarshalEasyJSON(w *jwriter.Writer) { - easyjsonDbb23193EncodeGithubComTidwallSjson(w, v) -} - -// UnmarshalJSON supports json.Unmarshaler interface -func (v *BenchStruct) UnmarshalEasyJSONFromData(data []byte) error { - r := jlexer.Lexer{Data: data} - easyjsonDbb23193DecodeGithubComTidwallSjson(&r, v) - return r.Error() -} - -// UnmarshalEasyJSON supports easyjson.Unmarshaler interface -func (v *BenchStruct) UnmarshalEasyJSON(l *jlexer.Lexer) { - easyjsonDbb23193DecodeGithubComTidwallSjson(l, v) -} -func easyjsonDbb23193Decode(in *jlexer.Lexer, out *struct { - Debug string "json:\"debug\"" - Window struct { - Title string "json:\"title\"" - Name string "json:\"name\"" - Width int "json:\"width\"" - Height int "json:\"height\"" - } "json:\"window\"" - Image struct { - Src string "json:\"src\"" - HOffset int "json:\"hOffset\"" - VOffset int "json:\"vOffset\"" - Alignment string "json:\"alignment\"" - } "json:\"image\"" - Text struct { - Data string "json:\"data\"" - Size int "json:\"size\"" - Style string "json:\"style\"" - VOffset int "json:\"vOffset\"" - Alignment string "json:\"alignment\"" - OnMouseUp string "json:\"onMouseUp\"" - } "json:\"text\"" -}) { - if in.IsNull() { - in.Skip() - return - } - in.Delim('{') - for !in.IsDelim('}') { - key := in.UnsafeString() - in.WantColon() - if in.IsNull() { - in.Skip() - in.WantComma() - continue - } - switch key { - case "debug": - out.Debug = string(in.String()) - case "window": - easyjsonDbb23193Decode1(in, &out.Window) - case "image": - easyjsonDbb23193Decode2(in, &out.Image) - case "text": - easyjsonDbb23193Decode3(in, &out.Text) - default: - in.SkipRecursive() - } - in.WantComma() - } - in.Delim('}') -} -func easyjsonDbb23193Encode(out *jwriter.Writer, in struct { - Debug string "json:\"debug\"" - Window struct { - Title string "json:\"title\"" - Name string "json:\"name\"" - Width int "json:\"width\"" - Height int "json:\"height\"" - } "json:\"window\"" - Image struct { - Src string "json:\"src\"" - HOffset int "json:\"hOffset\"" - VOffset int "json:\"vOffset\"" - Alignment string "json:\"alignment\"" - } "json:\"image\"" - Text struct { - Data string "json:\"data\"" - Size int "json:\"size\"" - Style string "json:\"style\"" - VOffset int "json:\"vOffset\"" - Alignment string "json:\"alignment\"" - OnMouseUp string "json:\"onMouseUp\"" - } "json:\"text\"" -}) { - out.RawByte('{') - first := true - _ = first - if !first { - out.RawByte(',') - } - first = false - out.RawString("\"debug\":") - out.String(string(in.Debug)) - if !first { - out.RawByte(',') - } - first = false - out.RawString("\"window\":") - easyjsonDbb23193Encode1(out, in.Window) - if !first { - out.RawByte(',') - } - first = false - out.RawString("\"image\":") - easyjsonDbb23193Encode2(out, in.Image) - if !first { - out.RawByte(',') - } - first = false - out.RawString("\"text\":") - easyjsonDbb23193Encode3(out, in.Text) - out.RawByte('}') -} -func easyjsonDbb23193Decode3(in *jlexer.Lexer, out *struct { - Data string "json:\"data\"" - Size int "json:\"size\"" - Style string "json:\"style\"" - VOffset int "json:\"vOffset\"" - Alignment string "json:\"alignment\"" - OnMouseUp string "json:\"onMouseUp\"" -}) { - if in.IsNull() { - in.Skip() - return - } - in.Delim('{') - for !in.IsDelim('}') { - key := in.UnsafeString() - in.WantColon() - if in.IsNull() { - in.Skip() - in.WantComma() - continue - } - switch key { - case "data": - out.Data = string(in.String()) - case "size": - out.Size = int(in.Int()) - case "style": - out.Style = string(in.String()) - case "vOffset": - out.VOffset = int(in.Int()) - case "alignment": - out.Alignment = string(in.String()) - case "onMouseUp": - out.OnMouseUp = string(in.String()) - default: - in.SkipRecursive() - } - in.WantComma() - } - in.Delim('}') -} -func easyjsonDbb23193Encode3(out *jwriter.Writer, in struct { - Data string "json:\"data\"" - Size int "json:\"size\"" - Style string "json:\"style\"" - VOffset int "json:\"vOffset\"" - Alignment string "json:\"alignment\"" - OnMouseUp string "json:\"onMouseUp\"" -}) { - out.RawByte('{') - first := true - _ = first - if !first { - out.RawByte(',') - } - first = false - out.RawString("\"data\":") - out.String(string(in.Data)) - if !first { - out.RawByte(',') - } - first = false - out.RawString("\"size\":") - out.Int(int(in.Size)) - if !first { - out.RawByte(',') - } - first = false - out.RawString("\"style\":") - out.String(string(in.Style)) - if !first { - out.RawByte(',') - } - first = false - out.RawString("\"vOffset\":") - out.Int(int(in.VOffset)) - if !first { - out.RawByte(',') - } - first = false - out.RawString("\"alignment\":") - out.String(string(in.Alignment)) - if !first { - out.RawByte(',') - } - first = false - out.RawString("\"onMouseUp\":") - out.String(string(in.OnMouseUp)) - out.RawByte('}') -} -func easyjsonDbb23193Decode2(in *jlexer.Lexer, out *struct { - Src string "json:\"src\"" - HOffset int "json:\"hOffset\"" - VOffset int "json:\"vOffset\"" - Alignment string "json:\"alignment\"" -}) { - if in.IsNull() { - in.Skip() - return - } - in.Delim('{') - for !in.IsDelim('}') { - key := in.UnsafeString() - in.WantColon() - if in.IsNull() { - in.Skip() - in.WantComma() - continue - } - switch key { - case "src": - out.Src = string(in.String()) - case "hOffset": - out.HOffset = int(in.Int()) - case "vOffset": - out.VOffset = int(in.Int()) - case "alignment": - out.Alignment = string(in.String()) - default: - in.SkipRecursive() - } - in.WantComma() - } - in.Delim('}') -} -func easyjsonDbb23193Encode2(out *jwriter.Writer, in struct { - Src string "json:\"src\"" - HOffset int "json:\"hOffset\"" - VOffset int "json:\"vOffset\"" - Alignment string "json:\"alignment\"" -}) { - out.RawByte('{') - first := true - _ = first - if !first { - out.RawByte(',') - } - first = false - out.RawString("\"src\":") - out.String(string(in.Src)) - if !first { - out.RawByte(',') - } - first = false - out.RawString("\"hOffset\":") - out.Int(int(in.HOffset)) - if !first { - out.RawByte(',') - } - first = false - out.RawString("\"vOffset\":") - out.Int(int(in.VOffset)) - if !first { - out.RawByte(',') - } - first = false - out.RawString("\"alignment\":") - out.String(string(in.Alignment)) - out.RawByte('}') -} -func easyjsonDbb23193Decode1(in *jlexer.Lexer, out *struct { - Title string "json:\"title\"" - Name string "json:\"name\"" - Width int "json:\"width\"" - Height int "json:\"height\"" -}) { - if in.IsNull() { - in.Skip() - return - } - in.Delim('{') - for !in.IsDelim('}') { - key := in.UnsafeString() - in.WantColon() - if in.IsNull() { - in.Skip() - in.WantComma() - continue - } - switch key { - case "title": - out.Title = string(in.String()) - case "name": - out.Name = string(in.String()) - case "width": - out.Width = int(in.Int()) - case "height": - out.Height = int(in.Int()) - default: - in.SkipRecursive() - } - in.WantComma() - } - in.Delim('}') -} -func easyjsonDbb23193Encode1(out *jwriter.Writer, in struct { - Title string "json:\"title\"" - Name string "json:\"name\"" - Width int "json:\"width\"" - Height int "json:\"height\"" -}) { - out.RawByte('{') - first := true - _ = first - if !first { - out.RawByte(',') - } - first = false - out.RawString("\"title\":") - out.String(string(in.Title)) - if !first { - out.RawByte(',') - } - first = false - out.RawString("\"name\":") - out.String(string(in.Name)) - if !first { - out.RawByte(',') - } - first = false - out.RawString("\"width\":") - out.Int(int(in.Width)) - if !first { - out.RawByte(',') - } - first = false - out.RawString("\"height\":") - out.Int(int(in.Height)) - out.RawByte('}') -} -func (mj *BenchStruct) MarshalFFJSONFromData() ([]byte, error) { - var buf fflib.Buffer - if mj == nil { - buf.WriteString("null") - return buf.Bytes(), nil - } - err := mj.MarshalJSONBufFFJSON(&buf) - if err != nil { - return nil, err - } - return buf.Bytes(), nil -} -func (mj *BenchStruct) MarshalJSONBufFFJSON(buf fflib.EncodingBuffer) error { - if mj == nil { - buf.WriteString("null") - return nil - } +func TestDeleteIssue21(t *testing.T) { + json := `{"country_code_from":"NZ","country_code_to":"SA","date_created":"2018-09-13T02:56:11.25783Z","date_updated":"2018-09-14T03:15:16.67356Z","disabled":false,"last_edited_by":"Developers","id":"a3e...bc454","merchant_id":"f2b...b91abf","signed_date":"2018-02-01T00:00:00Z","start_date":"2018-03-01T00:00:00Z","url":"https://www.google.com"}` + res1 := gjson.Get(json, "date_updated") var err error - var obj []byte - _ = obj - _ = err - /* Inline struct. type=struct { Debug string "json:\"debug\""; Window struct { Title string "json:\"title\""; Name string "json:\"name\""; Width int "json:\"width\""; Height int "json:\"height\"" } "json:\"window\""; Image struct { Src string "json:\"src\""; HOffset int "json:\"hOffset\""; VOffset int "json:\"vOffset\""; Alignment string "json:\"alignment\"" } "json:\"image\""; Text struct { Data string "json:\"data\""; Size int "json:\"size\""; Style string "json:\"style\""; VOffset int "json:\"vOffset\""; Alignment string "json:\"alignment\""; OnMouseUp string "json:\"onMouseUp\"" } "json:\"text\"" } kind=struct */ - buf.WriteString(`{"widget":{ "debug":`) - fflib.WriteJsonString(buf, string(mj.Widget.Debug)) - /* Inline struct. type=struct { Title string "json:\"title\""; Name string "json:\"name\""; Width int "json:\"width\""; Height int "json:\"height\"" } kind=struct */ - buf.WriteString(`,"window":{ "title":`) - fflib.WriteJsonString(buf, string(mj.Widget.Window.Title)) - buf.WriteString(`,"name":`) - fflib.WriteJsonString(buf, string(mj.Widget.Window.Name)) - buf.WriteString(`,"width":`) - fflib.FormatBits2(buf, uint64(mj.Widget.Window.Width), 10, mj.Widget.Window.Width < 0) - buf.WriteString(`,"height":`) - fflib.FormatBits2(buf, uint64(mj.Widget.Window.Height), 10, mj.Widget.Window.Height < 0) - buf.WriteByte('}') - /* Inline struct. type=struct { Src string "json:\"src\""; HOffset int "json:\"hOffset\""; VOffset int "json:\"vOffset\""; Alignment string "json:\"alignment\"" } kind=struct */ - buf.WriteString(`,"image":{ "src":`) - fflib.WriteJsonString(buf, string(mj.Widget.Image.Src)) - buf.WriteString(`,"hOffset":`) - fflib.FormatBits2(buf, uint64(mj.Widget.Image.HOffset), 10, mj.Widget.Image.HOffset < 0) - buf.WriteString(`,"vOffset":`) - fflib.FormatBits2(buf, uint64(mj.Widget.Image.VOffset), 10, mj.Widget.Image.VOffset < 0) - buf.WriteString(`,"alignment":`) - fflib.WriteJsonString(buf, string(mj.Widget.Image.Alignment)) - buf.WriteByte('}') - /* Inline struct. type=struct { Data string "json:\"data\""; Size int "json:\"size\""; Style string "json:\"style\""; VOffset int "json:\"vOffset\""; Alignment string "json:\"alignment\""; OnMouseUp string "json:\"onMouseUp\"" } kind=struct */ - buf.WriteString(`,"text":{ "data":`) - fflib.WriteJsonString(buf, string(mj.Widget.Text.Data)) - buf.WriteString(`,"size":`) - fflib.FormatBits2(buf, uint64(mj.Widget.Text.Size), 10, mj.Widget.Text.Size < 0) - buf.WriteString(`,"style":`) - fflib.WriteJsonString(buf, string(mj.Widget.Text.Style)) - buf.WriteString(`,"vOffset":`) - fflib.FormatBits2(buf, uint64(mj.Widget.Text.VOffset), 10, mj.Widget.Text.VOffset < 0) - buf.WriteString(`,"alignment":`) - fflib.WriteJsonString(buf, string(mj.Widget.Text.Alignment)) - buf.WriteString(`,"onMouseUp":`) - fflib.WriteJsonString(buf, string(mj.Widget.Text.OnMouseUp)) - buf.WriteByte('}') - buf.WriteByte('}') - buf.WriteByte('}') - return nil -} - -const ( - ffj_t_BenchStructbase = iota - ffj_t_BenchStructno_such_key - - ffj_t_BenchStruct_Widget -) - -var ffj_key_BenchStruct_Widget = []byte("widget") - -func (uj *BenchStruct) UnmarshalFFJSONFromData(input []byte) error { - fs := fflib.NewFFLexer(input) - return uj.UnmarshalJSONFFLexer(fs, fflib.FFParse_map_start) -} - -func (uj *BenchStruct) UnmarshalJSONFFLexer(fs *fflib.FFLexer, state fflib.FFParseState) error { - var err error = nil - currentKey := ffj_t_BenchStructbase - _ = currentKey - tok := fflib.FFTok_init - wantedTok := fflib.FFTok_init - -mainparse: - for { - tok = fs.Scan() - // println(fmt.Sprintf("debug: tok: %v state: %v", tok, state)) - if tok == fflib.FFTok_error { - goto tokerror - } - - switch state { - - case fflib.FFParse_map_start: - if tok != fflib.FFTok_left_bracket { - wantedTok = fflib.FFTok_left_bracket - goto wrongtokenerror - } - state = fflib.FFParse_want_key - continue - - case fflib.FFParse_after_value: - if tok == fflib.FFTok_comma { - state = fflib.FFParse_want_key - } else if tok == fflib.FFTok_right_bracket { - goto done - } else { - wantedTok = fflib.FFTok_comma - goto wrongtokenerror - } - - case fflib.FFParse_want_key: - // json {} ended. goto exit. woo. - if tok == fflib.FFTok_right_bracket { - goto done - } - if tok != fflib.FFTok_string { - wantedTok = fflib.FFTok_string - goto wrongtokenerror - } - - kn := fs.Output.Bytes() - if len(kn) <= 0 { - // "" case. hrm. - currentKey = ffj_t_BenchStructno_such_key - state = fflib.FFParse_want_colon - goto mainparse - } else { - switch kn[0] { - - case 'w': - - if bytes.Equal(ffj_key_BenchStruct_Widget, kn) { - currentKey = ffj_t_BenchStruct_Widget - state = fflib.FFParse_want_colon - goto mainparse - } - - } - - if fflib.SimpleLetterEqualFold(ffj_key_BenchStruct_Widget, kn) { - currentKey = ffj_t_BenchStruct_Widget - state = fflib.FFParse_want_colon - goto mainparse - } - - currentKey = ffj_t_BenchStructno_such_key - state = fflib.FFParse_want_colon - goto mainparse - } - - case fflib.FFParse_want_colon: - if tok != fflib.FFTok_colon { - wantedTok = fflib.FFTok_colon - goto wrongtokenerror - } - state = fflib.FFParse_want_value - continue - case fflib.FFParse_want_value: - - if tok == fflib.FFTok_left_brace || tok == fflib.FFTok_left_bracket || tok == fflib.FFTok_integer || tok == fflib.FFTok_double || tok == fflib.FFTok_string || tok == fflib.FFTok_bool || tok == fflib.FFTok_null { - switch currentKey { - - case ffj_t_BenchStruct_Widget: - goto handle_Widget - - case ffj_t_BenchStructno_such_key: - err = fs.SkipField(tok) - if err != nil { - return fs.WrapErr(err) - } - state = fflib.FFParse_after_value - goto mainparse - } - } else { - goto wantedvalue - } - } - } - -handle_Widget: - - /* handler: uj.Widget type=struct { Debug string "json:\"debug\""; Window struct { Title string "json:\"title\""; Name string "json:\"name\""; Width int "json:\"width\""; Height int "json:\"height\"" } "json:\"window\""; Image struct { Src string "json:\"src\""; HOffset int "json:\"hOffset\""; VOffset int "json:\"vOffset\""; Alignment string "json:\"alignment\"" } "json:\"image\""; Text struct { Data string "json:\"data\""; Size int "json:\"size\""; Style string "json:\"style\""; VOffset int "json:\"vOffset\""; Alignment string "json:\"alignment\""; OnMouseUp string "json:\"onMouseUp\"" } "json:\"text\"" } kind=struct quoted=false*/ - - { - /* Falling back. type=struct { Debug string "json:\"debug\""; Window struct { Title string "json:\"title\""; Name string "json:\"name\""; Width int "json:\"width\""; Height int "json:\"height\"" } "json:\"window\""; Image struct { Src string "json:\"src\""; HOffset int "json:\"hOffset\""; VOffset int "json:\"vOffset\""; Alignment string "json:\"alignment\"" } "json:\"image\""; Text struct { Data string "json:\"data\""; Size int "json:\"size\""; Style string "json:\"style\""; VOffset int "json:\"vOffset\""; Alignment string "json:\"alignment\""; OnMouseUp string "json:\"onMouseUp\"" } "json:\"text\"" } kind=struct */ - tbuf, err := fs.CaptureField(tok) - if err != nil { - return fs.WrapErr(err) - } - - err = gojson.Unmarshal(tbuf, &uj.Widget) - if err != nil { - return fs.WrapErr(err) - } - } - - state = fflib.FFParse_after_value - goto mainparse - -wantedvalue: - return fs.WrapErr(fmt.Errorf("wanted value token, but got token: %v", tok)) -wrongtokenerror: - return fs.WrapErr(fmt.Errorf("ffjson: wanted token: %v, but got token: %v output=%s", wantedTok, tok, fs.Output.String())) -tokerror: - if fs.BigError != nil { - return fs.WrapErr(fs.BigError) - } - err = fs.Error.ToError() + json, err = Delete(json, "date_updated") if err != nil { - return fs.WrapErr(err) + t.Fatal(err) + } + res2 := gjson.Get(json, "date_updated") + res3 := gjson.Get(json, "date_created") + if !res1.Exists() || res2.Exists() || !res3.Exists() { + t.Fatal("bad news") + } + + // We change the number of characters in this to make the section of the string before the section that we want to delete a certain length + + //--------------------------- + lenBeforeToDeleteIs307AsBytes := `{"1":"","0":"012345678901234567890123456789012345678901234567890123456789012345678901234567","to_delete":"0","2":""}` + + expectedForLenBefore307AsBytes := `{"1":"","0":"012345678901234567890123456789012345678901234567890123456789012345678901234567","2":""}` + //--------------------------- + + //--------------------------- + lenBeforeToDeleteIs308AsBytes := `{"1":"","0":"0123456789012345678901234567890123456789012345678901234567890123456789012345678","to_delete":"0","2":""}` + + expectedForLenBefore308AsBytes := `{"1":"","0":"0123456789012345678901234567890123456789012345678901234567890123456789012345678","2":""}` + //--------------------------- + + //--------------------------- + lenBeforeToDeleteIs309AsBytes := `{"1":"","0":"01234567890123456789012345678901234567890123456789012345678901234567890123456","to_delete":"0","2":""}` + + expectedForLenBefore309AsBytes := `{"1":"","0":"01234567890123456789012345678901234567890123456789012345678901234567890123456","2":""}` + //--------------------------- + + var data = []struct { + desc string + input string + expected string + }{ + { + desc: "len before \"to_delete\"... = 307", + input: lenBeforeToDeleteIs307AsBytes, + expected: expectedForLenBefore307AsBytes, + }, + { + desc: "len before \"to_delete\"... = 308", + input: lenBeforeToDeleteIs308AsBytes, + expected: expectedForLenBefore308AsBytes, + }, + { + desc: "len before \"to_delete\"... = 309", + input: lenBeforeToDeleteIs309AsBytes, + expected: expectedForLenBefore309AsBytes, + }, + } + + for i, d := range data { + result, err := Delete(d.input, "to_delete") + + if err != nil { + t.Error(fmtErrorf(testError{ + unexpected: "error", + desc: d.desc, + i: i, + lenInput: len(d.input), + input: d.input, + expected: d.expected, + result: result, + })) + } + if result != d.expected { + t.Error(fmtErrorf(testError{ + unexpected: "result", + desc: d.desc, + i: i, + lenInput: len(d.input), + input: d.input, + expected: d.expected, + result: result, + })) + } + } +} + +type testError struct { + unexpected string + desc string + i int + lenInput int + input interface{} + expected interface{} + result interface{} +} + +func fmtErrorf(e testError) string { + return fmt.Sprintf( + "Unexpected %s:\n\t"+ + "for=%q\n\t"+ + "i=%d\n\t"+ + "len(input)=%d\n\t"+ + "input=%v\n\t"+ + "expected=%v\n\t"+ + "result=%v", + e.unexpected, e.desc, e.i, e.lenInput, e.input, e.expected, e.result, + ) +} + +func TestSetDotKeyIssue10(t *testing.T) { + json := `{"app.token":"abc"}` + json, _ = Set(json, `app\.token`, "cde") + if json != `{"app.token":"cde"}` { + t.Fatalf("expected '%v', got '%v'", `{"app.token":"cde"}`, json) + } +} +func TestDeleteDotKeyIssue19(t *testing.T) { + json := []byte(`{"data":{"key1":"value1","key2.something":"value2"}}`) + json, _ = DeleteBytes(json, `data.key2\.something`) + if string(json) != `{"data":{"key1":"value1"}}` { + t.Fatalf("expected '%v', got '%v'", `{"data":{"key1":"value1"}}`, json) } - panic("ffjson-generated: unreachable, please report bug.") -done: - return nil }