From d6936636c217ef48c9bb29fa650faff928cb4659 Mon Sep 17 00:00:00 2001 From: Josh Baker Date: Fri, 29 Sep 2017 18:10:12 -0700 Subject: [PATCH] updated gjson --- vendor/github.com/tidwall/gjson/README.md | 112 ++- vendor/github.com/tidwall/gjson/gjson.go | 587 ++++++++++- vendor/github.com/tidwall/gjson/gjson_test.go | 932 ++++++++++-------- 3 files changed, 1130 insertions(+), 501 deletions(-) diff --git a/vendor/github.com/tidwall/gjson/README.md b/vendor/github.com/tidwall/gjson/README.md index 2733e586..5dbfed3c 100644 --- a/vendor/github.com/tidwall/gjson/README.md +++ b/vendor/github.com/tidwall/gjson/README.md @@ -3,17 +3,17 @@ src="logo.png" width="240" height="78" border="0" alt="GJSON">
-Build Status +Build Status GoDoc +GJSON Playground

+ +

get a json value quickly

-GJSON is a Go package that provides a [very fast](#performance) and simple way to get a value from a json document. The purpose for this library it to give efficient json indexing for the [BuntDB](https://github.com/tidwall/buntdb) project. - -For a command line interface check out [JSONed](https://github.com/tidwall/jsoned). +GJSON is a Go package that provides a [fast](#performance) and [simple](#get-a-value) way to get values from a json document. +It has features such as [one line retrieval](#get-a-value), [dot notation paths](#path-syntax), [iteration](#iterate-through-an-object-or-array). Getting Started =============== @@ -29,7 +29,7 @@ $ go get -u github.com/tidwall/gjson This will retrieve the library. ## Get a value -Get searches json for the specified path. A path is in dot syntax, such as "name.last" or "age". This function expects that the json is well-formed and validates. Invalid json will not panic, but it may return back unexpected results. When the value is found it's returned immediately. +Get searches json for the specified path. A path is in dot syntax, such as "name.last" or "age". This function expects that the json is well-formed. Bad json will not panic, but it may return back unexpected results. When the value is found it's returned immediately. ```go package main @@ -57,7 +57,7 @@ A path is a series of keys separated by a dot. A key may contain special wildcard characters '\*' and '?'. To access an array value use the index as the key. To get the number of elements in an array or to access a child path, use the '#' character. -The dot and wildcard characters can be escaped with '\'. +The dot and wildcard characters can be escaped with '\\'. ```json { @@ -122,12 +122,14 @@ result.Index // index of raw value in original json, zero means index unknown There are a variety of handy functions that work on a result: ```go +result.Exists() bool result.Value() interface{} result.Int() int64 result.Uint() uint64 result.Float() float64 result.String() string result.Bool() bool +result.Time() time.Time result.Array() []gjson.Result result.Map() map[string]gjson.Result result.Get(path string) Result @@ -137,8 +139,6 @@ result.Less(token Result, caseSensitive bool) bool The `result.Value()` function returns an `interface{}` which requires type assertion and is one of the following Go types: - - The `result.Array()` function returns back an array of values. If the result represents a non-existent value, then an empty array will be returned. If the result is not a JSON array, the return value will be an array containing one result. @@ -170,14 +170,14 @@ Suppose you want all the last names from the following json: "lastName": "Harold", } ] -}` +} ``` You would use the path "programmers.#.lastName" like such: ```go result := gjson.Get(json, "programmers.#.lastName") -for _,name := range result.Array() { +for _, name := range result.Array() { println(name.String()) } ``` @@ -198,7 +198,7 @@ Returning `false` from an iterator will stop iteration. ```go result := gjson.Get(json, "programmers") -result.ForEach(func(key, value Result) bool{ +result.ForEach(func(key, value gjson.Result) bool { println(value.String()) return true // keep iterating }) @@ -229,18 +229,65 @@ if !value.Exists() { } // Or as one step -if gjson.Get(json, "name.last").Exists(){ +if gjson.Get(json, "name.last").Exists() { println("has a last name") } ``` +## Unmarshalling + +There's a `gjson.Unmarshal` function which loads json data into a value. +It's a general replacement for `json.Unmarshal` and you can typically +see a 2-3x boost in performance without the need for external generators. + +This function works almost identically to `json.Unmarshal` except that +`gjson.Unmarshal` will automatically attempt to convert JSON values to any +Go type. For example, the JSON string "100" or the JSON number 100 can be +equally assigned to Go string, int, byte, uint64, etc. This rule applies to +all types. + + +```go +package main + +import ( + "fmt" + + "github.com/tidwall/gjson" +) + +type Animal struct { + Type string `json:"type"` + Sound string `json:"sound"` + Age int `json:"age"` +} + +var json = `{ + "type": "Dog", + "Sound": "Bark", + "Age": "11" +}` + +func main() { + var dog Animal + gjson.Unmarshal([]byte(json), &dog) + fmt.Printf("type: %s, sound: %s, age: %d\n", dog.Type, dog.Sound, dog.Age) +} +``` + +This will print: + +``` +type: Dog, sound: Bark, age: 11 +``` + ## Unmarshal to a map To unmarshal to a `map[string]interface{}`: ```go m, ok := gjson.Parse(json).Value().(map[string]interface{}) -if !ok{ +if !ok { // not a map } ``` @@ -284,28 +331,30 @@ The return value is a `[]Result`, which will always contain exactly the same num Benchmarks of GJSON alongside [encoding/json](https://golang.org/pkg/encoding/json/), [ffjson](https://github.com/pquerna/ffjson), [EasyJSON](https://github.com/mailru/easyjson), -and [jsonparser](https://github.com/buger/jsonparser) +[jsonparser](https://github.com/buger/jsonparser), +and [json-iterator](https://github.com/json-iterator/go) ``` -BenchmarkGJSONGet-8 15000000 333 ns/op 0 B/op 0 allocs/op -BenchmarkGJSONUnmarshalMap-8 900000 4188 ns/op 1920 B/op 26 allocs/op -BenchmarkJSONUnmarshalMap-8 600000 8908 ns/op 3048 B/op 69 allocs/op -BenchmarkJSONUnmarshalStruct-8 600000 9026 ns/op 1832 B/op 69 allocs/op -BenchmarkJSONDecoder-8 300000 14339 ns/op 4224 B/op 184 allocs/op -BenchmarkFFJSONLexer-8 1500000 3156 ns/op 896 B/op 8 allocs/op -BenchmarkEasyJSONLexer-8 3000000 938 ns/op 613 B/op 6 allocs/op -BenchmarkJSONParserGet-8 3000000 442 ns/op 21 B/op 0 allocs/op +BenchmarkGJSONGet-8 3000000 372 ns/op 0 B/op 0 allocs/op +BenchmarkGJSONUnmarshalMap-8 900000 4154 ns/op 1920 B/op 26 allocs/op +BenchmarkJSONUnmarshalMap-8 600000 9019 ns/op 3048 B/op 69 allocs/op +BenchmarkJSONUnmarshalStruct-8 600000 9268 ns/op 1832 B/op 69 allocs/op +BenchmarkJSONDecoder-8 300000 14120 ns/op 4224 B/op 184 allocs/op +BenchmarkFFJSONLexer-8 1500000 3111 ns/op 896 B/op 8 allocs/op +BenchmarkEasyJSONLexer-8 3000000 887 ns/op 613 B/op 6 allocs/op +BenchmarkJSONParserGet-8 3000000 499 ns/op 21 B/op 0 allocs/op +BenchmarkJSONIterator-8 3000000 812 ns/op 544 B/op 9 allocs/op ``` Benchmarks for the `GetMany` function: ``` -BenchmarkGJSONGetMany4Paths-8 4000000 319 ns/op 112 B/op 0 allocs/op -BenchmarkGJSONGetMany8Paths-8 8000000 218 ns/op 56 B/op 0 allocs/op -BenchmarkGJSONGetMany16Paths-8 16000000 160 ns/op 56 B/op 0 allocs/op -BenchmarkGJSONGetMany32Paths-8 32000000 130 ns/op 64 B/op 0 allocs/op -BenchmarkGJSONGetMany64Paths-8 64000000 117 ns/op 64 B/op 0 allocs/op -BenchmarkGJSONGetMany128Paths-8 128000000 109 ns/op 64 B/op 0 allocs/op +BenchmarkGJSONGetMany4Paths-8 4000000 303 ns/op 112 B/op 0 allocs/op +BenchmarkGJSONGetMany8Paths-8 8000000 208 ns/op 56 B/op 0 allocs/op +BenchmarkGJSONGetMany16Paths-8 16000000 156 ns/op 56 B/op 0 allocs/op +BenchmarkGJSONGetMany32Paths-8 32000000 127 ns/op 64 B/op 0 allocs/op +BenchmarkGJSONGetMany64Paths-8 64000000 117 ns/op 64 B/op 0 allocs/op +BenchmarkGJSONGetMany128Paths-8 128000000 109 ns/op 64 B/op 0 allocs/op ``` JSON document used: @@ -361,7 +410,8 @@ widget.text.data widget.text.size ``` -*These benchmarks were run on a MacBook Pro 15" 2.8 GHz Intel Core i7 using Go 1.7.* +*These benchmarks were run on a MacBook Pro 15" 2.8 GHz Intel Core i7 using Go 1.8 and can be be found [here](https://github.com/tidwall/gjson-benchmarks).* + ## Contact Josh Baker [@tidwall](http://twitter.com/tidwall) diff --git a/vendor/github.com/tidwall/gjson/gjson.go b/vendor/github.com/tidwall/gjson/gjson.go index 1ee26c92..fe4143c9 100644 --- a/vendor/github.com/tidwall/gjson/gjson.go +++ b/vendor/github.com/tidwall/gjson/gjson.go @@ -2,8 +2,17 @@ package gjson import ( + "encoding/base64" + "encoding/json" + "errors" "reflect" "strconv" + "strings" + "sync" + "sync/atomic" + "time" + "unicode/utf16" + "unicode/utf8" "unsafe" "github.com/tidwall/match" @@ -65,7 +74,7 @@ type Result struct { func (t Result) String() string { switch t.Type { default: - return "null" + return "" case False: return "false" case Number: @@ -101,10 +110,20 @@ func (t Result) Int() int64 { case True: return 1 case String: - n, _ := strconv.ParseInt(t.Str, 10, 64) + n, _ := parseInt(t.Str) return n case Number: - return int64(t.Num) + // try to directly convert the float64 to int64 + n, ok := floatToInt(t.Num) + if !ok { + // now try to parse the raw string + n, ok = parseInt(t.Raw) + if !ok { + // fallback to a standard conversion + return int64(t.Num) + } + } + return n } } @@ -116,10 +135,20 @@ func (t Result) Uint() uint64 { case True: return 1 case String: - n, _ := strconv.ParseUint(t.Str, 10, 64) + n, _ := parseUint(t.Str) return n case Number: - return uint64(t.Num) + // try to directly convert the float64 to uint64 + n, ok := floatToUint(t.Num) + if !ok { + // now try to parse the raw string + n, ok = parseUint(t.Raw) + if !ok { + // fallback to a standard conversion + return uint64(t.Num) + } + } + return n } } @@ -138,6 +167,12 @@ func (t Result) Float() float64 { } } +// Time returns a time.Time representation. +func (t Result) Time() time.Time { + res, _ := time.Parse(time.RFC3339, t.String()) + return res +} + // Array returns back an array of values. // If the result represents a non-existent value, then an empty array will be returned. // If the result is not a JSON array, the return value will be an array containing one result. @@ -152,6 +187,16 @@ func (t Result) Array() []Result { return r.a } +// IsObject returns true if the result value is a JSON object. +func (t Result) IsObject() bool { + return t.Type == JSON && len(t.Raw) > 0 && t.Raw[0] == '{' +} + +// IsObject returns true if the result value is a JSON array. +func (t Result) IsArray() bool { + return t.Type == JSON && len(t.Raw) > 0 && t.Raw[0] == '[' +} + // ForEach iterates through values. // If the result represents a non-existent value, then no values will be iterated. // If the result is an Object, the iterator will pass the key and value of each item. @@ -903,7 +948,7 @@ func parseObject(c *parseContext, i int, path string) (int, bool) { break } } - i, key, kesc, ok = i, c.json[s:], false, false + key, kesc, ok = c.json[s:], false, false parse_key_string_done: break } @@ -1076,8 +1121,8 @@ func parseArray(c *parseContext, i int, path string) (int, bool) { var multires []byte rp := parseArrayPath(path) if !rp.arrch { - n, err := strconv.ParseUint(rp.part, 10, 64) - if err != nil { + n, ok := parseUint(rp.part) + if !ok { partidx = -1 } else { partidx = int(n) @@ -1221,16 +1266,15 @@ func parseArray(c *parseContext, i int, path string) (int, bool) { c.value.Type = JSON c.value.Raw = string(jsons) return i + 1, true - } else { - if rp.alogok { - break - } - c.value.Raw = val - c.value.Type = Number - c.value.Num = float64(h - 1) - c.calcd = true - return i + 1, true } + if rp.alogok { + break + } + c.value.Raw = val + c.value.Type = Number + c.value.Num = float64(h - 1) + c.calcd = true + return i + 1, true } if len(multires) > 0 && !c.value.Exists() { c.value = Result{ @@ -1357,6 +1401,12 @@ func GetBytes(json []byte, path string) Result { return result } +// runeit returns the rune from the the \uXXXX +func runeit(json string) rune { + n, _ := strconv.ParseUint(json[:4], 16, 64) + return rune(n) +} + // unescape unescapes a string func unescape(json string) string { //, error) { var str = make([]byte, 0, len(json)) @@ -1365,15 +1415,15 @@ func unescape(json string) string { //, error) { default: str = append(str, json[i]) case json[i] < ' ': - return "" //, errors.New("invalid character in string") + return string(str) case json[i] == '\\': i++ if i >= len(json) { - return "" //, errors.New("invalid escape sequence") + return string(str) } switch json[i] { default: - return "" //, errors.New("invalid escape sequence") + return string(str) case '\\': str = append(str, '\\') case '/': @@ -1392,29 +1442,27 @@ func unescape(json string) string { //, error) { str = append(str, '"') case 'u': if i+5 > len(json) { - return "" //, errors.New("invalid escape sequence") + return string(str) } - i++ - // extract the codepoint - var code int - for j := i; j < i+4; j++ { - switch { - default: - return "" //, errors.New("invalid escape sequence") - case json[j] >= '0' && json[j] <= '9': - code += (int(json[j]) - '0') << uint(12-(j-i)*4) - case json[j] >= 'a' && json[j] <= 'f': - code += (int(json[j]) - 'a' + 10) << uint(12-(j-i)*4) - case json[j] >= 'a' && json[j] <= 'f': - code += (int(json[j]) - 'a' + 10) << uint(12-(j-i)*4) + r := runeit(json[i+1:]) + i += 5 + if utf16.IsSurrogate(r) { + // need another code + if len(json[i:]) >= 6 && json[i] == '\\' && json[i+1] == 'u' { + // we expect it to be correct so just consume it + r = utf16.DecodeRune(r, runeit(json[i+2:])) + i += 6 } } - str = append(str, []byte(string(code))...) - i += 3 // only 3 because we will increment on the for-loop + // provide enough space to encode the largest utf8 possible + str = append(str, 0, 0, 0, 0, 0, 0, 0, 0) + n := utf8.EncodeRune(str[len(str)-8:], r) + str = str[:len(str)-8+n] + i-- // backtrack index by one } } } - return string(str) //, nil + return string(str) } // Less return true if a token is less than another token. @@ -1742,7 +1790,6 @@ next_key: usedPaths++ continue } - // try to match the key to the path // this is spaghetti code but the idea is to minimize // calls and variable assignments when comparing the @@ -1757,12 +1804,17 @@ next_key: } if i < len(paths[j]) { if paths[j][i] == '.' { - // matched, but there still more keys in the path + // matched, but there are still more keys in path goto match_not_atend } } - // matched and at the end of the path - goto match_atend + if len(paths[j]) <= len(key) || kplen != 0 { + if len(paths[j]) != i { + goto nomatch + } + // matched and at the end of the path + goto match_atend + } } // no match, jump to the nomatch label goto nomatch @@ -1798,6 +1850,9 @@ next_key: nomatch: // noop label } + if !hasMatch && i < len(json) && json[i] == '}' { + return i + 1, true + } if !parsedVal { if hasMatch { // we found a match and the value has not been parsed yet. @@ -1940,3 +1995,453 @@ func getMany512(json string, i int, paths []string) ([]Result, bool) { _, ok := parseGetMany(json, i, 0, 0, paths, completed, matches, results) return results, ok } + +var fieldsmu sync.RWMutex +var fields = make(map[string]map[string]int) + +func assign(jsval Result, goval reflect.Value) { + if jsval.Type == Null { + return + } + switch goval.Kind() { + default: + case reflect.Ptr: + if !goval.IsNil() { + newval := reflect.New(goval.Elem().Type()) + assign(jsval, newval.Elem()) + goval.Elem().Set(newval.Elem()) + } else { + newval := reflect.New(goval.Type().Elem()) + assign(jsval, newval.Elem()) + goval.Set(newval) + } + case reflect.Struct: + fieldsmu.RLock() + sf := fields[goval.Type().String()] + fieldsmu.RUnlock() + if sf == nil { + fieldsmu.Lock() + sf = make(map[string]int) + for i := 0; i < goval.Type().NumField(); i++ { + f := goval.Type().Field(i) + tag := strings.Split(f.Tag.Get("json"), ",")[0] + if tag != "-" { + if tag != "" { + sf[tag] = i + sf[f.Name] = i + } else { + sf[f.Name] = i + } + } + } + fields[goval.Type().String()] = sf + fieldsmu.Unlock() + } + jsval.ForEach(func(key, value Result) bool { + if idx, ok := sf[key.Str]; ok { + f := goval.Field(idx) + if f.CanSet() { + assign(value, f) + } + } + return true + }) + case reflect.Slice: + if goval.Type().Elem().Kind() == reflect.Uint8 && jsval.Type == String { + data, _ := base64.StdEncoding.DecodeString(jsval.String()) + goval.Set(reflect.ValueOf(data)) + } else { + jsvals := jsval.Array() + slice := reflect.MakeSlice(goval.Type(), len(jsvals), len(jsvals)) + for i := 0; i < len(jsvals); i++ { + assign(jsvals[i], slice.Index(i)) + } + goval.Set(slice) + } + case reflect.Array: + i, n := 0, goval.Len() + jsval.ForEach(func(_, value Result) bool { + if i == n { + return false + } + assign(value, goval.Index(i)) + i++ + return true + }) + case reflect.Map: + if goval.Type().Key().Kind() == reflect.String && goval.Type().Elem().Kind() == reflect.Interface { + goval.Set(reflect.ValueOf(jsval.Value())) + } + case reflect.Interface: + goval.Set(reflect.ValueOf(jsval.Value())) + case reflect.Bool: + goval.SetBool(jsval.Bool()) + case reflect.Float32, reflect.Float64: + goval.SetFloat(jsval.Float()) + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + goval.SetInt(jsval.Int()) + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + goval.SetUint(jsval.Uint()) + case reflect.String: + goval.SetString(jsval.String()) + } + if len(goval.Type().PkgPath()) > 0 { + v := goval.Addr() + if v.Type().NumMethod() > 0 { + if u, ok := v.Interface().(json.Unmarshaler); ok { + u.UnmarshalJSON([]byte(jsval.Raw)) + } + } + } +} + +var validate uintptr = 1 + +// UnmarshalValidationEnabled provides the option to disable JSON validation +// during the Unmarshal routine. Validation is enabled by default. +func UnmarshalValidationEnabled(enabled bool) { + if enabled { + atomic.StoreUintptr(&validate, 1) + } else { + atomic.StoreUintptr(&validate, 0) + } +} + +// Unmarshal loads the JSON data into the value pointed to by v. +// +// This function works almost identically to json.Unmarshal except that +// gjson.Unmarshal will automatically attempt to convert JSON values to any Go +// type. For example, the JSON string "100" or the JSON number 100 can be equally +// assigned to Go string, int, byte, uint64, etc. This rule applies to all types. +func Unmarshal(data []byte, v interface{}) error { + if atomic.LoadUintptr(&validate) == 1 { + _, ok := validpayload(data, 0) + if !ok { + return errors.New("invalid json") + } + } + if v := reflect.ValueOf(v); v.Kind() == reflect.Ptr { + assign(ParseBytes(data), v) + } + return nil +} + +func validpayload(data []byte, i int) (outi int, ok bool) { + for ; i < len(data); i++ { + switch data[i] { + default: + i, ok = validany(data, i) + if !ok { + return i, false + } + for ; i < len(data); i++ { + switch data[i] { + default: + return i, false + case ' ', '\t', '\n', '\r': + continue + } + } + return i, true + case ' ', '\t', '\n', '\r': + continue + } + } + return i, false +} +func validany(data []byte, i int) (outi int, ok bool) { + for ; i < len(data); i++ { + switch data[i] { + default: + return i, false + case ' ', '\t', '\n', '\r': + continue + case '{': + return validobject(data, i+1) + case '[': + return validarray(data, i+1) + case '"': + return validstring(data, i+1) + case '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': + return validnumber(data, i+1) + case 't': + return validtrue(data, i+1) + case 'f': + return validfalse(data, i+1) + case 'n': + return validnull(data, i+1) + } + } + return i, false +} +func validobject(data []byte, i int) (outi int, ok bool) { + for ; i < len(data); i++ { + switch data[i] { + default: + return i, false + case ' ', '\t', '\n', '\r': + continue + case '}': + return i + 1, true + case '"': + key: + if i, ok = validstring(data, i+1); !ok { + return i, false + } + if i, ok = validcolon(data, i); !ok { + return i, false + } + if i, ok = validany(data, i); !ok { + return i, false + } + if i, ok = validcomma(data, i, '}'); !ok { + return i, false + } + if data[i] == '}' { + return i + 1, true + } + for ; i < len(data); i++ { + if data[i] == '"' { + goto key + } + } + return i, false + } + } + return i, false +} +func validcolon(data []byte, i int) (outi int, ok bool) { + for ; i < len(data); i++ { + switch data[i] { + default: + return i, false + case ' ', '\t', '\n', '\r': + continue + case ':': + return i + 1, true + } + } + return i, false +} +func validcomma(data []byte, i int, end byte) (outi int, ok bool) { + for ; i < len(data); i++ { + switch data[i] { + default: + return i, false + case ' ', '\t', '\n', '\r': + continue + case ',': + return i, true + case end: + return i, true + } + } + return i, false +} +func validarray(data []byte, i int) (outi int, ok bool) { + for ; i < len(data); i++ { + switch data[i] { + default: + for ; i < len(data); i++ { + if i, ok = validany(data, i); !ok { + return i, false + } + if i, ok = validcomma(data, i, ']'); !ok { + return i, false + } + if data[i] == ']' { + return i + 1, true + } + } + case ' ', '\t', '\n', '\r': + continue + case ']': + return i + 1, true + } + } + return i, false +} +func validstring(data []byte, i int) (outi int, ok bool) { + for ; i < len(data); i++ { + if data[i] < ' ' { + return i, false + } else if data[i] == '\\' { + i++ + if i == len(data) { + return i, false + } + switch data[i] { + default: + return i, false + case '"', '\\', '/', 'b', 'f', 'n', 'r', 't': + case 'u': + for j := 0; j < 4; j++ { + i++ + if i >= len(data) { + return i, false + } + if !((data[i] >= '0' && data[i] <= '9') || + (data[i] >= 'a' && data[i] <= 'f') || + (data[i] >= 'A' && data[i] <= 'F')) { + return i, false + } + } + } + } else if data[i] == '"' { + return i + 1, true + } + } + return i, false +} +func validnumber(data []byte, i int) (outi int, ok bool) { + i-- + // sign + if data[i] == '-' { + i++ + } + // int + if i == len(data) { + return i, false + } + if data[i] == '0' { + i++ + } else { + for ; i < len(data); i++ { + if data[i] >= '0' && data[i] <= '9' { + continue + } + break + } + } + // frac + if i == len(data) { + return i, true + } + if data[i] == '.' { + i++ + if i == len(data) { + return i, false + } + if data[i] < '0' || data[i] > '9' { + return i, false + } + i++ + for ; i < len(data); i++ { + if data[i] >= '0' && data[i] <= '9' { + continue + } + break + } + } + // exp + if i == len(data) { + return i, true + } + if data[i] == 'e' || data[i] == 'E' { + i++ + if i == len(data) { + return i, false + } + if data[i] == '+' || data[i] == '-' { + i++ + } + if i == len(data) { + return i, false + } + if data[i] < '0' || data[i] > '9' { + return i, false + } + i++ + for ; i < len(data); i++ { + if data[i] >= '0' && data[i] <= '9' { + continue + } + break + } + } + return i, true +} + +func validtrue(data []byte, i int) (outi int, ok bool) { + if i+3 <= len(data) && data[i] == 'r' && data[i+1] == 'u' && data[i+2] == 'e' { + return i + 3, true + } + return i, false +} +func validfalse(data []byte, i int) (outi int, ok bool) { + if i+4 <= len(data) && data[i] == 'a' && data[i+1] == 'l' && data[i+2] == 's' && data[i+3] == 'e' { + return i + 4, true + } + return i, false +} +func validnull(data []byte, i int) (outi int, ok bool) { + if i+3 <= len(data) && data[i] == 'u' && data[i+1] == 'l' && data[i+2] == 'l' { + return i + 3, true + } + return i, false +} + +// Valid returns true if the input is valid json. +func Valid(json string) bool { + _, ok := validpayload([]byte(json), 0) + return ok +} + +func parseUint(s string) (n uint64, ok bool) { + var i int + if i == len(s) { + return 0, false + } + for ; i < len(s); i++ { + if s[i] >= '0' && s[i] <= '9' { + n = n*10 + uint64(s[i]-'0') + } else { + return 0, false + } + } + return n, true +} + +func parseInt(s string) (n int64, ok bool) { + var i int + var sign bool + if len(s) > 0 && s[0] == '-' { + sign = true + i++ + } + if i == len(s) { + return 0, false + } + for ; i < len(s); i++ { + if s[i] >= '0' && s[i] <= '9' { + n = n*10 + int64(s[i]-'0') + } else { + return 0, false + } + } + if sign { + return n * -1, true + } + return n, true +} + +const minUint53 = 0 +const maxUint53 = 4503599627370495 +const minInt53 = -2251799813685248 +const maxInt53 = 2251799813685247 + +func floatToUint(f float64) (n uint64, ok bool) { + n = uint64(f) + if float64(n) == f && n >= minUint53 && n <= maxUint53 { + return n, true + } + return 0, false +} + +func floatToInt(f float64) (n int64, ok bool) { + n = int64(f) + if float64(n) == f && n >= minInt53 && n <= maxInt53 { + return n, true + } + return 0, false +} diff --git a/vendor/github.com/tidwall/gjson/gjson_test.go b/vendor/github.com/tidwall/gjson/gjson_test.go index 99b5048f..068e7426 100644 --- a/vendor/github.com/tidwall/gjson/gjson_test.go +++ b/vendor/github.com/tidwall/gjson/gjson_test.go @@ -5,15 +5,11 @@ import ( "encoding/hex" "encoding/json" "fmt" - "io" "math/rand" + "reflect" "strings" "testing" "time" - - "github.com/buger/jsonparser" - "github.com/mailru/easyjson/jlexer" - fflib "github.com/pquerna/ffjson/fflib/v1" ) // TestRandomData is a fuzzing test that throws random data at the Parse @@ -63,6 +59,17 @@ func TestRandomValidStrings(t *testing.T) { } } } + +func TestEmoji(t *testing.T) { + const input = `{"utf8":"Example emoji, KO: \ud83d\udd13, \ud83c\udfc3 OK: \u2764\ufe0f "}` + value := Get(input, "utf8") + var s string + json.Unmarshal([]byte(value.Raw), &s) + if value.String() != s { + t.Fatalf("expected '%v', got '%v'", s, value.String()) + } +} + func testEscapePath(t *testing.T, json, path, expect string) { if Get(json, path).String() != expect { t.Fatalf("expected '%v', got '%v'", expect, Get(json, path).String()) @@ -102,6 +109,7 @@ var basicJSON = `{"age":100, "name":{"here":"B\\\"R"}, "items":[1,2,3,{"tags":[1,2,3],"points":[[1,2],[3,4]]},4,5,6,7], "arr":["1",2,"3",{"hello":"world"},"4",5], "vals":[1,2,3,{"sadf":sdf"asdf"}],"name":{"first":"tom","last":null}, + "created":"2014-05-16T08:28:06.989Z", "loggy":{ "programmers": [ { @@ -127,10 +135,60 @@ var basicJSON = `{"age":100, "name":{"here":"B\\\"R"}, "age": 101 } ] - } + }, + "lastly":{"yay":"final"} }` var basicJSONB = []byte(basicJSON) +func TestTimeResult(t *testing.T) { + assert(t, Get(basicJSON, "created").String() == Get(basicJSON, "created").Time().Format(time.RFC3339Nano)) +} + +func TestParseAny(t *testing.T) { + assert(t, Parse("100").Float() == 100) + assert(t, Parse("true").Bool()) + assert(t, Parse("valse").Bool() == false) +} + +func TestManyVariousPathCounts(t *testing.T) { + json := `{"a":"a","b":"b","c":"c"}` + counts := []int{3, 4, 7, 8, 9, 15, 16, 17, 31, 32, 33, 63, 64, 65, 127, 128, 129, 255, 256, 257, 511, 512, 513} + paths := []string{"a", "b", "c"} + expects := []string{"a", "b", "c"} + for _, count := range counts { + var gpaths []string + var gexpects []string + for i := 0; i < count; i++ { + if i < len(paths) { + gpaths = append(gpaths, paths[i]) + gexpects = append(gexpects, expects[i]) + } else { + gpaths = append(gpaths, fmt.Sprintf("not%d", i)) + gexpects = append(gexpects, "null") + } + } + results := GetMany(json, gpaths...) + for i := 0; i < len(paths); i++ { + if results[i].String() != expects[i] { + t.Fatalf("expected '%v', got '%v'", expects[i], results[i].String()) + } + } + } +} +func TestManyRecursion(t *testing.T) { + var json string + var path string + for i := 0; i < 100; i++ { + json += `{"a":` + path += ".a" + } + json += `"b"` + for i := 0; i < 100; i++ { + json += `}` + } + path = path[1:] + assert(t, GetMany(json, path)[0].String() == "b") +} func TestByteSafety(t *testing.T) { jsonb := []byte(`{"name":"Janet","age":38}`) mtok := GetBytes(jsonb, "name") @@ -150,7 +208,7 @@ func TestByteSafety(t *testing.T) { } func get(json, path string) Result { - return GetBytes([]byte(basicJSONB), path) + return GetBytes([]byte(json), path) } func TestBasic(t *testing.T) { @@ -163,8 +221,138 @@ func TestBasic(t *testing.T) { if mtok.String() != `["Brett","Elliotte"]` { t.Fatalf("expected %v, got %v", `["Brett","Elliotte"]`, mtok.String()) } +} - mtok = get(basicJSON, `loggy.programmers`) +func TestIsArrayIsObject(t *testing.T) { + mtok := get(basicJSON, "loggy") + assert(t, mtok.IsObject()) + assert(t, !mtok.IsArray()) + + mtok = get(basicJSON, "loggy.programmers") + assert(t, !mtok.IsObject()) + assert(t, mtok.IsArray()) + + mtok = get(basicJSON, `loggy.programmers.#[tag="good"]#.firstName`) + assert(t, mtok.IsArray()) + + mtok = get(basicJSON, `loggy.programmers.0.firstName`) + assert(t, !mtok.IsObject()) + assert(t, !mtok.IsArray()) +} + +func TestPlus53BitInts(t *testing.T) { + json := `{"IdentityData":{"GameInstanceId":634866135153775564}}` + value := Get(json, "IdentityData.GameInstanceId") + assert(t, value.Uint() == 634866135153775564) + assert(t, value.Int() == 634866135153775564) + assert(t, value.Float() == 634866135153775616) + + json = `{"IdentityData":{"GameInstanceId":634866135153775564.88172}}` + value = Get(json, "IdentityData.GameInstanceId") + assert(t, value.Uint() == 634866135153775616) + assert(t, value.Int() == 634866135153775616) + assert(t, value.Float() == 634866135153775616.88172) + + json = `{ + "min_uint64": 0, + "max_uint64": 18446744073709551615, + "overflow_uint64": 18446744073709551616, + "min_int64": -9223372036854775808, + "max_int64": 9223372036854775807, + "overflow_int64": 9223372036854775808, + "min_uint53": 0, + "max_uint53": 4503599627370495, + "overflow_uint53": 4503599627370496, + "min_int53": -2251799813685248, + "max_int53": 2251799813685247, + "overflow_int53": 2251799813685248 + }` + + assert(t, Get(json, "min_uint53").Uint() == 0) + assert(t, Get(json, "max_uint53").Uint() == 4503599627370495) + assert(t, Get(json, "overflow_uint53").Int() == 4503599627370496) + assert(t, Get(json, "min_int53").Int() == -2251799813685248) + assert(t, Get(json, "max_int53").Int() == 2251799813685247) + assert(t, Get(json, "overflow_int53").Int() == 2251799813685248) + assert(t, Get(json, "min_uint64").Uint() == 0) + assert(t, Get(json, "max_uint64").Uint() == 18446744073709551615) + // this next value overflows the max uint64 by one which will just + // flip the number to zero + assert(t, Get(json, "overflow_uint64").Int() == 0) + assert(t, Get(json, "min_int64").Int() == -9223372036854775808) + assert(t, Get(json, "max_int64").Int() == 9223372036854775807) + // this next value overflows the max int64 by one which will just + // flip the number to the negative sign. + assert(t, Get(json, "overflow_int64").Int() == -9223372036854775808) +} +func TestIssue38(t *testing.T) { + // These should not fail, even though the unicode is invalid. + Get(`["S3O PEDRO DO BUTI\udf93"]`, "0") + Get(`["S3O PEDRO DO BUTI\udf93asdf"]`, "0") + Get(`["S3O PEDRO DO BUTI\udf93\u"]`, "0") + Get(`["S3O PEDRO DO BUTI\udf93\u1"]`, "0") + Get(`["S3O PEDRO DO BUTI\udf93\u13"]`, "0") + Get(`["S3O PEDRO DO BUTI\udf93\u134"]`, "0") + Get(`["S3O PEDRO DO BUTI\udf93\u1345"]`, "0") + Get(`["S3O PEDRO DO BUTI\udf93\u1345asd"]`, "0") +} +func TestTypes(t *testing.T) { + assert(t, (Result{Type: String}).Type.String() == "String") + assert(t, (Result{Type: Number}).Type.String() == "Number") + assert(t, (Result{Type: Null}).Type.String() == "Null") + assert(t, (Result{Type: False}).Type.String() == "False") + assert(t, (Result{Type: True}).Type.String() == "True") + assert(t, (Result{Type: JSON}).Type.String() == "JSON") + assert(t, (Result{Type: 100}).Type.String() == "") + // bool + assert(t, (Result{Type: String, Str: "true"}).Bool()) + assert(t, (Result{Type: True}).Bool()) + assert(t, (Result{Type: False}).Bool() == false) + assert(t, (Result{Type: Number, Num: 1}).Bool()) + // int + assert(t, (Result{Type: String, Str: "1"}).Int() == 1) + assert(t, (Result{Type: True}).Int() == 1) + assert(t, (Result{Type: False}).Int() == 0) + assert(t, (Result{Type: Number, Num: 1}).Int() == 1) + // uint + assert(t, (Result{Type: String, Str: "1"}).Uint() == 1) + assert(t, (Result{Type: True}).Uint() == 1) + assert(t, (Result{Type: False}).Uint() == 0) + assert(t, (Result{Type: Number, Num: 1}).Uint() == 1) + // float + assert(t, (Result{Type: String, Str: "1"}).Float() == 1) + assert(t, (Result{Type: True}).Float() == 1) + assert(t, (Result{Type: False}).Float() == 0) + assert(t, (Result{Type: Number, Num: 1}).Float() == 1) +} +func TestForEach(t *testing.T) { + Result{}.ForEach(nil) + Result{Type: String, Str: "Hello"}.ForEach(func(_, value Result) bool { + assert(t, value.String() == "Hello") + return false + }) + Result{Type: JSON, Raw: "*invalid*"}.ForEach(nil) + + json := ` {"name": {"first": "Janet","last": "Prichard"}, + "asd\nf":"\ud83d\udd13","age": 47}` + var count int + ParseBytes([]byte(json)).ForEach(func(key, value Result) bool { + count++ + return true + }) + assert(t, count == 3) + ParseBytes([]byte(`{"bad`)).ForEach(nil) + ParseBytes([]byte(`{"ok":"bad`)).ForEach(nil) +} +func TestMap(t *testing.T) { + assert(t, len(ParseBytes([]byte(`"asdf"`)).Map()) == 0) + assert(t, ParseBytes([]byte(`{"asdf":"ghjk"`)).Map()["asdf"].String() == "ghjk") + assert(t, len(Result{Type: JSON, Raw: "**invalid**"}.Map()) == 0) + assert(t, Result{Type: JSON, Raw: "**invalid**"}.Value() == nil) + assert(t, Result{Type: JSON, Raw: "{"}.Map() != nil) +} +func TestBasic1(t *testing.T) { + mtok := get(basicJSON, `loggy.programmers`) var count int mtok.ForEach(func(key, value Result) bool { if key.Exists() { @@ -200,7 +388,9 @@ func TestBasic(t *testing.T) { if count != 3 { t.Fatalf("expected %v, got %v", 3, count) } - mtok = get(basicJSON, `loggy.programmers.#[age=101].firstName`) +} +func TestBasic2(t *testing.T) { + mtok := get(basicJSON, `loggy.programmers.#[age=101].firstName`) if mtok.String() != "1002.3" { t.Fatalf("expected %v, got %v", "1002.3", mtok.String()) } @@ -227,25 +417,27 @@ func TestBasic(t *testing.T) { if programmers.Array()[1].Map()["firstName"].Str != "Jason" { t.Fatalf("expected %v, got %v", "Jason", mtok.Map()["programmers"].Array()[1].Map()["firstName"].Str) } - +} +func TestBasic3(t *testing.T) { + var mtok Result if Parse(basicJSON).Get("loggy.programmers").Get("1").Get("firstName").Str != "Jason" { t.Fatalf("expected %v, got %v", "Jason", Parse(basicJSON).Get("loggy.programmers").Get("1").Get("firstName").Str) } var token Result if token = Parse("-102"); token.Num != -102 { - t.Fatal("expected %v, got %v", -102, token.Num) + t.Fatalf("expected %v, got %v", -102, token.Num) } if token = Parse("102"); token.Num != 102 { - t.Fatal("expected %v, got %v", 102, token.Num) + t.Fatalf("expected %v, got %v", 102, token.Num) } if token = Parse("102.2"); token.Num != 102.2 { - t.Fatal("expected %v, got %v", 102.2, token.Num) + t.Fatalf("expected %v, got %v", 102.2, token.Num) } if token = Parse(`"hello"`); token.Str != "hello" { - t.Fatal("expected %v, got %v", "hello", token.Str) + t.Fatalf("expected %v, got %v", "hello", token.Str) } if token = Parse(`"\"he\nllo\""`); token.Str != "\"he\nllo\"" { - t.Fatal("expected %v, got %v", "\"he\nllo\"", token.Str) + t.Fatalf("expected %v, got %v", "\"he\nllo\"", token.Str) } mtok = get(basicJSON, "loggy.programmers.#.firstName") if len(mtok.Array()) != 4 { @@ -258,12 +450,13 @@ func TestBasic(t *testing.T) { } mtok = get(basicJSON, "loggy.programmers.#.asd") if mtok.Type != JSON { - t.Fatal("expected %v, got %v", JSON, mtok.Type) + t.Fatalf("expected %v, got %v", JSON, mtok.Type) } if len(mtok.Array()) != 0 { t.Fatalf("expected 0, got %v", len(mtok.Array())) } - +} +func TestBasic4(t *testing.T) { if get(basicJSON, "items.3.tags.#").Num != 3 { t.Fatalf("expected 3, got %v", get(basicJSON, "items.3.tags.#").Num) } @@ -279,7 +472,7 @@ func TestBasic(t *testing.T) { if !get(basicJSON, "name.last").Exists() { t.Fatal("expected true, got false") } - token = get(basicJSON, "name.here") + token := get(basicJSON, "name.here") if token.String() != "B\\\"R" { t.Fatal("expecting 'B\\\"R'", "got", token.String()) } @@ -298,13 +491,15 @@ func TestBasic(t *testing.T) { } _ = token.Value().(string) token = get(basicJSON, "name.last") - if token.String() != "null" { - t.Fatal("expecting 'null'", "got", token.String()) + if token.String() != "" { + t.Fatal("expecting ''", "got", token.String()) } if token.Value() != nil { t.Fatal("should be nil") } - token = get(basicJSON, "age") +} +func TestBasic5(t *testing.T) { + token := get(basicJSON, "age") if token.String() != "100" { t.Fatal("expecting '100'", "got", token.String()) } @@ -368,7 +563,7 @@ func TestUnescape(t *testing.T) { } func assert(t testing.TB, cond bool) { if !cond { - t.Fatal("assert failed") + panic("assert failed") } } func TestLess(t *testing.T) { @@ -507,17 +702,17 @@ func TestSingleArrayValue(t *testing.T) { t.Fatal("array is empty") } if array[0].String() != "value" { - t.Fatal("got %s, should be %s", array[0].String(), "value") + t.Fatalf("got %s, should be %s", array[0].String(), "value") } array = Get(json, "key2.#").Array() if len(array) != 1 { - t.Fatal("got '%v', expected '%v'", len(array), 1) + t.Fatalf("got '%v', expected '%v'", len(array), 1) } array = Get(json, "key3").Array() if len(array) != 0 { - t.Fatal("got '%v', expected '%v'", len(array), 0) + t.Fatalf("got '%v', expected '%v'", len(array), 0) } } @@ -549,32 +744,81 @@ func TestManyBasic(t *testing.T) { testWatchForFallback = false }() testMany := func(shouldFallback bool, expect string, paths ...string) { - results := GetMany( - manyJSON, + results := GetManyBytes( + []byte(manyJSON), paths..., ) if len(results) != len(paths) { t.Fatalf("expected %v, got %v", len(paths), len(results)) } if fmt.Sprintf("%v", results) != expect { + fmt.Printf("%v\n", paths) t.Fatalf("expected %v, got %v", expect, results) } - return - if testLastWasFallback != shouldFallback { - t.Fatalf("expected %v, got %v", shouldFallback, testLastWasFallback) - } + //if testLastWasFallback != shouldFallback { + // t.Fatalf("expected %v, got %v", shouldFallback, testLastWasFallback) + //} } testMany(false, "[Point]", "position.type") testMany(false, `[emptya ["world peace"] 31]`, ".a", "loves", "age") testMany(false, `[["world peace"]]`, "loves") testMany(false, `[{"last":"Anderson","first":"Nancy"} Nancy]`, "name", "name.first") - testMany(true, `[null]`, strings.Repeat("a.", 40)+"hello") + testMany(true, `[]`, strings.Repeat("a.", 40)+"hello") res := Get(manyJSON, strings.Repeat("a.", 48)+"a") testMany(true, `[`+res.String()+`]`, strings.Repeat("a.", 48)+"a") // these should fallback testMany(true, `[Cat Nancy]`, "name\\.first", "name.first") testMany(true, `[world]`, strings.Repeat("a.", 70)+"hello") } +func testMany(t *testing.T, json string, paths, expected []string) { + testManyAny(t, json, paths, expected, true) + testManyAny(t, json, paths, expected, false) +} +func testManyAny(t *testing.T, json string, paths, expected []string, bytes bool) { + var result []Result + for i := 0; i < 2; i++ { + var which string + if i == 0 { + which = "Get" + result = nil + for j := 0; j < len(expected); j++ { + if bytes { + result = append(result, GetBytes([]byte(json), paths[j])) + } else { + result = append(result, Get(json, paths[j])) + } + } + } else if i == 1 { + which = "GetMany" + if bytes { + result = GetManyBytes([]byte(json), paths...) + } else { + result = GetMany(json, paths...) + } + } + for j := 0; j < len(expected); j++ { + if result[j].String() != expected[j] { + t.Fatalf("Using key '%s' for '%s'\nexpected '%v', got '%v'", paths[j], which, expected[j], result[j].String()) + } + } + } +} +func TestIssue20(t *testing.T) { + json := `{ "name": "FirstName", "name1": "FirstName1", "address": "address1", "addressDetails": "address2", }` + paths := []string{"name", "name1", "address", "addressDetails"} + expected := []string{"FirstName", "FirstName1", "address1", "address2"} + t.Run("SingleMany", func(t *testing.T) { testMany(t, json, paths, expected) }) +} + +func TestIssue21(t *testing.T) { + json := `{ "Level1Field1":3, + "Level1Field4":4, + "Level1Field2":{ "Level2Field1":[ "value1", "value2" ], + "Level2Field2":{ "Level3Field1":[ { "key1":"value1" } ] } } }` + paths := []string{"Level1Field1", "Level1Field2.Level2Field1", "Level1Field2.Level2Field2.Level3Field1", "Level1Field4"} + expected := []string{"3", `[ "value1", "value2" ]`, `[ { "key1":"value1" } ]`, "4"} + t.Run("SingleMany", func(t *testing.T) { testMany(t, json, paths, expected) }) +} func TestRandomMany(t *testing.T) { var lstr string @@ -612,418 +856,248 @@ func TestRandomMany(t *testing.T) { } } -type BenchStruct struct { - Widget struct { - Window struct { - Name string `json:"name"` - } `json:"window"` - Image struct { - HOffset int `json:"hOffset"` - } `json:"image"` - Text struct { - OnMouseUp string `json:"onMouseUp"` - } `json:"text"` - } `json:"widget"` +type ComplicatedType struct { + unsettable int + Tagged string `json:"tagged"` + NotTagged bool + Nested struct { + Yellow string `json:"yellow"` + } + NestedTagged struct { + Green string + Map map[string]interface{} + Ints struct { + Int int `json:"int"` + Int8 int8 + Int16 int16 + Int32 int32 + Int64 int64 `json:"int64"` + } + Uints struct { + Uint uint + Uint8 uint8 + Uint16 uint16 + Uint32 uint32 + Uint64 uint64 + } + Floats struct { + Float64 float64 + Float32 float32 + } + Byte byte + Bool bool + } `json:"nestedTagged"` + LeftOut string `json:"-"` + SelfPtr *ComplicatedType + SelfSlice []ComplicatedType + SelfSlicePtr []*ComplicatedType + SelfPtrSlice *[]ComplicatedType + Interface interface{} `json:"interface"` + Array [3]int + Time time.Time `json:"time"` + Binary []byte + NonBinary []byte } -var benchPaths = []string{ - "widget.window.name", - "widget.image.hOffset", - "widget.text.onMouseUp", +var complicatedJSON = ` +{ + "tagged": "OK", + "Tagged": "KO", + "NotTagged": true, + "unsettable": 101, + "Nested": { + "Yellow": "Green", + "yellow": "yellow" + }, + "nestedTagged": { + "Green": "Green", + "Map": { + "this": "that", + "and": "the other thing" + }, + "Ints": { + "Uint": 99, + "Uint16": 16, + "Uint32": 32, + "Uint64": 65 + }, + "Uints": { + "int": -99, + "Int": -98, + "Int16": -16, + "Int32": -32, + "int64": -64, + "Int64": -65 + }, + "Uints": { + "Float32": 32.32, + "Float64": 64.64 + }, + "Byte": 254, + "Bool": true + }, + "LeftOut": "you shouldn't be here", + "SelfPtr": {"tagged":"OK","nestedTagged":{"Ints":{"Uint32":32}}}, + "SelfSlice": [{"tagged":"OK","nestedTagged":{"Ints":{"Uint32":32}}}], + "SelfSlicePtr": [{"tagged":"OK","nestedTagged":{"Ints":{"Uint32":32}}}], + "SelfPtrSlice": [{"tagged":"OK","nestedTagged":{"Ints":{"Uint32":32}}}], + "interface": "Tile38 Rocks!", + "Interface": "Please Download", + "Array": [0,2,3,4,5], + "time": "2017-05-07T13:24:43-07:00", + "Binary": "R0lGODlhPQBEAPeo", + "NonBinary": [9,3,100,115] +} +` + +func TestUnmarshal(t *testing.T) { + var s1 ComplicatedType + var s2 ComplicatedType + if err := json.Unmarshal([]byte(complicatedJSON), &s1); err != nil { + t.Fatal(err) + } + if err := Unmarshal([]byte(complicatedJSON), &s2); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(&s1, &s2) { + t.Fatal("not equal") + } + var str string + if err := json.Unmarshal([]byte(Get(complicatedJSON, "LeftOut").Raw), &str); err != nil { + t.Fatal(err) + } + assert(t, str == Get(complicatedJSON, "LeftOut").String()) } -var benchManyPaths = []string{ - "widget.window.name", - "widget.image.hOffset", - "widget.text.onMouseUp", - "widget.window.title", - "widget.image.alignment", - "widget.text.style", - "widget.window.height", - "widget.image.src", - "widget.text.data", - "widget.text.size", +func testvalid(json string, expect bool) { + _, ok := validpayload([]byte(json), 0) + if ok != expect { + panic("mismatch") + } } -func BenchmarkGJSONGet(t *testing.B) { - t.ReportAllocs() - t.ResetTimer() - for i := 0; i < t.N; i++ { - for j := 0; j < len(benchPaths); j++ { - if Get(exampleJSON, benchPaths[j]).Type == Null { - t.Fatal("did not find the value") - } - } - } - t.N *= len(benchPaths) // because we are running against 3 paths -} -func BenchmarkGJSONGetMany4Paths(t *testing.B) { - benchmarkGJSONGetManyN(t, 4) -} -func BenchmarkGJSONGetMany8Paths(t *testing.B) { - benchmarkGJSONGetManyN(t, 8) -} -func BenchmarkGJSONGetMany16Paths(t *testing.B) { - benchmarkGJSONGetManyN(t, 16) -} -func BenchmarkGJSONGetMany32Paths(t *testing.B) { - benchmarkGJSONGetManyN(t, 32) -} -func BenchmarkGJSONGetMany64Paths(t *testing.B) { - benchmarkGJSONGetManyN(t, 64) -} -func BenchmarkGJSONGetMany128Paths(t *testing.B) { - benchmarkGJSONGetManyN(t, 128) -} -func BenchmarkGJSONGetMany256Paths(t *testing.B) { - benchmarkGJSONGetManyN(t, 256) -} -func BenchmarkGJSONGetMany512Paths(t *testing.B) { - benchmarkGJSONGetManyN(t, 512) -} -func benchmarkGJSONGetManyN(t *testing.B, n int) { - var paths []string - for len(paths) < n { - paths = append(paths, benchManyPaths...) - } - paths = paths[:n] - t.ReportAllocs() - t.ResetTimer() - for i := 0; i < t.N; i++ { - results := GetMany(exampleJSON, paths...) - if len(results) == 0 { - t.Fatal("did not find the value") - } - for j := 0; j < len(results); j++ { - if results[j].Type == Null { - t.Fatal("did not find the value") - } - } - } - t.N *= len(paths) // because we are running against 3 paths +func TestValidBasic(t *testing.T) { + testvalid("0", true) + testvalid("00", false) + testvalid("-00", false) + testvalid("-.", false) + testvalid("0.0", true) + testvalid("10.0", true) + testvalid("10e1", true) + testvalid("10EE", false) + testvalid("10E-", false) + testvalid("10E+", false) + testvalid("10E123", true) + testvalid("10E-123", true) + testvalid("10E-0123", true) + testvalid("", false) + testvalid(" ", false) + testvalid("{}", true) + testvalid("{", false) + testvalid("-", false) + testvalid("-1", true) + testvalid("-1.", false) + testvalid("-1.0", true) + testvalid(" -1.0", true) + testvalid(" -1.0 ", true) + testvalid("-1.0 ", true) + testvalid("-1.0 i", false) + testvalid("-1.0 i", false) + testvalid("true", true) + testvalid(" true", true) + testvalid(" true ", true) + testvalid(" True ", false) + testvalid(" tru", false) + testvalid("false", true) + testvalid(" false", true) + testvalid(" false ", true) + testvalid(" False ", false) + testvalid(" fals", false) + testvalid("null", true) + testvalid(" null", true) + testvalid(" null ", true) + testvalid(" Null ", false) + testvalid(" nul", false) + testvalid(" []", true) + testvalid(" [true]", true) + testvalid(" [ true, null ]", true) + testvalid(" [ true,]", false) + testvalid(`{"hello":"world"}`, true) + testvalid(`{ "hello": "world" }`, true) + testvalid(`{ "hello": "world", }`, false) + testvalid(`{"a":"b",}`, false) + testvalid(`{"a":"b","a"}`, false) + testvalid(`{"a":"b","a":}`, false) + testvalid(`{"a":"b","a":1}`, true) + testvalid(`{"a":"b","a": 1, "c":{"hi":"there"} }`, true) + testvalid(`{"a":"b","a": 1, "c":{"hi":"there", "easy":["going",{"mixed":"bag"}]} }`, true) + testvalid(`""`, true) + testvalid(`"`, false) + testvalid(`"\n"`, true) + testvalid(`"\"`, false) + testvalid(`"\\"`, true) + testvalid(`"a\\b"`, true) + testvalid(`"a\\b\\\"a"`, true) + testvalid(`"a\\b\\\uFFAAa"`, true) + testvalid(`"a\\b\\\uFFAZa"`, false) + testvalid(`"a\\b\\\uFFA"`, false) + testvalid(string(complicatedJSON), true) + testvalid(string(exampleJSON), true) } -func BenchmarkGJSONUnmarshalMap(t *testing.B) { - t.ReportAllocs() - t.ResetTimer() - for i := 0; i < t.N; i++ { - for j := 0; j < len(benchPaths); j++ { - parts := strings.Split(benchPaths[j], ".") - m, _ := Parse(exampleJSON).Value().(map[string]interface{}) - var v interface{} - for len(parts) > 0 { - part := parts[0] - if len(parts) > 1 { - m = m[part].(map[string]interface{}) - if m == nil { - t.Fatal("did not find the value") - } - } else { - v = m[part] - if v == nil { - t.Fatal("did not find the value") - } - } - parts = parts[1:] - } - } +var jsonchars = []string{"{", "[", ",", ":", "}", "]", "1", "0", "true", "false", "null", `""`, `"\""`, `"a"`} + +func makeRandomJSONChars(b []byte) { + var bb []byte + for len(bb) < len(b) { + bb = append(bb, jsonchars[rand.Int()%len(jsonchars)]...) + } + copy(b, bb[:len(b)]) +} +func TestValidRandom(t *testing.T) { + rand.Seed(time.Now().UnixNano()) + b := make([]byte, 100000) + start := time.Now() + for time.Since(start) < time.Second*3 { + n := rand.Int() % len(b) + rand.Read(b[:n]) + validpayload(b[:n], 0) + } + + start = time.Now() + for time.Since(start) < time.Second*3 { + n := rand.Int() % len(b) + makeRandomJSONChars(b[:n]) + validpayload(b[:n], 0) } - t.N *= len(benchPaths) // because we are running against 3 paths } -func BenchmarkJSONUnmarshalMap(t *testing.B) { - t.ReportAllocs() - t.ResetTimer() - for i := 0; i < t.N; i++ { - for j := 0; j < len(benchPaths); j++ { - parts := strings.Split(benchPaths[j], ".") - var m map[string]interface{} - if err := json.Unmarshal([]byte(exampleJSON), &m); err != nil { - t.Fatal(err) - } - var v interface{} - for len(parts) > 0 { - part := parts[0] - if len(parts) > 1 { - m = m[part].(map[string]interface{}) - if m == nil { - t.Fatal("did not find the value") - } - } else { - v = m[part] - if v == nil { - t.Fatal("did not find the value") - } - } - parts = parts[1:] - } +func TestGetMany47(t *testing.T) { + json := `{"bar": {"id": 99, "mybar": "my mybar" }, "foo": {"myfoo": [605]}}` + paths := []string{"foo.myfoo", "bar.id", "bar.mybar", "bar.mybarx"} + expected := []string{"[605]", "99", "my mybar", ""} + results := GetMany(json, paths...) + if len(expected) != len(results) { + t.Fatalf("expected %v, got %v", len(expected), len(results)) + } + for i, path := range paths { + if results[i].String() != expected[i] { + t.Fatalf("expected '%v', got '%v' for path '%v'", expected[i], results[i].String(), path) } } - t.N *= len(benchPaths) // because we are running against 3 paths } -func BenchmarkJSONUnmarshalStruct(t *testing.B) { - t.ReportAllocs() - t.ResetTimer() - for i := 0; i < t.N; i++ { - for j := 0; j < len(benchPaths); j++ { - var s BenchStruct - if err := json.Unmarshal([]byte(exampleJSON), &s); err != nil { - t.Fatal(err) - } - switch benchPaths[j] { - case "widget.window.name": - if s.Widget.Window.Name == "" { - t.Fatal("did not find the value") - } - case "widget.image.hOffset": - if s.Widget.Image.HOffset == 0 { - t.Fatal("did not find the value") - } - case "widget.text.onMouseUp": - if s.Widget.Text.OnMouseUp == "" { - t.Fatal("did not find the value") - } - } +func TestGetMany48(t *testing.T) { + json := `{"bar": {"id": 99, "xyz": "my xyz"}, "foo": {"myfoo": [605]}}` + paths := []string{"foo.myfoo", "bar.id", "bar.xyz", "bar.abc"} + expected := []string{"[605]", "99", "my xyz", ""} + results := GetMany(json, paths...) + if len(expected) != len(results) { + t.Fatalf("expected %v, got %v", len(expected), len(results)) + } + for i, path := range paths { + if results[i].String() != expected[i] { + t.Fatalf("expected '%v', got '%v' for path '%v'", expected[i], results[i].String(), path) } } - t.N *= len(benchPaths) // because we are running against 3 paths -} - -func BenchmarkJSONDecoder(t *testing.B) { - t.ReportAllocs() - t.ResetTimer() - for i := 0; i < t.N; i++ { - for j := 0; j < len(benchPaths); j++ { - dec := json.NewDecoder(bytes.NewBuffer([]byte(exampleJSON))) - var found bool - outer: - for { - tok, err := dec.Token() - if err != nil { - if err == io.EOF { - break - } - t.Fatal(err) - } - switch v := tok.(type) { - case string: - if found { - // break out once we find the value. - break outer - } - switch benchPaths[j] { - case "widget.window.name": - if v == "name" { - found = true - } - case "widget.image.hOffset": - if v == "hOffset" { - found = true - } - case "widget.text.onMouseUp": - if v == "onMouseUp" { - found = true - } - } - } - } - if !found { - t.Fatal("field not found") - } - } - } - t.N *= len(benchPaths) // because we are running against 3 paths -} - -func BenchmarkFFJSONLexer(t *testing.B) { - t.ReportAllocs() - t.ResetTimer() - for i := 0; i < t.N; i++ { - for j := 0; j < len(benchPaths); j++ { - l := fflib.NewFFLexer([]byte(exampleJSON)) - var found bool - outer: - for { - t := l.Scan() - if t == fflib.FFTok_eof { - break - } - if t == fflib.FFTok_string { - b, _ := l.CaptureField(t) - v := string(b) - if found { - // break out once we find the value. - break outer - } - switch benchPaths[j] { - case "widget.window.name": - if v == "\"name\"" { - found = true - } - case "widget.image.hOffset": - if v == "\"hOffset\"" { - found = true - } - case "widget.text.onMouseUp": - if v == "\"onMouseUp\"" { - found = true - } - } - } - } - if !found { - t.Fatal("field not found") - } - } - } - t.N *= len(benchPaths) // because we are running against 3 paths -} - -func BenchmarkEasyJSONLexer(t *testing.B) { - skipCC := func(l *jlexer.Lexer, n int) { - for i := 0; i < n; i++ { - l.Skip() - l.WantColon() - l.Skip() - l.WantComma() - } - } - skipGroup := func(l *jlexer.Lexer, n int) { - l.WantColon() - l.Delim('{') - skipCC(l, n) - l.Delim('}') - l.WantComma() - } - t.ReportAllocs() - t.ResetTimer() - for i := 0; i < t.N; i++ { - for j := 0; j < len(benchPaths); j++ { - l := &jlexer.Lexer{Data: []byte(exampleJSON)} - l.Delim('{') - if l.String() == "widget" { - l.WantColon() - l.Delim('{') - switch benchPaths[j] { - case "widget.window.name": - skipCC(l, 1) - if l.String() == "window" { - l.WantColon() - l.Delim('{') - skipCC(l, 1) - if l.String() == "name" { - l.WantColon() - if l.String() == "" { - t.Fatal("did not find the value") - } - } - } - case "widget.image.hOffset": - skipCC(l, 1) - if l.String() == "window" { - skipGroup(l, 4) - } - if l.String() == "image" { - l.WantColon() - l.Delim('{') - skipCC(l, 1) - if l.String() == "hOffset" { - l.WantColon() - if l.Int() == 0 { - t.Fatal("did not find the value") - } - } - } - case "widget.text.onMouseUp": - skipCC(l, 1) - if l.String() == "window" { - skipGroup(l, 4) - } - if l.String() == "image" { - skipGroup(l, 4) - } - if l.String() == "text" { - l.WantColon() - l.Delim('{') - skipCC(l, 5) - if l.String() == "onMouseUp" { - l.WantColon() - if l.String() == "" { - t.Fatal("did not find the value") - } - } - } - } - } - } - } - t.N *= len(benchPaths) // because we are running against 3 paths -} - -func BenchmarkJSONParserGet(t *testing.B) { - data := []byte(exampleJSON) - keys := make([][]string, 0, len(benchPaths)) - for i := 0; i < len(benchPaths); i++ { - keys = append(keys, strings.Split(benchPaths[i], ".")) - } - t.ReportAllocs() - t.ResetTimer() - for i := 0; i < t.N; i++ { - for j, k := range keys { - if j == 1 { - // "widget.image.hOffset" is a number - v, _ := jsonparser.GetInt(data, k...) - if v == 0 { - t.Fatal("did not find the value") - } - } else { - // "widget.window.name", - // "widget.text.onMouseUp", - v, _ := jsonparser.GetString(data, k...) - if v == "" { - t.Fatal("did not find the value") - } - } - } - } - t.N *= len(benchPaths) // because we are running against 3 paths -} - -var massiveJSON = func() string { - var buf bytes.Buffer - buf.WriteString("[") - for i := 0; i < 100; i++ { - if i > 0 { - buf.WriteByte(',') - } - buf.WriteString(exampleJSON) - } - buf.WriteString("]") - return buf.String() -}() - -func BenchmarkConvertNone(t *testing.B) { - json := massiveJSON - t.ReportAllocs() - t.ResetTimer() - for i := 0; i < t.N; i++ { - Get(json, "50.widget.text.onMouseUp") - } -} -func BenchmarkConvertGet(t *testing.B) { - data := []byte(massiveJSON) - t.ReportAllocs() - t.ResetTimer() - for i := 0; i < t.N; i++ { - Get(string(data), "50.widget.text.onMouseUp") - } -} -func BenchmarkConvertGetBytes(t *testing.B) { - data := []byte(massiveJSON) - t.ReportAllocs() - t.ResetTimer() - for i := 0; i < t.N; i++ { - GetBytes(data, "50.widget.text.onMouseUp") - } }