From c784c417818f59d6597274642d8ac1d09efc9b01 Mon Sep 17 00:00:00 2001 From: Josh Baker Date: Thu, 25 May 2017 19:39:18 -0700 Subject: [PATCH] Allow parsing of large integers This commit fixes an issue in which GJSON was not representing integers correctly that were greater than 53-bits when calling the result.Int() and result.Uint() functions. This happened because GJSON stored all numbers as float64s in the result.Num field, and Int()/Uint() would simply try to convert the float64 to int64/uint64 by issuing int64(result.Num) or uint64(result.Num) operations. Now rather than a simple cast, GJSON checks to see if the float64 is a whole integer and if the integer can fit within 53-bits. If so, then the cast method can be used. Otherwise GJSON attempts to parse the result.Raw directly. If that fails too, it falls back to the original method. This fix should maintain compatibility with existing applications. thanks @joelpresence for reporting fixes #29 --- gjson.go | 94 +++++++++++++++++++++++++++++++++++++++++++++++---- gjson_test.go | 73 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 160 insertions(+), 7 deletions(-) diff --git a/gjson.go b/gjson.go index cc83364..1403506 100644 --- a/gjson.go +++ b/gjson.go @@ -110,10 +110,20 @@ func (t Result) Int() int64 { case True: return 1 case String: - n, _ := strconv.ParseInt(t.Str, 10, 64) + n, _ := parseInt(t.Str) return n case Number: - return int64(t.Num) + // try to directly convert the float64 to int64 + n, ok := floatToInt(t.Num) + if !ok { + // now try to parse the raw string + n, ok = parseInt(t.Raw) + if !ok { + // fallback to a standard conversion + return int64(t.Num) + } + } + return n } } @@ -125,10 +135,20 @@ func (t Result) Uint() uint64 { case True: return 1 case String: - n, _ := strconv.ParseUint(t.Str, 10, 64) + n, _ := parseUint(t.Str) return n case Number: - return uint64(t.Num) + // try to directly convert the float64 to uint64 + n, ok := floatToUint(t.Num) + if !ok { + // now try to parse the raw string + n, ok = parseUint(t.Raw) + if !ok { + // fallback to a standard conversion + return uint64(t.Num) + } + } + return n } } @@ -1091,8 +1111,8 @@ func parseArray(c *parseContext, i int, path string) (int, bool) { var multires []byte rp := parseArrayPath(path) if !rp.arrch { - n, err := strconv.ParseUint(rp.part, 10, 64) - if err != nil { + n, ok := parseUint(rp.part) + if !ok { partidx = -1 } else { partidx = int(n) @@ -2232,6 +2252,9 @@ func validstring(data []byte, i int) (outi int, ok bool) { return i, false } else if data[i] == '\\' { i++ + if i == len(data) { + return i, false + } switch data[i] { default: return i, false @@ -2348,3 +2371,62 @@ func Valid(json string) bool { _, ok := validpayload([]byte(json), 0) return ok } + +func parseUint(s string) (n uint64, ok bool) { + var i int + if i == len(s) { + return 0, false + } + for ; i < len(s); i++ { + if s[i] >= '0' && s[i] <= '9' { + n = n*10 + uint64(s[i]-'0') + } else { + return 0, false + } + } + return n, true +} + +func parseInt(s string) (n int64, ok bool) { + var i int + var sign bool + if len(s) > 0 && s[0] == '-' { + sign = true + i++ + } + if i == len(s) { + return 0, false + } + for ; i < len(s); i++ { + if s[i] >= '0' && s[i] <= '9' { + n = n*10 + int64(s[i]-'0') + } else { + return 0, false + } + } + if sign { + return n * -1, true + } + return n, true +} + +const minUint53 = 0 +const maxUint53 = 4503599627370495 +const minInt53 = -2251799813685248 +const maxInt53 = 2251799813685247 + +func floatToUint(f float64) (n uint64, ok bool) { + n = uint64(f) + if float64(n) == f && n >= minUint53 && n <= maxUint53 { + return n, true + } + return 0, false +} + +func floatToInt(f float64) (n int64, ok bool) { + n = int64(f) + if float64(n) == f && n >= minInt53 && n <= maxInt53 { + return n, true + } + return 0, false +} diff --git a/gjson_test.go b/gjson_test.go index 3a25f3d..00ab1bd 100644 --- a/gjson_test.go +++ b/gjson_test.go @@ -8,6 +8,7 @@ import ( "io" "math/rand" "reflect" + "strconv" "strings" "testing" "time" @@ -228,6 +229,52 @@ func TestBasic(t *testing.T) { t.Fatalf("expected %v, got %v", `["Brett","Elliotte"]`, mtok.String()) } } +func TestPlus53BitInts(t *testing.T) { + json := `{"IdentityData":{"GameInstanceId":634866135153775564}}` + value := Get(json, "IdentityData.GameInstanceId") + assert(t, value.Uint() == 634866135153775564) + assert(t, value.Int() == 634866135153775564) + assert(t, value.Float() == 634866135153775616) + + json = `{"IdentityData":{"GameInstanceId":634866135153775564.88172}}` + value = Get(json, "IdentityData.GameInstanceId") + assert(t, value.Uint() == 634866135153775616) + assert(t, value.Int() == 634866135153775616) + assert(t, value.Float() == 634866135153775616.88172) + + json = `{ + "min_uint64": 0, + "max_uint64": 18446744073709551615, + "overflow_uint64": 18446744073709551616, + "min_int64": -9223372036854775808, + "max_int64": 9223372036854775807, + "overflow_int64": 9223372036854775808, + "min_uint53": 0, + "max_uint53": 4503599627370495, + "overflow_uint53": 4503599627370496, + "min_int53": -2251799813685248, + "max_int53": 2251799813685247, + "overflow_int53": 2251799813685248 + }` + + assert(t, Get(json, "min_uint53").Uint() == 0) + assert(t, Get(json, "max_uint53").Uint() == 4503599627370495) + assert(t, Get(json, "overflow_uint53").Int() == 4503599627370496) + assert(t, Get(json, "min_int53").Int() == -2251799813685248) + assert(t, Get(json, "max_int53").Int() == 2251799813685247) + assert(t, Get(json, "overflow_int53").Int() == 2251799813685248) + assert(t, Get(json, "min_uint64").Uint() == 0) + assert(t, Get(json, "max_uint64").Uint() == 18446744073709551615) + // this next value overflows the max uint64 by one which will just + // flip the number to zero + assert(t, Get(json, "overflow_uint64").Int() == 0) + assert(t, Get(json, "min_int64").Int() == -9223372036854775808) + assert(t, Get(json, "max_int64").Int() == 9223372036854775807) + // this next value overflows the max int64 by one which will just + // flip the number to the negative sign. + assert(t, Get(json, "overflow_int64").Int() == -9223372036854775808) +} + func TestTypes(t *testing.T) { assert(t, (Result{Type: String}).Type.String() == "String") assert(t, (Result{Type: Number}).Type.String() == "Number") @@ -495,7 +542,7 @@ func TestUnescape(t *testing.T) { } func assert(t testing.TB, cond bool) { if !cond { - t.Fatal("assert failed") + panic("assert failed") } } func TestLess(t *testing.T) { @@ -1553,3 +1600,27 @@ func BenchmarkConvertGetBytes(t *testing.B) { GetBytes(data, "50.widget.text.onMouseUp") } } +func BenchmarkParseUintNumParser(t *testing.B) { + var s = "634866135153775564" + for i := 0; i < t.N; i++ { + parseUint(s) + } +} +func BenchmarkStdlibParseUintNumParser(t *testing.B) { + var s = "634866135153775564" + for i := 0; i < t.N; i++ { + strconv.ParseUint(s, 10, 64) + } +} +func BenchmarkParseIntNumParser(t *testing.B) { + var s = "-634866135153775564" + for i := 0; i < t.N; i++ { + parseInt(s) + } +} +func BenchmarkStdlibParseIntNumParser(t *testing.B) { + var s = "-634866135153775564" + for i := 0; i < t.N; i++ { + strconv.ParseInt(s, 10, 64) + } +}