Allow parsing of large integers

This commit fixes an issue in which GJSON was not representing integers
correctly that were greater than 53-bits when calling the result.Int()
and result.Uint() functions. This happened because GJSON stored all
numbers as float64s in the result.Num field, and Int()/Uint() would
simply try to convert the float64 to int64/uint64 by issuing
int64(result.Num) or uint64(result.Num) operations.

Now rather than a simple cast, GJSON checks to see if the float64 is a
whole integer and if the integer can fit within 53-bits. If so, then
the cast method can be used. Otherwise GJSON attempts to parse the
result.Raw directly. If that fails too, it falls back to the original
method.

This fix should maintain compatibility with existing applications.

thanks @joelpresence for reporting
fixes #29
This commit is contained in:
Josh Baker 2017-05-25 19:39:18 -07:00
parent 0623bd8fbd
commit c784c41781
2 changed files with 160 additions and 7 deletions

View File

@ -110,10 +110,20 @@ func (t Result) Int() int64 {
case True: case True:
return 1 return 1
case String: case String:
n, _ := strconv.ParseInt(t.Str, 10, 64) n, _ := parseInt(t.Str)
return n return n
case Number: case Number:
return int64(t.Num) // try to directly convert the float64 to int64
n, ok := floatToInt(t.Num)
if !ok {
// now try to parse the raw string
n, ok = parseInt(t.Raw)
if !ok {
// fallback to a standard conversion
return int64(t.Num)
}
}
return n
} }
} }
@ -125,10 +135,20 @@ func (t Result) Uint() uint64 {
case True: case True:
return 1 return 1
case String: case String:
n, _ := strconv.ParseUint(t.Str, 10, 64) n, _ := parseUint(t.Str)
return n return n
case Number: case Number:
return uint64(t.Num) // try to directly convert the float64 to uint64
n, ok := floatToUint(t.Num)
if !ok {
// now try to parse the raw string
n, ok = parseUint(t.Raw)
if !ok {
// fallback to a standard conversion
return uint64(t.Num)
}
}
return n
} }
} }
@ -1091,8 +1111,8 @@ func parseArray(c *parseContext, i int, path string) (int, bool) {
var multires []byte var multires []byte
rp := parseArrayPath(path) rp := parseArrayPath(path)
if !rp.arrch { if !rp.arrch {
n, err := strconv.ParseUint(rp.part, 10, 64) n, ok := parseUint(rp.part)
if err != nil { if !ok {
partidx = -1 partidx = -1
} else { } else {
partidx = int(n) partidx = int(n)
@ -2232,6 +2252,9 @@ func validstring(data []byte, i int) (outi int, ok bool) {
return i, false return i, false
} else if data[i] == '\\' { } else if data[i] == '\\' {
i++ i++
if i == len(data) {
return i, false
}
switch data[i] { switch data[i] {
default: default:
return i, false return i, false
@ -2348,3 +2371,62 @@ func Valid(json string) bool {
_, ok := validpayload([]byte(json), 0) _, ok := validpayload([]byte(json), 0)
return ok return ok
} }
func parseUint(s string) (n uint64, ok bool) {
var i int
if i == len(s) {
return 0, false
}
for ; i < len(s); i++ {
if s[i] >= '0' && s[i] <= '9' {
n = n*10 + uint64(s[i]-'0')
} else {
return 0, false
}
}
return n, true
}
func parseInt(s string) (n int64, ok bool) {
var i int
var sign bool
if len(s) > 0 && s[0] == '-' {
sign = true
i++
}
if i == len(s) {
return 0, false
}
for ; i < len(s); i++ {
if s[i] >= '0' && s[i] <= '9' {
n = n*10 + int64(s[i]-'0')
} else {
return 0, false
}
}
if sign {
return n * -1, true
}
return n, true
}
const minUint53 = 0
const maxUint53 = 4503599627370495
const minInt53 = -2251799813685248
const maxInt53 = 2251799813685247
func floatToUint(f float64) (n uint64, ok bool) {
n = uint64(f)
if float64(n) == f && n >= minUint53 && n <= maxUint53 {
return n, true
}
return 0, false
}
func floatToInt(f float64) (n int64, ok bool) {
n = int64(f)
if float64(n) == f && n >= minInt53 && n <= maxInt53 {
return n, true
}
return 0, false
}

View File

@ -8,6 +8,7 @@ import (
"io" "io"
"math/rand" "math/rand"
"reflect" "reflect"
"strconv"
"strings" "strings"
"testing" "testing"
"time" "time"
@ -228,6 +229,52 @@ func TestBasic(t *testing.T) {
t.Fatalf("expected %v, got %v", `["Brett","Elliotte"]`, mtok.String()) t.Fatalf("expected %v, got %v", `["Brett","Elliotte"]`, mtok.String())
} }
} }
func TestPlus53BitInts(t *testing.T) {
json := `{"IdentityData":{"GameInstanceId":634866135153775564}}`
value := Get(json, "IdentityData.GameInstanceId")
assert(t, value.Uint() == 634866135153775564)
assert(t, value.Int() == 634866135153775564)
assert(t, value.Float() == 634866135153775616)
json = `{"IdentityData":{"GameInstanceId":634866135153775564.88172}}`
value = Get(json, "IdentityData.GameInstanceId")
assert(t, value.Uint() == 634866135153775616)
assert(t, value.Int() == 634866135153775616)
assert(t, value.Float() == 634866135153775616.88172)
json = `{
"min_uint64": 0,
"max_uint64": 18446744073709551615,
"overflow_uint64": 18446744073709551616,
"min_int64": -9223372036854775808,
"max_int64": 9223372036854775807,
"overflow_int64": 9223372036854775808,
"min_uint53": 0,
"max_uint53": 4503599627370495,
"overflow_uint53": 4503599627370496,
"min_int53": -2251799813685248,
"max_int53": 2251799813685247,
"overflow_int53": 2251799813685248
}`
assert(t, Get(json, "min_uint53").Uint() == 0)
assert(t, Get(json, "max_uint53").Uint() == 4503599627370495)
assert(t, Get(json, "overflow_uint53").Int() == 4503599627370496)
assert(t, Get(json, "min_int53").Int() == -2251799813685248)
assert(t, Get(json, "max_int53").Int() == 2251799813685247)
assert(t, Get(json, "overflow_int53").Int() == 2251799813685248)
assert(t, Get(json, "min_uint64").Uint() == 0)
assert(t, Get(json, "max_uint64").Uint() == 18446744073709551615)
// this next value overflows the max uint64 by one which will just
// flip the number to zero
assert(t, Get(json, "overflow_uint64").Int() == 0)
assert(t, Get(json, "min_int64").Int() == -9223372036854775808)
assert(t, Get(json, "max_int64").Int() == 9223372036854775807)
// this next value overflows the max int64 by one which will just
// flip the number to the negative sign.
assert(t, Get(json, "overflow_int64").Int() == -9223372036854775808)
}
func TestTypes(t *testing.T) { func TestTypes(t *testing.T) {
assert(t, (Result{Type: String}).Type.String() == "String") assert(t, (Result{Type: String}).Type.String() == "String")
assert(t, (Result{Type: Number}).Type.String() == "Number") assert(t, (Result{Type: Number}).Type.String() == "Number")
@ -495,7 +542,7 @@ func TestUnescape(t *testing.T) {
} }
func assert(t testing.TB, cond bool) { func assert(t testing.TB, cond bool) {
if !cond { if !cond {
t.Fatal("assert failed") panic("assert failed")
} }
} }
func TestLess(t *testing.T) { func TestLess(t *testing.T) {
@ -1553,3 +1600,27 @@ func BenchmarkConvertGetBytes(t *testing.B) {
GetBytes(data, "50.widget.text.onMouseUp") GetBytes(data, "50.widget.text.onMouseUp")
} }
} }
func BenchmarkParseUintNumParser(t *testing.B) {
var s = "634866135153775564"
for i := 0; i < t.N; i++ {
parseUint(s)
}
}
func BenchmarkStdlibParseUintNumParser(t *testing.B) {
var s = "634866135153775564"
for i := 0; i < t.N; i++ {
strconv.ParseUint(s, 10, 64)
}
}
func BenchmarkParseIntNumParser(t *testing.B) {
var s = "-634866135153775564"
for i := 0; i < t.N; i++ {
parseInt(s)
}
}
func BenchmarkStdlibParseIntNumParser(t *testing.B) {
var s = "-634866135153775564"
for i := 0; i < t.N; i++ {
strconv.ParseInt(s, 10, 64)
}
}