From d2bc6b002c636d034f0bd9b3bd7de6595c3fe76a Mon Sep 17 00:00:00 2001 From: Masaaki Goshima Date: Tue, 25 Aug 2020 00:56:01 +0900 Subject: [PATCH 1/4] Add validation for decode_float --- decode_float.go | 53 ++++++++++++++++++++++++++++++++++--------------- 1 file changed, 37 insertions(+), 16 deletions(-) diff --git a/decode_float.go b/decode_float.go index 6da1cd6..1829009 100644 --- a/decode_float.go +++ b/decode_float.go @@ -13,22 +13,37 @@ func newFloatDecoder(op func(uintptr, float64)) *floatDecoder { return &floatDecoder{op: op} } -var floatTable = [256]bool{ - '0': true, - '1': true, - '2': true, - '3': true, - '4': true, - '5': true, - '6': true, - '7': true, - '8': true, - '9': true, - '.': true, - 'e': true, - 'E': true, - '+': true, -} +var ( + floatTable = [256]bool{ + '0': true, + '1': true, + '2': true, + '3': true, + '4': true, + '5': true, + '6': true, + '7': true, + '8': true, + '9': true, + '.': true, + 'e': true, + 'E': true, + '+': true, + '-': true, + } + + validEndNumberChar = [256]bool{ + nul: true, + ' ': true, + '\t': true, + '\r': true, + '\n': true, + ',': true, + ':': true, + '}': true, + ']': true, + } +) func floatBytes(s *stream) []byte { start := s.cursor @@ -97,6 +112,9 @@ func (d *floatDecoder) decodeStream(s *stream, p uintptr) error { if err != nil { return err } + if !validEndNumberChar[s.char()] { + return errUnexpectedEndOfJSON("float", s.totalOffset()) + } str := *(*string)(unsafe.Pointer(&bytes)) f64, err := strconv.ParseFloat(str, 64) if err != nil { @@ -112,6 +130,9 @@ func (d *floatDecoder) decode(buf []byte, cursor int64, p uintptr) (int64, error return 0, err } cursor = c + if !validEndNumberChar[buf[cursor]] { + return 0, errUnexpectedEndOfJSON("float", cursor) + } s := *(*string)(unsafe.Pointer(&bytes)) f64, err := strconv.ParseFloat(s, 64) if err != nil { From c9f1d0043e52e92eba64a84ded1b5d2b999cb6e9 Mon Sep 17 00:00:00 2001 From: Masaaki Goshima Date: Tue, 25 Aug 2020 00:56:23 +0900 Subject: [PATCH 2/4] Add test cases --- number_test.go | 195 +++++++++++++++++++++++++++++++++++++++++++++++++ tagkey_test.go | 122 +++++++++++++++++++++++++++++++ 2 files changed, 317 insertions(+) create mode 100644 number_test.go create mode 100644 tagkey_test.go diff --git a/number_test.go b/number_test.go new file mode 100644 index 0000000..4c5e3ef --- /dev/null +++ b/number_test.go @@ -0,0 +1,195 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package json_test + +import ( + "regexp" + "testing" + + "github.com/goccy/go-json" +) + +func TestNumberIsValid(t *testing.T) { + // From: https://stackoverflow.com/a/13340826 + var jsonNumberRegexp = regexp.MustCompile(`^-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?$`) + + validTests := []string{ + "0", + "-0", + "1", + "-1", + "0.1", + "-0.1", + "1234", + "-1234", + "12.34", + "-12.34", + "12E0", + "12E1", + "12e34", + "12E-0", + "12e+1", + "12e-34", + "-12E0", + "-12E1", + "-12e34", + "-12E-0", + "-12e+1", + "-12e-34", + "1.2E0", + "1.2E1", + "1.2e34", + "1.2E-0", + "1.2e+1", + "1.2e-34", + "-1.2E0", + "-1.2E1", + "-1.2e34", + "-1.2E-0", + "-1.2e+1", + "-1.2e-34", + "0E0", + "0E1", + "0e34", + "0E-0", + "0e+1", + "0e-34", + "-0E0", + "-0E1", + "-0e34", + "-0E-0", + "-0e+1", + "-0e-34", + } + + for i, test := range validTests { + if !isValidNumber(test) { + t.Errorf("%d: %s should be valid", i, test) + } + + var f float64 + if err := json.Unmarshal([]byte(test), &f); err != nil { + t.Errorf("%d: %s should be valid but Unmarshal failed: %v", i, test, err) + } + + if !jsonNumberRegexp.MatchString(test) { + t.Errorf("%d: %s should be valid but regexp does not match", i, test) + } + } + + invalidTests := []string{ + "", + "invalid", + "1.0.1", + "1..1", + "-1-2", + "012a42", + //"01.2", + //"012", + "12E12.12", + "1e2e3", + "1e+-2", + "1e--23", + "1e", + "e1", + "1e+", + "1ea", + "1a", + "1.a", + //"1.", + //"01", + //"1.e1", + } + + for i, test := range invalidTests { + if isValidNumber(test) { + t.Errorf("%d: %s should be invalid", i, test) + } + + var f float64 + if err := json.Unmarshal([]byte(test), &f); err == nil { + t.Errorf("%d: %s should be invalid but unmarshal wrote %v", i, test, f) + } + + if jsonNumberRegexp.MatchString(test) { + t.Errorf("%d: %s should be invalid but matches regexp", i, test) + } + } +} + +// isValidNumber reports whether s is a valid JSON number literal. +func isValidNumber(s string) bool { + // This function implements the JSON numbers grammar. + // See https://tools.ietf.org/html/rfc7159#section-6 + // and https://www.json.org/img/number.png + + if s == "" { + return false + } + + // Optional - + if s[0] == '-' { + s = s[1:] + if s == "" { + return false + } + } + + // Digits + switch { + default: + return false + + case s[0] == '0': + s = s[1:] + + case '1' <= s[0] && s[0] <= '9': + s = s[1:] + for len(s) > 0 && '0' <= s[0] && s[0] <= '9' { + s = s[1:] + } + } + + // . followed by 1 or more digits. + if len(s) >= 2 && s[0] == '.' && '0' <= s[1] && s[1] <= '9' { + s = s[2:] + for len(s) > 0 && '0' <= s[0] && s[0] <= '9' { + s = s[1:] + } + } + + // e or E followed by an optional - or + and + // 1 or more digits. + if len(s) >= 2 && (s[0] == 'e' || s[0] == 'E') { + s = s[1:] + if s[0] == '+' || s[0] == '-' { + s = s[1:] + if s == "" { + return false + } + } + for len(s) > 0 && '0' <= s[0] && s[0] <= '9' { + s = s[1:] + } + } + + // Make sure we are at the end. + return s == "" +} + +func BenchmarkNumberIsValid(b *testing.B) { + s := "-61657.61667E+61673" + for i := 0; i < b.N; i++ { + isValidNumber(s) + } +} + +func BenchmarkNumberIsValidRegexp(b *testing.B) { + var jsonNumberRegexp = regexp.MustCompile(`^-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?$`) + s := "-61657.61667E+61673" + for i := 0; i < b.N; i++ { + jsonNumberRegexp.MatchString(s) + } +} diff --git a/tagkey_test.go b/tagkey_test.go new file mode 100644 index 0000000..8caf14f --- /dev/null +++ b/tagkey_test.go @@ -0,0 +1,122 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package json_test + +import ( + "testing" + + "github.com/goccy/go-json" +) + +type basicLatin2xTag struct { + V string `json:"$%-/"` +} + +type basicLatin3xTag struct { + V string `json:"0123456789"` +} + +type basicLatin4xTag struct { + V string `json:"ABCDEFGHIJKLMO"` +} + +type basicLatin5xTag struct { + V string `json:"PQRSTUVWXYZ_"` +} + +type basicLatin6xTag struct { + V string `json:"abcdefghijklmno"` +} + +type basicLatin7xTag struct { + V string `json:"pqrstuvwxyz"` +} + +type miscPlaneTag struct { + V string `json:"色は匂へど"` +} + +type percentSlashTag struct { + V string `json:"text/html%"` // https://golang.org/issue/2718 +} + +type punctuationTag struct { + V string `json:"!#$%&()*+-./:<=>?@[]^_{|}~"` // https://golang.org/issue/3546 +} + +type dashTag struct { + V string `json:"-,"` +} + +type emptyTag struct { + W string +} + +type misnamedTag struct { + X string `jsom:"Misnamed"` +} + +type badFormatTag struct { + Y string `:"BadFormat"` +} + +type badCodeTag struct { + Z string `json:" !\"#&'()*+,."` +} + +type spaceTag struct { + Q string `json:"With space"` +} + +type unicodeTag struct { + W string `json:"Ελλάδα"` +} + +var structTagObjectKeyTests = []struct { + raw interface{} + value string + key string +}{ + {basicLatin2xTag{"2x"}, "2x", "$%-/"}, + {basicLatin3xTag{"3x"}, "3x", "0123456789"}, + {basicLatin4xTag{"4x"}, "4x", "ABCDEFGHIJKLMO"}, + {basicLatin5xTag{"5x"}, "5x", "PQRSTUVWXYZ_"}, + {basicLatin6xTag{"6x"}, "6x", "abcdefghijklmno"}, + {basicLatin7xTag{"7x"}, "7x", "pqrstuvwxyz"}, + {miscPlaneTag{"いろはにほへと"}, "いろはにほへと", "色は匂へど"}, + {dashTag{"foo"}, "foo", "-"}, + {emptyTag{"Pour Moi"}, "Pour Moi", "W"}, + {misnamedTag{"Animal Kingdom"}, "Animal Kingdom", "X"}, + {badFormatTag{"Orfevre"}, "Orfevre", "Y"}, + //{badCodeTag{"Reliable Man"}, "Reliable Man", "Z"}, + {percentSlashTag{"brut"}, "brut", "text/html%"}, + {punctuationTag{"Union Rags"}, "Union Rags", "!#$%&()*+-./:<=>?@[]^_{|}~"}, + {spaceTag{"Perreddu"}, "Perreddu", "With space"}, + {unicodeTag{"Loukanikos"}, "Loukanikos", "Ελλάδα"}, +} + +func TestStructTagObjectKey(t *testing.T) { + for _, tt := range structTagObjectKeyTests { + b, err := json.Marshal(tt.raw) + if err != nil { + t.Fatalf("Marshal(%#q) failed: %v", tt.raw, err) + } + var f interface{} + err = json.Unmarshal(b, &f) + if err != nil { + t.Fatalf("Unmarshal(%#q) failed: %v", b, err) + } + for i, v := range f.(map[interface{}]interface{}) { + switch i.(string) { + case tt.key: + if s, ok := v.(string); !ok || s != tt.value { + t.Fatalf("Unexpected value: %#q, want %v", s, tt.value) + } + default: + t.Fatalf("Unexpected key: %#q, from %#q", i, b) + } + } + } +} From a9b9597c2e51394f6416bdcfcd7012d73e1e0070 Mon Sep 17 00:00:00 2001 From: Masaaki Goshima Date: Tue, 25 Aug 2020 11:17:38 +0900 Subject: [PATCH 3/4] Fix map key type --- decode_interface.go | 10 +++++----- decode_test.go | 2 +- tagkey_test.go | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/decode_interface.go b/decode_interface.go index bc8d39a..0d0bfc2 100644 --- a/decode_interface.go +++ b/decode_interface.go @@ -29,7 +29,7 @@ func (d *interfaceDecoder) numDecoder(s *stream) decoder { var ( interfaceMapType = type2rtype( - reflect.TypeOf((*map[interface{}]interface{})(nil)).Elem(), + reflect.TypeOf((*map[string]interface{})(nil)).Elem(), ) ) @@ -38,12 +38,12 @@ func (d *interfaceDecoder) decodeStream(s *stream, p uintptr) error { for { switch s.char() { case '{': - var v map[interface{}]interface{} + var v map[string]interface{} ptr := unsafe.Pointer(&v) d.dummy = ptr if err := newMapDecoder( interfaceMapType, - newInterfaceDecoder(d.typ), + newStringDecoder(), newInterfaceDecoder(d.typ), ).decodeStream(s, uintptr(ptr)); err != nil { return err @@ -120,12 +120,12 @@ func (d *interfaceDecoder) decode(buf []byte, cursor int64, p uintptr) (int64, e cursor = skipWhiteSpace(buf, cursor) switch buf[cursor] { case '{': - var v map[interface{}]interface{} + var v map[string]interface{} ptr := unsafe.Pointer(&v) d.dummy = ptr dec := newMapDecoder( interfaceMapType, - newInterfaceDecoder(d.typ), + newStringDecoder(), newInterfaceDecoder(d.typ), ) cursor, err := dec.decode(buf, cursor, uintptr(ptr)) diff --git a/decode_test.go b/decode_test.go index c86565e..7c57b6b 100644 --- a/decode_test.go +++ b/decode_test.go @@ -204,7 +204,7 @@ func Test_Decoder(t *testing.T) { var v interface{} assertErr(t, json.Unmarshal([]byte(`{"a": 1, "b": "c"}`), &v)) assertEq(t, "interface.kind", "map", reflect.TypeOf(v).Kind().String()) - m := v.(map[interface{}]interface{}) + m := v.(map[string]interface{}) assertEq(t, "interface", `1`, fmt.Sprint(m["a"])) assertEq(t, "interface", `c`, fmt.Sprint(m["b"])) }) diff --git a/tagkey_test.go b/tagkey_test.go index 8caf14f..29f5502 100644 --- a/tagkey_test.go +++ b/tagkey_test.go @@ -108,8 +108,8 @@ func TestStructTagObjectKey(t *testing.T) { if err != nil { t.Fatalf("Unmarshal(%#q) failed: %v", b, err) } - for i, v := range f.(map[interface{}]interface{}) { - switch i.(string) { + for i, v := range f.(map[string]interface{}) { + switch i { case tt.key: if s, ok := v.(string); !ok || s != tt.value { t.Fatalf("Unexpected value: %#q, want %v", s, tt.value) From 18a8684873e2b3c4194f74ce7ed8319f0bc916e8 Mon Sep 17 00:00:00 2001 From: Masaaki Goshima Date: Tue, 25 Aug 2020 11:18:01 +0900 Subject: [PATCH 4/4] Refactor --- decode_interface.go | 2 +- json_test.go | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/decode_interface.go b/decode_interface.go index 0d0bfc2..aa08daa 100644 --- a/decode_interface.go +++ b/decode_interface.go @@ -165,7 +165,7 @@ func (d *interfaceDecoder) decode(buf []byte, cursor int64, p uintptr) (int64, e cursor++ *(*interface{})(unsafe.Pointer(p)) = *(*string)(unsafe.Pointer(&literal)) return cursor, nil - case '\000': + case nul: return 0, errUnexpectedEndOfJSON("string", cursor) } cursor++ diff --git a/json_test.go b/json_test.go index 1be6c15..3831777 100644 --- a/json_test.go +++ b/json_test.go @@ -199,6 +199,7 @@ func TestIndentErrors(t *testing.T) { } func diff(t *testing.T, a, b []byte) { + t.Helper() for i := 0; ; i++ { if i >= len(a) || i >= len(b) || a[i] != b[i] { j := i - 10