re
/
go-json
forked from mirror/go-json
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix []byte with unmarshaler

This commit is contained in:
Masaaki Goshima 2020-12-24 14:26:18 +09:00
parent 9f125be311
commit ba4d7d2885
4 changed files with 126 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

63
decode_bytes.go
View File

@ -6,16 +6,37 @@ import (
)
type bytesDecoder struct {
structName string
fieldName string
typ *rtype
sliceDecoder decoder
structName string
fieldName string
}
func newBytesDecoder(structName string, fieldName string) *bytesDecoder {
return &bytesDecoder{structName: structName, fieldName: fieldName}
func byteUnmarshalerSliceDecoder(typ *rtype, structName string, fieldName string) decoder {
var unmarshalDecoder decoder
switch {
case rtype_ptrTo(typ).Implements(unmarshalJSONType):
unmarshalDecoder = newUnmarshalJSONDecoder(rtype_ptrTo(typ), structName, fieldName)
case rtype_ptrTo(typ).Implements(unmarshalTextType):
unmarshalDecoder = newUnmarshalTextDecoder(rtype_ptrTo(typ), structName, fieldName)
}
if unmarshalDecoder == nil {
return nil
}
return newSliceDecoder(unmarshalDecoder, typ, 1, structName, fieldName)
}
func newBytesDecoder(typ *rtype, structName string, fieldName string) *bytesDecoder {
return &bytesDecoder{
typ: typ,
sliceDecoder: byteUnmarshalerSliceDecoder(typ, structName, fieldName),
structName: structName,
fieldName: fieldName,
}
}
func (d *bytesDecoder) decodeStream(s *stream, p unsafe.Pointer) error {
bytes, err := d.decodeStreamBinary(s)
bytes, err := d.decodeStreamBinary(s, p)
if err != nil {
return err
}
@ -34,10 +55,13 @@ func (d *bytesDecoder) decodeStream(s *stream, p unsafe.Pointer) error {
}
func (d *bytesDecoder) decode(buf []byte, cursor int64, p unsafe.Pointer) (int64, error) {
bytes, c, err := d.decodeBinary(buf, cursor)
bytes, c, err := d.decodeBinary(buf, cursor, p)
if err != nil {
return 0, err
}
if bytes == nil {
return c, nil
}
cursor = c
decodedLen := base64.StdEncoding.DecodedLen(len(bytes))
b := make([]byte, decodedLen)
@ -69,7 +93,7 @@ ERROR:
return nil, errUnexpectedEndOfJSON("[]byte", s.totalOffset())
}
func (d *bytesDecoder) decodeStreamBinary(s *stream) ([]byte, error) {
func (d *bytesDecoder) decodeStreamBinary(s *stream, p unsafe.Pointer) ([]byte, error) {
for {
switch s.char() {
case ' ', '\n', '\t', '\r':
@ -82,6 +106,17 @@ func (d *bytesDecoder) decodeStreamBinary(s *stream) ([]byte, error) {
return nil, err
}
return nil, nil
case '[':
if d.sliceDecoder == nil {
return nil, &UnmarshalTypeError{
Type: rtype2type(d.typ),
Offset: s.totalOffset(),
}
}
if err := d.sliceDecoder.decodeStream(s, p); err != nil {
return nil, err
}
return nil, nil
case nul:
if s.read() {
continue
@ -92,7 +127,7 @@ func (d *bytesDecoder) decodeStreamBinary(s *stream) ([]byte, error) {
return nil, errNotAtBeginningOfValue(s.totalOffset())
}
func (d *bytesDecoder) decodeBinary(buf []byte, cursor int64) ([]byte, int64, error) {
func (d *bytesDecoder) decodeBinary(buf []byte, cursor int64, p unsafe.Pointer) ([]byte, int64, error) {
for {
switch buf[cursor] {
case ' ', '\n', '\t', '\r':
@ -112,6 +147,18 @@ func (d *bytesDecoder) decodeBinary(buf []byte, cursor int64) ([]byte, int64, er
cursor++
}
return nil, 0, errUnexpectedEndOfJSON("[]byte", cursor)
case '[':
if d.sliceDecoder == nil {
return nil, 0, &UnmarshalTypeError{
Type: rtype2type(d.typ),
Offset: cursor,
}
}
c, err := d.sliceDecoder.decode(buf, cursor, p)
if err != nil {
return nil, 0, err
}
return nil, c, nil
case 'n':
buflen := int64(len(buf))
if cursor+3 >= buflen {

23
decode_compile.go
View File

@ -32,7 +32,7 @@ func (d *Decoder) compile(typ *rtype, structName, fieldName string) (decoder, er
case reflect.Slice:
elem := typ.Elem()
if elem.Kind() == reflect.Uint8 {
return d.compileBytes(structName, fieldName)
return d.compileBytes(elem, structName, fieldName)
}
return d.compileSlice(typ, structName, fieldName)
case reflect.Array:
@ -72,17 +72,24 @@ func (d *Decoder) compile(typ *rtype, structName, fieldName string) (decoder, er
case reflect.Float64:
return d.compileFloat64(structName, fieldName)
}
return nil, &UnsupportedTypeError{Type: rtype2type(typ)}
return nil, &UnmarshalTypeError{
Value: "object",
Type: rtype2type(typ),
Offset: 0,
}
}
func (d *Decoder) compileMapKey(typ *rtype, structName, fieldName string) (decoder, error) {
if rtype_ptrTo(typ).Implements(unmarshalTextType) {
return newUnmarshalTextDecoder(rtype_ptrTo(typ), structName, fieldName), nil
}
dec, err := d.compile(typ, structName, fieldName)
if err != nil {
return nil, err
}
for {
switch t := dec.(type) {
case *stringDecoder, *interfaceDecoder, *unmarshalJSONDecoder, *unmarshalTextDecoder:
case *stringDecoder, *interfaceDecoder:
return dec, nil
case *boolDecoder, *intDecoder, *uintDecoder, *numberDecoder:
return newWrappedStringDecoder(dec, structName, fieldName), nil
@ -93,7 +100,11 @@ func (d *Decoder) compileMapKey(typ *rtype, structName, fieldName string) (decod
}
}
ERROR:
return nil, &UnsupportedTypeError{Type: rtype2type(typ)}
return nil, &UnmarshalTypeError{
Value: "object",
Type: rtype2type(typ),
Offset: 0,
}
}
func (d *Decoder) compilePtr(typ *rtype, structName, fieldName string) (decoder, error) {
@ -184,8 +195,8 @@ func (d *Decoder) compileBool(structName, fieldName string) (decoder, error) {
return newBoolDecoder(structName, fieldName), nil
}
func (d *Decoder) compileBytes(structName, fieldName string) (decoder, error) {
return newBytesDecoder(structName, fieldName), nil
func (d *Decoder) compileBytes(typ *rtype, structName, fieldName string) (decoder, error) {
return newBytesDecoder(typ, structName, fieldName), nil
}
func (d *Decoder) compileSlice(typ *rtype, structName, fieldName string) (decoder, error) {

13
decode_string.go
View File

@ -4,6 +4,7 @@ import (
"reflect"
"unicode"
"unicode/utf16"
"unicode/utf8"
"unsafe"
)
@ -149,6 +150,16 @@ RETRY:
return nil
}
func appendCoerceInvalidUTF8(b []byte, s []byte) []byte {
c := [4]byte{}
for _, r := range string(s) {
b = append(b, c[:utf8.EncodeRune(c[:], r)]...)
}
return b
}
func stringBytes(s *stream) ([]byte, error) {
s.cursor++
start := s.cursor
@ -160,6 +171,8 @@ func stringBytes(s *stream) ([]byte, error) {
}
case '"':
literal := s.buf[start:s.cursor]
// TODO: this flow is so slow sequence.
// literal = appendCoerceInvalidUTF8(make([]byte, 0, len(literal)), literal)
s.cursor++
return literal, nil
case nul:

82
decode_test.go
View File

@ -1005,48 +1005,48 @@ var unmarshalTests = []unmarshalTest{
ptr: new(string),
out: "hello\ufffd\ufffdworld",
},
{
in: "\"hello\\ud800\\ud800world\"", // 101
ptr: new(string),
out: "hello\ufffd\ufffdworld",
},
/*
{
in: "\"hello\xed\xa0\x80\xed\xb0\x80world\"", // 102
ptr: new(string),
out: "hello\ufffd\ufffd\ufffd\ufffd\ufffd\ufffdworld",
},
*/
// Used to be issue 8305, but time.Time implements encoding.TextUnmarshaler so this works now.
{
in: `{"2009-11-10T23:00:00Z": "hello world"}`, // 103
ptr: new(map[time.Time]string),
out: map[time.Time]string{time.Date(2009, 11, 10, 23, 0, 0, 0, time.UTC): "hello world"},
},
// issue 8305
{
in: `{"2009-11-10T23:00:00Z": "hello world"}`, // 104
ptr: new(map[Point]string),
err: &json.UnmarshalTypeError{Value: "object", Type: reflect.TypeOf(Point{}), Offset: 0},
},
{
in: `{"asdf": "hello world"}`, // 105
ptr: new(map[unmarshaler]string),
err: &json.UnmarshalTypeError{Value: "object", Type: reflect.TypeOf(unmarshaler{}), Offset: 1},
},
// related to issue 13783.
// Go 1.7 changed marshaling a slice of typed byte to use the methods on the byte type,
// similar to marshaling a slice of typed int.
// These tests check that, assuming the byte type also has valid decoding methods,
// either the old base64 string encoding or the new per-element encoding can be
// successfully unmarshaled. The custom unmarshalers were accessible in earlier
// versions of Go, even though the custom marshaler was not.
{
in: `"AQID"`, // 106
ptr: new([]byteWithMarshalJSON),
out: []byteWithMarshalJSON{1, 2, 3},
},
/*
{
in: "\"hello\\ud800\\ud800world\"", // 101
ptr: new(string),
out: "hello\ufffd\ufffdworld",
},
{
in: "\"hello\xed\xa0\x80\xed\xb0\x80world\"", // 102
ptr: new(string),
out: "hello\ufffd\ufffd\ufffd\ufffd\ufffd\ufffdworld",
},
// Used to be issue 8305, but time.Time implements encoding.TextUnmarshaler so this works now.
{
in: `{"2009-11-10T23:00:00Z": "hello world"}`, // 103
ptr: new(map[time.Time]string),
out: map[time.Time]string{time.Date(2009, 11, 10, 23, 0, 0, 0, time.UTC): "hello world"},
},
// issue 8305
{
in: `{"2009-11-10T23:00:00Z": "hello world"}`, // 104
ptr: new(map[Point]string),
err: &json.UnmarshalTypeError{Value: "object", Type: reflect.TypeOf(map[Point]string{}), Offset: 1},
},
{
in: `{"asdf": "hello world"}`, // 105
ptr: new(map[unmarshaler]string),
err: &json.UnmarshalTypeError{Value: "object", Type: reflect.TypeOf(map[unmarshaler]string{}), Offset: 1},
},
// related to issue 13783.
// Go 1.7 changed marshaling a slice of typed byte to use the methods on the byte type,
// similar to marshaling a slice of typed int.
// These tests check that, assuming the byte type also has valid decoding methods,
// either the old base64 string encoding or the new per-element encoding can be
// successfully unmarshaled. The custom unmarshalers were accessible in earlier
// versions of Go, even though the custom marshaler was not.
{
in: `"AQID"`, // 106
ptr: new([]byteWithMarshalJSON),
out: []byteWithMarshalJSON{1, 2, 3},
},
{
in: `["Z01","Z02","Z03"]`, // 107
ptr: new([]byteWithMarshalJSON),