From 71d6f845e5c9afe461c467ee0e675860e294d75f Mon Sep 17 00:00:00 2001 From: Masaaki Goshima Date: Sat, 25 Apr 2020 19:55:05 +0900 Subject: [PATCH] Add decoder for Array or Slice type --- decode.go | 69 +++++++++++++++++++++++++++++++++++++---------- decode_array.go | 52 +++++++++++++++++++++++++++++++++++ decode_slice.go | 72 +++++++++++++++++++++++++++++++++++++++++++++++++ decode_test.go | 19 +++++++++---- 4 files changed, 193 insertions(+), 19 deletions(-) create mode 100644 decode_array.go create mode 100644 decode_slice.go diff --git a/decode.go b/decode.go index 158532c..2b18a56 100644 --- a/decode.go +++ b/decode.go @@ -1,6 +1,7 @@ package json import ( + "bytes" "io" "reflect" "strings" @@ -8,25 +9,26 @@ import ( "unsafe" ) +// A Token holds a value of one of these types: +// +// Delim, for the four JSON delimiters [ ] { } +// bool, for JSON booleans +// float64, for JSON numbers +// Number, for JSON numbers +// string, for JSON string literals +// nil, for JSON null +// type Token interface{} type Delim rune -const ( - stateNone int = iota - stateLiteral - stateObject - stateArray -) - type decoder interface { decode(*context, uintptr) error } type Decoder struct { - r io.Reader - state int - value []byte + r io.Reader + buffered func() io.Reader } var ( @@ -43,14 +45,18 @@ func init() { } } +// NewDecoder returns a new decoder that reads from r. +// +// The decoder introduces its own buffering and may +// read data from r beyond the JSON values requested. func NewDecoder(r io.Reader) *Decoder { - return &Decoder{ - r: r, - } + return &Decoder{r: r} } +// Buffered returns a reader of the data remaining in the Decoder's +// buffer. The reader is valid until the next call to Decode. func (d *Decoder) Buffered() io.Reader { - return d.r + return d.buffered() } func (d *Decoder) decode(src []byte, header *interfaceHeader) error { @@ -92,6 +98,11 @@ func (d *Decoder) decodeForUnmarshalNoEscape(src []byte, v interface{}) error { return d.decode(src, header) } +// Decode reads the next JSON-encoded value from its +// input and stores it in the value pointed to by v. +// +// See the documentation for Unmarshal for details about +// the conversion of JSON into a Go value. func (d *Decoder) Decode(v interface{}) error { header := (*interfaceHeader)(unsafe.Pointer(&v)) typ := header.typ @@ -113,6 +124,9 @@ func (d *Decoder) Decode(v interface{}) error { ptr := uintptr(header.ptr) ctx := ctxPool.Get().(*context) defer ctxPool.Put(ctx) + d.buffered = func() io.Reader { + return bytes.NewReader(ctx.buf[ctx.cursor:]) + } for { buf := make([]byte, 1024) n, err := d.r.Read(buf) @@ -136,6 +150,10 @@ func (d *Decoder) compile(typ *rtype) (decoder, error) { return d.compilePtr(typ) case reflect.Struct: return d.compileStruct(typ) + case reflect.Slice: + return d.compileSlice(typ) + case reflect.Array: + return d.compileArray(typ) case reflect.Int: return d.compileInt() case reflect.Int8: @@ -256,6 +274,24 @@ func (d *Decoder) compileBool() (decoder, error) { return newBoolDecoder(), nil } +func (d *Decoder) compileSlice(typ *rtype) (decoder, error) { + elem := typ.Elem() + decoder, err := d.compile(elem) + if err != nil { + return nil, err + } + return newSliceDecoder(decoder, elem, elem.Size()), nil +} + +func (d *Decoder) compileArray(typ *rtype) (decoder, error) { + elem := typ.Elem() + decoder, err := d.compile(elem) + if err != nil { + return nil, err + } + return newArrayDecoder(decoder, elem, typ.Len()), nil +} + func (d *Decoder) getTag(field reflect.StructField) string { return field.Tag.Get("json") } @@ -300,6 +336,9 @@ func (d *Decoder) compileStruct(typ *rtype) (decoder, error) { return newStructDecoder(fieldMap), nil } +// DisallowUnknownFields causes the Decoder to return an error when the destination +// is a struct and the input contains object keys which do not match any +// non-ignored, exported fields in the destination. func (d *Decoder) DisallowUnknownFields() { } @@ -316,6 +355,8 @@ func (d *Decoder) Token() (Token, error) { return nil, nil } +// UseNumber causes the Decoder to unmarshal a number into an interface{} as a +// Number instead of as a float64. func (d *Decoder) UseNumber() { } diff --git a/decode_array.go b/decode_array.go new file mode 100644 index 0000000..13258b2 --- /dev/null +++ b/decode_array.go @@ -0,0 +1,52 @@ +package json + +import ( + "errors" +) + +type arrayDecoder struct { + elemType *rtype + size uintptr + valueDecoder decoder + alen int +} + +func newArrayDecoder(dec decoder, elemType *rtype, alen int) *arrayDecoder { + return &arrayDecoder{ + valueDecoder: dec, + elemType: elemType, + size: elemType.Size(), + alen: alen, + } +} + +func (d *arrayDecoder) decode(ctx *context, p uintptr) error { + buf := ctx.buf + buflen := ctx.buflen + cursor := ctx.cursor + for ; cursor < buflen; cursor++ { + switch buf[cursor] { + case ' ', '\n', '\t', '\r': + continue + case '[': + idx := 0 + for { + ctx.cursor = cursor + 1 + if err := d.valueDecoder.decode(ctx, p+uintptr(idx)*d.size); err != nil { + return err + } + cursor = ctx.skipWhiteSpace() + switch buf[cursor] { + case ']': + return nil + case ',': + idx++ + continue + default: + return errors.New("syntax error array") + } + } + } + } + return errors.New("unexpected error array") +} diff --git a/decode_slice.go b/decode_slice.go new file mode 100644 index 0000000..ce01dff --- /dev/null +++ b/decode_slice.go @@ -0,0 +1,72 @@ +package json + +import ( + "errors" + "reflect" + "unsafe" +) + +type sliceDecoder struct { + elemType *rtype + valueDecoder decoder + size uintptr +} + +func newSliceDecoder(dec decoder, elemType *rtype, size uintptr) *sliceDecoder { + return &sliceDecoder{ + valueDecoder: dec, + elemType: elemType, + size: size, + } +} + +//go:linkname copySlice reflect.typedslicecopy +func copySlice(elemType *rtype, dst, src reflect.SliceHeader) int + +//go:linkname newArray reflect.unsafe_NewArray +func newArray(*rtype, int) unsafe.Pointer + +func (d *sliceDecoder) decode(ctx *context, p uintptr) error { + buf := ctx.buf + buflen := ctx.buflen + cursor := ctx.cursor + for ; cursor < buflen; cursor++ { + switch buf[cursor] { + case ' ', '\n', '\t', '\r': + continue + case '[': + idx := 0 + cap := 2 + data := uintptr(newArray(d.elemType, cap)) + for { + ctx.cursor = cursor + 1 + if cap <= idx { + src := reflect.SliceHeader{Data: data, Len: idx, Cap: cap} + cap *= 2 + data = uintptr(newArray(d.elemType, cap)) + dst := reflect.SliceHeader{Data: data, Len: idx, Cap: cap} + copySlice(d.elemType, dst, src) + } + if err := d.valueDecoder.decode(ctx, data+uintptr(idx)*d.size); err != nil { + return err + } + cursor = ctx.skipWhiteSpace() + switch buf[cursor] { + case ']': + *(*reflect.SliceHeader)(unsafe.Pointer(p)) = reflect.SliceHeader{ + Data: data, + Len: idx + 1, + Cap: cap, + } + return nil + case ',': + idx++ + continue + default: + return errors.New("syntax error slice") + } + } + } + } + return errors.New("unexpected error slice") +} diff --git a/decode_test.go b/decode_test.go index 6613989..9cac903 100644 --- a/decode_test.go +++ b/decode_test.go @@ -1,6 +1,7 @@ package json_test import ( + "fmt" "testing" "github.com/goccy/go-json" @@ -75,17 +76,25 @@ func Test_Decoder(t *testing.T) { assertEq(t, "string", "hello", v) }) t.Run("float32", func(t *testing.T) { - var v struct { - F float32 - } - assertErr(t, json.Unmarshal([]byte(`{"f": 3.14}`), &v)) - assertEq(t, "float32", float32(3.14), v.F) + var v float32 + assertErr(t, json.Unmarshal([]byte(`3.14`), &v)) + assertEq(t, "float32", float32(3.14), v) }) t.Run("float64", func(t *testing.T) { var v float64 assertErr(t, json.Unmarshal([]byte(`3.14`), &v)) assertEq(t, "float64", float64(3.14), v) }) + t.Run("slice", func(t *testing.T) { + var v []int + assertErr(t, json.Unmarshal([]byte(` [ 1 , 2 , 3 , 4 ] `), &v)) + assertEq(t, "slice", fmt.Sprint([]int{1, 2, 3, 4}), fmt.Sprint(v)) + }) + t.Run("array", func(t *testing.T) { + var v [4]int + assertErr(t, json.Unmarshal([]byte(` [ 1 , 2 , 3 , 4 ] `), &v)) + assertEq(t, "array", fmt.Sprint([4]int{1, 2, 3, 4}), fmt.Sprint(v)) + }) t.Run("struct", func(t *testing.T) { type T struct { AA int `json:"aa"`