From 20b67ad48d16faf3f6bcd1ab43d0e32f5019d694 Mon Sep 17 00:00:00 2001 From: Masaaki Goshima Date: Thu, 30 Jul 2020 22:41:53 +0900 Subject: [PATCH] Support Decoder.Token --- decode.go | 76 +++++++++++++++-------- decode_array.go | 30 +++++++++ decode_bool.go | 61 ++++++++++++++++++- decode_float.go | 58 ++++++++++++++++++ decode_int.go | 13 ++-- decode_interface.go | 101 +++++++++++++++++++++++++++++++ decode_map.go | 46 ++++++++++++++ decode_ptr.go | 9 +++ decode_slice.go | 54 +++++++++++++++++ decode_stream.go | 127 ++++++++++++++++++++++++++++++++++----- decode_string.go | 67 +++++++++++++++++++++ decode_struct.go | 45 ++++++++++++++ decode_test.go | 14 +++++ decode_uint.go | 33 ++++++++++ decode_unmarshal_json.go | 17 ++++++ decode_unmarshal_text.go | 17 ++++++ 16 files changed, 721 insertions(+), 47 deletions(-) diff --git a/decode.go b/decode.go index a9d261e..6975ad5 100644 --- a/decode.go +++ b/decode.go @@ -5,6 +5,7 @@ import ( "fmt" "io" "reflect" + "strconv" "sync" "unsafe" ) @@ -17,6 +18,7 @@ func (d Delim) String() string { type decoder interface { decode([]byte, int64, uintptr) (int64, error) + decodeStream(*stream, uintptr) error } type Decoder struct { @@ -53,7 +55,9 @@ func init() { // The decoder introduces its own buffering and may // read data from r beyond the JSON values requested. func NewDecoder(r io.Reader) *Decoder { - return &Decoder{s: &stream{r: r}} + s := &stream{r: r} + s.read() + return &Decoder{s: s} } // Buffered returns a reader of the data remaining in the Decoder's @@ -109,12 +113,13 @@ func (d *Decoder) decodeForUnmarshalNoEscape(src []byte, v interface{}) error { func (d *Decoder) prepareForDecode() error { s := d.s - for ; s.cursor < s.length || s.read(); s.cursor++ { + for { switch s.char() { case ' ', '\t', '\r', '\n': + s.progress() continue case ',', ':': - s.cursor++ + s.progress() return nil } break @@ -152,10 +157,7 @@ func (d *Decoder) Decode(v interface{}) error { return err } s := d.s - cursor, err := dec.decode(s.buf[s.cursor:], 0, ptr) - s.cursor += cursor - fmt.Println("cursor = ", cursor, "next buf = ", string(s.buf[s.cursor:])) - if err != nil { + if err := dec.decodeStream(s, ptr); err != nil { return err } return nil @@ -163,10 +165,12 @@ func (d *Decoder) Decode(v interface{}) error { func (d *Decoder) More() bool { s := d.s - for ; s.cursor < s.length || s.read(); s.cursor++ { + for { switch s.char() { case ' ', '\n', '\r', '\t': - continue + if s.progress() { + continue + } case '}', ']': return false } @@ -177,27 +181,51 @@ func (d *Decoder) More() bool { func (d *Decoder) Token() (Token, error) { s := d.s - for ; s.cursor < s.length || s.read(); s.cursor++ { - switch s.char() { + for { + c := s.char() + switch c { case ' ', '\n', '\r', '\t': - continue - case '{': - s.cursor++ - return Delim('{'), nil - case '[': - s.cursor++ - return Delim('['), nil - case '}': - s.cursor++ - return Delim('}'), nil - case ']': - s.cursor++ - return Delim(']'), nil + if s.progress() { + continue + } + case '{', '[', ']', '}': + s.progress() + return Delim(c), nil + case ',', ':': + if s.progress() { + continue + } case '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': + bytes := floatBytes(s) + s := *(*string)(unsafe.Pointer(&bytes)) + f64, err := strconv.ParseFloat(s, 64) + if err != nil { + return nil, err + } + return f64, nil case '"': + bytes, err := stringBytes(s) + if err != nil { + return nil, err + } + return string(bytes), nil case 't': + if err := trueBytes(s); err != nil { + return nil, err + } + return true, nil case 'f': + if err := falseBytes(s); err != nil { + return nil, err + } + return false, nil case 'n': + if err := nullBytes(s); err != nil { + return nil, err + } + return nil, nil + case '\000': + return nil, io.EOF default: return nil, errInvalidCharacter(s.char(), "token", s.totalOffset()) } diff --git a/decode_array.go b/decode_array.go index f4e3555..a1b89b8 100644 --- a/decode_array.go +++ b/decode_array.go @@ -16,6 +16,36 @@ func newArrayDecoder(dec decoder, elemType *rtype, alen int) *arrayDecoder { } } +func (d *arrayDecoder) decodeStream(s *stream, p uintptr) error { + for { + switch s.char() { + case ' ', '\n', '\t', '\r': + case '[': + idx := 0 + for { + s.progress() + if err := d.valueDecoder.decodeStream(s, p+uintptr(idx)*d.size); err != nil { + return err + } + s.skipWhiteSpace() + switch s.char() { + case ']': + s.progress() + return nil + case ',': + idx++ + default: + return errInvalidCharacter(s.char(), "array", s.offset) + } + } + default: + return errUnexpectedEndOfJSON("array", s.offset) + } + s.progress() + } + return errUnexpectedEndOfJSON("array", s.offset) +} + func (d *arrayDecoder) decode(buf []byte, cursor int64, p uintptr) (int64, error) { buflen := int64(len(buf)) for ; cursor < buflen; cursor++ { diff --git a/decode_bool.go b/decode_bool.go index 583bad5..a5dd822 100644 --- a/decode_bool.go +++ b/decode_bool.go @@ -10,6 +10,63 @@ func newBoolDecoder() *boolDecoder { return &boolDecoder{} } +func trueBytes(s *stream) error { + s.progress() + if s.char() != 'r' { + return errInvalidCharacter(s.char(), "bool(true)", s.totalOffset()) + } + s.progress() + if s.char() != 'u' { + return errInvalidCharacter(s.char(), "bool(true)", s.totalOffset()) + } + s.progress() + if s.char() != 'e' { + return errInvalidCharacter(s.char(), "bool(true)", s.totalOffset()) + } + s.progress() + return nil +} + +func falseBytes(s *stream) error { + s.progress() + if s.char() != 'a' { + return errInvalidCharacter(s.char(), "bool(false)", s.totalOffset()) + } + s.progress() + if s.char() != 'l' { + return errInvalidCharacter(s.char(), "bool(false)", s.totalOffset()) + } + s.progress() + if s.char() != 's' { + return errInvalidCharacter(s.char(), "bool(false)", s.totalOffset()) + } + s.progress() + if s.char() != 'e' { + return errInvalidCharacter(s.char(), "bool(false)", s.totalOffset()) + } + s.progress() + return nil +} + +func (d *boolDecoder) decodeStream(s *stream, p uintptr) error { + s.skipWhiteSpace() + switch s.char() { + case 't': + if err := trueBytes(s); err != nil { + return err + } + *(*bool)(unsafe.Pointer(p)) = true + return nil + case 'f': + if err := falseBytes(s); err != nil { + return err + } + *(*bool)(unsafe.Pointer(p)) = false + return nil + } + return errUnexpectedEndOfJSON("bool", s.totalOffset()) +} + func (d *boolDecoder) decode(buf []byte, cursor int64, p uintptr) (int64, error) { buflen := int64(len(buf)) cursor = skipWhiteSpace(buf, cursor) @@ -29,6 +86,7 @@ func (d *boolDecoder) decode(buf []byte, cursor int64, p uintptr) (int64, error) } cursor += 4 *(*bool)(unsafe.Pointer(p)) = true + return cursor, nil case 'f': if cursor+4 >= buflen { return 0, errUnexpectedEndOfJSON("bool(false)", cursor) @@ -47,6 +105,7 @@ func (d *boolDecoder) decode(buf []byte, cursor int64, p uintptr) (int64, error) } cursor += 5 *(*bool)(unsafe.Pointer(p)) = false + return cursor, nil } - return cursor, nil + return 0, errUnexpectedEndOfJSON("bool", cursor) } diff --git a/decode_float.go b/decode_float.go index 98b3088..6c7d12b 100644 --- a/decode_float.go +++ b/decode_float.go @@ -13,6 +13,48 @@ func newFloatDecoder(op func(uintptr, float64)) *floatDecoder { return &floatDecoder{op: op} } +var floatTable = [256]bool{ + '0': true, + '1': true, + '2': true, + '3': true, + '4': true, + '5': true, + '6': true, + '7': true, + '8': true, + '9': true, + '.': true, + 'e': true, + 'E': true, +} + +func floatBytes(s *stream) []byte { + start := s.cursor + for s.progress() { + if floatTable[s.char()] { + continue + } + break + } + return s.buf[start:s.cursor] +} + +func (d *floatDecoder) decodeStreamByte(s *stream) ([]byte, error) { + for { + switch s.char() { + case ' ', '\n', '\t', '\r': + s.progress() + continue + case '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': + return floatBytes(s), nil + default: + return nil, errUnexpectedEndOfJSON("float", s.offset) + } + } + return nil, errUnexpectedEndOfJSON("float", s.offset) +} + func (d *floatDecoder) decodeByte(buf []byte, cursor int64) ([]byte, int64, error) { buflen := int64(len(buf)) for ; cursor < buflen; cursor++ { @@ -31,11 +73,27 @@ func (d *floatDecoder) decodeByte(buf []byte, cursor int64) ([]byte, int64, erro } num := buf[start:cursor] return num, cursor, nil + default: + return nil, 0, errUnexpectedEndOfJSON("float", cursor) } } return nil, 0, errUnexpectedEndOfJSON("float", cursor) } +func (d *floatDecoder) decodeStream(s *stream, p uintptr) error { + bytes, err := d.decodeStreamByte(s) + if err != nil { + return err + } + str := *(*string)(unsafe.Pointer(&bytes)) + f64, err := strconv.ParseFloat(str, 64) + if err != nil { + return err + } + d.op(p, f64) + return nil +} + func (d *floatDecoder) decode(buf []byte, cursor int64, p uintptr) (int64, error) { bytes, c, err := d.decodeByte(buf, cursor) if err != nil { diff --git a/decode_int.go b/decode_int.go index d512cb8..5e9a3d6 100644 --- a/decode_int.go +++ b/decode_int.go @@ -49,15 +49,15 @@ var ( } ) -func (d *intDecoder) decodeByteStream(s *stream) ([]byte, error) { - for ; s.cursor < s.length || s.read(); s.cursor++ { +func (d *intDecoder) decodeStreamByte(s *stream) ([]byte, error) { + for { switch s.char() { case ' ', '\n', '\t', '\r': + s.progress() continue case '-': start := s.cursor - s.cursor++ - for ; s.cursor < s.length || s.read(); s.cursor++ { + for s.progress() { if numTable[s.char()] { continue } @@ -70,8 +70,7 @@ func (d *intDecoder) decodeByteStream(s *stream) ([]byte, error) { return num, nil case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': start := s.cursor - s.cursor++ - for ; s.cursor < s.length || s.read(); s.cursor++ { + for s.progress() { if numTable[s.char()] { continue } @@ -110,7 +109,7 @@ func (d *intDecoder) decodeByte(buf []byte, cursor int64) ([]byte, int64, error) } func (d *intDecoder) decodeStream(s *stream, p uintptr) error { - bytes, err := d.decodeByteStream(s) + bytes, err := d.decodeStreamByte(s) if err != nil { return err } diff --git a/decode_interface.go b/decode_interface.go index a0d94b3..eda9606 100644 --- a/decode_interface.go +++ b/decode_interface.go @@ -20,6 +20,107 @@ var ( ) ) +func (d *interfaceDecoder) decodeStream(s *stream, p uintptr) error { + s.skipWhiteSpace() + switch s.char() { + case '{': + var v map[interface{}]interface{} + ptr := unsafe.Pointer(&v) + d.dummy = ptr + dec := newMapDecoder(interfaceMapType, newInterfaceDecoder(d.typ), newInterfaceDecoder(d.typ)) + if err := dec.decodeStream(s, uintptr(ptr)); err != nil { + return err + } + *(*interface{})(unsafe.Pointer(p)) = v + return nil + case '[': + var v []interface{} + ptr := unsafe.Pointer(&v) + d.dummy = ptr // escape ptr + dec := newSliceDecoder(newInterfaceDecoder(d.typ), d.typ, d.typ.Size()) + if err := dec.decodeStream(s, uintptr(ptr)); err != nil { + return err + } + *(*interface{})(unsafe.Pointer(p)) = v + return nil + case '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': + return newFloatDecoder(func(p uintptr, v float64) { + *(*interface{})(unsafe.Pointer(p)) = v + }).decodeStream(s, p) + case '"': + s.progress() + start := s.cursor + for { + switch s.char() { + case '\\': + s.progress() + case '"': + literal := s.buf[start:s.cursor] + s.progress() + *(*interface{})(unsafe.Pointer(p)) = *(*string)(unsafe.Pointer(&literal)) + return nil + case '\000': + return errUnexpectedEndOfJSON("string", s.totalOffset()) + } + s.progress() + } + return errUnexpectedEndOfJSON("string", s.totalOffset()) + case 't': + s.progress() + if s.char() != 'r' { + return errInvalidCharacter(s.char(), "bool(true)", s.totalOffset()) + } + s.progress() + if s.char() != 'u' { + return errInvalidCharacter(s.char(), "bool(true)", s.totalOffset()) + } + s.progress() + if s.char() != 'e' { + return errInvalidCharacter(s.char(), "bool(true)", s.totalOffset()) + } + s.progress() + *(*interface{})(unsafe.Pointer(p)) = true + return nil + case 'f': + s.progress() + if s.char() != 'a' { + return errInvalidCharacter(s.char(), "bool(false)", s.totalOffset()) + } + s.progress() + if s.char() != 'l' { + return errInvalidCharacter(s.char(), "bool(false)", s.totalOffset()) + } + s.progress() + if s.char() != 's' { + return errInvalidCharacter(s.char(), "bool(false)", s.totalOffset()) + } + s.progress() + if s.char() != 'e' { + return errInvalidCharacter(s.char(), "bool(false)", s.totalOffset()) + } + s.progress() + *(*interface{})(unsafe.Pointer(p)) = false + return nil + case 'n': + s.progress() + if s.char() != 'u' { + return errInvalidCharacter(s.char(), "null", s.totalOffset()) + } + s.progress() + if s.char() != 'l' { + return errInvalidCharacter(s.char(), "null", s.totalOffset()) + } + s.progress() + if s.char() != 'l' { + return errInvalidCharacter(s.char(), "null", s.totalOffset()) + } + s.progress() + *(*interface{})(unsafe.Pointer(p)) = nil + return nil + } + return errNotAtBeginningOfValue(s.totalOffset()) +} + func (d *interfaceDecoder) decode(buf []byte, cursor int64, p uintptr) (int64, error) { cursor = skipWhiteSpace(buf, cursor) switch buf[cursor] { diff --git a/decode_map.go b/decode_map.go index 18b5119..239d9fa 100644 --- a/decode_map.go +++ b/decode_map.go @@ -35,6 +35,52 @@ func (d *mapDecoder) setValue(buf []byte, cursor int64, key interface{}) (int64, return d.valueDecoder.decode(buf, cursor, uintptr(header.ptr)) } +func (d *mapDecoder) setKeyStream(s *stream, key interface{}) error { + header := (*interfaceHeader)(unsafe.Pointer(&key)) + return d.keyDecoder.decodeStream(s, uintptr(header.ptr)) +} + +func (d *mapDecoder) setValueStream(s *stream, key interface{}) error { + header := (*interfaceHeader)(unsafe.Pointer(&key)) + return d.valueDecoder.decodeStream(s, uintptr(header.ptr)) +} + +func (d *mapDecoder) decodeStream(s *stream, p uintptr) error { + s.skipWhiteSpace() + if s.char() != '{' { + return errExpected("{ character for map value", s.totalOffset()) + } + mapValue := makemap(d.mapType, 0) + for s.progress() { + var key interface{} + if err := d.setKeyStream(s, &key); err != nil { + return err + } + s.skipWhiteSpace() + if s.char() != ':' { + return errExpected("colon after object key", s.totalOffset()) + } + s.progress() + if s.end() { + return errUnexpectedEndOfJSON("map", s.totalOffset()) + } + var value interface{} + if err := d.setValueStream(s, &value); err != nil { + return err + } + mapassign(d.mapType, mapValue, unsafe.Pointer(&key), unsafe.Pointer(&value)) + s.skipWhiteSpace() + if s.char() == '}' { + *(*unsafe.Pointer)(unsafe.Pointer(p)) = mapValue + return nil + } + if s.char() != ',' { + return errExpected("semicolon after object value", s.totalOffset()) + } + } + return nil +} + func (d *mapDecoder) decode(buf []byte, cursor int64, p uintptr) (int64, error) { cursor = skipWhiteSpace(buf, cursor) buflen := int64(len(buf)) diff --git a/decode_ptr.go b/decode_ptr.go index 6135872..4ce625d 100644 --- a/decode_ptr.go +++ b/decode_ptr.go @@ -16,6 +16,15 @@ func newPtrDecoder(dec decoder, typ *rtype) *ptrDecoder { //go:linkname unsafe_New reflect.unsafe_New func unsafe_New(*rtype) uintptr +func (d *ptrDecoder) decodeStream(s *stream, p uintptr) error { + newptr := unsafe_New(d.typ) + if err := d.dec.decodeStream(s, newptr); err != nil { + return err + } + *(*uintptr)(unsafe.Pointer(p)) = newptr + return nil +} + func (d *ptrDecoder) decode(buf []byte, cursor int64, p uintptr) (int64, error) { newptr := unsafe_New(d.typ) c, err := d.dec.decode(buf, cursor, newptr) diff --git a/decode_slice.go b/decode_slice.go index a8f6381..a76c3ae 100644 --- a/decode_slice.go +++ b/decode_slice.go @@ -47,6 +47,60 @@ func copySlice(elemType *rtype, dst, src reflect.SliceHeader) int //go:linkname newArray reflect.unsafe_NewArray func newArray(*rtype, int) unsafe.Pointer +func (d *sliceDecoder) decodeStream(s *stream, p uintptr) error { + for { + switch s.char() { + case ' ', '\n', '\t', '\r': + s.progress() + continue + case '[': + idx := 0 + slice := d.newSlice() + cap := slice.Cap + data := slice.Data + for s.progress() { + if cap <= idx { + src := reflect.SliceHeader{Data: data, Len: idx, Cap: cap} + cap *= 2 + data = uintptr(newArray(d.elemType, cap)) + dst := reflect.SliceHeader{Data: data, Len: idx, Cap: cap} + copySlice(d.elemType, dst, src) + } + if err := d.valueDecoder.decodeStream(s, data+uintptr(idx)*d.size); err != nil { + return err + } + s.skipWhiteSpace() + switch s.char() { + case ']': + slice.Cap = cap + slice.Len = idx + 1 + slice.Data = data + dstCap := idx + 1 + dst := reflect.SliceHeader{ + Data: uintptr(newArray(d.elemType, dstCap)), + Len: idx + 1, + Cap: dstCap, + } + copySlice(d.elemType, dst, *slice) + *(*reflect.SliceHeader)(unsafe.Pointer(p)) = dst + d.releaseSlice(slice) + s.progress() + return nil + case ',': + idx++ + continue + default: + slice.Cap = cap + slice.Data = data + d.releaseSlice(slice) + return errInvalidCharacter(s.char(), "slice", s.totalOffset()) + } + } + } + } + return errUnexpectedEndOfJSON("slice", s.totalOffset()) +} + func (d *sliceDecoder) decode(buf []byte, cursor int64, p uintptr) (int64, error) { buflen := int64(len(buf)) for ; cursor < buflen; cursor++ { diff --git a/decode_stream.go b/decode_stream.go index 37e48e5..817e612 100644 --- a/decode_stream.go +++ b/decode_stream.go @@ -6,16 +6,16 @@ import ( ) const ( - readChunkSize = 1024 + readChunkSize = 2 ) type stream struct { - buf []byte - length int64 - r io.Reader - decodedPos int64 - offset int64 - cursor int64 + buf []byte + length int64 + r io.Reader + offset int64 + cursor int64 + allRead bool } func (s *stream) buffered() io.Reader { @@ -26,24 +26,121 @@ func (s *stream) totalOffset() int64 { return s.offset + s.cursor } +func (s *stream) prevChar() byte { + return s.buf[s.cursor-1] +} + func (s *stream) char() byte { return s.buf[s.cursor] } +func (s *stream) end() bool { + return s.allRead && s.length <= s.cursor +} + +func (s *stream) progress() bool { + if s.cursor < s.length-1 || s.read() { + s.cursor++ + return true + } + s.cursor = s.length + return false +} + +func (s *stream) progressN(n int64) bool { + if s.cursor+n < s.length-1 || s.read() { + s.cursor += n + return true + } + s.cursor = s.length + return false +} + +func (s *stream) reset() { + s.buf = s.buf[s.cursor:] + s.length -= s.cursor + s.cursor = 0 +} + func (s *stream) read() bool { buf := make([]byte, readChunkSize) n, err := s.r.Read(buf) - if n == 0 || err == io.EOF { + if err != nil && err != io.EOF { return false } - remain := s.length - s.decodedPos - newBuf := make([]byte, remain+int64(n)) - copy(newBuf, s.buf[s.decodedPos:]) + remain := s.length + newBuf := make([]byte, remain+int64(n)+1) + copy(newBuf, s.buf) copy(newBuf[remain:], buf) s.buf = newBuf - s.length = int64(len(newBuf)) - s.offset += s.decodedPos - s.cursor = 0 - s.decodedPos = 0 + s.length = int64(len(newBuf)) - 1 + s.offset += s.cursor + if n == 0 || err == io.EOF { + s.allRead = true + return false + } return true } + +func (s *stream) skipWhiteSpace() { +LOOP: + if isWhiteSpace[s.char()] { + s.progress() + goto LOOP + } +} + +func (s *stream) skipValue() error { + s.skipWhiteSpace() + braceCount := 0 + bracketCount := 0 + for { + switch s.char() { + case '\000': + return errUnexpectedEndOfJSON("value of object", s.offset) + case '{': + braceCount++ + case '[': + bracketCount++ + case '}': + braceCount-- + if braceCount == -1 && bracketCount == 0 { + return nil + } + case ']': + bracketCount-- + case ',': + if bracketCount == 0 && braceCount == 0 { + return nil + } + case '"': + for s.progress() { + if s.char() != '"' { + continue + } + if s.prevChar() == '\\' { + continue + } + if bracketCount == 0 && braceCount == 0 { + s.progress() + return nil + } + break + } + case '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': + for s.progress() { + tk := int(s.char()) + if (int('0') <= tk && tk <= int('9')) || tk == '.' || tk == 'e' || tk == 'E' { + continue + } + break + } + if bracketCount == 0 && braceCount == 0 { + return nil + } + continue + } + s.progress() + } + return errUnexpectedEndOfJSON("value of object", s.offset) +} diff --git a/decode_string.go b/decode_string.go index 7b0aec0..0d92acd 100644 --- a/decode_string.go +++ b/decode_string.go @@ -11,6 +11,15 @@ func newStringDecoder() *stringDecoder { return &stringDecoder{} } +func (d *stringDecoder) decodeStream(s *stream, p uintptr) error { + bytes, err := d.decodeStreamByte(s) + if err != nil { + return err + } + *(*string)(unsafe.Pointer(p)) = *(*string)(unsafe.Pointer(&bytes)) + return nil +} + func (d *stringDecoder) decode(buf []byte, cursor int64, p uintptr) (int64, error) { bytes, c, err := d.decodeByte(buf, cursor) if err != nil { @@ -21,6 +30,64 @@ func (d *stringDecoder) decode(buf []byte, cursor int64, p uintptr) (int64, erro return cursor, nil } +func stringBytes(s *stream) ([]byte, error) { + s.progress() + start := s.cursor + for { + switch s.char() { + case '\\': + s.progress() + case '"': + literal := s.buf[start:s.cursor] + s.progress() + s.reset() + return literal, nil + case '\000': + goto ERROR + } + s.progress() + } +ERROR: + return nil, errUnexpectedEndOfJSON("string", s.totalOffset()) +} + +func nullBytes(s *stream) error { + s.progress() + if s.char() != 'u' { + return errInvalidCharacter(s.char(), "null", s.totalOffset()) + } + s.progress() + if s.char() != 'l' { + return errInvalidCharacter(s.char(), "null", s.totalOffset()) + } + s.progress() + if s.char() != 'l' { + return errInvalidCharacter(s.char(), "null", s.totalOffset()) + } + s.progress() + return nil +} + +func (d *stringDecoder) decodeStreamByte(s *stream) ([]byte, error) { + for { + switch s.char() { + case ' ', '\n', '\t', '\r': + s.progress() + case '"': + return stringBytes(s) + case 'n': + if err := nullBytes(s); err != nil { + return nil, err + } + return []byte{'n', 'u', 'l', 'l'}, nil + default: + goto ERROR + } + } +ERROR: + return nil, errNotAtBeginningOfValue(s.totalOffset()) +} + func (d *stringDecoder) decodeByte(buf []byte, cursor int64) ([]byte, int64, error) { for { switch buf[cursor] { diff --git a/decode_struct.go b/decode_struct.go index c065211..c0e7802 100644 --- a/decode_struct.go +++ b/decode_struct.go @@ -21,6 +21,51 @@ func newStructDecoder(fieldMap map[string]*structFieldSet) *structDecoder { } } +func (d *structDecoder) decodeStream(s *stream, p uintptr) error { + s.skipWhiteSpace() + if s.char() != '{' { + return errNotAtBeginningOfValue(s.totalOffset()) + } + s.progress() + for { + s.reset() + key, err := d.keyDecoder.decodeStreamByte(s) + if err != nil { + return err + } + s.skipWhiteSpace() + if s.char() != ':' { + return errExpected("colon after object key", s.totalOffset()) + } + s.progress() + if s.end() { + return errExpected("object value after colon", s.totalOffset()) + } + k := *(*string)(unsafe.Pointer(&key)) + field, exists := d.fieldMap[k] + if exists { + if err := field.dec.decodeStream(s, p+field.offset); err != nil { + return err + } + } else { + if err := s.skipValue(); err != nil { + return err + } + } + s.skipWhiteSpace() + c := s.char() + if c == '}' { + s.progress() + return nil + } + if c != ',' { + return errExpected("comma after object element", s.totalOffset()) + } + s.progress() + } + return nil +} + func (d *structDecoder) decode(buf []byte, cursor int64, p uintptr) (int64, error) { buflen := int64(len(buf)) cursor = skipWhiteSpace(buf, cursor) diff --git a/decode_test.go b/decode_test.go index 58b4cc9..f4f7bfb 100644 --- a/decode_test.go +++ b/decode_test.go @@ -233,6 +233,20 @@ func Test_InvalidUnmarshalError(t *testing.T) { }) } +func Test_Token(t *testing.T) { + dec := json.NewDecoder(strings.NewReader(`{"a": 1, "b": true, "c": [1, "two", null]}`)) + cnt := 0 + for { + if _, err := dec.Token(); err != nil { + break + } + cnt++ + } + if cnt != 12 { + t.Fatal("failed to parse token") + } +} + func Test_DecodeStream(t *testing.T) { const stream = ` [ diff --git a/decode_uint.go b/decode_uint.go index 1ba442d..49418f7 100644 --- a/decode_uint.go +++ b/decode_uint.go @@ -24,6 +24,30 @@ func (d *uintDecoder) parseUint(b []byte) uint64 { return sum } +func (d *uintDecoder) decodeStreamByte(s *stream) ([]byte, error) { + for { + switch s.char() { + case ' ', '\n', '\t', '\r': + s.progress() + continue + case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': + start := s.cursor + for s.progress() { + tk := int(s.char()) + if int('0') <= tk && tk <= int('9') { + continue + } + break + } + num := s.buf[start:s.cursor] + return num, nil + default: + return nil, errInvalidCharacter(s.char(), "number(unsigned integer)", s.totalOffset()) + } + } + return nil, errUnexpectedEndOfJSON("number(unsigned integer)", s.totalOffset()) +} + func (d *uintDecoder) decodeByte(buf []byte, cursor int64) ([]byte, int64, error) { buflen := int64(len(buf)) for ; cursor < buflen; cursor++ { @@ -49,6 +73,15 @@ func (d *uintDecoder) decodeByte(buf []byte, cursor int64) ([]byte, int64, error return nil, 0, errUnexpectedEndOfJSON("number(unsigned integer)", cursor) } +func (d *uintDecoder) decodeStream(s *stream, p uintptr) error { + bytes, err := d.decodeStreamByte(s) + if err != nil { + return err + } + d.op(p, d.parseUint(bytes)) + return nil +} + func (d *uintDecoder) decode(buf []byte, cursor int64, p uintptr) (int64, error) { bytes, c, err := d.decodeByte(buf, cursor) if err != nil { diff --git a/decode_unmarshal_json.go b/decode_unmarshal_json.go index dbd796f..f84f850 100644 --- a/decode_unmarshal_json.go +++ b/decode_unmarshal_json.go @@ -12,6 +12,23 @@ func newUnmarshalJSONDecoder(typ *rtype) *unmarshalJSONDecoder { return &unmarshalJSONDecoder{typ: typ} } +func (d *unmarshalJSONDecoder) decodeStream(s *stream, p uintptr) error { + s.skipWhiteSpace() + start := s.cursor + if err := s.skipValue(); err != nil { + return err + } + src := s.buf[start:s.cursor] + v := *(*interface{})(unsafe.Pointer(&interfaceHeader{ + typ: d.typ, + ptr: unsafe.Pointer(p), + })) + if err := v.(Unmarshaler).UnmarshalJSON(src); err != nil { + return err + } + return nil +} + func (d *unmarshalJSONDecoder) decode(buf []byte, cursor int64, p uintptr) (int64, error) { cursor = skipWhiteSpace(buf, cursor) start := cursor diff --git a/decode_unmarshal_text.go b/decode_unmarshal_text.go index dfcec9a..73ea335 100644 --- a/decode_unmarshal_text.go +++ b/decode_unmarshal_text.go @@ -13,6 +13,23 @@ func newUnmarshalTextDecoder(typ *rtype) *unmarshalTextDecoder { return &unmarshalTextDecoder{typ: typ} } +func (d *unmarshalTextDecoder) decodeStream(s *stream, p uintptr) error { + s.skipWhiteSpace() + start := s.cursor + if err := s.skipValue(); err != nil { + return err + } + src := s.buf[start:s.cursor] + v := *(*interface{})(unsafe.Pointer(&interfaceHeader{ + typ: d.typ, + ptr: unsafe.Pointer(p), + })) + if err := v.(encoding.TextUnmarshaler).UnmarshalText(src); err != nil { + return err + } + return nil +} + func (d *unmarshalTextDecoder) decode(buf []byte, cursor int64, p uintptr) (int64, error) { cursor = skipWhiteSpace(buf, cursor) start := cursor