Support Decoder.Token

decode.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"io"
 	"reflect"
+	"strconv"
 	"sync"
 	"unsafe"
 )
@@ -17,6 +18,7 @@ func (d Delim) String() string {
 
 type decoder interface {
 	decode([]byte, int64, uintptr) (int64, error)
+	decodeStream(*stream, uintptr) error
 }
 
 type Decoder struct {
@@ -53,7 +55,9 @@ func init() {
 // The decoder introduces its own buffering and may
 // read data from r beyond the JSON values requested.
 func NewDecoder(r io.Reader) *Decoder {
-	return &Decoder{s: &stream{r: r}}
+	s := &stream{r: r}
+	s.read()
+	return &Decoder{s: s}
 }
 
 // Buffered returns a reader of the data remaining in the Decoder's
@@ -109,12 +113,13 @@ func (d *Decoder) decodeForUnmarshalNoEscape(src []byte, v interface{}) error {
 
 func (d *Decoder) prepareForDecode() error {
 	s := d.s
-	for ; s.cursor < s.length || s.read(); s.cursor++ {
+	for {
 		switch s.char() {
 		case ' ', '\t', '\r', '\n':
+			s.progress()
 			continue
 		case ',', ':':
-			s.cursor++
+			s.progress()
 			return nil
 		}
 		break
@@ -152,10 +157,7 @@ func (d *Decoder) Decode(v interface{}) error {
 		return err
 	}
 	s := d.s
-	cursor, err := dec.decode(s.buf[s.cursor:], 0, ptr)
+	if err := dec.decodeStream(s, ptr); err != nil {
-	s.cursor += cursor
-	fmt.Println("cursor = ", cursor, "next buf = ", string(s.buf[s.cursor:]))
-	if err != nil {
 		return err
 	}
 	return nil
@@ -163,10 +165,12 @@ func (d *Decoder) Decode(v interface{}) error {
 
 func (d *Decoder) More() bool {
 	s := d.s
-	for ; s.cursor < s.length || s.read(); s.cursor++ {
+	for {
 		switch s.char() {
 		case ' ', '\n', '\r', '\t':
+			if s.progress() {
 				continue
+			}
 		case '}', ']':
 			return false
 		}
@@ -177,27 +181,51 @@ func (d *Decoder) More() bool {
 
 func (d *Decoder) Token() (Token, error) {
 	s := d.s
-	for ; s.cursor < s.length || s.read(); s.cursor++ {
+	for {
-		switch s.char() {
+		c := s.char()
+		switch c {
 		case ' ', '\n', '\r', '\t':
+			if s.progress() {
 				continue
-		case '{':
+			}
-			s.cursor++
+		case '{', '[', ']', '}':
-			return Delim('{'), nil
+			s.progress()
-		case '[':
+			return Delim(c), nil
-			s.cursor++
+		case ',', ':':
-			return Delim('['), nil
+			if s.progress() {
-		case '}':
+				continue
-			s.cursor++
+			}
-			return Delim('}'), nil
-		case ']':
-			s.cursor++
-			return Delim(']'), nil
 		case '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
+			bytes := floatBytes(s)
+			s := *(*string)(unsafe.Pointer(&bytes))
+			f64, err := strconv.ParseFloat(s, 64)
+			if err != nil {
+				return nil, err
+			}
+			return f64, nil
 		case '"':
+			bytes, err := stringBytes(s)
+			if err != nil {
+				return nil, err
+			}
+			return string(bytes), nil
 		case 't':
+			if err := trueBytes(s); err != nil {
+				return nil, err
+			}
+			return true, nil
 		case 'f':
+			if err := falseBytes(s); err != nil {
+				return nil, err
+			}
+			return false, nil
 		case 'n':
+			if err := nullBytes(s); err != nil {
+				return nil, err
+			}
+			return nil, nil
+		case '\000':
+			return nil, io.EOF
 		default:
 			return nil, errInvalidCharacter(s.char(), "token", s.totalOffset())
 		}

decode_array.go

@@ -16,6 +16,36 @@ func newArrayDecoder(dec decoder, elemType *rtype, alen int) *arrayDecoder {
 	}
 }
 
+func (d *arrayDecoder) decodeStream(s *stream, p uintptr) error {
+	for {
+		switch s.char() {
+		case ' ', '\n', '\t', '\r':
+		case '[':
+			idx := 0
+			for {
+				s.progress()
+				if err := d.valueDecoder.decodeStream(s, p+uintptr(idx)*d.size); err != nil {
+					return err
+				}
+				s.skipWhiteSpace()
+				switch s.char() {
+				case ']':
+					s.progress()
+					return nil
+				case ',':
+					idx++
+				default:
+					return errInvalidCharacter(s.char(), "array", s.offset)
+				}
+			}
+		default:
+			return errUnexpectedEndOfJSON("array", s.offset)
+		}
+		s.progress()
+	}
+	return errUnexpectedEndOfJSON("array", s.offset)
+}
+
 func (d *arrayDecoder) decode(buf []byte, cursor int64, p uintptr) (int64, error) {
 	buflen := int64(len(buf))
 	for ; cursor < buflen; cursor++ {

decode_bool.go

@@ -10,6 +10,63 @@ func newBoolDecoder() *boolDecoder {
 	return &boolDecoder{}
 }
 
+func trueBytes(s *stream) error {
+	s.progress()
+	if s.char() != 'r' {
+		return errInvalidCharacter(s.char(), "bool(true)", s.totalOffset())
+	}
+	s.progress()
+	if s.char() != 'u' {
+		return errInvalidCharacter(s.char(), "bool(true)", s.totalOffset())
+	}
+	s.progress()
+	if s.char() != 'e' {
+		return errInvalidCharacter(s.char(), "bool(true)", s.totalOffset())
+	}
+	s.progress()
+	return nil
+}
+
+func falseBytes(s *stream) error {
+	s.progress()
+	if s.char() != 'a' {
+		return errInvalidCharacter(s.char(), "bool(false)", s.totalOffset())
+	}
+	s.progress()
+	if s.char() != 'l' {
+		return errInvalidCharacter(s.char(), "bool(false)", s.totalOffset())
+	}
+	s.progress()
+	if s.char() != 's' {
+		return errInvalidCharacter(s.char(), "bool(false)", s.totalOffset())
+	}
+	s.progress()
+	if s.char() != 'e' {
+		return errInvalidCharacter(s.char(), "bool(false)", s.totalOffset())
+	}
+	s.progress()
+	return nil
+}
+
+func (d *boolDecoder) decodeStream(s *stream, p uintptr) error {
+	s.skipWhiteSpace()
+	switch s.char() {
+	case 't':
+		if err := trueBytes(s); err != nil {
+			return err
+		}
+		*(*bool)(unsafe.Pointer(p)) = true
+		return nil
+	case 'f':
+		if err := falseBytes(s); err != nil {
+			return err
+		}
+		*(*bool)(unsafe.Pointer(p)) = false
+		return nil
+	}
+	return errUnexpectedEndOfJSON("bool", s.totalOffset())
+}
+
 func (d *boolDecoder) decode(buf []byte, cursor int64, p uintptr) (int64, error) {
 	buflen := int64(len(buf))
 	cursor = skipWhiteSpace(buf, cursor)
@@ -29,6 +86,7 @@ func (d *boolDecoder) decode(buf []byte, cursor int64, p uintptr) (int64, error)
 		}
 		cursor += 4
 		*(*bool)(unsafe.Pointer(p)) = true
+		return cursor, nil
 	case 'f':
 		if cursor+4 >= buflen {
 			return 0, errUnexpectedEndOfJSON("bool(false)", cursor)
@@ -47,6 +105,7 @@ func (d *boolDecoder) decode(buf []byte, cursor int64, p uintptr) (int64, error)
 		}
 		cursor += 5
 		*(*bool)(unsafe.Pointer(p)) = false
-	}
 		return cursor, nil
+	}
+	return 0, errUnexpectedEndOfJSON("bool", cursor)
 }

decode_float.go

@@ -13,6 +13,48 @@ func newFloatDecoder(op func(uintptr, float64)) *floatDecoder {
 	return &floatDecoder{op: op}
 }
 
+var floatTable = [256]bool{
+	'0': true,
+	'1': true,
+	'2': true,
+	'3': true,
+	'4': true,
+	'5': true,
+	'6': true,
+	'7': true,
+	'8': true,
+	'9': true,
+	'.': true,
+	'e': true,
+	'E': true,
+}
+
+func floatBytes(s *stream) []byte {
+	start := s.cursor
+	for s.progress() {
+		if floatTable[s.char()] {
+			continue
+		}
+		break
+	}
+	return s.buf[start:s.cursor]
+}
+
+func (d *floatDecoder) decodeStreamByte(s *stream) ([]byte, error) {
+	for {
+		switch s.char() {
+		case ' ', '\n', '\t', '\r':
+			s.progress()
+			continue
+		case '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
+			return floatBytes(s), nil
+		default:
+			return nil, errUnexpectedEndOfJSON("float", s.offset)
+		}
+	}
+	return nil, errUnexpectedEndOfJSON("float", s.offset)
+}
+
 func (d *floatDecoder) decodeByte(buf []byte, cursor int64) ([]byte, int64, error) {
 	buflen := int64(len(buf))
 	for ; cursor < buflen; cursor++ {
@@ -31,11 +73,27 @@ func (d *floatDecoder) decodeByte(buf []byte, cursor int64) ([]byte, int64, erro
 			}
 			num := buf[start:cursor]
 			return num, cursor, nil
+		default:
+			return nil, 0, errUnexpectedEndOfJSON("float", cursor)
 		}
 	}
 	return nil, 0, errUnexpectedEndOfJSON("float", cursor)
 }
 
+func (d *floatDecoder) decodeStream(s *stream, p uintptr) error {
+	bytes, err := d.decodeStreamByte(s)
+	if err != nil {
+		return err
+	}
+	str := *(*string)(unsafe.Pointer(&bytes))
+	f64, err := strconv.ParseFloat(str, 64)
+	if err != nil {
+		return err
+	}
+	d.op(p, f64)
+	return nil
+}
+
 func (d *floatDecoder) decode(buf []byte, cursor int64, p uintptr) (int64, error) {
 	bytes, c, err := d.decodeByte(buf, cursor)
 	if err != nil {

decode_int.go

@@ -49,15 +49,15 @@ var (
 	}
 )
 
-func (d *intDecoder) decodeByteStream(s *stream) ([]byte, error) {
+func (d *intDecoder) decodeStreamByte(s *stream) ([]byte, error) {
-	for ; s.cursor < s.length || s.read(); s.cursor++ {
+	for {
 		switch s.char() {
 		case ' ', '\n', '\t', '\r':
+			s.progress()
 			continue
 		case '-':
 			start := s.cursor
-			s.cursor++
+			for s.progress() {
-			for ; s.cursor < s.length || s.read(); s.cursor++ {
 				if numTable[s.char()] {
 					continue
 				}
@@ -70,8 +70,7 @@ func (d *intDecoder) decodeByteStream(s *stream) ([]byte, error) {
 			return num, nil
 		case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
 			start := s.cursor
-			s.cursor++
+			for s.progress() {
-			for ; s.cursor < s.length || s.read(); s.cursor++ {
 				if numTable[s.char()] {
 					continue
 				}
@@ -110,7 +109,7 @@ func (d *intDecoder) decodeByte(buf []byte, cursor int64) ([]byte, int64, error)
 }
 
 func (d *intDecoder) decodeStream(s *stream, p uintptr) error {
-	bytes, err := d.decodeByteStream(s)
+	bytes, err := d.decodeStreamByte(s)
 	if err != nil {
 		return err
 	}

decode_interface.go

@@ -20,6 +20,107 @@ var (
 	)
 )
 
+func (d *interfaceDecoder) decodeStream(s *stream, p uintptr) error {
+	s.skipWhiteSpace()
+	switch s.char() {
+	case '{':
+		var v map[interface{}]interface{}
+		ptr := unsafe.Pointer(&v)
+		d.dummy = ptr
+		dec := newMapDecoder(interfaceMapType, newInterfaceDecoder(d.typ), newInterfaceDecoder(d.typ))
+		if err := dec.decodeStream(s, uintptr(ptr)); err != nil {
+			return err
+		}
+		*(*interface{})(unsafe.Pointer(p)) = v
+		return nil
+	case '[':
+		var v []interface{}
+		ptr := unsafe.Pointer(&v)
+		d.dummy = ptr // escape ptr
+		dec := newSliceDecoder(newInterfaceDecoder(d.typ), d.typ, d.typ.Size())
+		if err := dec.decodeStream(s, uintptr(ptr)); err != nil {
+			return err
+		}
+		*(*interface{})(unsafe.Pointer(p)) = v
+		return nil
+	case '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
+		return newFloatDecoder(func(p uintptr, v float64) {
+			*(*interface{})(unsafe.Pointer(p)) = v
+		}).decodeStream(s, p)
+	case '"':
+		s.progress()
+		start := s.cursor
+		for {
+			switch s.char() {
+			case '\\':
+				s.progress()
+			case '"':
+				literal := s.buf[start:s.cursor]
+				s.progress()
+				*(*interface{})(unsafe.Pointer(p)) = *(*string)(unsafe.Pointer(&literal))
+				return nil
+			case '\000':
+				return errUnexpectedEndOfJSON("string", s.totalOffset())
+			}
+			s.progress()
+		}
+		return errUnexpectedEndOfJSON("string", s.totalOffset())
+	case 't':
+		s.progress()
+		if s.char() != 'r' {
+			return errInvalidCharacter(s.char(), "bool(true)", s.totalOffset())
+		}
+		s.progress()
+		if s.char() != 'u' {
+			return errInvalidCharacter(s.char(), "bool(true)", s.totalOffset())
+		}
+		s.progress()
+		if s.char() != 'e' {
+			return errInvalidCharacter(s.char(), "bool(true)", s.totalOffset())
+		}
+		s.progress()
+		*(*interface{})(unsafe.Pointer(p)) = true
+		return nil
+	case 'f':
+		s.progress()
+		if s.char() != 'a' {
+			return errInvalidCharacter(s.char(), "bool(false)", s.totalOffset())
+		}
+		s.progress()
+		if s.char() != 'l' {
+			return errInvalidCharacter(s.char(), "bool(false)", s.totalOffset())
+		}
+		s.progress()
+		if s.char() != 's' {
+			return errInvalidCharacter(s.char(), "bool(false)", s.totalOffset())
+		}
+		s.progress()
+		if s.char() != 'e' {
+			return errInvalidCharacter(s.char(), "bool(false)", s.totalOffset())
+		}
+		s.progress()
+		*(*interface{})(unsafe.Pointer(p)) = false
+		return nil
+	case 'n':
+		s.progress()
+		if s.char() != 'u' {
+			return errInvalidCharacter(s.char(), "null", s.totalOffset())
+		}
+		s.progress()
+		if s.char() != 'l' {
+			return errInvalidCharacter(s.char(), "null", s.totalOffset())
+		}
+		s.progress()
+		if s.char() != 'l' {
+			return errInvalidCharacter(s.char(), "null", s.totalOffset())
+		}
+		s.progress()
+		*(*interface{})(unsafe.Pointer(p)) = nil
+		return nil
+	}
+	return errNotAtBeginningOfValue(s.totalOffset())
+}
+
 func (d *interfaceDecoder) decode(buf []byte, cursor int64, p uintptr) (int64, error) {
 	cursor = skipWhiteSpace(buf, cursor)
 	switch buf[cursor] {

decode_map.go

@@ -35,6 +35,52 @@ func (d *mapDecoder) setValue(buf []byte, cursor int64, key interface{}) (int64,
 	return d.valueDecoder.decode(buf, cursor, uintptr(header.ptr))
 }
 
+func (d *mapDecoder) setKeyStream(s *stream, key interface{}) error {
+	header := (*interfaceHeader)(unsafe.Pointer(&key))
+	return d.keyDecoder.decodeStream(s, uintptr(header.ptr))
+}
+
+func (d *mapDecoder) setValueStream(s *stream, key interface{}) error {
+	header := (*interfaceHeader)(unsafe.Pointer(&key))
+	return d.valueDecoder.decodeStream(s, uintptr(header.ptr))
+}
+
+func (d *mapDecoder) decodeStream(s *stream, p uintptr) error {
+	s.skipWhiteSpace()
+	if s.char() != '{' {
+		return errExpected("{ character for map value", s.totalOffset())
+	}
+	mapValue := makemap(d.mapType, 0)
+	for s.progress() {
+		var key interface{}
+		if err := d.setKeyStream(s, &key); err != nil {
+			return err
+		}
+		s.skipWhiteSpace()
+		if s.char() != ':' {
+			return errExpected("colon after object key", s.totalOffset())
+		}
+		s.progress()
+		if s.end() {
+			return errUnexpectedEndOfJSON("map", s.totalOffset())
+		}
+		var value interface{}
+		if err := d.setValueStream(s, &value); err != nil {
+			return err
+		}
+		mapassign(d.mapType, mapValue, unsafe.Pointer(&key), unsafe.Pointer(&value))
+		s.skipWhiteSpace()
+		if s.char() == '}' {
+			*(*unsafe.Pointer)(unsafe.Pointer(p)) = mapValue
+			return nil
+		}
+		if s.char() != ',' {
+			return errExpected("semicolon after object value", s.totalOffset())
+		}
+	}
+	return nil
+}
+
 func (d *mapDecoder) decode(buf []byte, cursor int64, p uintptr) (int64, error) {
 	cursor = skipWhiteSpace(buf, cursor)
 	buflen := int64(len(buf))

decode_ptr.go

@@ -16,6 +16,15 @@ func newPtrDecoder(dec decoder, typ *rtype) *ptrDecoder {
 //go:linkname unsafe_New reflect.unsafe_New
 func unsafe_New(*rtype) uintptr
 
+func (d *ptrDecoder) decodeStream(s *stream, p uintptr) error {
+	newptr := unsafe_New(d.typ)
+	if err := d.dec.decodeStream(s, newptr); err != nil {
+		return err
+	}
+	*(*uintptr)(unsafe.Pointer(p)) = newptr
+	return nil
+}
+
 func (d *ptrDecoder) decode(buf []byte, cursor int64, p uintptr) (int64, error) {
 	newptr := unsafe_New(d.typ)
 	c, err := d.dec.decode(buf, cursor, newptr)

decode_slice.go

@@ -47,6 +47,60 @@ func copySlice(elemType *rtype, dst, src reflect.SliceHeader) int
 //go:linkname newArray reflect.unsafe_NewArray
 func newArray(*rtype, int) unsafe.Pointer
 
+func (d *sliceDecoder) decodeStream(s *stream, p uintptr) error {
+	for {
+		switch s.char() {
+		case ' ', '\n', '\t', '\r':
+			s.progress()
+			continue
+		case '[':
+			idx := 0
+			slice := d.newSlice()
+			cap := slice.Cap
+			data := slice.Data
+			for s.progress() {
+				if cap <= idx {
+					src := reflect.SliceHeader{Data: data, Len: idx, Cap: cap}
+					cap *= 2
+					data = uintptr(newArray(d.elemType, cap))
+					dst := reflect.SliceHeader{Data: data, Len: idx, Cap: cap}
+					copySlice(d.elemType, dst, src)
+				}
+				if err := d.valueDecoder.decodeStream(s, data+uintptr(idx)*d.size); err != nil {
+					return err
+				}
+				s.skipWhiteSpace()
+				switch s.char() {
+				case ']':
+					slice.Cap = cap
+					slice.Len = idx + 1
+					slice.Data = data
+					dstCap := idx + 1
+					dst := reflect.SliceHeader{
+						Data: uintptr(newArray(d.elemType, dstCap)),
+						Len:  idx + 1,
+						Cap:  dstCap,
+					}
+					copySlice(d.elemType, dst, *slice)
+					*(*reflect.SliceHeader)(unsafe.Pointer(p)) = dst
+					d.releaseSlice(slice)
+					s.progress()
+					return nil
+				case ',':
+					idx++
+					continue
+				default:
+					slice.Cap = cap
+					slice.Data = data
+					d.releaseSlice(slice)
+					return errInvalidCharacter(s.char(), "slice", s.totalOffset())
+				}
+			}
+		}
+	}
+	return errUnexpectedEndOfJSON("slice", s.totalOffset())
+}
+
 func (d *sliceDecoder) decode(buf []byte, cursor int64, p uintptr) (int64, error) {
 	buflen := int64(len(buf))
 	for ; cursor < buflen; cursor++ {

decode_stream.go

@@ -6,16 +6,16 @@ import (
 )
 
 const (
-	readChunkSize = 1024
+	readChunkSize = 2
 )
 
 type stream struct {
 	buf     []byte
 	length  int64
 	r       io.Reader
-	decodedPos int64
 	offset  int64
 	cursor  int64
+	allRead bool
 }
 
 func (s *stream) buffered() io.Reader {
@@ -26,24 +26,121 @@ func (s *stream) totalOffset() int64 {
 	return s.offset + s.cursor
 }
 
+func (s *stream) prevChar() byte {
+	return s.buf[s.cursor-1]
+}
+
 func (s *stream) char() byte {
 	return s.buf[s.cursor]
 }
 
+func (s *stream) end() bool {
+	return s.allRead && s.length <= s.cursor
+}
+
+func (s *stream) progress() bool {
+	if s.cursor < s.length-1 || s.read() {
+		s.cursor++
+		return true
+	}
+	s.cursor = s.length
+	return false
+}
+
+func (s *stream) progressN(n int64) bool {
+	if s.cursor+n < s.length-1 || s.read() {
+		s.cursor += n
+		return true
+	}
+	s.cursor = s.length
+	return false
+}
+
+func (s *stream) reset() {
+	s.buf = s.buf[s.cursor:]
+	s.length -= s.cursor
+	s.cursor = 0
+}
+
 func (s *stream) read() bool {
 	buf := make([]byte, readChunkSize)
 	n, err := s.r.Read(buf)
-	if n == 0 || err == io.EOF {
+	if err != nil && err != io.EOF {
 		return false
 	}
-	remain := s.length - s.decodedPos
+	remain := s.length
-	newBuf := make([]byte, remain+int64(n))
+	newBuf := make([]byte, remain+int64(n)+1)
-	copy(newBuf, s.buf[s.decodedPos:])
+	copy(newBuf, s.buf)
 	copy(newBuf[remain:], buf)
 	s.buf = newBuf
-	s.length = int64(len(newBuf))
+	s.length = int64(len(newBuf)) - 1
-	s.offset += s.decodedPos
+	s.offset += s.cursor
-	s.cursor = 0
+	if n == 0 || err == io.EOF {
-	s.decodedPos = 0
+		s.allRead = true
+		return false
+	}
 	return true
 }
+
+func (s *stream) skipWhiteSpace() {
+LOOP:
+	if isWhiteSpace[s.char()] {
+		s.progress()
+		goto LOOP
+	}
+}
+
+func (s *stream) skipValue() error {
+	s.skipWhiteSpace()
+	braceCount := 0
+	bracketCount := 0
+	for {
+		switch s.char() {
+		case '\000':
+			return errUnexpectedEndOfJSON("value of object", s.offset)
+		case '{':
+			braceCount++
+		case '[':
+			bracketCount++
+		case '}':
+			braceCount--
+			if braceCount == -1 && bracketCount == 0 {
+				return nil
+			}
+		case ']':
+			bracketCount--
+		case ',':
+			if bracketCount == 0 && braceCount == 0 {
+				return nil
+			}
+		case '"':
+			for s.progress() {
+				if s.char() != '"' {
+					continue
+				}
+				if s.prevChar() == '\\' {
+					continue
+				}
+				if bracketCount == 0 && braceCount == 0 {
+					s.progress()
+					return nil
+				}
+				break
+			}
+		case '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
+			for s.progress() {
+				tk := int(s.char())
+				if (int('0') <= tk && tk <= int('9')) || tk == '.' || tk == 'e' || tk == 'E' {
+					continue
+				}
+				break
+			}
+			if bracketCount == 0 && braceCount == 0 {
+				return nil
+			}
+			continue
+		}
+		s.progress()
+	}
+	return errUnexpectedEndOfJSON("value of object", s.offset)
+}

decode_string.go

@@ -11,6 +11,15 @@ func newStringDecoder() *stringDecoder {
 	return &stringDecoder{}
 }
 
+func (d *stringDecoder) decodeStream(s *stream, p uintptr) error {
+	bytes, err := d.decodeStreamByte(s)
+	if err != nil {
+		return err
+	}
+	*(*string)(unsafe.Pointer(p)) = *(*string)(unsafe.Pointer(&bytes))
+	return nil
+}
+
 func (d *stringDecoder) decode(buf []byte, cursor int64, p uintptr) (int64, error) {
 	bytes, c, err := d.decodeByte(buf, cursor)
 	if err != nil {
@@ -21,6 +30,64 @@ func (d *stringDecoder) decode(buf []byte, cursor int64, p uintptr) (int64, erro
 	return cursor, nil
 }
 
+func stringBytes(s *stream) ([]byte, error) {
+	s.progress()
+	start := s.cursor
+	for {
+		switch s.char() {
+		case '\\':
+			s.progress()
+		case '"':
+			literal := s.buf[start:s.cursor]
+			s.progress()
+			s.reset()
+			return literal, nil
+		case '\000':
+			goto ERROR
+		}
+		s.progress()
+	}
+ERROR:
+	return nil, errUnexpectedEndOfJSON("string", s.totalOffset())
+}
+
+func nullBytes(s *stream) error {
+	s.progress()
+	if s.char() != 'u' {
+		return errInvalidCharacter(s.char(), "null", s.totalOffset())
+	}
+	s.progress()
+	if s.char() != 'l' {
+		return errInvalidCharacter(s.char(), "null", s.totalOffset())
+	}
+	s.progress()
+	if s.char() != 'l' {
+		return errInvalidCharacter(s.char(), "null", s.totalOffset())
+	}
+	s.progress()
+	return nil
+}
+
+func (d *stringDecoder) decodeStreamByte(s *stream) ([]byte, error) {
+	for {
+		switch s.char() {
+		case ' ', '\n', '\t', '\r':
+			s.progress()
+		case '"':
+			return stringBytes(s)
+		case 'n':
+			if err := nullBytes(s); err != nil {
+				return nil, err
+			}
+			return []byte{'n', 'u', 'l', 'l'}, nil
+		default:
+			goto ERROR
+		}
+	}
+ERROR:
+	return nil, errNotAtBeginningOfValue(s.totalOffset())
+}
+
 func (d *stringDecoder) decodeByte(buf []byte, cursor int64) ([]byte, int64, error) {
 	for {
 		switch buf[cursor] {

decode_struct.go

@@ -21,6 +21,51 @@ func newStructDecoder(fieldMap map[string]*structFieldSet) *structDecoder {
 	}
 }
 
+func (d *structDecoder) decodeStream(s *stream, p uintptr) error {
+	s.skipWhiteSpace()
+	if s.char() != '{' {
+		return errNotAtBeginningOfValue(s.totalOffset())
+	}
+	s.progress()
+	for {
+		s.reset()
+		key, err := d.keyDecoder.decodeStreamByte(s)
+		if err != nil {
+			return err
+		}
+		s.skipWhiteSpace()
+		if s.char() != ':' {
+			return errExpected("colon after object key", s.totalOffset())
+		}
+		s.progress()
+		if s.end() {
+			return errExpected("object value after colon", s.totalOffset())
+		}
+		k := *(*string)(unsafe.Pointer(&key))
+		field, exists := d.fieldMap[k]
+		if exists {
+			if err := field.dec.decodeStream(s, p+field.offset); err != nil {
+				return err
+			}
+		} else {
+			if err := s.skipValue(); err != nil {
+				return err
+			}
+		}
+		s.skipWhiteSpace()
+		c := s.char()
+		if c == '}' {
+			s.progress()
+			return nil
+		}
+		if c != ',' {
+			return errExpected("comma after object element", s.totalOffset())
+		}
+		s.progress()
+	}
+	return nil
+}
+
 func (d *structDecoder) decode(buf []byte, cursor int64, p uintptr) (int64, error) {
 	buflen := int64(len(buf))
 	cursor = skipWhiteSpace(buf, cursor)

decode_test.go

@@ -233,6 +233,20 @@ func Test_InvalidUnmarshalError(t *testing.T) {
 	})
 }
 
+func Test_Token(t *testing.T) {
+	dec := json.NewDecoder(strings.NewReader(`{"a": 1, "b": true, "c": [1, "two", null]}`))
+	cnt := 0
+	for {
+		if _, err := dec.Token(); err != nil {
+			break
+		}
+		cnt++
+	}
+	if cnt != 12 {
+		t.Fatal("failed to parse token")
+	}
+}
+
 func Test_DecodeStream(t *testing.T) {
 	const stream = `
 	[

decode_uint.go

@@ -24,6 +24,30 @@ func (d *uintDecoder) parseUint(b []byte) uint64 {
 	return sum
 }
 
+func (d *uintDecoder) decodeStreamByte(s *stream) ([]byte, error) {
+	for {
+		switch s.char() {
+		case ' ', '\n', '\t', '\r':
+			s.progress()
+			continue
+		case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
+			start := s.cursor
+			for s.progress() {
+				tk := int(s.char())
+				if int('0') <= tk && tk <= int('9') {
+					continue
+				}
+				break
+			}
+			num := s.buf[start:s.cursor]
+			return num, nil
+		default:
+			return nil, errInvalidCharacter(s.char(), "number(unsigned integer)", s.totalOffset())
+		}
+	}
+	return nil, errUnexpectedEndOfJSON("number(unsigned integer)", s.totalOffset())
+}
+
 func (d *uintDecoder) decodeByte(buf []byte, cursor int64) ([]byte, int64, error) {
 	buflen := int64(len(buf))
 	for ; cursor < buflen; cursor++ {
@@ -49,6 +73,15 @@ func (d *uintDecoder) decodeByte(buf []byte, cursor int64) ([]byte, int64, error
 	return nil, 0, errUnexpectedEndOfJSON("number(unsigned integer)", cursor)
 }
 
+func (d *uintDecoder) decodeStream(s *stream, p uintptr) error {
+	bytes, err := d.decodeStreamByte(s)
+	if err != nil {
+		return err
+	}
+	d.op(p, d.parseUint(bytes))
+	return nil
+}
+
 func (d *uintDecoder) decode(buf []byte, cursor int64, p uintptr) (int64, error) {
 	bytes, c, err := d.decodeByte(buf, cursor)
 	if err != nil {

decode_unmarshal_json.go

@@ -12,6 +12,23 @@ func newUnmarshalJSONDecoder(typ *rtype) *unmarshalJSONDecoder {
 	return &unmarshalJSONDecoder{typ: typ}
 }
 
+func (d *unmarshalJSONDecoder) decodeStream(s *stream, p uintptr) error {
+	s.skipWhiteSpace()
+	start := s.cursor
+	if err := s.skipValue(); err != nil {
+		return err
+	}
+	src := s.buf[start:s.cursor]
+	v := *(*interface{})(unsafe.Pointer(&interfaceHeader{
+		typ: d.typ,
+		ptr: unsafe.Pointer(p),
+	}))
+	if err := v.(Unmarshaler).UnmarshalJSON(src); err != nil {
+		return err
+	}
+	return nil
+}
+
 func (d *unmarshalJSONDecoder) decode(buf []byte, cursor int64, p uintptr) (int64, error) {
 	cursor = skipWhiteSpace(buf, cursor)
 	start := cursor

decode_unmarshal_text.go

@@ -13,6 +13,23 @@ func newUnmarshalTextDecoder(typ *rtype) *unmarshalTextDecoder {
 	return &unmarshalTextDecoder{typ: typ}
 }
 
+func (d *unmarshalTextDecoder) decodeStream(s *stream, p uintptr) error {
+	s.skipWhiteSpace()
+	start := s.cursor
+	if err := s.skipValue(); err != nil {
+		return err
+	}
+	src := s.buf[start:s.cursor]
+	v := *(*interface{})(unsafe.Pointer(&interfaceHeader{
+		typ: d.typ,
+		ptr: unsafe.Pointer(p),
+	}))
+	if err := v.(encoding.TextUnmarshaler).UnmarshalText(src); err != nil {
+		return err
+	}
+	return nil
+}
+
 func (d *unmarshalTextDecoder) decode(buf []byte, cursor int64, p uintptr) (int64, error) {
 	cursor = skipWhiteSpace(buf, cursor)
 	start := cursor