re
/
go-json
forked from mirror/go-json
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Support Decoder.Token

This commit is contained in:
Masaaki Goshima 2020-07-30 22:41:53 +09:00
parent b4b79620aa
commit 20b67ad48d
16 changed files with 721 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

72
decode.go
View File

@ -5,6 +5,7 @@ import (
"fmt"
"io"
"reflect"
"strconv"
"sync"
"unsafe"
)
@ -17,6 +18,7 @@ func (d Delim) String() string {
type decoder interface {
decode([]byte, int64, uintptr) (int64, error)
decodeStream(*stream, uintptr) error
}
type Decoder struct {
@ -53,7 +55,9 @@ func init() {
// The decoder introduces its own buffering and may
// read data from r beyond the JSON values requested.
func NewDecoder(r io.Reader) *Decoder {
return &Decoder{s: &stream{r: r}}
s := &stream{r: r}
s.read()
return &Decoder{s: s}
}
// Buffered returns a reader of the data remaining in the Decoder's
@ -109,12 +113,13 @@ func (d *Decoder) decodeForUnmarshalNoEscape(src []byte, v interface{}) error {
func (d *Decoder) prepareForDecode() error {
s := d.s
for ; s.cursor < s.length || s.read(); s.cursor++ {
for {
switch s.char() {
case ' ', '\t', '\r', '\n':
s.progress()
continue
case ',', ':':
s.cursor++
s.progress()
return nil
}
break
@ -152,10 +157,7 @@ func (d *Decoder) Decode(v interface{}) error {
return err
}
s := d.s
cursor, err := dec.decode(s.buf[s.cursor:], 0, ptr)
s.cursor += cursor
fmt.Println("cursor = ", cursor, "next buf = ", string(s.buf[s.cursor:]))
if err != nil {
if err := dec.decodeStream(s, ptr); err != nil {
return err
}
return nil
@ -163,10 +165,12 @@ func (d *Decoder) Decode(v interface{}) error {
func (d *Decoder) More() bool {
s := d.s
for ; s.cursor < s.length || s.read(); s.cursor++ {
for {
switch s.char() {
case ' ', '\n', '\r', '\t':
if s.progress() {
continue
}
case '}', ']':
return false
}
@ -177,27 +181,51 @@ func (d *Decoder) More() bool {
func (d *Decoder) Token() (Token, error) {
s := d.s
for ; s.cursor < s.length || s.read(); s.cursor++ {
switch s.char() {
for {
c := s.char()
switch c {
case ' ', '\n', '\r', '\t':
if s.progress() {
continue
case '{':
s.cursor++
return Delim('{'), nil
case '[':
s.cursor++
return Delim('['), nil
case '}':
s.cursor++
return Delim('}'), nil
case ']':
s.cursor++
return Delim(']'), nil
}
case '{', '[', ']', '}':
s.progress()
return Delim(c), nil
case ',', ':':
if s.progress() {
continue
}
case '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
bytes := floatBytes(s)
s := *(*string)(unsafe.Pointer(&bytes))
f64, err := strconv.ParseFloat(s, 64)
if err != nil {
return nil, err
}
return f64, nil
case '"':
bytes, err := stringBytes(s)
if err != nil {
return nil, err
}
return string(bytes), nil
case 't':
if err := trueBytes(s); err != nil {
return nil, err
}
return true, nil
case 'f':
if err := falseBytes(s); err != nil {
return nil, err
}
return false, nil
case 'n':
if err := nullBytes(s); err != nil {
return nil, err
}
return nil, nil
case '\000':
return nil, io.EOF
default:
return nil, errInvalidCharacter(s.char(), "token", s.totalOffset())
}

30
decode_array.go
View File

@ -16,6 +16,36 @@ func newArrayDecoder(dec decoder, elemType *rtype, alen int) *arrayDecoder {
}
}
func (d *arrayDecoder) decodeStream(s *stream, p uintptr) error {
for {
switch s.char() {
case ' ', '\n', '\t', '\r':
case '[':
idx := 0
for {
s.progress()
if err := d.valueDecoder.decodeStream(s, p+uintptr(idx)*d.size); err != nil {
return err
}
s.skipWhiteSpace()
switch s.char() {
case ']':
s.progress()
return nil
case ',':
idx++
default:
return errInvalidCharacter(s.char(), "array", s.offset)
}
}
default:
return errUnexpectedEndOfJSON("array", s.offset)
}
s.progress()
}
return errUnexpectedEndOfJSON("array", s.offset)
}
func (d *arrayDecoder) decode(buf []byte, cursor int64, p uintptr) (int64, error) {
buflen := int64(len(buf))
for ; cursor < buflen; cursor++ {

61
decode_bool.go
View File

@ -10,6 +10,63 @@ func newBoolDecoder() *boolDecoder {
return &boolDecoder{}
}
func trueBytes(s *stream) error {
s.progress()
if s.char() != 'r' {
return errInvalidCharacter(s.char(), "bool(true)", s.totalOffset())
}
s.progress()
if s.char() != 'u' {
return errInvalidCharacter(s.char(), "bool(true)", s.totalOffset())
}
s.progress()
if s.char() != 'e' {
return errInvalidCharacter(s.char(), "bool(true)", s.totalOffset())
}
s.progress()
return nil
}
func falseBytes(s *stream) error {
s.progress()
if s.char() != 'a' {
return errInvalidCharacter(s.char(), "bool(false)", s.totalOffset())
}
s.progress()
if s.char() != 'l' {
return errInvalidCharacter(s.char(), "bool(false)", s.totalOffset())
}
s.progress()
if s.char() != 's' {
return errInvalidCharacter(s.char(), "bool(false)", s.totalOffset())
}
s.progress()
if s.char() != 'e' {
return errInvalidCharacter(s.char(), "bool(false)", s.totalOffset())
}
s.progress()
return nil
}
func (d *boolDecoder) decodeStream(s *stream, p uintptr) error {
s.skipWhiteSpace()
switch s.char() {
case 't':
if err := trueBytes(s); err != nil {
return err
}
*(*bool)(unsafe.Pointer(p)) = true
return nil
case 'f':
if err := falseBytes(s); err != nil {
return err
}
*(*bool)(unsafe.Pointer(p)) = false
return nil
}
return errUnexpectedEndOfJSON("bool", s.totalOffset())
}
func (d *boolDecoder) decode(buf []byte, cursor int64, p uintptr) (int64, error) {
buflen := int64(len(buf))
cursor = skipWhiteSpace(buf, cursor)
@ -29,6 +86,7 @@ func (d *boolDecoder) decode(buf []byte, cursor int64, p uintptr) (int64, error)
}
cursor += 4
*(*bool)(unsafe.Pointer(p)) = true
return cursor, nil
case 'f':
if cursor+4 >= buflen {
return 0, errUnexpectedEndOfJSON("bool(false)", cursor)
@ -47,6 +105,7 @@ func (d *boolDecoder) decode(buf []byte, cursor int64, p uintptr) (int64, error)
}
cursor += 5
*(*bool)(unsafe.Pointer(p)) = false
}
return cursor, nil
}
return 0, errUnexpectedEndOfJSON("bool", cursor)
}

58
decode_float.go
View File

@ -13,6 +13,48 @@ func newFloatDecoder(op func(uintptr, float64)) *floatDecoder {
return &floatDecoder{op: op}
}
var floatTable = [256]bool{
'0': true,
'1': true,
'2': true,
'3': true,
'4': true,
'5': true,
'6': true,
'7': true,
'8': true,
'9': true,
'.': true,
'e': true,
'E': true,
}
func floatBytes(s *stream) []byte {
start := s.cursor
for s.progress() {
if floatTable[s.char()] {
continue
}
break
}
return s.buf[start:s.cursor]
}
func (d *floatDecoder) decodeStreamByte(s *stream) ([]byte, error) {
for {
switch s.char() {
case ' ', '\n', '\t', '\r':
s.progress()
continue
case '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
return floatBytes(s), nil
default:
return nil, errUnexpectedEndOfJSON("float", s.offset)
}
}
return nil, errUnexpectedEndOfJSON("float", s.offset)
}
func (d *floatDecoder) decodeByte(buf []byte, cursor int64) ([]byte, int64, error) {
buflen := int64(len(buf))
for ; cursor < buflen; cursor++ {
@ -31,11 +73,27 @@ func (d *floatDecoder) decodeByte(buf []byte, cursor int64) ([]byte, int64, erro
}
num := buf[start:cursor]
return num, cursor, nil
default:
return nil, 0, errUnexpectedEndOfJSON("float", cursor)
}
}
return nil, 0, errUnexpectedEndOfJSON("float", cursor)
}
func (d *floatDecoder) decodeStream(s *stream, p uintptr) error {
bytes, err := d.decodeStreamByte(s)
if err != nil {
return err
}
str := *(*string)(unsafe.Pointer(&bytes))
f64, err := strconv.ParseFloat(str, 64)
if err != nil {
return err
}
d.op(p, f64)
return nil
}
func (d *floatDecoder) decode(buf []byte, cursor int64, p uintptr) (int64, error) {
bytes, c, err := d.decodeByte(buf, cursor)
if err != nil {

13
decode_int.go
View File

@ -49,15 +49,15 @@ var (
}
)
func (d *intDecoder) decodeByteStream(s *stream) ([]byte, error) {
for ; s.cursor < s.length || s.read(); s.cursor++ {
func (d *intDecoder) decodeStreamByte(s *stream) ([]byte, error) {
for {
switch s.char() {
case ' ', '\n', '\t', '\r':
s.progress()
continue
case '-':
start := s.cursor
s.cursor++
for ; s.cursor < s.length || s.read(); s.cursor++ {
for s.progress() {
if numTable[s.char()] {
continue
}
@ -70,8 +70,7 @@ func (d *intDecoder) decodeByteStream(s *stream) ([]byte, error) {
return num, nil
case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
start := s.cursor
s.cursor++
for ; s.cursor < s.length || s.read(); s.cursor++ {
for s.progress() {
if numTable[s.char()] {
continue
}
@ -110,7 +109,7 @@ func (d *intDecoder) decodeByte(buf []byte, cursor int64) ([]byte, int64, error)
}
func (d *intDecoder) decodeStream(s *stream, p uintptr) error {
bytes, err := d.decodeByteStream(s)
bytes, err := d.decodeStreamByte(s)
if err != nil {
return err
}

101
decode_interface.go
View File

@ -20,6 +20,107 @@ var (
)
)
func (d *interfaceDecoder) decodeStream(s *stream, p uintptr) error {
s.skipWhiteSpace()
switch s.char() {
case '{':
var v map[interface{}]interface{}
ptr := unsafe.Pointer(&v)
d.dummy = ptr
dec := newMapDecoder(interfaceMapType, newInterfaceDecoder(d.typ), newInterfaceDecoder(d.typ))
if err := dec.decodeStream(s, uintptr(ptr)); err != nil {
return err
}
*(*interface{})(unsafe.Pointer(p)) = v
return nil
case '[':
var v []interface{}
ptr := unsafe.Pointer(&v)
d.dummy = ptr // escape ptr
dec := newSliceDecoder(newInterfaceDecoder(d.typ), d.typ, d.typ.Size())
if err := dec.decodeStream(s, uintptr(ptr)); err != nil {
return err
}
*(*interface{})(unsafe.Pointer(p)) = v
return nil
case '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
return newFloatDecoder(func(p uintptr, v float64) {
*(*interface{})(unsafe.Pointer(p)) = v
}).decodeStream(s, p)
case '"':
s.progress()
start := s.cursor
for {
switch s.char() {
case '\\':
s.progress()
case '"':
literal := s.buf[start:s.cursor]
s.progress()
*(*interface{})(unsafe.Pointer(p)) = *(*string)(unsafe.Pointer(&literal))
return nil
case '\000':
return errUnexpectedEndOfJSON("string", s.totalOffset())
}
s.progress()
}
return errUnexpectedEndOfJSON("string", s.totalOffset())
case 't':
s.progress()
if s.char() != 'r' {
return errInvalidCharacter(s.char(), "bool(true)", s.totalOffset())
}
s.progress()
if s.char() != 'u' {
return errInvalidCharacter(s.char(), "bool(true)", s.totalOffset())
}
s.progress()
if s.char() != 'e' {
return errInvalidCharacter(s.char(), "bool(true)", s.totalOffset())
}
s.progress()
*(*interface{})(unsafe.Pointer(p)) = true
return nil
case 'f':
s.progress()
if s.char() != 'a' {
return errInvalidCharacter(s.char(), "bool(false)", s.totalOffset())
}
s.progress()
if s.char() != 'l' {
return errInvalidCharacter(s.char(), "bool(false)", s.totalOffset())
}
s.progress()
if s.char() != 's' {
return errInvalidCharacter(s.char(), "bool(false)", s.totalOffset())
}
s.progress()
if s.char() != 'e' {
return errInvalidCharacter(s.char(), "bool(false)", s.totalOffset())
}
s.progress()
*(*interface{})(unsafe.Pointer(p)) = false
return nil
case 'n':
s.progress()
if s.char() != 'u' {
return errInvalidCharacter(s.char(), "null", s.totalOffset())
}
s.progress()
if s.char() != 'l' {
return errInvalidCharacter(s.char(), "null", s.totalOffset())
}
s.progress()
if s.char() != 'l' {
return errInvalidCharacter(s.char(), "null", s.totalOffset())
}
s.progress()
*(*interface{})(unsafe.Pointer(p)) = nil
return nil
}
return errNotAtBeginningOfValue(s.totalOffset())
}
func (d *interfaceDecoder) decode(buf []byte, cursor int64, p uintptr) (int64, error) {
cursor = skipWhiteSpace(buf, cursor)
switch buf[cursor] {

46
decode_map.go
View File

@ -35,6 +35,52 @@ func (d *mapDecoder) setValue(buf []byte, cursor int64, key interface{}) (int64,
return d.valueDecoder.decode(buf, cursor, uintptr(header.ptr))
}
func (d *mapDecoder) setKeyStream(s *stream, key interface{}) error {
header := (*interfaceHeader)(unsafe.Pointer(&key))
return d.keyDecoder.decodeStream(s, uintptr(header.ptr))
}
func (d *mapDecoder) setValueStream(s *stream, key interface{}) error {
header := (*interfaceHeader)(unsafe.Pointer(&key))
return d.valueDecoder.decodeStream(s, uintptr(header.ptr))
}
func (d *mapDecoder) decodeStream(s *stream, p uintptr) error {
s.skipWhiteSpace()
if s.char() != '{' {
return errExpected("{ character for map value", s.totalOffset())
}
mapValue := makemap(d.mapType, 0)
for s.progress() {
var key interface{}
if err := d.setKeyStream(s, &key); err != nil {
return err
}
s.skipWhiteSpace()
if s.char() != ':' {
return errExpected("colon after object key", s.totalOffset())
}
s.progress()
if s.end() {
return errUnexpectedEndOfJSON("map", s.totalOffset())
}
var value interface{}
if err := d.setValueStream(s, &value); err != nil {
return err
}
mapassign(d.mapType, mapValue, unsafe.Pointer(&key), unsafe.Pointer(&value))
s.skipWhiteSpace()
if s.char() == '}' {
*(*unsafe.Pointer)(unsafe.Pointer(p)) = mapValue
return nil
}
if s.char() != ',' {
return errExpected("semicolon after object value", s.totalOffset())
}
}
return nil
}
func (d *mapDecoder) decode(buf []byte, cursor int64, p uintptr) (int64, error) {
cursor = skipWhiteSpace(buf, cursor)
buflen := int64(len(buf))

9
decode_ptr.go
View File

@ -16,6 +16,15 @@ func newPtrDecoder(dec decoder, typ *rtype) *ptrDecoder {
//go:linkname unsafe_New reflect.unsafe_New
func unsafe_New(*rtype) uintptr
func (d *ptrDecoder) decodeStream(s *stream, p uintptr) error {
newptr := unsafe_New(d.typ)
if err := d.dec.decodeStream(s, newptr); err != nil {
return err
}
*(*uintptr)(unsafe.Pointer(p)) = newptr
return nil
}
func (d *ptrDecoder) decode(buf []byte, cursor int64, p uintptr) (int64, error) {
newptr := unsafe_New(d.typ)
c, err := d.dec.decode(buf, cursor, newptr)

54
decode_slice.go
View File

@ -47,6 +47,60 @@ func copySlice(elemType *rtype, dst, src reflect.SliceHeader) int
//go:linkname newArray reflect.unsafe_NewArray
func newArray(*rtype, int) unsafe.Pointer
func (d *sliceDecoder) decodeStream(s *stream, p uintptr) error {
for {
switch s.char() {
case ' ', '\n', '\t', '\r':
s.progress()
continue
case '[':
idx := 0
slice := d.newSlice()
cap := slice.Cap
data := slice.Data
for s.progress() {
if cap <= idx {
src := reflect.SliceHeader{Data: data, Len: idx, Cap: cap}
cap *= 2
data = uintptr(newArray(d.elemType, cap))
dst := reflect.SliceHeader{Data: data, Len: idx, Cap: cap}
copySlice(d.elemType, dst, src)
}
if err := d.valueDecoder.decodeStream(s, data+uintptr(idx)*d.size); err != nil {
return err
}
s.skipWhiteSpace()
switch s.char() {
case ']':
slice.Cap = cap
slice.Len = idx + 1
slice.Data = data
dstCap := idx + 1
dst := reflect.SliceHeader{
Data: uintptr(newArray(d.elemType, dstCap)),
Len: idx + 1,
Cap: dstCap,
}
copySlice(d.elemType, dst, *slice)
*(*reflect.SliceHeader)(unsafe.Pointer(p)) = dst
d.releaseSlice(slice)
s.progress()
return nil
case ',':
idx++
continue
default:
slice.Cap = cap
slice.Data = data
d.releaseSlice(slice)
return errInvalidCharacter(s.char(), "slice", s.totalOffset())
}
}
}
}
return errUnexpectedEndOfJSON("slice", s.totalOffset())
}
func (d *sliceDecoder) decode(buf []byte, cursor int64, p uintptr) (int64, error) {
buflen := int64(len(buf))
for ; cursor < buflen; cursor++ {

117
decode_stream.go
View File

@ -6,16 +6,16 @@ import (
)
const (
readChunkSize = 1024
readChunkSize = 2
)
type stream struct {
buf []byte
length int64
r io.Reader
decodedPos int64
offset int64
cursor int64
allRead bool
}
func (s *stream) buffered() io.Reader {
@ -26,24 +26,121 @@ func (s *stream) totalOffset() int64 {
return s.offset + s.cursor
}
func (s *stream) prevChar() byte {
return s.buf[s.cursor-1]
}
func (s *stream) char() byte {
return s.buf[s.cursor]
}
func (s *stream) end() bool {
return s.allRead && s.length <= s.cursor
}
func (s *stream) progress() bool {
if s.cursor < s.length-1 || s.read() {
s.cursor++
return true
}
s.cursor = s.length
return false
}
func (s *stream) progressN(n int64) bool {
if s.cursor+n < s.length-1 || s.read() {
s.cursor += n
return true
}
s.cursor = s.length
return false
}
func (s *stream) reset() {
s.buf = s.buf[s.cursor:]
s.length -= s.cursor
s.cursor = 0
}
func (s *stream) read() bool {
buf := make([]byte, readChunkSize)
n, err := s.r.Read(buf)
if n == 0 || err == io.EOF {
if err != nil && err != io.EOF {
return false
}
remain := s.length - s.decodedPos
newBuf := make([]byte, remain+int64(n))
copy(newBuf, s.buf[s.decodedPos:])
remain := s.length
newBuf := make([]byte, remain+int64(n)+1)
copy(newBuf, s.buf)
copy(newBuf[remain:], buf)
s.buf = newBuf
s.length = int64(len(newBuf))
s.offset += s.decodedPos
s.cursor = 0
s.decodedPos = 0
s.length = int64(len(newBuf)) - 1
s.offset += s.cursor
if n == 0 || err == io.EOF {
s.allRead = true
return false
}
return true
}
func (s *stream) skipWhiteSpace() {
LOOP:
if isWhiteSpace[s.char()] {
s.progress()
goto LOOP
}
}
func (s *stream) skipValue() error {
s.skipWhiteSpace()
braceCount := 0
bracketCount := 0
for {
switch s.char() {
case '\000':
return errUnexpectedEndOfJSON("value of object", s.offset)
case '{':
braceCount++
case '[':
bracketCount++
case '}':
braceCount--
if braceCount == -1 && bracketCount == 0 {
return nil
}
case ']':
bracketCount--
case ',':
if bracketCount == 0 && braceCount == 0 {
return nil
}
case '"':
for s.progress() {
if s.char() != '"' {
continue
}
if s.prevChar() == '\\' {
continue
}
if bracketCount == 0 && braceCount == 0 {
s.progress()
return nil
}
break
}
case '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
for s.progress() {
tk := int(s.char())
if (int('0') <= tk && tk <= int('9')) || tk == '.' || tk == 'e' || tk == 'E' {
continue
}
break
}
if bracketCount == 0 && braceCount == 0 {
return nil
}
continue
}
s.progress()
}
return errUnexpectedEndOfJSON("value of object", s.offset)
}

67
decode_string.go
View File

@ -11,6 +11,15 @@ func newStringDecoder() *stringDecoder {
return &stringDecoder{}
}
func (d *stringDecoder) decodeStream(s *stream, p uintptr) error {
bytes, err := d.decodeStreamByte(s)
if err != nil {
return err
}
*(*string)(unsafe.Pointer(p)) = *(*string)(unsafe.Pointer(&bytes))
return nil
}
func (d *stringDecoder) decode(buf []byte, cursor int64, p uintptr) (int64, error) {
bytes, c, err := d.decodeByte(buf, cursor)
if err != nil {
@ -21,6 +30,64 @@ func (d *stringDecoder) decode(buf []byte, cursor int64, p uintptr) (int64, erro
return cursor, nil
}
func stringBytes(s *stream) ([]byte, error) {
s.progress()
start := s.cursor
for {
switch s.char() {
case '\\':
s.progress()
case '"':
literal := s.buf[start:s.cursor]
s.progress()
s.reset()
return literal, nil
case '\000':
goto ERROR
}
s.progress()
}
ERROR:
return nil, errUnexpectedEndOfJSON("string", s.totalOffset())
}
func nullBytes(s *stream) error {
s.progress()
if s.char() != 'u' {
return errInvalidCharacter(s.char(), "null", s.totalOffset())
}
s.progress()
if s.char() != 'l' {
return errInvalidCharacter(s.char(), "null", s.totalOffset())
}
s.progress()
if s.char() != 'l' {
return errInvalidCharacter(s.char(), "null", s.totalOffset())
}
s.progress()
return nil
}
func (d *stringDecoder) decodeStreamByte(s *stream) ([]byte, error) {
for {
switch s.char() {
case ' ', '\n', '\t', '\r':
s.progress()
case '"':
return stringBytes(s)
case 'n':
if err := nullBytes(s); err != nil {
return nil, err
}
return []byte{'n', 'u', 'l', 'l'}, nil
default:
goto ERROR
}
}
ERROR:
return nil, errNotAtBeginningOfValue(s.totalOffset())
}
func (d *stringDecoder) decodeByte(buf []byte, cursor int64) ([]byte, int64, error) {
for {
switch buf[cursor] {

45
decode_struct.go
View File

@ -21,6 +21,51 @@ func newStructDecoder(fieldMap map[string]*structFieldSet) *structDecoder {
}
}
func (d *structDecoder) decodeStream(s *stream, p uintptr) error {
s.skipWhiteSpace()
if s.char() != '{' {
return errNotAtBeginningOfValue(s.totalOffset())
}
s.progress()
for {
s.reset()
key, err := d.keyDecoder.decodeStreamByte(s)
if err != nil {
return err
}
s.skipWhiteSpace()
if s.char() != ':' {
return errExpected("colon after object key", s.totalOffset())
}
s.progress()
if s.end() {
return errExpected("object value after colon", s.totalOffset())
}
k := *(*string)(unsafe.Pointer(&key))
field, exists := d.fieldMap[k]
if exists {
if err := field.dec.decodeStream(s, p+field.offset); err != nil {
return err
}
} else {
if err := s.skipValue(); err != nil {
return err
}
}
s.skipWhiteSpace()
c := s.char()
if c == '}' {
s.progress()
return nil
}
if c != ',' {
return errExpected("comma after object element", s.totalOffset())
}
s.progress()
}
return nil
}
func (d *structDecoder) decode(buf []byte, cursor int64, p uintptr) (int64, error) {
buflen := int64(len(buf))
cursor = skipWhiteSpace(buf, cursor)

14
decode_test.go
View File

@ -233,6 +233,20 @@ func Test_InvalidUnmarshalError(t *testing.T) {
})
}
func Test_Token(t *testing.T) {
dec := json.NewDecoder(strings.NewReader(`{"a": 1, "b": true, "c": [1, "two", null]}`))
cnt := 0
for {
if _, err := dec.Token(); err != nil {
break
}
cnt++
}
if cnt != 12 {
t.Fatal("failed to parse token")
}
}
func Test_DecodeStream(t *testing.T) {
const stream = `
[

33
decode_uint.go
View File

@ -24,6 +24,30 @@ func (d *uintDecoder) parseUint(b []byte) uint64 {
return sum
}
func (d *uintDecoder) decodeStreamByte(s *stream) ([]byte, error) {
for {
switch s.char() {
case ' ', '\n', '\t', '\r':
s.progress()
continue
case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
start := s.cursor
for s.progress() {
tk := int(s.char())
if int('0') <= tk && tk <= int('9') {
continue
}
break
}
num := s.buf[start:s.cursor]
return num, nil
default:
return nil, errInvalidCharacter(s.char(), "number(unsigned integer)", s.totalOffset())
}
}
return nil, errUnexpectedEndOfJSON("number(unsigned integer)", s.totalOffset())
}
func (d *uintDecoder) decodeByte(buf []byte, cursor int64) ([]byte, int64, error) {
buflen := int64(len(buf))
for ; cursor < buflen; cursor++ {
@ -49,6 +73,15 @@ func (d *uintDecoder) decodeByte(buf []byte, cursor int64) ([]byte, int64, error
return nil, 0, errUnexpectedEndOfJSON("number(unsigned integer)", cursor)
}
func (d *uintDecoder) decodeStream(s *stream, p uintptr) error {
bytes, err := d.decodeStreamByte(s)
if err != nil {
return err
}
d.op(p, d.parseUint(bytes))
return nil
}
func (d *uintDecoder) decode(buf []byte, cursor int64, p uintptr) (int64, error) {
bytes, c, err := d.decodeByte(buf, cursor)
if err != nil {

17
decode_unmarshal_json.go
View File

@ -12,6 +12,23 @@ func newUnmarshalJSONDecoder(typ *rtype) *unmarshalJSONDecoder {
return &unmarshalJSONDecoder{typ: typ}
}
func (d *unmarshalJSONDecoder) decodeStream(s *stream, p uintptr) error {
s.skipWhiteSpace()
start := s.cursor
if err := s.skipValue(); err != nil {
return err
}
src := s.buf[start:s.cursor]
v := *(*interface{})(unsafe.Pointer(&interfaceHeader{
typ: d.typ,
ptr: unsafe.Pointer(p),
}))
if err := v.(Unmarshaler).UnmarshalJSON(src); err != nil {
return err
}
return nil
}
func (d *unmarshalJSONDecoder) decode(buf []byte, cursor int64, p uintptr) (int64, error) {
cursor = skipWhiteSpace(buf, cursor)
start := cursor

17
decode_unmarshal_text.go
View File

@ -13,6 +13,23 @@ func newUnmarshalTextDecoder(typ *rtype) *unmarshalTextDecoder {
return &unmarshalTextDecoder{typ: typ}
}
func (d *unmarshalTextDecoder) decodeStream(s *stream, p uintptr) error {
s.skipWhiteSpace()
start := s.cursor
if err := s.skipValue(); err != nil {
return err
}
src := s.buf[start:s.cursor]
v := *(*interface{})(unsafe.Pointer(&interfaceHeader{
typ: d.typ,
ptr: unsafe.Pointer(p),
}))
if err := v.(encoding.TextUnmarshaler).UnmarshalText(src); err != nil {
return err
}
return nil
}
func (d *unmarshalTextDecoder) decode(buf []byte, cursor int64, p uintptr) (int64, error) {
cursor = skipWhiteSpace(buf, cursor)
start := cursor