From 5090a4bf5ea9d9bce62f4eb5e84f6a2947be0048 Mon Sep 17 00:00:00 2001 From: Masaaki Goshima Date: Sun, 27 Dec 2020 21:25:26 +0900 Subject: [PATCH] WIP --- decode.go | 60 +++++++++++++++++++++++++++- decode_stream.go | 10 +++++ error.go | 2 + stream_test.go | 102 +++++++++++++++++++++++------------------------ 4 files changed, 122 insertions(+), 52 deletions(-) diff --git a/decode.go b/decode.go index 255a7b5..998629c 100644 --- a/decode.go +++ b/decode.go @@ -117,8 +117,58 @@ func (d *Decoder) decodeForUnmarshalNoEscape(src []byte, v interface{}) error { return d.decode(src, header) } +func (d *Decoder) prepareForDecodeArray() error { + for { + switch s.char() { + case ' ', '\t', '\r', '\n': + s.cursor++ + continue + case ',', ':': + return errExpected("value in array element", s.totalOffset()) + case nul: + if s.read() { + continue + } + return io.EOF + } + break + } + s.context = streamContextTypeArrayDelim + return nil +} + +func (d *Decoder) prepareForDecodeArrayDelim() error { + for { + switch s.char() { + case ' ', '\t', '\r', '\n': + s.cursor++ + continue + case ',': + s.cursor++ + return nil + case ']': + s.cursor++ + s.context = streamContextTypeValue + return nil + case nul: + if s.read() { + continue + } + return io.EOF + } + break + } + return nil +} + func (d *Decoder) prepareForDecode() error { s := d.s + switch s.context { + case streamContextTypeArray: + return d.prepareForDecodeArray() + case streamContextTypeArrayDelim: + return d.prepareForDecodeArrayDelim() + } for { switch s.char() { case ' ', '\t', '\r', '\n': @@ -202,8 +252,16 @@ func (d *Decoder) Token() (Token, error) { switch c { case ' ', '\n', '\r', '\t': s.cursor++ - case '{', '[', ']', '}': + case ']', '}': s.cursor++ + d.s.context = streamContextTypeValue + return Delim(c), nil + case '{': + s.cursor++ + d.s.context = streamContextTypeObject + return Delim(c), nil + case '[': + d.s.context = streamContextTypeArray return Delim(c), nil case ',', ':': s.cursor++ diff --git a/decode_stream.go b/decode_stream.go index ca9d7d4..5cc6d02 100644 --- a/decode_stream.go +++ b/decode_stream.go @@ -20,8 +20,18 @@ type stream struct { allRead bool useNumber bool disallowUnknownFields bool + context streamContextType } +const ( + streamContextTypeValue streamContextType = iota + streamContextTypeArray + streamContextTypeArrayDelim /* expect ',' or ']' */ + streamContextTypeObject + streamContextTypeObjectColon /* expect ':' */ + streamContextTypeObjectDelim /* expect ',' or '}' */ +) + func newStream(r io.Reader) *stream { return &stream{ r: r, diff --git a/error.go b/error.go index 1a574ba..82028e5 100644 --- a/error.go +++ b/error.go @@ -3,6 +3,7 @@ package json import ( "fmt" "reflect" + "runtime" "strconv" ) @@ -122,6 +123,7 @@ func errNotAtBeginningOfValue(cursor int64) *SyntaxError { } func errUnexpectedEndOfJSON(msg string, cursor int64) *SyntaxError { + fmt.Println(runtime.Caller(1)) return &SyntaxError{ msg: fmt.Sprintf("json: %s unexpected end of JSON input", msg), Offset: cursor, diff --git a/stream_test.go b/stream_test.go index 8fee6cd..ffae01b 100644 --- a/stream_test.go +++ b/stream_test.go @@ -324,57 +324,59 @@ type decodeThis struct { var tokenStreamCases = []tokenStreamCase{ // streaming token cases - {json: `10`, expTokens: []interface{}{float64(10)}}, - {json: ` [10] `, expTokens: []interface{}{ - json.Delim('['), float64(10), json.Delim(']')}}, - {json: ` [false,10,"b"] `, expTokens: []interface{}{ - json.Delim('['), false, float64(10), "b", json.Delim(']')}}, - {json: `{ "a": 1 }`, expTokens: []interface{}{ - json.Delim('{'), "a", float64(1), json.Delim('}')}}, - {json: `{"a": 1, "b":"3"}`, expTokens: []interface{}{ - json.Delim('{'), "a", float64(1), "b", "3", json.Delim('}')}}, - {json: ` [{"a": 1},{"a": 2}] `, expTokens: []interface{}{ - json.Delim('['), - json.Delim('{'), "a", float64(1), json.Delim('}'), - json.Delim('{'), "a", float64(2), json.Delim('}'), - json.Delim(']')}}, - {json: `{"obj": {"a": 1}}`, expTokens: []interface{}{ - json.Delim('{'), "obj", json.Delim('{'), "a", float64(1), json.Delim('}'), - json.Delim('}')}}, - {json: `{"obj": [{"a": 1}]}`, expTokens: []interface{}{ - json.Delim('{'), "obj", json.Delim('['), - json.Delim('{'), "a", float64(1), json.Delim('}'), - json.Delim(']'), json.Delim('}')}}, + /* + {json: `10`, expTokens: []interface{}{float64(10)}}, + {json: ` [10] `, expTokens: []interface{}{ + json.Delim('['), float64(10), json.Delim(']')}}, + {json: ` [false,10,"b"] `, expTokens: []interface{}{ + json.Delim('['), false, float64(10), "b", json.Delim(']')}}, + {json: `{ "a": 1 }`, expTokens: []interface{}{ + json.Delim('{'), "a", float64(1), json.Delim('}')}}, + {json: `{"a": 1, "b":"3"}`, expTokens: []interface{}{ + json.Delim('{'), "a", float64(1), "b", "3", json.Delim('}')}}, + {json: ` [{"a": 1},{"a": 2}] `, expTokens: []interface{}{ + json.Delim('['), + json.Delim('{'), "a", float64(1), json.Delim('}'), + json.Delim('{'), "a", float64(2), json.Delim('}'), + json.Delim(']')}}, + {json: `{"obj": {"a": 1}}`, expTokens: []interface{}{ + json.Delim('{'), "obj", json.Delim('{'), "a", float64(1), json.Delim('}'), + json.Delim('}')}}, + {json: `{"obj": [{"a": 1}]}`, expTokens: []interface{}{ + json.Delim('{'), "obj", json.Delim('['), + json.Delim('{'), "a", float64(1), json.Delim('}'), + json.Delim(']'), json.Delim('}')}}, - // streaming tokens with intermittent Decode() - {json: `{ "a": 1 }`, expTokens: []interface{}{ - json.Delim('{'), "a", - decodeThis{float64(1)}, - json.Delim('}')}}, - {json: ` [ { "a" : 1 } ] `, expTokens: []interface{}{ - json.Delim('['), - decodeThis{map[string]interface{}{"a": float64(1)}}, - json.Delim(']')}}, - {json: ` [{"a": 1},{"a": 2}] `, expTokens: []interface{}{ - json.Delim('['), - decodeThis{map[string]interface{}{"a": float64(1)}}, - decodeThis{map[string]interface{}{"a": float64(2)}}, - json.Delim(']')}}, - {json: `{ "obj" : [ { "a" : 1 } ] }`, expTokens: []interface{}{ - json.Delim('{'), "obj", json.Delim('['), - decodeThis{map[string]interface{}{"a": float64(1)}}, - json.Delim(']'), json.Delim('}')}}, + // streaming tokens with intermittent Decode() + {json: `{ "a": 1 }`, expTokens: []interface{}{ + json.Delim('{'), "a", + decodeThis{float64(1)}, + json.Delim('}')}}, + {json: ` [ { "a" : 1 } ] `, expTokens: []interface{}{ + json.Delim('['), + decodeThis{map[string]interface{}{"a": float64(1)}}, + json.Delim(']')}}, + {json: ` [{"a": 1},{"a": 2}] `, expTokens: []interface{}{ + json.Delim('['), + decodeThis{map[string]interface{}{"a": float64(1)}}, + decodeThis{map[string]interface{}{"a": float64(2)}}, + json.Delim(']')}}, + {json: `{ "obj" : [ { "a" : 1 } ] }`, expTokens: []interface{}{ + json.Delim('{'), "obj", json.Delim('['), + decodeThis{map[string]interface{}{"a": float64(1)}}, + json.Delim(']'), json.Delim('}')}}, - {json: `{"obj": {"a": 1}}`, expTokens: []interface{}{ - json.Delim('{'), "obj", - decodeThis{map[string]interface{}{"a": float64(1)}}, - json.Delim('}')}}, - {json: `{"obj": [{"a": 1}]}`, expTokens: []interface{}{ - json.Delim('{'), "obj", - decodeThis{[]interface{}{ - map[string]interface{}{"a": float64(1)}, - }}, - json.Delim('}')}}, + {json: `{"obj": {"a": 1}}`, expTokens: []interface{}{ + json.Delim('{'), "obj", + decodeThis{map[string]interface{}{"a": float64(1)}}, + json.Delim('}')}}, + {json: `{"obj": [{"a": 1}]}`, expTokens: []interface{}{ + json.Delim('{'), "obj", + decodeThis{[]interface{}{ + map[string]interface{}{"a": float64(1)}, + }}, + json.Delim('}')}}, + */ {json: ` [{"a": 1} {"a": 2}] `, expTokens: []interface{}{ json.Delim('['), decodeThis{map[string]interface{}{"a": float64(1)}}, @@ -393,7 +395,6 @@ var tokenStreamCases = []tokenStreamCase{ }}, } -/* func TestDecodeInStream(t *testing.T) { for ci, tcase := range tokenStreamCases { @@ -428,7 +429,6 @@ func TestDecodeInStream(t *testing.T) { } } } -*/ // Test from golang.org/issue/11893 func TestHTTPDecoding(t *testing.T) {