mirror
/
go-json
mirror of https://github.com/goccy/go-json.git
1
0
Fork 1
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #242 from goccy/feature/improve-decoder-performance 

Add DecodeFieldPriorityFirstWin option
This commit is contained in:
Masaaki Goshima 2021-06-06 18:38:42 +09:00 committed by GitHub
parent 7d7af9fb36 8a3ef38852
commit b05a0c301e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
25 changed files with 327 additions and 117 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
benchmarks/decode_test.go
View File

@ -375,6 +375,34 @@ func Benchmark_Decode_LargeStruct_Unmarshal_GoJsonNoEscape(b *testing.B) {
}
}
func Benchmark_Decode_LargeStruct_Unmarshal_GoJsonFirstWinMode(b *testing.B) {
b.ReportAllocs()
for i := 0; i < b.N; i++ {
result := LargePayload{}
if err := gojson.UnmarshalWithOption(
LargeFixture,
&result,
gojson.DecodeFieldPriorityFirstWin(),
); err != nil {
b.Fatal(err)
}
}
}
func Benchmark_Decode_LargeStruct_Unmarshal_GoJsonNoEscapeFirstWinMode(b *testing.B) {
b.ReportAllocs()
for i := 0; i < b.N; i++ {
result := LargePayload{}
if err := gojson.UnmarshalNoEscape(
LargeFixture,
&result,
gojson.DecodeFieldPriorityFirstWin(),
); err != nil {
b.Fatal(err)
}
}
}
func Benchmark_Decode_LargeStruct_Stream_EncodingJson(b *testing.B) {
b.ReportAllocs()
reader := bytes.NewReader(LargeFixture)
@ -434,3 +462,18 @@ func Benchmark_Decode_LargeStruct_Stream_GoJson(b *testing.B) {
}
}
}
func Benchmark_Decode_LargeStruct_Stream_GoJsonFirstWinMode(b *testing.B) {
b.ReportAllocs()
reader := bytes.NewReader(LargeFixture)
for i := 0; i < b.N; i++ {
result := LargePayload{}
reader.Reset(LargeFixture)
if err := gojson.NewDecoder(reader).DecodeWithOption(
&result,
gojson.DecodeFieldPriorityFirstWin(),
); err != nil {
b.Fatal(err)
}
}
}

32
decode.go
View File

@ -24,7 +24,7 @@ type emptyInterface struct {
ptr unsafe.Pointer
}
func unmarshal(data []byte, v interface{}) error {
func unmarshal(data []byte, v interface{}, optFuncs ...DecodeOptionFunc) error {
src := make([]byte, len(data)+1) // append nul byte to the end
copy(src, data)
@ -37,14 +37,22 @@ func unmarshal(data []byte, v interface{}) error {
if err != nil {
return err
}
cursor, err := dec.Decode(src, 0, 0, header.ptr)
ctx := decoder.TakeRuntimeContext()
ctx.Buf = src
ctx.Option.Flag = 0
for _, optFunc := range optFuncs {
optFunc(ctx.Option)
}
cursor, err := dec.Decode(ctx, 0, 0, header.ptr)
if err != nil {
decoder.ReleaseRuntimeContext(ctx)
return err
}
decoder.ReleaseRuntimeContext(ctx)
return validateEndBuf(src, cursor)
}
func unmarshalNoEscape(data []byte, v interface{}) error {
func unmarshalNoEscape(data []byte, v interface{}, optFuncs ...DecodeOptionFunc) error {
src := make([]byte, len(data)+1) // append nul byte to the end
copy(src, data)
@ -57,10 +65,19 @@ func unmarshalNoEscape(data []byte, v interface{}) error {
if err != nil {
return err
}
cursor, err := dec.Decode(src, 0, 0, noescape(header.ptr))
ctx := decoder.TakeRuntimeContext()
ctx.Buf = src
ctx.Option.Flag = 0
for _, optFunc := range optFuncs {
optFunc(ctx.Option)
}
cursor, err := dec.Decode(ctx, 0, 0, noescape(header.ptr))
if err != nil {
decoder.ReleaseRuntimeContext(ctx)
return err
}
decoder.ReleaseRuntimeContext(ctx)
return validateEndBuf(src, cursor)
}
@ -117,6 +134,10 @@ func (d *Decoder) Buffered() io.Reader {
// See the documentation for Unmarshal for details about
// the conversion of JSON into a Go value.
func (d *Decoder) Decode(v interface{}) error {
return d.DecodeWithOption(v)
}
func (d *Decoder) DecodeWithOption(v interface{}, optFuncs ...DecodeOptionFunc) error {
header := (*emptyInterface)(unsafe.Pointer(&v))
typ := header.typ
ptr := uintptr(header.ptr)
@ -136,6 +157,9 @@ func (d *Decoder) Decode(v interface{}) error {
return err
}
s := d.s
for _, optFunc := range optFuncs {
optFunc(s.Option)
}
if err := dec.DecodeStream(s, 0, header.ptr); err != nil {
return err
}

4
internal/decoder/anonymous_field.go
View File

@ -28,10 +28,10 @@ func (d *anonymousFieldDecoder) DecodeStream(s *Stream, depth int64, p unsafe.Po
return d.dec.DecodeStream(s, depth, unsafe.Pointer(uintptr(p)+d.offset))
}
func (d *anonymousFieldDecoder) Decode(buf []byte, cursor, depth int64, p unsafe.Pointer) (int64, error) {
func (d *anonymousFieldDecoder) Decode(ctx *RuntimeContext, cursor, depth int64, p unsafe.Pointer) (int64, error) {
if *(*unsafe.Pointer)(p) == nil {
*(*unsafe.Pointer)(p) = unsafe_New(d.structType)
}
p = *(*unsafe.Pointer)(p)
return d.dec.Decode(buf, cursor, depth, unsafe.Pointer(uintptr(p)+d.offset))
return d.dec.Decode(ctx, cursor, depth, unsafe.Pointer(uintptr(p)+d.offset))
}

8
internal/decoder/array.go
View File

@ -58,8 +58,7 @@ func (d *arrayDecoder) DecodeStream(s *Stream, depth int64, p unsafe.Pointer) er
}
}
idx++
s.skipWhiteSpace()
switch s.char() {
switch s.skipWhiteSpace() {
case ']':
for idx < d.alen {
*(*unsafe.Pointer)(unsafe.Pointer(uintptr(p) + uintptr(idx)*d.size)) = d.zeroValue
@ -92,7 +91,8 @@ ERROR:
return errors.ErrUnexpectedEndOfJSON("array", s.totalOffset())
}
func (d *arrayDecoder) Decode(buf []byte, cursor, depth int64, p unsafe.Pointer) (int64, error) {
func (d *arrayDecoder) Decode(ctx *RuntimeContext, cursor, depth int64, p unsafe.Pointer) (int64, error) {
buf := ctx.Buf
depth++
if depth > maxDecodeNestingDepth {
return 0, errors.ErrExceededMaxDepth(buf[cursor], cursor)
@ -114,7 +114,7 @@ func (d *arrayDecoder) Decode(buf []byte, cursor, depth int64, p unsafe.Pointer)
for {
cursor++
if idx < d.alen {
c, err := d.valueDecoder.Decode(buf, cursor, depth, unsafe.Pointer(uintptr(p)+uintptr(idx)*d.size))
c, err := d.valueDecoder.Decode(ctx, cursor, depth, unsafe.Pointer(uintptr(p)+uintptr(idx)*d.size))
if err != nil {
return 0, err
}

8
internal/decoder/bool.go
View File

@ -16,9 +16,9 @@ func newBoolDecoder(structName, fieldName string) *boolDecoder {
}
func (d *boolDecoder) DecodeStream(s *Stream, depth int64, p unsafe.Pointer) error {
s.skipWhiteSpace()
c := s.skipWhiteSpace()
for {
switch s.char() {
switch c {
case 't':
if err := trueBytes(s); err != nil {
return err
@ -38,6 +38,7 @@ func (d *boolDecoder) DecodeStream(s *Stream, depth int64, p unsafe.Pointer) err
return nil
case nul:
if s.read() {
c = s.char()
continue
}
goto ERROR
@ -48,7 +49,8 @@ ERROR:
return errors.ErrUnexpectedEndOfJSON("bool", s.totalOffset())
}
func (d *boolDecoder) Decode(buf []byte, cursor, depth int64, p unsafe.Pointer) (int64, error) {
func (d *boolDecoder) Decode(ctx *RuntimeContext, cursor, depth int64, p unsafe.Pointer) (int64, error) {
buf := ctx.Buf
cursor = skipWhiteSpace(buf, cursor)
switch buf[cursor] {
case 't':

9
internal/decoder/bytes.go
View File

@ -57,8 +57,8 @@ func (d *bytesDecoder) DecodeStream(s *Stream, depth int64, p unsafe.Pointer) er
return nil
}
func (d *bytesDecoder) Decode(buf []byte, cursor, depth int64, p unsafe.Pointer) (int64, error) {
bytes, c, err := d.decodeBinary(buf, cursor, depth, p)
func (d *bytesDecoder) Decode(ctx *RuntimeContext, cursor, depth int64, p unsafe.Pointer) (int64, error) {
bytes, c, err := d.decodeBinary(ctx, cursor, depth, p)
if err != nil {
return 0, err
}
@ -131,7 +131,8 @@ func (d *bytesDecoder) decodeStreamBinary(s *Stream, depth int64, p unsafe.Point
return nil, errors.ErrNotAtBeginningOfValue(s.totalOffset())
}
func (d *bytesDecoder) decodeBinary(buf []byte, cursor, depth int64, p unsafe.Pointer) ([]byte, int64, error) {
func (d *bytesDecoder) decodeBinary(ctx *RuntimeContext, cursor, depth int64, p unsafe.Pointer) ([]byte, int64, error) {
buf := ctx.Buf
for {
switch buf[cursor] {
case ' ', '\n', '\t', '\r':
@ -157,7 +158,7 @@ func (d *bytesDecoder) decodeBinary(buf []byte, cursor, depth int64, p unsafe.Po
Offset: cursor,
}
}
c, err := d.sliceDecoder.Decode(buf, cursor, depth, p)
c, err := d.sliceDecoder.Decode(ctx, cursor, depth, p)
if err != nil {
return nil, 0, err
}

24
internal/decoder/context.go
View File

@ -1,11 +1,35 @@
package decoder
import (
"sync"
"unsafe"
"github.com/goccy/go-json/internal/errors"
)
type RuntimeContext struct {
Buf []byte
Option *Option
}
var (
runtimeContextPool = sync.Pool{
New: func() interface{} {
return &RuntimeContext{
Option: &Option{},
}
},
}
)
func TakeRuntimeContext() *RuntimeContext {
return runtimeContextPool.Get().(*RuntimeContext)
}
func ReleaseRuntimeContext(ctx *RuntimeContext) {
runtimeContextPool.Put(ctx)
}
var (
isWhiteSpace = [256]bool{}
)

3
internal/decoder/float.go
View File

@ -135,7 +135,8 @@ func (d *floatDecoder) DecodeStream(s *Stream, depth int64, p unsafe.Pointer) er
return nil
}
func (d *floatDecoder) Decode(buf []byte, cursor, depth int64, p unsafe.Pointer) (int64, error) {
func (d *floatDecoder) Decode(ctx *RuntimeContext, cursor, depth int64, p unsafe.Pointer) (int64, error) {
buf := ctx.Buf
bytes, c, err := d.decodeByte(buf, cursor)
if err != nil {
return 0, err

4
internal/decoder/int.go
View File

@ -209,8 +209,8 @@ func (d *intDecoder) DecodeStream(s *Stream, depth int64, p unsafe.Pointer) erro
return nil
}
func (d *intDecoder) Decode(buf []byte, cursor, depth int64, p unsafe.Pointer) (int64, error) {
bytes, c, err := d.decodeByte(buf, cursor)
func (d *intDecoder) Decode(ctx *RuntimeContext, cursor, depth int64, p unsafe.Pointer) (int64, error) {
bytes, c, err := d.decodeByte(ctx.Buf, cursor)
if err != nil {
return 0, err
}

33
internal/decoder/interface.go
View File

@ -176,9 +176,9 @@ func decodeTextUnmarshaler(buf []byte, cursor, depth int64, unmarshaler encoding
}
func (d *interfaceDecoder) decodeStreamEmptyInterface(s *Stream, depth int64, p unsafe.Pointer) error {
s.skipWhiteSpace()
c := s.skipWhiteSpace()
for {
switch s.char() {
switch c {
case '{':
var v map[string]interface{}
ptr := unsafe.Pointer(&v)
@ -203,7 +203,7 @@ func (d *interfaceDecoder) decodeStreamEmptyInterface(s *Stream, depth int64, p
for {
switch s.char() {
case '\\':
if err := decodeEscapeString(s); err != nil {
if _, err := decodeEscapeString(s, nil); err != nil {
return err
}
case '"':
@ -239,6 +239,7 @@ func (d *interfaceDecoder) decodeStreamEmptyInterface(s *Stream, depth int64, p
return nil
case nul:
if s.read() {
c = s.char()
continue
}
}
@ -265,8 +266,7 @@ func (d *interfaceDecoder) DecodeStream(s *Stream, depth int64, p unsafe.Pointer
if u, ok := rv.Interface().(encoding.TextUnmarshaler); ok {
return decodeStreamTextUnmarshaler(s, depth, u, p)
}
s.skipWhiteSpace()
if s.char() == 'n' {
if s.skipWhiteSpace() == 'n' {
if err := nullBytes(s); err != nil {
return err
}
@ -285,8 +285,7 @@ func (d *interfaceDecoder) DecodeStream(s *Stream, depth int64, p unsafe.Pointer
if typ.Kind() == reflect.Ptr && typ.Elem() == d.typ || typ.Kind() != reflect.Ptr {
return d.decodeStreamEmptyInterface(s, depth, p)
}
s.skipWhiteSpace()
if s.char() == 'n' {
if s.skipWhiteSpace() == 'n' {
if err := nullBytes(s); err != nil {
return err
}
@ -310,7 +309,8 @@ func (d *interfaceDecoder) errUnmarshalType(typ reflect.Type, offset int64) *err
}
}
func (d *interfaceDecoder) Decode(buf []byte, cursor, depth int64, p unsafe.Pointer) (int64, error) {
func (d *interfaceDecoder) Decode(ctx *RuntimeContext, cursor, depth int64, p unsafe.Pointer) (int64, error) {
buf := ctx.Buf
runtimeInterfaceValue := *(*interface{})(unsafe.Pointer(&emptyInterface{
typ: d.typ,
ptr: p,
@ -340,10 +340,10 @@ func (d *interfaceDecoder) Decode(buf []byte, cursor, depth int64, p unsafe.Poin
typ := ifaceHeader.typ
if ifaceHeader.ptr == nil || d.typ == typ || typ == nil {
// concrete type is empty interface
return d.decodeEmptyInterface(buf, cursor, depth, p)
return d.decodeEmptyInterface(ctx, cursor, depth, p)
}
if typ.Kind() == reflect.Ptr && typ.Elem() == d.typ || typ.Kind() != reflect.Ptr {
return d.decodeEmptyInterface(buf, cursor, depth, p)
return d.decodeEmptyInterface(ctx, cursor, depth, p)
}
cursor = skipWhiteSpace(buf, cursor)
if buf[cursor] == 'n' {
@ -358,16 +358,17 @@ func (d *interfaceDecoder) Decode(buf []byte, cursor, depth int64, p unsafe.Poin
if err != nil {
return 0, err
}
return decoder.Decode(buf, cursor, depth, ifaceHeader.ptr)
return decoder.Decode(ctx, cursor, depth, ifaceHeader.ptr)
}
func (d *interfaceDecoder) decodeEmptyInterface(buf []byte, cursor, depth int64, p unsafe.Pointer) (int64, error) {
func (d *interfaceDecoder) decodeEmptyInterface(ctx *RuntimeContext, cursor, depth int64, p unsafe.Pointer) (int64, error) {
buf := ctx.Buf
cursor = skipWhiteSpace(buf, cursor)
switch buf[cursor] {
case '{':
var v map[string]interface{}
ptr := unsafe.Pointer(&v)
cursor, err := d.mapDecoder.Decode(buf, cursor, depth, ptr)
cursor, err := d.mapDecoder.Decode(ctx, cursor, depth, ptr)
if err != nil {
return 0, err
}
@ -376,18 +377,18 @@ func (d *interfaceDecoder) decodeEmptyInterface(buf []byte, cursor, depth int64,
case '[':
var v []interface{}
ptr := unsafe.Pointer(&v)
cursor, err := d.sliceDecoder.Decode(buf, cursor, depth, ptr)
cursor, err := d.sliceDecoder.Decode(ctx, cursor, depth, ptr)
if err != nil {
return 0, err
}
**(**interface{})(unsafe.Pointer(&p)) = v
return cursor, nil
case '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
return d.floatDecoder.Decode(buf, cursor, depth, p)
return d.floatDecoder.Decode(ctx, cursor, depth, p)
case '"':
var v string
ptr := unsafe.Pointer(&v)
cursor, err := d.stringDecoder.Decode(buf, cursor, depth, ptr)
cursor, err := d.stringDecoder.Decode(ctx, cursor, depth, ptr)
if err != nil {
return 0, err
}

11
internal/decoder/map.go
View File

@ -42,8 +42,7 @@ func (d *mapDecoder) DecodeStream(s *Stream, depth int64, p unsafe.Pointer) erro
return errors.ErrExceededMaxDepth(s.char(), s.cursor)
}
s.skipWhiteSpace()
switch s.char() {
switch s.skipWhiteSpace() {
case 'n':
if err := nullBytes(s); err != nil {
return err
@ -54,7 +53,6 @@ func (d *mapDecoder) DecodeStream(s *Stream, depth int64, p unsafe.Pointer) erro
default:
return errors.ErrExpected("{ character for map value", s.totalOffset())
}
s.skipWhiteSpace()
mapValue := *(*unsafe.Pointer)(p)
if mapValue == nil {
mapValue = makemap(d.mapType, 0)
@ -92,7 +90,8 @@ func (d *mapDecoder) DecodeStream(s *Stream, depth int64, p unsafe.Pointer) erro
}
}
func (d *mapDecoder) Decode(buf []byte, cursor, depth int64, p unsafe.Pointer) (int64, error) {
func (d *mapDecoder) Decode(ctx *RuntimeContext, cursor, depth int64, p unsafe.Pointer) (int64, error) {
buf := ctx.Buf
depth++
if depth > maxDecodeNestingDepth {
return 0, errors.ErrExceededMaxDepth(buf[cursor], cursor)
@ -128,7 +127,7 @@ func (d *mapDecoder) Decode(buf []byte, cursor, depth int64, p unsafe.Pointer) (
}
for {
k := unsafe_New(d.keyType)
keyCursor, err := d.keyDecoder.Decode(buf, cursor, depth, k)
keyCursor, err := d.keyDecoder.Decode(ctx, cursor, depth, k)
if err != nil {
return 0, err
}
@ -138,7 +137,7 @@ func (d *mapDecoder) Decode(buf []byte, cursor, depth int64, p unsafe.Pointer) (
}
cursor++
v := unsafe_New(d.valueType)
valueCursor, err := d.valueDecoder.Decode(buf, cursor, depth, v)
valueCursor, err := d.valueDecoder.Decode(ctx, cursor, depth, v)
if err != nil {
return 0, err
}

4
internal/decoder/number.go
View File

@ -37,8 +37,8 @@ func (d *numberDecoder) DecodeStream(s *Stream, depth int64, p unsafe.Pointer) e
return nil
}
func (d *numberDecoder) Decode(buf []byte, cursor, depth int64, p unsafe.Pointer) (int64, error) {
bytes, c, err := d.decodeByte(buf, cursor)
func (d *numberDecoder) Decode(ctx *RuntimeContext, cursor, depth int64, p unsafe.Pointer) (int64, error) {
bytes, c, err := d.decodeByte(ctx.Buf, cursor)
if err != nil {
return 0, err
}

11
internal/decoder/option.go Normal file
View File

@ -0,0 +1,11 @@
package decoder
type OptionFlag int
const (
FirstWinOption OptionFlag = 1 << iota
)
type Option struct {
Flag OptionFlag
}

8
internal/decoder/ptr.go
View File

@ -35,8 +35,7 @@ func (d *ptrDecoder) contentDecoder() Decoder {
func unsafe_New(*runtime.Type) unsafe.Pointer
func (d *ptrDecoder) DecodeStream(s *Stream, depth int64, p unsafe.Pointer) error {
s.skipWhiteSpace()
if s.char() == nul {
if s.skipWhiteSpace() == nul {
s.read()
}
if s.char() == 'n' {
@ -59,7 +58,8 @@ func (d *ptrDecoder) DecodeStream(s *Stream, depth int64, p unsafe.Pointer) erro
return nil
}
func (d *ptrDecoder) Decode(buf []byte, cursor, depth int64, p unsafe.Pointer) (int64, error) {
func (d *ptrDecoder) Decode(ctx *RuntimeContext, cursor, depth int64, p unsafe.Pointer) (int64, error) {
buf := ctx.Buf
cursor = skipWhiteSpace(buf, cursor)
if buf[cursor] == 'n' {
if err := validateNull(buf, cursor); err != nil {
@ -78,7 +78,7 @@ func (d *ptrDecoder) Decode(buf []byte, cursor, depth int64, p unsafe.Pointer) (
} else {
newptr = *(*unsafe.Pointer)(p)
}
c, err := d.dec.Decode(buf, cursor, depth, newptr)
c, err := d.dec.Decode(ctx, cursor, depth, newptr)
if err != nil {
return 0, err
}

8
internal/decoder/slice.go
View File

@ -111,8 +111,7 @@ func (d *sliceDecoder) DecodeStream(s *Stream, depth int64, p unsafe.Pointer) er
return nil
case '[':
s.cursor++
s.skipWhiteSpace()
if s.char() == ']' {
if s.skipWhiteSpace() == ']' {
dst := (*sliceHeader)(p)
if dst.data == nil {
dst.data = newArray(d.elemType, 0)
@ -200,7 +199,8 @@ ERROR:
return errors.ErrUnexpectedEndOfJSON("slice", s.totalOffset())
}
func (d *sliceDecoder) Decode(buf []byte, cursor, depth int64, p unsafe.Pointer) (int64, error) {
func (d *sliceDecoder) Decode(ctx *RuntimeContext, cursor, depth int64, p unsafe.Pointer) (int64, error) {
buf := ctx.Buf
depth++
if depth > maxDecodeNestingDepth {
return 0, errors.ErrExceededMaxDepth(buf[cursor], cursor)
@ -254,7 +254,7 @@ func (d *sliceDecoder) Decode(buf []byte, cursor, depth int64, p unsafe.Pointer)
typedmemmove(d.elemType, ep, unsafe_New(d.elemType))
}
}
c, err := d.valueDecoder.Decode(buf, cursor, depth, ep)
c, err := d.valueDecoder.Decode(ctx, cursor, depth, ep)
if err != nil {
return 0, err
}

19
internal/decoder/stream.go
View File

@ -25,13 +25,15 @@ type Stream struct {
allRead bool
UseNumber bool
DisallowUnknownFields bool
Option *Option
}
func NewStream(r io.Reader) *Stream {
return &Stream{
r: r,
bufSize: initBufSize,
buf: []byte{nul},
buf: make([]byte, initBufSize),
Option: &Option{},
}
}
@ -90,6 +92,10 @@ func (s *Stream) stat() ([]byte, int64, unsafe.Pointer) {
return s.buf, s.cursor, (*sliceHeader)(unsafe.Pointer(&s.buf)).data
}
func (s *Stream) bufptr() unsafe.Pointer {
return (*sliceHeader)(unsafe.Pointer(&s.buf)).data
}
func (s *Stream) statForRetry() ([]byte, int64, unsafe.Pointer) {
s.cursor-- // for retry ( because caller progress cursor position in each loop )
return s.buf, s.cursor, (*sliceHeader)(unsafe.Pointer(&s.buf)).data
@ -194,6 +200,7 @@ func (s *Stream) readBuf() []byte {
}
remainNotNulCharNum++
}
s.length = s.cursor + remainNotNulCharNum
return s.buf[s.cursor+remainNotNulCharNum:]
}
@ -205,7 +212,7 @@ func (s *Stream) read() bool {
last := len(buf) - 1
buf[last] = nul
n, err := s.r.Read(buf[:last])
s.length = s.cursor + int64(n)
s.length += int64(n)
if n == last {
s.filledBuffer = true
} else {
@ -219,17 +226,21 @@ func (s *Stream) read() bool {
return true
}
func (s *Stream) skipWhiteSpace() {
func (s *Stream) skipWhiteSpace() byte {
p := s.bufptr()
LOOP:
switch s.char() {
c := char(p, s.cursor)
switch c {
case ' ', '\n', '\t', '\r':
s.cursor++
goto LOOP
case nul:
if s.read() {
p = s.bufptr()
goto LOOP
}
}
return c
}
func (s *Stream) skipObject(depth int64) error {

38
internal/decoder/string.go
View File

@ -45,8 +45,8 @@ func (d *stringDecoder) DecodeStream(s *Stream, depth int64, p unsafe.Pointer) e
return nil
}
func (d *stringDecoder) Decode(buf []byte, cursor, depth int64, p unsafe.Pointer) (int64, error) {
bytes, c, err := d.decodeByte(buf, cursor)
func (d *stringDecoder) Decode(ctx *RuntimeContext, cursor, depth int64, p unsafe.Pointer) (int64, error) {
bytes, c, err := d.decodeByte(ctx.Buf, cursor)
if err != nil {
return 0, err
}
@ -93,38 +93,40 @@ func unicodeToRune(code []byte) rune {
return r
}
func decodeUnicodeRune(s *Stream) (rune, int64, error) {
func decodeUnicodeRune(s *Stream, p unsafe.Pointer) (rune, int64, unsafe.Pointer, error) {
const defaultOffset = 5
const surrogateOffset = 11
if s.cursor+defaultOffset >= s.length {
if !s.read() {
return rune(0), 0, errors.ErrInvalidCharacter(s.char(), "escaped string", s.totalOffset())
return rune(0), 0, nil, errors.ErrInvalidCharacter(s.char(), "escaped string", s.totalOffset())
}
p = s.bufptr()
}
r := unicodeToRune(s.buf[s.cursor+1 : s.cursor+defaultOffset])
if utf16.IsSurrogate(r) {
if s.cursor+surrogateOffset >= s.length {
s.read()
p = s.bufptr()
}
if s.cursor+surrogateOffset >= s.length || s.buf[s.cursor+defaultOffset] != '\\' || s.buf[s.cursor+defaultOffset+1] != 'u' {
return unicode.ReplacementChar, defaultOffset, nil
return unicode.ReplacementChar, defaultOffset, p, nil
}
r2 := unicodeToRune(s.buf[s.cursor+defaultOffset+2 : s.cursor+surrogateOffset])
if r := utf16.DecodeRune(r, r2); r != unicode.ReplacementChar {
return r, surrogateOffset, nil
return r, surrogateOffset, p, nil
}
}
return r, defaultOffset, nil
return r, defaultOffset, p, nil
}
func decodeUnicode(s *Stream) error {
func decodeUnicode(s *Stream, p unsafe.Pointer) (unsafe.Pointer, error) {
const backSlashAndULen = 2 // length of \u
r, offset, err := decodeUnicodeRune(s)
r, offset, pp, err := decodeUnicodeRune(s, p)
if err != nil {
return err
return nil, err
}
unicode := []byte(string(r))
unicodeLen := int64(len(unicode))
@ -132,10 +134,10 @@ func decodeUnicode(s *Stream) error {
unicodeOrgLen := offset - 1
s.length = s.length - (backSlashAndULen + (unicodeOrgLen - unicodeLen))
s.cursor = s.cursor - backSlashAndULen + unicodeLen
return nil
return pp, nil
}
func decodeEscapeString(s *Stream) error {
func decodeEscapeString(s *Stream, p unsafe.Pointer) (unsafe.Pointer, error) {
s.cursor++
RETRY:
switch s.buf[s.cursor] {
@ -156,19 +158,19 @@ RETRY:
case 't':
s.buf[s.cursor] = '\t'
case 'u':
return decodeUnicode(s)
return decodeUnicode(s, p)
case nul:
if !s.read() {
return errors.ErrInvalidCharacter(s.char(), "escaped string", s.totalOffset())
return nil, errors.ErrInvalidCharacter(s.char(), "escaped string", s.totalOffset())
}
goto RETRY
default:
return errors.ErrUnexpectedEndOfJSON("string", s.totalOffset())
return nil, errors.ErrUnexpectedEndOfJSON("string", s.totalOffset())
}
s.buf = append(s.buf[:s.cursor-1], s.buf[s.cursor:]...)
s.length--
s.cursor--
return nil
return p, nil
}
var (
@ -184,9 +186,11 @@ func stringBytes(s *Stream) ([]byte, error) {
switch char(p, cursor) {
case '\\':
s.cursor = cursor
if err := decodeEscapeString(s); err != nil {
pp, err := decodeEscapeString(s, p)
if err != nil {
return nil, err
}
p = pp
cursor = s.cursor
case '"':
literal := s.buf[start:cursor]

126
internal/decoder/struct.go
View File

@ -17,22 +17,24 @@ type structFieldSet struct {
dec Decoder
offset uintptr
isTaggedKey bool
fieldIdx int
key string
keyLen int64
err error
}
type structDecoder struct {
fieldMap map[string]*structFieldSet
stringDecoder *stringDecoder
structName string
fieldName string
isTriedOptimize bool
keyBitmapUint8 [][256]uint8
keyBitmapUint16 [][256]uint16
sortedFieldSets []*structFieldSet
keyDecoder func(*structDecoder, []byte, int64) (int64, *structFieldSet, error)
keyStreamDecoder func(*structDecoder, *Stream) (*structFieldSet, string, error)
fieldMap map[string]*structFieldSet
fieldUniqueNameNum int
stringDecoder *stringDecoder
structName string
fieldName string
isTriedOptimize bool
keyBitmapUint8 [][256]uint8
keyBitmapUint16 [][256]uint16
sortedFieldSets []*structFieldSet
keyDecoder func(*structDecoder, []byte, int64) (int64, *structFieldSet, error)
keyStreamDecoder func(*structDecoder, *Stream) (*structFieldSet, string, error)
}
var (
@ -66,6 +68,21 @@ const (
)
func (d *structDecoder) tryOptimize() {
fieldUniqueNameMap := map[string]int{}
fieldIdx := -1
for k, v := range d.fieldMap {
lower := strings.ToLower(k)
idx, exists := fieldUniqueNameMap[lower]
if exists {
v.fieldIdx = idx
} else {
fieldIdx++
v.fieldIdx = fieldIdx
}
fieldUniqueNameMap[lower] = fieldIdx
}
d.fieldUniqueNameNum = len(fieldUniqueNameMap)
if d.isTriedOptimize {
return
}
@ -627,48 +644,64 @@ func (d *structDecoder) DecodeStream(s *Stream, depth int64, p unsafe.Pointer) e
return errors.ErrExceededMaxDepth(s.char(), s.cursor)
}
s.skipWhiteSpace()
switch s.char() {
c := s.skipWhiteSpace()
switch c {
case 'n':
if err := nullBytes(s); err != nil {
return err
}
return nil
case nul:
s.read()
default:
if s.char() != '{' {
return errors.ErrNotAtBeginningOfValue(s.totalOffset())
}
}
s.cursor++
s.skipWhiteSpace()
if s.char() == '}' {
if s.skipWhiteSpace() == '}' {
s.cursor++
return nil
}
var (
seenFields map[int]struct{}
seenFieldNum int
)
firstWin := (s.Option.Flag & FirstWinOption) != 0
if firstWin {
seenFields = make(map[int]struct{}, d.fieldUniqueNameNum)
}
for {
s.reset()
field, key, err := d.keyStreamDecoder(d, s)
if err != nil {
return err
}
s.skipWhiteSpace()
if s.char() != ':' {
if s.skipWhiteSpace() != ':' {
return errors.ErrExpected("colon after object key", s.totalOffset())
}
s.cursor++
if s.char() == nul {
if !s.read() {
return errors.ErrExpected("object value after colon", s.totalOffset())
}
}
if field != nil {
if field.err != nil {
return field.err
}
if err := field.dec.DecodeStream(s, depth, unsafe.Pointer(uintptr(p)+field.offset)); err != nil {
return err
if firstWin {
if _, exists := seenFields[field.fieldIdx]; exists {
if err := s.skipValue(depth); err != nil {
return err
}
} else {
if err := field.dec.DecodeStream(s, depth, unsafe.Pointer(uintptr(p)+field.offset)); err != nil {
return err
}
seenFieldNum++
if d.fieldUniqueNameNum <= seenFieldNum {
return s.skipObject(depth)
}
seenFields[field.fieldIdx] = struct{}{}
}
} else {
if err := field.dec.DecodeStream(s, depth, unsafe.Pointer(uintptr(p)+field.offset)); err != nil {
return err
}
}
} else if s.DisallowUnknownFields {
return fmt.Errorf("json: unknown field %q", key)
@ -677,8 +710,7 @@ func (d *structDecoder) DecodeStream(s *Stream, depth int64, p unsafe.Pointer) e
return err
}
}
s.skipWhiteSpace()
c := s.char()
c := s.skipWhiteSpace()
if c == '}' {
s.cursor++
return nil
@ -690,7 +722,8 @@ func (d *structDecoder) DecodeStream(s *Stream, depth int64, p unsafe.Pointer) e
}
}
func (d *structDecoder) Decode(buf []byte, cursor, depth int64, p unsafe.Pointer) (int64, error) {
func (d *structDecoder) Decode(ctx *RuntimeContext, cursor, depth int64, p unsafe.Pointer) (int64, error) {
buf := ctx.Buf
depth++
if depth > maxDecodeNestingDepth {
return 0, errors.ErrExceededMaxDepth(buf[cursor], cursor)
@ -715,6 +748,14 @@ func (d *structDecoder) Decode(buf []byte, cursor, depth int64, p unsafe.Pointer
cursor++
return cursor, nil
}
var (
seenFields map[int]struct{}
seenFieldNum int
)
firstWin := (ctx.Option.Flag & FirstWinOption) != 0
if firstWin {
seenFields = make(map[int]struct{}, d.fieldUniqueNameNum)
}
for {
c, field, err := d.keyDecoder(d, buf, cursor)
if err != nil {
@ -732,11 +773,32 @@ func (d *structDecoder) Decode(buf []byte, cursor, depth int64, p unsafe.Pointer
if field.err != nil {
return 0, field.err
}
c, err := field.dec.Decode(buf, cursor, depth, unsafe.Pointer(uintptr(p)+field.offset))
if err != nil {
return 0, err
if firstWin {
if _, exists := seenFields[field.fieldIdx]; exists {
c, err := skipValue(buf, cursor, depth)
if err != nil {
return 0, err
}
cursor = c
} else {
c, err := field.dec.Decode(ctx, cursor, depth, unsafe.Pointer(uintptr(p)+field.offset))
if err != nil {
return 0, err
}
cursor = c
seenFieldNum++
if d.fieldUniqueNameNum <= seenFieldNum {
return skipObject(buf, cursor, depth)
}
seenFields[field.fieldIdx] = struct{}{}
}
} else {
c, err := field.dec.Decode(ctx, cursor, depth, unsafe.Pointer(uintptr(p)+field.offset))
if err != nil {
return 0, err
}
cursor = c
}
cursor = c
} else {
c, err := skipValue(buf, cursor, depth)
if err != nil {

2
internal/decoder/type.go
View File

@ -8,7 +8,7 @@ import (
)
type Decoder interface {
Decode([]byte, int64, int64, unsafe.Pointer) (int64, error)
Decode(*RuntimeContext, int64, int64, unsafe.Pointer) (int64, error)
DecodeStream(*Stream, int64, unsafe.Pointer) error
}

4
internal/decoder/uint.go
View File

@ -158,8 +158,8 @@ func (d *uintDecoder) DecodeStream(s *Stream, depth int64, p unsafe.Pointer) err
return nil
}
func (d *uintDecoder) Decode(buf []byte, cursor, depth int64, p unsafe.Pointer) (int64, error) {
bytes, c, err := d.decodeByte(buf, cursor)
func (d *uintDecoder) Decode(ctx *RuntimeContext, cursor, depth int64, p unsafe.Pointer) (int64, error) {
bytes, c, err := d.decodeByte(ctx.Buf, cursor)
if err != nil {
return 0, err
}

3
internal/decoder/unmarshal_json.go
View File

@ -53,7 +53,8 @@ func (d *unmarshalJSONDecoder) DecodeStream(s *Stream, depth int64, p unsafe.Poi
return nil
}
func (d *unmarshalJSONDecoder) Decode(buf []byte, cursor, depth int64, p unsafe.Pointer) (int64, error) {
func (d *unmarshalJSONDecoder) Decode(ctx *RuntimeContext, cursor, depth int64, p unsafe.Pointer) (int64, error) {
buf := ctx.Buf
cursor = skipWhiteSpace(buf, cursor)
start := cursor
end, err := skipValue(buf, cursor, depth)

3
internal/decoder/unmarshal_text.go
View File

@ -91,7 +91,8 @@ func (d *unmarshalTextDecoder) DecodeStream(s *Stream, depth int64, p unsafe.Poi
return nil
}
func (d *unmarshalTextDecoder) Decode(buf []byte, cursor, depth int64, p unsafe.Pointer) (int64, error) {
func (d *unmarshalTextDecoder) Decode(ctx *RuntimeContext, cursor, depth int64, p unsafe.Pointer) (int64, error) {
buf := ctx.Buf
cursor = skipWhiteSpace(buf, cursor)
start := cursor
end, err := skipValue(buf, cursor, depth)

11
internal/decoder/wrapped_string.go
View File

@ -40,14 +40,14 @@ func (d *wrappedStringDecoder) DecodeStream(s *Stream, depth int64, p unsafe.Poi
}
b := make([]byte, len(bytes)+1)
copy(b, bytes)
if _, err := d.dec.Decode(b, 0, depth, p); err != nil {
if _, err := d.dec.Decode(&RuntimeContext{Buf: b}, 0, depth, p); err != nil {
return err
}
return nil
}
func (d *wrappedStringDecoder) Decode(buf []byte, cursor, depth int64, p unsafe.Pointer) (int64, error) {
bytes, c, err := d.stringDecoder.decodeByte(buf, cursor)
func (d *wrappedStringDecoder) Decode(ctx *RuntimeContext, cursor, depth int64, p unsafe.Pointer) (int64, error) {
bytes, c, err := d.stringDecoder.decodeByte(ctx.Buf, cursor)
if err != nil {
return 0, err
}
@ -58,8 +58,11 @@ func (d *wrappedStringDecoder) Decode(buf []byte, cursor, depth int64, p unsafe.
return c, nil
}
bytes = append(bytes, nul)
if _, err := d.dec.Decode(bytes, 0, depth, p); err != nil {
oldBuf := ctx.Buf
ctx.Buf = bytes
if _, err := d.dec.Decode(ctx, 0, depth, p); err != nil {
return 0, err
}
ctx.Buf = oldBuf
return c, nil
}

8
json.go
View File

@ -258,8 +258,12 @@ func Unmarshal(data []byte, v interface{}) error {
return unmarshal(data, v)
}
func UnmarshalNoEscape(data []byte, v interface{}) error {
return unmarshalNoEscape(data, v)
func UnmarshalWithOption(data []byte, v interface{}, optFuncs ...DecodeOptionFunc) error {
return unmarshal(data, v, optFuncs...)
}
func UnmarshalNoEscape(data []byte, v interface{}, optFuncs ...DecodeOptionFunc) error {
return unmarshalNoEscape(data, v, optFuncs...)
}
// A Token holds a value of one of these types:

20
option.go
View File

@ -1,28 +1,46 @@
package json
import (
"github.com/goccy/go-json/internal/decoder"
"github.com/goccy/go-json/internal/encoder"
)
type EncodeOption = encoder.Option
type EncodeOptionFunc func(*EncodeOption)
// UnorderedMap doesn't sort when encoding map type.
func UnorderedMap() EncodeOptionFunc {
return func(opt *EncodeOption) {
opt.Flag |= encoder.UnorderedMapOption
}
}
// Debug outputs debug information when panic occurs during encoding.
func Debug() EncodeOptionFunc {
return func(opt *EncodeOption) {
opt.Flag |= encoder.DebugOption
}
}
// Colorize add an identifier for coloring to the string of the encoded result.
func Colorize(scheme *ColorScheme) EncodeOptionFunc {
return func(opt *EncodeOption) {
opt.Flag |= encoder.ColorizeOption
opt.ColorScheme = scheme
}
}
type DecodeOption = decoder.Option
type DecodeOptionFunc func(*DecodeOption)
// DecodeFieldPriorityFirstWin
// in the default behavior, go-json, like encoding/json,
// will reflect the result of the last evaluation when a field with the same name exists.
// This option allow you to change this behavior.
// this option reflects the result of the first evaluation if a field with the same name exists.
// This behavior has a performance advantage as it allows the subsequent strings to be skipped if all fields have been evaluated.
func DecodeFieldPriorityFirstWin() DecodeOptionFunc {
return func(opt *DecodeOption) {
opt.Flag |= decoder.FirstWinOption
}
}