From 7ae14cd24c4ebdf7649910654580015ed51625ef Mon Sep 17 00:00:00 2001 From: Masaaki Goshima Date: Tue, 20 Apr 2021 20:12:32 +0900 Subject: [PATCH] Optimize json.Compact and json.Indent --- encode.go | 47 +++++------------- internal/encoder/compact.go | 19 ++++++-- internal/encoder/context.go | 53 ++++++++++++++++++++ internal/encoder/encoder.go | 37 ++++---------- internal/encoder/indent.go | 56 +++++++++++++++++----- internal/encoder/vm_escaped_indent/util.go | 13 ----- internal/encoder/vm_escaped_indent/vm.go | 9 ++-- internal/encoder/vm_indent/util.go | 13 ----- internal/encoder/vm_indent/vm.go | 9 ++-- 9 files changed, 142 insertions(+), 114 deletions(-) diff --git a/encode.go b/encode.go index f5c315c..e21a38e 100644 --- a/encode.go +++ b/encode.go @@ -2,7 +2,6 @@ package json import ( "io" - "sync" "unsafe" "github.com/goccy/go-json/internal/encoder" @@ -22,10 +21,6 @@ type Encoder struct { indentStr string } -const ( - bufSize = 1024 -) - type EncodeOption int const ( @@ -35,26 +30,6 @@ const ( EncodeOptionDebug ) -var ( - encRuntimeContextPool = sync.Pool{ - New: func() interface{} { - return &encoder.RuntimeContext{ - Buf: make([]byte, 0, bufSize), - Ptrs: make([]uintptr, 128), - KeepRefs: make([]unsafe.Pointer, 0, 8), - } - }, - } -) - -func takeEncodeRuntimeContext() *encoder.RuntimeContext { - return encRuntimeContextPool.Get().(*encoder.RuntimeContext) -} - -func releaseEncodeRuntimeContext(ctx *encoder.RuntimeContext) { - encRuntimeContextPool.Put(ctx) -} - // NewEncoder returns a new encoder that writes to w. func NewEncoder(w io.Writer) *Encoder { return &Encoder{w: w, enabledHTMLEscape: true} @@ -69,11 +44,11 @@ func (e *Encoder) Encode(v interface{}) error { // EncodeWithOption call Encode with EncodeOption. func (e *Encoder) EncodeWithOption(v interface{}, optFuncs ...EncodeOptionFunc) error { - ctx := takeEncodeRuntimeContext() + ctx := encoder.TakeRuntimeContext() err := e.encodeWithOption(ctx, v, optFuncs...) - releaseEncodeRuntimeContext(ctx) + encoder.ReleaseRuntimeContext(ctx) return err } @@ -130,11 +105,11 @@ func (e *Encoder) SetIndent(prefix, indent string) { } func marshal(v interface{}, opt EncodeOption) ([]byte, error) { - ctx := takeEncodeRuntimeContext() + ctx := encoder.TakeRuntimeContext() buf, err := encode(ctx, v, opt|EncodeOptionHTMLEscape) if err != nil { - releaseEncodeRuntimeContext(ctx) + encoder.ReleaseRuntimeContext(ctx) return nil, err } @@ -146,16 +121,16 @@ func marshal(v interface{}, opt EncodeOption) ([]byte, error) { copied := make([]byte, len(buf)) copy(copied, buf) - releaseEncodeRuntimeContext(ctx) + encoder.ReleaseRuntimeContext(ctx) return copied, nil } func marshalNoEscape(v interface{}, opt EncodeOption) ([]byte, error) { - ctx := takeEncodeRuntimeContext() + ctx := encoder.TakeRuntimeContext() buf, err := encodeNoEscape(ctx, v, opt|EncodeOptionHTMLEscape) if err != nil { - releaseEncodeRuntimeContext(ctx) + encoder.ReleaseRuntimeContext(ctx) return nil, err } @@ -167,16 +142,16 @@ func marshalNoEscape(v interface{}, opt EncodeOption) ([]byte, error) { copied := make([]byte, len(buf)) copy(copied, buf) - releaseEncodeRuntimeContext(ctx) + encoder.ReleaseRuntimeContext(ctx) return copied, nil } func marshalIndent(v interface{}, prefix, indent string, opt EncodeOption) ([]byte, error) { - ctx := takeEncodeRuntimeContext() + ctx := encoder.TakeRuntimeContext() buf, err := encodeIndent(ctx, v, prefix, indent, opt|EncodeOptionHTMLEscape) if err != nil { - releaseEncodeRuntimeContext(ctx) + encoder.ReleaseRuntimeContext(ctx) return nil, err } @@ -184,7 +159,7 @@ func marshalIndent(v interface{}, prefix, indent string, opt EncodeOption) ([]by copied := make([]byte, len(buf)) copy(copied, buf) - releaseEncodeRuntimeContext(ctx) + encoder.ReleaseRuntimeContext(ctx) return copied, nil } diff --git a/internal/encoder/compact.go b/internal/encoder/compact.go index 4f1bd79..0eb9545 100644 --- a/internal/encoder/compact.go +++ b/internal/encoder/compact.go @@ -30,9 +30,22 @@ func Compact(buf *bytes.Buffer, src []byte, escape bool) error { } buf.Grow(len(src)) dst := buf.Bytes() - newSrc := make([]byte, len(src)+1) // append nul byte to the end - copy(newSrc, src) - dst, err := compact(dst, newSrc, escape) + + ctx := TakeRuntimeContext() + ctxBuf := ctx.Buf[:0] + ctxBuf = append(append(ctxBuf, src...), nul) + ctx.Buf = ctxBuf + + if err := compactAndWrite(buf, dst, ctxBuf, escape); err != nil { + ReleaseRuntimeContext(ctx) + return err + } + ReleaseRuntimeContext(ctx) + return nil +} + +func compactAndWrite(buf *bytes.Buffer, dst []byte, src []byte, escape bool) error { + dst, err := compact(dst, src, escape) if err != nil { return err } diff --git a/internal/encoder/context.go b/internal/encoder/context.go index d21c008..d7e030e 100644 --- a/internal/encoder/context.go +++ b/internal/encoder/context.go @@ -1,6 +1,9 @@ package encoder import ( + "sync" + "unsafe" + "github.com/goccy/go-json/internal/runtime" ) @@ -80,3 +83,53 @@ func (c *compileContext) decPtrIndex() { c.parent.decPtrIndex() } } + +const ( + bufSize = 1024 +) + +var ( + runtimeContextPool = sync.Pool{ + New: func() interface{} { + return &RuntimeContext{ + Buf: make([]byte, 0, bufSize), + Ptrs: make([]uintptr, 128), + KeepRefs: make([]unsafe.Pointer, 0, 8), + } + }, + } +) + +type RuntimeContext struct { + Buf []byte + MarshalBuf []byte + Ptrs []uintptr + KeepRefs []unsafe.Pointer + SeenPtr []uintptr + BaseIndent int + Prefix []byte + IndentStr []byte +} + +func (c *RuntimeContext) Init(p uintptr, codelen int) { + if len(c.Ptrs) < codelen { + c.Ptrs = make([]uintptr, codelen) + } + c.Ptrs[0] = p + c.KeepRefs = c.KeepRefs[:0] + c.SeenPtr = c.SeenPtr[:0] + c.BaseIndent = 0 +} + +func (c *RuntimeContext) Ptr() uintptr { + header := (*runtime.SliceHeader)(unsafe.Pointer(&c.Ptrs)) + return uintptr(header.Data) +} + +func TakeRuntimeContext() *RuntimeContext { + return runtimeContextPool.Get().(*RuntimeContext) +} + +func ReleaseRuntimeContext(ctx *RuntimeContext) { + runtimeContextPool.Put(ctx) +} diff --git a/internal/encoder/encoder.go b/internal/encoder/encoder.go index 44adee0..7f6c294 100644 --- a/internal/encoder/encoder.go +++ b/internal/encoder/encoder.go @@ -304,32 +304,6 @@ func MapIterNext(it unsafe.Pointer) //go:noescape func MapLen(m unsafe.Pointer) int -type RuntimeContext struct { - Buf []byte - MarshalBuf []byte - Ptrs []uintptr - KeepRefs []unsafe.Pointer - SeenPtr []uintptr - BaseIndent int - Prefix []byte - IndentStr []byte -} - -func (c *RuntimeContext) Init(p uintptr, codelen int) { - if len(c.Ptrs) < codelen { - c.Ptrs = make([]uintptr, codelen) - } - c.Ptrs[0] = p - c.KeepRefs = c.KeepRefs[:0] - c.SeenPtr = c.SeenPtr[:0] - c.BaseIndent = 0 -} - -func (c *RuntimeContext) Ptr() uintptr { - header := (*runtime.SliceHeader)(unsafe.Pointer(&c.Ptrs)) - return uintptr(header.Data) -} - func AppendByteSlice(b []byte, src []byte) []byte { if src == nil { return append(b, `null`...) @@ -551,13 +525,20 @@ func AppendStructEnd(b []byte) []byte { func AppendStructEndIndent(ctx *RuntimeContext, b []byte, indent int) []byte { b = append(b, '\n') b = append(b, ctx.Prefix...) - b = append(b, bytes.Repeat(ctx.IndentStr, ctx.BaseIndent+indent)...) + indentNum := ctx.BaseIndent + indent + for i := 0; i < indentNum; i++ { + b = append(b, ctx.IndentStr...) + } return append(b, '}', ',', '\n') } func AppendIndent(ctx *RuntimeContext, b []byte, indent int) []byte { b = append(b, ctx.Prefix...) - return append(b, bytes.Repeat(ctx.IndentStr, ctx.BaseIndent+indent)...) + indentNum := ctx.BaseIndent + indent + for i := 0; i < indentNum; i++ { + b = append(b, ctx.IndentStr...) + } + return b } func IsNilForMarshaler(v interface{}) bool { diff --git a/internal/encoder/indent.go b/internal/encoder/indent.go index d593301..dfe04b5 100644 --- a/internal/encoder/indent.go +++ b/internal/encoder/indent.go @@ -7,22 +7,44 @@ import ( "github.com/goccy/go-json/internal/errors" ) +func takeIndentSrcRuntimeContext(src []byte) (*RuntimeContext, []byte) { + ctx := TakeRuntimeContext() + buf := ctx.Buf[:0] + buf = append(append(buf, src...), nul) + ctx.Buf = buf + return ctx, buf +} + func Indent(buf *bytes.Buffer, src []byte, prefix, indentStr string) error { if len(src) == 0 { return errors.ErrUnexpectedEndOfJSON("", 0) } - buf.Grow(len(src)) - dst := buf.Bytes() - newSrc := make([]byte, len(src)+1) // append nul byte to the end - copy(newSrc, src) - dst, err := doIndent(dst, newSrc, prefix, indentStr, false) + + srcCtx, srcBuf := takeIndentSrcRuntimeContext(src) + dstCtx := TakeRuntimeContext() + dst := dstCtx.Buf[:0] + + dst, err := indentAndWrite(buf, dst, srcBuf, prefix, indentStr) if err != nil { + ReleaseRuntimeContext(srcCtx) + ReleaseRuntimeContext(dstCtx) return err } + dstCtx.Buf = dst + ReleaseRuntimeContext(srcCtx) + ReleaseRuntimeContext(dstCtx) + return nil +} + +func indentAndWrite(buf *bytes.Buffer, dst []byte, src []byte, prefix, indentStr string) ([]byte, error) { + dst, err := doIndent(dst, src, prefix, indentStr, false) + if err != nil { + return nil, err + } if _, err := buf.Write(dst); err != nil { - return err + return nil, err } - return nil + return dst, nil } func doIndent(dst, src []byte, prefix, indentStr string, escape bool) ([]byte, error) { @@ -94,7 +116,10 @@ func indentObject( indentNum++ var err error for { - dst = append(append(append(dst, '\n'), prefix...), bytes.Repeat(indentBytes, indentNum)...) + dst = append(append(dst, '\n'), prefix...) + for i := 0; i < indentNum; i++ { + dst = append(dst, indentBytes...) + } cursor = skipWhiteSpace(src, cursor) dst, cursor, err = compactString(dst, src, cursor, escape) if err != nil { @@ -115,7 +140,10 @@ func indentObject( cursor = skipWhiteSpace(src, cursor) switch src[cursor] { case '}': - dst = append(append(append(dst, '\n'), prefix...), bytes.Repeat(indentBytes, indentNum-1)...) + dst = append(append(dst, '\n'), prefix...) + for i := 0; i < indentNum-1; i++ { + dst = append(dst, indentBytes...) + } dst = append(dst, '}') cursor++ return dst, cursor, nil @@ -152,7 +180,10 @@ func indentArray( indentNum++ var err error for { - dst = append(append(append(dst, '\n'), prefix...), bytes.Repeat(indentBytes, indentNum)...) + dst = append(append(dst, '\n'), prefix...) + for i := 0; i < indentNum; i++ { + dst = append(dst, indentBytes...) + } dst, cursor, err = indentValue(dst, src, indentNum, cursor, prefix, indentBytes, escape) if err != nil { return nil, 0, err @@ -160,7 +191,10 @@ func indentArray( cursor = skipWhiteSpace(src, cursor) switch src[cursor] { case ']': - dst = append(append(append(dst, '\n'), prefix...), bytes.Repeat(indentBytes, indentNum-1)...) + dst = append(append(dst, '\n'), prefix...) + for i := 0; i < indentNum-1; i++ { + dst = append(dst, indentBytes...) + } dst = append(dst, ']') cursor++ return dst, cursor, nil diff --git a/internal/encoder/vm_escaped_indent/util.go b/internal/encoder/vm_escaped_indent/util.go index 383105b..e238cf6 100644 --- a/internal/encoder/vm_escaped_indent/util.go +++ b/internal/encoder/vm_escaped_indent/util.go @@ -1,7 +1,6 @@ package vm_escaped_indent import ( - "bytes" "encoding/json" "unsafe" @@ -76,15 +75,3 @@ func appendNull(b []byte) []byte { func appendComma(b []byte) []byte { return append(b, ',', '\n') } - -func appendStructEnd(ctx *encoder.RuntimeContext, b []byte, indent int) []byte { - b = append(b, '\n') - b = append(b, ctx.Prefix...) - b = append(b, bytes.Repeat(ctx.IndentStr, ctx.BaseIndent+indent)...) - return append(b, '}', ',', '\n') -} - -func appendIndent(ctx *encoder.RuntimeContext, b []byte, indent int) []byte { - b = append(b, ctx.Prefix...) - return append(b, bytes.Repeat(ctx.IndentStr, ctx.BaseIndent+indent)...) -} diff --git a/internal/encoder/vm_escaped_indent/vm.go b/internal/encoder/vm_escaped_indent/vm.go index bcddbc1..aee8f5f 100644 --- a/internal/encoder/vm_escaped_indent/vm.go +++ b/internal/encoder/vm_escaped_indent/vm.go @@ -1,7 +1,6 @@ package vm_escaped_indent import ( - "bytes" "fmt" "math" "sort" @@ -29,6 +28,8 @@ var ( appendNumber = encoder.AppendNumber appendMarshalJSON = encoder.AppendMarshalJSONIndent appendMarshalText = encoder.AppendMarshalTextIndent + appendStructEnd = encoder.AppendStructEndIndent + appendIndent = encoder.AppendIndent errUnsupportedValue = encoder.ErrUnsupportedValue errUnsupportedFloat = encoder.ErrUnsupportedFloat mapiterinit = encoder.MapIterInit @@ -518,8 +519,7 @@ func Run(ctx *encoder.RuntimeContext, b []byte, codeSet *encoder.OpcodeSet, opt sort.Sort(mapCtx.Slice) buf := mapCtx.Buf for _, item := range mapCtx.Slice.Items { - buf = append(buf, ctx.Prefix...) - buf = append(buf, bytes.Repeat(ctx.IndentStr, ctx.BaseIndent+code.Indent+1)...) + buf = appendIndent(ctx, buf, code.Indent+1) buf = append(buf, item.Key...) buf[len(buf)-2] = ':' buf[len(buf)-1] = ' ' @@ -527,8 +527,7 @@ func Run(ctx *encoder.RuntimeContext, b []byte, codeSet *encoder.OpcodeSet, opt } buf = buf[:len(buf)-2] buf = append(buf, '\n') - buf = append(buf, ctx.Prefix...) - buf = append(buf, bytes.Repeat(ctx.IndentStr, ctx.BaseIndent+code.Indent)...) + buf = appendIndent(ctx, buf, code.Indent) buf = append(buf, '}', ',', '\n') b = b[:pos[0]] diff --git a/internal/encoder/vm_indent/util.go b/internal/encoder/vm_indent/util.go index cbcf54f..e678217 100644 --- a/internal/encoder/vm_indent/util.go +++ b/internal/encoder/vm_indent/util.go @@ -1,7 +1,6 @@ package vm_indent import ( - "bytes" "encoding/json" "unsafe" @@ -76,15 +75,3 @@ func appendNull(b []byte) []byte { func appendComma(b []byte) []byte { return append(b, ',', '\n') } - -func appendStructEnd(ctx *encoder.RuntimeContext, b []byte, indent int) []byte { - b = append(b, '\n') - b = append(b, ctx.Prefix...) - b = append(b, bytes.Repeat(ctx.IndentStr, ctx.BaseIndent+indent)...) - return append(b, '}', ',', '\n') -} - -func appendIndent(ctx *encoder.RuntimeContext, b []byte, indent int) []byte { - b = append(b, ctx.Prefix...) - return append(b, bytes.Repeat(ctx.IndentStr, ctx.BaseIndent+indent)...) -} diff --git a/internal/encoder/vm_indent/vm.go b/internal/encoder/vm_indent/vm.go index eddffcc..fc27472 100644 --- a/internal/encoder/vm_indent/vm.go +++ b/internal/encoder/vm_indent/vm.go @@ -1,7 +1,6 @@ package vm_indent import ( - "bytes" "fmt" "math" "sort" @@ -29,6 +28,8 @@ var ( appendNumber = encoder.AppendNumber appendMarshalJSON = encoder.AppendMarshalJSONIndent appendMarshalText = encoder.AppendMarshalTextIndent + appendStructEnd = encoder.AppendStructEndIndent + appendIndent = encoder.AppendIndent errUnsupportedValue = encoder.ErrUnsupportedValue errUnsupportedFloat = encoder.ErrUnsupportedFloat mapiterinit = encoder.MapIterInit @@ -518,8 +519,7 @@ func Run(ctx *encoder.RuntimeContext, b []byte, codeSet *encoder.OpcodeSet, opt sort.Sort(mapCtx.Slice) buf := mapCtx.Buf for _, item := range mapCtx.Slice.Items { - buf = append(buf, ctx.Prefix...) - buf = append(buf, bytes.Repeat(ctx.IndentStr, ctx.BaseIndent+code.Indent+1)...) + buf = appendIndent(ctx, buf, code.Indent+1) buf = append(buf, item.Key...) buf[len(buf)-2] = ':' buf[len(buf)-1] = ' ' @@ -527,8 +527,7 @@ func Run(ctx *encoder.RuntimeContext, b []byte, codeSet *encoder.OpcodeSet, opt } buf = buf[:len(buf)-2] buf = append(buf, '\n') - buf = append(buf, ctx.Prefix...) - buf = append(buf, bytes.Repeat(ctx.IndentStr, ctx.BaseIndent+code.Indent)...) + buf = appendIndent(ctx, buf, code.Indent) buf = append(buf, '}', ',', '\n') b = b[:pos[0]]