Merge branch 'master' into feat-case-271

CHANGELOG.md

@@ -1,3 +1,32 @@
+# v0.9.10 - 2022/07/15
+
+### Fix bugs
+
+* Fix boundary exception of type caching ( #382 )
+
+# v0.9.9 - 2022/07/15
+
+### Fix bugs
+
+* Fix encoding of directed interface with typed nil ( #377 )
+* Fix embedded primitive type encoding using alias ( #378 )
+* Fix slice/array type encoding with types implementing MarshalJSON ( #379 )
+* Fix unicode decoding when the expected buffer state is not met after reading ( #380 )
+
+# v0.9.8 - 2022/06/30
+
+### Fix bugs
+
+* Fix decoding of surrogate-pair ( #365 )
+* Fix handling of embedded primitive type ( #366 )
+* Add validation of escape sequence for decoder ( #367 )
+* Fix stream tokenizing respecting UseNumber ( #369 )
+* Fix encoding when struct pointer type that implements Marshal JSON is embedded ( #375 )
+
+### Improve performance
+
+* Improve performance of linkRecursiveCode ( #368 )
+
 # v0.9.7 - 2022/04/22
 
 ### Fix bugs

decode_test.go

@@ -3968,3 +3968,20 @@ func TestIssue335(t *testing.T) {
 		t.Errorf("unexpected success")
 	}
 }
+
+func TestIssue372(t *testing.T) {
+	type A int
+	type T struct {
+		_ int
+		*A
+	}
+	var v T
+	err := json.Unmarshal([]byte(`{"A":7}`), &v)
+	assertErr(t, err)
+
+	got := *v.A
+	expected := A(7)
+	if got != expected {
+		t.Errorf("unexpected result: %v != %v", got, expected)
+	}
+}

encode_test.go

@@ -7,11 +7,14 @@ import (
 	stdjson "encoding/json"
 	"errors"
 	"fmt"
+	"io"
 	"log"
 	"math"
+	"math/big"
 	"reflect"
 	"regexp"
 	"strconv"
+	"strings"
 	"testing"
 	"time"
 
@@ -2412,5 +2415,203 @@ func TestCamelCaseField(t *testing.T) {
 
 	if !bytes.Equal(expected, got) {
 		t.Fatalf("failed to encode. expected %q but got %q", expected, got)
+
+func TestIssue339(t *testing.T) {
+	type T1 struct {
+		*big.Int
+	}
+	type T2 struct {
+		T1 T1 `json:"T1"`
+	}
+	v := T2{T1{Int: big.NewInt(10000)}}
+	b, err := json.Marshal(&v)
+	assertErr(t, err)
+	got := string(b)
+	expected := `{"T1":10000}`
+	if got != expected {
+		t.Errorf("unexpected result: %v != %v", got, expected)
+	}
+}
+
+func TestIssue376(t *testing.T) {
+	type Container struct {
+		V interface{} `json:"value"`
+	}
+	type MapOnly struct {
+		Map map[string]int64 `json:"map"`
+	}
+	b, err := json.Marshal(Container{MapOnly{}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	got := string(b)
+	expected := `{"value":{"map":null}}`
+	if got != expected {
+		t.Errorf("unexpected result: %v != %v", got, expected)
+	}
+}
+
+type Issue370 struct {
+	String string
+	Valid  bool
+}
+
+func (i *Issue370) MarshalJSON() ([]byte, error) {
+	if !i.Valid {
+		return json.Marshal(nil)
+	}
+	return json.Marshal(i.String)
+}
+
+func TestIssue370(t *testing.T) {
+	v := []struct {
+		V Issue370
+	}{
+		{V: Issue370{String: "test", Valid: true}},
+	}
+	b, err := json.Marshal(v)
+	if err != nil {
+		t.Fatal(err)
+	}
+	got := string(b)
+	expected := `[{"V":"test"}]`
+	if got != expected {
+		t.Errorf("unexpected result: %v != %v", got, expected)
+	}
+}
+
+func TestIssue374(t *testing.T) {
+	r := io.MultiReader(strings.NewReader(strings.Repeat(" ", 505)+`"\u`), strings.NewReader(`0000"`))
+	var v interface{}
+	if err := json.NewDecoder(r).Decode(&v); err != nil {
+		t.Fatal(err)
+	}
+	got := v.(string)
+	expected := "\u0000"
+	if got != expected {
+		t.Errorf("unexpected result: %q != %q", got, expected)
+	}
+}
+
+func TestIssue381(t *testing.T) {
+	var v struct {
+		Field0  bool
+		Field1  bool
+		Field2  bool
+		Field3  bool
+		Field4  bool
+		Field5  bool
+		Field6  bool
+		Field7  bool
+		Field8  bool
+		Field9  bool
+		Field10 bool
+		Field11 bool
+		Field12 bool
+		Field13 bool
+		Field14 bool
+		Field15 bool
+		Field16 bool
+		Field17 bool
+		Field18 bool
+		Field19 bool
+		Field20 bool
+		Field21 bool
+		Field22 bool
+		Field23 bool
+		Field24 bool
+		Field25 bool
+		Field26 bool
+		Field27 bool
+		Field28 bool
+		Field29 bool
+		Field30 bool
+		Field31 bool
+		Field32 bool
+		Field33 bool
+		Field34 bool
+		Field35 bool
+		Field36 bool
+		Field37 bool
+		Field38 bool
+		Field39 bool
+		Field40 bool
+		Field41 bool
+		Field42 bool
+		Field43 bool
+		Field44 bool
+		Field45 bool
+		Field46 bool
+		Field47 bool
+		Field48 bool
+		Field49 bool
+		Field50 bool
+		Field51 bool
+		Field52 bool
+		Field53 bool
+		Field54 bool
+		Field55 bool
+		Field56 bool
+		Field57 bool
+		Field58 bool
+		Field59 bool
+		Field60 bool
+		Field61 bool
+		Field62 bool
+		Field63 bool
+		Field64 bool
+		Field65 bool
+		Field66 bool
+		Field67 bool
+		Field68 bool
+		Field69 bool
+		Field70 bool
+		Field71 bool
+		Field72 bool
+		Field73 bool
+		Field74 bool
+		Field75 bool
+		Field76 bool
+		Field77 bool
+		Field78 bool
+		Field79 bool
+		Field80 bool
+		Field81 bool
+		Field82 bool
+		Field83 bool
+		Field84 bool
+		Field85 bool
+		Field86 bool
+		Field87 bool
+		Field88 bool
+		Field89 bool
+		Field90 bool
+		Field91 bool
+		Field92 bool
+		Field93 bool
+		Field94 bool
+		Field95 bool
+		Field96 bool
+		Field97 bool
+		Field98 bool
+		Field99 bool
+	}
+
+	// test encoder cache issue, not related to encoder
+	b, err := json.Marshal(v)
+	if err != nil {
+		t.Errorf("failed to marshal %s", err.Error())
+		t.FailNow()
+	}
+
+	std, err := stdjson.Marshal(v)
+	if err != nil {
+		t.Errorf("failed to marshal with encoding/json %s", err.Error())
+		t.FailNow()
+	}
+
+	if !bytes.Equal(std, b) {
+		t.Errorf("encoding result not equal to encoding/json")
+		t.FailNow()
 	}
 }

internal/cmd/generator/vm.go.tmpl

@@ -3,6 +3,7 @@ package vm
 
 import (
 	"math"
+	"reflect"
 	"sort"
 	"unsafe"
 
@@ -194,10 +195,13 @@ func Run(ctx *encoder.RuntimeContext, b []byte, codeSet *encoder.OpcodeSet) ([]b
 				typ = iface.typ
 			}
 			if ifacePtr == nil {
+				isDirectedNil := typ != nil && typ.Kind() == reflect.Struct && !runtime.IfaceIndir(typ)
+				if !isDirectedNil {
 					b = appendNullComma(ctx, b)
 					code = code.Next
 					break
 				}
+			}
 			ctx.KeepRefs = append(ctx.KeepRefs, up)
 			ifaceCodeSet, err := encoder.CompileToGetCodeSet(ctx, uintptr(unsafe.Pointer(typ)))
 			if err != nil {

internal/decoder/compile.go

@@ -24,7 +24,7 @@ func init() {
 	if typeAddr == nil {
 		typeAddr = &runtime.TypeAddr{}
 	}
-	cachedDecoder = make([]Decoder, typeAddr.AddrRange>>typeAddr.AddrShift)
+	cachedDecoder = make([]Decoder, typeAddr.AddrRange>>typeAddr.AddrShift+1)
 }
 
 func loadDecoderMap() map[uintptr]Decoder {
@@ -393,6 +393,15 @@ func compileStruct(typ *runtime.Type, structName, fieldName string, structTypeTo
 						}
 						allFields = append(allFields, fieldSet)
 					}
+				} else {
+					fieldSet := &structFieldSet{
+						dec:         pdec,
+						offset:      field.Offset,
+						isTaggedKey: tag.IsTaggedKey,
+						key:         field.Name,
+						keyLen:      int64(len(field.Name)),
+					}
+					allFields = append(allFields, fieldSet)
 				}
 			} else {
 				fieldSet := &structFieldSet{

internal/decoder/string.go

@@ -95,24 +95,30 @@ func unicodeToRune(code []byte) rune {
 	return r
 }
 
+func readAtLeast(s *Stream, n int64, p *unsafe.Pointer) bool {
+	for s.cursor+n >= s.length {
+		if !s.read() {
+			return false
+		}
+		*p = s.bufptr()
+	}
+	return true
+}
+
 func decodeUnicodeRune(s *Stream, p unsafe.Pointer) (rune, int64, unsafe.Pointer, error) {
 	const defaultOffset = 5
 	const surrogateOffset = 11
 
-	if s.cursor+defaultOffset >= s.length {
+	if !readAtLeast(s, defaultOffset, &p) {
-		if !s.read() {
 		return rune(0), 0, nil, errors.ErrInvalidCharacter(s.char(), "escaped string", s.totalOffset())
 	}
-		p = s.bufptr()
-	}
 
 	r := unicodeToRune(s.buf[s.cursor+1 : s.cursor+defaultOffset])
 	if utf16.IsSurrogate(r) {
-		if s.cursor+surrogateOffset >= s.length {
+		if !readAtLeast(s, surrogateOffset, &p) {
-			s.read()
+			return unicode.ReplacementChar, defaultOffset, p, nil
-			p = s.bufptr()
 		}
-		if s.cursor+surrogateOffset >= s.length || s.buf[s.cursor+defaultOffset] != '\\' || s.buf[s.cursor+defaultOffset+1] != 'u' {
+		if s.buf[s.cursor+defaultOffset] != '\\' || s.buf[s.cursor+defaultOffset+1] != 'u' {
 			return unicode.ReplacementChar, defaultOffset, p, nil
 		}
 		r2 := unicodeToRune(s.buf[s.cursor+defaultOffset+2 : s.cursor+surrogateOffset])

internal/encoder/compiler.go

@@ -31,7 +31,7 @@ func init() {
 	if typeAddr == nil {
 		typeAddr = &runtime.TypeAddr{}
 	}
-	cachedOpcodeSets = make([]*OpcodeSet, typeAddr.AddrRange>>typeAddr.AddrShift)
+	cachedOpcodeSets = make([]*OpcodeSet, typeAddr.AddrRange>>typeAddr.AddrShift+1)
 }
 
 func loadOpcodeMap() map[uintptr]*OpcodeSet {
@@ -487,7 +487,10 @@ func (c *Compiler) listElemCode(typ *runtime.Type) (Code, error) {
 	case typ.Kind() == reflect.Map:
 		return c.ptrCode(runtime.PtrTo(typ))
 	default:
-		code, err := c.typeToCodeWithPtr(typ, false)
+		// isPtr was originally used to indicate whether the type of top level is pointer.
+		// However, since the slice/array element is a specification that can get the pointer address, explicitly set isPtr to true.
+		// See here for related issues: https://github.com/goccy/go-json/issues/370
+		code, err := c.typeToCodeWithPtr(typ, true)
 		if err != nil {
 			return nil, err
 		}
@@ -853,6 +856,9 @@ func (c *Compiler) implementsMarshalText(typ *runtime.Type) bool {
 }
 
 func (c *Compiler) isNilableType(typ *runtime.Type) bool {
+	if !runtime.IfaceIndir(typ) {
+		return true
+	}
 	switch typ.Kind() {
 	case reflect.Ptr:
 		return true

internal/encoder/vm/vm.go

@@ -3,6 +3,7 @@ package vm
 
 import (
 	"math"
+	"reflect"
 	"sort"
 	"unsafe"
 
@@ -194,10 +195,13 @@ func Run(ctx *encoder.RuntimeContext, b []byte, codeSet *encoder.OpcodeSet) ([]b
 				typ = iface.typ
 			}
 			if ifacePtr == nil {
+				isDirectedNil := typ != nil && typ.Kind() == reflect.Struct && !runtime.IfaceIndir(typ)
+				if !isDirectedNil {
 					b = appendNullComma(ctx, b)
 					code = code.Next
 					break
 				}
+			}
 			ctx.KeepRefs = append(ctx.KeepRefs, up)
 			ifaceCodeSet, err := encoder.CompileToGetCodeSet(ctx, uintptr(unsafe.Pointer(typ)))
 			if err != nil {

internal/encoder/vm_color/vm.go

@@ -3,6 +3,7 @@ package vm_color
 
 import (
 	"math"
+	"reflect"
 	"sort"
 	"unsafe"
 
@@ -194,10 +195,13 @@ func Run(ctx *encoder.RuntimeContext, b []byte, codeSet *encoder.OpcodeSet) ([]b
 				typ = iface.typ
 			}
 			if ifacePtr == nil {
+				isDirectedNil := typ != nil && typ.Kind() == reflect.Struct && !runtime.IfaceIndir(typ)
+				if !isDirectedNil {
 					b = appendNullComma(ctx, b)
 					code = code.Next
 					break
 				}
+			}
 			ctx.KeepRefs = append(ctx.KeepRefs, up)
 			ifaceCodeSet, err := encoder.CompileToGetCodeSet(ctx, uintptr(unsafe.Pointer(typ)))
 			if err != nil {

internal/encoder/vm_color_indent/vm.go

@@ -3,6 +3,7 @@ package vm_color_indent
 
 import (
 	"math"
+	"reflect"
 	"sort"
 	"unsafe"
 
@@ -194,10 +195,13 @@ func Run(ctx *encoder.RuntimeContext, b []byte, codeSet *encoder.OpcodeSet) ([]b
 				typ = iface.typ
 			}
 			if ifacePtr == nil {
+				isDirectedNil := typ != nil && typ.Kind() == reflect.Struct && !runtime.IfaceIndir(typ)
+				if !isDirectedNil {
 					b = appendNullComma(ctx, b)
 					code = code.Next
 					break
 				}
+			}
 			ctx.KeepRefs = append(ctx.KeepRefs, up)
 			ifaceCodeSet, err := encoder.CompileToGetCodeSet(ctx, uintptr(unsafe.Pointer(typ)))
 			if err != nil {

internal/encoder/vm_indent/vm.go

@@ -3,6 +3,7 @@ package vm_indent
 
 import (
 	"math"
+	"reflect"
 	"sort"
 	"unsafe"
 
@@ -194,10 +195,13 @@ func Run(ctx *encoder.RuntimeContext, b []byte, codeSet *encoder.OpcodeSet) ([]b
 				typ = iface.typ
 			}
 			if ifacePtr == nil {
+				isDirectedNil := typ != nil && typ.Kind() == reflect.Struct && !runtime.IfaceIndir(typ)
+				if !isDirectedNil {
 					b = appendNullComma(ctx, b)
 					code = code.Next
 					break
 				}
+			}
 			ctx.KeepRefs = append(ctx.KeepRefs, up)
 			ifaceCodeSet, err := encoder.CompileToGetCodeSet(ctx, uintptr(unsafe.Pointer(typ)))
 			if err != nil {