diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c4a94f..20d13e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,32 @@ +# v0.9.10 - 2022/07/15 + +### Fix bugs + +* Fix boundary exception of type caching ( #382 ) + +# v0.9.9 - 2022/07/15 + +### Fix bugs + +* Fix encoding of directed interface with typed nil ( #377 ) +* Fix embedded primitive type encoding using alias ( #378 ) +* Fix slice/array type encoding with types implementing MarshalJSON ( #379 ) +* Fix unicode decoding when the expected buffer state is not met after reading ( #380 ) + +# v0.9.8 - 2022/06/30 + +### Fix bugs + +* Fix decoding of surrogate-pair ( #365 ) +* Fix handling of embedded primitive type ( #366 ) +* Add validation of escape sequence for decoder ( #367 ) +* Fix stream tokenizing respecting UseNumber ( #369 ) +* Fix encoding when struct pointer type that implements Marshal JSON is embedded ( #375 ) + +### Improve performance + +* Improve performance of linkRecursiveCode ( #368 ) + # v0.9.7 - 2022/04/22 ### Fix bugs diff --git a/decode_test.go b/decode_test.go index f5e2563..c276a32 100644 --- a/decode_test.go +++ b/decode_test.go @@ -3968,3 +3968,20 @@ func TestIssue335(t *testing.T) { t.Errorf("unexpected success") } } + +func TestIssue372(t *testing.T) { + type A int + type T struct { + _ int + *A + } + var v T + err := json.Unmarshal([]byte(`{"A":7}`), &v) + assertErr(t, err) + + got := *v.A + expected := A(7) + if got != expected { + t.Errorf("unexpected result: %v != %v", got, expected) + } +} diff --git a/encode_test.go b/encode_test.go index 4bc4a60..384292e 100644 --- a/encode_test.go +++ b/encode_test.go @@ -7,11 +7,14 @@ import ( stdjson "encoding/json" "errors" "fmt" + "io" "log" "math" + "math/big" "reflect" "regexp" "strconv" + "strings" "testing" "time" @@ -2412,5 +2415,203 @@ func TestCamelCaseField(t *testing.T) { if !bytes.Equal(expected, got) { t.Fatalf("failed to encode. expected %q but got %q", expected, got) + +func TestIssue339(t *testing.T) { + type T1 struct { + *big.Int + } + type T2 struct { + T1 T1 `json:"T1"` + } + v := T2{T1{Int: big.NewInt(10000)}} + b, err := json.Marshal(&v) + assertErr(t, err) + got := string(b) + expected := `{"T1":10000}` + if got != expected { + t.Errorf("unexpected result: %v != %v", got, expected) + } +} + +func TestIssue376(t *testing.T) { + type Container struct { + V interface{} `json:"value"` + } + type MapOnly struct { + Map map[string]int64 `json:"map"` + } + b, err := json.Marshal(Container{MapOnly{}}) + if err != nil { + t.Fatal(err) + } + got := string(b) + expected := `{"value":{"map":null}}` + if got != expected { + t.Errorf("unexpected result: %v != %v", got, expected) + } +} + +type Issue370 struct { + String string + Valid bool +} + +func (i *Issue370) MarshalJSON() ([]byte, error) { + if !i.Valid { + return json.Marshal(nil) + } + return json.Marshal(i.String) +} + +func TestIssue370(t *testing.T) { + v := []struct { + V Issue370 + }{ + {V: Issue370{String: "test", Valid: true}}, + } + b, err := json.Marshal(v) + if err != nil { + t.Fatal(err) + } + got := string(b) + expected := `[{"V":"test"}]` + if got != expected { + t.Errorf("unexpected result: %v != %v", got, expected) + } +} + +func TestIssue374(t *testing.T) { + r := io.MultiReader(strings.NewReader(strings.Repeat(" ", 505)+`"\u`), strings.NewReader(`0000"`)) + var v interface{} + if err := json.NewDecoder(r).Decode(&v); err != nil { + t.Fatal(err) + } + got := v.(string) + expected := "\u0000" + if got != expected { + t.Errorf("unexpected result: %q != %q", got, expected) + } +} + +func TestIssue381(t *testing.T) { + var v struct { + Field0 bool + Field1 bool + Field2 bool + Field3 bool + Field4 bool + Field5 bool + Field6 bool + Field7 bool + Field8 bool + Field9 bool + Field10 bool + Field11 bool + Field12 bool + Field13 bool + Field14 bool + Field15 bool + Field16 bool + Field17 bool + Field18 bool + Field19 bool + Field20 bool + Field21 bool + Field22 bool + Field23 bool + Field24 bool + Field25 bool + Field26 bool + Field27 bool + Field28 bool + Field29 bool + Field30 bool + Field31 bool + Field32 bool + Field33 bool + Field34 bool + Field35 bool + Field36 bool + Field37 bool + Field38 bool + Field39 bool + Field40 bool + Field41 bool + Field42 bool + Field43 bool + Field44 bool + Field45 bool + Field46 bool + Field47 bool + Field48 bool + Field49 bool + Field50 bool + Field51 bool + Field52 bool + Field53 bool + Field54 bool + Field55 bool + Field56 bool + Field57 bool + Field58 bool + Field59 bool + Field60 bool + Field61 bool + Field62 bool + Field63 bool + Field64 bool + Field65 bool + Field66 bool + Field67 bool + Field68 bool + Field69 bool + Field70 bool + Field71 bool + Field72 bool + Field73 bool + Field74 bool + Field75 bool + Field76 bool + Field77 bool + Field78 bool + Field79 bool + Field80 bool + Field81 bool + Field82 bool + Field83 bool + Field84 bool + Field85 bool + Field86 bool + Field87 bool + Field88 bool + Field89 bool + Field90 bool + Field91 bool + Field92 bool + Field93 bool + Field94 bool + Field95 bool + Field96 bool + Field97 bool + Field98 bool + Field99 bool + } + + // test encoder cache issue, not related to encoder + b, err := json.Marshal(v) + if err != nil { + t.Errorf("failed to marshal %s", err.Error()) + t.FailNow() + } + + std, err := stdjson.Marshal(v) + if err != nil { + t.Errorf("failed to marshal with encoding/json %s", err.Error()) + t.FailNow() + } + + if !bytes.Equal(std, b) { + t.Errorf("encoding result not equal to encoding/json") + t.FailNow() } } diff --git a/internal/cmd/generator/vm.go.tmpl b/internal/cmd/generator/vm.go.tmpl index 91b11e1..645d20f 100644 --- a/internal/cmd/generator/vm.go.tmpl +++ b/internal/cmd/generator/vm.go.tmpl @@ -3,6 +3,7 @@ package vm import ( "math" + "reflect" "sort" "unsafe" @@ -194,9 +195,12 @@ func Run(ctx *encoder.RuntimeContext, b []byte, codeSet *encoder.OpcodeSet) ([]b typ = iface.typ } if ifacePtr == nil { - b = appendNullComma(ctx, b) - code = code.Next - break + isDirectedNil := typ != nil && typ.Kind() == reflect.Struct && !runtime.IfaceIndir(typ) + if !isDirectedNil { + b = appendNullComma(ctx, b) + code = code.Next + break + } } ctx.KeepRefs = append(ctx.KeepRefs, up) ifaceCodeSet, err := encoder.CompileToGetCodeSet(ctx, uintptr(unsafe.Pointer(typ))) diff --git a/internal/decoder/compile.go b/internal/decoder/compile.go index f13b43b..fab6437 100644 --- a/internal/decoder/compile.go +++ b/internal/decoder/compile.go @@ -24,7 +24,7 @@ func init() { if typeAddr == nil { typeAddr = &runtime.TypeAddr{} } - cachedDecoder = make([]Decoder, typeAddr.AddrRange>>typeAddr.AddrShift) + cachedDecoder = make([]Decoder, typeAddr.AddrRange>>typeAddr.AddrShift+1) } func loadDecoderMap() map[uintptr]Decoder { @@ -393,6 +393,15 @@ func compileStruct(typ *runtime.Type, structName, fieldName string, structTypeTo } allFields = append(allFields, fieldSet) } + } else { + fieldSet := &structFieldSet{ + dec: pdec, + offset: field.Offset, + isTaggedKey: tag.IsTaggedKey, + key: field.Name, + keyLen: int64(len(field.Name)), + } + allFields = append(allFields, fieldSet) } } else { fieldSet := &structFieldSet{ diff --git a/internal/decoder/string.go b/internal/decoder/string.go index cef6688..871ab3d 100644 --- a/internal/decoder/string.go +++ b/internal/decoder/string.go @@ -95,24 +95,30 @@ func unicodeToRune(code []byte) rune { return r } +func readAtLeast(s *Stream, n int64, p *unsafe.Pointer) bool { + for s.cursor+n >= s.length { + if !s.read() { + return false + } + *p = s.bufptr() + } + return true +} + func decodeUnicodeRune(s *Stream, p unsafe.Pointer) (rune, int64, unsafe.Pointer, error) { const defaultOffset = 5 const surrogateOffset = 11 - if s.cursor+defaultOffset >= s.length { - if !s.read() { - return rune(0), 0, nil, errors.ErrInvalidCharacter(s.char(), "escaped string", s.totalOffset()) - } - p = s.bufptr() + if !readAtLeast(s, defaultOffset, &p) { + return rune(0), 0, nil, errors.ErrInvalidCharacter(s.char(), "escaped string", s.totalOffset()) } r := unicodeToRune(s.buf[s.cursor+1 : s.cursor+defaultOffset]) if utf16.IsSurrogate(r) { - if s.cursor+surrogateOffset >= s.length { - s.read() - p = s.bufptr() + if !readAtLeast(s, surrogateOffset, &p) { + return unicode.ReplacementChar, defaultOffset, p, nil } - if s.cursor+surrogateOffset >= s.length || s.buf[s.cursor+defaultOffset] != '\\' || s.buf[s.cursor+defaultOffset+1] != 'u' { + if s.buf[s.cursor+defaultOffset] != '\\' || s.buf[s.cursor+defaultOffset+1] != 'u' { return unicode.ReplacementChar, defaultOffset, p, nil } r2 := unicodeToRune(s.buf[s.cursor+defaultOffset+2 : s.cursor+surrogateOffset]) diff --git a/internal/encoder/compiler.go b/internal/encoder/compiler.go index 1be01aa..bf5e0f9 100644 --- a/internal/encoder/compiler.go +++ b/internal/encoder/compiler.go @@ -31,7 +31,7 @@ func init() { if typeAddr == nil { typeAddr = &runtime.TypeAddr{} } - cachedOpcodeSets = make([]*OpcodeSet, typeAddr.AddrRange>>typeAddr.AddrShift) + cachedOpcodeSets = make([]*OpcodeSet, typeAddr.AddrRange>>typeAddr.AddrShift+1) } func loadOpcodeMap() map[uintptr]*OpcodeSet { @@ -487,7 +487,10 @@ func (c *Compiler) listElemCode(typ *runtime.Type) (Code, error) { case typ.Kind() == reflect.Map: return c.ptrCode(runtime.PtrTo(typ)) default: - code, err := c.typeToCodeWithPtr(typ, false) + // isPtr was originally used to indicate whether the type of top level is pointer. + // However, since the slice/array element is a specification that can get the pointer address, explicitly set isPtr to true. + // See here for related issues: https://github.com/goccy/go-json/issues/370 + code, err := c.typeToCodeWithPtr(typ, true) if err != nil { return nil, err } @@ -853,6 +856,9 @@ func (c *Compiler) implementsMarshalText(typ *runtime.Type) bool { } func (c *Compiler) isNilableType(typ *runtime.Type) bool { + if !runtime.IfaceIndir(typ) { + return true + } switch typ.Kind() { case reflect.Ptr: return true diff --git a/internal/encoder/vm/vm.go b/internal/encoder/vm/vm.go index 91b11e1..645d20f 100644 --- a/internal/encoder/vm/vm.go +++ b/internal/encoder/vm/vm.go @@ -3,6 +3,7 @@ package vm import ( "math" + "reflect" "sort" "unsafe" @@ -194,9 +195,12 @@ func Run(ctx *encoder.RuntimeContext, b []byte, codeSet *encoder.OpcodeSet) ([]b typ = iface.typ } if ifacePtr == nil { - b = appendNullComma(ctx, b) - code = code.Next - break + isDirectedNil := typ != nil && typ.Kind() == reflect.Struct && !runtime.IfaceIndir(typ) + if !isDirectedNil { + b = appendNullComma(ctx, b) + code = code.Next + break + } } ctx.KeepRefs = append(ctx.KeepRefs, up) ifaceCodeSet, err := encoder.CompileToGetCodeSet(ctx, uintptr(unsafe.Pointer(typ))) diff --git a/internal/encoder/vm_color/vm.go b/internal/encoder/vm_color/vm.go index 5c6c52c..a63e83e 100644 --- a/internal/encoder/vm_color/vm.go +++ b/internal/encoder/vm_color/vm.go @@ -3,6 +3,7 @@ package vm_color import ( "math" + "reflect" "sort" "unsafe" @@ -194,9 +195,12 @@ func Run(ctx *encoder.RuntimeContext, b []byte, codeSet *encoder.OpcodeSet) ([]b typ = iface.typ } if ifacePtr == nil { - b = appendNullComma(ctx, b) - code = code.Next - break + isDirectedNil := typ != nil && typ.Kind() == reflect.Struct && !runtime.IfaceIndir(typ) + if !isDirectedNil { + b = appendNullComma(ctx, b) + code = code.Next + break + } } ctx.KeepRefs = append(ctx.KeepRefs, up) ifaceCodeSet, err := encoder.CompileToGetCodeSet(ctx, uintptr(unsafe.Pointer(typ))) diff --git a/internal/encoder/vm_color_indent/vm.go b/internal/encoder/vm_color_indent/vm.go index 42dc11c..3b4e22e 100644 --- a/internal/encoder/vm_color_indent/vm.go +++ b/internal/encoder/vm_color_indent/vm.go @@ -3,6 +3,7 @@ package vm_color_indent import ( "math" + "reflect" "sort" "unsafe" @@ -194,9 +195,12 @@ func Run(ctx *encoder.RuntimeContext, b []byte, codeSet *encoder.OpcodeSet) ([]b typ = iface.typ } if ifacePtr == nil { - b = appendNullComma(ctx, b) - code = code.Next - break + isDirectedNil := typ != nil && typ.Kind() == reflect.Struct && !runtime.IfaceIndir(typ) + if !isDirectedNil { + b = appendNullComma(ctx, b) + code = code.Next + break + } } ctx.KeepRefs = append(ctx.KeepRefs, up) ifaceCodeSet, err := encoder.CompileToGetCodeSet(ctx, uintptr(unsafe.Pointer(typ))) diff --git a/internal/encoder/vm_indent/vm.go b/internal/encoder/vm_indent/vm.go index dfe0cc6..836c5c8 100644 --- a/internal/encoder/vm_indent/vm.go +++ b/internal/encoder/vm_indent/vm.go @@ -3,6 +3,7 @@ package vm_indent import ( "math" + "reflect" "sort" "unsafe" @@ -194,9 +195,12 @@ func Run(ctx *encoder.RuntimeContext, b []byte, codeSet *encoder.OpcodeSet) ([]b typ = iface.typ } if ifacePtr == nil { - b = appendNullComma(ctx, b) - code = code.Next - break + isDirectedNil := typ != nil && typ.Kind() == reflect.Struct && !runtime.IfaceIndir(typ) + if !isDirectedNil { + b = appendNullComma(ctx, b) + code = code.Next + break + } } ctx.KeepRefs = append(ctx.KeepRefs, up) ifaceCodeSet, err := encoder.CompileToGetCodeSet(ctx, uintptr(unsafe.Pointer(typ)))