mirror
/
go-json
mirror of https://github.com/goccy/go-json.git
1
0
Fork 1
Code Issues Packages Projects Releases Wiki Activity

Improve performance of streaming decoder

This commit is contained in:
Masaaki Goshima 2021-02-06 14:24:07 +09:00
parent e0f5f9764f
commit aef2390074
2 changed files with 251 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
decode_compile.go
View File

@ -272,6 +272,7 @@ func (d *Decoder) removeConflictFields(fieldMap map[string]*structFieldSet, conf
if v.isTaggedKey {
// conflict tag key
delete(fieldMap, k)
delete(fieldMap, strings.ToLower(k))
conflictedMap[k] = struct{}{}
conflictedMap[strings.ToLower(k)] = struct{}{}
}
@ -290,6 +291,7 @@ func (d *Decoder) removeConflictFields(fieldMap map[string]*structFieldSet, conf
} else {
// conflict tag key
delete(fieldMap, k)
delete(fieldMap, strings.ToLower(k))
conflictedMap[k] = struct{}{}
conflictedMap[strings.ToLower(k)] = struct{}{}
}
@ -355,6 +357,7 @@ func (d *Decoder) compileStruct(typ *rtype, structName, fieldName string) (decod
if v.isTaggedKey {
// conflict tag key
delete(fieldMap, k)
delete(fieldMap, strings.ToLower(k))
conflictedMap[k] = struct{}{}
conflictedMap[strings.ToLower(k)] = struct{}{}
}
@ -373,6 +376,7 @@ func (d *Decoder) compileStruct(typ *rtype, structName, fieldName string) (decod
} else {
// conflict tag key
delete(fieldMap, k)
delete(fieldMap, strings.ToLower(k))
conflictedMap[k] = struct{}{}
conflictedMap[strings.ToLower(k)] = struct{}{}
}

268
decode_struct.go
View File

@ -15,15 +15,16 @@ type structFieldSet struct {
}
type structDecoder struct {
fieldMap map[string]*structFieldSet
stringDecoder *stringDecoder
structName string
fieldName string
isTriedOptimize bool
keyBitmapInt8 [][256]int8
keyBitmapInt16 [][256]int16
sortedFieldSets []*structFieldSet
keyDecoder func(*structDecoder, []byte, int64) (int64, *structFieldSet, error)
fieldMap map[string]*structFieldSet
stringDecoder *stringDecoder
structName string
fieldName string
isTriedOptimize bool
keyBitmapInt8 [][256]int8
keyBitmapInt16 [][256]int16
sortedFieldSets []*structFieldSet
keyDecoder func(*structDecoder, []byte, int64) (int64, *structFieldSet, error)
keyStreamDecoder func(*structDecoder, *stream) (*structFieldSet, string, error)
}
var (
@ -48,11 +49,12 @@ func init() {
func newStructDecoder(structName, fieldName string, fieldMap map[string]*structFieldSet) *structDecoder {
return &structDecoder{
fieldMap: fieldMap,
stringDecoder: newStringDecoder(structName, fieldName),
structName: structName,
fieldName: fieldName,
keyDecoder: decodeKey,
fieldMap: fieldMap,
stringDecoder: newStringDecoder(structName, fieldName),
structName: structName,
fieldName: fieldName,
keyDecoder: decodeKey,
keyStreamDecoder: decodeKeyStream,
}
}
@ -66,9 +68,24 @@ func (d *structDecoder) tryOptimize() {
return
}
fieldMap := map[string]*structFieldSet{}
conflicted := map[string]struct{}{}
for k, v := range d.fieldMap {
k := strings.ToLower(k)
fieldMap[k] = v
key := strings.ToLower(k)
if key != k {
// already exists same key (e.g. Hello and HELLO has same lower case key
if _, exists := conflicted[key]; exists {
d.isTriedOptimize = true
return
}
conflicted[key] = struct{}{}
}
if field, exists := fieldMap[key]; exists {
if field != v {
d.isTriedOptimize = true
return
}
}
fieldMap[key] = v
}
if len(fieldMap) > allowOptimizeMaxFieldLen {
@ -101,6 +118,7 @@ func (d *structDecoder) tryOptimize() {
}
d.keyBitmapInt8 = keyBitmap
d.keyDecoder = decodeKeyByBitmapInt8
d.keyStreamDecoder = decodeKeyByBitmapInt8Stream
} else {
keyBitmap := make([][256]int16, maxKeyLen)
for i, key := range sortedKeys {
@ -112,6 +130,7 @@ func (d *structDecoder) tryOptimize() {
}
d.keyBitmapInt16 = keyBitmap
d.keyDecoder = decodeKeyByBitmapInt16
d.keyStreamDecoder = decodeKeyByBitmapInt16Stream
}
}
@ -289,6 +308,215 @@ func decodeKey(d *structDecoder, buf []byte, cursor int64) (int64, *structFieldS
return cursor, field, nil
}
func decodeKeyByBitmapInt8Stream(d *structDecoder, s *stream) (*structFieldSet, string, error) {
var (
field *structFieldSet
curBit int8 = math.MaxInt8
)
for {
switch s.char() {
case ' ', '\n', '\t', '\r':
s.cursor++
case '"':
s.cursor++
FIRST_CHAR:
start := s.cursor
switch s.char() {
case '"':
s.cursor++
return field, "", nil
case nul:
if s.read() {
goto FIRST_CHAR
}
return nil, "", errUnexpectedEndOfJSON("string", s.totalOffset())
}
keyIdx := 0
bitmap := d.keyBitmapInt8
keyBitmapLen := len(bitmap)
for {
c := s.char()
switch c {
case '"':
x := uint64(curBit & -curBit)
fieldSetIndex := bitHashTable[(x*0x03F566ED27179461)>>58]
field = d.sortedFieldSets[fieldSetIndex]
b := s.buf[start:s.cursor]
key := *(*string)(unsafe.Pointer(&b))
s.cursor++
return field, key, nil
case nul:
if s.read() {
continue
}
return nil, "", errUnexpectedEndOfJSON("string", s.totalOffset())
default:
if keyIdx >= keyBitmapLen {
for {
s.cursor++
switch s.char() {
case '"':
b := s.buf[start:s.cursor]
key := *(*string)(unsafe.Pointer(&b))
s.cursor++
return field, key, nil
case '\\':
s.cursor++
if s.char() == nul {
if !s.read() {
return nil, "", errUnexpectedEndOfJSON("string", s.totalOffset())
}
}
case nul:
if !s.read() {
return nil, "", errUnexpectedEndOfJSON("string", s.totalOffset())
}
}
}
}
curBit &= bitmap[keyIdx][largeToSmallTable[c]]
if curBit == 0 {
for {
s.cursor++
switch s.char() {
case '"':
b := s.buf[start:s.cursor]
key := *(*string)(unsafe.Pointer(&b))
s.cursor++
return field, key, nil
case '\\':
s.cursor++
if s.char() == nul {
if !s.read() {
return nil, "", errUnexpectedEndOfJSON("string", s.totalOffset())
}
}
case nul:
if !s.read() {
return nil, "", errUnexpectedEndOfJSON("string", s.totalOffset())
}
}
}
}
keyIdx++
}
s.cursor++
}
default:
return nil, "", errNotAtBeginningOfValue(s.totalOffset())
}
}
}
func decodeKeyByBitmapInt16Stream(d *structDecoder, s *stream) (*structFieldSet, string, error) {
var (
field *structFieldSet
curBit int16 = math.MaxInt16
)
for {
switch s.char() {
case ' ', '\n', '\t', '\r':
s.cursor++
case '"':
s.cursor++
FIRST_CHAR:
start := s.cursor
switch s.char() {
case '"':
s.cursor++
return field, "", nil
case nul:
if s.read() {
goto FIRST_CHAR
}
return nil, "", errUnexpectedEndOfJSON("string", s.totalOffset())
}
keyIdx := 0
bitmap := d.keyBitmapInt16
keyBitmapLen := len(bitmap)
for {
c := s.char()
switch c {
case '"':
x := uint64(curBit & -curBit)
fieldSetIndex := bitHashTable[(x*0x03F566ED27179461)>>58]
field = d.sortedFieldSets[fieldSetIndex]
b := s.buf[start:s.cursor]
key := *(*string)(unsafe.Pointer(&b))
s.cursor++
return field, key, nil
case nul:
if s.read() {
continue
}
return nil, "", errUnexpectedEndOfJSON("string", s.totalOffset())
default:
if keyIdx >= keyBitmapLen {
for {
s.cursor++
switch s.char() {
case '"':
b := s.buf[start:s.cursor]
key := *(*string)(unsafe.Pointer(&b))
s.cursor++
return field, key, nil
case '\\':
s.cursor++
if s.char() == nul {
if !s.read() {
return nil, "", errUnexpectedEndOfJSON("string", s.totalOffset())
}
}
case nul:
if !s.read() {
return nil, "", errUnexpectedEndOfJSON("string", s.totalOffset())
}
}
}
}
curBit &= bitmap[keyIdx][largeToSmallTable[c]]
if curBit == 0 {
for {
s.cursor++
switch s.char() {
case '"':
b := s.buf[start:s.cursor]
key := *(*string)(unsafe.Pointer(&b))
s.cursor++
return field, key, nil
case '\\':
s.cursor++
if s.char() == nul {
if !s.read() {
return nil, "", errUnexpectedEndOfJSON("string", s.totalOffset())
}
}
case nul:
if !s.read() {
return nil, "", errUnexpectedEndOfJSON("string", s.totalOffset())
}
}
}
}
keyIdx++
}
s.cursor++
}
default:
return nil, "", errNotAtBeginningOfValue(s.totalOffset())
}
}
}
func decodeKeyStream(d *structDecoder, s *stream) (*structFieldSet, string, error) {
key, err := d.stringDecoder.decodeStreamByte(s)
if err != nil {
return nil, "", err
}
k := *(*string)(unsafe.Pointer(&key))
return d.fieldMap[k], k, nil
}
func (d *structDecoder) decodeStream(s *stream, p unsafe.Pointer) error {
s.skipWhiteSpace()
switch s.char() {
@ -311,7 +539,7 @@ func (d *structDecoder) decodeStream(s *stream, p unsafe.Pointer) error {
}
for {
s.reset()
key, err := d.stringDecoder.decodeStreamByte(s)
field, key, err := d.keyStreamDecoder(d, s)
if err != nil {
return err
}
@ -328,14 +556,12 @@ func (d *structDecoder) decodeStream(s *stream, p unsafe.Pointer) error {
return errExpected("object value after colon", s.totalOffset())
}
}
k := *(*string)(unsafe.Pointer(&key))
field, exists := d.fieldMap[k]
if exists {
if field != nil {
if err := field.dec.decodeStream(s, unsafe.Pointer(uintptr(p)+field.offset)); err != nil {
return err
}
} else if s.disallowUnknownFields {
return fmt.Errorf("json: unknown field %q", k)
return fmt.Errorf("json: unknown field %q", key)
} else {
if err := s.skipValue(); err != nil {
return err