diff --git a/decode_compile.go b/decode_compile.go index 982e929..c494f90 100644 --- a/decode_compile.go +++ b/decode_compile.go @@ -272,6 +272,7 @@ func (d *Decoder) removeConflictFields(fieldMap map[string]*structFieldSet, conf if v.isTaggedKey { // conflict tag key delete(fieldMap, k) + delete(fieldMap, strings.ToLower(k)) conflictedMap[k] = struct{}{} conflictedMap[strings.ToLower(k)] = struct{}{} } @@ -290,6 +291,7 @@ func (d *Decoder) removeConflictFields(fieldMap map[string]*structFieldSet, conf } else { // conflict tag key delete(fieldMap, k) + delete(fieldMap, strings.ToLower(k)) conflictedMap[k] = struct{}{} conflictedMap[strings.ToLower(k)] = struct{}{} } @@ -355,6 +357,7 @@ func (d *Decoder) compileStruct(typ *rtype, structName, fieldName string) (decod if v.isTaggedKey { // conflict tag key delete(fieldMap, k) + delete(fieldMap, strings.ToLower(k)) conflictedMap[k] = struct{}{} conflictedMap[strings.ToLower(k)] = struct{}{} } @@ -373,6 +376,7 @@ func (d *Decoder) compileStruct(typ *rtype, structName, fieldName string) (decod } else { // conflict tag key delete(fieldMap, k) + delete(fieldMap, strings.ToLower(k)) conflictedMap[k] = struct{}{} conflictedMap[strings.ToLower(k)] = struct{}{} } diff --git a/decode_struct.go b/decode_struct.go index 5f9c6a6..3395133 100644 --- a/decode_struct.go +++ b/decode_struct.go @@ -15,15 +15,16 @@ type structFieldSet struct { } type structDecoder struct { - fieldMap map[string]*structFieldSet - stringDecoder *stringDecoder - structName string - fieldName string - isTriedOptimize bool - keyBitmapInt8 [][256]int8 - keyBitmapInt16 [][256]int16 - sortedFieldSets []*structFieldSet - keyDecoder func(*structDecoder, []byte, int64) (int64, *structFieldSet, error) + fieldMap map[string]*structFieldSet + stringDecoder *stringDecoder + structName string + fieldName string + isTriedOptimize bool + keyBitmapInt8 [][256]int8 + keyBitmapInt16 [][256]int16 + sortedFieldSets []*structFieldSet + keyDecoder func(*structDecoder, []byte, int64) (int64, *structFieldSet, error) + keyStreamDecoder func(*structDecoder, *stream) (*structFieldSet, string, error) } var ( @@ -48,11 +49,12 @@ func init() { func newStructDecoder(structName, fieldName string, fieldMap map[string]*structFieldSet) *structDecoder { return &structDecoder{ - fieldMap: fieldMap, - stringDecoder: newStringDecoder(structName, fieldName), - structName: structName, - fieldName: fieldName, - keyDecoder: decodeKey, + fieldMap: fieldMap, + stringDecoder: newStringDecoder(structName, fieldName), + structName: structName, + fieldName: fieldName, + keyDecoder: decodeKey, + keyStreamDecoder: decodeKeyStream, } } @@ -66,9 +68,24 @@ func (d *structDecoder) tryOptimize() { return } fieldMap := map[string]*structFieldSet{} + conflicted := map[string]struct{}{} for k, v := range d.fieldMap { - k := strings.ToLower(k) - fieldMap[k] = v + key := strings.ToLower(k) + if key != k { + // already exists same key (e.g. Hello and HELLO has same lower case key + if _, exists := conflicted[key]; exists { + d.isTriedOptimize = true + return + } + conflicted[key] = struct{}{} + } + if field, exists := fieldMap[key]; exists { + if field != v { + d.isTriedOptimize = true + return + } + } + fieldMap[key] = v } if len(fieldMap) > allowOptimizeMaxFieldLen { @@ -101,6 +118,7 @@ func (d *structDecoder) tryOptimize() { } d.keyBitmapInt8 = keyBitmap d.keyDecoder = decodeKeyByBitmapInt8 + d.keyStreamDecoder = decodeKeyByBitmapInt8Stream } else { keyBitmap := make([][256]int16, maxKeyLen) for i, key := range sortedKeys { @@ -112,6 +130,7 @@ func (d *structDecoder) tryOptimize() { } d.keyBitmapInt16 = keyBitmap d.keyDecoder = decodeKeyByBitmapInt16 + d.keyStreamDecoder = decodeKeyByBitmapInt16Stream } } @@ -289,6 +308,215 @@ func decodeKey(d *structDecoder, buf []byte, cursor int64) (int64, *structFieldS return cursor, field, nil } +func decodeKeyByBitmapInt8Stream(d *structDecoder, s *stream) (*structFieldSet, string, error) { + var ( + field *structFieldSet + curBit int8 = math.MaxInt8 + ) + for { + switch s.char() { + case ' ', '\n', '\t', '\r': + s.cursor++ + case '"': + s.cursor++ + FIRST_CHAR: + start := s.cursor + switch s.char() { + case '"': + s.cursor++ + return field, "", nil + case nul: + if s.read() { + goto FIRST_CHAR + } + return nil, "", errUnexpectedEndOfJSON("string", s.totalOffset()) + } + keyIdx := 0 + bitmap := d.keyBitmapInt8 + keyBitmapLen := len(bitmap) + for { + c := s.char() + switch c { + case '"': + x := uint64(curBit & -curBit) + fieldSetIndex := bitHashTable[(x*0x03F566ED27179461)>>58] + field = d.sortedFieldSets[fieldSetIndex] + b := s.buf[start:s.cursor] + key := *(*string)(unsafe.Pointer(&b)) + s.cursor++ + return field, key, nil + case nul: + if s.read() { + continue + } + return nil, "", errUnexpectedEndOfJSON("string", s.totalOffset()) + default: + if keyIdx >= keyBitmapLen { + for { + s.cursor++ + switch s.char() { + case '"': + b := s.buf[start:s.cursor] + key := *(*string)(unsafe.Pointer(&b)) + s.cursor++ + return field, key, nil + case '\\': + s.cursor++ + if s.char() == nul { + if !s.read() { + return nil, "", errUnexpectedEndOfJSON("string", s.totalOffset()) + } + } + case nul: + if !s.read() { + return nil, "", errUnexpectedEndOfJSON("string", s.totalOffset()) + } + } + } + } + curBit &= bitmap[keyIdx][largeToSmallTable[c]] + if curBit == 0 { + for { + s.cursor++ + switch s.char() { + case '"': + b := s.buf[start:s.cursor] + key := *(*string)(unsafe.Pointer(&b)) + s.cursor++ + return field, key, nil + case '\\': + s.cursor++ + if s.char() == nul { + if !s.read() { + return nil, "", errUnexpectedEndOfJSON("string", s.totalOffset()) + } + } + case nul: + if !s.read() { + return nil, "", errUnexpectedEndOfJSON("string", s.totalOffset()) + } + } + } + } + keyIdx++ + } + s.cursor++ + } + default: + return nil, "", errNotAtBeginningOfValue(s.totalOffset()) + } + } +} + +func decodeKeyByBitmapInt16Stream(d *structDecoder, s *stream) (*structFieldSet, string, error) { + var ( + field *structFieldSet + curBit int16 = math.MaxInt16 + ) + for { + switch s.char() { + case ' ', '\n', '\t', '\r': + s.cursor++ + case '"': + s.cursor++ + FIRST_CHAR: + start := s.cursor + switch s.char() { + case '"': + s.cursor++ + return field, "", nil + case nul: + if s.read() { + goto FIRST_CHAR + } + return nil, "", errUnexpectedEndOfJSON("string", s.totalOffset()) + } + keyIdx := 0 + bitmap := d.keyBitmapInt16 + keyBitmapLen := len(bitmap) + for { + c := s.char() + switch c { + case '"': + x := uint64(curBit & -curBit) + fieldSetIndex := bitHashTable[(x*0x03F566ED27179461)>>58] + field = d.sortedFieldSets[fieldSetIndex] + b := s.buf[start:s.cursor] + key := *(*string)(unsafe.Pointer(&b)) + s.cursor++ + return field, key, nil + case nul: + if s.read() { + continue + } + return nil, "", errUnexpectedEndOfJSON("string", s.totalOffset()) + default: + if keyIdx >= keyBitmapLen { + for { + s.cursor++ + switch s.char() { + case '"': + b := s.buf[start:s.cursor] + key := *(*string)(unsafe.Pointer(&b)) + s.cursor++ + return field, key, nil + case '\\': + s.cursor++ + if s.char() == nul { + if !s.read() { + return nil, "", errUnexpectedEndOfJSON("string", s.totalOffset()) + } + } + case nul: + if !s.read() { + return nil, "", errUnexpectedEndOfJSON("string", s.totalOffset()) + } + } + } + } + curBit &= bitmap[keyIdx][largeToSmallTable[c]] + if curBit == 0 { + for { + s.cursor++ + switch s.char() { + case '"': + b := s.buf[start:s.cursor] + key := *(*string)(unsafe.Pointer(&b)) + s.cursor++ + return field, key, nil + case '\\': + s.cursor++ + if s.char() == nul { + if !s.read() { + return nil, "", errUnexpectedEndOfJSON("string", s.totalOffset()) + } + } + case nul: + if !s.read() { + return nil, "", errUnexpectedEndOfJSON("string", s.totalOffset()) + } + } + } + } + keyIdx++ + } + s.cursor++ + } + default: + return nil, "", errNotAtBeginningOfValue(s.totalOffset()) + } + } +} + +func decodeKeyStream(d *structDecoder, s *stream) (*structFieldSet, string, error) { + key, err := d.stringDecoder.decodeStreamByte(s) + if err != nil { + return nil, "", err + } + k := *(*string)(unsafe.Pointer(&key)) + return d.fieldMap[k], k, nil +} + func (d *structDecoder) decodeStream(s *stream, p unsafe.Pointer) error { s.skipWhiteSpace() switch s.char() { @@ -311,7 +539,7 @@ func (d *structDecoder) decodeStream(s *stream, p unsafe.Pointer) error { } for { s.reset() - key, err := d.stringDecoder.decodeStreamByte(s) + field, key, err := d.keyStreamDecoder(d, s) if err != nil { return err } @@ -328,14 +556,12 @@ func (d *structDecoder) decodeStream(s *stream, p unsafe.Pointer) error { return errExpected("object value after colon", s.totalOffset()) } } - k := *(*string)(unsafe.Pointer(&key)) - field, exists := d.fieldMap[k] - if exists { + if field != nil { if err := field.dec.decodeStream(s, unsafe.Pointer(uintptr(p)+field.offset)); err != nil { return err } } else if s.disallowUnknownFields { - return fmt.Errorf("json: unknown field %q", k) + return fmt.Errorf("json: unknown field %q", key) } else { if err := s.skipValue(); err != nil { return err