New gjson.Unmarshal function

It's a drop in replacement for json.Unmarshal and you can typically see
a 3 to 4 times boost in performance without the need for external tools
or generators.

This function works almost identically to json.Unmarshal except that
it expects the json to be well-formed prior to being called. Invalid
json will not panic, but it may return back unexpected results.
Therefore the return value of this function will always be nil.

Another difference is that gjson.Unmarshal will automatically attempt
to convert JSON values to any Go type. For example, the JSON string
"100" or the JSON number 100 can be equally assigned to Go string,
int, byte, uint64, etc. This rule applies to all types.
This commit is contained in:
Josh Baker 2017-05-07 18:26:54 -07:00
parent e30a9c1037
commit 3f5adf1ba9
3 changed files with 316 additions and 1 deletions

View File

@ -10,7 +10,7 @@
<p align="center">get a json value quickly</a></p> <p align="center">get a json value quickly</a></p>
GJSON is a Go package that provides a [fast](#performance) and [simple](#get-a-value) way to get values from a json document. GJSON is a Go package that provides a [fast](#performance) and [simple](#get-a-value) way to get values from a json document.
It has features such as [one line retrieval](#get-a-value), [dot notation paths](#path-syntax), [iteration](#iterate-through-an-object-or-array), and [map unmarshalling](#unmarshal-to-a-map). It has features such as [one line retrieval](#get-a-value), [dot notation paths](#path-syntax), [iteration](#iterate-through-an-object-or-array). It can also [unmarshal](#unmarshalling) 3 to 4 times faster than the standard Go `json/encoding` unmarshaller.
Getting Started Getting Started
=============== ===============
@ -233,6 +233,58 @@ if gjson.Get(json, "name.last").Exists(){
} }
``` ```
## Unmarshalling
There's a `gjson.Unmarshal` function that loads json data into the value.
It's a drop in replacement for `json.Unmarshal` and you can typically see a
3 to 4 times boost in performance without the need for external tools or
generators.
This function works almost identically to `json.Unmarshal` except that it
expects the json to be well-formed prior to being called. Invalid json
will not panic, but it may return back unexpected results. Therefore the
return value of this function will always be nil.
Another difference is that `gjson.Unmarshal` will automatically attempt to
convert JSON values to any Go type. For example, the JSON string "100" or
the JSON number 100 can be equally assigned to Go string, int, byte, uint64,
etc. This rule applies to all types.
```go
package main
import (
"fmt"
"github.com/tidwall/gjson"
)
type Animal struct {
Type string `json:"type"`
Sound string `json:"sound"`
Age int `json:"age"`
}
var json = `{
"type": "Dog",
"Sound": "Bark",
"Age": "11"
}`
func main() {
var dog Animal
gjson.Unmarshal([]byte(json), &dog)
fmt.Printf("type: %s, sound: %s, age: %d\n", dog.Type, dog.Sound, dog.Age)
}
```
This will print:
```
type: Dog, sound: Bark, age: 11
```
## Unmarshal to a map ## Unmarshal to a map
To unmarshal to a `map[string]interface{}`: To unmarshal to a `map[string]interface{}`:

121
gjson.go
View File

@ -2,8 +2,12 @@
package gjson package gjson
import ( import (
"encoding/base64"
"encoding/json"
"reflect" "reflect"
"strconv" "strconv"
"strings"
"sync"
"time" "time"
"unicode/utf16" "unicode/utf16"
"unicode/utf8" "unicode/utf8"
@ -1954,3 +1958,120 @@ func getMany512(json string, i int, paths []string) ([]Result, bool) {
_, ok := parseGetMany(json, i, 0, 0, paths, completed, matches, results) _, ok := parseGetMany(json, i, 0, 0, paths, completed, matches, results)
return results, ok return results, ok
} }
var fieldsmu sync.RWMutex
var fields = make(map[string]map[string]int)
func assign(jsval Result, goval reflect.Value) {
if jsval.Type == Null {
return
}
switch goval.Kind() {
default:
case reflect.Ptr:
if !goval.IsNil() {
newval := reflect.New(goval.Elem().Type())
assign(jsval, newval.Elem())
goval.Elem().Set(newval.Elem())
} else {
newval := reflect.New(goval.Type().Elem())
assign(jsval, newval.Elem())
goval.Set(newval)
}
case reflect.Struct:
fieldsmu.RLock()
sf := fields[goval.Type().String()]
fieldsmu.RUnlock()
if sf == nil {
fieldsmu.Lock()
sf = make(map[string]int)
for i := 0; i < goval.Type().NumField(); i++ {
f := goval.Type().Field(i)
tag := strings.Split(f.Tag.Get("json"), ",")[0]
if tag != "-" {
if tag != "" {
sf[tag] = i
sf[f.Name] = i
} else {
sf[f.Name] = i
}
}
}
fields[goval.Type().String()] = sf
fieldsmu.Unlock()
}
jsval.ForEach(func(key, value Result) bool {
if idx, ok := sf[key.Str]; ok {
f := goval.Field(idx)
if f.CanSet() {
assign(value, f)
}
}
return true
})
case reflect.Slice:
if goval.Type().Elem().Kind() == reflect.Uint8 && jsval.Type == String {
data, _ := base64.StdEncoding.DecodeString(jsval.String())
goval.Set(reflect.ValueOf(data))
} else {
jsvals := jsval.Array()
slice := reflect.MakeSlice(goval.Type(), len(jsvals), len(jsvals))
for i := 0; i < len(jsvals); i++ {
assign(jsvals[i], slice.Index(i))
}
goval.Set(slice)
}
case reflect.Array:
i, n := 0, goval.Len()
jsval.ForEach(func(_, value Result) bool {
if i == n {
return false
}
assign(value, goval.Index(i))
i++
return true
})
case reflect.Map:
if goval.Type().Key().Kind() == reflect.String && goval.Type().Elem().Kind() == reflect.Interface {
goval.Set(reflect.ValueOf(jsval.Value()))
}
case reflect.Interface:
goval.Set(reflect.ValueOf(jsval.Value()))
case reflect.Bool:
goval.SetBool(jsval.Bool())
case reflect.Float32, reflect.Float64:
goval.SetFloat(jsval.Float())
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
goval.SetInt(jsval.Int())
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
goval.SetUint(jsval.Uint())
case reflect.String:
goval.SetString(jsval.String())
}
if len(goval.Type().PkgPath()) > 0 {
v := goval.Addr()
if v.Type().NumMethod() > 0 {
if u, ok := v.Interface().(json.Unmarshaler); ok {
u.UnmarshalJSON([]byte(jsval.Raw))
}
}
}
}
// Unmarshal loads the JSON data into the value pointed to by v.
//
// This function works almost identically to json.Unmarshal except that it
// expects that the json is well-formed prior to being called. Invalid json
// will not panic, but it may return back unexpected results. Therefore the
// return value of this function will always be nil.
//
// Another difference is that gjson.Unmarshal will automatically attempt to
// convert JSON values to any Go type. For example, the JSON string "100" or
// the JSON number 100 can be equally assigned to Go string, int, byte, uint64,
// etc. This rule applies to all types.
func Unmarshal(data []byte, v interface{}) error {
if v := reflect.ValueOf(v); v.Kind() == reflect.Ptr {
assign(ParseBytes(data), v)
}
return nil
}

View File

@ -7,6 +7,7 @@ import (
"fmt" "fmt"
"io" "io"
"math/rand" "math/rand"
"reflect"
"strings" "strings"
"testing" "testing"
"time" "time"
@ -787,6 +788,119 @@ func TestRandomMany(t *testing.T) {
} }
} }
type ComplicatedType struct {
unsettable int
Tagged string `json:"tagged"`
NotTagged bool
Nested struct {
Yellow string `json:"yellow"`
}
NestedTagged struct {
Green string
Map map[string]interface{}
Ints struct {
Int int `json:"int"`
Int8 int8
Int16 int16
Int32 int32
Int64 int64 `json:"int64"`
}
Uints struct {
Uint uint
Uint8 uint8
Uint16 uint16
Uint32 uint32
Uint64 uint64
}
Floats struct {
Float64 float64
Float32 float32
}
Byte byte
Bool bool
} `json:"nestedTagged"`
LeftOut string `json:"-"`
SelfPtr *ComplicatedType
SelfSlice []ComplicatedType
SelfSlicePtr []*ComplicatedType
SelfPtrSlice *[]ComplicatedType
Interface interface{} `json:"interface"`
Array [3]int
Time time.Time `json:"time"`
Binary []byte
NonBinary []byte
}
var complicatedJSON = `
{
"tagged": "OK",
"Tagged": "KO",
"NotTagged": true,
"unsettable": 101,
"Nested": {
"Yellow": "Green",
"yellow": "yellow"
},
"nestedTagged": {
"Green": "Green",
"Map": {
"this": "that",
"and": "the other thing"
},
"Ints": {
"Uint": 99,
"Uint16": 16,
"Uint32": 32,
"Uint64": 65
},
"Uints": {
"int": -99,
"Int": -98,
"Int16": -16,
"Int32": -32,
"int64": -64,
"Int64": -65
},
"Uints": {
"Float32": 32.32,
"Float64": 64.64
},
"Byte": 254,
"Bool": true
},
"LeftOut": "you shouldn't be here",
"SelfPtr": {"tagged":"OK","nestedTagged":{"Ints":{"Uint32":32}}},
"SelfSlice": [{"tagged":"OK","nestedTagged":{"Ints":{"Uint32":32}}}],
"SelfSlicePtr": [{"tagged":"OK","nestedTagged":{"Ints":{"Uint32":32}}}],
"SelfPtrSlice": [{"tagged":"OK","nestedTagged":{"Ints":{"Uint32":32}}}],
"interface": "Tile38 Rocks!",
"Interface": "Please Download",
"Array": [0,2,3,4,5],
"time": "2017-05-07T13:24:43-07:00",
"Binary": "R0lGODlhPQBEAPeo",
"NonBinary": [9,3,100,115]
}
`
func TestUnmarshal(t *testing.T) {
var s1 ComplicatedType
var s2 ComplicatedType
if err := json.Unmarshal([]byte(complicatedJSON), &s1); err != nil {
t.Fatal(err)
}
if err := Unmarshal([]byte(complicatedJSON), &s2); err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(&s1, &s2) {
t.Fatal("not equal")
}
var str string
if err := json.Unmarshal([]byte(Get(complicatedJSON, "LeftOut").Raw), &str); err != nil {
t.Fatal(err)
}
assert(t, str == Get(complicatedJSON, "LeftOut").String())
}
type BenchStruct struct { type BenchStruct struct {
Widget struct { Widget struct {
Window struct { Window struct {
@ -900,6 +1014,34 @@ func BenchmarkGJSONUnmarshalMap(t *testing.B) {
t.N *= len(benchPaths) // because we are running against 3 paths t.N *= len(benchPaths) // because we are running against 3 paths
} }
func BenchmarkGJSONUnmarshalStruct(t *testing.B) {
t.ReportAllocs()
t.ResetTimer()
for i := 0; i < t.N; i++ {
for j := 0; j < len(benchPaths); j++ {
var s BenchStruct
if err := Unmarshal([]byte(exampleJSON), &s); err != nil {
t.Fatal(err)
}
switch benchPaths[j] {
case "widget.window.name":
if s.Widget.Window.Name == "" {
t.Fatal("did not find the value")
}
case "widget.image.hOffset":
if s.Widget.Image.HOffset == 0 {
t.Fatal("did not find the value")
}
case "widget.text.onMouseUp":
if s.Widget.Text.OnMouseUp == "" {
t.Fatal("did not find the value")
}
}
}
}
t.N *= len(benchPaths) // because we are running against 3 paths
}
func BenchmarkJSONUnmarshalMap(t *testing.B) { func BenchmarkJSONUnmarshalMap(t *testing.B) {
t.ReportAllocs() t.ReportAllocs()
t.ResetTimer() t.ResetTimer()