mirror
/
gjson
mirror of https://github.com/tidwall/gjson.git
1
0
Fork 1
Code Issues Packages Projects Releases Wiki Activity

Added modifiers and path chaining 

A modifier is a path component that performs custom processing on
the json.

Multiple paths can be "chained" together using the pipe character.
This is useful for getting results from a modified query.

See the README file for more information.
This commit is contained in:
tidwall 2019-02-16 18:29:39 -07:00
parent 5d7556ad3d
commit 1ed2249f74
5 changed files with 351 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

138
README.md
View File

@ -98,36 +98,6 @@ friends.#[first%"D*"].last >> "Murphy"
friends.#[first!%"D*"].last >> "Craig"
```
## JSON Lines
There's support for [JSON Lines](http://jsonlines.org/) using the `..` prefix, which treats a multilined document as an array.
For example:
```
{"name": "Gilbert", "age": 61}
{"name": "Alexa", "age": 34}
{"name": "May", "age": 57}
{"name": "Deloise", "age": 44}
```
```
..# >> 4
..1 >> {"name": "Alexa", "age": 34}
..3 >> {"name": "Deloise", "age": 44}
..#.name >> ["Gilbert","Alexa","May","Deloise"]
..#[name="May"].age >> 57
```
The `ForEachLines` function will iterate through JSON lines.
```go
gjson.ForEachLine(json, func(line gjson.Result) bool{
println(line.String())
return true
})
```
## Result Type
GJSON supports the json types `string`, `number`, `bool`, and `null`.
@ -194,6 +164,114 @@ result.Int() int64 // -9223372036854775808 to 9223372036854775807
result.Uint() int64 // 0 to 18446744073709551615
```
## Modifiers and path chaining
New in version 1.2 is support for modifier functions and path chaining.
A modifier is a path component that performs custom processing on the
json.
Multiple paths can be "chained" together using the pipe character.
This is useful for getting results from a modified query.
For example, using the built-in `@reverse` modifier on the above json document,
we'll get `children` array and reverse the order:
```
"children|@reverse" >> ["Jack","Alex","Sara"]
"children|@reverse|#" >> "Jack"
```
There are currently three built-in modifiers:
- `@reverse`: Reverse an array or the members of an object.
- `@ugly`: Remove all whitespace from a json document.
- `@pretty`: Make the json document more human readable.
### Modifier arguments
A modifier may accept an optional argument. The argument can be a valid JSON
document or just characters.
For example, the `@pretty` modifier takes a json object as its argument.
```
@pretty:{"sortKeys":true}
```
Which makes the json pretty and orders all of its keys.
```json
{
"age":37,
"children": ["Sara","Alex","Jack"],
"fav.movie": "Deer Hunter",
"friends": [
{"age": 44, "first": "Dale", "last": "Murphy"},
{"age": 68, "first": "Roger", "last": "Craig"},
{"age": 47, "first": "Jane", "last": "Murphy"}
],
"name": {"first": "Tom", "last": "Anderson"}
}
```
*The full list of `@pretty` options are `sortKeys`, `indent`, `prefix`, and `width`.
Please see [Pretty Options](https://github.com/tidwall/pretty#customized-output) for more information.*
### Custom modifiers
You can also add custom modifiers.
For example, here we create a modifier that makes the entire json document upper
or lower case.
```go
gjson.AddModifier("case", func(json, arg string) string {
if arg == "upper" {
return strings.ToUpper(json)
}
if arg == "lower" {
return strings.ToLower(json)
}
return json
})
```
```
"children|@case:upper" >> ["SARA","ALEX","JACK"]
"children|@case:lower|@reverse" >> ["jack","alex","sara"]
```
## JSON Lines
There's support for [JSON Lines](http://jsonlines.org/) using the `..` prefix, which treats a multilined document as an array.
For example:
```
{"name": "Gilbert", "age": 61}
{"name": "Alexa", "age": 34}
{"name": "May", "age": 57}
{"name": "Deloise", "age": 44}
```
```
..# >> 4
..1 >> {"name": "Alexa", "age": 34}
..3 >> {"name": "Deloise", "age": 44}
..#.name >> ["Gilbert","Alexa","May","Deloise"]
..#[name="May"].age >> 57
```
The `ForEachLines` function will iterate through JSON lines.
```go
gjson.ForEachLine(json, func(line gjson.Result) bool{
println(line.String())
return true
})
```
## Get nested array values
Suppose you want all the last names from the following json:

205
gjson.go
View File

@ -15,6 +15,7 @@ import (
"unicode/utf8"
"github.com/tidwall/match"
"github.com/tidwall/pretty"
)
// Type is Result type
@ -695,6 +696,8 @@ func parseLiteral(json string, i int) (int, string) {
type arrayPathResult struct {
part string
path string
pipe string
piped bool
more bool
alogok bool
arrch bool
@ -710,6 +713,13 @@ type arrayPathResult struct {
func parseArrayPath(path string) (r arrayPathResult) {
for i := 0; i < len(path); i++ {
if !DisableChaining {
if path[i] == '|' {
r.part = path[:i]
r.pipe = path[i+1:]
return
}
}
if path[i] == '.' {
r.part = path[:i]
r.path = path[i+1:]
@ -828,15 +838,28 @@ func parseArrayPath(path string) (r arrayPathResult) {
return
}
// DisableChaining will disable the chaining (pipe) syntax
var DisableChaining = false
type objectPathResult struct {
part string
path string
wild bool
more bool
part string
path string
pipe string
piped bool
wild bool
more bool
}
func parseObjectPath(path string) (r objectPathResult) {
for i := 0; i < len(path); i++ {
if !DisableChaining {
if path[i] == '|' {
r.part = path[:i]
r.pipe = path[i+1:]
r.piped = true
return
}
}
if path[i] == '.' {
r.part = path[:i]
r.path = path[i+1:]
@ -934,6 +957,10 @@ func parseObject(c *parseContext, i int, path string) (int, bool) {
var pmatch, kesc, vesc, ok, hit bool
var key, val string
rp := parseObjectPath(path)
if !rp.more && rp.piped {
c.pipe = rp.pipe
c.piped = true
}
for i < len(c.json) {
for ; i < len(c.json); i++ {
if c.json[i] == '"' {
@ -1159,6 +1186,10 @@ func parseArray(c *parseContext, i int, path string) (int, bool) {
partidx = int(n)
}
}
if !rp.more && rp.piped {
c.pipe = rp.pipe
c.piped = true
}
for i < len(c.json)+1 {
if !rp.arrch {
pmatch = partidx == h
@ -1353,6 +1384,8 @@ func ForEachLine(json string, iterator func(line Result) bool) {
type parseContext struct {
json string
value Result
pipe string
piped bool
calcd bool
lines bool
}
@ -1390,6 +1423,22 @@ type parseContext struct {
// If you are consuming JSON from an unpredictable source then you may want to
// use the Valid function first.
func Get(json, path string) Result {
if !DisableModifiers {
if len(path) > 1 && path[0] == '@' {
// possible modifier
var ok bool
var rjson string
path, rjson, ok = execModifier(json, path)
if ok {
if len(path) > 0 && path[0] == '|' {
res := Get(rjson, path[1:])
res.Index = 0
return res
}
return Parse(rjson)
}
}
}
var i int
var c = &parseContext{json: json}
if len(path) >= 2 && path[0] == '.' && path[1] == '.' {
@ -1409,6 +1458,11 @@ func Get(json, path string) Result {
}
}
}
if c.piped {
res := c.value.Get(c.pipe)
res.Index = 0
return res
}
fillIndex(json, c)
return c.value
}
@ -2114,3 +2168,146 @@ func floatToInt(f float64) (n int64, ok bool) {
}
return 0, false
}
// execModifier parses the path to find a matching modifier function.
// then input expects that the path already starts with a '@'
func execModifier(json, path string) (pathOut, res string, ok bool) {
name := path[1:]
var hasArgs bool
for i := 1; i < len(path); i++ {
if path[i] == ':' {
pathOut = path[i+1:]
name = path[1:i]
hasArgs = len(pathOut) > 0
break
}
if !DisableChaining {
if path[i] == '|' {
pathOut = path[i:]
name = path[1:i]
break
}
}
}
if fn, ok := modifiers[name]; ok {
var args string
if hasArgs {
var parsedArgs bool
switch pathOut[0] {
case '{', '[', '"':
res := Parse(pathOut)
if res.Exists() {
_, args = parseSquash(pathOut, 0)
pathOut = pathOut[len(args):]
parsedArgs = true
}
}
if !parsedArgs {
idx := -1
if !DisableChaining {
idx = strings.IndexByte(pathOut, '|')
}
if idx == -1 {
args = pathOut
pathOut = ""
} else {
args = pathOut[:idx]
pathOut = pathOut[idx:]
}
}
}
return pathOut, fn(json, args), true
}
return pathOut, res, false
}
// DisableModifiers will disable the modifier syntax
var DisableModifiers = false
var modifiers = map[string]func(json, arg string) string{
"pretty": modPretty,
"ugly": modUgly,
"reverse": modReverse,
}
// AddModifier binds a custom modifier command to the GJSON syntax.
// This operation is not thread safe and should be executed prior to
// using all other gjson function.
func AddModifier(name string, fn func(json, arg string) string) {
modifiers[name] = fn
}
// ModifierExists returns true when the specified modifier exists.
func ModifierExists(name string, fn func(json, arg string) string) bool {
_, ok := modifiers[name]
return ok
}
// @pretty modifier makes the json look nice.
func modPretty(json, arg string) string {
if len(arg) > 0 {
opts := *pretty.DefaultOptions
Parse(arg).ForEach(func(key, value Result) bool {
switch key.String() {
case "sortKeys":
opts.SortKeys = value.Bool()
case "indent":
opts.Indent = value.String()
case "prefix":
opts.Prefix = value.String()
case "width":
opts.Width = int(value.Int())
}
return true
})
return bytesString(pretty.PrettyOptions(stringBytes(json), &opts))
}
return bytesString(pretty.Pretty(stringBytes(json)))
}
// @ugly modifier removes all whitespace.
func modUgly(json, arg string) string {
return bytesString(pretty.Ugly(stringBytes(json)))
}
// @reverse reverses array elements or root object members.
func modReverse(json, arg string) string {
res := Parse(json)
if res.IsArray() {
var values []Result
res.ForEach(func(_, value Result) bool {
values = append(values, value)
return true
})
out := make([]byte, 0, len(json))
out = append(out, '[')
for i, j := len(values)-1, 0; i >= 0; i, j = i-1, j+1 {
if j > 0 {
out = append(out, ',')
}
out = append(out, values[i].Raw...)
}
out = append(out, ']')
return bytesString(out)
}
if res.IsObject() {
var keyValues []Result
res.ForEach(func(key, value Result) bool {
keyValues = append(keyValues, key, value)
return true
})
out := make([]byte, 0, len(json))
out = append(out, '{')
for i, j := len(keyValues)-2, 0; i >= 0; i, j = i-2, j+1 {
if j > 0 {
out = append(out, ',')
}
out = append(out, keyValues[i+0].Raw...)
out = append(out, ':')
out = append(out, keyValues[i+1].Raw...)
}
out = append(out, '}')
return bytesString(out)
}
return json
}

4
gjson_gae.go
View File

@ -12,3 +12,7 @@ func fillIndex(json string, c *parseContext) {
func stringBytes(s string) []byte {
return []byte(s)
}
func bytesString(b []byte) string {
return string(b)
}

4
gjson_ngae.go
View File

@ -75,3 +75,7 @@ func stringBytes(s string) []byte {
Cap: len(s),
}))
}
func bytesString(b []byte) string {
return *(*string)(unsafe.Pointer(&b))
}

34
gjson_test.go
View File

@ -11,6 +11,8 @@ import (
"strings"
"testing"
"time"
"github.com/tidwall/pretty"
)
// TestRandomData is a fuzzing test that throws random data at the Parse
@ -1451,3 +1453,35 @@ func BenchmarkGoStdlibValidBytes(b *testing.B) {
json.Valid(complicatedJSON)
}
}
func TestModifier(t *testing.T) {
json := `{"other":{"hello":"world"},"arr":[1,2,3,4,5,6]}`
opts := *pretty.DefaultOptions
opts.SortKeys = true
exp := string(pretty.PrettyOptions([]byte(json), &opts))
res := Get(json, `@pretty:{"sortKeys":true}`).String()
if res != exp {
t.Fatalf("expected '%v', got '%v'", exp, res)
}
res = Get(res, "@pretty|@reverse|@ugly").String()
if res != json {
t.Fatalf("expected '%v', got '%v'", json, res)
}
res = Get(res, "@pretty|@reverse|arr|@reverse|2").String()
if res != "4" {
t.Fatalf("expected '%v', got '%v'", "4", res)
}
AddModifier("case", func(json, arg string) string {
if arg == "upper" {
return strings.ToUpper(json)
}
if arg == "lower" {
return strings.ToLower(json)
}
return json
})
res = Get(json, "other|@case:upper").String()
if res != `{"HELLO":"WORLD"}` {
t.Fatalf("expected '%v', got '%v'", `{"HELLO":"WORLD"}`, res)
}
}