diff --git a/README.md b/README.md index a7484c6..1e38644 100644 --- a/README.md +++ b/README.md @@ -98,36 +98,6 @@ friends.#[first%"D*"].last >> "Murphy" friends.#[first!%"D*"].last >> "Craig" ``` -## JSON Lines - -There's support for [JSON Lines](http://jsonlines.org/) using the `..` prefix, which treats a multilined document as an array. - -For example: - -``` -{"name": "Gilbert", "age": 61} -{"name": "Alexa", "age": 34} -{"name": "May", "age": 57} -{"name": "Deloise", "age": 44} -``` - -``` -..# >> 4 -..1 >> {"name": "Alexa", "age": 34} -..3 >> {"name": "Deloise", "age": 44} -..#.name >> ["Gilbert","Alexa","May","Deloise"] -..#[name="May"].age >> 57 -``` - -The `ForEachLines` function will iterate through JSON lines. - -```go -gjson.ForEachLine(json, func(line gjson.Result) bool{ - println(line.String()) - return true -}) -``` - ## Result Type GJSON supports the json types `string`, `number`, `bool`, and `null`. @@ -194,6 +164,114 @@ result.Int() int64 // -9223372036854775808 to 9223372036854775807 result.Uint() int64 // 0 to 18446744073709551615 ``` +## Modifiers and path chaining + +New in version 1.2 is support for modifier functions and path chaining. + +A modifier is a path component that performs custom processing on the +json. + +Multiple paths can be "chained" together using the pipe character. +This is useful for getting results from a modified query. + +For example, using the built-in `@reverse` modifier on the above json document, +we'll get `children` array and reverse the order: + +``` +"children|@reverse" >> ["Jack","Alex","Sara"] +"children|@reverse|#" >> "Jack" +``` + +There are currently three built-in modifiers: + +- `@reverse`: Reverse an array or the members of an object. +- `@ugly`: Remove all whitespace from a json document. +- `@pretty`: Make the json document more human readable. + +### Modifier arguments + +A modifier may accept an optional argument. The argument can be a valid JSON +document or just characters. + +For example, the `@pretty` modifier takes a json object as its argument. + +``` +@pretty:{"sortKeys":true} +``` + +Which makes the json pretty and orders all of its keys. + +```json +{ + "age":37, + "children": ["Sara","Alex","Jack"], + "fav.movie": "Deer Hunter", + "friends": [ + {"age": 44, "first": "Dale", "last": "Murphy"}, + {"age": 68, "first": "Roger", "last": "Craig"}, + {"age": 47, "first": "Jane", "last": "Murphy"} + ], + "name": {"first": "Tom", "last": "Anderson"} +} +``` + +*The full list of `@pretty` options are `sortKeys`, `indent`, `prefix`, and `width`. +Please see [Pretty Options](https://github.com/tidwall/pretty#customized-output) for more information.* + +### Custom modifiers + +You can also add custom modifiers. + +For example, here we create a modifier that makes the entire json document upper +or lower case. + +```go +gjson.AddModifier("case", func(json, arg string) string { + if arg == "upper" { + return strings.ToUpper(json) + } + if arg == "lower" { + return strings.ToLower(json) + } + return json +}) +``` + +``` +"children|@case:upper" >> ["SARA","ALEX","JACK"] +"children|@case:lower|@reverse" >> ["jack","alex","sara"] +``` + +## JSON Lines + +There's support for [JSON Lines](http://jsonlines.org/) using the `..` prefix, which treats a multilined document as an array. + +For example: + +``` +{"name": "Gilbert", "age": 61} +{"name": "Alexa", "age": 34} +{"name": "May", "age": 57} +{"name": "Deloise", "age": 44} +``` + +``` +..# >> 4 +..1 >> {"name": "Alexa", "age": 34} +..3 >> {"name": "Deloise", "age": 44} +..#.name >> ["Gilbert","Alexa","May","Deloise"] +..#[name="May"].age >> 57 +``` + +The `ForEachLines` function will iterate through JSON lines. + +```go +gjson.ForEachLine(json, func(line gjson.Result) bool{ + println(line.String()) + return true +}) +``` + ## Get nested array values Suppose you want all the last names from the following json: diff --git a/gjson.go b/gjson.go index 7722ad9..eba5284 100644 --- a/gjson.go +++ b/gjson.go @@ -15,6 +15,7 @@ import ( "unicode/utf8" "github.com/tidwall/match" + "github.com/tidwall/pretty" ) // Type is Result type @@ -695,6 +696,8 @@ func parseLiteral(json string, i int) (int, string) { type arrayPathResult struct { part string path string + pipe string + piped bool more bool alogok bool arrch bool @@ -710,6 +713,13 @@ type arrayPathResult struct { func parseArrayPath(path string) (r arrayPathResult) { for i := 0; i < len(path); i++ { + if !DisableChaining { + if path[i] == '|' { + r.part = path[:i] + r.pipe = path[i+1:] + return + } + } if path[i] == '.' { r.part = path[:i] r.path = path[i+1:] @@ -828,15 +838,28 @@ func parseArrayPath(path string) (r arrayPathResult) { return } +// DisableChaining will disable the chaining (pipe) syntax +var DisableChaining = false + type objectPathResult struct { - part string - path string - wild bool - more bool + part string + path string + pipe string + piped bool + wild bool + more bool } func parseObjectPath(path string) (r objectPathResult) { for i := 0; i < len(path); i++ { + if !DisableChaining { + if path[i] == '|' { + r.part = path[:i] + r.pipe = path[i+1:] + r.piped = true + return + } + } if path[i] == '.' { r.part = path[:i] r.path = path[i+1:] @@ -934,6 +957,10 @@ func parseObject(c *parseContext, i int, path string) (int, bool) { var pmatch, kesc, vesc, ok, hit bool var key, val string rp := parseObjectPath(path) + if !rp.more && rp.piped { + c.pipe = rp.pipe + c.piped = true + } for i < len(c.json) { for ; i < len(c.json); i++ { if c.json[i] == '"' { @@ -1159,6 +1186,10 @@ func parseArray(c *parseContext, i int, path string) (int, bool) { partidx = int(n) } } + if !rp.more && rp.piped { + c.pipe = rp.pipe + c.piped = true + } for i < len(c.json)+1 { if !rp.arrch { pmatch = partidx == h @@ -1353,6 +1384,8 @@ func ForEachLine(json string, iterator func(line Result) bool) { type parseContext struct { json string value Result + pipe string + piped bool calcd bool lines bool } @@ -1390,6 +1423,22 @@ type parseContext struct { // If you are consuming JSON from an unpredictable source then you may want to // use the Valid function first. func Get(json, path string) Result { + if !DisableModifiers { + if len(path) > 1 && path[0] == '@' { + // possible modifier + var ok bool + var rjson string + path, rjson, ok = execModifier(json, path) + if ok { + if len(path) > 0 && path[0] == '|' { + res := Get(rjson, path[1:]) + res.Index = 0 + return res + } + return Parse(rjson) + } + } + } var i int var c = &parseContext{json: json} if len(path) >= 2 && path[0] == '.' && path[1] == '.' { @@ -1409,6 +1458,11 @@ func Get(json, path string) Result { } } } + if c.piped { + res := c.value.Get(c.pipe) + res.Index = 0 + return res + } fillIndex(json, c) return c.value } @@ -2114,3 +2168,146 @@ func floatToInt(f float64) (n int64, ok bool) { } return 0, false } + +// execModifier parses the path to find a matching modifier function. +// then input expects that the path already starts with a '@' +func execModifier(json, path string) (pathOut, res string, ok bool) { + name := path[1:] + var hasArgs bool + for i := 1; i < len(path); i++ { + if path[i] == ':' { + pathOut = path[i+1:] + name = path[1:i] + hasArgs = len(pathOut) > 0 + break + } + if !DisableChaining { + if path[i] == '|' { + pathOut = path[i:] + name = path[1:i] + break + } + } + } + if fn, ok := modifiers[name]; ok { + var args string + if hasArgs { + var parsedArgs bool + switch pathOut[0] { + case '{', '[', '"': + res := Parse(pathOut) + if res.Exists() { + _, args = parseSquash(pathOut, 0) + pathOut = pathOut[len(args):] + parsedArgs = true + } + } + if !parsedArgs { + idx := -1 + if !DisableChaining { + idx = strings.IndexByte(pathOut, '|') + } + if idx == -1 { + args = pathOut + pathOut = "" + } else { + args = pathOut[:idx] + pathOut = pathOut[idx:] + } + } + } + return pathOut, fn(json, args), true + } + return pathOut, res, false +} + +// DisableModifiers will disable the modifier syntax +var DisableModifiers = false + +var modifiers = map[string]func(json, arg string) string{ + "pretty": modPretty, + "ugly": modUgly, + "reverse": modReverse, +} + +// AddModifier binds a custom modifier command to the GJSON syntax. +// This operation is not thread safe and should be executed prior to +// using all other gjson function. +func AddModifier(name string, fn func(json, arg string) string) { + modifiers[name] = fn +} + +// ModifierExists returns true when the specified modifier exists. +func ModifierExists(name string, fn func(json, arg string) string) bool { + _, ok := modifiers[name] + return ok +} + +// @pretty modifier makes the json look nice. +func modPretty(json, arg string) string { + if len(arg) > 0 { + opts := *pretty.DefaultOptions + Parse(arg).ForEach(func(key, value Result) bool { + switch key.String() { + case "sortKeys": + opts.SortKeys = value.Bool() + case "indent": + opts.Indent = value.String() + case "prefix": + opts.Prefix = value.String() + case "width": + opts.Width = int(value.Int()) + } + return true + }) + return bytesString(pretty.PrettyOptions(stringBytes(json), &opts)) + } + return bytesString(pretty.Pretty(stringBytes(json))) +} + +// @ugly modifier removes all whitespace. +func modUgly(json, arg string) string { + return bytesString(pretty.Ugly(stringBytes(json))) +} + +// @reverse reverses array elements or root object members. +func modReverse(json, arg string) string { + res := Parse(json) + if res.IsArray() { + var values []Result + res.ForEach(func(_, value Result) bool { + values = append(values, value) + return true + }) + out := make([]byte, 0, len(json)) + out = append(out, '[') + for i, j := len(values)-1, 0; i >= 0; i, j = i-1, j+1 { + if j > 0 { + out = append(out, ',') + } + out = append(out, values[i].Raw...) + } + out = append(out, ']') + return bytesString(out) + } + if res.IsObject() { + var keyValues []Result + res.ForEach(func(key, value Result) bool { + keyValues = append(keyValues, key, value) + return true + }) + out := make([]byte, 0, len(json)) + out = append(out, '{') + for i, j := len(keyValues)-2, 0; i >= 0; i, j = i-2, j+1 { + if j > 0 { + out = append(out, ',') + } + out = append(out, keyValues[i+0].Raw...) + out = append(out, ':') + out = append(out, keyValues[i+1].Raw...) + } + out = append(out, '}') + return bytesString(out) + } + return json +} diff --git a/gjson_gae.go b/gjson_gae.go index 6a316e5..9586903 100644 --- a/gjson_gae.go +++ b/gjson_gae.go @@ -12,3 +12,7 @@ func fillIndex(json string, c *parseContext) { func stringBytes(s string) []byte { return []byte(s) } + +func bytesString(b []byte) string { + return string(b) +} diff --git a/gjson_ngae.go b/gjson_ngae.go index f606581..bc608b5 100644 --- a/gjson_ngae.go +++ b/gjson_ngae.go @@ -75,3 +75,7 @@ func stringBytes(s string) []byte { Cap: len(s), })) } + +func bytesString(b []byte) string { + return *(*string)(unsafe.Pointer(&b)) +} diff --git a/gjson_test.go b/gjson_test.go index d718ba0..142067a 100644 --- a/gjson_test.go +++ b/gjson_test.go @@ -11,6 +11,8 @@ import ( "strings" "testing" "time" + + "github.com/tidwall/pretty" ) // TestRandomData is a fuzzing test that throws random data at the Parse @@ -1451,3 +1453,35 @@ func BenchmarkGoStdlibValidBytes(b *testing.B) { json.Valid(complicatedJSON) } } + +func TestModifier(t *testing.T) { + json := `{"other":{"hello":"world"},"arr":[1,2,3,4,5,6]}` + opts := *pretty.DefaultOptions + opts.SortKeys = true + exp := string(pretty.PrettyOptions([]byte(json), &opts)) + res := Get(json, `@pretty:{"sortKeys":true}`).String() + if res != exp { + t.Fatalf("expected '%v', got '%v'", exp, res) + } + res = Get(res, "@pretty|@reverse|@ugly").String() + if res != json { + t.Fatalf("expected '%v', got '%v'", json, res) + } + res = Get(res, "@pretty|@reverse|arr|@reverse|2").String() + if res != "4" { + t.Fatalf("expected '%v', got '%v'", "4", res) + } + AddModifier("case", func(json, arg string) string { + if arg == "upper" { + return strings.ToUpper(json) + } + if arg == "lower" { + return strings.ToLower(json) + } + return json + }) + res = Get(json, "other|@case:upper").String() + if res != `{"HELLO":"WORLD"}` { + t.Fatalf("expected '%v', got '%v'", `{"HELLO":"WORLD"}`, res) + } +}