Switch from encoding/json -> jsoniter (#570)

* Switch from encoding/json -> jsoniter

Signed-off-by: Thomas Jackson <jacksontj.89@gmail.com>
This commit is contained in:
Thomas Jackson 2019-05-28 04:45:06 -07:00 committed by Krasi Georgiev
parent 388f9869ac
commit 7e60220fd3
6 changed files with 328 additions and 16 deletions

View File

@ -20,8 +20,6 @@ import (
"net/http/httptest"
"net/url"
"testing"
"github.com/prometheus/tsdb/testutil"
)
func TestConfig(t *testing.T) {
@ -148,7 +146,9 @@ func TestDoGetFallback(t *testing.T) {
defer server.Close()
u, err := url.Parse(server.URL)
testutil.Ok(t, err)
if err != nil {
t.Fatal(err)
}
client := &httpClient{client: *(server.Client())}
// Do a post, and ensure that the post succeeds.
@ -158,7 +158,7 @@ func TestDoGetFallback(t *testing.T) {
}
resp := &testResponse{}
if err := json.Unmarshal(b, resp); err != nil {
testutil.Ok(t, err)
t.Fatal(err)
}
if resp.Method != http.MethodPost {
t.Fatalf("Mismatch method")
@ -174,7 +174,7 @@ func TestDoGetFallback(t *testing.T) {
t.Fatalf("Error doing local request: %v", err)
}
if err := json.Unmarshal(b, resp); err != nil {
testutil.Ok(t, err)
t.Fatal(err)
}
if resp.Method != http.MethodGet {
t.Fatalf("Mismatch method")

View File

@ -17,18 +17,105 @@ package v1
import (
"context"
"encoding/json"
"errors"
"fmt"
"math"
"net/http"
"strconv"
"strings"
"time"
"unsafe"
json "github.com/json-iterator/go"
"github.com/prometheus/common/model"
"github.com/prometheus/client_golang/api"
"github.com/prometheus/common/model"
)
func init() {
json.RegisterTypeEncoderFunc("model.SamplePair", marshalPointJSON, marshalPointJSONIsEmpty)
json.RegisterTypeDecoderFunc("model.SamplePair", unMarshalPointJSON)
}
func unMarshalPointJSON(ptr unsafe.Pointer, iter *json.Iterator) {
p := (*model.SamplePair)(ptr)
if !iter.ReadArray() {
iter.ReportError("unmarshal model.SamplePair", "SamplePair must be [timestamp, value]")
return
}
t := iter.ReadNumber()
if err := p.Timestamp.UnmarshalJSON([]byte(t)); err != nil {
iter.ReportError("unmarshal model.SamplePair", err.Error())
return
}
if !iter.ReadArray() {
iter.ReportError("unmarshal model.SamplePair", "SamplePair missing value")
return
}
f, err := strconv.ParseFloat(iter.ReadString(), 64)
if err != nil {
iter.ReportError("unmarshal model.SamplePair", err.Error())
return
}
p.Value = model.SampleValue(f)
if iter.ReadArray() {
iter.ReportError("unmarshal model.SamplePair", "SamplePair has too many values, must be [timestamp, value]")
return
}
}
func marshalPointJSON(ptr unsafe.Pointer, stream *json.Stream) {
p := *((*model.SamplePair)(ptr))
stream.WriteArrayStart()
// Write out the timestamp as a float divided by 1000.
// This is ~3x faster than converting to a float.
t := int64(p.Timestamp)
if t < 0 {
stream.WriteRaw(`-`)
t = -t
}
stream.WriteInt64(t / 1000)
fraction := t % 1000
if fraction != 0 {
stream.WriteRaw(`.`)
if fraction < 100 {
stream.WriteRaw(`0`)
}
if fraction < 10 {
stream.WriteRaw(`0`)
}
stream.WriteInt64(fraction)
}
stream.WriteMore()
stream.WriteRaw(`"`)
// Taken from https://github.com/json-iterator/go/blob/master/stream_float.go#L71 as a workaround
// to https://github.com/json-iterator/go/issues/365 (jsoniter, to follow json standard, doesn't allow inf/nan)
buf := stream.Buffer()
abs := math.Abs(float64(p.Value))
fmt := byte('f')
// Note: Must use float32 comparisons for underlying float32 value to get precise cutoffs right.
if abs != 0 {
if abs < 1e-6 || abs >= 1e21 {
fmt = 'e'
fmt = 'e'
}
}
buf = strconv.AppendFloat(buf, float64(p.Value), fmt, -1, 64)
stream.SetBuffer(buf)
stream.WriteRaw(`"`)
stream.WriteArrayEnd()
}
func marshalPointJSONIsEmpty(ptr unsafe.Pointer) bool {
return false
}
const (
statusAPIError = 422

View File

@ -0,0 +1,112 @@
// Copyright 2019 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package v1
import (
"encoding/json"
"strconv"
"testing"
"time"
jsoniter "github.com/json-iterator/go"
"github.com/prometheus/common/model"
)
func generateData(timeseries, datapoints int) model.Matrix {
m := make(model.Matrix, 0)
for i := 0; i < timeseries; i++ {
lset := map[model.LabelName]model.LabelValue{
model.MetricNameLabel: model.LabelValue("timeseries_" + strconv.Itoa(i)),
}
now := model.Now()
values := make([]model.SamplePair, datapoints)
for x := datapoints; x > 0; x-- {
values[x-1] = model.SamplePair{
// Set the time back assuming a 15s interval. Since this is used for
// Marshal/Unmarshal testing the actual interval doesn't matter.
Timestamp: now.Add(time.Second * -15 * time.Duration(x)),
Value: model.SampleValue(float64(x)),
}
}
ss := &model.SampleStream{
Metric: model.Metric(lset),
Values: values,
}
m = append(m, ss)
}
return m
}
func BenchmarkSamplesJsonSerialization(b *testing.B) {
for _, timeseriesCount := range []int{10, 100, 1000} {
b.Run(strconv.Itoa(timeseriesCount), func(b *testing.B) {
for _, datapointCount := range []int{10, 100, 1000} {
b.Run(strconv.Itoa(datapointCount), func(b *testing.B) {
data := generateData(timeseriesCount, datapointCount)
dataBytes, err := json.Marshal(data)
if err != nil {
b.Fatalf("Error marshaling: %v", err)
}
b.Run("marshal", func(b *testing.B) {
b.Run("encoding/json", func(b *testing.B) {
b.ReportAllocs()
for i := 0; i < b.N; i++ {
if _, err := json.Marshal(data); err != nil {
b.Fatal(err)
}
}
})
b.Run("jsoniter", func(b *testing.B) {
b.ReportAllocs()
for i := 0; i < b.N; i++ {
if _, err := jsoniter.Marshal(data); err != nil {
b.Fatal(err)
}
}
})
})
b.Run("unmarshal", func(b *testing.B) {
b.Run("encoding/json", func(b *testing.B) {
b.ReportAllocs()
var m model.Matrix
for i := 0; i < b.N; i++ {
if err := json.Unmarshal(dataBytes, &m); err != nil {
b.Fatal(err)
}
}
})
b.Run("jsoniter", func(b *testing.B) {
b.ReportAllocs()
var m model.Matrix
for i := 0; i < b.N; i++ {
if err := jsoniter.Unmarshal(dataBytes, &m); err != nil {
b.Fatal(err)
}
}
})
})
})
}
})
}
}

View File

@ -15,9 +15,9 @@ package v1
import (
"context"
"encoding/json"
"errors"
"fmt"
"math"
"net/http"
"net/url"
"reflect"
@ -25,9 +25,12 @@ import (
"testing"
"time"
"github.com/prometheus/client_golang/api"
json "github.com/json-iterator/go"
"github.com/prometheus/common/model"
"github.com/prometheus/tsdb/testutil"
"github.com/prometheus/client_golang/api"
)
type apiTest struct {
@ -792,7 +795,7 @@ func TestAPIClientDo(t *testing.T) {
response: "bad json",
expectedErr: &Error{
Type: ErrBadResponse,
Msg: "invalid character 'b' looking for beginning of value",
Msg: "readObjectStart: expect { or n, but found b, error found in #1 byte of ...|bad json|..., bigger context ...|bad json|...",
},
},
{
@ -882,3 +885,99 @@ func TestAPIClientDo(t *testing.T) {
}
}
func TestSamplesJsonSerialization(t *testing.T) {
tests := []struct {
point model.SamplePair
expected string
}{
{
point: model.SamplePair{0, 0},
expected: `[0,"0"]`,
},
{
point: model.SamplePair{1, 20},
expected: `[0.001,"20"]`,
},
{
point: model.SamplePair{10, 20},
expected: `[0.010,"20"]`,
},
{
point: model.SamplePair{100, 20},
expected: `[0.100,"20"]`,
},
{
point: model.SamplePair{1001, 20},
expected: `[1.001,"20"]`,
},
{
point: model.SamplePair{1010, 20},
expected: `[1.010,"20"]`,
},
{
point: model.SamplePair{1100, 20},
expected: `[1.100,"20"]`,
},
{
point: model.SamplePair{12345678123456555, 20},
expected: `[12345678123456.555,"20"]`,
},
{
point: model.SamplePair{-1, 20},
expected: `[-0.001,"20"]`,
},
{
point: model.SamplePair{0, model.SampleValue(math.NaN())},
expected: `[0,"NaN"]`,
},
{
point: model.SamplePair{0, model.SampleValue(math.Inf(1))},
expected: `[0,"+Inf"]`,
},
{
point: model.SamplePair{0, model.SampleValue(math.Inf(-1))},
expected: `[0,"-Inf"]`,
},
{
point: model.SamplePair{0, model.SampleValue(1.2345678e6)},
expected: `[0,"1234567.8"]`,
},
{
point: model.SamplePair{0, 1.2345678e-6},
expected: `[0,"0.0000012345678"]`,
},
{
point: model.SamplePair{0, 1.2345678e-67},
expected: `[0,"1.2345678e-67"]`,
},
}
for _, test := range tests {
t.Run(test.expected, func(t *testing.T) {
b, err := json.Marshal(test.point)
if err != nil {
t.Fatal(err)
}
if string(b) != test.expected {
t.Fatalf("Mismatch marshal expected=%s actual=%s", test.expected, string(b))
}
// To test Unmarshal we will Unmarshal then re-Marshal this way we
// can do a string compare, otherwise Nan values don't show equivalence
// properly.
var sp model.SamplePair
if err = json.Unmarshal(b, &sp); err != nil {
t.Fatal(err)
}
b, err = json.Marshal(sp)
if err != nil {
t.Fatal(err)
}
if string(b) != test.expected {
t.Fatalf("Mismatch marshal expected=%s actual=%s", test.expected, string(b))
}
})
}
}

8
go.mod
View File

@ -4,8 +4,12 @@ require (
github.com/beorn7/perks v1.0.0
github.com/go-logfmt/logfmt v0.4.0 // indirect
github.com/golang/protobuf v1.3.1
github.com/json-iterator/go v1.1.6
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.1 // indirect
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90
github.com/prometheus/common v0.4.0
github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084
github.com/prometheus/common v0.4.1
github.com/prometheus/procfs v0.0.0-20190522114515-bc1a522cf7b1
github.com/prometheus/tsdb v0.7.1
github.com/stretchr/testify v1.3.0 // indirect
)

18
go.sum
View File

@ -8,6 +8,7 @@ github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
@ -24,12 +25,18 @@ github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/json-iterator/go v1.1.6 h1:MrUvLMLTMxbqFJ9kzlvat/rYZqZnW3u4wkLzWTaFwKs=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY=
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@ -41,19 +48,22 @@ github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE=
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
github.com/prometheus/common v0.4.0 h1:7etb9YClo3a6HjLzfl6rIQaU+FDfi0VSX39io3aQ+DM=
github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
github.com/prometheus/common v0.4.1 h1:K0MGApIoQvMw27RTdJkPbr3JZ7DNbtxQNyi5STVM6Kw=
github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084 h1:sofwID9zm4tzrgykg80hfFph1mryUeLRsUfoocVVmRY=
github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
github.com/prometheus/procfs v0.0.0-20190522114515-bc1a522cf7b1 h1:Lo6mRUjdS99f3zxYOUalftWHUoOGaDRqFk1+j0Q57/I=
github.com/prometheus/procfs v0.0.0-20190522114515-bc1a522cf7b1/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
github.com/prometheus/tsdb v0.7.1 h1:YZcsG11NqnK4czYLrWd9mpEuAJIHVQLwdrleYfszMAA=
github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f h1:Bl/8QSvNqXvPGPGXa2z5xUTmV7VDcZyvRZ+QQXkXTZQ=