From 7e60220fd3ab4cdeb15bf0603345967b4268a80f Mon Sep 17 00:00:00 2001 From: Thomas Jackson Date: Tue, 28 May 2019 04:45:06 -0700 Subject: [PATCH] Switch from encoding/json -> jsoniter (#570) * Switch from encoding/json -> jsoniter Signed-off-by: Thomas Jackson --- api/client_test.go | 10 +-- api/prometheus/v1/api.go | 91 +++++++++++++++++++++- api/prometheus/v1/api_bench_test.go | 112 ++++++++++++++++++++++++++++ api/prometheus/v1/api_test.go | 105 +++++++++++++++++++++++++- go.mod | 8 +- go.sum | 18 ++++- 6 files changed, 328 insertions(+), 16 deletions(-) create mode 100644 api/prometheus/v1/api_bench_test.go diff --git a/api/client_test.go b/api/client_test.go index 7877e6a..b0ea306 100644 --- a/api/client_test.go +++ b/api/client_test.go @@ -20,8 +20,6 @@ import ( "net/http/httptest" "net/url" "testing" - - "github.com/prometheus/tsdb/testutil" ) func TestConfig(t *testing.T) { @@ -148,7 +146,9 @@ func TestDoGetFallback(t *testing.T) { defer server.Close() u, err := url.Parse(server.URL) - testutil.Ok(t, err) + if err != nil { + t.Fatal(err) + } client := &httpClient{client: *(server.Client())} // Do a post, and ensure that the post succeeds. @@ -158,7 +158,7 @@ func TestDoGetFallback(t *testing.T) { } resp := &testResponse{} if err := json.Unmarshal(b, resp); err != nil { - testutil.Ok(t, err) + t.Fatal(err) } if resp.Method != http.MethodPost { t.Fatalf("Mismatch method") @@ -174,7 +174,7 @@ func TestDoGetFallback(t *testing.T) { t.Fatalf("Error doing local request: %v", err) } if err := json.Unmarshal(b, resp); err != nil { - testutil.Ok(t, err) + t.Fatal(err) } if resp.Method != http.MethodGet { t.Fatalf("Mismatch method") diff --git a/api/prometheus/v1/api.go b/api/prometheus/v1/api.go index decd09b..1ff4402 100644 --- a/api/prometheus/v1/api.go +++ b/api/prometheus/v1/api.go @@ -17,18 +17,105 @@ package v1 import ( "context" - "encoding/json" "errors" "fmt" + "math" "net/http" "strconv" "strings" "time" + "unsafe" + + json "github.com/json-iterator/go" + + "github.com/prometheus/common/model" "github.com/prometheus/client_golang/api" - "github.com/prometheus/common/model" ) +func init() { + json.RegisterTypeEncoderFunc("model.SamplePair", marshalPointJSON, marshalPointJSONIsEmpty) + json.RegisterTypeDecoderFunc("model.SamplePair", unMarshalPointJSON) +} + +func unMarshalPointJSON(ptr unsafe.Pointer, iter *json.Iterator) { + p := (*model.SamplePair)(ptr) + if !iter.ReadArray() { + iter.ReportError("unmarshal model.SamplePair", "SamplePair must be [timestamp, value]") + return + } + t := iter.ReadNumber() + if err := p.Timestamp.UnmarshalJSON([]byte(t)); err != nil { + iter.ReportError("unmarshal model.SamplePair", err.Error()) + return + } + if !iter.ReadArray() { + iter.ReportError("unmarshal model.SamplePair", "SamplePair missing value") + return + } + + f, err := strconv.ParseFloat(iter.ReadString(), 64) + if err != nil { + iter.ReportError("unmarshal model.SamplePair", err.Error()) + return + } + p.Value = model.SampleValue(f) + + if iter.ReadArray() { + iter.ReportError("unmarshal model.SamplePair", "SamplePair has too many values, must be [timestamp, value]") + return + } +} + +func marshalPointJSON(ptr unsafe.Pointer, stream *json.Stream) { + p := *((*model.SamplePair)(ptr)) + stream.WriteArrayStart() + // Write out the timestamp as a float divided by 1000. + // This is ~3x faster than converting to a float. + t := int64(p.Timestamp) + if t < 0 { + stream.WriteRaw(`-`) + t = -t + } + stream.WriteInt64(t / 1000) + fraction := t % 1000 + if fraction != 0 { + stream.WriteRaw(`.`) + if fraction < 100 { + stream.WriteRaw(`0`) + } + if fraction < 10 { + stream.WriteRaw(`0`) + } + stream.WriteInt64(fraction) + } + stream.WriteMore() + stream.WriteRaw(`"`) + + // Taken from https://github.com/json-iterator/go/blob/master/stream_float.go#L71 as a workaround + // to https://github.com/json-iterator/go/issues/365 (jsoniter, to follow json standard, doesn't allow inf/nan) + buf := stream.Buffer() + abs := math.Abs(float64(p.Value)) + fmt := byte('f') + // Note: Must use float32 comparisons for underlying float32 value to get precise cutoffs right. + if abs != 0 { + if abs < 1e-6 || abs >= 1e21 { + fmt = 'e' + fmt = 'e' + } + } + buf = strconv.AppendFloat(buf, float64(p.Value), fmt, -1, 64) + stream.SetBuffer(buf) + + stream.WriteRaw(`"`) + stream.WriteArrayEnd() + +} + +func marshalPointJSONIsEmpty(ptr unsafe.Pointer) bool { + return false +} + const ( statusAPIError = 422 diff --git a/api/prometheus/v1/api_bench_test.go b/api/prometheus/v1/api_bench_test.go new file mode 100644 index 0000000..764895a --- /dev/null +++ b/api/prometheus/v1/api_bench_test.go @@ -0,0 +1,112 @@ +// Copyright 2019 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package v1 + +import ( + "encoding/json" + "strconv" + "testing" + "time" + + jsoniter "github.com/json-iterator/go" + + "github.com/prometheus/common/model" +) + +func generateData(timeseries, datapoints int) model.Matrix { + m := make(model.Matrix, 0) + + for i := 0; i < timeseries; i++ { + lset := map[model.LabelName]model.LabelValue{ + model.MetricNameLabel: model.LabelValue("timeseries_" + strconv.Itoa(i)), + } + now := model.Now() + values := make([]model.SamplePair, datapoints) + + for x := datapoints; x > 0; x-- { + values[x-1] = model.SamplePair{ + // Set the time back assuming a 15s interval. Since this is used for + // Marshal/Unmarshal testing the actual interval doesn't matter. + Timestamp: now.Add(time.Second * -15 * time.Duration(x)), + Value: model.SampleValue(float64(x)), + } + } + + ss := &model.SampleStream{ + Metric: model.Metric(lset), + Values: values, + } + + m = append(m, ss) + } + return m +} + +func BenchmarkSamplesJsonSerialization(b *testing.B) { + for _, timeseriesCount := range []int{10, 100, 1000} { + b.Run(strconv.Itoa(timeseriesCount), func(b *testing.B) { + for _, datapointCount := range []int{10, 100, 1000} { + b.Run(strconv.Itoa(datapointCount), func(b *testing.B) { + data := generateData(timeseriesCount, datapointCount) + + dataBytes, err := json.Marshal(data) + if err != nil { + b.Fatalf("Error marshaling: %v", err) + } + + b.Run("marshal", func(b *testing.B) { + b.Run("encoding/json", func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + if _, err := json.Marshal(data); err != nil { + b.Fatal(err) + } + } + }) + + b.Run("jsoniter", func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + if _, err := jsoniter.Marshal(data); err != nil { + b.Fatal(err) + } + } + }) + }) + + b.Run("unmarshal", func(b *testing.B) { + b.Run("encoding/json", func(b *testing.B) { + b.ReportAllocs() + var m model.Matrix + for i := 0; i < b.N; i++ { + if err := json.Unmarshal(dataBytes, &m); err != nil { + b.Fatal(err) + } + } + }) + + b.Run("jsoniter", func(b *testing.B) { + b.ReportAllocs() + var m model.Matrix + for i := 0; i < b.N; i++ { + if err := jsoniter.Unmarshal(dataBytes, &m); err != nil { + b.Fatal(err) + } + } + }) + }) + }) + } + }) + } +} diff --git a/api/prometheus/v1/api_test.go b/api/prometheus/v1/api_test.go index 2429e05..0e24b49 100644 --- a/api/prometheus/v1/api_test.go +++ b/api/prometheus/v1/api_test.go @@ -15,9 +15,9 @@ package v1 import ( "context" - "encoding/json" "errors" "fmt" + "math" "net/http" "net/url" "reflect" @@ -25,9 +25,12 @@ import ( "testing" "time" - "github.com/prometheus/client_golang/api" + json "github.com/json-iterator/go" + "github.com/prometheus/common/model" "github.com/prometheus/tsdb/testutil" + + "github.com/prometheus/client_golang/api" ) type apiTest struct { @@ -792,7 +795,7 @@ func TestAPIClientDo(t *testing.T) { response: "bad json", expectedErr: &Error{ Type: ErrBadResponse, - Msg: "invalid character 'b' looking for beginning of value", + Msg: "readObjectStart: expect { or n, but found b, error found in #1 byte of ...|bad json|..., bigger context ...|bad json|...", }, }, { @@ -882,3 +885,99 @@ func TestAPIClientDo(t *testing.T) { } } + +func TestSamplesJsonSerialization(t *testing.T) { + tests := []struct { + point model.SamplePair + expected string + }{ + { + point: model.SamplePair{0, 0}, + expected: `[0,"0"]`, + }, + { + point: model.SamplePair{1, 20}, + expected: `[0.001,"20"]`, + }, + { + point: model.SamplePair{10, 20}, + expected: `[0.010,"20"]`, + }, + { + point: model.SamplePair{100, 20}, + expected: `[0.100,"20"]`, + }, + { + point: model.SamplePair{1001, 20}, + expected: `[1.001,"20"]`, + }, + { + point: model.SamplePair{1010, 20}, + expected: `[1.010,"20"]`, + }, + { + point: model.SamplePair{1100, 20}, + expected: `[1.100,"20"]`, + }, + { + point: model.SamplePair{12345678123456555, 20}, + expected: `[12345678123456.555,"20"]`, + }, + { + point: model.SamplePair{-1, 20}, + expected: `[-0.001,"20"]`, + }, + { + point: model.SamplePair{0, model.SampleValue(math.NaN())}, + expected: `[0,"NaN"]`, + }, + { + point: model.SamplePair{0, model.SampleValue(math.Inf(1))}, + expected: `[0,"+Inf"]`, + }, + { + point: model.SamplePair{0, model.SampleValue(math.Inf(-1))}, + expected: `[0,"-Inf"]`, + }, + { + point: model.SamplePair{0, model.SampleValue(1.2345678e6)}, + expected: `[0,"1234567.8"]`, + }, + { + point: model.SamplePair{0, 1.2345678e-6}, + expected: `[0,"0.0000012345678"]`, + }, + { + point: model.SamplePair{0, 1.2345678e-67}, + expected: `[0,"1.2345678e-67"]`, + }, + } + + for _, test := range tests { + t.Run(test.expected, func(t *testing.T) { + b, err := json.Marshal(test.point) + if err != nil { + t.Fatal(err) + } + if string(b) != test.expected { + t.Fatalf("Mismatch marshal expected=%s actual=%s", test.expected, string(b)) + } + + // To test Unmarshal we will Unmarshal then re-Marshal this way we + // can do a string compare, otherwise Nan values don't show equivalence + // properly. + var sp model.SamplePair + if err = json.Unmarshal(b, &sp); err != nil { + t.Fatal(err) + } + + b, err = json.Marshal(sp) + if err != nil { + t.Fatal(err) + } + if string(b) != test.expected { + t.Fatalf("Mismatch marshal expected=%s actual=%s", test.expected, string(b)) + } + }) + } +} diff --git a/go.mod b/go.mod index e2c8b8b..abacaaa 100644 --- a/go.mod +++ b/go.mod @@ -4,8 +4,12 @@ require ( github.com/beorn7/perks v1.0.0 github.com/go-logfmt/logfmt v0.4.0 // indirect github.com/golang/protobuf v1.3.1 + github.com/json-iterator/go v1.1.6 + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.1 // indirect github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 - github.com/prometheus/common v0.4.0 - github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084 + github.com/prometheus/common v0.4.1 + github.com/prometheus/procfs v0.0.0-20190522114515-bc1a522cf7b1 github.com/prometheus/tsdb v0.7.1 + github.com/stretchr/testify v1.3.0 // indirect ) diff --git a/go.sum b/go.sum index 05f965c..81e2ba1 100644 --- a/go.sum +++ b/go.sum @@ -8,6 +8,7 @@ github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= @@ -24,12 +25,18 @@ github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/json-iterator/go v1.1.6 h1:MrUvLMLTMxbqFJ9kzlvat/rYZqZnW3u4wkLzWTaFwKs= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -41,19 +48,22 @@ github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1: github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.4.0 h1:7etb9YClo3a6HjLzfl6rIQaU+FDfi0VSX39io3aQ+DM= -github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.4.1 h1:K0MGApIoQvMw27RTdJkPbr3JZ7DNbtxQNyi5STVM6Kw= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084 h1:sofwID9zm4tzrgykg80hfFph1mryUeLRsUfoocVVmRY= -github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.0-20190522114515-bc1a522cf7b1 h1:Lo6mRUjdS99f3zxYOUalftWHUoOGaDRqFk1+j0Q57/I= +github.com/prometheus/procfs v0.0.0-20190522114515-bc1a522cf7b1/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/tsdb v0.7.1 h1:YZcsG11NqnK4czYLrWd9mpEuAJIHVQLwdrleYfszMAA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f h1:Bl/8QSvNqXvPGPGXa2z5xUTmV7VDcZyvRZ+QQXkXTZQ=