Extend Counters, Summaries and Histograms with creation timestamp (#1313)
* Extend Counters, Summaries and Histograms with creation timestamp Signed-off-by: Arthur Silva Sens <arthur.sens@coralogix.com> * Backport created timestamp to existing tests Signed-off-by: Arthur Silva Sens <arthur.sens@coralogix.com> * Last touches (readability and consistency) Changes: * Comments for "now" are more explicit and not inlined. * populateMetrics is simpler and bit more efficient without timestamp to time to timestamp conversionts for more common code. * Test consistency and simplicity - the fewer variables the better. * Fixed inconsistency for v2 and MetricVec - let's pass opt.now consistently. * We don't need TestCounterXXXTimestamp - we test CT in many other places already. * Added more involved test for counter vectors with created timestamp. * Refactored normalization for simplicity. * Make histogram, summaries now consistent. * Simplified histograms CT flow and implemented proper CT on reset. TODO for next PRs: * NewConstSummary and NewConstHistogram - ability to specify CTs there. Signed-off-by: bwplotka <bwplotka@gmail.com> * Update prometheus/counter_test.go Co-authored-by: Arthur Silva Sens <arthursens2005@gmail.com> Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com> --------- Signed-off-by: Arthur Silva Sens <arthur.sens@coralogix.com> Signed-off-by: bwplotka <bwplotka@gmail.com> Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com> Co-authored-by: bwplotka <bwplotka@gmail.com>
This commit is contained in:
Arthur Silva Sens
GitHub
parent
74cc26257c
commit
df7fa49417
2
go.mod
2
go.mod
|
|
@ -7,7 +7,7 @@ require (
|
|||
github.com/cespare/xxhash/v2 v2.2.0
|
||||
github.com/davecgh/go-spew v1.1.1
|
||||
github.com/json-iterator/go v1.1.12
|
||||
github.com/prometheus/client_model v0.4.0
|
||||
github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16
|
||||
github.com/prometheus/common v0.44.0
|
||||
github.com/prometheus/procfs v0.11.1
|
||||
golang.org/x/sys v0.11.0
|
||||
|
|
|
|||
4
go.sum
4
go.sum
|
|
@ -34,8 +34,8 @@ github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRW
|
|||
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY=
|
||||
github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU=
|
||||
github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 h1:v7DLqVdK4VrYkVD5diGdl4sxJurKJEMnODWRJlxV9oM=
|
||||
github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU=
|
||||
github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY=
|
||||
github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY=
|
||||
github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI=
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ import (
|
|||
"time"
|
||||
|
||||
dto "github.com/prometheus/client_model/go"
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
)
|
||||
|
||||
// Counter is a Metric that represents a single numerical value that only ever
|
||||
|
|
@ -90,8 +91,12 @@ func NewCounter(opts CounterOpts) Counter {
|
|||
nil,
|
||||
opts.ConstLabels,
|
||||
)
|
||||
result := &counter{desc: desc, labelPairs: desc.constLabelPairs, now: time.Now}
|
||||
if opts.now == nil {
|
||||
opts.now = time.Now
|
||||
}
|
||||
result := &counter{desc: desc, labelPairs: desc.constLabelPairs, now: opts.now}
|
||||
result.init(result) // Init self-collection.
|
||||
result.createdTs = timestamppb.New(opts.now())
|
||||
return result
|
||||
}
|
||||
|
||||
|
|
@ -106,10 +111,12 @@ type counter struct {
|
|||
selfCollector
|
||||
desc *Desc
|
||||
|
||||
createdTs *timestamppb.Timestamp
|
||||
labelPairs []*dto.LabelPair
|
||||
exemplar atomic.Value // Containing nil or a *dto.Exemplar.
|
||||
|
||||
now func() time.Time // To mock out time.Now() for testing.
|
||||
// now is for testing purposes, by default it's time.Now.
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
func (c *counter) Desc() *Desc {
|
||||
|
|
@ -159,8 +166,7 @@ func (c *counter) Write(out *dto.Metric) error {
|
|||
exemplar = e.(*dto.Exemplar)
|
||||
}
|
||||
val := c.get()
|
||||
|
||||
return populateMetric(CounterValue, val, c.labelPairs, exemplar, out)
|
||||
return populateMetric(CounterValue, val, c.labelPairs, exemplar, out, c.createdTs)
|
||||
}
|
||||
|
||||
func (c *counter) updateExemplar(v float64, l Labels) {
|
||||
|
|
@ -200,13 +206,17 @@ func (v2) NewCounterVec(opts CounterVecOpts) *CounterVec {
|
|||
opts.VariableLabels,
|
||||
opts.ConstLabels,
|
||||
)
|
||||
if opts.now == nil {
|
||||
opts.now = time.Now
|
||||
}
|
||||
return &CounterVec{
|
||||
MetricVec: NewMetricVec(desc, func(lvs ...string) Metric {
|
||||
if len(lvs) != len(desc.variableLabels.names) {
|
||||
panic(makeInconsistentCardinalityError(desc.fqName, desc.variableLabels.names, lvs))
|
||||
}
|
||||
result := &counter{desc: desc, labelPairs: MakeLabelPairs(desc, lvs), now: time.Now}
|
||||
result := &counter{desc: desc, labelPairs: MakeLabelPairs(desc, lvs), now: opts.now}
|
||||
result.init(result) // Init self-collection.
|
||||
result.createdTs = timestamppb.New(opts.now())
|
||||
return result
|
||||
}),
|
||||
}
|
||||
|
|
|
|||
|
|
@ -26,10 +26,13 @@ import (
|
|||
)
|
||||
|
||||
func TestCounterAdd(t *testing.T) {
|
||||
now := time.Now()
|
||||
|
||||
counter := NewCounter(CounterOpts{
|
||||
Name: "test",
|
||||
Help: "test help",
|
||||
ConstLabels: Labels{"a": "1", "b": "2"},
|
||||
now: func() time.Time { return now },
|
||||
}).(*counter)
|
||||
counter.Inc()
|
||||
if expected, got := 0.0, math.Float64frombits(counter.valBits); expected != got {
|
||||
|
|
@ -66,7 +69,10 @@ func TestCounterAdd(t *testing.T) {
|
|||
{Name: proto.String("a"), Value: proto.String("1")},
|
||||
{Name: proto.String("b"), Value: proto.String("2")},
|
||||
},
|
||||
Counter: &dto.Counter{Value: proto.Float64(67.42)},
|
||||
Counter: &dto.Counter{
|
||||
Value: proto.Float64(67.42),
|
||||
CreatedTimestamp: timestamppb.New(now),
|
||||
},
|
||||
}
|
||||
if !proto.Equal(expected, m) {
|
||||
t.Errorf("expected %q, got %q", expected, m)
|
||||
|
|
@ -139,9 +145,12 @@ func expectPanic(t *testing.T, op func(), errorMsg string) {
|
|||
}
|
||||
|
||||
func TestCounterAddInf(t *testing.T) {
|
||||
now := time.Now()
|
||||
|
||||
counter := NewCounter(CounterOpts{
|
||||
Name: "test",
|
||||
Help: "test help",
|
||||
now: func() time.Time { return now },
|
||||
}).(*counter)
|
||||
|
||||
counter.Inc()
|
||||
|
|
@ -173,7 +182,8 @@ func TestCounterAddInf(t *testing.T) {
|
|||
|
||||
expected := &dto.Metric{
|
||||
Counter: &dto.Counter{
|
||||
Value: proto.Float64(math.Inf(1)),
|
||||
Value: proto.Float64(math.Inf(1)),
|
||||
CreatedTimestamp: timestamppb.New(now),
|
||||
},
|
||||
}
|
||||
|
||||
|
|
@ -183,9 +193,12 @@ func TestCounterAddInf(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestCounterAddLarge(t *testing.T) {
|
||||
now := time.Now()
|
||||
|
||||
counter := NewCounter(CounterOpts{
|
||||
Name: "test",
|
||||
Help: "test help",
|
||||
now: func() time.Time { return now },
|
||||
}).(*counter)
|
||||
|
||||
// large overflows the underlying type and should therefore be stored in valBits.
|
||||
|
|
@ -203,7 +216,8 @@ func TestCounterAddLarge(t *testing.T) {
|
|||
|
||||
expected := &dto.Metric{
|
||||
Counter: &dto.Counter{
|
||||
Value: proto.Float64(large),
|
||||
Value: proto.Float64(large),
|
||||
CreatedTimestamp: timestamppb.New(now),
|
||||
},
|
||||
}
|
||||
|
||||
|
|
@ -213,10 +227,14 @@ func TestCounterAddLarge(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestCounterAddSmall(t *testing.T) {
|
||||
now := time.Now()
|
||||
|
||||
counter := NewCounter(CounterOpts{
|
||||
Name: "test",
|
||||
Help: "test help",
|
||||
now: func() time.Time { return now },
|
||||
}).(*counter)
|
||||
|
||||
small := 0.000000000001
|
||||
counter.Add(small)
|
||||
if expected, got := small, math.Float64frombits(counter.valBits); expected != got {
|
||||
|
|
@ -231,7 +249,8 @@ func TestCounterAddSmall(t *testing.T) {
|
|||
|
||||
expected := &dto.Metric{
|
||||
Counter: &dto.Counter{
|
||||
Value: proto.Float64(small),
|
||||
Value: proto.Float64(small),
|
||||
CreatedTimestamp: timestamppb.New(now),
|
||||
},
|
||||
}
|
||||
|
||||
|
|
@ -246,8 +265,8 @@ func TestCounterExemplar(t *testing.T) {
|
|||
counter := NewCounter(CounterOpts{
|
||||
Name: "test",
|
||||
Help: "test help",
|
||||
now: func() time.Time { return now },
|
||||
}).(*counter)
|
||||
counter.now = func() time.Time { return now }
|
||||
|
||||
ts := timestamppb.New(now)
|
||||
if err := ts.CheckValid(); err != nil {
|
||||
|
|
@ -298,3 +317,72 @@ func TestCounterExemplar(t *testing.T) {
|
|||
t.Error("adding exemplar with oversized labels succeeded")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCounterVecCreatedTimestampWithDeletes(t *testing.T) {
|
||||
now := time.Now()
|
||||
|
||||
counterVec := NewCounterVec(CounterOpts{
|
||||
Name: "test",
|
||||
Help: "test help",
|
||||
now: func() time.Time { return now },
|
||||
}, []string{"label"})
|
||||
|
||||
// First use of "With" should populate CT.
|
||||
counterVec.WithLabelValues("1")
|
||||
expected := map[string]time.Time{"1": now}
|
||||
|
||||
now = now.Add(1 * time.Hour)
|
||||
expectCTsForMetricVecValues(t, counterVec.MetricVec, dto.MetricType_COUNTER, expected)
|
||||
|
||||
// Two more labels at different times.
|
||||
counterVec.WithLabelValues("2")
|
||||
expected["2"] = now
|
||||
|
||||
now = now.Add(1 * time.Hour)
|
||||
|
||||
counterVec.WithLabelValues("3")
|
||||
expected["3"] = now
|
||||
|
||||
now = now.Add(1 * time.Hour)
|
||||
expectCTsForMetricVecValues(t, counterVec.MetricVec, dto.MetricType_COUNTER, expected)
|
||||
|
||||
// Recreate metric instance should reset created timestamp to now.
|
||||
counterVec.DeleteLabelValues("1")
|
||||
counterVec.WithLabelValues("1")
|
||||
expected["1"] = now
|
||||
|
||||
now = now.Add(1 * time.Hour)
|
||||
expectCTsForMetricVecValues(t, counterVec.MetricVec, dto.MetricType_COUNTER, expected)
|
||||
}
|
||||
|
||||
func expectCTsForMetricVecValues(t testing.TB, vec *MetricVec, typ dto.MetricType, ctsPerLabelValue map[string]time.Time) {
|
||||
t.Helper()
|
||||
|
||||
for val, ct := range ctsPerLabelValue {
|
||||
var metric dto.Metric
|
||||
m, err := vec.GetMetricWithLabelValues(val)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := m.Write(&metric); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var gotTs time.Time
|
||||
switch typ {
|
||||
case dto.MetricType_COUNTER:
|
||||
gotTs = metric.Counter.CreatedTimestamp.AsTime()
|
||||
case dto.MetricType_HISTOGRAM:
|
||||
gotTs = metric.Histogram.CreatedTimestamp.AsTime()
|
||||
case dto.MetricType_SUMMARY:
|
||||
gotTs = metric.Summary.CreatedTimestamp.AsTime()
|
||||
default:
|
||||
t.Fatalf("unknown metric type %v", typ)
|
||||
}
|
||||
|
||||
if !gotTs.Equal(ct) {
|
||||
t.Errorf("expected created timestamp for %s with label value %q: %s, got %s", typ, val, ct, gotTs)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,6 +14,8 @@
|
|||
package prometheus_test
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"google.golang.org/protobuf/proto"
|
||||
|
||||
dto "github.com/prometheus/client_model/go"
|
||||
|
|
@ -124,7 +126,7 @@ func ExampleMetricVec() {
|
|||
if err != nil || len(metricFamilies) != 1 {
|
||||
panic("unexpected behavior of custom test registry")
|
||||
}
|
||||
printlnNormalized(metricFamilies[0])
|
||||
fmt.Println(toNormalizedJSON(metricFamilies[0]))
|
||||
|
||||
// Output:
|
||||
// {"name":"library_version_info","help":"Versions of the libraries used in this binary.","type":"GAUGE","metric":[{"label":[{"name":"library","value":"k8s.io/client-go"},{"name":"version","value":"0.18.8"}],"gauge":{"value":1}},{"label":[{"name":"library","value":"prometheus/client_golang"},{"name":"version","value":"1.7.1"}],"gauge":{"value":1}}]}
|
||||
|
|
|
|||
|
|
@ -153,6 +153,22 @@ func ExampleCounterVec() {
|
|||
httpReqs.DeleteLabelValues("200", "GET")
|
||||
// Same thing with the more verbose Labels syntax.
|
||||
httpReqs.Delete(prometheus.Labels{"method": "GET", "code": "200"})
|
||||
|
||||
// Just for demonstration, let's check the state of the counter vector
|
||||
// by registering it with a custom registry and then let it collect the
|
||||
// metrics.
|
||||
reg := prometheus.NewRegistry()
|
||||
reg.MustRegister(httpReqs)
|
||||
|
||||
metricFamilies, err := reg.Gather()
|
||||
if err != nil || len(metricFamilies) != 1 {
|
||||
panic("unexpected behavior of custom test registry")
|
||||
}
|
||||
|
||||
fmt.Println(toNormalizedJSON(sanitizeMetricFamily(metricFamilies[0])))
|
||||
|
||||
// Output:
|
||||
// {"name":"http_requests_total","help":"How many HTTP requests processed, partitioned by status code and HTTP method.","type":"COUNTER","metric":[{"label":[{"name":"code","value":"404"},{"name":"method","value":"POST"}],"counter":{"value":42,"createdTimestamp":"1970-01-01T00:00:10Z"}}]}
|
||||
}
|
||||
|
||||
func ExampleRegister() {
|
||||
|
|
@ -320,10 +336,10 @@ func ExampleSummary() {
|
|||
metric := &dto.Metric{}
|
||||
temps.Write(metric)
|
||||
|
||||
printlnNormalized(metric)
|
||||
fmt.Println(toNormalizedJSON(sanitizeMetric(metric)))
|
||||
|
||||
// Output:
|
||||
// {"summary":{"sampleCount":"1000","sampleSum":29969.50000000001,"quantile":[{"quantile":0.5,"value":31.1},{"quantile":0.9,"value":41.3},{"quantile":0.99,"value":41.9}]}}
|
||||
// {"summary":{"sampleCount":"1000","sampleSum":29969.50000000001,"quantile":[{"quantile":0.5,"value":31.1},{"quantile":0.9,"value":41.3},{"quantile":0.99,"value":41.9}],"createdTimestamp":"1970-01-01T00:00:10Z"}}
|
||||
}
|
||||
|
||||
func ExampleSummaryVec() {
|
||||
|
|
@ -355,10 +371,11 @@ func ExampleSummaryVec() {
|
|||
if err != nil || len(metricFamilies) != 1 {
|
||||
panic("unexpected behavior of custom test registry")
|
||||
}
|
||||
printlnNormalized(metricFamilies[0])
|
||||
|
||||
fmt.Println(toNormalizedJSON(sanitizeMetricFamily(metricFamilies[0])))
|
||||
|
||||
// Output:
|
||||
// {"name":"pond_temperature_celsius","help":"The temperature of the frog pond.","type":"SUMMARY","metric":[{"label":[{"name":"species","value":"leiopelma-hochstetteri"}],"summary":{"sampleCount":"0","sampleSum":0,"quantile":[{"quantile":0.5,"value":"NaN"},{"quantile":0.9,"value":"NaN"},{"quantile":0.99,"value":"NaN"}]}},{"label":[{"name":"species","value":"lithobates-catesbeianus"}],"summary":{"sampleCount":"1000","sampleSum":31956.100000000017,"quantile":[{"quantile":0.5,"value":32.4},{"quantile":0.9,"value":41.4},{"quantile":0.99,"value":41.9}]}},{"label":[{"name":"species","value":"litoria-caerulea"}],"summary":{"sampleCount":"1000","sampleSum":29969.50000000001,"quantile":[{"quantile":0.5,"value":31.1},{"quantile":0.9,"value":41.3},{"quantile":0.99,"value":41.9}]}}]}
|
||||
// {"name":"pond_temperature_celsius","help":"The temperature of the frog pond.","type":"SUMMARY","metric":[{"label":[{"name":"species","value":"leiopelma-hochstetteri"}],"summary":{"sampleCount":"0","sampleSum":0,"quantile":[{"quantile":0.5,"value":"NaN"},{"quantile":0.9,"value":"NaN"},{"quantile":0.99,"value":"NaN"}],"createdTimestamp":"1970-01-01T00:00:10Z"}},{"label":[{"name":"species","value":"lithobates-catesbeianus"}],"summary":{"sampleCount":"1000","sampleSum":31956.100000000017,"quantile":[{"quantile":0.5,"value":32.4},{"quantile":0.9,"value":41.4},{"quantile":0.99,"value":41.9}],"createdTimestamp":"1970-01-01T00:00:10Z"}},{"label":[{"name":"species","value":"litoria-caerulea"}],"summary":{"sampleCount":"1000","sampleSum":29969.50000000001,"quantile":[{"quantile":0.5,"value":31.1},{"quantile":0.9,"value":41.3},{"quantile":0.99,"value":41.9}],"createdTimestamp":"1970-01-01T00:00:10Z"}}]}
|
||||
}
|
||||
|
||||
func ExampleNewConstSummary() {
|
||||
|
|
@ -382,7 +399,7 @@ func ExampleNewConstSummary() {
|
|||
// internally).
|
||||
metric := &dto.Metric{}
|
||||
s.Write(metric)
|
||||
printlnNormalized(metric)
|
||||
fmt.Println(toNormalizedJSON(metric))
|
||||
|
||||
// Output:
|
||||
// {"label":[{"name":"code","value":"200"},{"name":"method","value":"get"},{"name":"owner","value":"example"}],"summary":{"sampleCount":"4711","sampleSum":403.34,"quantile":[{"quantile":0.5,"value":42.3},{"quantile":0.9,"value":323.3}]}}
|
||||
|
|
@ -405,10 +422,11 @@ func ExampleHistogram() {
|
|||
// internally).
|
||||
metric := &dto.Metric{}
|
||||
temps.Write(metric)
|
||||
printlnNormalized(metric)
|
||||
|
||||
fmt.Println(toNormalizedJSON(sanitizeMetric(metric)))
|
||||
|
||||
// Output:
|
||||
// {"histogram":{"sampleCount":"1000","sampleSum":29969.50000000001,"bucket":[{"cumulativeCount":"192","upperBound":20},{"cumulativeCount":"366","upperBound":25},{"cumulativeCount":"501","upperBound":30},{"cumulativeCount":"638","upperBound":35},{"cumulativeCount":"816","upperBound":40}]}}
|
||||
// {"histogram":{"sampleCount":"1000","sampleSum":29969.50000000001,"bucket":[{"cumulativeCount":"192","upperBound":20},{"cumulativeCount":"366","upperBound":25},{"cumulativeCount":"501","upperBound":30},{"cumulativeCount":"638","upperBound":35},{"cumulativeCount":"816","upperBound":40}],"createdTimestamp":"1970-01-01T00:00:10Z"}}
|
||||
}
|
||||
|
||||
func ExampleNewConstHistogram() {
|
||||
|
|
@ -432,7 +450,7 @@ func ExampleNewConstHistogram() {
|
|||
// internally).
|
||||
metric := &dto.Metric{}
|
||||
h.Write(metric)
|
||||
printlnNormalized(metric)
|
||||
fmt.Println(toNormalizedJSON(metric))
|
||||
|
||||
// Output:
|
||||
// {"label":[{"name":"code","value":"200"},{"name":"method","value":"get"},{"name":"owner","value":"example"}],"histogram":{"sampleCount":"4711","sampleSum":403.34,"bucket":[{"cumulativeCount":"121","upperBound":25},{"cumulativeCount":"2403","upperBound":50},{"cumulativeCount":"3221","upperBound":100},{"cumulativeCount":"4233","upperBound":200}]}}
|
||||
|
|
@ -470,7 +488,7 @@ func ExampleNewConstHistogram_WithExemplar() {
|
|||
// internally).
|
||||
metric := &dto.Metric{}
|
||||
h.Write(metric)
|
||||
printlnNormalized(metric)
|
||||
fmt.Println(toNormalizedJSON(metric))
|
||||
|
||||
// Output:
|
||||
// {"label":[{"name":"code","value":"200"},{"name":"method","value":"get"},{"name":"owner","value":"example"}],"histogram":{"sampleCount":"4711","sampleSum":403.34,"bucket":[{"cumulativeCount":"121","upperBound":25,"exemplar":{"label":[{"name":"testName","value":"testVal"}],"value":24,"timestamp":"2006-01-02T15:04:05Z"}},{"cumulativeCount":"2403","upperBound":50,"exemplar":{"label":[{"name":"testName","value":"testVal"}],"value":42,"timestamp":"2006-01-02T15:04:05Z"}},{"cumulativeCount":"3221","upperBound":100,"exemplar":{"label":[{"name":"testName","value":"testVal"}],"value":89,"timestamp":"2006-01-02T15:04:05Z"}},{"cumulativeCount":"4233","upperBound":200,"exemplar":{"label":[{"name":"testName","value":"testVal"}],"value":157,"timestamp":"2006-01-02T15:04:05Z"}}]}}
|
||||
|
|
@ -632,7 +650,7 @@ func ExampleNewMetricWithTimestamp() {
|
|||
// internally).
|
||||
metric := &dto.Metric{}
|
||||
s.Write(metric)
|
||||
printlnNormalized(metric)
|
||||
fmt.Println(toNormalizedJSON(metric))
|
||||
|
||||
// Output:
|
||||
// {"gauge":{"value":298.15},"timestampMs":"1257894000012"}
|
||||
|
|
|
|||
|
|
@ -81,7 +81,7 @@ func ExampleNewExpvarCollector() {
|
|||
if !strings.Contains(m.Desc().String(), "expvar_memstats") {
|
||||
metric.Reset()
|
||||
m.Write(&metric)
|
||||
metricStrings = append(metricStrings, protoToNormalizedJSON(&metric))
|
||||
metricStrings = append(metricStrings, toNormalizedJSON(&metric))
|
||||
}
|
||||
}
|
||||
sort.Strings(metricStrings)
|
||||
|
|
|
|||
|
|
@ -135,7 +135,7 @@ func (g *gauge) Sub(val float64) {
|
|||
|
||||
func (g *gauge) Write(out *dto.Metric) error {
|
||||
val := math.Float64frombits(atomic.LoadUint64(&g.valBits))
|
||||
return populateMetric(GaugeValue, val, g.labelPairs, nil, out)
|
||||
return populateMetric(GaugeValue, val, g.labelPairs, nil, out, nil)
|
||||
}
|
||||
|
||||
// GaugeVec is a Collector that bundles a set of Gauges that all share the same
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ import (
|
|||
dto "github.com/prometheus/client_model/go"
|
||||
|
||||
"google.golang.org/protobuf/proto"
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
)
|
||||
|
||||
// nativeHistogramBounds for the frac of observed values. Only relevant for
|
||||
|
|
@ -471,6 +472,9 @@ type HistogramOpts struct {
|
|||
NativeHistogramMaxBucketNumber uint32
|
||||
NativeHistogramMinResetDuration time.Duration
|
||||
NativeHistogramMaxZeroThreshold float64
|
||||
|
||||
// now is for testing purposes, by default it's time.Now.
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
// HistogramVecOpts bundles the options to create a HistogramVec metric.
|
||||
|
|
@ -519,6 +523,10 @@ func newHistogram(desc *Desc, opts HistogramOpts, labelValues ...string) Histogr
|
|||
}
|
||||
}
|
||||
|
||||
if opts.now == nil {
|
||||
opts.now = time.Now
|
||||
}
|
||||
|
||||
h := &histogram{
|
||||
desc: desc,
|
||||
upperBounds: opts.Buckets,
|
||||
|
|
@ -526,8 +534,8 @@ func newHistogram(desc *Desc, opts HistogramOpts, labelValues ...string) Histogr
|
|||
nativeHistogramMaxBuckets: opts.NativeHistogramMaxBucketNumber,
|
||||
nativeHistogramMaxZeroThreshold: opts.NativeHistogramMaxZeroThreshold,
|
||||
nativeHistogramMinResetDuration: opts.NativeHistogramMinResetDuration,
|
||||
lastResetTime: time.Now(),
|
||||
now: time.Now,
|
||||
lastResetTime: opts.now(),
|
||||
now: opts.now,
|
||||
}
|
||||
if len(h.upperBounds) == 0 && opts.NativeHistogramBucketFactor <= 1 {
|
||||
h.upperBounds = DefBuckets
|
||||
|
|
@ -706,9 +714,11 @@ type histogram struct {
|
|||
nativeHistogramMaxZeroThreshold float64
|
||||
nativeHistogramMaxBuckets uint32
|
||||
nativeHistogramMinResetDuration time.Duration
|
||||
lastResetTime time.Time // Protected by mtx.
|
||||
// lastResetTime is protected by mtx. It is also used as created timestamp.
|
||||
lastResetTime time.Time
|
||||
|
||||
now func() time.Time // To mock out time.Now() for testing.
|
||||
// now is for testing purposes, by default it's time.Now.
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
func (h *histogram) Desc() *Desc {
|
||||
|
|
@ -747,9 +757,10 @@ func (h *histogram) Write(out *dto.Metric) error {
|
|||
waitForCooldown(count, coldCounts)
|
||||
|
||||
his := &dto.Histogram{
|
||||
Bucket: make([]*dto.Bucket, len(h.upperBounds)),
|
||||
SampleCount: proto.Uint64(count),
|
||||
SampleSum: proto.Float64(math.Float64frombits(atomic.LoadUint64(&coldCounts.sumBits))),
|
||||
Bucket: make([]*dto.Bucket, len(h.upperBounds)),
|
||||
SampleCount: proto.Uint64(count),
|
||||
SampleSum: proto.Float64(math.Float64frombits(atomic.LoadUint64(&coldCounts.sumBits))),
|
||||
CreatedTimestamp: timestamppb.New(h.lastResetTime),
|
||||
}
|
||||
out.Histogram = his
|
||||
out.Label = h.labelPairs
|
||||
|
|
@ -1194,6 +1205,7 @@ type constHistogram struct {
|
|||
sum float64
|
||||
buckets map[float64]uint64
|
||||
labelPairs []*dto.LabelPair
|
||||
createdTs *timestamppb.Timestamp
|
||||
}
|
||||
|
||||
func (h *constHistogram) Desc() *Desc {
|
||||
|
|
@ -1201,7 +1213,9 @@ func (h *constHistogram) Desc() *Desc {
|
|||
}
|
||||
|
||||
func (h *constHistogram) Write(out *dto.Metric) error {
|
||||
his := &dto.Histogram{}
|
||||
his := &dto.Histogram{
|
||||
CreatedTimestamp: h.createdTs,
|
||||
}
|
||||
|
||||
buckets := make([]*dto.Bucket, 0, len(h.buckets))
|
||||
|
||||
|
|
|
|||
|
|
@ -416,8 +416,8 @@ func TestHistogramExemplar(t *testing.T) {
|
|||
Name: "test",
|
||||
Help: "test help",
|
||||
Buckets: []float64{1, 2, 3, 4},
|
||||
now: func() time.Time { return now },
|
||||
}).(*histogram)
|
||||
histogram.now = func() time.Time { return now }
|
||||
|
||||
ts := timestamppb.New(now)
|
||||
if err := ts.CheckValid(); err != nil {
|
||||
|
|
@ -469,6 +469,8 @@ func TestHistogramExemplar(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestNativeHistogram(t *testing.T) {
|
||||
now := time.Now()
|
||||
|
||||
scenarios := []struct {
|
||||
name string
|
||||
observations []float64 // With simulated interval of 1m.
|
||||
|
|
@ -499,17 +501,19 @@ func TestNativeHistogram(t *testing.T) {
|
|||
{CumulativeCount: proto.Uint64(3), UpperBound: proto.Float64(5)},
|
||||
{CumulativeCount: proto.Uint64(3), UpperBound: proto.Float64(10)},
|
||||
},
|
||||
CreatedTimestamp: timestamppb.New(now),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "no observations",
|
||||
factor: 1.1,
|
||||
want: &dto.Histogram{
|
||||
SampleCount: proto.Uint64(0),
|
||||
SampleSum: proto.Float64(0),
|
||||
Schema: proto.Int32(3),
|
||||
ZeroThreshold: proto.Float64(2.938735877055719e-39),
|
||||
ZeroCount: proto.Uint64(0),
|
||||
SampleCount: proto.Uint64(0),
|
||||
SampleSum: proto.Float64(0),
|
||||
Schema: proto.Int32(3),
|
||||
ZeroThreshold: proto.Float64(2.938735877055719e-39),
|
||||
ZeroCount: proto.Uint64(0),
|
||||
CreatedTimestamp: timestamppb.New(now),
|
||||
},
|
||||
},
|
||||
{
|
||||
|
|
@ -525,6 +529,7 @@ func TestNativeHistogram(t *testing.T) {
|
|||
PositiveSpan: []*dto.BucketSpan{
|
||||
{Offset: proto.Int32(0), Length: proto.Uint32(0)},
|
||||
},
|
||||
CreatedTimestamp: timestamppb.New(now),
|
||||
},
|
||||
},
|
||||
{
|
||||
|
|
@ -542,7 +547,8 @@ func TestNativeHistogram(t *testing.T) {
|
|||
{Offset: proto.Int32(7), Length: proto.Uint32(1)},
|
||||
{Offset: proto.Int32(4), Length: proto.Uint32(1)},
|
||||
},
|
||||
PositiveDelta: []int64{1, 0, 0},
|
||||
PositiveDelta: []int64{1, 0, 0},
|
||||
CreatedTimestamp: timestamppb.New(now),
|
||||
},
|
||||
},
|
||||
{
|
||||
|
|
@ -558,7 +564,8 @@ func TestNativeHistogram(t *testing.T) {
|
|||
PositiveSpan: []*dto.BucketSpan{
|
||||
{Offset: proto.Int32(0), Length: proto.Uint32(5)},
|
||||
},
|
||||
PositiveDelta: []int64{1, -1, 2, -2, 2},
|
||||
PositiveDelta: []int64{1, -1, 2, -2, 2},
|
||||
CreatedTimestamp: timestamppb.New(now),
|
||||
},
|
||||
},
|
||||
{
|
||||
|
|
@ -581,7 +588,8 @@ func TestNativeHistogram(t *testing.T) {
|
|||
PositiveSpan: []*dto.BucketSpan{
|
||||
{Offset: proto.Int32(-2), Length: proto.Uint32(6)},
|
||||
},
|
||||
PositiveDelta: []int64{2, 0, 0, 2, -1, -2},
|
||||
PositiveDelta: []int64{2, 0, 0, 2, -1, -2},
|
||||
CreatedTimestamp: timestamppb.New(now),
|
||||
},
|
||||
},
|
||||
{
|
||||
|
|
@ -602,7 +610,8 @@ func TestNativeHistogram(t *testing.T) {
|
|||
PositiveSpan: []*dto.BucketSpan{
|
||||
{Offset: proto.Int32(-1), Length: proto.Uint32(4)},
|
||||
},
|
||||
PositiveDelta: []int64{2, 2, 3, -6},
|
||||
PositiveDelta: []int64{2, 2, 3, -6},
|
||||
CreatedTimestamp: timestamppb.New(now),
|
||||
},
|
||||
},
|
||||
{
|
||||
|
|
@ -618,7 +627,8 @@ func TestNativeHistogram(t *testing.T) {
|
|||
NegativeSpan: []*dto.BucketSpan{
|
||||
{Offset: proto.Int32(0), Length: proto.Uint32(5)},
|
||||
},
|
||||
NegativeDelta: []int64{1, -1, 2, -2, 2},
|
||||
NegativeDelta: []int64{1, -1, 2, -2, 2},
|
||||
CreatedTimestamp: timestamppb.New(now),
|
||||
},
|
||||
},
|
||||
{
|
||||
|
|
@ -638,7 +648,8 @@ func TestNativeHistogram(t *testing.T) {
|
|||
PositiveSpan: []*dto.BucketSpan{
|
||||
{Offset: proto.Int32(0), Length: proto.Uint32(5)},
|
||||
},
|
||||
PositiveDelta: []int64{1, -1, 2, -2, 2},
|
||||
PositiveDelta: []int64{1, -1, 2, -2, 2},
|
||||
CreatedTimestamp: timestamppb.New(now),
|
||||
},
|
||||
},
|
||||
{
|
||||
|
|
@ -659,7 +670,8 @@ func TestNativeHistogram(t *testing.T) {
|
|||
PositiveSpan: []*dto.BucketSpan{
|
||||
{Offset: proto.Int32(4), Length: proto.Uint32(1)},
|
||||
},
|
||||
PositiveDelta: []int64{2},
|
||||
PositiveDelta: []int64{2},
|
||||
CreatedTimestamp: timestamppb.New(now),
|
||||
},
|
||||
},
|
||||
{
|
||||
|
|
@ -675,7 +687,8 @@ func TestNativeHistogram(t *testing.T) {
|
|||
PositiveSpan: []*dto.BucketSpan{
|
||||
{Offset: proto.Int32(0), Length: proto.Uint32(5)},
|
||||
},
|
||||
PositiveDelta: []int64{1, -1, 2, -2, 2},
|
||||
PositiveDelta: []int64{1, -1, 2, -2, 2},
|
||||
CreatedTimestamp: timestamppb.New(now),
|
||||
},
|
||||
},
|
||||
{
|
||||
|
|
@ -692,7 +705,8 @@ func TestNativeHistogram(t *testing.T) {
|
|||
{Offset: proto.Int32(0), Length: proto.Uint32(5)},
|
||||
{Offset: proto.Int32(4092), Length: proto.Uint32(1)},
|
||||
},
|
||||
PositiveDelta: []int64{1, -1, 2, -2, 2, -1},
|
||||
PositiveDelta: []int64{1, -1, 2, -2, 2, -1},
|
||||
CreatedTimestamp: timestamppb.New(now),
|
||||
},
|
||||
},
|
||||
{
|
||||
|
|
@ -712,7 +726,8 @@ func TestNativeHistogram(t *testing.T) {
|
|||
PositiveSpan: []*dto.BucketSpan{
|
||||
{Offset: proto.Int32(0), Length: proto.Uint32(5)},
|
||||
},
|
||||
PositiveDelta: []int64{1, -1, 2, -2, 2},
|
||||
PositiveDelta: []int64{1, -1, 2, -2, 2},
|
||||
CreatedTimestamp: timestamppb.New(now),
|
||||
},
|
||||
},
|
||||
{
|
||||
|
|
@ -729,7 +744,8 @@ func TestNativeHistogram(t *testing.T) {
|
|||
PositiveSpan: []*dto.BucketSpan{
|
||||
{Offset: proto.Int32(0), Length: proto.Uint32(5)},
|
||||
},
|
||||
PositiveDelta: []int64{1, -1, 2, -2, 2},
|
||||
PositiveDelta: []int64{1, -1, 2, -2, 2},
|
||||
CreatedTimestamp: timestamppb.New(now),
|
||||
},
|
||||
},
|
||||
{
|
||||
|
|
@ -746,7 +762,8 @@ func TestNativeHistogram(t *testing.T) {
|
|||
PositiveSpan: []*dto.BucketSpan{
|
||||
{Offset: proto.Int32(0), Length: proto.Uint32(5)},
|
||||
},
|
||||
PositiveDelta: []int64{1, 2, -1, -2, 1},
|
||||
PositiveDelta: []int64{1, 2, -1, -2, 1},
|
||||
CreatedTimestamp: timestamppb.New(now),
|
||||
},
|
||||
},
|
||||
{
|
||||
|
|
@ -764,7 +781,8 @@ func TestNativeHistogram(t *testing.T) {
|
|||
PositiveSpan: []*dto.BucketSpan{
|
||||
{Offset: proto.Int32(1), Length: proto.Uint32(7)},
|
||||
},
|
||||
PositiveDelta: []int64{1, 1, -2, 2, -2, 0, 1},
|
||||
PositiveDelta: []int64{1, 1, -2, 2, -2, 0, 1},
|
||||
CreatedTimestamp: timestamppb.New(now),
|
||||
},
|
||||
},
|
||||
{
|
||||
|
|
@ -782,7 +800,8 @@ func TestNativeHistogram(t *testing.T) {
|
|||
PositiveSpan: []*dto.BucketSpan{
|
||||
{Offset: proto.Int32(2), Length: proto.Uint32(7)},
|
||||
},
|
||||
PositiveDelta: []int64{2, -2, 2, -2, 0, 1, 0},
|
||||
PositiveDelta: []int64{2, -2, 2, -2, 0, 1, 0},
|
||||
CreatedTimestamp: timestamppb.New(now),
|
||||
},
|
||||
},
|
||||
{
|
||||
|
|
@ -801,7 +820,8 @@ func TestNativeHistogram(t *testing.T) {
|
|||
PositiveSpan: []*dto.BucketSpan{
|
||||
{Offset: proto.Int32(7), Length: proto.Uint32(2)},
|
||||
},
|
||||
PositiveDelta: []int64{1, 0},
|
||||
PositiveDelta: []int64{1, 0},
|
||||
CreatedTimestamp: timestamppb.New(now.Add(8 * time.Minute)), // We expect reset to happen after 8 observations.
|
||||
},
|
||||
},
|
||||
{
|
||||
|
|
@ -818,7 +838,8 @@ func TestNativeHistogram(t *testing.T) {
|
|||
NegativeSpan: []*dto.BucketSpan{
|
||||
{Offset: proto.Int32(0), Length: proto.Uint32(5)},
|
||||
},
|
||||
NegativeDelta: []int64{1, -1, 2, -2, 2},
|
||||
NegativeDelta: []int64{1, -1, 2, -2, 2},
|
||||
CreatedTimestamp: timestamppb.New(now),
|
||||
},
|
||||
},
|
||||
{
|
||||
|
|
@ -835,7 +856,8 @@ func TestNativeHistogram(t *testing.T) {
|
|||
NegativeSpan: []*dto.BucketSpan{
|
||||
{Offset: proto.Int32(0), Length: proto.Uint32(5)},
|
||||
},
|
||||
NegativeDelta: []int64{1, 2, -1, -2, 1},
|
||||
NegativeDelta: []int64{1, 2, -1, -2, 1},
|
||||
CreatedTimestamp: timestamppb.New(now),
|
||||
},
|
||||
},
|
||||
{
|
||||
|
|
@ -853,7 +875,8 @@ func TestNativeHistogram(t *testing.T) {
|
|||
NegativeSpan: []*dto.BucketSpan{
|
||||
{Offset: proto.Int32(1), Length: proto.Uint32(7)},
|
||||
},
|
||||
NegativeDelta: []int64{1, 1, -2, 2, -2, 0, 1},
|
||||
NegativeDelta: []int64{1, 1, -2, 2, -2, 0, 1},
|
||||
CreatedTimestamp: timestamppb.New(now),
|
||||
},
|
||||
},
|
||||
{
|
||||
|
|
@ -871,7 +894,8 @@ func TestNativeHistogram(t *testing.T) {
|
|||
NegativeSpan: []*dto.BucketSpan{
|
||||
{Offset: proto.Int32(2), Length: proto.Uint32(7)},
|
||||
},
|
||||
NegativeDelta: []int64{2, -2, 2, -2, 0, 1, 0},
|
||||
NegativeDelta: []int64{2, -2, 2, -2, 0, 1, 0},
|
||||
CreatedTimestamp: timestamppb.New(now),
|
||||
},
|
||||
},
|
||||
{
|
||||
|
|
@ -890,7 +914,8 @@ func TestNativeHistogram(t *testing.T) {
|
|||
NegativeSpan: []*dto.BucketSpan{
|
||||
{Offset: proto.Int32(7), Length: proto.Uint32(2)},
|
||||
},
|
||||
NegativeDelta: []int64{1, 0},
|
||||
NegativeDelta: []int64{1, 0},
|
||||
CreatedTimestamp: timestamppb.New(now.Add(8 * time.Minute)), // We expect reset to happen after 8 observations.
|
||||
},
|
||||
},
|
||||
{
|
||||
|
|
@ -908,7 +933,8 @@ func TestNativeHistogram(t *testing.T) {
|
|||
PositiveSpan: []*dto.BucketSpan{
|
||||
{Offset: proto.Int32(7), Length: proto.Uint32(2)},
|
||||
},
|
||||
PositiveDelta: []int64{1, 0},
|
||||
PositiveDelta: []int64{1, 0},
|
||||
CreatedTimestamp: timestamppb.New(now.Add(10 * time.Minute)), // We expect reset to happen after 9 minutes.
|
||||
},
|
||||
},
|
||||
{
|
||||
|
|
@ -927,13 +953,16 @@ func TestNativeHistogram(t *testing.T) {
|
|||
PositiveSpan: []*dto.BucketSpan{
|
||||
{Offset: proto.Int32(7), Length: proto.Uint32(2)},
|
||||
},
|
||||
PositiveDelta: []int64{1, 0},
|
||||
PositiveDelta: []int64{1, 0},
|
||||
CreatedTimestamp: timestamppb.New(now.Add(10 * time.Minute)), // We expect reset to happen after 9 minutes.
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, s := range scenarios {
|
||||
t.Run(s.name, func(t *testing.T) {
|
||||
ts := now
|
||||
|
||||
his := NewHistogram(HistogramOpts{
|
||||
Name: "name",
|
||||
Help: "help",
|
||||
|
|
@ -942,12 +971,10 @@ func TestNativeHistogram(t *testing.T) {
|
|||
NativeHistogramMaxBucketNumber: s.maxBuckets,
|
||||
NativeHistogramMinResetDuration: s.minResetDuration,
|
||||
NativeHistogramMaxZeroThreshold: s.maxZeroThreshold,
|
||||
now: func() time.Time { return ts },
|
||||
})
|
||||
ts := time.Now().Add(30 * time.Second)
|
||||
now := func() time.Time {
|
||||
return ts
|
||||
}
|
||||
his.(*histogram).now = now
|
||||
|
||||
ts = ts.Add(time.Minute)
|
||||
for _, o := range s.observations {
|
||||
his.Observe(o)
|
||||
ts = ts.Add(time.Minute)
|
||||
|
|
@ -972,6 +999,8 @@ func TestNativeHistogramConcurrency(t *testing.T) {
|
|||
rand.Seed(42)
|
||||
|
||||
it := func(n uint32) bool {
|
||||
ts := time.Now().Add(30 * time.Second).Unix()
|
||||
|
||||
mutations := int(n%1e4 + 1e4)
|
||||
concLevel := int(n%5 + 1)
|
||||
total := mutations * concLevel
|
||||
|
|
@ -988,14 +1017,11 @@ func TestNativeHistogramConcurrency(t *testing.T) {
|
|||
NativeHistogramMaxBucketNumber: 50,
|
||||
NativeHistogramMinResetDuration: time.Hour, // Comment out to test for totals below.
|
||||
NativeHistogramMaxZeroThreshold: 0.001,
|
||||
now: func() time.Time {
|
||||
return time.Unix(atomic.LoadInt64(&ts), 0)
|
||||
},
|
||||
})
|
||||
|
||||
ts := time.Now().Add(30 * time.Second).Unix()
|
||||
now := func() time.Time {
|
||||
return time.Unix(atomic.LoadInt64(&ts), 0)
|
||||
}
|
||||
his.(*histogram).now = now
|
||||
|
||||
allVars := make([]float64, total)
|
||||
var sampleSum float64
|
||||
for i := 0; i < concLevel; i++ {
|
||||
|
|
@ -1152,3 +1178,82 @@ func TestGetLe(t *testing.T) {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHistogramCreatedTimestamp(t *testing.T) {
|
||||
now := time.Now()
|
||||
|
||||
histogram := NewHistogram(HistogramOpts{
|
||||
Name: "test",
|
||||
Help: "test help",
|
||||
Buckets: []float64{1, 2, 3, 4},
|
||||
now: func() time.Time { return now },
|
||||
})
|
||||
|
||||
var metric dto.Metric
|
||||
if err := histogram.Write(&metric); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if metric.Histogram.CreatedTimestamp.AsTime().Unix() != now.Unix() {
|
||||
t.Errorf("expected created timestamp %d, got %d", now.Unix(), metric.Histogram.CreatedTimestamp.AsTime().Unix())
|
||||
}
|
||||
}
|
||||
|
||||
func TestHistogramVecCreatedTimestamp(t *testing.T) {
|
||||
now := time.Now()
|
||||
|
||||
histogramVec := NewHistogramVec(HistogramOpts{
|
||||
Name: "test",
|
||||
Help: "test help",
|
||||
Buckets: []float64{1, 2, 3, 4},
|
||||
now: func() time.Time { return now },
|
||||
}, []string{"label"})
|
||||
histogram := histogramVec.WithLabelValues("value").(Histogram)
|
||||
|
||||
var metric dto.Metric
|
||||
if err := histogram.Write(&metric); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if metric.Histogram.CreatedTimestamp.AsTime().Unix() != now.Unix() {
|
||||
t.Errorf("expected created timestamp %d, got %d", now.Unix(), metric.Histogram.CreatedTimestamp.AsTime().Unix())
|
||||
}
|
||||
}
|
||||
|
||||
func TestHistogramVecCreatedTimestampWithDeletes(t *testing.T) {
|
||||
now := time.Now()
|
||||
|
||||
histogramVec := NewHistogramVec(HistogramOpts{
|
||||
Name: "test",
|
||||
Help: "test help",
|
||||
Buckets: []float64{1, 2, 3, 4},
|
||||
now: func() time.Time { return now },
|
||||
}, []string{"label"})
|
||||
|
||||
// First use of "With" should populate CT.
|
||||
histogramVec.WithLabelValues("1")
|
||||
expected := map[string]time.Time{"1": now}
|
||||
|
||||
now = now.Add(1 * time.Hour)
|
||||
expectCTsForMetricVecValues(t, histogramVec.MetricVec, dto.MetricType_HISTOGRAM, expected)
|
||||
|
||||
// Two more labels at different times.
|
||||
histogramVec.WithLabelValues("2")
|
||||
expected["2"] = now
|
||||
|
||||
now = now.Add(1 * time.Hour)
|
||||
|
||||
histogramVec.WithLabelValues("3")
|
||||
expected["3"] = now
|
||||
|
||||
now = now.Add(1 * time.Hour)
|
||||
expectCTsForMetricVecValues(t, histogramVec.MetricVec, dto.MetricType_HISTOGRAM, expected)
|
||||
|
||||
// Recreate metric instance should reset created timestamp to now.
|
||||
histogramVec.DeleteLabelValues("1")
|
||||
histogramVec.WithLabelValues("1")
|
||||
expected["1"] = now
|
||||
|
||||
now = now.Add(1 * time.Hour)
|
||||
expectCTsForMetricVecValues(t, histogramVec.MetricVec, dto.MetricType_HISTOGRAM, expected)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -92,6 +92,9 @@ type Opts struct {
|
|||
// machine_role metric). See also
|
||||
// https://prometheus.io/docs/instrumenting/writing_exporters/#target-labels-not-static-scraped-labels
|
||||
ConstLabels Labels
|
||||
|
||||
// now is for testing purposes, by default it's time.Now.
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
// BuildFQName joins the given three name components by "_". Empty name
|
||||
|
|
|
|||
|
|
@ -37,6 +37,7 @@ import (
|
|||
dto "github.com/prometheus/client_model/go"
|
||||
"github.com/prometheus/common/expfmt"
|
||||
"google.golang.org/protobuf/proto"
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
)
|
||||
|
||||
// uncheckedCollector wraps a Collector but its Describe method yields no Desc.
|
||||
|
|
@ -138,7 +139,8 @@ metric: <
|
|||
},
|
||||
},
|
||||
Counter: &dto.Counter{
|
||||
Value: proto.Float64(1),
|
||||
Value: proto.Float64(1),
|
||||
CreatedTimestamp: timestamppb.New(time.Now()),
|
||||
},
|
||||
},
|
||||
{
|
||||
|
|
@ -153,7 +155,8 @@ metric: <
|
|||
},
|
||||
},
|
||||
Counter: &dto.Counter{
|
||||
Value: proto.Float64(1),
|
||||
Value: proto.Float64(1),
|
||||
CreatedTimestamp: timestamppb.New(time.Now()),
|
||||
},
|
||||
},
|
||||
},
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ import (
|
|||
|
||||
"github.com/beorn7/perks/quantile"
|
||||
"google.golang.org/protobuf/proto"
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
)
|
||||
|
||||
// quantileLabel is used for the label that defines the quantile in a
|
||||
|
|
@ -145,6 +146,9 @@ type SummaryOpts struct {
|
|||
// is the internal buffer size of the underlying package
|
||||
// "github.com/bmizerany/perks/quantile").
|
||||
BufCap uint32
|
||||
|
||||
// now is for testing purposes, by default it's time.Now.
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
// SummaryVecOpts bundles the options to create a SummaryVec metric.
|
||||
|
|
@ -222,6 +226,9 @@ func newSummary(desc *Desc, opts SummaryOpts, labelValues ...string) Summary {
|
|||
opts.BufCap = DefBufCap
|
||||
}
|
||||
|
||||
if opts.now == nil {
|
||||
opts.now = time.Now
|
||||
}
|
||||
if len(opts.Objectives) == 0 {
|
||||
// Use the lock-free implementation of a Summary without objectives.
|
||||
s := &noObjectivesSummary{
|
||||
|
|
@ -230,6 +237,7 @@ func newSummary(desc *Desc, opts SummaryOpts, labelValues ...string) Summary {
|
|||
counts: [2]*summaryCounts{{}, {}},
|
||||
}
|
||||
s.init(s) // Init self-collection.
|
||||
s.createdTs = timestamppb.New(opts.now())
|
||||
return s
|
||||
}
|
||||
|
||||
|
|
@ -245,7 +253,7 @@ func newSummary(desc *Desc, opts SummaryOpts, labelValues ...string) Summary {
|
|||
coldBuf: make([]float64, 0, opts.BufCap),
|
||||
streamDuration: opts.MaxAge / time.Duration(opts.AgeBuckets),
|
||||
}
|
||||
s.headStreamExpTime = time.Now().Add(s.streamDuration)
|
||||
s.headStreamExpTime = opts.now().Add(s.streamDuration)
|
||||
s.hotBufExpTime = s.headStreamExpTime
|
||||
|
||||
for i := uint32(0); i < opts.AgeBuckets; i++ {
|
||||
|
|
@ -259,6 +267,7 @@ func newSummary(desc *Desc, opts SummaryOpts, labelValues ...string) Summary {
|
|||
sort.Float64s(s.sortedObjectives)
|
||||
|
||||
s.init(s) // Init self-collection.
|
||||
s.createdTs = timestamppb.New(opts.now())
|
||||
return s
|
||||
}
|
||||
|
||||
|
|
@ -286,6 +295,8 @@ type summary struct {
|
|||
headStream *quantile.Stream
|
||||
headStreamIdx int
|
||||
headStreamExpTime, hotBufExpTime time.Time
|
||||
|
||||
createdTs *timestamppb.Timestamp
|
||||
}
|
||||
|
||||
func (s *summary) Desc() *Desc {
|
||||
|
|
@ -307,7 +318,9 @@ func (s *summary) Observe(v float64) {
|
|||
}
|
||||
|
||||
func (s *summary) Write(out *dto.Metric) error {
|
||||
sum := &dto.Summary{}
|
||||
sum := &dto.Summary{
|
||||
CreatedTimestamp: s.createdTs,
|
||||
}
|
||||
qs := make([]*dto.Quantile, 0, len(s.objectives))
|
||||
|
||||
s.bufMtx.Lock()
|
||||
|
|
@ -440,6 +453,8 @@ type noObjectivesSummary struct {
|
|||
counts [2]*summaryCounts
|
||||
|
||||
labelPairs []*dto.LabelPair
|
||||
|
||||
createdTs *timestamppb.Timestamp
|
||||
}
|
||||
|
||||
func (s *noObjectivesSummary) Desc() *Desc {
|
||||
|
|
@ -490,8 +505,9 @@ func (s *noObjectivesSummary) Write(out *dto.Metric) error {
|
|||
}
|
||||
|
||||
sum := &dto.Summary{
|
||||
SampleCount: proto.Uint64(count),
|
||||
SampleSum: proto.Float64(math.Float64frombits(atomic.LoadUint64(&coldCounts.sumBits))),
|
||||
SampleCount: proto.Uint64(count),
|
||||
SampleSum: proto.Float64(math.Float64frombits(atomic.LoadUint64(&coldCounts.sumBits))),
|
||||
CreatedTimestamp: s.createdTs,
|
||||
}
|
||||
|
||||
out.Summary = sum
|
||||
|
|
@ -681,6 +697,7 @@ type constSummary struct {
|
|||
sum float64
|
||||
quantiles map[float64]float64
|
||||
labelPairs []*dto.LabelPair
|
||||
createdTs *timestamppb.Timestamp
|
||||
}
|
||||
|
||||
func (s *constSummary) Desc() *Desc {
|
||||
|
|
@ -688,7 +705,9 @@ func (s *constSummary) Desc() *Desc {
|
|||
}
|
||||
|
||||
func (s *constSummary) Write(out *dto.Metric) error {
|
||||
sum := &dto.Summary{}
|
||||
sum := &dto.Summary{
|
||||
CreatedTimestamp: s.createdTs,
|
||||
}
|
||||
qs := make([]*dto.Quantile, 0, len(s.quantiles))
|
||||
|
||||
sum.SampleCount = proto.Uint64(s.count)
|
||||
|
|
|
|||
|
|
@ -26,10 +26,13 @@ import (
|
|||
)
|
||||
|
||||
func TestSummaryWithDefaultObjectives(t *testing.T) {
|
||||
now := time.Now()
|
||||
|
||||
reg := NewRegistry()
|
||||
summaryWithDefaultObjectives := NewSummary(SummaryOpts{
|
||||
Name: "default_objectives",
|
||||
Help: "Test help.",
|
||||
now: func() time.Time { return now },
|
||||
})
|
||||
if err := reg.Register(summaryWithDefaultObjectives); err != nil {
|
||||
t.Error(err)
|
||||
|
|
@ -42,6 +45,10 @@ func TestSummaryWithDefaultObjectives(t *testing.T) {
|
|||
if len(m.GetSummary().Quantile) != 0 {
|
||||
t.Error("expected no objectives in summary")
|
||||
}
|
||||
|
||||
if !m.Summary.CreatedTimestamp.AsTime().Equal(now) {
|
||||
t.Errorf("expected created timestamp %s, got %s", now, m.Summary.CreatedTimestamp.AsTime())
|
||||
}
|
||||
}
|
||||
|
||||
func TestSummaryWithoutObjectives(t *testing.T) {
|
||||
|
|
@ -420,3 +427,50 @@ func getBounds(vars []float64, q, ε float64) (min, max float64) {
|
|||
}
|
||||
return
|
||||
}
|
||||
|
||||
func TestSummaryVecCreatedTimestampWithDeletes(t *testing.T) {
|
||||
for _, tcase := range []struct {
|
||||
desc string
|
||||
objectives map[float64]float64
|
||||
}{
|
||||
{desc: "summary with objectives", objectives: map[float64]float64{1.0: 1.0}},
|
||||
{desc: "no objectives summary", objectives: nil},
|
||||
} {
|
||||
now := time.Now()
|
||||
t.Run(tcase.desc, func(t *testing.T) {
|
||||
summaryVec := NewSummaryVec(SummaryOpts{
|
||||
Name: "test",
|
||||
Help: "test help",
|
||||
Objectives: tcase.objectives,
|
||||
now: func() time.Time { return now },
|
||||
}, []string{"label"})
|
||||
|
||||
// First use of "With" should populate CT.
|
||||
summaryVec.WithLabelValues("1")
|
||||
expected := map[string]time.Time{"1": now}
|
||||
|
||||
now = now.Add(1 * time.Hour)
|
||||
expectCTsForMetricVecValues(t, summaryVec.MetricVec, dto.MetricType_SUMMARY, expected)
|
||||
|
||||
// Two more labels at different times.
|
||||
summaryVec.WithLabelValues("2")
|
||||
expected["2"] = now
|
||||
|
||||
now = now.Add(1 * time.Hour)
|
||||
|
||||
summaryVec.WithLabelValues("3")
|
||||
expected["3"] = now
|
||||
|
||||
now = now.Add(1 * time.Hour)
|
||||
expectCTsForMetricVecValues(t, summaryVec.MetricVec, dto.MetricType_SUMMARY, expected)
|
||||
|
||||
// Recreate metric instance should reset created timestamp to now.
|
||||
summaryVec.DeleteLabelValues("1")
|
||||
summaryVec.WithLabelValues("1")
|
||||
expected["1"] = now
|
||||
|
||||
now = now.Add(1 * time.Hour)
|
||||
expectCTsForMetricVecValues(t, summaryVec.MetricVec, dto.MetricType_SUMMARY, expected)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -15,21 +15,42 @@ package prometheus_test
|
|||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
dto "github.com/prometheus/client_model/go"
|
||||
"google.golang.org/protobuf/encoding/protojson"
|
||||
"google.golang.org/protobuf/proto"
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
)
|
||||
|
||||
// printlnNormalized is a helper function to compare proto messages in json format.
|
||||
// Without removing brittle, we can't assert that two proto messages in json/text format are equal.
|
||||
// Read more in https://github.com/golang/protobuf/issues/1121
|
||||
func printlnNormalized(m proto.Message) {
|
||||
fmt.Println(protoToNormalizedJSON(m))
|
||||
// sanitizeMetric injects expected fake created timestamp value "1970-01-01T00:00:10Z",
|
||||
// so we can compare it in examples. It modifies metric in-place, the returned pointer
|
||||
// is for convenience.
|
||||
func sanitizeMetric(metric *dto.Metric) *dto.Metric {
|
||||
if metric.Counter != nil && metric.Counter.CreatedTimestamp != nil {
|
||||
metric.Counter.CreatedTimestamp = timestamppb.New(time.Unix(10, 0))
|
||||
}
|
||||
if metric.Summary != nil && metric.Summary.CreatedTimestamp != nil {
|
||||
metric.Summary.CreatedTimestamp = timestamppb.New(time.Unix(10, 0))
|
||||
}
|
||||
if metric.Histogram != nil && metric.Histogram.CreatedTimestamp != nil {
|
||||
metric.Histogram.CreatedTimestamp = timestamppb.New(time.Unix(10, 0))
|
||||
}
|
||||
return metric
|
||||
}
|
||||
|
||||
// protoToNormalizedJSON works as printlnNormalized, but returns the string instead of printing.
|
||||
func protoToNormalizedJSON(m proto.Message) string {
|
||||
// sanitizeMetricFamily is like sanitizeMetric, but for multiple metrics.
|
||||
func sanitizeMetricFamily(f *dto.MetricFamily) *dto.MetricFamily {
|
||||
for _, m := range f.Metric {
|
||||
sanitizeMetric(m)
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
// toNormalizedJSON removes fake random space from proto JSON original marshaller.
|
||||
// It is required, so we can compare proto messages in json format.
|
||||
// Read more in https://github.com/golang/protobuf/issues/1121
|
||||
func toNormalizedJSON(m proto.Message) string {
|
||||
mAsJSON, err := protojson.Marshal(m)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@
|
|||
package prometheus
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"time"
|
||||
|
|
@ -91,7 +92,7 @@ func (v *valueFunc) Desc() *Desc {
|
|||
}
|
||||
|
||||
func (v *valueFunc) Write(out *dto.Metric) error {
|
||||
return populateMetric(v.valType, v.function(), v.labelPairs, nil, out)
|
||||
return populateMetric(v.valType, v.function(), v.labelPairs, nil, out, nil)
|
||||
}
|
||||
|
||||
// NewConstMetric returns a metric with one fixed value that cannot be
|
||||
|
|
@ -110,7 +111,7 @@ func NewConstMetric(desc *Desc, valueType ValueType, value float64, labelValues
|
|||
}
|
||||
|
||||
metric := &dto.Metric{}
|
||||
if err := populateMetric(valueType, value, MakeLabelPairs(desc, labelValues), nil, metric); err != nil {
|
||||
if err := populateMetric(valueType, value, MakeLabelPairs(desc, labelValues), nil, metric, nil); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
|
|
@ -130,6 +131,43 @@ func MustNewConstMetric(desc *Desc, valueType ValueType, value float64, labelVal
|
|||
return m
|
||||
}
|
||||
|
||||
// NewConstMetricWithCreatedTimestamp does the same thing as NewConstMetric, but generates Counters
|
||||
// with created timestamp set and returns an error for other metric types.
|
||||
func NewConstMetricWithCreatedTimestamp(desc *Desc, valueType ValueType, value float64, ct time.Time, labelValues ...string) (Metric, error) {
|
||||
if desc.err != nil {
|
||||
return nil, desc.err
|
||||
}
|
||||
if err := validateLabelValues(labelValues, len(desc.variableLabels.names)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
switch valueType {
|
||||
case CounterValue:
|
||||
break
|
||||
default:
|
||||
return nil, errors.New("created timestamps are only supported for counters")
|
||||
}
|
||||
|
||||
metric := &dto.Metric{}
|
||||
if err := populateMetric(valueType, value, MakeLabelPairs(desc, labelValues), nil, metric, timestamppb.New(ct)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &constMetric{
|
||||
desc: desc,
|
||||
metric: metric,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// MustNewConstMetricWithCreatedTimestamp is a version of NewConstMetricWithCreatedTimestamp that panics where
|
||||
// NewConstMetricWithCreatedTimestamp would have returned an error.
|
||||
func MustNewConstMetricWithCreatedTimestamp(desc *Desc, valueType ValueType, value float64, ct time.Time, labelValues ...string) Metric {
|
||||
m, err := NewConstMetricWithCreatedTimestamp(desc, valueType, value, ct, labelValues...)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
type constMetric struct {
|
||||
desc *Desc
|
||||
metric *dto.Metric
|
||||
|
|
@ -153,11 +191,12 @@ func populateMetric(
|
|||
labelPairs []*dto.LabelPair,
|
||||
e *dto.Exemplar,
|
||||
m *dto.Metric,
|
||||
ct *timestamppb.Timestamp,
|
||||
) error {
|
||||
m.Label = labelPairs
|
||||
switch t {
|
||||
case CounterValue:
|
||||
m.Counter = &dto.Counter{Value: proto.Float64(v), Exemplar: e}
|
||||
m.Counter = &dto.Counter{Value: proto.Float64(v), Exemplar: e, CreatedTimestamp: ct}
|
||||
case GaugeValue:
|
||||
m.Gauge = &dto.Gauge{Value: proto.Float64(v)}
|
||||
case UntypedValue:
|
||||
|
|
|
|||
|
|
@ -16,6 +16,10 @@ package prometheus
|
|||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
dto "github.com/prometheus/client_model/go"
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
)
|
||||
|
||||
func TestNewConstMetricInvalidLabelValues(t *testing.T) {
|
||||
|
|
@ -54,3 +58,54 @@ func TestNewConstMetricInvalidLabelValues(t *testing.T) {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewConstMetricWithCreatedTimestamp(t *testing.T) {
|
||||
now := time.Now()
|
||||
|
||||
for _, tcase := range []struct {
|
||||
desc string
|
||||
metricType ValueType
|
||||
createdTimestamp time.Time
|
||||
expecErr bool
|
||||
expectedCt *timestamppb.Timestamp
|
||||
}{
|
||||
{
|
||||
desc: "gauge with CT",
|
||||
metricType: GaugeValue,
|
||||
createdTimestamp: now,
|
||||
expecErr: true,
|
||||
expectedCt: nil,
|
||||
},
|
||||
{
|
||||
desc: "counter with CT",
|
||||
metricType: CounterValue,
|
||||
createdTimestamp: now,
|
||||
expecErr: false,
|
||||
expectedCt: timestamppb.New(now),
|
||||
},
|
||||
} {
|
||||
t.Run(tcase.desc, func(t *testing.T) {
|
||||
metricDesc := NewDesc(
|
||||
"sample_value",
|
||||
"sample value",
|
||||
nil,
|
||||
nil,
|
||||
)
|
||||
m, err := NewConstMetricWithCreatedTimestamp(metricDesc, tcase.metricType, float64(1), tcase.createdTimestamp)
|
||||
if tcase.expecErr && err == nil {
|
||||
t.Errorf("Expected error is test %s, got no err", tcase.desc)
|
||||
}
|
||||
if !tcase.expecErr && err != nil {
|
||||
t.Errorf("Didn't expect error in test %s, got %s", tcase.desc, err.Error())
|
||||
}
|
||||
|
||||
if tcase.expectedCt != nil {
|
||||
var metric dto.Metric
|
||||
m.Write(&metric)
|
||||
if metric.Counter.CreatedTimestamp.AsTime() != tcase.expectedCt.AsTime() {
|
||||
t.Errorf("Expected timestamp %v, got %v", tcase.expectedCt, &metric.Counter.CreatedTimestamp)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ import (
|
|||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
dto "github.com/prometheus/client_model/go"
|
||||
"google.golang.org/protobuf/proto"
|
||||
|
|
@ -43,9 +44,12 @@ func toMetricFamilies(cs ...Collector) []*dto.MetricFamily {
|
|||
}
|
||||
|
||||
func TestWrap(t *testing.T) {
|
||||
now := time.Now()
|
||||
nowFn := func() time.Time { return now }
|
||||
simpleCnt := NewCounter(CounterOpts{
|
||||
Name: "simpleCnt",
|
||||
Help: "helpSimpleCnt",
|
||||
now: nowFn,
|
||||
})
|
||||
simpleCnt.Inc()
|
||||
|
||||
|
|
@ -58,6 +62,7 @@ func TestWrap(t *testing.T) {
|
|||
preCnt := NewCounter(CounterOpts{
|
||||
Name: "pre_simpleCnt",
|
||||
Help: "helpSimpleCnt",
|
||||
now: nowFn,
|
||||
})
|
||||
preCnt.Inc()
|
||||
|
||||
|
|
@ -65,6 +70,7 @@ func TestWrap(t *testing.T) {
|
|||
Name: "simpleCnt",
|
||||
Help: "helpSimpleCnt",
|
||||
ConstLabels: Labels{"foo": "bar"},
|
||||
now: nowFn,
|
||||
})
|
||||
barLabeledCnt.Inc()
|
||||
|
||||
|
|
@ -72,6 +78,7 @@ func TestWrap(t *testing.T) {
|
|||
Name: "simpleCnt",
|
||||
Help: "helpSimpleCnt",
|
||||
ConstLabels: Labels{"foo": "baz"},
|
||||
now: nowFn,
|
||||
})
|
||||
bazLabeledCnt.Inc()
|
||||
|
||||
|
|
@ -79,6 +86,7 @@ func TestWrap(t *testing.T) {
|
|||
Name: "pre_simpleCnt",
|
||||
Help: "helpSimpleCnt",
|
||||
ConstLabels: Labels{"foo": "bar"},
|
||||
now: nowFn,
|
||||
})
|
||||
labeledPreCnt.Inc()
|
||||
|
||||
|
|
@ -86,6 +94,7 @@ func TestWrap(t *testing.T) {
|
|||
Name: "pre_simpleCnt",
|
||||
Help: "helpSimpleCnt",
|
||||
ConstLabels: Labels{"foo": "bar", "dings": "bums"},
|
||||
now: nowFn,
|
||||
})
|
||||
twiceLabeledPreCnt.Inc()
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue