Extend Counters, Summaries and Histograms with creation timestamp (#1313)

* Extend Counters, Summaries and Histograms with creation timestamp

Signed-off-by: Arthur Silva Sens <arthur.sens@coralogix.com>

* Backport created timestamp to existing tests

Signed-off-by: Arthur Silva Sens <arthur.sens@coralogix.com>

* Last touches (readability and consistency)

Changes:

* Comments for "now" are more explicit and not inlined.
* populateMetrics is simpler and bit more efficient without timestamp to time to timestamp conversionts for more common code.
* Test consistency and simplicity - the fewer variables the better.
* Fixed inconsistency for v2 and MetricVec - let's pass opt.now consistently.
* We don't need TestCounterXXXTimestamp - we test CT in many other places already.
* Added more involved test for counter vectors with created timestamp.
* Refactored normalization for simplicity.
* Make histogram, summaries now consistent.
* Simplified histograms CT flow and implemented proper CT on reset.

TODO for next PRs:
* NewConstSummary and NewConstHistogram - ability to specify CTs there.

Signed-off-by: bwplotka <bwplotka@gmail.com>

* Update prometheus/counter_test.go

Co-authored-by: Arthur Silva Sens <arthursens2005@gmail.com>
Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com>

---------

Signed-off-by: Arthur Silva Sens <arthur.sens@coralogix.com>
Signed-off-by: bwplotka <bwplotka@gmail.com>
Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com>
Co-authored-by: bwplotka <bwplotka@gmail.com>

go.mod

@@ -7,7 +7,7 @@ require (
 	github.com/cespare/xxhash/v2 v2.2.0
 	github.com/davecgh/go-spew v1.1.1
 	github.com/json-iterator/go v1.1.12
-	github.com/prometheus/client_model v0.4.0
+	github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16
 	github.com/prometheus/common v0.44.0
 	github.com/prometheus/procfs v0.11.1
 	golang.org/x/sys v0.11.0

go.sum

@@ -34,8 +34,8 @@ github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRW
 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY=
+github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 h1:v7DLqVdK4VrYkVD5diGdl4sxJurKJEMnODWRJlxV9oM=
-github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU=
+github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU=
 github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY=
 github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY=
 github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI=

prometheus/counter.go

@@ -20,6 +20,7 @@ import (
 	"time"
 
 	dto "github.com/prometheus/client_model/go"
+	"google.golang.org/protobuf/types/known/timestamppb"
 )
 
 // Counter is a Metric that represents a single numerical value that only ever
@@ -90,8 +91,12 @@ func NewCounter(opts CounterOpts) Counter {
 		nil,
 		opts.ConstLabels,
 	)
-	result := &counter{desc: desc, labelPairs: desc.constLabelPairs, now: time.Now}
+	if opts.now == nil {
+		opts.now = time.Now
+	}
+	result := &counter{desc: desc, labelPairs: desc.constLabelPairs, now: opts.now}
 	result.init(result) // Init self-collection.
+	result.createdTs = timestamppb.New(opts.now())
 	return result
 }
 
@@ -106,10 +111,12 @@ type counter struct {
 	selfCollector
 	desc *Desc
 
+	createdTs  *timestamppb.Timestamp
 	labelPairs []*dto.LabelPair
 	exemplar   atomic.Value // Containing nil or a *dto.Exemplar.
 
-	now func() time.Time // To mock out time.Now() for testing.
+	// now is for testing purposes, by default it's time.Now.
+	now func() time.Time
 }
 
 func (c *counter) Desc() *Desc {
@@ -159,8 +166,7 @@ func (c *counter) Write(out *dto.Metric) error {
 		exemplar = e.(*dto.Exemplar)
 	}
 	val := c.get()
-
+	return populateMetric(CounterValue, val, c.labelPairs, exemplar, out, c.createdTs)
-	return populateMetric(CounterValue, val, c.labelPairs, exemplar, out)
 }
 
 func (c *counter) updateExemplar(v float64, l Labels) {
@@ -200,13 +206,17 @@ func (v2) NewCounterVec(opts CounterVecOpts) *CounterVec {
 		opts.VariableLabels,
 		opts.ConstLabels,
 	)
+	if opts.now == nil {
+		opts.now = time.Now
+	}
 	return &CounterVec{
 		MetricVec: NewMetricVec(desc, func(lvs ...string) Metric {
 			if len(lvs) != len(desc.variableLabels.names) {
 				panic(makeInconsistentCardinalityError(desc.fqName, desc.variableLabels.names, lvs))
 			}
-			result := &counter{desc: desc, labelPairs: MakeLabelPairs(desc, lvs), now: time.Now}
+			result := &counter{desc: desc, labelPairs: MakeLabelPairs(desc, lvs), now: opts.now}
 			result.init(result) // Init self-collection.
+			result.createdTs = timestamppb.New(opts.now())
 			return result
 		}),
 	}

prometheus/counter_test.go

@@ -26,10 +26,13 @@ import (
 )
 
 func TestCounterAdd(t *testing.T) {
+	now := time.Now()
+
 	counter := NewCounter(CounterOpts{
 		Name:        "test",
 		Help:        "test help",
 		ConstLabels: Labels{"a": "1", "b": "2"},
+		now:         func() time.Time { return now },
 	}).(*counter)
 	counter.Inc()
 	if expected, got := 0.0, math.Float64frombits(counter.valBits); expected != got {
@@ -66,7 +69,10 @@ func TestCounterAdd(t *testing.T) {
 			{Name: proto.String("a"), Value: proto.String("1")},
 			{Name: proto.String("b"), Value: proto.String("2")},
 		},
-		Counter: &dto.Counter{Value: proto.Float64(67.42)},
+		Counter: &dto.Counter{
+			Value:            proto.Float64(67.42),
+			CreatedTimestamp: timestamppb.New(now),
+		},
 	}
 	if !proto.Equal(expected, m) {
 		t.Errorf("expected %q, got %q", expected, m)
@@ -139,9 +145,12 @@ func expectPanic(t *testing.T, op func(), errorMsg string) {
 }
 
 func TestCounterAddInf(t *testing.T) {
+	now := time.Now()
+
 	counter := NewCounter(CounterOpts{
 		Name: "test",
 		Help: "test help",
+		now:  func() time.Time { return now },
 	}).(*counter)
 
 	counter.Inc()
@@ -174,6 +183,7 @@ func TestCounterAddInf(t *testing.T) {
 	expected := &dto.Metric{
 		Counter: &dto.Counter{
 			Value:            proto.Float64(math.Inf(1)),
+			CreatedTimestamp: timestamppb.New(now),
 		},
 	}
 
@@ -183,9 +193,12 @@ func TestCounterAddInf(t *testing.T) {
 }
 
 func TestCounterAddLarge(t *testing.T) {
+	now := time.Now()
+
 	counter := NewCounter(CounterOpts{
 		Name: "test",
 		Help: "test help",
+		now:  func() time.Time { return now },
 	}).(*counter)
 
 	// large overflows the underlying type and should therefore be stored in valBits.
@@ -204,6 +217,7 @@ func TestCounterAddLarge(t *testing.T) {
 	expected := &dto.Metric{
 		Counter: &dto.Counter{
 			Value:            proto.Float64(large),
+			CreatedTimestamp: timestamppb.New(now),
 		},
 	}
 
@@ -213,10 +227,14 @@ func TestCounterAddLarge(t *testing.T) {
 }
 
 func TestCounterAddSmall(t *testing.T) {
+	now := time.Now()
+
 	counter := NewCounter(CounterOpts{
 		Name: "test",
 		Help: "test help",
+		now:  func() time.Time { return now },
 	}).(*counter)
+
 	small := 0.000000000001
 	counter.Add(small)
 	if expected, got := small, math.Float64frombits(counter.valBits); expected != got {
@@ -232,6 +250,7 @@ func TestCounterAddSmall(t *testing.T) {
 	expected := &dto.Metric{
 		Counter: &dto.Counter{
 			Value:            proto.Float64(small),
+			CreatedTimestamp: timestamppb.New(now),
 		},
 	}
 
@@ -246,8 +265,8 @@ func TestCounterExemplar(t *testing.T) {
 	counter := NewCounter(CounterOpts{
 		Name: "test",
 		Help: "test help",
+		now:  func() time.Time { return now },
 	}).(*counter)
-	counter.now = func() time.Time { return now }
 
 	ts := timestamppb.New(now)
 	if err := ts.CheckValid(); err != nil {
@@ -298,3 +317,72 @@ func TestCounterExemplar(t *testing.T) {
 		t.Error("adding exemplar with oversized labels succeeded")
 	}
 }
+
+func TestCounterVecCreatedTimestampWithDeletes(t *testing.T) {
+	now := time.Now()
+
+	counterVec := NewCounterVec(CounterOpts{
+		Name: "test",
+		Help: "test help",
+		now:  func() time.Time { return now },
+	}, []string{"label"})
+
+	// First use of "With" should populate CT.
+	counterVec.WithLabelValues("1")
+	expected := map[string]time.Time{"1": now}
+
+	now = now.Add(1 * time.Hour)
+	expectCTsForMetricVecValues(t, counterVec.MetricVec, dto.MetricType_COUNTER, expected)
+
+	// Two more labels at different times.
+	counterVec.WithLabelValues("2")
+	expected["2"] = now
+
+	now = now.Add(1 * time.Hour)
+
+	counterVec.WithLabelValues("3")
+	expected["3"] = now
+
+	now = now.Add(1 * time.Hour)
+	expectCTsForMetricVecValues(t, counterVec.MetricVec, dto.MetricType_COUNTER, expected)
+
+	// Recreate metric instance should reset created timestamp to now.
+	counterVec.DeleteLabelValues("1")
+	counterVec.WithLabelValues("1")
+	expected["1"] = now
+
+	now = now.Add(1 * time.Hour)
+	expectCTsForMetricVecValues(t, counterVec.MetricVec, dto.MetricType_COUNTER, expected)
+}
+
+func expectCTsForMetricVecValues(t testing.TB, vec *MetricVec, typ dto.MetricType, ctsPerLabelValue map[string]time.Time) {
+	t.Helper()
+
+	for val, ct := range ctsPerLabelValue {
+		var metric dto.Metric
+		m, err := vec.GetMetricWithLabelValues(val)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if err := m.Write(&metric); err != nil {
+			t.Fatal(err)
+		}
+
+		var gotTs time.Time
+		switch typ {
+		case dto.MetricType_COUNTER:
+			gotTs = metric.Counter.CreatedTimestamp.AsTime()
+		case dto.MetricType_HISTOGRAM:
+			gotTs = metric.Histogram.CreatedTimestamp.AsTime()
+		case dto.MetricType_SUMMARY:
+			gotTs = metric.Summary.CreatedTimestamp.AsTime()
+		default:
+			t.Fatalf("unknown metric type %v", typ)
+		}
+
+		if !gotTs.Equal(ct) {
+			t.Errorf("expected created timestamp for %s with label value %q: %s, got %s", typ, val, ct, gotTs)
+		}
+	}
+}

prometheus/example_metricvec_test.go

@@ -14,6 +14,8 @@
 package prometheus_test
 
 import (
+	"fmt"
+
 	"google.golang.org/protobuf/proto"
 
 	dto "github.com/prometheus/client_model/go"
@@ -124,7 +126,7 @@ func ExampleMetricVec() {
 	if err != nil || len(metricFamilies) != 1 {
 		panic("unexpected behavior of custom test registry")
 	}
-	printlnNormalized(metricFamilies[0])
+	fmt.Println(toNormalizedJSON(metricFamilies[0]))
 
 	// Output:
 	// {"name":"library_version_info","help":"Versions of the libraries used in this binary.","type":"GAUGE","metric":[{"label":[{"name":"library","value":"k8s.io/client-go"},{"name":"version","value":"0.18.8"}],"gauge":{"value":1}},{"label":[{"name":"library","value":"prometheus/client_golang"},{"name":"version","value":"1.7.1"}],"gauge":{"value":1}}]}

prometheus/examples_test.go

@@ -153,6 +153,22 @@ func ExampleCounterVec() {
 	httpReqs.DeleteLabelValues("200", "GET")
 	// Same thing with the more verbose Labels syntax.
 	httpReqs.Delete(prometheus.Labels{"method": "GET", "code": "200"})
+
+	// Just for demonstration, let's check the state of the counter vector
+	// by registering it with a custom registry and then let it collect the
+	// metrics.
+	reg := prometheus.NewRegistry()
+	reg.MustRegister(httpReqs)
+
+	metricFamilies, err := reg.Gather()
+	if err != nil || len(metricFamilies) != 1 {
+		panic("unexpected behavior of custom test registry")
+	}
+
+	fmt.Println(toNormalizedJSON(sanitizeMetricFamily(metricFamilies[0])))
+
+	// Output:
+	// {"name":"http_requests_total","help":"How many HTTP requests processed, partitioned by status code and HTTP method.","type":"COUNTER","metric":[{"label":[{"name":"code","value":"404"},{"name":"method","value":"POST"}],"counter":{"value":42,"createdTimestamp":"1970-01-01T00:00:10Z"}}]}
 }
 
 func ExampleRegister() {
@@ -320,10 +336,10 @@ func ExampleSummary() {
 	metric := &dto.Metric{}
 	temps.Write(metric)
 
-	printlnNormalized(metric)
+	fmt.Println(toNormalizedJSON(sanitizeMetric(metric)))
 
 	// Output:
-	// {"summary":{"sampleCount":"1000","sampleSum":29969.50000000001,"quantile":[{"quantile":0.5,"value":31.1},{"quantile":0.9,"value":41.3},{"quantile":0.99,"value":41.9}]}}
+	// {"summary":{"sampleCount":"1000","sampleSum":29969.50000000001,"quantile":[{"quantile":0.5,"value":31.1},{"quantile":0.9,"value":41.3},{"quantile":0.99,"value":41.9}],"createdTimestamp":"1970-01-01T00:00:10Z"}}
 }
 
 func ExampleSummaryVec() {
@@ -355,10 +371,11 @@ func ExampleSummaryVec() {
 	if err != nil || len(metricFamilies) != 1 {
 		panic("unexpected behavior of custom test registry")
 	}
-	printlnNormalized(metricFamilies[0])
+
+	fmt.Println(toNormalizedJSON(sanitizeMetricFamily(metricFamilies[0])))
 
 	// Output:
-	// {"name":"pond_temperature_celsius","help":"The temperature of the frog pond.","type":"SUMMARY","metric":[{"label":[{"name":"species","value":"leiopelma-hochstetteri"}],"summary":{"sampleCount":"0","sampleSum":0,"quantile":[{"quantile":0.5,"value":"NaN"},{"quantile":0.9,"value":"NaN"},{"quantile":0.99,"value":"NaN"}]}},{"label":[{"name":"species","value":"lithobates-catesbeianus"}],"summary":{"sampleCount":"1000","sampleSum":31956.100000000017,"quantile":[{"quantile":0.5,"value":32.4},{"quantile":0.9,"value":41.4},{"quantile":0.99,"value":41.9}]}},{"label":[{"name":"species","value":"litoria-caerulea"}],"summary":{"sampleCount":"1000","sampleSum":29969.50000000001,"quantile":[{"quantile":0.5,"value":31.1},{"quantile":0.9,"value":41.3},{"quantile":0.99,"value":41.9}]}}]}
+	// {"name":"pond_temperature_celsius","help":"The temperature of the frog pond.","type":"SUMMARY","metric":[{"label":[{"name":"species","value":"leiopelma-hochstetteri"}],"summary":{"sampleCount":"0","sampleSum":0,"quantile":[{"quantile":0.5,"value":"NaN"},{"quantile":0.9,"value":"NaN"},{"quantile":0.99,"value":"NaN"}],"createdTimestamp":"1970-01-01T00:00:10Z"}},{"label":[{"name":"species","value":"lithobates-catesbeianus"}],"summary":{"sampleCount":"1000","sampleSum":31956.100000000017,"quantile":[{"quantile":0.5,"value":32.4},{"quantile":0.9,"value":41.4},{"quantile":0.99,"value":41.9}],"createdTimestamp":"1970-01-01T00:00:10Z"}},{"label":[{"name":"species","value":"litoria-caerulea"}],"summary":{"sampleCount":"1000","sampleSum":29969.50000000001,"quantile":[{"quantile":0.5,"value":31.1},{"quantile":0.9,"value":41.3},{"quantile":0.99,"value":41.9}],"createdTimestamp":"1970-01-01T00:00:10Z"}}]}
 }
 
 func ExampleNewConstSummary() {
@@ -382,7 +399,7 @@ func ExampleNewConstSummary() {
 	// internally).
 	metric := &dto.Metric{}
 	s.Write(metric)
-	printlnNormalized(metric)
+	fmt.Println(toNormalizedJSON(metric))
 
 	// Output:
 	// {"label":[{"name":"code","value":"200"},{"name":"method","value":"get"},{"name":"owner","value":"example"}],"summary":{"sampleCount":"4711","sampleSum":403.34,"quantile":[{"quantile":0.5,"value":42.3},{"quantile":0.9,"value":323.3}]}}
@@ -405,10 +422,11 @@ func ExampleHistogram() {
 	// internally).
 	metric := &dto.Metric{}
 	temps.Write(metric)
-	printlnNormalized(metric)
+
+	fmt.Println(toNormalizedJSON(sanitizeMetric(metric)))
 
 	// Output:
-	// {"histogram":{"sampleCount":"1000","sampleSum":29969.50000000001,"bucket":[{"cumulativeCount":"192","upperBound":20},{"cumulativeCount":"366","upperBound":25},{"cumulativeCount":"501","upperBound":30},{"cumulativeCount":"638","upperBound":35},{"cumulativeCount":"816","upperBound":40}]}}
+	// {"histogram":{"sampleCount":"1000","sampleSum":29969.50000000001,"bucket":[{"cumulativeCount":"192","upperBound":20},{"cumulativeCount":"366","upperBound":25},{"cumulativeCount":"501","upperBound":30},{"cumulativeCount":"638","upperBound":35},{"cumulativeCount":"816","upperBound":40}],"createdTimestamp":"1970-01-01T00:00:10Z"}}
 }
 
 func ExampleNewConstHistogram() {
@@ -432,7 +450,7 @@ func ExampleNewConstHistogram() {
 	// internally).
 	metric := &dto.Metric{}
 	h.Write(metric)
-	printlnNormalized(metric)
+	fmt.Println(toNormalizedJSON(metric))
 
 	// Output:
 	// {"label":[{"name":"code","value":"200"},{"name":"method","value":"get"},{"name":"owner","value":"example"}],"histogram":{"sampleCount":"4711","sampleSum":403.34,"bucket":[{"cumulativeCount":"121","upperBound":25},{"cumulativeCount":"2403","upperBound":50},{"cumulativeCount":"3221","upperBound":100},{"cumulativeCount":"4233","upperBound":200}]}}
@@ -470,7 +488,7 @@ func ExampleNewConstHistogram_WithExemplar() {
 	// internally).
 	metric := &dto.Metric{}
 	h.Write(metric)
-	printlnNormalized(metric)
+	fmt.Println(toNormalizedJSON(metric))
 
 	// Output:
 	// {"label":[{"name":"code","value":"200"},{"name":"method","value":"get"},{"name":"owner","value":"example"}],"histogram":{"sampleCount":"4711","sampleSum":403.34,"bucket":[{"cumulativeCount":"121","upperBound":25,"exemplar":{"label":[{"name":"testName","value":"testVal"}],"value":24,"timestamp":"2006-01-02T15:04:05Z"}},{"cumulativeCount":"2403","upperBound":50,"exemplar":{"label":[{"name":"testName","value":"testVal"}],"value":42,"timestamp":"2006-01-02T15:04:05Z"}},{"cumulativeCount":"3221","upperBound":100,"exemplar":{"label":[{"name":"testName","value":"testVal"}],"value":89,"timestamp":"2006-01-02T15:04:05Z"}},{"cumulativeCount":"4233","upperBound":200,"exemplar":{"label":[{"name":"testName","value":"testVal"}],"value":157,"timestamp":"2006-01-02T15:04:05Z"}}]}}
@@ -632,7 +650,7 @@ func ExampleNewMetricWithTimestamp() {
 	// internally).
 	metric := &dto.Metric{}
 	s.Write(metric)
-	printlnNormalized(metric)
+	fmt.Println(toNormalizedJSON(metric))
 
 	// Output:
 	// {"gauge":{"value":298.15},"timestampMs":"1257894000012"}

prometheus/expvar_collector_test.go

@@ -81,7 +81,7 @@ func ExampleNewExpvarCollector() {
 		if !strings.Contains(m.Desc().String(), "expvar_memstats") {
 			metric.Reset()
 			m.Write(&metric)
-			metricStrings = append(metricStrings, protoToNormalizedJSON(&metric))
+			metricStrings = append(metricStrings, toNormalizedJSON(&metric))
 		}
 	}
 	sort.Strings(metricStrings)

prometheus/gauge.go

@@ -135,7 +135,7 @@ func (g *gauge) Sub(val float64) {
 
 func (g *gauge) Write(out *dto.Metric) error {
 	val := math.Float64frombits(atomic.LoadUint64(&g.valBits))
-	return populateMetric(GaugeValue, val, g.labelPairs, nil, out)
+	return populateMetric(GaugeValue, val, g.labelPairs, nil, out, nil)
 }
 
 // GaugeVec is a Collector that bundles a set of Gauges that all share the same

prometheus/histogram.go

@@ -25,6 +25,7 @@ import (
 	dto "github.com/prometheus/client_model/go"
 
 	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/types/known/timestamppb"
 )
 
 // nativeHistogramBounds for the frac of observed values. Only relevant for
@@ -471,6 +472,9 @@ type HistogramOpts struct {
 	NativeHistogramMaxBucketNumber  uint32
 	NativeHistogramMinResetDuration time.Duration
 	NativeHistogramMaxZeroThreshold float64
+
+	// now is for testing purposes, by default it's time.Now.
+	now func() time.Time
 }
 
 // HistogramVecOpts bundles the options to create a HistogramVec metric.
@@ -519,6 +523,10 @@ func newHistogram(desc *Desc, opts HistogramOpts, labelValues ...string) Histogr
 		}
 	}
 
+	if opts.now == nil {
+		opts.now = time.Now
+	}
+
 	h := &histogram{
 		desc:                            desc,
 		upperBounds:                     opts.Buckets,
@@ -526,8 +534,8 @@ func newHistogram(desc *Desc, opts HistogramOpts, labelValues ...string) Histogr
 		nativeHistogramMaxBuckets:       opts.NativeHistogramMaxBucketNumber,
 		nativeHistogramMaxZeroThreshold: opts.NativeHistogramMaxZeroThreshold,
 		nativeHistogramMinResetDuration: opts.NativeHistogramMinResetDuration,
-		lastResetTime:                   time.Now(),
+		lastResetTime:                   opts.now(),
-		now:                             time.Now,
+		now:                             opts.now,
 	}
 	if len(h.upperBounds) == 0 && opts.NativeHistogramBucketFactor <= 1 {
 		h.upperBounds = DefBuckets
@@ -706,9 +714,11 @@ type histogram struct {
 	nativeHistogramMaxZeroThreshold float64
 	nativeHistogramMaxBuckets       uint32
 	nativeHistogramMinResetDuration time.Duration
-	lastResetTime                   time.Time // Protected by mtx.
+	// lastResetTime is protected by mtx. It is also used as created timestamp.
+	lastResetTime time.Time
 
-	now func() time.Time // To mock out time.Now() for testing.
+	// now is for testing purposes, by default it's time.Now.
+	now func() time.Time
 }
 
 func (h *histogram) Desc() *Desc {
@@ -750,6 +760,7 @@ func (h *histogram) Write(out *dto.Metric) error {
 		Bucket:           make([]*dto.Bucket, len(h.upperBounds)),
 		SampleCount:      proto.Uint64(count),
 		SampleSum:        proto.Float64(math.Float64frombits(atomic.LoadUint64(&coldCounts.sumBits))),
+		CreatedTimestamp: timestamppb.New(h.lastResetTime),
 	}
 	out.Histogram = his
 	out.Label = h.labelPairs
@@ -1194,6 +1205,7 @@ type constHistogram struct {
 	sum        float64
 	buckets    map[float64]uint64
 	labelPairs []*dto.LabelPair
+	createdTs  *timestamppb.Timestamp
 }
 
 func (h *constHistogram) Desc() *Desc {
@@ -1201,7 +1213,9 @@ func (h *constHistogram) Desc() *Desc {
 }
 
 func (h *constHistogram) Write(out *dto.Metric) error {
-	his := &dto.Histogram{}
+	his := &dto.Histogram{
+		CreatedTimestamp: h.createdTs,
+	}
 
 	buckets := make([]*dto.Bucket, 0, len(h.buckets))
 

prometheus/histogram_test.go

@@ -416,8 +416,8 @@ func TestHistogramExemplar(t *testing.T) {
 		Name:    "test",
 		Help:    "test help",
 		Buckets: []float64{1, 2, 3, 4},
+		now:     func() time.Time { return now },
 	}).(*histogram)
-	histogram.now = func() time.Time { return now }
 
 	ts := timestamppb.New(now)
 	if err := ts.CheckValid(); err != nil {
@@ -469,6 +469,8 @@ func TestHistogramExemplar(t *testing.T) {
 }
 
 func TestNativeHistogram(t *testing.T) {
+	now := time.Now()
+
 	scenarios := []struct {
 		name             string
 		observations     []float64 // With simulated interval of 1m.
@@ -499,6 +501,7 @@ func TestNativeHistogram(t *testing.T) {
 					{CumulativeCount: proto.Uint64(3), UpperBound: proto.Float64(5)},
 					{CumulativeCount: proto.Uint64(3), UpperBound: proto.Float64(10)},
 				},
+				CreatedTimestamp: timestamppb.New(now),
 			},
 		},
 		{
@@ -510,6 +513,7 @@ func TestNativeHistogram(t *testing.T) {
 				Schema:           proto.Int32(3),
 				ZeroThreshold:    proto.Float64(2.938735877055719e-39),
 				ZeroCount:        proto.Uint64(0),
+				CreatedTimestamp: timestamppb.New(now),
 			},
 		},
 		{
@@ -525,6 +529,7 @@ func TestNativeHistogram(t *testing.T) {
 				PositiveSpan: []*dto.BucketSpan{
 					{Offset: proto.Int32(0), Length: proto.Uint32(0)},
 				},
+				CreatedTimestamp: timestamppb.New(now),
 			},
 		},
 		{
@@ -543,6 +548,7 @@ func TestNativeHistogram(t *testing.T) {
 					{Offset: proto.Int32(4), Length: proto.Uint32(1)},
 				},
 				PositiveDelta:    []int64{1, 0, 0},
+				CreatedTimestamp: timestamppb.New(now),
 			},
 		},
 		{
@@ -559,6 +565,7 @@ func TestNativeHistogram(t *testing.T) {
 					{Offset: proto.Int32(0), Length: proto.Uint32(5)},
 				},
 				PositiveDelta:    []int64{1, -1, 2, -2, 2},
+				CreatedTimestamp: timestamppb.New(now),
 			},
 		},
 		{
@@ -582,6 +589,7 @@ func TestNativeHistogram(t *testing.T) {
 					{Offset: proto.Int32(-2), Length: proto.Uint32(6)},
 				},
 				PositiveDelta:    []int64{2, 0, 0, 2, -1, -2},
+				CreatedTimestamp: timestamppb.New(now),
 			},
 		},
 		{
@@ -603,6 +611,7 @@ func TestNativeHistogram(t *testing.T) {
 					{Offset: proto.Int32(-1), Length: proto.Uint32(4)},
 				},
 				PositiveDelta:    []int64{2, 2, 3, -6},
+				CreatedTimestamp: timestamppb.New(now),
 			},
 		},
 		{
@@ -619,6 +628,7 @@ func TestNativeHistogram(t *testing.T) {
 					{Offset: proto.Int32(0), Length: proto.Uint32(5)},
 				},
 				NegativeDelta:    []int64{1, -1, 2, -2, 2},
+				CreatedTimestamp: timestamppb.New(now),
 			},
 		},
 		{
@@ -639,6 +649,7 @@ func TestNativeHistogram(t *testing.T) {
 					{Offset: proto.Int32(0), Length: proto.Uint32(5)},
 				},
 				PositiveDelta:    []int64{1, -1, 2, -2, 2},
+				CreatedTimestamp: timestamppb.New(now),
 			},
 		},
 		{
@@ -660,6 +671,7 @@ func TestNativeHistogram(t *testing.T) {
 					{Offset: proto.Int32(4), Length: proto.Uint32(1)},
 				},
 				PositiveDelta:    []int64{2},
+				CreatedTimestamp: timestamppb.New(now),
 			},
 		},
 		{
@@ -676,6 +688,7 @@ func TestNativeHistogram(t *testing.T) {
 					{Offset: proto.Int32(0), Length: proto.Uint32(5)},
 				},
 				PositiveDelta:    []int64{1, -1, 2, -2, 2},
+				CreatedTimestamp: timestamppb.New(now),
 			},
 		},
 		{
@@ -693,6 +706,7 @@ func TestNativeHistogram(t *testing.T) {
 					{Offset: proto.Int32(4092), Length: proto.Uint32(1)},
 				},
 				PositiveDelta:    []int64{1, -1, 2, -2, 2, -1},
+				CreatedTimestamp: timestamppb.New(now),
 			},
 		},
 		{
@@ -713,6 +727,7 @@ func TestNativeHistogram(t *testing.T) {
 					{Offset: proto.Int32(0), Length: proto.Uint32(5)},
 				},
 				PositiveDelta:    []int64{1, -1, 2, -2, 2},
+				CreatedTimestamp: timestamppb.New(now),
 			},
 		},
 		{
@@ -730,6 +745,7 @@ func TestNativeHistogram(t *testing.T) {
 					{Offset: proto.Int32(0), Length: proto.Uint32(5)},
 				},
 				PositiveDelta:    []int64{1, -1, 2, -2, 2},
+				CreatedTimestamp: timestamppb.New(now),
 			},
 		},
 		{
@@ -747,6 +763,7 @@ func TestNativeHistogram(t *testing.T) {
 					{Offset: proto.Int32(0), Length: proto.Uint32(5)},
 				},
 				PositiveDelta:    []int64{1, 2, -1, -2, 1},
+				CreatedTimestamp: timestamppb.New(now),
 			},
 		},
 		{
@@ -765,6 +782,7 @@ func TestNativeHistogram(t *testing.T) {
 					{Offset: proto.Int32(1), Length: proto.Uint32(7)},
 				},
 				PositiveDelta:    []int64{1, 1, -2, 2, -2, 0, 1},
+				CreatedTimestamp: timestamppb.New(now),
 			},
 		},
 		{
@@ -783,6 +801,7 @@ func TestNativeHistogram(t *testing.T) {
 					{Offset: proto.Int32(2), Length: proto.Uint32(7)},
 				},
 				PositiveDelta:    []int64{2, -2, 2, -2, 0, 1, 0},
+				CreatedTimestamp: timestamppb.New(now),
 			},
 		},
 		{
@@ -802,6 +821,7 @@ func TestNativeHistogram(t *testing.T) {
 					{Offset: proto.Int32(7), Length: proto.Uint32(2)},
 				},
 				PositiveDelta:    []int64{1, 0},
+				CreatedTimestamp: timestamppb.New(now.Add(8 * time.Minute)), // We expect reset to happen after 8 observations.
 			},
 		},
 		{
@@ -819,6 +839,7 @@ func TestNativeHistogram(t *testing.T) {
 					{Offset: proto.Int32(0), Length: proto.Uint32(5)},
 				},
 				NegativeDelta:    []int64{1, -1, 2, -2, 2},
+				CreatedTimestamp: timestamppb.New(now),
 			},
 		},
 		{
@@ -836,6 +857,7 @@ func TestNativeHistogram(t *testing.T) {
 					{Offset: proto.Int32(0), Length: proto.Uint32(5)},
 				},
 				NegativeDelta:    []int64{1, 2, -1, -2, 1},
+				CreatedTimestamp: timestamppb.New(now),
 			},
 		},
 		{
@@ -854,6 +876,7 @@ func TestNativeHistogram(t *testing.T) {
 					{Offset: proto.Int32(1), Length: proto.Uint32(7)},
 				},
 				NegativeDelta:    []int64{1, 1, -2, 2, -2, 0, 1},
+				CreatedTimestamp: timestamppb.New(now),
 			},
 		},
 		{
@@ -872,6 +895,7 @@ func TestNativeHistogram(t *testing.T) {
 					{Offset: proto.Int32(2), Length: proto.Uint32(7)},
 				},
 				NegativeDelta:    []int64{2, -2, 2, -2, 0, 1, 0},
+				CreatedTimestamp: timestamppb.New(now),
 			},
 		},
 		{
@@ -891,6 +915,7 @@ func TestNativeHistogram(t *testing.T) {
 					{Offset: proto.Int32(7), Length: proto.Uint32(2)},
 				},
 				NegativeDelta:    []int64{1, 0},
+				CreatedTimestamp: timestamppb.New(now.Add(8 * time.Minute)), // We expect reset to happen after 8 observations.
 			},
 		},
 		{
@@ -909,6 +934,7 @@ func TestNativeHistogram(t *testing.T) {
 					{Offset: proto.Int32(7), Length: proto.Uint32(2)},
 				},
 				PositiveDelta:    []int64{1, 0},
+				CreatedTimestamp: timestamppb.New(now.Add(10 * time.Minute)), // We expect reset to happen after 9 minutes.
 			},
 		},
 		{
@@ -928,12 +954,15 @@ func TestNativeHistogram(t *testing.T) {
 					{Offset: proto.Int32(7), Length: proto.Uint32(2)},
 				},
 				PositiveDelta:    []int64{1, 0},
+				CreatedTimestamp: timestamppb.New(now.Add(10 * time.Minute)), // We expect reset to happen after 9 minutes.
 			},
 		},
 	}
 
 	for _, s := range scenarios {
 		t.Run(s.name, func(t *testing.T) {
+			ts := now
+
 			his := NewHistogram(HistogramOpts{
 				Name:                            "name",
 				Help:                            "help",
@@ -942,12 +971,10 @@ func TestNativeHistogram(t *testing.T) {
 				NativeHistogramMaxBucketNumber:  s.maxBuckets,
 				NativeHistogramMinResetDuration: s.minResetDuration,
 				NativeHistogramMaxZeroThreshold: s.maxZeroThreshold,
+				now:                             func() time.Time { return ts },
 			})
-			ts := time.Now().Add(30 * time.Second)
+
-			now := func() time.Time {
+			ts = ts.Add(time.Minute)
-				return ts
-			}
-			his.(*histogram).now = now
 			for _, o := range s.observations {
 				his.Observe(o)
 				ts = ts.Add(time.Minute)
@@ -972,6 +999,8 @@ func TestNativeHistogramConcurrency(t *testing.T) {
 	rand.Seed(42)
 
 	it := func(n uint32) bool {
+		ts := time.Now().Add(30 * time.Second).Unix()
+
 		mutations := int(n%1e4 + 1e4)
 		concLevel := int(n%5 + 1)
 		total := mutations * concLevel
@@ -988,13 +1017,10 @@ func TestNativeHistogramConcurrency(t *testing.T) {
 			NativeHistogramMaxBucketNumber:  50,
 			NativeHistogramMinResetDuration: time.Hour, // Comment out to test for totals below.
 			NativeHistogramMaxZeroThreshold: 0.001,
-		})
+			now: func() time.Time {
-
-		ts := time.Now().Add(30 * time.Second).Unix()
-		now := func() time.Time {
 				return time.Unix(atomic.LoadInt64(&ts), 0)
-		}
+			},
-		his.(*histogram).now = now
+		})
 
 		allVars := make([]float64, total)
 		var sampleSum float64
@@ -1152,3 +1178,82 @@ func TestGetLe(t *testing.T) {
 		}
 	}
 }
+
+func TestHistogramCreatedTimestamp(t *testing.T) {
+	now := time.Now()
+
+	histogram := NewHistogram(HistogramOpts{
+		Name:    "test",
+		Help:    "test help",
+		Buckets: []float64{1, 2, 3, 4},
+		now:     func() time.Time { return now },
+	})
+
+	var metric dto.Metric
+	if err := histogram.Write(&metric); err != nil {
+		t.Fatal(err)
+	}
+
+	if metric.Histogram.CreatedTimestamp.AsTime().Unix() != now.Unix() {
+		t.Errorf("expected created timestamp %d, got %d", now.Unix(), metric.Histogram.CreatedTimestamp.AsTime().Unix())
+	}
+}
+
+func TestHistogramVecCreatedTimestamp(t *testing.T) {
+	now := time.Now()
+
+	histogramVec := NewHistogramVec(HistogramOpts{
+		Name:    "test",
+		Help:    "test help",
+		Buckets: []float64{1, 2, 3, 4},
+		now:     func() time.Time { return now },
+	}, []string{"label"})
+	histogram := histogramVec.WithLabelValues("value").(Histogram)
+
+	var metric dto.Metric
+	if err := histogram.Write(&metric); err != nil {
+		t.Fatal(err)
+	}
+
+	if metric.Histogram.CreatedTimestamp.AsTime().Unix() != now.Unix() {
+		t.Errorf("expected created timestamp %d, got %d", now.Unix(), metric.Histogram.CreatedTimestamp.AsTime().Unix())
+	}
+}
+
+func TestHistogramVecCreatedTimestampWithDeletes(t *testing.T) {
+	now := time.Now()
+
+	histogramVec := NewHistogramVec(HistogramOpts{
+		Name:    "test",
+		Help:    "test help",
+		Buckets: []float64{1, 2, 3, 4},
+		now:     func() time.Time { return now },
+	}, []string{"label"})
+
+	// First use of "With" should populate CT.
+	histogramVec.WithLabelValues("1")
+	expected := map[string]time.Time{"1": now}
+
+	now = now.Add(1 * time.Hour)
+	expectCTsForMetricVecValues(t, histogramVec.MetricVec, dto.MetricType_HISTOGRAM, expected)
+
+	// Two more labels at different times.
+	histogramVec.WithLabelValues("2")
+	expected["2"] = now
+
+	now = now.Add(1 * time.Hour)
+
+	histogramVec.WithLabelValues("3")
+	expected["3"] = now
+
+	now = now.Add(1 * time.Hour)
+	expectCTsForMetricVecValues(t, histogramVec.MetricVec, dto.MetricType_HISTOGRAM, expected)
+
+	// Recreate metric instance should reset created timestamp to now.
+	histogramVec.DeleteLabelValues("1")
+	histogramVec.WithLabelValues("1")
+	expected["1"] = now
+
+	now = now.Add(1 * time.Hour)
+	expectCTsForMetricVecValues(t, histogramVec.MetricVec, dto.MetricType_HISTOGRAM, expected)
+}

prometheus/metric.go

@@ -92,6 +92,9 @@ type Opts struct {
 	// machine_role metric). See also
 	// https://prometheus.io/docs/instrumenting/writing_exporters/#target-labels-not-static-scraped-labels
 	ConstLabels Labels
+
+	// now is for testing purposes, by default it's time.Now.
+	now func() time.Time
 }
 
 // BuildFQName joins the given three name components by "_". Empty name

prometheus/registry_test.go

@@ -37,6 +37,7 @@ import (
 	dto "github.com/prometheus/client_model/go"
 	"github.com/prometheus/common/expfmt"
 	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/types/known/timestamppb"
 )
 
 // uncheckedCollector wraps a Collector but its Describe method yields no Desc.
@@ -139,6 +140,7 @@ metric: <
 				},
 				Counter: &dto.Counter{
 					Value:            proto.Float64(1),
+					CreatedTimestamp: timestamppb.New(time.Now()),
 				},
 			},
 			{
@@ -154,6 +156,7 @@ metric: <
 				},
 				Counter: &dto.Counter{
 					Value:            proto.Float64(1),
+					CreatedTimestamp: timestamppb.New(time.Now()),
 				},
 			},
 		},

prometheus/summary.go

@@ -26,6 +26,7 @@ import (
 
 	"github.com/beorn7/perks/quantile"
 	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/types/known/timestamppb"
 )
 
 // quantileLabel is used for the label that defines the quantile in a
@@ -145,6 +146,9 @@ type SummaryOpts struct {
 	// is the internal buffer size of the underlying package
 	// "github.com/bmizerany/perks/quantile").
 	BufCap uint32
+
+	// now is for testing purposes, by default it's time.Now.
+	now func() time.Time
 }
 
 // SummaryVecOpts bundles the options to create a SummaryVec metric.
@@ -222,6 +226,9 @@ func newSummary(desc *Desc, opts SummaryOpts, labelValues ...string) Summary {
 		opts.BufCap = DefBufCap
 	}
 
+	if opts.now == nil {
+		opts.now = time.Now
+	}
 	if len(opts.Objectives) == 0 {
 		// Use the lock-free implementation of a Summary without objectives.
 		s := &noObjectivesSummary{
@@ -230,6 +237,7 @@ func newSummary(desc *Desc, opts SummaryOpts, labelValues ...string) Summary {
 			counts:     [2]*summaryCounts{{}, {}},
 		}
 		s.init(s) // Init self-collection.
+		s.createdTs = timestamppb.New(opts.now())
 		return s
 	}
 
@@ -245,7 +253,7 @@ func newSummary(desc *Desc, opts SummaryOpts, labelValues ...string) Summary {
 		coldBuf:        make([]float64, 0, opts.BufCap),
 		streamDuration: opts.MaxAge / time.Duration(opts.AgeBuckets),
 	}
-	s.headStreamExpTime = time.Now().Add(s.streamDuration)
+	s.headStreamExpTime = opts.now().Add(s.streamDuration)
 	s.hotBufExpTime = s.headStreamExpTime
 
 	for i := uint32(0); i < opts.AgeBuckets; i++ {
@@ -259,6 +267,7 @@ func newSummary(desc *Desc, opts SummaryOpts, labelValues ...string) Summary {
 	sort.Float64s(s.sortedObjectives)
 
 	s.init(s) // Init self-collection.
+	s.createdTs = timestamppb.New(opts.now())
 	return s
 }
 
@@ -286,6 +295,8 @@ type summary struct {
 	headStream                       *quantile.Stream
 	headStreamIdx                    int
 	headStreamExpTime, hotBufExpTime time.Time
+
+	createdTs *timestamppb.Timestamp
 }
 
 func (s *summary) Desc() *Desc {
@@ -307,7 +318,9 @@ func (s *summary) Observe(v float64) {
 }
 
 func (s *summary) Write(out *dto.Metric) error {
-	sum := &dto.Summary{}
+	sum := &dto.Summary{
+		CreatedTimestamp: s.createdTs,
+	}
 	qs := make([]*dto.Quantile, 0, len(s.objectives))
 
 	s.bufMtx.Lock()
@@ -440,6 +453,8 @@ type noObjectivesSummary struct {
 	counts [2]*summaryCounts
 
 	labelPairs []*dto.LabelPair
+
+	createdTs *timestamppb.Timestamp
 }
 
 func (s *noObjectivesSummary) Desc() *Desc {
@@ -492,6 +507,7 @@ func (s *noObjectivesSummary) Write(out *dto.Metric) error {
 	sum := &dto.Summary{
 		SampleCount:      proto.Uint64(count),
 		SampleSum:        proto.Float64(math.Float64frombits(atomic.LoadUint64(&coldCounts.sumBits))),
+		CreatedTimestamp: s.createdTs,
 	}
 
 	out.Summary = sum
@@ -681,6 +697,7 @@ type constSummary struct {
 	sum        float64
 	quantiles  map[float64]float64
 	labelPairs []*dto.LabelPair
+	createdTs  *timestamppb.Timestamp
 }
 
 func (s *constSummary) Desc() *Desc {
@@ -688,7 +705,9 @@ func (s *constSummary) Desc() *Desc {
 }
 
 func (s *constSummary) Write(out *dto.Metric) error {
-	sum := &dto.Summary{}
+	sum := &dto.Summary{
+		CreatedTimestamp: s.createdTs,
+	}
 	qs := make([]*dto.Quantile, 0, len(s.quantiles))
 
 	sum.SampleCount = proto.Uint64(s.count)

prometheus/summary_test.go

@@ -26,10 +26,13 @@ import (
 )
 
 func TestSummaryWithDefaultObjectives(t *testing.T) {
+	now := time.Now()
+
 	reg := NewRegistry()
 	summaryWithDefaultObjectives := NewSummary(SummaryOpts{
 		Name: "default_objectives",
 		Help: "Test help.",
+		now:  func() time.Time { return now },
 	})
 	if err := reg.Register(summaryWithDefaultObjectives); err != nil {
 		t.Error(err)
@@ -42,6 +45,10 @@ func TestSummaryWithDefaultObjectives(t *testing.T) {
 	if len(m.GetSummary().Quantile) != 0 {
 		t.Error("expected no objectives in summary")
 	}
+
+	if !m.Summary.CreatedTimestamp.AsTime().Equal(now) {
+		t.Errorf("expected created timestamp %s, got %s", now, m.Summary.CreatedTimestamp.AsTime())
+	}
 }
 
 func TestSummaryWithoutObjectives(t *testing.T) {
@@ -420,3 +427,50 @@ func getBounds(vars []float64, q, ε float64) (min, max float64) {
 	}
 	return
 }
+
+func TestSummaryVecCreatedTimestampWithDeletes(t *testing.T) {
+	for _, tcase := range []struct {
+		desc       string
+		objectives map[float64]float64
+	}{
+		{desc: "summary with objectives", objectives: map[float64]float64{1.0: 1.0}},
+		{desc: "no objectives summary", objectives: nil},
+	} {
+		now := time.Now()
+		t.Run(tcase.desc, func(t *testing.T) {
+			summaryVec := NewSummaryVec(SummaryOpts{
+				Name:       "test",
+				Help:       "test help",
+				Objectives: tcase.objectives,
+				now:        func() time.Time { return now },
+			}, []string{"label"})
+
+			// First use of "With" should populate CT.
+			summaryVec.WithLabelValues("1")
+			expected := map[string]time.Time{"1": now}
+
+			now = now.Add(1 * time.Hour)
+			expectCTsForMetricVecValues(t, summaryVec.MetricVec, dto.MetricType_SUMMARY, expected)
+
+			// Two more labels at different times.
+			summaryVec.WithLabelValues("2")
+			expected["2"] = now
+
+			now = now.Add(1 * time.Hour)
+
+			summaryVec.WithLabelValues("3")
+			expected["3"] = now
+
+			now = now.Add(1 * time.Hour)
+			expectCTsForMetricVecValues(t, summaryVec.MetricVec, dto.MetricType_SUMMARY, expected)
+
+			// Recreate metric instance should reset created timestamp to now.
+			summaryVec.DeleteLabelValues("1")
+			summaryVec.WithLabelValues("1")
+			expected["1"] = now
+
+			now = now.Add(1 * time.Hour)
+			expectCTsForMetricVecValues(t, summaryVec.MetricVec, dto.MetricType_SUMMARY, expected)
+		})
+	}
+}

prometheus/utils_test.go

@@ -15,21 +15,42 @@ package prometheus_test
 import (
 	"bytes"
 	"encoding/json"
-	"fmt"
+	"time"
 
+	dto "github.com/prometheus/client_model/go"
 	"google.golang.org/protobuf/encoding/protojson"
 	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/types/known/timestamppb"
 )
 
-// printlnNormalized is a helper function to compare proto messages in json format.
+// sanitizeMetric injects expected fake created timestamp value "1970-01-01T00:00:10Z",
-// Without removing brittle, we can't assert that two proto messages in json/text format are equal.
+// so we can compare it in examples. It modifies metric in-place, the returned pointer
-// Read more in https://github.com/golang/protobuf/issues/1121
+// is for convenience.
-func printlnNormalized(m proto.Message) {
+func sanitizeMetric(metric *dto.Metric) *dto.Metric {
-	fmt.Println(protoToNormalizedJSON(m))
+	if metric.Counter != nil && metric.Counter.CreatedTimestamp != nil {
+		metric.Counter.CreatedTimestamp = timestamppb.New(time.Unix(10, 0))
+	}
+	if metric.Summary != nil && metric.Summary.CreatedTimestamp != nil {
+		metric.Summary.CreatedTimestamp = timestamppb.New(time.Unix(10, 0))
+	}
+	if metric.Histogram != nil && metric.Histogram.CreatedTimestamp != nil {
+		metric.Histogram.CreatedTimestamp = timestamppb.New(time.Unix(10, 0))
+	}
+	return metric
 }
 
-// protoToNormalizedJSON works as printlnNormalized, but returns the string instead of printing.
+// sanitizeMetricFamily is like sanitizeMetric, but for multiple metrics.
-func protoToNormalizedJSON(m proto.Message) string {
+func sanitizeMetricFamily(f *dto.MetricFamily) *dto.MetricFamily {
+	for _, m := range f.Metric {
+		sanitizeMetric(m)
+	}
+	return f
+}
+
+// toNormalizedJSON removes fake random space from proto JSON original marshaller.
+// It is required, so we can compare proto messages in json format.
+// Read more in https://github.com/golang/protobuf/issues/1121
+func toNormalizedJSON(m proto.Message) string {
 	mAsJSON, err := protojson.Marshal(m)
 	if err != nil {
 		panic(err)

prometheus/value.go

@@ -14,6 +14,7 @@
 package prometheus
 
 import (
+	"errors"
 	"fmt"
 	"sort"
 	"time"
@@ -91,7 +92,7 @@ func (v *valueFunc) Desc() *Desc {
 }
 
 func (v *valueFunc) Write(out *dto.Metric) error {
-	return populateMetric(v.valType, v.function(), v.labelPairs, nil, out)
+	return populateMetric(v.valType, v.function(), v.labelPairs, nil, out, nil)
 }
 
 // NewConstMetric returns a metric with one fixed value that cannot be
@@ -110,7 +111,7 @@ func NewConstMetric(desc *Desc, valueType ValueType, value float64, labelValues
 	}
 
 	metric := &dto.Metric{}
-	if err := populateMetric(valueType, value, MakeLabelPairs(desc, labelValues), nil, metric); err != nil {
+	if err := populateMetric(valueType, value, MakeLabelPairs(desc, labelValues), nil, metric, nil); err != nil {
 		return nil, err
 	}
 
@@ -130,6 +131,43 @@ func MustNewConstMetric(desc *Desc, valueType ValueType, value float64, labelVal
 	return m
 }
 
+// NewConstMetricWithCreatedTimestamp does the same thing as NewConstMetric, but generates Counters
+// with created timestamp set and returns an error for other metric types.
+func NewConstMetricWithCreatedTimestamp(desc *Desc, valueType ValueType, value float64, ct time.Time, labelValues ...string) (Metric, error) {
+	if desc.err != nil {
+		return nil, desc.err
+	}
+	if err := validateLabelValues(labelValues, len(desc.variableLabels.names)); err != nil {
+		return nil, err
+	}
+	switch valueType {
+	case CounterValue:
+		break
+	default:
+		return nil, errors.New("created timestamps are only supported for counters")
+	}
+
+	metric := &dto.Metric{}
+	if err := populateMetric(valueType, value, MakeLabelPairs(desc, labelValues), nil, metric, timestamppb.New(ct)); err != nil {
+		return nil, err
+	}
+
+	return &constMetric{
+		desc:   desc,
+		metric: metric,
+	}, nil
+}
+
+// MustNewConstMetricWithCreatedTimestamp is a version of NewConstMetricWithCreatedTimestamp that panics where
+// NewConstMetricWithCreatedTimestamp would have returned an error.
+func MustNewConstMetricWithCreatedTimestamp(desc *Desc, valueType ValueType, value float64, ct time.Time, labelValues ...string) Metric {
+	m, err := NewConstMetricWithCreatedTimestamp(desc, valueType, value, ct, labelValues...)
+	if err != nil {
+		panic(err)
+	}
+	return m
+}
+
 type constMetric struct {
 	desc   *Desc
 	metric *dto.Metric
@@ -153,11 +191,12 @@ func populateMetric(
 	labelPairs []*dto.LabelPair,
 	e *dto.Exemplar,
 	m *dto.Metric,
+	ct *timestamppb.Timestamp,
 ) error {
 	m.Label = labelPairs
 	switch t {
 	case CounterValue:
-		m.Counter = &dto.Counter{Value: proto.Float64(v), Exemplar: e}
+		m.Counter = &dto.Counter{Value: proto.Float64(v), Exemplar: e, CreatedTimestamp: ct}
 	case GaugeValue:
 		m.Gauge = &dto.Gauge{Value: proto.Float64(v)}
 	case UntypedValue:

prometheus/value_test.go

@@ -16,6 +16,10 @@ package prometheus
 import (
 	"fmt"
 	"testing"
+	"time"
+
+	dto "github.com/prometheus/client_model/go"
+	"google.golang.org/protobuf/types/known/timestamppb"
 )
 
 func TestNewConstMetricInvalidLabelValues(t *testing.T) {
@@ -54,3 +58,54 @@ func TestNewConstMetricInvalidLabelValues(t *testing.T) {
 		}
 	}
 }
+
+func TestNewConstMetricWithCreatedTimestamp(t *testing.T) {
+	now := time.Now()
+
+	for _, tcase := range []struct {
+		desc             string
+		metricType       ValueType
+		createdTimestamp time.Time
+		expecErr         bool
+		expectedCt       *timestamppb.Timestamp
+	}{
+		{
+			desc:             "gauge with CT",
+			metricType:       GaugeValue,
+			createdTimestamp: now,
+			expecErr:         true,
+			expectedCt:       nil,
+		},
+		{
+			desc:             "counter with CT",
+			metricType:       CounterValue,
+			createdTimestamp: now,
+			expecErr:         false,
+			expectedCt:       timestamppb.New(now),
+		},
+	} {
+		t.Run(tcase.desc, func(t *testing.T) {
+			metricDesc := NewDesc(
+				"sample_value",
+				"sample value",
+				nil,
+				nil,
+			)
+			m, err := NewConstMetricWithCreatedTimestamp(metricDesc, tcase.metricType, float64(1), tcase.createdTimestamp)
+			if tcase.expecErr && err == nil {
+				t.Errorf("Expected error is test %s, got no err", tcase.desc)
+			}
+			if !tcase.expecErr && err != nil {
+				t.Errorf("Didn't expect error in test %s, got %s", tcase.desc, err.Error())
+			}
+
+			if tcase.expectedCt != nil {
+				var metric dto.Metric
+				m.Write(&metric)
+				if metric.Counter.CreatedTimestamp.AsTime() != tcase.expectedCt.AsTime() {
+					t.Errorf("Expected timestamp %v, got %v", tcase.expectedCt, &metric.Counter.CreatedTimestamp)
+				}
+			}
+		})
+	}
+}

prometheus/wrap_test.go

@@ -17,6 +17,7 @@ import (
 	"fmt"
 	"strings"
 	"testing"
+	"time"
 
 	dto "github.com/prometheus/client_model/go"
 	"google.golang.org/protobuf/proto"
@@ -43,9 +44,12 @@ func toMetricFamilies(cs ...Collector) []*dto.MetricFamily {
 }
 
 func TestWrap(t *testing.T) {
+	now := time.Now()
+	nowFn := func() time.Time { return now }
 	simpleCnt := NewCounter(CounterOpts{
 		Name: "simpleCnt",
 		Help: "helpSimpleCnt",
+		now:  nowFn,
 	})
 	simpleCnt.Inc()
 
@@ -58,6 +62,7 @@ func TestWrap(t *testing.T) {
 	preCnt := NewCounter(CounterOpts{
 		Name: "pre_simpleCnt",
 		Help: "helpSimpleCnt",
+		now:  nowFn,
 	})
 	preCnt.Inc()
 
@@ -65,6 +70,7 @@ func TestWrap(t *testing.T) {
 		Name:        "simpleCnt",
 		Help:        "helpSimpleCnt",
 		ConstLabels: Labels{"foo": "bar"},
+		now:         nowFn,
 	})
 	barLabeledCnt.Inc()
 
@@ -72,6 +78,7 @@ func TestWrap(t *testing.T) {
 		Name:        "simpleCnt",
 		Help:        "helpSimpleCnt",
 		ConstLabels: Labels{"foo": "baz"},
+		now:         nowFn,
 	})
 	bazLabeledCnt.Inc()
 
@@ -79,6 +86,7 @@ func TestWrap(t *testing.T) {
 		Name:        "pre_simpleCnt",
 		Help:        "helpSimpleCnt",
 		ConstLabels: Labels{"foo": "bar"},
+		now:         nowFn,
 	})
 	labeledPreCnt.Inc()
 
@@ -86,6 +94,7 @@ func TestWrap(t *testing.T) {
 		Name:        "pre_simpleCnt",
 		Help:        "helpSimpleCnt",
 		ConstLabels: Labels{"foo": "bar", "dings": "bums"},
+		now:         nowFn,
 	})
 	twiceLabeledPreCnt.Inc()