Optinally print OM created lines

Signed-off-by: Arthur Silva Sens <arthur.sens@coralogix.com>

examples/createdtimestamps/main.go

@@ -0,0 +1,62 @@
+// Copyright 2022 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// A simple example of how to exposed created timestamps in OpenMetrics format.
+
+package main
+
+import (
+	"log"
+	"net/http"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+)
+
+func main() {
+	requestDurations := prometheus.NewHistogram(prometheus.HistogramOpts{
+		Name:    "http_request_duration_seconds",
+		Help:    "A histogram of the HTTP request durations in seconds.",
+		Buckets: prometheus.ExponentialBuckets(0.1, 1.5, 5),
+	})
+
+	// Create non-global registry.
+	registry := prometheus.NewRegistry()
+	registry.MustRegister(
+		requestDurations,
+	)
+
+	go func() {
+		for {
+			// Record fictional latency.
+			now := time.Now()
+			requestDurations.Observe(time.Since(now).Seconds())
+			time.Sleep(600 * time.Millisecond)
+		}
+	}()
+
+	// Expose /metrics HTTP endpoint using the created custom registry.
+	http.Handle(
+		"/metrics", promhttp.HandlerFor(
+			registry,
+			promhttp.HandlerOpts{
+				OpenMetricsOptions: promhttp.OpenMetricsOptions{
+					Enable:                  true,
+					EnableCreatedTimestamps: true,
+				},
+			}),
+	)
+	// To test: curl -H 'Accept: application/openmetrics-text' localhost:8080/metrics
+	log.Fatalln(http.ListenAndServe(":8080", nil))
+}

examples/exemplars/main.go

@@ -61,7 +61,9 @@ func main() {
 		"/metrics", promhttp.HandlerFor(
 			registry,
 			promhttp.HandlerOpts{
-				EnableOpenMetrics: true,
+				OpenMetricsOptions: promhttp.OpenMetricsOptions{
+					Enable: true,
+				},
 			}),
 	)
 	// To test: curl -H 'Accept: application/openmetrics-text' localhost:8080/metrics

examples/gocollector/main.go

@@ -54,8 +54,10 @@ func main() {
 	http.Handle("/metrics", promhttp.HandlerFor(
 		reg,
 		promhttp.HandlerOpts{
-			// Opt into OpenMetrics to support exemplars.
+			OpenMetricsOptions: promhttp.OpenMetricsOptions{
-			EnableOpenMetrics: true,
+				// Opt into OpenMetrics to support exemplars.
+				Enable: true,
+			},
 		},
 	))
 	fmt.Println("Hello world from new Go Collector!")

examples/random/main.go

@@ -134,8 +134,10 @@ func main() {
 	http.Handle("/metrics", promhttp.HandlerFor(
 		reg,
 		promhttp.HandlerOpts{
-			// Opt into OpenMetrics to support exemplars.
+			OpenMetricsOptions: promhttp.OpenMetricsOptions{
-			EnableOpenMetrics: true,
+				// Opt into OpenMetrics to support exemplars.
+				Enable: true,
+			},
 			// Pass custom registry
 			Registry: reg,
 		},

prometheus/promhttp/http.go

@@ -185,7 +185,7 @@ func HandlerForTransactional(reg prometheus.TransactionalGatherer, opts HandlerO
 		}
 
 		var contentType expfmt.Format
-		if opts.EnableOpenMetrics {
+		if opts.EnableOpenMetrics || opts.OpenMetricsOptions.Enable {
 			contentType = expfmt.NegotiateIncludingOpenMetrics(req.Header)
 		} else {
 			contentType = expfmt.Negotiate(req.Header)
@@ -207,7 +207,13 @@ func HandlerForTransactional(reg prometheus.TransactionalGatherer, opts HandlerO
 		if encodingHeader != string(Identity) {
 			rsp.Header().Set(contentEncodingHeader, encodingHeader)
 		}
-		enc := expfmt.NewEncoder(w, contentType)
+
+		var enc expfmt.Encoder
+		if opts.OpenMetricsOptions.EnableCreatedTimestamps {
+			enc = expfmt.NewEncoder(w, contentType, expfmt.WithCreatedLines())
+		} else {
+			enc = expfmt.NewEncoder(w, contentType)
+		}
 
 		// handleError handles the error according to opts.ErrorHandling
 		// and returns true if we have to abort after the handling.
@@ -398,16 +404,11 @@ type HandlerOpts struct {
 	// away). Until the implementation is improved, it is recommended to
 	// implement a separate timeout in potentially slow Collectors.
 	Timeout time.Duration
-	// If true, the experimental OpenMetrics encoding is added to the
+	// Deprecated: Use OpenMetricsOptions.Enable instead.
-	// possible options during content negotiation. Note that Prometheus
-	// 2.5.0+ will negotiate OpenMetrics as first priority. OpenMetrics is
-	// the only way to transmit exemplars. However, the move to OpenMetrics
-	// is not completely transparent. Most notably, the values of "quantile"
-	// labels of Summaries and "le" labels of Histograms are formatted with
-	// a trailing ".0" if they would otherwise look like integer numbers
-	// (which changes the identity of the resulting series on the Prometheus
-	// server).
 	EnableOpenMetrics bool
+	// OpenMetricsOptions holds settings for the experimental OpenMetrics encoding.
+	// It can be used to enable OpenMetrics encoding and for setting extra options.
+	OpenMetricsOptions OpenMetricsOptions
 	// ProcessStartTime allows setting process start timevalue that will be exposed
 	// with "Process-Start-Time-Unix" response header along with the metrics
 	// payload. This allow callers to have efficient transformations to cumulative
@@ -418,6 +419,43 @@ type HandlerOpts struct {
 	ProcessStartTime time.Time
 }
 
+type OpenMetricsOptions struct {
+	// Enable specifies if the experimental OpenMetrics encoding is added to the
+	// possible options during content negotiation.
+	//
+	// Note that Prometheus 2.5.0+ might negotiate OpenMetrics Text format
+	// as first priority unless user uses custom scrape protocol prioritization or
+	// histograms feature is enabled (then Prometheus proto format is prioritized,
+	// which client_golang supports).
+	//
+	// Keep in mind that the move to OpenMetrics is not completely transparent. Most notably,
+	// the values of "quantile" labels of Summaries and "le" labels of Histograms are
+	// formatted with a trailing ".0" if they would otherwise look like integer numbers
+	// (which changes the identity of the resulting series on the Prometheus
+	// server).
+	//
+	// See other options in OpenMetricsOptions to learn how to enable some special
+	// features e.g. potentially dangerous created timestamp series.
+	Enable bool
+	// EnableCreatedTimestamps specifies if this handler should add, extra, synthetic
+	// Created Timestamps for counters, histograms and summaries, which for the current
+	// version of OpenMetrics are defined as extra series with the same name and "_created"
+	// suffix. See also the OpenMetrics specification for more details
+	// https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#counter-1
+	//
+	// Created timestamps are used to improve the accuracy of reset detection,
+	// but the way it's designed in OpenMetrics 1.0 it also dramatically increases cardinality
+	// if the scraper does not handle those metrics correctly (converting to created timestamp
+	// instead of leaving those series as-is). New OpenMetrics versions might improve
+	// this situation.
+	//
+	// Prometheus introduced the feature flag 'created-timestamp-zero-ingestion'
+	// in version 2.50.0, but only for the Prometheus protobuf format. Starting in
+	// future Prometheus version, the feature flag will be extended to the OpenMetrics
+	// text format, thus safe to be enabled to improve accuracy of counters in Prometheus.
+	EnableCreatedTimestamps bool
+}
+
 // httpError removes any content-encoding header and then calls http.Error with
 // the provided error and http.StatusInternalServerError. Error contents is
 // supposed to be uncompressed plain text. Same as with a plain http.Error, this