Merge pull request #706 from prometheus/beorn7/exemplars

Add exemplars to counter and histogram
This commit is contained in:
Björn Rabenstein 2020-01-24 17:17:39 +01:00 committed by GitHub
commit ba7901740d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 369 additions and 64 deletions

View File

@ -18,6 +18,7 @@ package main
import ( import (
"flag" "flag"
"fmt"
"log" "log"
"math" "math"
"math/rand" "math/rand"
@ -89,7 +90,11 @@ func main() {
for { for {
v := (rand.NormFloat64() * *normDomain) + *normMean v := (rand.NormFloat64() * *normDomain) + *normMean
rpcDurations.WithLabelValues("normal").Observe(v) rpcDurations.WithLabelValues("normal").Observe(v)
rpcDurationsHistogram.Observe(v) rpcDurationsHistogram.ObserveWithExemplar(
// Demonstrate exemplar support with a dummy ID. This would be
// something like a trace ID in a real application.
v, prometheus.Labels{"dummyID": fmt.Sprint(rand.Intn(100000))},
)
time.Sleep(time.Duration(75*oscillationFactor()) * time.Millisecond) time.Sleep(time.Duration(75*oscillationFactor()) * time.Millisecond)
} }
}() }()
@ -103,6 +108,12 @@ func main() {
}() }()
// Expose the registered metrics via HTTP. // Expose the registered metrics via HTTP.
http.Handle("/metrics", promhttp.Handler()) http.Handle("/metrics", promhttp.HandlerFor(
prometheus.DefaultGatherer,
promhttp.HandlerOpts{
// Opt into OpenMetrics to support exemplars.
EnableOpenMetrics: true,
},
))
log.Fatal(http.ListenAndServe(*addr, nil)) log.Fatal(http.ListenAndServe(*addr, nil))
} }

4
go.mod
View File

@ -5,8 +5,8 @@ require (
github.com/cespare/xxhash/v2 v2.1.1 github.com/cespare/xxhash/v2 v2.1.1
github.com/golang/protobuf v1.3.2 github.com/golang/protobuf v1.3.2
github.com/json-iterator/go v1.1.8 github.com/json-iterator/go v1.1.8
github.com/prometheus/client_model v0.1.0 github.com/prometheus/client_model v0.2.0
github.com/prometheus/common v0.7.0 github.com/prometheus/common v0.9.0
github.com/prometheus/procfs v0.0.8 github.com/prometheus/procfs v0.0.8
golang.org/x/sys v0.0.0-20191220142924-d4481acd189f golang.org/x/sys v0.0.0-20191220142924-d4481acd189f
) )

10
go.sum
View File

@ -58,12 +58,12 @@ github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 h1:idejC8f
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE=
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/client_model v0.1.0 h1:ElTg5tNp4DqfV7UQjDqv2+RJlNzsDtvNAWccbItceIE= github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M=
github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/common v0.4.1 h1:K0MGApIoQvMw27RTdJkPbr3JZ7DNbtxQNyi5STVM6Kw= github.com/prometheus/common v0.4.1 h1:K0MGApIoQvMw27RTdJkPbr3JZ7DNbtxQNyi5STVM6Kw=
github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
github.com/prometheus/common v0.7.0 h1:L+1lyG48J1zAQXA3RBX/nG/B3gjlHq0zTt2tlbJLyCY= github.com/prometheus/common v0.9.0 h1:yg//x/8DqN+PxXTBFMwVCopGqDn3wSxmbF/3PCuu1bk=
github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= github.com/prometheus/common v0.9.0/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4=
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNGfs= github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNGfs=
github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
@ -97,4 +97,4 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

View File

@ -17,6 +17,7 @@ import (
"errors" "errors"
"math" "math"
"sync/atomic" "sync/atomic"
"time"
dto "github.com/prometheus/client_model/go" dto "github.com/prometheus/client_model/go"
) )
@ -40,6 +41,14 @@ type Counter interface {
// Add adds the given value to the counter. It panics if the value is < // Add adds the given value to the counter. It panics if the value is <
// 0. // 0.
Add(float64) Add(float64)
// AddWithExemplar works like Add but also replaces the currently saved
// exemplar (if any) with a new one, created from the provided value,
// the current time as timestamp, and the provided labels. Empty Labels
// will lead to a valid (label-less) exemplar. But if Labels is nil, the
// current exemplar is left in place. This method panics if the value is
// < 0, if any of the provided labels are invalid, or if the provided
// labels contain more than 64 runes in total.
AddWithExemplar(value float64, exemplar Labels)
} }
// CounterOpts is an alias for Opts. See there for doc comments. // CounterOpts is an alias for Opts. See there for doc comments.
@ -61,7 +70,7 @@ func NewCounter(opts CounterOpts) Counter {
nil, nil,
opts.ConstLabels, opts.ConstLabels,
) )
result := &counter{desc: desc, labelPairs: desc.constLabelPairs} result := &counter{desc: desc, labelPairs: desc.constLabelPairs, now: time.Now}
result.init(result) // Init self-collection. result.init(result) // Init self-collection.
return result return result
} }
@ -78,6 +87,9 @@ type counter struct {
desc *Desc desc *Desc
labelPairs []*dto.LabelPair labelPairs []*dto.LabelPair
exemplar atomic.Value // Containing nil or a *dto.Exemplar.
now func() time.Time // To mock out time.Now() for testing.
} }
func (c *counter) Desc() *Desc { func (c *counter) Desc() *Desc {
@ -88,6 +100,7 @@ func (c *counter) Add(v float64) {
if v < 0 { if v < 0 {
panic(errors.New("counter cannot decrease in value")) panic(errors.New("counter cannot decrease in value"))
} }
ival := uint64(v) ival := uint64(v)
if float64(ival) == v { if float64(ival) == v {
atomic.AddUint64(&c.valInt, ival) atomic.AddUint64(&c.valInt, ival)
@ -103,6 +116,11 @@ func (c *counter) Add(v float64) {
} }
} }
func (c *counter) AddWithExemplar(v float64, e Labels) {
c.Add(v)
c.updateExemplar(v, e)
}
func (c *counter) Inc() { func (c *counter) Inc() {
atomic.AddUint64(&c.valInt, 1) atomic.AddUint64(&c.valInt, 1)
} }
@ -112,7 +130,23 @@ func (c *counter) Write(out *dto.Metric) error {
ival := atomic.LoadUint64(&c.valInt) ival := atomic.LoadUint64(&c.valInt)
val := fval + float64(ival) val := fval + float64(ival)
return populateMetric(CounterValue, val, c.labelPairs, out) var exemplar *dto.Exemplar
if e := c.exemplar.Load(); e != nil {
exemplar = e.(*dto.Exemplar)
}
return populateMetric(CounterValue, val, c.labelPairs, exemplar, out)
}
func (c *counter) updateExemplar(v float64, l Labels) {
if l == nil {
return
}
e, err := newExemplar(v, c.now(), l)
if err != nil {
panic(err)
}
c.exemplar.Store(e)
} }
// CounterVec is a Collector that bundles a set of Counters that all share the // CounterVec is a Collector that bundles a set of Counters that all share the

View File

@ -17,6 +17,10 @@ import (
"fmt" "fmt"
"math" "math"
"testing" "testing"
"time"
"github.com/golang/protobuf/proto"
"github.com/golang/protobuf/ptypes"
dto "github.com/prometheus/client_model/go" dto "github.com/prometheus/client_model/go"
) )
@ -210,3 +214,61 @@ func TestCounterAddSmall(t *testing.T) {
t.Errorf("expected %q, got %q", expected, got) t.Errorf("expected %q, got %q", expected, got)
} }
} }
func TestCounterExemplar(t *testing.T) {
now := time.Now()
counter := NewCounter(CounterOpts{
Name: "test",
Help: "test help",
}).(*counter)
counter.now = func() time.Time { return now }
ts, err := ptypes.TimestampProto(now)
if err != nil {
t.Fatal(err)
}
expectedExemplar := &dto.Exemplar{
Label: []*dto.LabelPair{
&dto.LabelPair{Name: proto.String("foo"), Value: proto.String("bar")},
},
Value: proto.Float64(42),
Timestamp: ts,
}
counter.AddWithExemplar(42, Labels{"foo": "bar"})
if expected, got := expectedExemplar.String(), counter.exemplar.Load().(*dto.Exemplar).String(); expected != got {
t.Errorf("expected exemplar %s, got %s.", expected, got)
}
addExemplarWithInvalidLabel := func() (err error) {
defer func() {
if e := recover(); e != nil {
err = e.(error)
}
}()
// Should panic because of invalid label name.
counter.AddWithExemplar(42, Labels{":o)": "smile"})
return nil
}
if addExemplarWithInvalidLabel() == nil {
t.Error("adding exemplar with invalid label succeeded")
}
addExemplarWithOversizedLabels := func() (err error) {
defer func() {
if e := recover(); e != nil {
err = e.(error)
}
}()
// Should panic because of 65 runes.
counter.AddWithExemplar(42, Labels{
"abcdefghijklmnopqrstuvwxyz": "26+16 characters",
"x1234567": "8+15 characters",
})
return nil
}
if addExemplarWithOversizedLabels() == nil {
t.Error("adding exemplar with oversized labels succeeded")
}
}

View File

@ -123,7 +123,7 @@ func (g *gauge) Sub(val float64) {
func (g *gauge) Write(out *dto.Metric) error { func (g *gauge) Write(out *dto.Metric) error {
val := math.Float64frombits(atomic.LoadUint64(&g.valBits)) val := math.Float64frombits(atomic.LoadUint64(&g.valBits))
return populateMetric(GaugeValue, val, g.labelPairs, out) return populateMetric(GaugeValue, val, g.labelPairs, nil, out)
} }
// GaugeVec is a Collector that bundles a set of Gauges that all share the same // GaugeVec is a Collector that bundles a set of Gauges that all share the same

View File

@ -20,6 +20,7 @@ import (
"sort" "sort"
"sync" "sync"
"sync/atomic" "sync/atomic"
"time"
"github.com/golang/protobuf/proto" "github.com/golang/protobuf/proto"
@ -47,6 +48,15 @@ type Histogram interface {
// Observe adds a single observation to the histogram. // Observe adds a single observation to the histogram.
Observe(float64) Observe(float64)
// ObserveWithExemplar works like Observe but also replaces the
// currently saved exemplar for the relevant bucket (possibly none) with
// a new one, created from the provided value, the current time as
// timestamp, and the provided Labels. Empty Labels will lead to a valid
// (label-less) exemplar. But if Labels is nil, the current exemplar in
// the relevant bucket is left in place. This method panics if any of
// the provided labels are invalid or if the provided labels contain
// more than 64 runes in total.
ObserveWithExemplar(value float64, exemplar Labels)
} }
// bucketLabel is used for the label that defines the upper bound of a // bucketLabel is used for the label that defines the upper bound of a
@ -188,6 +198,7 @@ func newHistogram(desc *Desc, opts HistogramOpts, labelValues ...string) Histogr
upperBounds: opts.Buckets, upperBounds: opts.Buckets,
labelPairs: makeLabelPairs(desc, labelValues), labelPairs: makeLabelPairs(desc, labelValues),
counts: [2]*histogramCounts{{}, {}}, counts: [2]*histogramCounts{{}, {}},
now: time.Now,
} }
for i, upperBound := range h.upperBounds { for i, upperBound := range h.upperBounds {
if i < len(h.upperBounds)-1 { if i < len(h.upperBounds)-1 {
@ -205,9 +216,10 @@ func newHistogram(desc *Desc, opts HistogramOpts, labelValues ...string) Histogr
} }
} }
// Finally we know the final length of h.upperBounds and can make buckets // Finally we know the final length of h.upperBounds and can make buckets
// for both counts: // for both counts as well as exemplars:
h.counts[0].buckets = make([]uint64, len(h.upperBounds)) h.counts[0].buckets = make([]uint64, len(h.upperBounds))
h.counts[1].buckets = make([]uint64, len(h.upperBounds)) h.counts[1].buckets = make([]uint64, len(h.upperBounds))
h.exemplars = make([]atomic.Value, len(h.upperBounds)+1)
h.init(h) // Init self-collection. h.init(h) // Init self-collection.
return h return h
@ -254,6 +266,9 @@ type histogram struct {
upperBounds []float64 upperBounds []float64
labelPairs []*dto.LabelPair labelPairs []*dto.LabelPair
exemplars []atomic.Value // One more than buckets (to include +Inf), each a *dto.Exemplar.
now func() time.Time // To mock out time.Now() for testing.
} }
func (h *histogram) Desc() *Desc { func (h *histogram) Desc() *Desc {
@ -261,36 +276,13 @@ func (h *histogram) Desc() *Desc {
} }
func (h *histogram) Observe(v float64) { func (h *histogram) Observe(v float64) {
// TODO(beorn7): For small numbers of buckets (<30), a linear search is h.observe(v, h.findBucket(v))
// slightly faster than the binary search. If we really care, we could }
// switch from one search strategy to the other depending on the number
// of buckets.
//
// Microbenchmarks (BenchmarkHistogramNoLabels):
// 11 buckets: 38.3 ns/op linear - binary 48.7 ns/op
// 100 buckets: 78.1 ns/op linear - binary 54.9 ns/op
// 300 buckets: 154 ns/op linear - binary 61.6 ns/op
i := sort.SearchFloat64s(h.upperBounds, v)
// We increment h.countAndHotIdx so that the counter in the lower func (h *histogram) ObserveWithExemplar(v float64, e Labels) {
// 63 bits gets incremented. At the same time, we get the new value i := h.findBucket(v)
// back, which we can use to find the currently-hot counts. h.observe(v, i)
n := atomic.AddUint64(&h.countAndHotIdx, 1) h.updateExemplar(v, i, e)
hotCounts := h.counts[n>>63]
if i < len(h.upperBounds) {
atomic.AddUint64(&hotCounts.buckets[i], 1)
}
for {
oldBits := atomic.LoadUint64(&hotCounts.sumBits)
newBits := math.Float64bits(math.Float64frombits(oldBits) + v)
if atomic.CompareAndSwapUint64(&hotCounts.sumBits, oldBits, newBits) {
break
}
}
// Increment count last as we take it as a signal that the observation
// is complete.
atomic.AddUint64(&hotCounts.count, 1)
} }
func (h *histogram) Write(out *dto.Metric) error { func (h *histogram) Write(out *dto.Metric) error {
@ -329,6 +321,18 @@ func (h *histogram) Write(out *dto.Metric) error {
CumulativeCount: proto.Uint64(cumCount), CumulativeCount: proto.Uint64(cumCount),
UpperBound: proto.Float64(upperBound), UpperBound: proto.Float64(upperBound),
} }
if e := h.exemplars[i].Load(); e != nil {
his.Bucket[i].Exemplar = e.(*dto.Exemplar)
}
}
// If there is an exemplar for the +Inf bucket, we have to add that bucket explicitly.
if e := h.exemplars[len(h.upperBounds)].Load(); e != nil {
b := &dto.Bucket{
CumulativeCount: proto.Uint64(count),
UpperBound: proto.Float64(math.Inf(1)),
Exemplar: e.(*dto.Exemplar),
}
his.Bucket = append(his.Bucket, b)
} }
out.Histogram = his out.Histogram = his
@ -352,6 +356,57 @@ func (h *histogram) Write(out *dto.Metric) error {
return nil return nil
} }
// findBucket returns the index of the bucket for the provided value, or
// len(h.upperBounds) for the +Inf bucket.
func (h *histogram) findBucket(v float64) int {
// TODO(beorn7): For small numbers of buckets (<30), a linear search is
// slightly faster than the binary search. If we really care, we could
// switch from one search strategy to the other depending on the number
// of buckets.
//
// Microbenchmarks (BenchmarkHistogramNoLabels):
// 11 buckets: 38.3 ns/op linear - binary 48.7 ns/op
// 100 buckets: 78.1 ns/op linear - binary 54.9 ns/op
// 300 buckets: 154 ns/op linear - binary 61.6 ns/op
return sort.SearchFloat64s(h.upperBounds, v)
}
// observe is the implementation for Observe without the findBucket part.
func (h *histogram) observe(v float64, bucket int) {
// We increment h.countAndHotIdx so that the counter in the lower
// 63 bits gets incremented. At the same time, we get the new value
// back, which we can use to find the currently-hot counts.
n := atomic.AddUint64(&h.countAndHotIdx, 1)
hotCounts := h.counts[n>>63]
if bucket < len(h.upperBounds) {
atomic.AddUint64(&hotCounts.buckets[bucket], 1)
}
for {
oldBits := atomic.LoadUint64(&hotCounts.sumBits)
newBits := math.Float64bits(math.Float64frombits(oldBits) + v)
if atomic.CompareAndSwapUint64(&hotCounts.sumBits, oldBits, newBits) {
break
}
}
// Increment count last as we take it as a signal that the observation
// is complete.
atomic.AddUint64(&hotCounts.count, 1)
}
// updateExemplar replaces the exemplar for the provided bucket. With empty
// labels, it's a no-op. It panics if any of the labels is invalid.
func (h *histogram) updateExemplar(v float64, bucket int, l Labels) {
if l == nil {
return
}
e, err := newExemplar(v, h.now(), l)
if err != nil {
panic(err)
}
h.exemplars[bucket].Store(e)
}
// HistogramVec is a Collector that bundles a set of Histograms that all share the // HistogramVec is a Collector that bundles a set of Histograms that all share the
// same Desc, but have different values for their variable labels. This is used // same Desc, but have different values for their variable labels. This is used
// if you want to count the same thing partitioned by various dimensions // if you want to count the same thing partitioned by various dimensions

View File

@ -22,6 +22,10 @@ import (
"sync" "sync"
"testing" "testing"
"testing/quick" "testing/quick"
"time"
"github.com/golang/protobuf/proto"
"github.com/golang/protobuf/ptypes"
dto "github.com/prometheus/client_model/go" dto "github.com/prometheus/client_model/go"
) )
@ -182,7 +186,11 @@ func TestHistogramConcurrency(t *testing.T) {
go func(vals []float64) { go func(vals []float64) {
start.Wait() start.Wait()
for _, v := range vals { for _, v := range vals {
if n%2 == 0 {
sum.Observe(v) sum.Observe(v)
} else {
sum.ObserveWithExemplar(v, Labels{"foo": "bar"})
}
} }
end.Done() end.Done()
}(vals) }(vals)
@ -201,9 +209,13 @@ func TestHistogramConcurrency(t *testing.T) {
} }
wantCounts := getCumulativeCounts(allVars) wantCounts := getCumulativeCounts(allVars)
wantBuckets := len(testBuckets)
if !math.IsInf(m.Histogram.Bucket[len(m.Histogram.Bucket)-1].GetUpperBound(), +1) {
wantBuckets--
}
if got, want := len(m.Histogram.Bucket), len(testBuckets)-1; got != want { if got := len(m.Histogram.Bucket); got != wantBuckets {
t.Errorf("got %d buckets in protobuf, want %d", got, want) t.Errorf("got %d buckets in protobuf, want %d", got, wantBuckets)
} }
for i, wantBound := range testBuckets { for i, wantBound := range testBuckets {
if i == len(testBuckets)-1 { if i == len(testBuckets)-1 {
@ -384,3 +396,62 @@ func TestHistogramAtomicObserve(t *testing.T) {
runtime.Gosched() runtime.Gosched()
} }
} }
func TestHistogramExemplar(t *testing.T) {
now := time.Now()
histogram := NewHistogram(HistogramOpts{
Name: "test",
Help: "test help",
Buckets: []float64{1, 2, 3, 4},
}).(*histogram)
histogram.now = func() time.Time { return now }
ts, err := ptypes.TimestampProto(now)
if err != nil {
t.Fatal(err)
}
expectedExemplars := []*dto.Exemplar{
nil,
&dto.Exemplar{
Label: []*dto.LabelPair{
&dto.LabelPair{Name: proto.String("id"), Value: proto.String("2")},
},
Value: proto.Float64(1.6),
Timestamp: ts,
},
nil,
&dto.Exemplar{
Label: []*dto.LabelPair{
&dto.LabelPair{Name: proto.String("id"), Value: proto.String("3")},
},
Value: proto.Float64(4),
Timestamp: ts,
},
&dto.Exemplar{
Label: []*dto.LabelPair{
&dto.LabelPair{Name: proto.String("id"), Value: proto.String("4")},
},
Value: proto.Float64(4.5),
Timestamp: ts,
},
}
histogram.ObserveWithExemplar(1.5, Labels{"id": "1"})
histogram.ObserveWithExemplar(1.6, Labels{"id": "2"}) // To replace exemplar in bucket 0.
histogram.ObserveWithExemplar(4, Labels{"id": "3"})
histogram.ObserveWithExemplar(4.5, Labels{"id": "4"}) // Should go to +Inf bucket.
for i, ex := range histogram.exemplars {
var got, expected string
if val := ex.Load(); val != nil {
got = val.(*dto.Exemplar).String()
}
if expectedExemplars[i] != nil {
expected = expectedExemplars[i].String()
}
if got != expected {
t.Errorf("expected exemplar %s, got %s.", expected, got)
}
}
}

View File

@ -144,7 +144,12 @@ func HandlerFor(reg prometheus.Gatherer, opts HandlerOpts) http.Handler {
} }
} }
contentType := expfmt.Negotiate(req.Header) var contentType expfmt.Format
if opts.EnableOpenMetrics {
contentType = expfmt.NegotiateIncludingOpenMetrics(req.Header)
} else {
contentType = expfmt.Negotiate(req.Header)
}
header := rsp.Header() header := rsp.Header()
header.Set(contentTypeHeader, string(contentType)) header.Set(contentTypeHeader, string(contentType))
@ -163,8 +168,13 @@ func HandlerFor(reg prometheus.Gatherer, opts HandlerOpts) http.Handler {
enc := expfmt.NewEncoder(w, contentType) enc := expfmt.NewEncoder(w, contentType)
var lastErr error var lastErr error
for _, mf := range mfs {
if err := enc.Encode(mf); err != nil { // handleError handles the error according to opts.ErrorHandling
// and returns true if we have to abort after the handling.
handleError := func(err error) bool {
if err == nil {
return false
}
lastErr = err lastErr = err
if opts.ErrorLog != nil { if opts.ErrorLog != nil {
opts.ErrorLog.Println("error encoding and sending metric family:", err) opts.ErrorLog.Println("error encoding and sending metric family:", err)
@ -173,13 +183,24 @@ func HandlerFor(reg prometheus.Gatherer, opts HandlerOpts) http.Handler {
switch opts.ErrorHandling { switch opts.ErrorHandling {
case PanicOnError: case PanicOnError:
panic(err) panic(err)
case ContinueOnError:
// Handled later.
case HTTPErrorOnError: case HTTPErrorOnError:
httpError(rsp, err) httpError(rsp, err)
return true
}
// Do nothing in all other cases, including ContinueOnError.
return false
}
for _, mf := range mfs {
if handleError(enc.Encode(mf)) {
return return
} }
} }
if closer, ok := enc.(expfmt.Closer); ok {
// This in particular takes care of the final "# EOF\n" line for OpenMetrics.
if handleError(closer.Close()) {
return
}
} }
if lastErr != nil { if lastErr != nil {
@ -318,6 +339,16 @@ type HandlerOpts struct {
// away). Until the implementation is improved, it is recommended to // away). Until the implementation is improved, it is recommended to
// implement a separate timeout in potentially slow Collectors. // implement a separate timeout in potentially slow Collectors.
Timeout time.Duration Timeout time.Duration
// If true, the experimental OpenMetrics encoding is added to the
// possible options during content negotiation. Note that Prometheus
// 2.5.0+ will negotiate OpenMetrics as first priority. OpenMetrics is
// the only way to transmit exemplars. However, the move to OpenMetrics
// is not completely transparent. Most notably, the values of "quantile"
// labels of Summaries and "le" labels of Histograms are formatted with
// a trailing ".0" if they would otherwise look like integer numbers
// (which changes the identity of the resulting series on the Prometheus
// server).
EnableOpenMetrics bool
} }
// gzipAccepted returns whether the client will accept gzip-encoded content. // gzipAccepted returns whether the client will accept gzip-encoded content.

View File

@ -16,8 +16,11 @@ package prometheus
import ( import (
"fmt" "fmt"
"sort" "sort"
"time"
"unicode/utf8"
"github.com/golang/protobuf/proto" "github.com/golang/protobuf/proto"
"github.com/golang/protobuf/ptypes"
dto "github.com/prometheus/client_model/go" dto "github.com/prometheus/client_model/go"
) )
@ -69,7 +72,7 @@ func (v *valueFunc) Desc() *Desc {
} }
func (v *valueFunc) Write(out *dto.Metric) error { func (v *valueFunc) Write(out *dto.Metric) error {
return populateMetric(v.valType, v.function(), v.labelPairs, out) return populateMetric(v.valType, v.function(), v.labelPairs, nil, out)
} }
// NewConstMetric returns a metric with one fixed value that cannot be // NewConstMetric returns a metric with one fixed value that cannot be
@ -116,19 +119,20 @@ func (m *constMetric) Desc() *Desc {
} }
func (m *constMetric) Write(out *dto.Metric) error { func (m *constMetric) Write(out *dto.Metric) error {
return populateMetric(m.valType, m.val, m.labelPairs, out) return populateMetric(m.valType, m.val, m.labelPairs, nil, out)
} }
func populateMetric( func populateMetric(
t ValueType, t ValueType,
v float64, v float64,
labelPairs []*dto.LabelPair, labelPairs []*dto.LabelPair,
e *dto.Exemplar,
m *dto.Metric, m *dto.Metric,
) error { ) error {
m.Label = labelPairs m.Label = labelPairs
switch t { switch t {
case CounterValue: case CounterValue:
m.Counter = &dto.Counter{Value: proto.Float64(v)} m.Counter = &dto.Counter{Value: proto.Float64(v), Exemplar: e}
case GaugeValue: case GaugeValue:
m.Gauge = &dto.Gauge{Value: proto.Float64(v)} m.Gauge = &dto.Gauge{Value: proto.Float64(v)}
case UntypedValue: case UntypedValue:
@ -160,3 +164,40 @@ func makeLabelPairs(desc *Desc, labelValues []string) []*dto.LabelPair {
sort.Sort(labelPairSorter(labelPairs)) sort.Sort(labelPairSorter(labelPairs))
return labelPairs return labelPairs
} }
// ExemplarMaxRunes is the max total number of runes allowed in exemplar labels.
const ExemplarMaxRunes = 64
// newExemplar creates a new dto.Exemplar from the provided values. An error is
// returned if any of the label names or values are invalid or if the total
// number of runes in the label names and values exceeds ExemplarMaxRunes.
func newExemplar(value float64, ts time.Time, l Labels) (*dto.Exemplar, error) {
e := &dto.Exemplar{}
e.Value = proto.Float64(value)
tsProto, err := ptypes.TimestampProto(ts)
if err != nil {
return nil, err
}
e.Timestamp = tsProto
labelPairs := make([]*dto.LabelPair, 0, len(l))
var runes int
for name, value := range l {
if !checkLabelName(name) {
return nil, fmt.Errorf("exemplar label name %q is invalid", name)
}
runes += utf8.RuneCountInString(name)
if !utf8.ValidString(value) {
return nil, fmt.Errorf("exemplar label value %q is not valid UTF-8", value)
}
runes += utf8.RuneCountInString(value)
labelPairs = append(labelPairs, &dto.LabelPair{
Name: proto.String(name),
Value: proto.String(value),
})
}
if runes > ExemplarMaxRunes {
return nil, fmt.Errorf("exemplar labels have %d runes, exceeding the limit of %d", runes, ExemplarMaxRunes)
}
e.Label = labelPairs
return e, nil
}