2013-04-03 20:33:32 +04:00
|
|
|
// Copyright (c) 2013, Prometheus Team
|
2013-02-12 05:36:06 +04:00
|
|
|
// All rights reserved.
|
|
|
|
//
|
|
|
|
// Use of this source code is governed by a BSD-style
|
|
|
|
// license that can be found in the LICENSE file.
|
2012-05-20 01:59:25 +04:00
|
|
|
|
2013-04-03 20:33:32 +04:00
|
|
|
package prometheus
|
2012-05-20 01:59:25 +04:00
|
|
|
|
|
|
|
import (
|
|
|
|
"bytes"
|
2013-04-19 16:11:01 +04:00
|
|
|
"encoding/json"
|
2012-05-20 01:59:25 +04:00
|
|
|
"fmt"
|
|
|
|
"math"
|
|
|
|
"strconv"
|
2013-01-19 17:48:30 +04:00
|
|
|
"sync"
|
2013-07-21 19:34:37 +04:00
|
|
|
"time"
|
2013-06-27 20:46:16 +04:00
|
|
|
|
|
|
|
dto "github.com/prometheus/client_model/go"
|
|
|
|
|
|
|
|
"code.google.com/p/goprotobuf/proto"
|
2012-05-20 01:59:25 +04:00
|
|
|
)
|
|
|
|
|
2013-02-12 05:36:06 +04:00
|
|
|
// This generates count-buckets of equal size distributed along the open
|
|
|
|
// interval of lower to upper. For instance, {lower=0, upper=10, count=5}
|
|
|
|
// yields the following: [0, 2, 4, 6, 8].
|
2012-05-20 01:59:25 +04:00
|
|
|
func EquallySizedBucketsFor(lower, upper float64, count int) []float64 {
|
|
|
|
buckets := make([]float64, count)
|
|
|
|
|
|
|
|
partitionSize := (upper - lower) / float64(count)
|
|
|
|
|
|
|
|
for i := 0; i < count; i++ {
|
|
|
|
m := float64(i)
|
|
|
|
buckets[i] = lower + (m * partitionSize)
|
|
|
|
}
|
|
|
|
|
|
|
|
return buckets
|
|
|
|
}
|
|
|
|
|
2013-02-12 05:36:06 +04:00
|
|
|
// This generates log2-sized buckets spanning from lower to upper inclusively
|
|
|
|
// as well as values beyond it.
|
2012-05-20 01:59:25 +04:00
|
|
|
func LogarithmicSizedBucketsFor(lower, upper float64) []float64 {
|
|
|
|
bucketCount := int(math.Ceil(math.Log2(upper)))
|
|
|
|
|
|
|
|
buckets := make([]float64, bucketCount)
|
|
|
|
|
|
|
|
for i, j := 0, 0.0; i < bucketCount; i, j = i+1, math.Pow(2, float64(i+1.0)) {
|
|
|
|
buckets[i] = j
|
|
|
|
}
|
|
|
|
|
|
|
|
return buckets
|
|
|
|
}
|
|
|
|
|
2013-02-12 05:36:06 +04:00
|
|
|
// A HistogramSpecification defines how a Histogram is to be built.
|
2012-05-20 01:59:25 +04:00
|
|
|
type HistogramSpecification struct {
|
2013-01-19 17:48:30 +04:00
|
|
|
BucketBuilder BucketBuilder
|
2012-05-20 01:59:25 +04:00
|
|
|
ReportablePercentiles []float64
|
2013-01-19 17:48:30 +04:00
|
|
|
Starts []float64
|
2013-07-21 19:34:37 +04:00
|
|
|
PurgeInterval time.Duration
|
2013-01-19 17:48:30 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
type Histogram interface {
|
2013-04-19 15:22:59 +04:00
|
|
|
Metric
|
2013-01-19 17:48:30 +04:00
|
|
|
Add(labels map[string]string, value float64)
|
2012-05-20 01:59:25 +04:00
|
|
|
}
|
|
|
|
|
2013-02-12 05:36:06 +04:00
|
|
|
// The histogram is an accumulator for samples. It merely routes into which
|
|
|
|
// bucket to capture an event and provides a percentile calculation mechanism.
|
2013-01-19 17:48:30 +04:00
|
|
|
type histogram struct {
|
|
|
|
bucketMaker BucketBuilder
|
2013-02-12 05:36:06 +04:00
|
|
|
// This represents the open interval's start at which values shall be added to
|
|
|
|
// the bucket. The interval continues until the beginning of the next bucket
|
|
|
|
// exclusive or positive infinity.
|
|
|
|
//
|
|
|
|
// N.B.
|
|
|
|
// - bucketStarts should be sorted in ascending order;
|
|
|
|
// - len(bucketStarts) must be equivalent to len(buckets);
|
|
|
|
// - The index of a given bucketStarts' element is presumed to
|
|
|
|
// correspond to the appropriate element in buckets.
|
2012-05-20 01:59:25 +04:00
|
|
|
bucketStarts []float64
|
2013-01-19 17:48:30 +04:00
|
|
|
mutex sync.RWMutex
|
2013-02-12 05:36:06 +04:00
|
|
|
// These are the buckets that capture samples as they are emitted to the
|
|
|
|
// histogram. Please consult the reference interface and its implements for
|
|
|
|
// further details about behavior expectations.
|
2013-06-27 20:46:16 +04:00
|
|
|
values map[uint64]*histogramVector
|
2013-02-12 05:36:06 +04:00
|
|
|
// These are the percentile values that will be reported on marshalling.
|
2012-05-20 01:59:25 +04:00
|
|
|
reportablePercentiles []float64
|
2013-07-21 19:34:37 +04:00
|
|
|
|
|
|
|
purgeInterval time.Duration
|
|
|
|
lastPurge time.Time
|
2012-05-20 01:59:25 +04:00
|
|
|
}
|
|
|
|
|
2013-02-12 05:36:06 +04:00
|
|
|
type histogramVector struct {
|
2013-01-19 17:48:30 +04:00
|
|
|
buckets []Bucket
|
|
|
|
labels map[string]string
|
2013-07-21 19:14:29 +04:00
|
|
|
sum float64
|
|
|
|
count uint64
|
2013-01-19 17:48:30 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
func (h *histogram) Add(labels map[string]string, value float64) {
|
|
|
|
if labels == nil {
|
2014-04-14 20:45:16 +04:00
|
|
|
labels = blankLabelsSingleton
|
2013-01-19 17:48:30 +04:00
|
|
|
}
|
|
|
|
|
2014-04-14 20:45:16 +04:00
|
|
|
signature := labelValuesToSignature(labels)
|
2013-02-12 05:36:06 +04:00
|
|
|
var histogram *histogramVector = nil
|
2014-04-14 20:45:16 +04:00
|
|
|
|
|
|
|
h.mutex.Lock()
|
|
|
|
defer h.mutex.Unlock()
|
|
|
|
|
2013-01-19 17:48:30 +04:00
|
|
|
if original, ok := h.values[signature]; ok {
|
|
|
|
histogram = original
|
|
|
|
} else {
|
|
|
|
bucketCount := len(h.bucketStarts)
|
2013-02-12 05:36:06 +04:00
|
|
|
histogram = &histogramVector{
|
2013-01-19 17:48:30 +04:00
|
|
|
buckets: make([]Bucket, bucketCount),
|
|
|
|
labels: labels,
|
|
|
|
}
|
|
|
|
for i := 0; i < bucketCount; i++ {
|
|
|
|
histogram.buckets[i] = h.bucketMaker()
|
|
|
|
}
|
|
|
|
h.values[signature] = histogram
|
|
|
|
}
|
|
|
|
|
2012-05-20 01:59:25 +04:00
|
|
|
lastIndex := 0
|
|
|
|
|
|
|
|
for i, bucketStart := range h.bucketStarts {
|
|
|
|
if value < bucketStart {
|
|
|
|
break
|
|
|
|
}
|
|
|
|
|
|
|
|
lastIndex = i
|
|
|
|
}
|
|
|
|
|
2013-01-19 17:48:30 +04:00
|
|
|
histogram.buckets[lastIndex].Add(value)
|
2013-07-21 19:14:29 +04:00
|
|
|
|
|
|
|
histogram.sum += value
|
|
|
|
histogram.count++
|
2012-05-20 01:59:25 +04:00
|
|
|
}
|
|
|
|
|
2013-05-03 18:02:03 +04:00
|
|
|
func (h *histogram) String() string {
|
2013-01-19 17:48:30 +04:00
|
|
|
h.mutex.RLock()
|
|
|
|
defer h.mutex.RUnlock()
|
|
|
|
|
|
|
|
stringBuffer := &bytes.Buffer{}
|
2012-05-20 01:59:25 +04:00
|
|
|
stringBuffer.WriteString("[Histogram { ")
|
|
|
|
|
2013-01-19 17:48:30 +04:00
|
|
|
for _, histogram := range h.values {
|
|
|
|
fmt.Fprintf(stringBuffer, "Labels: %s ", histogram.labels)
|
|
|
|
for i, bucketStart := range h.bucketStarts {
|
|
|
|
bucket := histogram.buckets[i]
|
|
|
|
fmt.Fprintf(stringBuffer, "[%f, inf) = %s, ", bucketStart, bucket)
|
|
|
|
}
|
2012-05-20 01:59:25 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
stringBuffer.WriteString("}]")
|
|
|
|
|
2013-01-19 17:48:30 +04:00
|
|
|
return stringBuffer.String()
|
2012-05-20 01:59:25 +04:00
|
|
|
}
|
|
|
|
|
2013-02-12 05:36:06 +04:00
|
|
|
// Determine the number of previous observations up to a given index.
|
2012-05-20 01:59:25 +04:00
|
|
|
func previousCumulativeObservations(cumulativeObservations []int, bucketIndex int) int {
|
|
|
|
if bucketIndex == 0 {
|
|
|
|
return 0
|
|
|
|
}
|
|
|
|
|
|
|
|
return cumulativeObservations[bucketIndex-1]
|
|
|
|
}
|
|
|
|
|
2013-02-12 05:36:06 +04:00
|
|
|
// Determine the index for an element given a percentage of length.
|
2012-05-20 01:59:25 +04:00
|
|
|
func prospectiveIndexForPercentile(percentile float64, totalObservations int) int {
|
2012-05-22 11:20:09 +04:00
|
|
|
return int(percentile * float64(totalObservations-1))
|
|
|
|
}
|
|
|
|
|
2013-02-12 05:36:06 +04:00
|
|
|
// Determine the next bucket element when interim bucket intervals may be empty.
|
2013-06-27 20:46:16 +04:00
|
|
|
func (h histogram) nextNonEmptyBucketElement(signature uint64, currentIndex, bucketCount int, observationsByBucket []int) (*Bucket, int) {
|
2012-05-22 11:20:09 +04:00
|
|
|
for i := currentIndex; i < bucketCount; i++ {
|
|
|
|
if observationsByBucket[i] == 0 {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
2013-01-19 17:48:30 +04:00
|
|
|
histogram := h.values[signature]
|
|
|
|
|
|
|
|
return &histogram.buckets[i], 0
|
2012-05-22 11:20:09 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
panic("Illegal Condition: There were no remaining buckets to provide a value.")
|
2012-05-20 01:59:25 +04:00
|
|
|
}
|
|
|
|
|
2013-02-12 05:36:06 +04:00
|
|
|
// Find what bucket and element index contains a given percentile value.
|
|
|
|
// If a percentile is requested that results in a corresponding index that is no
|
|
|
|
// longer contained by the bucket, the index of the last item is returned. This
|
|
|
|
// may occur if the underlying bucket catalogs values and employs an eviction
|
|
|
|
// strategy.
|
2013-06-27 20:46:16 +04:00
|
|
|
func (h histogram) bucketForPercentile(signature uint64, percentile float64) (*Bucket, int) {
|
2013-01-19 17:48:30 +04:00
|
|
|
bucketCount := len(h.bucketStarts)
|
2012-05-20 01:59:25 +04:00
|
|
|
|
2013-02-12 05:36:06 +04:00
|
|
|
// This captures the quantity of samples in a given bucket's range.
|
2012-05-20 01:59:25 +04:00
|
|
|
observationsByBucket := make([]int, bucketCount)
|
2013-02-12 05:36:06 +04:00
|
|
|
// This captures the cumulative quantity of observations from all preceding
|
|
|
|
// buckets up and to the end of this bucket.
|
2012-05-20 01:59:25 +04:00
|
|
|
cumulativeObservationsByBucket := make([]int, bucketCount)
|
|
|
|
|
2013-01-19 17:48:30 +04:00
|
|
|
totalObservations := 0
|
|
|
|
|
|
|
|
histogram := h.values[signature]
|
2012-05-20 01:59:25 +04:00
|
|
|
|
2013-01-19 17:48:30 +04:00
|
|
|
for i, bucket := range histogram.buckets {
|
2012-05-20 01:59:25 +04:00
|
|
|
observations := bucket.Observations()
|
|
|
|
observationsByBucket[i] = observations
|
|
|
|
totalObservations += bucket.Observations()
|
|
|
|
cumulativeObservationsByBucket[i] = totalObservations
|
|
|
|
}
|
|
|
|
|
2013-02-12 05:36:06 +04:00
|
|
|
// This captures the index offset where the given percentile value would be
|
|
|
|
// were all submitted samples stored and never down-/re-sampled nor deleted
|
|
|
|
// and housed in a singular array.
|
2012-05-20 01:59:25 +04:00
|
|
|
prospectiveIndex := prospectiveIndexForPercentile(percentile, totalObservations)
|
|
|
|
|
|
|
|
for i, cumulativeObservation := range cumulativeObservationsByBucket {
|
|
|
|
if cumulativeObservation == 0 {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
2013-02-12 05:36:06 +04:00
|
|
|
// Find the bucket that contains the given index.
|
2012-05-20 01:59:25 +04:00
|
|
|
if cumulativeObservation >= prospectiveIndex {
|
|
|
|
var subIndex int
|
2013-02-12 05:36:06 +04:00
|
|
|
// This calculates the index within the current bucket where the given
|
|
|
|
// percentile may be found.
|
2012-05-20 01:59:25 +04:00
|
|
|
subIndex = prospectiveIndex - previousCumulativeObservations(cumulativeObservationsByBucket, i)
|
2012-05-22 11:20:09 +04:00
|
|
|
|
2013-02-12 05:36:06 +04:00
|
|
|
// Sometimes the index may be the last item, in which case we need to
|
|
|
|
// take this into account.
|
2012-05-20 01:59:25 +04:00
|
|
|
if observationsByBucket[i] == subIndex {
|
2013-01-19 17:48:30 +04:00
|
|
|
return h.nextNonEmptyBucketElement(signature, i+1, bucketCount, observationsByBucket)
|
2012-05-20 01:59:25 +04:00
|
|
|
}
|
|
|
|
|
2013-01-19 17:48:30 +04:00
|
|
|
return &histogram.buckets[i], subIndex
|
2012-05-20 01:59:25 +04:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2013-01-19 17:48:30 +04:00
|
|
|
return &histogram.buckets[0], 0
|
2012-05-20 01:59:25 +04:00
|
|
|
}
|
|
|
|
|
2013-02-12 05:36:06 +04:00
|
|
|
// Return the histogram's estimate of the value for a given percentile of
|
|
|
|
// collected samples. The requested percentile is expected to be a real
|
|
|
|
// value within (0, 1.0].
|
2013-06-27 20:46:16 +04:00
|
|
|
func (h histogram) percentile(signature uint64, percentile float64) float64 {
|
2013-01-19 17:48:30 +04:00
|
|
|
bucket, index := h.bucketForPercentile(signature, percentile)
|
2012-05-20 01:59:25 +04:00
|
|
|
|
|
|
|
return (*bucket).ValueForIndex(index)
|
|
|
|
}
|
|
|
|
|
2012-05-24 22:02:44 +04:00
|
|
|
func formatFloat(value float64) string {
|
|
|
|
return strconv.FormatFloat(value, floatFormat, floatPrecision, floatBitCount)
|
|
|
|
}
|
|
|
|
|
2013-05-03 18:02:03 +04:00
|
|
|
func (h *histogram) MarshalJSON() ([]byte, error) {
|
2013-07-21 19:34:37 +04:00
|
|
|
h.Purge()
|
|
|
|
|
2013-01-19 17:48:30 +04:00
|
|
|
h.mutex.RLock()
|
|
|
|
defer h.mutex.RUnlock()
|
2012-05-20 01:59:25 +04:00
|
|
|
|
2013-01-19 17:48:30 +04:00
|
|
|
values := make([]map[string]interface{}, 0, len(h.values))
|
|
|
|
|
|
|
|
for signature, value := range h.values {
|
2013-04-19 16:11:01 +04:00
|
|
|
percentiles := make(map[string]float64, len(h.reportablePercentiles))
|
|
|
|
|
2013-01-19 17:48:30 +04:00
|
|
|
for _, percentile := range h.reportablePercentiles {
|
|
|
|
formatted := formatFloat(percentile)
|
2013-04-19 16:11:01 +04:00
|
|
|
percentiles[formatted] = h.percentile(signature, percentile)
|
2013-01-19 17:48:30 +04:00
|
|
|
}
|
2012-05-20 01:59:25 +04:00
|
|
|
|
2013-04-19 16:11:01 +04:00
|
|
|
values = append(values, map[string]interface{}{
|
|
|
|
labelsKey: value.labels,
|
|
|
|
valueKey: percentiles,
|
|
|
|
})
|
|
|
|
}
|
2012-05-20 01:59:25 +04:00
|
|
|
|
2013-04-19 16:11:01 +04:00
|
|
|
return json.Marshal(map[string]interface{}{
|
|
|
|
typeKey: histogramTypeValue,
|
|
|
|
valueKey: values,
|
|
|
|
})
|
2012-05-20 01:59:25 +04:00
|
|
|
}
|
|
|
|
|
2013-07-21 19:34:37 +04:00
|
|
|
func (h *histogram) Purge() {
|
|
|
|
if h.purgeInterval == 0 {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
h.mutex.Lock()
|
|
|
|
defer h.mutex.Unlock()
|
|
|
|
if time.Since(h.lastPurge) < h.purgeInterval {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
h.resetAll()
|
|
|
|
h.lastPurge = time.Now()
|
|
|
|
}
|
|
|
|
|
2014-02-19 17:53:34 +04:00
|
|
|
func (h *histogram) Reset(labels map[string]string) {
|
2014-04-14 20:45:16 +04:00
|
|
|
signature := labelValuesToSignature(labels)
|
|
|
|
|
2014-04-15 03:49:28 +04:00
|
|
|
h.mutex.Lock()
|
|
|
|
defer h.mutex.Unlock()
|
2014-02-19 17:53:34 +04:00
|
|
|
|
|
|
|
value, ok := h.values[signature]
|
|
|
|
|
|
|
|
if !ok {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
for _, bucket := range value.buckets {
|
|
|
|
bucket.Reset()
|
|
|
|
}
|
2014-04-15 03:49:28 +04:00
|
|
|
|
2014-02-19 17:53:34 +04:00
|
|
|
delete(h.values, signature)
|
|
|
|
}
|
|
|
|
|
2013-01-19 17:48:30 +04:00
|
|
|
func (h *histogram) ResetAll() {
|
|
|
|
h.mutex.Lock()
|
|
|
|
defer h.mutex.Unlock()
|
|
|
|
|
2013-07-21 19:34:37 +04:00
|
|
|
h.resetAll()
|
|
|
|
}
|
|
|
|
|
|
|
|
func (h *histogram) resetAll() {
|
2013-01-19 17:48:30 +04:00
|
|
|
for signature, value := range h.values {
|
|
|
|
for _, bucket := range value.buckets {
|
|
|
|
bucket.Reset()
|
|
|
|
}
|
|
|
|
|
|
|
|
delete(h.values, signature)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2013-02-12 05:36:06 +04:00
|
|
|
// Produce a histogram from a given specification.
|
2013-01-19 17:48:30 +04:00
|
|
|
func NewHistogram(specification *HistogramSpecification) Histogram {
|
|
|
|
metric := &histogram{
|
|
|
|
bucketMaker: specification.BucketBuilder,
|
2012-05-20 01:59:25 +04:00
|
|
|
bucketStarts: specification.Starts,
|
|
|
|
reportablePercentiles: specification.ReportablePercentiles,
|
2013-06-27 20:46:16 +04:00
|
|
|
values: map[uint64]*histogramVector{},
|
2013-07-21 19:34:37 +04:00
|
|
|
lastPurge: time.Now(),
|
|
|
|
purgeInterval: specification.PurgeInterval,
|
2012-05-20 01:59:25 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
return metric
|
|
|
|
}
|
2013-02-12 05:36:06 +04:00
|
|
|
|
|
|
|
// Furnish a Histogram with unsensible default values and behaviors that is
|
|
|
|
// strictly useful for prototyping purposes.
|
|
|
|
func NewDefaultHistogram() Histogram {
|
|
|
|
return NewHistogram(
|
|
|
|
&HistogramSpecification{
|
|
|
|
Starts: LogarithmicSizedBucketsFor(0, 4096),
|
2013-04-03 20:33:32 +04:00
|
|
|
BucketBuilder: AccumulatingBucketBuilder(EvictAndReplaceWith(10, AverageReducer), 50),
|
2013-02-12 05:36:06 +04:00
|
|
|
ReportablePercentiles: []float64{0.01, 0.05, 0.5, 0.90, 0.99},
|
2013-07-21 19:34:37 +04:00
|
|
|
PurgeInterval: 15 * time.Minute,
|
2013-02-12 05:36:06 +04:00
|
|
|
},
|
|
|
|
)
|
|
|
|
}
|
2013-06-27 20:46:16 +04:00
|
|
|
|
|
|
|
func (metric *histogram) dumpChildren(f *dto.MetricFamily) {
|
2013-07-21 19:34:37 +04:00
|
|
|
metric.Purge()
|
|
|
|
|
2013-06-27 20:46:16 +04:00
|
|
|
metric.mutex.RLock()
|
|
|
|
defer metric.mutex.RUnlock()
|
|
|
|
|
|
|
|
f.Type = dto.MetricType_SUMMARY.Enum()
|
|
|
|
|
|
|
|
for signature, child := range metric.values {
|
2013-07-21 19:14:29 +04:00
|
|
|
c := &dto.Summary{
|
|
|
|
SampleSum: proto.Float64(child.sum),
|
|
|
|
SampleCount: proto.Uint64(child.count),
|
|
|
|
}
|
2013-06-27 20:46:16 +04:00
|
|
|
|
|
|
|
m := &dto.Metric{
|
|
|
|
Summary: c,
|
|
|
|
}
|
|
|
|
|
|
|
|
for name, value := range child.labels {
|
|
|
|
p := &dto.LabelPair{
|
|
|
|
Name: proto.String(name),
|
|
|
|
Value: proto.String(value),
|
|
|
|
}
|
|
|
|
|
|
|
|
m.Label = append(m.Label, p)
|
|
|
|
}
|
|
|
|
|
|
|
|
for _, percentile := range metric.reportablePercentiles {
|
|
|
|
q := &dto.Quantile{
|
|
|
|
Quantile: proto.Float64(percentile),
|
|
|
|
Value: proto.Float64(metric.percentile(signature, percentile)),
|
|
|
|
}
|
|
|
|
|
|
|
|
c.Quantile = append(c.Quantile, q)
|
|
|
|
}
|
|
|
|
|
|
|
|
f.Metric = append(f.Metric, m)
|
|
|
|
}
|
|
|
|
}
|