diff --git a/prometheus/histogram.go b/prometheus/histogram.go index b5c8bcb..6ee6bdb 100644 --- a/prometheus/histogram.go +++ b/prometheus/histogram.go @@ -472,6 +472,8 @@ type HistogramOpts struct { NativeHistogramMaxBucketNumber uint32 NativeHistogramMinResetDuration time.Duration NativeHistogramMaxZeroThreshold float64 + NativeHistogramMaxExemplarCount uint32 + NativeHistogramExemplarTTL time.Duration // now is for testing purposes, by default it's time.Now. now func() time.Time @@ -556,6 +558,7 @@ func newHistogram(desc *Desc, opts HistogramOpts, labelValues ...string) Histogr h.nativeHistogramZeroThreshold = DefNativeHistogramZeroThreshold } // Leave h.nativeHistogramZeroThreshold at 0 otherwise. h.nativeHistogramSchema = pickSchema(opts.NativeHistogramBucketFactor) + h.nativeExemplars = newNativeExemplars(opts.NativeHistogramExemplarTTL, opts.NativeHistogramMaxExemplarCount) } for i, upperBound := range h.upperBounds { if i < len(h.upperBounds)-1 { @@ -732,6 +735,8 @@ type histogram struct { // afterFunc is for testing purposes, by default it's time.AfterFunc. afterFunc func(time.Duration, func()) *time.Timer + + nativeExemplars nativeExemplars } func (h *histogram) Desc() *Desc { @@ -821,6 +826,8 @@ func (h *histogram) Write(out *dto.Metric) error { Length: proto.Uint32(0), }} } + + his.Exemplars = append(his.Exemplars, h.nativeExemplars.exemplars...) } addAndResetCounts(hotCounts, coldCounts) return nil @@ -1102,6 +1109,10 @@ func (h *histogram) updateExemplar(v float64, bucket int, l Labels) { panic(err) } h.exemplars[bucket].Store(e) + doSparse := h.nativeHistogramSchema > math.MinInt32 && !math.IsNaN(v) + if doSparse { + h.nativeExemplars.addExemplar(e) + } } // HistogramVec is a Collector that bundles a set of Histograms that all share the @@ -1575,3 +1586,58 @@ func addAndResetCounts(hot, cold *histogramCounts) { atomic.AddUint64(&hot.nativeHistogramZeroBucket, atomic.LoadUint64(&cold.nativeHistogramZeroBucket)) atomic.StoreUint64(&cold.nativeHistogramZeroBucket, 0) } + +type nativeExemplars struct { + nativeHistogramExemplarTTL time.Duration + nativeHistogramMaxExemplarCount uint32 + + exemplars []*dto.Exemplar + + lock sync.Mutex +} + +func newNativeExemplars(ttl time.Duration, count uint32) nativeExemplars { + return nativeExemplars{ + nativeHistogramExemplarTTL: ttl, + nativeHistogramMaxExemplarCount: count, + exemplars: make([]*dto.Exemplar, 0), + lock: sync.Mutex{}, + } +} + +func (n *nativeExemplars) addExemplar(e *dto.Exemplar) { + n.lock.Lock() + defer n.lock.Unlock() + + elogarithm := math.Log(e.GetValue()) + if len(n.exemplars) == int(n.nativeHistogramMaxExemplarCount) { + // check if oldestIndex is beyond TTL, + // if so, find the oldest exemplar, and nearest exemplar + oldestTimestamp := time.Now() + oldestIndex := -1 + nearestValue := -1.0 + nearestIndex := -1 + + for i, exemplar := range n.exemplars { + if exemplar.Timestamp.AsTime().Before(oldestTimestamp) { + oldestTimestamp = exemplar.Timestamp.AsTime() + oldestIndex = i + } + logarithm := math.Log(exemplar.GetValue()) + if nearestValue == -1 || math.Abs(elogarithm-logarithm) < nearestValue { + fmt.Printf("gap: %f", math.Abs(elogarithm-logarithm)) + nearestValue = math.Abs(elogarithm - logarithm) + nearestIndex = i + } + } + + if oldestIndex != -1 && time.Since(oldestTimestamp) > n.nativeHistogramExemplarTTL { + n.exemplars[oldestIndex] = e + } else { + n.exemplars[nearestIndex] = e + } + return + } + + n.exemplars = append(n.exemplars, e) +} diff --git a/prometheus/histogram_test.go b/prometheus/histogram_test.go index 39bb0dc..5578094 100644 --- a/prometheus/histogram_test.go +++ b/prometheus/histogram_test.go @@ -1271,3 +1271,91 @@ func TestHistogramVecCreatedTimestampWithDeletes(t *testing.T) { now = now.Add(1 * time.Hour) expectCTsForMetricVecValues(t, histogramVec.MetricVec, dto.MetricType_HISTOGRAM, expected) } + +func TestNativeHistogramExemplar(t *testing.T) { + histogram := NewHistogram(HistogramOpts{ + Name: "test", + Help: "test help", + Buckets: []float64{1, 2, 3, 4}, + NativeHistogramBucketFactor: 1.1, + NativeHistogramMaxExemplarCount: 3, + NativeHistogramExemplarTTL: 10 * time.Second, + }).(*histogram) + + // expectedExemplars := []*dto.Exemplar{ + // { + // Label: []*dto.LabelPair{ + // {Name: proto.String("id"), Value: proto.String("1")}, + // }, + // Value: proto.Float64(1), + // }, + // { + // Label: []*dto.LabelPair{ + // {Name: proto.String("id"), Value: proto.String("2")}, + // }, + // Value: proto.Float64(3), + // }, + // { + // Label: []*dto.LabelPair{ + // {Name: proto.String("id"), Value: proto.String("3")}, + // }, + // Value: proto.Float64(5), + // }, + // } + + histogram.ObserveWithExemplar(1, Labels{"id": "1"}) + histogram.ObserveWithExemplar(3, Labels{"id": "1"}) + histogram.ObserveWithExemplar(5, Labels{"id": "1"}) + + if len(histogram.nativeExemplars.exemplars) != 3 { + t.Errorf("the count of exemplars is not 3") + } + + expectedValues := map[float64]struct{}{ + 1: {}, + 3: {}, + 5: {}, + } + + for _, e := range histogram.nativeExemplars.exemplars { + if _, ok := expectedValues[e.GetValue()]; !ok { + t.Errorf("the value is not in expected value") + } + } + + histogram.ObserveWithExemplar(4, Labels{"id": "1"}) + + if len(histogram.nativeExemplars.exemplars) != 3 { + t.Errorf("the count of exemplars is not 3") + } + + expectedValues = map[float64]struct{}{ + 1: {}, + 3: {}, + 4: {}, + } + + for _, e := range histogram.nativeExemplars.exemplars { + if _, ok := expectedValues[e.GetValue()]; !ok { + t.Errorf("the value is not in expected value") + } + } + + time.Sleep(10 * time.Second) + histogram.ObserveWithExemplar(6, Labels{"id": "1"}) + + if len(histogram.nativeExemplars.exemplars) != 3 { + t.Errorf("the count of exemplars is not 3") + } + + expectedValues = map[float64]struct{}{ + 6: {}, + 3: {}, + 4: {}, + } + for _, e := range histogram.nativeExemplars.exemplars { + if _, ok := expectedValues[e.GetValue()]; !ok { + t.Errorf("the value is not in expected value") + } + } +}