diff --git a/go.mod b/go.mod index 24d9fa6..be69e3a 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/davecgh/go-spew v1.1.1 github.com/golang/protobuf v1.5.2 github.com/json-iterator/go v1.1.12 - github.com/prometheus/client_model v0.2.1-0.20210624201024-61b6c1aac064 + github.com/prometheus/client_model v0.2.1-0.20220719122737-1f8dcad1221e github.com/prometheus/common v0.35.0 github.com/prometheus/procfs v0.7.3 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a diff --git a/go.sum b/go.sum index dc6ec52..9e02d7f 100644 --- a/go.sum +++ b/go.sum @@ -135,8 +135,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.1-0.20210624201024-61b6c1aac064 h1:Kyx21CLOfWDA4e2TcOcupRl2g/Bmddu0AL0hR1BldEw= -github.com/prometheus/client_model v0.2.1-0.20210624201024-61b6c1aac064/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= +github.com/prometheus/client_model v0.2.1-0.20220719122737-1f8dcad1221e h1:KjoQdMEQmNC8smQ731iHAXnbFbApg4uu60fNcWHs3Bk= +github.com/prometheus/client_model v0.2.1-0.20220719122737-1f8dcad1221e/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= github.com/prometheus/common v0.35.0 h1:Eyr+Pw2VymWejHqCugNaQXkAi6KayVNxaHeu6khmFBE= github.com/prometheus/common v0.35.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= diff --git a/prometheus/histogram.go b/prometheus/histogram.go index fea0142..eb80fad 100644 --- a/prometheus/histogram.go +++ b/prometheus/histogram.go @@ -382,19 +382,20 @@ type HistogramOpts struct { Buckets []float64 // If SparseBucketsFactor is greater than one, sparse buckets are used - // (in addition to the regular buckets, if defined above). Sparse - // buckets are exponential buckets covering the whole float64 range - // (with the exception of the “zero” bucket, see - // SparseBucketsZeroThreshold below). From any one bucket to the next, - // the width of the bucket grows by a constant factor. - // SparseBucketsFactor provides an upper bound for this factor - // (exception see below). The smaller SparseBucketsFactor, the more - // buckets will be used and thus the more costly the histogram will - // become. A generally good trade-off between cost and accuracy is a - // value of 1.1 (each bucket is at most 10% wider than the previous - // one), which will result in each power of two divided into 8 buckets - // (e.g. there will be 8 buckets between 1 and 2, same as between 2 and - // 4, and 4 and 8, etc.). + // (in addition to the regular buckets, if defined above). A histogram + // with sparse buckets will be ingested as a native histogram by a + // Prometheus server with that feature enable. Sparse buckets are + // exponential buckets covering the whole float64 range (with the + // exception of the “zero” bucket, see SparseBucketsZeroThreshold + // below). From any one bucket to the next, the width of the bucket + // grows by a constant factor. SparseBucketsFactor provides an upper + // bound for this factor (exception see below). The smaller + // SparseBucketsFactor, the more buckets will be used and thus the more + // costly the histogram will become. A generally good trade-off between + // cost and accuracy is a value of 1.1 (each bucket is at most 10% wider + // than the previous one), which will result in each power of two + // divided into 8 buckets (e.g. there will be 8 buckets between 1 and 2, + // same as between 2 and 4, and 4 and 8, etc.). // // Details about the actually used factor: The factor is calculated as // 2^(2^n), where n is an integer number between (and including) -8 and @@ -723,8 +724,8 @@ func (h *histogram) Write(out *dto.Metric) error { his.Bucket = append(his.Bucket, b) } if h.sparseSchema > math.MinInt32 { - his.SbZeroThreshold = proto.Float64(math.Float64frombits(atomic.LoadUint64(&coldCounts.sparseZeroThresholdBits))) - his.SbSchema = proto.Int32(atomic.LoadInt32(&coldCounts.sparseSchema)) + his.ZeroThreshold = proto.Float64(math.Float64frombits(atomic.LoadUint64(&coldCounts.sparseZeroThresholdBits))) + his.Schema = proto.Int32(atomic.LoadInt32(&coldCounts.sparseSchema)) zeroBucket := atomic.LoadUint64(&coldCounts.sparseZeroBucket) defer func() { @@ -732,9 +733,9 @@ func (h *histogram) Write(out *dto.Metric) error { coldCounts.sparseBucketsNegative.Range(addAndReset(&hotCounts.sparseBucketsNegative, &hotCounts.sparseBucketsNumber)) }() - his.SbZeroCount = proto.Uint64(zeroBucket) - his.SbNegative = makeSparseBuckets(&coldCounts.sparseBucketsNegative) - his.SbPositive = makeSparseBuckets(&coldCounts.sparseBucketsPositive) + his.ZeroCount = proto.Uint64(zeroBucket) + his.NegativeSpan, his.NegativeDelta = makeSparseBuckets(&coldCounts.sparseBucketsNegative) + his.PositiveSpan, his.PositiveDelta = makeSparseBuckets(&coldCounts.sparseBucketsPositive) } addAndResetCounts(hotCounts, coldCounts) return nil @@ -1235,7 +1236,7 @@ func pickSparseSchema(bucketFactor float64) int32 { } } -func makeSparseBuckets(buckets *sync.Map) *dto.SparseBuckets { +func makeSparseBuckets(buckets *sync.Map) ([]*dto.BucketSpan, []int64) { var ii []int buckets.Range(func(k, v interface{}) bool { ii = append(ii, k.(int)) @@ -1244,16 +1245,19 @@ func makeSparseBuckets(buckets *sync.Map) *dto.SparseBuckets { sort.Ints(ii) if len(ii) == 0 { - return nil + return nil, nil } - sbs := dto.SparseBuckets{} - var prevCount int64 - var nextI int + var ( + spans []*dto.BucketSpan + deltas []int64 + prevCount int64 + nextI int + ) appendDelta := func(count int64) { - *sbs.Span[len(sbs.Span)-1].Length++ - sbs.Delta = append(sbs.Delta, count-prevCount) + *spans[len(spans)-1].Length++ + deltas = append(deltas, count-prevCount) prevCount = count } @@ -1270,7 +1274,7 @@ func makeSparseBuckets(buckets *sync.Map) *dto.SparseBuckets { // We have to create a new span, either because we are // at the very beginning, or because we have found a gap // of more than two buckets. - sbs.Span = append(sbs.Span, &dto.SparseBuckets_Span{ + spans = append(spans, &dto.BucketSpan{ Offset: proto.Int32(iDelta), Length: proto.Uint32(0), }) @@ -1284,7 +1288,7 @@ func makeSparseBuckets(buckets *sync.Map) *dto.SparseBuckets { appendDelta(count) nextI = i + 1 } - return &sbs + return spans, deltas } // addToSparseBucket increments the sparse bucket at key by the provided diff --git a/prometheus/histogram_test.go b/prometheus/histogram_test.go index 81803d5..5b26fb4 100644 --- a/prometheus/histogram_test.go +++ b/prometheus/histogram_test.go @@ -490,13 +490,13 @@ func TestSparseHistogram(t *testing.T) { name: "factor 1.1 results in schema 3", observations: []float64{0, 1, 2, 3}, factor: 1.1, - want: `sample_count:4 sample_sum:6 sb_schema:3 sb_zero_threshold:2.938735877055719e-39 sb_zero_count:1 sb_positive: span: span: delta:1 delta:0 delta:0 > `, + want: `sample_count:4 sample_sum:6 schema:3 zero_threshold:2.938735877055719e-39 zero_count:1 positive_span: positive_span: positive_span: positive_delta:1 positive_delta:0 positive_delta:0 `, }, { name: "factor 1.2 results in schema 2", observations: []float64{0, 1, 1.2, 1.4, 1.8, 2}, factor: 1.2, - want: `sample_count:6 sample_sum:7.4 sb_schema:2 sb_zero_threshold:2.938735877055719e-39 sb_zero_count:1 sb_positive: delta:1 delta:-1 delta:2 delta:-2 delta:2 > `, + want: `sample_count:6 sample_sum:7.4 schema:2 zero_threshold:2.938735877055719e-39 zero_count:1 positive_span: positive_delta:1 positive_delta:-1 positive_delta:2 positive_delta:-2 positive_delta:2 `, }, { name: "factor 4 results in schema -1", @@ -507,7 +507,7 @@ func TestSparseHistogram(t *testing.T) { 33.33, // Bucket 3: (16, 64] }, factor: 4, - want: `sample_count:10 sample_sum:62.83 sb_schema:-1 sb_zero_threshold:2.938735877055719e-39 sb_zero_count:0 sb_positive: delta:2 delta:2 delta:-1 delta:-2 > `, + want: `sample_count:10 sample_sum:62.83 schema:-1 zero_threshold:2.938735877055719e-39 zero_count:0 positive_span: positive_delta:2 positive_delta:2 positive_delta:-1 positive_delta:-2 `, }, { name: "factor 17 results in schema -2", @@ -517,58 +517,58 @@ func TestSparseHistogram(t *testing.T) { 33.33, // Bucket 2: (16, 256] }, factor: 17, - want: `sample_count:10 sample_sum:62.83 sb_schema:-2 sb_zero_threshold:2.938735877055719e-39 sb_zero_count:0 sb_positive: delta:2 delta:5 delta:-6 > `, + want: `sample_count:10 sample_sum:62.83 schema:-2 zero_threshold:2.938735877055719e-39 zero_count:0 positive_span: positive_delta:2 positive_delta:5 positive_delta:-6 `, }, { name: "negative buckets", observations: []float64{0, -1, -1.2, -1.4, -1.8, -2}, factor: 1.2, - want: `sample_count:6 sample_sum:-7.4 sb_schema:2 sb_zero_threshold:2.938735877055719e-39 sb_zero_count:1 sb_negative: delta:1 delta:-1 delta:2 delta:-2 delta:2 > `, + want: `sample_count:6 sample_sum:-7.4 schema:2 zero_threshold:2.938735877055719e-39 zero_count:1 negative_span: negative_delta:1 negative_delta:-1 negative_delta:2 negative_delta:-2 negative_delta:2 `, }, { name: "negative and positive buckets", observations: []float64{0, -1, -1.2, -1.4, -1.8, -2, 1, 1.2, 1.4, 1.8, 2}, factor: 1.2, - want: `sample_count:11 sample_sum:0 sb_schema:2 sb_zero_threshold:2.938735877055719e-39 sb_zero_count:1 sb_negative: delta:1 delta:-1 delta:2 delta:-2 delta:2 > sb_positive: delta:1 delta:-1 delta:2 delta:-2 delta:2 > `, + want: `sample_count:11 sample_sum:0 schema:2 zero_threshold:2.938735877055719e-39 zero_count:1 negative_span: negative_delta:1 negative_delta:-1 negative_delta:2 negative_delta:-2 negative_delta:2 positive_span: positive_delta:1 positive_delta:-1 positive_delta:2 positive_delta:-2 positive_delta:2 `, }, { name: "wide zero bucket", observations: []float64{0, -1, -1.2, -1.4, -1.8, -2, 1, 1.2, 1.4, 1.8, 2}, factor: 1.2, zeroThreshold: 1.4, - want: `sample_count:11 sample_sum:0 sb_schema:2 sb_zero_threshold:1.4 sb_zero_count:7 sb_negative: delta:2 > sb_positive: delta:2 > `, + want: `sample_count:11 sample_sum:0 schema:2 zero_threshold:1.4 zero_count:7 negative_span: negative_delta:2 positive_span: positive_delta:2 `, }, { name: "NaN observation", observations: []float64{0, 1, 1.2, 1.4, 1.8, 2, math.NaN()}, factor: 1.2, - want: `sample_count:7 sample_sum:nan sb_schema:2 sb_zero_threshold:2.938735877055719e-39 sb_zero_count:1 sb_positive: delta:1 delta:-1 delta:2 delta:-2 delta:2 > `, + want: `sample_count:7 sample_sum:nan schema:2 zero_threshold:2.938735877055719e-39 zero_count:1 positive_span: positive_delta:1 positive_delta:-1 positive_delta:2 positive_delta:-2 positive_delta:2 `, }, { name: "+Inf observation", observations: []float64{0, 1, 1.2, 1.4, 1.8, 2, math.Inf(+1)}, factor: 1.2, - want: `sample_count:7 sample_sum:inf sb_schema:2 sb_zero_threshold:2.938735877055719e-39 sb_zero_count:1 sb_positive: span: delta:1 delta:-1 delta:2 delta:-2 delta:2 delta:-1 > `, + want: `sample_count:7 sample_sum:inf schema:2 zero_threshold:2.938735877055719e-39 zero_count:1 positive_span: positive_span: positive_delta:1 positive_delta:-1 positive_delta:2 positive_delta:-2 positive_delta:2 positive_delta:-1 `, }, { name: "-Inf observation", observations: []float64{0, 1, 1.2, 1.4, 1.8, 2, math.Inf(-1)}, factor: 1.2, - want: `sample_count:7 sample_sum:-inf sb_schema:2 sb_zero_threshold:2.938735877055719e-39 sb_zero_count:1 sb_negative: delta:1 > sb_positive: delta:1 delta:-1 delta:2 delta:-2 delta:2 > `, + want: `sample_count:7 sample_sum:-inf schema:2 zero_threshold:2.938735877055719e-39 zero_count:1 negative_span: negative_delta:1 positive_span: positive_delta:1 positive_delta:-1 positive_delta:2 positive_delta:-2 positive_delta:2 `, }, { name: "limited buckets but nothing triggered", observations: []float64{0, 1, 1.2, 1.4, 1.8, 2}, factor: 1.2, maxBuckets: 4, - want: `sample_count:6 sample_sum:7.4 sb_schema:2 sb_zero_threshold:2.938735877055719e-39 sb_zero_count:1 sb_positive: delta:1 delta:-1 delta:2 delta:-2 delta:2 > `, + want: `sample_count:6 sample_sum:7.4 schema:2 zero_threshold:2.938735877055719e-39 zero_count:1 positive_span: positive_delta:1 positive_delta:-1 positive_delta:2 positive_delta:-2 positive_delta:2 `, }, { name: "buckets limited by halving resolution", observations: []float64{0, 1, 1.1, 1.2, 1.4, 1.8, 2, 3}, factor: 1.2, maxBuckets: 4, - want: `sample_count:8 sample_sum:11.5 sb_schema:1 sb_zero_threshold:2.938735877055719e-39 sb_zero_count:1 sb_positive: delta:1 delta:2 delta:-1 delta:-2 delta:1 > `, + want: `sample_count:8 sample_sum:11.5 schema:1 zero_threshold:2.938735877055719e-39 zero_count:1 positive_span: positive_delta:1 positive_delta:2 positive_delta:-1 positive_delta:-2 positive_delta:1 `, }, { name: "buckets limited by widening the zero bucket", @@ -576,7 +576,7 @@ func TestSparseHistogram(t *testing.T) { factor: 1.2, maxBuckets: 4, maxZeroThreshold: 1.2, - want: `sample_count:8 sample_sum:11.5 sb_schema:2 sb_zero_threshold:1 sb_zero_count:2 sb_positive: delta:1 delta:1 delta:-2 delta:2 delta:-2 delta:0 delta:1 > `, + want: `sample_count:8 sample_sum:11.5 schema:2 zero_threshold:1 zero_count:2 positive_span: positive_delta:1 positive_delta:1 positive_delta:-2 positive_delta:2 positive_delta:-2 positive_delta:0 positive_delta:1 `, }, { name: "buckets limited by widening the zero bucket twice", @@ -584,7 +584,7 @@ func TestSparseHistogram(t *testing.T) { factor: 1.2, maxBuckets: 4, maxZeroThreshold: 1.2, - want: `sample_count:9 sample_sum:15.5 sb_schema:2 sb_zero_threshold:1.189207115002721 sb_zero_count:3 sb_positive: delta:2 delta:-2 delta:2 delta:-2 delta:0 delta:1 delta:0 > `, + want: `sample_count:9 sample_sum:15.5 schema:2 zero_threshold:1.189207115002721 zero_count:3 positive_span: positive_delta:2 positive_delta:-2 positive_delta:2 positive_delta:-2 positive_delta:0 positive_delta:1 positive_delta:0 `, }, { name: "buckets limited by reset", @@ -593,21 +593,21 @@ func TestSparseHistogram(t *testing.T) { maxBuckets: 4, maxZeroThreshold: 1.2, minResetDuration: 5 * time.Minute, - want: `sample_count:2 sample_sum:7 sb_schema:2 sb_zero_threshold:2.938735877055719e-39 sb_zero_count:0 sb_positive: delta:1 delta:0 > `, + want: `sample_count:2 sample_sum:7 schema:2 zero_threshold:2.938735877055719e-39 zero_count:0 positive_span: positive_delta:1 positive_delta:0 `, }, { name: "limited buckets but nothing triggered, negative observations", observations: []float64{0, -1, -1.2, -1.4, -1.8, -2}, factor: 1.2, maxBuckets: 4, - want: `sample_count:6 sample_sum:-7.4 sb_schema:2 sb_zero_threshold:2.938735877055719e-39 sb_zero_count:1 sb_negative: delta:1 delta:-1 delta:2 delta:-2 delta:2 > `, + want: `sample_count:6 sample_sum:-7.4 schema:2 zero_threshold:2.938735877055719e-39 zero_count:1 negative_span: negative_delta:1 negative_delta:-1 negative_delta:2 negative_delta:-2 negative_delta:2 `, }, { name: "buckets limited by halving resolution, negative observations", observations: []float64{0, -1, -1.1, -1.2, -1.4, -1.8, -2, -3}, factor: 1.2, maxBuckets: 4, - want: `sample_count:8 sample_sum:-11.5 sb_schema:1 sb_zero_threshold:2.938735877055719e-39 sb_zero_count:1 sb_negative: delta:1 delta:2 delta:-1 delta:-2 delta:1 > `, + want: `sample_count:8 sample_sum:-11.5 schema:1 zero_threshold:2.938735877055719e-39 zero_count:1 negative_span: negative_delta:1 negative_delta:2 negative_delta:-1 negative_delta:-2 negative_delta:1 `, }, { name: "buckets limited by widening the zero bucket, negative observations", @@ -615,7 +615,7 @@ func TestSparseHistogram(t *testing.T) { factor: 1.2, maxBuckets: 4, maxZeroThreshold: 1.2, - want: `sample_count:8 sample_sum:-11.5 sb_schema:2 sb_zero_threshold:1 sb_zero_count:2 sb_negative: delta:1 delta:1 delta:-2 delta:2 delta:-2 delta:0 delta:1 > `, + want: `sample_count:8 sample_sum:-11.5 schema:2 zero_threshold:1 zero_count:2 negative_span: negative_delta:1 negative_delta:1 negative_delta:-2 negative_delta:2 negative_delta:-2 negative_delta:0 negative_delta:1 `, }, { name: "buckets limited by widening the zero bucket twice, negative observations", @@ -623,7 +623,7 @@ func TestSparseHistogram(t *testing.T) { factor: 1.2, maxBuckets: 4, maxZeroThreshold: 1.2, - want: `sample_count:9 sample_sum:-15.5 sb_schema:2 sb_zero_threshold:1.189207115002721 sb_zero_count:3 sb_negative: delta:2 delta:-2 delta:2 delta:-2 delta:0 delta:1 delta:0 > `, + want: `sample_count:9 sample_sum:-15.5 schema:2 zero_threshold:1.189207115002721 zero_count:3 negative_span: negative_delta:2 negative_delta:-2 negative_delta:2 negative_delta:-2 negative_delta:0 negative_delta:1 negative_delta:0 `, }, { name: "buckets limited by reset, negative observations", @@ -632,7 +632,7 @@ func TestSparseHistogram(t *testing.T) { maxBuckets: 4, maxZeroThreshold: 1.2, minResetDuration: 5 * time.Minute, - want: `sample_count:2 sample_sum:-7 sb_schema:2 sb_zero_threshold:2.938735877055719e-39 sb_zero_count:0 sb_negative: delta:1 delta:0 > `, + want: `sample_count:2 sample_sum:-7 schema:2 zero_threshold:2.938735877055719e-39 zero_count:0 negative_span: negative_delta:1 negative_delta:0 `, }, { name: "buckets limited by halving resolution, then reset", @@ -640,7 +640,7 @@ func TestSparseHistogram(t *testing.T) { factor: 1.2, maxBuckets: 4, minResetDuration: 9 * time.Minute, - want: `sample_count:2 sample_sum:7 sb_schema:2 sb_zero_threshold:2.938735877055719e-39 sb_zero_count:0 sb_positive: delta:1 delta:0 > `, + want: `sample_count:2 sample_sum:7 schema:2 zero_threshold:2.938735877055719e-39 zero_count:0 positive_span: positive_delta:1 positive_delta:0 `, }, { name: "buckets limited by widening the zero bucket, then reset", @@ -649,7 +649,7 @@ func TestSparseHistogram(t *testing.T) { maxBuckets: 4, maxZeroThreshold: 1.2, minResetDuration: 9 * time.Minute, - want: `sample_count:2 sample_sum:7 sb_schema:2 sb_zero_threshold:2.938735877055719e-39 sb_zero_count:0 sb_positive: delta:1 delta:0 > `, + want: `sample_count:2 sample_sum:7 schema:2 zero_threshold:2.938735877055719e-39 zero_count:0 positive_span: positive_delta:1 positive_delta:0 `, }, } @@ -754,9 +754,9 @@ func TestSparseHistogramConcurrency(t *testing.T) { // t.Errorf("got sample sum %f, want %f", got, want) // } - sumBuckets := int(m.Histogram.GetSbZeroCount()) + sumBuckets := int(m.Histogram.GetZeroCount()) current := 0 - for _, delta := range m.Histogram.GetSbNegative().GetDelta() { + for _, delta := range m.Histogram.GetNegativeDelta() { current += int(delta) if current < 0 { t.Fatalf("negative bucket population negative: %d", current) @@ -764,7 +764,7 @@ func TestSparseHistogramConcurrency(t *testing.T) { sumBuckets += current } current = 0 - for _, delta := range m.Histogram.GetSbPositive().GetDelta() { + for _, delta := range m.Histogram.GetPositiveDelta() { current += int(delta) if current < 0 { t.Fatalf("positive bucket population negative: %d", current)