mirror
/
client_golang
mirror of https://github.com/prometheus/client_golang.git
1
0
Fork 1
Code Issues Packages Projects Releases Wiki Activity

metricvec: handle hash collision for labeled metrics 

While hash collisions are quite rare, the current state of the client
library carries a risk of merging two separate label values into a
single metric bucket. The effects are near impossible to detect and the
result will be missing or incorrect counters.

This changeset handles hash collisions by falling back to collision
resolution if multiple label values hash to the same value. This works
similar to separate chaining using a slice. Extra storage is minimized
to only the value key slice to that metrics can be differentiated
within a bucket.

In general, the cost of handling collisions is completely minimized
under normal operation. Performance does show slight increases in
certain areas, but these are more likely statistically anomalies. More
importantly, zero allocation behavior for metrics is preserved on the
fast path. Minimal allocations may be made during collision handling but
this has minimal effect.

Benchmark comparisons with and without collision resolution follow.

```
benchmark                                                         old ns/op     new ns/op     delta
BenchmarkCounterWithLabelValues-4                                 99.0          107           +8.08%
BenchmarkCounterWithLabelValuesConcurrent-4                       79.6          91.0          +14.32%
BenchmarkCounterWithMappedLabels-4                                518           542           +4.63%
BenchmarkCounterWithPreparedMappedLabels-4                        127           137           +7.87%
BenchmarkCounterNoLabels-4                                        19.5          19.1          -2.05%
BenchmarkGaugeWithLabelValues-4                                   97.4          110           +12.94%
BenchmarkGaugeNoLabels-4                                          12.4          10.3          -16.94%
BenchmarkSummaryWithLabelValues-4                                 1204          915           -24.00%
BenchmarkSummaryNoLabels-4                                        936           847           -9.51%
BenchmarkHistogramWithLabelValues-4                               147           147           +0.00%
BenchmarkHistogramNoLabels-4                                      50.6          49.3          -2.57%
BenchmarkHistogramObserve1-4                                      37.9          37.5          -1.06%
BenchmarkHistogramObserve2-4                                      122           137           +12.30%
BenchmarkHistogramObserve4-4                                      310           352           +13.55%
BenchmarkHistogramObserve8-4                                      691           729           +5.50%
BenchmarkHistogramWrite1-4                                        3374          3097          -8.21%
BenchmarkHistogramWrite2-4                                        5310          5051          -4.88%
BenchmarkHistogramWrite4-4                                        12094         10690         -11.61%
BenchmarkHistogramWrite8-4                                        19416         17755         -8.55%
BenchmarkHandler-4                                                11934304      13765894      +15.35%
BenchmarkSummaryObserve1-4                                        1119          1105          -1.25%
BenchmarkSummaryObserve2-4                                        3679          3430          -6.77%
BenchmarkSummaryObserve4-4                                        10678         7982          -25.25%
BenchmarkSummaryObserve8-4                                        22974         16689         -27.36%
BenchmarkSummaryWrite1-4                                          25962         14680         -43.46%
BenchmarkSummaryWrite2-4                                          38019         35073         -7.75%
BenchmarkSummaryWrite4-4                                          78027         56816         -27.18%
BenchmarkSummaryWrite8-4                                          117220        132248        +12.82%
BenchmarkMetricVecWithLabelValuesBasic-4                          138           133           -3.62%
BenchmarkMetricVecWithLabelValues2Keys10ValueCardinality-4        150           144           -4.00%
BenchmarkMetricVecWithLabelValues4Keys10ValueCardinality-4        263           256           -2.66%
BenchmarkMetricVecWithLabelValues2Keys100ValueCardinality-4       145           155           +6.90%
BenchmarkMetricVecWithLabelValues10Keys100ValueCardinality-4      606           634           +4.62%
BenchmarkMetricVecWithLabelValues10Keys1000ValueCardinality-4     746           641           -14.08%

benchmark                                                         old allocs     new allocs     delta
BenchmarkCounterWithLabelValues-4                                 0              0              +0.00%
BenchmarkCounterWithLabelValuesConcurrent-4                       0              0              +0.00%
BenchmarkCounterWithMappedLabels-4                                2              2              +0.00%
BenchmarkCounterWithPreparedMappedLabels-4                        0              0              +0.00%
BenchmarkCounterNoLabels-4                                        0              0              +0.00%
BenchmarkGaugeWithLabelValues-4                                   0              0              +0.00%
BenchmarkGaugeNoLabels-4                                          0              0              +0.00%
BenchmarkSummaryWithLabelValues-4                                 0              0              +0.00%
BenchmarkSummaryNoLabels-4                                        0              0              +0.00%
BenchmarkHistogramWithLabelValues-4                               0              0              +0.00%
BenchmarkHistogramNoLabels-4                                      0              0              +0.00%
BenchmarkMetricVecWithLabelValuesBasic-4                          0              0              +0.00%
BenchmarkMetricVecWithLabelValues2Keys10ValueCardinality-4        0              0              +0.00%
BenchmarkMetricVecWithLabelValues4Keys10ValueCardinality-4        0              0              +0.00%
BenchmarkMetricVecWithLabelValues2Keys100ValueCardinality-4       0              0              +0.00%
BenchmarkMetricVecWithLabelValues10Keys100ValueCardinality-4      0              0              +0.00%
BenchmarkMetricVecWithLabelValues10Keys1000ValueCardinality-4     0              0              +0.00%

benchmark                                                         old bytes     new bytes     delta
BenchmarkCounterWithLabelValues-4                                 0             0             +0.00%
BenchmarkCounterWithLabelValuesConcurrent-4                       0             0             +0.00%
BenchmarkCounterWithMappedLabels-4                                336           336           +0.00%
BenchmarkCounterWithPreparedMappedLabels-4                        0             0             +0.00%
BenchmarkCounterNoLabels-4                                        0             0             +0.00%
BenchmarkGaugeWithLabelValues-4                                   0             0             +0.00%
BenchmarkGaugeNoLabels-4                                          0             0             +0.00%
BenchmarkSummaryWithLabelValues-4                                 0             0             +0.00%
BenchmarkSummaryNoLabels-4                                        0             0             +0.00%
BenchmarkHistogramWithLabelValues-4                               0             0             +0.00%
BenchmarkHistogramNoLabels-4                                      0             0             +0.00%
BenchmarkMetricVecWithLabelValuesBasic-4                          0             0             +0.00%
BenchmarkMetricVecWithLabelValues2Keys10ValueCardinality-4        0             0             +0.00%
BenchmarkMetricVecWithLabelValues4Keys10ValueCardinality-4        0             0             +0.00%
BenchmarkMetricVecWithLabelValues2Keys100ValueCardinality-4       0             0             +0.00%
BenchmarkMetricVecWithLabelValues10Keys100ValueCardinality-4      0             0             +0.00%
BenchmarkMetricVecWithLabelValues10Keys1000ValueCardinality-4     0             0             +0.00%
```

Signed-off-by: Stephen J Day <stephen.day@docker.com>
This commit is contained in:
Stephen J Day 2016-08-11 17:06:03 -07:00
parent c4004ef5f6
commit 3cf50db5fd
No known key found for this signature in database
GPG Key ID: FB5F6B2905D7ECF3
2 changed files with 114 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

121
prometheus/vec.go
View File

@ -25,22 +25,29 @@ import (
// provided in this package. // provided in this package.
type MetricVec struct { type MetricVec struct {
mtx sync.RWMutex // Protects the children. mtx sync.RWMutex // Protects the children.
children map[uint64]Metric children map[uint64][]metricLabelValue
desc *Desc desc *Desc
newMetric func(labelValues ...string) Metric newMetric func(labelValues ...string) Metric
hashAdd func(h uint64, s string) uint64 // replace hash function for testing collision handling
} }
// newMetricVec returns an initialized MetricVec. The concrete value is // newMetricVec returns an initialized MetricVec. The concrete value is
// returned for embedding into another struct. // returned for embedding into another struct.
func newMetricVec(desc *Desc, newMetric func(lvs ...string) Metric) MetricVec { func newMetricVec(desc *Desc, newMetric func(lvs ...string) Metric) MetricVec {
return MetricVec{ return MetricVec{
children: map[uint64]Metric{}, children: map[uint64][]metricLabelValue{},
desc: desc, desc: desc,
newMetric: newMetric, newMetric: newMetric,
hashAdd: hashAdd,
} }
} }
type metricLabelValue struct {
values []string
metric Metric
}
// Describe implements Collector. The length of the returned slice // Describe implements Collector. The length of the returned slice
// is always one. // is always one.
func (m *MetricVec) Describe(ch chan<- *Desc) { func (m *MetricVec) Describe(ch chan<- *Desc) {
@ -52,8 +59,10 @@ func (m *MetricVec) Collect(ch chan<- Metric) {
m.mtx.RLock() m.mtx.RLock()
defer m.mtx.RUnlock() defer m.mtx.RUnlock()
for _, metric := range m.children { for _, metrics := range m.children {
ch <- metric for _, metric := range metrics {
ch <- metric.metric
}
} }
} }
@ -88,7 +97,7 @@ func (m *MetricVec) GetMetricWithLabelValues(lvs ...string) (Metric, error) {
} }
m.mtx.RLock() m.mtx.RLock()
metric, ok := m.children[h] metric, ok := m.getMetric(h, lvs...)
m.mtx.RUnlock() m.mtx.RUnlock()
if ok { if ok {
return metric, nil return metric, nil
@ -118,11 +127,12 @@ func (m *MetricVec) GetMetricWith(labels Labels) (Metric, error) {
} }
m.mtx.RLock() m.mtx.RLock()
metric, ok := m.children[h] metrics, ok := m.children[h]
if ok && len(metrics) == 1 {
m.mtx.RUnlock() m.mtx.RUnlock()
if ok { return metrics[0].metric, nil
return metric, nil
} }
m.mtx.RUnlock()
lvs := make([]string, len(labels)) lvs := make([]string, len(labels))
for i, label := range m.desc.variableLabels { for i, label := range m.desc.variableLabels {
@ -178,11 +188,7 @@ func (m *MetricVec) DeleteLabelValues(lvs ...string) bool {
if err != nil { if err != nil {
return false return false
} }
if _, ok := m.children[h]; !ok { return m.deleteByHash(h, lvs...)
return false
}
delete(m.children, h)
return true
} }
// Delete deletes the metric where the variable labels are the same as those // Delete deletes the metric where the variable labels are the same as those
@ -203,10 +209,38 @@ func (m *MetricVec) Delete(labels Labels) bool {
if err != nil { if err != nil {
return false return false
} }
if _, ok := m.children[h]; !ok {
var lvs []string
for _, k := range m.desc.variableLabels {
lvs = append(lvs, labels[k])
}
return m.deleteByHash(h, lvs...)
}
// deleteByHash removes the metric from the hash bucket h. If there are
// multiple matches in the bucket, use lvs to select a metric and remove only
// that metric.
func (m *MetricVec) deleteByHash(h uint64, lvs ...string) bool {
metrics, ok := m.children[h]
if !ok {
return false return false
} }
if len(metrics) < 2 {
delete(m.children, h) delete(m.children, h)
}
i := findMetric(lvs, metrics)
if i >= len(metrics) {
return false
}
if len(metrics) > 1 {
m.children[h] = append(metrics[:i], metrics[i+1:]...)
} else {
delete(m.children, h)
}
return true return true
} }
@ -226,7 +260,7 @@ func (m *MetricVec) hashLabelValues(vals []string) (uint64, error) {
} }
h := hashNew() h := hashNew()
for _, val := range vals { for _, val := range vals {
h = hashAdd(h, val) h = m.hashAdd(h, val)
} }
return h, nil return h, nil
} }
@ -241,19 +275,70 @@ func (m *MetricVec) hashLabels(labels Labels) (uint64, error) {
if !ok { if !ok {
return 0, fmt.Errorf("label name %q missing in label map", label) return 0, fmt.Errorf("label name %q missing in label map", label)
} }
h = hashAdd(h, val) h = m.hashAdd(h, val)
} }
return h, nil return h, nil
} }
func (m *MetricVec) getOrCreateMetric(hash uint64, labelValues ...string) Metric { func (m *MetricVec) getOrCreateMetric(hash uint64, labelValues ...string) Metric {
metric, ok := m.children[hash] metric, ok := m.getMetric(hash, labelValues...)
if !ok { if !ok {
// Copy labelValues. Otherwise, they would be allocated even if we don't go // Copy labelValues. Otherwise, they would be allocated even if we don't go
// down this code path. // down this code path.
copiedLabelValues := append(make([]string, 0, len(labelValues)), labelValues...) copiedLabelValues := append(make([]string, 0, len(labelValues)), labelValues...)
metric = m.newMetric(copiedLabelValues...) metric = m.newMetric(copiedLabelValues...)
m.children[hash] = metric m.children[hash] = append(m.children[hash], metricLabelValue{values: copiedLabelValues, metric: metric})
} }
return metric return metric
} }
// getMetric while handling possible collisions in the hash space. Must be
// called while holding read mutex.
func (m *MetricVec) getMetric(h uint64, lvs ...string) (Metric, bool) {
metrics, ok := m.children[h]
if ok {
return m.selectMetric(lvs, metrics)
}
return nil, false
}
func (m *MetricVec) selectMetric(lvs []string, metrics []metricLabelValue) (Metric, bool) {
switch len(metrics) {
case 0:
return nil, false
case 1:
// collisions are rare, this should be the fast path.
return metrics[0].metric, true
}
i := findMetric(lvs, metrics)
if i < len(metrics) {
return metrics[i].metric, true
}
return nil, false
}
// findMetric returns the index of the matching metric or len(metrics) if not
// found.
func findMetric(lvs []string, metrics []metricLabelValue) int {
next:
for i, metric := range metrics {
if len(metric.values) != len(lvs) {
continue
}
for j, v := range metric.values {
if v != lvs[j] {
continue next
}
}
// falling out of the loop here means we have a match!
return i
}
return len(metrics)
}

9
prometheus/vec_test.go
View File

@ -52,7 +52,16 @@ func TestDelete(t *testing.T) {
func TestDeleteLabelValues(t *testing.T) { func TestDeleteLabelValues(t *testing.T) {
vec := newUntypedMetricVec("test", "helpless", []string{"l1", "l2"}) vec := newUntypedMetricVec("test", "helpless", []string{"l1", "l2"})
testDeleteLabelValues(t, vec)
}
func TestDeleteLabelValuesWithCollisions(t *testing.T) {
vec := newUntypedMetricVec("test", "helpless", []string{"l1", "l2"})
vec.hashAdd = func(h uint64, s string) uint64 { return 1 }
testDeleteLabelValues(t, vec)
}
func testDeleteLabelValues(t *testing.T, vec *MetricVec) {
if got, want := vec.DeleteLabelValues("v1", "v2"), false; got != want { if got, want := vec.DeleteLabelValues("v1", "v2"), false; got != want {
t.Errorf("got %v, want %v", got, want) t.Errorf("got %v, want %v", got, want)
} }