Adjust various things required for the new storage backend.

- Change Fingerprints to be simple uint64s.
- Deal sensibly with missing metric names.
- Enable finer-grained time resolution.

Merge this concurrently with the merge of the new storage backend into
prometheus/prometheus.

Change-Id: Idd82f137aa0c4286df422c53ce3c62e0de285360
This commit is contained in:
Julius Volz 2014-07-22 17:52:10 +02:00 committed by Bjoern Rabenstein
parent c53c07a719
commit 87a585def8
7 changed files with 124 additions and 303 deletions

View File

@ -19,96 +19,35 @@ import (
"hash/fnv" "hash/fnv"
"sort" "sort"
"strconv" "strconv"
"strings"
) )
// Fingerprint provides a hash-capable representation of a Metric. // Fingerprint provides a hash-capable representation of a Metric.
type Fingerprint struct { // For our purposes, FNV-1A 64-bit is used.
// A hashed representation of the underyling entity. For our purposes, FNV-1A type Fingerprint uint64
// 64-bit is used.
Hash uint64 func (f Fingerprint) String() string {
FirstCharacterOfFirstLabelName string return fmt.Sprintf("%016x", uint64(f))
LabelMatterLength uint
LastCharacterOfLastLabelValue string
} }
func (f *Fingerprint) String() string { // Less implements sort.Interface.
return strings.Join([]string{fmt.Sprintf("%020d", f.Hash), f.FirstCharacterOfFirstLabelName, fmt.Sprint(f.LabelMatterLength), f.LastCharacterOfLastLabelValue}, "-") func (f Fingerprint) Less(o Fingerprint) bool {
return f < o
} }
// Less compares first the Hash, then the FirstCharacterOfFirstLabelName, then // Equal implements sort.Interface.
// the LabelMatterLength, then the LastCharacterOfLastLabelValue. func (f Fingerprint) Equal(o Fingerprint) bool {
func (f *Fingerprint) Less(o *Fingerprint) bool { return f == o
if f.Hash < o.Hash {
return true
}
if f.Hash > o.Hash {
return false
} }
if f.FirstCharacterOfFirstLabelName < o.FirstCharacterOfFirstLabelName { // LoadFromString transforms a string representation into a Fingerprint.
return true
}
if f.FirstCharacterOfFirstLabelName > o.FirstCharacterOfFirstLabelName {
return false
}
if f.LabelMatterLength < o.LabelMatterLength {
return true
}
if f.LabelMatterLength > o.LabelMatterLength {
return false
}
if f.LastCharacterOfLastLabelValue < o.LastCharacterOfLastLabelValue {
return true
}
if f.LastCharacterOfLastLabelValue > o.LastCharacterOfLastLabelValue {
return false
}
return false
}
// Equal uses the same semantics as Less.
func (f *Fingerprint) Equal(o *Fingerprint) bool {
if f.Hash != o.Hash {
return false
}
if f.FirstCharacterOfFirstLabelName != o.FirstCharacterOfFirstLabelName {
return false
}
if f.LabelMatterLength != o.LabelMatterLength {
return false
}
return f.LastCharacterOfLastLabelValue == o.LastCharacterOfLastLabelValue
}
const rowKeyDelimiter = "-"
// LoadFromString transforms a string representation into a Fingerprint,
// resetting any previous attributes.
func (f *Fingerprint) LoadFromString(s string) { func (f *Fingerprint) LoadFromString(s string) {
components := strings.Split(s, rowKeyDelimiter) num, err := strconv.ParseUint(s, 16, 64)
hash, err := strconv.ParseUint(components[0], 10, 64)
if err != nil { if err != nil {
panic(err) panic(err)
} }
labelMatterLength, err := strconv.ParseUint(components[2], 10, 0) *f = Fingerprint(num)
if err != nil {
panic(err)
} }
f.Hash = hash
f.FirstCharacterOfFirstLabelName = components[1]
f.LabelMatterLength = uint(labelMatterLength)
f.LastCharacterOfLastLabelValue = components[3]
}
const reservedDelimiter = `"`
// LoadFromMetric decomposes a Metric into this Fingerprint // LoadFromMetric decomposes a Metric into this Fingerprint
func (f *Fingerprint) LoadFromMetric(m Metric) { func (f *Fingerprint) LoadFromMetric(m Metric) {
labelLength := len(m) labelLength := len(m)
@ -121,46 +60,33 @@ func (f *Fingerprint) LoadFromMetric(m Metric) {
sort.Strings(labelNames) sort.Strings(labelNames)
summer := fnv.New64a() summer := fnv.New64a()
firstCharacterOfFirstLabelName := ""
lastCharacterOfLastLabelValue := ""
labelMatterLength := 0
for i, labelName := range labelNames { for _, labelName := range labelNames {
labelValue := m[LabelName(labelName)] labelValue := m[LabelName(labelName)]
labelNameLength := len(labelName)
labelValueLength := len(labelValue)
labelMatterLength += labelNameLength + labelValueLength
if i == 0 {
firstCharacterOfFirstLabelName = labelName[0:1]
}
if i == labelLength-1 {
lastCharacterOfLastLabelValue = string(labelValue[labelValueLength-1 : labelValueLength])
}
summer.Write([]byte(labelName)) summer.Write([]byte(labelName))
summer.Write([]byte(reservedDelimiter)) summer.Write([]byte{0})
summer.Write([]byte(labelValue)) summer.Write([]byte(labelValue))
} }
f.FirstCharacterOfFirstLabelName = firstCharacterOfFirstLabelName *f = Fingerprint(binary.LittleEndian.Uint64(summer.Sum(nil)))
f.Hash = binary.LittleEndian.Uint64(summer.Sum(nil))
f.LabelMatterLength = uint(labelMatterLength % 10)
f.LastCharacterOfLastLabelValue = lastCharacterOfLastLabelValue
} }
// Fingerprints represents a collection of Fingerprint subject to a given // Fingerprints represents a collection of Fingerprint subject to a given
// natural sorting scheme. It implements sort.Interface. // natural sorting scheme. It implements sort.Interface.
type Fingerprints []*Fingerprint type Fingerprints []Fingerprint
// Len implements sort.Interface.
func (f Fingerprints) Len() int { func (f Fingerprints) Len() int {
return len(f) return len(f)
} }
// Less implements sort.Interface.
func (f Fingerprints) Less(i, j int) bool { func (f Fingerprints) Less(i, j int) bool {
return f[i].Less(f[j]) return f[i] < f[j]
} }
// Swap implements sort.Interface.
func (f Fingerprints) Swap(i, j int) { func (f Fingerprints) Swap(i, j int) {
f[i], f[j] = f[j], f[i] f[i], f[j] = f[j], f[i]
} }

View File

@ -25,47 +25,46 @@ type Metric map[LabelName]LabelValue
// Equal compares the fingerprints of both metrics. // Equal compares the fingerprints of both metrics.
func (m Metric) Equal(o Metric) bool { func (m Metric) Equal(o Metric) bool {
lFingerprint := &Fingerprint{} return m.Fingerprint().Equal(o.Fingerprint())
rFingerprint := &Fingerprint{}
lFingerprint.LoadFromMetric(m)
rFingerprint.LoadFromMetric(o)
return lFingerprint.Equal(rFingerprint)
} }
// Before compares the fingerprints of both metrics. // Before compares the fingerprints of both metrics.
func (m Metric) Before(o Metric) bool { func (m Metric) Before(o Metric) bool {
lFingerprint := &Fingerprint{} return m.Fingerprint().Less(o.Fingerprint())
rFingerprint := &Fingerprint{}
lFingerprint.LoadFromMetric(m)
rFingerprint.LoadFromMetric(o)
return lFingerprint.Less(rFingerprint)
} }
func (m Metric) String() string { func (m Metric) String() string {
metricName, ok := m[MetricNameLabel] metricName, hasName := m[MetricNameLabel]
if !ok { numLabels := len(m) - 1
panic("Tried to print metric without name") if !hasName {
numLabels = len(m)
} }
labelStrings := make([]string, 0, len(m)-1) labelStrings := make([]string, 0, numLabels)
for label, value := range m { for label, value := range m {
if label != MetricNameLabel { if label != MetricNameLabel {
labelStrings = append(labelStrings, fmt.Sprintf("%s=%q", label, value)) labelStrings = append(labelStrings, fmt.Sprintf("%s=%q", label, value))
} }
} }
switch len(labelStrings) { switch numLabels {
case 0: case 0:
if hasName {
return string(metricName) return string(metricName)
} else {
return "{}"
}
default: default:
sort.Strings(labelStrings) sort.Strings(labelStrings)
return fmt.Sprintf("%s{%s}", metricName, strings.Join(labelStrings, ", ")) return fmt.Sprintf("%s{%s}", metricName, strings.Join(labelStrings, ", "))
} }
} }
func (m Metric) Fingerprint() Fingerprint {
var fp Fingerprint
fp.LoadFromMetric(m)
return fp
}
// MergeFromLabelSet merges a label set into this Metric, prefixing a collision // MergeFromLabelSet merges a label set into this Metric, prefixing a collision
// prefix to the label names merged from the label set where required. // prefix to the label names merged from the label set where required.
func (m Metric) MergeFromLabelSet(labels LabelSet, collisionPrefix LabelName) { func (m Metric) MergeFromLabelSet(labels LabelSet, collisionPrefix LabelName) {

View File

@ -17,51 +17,32 @@ import "testing"
func testMetric(t testing.TB) { func testMetric(t testing.TB) {
var scenarios = []struct { var scenarios = []struct {
input map[string]string input Metric
hash uint64 fingerprint Fingerprint
rowkey string
}{ }{
{ {
input: map[string]string{}, input: Metric{},
rowkey: "02676020557754725067--0-", fingerprint: 2676020557754725067,
hash: 2676020557754725067,
}, },
{ {
input: map[string]string{ input: Metric{
"first_name": "electro", "first_name": "electro",
"occupation": "robot", "occupation": "robot",
"manufacturer": "westinghouse", "manufacturer": "westinghouse",
}, },
rowkey: "04776841610193542734-f-6-t", fingerprint: 13260944541294022935,
hash: 4776841610193542734,
}, },
{ {
input: map[string]string{ input: Metric{
"x": "y", "x": "y",
}, },
rowkey: "01306929544689993150-x-2-y", fingerprint: 1470933794305433534,
hash: 1306929544689993150,
}, },
} }
for i, scenario := range scenarios { for i, scenario := range scenarios {
metric := Metric{} if scenario.fingerprint != scenario.input.Fingerprint() {
for key, value := range scenario.input { t.Errorf("%d. expected %d, got %d", i, scenario.fingerprint, scenario.input.Fingerprint())
metric[LabelName(key)] = LabelValue(value)
}
expectedRowKey := scenario.rowkey
expectedHash := scenario.hash
fingerprint := &Fingerprint{}
fingerprint.LoadFromMetric(metric)
actualRowKey := fingerprint.String()
actualHash := fingerprint.Hash
if expectedRowKey != actualRowKey {
t.Errorf("%d. expected %s, got %s", i, expectedRowKey, actualRowKey)
}
if actualHash != expectedHash {
t.Errorf("%d. expected %d, got %d", i, expectedHash, actualHash)
} }
} }
} }

View File

@ -1,104 +0,0 @@
// Copyright 2013 Prometheus Team
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package model
import (
"runtime"
"testing"
)
func TestFingerprintComparison(t *testing.T) {
fingerprints := []*Fingerprint{
{
Hash: 0,
FirstCharacterOfFirstLabelName: "b",
LabelMatterLength: 1,
LastCharacterOfLastLabelValue: "b",
},
{
Hash: 1,
FirstCharacterOfFirstLabelName: "a",
LabelMatterLength: 0,
LastCharacterOfLastLabelValue: "a",
},
{
Hash: 1,
FirstCharacterOfFirstLabelName: "a",
LabelMatterLength: 1000,
LastCharacterOfLastLabelValue: "b",
},
{
Hash: 1,
FirstCharacterOfFirstLabelName: "b",
LabelMatterLength: 0,
LastCharacterOfLastLabelValue: "a",
},
{
Hash: 1,
FirstCharacterOfFirstLabelName: "b",
LabelMatterLength: 1,
LastCharacterOfLastLabelValue: "a",
},
{
Hash: 1,
FirstCharacterOfFirstLabelName: "b",
LabelMatterLength: 1,
LastCharacterOfLastLabelValue: "b",
},
}
for i := range fingerprints {
if i == 0 {
continue
}
if !fingerprints[i-1].Less(fingerprints[i]) {
t.Errorf("%d expected %s < %s", i, fingerprints[i-1], fingerprints[i])
}
}
}
func BenchmarkFingerprinting(b *testing.B) {
b.StopTimer()
fps := []*Fingerprint{
{
Hash: 0,
FirstCharacterOfFirstLabelName: "a",
LabelMatterLength: 2,
LastCharacterOfLastLabelValue: "z",
},
{
Hash: 0,
FirstCharacterOfFirstLabelName: "a",
LabelMatterLength: 2,
LastCharacterOfLastLabelValue: "z",
},
}
for i := 0; i < 10; i++ {
fps[0].Less(fps[1])
}
b.Logf("N: %v", b.N)
b.StartTimer()
var pre runtime.MemStats
runtime.ReadMemStats(&pre)
for i := 0; i < b.N; i++ {
fps[0].Less(fps[1])
}
var post runtime.MemStats
runtime.ReadMemStats(&post)
b.Logf("allocs: %d items: ", post.TotalAlloc-pre.TotalAlloc)
}

View File

@ -21,42 +21,42 @@ import (
func TestSamplesSort(t *testing.T) { func TestSamplesSort(t *testing.T) {
input := Samples{ input := Samples{
&Sample{ &Sample{
// Fingerprint: 5735370332639503759 // Fingerprint: 81f9c9ed24563f8f.
Metric: Metric{ Metric: Metric{
MetricNameLabel: "A", MetricNameLabel: "A",
}, },
Timestamp: 1, Timestamp: 1,
}, },
&Sample{ &Sample{
// Fingerprint: 5735370332639503759 // Fingerprint: 81f9c9ed24563f8f.
Metric: Metric{ Metric: Metric{
MetricNameLabel: "A", MetricNameLabel: "A",
}, },
Timestamp: 2, Timestamp: 2,
}, },
&Sample{ &Sample{
// Fingerprint: 13086089349438416271 // Fingerprint: 1bf6c9ed24543f8f.
Metric: Metric{ Metric: Metric{
MetricNameLabel: "C", MetricNameLabel: "C",
}, },
Timestamp: 1, Timestamp: 1,
}, },
&Sample{ &Sample{
// Fingerprint: 13086089349438416271 // Fingerprint: 1bf6c9ed24543f8f.
Metric: Metric{ Metric: Metric{
MetricNameLabel: "C", MetricNameLabel: "C",
}, },
Timestamp: 2, Timestamp: 2,
}, },
&Sample{ &Sample{
// Fingerprint: 187498541672539535 // Fingerprint: 68f4c9ed24533f8f.
Metric: Metric{ Metric: Metric{
MetricNameLabel: "B", MetricNameLabel: "B",
}, },
Timestamp: 1, Timestamp: 1,
}, },
&Sample{ &Sample{
// Fingerprint: 187498541672539535 // Fingerprint: 68f4c9ed24533f8f.
Metric: Metric{ Metric: Metric{
MetricNameLabel: "B", MetricNameLabel: "B",
}, },
@ -66,59 +66,56 @@ func TestSamplesSort(t *testing.T) {
expected := Samples{ expected := Samples{
&Sample{ &Sample{
// Fingerprint: 187498541672539535 // Fingerprint: 1bf6c9ed24543f8f.
Metric: Metric{
MetricNameLabel: "B",
},
Timestamp: 1,
},
&Sample{
// Fingerprint: 187498541672539535
Metric: Metric{
MetricNameLabel: "B",
},
Timestamp: 2,
},
&Sample{
// Fingerprint: 5735370332639503759
Metric: Metric{
MetricNameLabel: "A",
},
Timestamp: 1,
},
&Sample{
// Fingerprint: 5735370332639503759
Metric: Metric{
MetricNameLabel: "A",
},
Timestamp: 2,
},
&Sample{
// Fingerprint: 13086089349438416271
Metric: Metric{ Metric: Metric{
MetricNameLabel: "C", MetricNameLabel: "C",
}, },
Timestamp: 1, Timestamp: 1,
}, },
&Sample{ &Sample{
// Fingerprint: 13086089349438416271 // Fingerprint: 1bf6c9ed24543f8f.
Metric: Metric{ Metric: Metric{
MetricNameLabel: "C", MetricNameLabel: "C",
}, },
Timestamp: 2, Timestamp: 2,
}, },
&Sample{
// Fingerprint: 68f4c9ed24533f8f.
Metric: Metric{
MetricNameLabel: "B",
},
Timestamp: 1,
},
&Sample{
// Fingerprint: 68f4c9ed24533f8f.
Metric: Metric{
MetricNameLabel: "B",
},
Timestamp: 2,
},
&Sample{
// Fingerprint: 81f9c9ed24563f8f.
Metric: Metric{
MetricNameLabel: "A",
},
Timestamp: 1,
},
&Sample{
// Fingerprint: 81f9c9ed24563f8f.
Metric: Metric{
MetricNameLabel: "A",
},
Timestamp: 2,
},
} }
sort.Sort(input) sort.Sort(input)
for i, actual := range input { for i, actual := range input {
actualFp := Fingerprint{} actualFp := actual.Metric.Fingerprint()
actualFp.LoadFromMetric(actual.Metric) expectedFp := expected[i].Metric.Fingerprint()
expectedFp := Fingerprint{} if !actualFp.Equal(expectedFp) {
expectedFp.LoadFromMetric(expected[i].Metric)
if !actualFp.Equal(&expectedFp) {
t.Fatalf("%d. Incorrect fingerprint. Got %s; want %s", i, actualFp.String(), expectedFp.String()) t.Fatalf("%d. Incorrect fingerprint. Got %s; want %s", i, actualFp.String(), expectedFp.String())
} }

View File

@ -30,9 +30,11 @@ type Timestamp int64
const ( const (
// MinimumTick is the minimum supported time resolution. This has to be // MinimumTick is the minimum supported time resolution. This has to be
// at least native_time.Second in order for the code below to work. // at least native_time.Second in order for the code below to work.
MinimumTick = native_time.Second MinimumTick = native_time.Millisecond
// second is the timestamp duration equivalent to one second. // second is the timestamp duration equivalent to one second.
second = int64(native_time.Second / MinimumTick) second = int64(native_time.Second / MinimumTick)
// The number of nanoseconds per minimum tick.
nanosPerTick = int64(MinimumTick / native_time.Nanosecond)
) )
// Equal reports whether two timestamps represent the same instant. // Equal reports whether two timestamps represent the same instant.
@ -62,7 +64,7 @@ func (t Timestamp) Sub(o Timestamp) native_time.Duration {
// Time returns the time.Time representation of t. // Time returns the time.Time representation of t.
func (t Timestamp) Time() native_time.Time { func (t Timestamp) Time() native_time.Time {
return native_time.Unix(int64(t)/second, (int64(t) % second)) return native_time.Unix(int64(t)/second, (int64(t)%second)*nanosPerTick)
} }
// Unix returns t as a Unix time, the number of seconds elapsed // Unix returns t as a Unix time, the number of seconds elapsed
@ -71,9 +73,19 @@ func (t Timestamp) Unix() int64 {
return int64(t) / second return int64(t) / second
} }
// UnixNano returns t as a Unix time, the number of nanoseconds elapsed
// since January 1, 1970 UTC.
func (t Timestamp) UnixNano() int64 {
return int64(t) * nanosPerTick
}
// String returns a string representation of the timestamp. // String returns a string representation of the timestamp.
func (t Timestamp) String() string { func (t Timestamp) String() string {
return fmt.Sprint(int64(t)) return fmt.Sprintf("%f", float64(t)/float64(second))
}
func (t Timestamp) MarshalJSON() ([]byte, error) {
return []byte(t.String()), nil
} }
// Now returns the current time as a Timestamp. // Now returns the current time as a Timestamp.
@ -83,10 +95,17 @@ func Now() Timestamp {
// TimestampFromTime returns the Timestamp equivalent to the time.Time t. // TimestampFromTime returns the Timestamp equivalent to the time.Time t.
func TimestampFromTime(t native_time.Time) Timestamp { func TimestampFromTime(t native_time.Time) Timestamp {
return TimestampFromUnix(t.Unix()) return TimestampFromUnixNano(t.UnixNano())
} }
// TimestampFromUnix returns the Timestamp equivalent to the Unix timestamp t. // TimestampFromUnix returns the Timestamp equivalent to the Unix timestamp t
// provided in seconds.
func TimestampFromUnix(t int64) Timestamp { func TimestampFromUnix(t int64) Timestamp {
return Timestamp(t * second) return Timestamp(t * second)
} }
// TimestampFromUnixNano returns the Timestamp equivalent to the Unix timestamp
// t provided in nanoseconds.
func TimestampFromUnixNano(t int64) Timestamp {
return Timestamp(t / nanosPerTick)
}

View File

@ -46,13 +46,16 @@ func TestComparators(t *testing.T) {
} }
func TestTimestampConversions(t *testing.T) { func TestTimestampConversions(t *testing.T) {
unix := int64(1136239445) unixSecs := int64(1136239445)
t1 := native_time.Unix(unix, 0) unixNsecs := int64(123456789)
t2 := native_time.Unix(unix, second-1) unixNano := unixSecs*1000000000 + unixNsecs
ts := TimestampFromUnix(unix) t1 := native_time.Unix(unixSecs, unixNsecs-unixNsecs%nanosPerTick)
t2 := native_time.Unix(unixSecs, unixNsecs)
ts := TimestampFromUnixNano(unixNano)
if !ts.Time().Equal(t1) { if !ts.Time().Equal(t1) {
t.Fatalf("Expected %s, got %s", t1, ts.Time()) t.Fatalf("Expected %s, got %s %d", t1, ts.Time())
} }
// Test available precision. // Test available precision.
@ -61,8 +64,8 @@ func TestTimestampConversions(t *testing.T) {
t.Fatalf("Expected %s, got %s", t1, ts.Time()) t.Fatalf("Expected %s, got %s", t1, ts.Time())
} }
if ts.Unix() != unix { if ts.UnixNano() != unixNano-unixNano%nanosPerTick {
t.Fatalf("Expected %d, got %d", unix, ts.Unix()) t.Fatalf("Expected %d, got %d", unixNano, ts.UnixNano())
} }
} }