From 87a585def8ac647af7054f11a1fb505a151f3198 Mon Sep 17 00:00:00 2001 From: Julius Volz Date: Tue, 22 Jul 2014 17:52:10 +0200 Subject: [PATCH] Adjust various things required for the new storage backend. - Change Fingerprints to be simple uint64s. - Deal sensibly with missing metric names. - Enable finer-grained time resolution. Merge this concurrently with the merge of the new storage backend into prometheus/prometheus. Change-Id: Idd82f137aa0c4286df422c53ce3c62e0de285360 --- model/fingerprinting.go | 118 ++++++++-------------------------------- model/metric.go | 39 +++++++------ model/metric_test.go | 39 ++++--------- model/model_test.go | 104 ----------------------------------- model/sample_test.go | 81 +++++++++++++-------------- model/timestamp.go | 29 ++++++++-- model/timestamp_test.go | 17 +++--- 7 files changed, 124 insertions(+), 303 deletions(-) delete mode 100644 model/model_test.go diff --git a/model/fingerprinting.go b/model/fingerprinting.go index 313ff30..670756e 100644 --- a/model/fingerprinting.go +++ b/model/fingerprinting.go @@ -19,96 +19,35 @@ import ( "hash/fnv" "sort" "strconv" - "strings" ) // Fingerprint provides a hash-capable representation of a Metric. -type Fingerprint struct { - // A hashed representation of the underyling entity. For our purposes, FNV-1A - // 64-bit is used. - Hash uint64 - FirstCharacterOfFirstLabelName string - LabelMatterLength uint - LastCharacterOfLastLabelValue string +// For our purposes, FNV-1A 64-bit is used. +type Fingerprint uint64 + +func (f Fingerprint) String() string { + return fmt.Sprintf("%016x", uint64(f)) } -func (f *Fingerprint) String() string { - return strings.Join([]string{fmt.Sprintf("%020d", f.Hash), f.FirstCharacterOfFirstLabelName, fmt.Sprint(f.LabelMatterLength), f.LastCharacterOfLastLabelValue}, "-") +// Less implements sort.Interface. +func (f Fingerprint) Less(o Fingerprint) bool { + return f < o } -// Less compares first the Hash, then the FirstCharacterOfFirstLabelName, then -// the LabelMatterLength, then the LastCharacterOfLastLabelValue. -func (f *Fingerprint) Less(o *Fingerprint) bool { - if f.Hash < o.Hash { - return true - } - if f.Hash > o.Hash { - return false - } - - if f.FirstCharacterOfFirstLabelName < o.FirstCharacterOfFirstLabelName { - return true - } - if f.FirstCharacterOfFirstLabelName > o.FirstCharacterOfFirstLabelName { - return false - } - - if f.LabelMatterLength < o.LabelMatterLength { - return true - } - if f.LabelMatterLength > o.LabelMatterLength { - return false - } - - if f.LastCharacterOfLastLabelValue < o.LastCharacterOfLastLabelValue { - return true - } - if f.LastCharacterOfLastLabelValue > o.LastCharacterOfLastLabelValue { - return false - } - return false +// Equal implements sort.Interface. +func (f Fingerprint) Equal(o Fingerprint) bool { + return f == o } -// Equal uses the same semantics as Less. -func (f *Fingerprint) Equal(o *Fingerprint) bool { - if f.Hash != o.Hash { - return false - } - - if f.FirstCharacterOfFirstLabelName != o.FirstCharacterOfFirstLabelName { - return false - } - - if f.LabelMatterLength != o.LabelMatterLength { - return false - } - - return f.LastCharacterOfLastLabelValue == o.LastCharacterOfLastLabelValue -} - -const rowKeyDelimiter = "-" - -// LoadFromString transforms a string representation into a Fingerprint, -// resetting any previous attributes. +// LoadFromString transforms a string representation into a Fingerprint. func (f *Fingerprint) LoadFromString(s string) { - components := strings.Split(s, rowKeyDelimiter) - hash, err := strconv.ParseUint(components[0], 10, 64) + num, err := strconv.ParseUint(s, 16, 64) if err != nil { panic(err) } - labelMatterLength, err := strconv.ParseUint(components[2], 10, 0) - if err != nil { - panic(err) - } - - f.Hash = hash - f.FirstCharacterOfFirstLabelName = components[1] - f.LabelMatterLength = uint(labelMatterLength) - f.LastCharacterOfLastLabelValue = components[3] + *f = Fingerprint(num) } -const reservedDelimiter = `"` - // LoadFromMetric decomposes a Metric into this Fingerprint func (f *Fingerprint) LoadFromMetric(m Metric) { labelLength := len(m) @@ -121,46 +60,33 @@ func (f *Fingerprint) LoadFromMetric(m Metric) { sort.Strings(labelNames) summer := fnv.New64a() - firstCharacterOfFirstLabelName := "" - lastCharacterOfLastLabelValue := "" - labelMatterLength := 0 - for i, labelName := range labelNames { + for _, labelName := range labelNames { labelValue := m[LabelName(labelName)] - labelNameLength := len(labelName) - labelValueLength := len(labelValue) - labelMatterLength += labelNameLength + labelValueLength - - if i == 0 { - firstCharacterOfFirstLabelName = labelName[0:1] - } - if i == labelLength-1 { - lastCharacterOfLastLabelValue = string(labelValue[labelValueLength-1 : labelValueLength]) - } summer.Write([]byte(labelName)) - summer.Write([]byte(reservedDelimiter)) + summer.Write([]byte{0}) summer.Write([]byte(labelValue)) } - f.FirstCharacterOfFirstLabelName = firstCharacterOfFirstLabelName - f.Hash = binary.LittleEndian.Uint64(summer.Sum(nil)) - f.LabelMatterLength = uint(labelMatterLength % 10) - f.LastCharacterOfLastLabelValue = lastCharacterOfLastLabelValue + *f = Fingerprint(binary.LittleEndian.Uint64(summer.Sum(nil))) } // Fingerprints represents a collection of Fingerprint subject to a given // natural sorting scheme. It implements sort.Interface. -type Fingerprints []*Fingerprint +type Fingerprints []Fingerprint +// Len implements sort.Interface. func (f Fingerprints) Len() int { return len(f) } +// Less implements sort.Interface. func (f Fingerprints) Less(i, j int) bool { - return f[i].Less(f[j]) + return f[i] < f[j] } +// Swap implements sort.Interface. func (f Fingerprints) Swap(i, j int) { f[i], f[j] = f[j], f[i] } diff --git a/model/metric.go b/model/metric.go index 3379f95..d3671a1 100644 --- a/model/metric.go +++ b/model/metric.go @@ -25,47 +25,46 @@ type Metric map[LabelName]LabelValue // Equal compares the fingerprints of both metrics. func (m Metric) Equal(o Metric) bool { - lFingerprint := &Fingerprint{} - rFingerprint := &Fingerprint{} - - lFingerprint.LoadFromMetric(m) - rFingerprint.LoadFromMetric(o) - - return lFingerprint.Equal(rFingerprint) + return m.Fingerprint().Equal(o.Fingerprint()) } // Before compares the fingerprints of both metrics. func (m Metric) Before(o Metric) bool { - lFingerprint := &Fingerprint{} - rFingerprint := &Fingerprint{} - - lFingerprint.LoadFromMetric(m) - rFingerprint.LoadFromMetric(o) - - return lFingerprint.Less(rFingerprint) + return m.Fingerprint().Less(o.Fingerprint()) } func (m Metric) String() string { - metricName, ok := m[MetricNameLabel] - if !ok { - panic("Tried to print metric without name") + metricName, hasName := m[MetricNameLabel] + numLabels := len(m) - 1 + if !hasName { + numLabels = len(m) } - labelStrings := make([]string, 0, len(m)-1) + labelStrings := make([]string, 0, numLabels) for label, value := range m { if label != MetricNameLabel { labelStrings = append(labelStrings, fmt.Sprintf("%s=%q", label, value)) } } - switch len(labelStrings) { + switch numLabels { case 0: - return string(metricName) + if hasName { + return string(metricName) + } else { + return "{}" + } default: sort.Strings(labelStrings) return fmt.Sprintf("%s{%s}", metricName, strings.Join(labelStrings, ", ")) } } +func (m Metric) Fingerprint() Fingerprint { + var fp Fingerprint + fp.LoadFromMetric(m) + return fp +} + // MergeFromLabelSet merges a label set into this Metric, prefixing a collision // prefix to the label names merged from the label set where required. func (m Metric) MergeFromLabelSet(labels LabelSet, collisionPrefix LabelName) { diff --git a/model/metric_test.go b/model/metric_test.go index 2d8eedd..db1b4a7 100644 --- a/model/metric_test.go +++ b/model/metric_test.go @@ -17,51 +17,32 @@ import "testing" func testMetric(t testing.TB) { var scenarios = []struct { - input map[string]string - hash uint64 - rowkey string + input Metric + fingerprint Fingerprint }{ { - input: map[string]string{}, - rowkey: "02676020557754725067--0-", - hash: 2676020557754725067, + input: Metric{}, + fingerprint: 2676020557754725067, }, { - input: map[string]string{ + input: Metric{ "first_name": "electro", "occupation": "robot", "manufacturer": "westinghouse", }, - rowkey: "04776841610193542734-f-6-t", - hash: 4776841610193542734, + fingerprint: 13260944541294022935, }, { - input: map[string]string{ + input: Metric{ "x": "y", }, - rowkey: "01306929544689993150-x-2-y", - hash: 1306929544689993150, + fingerprint: 1470933794305433534, }, } for i, scenario := range scenarios { - metric := Metric{} - for key, value := range scenario.input { - metric[LabelName(key)] = LabelValue(value) - } - - expectedRowKey := scenario.rowkey - expectedHash := scenario.hash - fingerprint := &Fingerprint{} - fingerprint.LoadFromMetric(metric) - actualRowKey := fingerprint.String() - actualHash := fingerprint.Hash - - if expectedRowKey != actualRowKey { - t.Errorf("%d. expected %s, got %s", i, expectedRowKey, actualRowKey) - } - if actualHash != expectedHash { - t.Errorf("%d. expected %d, got %d", i, expectedHash, actualHash) + if scenario.fingerprint != scenario.input.Fingerprint() { + t.Errorf("%d. expected %d, got %d", i, scenario.fingerprint, scenario.input.Fingerprint()) } } } diff --git a/model/model_test.go b/model/model_test.go deleted file mode 100644 index e9d77b9..0000000 --- a/model/model_test.go +++ /dev/null @@ -1,104 +0,0 @@ -// Copyright 2013 Prometheus Team -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package model - -import ( - "runtime" - "testing" -) - -func TestFingerprintComparison(t *testing.T) { - fingerprints := []*Fingerprint{ - { - Hash: 0, - FirstCharacterOfFirstLabelName: "b", - LabelMatterLength: 1, - LastCharacterOfLastLabelValue: "b", - }, - { - Hash: 1, - FirstCharacterOfFirstLabelName: "a", - LabelMatterLength: 0, - LastCharacterOfLastLabelValue: "a", - }, - { - Hash: 1, - FirstCharacterOfFirstLabelName: "a", - LabelMatterLength: 1000, - LastCharacterOfLastLabelValue: "b", - }, - { - Hash: 1, - FirstCharacterOfFirstLabelName: "b", - LabelMatterLength: 0, - LastCharacterOfLastLabelValue: "a", - }, - { - Hash: 1, - FirstCharacterOfFirstLabelName: "b", - LabelMatterLength: 1, - LastCharacterOfLastLabelValue: "a", - }, - { - Hash: 1, - FirstCharacterOfFirstLabelName: "b", - LabelMatterLength: 1, - LastCharacterOfLastLabelValue: "b", - }, - } - for i := range fingerprints { - if i == 0 { - continue - } - - if !fingerprints[i-1].Less(fingerprints[i]) { - t.Errorf("%d expected %s < %s", i, fingerprints[i-1], fingerprints[i]) - } - } -} - -func BenchmarkFingerprinting(b *testing.B) { - b.StopTimer() - fps := []*Fingerprint{ - { - Hash: 0, - FirstCharacterOfFirstLabelName: "a", - LabelMatterLength: 2, - LastCharacterOfLastLabelValue: "z", - }, - { - Hash: 0, - FirstCharacterOfFirstLabelName: "a", - LabelMatterLength: 2, - LastCharacterOfLastLabelValue: "z", - }, - } - for i := 0; i < 10; i++ { - fps[0].Less(fps[1]) - } - b.Logf("N: %v", b.N) - b.StartTimer() - - var pre runtime.MemStats - runtime.ReadMemStats(&pre) - - for i := 0; i < b.N; i++ { - fps[0].Less(fps[1]) - } - - var post runtime.MemStats - runtime.ReadMemStats(&post) - - b.Logf("allocs: %d items: ", post.TotalAlloc-pre.TotalAlloc) -} diff --git a/model/sample_test.go b/model/sample_test.go index fbb4f0c..3cc5650 100644 --- a/model/sample_test.go +++ b/model/sample_test.go @@ -21,42 +21,42 @@ import ( func TestSamplesSort(t *testing.T) { input := Samples{ &Sample{ - // Fingerprint: 5735370332639503759 + // Fingerprint: 81f9c9ed24563f8f. Metric: Metric{ MetricNameLabel: "A", }, Timestamp: 1, }, &Sample{ - // Fingerprint: 5735370332639503759 + // Fingerprint: 81f9c9ed24563f8f. Metric: Metric{ MetricNameLabel: "A", }, Timestamp: 2, }, &Sample{ - // Fingerprint: 13086089349438416271 + // Fingerprint: 1bf6c9ed24543f8f. Metric: Metric{ MetricNameLabel: "C", }, Timestamp: 1, }, &Sample{ - // Fingerprint: 13086089349438416271 + // Fingerprint: 1bf6c9ed24543f8f. Metric: Metric{ MetricNameLabel: "C", }, Timestamp: 2, }, &Sample{ - // Fingerprint: 187498541672539535 + // Fingerprint: 68f4c9ed24533f8f. Metric: Metric{ MetricNameLabel: "B", }, Timestamp: 1, }, &Sample{ - // Fingerprint: 187498541672539535 + // Fingerprint: 68f4c9ed24533f8f. Metric: Metric{ MetricNameLabel: "B", }, @@ -66,59 +66,56 @@ func TestSamplesSort(t *testing.T) { expected := Samples{ &Sample{ - // Fingerprint: 187498541672539535 - Metric: Metric{ - MetricNameLabel: "B", - }, - Timestamp: 1, - }, - &Sample{ - // Fingerprint: 187498541672539535 - Metric: Metric{ - MetricNameLabel: "B", - }, - Timestamp: 2, - }, - &Sample{ - // Fingerprint: 5735370332639503759 - Metric: Metric{ - MetricNameLabel: "A", - }, - Timestamp: 1, - }, - &Sample{ - // Fingerprint: 5735370332639503759 - Metric: Metric{ - MetricNameLabel: "A", - }, - Timestamp: 2, - }, - &Sample{ - // Fingerprint: 13086089349438416271 + // Fingerprint: 1bf6c9ed24543f8f. Metric: Metric{ MetricNameLabel: "C", }, Timestamp: 1, }, &Sample{ - // Fingerprint: 13086089349438416271 + // Fingerprint: 1bf6c9ed24543f8f. Metric: Metric{ MetricNameLabel: "C", }, Timestamp: 2, }, + &Sample{ + // Fingerprint: 68f4c9ed24533f8f. + Metric: Metric{ + MetricNameLabel: "B", + }, + Timestamp: 1, + }, + &Sample{ + // Fingerprint: 68f4c9ed24533f8f. + Metric: Metric{ + MetricNameLabel: "B", + }, + Timestamp: 2, + }, + &Sample{ + // Fingerprint: 81f9c9ed24563f8f. + Metric: Metric{ + MetricNameLabel: "A", + }, + Timestamp: 1, + }, + &Sample{ + // Fingerprint: 81f9c9ed24563f8f. + Metric: Metric{ + MetricNameLabel: "A", + }, + Timestamp: 2, + }, } sort.Sort(input) for i, actual := range input { - actualFp := Fingerprint{} - actualFp.LoadFromMetric(actual.Metric) + actualFp := actual.Metric.Fingerprint() + expectedFp := expected[i].Metric.Fingerprint() - expectedFp := Fingerprint{} - expectedFp.LoadFromMetric(expected[i].Metric) - - if !actualFp.Equal(&expectedFp) { + if !actualFp.Equal(expectedFp) { t.Fatalf("%d. Incorrect fingerprint. Got %s; want %s", i, actualFp.String(), expectedFp.String()) } diff --git a/model/timestamp.go b/model/timestamp.go index a12478f..e0c6dd4 100644 --- a/model/timestamp.go +++ b/model/timestamp.go @@ -30,9 +30,11 @@ type Timestamp int64 const ( // MinimumTick is the minimum supported time resolution. This has to be // at least native_time.Second in order for the code below to work. - MinimumTick = native_time.Second + MinimumTick = native_time.Millisecond // second is the timestamp duration equivalent to one second. second = int64(native_time.Second / MinimumTick) + // The number of nanoseconds per minimum tick. + nanosPerTick = int64(MinimumTick / native_time.Nanosecond) ) // Equal reports whether two timestamps represent the same instant. @@ -62,7 +64,7 @@ func (t Timestamp) Sub(o Timestamp) native_time.Duration { // Time returns the time.Time representation of t. func (t Timestamp) Time() native_time.Time { - return native_time.Unix(int64(t)/second, (int64(t) % second)) + return native_time.Unix(int64(t)/second, (int64(t)%second)*nanosPerTick) } // Unix returns t as a Unix time, the number of seconds elapsed @@ -71,9 +73,19 @@ func (t Timestamp) Unix() int64 { return int64(t) / second } +// UnixNano returns t as a Unix time, the number of nanoseconds elapsed +// since January 1, 1970 UTC. +func (t Timestamp) UnixNano() int64 { + return int64(t) * nanosPerTick +} + // String returns a string representation of the timestamp. func (t Timestamp) String() string { - return fmt.Sprint(int64(t)) + return fmt.Sprintf("%f", float64(t)/float64(second)) +} + +func (t Timestamp) MarshalJSON() ([]byte, error) { + return []byte(t.String()), nil } // Now returns the current time as a Timestamp. @@ -83,10 +95,17 @@ func Now() Timestamp { // TimestampFromTime returns the Timestamp equivalent to the time.Time t. func TimestampFromTime(t native_time.Time) Timestamp { - return TimestampFromUnix(t.Unix()) + return TimestampFromUnixNano(t.UnixNano()) } -// TimestampFromUnix returns the Timestamp equivalent to the Unix timestamp t. +// TimestampFromUnix returns the Timestamp equivalent to the Unix timestamp t +// provided in seconds. func TimestampFromUnix(t int64) Timestamp { return Timestamp(t * second) } + +// TimestampFromUnixNano returns the Timestamp equivalent to the Unix timestamp +// t provided in nanoseconds. +func TimestampFromUnixNano(t int64) Timestamp { + return Timestamp(t / nanosPerTick) +} diff --git a/model/timestamp_test.go b/model/timestamp_test.go index 12d97bd..f146a7b 100644 --- a/model/timestamp_test.go +++ b/model/timestamp_test.go @@ -46,13 +46,16 @@ func TestComparators(t *testing.T) { } func TestTimestampConversions(t *testing.T) { - unix := int64(1136239445) - t1 := native_time.Unix(unix, 0) - t2 := native_time.Unix(unix, second-1) + unixSecs := int64(1136239445) + unixNsecs := int64(123456789) + unixNano := unixSecs*1000000000 + unixNsecs - ts := TimestampFromUnix(unix) + t1 := native_time.Unix(unixSecs, unixNsecs-unixNsecs%nanosPerTick) + t2 := native_time.Unix(unixSecs, unixNsecs) + + ts := TimestampFromUnixNano(unixNano) if !ts.Time().Equal(t1) { - t.Fatalf("Expected %s, got %s", t1, ts.Time()) + t.Fatalf("Expected %s, got %s %d", t1, ts.Time()) } // Test available precision. @@ -61,8 +64,8 @@ func TestTimestampConversions(t *testing.T) { t.Fatalf("Expected %s, got %s", t1, ts.Time()) } - if ts.Unix() != unix { - t.Fatalf("Expected %d, got %d", unix, ts.Unix()) + if ts.UnixNano() != unixNano-unixNano%nanosPerTick { + t.Fatalf("Expected %d, got %d", unixNano, ts.UnixNano()) } }