Adjust various things required for the new storage backend.

- Change Fingerprints to be simple uint64s.
- Deal sensibly with missing metric names.
- Enable finer-grained time resolution.

Merge this concurrently with the merge of the new storage backend into
prometheus/prometheus.

Change-Id: Idd82f137aa0c4286df422c53ce3c62e0de285360
This commit is contained in:
Julius Volz 2014-07-22 17:52:10 +02:00 committed by Bjoern Rabenstein
parent c53c07a719
commit 87a585def8
7 changed files with 124 additions and 303 deletions

View File

@ -19,96 +19,35 @@ import (
"hash/fnv"
"sort"
"strconv"
"strings"
)
// Fingerprint provides a hash-capable representation of a Metric.
type Fingerprint struct {
// A hashed representation of the underyling entity. For our purposes, FNV-1A
// 64-bit is used.
Hash uint64
FirstCharacterOfFirstLabelName string
LabelMatterLength uint
LastCharacterOfLastLabelValue string
// For our purposes, FNV-1A 64-bit is used.
type Fingerprint uint64
func (f Fingerprint) String() string {
return fmt.Sprintf("%016x", uint64(f))
}
func (f *Fingerprint) String() string {
return strings.Join([]string{fmt.Sprintf("%020d", f.Hash), f.FirstCharacterOfFirstLabelName, fmt.Sprint(f.LabelMatterLength), f.LastCharacterOfLastLabelValue}, "-")
// Less implements sort.Interface.
func (f Fingerprint) Less(o Fingerprint) bool {
return f < o
}
// Less compares first the Hash, then the FirstCharacterOfFirstLabelName, then
// the LabelMatterLength, then the LastCharacterOfLastLabelValue.
func (f *Fingerprint) Less(o *Fingerprint) bool {
if f.Hash < o.Hash {
return true
}
if f.Hash > o.Hash {
return false
}
if f.FirstCharacterOfFirstLabelName < o.FirstCharacterOfFirstLabelName {
return true
}
if f.FirstCharacterOfFirstLabelName > o.FirstCharacterOfFirstLabelName {
return false
}
if f.LabelMatterLength < o.LabelMatterLength {
return true
}
if f.LabelMatterLength > o.LabelMatterLength {
return false
}
if f.LastCharacterOfLastLabelValue < o.LastCharacterOfLastLabelValue {
return true
}
if f.LastCharacterOfLastLabelValue > o.LastCharacterOfLastLabelValue {
return false
}
return false
// Equal implements sort.Interface.
func (f Fingerprint) Equal(o Fingerprint) bool {
return f == o
}
// Equal uses the same semantics as Less.
func (f *Fingerprint) Equal(o *Fingerprint) bool {
if f.Hash != o.Hash {
return false
}
if f.FirstCharacterOfFirstLabelName != o.FirstCharacterOfFirstLabelName {
return false
}
if f.LabelMatterLength != o.LabelMatterLength {
return false
}
return f.LastCharacterOfLastLabelValue == o.LastCharacterOfLastLabelValue
}
const rowKeyDelimiter = "-"
// LoadFromString transforms a string representation into a Fingerprint,
// resetting any previous attributes.
// LoadFromString transforms a string representation into a Fingerprint.
func (f *Fingerprint) LoadFromString(s string) {
components := strings.Split(s, rowKeyDelimiter)
hash, err := strconv.ParseUint(components[0], 10, 64)
num, err := strconv.ParseUint(s, 16, 64)
if err != nil {
panic(err)
}
labelMatterLength, err := strconv.ParseUint(components[2], 10, 0)
if err != nil {
panic(err)
}
f.Hash = hash
f.FirstCharacterOfFirstLabelName = components[1]
f.LabelMatterLength = uint(labelMatterLength)
f.LastCharacterOfLastLabelValue = components[3]
*f = Fingerprint(num)
}
const reservedDelimiter = `"`
// LoadFromMetric decomposes a Metric into this Fingerprint
func (f *Fingerprint) LoadFromMetric(m Metric) {
labelLength := len(m)
@ -121,46 +60,33 @@ func (f *Fingerprint) LoadFromMetric(m Metric) {
sort.Strings(labelNames)
summer := fnv.New64a()
firstCharacterOfFirstLabelName := ""
lastCharacterOfLastLabelValue := ""
labelMatterLength := 0
for i, labelName := range labelNames {
for _, labelName := range labelNames {
labelValue := m[LabelName(labelName)]
labelNameLength := len(labelName)
labelValueLength := len(labelValue)
labelMatterLength += labelNameLength + labelValueLength
if i == 0 {
firstCharacterOfFirstLabelName = labelName[0:1]
}
if i == labelLength-1 {
lastCharacterOfLastLabelValue = string(labelValue[labelValueLength-1 : labelValueLength])
}
summer.Write([]byte(labelName))
summer.Write([]byte(reservedDelimiter))
summer.Write([]byte{0})
summer.Write([]byte(labelValue))
}
f.FirstCharacterOfFirstLabelName = firstCharacterOfFirstLabelName
f.Hash = binary.LittleEndian.Uint64(summer.Sum(nil))
f.LabelMatterLength = uint(labelMatterLength % 10)
f.LastCharacterOfLastLabelValue = lastCharacterOfLastLabelValue
*f = Fingerprint(binary.LittleEndian.Uint64(summer.Sum(nil)))
}
// Fingerprints represents a collection of Fingerprint subject to a given
// natural sorting scheme. It implements sort.Interface.
type Fingerprints []*Fingerprint
type Fingerprints []Fingerprint
// Len implements sort.Interface.
func (f Fingerprints) Len() int {
return len(f)
}
// Less implements sort.Interface.
func (f Fingerprints) Less(i, j int) bool {
return f[i].Less(f[j])
return f[i] < f[j]
}
// Swap implements sort.Interface.
func (f Fingerprints) Swap(i, j int) {
f[i], f[j] = f[j], f[i]
}

View File

@ -25,47 +25,46 @@ type Metric map[LabelName]LabelValue
// Equal compares the fingerprints of both metrics.
func (m Metric) Equal(o Metric) bool {
lFingerprint := &Fingerprint{}
rFingerprint := &Fingerprint{}
lFingerprint.LoadFromMetric(m)
rFingerprint.LoadFromMetric(o)
return lFingerprint.Equal(rFingerprint)
return m.Fingerprint().Equal(o.Fingerprint())
}
// Before compares the fingerprints of both metrics.
func (m Metric) Before(o Metric) bool {
lFingerprint := &Fingerprint{}
rFingerprint := &Fingerprint{}
lFingerprint.LoadFromMetric(m)
rFingerprint.LoadFromMetric(o)
return lFingerprint.Less(rFingerprint)
return m.Fingerprint().Less(o.Fingerprint())
}
func (m Metric) String() string {
metricName, ok := m[MetricNameLabel]
if !ok {
panic("Tried to print metric without name")
metricName, hasName := m[MetricNameLabel]
numLabels := len(m) - 1
if !hasName {
numLabels = len(m)
}
labelStrings := make([]string, 0, len(m)-1)
labelStrings := make([]string, 0, numLabels)
for label, value := range m {
if label != MetricNameLabel {
labelStrings = append(labelStrings, fmt.Sprintf("%s=%q", label, value))
}
}
switch len(labelStrings) {
switch numLabels {
case 0:
return string(metricName)
if hasName {
return string(metricName)
} else {
return "{}"
}
default:
sort.Strings(labelStrings)
return fmt.Sprintf("%s{%s}", metricName, strings.Join(labelStrings, ", "))
}
}
func (m Metric) Fingerprint() Fingerprint {
var fp Fingerprint
fp.LoadFromMetric(m)
return fp
}
// MergeFromLabelSet merges a label set into this Metric, prefixing a collision
// prefix to the label names merged from the label set where required.
func (m Metric) MergeFromLabelSet(labels LabelSet, collisionPrefix LabelName) {

View File

@ -17,51 +17,32 @@ import "testing"
func testMetric(t testing.TB) {
var scenarios = []struct {
input map[string]string
hash uint64
rowkey string
input Metric
fingerprint Fingerprint
}{
{
input: map[string]string{},
rowkey: "02676020557754725067--0-",
hash: 2676020557754725067,
input: Metric{},
fingerprint: 2676020557754725067,
},
{
input: map[string]string{
input: Metric{
"first_name": "electro",
"occupation": "robot",
"manufacturer": "westinghouse",
},
rowkey: "04776841610193542734-f-6-t",
hash: 4776841610193542734,
fingerprint: 13260944541294022935,
},
{
input: map[string]string{
input: Metric{
"x": "y",
},
rowkey: "01306929544689993150-x-2-y",
hash: 1306929544689993150,
fingerprint: 1470933794305433534,
},
}
for i, scenario := range scenarios {
metric := Metric{}
for key, value := range scenario.input {
metric[LabelName(key)] = LabelValue(value)
}
expectedRowKey := scenario.rowkey
expectedHash := scenario.hash
fingerprint := &Fingerprint{}
fingerprint.LoadFromMetric(metric)
actualRowKey := fingerprint.String()
actualHash := fingerprint.Hash
if expectedRowKey != actualRowKey {
t.Errorf("%d. expected %s, got %s", i, expectedRowKey, actualRowKey)
}
if actualHash != expectedHash {
t.Errorf("%d. expected %d, got %d", i, expectedHash, actualHash)
if scenario.fingerprint != scenario.input.Fingerprint() {
t.Errorf("%d. expected %d, got %d", i, scenario.fingerprint, scenario.input.Fingerprint())
}
}
}

View File

@ -1,104 +0,0 @@
// Copyright 2013 Prometheus Team
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package model
import (
"runtime"
"testing"
)
func TestFingerprintComparison(t *testing.T) {
fingerprints := []*Fingerprint{
{
Hash: 0,
FirstCharacterOfFirstLabelName: "b",
LabelMatterLength: 1,
LastCharacterOfLastLabelValue: "b",
},
{
Hash: 1,
FirstCharacterOfFirstLabelName: "a",
LabelMatterLength: 0,
LastCharacterOfLastLabelValue: "a",
},
{
Hash: 1,
FirstCharacterOfFirstLabelName: "a",
LabelMatterLength: 1000,
LastCharacterOfLastLabelValue: "b",
},
{
Hash: 1,
FirstCharacterOfFirstLabelName: "b",
LabelMatterLength: 0,
LastCharacterOfLastLabelValue: "a",
},
{
Hash: 1,
FirstCharacterOfFirstLabelName: "b",
LabelMatterLength: 1,
LastCharacterOfLastLabelValue: "a",
},
{
Hash: 1,
FirstCharacterOfFirstLabelName: "b",
LabelMatterLength: 1,
LastCharacterOfLastLabelValue: "b",
},
}
for i := range fingerprints {
if i == 0 {
continue
}
if !fingerprints[i-1].Less(fingerprints[i]) {
t.Errorf("%d expected %s < %s", i, fingerprints[i-1], fingerprints[i])
}
}
}
func BenchmarkFingerprinting(b *testing.B) {
b.StopTimer()
fps := []*Fingerprint{
{
Hash: 0,
FirstCharacterOfFirstLabelName: "a",
LabelMatterLength: 2,
LastCharacterOfLastLabelValue: "z",
},
{
Hash: 0,
FirstCharacterOfFirstLabelName: "a",
LabelMatterLength: 2,
LastCharacterOfLastLabelValue: "z",
},
}
for i := 0; i < 10; i++ {
fps[0].Less(fps[1])
}
b.Logf("N: %v", b.N)
b.StartTimer()
var pre runtime.MemStats
runtime.ReadMemStats(&pre)
for i := 0; i < b.N; i++ {
fps[0].Less(fps[1])
}
var post runtime.MemStats
runtime.ReadMemStats(&post)
b.Logf("allocs: %d items: ", post.TotalAlloc-pre.TotalAlloc)
}

View File

@ -21,42 +21,42 @@ import (
func TestSamplesSort(t *testing.T) {
input := Samples{
&Sample{
// Fingerprint: 5735370332639503759
// Fingerprint: 81f9c9ed24563f8f.
Metric: Metric{
MetricNameLabel: "A",
},
Timestamp: 1,
},
&Sample{
// Fingerprint: 5735370332639503759
// Fingerprint: 81f9c9ed24563f8f.
Metric: Metric{
MetricNameLabel: "A",
},
Timestamp: 2,
},
&Sample{
// Fingerprint: 13086089349438416271
// Fingerprint: 1bf6c9ed24543f8f.
Metric: Metric{
MetricNameLabel: "C",
},
Timestamp: 1,
},
&Sample{
// Fingerprint: 13086089349438416271
// Fingerprint: 1bf6c9ed24543f8f.
Metric: Metric{
MetricNameLabel: "C",
},
Timestamp: 2,
},
&Sample{
// Fingerprint: 187498541672539535
// Fingerprint: 68f4c9ed24533f8f.
Metric: Metric{
MetricNameLabel: "B",
},
Timestamp: 1,
},
&Sample{
// Fingerprint: 187498541672539535
// Fingerprint: 68f4c9ed24533f8f.
Metric: Metric{
MetricNameLabel: "B",
},
@ -66,59 +66,56 @@ func TestSamplesSort(t *testing.T) {
expected := Samples{
&Sample{
// Fingerprint: 187498541672539535
Metric: Metric{
MetricNameLabel: "B",
},
Timestamp: 1,
},
&Sample{
// Fingerprint: 187498541672539535
Metric: Metric{
MetricNameLabel: "B",
},
Timestamp: 2,
},
&Sample{
// Fingerprint: 5735370332639503759
Metric: Metric{
MetricNameLabel: "A",
},
Timestamp: 1,
},
&Sample{
// Fingerprint: 5735370332639503759
Metric: Metric{
MetricNameLabel: "A",
},
Timestamp: 2,
},
&Sample{
// Fingerprint: 13086089349438416271
// Fingerprint: 1bf6c9ed24543f8f.
Metric: Metric{
MetricNameLabel: "C",
},
Timestamp: 1,
},
&Sample{
// Fingerprint: 13086089349438416271
// Fingerprint: 1bf6c9ed24543f8f.
Metric: Metric{
MetricNameLabel: "C",
},
Timestamp: 2,
},
&Sample{
// Fingerprint: 68f4c9ed24533f8f.
Metric: Metric{
MetricNameLabel: "B",
},
Timestamp: 1,
},
&Sample{
// Fingerprint: 68f4c9ed24533f8f.
Metric: Metric{
MetricNameLabel: "B",
},
Timestamp: 2,
},
&Sample{
// Fingerprint: 81f9c9ed24563f8f.
Metric: Metric{
MetricNameLabel: "A",
},
Timestamp: 1,
},
&Sample{
// Fingerprint: 81f9c9ed24563f8f.
Metric: Metric{
MetricNameLabel: "A",
},
Timestamp: 2,
},
}
sort.Sort(input)
for i, actual := range input {
actualFp := Fingerprint{}
actualFp.LoadFromMetric(actual.Metric)
actualFp := actual.Metric.Fingerprint()
expectedFp := expected[i].Metric.Fingerprint()
expectedFp := Fingerprint{}
expectedFp.LoadFromMetric(expected[i].Metric)
if !actualFp.Equal(&expectedFp) {
if !actualFp.Equal(expectedFp) {
t.Fatalf("%d. Incorrect fingerprint. Got %s; want %s", i, actualFp.String(), expectedFp.String())
}

View File

@ -30,9 +30,11 @@ type Timestamp int64
const (
// MinimumTick is the minimum supported time resolution. This has to be
// at least native_time.Second in order for the code below to work.
MinimumTick = native_time.Second
MinimumTick = native_time.Millisecond
// second is the timestamp duration equivalent to one second.
second = int64(native_time.Second / MinimumTick)
// The number of nanoseconds per minimum tick.
nanosPerTick = int64(MinimumTick / native_time.Nanosecond)
)
// Equal reports whether two timestamps represent the same instant.
@ -62,7 +64,7 @@ func (t Timestamp) Sub(o Timestamp) native_time.Duration {
// Time returns the time.Time representation of t.
func (t Timestamp) Time() native_time.Time {
return native_time.Unix(int64(t)/second, (int64(t) % second))
return native_time.Unix(int64(t)/second, (int64(t)%second)*nanosPerTick)
}
// Unix returns t as a Unix time, the number of seconds elapsed
@ -71,9 +73,19 @@ func (t Timestamp) Unix() int64 {
return int64(t) / second
}
// UnixNano returns t as a Unix time, the number of nanoseconds elapsed
// since January 1, 1970 UTC.
func (t Timestamp) UnixNano() int64 {
return int64(t) * nanosPerTick
}
// String returns a string representation of the timestamp.
func (t Timestamp) String() string {
return fmt.Sprint(int64(t))
return fmt.Sprintf("%f", float64(t)/float64(second))
}
func (t Timestamp) MarshalJSON() ([]byte, error) {
return []byte(t.String()), nil
}
// Now returns the current time as a Timestamp.
@ -83,10 +95,17 @@ func Now() Timestamp {
// TimestampFromTime returns the Timestamp equivalent to the time.Time t.
func TimestampFromTime(t native_time.Time) Timestamp {
return TimestampFromUnix(t.Unix())
return TimestampFromUnixNano(t.UnixNano())
}
// TimestampFromUnix returns the Timestamp equivalent to the Unix timestamp t.
// TimestampFromUnix returns the Timestamp equivalent to the Unix timestamp t
// provided in seconds.
func TimestampFromUnix(t int64) Timestamp {
return Timestamp(t * second)
}
// TimestampFromUnixNano returns the Timestamp equivalent to the Unix timestamp
// t provided in nanoseconds.
func TimestampFromUnixNano(t int64) Timestamp {
return Timestamp(t / nanosPerTick)
}

View File

@ -46,13 +46,16 @@ func TestComparators(t *testing.T) {
}
func TestTimestampConversions(t *testing.T) {
unix := int64(1136239445)
t1 := native_time.Unix(unix, 0)
t2 := native_time.Unix(unix, second-1)
unixSecs := int64(1136239445)
unixNsecs := int64(123456789)
unixNano := unixSecs*1000000000 + unixNsecs
ts := TimestampFromUnix(unix)
t1 := native_time.Unix(unixSecs, unixNsecs-unixNsecs%nanosPerTick)
t2 := native_time.Unix(unixSecs, unixNsecs)
ts := TimestampFromUnixNano(unixNano)
if !ts.Time().Equal(t1) {
t.Fatalf("Expected %s, got %s", t1, ts.Time())
t.Fatalf("Expected %s, got %s %d", t1, ts.Time())
}
// Test available precision.
@ -61,8 +64,8 @@ func TestTimestampConversions(t *testing.T) {
t.Fatalf("Expected %s, got %s", t1, ts.Time())
}
if ts.Unix() != unix {
t.Fatalf("Expected %d, got %d", unix, ts.Unix())
if ts.UnixNano() != unixNano-unixNano%nanosPerTick {
t.Fatalf("Expected %d, got %d", unixNano, ts.UnixNano())
}
}