Adjust various things required for the new storage backend.

- Change Fingerprints to be simple uint64s.
- Deal sensibly with missing metric names.
- Enable finer-grained time resolution.

Merge this concurrently with the merge of the new storage backend into
prometheus/prometheus.

Change-Id: Idd82f137aa0c4286df422c53ce3c62e0de285360

model/fingerprinting.go

@@ -19,96 +19,35 @@ import (
 	"hash/fnv"
 	"sort"
 	"strconv"
-	"strings"
 )
 
 // Fingerprint provides a hash-capable representation of a Metric.
-type Fingerprint struct {
+// For our purposes, FNV-1A 64-bit is used.
-	// A hashed representation of the underyling entity.  For our purposes, FNV-1A
+type Fingerprint uint64
-	// 64-bit is used.
+
-	Hash                           uint64
+func (f Fingerprint) String() string {
-	FirstCharacterOfFirstLabelName string
+	return fmt.Sprintf("%016x", uint64(f))
-	LabelMatterLength              uint
-	LastCharacterOfLastLabelValue  string
 }
 
-func (f *Fingerprint) String() string {
+// Less implements sort.Interface.
-	return strings.Join([]string{fmt.Sprintf("%020d", f.Hash), f.FirstCharacterOfFirstLabelName, fmt.Sprint(f.LabelMatterLength), f.LastCharacterOfLastLabelValue}, "-")
+func (f Fingerprint) Less(o Fingerprint) bool {
+	return f < o
 }
 
-// Less compares first the Hash, then the FirstCharacterOfFirstLabelName, then
+// Equal implements sort.Interface.
-// the LabelMatterLength, then the LastCharacterOfLastLabelValue.
+func (f Fingerprint) Equal(o Fingerprint) bool {
-func (f *Fingerprint) Less(o *Fingerprint) bool {
+	return f == o
-	if f.Hash < o.Hash {
-		return true
-	}
-	if f.Hash > o.Hash {
-		return false
-	}
-
-	if f.FirstCharacterOfFirstLabelName < o.FirstCharacterOfFirstLabelName {
-		return true
-	}
-	if f.FirstCharacterOfFirstLabelName > o.FirstCharacterOfFirstLabelName {
-		return false
-	}
-
-	if f.LabelMatterLength < o.LabelMatterLength {
-		return true
-	}
-	if f.LabelMatterLength > o.LabelMatterLength {
-		return false
-	}
-
-	if f.LastCharacterOfLastLabelValue < o.LastCharacterOfLastLabelValue {
-		return true
-	}
-	if f.LastCharacterOfLastLabelValue > o.LastCharacterOfLastLabelValue {
-		return false
-	}
-	return false
 }
 
-// Equal uses the same semantics as Less.
+// LoadFromString transforms a string representation into a Fingerprint.
-func (f *Fingerprint) Equal(o *Fingerprint) bool {
-	if f.Hash != o.Hash {
-		return false
-	}
-
-	if f.FirstCharacterOfFirstLabelName != o.FirstCharacterOfFirstLabelName {
-		return false
-	}
-
-	if f.LabelMatterLength != o.LabelMatterLength {
-		return false
-	}
-
-	return f.LastCharacterOfLastLabelValue == o.LastCharacterOfLastLabelValue
-}
-
-const rowKeyDelimiter = "-"
-
-// LoadFromString transforms a string representation into a Fingerprint,
-// resetting any previous attributes.
 func (f *Fingerprint) LoadFromString(s string) {
-	components := strings.Split(s, rowKeyDelimiter)
+	num, err := strconv.ParseUint(s, 16, 64)
-	hash, err := strconv.ParseUint(components[0], 10, 64)
 	if err != nil {
 		panic(err)
 	}
-	labelMatterLength, err := strconv.ParseUint(components[2], 10, 0)
+	*f = Fingerprint(num)
-	if err != nil {
-		panic(err)
-	}
-
-	f.Hash = hash
-	f.FirstCharacterOfFirstLabelName = components[1]
-	f.LabelMatterLength = uint(labelMatterLength)
-	f.LastCharacterOfLastLabelValue = components[3]
 }
 
-const reservedDelimiter = `"`
-
 // LoadFromMetric decomposes a Metric into this Fingerprint
 func (f *Fingerprint) LoadFromMetric(m Metric) {
 	labelLength := len(m)
@@ -121,46 +60,33 @@ func (f *Fingerprint) LoadFromMetric(m Metric) {
 	sort.Strings(labelNames)
 
 	summer := fnv.New64a()
-	firstCharacterOfFirstLabelName := ""
-	lastCharacterOfLastLabelValue := ""
-	labelMatterLength := 0
 
-	for i, labelName := range labelNames {
+	for _, labelName := range labelNames {
 		labelValue := m[LabelName(labelName)]
-		labelNameLength := len(labelName)
-		labelValueLength := len(labelValue)
-		labelMatterLength += labelNameLength + labelValueLength
-
-		if i == 0 {
-			firstCharacterOfFirstLabelName = labelName[0:1]
-		}
-		if i == labelLength-1 {
-			lastCharacterOfLastLabelValue = string(labelValue[labelValueLength-1 : labelValueLength])
-		}
 
 		summer.Write([]byte(labelName))
-		summer.Write([]byte(reservedDelimiter))
+		summer.Write([]byte{0})
 		summer.Write([]byte(labelValue))
 	}
 
-	f.FirstCharacterOfFirstLabelName = firstCharacterOfFirstLabelName
+	*f = Fingerprint(binary.LittleEndian.Uint64(summer.Sum(nil)))
-	f.Hash = binary.LittleEndian.Uint64(summer.Sum(nil))
-	f.LabelMatterLength = uint(labelMatterLength % 10)
-	f.LastCharacterOfLastLabelValue = lastCharacterOfLastLabelValue
 }
 
 // Fingerprints represents a collection of Fingerprint subject to a given
 // natural sorting scheme. It implements sort.Interface.
-type Fingerprints []*Fingerprint
+type Fingerprints []Fingerprint
 
+// Len implements sort.Interface.
 func (f Fingerprints) Len() int {
 	return len(f)
 }
 
+// Less implements sort.Interface.
 func (f Fingerprints) Less(i, j int) bool {
-	return f[i].Less(f[j])
+	return f[i] < f[j]
 }
 
+// Swap implements sort.Interface.
 func (f Fingerprints) Swap(i, j int) {
 	f[i], f[j] = f[j], f[i]
 }

model/metric.go

@@ -25,47 +25,46 @@ type Metric map[LabelName]LabelValue
 
 // Equal compares the fingerprints of both metrics.
 func (m Metric) Equal(o Metric) bool {
-	lFingerprint := &Fingerprint{}
+	return m.Fingerprint().Equal(o.Fingerprint())
-	rFingerprint := &Fingerprint{}
-
-	lFingerprint.LoadFromMetric(m)
-	rFingerprint.LoadFromMetric(o)
-
-	return lFingerprint.Equal(rFingerprint)
 }
 
 // Before compares the fingerprints of both metrics.
 func (m Metric) Before(o Metric) bool {
-	lFingerprint := &Fingerprint{}
+	return m.Fingerprint().Less(o.Fingerprint())
-	rFingerprint := &Fingerprint{}
-
-	lFingerprint.LoadFromMetric(m)
-	rFingerprint.LoadFromMetric(o)
-
-	return lFingerprint.Less(rFingerprint)
 }
 
 func (m Metric) String() string {
-	metricName, ok := m[MetricNameLabel]
+	metricName, hasName := m[MetricNameLabel]
-	if !ok {
+	numLabels := len(m) - 1
-		panic("Tried to print metric without name")
+	if !hasName {
+		numLabels = len(m)
 	}
-	labelStrings := make([]string, 0, len(m)-1)
+	labelStrings := make([]string, 0, numLabels)
 	for label, value := range m {
 		if label != MetricNameLabel {
 			labelStrings = append(labelStrings, fmt.Sprintf("%s=%q", label, value))
 		}
 	}
 
-	switch len(labelStrings) {
+	switch numLabels {
 	case 0:
-		return string(metricName)
+		if hasName {
+			return string(metricName)
+		} else {
+			return "{}"
+		}
 	default:
 		sort.Strings(labelStrings)
 		return fmt.Sprintf("%s{%s}", metricName, strings.Join(labelStrings, ", "))
 	}
 }
 
+func (m Metric) Fingerprint() Fingerprint {
+	var fp Fingerprint
+	fp.LoadFromMetric(m)
+	return fp
+}
+
 // MergeFromLabelSet merges a label set into this Metric, prefixing a collision
 // prefix to the label names merged from the label set where required.
 func (m Metric) MergeFromLabelSet(labels LabelSet, collisionPrefix LabelName) {

model/metric_test.go

@@ -17,51 +17,32 @@ import "testing"
 
 func testMetric(t testing.TB) {
 	var scenarios = []struct {
-		input  map[string]string
+		input       Metric
-		hash   uint64
+		fingerprint Fingerprint
-		rowkey string
 	}{
 		{
-			input:  map[string]string{},
+			input:       Metric{},
-			rowkey: "02676020557754725067--0-",
+			fingerprint: 2676020557754725067,
-			hash:   2676020557754725067,
 		},
 		{
-			input: map[string]string{
+			input: Metric{
 				"first_name":   "electro",
 				"occupation":   "robot",
 				"manufacturer": "westinghouse",
 			},
-			rowkey: "04776841610193542734-f-6-t",
+			fingerprint: 13260944541294022935,
-			hash:   4776841610193542734,
 		},
 		{
-			input: map[string]string{
+			input: Metric{
 				"x": "y",
 			},
-			rowkey: "01306929544689993150-x-2-y",
+			fingerprint: 1470933794305433534,
-			hash:   1306929544689993150,
 		},
 	}
 
 	for i, scenario := range scenarios {
-		metric := Metric{}
+		if scenario.fingerprint != scenario.input.Fingerprint() {
-		for key, value := range scenario.input {
+			t.Errorf("%d. expected %d, got %d", i, scenario.fingerprint, scenario.input.Fingerprint())
-			metric[LabelName(key)] = LabelValue(value)
-		}
-
-		expectedRowKey := scenario.rowkey
-		expectedHash := scenario.hash
-		fingerprint := &Fingerprint{}
-		fingerprint.LoadFromMetric(metric)
-		actualRowKey := fingerprint.String()
-		actualHash := fingerprint.Hash
-
-		if expectedRowKey != actualRowKey {
-			t.Errorf("%d. expected %s, got %s", i, expectedRowKey, actualRowKey)
-		}
-		if actualHash != expectedHash {
-			t.Errorf("%d. expected %d, got %d", i, expectedHash, actualHash)
 		}
 	}
 }

model/model_test.go

@@ -1,104 +0,0 @@
-// Copyright 2013 Prometheus Team
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package model
-
-import (
-	"runtime"
-	"testing"
-)
-
-func TestFingerprintComparison(t *testing.T) {
-	fingerprints := []*Fingerprint{
-		{
-			Hash: 0,
-			FirstCharacterOfFirstLabelName: "b",
-			LabelMatterLength:              1,
-			LastCharacterOfLastLabelValue:  "b",
-		},
-		{
-			Hash: 1,
-			FirstCharacterOfFirstLabelName: "a",
-			LabelMatterLength:              0,
-			LastCharacterOfLastLabelValue:  "a",
-		},
-		{
-			Hash: 1,
-			FirstCharacterOfFirstLabelName: "a",
-			LabelMatterLength:              1000,
-			LastCharacterOfLastLabelValue:  "b",
-		},
-		{
-			Hash: 1,
-			FirstCharacterOfFirstLabelName: "b",
-			LabelMatterLength:              0,
-			LastCharacterOfLastLabelValue:  "a",
-		},
-		{
-			Hash: 1,
-			FirstCharacterOfFirstLabelName: "b",
-			LabelMatterLength:              1,
-			LastCharacterOfLastLabelValue:  "a",
-		},
-		{
-			Hash: 1,
-			FirstCharacterOfFirstLabelName: "b",
-			LabelMatterLength:              1,
-			LastCharacterOfLastLabelValue:  "b",
-		},
-	}
-	for i := range fingerprints {
-		if i == 0 {
-			continue
-		}
-
-		if !fingerprints[i-1].Less(fingerprints[i]) {
-			t.Errorf("%d expected %s < %s", i, fingerprints[i-1], fingerprints[i])
-		}
-	}
-}
-
-func BenchmarkFingerprinting(b *testing.B) {
-	b.StopTimer()
-	fps := []*Fingerprint{
-		{
-			Hash: 0,
-			FirstCharacterOfFirstLabelName: "a",
-			LabelMatterLength:              2,
-			LastCharacterOfLastLabelValue:  "z",
-		},
-		{
-			Hash: 0,
-			FirstCharacterOfFirstLabelName: "a",
-			LabelMatterLength:              2,
-			LastCharacterOfLastLabelValue:  "z",
-		},
-	}
-	for i := 0; i < 10; i++ {
-		fps[0].Less(fps[1])
-	}
-	b.Logf("N: %v", b.N)
-	b.StartTimer()
-
-	var pre runtime.MemStats
-	runtime.ReadMemStats(&pre)
-
-	for i := 0; i < b.N; i++ {
-		fps[0].Less(fps[1])
-	}
-
-	var post runtime.MemStats
-	runtime.ReadMemStats(&post)
-
-	b.Logf("allocs: %d items: ", post.TotalAlloc-pre.TotalAlloc)
-}

model/sample_test.go

@@ -21,42 +21,42 @@ import (
 func TestSamplesSort(t *testing.T) {
 	input := Samples{
 		&Sample{
-			// Fingerprint: 5735370332639503759
+			// Fingerprint: 81f9c9ed24563f8f.
 			Metric: Metric{
 				MetricNameLabel: "A",
 			},
 			Timestamp: 1,
 		},
 		&Sample{
-			// Fingerprint: 5735370332639503759
+			// Fingerprint: 81f9c9ed24563f8f.
 			Metric: Metric{
 				MetricNameLabel: "A",
 			},
 			Timestamp: 2,
 		},
 		&Sample{
-			// Fingerprint: 13086089349438416271
+			// Fingerprint: 1bf6c9ed24543f8f.
 			Metric: Metric{
 				MetricNameLabel: "C",
 			},
 			Timestamp: 1,
 		},
 		&Sample{
-			// Fingerprint: 13086089349438416271
+			// Fingerprint: 1bf6c9ed24543f8f.
 			Metric: Metric{
 				MetricNameLabel: "C",
 			},
 			Timestamp: 2,
 		},
 		&Sample{
-			// Fingerprint: 187498541672539535
+			// Fingerprint: 68f4c9ed24533f8f.
 			Metric: Metric{
 				MetricNameLabel: "B",
 			},
 			Timestamp: 1,
 		},
 		&Sample{
-			// Fingerprint: 187498541672539535
+			// Fingerprint: 68f4c9ed24533f8f.
 			Metric: Metric{
 				MetricNameLabel: "B",
 			},
@@ -66,59 +66,56 @@ func TestSamplesSort(t *testing.T) {
 
 	expected := Samples{
 		&Sample{
-			// Fingerprint: 187498541672539535
+			// Fingerprint: 1bf6c9ed24543f8f.
-			Metric: Metric{
-				MetricNameLabel: "B",
-			},
-			Timestamp: 1,
-		},
-		&Sample{
-			// Fingerprint: 187498541672539535
-			Metric: Metric{
-				MetricNameLabel: "B",
-			},
-			Timestamp: 2,
-		},
-		&Sample{
-			// Fingerprint: 5735370332639503759
-			Metric: Metric{
-				MetricNameLabel: "A",
-			},
-			Timestamp: 1,
-		},
-		&Sample{
-			// Fingerprint: 5735370332639503759
-			Metric: Metric{
-				MetricNameLabel: "A",
-			},
-			Timestamp: 2,
-		},
-		&Sample{
-			// Fingerprint: 13086089349438416271
 			Metric: Metric{
 				MetricNameLabel: "C",
 			},
 			Timestamp: 1,
 		},
 		&Sample{
-			// Fingerprint: 13086089349438416271
+			// Fingerprint: 1bf6c9ed24543f8f.
 			Metric: Metric{
 				MetricNameLabel: "C",
 			},
 			Timestamp: 2,
 		},
+		&Sample{
+			// Fingerprint: 68f4c9ed24533f8f.
+			Metric: Metric{
+				MetricNameLabel: "B",
+			},
+			Timestamp: 1,
+		},
+		&Sample{
+			// Fingerprint: 68f4c9ed24533f8f.
+			Metric: Metric{
+				MetricNameLabel: "B",
+			},
+			Timestamp: 2,
+		},
+		&Sample{
+			// Fingerprint: 81f9c9ed24563f8f.
+			Metric: Metric{
+				MetricNameLabel: "A",
+			},
+			Timestamp: 1,
+		},
+		&Sample{
+			// Fingerprint: 81f9c9ed24563f8f.
+			Metric: Metric{
+				MetricNameLabel: "A",
+			},
+			Timestamp: 2,
+		},
 	}
 
 	sort.Sort(input)
 
 	for i, actual := range input {
-		actualFp := Fingerprint{}
+		actualFp := actual.Metric.Fingerprint()
-		actualFp.LoadFromMetric(actual.Metric)
+		expectedFp := expected[i].Metric.Fingerprint()
 
-		expectedFp := Fingerprint{}
+		if !actualFp.Equal(expectedFp) {
-		expectedFp.LoadFromMetric(expected[i].Metric)
-
-		if !actualFp.Equal(&expectedFp) {
 			t.Fatalf("%d. Incorrect fingerprint. Got %s; want %s", i, actualFp.String(), expectedFp.String())
 		}
 

model/timestamp.go

@@ -30,9 +30,11 @@ type Timestamp int64
 const (
 	// MinimumTick is the minimum supported time resolution. This has to be
 	// at least native_time.Second in order for the code below to work.
-	MinimumTick = native_time.Second
+	MinimumTick = native_time.Millisecond
 	// second is the timestamp duration equivalent to one second.
 	second = int64(native_time.Second / MinimumTick)
+	// The number of nanoseconds per minimum tick.
+	nanosPerTick = int64(MinimumTick / native_time.Nanosecond)
 )
 
 // Equal reports whether two timestamps represent the same instant.
@@ -62,7 +64,7 @@ func (t Timestamp) Sub(o Timestamp) native_time.Duration {
 
 // Time returns the time.Time representation of t.
 func (t Timestamp) Time() native_time.Time {
-	return native_time.Unix(int64(t)/second, (int64(t) % second))
+	return native_time.Unix(int64(t)/second, (int64(t)%second)*nanosPerTick)
 }
 
 // Unix returns t as a Unix time, the number of seconds elapsed
@@ -71,9 +73,19 @@ func (t Timestamp) Unix() int64 {
 	return int64(t) / second
 }
 
+// UnixNano returns t as a Unix time, the number of nanoseconds elapsed
+// since January 1, 1970 UTC.
+func (t Timestamp) UnixNano() int64 {
+	return int64(t) * nanosPerTick
+}
+
 // String returns a string representation of the timestamp.
 func (t Timestamp) String() string {
-	return fmt.Sprint(int64(t))
+	return fmt.Sprintf("%f", float64(t)/float64(second))
+}
+
+func (t Timestamp) MarshalJSON() ([]byte, error) {
+	return []byte(t.String()), nil
 }
 
 // Now returns the current time as a Timestamp.
@@ -83,10 +95,17 @@ func Now() Timestamp {
 
 // TimestampFromTime returns the Timestamp equivalent to the time.Time t.
 func TimestampFromTime(t native_time.Time) Timestamp {
-	return TimestampFromUnix(t.Unix())
+	return TimestampFromUnixNano(t.UnixNano())
 }
 
-// TimestampFromUnix returns the Timestamp equivalent to the Unix timestamp t.
+// TimestampFromUnix returns the Timestamp equivalent to the Unix timestamp t
+// provided in seconds.
 func TimestampFromUnix(t int64) Timestamp {
 	return Timestamp(t * second)
 }
+
+// TimestampFromUnixNano returns the Timestamp equivalent to the Unix timestamp
+// t provided in nanoseconds.
+func TimestampFromUnixNano(t int64) Timestamp {
+	return Timestamp(t / nanosPerTick)
+}

model/timestamp_test.go

@@ -46,13 +46,16 @@ func TestComparators(t *testing.T) {
 }
 
 func TestTimestampConversions(t *testing.T) {
-	unix := int64(1136239445)
+	unixSecs := int64(1136239445)
-	t1 := native_time.Unix(unix, 0)
+	unixNsecs := int64(123456789)
-	t2 := native_time.Unix(unix, second-1)
+	unixNano := unixSecs*1000000000 + unixNsecs
 
-	ts := TimestampFromUnix(unix)
+	t1 := native_time.Unix(unixSecs, unixNsecs-unixNsecs%nanosPerTick)
+	t2 := native_time.Unix(unixSecs, unixNsecs)
+
+	ts := TimestampFromUnixNano(unixNano)
 	if !ts.Time().Equal(t1) {
-		t.Fatalf("Expected %s, got %s", t1, ts.Time())
+		t.Fatalf("Expected %s, got %s %d", t1, ts.Time())
 	}
 
 	// Test available precision.
@@ -61,8 +64,8 @@ func TestTimestampConversions(t *testing.T) {
 		t.Fatalf("Expected %s, got %s", t1, ts.Time())
 	}
 
-	if ts.Unix() != unix {
+	if ts.UnixNano() != unixNano-unixNano%nanosPerTick {
-		t.Fatalf("Expected %d, got %d", unix, ts.Unix())
+		t.Fatalf("Expected %d, got %d", unixNano, ts.UnixNano())
 	}
 }