Bump prometheus/client_model (#1323)

By upgrading prometheus/client_model, several test functions had to be re-written due to 2 breaking changes made in protobuf when parsing messages to text:
1. '<' and '>' characters were replaced with '{' and '}' respectively.
2. The text format is non-deterministic. More information in https://github.com/golang/protobuf/issues/1121

Signed-off-by: Arthur Silva Sens <arthur.sens@coralogix.com>

go.mod

@@ -7,7 +7,7 @@ require (
 	github.com/cespare/xxhash/v2 v2.2.0
 	github.com/davecgh/go-spew v1.1.1
 	github.com/json-iterator/go v1.1.12
-	github.com/prometheus/client_model v0.3.0
+	github.com/prometheus/client_model v0.4.0
 	github.com/prometheus/common v0.42.0
 	github.com/prometheus/procfs v0.11.1
 	golang.org/x/sys v0.10.0

go.sum

@@ -151,8 +151,9 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
 github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4=
 github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w=
+github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY=
+github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU=
 github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA=
 github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM=
 github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc=

prometheus/counter_test.go

@@ -61,8 +61,15 @@ func TestCounterAdd(t *testing.T) {
 	m := &dto.Metric{}
 	counter.Write(m)
 
-	if expected, got := `label:<name:"a" value:"1" > label:<name:"b" value:"2" > counter:<value:67.42 > `, m.String(); expected != got {
+	expected := &dto.Metric{
-		t.Errorf("expected %q, got %q", expected, got)
+		Label: []*dto.LabelPair{
+			{Name: proto.String("a"), Value: proto.String("1")},
+			{Name: proto.String("b"), Value: proto.String("2")},
+		},
+		Counter: &dto.Counter{Value: proto.Float64(67.42)},
+	}
+	if !proto.Equal(expected, m) {
+		t.Errorf("expected %q, got %q", expected, m)
 	}
 }
 
@@ -164,8 +171,14 @@ func TestCounterAddInf(t *testing.T) {
 	m := &dto.Metric{}
 	counter.Write(m)
 
-	if expected, got := `counter:<value:inf > `, m.String(); expected != got {
+	expected := &dto.Metric{
-		t.Errorf("expected %q, got %q", expected, got)
+		Counter: &dto.Counter{
+			Value: proto.Float64(math.Inf(1)),
+		},
+	}
+
+	if !proto.Equal(expected, m) {
+		t.Errorf("expected %q, got %q", expected, m)
 	}
 }
 
@@ -188,8 +201,14 @@ func TestCounterAddLarge(t *testing.T) {
 	m := &dto.Metric{}
 	counter.Write(m)
 
-	if expected, got := fmt.Sprintf("counter:<value:%0.16e > ", large), m.String(); expected != got {
+	expected := &dto.Metric{
-		t.Errorf("expected %q, got %q", expected, got)
+		Counter: &dto.Counter{
+			Value: proto.Float64(large),
+		},
+	}
+
+	if !proto.Equal(expected, m) {
+		t.Errorf("expected %q, got %q", expected, m)
 	}
 }
 
@@ -210,8 +229,14 @@ func TestCounterAddSmall(t *testing.T) {
 	m := &dto.Metric{}
 	counter.Write(m)
 
-	if expected, got := fmt.Sprintf("counter:<value:%0.0e > ", small), m.String(); expected != got {
+	expected := &dto.Metric{
-		t.Errorf("expected %q, got %q", expected, got)
+		Counter: &dto.Counter{
+			Value: proto.Float64(small),
+		},
+	}
+
+	if !proto.Equal(expected, m) {
+		t.Errorf("expected %q, got %q", expected, m)
 	}
 }
 

prometheus/example_metricvec_test.go

@@ -14,8 +14,6 @@
 package prometheus_test
 
 import (
-	"fmt"
-
 	"google.golang.org/protobuf/proto"
 
 	dto "github.com/prometheus/client_model/go"
@@ -126,8 +124,8 @@ func ExampleMetricVec() {
 	if err != nil || len(metricFamilies) != 1 {
 		panic("unexpected behavior of custom test registry")
 	}
-	fmt.Println(metricFamilies[0].String())
+	printlnNormalized(metricFamilies[0])
 
 	// Output:
-	// name:"library_version_info" help:"Versions of the libraries used in this binary." type:GAUGE metric:<label:<name:"library" value:"k8s.io/client-go" > label:<name:"version" value:"0.18.8" > gauge:<value:1 > > metric:<label:<name:"library" value:"prometheus/client_golang" > label:<name:"version" value:"1.7.1" > gauge:<value:1 > >
+	// {"name":"library_version_info","help":"Versions of the libraries used in this binary.","type":"GAUGE","metric":[{"label":[{"name":"library","value":"k8s.io/client-go"},{"name":"version","value":"0.18.8"}],"gauge":{"value":1}},{"label":[{"name":"library","value":"prometheus/client_golang"},{"name":"version","value":"1.7.1"}],"gauge":{"value":1}}]}
 }

prometheus/examples_test.go

@@ -319,10 +319,11 @@ func ExampleSummary() {
 	// internally).
 	metric := &dto.Metric{}
 	temps.Write(metric)
-	fmt.Println(metric.String())
+
+	printlnNormalized(metric)
 
 	// Output:
-	// summary:<sample_count:1000 sample_sum:29969.50000000001 quantile:<quantile:0.5 value:31.1 > quantile:<quantile:0.9 value:41.3 > quantile:<quantile:0.99 value:41.9 > >
+	// {"summary":{"sampleCount":"1000","sampleSum":29969.50000000001,"quantile":[{"quantile":0.5,"value":31.1},{"quantile":0.9,"value":41.3},{"quantile":0.99,"value":41.9}]}}
 }
 
 func ExampleSummaryVec() {
@@ -354,10 +355,10 @@ func ExampleSummaryVec() {
 	if err != nil || len(metricFamilies) != 1 {
 		panic("unexpected behavior of custom test registry")
 	}
-	fmt.Println(metricFamilies[0].String())
+	printlnNormalized(metricFamilies[0])
 
 	// Output:
-	// name:"pond_temperature_celsius" help:"The temperature of the frog pond." type:SUMMARY metric:<label:<name:"species" value:"leiopelma-hochstetteri" > summary:<sample_count:0 sample_sum:0 quantile:<quantile:0.5 value:nan > quantile:<quantile:0.9 value:nan > quantile:<quantile:0.99 value:nan > > > metric:<label:<name:"species" value:"lithobates-catesbeianus" > summary:<sample_count:1000 sample_sum:31956.100000000017 quantile:<quantile:0.5 value:32.4 > quantile:<quantile:0.9 value:41.4 > quantile:<quantile:0.99 value:41.9 > > > metric:<label:<name:"species" value:"litoria-caerulea" > summary:<sample_count:1000 sample_sum:29969.50000000001 quantile:<quantile:0.5 value:31.1 > quantile:<quantile:0.9 value:41.3 > quantile:<quantile:0.99 value:41.9 > > >
+	// {"name":"pond_temperature_celsius","help":"The temperature of the frog pond.","type":"SUMMARY","metric":[{"label":[{"name":"species","value":"leiopelma-hochstetteri"}],"summary":{"sampleCount":"0","sampleSum":0,"quantile":[{"quantile":0.5,"value":"NaN"},{"quantile":0.9,"value":"NaN"},{"quantile":0.99,"value":"NaN"}]}},{"label":[{"name":"species","value":"lithobates-catesbeianus"}],"summary":{"sampleCount":"1000","sampleSum":31956.100000000017,"quantile":[{"quantile":0.5,"value":32.4},{"quantile":0.9,"value":41.4},{"quantile":0.99,"value":41.9}]}},{"label":[{"name":"species","value":"litoria-caerulea"}],"summary":{"sampleCount":"1000","sampleSum":29969.50000000001,"quantile":[{"quantile":0.5,"value":31.1},{"quantile":0.9,"value":41.3},{"quantile":0.99,"value":41.9}]}}]}
 }
 
 func ExampleNewConstSummary() {
@@ -381,10 +382,10 @@ func ExampleNewConstSummary() {
 	// internally).
 	metric := &dto.Metric{}
 	s.Write(metric)
-	fmt.Println(metric.String())
+	printlnNormalized(metric)
 
 	// Output:
-	// label:<name:"code" value:"200" > label:<name:"method" value:"get" > label:<name:"owner" value:"example" > summary:<sample_count:4711 sample_sum:403.34 quantile:<quantile:0.5 value:42.3 > quantile:<quantile:0.9 value:323.3 > >
+	// {"label":[{"name":"code","value":"200"},{"name":"method","value":"get"},{"name":"owner","value":"example"}],"summary":{"sampleCount":"4711","sampleSum":403.34,"quantile":[{"quantile":0.5,"value":42.3},{"quantile":0.9,"value":323.3}]}}
 }
 
 func ExampleHistogram() {
@@ -404,10 +405,10 @@ func ExampleHistogram() {
 	// internally).
 	metric := &dto.Metric{}
 	temps.Write(metric)
-	fmt.Println(metric.String())
+	printlnNormalized(metric)
 
 	// Output:
-	// histogram:<sample_count:1000 sample_sum:29969.50000000001 bucket:<cumulative_count:192 upper_bound:20 > bucket:<cumulative_count:366 upper_bound:25 > bucket:<cumulative_count:501 upper_bound:30 > bucket:<cumulative_count:638 upper_bound:35 > bucket:<cumulative_count:816 upper_bound:40 > >
+	// {"histogram":{"sampleCount":"1000","sampleSum":29969.50000000001,"bucket":[{"cumulativeCount":"192","upperBound":20},{"cumulativeCount":"366","upperBound":25},{"cumulativeCount":"501","upperBound":30},{"cumulativeCount":"638","upperBound":35},{"cumulativeCount":"816","upperBound":40}]}}
 }
 
 func ExampleNewConstHistogram() {
@@ -431,10 +432,10 @@ func ExampleNewConstHistogram() {
 	// internally).
 	metric := &dto.Metric{}
 	h.Write(metric)
-	fmt.Println(metric.String())
+	printlnNormalized(metric)
 
 	// Output:
-	// label:<name:"code" value:"200" > label:<name:"method" value:"get" > label:<name:"owner" value:"example" > histogram:<sample_count:4711 sample_sum:403.34 bucket:<cumulative_count:121 upper_bound:25 > bucket:<cumulative_count:2403 upper_bound:50 > bucket:<cumulative_count:3221 upper_bound:100 > bucket:<cumulative_count:4233 upper_bound:200 > >
+	// {"label":[{"name":"code","value":"200"},{"name":"method","value":"get"},{"name":"owner","value":"example"}],"histogram":{"sampleCount":"4711","sampleSum":403.34,"bucket":[{"cumulativeCount":"121","upperBound":25},{"cumulativeCount":"2403","upperBound":50},{"cumulativeCount":"3221","upperBound":100},{"cumulativeCount":"4233","upperBound":200}]}}
 }
 
 func ExampleNewConstHistogram_WithExemplar() {
@@ -469,10 +470,10 @@ func ExampleNewConstHistogram_WithExemplar() {
 	// internally).
 	metric := &dto.Metric{}
 	h.Write(metric)
-	fmt.Println(metric.String())
+	printlnNormalized(metric)
 
 	// Output:
-	// label:<name:"code" value:"200" > label:<name:"method" value:"get" > label:<name:"owner" value:"example" > histogram:<sample_count:4711 sample_sum:403.34 bucket:<cumulative_count:121 upper_bound:25 exemplar:<label:<name:"testName" value:"testVal" > value:24 timestamp:<seconds:1136214245 > > > bucket:<cumulative_count:2403 upper_bound:50 exemplar:<label:<name:"testName" value:"testVal" > value:42 timestamp:<seconds:1136214245 > > > bucket:<cumulative_count:3221 upper_bound:100 exemplar:<label:<name:"testName" value:"testVal" > value:89 timestamp:<seconds:1136214245 > > > bucket:<cumulative_count:4233 upper_bound:200 exemplar:<label:<name:"testName" value:"testVal" > value:157 timestamp:<seconds:1136214245 > > > >
+	// {"label":[{"name":"code","value":"200"},{"name":"method","value":"get"},{"name":"owner","value":"example"}],"histogram":{"sampleCount":"4711","sampleSum":403.34,"bucket":[{"cumulativeCount":"121","upperBound":25,"exemplar":{"label":[{"name":"testName","value":"testVal"}],"value":24,"timestamp":"2006-01-02T15:04:05Z"}},{"cumulativeCount":"2403","upperBound":50,"exemplar":{"label":[{"name":"testName","value":"testVal"}],"value":42,"timestamp":"2006-01-02T15:04:05Z"}},{"cumulativeCount":"3221","upperBound":100,"exemplar":{"label":[{"name":"testName","value":"testVal"}],"value":89,"timestamp":"2006-01-02T15:04:05Z"}},{"cumulativeCount":"4233","upperBound":200,"exemplar":{"label":[{"name":"testName","value":"testVal"}],"value":157,"timestamp":"2006-01-02T15:04:05Z"}}]}}
 }
 
 func ExampleAlreadyRegisteredError() {
@@ -567,7 +568,13 @@ temperature_kelvin 4.5
 
 	gathering, err = gatherers.Gather()
 	if err != nil {
-		fmt.Println(err)
+		// We expect error collected metric "temperature_kelvin" { label:<name:"location" value:"outside" > gauge:<value:265.3 > } was collected before with the same name and label values
+		// We cannot assert it because of https://github.com/golang/protobuf/issues/1121
+		if strings.HasPrefix(err.Error(), `collected metric "temperature_kelvin" `) {
+			fmt.Println("Found duplicated metric `temperature_kelvin`")
+		} else {
+			fmt.Print(err)
+		}
 	}
 	// Note that still as many metrics as possible are returned:
 	out.Reset()
@@ -589,7 +596,7 @@ temperature_kelvin 4.5
 	// temperature_kelvin{location="outside"} 273.14
 	// temperature_kelvin{location="somewhere else"} 4.5
 	// ----------
-	// collected metric "temperature_kelvin" { label:<name:"location" value:"outside" > gauge:<value:265.3 > } was collected before with the same name and label values
+	// Found duplicated metric `temperature_kelvin`
 	// # HELP humidity_percent Humidity in %.
 	// # TYPE humidity_percent gauge
 	// humidity_percent{location="inside"} 33.2
@@ -625,8 +632,8 @@ func ExampleNewMetricWithTimestamp() {
 	// internally).
 	metric := &dto.Metric{}
 	s.Write(metric)
-	fmt.Println(metric.String())
+	printlnNormalized(metric)
 
 	// Output:
-	// gauge:<value:298.15 > timestamp_ms:1257894000012
+	// {"gauge":{"value":298.15},"timestampMs":"1257894000012"}
 }

prometheus/expvar_collector_test.go

@@ -81,17 +81,17 @@ func ExampleNewExpvarCollector() {
 		if !strings.Contains(m.Desc().String(), "expvar_memstats") {
 			metric.Reset()
 			m.Write(&metric)
-			metricStrings = append(metricStrings, metric.String())
+			metricStrings = append(metricStrings, protoToNormalizedJSON(&metric))
 		}
 	}
 	sort.Strings(metricStrings)
 	for _, s := range metricStrings {
-		fmt.Println(strings.TrimRight(s, " "))
+		fmt.Println(s)
 	}
 	// Output:
-	// label:<name:"code" value:"200" > label:<name:"method" value:"GET" > untyped:<value:212 >
+	// {"label":[{"name":"code","value":"200"},{"name":"method","value":"GET"}],"untyped":{"value":212}}
-	// label:<name:"code" value:"200" > label:<name:"method" value:"POST" > untyped:<value:11 >
+	// {"label":[{"name":"code","value":"200"},{"name":"method","value":"POST"}],"untyped":{"value":11}}
-	// label:<name:"code" value:"404" > label:<name:"method" value:"GET" > untyped:<value:13 >
+	// {"label":[{"name":"code","value":"404"},{"name":"method","value":"GET"}],"untyped":{"value":13}}
-	// label:<name:"code" value:"404" > label:<name:"method" value:"POST" > untyped:<value:3 >
+	// {"label":[{"name":"code","value":"404"},{"name":"method","value":"POST"}],"untyped":{"value":3}}
-	// untyped:<value:42 >
+	// {"untyped":{"value":42}}
 }

prometheus/gauge_test.go

@@ -22,6 +22,7 @@ import (
 	"time"
 
 	dto "github.com/prometheus/client_model/go"
+	"google.golang.org/protobuf/proto"
 )
 
 func listenGaugeStream(vals, result chan float64, done chan struct{}) {
@@ -177,8 +178,18 @@ func TestGaugeFunc(t *testing.T) {
 	m := &dto.Metric{}
 	gf.Write(m)
 
-	if expected, got := `label:<name:"a" value:"1" > label:<name:"b" value:"2" > gauge:<value:3.1415 > `, m.String(); expected != got {
+	expected := &dto.Metric{
-		t.Errorf("expected %q, got %q", expected, got)
+		Label: []*dto.LabelPair{
+			{Name: proto.String("a"), Value: proto.String("1")},
+			{Name: proto.String("b"), Value: proto.String("2")},
+		},
+		Gauge: &dto.Gauge{
+			Value: proto.Float64(3.1415),
+		},
+	}
+
+	if !proto.Equal(expected, m) {
+		t.Errorf("expected %q, got %q", expected, m)
 	}
 }
 

prometheus/histogram_test.go

@@ -477,36 +477,89 @@ func TestNativeHistogram(t *testing.T) {
 		maxBuckets       uint32
 		minResetDuration time.Duration
 		maxZeroThreshold float64
-		want             string // String representation of protobuf.
+		want             *dto.Histogram
 	}{
 		{
 			name:         "no sparse buckets",
 			observations: []float64{1, 2, 3},
 			factor:       1,
-			want:         `sample_count:3 sample_sum:6 bucket:<cumulative_count:0 upper_bound:0.005 > bucket:<cumulative_count:0 upper_bound:0.01 > bucket:<cumulative_count:0 upper_bound:0.025 > bucket:<cumulative_count:0 upper_bound:0.05 > bucket:<cumulative_count:0 upper_bound:0.1 > bucket:<cumulative_count:0 upper_bound:0.25 > bucket:<cumulative_count:0 upper_bound:0.5 > bucket:<cumulative_count:1 upper_bound:1 > bucket:<cumulative_count:2 upper_bound:2.5 > bucket:<cumulative_count:3 upper_bound:5 > bucket:<cumulative_count:3 upper_bound:10 > `, // Has conventional buckets because there are no sparse buckets.
+			want: &dto.Histogram{
+				SampleCount: proto.Uint64(3),
+				SampleSum:   proto.Float64(6),
+				Bucket: []*dto.Bucket{
+					{CumulativeCount: proto.Uint64(0), UpperBound: proto.Float64(0.005)},
+					{CumulativeCount: proto.Uint64(0), UpperBound: proto.Float64(0.01)},
+					{CumulativeCount: proto.Uint64(0), UpperBound: proto.Float64(0.025)},
+					{CumulativeCount: proto.Uint64(0), UpperBound: proto.Float64(0.05)},
+					{CumulativeCount: proto.Uint64(0), UpperBound: proto.Float64(0.1)},
+					{CumulativeCount: proto.Uint64(0), UpperBound: proto.Float64(0.25)},
+					{CumulativeCount: proto.Uint64(0), UpperBound: proto.Float64(0.5)},
+					{CumulativeCount: proto.Uint64(1), UpperBound: proto.Float64(1)},
+					{CumulativeCount: proto.Uint64(2), UpperBound: proto.Float64(2.5)},
+					{CumulativeCount: proto.Uint64(3), UpperBound: proto.Float64(5)},
+					{CumulativeCount: proto.Uint64(3), UpperBound: proto.Float64(10)},
+				},
+			},
 		},
 		{
 			name:   "no observations",
 			factor: 1.1,
-			want:   `sample_count:0 sample_sum:0 schema:3 zero_threshold:2.938735877055719e-39 zero_count:0 `,
+			want: &dto.Histogram{
+				SampleCount:   proto.Uint64(0),
+				SampleSum:     proto.Float64(0),
+				Schema:        proto.Int32(3),
+				ZeroThreshold: proto.Float64(2.938735877055719e-39),
+				ZeroCount:     proto.Uint64(0),
+			},
 		},
 		{
 			name:          "no observations and zero threshold of zero resulting in no-op span",
 			factor:        1.1,
 			zeroThreshold: NativeHistogramZeroThresholdZero,
-			want:          `sample_count:0 sample_sum:0 schema:3 zero_threshold:0 zero_count:0 positive_span:<offset:0 length:0 > `,
+			want: &dto.Histogram{
+				SampleCount:   proto.Uint64(0),
+				SampleSum:     proto.Float64(0),
+				Schema:        proto.Int32(3),
+				ZeroThreshold: proto.Float64(0),
+				ZeroCount:     proto.Uint64(0),
+				PositiveSpan: []*dto.BucketSpan{
+					{Offset: proto.Int32(0), Length: proto.Uint32(0)},
+				},
+			},
 		},
 		{
 			name:         "factor 1.1 results in schema 3",
 			observations: []float64{0, 1, 2, 3},
 			factor:       1.1,
-			want:         `sample_count:4 sample_sum:6 schema:3 zero_threshold:2.938735877055719e-39 zero_count:1 positive_span:<offset:0 length:1 > positive_span:<offset:7 length:1 > positive_span:<offset:4 length:1 > positive_delta:1 positive_delta:0 positive_delta:0 `,
+			want: &dto.Histogram{
+				SampleCount:   proto.Uint64(4),
+				SampleSum:     proto.Float64(6),
+				Schema:        proto.Int32(3),
+				ZeroThreshold: proto.Float64(2.938735877055719e-39),
+				ZeroCount:     proto.Uint64(1),
+				PositiveSpan: []*dto.BucketSpan{
+					{Offset: proto.Int32(0), Length: proto.Uint32(1)},
+					{Offset: proto.Int32(7), Length: proto.Uint32(1)},
+					{Offset: proto.Int32(4), Length: proto.Uint32(1)},
+				},
+				PositiveDelta: []int64{1, 0, 0},
+			},
 		},
 		{
 			name:         "factor 1.2 results in schema 2",
 			observations: []float64{0, 1, 1.2, 1.4, 1.8, 2},
 			factor:       1.2,
-			want:         `sample_count:6 sample_sum:7.4 schema:2 zero_threshold:2.938735877055719e-39 zero_count:1 positive_span:<offset:0 length:5 > positive_delta:1 positive_delta:-1 positive_delta:2 positive_delta:-2 positive_delta:2 `,
+			want: &dto.Histogram{
+				SampleCount:   proto.Uint64(6),
+				SampleSum:     proto.Float64(7.4),
+				Schema:        proto.Int32(2),
+				ZeroThreshold: proto.Float64(2.938735877055719e-39),
+				ZeroCount:     proto.Uint64(1),
+				PositiveSpan: []*dto.BucketSpan{
+					{Offset: proto.Int32(0), Length: proto.Uint32(5)},
+				},
+				PositiveDelta: []int64{1, -1, 2, -2, 2},
+			},
 		},
 		{
 			name: "factor 4 results in schema -1",
@@ -519,7 +572,17 @@ func TestNativeHistogram(t *testing.T) {
 				33.33, // Bucket 3: (16, 64]
 			},
 			factor: 4,
-			want:   `sample_count:14 sample_sum:63.2581251 schema:-1 zero_threshold:2.938735877055719e-39 zero_count:0 positive_span:<offset:-2 length:6 > positive_delta:2 positive_delta:0 positive_delta:0 positive_delta:2 positive_delta:-1 positive_delta:-2 `,
+			want: &dto.Histogram{
+				SampleCount:   proto.Uint64(14),
+				SampleSum:     proto.Float64(63.2581251),
+				Schema:        proto.Int32(-1),
+				ZeroThreshold: proto.Float64(2.938735877055719e-39),
+				ZeroCount:     proto.Uint64(0),
+				PositiveSpan: []*dto.BucketSpan{
+					{Offset: proto.Int32(-2), Length: proto.Uint32(6)},
+				},
+				PositiveDelta: []int64{2, 0, 0, 2, -1, -2},
+			},
 		},
 		{
 			name: "factor 17 results in schema -2",
@@ -530,58 +593,161 @@ func TestNativeHistogram(t *testing.T) {
 				33.33, // Bucket 2: (16, 256]
 			},
 			factor: 17,
-			want:   `sample_count:14 sample_sum:63.2581251 schema:-2 zero_threshold:2.938735877055719e-39 zero_count:0 positive_span:<offset:-1 length:4 > positive_delta:2 positive_delta:2 positive_delta:3 positive_delta:-6 `,
+			want: &dto.Histogram{
+				SampleCount:   proto.Uint64(14),
+				SampleSum:     proto.Float64(63.2581251),
+				Schema:        proto.Int32(-2),
+				ZeroThreshold: proto.Float64(2.938735877055719e-39),
+				ZeroCount:     proto.Uint64(0),
+				PositiveSpan: []*dto.BucketSpan{
+					{Offset: proto.Int32(-1), Length: proto.Uint32(4)},
+				},
+				PositiveDelta: []int64{2, 2, 3, -6},
+			},
 		},
 		{
 			name:         "negative buckets",
 			observations: []float64{0, -1, -1.2, -1.4, -1.8, -2},
 			factor:       1.2,
-			want:         `sample_count:6 sample_sum:-7.4 schema:2 zero_threshold:2.938735877055719e-39 zero_count:1 negative_span:<offset:0 length:5 > negative_delta:1 negative_delta:-1 negative_delta:2 negative_delta:-2 negative_delta:2 `,
+			want: &dto.Histogram{
+				SampleCount:   proto.Uint64(6),
+				SampleSum:     proto.Float64(-7.4),
+				Schema:        proto.Int32(2),
+				ZeroThreshold: proto.Float64(2.938735877055719e-39),
+				ZeroCount:     proto.Uint64(1),
+				NegativeSpan: []*dto.BucketSpan{
+					{Offset: proto.Int32(0), Length: proto.Uint32(5)},
+				},
+				NegativeDelta: []int64{1, -1, 2, -2, 2},
+			},
 		},
 		{
 			name:         "negative and positive buckets",
 			observations: []float64{0, -1, -1.2, -1.4, -1.8, -2, 1, 1.2, 1.4, 1.8, 2},
 			factor:       1.2,
-			want:         `sample_count:11 sample_sum:0 schema:2 zero_threshold:2.938735877055719e-39 zero_count:1 negative_span:<offset:0 length:5 > negative_delta:1 negative_delta:-1 negative_delta:2 negative_delta:-2 negative_delta:2 positive_span:<offset:0 length:5 > positive_delta:1 positive_delta:-1 positive_delta:2 positive_delta:-2 positive_delta:2 `,
+			want: &dto.Histogram{
+				SampleCount:   proto.Uint64(11),
+				SampleSum:     proto.Float64(0),
+				Schema:        proto.Int32(2),
+				ZeroThreshold: proto.Float64(2.938735877055719e-39),
+				ZeroCount:     proto.Uint64(1),
+				NegativeSpan: []*dto.BucketSpan{
+					{Offset: proto.Int32(0), Length: proto.Uint32(5)},
+				},
+				NegativeDelta: []int64{1, -1, 2, -2, 2},
+				PositiveSpan: []*dto.BucketSpan{
+					{Offset: proto.Int32(0), Length: proto.Uint32(5)},
+				},
+				PositiveDelta: []int64{1, -1, 2, -2, 2},
+			},
 		},
 		{
 			name:          "wide zero bucket",
 			observations:  []float64{0, -1, -1.2, -1.4, -1.8, -2, 1, 1.2, 1.4, 1.8, 2},
 			factor:        1.2,
 			zeroThreshold: 1.4,
-			want:          `sample_count:11 sample_sum:0 schema:2 zero_threshold:1.4 zero_count:7 negative_span:<offset:4 length:1 > negative_delta:2 positive_span:<offset:4 length:1 > positive_delta:2 `,
+			want: &dto.Histogram{
+				SampleCount:   proto.Uint64(11),
+				SampleSum:     proto.Float64(0),
+				Schema:        proto.Int32(2),
+				ZeroThreshold: proto.Float64(1.4),
+				ZeroCount:     proto.Uint64(7),
+				NegativeSpan: []*dto.BucketSpan{
+					{Offset: proto.Int32(4), Length: proto.Uint32(1)},
+				},
+				NegativeDelta: []int64{2},
+				PositiveSpan: []*dto.BucketSpan{
+					{Offset: proto.Int32(4), Length: proto.Uint32(1)},
+				},
+				PositiveDelta: []int64{2},
+			},
 		},
 		{
 			name:         "NaN observation",
 			observations: []float64{0, 1, 1.2, 1.4, 1.8, 2, math.NaN()},
 			factor:       1.2,
-			want:         `sample_count:7 sample_sum:nan schema:2 zero_threshold:2.938735877055719e-39 zero_count:1 positive_span:<offset:0 length:5 > positive_delta:1 positive_delta:-1 positive_delta:2 positive_delta:-2 positive_delta:2 `,
+			want: &dto.Histogram{
+				SampleCount:   proto.Uint64(7),
+				SampleSum:     proto.Float64(math.NaN()),
+				Schema:        proto.Int32(2),
+				ZeroThreshold: proto.Float64(2.938735877055719e-39),
+				ZeroCount:     proto.Uint64(1),
+				PositiveSpan: []*dto.BucketSpan{
+					{Offset: proto.Int32(0), Length: proto.Uint32(5)},
+				},
+				PositiveDelta: []int64{1, -1, 2, -2, 2},
+			},
 		},
 		{
 			name:         "+Inf observation",
 			observations: []float64{0, 1, 1.2, 1.4, 1.8, 2, math.Inf(+1)},
 			factor:       1.2,
-			want:         `sample_count:7 sample_sum:inf schema:2 zero_threshold:2.938735877055719e-39 zero_count:1 positive_span:<offset:0 length:5 > positive_span:<offset:4092 length:1 > positive_delta:1 positive_delta:-1 positive_delta:2 positive_delta:-2 positive_delta:2 positive_delta:-1 `,
+			want: &dto.Histogram{
+				SampleCount:   proto.Uint64(7),
+				SampleSum:     proto.Float64(math.Inf(+1)),
+				Schema:        proto.Int32(2),
+				ZeroThreshold: proto.Float64(2.938735877055719e-39),
+				ZeroCount:     proto.Uint64(1),
+				PositiveSpan: []*dto.BucketSpan{
+					{Offset: proto.Int32(0), Length: proto.Uint32(5)},
+					{Offset: proto.Int32(4092), Length: proto.Uint32(1)},
+				},
+				PositiveDelta: []int64{1, -1, 2, -2, 2, -1},
+			},
 		},
 		{
 			name:         "-Inf observation",
 			observations: []float64{0, 1, 1.2, 1.4, 1.8, 2, math.Inf(-1)},
 			factor:       1.2,
-			want:         `sample_count:7 sample_sum:-inf schema:2 zero_threshold:2.938735877055719e-39 zero_count:1 negative_span:<offset:4097 length:1 > negative_delta:1 positive_span:<offset:0 length:5 > positive_delta:1 positive_delta:-1 positive_delta:2 positive_delta:-2 positive_delta:2 `,
+			want: &dto.Histogram{
+				SampleCount:   proto.Uint64(7),
+				SampleSum:     proto.Float64(math.Inf(-1)),
+				Schema:        proto.Int32(2),
+				ZeroThreshold: proto.Float64(2.938735877055719e-39),
+				ZeroCount:     proto.Uint64(1),
+				NegativeSpan: []*dto.BucketSpan{
+					{Offset: proto.Int32(4097), Length: proto.Uint32(1)},
+				},
+				NegativeDelta: []int64{1},
+				PositiveSpan: []*dto.BucketSpan{
+					{Offset: proto.Int32(0), Length: proto.Uint32(5)},
+				},
+				PositiveDelta: []int64{1, -1, 2, -2, 2},
+			},
 		},
 		{
 			name:         "limited buckets but nothing triggered",
 			observations: []float64{0, 1, 1.2, 1.4, 1.8, 2},
 			factor:       1.2,
 			maxBuckets:   4,
-			want:         `sample_count:6 sample_sum:7.4 schema:2 zero_threshold:2.938735877055719e-39 zero_count:1 positive_span:<offset:0 length:5 > positive_delta:1 positive_delta:-1 positive_delta:2 positive_delta:-2 positive_delta:2 `,
+			want: &dto.Histogram{
+				SampleCount:   proto.Uint64(6),
+				SampleSum:     proto.Float64(7.4),
+				Schema:        proto.Int32(2),
+				ZeroThreshold: proto.Float64(2.938735877055719e-39),
+				ZeroCount:     proto.Uint64(1),
+				PositiveSpan: []*dto.BucketSpan{
+					{Offset: proto.Int32(0), Length: proto.Uint32(5)},
+				},
+				PositiveDelta: []int64{1, -1, 2, -2, 2},
+			},
 		},
 		{
 			name:         "buckets limited by halving resolution",
 			observations: []float64{0, 1, 1.1, 1.2, 1.4, 1.8, 2, 3},
 			factor:       1.2,
 			maxBuckets:   4,
-			want:         `sample_count:8 sample_sum:11.5 schema:1 zero_threshold:2.938735877055719e-39 zero_count:1 positive_span:<offset:0 length:5 > positive_delta:1 positive_delta:2 positive_delta:-1 positive_delta:-2 positive_delta:1 `,
+			want: &dto.Histogram{
+				SampleCount:   proto.Uint64(8),
+				SampleSum:     proto.Float64(11.5),
+				Schema:        proto.Int32(1),
+				ZeroThreshold: proto.Float64(2.938735877055719e-39),
+				ZeroCount:     proto.Uint64(1),
+				PositiveSpan: []*dto.BucketSpan{
+					{Offset: proto.Int32(0), Length: proto.Uint32(5)},
+				},
+				PositiveDelta: []int64{1, 2, -1, -2, 1},
+			},
 		},
 		{
 			name:             "buckets limited by widening the zero bucket",
@@ -589,7 +755,17 @@ func TestNativeHistogram(t *testing.T) {
 			factor:           1.2,
 			maxBuckets:       4,
 			maxZeroThreshold: 1.2,
-			want:             `sample_count:8 sample_sum:11.5 schema:2 zero_threshold:1 zero_count:2 positive_span:<offset:1 length:7 > positive_delta:1 positive_delta:1 positive_delta:-2 positive_delta:2 positive_delta:-2 positive_delta:0 positive_delta:1 `,
+			want: &dto.Histogram{
+				SampleCount:   proto.Uint64(8),
+				SampleSum:     proto.Float64(11.5),
+				Schema:        proto.Int32(2),
+				ZeroThreshold: proto.Float64(1),
+				ZeroCount:     proto.Uint64(2),
+				PositiveSpan: []*dto.BucketSpan{
+					{Offset: proto.Int32(1), Length: proto.Uint32(7)},
+				},
+				PositiveDelta: []int64{1, 1, -2, 2, -2, 0, 1},
+			},
 		},
 		{
 			name:             "buckets limited by widening the zero bucket twice",
@@ -597,7 +773,17 @@ func TestNativeHistogram(t *testing.T) {
 			factor:           1.2,
 			maxBuckets:       4,
 			maxZeroThreshold: 1.2,
-			want:             `sample_count:9 sample_sum:15.5 schema:2 zero_threshold:1.189207115002721 zero_count:3 positive_span:<offset:2 length:7 > positive_delta:2 positive_delta:-2 positive_delta:2 positive_delta:-2 positive_delta:0 positive_delta:1 positive_delta:0 `,
+			want: &dto.Histogram{
+				SampleCount:   proto.Uint64(9),
+				SampleSum:     proto.Float64(15.5),
+				Schema:        proto.Int32(2),
+				ZeroThreshold: proto.Float64(1.189207115002721),
+				ZeroCount:     proto.Uint64(3),
+				PositiveSpan: []*dto.BucketSpan{
+					{Offset: proto.Int32(2), Length: proto.Uint32(7)},
+				},
+				PositiveDelta: []int64{2, -2, 2, -2, 0, 1, 0},
+			},
 		},
 		{
 			name:             "buckets limited by reset",
@@ -606,21 +792,51 @@ func TestNativeHistogram(t *testing.T) {
 			maxBuckets:       4,
 			maxZeroThreshold: 1.2,
 			minResetDuration: 5 * time.Minute,
-			want:             `sample_count:2 sample_sum:7 schema:2 zero_threshold:2.938735877055719e-39 zero_count:0 positive_span:<offset:7 length:2 > positive_delta:1 positive_delta:0 `,
+			want: &dto.Histogram{
+				SampleCount:   proto.Uint64(2),
+				SampleSum:     proto.Float64(7),
+				Schema:        proto.Int32(2),
+				ZeroThreshold: proto.Float64(2.938735877055719e-39),
+				ZeroCount:     proto.Uint64(0),
+				PositiveSpan: []*dto.BucketSpan{
+					{Offset: proto.Int32(7), Length: proto.Uint32(2)},
+				},
+				PositiveDelta: []int64{1, 0},
+			},
 		},
 		{
 			name:         "limited buckets but nothing triggered, negative observations",
 			observations: []float64{0, -1, -1.2, -1.4, -1.8, -2},
 			factor:       1.2,
 			maxBuckets:   4,
-			want:         `sample_count:6 sample_sum:-7.4 schema:2 zero_threshold:2.938735877055719e-39 zero_count:1 negative_span:<offset:0 length:5 > negative_delta:1 negative_delta:-1 negative_delta:2 negative_delta:-2 negative_delta:2 `,
+			want: &dto.Histogram{
+				SampleCount:   proto.Uint64(6),
+				SampleSum:     proto.Float64(-7.4),
+				Schema:        proto.Int32(2),
+				ZeroThreshold: proto.Float64(2.938735877055719e-39),
+				ZeroCount:     proto.Uint64(1),
+				NegativeSpan: []*dto.BucketSpan{
+					{Offset: proto.Int32(0), Length: proto.Uint32(5)},
+				},
+				NegativeDelta: []int64{1, -1, 2, -2, 2},
+			},
 		},
 		{
 			name:         "buckets limited by halving resolution, negative observations",
 			observations: []float64{0, -1, -1.1, -1.2, -1.4, -1.8, -2, -3},
 			factor:       1.2,
 			maxBuckets:   4,
-			want:         `sample_count:8 sample_sum:-11.5 schema:1 zero_threshold:2.938735877055719e-39 zero_count:1 negative_span:<offset:0 length:5 > negative_delta:1 negative_delta:2 negative_delta:-1 negative_delta:-2 negative_delta:1 `,
+			want: &dto.Histogram{
+				SampleCount:   proto.Uint64(8),
+				SampleSum:     proto.Float64(-11.5),
+				Schema:        proto.Int32(1),
+				ZeroThreshold: proto.Float64(2.938735877055719e-39),
+				ZeroCount:     proto.Uint64(1),
+				NegativeSpan: []*dto.BucketSpan{
+					{Offset: proto.Int32(0), Length: proto.Uint32(5)},
+				},
+				NegativeDelta: []int64{1, 2, -1, -2, 1},
+			},
 		},
 		{
 			name:             "buckets limited by widening the zero bucket, negative observations",
@@ -628,7 +844,17 @@ func TestNativeHistogram(t *testing.T) {
 			factor:           1.2,
 			maxBuckets:       4,
 			maxZeroThreshold: 1.2,
-			want:             `sample_count:8 sample_sum:-11.5 schema:2 zero_threshold:1 zero_count:2 negative_span:<offset:1 length:7 > negative_delta:1 negative_delta:1 negative_delta:-2 negative_delta:2 negative_delta:-2 negative_delta:0 negative_delta:1 `,
+			want: &dto.Histogram{
+				SampleCount:   proto.Uint64(8),
+				SampleSum:     proto.Float64(-11.5),
+				Schema:        proto.Int32(2),
+				ZeroThreshold: proto.Float64(1),
+				ZeroCount:     proto.Uint64(2),
+				NegativeSpan: []*dto.BucketSpan{
+					{Offset: proto.Int32(1), Length: proto.Uint32(7)},
+				},
+				NegativeDelta: []int64{1, 1, -2, 2, -2, 0, 1},
+			},
 		},
 		{
 			name:             "buckets limited by widening the zero bucket twice, negative observations",
@@ -636,7 +862,17 @@ func TestNativeHistogram(t *testing.T) {
 			factor:           1.2,
 			maxBuckets:       4,
 			maxZeroThreshold: 1.2,
-			want:             `sample_count:9 sample_sum:-15.5 schema:2 zero_threshold:1.189207115002721 zero_count:3 negative_span:<offset:2 length:7 > negative_delta:2 negative_delta:-2 negative_delta:2 negative_delta:-2 negative_delta:0 negative_delta:1 negative_delta:0 `,
+			want: &dto.Histogram{
+				SampleCount:   proto.Uint64(9),
+				SampleSum:     proto.Float64(-15.5),
+				Schema:        proto.Int32(2),
+				ZeroThreshold: proto.Float64(1.189207115002721),
+				ZeroCount:     proto.Uint64(3),
+				NegativeSpan: []*dto.BucketSpan{
+					{Offset: proto.Int32(2), Length: proto.Uint32(7)},
+				},
+				NegativeDelta: []int64{2, -2, 2, -2, 0, 1, 0},
+			},
 		},
 		{
 			name:             "buckets limited by reset, negative observations",
@@ -645,7 +881,17 @@ func TestNativeHistogram(t *testing.T) {
 			maxBuckets:       4,
 			maxZeroThreshold: 1.2,
 			minResetDuration: 5 * time.Minute,
-			want:             `sample_count:2 sample_sum:-7 schema:2 zero_threshold:2.938735877055719e-39 zero_count:0 negative_span:<offset:7 length:2 > negative_delta:1 negative_delta:0 `,
+			want: &dto.Histogram{
+				SampleCount:   proto.Uint64(2),
+				SampleSum:     proto.Float64(-7),
+				Schema:        proto.Int32(2),
+				ZeroThreshold: proto.Float64(2.938735877055719e-39),
+				ZeroCount:     proto.Uint64(0),
+				NegativeSpan: []*dto.BucketSpan{
+					{Offset: proto.Int32(7), Length: proto.Uint32(2)},
+				},
+				NegativeDelta: []int64{1, 0},
+			},
 		},
 		{
 			name:             "buckets limited by halving resolution, then reset",
@@ -653,7 +899,17 @@ func TestNativeHistogram(t *testing.T) {
 			factor:           1.2,
 			maxBuckets:       4,
 			minResetDuration: 9 * time.Minute,
-			want:             `sample_count:2 sample_sum:7 schema:2 zero_threshold:2.938735877055719e-39 zero_count:0 positive_span:<offset:7 length:2 > positive_delta:1 positive_delta:0 `,
+			want: &dto.Histogram{
+				SampleCount:   proto.Uint64(2),
+				SampleSum:     proto.Float64(7),
+				Schema:        proto.Int32(2),
+				ZeroThreshold: proto.Float64(2.938735877055719e-39),
+				ZeroCount:     proto.Uint64(0),
+				PositiveSpan: []*dto.BucketSpan{
+					{Offset: proto.Int32(7), Length: proto.Uint32(2)},
+				},
+				PositiveDelta: []int64{1, 0},
+			},
 		},
 		{
 			name:             "buckets limited by widening the zero bucket, then reset",
@@ -662,7 +918,17 @@ func TestNativeHistogram(t *testing.T) {
 			maxBuckets:       4,
 			maxZeroThreshold: 1.2,
 			minResetDuration: 9 * time.Minute,
-			want:             `sample_count:2 sample_sum:7 schema:2 zero_threshold:2.938735877055719e-39 zero_count:0 positive_span:<offset:7 length:2 > positive_delta:1 positive_delta:0 `,
+			want: &dto.Histogram{
+				SampleCount:   proto.Uint64(2),
+				SampleSum:     proto.Float64(7),
+				Schema:        proto.Int32(2),
+				ZeroThreshold: proto.Float64(2.938735877055719e-39),
+				ZeroCount:     proto.Uint64(0),
+				PositiveSpan: []*dto.BucketSpan{
+					{Offset: proto.Int32(7), Length: proto.Uint32(2)},
+				},
+				PositiveDelta: []int64{1, 0},
+			},
 		},
 	}
 
@@ -690,8 +956,8 @@ func TestNativeHistogram(t *testing.T) {
 			if err := his.Write(m); err != nil {
 				t.Fatal("unexpected error writing metric", err)
 			}
-			got := m.Histogram.String()
+			got := m.Histogram
-			if s.want != got {
+			if !proto.Equal(s.want, got) {
 				t.Errorf("want histogram %q, got %q", s.want, got)
 			}
 		})

prometheus/registry_test.go

@@ -725,7 +725,11 @@ collected metric "broken_metric" { label:<name:"foo" value:"bar" > label:<name:"
 			}
 		}
 
-		if !bytes.Equal(scenario.out.body, writer.Body.Bytes()) {
+		var outMF dto.MetricFamily
+		var writerMF dto.MetricFamily
+		proto.Unmarshal(scenario.out.body, &outMF)
+		proto.Unmarshal(writer.Body.Bytes(), &writerMF)
+		if !proto.Equal(&outMF, &writerMF) {
 			t.Errorf(
 				"%d. expected body:\n%s\ngot body:\n%s\n",
 				i, scenario.out.body, writer.Body.Bytes(),

prometheus/utils_test.go

@@ -0,0 +1,43 @@
+// Copyright 2018 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package prometheus_test
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+
+	"google.golang.org/protobuf/encoding/protojson"
+	"google.golang.org/protobuf/proto"
+)
+
+// printlnNormalized is a helper function to compare proto messages in json format.
+// Without removing brittle, we can't assert that two proto messages in json/text format are equal.
+// Read more in https://github.com/golang/protobuf/issues/1121
+func printlnNormalized(m proto.Message) {
+	fmt.Println(protoToNormalizedJSON(m))
+}
+
+// protoToNormalizedJSON works as printlnNormalized, but returns the string instead of printing.
+func protoToNormalizedJSON(m proto.Message) string {
+	mAsJSON, err := protojson.Marshal(m)
+	if err != nil {
+		panic(err)
+	}
+
+	buffer := new(bytes.Buffer)
+	if err := json.Compact(buffer, mAsJSON); err != nil {
+		panic(err)
+	}
+	return buffer.String()
+}

prometheus/wrap_test.go

@@ -15,11 +15,11 @@ package prometheus
 
 import (
 	"fmt"
-	"reflect"
 	"strings"
 	"testing"
 
 	dto "github.com/prometheus/client_model/go"
+	"google.golang.org/protobuf/proto"
 )
 
 // uncheckedCollector wraps a Collector but its Describe method yields no Desc.
@@ -297,21 +297,26 @@ func TestWrap(t *testing.T) {
 			if !s.gatherFails && err != nil {
 				t.Fatal("gathering failed:", err)
 			}
-			if !reflect.DeepEqual(gotMF, wantMF) {
+			if len(wantMF) != len(gotMF) {
-				var want, got []string
+				t.Fatalf("Expected %d metricFamilies, got %d", len(wantMF), len(gotMF))
+			}
+			for i := range gotMF {
+				if !proto.Equal(gotMF[i], wantMF[i]) {
+					var want, got []string
 
-				for i, mf := range wantMF {
+					for i, mf := range wantMF {
-					want = append(want, fmt.Sprintf("%3d: %s", i, mf))
+						want = append(want, fmt.Sprintf("%3d: %s", i, mf))
-				}
+					}
-				for i, mf := range gotMF {
+					for i, mf := range gotMF {
-					got = append(got, fmt.Sprintf("%3d: %s", i, mf))
+						got = append(got, fmt.Sprintf("%3d: %s", i, mf))
-				}
+					}
 
-				t.Fatalf(
+					t.Fatalf(
-					"unexpected output of gathering:\n\nWANT:\n%s\n\nGOT:\n%s\n",
+						"unexpected output of gathering:\n\nWANT:\n%s\n\nGOT:\n%s\n",
-					strings.Join(want, "\n"),
+						strings.Join(want, "\n"),
-					strings.Join(got, "\n"),
+						strings.Join(got, "\n"),
-				)
+					)
+				}
 			}
 		})
 	}