Enable the Golang client library to create the new text formats.

Most important here is the simple & flat text format, but while I'm on
it, I have also added the text representations for protobufs (which is
purely meant for debugging purposes). I hope my basic idea about
handling those various protocols (and the text package) becomes
clearer now.

Change-Id: I7299853eadc82a426101e907f2b3d4e37f9e4c71

prometheus/constants.go

@@ -26,14 +26,27 @@ const (
 	// APIVersion is the version of the format of the exported data.  This
 	// will match this library's version, which subscribes to the Semantic
 	// Versioning scheme.
-	APIVersion = "0.0.2"
+	APIVersion = "0.0.4"
 
-	// TelemetryContentType is the content type and schema information set
+	// JSONAPIVersion is the version of the JSON export format.
-	// on telemetry data responses.
+	JSONAPIVersion = "0.0.2"
-	TelemetryContentType = `application/json; schema="prometheus/telemetry"; version=` + APIVersion
+
-	// DelimitedTelemetryContentType is the content type and schema
+	// DelimitedTelemetryContentType is the content type set on telemetry
-	// information set on telemetry data responses.
+	// data responses in delimited protobuf format.
 	DelimitedTelemetryContentType = `application/vnd.google.protobuf; proto="io.prometheus.client.MetricFamily"; encoding="delimited"`
+	// TextTelemetryContentType is the content type set on telemetry data
+	// responses in text format.
+	TextTelemetryContentType = `text/plain; version=` + APIVersion
+	// ProtoTextTelemetryContentType is the content type set on telemetry
+	// data responses in protobuf text format.  (Only used for debugging.)
+	ProtoTextTelemetryContentType = `application/vnd.google.protobuf; proto="io.prometheus.client.MetricFamily"; encoding="text"`
+	// ProtoCompactTextTelemetryContentType is the content type set on
+	// telemetry data responses in protobuf compact text format.  (Only used
+	// for debugging.)
+	ProtoCompactTextTelemetryContentType = `application/vnd.google.protobuf; proto="io.prometheus.client.MetricFamily"; encoding="compact-text"`
+	// JSONTelemetryContentType is the content type set on telemetry data
+	// responses formatted as JSON.
+	JSONTelemetryContentType = `application/json; schema="prometheus/telemetry"; version=` + JSONAPIVersion
 
 	// ExpositionResource is the customary web services endpoint on which
 	// telemetric data is exposed.

prometheus/registry.go

@@ -26,6 +26,7 @@ import (
 	"github.com/matttproud/golang_protobuf_extensions/ext"
 
 	"github.com/prometheus/client_golang/model"
+	"github.com/prometheus/client_golang/text"
 	"github.com/prometheus/client_golang/vendor/goautoneg"
 )
 
@@ -42,6 +43,12 @@ const (
 	jsonContentType          = "application/json"
 )
 
+// encoder is a function that writes a proto.Message to an io.Writer in a
+// certain encoding. It returns the number of bytes written and any error
+// encountered.  Note that ext.WriteDelimited and text.MetricFamilyToText are
+// encoders.
+type encoder func(io.Writer, proto.Message) (int, error)
+
 // container represents a top-level registered metric that encompasses its
 // static metadata.
 type container struct {
@@ -267,7 +274,7 @@ func (r *registry) YieldExporter() http.HandlerFunc {
 	return r.Handler()
 }
 
-func (r *registry) dumpDelimitedPB(w io.Writer) {
+func (r *registry) dumpPB(w io.Writer, writeEncoded encoder) {
 	r.mutex.RLock()
 	defer r.mutex.RUnlock()
 
@@ -298,16 +305,16 @@ func (r *registry) dumpDelimitedPB(w io.Writer) {
 			}
 		}
 
-		ext.WriteDelimited(w, f)
+		writeEncoded(w, f)
 	}
 }
 
-func (r *registry) dumpDelimitedExternalPB(w io.Writer) {
+func (r *registry) dumpExternalPB(w io.Writer, writeEncoded encoder) {
 	if r.metricFamilyInjectionHook == nil {
 		return
 	}
 	for _, f := range r.metricFamilyInjectionHook() {
-		ext.WriteDelimited(w, f)
+		writeEncoded(w, f)
 	}
 }
 
@@ -326,26 +333,39 @@ func (r *registry) Handler() http.HandlerFunc {
 
 		accepts := goautoneg.ParseAccept(req.Header.Get("Accept"))
 		for _, accept := range accepts {
-			if accept.Type != "application" {
+			var enc encoder
+			switch {
+			case accept.Type == "application" &&
+				accept.SubType == "vnd.google.protobuf" &&
+				accept.Params["proto"] == "io.prometheus.client.MetricFamily":
+				switch accept.Params["encoding"] {
+				case "delimited":
+					header.Set(contentTypeHeader, DelimitedTelemetryContentType)
+					enc = ext.WriteDelimited
+				case "text":
+					header.Set(contentTypeHeader, ProtoTextTelemetryContentType)
+					enc = text.WriteProtoText
+				case "compact-text":
+					header.Set(contentTypeHeader, ProtoCompactTextTelemetryContentType)
+					enc = text.WriteProtoCompactText
+				default:
+					continue
+				}
+			case accept.Type == "text" &&
+				accept.SubType == "plain" &&
+				(accept.Params["version"] == "0.0.4" || accept.Params["version"] == ""):
+				header.Set(contentTypeHeader, TextTelemetryContentType)
+				enc = text.MetricFamilyToText
+			default:
 				continue
 			}
-
+			r.dumpPB(writer, enc)
-			if accept.SubType == "vnd.google.protobuf" {
+			r.dumpExternalPB(writer, enc)
-				if accept.Params["proto"] != "io.prometheus.client.MetricFamily" {
+			return
-					continue
-				}
-				if accept.Params["encoding"] != "delimited" {
-					continue
-				}
-
-				header.Set(contentTypeHeader, DelimitedTelemetryContentType)
-				r.dumpDelimitedPB(writer)
-				r.dumpDelimitedExternalPB(writer)
-				return
-			}
 		}
-
+		// TODO: Once JSON deprecation is completed, use text format as
-		header.Set(contentTypeHeader, TelemetryContentType)
+		// fall-back.
+		header.Set(contentTypeHeader, JSONTelemetryContentType)
 		json.NewEncoder(writer).Encode(r)
 	}
 }

prometheus/registry_test.go

@@ -253,6 +253,30 @@ func testHandler(t test.Tester) {
 		t.Fatal(err)
 	}
 	externalMetricFamilyAsBytes := externalBuf.Bytes()
+	externalMetricFamilyAsText := []byte(`# HELP externalname externaldocstring
+# TYPE externalname counter
+externalname{externallabelname="externalval1",externalbasename="externalbasevalue"} 1
+`)
+	externalMetricFamilyAsProtoText := []byte(`name: "externalname"
+help: "externaldocstring"
+type: COUNTER
+metric: <
+  label: <
+    name: "externallabelname"
+    value: "externalval1"
+  >
+  label: <
+    name: "externalbasename"
+    value: "externalbasevalue"
+  >
+  counter: <
+    value: 1
+  >
+>
+
+`)
+	externalMetricFamilyAsProtoCompactText := []byte(`name:"externalname" help:"externaldocstring" type:COUNTER metric:<label:<name:"externallabelname" value:"externalval1" > label:<name:"externalbasename" value:"externalbasevalue" > counter:<value:1 > > 
+`)
 
 	expectedMetricFamily := &dto.MetricFamily{
 		Name: proto.String("name"),
@@ -306,6 +330,44 @@ func testHandler(t test.Tester) {
 		t.Fatal(err)
 	}
 	expectedMetricFamilyAsBytes := buf.Bytes()
+	expectedMetricFamilyAsText := []byte(`# HELP name docstring
+# TYPE name counter
+name{labelname="val1",basename="basevalue"} 1
+name{labelname="val2",basename="basevalue"} 1
+`)
+	expectedMetricFamilyAsProtoText := []byte(`name: "name"
+help: "docstring"
+type: COUNTER
+metric: <
+  label: <
+    name: "labelname"
+    value: "val1"
+  >
+  label: <
+    name: "basename"
+    value: "basevalue"
+  >
+  counter: <
+    value: 1
+  >
+>
+metric: <
+  label: <
+    name: "labelname"
+    value: "val2"
+  >
+  label: <
+    name: "basename"
+    value: "basevalue"
+  >
+  counter: <
+    value: 1
+  >
+>
+
+`)
+	expectedMetricFamilyAsProtoCompactText := []byte(`name:"name" help:"docstring" type:COUNTER metric:<label:<name:"labelname" value:"val1" > label:<name:"basename" value:"basevalue" > counter:<value:1 > > metric:<label:<name:"labelname" value:"val2" > label:<name:"basename" value:"basevalue" > counter:<value:1 > > 
+`)
 
 	type output struct {
 		headers map[string]string
@@ -318,7 +380,7 @@ func testHandler(t test.Tester) {
 		withCounter    bool
 		withExternalMF bool
 	}{
-		{
+		{ // 0
 			headers: map[string]string{
 				"Accept": "foo/bar;q=0.2, dings/bums;q=0.8",
 			},
@@ -329,7 +391,7 @@ func testHandler(t test.Tester) {
 				body: []byte("[]\n"),
 			},
 		},
-		{
+		{ // 1
 			headers: map[string]string{
 				"Accept": "foo/bar;q=0.2, application/quark;q=0.8",
 			},
@@ -340,7 +402,7 @@ func testHandler(t test.Tester) {
 				body: []byte("[]\n"),
 			},
 		},
-		{
+		{ // 2
 			headers: map[string]string{
 				"Accept": "foo/bar;q=0.2, application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=bla;q=0.8",
 			},
@@ -351,9 +413,9 @@ func testHandler(t test.Tester) {
 				body: []byte("[]\n"),
 			},
 		},
-		{
+		{ // 3
 			headers: map[string]string{
-				"Accept": "foo/bar;q=0.2, application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited;q=0.8",
+				"Accept": "text/plain;q=0.2, application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited;q=0.8",
 			},
 			out: output{
 				headers: map[string]string{
@@ -362,7 +424,7 @@ func testHandler(t test.Tester) {
 				body: []byte{},
 			},
 		},
-		{
+		{ // 4
 			headers: map[string]string{
 				"Accept": "application/json",
 			},
@@ -375,7 +437,7 @@ func testHandler(t test.Tester) {
 			},
 			withCounter: true,
 		},
-		{
+		{ // 5
 			headers: map[string]string{
 				"Accept": "application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited",
 			},
@@ -387,7 +449,7 @@ func testHandler(t test.Tester) {
 			},
 			withCounter: true,
 		},
-		{
+		{ // 6
 			headers: map[string]string{
 				"Accept": "application/json",
 			},
@@ -399,7 +461,7 @@ func testHandler(t test.Tester) {
 			},
 			withExternalMF: true,
 		},
-		{
+		{ // 7
 			headers: map[string]string{
 				"Accept": "application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited",
 			},
@@ -411,7 +473,7 @@ func testHandler(t test.Tester) {
 			},
 			withExternalMF: true,
 		},
-		{
+		{ // 8
 			headers: map[string]string{
 				"Accept": "application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited",
 			},
@@ -430,6 +492,105 @@ func testHandler(t test.Tester) {
 			withCounter:    true,
 			withExternalMF: true,
 		},
+		{ // 9
+			headers: map[string]string{
+				"Accept": "text/plain",
+			},
+			out: output{
+				headers: map[string]string{
+					"Content-Type": `text/plain; version=0.0.4`,
+				},
+				body: []byte{},
+			},
+		},
+		{ // 10
+			headers: map[string]string{
+				"Accept": "application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=bla;q=0.2, text/plain;q=0.5",
+			},
+			out: output{
+				headers: map[string]string{
+					"Content-Type": `text/plain; version=0.0.4`,
+				},
+				body: expectedMetricFamilyAsText,
+			},
+			withCounter: true,
+		},
+		{ // 11
+			headers: map[string]string{
+				"Accept": "application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=bla;q=0.2, text/plain;q=0.5;version=0.0.4",
+			},
+			out: output{
+				headers: map[string]string{
+					"Content-Type": `text/plain; version=0.0.4`,
+				},
+				body: bytes.Join(
+					[][]byte{
+						expectedMetricFamilyAsText,
+						externalMetricFamilyAsText,
+					},
+					[]byte{},
+				),
+			},
+			withCounter:    true,
+			withExternalMF: true,
+		},
+		{ // 12
+			headers: map[string]string{
+				"Accept": "application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited;q=0.2, text/plain;q=0.5;version=0.0.2",
+			},
+			out: output{
+				headers: map[string]string{
+					"Content-Type": `application/vnd.google.protobuf; proto="io.prometheus.client.MetricFamily"; encoding="delimited"`,
+				},
+				body: bytes.Join(
+					[][]byte{
+						expectedMetricFamilyAsBytes,
+						externalMetricFamilyAsBytes,
+					},
+					[]byte{},
+				),
+			},
+			withCounter:    true,
+			withExternalMF: true,
+		},
+		{ // 13
+			headers: map[string]string{
+				"Accept": "application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=text;q=0.5, application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited;q=0.4",
+			},
+			out: output{
+				headers: map[string]string{
+					"Content-Type": `application/vnd.google.protobuf; proto="io.prometheus.client.MetricFamily"; encoding="text"`,
+				},
+				body: bytes.Join(
+					[][]byte{
+						expectedMetricFamilyAsProtoText,
+						externalMetricFamilyAsProtoText,
+					},
+					[]byte{},
+				),
+			},
+			withCounter:    true,
+			withExternalMF: true,
+		},
+		{ // 14
+			headers: map[string]string{
+				"Accept": "application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=compact-text",
+			},
+			out: output{
+				headers: map[string]string{
+					"Content-Type": `application/vnd.google.protobuf; proto="io.prometheus.client.MetricFamily"; encoding="compact-text"`,
+				},
+				body: bytes.Join(
+					[][]byte{
+						expectedMetricFamilyAsProtoCompactText,
+						externalMetricFamilyAsProtoCompactText,
+					},
+					[]byte{},
+				),
+			},
+			withCounter:    true,
+			withExternalMF: true,
+		},
 	}
 	for i, scenario := range scenarios {
 		registry := NewRegistry().(*registry)

text/create.go

@@ -11,8 +11,13 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// Package text contains functions to parse and create the simple and flat
+// Package text contains helper functions to parse and create text-based
-// text-based exchange format.
+// exchange formats. The package currently supports (only) version 0.0.4 of the
+// exchange format. Should other versions be supported in the future, some
+// versioning scheme has to be applied. Possibilities include separate packages
+// or separate functions. The best way depends on the nature of future changes,
+// which is the reason why no versioning scheme has been applied prematurely
+// here.
 package text
 
 import (
@@ -20,6 +25,7 @@ import (
 	"fmt"
 	"io"
 	"strings"
+	"code.google.com/p/goprotobuf/proto"
 
 	dto "github.com/prometheus/client_model/go"
 )
@@ -29,32 +35,33 @@ import (
 // and any error encountered.  This function does not perform checks on the
 // content of the metric and label names, i.e. invalid metric or label names
 // will result in invalid text format output.
-func MetricFamilyToText(in *dto.MetricFamily, out io.Writer) (int, error) {
+func MetricFamilyToText(out io.Writer, in proto.Message) (int, error) {
+	mf := in.(*dto.MetricFamily)
 	var written int
 
 	// Fail-fast checks.
-	if len(in.Metric) == 0 {
+	if len(mf.Metric) == 0 {
 		return written, fmt.Errorf("MetricFamily has no metrics: %s", in)
 	}
-	name := in.GetName()
+	name := mf.GetName()
 	if name == "" {
 		return written, fmt.Errorf("MetricFamily has no name: %s", in)
 	}
-	if in.Type == nil {
+	if mf.Type == nil {
 		return written, fmt.Errorf("MetricFamily has no type: %s", in)
 	}
 
 	// Comments, first HELP, then TYPE.
-	if in.Help != nil {
+	if mf.Help != nil {
 		n, err := fmt.Fprintf(
 			out, "# HELP %s %s\n",
-			name, strings.Replace(*in.Help, "\n", `\n`, -1))
+			name, strings.Replace(*mf.Help, "\n", `\n`, -1))
 		written += n
 		if err != nil {
 			return written, err
 		}
 	}
-	metricType := in.GetType()
+	metricType := mf.GetType()
 	n, err := fmt.Fprintf(
 		out, "# TYPE %s %s\n",
 		name, strings.ToLower(metricType.String()),
@@ -65,7 +72,7 @@ func MetricFamilyToText(in *dto.MetricFamily, out io.Writer) (int, error) {
 	}
 
 	// Finally the samples, one line for each.
-	for _, metric := range in.Metric {
+	for _, metric := range mf.Metric {
 		switch metricType {
 		case dto.MetricType_COUNTER:
 			if metric.Counter == nil {

text/create_test.go

@@ -226,7 +226,7 @@ summary_name_count{name_1="value 1",name_2="value 2"} 4711
 
 	for i, scenario := range scenarios {
 		out := bytes.NewBuffer(make([]byte, 0, len(scenario.out)))
-		n, err := MetricFamilyToText(scenario.in, out)
+		n, err := MetricFamilyToText(out, scenario.in)
 		if err != nil {
 			t.Errorf("%d. error: %s", i, err)
 			continue
@@ -322,7 +322,7 @@ func testCreateError(t test.Tester) {
 
 	for i, scenario := range scenarios {
 		var out bytes.Buffer
-		_, err := MetricFamilyToText(scenario.in, &out)
+		_, err := MetricFamilyToText(&out, scenario.in)
 		if err == nil {
 			t.Errorf("%d. expected error, got nil", i)
 			continue

text/proto.go

@@ -0,0 +1,33 @@
+// Copyright 2014 Prometheus Team
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package text
+
+import (
+	"fmt"
+	"io"
+
+	"code.google.com/p/goprotobuf/proto"
+)
+
+// WriteProtoText writes the proto.Message to the writer in text format and
+// returns the number of bytes written and any error encountered.
+func WriteProtoText(w io.Writer, p proto.Message) (int, error) {
+	return fmt.Fprintf(w, "%s\n", proto.MarshalTextString(p))
+}
+
+// WriteProtoCompactText writes the proto.Message to the writer in compact text
+// format and returns the number of bytes written and any error encountered.
+func WriteProtoCompactText(w io.Writer, p proto.Message) (int, error) {
+	return fmt.Fprintf(w, "%s\n", p)
+}