Enable the Golang client library to create the new text formats.

Most important here is the simple & flat text format, but while I'm on
it, I have also added the text representations for protobufs (which is
purely meant for debugging purposes). I hope my basic idea about
handling those various protocols (and the text package) becomes
clearer now.

Change-Id: I7299853eadc82a426101e907f2b3d4e37f9e4c71
This commit is contained in:
Bjoern Rabenstein 2014-04-23 13:40:37 +02:00
parent 9da2fbcce3
commit 84dc53148d
6 changed files with 283 additions and 49 deletions

View File

@ -26,14 +26,27 @@ const (
// APIVersion is the version of the format of the exported data. This // APIVersion is the version of the format of the exported data. This
// will match this library's version, which subscribes to the Semantic // will match this library's version, which subscribes to the Semantic
// Versioning scheme. // Versioning scheme.
APIVersion = "0.0.2" APIVersion = "0.0.4"
// TelemetryContentType is the content type and schema information set // JSONAPIVersion is the version of the JSON export format.
// on telemetry data responses. JSONAPIVersion = "0.0.2"
TelemetryContentType = `application/json; schema="prometheus/telemetry"; version=` + APIVersion
// DelimitedTelemetryContentType is the content type and schema // DelimitedTelemetryContentType is the content type set on telemetry
// information set on telemetry data responses. // data responses in delimited protobuf format.
DelimitedTelemetryContentType = `application/vnd.google.protobuf; proto="io.prometheus.client.MetricFamily"; encoding="delimited"` DelimitedTelemetryContentType = `application/vnd.google.protobuf; proto="io.prometheus.client.MetricFamily"; encoding="delimited"`
// TextTelemetryContentType is the content type set on telemetry data
// responses in text format.
TextTelemetryContentType = `text/plain; version=` + APIVersion
// ProtoTextTelemetryContentType is the content type set on telemetry
// data responses in protobuf text format. (Only used for debugging.)
ProtoTextTelemetryContentType = `application/vnd.google.protobuf; proto="io.prometheus.client.MetricFamily"; encoding="text"`
// ProtoCompactTextTelemetryContentType is the content type set on
// telemetry data responses in protobuf compact text format. (Only used
// for debugging.)
ProtoCompactTextTelemetryContentType = `application/vnd.google.protobuf; proto="io.prometheus.client.MetricFamily"; encoding="compact-text"`
// JSONTelemetryContentType is the content type set on telemetry data
// responses formatted as JSON.
JSONTelemetryContentType = `application/json; schema="prometheus/telemetry"; version=` + JSONAPIVersion
// ExpositionResource is the customary web services endpoint on which // ExpositionResource is the customary web services endpoint on which
// telemetric data is exposed. // telemetric data is exposed.

View File

@ -26,6 +26,7 @@ import (
"github.com/matttproud/golang_protobuf_extensions/ext" "github.com/matttproud/golang_protobuf_extensions/ext"
"github.com/prometheus/client_golang/model" "github.com/prometheus/client_golang/model"
"github.com/prometheus/client_golang/text"
"github.com/prometheus/client_golang/vendor/goautoneg" "github.com/prometheus/client_golang/vendor/goautoneg"
) )
@ -42,6 +43,12 @@ const (
jsonContentType = "application/json" jsonContentType = "application/json"
) )
// encoder is a function that writes a proto.Message to an io.Writer in a
// certain encoding. It returns the number of bytes written and any error
// encountered. Note that ext.WriteDelimited and text.MetricFamilyToText are
// encoders.
type encoder func(io.Writer, proto.Message) (int, error)
// container represents a top-level registered metric that encompasses its // container represents a top-level registered metric that encompasses its
// static metadata. // static metadata.
type container struct { type container struct {
@ -267,7 +274,7 @@ func (r *registry) YieldExporter() http.HandlerFunc {
return r.Handler() return r.Handler()
} }
func (r *registry) dumpDelimitedPB(w io.Writer) { func (r *registry) dumpPB(w io.Writer, writeEncoded encoder) {
r.mutex.RLock() r.mutex.RLock()
defer r.mutex.RUnlock() defer r.mutex.RUnlock()
@ -298,16 +305,16 @@ func (r *registry) dumpDelimitedPB(w io.Writer) {
} }
} }
ext.WriteDelimited(w, f) writeEncoded(w, f)
} }
} }
func (r *registry) dumpDelimitedExternalPB(w io.Writer) { func (r *registry) dumpExternalPB(w io.Writer, writeEncoded encoder) {
if r.metricFamilyInjectionHook == nil { if r.metricFamilyInjectionHook == nil {
return return
} }
for _, f := range r.metricFamilyInjectionHook() { for _, f := range r.metricFamilyInjectionHook() {
ext.WriteDelimited(w, f) writeEncoded(w, f)
} }
} }
@ -326,26 +333,39 @@ func (r *registry) Handler() http.HandlerFunc {
accepts := goautoneg.ParseAccept(req.Header.Get("Accept")) accepts := goautoneg.ParseAccept(req.Header.Get("Accept"))
for _, accept := range accepts { for _, accept := range accepts {
if accept.Type != "application" { var enc encoder
switch {
case accept.Type == "application" &&
accept.SubType == "vnd.google.protobuf" &&
accept.Params["proto"] == "io.prometheus.client.MetricFamily":
switch accept.Params["encoding"] {
case "delimited":
header.Set(contentTypeHeader, DelimitedTelemetryContentType)
enc = ext.WriteDelimited
case "text":
header.Set(contentTypeHeader, ProtoTextTelemetryContentType)
enc = text.WriteProtoText
case "compact-text":
header.Set(contentTypeHeader, ProtoCompactTextTelemetryContentType)
enc = text.WriteProtoCompactText
default:
continue
}
case accept.Type == "text" &&
accept.SubType == "plain" &&
(accept.Params["version"] == "0.0.4" || accept.Params["version"] == ""):
header.Set(contentTypeHeader, TextTelemetryContentType)
enc = text.MetricFamilyToText
default:
continue continue
} }
r.dumpPB(writer, enc)
if accept.SubType == "vnd.google.protobuf" { r.dumpExternalPB(writer, enc)
if accept.Params["proto"] != "io.prometheus.client.MetricFamily" { return
continue
}
if accept.Params["encoding"] != "delimited" {
continue
}
header.Set(contentTypeHeader, DelimitedTelemetryContentType)
r.dumpDelimitedPB(writer)
r.dumpDelimitedExternalPB(writer)
return
}
} }
// TODO: Once JSON deprecation is completed, use text format as
header.Set(contentTypeHeader, TelemetryContentType) // fall-back.
header.Set(contentTypeHeader, JSONTelemetryContentType)
json.NewEncoder(writer).Encode(r) json.NewEncoder(writer).Encode(r)
} }
} }

View File

@ -253,6 +253,30 @@ func testHandler(t test.Tester) {
t.Fatal(err) t.Fatal(err)
} }
externalMetricFamilyAsBytes := externalBuf.Bytes() externalMetricFamilyAsBytes := externalBuf.Bytes()
externalMetricFamilyAsText := []byte(`# HELP externalname externaldocstring
# TYPE externalname counter
externalname{externallabelname="externalval1",externalbasename="externalbasevalue"} 1
`)
externalMetricFamilyAsProtoText := []byte(`name: "externalname"
help: "externaldocstring"
type: COUNTER
metric: <
label: <
name: "externallabelname"
value: "externalval1"
>
label: <
name: "externalbasename"
value: "externalbasevalue"
>
counter: <
value: 1
>
>
`)
externalMetricFamilyAsProtoCompactText := []byte(`name:"externalname" help:"externaldocstring" type:COUNTER metric:<label:<name:"externallabelname" value:"externalval1" > label:<name:"externalbasename" value:"externalbasevalue" > counter:<value:1 > >
`)
expectedMetricFamily := &dto.MetricFamily{ expectedMetricFamily := &dto.MetricFamily{
Name: proto.String("name"), Name: proto.String("name"),
@ -306,6 +330,44 @@ func testHandler(t test.Tester) {
t.Fatal(err) t.Fatal(err)
} }
expectedMetricFamilyAsBytes := buf.Bytes() expectedMetricFamilyAsBytes := buf.Bytes()
expectedMetricFamilyAsText := []byte(`# HELP name docstring
# TYPE name counter
name{labelname="val1",basename="basevalue"} 1
name{labelname="val2",basename="basevalue"} 1
`)
expectedMetricFamilyAsProtoText := []byte(`name: "name"
help: "docstring"
type: COUNTER
metric: <
label: <
name: "labelname"
value: "val1"
>
label: <
name: "basename"
value: "basevalue"
>
counter: <
value: 1
>
>
metric: <
label: <
name: "labelname"
value: "val2"
>
label: <
name: "basename"
value: "basevalue"
>
counter: <
value: 1
>
>
`)
expectedMetricFamilyAsProtoCompactText := []byte(`name:"name" help:"docstring" type:COUNTER metric:<label:<name:"labelname" value:"val1" > label:<name:"basename" value:"basevalue" > counter:<value:1 > > metric:<label:<name:"labelname" value:"val2" > label:<name:"basename" value:"basevalue" > counter:<value:1 > >
`)
type output struct { type output struct {
headers map[string]string headers map[string]string
@ -318,7 +380,7 @@ func testHandler(t test.Tester) {
withCounter bool withCounter bool
withExternalMF bool withExternalMF bool
}{ }{
{ { // 0
headers: map[string]string{ headers: map[string]string{
"Accept": "foo/bar;q=0.2, dings/bums;q=0.8", "Accept": "foo/bar;q=0.2, dings/bums;q=0.8",
}, },
@ -329,7 +391,7 @@ func testHandler(t test.Tester) {
body: []byte("[]\n"), body: []byte("[]\n"),
}, },
}, },
{ { // 1
headers: map[string]string{ headers: map[string]string{
"Accept": "foo/bar;q=0.2, application/quark;q=0.8", "Accept": "foo/bar;q=0.2, application/quark;q=0.8",
}, },
@ -340,7 +402,7 @@ func testHandler(t test.Tester) {
body: []byte("[]\n"), body: []byte("[]\n"),
}, },
}, },
{ { // 2
headers: map[string]string{ headers: map[string]string{
"Accept": "foo/bar;q=0.2, application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=bla;q=0.8", "Accept": "foo/bar;q=0.2, application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=bla;q=0.8",
}, },
@ -351,9 +413,9 @@ func testHandler(t test.Tester) {
body: []byte("[]\n"), body: []byte("[]\n"),
}, },
}, },
{ { // 3
headers: map[string]string{ headers: map[string]string{
"Accept": "foo/bar;q=0.2, application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited;q=0.8", "Accept": "text/plain;q=0.2, application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited;q=0.8",
}, },
out: output{ out: output{
headers: map[string]string{ headers: map[string]string{
@ -362,7 +424,7 @@ func testHandler(t test.Tester) {
body: []byte{}, body: []byte{},
}, },
}, },
{ { // 4
headers: map[string]string{ headers: map[string]string{
"Accept": "application/json", "Accept": "application/json",
}, },
@ -375,7 +437,7 @@ func testHandler(t test.Tester) {
}, },
withCounter: true, withCounter: true,
}, },
{ { // 5
headers: map[string]string{ headers: map[string]string{
"Accept": "application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited", "Accept": "application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited",
}, },
@ -387,7 +449,7 @@ func testHandler(t test.Tester) {
}, },
withCounter: true, withCounter: true,
}, },
{ { // 6
headers: map[string]string{ headers: map[string]string{
"Accept": "application/json", "Accept": "application/json",
}, },
@ -399,7 +461,7 @@ func testHandler(t test.Tester) {
}, },
withExternalMF: true, withExternalMF: true,
}, },
{ { // 7
headers: map[string]string{ headers: map[string]string{
"Accept": "application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited", "Accept": "application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited",
}, },
@ -411,7 +473,7 @@ func testHandler(t test.Tester) {
}, },
withExternalMF: true, withExternalMF: true,
}, },
{ { // 8
headers: map[string]string{ headers: map[string]string{
"Accept": "application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited", "Accept": "application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited",
}, },
@ -430,6 +492,105 @@ func testHandler(t test.Tester) {
withCounter: true, withCounter: true,
withExternalMF: true, withExternalMF: true,
}, },
{ // 9
headers: map[string]string{
"Accept": "text/plain",
},
out: output{
headers: map[string]string{
"Content-Type": `text/plain; version=0.0.4`,
},
body: []byte{},
},
},
{ // 10
headers: map[string]string{
"Accept": "application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=bla;q=0.2, text/plain;q=0.5",
},
out: output{
headers: map[string]string{
"Content-Type": `text/plain; version=0.0.4`,
},
body: expectedMetricFamilyAsText,
},
withCounter: true,
},
{ // 11
headers: map[string]string{
"Accept": "application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=bla;q=0.2, text/plain;q=0.5;version=0.0.4",
},
out: output{
headers: map[string]string{
"Content-Type": `text/plain; version=0.0.4`,
},
body: bytes.Join(
[][]byte{
expectedMetricFamilyAsText,
externalMetricFamilyAsText,
},
[]byte{},
),
},
withCounter: true,
withExternalMF: true,
},
{ // 12
headers: map[string]string{
"Accept": "application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited;q=0.2, text/plain;q=0.5;version=0.0.2",
},
out: output{
headers: map[string]string{
"Content-Type": `application/vnd.google.protobuf; proto="io.prometheus.client.MetricFamily"; encoding="delimited"`,
},
body: bytes.Join(
[][]byte{
expectedMetricFamilyAsBytes,
externalMetricFamilyAsBytes,
},
[]byte{},
),
},
withCounter: true,
withExternalMF: true,
},
{ // 13
headers: map[string]string{
"Accept": "application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=text;q=0.5, application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited;q=0.4",
},
out: output{
headers: map[string]string{
"Content-Type": `application/vnd.google.protobuf; proto="io.prometheus.client.MetricFamily"; encoding="text"`,
},
body: bytes.Join(
[][]byte{
expectedMetricFamilyAsProtoText,
externalMetricFamilyAsProtoText,
},
[]byte{},
),
},
withCounter: true,
withExternalMF: true,
},
{ // 14
headers: map[string]string{
"Accept": "application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=compact-text",
},
out: output{
headers: map[string]string{
"Content-Type": `application/vnd.google.protobuf; proto="io.prometheus.client.MetricFamily"; encoding="compact-text"`,
},
body: bytes.Join(
[][]byte{
expectedMetricFamilyAsProtoCompactText,
externalMetricFamilyAsProtoCompactText,
},
[]byte{},
),
},
withCounter: true,
withExternalMF: true,
},
} }
for i, scenario := range scenarios { for i, scenario := range scenarios {
registry := NewRegistry().(*registry) registry := NewRegistry().(*registry)

View File

@ -11,8 +11,13 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
// Package text contains functions to parse and create the simple and flat // Package text contains helper functions to parse and create text-based
// text-based exchange format. // exchange formats. The package currently supports (only) version 0.0.4 of the
// exchange format. Should other versions be supported in the future, some
// versioning scheme has to be applied. Possibilities include separate packages
// or separate functions. The best way depends on the nature of future changes,
// which is the reason why no versioning scheme has been applied prematurely
// here.
package text package text
import ( import (
@ -20,6 +25,7 @@ import (
"fmt" "fmt"
"io" "io"
"strings" "strings"
"code.google.com/p/goprotobuf/proto"
dto "github.com/prometheus/client_model/go" dto "github.com/prometheus/client_model/go"
) )
@ -29,32 +35,33 @@ import (
// and any error encountered. This function does not perform checks on the // and any error encountered. This function does not perform checks on the
// content of the metric and label names, i.e. invalid metric or label names // content of the metric and label names, i.e. invalid metric or label names
// will result in invalid text format output. // will result in invalid text format output.
func MetricFamilyToText(in *dto.MetricFamily, out io.Writer) (int, error) { func MetricFamilyToText(out io.Writer, in proto.Message) (int, error) {
mf := in.(*dto.MetricFamily)
var written int var written int
// Fail-fast checks. // Fail-fast checks.
if len(in.Metric) == 0 { if len(mf.Metric) == 0 {
return written, fmt.Errorf("MetricFamily has no metrics: %s", in) return written, fmt.Errorf("MetricFamily has no metrics: %s", in)
} }
name := in.GetName() name := mf.GetName()
if name == "" { if name == "" {
return written, fmt.Errorf("MetricFamily has no name: %s", in) return written, fmt.Errorf("MetricFamily has no name: %s", in)
} }
if in.Type == nil { if mf.Type == nil {
return written, fmt.Errorf("MetricFamily has no type: %s", in) return written, fmt.Errorf("MetricFamily has no type: %s", in)
} }
// Comments, first HELP, then TYPE. // Comments, first HELP, then TYPE.
if in.Help != nil { if mf.Help != nil {
n, err := fmt.Fprintf( n, err := fmt.Fprintf(
out, "# HELP %s %s\n", out, "# HELP %s %s\n",
name, strings.Replace(*in.Help, "\n", `\n`, -1)) name, strings.Replace(*mf.Help, "\n", `\n`, -1))
written += n written += n
if err != nil { if err != nil {
return written, err return written, err
} }
} }
metricType := in.GetType() metricType := mf.GetType()
n, err := fmt.Fprintf( n, err := fmt.Fprintf(
out, "# TYPE %s %s\n", out, "# TYPE %s %s\n",
name, strings.ToLower(metricType.String()), name, strings.ToLower(metricType.String()),
@ -65,7 +72,7 @@ func MetricFamilyToText(in *dto.MetricFamily, out io.Writer) (int, error) {
} }
// Finally the samples, one line for each. // Finally the samples, one line for each.
for _, metric := range in.Metric { for _, metric := range mf.Metric {
switch metricType { switch metricType {
case dto.MetricType_COUNTER: case dto.MetricType_COUNTER:
if metric.Counter == nil { if metric.Counter == nil {

View File

@ -226,7 +226,7 @@ summary_name_count{name_1="value 1",name_2="value 2"} 4711
for i, scenario := range scenarios { for i, scenario := range scenarios {
out := bytes.NewBuffer(make([]byte, 0, len(scenario.out))) out := bytes.NewBuffer(make([]byte, 0, len(scenario.out)))
n, err := MetricFamilyToText(scenario.in, out) n, err := MetricFamilyToText(out, scenario.in)
if err != nil { if err != nil {
t.Errorf("%d. error: %s", i, err) t.Errorf("%d. error: %s", i, err)
continue continue
@ -322,7 +322,7 @@ func testCreateError(t test.Tester) {
for i, scenario := range scenarios { for i, scenario := range scenarios {
var out bytes.Buffer var out bytes.Buffer
_, err := MetricFamilyToText(scenario.in, &out) _, err := MetricFamilyToText(&out, scenario.in)
if err == nil { if err == nil {
t.Errorf("%d. expected error, got nil", i) t.Errorf("%d. expected error, got nil", i)
continue continue

33
text/proto.go Normal file
View File

@ -0,0 +1,33 @@
// Copyright 2014 Prometheus Team
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package text
import (
"fmt"
"io"
"code.google.com/p/goprotobuf/proto"
)
// WriteProtoText writes the proto.Message to the writer in text format and
// returns the number of bytes written and any error encountered.
func WriteProtoText(w io.Writer, p proto.Message) (int, error) {
return fmt.Fprintf(w, "%s\n", proto.MarshalTextString(p))
}
// WriteProtoCompactText writes the proto.Message to the writer in compact text
// format and returns the number of bytes written and any error encountered.
func WriteProtoCompactText(w io.Writer, p proto.Message) (int, error) {
return fmt.Fprintf(w, "%s\n", p)
}