From 84dc53148d6abf392427cb334e15cdac834d651a Mon Sep 17 00:00:00 2001 From: Bjoern Rabenstein Date: Wed, 23 Apr 2014 13:40:37 +0200 Subject: [PATCH] Enable the Golang client library to create the new text formats. Most important here is the simple & flat text format, but while I'm on it, I have also added the text representations for protobufs (which is purely meant for debugging purposes). I hope my basic idea about handling those various protocols (and the text package) becomes clearer now. Change-Id: I7299853eadc82a426101e907f2b3d4e37f9e4c71 --- prometheus/constants.go | 25 +++-- prometheus/registry.go | 62 +++++++----- prometheus/registry_test.go | 181 ++++++++++++++++++++++++++++++++++-- text/create.go | 27 ++++-- text/create_test.go | 4 +- text/proto.go | 33 +++++++ 6 files changed, 283 insertions(+), 49 deletions(-) create mode 100644 text/proto.go diff --git a/prometheus/constants.go b/prometheus/constants.go index 385b467..436daaf 100644 --- a/prometheus/constants.go +++ b/prometheus/constants.go @@ -26,14 +26,27 @@ const ( // APIVersion is the version of the format of the exported data. This // will match this library's version, which subscribes to the Semantic // Versioning scheme. - APIVersion = "0.0.2" + APIVersion = "0.0.4" - // TelemetryContentType is the content type and schema information set - // on telemetry data responses. - TelemetryContentType = `application/json; schema="prometheus/telemetry"; version=` + APIVersion - // DelimitedTelemetryContentType is the content type and schema - // information set on telemetry data responses. + // JSONAPIVersion is the version of the JSON export format. + JSONAPIVersion = "0.0.2" + + // DelimitedTelemetryContentType is the content type set on telemetry + // data responses in delimited protobuf format. DelimitedTelemetryContentType = `application/vnd.google.protobuf; proto="io.prometheus.client.MetricFamily"; encoding="delimited"` + // TextTelemetryContentType is the content type set on telemetry data + // responses in text format. + TextTelemetryContentType = `text/plain; version=` + APIVersion + // ProtoTextTelemetryContentType is the content type set on telemetry + // data responses in protobuf text format. (Only used for debugging.) + ProtoTextTelemetryContentType = `application/vnd.google.protobuf; proto="io.prometheus.client.MetricFamily"; encoding="text"` + // ProtoCompactTextTelemetryContentType is the content type set on + // telemetry data responses in protobuf compact text format. (Only used + // for debugging.) + ProtoCompactTextTelemetryContentType = `application/vnd.google.protobuf; proto="io.prometheus.client.MetricFamily"; encoding="compact-text"` + // JSONTelemetryContentType is the content type set on telemetry data + // responses formatted as JSON. + JSONTelemetryContentType = `application/json; schema="prometheus/telemetry"; version=` + JSONAPIVersion // ExpositionResource is the customary web services endpoint on which // telemetric data is exposed. diff --git a/prometheus/registry.go b/prometheus/registry.go index 31b06f8..e128b77 100644 --- a/prometheus/registry.go +++ b/prometheus/registry.go @@ -26,6 +26,7 @@ import ( "github.com/matttproud/golang_protobuf_extensions/ext" "github.com/prometheus/client_golang/model" + "github.com/prometheus/client_golang/text" "github.com/prometheus/client_golang/vendor/goautoneg" ) @@ -42,6 +43,12 @@ const ( jsonContentType = "application/json" ) +// encoder is a function that writes a proto.Message to an io.Writer in a +// certain encoding. It returns the number of bytes written and any error +// encountered. Note that ext.WriteDelimited and text.MetricFamilyToText are +// encoders. +type encoder func(io.Writer, proto.Message) (int, error) + // container represents a top-level registered metric that encompasses its // static metadata. type container struct { @@ -267,7 +274,7 @@ func (r *registry) YieldExporter() http.HandlerFunc { return r.Handler() } -func (r *registry) dumpDelimitedPB(w io.Writer) { +func (r *registry) dumpPB(w io.Writer, writeEncoded encoder) { r.mutex.RLock() defer r.mutex.RUnlock() @@ -298,16 +305,16 @@ func (r *registry) dumpDelimitedPB(w io.Writer) { } } - ext.WriteDelimited(w, f) + writeEncoded(w, f) } } -func (r *registry) dumpDelimitedExternalPB(w io.Writer) { +func (r *registry) dumpExternalPB(w io.Writer, writeEncoded encoder) { if r.metricFamilyInjectionHook == nil { return } for _, f := range r.metricFamilyInjectionHook() { - ext.WriteDelimited(w, f) + writeEncoded(w, f) } } @@ -326,26 +333,39 @@ func (r *registry) Handler() http.HandlerFunc { accepts := goautoneg.ParseAccept(req.Header.Get("Accept")) for _, accept := range accepts { - if accept.Type != "application" { + var enc encoder + switch { + case accept.Type == "application" && + accept.SubType == "vnd.google.protobuf" && + accept.Params["proto"] == "io.prometheus.client.MetricFamily": + switch accept.Params["encoding"] { + case "delimited": + header.Set(contentTypeHeader, DelimitedTelemetryContentType) + enc = ext.WriteDelimited + case "text": + header.Set(contentTypeHeader, ProtoTextTelemetryContentType) + enc = text.WriteProtoText + case "compact-text": + header.Set(contentTypeHeader, ProtoCompactTextTelemetryContentType) + enc = text.WriteProtoCompactText + default: + continue + } + case accept.Type == "text" && + accept.SubType == "plain" && + (accept.Params["version"] == "0.0.4" || accept.Params["version"] == ""): + header.Set(contentTypeHeader, TextTelemetryContentType) + enc = text.MetricFamilyToText + default: continue } - - if accept.SubType == "vnd.google.protobuf" { - if accept.Params["proto"] != "io.prometheus.client.MetricFamily" { - continue - } - if accept.Params["encoding"] != "delimited" { - continue - } - - header.Set(contentTypeHeader, DelimitedTelemetryContentType) - r.dumpDelimitedPB(writer) - r.dumpDelimitedExternalPB(writer) - return - } + r.dumpPB(writer, enc) + r.dumpExternalPB(writer, enc) + return } - - header.Set(contentTypeHeader, TelemetryContentType) + // TODO: Once JSON deprecation is completed, use text format as + // fall-back. + header.Set(contentTypeHeader, JSONTelemetryContentType) json.NewEncoder(writer).Encode(r) } } diff --git a/prometheus/registry_test.go b/prometheus/registry_test.go index 85cab7c..a070d26 100644 --- a/prometheus/registry_test.go +++ b/prometheus/registry_test.go @@ -253,6 +253,30 @@ func testHandler(t test.Tester) { t.Fatal(err) } externalMetricFamilyAsBytes := externalBuf.Bytes() + externalMetricFamilyAsText := []byte(`# HELP externalname externaldocstring +# TYPE externalname counter +externalname{externallabelname="externalval1",externalbasename="externalbasevalue"} 1 +`) + externalMetricFamilyAsProtoText := []byte(`name: "externalname" +help: "externaldocstring" +type: COUNTER +metric: < + label: < + name: "externallabelname" + value: "externalval1" + > + label: < + name: "externalbasename" + value: "externalbasevalue" + > + counter: < + value: 1 + > +> + +`) + externalMetricFamilyAsProtoCompactText := []byte(`name:"externalname" help:"externaldocstring" type:COUNTER metric: label: counter: > +`) expectedMetricFamily := &dto.MetricFamily{ Name: proto.String("name"), @@ -306,6 +330,44 @@ func testHandler(t test.Tester) { t.Fatal(err) } expectedMetricFamilyAsBytes := buf.Bytes() + expectedMetricFamilyAsText := []byte(`# HELP name docstring +# TYPE name counter +name{labelname="val1",basename="basevalue"} 1 +name{labelname="val2",basename="basevalue"} 1 +`) + expectedMetricFamilyAsProtoText := []byte(`name: "name" +help: "docstring" +type: COUNTER +metric: < + label: < + name: "labelname" + value: "val1" + > + label: < + name: "basename" + value: "basevalue" + > + counter: < + value: 1 + > +> +metric: < + label: < + name: "labelname" + value: "val2" + > + label: < + name: "basename" + value: "basevalue" + > + counter: < + value: 1 + > +> + +`) + expectedMetricFamilyAsProtoCompactText := []byte(`name:"name" help:"docstring" type:COUNTER metric: label: counter: > metric: label: counter: > +`) type output struct { headers map[string]string @@ -318,7 +380,7 @@ func testHandler(t test.Tester) { withCounter bool withExternalMF bool }{ - { + { // 0 headers: map[string]string{ "Accept": "foo/bar;q=0.2, dings/bums;q=0.8", }, @@ -329,7 +391,7 @@ func testHandler(t test.Tester) { body: []byte("[]\n"), }, }, - { + { // 1 headers: map[string]string{ "Accept": "foo/bar;q=0.2, application/quark;q=0.8", }, @@ -340,7 +402,7 @@ func testHandler(t test.Tester) { body: []byte("[]\n"), }, }, - { + { // 2 headers: map[string]string{ "Accept": "foo/bar;q=0.2, application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=bla;q=0.8", }, @@ -351,9 +413,9 @@ func testHandler(t test.Tester) { body: []byte("[]\n"), }, }, - { + { // 3 headers: map[string]string{ - "Accept": "foo/bar;q=0.2, application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited;q=0.8", + "Accept": "text/plain;q=0.2, application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited;q=0.8", }, out: output{ headers: map[string]string{ @@ -362,7 +424,7 @@ func testHandler(t test.Tester) { body: []byte{}, }, }, - { + { // 4 headers: map[string]string{ "Accept": "application/json", }, @@ -375,7 +437,7 @@ func testHandler(t test.Tester) { }, withCounter: true, }, - { + { // 5 headers: map[string]string{ "Accept": "application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited", }, @@ -387,7 +449,7 @@ func testHandler(t test.Tester) { }, withCounter: true, }, - { + { // 6 headers: map[string]string{ "Accept": "application/json", }, @@ -399,7 +461,7 @@ func testHandler(t test.Tester) { }, withExternalMF: true, }, - { + { // 7 headers: map[string]string{ "Accept": "application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited", }, @@ -411,7 +473,7 @@ func testHandler(t test.Tester) { }, withExternalMF: true, }, - { + { // 8 headers: map[string]string{ "Accept": "application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited", }, @@ -430,6 +492,105 @@ func testHandler(t test.Tester) { withCounter: true, withExternalMF: true, }, + { // 9 + headers: map[string]string{ + "Accept": "text/plain", + }, + out: output{ + headers: map[string]string{ + "Content-Type": `text/plain; version=0.0.4`, + }, + body: []byte{}, + }, + }, + { // 10 + headers: map[string]string{ + "Accept": "application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=bla;q=0.2, text/plain;q=0.5", + }, + out: output{ + headers: map[string]string{ + "Content-Type": `text/plain; version=0.0.4`, + }, + body: expectedMetricFamilyAsText, + }, + withCounter: true, + }, + { // 11 + headers: map[string]string{ + "Accept": "application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=bla;q=0.2, text/plain;q=0.5;version=0.0.4", + }, + out: output{ + headers: map[string]string{ + "Content-Type": `text/plain; version=0.0.4`, + }, + body: bytes.Join( + [][]byte{ + expectedMetricFamilyAsText, + externalMetricFamilyAsText, + }, + []byte{}, + ), + }, + withCounter: true, + withExternalMF: true, + }, + { // 12 + headers: map[string]string{ + "Accept": "application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited;q=0.2, text/plain;q=0.5;version=0.0.2", + }, + out: output{ + headers: map[string]string{ + "Content-Type": `application/vnd.google.protobuf; proto="io.prometheus.client.MetricFamily"; encoding="delimited"`, + }, + body: bytes.Join( + [][]byte{ + expectedMetricFamilyAsBytes, + externalMetricFamilyAsBytes, + }, + []byte{}, + ), + }, + withCounter: true, + withExternalMF: true, + }, + { // 13 + headers: map[string]string{ + "Accept": "application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=text;q=0.5, application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited;q=0.4", + }, + out: output{ + headers: map[string]string{ + "Content-Type": `application/vnd.google.protobuf; proto="io.prometheus.client.MetricFamily"; encoding="text"`, + }, + body: bytes.Join( + [][]byte{ + expectedMetricFamilyAsProtoText, + externalMetricFamilyAsProtoText, + }, + []byte{}, + ), + }, + withCounter: true, + withExternalMF: true, + }, + { // 14 + headers: map[string]string{ + "Accept": "application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=compact-text", + }, + out: output{ + headers: map[string]string{ + "Content-Type": `application/vnd.google.protobuf; proto="io.prometheus.client.MetricFamily"; encoding="compact-text"`, + }, + body: bytes.Join( + [][]byte{ + expectedMetricFamilyAsProtoCompactText, + externalMetricFamilyAsProtoCompactText, + }, + []byte{}, + ), + }, + withCounter: true, + withExternalMF: true, + }, } for i, scenario := range scenarios { registry := NewRegistry().(*registry) diff --git a/text/create.go b/text/create.go index d39eee9..3f28871 100644 --- a/text/create.go +++ b/text/create.go @@ -11,8 +11,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package text contains functions to parse and create the simple and flat -// text-based exchange format. +// Package text contains helper functions to parse and create text-based +// exchange formats. The package currently supports (only) version 0.0.4 of the +// exchange format. Should other versions be supported in the future, some +// versioning scheme has to be applied. Possibilities include separate packages +// or separate functions. The best way depends on the nature of future changes, +// which is the reason why no versioning scheme has been applied prematurely +// here. package text import ( @@ -20,6 +25,7 @@ import ( "fmt" "io" "strings" + "code.google.com/p/goprotobuf/proto" dto "github.com/prometheus/client_model/go" ) @@ -29,32 +35,33 @@ import ( // and any error encountered. This function does not perform checks on the // content of the metric and label names, i.e. invalid metric or label names // will result in invalid text format output. -func MetricFamilyToText(in *dto.MetricFamily, out io.Writer) (int, error) { +func MetricFamilyToText(out io.Writer, in proto.Message) (int, error) { + mf := in.(*dto.MetricFamily) var written int // Fail-fast checks. - if len(in.Metric) == 0 { + if len(mf.Metric) == 0 { return written, fmt.Errorf("MetricFamily has no metrics: %s", in) } - name := in.GetName() + name := mf.GetName() if name == "" { return written, fmt.Errorf("MetricFamily has no name: %s", in) } - if in.Type == nil { + if mf.Type == nil { return written, fmt.Errorf("MetricFamily has no type: %s", in) } // Comments, first HELP, then TYPE. - if in.Help != nil { + if mf.Help != nil { n, err := fmt.Fprintf( out, "# HELP %s %s\n", - name, strings.Replace(*in.Help, "\n", `\n`, -1)) + name, strings.Replace(*mf.Help, "\n", `\n`, -1)) written += n if err != nil { return written, err } } - metricType := in.GetType() + metricType := mf.GetType() n, err := fmt.Fprintf( out, "# TYPE %s %s\n", name, strings.ToLower(metricType.String()), @@ -65,7 +72,7 @@ func MetricFamilyToText(in *dto.MetricFamily, out io.Writer) (int, error) { } // Finally the samples, one line for each. - for _, metric := range in.Metric { + for _, metric := range mf.Metric { switch metricType { case dto.MetricType_COUNTER: if metric.Counter == nil { diff --git a/text/create_test.go b/text/create_test.go index 9c8d10e..c08d421 100644 --- a/text/create_test.go +++ b/text/create_test.go @@ -226,7 +226,7 @@ summary_name_count{name_1="value 1",name_2="value 2"} 4711 for i, scenario := range scenarios { out := bytes.NewBuffer(make([]byte, 0, len(scenario.out))) - n, err := MetricFamilyToText(scenario.in, out) + n, err := MetricFamilyToText(out, scenario.in) if err != nil { t.Errorf("%d. error: %s", i, err) continue @@ -322,7 +322,7 @@ func testCreateError(t test.Tester) { for i, scenario := range scenarios { var out bytes.Buffer - _, err := MetricFamilyToText(scenario.in, &out) + _, err := MetricFamilyToText(&out, scenario.in) if err == nil { t.Errorf("%d. expected error, got nil", i) continue diff --git a/text/proto.go b/text/proto.go new file mode 100644 index 0000000..3edd9d0 --- /dev/null +++ b/text/proto.go @@ -0,0 +1,33 @@ +// Copyright 2014 Prometheus Team +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package text + +import ( + "fmt" + "io" + + "code.google.com/p/goprotobuf/proto" +) + +// WriteProtoText writes the proto.Message to the writer in text format and +// returns the number of bytes written and any error encountered. +func WriteProtoText(w io.Writer, p proto.Message) (int, error) { + return fmt.Fprintf(w, "%s\n", proto.MarshalTextString(p)) +} + +// WriteProtoCompactText writes the proto.Message to the writer in compact text +// format and returns the number of bytes written and any error encountered. +func WriteProtoCompactText(w io.Writer, p proto.Message) (int, error) { + return fmt.Fprintf(w, "%s\n", p) +}