From d300d5cf215ea3b986927c0a577086b494853022 Mon Sep 17 00:00:00 2001 From: stuart nelson Date: Tue, 9 May 2017 12:46:09 -0400 Subject: [PATCH] Instrument RoundTripper via middleware (#295) Instrument RoundTripper via middleware --- .travis.yml | 3 +- prometheus/promhttp/http.go | 5 + prometheus/promhttp/instrument_client.go | 95 +++++++++ prometheus/promhttp/instrument_client_1_8.go | 142 +++++++++++++ .../promhttp/instrument_client_1_8_test.go | 193 ++++++++++++++++++ prometheus/promhttp/instrument_server.go | 2 +- 6 files changed, 438 insertions(+), 2 deletions(-) create mode 100644 prometheus/promhttp/instrument_client.go create mode 100644 prometheus/promhttp/instrument_client_1_8.go create mode 100644 prometheus/promhttp/instrument_client_1_8_test.go diff --git a/.travis.yml b/.travis.yml index d9969ce..85b5115 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,7 @@ language: go go: - 1.6.3 - 1.7 + - 1.8.1 script: - - go test -short ./... + - go test -short ./... diff --git a/prometheus/promhttp/http.go b/prometheus/promhttp/http.go index 82d5656..4c70a7a 100644 --- a/prometheus/promhttp/http.go +++ b/prometheus/promhttp/http.go @@ -24,6 +24,11 @@ // via middleware. Middleware wrappers follow the naming scheme // InstrumentHandlerX, where X describes the intended use of the middleware. // See each function's doc comment for specific details. +// +// Finally, the package allows for an http.RoundTripper to be instrumented via +// middleware. Middleware wrappers follow the naming scheme +// InstrumentRoundTripperX, where X describes the intended use of the +// middleware. See each function's doc comment for specific details. package promhttp import ( diff --git a/prometheus/promhttp/instrument_client.go b/prometheus/promhttp/instrument_client.go new file mode 100644 index 0000000..1cf21f2 --- /dev/null +++ b/prometheus/promhttp/instrument_client.go @@ -0,0 +1,95 @@ +// Copyright 2017 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package promhttp + +import ( + "net/http" + "time" + + "github.com/prometheus/client_golang/prometheus" +) + +// The RoundTripperFunc type is an adapter to allow the use of ordinary +// functions as RoundTrippers. If f is a function with the appropriate +// signature, RountTripperFunc(f) is a RoundTripper that calls f. +type RoundTripperFunc func(req *http.Request) (*http.Response, error) + +// RoundTrip implements the RoundTripper interface. +func (rt RoundTripperFunc) RoundTrip(r *http.Request) (*http.Response, error) { + return rt(r) +} + +// InstrumentRoundTripperInFlight is a middleware that wraps the provided +// http.RoundTripper. It sets the provided prometheus.Gauge to the number of +// requests currently handled by the wrapped http.RoundTripper. +// +// See the example for ExampleInstrumentRoundTripperDuration for example usage. +func InstrumentRoundTripperInFlight(gauge prometheus.Gauge, next http.RoundTripper) RoundTripperFunc { + return RoundTripperFunc(func(r *http.Request) (*http.Response, error) { + gauge.Inc() + defer gauge.Dec() + return next.RoundTrip(r) + }) +} + +// InstrumentRoundTripperCounter is a middleware that wraps the provided +// http.RoundTripper to observe the request result with the provided CounterVec. +// The CounterVec must have zero, one, or two labels. The only allowed label +// names are "code" and "method". The function panics if any other instance +// labels are provided. Partitioning of the CounterVec happens by HTTP status +// code and/or HTTP method if the respective instance label names are present +// in the CounterVec. For unpartitioned counting, use a CounterVec with +// zero labels. +// +// If the wrapped RoundTripper panics or returns a non-nil error, the Counter +// is not incremented. +// +// See the example for ExampleInstrumentRoundTripperDuration for example usage. +func InstrumentRoundTripperCounter(counter *prometheus.CounterVec, next http.RoundTripper) RoundTripperFunc { + code, method := checkLabels(counter) + + return RoundTripperFunc(func(r *http.Request) (*http.Response, error) { + resp, err := next.RoundTrip(r) + if err == nil { + counter.With(labels(code, method, r.Method, resp.StatusCode)).Inc() + } + return resp, err + }) +} + +// InstrumentRoundTripperDuration is a middleware that wraps the provided +// http.RoundTripper to observe the request duration with the provided ObserverVec. +// The ObserverVec must have zero, one, or two labels. The only allowed label +// names are "code" and "method". The function panics if any other instance +// labels are provided. The Observe method of the Observer in the ObserverVec +// is called with the request duration in seconds. Partitioning happens by HTTP +// status code and/or HTTP method if the respective instance label names are +// present in the ObserverVec. For unpartitioned observations, use an +// ObserverVec with zero labels. Note that partitioning of Histograms is +// expensive and should be used judiciously. +// +// If the wrapped RoundTripper panics or returns a non-nil error, no values are +// reported. +func InstrumentRoundTripperDuration(obs prometheus.ObserverVec, next http.RoundTripper) RoundTripperFunc { + code, method := checkLabels(obs) + + return RoundTripperFunc(func(r *http.Request) (*http.Response, error) { + start := time.Now() + resp, err := next.RoundTrip(r) + if err == nil { + obs.With(labels(code, method, r.Method, resp.StatusCode)).Observe(time.Since(start).Seconds()) + } + return resp, err + }) +} diff --git a/prometheus/promhttp/instrument_client_1_8.go b/prometheus/promhttp/instrument_client_1_8.go new file mode 100644 index 0000000..b51d910 --- /dev/null +++ b/prometheus/promhttp/instrument_client_1_8.go @@ -0,0 +1,142 @@ +// Copyright 2017 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build go1.8 + +package promhttp + +import ( + "context" + "crypto/tls" + "net/http" + "net/http/httptrace" + "time" +) + +// InstrumentTrace is used to offer flexibility in instrumenting the available +// httptrace.ClientTrace hook functions. Each function is passed a float64 +// representing the time in seconds since the start of the http request. A user +// may choose to use separately buckets Histograms, or implement custom +// instance labels on a per function basis. +type InstrumentTrace struct { + GotConn func(float64) + PutIdleConn func(float64) + GotFirstResponseByte func(float64) + Got100Continue func(float64) + DNSStart func(float64) + DNSDone func(float64) + ConnectStart func(float64) + ConnectDone func(float64) + TLSHandshakeStart func(float64) + TLSHandshakeDone func(float64) + WroteHeaders func(float64) + Wait100Continue func(float64) + WroteRequest func(float64) +} + +// InstrumentRoundTripperTrace is a middleware that wraps the provided +// RoundTripper and reports times to hook functions provided in the +// InstrumentTrace struct. Hook functions that are not present in the provided +// InstrumentTrace struct are ignored. Times reported to the hook functions are +// time since the start of the request. Note that partitioning of Histograms +// is expensive and should be used judiciously. +// +// For hook functions that receive an error as an argument, no observations are +// made in the event of a non-nil error value. +// +// See the example for ExampleInstrumentRoundTripperDuration for example usage. +func InstrumentRoundTripperTrace(it *InstrumentTrace, next http.RoundTripper) RoundTripperFunc { + return RoundTripperFunc(func(r *http.Request) (*http.Response, error) { + start := time.Now() + + trace := &httptrace.ClientTrace{ + GotConn: func(_ httptrace.GotConnInfo) { + if it.GotConn != nil { + it.GotConn(time.Since(start).Seconds()) + } + }, + PutIdleConn: func(err error) { + if err != nil { + return + } + if it.PutIdleConn != nil { + it.PutIdleConn(time.Since(start).Seconds()) + } + }, + DNSStart: func(_ httptrace.DNSStartInfo) { + if it.DNSStart != nil { + it.DNSStart(time.Since(start).Seconds()) + } + }, + DNSDone: func(_ httptrace.DNSDoneInfo) { + if it.DNSStart != nil { + it.DNSStart(time.Since(start).Seconds()) + } + }, + ConnectStart: func(_, _ string) { + if it.ConnectStart != nil { + it.ConnectStart(time.Since(start).Seconds()) + } + }, + ConnectDone: func(_, _ string, err error) { + if err != nil { + return + } + if it.ConnectDone != nil { + it.ConnectDone(time.Since(start).Seconds()) + } + }, + GotFirstResponseByte: func() { + if it.GotFirstResponseByte != nil { + it.GotFirstResponseByte(time.Since(start).Seconds()) + } + }, + Got100Continue: func() { + if it.Got100Continue != nil { + it.Got100Continue(time.Since(start).Seconds()) + } + }, + TLSHandshakeStart: func() { + if it.TLSHandshakeStart != nil { + it.TLSHandshakeStart(time.Since(start).Seconds()) + } + }, + TLSHandshakeDone: func(_ tls.ConnectionState, err error) { + if err != nil { + return + } + if it.TLSHandshakeDone != nil { + it.TLSHandshakeDone(time.Since(start).Seconds()) + } + }, + WroteHeaders: func() { + if it.WroteHeaders != nil { + it.WroteHeaders(time.Since(start).Seconds()) + } + }, + Wait100Continue: func() { + if it.Wait100Continue != nil { + it.Wait100Continue(time.Since(start).Seconds()) + } + }, + WroteRequest: func(_ httptrace.WroteRequestInfo) { + if it.WroteRequest != nil { + it.WroteRequest(time.Since(start).Seconds()) + } + }, + } + r = r.WithContext(httptrace.WithClientTrace(context.Background(), trace)) + + return next.RoundTrip(r) + }) +} diff --git a/prometheus/promhttp/instrument_client_1_8_test.go b/prometheus/promhttp/instrument_client_1_8_test.go new file mode 100644 index 0000000..235a3d2 --- /dev/null +++ b/prometheus/promhttp/instrument_client_1_8_test.go @@ -0,0 +1,193 @@ +// Copyright 2017 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build go1.8 + +package promhttp + +import ( + "log" + "net/http" + "testing" + "time" + + "github.com/prometheus/client_golang/prometheus" +) + +func TestClientMiddlewareAPI(t *testing.T) { + client := http.DefaultClient + client.Timeout = 1 * time.Second + + inFlightGauge := prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "client_in_flight_requests", + Help: "A gauge of in-flight requests for the wrapped client.", + }) + + counter := prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "client_api_requests_total", + Help: "A counter for requests from the wrapped client.", + }, + []string{"code", "method"}, + ) + + dnsLatencyVec := prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "dns_duration_seconds", + Help: "Trace dns latency histogram.", + Buckets: []float64{.005, .01, .025, .05}, + }, + []string{"event"}, + ) + + tlsLatencyVec := prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "tls_duration_seconds", + Help: "Trace tls latency histogram.", + Buckets: []float64{.05, .1, .25, .5}, + }, + []string{"event"}, + ) + + histVec := prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "request_duration_seconds", + Help: "A histogram of request latencies.", + Buckets: prometheus.DefBuckets, + }, + []string{"method"}, + ) + + prometheus.MustRegister(counter, tlsLatencyVec, dnsLatencyVec, histVec, inFlightGauge) + + trace := &InstrumentTrace{ + DNSStart: func(t float64) { + dnsLatencyVec.WithLabelValues("dns_start") + }, + DNSDone: func(t float64) { + dnsLatencyVec.WithLabelValues("dns_done") + }, + TLSHandshakeStart: func(t float64) { + tlsLatencyVec.WithLabelValues("tls_handshake_start") + }, + TLSHandshakeDone: func(t float64) { + tlsLatencyVec.WithLabelValues("tls_handshake_done") + }, + } + + client.Transport = InstrumentRoundTripperInFlight(inFlightGauge, + InstrumentRoundTripperCounter(counter, + InstrumentRoundTripperTrace(trace, + InstrumentRoundTripperDuration(histVec, http.DefaultTransport), + ), + ), + ) + + resp, err := client.Get("http://google.com") + if err != nil { + t.Fatalf("%v", err) + } + defer resp.Body.Close() +} + +func ExampleInstrumentRoundTripperDuration() { + client := http.DefaultClient + client.Timeout = 1 * time.Second + + inFlightGauge := prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "client_in_flight_requests", + Help: "A gauge of in-flight requests for the wrapped client.", + }) + + counter := prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "client_api_requests_total", + Help: "A counter for requests from the wrapped client.", + }, + []string{"code", "method"}, + ) + + // dnsLatencyVec uses custom buckets based on expected dns durations. + // It has an instance label "event", which is set in the + // DNSStart and DNSDonehook functions defined in the + // InstrumentTrace struct below. + dnsLatencyVec := prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "dns_duration_seconds", + Help: "Trace dns latency histogram.", + Buckets: []float64{.005, .01, .025, .05}, + }, + []string{"event"}, + ) + + // tlsLatencyVec uses custom buckets based on expected tls durations. + // It has an instance label "event", which is set in the + // TLSHandshakeStart and TLSHandshakeDone hook functions defined in the + // InstrumentTrace struct below. + tlsLatencyVec := prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "tls_duration_seconds", + Help: "Trace tls latency histogram.", + Buckets: []float64{.05, .1, .25, .5}, + }, + []string{"event"}, + ) + + // histVec has no labels, making it a zero-dimensional ObserverVec. + histVec := prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "request_duration_seconds", + Help: "A histogram of request latencies.", + Buckets: prometheus.DefBuckets, + }, + []string{}, + ) + + // Register all of the metrics in the standard registry. + prometheus.MustRegister(counter, tlsLatencyVec, dnsLatencyVec, histVec, inFlightGauge) + + // Define functions for the available httptrace.ClientTrace hook + // functions that we want to instrument. + trace := &InstrumentTrace{ + DNSStart: func(t float64) { + dnsLatencyVec.WithLabelValues("dns_start") + }, + DNSDone: func(t float64) { + dnsLatencyVec.WithLabelValues("dns_done") + }, + TLSHandshakeStart: func(t float64) { + tlsLatencyVec.WithLabelValues("tls_handshake_start") + }, + TLSHandshakeDone: func(t float64) { + tlsLatencyVec.WithLabelValues("tls_handshake_done") + }, + } + + // Wrap the default RoundTripper with middleware. + roundTripper := InstrumentRoundTripperInFlight(inFlightGauge, + InstrumentRoundTripperCounter(counter, + InstrumentRoundTripperTrace(trace, + InstrumentRoundTripperDuration(histVec, http.DefaultTransport), + ), + ), + ) + + // Set the RoundTripper on our client. + client.Transport = roundTripper + + resp, err := client.Get("http://google.com") + if err != nil { + log.Printf("error: %v", err) + } + defer resp.Body.Close() +} diff --git a/prometheus/promhttp/instrument_server.go b/prometheus/promhttp/instrument_server.go index 31e15cb..193598d 100644 --- a/prometheus/promhttp/instrument_server.go +++ b/prometheus/promhttp/instrument_server.go @@ -80,7 +80,7 @@ func InstrumentHandlerDuration(obs prometheus.ObserverVec, next http.Handler) ht // names are "code" and "method". The function panics if any other instance // labels are provided. Partitioning of the CounterVec happens by HTTP status // code and/or HTTP method if the respective instance label names are present -// in the CounterVec. For unpartitioned observations, use a CounterVec with +// in the CounterVec. For unpartitioned counting, use a CounterVec with // zero labels. // // If the wrapped Handler does not set a status code, a status code of 200 is assumed.