Instrument RoundTripper via middleware (#295)

Instrument RoundTripper via middleware
This commit is contained in:
stuart nelson 2017-05-09 12:46:09 -04:00 committed by Björn Rabenstein
parent 7d9484283e
commit d300d5cf21
6 changed files with 438 additions and 2 deletions

View File

@ -4,6 +4,7 @@ language: go
go: go:
- 1.6.3 - 1.6.3
- 1.7 - 1.7
- 1.8.1
script: script:
- go test -short ./... - go test -short ./...

View File

@ -24,6 +24,11 @@
// via middleware. Middleware wrappers follow the naming scheme // via middleware. Middleware wrappers follow the naming scheme
// InstrumentHandlerX, where X describes the intended use of the middleware. // InstrumentHandlerX, where X describes the intended use of the middleware.
// See each function's doc comment for specific details. // See each function's doc comment for specific details.
//
// Finally, the package allows for an http.RoundTripper to be instrumented via
// middleware. Middleware wrappers follow the naming scheme
// InstrumentRoundTripperX, where X describes the intended use of the
// middleware. See each function's doc comment for specific details.
package promhttp package promhttp
import ( import (

View File

@ -0,0 +1,95 @@
// Copyright 2017 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package promhttp
import (
"net/http"
"time"
"github.com/prometheus/client_golang/prometheus"
)
// The RoundTripperFunc type is an adapter to allow the use of ordinary
// functions as RoundTrippers. If f is a function with the appropriate
// signature, RountTripperFunc(f) is a RoundTripper that calls f.
type RoundTripperFunc func(req *http.Request) (*http.Response, error)
// RoundTrip implements the RoundTripper interface.
func (rt RoundTripperFunc) RoundTrip(r *http.Request) (*http.Response, error) {
return rt(r)
}
// InstrumentRoundTripperInFlight is a middleware that wraps the provided
// http.RoundTripper. It sets the provided prometheus.Gauge to the number of
// requests currently handled by the wrapped http.RoundTripper.
//
// See the example for ExampleInstrumentRoundTripperDuration for example usage.
func InstrumentRoundTripperInFlight(gauge prometheus.Gauge, next http.RoundTripper) RoundTripperFunc {
return RoundTripperFunc(func(r *http.Request) (*http.Response, error) {
gauge.Inc()
defer gauge.Dec()
return next.RoundTrip(r)
})
}
// InstrumentRoundTripperCounter is a middleware that wraps the provided
// http.RoundTripper to observe the request result with the provided CounterVec.
// The CounterVec must have zero, one, or two labels. The only allowed label
// names are "code" and "method". The function panics if any other instance
// labels are provided. Partitioning of the CounterVec happens by HTTP status
// code and/or HTTP method if the respective instance label names are present
// in the CounterVec. For unpartitioned counting, use a CounterVec with
// zero labels.
//
// If the wrapped RoundTripper panics or returns a non-nil error, the Counter
// is not incremented.
//
// See the example for ExampleInstrumentRoundTripperDuration for example usage.
func InstrumentRoundTripperCounter(counter *prometheus.CounterVec, next http.RoundTripper) RoundTripperFunc {
code, method := checkLabels(counter)
return RoundTripperFunc(func(r *http.Request) (*http.Response, error) {
resp, err := next.RoundTrip(r)
if err == nil {
counter.With(labels(code, method, r.Method, resp.StatusCode)).Inc()
}
return resp, err
})
}
// InstrumentRoundTripperDuration is a middleware that wraps the provided
// http.RoundTripper to observe the request duration with the provided ObserverVec.
// The ObserverVec must have zero, one, or two labels. The only allowed label
// names are "code" and "method". The function panics if any other instance
// labels are provided. The Observe method of the Observer in the ObserverVec
// is called with the request duration in seconds. Partitioning happens by HTTP
// status code and/or HTTP method if the respective instance label names are
// present in the ObserverVec. For unpartitioned observations, use an
// ObserverVec with zero labels. Note that partitioning of Histograms is
// expensive and should be used judiciously.
//
// If the wrapped RoundTripper panics or returns a non-nil error, no values are
// reported.
func InstrumentRoundTripperDuration(obs prometheus.ObserverVec, next http.RoundTripper) RoundTripperFunc {
code, method := checkLabels(obs)
return RoundTripperFunc(func(r *http.Request) (*http.Response, error) {
start := time.Now()
resp, err := next.RoundTrip(r)
if err == nil {
obs.With(labels(code, method, r.Method, resp.StatusCode)).Observe(time.Since(start).Seconds())
}
return resp, err
})
}

View File

@ -0,0 +1,142 @@
// Copyright 2017 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +build go1.8
package promhttp
import (
"context"
"crypto/tls"
"net/http"
"net/http/httptrace"
"time"
)
// InstrumentTrace is used to offer flexibility in instrumenting the available
// httptrace.ClientTrace hook functions. Each function is passed a float64
// representing the time in seconds since the start of the http request. A user
// may choose to use separately buckets Histograms, or implement custom
// instance labels on a per function basis.
type InstrumentTrace struct {
GotConn func(float64)
PutIdleConn func(float64)
GotFirstResponseByte func(float64)
Got100Continue func(float64)
DNSStart func(float64)
DNSDone func(float64)
ConnectStart func(float64)
ConnectDone func(float64)
TLSHandshakeStart func(float64)
TLSHandshakeDone func(float64)
WroteHeaders func(float64)
Wait100Continue func(float64)
WroteRequest func(float64)
}
// InstrumentRoundTripperTrace is a middleware that wraps the provided
// RoundTripper and reports times to hook functions provided in the
// InstrumentTrace struct. Hook functions that are not present in the provided
// InstrumentTrace struct are ignored. Times reported to the hook functions are
// time since the start of the request. Note that partitioning of Histograms
// is expensive and should be used judiciously.
//
// For hook functions that receive an error as an argument, no observations are
// made in the event of a non-nil error value.
//
// See the example for ExampleInstrumentRoundTripperDuration for example usage.
func InstrumentRoundTripperTrace(it *InstrumentTrace, next http.RoundTripper) RoundTripperFunc {
return RoundTripperFunc(func(r *http.Request) (*http.Response, error) {
start := time.Now()
trace := &httptrace.ClientTrace{
GotConn: func(_ httptrace.GotConnInfo) {
if it.GotConn != nil {
it.GotConn(time.Since(start).Seconds())
}
},
PutIdleConn: func(err error) {
if err != nil {
return
}
if it.PutIdleConn != nil {
it.PutIdleConn(time.Since(start).Seconds())
}
},
DNSStart: func(_ httptrace.DNSStartInfo) {
if it.DNSStart != nil {
it.DNSStart(time.Since(start).Seconds())
}
},
DNSDone: func(_ httptrace.DNSDoneInfo) {
if it.DNSStart != nil {
it.DNSStart(time.Since(start).Seconds())
}
},
ConnectStart: func(_, _ string) {
if it.ConnectStart != nil {
it.ConnectStart(time.Since(start).Seconds())
}
},
ConnectDone: func(_, _ string, err error) {
if err != nil {
return
}
if it.ConnectDone != nil {
it.ConnectDone(time.Since(start).Seconds())
}
},
GotFirstResponseByte: func() {
if it.GotFirstResponseByte != nil {
it.GotFirstResponseByte(time.Since(start).Seconds())
}
},
Got100Continue: func() {
if it.Got100Continue != nil {
it.Got100Continue(time.Since(start).Seconds())
}
},
TLSHandshakeStart: func() {
if it.TLSHandshakeStart != nil {
it.TLSHandshakeStart(time.Since(start).Seconds())
}
},
TLSHandshakeDone: func(_ tls.ConnectionState, err error) {
if err != nil {
return
}
if it.TLSHandshakeDone != nil {
it.TLSHandshakeDone(time.Since(start).Seconds())
}
},
WroteHeaders: func() {
if it.WroteHeaders != nil {
it.WroteHeaders(time.Since(start).Seconds())
}
},
Wait100Continue: func() {
if it.Wait100Continue != nil {
it.Wait100Continue(time.Since(start).Seconds())
}
},
WroteRequest: func(_ httptrace.WroteRequestInfo) {
if it.WroteRequest != nil {
it.WroteRequest(time.Since(start).Seconds())
}
},
}
r = r.WithContext(httptrace.WithClientTrace(context.Background(), trace))
return next.RoundTrip(r)
})
}

View File

@ -0,0 +1,193 @@
// Copyright 2017 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +build go1.8
package promhttp
import (
"log"
"net/http"
"testing"
"time"
"github.com/prometheus/client_golang/prometheus"
)
func TestClientMiddlewareAPI(t *testing.T) {
client := http.DefaultClient
client.Timeout = 1 * time.Second
inFlightGauge := prometheus.NewGauge(prometheus.GaugeOpts{
Name: "client_in_flight_requests",
Help: "A gauge of in-flight requests for the wrapped client.",
})
counter := prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "client_api_requests_total",
Help: "A counter for requests from the wrapped client.",
},
[]string{"code", "method"},
)
dnsLatencyVec := prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Name: "dns_duration_seconds",
Help: "Trace dns latency histogram.",
Buckets: []float64{.005, .01, .025, .05},
},
[]string{"event"},
)
tlsLatencyVec := prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Name: "tls_duration_seconds",
Help: "Trace tls latency histogram.",
Buckets: []float64{.05, .1, .25, .5},
},
[]string{"event"},
)
histVec := prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Name: "request_duration_seconds",
Help: "A histogram of request latencies.",
Buckets: prometheus.DefBuckets,
},
[]string{"method"},
)
prometheus.MustRegister(counter, tlsLatencyVec, dnsLatencyVec, histVec, inFlightGauge)
trace := &InstrumentTrace{
DNSStart: func(t float64) {
dnsLatencyVec.WithLabelValues("dns_start")
},
DNSDone: func(t float64) {
dnsLatencyVec.WithLabelValues("dns_done")
},
TLSHandshakeStart: func(t float64) {
tlsLatencyVec.WithLabelValues("tls_handshake_start")
},
TLSHandshakeDone: func(t float64) {
tlsLatencyVec.WithLabelValues("tls_handshake_done")
},
}
client.Transport = InstrumentRoundTripperInFlight(inFlightGauge,
InstrumentRoundTripperCounter(counter,
InstrumentRoundTripperTrace(trace,
InstrumentRoundTripperDuration(histVec, http.DefaultTransport),
),
),
)
resp, err := client.Get("http://google.com")
if err != nil {
t.Fatalf("%v", err)
}
defer resp.Body.Close()
}
func ExampleInstrumentRoundTripperDuration() {
client := http.DefaultClient
client.Timeout = 1 * time.Second
inFlightGauge := prometheus.NewGauge(prometheus.GaugeOpts{
Name: "client_in_flight_requests",
Help: "A gauge of in-flight requests for the wrapped client.",
})
counter := prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "client_api_requests_total",
Help: "A counter for requests from the wrapped client.",
},
[]string{"code", "method"},
)
// dnsLatencyVec uses custom buckets based on expected dns durations.
// It has an instance label "event", which is set in the
// DNSStart and DNSDonehook functions defined in the
// InstrumentTrace struct below.
dnsLatencyVec := prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Name: "dns_duration_seconds",
Help: "Trace dns latency histogram.",
Buckets: []float64{.005, .01, .025, .05},
},
[]string{"event"},
)
// tlsLatencyVec uses custom buckets based on expected tls durations.
// It has an instance label "event", which is set in the
// TLSHandshakeStart and TLSHandshakeDone hook functions defined in the
// InstrumentTrace struct below.
tlsLatencyVec := prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Name: "tls_duration_seconds",
Help: "Trace tls latency histogram.",
Buckets: []float64{.05, .1, .25, .5},
},
[]string{"event"},
)
// histVec has no labels, making it a zero-dimensional ObserverVec.
histVec := prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Name: "request_duration_seconds",
Help: "A histogram of request latencies.",
Buckets: prometheus.DefBuckets,
},
[]string{},
)
// Register all of the metrics in the standard registry.
prometheus.MustRegister(counter, tlsLatencyVec, dnsLatencyVec, histVec, inFlightGauge)
// Define functions for the available httptrace.ClientTrace hook
// functions that we want to instrument.
trace := &InstrumentTrace{
DNSStart: func(t float64) {
dnsLatencyVec.WithLabelValues("dns_start")
},
DNSDone: func(t float64) {
dnsLatencyVec.WithLabelValues("dns_done")
},
TLSHandshakeStart: func(t float64) {
tlsLatencyVec.WithLabelValues("tls_handshake_start")
},
TLSHandshakeDone: func(t float64) {
tlsLatencyVec.WithLabelValues("tls_handshake_done")
},
}
// Wrap the default RoundTripper with middleware.
roundTripper := InstrumentRoundTripperInFlight(inFlightGauge,
InstrumentRoundTripperCounter(counter,
InstrumentRoundTripperTrace(trace,
InstrumentRoundTripperDuration(histVec, http.DefaultTransport),
),
),
)
// Set the RoundTripper on our client.
client.Transport = roundTripper
resp, err := client.Get("http://google.com")
if err != nil {
log.Printf("error: %v", err)
}
defer resp.Body.Close()
}

View File

@ -80,7 +80,7 @@ func InstrumentHandlerDuration(obs prometheus.ObserverVec, next http.Handler) ht
// names are "code" and "method". The function panics if any other instance // names are "code" and "method". The function panics if any other instance
// labels are provided. Partitioning of the CounterVec happens by HTTP status // labels are provided. Partitioning of the CounterVec happens by HTTP status
// code and/or HTTP method if the respective instance label names are present // code and/or HTTP method if the respective instance label names are present
// in the CounterVec. For unpartitioned observations, use a CounterVec with // in the CounterVec. For unpartitioned counting, use a CounterVec with
// zero labels. // zero labels.
// //
// If the wrapped Handler does not set a status code, a status code of 200 is assumed. // If the wrapped Handler does not set a status code, a status code of 200 is assumed.