From 8fd47d2e8fb8b2c27b899593ae9237aae7bb4191 Mon Sep 17 00:00:00 2001 From: beorn7 Date: Wed, 7 Feb 2018 17:55:45 +0100 Subject: [PATCH] Provide an InstrumentedHandler See https://github.com/prometheus/client_golang/issues/316 for details. --- prometheus/http.go | 5 +- prometheus/promhttp/http.go | 81 ++++++++++++++++++++++++++++---- prometheus/promhttp/http_test.go | 40 ++++++++++++++++ 3 files changed, 114 insertions(+), 12 deletions(-) diff --git a/prometheus/http.go b/prometheus/http.go index bfee5c6..dd0f819 100644 --- a/prometheus/http.go +++ b/prometheus/http.go @@ -61,9 +61,8 @@ func giveBuf(buf *bytes.Buffer) { // name). // // Deprecated: Please note the issues described in the doc comment of -// InstrumentHandler. You might want to consider using promhttp.Handler instead -// (which is not instrumented, but can be instrumented with the tooling provided -// in package promhttp). +// InstrumentHandler. You might want to consider using +// promhttp.InstrumentedHandler instead. func Handler() http.Handler { return InstrumentHandler("prometheus", UninstrumentedHandler()) } diff --git a/prometheus/promhttp/http.go b/prometheus/promhttp/http.go index 2d67f24..cdca3e3 100644 --- a/prometheus/promhttp/http.go +++ b/prometheus/promhttp/http.go @@ -67,19 +67,32 @@ func giveBuf(buf *bytes.Buffer) { bufPool.Put(buf) } -// Handler returns an HTTP handler for the prometheus.DefaultGatherer. The -// Handler uses the default HandlerOpts, i.e. report the first error as an HTTP -// error, no error logging, and compression if requested by the client. +// Handler returns an http.Handler for the prometheus.DefaultGatherer, using +// default HandlerOpts, i.e. it reports the first error as an HTTP error, it has +// no error logging, and it applies compression if requested by the client. // -// If you want to create a Handler for the DefaultGatherer with different -// HandlerOpts, create it with HandlerFor with prometheus.DefaultGatherer and -// your desired HandlerOpts. +// The returned http.Handler is already instrumented using the +// InstrumentMetricHandler function and the prometheus.DefaultRegisterer. If you +// create multiple http.Handlers by separate calls of the Handler function, the +// metrics used for instrumentation will be shared between them, providing +// global scrape counts. +// +// This function is meant to cover the bulk of basic use cases. If you are doing +// anything that requires more customization (including using a non-default +// Gatherer, different instrumentation, and non-default HandlerOpts), use the +// HandlerFor function. See there for details. func Handler() http.Handler { - return HandlerFor(prometheus.DefaultGatherer, HandlerOpts{}) + return InstrumentMetricHandler( + prometheus.DefaultRegisterer, HandlerFor(prometheus.DefaultGatherer, HandlerOpts{}), + ) } -// HandlerFor returns an http.Handler for the provided Gatherer. The behavior -// of the Handler is defined by the provided HandlerOpts. +// HandlerFor returns an uninstrumented http.Handler for the provided +// Gatherer. The behavior of the Handler is defined by the provided +// HandlerOpts. Thus, HandlerFor is useful to create http.Handlers for custom +// Gatherers, with non-default HandlerOpts, and/or with custom (or no) +// instrumentation. Use the InstrumentMetricHandler function to apply the same +// kind of instrumentation as it is used by the Handler function. func HandlerFor(reg prometheus.Gatherer, opts HandlerOpts) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { mfs, err := reg.Gather() @@ -142,6 +155,56 @@ func HandlerFor(reg prometheus.Gatherer, opts HandlerOpts) http.Handler { }) } +// InstrumentMetricHandler is usually used with an http.Handler returned by the +// HandlerFor function. It instruments the provided http.Handler with two +// metrics: A counter vector "promhttp_metric_handler_requests_total" to count +// scrapes partitioned by HTTP status code, and a gauge +// "promhttp_metric_handler_requests_in_flight" to track the number of +// simultaneous scrapes. This function idempotently registers collectors for +// both metrics with the provided Registerer. It panics if the registration +// fails. The provided metrics are useful to see how many scrapes hit the +// monitored target (which could be from different Prometheus servers or other +// scrapers), and how often they overlap (which would result in more than one +// scrape in flight at the same time). Note that the scrapes-in-flight gauge +// will contain the scrape by which it is exposed, while the scrape counter will +// only get incremented after the scrape is complete (as only then the status +// code is known). For tracking scrape durations, use the +// "scrape_duration_seconds" gauge created by the Prometheus server upon each +// scrape. +func InstrumentMetricHandler(reg prometheus.Registerer, handler http.Handler) http.Handler { + cnt := prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "promhttp_metric_handler_requests_total", + Help: "Total number of scrapes by HTTP status code.", + }, + []string{"code"}, + ) + // Initialize the most likely HTTP status codes. + cnt.WithLabelValues("200") + cnt.WithLabelValues("500") + if err := reg.Register(cnt); err != nil { + if are, ok := err.(prometheus.AlreadyRegisteredError); ok { + cnt = are.ExistingCollector.(*prometheus.CounterVec) + } else { + panic(err) + } + } + + gge := prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "promhttp_metric_handler_requests_in_flight", + Help: "Current number of scrapes being served.", + }) + if err := reg.Register(gge); err != nil { + if are, ok := err.(prometheus.AlreadyRegisteredError); ok { + gge = are.ExistingCollector.(prometheus.Gauge) + } else { + panic(err) + } + } + + return InstrumentHandlerCounter(cnt, InstrumentHandlerInFlight(gge, handler)) +} + // HandlerErrorHandling defines how a Handler serving metrics will handle // errors. type HandlerErrorHandling int diff --git a/prometheus/promhttp/http_test.go b/prometheus/promhttp/http_test.go index 413ff7b..a47ca7b 100644 --- a/prometheus/promhttp/http_test.go +++ b/prometheus/promhttp/http_test.go @@ -19,6 +19,7 @@ import ( "log" "net/http" "net/http/httptest" + "strings" "testing" "github.com/prometheus/client_golang/prometheus" @@ -129,3 +130,42 @@ the_count 0 }() panicHandler.ServeHTTP(writer, request) } + +func TestInstrumentMetricHandler(t *testing.T) { + reg := prometheus.NewRegistry() + handler := InstrumentMetricHandler(reg, HandlerFor(reg, HandlerOpts{})) + // Do it again to test idempotency. + InstrumentMetricHandler(reg, HandlerFor(reg, HandlerOpts{})) + writer := httptest.NewRecorder() + request, _ := http.NewRequest("GET", "/", nil) + request.Header.Add("Accept", "test/plain") + + handler.ServeHTTP(writer, request) + if got, want := writer.Code, http.StatusOK; got != want { + t.Errorf("got HTTP status code %d, want %d", got, want) + } + + want := "promhttp_metric_handler_requests_in_flight 1\n" + if got := writer.Body.String(); !strings.Contains(got, want) { + t.Errorf("got body %q, does not contain %q", got, want) + } + want = "promhttp_metric_handler_requests_total{code=\"200\"} 0\n" + if got := writer.Body.String(); !strings.Contains(got, want) { + t.Errorf("got body %q, does not contain %q", got, want) + } + + writer.Body.Reset() + handler.ServeHTTP(writer, request) + if got, want := writer.Code, http.StatusOK; got != want { + t.Errorf("got HTTP status code %d, want %d", got, want) + } + + want = "promhttp_metric_handler_requests_in_flight 1\n" + if got := writer.Body.String(); !strings.Contains(got, want) { + t.Errorf("got body %q, does not contain %q", got, want) + } + want = "promhttp_metric_handler_requests_total{code=\"200\"} 1\n" + if got := writer.Body.String(); !strings.Contains(got, want) { + t.Errorf("got body %q, does not contain %q", got, want) + } +}