Merge pull request #377 from prometheus/beorn7/http

Provide an InstrumentedHandler

prometheus/http.go

@@ -61,9 +61,8 @@ func giveBuf(buf *bytes.Buffer) {
 // name).
 //
 // Deprecated: Please note the issues described in the doc comment of
-// InstrumentHandler. You might want to consider using promhttp.Handler instead
+// InstrumentHandler. You might want to consider using
-// (which is not instrumented, but can be instrumented with the tooling provided
+// promhttp.InstrumentedHandler instead.
-// in package promhttp).
 func Handler() http.Handler {
 	return InstrumentHandler("prometheus", UninstrumentedHandler())
 }

prometheus/promhttp/http.go

@@ -67,19 +67,32 @@ func giveBuf(buf *bytes.Buffer) {
 	bufPool.Put(buf)
 }
 
-// Handler returns an HTTP handler for the prometheus.DefaultGatherer. The
+// Handler returns an http.Handler for the prometheus.DefaultGatherer, using
-// Handler uses the default HandlerOpts, i.e. report the first error as an HTTP
+// default HandlerOpts, i.e. it reports the first error as an HTTP error, it has
-// error, no error logging, and compression if requested by the client.
+// no error logging, and it applies compression if requested by the client.
 //
-// If you want to create a Handler for the DefaultGatherer with different
+// The returned http.Handler is already instrumented using the
-// HandlerOpts, create it with HandlerFor with prometheus.DefaultGatherer and
+// InstrumentMetricHandler function and the prometheus.DefaultRegisterer. If you
-// your desired HandlerOpts.
+// create multiple http.Handlers by separate calls of the Handler function, the
+// metrics used for instrumentation will be shared between them, providing
+// global scrape counts.
+//
+// This function is meant to cover the bulk of basic use cases. If you are doing
+// anything that requires more customization (including using a non-default
+// Gatherer, different instrumentation, and non-default HandlerOpts), use the
+// HandlerFor function. See there for details.
 func Handler() http.Handler {
-	return HandlerFor(prometheus.DefaultGatherer, HandlerOpts{})
+	return InstrumentMetricHandler(
+		prometheus.DefaultRegisterer, HandlerFor(prometheus.DefaultGatherer, HandlerOpts{}),
+	)
 }
 
-// HandlerFor returns an http.Handler for the provided Gatherer. The behavior
+// HandlerFor returns an uninstrumented http.Handler for the provided
-// of the Handler is defined by the provided HandlerOpts.
+// Gatherer. The behavior of the Handler is defined by the provided
+// HandlerOpts. Thus, HandlerFor is useful to create http.Handlers for custom
+// Gatherers, with non-default HandlerOpts, and/or with custom (or no)
+// instrumentation. Use the InstrumentMetricHandler function to apply the same
+// kind of instrumentation as it is used by the Handler function.
 func HandlerFor(reg prometheus.Gatherer, opts HandlerOpts) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
 		mfs, err := reg.Gather()
@@ -142,6 +155,56 @@ func HandlerFor(reg prometheus.Gatherer, opts HandlerOpts) http.Handler {
 	})
 }
 
+// InstrumentMetricHandler is usually used with an http.Handler returned by the
+// HandlerFor function. It instruments the provided http.Handler with two
+// metrics: A counter vector "promhttp_metric_handler_requests_total" to count
+// scrapes partitioned by HTTP status code, and a gauge
+// "promhttp_metric_handler_requests_in_flight" to track the number of
+// simultaneous scrapes. This function idempotently registers collectors for
+// both metrics with the provided Registerer. It panics if the registration
+// fails. The provided metrics are useful to see how many scrapes hit the
+// monitored target (which could be from different Prometheus servers or other
+// scrapers), and how often they overlap (which would result in more than one
+// scrape in flight at the same time). Note that the scrapes-in-flight gauge
+// will contain the scrape by which it is exposed, while the scrape counter will
+// only get incremented after the scrape is complete (as only then the status
+// code is known). For tracking scrape durations, use the
+// "scrape_duration_seconds" gauge created by the Prometheus server upon each
+// scrape.
+func InstrumentMetricHandler(reg prometheus.Registerer, handler http.Handler) http.Handler {
+	cnt := prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "promhttp_metric_handler_requests_total",
+			Help: "Total number of scrapes by HTTP status code.",
+		},
+		[]string{"code"},
+	)
+	// Initialize the most likely HTTP status codes.
+	cnt.WithLabelValues("200")
+	cnt.WithLabelValues("500")
+	if err := reg.Register(cnt); err != nil {
+		if are, ok := err.(prometheus.AlreadyRegisteredError); ok {
+			cnt = are.ExistingCollector.(*prometheus.CounterVec)
+		} else {
+			panic(err)
+		}
+	}
+
+	gge := prometheus.NewGauge(prometheus.GaugeOpts{
+		Name: "promhttp_metric_handler_requests_in_flight",
+		Help: "Current number of scrapes being served.",
+	})
+	if err := reg.Register(gge); err != nil {
+		if are, ok := err.(prometheus.AlreadyRegisteredError); ok {
+			gge = are.ExistingCollector.(prometheus.Gauge)
+		} else {
+			panic(err)
+		}
+	}
+
+	return InstrumentHandlerCounter(cnt, InstrumentHandlerInFlight(gge, handler))
+}
+
 // HandlerErrorHandling defines how a Handler serving metrics will handle
 // errors.
 type HandlerErrorHandling int

prometheus/promhttp/http_test.go

@@ -19,6 +19,7 @@ import (
 	"log"
 	"net/http"
 	"net/http/httptest"
+	"strings"
 	"testing"
 
 	"github.com/prometheus/client_golang/prometheus"
@@ -129,3 +130,42 @@ the_count 0
 	}()
 	panicHandler.ServeHTTP(writer, request)
 }
+
+func TestInstrumentMetricHandler(t *testing.T) {
+	reg := prometheus.NewRegistry()
+	handler := InstrumentMetricHandler(reg, HandlerFor(reg, HandlerOpts{}))
+	// Do it again to test idempotency.
+	InstrumentMetricHandler(reg, HandlerFor(reg, HandlerOpts{}))
+	writer := httptest.NewRecorder()
+	request, _ := http.NewRequest("GET", "/", nil)
+	request.Header.Add("Accept", "test/plain")
+
+	handler.ServeHTTP(writer, request)
+	if got, want := writer.Code, http.StatusOK; got != want {
+		t.Errorf("got HTTP status code %d, want %d", got, want)
+	}
+
+	want := "promhttp_metric_handler_requests_in_flight 1\n"
+	if got := writer.Body.String(); !strings.Contains(got, want) {
+		t.Errorf("got body %q, does not contain %q", got, want)
+	}
+	want = "promhttp_metric_handler_requests_total{code=\"200\"} 0\n"
+	if got := writer.Body.String(); !strings.Contains(got, want) {
+		t.Errorf("got body %q, does not contain %q", got, want)
+	}
+
+	writer.Body.Reset()
+	handler.ServeHTTP(writer, request)
+	if got, want := writer.Code, http.StatusOK; got != want {
+		t.Errorf("got HTTP status code %d, want %d", got, want)
+	}
+
+	want = "promhttp_metric_handler_requests_in_flight 1\n"
+	if got := writer.Body.String(); !strings.Contains(got, want) {
+		t.Errorf("got body %q, does not contain %q", got, want)
+	}
+	want = "promhttp_metric_handler_requests_total{code=\"200\"} 1\n"
+	if got := writer.Body.String(); !strings.Contains(got, want) {
+		t.Errorf("got body %q, does not contain %q", got, want)
+	}
+}