forked from mirror/client_golang
316 lines
12 KiB
Go
316 lines
12 KiB
Go
// Copyright 2016 The Prometheus Authors
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
// Package promhttp provides tooling around HTTP servers and clients.
|
|
//
|
|
// First, the package allows the creation of http.Handler instances to expose
|
|
// Prometheus metrics via HTTP. promhttp.Handler acts on the
|
|
// prometheus.DefaultGatherer. With HandlerFor, you can create a handler for a
|
|
// custom registry or anything that implements the Gatherer interface. It also
|
|
// allows the creation of handlers that act differently on errors or allow to
|
|
// log errors.
|
|
//
|
|
// Second, the package provides tooling to instrument instances of http.Handler
|
|
// via middleware. Middleware wrappers follow the naming scheme
|
|
// InstrumentHandlerX, where X describes the intended use of the middleware.
|
|
// See each function's doc comment for specific details.
|
|
//
|
|
// Finally, the package allows for an http.RoundTripper to be instrumented via
|
|
// middleware. Middleware wrappers follow the naming scheme
|
|
// InstrumentRoundTripperX, where X describes the intended use of the
|
|
// middleware. See each function's doc comment for specific details.
|
|
package promhttp
|
|
|
|
import (
|
|
"bytes"
|
|
"compress/gzip"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/prometheus/common/expfmt"
|
|
|
|
"github.com/prometheus/client_golang/prometheus"
|
|
)
|
|
|
|
const (
|
|
contentTypeHeader = "Content-Type"
|
|
contentLengthHeader = "Content-Length"
|
|
contentEncodingHeader = "Content-Encoding"
|
|
acceptEncodingHeader = "Accept-Encoding"
|
|
)
|
|
|
|
var gzipPool = sync.Pool{
|
|
New: func() interface{} {
|
|
return gzip.NewWriter(nil)
|
|
},
|
|
}
|
|
|
|
// Handler returns an http.Handler for the prometheus.DefaultGatherer, using
|
|
// default HandlerOpts, i.e. it reports the first error as an HTTP error, it has
|
|
// no error logging, and it applies compression if requested by the client.
|
|
//
|
|
// The returned http.Handler is already instrumented using the
|
|
// InstrumentMetricHandler function and the prometheus.DefaultRegisterer. If you
|
|
// create multiple http.Handlers by separate calls of the Handler function, the
|
|
// metrics used for instrumentation will be shared between them, providing
|
|
// global scrape counts.
|
|
//
|
|
// This function is meant to cover the bulk of basic use cases. If you are doing
|
|
// anything that requires more customization (including using a non-default
|
|
// Gatherer, different instrumentation, and non-default HandlerOpts), use the
|
|
// HandlerFor function. See there for details.
|
|
func Handler() http.Handler {
|
|
return InstrumentMetricHandler(
|
|
prometheus.DefaultRegisterer, HandlerFor(prometheus.DefaultGatherer, HandlerOpts{}),
|
|
)
|
|
}
|
|
|
|
// HandlerFor returns an uninstrumented http.Handler for the provided
|
|
// Gatherer. The behavior of the Handler is defined by the provided
|
|
// HandlerOpts. Thus, HandlerFor is useful to create http.Handlers for custom
|
|
// Gatherers, with non-default HandlerOpts, and/or with custom (or no)
|
|
// instrumentation. Use the InstrumentMetricHandler function to apply the same
|
|
// kind of instrumentation as it is used by the Handler function.
|
|
func HandlerFor(reg prometheus.Gatherer, opts HandlerOpts) http.Handler {
|
|
var inFlightSem chan struct{}
|
|
if opts.MaxRequestsInFlight > 0 {
|
|
inFlightSem = make(chan struct{}, opts.MaxRequestsInFlight)
|
|
}
|
|
|
|
h := http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
|
|
if inFlightSem != nil {
|
|
select {
|
|
case inFlightSem <- struct{}{}: // All good, carry on.
|
|
defer func() { <-inFlightSem }()
|
|
default:
|
|
http.Error(w, fmt.Sprintf(
|
|
"Limit of concurrent requests reached (%d), try again later.", opts.MaxRequestsInFlight,
|
|
), http.StatusServiceUnavailable)
|
|
return
|
|
}
|
|
}
|
|
|
|
mfs, err := reg.Gather()
|
|
if err != nil {
|
|
if opts.ErrorLog != nil {
|
|
opts.ErrorLog.Println("error gathering metrics:", err)
|
|
}
|
|
switch opts.ErrorHandling {
|
|
case PanicOnError:
|
|
panic(err)
|
|
case ContinueOnError:
|
|
if len(mfs) == 0 {
|
|
http.Error(w, "No metrics gathered, last error:\n\n"+err.Error(), http.StatusInternalServerError)
|
|
return
|
|
}
|
|
case HTTPErrorOnError:
|
|
http.Error(w, "An error has occurred during metrics gathering:\n\n"+err.Error(), http.StatusInternalServerError)
|
|
return
|
|
}
|
|
}
|
|
|
|
contentType := expfmt.Negotiate(req.Header)
|
|
buf := &bytes.Buffer{}
|
|
enc := expfmt.NewEncoder(buf, contentType)
|
|
|
|
var lastErr error
|
|
for _, mf := range mfs {
|
|
if err := enc.Encode(mf); err != nil {
|
|
lastErr = err
|
|
if opts.ErrorLog != nil {
|
|
opts.ErrorLog.Println("error encoding metric family:", err)
|
|
}
|
|
switch opts.ErrorHandling {
|
|
case PanicOnError:
|
|
panic(err)
|
|
case ContinueOnError:
|
|
// Handled later.
|
|
case HTTPErrorOnError:
|
|
http.Error(w, "An error has occurred during metrics encoding:\n\n"+err.Error(), http.StatusInternalServerError)
|
|
return
|
|
}
|
|
}
|
|
}
|
|
|
|
if lastErr != nil && buf.Len() == 0 {
|
|
http.Error(w, "No metrics encoded, last error:\n\n"+lastErr.Error(), http.StatusInternalServerError)
|
|
return
|
|
}
|
|
|
|
w.Header().Set(contentTypeHeader, string(contentType))
|
|
|
|
if _, err := w.Write(buf.Bytes()); err != nil && opts.ErrorLog != nil {
|
|
opts.ErrorLog.Println("error while sending encoded metrics:", err)
|
|
}
|
|
// TODO(beorn7): Consider streaming serving of metrics.
|
|
})
|
|
|
|
gzipHandler := http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
|
|
if opts.DisableCompression {
|
|
h.ServeHTTP(w, req)
|
|
return
|
|
}
|
|
header := req.Header.Get(acceptEncodingHeader)
|
|
parts := strings.Split(header, ",")
|
|
for _, part := range parts {
|
|
part = strings.TrimSpace(part)
|
|
if part == "gzip" || strings.HasPrefix(part, "gzip;") {
|
|
|
|
w.Header().Set(contentEncodingHeader, "gzip")
|
|
gz := gzipPool.Get().(*gzip.Writer)
|
|
defer gzipPool.Put(gz)
|
|
|
|
gz.Reset(w)
|
|
defer gz.Close()
|
|
|
|
h.ServeHTTP(gzipResponseWriter{gz, w}, req)
|
|
return
|
|
}
|
|
}
|
|
h.ServeHTTP(w, req)
|
|
return
|
|
})
|
|
|
|
if opts.Timeout <= 0 {
|
|
return gzipHandler
|
|
}
|
|
return http.TimeoutHandler(gzipHandler, opts.Timeout, fmt.Sprintf(
|
|
"Exceeded configured timeout of %v.\n",
|
|
opts.Timeout,
|
|
))
|
|
}
|
|
|
|
// InstrumentMetricHandler is usually used with an http.Handler returned by the
|
|
// HandlerFor function. It instruments the provided http.Handler with two
|
|
// metrics: A counter vector "promhttp_metric_handler_requests_total" to count
|
|
// scrapes partitioned by HTTP status code, and a gauge
|
|
// "promhttp_metric_handler_requests_in_flight" to track the number of
|
|
// simultaneous scrapes. This function idempotently registers collectors for
|
|
// both metrics with the provided Registerer. It panics if the registration
|
|
// fails. The provided metrics are useful to see how many scrapes hit the
|
|
// monitored target (which could be from different Prometheus servers or other
|
|
// scrapers), and how often they overlap (which would result in more than one
|
|
// scrape in flight at the same time). Note that the scrapes-in-flight gauge
|
|
// will contain the scrape by which it is exposed, while the scrape counter will
|
|
// only get incremented after the scrape is complete (as only then the status
|
|
// code is known). For tracking scrape durations, use the
|
|
// "scrape_duration_seconds" gauge created by the Prometheus server upon each
|
|
// scrape.
|
|
func InstrumentMetricHandler(reg prometheus.Registerer, handler http.Handler) http.Handler {
|
|
cnt := prometheus.NewCounterVec(
|
|
prometheus.CounterOpts{
|
|
Name: "promhttp_metric_handler_requests_total",
|
|
Help: "Total number of scrapes by HTTP status code.",
|
|
},
|
|
[]string{"code"},
|
|
)
|
|
// Initialize the most likely HTTP status codes.
|
|
cnt.WithLabelValues("200")
|
|
cnt.WithLabelValues("500")
|
|
cnt.WithLabelValues("503")
|
|
if err := reg.Register(cnt); err != nil {
|
|
if are, ok := err.(prometheus.AlreadyRegisteredError); ok {
|
|
cnt = are.ExistingCollector.(*prometheus.CounterVec)
|
|
} else {
|
|
panic(err)
|
|
}
|
|
}
|
|
|
|
gge := prometheus.NewGauge(prometheus.GaugeOpts{
|
|
Name: "promhttp_metric_handler_requests_in_flight",
|
|
Help: "Current number of scrapes being served.",
|
|
})
|
|
if err := reg.Register(gge); err != nil {
|
|
if are, ok := err.(prometheus.AlreadyRegisteredError); ok {
|
|
gge = are.ExistingCollector.(prometheus.Gauge)
|
|
} else {
|
|
panic(err)
|
|
}
|
|
}
|
|
|
|
return InstrumentHandlerCounter(cnt, InstrumentHandlerInFlight(gge, handler))
|
|
}
|
|
|
|
// HandlerErrorHandling defines how a Handler serving metrics will handle
|
|
// errors.
|
|
type HandlerErrorHandling int
|
|
|
|
// These constants cause handlers serving metrics to behave as described if
|
|
// errors are encountered.
|
|
const (
|
|
// Serve an HTTP status code 500 upon the first error
|
|
// encountered. Report the error message in the body.
|
|
HTTPErrorOnError HandlerErrorHandling = iota
|
|
// Ignore errors and try to serve as many metrics as possible. However,
|
|
// if no metrics can be served, serve an HTTP status code 500 and the
|
|
// last error message in the body. Only use this in deliberate "best
|
|
// effort" metrics collection scenarios. It is recommended to at least
|
|
// log errors (by providing an ErrorLog in HandlerOpts) to not mask
|
|
// errors completely.
|
|
ContinueOnError
|
|
// Panic upon the first error encountered (useful for "crash only" apps).
|
|
PanicOnError
|
|
)
|
|
|
|
// Logger is the minimal interface HandlerOpts needs for logging. Note that
|
|
// log.Logger from the standard library implements this interface, and it is
|
|
// easy to implement by custom loggers, if they don't do so already anyway.
|
|
type Logger interface {
|
|
Println(v ...interface{})
|
|
}
|
|
|
|
// HandlerOpts specifies options how to serve metrics via an http.Handler. The
|
|
// zero value of HandlerOpts is a reasonable default.
|
|
type HandlerOpts struct {
|
|
// ErrorLog specifies an optional logger for errors collecting and
|
|
// serving metrics. If nil, errors are not logged at all.
|
|
ErrorLog Logger
|
|
// ErrorHandling defines how errors are handled. Note that errors are
|
|
// logged regardless of the configured ErrorHandling provided ErrorLog
|
|
// is not nil.
|
|
ErrorHandling HandlerErrorHandling
|
|
// If DisableCompression is true, the handler will never compress the
|
|
// response, even if requested by the client.
|
|
DisableCompression bool
|
|
// The number of concurrent HTTP requests is limited to
|
|
// MaxRequestsInFlight. Additional requests are responded to with 503
|
|
// Service Unavailable and a suitable message in the body. If
|
|
// MaxRequestsInFlight is 0 or negative, no limit is applied.
|
|
MaxRequestsInFlight int
|
|
// If handling a request takes longer than Timeout, it is responded to
|
|
// with 503 ServiceUnavailable and a suitable Message. No timeout is
|
|
// applied if Timeout is 0 or negative. Note that with the current
|
|
// implementation, reaching the timeout simply ends the HTTP requests as
|
|
// described above (and even that only if sending of the body hasn't
|
|
// started yet), while the bulk work of gathering all the metrics keeps
|
|
// running in the background (with the eventual result to be thrown
|
|
// away). Until the implementation is improved, it is recommended to
|
|
// implement a separate timeout in potentially slow Collectors.
|
|
Timeout time.Duration
|
|
}
|
|
|
|
// gzipResponseWriter in charge of wrapping io.Writer and http.ReponseWriter
|
|
// together, allowing to get a single struct which implements both interface.
|
|
type gzipResponseWriter struct {
|
|
io.Writer
|
|
http.ResponseWriter
|
|
}
|
|
|
|
func (w gzipResponseWriter) Write(b []byte) (int, error) {
|
|
return w.Writer.Write(b)
|
|
}
|