From 1db43792dbc2308747636f201649e6e7ede4ff6d Mon Sep 17 00:00:00 2001 From: beorn7 Date: Tue, 13 Nov 2018 15:48:48 +0100 Subject: [PATCH 01/73] Expose bug #494 Signed-off-by: beorn7 --- prometheus/testutil/testutil_test.go | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/prometheus/testutil/testutil_test.go b/prometheus/testutil/testutil_test.go index e25b130..0b8cf09 100644 --- a/prometheus/testutil/testutil_test.go +++ b/prometheus/testutil/testutil_test.go @@ -143,6 +143,28 @@ func TestCollectAndCompare(t *testing.T) { } } +func TestCollectAndCompareNoLabel(t *testing.T) { + const metadata = ` + # HELP some_total A value that represents a counter. + # TYPE some_total counter + ` + + c := prometheus.NewCounter(prometheus.CounterOpts{ + Name: "some_total", + Help: "A value that represents a counter.", + }) + c.Inc() + + expected := ` + + some_total 1 + ` + + if err := CollectAndCompare(c, strings.NewReader(metadata+expected), "some_total"); err != nil { + t.Errorf("unexpected collecting result:\n%s", err) + } +} + func TestNoMetricFilter(t *testing.T) { const metadata = ` # HELP some_total A value that represents a counter. From 86702ea6b427ab178f0353345fc69cfd80b63d8f Mon Sep 17 00:00:00 2001 From: beorn7 Date: Tue, 13 Nov 2018 16:16:43 +0100 Subject: [PATCH 02/73] Fix metric comparison for empty labels reflect.DeepEqual is not suitable for zero occurrences of repeated proto messages. This changes the comparison to act on the string representation of proto messages. Signed-off-by: beorn7 --- prometheus/testutil/testutil.go | 57 ++++++++++++++++++--------------- 1 file changed, 32 insertions(+), 25 deletions(-) diff --git a/prometheus/testutil/testutil.go b/prometheus/testutil/testutil.go index d148af9..e7af303 100644 --- a/prometheus/testutil/testutil.go +++ b/prometheus/testutil/testutil.go @@ -37,7 +37,6 @@ import ( "bytes" "fmt" "io" - "reflect" "github.com/prometheus/common/expfmt" @@ -125,39 +124,49 @@ func CollectAndCompare(c prometheus.Collector, expected io.Reader, metricNames . // exposition format. If any metricNames are provided, only metrics with those // names are compared. func GatherAndCompare(g prometheus.Gatherer, expected io.Reader, metricNames ...string) error { - metrics, err := g.Gather() + got, err := g.Gather() if err != nil { return fmt.Errorf("gathering metrics failed: %s", err) } if metricNames != nil { - metrics = filterMetrics(metrics, metricNames) + got = filterMetrics(got, metricNames) } var tp expfmt.TextParser - expectedMetrics, err := tp.TextToMetricFamilies(expected) + wantRaw, err := tp.TextToMetricFamilies(expected) if err != nil { return fmt.Errorf("parsing expected metrics failed: %s", err) } + want := internal.NormalizeMetricFamilies(wantRaw) - if !reflect.DeepEqual(metrics, internal.NormalizeMetricFamilies(expectedMetrics)) { - // Encode the gathered output to the readable text format for comparison. - var buf1 bytes.Buffer - enc := expfmt.NewEncoder(&buf1, expfmt.FmtText) - for _, mf := range metrics { - if err := enc.Encode(mf); err != nil { - return fmt.Errorf("encoding result failed: %s", err) - } - } - // Encode normalized expected metrics again to generate them in the same ordering - // the registry does to spot differences more easily. - var buf2 bytes.Buffer - enc = expfmt.NewEncoder(&buf2, expfmt.FmtText) - for _, mf := range internal.NormalizeMetricFamilies(expectedMetrics) { - if err := enc.Encode(mf); err != nil { - return fmt.Errorf("encoding result failed: %s", err) - } + if len(got) != len(want) { + return notMatchingError(got, want) + } + for i := range got { + if got[i].String() != want[i].String() { + return notMatchingError(got, want) } + } + return nil +} - return fmt.Errorf(` +// notMatchingError encodes both provided slices of metric families into the +// text format and creates a readable error message from the result. +func notMatchingError(got, want []*dto.MetricFamily) error { + var gotBuf, wantBuf bytes.Buffer + enc := expfmt.NewEncoder(&gotBuf, expfmt.FmtText) + for _, mf := range got { + if err := enc.Encode(mf); err != nil { + return fmt.Errorf("encoding gathered metrics failed: %s", err) + } + } + enc = expfmt.NewEncoder(&wantBuf, expfmt.FmtText) + for _, mf := range want { + if err := enc.Encode(mf); err != nil { + return fmt.Errorf("encoding expected metrics failed: %s", err) + } + } + + return fmt.Errorf(` metric output does not match expectation; want: %s @@ -165,9 +174,7 @@ metric output does not match expectation; want: got: %s -`, buf2.String(), buf1.String()) - } - return nil +`, wantBuf.String(), gotBuf.String()) } func filterMetrics(metrics []*dto.MetricFamily, names []string) []*dto.MetricFamily { From 30503fe98e14c8f138f460d2eb1ccfa27b57beae Mon Sep 17 00:00:00 2001 From: PhilipGough Date: Wed, 14 Nov 2018 11:33:19 +0000 Subject: [PATCH 03/73] Exposes bug #498 - unexpected results for Histograms in testutil Signed-off-by: PhilipGough --- prometheus/testutil/testutil_test.go | 76 ++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/prometheus/testutil/testutil_test.go b/prometheus/testutil/testutil_test.go index 0b8cf09..f960077 100644 --- a/prometheus/testutil/testutil_test.go +++ b/prometheus/testutil/testutil_test.go @@ -165,6 +165,82 @@ func TestCollectAndCompareNoLabel(t *testing.T) { } } +func TestCollectAndCompareHistogram(t *testing.T) { + inputs := []struct { + name string + c prometheus.Collector + metadata string + expect string + labels []string + observation float64 + }{ + { + name: "Testing Histogram Collector", + c: prometheus.NewHistogram(prometheus.HistogramOpts{ + Name: "some_histogram", + Help: "An example of a histogram", + Buckets: []float64{1, 2, 3}, + }), + metadata: ` + # HELP some_histogram An example of a histogram + # TYPE some_histogram histogram + `, + expect: ` + some_histogram{le="1"} 0 + some_histogram{le="2"} 0 + some_histogram{le="3"} 1 + some_histogram_bucket{le="+Inf"} 1 + some_histogram_sum 2.5 + some_histogram_count 1 + + `, + observation: 2.5, + }, + { + name: "Testing HistogramVec Collector", + c: prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Name: "some_histogram", + Help: "An example of a histogram", + Buckets: []float64{1, 2, 3}, + }, []string{"test"}), + + metadata: ` + # HELP some_histogram An example of a histogram + # TYPE some_histogram histogram + `, + expect: ` + some_histogram_bucket{test="test",le="1"} 0 + some_histogram_bucket{test="test",le="2"} 0 + some_histogram_bucket{test="test",le="3"} 1 + some_histogram_bucket{test="test",le="+Inf"} 1 + some_histogram_sum{test="test"} 2.5 + some_histogram_count{test="test"} 1 + + `, + observation: 2.5, + }, + } + + for _, input := range inputs { + switch collector := input.c.(type) { + case prometheus.Histogram: + collector.Observe(input.observation) + case *prometheus.HistogramVec: + collector.WithLabelValues("test").Observe(input.observation) + default: + t.Fatalf("unsuported collector tested") + + } + + t.Run(input.name, func(t *testing.T) { + if err := CollectAndCompare(input.c, strings.NewReader(input.metadata+input.expect)); err != nil { + t.Errorf("unexpected collecting result:\n%s", err) + } + }) + + } +} + func TestNoMetricFilter(t *testing.T) { const metadata = ` # HELP some_total A value that represents a counter. From f9739b3d9725e4254ae615c83042741091f5222e Mon Sep 17 00:00:00 2001 From: PhilipGough Date: Wed, 14 Nov 2018 11:55:09 +0000 Subject: [PATCH 04/73] Compare text strings directly in testutil Signed-off-by: PhilipGough --- prometheus/testutil/testutil.go | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/prometheus/testutil/testutil.go b/prometheus/testutil/testutil.go index e7af303..973b82d 100644 --- a/prometheus/testutil/testutil.go +++ b/prometheus/testutil/testutil.go @@ -138,20 +138,13 @@ func GatherAndCompare(g prometheus.Gatherer, expected io.Reader, metricNames ... } want := internal.NormalizeMetricFamilies(wantRaw) - if len(got) != len(want) { - return notMatchingError(got, want) - } - for i := range got { - if got[i].String() != want[i].String() { - return notMatchingError(got, want) - } - } - return nil + return compare(got, want) } -// notMatchingError encodes both provided slices of metric families into the -// text format and creates a readable error message from the result. -func notMatchingError(got, want []*dto.MetricFamily) error { +// compare encodes both provided slices of metric families into the text format, +// compares their string message, and returns an error if they do not match. +// The error contains the encoded text, of both the desired, and actual result. +func compare(got, want []*dto.MetricFamily) error { var gotBuf, wantBuf bytes.Buffer enc := expfmt.NewEncoder(&gotBuf, expfmt.FmtText) for _, mf := range got { @@ -166,7 +159,8 @@ func notMatchingError(got, want []*dto.MetricFamily) error { } } - return fmt.Errorf(` + if wantBuf.String() != gotBuf.String() { + return fmt.Errorf(` metric output does not match expectation; want: %s @@ -175,6 +169,9 @@ got: %s `, wantBuf.String(), gotBuf.String()) + + } + return nil } func filterMetrics(metrics []*dto.MetricFamily, names []string) []*dto.MetricFamily { From c4c1f3461e40e5c6a4f570f5b00415cf5eacbd79 Mon Sep 17 00:00:00 2001 From: beorn7 Date: Thu, 15 Nov 2018 18:40:33 +0100 Subject: [PATCH 05/73] Fix tiny punctuation issue in doc comment Signed-off-by: beorn7 --- prometheus/testutil/testutil.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/prometheus/testutil/testutil.go b/prometheus/testutil/testutil.go index 973b82d..e344cc8 100644 --- a/prometheus/testutil/testutil.go +++ b/prometheus/testutil/testutil.go @@ -143,7 +143,8 @@ func GatherAndCompare(g prometheus.Gatherer, expected io.Reader, metricNames ... // compare encodes both provided slices of metric families into the text format, // compares their string message, and returns an error if they do not match. -// The error contains the encoded text, of both the desired, and actual result. +// The error contains the encoded text of both the desired and the actual +// result. func compare(got, want []*dto.MetricFamily) error { var gotBuf, wantBuf bytes.Buffer enc := expfmt.NewEncoder(&gotBuf, expfmt.FmtText) From 8c67c139e284d15287605f4d9bd69ca2fbee3195 Mon Sep 17 00:00:00 2001 From: beorn7 Date: Fri, 16 Nov 2018 13:23:08 +0100 Subject: [PATCH 06/73] Remove accidental gogo/protobuf import Signed-off-by: beorn7 --- prometheus/wrap_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prometheus/wrap_test.go b/prometheus/wrap_test.go index bed103e..28e2470 100644 --- a/prometheus/wrap_test.go +++ b/prometheus/wrap_test.go @@ -19,7 +19,7 @@ import ( "strings" "testing" - "github.com/gogo/protobuf/proto" + "github.com/golang/protobuf/proto" dto "github.com/prometheus/client_model/go" ) From f626bd2850300d4c8aafd15da82133a416ff9688 Mon Sep 17 00:00:00 2001 From: Simon Pasquier Date: Thu, 15 Nov 2018 17:51:14 +0100 Subject: [PATCH 07/73] *: support Go modules Signed-off-by: Simon Pasquier --- Makefile.common | 113 +++++++++++++++++++++++++++++++++++---- api/prometheus/v1/api.go | 12 ++--- go.mod | 12 +++++ go.sum | 16 ++++++ 4 files changed, 136 insertions(+), 17 deletions(-) create mode 100644 go.mod create mode 100644 go.sum diff --git a/Makefile.common b/Makefile.common index c9d8323..0248fee 100644 --- a/Makefile.common +++ b/Makefile.common @@ -16,7 +16,7 @@ # !!! Open PRs only against the prometheus/prometheus/Makefile.common repository! # Example usage : -# Create the main Makefile in the root project directory. +# Create the main Makefile in the root project directory. # include Makefile.common # customTarget: # @echo ">> Running customTarget" @@ -28,18 +28,53 @@ unexport GOBIN GO ?= go GOFMT ?= $(GO)fmt FIRST_GOPATH := $(firstword $(subst :, ,$(shell $(GO) env GOPATH))) +GOOPTS ?= + +GO_VERSION ?= $(shell $(GO) version) +GO_VERSION_NUMBER ?= $(word 3, $(GO_VERSION)) +PRE_GO_111 ?= $(shell echo $(GO_VERSION_NUMBER) | grep -E 'go1\.(10|[0-9])\.') + +unexport GOVENDOR +ifeq (, $(PRE_GO_111)) + ifneq (,$(wildcard go.mod)) + # Enforce Go modules support just in case the directory is inside GOPATH (and for Travis CI). + GO111MODULE := on + + ifneq (,$(wildcard vendor)) + # Always use the local vendor/ directory to satisfy the dependencies. + GOOPTS := $(GOOPTS) -mod=vendor + endif + endif +else + ifneq (,$(wildcard go.mod)) + ifneq (,$(wildcard vendor)) +$(warning This repository requires Go >= 1.11 because of Go modules) +$(warning Some recipes may not work as expected as the current Go runtime is '$(GO_VERSION_NUMBER)') + endif + else + # This repository isn't using Go modules (yet). + GOVENDOR := $(FIRST_GOPATH)/bin/govendor + endif + + unexport GO111MODULE +endif PROMU := $(FIRST_GOPATH)/bin/promu STATICCHECK := $(FIRST_GOPATH)/bin/staticcheck -GOVENDOR := $(FIRST_GOPATH)/bin/govendor pkgs = ./... +GO_VERSION ?= $(shell $(GO) version) +GO_BUILD_PLATFORM ?= $(subst /,-,$(lastword $(GO_VERSION))) + +PROMU_VERSION ?= 0.2.0 +PROMU_URL := https://github.com/prometheus/promu/releases/download/v$(PROMU_VERSION)/promu-$(PROMU_VERSION).$(GO_BUILD_PLATFORM).tar.gz + PREFIX ?= $(shell pwd) BIN_DIR ?= $(shell pwd) DOCKER_IMAGE_TAG ?= $(subst /,-,$(shell git rev-parse --abbrev-ref HEAD)) DOCKER_REPO ?= prom .PHONY: all -all: style staticcheck unused build test +all: precheck style staticcheck unused build test # This rule is used to forward a target like "build" to "common-build". This # allows a new "build" target to be defined in a Makefile which includes this @@ -70,37 +105,54 @@ common-check_license: .PHONY: common-test-short common-test-short: @echo ">> running short tests" - $(GO) test -short $(pkgs) + GO111MODULE=$(GO111MODULE) $(GO) test -short $(GOOPTS) $(pkgs) .PHONY: common-test common-test: @echo ">> running all tests" - $(GO) test -race $(pkgs) + GO111MODULE=$(GO111MODULE) $(GO) test -race $(GOOPTS) $(pkgs) .PHONY: common-format common-format: @echo ">> formatting code" - $(GO) fmt $(pkgs) + GO111MODULE=$(GO111MODULE) $(GO) fmt $(GOOPTS) $(pkgs) .PHONY: common-vet common-vet: @echo ">> vetting code" - $(GO) vet $(pkgs) + GO111MODULE=$(GO111MODULE) $(GO) vet $(GOOPTS) $(pkgs) .PHONY: common-staticcheck common-staticcheck: $(STATICCHECK) @echo ">> running staticcheck" +ifdef GO111MODULE + GO111MODULE=$(GO111MODULE) $(STATICCHECK) -ignore "$(STATICCHECK_IGNORE)" -checks "SA*" $(pkgs) +else $(STATICCHECK) -ignore "$(STATICCHECK_IGNORE)" $(pkgs) +endif .PHONY: common-unused common-unused: $(GOVENDOR) +ifdef GOVENDOR @echo ">> running check for unused packages" @$(GOVENDOR) list +unused | grep . && exit 1 || echo 'No unused packages' +else +ifdef GO111MODULE + @echo ">> running check for unused/missing packages in go.mod" + GO111MODULE=$(GO111MODULE) $(GO) mod tidy + @git diff --exit-code -- go.sum go.mod +ifneq (,$(wildcard vendor)) + @echo ">> running check for unused packages in vendor/" + GO111MODULE=$(GO111MODULE) $(GO) mod vendor + @git diff --exit-code -- go.sum go.mod vendor/ +endif +endif +endif .PHONY: common-build common-build: promu @echo ">> building binaries" - $(PROMU) build --prefix $(PREFIX) + GO111MODULE=$(GO111MODULE) $(PROMU) build --prefix $(PREFIX) .PHONY: common-tarball common-tarball: promu @@ -120,13 +172,52 @@ common-docker-tag-latest: docker tag "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_TAG)" "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):latest" .PHONY: promu -promu: - GOOS= GOARCH= $(GO) get -u github.com/prometheus/promu +promu: $(PROMU) + +$(PROMU): + curl -s -L $(PROMU_URL) | tar -xvz -C /tmp + mkdir -v -p $(FIRST_GOPATH)/bin + cp -v /tmp/promu-$(PROMU_VERSION).$(GO_BUILD_PLATFORM)/promu $(PROMU) + +.PHONY: proto +proto: + @echo ">> generating code from proto files" + @./scripts/genproto.sh .PHONY: $(STATICCHECK) $(STATICCHECK): - GOOS= GOARCH= $(GO) get -u honnef.co/go/tools/cmd/staticcheck +ifdef GO111MODULE +# Get staticcheck from a temporary directory to avoid modifying the local go.{mod,sum}. +# See https://github.com/golang/go/issues/27643. +# For now, we are using the next branch of staticcheck because master isn't compatible yet with Go modules. + tmpModule=$$(mktemp -d 2>&1) && \ + mkdir -p $${tmpModule}/staticcheck && \ + cd "$${tmpModule}"/staticcheck && \ + GO111MODULE=on $(GO) mod init example.com/staticcheck && \ + GO111MODULE=on GOOS= GOARCH= $(GO) get -u honnef.co/go/tools/cmd/staticcheck@next && \ + rm -rf $${tmpModule}; +else + GOOS= GOARCH= GO111MODULE=off $(GO) get -u honnef.co/go/tools/cmd/staticcheck +endif +ifdef GOVENDOR .PHONY: $(GOVENDOR) $(GOVENDOR): GOOS= GOARCH= $(GO) get -u github.com/kardianos/govendor +endif + +.PHONY: precheck +precheck:: + +define PRECHECK_COMMAND_template = +precheck:: $(1)_precheck + + +PRECHECK_COMMAND_$(1) ?= $(1) $$(strip $$(PRECHECK_OPTIONS_$(1))) +.PHONE: $(1)_precheck +$(1)_precheck: + @if ! $$(PRECHECK_COMMAND_$(1)) 1>/dev/null 2>&1; then \ + echo "Execution of '$$(PRECHECK_COMMAND_$(1))' command failed. Is $(1) installed?"; \ + exit 1; \ + fi +endef diff --git a/api/prometheus/v1/api.go b/api/prometheus/v1/api.go index 255f3ba..6424188 100644 --- a/api/prometheus/v1/api.go +++ b/api/prometheus/v1/api.go @@ -56,12 +56,12 @@ type HealthStatus string const ( // Possible values for ErrorType. ErrBadData ErrorType = "bad_data" - ErrTimeout = "timeout" - ErrCanceled = "canceled" - ErrExec = "execution" - ErrBadResponse = "bad_response" - ErrServer = "server_error" - ErrClient = "client_error" + ErrTimeout ErrorType = "timeout" + ErrCanceled ErrorType = "canceled" + ErrExec ErrorType = "execution" + ErrBadResponse ErrorType = "bad_response" + ErrServer ErrorType = "server_error" + ErrClient ErrorType = "client_error" // Possible values for HealthStatus. HealthGood HealthStatus = "up" diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..69f9607 --- /dev/null +++ b/go.mod @@ -0,0 +1,12 @@ +module github.com/prometheus/client_golang + +require ( + github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 + github.com/golang/protobuf v1.2.0 + github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect + github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 + github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce + github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d + golang.org/x/net v0.0.0-20181114220301-adae6a3d119a + golang.org/x/sync v0.0.0-20181108010431-42b317875d0f // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..52c8dd1 --- /dev/null +++ b/go.sum @@ -0,0 +1,16 @@ +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 h1:idejC8f05m9MGOsuEi1ATq9shN03HrxNkD/luQvxCv8= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce h1:X0jFYGnHemYDIW6jlc+fSI8f9Cg+jqCnClYP2WgZT/A= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d h1:GoAlyOgbOEIFdaDqxJVlbOQ1DtGmZWs/Qau0hIlk+WQ= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a h1:gOpx8G595UYyvj8UK4+OFyY4rx037g3fmfhe5SasG3U= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f h1:Bl/8QSvNqXvPGPGXa2z5xUTmV7VDcZyvRZ+QQXkXTZQ= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= From a199184c14230ae23cb6edea6bc3b95b1a0fe924 Mon Sep 17 00:00:00 2001 From: Peter Jausovec Date: Mon, 19 Nov 2018 16:51:57 -0800 Subject: [PATCH 08/73] Update the label is not valid error message to include the fqName Fixes #504. Signed-off-by: Peter Jausovec --- prometheus/desc.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/prometheus/desc.go b/prometheus/desc.go index 7b8827f..1d034f8 100644 --- a/prometheus/desc.go +++ b/prometheus/desc.go @@ -93,7 +93,7 @@ func NewDesc(fqName, help string, variableLabels []string, constLabels Labels) * // First add only the const label names and sort them... for labelName := range constLabels { if !checkLabelName(labelName) { - d.err = fmt.Errorf("%q is not a valid label name", labelName) + d.err = fmt.Errorf("%q is not a valid label name for metric %q", labelName, fqName) return d } labelNames = append(labelNames, labelName) @@ -115,7 +115,7 @@ func NewDesc(fqName, help string, variableLabels []string, constLabels Labels) * // dimension with a different mix between preset and variable labels. for _, labelName := range variableLabels { if !checkLabelName(labelName) { - d.err = fmt.Errorf("%q is not a valid label name", labelName) + d.err = fmt.Errorf("%q is not a valid label name for metric %q", labelName, fqName) return d } labelNames = append(labelNames, "$"+labelName) From 8324e65597c59ecabe219affea5c64db2d8843ce Mon Sep 17 00:00:00 2001 From: "Dowideit, Sven (O&A, St. Lucia)" Date: Mon, 12 Nov 2018 15:19:09 +1000 Subject: [PATCH 09/73] Add the last few lines needed to show a new user how the Collector/Gatherer fits together Signed-off-by: Dowideit, Sven (O&A, St. Lucia) --- prometheus/example_clustermanager_test.go | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/prometheus/example_clustermanager_test.go b/prometheus/example_clustermanager_test.go index 9a5a4b8..4cafa55 100644 --- a/prometheus/example_clustermanager_test.go +++ b/prometheus/example_clustermanager_test.go @@ -13,7 +13,13 @@ package prometheus_test -import "github.com/prometheus/client_golang/prometheus" +import ( + "log" + "net/http" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" +) // ClusterManager is an example for a system that might have been built without // Prometheus in mind. It models a central manager of jobs running in a @@ -124,4 +130,13 @@ func ExampleCollector() { // variables to then do something with them. NewClusterManager("db", reg) NewClusterManager("ca", reg) + + // Add the built in process and golang metrics to this registry + reg.MustRegister( + prometheus.NewProcessCollector(prometheus.ProcessCollectorOpts{}), + prometheus.NewGoCollector(), + ) + + http.Handle("/metrics", promhttp.HandlerFor(reg, promhttp.HandlerOpts{})) + log.Fatal(http.ListenAndServe(":8080", nil)) } From 8d9c21f0cf5f73718ee86bb3a8e6432b88e9291d Mon Sep 17 00:00:00 2001 From: beorn7 Date: Tue, 20 Nov 2018 13:41:38 +0100 Subject: [PATCH 10/73] Nitpicking a doc comment Signed-off-by: beorn7 --- prometheus/example_clustermanager_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prometheus/example_clustermanager_test.go b/prometheus/example_clustermanager_test.go index 4cafa55..92b61ca 100644 --- a/prometheus/example_clustermanager_test.go +++ b/prometheus/example_clustermanager_test.go @@ -131,7 +131,7 @@ func ExampleCollector() { NewClusterManager("db", reg) NewClusterManager("ca", reg) - // Add the built in process and golang metrics to this registry + // Add the standard process and Go metrics to the custom registry. reg.MustRegister( prometheus.NewProcessCollector(prometheus.ProcessCollectorOpts{}), prometheus.NewGoCollector(), From 19f3737c3392d33dd2ba638919983643726e6bb6 Mon Sep 17 00:00:00 2001 From: beorn7 Date: Tue, 20 Nov 2018 19:39:44 +0100 Subject: [PATCH 11/73] Propagate Makefile.common fix Signed-off-by: beorn7 --- Makefile.common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.common b/Makefile.common index 0248fee..741579e 100644 --- a/Makefile.common +++ b/Makefile.common @@ -214,7 +214,7 @@ precheck:: $(1)_precheck PRECHECK_COMMAND_$(1) ?= $(1) $$(strip $$(PRECHECK_OPTIONS_$(1))) -.PHONE: $(1)_precheck +.PHONY: $(1)_precheck $(1)_precheck: @if ! $$(PRECHECK_COMMAND_$(1)) 1>/dev/null 2>&1; then \ echo "Execution of '$$(PRECHECK_COMMAND_$(1))' command failed. Is $(1) installed?"; \ From 48d3ae798bd09680713b606fab423f55d8fe6b40 Mon Sep 17 00:00:00 2001 From: beorn7 Date: Mon, 3 Dec 2018 21:10:34 +0100 Subject: [PATCH 12/73] Return observed duration from Timer.ObserveDuration Signed-off-by: beorn7 --- prometheus/timer.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/prometheus/timer.go b/prometheus/timer.go index b8fc5f1..8d5f105 100644 --- a/prometheus/timer.go +++ b/prometheus/timer.go @@ -39,13 +39,16 @@ func NewTimer(o Observer) *Timer { // ObserveDuration records the duration passed since the Timer was created with // NewTimer. It calls the Observe method of the Observer provided during -// construction with the duration in seconds as an argument. ObserveDuration is -// usually called with a defer statement. +// construction with the duration in seconds as an argument. The observed +// duration is also returned. ObserveDuration is usually called with a defer +// statement. // // Note that this method is only guaranteed to never observe negative durations // if used with Go1.9+. -func (t *Timer) ObserveDuration() { +func (t *Timer) ObserveDuration() time.Duration { + d := time.Since(t.begin) if t.observer != nil { - t.observer.Observe(time.Since(t.begin).Seconds()) + t.observer.Observe(d.Seconds()) } + return d } From 9542e4005c397e93b9fa31a4df1784693473d24d Mon Sep 17 00:00:00 2001 From: beorn7 Date: Thu, 6 Dec 2018 11:22:18 +0100 Subject: [PATCH 13/73] Expose #512 Signed-off-by: beorn7 --- prometheus/registry_test.go | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/prometheus/registry_test.go b/prometheus/registry_test.go index a96cb10..b9d9f13 100644 --- a/prometheus/registry_test.go +++ b/prometheus/registry_test.go @@ -21,6 +21,7 @@ package prometheus_test import ( "bytes" + "fmt" "io/ioutil" "math/rand" "net/http" @@ -783,6 +784,11 @@ func TestAlreadyRegistered(t *testing.T) { // same HistogramVec is registered concurrently and the Gather method of the // registry is called concurrently. func TestHistogramVecRegisterGatherConcurrency(t *testing.T) { + labelNames := make([]string, 16) // Need at least 13 to expose #512. + for i := range labelNames { + labelNames[i] = fmt.Sprint("label_", i) + } + var ( reg = prometheus.NewPedanticRegistry() hv = prometheus.NewHistogramVec( @@ -791,7 +797,7 @@ func TestHistogramVecRegisterGatherConcurrency(t *testing.T) { Help: "This helps testing.", ConstLabels: prometheus.Labels{"foo": "bar"}, }, - []string{"one", "two", "three"}, + labelNames, ) labelValues = []string{"a", "b", "c", "alpha", "beta", "gamma", "aleph", "beth", "gimel"} quit = make(chan struct{}) @@ -806,11 +812,11 @@ func TestHistogramVecRegisterGatherConcurrency(t *testing.T) { return default: obs := rand.NormFloat64()*.1 + .2 - hv.WithLabelValues( - labelValues[rand.Intn(len(labelValues))], - labelValues[rand.Intn(len(labelValues))], - labelValues[rand.Intn(len(labelValues))], - ).Observe(obs) + values := make([]string, 0, len(labelNames)) + for range labelNames { + values = append(values, labelValues[rand.Intn(len(labelValues))]) + } + hv.WithLabelValues(values...).Observe(obs) } } } @@ -848,7 +854,7 @@ func TestHistogramVecRegisterGatherConcurrency(t *testing.T) { if len(g) != 1 { t.Error("Gathered unexpected number of metric families:", len(g)) } - if len(g[0].Metric[0].Label) != 4 { + if len(g[0].Metric[0].Label) != len(labelNames)+1 { t.Error("Gathered unexpected number of label pairs:", len(g[0].Metric[0].Label)) } } From fae889635ce96b79dbe0882363682a9bf45b7268 Mon Sep 17 00:00:00 2001 From: beorn7 Date: Thu, 6 Dec 2018 11:35:30 +0100 Subject: [PATCH 14/73] Fix #512 Signed-off-by: beorn7 --- prometheus/registry.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/prometheus/registry.go b/prometheus/registry.go index f98c81a..8301fc8 100644 --- a/prometheus/registry.go +++ b/prometheus/registry.go @@ -872,7 +872,13 @@ func checkMetricConsistency( h = hashAddByte(h, separatorByte) // Make sure label pairs are sorted. We depend on it for the consistency // check. - sort.Sort(labelPairSorter(dtoMetric.Label)) + if !sort.IsSorted(labelPairSorter(dtoMetric.Label)) { + // We cannot sort dtoMetric.Label in place as it is immutable by contract. + copiedLabels := make([]*dto.LabelPair, len(dtoMetric.Label)) + copy(copiedLabels, dtoMetric.Label) + sort.Sort(labelPairSorter(copiedLabels)) + dtoMetric.Label = copiedLabels + } for _, lp := range dtoMetric.Label { h = hashAdd(h, lp.GetName()) h = hashAddByte(h, separatorByte) From 619eb595bafb3c0b2b1d94c22be1bd74a65c1a7c Mon Sep 17 00:00:00 2001 From: beorn7 Date: Thu, 6 Dec 2018 11:37:34 +0100 Subject: [PATCH 15/73] Simplify an `append` to `copy` Signed-off-by: beorn7 --- prometheus/registry.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/prometheus/registry.go b/prometheus/registry.go index 8301fc8..b5e70b9 100644 --- a/prometheus/registry.go +++ b/prometheus/registry.go @@ -909,8 +909,8 @@ func checkDescConsistency( } // Is the desc consistent with the content of the metric? - lpsFromDesc := make([]*dto.LabelPair, 0, len(dtoMetric.Label)) - lpsFromDesc = append(lpsFromDesc, desc.constLabelPairs...) + lpsFromDesc := make([]*dto.LabelPair, len(desc.constLabelPairs), len(dtoMetric.Label)) + copy(lpsFromDesc, desc.constLabelPairs) for _, l := range desc.variableLabels { lpsFromDesc = append(lpsFromDesc, &dto.LabelPair{ Name: proto.String(l), From e3fab98d54d430866cebff1f2024185847c6641c Mon Sep 17 00:00:00 2001 From: beorn7 Date: Thu, 6 Dec 2018 22:12:18 +0100 Subject: [PATCH 16/73] Cut v0.9.2 At this time, also update dependencies in go.mod (which I didn't explicitly mention in the CHANGELOG as this is the first release with Go modulie support anyway). Signed-off-by: beorn7 --- CHANGELOG.md | 8 ++++++++ VERSION | 2 +- go.mod | 6 +++--- go.sum | 12 ++++++------ 4 files changed, 18 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d93880..fcfa112 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,11 @@ +## 0.9.2 / 2018-12-06 +* [FEATURE] Support for Go modules. #501 +* [FEATURE] `Timer.ObserveDuration` returns observed duration. #509 +* [ENHANCEMENT] Improved doc comments and error messages. #504 +* [BUGFIX] Fix race condition during metrics gathering. #512 +* [BUGFIX] Fix testutil metric comparison for Histograms and empty labels. #494 + #498 + ## 0.9.1 / 2018-11-03 * [FEATURE] Add `WriteToTextfile` function to facilitate the creation of *.prom files for the textfile collector of the node exporter. #489 diff --git a/VERSION b/VERSION index f374f66..2003b63 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.9.1 +0.9.2 diff --git a/go.mod b/go.mod index 69f9607..6ec1588 100644 --- a/go.mod +++ b/go.mod @@ -5,8 +5,8 @@ require ( github.com/golang/protobuf v1.2.0 github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 - github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce - github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d - golang.org/x/net v0.0.0-20181114220301-adae6a3d119a + github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 + github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a + golang.org/x/net v0.0.0-20181201002055-351d144fa1fc golang.org/x/sync v0.0.0-20181108010431-42b317875d0f // indirect ) diff --git a/go.sum b/go.sum index 52c8dd1..181c46d 100644 --- a/go.sum +++ b/go.sum @@ -6,11 +6,11 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0j github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 h1:idejC8f05m9MGOsuEi1ATq9shN03HrxNkD/luQvxCv8= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce h1:X0jFYGnHemYDIW6jlc+fSI8f9Cg+jqCnClYP2WgZT/A= -github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d h1:GoAlyOgbOEIFdaDqxJVlbOQ1DtGmZWs/Qau0hIlk+WQ= -github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -golang.org/x/net v0.0.0-20181114220301-adae6a3d119a h1:gOpx8G595UYyvj8UK4+OFyY4rx037g3fmfhe5SasG3U= -golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 h1:PnBWHBf+6L0jOqq0gIVUe6Yk0/QMZ640k6NvkxcBf+8= +github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a h1:9a8MnZMP0X2nLJdBg+pBmGgkJlSaKC2KaQmTCk1XDtE= +github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc h1:a3CU5tJYVj92DY2LaA1kUkrsqD5/3mLDhx2NcNqyW+0= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f h1:Bl/8QSvNqXvPGPGXa2z5xUTmV7VDcZyvRZ+QQXkXTZQ= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= From 000ceb266b84c4d1c71f9ac92e937a19632d30d2 Mon Sep 17 00:00:00 2001 From: beorn7 Date: Tue, 18 Dec 2018 16:30:34 +0100 Subject: [PATCH 17/73] Fix doc comment typo Signed-off-by: beorn7 --- prometheus/registry.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prometheus/registry.go b/prometheus/registry.go index b5e70b9..f2fb67a 100644 --- a/prometheus/registry.go +++ b/prometheus/registry.go @@ -680,7 +680,7 @@ func processMetric( // Gatherers is a slice of Gatherer instances that implements the Gatherer // interface itself. Its Gather method calls Gather on all Gatherers in the // slice in order and returns the merged results. Errors returned from the -// Gather calles are all returned in a flattened MultiError. Duplicate and +// Gather calls are all returned in a flattened MultiError. Duplicate and // inconsistent Metrics are skipped (first occurrence in slice order wins) and // reported in the returned error. // From cb063c2bf24a8bbb45f88164ea26fb027563d36b Mon Sep 17 00:00:00 2001 From: beorn7 Date: Tue, 18 Dec 2018 17:11:19 +0100 Subject: [PATCH 18/73] Update vendoring Signed-off-by: beorn7 --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 6ec1588..12321ef 100644 --- a/go.mod +++ b/go.mod @@ -5,8 +5,8 @@ require ( github.com/golang/protobuf v1.2.0 github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 - github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 + github.com/prometheus/common v0.0.0-20181218105931-67670fe90761 github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a - golang.org/x/net v0.0.0-20181201002055-351d144fa1fc + golang.org/x/net v0.0.0-20181217023233-e147a9138326 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f // indirect ) diff --git a/go.sum b/go.sum index 181c46d..223eaee 100644 --- a/go.sum +++ b/go.sum @@ -6,11 +6,11 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0j github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 h1:idejC8f05m9MGOsuEi1ATq9shN03HrxNkD/luQvxCv8= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 h1:PnBWHBf+6L0jOqq0gIVUe6Yk0/QMZ640k6NvkxcBf+8= -github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.0.0-20181218105931-67670fe90761 h1:z6tvbDJ5OLJ48FFmnksv04a78maSTRBUIhkdHYV5Y98= +github.com/prometheus/common v0.0.0-20181218105931-67670fe90761/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a h1:9a8MnZMP0X2nLJdBg+pBmGgkJlSaKC2KaQmTCk1XDtE= github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -golang.org/x/net v0.0.0-20181201002055-351d144fa1fc h1:a3CU5tJYVj92DY2LaA1kUkrsqD5/3mLDhx2NcNqyW+0= -golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181217023233-e147a9138326 h1:iCzOf0xz39Tstp+Tu/WwyGjUXCk34QhQORRxBeXXTA4= +golang.org/x/net v0.0.0-20181217023233-e147a9138326/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f h1:Bl/8QSvNqXvPGPGXa2z5xUTmV7VDcZyvRZ+QQXkXTZQ= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= From 2c9811f88eb56c7eeb1e2262b10a17ef727cfc91 Mon Sep 17 00:00:00 2001 From: beorn7 Date: Tue, 18 Dec 2018 17:20:25 +0100 Subject: [PATCH 19/73] Fix tests to adhere to the recent change in prometheus/common Signed-off-by: beorn7 --- prometheus/promhttp/http_test.go | 14 ++++---- prometheus/registry_test.go | 52 ++++++++++++++-------------- prometheus/testutil/testutil_test.go | 30 ++++++++-------- 3 files changed, 48 insertions(+), 48 deletions(-) diff --git a/prometheus/promhttp/http_test.go b/prometheus/promhttp/http_test.go index 6e23e6c..0738276 100644 --- a/prometheus/promhttp/http_test.go +++ b/prometheus/promhttp/http_test.go @@ -109,11 +109,11 @@ error collecting metric Desc{fqName: "invalid_metric", help: "not helpful", cons ` wantOKBody := `# HELP name docstring # TYPE name counter -name{constname="constvalue",labelname="val1"} 1 -name{constname="constvalue",labelname="val2"} 1 +name{constname="constvalue",labelname="val1"} 1.0 +name{constname="constvalue",labelname="val2"} 1.0 # HELP the_count Ah-ah-ah! Thunder and lightning! # TYPE the_count counter -the_count 0 +the_count 0.0 ` errorHandler.ServeHTTP(writer, request) @@ -163,11 +163,11 @@ func TestInstrumentMetricHandler(t *testing.T) { t.Errorf("got HTTP status code %d, want %d", got, want) } - want := "promhttp_metric_handler_requests_in_flight 1\n" + want := "promhttp_metric_handler_requests_in_flight 1.0\n" if got := writer.Body.String(); !strings.Contains(got, want) { t.Errorf("got body %q, does not contain %q", got, want) } - want = "promhttp_metric_handler_requests_total{code=\"200\"} 0\n" + want = "promhttp_metric_handler_requests_total{code=\"200\"} 0.0\n" if got := writer.Body.String(); !strings.Contains(got, want) { t.Errorf("got body %q, does not contain %q", got, want) } @@ -178,11 +178,11 @@ func TestInstrumentMetricHandler(t *testing.T) { t.Errorf("got HTTP status code %d, want %d", got, want) } - want = "promhttp_metric_handler_requests_in_flight 1\n" + want = "promhttp_metric_handler_requests_in_flight 1.0\n" if got := writer.Body.String(); !strings.Contains(got, want) { t.Errorf("got body %q, does not contain %q", got, want) } - want = "promhttp_metric_handler_requests_total{code=\"200\"} 1\n" + want = "promhttp_metric_handler_requests_total{code=\"200\"} 1.0\n" if got := writer.Body.String(); !strings.Contains(got, want) { t.Errorf("got body %q, does not contain %q", got, want) } diff --git a/prometheus/registry_test.go b/prometheus/registry_test.go index b9d9f13..6b6aaf3 100644 --- a/prometheus/registry_test.go +++ b/prometheus/registry_test.go @@ -99,7 +99,7 @@ func testHandler(t testing.TB) { externalMetricFamilyAsBytes := externalBuf.Bytes() externalMetricFamilyAsText := []byte(`# HELP externalname externaldocstring # TYPE externalname counter -externalname{externalconstname="externalconstvalue",externallabelname="externalval1"} 1 +externalname{externalconstname="externalconstvalue",externallabelname="externalval1"} 1.0 `) externalMetricFamilyAsProtoText := []byte(`name: "externalname" help: "externaldocstring" @@ -167,8 +167,8 @@ metric: < expectedMetricFamilyAsBytes := buf.Bytes() expectedMetricFamilyAsText := []byte(`# HELP name docstring # TYPE name counter -name{constname="constvalue",labelname="val1"} 1 -name{constname="constvalue",labelname="val2"} 1 +name{constname="constvalue",labelname="val1"} 1.0 +name{constname="constvalue",labelname="val2"} 1.0 `) expectedMetricFamilyAsProtoText := []byte(`name: "name" help: "docstring" @@ -267,8 +267,8 @@ collected metric "name" { label: label: Date: Fri, 21 Dec 2018 15:40:04 -0800 Subject: [PATCH 20/73] Fix typo Signed-off-by: gkze --- prometheus/doc.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prometheus/doc.go b/prometheus/doc.go index 5d9525d..11c1766 100644 --- a/prometheus/doc.go +++ b/prometheus/doc.go @@ -128,7 +128,7 @@ // errors. Thus, with unchecked Collectors, the responsibility to not collect // metrics that lead to inconsistencies in the total scrape result lies with the // implementer of the Collector. While this is not a desirable state, it is -// sometimes necessary. The typical use case is a situatios where the exact +// sometimes necessary. The typical use case is a situations where the exact // metrics to be returned by a Collector cannot be predicted at registration // time, but the implementer has sufficient knowledge of the whole system to // guarantee metric consistency. From a006a7550a7ee632d7e836c7500cedaa1aab9ccc Mon Sep 17 00:00:00 2001 From: beorn7 Date: Mon, 24 Dec 2018 11:22:57 +0100 Subject: [PATCH 21/73] Remove unused constants and fields Signed-off-by: beorn7 --- prometheus/http.go | 1 - prometheus/promhttp/delegator.go | 1 - prometheus/promhttp/http.go | 1 - 3 files changed, 3 deletions(-) diff --git a/prometheus/http.go b/prometheus/http.go index 9f0875b..0fa339a 100644 --- a/prometheus/http.go +++ b/prometheus/http.go @@ -34,7 +34,6 @@ import ( const ( contentTypeHeader = "Content-Type" - contentLengthHeader = "Content-Length" contentEncodingHeader = "Content-Encoding" acceptEncodingHeader = "Accept-Encoding" ) diff --git a/prometheus/promhttp/delegator.go b/prometheus/promhttp/delegator.go index 67b56d3..5de5bbc 100644 --- a/prometheus/promhttp/delegator.go +++ b/prometheus/promhttp/delegator.go @@ -38,7 +38,6 @@ type delegator interface { type responseWriterDelegator struct { http.ResponseWriter - handler, method string status int written int64 wroteHeader bool diff --git a/prometheus/promhttp/http.go b/prometheus/promhttp/http.go index 668eb6b..b137c88 100644 --- a/prometheus/promhttp/http.go +++ b/prometheus/promhttp/http.go @@ -47,7 +47,6 @@ import ( const ( contentTypeHeader = "Content-Type" - contentLengthHeader = "Content-Length" contentEncodingHeader = "Content-Encoding" acceptEncodingHeader = "Accept-Encoding" ) From 226b83ac2ed99358992f6571f36597b6391f2058 Mon Sep 17 00:00:00 2001 From: beorn7 Date: Mon, 24 Dec 2018 11:23:13 +0100 Subject: [PATCH 22/73] Provide lock-free implementation for Summary without objectives Signed-off-by: beorn7 --- prometheus/histogram.go | 4 +- prometheus/summary.go | 149 +++++++++++++++++++++++++++++++++++++ prometheus/summary_test.go | 8 ++ 3 files changed, 159 insertions(+), 2 deletions(-) diff --git a/prometheus/histogram.go b/prometheus/histogram.go index f88da70..20ee1af 100644 --- a/prometheus/histogram.go +++ b/prometheus/histogram.go @@ -204,8 +204,8 @@ func newHistogram(desc *Desc, opts HistogramOpts, labelValues ...string) Histogr } } } - // Finally we know the final length of h.upperBounds and can make counts - // for both states: + // Finally we know the final length of h.upperBounds and can make buckets + // for both counts: h.counts[0].buckets = make([]uint64, len(h.upperBounds)) h.counts[1].buckets = make([]uint64, len(h.upperBounds)) diff --git a/prometheus/summary.go b/prometheus/summary.go index 2980614..577fd49 100644 --- a/prometheus/summary.go +++ b/prometheus/summary.go @@ -16,8 +16,10 @@ package prometheus import ( "fmt" "math" + "runtime" "sort" "sync" + "sync/atomic" "time" "github.com/beorn7/perks/quantile" @@ -214,6 +216,17 @@ func newSummary(desc *Desc, opts SummaryOpts, labelValues ...string) Summary { opts.BufCap = DefBufCap } + if len(opts.Objectives) == 0 { + // Use the lock-free implementation of a Summary without objectives. + s := &noObjectivesSummary{ + desc: desc, + labelPairs: makeLabelPairs(desc, labelValues), + counts: [2]*summaryCounts{&summaryCounts{}, &summaryCounts{}}, + } + s.init(s) // Init self-collection. + return s + } + s := &summary{ desc: desc, @@ -382,6 +395,142 @@ func (s *summary) swapBufs(now time.Time) { } } +type summaryCounts struct { + // sumBits contains the bits of the float64 representing the sum of all + // observations. sumBits and count have to go first in the struct to + // guarantee alignment for atomic operations. + // http://golang.org/pkg/sync/atomic/#pkg-note-BUG + sumBits uint64 + count uint64 +} + +type noObjectivesSummary struct { + // countAndHotIdx is a complicated one. For lock-free yet atomic + // observations, we need to save the total count of observations again, + // combined with the index of the currently-hot counts struct, so that + // we can perform the operation on both values atomically. The least + // significant bit defines the hot counts struct. The remaining 63 bits + // represent the total count of observations. This happens under the + // assumption that the 63bit count will never overflow. Rationale: An + // observations takes about 30ns. Let's assume it could happen in + // 10ns. Overflowing the counter will then take at least (2^63)*10ns, + // which is about 3000 years. + // + // This has to be first in the struct for 64bit alignment. See + // http://golang.org/pkg/sync/atomic/#pkg-note-BUG + countAndHotIdx uint64 + + selfCollector + desc *Desc + writeMtx sync.Mutex // Only used in the Write method. + + // Two counts, one is "hot" for lock-free observations, the other is + // "cold" for writing out a dto.Metric. It has to be an array of + // pointers to guarantee 64bit alignment of the histogramCounts, see + // http://golang.org/pkg/sync/atomic/#pkg-note-BUG. + counts [2]*summaryCounts + hotIdx int // Index of currently-hot counts. Only used within Write. + + labelPairs []*dto.LabelPair +} + +func (s *noObjectivesSummary) Desc() *Desc { + return s.desc +} + +func (s *noObjectivesSummary) Observe(v float64) { + // We increment s.countAndHotIdx by 2 so that the counter in the upper + // 63 bits gets incremented by 1. At the same time, we get the new value + // back, which we can use to find the currently-hot counts. + n := atomic.AddUint64(&s.countAndHotIdx, 2) + hotCounts := s.counts[n%2] + + for { + oldBits := atomic.LoadUint64(&hotCounts.sumBits) + newBits := math.Float64bits(math.Float64frombits(oldBits) + v) + if atomic.CompareAndSwapUint64(&hotCounts.sumBits, oldBits, newBits) { + break + } + } + // Increment count last as we take it as a signal that the observation + // is complete. + atomic.AddUint64(&hotCounts.count, 1) +} + +func (s *noObjectivesSummary) Write(out *dto.Metric) error { + var ( + sum = &dto.Summary{} + hotCounts, coldCounts *summaryCounts + count uint64 + ) + + // For simplicity, we mutex the rest of this method. It is not in the + // hot path, i.e. Observe is called much more often than Write. The + // complication of making Write lock-free isn't worth it. + s.writeMtx.Lock() + defer s.writeMtx.Unlock() + + // This is a bit arcane, which is why the following spells out this if + // clause in English: + // + // If the currently-hot counts struct is #0, we atomically increment + // s.countAndHotIdx by 1 so that from now on Observe will use the counts + // struct #1. Furthermore, the atomic increment gives us the new value, + // which, in its most significant 63 bits, tells us the count of + // observations done so far up to and including currently ongoing + // observations still using the counts struct just changed from hot to + // cold. To have a normal uint64 for the count, we bitshift by 1 and + // save the result in count. We also set s.hotIdx to 1 for the next + // Write call, and we will refer to counts #1 as hotCounts and to counts + // #0 as coldCounts. + // + // If the currently-hot counts struct is #1, we do the corresponding + // things the other way round. We have to _decrement_ s.countAndHotIdx + // (which is a bit arcane in itself, as we have to express -1 with an + // unsigned int...). + if s.hotIdx == 0 { + count = atomic.AddUint64(&s.countAndHotIdx, 1) >> 1 + s.hotIdx = 1 + hotCounts = s.counts[1] + coldCounts = s.counts[0] + } else { + count = atomic.AddUint64(&s.countAndHotIdx, ^uint64(0)) >> 1 // Decrement. + s.hotIdx = 0 + hotCounts = s.counts[0] + coldCounts = s.counts[1] + } + + // Now we have to wait for the now-declared-cold counts to actually cool + // down, i.e. wait for all observations still using it to finish. That's + // the case once the count in the cold counts struct is the same as the + // one atomically retrieved from the upper 63bits of s.countAndHotIdx. + for { + if count == atomic.LoadUint64(&coldCounts.count) { + break + } + runtime.Gosched() // Let observations get work done. + } + + sum.SampleCount = proto.Uint64(count) + sum.SampleSum = proto.Float64(math.Float64frombits(atomic.LoadUint64(&coldCounts.sumBits))) + + out.Summary = sum + out.Label = s.labelPairs + + // Finally add all the cold counts to the new hot counts and reset the cold counts. + atomic.AddUint64(&hotCounts.count, count) + atomic.StoreUint64(&coldCounts.count, 0) + for { + oldBits := atomic.LoadUint64(&hotCounts.sumBits) + newBits := math.Float64bits(math.Float64frombits(oldBits) + sum.GetSampleSum()) + if atomic.CompareAndSwapUint64(&hotCounts.sumBits, oldBits, newBits) { + atomic.StoreUint64(&coldCounts.sumBits, 0) + break + } + } + return nil +} + type quantSort []*dto.Quantile func (s quantSort) Len() int { diff --git a/prometheus/summary_test.go b/prometheus/summary_test.go index 8b1a62e..26f693d 100644 --- a/prometheus/summary_test.go +++ b/prometheus/summary_test.go @@ -54,11 +54,19 @@ func TestSummaryWithoutObjectives(t *testing.T) { if err := reg.Register(summaryWithEmptyObjectives); err != nil { t.Error(err) } + summaryWithEmptyObjectives.Observe(3) + summaryWithEmptyObjectives.Observe(0.14) m := &dto.Metric{} if err := summaryWithEmptyObjectives.Write(m); err != nil { t.Error(err) } + if got, want := m.GetSummary().GetSampleSum(), 3.14; got != want { + t.Errorf("got sample sum %f, want %f", got, want) + } + if got, want := m.GetSummary().GetSampleCount(), uint64(2); got != want { + t.Errorf("got sample sum %d, want %d", got, want) + } if len(m.GetSummary().Quantile) != 0 { t.Error("expected no objectives in summary") } From ee4e5e071ede9a2cbcbfa9ccfcdb9d561c928b51 Mon Sep 17 00:00:00 2001 From: gkze Date: Wed, 26 Dec 2018 17:39:36 -0800 Subject: [PATCH 23/73] Fix typo again Signed-off-by: gkze --- prometheus/doc.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prometheus/doc.go b/prometheus/doc.go index 11c1766..e7f86dd 100644 --- a/prometheus/doc.go +++ b/prometheus/doc.go @@ -128,7 +128,7 @@ // errors. Thus, with unchecked Collectors, the responsibility to not collect // metrics that lead to inconsistencies in the total scrape result lies with the // implementer of the Collector. While this is not a desirable state, it is -// sometimes necessary. The typical use case is a situations where the exact +// sometimes necessary. The typical use case is a situation where the exact // metrics to be returned by a Collector cannot be predicted at registration // time, but the implementer has sufficient knowledge of the whole system to // guarantee metric consistency. From fb3d5cb2ad5789367093b409855a3937d651b572 Mon Sep 17 00:00:00 2001 From: JoeWrightss <42261994+JoeWrightss@users.noreply.github.com> Date: Mon, 31 Dec 2018 04:31:21 +0800 Subject: [PATCH 24/73] Fix some spelling errors (#523) * Fix some spelling errors Signed-off-by: JoeWrightss --- prometheus/doc.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/prometheus/doc.go b/prometheus/doc.go index e7f86dd..1e0d578 100644 --- a/prometheus/doc.go +++ b/prometheus/doc.go @@ -122,8 +122,8 @@ // the Collect method. The Describe method has to return separate Desc // instances, representative of the “throw-away” metrics to be created later. // NewDesc comes in handy to create those Desc instances. Alternatively, you -// could return no Desc at all, which will marke the Collector “unchecked”. No -// checks are porformed at registration time, but metric consistency will still +// could return no Desc at all, which will mark the Collector “unchecked”. No +// checks are performed at registration time, but metric consistency will still // be ensured at scrape time, i.e. any inconsistencies will lead to scrape // errors. Thus, with unchecked Collectors, the responsibility to not collect // metrics that lead to inconsistencies in the total scrape result lies with the From c51ed6941c1016c327e930580a70492641b24da2 Mon Sep 17 00:00:00 2001 From: Roman Krasavtsev Date: Fri, 4 Jan 2019 21:07:29 +0100 Subject: [PATCH 25/73] fix typo Signed-off-by: Roman Krasavtsev --- prometheus/collector.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prometheus/collector.go b/prometheus/collector.go index c0d70b2..1e83965 100644 --- a/prometheus/collector.go +++ b/prometheus/collector.go @@ -79,7 +79,7 @@ type Collector interface { // of the Describe method. If a Collector sometimes collects no metrics at all // (for example vectors like CounterVec, GaugeVec, etc., which only collect // metrics after a metric with a fully specified label set has been accessed), -// it might even get registered as an unchecked Collecter (cf. the Register +// it might even get registered as an unchecked Collector (cf. the Register // method of the Registerer interface). Hence, only use this shortcut // implementation of Describe if you are certain to fulfill the contract. // From 1749edba401a182b6c4bb4931441e09cbe4e7f48 Mon Sep 17 00:00:00 2001 From: beorn7 Date: Sat, 5 Jan 2019 21:56:06 +0100 Subject: [PATCH 26/73] Update Makefile.common to current upstream version This should fix the staticcheck test failures with Go 1.10 Signed-off-by: beorn7 --- Makefile.common | 56 +++++++++++++++++++++++++++++-------------------- 1 file changed, 33 insertions(+), 23 deletions(-) diff --git a/Makefile.common b/Makefile.common index 741579e..fff85f9 100644 --- a/Makefile.common +++ b/Makefile.common @@ -29,6 +29,8 @@ GO ?= go GOFMT ?= $(GO)fmt FIRST_GOPATH := $(firstword $(subst :, ,$(shell $(GO) env GOPATH))) GOOPTS ?= +GOHOSTOS ?= $(shell $(GO) env GOHOSTOS) +GOHOSTARCH ?= $(shell $(GO) env GOHOSTARCH) GO_VERSION ?= $(shell $(GO) version) GO_VERSION_NUMBER ?= $(word 3, $(GO_VERSION)) @@ -62,17 +64,30 @@ PROMU := $(FIRST_GOPATH)/bin/promu STATICCHECK := $(FIRST_GOPATH)/bin/staticcheck pkgs = ./... -GO_VERSION ?= $(shell $(GO) version) -GO_BUILD_PLATFORM ?= $(subst /,-,$(lastword $(GO_VERSION))) +ifeq (arm, $(GOHOSTARCH)) + GOHOSTARM ?= $(shell GOARM= $(GO) env GOARM) + GO_BUILD_PLATFORM ?= $(GOHOSTOS)-$(GOHOSTARCH)v$(GOHOSTARM) +else + GO_BUILD_PLATFORM ?= $(GOHOSTOS)-$(GOHOSTARCH) +endif PROMU_VERSION ?= 0.2.0 PROMU_URL := https://github.com/prometheus/promu/releases/download/v$(PROMU_VERSION)/promu-$(PROMU_VERSION).$(GO_BUILD_PLATFORM).tar.gz +STATICCHECK_VERSION ?= 2019.1 +STATICCHECK_URL := https://github.com/dominikh/go-tools/releases/download/$(STATICCHECK_VERSION)/staticcheck_$(GOHOSTOS)_$(GOHOSTARCH) PREFIX ?= $(shell pwd) BIN_DIR ?= $(shell pwd) DOCKER_IMAGE_TAG ?= $(subst /,-,$(shell git rev-parse --abbrev-ref HEAD)) DOCKER_REPO ?= prom +ifeq ($(GOHOSTARCH),amd64) + ifeq ($(GOHOSTOS),$(filter $(GOHOSTOS),linux freebsd darwin windows)) + # Only supported on amd64 + test-flags := -race + endif +endif + .PHONY: all all: precheck style staticcheck unused build test @@ -110,12 +125,12 @@ common-test-short: .PHONY: common-test common-test: @echo ">> running all tests" - GO111MODULE=$(GO111MODULE) $(GO) test -race $(GOOPTS) $(pkgs) + GO111MODULE=$(GO111MODULE) $(GO) test $(test-flags) $(GOOPTS) $(pkgs) .PHONY: common-format common-format: @echo ">> formatting code" - GO111MODULE=$(GO111MODULE) $(GO) fmt $(GOOPTS) $(pkgs) + GO111MODULE=$(GO111MODULE) $(GO) fmt $(pkgs) .PHONY: common-vet common-vet: @@ -125,8 +140,12 @@ common-vet: .PHONY: common-staticcheck common-staticcheck: $(STATICCHECK) @echo ">> running staticcheck" + chmod +x $(STATICCHECK) ifdef GO111MODULE - GO111MODULE=$(GO111MODULE) $(STATICCHECK) -ignore "$(STATICCHECK_IGNORE)" -checks "SA*" $(pkgs) +# 'go list' needs to be executed before staticcheck to prepopulate the modules cache. +# Otherwise staticcheck might fail randomly for some reason not yet explained. + GO111MODULE=$(GO111MODULE) $(GO) list -e -compiled -test=true -export=false -deps=true -find=false -tags= -- ./... > /dev/null + GO111MODULE=$(GO111MODULE) $(STATICCHECK) -ignore "$(STATICCHECK_IGNORE)" $(pkgs) else $(STATICCHECK) -ignore "$(STATICCHECK_IGNORE)" $(pkgs) endif @@ -140,8 +159,9 @@ else ifdef GO111MODULE @echo ">> running check for unused/missing packages in go.mod" GO111MODULE=$(GO111MODULE) $(GO) mod tidy +ifeq (,$(wildcard vendor)) @git diff --exit-code -- go.sum go.mod -ifneq (,$(wildcard vendor)) +else @echo ">> running check for unused packages in vendor/" GO111MODULE=$(GO111MODULE) $(GO) mod vendor @git diff --exit-code -- go.sum go.mod vendor/ @@ -175,30 +195,20 @@ common-docker-tag-latest: promu: $(PROMU) $(PROMU): - curl -s -L $(PROMU_URL) | tar -xvz -C /tmp - mkdir -v -p $(FIRST_GOPATH)/bin - cp -v /tmp/promu-$(PROMU_VERSION).$(GO_BUILD_PLATFORM)/promu $(PROMU) + $(eval PROMU_TMP := $(shell mktemp -d)) + curl -s -L $(PROMU_URL) | tar -xvzf - -C $(PROMU_TMP) + mkdir -p $(FIRST_GOPATH)/bin + cp $(PROMU_TMP)/promu-$(PROMU_VERSION).$(GO_BUILD_PLATFORM)/promu $(FIRST_GOPATH)/bin/promu + rm -r $(PROMU_TMP) .PHONY: proto proto: @echo ">> generating code from proto files" @./scripts/genproto.sh -.PHONY: $(STATICCHECK) $(STATICCHECK): -ifdef GO111MODULE -# Get staticcheck from a temporary directory to avoid modifying the local go.{mod,sum}. -# See https://github.com/golang/go/issues/27643. -# For now, we are using the next branch of staticcheck because master isn't compatible yet with Go modules. - tmpModule=$$(mktemp -d 2>&1) && \ - mkdir -p $${tmpModule}/staticcheck && \ - cd "$${tmpModule}"/staticcheck && \ - GO111MODULE=on $(GO) mod init example.com/staticcheck && \ - GO111MODULE=on GOOS= GOARCH= $(GO) get -u honnef.co/go/tools/cmd/staticcheck@next && \ - rm -rf $${tmpModule}; -else - GOOS= GOARCH= GO111MODULE=off $(GO) get -u honnef.co/go/tools/cmd/staticcheck -endif + mkdir -p $(FIRST_GOPATH)/bin + curl -s -L $(STATICCHECK_URL) > $(STATICCHECK) ifdef GOVENDOR .PHONY: $(GOVENDOR) From 91b385db444376f4309468ee207db596a2f8e56c Mon Sep 17 00:00:00 2001 From: beorn7 Date: Sat, 5 Jan 2019 22:11:07 +0100 Subject: [PATCH 27/73] Fix staticcheck errors in push_test.go Signed-off-by: beorn7 --- prometheus/push/push_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/prometheus/push/push_test.go b/prometheus/push/push_test.go index 34ec334..5e19f39 100644 --- a/prometheus/push/push_test.go +++ b/prometheus/push/push_test.go @@ -96,7 +96,7 @@ func TestPush(t *testing.T) { if lastMethod != "PUT" { t.Error("want method PUT for Push, got", lastMethod) } - if bytes.Compare(lastBody, wantBody) != 0 { + if !bytes.Equal(lastBody, wantBody) { t.Errorf("got body %v, want %v", lastBody, wantBody) } if lastPath != "/metrics/job/testjob" { @@ -113,7 +113,7 @@ func TestPush(t *testing.T) { if lastMethod != "POST" { t.Error("want method POST for Add, got", lastMethod) } - if bytes.Compare(lastBody, wantBody) != 0 { + if !bytes.Equal(lastBody, wantBody) { t.Errorf("got body %v, want %v", lastBody, wantBody) } if lastPath != "/metrics/job/testjob" { @@ -170,7 +170,7 @@ func TestPush(t *testing.T) { if lastMethod != "PUT" { t.Error("want method PUT for Push, got", lastMethod) } - if bytes.Compare(lastBody, wantBody) != 0 { + if !bytes.Equal(lastBody, wantBody) { t.Errorf("got body %v, want %v", lastBody, wantBody) } @@ -185,7 +185,7 @@ func TestPush(t *testing.T) { if lastMethod != "POST" { t.Error("want method POST for Add, got", lastMethod) } - if bytes.Compare(lastBody, wantBody) != 0 { + if !bytes.Equal(lastBody, wantBody) { t.Errorf("got body %v, want %v", lastBody, wantBody) } if lastPath != "/metrics/job/testjob/a/x/b/y" && lastPath != "/metrics/job/testjob/b/y/a/x" { From b3d60964321a277b1ab2c18d912f204acc30a7c3 Mon Sep 17 00:00:00 2001 From: beorn7 Date: Sun, 6 Jan 2019 00:22:03 +0100 Subject: [PATCH 28/73] Improve testutil error messages This also satisfy newer staticcheck versions. Signed-off-by: beorn7 --- prometheus/testutil/testutil.go | 4 +--- prometheus/testutil/testutil_test.go | 5 +---- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/prometheus/testutil/testutil.go b/prometheus/testutil/testutil.go index e344cc8..4c72bc6 100644 --- a/prometheus/testutil/testutil.go +++ b/prometheus/testutil/testutil.go @@ -165,11 +165,9 @@ func compare(got, want []*dto.MetricFamily) error { metric output does not match expectation; want: %s - got: -%s -`, wantBuf.String(), gotBuf.String()) +%s`, wantBuf.String(), gotBuf.String()) } return nil diff --git a/prometheus/testutil/testutil_test.go b/prometheus/testutil/testutil_test.go index 4812247..e1f7aab 100644 --- a/prometheus/testutil/testutil_test.go +++ b/prometheus/testutil/testutil_test.go @@ -171,7 +171,6 @@ func TestCollectAndCompareHistogram(t *testing.T) { c prometheus.Collector metadata string expect string - labels []string observation float64 }{ { @@ -291,13 +290,11 @@ metric output does not match expectation; want: # TYPE some_other_metric counter some_other_metric{label1="value1"} 1.0 - got: # HELP some_total A value that represents a counter. # TYPE some_total counter some_total{label1="value1"} 1.0 - ` err := CollectAndCompare(c, strings.NewReader(metadata+expected)) @@ -306,6 +303,6 @@ some_total{label1="value1"} 1.0 } if err.Error() != expectedError { - t.Errorf("Expected\n%#+v\nGot:\n%#+v\n", expectedError, err.Error()) + t.Errorf("Expected\n%#+v\nGot:\n%#+v", expectedError, err.Error()) } } From 18d13eacc9cce330610a70daf4ed0fef2e846589 Mon Sep 17 00:00:00 2001 From: Bob Shannon Date: Tue, 15 Jan 2019 16:34:51 -0500 Subject: [PATCH 29/73] Add support for fetching rules with API client (#508) This PR adds support for fetching [rules](https://prometheus.io/docs/prometheus/latest/querying/api/#rules) via the `/api/v1/rules` endpoint using the API client. Currently, the API client exposes no way to do this and it would be nice to have for external systems that wish to retrieve this information. --- api/prometheus/v1/api.go | 207 ++++++++++++++++++++++++++++++++++ api/prometheus/v1/api_test.go | 108 ++++++++++++++++++ 2 files changed, 315 insertions(+) diff --git a/api/prometheus/v1/api.go b/api/prometheus/v1/api.go index 6424188..345c0de 100644 --- a/api/prometheus/v1/api.go +++ b/api/prometheus/v1/api.go @@ -20,6 +20,7 @@ package v1 import ( "context" "encoding/json" + "errors" "fmt" "net/http" "strconv" @@ -40,6 +41,7 @@ const ( epLabelValues = apiPrefix + "/label/:name/values" epSeries = apiPrefix + "/series" epTargets = apiPrefix + "/targets" + epRules = apiPrefix + "/rules" epSnapshot = apiPrefix + "/admin/tsdb/snapshot" epDeleteSeries = apiPrefix + "/admin/tsdb/delete_series" epCleanTombstones = apiPrefix + "/admin/tsdb/clean_tombstones" @@ -47,13 +49,27 @@ const ( epFlags = apiPrefix + "/status/flags" ) +// AlertState models the state of an alert. +type AlertState string + // ErrorType models the different API error types. type ErrorType string // HealthStatus models the health status of a scrape target. type HealthStatus string +// RuleType models the type of a rule. +type RuleType string + +// RuleHealth models the health status of a rule. +type RuleHealth string + const ( + // Possible values for AlertState. + AlertStateFiring AlertState = "firing" + AlertStateInactive AlertState = "inactive" + AlertStatePending AlertState = "pending" + // Possible values for ErrorType. ErrBadData ErrorType = "bad_data" ErrTimeout ErrorType = "timeout" @@ -67,6 +83,15 @@ const ( HealthGood HealthStatus = "up" HealthUnknown HealthStatus = "unknown" HealthBad HealthStatus = "down" + + // Possible values for RuleType. + RuleTypeRecording RuleType = "recording" + RuleTypeAlerting RuleType = "alerting" + + // Possible values for RuleHealth. + RuleHealthGood = "ok" + RuleHealthUnknown = "unknown" + RuleHealthBad = "err" ) // Error is an error returned by the API. @@ -111,6 +136,8 @@ type API interface { // Snapshot creates a snapshot of all current data into snapshots/- // under the TSDB's data directory and returns the directory as response. Snapshot(ctx context.Context, skipHead bool) (SnapshotResult, error) + // Rules returns a list of alerting and recording rules that are currently loaded. + Rules(ctx context.Context) (RulesResult, error) // Targets returns an overview of the current state of the Prometheus target discovery. Targets(ctx context.Context) (TargetsResult, error) } @@ -139,6 +166,63 @@ type SnapshotResult struct { Name string `json:"name"` } +// RulesResult contains the result from querying the rules endpoint. +type RulesResult struct { + Groups []RuleGroup `json:"groups"` +} + +// RuleGroup models a rule group that contains a set of recording and alerting rules. +type RuleGroup struct { + Name string `json:"name"` + File string `json:"file"` + Interval float64 `json:"interval"` + Rules Rules `json:"rules"` +} + +// Recording and alerting rules are stored in the same slice to preserve the order +// that rules are returned in by the API. +// +// Rule types can be determined using a type switch: +// switch v := rule.(type) { +// case RecordingRule: +// fmt.Print("got a recording rule") +// case AlertingRule: +// fmt.Print("got a alerting rule") +// default: +// fmt.Printf("unknown rule type %s", v) +// } +type Rules []interface{} + +// AlertingRule models a alerting rule. +type AlertingRule struct { + Name string `json:"name"` + Query string `json:"query"` + Duration float64 `json:"duration"` + Labels model.LabelSet `json:"labels"` + Annotations model.LabelSet `json:"annotations"` + Alerts []*Alert `json:"alerts"` + Health RuleHealth `json:"health"` + LastError string `json:"lastError,omitempty"` +} + +// RecordingRule models a recording rule. +type RecordingRule struct { + Name string `json:"name"` + Query string `json:"query"` + Labels model.LabelSet `json:"labels,omitempty"` + Health RuleHealth `json:"health"` + LastError string `json:"lastError,omitempty"` +} + +// Alert models an active alert. +type Alert struct { + ActiveAt time.Time `json:"activeAt"` + Annotations model.LabelSet + Labels model.LabelSet + State AlertState + Value float64 +} + // TargetsResult contains the result from querying the targets endpoint. type TargetsResult struct { Active []ActiveTarget `json:"activeTargets"` @@ -169,6 +253,111 @@ type queryResult struct { v model.Value } +func (rg *RuleGroup) UnmarshalJSON(b []byte) error { + v := struct { + Name string `json:"name"` + File string `json:"file"` + Interval float64 `json:"interval"` + Rules []json.RawMessage `json:"rules"` + }{} + + if err := json.Unmarshal(b, &v); err != nil { + return err + } + + rg.Name = v.Name + rg.File = v.File + rg.Interval = v.Interval + + for _, rule := range v.Rules { + alertingRule := AlertingRule{} + if err := json.Unmarshal(rule, &alertingRule); err == nil { + rg.Rules = append(rg.Rules, alertingRule) + continue + } + recordingRule := RecordingRule{} + if err := json.Unmarshal(rule, &recordingRule); err == nil { + rg.Rules = append(rg.Rules, recordingRule) + continue + } + return errors.New("failed to decode JSON into an alerting or recording rule") + } + + return nil +} + +func (r *AlertingRule) UnmarshalJSON(b []byte) error { + v := struct { + Type string `json:"type"` + }{} + if err := json.Unmarshal(b, &v); err != nil { + return err + } + if v.Type == "" { + return errors.New("type field not present in rule") + } + if v.Type != string(RuleTypeAlerting) { + return fmt.Errorf("expected rule of type %s but got %s", string(RuleTypeAlerting), v.Type) + } + + rule := struct { + Name string `json:"name"` + Query string `json:"query"` + Duration float64 `json:"duration"` + Labels model.LabelSet `json:"labels"` + Annotations model.LabelSet `json:"annotations"` + Alerts []*Alert `json:"alerts"` + Health RuleHealth `json:"health"` + LastError string `json:"lastError,omitempty"` + }{} + if err := json.Unmarshal(b, &rule); err != nil { + return err + } + r.Health = rule.Health + r.Annotations = rule.Annotations + r.Name = rule.Name + r.Query = rule.Query + r.Alerts = rule.Alerts + r.Duration = rule.Duration + r.Labels = rule.Labels + r.LastError = rule.LastError + + return nil +} + +func (r *RecordingRule) UnmarshalJSON(b []byte) error { + v := struct { + Type string `json:"type"` + }{} + if err := json.Unmarshal(b, &v); err != nil { + return err + } + if v.Type == "" { + return errors.New("type field not present in rule") + } + if v.Type != string(RuleTypeRecording) { + return fmt.Errorf("expected rule of type %s but got %s", string(RuleTypeRecording), v.Type) + } + + rule := struct { + Name string `json:"name"` + Query string `json:"query"` + Labels model.LabelSet `json:"labels,omitempty"` + Health RuleHealth `json:"health"` + LastError string `json:"lastError,omitempty"` + }{} + if err := json.Unmarshal(b, &rule); err != nil { + return err + } + r.Health = rule.Health + r.Labels = rule.Labels + r.Name = rule.Name + r.LastError = rule.LastError + r.Query = rule.Query + + return nil +} + func (qr *queryResult) UnmarshalJSON(b []byte) error { v := struct { Type model.ValueType `json:"resultType"` @@ -427,6 +616,24 @@ func (h *httpAPI) Snapshot(ctx context.Context, skipHead bool) (SnapshotResult, return res, err } +func (h *httpAPI) Rules(ctx context.Context) (RulesResult, error) { + u := h.client.URL(epRules, nil) + + req, err := http.NewRequest(http.MethodGet, u.String(), nil) + if err != nil { + return RulesResult{}, err + } + + _, body, err := h.client.Do(ctx, req) + if err != nil { + return RulesResult{}, err + } + + var res RulesResult + err = json.Unmarshal(body, &res) + return res, err +} + func (h *httpAPI) Targets(ctx context.Context) (TargetsResult, error) { u := h.client.URL(epTargets, nil) diff --git a/api/prometheus/v1/api_test.go b/api/prometheus/v1/api_test.go index 8492a5c..c95ab3e 100644 --- a/api/prometheus/v1/api_test.go +++ b/api/prometheus/v1/api_test.go @@ -158,6 +158,12 @@ func TestAPIs(t *testing.T) { } } + doRules := func() func() (interface{}, error) { + return func() (interface{}, error) { + return promAPI.Rules(context.Background()) + } + } + doTargets := func() func() (interface{}, error) { return func() (interface{}, error) { return promAPI.Targets(context.Background()) @@ -460,6 +466,108 @@ func TestAPIs(t *testing.T) { err: fmt.Errorf("some error"), }, + { + do: doRules(), + reqMethod: "GET", + reqPath: "/api/v1/rules", + inRes: map[string]interface{}{ + "groups": []map[string]interface{}{ + { + "file": "/rules.yaml", + "interval": 60, + "name": "example", + "rules": []map[string]interface{}{ + { + "alerts": []map[string]interface{}{ + { + "activeAt": testTime.UTC().Format(time.RFC3339Nano), + "annotations": map[string]interface{}{ + "summary": "High request latency", + }, + "labels": map[string]interface{}{ + "alertname": "HighRequestLatency", + "severity": "page", + }, + "state": "firing", + "value": 1, + }, + }, + "annotations": map[string]interface{}{ + "summary": "High request latency", + }, + "duration": 600, + "health": "ok", + "labels": map[string]interface{}{ + "severity": "page", + }, + "name": "HighRequestLatency", + "query": "job:request_latency_seconds:mean5m{job=\"myjob\"} > 0.5", + "type": "alerting", + }, + { + "health": "ok", + "name": "job:http_inprogress_requests:sum", + "query": "sum(http_inprogress_requests) by (job)", + "type": "recording", + }, + }, + }, + }, + }, + res: RulesResult{ + Groups: []RuleGroup{ + { + Name: "example", + File: "/rules.yaml", + Interval: 60, + Rules: []interface{}{ + AlertingRule{ + Alerts: []*Alert{ + { + ActiveAt: testTime.UTC(), + Annotations: model.LabelSet{ + "summary": "High request latency", + }, + Labels: model.LabelSet{ + "alertname": "HighRequestLatency", + "severity": "page", + }, + State: AlertStateFiring, + Value: 1, + }, + }, + Annotations: model.LabelSet{ + "summary": "High request latency", + }, + Labels: model.LabelSet{ + "severity": "page", + }, + Duration: 600, + Health: RuleHealthGood, + Name: "HighRequestLatency", + Query: "job:request_latency_seconds:mean5m{job=\"myjob\"} > 0.5", + LastError: "", + }, + RecordingRule{ + Health: RuleHealthGood, + Name: "job:http_inprogress_requests:sum", + Query: "sum(http_inprogress_requests) by (job)", + LastError: "", + }, + }, + }, + }, + }, + }, + + { + do: doRules(), + reqMethod: "GET", + reqPath: "/api/v1/rules", + inErr: fmt.Errorf("some error"), + err: fmt.Errorf("some error"), + }, + { do: doTargets(), reqMethod: "GET", From da7d5640a22352fbb75c57fe8e18f3ab17f06a2b Mon Sep 17 00:00:00 2001 From: Matt Layher Date: Wed, 16 Jan 2019 17:02:02 -0500 Subject: [PATCH 30/73] prometheus: reword comment to avoid cursing Signed-off-by: Matt Layher --- prometheus/summary.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prometheus/summary.go b/prometheus/summary.go index 577fd49..e4c8714 100644 --- a/prometheus/summary.go +++ b/prometheus/summary.go @@ -153,7 +153,7 @@ type SummaryOpts struct { BufCap uint32 } -// Great fuck-up with the sliding-window decay algorithm... The Merge method of +// Problem with the sliding-window decay algorithm... The Merge method of // perk/quantile is actually not working as advertised - and it might be // unfixable, as the underlying algorithm is apparently not capable of merging // summaries in the first place. To avoid using Merge, we are currently adding From d5f63107bfcac90d692baa94bd0064132e1f1d0d Mon Sep 17 00:00:00 2001 From: Oleksandr Kushchenko Date: Wed, 23 Jan 2019 15:39:45 +0000 Subject: [PATCH 31/73] Use a map to describe discovered labels, as they are not validated by the server (#529) Signed-off-by: Oleksandr Kushchenko --- api/prometheus/v1/api.go | 14 +++++++------- api/prometheus/v1/api_test.go | 4 ++-- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/api/prometheus/v1/api.go b/api/prometheus/v1/api.go index 345c0de..5078df1 100644 --- a/api/prometheus/v1/api.go +++ b/api/prometheus/v1/api.go @@ -231,17 +231,17 @@ type TargetsResult struct { // ActiveTarget models an active Prometheus scrape target. type ActiveTarget struct { - DiscoveredLabels model.LabelSet `json:"discoveredLabels"` - Labels model.LabelSet `json:"labels"` - ScrapeURL string `json:"scrapeUrl"` - LastError string `json:"lastError"` - LastScrape time.Time `json:"lastScrape"` - Health HealthStatus `json:"health"` + DiscoveredLabels map[string]string `json:"discoveredLabels"` + Labels model.LabelSet `json:"labels"` + ScrapeURL string `json:"scrapeUrl"` + LastError string `json:"lastError"` + LastScrape time.Time `json:"lastScrape"` + Health HealthStatus `json:"health"` } // DroppedTarget models a dropped Prometheus scrape target. type DroppedTarget struct { - DiscoveredLabels model.LabelSet `json:"discoveredLabels"` + DiscoveredLabels map[string]string `json:"discoveredLabels"` } // queryResult contains result data for a query. diff --git a/api/prometheus/v1/api_test.go b/api/prometheus/v1/api_test.go index c95ab3e..fec170e 100644 --- a/api/prometheus/v1/api_test.go +++ b/api/prometheus/v1/api_test.go @@ -605,7 +605,7 @@ func TestAPIs(t *testing.T) { res: TargetsResult{ Active: []ActiveTarget{ { - DiscoveredLabels: model.LabelSet{ + DiscoveredLabels: map[string]string{ "__address__": "127.0.0.1:9090", "__metrics_path__": "/metrics", "__scheme__": "http", @@ -623,7 +623,7 @@ func TestAPIs(t *testing.T) { }, Dropped: []DroppedTarget{ { - DiscoveredLabels: model.LabelSet{ + DiscoveredLabels: map[string]string{ "__address__": "127.0.0.1:9100", "__metrics_path__": "/metrics", "__scheme__": "http", From 3c4408c8b8293c18b75e67c53b4d21131c169db9 Mon Sep 17 00:00:00 2001 From: beorn7 Date: Sun, 27 Jan 2019 23:13:11 +0100 Subject: [PATCH 32/73] Update dependencies This includes the infamous rollback of prometheus/common Signed-off-by: beorn7 --- go.mod | 11 +++++------ go.sum | 41 +++++++++++++++++++++++++++++++++++------ 2 files changed, 40 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 12321ef..0c16bac 100644 --- a/go.mod +++ b/go.mod @@ -3,10 +3,9 @@ module github.com/prometheus/client_golang require ( github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 github.com/golang/protobuf v1.2.0 - github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect - github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 - github.com/prometheus/common v0.0.0-20181218105931-67670fe90761 - github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a - golang.org/x/net v0.0.0-20181217023233-e147a9138326 - golang.org/x/sync v0.0.0-20181108010431-42b317875d0f // indirect + github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f + github.com/prometheus/common v0.2.0 + github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1 + golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3 + golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 // indirect ) diff --git a/go.sum b/go.sum index 223eaee..3ee56ae 100644 --- a/go.sum +++ b/go.sum @@ -1,16 +1,45 @@ +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 h1:idejC8f05m9MGOsuEi1ATq9shN03HrxNkD/luQvxCv8= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/common v0.0.0-20181218105931-67670fe90761 h1:z6tvbDJ5OLJ48FFmnksv04a78maSTRBUIhkdHYV5Y98= -github.com/prometheus/common v0.0.0-20181218105931-67670fe90761/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a h1:9a8MnZMP0X2nLJdBg+pBmGgkJlSaKC2KaQmTCk1XDtE= -github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -golang.org/x/net v0.0.0-20181217023233-e147a9138326 h1:iCzOf0xz39Tstp+Tu/WwyGjUXCk34QhQORRxBeXXTA4= -golang.org/x/net v0.0.0-20181217023233-e147a9138326/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f h1:BVwpUVJDADN2ufcGik7W992pyps0wZ888b/y9GXcLTU= +github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/common v0.2.0 h1:kUZDBDTdBVBYBj5Tmh2NZLlF60mfjA27rM34b+cVwNU= +github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1 h1:/K3IL0Z1quvmJ7X0A1AwNEK7CRkVK3YwfOU/QAL4WGg= +github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3 h1:ulvT7fqt0yHWzpJwI57MezWnYDVpCAYBVuYst/L+fAY= +golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f h1:Bl/8QSvNqXvPGPGXa2z5xUTmV7VDcZyvRZ+QQXkXTZQ= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= From 8785922956813fc994bbaa1d149b972ff23ba16d Mon Sep 17 00:00:00 2001 From: beorn7 Date: Sun, 27 Jan 2019 23:18:44 +0100 Subject: [PATCH 33/73] Revert "Fix tests to adhere to the recent change in prometheus/common" This reverts commit 2c9811f88eb56c7eeb1e2262b10a17ef727cfc91. This is necessary because the changes in prometheus/common got reverted. Signed-off-by: beorn7 --- prometheus/promhttp/http_test.go | 14 ++++---- prometheus/registry_test.go | 52 ++++++++++++++-------------- prometheus/testutil/testutil_test.go | 30 ++++++++-------- 3 files changed, 48 insertions(+), 48 deletions(-) diff --git a/prometheus/promhttp/http_test.go b/prometheus/promhttp/http_test.go index 0738276..6e23e6c 100644 --- a/prometheus/promhttp/http_test.go +++ b/prometheus/promhttp/http_test.go @@ -109,11 +109,11 @@ error collecting metric Desc{fqName: "invalid_metric", help: "not helpful", cons ` wantOKBody := `# HELP name docstring # TYPE name counter -name{constname="constvalue",labelname="val1"} 1.0 -name{constname="constvalue",labelname="val2"} 1.0 +name{constname="constvalue",labelname="val1"} 1 +name{constname="constvalue",labelname="val2"} 1 # HELP the_count Ah-ah-ah! Thunder and lightning! # TYPE the_count counter -the_count 0.0 +the_count 0 ` errorHandler.ServeHTTP(writer, request) @@ -163,11 +163,11 @@ func TestInstrumentMetricHandler(t *testing.T) { t.Errorf("got HTTP status code %d, want %d", got, want) } - want := "promhttp_metric_handler_requests_in_flight 1.0\n" + want := "promhttp_metric_handler_requests_in_flight 1\n" if got := writer.Body.String(); !strings.Contains(got, want) { t.Errorf("got body %q, does not contain %q", got, want) } - want = "promhttp_metric_handler_requests_total{code=\"200\"} 0.0\n" + want = "promhttp_metric_handler_requests_total{code=\"200\"} 0\n" if got := writer.Body.String(); !strings.Contains(got, want) { t.Errorf("got body %q, does not contain %q", got, want) } @@ -178,11 +178,11 @@ func TestInstrumentMetricHandler(t *testing.T) { t.Errorf("got HTTP status code %d, want %d", got, want) } - want = "promhttp_metric_handler_requests_in_flight 1.0\n" + want = "promhttp_metric_handler_requests_in_flight 1\n" if got := writer.Body.String(); !strings.Contains(got, want) { t.Errorf("got body %q, does not contain %q", got, want) } - want = "promhttp_metric_handler_requests_total{code=\"200\"} 1.0\n" + want = "promhttp_metric_handler_requests_total{code=\"200\"} 1\n" if got := writer.Body.String(); !strings.Contains(got, want) { t.Errorf("got body %q, does not contain %q", got, want) } diff --git a/prometheus/registry_test.go b/prometheus/registry_test.go index 6b6aaf3..b9d9f13 100644 --- a/prometheus/registry_test.go +++ b/prometheus/registry_test.go @@ -99,7 +99,7 @@ func testHandler(t testing.TB) { externalMetricFamilyAsBytes := externalBuf.Bytes() externalMetricFamilyAsText := []byte(`# HELP externalname externaldocstring # TYPE externalname counter -externalname{externalconstname="externalconstvalue",externallabelname="externalval1"} 1.0 +externalname{externalconstname="externalconstvalue",externallabelname="externalval1"} 1 `) externalMetricFamilyAsProtoText := []byte(`name: "externalname" help: "externaldocstring" @@ -167,8 +167,8 @@ metric: < expectedMetricFamilyAsBytes := buf.Bytes() expectedMetricFamilyAsText := []byte(`# HELP name docstring # TYPE name counter -name{constname="constvalue",labelname="val1"} 1.0 -name{constname="constvalue",labelname="val2"} 1.0 +name{constname="constvalue",labelname="val1"} 1 +name{constname="constvalue",labelname="val2"} 1 `) expectedMetricFamilyAsProtoText := []byte(`name: "name" help: "docstring" @@ -267,8 +267,8 @@ collected metric "name" { label: label: Date: Mon, 28 Jan 2019 03:53:16 -0500 Subject: [PATCH 34/73] wait for done before writing to shared resp body (#532) Signed-off-by: Louis Delossantos --- api/client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/client.go b/api/client.go index 09af749..9d5b187 100644 --- a/api/client.go +++ b/api/client.go @@ -119,8 +119,8 @@ func (c *httpClient) Do(ctx context.Context, req *http.Request) (*http.Response, select { case <-ctx.Done(): - err = resp.Body.Close() <-done + err = resp.Body.Close() if err == nil { err = ctx.Err() } From a11607d7ae9dc5bc4d100a3c029e4b6a950608ea Mon Sep 17 00:00:00 2001 From: kklipsch Date: Mon, 4 Feb 2019 20:30:19 +0000 Subject: [PATCH 35/73] fix client trace examples to actually capture metrics Signed-off-by: kklipsch --- prometheus/promhttp/instrument_client_1_8_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/prometheus/promhttp/instrument_client_1_8_test.go b/prometheus/promhttp/instrument_client_1_8_test.go index 7e3f522..fad9ecb 100644 --- a/prometheus/promhttp/instrument_client_1_8_test.go +++ b/prometheus/promhttp/instrument_client_1_8_test.go @@ -74,16 +74,16 @@ func TestClientMiddlewareAPI(t *testing.T) { trace := &InstrumentTrace{ DNSStart: func(t float64) { - dnsLatencyVec.WithLabelValues("dns_start") + dnsLatencyVec.WithLabelValues("dns_start").Observe(t) }, DNSDone: func(t float64) { - dnsLatencyVec.WithLabelValues("dns_done") + dnsLatencyVec.WithLabelValues("dns_done").Observe(t) }, TLSHandshakeStart: func(t float64) { - tlsLatencyVec.WithLabelValues("tls_handshake_start") + tlsLatencyVec.WithLabelValues("tls_handshake_start").Observe(t) }, TLSHandshakeDone: func(t float64) { - tlsLatencyVec.WithLabelValues("tls_handshake_done") + tlsLatencyVec.WithLabelValues("tls_handshake_done").Observe(t) }, } From 069235ab8ceb55363034de865b6eb645fc7d57d1 Mon Sep 17 00:00:00 2001 From: beorn7 Date: Mon, 11 Feb 2019 16:24:22 +0100 Subject: [PATCH 36/73] Pull in new Makefile.common from prometheus/prometheus This should fix the currently broken Travis builds for Go versions before 1.11. Signed-off-by: beorn7 --- Makefile.common | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Makefile.common b/Makefile.common index fff85f9..a422e1b 100644 --- a/Makefile.common +++ b/Makefile.common @@ -36,7 +36,8 @@ GO_VERSION ?= $(shell $(GO) version) GO_VERSION_NUMBER ?= $(word 3, $(GO_VERSION)) PRE_GO_111 ?= $(shell echo $(GO_VERSION_NUMBER) | grep -E 'go1\.(10|[0-9])\.') -unexport GOVENDOR +GOVENDOR := +GO111MODULE := ifeq (, $(PRE_GO_111)) ifneq (,$(wildcard go.mod)) # Enforce Go modules support just in case the directory is inside GOPATH (and for Travis CI). @@ -57,8 +58,6 @@ $(warning Some recipes may not work as expected as the current Go runtime is '$( # This repository isn't using Go modules (yet). GOVENDOR := $(FIRST_GOPATH)/bin/govendor endif - - unexport GO111MODULE endif PROMU := $(FIRST_GOPATH)/bin/promu STATICCHECK := $(FIRST_GOPATH)/bin/staticcheck From 19ff2774833b179baa76643d62398dc4a3cb9cf8 Mon Sep 17 00:00:00 2001 From: "Pascal S. de Kloe" Date: Mon, 11 Feb 2019 18:29:02 +0100 Subject: [PATCH 37/73] Save memory on histograms: slightly faster with less code (#536) Use the highest bit for buffer switch tracking in histograms Signed-off-by: Pascal S. de Kloe --- prometheus/histogram.go | 118 +++++++++++++++------------------------- 1 file changed, 43 insertions(+), 75 deletions(-) diff --git a/prometheus/histogram.go b/prometheus/histogram.go index 20ee1af..98e06bf 100644 --- a/prometheus/histogram.go +++ b/prometheus/histogram.go @@ -224,33 +224,34 @@ type histogramCounts struct { } type histogram struct { - // countAndHotIdx is a complicated one. For lock-free yet atomic - // observations, we need to save the total count of observations again, - // combined with the index of the currently-hot counts struct, so that - // we can perform the operation on both values atomically. The least - // significant bit defines the hot counts struct. The remaining 63 bits - // represent the total count of observations. This happens under the - // assumption that the 63bit count will never overflow. Rationale: An - // observations takes about 30ns. Let's assume it could happen in - // 10ns. Overflowing the counter will then take at least (2^63)*10ns, - // which is about 3000 years. + // CountAndHotIdx enables lock-free writes with use of atomic updates. + // The most significant bit is the hot index [0 or 1] of the count field + // below. Writes update the hot one. All remaining bits count the number + // of writes initiated. Write transactions start by incrementing this + // counter, and finish by incrementing the count field in the respective + // histogramCounts, as a marker for completion. // - // This has to be first in the struct for 64bit alignment. See + // Reads swap the hot–cold in a switchMutex lock. A cooldown is awaited + // (in such lock) by comparing the number of writes with the initiation + // count. Once they match, then the last write transaction on the now + // cool one completed. All cool fields must be merged into the new hot + // before the unlock of switchMutex. + // + // Fields with atomic access first! See alignment constraint: // http://golang.org/pkg/sync/atomic/#pkg-note-BUG countAndHotIdx uint64 - selfCollector - desc *Desc - writeMtx sync.Mutex // Only used in the Write method. - - upperBounds []float64 - - // Two counts, one is "hot" for lock-free observations, the other is - // "cold" for writing out a dto.Metric. It has to be an array of - // pointers to guarantee 64bit alignment of the histogramCounts, see + // Counts has to be an array of pointers to guarantee 64bit alignment of + // the histogramCounts, see // http://golang.org/pkg/sync/atomic/#pkg-note-BUG. counts [2]*histogramCounts - hotIdx int // Index of currently-hot counts. Only used within Write. + + switchMtx sync.Mutex + + selfCollector + desc *Desc + + upperBounds []float64 labelPairs []*dto.LabelPair } @@ -271,11 +272,11 @@ func (h *histogram) Observe(v float64) { // 300 buckets: 154 ns/op linear - binary 61.6 ns/op i := sort.SearchFloat64s(h.upperBounds, v) - // We increment h.countAndHotIdx by 2 so that the counter in the upper - // 63 bits gets incremented by 1. At the same time, we get the new value + // We increment h.countAndHotIdx so that the counter in the lower + // 63 bits gets incremented. At the same time, we get the new value // back, which we can use to find the currently-hot counts. - n := atomic.AddUint64(&h.countAndHotIdx, 2) - hotCounts := h.counts[n%2] + n := atomic.AddUint64(&h.countAndHotIdx, 1) + hotCounts := h.counts[n>>63] if i < len(h.upperBounds) { atomic.AddUint64(&hotCounts.buckets[i], 1) @@ -293,72 +294,39 @@ func (h *histogram) Observe(v float64) { } func (h *histogram) Write(out *dto.Metric) error { - var ( - his = &dto.Histogram{} - buckets = make([]*dto.Bucket, len(h.upperBounds)) - hotCounts, coldCounts *histogramCounts - count uint64 - ) - // For simplicity, we mutex the rest of this method. It is not in the // hot path, i.e. Observe is called much more often than Write. The // complication of making Write lock-free isn't worth it. - h.writeMtx.Lock() - defer h.writeMtx.Unlock() + h.switchMtx.Lock() + defer h.switchMtx.Unlock() - // This is a bit arcane, which is why the following spells out this if - // clause in English: - // - // If the currently-hot counts struct is #0, we atomically increment - // h.countAndHotIdx by 1 so that from now on Observe will use the counts - // struct #1. Furthermore, the atomic increment gives us the new value, - // which, in its most significant 63 bits, tells us the count of - // observations done so far up to and including currently ongoing - // observations still using the counts struct just changed from hot to - // cold. To have a normal uint64 for the count, we bitshift by 1 and - // save the result in count. We also set h.hotIdx to 1 for the next - // Write call, and we will refer to counts #1 as hotCounts and to counts - // #0 as coldCounts. - // - // If the currently-hot counts struct is #1, we do the corresponding - // things the other way round. We have to _decrement_ h.countAndHotIdx - // (which is a bit arcane in itself, as we have to express -1 with an - // unsigned int...). - if h.hotIdx == 0 { - count = atomic.AddUint64(&h.countAndHotIdx, 1) >> 1 - h.hotIdx = 1 - hotCounts = h.counts[1] - coldCounts = h.counts[0] - } else { - count = atomic.AddUint64(&h.countAndHotIdx, ^uint64(0)) >> 1 // Decrement. - h.hotIdx = 0 - hotCounts = h.counts[0] - coldCounts = h.counts[1] - } + // Adding 1<<63 switches the hot index (from 0 to 1 or from 1 to 0) + // without touching the count bits. See the struct comments for a full + // description of the algorithm. + n := atomic.AddUint64(&h.countAndHotIdx, 1<<63) + count := n & ((1 << 63) - 1) + hotCounts := h.counts[n>>63] + coldCounts := h.counts[(^n)>>63] - // Now we have to wait for the now-declared-cold counts to actually cool - // down, i.e. wait for all observations still using it to finish. That's - // the case once the count in the cold counts struct is the same as the - // one atomically retrieved from the upper 63bits of h.countAndHotIdx. - for { - if count == atomic.LoadUint64(&coldCounts.count) { - break - } + // await cooldown + for count != atomic.LoadUint64(&coldCounts.count) { runtime.Gosched() // Let observations get work done. } - his.SampleCount = proto.Uint64(count) - his.SampleSum = proto.Float64(math.Float64frombits(atomic.LoadUint64(&coldCounts.sumBits))) + his := &dto.Histogram{ + Bucket: make([]*dto.Bucket, len(h.upperBounds)), + SampleCount: proto.Uint64(count), + SampleSum: proto.Float64(math.Float64frombits(atomic.LoadUint64(&coldCounts.sumBits))), + } var cumCount uint64 for i, upperBound := range h.upperBounds { cumCount += atomic.LoadUint64(&coldCounts.buckets[i]) - buckets[i] = &dto.Bucket{ + his.Bucket[i] = &dto.Bucket{ CumulativeCount: proto.Uint64(cumCount), UpperBound: proto.Float64(upperBound), } } - his.Bucket = buckets out.Histogram = his out.Label = h.labelPairs From 295f7e4861b2dfe94f0c964902c39567dce69241 Mon Sep 17 00:00:00 2001 From: beorn7 Date: Mon, 11 Feb 2019 18:48:35 +0100 Subject: [PATCH 38/73] Fix some comment and naming nits as leftover from #536 Signed-off-by: beorn7 --- prometheus/histogram.go | 52 ++++++++++++++++++++++------------------- 1 file changed, 28 insertions(+), 24 deletions(-) diff --git a/prometheus/histogram.go b/prometheus/histogram.go index 98e06bf..d7ea67b 100644 --- a/prometheus/histogram.go +++ b/prometheus/histogram.go @@ -224,36 +224,36 @@ type histogramCounts struct { } type histogram struct { - // CountAndHotIdx enables lock-free writes with use of atomic updates. + // countAndHotIdx enables lock-free writes with use of atomic updates. // The most significant bit is the hot index [0 or 1] of the count field - // below. Writes update the hot one. All remaining bits count the number - // of writes initiated. Write transactions start by incrementing this - // counter, and finish by incrementing the count field in the respective + // below. Observe calls update the hot one. All remaining bits count the + // number of Observe calls. Observe starts by incrementing this counter, + // and finish by incrementing the count field in the respective // histogramCounts, as a marker for completion. // - // Reads swap the hot–cold in a switchMutex lock. A cooldown is awaited - // (in such lock) by comparing the number of writes with the initiation - // count. Once they match, then the last write transaction on the now - // cool one completed. All cool fields must be merged into the new hot - // before the unlock of switchMutex. + // Calls of the Write method (which are non-mutating reads from the + // perspective of the histogram) swap the hot–cold under the writeMtx + // lock. A cooldown is awaited (while locked) by comparing the number of + // observations with the initiation count. Once they match, then the + // last observation on the now cool one has completed. All cool fields must + // be merged into the new hot before releasing writeMtx. // // Fields with atomic access first! See alignment constraint: // http://golang.org/pkg/sync/atomic/#pkg-note-BUG countAndHotIdx uint64 - // Counts has to be an array of pointers to guarantee 64bit alignment of - // the histogramCounts, see + selfCollector + desc *Desc + writeMtx sync.Mutex // Only used in the Write method. + + // Two counts, one is "hot" for lock-free observations, the other is + // "cold" for writing out a dto.Metric. It has to be an array of + // pointers to guarantee 64bit alignment of the histogramCounts, see // http://golang.org/pkg/sync/atomic/#pkg-note-BUG. counts [2]*histogramCounts - switchMtx sync.Mutex - - selfCollector - desc *Desc - upperBounds []float64 - - labelPairs []*dto.LabelPair + labelPairs []*dto.LabelPair } func (h *histogram) Desc() *Desc { @@ -294,21 +294,25 @@ func (h *histogram) Observe(v float64) { } func (h *histogram) Write(out *dto.Metric) error { - // For simplicity, we mutex the rest of this method. It is not in the - // hot path, i.e. Observe is called much more often than Write. The - // complication of making Write lock-free isn't worth it. - h.switchMtx.Lock() - defer h.switchMtx.Unlock() + // For simplicity, we protect this whole method by a mutex. It is not in + // the hot path, i.e. Observe is called much more often than Write. The + // complication of making Write lock-free isn't worth it, if possible at + // all. + h.writeMtx.Lock() + defer h.writeMtx.Unlock() // Adding 1<<63 switches the hot index (from 0 to 1 or from 1 to 0) // without touching the count bits. See the struct comments for a full // description of the algorithm. n := atomic.AddUint64(&h.countAndHotIdx, 1<<63) + // count is contained unchanged in the lower 63 bits. count := n & ((1 << 63) - 1) + // The most significant bit tells us which counts is hot. The complement + // is thus the cold one. hotCounts := h.counts[n>>63] coldCounts := h.counts[(^n)>>63] - // await cooldown + // Await cooldown. for count != atomic.LoadUint64(&coldCounts.count) { runtime.Gosched() // Let observations get work done. } From 4c99dd66303a54cbf8559dd6110d5c30b1819e4c Mon Sep 17 00:00:00 2001 From: beorn7 Date: Mon, 11 Feb 2019 19:10:17 +0100 Subject: [PATCH 39/73] Port histogram improvements into noObjectivesSummary Signed-off-by: beorn7 --- prometheus/summary.go | 102 ++++++++++++++++-------------------------- 1 file changed, 38 insertions(+), 64 deletions(-) diff --git a/prometheus/summary.go b/prometheus/summary.go index e4c8714..1bf3e19 100644 --- a/prometheus/summary.go +++ b/prometheus/summary.go @@ -405,18 +405,21 @@ type summaryCounts struct { } type noObjectivesSummary struct { - // countAndHotIdx is a complicated one. For lock-free yet atomic - // observations, we need to save the total count of observations again, - // combined with the index of the currently-hot counts struct, so that - // we can perform the operation on both values atomically. The least - // significant bit defines the hot counts struct. The remaining 63 bits - // represent the total count of observations. This happens under the - // assumption that the 63bit count will never overflow. Rationale: An - // observations takes about 30ns. Let's assume it could happen in - // 10ns. Overflowing the counter will then take at least (2^63)*10ns, - // which is about 3000 years. + // countAndHotIdx enables lock-free writes with use of atomic updates. + // The most significant bit is the hot index [0 or 1] of the count field + // below. Observe calls update the hot one. All remaining bits count the + // number of Observe calls. Observe starts by incrementing this counter, + // and finish by incrementing the count field in the respective + // summaryCounts, as a marker for completion. // - // This has to be first in the struct for 64bit alignment. See + // Calls of the Write method (which are non-mutating reads from the + // perspective of the summary) swap the hot–cold under the writeMtx + // lock. A cooldown is awaited (while locked) by comparing the number of + // observations with the initiation count. Once they match, then the + // last observation on the now cool one has completed. All cool fields must + // be merged into the new hot before releasing writeMtx. + + // Fields with atomic access first! See alignment constraint: // http://golang.org/pkg/sync/atomic/#pkg-note-BUG countAndHotIdx uint64 @@ -429,7 +432,6 @@ type noObjectivesSummary struct { // pointers to guarantee 64bit alignment of the histogramCounts, see // http://golang.org/pkg/sync/atomic/#pkg-note-BUG. counts [2]*summaryCounts - hotIdx int // Index of currently-hot counts. Only used within Write. labelPairs []*dto.LabelPair } @@ -439,11 +441,11 @@ func (s *noObjectivesSummary) Desc() *Desc { } func (s *noObjectivesSummary) Observe(v float64) { - // We increment s.countAndHotIdx by 2 so that the counter in the upper - // 63 bits gets incremented by 1. At the same time, we get the new value + // We increment h.countAndHotIdx so that the counter in the lower + // 63 bits gets incremented. At the same time, we get the new value // back, which we can use to find the currently-hot counts. - n := atomic.AddUint64(&s.countAndHotIdx, 2) - hotCounts := s.counts[n%2] + n := atomic.AddUint64(&s.countAndHotIdx, 1) + hotCounts := s.counts[n>>63] for { oldBits := atomic.LoadUint64(&hotCounts.sumBits) @@ -458,61 +460,33 @@ func (s *noObjectivesSummary) Observe(v float64) { } func (s *noObjectivesSummary) Write(out *dto.Metric) error { - var ( - sum = &dto.Summary{} - hotCounts, coldCounts *summaryCounts - count uint64 - ) - - // For simplicity, we mutex the rest of this method. It is not in the - // hot path, i.e. Observe is called much more often than Write. The - // complication of making Write lock-free isn't worth it. + // For simplicity, we protect this whole method by a mutex. It is not in + // the hot path, i.e. Observe is called much more often than Write. The + // complication of making Write lock-free isn't worth it, if possible at + // all. s.writeMtx.Lock() defer s.writeMtx.Unlock() - // This is a bit arcane, which is why the following spells out this if - // clause in English: - // - // If the currently-hot counts struct is #0, we atomically increment - // s.countAndHotIdx by 1 so that from now on Observe will use the counts - // struct #1. Furthermore, the atomic increment gives us the new value, - // which, in its most significant 63 bits, tells us the count of - // observations done so far up to and including currently ongoing - // observations still using the counts struct just changed from hot to - // cold. To have a normal uint64 for the count, we bitshift by 1 and - // save the result in count. We also set s.hotIdx to 1 for the next - // Write call, and we will refer to counts #1 as hotCounts and to counts - // #0 as coldCounts. - // - // If the currently-hot counts struct is #1, we do the corresponding - // things the other way round. We have to _decrement_ s.countAndHotIdx - // (which is a bit arcane in itself, as we have to express -1 with an - // unsigned int...). - if s.hotIdx == 0 { - count = atomic.AddUint64(&s.countAndHotIdx, 1) >> 1 - s.hotIdx = 1 - hotCounts = s.counts[1] - coldCounts = s.counts[0] - } else { - count = atomic.AddUint64(&s.countAndHotIdx, ^uint64(0)) >> 1 // Decrement. - s.hotIdx = 0 - hotCounts = s.counts[0] - coldCounts = s.counts[1] - } + // Adding 1<<63 switches the hot index (from 0 to 1 or from 1 to 0) + // without touching the count bits. See the struct comments for a full + // description of the algorithm. + n := atomic.AddUint64(&s.countAndHotIdx, 1<<63) + // count is contained unchanged in the lower 63 bits. + count := n & ((1 << 63) - 1) + // The most significant bit tells us which counts is hot. The complement + // is thus the cold one. + hotCounts := s.counts[n>>63] + coldCounts := s.counts[(^n)>>63] - // Now we have to wait for the now-declared-cold counts to actually cool - // down, i.e. wait for all observations still using it to finish. That's - // the case once the count in the cold counts struct is the same as the - // one atomically retrieved from the upper 63bits of s.countAndHotIdx. - for { - if count == atomic.LoadUint64(&coldCounts.count) { - break - } + // Await cooldown. + for count != atomic.LoadUint64(&coldCounts.count) { runtime.Gosched() // Let observations get work done. } - sum.SampleCount = proto.Uint64(count) - sum.SampleSum = proto.Float64(math.Float64frombits(atomic.LoadUint64(&coldCounts.sumBits))) + sum := &dto.Summary{ + SampleCount: proto.Uint64(count), + SampleSum: proto.Float64(math.Float64frombits(atomic.LoadUint64(&coldCounts.sumBits))), + } out.Summary = sum out.Label = s.labelPairs From 7b127b3fe5913993c4125afada8a5e0aa203c5b5 Mon Sep 17 00:00:00 2001 From: Frank Spitulski Date: Sun, 24 Feb 2019 12:51:45 -0800 Subject: [PATCH 40/73] feat(push): add format builder option Allow users to specify which format the pusher should push in. Signed-off-by: Frank Spitulski --- prometheus/push/push.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/prometheus/push/push.go b/prometheus/push/push.go index 3721ff1..ccfca3a 100644 --- a/prometheus/push/push.go +++ b/prometheus/push/push.go @@ -64,6 +64,8 @@ type Pusher struct { client *http.Client useBasicAuth bool username, password string + + expfmt expfmt.Format } // New creates a new Pusher to push to the provided URL with the provided job @@ -96,6 +98,7 @@ func New(url, job string) *Pusher { gatherers: prometheus.Gatherers{reg}, registerer: reg, client: &http.Client{}, + expfmt: expfmt.FmtProtoDelim, } } @@ -182,6 +185,14 @@ func (p *Pusher) BasicAuth(username, password string) *Pusher { return p } +// Format configures the Pusher to use an encoding format given by the +// provided expfmt.Format. For convenience, this method returns a +// pointer to the Pusher itself. +func (p *Pusher) Format(format expfmt.Format) *Pusher { + p.expfmt = format + return p +} + func (p *Pusher) push(method string) error { if p.error != nil { return p.error @@ -197,7 +208,7 @@ func (p *Pusher) push(method string) error { return err } buf := &bytes.Buffer{} - enc := expfmt.NewEncoder(buf, expfmt.FmtProtoDelim) + enc := expfmt.NewEncoder(buf, p.expfmt) // Check for pre-existing grouping labels: for _, mf := range mfs { for _, m := range mf.GetMetric() { From 10e34c668faa0b614e94e466777585a24bdf5270 Mon Sep 17 00:00:00 2001 From: Frank Spitulski Date: Sun, 24 Feb 2019 15:15:35 -0800 Subject: [PATCH 41/73] fix(push): incorporate changes from review Signed-off-by: Frank Spitulski --- prometheus/push/push.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/prometheus/push/push.go b/prometheus/push/push.go index ccfca3a..6636ffc 100644 --- a/prometheus/push/push.go +++ b/prometheus/push/push.go @@ -186,8 +186,10 @@ func (p *Pusher) BasicAuth(username, password string) *Pusher { } // Format configures the Pusher to use an encoding format given by the -// provided expfmt.Format. For convenience, this method returns a -// pointer to the Pusher itself. +// provided expfmt.Format. The default format is expfmt.FmtProtoDelim and +// should be used with the standard Prometheus Pushgateway. Custom +// implementations may require different formats. For convenience, this +// method returns a pointer to the Pusher itself. func (p *Pusher) Format(format expfmt.Format) *Pusher { p.expfmt = format return p @@ -233,7 +235,7 @@ func (p *Pusher) push(method string) error { if p.useBasicAuth { req.SetBasicAuth(p.username, p.password) } - req.Header.Set(contentTypeHeader, string(expfmt.FmtProtoDelim)) + req.Header.Set(contentTypeHeader, string(p.expfmt)) resp, err := p.client.Do(req) if err != nil { return err From e490cf5202d2f5123fbe97f601b9445b393e59ba Mon Sep 17 00:00:00 2001 From: Simon Pasquier Date: Mon, 4 Mar 2019 14:17:23 +0100 Subject: [PATCH 42/73] Add Go 1.12 to Travis CI Also update to the latest version of Makefile.common. Signed-off-by: Simon Pasquier --- .travis.yml | 6 ++++-- Makefile.common | 8 ++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index a5cbf75..502935d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,3 @@ -sudo: false language: go go: @@ -7,7 +6,10 @@ go: - 1.9.x - 1.10.x - 1.11.x + - 1.12.x script: - - make check_license style unused test-short + - make check_license unused test-short - if [[ ! $TRAVIS_GO_VERSION =~ ^1\.(7|8|9)\.[x0-9]+$ ]]; then make staticcheck; fi + # style is only checked against the latest supported Go version. + - if [[ $TRAVIS_GO_VERSION =~ ^1\.(12)\. ]]; then make style; fi diff --git a/Makefile.common b/Makefile.common index a422e1b..7105cff 100644 --- a/Makefile.common +++ b/Makefile.common @@ -70,7 +70,7 @@ else GO_BUILD_PLATFORM ?= $(GOHOSTOS)-$(GOHOSTARCH) endif -PROMU_VERSION ?= 0.2.0 +PROMU_VERSION ?= 0.3.0 PROMU_URL := https://github.com/prometheus/promu/releases/download/v$(PROMU_VERSION)/promu-$(PROMU_VERSION).$(GO_BUILD_PLATFORM).tar.gz STATICCHECK_VERSION ?= 2019.1 STATICCHECK_URL := https://github.com/dominikh/go-tools/releases/download/$(STATICCHECK_VERSION)/staticcheck_$(GOHOSTOS)_$(GOHOSTARCH) @@ -87,14 +87,14 @@ ifeq ($(GOHOSTARCH),amd64) endif endif -.PHONY: all -all: precheck style staticcheck unused build test - # This rule is used to forward a target like "build" to "common-build". This # allows a new "build" target to be defined in a Makefile which includes this # one and override "common-build" without override warnings. %: common-% ; +.PHONY: common-all +common-all: precheck style check_license staticcheck unused build test + .PHONY: common-style common-style: @echo ">> checking code style" From 60728f445d290e966e430abeb60b0e846267d677 Mon Sep 17 00:00:00 2001 From: Sergiusz Urbaniak Date: Wed, 6 Mar 2019 13:58:46 +0100 Subject: [PATCH 43/73] prometheus/promhttp: actually observe values in trace in example This is similar to https://github.com/prometheus/client_golang/pull/535 fixing ExampleInstrumentRoundTripperDuration. Signed-off-by: Sergiusz Urbaniak --- prometheus/promhttp/instrument_client_1_8_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/prometheus/promhttp/instrument_client_1_8_test.go b/prometheus/promhttp/instrument_client_1_8_test.go index fad9ecb..2d0ff4f 100644 --- a/prometheus/promhttp/instrument_client_1_8_test.go +++ b/prometheus/promhttp/instrument_client_1_8_test.go @@ -162,16 +162,16 @@ func ExampleInstrumentRoundTripperDuration() { // functions that we want to instrument. trace := &InstrumentTrace{ DNSStart: func(t float64) { - dnsLatencyVec.WithLabelValues("dns_start") + dnsLatencyVec.WithLabelValues("dns_start").Observe(t) }, DNSDone: func(t float64) { - dnsLatencyVec.WithLabelValues("dns_done") + dnsLatencyVec.WithLabelValues("dns_done").Observe(t) }, TLSHandshakeStart: func(t float64) { - tlsLatencyVec.WithLabelValues("tls_handshake_start") + tlsLatencyVec.WithLabelValues("tls_handshake_start").Observe(t) }, TLSHandshakeDone: func(t float64) { - tlsLatencyVec.WithLabelValues("tls_handshake_done") + tlsLatencyVec.WithLabelValues("tls_handshake_done").Observe(t) }, } From a2c4da068f535a2ad81572e98a1f7d3bea07e495 Mon Sep 17 00:00:00 2001 From: Simon Pasquier Date: Wed, 13 Mar 2019 09:14:47 +0100 Subject: [PATCH 44/73] Update Makefile.common Signed-off-by: Simon Pasquier --- Makefile | 9 ++------- Makefile.common | 10 +++++++++- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index 443c360..b25fb83 100644 --- a/Makefile +++ b/Makefile @@ -20,13 +20,8 @@ STATICCHECK_IGNORE = \ github.com/prometheus/client_golang/prometheus/promhttp/instrument_server_test.go:SA1019 \ github.com/prometheus/client_golang/prometheus/http.go:SA1019 -.PHONY: get_dep -get_dep: - @echo ">> getting dependencies" - $(GO) get -t ./... - .PHONY: test -test: get_dep common-test +test: deps common-test .PHONY: test-short -test-short: get_dep common-test-short +test-short: deps common-test-short diff --git a/Makefile.common b/Makefile.common index 7105cff..108e494 100644 --- a/Makefile.common +++ b/Makefile.common @@ -116,6 +116,15 @@ common-check_license: exit 1; \ fi +.PHONY: common-deps +common-deps: + @echo ">> getting dependencies" +ifdef GO111MODULE + GO111MODULE=$(GO111MODULE) $(GO) mod download +else + $(GO) get $(GOOPTS) -t ./... +endif + .PHONY: common-test-short common-test-short: @echo ">> running short tests" @@ -221,7 +230,6 @@ precheck:: define PRECHECK_COMMAND_template = precheck:: $(1)_precheck - PRECHECK_COMMAND_$(1) ?= $(1) $$(strip $$(PRECHECK_OPTIONS_$(1))) .PHONY: $(1)_precheck $(1)_precheck: From 7490f0a74525a1f863eaf81b42f3ead3a1ecc43a Mon Sep 17 00:00:00 2001 From: Roald Nefs Date: Mon, 25 Mar 2019 09:23:28 +0100 Subject: [PATCH 45/73] Add alerts endpoint to API client (#552) Add alerts endpoint to API client. The alerts endpoint returns a list of all active alerts. Signed-off-by: Roald Nefs --- api/prometheus/v1/api.go | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/api/prometheus/v1/api.go b/api/prometheus/v1/api.go index 5078df1..c048179 100644 --- a/api/prometheus/v1/api.go +++ b/api/prometheus/v1/api.go @@ -35,6 +35,7 @@ const ( apiPrefix = "/api/v1" + epAlerts = apiPrefix + "/alerts" epAlertManagers = apiPrefix + "/alertmanagers" epQuery = apiPrefix + "/query" epQueryRange = apiPrefix + "/query_range" @@ -115,6 +116,8 @@ type Range struct { // API provides bindings for Prometheus's v1 API. type API interface { + // Alerts returns a list of all active alerts. + Alerts(ctx context.Context) (AlertsResult, error) // AlertManagers returns an overview of the current state of the Prometheus alert manager discovery. AlertManagers(ctx context.Context) (AlertManagersResult, error) // CleanTombstones removes the deleted data from disk and cleans up the existing tombstones. @@ -142,6 +145,11 @@ type API interface { Targets(ctx context.Context) (TargetsResult, error) } +// AlertsResult contains the result from querying the alerts endpoint. +type AlertsResult struct { + Alerts []Alert `json:"alerts"` +} + // AlertManagersResult contains the result from querying the alertmanagers endpoint. type AlertManagersResult struct { Active []AlertManager `json:"activeAlertManagers"` @@ -402,6 +410,24 @@ type httpAPI struct { client api.Client } +func (h *httpAPI) Alerts(ctx context.Context) (AlertsResult, error) { + u := h.client.URL(epAlerts, nil) + + req, err := http.NewRequest(http.MethodGet, u.String(), nil) + if err != nil { + return AlertsResult{}, err + } + + _, body, err := h.client.Do(ctx, req) + if err != nil { + return AlertsResult{}, err + } + + var res AlertsResult + err = json.Unmarshal(body, &res) + return res, err +} + func (h *httpAPI) AlertManagers(ctx context.Context) (AlertManagersResult, error) { u := h.client.URL(epAlertManagers, nil) From 15da21f1e4581dc21b9c5a795d7c06a565eb2bc5 Mon Sep 17 00:00:00 2001 From: Simon Pasquier Date: Tue, 9 Apr 2019 15:10:51 +0200 Subject: [PATCH 46/73] *: replace golang.org/x/net/context by context The context package is available since Go 1.7 which is the minimal version supported by client_golang. Signed-off-by: Simon Pasquier --- go.mod | 1 - go.sum | 2 -- prometheus/graphite/bridge.go | 2 +- prometheus/graphite/bridge_test.go | 2 +- 4 files changed, 2 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 0c16bac..a8ca63c 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,5 @@ require ( github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f github.com/prometheus/common v0.2.0 github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1 - golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 // indirect ) diff --git a/go.sum b/go.sum index 3ee56ae..003625a 100644 --- a/go.sum +++ b/go.sum @@ -32,8 +32,6 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3 h1:ulvT7fqt0yHWzpJwI57MezWnYDVpCAYBVuYst/L+fAY= -golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f h1:Bl/8QSvNqXvPGPGXa2z5xUTmV7VDcZyvRZ+QQXkXTZQ= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw= diff --git a/prometheus/graphite/bridge.go b/prometheus/graphite/bridge.go index 466e229..43bfd8d 100644 --- a/prometheus/graphite/bridge.go +++ b/prometheus/graphite/bridge.go @@ -17,6 +17,7 @@ package graphite import ( "bufio" + "context" "errors" "fmt" "io" @@ -26,7 +27,6 @@ import ( "github.com/prometheus/common/expfmt" "github.com/prometheus/common/model" - "golang.org/x/net/context" dto "github.com/prometheus/client_model/go" diff --git a/prometheus/graphite/bridge_test.go b/prometheus/graphite/bridge_test.go index 471edfe..7728d05 100644 --- a/prometheus/graphite/bridge_test.go +++ b/prometheus/graphite/bridge_test.go @@ -16,6 +16,7 @@ package graphite import ( "bufio" "bytes" + "context" "fmt" "io" "log" @@ -26,7 +27,6 @@ import ( "time" "github.com/prometheus/common/model" - "golang.org/x/net/context" "github.com/prometheus/client_golang/prometheus" ) From deac34f26f23d0502cb7aff923cc50fa859d25cf Mon Sep 17 00:00:00 2001 From: prombot Date: Fri, 12 Apr 2019 00:00:36 +0000 Subject: [PATCH 47/73] makefile: update Makefile.common with newer version Signed-off-by: prombot --- Makefile.common | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/Makefile.common b/Makefile.common index 108e494..ed29aa8 100644 --- a/Makefile.common +++ b/Makefile.common @@ -60,7 +60,6 @@ $(warning Some recipes may not work as expected as the current Go runtime is '$( endif endif PROMU := $(FIRST_GOPATH)/bin/promu -STATICCHECK := $(FIRST_GOPATH)/bin/staticcheck pkgs = ./... ifeq (arm, $(GOHOSTARCH)) @@ -72,8 +71,17 @@ endif PROMU_VERSION ?= 0.3.0 PROMU_URL := https://github.com/prometheus/promu/releases/download/v$(PROMU_VERSION)/promu-$(PROMU_VERSION).$(GO_BUILD_PLATFORM).tar.gz -STATICCHECK_VERSION ?= 2019.1 -STATICCHECK_URL := https://github.com/dominikh/go-tools/releases/download/$(STATICCHECK_VERSION)/staticcheck_$(GOHOSTOS)_$(GOHOSTARCH) + +STATICCHECK := +# staticcheck only supports linux, freebsd, darwin and windows platforms on i386/amd64 +# windows isn't included here because of the path separator being different. +ifeq ($(GOHOSTOS),$(filter $(GOHOSTOS),linux freebsd darwin)) + ifeq ($(GOHOSTARCH),$(filter $(GOHOSTARCH),amd64 i386)) + STATICCHECK := $(FIRST_GOPATH)/bin/staticcheck + STATICCHECK_VERSION ?= 2019.1 + STATICCHECK_URL := https://github.com/dominikh/go-tools/releases/download/$(STATICCHECK_VERSION)/staticcheck_$(GOHOSTOS)_$(GOHOSTARCH) + endif +endif PREFIX ?= $(shell pwd) BIN_DIR ?= $(shell pwd) @@ -147,6 +155,7 @@ common-vet: .PHONY: common-staticcheck common-staticcheck: $(STATICCHECK) +ifdef STATICCHECK @echo ">> running staticcheck" chmod +x $(STATICCHECK) ifdef GO111MODULE @@ -157,6 +166,7 @@ ifdef GO111MODULE else $(STATICCHECK) -ignore "$(STATICCHECK_IGNORE)" $(pkgs) endif +endif .PHONY: common-unused common-unused: $(GOVENDOR) @@ -214,9 +224,11 @@ proto: @echo ">> generating code from proto files" @./scripts/genproto.sh +ifdef STATICCHECK $(STATICCHECK): mkdir -p $(FIRST_GOPATH)/bin curl -s -L $(STATICCHECK_URL) > $(STATICCHECK) +endif ifdef GOVENDOR .PHONY: $(GOVENDOR) From 2d69188b26487fbd148d5316378e014c3676d730 Mon Sep 17 00:00:00 2001 From: Bjoern Rabenstein Date: Mon, 15 Apr 2019 00:16:44 +0200 Subject: [PATCH 48/73] Update dependencies Mostly to use the new module support for prometheus/common and beorn7/perks. Signed-off-by: Bjoern Rabenstein --- go.mod | 11 +++++------ go.sum | 16 ++++++++++------ 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index a8ca63c..37f7f40 100644 --- a/go.mod +++ b/go.mod @@ -1,10 +1,9 @@ module github.com/prometheus/client_golang require ( - github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 - github.com/golang/protobuf v1.2.0 - github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f - github.com/prometheus/common v0.2.0 - github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1 - golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 // indirect + github.com/beorn7/perks v1.0.0 + github.com/golang/protobuf v1.3.1 + github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 + github.com/prometheus/common v0.3.0 + github.com/prometheus/procfs v0.0.0-20190412120340-e22ddced7142 ) diff --git a/go.sum b/go.sum index 003625a..235ab95 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,8 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= @@ -9,6 +11,8 @@ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/me github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= @@ -20,13 +24,13 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 h1:idejC8f05m9MGOsuEi1ATq9shN03HrxNkD/luQvxCv8= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f h1:BVwpUVJDADN2ufcGik7W992pyps0wZ888b/y9GXcLTU= -github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/common v0.2.0 h1:kUZDBDTdBVBYBj5Tmh2NZLlF60mfjA27rM34b+cVwNU= -github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.3.0 h1:taZ4h8Tkxv2kNyoSctBvfXEHmBmxrwmIidZTIaHons4= +github.com/prometheus/common v0.3.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1 h1:/K3IL0Z1quvmJ7X0A1AwNEK7CRkVK3YwfOU/QAL4WGg= -github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190412120340-e22ddced7142 h1:JO6VBMEDSBX/LT4GKwSdvuFigZNwVD4lkPyUE4BDCKE= +github.com/prometheus/procfs v0.0.0-20190412120340-e22ddced7142/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= From 9a98fd8405751d55dd858b30722cff1a076358d3 Mon Sep 17 00:00:00 2001 From: prombot Date: Tue, 16 Apr 2019 00:00:39 +0000 Subject: [PATCH 49/73] makefile: update Makefile.common with newer version Signed-off-by: prombot --- Makefile.common | 35 ++++++++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/Makefile.common b/Makefile.common index ed29aa8..873964f 100644 --- a/Makefile.common +++ b/Makefile.common @@ -88,6 +88,12 @@ BIN_DIR ?= $(shell pwd) DOCKER_IMAGE_TAG ?= $(subst /,-,$(shell git rev-parse --abbrev-ref HEAD)) DOCKER_REPO ?= prom +DOCKER_ARCHS ?= amd64 + +BUILD_DOCKER_ARCHS = $(addprefix common-docker-,$(DOCKER_ARCHS)) +PUBLISH_DOCKER_ARCHS = $(addprefix common-docker-publish-,$(DOCKER_ARCHS)) +TAG_DOCKER_ARCHS = $(addprefix common-docker-tag-latest-,$(DOCKER_ARCHS)) + ifeq ($(GOHOSTARCH),amd64) ifeq ($(GOHOSTOS),$(filter $(GOHOSTOS),linux freebsd darwin windows)) # Only supported on amd64 @@ -197,17 +203,28 @@ common-tarball: promu @echo ">> building release tarball" $(PROMU) tarball --prefix $(PREFIX) $(BIN_DIR) -.PHONY: common-docker -common-docker: - docker build -t "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_TAG)" . +.PHONY: common-docker $(BUILD_DOCKER_ARCHS) +common-docker: $(BUILD_DOCKER_ARCHS) +$(BUILD_DOCKER_ARCHS): common-docker-%: + docker build -t "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:$(DOCKER_IMAGE_TAG)" \ + --build-arg ARCH="$*" \ + --build-arg OS="linux" \ + . -.PHONY: common-docker-publish -common-docker-publish: - docker push "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)" +.PHONY: common-docker-publish $(PUBLISH_DOCKER_ARCHS) +common-docker-publish: $(PUBLISH_DOCKER_ARCHS) +$(PUBLISH_DOCKER_ARCHS): common-docker-publish-%: + docker push "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:$(DOCKER_IMAGE_TAG)" -.PHONY: common-docker-tag-latest -common-docker-tag-latest: - docker tag "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_TAG)" "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):latest" +.PHONY: common-docker-tag-latest $(TAG_DOCKER_ARCHS) +common-docker-tag-latest: $(TAG_DOCKER_ARCHS) +$(TAG_DOCKER_ARCHS): common-docker-tag-latest-%: + docker tag "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:$(DOCKER_IMAGE_TAG)" "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:latest" + +.PHONY: common-docker-manifest +common-docker-manifest: + DOCKER_CLI_EXPERIMENTAL=enabled docker manifest create -a "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_TAG)" $(foreach ARCH,$(DOCKER_ARCHS),$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$(ARCH):$(DOCKER_IMAGE_TAG)) + DOCKER_CLI_EXPERIMENTAL=enabled docker manifest push "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_TAG)" .PHONY: promu promu: $(PROMU) From 53800fb9d3e532a75143073a0972992275ee1240 Mon Sep 17 00:00:00 2001 From: prombot Date: Thu, 25 Apr 2019 00:00:38 +0000 Subject: [PATCH 50/73] makefile: update Makefile.common with newer version Signed-off-by: prombot --- Makefile.common | 36 +++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/Makefile.common b/Makefile.common index 873964f..73052b3 100644 --- a/Makefile.common +++ b/Makefile.common @@ -72,14 +72,13 @@ endif PROMU_VERSION ?= 0.3.0 PROMU_URL := https://github.com/prometheus/promu/releases/download/v$(PROMU_VERSION)/promu-$(PROMU_VERSION).$(GO_BUILD_PLATFORM).tar.gz -STATICCHECK := -# staticcheck only supports linux, freebsd, darwin and windows platforms on i386/amd64 +GOLANGCI_LINT := +GOLANGCI_LINT_VERSION ?= v1.16.0 +# golangci-lint only supports linux, darwin and windows platforms on i386/amd64. # windows isn't included here because of the path separator being different. -ifeq ($(GOHOSTOS),$(filter $(GOHOSTOS),linux freebsd darwin)) +ifeq ($(GOHOSTOS),$(filter $(GOHOSTOS),linux darwin)) ifeq ($(GOHOSTARCH),$(filter $(GOHOSTARCH),amd64 i386)) - STATICCHECK := $(FIRST_GOPATH)/bin/staticcheck - STATICCHECK_VERSION ?= 2019.1 - STATICCHECK_URL := https://github.com/dominikh/go-tools/releases/download/$(STATICCHECK_VERSION)/staticcheck_$(GOHOSTOS)_$(GOHOSTARCH) + GOLANGCI_LINT := $(FIRST_GOPATH)/bin/golangci-lint endif endif @@ -107,7 +106,7 @@ endif %: common-% ; .PHONY: common-all -common-all: precheck style check_license staticcheck unused build test +common-all: precheck style check_license lint unused build test .PHONY: common-style common-style: @@ -159,21 +158,24 @@ common-vet: @echo ">> vetting code" GO111MODULE=$(GO111MODULE) $(GO) vet $(GOOPTS) $(pkgs) -.PHONY: common-staticcheck -common-staticcheck: $(STATICCHECK) -ifdef STATICCHECK - @echo ">> running staticcheck" - chmod +x $(STATICCHECK) +.PHONY: common-lint +common-lint: $(GOLANGCI_LINT) +ifdef GOLANGCI_LINT + @echo ">> running golangci-lint" ifdef GO111MODULE # 'go list' needs to be executed before staticcheck to prepopulate the modules cache. # Otherwise staticcheck might fail randomly for some reason not yet explained. GO111MODULE=$(GO111MODULE) $(GO) list -e -compiled -test=true -export=false -deps=true -find=false -tags= -- ./... > /dev/null - GO111MODULE=$(GO111MODULE) $(STATICCHECK) -ignore "$(STATICCHECK_IGNORE)" $(pkgs) + GO111MODULE=$(GO111MODULE) $(GOLANGCI_LINT) run $(pkgs) else - $(STATICCHECK) -ignore "$(STATICCHECK_IGNORE)" $(pkgs) + $(GOLANGCI_LINT) run $(pkgs) endif endif +# For backward-compatibility. +.PHONY: common-staticcheck +common-staticcheck: lint + .PHONY: common-unused common-unused: $(GOVENDOR) ifdef GOVENDOR @@ -241,10 +243,10 @@ proto: @echo ">> generating code from proto files" @./scripts/genproto.sh -ifdef STATICCHECK -$(STATICCHECK): +ifdef GOLANGCI_LINT +$(GOLANGCI_LINT): mkdir -p $(FIRST_GOPATH)/bin - curl -s -L $(STATICCHECK_URL) > $(STATICCHECK) + curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b $(FIRST_GOPATH)/bin $(GOLANGCI_LINT_VERSION) endif ifdef GOVENDOR From 6894bb3c7c28563f32c702f211541d4c20440acc Mon Sep 17 00:00:00 2001 From: Simon Pasquier Date: Thu, 25 Apr 2019 10:38:07 +0200 Subject: [PATCH 51/73] Add .golangci.yml Signed-off-by: Simon Pasquier --- .golangci.yml | 5 +++++ .travis.yml | 2 +- prometheus/http.go | 2 ++ prometheus/promhttp/delegator.go | 2 ++ prometheus/promhttp/delegator_1_8.go | 2 ++ prometheus/promhttp/instrument_server_test.go | 4 ++++ 6 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 .golangci.yml diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..d9efa75 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,5 @@ +# Run only staticcheck for now. Additional linters will be enabled one-by-one. +linters: + enable: + - staticcheck + disable-all: true diff --git a/.travis.yml b/.travis.yml index 502935d..ec1220d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,6 +10,6 @@ go: script: - make check_license unused test-short - - if [[ ! $TRAVIS_GO_VERSION =~ ^1\.(7|8|9)\.[x0-9]+$ ]]; then make staticcheck; fi + - if [[ ! $TRAVIS_GO_VERSION =~ ^1\.(7|8|9)\.[x0-9]+$ ]]; then make lint; fi # style is only checked against the latest supported Go version. - if [[ $TRAVIS_GO_VERSION =~ ^1\.(12)\. ]]; then make style; fi diff --git a/prometheus/http.go b/prometheus/http.go index 0fa339a..19a3e8f 100644 --- a/prometheus/http.go +++ b/prometheus/http.go @@ -330,6 +330,8 @@ type fancyResponseWriterDelegator struct { } func (f *fancyResponseWriterDelegator) CloseNotify() <-chan bool { + //lint:ignore SA1019 http.CloseNotifier is deprecated but we don't want to + //remove support from client_golang yet. return f.ResponseWriter.(http.CloseNotifier).CloseNotify() } diff --git a/prometheus/promhttp/delegator.go b/prometheus/promhttp/delegator.go index 5de5bbc..1f38090 100644 --- a/prometheus/promhttp/delegator.go +++ b/prometheus/promhttp/delegator.go @@ -76,6 +76,8 @@ type hijackerDelegator struct{ *responseWriterDelegator } type readerFromDelegator struct{ *responseWriterDelegator } func (d closeNotifierDelegator) CloseNotify() <-chan bool { + //lint:ignore SA1019 http.CloseNotifier is deprecated but we don't want to + //remove support from client_golang yet. return d.ResponseWriter.(http.CloseNotifier).CloseNotify() } func (d flusherDelegator) Flush() { diff --git a/prometheus/promhttp/delegator_1_8.go b/prometheus/promhttp/delegator_1_8.go index 31a7069..79dbd20 100644 --- a/prometheus/promhttp/delegator_1_8.go +++ b/prometheus/promhttp/delegator_1_8.go @@ -161,6 +161,8 @@ func newDelegator(w http.ResponseWriter, observeWriteHeaderFunc func(int)) deleg } id := 0 + //lint:ignore SA1019 http.CloseNotifier is deprecated but we don't want to + //remove support from client_golang yet. if _, ok := w.(http.CloseNotifier); ok { id += closeNotifier } diff --git a/prometheus/promhttp/instrument_server_test.go b/prometheus/promhttp/instrument_server_test.go index 716c6f4..11e42f2 100644 --- a/prometheus/promhttp/instrument_server_test.go +++ b/prometheus/promhttp/instrument_server_test.go @@ -294,6 +294,8 @@ func (t *testFlusher) Flush() { t.flushCalled = true } func TestInterfaceUpgrade(t *testing.T) { w := &testResponseWriter{} d := newDelegator(w, nil) + //lint:ignore SA1019 http.CloseNotifier is deprecated but we don't want to + //remove support from client_golang yet. d.(http.CloseNotifier).CloseNotify() if !w.closeNotifyCalled { t.Error("CloseNotify not called") @@ -312,6 +314,8 @@ func TestInterfaceUpgrade(t *testing.T) { f := &testFlusher{} d = newDelegator(f, nil) + //lint:ignore SA1019 http.CloseNotifier is deprecated but we don't want to + //remove support from client_golang yet. if _, ok := d.(http.CloseNotifier); ok { t.Error("delegator unexpectedly implements http.CloseNotifier") } From c9d77912ac86cbd90a802b36ffe3663c931cb394 Mon Sep 17 00:00:00 2001 From: NghiaLT Date: Thu, 25 Apr 2019 09:50:46 +0700 Subject: [PATCH 52/73] Add HTTPClient interface to Pusher struct Signed-off-by: NghiaLT --- prometheus/push/http.go | 21 +++++++++++++++++++++ prometheus/push/push.go | 4 ++-- 2 files changed, 23 insertions(+), 2 deletions(-) create mode 100644 prometheus/push/http.go diff --git a/prometheus/push/http.go b/prometheus/push/http.go new file mode 100644 index 0000000..208adf3 --- /dev/null +++ b/prometheus/push/http.go @@ -0,0 +1,21 @@ +// Copyright 2019 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package push + +import "net/http" + +// HTTPClient is a interface for http client +type HTTPClient interface { + Do(*http.Request) (*http.Response, error) +} diff --git a/prometheus/push/push.go b/prometheus/push/push.go index 6636ffc..f145919 100644 --- a/prometheus/push/push.go +++ b/prometheus/push/push.go @@ -61,7 +61,7 @@ type Pusher struct { gatherers prometheus.Gatherers registerer prometheus.Registerer - client *http.Client + client HTTPClient useBasicAuth bool username, password string @@ -170,7 +170,7 @@ func (p *Pusher) Grouping(name, value string) *Pusher { // Client sets a custom HTTP client for the Pusher. For convenience, this method // returns a pointer to the Pusher itself. -func (p *Pusher) Client(c *http.Client) *Pusher { +func (p *Pusher) Client(c HTTPClient) *Pusher { p.client = c return p } From 1173d734050af725504a9b2fbe93bff0b16f1b53 Mon Sep 17 00:00:00 2001 From: Bjoern Rabenstein Date: Sun, 28 Apr 2019 23:26:36 +0200 Subject: [PATCH 53/73] Increase minimum required Go version to 1.9 This allows us to simplify a bunch of code while still supporting the last four Go minor versions. We have also run into minor annoyances a couple of times by now to keep supporting 1.7 and 1.8. It's time to pull the plug! Signed-off-by: Bjoern Rabenstein --- .travis.yml | 8 +- README.md | 2 +- api/client.go | 2 - api/client_test.go | 2 - api/prometheus/v1/api.go | 2 - api/prometheus/v1/api_test.go | 2 - prometheus/promhttp/delegator.go | 29 +++ prometheus/promhttp/delegator_1_8.go | 183 ------------------- prometheus/promhttp/delegator_pre_1_8.go | 44 ----- prometheus/promhttp/instrument_client.go | 123 +++++++++++++ prometheus/promhttp/instrument_client_1_8.go | 144 --------------- 11 files changed, 156 insertions(+), 385 deletions(-) delete mode 100644 prometheus/promhttp/delegator_1_8.go delete mode 100644 prometheus/promhttp/delegator_pre_1_8.go delete mode 100644 prometheus/promhttp/instrument_client_1_8.go diff --git a/.travis.yml b/.travis.yml index ec1220d..9536ebc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,15 +1,13 @@ language: go go: - - 1.7.x # See README.md for current minimum version. - - 1.8.x - - 1.9.x + - 1.9.x # See README.md for current minimum version. - 1.10.x - 1.11.x - 1.12.x script: - make check_license unused test-short - - if [[ ! $TRAVIS_GO_VERSION =~ ^1\.(7|8|9)\.[x0-9]+$ ]]; then make lint; fi - # style is only checked against the latest supported Go version. + - if [[ ! $TRAVIS_GO_VERSION =~ ^1\.9\.[x0-9]+$ ]]; then make lint; fi + # Style is only checked against the latest supported Go version. - if [[ $TRAVIS_GO_VERSION =~ ^1\.(12)\. ]]; then make style; fi diff --git a/README.md b/README.md index 894a6a3..e2b464f 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ This is the [Go](http://golang.org) client library for instrumenting application code, and one for creating clients that talk to the Prometheus HTTP API. -__This library requires Go1.7 or later.__ +__This library requires Go1.9 or later.__ ## Important note about releases, versioning, tagging, and stability diff --git a/api/client.go b/api/client.go index 9d5b187..f7ca60b 100644 --- a/api/client.go +++ b/api/client.go @@ -11,8 +11,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -// +build go1.7 - // Package api provides clients for the HTTP APIs. package api diff --git a/api/client_test.go b/api/client_test.go index b1bcfc9..47094fc 100644 --- a/api/client_test.go +++ b/api/client_test.go @@ -11,8 +11,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -// +build go1.7 - package api import ( diff --git a/api/prometheus/v1/api.go b/api/prometheus/v1/api.go index c048179..c1cf47e 100644 --- a/api/prometheus/v1/api.go +++ b/api/prometheus/v1/api.go @@ -11,8 +11,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -// +build go1.7 - // Package v1 provides bindings to the Prometheus HTTP API v1: // http://prometheus.io/docs/querying/api/ package v1 diff --git a/api/prometheus/v1/api_test.go b/api/prometheus/v1/api_test.go index fec170e..b3204a9 100644 --- a/api/prometheus/v1/api_test.go +++ b/api/prometheus/v1/api_test.go @@ -11,8 +11,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -// +build go1.7 - package v1 import ( diff --git a/prometheus/promhttp/delegator.go b/prometheus/promhttp/delegator.go index 1f38090..7f7dd3f 100644 --- a/prometheus/promhttp/delegator.go +++ b/prometheus/promhttp/delegator.go @@ -74,6 +74,7 @@ type closeNotifierDelegator struct{ *responseWriterDelegator } type flusherDelegator struct{ *responseWriterDelegator } type hijackerDelegator struct{ *responseWriterDelegator } type readerFromDelegator struct{ *responseWriterDelegator } +type pusherDelegator struct{ *responseWriterDelegator } func (d closeNotifierDelegator) CloseNotify() <-chan bool { //lint:ignore SA1019 http.CloseNotifier is deprecated but we don't want to @@ -198,3 +199,31 @@ func init() { }{d, readerFromDelegator{d}, hijackerDelegator{d}, flusherDelegator{d}, closeNotifierDelegator{d}} } } + +func newDelegator(w http.ResponseWriter, observeWriteHeaderFunc func(int)) delegator { + d := &responseWriterDelegator{ + ResponseWriter: w, + observeWriteHeader: observeWriteHeaderFunc, + } + + id := 0 + //lint:ignore SA1019 http.CloseNotifier is deprecated but we don't want to + //remove support from client_golang yet. + if _, ok := w.(http.CloseNotifier); ok { + id += closeNotifier + } + if _, ok := w.(http.Flusher); ok { + id += flusher + } + if _, ok := w.(http.Hijacker); ok { + id += hijacker + } + if _, ok := w.(io.ReaderFrom); ok { + id += readerFrom + } + if _, ok := w.(http.Pusher); ok { + id += pusher + } + + return pickDelegator[id](d) +} diff --git a/prometheus/promhttp/delegator_1_8.go b/prometheus/promhttp/delegator_1_8.go deleted file mode 100644 index 79dbd20..0000000 --- a/prometheus/promhttp/delegator_1_8.go +++ /dev/null @@ -1,183 +0,0 @@ -// Copyright 2017 The Prometheus Authors -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// +build go1.8 - -package promhttp - -import ( - "io" - "net/http" -) - -type pusherDelegator struct{ *responseWriterDelegator } - -func (d pusherDelegator) Push(target string, opts *http.PushOptions) error { - return d.ResponseWriter.(http.Pusher).Push(target, opts) -} - -func init() { - pickDelegator[pusher] = func(d *responseWriterDelegator) delegator { // 16 - return pusherDelegator{d} - } - pickDelegator[pusher+closeNotifier] = func(d *responseWriterDelegator) delegator { // 17 - return struct { - *responseWriterDelegator - http.Pusher - http.CloseNotifier - }{d, pusherDelegator{d}, closeNotifierDelegator{d}} - } - pickDelegator[pusher+flusher] = func(d *responseWriterDelegator) delegator { // 18 - return struct { - *responseWriterDelegator - http.Pusher - http.Flusher - }{d, pusherDelegator{d}, flusherDelegator{d}} - } - pickDelegator[pusher+flusher+closeNotifier] = func(d *responseWriterDelegator) delegator { // 19 - return struct { - *responseWriterDelegator - http.Pusher - http.Flusher - http.CloseNotifier - }{d, pusherDelegator{d}, flusherDelegator{d}, closeNotifierDelegator{d}} - } - pickDelegator[pusher+hijacker] = func(d *responseWriterDelegator) delegator { // 20 - return struct { - *responseWriterDelegator - http.Pusher - http.Hijacker - }{d, pusherDelegator{d}, hijackerDelegator{d}} - } - pickDelegator[pusher+hijacker+closeNotifier] = func(d *responseWriterDelegator) delegator { // 21 - return struct { - *responseWriterDelegator - http.Pusher - http.Hijacker - http.CloseNotifier - }{d, pusherDelegator{d}, hijackerDelegator{d}, closeNotifierDelegator{d}} - } - pickDelegator[pusher+hijacker+flusher] = func(d *responseWriterDelegator) delegator { // 22 - return struct { - *responseWriterDelegator - http.Pusher - http.Hijacker - http.Flusher - }{d, pusherDelegator{d}, hijackerDelegator{d}, flusherDelegator{d}} - } - pickDelegator[pusher+hijacker+flusher+closeNotifier] = func(d *responseWriterDelegator) delegator { //23 - return struct { - *responseWriterDelegator - http.Pusher - http.Hijacker - http.Flusher - http.CloseNotifier - }{d, pusherDelegator{d}, hijackerDelegator{d}, flusherDelegator{d}, closeNotifierDelegator{d}} - } - pickDelegator[pusher+readerFrom] = func(d *responseWriterDelegator) delegator { // 24 - return struct { - *responseWriterDelegator - http.Pusher - io.ReaderFrom - }{d, pusherDelegator{d}, readerFromDelegator{d}} - } - pickDelegator[pusher+readerFrom+closeNotifier] = func(d *responseWriterDelegator) delegator { // 25 - return struct { - *responseWriterDelegator - http.Pusher - io.ReaderFrom - http.CloseNotifier - }{d, pusherDelegator{d}, readerFromDelegator{d}, closeNotifierDelegator{d}} - } - pickDelegator[pusher+readerFrom+flusher] = func(d *responseWriterDelegator) delegator { // 26 - return struct { - *responseWriterDelegator - http.Pusher - io.ReaderFrom - http.Flusher - }{d, pusherDelegator{d}, readerFromDelegator{d}, flusherDelegator{d}} - } - pickDelegator[pusher+readerFrom+flusher+closeNotifier] = func(d *responseWriterDelegator) delegator { // 27 - return struct { - *responseWriterDelegator - http.Pusher - io.ReaderFrom - http.Flusher - http.CloseNotifier - }{d, pusherDelegator{d}, readerFromDelegator{d}, flusherDelegator{d}, closeNotifierDelegator{d}} - } - pickDelegator[pusher+readerFrom+hijacker] = func(d *responseWriterDelegator) delegator { // 28 - return struct { - *responseWriterDelegator - http.Pusher - io.ReaderFrom - http.Hijacker - }{d, pusherDelegator{d}, readerFromDelegator{d}, hijackerDelegator{d}} - } - pickDelegator[pusher+readerFrom+hijacker+closeNotifier] = func(d *responseWriterDelegator) delegator { // 29 - return struct { - *responseWriterDelegator - http.Pusher - io.ReaderFrom - http.Hijacker - http.CloseNotifier - }{d, pusherDelegator{d}, readerFromDelegator{d}, hijackerDelegator{d}, closeNotifierDelegator{d}} - } - pickDelegator[pusher+readerFrom+hijacker+flusher] = func(d *responseWriterDelegator) delegator { // 30 - return struct { - *responseWriterDelegator - http.Pusher - io.ReaderFrom - http.Hijacker - http.Flusher - }{d, pusherDelegator{d}, readerFromDelegator{d}, hijackerDelegator{d}, flusherDelegator{d}} - } - pickDelegator[pusher+readerFrom+hijacker+flusher+closeNotifier] = func(d *responseWriterDelegator) delegator { // 31 - return struct { - *responseWriterDelegator - http.Pusher - io.ReaderFrom - http.Hijacker - http.Flusher - http.CloseNotifier - }{d, pusherDelegator{d}, readerFromDelegator{d}, hijackerDelegator{d}, flusherDelegator{d}, closeNotifierDelegator{d}} - } -} - -func newDelegator(w http.ResponseWriter, observeWriteHeaderFunc func(int)) delegator { - d := &responseWriterDelegator{ - ResponseWriter: w, - observeWriteHeader: observeWriteHeaderFunc, - } - - id := 0 - //lint:ignore SA1019 http.CloseNotifier is deprecated but we don't want to - //remove support from client_golang yet. - if _, ok := w.(http.CloseNotifier); ok { - id += closeNotifier - } - if _, ok := w.(http.Flusher); ok { - id += flusher - } - if _, ok := w.(http.Hijacker); ok { - id += hijacker - } - if _, ok := w.(io.ReaderFrom); ok { - id += readerFrom - } - if _, ok := w.(http.Pusher); ok { - id += pusher - } - - return pickDelegator[id](d) -} diff --git a/prometheus/promhttp/delegator_pre_1_8.go b/prometheus/promhttp/delegator_pre_1_8.go deleted file mode 100644 index 8bb9b8b..0000000 --- a/prometheus/promhttp/delegator_pre_1_8.go +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright 2017 The Prometheus Authors -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// +build !go1.8 - -package promhttp - -import ( - "io" - "net/http" -) - -func newDelegator(w http.ResponseWriter, observeWriteHeaderFunc func(int)) delegator { - d := &responseWriterDelegator{ - ResponseWriter: w, - observeWriteHeader: observeWriteHeaderFunc, - } - - id := 0 - if _, ok := w.(http.CloseNotifier); ok { - id += closeNotifier - } - if _, ok := w.(http.Flusher); ok { - id += flusher - } - if _, ok := w.(http.Hijacker); ok { - id += hijacker - } - if _, ok := w.(io.ReaderFrom); ok { - id += readerFrom - } - - return pickDelegator[id](d) -} diff --git a/prometheus/promhttp/instrument_client.go b/prometheus/promhttp/instrument_client.go index 86fd564..d5172db 100644 --- a/prometheus/promhttp/instrument_client.go +++ b/prometheus/promhttp/instrument_client.go @@ -14,7 +14,10 @@ package promhttp import ( + "context" + "crypto/tls" "net/http" + "net/http/httptrace" "time" "github.com/prometheus/client_golang/prometheus" @@ -95,3 +98,123 @@ func InstrumentRoundTripperDuration(obs prometheus.ObserverVec, next http.RoundT return resp, err }) } + +// InstrumentTrace is used to offer flexibility in instrumenting the available +// httptrace.ClientTrace hook functions. Each function is passed a float64 +// representing the time in seconds since the start of the http request. A user +// may choose to use separately buckets Histograms, or implement custom +// instance labels on a per function basis. +type InstrumentTrace struct { + GotConn func(float64) + PutIdleConn func(float64) + GotFirstResponseByte func(float64) + Got100Continue func(float64) + DNSStart func(float64) + DNSDone func(float64) + ConnectStart func(float64) + ConnectDone func(float64) + TLSHandshakeStart func(float64) + TLSHandshakeDone func(float64) + WroteHeaders func(float64) + Wait100Continue func(float64) + WroteRequest func(float64) +} + +// InstrumentRoundTripperTrace is a middleware that wraps the provided +// RoundTripper and reports times to hook functions provided in the +// InstrumentTrace struct. Hook functions that are not present in the provided +// InstrumentTrace struct are ignored. Times reported to the hook functions are +// time since the start of the request. Only with Go1.9+, those times are +// guaranteed to never be negative. (Earlier Go versions are not using a +// monotonic clock.) Note that partitioning of Histograms is expensive and +// should be used judiciously. +// +// For hook functions that receive an error as an argument, no observations are +// made in the event of a non-nil error value. +// +// See the example for ExampleInstrumentRoundTripperDuration for example usage. +func InstrumentRoundTripperTrace(it *InstrumentTrace, next http.RoundTripper) RoundTripperFunc { + return RoundTripperFunc(func(r *http.Request) (*http.Response, error) { + start := time.Now() + + trace := &httptrace.ClientTrace{ + GotConn: func(_ httptrace.GotConnInfo) { + if it.GotConn != nil { + it.GotConn(time.Since(start).Seconds()) + } + }, + PutIdleConn: func(err error) { + if err != nil { + return + } + if it.PutIdleConn != nil { + it.PutIdleConn(time.Since(start).Seconds()) + } + }, + DNSStart: func(_ httptrace.DNSStartInfo) { + if it.DNSStart != nil { + it.DNSStart(time.Since(start).Seconds()) + } + }, + DNSDone: func(_ httptrace.DNSDoneInfo) { + if it.DNSDone != nil { + it.DNSDone(time.Since(start).Seconds()) + } + }, + ConnectStart: func(_, _ string) { + if it.ConnectStart != nil { + it.ConnectStart(time.Since(start).Seconds()) + } + }, + ConnectDone: func(_, _ string, err error) { + if err != nil { + return + } + if it.ConnectDone != nil { + it.ConnectDone(time.Since(start).Seconds()) + } + }, + GotFirstResponseByte: func() { + if it.GotFirstResponseByte != nil { + it.GotFirstResponseByte(time.Since(start).Seconds()) + } + }, + Got100Continue: func() { + if it.Got100Continue != nil { + it.Got100Continue(time.Since(start).Seconds()) + } + }, + TLSHandshakeStart: func() { + if it.TLSHandshakeStart != nil { + it.TLSHandshakeStart(time.Since(start).Seconds()) + } + }, + TLSHandshakeDone: func(_ tls.ConnectionState, err error) { + if err != nil { + return + } + if it.TLSHandshakeDone != nil { + it.TLSHandshakeDone(time.Since(start).Seconds()) + } + }, + WroteHeaders: func() { + if it.WroteHeaders != nil { + it.WroteHeaders(time.Since(start).Seconds()) + } + }, + Wait100Continue: func() { + if it.Wait100Continue != nil { + it.Wait100Continue(time.Since(start).Seconds()) + } + }, + WroteRequest: func(_ httptrace.WroteRequestInfo) { + if it.WroteRequest != nil { + it.WroteRequest(time.Since(start).Seconds()) + } + }, + } + r = r.WithContext(httptrace.WithClientTrace(context.Background(), trace)) + + return next.RoundTrip(r) + }) +} diff --git a/prometheus/promhttp/instrument_client_1_8.go b/prometheus/promhttp/instrument_client_1_8.go deleted file mode 100644 index a034d1e..0000000 --- a/prometheus/promhttp/instrument_client_1_8.go +++ /dev/null @@ -1,144 +0,0 @@ -// Copyright 2017 The Prometheus Authors -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// +build go1.8 - -package promhttp - -import ( - "context" - "crypto/tls" - "net/http" - "net/http/httptrace" - "time" -) - -// InstrumentTrace is used to offer flexibility in instrumenting the available -// httptrace.ClientTrace hook functions. Each function is passed a float64 -// representing the time in seconds since the start of the http request. A user -// may choose to use separately buckets Histograms, or implement custom -// instance labels on a per function basis. -type InstrumentTrace struct { - GotConn func(float64) - PutIdleConn func(float64) - GotFirstResponseByte func(float64) - Got100Continue func(float64) - DNSStart func(float64) - DNSDone func(float64) - ConnectStart func(float64) - ConnectDone func(float64) - TLSHandshakeStart func(float64) - TLSHandshakeDone func(float64) - WroteHeaders func(float64) - Wait100Continue func(float64) - WroteRequest func(float64) -} - -// InstrumentRoundTripperTrace is a middleware that wraps the provided -// RoundTripper and reports times to hook functions provided in the -// InstrumentTrace struct. Hook functions that are not present in the provided -// InstrumentTrace struct are ignored. Times reported to the hook functions are -// time since the start of the request. Only with Go1.9+, those times are -// guaranteed to never be negative. (Earlier Go versions are not using a -// monotonic clock.) Note that partitioning of Histograms is expensive and -// should be used judiciously. -// -// For hook functions that receive an error as an argument, no observations are -// made in the event of a non-nil error value. -// -// See the example for ExampleInstrumentRoundTripperDuration for example usage. -func InstrumentRoundTripperTrace(it *InstrumentTrace, next http.RoundTripper) RoundTripperFunc { - return RoundTripperFunc(func(r *http.Request) (*http.Response, error) { - start := time.Now() - - trace := &httptrace.ClientTrace{ - GotConn: func(_ httptrace.GotConnInfo) { - if it.GotConn != nil { - it.GotConn(time.Since(start).Seconds()) - } - }, - PutIdleConn: func(err error) { - if err != nil { - return - } - if it.PutIdleConn != nil { - it.PutIdleConn(time.Since(start).Seconds()) - } - }, - DNSStart: func(_ httptrace.DNSStartInfo) { - if it.DNSStart != nil { - it.DNSStart(time.Since(start).Seconds()) - } - }, - DNSDone: func(_ httptrace.DNSDoneInfo) { - if it.DNSDone != nil { - it.DNSDone(time.Since(start).Seconds()) - } - }, - ConnectStart: func(_, _ string) { - if it.ConnectStart != nil { - it.ConnectStart(time.Since(start).Seconds()) - } - }, - ConnectDone: func(_, _ string, err error) { - if err != nil { - return - } - if it.ConnectDone != nil { - it.ConnectDone(time.Since(start).Seconds()) - } - }, - GotFirstResponseByte: func() { - if it.GotFirstResponseByte != nil { - it.GotFirstResponseByte(time.Since(start).Seconds()) - } - }, - Got100Continue: func() { - if it.Got100Continue != nil { - it.Got100Continue(time.Since(start).Seconds()) - } - }, - TLSHandshakeStart: func() { - if it.TLSHandshakeStart != nil { - it.TLSHandshakeStart(time.Since(start).Seconds()) - } - }, - TLSHandshakeDone: func(_ tls.ConnectionState, err error) { - if err != nil { - return - } - if it.TLSHandshakeDone != nil { - it.TLSHandshakeDone(time.Since(start).Seconds()) - } - }, - WroteHeaders: func() { - if it.WroteHeaders != nil { - it.WroteHeaders(time.Since(start).Seconds()) - } - }, - Wait100Continue: func() { - if it.Wait100Continue != nil { - it.Wait100Continue(time.Since(start).Seconds()) - } - }, - WroteRequest: func(_ httptrace.WroteRequestInfo) { - if it.WroteRequest != nil { - it.WroteRequest(time.Since(start).Seconds()) - } - }, - } - r = r.WithContext(httptrace.WithClientTrace(context.Background(), trace)) - - return next.RoundTrip(r) - }) -} From a4daf0098cbdb8d67db6a1cef73c7e7956edef05 Mon Sep 17 00:00:00 2001 From: Thomas Jackson Date: Tue, 30 Apr 2019 00:13:48 -0700 Subject: [PATCH 54/73] Implement POST with get fallback for Query/QueryRange (#557) * Implement POST with get fallback for Query/QueryRange Signed-off-by: Thomas Jackson --- api/client.go | 22 +++++++++++ api/client_test.go | 72 +++++++++++++++++++++++++++++++++++ api/prometheus/v1/api.go | 16 +------- api/prometheus/v1/api_test.go | 10 ++--- go.mod | 1 + go.sum | 13 +++++++ 6 files changed, 115 insertions(+), 19 deletions(-) diff --git a/api/client.go b/api/client.go index f7ca60b..db78ce2 100644 --- a/api/client.go +++ b/api/client.go @@ -58,6 +58,28 @@ type Client interface { Do(context.Context, *http.Request) (*http.Response, []byte, error) } +// DoGetFallback will attempt to do the request as-is, and on a 405 it will fallback to a GET request. +func DoGetFallback(c Client, ctx context.Context, u *url.URL, args url.Values) (*http.Response, []byte, error) { + req, err := http.NewRequest(http.MethodPost, u.String(), strings.NewReader(args.Encode())) + if err != nil { + return nil, nil, err + } + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + + resp, body, err := c.Do(ctx, req) + if resp != nil && resp.StatusCode == http.StatusMethodNotAllowed { + u.RawQuery = args.Encode() + req, err = http.NewRequest(http.MethodGet, u.String(), nil) + if err != nil { + return nil, nil, err + } + + } else { + return resp, body, err + } + return c.Do(ctx, req) +} + // NewClient returns a new Client. // // It is safe to use the returned Client from multiple goroutines. diff --git a/api/client_test.go b/api/client_test.go index 47094fc..7877e6a 100644 --- a/api/client_test.go +++ b/api/client_test.go @@ -14,9 +14,14 @@ package api import ( + "context" + "encoding/json" "net/http" + "net/http/httptest" "net/url" "testing" + + "github.com/prometheus/tsdb/testutil" ) func TestConfig(t *testing.T) { @@ -111,3 +116,70 @@ func TestClientURL(t *testing.T) { } } } + +func TestDoGetFallback(t *testing.T) { + v := url.Values{"a": []string{"1", "2"}} + + type testResponse struct { + Values string + Method string + } + + // Start a local HTTP server. + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + req.ParseForm() + r := &testResponse{ + Values: req.Form.Encode(), + Method: req.Method, + } + + body, _ := json.Marshal(r) + + if req.Method == http.MethodPost { + if req.URL.Path == "/blockPost" { + http.Error(w, string(body), http.StatusMethodNotAllowed) + return + } + } + + w.Write(body) + })) + // Close the server when test finishes. + defer server.Close() + + u, err := url.Parse(server.URL) + testutil.Ok(t, err) + client := &httpClient{client: *(server.Client())} + + // Do a post, and ensure that the post succeeds. + _, b, err := DoGetFallback(client, context.TODO(), u, v) + if err != nil { + t.Fatalf("Error doing local request: %v", err) + } + resp := &testResponse{} + if err := json.Unmarshal(b, resp); err != nil { + testutil.Ok(t, err) + } + if resp.Method != http.MethodPost { + t.Fatalf("Mismatch method") + } + if resp.Values != v.Encode() { + t.Fatalf("Mismatch in values") + } + + // Do a fallbcak to a get. + u.Path = "/blockPost" + _, b, err = DoGetFallback(client, context.TODO(), u, v) + if err != nil { + t.Fatalf("Error doing local request: %v", err) + } + if err := json.Unmarshal(b, resp); err != nil { + testutil.Ok(t, err) + } + if resp.Method != http.MethodGet { + t.Fatalf("Mismatch method") + } + if resp.Values != v.Encode() { + t.Fatalf("Mismatch in values") + } +} diff --git a/api/prometheus/v1/api.go b/api/prometheus/v1/api.go index c1cf47e..187527f 100644 --- a/api/prometheus/v1/api.go +++ b/api/prometheus/v1/api.go @@ -540,12 +540,7 @@ func (h *httpAPI) Query(ctx context.Context, query string, ts time.Time) (model. u.RawQuery = q.Encode() - req, err := http.NewRequest(http.MethodGet, u.String(), nil) - if err != nil { - return nil, err - } - - _, body, err := h.client.Do(ctx, req) + _, body, err := api.DoGetFallback(h.client, ctx, u, q) if err != nil { return nil, err } @@ -571,14 +566,7 @@ func (h *httpAPI) QueryRange(ctx context.Context, query string, r Range) (model. q.Set("end", end) q.Set("step", step) - u.RawQuery = q.Encode() - - req, err := http.NewRequest(http.MethodGet, u.String(), nil) - if err != nil { - return nil, err - } - - _, body, err := h.client.Do(ctx, req) + _, body, err := api.DoGetFallback(h.client, ctx, u, q) if err != nil { return nil, err } diff --git a/api/prometheus/v1/api_test.go b/api/prometheus/v1/api_test.go index b3204a9..f195f2b 100644 --- a/api/prometheus/v1/api_test.go +++ b/api/prometheus/v1/api_test.go @@ -179,7 +179,7 @@ func TestAPIs(t *testing.T) { }, }, - reqMethod: "GET", + reqMethod: "POST", reqPath: "/api/v1/query", reqParam: url.Values{ "query": []string{"2"}, @@ -194,7 +194,7 @@ func TestAPIs(t *testing.T) { do: doQuery("2", testTime), inErr: fmt.Errorf("some error"), - reqMethod: "GET", + reqMethod: "POST", reqPath: "/api/v1/query", reqParam: url.Values{ "query": []string{"2"}, @@ -212,7 +212,7 @@ func TestAPIs(t *testing.T) { Detail: "some body", }, - reqMethod: "GET", + reqMethod: "POST", reqPath: "/api/v1/query", reqParam: url.Values{ "query": []string{"2"}, @@ -230,7 +230,7 @@ func TestAPIs(t *testing.T) { Detail: "some body", }, - reqMethod: "GET", + reqMethod: "POST", reqPath: "/api/v1/query", reqParam: url.Values{ "query": []string{"2"}, @@ -247,7 +247,7 @@ func TestAPIs(t *testing.T) { }), inErr: fmt.Errorf("some error"), - reqMethod: "GET", + reqMethod: "POST", reqPath: "/api/v1/query_range", reqParam: url.Values{ "query": []string{"2"}, diff --git a/go.mod b/go.mod index 37f7f40..2168f76 100644 --- a/go.mod +++ b/go.mod @@ -6,4 +6,5 @@ require ( github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 github.com/prometheus/common v0.3.0 github.com/prometheus/procfs v0.0.0-20190412120340-e22ddced7142 + github.com/prometheus/tsdb v0.7.1 ) diff --git a/go.sum b/go.sum index 235ab95..d998271 100644 --- a/go.sum +++ b/go.sum @@ -1,12 +1,18 @@ +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/go-kit/kit v0.8.0 h1:Wz+5lgoB0kkuqLEc6NVmwRknTKP6dTGbSqvhZtBI/j0= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0 h1:8HUsc87TaSWLKwrnumgC8/YconD2fJQsRJAsWaPg2ic= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= @@ -15,10 +21,12 @@ github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -26,12 +34,16 @@ github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 h1:idejC8f github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.3.0 h1:taZ4h8Tkxv2kNyoSctBvfXEHmBmxrwmIidZTIaHons4= github.com/prometheus/common v0.3.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190412120340-e22ddced7142 h1:JO6VBMEDSBX/LT4GKwSdvuFigZNwVD4lkPyUE4BDCKE= github.com/prometheus/procfs v0.0.0-20190412120340-e22ddced7142/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/tsdb v0.7.1 h1:YZcsG11NqnK4czYLrWd9mpEuAJIHVQLwdrleYfszMAA= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -41,6 +53,7 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 3f6cbd95606771ac9f7b1c9247d2ca186cb72cb9 Mon Sep 17 00:00:00 2001 From: Thomas Jackson Date: Tue, 30 Apr 2019 23:28:05 -0700 Subject: [PATCH 55/73] Remove encode of params in query before DoGetFallback (#563) Signed-off-by: Thomas Jackson --- api/prometheus/v1/api.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/api/prometheus/v1/api.go b/api/prometheus/v1/api.go index 187527f..8394c97 100644 --- a/api/prometheus/v1/api.go +++ b/api/prometheus/v1/api.go @@ -538,8 +538,6 @@ func (h *httpAPI) Query(ctx context.Context, query string, ts time.Time) (model. q.Set("time", ts.Format(time.RFC3339Nano)) } - u.RawQuery = q.Encode() - _, body, err := api.DoGetFallback(h.client, ctx, u, q) if err != nil { return nil, err From a325fb756688945a2b742d5fafc765081f4e9023 Mon Sep 17 00:00:00 2001 From: KevinBetterQ <1093850932@qq.com> Date: Wed, 1 May 2019 17:50:23 +0800 Subject: [PATCH 56/73] fix: fix a typo Signed-off-by: KevinBetterQ <1093850932@qq.com> --- prometheus/wrap_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/prometheus/wrap_test.go b/prometheus/wrap_test.go index 28e2470..0fbb78c 100644 --- a/prometheus/wrap_test.go +++ b/prometheus/wrap_test.go @@ -286,7 +286,7 @@ func TestWrap(t *testing.T) { err = lReg.Register(tr.collector) } if tr.registrationFails && err == nil { - t.Fatalf("registration with wrapping registry unexpectedly succeded for collector #%d", i) + t.Fatalf("registration with wrapping registry unexpectedly succeeded for collector #%d", i) } if !tr.registrationFails && err != nil { t.Fatalf("registration with wrapping registry failed for collector #%d: %s", i, err) @@ -295,7 +295,7 @@ func TestWrap(t *testing.T) { wantMF := toMetricFamilies(s.output...) gotMF, err := reg.Gather() if s.gatherFails && err == nil { - t.Fatal("gathering unexpectedly succeded") + t.Fatal("gathering unexpectedly succeeded") } if !s.gatherFails && err != nil { t.Fatal("gathering failed:", err) From 57f7bd35fd4585f4d54cddf0b56db07f13a40e9e Mon Sep 17 00:00:00 2001 From: Bjoern Rabenstein Date: Fri, 3 May 2019 21:30:30 +0200 Subject: [PATCH 57/73] Remove remaining traces of pre-go-1.8 handling Signed-off-by: Bjoern Rabenstein --- ...{instrument_client_1_8_test.go => instrument_client_test.go} | 2 -- 1 file changed, 2 deletions(-) rename prometheus/promhttp/{instrument_client_1_8_test.go => instrument_client_test.go} (99%) diff --git a/prometheus/promhttp/instrument_client_1_8_test.go b/prometheus/promhttp/instrument_client_test.go similarity index 99% rename from prometheus/promhttp/instrument_client_1_8_test.go rename to prometheus/promhttp/instrument_client_test.go index 2d0ff4f..59c0a2f 100644 --- a/prometheus/promhttp/instrument_client_1_8_test.go +++ b/prometheus/promhttp/instrument_client_test.go @@ -11,8 +11,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -// +build go1.8 - package promhttp import ( From bf1f4e4a246d1094eb9bf44b48b8793fb1df9413 Mon Sep 17 00:00:00 2001 From: Bjoern Rabenstein Date: Fri, 3 May 2019 22:25:32 +0200 Subject: [PATCH 58/73] Make TestCounterAddLarge more robust The previous `float64(math.MaxUint64 + 1)` is too close to `float64(math.MaxUint64)` to actually overflow as indended. The counter code is actually converting forward and backward and compare the original and twice-converted value. On most platform, this will create a deviation and thus trigger the expected behavior. By sheer "luck", one might end up with the same value and thus still use the uint64 representation. Which is OK within the precision we can expect. But it breaks the test. With this change, the next representable floating point value greater than the floating point value used to represent math.MaxUint64 is used. Signed-off-by: Bjoern Rabenstein --- prometheus/counter_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prometheus/counter_test.go b/prometheus/counter_test.go index 5062f51..fd98fb1 100644 --- a/prometheus/counter_test.go +++ b/prometheus/counter_test.go @@ -172,7 +172,7 @@ func TestCounterAddLarge(t *testing.T) { }).(*counter) // large overflows the underlying type and should therefore be stored in valBits. - large := float64(math.MaxUint64 + 1) + large := math.Nextafter(float64(math.MaxUint64), 1e20) counter.Add(large) if expected, got := large, math.Float64frombits(counter.valBits); expected != got { t.Errorf("valBits expected %f, got %f.", expected, got) From da0c9e549ac3d86506985f53f8b831bbf4ca80da Mon Sep 17 00:00:00 2001 From: prombot Date: Sat, 4 May 2019 00:00:58 +0000 Subject: [PATCH 59/73] makefile: update Makefile.common with newer version Signed-off-by: prombot --- Makefile.common | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile.common b/Makefile.common index 73052b3..4f18ea5 100644 --- a/Makefile.common +++ b/Makefile.common @@ -73,6 +73,7 @@ PROMU_VERSION ?= 0.3.0 PROMU_URL := https://github.com/prometheus/promu/releases/download/v$(PROMU_VERSION)/promu-$(PROMU_VERSION).$(GO_BUILD_PLATFORM).tar.gz GOLANGCI_LINT := +GOLANGCI_LINT_OPTS ?= GOLANGCI_LINT_VERSION ?= v1.16.0 # golangci-lint only supports linux, darwin and windows platforms on i386/amd64. # windows isn't included here because of the path separator being different. @@ -166,7 +167,7 @@ ifdef GO111MODULE # 'go list' needs to be executed before staticcheck to prepopulate the modules cache. # Otherwise staticcheck might fail randomly for some reason not yet explained. GO111MODULE=$(GO111MODULE) $(GO) list -e -compiled -test=true -export=false -deps=true -find=false -tags= -- ./... > /dev/null - GO111MODULE=$(GO111MODULE) $(GOLANGCI_LINT) run $(pkgs) + GO111MODULE=$(GO111MODULE) $(GOLANGCI_LINT) run $(GOLANGCI_LINT_OPTS) $(pkgs) else $(GOLANGCI_LINT) run $(pkgs) endif From 6ea6f0791825cf7e5de30836a66e0574d817366a Mon Sep 17 00:00:00 2001 From: NghiaLT Date: Mon, 6 May 2019 04:18:09 +0700 Subject: [PATCH 60/73] Replace HTTPClient with HTTPDoer; Update document accordingly Signed-off-by: NghiaLT --- prometheus/push/http.go | 21 --------------------- prometheus/push/push.go | 13 +++++++++++-- 2 files changed, 11 insertions(+), 23 deletions(-) delete mode 100644 prometheus/push/http.go diff --git a/prometheus/push/http.go b/prometheus/push/http.go deleted file mode 100644 index 208adf3..0000000 --- a/prometheus/push/http.go +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright 2019 The Prometheus Authors -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package push - -import "net/http" - -// HTTPClient is a interface for http client -type HTTPClient interface { - Do(*http.Request) (*http.Response, error) -} diff --git a/prometheus/push/push.go b/prometheus/push/push.go index f145919..3de115d 100644 --- a/prometheus/push/push.go +++ b/prometheus/push/push.go @@ -50,6 +50,11 @@ import ( const contentTypeHeader = "Content-Type" +// HTTPDoer is an interface for the one method of http.Client that is used by Pusher +type HTTPDoer interface { + Do(*http.Request) (*http.Response, error) +} + // Pusher manages a push to the Pushgateway. Use New to create one, configure it // with its methods, and finally use the Add or Push method to push. type Pusher struct { @@ -61,7 +66,7 @@ type Pusher struct { gatherers prometheus.Gatherers registerer prometheus.Registerer - client HTTPClient + client HTTPDoer useBasicAuth bool username, password string @@ -170,7 +175,11 @@ func (p *Pusher) Grouping(name, value string) *Pusher { // Client sets a custom HTTP client for the Pusher. For convenience, this method // returns a pointer to the Pusher itself. -func (p *Pusher) Client(c HTTPClient) *Pusher { +// Pusher only needs one method of the custom HTTP client: Do(*http.Request). +// Thus, rather than requiring a fully fledged http.Client, +// the provided client only needs to implement the HTTPDoer interface. +// Since *http.Client naturally implements that interface, it can still be used normally. +func (p *Pusher) Client(c HTTPDoer) *Pusher { p.client = c return p } From 016273b1f98f93a268e8eba12a5548c315fc0a02 Mon Sep 17 00:00:00 2001 From: beorn7 Date: Sun, 5 May 2019 23:44:22 +0200 Subject: [PATCH 61/73] Fix and tweak Go collector tests Signed-off-by: beorn7 --- prometheus/go_collector_test.go | 86 +++++++++++++++++++-------------- 1 file changed, 51 insertions(+), 35 deletions(-) diff --git a/prometheus/go_collector_test.go b/prometheus/go_collector_test.go index f93dcdc..7ab3968 100644 --- a/prometheus/go_collector_test.go +++ b/prometheus/go_collector_test.go @@ -21,28 +21,40 @@ import ( dto "github.com/prometheus/client_model/go" ) -func TestGoCollector(t *testing.T) { +func TestGoCollectorGoroutines(t *testing.T) { var ( - c = NewGoCollector() - ch = make(chan Metric) - waitc = make(chan struct{}) - closec = make(chan struct{}) - old = -1 + c = NewGoCollector() + metricCh = make(chan Metric) + waitCh = make(chan struct{}) + endGoroutineCh = make(chan struct{}) + endCollectionCh = make(chan struct{}) + old = -1 ) - defer close(closec) + defer func() { + close(endGoroutineCh) + // Drain the collect channel to prevent goroutine leak. + for { + select { + case <-metricCh: + case <-endCollectionCh: + return + } + } + }() go func() { - c.Collect(ch) + c.Collect(metricCh) go func(c <-chan struct{}) { <-c - }(closec) - <-waitc - c.Collect(ch) + }(endGoroutineCh) + <-waitCh + c.Collect(metricCh) + close(endCollectionCh) }() for { select { - case m := <-ch: + case m := <-metricCh: // m can be Gauge or Counter, // currently just test the go_goroutines Gauge // and ignore others. @@ -57,7 +69,7 @@ func TestGoCollector(t *testing.T) { if old == -1 { old = int(pb.GetGauge().GetValue()) - close(waitc) + close(waitCh) continue } @@ -65,43 +77,47 @@ func TestGoCollector(t *testing.T) { // TODO: This is flaky in highly concurrent situations. t.Errorf("want 1 new goroutine, got %d", diff) } - - // GoCollector performs three sends per call. - // On line 27 we need to receive three more sends - // to shut down cleanly. - <-ch - <-ch - <-ch - return case <-time.After(1 * time.Second): t.Fatalf("expected collect timed out") } + break } } -func TestGCCollector(t *testing.T) { +func TestGoCollectorGC(t *testing.T) { var ( - c = NewGoCollector() - ch = make(chan Metric) - waitc = make(chan struct{}) - closec = make(chan struct{}) - oldGC uint64 - oldPause float64 + c = NewGoCollector() + metricCh = make(chan Metric) + waitCh = make(chan struct{}) + endCollectionCh = make(chan struct{}) + oldGC uint64 + oldPause float64 ) - defer close(closec) go func() { - c.Collect(ch) + c.Collect(metricCh) // force GC runtime.GC() - <-waitc - c.Collect(ch) + <-waitCh + c.Collect(metricCh) + close(endCollectionCh) + }() + + defer func() { + // Drain the collect channel to prevent goroutine leak. + for { + select { + case <-metricCh: + case <-endCollectionCh: + return + } + } }() first := true for { select { - case metric := <-ch: + case metric := <-metricCh: pb := &dto.Metric{} metric.Write(pb) if pb.GetSummary() == nil { @@ -119,7 +135,7 @@ func TestGCCollector(t *testing.T) { first = false oldGC = *pb.GetSummary().SampleCount oldPause = *pb.GetSummary().SampleSum - close(waitc) + close(waitCh) continue } if diff := *pb.GetSummary().SampleCount - oldGC; diff != 1 { @@ -128,9 +144,9 @@ func TestGCCollector(t *testing.T) { if diff := *pb.GetSummary().SampleSum - oldPause; diff <= 0 { t.Errorf("want moar pause, got %f", diff) } - return case <-time.After(1 * time.Second): t.Fatalf("expected collect timed out") } + break } } From 547c945a6226045c65f4f8a840d96a456db97651 Mon Sep 17 00:00:00 2001 From: beorn7 Date: Mon, 6 May 2019 00:39:02 +0200 Subject: [PATCH 62/73] Replace fmt.Sprintf by simple concatenation In this simple case, it's the fastest and easiest. Signed-off-by: beorn7 --- prometheus/go_collector.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/prometheus/go_collector.go b/prometheus/go_collector.go index ba3b933..fcb5ccb 100644 --- a/prometheus/go_collector.go +++ b/prometheus/go_collector.go @@ -14,7 +14,6 @@ package prometheus import ( - "fmt" "runtime" "runtime/debug" "time" @@ -253,7 +252,7 @@ func NewGoCollector() Collector { } func memstatNamespace(s string) string { - return fmt.Sprintf("go_memstats_%s", s) + return "go_memstats_" + s } // Describe returns all descriptions of the collector. From 7cf09554214c5d2a1cc31d78629b92bd97edebe4 Mon Sep 17 00:00:00 2001 From: Bjoern Rabenstein Date: Sat, 4 May 2019 01:00:59 +0200 Subject: [PATCH 63/73] Handle long ReadMemStats duration in Go collector tl;dr: Return previous memstats if reading new ones takes longer than 1s. See the doc comment of NewGoCollector for details. Signed-off-by: beorn7 --- prometheus/go_collector.go | 86 +++++++++++++++++++++++++++++---- prometheus/go_collector_test.go | 85 ++++++++++++++++++++++++++++++++ 2 files changed, 161 insertions(+), 10 deletions(-) diff --git a/prometheus/go_collector.go b/prometheus/go_collector.go index fcb5ccb..b108ec5 100644 --- a/prometheus/go_collector.go +++ b/prometheus/go_collector.go @@ -16,6 +16,7 @@ package prometheus import ( "runtime" "runtime/debug" + "sync" "time" ) @@ -25,16 +26,41 @@ type goCollector struct { gcDesc *Desc goInfoDesc *Desc - // metrics to describe and collect - metrics memStatsMetrics + // ms... are memstats related. + msLast *runtime.MemStats // Previously collected memstats. + msLastTimestamp time.Time + msMtx sync.Mutex // Protects msLast and msLastTimestamp. + msMetrics memStatsMetrics + msRead func(*runtime.MemStats) // For mocking in tests. + msMaxWait time.Duration // Wait time for fresh memstats. + msMaxAge time.Duration // Maximum allowed age of old memstats. } // NewGoCollector returns a collector which exports metrics about the current Go // process. This includes memory stats. To collect those, runtime.ReadMemStats -// is called. This causes a stop-the-world, which is very short with Go1.9+ -// (~25µs). However, with older Go versions, the stop-the-world duration depends -// on the heap size and can be quite significant (~1.7 ms/GiB as per +// is called. This requires to “stop the world”, which usually only happens for +// garbage collection (GC). Take the following implications into account when +// deciding whether to use the Go collector: +// +// 1. The performance impact of stopping the world is the more relevant the more +// frequently metrics are collected. However, with Go1.9 or later the +// stop-the-world time per metrics collection is very short (~25µs) so that the +// performance impact will only matter in rare cases. However, with older Go +// versions, the stop-the-world duration depends on the heap size and can be +// quite significant (~1.7 ms/GiB as per // https://go-review.googlesource.com/c/go/+/34937). +// +// 2. During an ongoing GC, nothing else can stop the world. Therefore, if the +// metrics collection happens to coincide with GC, it will only complete after +// GC has finished. Usually, GC is fast enough to not cause problems. However, +// with a very large heap, GC might take multiple seconds, which is enough to +// cause scrape timeouts in common setups. To avoid this problem, the Go +// collector will use the memstats from a previous collection if +// runtime.ReadMemStats takes more than 1s. However, if there are no previously +// collected memstats, or their collection is more than 5m ago, the collection +// will block until runtime.ReadMemStats succeeds. (The problem might be solved +// in Go1.13, see https://github.com/golang/go/issues/19812 for the related Go +// issue.) func NewGoCollector() Collector { return &goCollector{ goroutinesDesc: NewDesc( @@ -53,7 +79,11 @@ func NewGoCollector() Collector { "go_info", "Information about the Go environment.", nil, Labels{"version": runtime.Version()}), - metrics: memStatsMetrics{ + msLast: &runtime.MemStats{}, + msRead: runtime.ReadMemStats, + msMaxWait: time.Second, + msMaxAge: 5 * time.Minute, + msMetrics: memStatsMetrics{ { desc: NewDesc( memstatNamespace("alloc_bytes"), @@ -261,13 +291,27 @@ func (c *goCollector) Describe(ch chan<- *Desc) { ch <- c.threadsDesc ch <- c.gcDesc ch <- c.goInfoDesc - for _, i := range c.metrics { + for _, i := range c.msMetrics { ch <- i.desc } } // Collect returns the current state of all metrics of the collector. func (c *goCollector) Collect(ch chan<- Metric) { + var ( + ms = &runtime.MemStats{} + done = make(chan struct{}) + ) + // Start reading memstats first as it might take a while. + go func() { + c.msRead(ms) + c.msMtx.Lock() + c.msLast = ms + c.msLastTimestamp = time.Now() + c.msMtx.Unlock() + close(done) + }() + ch <- MustNewConstMetric(c.goroutinesDesc, GaugeValue, float64(runtime.NumGoroutine())) n, _ := runtime.ThreadCreateProfile(nil) ch <- MustNewConstMetric(c.threadsDesc, GaugeValue, float64(n)) @@ -285,9 +329,31 @@ func (c *goCollector) Collect(ch chan<- Metric) { ch <- MustNewConstMetric(c.goInfoDesc, GaugeValue, 1) - ms := &runtime.MemStats{} - runtime.ReadMemStats(ms) - for _, i := range c.metrics { + timer := time.NewTimer(c.msMaxWait) + select { + case <-done: // Our own ReadMemStats succeeded in time. Use it. + timer.Stop() // Important for high collection frequencies to not pile up timers. + c.msCollect(ch, ms) + return + case <-timer.C: // Time out, use last memstats if possible. Continue below. + } + c.msMtx.Lock() + if time.Since(c.msLastTimestamp) < c.msMaxAge { + // Last memstats are recent enough. Collect from them under the lock. + c.msCollect(ch, c.msLast) + c.msMtx.Unlock() + return + } + // If we are here, the last memstats are too old or don't exist. We have + // to wait until our own ReadMemStats finally completes. For that to + // happen, we have to release the lock. + c.msMtx.Unlock() + <-done + c.msCollect(ch, ms) +} + +func (c *goCollector) msCollect(ch chan<- Metric, ms *runtime.MemStats) { + for _, i := range c.msMetrics { ch <- MustNewConstMetric(i.desc, i.valType, i.eval(ms)) } } diff --git a/prometheus/go_collector_test.go b/prometheus/go_collector_test.go index 7ab3968..5a89b25 100644 --- a/prometheus/go_collector_test.go +++ b/prometheus/go_collector_test.go @@ -150,3 +150,88 @@ func TestGoCollectorGC(t *testing.T) { break } } + +func TestGoCollectorMemStats(t *testing.T) { + var ( + c = NewGoCollector().(*goCollector) + got uint64 + ) + + checkCollect := func(want uint64) { + metricCh := make(chan Metric) + endCh := make(chan struct{}) + + go func() { + c.Collect(metricCh) + close(endCh) + }() + Collect: + for { + select { + case metric := <-metricCh: + if metric.Desc().fqName != "go_memstats_alloc_bytes" { + continue Collect + } + pb := &dto.Metric{} + metric.Write(pb) + got = uint64(pb.GetGauge().GetValue()) + case <-endCh: + break Collect + } + } + if want != got { + t.Errorf("unexpected value of go_memstats_alloc_bytes, want %d, got %d", want, got) + } + } + + // Speed up the timing to make the tast faster. + c.msMaxWait = time.Millisecond + c.msMaxAge = 10 * time.Millisecond + + // Scenario 1: msRead responds slowly, no previous memstats available, + // msRead is executed anyway. + c.msRead = func(ms *runtime.MemStats) { + time.Sleep(3 * time.Millisecond) + ms.Alloc = 1 + } + checkCollect(1) + // Now msLast is set. + if want, got := uint64(1), c.msLast.Alloc; want != got { + t.Errorf("unexpected of msLast.Alloc, want %d, got %d", want, got) + } + + // Scenario 2: msRead responds fast, previous memstats available, new + // value collected. + c.msRead = func(ms *runtime.MemStats) { + ms.Alloc = 2 + } + checkCollect(2) + // msLast is set, too. + if want, got := uint64(2), c.msLast.Alloc; want != got { + t.Errorf("unexpected of msLast.Alloc, want %d, got %d", want, got) + } + + // Scenario 3: msRead responds slowly, previous memstats available, old + // value collected. + c.msRead = func(ms *runtime.MemStats) { + time.Sleep(3 * time.Millisecond) + ms.Alloc = 3 + } + checkCollect(2) + // After waiting, new value is still set in msLast. + time.Sleep(12 * time.Millisecond) + if want, got := uint64(3), c.msLast.Alloc; want != got { + t.Errorf("unexpected of msLast.Alloc, want %d, got %d", want, got) + } + + // Scenario 4: msRead responds slowly, previous memstats is too old, new + // value collected. + c.msRead = func(ms *runtime.MemStats) { + time.Sleep(3 * time.Millisecond) + ms.Alloc = 4 + } + checkCollect(4) + if want, got := uint64(4), c.msLast.Alloc; want != got { + t.Errorf("unexpected of msLast.Alloc, want %d, got %d", want, got) + } +} From 886e2ee0c0e46f378032a7846df227113b777949 Mon Sep 17 00:00:00 2001 From: beorn7 Date: Wed, 8 May 2019 13:38:43 +0200 Subject: [PATCH 64/73] Clarify deprecation of `DefObjectives` Previously, the whole `Objectives` field was marked as deprecated by linters. Signed-off-by: beorn7 --- prometheus/summary.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/prometheus/summary.go b/prometheus/summary.go index 1bf3e19..1574b0f 100644 --- a/prometheus/summary.go +++ b/prometheus/summary.go @@ -127,9 +127,10 @@ type SummaryOpts struct { // its zero value (i.e. nil). To create a Summary without Objectives, // set it to an empty map (i.e. map[float64]float64{}). // - // Deprecated: Note that the current value of DefObjectives is - // deprecated. It will be replaced by an empty map in v0.10 of the - // library. Please explicitly set Objectives to the desired value. + // Note that the current value of DefObjectives is deprecated. It will + // be replaced by an empty map in v0.10 of the library. Please + // explicitly set Objectives to the desired value to avoid problems + // during the transition. Objectives map[float64]float64 // MaxAge defines the duration for which an observation stays relevant From facfcc21f8116d00e3a16650a1b15febfca36de5 Mon Sep 17 00:00:00 2001 From: Camilo Viecco Date: Tue, 14 May 2019 11:38:34 -0700 Subject: [PATCH 65/73] fix regression issue 574 Signed-off-by: Camilo Viecco --- prometheus/promhttp/delegator.go | 128 +++++++++++++++++++++++++++++++ 1 file changed, 128 insertions(+) diff --git a/prometheus/promhttp/delegator.go b/prometheus/promhttp/delegator.go index 7f7dd3f..fa53568 100644 --- a/prometheus/promhttp/delegator.go +++ b/prometheus/promhttp/delegator.go @@ -95,6 +95,9 @@ func (d readerFromDelegator) ReadFrom(re io.Reader) (int64, error) { d.written += n return n, err } +func (d pusherDelegator) Push(target string, opts *http.PushOptions) error { + return d.ResponseWriter.(http.Pusher).Push(target, opts) +} var pickDelegator = make([]func(*responseWriterDelegator) delegator, 32) @@ -198,6 +201,131 @@ func init() { http.CloseNotifier }{d, readerFromDelegator{d}, hijackerDelegator{d}, flusherDelegator{d}, closeNotifierDelegator{d}} } + pickDelegator[pusher] = func(d *responseWriterDelegator) delegator { // 16 + return pusherDelegator{d} + } + pickDelegator[pusher+closeNotifier] = func(d *responseWriterDelegator) delegator { // 17 + return struct { + *responseWriterDelegator + http.Pusher + http.CloseNotifier + }{d, pusherDelegator{d}, closeNotifierDelegator{d}} + } + pickDelegator[pusher+flusher] = func(d *responseWriterDelegator) delegator { // 18 + return struct { + *responseWriterDelegator + http.Pusher + http.Flusher + }{d, pusherDelegator{d}, flusherDelegator{d}} + } + pickDelegator[pusher+flusher+closeNotifier] = func(d *responseWriterDelegator) delegator { // 19 + return struct { + *responseWriterDelegator + http.Pusher + http.Flusher + http.CloseNotifier + }{d, pusherDelegator{d}, flusherDelegator{d}, closeNotifierDelegator{d}} + } + pickDelegator[pusher+hijacker] = func(d *responseWriterDelegator) delegator { // 20 + return struct { + *responseWriterDelegator + http.Pusher + http.Hijacker + }{d, pusherDelegator{d}, hijackerDelegator{d}} + } + pickDelegator[pusher+hijacker+closeNotifier] = func(d *responseWriterDelegator) delegator { // 21 + return struct { + *responseWriterDelegator + http.Pusher + http.Hijacker + http.CloseNotifier + }{d, pusherDelegator{d}, hijackerDelegator{d}, closeNotifierDelegator{d}} + } + pickDelegator[pusher+hijacker+flusher] = func(d *responseWriterDelegator) delegator { // 22 + return struct { + *responseWriterDelegator + http.Pusher + http.Hijacker + http.Flusher + }{d, pusherDelegator{d}, hijackerDelegator{d}, flusherDelegator{d}} + } + pickDelegator[pusher+hijacker+flusher+closeNotifier] = func(d *responseWriterDelegator) delegator { //23 + return struct { + *responseWriterDelegator + http.Pusher + http.Hijacker + http.Flusher + http.CloseNotifier + }{d, pusherDelegator{d}, hijackerDelegator{d}, flusherDelegator{d}, closeNotifierDelegator{d}} + } + pickDelegator[pusher+readerFrom] = func(d *responseWriterDelegator) delegator { // 24 + return struct { + *responseWriterDelegator + http.Pusher + io.ReaderFrom + }{d, pusherDelegator{d}, readerFromDelegator{d}} + } + pickDelegator[pusher+readerFrom+closeNotifier] = func(d *responseWriterDelegator) delegator { // 25 + return struct { + *responseWriterDelegator + http.Pusher + io.ReaderFrom + http.CloseNotifier + }{d, pusherDelegator{d}, readerFromDelegator{d}, closeNotifierDelegator{d}} + } + pickDelegator[pusher+readerFrom+flusher] = func(d *responseWriterDelegator) delegator { // 26 + return struct { + *responseWriterDelegator + http.Pusher + io.ReaderFrom + http.Flusher + }{d, pusherDelegator{d}, readerFromDelegator{d}, flusherDelegator{d}} + } + pickDelegator[pusher+readerFrom+flusher+closeNotifier] = func(d *responseWriterDelegator) delegator { // 27 + return struct { + *responseWriterDelegator + http.Pusher + io.ReaderFrom + http.Flusher + http.CloseNotifier + }{d, pusherDelegator{d}, readerFromDelegator{d}, flusherDelegator{d}, closeNotifierDelegator{d}} + } + pickDelegator[pusher+readerFrom+hijacker] = func(d *responseWriterDelegator) delegator { // 28 + return struct { + *responseWriterDelegator + http.Pusher + io.ReaderFrom + http.Hijacker + }{d, pusherDelegator{d}, readerFromDelegator{d}, hijackerDelegator{d}} + } + pickDelegator[pusher+readerFrom+hijacker+closeNotifier] = func(d *responseWriterDelegator) delegator { // 29 + return struct { + *responseWriterDelegator + http.Pusher + io.ReaderFrom + http.Hijacker + http.CloseNotifier + }{d, pusherDelegator{d}, readerFromDelegator{d}, hijackerDelegator{d}, closeNotifierDelegator{d}} + } + pickDelegator[pusher+readerFrom+hijacker+flusher] = func(d *responseWriterDelegator) delegator { // 30 + return struct { + *responseWriterDelegator + http.Pusher + io.ReaderFrom + http.Hijacker + http.Flusher + }{d, pusherDelegator{d}, readerFromDelegator{d}, hijackerDelegator{d}, flusherDelegator{d}} + } + pickDelegator[pusher+readerFrom+hijacker+flusher+closeNotifier] = func(d *responseWriterDelegator) delegator { // 31 + return struct { + *responseWriterDelegator + http.Pusher + io.ReaderFrom + http.Hijacker + http.Flusher + http.CloseNotifier + }{d, pusherDelegator{d}, readerFromDelegator{d}, hijackerDelegator{d}, flusherDelegator{d}, closeNotifierDelegator{d}} + } } func newDelegator(w http.ResponseWriter, observeWriteHeaderFunc func(int)) delegator { From af332dff97e0c0506bc0ad2b1856324241117c23 Mon Sep 17 00:00:00 2001 From: prombot Date: Wed, 15 May 2019 00:00:42 +0000 Subject: [PATCH 66/73] makefile: update Makefile.common with newer version Signed-off-by: prombot --- Makefile.common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.common b/Makefile.common index 4f18ea5..c7f9ea6 100644 --- a/Makefile.common +++ b/Makefile.common @@ -69,7 +69,7 @@ else GO_BUILD_PLATFORM ?= $(GOHOSTOS)-$(GOHOSTARCH) endif -PROMU_VERSION ?= 0.3.0 +PROMU_VERSION ?= 0.4.0 PROMU_URL := https://github.com/prometheus/promu/releases/download/v$(PROMU_VERSION)/promu-$(PROMU_VERSION).$(GO_BUILD_PLATFORM).tar.gz GOLANGCI_LINT := From 709dc4b1de9e19ca39244c1b076822c01a452a2f Mon Sep 17 00:00:00 2001 From: beorn7 Date: Thu, 16 May 2019 10:37:37 +0200 Subject: [PATCH 67/73] Fix race in TestGoCollectorMemStats Related to #573 (but only partially fixes it). Signed-off-by: beorn7 --- prometheus/go_collector_test.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/prometheus/go_collector_test.go b/prometheus/go_collector_test.go index 5a89b25..f55aff9 100644 --- a/prometheus/go_collector_test.go +++ b/prometheus/go_collector_test.go @@ -196,9 +196,11 @@ func TestGoCollectorMemStats(t *testing.T) { } checkCollect(1) // Now msLast is set. + c.msMtx.Lock() if want, got := uint64(1), c.msLast.Alloc; want != got { t.Errorf("unexpected of msLast.Alloc, want %d, got %d", want, got) } + c.msMtx.Unlock() // Scenario 2: msRead responds fast, previous memstats available, new // value collected. @@ -207,9 +209,11 @@ func TestGoCollectorMemStats(t *testing.T) { } checkCollect(2) // msLast is set, too. + c.msMtx.Lock() if want, got := uint64(2), c.msLast.Alloc; want != got { t.Errorf("unexpected of msLast.Alloc, want %d, got %d", want, got) } + c.msMtx.Unlock() // Scenario 3: msRead responds slowly, previous memstats available, old // value collected. @@ -220,9 +224,11 @@ func TestGoCollectorMemStats(t *testing.T) { checkCollect(2) // After waiting, new value is still set in msLast. time.Sleep(12 * time.Millisecond) + c.msMtx.Lock() if want, got := uint64(3), c.msLast.Alloc; want != got { t.Errorf("unexpected of msLast.Alloc, want %d, got %d", want, got) } + c.msMtx.Unlock() // Scenario 4: msRead responds slowly, previous memstats is too old, new // value collected. @@ -231,7 +237,9 @@ func TestGoCollectorMemStats(t *testing.T) { ms.Alloc = 4 } checkCollect(4) + c.msMtx.Lock() if want, got := uint64(4), c.msLast.Alloc; want != got { t.Errorf("unexpected of msLast.Alloc, want %d, got %d", want, got) } + c.msMtx.Unlock() } From e91cd8163290fdc44fc984768703aa33a04ec6db Mon Sep 17 00:00:00 2001 From: beorn7 Date: Thu, 16 May 2019 11:23:48 +0200 Subject: [PATCH 68/73] Remove outdated use-case description from doc comment We stopped advertising binary-wide setting of a label quite a while ago. This doc comment was missed in the cleanup. Signed-off-by: beorn7 --- prometheus/examples_test.go | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/prometheus/examples_test.go b/prometheus/examples_test.go index a382479..91fef3d 100644 --- a/prometheus/examples_test.go +++ b/prometheus/examples_test.go @@ -232,16 +232,11 @@ func ExampleRegister() { // A different (and somewhat tricky) approach is to use // ConstLabels. ConstLabels are pairs of label names and label values - // that never change. You might ask what those labels are good for (and - // rightfully so - if they never change, they could as well be part of - // the metric name). There are essentially two use-cases: The first is - // if labels are constant throughout the lifetime of a binary execution, - // but they vary over time or between different instances of a running - // binary. The second is what we have here: Each worker creates and - // registers an own Counter instance where the only difference is in the - // value of the ConstLabels. Those Counters can all be registered - // because the different ConstLabel values guarantee that each worker - // will increment a different Counter metric. + // that never change. Each worker creates and registers an own Counter + // instance where the only difference is in the value of the + // ConstLabels. Those Counters can all be registered because the + // different ConstLabel values guarantee that each worker will increment + // a different Counter metric. counterOpts := prometheus.CounterOpts{ Subsystem: "worker_pool", Name: "completed_tasks", From 6c520f6aca8423c1c8c73f3b4a967bd1f9969f7b Mon Sep 17 00:00:00 2001 From: beorn7 Date: Thu, 16 May 2019 17:46:32 +0200 Subject: [PATCH 69/73] Add test to expose #580 Tests are heavily inspired by @shturec, see #584. Signed-off-by: beorn7 --- prometheus/promhttp/instrument_client_test.go | 95 ++++++++++++++++++- 1 file changed, 93 insertions(+), 2 deletions(-) diff --git a/prometheus/promhttp/instrument_client_test.go b/prometheus/promhttp/instrument_client_test.go index 59c0a2f..50d64bd 100644 --- a/prometheus/promhttp/instrument_client_test.go +++ b/prometheus/promhttp/instrument_client_test.go @@ -14,15 +14,18 @@ package promhttp import ( + "context" + "fmt" "log" "net/http" + "net/http/httptest" "testing" "time" "github.com/prometheus/client_golang/prometheus" ) -func TestClientMiddlewareAPI(t *testing.T) { +func makeInstrumentedClient() (*http.Client, *prometheus.Registry) { client := http.DefaultClient client.Timeout = 1 * time.Second @@ -92,12 +95,100 @@ func TestClientMiddlewareAPI(t *testing.T) { ), ), ) + return client, reg +} - resp, err := client.Get("http://google.com") +func TestClientMiddlewareAPI(t *testing.T) { + client, reg := makeInstrumentedClient() + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + })) + defer backend.Close() + + resp, err := client.Get(backend.URL) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + + mfs, err := reg.Gather() + if err != nil { + t.Fatal(err) + } + if want, got := 3, len(mfs); want != got { + t.Fatalf("unexpected number of metric families gathered, want %d, got %d", want, got) + } + for _, mf := range mfs { + if len(mf.Metric) == 0 { + t.Errorf("metric family %s must not be empty", mf.GetName()) + } + } +} + +func TestClientMiddlewareAPIWithRequestContext(t *testing.T) { + client, reg := makeInstrumentedClient() + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + })) + defer backend.Close() + + req, err := http.NewRequest("GET", backend.URL, nil) if err != nil { t.Fatalf("%v", err) } + + // Set a context with a long timeout. + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + req = req.WithContext(ctx) + + resp, err := client.Do(req) + if err != nil { + t.Fatal(err) + } defer resp.Body.Close() + + mfs, err := reg.Gather() + if err != nil { + t.Fatal(err) + } + if want, got := 3, len(mfs); want != got { + t.Fatalf("unexpected number of metric families gathered, want %d, got %d", want, got) + } + for _, mf := range mfs { + if len(mf.Metric) == 0 { + t.Errorf("metric family %s must not be empty", mf.GetName()) + } + } +} + +func TestClientMiddlewareAPIWithRequestContextTimeout(t *testing.T) { + client, _ := makeInstrumentedClient() + + // Slow testserver responding in 100ms. + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + time.Sleep(100 * time.Millisecond) + w.WriteHeader(http.StatusOK) + })) + defer backend.Close() + + req, err := http.NewRequest("GET", backend.URL, nil) + if err != nil { + t.Fatalf("%v", err) + } + + // Set a context with a short timeout. + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond) + defer cancel() + req = req.WithContext(ctx) + + _, err = client.Do(req) + if err == nil { + t.Fatal("did not get timeout error") + } + if want, got := fmt.Sprintf("Get %s: context deadline exceeded", backend.URL), err.Error(); want != got { + t.Fatalf("want error %q, got %q", want, got) + } } func ExampleInstrumentRoundTripperDuration() { From bc54582c5ec9f1b8e0383eaa8628db3bca88f041 Mon Sep 17 00:00:00 2001 From: beorn7 Date: Thu, 16 May 2019 11:03:34 +0200 Subject: [PATCH 70/73] Make use of pre-existing context in InstrumentRoundTripperTrace Fixes #580 Signed-off-by: beorn7 --- prometheus/promhttp/instrument_client.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/prometheus/promhttp/instrument_client.go b/prometheus/promhttp/instrument_client.go index d5172db..83c49b6 100644 --- a/prometheus/promhttp/instrument_client.go +++ b/prometheus/promhttp/instrument_client.go @@ -14,7 +14,6 @@ package promhttp import ( - "context" "crypto/tls" "net/http" "net/http/httptrace" @@ -213,7 +212,7 @@ func InstrumentRoundTripperTrace(it *InstrumentTrace, next http.RoundTripper) Ro } }, } - r = r.WithContext(httptrace.WithClientTrace(context.Background(), trace)) + r = r.WithContext(httptrace.WithClientTrace(r.Context(), trace)) return next.RoundTrip(r) }) From 9a6fc61a7b0b8195152a981e26e5cd68a91aa442 Mon Sep 17 00:00:00 2001 From: beorn7 Date: Wed, 15 May 2019 23:21:43 +0200 Subject: [PATCH 71/73] Update versioning plan with Go Modules in mind Signed-off-by: beorn7 --- README.md | 72 +++++++++++++++++++++++-------------------------------- 1 file changed, 30 insertions(+), 42 deletions(-) diff --git a/README.md b/README.md index e2b464f..2336eb1 100644 --- a/README.md +++ b/README.md @@ -13,51 +13,39 @@ __This library requires Go1.9 or later.__ ## Important note about releases, versioning, tagging, and stability -While our goal is to follow [Semantic Versioning](https://semver.org/), this -repository is still pre-1.0.0. To quote the -[Semantic Versioning spec](https://semver.org/#spec-item-4): “Anything may -change at any time. The public API should not be considered stable.” We know -that this is at odds with the widespread use of this library. However, just -declaring something 1.0.0 doesn't make it 1.0.0. Instead, we are working -towards a 1.0.0 release that actually deserves its major version number. +In this repository, we used to mostly ignore the many coming and going +dependency management tools for Go and instead wait for a tool that most of the +community would converge on. Our bet is that this tool has arrived now in the +form of [Go +Modules](https://github.com/golang/go/wiki/Modules#how-to-upgrade-and-downgrade-dependencies). -Having said that, we aim for always keeping the tip of master in a workable -state. We occasionally tag versions and track their changes in CHANGELOG.md, -but this happens mostly to keep dependency management tools happy and to give -people a handle they can talk about easily. In particular, all commits in the -master branch have passed the same testing and reviewing. There is no QA -process in place that would render tagged commits more stable or better tested -than others. +To make full use of what Go Modules are offering, the previous versioning +roadmap for this repository had to be changed. In particular, Go Modules +finally provide a way for incompatible versions of the same package to coexist +in the same binary. For that, however, the versions must be tagged with +different major versions of 1 or greater (following [Semantic +Versioning](https://semver.org/)). Thus, we decided to abandon the original +plan of introducing a lot of breaking changes _before_ releasing v1 of this +repository, mostly driven by the widespread use this repository already has and +the relatively stable state it is in. -There is a plan behind the current (pre-1.0.0) versioning, though: +To leverage the mechanism Go Modules offers for a transition between major +version, the current plan is the following: -- v0.9 is the “production release”, currently tracked in the master - branch. “Patch” releases will usually be just bug fixes, indeed, but - important new features that do not require invasive code changes might also - be included in those. We do not plan any breaking changes from one v0.9.x - release to any later v0.9.y release, but nothing is guaranteed. Since the - master branch will eventually be switched over to track the upcoming v0.10 - (see below), we recommend to tell your dependency management tool of choice - to use the latest v0.9.x release, at least for your production software. In - that way, you should get bug fixes and non-invasive, low-risk new features - without the need to change anything on your part. -- v0.10 is a planned release that will have a _lot_ of breaking changes - (despite being only a “minor” release in the Semantic Versioning terminology, - but as said, pre-1.0.0 means nothing is guaranteed). Essentially, we have - been piling up feature requests that require breaking changes for a while, - and they are all collected in the - [v0.10 milestone](https://github.com/prometheus/client_golang/milestone/2). - Since there will be so many breaking changes, the development for v0.10 is - currently not happening in the master branch, but in the - [dev-0.10 branch](https://github.com/prometheus/client_golang/tree/dev-0.10). - It will violently change for a while, and it will definitely be in a - non-working state now and then. It should only be used for sneak-peaks and - discussions of the new features and designs. -- Once v0.10 is ready for real-life use, it will be merged into the master - branch (which is the reason why you should lock your dependency management - tool to v0.9.x and only migrate to v0.10 when both you and v0.10 are ready - for it). In the ideal case, v0.10 will be the basis for the future v1.0 - release, but we cannot provide an ETA at this time. +- The v0.9.x series of releases will see a small number of bugfix releases to + deal with a few remaining minor issues (#543, #542, #539). +- After that, all features currently marked as _deprecated_ will be removed, + and the result will be released as v1.0.0. +- The planned breaking changes previously gathered as part of the v0.10 + milestone will now go into the v2 milestone. The v2 development happens in a + [separate branch](https://github.com/prometheus/client_golang/tree/dev-v2) + for the time being. v2 releases off that branch will happen once sufficient + stability is reached. v1 and v2 will coexist for a while to enable a + convenient transition. +- The API client in prometheus/client_golang/api/… is still considered + experimental. While it will be tagged alongside the rest of the code + according to the plan above, we cannot strictly guarantee semver semantics + for it. ## Instrumenting applications From d586b4b2d5a9972e498638be3146aa6296af0bec Mon Sep 17 00:00:00 2001 From: beorn7 Date: Wed, 15 May 2019 23:42:52 +0200 Subject: [PATCH 72/73] Update dependencies Signed-off-by: beorn7 --- go.mod | 5 +++-- go.sum | 10 ++++++---- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 2168f76..e2c8b8b 100644 --- a/go.mod +++ b/go.mod @@ -2,9 +2,10 @@ module github.com/prometheus/client_golang require ( github.com/beorn7/perks v1.0.0 + github.com/go-logfmt/logfmt v0.4.0 // indirect github.com/golang/protobuf v1.3.1 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 - github.com/prometheus/common v0.3.0 - github.com/prometheus/procfs v0.0.0-20190412120340-e22ddced7142 + github.com/prometheus/common v0.4.0 + github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084 github.com/prometheus/tsdb v0.7.1 ) diff --git a/go.sum b/go.sum index d998271..00c60f6 100644 --- a/go.sum +++ b/go.sum @@ -12,6 +12,8 @@ github.com/go-kit/kit v0.8.0 h1:Wz+5lgoB0kkuqLEc6NVmwRknTKP6dTGbSqvhZtBI/j0= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0 h1:8HUsc87TaSWLKwrnumgC8/YconD2fJQsRJAsWaPg2ic= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0 h1:MP4Eh7ZCb31lleYCFuwm0oe4/YGak+5l1vA2NOE80nA= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -35,11 +37,11 @@ github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1: github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.3.0 h1:taZ4h8Tkxv2kNyoSctBvfXEHmBmxrwmIidZTIaHons4= -github.com/prometheus/common v0.3.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.4.0 h1:7etb9YClo3a6HjLzfl6rIQaU+FDfi0VSX39io3aQ+DM= +github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190412120340-e22ddced7142 h1:JO6VBMEDSBX/LT4GKwSdvuFigZNwVD4lkPyUE4BDCKE= -github.com/prometheus/procfs v0.0.0-20190412120340-e22ddced7142/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084 h1:sofwID9zm4tzrgykg80hfFph1mryUeLRsUfoocVVmRY= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/tsdb v0.7.1 h1:YZcsG11NqnK4czYLrWd9mpEuAJIHVQLwdrleYfszMAA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= From d980b1cf3b095c9d346faa936b13fb914611a86c Mon Sep 17 00:00:00 2001 From: beorn7 Date: Wed, 15 May 2019 23:41:43 +0200 Subject: [PATCH 73/73] Cut v0.9.3 Signed-off-by: beorn7 --- CHANGELOG.md | 21 +++++++++++++++++++++ VERSION | 2 +- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fcfa112..e5e33f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,24 @@ +## 0.9.3 / 2019-05-16 +* [CHANGE] Required Go version is now 1.9+. #561 +* [FEATURE] API client: Add POST with get fallback for Query/QueryRange. #557 +* [FEATURE] API client: Add alerts endpoint. #552 +* [FEATURE] API client: Add rules endpoint. #508 +* [FEATURE] push: Add option to pick metrics format. #540 +* [ENHANCEMENT] Limit time the Go collector may take to collect memstats, + returning results from the previous collection in case of a timeout. #568 +* [ENHANCEMENT] Pusher now requires only a thin interface instead of a full + `http.Client`, facilitating mocking and custom HTTP client implementation. + #559 +* [ENHANCEMENT] Memory usage improvement for histograms and summaries without + objectives. #536 +* [ENHANCEMENT] Summaries without objectives are now lock-free. #521 +* [BUGFIX] promhttp: `InstrumentRoundTripperTrace` now takes into account a pre-set context. #582 +* [BUGFIX] `TestCounterAddLarge` now works on all platforms. #567 +* [BUGFIX] Fix `promhttp` examples. #535 #544 +* [BUGFIX] API client: Wait for done before writing to shared response + body. #532 +* [BUGFIX] API client: Deal with discovered labels properly. #529 + ## 0.9.2 / 2018-12-06 * [FEATURE] Support for Go modules. #501 * [FEATURE] `Timer.ObserveDuration` returns observed duration. #509 diff --git a/VERSION b/VERSION index 2003b63..965065d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.9.2 +0.9.3