Merge branch 'main' into sparsehistogram

.go-version

@@ -0,0 +1 @@
+1.18

.golangci.yml

@@ -8,10 +8,26 @@ output:
 
 linters:
   enable:
+    - deadcode
+    - depguard
+    - durationcheck
+    - errorlint
+    - exportloopref
+    - gofmt
     - gofumpt
     - goimports
-    - revive
+    - gosimple
+    - ineffassign
     - misspell
+    - nolintlint
+    - predeclared
+    - revive
+    - staticcheck
+    - structcheck
+    - unconvert
+    - unused
+    - varcheck
+    - wastedassign
 
 issues:
   max-same-issues: 0

CHANGELOG.md

@@ -1,10 +1,26 @@
 ## Unreleased
 
-* [CHANGE] Minimum required Go version is now 1.16.
+## 1.13.0 / 2022-08-05
+
+* [CHANGE] Minimum required Go version is now 1.17 (we also test client_golang against new 1.19 version).
+* [ENHANCEMENT] Added `prometheus.TransactionalGatherer` interface for `promhttp.Handler` use which allows using low allocation update techniques for custom collectors. #989
+* [ENHANCEMENT] Added exemplar support to `prometheus.NewConstHistogram`. See [`ExampleNewConstHistogram_WithExemplar`](prometheus/examples_test.go#L602) example on how to use it. #986
+* [ENHANCEMENT] `prometheus/push.Pusher` has now context aware methods that pass context to HTTP request. #1028
+* [ENHANCEMENT] `prometheus/push.Pusher` has now `Error` method that retrieve last error. #1075
+* [ENHANCEMENT] `testutil.GatherAndCompare` provides now readable diff on failed comparisons. #998
+* [ENHANCEMENT] Query API now supports timeouts. #1014
+* [ENHANCEMENT] New `MetricVec` method `DeletePartialMatch(labels Labels)` for deleting all metrics that match provided labels. #1013
+* [ENHANCEMENT] `api.Config` now accepts passing custom `*http.Client`. #1025
+* [BUGFIX] Raise exemplar labels limit from 64 to 128 bytes as specified in OpenMetrics spec. #1091
+* [BUGFIX] Allow adding exemplar to +Inf bucket to const histograms. #1094
+* [ENHANCEMENT] Most `promhttp.Instrument*` middlewares now supports adding exemplars to metrics. This allows hooking those to your tracing middleware that retrieves trace ID and put it in exemplar if present. #1055
+* [ENHANCEMENT] Added `testutil.ScrapeAndCompare` method. #1043
+* [BUGFIX] Fixed `GopherJS` build support. #897
+* [ENHANCEMENT] :warning: Added way to specify what `runtime/metrics`  `collectors.NewGoCollector` should use. See [`ExampleGoCollector_WithAdvancedGoMetrics`](prometheus/collectors/go_collector_latest_test.go#L263). #1102
 
 ## 1.12.2 / 2022-05-13
 
-* [CHANGE] Added `collectors.WithGoCollections` that allows to choose what collection of Go runtime metrics user wants: Equivalent of [`MemStats` structure](https://pkg.go.dev/runtime#MemStats) configured using `GoRuntimeMemStatsCollection`, new based on dedicated [runtime/metrics](https://pkg.go.dev/runtime/metrics) metrics represented by `GoRuntimeMetricsCollection` option, or both by specifying `GoRuntimeMemStatsCollection | GoRuntimeMetricsCollection` flag.
+* [CHANGE] Added `collectors.WithGoCollections` that allows to choose what collection of Go runtime metrics user wants: Equivalent of [`MemStats` structure](https://pkg.go.dev/runtime#MemStats) configured using `GoRuntimeMemStatsCollection`, new based on dedicated [runtime/metrics](https://pkg.go.dev/runtime/metrics) metrics represented by `GoRuntimeMetricsCollection` option, or both by specifying `GoRuntimeMemStatsCollection | GoRuntimeMetricsCollection` flag. #1031
 * [CHANGE] :warning: Change in `collectors.NewGoCollector` metrics: Reverting addition of new ~80 runtime metrics by default. You can enable this back with `GoRuntimeMetricsCollection` option or `GoRuntimeMemStatsCollection | GoRuntimeMetricsCollection` for smooth transition.
 * [BUGFIX] Fixed the bug that causes generated histogram metric names to end with `_total`. ⚠️ This changes 3 metric names in the new Go collector that was reverted from default in this release.
   * `go_gc_heap_allocs_by_size_bytes_total` -> `go_gc_heap_allocs_by_size_bytes`,

Makefile

@@ -18,3 +18,11 @@ test: deps common-test
 
 .PHONY: test-short
 test-short: deps common-test-short
+
+.PHONY: generate-go-collector-test-files
+VERSIONS := 1.17 1.18 1.19
+generate-go-collector-test-files:
+	for GO_VERSION in $(VERSIONS); do \
+    	docker run --rm -v $(PWD):/workspace -w /workspace golang:$$GO_VERSION go run prometheus/gen_go_collector_metrics_set.go; \
+		mv -f go_collector_metrics* prometheus; \
+    done

VERSION

@@ -1 +1 @@
-1.12.2
+1.13.0

api/client.go

@@ -109,7 +109,7 @@ func (c *httpClient) URL(ep string, args map[string]string) *url.URL {
 
 	for arg, val := range args {
 		arg = ":" + arg
-		p = strings.Replace(p, arg, val, -1)
+		p = strings.ReplaceAll(p, arg, val)
 	}
 
 	u := *c.endpoint

api/prometheus/v1/api.go

@@ -856,7 +856,7 @@ func (h *httpAPI) Query(ctx context.Context, query string, ts time.Time, opts ..
 	}
 
 	var qres queryResult
-	return model.Value(qres.v), warnings, json.Unmarshal(body, &qres)
+	return qres.v, warnings, json.Unmarshal(body, &qres)
 }
 
 func (h *httpAPI) QueryRange(ctx context.Context, query string, r Range, opts ...Option) (model.Value, Warnings, error) {
@@ -885,7 +885,7 @@ func (h *httpAPI) QueryRange(ctx context.Context, query string, r Range, opts ..
 
 	var qres queryResult
 
-	return model.Value(qres.v), warnings, json.Unmarshal(body, &qres)
+	return qres.v, warnings, json.Unmarshal(body, &qres)
 }
 
 func (h *httpAPI) Series(ctx context.Context, matches []string, startTime, endTime time.Time) ([]model.LabelSet, Warnings, error) {

api/prometheus/v1/api_test.go

@@ -17,7 +17,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"math"
 	"net/http"
 	"net/http/httptest"
@@ -40,10 +40,8 @@ type apiTest struct {
 	inRes        interface{}
 
 	reqPath   string
-	reqParam  url.Values
 	reqMethod string
 	res       interface{}
-	warnings  Warnings
 	err       error
 }
 
@@ -55,7 +53,7 @@ type apiTestClient struct {
 func (c *apiTestClient) URL(ep string, args map[string]string) *url.URL {
 	path := ep
 	for k, v := range args {
-		path = strings.Replace(path, ":"+k, v, -1)
+		path = strings.ReplaceAll(path, ":"+k, v)
 	}
 	u := &url.URL{
 		Host: "test:9090",
@@ -64,7 +62,7 @@ func (c *apiTestClient) URL(ep string, args map[string]string) *url.URL {
 	return u
 }
 
-func (c *apiTestClient) Do(ctx context.Context, req *http.Request) (*http.Response, []byte, Warnings, error) {
+func (c *apiTestClient) Do(_ context.Context, req *http.Request) (*http.Response, []byte, Warnings, error) {
 	test := c.curTest
 
 	if req.URL.Path != test.reqPath {
@@ -156,15 +154,15 @@ func TestAPIs(t *testing.T) {
 		}
 	}
 
-	doLabelNames := func(matches []string) func() (interface{}, Warnings, error) {
+	doLabelNames := func(matches []string, startTime, endTime time.Time) func() (interface{}, Warnings, error) {
 		return func() (interface{}, Warnings, error) {
-			return promAPI.LabelNames(context.Background(), matches, time.Now().Add(-100*time.Hour), time.Now())
+			return promAPI.LabelNames(context.Background(), matches, startTime, endTime)
 		}
 	}
 
-	doLabelValues := func(matches []string, label string) func() (interface{}, Warnings, error) {
+	doLabelValues := func(matches []string, label string, startTime, endTime time.Time) func() (interface{}, Warnings, error) {
 		return func() (interface{}, Warnings, error) {
-			return promAPI.LabelValues(context.Background(), label, matches, time.Now().Add(-100*time.Hour), time.Now())
+			return promAPI.LabelValues(context.Background(), label, matches, startTime, endTime)
 		}
 	}
 
@@ -255,11 +253,6 @@ func TestAPIs(t *testing.T) {
 
 			reqMethod: "POST",
 			reqPath:   "/api/v1/query",
-			reqParam: url.Values{
-				"query":   []string{"2"},
-				"time":    []string{testTime.Format(time.RFC3339Nano)},
-				"timeout": []string{(5 * time.Second).String()},
-			},
 			res: &model.Scalar{
 				Value:     2,
 				Timestamp: model.TimeFromUnix(testTime.Unix()),
@@ -271,11 +264,7 @@ func TestAPIs(t *testing.T) {
 
 			reqMethod: "POST",
 			reqPath:   "/api/v1/query",
-			reqParam: url.Values{
-				"query": []string{"2"},
-				"time":  []string{testTime.Format(time.RFC3339Nano)},
-			},
-			err: fmt.Errorf("some error"),
+			err:       errors.New("some error"),
 		},
 		{
 			do:           doQuery("2", testTime),
@@ -289,10 +278,6 @@ func TestAPIs(t *testing.T) {
 
 			reqMethod: "POST",
 			reqPath:   "/api/v1/query",
-			reqParam: url.Values{
-				"query": []string{"2"},
-				"time":  []string{testTime.Format(time.RFC3339Nano)},
-			},
 			err:       errors.New("server_error: server error: 500"),
 		},
 		{
@@ -307,10 +292,6 @@ func TestAPIs(t *testing.T) {
 
 			reqMethod: "POST",
 			reqPath:   "/api/v1/query",
-			reqParam: url.Values{
-				"query": []string{"2"},
-				"time":  []string{testTime.Format(time.RFC3339Nano)},
-			},
 			err:       errors.New("client_error: client error: 404"),
 		},
 		// Warning only.
@@ -327,15 +308,10 @@ func TestAPIs(t *testing.T) {
 
 			reqMethod: "POST",
 			reqPath:   "/api/v1/query",
-			reqParam: url.Values{
-				"query": []string{"2"},
-				"time":  []string{testTime.Format(time.RFC3339Nano)},
-			},
 			res: &model.Scalar{
 				Value:     2,
 				Timestamp: model.TimeFromUnix(testTime.Unix()),
 			},
-			warnings: []string{"warning"},
 		},
 		// Warning + error.
 		{
@@ -351,115 +327,97 @@ func TestAPIs(t *testing.T) {
 
 			reqMethod: "POST",
 			reqPath:   "/api/v1/query",
-			reqParam: url.Values{
-				"query": []string{"2"},
-				"time":  []string{testTime.Format(time.RFC3339Nano)},
-			},
 			err:       errors.New("client_error: client error: 404"),
-			warnings: []string{"warning"},
 		},
 
 		{
 			do: doQueryRange("2", Range{
 				Start: testTime.Add(-time.Minute),
 				End:   testTime,
-				Step:  time.Minute,
+				Step:  1 * time.Minute,
 			}, WithTimeout(5*time.Second)),
 			inErr: fmt.Errorf("some error"),
 
 			reqMethod: "POST",
 			reqPath:   "/api/v1/query_range",
-			reqParam: url.Values{
-				"query":   []string{"2"},
-				"start":   []string{testTime.Add(-time.Minute).Format(time.RFC3339Nano)},
-				"end":     []string{testTime.Format(time.RFC3339Nano)},
-				"step":    []string{time.Minute.String()},
-				"timeout": []string{(5 * time.Second).String()},
-			},
-			err: fmt.Errorf("some error"),
+			err:       errors.New("some error"),
 		},
 
 		{
-			do:        doLabelNames(nil),
+			do:        doLabelNames(nil, testTime.Add(-100*time.Hour), testTime),
 			inRes:     []string{"val1", "val2"},
 			reqMethod: "GET",
 			reqPath:   "/api/v1/labels",
 			res:       []string{"val1", "val2"},
 		},
 		{
-			do:         doLabelNames(nil),
+			do:         doLabelNames(nil, testTime.Add(-100*time.Hour), testTime),
 			inRes:      []string{"val1", "val2"},
 			inWarnings: []string{"a"},
 			reqMethod:  "GET",
 			reqPath:    "/api/v1/labels",
 			res:        []string{"val1", "val2"},
-			warnings:   []string{"a"},
 		},
 
 		{
-			do:        doLabelNames(nil),
+			do:        doLabelNames(nil, testTime.Add(-100*time.Hour), testTime),
 			inErr:     fmt.Errorf("some error"),
 			reqMethod: "GET",
 			reqPath:   "/api/v1/labels",
-			err:       fmt.Errorf("some error"),
+			err:       errors.New("some error"),
 		},
 		{
-			do:         doLabelNames(nil),
+			do:         doLabelNames(nil, testTime.Add(-100*time.Hour), testTime),
 			inErr:      fmt.Errorf("some error"),
 			inWarnings: []string{"a"},
 			reqMethod:  "GET",
 			reqPath:    "/api/v1/labels",
-			err:        fmt.Errorf("some error"),
-			warnings:   []string{"a"},
+			err:        errors.New("some error"),
 		},
 		{
-			do:        doLabelNames([]string{"up"}),
+			do:        doLabelNames([]string{"up"}, testTime.Add(-100*time.Hour), testTime),
 			inRes:     []string{"val1", "val2"},
 			reqMethod: "GET",
 			reqPath:   "/api/v1/labels",
-			reqParam:  url.Values{"match[]": {"up"}},
 			res:       []string{"val1", "val2"},
 		},
 
 		{
-			do:        doLabelValues(nil, "mylabel"),
+			do:        doLabelValues(nil, "mylabel", testTime.Add(-100*time.Hour), testTime),
 			inRes:     []string{"val1", "val2"},
 			reqMethod: "GET",
 			reqPath:   "/api/v1/label/mylabel/values",
 			res:       model.LabelValues{"val1", "val2"},
 		},
 		{
-			do:         doLabelValues(nil, "mylabel"),
+			do:         doLabelValues(nil, "mylabel", testTime.Add(-100*time.Hour), testTime),
 			inRes:      []string{"val1", "val2"},
 			inWarnings: []string{"a"},
 			reqMethod:  "GET",
 			reqPath:    "/api/v1/label/mylabel/values",
 			res:        model.LabelValues{"val1", "val2"},
-			warnings:   []string{"a"},
 		},
 
 		{
-			do:        doLabelValues(nil, "mylabel"),
+			do:        doLabelValues(nil, "mylabel", testTime.Add(-100*time.Hour), testTime),
 			inErr:     fmt.Errorf("some error"),
 			reqMethod: "GET",
 			reqPath:   "/api/v1/label/mylabel/values",
-			err:       fmt.Errorf("some error"),
+			err:       errors.New("some error"),
 		},
 		{
-			do:         doLabelValues(nil, "mylabel"),
+			do:         doLabelValues(nil, "mylabel", testTime.Add(-100*time.Hour), testTime),
 			inErr:      fmt.Errorf("some error"),
 			inWarnings: []string{"a"},
 			reqMethod:  "GET",
 			reqPath:    "/api/v1/label/mylabel/values",
-			err:        fmt.Errorf("some error"),
-			warnings:   []string{"a"},
+			err:        errors.New("some error"),
 		},
 		{
-			do:        doLabelValues([]string{"up"}, "mylabel"),
+			do:        doLabelValues([]string{"up"}, "mylabel", testTime.Add(-100*time.Hour), testTime),
 			inRes:     []string{"val1", "val2"},
 			reqMethod: "GET",
 			reqPath:   "/api/v1/label/mylabel/values",
-			reqParam:  url.Values{"match[]": {"up"}},
 			res:       model.LabelValues{"val1", "val2"},
 		},
 
@@ -474,11 +432,6 @@ func TestAPIs(t *testing.T) {
 			},
 			reqMethod: "GET",
 			reqPath:   "/api/v1/series",
-			reqParam: url.Values{
-				"match": []string{"up"},
-				"start": []string{testTime.Add(-time.Minute).Format(time.RFC3339Nano)},
-				"end":   []string{testTime.Format(time.RFC3339Nano)},
-			},
 			res: []model.LabelSet{
 				{
 					"__name__": "up",
@@ -500,11 +453,6 @@ func TestAPIs(t *testing.T) {
 			inWarnings: []string{"a"},
 			reqMethod:  "GET",
 			reqPath:    "/api/v1/series",
-			reqParam: url.Values{
-				"match": []string{"up"},
-				"start": []string{testTime.Add(-time.Minute).Format(time.RFC3339Nano)},
-				"end":   []string{testTime.Format(time.RFC3339Nano)},
-			},
 			res: []model.LabelSet{
 				{
 					"__name__": "up",
@@ -512,7 +460,6 @@ func TestAPIs(t *testing.T) {
 					"instance": "localhost:9090",
 				},
 			},
-			warnings: []string{"a"},
 		},
 
 		{
@@ -520,12 +467,7 @@ func TestAPIs(t *testing.T) {
 			inErr:     fmt.Errorf("some error"),
 			reqMethod: "GET",
 			reqPath:   "/api/v1/series",
-			reqParam: url.Values{
-				"match": []string{"up"},
-				"start": []string{testTime.Add(-time.Minute).Format(time.RFC3339Nano)},
-				"end":   []string{testTime.Format(time.RFC3339Nano)},
-			},
-			err: fmt.Errorf("some error"),
+			err:       errors.New("some error"),
 		},
 		// Series with error and warning.
 		{
@@ -534,13 +476,7 @@ func TestAPIs(t *testing.T) {
 			inWarnings: []string{"a"},
 			reqMethod:  "GET",
 			reqPath:    "/api/v1/series",
-			reqParam: url.Values{
-				"match": []string{"up"},
-				"start": []string{testTime.Add(-time.Minute).Format(time.RFC3339Nano)},
-				"end":   []string{testTime.Format(time.RFC3339Nano)},
-			},
-			err:      fmt.Errorf("some error"),
-			warnings: []string{"a"},
+			err:        errors.New("some error"),
 		},
 
 		{
@@ -550,9 +486,6 @@ func TestAPIs(t *testing.T) {
 			},
 			reqMethod: "POST",
 			reqPath:   "/api/v1/admin/tsdb/snapshot",
-			reqParam: url.Values{
-				"skip_head": []string{"true"},
-			},
 			res: SnapshotResult{
 				Name: "20171210T211224Z-2be650b6d019eb54",
 			},
@@ -563,7 +496,7 @@ func TestAPIs(t *testing.T) {
 			inErr:     fmt.Errorf("some error"),
 			reqMethod: "POST",
 			reqPath:   "/api/v1/admin/tsdb/snapshot",
-			err:       fmt.Errorf("some error"),
+			err:       errors.New("some error"),
 		},
 
 		{
@@ -577,7 +510,7 @@ func TestAPIs(t *testing.T) {
 			inErr:     fmt.Errorf("some error"),
 			reqMethod: "POST",
 			reqPath:   "/api/v1/admin/tsdb/clean_tombstones",
-			err:       fmt.Errorf("some error"),
+			err:       errors.New("some error"),
 		},
 
 		{
@@ -591,11 +524,6 @@ func TestAPIs(t *testing.T) {
 			},
 			reqMethod: "POST",
 			reqPath:   "/api/v1/admin/tsdb/delete_series",
-			reqParam: url.Values{
-				"match": []string{"up"},
-				"start": []string{testTime.Add(-time.Minute).Format(time.RFC3339Nano)},
-				"end":   []string{testTime.Format(time.RFC3339Nano)},
-			},
 		},
 
 		{
@@ -603,12 +531,7 @@ func TestAPIs(t *testing.T) {
 			inErr:     fmt.Errorf("some error"),
 			reqMethod: "POST",
 			reqPath:   "/api/v1/admin/tsdb/delete_series",
-			reqParam: url.Values{
-				"match": []string{"up"},
-				"start": []string{testTime.Add(-time.Minute).Format(time.RFC3339Nano)},
-				"end":   []string{testTime.Format(time.RFC3339Nano)},
-			},
-			err: fmt.Errorf("some error"),
+			err:       errors.New("some error"),
 		},
 
 		{
@@ -1067,11 +990,6 @@ func TestAPIs(t *testing.T) {
 			},
 			reqMethod: "GET",
 			reqPath:   "/api/v1/targets/metadata",
-			reqParam: url.Values{
-				"match_target": []string{"{job=\"prometheus\"}"},
-				"metric":       []string{"go_goroutines"},
-				"limit":        []string{"1"},
-			},
 			res: []MetricMetadata{
 				{
 					Target: map[string]string{
@@ -1090,12 +1008,7 @@ func TestAPIs(t *testing.T) {
 			inErr:     fmt.Errorf("some error"),
 			reqMethod: "GET",
 			reqPath:   "/api/v1/targets/metadata",
-			reqParam: url.Values{
-				"match_target": []string{"{job=\"prometheus\"}"},
-				"metric":       []string{"go_goroutines"},
-				"limit":        []string{"1"},
-			},
-			err: fmt.Errorf("some error"),
+			err:       errors.New("some error"),
 		},
 
 		{
@@ -1111,10 +1024,6 @@ func TestAPIs(t *testing.T) {
 			},
 			reqMethod: "GET",
 			reqPath:   "/api/v1/metadata",
-			reqParam: url.Values{
-				"metric": []string{"go_goroutines"},
-				"limit":  []string{"1"},
-			},
 			res: map[string][]Metadata{
 				"go_goroutines": {
 					{
@@ -1131,11 +1040,7 @@ func TestAPIs(t *testing.T) {
 			inErr:     fmt.Errorf("some error"),
 			reqMethod: "GET",
 			reqPath:   "/api/v1/metadata",
-			reqParam: url.Values{
-				"metric": []string{""},
-				"limit":  []string{"1"},
-			},
-			err: fmt.Errorf("some error"),
+			err:       errors.New("some error"),
 		},
 
 		{
@@ -1246,8 +1151,8 @@ func TestAPIs(t *testing.T) {
 			do:        doQueryExemplars("tns_request_duration_seconds_bucket", testTime.Add(-1*time.Minute), testTime),
 			reqMethod: "GET",
 			reqPath:   "/api/v1/query_exemplars",
-			inErr:     fmt.Errorf("some error"),
-			err:       fmt.Errorf("some error"),
+			inErr:     errors.New("some error"),
+			err:       errors.New("some error"),
 		},
 
 		{
@@ -1323,7 +1228,9 @@ func TestAPIs(t *testing.T) {
 				if err.Error() != test.err.Error() {
 					t.Errorf("unexpected error: want %s, got %s", test.err, err)
 				}
-				if apiErr, ok := err.(*Error); ok {
+
+				apiErr := &Error{}
+				if ok := errors.As(err, &apiErr); ok {
 					if apiErr.Detail != test.inRes {
 						t.Errorf("%q should be %q", apiErr.Detail, test.inRes)
 					}
@@ -1548,10 +1455,14 @@ func TestAPIClientDo(t *testing.T) {
 				}
 
 				if test.expectedErr.Detail != "" {
-					apiErr := err.(*Error)
+					apiErr := &Error{}
+					if errors.As(err, &apiErr) {
 						if apiErr.Detail != test.expectedErr.Detail {
 							t.Fatalf("expected error detail :%v, but got:%v", apiErr.Detail, test.expectedErr.Detail)
 						}
+					} else {
+						t.Fatalf("expected v1.Error instance, but got:%T", err)
+					}
 				}
 
 				return
@@ -1680,7 +1591,7 @@ func (c *httpTestClient) Do(ctx context.Context, req *http.Request) (*http.Respo
 	var body []byte
 	done := make(chan struct{})
 	go func() {
-		body, err = ioutil.ReadAll(resp.Body)
+		body, err = io.ReadAll(resp.Body)
 		close(done)
 	}()
 

go.mod

@@ -9,10 +9,10 @@ require (
 	github.com/golang/protobuf v1.5.2
 	github.com/json-iterator/go v1.1.12
 	github.com/prometheus/client_model v0.2.1-0.20220719122737-1f8dcad1221e
-	github.com/prometheus/common v0.35.0
-	github.com/prometheus/procfs v0.7.3
+	github.com/prometheus/common v0.37.0
+	github.com/prometheus/procfs v0.8.0
 	golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a
-	google.golang.org/protobuf v1.28.0
+	google.golang.org/protobuf v1.28.1
 )
 
 require (

go.sum

@@ -91,9 +91,9 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
+github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
@@ -137,10 +137,10 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:
 github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.2.1-0.20220719122737-1f8dcad1221e h1:KjoQdMEQmNC8smQ731iHAXnbFbApg4uu60fNcWHs3Bk=
 github.com/prometheus/client_model v0.2.1-0.20220719122737-1f8dcad1221e/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w=
-github.com/prometheus/common v0.35.0 h1:Eyr+Pw2VymWejHqCugNaQXkAi6KayVNxaHeu6khmFBE=
-github.com/prometheus/common v0.35.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA=
-github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU=
-github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
+github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE=
+github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA=
+github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo=
+github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
@@ -233,7 +233,7 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -259,7 +259,6 @@ golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k=
@@ -318,7 +317,6 @@ golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
 google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
@@ -396,8 +394,8 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw=
-google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w=
+google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=

prometheus/collectors/go_collector_latest.go

@@ -16,16 +16,29 @@
 
 package collectors
 
-import "github.com/prometheus/client_golang/prometheus"
+import (
+	"regexp"
 
-//nolint:staticcheck // Ignore SA1019 until v2.
-type goOptions = prometheus.GoCollectorOptions
-type goOption func(o *goOptions)
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/internal"
+)
 
-type GoCollectionOption uint32
+var (
+	// MetricsAll allows all the metrics to be collected from Go runtime.
+	MetricsAll = GoRuntimeMetricsRule{regexp.MustCompile("/.*")}
+	// MetricsGC allows only GC metrics to be collected from Go runtime.
+	// e.g. go_gc_cycles_automatic_gc_cycles_total
+	MetricsGC = GoRuntimeMetricsRule{regexp.MustCompile(`^/gc/.*`)}
+	// MetricsMemory allows only memory metrics to be collected from Go runtime.
+	// e.g. go_memory_classes_heap_free_bytes
+	MetricsMemory = GoRuntimeMetricsRule{regexp.MustCompile(`^/memory/.*`)}
+	// MetricsScheduler allows only scheduler metrics to be collected from Go runtime.
+	// e.g. go_sched_goroutines_goroutines
+	MetricsScheduler = GoRuntimeMetricsRule{regexp.MustCompile(`^/sched/.*`)}
+)
 
-const (
-	// GoRuntimeMemStatsCollection represents the metrics represented by runtime.MemStats structure such as
+// WithGoCollectorMemStatsMetricsDisabled disables metrics that is gathered in runtime.MemStats structure such as:
+//
 // go_memstats_alloc_bytes
 // go_memstats_alloc_bytes_total
 // go_memstats_sys_bytes
@@ -48,45 +61,100 @@ const (
 // go_memstats_gc_sys_bytes
 // go_memstats_other_sys_bytes
 // go_memstats_next_gc_bytes
-	// so the metrics known from pre client_golang v1.12.0, except skipped go_memstats_gc_cpu_fraction (see
-	// https://github.com/prometheus/client_golang/issues/842#issuecomment-861812034 for explanation.
 //
-	// NOTE that this mode represents runtime.MemStats statistics, but they are
-	// actually implemented using new runtime/metrics package.
-	// Deprecated: Use GoRuntimeMetricsCollection instead going forward.
+// so the metrics known from pre client_golang v1.12.0,
+//
+// NOTE(bwplotka): The above represents runtime.MemStats statistics, but they are
+// actually implemented using new runtime/metrics package. (except skipped go_memstats_gc_cpu_fraction
+// -- see  https://github.com/prometheus/client_golang/issues/842#issuecomment-861812034 for explanation).
+//
+// Some users might want to disable this on collector level (although you can use scrape relabelling on Prometheus),
+// because similar metrics can be now obtained using WithGoCollectorRuntimeMetrics. Note that the semantics of new
+// metrics might be different, plus the names can be change over time with different Go version.
+//
+// NOTE(bwplotka): Changing metric names can be tedious at times as the alerts, recording rules and dashboards have to be adjusted.
+// The old metrics are also very useful, with many guides and books written about how to interpret them.
+//
+// As a result our recommendation would be to stick with MemStats like metrics and enable other runtime/metrics if you are interested
+// in advanced insights Go provides. See ExampleGoCollector_WithAdvancedGoMetrics.
+func WithGoCollectorMemStatsMetricsDisabled() func(options *internal.GoCollectorOptions) {
+	return func(o *internal.GoCollectorOptions) {
+		o.DisableMemStatsLikeMetrics = true
+	}
+}
+
+// GoRuntimeMetricsRule allow enabling and configuring particular group of runtime/metrics.
+// TODO(bwplotka): Consider adding ability to adjust buckets.
+type GoRuntimeMetricsRule struct {
+	// Matcher represents RE2 expression will match the runtime/metrics from https://golang.bg/src/runtime/metrics/description.go
+	// Use `regexp.MustCompile` or `regexp.Compile` to create this field.
+	Matcher *regexp.Regexp
+}
+
+// WithGoCollectorRuntimeMetrics allows enabling and configuring particular group of runtime/metrics.
+// See the list of metrics https://golang.bg/src/runtime/metrics/description.go (pick the Go version you use there!).
+// You can use this option in repeated manner, which will add new rules. The order of rules is important, the last rule
+// that matches particular metrics is applied.
+func WithGoCollectorRuntimeMetrics(rules ...GoRuntimeMetricsRule) func(options *internal.GoCollectorOptions) {
+	rs := make([]internal.GoCollectorRule, len(rules))
+	for i, r := range rules {
+		rs[i] = internal.GoCollectorRule{
+			Matcher: r.Matcher,
+		}
+	}
+
+	return func(o *internal.GoCollectorOptions) {
+		o.RuntimeMetricRules = append(o.RuntimeMetricRules, rs...)
+	}
+}
+
+// WithoutGoCollectorRuntimeMetrics allows disabling group of runtime/metrics that you might have added in WithGoCollectorRuntimeMetrics.
+// It behaves similarly to WithGoCollectorRuntimeMetrics just with deny-list semantics.
+func WithoutGoCollectorRuntimeMetrics(matchers ...*regexp.Regexp) func(options *internal.GoCollectorOptions) {
+	rs := make([]internal.GoCollectorRule, len(matchers))
+	for i, m := range matchers {
+		rs[i] = internal.GoCollectorRule{
+			Matcher: m,
+			Deny:    true,
+		}
+	}
+
+	return func(o *internal.GoCollectorOptions) {
+		o.RuntimeMetricRules = append(o.RuntimeMetricRules, rs...)
+	}
+}
+
+// GoCollectionOption represents Go collection option flag.
+// Deprecated.
+type GoCollectionOption uint32
+
+const (
+	// GoRuntimeMemStatsCollection represents the metrics represented by runtime.MemStats structure.
+	// Deprecated. Use WithGoCollectorMemStatsMetricsDisabled() function to disable those metrics in the collector.
 	GoRuntimeMemStatsCollection GoCollectionOption = 1 << iota
-	// GoRuntimeMetricsCollection is the new set of metrics represented by runtime/metrics package and follows
-	// consistent naming. The exposed metric set depends on Go version, but it is controlled against
-	// unexpected cardinality. This set has overlapping information with GoRuntimeMemStatsCollection, just with
-	// new names. GoRuntimeMetricsCollection is what is recommended for using going forward.
+	// GoRuntimeMetricsCollection is the new set of metrics represented by runtime/metrics package.
+	// Deprecated. Use WithGoCollectorRuntimeMetrics(GoRuntimeMetricsRule{Matcher: regexp.MustCompile("/.*")})
+	// function to enable those metrics in the collector.
 	GoRuntimeMetricsCollection
 )
 
-// WithGoCollections allows enabling different collections for Go collector on top of base metrics
-// like go_goroutines, go_threads, go_gc_duration_seconds, go_memstats_last_gc_time_seconds, go_info.
-//
-// Check GoRuntimeMemStatsCollection and GoRuntimeMetricsCollection for more details. You can use none,
-// one or more collections at once. For example:
-// WithGoCollections(GoRuntimeMemStatsCollection | GoRuntimeMetricsCollection) means both GoRuntimeMemStatsCollection
-// metrics and GoRuntimeMetricsCollection will be exposed.
-//
-// The current default is GoRuntimeMemStatsCollection, so the compatibility mode with
-// client_golang pre v1.12 (move to runtime/metrics).
-//nolint:staticcheck // Ignore SA1019 until v2.
-func WithGoCollections(flags GoCollectionOption) func(options *prometheus.GoCollectorOptions) {
-	return func(o *goOptions) {
-		o.EnabledCollections = uint32(flags)
+// WithGoCollections allows enabling different collections for Go collector on top of base metrics.
+// Deprecated. Use WithGoCollectorRuntimeMetrics() and WithGoCollectorMemStatsMetricsDisabled() instead to control metrics.
+func WithGoCollections(flags GoCollectionOption) func(options *internal.GoCollectorOptions) {
+	return func(options *internal.GoCollectorOptions) {
+		if flags&GoRuntimeMemStatsCollection == 0 {
+			WithGoCollectorMemStatsMetricsDisabled()(options)
+		}
+
+		if flags&GoRuntimeMetricsCollection != 0 {
+			WithGoCollectorRuntimeMetrics(GoRuntimeMetricsRule{Matcher: regexp.MustCompile("/.*")})(options)
+		}
 	}
 }
 
 // NewGoCollector returns a collector that exports metrics about the current Go
-// process using debug.GCStats using runtime/metrics.
-func NewGoCollector(opts ...goOption) prometheus.Collector {
+// process using debug.GCStats (base metrics) and runtime/metrics (both in MemStats style and new ones).
+func NewGoCollector(opts ...func(o *internal.GoCollectorOptions)) prometheus.Collector {
 	//nolint:staticcheck // Ignore SA1019 until v2.
-	promPkgOpts := make([]func(o *prometheus.GoCollectorOptions), len(opts))
-	for i, opt := range opts {
-		promPkgOpts[i] = opt
-	}
-	//nolint:staticcheck // Ignore SA1019 until v2.
-	return prometheus.NewGoCollector(promPkgOpts...)
+	return prometheus.NewGoCollector(opts...)
 }

prometheus/collectors/go_collector_latest_test.go

@@ -18,15 +18,31 @@ package collectors
 
 import (
 	"encoding/json"
+	"log"
+	"net/http"
+	"reflect"
+	"regexp"
+	"sort"
 	"testing"
 
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
 
+var baseMetrics = []string{
+	"go_gc_duration_seconds",
+	"go_goroutines",
+	"go_info",
+	"go_memstats_last_gc_time_seconds",
+	"go_threads",
+}
+
 func TestGoCollectorMarshalling(t *testing.T) {
 	reg := prometheus.NewRegistry()
 	reg.MustRegister(NewGoCollector(
-		WithGoCollections(GoRuntimeMemStatsCollection | GoRuntimeMetricsCollection),
+		WithGoCollectorRuntimeMetrics(GoRuntimeMetricsRule{
+			Matcher: regexp.MustCompile("/.*"),
+		}),
 	))
 	result, err := reg.Gather()
 	if err != nil {
@@ -37,3 +53,248 @@ func TestGoCollectorMarshalling(t *testing.T) {
 		t.Errorf("json marshalling shoud not fail, %v", err)
 	}
 }
+
+func TestWithGoCollectorMemStatsMetricsDisabled(t *testing.T) {
+	reg := prometheus.NewRegistry()
+	reg.MustRegister(NewGoCollector(
+		WithGoCollectorMemStatsMetricsDisabled(),
+	))
+	result, err := reg.Gather()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	got := []string{}
+	for _, r := range result {
+		got = append(got, r.GetName())
+	}
+
+	if !reflect.DeepEqual(got, baseMetrics) {
+		t.Errorf("got %v, want %v", got, baseMetrics)
+	}
+}
+
+func TestGoCollectorAllowList(t *testing.T) {
+	for _, test := range []struct {
+		name     string
+		rules    []GoRuntimeMetricsRule
+		expected []string
+	}{
+		{
+			name:     "Without any rules",
+			rules:    nil,
+			expected: baseMetrics,
+		},
+		{
+			name:  "allow all",
+			rules: []GoRuntimeMetricsRule{MetricsAll},
+			expected: withBaseMetrics([]string{
+				"go_gc_cycles_automatic_gc_cycles_total",
+				"go_gc_cycles_forced_gc_cycles_total",
+				"go_gc_cycles_total_gc_cycles_total",
+				"go_gc_heap_allocs_by_size_bytes",
+				"go_gc_heap_allocs_bytes_total",
+				"go_gc_heap_allocs_objects_total",
+				"go_gc_heap_frees_by_size_bytes",
+				"go_gc_heap_frees_bytes_total",
+				"go_gc_heap_frees_objects_total",
+				"go_gc_heap_goal_bytes",
+				"go_gc_heap_objects_objects",
+				"go_gc_heap_tiny_allocs_objects_total",
+				"go_gc_pauses_seconds",
+				"go_memory_classes_heap_free_bytes",
+				"go_memory_classes_heap_objects_bytes",
+				"go_memory_classes_heap_released_bytes",
+				"go_memory_classes_heap_stacks_bytes",
+				"go_memory_classes_heap_unused_bytes",
+				"go_memory_classes_metadata_mcache_free_bytes",
+				"go_memory_classes_metadata_mcache_inuse_bytes",
+				"go_memory_classes_metadata_mspan_free_bytes",
+				"go_memory_classes_metadata_mspan_inuse_bytes",
+				"go_memory_classes_metadata_other_bytes",
+				"go_memory_classes_os_stacks_bytes",
+				"go_memory_classes_other_bytes",
+				"go_memory_classes_profiling_buckets_bytes",
+				"go_memory_classes_total_bytes",
+				"go_sched_goroutines_goroutines",
+				"go_sched_latencies_seconds",
+			}),
+		},
+		{
+			name:  "allow GC",
+			rules: []GoRuntimeMetricsRule{MetricsGC},
+			expected: withBaseMetrics([]string{
+				"go_gc_cycles_automatic_gc_cycles_total",
+				"go_gc_cycles_forced_gc_cycles_total",
+				"go_gc_cycles_total_gc_cycles_total",
+				"go_gc_heap_allocs_by_size_bytes",
+				"go_gc_heap_allocs_bytes_total",
+				"go_gc_heap_allocs_objects_total",
+				"go_gc_heap_frees_by_size_bytes",
+				"go_gc_heap_frees_bytes_total",
+				"go_gc_heap_frees_objects_total",
+				"go_gc_heap_goal_bytes",
+				"go_gc_heap_objects_objects",
+				"go_gc_heap_tiny_allocs_objects_total",
+				"go_gc_pauses_seconds",
+			}),
+		},
+		{
+			name:  "allow Memory",
+			rules: []GoRuntimeMetricsRule{MetricsMemory},
+			expected: withBaseMetrics([]string{
+				"go_memory_classes_heap_free_bytes",
+				"go_memory_classes_heap_objects_bytes",
+				"go_memory_classes_heap_released_bytes",
+				"go_memory_classes_heap_stacks_bytes",
+				"go_memory_classes_heap_unused_bytes",
+				"go_memory_classes_metadata_mcache_free_bytes",
+				"go_memory_classes_metadata_mcache_inuse_bytes",
+				"go_memory_classes_metadata_mspan_free_bytes",
+				"go_memory_classes_metadata_mspan_inuse_bytes",
+				"go_memory_classes_metadata_other_bytes",
+				"go_memory_classes_os_stacks_bytes",
+				"go_memory_classes_other_bytes",
+				"go_memory_classes_profiling_buckets_bytes",
+				"go_memory_classes_total_bytes",
+			}),
+		},
+		{
+			name:  "allow Scheduler",
+			rules: []GoRuntimeMetricsRule{MetricsScheduler},
+			expected: []string{
+				"go_gc_duration_seconds",
+				"go_goroutines",
+				"go_info",
+				"go_memstats_last_gc_time_seconds",
+				"go_sched_goroutines_goroutines",
+				"go_sched_latencies_seconds",
+				"go_threads",
+			},
+		},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			reg := prometheus.NewRegistry()
+			reg.MustRegister(NewGoCollector(
+				WithGoCollectorMemStatsMetricsDisabled(),
+				WithGoCollectorRuntimeMetrics(test.rules...),
+			))
+			result, err := reg.Gather()
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			got := []string{}
+			for _, r := range result {
+				got = append(got, r.GetName())
+			}
+
+			if !reflect.DeepEqual(got, test.expected) {
+				t.Errorf("got %v, want %v", got, test.expected)
+			}
+		})
+	}
+}
+
+func withBaseMetrics(metricNames []string) []string {
+	metricNames = append(metricNames, baseMetrics...)
+	sort.Strings(metricNames)
+	return metricNames
+}
+
+func TestGoCollectorDenyList(t *testing.T) {
+	for _, test := range []struct {
+		name     string
+		matchers []*regexp.Regexp
+		expected []string
+	}{
+		{
+			name:     "Without any matchers",
+			matchers: nil,
+			expected: baseMetrics,
+		},
+		{
+			name:     "deny all",
+			matchers: []*regexp.Regexp{regexp.MustCompile("/.*")},
+			expected: baseMetrics,
+		},
+		{
+			name: "deny gc and scheduler latency",
+			matchers: []*regexp.Regexp{
+				regexp.MustCompile("^/gc/.*"),
+				regexp.MustCompile("^/sched/latencies:.*"),
+			},
+			expected: baseMetrics,
+		},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			reg := prometheus.NewRegistry()
+			reg.MustRegister(NewGoCollector(
+				WithGoCollectorMemStatsMetricsDisabled(),
+				WithoutGoCollectorRuntimeMetrics(test.matchers...),
+			))
+			result, err := reg.Gather()
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			got := []string{}
+			for _, r := range result {
+				got = append(got, r.GetName())
+			}
+
+			if !reflect.DeepEqual(got, test.expected) {
+				t.Errorf("got %v, want %v", got, test.expected)
+			}
+		})
+	}
+}
+
+func ExampleGoCollector() {
+	reg := prometheus.NewRegistry()
+
+	// Register the GoCollector with the default options. Only the base metrics will be enabled.
+	reg.MustRegister(NewGoCollector())
+
+	http.Handle("/metrics", promhttp.HandlerFor(reg, promhttp.HandlerOpts{}))
+	log.Fatal(http.ListenAndServe(":8080", nil))
+}
+
+func ExampleGoCollector_WithAdvancedGoMetrics() {
+	reg := prometheus.NewRegistry()
+
+	// Enable Go metrics with pre-defined rules. Or your custom rules.
+	reg.MustRegister(
+		NewGoCollector(
+			WithGoCollectorMemStatsMetricsDisabled(),
+			WithGoCollectorRuntimeMetrics(
+				MetricsScheduler,
+				MetricsMemory,
+				GoRuntimeMetricsRule{
+					Matcher: regexp.MustCompile("^/mycustomrule.*"),
+				},
+			),
+			WithoutGoCollectorRuntimeMetrics(regexp.MustCompile("^/gc/.*")),
+		))
+
+	http.Handle("/metrics", promhttp.HandlerFor(reg, promhttp.HandlerOpts{}))
+	log.Fatal(http.ListenAndServe(":8080", nil))
+}
+
+func ExampleGoCollector_DefaultRegister() {
+	// Unregister the default GoCollector.
+	prometheus.Unregister(NewGoCollector())
+
+	// Register the default GoCollector with a custom config.
+	prometheus.MustRegister(NewGoCollector(WithGoCollectorRuntimeMetrics(
+		MetricsScheduler,
+		MetricsGC,
+		GoRuntimeMetricsRule{
+			Matcher: regexp.MustCompile("^/mycustomrule.*"),
+		},
+	),
+	))
+
+	http.Handle("/metrics", promhttp.Handler())
+	log.Fatal(http.ListenAndServe(":8080", nil))
+}

prometheus/counter.go

@@ -51,7 +51,7 @@ type Counter interface {
 // will lead to a valid (label-less) exemplar. But if Labels is nil, the current
 // exemplar is left in place. AddWithExemplar panics if the value is < 0, if any
 // of the provided labels are invalid, or if the provided labels contain more
-// than 64 runes in total.
+// than 128 runes in total.
 type ExemplarAdder interface {
 	AddWithExemplar(value float64, exemplar Labels)
 }

prometheus/counter_test.go

@@ -16,6 +16,7 @@ package prometheus
 import (
 	"fmt"
 	"math"
+	"strings"
 	"testing"
 	"time"
 
@@ -262,10 +263,11 @@ func TestCounterExemplar(t *testing.T) {
 				err = e.(error)
 			}
 		}()
-		// Should panic because of 65 runes.
+		// Should panic because of 129 runes.
 		counter.AddWithExemplar(42, Labels{
 			"abcdefghijklmnopqrstuvwxyz": "26+16 characters",
 			"x1234567":                   "8+15 characters",
+			"z":                          strings.Repeat("x", 63),
 		})
 		return nil
 	}

prometheus/examples_test.go

@@ -15,6 +15,7 @@ package prometheus_test
 
 import (
 	"bytes"
+	"errors"
 	"fmt"
 	"math"
 	"net/http"
@@ -713,7 +714,8 @@ func ExampleAlreadyRegisteredError() {
 		Help: "The total number of requests served.",
 	})
 	if err := prometheus.Register(reqCounter); err != nil {
-		if are, ok := err.(prometheus.AlreadyRegisteredError); ok {
+		are := &prometheus.AlreadyRegisteredError{}
+		if errors.As(err, are) {
 			// A counter for that metric has been registered before.
 			// Use the old counter from now on.
 			reqCounter = are.ExistingCollector.(prometheus.Counter)

prometheus/gen_go_collector_metrics_set.go

@@ -25,28 +25,41 @@ import (
 	"os"
 	"runtime"
 	"runtime/metrics"
-	"strconv"
 	"strings"
 	"text/template"
 
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/internal"
+
+	"github.com/hashicorp/go-version"
 )
 
 func main() {
-	if len(os.Args) != 2 {
-		log.Fatal("requires Go version (e.g. go1.17) as an argument")
-	}
+	var givenVersion string
 	toolVersion := runtime.Version()
-	mtv := majorVersion(toolVersion)
-	mv := majorVersion(os.Args[1])
-	if mtv != mv {
-		log.Fatalf("using Go version %q but expected Go version %q", mtv, mv)
+	if len(os.Args) != 2 {
+		log.Printf("requires Go version (e.g. go1.17) as an argument. Since it is not specified, assuming %s.", toolVersion)
+		givenVersion = toolVersion
+	} else {
+		givenVersion = os.Args[1]
 	}
-	version, err := parseVersion(mv)
+	log.Printf("given version for Go: %s", givenVersion)
+	log.Printf("tool version for Go: %s", toolVersion)
+
+	tv, err := version.NewVersion(strings.TrimPrefix(givenVersion, "go"))
 	if err != nil {
-		log.Fatalf("parsing Go version: %v", err)
+		log.Fatal(err)
 	}
+	gv, err := version.NewVersion(strings.TrimPrefix(toolVersion, "go"))
+	if err != nil {
+		log.Fatal(err)
+	}
+	if !gv.Equal(tv) {
+		log.Fatalf("using Go version %q but expected Go version %q", tv, gv)
+	}
+
+	v := goVersion(gv.Segments()[1])
+	log.Printf("generating metrics for Go version %q", v)
 
 	// Generate code.
 	var buf bytes.Buffer
@@ -56,7 +69,7 @@ func main() {
 		Cardinality  int
 	}{
 		Descriptions: metrics.All(),
-		GoVersion:    version,
+		GoVersion:    v,
 		Cardinality:  rmCardinality(),
 	})
 	if err != nil {
@@ -70,7 +83,7 @@ func main() {
 	}
 
 	// Write it to a file.
-	fname := fmt.Sprintf("go_collector_metrics_%s_test.go", version.Abbr())
+	fname := fmt.Sprintf("go_collector_metrics_%s_test.go", v.Abbr())
 	if err := os.WriteFile(fname, result, 0o644); err != nil {
 		log.Fatalf("writing file: %v", err)
 	}
@@ -86,19 +99,6 @@ func (g goVersion) Abbr() string {
 	return fmt.Sprintf("go1%d", g)
 }
 
-func parseVersion(s string) (goVersion, error) {
-	i := strings.IndexRune(s, '.')
-	if i < 0 {
-		return goVersion(-1), fmt.Errorf("bad Go version format")
-	}
-	i, err := strconv.Atoi(s[i+1:])
-	return goVersion(i), err
-}
-
-func majorVersion(v string) string {
-	return v[:strings.LastIndexByte(v, '.')]
-}
-
 func rmCardinality() int {
 	cardinality := 0
 
@@ -123,6 +123,7 @@ func rmCardinality() int {
 			name[strings.IndexRune(name, ':')+1:],
 		)
 		cardinality += len(buckets) + 3 // Plus total count, sum, and the implicit infinity bucket.
+
 		// runtime/metrics bucket boundaries are lower-bound-inclusive, but
 		// always represents each actual *boundary* so Buckets is always
 		// 1 longer than Counts, while in Prometheus the mapping is one-to-one,
@@ -134,6 +135,12 @@ func rmCardinality() int {
 			// We already counted the infinity bucket separately.
 			cardinality--
 		}
+		// Prometheus also doesn't have buckets for -Inf, so they need to be omitted.
+		// See the following PR for more information:
+		// https://github.com/prometheus/client_golang/pull/1049
+		if buckets[0] == math.Inf(-1) {
+			cardinality--
+		}
 	}
 
 	return cardinality

prometheus/get_pid.go

@@ -0,0 +1,26 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build !js || wasm
+// +build !js wasm
+
+package prometheus
+
+import "os"
+
+func getPIDFn() func() (int, error) {
+	pid := os.Getpid()
+	return func() (int, error) {
+		return pid, nil
+	}
+}

prometheus/get_pid_gopherjs.go

@@ -0,0 +1,23 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build js && !wasm
+// +build js,!wasm
+
+package prometheus
+
+func getPIDFn() func() (int, error) {
+	return func() (int, error) {
+		return 1, nil
+	}
+}

prometheus/go_collector.go

@@ -19,6 +19,10 @@ import (
 	"time"
 )
 
+// goRuntimeMemStats provides the metrics initially provided by runtime.ReadMemStats.
+// From Go 1.17 those similar (and better) statistics are provided by runtime/metrics, so
+// while eval closure works on runtime.MemStats, the struct from Go 1.17+ is
+// populated using runtime/metrics.
 func goRuntimeMemStats() memStatsMetrics {
 	return memStatsMetrics{
 		{
@@ -224,7 +228,7 @@ func newBaseGoCollector() baseGoCollector {
 			"A summary of the pause duration of garbage collection cycles.",
 			nil, nil),
 		gcLastTimeDesc: NewDesc(
-			memstatNamespace("last_gc_time_seconds"),
+			"go_memstats_last_gc_time_seconds",
 			"Number of seconds since 1970 of last garbage collection.",
 			nil, nil),
 		goInfoDesc: NewDesc(
@@ -246,8 +250,9 @@ func (c *baseGoCollector) Describe(ch chan<- *Desc) {
 // Collect returns the current state of all metrics of the collector.
 func (c *baseGoCollector) Collect(ch chan<- Metric) {
 	ch <- MustNewConstMetric(c.goroutinesDesc, GaugeValue, float64(runtime.NumGoroutine()))
-	n, _ := runtime.ThreadCreateProfile(nil)
-	ch <- MustNewConstMetric(c.threadsDesc, GaugeValue, float64(n))
+
+	n := getRuntimeNumThreads()
+	ch <- MustNewConstMetric(c.threadsDesc, GaugeValue, n)
 
 	var stats debug.GCStats
 	stats.PauseQuantiles = make([]time.Duration, 5)
@@ -269,7 +274,6 @@ func memstatNamespace(s string) string {
 
 // memStatsMetrics provide description, evaluator, runtime/metrics name, and
 // value type for memstat metrics.
-// TODO(bwplotka): Remove with end Go 1.16 EOL and replace with runtime/metrics.Description
 type memStatsMetrics []struct {
 	desc    *Desc
 	eval    func(*runtime.MemStats) float64

prometheus/go_collector_latest.go

@@ -31,9 +31,11 @@ import (
 )
 
 const (
+	// constants for strings referenced more than once.
 	goGCHeapTinyAllocsObjects               = "/gc/heap/tiny/allocs:objects"
 	goGCHeapAllocsObjects                   = "/gc/heap/allocs:objects"
 	goGCHeapFreesObjects                    = "/gc/heap/frees:objects"
+	goGCHeapFreesBytes                      = "/gc/heap/frees:bytes"
 	goGCHeapAllocsBytes                     = "/gc/heap/allocs:bytes"
 	goGCHeapObjects                         = "/gc/heap/objects:objects"
 	goGCHeapGoalBytes                       = "/gc/heap/goal:bytes"
@@ -53,8 +55,8 @@ const (
 	goMemoryClassesOtherBytes               = "/memory/classes/other:bytes"
 )
 
-// runtime/metrics names required for runtimeMemStats like logic.
-var rmForMemStats = []string{
+// rmNamesForMemStatsMetrics represents runtime/metrics names required to populate goRuntimeMemStats from like logic.
+var rmNamesForMemStatsMetrics = []string{
 	goGCHeapTinyAllocsObjects,
 	goGCHeapAllocsObjects,
 	goGCHeapFreesObjects,
@@ -90,74 +92,90 @@ func bestEffortLookupRM(lookup []string) []metrics.Description {
 }
 
 type goCollector struct {
-	opt  GoCollectorOptions
 	base baseGoCollector
 
 	// mu protects updates to all fields ensuring a consistent
 	// snapshot is always produced by Collect.
 	mu sync.Mutex
 
-	// rm... fields all pertain to the runtime/metrics package.
-	rmSampleBuf []metrics.Sample
-	rmSampleMap map[string]*metrics.Sample
-	rmMetrics   []collectorMetric
+	// Contains all samples that has to retrieved from runtime/metrics (not all of them will be exposed).
+	sampleBuf []metrics.Sample
+	// sampleMap allows lookup for MemStats metrics and runtime/metrics histograms for exact sums.
+	sampleMap map[string]*metrics.Sample
+
+	// rmExposedMetrics represents all runtime/metrics package metrics
+	// that were configured to be exposed.
+	rmExposedMetrics     []collectorMetric
+	rmExactSumMapForHist map[string]string
 
 	// With Go 1.17, the runtime/metrics package was introduced.
 	// From that point on, metric names produced by the runtime/metrics
 	// package could be generated from runtime/metrics names. However,
 	// these differ from the old names for the same values.
 	//
-	// This field exist to export the same values under the old names
+	// This field exists to export the same values under the old names
 	// as well.
 	msMetrics        memStatsMetrics
+	msMetricsEnabled bool
 }
 
-const (
-	// Those are not exposed due to need to move Go collector to another package in v2.
-	// See issue https://github.com/prometheus/client_golang/issues/1030.
-	goRuntimeMemStatsCollection uint32 = 1 << iota
-	goRuntimeMetricsCollection
+type rmMetricDesc struct {
+	metrics.Description
+}
+
+func matchRuntimeMetricsRules(rules []internal.GoCollectorRule) []rmMetricDesc {
+	var descs []rmMetricDesc
+	for _, d := range metrics.All() {
+		var (
+			deny = true
+			desc rmMetricDesc
 		)
 
-// GoCollectorOptions should not be used be directly by anything, except `collectors` package.
-// Use it via collectors package instead. See issue
-// https://github.com/prometheus/client_golang/issues/1030.
-//
-// Deprecated: Use collectors.WithGoCollections
-type GoCollectorOptions struct {
-	// EnabledCollection sets what type of collections collector should expose on top of base collection.
-	// By default it's goMemStatsCollection | goRuntimeMetricsCollection.
-	EnabledCollections uint32
+		for _, r := range rules {
+			if !r.Matcher.MatchString(d.Name) {
+				continue
+			}
+			deny = r.Deny
+		}
+		if deny {
+			continue
 		}
 
-func (c GoCollectorOptions) isEnabled(flag uint32) bool {
-	return c.EnabledCollections&flag != 0
+		desc.Description = d
+		descs = append(descs, desc)
+	}
+	return descs
 }
 
-const defaultGoCollections = goRuntimeMemStatsCollection
+func defaultGoCollectorOptions() internal.GoCollectorOptions {
+	return internal.GoCollectorOptions{
+		RuntimeMetricSumForHist: map[string]string{
+			"/gc/heap/allocs-by-size:bytes": goGCHeapAllocsBytes,
+			"/gc/heap/frees-by-size:bytes":  goGCHeapFreesBytes,
+		},
+		RuntimeMetricRules: []internal.GoCollectorRule{
+			//{Matcher: regexp.MustCompile("")},
+		},
+	}
+}
 
 // NewGoCollector is the obsolete version of collectors.NewGoCollector.
 // See there for documentation.
 //
 // Deprecated: Use collectors.NewGoCollector instead.
-func NewGoCollector(opts ...func(o *GoCollectorOptions)) Collector {
-	opt := GoCollectorOptions{EnabledCollections: defaultGoCollections}
+func NewGoCollector(opts ...func(o *internal.GoCollectorOptions)) Collector {
+	opt := defaultGoCollectorOptions()
 	for _, o := range opts {
 		o(&opt)
 	}
 
-	var descriptions []metrics.Description
-	if opt.isEnabled(goRuntimeMetricsCollection) {
-		descriptions = metrics.All()
-	} else if opt.isEnabled(goRuntimeMemStatsCollection) {
-		descriptions = bestEffortLookupRM(rmForMemStats)
-	}
+	exposedDescriptions := matchRuntimeMetricsRules(opt.RuntimeMetricRules)
 
 	// Collect all histogram samples so that we can get their buckets.
 	// The API guarantees that the buckets are always fixed for the lifetime
 	// of the process.
 	var histograms []metrics.Sample
-	for _, d := range descriptions {
+	for _, d := range exposedDescriptions {
 		if d.Kind == metrics.KindFloat64Histogram {
 			histograms = append(histograms, metrics.Sample{Name: d.Name})
 		}
@@ -172,13 +190,14 @@ func NewGoCollector(opts ...func(o *GoCollectorOptions)) Collector {
 		bucketsMap[histograms[i].Name] = histograms[i].Value.Float64Histogram().Buckets
 	}
 
-	// Generate a Desc and ValueType for each runtime/metrics metric.
-	metricSet := make([]collectorMetric, 0, len(descriptions))
-	sampleBuf := make([]metrics.Sample, 0, len(descriptions))
-	sampleMap := make(map[string]*metrics.Sample, len(descriptions))
-	for i := range descriptions {
-		d := &descriptions[i]
-		namespace, subsystem, name, ok := internal.RuntimeMetricsToProm(d)
+	// Generate a collector for each exposed runtime/metrics metric.
+	metricSet := make([]collectorMetric, 0, len(exposedDescriptions))
+	// SampleBuf is used for reading from runtime/metrics.
+	// We are assuming the largest case to have stable pointers for sampleMap purposes.
+	sampleBuf := make([]metrics.Sample, 0, len(exposedDescriptions)+len(opt.RuntimeMetricSumForHist)+len(rmNamesForMemStatsMetrics))
+	sampleMap := make(map[string]*metrics.Sample, len(exposedDescriptions))
+	for _, d := range exposedDescriptions {
+		namespace, subsystem, name, ok := internal.RuntimeMetricsToProm(&d.Description)
 		if !ok {
 			// Just ignore this metric; we can't do anything with it here.
 			// If a user decides to use the latest version of Go, we don't want
@@ -186,19 +205,17 @@ func NewGoCollector(opts ...func(o *GoCollectorOptions)) Collector {
 			continue
 		}
 
-		// Set up sample buffer for reading, and a map
-		// for quick lookup of sample values.
 		sampleBuf = append(sampleBuf, metrics.Sample{Name: d.Name})
 		sampleMap[d.Name] = &sampleBuf[len(sampleBuf)-1]
 
 		var m collectorMetric
 		if d.Kind == metrics.KindFloat64Histogram {
-			_, hasSum := rmExactSumMap[d.Name]
+			_, hasSum := opt.RuntimeMetricSumForHist[d.Name]
 			unit := d.Name[strings.IndexRune(d.Name, ':')+1:]
 			m = newBatchHistogram(
 				NewDesc(
 					BuildFQName(namespace, subsystem, name),
-					d.Description,
+					d.Description.Description,
 					nil,
 					nil,
 				),
@@ -210,30 +227,61 @@ func NewGoCollector(opts ...func(o *GoCollectorOptions)) Collector {
 				Namespace: namespace,
 				Subsystem: subsystem,
 				Name:      name,
-				Help:      d.Description,
-			})
+				Help:      d.Description.Description,
+			},
+			)
 		} else {
 			m = NewGauge(GaugeOpts{
 				Namespace: namespace,
 				Subsystem: subsystem,
 				Name:      name,
-				Help:      d.Description,
+				Help:      d.Description.Description,
 			})
 		}
 		metricSet = append(metricSet, m)
 	}
 
-	var msMetrics memStatsMetrics
-	if opt.isEnabled(goRuntimeMemStatsCollection) {
-		msMetrics = goRuntimeMemStats()
+	// Add exact sum metrics to sampleBuf if not added before.
+	for _, h := range histograms {
+		sumMetric, ok := opt.RuntimeMetricSumForHist[h.Name]
+		if !ok {
+			continue
 		}
+
+		if _, ok := sampleMap[sumMetric]; ok {
+			continue
+		}
+		sampleBuf = append(sampleBuf, metrics.Sample{Name: sumMetric})
+		sampleMap[sumMetric] = &sampleBuf[len(sampleBuf)-1]
+	}
+
+	var (
+		msMetrics      memStatsMetrics
+		msDescriptions []metrics.Description
+	)
+
+	if !opt.DisableMemStatsLikeMetrics {
+		msMetrics = goRuntimeMemStats()
+		msDescriptions = bestEffortLookupRM(rmNamesForMemStatsMetrics)
+
+		// Check if metric was not exposed before and if not, add to sampleBuf.
+		for _, mdDesc := range msDescriptions {
+			if _, ok := sampleMap[mdDesc.Name]; ok {
+				continue
+			}
+			sampleBuf = append(sampleBuf, metrics.Sample{Name: mdDesc.Name})
+			sampleMap[mdDesc.Name] = &sampleBuf[len(sampleBuf)-1]
+		}
+	}
+
 	return &goCollector{
-		opt:         opt,
 		base:                 newBaseGoCollector(),
-		rmSampleBuf: sampleBuf,
-		rmSampleMap: sampleMap,
-		rmMetrics:   metricSet,
+		sampleBuf:            sampleBuf,
+		sampleMap:            sampleMap,
+		rmExposedMetrics:     metricSet,
+		rmExactSumMapForHist: opt.RuntimeMetricSumForHist,
 		msMetrics:            msMetrics,
+		msMetricsEnabled:     !opt.DisableMemStatsLikeMetrics,
 	}
 }
 
@@ -243,7 +291,7 @@ func (c *goCollector) Describe(ch chan<- *Desc) {
 	for _, i := range c.msMetrics {
 		ch <- i.desc
 	}
-	for _, m := range c.rmMetrics {
+	for _, m := range c.rmExposedMetrics {
 		ch <- m.Desc()
 	}
 }
@@ -253,8 +301,12 @@ func (c *goCollector) Collect(ch chan<- Metric) {
 	// Collect base non-memory metrics.
 	c.base.Collect(ch)
 
+	if len(c.sampleBuf) == 0 {
+		return
+	}
+
 	// Collect must be thread-safe, so prevent concurrent use of
-	// rmSampleBuf. Just read into rmSampleBuf but write all the data
+	// sampleBuf elements. Just read into sampleBuf but write all the data
 	// we get into our Metrics or MemStats.
 	//
 	// This lock also ensures that the Metrics we send out are all from
@@ -268,18 +320,18 @@ func (c *goCollector) Collect(ch chan<- Metric) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 
-	if len(c.rmSampleBuf) > 0 {
 	// Populate runtime/metrics sample buffer.
-		metrics.Read(c.rmSampleBuf)
-	}
+	metrics.Read(c.sampleBuf)
+
+	// Collect all our runtime/metrics user chose to expose from sampleBuf (if any).
+	for i, metric := range c.rmExposedMetrics {
+		// We created samples for exposed metrics first in order, so indexes match.
+		sample := c.sampleBuf[i]
 
-	if c.opt.isEnabled(goRuntimeMetricsCollection) {
-		// Collect all our metrics from rmSampleBuf.
-		for i, sample := range c.rmSampleBuf {
 		// N.B. switch on concrete type because it's significantly more efficient
 		// than checking for the Counter and Gauge interface implementations. In
 		// this case, we control all the types here.
-			switch m := c.rmMetrics[i].(type) {
+		switch m := metric.(type) {
 		case *counter:
 			// Guard against decreases. This should never happen, but a failure
 			// to do so will result in a panic, which is a harsh consequence for
@@ -299,13 +351,12 @@ func (c *goCollector) Collect(ch chan<- Metric) {
 			panic("unexpected metric type")
 		}
 	}
-	}
 
+	if c.msMetricsEnabled {
 		// ms is a dummy MemStats that we populate ourselves so that we can
 		// populate the old metrics from it if goMemStatsCollection is enabled.
-	if c.opt.isEnabled(goRuntimeMemStatsCollection) {
 		var ms runtime.MemStats
-		memStatsFromRM(&ms, c.rmSampleMap)
+		memStatsFromRM(&ms, c.sampleMap)
 		for _, i := range c.msMetrics {
 			ch <- MustNewConstMetric(i.desc, i.valType, i.eval(&ms))
 		}
@@ -336,11 +387,6 @@ func unwrapScalarRMValue(v metrics.Value) float64 {
 	}
 }
 
-var rmExactSumMap = map[string]string{
-	"/gc/heap/allocs-by-size:bytes": "/gc/heap/allocs:bytes",
-	"/gc/heap/frees-by-size:bytes":  "/gc/heap/frees:bytes",
-}
-
 // exactSumFor takes a runtime/metrics metric name (that is assumed to
 // be of kind KindFloat64Histogram) and returns its exact sum and whether
 // its exact sum exists.
@@ -348,11 +394,11 @@ var rmExactSumMap = map[string]string{
 // The runtime/metrics API for histograms doesn't currently expose exact
 // sums, but some of the other metrics are in fact exact sums of histograms.
 func (c *goCollector) exactSumFor(rmName string) float64 {
-	sumName, ok := rmExactSumMap[rmName]
+	sumName, ok := c.rmExactSumMapForHist[rmName]
 	if !ok {
 		return 0
 	}
-	s, ok := c.rmSampleMap[sumName]
+	s, ok := c.sampleMap[sumName]
 	if !ok {
 		return 0
 	}

prometheus/go_collector_latest_test.go

@@ -19,6 +19,7 @@ package prometheus
 import (
 	"math"
 	"reflect"
+	"regexp"
 	"runtime"
 	"runtime/metrics"
 	"sync"
@@ -30,9 +31,18 @@ import (
 )
 
 func TestRmForMemStats(t *testing.T) {
-	if got, want := len(bestEffortLookupRM(rmForMemStats)), len(rmForMemStats); got != want {
+	descs := bestEffortLookupRM(rmNamesForMemStatsMetrics)
+
+	if got, want := len(descs), len(rmNamesForMemStatsMetrics); got != want {
 		t.Errorf("got %d, want %d metrics", got, want)
 	}
+
+	for _, d := range descs {
+		// We don't expect histograms there.
+		if d.Kind == metrics.KindFloat64Histogram {
+			t.Errorf("we don't expect to use histograms for MemStats metrics, got %v", d.Name)
+		}
+	}
 }
 
 func expectedBaseMetrics() map[string]struct{} {
@@ -64,30 +74,43 @@ func addExpectedRuntimeMetrics(metrics map[string]struct{}) map[string]struct{}
 	return metrics
 }
 
-func TestGoCollector(t *testing.T) {
+func TestGoCollector_ExposedMetrics(t *testing.T) {
 	for _, tcase := range []struct {
-		collections       uint32
+		opts              internal.GoCollectorOptions
 		expectedFQNameSet map[string]struct{}
 	}{
 		{
-			collections:       0,
+			opts: internal.GoCollectorOptions{
+				DisableMemStatsLikeMetrics: true,
+			},
 			expectedFQNameSet: expectedBaseMetrics(),
 		},
 		{
-			collections:       goRuntimeMemStatsCollection,
+			// Default, only MemStats.
 			expectedFQNameSet: addExpectedRuntimeMemStats(expectedBaseMetrics()),
 		},
 		{
-			collections:       goRuntimeMetricsCollection,
+			// Get all runtime/metrics without MemStats.
+			opts: internal.GoCollectorOptions{
+				DisableMemStatsLikeMetrics: true,
+				RuntimeMetricRules: []internal.GoCollectorRule{
+					{Matcher: regexp.MustCompile("/.*")},
+				},
+			},
 			expectedFQNameSet: addExpectedRuntimeMetrics(expectedBaseMetrics()),
 		},
 		{
-			collections:       goRuntimeMemStatsCollection | goRuntimeMetricsCollection,
+			// Get all runtime/metrics and MemStats.
+			opts: internal.GoCollectorOptions{
+				RuntimeMetricRules: []internal.GoCollectorRule{
+					{Matcher: regexp.MustCompile("/.*")},
+				},
+			},
 			expectedFQNameSet: addExpectedRuntimeMemStats(addExpectedRuntimeMetrics(expectedBaseMetrics())),
 		},
 	} {
 		if ok := t.Run("", func(t *testing.T) {
-			goMetrics := collectGoMetrics(t, tcase.collections)
+			goMetrics := collectGoMetrics(t, tcase.opts)
 			goMetricSet := make(map[string]Metric)
 			for _, m := range goMetrics {
 				goMetricSet[m.Desc().fqName] = m
@@ -118,7 +141,11 @@ func TestGoCollector(t *testing.T) {
 var sink interface{}
 
 func TestBatchHistogram(t *testing.T) {
-	goMetrics := collectGoMetrics(t, goRuntimeMetricsCollection)
+	goMetrics := collectGoMetrics(t, internal.GoCollectorOptions{
+		RuntimeMetricRules: []internal.GoCollectorRule{
+			{Matcher: regexp.MustCompile("/.*")},
+		},
+	})
 
 	var mhist Metric
 	for _, m := range goMetrics {
@@ -145,7 +172,8 @@ func TestBatchHistogram(t *testing.T) {
 	for i := 0; i < 100; i++ {
 		sink = make([]byte, 128)
 	}
-	collectGoMetrics(t, defaultGoCollections)
+
+	collectGoMetrics(t, defaultGoCollectorOptions())
 	for i, v := range hist.counts {
 		if v != countsCopy[i] {
 			t.Error("counts changed during new collection")
@@ -194,11 +222,13 @@ func TestBatchHistogram(t *testing.T) {
 	}
 }
 
-func collectGoMetrics(t *testing.T, enabledCollections uint32) []Metric {
+func collectGoMetrics(t *testing.T, opts internal.GoCollectorOptions) []Metric {
 	t.Helper()
 
-	c := NewGoCollector(func(o *GoCollectorOptions) {
-		o.EnabledCollections = enabledCollections
+	c := NewGoCollector(func(o *internal.GoCollectorOptions) {
+		o.DisableMemStatsLikeMetrics = opts.DisableMemStatsLikeMetrics
+		o.RuntimeMetricSumForHist = opts.RuntimeMetricSumForHist
+		o.RuntimeMetricRules = opts.RuntimeMetricRules
 	}).(*goCollector)
 
 	// Collect all metrics.
@@ -222,7 +252,7 @@ func collectGoMetrics(t *testing.T, enabledCollections uint32) []Metric {
 
 func TestMemStatsEquivalence(t *testing.T) {
 	var msReal, msFake runtime.MemStats
-	descs := bestEffortLookupRM(rmForMemStats)
+	descs := bestEffortLookupRM(rmNamesForMemStatsMetrics)
 
 	samples := make([]metrics.Sample, len(descs))
 	samplesMap := make(map[string]*metrics.Sample)
@@ -269,7 +299,12 @@ func TestMemStatsEquivalence(t *testing.T) {
 }
 
 func TestExpectedRuntimeMetrics(t *testing.T) {
-	goMetrics := collectGoMetrics(t, goRuntimeMetricsCollection)
+	goMetrics := collectGoMetrics(t, internal.GoCollectorOptions{
+		DisableMemStatsLikeMetrics: true,
+		RuntimeMetricRules: []internal.GoCollectorRule{
+			{Matcher: regexp.MustCompile("/.*")},
+		},
+	})
 	goMetricSet := make(map[string]Metric)
 	for _, m := range goMetrics {
 		goMetricSet[m.Desc().fqName] = m

prometheus/go_collector_metrics_go119_test.go

@@ -0,0 +1,45 @@
+// Code generated by gen_go_collector_metrics_set.go; DO NOT EDIT.
+//go:generate go run gen_go_collector_metrics_set.go go1.19
+
+//go:build go1.19 && !go1.20
+// +build go1.19,!go1.20
+
+package prometheus
+
+var expectedRuntimeMetrics = map[string]string{
+	"/cgo/go-to-c-calls:calls":                    "go_cgo_go_to_c_calls_calls_total",
+	"/gc/cycles/automatic:gc-cycles":              "go_gc_cycles_automatic_gc_cycles_total",
+	"/gc/cycles/forced:gc-cycles":                 "go_gc_cycles_forced_gc_cycles_total",
+	"/gc/cycles/total:gc-cycles":                  "go_gc_cycles_total_gc_cycles_total",
+	"/gc/heap/allocs-by-size:bytes":               "go_gc_heap_allocs_by_size_bytes",
+	"/gc/heap/allocs:bytes":                       "go_gc_heap_allocs_bytes_total",
+	"/gc/heap/allocs:objects":                     "go_gc_heap_allocs_objects_total",
+	"/gc/heap/frees-by-size:bytes":                "go_gc_heap_frees_by_size_bytes",
+	"/gc/heap/frees:bytes":                        "go_gc_heap_frees_bytes_total",
+	"/gc/heap/frees:objects":                      "go_gc_heap_frees_objects_total",
+	"/gc/heap/goal:bytes":                         "go_gc_heap_goal_bytes",
+	"/gc/heap/objects:objects":                    "go_gc_heap_objects_objects",
+	"/gc/heap/tiny/allocs:objects":                "go_gc_heap_tiny_allocs_objects_total",
+	"/gc/limiter/last-enabled:gc-cycle":           "go_gc_limiter_last_enabled_gc_cycle",
+	"/gc/pauses:seconds":                          "go_gc_pauses_seconds",
+	"/gc/stack/starting-size:bytes":               "go_gc_stack_starting_size_bytes",
+	"/memory/classes/heap/free:bytes":             "go_memory_classes_heap_free_bytes",
+	"/memory/classes/heap/objects:bytes":          "go_memory_classes_heap_objects_bytes",
+	"/memory/classes/heap/released:bytes":         "go_memory_classes_heap_released_bytes",
+	"/memory/classes/heap/stacks:bytes":           "go_memory_classes_heap_stacks_bytes",
+	"/memory/classes/heap/unused:bytes":           "go_memory_classes_heap_unused_bytes",
+	"/memory/classes/metadata/mcache/free:bytes":  "go_memory_classes_metadata_mcache_free_bytes",
+	"/memory/classes/metadata/mcache/inuse:bytes": "go_memory_classes_metadata_mcache_inuse_bytes",
+	"/memory/classes/metadata/mspan/free:bytes":   "go_memory_classes_metadata_mspan_free_bytes",
+	"/memory/classes/metadata/mspan/inuse:bytes":  "go_memory_classes_metadata_mspan_inuse_bytes",
+	"/memory/classes/metadata/other:bytes":        "go_memory_classes_metadata_other_bytes",
+	"/memory/classes/os-stacks:bytes":             "go_memory_classes_os_stacks_bytes",
+	"/memory/classes/other:bytes":                 "go_memory_classes_other_bytes",
+	"/memory/classes/profiling/buckets:bytes":     "go_memory_classes_profiling_buckets_bytes",
+	"/memory/classes/total:bytes":                 "go_memory_classes_total_bytes",
+	"/sched/gomaxprocs:threads":                   "go_sched_gomaxprocs_threads",
+	"/sched/goroutines:goroutines":                "go_sched_goroutines_goroutines",
+	"/sched/latencies:seconds":                    "go_sched_latencies_seconds",
+}
+
+const expectedRuntimeMetricsCardinality = 81

prometheus/internal/difflib.go

@@ -485,7 +485,7 @@ func (m *SequenceMatcher) QuickRatio() float64 {
 	if m.fullBCount == nil {
 		m.fullBCount = map[string]int{}
 		for _, s := range m.b {
-			m.fullBCount[s] = m.fullBCount[s] + 1
+			m.fullBCount[s]++
 		}
 	}
 

prometheus/internal/difflib_test.go

@@ -134,7 +134,7 @@ four`
 		Context:  3,
 	}
 	result, _ := GetUnifiedDiffString(diff)
-	fmt.Println(strings.Replace(result, "\t", " ", -1))
+	fmt.Println(strings.ReplaceAll(result, "\t", " "))
 	// Output:
 	// --- Original 2005-01-26 23:30:50
 	// +++ Current 2010-04-02 10:20:52

prometheus/internal/go_collector_options.go

@@ -0,0 +1,32 @@
+// Copyright 2021 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package internal
+
+import "regexp"
+
+type GoCollectorRule struct {
+	Matcher *regexp.Regexp
+	Deny    bool
+}
+
+// GoCollectorOptions should not be used be directly by anything, except `collectors` package.
+// Use it via collectors package instead. See issue
+// https://github.com/prometheus/client_golang/issues/1030.
+//
+// This is internal, so external users only can use it via `collector.WithGoCollector*` methods
+type GoCollectorOptions struct {
+	DisableMemStatsLikeMetrics bool
+	RuntimeMetricSumForHist    map[string]string
+	RuntimeMetricRules         []GoCollectorRule
+}

prometheus/internal/go_runtime_metrics.go

@@ -61,9 +61,9 @@ func RuntimeMetricsToProm(d *metrics.Description) (string, string, string, bool)
 	// name has - replaced with _ and is concatenated with the unit and
 	// other data.
 	name = strings.ReplaceAll(name, "-", "_")
-	name = name + "_" + unit
+	name += "_" + unit
 	if d.Cumulative && d.Kind != metrics.KindFloat64Histogram {
-		name = name + "_total"
+		name += "_total"
 	}
 
 	valid := model.IsValidMetricName(model.LabelValue(namespace + "_" + subsystem + "_" + name))

prometheus/labels.go

@@ -39,7 +39,7 @@ var errInconsistentCardinality = errors.New("inconsistent label cardinality")
 
 func makeInconsistentCardinalityError(fqName string, labels, labelValues []string) error {
 	return fmt.Errorf(
-		"%s: %q has %d variable labels named %q but %d values %q were provided",
+		"%w: %q has %d variable labels named %q but %d values %q were provided",
 		errInconsistentCardinality, fqName,
 		len(labels), labels,
 		len(labelValues), labelValues,
@@ -49,7 +49,7 @@ func makeInconsistentCardinalityError(fqName string, labels, labelValues []strin
 func validateValuesInLabels(labels Labels, expectedNumberOfValues int) error {
 	if len(labels) != expectedNumberOfValues {
 		return fmt.Errorf(
-			"%s: expected %d label values but got %d in %#v",
+			"%w: expected %d label values but got %d in %#v",
 			errInconsistentCardinality, expectedNumberOfValues,
 			len(labels), labels,
 		)
@@ -67,7 +67,7 @@ func validateValuesInLabels(labels Labels, expectedNumberOfValues int) error {
 func validateLabelValues(vals []string, expectedNumberOfValues int) error {
 	if len(vals) != expectedNumberOfValues {
 		return fmt.Errorf(
-			"%s: expected %d label values but got %d in %#v",
+			"%w: expected %d label values but got %d in %#v",
 			errInconsistentCardinality, expectedNumberOfValues,
 			len(vals), vals,
 		)

prometheus/metric.go

@@ -15,6 +15,7 @@ package prometheus
 
 import (
 	"errors"
+	"math"
 	"sort"
 	"strings"
 	"time"
@@ -184,8 +185,13 @@ func (m *withExemplarsMetric) Write(pb *dto.Metric) error {
 			if i < len(pb.Histogram.Bucket) {
 				pb.Histogram.Bucket[i].Exemplar = e
 			} else {
-				// This is not possible as last bucket is Inf.
-				panic("no bucket was found for given exemplar value")
+				// The +Inf bucket should be explicitly added if there is an exemplar for it, similar to non-const histogram logic in https://github.com/prometheus/client_golang/blob/main/prometheus/histogram.go#L357-L365.
+				b := &dto.Bucket{
+					CumulativeCount: proto.Uint64(pb.Histogram.Bucket[len(pb.Histogram.GetBucket())-1].GetCumulativeCount()),
+					UpperBound:      proto.Float64(math.Inf(1)),
+					Exemplar:        e,
+				}
+				pb.Histogram.Bucket = append(pb.Histogram.Bucket, b)
 			}
 		}
 	default:

prometheus/metric_test.go

@@ -14,6 +14,7 @@
 package prometheus
 
 import (
+	"math"
 	"testing"
 
 	//nolint:staticcheck // Ignore SA1019. Need to keep deprecated package for compatibility.
@@ -46,26 +47,29 @@ func TestWithExemplarsMetric(t *testing.T) {
 		h := MustNewConstHistogram(
 			NewDesc("http_request_duration_seconds", "A histogram of the HTTP request durations.", nil, nil),
 			4711, 403.34,
+			// Four buckets, but we expect five as the +Inf bucket will be created if we see value outside of those buckets.
 			map[float64]uint64{25: 121, 50: 2403, 100: 3221, 200: 4233},
 		)
 
 		m := &withExemplarsMetric{Metric: h, exemplars: []*dto.Exemplar{
+			{Value: proto.Float64(2000.0)}, // Unordered exemplars.
+			{Value: proto.Float64(500.0)},
+			{Value: proto.Float64(42.0)},
+			{Value: proto.Float64(157.0)},
+			{Value: proto.Float64(100.0)},
+			{Value: proto.Float64(89.0)},
 			{Value: proto.Float64(24.0)},
 			{Value: proto.Float64(25.1)},
-			{Value: proto.Float64(42.0)},
-			{Value: proto.Float64(89.0)},
-			{Value: proto.Float64(100.0)},
-			{Value: proto.Float64(157.0)},
 		}}
 		metric := dto.Metric{}
 		if err := m.Write(&metric); err != nil {
 			t.Fatal(err)
 		}
-		if want, got := 4, len(metric.GetHistogram().Bucket); want != got {
+		if want, got := 5, len(metric.GetHistogram().Bucket); want != got {
 			t.Errorf("want %v, got %v", want, got)
 		}
 
-		expectedExemplarVals := []float64{24.0, 42.0, 100.0, 157.0}
+		expectedExemplarVals := []float64{24.0, 25.1, 89.0, 157.0, 500.0}
 		for i, b := range metric.GetHistogram().Bucket {
 			if b.Exemplar == nil {
 				t.Errorf("Expected exemplar for bucket %v, got nil", i)
@@ -74,5 +78,11 @@ func TestWithExemplarsMetric(t *testing.T) {
 				t.Errorf("%v: want %v, got %v", i, want, got)
 			}
 		}
+
+		infBucket := metric.GetHistogram().Bucket[len(metric.GetHistogram().Bucket)-1].GetUpperBound()
+
+		if infBucket != math.Inf(1) {
+			t.Errorf("want %v, got %v", math.Inf(1), infBucket)
+		}
 	})
 }

prometheus/num_threads.go

@@ -0,0 +1,25 @@
+// Copyright 2018 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build !js || wasm
+// +build !js wasm
+
+package prometheus
+
+import "runtime"
+
+// getRuntimeNumThreads returns the number of open OS threads.
+func getRuntimeNumThreads() float64 {
+	n, _ := runtime.ThreadCreateProfile(nil)
+	return float64(n)
+}

prometheus/num_threads_gopherjs.go

@@ -0,0 +1,22 @@
+// Copyright 2018 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build js && !wasm
+// +build js,!wasm
+
+package prometheus
+
+// getRuntimeNumThreads returns the number of open OS threads.
+func getRuntimeNumThreads() float64 {
+	return 1
+}

prometheus/observer.go

@@ -58,7 +58,7 @@ type ObserverVec interface {
 // current time as timestamp, and the provided Labels. Empty Labels will lead to
 // a valid (label-less) exemplar. But if Labels is nil, the current exemplar is
 // left in place. ObserveWithExemplar panics if any of the provided labels are
-// invalid or if the provided labels contain more than 64 runes in total.
+// invalid or if the provided labels contain more than 128 runes in total.
 type ExemplarObserver interface {
 	ObserveWithExemplar(value float64, exemplar Labels)
 }

prometheus/process_collector.go

@@ -16,7 +16,6 @@ package prometheus
 import (
 	"errors"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"strconv"
 	"strings"
@@ -104,8 +103,7 @@ func NewProcessCollector(opts ProcessCollectorOpts) Collector {
 	}
 
 	if opts.PidFn == nil {
-		pid := os.Getpid()
-		c.pidFn = func() (int, error) { return pid, nil }
+		c.pidFn = getPIDFn()
 	} else {
 		c.pidFn = opts.PidFn
 	}
@@ -152,13 +150,13 @@ func (c *processCollector) reportError(ch chan<- Metric, desc *Desc, err error)
 // It is meant to be used for the PidFn field in ProcessCollectorOpts.
 func NewPidFileFn(pidFilePath string) func() (int, error) {
 	return func() (int, error) {
-		content, err := ioutil.ReadFile(pidFilePath)
+		content, err := os.ReadFile(pidFilePath)
 		if err != nil {
-			return 0, fmt.Errorf("can't read pid file %q: %+v", pidFilePath, err)
+			return 0, fmt.Errorf("can't read pid file %q: %w", pidFilePath, err)
 		}
 		pid, err := strconv.Atoi(strings.TrimSpace(string(content)))
 		if err != nil {
-			return 0, fmt.Errorf("can't parse pid file %q: %+v", pidFilePath, err)
+			return 0, fmt.Errorf("can't parse pid file %q: %w", pidFilePath, err)
 		}
 
 		return pid, nil

prometheus/process_collector_js.go

@@ -0,0 +1,26 @@
+// Copyright 2019 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build js
+// +build js
+
+package prometheus
+
+func canCollectProcess() bool {
+	return false
+}
+
+func (c *processCollector) processCollect(ch chan<- Metric) {
+	// noop on this platform
+	return
+}

prometheus/process_collector_other.go

@@ -11,8 +11,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-//go:build !windows
-// +build !windows
+//go:build !windows && !js
+// +build !windows,!js
 
 package prometheus
 

prometheus/promhttp/http.go

@@ -33,6 +33,7 @@ package promhttp
 
 import (
 	"compress/gzip"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -110,7 +111,8 @@ func HandlerForTransactional(reg prometheus.TransactionalGatherer, opts HandlerO
 		errCnt.WithLabelValues("gathering")
 		errCnt.WithLabelValues("encoding")
 		if err := opts.Registry.Register(errCnt); err != nil {
-			if are, ok := err.(prometheus.AlreadyRegisteredError); ok {
+			are := &prometheus.AlreadyRegisteredError{}
+			if errors.As(err, are) {
 				errCnt = are.ExistingCollector.(*prometheus.CounterVec)
 			} else {
 				panic(err)
@@ -250,7 +252,8 @@ func InstrumentMetricHandler(reg prometheus.Registerer, handler http.Handler) ht
 	cnt.WithLabelValues("500")
 	cnt.WithLabelValues("503")
 	if err := reg.Register(cnt); err != nil {
-		if are, ok := err.(prometheus.AlreadyRegisteredError); ok {
+		are := &prometheus.AlreadyRegisteredError{}
+		if errors.As(err, are) {
 			cnt = are.ExistingCollector.(*prometheus.CounterVec)
 		} else {
 			panic(err)
@@ -262,7 +265,8 @@ func InstrumentMetricHandler(reg prometheus.Registerer, handler http.Handler) ht
 		Help: "Current number of scrapes being served.",
 	})
 	if err := reg.Register(gge); err != nil {
-		if are, ok := err.(prometheus.AlreadyRegisteredError); ok {
+		are := &prometheus.AlreadyRegisteredError{}
+		if errors.As(err, are) {
 			gge = are.ExistingCollector.(prometheus.Gauge)
 		} else {
 			panic(err)

prometheus/promhttp/instrument_client.go

@@ -38,11 +38,11 @@ func (rt RoundTripperFunc) RoundTrip(r *http.Request) (*http.Response, error) {
 //
 // See the example for ExampleInstrumentRoundTripperDuration for example usage.
 func InstrumentRoundTripperInFlight(gauge prometheus.Gauge, next http.RoundTripper) RoundTripperFunc {
-	return RoundTripperFunc(func(r *http.Request) (*http.Response, error) {
+	return func(r *http.Request) (*http.Response, error) {
 		gauge.Inc()
 		defer gauge.Dec()
 		return next.RoundTrip(r)
-	})
+	}
 }
 
 // InstrumentRoundTripperCounter is a middleware that wraps the provided
@@ -59,22 +59,28 @@ func InstrumentRoundTripperInFlight(gauge prometheus.Gauge, next http.RoundTripp
 // If the wrapped RoundTripper panics or returns a non-nil error, the Counter
 // is not incremented.
 //
+// Use with WithExemplarFromContext to instrument the exemplars on the counter of requests.
+//
 // See the example for ExampleInstrumentRoundTripperDuration for example usage.
 func InstrumentRoundTripperCounter(counter *prometheus.CounterVec, next http.RoundTripper, opts ...Option) RoundTripperFunc {
-	rtOpts := &option{}
+	rtOpts := defaultOptions()
 	for _, o := range opts {
-		o(rtOpts)
+		o.apply(rtOpts)
 	}
 
 	code, method := checkLabels(counter)
 
-	return RoundTripperFunc(func(r *http.Request) (*http.Response, error) {
+	return func(r *http.Request) (*http.Response, error) {
 		resp, err := next.RoundTrip(r)
 		if err == nil {
-			counter.With(labels(code, method, r.Method, resp.StatusCode, rtOpts.extraMethods...)).Inc()
+			addWithExemplar(
+				counter.With(labels(code, method, r.Method, resp.StatusCode, rtOpts.extraMethods...)),
+				1,
+				rtOpts.getExemplarFn(r.Context()),
+			)
 		}
 		return resp, err
-	})
+	}
 }
 
 // InstrumentRoundTripperDuration is a middleware that wraps the provided
@@ -94,24 +100,30 @@ func InstrumentRoundTripperCounter(counter *prometheus.CounterVec, next http.Rou
 // If the wrapped RoundTripper panics or returns a non-nil error, no values are
 // reported.
 //
+// Use with WithExemplarFromContext to instrument the exemplars on the duration histograms.
+//
 // Note that this method is only guaranteed to never observe negative durations
 // if used with Go1.9+.
 func InstrumentRoundTripperDuration(obs prometheus.ObserverVec, next http.RoundTripper, opts ...Option) RoundTripperFunc {
-	rtOpts := &option{}
+	rtOpts := defaultOptions()
 	for _, o := range opts {
-		o(rtOpts)
+		o.apply(rtOpts)
 	}
 
 	code, method := checkLabels(obs)
 
-	return RoundTripperFunc(func(r *http.Request) (*http.Response, error) {
+	return func(r *http.Request) (*http.Response, error) {
 		start := time.Now()
 		resp, err := next.RoundTrip(r)
 		if err == nil {
-			obs.With(labels(code, method, r.Method, resp.StatusCode, rtOpts.extraMethods...)).Observe(time.Since(start).Seconds())
+			observeWithExemplar(
+				obs.With(labels(code, method, r.Method, resp.StatusCode, rtOpts.extraMethods...)),
+				time.Since(start).Seconds(),
+				rtOpts.getExemplarFn(r.Context()),
+			)
 		}
 		return resp, err
-	})
+	}
 }
 
 // InstrumentTrace is used to offer flexibility in instrumenting the available
@@ -149,7 +161,7 @@ type InstrumentTrace struct {
 //
 // See the example for ExampleInstrumentRoundTripperDuration for example usage.
 func InstrumentRoundTripperTrace(it *InstrumentTrace, next http.RoundTripper) RoundTripperFunc {
-	return RoundTripperFunc(func(r *http.Request) (*http.Response, error) {
+	return func(r *http.Request) (*http.Response, error) {
 		start := time.Now()
 
 		trace := &httptrace.ClientTrace{
@@ -231,5 +243,5 @@ func InstrumentRoundTripperTrace(it *InstrumentTrace, next http.RoundTripper) Ro
 		r = r.WithContext(httptrace.WithClientTrace(r.Context(), trace))
 
 		return next.RoundTrip(r)
-	})
+	}
 }

prometheus/promhttp/instrument_client_test.go

@@ -18,14 +18,20 @@ import (
 	"log"
 	"net/http"
 	"net/http/httptest"
+	"reflect"
+	"sort"
 	"strings"
 	"testing"
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/testutil"
+
+	dto "github.com/prometheus/client_model/go"
+	"google.golang.org/protobuf/proto"
 )
 
-func makeInstrumentedClient() (*http.Client, *prometheus.Registry) {
+func makeInstrumentedClient(opts ...Option) (*http.Client, *prometheus.Registry) {
 	client := http.DefaultClient
 	client.Timeout = 1 * time.Second
 
@@ -91,13 +97,91 @@ func makeInstrumentedClient() (*http.Client, *prometheus.Registry) {
 	client.Transport = InstrumentRoundTripperInFlight(inFlightGauge,
 		InstrumentRoundTripperCounter(counter,
 			InstrumentRoundTripperTrace(trace,
-				InstrumentRoundTripperDuration(histVec, http.DefaultTransport),
-			),
+				InstrumentRoundTripperDuration(histVec, http.DefaultTransport, opts...),
 			),
+			opts...),
 	)
 	return client, reg
 }
 
+func labelsToLabelPair(l prometheus.Labels) []*dto.LabelPair {
+	ret := make([]*dto.LabelPair, 0, len(l))
+	for k, v := range l {
+		ret = append(ret, &dto.LabelPair{Name: proto.String(k), Value: proto.String(v)})
+	}
+	sort.Slice(ret, func(i, j int) bool {
+		return *ret[i].Name < *ret[j].Name
+	})
+	return ret
+}
+
+func assetMetricAndExemplars(
+	t *testing.T,
+	reg *prometheus.Registry,
+	expectedNumMetrics int,
+	expectedExemplar []*dto.LabelPair,
+) {
+	t.Helper()
+
+	mfs, err := reg.Gather()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if want, got := expectedNumMetrics, len(mfs); want != got {
+		t.Fatalf("unexpected number of metric families gathered, want %d, got %d", want, got)
+	}
+
+	for _, mf := range mfs {
+		if len(mf.Metric) == 0 {
+			t.Errorf("metric family %s must not be empty", mf.GetName())
+		}
+		for _, m := range mf.GetMetric() {
+			if c := m.GetCounter(); c != nil {
+				if len(expectedExemplar) == 0 {
+					if c.Exemplar != nil {
+						t.Errorf("expected no exemplar on the counter %v%v, got %v", mf.GetName(), m.Label, c.Exemplar.String())
+					}
+					continue
+				}
+
+				if c.Exemplar == nil {
+					t.Errorf("expected exemplar %v on the counter %v%v, got none", expectedExemplar, mf.GetName(), m.Label)
+					continue
+				}
+				if got := c.Exemplar.Label; !reflect.DeepEqual(expectedExemplar, got) {
+					t.Errorf("expected exemplar %v on the counter %v%v, got %v", expectedExemplar, mf.GetName(), m.Label, got)
+				}
+				continue
+			}
+			if h := m.GetHistogram(); h != nil {
+				found := false
+				for _, b := range h.GetBucket() {
+					if len(expectedExemplar) == 0 {
+						if b.Exemplar != nil {
+							t.Errorf("expected no exemplar on histogram %v%v bkt %v, got %v", mf.GetName(), m.Label, b.GetUpperBound(), b.Exemplar.String())
+						}
+						continue
+					}
+
+					if b.Exemplar == nil {
+						continue
+					}
+					if got := b.Exemplar.Label; !reflect.DeepEqual(expectedExemplar, got) {
+						t.Errorf("expected exemplar %v on the histogram %v%v on bkt %v, got %v", expectedExemplar, mf.GetName(), m.Label, b.GetUpperBound(), got)
+						continue
+					}
+					found = true
+					break
+				}
+
+				if len(expectedExemplar) > 0 && !found {
+					t.Errorf("expected exemplar %v on at least one bucket of the histogram %v%v, got none", expectedExemplar, mf.GetName(), m.Label)
+				}
+			}
+		}
+	}
+}
+
 func TestClientMiddlewareAPI(t *testing.T) {
 	client, reg := makeInstrumentedClient()
 	backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -111,21 +195,28 @@ func TestClientMiddlewareAPI(t *testing.T) {
 	}
 	defer resp.Body.Close()
 
-	mfs, err := reg.Gather()
+	assetMetricAndExemplars(t, reg, 3, nil)
+}
+
+func TestClientMiddlewareAPI_WithExemplars(t *testing.T) {
+	exemplar := prometheus.Labels{"traceID": "example situation observed by this metric"}
+
+	client, reg := makeInstrumentedClient(WithExemplarFromContext(func(_ context.Context) prometheus.Labels { return exemplar }))
+	backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+	}))
+	defer backend.Close()
+
+	resp, err := client.Get(backend.URL)
 	if err != nil {
 		t.Fatal(err)
 	}
-	if want, got := 3, len(mfs); want != got {
-		t.Fatalf("unexpected number of metric families gathered, want %d, got %d", want, got)
-	}
-	for _, mf := range mfs {
-		if len(mf.Metric) == 0 {
-			t.Errorf("metric family %s must not be empty", mf.GetName())
-		}
-	}
+	defer resp.Body.Close()
+
+	assetMetricAndExemplars(t, reg, 3, labelsToLabelPair(exemplar))
 }
 
-func TestClientMiddlewareAPIWithRequestContext(t *testing.T) {
+func TestClientMiddlewareAPI_WithRequestContext(t *testing.T) {
 	client, reg := makeInstrumentedClient()
 	backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		w.WriteHeader(http.StatusOK)
@@ -160,6 +251,19 @@ func TestClientMiddlewareAPIWithRequestContext(t *testing.T) {
 			t.Errorf("metric family %s must not be empty", mf.GetName())
 		}
 	}
+
+	// make sure counters aren't double-incremented (see #1117)
+	expected := `
+		# HELP client_api_requests_total A counter for requests from the wrapped client.
+		# TYPE client_api_requests_total counter
+		client_api_requests_total{code="200",method="get"} 1
+	`
+
+	if err := testutil.GatherAndCompare(reg, strings.NewReader(expected),
+		"client_api_requests_total",
+	); err != nil {
+		t.Fatal(err)
+	}
 }
 
 func TestClientMiddlewareAPIWithRequestContextTimeout(t *testing.T) {

prometheus/promhttp/instrument_server.go

@@ -28,6 +28,26 @@ import (
 // magicString is used for the hacky label test in checkLabels. Remove once fixed.
 const magicString = "zZgWfBxLqvG8kc8IMv3POi2Bb0tZI3vAnBx+gBaFi9FyPzB/CzKUer1yufDa"
 
+// observeWithExemplar is a wrapper for [prometheus.ExemplarAdder.ExemplarObserver],
+// which falls back to [prometheus.Observer.Observe] if no labels are provided.
+func observeWithExemplar(obs prometheus.Observer, val float64, labels map[string]string) {
+	if labels == nil {
+		obs.Observe(val)
+		return
+	}
+	obs.(prometheus.ExemplarObserver).ObserveWithExemplar(val, labels)
+}
+
+// addWithExemplar is a wrapper for [prometheus.ExemplarAdder.AddWithExemplar],
+// which falls back to [prometheus.Counter.Add] if no labels are provided.
+func addWithExemplar(obs prometheus.Counter, val float64, labels map[string]string) {
+	if labels == nil {
+		obs.Add(val)
+		return
+	}
+	obs.(prometheus.ExemplarAdder).AddWithExemplar(val, labels)
+}
+
 // InstrumentHandlerInFlight is a middleware that wraps the provided
 // http.Handler. It sets the provided prometheus.Gauge to the number of
 // requests currently handled by the wrapped http.Handler.
@@ -62,28 +82,37 @@ func InstrumentHandlerInFlight(g prometheus.Gauge, next http.Handler) http.Handl
 // Note that this method is only guaranteed to never observe negative durations
 // if used with Go1.9+.
 func InstrumentHandlerDuration(obs prometheus.ObserverVec, next http.Handler, opts ...Option) http.HandlerFunc {
-	mwOpts := &option{}
+	hOpts := defaultOptions()
 	for _, o := range opts {
-		o(mwOpts)
+		o.apply(hOpts)
 	}
 
 	code, method := checkLabels(obs)
 
 	if code {
-		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		return func(w http.ResponseWriter, r *http.Request) {
 			now := time.Now()
 			d := newDelegator(w, nil)
 			next.ServeHTTP(d, r)
 
-			obs.With(labels(code, method, r.Method, d.Status(), mwOpts.extraMethods...)).Observe(time.Since(now).Seconds())
-		})
+			observeWithExemplar(
+				obs.With(labels(code, method, r.Method, d.Status(), hOpts.extraMethods...)),
+				time.Since(now).Seconds(),
+				hOpts.getExemplarFn(r.Context()),
+			)
+		}
 	}
 
-	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+	return func(w http.ResponseWriter, r *http.Request) {
 		now := time.Now()
 		next.ServeHTTP(w, r)
-		obs.With(labels(code, method, r.Method, 0, mwOpts.extraMethods...)).Observe(time.Since(now).Seconds())
-	})
+
+		observeWithExemplar(
+			obs.With(labels(code, method, r.Method, 0, hOpts.extraMethods...)),
+			time.Since(now).Seconds(),
+			hOpts.getExemplarFn(r.Context()),
+		)
+	}
 }
 
 // InstrumentHandlerCounter is a middleware that wraps the provided http.Handler
@@ -104,25 +133,34 @@ func InstrumentHandlerDuration(obs prometheus.ObserverVec, next http.Handler, op
 //
 // See the example for InstrumentHandlerDuration for example usage.
 func InstrumentHandlerCounter(counter *prometheus.CounterVec, next http.Handler, opts ...Option) http.HandlerFunc {
-	mwOpts := &option{}
+	hOpts := defaultOptions()
 	for _, o := range opts {
-		o(mwOpts)
+		o.apply(hOpts)
 	}
 
 	code, method := checkLabels(counter)
 
 	if code {
-		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		return func(w http.ResponseWriter, r *http.Request) {
 			d := newDelegator(w, nil)
 			next.ServeHTTP(d, r)
-			counter.With(labels(code, method, r.Method, d.Status(), mwOpts.extraMethods...)).Inc()
-		})
+
+			addWithExemplar(
+				counter.With(labels(code, method, r.Method, d.Status(), hOpts.extraMethods...)),
+				1,
+				hOpts.getExemplarFn(r.Context()),
+			)
+		}
 	}
 
-	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+	return func(w http.ResponseWriter, r *http.Request) {
 		next.ServeHTTP(w, r)
-		counter.With(labels(code, method, r.Method, 0, mwOpts.extraMethods...)).Inc()
-	})
+		addWithExemplar(
+			counter.With(labels(code, method, r.Method, 0, hOpts.extraMethods...)),
+			1,
+			hOpts.getExemplarFn(r.Context()),
+		)
+	}
 }
 
 // InstrumentHandlerTimeToWriteHeader is a middleware that wraps the provided
@@ -148,20 +186,24 @@ func InstrumentHandlerCounter(counter *prometheus.CounterVec, next http.Handler,
 //
 // See the example for InstrumentHandlerDuration for example usage.
 func InstrumentHandlerTimeToWriteHeader(obs prometheus.ObserverVec, next http.Handler, opts ...Option) http.HandlerFunc {
-	mwOpts := &option{}
+	hOpts := defaultOptions()
 	for _, o := range opts {
-		o(mwOpts)
+		o.apply(hOpts)
 	}
 
 	code, method := checkLabels(obs)
 
-	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+	return func(w http.ResponseWriter, r *http.Request) {
 		now := time.Now()
 		d := newDelegator(w, func(status int) {
-			obs.With(labels(code, method, r.Method, status, mwOpts.extraMethods...)).Observe(time.Since(now).Seconds())
+			observeWithExemplar(
+				obs.With(labels(code, method, r.Method, status, hOpts.extraMethods...)),
+				time.Since(now).Seconds(),
+				hOpts.getExemplarFn(r.Context()),
+			)
 		})
 		next.ServeHTTP(d, r)
-	})
+	}
 }
 
 // InstrumentHandlerRequestSize is a middleware that wraps the provided
@@ -184,27 +226,34 @@ func InstrumentHandlerTimeToWriteHeader(obs prometheus.ObserverVec, next http.Ha
 //
 // See the example for InstrumentHandlerDuration for example usage.
 func InstrumentHandlerRequestSize(obs prometheus.ObserverVec, next http.Handler, opts ...Option) http.HandlerFunc {
-	mwOpts := &option{}
+	hOpts := defaultOptions()
 	for _, o := range opts {
-		o(mwOpts)
+		o.apply(hOpts)
 	}
 
 	code, method := checkLabels(obs)
-
 	if code {
-		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		return func(w http.ResponseWriter, r *http.Request) {
 			d := newDelegator(w, nil)
 			next.ServeHTTP(d, r)
 			size := computeApproximateRequestSize(r)
-			obs.With(labels(code, method, r.Method, d.Status(), mwOpts.extraMethods...)).Observe(float64(size))
-		})
+			observeWithExemplar(
+				obs.With(labels(code, method, r.Method, d.Status(), hOpts.extraMethods...)),
+				float64(size),
+				hOpts.getExemplarFn(r.Context()),
+			)
+		}
 	}
 
-	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+	return func(w http.ResponseWriter, r *http.Request) {
 		next.ServeHTTP(w, r)
 		size := computeApproximateRequestSize(r)
-		obs.With(labels(code, method, r.Method, 0, mwOpts.extraMethods...)).Observe(float64(size))
-	})
+		observeWithExemplar(
+			obs.With(labels(code, method, r.Method, 0, hOpts.extraMethods...)),
+			float64(size),
+			hOpts.getExemplarFn(r.Context()),
+		)
+	}
 }
 
 // InstrumentHandlerResponseSize is a middleware that wraps the provided
@@ -227,9 +276,9 @@ func InstrumentHandlerRequestSize(obs prometheus.ObserverVec, next http.Handler,
 //
 // See the example for InstrumentHandlerDuration for example usage.
 func InstrumentHandlerResponseSize(obs prometheus.ObserverVec, next http.Handler, opts ...Option) http.Handler {
-	mwOpts := &option{}
+	hOpts := defaultOptions()
 	for _, o := range opts {
-		o(mwOpts)
+		o.apply(hOpts)
 	}
 
 	code, method := checkLabels(obs)
@@ -237,7 +286,11 @@ func InstrumentHandlerResponseSize(obs prometheus.ObserverVec, next http.Handler
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		d := newDelegator(w, nil)
 		next.ServeHTTP(d, r)
-		obs.With(labels(code, method, r.Method, d.Status(), mwOpts.extraMethods...)).Observe(float64(d.Written()))
+		observeWithExemplar(
+			obs.With(labels(code, method, r.Method, d.Status(), hOpts.extraMethods...)),
+			float64(d.Written()),
+			hOpts.getExemplarFn(r.Context()),
+		)
 	})
 }
 

prometheus/promhttp/instrument_server_test.go

@@ -14,6 +14,7 @@
 package promhttp
 
 import (
+	"context"
 	"io"
 	"log"
 	"net/http"
@@ -145,7 +146,6 @@ func TestLabelCheck(t *testing.T) {
 				},
 				append(sc.varLabels, sc.curriedLabels...),
 			))
-			//nolint:typecheck // Ignore declared but unused error.
 			for _, l := range sc.curriedLabels {
 				c = c.MustCurryWith(prometheus.Labels{l: "dummy"})
 				o = o.MustCurryWith(prometheus.Labels{l: "dummy"})
@@ -321,7 +321,7 @@ func TestLabels(t *testing.T) {
 	}
 }
 
-func TestMiddlewareAPI(t *testing.T) {
+func makeInstrumentedHandler(handler http.HandlerFunc, opts ...Option) (http.Handler, *prometheus.Registry) {
 	reg := prometheus.NewRegistry()
 
 	inFlightGauge := prometheus.NewGauge(prometheus.GaugeOpts{
@@ -366,25 +366,43 @@ func TestMiddlewareAPI(t *testing.T) {
 		[]string{},
 	)
 
-	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		w.Write([]byte("OK"))
-	})
-
 	reg.MustRegister(inFlightGauge, counter, histVec, responseSize, writeHeaderVec)
 
-	chain := InstrumentHandlerInFlight(inFlightGauge,
+	return InstrumentHandlerInFlight(inFlightGauge,
 		InstrumentHandlerCounter(counter,
 			InstrumentHandlerDuration(histVec,
 				InstrumentHandlerTimeToWriteHeader(writeHeaderVec,
-					InstrumentHandlerResponseSize(responseSize, handler),
-				),
-			),
-		),
-	)
+					InstrumentHandlerResponseSize(responseSize, handler, opts...),
+					opts...),
+				opts...),
+			opts...),
+	), reg
+}
+
+func TestMiddlewareAPI(t *testing.T) {
+	chain, reg := makeInstrumentedHandler(func(w http.ResponseWriter, r *http.Request) {
+		_, _ = w.Write([]byte("OK"))
+	})
 
 	r, _ := http.NewRequest("GET", "www.example.com", nil)
 	w := httptest.NewRecorder()
 	chain.ServeHTTP(w, r)
+
+	assetMetricAndExemplars(t, reg, 5, nil)
+}
+
+func TestMiddlewareAPI_WithExemplars(t *testing.T) {
+	exemplar := prometheus.Labels{"traceID": "example situation observed by this metric"}
+
+	chain, reg := makeInstrumentedHandler(func(w http.ResponseWriter, r *http.Request) {
+		_, _ = w.Write([]byte("OK"))
+	}, WithExemplarFromContext(func(_ context.Context) prometheus.Labels { return exemplar }))
+
+	r, _ := http.NewRequest("GET", "www.example.com", nil)
+	w := httptest.NewRecorder()
+	chain.ServeHTTP(w, r)
+
+	assetMetricAndExemplars(t, reg, 5, labelsToLabelPair(exemplar))
 }
 
 func TestInstrumentTimeToFirstWrite(t *testing.T) {

prometheus/promhttp/option.go

@@ -13,19 +13,46 @@
 
 package promhttp
 
-// Option are used to configure a middleware or round tripper..
-type Option func(*option)
+import (
+	"context"
 
-type option struct {
-	extraMethods []string
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+// Option are used to configure both handler (middleware) or round tripper.
+type Option interface {
+	apply(*options)
 }
 
+// options store options for both a handler or round tripper.
+type options struct {
+	extraMethods  []string
+	getExemplarFn func(requestCtx context.Context) prometheus.Labels
+}
+
+func defaultOptions() *options {
+	return &options{getExemplarFn: func(ctx context.Context) prometheus.Labels { return nil }}
+}
+
+type optionApplyFunc func(*options)
+
+func (o optionApplyFunc) apply(opt *options) { o(opt) }
+
 // WithExtraMethods adds additional HTTP methods to the list of allowed methods.
 // See https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods for the default list.
 //
 // See the example for ExampleInstrumentHandlerWithExtraMethods for example usage.
 func WithExtraMethods(methods ...string) Option {
-	return func(o *option) {
+	return optionApplyFunc(func(o *options) {
 		o.extraMethods = methods
+	})
 }
+
+// WithExemplarFromContext adds allows to put a hook to all counter and histogram metrics.
+// If the hook function returns non-nil labels, exemplars will be added for that request, otherwise metric
+// will get instrumented without exemplar.
+func WithExemplarFromContext(getExemplarFn func(requestCtx context.Context) prometheus.Labels) Option {
+	return optionApplyFunc(func(o *options) {
+		o.getExemplarFn = getExemplarFn
+	})
 }

prometheus/push/push.go

@@ -40,7 +40,7 @@ import (
 	"encoding/base64"
 	"errors"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"net/url"
 	"strings"
@@ -245,7 +245,7 @@ func (p *Pusher) Delete() error {
 	}
 	defer resp.Body.Close()
 	if resp.StatusCode != http.StatusAccepted {
-		body, _ := ioutil.ReadAll(resp.Body) // Ignore any further error as this is for an error message only.
+		body, _ := io.ReadAll(resp.Body) // Ignore any further error as this is for an error message only.
 		return fmt.Errorf("unexpected status code %d while deleting %s: %s", resp.StatusCode, p.fullURL(), body)
 	}
 	return nil
@@ -297,7 +297,7 @@ func (p *Pusher) push(ctx context.Context, method string) error {
 	defer resp.Body.Close()
 	// Depending on version and configuration of the PGW, StatusOK or StatusAccepted may be returned.
 	if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted {
-		body, _ := ioutil.ReadAll(resp.Body) // Ignore any further error as this is for an error message only.
+		body, _ := io.ReadAll(resp.Body) // Ignore any further error as this is for an error message only.
 		return fmt.Errorf("unexpected status code %d while pushing to %s: %s", resp.StatusCode, p.fullURL(), body)
 	}
 	return nil

prometheus/push/push_test.go

@@ -15,7 +15,8 @@ package push
 
 import (
 	"bytes"
-	"io/ioutil"
+	"errors"
+	"io"
 	"net/http"
 	"net/http/httptest"
 	"testing"
@@ -38,7 +39,7 @@ func TestPush(t *testing.T) {
 		http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 			lastMethod = r.Method
 			var err error
-			lastBody, err = ioutil.ReadAll(r.Body)
+			lastBody, err = io.ReadAll(r.Body)
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -200,8 +201,8 @@ func TestPush(t *testing.T) {
 		Push(); err == nil {
 		t.Error("push with empty job succeeded")
 	} else {
-		if got, want := err, errJobEmpty; got != want {
-			t.Errorf("got error %q, want %q", got, want)
+		if want := errJobEmpty; !errors.Is(err, want) {
+			t.Errorf("got error %q, want %q", err, want)
 		}
 	}
 

prometheus/registry.go

@@ -15,8 +15,8 @@ package prometheus
 
 import (
 	"bytes"
+	"errors"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"runtime"
@@ -252,9 +252,12 @@ func (errs MultiError) MaybeUnwrap() error {
 }
 
 // Registry registers Prometheus collectors, collects their metrics, and gathers
-// them into MetricFamilies for exposition. It implements both Registerer and
-// Gatherer. The zero value is not usable. Create instances with NewRegistry or
-// NewPedanticRegistry.
+// them into MetricFamilies for exposition. It implements Registerer, Gatherer,
+// and Collector. The zero value is not usable. Create instances with
+// NewRegistry or NewPedanticRegistry.
+//
+// Registry implements Collector to allow it to be used for creating groups of
+// metrics. See the Grouping example for how this can be done.
 type Registry struct {
 	mtx                   sync.RWMutex
 	collectorsByID        map[uint64]Collector // ID is a hash of the descIDs.
@@ -289,7 +292,7 @@ func (r *Registry) Register(c Collector) error {
 
 		// Is the descriptor valid at all?
 		if desc.err != nil {
-			return fmt.Errorf("descriptor %s is invalid: %s", desc, desc.err)
+			return fmt.Errorf("descriptor %s is invalid: %w", desc, desc.err)
 		}
 
 		// Is the descID unique?
@@ -556,6 +559,31 @@ func (r *Registry) Gather() ([]*dto.MetricFamily, error) {
 	return internal.NormalizeMetricFamilies(metricFamiliesByName), errs.MaybeUnwrap()
 }
 
+// Describe implements Collector.
+func (r *Registry) Describe(ch chan<- *Desc) {
+	r.mtx.RLock()
+	defer r.mtx.RUnlock()
+
+	// Only report the checked Collectors; unchecked collectors don't report any
+	// Desc.
+	for _, c := range r.collectorsByID {
+		c.Describe(ch)
+	}
+}
+
+// Collect implements Collector.
+func (r *Registry) Collect(ch chan<- Metric) {
+	r.mtx.RLock()
+	defer r.mtx.RUnlock()
+
+	for _, c := range r.collectorsByID {
+		c.Collect(ch)
+	}
+	for _, c := range r.uncheckedCollectors {
+		c.Collect(ch)
+	}
+}
+
 // WriteToTextfile calls Gather on the provided Gatherer, encodes the result in the
 // Prometheus text format, and writes it to a temporary file. Upon success, the
 // temporary file is renamed to the provided filename.
@@ -563,7 +591,7 @@ func (r *Registry) Gather() ([]*dto.MetricFamily, error) {
 // This is intended for use with the textfile collector of the node exporter.
 // Note that the node exporter expects the filename to be suffixed with ".prom".
 func WriteToTextfile(filename string, g Gatherer) error {
-	tmp, err := ioutil.TempFile(filepath.Dir(filename), filepath.Base(filename))
+	tmp, err := os.CreateTemp(filepath.Dir(filename), filepath.Base(filename))
 	if err != nil {
 		return err
 	}
@@ -603,7 +631,7 @@ func processMetric(
 	}
 	dtoMetric := &dto.Metric{}
 	if err := metric.Write(dtoMetric); err != nil {
-		return fmt.Errorf("error collecting metric %v: %s", desc, err)
+		return fmt.Errorf("error collecting metric %v: %w", desc, err)
 	}
 	metricFamily, ok := metricFamiliesByName[desc.fqName]
 	if ok { // Existing name.
@@ -725,12 +753,13 @@ func (gs Gatherers) Gather() ([]*dto.MetricFamily, error) {
 	for i, g := range gs {
 		mfs, err := g.Gather()
 		if err != nil {
-			if multiErr, ok := err.(MultiError); ok {
+			multiErr := MultiError{}
+			if errors.As(err, &multiErr) {
 				for _, err := range multiErr {
-					errs = append(errs, fmt.Errorf("[from Gatherer #%d] %s", i+1, err))
+					errs = append(errs, fmt.Errorf("[from Gatherer #%d] %w", i+1, err))
 				}
 			} else {
-				errs = append(errs, fmt.Errorf("[from Gatherer #%d] %s", i+1, err))
+				errs = append(errs, fmt.Errorf("[from Gatherer #%d] %w", i+1, err))
 			}
 		}
 		for _, mf := range mfs {

prometheus/registry_test.go

@@ -23,7 +23,6 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
-	"io/ioutil"
 	"math/rand"
 	"net/http"
 	"net/http/httptest"
@@ -121,8 +120,8 @@ metric: <
 >
 
 `)
-	externalMetricFamilyAsProtoCompactText := []byte(`name:"externalname" help:"externaldocstring" type:COUNTER metric:<label:<name:"externalconstname" value:"externalconstvalue" > label:<name:"externallabelname" value:"externalval1" > counter:<value:1 > > 
-`)
+	externalMetricFamilyAsProtoCompactText := []byte(`name:"externalname" help:"externaldocstring" type:COUNTER metric:<label:<name:"externalconstname" value:"externalconstvalue" > label:<name:"externallabelname" value:"externalval1" > counter:<value:1 > >`)
+	externalMetricFamilyAsProtoCompactText = append(externalMetricFamilyAsProtoCompactText, []byte(" \n")...)
 
 	expectedMetricFamily := &dto.MetricFamily{
 		Name: proto.String("name"),
@@ -203,8 +202,8 @@ metric: <
 >
 
 `)
-	expectedMetricFamilyAsProtoCompactText := []byte(`name:"name" help:"docstring" type:COUNTER metric:<label:<name:"constname" value:"constvalue" > label:<name:"labelname" value:"val1" > counter:<value:1 > > metric:<label:<name:"constname" value:"constvalue" > label:<name:"labelname" value:"val2" > counter:<value:1 > > 
-`)
+	expectedMetricFamilyAsProtoCompactText := []byte(`name:"name" help:"docstring" type:COUNTER metric:<label:<name:"constname" value:"constvalue" > label:<name:"labelname" value:"val1" > counter:<value:1 > > metric:<label:<name:"constname" value:"constvalue" > label:<name:"labelname" value:"val2" > counter:<value:1 > >`)
+	expectedMetricFamilyAsProtoCompactText = append(expectedMetricFamilyAsProtoCompactText, []byte(" \n")...)
 
 	externalMetricFamilyWithSameName := &dto.MetricFamily{
 		Name: proto.String("name"),
@@ -229,8 +228,8 @@ metric: <
 		},
 	}
 
-	expectedMetricFamilyMergedWithExternalAsProtoCompactText := []byte(`name:"name" help:"docstring" type:COUNTER metric:<label:<name:"constname" value:"constvalue" > label:<name:"labelname" value:"different_val" > counter:<value:42 > > metric:<label:<name:"constname" value:"constvalue" > label:<name:"labelname" value:"val1" > counter:<value:1 > > metric:<label:<name:"constname" value:"constvalue" > label:<name:"labelname" value:"val2" > counter:<value:1 > > 
-`)
+	expectedMetricFamilyMergedWithExternalAsProtoCompactText := []byte(`name:"name" help:"docstring" type:COUNTER metric:<label:<name:"constname" value:"constvalue" > label:<name:"labelname" value:"different_val" > counter:<value:42 > > metric:<label:<name:"constname" value:"constvalue" > label:<name:"labelname" value:"val1" > counter:<value:1 > > metric:<label:<name:"constname" value:"constvalue" > label:<name:"labelname" value:"val2" > counter:<value:1 > >`)
+	expectedMetricFamilyMergedWithExternalAsProtoCompactText = append(expectedMetricFamilyMergedWithExternalAsProtoCompactText, []byte(" \n")...)
 
 	externalMetricFamilyWithInvalidLabelValue := &dto.MetricFamily{
 		Name: proto.String("name"),
@@ -851,7 +850,8 @@ func TestAlreadyRegistered(t *testing.T) {
 			if err = s.reRegisterWith(reg).Register(s.newCollector); err == nil {
 				t.Fatal("expected error when registering new collector")
 			}
-			if are, ok := err.(prometheus.AlreadyRegisteredError); ok {
+			are := &prometheus.AlreadyRegisteredError{}
+			if errors.As(err, are) {
 				if are.ExistingCollector != s.originalCollector {
 					t.Error("expected original collector but got something else")
 				}
@@ -932,7 +932,7 @@ func TestHistogramVecRegisterGatherConcurrency(t *testing.T) {
 				return
 			default:
 				if err := reg.Register(hv); err != nil {
-					if _, ok := err.(prometheus.AlreadyRegisteredError); !ok {
+					if !errors.As(err, &prometheus.AlreadyRegisteredError{}) {
 						t.Error("Registering failed:", err)
 					}
 				}
@@ -1066,7 +1066,7 @@ test_summary_count{name="foo"} 2
 	gauge.With(prometheus.Labels{"name": "baz"}).Set(1.1)
 	counter.With(prometheus.Labels{"name": "qux"}).Inc()
 
-	tmpfile, err := ioutil.TempFile("", "prom_registry_test")
+	tmpfile, err := os.CreateTemp("", "prom_registry_test")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -1076,7 +1076,7 @@ test_summary_count{name="foo"} 2
 		t.Fatal(err)
 	}
 
-	fileBytes, err := ioutil.ReadFile(tmpfile.Name())
+	fileBytes, err := os.ReadFile(tmpfile.Name())
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -1163,15 +1163,15 @@ func TestAlreadyRegisteredCollision(t *testing.T) {
 		// Register should not fail, since each collector has a unique
 		// set of sub-collectors, determined by their names and const label values.
 		if err := reg.Register(&collector); err != nil {
-			alreadyRegErr, ok := err.(prometheus.AlreadyRegisteredError)
-			if !ok {
+			are := &prometheus.AlreadyRegisteredError{}
+			if !errors.As(err, are) {
 				t.Fatal(err)
 			}
 
-			previous := alreadyRegErr.ExistingCollector.(*collidingCollector)
-			current := alreadyRegErr.NewCollector.(*collidingCollector)
+			previous := are.ExistingCollector.(*collidingCollector)
+			current := are.NewCollector.(*collidingCollector)
 
-			t.Errorf("Unexpected registration error: %q\nprevious collector: %s (i=%d)\ncurrent collector %s (i=%d)", alreadyRegErr, previous.name, previous.i, current.name, current.i)
+			t.Errorf("Unexpected registration error: %q\nprevious collector: %s (i=%d)\ncurrent collector %s (i=%d)", are, previous.name, previous.i, current.name, current.i)
 		}
 	}
 }
@@ -1241,7 +1241,7 @@ func TestNewMultiTRegistry(t *testing.T) {
 	t.Run("two registries, one with error", func(t *testing.T) {
 		m := prometheus.NewMultiTRegistry(prometheus.ToTransactionalGatherer(reg), treg)
 		ret, done, err := m.Gather()
-		if err != treg.err {
+		if !errors.Is(err, treg.err) {
 			t.Error("unexpected error:", err)
 		}
 		done()
@@ -1254,3 +1254,37 @@ func TestNewMultiTRegistry(t *testing.T) {
 		}
 	})
 }
+
+// This example shows how to use multiple registries for registering and
+// unregistering groups of metrics.
+func ExampleRegistry_grouping() {
+	// Create a global registry.
+	globalReg := prometheus.NewRegistry()
+
+	// Spawn 10 workers, each of which will have their own group of metrics.
+	for i := 0; i < 10; i++ {
+		// Create a new registry for each worker, which acts as a group of
+		// worker-specific metrics.
+		workerReg := prometheus.NewRegistry()
+		globalReg.Register(workerReg)
+
+		go func(workerID int) {
+			// Once the worker is done, it can unregister itself.
+			defer globalReg.Unregister(workerReg)
+
+			workTime := prometheus.NewCounter(prometheus.CounterOpts{
+				Name: "worker_total_work_time_milliseconds",
+				ConstLabels: prometheus.Labels{
+					// Generate a label unique to this worker so its metric doesn't
+					// collide with the metrics from other workers.
+					"worker_id": fmt.Sprintf("%d", workerID),
+				},
+			})
+			workerReg.MustRegister(workTime)
+
+			start := time.Now()
+			time.Sleep(time.Millisecond * time.Duration(rand.Intn(100)))
+			workTime.Add(float64(time.Since(start).Milliseconds()))
+		}(i)
+	}
+}

prometheus/testutil/lint.go

@@ -26,7 +26,7 @@ import (
 func CollectAndLint(c prometheus.Collector, metricNames ...string) ([]promlint.Problem, error) {
 	reg := prometheus.NewPedanticRegistry()
 	if err := reg.Register(c); err != nil {
-		return nil, fmt.Errorf("registering collector failed: %s", err)
+		return nil, fmt.Errorf("registering collector failed: %w", err)
 	}
 	return GatherAndLint(reg, metricNames...)
 }
@@ -37,7 +37,7 @@ func CollectAndLint(c prometheus.Collector, metricNames ...string) ([]promlint.P
 func GatherAndLint(g prometheus.Gatherer, metricNames ...string) ([]promlint.Problem, error) {
 	got, err := g.Gather()
 	if err != nil {
-		return nil, fmt.Errorf("gathering metrics failed: %s", err)
+		return nil, fmt.Errorf("gathering metrics failed: %w", err)
 	}
 	if metricNames != nil {
 		got = filterMetrics(got, metricNames)

prometheus/testutil/promlint/promlint.go

@@ -15,6 +15,7 @@
 package promlint
 
 import (
+	"errors"
 	"fmt"
 	"io"
 	"regexp"
@@ -83,7 +84,7 @@ func (l *Linter) Lint() ([]Problem, error) {
 		mf := &dto.MetricFamily{}
 		for {
 			if err := d.Decode(mf); err != nil {
-				if err == io.EOF {
+				if errors.Is(err, io.EOF) {
 					break
 				}
 

prometheus/testutil/testutil.go

@@ -41,12 +41,12 @@ import (
 	"bytes"
 	"fmt"
 	"io"
+	"net/http"
 	"reflect"
 
 	"github.com/davecgh/go-spew/spew"
-	"github.com/prometheus/common/expfmt"
-
 	dto "github.com/prometheus/client_model/go"
+	"github.com/prometheus/common/expfmt"
 
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/internal"
@@ -126,7 +126,7 @@ func ToFloat64(c prometheus.Collector) float64 {
 func CollectAndCount(c prometheus.Collector, metricNames ...string) int {
 	reg := prometheus.NewPedanticRegistry()
 	if err := reg.Register(c); err != nil {
-		panic(fmt.Errorf("registering collector failed: %s", err))
+		panic(fmt.Errorf("registering collector failed: %w", err))
 	}
 	result, err := GatherAndCount(reg, metricNames...)
 	if err != nil {
@@ -142,7 +142,7 @@ func CollectAndCount(c prometheus.Collector, metricNames ...string) int {
 func GatherAndCount(g prometheus.Gatherer, metricNames ...string) (int, error) {
 	got, err := g.Gather()
 	if err != nil {
-		return 0, fmt.Errorf("gathering metrics failed: %s", err)
+		return 0, fmt.Errorf("gathering metrics failed: %w", err)
 	}
 	if metricNames != nil {
 		got = filterMetrics(got, metricNames)
@@ -155,13 +155,41 @@ func GatherAndCount(g prometheus.Gatherer, metricNames ...string) (int, error) {
 	return result, nil
 }
 
+// ScrapeAndCompare calls a remote exporter's endpoint which is expected to return some metrics in
+// plain text format. Then it compares it with the results that the `expected` would return.
+// If the `metricNames` is not empty it would filter the comparison only to the given metric names.
+func ScrapeAndCompare(url string, expected io.Reader, metricNames ...string) error {
+	resp, err := http.Get(url)
+	if err != nil {
+		return fmt.Errorf("scraping metrics failed: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("the scraping target returned a status code other than 200: %d",
+			resp.StatusCode)
+	}
+
+	scraped, err := convertReaderToMetricFamily(resp.Body)
+	if err != nil {
+		return err
+	}
+
+	wanted, err := convertReaderToMetricFamily(expected)
+	if err != nil {
+		return err
+	}
+
+	return compareMetricFamilies(scraped, wanted, metricNames...)
+}
+
 // CollectAndCompare registers the provided Collector with a newly created
 // pedantic Registry. It then calls GatherAndCompare with that Registry and with
 // the provided metricNames.
 func CollectAndCompare(c prometheus.Collector, expected io.Reader, metricNames ...string) error {
 	reg := prometheus.NewPedanticRegistry()
 	if err := reg.Register(c); err != nil {
-		return fmt.Errorf("registering collector failed: %s", err)
+		return fmt.Errorf("registering collector failed: %w", err)
 	}
 	return GatherAndCompare(reg, expected, metricNames...)
 }
@@ -182,19 +210,37 @@ func TransactionalGatherAndCompare(g prometheus.TransactionalGatherer, expected
 	got, done, err := g.Gather()
 	defer done()
 	if err != nil {
-		return fmt.Errorf("gathering metrics failed: %s", err)
+		return fmt.Errorf("gathering metrics failed: %w", err)
 	}
+
+	wanted, err := convertReaderToMetricFamily(expected)
+	if err != nil {
+		return err
+	}
+
+	return compareMetricFamilies(got, wanted, metricNames...)
+}
+
+// convertReaderToMetricFamily would read from a io.Reader object and convert it to a slice of
+// dto.MetricFamily.
+func convertReaderToMetricFamily(reader io.Reader) ([]*dto.MetricFamily, error) {
+	var tp expfmt.TextParser
+	notNormalized, err := tp.TextToMetricFamilies(reader)
+	if err != nil {
+		return nil, fmt.Errorf("converting reader to metric families failed: %w", err)
+	}
+
+	return internal.NormalizeMetricFamilies(notNormalized), nil
+}
+
+// compareMetricFamilies would compare 2 slices of metric families, and optionally filters both of
+// them to the `metricNames` provided.
+func compareMetricFamilies(got, expected []*dto.MetricFamily, metricNames ...string) error {
 	if metricNames != nil {
 		got = filterMetrics(got, metricNames)
 	}
-	var tp expfmt.TextParser
-	wantRaw, err := tp.TextToMetricFamilies(expected)
-	if err != nil {
-		return fmt.Errorf("parsing expected metrics failed: %s", err)
-	}
-	want := internal.NormalizeMetricFamilies(wantRaw)
 
-	return compare(got, want)
+	return compare(got, expected)
 }
 
 // compare encodes both provided slices of metric families into the text format,
@@ -206,13 +252,13 @@ func compare(got, want []*dto.MetricFamily) error {
 	enc := expfmt.NewEncoder(&gotBuf, expfmt.FmtText)
 	for _, mf := range got {
 		if err := enc.Encode(mf); err != nil {
-			return fmt.Errorf("encoding gathered metrics failed: %s", err)
+			return fmt.Errorf("encoding gathered metrics failed: %w", err)
 		}
 	}
 	enc = expfmt.NewEncoder(&wantBuf, expfmt.FmtText)
 	for _, mf := range want {
 		if err := enc.Encode(mf); err != nil {
-			return fmt.Errorf("encoding expected metrics failed: %s", err)
+			return fmt.Errorf("encoding expected metrics failed: %w", err)
 		}
 	}
 	if diffErr := diff(wantBuf, gotBuf); diffErr != "" {

prometheus/testutil/testutil_test.go

@@ -14,6 +14,9 @@
 package testutil
 
 import (
+	"fmt"
+	"net/http"
+	"net/http/httptest"
 	"strings"
 	"testing"
 
@@ -308,6 +311,61 @@ Diff:
 	}
 }
 
+func TestScrapeAndCompare(t *testing.T) {
+	const expected = `
+		# HELP some_total A value that represents a counter.
+		# TYPE some_total counter
+
+		some_total{ label1 = "value1" } 1
+	`
+
+	expectedReader := strings.NewReader(expected)
+
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprintln(w, expected)
+	}))
+	defer ts.Close()
+
+	if err := ScrapeAndCompare(ts.URL, expectedReader, "some_total"); err != nil {
+		t.Errorf("unexpected scraping result:\n%s", err)
+	}
+}
+
+func TestScrapeAndCompareFetchingFail(t *testing.T) {
+	err := ScrapeAndCompare("some_url", strings.NewReader("some expectation"), "some_total")
+	if err == nil {
+		t.Errorf("expected an error but got nil")
+	}
+	if !strings.HasPrefix(err.Error(), "scraping metrics failed") {
+		t.Errorf("unexpected error happened: %s", err)
+	}
+}
+
+func TestScrapeAndCompareBadStatusCode(t *testing.T) {
+	const expected = `
+		# HELP some_total A value that represents a counter.
+		# TYPE some_total counter
+
+		some_total{ label1 = "value1" } 1
+	`
+
+	expectedReader := strings.NewReader(expected)
+
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusBadGateway)
+		fmt.Fprintln(w, expected)
+	}))
+	defer ts.Close()
+
+	err := ScrapeAndCompare(ts.URL, expectedReader, "some_total")
+	if err == nil {
+		t.Errorf("expected an error but got nil")
+	}
+	if !strings.HasPrefix(err.Error(), "the scraping target returned a status code other than 200") {
+		t.Errorf("unexpected error happened: %s", err)
+	}
+}
+
 func TestCollectAndCount(t *testing.T) {
 	c := prometheus.NewCounterVec(
 		prometheus.CounterOpts{

prometheus/value.go

@@ -200,7 +200,7 @@ func MakeLabelPairs(desc *Desc, labelValues []string) []*dto.LabelPair {
 }
 
 // ExemplarMaxRunes is the max total number of runes allowed in exemplar labels.
-const ExemplarMaxRunes = 64
+const ExemplarMaxRunes = 128
 
 // newExemplar creates a new dto.Exemplar from the provided values. An error is
 // returned if any of the label names or values are invalid or if the total