From 22da9497b8f0d53072dfc4721904faa7395d8318 Mon Sep 17 00:00:00 2001 From: Michael Knyszek Date: Sun, 16 Jan 2022 11:41:56 -0500 Subject: [PATCH 1/9] Use the runtime/metrics package for the Go collector for 1.17+ (#955) This change introduces use of the runtime/metrics package in place of runtime.MemStats for Go 1.17 or later. The runtime/metrics package was introduced in Go 1.16, but not all the old metrics were accounted for until 1.17. The runtime/metrics package offers several advantages over using runtime.MemStats: * The list of metrics and their descriptions are machine-readable, allowing new metrics to get added without any additional work. * Detailed histogram-based metrics are now available, offering much deeper insights into the Go runtime. * The runtime/metrics API is significantly more efficient than runtime.MemStats, even with the additional metrics added, because it does not require any stop-the-world events. That being said, integrating the package comes with some caveats, some of which were discussed in #842. Namely: * The old MemStats-based metrics need to continue working, so they're exported under their old names backed by equivalent runtime/metrics metrics. * Earlier versions of Go need to continue working, so the old code remains, but behind a build tag. Finally, a few notes about the implementation: * This change includes a whole bunch of refactoring to avoid significant code duplication. * This change adds a new histogram metric type specifically optimized for runtime/metrics histograms. This type's methods also include additional logic to deal with differences in bounds conventions. * This change makes a whole bunch of decisions about how runtime/metrics names are translated. * This change adds a `go generate` script to generate a list of expected runtime/metrics names for a given Go version for auditing. Users of new versions of Go will transparently be allowed to use new metrics, however. Signed-off-by: Michael Anthony Knyszek --- prometheus/build_info_collector.go | 38 ++ prometheus/counter.go | 8 +- prometheus/gen_go_collector_metrics_set.go | 115 ++++ prometheus/go_collector.go | 494 ++++++++---------- prometheus/go_collector_go116.go | 107 ++++ prometheus/go_collector_go116_test.go | 118 +++++ prometheus/go_collector_go117.go | 364 +++++++++++++ prometheus/go_collector_go117_test.go | 282 ++++++++++ prometheus/go_collector_metrics_go117_test.go | 39 ++ prometheus/go_collector_test.go | 93 ---- prometheus/internal/go_runtime_metrics.go | 77 +++ .../internal/go_runtime_metrics_test.go | 97 ++++ 12 files changed, 1449 insertions(+), 383 deletions(-) create mode 100644 prometheus/build_info_collector.go create mode 100644 prometheus/gen_go_collector_metrics_set.go create mode 100644 prometheus/go_collector_go116.go create mode 100644 prometheus/go_collector_go116_test.go create mode 100644 prometheus/go_collector_go117.go create mode 100644 prometheus/go_collector_go117_test.go create mode 100644 prometheus/go_collector_metrics_go117_test.go create mode 100644 prometheus/internal/go_runtime_metrics.go create mode 100644 prometheus/internal/go_runtime_metrics_test.go diff --git a/prometheus/build_info_collector.go b/prometheus/build_info_collector.go new file mode 100644 index 0000000..450189f --- /dev/null +++ b/prometheus/build_info_collector.go @@ -0,0 +1,38 @@ +// Copyright 2021 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package prometheus + +import "runtime/debug" + +// NewBuildInfoCollector is the obsolete version of collectors.NewBuildInfoCollector. +// See there for documentation. +// +// Deprecated: Use collectors.NewBuildInfoCollector instead. +func NewBuildInfoCollector() Collector { + path, version, sum := "unknown", "unknown", "unknown" + if bi, ok := debug.ReadBuildInfo(); ok { + path = bi.Main.Path + version = bi.Main.Version + sum = bi.Main.Sum + } + c := &selfCollector{MustNewConstMetric( + NewDesc( + "go_build_info", + "Build information about the main Go module.", + nil, Labels{"path": path, "version": version, "checksum": sum}, + ), + GaugeValue, 1)} + c.init(c.self) + return c +} diff --git a/prometheus/counter.go b/prometheus/counter.go index 3f8fd79..00d70f0 100644 --- a/prometheus/counter.go +++ b/prometheus/counter.go @@ -133,10 +133,14 @@ func (c *counter) Inc() { atomic.AddUint64(&c.valInt, 1) } -func (c *counter) Write(out *dto.Metric) error { +func (c *counter) get() float64 { fval := math.Float64frombits(atomic.LoadUint64(&c.valBits)) ival := atomic.LoadUint64(&c.valInt) - val := fval + float64(ival) + return fval + float64(ival) +} + +func (c *counter) Write(out *dto.Metric) error { + val := c.get() var exemplar *dto.Exemplar if e := c.exemplar.Load(); e != nil { diff --git a/prometheus/gen_go_collector_metrics_set.go b/prometheus/gen_go_collector_metrics_set.go new file mode 100644 index 0000000..ac63777 --- /dev/null +++ b/prometheus/gen_go_collector_metrics_set.go @@ -0,0 +1,115 @@ +// Copyright 2021 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build ignore +// +build ignore + +package main + +import ( + "bytes" + "fmt" + "go/format" + "log" + "os" + "runtime/metrics" + "strconv" + "strings" + "text/template" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/internal" +) + +func main() { + if len(os.Args) != 2 { + log.Fatal("requires Go version (e.g. go1.17) as an argument") + } + version, err := parseVersion(os.Args[1]) + if err != nil { + log.Fatalf("parsing Go version: %v", err) + } + + // Generate code. + var buf bytes.Buffer + err = testFile.Execute(&buf, struct { + Descriptions []metrics.Description + GoVersion goVersion + }{ + Descriptions: metrics.All(), + GoVersion: version, + }) + if err != nil { + log.Fatalf("executing template: %v", err) + } + + // Format it. + result, err := format.Source(buf.Bytes()) + if err != nil { + log.Fatalf("formatting code: %v", err) + } + + // Write it to a file. + fname := fmt.Sprintf("go_collector_metrics_%s_test.go", version.Abbr()) + if err := os.WriteFile(fname, result, 0o644); err != nil { + log.Fatalf("writing file: %v", err) + } +} + +type goVersion int + +func (g goVersion) String() string { + return fmt.Sprintf("go1.%d", g) +} + +func (g goVersion) Abbr() string { + return fmt.Sprintf("go1%d", g) +} + +func parseVersion(s string) (goVersion, error) { + i := strings.IndexRune(s, '.') + if i < 0 { + return goVersion(-1), fmt.Errorf("bad Go version format") + } + i, err := strconv.Atoi(s[i+1:]) + return goVersion(i), err +} + +var testFile = template.Must(template.New("testFile").Funcs(map[string]interface{}{ + "rm2prom": func(d metrics.Description) string { + ns, ss, n, ok := internal.RuntimeMetricsToProm(&d) + if !ok { + return "" + } + return prometheus.BuildFQName(ns, ss, n) + }, + "nextVersion": func(version goVersion) string { + return (version + goVersion(1)).String() + }, +}).Parse(`// Code generated by gen_go_collector_metrics_set.go; DO NOT EDIT. +//go:generate go run gen_go_collector_metrics_set.go {{.GoVersion}} + +//go:build {{.GoVersion}} && !{{nextVersion .GoVersion}} +// +build {{.GoVersion}},!{{nextVersion .GoVersion}} + +package prometheus + +var expectedRuntimeMetrics = map[string]string{ +{{- range .Descriptions -}} + {{- $trans := rm2prom . -}} + {{- if ne $trans "" }} + {{.Name | printf "%q"}}: {{$trans | printf "%q"}}, + {{- end -}} +{{end}} +} +`)) diff --git a/prometheus/go_collector.go b/prometheus/go_collector.go index a96ed1c..08195b4 100644 --- a/prometheus/go_collector.go +++ b/prometheus/go_collector.go @@ -16,32 +16,209 @@ package prometheus import ( "runtime" "runtime/debug" - "sync" "time" ) -type goCollector struct { +func goRuntimeMemStats() memStatsMetrics { + return memStatsMetrics{ + { + desc: NewDesc( + memstatNamespace("alloc_bytes"), + "Number of bytes allocated and still in use.", + nil, nil, + ), + eval: func(ms *runtime.MemStats) float64 { return float64(ms.Alloc) }, + valType: GaugeValue, + }, { + desc: NewDesc( + memstatNamespace("alloc_bytes_total"), + "Total number of bytes allocated, even if freed.", + nil, nil, + ), + eval: func(ms *runtime.MemStats) float64 { return float64(ms.TotalAlloc) }, + valType: CounterValue, + }, { + desc: NewDesc( + memstatNamespace("sys_bytes"), + "Number of bytes obtained from system.", + nil, nil, + ), + eval: func(ms *runtime.MemStats) float64 { return float64(ms.Sys) }, + valType: GaugeValue, + }, { + desc: NewDesc( + memstatNamespace("lookups_total"), + "Total number of pointer lookups.", + nil, nil, + ), + eval: func(ms *runtime.MemStats) float64 { return float64(ms.Lookups) }, + valType: CounterValue, + }, { + desc: NewDesc( + memstatNamespace("mallocs_total"), + "Total number of mallocs.", + nil, nil, + ), + eval: func(ms *runtime.MemStats) float64 { return float64(ms.Mallocs) }, + valType: CounterValue, + }, { + desc: NewDesc( + memstatNamespace("frees_total"), + "Total number of frees.", + nil, nil, + ), + eval: func(ms *runtime.MemStats) float64 { return float64(ms.Frees) }, + valType: CounterValue, + }, { + desc: NewDesc( + memstatNamespace("heap_alloc_bytes"), + "Number of heap bytes allocated and still in use.", + nil, nil, + ), + eval: func(ms *runtime.MemStats) float64 { return float64(ms.HeapAlloc) }, + valType: GaugeValue, + }, { + desc: NewDesc( + memstatNamespace("heap_sys_bytes"), + "Number of heap bytes obtained from system.", + nil, nil, + ), + eval: func(ms *runtime.MemStats) float64 { return float64(ms.HeapSys) }, + valType: GaugeValue, + }, { + desc: NewDesc( + memstatNamespace("heap_idle_bytes"), + "Number of heap bytes waiting to be used.", + nil, nil, + ), + eval: func(ms *runtime.MemStats) float64 { return float64(ms.HeapIdle) }, + valType: GaugeValue, + }, { + desc: NewDesc( + memstatNamespace("heap_inuse_bytes"), + "Number of heap bytes that are in use.", + nil, nil, + ), + eval: func(ms *runtime.MemStats) float64 { return float64(ms.HeapInuse) }, + valType: GaugeValue, + }, { + desc: NewDesc( + memstatNamespace("heap_released_bytes"), + "Number of heap bytes released to OS.", + nil, nil, + ), + eval: func(ms *runtime.MemStats) float64 { return float64(ms.HeapReleased) }, + valType: GaugeValue, + }, { + desc: NewDesc( + memstatNamespace("heap_objects"), + "Number of allocated objects.", + nil, nil, + ), + eval: func(ms *runtime.MemStats) float64 { return float64(ms.HeapObjects) }, + valType: GaugeValue, + }, { + desc: NewDesc( + memstatNamespace("stack_inuse_bytes"), + "Number of bytes in use by the stack allocator.", + nil, nil, + ), + eval: func(ms *runtime.MemStats) float64 { return float64(ms.StackInuse) }, + valType: GaugeValue, + }, { + desc: NewDesc( + memstatNamespace("stack_sys_bytes"), + "Number of bytes obtained from system for stack allocator.", + nil, nil, + ), + eval: func(ms *runtime.MemStats) float64 { return float64(ms.StackSys) }, + valType: GaugeValue, + }, { + desc: NewDesc( + memstatNamespace("mspan_inuse_bytes"), + "Number of bytes in use by mspan structures.", + nil, nil, + ), + eval: func(ms *runtime.MemStats) float64 { return float64(ms.MSpanInuse) }, + valType: GaugeValue, + }, { + desc: NewDesc( + memstatNamespace("mspan_sys_bytes"), + "Number of bytes used for mspan structures obtained from system.", + nil, nil, + ), + eval: func(ms *runtime.MemStats) float64 { return float64(ms.MSpanSys) }, + valType: GaugeValue, + }, { + desc: NewDesc( + memstatNamespace("mcache_inuse_bytes"), + "Number of bytes in use by mcache structures.", + nil, nil, + ), + eval: func(ms *runtime.MemStats) float64 { return float64(ms.MCacheInuse) }, + valType: GaugeValue, + }, { + desc: NewDesc( + memstatNamespace("mcache_sys_bytes"), + "Number of bytes used for mcache structures obtained from system.", + nil, nil, + ), + eval: func(ms *runtime.MemStats) float64 { return float64(ms.MCacheSys) }, + valType: GaugeValue, + }, { + desc: NewDesc( + memstatNamespace("buck_hash_sys_bytes"), + "Number of bytes used by the profiling bucket hash table.", + nil, nil, + ), + eval: func(ms *runtime.MemStats) float64 { return float64(ms.BuckHashSys) }, + valType: GaugeValue, + }, { + desc: NewDesc( + memstatNamespace("gc_sys_bytes"), + "Number of bytes used for garbage collection system metadata.", + nil, nil, + ), + eval: func(ms *runtime.MemStats) float64 { return float64(ms.GCSys) }, + valType: GaugeValue, + }, { + desc: NewDesc( + memstatNamespace("other_sys_bytes"), + "Number of bytes used for other system allocations.", + nil, nil, + ), + eval: func(ms *runtime.MemStats) float64 { return float64(ms.OtherSys) }, + valType: GaugeValue, + }, { + desc: NewDesc( + memstatNamespace("next_gc_bytes"), + "Number of heap bytes when next garbage collection will take place.", + nil, nil, + ), + eval: func(ms *runtime.MemStats) float64 { return float64(ms.NextGC) }, + valType: GaugeValue, + }, { + desc: NewDesc( + memstatNamespace("gc_cpu_fraction"), + "The fraction of this program's available CPU time used by the GC since the program started.", + nil, nil, + ), + eval: func(ms *runtime.MemStats) float64 { return ms.GCCPUFraction }, + valType: GaugeValue, + }, + } +} + +type baseGoCollector struct { goroutinesDesc *Desc threadsDesc *Desc gcDesc *Desc + gcLastTimeDesc *Desc goInfoDesc *Desc - - // ms... are memstats related. - msLast *runtime.MemStats // Previously collected memstats. - msLastTimestamp time.Time - msMtx sync.Mutex // Protects msLast and msLastTimestamp. - msMetrics memStatsMetrics - msRead func(*runtime.MemStats) // For mocking in tests. - msMaxWait time.Duration // Wait time for fresh memstats. - msMaxAge time.Duration // Maximum allowed age of old memstats. } -// NewGoCollector is the obsolete version of collectors.NewGoCollector. -// See there for documentation. -// -// Deprecated: Use collectors.NewGoCollector instead. -func NewGoCollector() Collector { - return &goCollector{ +func newBaseGoCollector() baseGoCollector { + return baseGoCollector{ goroutinesDesc: NewDesc( "go_goroutines", "Number of goroutines that currently exist.", @@ -54,243 +231,28 @@ func NewGoCollector() Collector { "go_gc_duration_seconds", "A summary of the pause duration of garbage collection cycles.", nil, nil), + gcLastTimeDesc: NewDesc( + memstatNamespace("last_gc_time_seconds"), + "Number of seconds since 1970 of last garbage collection.", + nil, nil), goInfoDesc: NewDesc( "go_info", "Information about the Go environment.", nil, Labels{"version": runtime.Version()}), - msLast: &runtime.MemStats{}, - msRead: runtime.ReadMemStats, - msMaxWait: time.Second, - msMaxAge: 5 * time.Minute, - msMetrics: memStatsMetrics{ - { - desc: NewDesc( - memstatNamespace("alloc_bytes"), - "Number of bytes allocated and still in use.", - nil, nil, - ), - eval: func(ms *runtime.MemStats) float64 { return float64(ms.Alloc) }, - valType: GaugeValue, - }, { - desc: NewDesc( - memstatNamespace("alloc_bytes_total"), - "Total number of bytes allocated, even if freed.", - nil, nil, - ), - eval: func(ms *runtime.MemStats) float64 { return float64(ms.TotalAlloc) }, - valType: CounterValue, - }, { - desc: NewDesc( - memstatNamespace("sys_bytes"), - "Number of bytes obtained from system.", - nil, nil, - ), - eval: func(ms *runtime.MemStats) float64 { return float64(ms.Sys) }, - valType: GaugeValue, - }, { - desc: NewDesc( - memstatNamespace("lookups_total"), - "Total number of pointer lookups.", - nil, nil, - ), - eval: func(ms *runtime.MemStats) float64 { return float64(ms.Lookups) }, - valType: CounterValue, - }, { - desc: NewDesc( - memstatNamespace("mallocs_total"), - "Total number of mallocs.", - nil, nil, - ), - eval: func(ms *runtime.MemStats) float64 { return float64(ms.Mallocs) }, - valType: CounterValue, - }, { - desc: NewDesc( - memstatNamespace("frees_total"), - "Total number of frees.", - nil, nil, - ), - eval: func(ms *runtime.MemStats) float64 { return float64(ms.Frees) }, - valType: CounterValue, - }, { - desc: NewDesc( - memstatNamespace("heap_alloc_bytes"), - "Number of heap bytes allocated and still in use.", - nil, nil, - ), - eval: func(ms *runtime.MemStats) float64 { return float64(ms.HeapAlloc) }, - valType: GaugeValue, - }, { - desc: NewDesc( - memstatNamespace("heap_sys_bytes"), - "Number of heap bytes obtained from system.", - nil, nil, - ), - eval: func(ms *runtime.MemStats) float64 { return float64(ms.HeapSys) }, - valType: GaugeValue, - }, { - desc: NewDesc( - memstatNamespace("heap_idle_bytes"), - "Number of heap bytes waiting to be used.", - nil, nil, - ), - eval: func(ms *runtime.MemStats) float64 { return float64(ms.HeapIdle) }, - valType: GaugeValue, - }, { - desc: NewDesc( - memstatNamespace("heap_inuse_bytes"), - "Number of heap bytes that are in use.", - nil, nil, - ), - eval: func(ms *runtime.MemStats) float64 { return float64(ms.HeapInuse) }, - valType: GaugeValue, - }, { - desc: NewDesc( - memstatNamespace("heap_released_bytes"), - "Number of heap bytes released to OS.", - nil, nil, - ), - eval: func(ms *runtime.MemStats) float64 { return float64(ms.HeapReleased) }, - valType: GaugeValue, - }, { - desc: NewDesc( - memstatNamespace("heap_objects"), - "Number of allocated objects.", - nil, nil, - ), - eval: func(ms *runtime.MemStats) float64 { return float64(ms.HeapObjects) }, - valType: GaugeValue, - }, { - desc: NewDesc( - memstatNamespace("stack_inuse_bytes"), - "Number of bytes in use by the stack allocator.", - nil, nil, - ), - eval: func(ms *runtime.MemStats) float64 { return float64(ms.StackInuse) }, - valType: GaugeValue, - }, { - desc: NewDesc( - memstatNamespace("stack_sys_bytes"), - "Number of bytes obtained from system for stack allocator.", - nil, nil, - ), - eval: func(ms *runtime.MemStats) float64 { return float64(ms.StackSys) }, - valType: GaugeValue, - }, { - desc: NewDesc( - memstatNamespace("mspan_inuse_bytes"), - "Number of bytes in use by mspan structures.", - nil, nil, - ), - eval: func(ms *runtime.MemStats) float64 { return float64(ms.MSpanInuse) }, - valType: GaugeValue, - }, { - desc: NewDesc( - memstatNamespace("mspan_sys_bytes"), - "Number of bytes used for mspan structures obtained from system.", - nil, nil, - ), - eval: func(ms *runtime.MemStats) float64 { return float64(ms.MSpanSys) }, - valType: GaugeValue, - }, { - desc: NewDesc( - memstatNamespace("mcache_inuse_bytes"), - "Number of bytes in use by mcache structures.", - nil, nil, - ), - eval: func(ms *runtime.MemStats) float64 { return float64(ms.MCacheInuse) }, - valType: GaugeValue, - }, { - desc: NewDesc( - memstatNamespace("mcache_sys_bytes"), - "Number of bytes used for mcache structures obtained from system.", - nil, nil, - ), - eval: func(ms *runtime.MemStats) float64 { return float64(ms.MCacheSys) }, - valType: GaugeValue, - }, { - desc: NewDesc( - memstatNamespace("buck_hash_sys_bytes"), - "Number of bytes used by the profiling bucket hash table.", - nil, nil, - ), - eval: func(ms *runtime.MemStats) float64 { return float64(ms.BuckHashSys) }, - valType: GaugeValue, - }, { - desc: NewDesc( - memstatNamespace("gc_sys_bytes"), - "Number of bytes used for garbage collection system metadata.", - nil, nil, - ), - eval: func(ms *runtime.MemStats) float64 { return float64(ms.GCSys) }, - valType: GaugeValue, - }, { - desc: NewDesc( - memstatNamespace("other_sys_bytes"), - "Number of bytes used for other system allocations.", - nil, nil, - ), - eval: func(ms *runtime.MemStats) float64 { return float64(ms.OtherSys) }, - valType: GaugeValue, - }, { - desc: NewDesc( - memstatNamespace("next_gc_bytes"), - "Number of heap bytes when next garbage collection will take place.", - nil, nil, - ), - eval: func(ms *runtime.MemStats) float64 { return float64(ms.NextGC) }, - valType: GaugeValue, - }, { - desc: NewDesc( - memstatNamespace("last_gc_time_seconds"), - "Number of seconds since 1970 of last garbage collection.", - nil, nil, - ), - eval: func(ms *runtime.MemStats) float64 { return float64(ms.LastGC) / 1e9 }, - valType: GaugeValue, - }, { - desc: NewDesc( - memstatNamespace("gc_cpu_fraction"), - "The fraction of this program's available CPU time used by the GC since the program started.", - nil, nil, - ), - eval: func(ms *runtime.MemStats) float64 { return ms.GCCPUFraction }, - valType: GaugeValue, - }, - }, } } -func memstatNamespace(s string) string { - return "go_memstats_" + s -} - // Describe returns all descriptions of the collector. -func (c *goCollector) Describe(ch chan<- *Desc) { +func (c *baseGoCollector) Describe(ch chan<- *Desc) { ch <- c.goroutinesDesc ch <- c.threadsDesc ch <- c.gcDesc + ch <- c.gcLastTimeDesc ch <- c.goInfoDesc - for _, i := range c.msMetrics { - ch <- i.desc - } } // Collect returns the current state of all metrics of the collector. -func (c *goCollector) Collect(ch chan<- Metric) { - var ( - ms = &runtime.MemStats{} - done = make(chan struct{}) - ) - // Start reading memstats first as it might take a while. - go func() { - c.msRead(ms) - c.msMtx.Lock() - c.msLast = ms - c.msLastTimestamp = time.Now() - c.msMtx.Unlock() - close(done) - }() - +func (c *baseGoCollector) Collect(ch chan<- Metric) { ch <- MustNewConstMetric(c.goroutinesDesc, GaugeValue, float64(runtime.NumGoroutine())) n, _ := runtime.ThreadCreateProfile(nil) ch <- MustNewConstMetric(c.threadsDesc, GaugeValue, float64(n)) @@ -305,63 +267,19 @@ func (c *goCollector) Collect(ch chan<- Metric) { } quantiles[0.0] = stats.PauseQuantiles[0].Seconds() ch <- MustNewConstSummary(c.gcDesc, uint64(stats.NumGC), stats.PauseTotal.Seconds(), quantiles) + ch <- MustNewConstMetric(c.gcLastTimeDesc, GaugeValue, float64(stats.LastGC.UnixNano())/1e9) ch <- MustNewConstMetric(c.goInfoDesc, GaugeValue, 1) - - timer := time.NewTimer(c.msMaxWait) - select { - case <-done: // Our own ReadMemStats succeeded in time. Use it. - timer.Stop() // Important for high collection frequencies to not pile up timers. - c.msCollect(ch, ms) - return - case <-timer.C: // Time out, use last memstats if possible. Continue below. - } - c.msMtx.Lock() - if time.Since(c.msLastTimestamp) < c.msMaxAge { - // Last memstats are recent enough. Collect from them under the lock. - c.msCollect(ch, c.msLast) - c.msMtx.Unlock() - return - } - // If we are here, the last memstats are too old or don't exist. We have - // to wait until our own ReadMemStats finally completes. For that to - // happen, we have to release the lock. - c.msMtx.Unlock() - <-done - c.msCollect(ch, ms) } -func (c *goCollector) msCollect(ch chan<- Metric, ms *runtime.MemStats) { - for _, i := range c.msMetrics { - ch <- MustNewConstMetric(i.desc, i.valType, i.eval(ms)) - } +func memstatNamespace(s string) string { + return "go_memstats_" + s } -// memStatsMetrics provide description, value, and value type for memstat metrics. +// memStatsMetrics provide description, evaluator, runtime/metrics name, and +// value type for memstat metrics. type memStatsMetrics []struct { desc *Desc eval func(*runtime.MemStats) float64 valType ValueType } - -// NewBuildInfoCollector is the obsolete version of collectors.NewBuildInfoCollector. -// See there for documentation. -// -// Deprecated: Use collectors.NewBuildInfoCollector instead. -func NewBuildInfoCollector() Collector { - path, version, sum := "unknown", "unknown", "unknown" - if bi, ok := debug.ReadBuildInfo(); ok { - path = bi.Main.Path - version = bi.Main.Version - sum = bi.Main.Sum - } - c := &selfCollector{MustNewConstMetric( - NewDesc( - "go_build_info", - "Build information about the main Go module.", - nil, Labels{"path": path, "version": version, "checksum": sum}, - ), - GaugeValue, 1)} - c.init(c.self) - return c -} diff --git a/prometheus/go_collector_go116.go b/prometheus/go_collector_go116.go new file mode 100644 index 0000000..2452613 --- /dev/null +++ b/prometheus/go_collector_go116.go @@ -0,0 +1,107 @@ +// Copyright 2021 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !go1.17 +// +build !go1.17 + +package prometheus + +import ( + "runtime" + "sync" + "time" +) + +type goCollector struct { + base baseGoCollector + + // ms... are memstats related. + msLast *runtime.MemStats // Previously collected memstats. + msLastTimestamp time.Time + msMtx sync.Mutex // Protects msLast and msLastTimestamp. + msMetrics memStatsMetrics + msRead func(*runtime.MemStats) // For mocking in tests. + msMaxWait time.Duration // Wait time for fresh memstats. + msMaxAge time.Duration // Maximum allowed age of old memstats. +} + +// NewGoCollector is the obsolete version of collectors.NewGoCollector. +// See there for documentation. +// +// Deprecated: Use collectors.NewGoCollector instead. +func NewGoCollector() Collector { + return &goCollector{ + base: newBaseGoCollector(), + msLast: &runtime.MemStats{}, + msRead: runtime.ReadMemStats, + msMaxWait: time.Second, + msMaxAge: 5 * time.Minute, + msMetrics: goRuntimeMemStats(), + } +} + +// Describe returns all descriptions of the collector. +func (c *goCollector) Describe(ch chan<- *Desc) { + c.base.Describe(ch) + for _, i := range c.msMetrics { + ch <- i.desc + } +} + +// Collect returns the current state of all metrics of the collector. +func (c *goCollector) Collect(ch chan<- Metric) { + var ( + ms = &runtime.MemStats{} + done = make(chan struct{}) + ) + // Start reading memstats first as it might take a while. + go func() { + c.msRead(ms) + c.msMtx.Lock() + c.msLast = ms + c.msLastTimestamp = time.Now() + c.msMtx.Unlock() + close(done) + }() + + // Collect base non-memory metrics. + c.base.Collect(ch) + + timer := time.NewTimer(c.msMaxWait) + select { + case <-done: // Our own ReadMemStats succeeded in time. Use it. + timer.Stop() // Important for high collection frequencies to not pile up timers. + c.msCollect(ch, ms) + return + case <-timer.C: // Time out, use last memstats if possible. Continue below. + } + c.msMtx.Lock() + if time.Since(c.msLastTimestamp) < c.msMaxAge { + // Last memstats are recent enough. Collect from them under the lock. + c.msCollect(ch, c.msLast) + c.msMtx.Unlock() + return + } + // If we are here, the last memstats are too old or don't exist. We have + // to wait until our own ReadMemStats finally completes. For that to + // happen, we have to release the lock. + c.msMtx.Unlock() + <-done + c.msCollect(ch, ms) +} + +func (c *goCollector) msCollect(ch chan<- Metric, ms *runtime.MemStats) { + for _, i := range c.msMetrics { + ch <- MustNewConstMetric(i.desc, i.valType, i.eval(ms)) + } +} diff --git a/prometheus/go_collector_go116_test.go b/prometheus/go_collector_go116_test.go new file mode 100644 index 0000000..8969f38 --- /dev/null +++ b/prometheus/go_collector_go116_test.go @@ -0,0 +1,118 @@ +// Copyright 2021 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !go1.17 +// +build !go1.17 + +package prometheus + +import ( + "runtime" + "testing" + "time" + + dto "github.com/prometheus/client_model/go" +) + +func TestGoCollectorMemStats(t *testing.T) { + var ( + c = NewGoCollector().(*goCollector) + got uint64 + ) + + checkCollect := func(want uint64) { + metricCh := make(chan Metric) + endCh := make(chan struct{}) + + go func() { + c.Collect(metricCh) + close(endCh) + }() + Collect: + for { + select { + case metric := <-metricCh: + if metric.Desc().fqName != "go_memstats_alloc_bytes" { + continue Collect + } + pb := &dto.Metric{} + metric.Write(pb) + got = uint64(pb.GetGauge().GetValue()) + case <-endCh: + break Collect + } + } + if want != got { + t.Errorf("unexpected value of go_memstats_alloc_bytes, want %d, got %d", want, got) + } + } + + // Speed up the timing to make the test faster. + c.msMaxWait = 5 * time.Millisecond + c.msMaxAge = 50 * time.Millisecond + + // Scenario 1: msRead responds slowly, no previous memstats available, + // msRead is executed anyway. + c.msRead = func(ms *runtime.MemStats) { + time.Sleep(20 * time.Millisecond) + ms.Alloc = 1 + } + checkCollect(1) + // Now msLast is set. + c.msMtx.Lock() + if want, got := uint64(1), c.msLast.Alloc; want != got { + t.Errorf("unexpected of msLast.Alloc, want %d, got %d", want, got) + } + c.msMtx.Unlock() + + // Scenario 2: msRead responds fast, previous memstats available, new + // value collected. + c.msRead = func(ms *runtime.MemStats) { + ms.Alloc = 2 + } + checkCollect(2) + // msLast is set, too. + c.msMtx.Lock() + if want, got := uint64(2), c.msLast.Alloc; want != got { + t.Errorf("unexpected of msLast.Alloc, want %d, got %d", want, got) + } + c.msMtx.Unlock() + + // Scenario 3: msRead responds slowly, previous memstats available, old + // value collected. + c.msRead = func(ms *runtime.MemStats) { + time.Sleep(20 * time.Millisecond) + ms.Alloc = 3 + } + checkCollect(2) + // After waiting, new value is still set in msLast. + time.Sleep(80 * time.Millisecond) + c.msMtx.Lock() + if want, got := uint64(3), c.msLast.Alloc; want != got { + t.Errorf("unexpected of msLast.Alloc, want %d, got %d", want, got) + } + c.msMtx.Unlock() + + // Scenario 4: msRead responds slowly, previous memstats is too old, new + // value collected. + c.msRead = func(ms *runtime.MemStats) { + time.Sleep(20 * time.Millisecond) + ms.Alloc = 4 + } + checkCollect(4) + c.msMtx.Lock() + if want, got := uint64(4), c.msLast.Alloc; want != got { + t.Errorf("unexpected of msLast.Alloc, want %d, got %d", want, got) + } + c.msMtx.Unlock() +} diff --git a/prometheus/go_collector_go117.go b/prometheus/go_collector_go117.go new file mode 100644 index 0000000..d534742 --- /dev/null +++ b/prometheus/go_collector_go117.go @@ -0,0 +1,364 @@ +// Copyright 2021 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build go1.17 +// +build go1.17 + +package prometheus + +import ( + "math" + "runtime" + "runtime/metrics" + "sync" + + //nolint:staticcheck // Ignore SA1019. Need to keep deprecated package for compatibility. + "github.com/golang/protobuf/proto" + "github.com/prometheus/client_golang/prometheus/internal" + dto "github.com/prometheus/client_model/go" +) + +type goCollector struct { + base baseGoCollector + + // rm... fields all pertain to the runtime/metrics package. + rmSampleBuf []metrics.Sample + rmSampleMap map[string]*metrics.Sample + rmMetrics []Metric + + // With Go 1.17, the runtime/metrics package was introduced. + // From that point on, metric names produced by the runtime/metrics + // package could be generated from runtime/metrics names. However, + // these differ from the old names for the same values. + // + // This field exist to export the same values under the old names + // as well. + msMetrics memStatsMetrics +} + +// NewGoCollector is the obsolete version of collectors.NewGoCollector. +// See there for documentation. +// +// Deprecated: Use collectors.NewGoCollector instead. +func NewGoCollector() Collector { + descriptions := metrics.All() + descMap := make(map[string]*metrics.Description) + for i := range descriptions { + descMap[descriptions[i].Name] = &descriptions[i] + } + + // Generate a Desc and ValueType for each runtime/metrics metric. + metricSet := make([]Metric, 0, len(descriptions)) + sampleBuf := make([]metrics.Sample, 0, len(descriptions)) + sampleMap := make(map[string]*metrics.Sample, len(descriptions)) + for i := range descriptions { + d := &descriptions[i] + namespace, subsystem, name, ok := internal.RuntimeMetricsToProm(d) + if !ok { + // Just ignore this metric; we can't do anything with it here. + // If a user decides to use the latest version of Go, we don't want + // to fail here. This condition is tested elsewhere. + continue + } + + // Set up sample buffer for reading, and a map + // for quick lookup of sample values. + sampleBuf = append(sampleBuf, metrics.Sample{Name: d.Name}) + sampleMap[d.Name] = &sampleBuf[len(sampleBuf)-1] + + var m Metric + if d.Kind == metrics.KindFloat64Histogram { + _, hasSum := rmExactSumMap[d.Name] + m = newBatchHistogram( + NewDesc( + BuildFQName(namespace, subsystem, name), + d.Description, + nil, + nil, + ), + hasSum, + ) + } else if d.Cumulative { + m = NewCounter(CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: name, + Help: d.Description, + }) + } else { + m = NewGauge(GaugeOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: name, + Help: d.Description, + }) + } + metricSet = append(metricSet, m) + } + return &goCollector{ + base: newBaseGoCollector(), + rmSampleBuf: sampleBuf, + rmSampleMap: sampleMap, + rmMetrics: metricSet, + msMetrics: goRuntimeMemStats(), + } +} + +// Describe returns all descriptions of the collector. +func (c *goCollector) Describe(ch chan<- *Desc) { + c.base.Describe(ch) + for _, i := range c.msMetrics { + ch <- i.desc + } + for _, m := range c.rmMetrics { + ch <- m.Desc() + } +} + +// Collect returns the current state of all metrics of the collector. +func (c *goCollector) Collect(ch chan<- Metric) { + // Collect base non-memory metrics. + c.base.Collect(ch) + + // Populate runtime/metrics sample buffer. + metrics.Read(c.rmSampleBuf) + + for i, sample := range c.rmSampleBuf { + // N.B. switch on concrete type because it's significantly more efficient + // than checking for the Counter and Gauge interface implementations. In + // this case, we control all the types here. + switch m := c.rmMetrics[i].(type) { + case *counter: + // Guard against decreases. This should never happen, but a failure + // to do so will result in a panic, which is a harsh consequence for + // a metrics collection bug. + v0, v1 := m.get(), unwrapScalarRMValue(sample.Value) + if v1 > v0 { + m.Add(unwrapScalarRMValue(sample.Value) - m.get()) + } + m.Collect(ch) + case *gauge: + m.Set(unwrapScalarRMValue(sample.Value)) + m.Collect(ch) + case *batchHistogram: + m.update(sample.Value.Float64Histogram(), c.exactSumFor(sample.Name)) + m.Collect(ch) + default: + panic("unexpected metric type") + } + } + + // ms is a dummy MemStats that we populate ourselves so that we can + // populate the old metrics from it. + var ms runtime.MemStats + memStatsFromRM(&ms, c.rmSampleMap) + for _, i := range c.msMetrics { + ch <- MustNewConstMetric(i.desc, i.valType, i.eval(&ms)) + } +} + +// unwrapScalarRMValue unwraps a runtime/metrics value that is assumed +// to be scalar and returns the equivalent float64 value. Panics if the +// value is not scalar. +func unwrapScalarRMValue(v metrics.Value) float64 { + switch v.Kind() { + case metrics.KindUint64: + return float64(v.Uint64()) + case metrics.KindFloat64: + return v.Float64() + case metrics.KindBad: + // Unsupported metric. + // + // This should never happen because we always populate our metric + // set from the runtime/metrics package. + panic("unexpected unsupported metric") + default: + // Unsupported metric kind. + // + // This should never happen because we check for this during initialization + // and flag and filter metrics whose kinds we don't understand. + panic("unexpected unsupported metric kind") + } +} + +var rmExactSumMap = map[string]string{ + "/gc/heap/allocs-by-size:bytes": "/gc/heap/allocs:bytes", + "/gc/heap/frees-by-size:bytes": "/gc/heap/frees:bytes", +} + +// exactSumFor takes a runtime/metrics metric name (that is assumed to +// be of kind KindFloat64Histogram) and returns its exact sum and whether +// its exact sum exists. +// +// The runtime/metrics API for histograms doesn't currently expose exact +// sums, but some of the other metrics are in fact exact sums of histograms. +func (c *goCollector) exactSumFor(rmName string) float64 { + sumName, ok := rmExactSumMap[rmName] + if !ok { + return 0 + } + s, ok := c.rmSampleMap[sumName] + if !ok { + return 0 + } + return unwrapScalarRMValue(s.Value) +} + +func memStatsFromRM(ms *runtime.MemStats, rm map[string]*metrics.Sample) { + lookupOrZero := func(name string) uint64 { + if s, ok := rm[name]; ok { + return s.Value.Uint64() + } + return 0 + } + + // Currently, MemStats adds tiny alloc count to both Mallocs AND Frees. + // The reason for this is because MemStats couldn't be extended at the time + // but there was a desire to have Mallocs at least be a little more representative, + // while having Mallocs - Frees still represent a live object count. + // Unfortunately, MemStats doesn't actually export a large allocation count, + // so it's impossible to pull this number out directly. + tinyAllocs := lookupOrZero("/gc/heap/tiny/allocs:objects") + ms.Mallocs = lookupOrZero("/gc/heap/allocs:objects") + tinyAllocs + ms.Frees = lookupOrZero("/gc/heap/frees:objects") + tinyAllocs + + ms.TotalAlloc = lookupOrZero("/gc/heap/allocs:bytes") + ms.Sys = lookupOrZero("/memory/classes/total:bytes") + ms.Lookups = 0 // Already always zero. + ms.HeapAlloc = lookupOrZero("/memory/classes/heap/objects:bytes") + ms.Alloc = ms.HeapAlloc + ms.HeapInuse = ms.HeapAlloc + lookupOrZero("/memory/classes/heap/unused:bytes") + ms.HeapReleased = lookupOrZero("/memory/classes/heap/released:bytes") + ms.HeapIdle = ms.HeapReleased + lookupOrZero("/memory/classes/heap/free:bytes") + ms.HeapSys = ms.HeapInuse + ms.HeapIdle + ms.HeapObjects = lookupOrZero("/gc/heap/objects:objects") + ms.StackInuse = lookupOrZero("/memory/classes/heap/stacks:bytes") + ms.StackSys = ms.StackInuse + lookupOrZero("/memory/classes/os-stacks:bytes") + ms.MSpanInuse = lookupOrZero("/memory/classes/metadata/mspan/inuse:bytes") + ms.MSpanSys = ms.MSpanInuse + lookupOrZero("/memory/classes/metadata/mspan/free:bytes") + ms.MCacheInuse = lookupOrZero("/memory/classes/metadata/mcache/inuse:bytes") + ms.MCacheSys = ms.MCacheInuse + lookupOrZero("/memory/classes/metadata/mcache/free:bytes") + ms.BuckHashSys = lookupOrZero("/memory/classes/profiling/buckets:bytes") + ms.GCSys = lookupOrZero("/memory/classes/metadata/other:bytes") + ms.OtherSys = lookupOrZero("/memory/classes/other:bytes") + ms.NextGC = lookupOrZero("/gc/heap/goal:bytes") + + // N.B. LastGC is omitted because runtime.GCStats already has this. + // See https://github.com/prometheus/client_golang/issues/842#issuecomment-861812034 + // for more details. + ms.LastGC = 0 + + // N.B. GCCPUFraction is intentionally omitted. This metric is not useful, + // and often misleading due to the fact that it's an average over the lifetime + // of the process. + // See https://github.com/prometheus/client_golang/issues/842#issuecomment-861812034 + // for more details. + ms.GCCPUFraction = 0 +} + +// batchHistogram is a mutable histogram that is updated +// in batches. +type batchHistogram struct { + selfCollector + + // Static fields updated only once. + desc *Desc + hasSum bool + + // Because this histogram operates in batches, it just uses a + // single mutex for everything. updates are always serialized + // but Write calls may operate concurrently with updates. + // Contention between these two sources should be rare. + mu sync.Mutex + buckets []float64 // Inclusive lower bounds. + counts []uint64 + sum float64 // Used if hasSum is true. +} + +func newBatchHistogram(desc *Desc, hasSum bool) *batchHistogram { + h := &batchHistogram{desc: desc, hasSum: hasSum} + h.init(h) + return h +} + +// update updates the batchHistogram from a runtime/metrics histogram. +// +// sum must be provided if the batchHistogram was created to have an exact sum. +func (h *batchHistogram) update(his *metrics.Float64Histogram, sum float64) { + counts, buckets := his.Counts, his.Buckets + // Skip a -Inf bucket altogether. It's not clear how to represent that. + if math.IsInf(buckets[0], -1) { + buckets = buckets[1:] + counts = counts[1:] + } + + h.mu.Lock() + defer h.mu.Unlock() + + // Check if we're initialized. + if h.buckets == nil { + // Make copies of counts and buckets. It's really important + // that we don't retain his.Counts or his.Buckets anywhere since + // it's going to get reused. + h.buckets = make([]float64, len(buckets)) + copy(h.buckets, buckets) + + h.counts = make([]uint64, len(counts)) + } + copy(h.counts, counts) + if h.hasSum { + h.sum = sum + } +} + +func (h *batchHistogram) Desc() *Desc { + return h.desc +} + +func (h *batchHistogram) Write(out *dto.Metric) error { + h.mu.Lock() + defer h.mu.Unlock() + + sum := float64(0) + if h.hasSum { + sum = h.sum + } + dtoBuckets := make([]*dto.Bucket, 0, len(h.counts)) + totalCount := uint64(0) + for i, count := range h.counts { + totalCount += count + if !h.hasSum { + // N.B. This computed sum is an underestimate. + sum += h.buckets[i] * float64(count) + } + + // Skip the +Inf bucket, but only for the bucket list. + // It must still count for sum and totalCount. + if math.IsInf(h.buckets[i+1], 1) { + break + } + // Float64Histogram's upper bound is exclusive, so make it inclusive + // by obtaining the next float64 value down, in order. + upperBound := math.Nextafter(h.buckets[i+1], h.buckets[i]) + dtoBuckets = append(dtoBuckets, &dto.Bucket{ + CumulativeCount: proto.Uint64(totalCount), + UpperBound: proto.Float64(upperBound), + }) + } + out.Histogram = &dto.Histogram{ + Bucket: dtoBuckets, + SampleCount: proto.Uint64(totalCount), + SampleSum: proto.Float64(sum), + } + return nil +} diff --git a/prometheus/go_collector_go117_test.go b/prometheus/go_collector_go117_test.go new file mode 100644 index 0000000..653e332 --- /dev/null +++ b/prometheus/go_collector_go117_test.go @@ -0,0 +1,282 @@ +// Copyright 2021 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build go1.17 +// +build go1.17 + +package prometheus + +import ( + "math" + "reflect" + "runtime" + "runtime/metrics" + "sync" + "testing" + + "github.com/prometheus/client_golang/prometheus/internal" + dto "github.com/prometheus/client_model/go" +) + +func TestGoCollectorRuntimeMetrics(t *testing.T) { + metrics := collectGoMetrics(t) + + msChecklist := make(map[string]bool) + for _, m := range goRuntimeMemStats() { + msChecklist[m.desc.fqName] = false + } + + if len(metrics) == 0 { + t.Fatal("no metrics created by Collect") + } + + // Check a few specific metrics. + // + // Checking them all is somewhat pointless because the runtime/metrics + // metrics are going to shift underneath us. Also if we try to check + // against the runtime/metrics package in an automated fashion we're kind + // of missing the point, because we have to do all the same work the code + // has to do to perform the translation. Same for supporting old metric + // names (the best we can do here is make sure they're all accounted for). + var sysBytes, allocs float64 + for _, m := range metrics { + name := m.Desc().fqName + switch name { + case "go_memory_classes_total_bytes": + checkMemoryMetric(t, m, &sysBytes) + case "go_sys_bytes": + checkMemoryMetric(t, m, &sysBytes) + case "go_gc_heap_allocs_bytes_total": + checkMemoryMetric(t, m, &allocs) + case "go_alloc_bytes_total": + checkMemoryMetric(t, m, &allocs) + } + if present, ok := msChecklist[name]; ok { + if present { + t.Errorf("memstats metric %s found more than once", name) + } + msChecklist[name] = true + } + } + for name := range msChecklist { + if present := msChecklist[name]; !present { + t.Errorf("memstats metric %s not collected", name) + } + } +} + +func checkMemoryMetric(t *testing.T, m Metric, expValue *float64) { + t.Helper() + + pb := &dto.Metric{} + m.Write(pb) + var value float64 + if g := pb.GetGauge(); g != nil { + value = g.GetValue() + } else { + value = pb.GetCounter().GetValue() + } + if value <= 0 { + t.Error("bad value for total memory") + } + if *expValue == 0 { + *expValue = value + } else if value != *expValue { + t.Errorf("legacy metric and runtime/metrics metric do not match: want %d, got %d", int64(*expValue), int64(value)) + } +} + +var sink interface{} + +func TestBatchHistogram(t *testing.T) { + goMetrics := collectGoMetrics(t) + + var mhist Metric + for _, m := range goMetrics { + if m.Desc().fqName == "go_gc_heap_allocs_by_size_bytes_total" { + mhist = m + break + } + } + if mhist == nil { + t.Fatal("failed to find metric to test") + } + hist, ok := mhist.(*batchHistogram) + if !ok { + t.Fatal("found metric is not a runtime/metrics histogram") + } + + // Make a bunch of allocations then do another collection. + // + // The runtime/metrics API tries to reuse memory where possible, + // so make sure that we didn't hang on to any of that memory in + // hist. + countsCopy := make([]uint64, len(hist.counts)) + copy(countsCopy, hist.counts) + for i := 0; i < 100; i++ { + sink = make([]byte, 128) + } + collectGoMetrics(t) + for i, v := range hist.counts { + if v != countsCopy[i] { + t.Error("counts changed during new collection") + break + } + } + + // Get the runtime/metrics copy. + s := []metrics.Sample{ + {Name: "/gc/heap/allocs-by-size:bytes"}, + } + metrics.Read(s) + rmHist := s[0].Value.Float64Histogram() + // runtime/metrics histograms always have -Inf and +Inf buckets. + // We never handle -Inf and +Inf is implicit. + wantBuckets := len(rmHist.Buckets) - 2 + + // Check to make sure the output proto makes sense. + pb := &dto.Metric{} + hist.Write(pb) + + if math.IsInf(pb.Histogram.Bucket[len(pb.Histogram.Bucket)-1].GetUpperBound(), +1) { + t.Errorf("found +Inf bucket") + } + if got := len(pb.Histogram.Bucket); got != wantBuckets { + t.Errorf("got %d buckets in protobuf, want %d", got, wantBuckets) + } + for i, bucket := range pb.Histogram.Bucket { + // runtime/metrics histograms are lower-bound inclusive, but we're + // upper-bound inclusive. So just make sure the new inclusive upper + // bound is somewhere close by (in some cases it's equal). + wantBound := rmHist.Buckets[i+1] + if gotBound := *bucket.UpperBound; (wantBound-gotBound)/wantBound > 0.001 { + t.Errorf("got bound %f, want within 0.1%% of %f", gotBound, wantBound) + } + // Make sure counts are cumulative. Because of the consistency guarantees + // made by the runtime/metrics package, we're really not guaranteed to get + // anything even remotely the same here. + if i > 0 && *bucket.CumulativeCount < *pb.Histogram.Bucket[i-1].CumulativeCount { + t.Error("cumulative counts are non-monotonic") + } + } +} + +func collectGoMetrics(t *testing.T) []Metric { + t.Helper() + + c := NewGoCollector().(*goCollector) + + // Collect all metrics. + ch := make(chan Metric) + var wg sync.WaitGroup + var metrics []Metric + wg.Add(1) + go func() { + defer wg.Done() + for metric := range ch { + metrics = append(metrics, metric) + } + }() + c.Collect(ch) + close(ch) + + wg.Wait() + + return metrics +} + +func TestMemStatsEquivalence(t *testing.T) { + var msReal, msFake runtime.MemStats + descs := metrics.All() + samples := make([]metrics.Sample, len(descs)) + samplesMap := make(map[string]*metrics.Sample) + for i := range descs { + samples[i].Name = descs[i].Name + samplesMap[descs[i].Name] = &samples[i] + } + + // Force a GC cycle to try to reach a clean slate. + runtime.GC() + + // Populate msReal. + runtime.ReadMemStats(&msReal) + + // Populate msFake. + metrics.Read(samples) + memStatsFromRM(&msFake, samplesMap) + + // Iterate over them and make sure they're somewhat close. + msRealValue := reflect.ValueOf(msReal) + msFakeValue := reflect.ValueOf(msFake) + + typ := msRealValue.Type() + for i := 0; i < msRealValue.NumField(); i++ { + fr := msRealValue.Field(i) + ff := msFakeValue.Field(i) + switch typ.Kind() { + case reflect.Uint64: + // N.B. Almost all fields of MemStats are uint64s. + vr := fr.Interface().(uint64) + vf := ff.Interface().(uint64) + if float64(vr-vf)/float64(vf) > 0.05 { + t.Errorf("wrong value for %s: got %d, want %d", typ.Field(i).Name, vf, vr) + } + } + } +} + +func TestExpectedRuntimeMetrics(t *testing.T) { + goMetrics := collectGoMetrics(t) + goMetricSet := make(map[string]Metric) + for _, m := range goMetrics { + goMetricSet[m.Desc().fqName] = m + } + + descs := metrics.All() + rmSet := make(map[string]struct{}) + for i := range descs { + rmName := descs[i].Name + rmSet[rmName] = struct{}{} + + expFQName, ok := expectedRuntimeMetrics[rmName] + if !ok { + t.Errorf("found new runtime/metrics metric %s", rmName) + _, _, _, ok := internal.RuntimeMetricsToProm(&descs[i]) + if !ok { + t.Errorf("new metric has name that can't be converted, or has an unsupported Kind") + } + continue + } + _, ok = goMetricSet[expFQName] + if !ok { + t.Errorf("existing runtime/metrics metric %s (expected fq name %s) not collected", rmName, expFQName) + continue + } + } + for rmName, fqName := range expectedRuntimeMetrics { + if _, ok := rmSet[rmName]; !ok { + t.Errorf("runtime/metrics metric %s removed", rmName) + continue + } + if _, ok := goMetricSet[fqName]; !ok { + t.Errorf("runtime/metrics metric %s not appearing under expected name %s", rmName, fqName) + continue + } + } + + if t.Failed() { + t.Log("a new Go version may have been detected, please run") + t.Log("\tgo run gen_go_collector_metrics_set.go go1.X") + t.Log("where X is the Go version you are currently using") + } +} diff --git a/prometheus/go_collector_metrics_go117_test.go b/prometheus/go_collector_metrics_go117_test.go new file mode 100644 index 0000000..6c0a693 --- /dev/null +++ b/prometheus/go_collector_metrics_go117_test.go @@ -0,0 +1,39 @@ +// Code generated by gen_go_collector_metrics_set.go; DO NOT EDIT. +//go:generate go run gen_go_collector_metrics_set.go go1.17 + +//go:build go1.17 && !go1.18 +// +build go1.17,!go1.18 + +package prometheus + +var expectedRuntimeMetrics = map[string]string{ + "/gc/cycles/automatic:gc-cycles": "go_gc_cycles_automatic_gc_cycles_total", + "/gc/cycles/forced:gc-cycles": "go_gc_cycles_forced_gc_cycles_total", + "/gc/cycles/total:gc-cycles": "go_gc_cycles_total_gc_cycles_total", + "/gc/heap/allocs-by-size:bytes": "go_gc_heap_allocs_by_size_bytes_total", + "/gc/heap/allocs:bytes": "go_gc_heap_allocs_bytes_total", + "/gc/heap/allocs:objects": "go_gc_heap_allocs_objects_total", + "/gc/heap/frees-by-size:bytes": "go_gc_heap_frees_by_size_bytes_total", + "/gc/heap/frees:bytes": "go_gc_heap_frees_bytes_total", + "/gc/heap/frees:objects": "go_gc_heap_frees_objects_total", + "/gc/heap/goal:bytes": "go_gc_heap_goal_bytes", + "/gc/heap/objects:objects": "go_gc_heap_objects_objects", + "/gc/heap/tiny/allocs:objects": "go_gc_heap_tiny_allocs_objects_total", + "/gc/pauses:seconds": "go_gc_pauses_seconds_total", + "/memory/classes/heap/free:bytes": "go_memory_classes_heap_free_bytes", + "/memory/classes/heap/objects:bytes": "go_memory_classes_heap_objects_bytes", + "/memory/classes/heap/released:bytes": "go_memory_classes_heap_released_bytes", + "/memory/classes/heap/stacks:bytes": "go_memory_classes_heap_stacks_bytes", + "/memory/classes/heap/unused:bytes": "go_memory_classes_heap_unused_bytes", + "/memory/classes/metadata/mcache/free:bytes": "go_memory_classes_metadata_mcache_free_bytes", + "/memory/classes/metadata/mcache/inuse:bytes": "go_memory_classes_metadata_mcache_inuse_bytes", + "/memory/classes/metadata/mspan/free:bytes": "go_memory_classes_metadata_mspan_free_bytes", + "/memory/classes/metadata/mspan/inuse:bytes": "go_memory_classes_metadata_mspan_inuse_bytes", + "/memory/classes/metadata/other:bytes": "go_memory_classes_metadata_other_bytes", + "/memory/classes/os-stacks:bytes": "go_memory_classes_os_stacks_bytes", + "/memory/classes/other:bytes": "go_memory_classes_other_bytes", + "/memory/classes/profiling/buckets:bytes": "go_memory_classes_profiling_buckets_bytes", + "/memory/classes/total:bytes": "go_memory_classes_total_bytes", + "/sched/goroutines:goroutines": "go_sched_goroutines_goroutines", + "/sched/latencies:seconds": "go_sched_latencies_seconds", +} diff --git a/prometheus/go_collector_test.go b/prometheus/go_collector_test.go index a929acf..9cc1b2e 100644 --- a/prometheus/go_collector_test.go +++ b/prometheus/go_collector_test.go @@ -154,96 +154,3 @@ func TestGoCollectorGC(t *testing.T) { break } } - -func TestGoCollectorMemStats(t *testing.T) { - var ( - c = NewGoCollector().(*goCollector) - got uint64 - ) - - checkCollect := func(want uint64) { - metricCh := make(chan Metric) - endCh := make(chan struct{}) - - go func() { - c.Collect(metricCh) - close(endCh) - }() - Collect: - for { - select { - case metric := <-metricCh: - if metric.Desc().fqName != "go_memstats_alloc_bytes" { - continue Collect - } - pb := &dto.Metric{} - metric.Write(pb) - got = uint64(pb.GetGauge().GetValue()) - case <-endCh: - break Collect - } - } - if want != got { - t.Errorf("unexpected value of go_memstats_alloc_bytes, want %d, got %d", want, got) - } - } - - // Speed up the timing to make the test faster. - c.msMaxWait = 5 * time.Millisecond - c.msMaxAge = 50 * time.Millisecond - - // Scenario 1: msRead responds slowly, no previous memstats available, - // msRead is executed anyway. - c.msRead = func(ms *runtime.MemStats) { - time.Sleep(20 * time.Millisecond) - ms.Alloc = 1 - } - checkCollect(1) - // Now msLast is set. - c.msMtx.Lock() - if want, got := uint64(1), c.msLast.Alloc; want != got { - t.Errorf("unexpected of msLast.Alloc, want %d, got %d", want, got) - } - c.msMtx.Unlock() - - // Scenario 2: msRead responds fast, previous memstats available, new - // value collected. - c.msRead = func(ms *runtime.MemStats) { - ms.Alloc = 2 - } - checkCollect(2) - // msLast is set, too. - c.msMtx.Lock() - if want, got := uint64(2), c.msLast.Alloc; want != got { - t.Errorf("unexpected of msLast.Alloc, want %d, got %d", want, got) - } - c.msMtx.Unlock() - - // Scenario 3: msRead responds slowly, previous memstats available, old - // value collected. - c.msRead = func(ms *runtime.MemStats) { - time.Sleep(20 * time.Millisecond) - ms.Alloc = 3 - } - checkCollect(2) - // After waiting, new value is still set in msLast. - time.Sleep(80 * time.Millisecond) - c.msMtx.Lock() - if want, got := uint64(3), c.msLast.Alloc; want != got { - t.Errorf("unexpected of msLast.Alloc, want %d, got %d", want, got) - } - c.msMtx.Unlock() - - // Scenario 4: msRead responds slowly, previous memstats is too old, new - // value collected. - c.msRead = func(ms *runtime.MemStats) { - time.Sleep(20 * time.Millisecond) - ms.Alloc = 4 - } - checkCollect(4) - c.msMtx.Lock() - if want, got := uint64(4), c.msLast.Alloc; want != got { - t.Errorf("unexpected of msLast.Alloc, want %d, got %d", want, got) - } - c.msMtx.Unlock() -} diff --git a/prometheus/internal/go_runtime_metrics.go b/prometheus/internal/go_runtime_metrics.go new file mode 100644 index 0000000..afc8dff --- /dev/null +++ b/prometheus/internal/go_runtime_metrics.go @@ -0,0 +1,77 @@ +// Copyright 2021 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build go1.17 +// +build go1.17 + +package internal + +import ( + "path" + "runtime/metrics" + "strings" + + "github.com/prometheus/common/model" +) + +// RuntimeMetricsToProm produces a Prometheus metric name from a runtime/metrics +// metric description and validates whether the metric is suitable for integration +// with Prometheus. +// +// Returns false if a name could not be produced, or if Prometheus does not understand +// the runtime/metrics Kind. +// +// Note that the main reason a name couldn't be produced is if the runtime/metrics +// package exports a name with characters outside the valid Prometheus metric name +// character set. This is theoretically possible, but should never happen in practice. +// Still, don't rely on it. +func RuntimeMetricsToProm(d *metrics.Description) (string, string, string, bool) { + namespace := "go" + + comp := strings.SplitN(d.Name, ":", 2) + key := comp[0] + unit := comp[1] + + // The last path element in the key is the name, + // the rest is the subsystem. + subsystem := path.Dir(key[1:] /* remove leading / */) + name := path.Base(key) + + // subsystem is translated by replacing all / and - with _. + subsystem = strings.ReplaceAll(subsystem, "/", "_") + subsystem = strings.ReplaceAll(subsystem, "-", "_") + + // unit is translated assuming that the unit contains no + // non-ASCII characters. + unit = strings.ReplaceAll(unit, "-", "_") + unit = strings.ReplaceAll(unit, "*", "_") + unit = strings.ReplaceAll(unit, "/", "_per_") + + // name has - replaced with _ and is concatenated with the unit and + // other data. + name = strings.ReplaceAll(name, "-", "_") + name = name + "_" + unit + if d.Cumulative { + name = name + "_total" + } + + valid := model.IsValidMetricName(model.LabelValue(namespace + "_" + subsystem + "_" + name)) + switch d.Kind { + case metrics.KindUint64: + case metrics.KindFloat64: + case metrics.KindFloat64Histogram: + default: + valid = false + } + return namespace, subsystem, name, valid +} diff --git a/prometheus/internal/go_runtime_metrics_test.go b/prometheus/internal/go_runtime_metrics_test.go new file mode 100644 index 0000000..2020a2d --- /dev/null +++ b/prometheus/internal/go_runtime_metrics_test.go @@ -0,0 +1,97 @@ +// Copyright 2021 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build go1.17 +// +build go1.17 + +package internal + +import ( + "runtime/metrics" + "testing" +) + +func TestRuntimeMetricsToProm(t *testing.T) { + tests := []struct { + got metrics.Description + expect string + }{ + { + metrics.Description{ + Name: "/memory/live:bytes", + Kind: metrics.KindUint64, + }, + "go_memory_live_bytes", + }, + { + metrics.Description{ + Name: "/memory/allocs:bytes", + Kind: metrics.KindUint64, + Cumulative: true, + }, + "go_memory_allocs_bytes_total", + }, + { + metrics.Description{ + Name: "/memory/alloc-rate:bytes/second", + Kind: metrics.KindFloat64, + }, + "go_memory_alloc_rate_bytes_per_second", + }, + { + metrics.Description{ + Name: "/gc/time:cpu*seconds", + Kind: metrics.KindFloat64, + Cumulative: true, + }, + "go_gc_time_cpu_seconds_total", + }, + { + metrics.Description{ + Name: "/this/is/a/very/deep/metric:metrics", + Kind: metrics.KindFloat64, + }, + "go_this_is_a_very_deep_metric_metrics", + }, + { + metrics.Description{ + Name: "/this*is*an*invalid...:µname", + Kind: metrics.KindUint64, + }, + "", + }, + { + metrics.Description{ + Name: "/this/is/a/valid/name:objects", + Kind: metrics.KindBad, + }, + "", + }, + } + for _, test := range tests { + ns, ss, n, ok := RuntimeMetricsToProm(&test.got) + name := ns + "_" + ss + "_" + n + if test.expect == "" && ok { + t.Errorf("bad input expected a bad output: input %s, got %s", test.got.Name, name) + continue + } + if test.expect != "" && !ok { + t.Errorf("unexpected bad output on good input: input %s", test.got.Name) + continue + } + if test.expect != "" && name != test.expect { + t.Errorf("expected %s, got %s", test.expect, name) + continue + } + } +} From 9075cdf61646b5adf54d3ba77a0e4f6c65cb4fd7 Mon Sep 17 00:00:00 2001 From: Kemal Akkoyun Date: Tue, 18 Jan 2022 10:19:28 +0100 Subject: [PATCH 2/9] promhttp: Check validity of method and code label values (#962) * Check validity of method and code label values Signed-off-by: Kemal Akkoyun * Use more flexibly functional option pattern for configuration Signed-off-by: Kemal Akkoyun * Update documentation Signed-off-by: Kemal Akkoyun * Simplify Signed-off-by: Kemal Akkoyun * Fix inconsistent method naming Signed-off-by: Kemal Akkoyun --- prometheus/promhttp/instrument_client.go | 28 ++++- prometheus/promhttp/instrument_server.go | 111 +++++++++++++---- prometheus/promhttp/instrument_server_test.go | 116 ++++++++++++++++++ prometheus/promhttp/option.go | 31 +++++ prometheus/promhttp/option_test.go | 65 ++++++++++ 5 files changed, 319 insertions(+), 32 deletions(-) create mode 100644 prometheus/promhttp/option.go create mode 100644 prometheus/promhttp/option_test.go diff --git a/prometheus/promhttp/instrument_client.go b/prometheus/promhttp/instrument_client.go index 83c49b6..861b4d2 100644 --- a/prometheus/promhttp/instrument_client.go +++ b/prometheus/promhttp/instrument_client.go @@ -49,7 +49,10 @@ func InstrumentRoundTripperInFlight(gauge prometheus.Gauge, next http.RoundTripp // http.RoundTripper to observe the request result with the provided CounterVec. // The CounterVec must have zero, one, or two non-const non-curried labels. For // those, the only allowed label names are "code" and "method". The function -// panics otherwise. Partitioning of the CounterVec happens by HTTP status code +// panics otherwise. For the "method" label a predefined default label value set +// is used to filter given values. Values besides predefined values will count +// as `unknown` method.`WithExtraMethods` can be used to add more +// methods to the set. Partitioning of the CounterVec happens by HTTP status code // and/or HTTP method if the respective instance label names are present in the // CounterVec. For unpartitioned counting, use a CounterVec with zero labels. // @@ -57,13 +60,18 @@ func InstrumentRoundTripperInFlight(gauge prometheus.Gauge, next http.RoundTripp // is not incremented. // // See the example for ExampleInstrumentRoundTripperDuration for example usage. -func InstrumentRoundTripperCounter(counter *prometheus.CounterVec, next http.RoundTripper) RoundTripperFunc { +func InstrumentRoundTripperCounter(counter *prometheus.CounterVec, next http.RoundTripper, opts ...Option) RoundTripperFunc { + rtOpts := &option{} + for _, o := range opts { + o(rtOpts) + } + code, method := checkLabels(counter) return RoundTripperFunc(func(r *http.Request) (*http.Response, error) { resp, err := next.RoundTrip(r) if err == nil { - counter.With(labels(code, method, r.Method, resp.StatusCode)).Inc() + counter.With(labels(code, method, r.Method, resp.StatusCode, rtOpts.extraMethods...)).Inc() } return resp, err }) @@ -73,7 +81,10 @@ func InstrumentRoundTripperCounter(counter *prometheus.CounterVec, next http.Rou // http.RoundTripper to observe the request duration with the provided // ObserverVec. The ObserverVec must have zero, one, or two non-const // non-curried labels. For those, the only allowed label names are "code" and -// "method". The function panics otherwise. The Observe method of the Observer +// "method". The function panics otherwise. For the "method" label a predefined +// default label value set is used to filter given values. Values besides +// predefined values will count as `unknown` method. `WithExtraMethods` +// can be used to add more methods to the set. The Observe method of the Observer // in the ObserverVec is called with the request duration in // seconds. Partitioning happens by HTTP status code and/or HTTP method if the // respective instance label names are present in the ObserverVec. For @@ -85,14 +96,19 @@ func InstrumentRoundTripperCounter(counter *prometheus.CounterVec, next http.Rou // // Note that this method is only guaranteed to never observe negative durations // if used with Go1.9+. -func InstrumentRoundTripperDuration(obs prometheus.ObserverVec, next http.RoundTripper) RoundTripperFunc { +func InstrumentRoundTripperDuration(obs prometheus.ObserverVec, next http.RoundTripper, opts ...Option) RoundTripperFunc { + rtOpts := &option{} + for _, o := range opts { + o(rtOpts) + } + code, method := checkLabels(obs) return RoundTripperFunc(func(r *http.Request) (*http.Response, error) { start := time.Now() resp, err := next.RoundTrip(r) if err == nil { - obs.With(labels(code, method, r.Method, resp.StatusCode)).Observe(time.Since(start).Seconds()) + obs.With(labels(code, method, r.Method, resp.StatusCode, rtOpts.extraMethods...)).Observe(time.Since(start).Seconds()) } return resp, err }) diff --git a/prometheus/promhttp/instrument_server.go b/prometheus/promhttp/instrument_server.go index ab037db..a23f0ed 100644 --- a/prometheus/promhttp/instrument_server.go +++ b/prometheus/promhttp/instrument_server.go @@ -45,7 +45,10 @@ func InstrumentHandlerInFlight(g prometheus.Gauge, next http.Handler) http.Handl // http.Handler to observe the request duration with the provided ObserverVec. // The ObserverVec must have valid metric and label names and must have zero, // one, or two non-const non-curried labels. For those, the only allowed label -// names are "code" and "method". The function panics otherwise. The Observe +// names are "code" and "method". The function panics otherwise. For the "method" +// label a predefined default label value set is used to filter given values. +// Values besides predefined values will count as `unknown` method. +//`WithExtraMethods` can be used to add more methods to the set. The Observe // method of the Observer in the ObserverVec is called with the request duration // in seconds. Partitioning happens by HTTP status code and/or HTTP method if // the respective instance label names are present in the ObserverVec. For @@ -58,7 +61,12 @@ func InstrumentHandlerInFlight(g prometheus.Gauge, next http.Handler) http.Handl // // Note that this method is only guaranteed to never observe negative durations // if used with Go1.9+. -func InstrumentHandlerDuration(obs prometheus.ObserverVec, next http.Handler) http.HandlerFunc { +func InstrumentHandlerDuration(obs prometheus.ObserverVec, next http.Handler, opts ...Option) http.HandlerFunc { + mwOpts := &option{} + for _, o := range opts { + o(mwOpts) + } + code, method := checkLabels(obs) if code { @@ -67,14 +75,14 @@ func InstrumentHandlerDuration(obs prometheus.ObserverVec, next http.Handler) ht d := newDelegator(w, nil) next.ServeHTTP(d, r) - obs.With(labels(code, method, r.Method, d.Status())).Observe(time.Since(now).Seconds()) + obs.With(labels(code, method, r.Method, d.Status(), mwOpts.extraMethods...)).Observe(time.Since(now).Seconds()) }) } return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { now := time.Now() next.ServeHTTP(w, r) - obs.With(labels(code, method, r.Method, 0)).Observe(time.Since(now).Seconds()) + obs.With(labels(code, method, r.Method, 0, mwOpts.extraMethods...)).Observe(time.Since(now).Seconds()) }) } @@ -82,7 +90,10 @@ func InstrumentHandlerDuration(obs prometheus.ObserverVec, next http.Handler) ht // to observe the request result with the provided CounterVec. The CounterVec // must have valid metric and label names and must have zero, one, or two // non-const non-curried labels. For those, the only allowed label names are -// "code" and "method". The function panics otherwise. Partitioning of the +// "code" and "method". The function panics otherwise. For the "method" +// label a predefined default label value set is used to filter given values. +// Values besides predefined values will count as `unknown` method. +// `WithExtraMethods` can be used to add more methods to the set. Partitioning of the // CounterVec happens by HTTP status code and/or HTTP method if the respective // instance label names are present in the CounterVec. For unpartitioned // counting, use a CounterVec with zero labels. @@ -92,20 +103,25 @@ func InstrumentHandlerDuration(obs prometheus.ObserverVec, next http.Handler) ht // If the wrapped Handler panics, the Counter is not incremented. // // See the example for InstrumentHandlerDuration for example usage. -func InstrumentHandlerCounter(counter *prometheus.CounterVec, next http.Handler) http.HandlerFunc { +func InstrumentHandlerCounter(counter *prometheus.CounterVec, next http.Handler, opts ...Option) http.HandlerFunc { + mwOpts := &option{} + for _, o := range opts { + o(mwOpts) + } + code, method := checkLabels(counter) if code { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { d := newDelegator(w, nil) next.ServeHTTP(d, r) - counter.With(labels(code, method, r.Method, d.Status())).Inc() + counter.With(labels(code, method, r.Method, d.Status(), mwOpts.extraMethods...)).Inc() }) } return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { next.ServeHTTP(w, r) - counter.With(labels(code, method, r.Method, 0)).Inc() + counter.With(labels(code, method, r.Method, 0, mwOpts.extraMethods...)).Inc() }) } @@ -114,7 +130,10 @@ func InstrumentHandlerCounter(counter *prometheus.CounterVec, next http.Handler) // until the response headers are written. The ObserverVec must have valid // metric and label names and must have zero, one, or two non-const non-curried // labels. For those, the only allowed label names are "code" and "method". The -// function panics otherwise. The Observe method of the Observer in the +// function panics otherwise. For the "method" label a predefined default label +// value set is used to filter given values. Values besides predefined values +// will count as `unknown` method.`WithExtraMethods` can be used to add more +// methods to the set. The Observe method of the Observer in the // ObserverVec is called with the request duration in seconds. Partitioning // happens by HTTP status code and/or HTTP method if the respective instance // label names are present in the ObserverVec. For unpartitioned observations, @@ -128,13 +147,18 @@ func InstrumentHandlerCounter(counter *prometheus.CounterVec, next http.Handler) // if used with Go1.9+. // // See the example for InstrumentHandlerDuration for example usage. -func InstrumentHandlerTimeToWriteHeader(obs prometheus.ObserverVec, next http.Handler) http.HandlerFunc { +func InstrumentHandlerTimeToWriteHeader(obs prometheus.ObserverVec, next http.Handler, opts ...Option) http.HandlerFunc { + mwOpts := &option{} + for _, o := range opts { + o(mwOpts) + } + code, method := checkLabels(obs) return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { now := time.Now() d := newDelegator(w, func(status int) { - obs.With(labels(code, method, r.Method, status)).Observe(time.Since(now).Seconds()) + obs.With(labels(code, method, r.Method, status, mwOpts.extraMethods...)).Observe(time.Since(now).Seconds()) }) next.ServeHTTP(d, r) }) @@ -144,8 +168,11 @@ func InstrumentHandlerTimeToWriteHeader(obs prometheus.ObserverVec, next http.Ha // http.Handler to observe the request size with the provided ObserverVec. The // ObserverVec must have valid metric and label names and must have zero, one, // or two non-const non-curried labels. For those, the only allowed label names -// are "code" and "method". The function panics otherwise. The Observe method of -// the Observer in the ObserverVec is called with the request size in +// are "code" and "method". The function panics otherwise. For the "method" +// label a predefined default label value set is used to filter given values. +// Values besides predefined values will count as `unknown` method. +// `WithExtraMethods` can be used to add more methods to the set. The Observe +// method of the Observer in the ObserverVec is called with the request size in // bytes. Partitioning happens by HTTP status code and/or HTTP method if the // respective instance label names are present in the ObserverVec. For // unpartitioned observations, use an ObserverVec with zero labels. Note that @@ -156,7 +183,12 @@ func InstrumentHandlerTimeToWriteHeader(obs prometheus.ObserverVec, next http.Ha // If the wrapped Handler panics, no values are reported. // // See the example for InstrumentHandlerDuration for example usage. -func InstrumentHandlerRequestSize(obs prometheus.ObserverVec, next http.Handler) http.HandlerFunc { +func InstrumentHandlerRequestSize(obs prometheus.ObserverVec, next http.Handler, opts ...Option) http.HandlerFunc { + mwOpts := &option{} + for _, o := range opts { + o(mwOpts) + } + code, method := checkLabels(obs) if code { @@ -164,14 +196,14 @@ func InstrumentHandlerRequestSize(obs prometheus.ObserverVec, next http.Handler) d := newDelegator(w, nil) next.ServeHTTP(d, r) size := computeApproximateRequestSize(r) - obs.With(labels(code, method, r.Method, d.Status())).Observe(float64(size)) + obs.With(labels(code, method, r.Method, d.Status(), mwOpts.extraMethods...)).Observe(float64(size)) }) } return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { next.ServeHTTP(w, r) size := computeApproximateRequestSize(r) - obs.With(labels(code, method, r.Method, 0)).Observe(float64(size)) + obs.With(labels(code, method, r.Method, 0, mwOpts.extraMethods...)).Observe(float64(size)) }) } @@ -179,8 +211,11 @@ func InstrumentHandlerRequestSize(obs prometheus.ObserverVec, next http.Handler) // http.Handler to observe the response size with the provided ObserverVec. The // ObserverVec must have valid metric and label names and must have zero, one, // or two non-const non-curried labels. For those, the only allowed label names -// are "code" and "method". The function panics otherwise. The Observe method of -// the Observer in the ObserverVec is called with the response size in +// are "code" and "method". The function panics otherwise. For the "method" +// label a predefined default label value set is used to filter given values. +// Values besides predefined values will count as `unknown` method. +// `WithExtraMethods` can be used to add more methods to the set. The Observe +// method of the Observer in the ObserverVec is called with the response size in // bytes. Partitioning happens by HTTP status code and/or HTTP method if the // respective instance label names are present in the ObserverVec. For // unpartitioned observations, use an ObserverVec with zero labels. Note that @@ -191,12 +226,18 @@ func InstrumentHandlerRequestSize(obs prometheus.ObserverVec, next http.Handler) // If the wrapped Handler panics, no values are reported. // // See the example for InstrumentHandlerDuration for example usage. -func InstrumentHandlerResponseSize(obs prometheus.ObserverVec, next http.Handler) http.Handler { +func InstrumentHandlerResponseSize(obs prometheus.ObserverVec, next http.Handler, opts ...Option) http.Handler { + mwOpts := &option{} + for _, o := range opts { + o(mwOpts) + } + code, method := checkLabels(obs) + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { d := newDelegator(w, nil) next.ServeHTTP(d, r) - obs.With(labels(code, method, r.Method, d.Status())).Observe(float64(d.Written())) + obs.With(labels(code, method, r.Method, d.Status(), mwOpts.extraMethods...)).Observe(float64(d.Written())) }) } @@ -290,7 +331,7 @@ func isLabelCurried(c prometheus.Collector, label string) bool { // unnecessary allocations on each request. var emptyLabels = prometheus.Labels{} -func labels(code, method bool, reqMethod string, status int) prometheus.Labels { +func labels(code, method bool, reqMethod string, status int, extraMethods ...string) prometheus.Labels { if !(code || method) { return emptyLabels } @@ -300,7 +341,7 @@ func labels(code, method bool, reqMethod string, status int) prometheus.Labels { labels["code"] = sanitizeCode(status) } if method { - labels["method"] = sanitizeMethod(reqMethod) + labels["method"] = sanitizeMethod(reqMethod, extraMethods...) } return labels @@ -330,7 +371,12 @@ func computeApproximateRequestSize(r *http.Request) int { return s } -func sanitizeMethod(m string) string { +// If the wrapped http.Handler has a known method, it will be sanitized and returned. +// Otherwise, "unknown" will be returned. The known method list can be extended +// as needed by using extraMethods parameter. +func sanitizeMethod(m string, extraMethods ...string) string { + // See https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods for + // the methods chosen as default. switch m { case "GET", "get": return "get" @@ -348,15 +394,25 @@ func sanitizeMethod(m string) string { return "options" case "NOTIFY", "notify": return "notify" + case "TRACE", "trace": + return "trace" + case "PATCH", "patch": + return "patch" default: - return strings.ToLower(m) + for _, method := range extraMethods { + if strings.EqualFold(m, method) { + return strings.ToLower(m) + } + } + return "unknown" } } // If the wrapped http.Handler has not set a status code, i.e. the value is -// currently 0, santizeCode will return 200, for consistency with behavior in +// currently 0, sanitizeCode will return 200, for consistency with behavior in // the stdlib. func sanitizeCode(s int) string { + // See for accepted codes https://www.iana.org/assignments/http-status-codes/http-status-codes.xhtml switch s { case 100: return "100" @@ -453,6 +509,9 @@ func sanitizeCode(s int) string { return "511" default: - return strconv.Itoa(s) + if s >= 100 && s <= 599 { + return strconv.Itoa(s) + } + return "unknown" } } diff --git a/prometheus/promhttp/instrument_server_test.go b/prometheus/promhttp/instrument_server_test.go index 545ae4c..5db50fb 100644 --- a/prometheus/promhttp/instrument_server_test.go +++ b/prometheus/promhttp/instrument_server_test.go @@ -204,6 +204,122 @@ func TestLabelCheck(t *testing.T) { } } +func TestLabels(t *testing.T) { + scenarios := map[string]struct { + varLabels []string + reqMethod string + respStatus int + extraMethods []string + wantLabels prometheus.Labels + ok bool + }{ + "empty": { + varLabels: []string{}, + wantLabels: emptyLabels, + reqMethod: "GET", + respStatus: 200, + ok: true, + }, + "code as single var label": { + varLabels: []string{"code"}, + reqMethod: "GET", + respStatus: 200, + wantLabels: prometheus.Labels{"code": "200"}, + ok: true, + }, + "code as single var label and out-of-range code": { + varLabels: []string{"code"}, + reqMethod: "GET", + respStatus: 99, + wantLabels: prometheus.Labels{"code": "unknown"}, + ok: true, + }, + "code as single var label and in-range but unrecognized code": { + varLabels: []string{"code"}, + reqMethod: "GET", + respStatus: 308, + wantLabels: prometheus.Labels{"code": "308"}, + ok: true, + }, + "method as single var label": { + varLabels: []string{"method"}, + reqMethod: "GET", + respStatus: 200, + wantLabels: prometheus.Labels{"method": "get"}, + ok: true, + }, + "method as single var label and unknown method": { + varLabels: []string{"method"}, + reqMethod: "CUSTOM_METHOD", + respStatus: 200, + wantLabels: prometheus.Labels{"method": "unknown"}, + ok: true, + }, + "code and method as var labels": { + varLabels: []string{"method", "code"}, + reqMethod: "GET", + respStatus: 200, + wantLabels: prometheus.Labels{"method": "get", "code": "200"}, + ok: true, + }, + "method as single var label with extra methods specified": { + varLabels: []string{"method"}, + reqMethod: "CUSTOM_METHOD", + respStatus: 200, + extraMethods: []string{"CUSTOM_METHOD", "CUSTOM_METHOD_1"}, + wantLabels: prometheus.Labels{"method": "custom_method"}, + ok: true, + }, + "all labels used with an unknown method and out-of-range code": { + varLabels: []string{"code", "method"}, + reqMethod: "CUSTOM_METHOD", + respStatus: 99, + wantLabels: prometheus.Labels{"method": "unknown", "code": "unknown"}, + ok: false, + }, + } + checkLabels := func(labels []string) (gotCode bool, gotMethod bool) { + for _, label := range labels { + switch label { + case "code": + gotCode = true + case "method": + gotMethod = true + default: + panic("metric partitioned with non-supported labels for this test") + } + } + return + } + equalLabels := func(gotLabels, wantLabels prometheus.Labels) bool { + if len(gotLabels) != len(wantLabels) { + return false + } + for ln, lv := range gotLabels { + olv, ok := wantLabels[ln] + if !ok { + return false + } + if olv != lv { + return false + } + } + return true + } + + for name, sc := range scenarios { + t.Run(name, func(t *testing.T) { + if sc.ok { + gotCode, gotMethod := checkLabels(sc.varLabels) + gotLabels := labels(gotCode, gotMethod, sc.reqMethod, sc.respStatus, sc.extraMethods...) + if !equalLabels(gotLabels, sc.wantLabels) { + t.Errorf("wanted labels=%v for counter, got code=%v", sc.wantLabels, gotLabels) + } + } + }) + } +} + func TestMiddlewareAPI(t *testing.T) { reg := prometheus.NewRegistry() diff --git a/prometheus/promhttp/option.go b/prometheus/promhttp/option.go new file mode 100644 index 0000000..35e41bd --- /dev/null +++ b/prometheus/promhttp/option.go @@ -0,0 +1,31 @@ +// Copyright 2022 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package promhttp + +// Option are used to configure a middleware or round tripper.. +type Option func(*option) + +type option struct { + extraMethods []string +} + +// WithExtraMethods adds additional HTTP methods to the list of allowed methods. +// See https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods for the default list. +// +// See the example for ExampleInstrumentHandlerWithExtraMethods for example usage. +func WithExtraMethods(methods ...string) Option { + return func(o *option) { + o.extraMethods = methods + } +} diff --git a/prometheus/promhttp/option_test.go b/prometheus/promhttp/option_test.go new file mode 100644 index 0000000..301fd06 --- /dev/null +++ b/prometheus/promhttp/option_test.go @@ -0,0 +1,65 @@ +// Copyright 2022 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package promhttp + +import ( + "log" + "net/http" + + "github.com/prometheus/client_golang/prometheus" +) + +func ExampleInstrumentHandlerWithExtraMethods() { + counter := prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "api_requests_total", + Help: "A counter for requests to the wrapped handler.", + }, + []string{"code", "method"}, + ) + + // duration is partitioned by the HTTP method and handler. It uses custom + // buckets based on the expected request duration. + duration := prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "request_duration_seconds", + Help: "A histogram of latencies for requests.", + Buckets: []float64{.25, .5, 1, 2.5, 5, 10}, + }, + []string{"handler", "method"}, + ) + + // Create the handlers that will be wrapped by the middleware. + pullHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("Pull")) + }) + + // Specify additional HTTP methods to be added to the label allow list. + opts := WithExtraMethods("CUSTOM_METHOD") + + // Instrument the handlers with all the metrics, injecting the "handler" + // label by currying. + pullChain := + InstrumentHandlerDuration(duration.MustCurryWith(prometheus.Labels{"handler": "pull"}), + InstrumentHandlerCounter(counter, pullHandler, opts), + opts, + ) + + http.Handle("/metrics", Handler()) + http.Handle("/pull", pullChain) + + if err := http.ListenAndServe(":3000", nil); err != nil { + log.Fatal(err) + } +} From 5bd9ee52dcd8aaa216b74e7fda166fcad25643c9 Mon Sep 17 00:00:00 2001 From: Kemal Akkoyun Date: Tue, 18 Jan 2022 20:32:44 +0100 Subject: [PATCH 3/9] go.*: Update dependencies (#965) * Update dependencies Signed-off-by: Kemal Akkoyun * Update deprecated APIs Signed-off-by: Kemal Akkoyun * Add missing dependency explicitly Signed-off-by: Kemal Akkoyun --- go.mod | 11 +++++---- go.sum | 23 ++++++++++++------- prometheus/counter_test.go | 6 ++--- prometheus/histogram_test.go | 6 ++--- prometheus/promhttp/instrument_server_test.go | 1 + prometheus/value.go | 6 ++--- 6 files changed, 31 insertions(+), 22 deletions(-) diff --git a/go.mod b/go.mod index c2b9327..40ff4fc 100644 --- a/go.mod +++ b/go.mod @@ -3,12 +3,13 @@ module github.com/prometheus/client_golang require ( github.com/beorn7/perks v1.0.1 github.com/cespare/xxhash/v2 v2.1.2 - github.com/golang/protobuf v1.4.3 - github.com/json-iterator/go v1.1.11 + github.com/golang/protobuf v1.5.2 + github.com/json-iterator/go v1.1.12 github.com/prometheus/client_model v0.2.0 - github.com/prometheus/common v0.29.0 - github.com/prometheus/procfs v0.6.0 - golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40 + github.com/prometheus/common v0.32.1 + github.com/prometheus/procfs v0.7.3 + golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 + google.golang.org/protobuf v1.26.0 ) go 1.13 diff --git a/go.sum b/go.sum index 6d10b69..7aae554 100644 --- a/go.sum +++ b/go.sum @@ -93,8 +93,10 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -127,8 +129,9 @@ github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2E github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= @@ -148,8 +151,9 @@ github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= @@ -170,13 +174,14 @@ github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6T github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= -github.com/prometheus/common v0.29.0 h1:3jqPBvKT4OHAbje2Ql7KeaaSicDBCxMYwEJU1zRJceE= -github.com/prometheus/common v0.29.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= +github.com/prometheus/common v0.32.1 h1:hWIdL3N2HoUx3B8j3YN9mWor0qhY/NlEKZEaXxuIRh4= +github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= +github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= @@ -311,8 +316,9 @@ golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40 h1:JWgyZ1qgdTaF3N3oxC+MdTV7qvEEgHo3otj+HB5CM7Q= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 h1:XfKQ4OlFl8okEOr5UvAqFRVj8pY/4yfcXrddB8qAbU0= +golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -443,8 +449,9 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -google.golang.org/protobuf v1.26.0-rc.1 h1:7QnIQpGRHE5RnLKnESfDoxm2dTapTZua5a0kS0A+VXQ= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/prometheus/counter_test.go b/prometheus/counter_test.go index 6b359b1..9d099dc 100644 --- a/prometheus/counter_test.go +++ b/prometheus/counter_test.go @@ -21,7 +21,7 @@ import ( //nolint:staticcheck // Ignore SA1019. Need to keep deprecated package for compatibility. "github.com/golang/protobuf/proto" - "github.com/golang/protobuf/ptypes" + "google.golang.org/protobuf/types/known/timestamppb" dto "github.com/prometheus/client_model/go" ) @@ -225,8 +225,8 @@ func TestCounterExemplar(t *testing.T) { }).(*counter) counter.now = func() time.Time { return now } - ts, err := ptypes.TimestampProto(now) - if err != nil { + ts := timestamppb.New(now) + if err := ts.CheckValid(); err != nil { t.Fatal(err) } expectedExemplar := &dto.Exemplar{ diff --git a/prometheus/histogram_test.go b/prometheus/histogram_test.go index 93252aa..da57715 100644 --- a/prometheus/histogram_test.go +++ b/prometheus/histogram_test.go @@ -26,7 +26,7 @@ import ( //nolint:staticcheck // Ignore SA1019. Need to keep deprecated package for compatibility. "github.com/golang/protobuf/proto" - "github.com/golang/protobuf/ptypes" + "google.golang.org/protobuf/types/known/timestamppb" dto "github.com/prometheus/client_model/go" ) @@ -418,8 +418,8 @@ func TestHistogramExemplar(t *testing.T) { }).(*histogram) histogram.now = func() time.Time { return now } - ts, err := ptypes.TimestampProto(now) - if err != nil { + ts := timestamppb.New(now) + if err := ts.CheckValid(); err != nil { t.Fatal(err) } expectedExemplars := []*dto.Exemplar{ diff --git a/prometheus/promhttp/instrument_server_test.go b/prometheus/promhttp/instrument_server_test.go index 5db50fb..fbd8fa9 100644 --- a/prometheus/promhttp/instrument_server_test.go +++ b/prometheus/promhttp/instrument_server_test.go @@ -145,6 +145,7 @@ func TestLabelCheck(t *testing.T) { }, append(sc.varLabels, sc.curriedLabels...), )) + //nolint:typecheck // Ignore declared but unused error. for _, l := range sc.curriedLabels { c = c.MustCurryWith(prometheus.Labels{l: "dummy"}) o = o.MustCurryWith(prometheus.Labels{l: "dummy"}) diff --git a/prometheus/value.go b/prometheus/value.go index c778711..b4e0ae1 100644 --- a/prometheus/value.go +++ b/prometheus/value.go @@ -21,7 +21,7 @@ import ( //nolint:staticcheck // Ignore SA1019. Need to keep deprecated package for compatibility. "github.com/golang/protobuf/proto" - "github.com/golang/protobuf/ptypes" + "google.golang.org/protobuf/types/known/timestamppb" dto "github.com/prometheus/client_model/go" ) @@ -183,8 +183,8 @@ const ExemplarMaxRunes = 64 func newExemplar(value float64, ts time.Time, l Labels) (*dto.Exemplar, error) { e := &dto.Exemplar{} e.Value = proto.Float64(value) - tsProto, err := ptypes.TimestampProto(ts) - if err != nil { + tsProto := timestamppb.New(ts) + if err := tsProto.CheckValid(); err != nil { return nil, err } e.Timestamp = tsProto From 01087964d02726ef7438fdb19c2672e6e5d0b48a Mon Sep 17 00:00:00 2001 From: Kemal Akkoyun Date: Wed, 19 Jan 2022 08:59:25 +0100 Subject: [PATCH 4/9] Cut v1.12.0 (#966) * Cut v1.12.0 Signed-off-by: Kemal Akkoyun * Bump the day Signed-off-by: Kemal Akkoyun --- CHANGELOG.md | 8 ++++++++ VERSION | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b5d774d..2740082 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,11 @@ +## 1.12.0 / 2022-01-19 + +* [CHANGE] example/random: Move flags and metrics into main() #935 +* [FEATURE] API client: Support wal replay status api #944 +* [FEATURE] Use the runtime/metrics package for the Go collector for 1.17+ #955 +* [ENHANCEMENT] API client: Update /api/v1/status/tsdb to include headStats #925 +* [ENHANCEMENT] promhttp: Check validity of method and code label values #962 + ## 1.11.0 / 2021-06-07 * [CHANGE] Add new collectors package. #862 diff --git a/VERSION b/VERSION index 1cac385..0eed1a2 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.11.0 +1.12.0 From f63e219e6b9074f8a55c8475e7b11720bdfc3737 Mon Sep 17 00:00:00 2001 From: Michael Knyszek Date: Fri, 21 Jan 2022 02:34:45 -0500 Subject: [PATCH 5/9] Make the Go 1.17 collector thread-safe (#969) --- prometheus/collector.go | 8 +++++++ prometheus/go_collector_go117.go | 32 +++++++++++++++++++++------ prometheus/go_collector_go117_test.go | 22 ++++++++++++++++++ 3 files changed, 55 insertions(+), 7 deletions(-) diff --git a/prometheus/collector.go b/prometheus/collector.go index 1e83965..ac1ca3c 100644 --- a/prometheus/collector.go +++ b/prometheus/collector.go @@ -118,3 +118,11 @@ func (c *selfCollector) Describe(ch chan<- *Desc) { func (c *selfCollector) Collect(ch chan<- Metric) { ch <- c.self } + +// collectorMetric is a metric that is also a collector. +// Because of selfCollector, most (if not all) Metrics in +// this package are also collectors. +type collectorMetric interface { + Metric + Collector +} diff --git a/prometheus/go_collector_go117.go b/prometheus/go_collector_go117.go index d534742..5046840 100644 --- a/prometheus/go_collector_go117.go +++ b/prometheus/go_collector_go117.go @@ -32,9 +32,10 @@ type goCollector struct { base baseGoCollector // rm... fields all pertain to the runtime/metrics package. + rmSampleMu sync.Mutex rmSampleBuf []metrics.Sample rmSampleMap map[string]*metrics.Sample - rmMetrics []Metric + rmMetrics []collectorMetric // With Go 1.17, the runtime/metrics package was introduced. // From that point on, metric names produced by the runtime/metrics @@ -58,7 +59,7 @@ func NewGoCollector() Collector { } // Generate a Desc and ValueType for each runtime/metrics metric. - metricSet := make([]Metric, 0, len(descriptions)) + metricSet := make([]collectorMetric, 0, len(descriptions)) sampleBuf := make([]metrics.Sample, 0, len(descriptions)) sampleMap := make(map[string]*metrics.Sample, len(descriptions)) for i := range descriptions { @@ -76,7 +77,7 @@ func NewGoCollector() Collector { sampleBuf = append(sampleBuf, metrics.Sample{Name: d.Name}) sampleMap[d.Name] = &sampleBuf[len(sampleBuf)-1] - var m Metric + var m collectorMetric if d.Kind == metrics.KindFloat64Histogram { _, hasSum := rmExactSumMap[d.Name] m = newBatchHistogram( @@ -130,9 +131,19 @@ func (c *goCollector) Collect(ch chan<- Metric) { // Collect base non-memory metrics. c.base.Collect(ch) + // Collect must be thread-safe, so prevent concurrent use of + // rmSampleBuf. Just read into rmSampleBuf but write all the data + // we get into our Metrics or MemStats. + // + // Note that we cannot simply read and then clone rmSampleBuf + // because we'd need to perform a deep clone of it, which is likely + // not worth it. + c.rmSampleMu.Lock() + // Populate runtime/metrics sample buffer. metrics.Read(c.rmSampleBuf) + // Update all our metrics from rmSampleBuf. for i, sample := range c.rmSampleBuf { // N.B. switch on concrete type because it's significantly more efficient // than checking for the Counter and Gauge interface implementations. In @@ -146,22 +157,29 @@ func (c *goCollector) Collect(ch chan<- Metric) { if v1 > v0 { m.Add(unwrapScalarRMValue(sample.Value) - m.get()) } - m.Collect(ch) case *gauge: m.Set(unwrapScalarRMValue(sample.Value)) - m.Collect(ch) case *batchHistogram: m.update(sample.Value.Float64Histogram(), c.exactSumFor(sample.Name)) - m.Collect(ch) default: panic("unexpected metric type") } } - // ms is a dummy MemStats that we populate ourselves so that we can // populate the old metrics from it. var ms runtime.MemStats memStatsFromRM(&ms, c.rmSampleMap) + + c.rmSampleMu.Unlock() + + // Export all the metrics to ch. + // At this point we must not access rmSampleBuf or rmSampleMap, because + // a concurrent caller could use it. It's safe to Collect all our Metrics, + // however, because they're updated in a thread-safe way while MemStats + // is local to this call of Collect. + for _, m := range c.rmMetrics { + m.Collect(ch) + } for _, i := range c.msMetrics { ch <- MustNewConstMetric(i.desc, i.valType, i.eval(&ms)) } diff --git a/prometheus/go_collector_go117_test.go b/prometheus/go_collector_go117_test.go index 653e332..e780bce 100644 --- a/prometheus/go_collector_go117_test.go +++ b/prometheus/go_collector_go117_test.go @@ -280,3 +280,25 @@ func TestExpectedRuntimeMetrics(t *testing.T) { t.Log("where X is the Go version you are currently using") } } + +func TestGoCollectorConcurrency(t *testing.T) { + c := NewGoCollector().(*goCollector) + + // Set up multiple goroutines to Collect from the + // same GoCollector. In race mode with GOMAXPROCS > 1, + // this test should fail often if Collect is not + // concurrent-safe. + for i := 0; i < 4; i++ { + go func() { + ch := make(chan Metric) + go func() { + // Drain all metrics recieved until the + // channel is closed. + for range ch { + } + }() + c.Collect(ch) + close(ch) + }() + } +} From 85206714ae0528f3c7b303a69c0a7383e1f99673 Mon Sep 17 00:00:00 2001 From: Michael Knyszek Date: Tue, 25 Jan 2022 02:43:45 -0500 Subject: [PATCH 6/9] Use simpler locking in the Go 1.17 collector (#975) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A previous PR made it so that the Go 1.17 collector locked only around uses of rmSampleBuf, but really that means that Metric values may be sent over the channel containing some values from future metrics.Read calls. While generally-speaking this isn't a problem, we lose any consistency guarantees provided by the runtime/metrics package. Also, that optimization to not just lock around all of Collect was premature. Truthfully, Collect is called relatively infrequently, and its critical path is fairly fast (10s of µs). To prove it, this change also adds a benchmark. name old time/op new time/op delta GoCollector-16 43.7µs ± 2% 43.2µs ± 2% ~ (p=0.190 n=9+9) Note that because the benchmark is single-threaded it actually looks like it might be getting *slightly* faster, because all those Collect calls for the Metrics are direct calls instead of interface calls. Signed-off-by: Michael Anthony Knyszek --- prometheus/go_collector_go117.go | 33 ++++++++++++++------------- prometheus/go_collector_go117_test.go | 2 +- prometheus/go_collector_test.go | 17 ++++++++++++++ 3 files changed, 35 insertions(+), 17 deletions(-) diff --git a/prometheus/go_collector_go117.go b/prometheus/go_collector_go117.go index 5046840..a9b1fba 100644 --- a/prometheus/go_collector_go117.go +++ b/prometheus/go_collector_go117.go @@ -31,8 +31,11 @@ import ( type goCollector struct { base baseGoCollector + // mu protects updates to all fields ensuring a consistent + // snapshot is always produced by Collect. + mu sync.Mutex + // rm... fields all pertain to the runtime/metrics package. - rmSampleMu sync.Mutex rmSampleBuf []metrics.Sample rmSampleMap map[string]*metrics.Sample rmMetrics []collectorMetric @@ -135,10 +138,16 @@ func (c *goCollector) Collect(ch chan<- Metric) { // rmSampleBuf. Just read into rmSampleBuf but write all the data // we get into our Metrics or MemStats. // - // Note that we cannot simply read and then clone rmSampleBuf - // because we'd need to perform a deep clone of it, which is likely - // not worth it. - c.rmSampleMu.Lock() + // This lock also ensures that the Metrics we send out are all from + // the same updates, ensuring their mutual consistency insofar as + // is guaranteed by the runtime/metrics package. + // + // N.B. This locking is heavy-handed, but Collect is expected to be called + // relatively infrequently. Also the core operation here, metrics.Read, + // is fast (O(tens of microseconds)) so contention should certainly be + // low, though channel operations and any allocations may add to that. + c.mu.Lock() + defer c.mu.Unlock() // Populate runtime/metrics sample buffer. metrics.Read(c.rmSampleBuf) @@ -157,10 +166,13 @@ func (c *goCollector) Collect(ch chan<- Metric) { if v1 > v0 { m.Add(unwrapScalarRMValue(sample.Value) - m.get()) } + m.Collect(ch) case *gauge: m.Set(unwrapScalarRMValue(sample.Value)) + m.Collect(ch) case *batchHistogram: m.update(sample.Value.Float64Histogram(), c.exactSumFor(sample.Name)) + m.Collect(ch) default: panic("unexpected metric type") } @@ -169,17 +181,6 @@ func (c *goCollector) Collect(ch chan<- Metric) { // populate the old metrics from it. var ms runtime.MemStats memStatsFromRM(&ms, c.rmSampleMap) - - c.rmSampleMu.Unlock() - - // Export all the metrics to ch. - // At this point we must not access rmSampleBuf or rmSampleMap, because - // a concurrent caller could use it. It's safe to Collect all our Metrics, - // however, because they're updated in a thread-safe way while MemStats - // is local to this call of Collect. - for _, m := range c.rmMetrics { - m.Collect(ch) - } for _, i := range c.msMetrics { ch <- MustNewConstMetric(i.desc, i.valType, i.eval(&ms)) } diff --git a/prometheus/go_collector_go117_test.go b/prometheus/go_collector_go117_test.go index e780bce..fe715fc 100644 --- a/prometheus/go_collector_go117_test.go +++ b/prometheus/go_collector_go117_test.go @@ -292,7 +292,7 @@ func TestGoCollectorConcurrency(t *testing.T) { go func() { ch := make(chan Metric) go func() { - // Drain all metrics recieved until the + // Drain all metrics received until the // channel is closed. for range ch { } diff --git a/prometheus/go_collector_test.go b/prometheus/go_collector_test.go index 9cc1b2e..47b944d 100644 --- a/prometheus/go_collector_test.go +++ b/prometheus/go_collector_test.go @@ -154,3 +154,20 @@ func TestGoCollectorGC(t *testing.T) { break } } + +func BenchmarkGoCollector(b *testing.B) { + c := NewGoCollector().(*goCollector) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + ch := make(chan Metric, 8) + go func() { + // Drain all metrics received until the + // channel is closed. + for range ch { + } + }() + c.Collect(ch) + close(ch) + } +} From 4dd3cbb4ab52a73d9046b721b5b5bdffa9e10922 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Tue, 25 Jan 2022 10:16:10 +0000 Subject: [PATCH 7/9] API client: make http reads more efficient (#976) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace `io.ReadAll` with `bytes.Buffer.ReadFrom`. Both need to resize a buffer until they have finished reading; the former increases by 1.25x each time while the latter uses 2x. Also added a benchmark to demonstrate the benefit: name old time/op new time/op delta Client/4KB-8 35.9µs ± 4% 35.3µs ± 3% ~ (p=0.310 n=5+5) Client/50KB-8 83.1µs ± 8% 69.5µs ± 1% -16.37% (p=0.008 n=5+5) Client/1000KB-8 891µs ± 6% 750µs ± 0% -15.83% (p=0.016 n=5+4) Client/2000KB-8 1.74ms ± 2% 1.35ms ± 1% -22.72% (p=0.008 n=5+5) name old alloc/op new alloc/op delta Client/4KB-8 20.2kB ± 0% 20.4kB ± 0% +1.26% (p=0.008 n=5+5) Client/50KB-8 218kB ± 0% 136kB ± 0% -37.65% (p=0.008 n=5+5) Client/1000KB-8 5.88MB ± 0% 2.11MB ± 0% -64.10% (p=0.008 n=5+5) Client/2000KB-8 11.7MB ± 0% 4.2MB ± 0% -63.93% (p=0.008 n=5+5) name old allocs/op new allocs/op delta Client/4KB-8 75.0 ± 0% 72.0 ± 0% -4.00% (p=0.008 n=5+5) Client/50KB-8 109 ± 0% 98 ± 0% -10.09% (p=0.079 n=4+5) Client/1000KB-8 617 ± 0% 593 ± 0% -3.89% (p=0.008 n=5+5) Client/2000KB-8 1.13k ± 0% 1.09k ± 0% -3.27% (p=0.008 n=5+5) Signed-off-by: Bryan Boreham --- api/client.go | 6 ++++-- api/client_test.go | 51 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 2 deletions(-) diff --git a/api/client.go b/api/client.go index f7ca60b..1413f65 100644 --- a/api/client.go +++ b/api/client.go @@ -15,8 +15,8 @@ package api import ( + "bytes" "context" - "io/ioutil" "net" "net/http" "net/url" @@ -111,7 +111,9 @@ func (c *httpClient) Do(ctx context.Context, req *http.Request) (*http.Response, var body []byte done := make(chan struct{}) go func() { - body, err = ioutil.ReadAll(resp.Body) + var buf bytes.Buffer + _, err = buf.ReadFrom(resp.Body) + body = buf.Bytes() close(done) }() diff --git a/api/client_test.go b/api/client_test.go index 47094fc..4215c73 100644 --- a/api/client_test.go +++ b/api/client_test.go @@ -14,7 +14,11 @@ package api import ( + "bytes" + "context" + "fmt" "net/http" + "net/http/httptest" "net/url" "testing" ) @@ -111,3 +115,50 @@ func TestClientURL(t *testing.T) { } } } + +// Serve any http request with a response of N KB of spaces. +type serveSpaces struct { + sizeKB int +} + +func (t serveSpaces) ServeHTTP(w http.ResponseWriter, req *http.Request) { + kb := bytes.Repeat([]byte{' '}, 1024) + for i := 0; i < t.sizeKB; i++ { + w.Write(kb) + } +} + +func BenchmarkClient(b *testing.B) { + b.ReportAllocs() + ctx := context.Background() + + for _, sizeKB := range []int{4, 50, 1000, 2000} { + b.Run(fmt.Sprintf("%dKB", sizeKB), func(b *testing.B) { + + testServer := httptest.NewServer(serveSpaces{sizeKB}) + defer testServer.Close() + + client, err := NewClient(Config{ + Address: testServer.URL, + }) + if err != nil { + b.Fatalf("Failed to initialize client: %v", err) + } + url, err := url.Parse(testServer.URL + "/prometheus/api/v1/query?query=up") + if err != nil { + b.Fatalf("Failed to parse url: %v", err) + } + req := &http.Request{ + URL: url, + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, _, err := client.Do(ctx, req) + if err != nil { + b.Fatalf("Query failed: %v", err) + } + } + b.StopTimer() + }) + } +} From 77626d64fa02954e546be20b688e235617e42c7e Mon Sep 17 00:00:00 2001 From: Michael Knyszek Date: Thu, 27 Jan 2022 23:46:45 -0500 Subject: [PATCH 8/9] Reduce granularity of histogram buckets for Go 1.17 collector (#974) The Go runtime/metrics package currently exports extremely granular histograms. Exponentially bucket any histogram with unit "seconds" or "bytes" instead to dramatically reduce the number of buckets, and thus the number of metrics. This change also adds a test to check for expected cardinality to prevent cardinality surprises in the future. Signed-off-by: Michael Anthony Knyszek --- prometheus/gen_go_collector_metrics_set.go | 50 ++++++++++++++ prometheus/go_collector_go117.go | 67 +++++++++++++------ prometheus/go_collector_go117_test.go | 48 +++++++++++-- prometheus/go_collector_metrics_go117_test.go | 2 + prometheus/internal/go_runtime_metrics.go | 65 ++++++++++++++++++ 5 files changed, 205 insertions(+), 27 deletions(-) diff --git a/prometheus/gen_go_collector_metrics_set.go b/prometheus/gen_go_collector_metrics_set.go index ac63777..e33b397 100644 --- a/prometheus/gen_go_collector_metrics_set.go +++ b/prometheus/gen_go_collector_metrics_set.go @@ -21,7 +21,9 @@ import ( "fmt" "go/format" "log" + "math" "os" + "runtime" "runtime/metrics" "strconv" "strings" @@ -35,6 +37,10 @@ func main() { if len(os.Args) != 2 { log.Fatal("requires Go version (e.g. go1.17) as an argument") } + toolVersion := runtime.Version() + if majorVersion := toolVersion[:strings.LastIndexByte(toolVersion, '.')]; majorVersion != os.Args[1] { + log.Fatalf("using Go version %q but expected Go version %q", majorVersion, os.Args[1]) + } version, err := parseVersion(os.Args[1]) if err != nil { log.Fatalf("parsing Go version: %v", err) @@ -45,9 +51,11 @@ func main() { err = testFile.Execute(&buf, struct { Descriptions []metrics.Description GoVersion goVersion + Cardinality int }{ Descriptions: metrics.All(), GoVersion: version, + Cardinality: rmCardinality(), }) if err != nil { log.Fatalf("executing template: %v", err) @@ -85,6 +93,46 @@ func parseVersion(s string) (goVersion, error) { return goVersion(i), err } +func rmCardinality() int { + cardinality := 0 + + // Collect all histogram samples so that we can get their buckets. + // The API guarantees that the buckets are always fixed for the lifetime + // of the process. + var histograms []metrics.Sample + for _, d := range metrics.All() { + if d.Kind == metrics.KindFloat64Histogram { + histograms = append(histograms, metrics.Sample{Name: d.Name}) + } else { + cardinality++ + } + } + + // Handle histograms. + metrics.Read(histograms) + for i := range histograms { + name := histograms[i].Name + buckets := internal.RuntimeMetricsBucketsForUnit( + histograms[i].Value.Float64Histogram().Buckets, + name[strings.IndexRune(name, ':')+1:], + ) + cardinality += len(buckets) + 3 // Plus total count, sum, and the implicit infinity bucket. + // runtime/metrics bucket boundaries are lower-bound-inclusive, but + // always represents each actual *boundary* so Buckets is always + // 1 longer than Counts, while in Prometheus the mapping is one-to-one, + // as the bottom bucket extends to -Inf, and the top infinity bucket is + // implicit. Therefore, we should have one fewer bucket than is listed + // above. + cardinality-- + if buckets[len(buckets)-1] == math.Inf(1) { + // We already counted the infinity bucket separately. + cardinality-- + } + } + + return cardinality +} + var testFile = template.Must(template.New("testFile").Funcs(map[string]interface{}{ "rm2prom": func(d metrics.Description) string { ns, ss, n, ok := internal.RuntimeMetricsToProm(&d) @@ -112,4 +160,6 @@ var expectedRuntimeMetrics = map[string]string{ {{- end -}} {{end}} } + +const expectedRuntimeMetricsCardinality = {{.Cardinality}} `)) diff --git a/prometheus/go_collector_go117.go b/prometheus/go_collector_go117.go index a9b1fba..d43bdcd 100644 --- a/prometheus/go_collector_go117.go +++ b/prometheus/go_collector_go117.go @@ -20,6 +20,7 @@ import ( "math" "runtime" "runtime/metrics" + "strings" "sync" //nolint:staticcheck // Ignore SA1019. Need to keep deprecated package for compatibility. @@ -56,9 +57,20 @@ type goCollector struct { // Deprecated: Use collectors.NewGoCollector instead. func NewGoCollector() Collector { descriptions := metrics.All() - descMap := make(map[string]*metrics.Description) - for i := range descriptions { - descMap[descriptions[i].Name] = &descriptions[i] + + // Collect all histogram samples so that we can get their buckets. + // The API guarantees that the buckets are always fixed for the lifetime + // of the process. + var histograms []metrics.Sample + for _, d := range descriptions { + if d.Kind == metrics.KindFloat64Histogram { + histograms = append(histograms, metrics.Sample{Name: d.Name}) + } + } + metrics.Read(histograms) + bucketsMap := make(map[string][]float64) + for i := range histograms { + bucketsMap[histograms[i].Name] = histograms[i].Value.Float64Histogram().Buckets } // Generate a Desc and ValueType for each runtime/metrics metric. @@ -83,6 +95,7 @@ func NewGoCollector() Collector { var m collectorMetric if d.Kind == metrics.KindFloat64Histogram { _, hasSum := rmExactSumMap[d.Name] + unit := d.Name[strings.IndexRune(d.Name, ':')+1:] m = newBatchHistogram( NewDesc( BuildFQName(namespace, subsystem, name), @@ -90,6 +103,7 @@ func NewGoCollector() Collector { nil, nil, ), + internal.RuntimeMetricsBucketsForUnit(bucketsMap[d.Name], unit), hasSum, ) } else if d.Cumulative { @@ -299,13 +313,27 @@ type batchHistogram struct { // but Write calls may operate concurrently with updates. // Contention between these two sources should be rare. mu sync.Mutex - buckets []float64 // Inclusive lower bounds. + buckets []float64 // Inclusive lower bounds, like runtime/metrics. counts []uint64 sum float64 // Used if hasSum is true. } -func newBatchHistogram(desc *Desc, hasSum bool) *batchHistogram { - h := &batchHistogram{desc: desc, hasSum: hasSum} +// newBatchHistogram creates a new batch histogram value with the given +// Desc, buckets, and whether or not it has an exact sum available. +// +// buckets must always be from the runtime/metrics package, following +// the same conventions. +func newBatchHistogram(desc *Desc, buckets []float64, hasSum bool) *batchHistogram { + h := &batchHistogram{ + desc: desc, + buckets: buckets, + // Because buckets follows runtime/metrics conventions, there's + // 1 more value in the buckets list than there are buckets represented, + // because in runtime/metrics, the bucket values represent *boundaries*, + // and non-Inf boundaries are inclusive lower bounds for that bucket. + counts: make([]uint64, len(buckets)-1), + hasSum: hasSum, + } h.init(h) return h } @@ -313,28 +341,25 @@ func newBatchHistogram(desc *Desc, hasSum bool) *batchHistogram { // update updates the batchHistogram from a runtime/metrics histogram. // // sum must be provided if the batchHistogram was created to have an exact sum. +// h.buckets must be a strict subset of his.Buckets. func (h *batchHistogram) update(his *metrics.Float64Histogram, sum float64) { counts, buckets := his.Counts, his.Buckets - // Skip a -Inf bucket altogether. It's not clear how to represent that. - if math.IsInf(buckets[0], -1) { - buckets = buckets[1:] - counts = counts[1:] - } h.mu.Lock() defer h.mu.Unlock() - // Check if we're initialized. - if h.buckets == nil { - // Make copies of counts and buckets. It's really important - // that we don't retain his.Counts or his.Buckets anywhere since - // it's going to get reused. - h.buckets = make([]float64, len(buckets)) - copy(h.buckets, buckets) - - h.counts = make([]uint64, len(counts)) + // Clear buckets. + for i := range h.counts { + h.counts[i] = 0 + } + // Copy and reduce buckets. + var j int + for i, count := range counts { + h.counts[j] += count + if buckets[i+1] == h.buckets[j+1] { + j++ + } } - copy(h.counts, counts) if h.hasSum { h.sum = sum } diff --git a/prometheus/go_collector_go117_test.go b/prometheus/go_collector_go117_test.go index fe715fc..9c5218f 100644 --- a/prometheus/go_collector_go117_test.go +++ b/prometheus/go_collector_go117_test.go @@ -140,9 +140,13 @@ func TestBatchHistogram(t *testing.T) { } metrics.Read(s) rmHist := s[0].Value.Float64Histogram() - // runtime/metrics histograms always have -Inf and +Inf buckets. - // We never handle -Inf and +Inf is implicit. - wantBuckets := len(rmHist.Buckets) - 2 + wantBuckets := internal.RuntimeMetricsBucketsForUnit(rmHist.Buckets, "bytes") + // runtime/metrics histograms always have a +Inf bucket and are lower + // bound inclusive. In contrast, we have an implicit +Inf bucket and + // are upper bound inclusive, so we can chop off the first bucket + // (since the conversion to upper bound inclusive will shift all buckets + // down one index) and the +Inf for the last bucket. + wantBuckets = wantBuckets[1 : len(wantBuckets)-1] // Check to make sure the output proto makes sense. pb := &dto.Metric{} @@ -151,14 +155,14 @@ func TestBatchHistogram(t *testing.T) { if math.IsInf(pb.Histogram.Bucket[len(pb.Histogram.Bucket)-1].GetUpperBound(), +1) { t.Errorf("found +Inf bucket") } - if got := len(pb.Histogram.Bucket); got != wantBuckets { - t.Errorf("got %d buckets in protobuf, want %d", got, wantBuckets) + if got := len(pb.Histogram.Bucket); got != len(wantBuckets) { + t.Errorf("got %d buckets in protobuf, want %d", got, len(wantBuckets)) } for i, bucket := range pb.Histogram.Bucket { // runtime/metrics histograms are lower-bound inclusive, but we're // upper-bound inclusive. So just make sure the new inclusive upper // bound is somewhere close by (in some cases it's equal). - wantBound := rmHist.Buckets[i+1] + wantBound := wantBuckets[i] if gotBound := *bucket.UpperBound; (wantBound-gotBound)/wantBound > 0.001 { t.Errorf("got bound %f, want within 0.1%% of %f", gotBound, wantBound) } @@ -244,6 +248,7 @@ func TestExpectedRuntimeMetrics(t *testing.T) { descs := metrics.All() rmSet := make(map[string]struct{}) + // Iterate over runtime-reported descriptions to find new metrics. for i := range descs { rmName := descs[i].Name rmSet[rmName] = struct{}{} @@ -263,6 +268,8 @@ func TestExpectedRuntimeMetrics(t *testing.T) { continue } } + // Now iterate over the expected metrics and look for removals. + cardinality := 0 for rmName, fqName := range expectedRuntimeMetrics { if _, ok := rmSet[rmName]; !ok { t.Errorf("runtime/metrics metric %s removed", rmName) @@ -272,6 +279,30 @@ func TestExpectedRuntimeMetrics(t *testing.T) { t.Errorf("runtime/metrics metric %s not appearing under expected name %s", rmName, fqName) continue } + + // While we're at it, check to make sure expected cardinality lines + // up, but at the point of the protobuf write to get as close to the + // real deal as possible. + // + // Note that we filter out non-runtime/metrics metrics here, because + // those are manually managed. + var m dto.Metric + if err := goMetricSet[fqName].Write(&m); err != nil { + t.Errorf("writing metric %s: %v", fqName, err) + continue + } + // N.B. These are the only fields populated by runtime/metrics metrics specifically. + // Other fields are populated by e.g. GCStats metrics. + switch { + case m.Counter != nil: + fallthrough + case m.Gauge != nil: + cardinality++ + case m.Histogram != nil: + cardinality += len(m.Histogram.Bucket) + 3 // + sum, count, and +inf + default: + t.Errorf("unexpected protobuf structure for metric %s", fqName) + } } if t.Failed() { @@ -279,6 +310,11 @@ func TestExpectedRuntimeMetrics(t *testing.T) { t.Log("\tgo run gen_go_collector_metrics_set.go go1.X") t.Log("where X is the Go version you are currently using") } + + expectCardinality := expectedRuntimeMetricsCardinality + if cardinality != expectCardinality { + t.Errorf("unexpected cardinality for runtime/metrics metrics: got %d, want %d", cardinality, expectCardinality) + } } func TestGoCollectorConcurrency(t *testing.T) { diff --git a/prometheus/go_collector_metrics_go117_test.go b/prometheus/go_collector_metrics_go117_test.go index 6c0a693..20e98ef 100644 --- a/prometheus/go_collector_metrics_go117_test.go +++ b/prometheus/go_collector_metrics_go117_test.go @@ -37,3 +37,5 @@ var expectedRuntimeMetrics = map[string]string{ "/sched/goroutines:goroutines": "go_sched_goroutines_goroutines", "/sched/latencies:seconds": "go_sched_latencies_seconds", } + +const expectedRuntimeMetricsCardinality = 79 diff --git a/prometheus/internal/go_runtime_metrics.go b/prometheus/internal/go_runtime_metrics.go index afc8dff..fe0a521 100644 --- a/prometheus/internal/go_runtime_metrics.go +++ b/prometheus/internal/go_runtime_metrics.go @@ -17,6 +17,7 @@ package internal import ( + "math" "path" "runtime/metrics" "strings" @@ -75,3 +76,67 @@ func RuntimeMetricsToProm(d *metrics.Description) (string, string, string, bool) } return namespace, subsystem, name, valid } + +// RuntimeMetricsBucketsForUnit takes a set of buckets obtained for a runtime/metrics histogram +// type (so, lower-bound inclusive) and a unit from a runtime/metrics name, and produces +// a reduced set of buckets. This function always removes any -Inf bucket as it's represented +// as the bottom-most upper-bound inclusive bucket in Prometheus. +func RuntimeMetricsBucketsForUnit(buckets []float64, unit string) []float64 { + switch unit { + case "bytes": + // Rebucket as powers of 2. + return rebucketExp(buckets, 2) + case "seconds": + // Rebucket as powers of 10 and then merge all buckets greater + // than 1 second into the +Inf bucket. + b := rebucketExp(buckets, 10) + for i := range b { + if b[i] <= 1 { + continue + } + b[i] = math.Inf(1) + b = b[:i+1] + break + } + return b + } + return buckets +} + +// rebucketExp takes a list of bucket boundaries (lower bound inclusive) and +// downsamples the buckets to those a multiple of base apart. The end result +// is a roughly exponential (in many cases, perfectly exponential) bucketing +// scheme. +func rebucketExp(buckets []float64, base float64) []float64 { + bucket := buckets[0] + var newBuckets []float64 + // We may see a -Inf here, in which case, add it and skip it + // since we risk producing NaNs otherwise. + // + // We need to preserve -Inf values to maintain runtime/metrics + // conventions. We'll strip it out later. + if bucket == math.Inf(-1) { + newBuckets = append(newBuckets, bucket) + buckets = buckets[1:] + bucket = buckets[0] + } + // From now on, bucket should always have a non-Inf value because + // Infs are only ever at the ends of the bucket lists, so + // arithmetic operations on it are non-NaN. + for i := 1; i < len(buckets); i++ { + if bucket >= 0 && buckets[i] < bucket*base { + // The next bucket we want to include is at least bucket*base. + continue + } else if bucket < 0 && buckets[i] < bucket/base { + // In this case the bucket we're targeting is negative, and since + // we're ascending through buckets here, we need to divide to get + // closer to zero exponentially. + continue + } + // The +Inf bucket will always be the last one, and we'll always + // end up including it here because bucket + newBuckets = append(newBuckets, bucket) + bucket = buckets[i] + } + return append(newBuckets, bucket) +} From 2e1c4818ccfdcf953ce399cadad615ff2bed968c Mon Sep 17 00:00:00 2001 From: Kemal Akkoyun Date: Sat, 29 Jan 2022 15:48:34 +0100 Subject: [PATCH 9/9] Cut v1.12.1 (#978) * Cut v1.12.1 Signed-off-by: Kemal Akkoyun * Apply review suggestions Signed-off-by: Kemal Akkoyun --- CHANGELOG.md | 7 +++++++ VERSION | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2740082..cf231ff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## 1.12.1 / 2022-01-29 + +* [BUGFIX] Make the Go 1.17 collector concurrency-safe #969 + * Use simpler locking in the Go 1.17 collector #975 +* [BUGFIX] Reduce granularity of histogram buckets for Go 1.17 collector #974 +* [ENHANCEMENT] API client: make HTTP reads more efficient #976 + ## 1.12.0 / 2022-01-19 * [CHANGE] example/random: Move flags and metrics into main() #935 diff --git a/VERSION b/VERSION index 0eed1a2..f8f4f03 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.12.0 +1.12.1