re
/
client_golang
forked from mirror/client_golang
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

collectors.GoCollector: Added rule support for granular metric configuration. (#1102) 

* goCollector: Added rule support for granular metric configuration.

Fixes: https://github.com/prometheus/client_golang/issues/1089

Signed-off-by: bwplotka <bwplotka@gmail.com>

* Added compatibility mode with old options. (#1107)

* Added compatibility mode with old options.

Signed-off-by: bwplotka <bwplotka@gmail.com>

* Copyright header.

Signed-off-by: bwplotka <bwplotka@gmail.com>

* Remove bucket option for now. (#1108)

Signed-off-by: bwplotka <bwplotka@gmail.com>

* collectors/GoCollector: Add tests and examples (#1109)

* Add tests and examples

Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com>

* Add docs for the presets

Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com>

Co-authored-by: Kemal Akkoyun <kakkoyun@users.noreply.github.com>
This commit is contained in:
Bartlomiej Plotka 2022-08-05 19:37:46 +02:00 committed by GitHub
parent d44fbbefdd
commit 5b7e8b2e67
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 626 additions and 181 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

186
prometheus/collectors/go_collector_latest.go
View File

@ -16,77 +16,145 @@
package collectors
import "github.com/prometheus/client_golang/prometheus"
import (
"regexp"
//nolint:staticcheck // Ignore SA1019 until v2.
type goOptions = prometheus.GoCollectorOptions
type goOption func(o *goOptions)
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/internal"
)
var (
// MetricsAll allows all the metrics to be collected from Go runtime.
MetricsAll = GoRuntimeMetricsRule{regexp.MustCompile("/.*")}
// MetricsGC allows only GC metrics to be collected from Go runtime.
// e.g. go_gc_cycles_automatic_gc_cycles_total
MetricsGC = GoRuntimeMetricsRule{regexp.MustCompile(`^/gc/.*`)}
// MetricsMemory allows only memory metrics to be collected from Go runtime.
// e.g. go_memory_classes_heap_free_bytes
MetricsMemory = GoRuntimeMetricsRule{regexp.MustCompile(`^/memory/.*`)}
// MetricsScheduler allows only scheduler metrics to be collected from Go runtime.
// e.g. go_sched_goroutines_goroutines
MetricsScheduler = GoRuntimeMetricsRule{regexp.MustCompile(`^/sched/.*`)}
)
// WithGoCollectorMemStatsMetricsDisabled disables metrics that is gathered in runtime.MemStats structure such as:
//
// go_memstats_alloc_bytes
// go_memstats_alloc_bytes_total
// go_memstats_sys_bytes
// go_memstats_lookups_total
// go_memstats_mallocs_total
// go_memstats_frees_total
// go_memstats_heap_alloc_bytes
// go_memstats_heap_sys_bytes
// go_memstats_heap_idle_bytes
// go_memstats_heap_inuse_bytes
// go_memstats_heap_released_bytes
// go_memstats_heap_objects
// go_memstats_stack_inuse_bytes
// go_memstats_stack_sys_bytes
// go_memstats_mspan_inuse_bytes
// go_memstats_mspan_sys_bytes
// go_memstats_mcache_inuse_bytes
// go_memstats_mcache_sys_bytes
// go_memstats_buck_hash_sys_bytes
// go_memstats_gc_sys_bytes
// go_memstats_other_sys_bytes
// go_memstats_next_gc_bytes
//
// so the metrics known from pre client_golang v1.12.0,
//
// NOTE(bwplotka): The above represents runtime.MemStats statistics, but they are
// actually implemented using new runtime/metrics package. (except skipped go_memstats_gc_cpu_fraction
// -- see https://github.com/prometheus/client_golang/issues/842#issuecomment-861812034 for explanation).
//
// Some users might want to disable this on collector level (although you can use scrape relabelling on Prometheus),
// because similar metrics can be now obtained using WithGoCollectorRuntimeMetrics. Note that the semantics of new
// metrics might be different, plus the names can be change over time with different Go version.
//
// NOTE(bwplotka): Changing metric names can be tedious at times as the alerts, recording rules and dashboards have to be adjusted.
// The old metrics are also very useful, with many guides and books written about how to interpret them.
//
// As a result our recommendation would be to stick with MemStats like metrics and enable other runtime/metrics if you are interested
// in advanced insights Go provides. See ExampleGoCollector_WithAdvancedGoMetrics.
func WithGoCollectorMemStatsMetricsDisabled() func(options *internal.GoCollectorOptions) {
return func(o *internal.GoCollectorOptions) {
o.DisableMemStatsLikeMetrics = true
}
}
// GoRuntimeMetricsRule allow enabling and configuring particular group of runtime/metrics.
// TODO(bwplotka): Consider adding ability to adjust buckets.
type GoRuntimeMetricsRule struct {
// Matcher represents RE2 expression will match the runtime/metrics from https://golang.bg/src/runtime/metrics/description.go
// Use `regexp.MustCompile` or `regexp.Compile` to create this field.
Matcher *regexp.Regexp
}
// WithGoCollectorRuntimeMetrics allows enabling and configuring particular group of runtime/metrics.
// See the list of metrics https://golang.bg/src/runtime/metrics/description.go (pick the Go version you use there!).
// You can use this option in repeated manner, which will add new rules. The order of rules is important, the last rule
// that matches particular metrics is applied.
func WithGoCollectorRuntimeMetrics(rules ...GoRuntimeMetricsRule) func(options *internal.GoCollectorOptions) {
rs := make([]internal.GoCollectorRule, len(rules))
for i, r := range rules {
rs[i] = internal.GoCollectorRule{
Matcher: r.Matcher,
}
}
return func(o *internal.GoCollectorOptions) {
o.RuntimeMetricRules = append(o.RuntimeMetricRules, rs...)
}
}
// WithoutGoCollectorRuntimeMetrics allows disabling group of runtime/metrics that you might have added in WithGoCollectorRuntimeMetrics.
// It behaves similarly to WithGoCollectorRuntimeMetrics just with deny-list semantics.
func WithoutGoCollectorRuntimeMetrics(matchers ...*regexp.Regexp) func(options *internal.GoCollectorOptions) {
rs := make([]internal.GoCollectorRule, len(matchers))
for i, m := range matchers {
rs[i] = internal.GoCollectorRule{
Matcher: m,
Deny: true,
}
}
return func(o *internal.GoCollectorOptions) {
o.RuntimeMetricRules = append(o.RuntimeMetricRules, rs...)
}
}
// GoCollectionOption represents Go collection option flag.
// Deprecated.
type GoCollectionOption uint32
const (
// GoRuntimeMemStatsCollection represents the metrics represented by runtime.MemStats structure such as
// go_memstats_alloc_bytes
// go_memstats_alloc_bytes_total
// go_memstats_sys_bytes
// go_memstats_lookups_total
// go_memstats_mallocs_total
// go_memstats_frees_total
// go_memstats_heap_alloc_bytes
// go_memstats_heap_sys_bytes
// go_memstats_heap_idle_bytes
// go_memstats_heap_inuse_bytes
// go_memstats_heap_released_bytes
// go_memstats_heap_objects
// go_memstats_stack_inuse_bytes
// go_memstats_stack_sys_bytes
// go_memstats_mspan_inuse_bytes
// go_memstats_mspan_sys_bytes
// go_memstats_mcache_inuse_bytes
// go_memstats_mcache_sys_bytes
// go_memstats_buck_hash_sys_bytes
// go_memstats_gc_sys_bytes
// go_memstats_other_sys_bytes
// go_memstats_next_gc_bytes
// so the metrics known from pre client_golang v1.12.0, except skipped go_memstats_gc_cpu_fraction (see
// https://github.com/prometheus/client_golang/issues/842#issuecomment-861812034 for explanation.
//
// NOTE that this mode represents runtime.MemStats statistics, but they are
// actually implemented using new runtime/metrics package.
// Deprecated: Use GoRuntimeMetricsCollection instead going forward.
// GoRuntimeMemStatsCollection represents the metrics represented by runtime.MemStats structure.
// Deprecated. Use WithGoCollectorMemStatsMetricsDisabled() function to disable those metrics in the collector.
GoRuntimeMemStatsCollection GoCollectionOption = 1 << iota
// GoRuntimeMetricsCollection is the new set of metrics represented by runtime/metrics package and follows
// consistent naming. The exposed metric set depends on Go version, but it is controlled against
// unexpected cardinality. This set has overlapping information with GoRuntimeMemStatsCollection, just with
// new names. GoRuntimeMetricsCollection is what is recommended for using going forward.
// GoRuntimeMetricsCollection is the new set of metrics represented by runtime/metrics package.
// Deprecated. Use WithGoCollectorRuntimeMetrics(GoRuntimeMetricsRule{Matcher: regexp.MustCompile("/.*")})
// function to enable those metrics in the collector.
GoRuntimeMetricsCollection
)
// WithGoCollections allows enabling different collections for Go collector on top of base metrics
// like go_goroutines, go_threads, go_gc_duration_seconds, go_memstats_last_gc_time_seconds, go_info.
//
// Check GoRuntimeMemStatsCollection and GoRuntimeMetricsCollection for more details. You can use none,
// one or more collections at once. For example:
// WithGoCollections(GoRuntimeMemStatsCollection | GoRuntimeMetricsCollection) means both GoRuntimeMemStatsCollection
// metrics and GoRuntimeMetricsCollection will be exposed.
//
// The current default is GoRuntimeMemStatsCollection, so the compatibility mode with
// client_golang pre v1.12 (move to runtime/metrics).
//nolint:staticcheck // Ignore SA1019 until v2.
func WithGoCollections(flags GoCollectionOption) func(options *prometheus.GoCollectorOptions) {
return func(o *goOptions) {
o.EnabledCollections = uint32(flags)
// WithGoCollections allows enabling different collections for Go collector on top of base metrics.
// Deprecated. Use WithGoCollectorRuntimeMetrics() and WithGoCollectorMemStatsMetricsDisabled() instead to control metrics.
func WithGoCollections(flags GoCollectionOption) func(options *internal.GoCollectorOptions) {
return func(options *internal.GoCollectorOptions) {
if flags&GoRuntimeMemStatsCollection == 0 {
WithGoCollectorMemStatsMetricsDisabled()(options)
}
if flags&GoRuntimeMetricsCollection != 0 {
WithGoCollectorRuntimeMetrics(GoRuntimeMetricsRule{Matcher: regexp.MustCompile("/.*")})(options)
}
}
}
// NewGoCollector returns a collector that exports metrics about the current Go
// process using debug.GCStats using runtime/metrics.
func NewGoCollector(opts ...goOption) prometheus.Collector {
// process using debug.GCStats (base metrics) and runtime/metrics (both in MemStats style and new ones).
func NewGoCollector(opts ...func(o *internal.GoCollectorOptions)) prometheus.Collector {
//nolint:staticcheck // Ignore SA1019 until v2.
promPkgOpts := make([]func(o *prometheus.GoCollectorOptions), len(opts))
for i, opt := range opts {
promPkgOpts[i] = opt
}
//nolint:staticcheck // Ignore SA1019 until v2.
return prometheus.NewGoCollector(promPkgOpts...)
return prometheus.NewGoCollector(opts...)
}

263
prometheus/collectors/go_collector_latest_test.go
View File

@ -18,15 +18,31 @@ package collectors
import (
"encoding/json"
"log"
"net/http"
"reflect"
"regexp"
"sort"
"testing"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
)
var baseMetrics = []string{
"go_gc_duration_seconds",
"go_goroutines",
"go_info",
"go_memstats_last_gc_time_seconds",
"go_threads",
}
func TestGoCollectorMarshalling(t *testing.T) {
reg := prometheus.NewRegistry()
reg.MustRegister(NewGoCollector(
WithGoCollections(GoRuntimeMemStatsCollection | GoRuntimeMetricsCollection),
WithGoCollectorRuntimeMetrics(GoRuntimeMetricsRule{
Matcher: regexp.MustCompile("/.*"),
}),
))
result, err := reg.Gather()
if err != nil {
@ -37,3 +53,248 @@ func TestGoCollectorMarshalling(t *testing.T) {
t.Errorf("json marshalling shoud not fail, %v", err)
}
}
func TestWithGoCollectorMemStatsMetricsDisabled(t *testing.T) {
reg := prometheus.NewRegistry()
reg.MustRegister(NewGoCollector(
WithGoCollectorMemStatsMetricsDisabled(),
))
result, err := reg.Gather()
if err != nil {
t.Fatal(err)
}
got := []string{}
for _, r := range result {
got = append(got, r.GetName())
}
if !reflect.DeepEqual(got, baseMetrics) {
t.Errorf("got %v, want %v", got, baseMetrics)
}
}
func TestGoCollectorAllowList(t *testing.T) {
for _, test := range []struct {
name string
rules []GoRuntimeMetricsRule
expected []string
}{
{
name: "Without any rules",
rules: nil,
expected: baseMetrics,
},
{
name: "allow all",
rules: []GoRuntimeMetricsRule{MetricsAll},
expected: withBaseMetrics([]string{
"go_gc_cycles_automatic_gc_cycles_total",
"go_gc_cycles_forced_gc_cycles_total",
"go_gc_cycles_total_gc_cycles_total",
"go_gc_heap_allocs_by_size_bytes",
"go_gc_heap_allocs_bytes_total",
"go_gc_heap_allocs_objects_total",
"go_gc_heap_frees_by_size_bytes",
"go_gc_heap_frees_bytes_total",
"go_gc_heap_frees_objects_total",
"go_gc_heap_goal_bytes",
"go_gc_heap_objects_objects",
"go_gc_heap_tiny_allocs_objects_total",
"go_gc_pauses_seconds",
"go_memory_classes_heap_free_bytes",
"go_memory_classes_heap_objects_bytes",
"go_memory_classes_heap_released_bytes",
"go_memory_classes_heap_stacks_bytes",
"go_memory_classes_heap_unused_bytes",
"go_memory_classes_metadata_mcache_free_bytes",
"go_memory_classes_metadata_mcache_inuse_bytes",
"go_memory_classes_metadata_mspan_free_bytes",
"go_memory_classes_metadata_mspan_inuse_bytes",
"go_memory_classes_metadata_other_bytes",
"go_memory_classes_os_stacks_bytes",
"go_memory_classes_other_bytes",
"go_memory_classes_profiling_buckets_bytes",
"go_memory_classes_total_bytes",
"go_sched_goroutines_goroutines",
"go_sched_latencies_seconds",
}),
},
{
name: "allow GC",
rules: []GoRuntimeMetricsRule{MetricsGC},
expected: withBaseMetrics([]string{
"go_gc_cycles_automatic_gc_cycles_total",
"go_gc_cycles_forced_gc_cycles_total",
"go_gc_cycles_total_gc_cycles_total",
"go_gc_heap_allocs_by_size_bytes",
"go_gc_heap_allocs_bytes_total",
"go_gc_heap_allocs_objects_total",
"go_gc_heap_frees_by_size_bytes",
"go_gc_heap_frees_bytes_total",
"go_gc_heap_frees_objects_total",
"go_gc_heap_goal_bytes",
"go_gc_heap_objects_objects",
"go_gc_heap_tiny_allocs_objects_total",
"go_gc_pauses_seconds",
}),
},
{
name: "allow Memory",
rules: []GoRuntimeMetricsRule{MetricsMemory},
expected: withBaseMetrics([]string{
"go_memory_classes_heap_free_bytes",
"go_memory_classes_heap_objects_bytes",
"go_memory_classes_heap_released_bytes",
"go_memory_classes_heap_stacks_bytes",
"go_memory_classes_heap_unused_bytes",
"go_memory_classes_metadata_mcache_free_bytes",
"go_memory_classes_metadata_mcache_inuse_bytes",
"go_memory_classes_metadata_mspan_free_bytes",
"go_memory_classes_metadata_mspan_inuse_bytes",
"go_memory_classes_metadata_other_bytes",
"go_memory_classes_os_stacks_bytes",
"go_memory_classes_other_bytes",
"go_memory_classes_profiling_buckets_bytes",
"go_memory_classes_total_bytes",
}),
},
{
name: "allow Scheduler",
rules: []GoRuntimeMetricsRule{MetricsScheduler},
expected: []string{
"go_gc_duration_seconds",
"go_goroutines",
"go_info",
"go_memstats_last_gc_time_seconds",
"go_sched_goroutines_goroutines",
"go_sched_latencies_seconds",
"go_threads",
},
},
} {
t.Run(test.name, func(t *testing.T) {
reg := prometheus.NewRegistry()
reg.MustRegister(NewGoCollector(
WithGoCollectorMemStatsMetricsDisabled(),
WithGoCollectorRuntimeMetrics(test.rules...),
))
result, err := reg.Gather()
if err != nil {
t.Fatal(err)
}
got := []string{}
for _, r := range result {
got = append(got, r.GetName())
}
if !reflect.DeepEqual(got, test.expected) {
t.Errorf("got %v, want %v", got, test.expected)
}
})
}
}
func withBaseMetrics(metricNames []string) []string {
metricNames = append(metricNames, baseMetrics...)
sort.Strings(metricNames)
return metricNames
}
func TestGoCollectorDenyList(t *testing.T) {
for _, test := range []struct {
name string
matchers []*regexp.Regexp
expected []string
}{
{
name: "Without any matchers",
matchers: nil,
expected: baseMetrics,
},
{
name: "deny all",
matchers: []*regexp.Regexp{regexp.MustCompile("/.*")},
expected: baseMetrics,
},
{
name: "deny gc and scheduler latency",
matchers: []*regexp.Regexp{
regexp.MustCompile("^/gc/.*"),
regexp.MustCompile("^/sched/latencies:.*"),
},
expected: baseMetrics,
},
} {
t.Run(test.name, func(t *testing.T) {
reg := prometheus.NewRegistry()
reg.MustRegister(NewGoCollector(
WithGoCollectorMemStatsMetricsDisabled(),
WithoutGoCollectorRuntimeMetrics(test.matchers...),
))
result, err := reg.Gather()
if err != nil {
t.Fatal(err)
}
got := []string{}
for _, r := range result {
got = append(got, r.GetName())
}
if !reflect.DeepEqual(got, test.expected) {
t.Errorf("got %v, want %v", got, test.expected)
}
})
}
}
func ExampleGoCollector() {
reg := prometheus.NewRegistry()
// Register the GoCollector with the default options. Only the base metrics will be enabled.
reg.MustRegister(NewGoCollector())
http.Handle("/metrics", promhttp.HandlerFor(reg, promhttp.HandlerOpts{}))
log.Fatal(http.ListenAndServe(":8080", nil))
}
func ExampleGoCollector_WithAdvancedGoMetrics() {
reg := prometheus.NewRegistry()
// Enable Go metrics with pre-defined rules. Or your custom rules.
reg.MustRegister(
NewGoCollector(
WithGoCollectorMemStatsMetricsDisabled(),
WithGoCollectorRuntimeMetrics(
MetricsScheduler,
MetricsMemory,
GoRuntimeMetricsRule{
Matcher: regexp.MustCompile("^/mycustomrule.*"),
},
),
WithoutGoCollectorRuntimeMetrics(regexp.MustCompile("^/gc/.*")),
))
http.Handle("/metrics", promhttp.HandlerFor(reg, promhttp.HandlerOpts{}))
log.Fatal(http.ListenAndServe(":8080", nil))
}
func ExampleGoCollector_DefaultRegister() {
// Unregister the default GoCollector.
prometheus.Unregister(NewGoCollector())
// Register the default GoCollector with a custom config.
prometheus.MustRegister(NewGoCollector(WithGoCollectorRuntimeMetrics(
MetricsScheduler,
MetricsGC,
GoRuntimeMetricsRule{
Matcher: regexp.MustCompile("^/mycustomrule.*"),
},
),
))
http.Handle("/metrics", promhttp.Handler())
log.Fatal(http.ListenAndServe(":8080", nil))
}

7
prometheus/go_collector.go
View File

@ -19,6 +19,10 @@ import (
"time"
)
// goRuntimeMemStats provides the metrics initially provided by runtime.ReadMemStats.
// From Go 1.17 those similar (and better) statistics are provided by runtime/metrics, so
// while eval closure works on runtime.MemStats, the struct from Go 1.17+ is
// populated using runtime/metrics.
func goRuntimeMemStats() memStatsMetrics {
return memStatsMetrics{
{
@ -224,7 +228,7 @@ func newBaseGoCollector() baseGoCollector {
"A summary of the pause duration of garbage collection cycles.",
nil, nil),
gcLastTimeDesc: NewDesc(
memstatNamespace("last_gc_time_seconds"),
"go_memstats_last_gc_time_seconds",
"Number of seconds since 1970 of last garbage collection.",
nil, nil),
goInfoDesc: NewDesc(
@ -270,7 +274,6 @@ func memstatNamespace(s string) string {
// memStatsMetrics provide description, evaluator, runtime/metrics name, and
// value type for memstat metrics.
// TODO(bwplotka): Remove with end Go 1.16 EOL and replace with runtime/metrics.Description
type memStatsMetrics []struct {
desc *Desc
eval func(*runtime.MemStats) float64

198
prometheus/go_collector_latest.go
View File

@ -31,9 +31,11 @@ import (
)
const (
// constants for strings referenced more than once.
goGCHeapTinyAllocsObjects = "/gc/heap/tiny/allocs:objects"
goGCHeapAllocsObjects = "/gc/heap/allocs:objects"
goGCHeapFreesObjects = "/gc/heap/frees:objects"
goGCHeapFreesBytes = "/gc/heap/frees:bytes"
goGCHeapAllocsBytes = "/gc/heap/allocs:bytes"
goGCHeapObjects = "/gc/heap/objects:objects"
goGCHeapGoalBytes = "/gc/heap/goal:bytes"
@ -53,8 +55,8 @@ const (
goMemoryClassesOtherBytes = "/memory/classes/other:bytes"
)
// runtime/metrics names required for runtimeMemStats like logic.
var rmForMemStats = []string{
// rmNamesForMemStatsMetrics represents runtime/metrics names required to populate goRuntimeMemStats from like logic.
var rmNamesForMemStatsMetrics = []string{
goGCHeapTinyAllocsObjects,
goGCHeapAllocsObjects,
goGCHeapFreesObjects,
@ -90,74 +92,90 @@ func bestEffortLookupRM(lookup []string) []metrics.Description {
}
type goCollector struct {
opt GoCollectorOptions
base baseGoCollector
// mu protects updates to all fields ensuring a consistent
// snapshot is always produced by Collect.
mu sync.Mutex
// rm... fields all pertain to the runtime/metrics package.
rmSampleBuf []metrics.Sample
rmSampleMap map[string]*metrics.Sample
rmMetrics []collectorMetric
// Contains all samples that has to retrieved from runtime/metrics (not all of them will be exposed).
sampleBuf []metrics.Sample
// sampleMap allows lookup for MemStats metrics and runtime/metrics histograms for exact sums.
sampleMap map[string]*metrics.Sample
// rmExposedMetrics represents all runtime/metrics package metrics
// that were configured to be exposed.
rmExposedMetrics []collectorMetric
rmExactSumMapForHist map[string]string
// With Go 1.17, the runtime/metrics package was introduced.
// From that point on, metric names produced by the runtime/metrics
// package could be generated from runtime/metrics names. However,
// these differ from the old names for the same values.
//
// This field exist to export the same values under the old names
// This field exists to export the same values under the old names
// as well.
msMetrics memStatsMetrics
msMetricsEnabled bool
}
const (
// Those are not exposed due to need to move Go collector to another package in v2.
// See issue https://github.com/prometheus/client_golang/issues/1030.
goRuntimeMemStatsCollection uint32 = 1 << iota
goRuntimeMetricsCollection
)
// GoCollectorOptions should not be used be directly by anything, except `collectors` package.
// Use it via collectors package instead. See issue
// https://github.com/prometheus/client_golang/issues/1030.
//
// Deprecated: Use collectors.WithGoCollections
type GoCollectorOptions struct {
// EnabledCollection sets what type of collections collector should expose on top of base collection.
// By default it's goMemStatsCollection | goRuntimeMetricsCollection.
EnabledCollections uint32
type rmMetricDesc struct {
metrics.Description
}
func (c GoCollectorOptions) isEnabled(flag uint32) bool {
return c.EnabledCollections&flag != 0
func matchRuntimeMetricsRules(rules []internal.GoCollectorRule) []rmMetricDesc {
var descs []rmMetricDesc
for _, d := range metrics.All() {
var (
deny = true
desc rmMetricDesc
)
for _, r := range rules {
if !r.Matcher.MatchString(d.Name) {
continue
}
deny = r.Deny
}
if deny {
continue
}
desc.Description = d
descs = append(descs, desc)
}
return descs
}
const defaultGoCollections = goRuntimeMemStatsCollection
func defaultGoCollectorOptions() internal.GoCollectorOptions {
return internal.GoCollectorOptions{
RuntimeMetricSumForHist: map[string]string{
"/gc/heap/allocs-by-size:bytes": goGCHeapAllocsBytes,
"/gc/heap/frees-by-size:bytes": goGCHeapFreesBytes,
},
RuntimeMetricRules: []internal.GoCollectorRule{
//{Matcher: regexp.MustCompile("")},
},
}
}
// NewGoCollector is the obsolete version of collectors.NewGoCollector.
// See there for documentation.
//
// Deprecated: Use collectors.NewGoCollector instead.
func NewGoCollector(opts ...func(o *GoCollectorOptions)) Collector {
opt := GoCollectorOptions{EnabledCollections: defaultGoCollections}
func NewGoCollector(opts ...func(o *internal.GoCollectorOptions)) Collector {
opt := defaultGoCollectorOptions()
for _, o := range opts {
o(&opt)
}
var descriptions []metrics.Description
if opt.isEnabled(goRuntimeMetricsCollection) {
descriptions = metrics.All()
} else if opt.isEnabled(goRuntimeMemStatsCollection) {
descriptions = bestEffortLookupRM(rmForMemStats)
}
exposedDescriptions := matchRuntimeMetricsRules(opt.RuntimeMetricRules)
// Collect all histogram samples so that we can get their buckets.
// The API guarantees that the buckets are always fixed for the lifetime
// of the process.
var histograms []metrics.Sample
for _, d := range descriptions {
for _, d := range exposedDescriptions {
if d.Kind == metrics.KindFloat64Histogram {
histograms = append(histograms, metrics.Sample{Name: d.Name})
}
@ -172,13 +190,14 @@ func NewGoCollector(opts ...func(o *GoCollectorOptions)) Collector {
bucketsMap[histograms[i].Name] = histograms[i].Value.Float64Histogram().Buckets
}
// Generate a Desc and ValueType for each runtime/metrics metric.
metricSet := make([]collectorMetric, 0, len(descriptions))
sampleBuf := make([]metrics.Sample, 0, len(descriptions))
sampleMap := make(map[string]*metrics.Sample, len(descriptions))
for i := range descriptions {
d := &descriptions[i]
namespace, subsystem, name, ok := internal.RuntimeMetricsToProm(d)
// Generate a collector for each exposed runtime/metrics metric.
metricSet := make([]collectorMetric, 0, len(exposedDescriptions))
// SampleBuf is used for reading from runtime/metrics.
// We are assuming the largest case to have stable pointers for sampleMap purposes.
sampleBuf := make([]metrics.Sample, 0, len(exposedDescriptions)+len(opt.RuntimeMetricSumForHist)+len(rmNamesForMemStatsMetrics))
sampleMap := make(map[string]*metrics.Sample, len(exposedDescriptions))
for _, d := range exposedDescriptions {
namespace, subsystem, name, ok := internal.RuntimeMetricsToProm(&d.Description)
if !ok {
// Just ignore this metric; we can't do anything with it here.
// If a user decides to use the latest version of Go, we don't want
@ -186,19 +205,17 @@ func NewGoCollector(opts ...func(o *GoCollectorOptions)) Collector {
continue
}
// Set up sample buffer for reading, and a map
// for quick lookup of sample values.
sampleBuf = append(sampleBuf, metrics.Sample{Name: d.Name})
sampleMap[d.Name] = &sampleBuf[len(sampleBuf)-1]
var m collectorMetric
if d.Kind == metrics.KindFloat64Histogram {
_, hasSum := rmExactSumMap[d.Name]
_, hasSum := opt.RuntimeMetricSumForHist[d.Name]
unit := d.Name[strings.IndexRune(d.Name, ':')+1:]
m = newBatchHistogram(
NewDesc(
BuildFQName(namespace, subsystem, name),
d.Description,
d.Description.Description,
nil,
nil,
),
@ -210,30 +227,61 @@ func NewGoCollector(opts ...func(o *GoCollectorOptions)) Collector {
Namespace: namespace,
Subsystem: subsystem,
Name: name,
Help: d.Description,
})
Help: d.Description.Description,
},
)
} else {
m = NewGauge(GaugeOpts{
Namespace: namespace,
Subsystem: subsystem,
Name: name,
Help: d.Description,
Help: d.Description.Description,
})
}
metricSet = append(metricSet, m)
}
var msMetrics memStatsMetrics
if opt.isEnabled(goRuntimeMemStatsCollection) {
msMetrics = goRuntimeMemStats()
// Add exact sum metrics to sampleBuf if not added before.
for _, h := range histograms {
sumMetric, ok := opt.RuntimeMetricSumForHist[h.Name]
if !ok {
continue
}
if _, ok := sampleMap[sumMetric]; ok {
continue
}
sampleBuf = append(sampleBuf, metrics.Sample{Name: sumMetric})
sampleMap[sumMetric] = &sampleBuf[len(sampleBuf)-1]
}
var (
msMetrics memStatsMetrics
msDescriptions []metrics.Description
)
if !opt.DisableMemStatsLikeMetrics {
msMetrics = goRuntimeMemStats()
msDescriptions = bestEffortLookupRM(rmNamesForMemStatsMetrics)
// Check if metric was not exposed before and if not, add to sampleBuf.
for _, mdDesc := range msDescriptions {
if _, ok := sampleMap[mdDesc.Name]; ok {
continue
}
sampleBuf = append(sampleBuf, metrics.Sample{Name: mdDesc.Name})
sampleMap[mdDesc.Name] = &sampleBuf[len(sampleBuf)-1]
}
}
return &goCollector{
opt: opt,
base: newBaseGoCollector(),
rmSampleBuf: sampleBuf,
rmSampleMap: sampleMap,
rmMetrics: metricSet,
sampleBuf: sampleBuf,
sampleMap: sampleMap,
rmExposedMetrics: metricSet,
rmExactSumMapForHist: opt.RuntimeMetricSumForHist,
msMetrics: msMetrics,
msMetricsEnabled: !opt.DisableMemStatsLikeMetrics,
}
}
@ -243,7 +291,7 @@ func (c *goCollector) Describe(ch chan<- *Desc) {
for _, i := range c.msMetrics {
ch <- i.desc
}
for _, m := range c.rmMetrics {
for _, m := range c.rmExposedMetrics {
ch <- m.Desc()
}
}
@ -253,8 +301,12 @@ func (c *goCollector) Collect(ch chan<- Metric) {
// Collect base non-memory metrics.
c.base.Collect(ch)
if len(c.sampleBuf) == 0 {
return
}
// Collect must be thread-safe, so prevent concurrent use of
// rmSampleBuf. Just read into rmSampleBuf but write all the data
// sampleBuf elements. Just read into sampleBuf but write all the data
// we get into our Metrics or MemStats.
//
// This lock also ensures that the Metrics we send out are all from
@ -268,18 +320,18 @@ func (c *goCollector) Collect(ch chan<- Metric) {
c.mu.Lock()
defer c.mu.Unlock()
if len(c.rmSampleBuf) > 0 {
// Populate runtime/metrics sample buffer.
metrics.Read(c.rmSampleBuf)
}
metrics.Read(c.sampleBuf)
// Collect all our runtime/metrics user chose to expose from sampleBuf (if any).
for i, metric := range c.rmExposedMetrics {
// We created samples for exposed metrics first in order, so indexes match.
sample := c.sampleBuf[i]
if c.opt.isEnabled(goRuntimeMetricsCollection) {
// Collect all our metrics from rmSampleBuf.
for i, sample := range c.rmSampleBuf {
// N.B. switch on concrete type because it's significantly more efficient
// than checking for the Counter and Gauge interface implementations. In
// this case, we control all the types here.
switch m := c.rmMetrics[i].(type) {
switch m := metric.(type) {
case *counter:
// Guard against decreases. This should never happen, but a failure
// to do so will result in a panic, which is a harsh consequence for
@ -299,13 +351,12 @@ func (c *goCollector) Collect(ch chan<- Metric) {
panic("unexpected metric type")
}
}
}
if c.msMetricsEnabled {
// ms is a dummy MemStats that we populate ourselves so that we can
// populate the old metrics from it if goMemStatsCollection is enabled.
if c.opt.isEnabled(goRuntimeMemStatsCollection) {
var ms runtime.MemStats
memStatsFromRM(&ms, c.rmSampleMap)
memStatsFromRM(&ms, c.sampleMap)
for _, i := range c.msMetrics {
ch <- MustNewConstMetric(i.desc, i.valType, i.eval(&ms))
}
@ -336,11 +387,6 @@ func unwrapScalarRMValue(v metrics.Value) float64 {
}
}
var rmExactSumMap = map[string]string{
"/gc/heap/allocs-by-size:bytes": "/gc/heap/allocs:bytes",
"/gc/heap/frees-by-size:bytes": "/gc/heap/frees:bytes",
}
// exactSumFor takes a runtime/metrics metric name (that is assumed to
// be of kind KindFloat64Histogram) and returns its exact sum and whether
// its exact sum exists.
@ -348,11 +394,11 @@ var rmExactSumMap = map[string]string{
// The runtime/metrics API for histograms doesn't currently expose exact
// sums, but some of the other metrics are in fact exact sums of histograms.
func (c *goCollector) exactSumFor(rmName string) float64 {
sumName, ok := rmExactSumMap[rmName]
sumName, ok := c.rmExactSumMapForHist[rmName]
if !ok {
return 0
}
s, ok := c.rmSampleMap[sumName]
s, ok := c.sampleMap[sumName]
if !ok {
return 0
}

65
prometheus/go_collector_latest_test.go
View File

@ -19,6 +19,7 @@ package prometheus
import (
"math"
"reflect"
"regexp"
"runtime"
"runtime/metrics"
"sync"
@ -30,9 +31,18 @@ import (
)
func TestRmForMemStats(t *testing.T) {
if got, want := len(bestEffortLookupRM(rmForMemStats)), len(rmForMemStats); got != want {
descs := bestEffortLookupRM(rmNamesForMemStatsMetrics)
if got, want := len(descs), len(rmNamesForMemStatsMetrics); got != want {
t.Errorf("got %d, want %d metrics", got, want)
}
for _, d := range descs {
// We don't expect histograms there.
if d.Kind == metrics.KindFloat64Histogram {
t.Errorf("we don't expect to use histograms for MemStats metrics, got %v", d.Name)
}
}
}
func expectedBaseMetrics() map[string]struct{} {
@ -64,30 +74,43 @@ func addExpectedRuntimeMetrics(metrics map[string]struct{}) map[string]struct{}
return metrics
}
func TestGoCollector(t *testing.T) {
func TestGoCollector_ExposedMetrics(t *testing.T) {
for _, tcase := range []struct {
collections uint32
opts internal.GoCollectorOptions
expectedFQNameSet map[string]struct{}
}{
{
collections: 0,
opts: internal.GoCollectorOptions{
DisableMemStatsLikeMetrics: true,
},
expectedFQNameSet: expectedBaseMetrics(),
},
{
collections: goRuntimeMemStatsCollection,
// Default, only MemStats.
expectedFQNameSet: addExpectedRuntimeMemStats(expectedBaseMetrics()),
},
{
collections: goRuntimeMetricsCollection,
// Get all runtime/metrics without MemStats.
opts: internal.GoCollectorOptions{
DisableMemStatsLikeMetrics: true,
RuntimeMetricRules: []internal.GoCollectorRule{
{Matcher: regexp.MustCompile("/.*")},
},
},
expectedFQNameSet: addExpectedRuntimeMetrics(expectedBaseMetrics()),
},
{
collections: goRuntimeMemStatsCollection | goRuntimeMetricsCollection,
// Get all runtime/metrics and MemStats.
opts: internal.GoCollectorOptions{
RuntimeMetricRules: []internal.GoCollectorRule{
{Matcher: regexp.MustCompile("/.*")},
},
},
expectedFQNameSet: addExpectedRuntimeMemStats(addExpectedRuntimeMetrics(expectedBaseMetrics())),
},
} {
if ok := t.Run("", func(t *testing.T) {
goMetrics := collectGoMetrics(t, tcase.collections)
goMetrics := collectGoMetrics(t, tcase.opts)
goMetricSet := make(map[string]Metric)
for _, m := range goMetrics {
goMetricSet[m.Desc().fqName] = m
@ -118,7 +141,11 @@ func TestGoCollector(t *testing.T) {
var sink interface{}
func TestBatchHistogram(t *testing.T) {
goMetrics := collectGoMetrics(t, goRuntimeMetricsCollection)
goMetrics := collectGoMetrics(t, internal.GoCollectorOptions{
RuntimeMetricRules: []internal.GoCollectorRule{
{Matcher: regexp.MustCompile("/.*")},
},
})
var mhist Metric
for _, m := range goMetrics {
@ -145,7 +172,8 @@ func TestBatchHistogram(t *testing.T) {
for i := 0; i < 100; i++ {
sink = make([]byte, 128)
}
collectGoMetrics(t, defaultGoCollections)
collectGoMetrics(t, defaultGoCollectorOptions())
for i, v := range hist.counts {
if v != countsCopy[i] {
t.Error("counts changed during new collection")
@ -194,11 +222,13 @@ func TestBatchHistogram(t *testing.T) {
}
}
func collectGoMetrics(t *testing.T, enabledCollections uint32) []Metric {
func collectGoMetrics(t *testing.T, opts internal.GoCollectorOptions) []Metric {
t.Helper()
c := NewGoCollector(func(o *GoCollectorOptions) {
o.EnabledCollections = enabledCollections
c := NewGoCollector(func(o *internal.GoCollectorOptions) {
o.DisableMemStatsLikeMetrics = opts.DisableMemStatsLikeMetrics
o.RuntimeMetricSumForHist = opts.RuntimeMetricSumForHist
o.RuntimeMetricRules = opts.RuntimeMetricRules
}).(*goCollector)
// Collect all metrics.
@ -222,7 +252,7 @@ func collectGoMetrics(t *testing.T, enabledCollections uint32) []Metric {
func TestMemStatsEquivalence(t *testing.T) {
var msReal, msFake runtime.MemStats
descs := bestEffortLookupRM(rmForMemStats)
descs := bestEffortLookupRM(rmNamesForMemStatsMetrics)
samples := make([]metrics.Sample, len(descs))
samplesMap := make(map[string]*metrics.Sample)
@ -269,7 +299,12 @@ func TestMemStatsEquivalence(t *testing.T) {
}
func TestExpectedRuntimeMetrics(t *testing.T) {
goMetrics := collectGoMetrics(t, goRuntimeMemStatsCollection|goRuntimeMetricsCollection)
goMetrics := collectGoMetrics(t, internal.GoCollectorOptions{
DisableMemStatsLikeMetrics: true,
RuntimeMetricRules: []internal.GoCollectorRule{
{Matcher: regexp.MustCompile("/.*")},
},
})
goMetricSet := make(map[string]Metric)
for _, m := range goMetrics {
goMetricSet[m.Desc().fqName] = m

32
prometheus/internal/go_collector_options.go Normal file
View File

@ -0,0 +1,32 @@
// Copyright 2021 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package internal
import "regexp"
type GoCollectorRule struct {
Matcher *regexp.Regexp
Deny bool
}
// GoCollectorOptions should not be used be directly by anything, except `collectors` package.
// Use it via collectors package instead. See issue
// https://github.com/prometheus/client_golang/issues/1030.
//
// This is internal, so external users only can use it via `collector.WithGoCollector*` methods
type GoCollectorOptions struct {
DisableMemStatsLikeMetrics bool
RuntimeMetricSumForHist map[string]string
RuntimeMetricRules []GoCollectorRule
}