Add default Go runtime metrics for /gc/gogc:percent, /gc/gomemlimit:bytes, /sched/gomaxprocs:threads (#1559)

* Add go_gomaxprocs, go_gogc_percent and go_gomemlimit to the default Go runtime metrics

Signed-off-by: Arianna Vespri <arianna.vespri@yahoo.it>

* Move newly added metrics out of base metrics and into goCollector

Signed-off-by: Arianna Vespri <arianna.vespri@yahoo.it>

* Rethink struct for newly added metrics, adapt and add tests

Signed-off-by: Arianna Vespri <arianna.vespri@yahoo.it>

* Simplify new metrics reading

Signed-off-by: Arianna Vespri <arianna.vespri@yahoo.it>

* Correct loop, add debugging lines

Signed-off-by: Arianna Vespri <arianna.vespri@yahoo.it>

* Make goRuntimeEnvVarsMetrics function Go version dependent

Signed-off-by: Arianna Vespri <arianna.vespri@yahoo.it>

* Fix go mod

Signed-off-by: Arianna Vespri <arianna.vespri@yahoo.it>

* Remove debuggin line

Signed-off-by: Arianna Vespri <arianna.vespri@yahoo.it>

* Move default runtime metrics into the runtime metrics flow, change tests accordingly

Signed-off-by: Arianna Vespri <arianna.vespri@yahoo.it>

* Go version expected default runtime metrics map for tests

Signed-off-by: Arianna Vespri <arianna.vespri@yahoo.it>

* 1.21 update.

Signed-off-by: bwplotka <bwplotka@gmail.com>

* Addressed comments on Arianna's PR.

Signed-off-by: bwplotka <bwplotka@gmail.com>

* Use default GoCollector func in test

Signed-off-by: Arianna Vespri <arianna.vespri@yahoo.it>

---------

Signed-off-by: Arianna Vespri <arianna.vespri@yahoo.it>
Signed-off-by: bwplotka <bwplotka@gmail.com>
Co-authored-by: bwplotka <bwplotka@gmail.com>
This commit is contained in:
Arianna Vespri 2024-08-09 21:27:25 +02:00 committed by GitHub
parent 071572721c
commit 3ad272204b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
13 changed files with 389 additions and 227 deletions

View File

@ -102,3 +102,9 @@ func withSchedulerMetrics() []string {
func withDebugMetrics() []string { func withDebugMetrics() []string {
return withBaseMetrics([]string{}) return withBaseMetrics([]string{})
} }
var defaultRuntimeMetrics = []string{}
func withDefaultRuntimeMetrics(metricNames []string, withoutGC, withoutSched bool) []string {
return metricNames
}

View File

@ -16,6 +16,8 @@
package collectors package collectors
import "sort"
func withAllMetrics() []string { func withAllMetrics() []string {
return withBaseMetrics([]string{ return withBaseMetrics([]string{
"go_cgo_go_to_c_calls_calls_total", "go_cgo_go_to_c_calls_calls_total",
@ -109,3 +111,18 @@ func withSchedulerMetrics() []string {
func withDebugMetrics() []string { func withDebugMetrics() []string {
return withBaseMetrics([]string{}) return withBaseMetrics([]string{})
} }
var defaultRuntimeMetrics = []string{
"go_sched_gomaxprocs_threads",
}
func withDefaultRuntimeMetrics(metricNames []string, withoutGC, withoutSched bool) []string {
// If withoutSched is true, exclude "go_sched_gomaxprocs_threads".
if withoutSched {
return metricNames
}
metricNames = append(metricNames, defaultRuntimeMetrics...)
// sorting is required
sort.Strings(metricNames)
return metricNames
}

View File

@ -16,6 +16,8 @@
package collectors package collectors
import "sort"
func withAllMetrics() []string { func withAllMetrics() []string {
return withBaseMetrics([]string{ return withBaseMetrics([]string{
"go_cgo_go_to_c_calls_calls_total", "go_cgo_go_to_c_calls_calls_total",
@ -116,3 +118,18 @@ func withSchedulerMetrics() []string {
func withDebugMetrics() []string { func withDebugMetrics() []string {
return withBaseMetrics([]string{}) return withBaseMetrics([]string{})
} }
var defaultRuntimeMetrics = []string{
"go_sched_gomaxprocs_threads",
}
func withDefaultRuntimeMetrics(metricNames []string, withoutGC, withoutSched bool) []string {
// If withoutSched is true, exclude "go_sched_gomaxprocs_threads".
if withoutSched {
return metricNames
}
metricNames = append(metricNames, defaultRuntimeMetrics...)
// sorting is required
sort.Strings(metricNames)
return metricNames
}

View File

@ -16,6 +16,8 @@
package collectors package collectors
import "sort"
func withAllMetrics() []string { func withAllMetrics() []string {
return withBaseMetrics([]string{ return withBaseMetrics([]string{
"go_cgo_go_to_c_calls_calls_total", "go_cgo_go_to_c_calls_calls_total",
@ -169,3 +171,28 @@ func withDebugMetrics() []string {
"go_godebug_non_default_behavior_zipinsecurepath_events_total", "go_godebug_non_default_behavior_zipinsecurepath_events_total",
}) })
} }
var defaultRuntimeMetrics = []string{
"go_gc_gogc_percent",
"go_gc_gomemlimit_bytes",
"go_sched_gomaxprocs_threads",
}
func withDefaultRuntimeMetrics(metricNames []string, withoutGC, withoutSched bool) []string {
if withoutGC && withoutSched {
// If both flags are true, return the metricNames as is.
return metricNames
} else if withoutGC && !withoutSched {
// If only withoutGC is true, include "go_sched_gomaxprocs_threads" only.
metricNames = append(metricNames, []string{"go_sched_gomaxprocs_threads"}...)
} else if withoutSched && !withoutGC {
// If only withoutSched is true, exclude "go_sched_gomaxprocs_threads".
metricNames = append(metricNames, []string{"go_gc_gogc_percent", "go_gc_gomemlimit_bytes"}...)
} else {
// If neither flag is true, use the default metrics.
metricNames = append(metricNames, defaultRuntimeMetrics...)
}
// sorting is required
sort.Strings(metricNames)
return metricNames
}

View File

@ -16,6 +16,8 @@
package collectors package collectors
import "sort"
func withAllMetrics() []string { func withAllMetrics() []string {
return withBaseMetrics([]string{ return withBaseMetrics([]string{
"go_cgo_go_to_c_calls_calls_total", "go_cgo_go_to_c_calls_calls_total",
@ -191,3 +193,28 @@ func withDebugMetrics() []string {
"go_godebug_non_default_behavior_zipinsecurepath_events_total", "go_godebug_non_default_behavior_zipinsecurepath_events_total",
}) })
} }
var defaultRuntimeMetrics = []string{
"go_gc_gogc_percent",
"go_gc_gomemlimit_bytes",
"go_sched_gomaxprocs_threads",
}
func withDefaultRuntimeMetrics(metricNames []string, withoutGC, withoutSched bool) []string {
if withoutGC && withoutSched {
// If both flags are true, return the metricNames as is.
return metricNames
} else if withoutGC && !withoutSched {
// If only withoutGC is true, include "go_sched_gomaxprocs_threads" only.
metricNames = append(metricNames, []string{"go_sched_gomaxprocs_threads"}...)
} else if withoutSched && !withoutGC {
// If only withoutSched is true, exclude "go_sched_gomaxprocs_threads".
metricNames = append(metricNames, []string{"go_gc_gogc_percent", "go_gc_gomemlimit_bytes"}...)
} else {
// If neither flag is true, use the default metrics.
metricNames = append(metricNames, defaultRuntimeMetrics...)
}
// sorting is required
sort.Strings(metricNames)
return metricNames
}

View File

@ -93,7 +93,9 @@ func TestWithGoCollectorDefault(t *testing.T) {
got = append(got, r.GetName()) got = append(got, r.GetName())
} }
if diff := cmp.Diff(got, withBaseMetrics(memstatMetrics)); diff != "" { expected := append(withBaseMetrics(memstatMetrics), defaultRuntimeMetrics...)
sort.Strings(expected)
if diff := cmp.Diff(got, expected); diff != "" {
t.Errorf("[IMPORTANT, those are default metrics, can't change in 1.x] missmatch (-want +got):\n%s", diff) t.Errorf("[IMPORTANT, those are default metrics, can't change in 1.x] missmatch (-want +got):\n%s", diff)
} }
} }
@ -113,7 +115,7 @@ func TestWithGoCollectorMemStatsMetricsDisabled(t *testing.T) {
got = append(got, r.GetName()) got = append(got, r.GetName())
} }
if diff := cmp.Diff(got, baseMetrics); diff != "" { if diff := cmp.Diff(got, withBaseMetrics(defaultRuntimeMetrics)); diff != "" {
t.Errorf("missmatch (-want +got):\n%s", diff) t.Errorf("missmatch (-want +got):\n%s", diff)
} }
} }
@ -127,7 +129,7 @@ func TestGoCollectorAllowList(t *testing.T) {
{ {
name: "Without any rules", name: "Without any rules",
rules: nil, rules: nil,
expected: baseMetrics, expected: withBaseMetrics(defaultRuntimeMetrics),
}, },
{ {
name: "allow all", name: "allow all",
@ -137,22 +139,22 @@ func TestGoCollectorAllowList(t *testing.T) {
{ {
name: "allow GC", name: "allow GC",
rules: []GoRuntimeMetricsRule{MetricsGC}, rules: []GoRuntimeMetricsRule{MetricsGC},
expected: withGCMetrics(), expected: withDefaultRuntimeMetrics(withGCMetrics(), true, false),
}, },
{ {
name: "allow Memory", name: "allow Memory",
rules: []GoRuntimeMetricsRule{MetricsMemory}, rules: []GoRuntimeMetricsRule{MetricsMemory},
expected: withMemoryMetrics(), expected: withDefaultRuntimeMetrics(withMemoryMetrics(), false, false),
}, },
{ {
name: "allow Scheduler", name: "allow Scheduler",
rules: []GoRuntimeMetricsRule{MetricsScheduler}, rules: []GoRuntimeMetricsRule{MetricsScheduler},
expected: withSchedulerMetrics(), expected: withDefaultRuntimeMetrics(withSchedulerMetrics(), false, true),
}, },
{ {
name: "allow debug", name: "allow debug",
rules: []GoRuntimeMetricsRule{MetricsDebug}, rules: []GoRuntimeMetricsRule{MetricsDebug},
expected: withDebugMetrics(), expected: withDefaultRuntimeMetrics(withDebugMetrics(), false, false),
}, },
} { } {
t.Run(test.name, func(t *testing.T) { t.Run(test.name, func(t *testing.T) {
@ -193,7 +195,7 @@ func TestGoCollectorDenyList(t *testing.T) {
{ {
name: "Without any matchers", name: "Without any matchers",
matchers: nil, matchers: nil,
expected: baseMetrics, expected: withBaseMetrics(defaultRuntimeMetrics),
}, },
{ {
name: "deny all", name: "deny all",
@ -206,6 +208,14 @@ func TestGoCollectorDenyList(t *testing.T) {
regexp.MustCompile("^/gc/.*"), regexp.MustCompile("^/gc/.*"),
regexp.MustCompile("^/sched/latencies:.*"), regexp.MustCompile("^/sched/latencies:.*"),
}, },
expected: withDefaultRuntimeMetrics(baseMetrics, true, false),
},
{
name: "deny gc and scheduler",
matchers: []*regexp.Regexp{
regexp.MustCompile("^/gc/.*"),
regexp.MustCompile("^/sched/.*"),
},
expected: baseMetrics, expected: baseMetrics,
}, },
} { } {
@ -235,7 +245,7 @@ func TestGoCollectorDenyList(t *testing.T) {
func ExampleGoCollector() { func ExampleGoCollector() {
reg := prometheus.NewRegistry() reg := prometheus.NewRegistry()
// Register the GoCollector with the default options. Only the base metrics and memstats are enabled. // Register the GoCollector with the default options. Only the base metrics, default runtime metrics and memstats are enabled.
reg.MustRegister(NewGoCollector()) reg.MustRegister(NewGoCollector())
http.Handle("/metrics", promhttp.HandlerFor(reg, promhttp.HandlerOpts{})) http.Handle("/metrics", promhttp.HandlerFor(reg, promhttp.HandlerOpts{}))

View File

@ -63,14 +63,27 @@ func main() {
v := goVersion(gv.Segments()[1]) v := goVersion(gv.Segments()[1])
log.Printf("generating metrics for Go version %q", v) log.Printf("generating metrics for Go version %q", v)
allDesc := metrics.All()
// Find default metrics.
var defaultDesc []metrics.Description
for _, d := range allDesc {
if !internal.GoCollectorDefaultRuntimeMetrics.MatchString(d.Name) {
continue
}
defaultDesc = append(defaultDesc, d)
}
// Generate code. // Generate code.
var buf bytes.Buffer var buf bytes.Buffer
err = testFile.Execute(&buf, struct { err = testFile.Execute(&buf, struct {
Descriptions []metrics.Description AllDescriptions []metrics.Description
DefaultDescriptions []metrics.Description
GoVersion goVersion GoVersion goVersion
Cardinality int Cardinality int
}{ }{
Descriptions: metrics.All(), AllDescriptions: allDesc,
DefaultDescriptions: defaultDesc,
GoVersion: v, GoVersion: v,
Cardinality: rmCardinality(), Cardinality: rmCardinality(),
}) })
@ -167,14 +180,25 @@ var testFile = template.Must(template.New("testFile").Funcs(map[string]interface
package prometheus package prometheus
var expectedRuntimeMetrics = map[string]string{ var (
{{- range .Descriptions -}} expectedRuntimeMetrics = map[string]string{
{{- range .AllDescriptions -}}
{{- $trans := rm2prom . -}} {{- $trans := rm2prom . -}}
{{- if ne $trans "" }} {{- if ne $trans "" }}
{{.Name | printf "%q"}}: {{$trans | printf "%q"}}, {{.Name | printf "%q"}}: {{$trans | printf "%q"}},
{{- end -}} {{- end -}}
{{end}} {{end}}
} }
expMetrics = map[string]string{
{{- range .DefaultDescriptions -}}
{{- $trans := rm2prom . -}}
{{- if ne $trans "" }}
{{.Name | printf "%q"}}: {{$trans | printf "%q"}},
{{- end -}}
{{end}}
}
)
const expectedRuntimeMetricsCardinality = {{.Cardinality}} const expectedRuntimeMetricsCardinality = {{.Cardinality}}
`)) `))

View File

@ -17,6 +17,7 @@
package prometheus package prometheus
import ( import (
"fmt"
"math" "math"
"runtime" "runtime"
"runtime/metrics" "runtime/metrics"
@ -153,7 +154,8 @@ func defaultGoCollectorOptions() internal.GoCollectorOptions {
"/gc/heap/frees-by-size:bytes": goGCHeapFreesBytes, "/gc/heap/frees-by-size:bytes": goGCHeapFreesBytes,
}, },
RuntimeMetricRules: []internal.GoCollectorRule{ RuntimeMetricRules: []internal.GoCollectorRule{
//{Matcher: regexp.MustCompile("")}, // Recommended metrics we want by default from runtime/metrics.
{Matcher: internal.GoCollectorDefaultRuntimeMetrics},
}, },
} }
} }
@ -376,13 +378,13 @@ func unwrapScalarRMValue(v metrics.Value) float64 {
// //
// This should never happen because we always populate our metric // This should never happen because we always populate our metric
// set from the runtime/metrics package. // set from the runtime/metrics package.
panic("unexpected unsupported metric") panic("unexpected bad kind metric")
default: default:
// Unsupported metric kind. // Unsupported metric kind.
// //
// This should never happen because we check for this during initialization // This should never happen because we check for this during initialization
// and flag and filter metrics whose kinds we don't understand. // and flag and filter metrics whose kinds we don't understand.
panic("unexpected unsupported metric kind") panic(fmt.Sprintf("unexpected unsupported metric: %v", v.Kind()))
} }
} }

View File

@ -74,6 +74,13 @@ func addExpectedRuntimeMetrics(metrics map[string]struct{}) map[string]struct{}
return metrics return metrics
} }
func addExpectedDefaultRuntimeMetrics(metrics map[string]struct{}) map[string]struct{} {
for _, e := range expMetrics {
metrics[e] = struct{}{}
}
return metrics
}
func TestGoCollector_ExposedMetrics(t *testing.T) { func TestGoCollector_ExposedMetrics(t *testing.T) {
for _, tcase := range []struct { for _, tcase := range []struct {
opts internal.GoCollectorOptions opts internal.GoCollectorOptions
@ -86,8 +93,9 @@ func TestGoCollector_ExposedMetrics(t *testing.T) {
expectedFQNameSet: expectedBaseMetrics(), expectedFQNameSet: expectedBaseMetrics(),
}, },
{ {
// Default, only MemStats. // Default, only MemStats and default Runtime metrics.
expectedFQNameSet: addExpectedRuntimeMemStats(expectedBaseMetrics()), opts: defaultGoCollectorOptions(),
expectedFQNameSet: addExpectedDefaultRuntimeMetrics(addExpectedRuntimeMemStats(expectedBaseMetrics())),
}, },
{ {
// Get all runtime/metrics without MemStats. // Get all runtime/metrics without MemStats.

View File

@ -6,7 +6,8 @@
package prometheus package prometheus
var expectedRuntimeMetrics = map[string]string{ var (
expectedRuntimeMetrics = map[string]string{
"/cgo/go-to-c-calls:calls": "go_cgo_go_to_c_calls_calls_total", "/cgo/go-to-c-calls:calls": "go_cgo_go_to_c_calls_calls_total",
"/cpu/classes/gc/mark/assist:cpu-seconds": "go_cpu_classes_gc_mark_assist_cpu_seconds_total", "/cpu/classes/gc/mark/assist:cpu-seconds": "go_cpu_classes_gc_mark_assist_cpu_seconds_total",
"/cpu/classes/gc/mark/dedicated:cpu-seconds": "go_cpu_classes_gc_mark_dedicated_cpu_seconds_total", "/cpu/classes/gc/mark/dedicated:cpu-seconds": "go_cpu_classes_gc_mark_dedicated_cpu_seconds_total",
@ -52,6 +53,11 @@ var expectedRuntimeMetrics = map[string]string{
"/sched/goroutines:goroutines": "go_sched_goroutines_goroutines", "/sched/goroutines:goroutines": "go_sched_goroutines_goroutines",
"/sched/latencies:seconds": "go_sched_latencies_seconds", "/sched/latencies:seconds": "go_sched_latencies_seconds",
"/sync/mutex/wait/total:seconds": "go_sync_mutex_wait_total_seconds_total", "/sync/mutex/wait/total:seconds": "go_sync_mutex_wait_total_seconds_total",
} }
expMetrics = map[string]string{
"/sched/gomaxprocs:threads": "go_sched_gomaxprocs_threads",
}
)
const expectedRuntimeMetricsCardinality = 89 const expectedRuntimeMetricsCardinality = 89

View File

@ -6,7 +6,8 @@
package prometheus package prometheus
var expectedRuntimeMetrics = map[string]string{ var (
expectedRuntimeMetrics = map[string]string{
"/cgo/go-to-c-calls:calls": "go_cgo_go_to_c_calls_calls_total", "/cgo/go-to-c-calls:calls": "go_cgo_go_to_c_calls_calls_total",
"/cpu/classes/gc/mark/assist:cpu-seconds": "go_cpu_classes_gc_mark_assist_cpu_seconds_total", "/cpu/classes/gc/mark/assist:cpu-seconds": "go_cpu_classes_gc_mark_assist_cpu_seconds_total",
"/cpu/classes/gc/mark/dedicated:cpu-seconds": "go_cpu_classes_gc_mark_dedicated_cpu_seconds_total", "/cpu/classes/gc/mark/dedicated:cpu-seconds": "go_cpu_classes_gc_mark_dedicated_cpu_seconds_total",
@ -78,6 +79,13 @@ var expectedRuntimeMetrics = map[string]string{
"/sched/goroutines:goroutines": "go_sched_goroutines_goroutines", "/sched/goroutines:goroutines": "go_sched_goroutines_goroutines",
"/sched/latencies:seconds": "go_sched_latencies_seconds", "/sched/latencies:seconds": "go_sched_latencies_seconds",
"/sync/mutex/wait/total:seconds": "go_sync_mutex_wait_total_seconds_total", "/sync/mutex/wait/total:seconds": "go_sync_mutex_wait_total_seconds_total",
} }
expMetrics = map[string]string{
"/gc/gogc:percent": "go_gc_gogc_percent",
"/gc/gomemlimit:bytes": "go_gc_gomemlimit_bytes",
"/sched/gomaxprocs:threads": "go_sched_gomaxprocs_threads",
}
)
const expectedRuntimeMetricsCardinality = 115 const expectedRuntimeMetricsCardinality = 115

View File

@ -6,7 +6,8 @@
package prometheus package prometheus
var expectedRuntimeMetrics = map[string]string{ var (
expectedRuntimeMetrics = map[string]string{
"/cgo/go-to-c-calls:calls": "go_cgo_go_to_c_calls_calls_total", "/cgo/go-to-c-calls:calls": "go_cgo_go_to_c_calls_calls_total",
"/cpu/classes/gc/mark/assist:cpu-seconds": "go_cpu_classes_gc_mark_assist_cpu_seconds_total", "/cpu/classes/gc/mark/assist:cpu-seconds": "go_cpu_classes_gc_mark_assist_cpu_seconds_total",
"/cpu/classes/gc/mark/dedicated:cpu-seconds": "go_cpu_classes_gc_mark_dedicated_cpu_seconds_total", "/cpu/classes/gc/mark/dedicated:cpu-seconds": "go_cpu_classes_gc_mark_dedicated_cpu_seconds_total",
@ -89,6 +90,13 @@ var expectedRuntimeMetrics = map[string]string{
"/sched/pauses/total/gc:seconds": "go_sched_pauses_total_gc_seconds", "/sched/pauses/total/gc:seconds": "go_sched_pauses_total_gc_seconds",
"/sched/pauses/total/other:seconds": "go_sched_pauses_total_other_seconds", "/sched/pauses/total/other:seconds": "go_sched_pauses_total_other_seconds",
"/sync/mutex/wait/total:seconds": "go_sync_mutex_wait_total_seconds_total", "/sync/mutex/wait/total:seconds": "go_sync_mutex_wait_total_seconds_total",
} }
expMetrics = map[string]string{
"/gc/gogc:percent": "go_gc_gogc_percent",
"/gc/gomemlimit:bytes": "go_gc_gomemlimit_bytes",
"/sched/gomaxprocs:threads": "go_sched_gomaxprocs_threads",
}
)
const expectedRuntimeMetricsCardinality = 162 const expectedRuntimeMetricsCardinality = 162

View File

@ -30,3 +30,5 @@ type GoCollectorOptions struct {
RuntimeMetricSumForHist map[string]string RuntimeMetricSumForHist map[string]string
RuntimeMetricRules []GoCollectorRule RuntimeMetricRules []GoCollectorRule
} }
var GoCollectorDefaultRuntimeMetrics = regexp.MustCompile(`/gc/gogc:percent|/gc/gomemlimit:bytes|/sched/gomaxprocs:threads`)