From 50ab457fb7e73866c181943e1aaf88ee0ccacc61 Mon Sep 17 00:00:00 2001 From: Sachin Sahu <75629410+SachinSahu431@users.noreply.github.com> Date: Tue, 26 Mar 2024 18:12:34 +0530 Subject: [PATCH] Auto-update Go Collector Metrics for new Go versions (#1476) * Autogenerate go_collector__test.go files Signed-off-by: Sachin Sahu <75629410+SachinSahu431@users.noreply.github.com> * Add latest Go version Signed-off-by: Sachin Sahu <75629410+SachinSahu431@users.noreply.github.com> * nit: Script to check new Go version Signed-off-by: Sachin Sahu <75629410+SachinSahu431@users.noreply.github.com> * Rename file, fix linting issue Signed-off-by: Sachin Sahu <75629410+SachinSahu431@users.noreply.github.com> --------- Signed-off-by: Sachin Sahu <75629410+SachinSahu431@users.noreply.github.com> --- Makefile | 4 +- generate-go-collector.bash | 2 + prometheus/collectors/gen_go_collector_set.go | 214 ++++++++++++++++++ supported_go_versions.txt | 3 + 4 files changed, 222 insertions(+), 1 deletion(-) create mode 100644 prometheus/collectors/gen_go_collector_set.go create mode 100644 supported_go_versions.txt diff --git a/Makefile b/Makefile index 1c5eb70..6fa529a 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,9 @@ test: deps common-test test-short: deps common-test-short .PHONY: generate-go-collector-test-files -VERSIONS := 1.20 1.21 1.22 +file := supported_go_versions.txt +# take top 3 versions +VERSIONS := $(shell cat ${file} | head -n 3) generate-go-collector-test-files: for GO_VERSION in $(VERSIONS); do \ docker run \ diff --git a/generate-go-collector.bash b/generate-go-collector.bash index 3fa822a..a9e2fa5 100644 --- a/generate-go-collector.bash +++ b/generate-go-collector.bash @@ -5,3 +5,5 @@ set -e go get github.com/hashicorp/go-version@v1.6.0 go run prometheus/gen_go_collector_metrics_set.go mv -f go_collector_metrics_* prometheus +go run prometheus/collectors/gen_go_collector_set.go +mv -f go_collector_* prometheus/collectors diff --git a/prometheus/collectors/gen_go_collector_set.go b/prometheus/collectors/gen_go_collector_set.go new file mode 100644 index 0000000..61e09bb --- /dev/null +++ b/prometheus/collectors/gen_go_collector_set.go @@ -0,0 +1,214 @@ +// Copyright 2021 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build ignore +// +build ignore + +package main + +import ( + "bytes" + "fmt" + "go/format" + "log" + "os" + "regexp" + "runtime" + "runtime/metrics" + "sort" + "strings" + "text/template" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/internal" + + version "github.com/hashicorp/go-version" +) + +type metricGroup struct { + Name string + Regex *regexp.Regexp + Metrics []string +} + +var metricGroups = []metricGroup{ + {"withAllMetrics", nil, nil}, + {"withGCMetrics", regexp.MustCompile("^go_gc_.*"), nil}, + {"withMemoryMetrics", regexp.MustCompile("^go_memory_classes_.*"), nil}, + {"withSchedulerMetrics", regexp.MustCompile("^go_sched_.*"), nil}, + {"withDebugMetrics", regexp.MustCompile("^go_godebug_non_default_behavior_.*"), nil}, +} + +func main() { + var givenVersion string + toolVersion := runtime.Version() + if len(os.Args) != 2 { + log.Printf("requires Go version (e.g. go1.17) as an argument. Since it is not specified, assuming %s.", toolVersion) + givenVersion = toolVersion + } else { + givenVersion = os.Args[1] + } + log.Printf("given version for Go: %s", givenVersion) + log.Printf("tool version for Go: %s", toolVersion) + + tv, err := version.NewVersion(strings.TrimPrefix(givenVersion, "go")) + if err != nil { + log.Fatal(err) + } + + toolVersion = strings.Split(strings.TrimPrefix(toolVersion, "go"), " ")[0] + gv, err := version.NewVersion(toolVersion) + if err != nil { + log.Fatal(err) + } + if !gv.Equal(tv) { + log.Fatalf("using Go version %q but expected Go version %q", tv, gv) + } + + v := goVersion(gv.Segments()[1]) + log.Printf("generating metrics for Go version %q", v) + + descriptions := computeMetricsList() + groupedMetrics := groupMetrics(descriptions) + + // Generate code. + var buf bytes.Buffer + err = testFile.Execute(&buf, struct { + GoVersion goVersion + Groups []metricGroup + }{ + GoVersion: v, + Groups: groupedMetrics, + }) + if err != nil { + log.Fatalf("executing template: %v", err) + } + + // Format it. + result, err := format.Source(buf.Bytes()) + if err != nil { + log.Fatalf("formatting code: %v", err) + } + + // Write it to a file. + fname := fmt.Sprintf("go_collector_%s_test.go", v.Abbr()) + if err := os.WriteFile(fname, result, 0o644); err != nil { + log.Fatalf("writing file: %v", err) + } +} + +func computeMetricsList() []string { + var metricsList []string + for _, d := range metrics.All() { + if trans := rm2prom(d); trans != "" { + metricsList = append(metricsList, trans) + } + } + return metricsList +} + +func rm2prom(d metrics.Description) string { + ns, ss, n, ok := internal.RuntimeMetricsToProm(&d) + if !ok { + return "" + } + return prometheus.BuildFQName(ns, ss, n) +} + +func groupMetrics(metricsList []string) []metricGroup { + var groupedMetrics []metricGroup + for _, group := range metricGroups { + var matchedMetrics []string + for _, metric := range metricsList { + if group.Regex == nil || group.Regex.MatchString(metric) { + matchedMetrics = append(matchedMetrics, metric) + } + } + + // Scheduler metrics is `sched` regex plus base metrics + // List of base metrics are taken from here: https://github.com/prometheus/client_golang/blob/26e3055e5133a9d64e8e5a07a7cf026875d5f55d/prometheus/go_collector.go#L208 + if group.Name == "withSchedulerMetrics" { + baseMatrices := []string{ + "go_gc_duration_seconds", + "go_goroutines", + "go_info", + "go_memstats_last_gc_time_seconds", + "go_threads", + } + matchedMetrics = append(matchedMetrics, baseMatrices...) + } + sort.Strings(matchedMetrics) + if len(matchedMetrics) > 0 { + groupedMetrics = append(groupedMetrics, metricGroup{ + Name: group.Name, + Regex: group.Regex, + Metrics: matchedMetrics, + }) + } + } + return groupedMetrics +} + +type goVersion int + +func (g goVersion) String() string { + return fmt.Sprintf("go1.%d", g) +} + +func (g goVersion) Abbr() string { + return fmt.Sprintf("go1%d", g) +} + +var testFile = template.Must(template.New("testFile").Funcs(map[string]interface{}{ + "nextVersion": func(version goVersion) string { + return (version + goVersion(1)).String() + }, + "needsBaseMetrics": func(groupName string) bool { + return groupName == "withAllMetrics" || groupName == "withGCMetrics" || groupName == "withMemoryMetrics" + }, +}).Parse(`// Copyright 2022 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build {{.GoVersion}} && !{{nextVersion .GoVersion}} +// +build {{.GoVersion}},!{{nextVersion .GoVersion}} + +package collectors + +{{- range .Groups }} +func {{ .Name }}() []string { + {{- if needsBaseMetrics .Name }} + return withBaseMetrics([]string{ + {{- range $metric := .Metrics }} + {{ $metric | printf "%q" }}, + {{- end }} + }) + {{- else }} + return []string{ + {{- range $metric := .Metrics }} + {{ $metric | printf "%q" }}, + {{- end }} + } + {{- end }} +} +{{ end }} +`)) diff --git a/supported_go_versions.txt b/supported_go_versions.txt new file mode 100644 index 0000000..321c2b6 --- /dev/null +++ b/supported_go_versions.txt @@ -0,0 +1,3 @@ +1.22 +1.21 +1.20 \ No newline at end of file