re
/
client_golang
forked from mirror/client_golang
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add a safety goroutine budget 

This makes sure we don't spin up a possibly infinite number of
goroutines in `Gather`, which could theoretically happen with unlucky
scheduling.
This commit is contained in:
beorn7 2018-01-30 15:23:23 +01:00
parent e04451f4be
commit 4957f7bba4
1 changed files with 11 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
prometheus/registry.go
View File

@ -376,7 +376,7 @@ func (r *Registry) Gather() ([]*dto.MetricFamily, error) {
) )
r.mtx.RLock() r.mtx.RLock()
wg.Add(len(r.collectorsByID)) goroutineBudget := len(r.collectorsByID)
metricFamiliesByName := make(map[string]*dto.MetricFamily, len(r.dimHashesByName)) metricFamiliesByName := make(map[string]*dto.MetricFamily, len(r.dimHashesByName))
collectors := make(chan Collector, len(r.collectorsByID)) collectors := make(chan Collector, len(r.collectorsByID))
for _, collector := range r.collectorsByID { for _, collector := range r.collectorsByID {
@ -392,6 +392,8 @@ func (r *Registry) Gather() ([]*dto.MetricFamily, error) {
} }
r.mtx.RUnlock() r.mtx.RUnlock()
wg.Add(goroutineBudget)
collectWorker := func() { collectWorker := func() {
for { for {
select { select {
@ -404,8 +406,9 @@ func (r *Registry) Gather() ([]*dto.MetricFamily, error) {
} }
} }
// Start the first worker now to make sure at least is running. // Start the first worker now to make sure at least one is running.
go collectWorker() go collectWorker()
goroutineBudget--
// Close the metricChan once all collectors are collected. // Close the metricChan once all collectors are collected.
go func() { go func() {
@ -433,9 +436,11 @@ collectLoop:
registeredDescIDs, registeredDescIDs,
)) ))
default: default:
if len(collectors) == 0 { if goroutineBudget <= 0 || len(collectors) == 0 {
// All collectors are being worked on. Just // All collectors are aleady being worked on or
// process metrics from now on. // we have already as many goroutines started as
// there are collectors. Just process metrics
// from now on.
for metric := range metricChan { for metric := range metricChan {
errs.Append(processMetric( errs.Append(processMetric(
metric, metricFamiliesByName, metric, metricFamiliesByName,
@ -447,6 +452,7 @@ collectLoop:
} }
// Start more workers. // Start more workers.
go collectWorker() go collectWorker()
goroutineBudget--
runtime.Gosched() runtime.Gosched()
} }
} }