Add a concurrent end-to-end test for observe-register-gather

This is an attempt to expose
https://github.com/istio/istio/issues/8906 .  The failure to do so
makes me believe the error is either already fixed in current
client_golang, or something weird I haven't spotted yet is happening
in the istio code.

Signed-off-by: beorn7 <beorn@soundcloud.com>

prometheus/registry_test.go

@@ -21,9 +21,12 @@ package prometheus_test
 
 import (
 	"bytes"
+	"math/rand"
 	"net/http"
 	"net/http/httptest"
+	"sync"
 	"testing"
+	"time"
 
 	dto "github.com/prometheus/client_model/go"
 
@@ -772,3 +775,99 @@ func TestAlreadyRegistered(t *testing.T) {
 		t.Error("unexpected error:", err)
 	}
 }
+
+// TestHistogramVecRegisterGatherConcurrency is an end-to-end test that
+// concurrently calls Observe on random elements of a HistogramVec while the
+// same HistogramVec is registered concurrently and the Gather method of the
+// registry is called concurrently.
+func TestHistogramVecRegisterGatherConcurrency(t *testing.T) {
+	var (
+		reg = prometheus.NewPedanticRegistry()
+		hv  = prometheus.NewHistogramVec(
+			prometheus.HistogramOpts{
+				Name:        "test_histogram",
+				Help:        "This helps testing.",
+				ConstLabels: prometheus.Labels{"foo": "bar"},
+			},
+			[]string{"one", "two", "three"},
+		)
+		labelValues = []string{"a", "b", "c", "alpha", "beta", "gamma", "aleph", "beth", "gimel"}
+		quit        = make(chan struct{})
+		wg          sync.WaitGroup
+	)
+
+	observe := func() {
+		defer wg.Done()
+		for {
+			select {
+			case <-quit:
+				return
+			default:
+				obs := rand.NormFloat64()*.1 + .2
+				hv.WithLabelValues(
+					labelValues[rand.Intn(len(labelValues))],
+					labelValues[rand.Intn(len(labelValues))],
+					labelValues[rand.Intn(len(labelValues))],
+				).Observe(obs)
+			}
+		}
+	}
+
+	register := func() {
+		defer wg.Done()
+		for {
+			select {
+			case <-quit:
+				return
+			default:
+				if err := reg.Register(hv); err != nil {
+					if _, ok := err.(prometheus.AlreadyRegisteredError); !ok {
+						t.Error("Registering failed:", err)
+					}
+				}
+				time.Sleep(7 * time.Millisecond)
+			}
+		}
+	}
+
+	gather := func() {
+		defer wg.Done()
+		for {
+			select {
+			case <-quit:
+				return
+			default:
+				if g, err := reg.Gather(); err != nil {
+					t.Error("Gathering failed:", err)
+				} else {
+					if len(g) == 0 {
+						continue
+					}
+					if len(g) != 1 {
+						t.Error("Gathered unexpected number of metric families:", len(g))
+					}
+					if len(g[0].Metric[0].Label) != 4 {
+						t.Error("Gathered unexpected number of label pairs:", len(g[0].Metric[0].Label))
+					}
+				}
+				time.Sleep(4 * time.Millisecond)
+			}
+		}
+	}
+
+	wg.Add(10)
+	go observe()
+	go observe()
+	go register()
+	go observe()
+	go gather()
+	go observe()
+	go register()
+	go observe()
+	go gather()
+	go observe()
+
+	time.Sleep(time.Second)
+	close(quit)
+	wg.Wait()
+}