Update comments with more explanations

Signed-off-by: György Krajcsovits <gyorgy.krajcsovits@grafana.com>

prometheus/histogram.go

@@ -1658,7 +1658,10 @@ func addAndResetCounts(hot, cold *histogramCounts) {
 type nativeExemplars struct {
 	sync.Mutex
 
-	ttl       time.Duration
+	// Time-to-live for exemplars, it is set to -1 if exemplars are disabled, that is NativeHistogramMaxExemplars is below 0.
+	// The ttl is used on insertion to remove an exemplar that is older than ttl, if present.
+	ttl time.Duration
+
 	exemplars []*dto.Exemplar
 }
 
@@ -1690,13 +1693,11 @@ func (n *nativeExemplars) addExemplar(e *dto.Exemplar) {
 	n.Lock()
 	defer n.Unlock()
 
-	// The index where to insert the new exemplar.
-	var nIdx int = -1
-
 	// When the number of exemplars has not yet exceeded or
 	// is equal to cap(n.exemplars), then
 	// insert the new exemplar directly.
 	if len(n.exemplars) < cap(n.exemplars) {
+		var nIdx int
 		for nIdx = 0; nIdx < len(n.exemplars); nIdx++ {
 			if *e.Value < *n.exemplars[nIdx].Value {
 				break
@@ -1716,11 +1717,34 @@ func (n *nativeExemplars) addExemplar(e *dto.Exemplar) {
 
 	// When the number of exemplars exceeds the limit, remove one exemplar.
 	var (
-		ot    = time.Now() // Oldest timestamp seen.
+		ot    = time.Time{} // Oldest timestamp seen. Initial value doesn't matter as we replace it due to otIdx == -1 in the loop.
-		otIdx = -1         // Index of the exemplar with the oldest timestamp.
+		otIdx = -1          // Index of the exemplar with the oldest timestamp.
 
-		md   = -1.0  // Logarithm of the delta of the closest pair of exemplars.
+		md = -1.0 // Logarithm of the delta of the closest pair of exemplars.
-		rIdx = -1    // Index of the older exemplar within the closest pair and where we need to insert the new exemplar.
+
+		// The insertion point of the new exemplar in the exemplars slice after insertion.
+		// This is calculated purely based on the order of the exemplars by value.
+		// nIdx == len(n.exemplars) means the new exemplar is to be inserted after the end.
+		nIdx = -1
+
+		// rIdx is ultimately the index for the exemplar that we are replacing with the new exemplar.
+		// The aim is to keep a good spread of exemplars by value and not let them bunch up too much.
+		// It is calculated in 3 steps:
+		//   1. First we set rIdx to the index of the older exemplar within the closest pair by value.
+		//      That is the following will be true (on log scale):
+		//      either the exemplar pair on index (rIdx-1, rIdx) or (rIdx, rIdx+1) will have
+		//      the closest values to each other from all pairs.
+		//      For example, suppose the values are distributed like this:
+		//        |-----------x-------------x----------------x----x-----|
+		//                                                   ^--rIdx as this is older.
+		//      Or like this:
+		//        |-----------x-------------x----------------x----x-----|
+		//                                                        ^--rIdx as this is older.
+		//   2. If there is an exemplar that expired, then we simple reset rIdx to that index.
+		//   3. We check if by inserting the new exemplar we would create a closer pair at
+		//      (nIdx-1, nIdx) or (nIdx, nIdx+1) and set rIdx to nIdx-1 or nIdx accordingly to
+		//      keep the spread of exemplars by value; otherwise we keep rIdx as it is.
+		rIdx = -1
 		cLog float64 // Logarithm of the current exemplar.
 		pLog float64 // Logarithm of the previous exemplar.
 	)
@@ -1745,7 +1769,7 @@ func (n *nativeExemplars) addExemplar(e *dto.Exemplar) {
 		}
 		diff := math.Abs(cLog - pLog)
 		if md == -1 || diff < md {
-			// The closest exemplar pair is this: |exemplar.[i] - n.exemplars[i-1].Value| is minimal.
+			// The closest exemplar pair is at index: i-1, i.
 			// Choose the exemplar with the older timestamp for replacement.
 			md = diff
 			if n.exemplars[i].Timestamp.AsTime().Before(n.exemplars[i-1].Timestamp.AsTime()) {
@@ -1763,7 +1787,8 @@ func (n *nativeExemplars) addExemplar(e *dto.Exemplar) {
 		nIdx = len(n.exemplars)
 	}
 	// Here, we have the following relationships:
-	// n.exemplars[nIdx-1].Value < e.Value <= n.exemplars[nIdx].Value
+	// n.exemplars[nIdx-1].Value < e.Value (if nIdx > 0)
+	// e.Value <= n.exemplars[nIdx].Value (if nIdx < len(n.exemplars))
 
 	if otIdx != -1 && e.Timestamp.AsTime().Sub(ot) > n.ttl {
 		// If the oldest exemplar has expired, then replace it with the new exemplar.
@@ -1776,9 +1801,11 @@ func (n *nativeExemplars) addExemplar(e *dto.Exemplar) {
 		if nIdx > 0 {
 			diff := math.Abs(elog - math.Log(n.exemplars[nIdx-1].GetValue()))
 			if diff < md {
-				// The closest exemplar pair is this: |e.Value - n.exemplars[nIdx-1].Value| is minimal.
+				// The value we are about to insert is closer to the previous exemplar at the insertion point than what we calculated before in rIdx.
-				// Assume that the exemplar we are inserting has a newer timestamp. This is not always
+				//                                            v--rIdx
-				// true, due to concurrency, but it's a good enough approximation.
+				// |-----------x-n-----------x----------------x----x-----|
+				//     nIdx-1--^ ^--new exemplar value
+				// Do not make the spread worse, replace nIdx-1 and not rIdx.
 				md = diff
 				rIdx = nIdx - 1
 			}
@@ -1786,9 +1813,11 @@ func (n *nativeExemplars) addExemplar(e *dto.Exemplar) {
 		if nIdx < len(n.exemplars) {
 			diff := math.Abs(math.Log(n.exemplars[nIdx].GetValue()) - elog)
 			if diff < md {
-				// The closest exemplar pair is this: |n.exemplars[nIdx].Value - e.Value| is minimal.
+				// The value we are about to insert is closer to the next exemplar at the insertion point than what we calculated before in rIdx.
-				// Assume that the exemplar we are inserting has a newer timestamp. This is not always
+				//                                            v--rIdx
-				// true, due to concurrency, but it's a good enough approximation.
+				// |-----------x-----------n-x----------------x----x-----|
+				//     new exemplar value--^ ^--nIdx
+				// Do not make the spread worse, replace nIdx-1 and not rIdx.
 				rIdx = nIdx
 			}
 		}