Add better retry backoff

main.go

@@ -19,10 +19,12 @@
 package main
 
 import (
+	"context"
 	"errors"
 	"flag"
 	"fmt"
 	"log"
+	"math/rand"
 	"net/http"
 	"strings"
 	"sync"
@@ -202,6 +204,37 @@ func scrape() error {
 	return nil
 }
 
+func retries(ctx context.Context, tries int, base time.Duration, mul float64) <-chan int {
+	ch := make(chan int)
+	go func() {
+		defer close(ch)
+		i := 0
+		for {
+			select {
+			case ch <- i:
+				i++
+				if i == tries {
+					return
+				}
+
+				t := time.NewTimer(time.Duration(rand.Int63n(int64(base))))
+				select {
+				case <-t.C:
+					// next iteration
+				case <-ctx.Done():
+					t.Stop()
+					return
+				}
+
+				base = time.Duration(mul * float64(base))
+			case <-ctx.Done():
+				return
+			}
+		}
+	}()
+	return ch
+}
+
 func metricsHandler(w http.ResponseWriter, r *http.Request) {
 	// In normal mode, always serve metrics
 	defer promHandler.ServeHTTP(w, r)
@@ -213,7 +246,9 @@ func metricsHandler(w http.ResponseWriter, r *http.Request) {
 	}
 
 	var lastErr error
-	for i := 0; i < 3; i++ {
+	ctx, canc := context.WithCancel(context.Background())
+	defer canc()
+	for range retries(ctx, 3, time.Second, 2) {
 		if err := scrape(); err != nil {
 			log.Printf("Scrape error: %v", err)
 			lastErr = err