From 9d42148c648146e44fc1a7ba2e8365297ec98d91 Mon Sep 17 00:00:00 2001
From: Josh Deprez <josh.deprez@gmail.com>
Date: Sun, 22 Jan 2023 19:21:49 +1100
Subject: [PATCH] Add better retry backoff

---
 main.go | 37 ++++++++++++++++++++++++++++++++++++-
 1 file changed, 36 insertions(+), 1 deletion(-)

diff --git a/main.go b/main.go
index 33ee914..ed0956d 100644
--- a/main.go
+++ b/main.go
@@ -19,10 +19,12 @@
 package main
 
 import (
+	"context"
 	"errors"
 	"flag"
 	"fmt"
 	"log"
+	"math/rand"
 	"net/http"
 	"strings"
 	"sync"
@@ -202,6 +204,37 @@ func scrape() error {
 	return nil
 }
 
+func retries(ctx context.Context, tries int, base time.Duration, mul float64) <-chan int {
+	ch := make(chan int)
+	go func() {
+		defer close(ch)
+		i := 0
+		for {
+			select {
+			case ch <- i:
+				i++
+				if i == tries {
+					return
+				}
+
+				t := time.NewTimer(time.Duration(rand.Int63n(int64(base))))
+				select {
+				case <-t.C:
+					// next iteration
+				case <-ctx.Done():
+					t.Stop()
+					return
+				}
+
+				base = time.Duration(mul * float64(base))
+			case <-ctx.Done():
+				return
+			}
+		}
+	}()
+	return ch
+}
+
 func metricsHandler(w http.ResponseWriter, r *http.Request) {
 	// In normal mode, always serve metrics
 	defer promHandler.ServeHTTP(w, r)
@@ -213,7 +246,9 @@ func metricsHandler(w http.ResponseWriter, r *http.Request) {
 	}
 
 	var lastErr error
-	for i := 0; i < 3; i++ {
+	ctx, canc := context.WithCancel(context.Background())
+	defer canc()
+	for range retries(ctx, 3, time.Second, 2) {
 		if err := scrape(); err != nil {
 			log.Printf("Scrape error: %v", err)
 			lastErr = err