Retry on plugin API calls

go.mod

@@ -31,6 +31,7 @@ require (
 	github.com/hashicorp/consul/api v1.14.0
 	github.com/hashicorp/go-hclog v1.2.0
 	github.com/hashicorp/go-multierror v1.1.1
+	github.com/hashicorp/go-retryablehttp v0.7.1
 	github.com/hashicorp/go-version v1.6.0
 	github.com/hashicorp/nomad/api v0.0.0-20220506174431-b5665129cd1f
 	github.com/improbable-eng/grpc-web v0.15.0
@@ -202,7 +203,6 @@ require (
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
 	github.com/hashicorp/go-msgpack v0.5.5 // indirect
-	github.com/hashicorp/go-retryablehttp v0.7.1 // indirect
 	github.com/hashicorp/go-rootcerts v1.0.2 // indirect
 	github.com/hashicorp/go-sockaddr v1.0.2 // indirect
 	github.com/hashicorp/go-uuid v1.0.2 // indirect

pkg/logs/hclog.go

@@ -0,0 +1,75 @@
+package logs
+
+import (
+	"fmt"
+	"strings"
+	"unicode"
+
+	"github.com/rs/zerolog"
+)
+
+// RetryableHTTPLogger wraps our logger and implements retryablehttp.LeveledLogger.
+// The retry library sends fields as pairs of keys and values as structured logging,
+// so we need to adapt them to our logger.
+type RetryableHTTPLogger struct {
+	logger zerolog.Logger
+}
+
+// NewRetryableHTTPLogger creates an implementation of the retryablehttp.LeveledLogger.
+func NewRetryableHTTPLogger(logger zerolog.Logger) *RetryableHTTPLogger {
+	return &RetryableHTTPLogger{logger: logger}
+}
+
+// Error starts a new message with error level.
+func (l RetryableHTTPLogger) Error(msg string, keysAndValues ...interface{}) {
+	logWithLevel(l.logger.Error().CallerSkipFrame(2), msg, keysAndValues...)
+}
+
+// Info starts a new message with info level.
+func (l RetryableHTTPLogger) Info(msg string, keysAndValues ...interface{}) {
+	logWithLevel(l.logger.Info().CallerSkipFrame(2), msg, keysAndValues...)
+}
+
+// Debug starts a new message with debug level.
+func (l RetryableHTTPLogger) Debug(msg string, keysAndValues ...interface{}) {
+	logWithLevel(l.logger.Debug().CallerSkipFrame(2), msg, keysAndValues...)
+}
+
+// Warn starts a new message with warn level.
+func (l RetryableHTTPLogger) Warn(msg string, keysAndValues ...interface{}) {
+	logWithLevel(l.logger.Warn().CallerSkipFrame(2), msg, keysAndValues...)
+}
+
+func logWithLevel(ev *zerolog.Event, msg string, kvs ...interface{}) {
+	if len(kvs)%2 == 0 {
+		for i := 0; i < len(kvs)-1; i += 2 {
+			// The first item of the pair (the key) is supposed to be a string.
+			key, ok := kvs[i].(string)
+			if !ok {
+				continue
+			}
+
+			val := kvs[i+1]
+
+			var s fmt.Stringer
+			if s, ok = val.(fmt.Stringer); ok {
+				ev.Str(key, s.String())
+			} else {
+				ev.Interface(key, val)
+			}
+		}
+	}
+
+	// Capitalize first character.
+	first := true
+	msg = strings.Map(func(r rune) rune {
+		if first {
+			first = false
+			return unicode.ToTitle(r)
+		}
+
+		return r
+	}, msg)
+
+	ev.Msg(msg)
+}

pkg/logs/hclog_test.go

@@ -0,0 +1,17 @@
+package logs
+
+import (
+	"os"
+	"testing"
+	"time"
+
+	"github.com/rs/zerolog"
+)
+
+func TestNewRetryableHTTPLogger(t *testing.T) {
+	out := zerolog.ConsoleWriter{Out: os.Stderr, TimeFormat: time.RFC3339}
+
+	logger := NewRetryableHTTPLogger(zerolog.New(out).With().Caller().Logger())
+
+	logger.Info("foo")
+}

pkg/plugins/client.go

@@ -16,6 +16,9 @@ import (
 	"strings"
 	"time"
 
+	"github.com/hashicorp/go-retryablehttp"
+	"github.com/rs/zerolog/log"
+	"github.com/traefik/traefik/v2/pkg/logs"
 	"golang.org/x/mod/module"
 	"golang.org/x/mod/zip"
 	"gopkg.in/yaml.v3"
@@ -77,8 +80,13 @@ func NewClient(opts ClientOptions) (*Client, error) {
 		return nil, fmt.Errorf("failed to create archives directory %s: %w", archivesPath, err)
 	}
 
+	client := retryablehttp.NewClient()
+	client.Logger = logs.NewRetryableHTTPLogger(log.Logger)
+	client.HTTPClient = &http.Client{Timeout: 10 * time.Second}
+	client.RetryMax = 3
+
 	return &Client{
-		HTTPClient: &http.Client{Timeout: 10 * time.Second},
+		HTTPClient: client.StandardClient(),
 		baseURL:    baseURL,
 
 		archives:  archivesPath,