From be362f0d9f0534843cadd5f8f571c2d1740d768f Mon Sep 17 00:00:00 2001 From: Emile Vauge Date: Thu, 8 Dec 2016 13:32:12 +0100 Subject: [PATCH] Add Operation with recover --- acme/challengeProvider.go | 3 ++- cluster/datastore.go | 3 ++- cluster/leadership.go | 2 +- provider/consul_catalog.go | 2 +- provider/docker.go | 2 +- provider/kubernetes.go | 2 +- provider/kv.go | 4 ++-- provider/marathon.go | 2 +- provider/mesos.go | 2 +- safe/routine.go | 19 +++++++++++++++++-- safe/routine_test.go | 16 ++++++++++++++++ 11 files changed, 45 insertions(+), 12 deletions(-) create mode 100644 safe/routine_test.go diff --git a/acme/challengeProvider.go b/acme/challengeProvider.go index 464179760..2b1f2f37e 100644 --- a/acme/challengeProvider.go +++ b/acme/challengeProvider.go @@ -10,6 +10,7 @@ import ( "github.com/cenk/backoff" "github.com/containous/traefik/cluster" "github.com/containous/traefik/log" + "github.com/containous/traefik/safe" "github.com/xenolf/lego/acme" ) @@ -49,7 +50,7 @@ func (c *challengeProvider) getCertificate(domain string) (cert *tls.Certificate } ebo := backoff.NewExponentialBackOff() ebo.MaxElapsedTime = 60 * time.Second - err := backoff.RetryNotify(operation, ebo, notify) + err := backoff.RetryNotify(safe.OperationWithRecover(operation), ebo, notify) if err != nil { log.Errorf("Error getting cert: %v", err) return nil, false diff --git a/cluster/datastore.go b/cluster/datastore.go index 88f453b4d..fe552d897 100644 --- a/cluster/datastore.go +++ b/cluster/datastore.go @@ -11,6 +11,7 @@ import ( "github.com/containous/staert" "github.com/containous/traefik/job" "github.com/containous/traefik/log" + "github.com/containous/traefik/safe" "github.com/docker/libkv/store" "github.com/satori/go.uuid" ) @@ -109,7 +110,7 @@ func (d *Datastore) watchChanges() error { notify := func(err error, time time.Duration) { log.Errorf("Error in watch datastore: %+v, retrying in %s", err, time) } - err := backoff.RetryNotify(operation, job.NewBackOff(backoff.NewExponentialBackOff()), notify) + err := backoff.RetryNotify(safe.OperationWithRecover(operation), job.NewBackOff(backoff.NewExponentialBackOff()), notify) if err != nil { log.Errorf("Error in watch datastore: %v", err) } diff --git a/cluster/leadership.go b/cluster/leadership.go index 5173dd421..e091fc8cc 100644 --- a/cluster/leadership.go +++ b/cluster/leadership.go @@ -47,7 +47,7 @@ func (l *Leadership) Participate(pool *safe.Pool) { notify := func(err error, time time.Duration) { log.Errorf("Leadership election error %+v, retrying in %s", err, time) } - err := backoff.RetryNotify(operation, backOff, notify) + err := backoff.RetryNotify(safe.OperationWithRecover(operation), backOff, notify) if err != nil { log.Errorf("Cannot elect leadership %+v", err) } diff --git a/provider/consul_catalog.go b/provider/consul_catalog.go index b83da3b1d..981ecda74 100644 --- a/provider/consul_catalog.go +++ b/provider/consul_catalog.go @@ -334,7 +334,7 @@ func (provider *ConsulCatalog) Provide(configurationChan chan<- types.ConfigMess operation := func() error { return provider.watch(configurationChan, stop) } - err := backoff.RetryNotify(operation, job.NewBackOff(backoff.NewExponentialBackOff()), notify) + err := backoff.RetryNotify(safe.OperationWithRecover(operation), job.NewBackOff(backoff.NewExponentialBackOff()), notify) if err != nil { log.Errorf("Cannot connect to consul server %+v", err) } diff --git a/provider/docker.go b/provider/docker.go index e93d603fd..630bfafe0 100644 --- a/provider/docker.go +++ b/provider/docker.go @@ -229,7 +229,7 @@ func (provider *Docker) Provide(configurationChan chan<- types.ConfigMessage, po notify := func(err error, time time.Duration) { log.Errorf("Docker connection error %+v, retrying in %s", err, time) } - err := backoff.RetryNotify(operation, job.NewBackOff(backoff.NewExponentialBackOff()), notify) + err := backoff.RetryNotify(safe.OperationWithRecover(operation), job.NewBackOff(backoff.NewExponentialBackOff()), notify) if err != nil { log.Errorf("Cannot connect to docker server %+v", err) } diff --git a/provider/kubernetes.go b/provider/kubernetes.go index 65ec1f0d4..cb1c367a2 100644 --- a/provider/kubernetes.go +++ b/provider/kubernetes.go @@ -91,7 +91,7 @@ func (provider *Kubernetes) Provide(configurationChan chan<- types.ConfigMessage notify := func(err error, time time.Duration) { log.Errorf("Kubernetes connection error %+v, retrying in %s", err, time) } - err := backoff.RetryNotify(operation, job.NewBackOff(backoff.NewExponentialBackOff()), notify) + err := backoff.RetryNotify(safe.OperationWithRecover(operation), job.NewBackOff(backoff.NewExponentialBackOff()), notify) if err != nil { log.Errorf("Cannot connect to Kubernetes server %+v", err) } diff --git a/provider/kv.go b/provider/kv.go index 566188b45..6885d09b3 100644 --- a/provider/kv.go +++ b/provider/kv.go @@ -76,7 +76,7 @@ func (provider *Kv) watchKv(configurationChan chan<- types.ConfigMessage, prefix notify := func(err error, time time.Duration) { log.Errorf("KV connection error: %+v, retrying in %s", err, time) } - err := backoff.RetryNotify(operation, job.NewBackOff(backoff.NewExponentialBackOff()), notify) + err := backoff.RetryNotify(safe.OperationWithRecover(operation), job.NewBackOff(backoff.NewExponentialBackOff()), notify) if err != nil { return fmt.Errorf("Cannot connect to KV server: %v", err) } @@ -107,7 +107,7 @@ func (provider *Kv) provide(configurationChan chan<- types.ConfigMessage, pool * notify := func(err error, time time.Duration) { log.Errorf("KV connection error: %+v, retrying in %s", err, time) } - err := backoff.RetryNotify(operation, job.NewBackOff(backoff.NewExponentialBackOff()), notify) + err := backoff.RetryNotify(safe.OperationWithRecover(operation), job.NewBackOff(backoff.NewExponentialBackOff()), notify) if err != nil { return fmt.Errorf("Cannot connect to KV server: %v", err) } diff --git a/provider/marathon.go b/provider/marathon.go index 3e98644a5..b7c84a589 100644 --- a/provider/marathon.go +++ b/provider/marathon.go @@ -120,7 +120,7 @@ func (provider *Marathon) Provide(configurationChan chan<- types.ConfigMessage, notify := func(err error, time time.Duration) { log.Errorf("Marathon connection error %+v, retrying in %s", err, time) } - err := backoff.RetryNotify(operation, job.NewBackOff(backoff.NewExponentialBackOff()), notify) + err := backoff.RetryNotify(safe.OperationWithRecover(operation), job.NewBackOff(backoff.NewExponentialBackOff()), notify) if err != nil { log.Errorf("Cannot connect to Marathon server %+v", err) } diff --git a/provider/mesos.go b/provider/mesos.go index 6935712d8..1835266b3 100644 --- a/provider/mesos.go +++ b/provider/mesos.go @@ -113,7 +113,7 @@ func (provider *Mesos) Provide(configurationChan chan<- types.ConfigMessage, poo notify := func(err error, time time.Duration) { log.Errorf("mesos connection error %+v, retrying in %s", err, time) } - err := backoff.RetryNotify(operation, job.NewBackOff(backoff.NewExponentialBackOff()), notify) + err := backoff.RetryNotify(safe.OperationWithRecover(operation), job.NewBackOff(backoff.NewExponentialBackOff()), notify) if err != nil { log.Errorf("Cannot connect to mesos server %+v", err) } diff --git a/safe/routine.go b/safe/routine.go index 290f3fe23..62d666963 100644 --- a/safe/routine.go +++ b/safe/routine.go @@ -1,11 +1,12 @@ package safe import ( + "fmt" + "github.com/cenk/backoff" + "github.com/containous/traefik/log" "context" "runtime/debug" "sync" - - "github.com/containous/traefik/log" ) type routine struct { @@ -145,3 +146,17 @@ func defaultRecoverGoroutine(err interface{}) { log.Errorf("Error in Go routine: %s", err) debug.PrintStack() } + +// OperationWithRecover wrap a backoff operation in a Recover +func OperationWithRecover(operation backoff.Operation) backoff.Operation { + return func() (err error) { + defer func() { + if res := recover(); res != nil { + defaultRecoverGoroutine(err) + err = fmt.Errorf("Panic in operation: %s", err) + } + }() + err = operation() + return nil + } +} diff --git a/safe/routine_test.go b/safe/routine_test.go new file mode 100644 index 000000000..07fa89193 --- /dev/null +++ b/safe/routine_test.go @@ -0,0 +1,16 @@ +package safe + +import ( + "github.com/cenk/backoff" + "testing" +) + +func TestOperationWithRecover(t *testing.T) { + operation := func() error { + panic("BOOM") + } + err := backoff.Retry(OperationWithRecover(operation), &backoff.StopBackOff{}) + if err == nil { + t.Fatalf("Error in OperationWithRecover: %s", err) + } +}