baalajimaestro/traefik
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

Merge pull request #627 from containous/add-long-job-exponential-backoff

Browse source 
Add long job exponential backoff
This commit is contained in:
Vincent Demeester 2016-08-19 11:40:16 +02:00 committed by GitHub
parent fc19ab2868 97ddfcb17a
commit 95e8f0a31e
10 changed files with 68 additions and 27 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
middlewares/logger.go
View file

@ -56,7 +56,7 @@ func NewLogger(file string) *Logger {
if len(file) > 0 {
fi, err := os.OpenFile(file, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
if err != nil {
log.Fatal("Error opening file", err)
log.Error("Error opening file", err)
}
return &Logger{fi}
}

5
provider/consul_catalog.go
View file

@ -13,6 +13,7 @@ import (
"github.com/cenkalti/backoff"
"github.com/containous/traefik/safe"
"github.com/containous/traefik/types"
"github.com/containous/traefik/utils"
"github.com/hashicorp/consul/api"
)
@ -320,9 +321,9 @@ func (provider *ConsulCatalog) Provide(configurationChan chan<- types.ConfigMess
worker := func() error {
return provider.watch(configurationChan, stop)
}
err := backoff.RetryNotify(worker, backoff.NewExponentialBackOff(), notify)
err := utils.RetryNotifyJob(worker, backoff.NewExponentialBackOff(), notify)
if err != nil {
log.Fatalf("Cannot connect to consul server %+v", err)
log.Errorf("Cannot connect to consul server %+v", err)
}
})

5
provider/docker.go
View file

@ -15,6 +15,7 @@ import (
"github.com/cenkalti/backoff"
"github.com/containous/traefik/safe"
"github.com/containous/traefik/types"
"github.com/containous/traefik/utils"
"github.com/containous/traefik/version"
"github.com/docker/engine-api/client"
dockertypes "github.com/docker/engine-api/types"
@ -139,9 +140,9 @@ func (provider *Docker) Provide(configurationChan chan<- types.ConfigMessage, po
notify := func(err error, time time.Duration) {
log.Errorf("Docker connection error %+v, retrying in %s", err, time)
}
err := backoff.RetryNotify(operation, backoff.NewExponentialBackOff(), notify)
err := utils.RetryNotifyJob(operation, backoff.NewExponentialBackOff(), notify)
if err != nil {
log.Fatalf("Cannot connect to docker server %+v", err)
log.Errorf("Cannot connect to docker server %+v", err)
}
})

15
provider/kubernetes.go
View file

@ -7,7 +7,7 @@ import (
"github.com/containous/traefik/provider/k8s"
"github.com/containous/traefik/safe"
"github.com/containous/traefik/types"
"io"
"github.com/containous/traefik/utils"
"io/ioutil"
"os"
"reflect"
@ -104,7 +104,7 @@ func (provider *Kubernetes) Provide(configurationChan chan<- types.ConfigMessage
for {
stopWatch := make(chan bool, 5)
defer close(stopWatch)
log.Debugf("Using lable selector: %s", provider.LabelSelector)
log.Debugf("Using label selector: '%s'", provider.LabelSelector)
eventsChan, errEventsChan, err := k8sClient.WatchAll(provider.LabelSelector, stopWatch)
if err != nil {
log.Errorf("Error watching kubernetes events: %v", err)
@ -116,18 +116,13 @@ func (provider *Kubernetes) Provide(configurationChan chan<- types.ConfigMessage
return nil
}
}
Watch:
for {
select {
case <-stop:
stopWatch <- true
return nil
case err, ok := <-errEventsChan:
case err, _ := <-errEventsChan:
stopWatch <- true
if ok && strings.Contains(err.Error(), io.EOF.Error()) {
// edge case, kubernetes long-polling disconnection
break Watch
}
return err
case event := <-eventsChan:
log.Debugf("Received event from kubernetes %+v", event)
@ -152,9 +147,9 @@ func (provider *Kubernetes) Provide(configurationChan chan<- types.ConfigMessage
notify := func(err error, time time.Duration) {
log.Errorf("Kubernetes connection error %+v, retrying in %s", err, time)
}
err := backoff.RetryNotify(operation, backOff, notify)
err := utils.RetryNotifyJob(operation, backOff, notify)
if err != nil {
log.Fatalf("Cannot connect to Kubernetes server %+v", err)
log.Errorf("Cannot connect to Kubernetes server %+v", err)
}
})

5
provider/kv.go
View file

@ -13,6 +13,7 @@ import (
"github.com/cenkalti/backoff"
"github.com/containous/traefik/safe"
"github.com/containous/traefik/types"
"github.com/containous/traefik/utils"
"github.com/docker/libkv"
"github.com/docker/libkv/store"
)
@ -75,7 +76,7 @@ func (provider *Kv) watchKv(configurationChan chan<- types.ConfigMessage, prefix
notify := func(err error, time time.Duration) {
log.Errorf("KV connection error: %+v, retrying in %s", err, time)
}
err := backoff.RetryNotify(operation, backoff.NewExponentialBackOff(), notify)
err := utils.RetryNotifyJob(operation, backoff.NewExponentialBackOff(), notify)
if err != nil {
return fmt.Errorf("Cannot connect to KV server: %v", err)
}
@ -105,7 +106,7 @@ func (provider *Kv) provide(configurationChan chan<- types.ConfigMessage, pool *
notify := func(err error, time time.Duration) {
log.Errorf("KV connection error: %+v, retrying in %s", err, time)
}
err := backoff.RetryNotify(operation, backoff.NewExponentialBackOff(), notify)
err := utils.RetryNotifyJob(operation, backoff.NewExponentialBackOff(), notify)
if err != nil {
return fmt.Errorf("Cannot connect to KV server: %v", err)
}

5
provider/marathon.go
View file

@ -13,6 +13,7 @@ import (
"github.com/cenkalti/backoff"
"github.com/containous/traefik/safe"
"github.com/containous/traefik/types"
"github.com/containous/traefik/utils"
"github.com/gambol99/go-marathon"
"net/http"
"time"
@ -108,9 +109,9 @@ func (provider *Marathon) Provide(configurationChan chan<- types.ConfigMessage,
notify := func(err error, time time.Duration) {
log.Errorf("Marathon connection error %+v, retrying in %s", err, time)
}
err := backoff.RetryNotify(operation, backoff.NewExponentialBackOff(), notify)
err := utils.RetryNotifyJob(operation, backoff.NewExponentialBackOff(), notify)
if err != nil {
log.Fatalf("Cannot connect to Marathon server %+v", err)
log.Errorf("Cannot connect to Marathon server %+v", err)
}
return nil
}

9
provider/mesos.go
View file

@ -12,6 +12,7 @@ import (
"github.com/cenkalti/backoff"
"github.com/containous/traefik/safe"
"github.com/containous/traefik/types"
"github.com/containous/traefik/utils"
"github.com/mesos/mesos-go/detector"
_ "github.com/mesos/mesos-go/detector/zoo" // Registers the ZK detector
"github.com/mesosphere/mesos-dns/detect"
@ -110,9 +111,9 @@ func (provider *Mesos) Provide(configurationChan chan<- types.ConfigMessage, poo
notify := func(err error, time time.Duration) {
log.Errorf("mesos connection error %+v, retrying in %s", err, time)
}
err := backoff.RetryNotify(operation, backoff.NewExponentialBackOff(), notify)
err := utils.RetryNotifyJob(operation, backoff.NewExponentialBackOff(), notify)
if err != nil {
log.Fatalf("Cannot connect to mesos server %+v", err)
log.Errorf("Cannot connect to mesos server %+v", err)
}
return nil
}
@ -393,9 +394,9 @@ func detectMasters(zk string, masters []string) <-chan []string {
if zk != "" {
log.Debugf("Starting master detector for ZK ", zk)
if md, err := detector.New(zk); err != nil {
log.Fatalf("failed to create master detector: %v", err)
log.Errorf("failed to create master detector: %v", err)
} else if err := md.Detect(detect.NewMasters(masters, changed)); err != nil {
log.Fatalf("failed to initialize master detector: %v", err)
log.Errorf("failed to initialize master detector: %v", err)
}
} else {
changed <- masters

4
server.go
View file

@ -413,7 +413,7 @@ func (server *Server) prepareServer(entryPointName string, router *middlewares.H
negroni.UseHandler(router)
tlsConfig, err := server.createTLSConfig(entryPointName, entryPoint.TLS, router)
if err != nil {
log.Fatalf("Error creating TLS config %s", err)
log.Errorf("Error creating TLS config %s", err)
return nil, err
}
@ -431,7 +431,7 @@ func (server *Server) prepareServer(entryPointName string, router *middlewares.H
TLSConfig: tlsConfig,
}, tlsConfig)
if err != nil {
log.Fatalf("Error hijacking server %s", err)
log.Errorf("Error hijacking server %s", err)
return nil, err
}
return gracefulServer, nil

4
traefik.go
View file

@ -203,7 +203,7 @@ func run(traefikConfiguration *TraefikConfiguration) {
// logging
level, err := log.ParseLevel(strings.ToLower(globalConfiguration.LogLevel))
if err != nil {
log.Fatal("Error getting level", err)
log.Error("Error getting level", err)
}
log.SetLevel(level)
if len(globalConfiguration.TraefikLogsFile) > 0 {
@ -214,7 +214,7 @@ func run(traefikConfiguration *TraefikConfiguration) {
}
}()
if err != nil {
log.Fatal("Error opening file", err)
log.Error("Error opening file", err)
} else {
log.SetOutput(fi)
log.SetFormatter(&log.TextFormatter{DisableColors: true, FullTimestamp: true, DisableSorting: true})

41
utils/retry.go Normal file
View file

@ -0,0 +1,41 @@
package utils
import (
"github.com/cenkalti/backoff"
"time"
)
const (
minLongJobInterval = 30 * time.Second
)
// RetryNotifyJob calls notify function with the error and wait duration
// for each failed attempt before sleep.
func RetryNotifyJob(operation backoff.Operation, b backoff.BackOff, notify backoff.Notify) error {
var err error
var next time.Duration
b.Reset()
for {
before := time.Now()
if err = operation(); err == nil {
return nil
}
elapsed := time.Since(before)
// If long job, we reset the backoff
if elapsed >= minLongJobInterval {
b.Reset()
}
if next = b.NextBackOff(); next == backoff.Stop {
return err
}
if notify != nil {
notify(err, next)
}
time.Sleep(next)
}
}