Move jobs backoff back into traefik

https://github.com/cenk/backoff/pull/27#issuecomment-245842725
This commit is contained in:
Emile Vauge 2016-09-19 19:08:39 +02:00
parent 1b6af2045e
commit 364958cbaf
No known key found for this signature in database
GPG key ID: D808B4C167352E59
11 changed files with 213 additions and 124 deletions

218
glide.lock generated
View file

@ -1,22 +1,24 @@
hash: 809b3fa812ca88940fdc15530804a4bcd881708e4819fed5aa45c42c871ba5cf
updated: 2016-09-20T14:50:04.029710103+02:00
hash: 7382685091734eeeef590d8d298600abfaa21c40c8714ef570b8006e4f8cc846
updated: 2016-08-19T14:15:17.22440385+02:00
imports:
- name: github.com/abbot/go-http-auth
version: cb4372376e1e00e9f6ab9ec142e029302c9e7140
- name: github.com/boltdb/bolt
version: 5cc10bbbc5c141029940133bb33c9e969512a698
version: fff57c100f4dea1905678da7e90d92429dff2904
- name: github.com/BurntSushi/toml
version: 99064174e013895bbd9b025c31100bd1d9b590ca
- name: github.com/BurntSushi/ty
version: 6add9cd6ad42d389d6ead1dde60b4ad71e46fd74
subpackages:
- fun
- name: github.com/cenkalti/backoff
version: cdf48bbc1eb78d1349cbda326a4a037f7ba565c6
- name: github.com/codahale/hdrhistogram
version: f8ad88b59a584afeee9d334eff879b104439117b
- name: github.com/codegangsta/cli
version: 1efa31f08b9333f1bd4882d61f9d668a70cd902e
version: d53eb991652b1d438abdd34ce4bfa3ef1539108e
- name: github.com/codegangsta/negroni
version: dc6b9d037e8dab60cbfc09c61d6932537829be8b
version: 3f7ce7b928e14ff890b067e5bbbc80af73690a9c
- name: github.com/containous/flaeg
version: a731c034dda967333efce5f8d276aeff11f8ff87
- name: github.com/containous/mux
@ -24,122 +26,122 @@ imports:
- name: github.com/containous/staert
version: 044bdfee6c8f5e8fb71f70d5ba1cf4cb11a94e97
- name: github.com/coreos/etcd
version: 1c9e0a0e33051fed6c05c141e6fcbfe5c7f2a899
version: a70513621cf914e2de4dbf90d8a0328e807ffd12
subpackages:
- client
- pkg/pathutil
- pkg/types
- name: github.com/davecgh/go-spew
version: 5215b55f46b2b919f50a1df0eaa5886afe4e3b3d
version: 6d212800a42e8ab5c146b8ace3490ee17e5225f9
subpackages:
- spew
- name: github.com/docker/distribution
version: 857d0f15c0a4d8037175642e0ca3660829551cb5
version: 2b72dd3927b2958160a2336f16145c0c421aa6a4
subpackages:
- reference
- context
- digest
- reference
- registry/api/errcode
- registry/api/v2
- registry/client
- registry/client/auth
- registry/client/transport
- registry/client
- context
- registry/api/v2
- registry/storage/cache
- registry/storage/cache/memory
- uuid
- name: github.com/docker/docker
version: 534753663161334baba06f13b8efa4cad22b5bc5
subpackages:
- namesgenerator
- pkg/namesgenerator
- pkg/random
- pkg/urlutil
- api/types/backend
- builder
- builder/dockerignore
- cliconfig
- cliconfig/configfile
- daemon/graphdriver
- image
- image/v1
- layer
- namesgenerator
- opts
- pkg/archive
- pkg/chrootarchive
- pkg/fileutils
- pkg/gitutils
- pkg/homedir
- pkg/httputils
- pkg/idtools
- pkg/ioutils
- pkg/jsonlog
- pkg/jsonmessage
- pkg/longpath
- pkg/mflag
- pkg/mount
- pkg/namesgenerator
- pkg/plugins
- pkg/plugins/transport
- pkg/pools
- pkg/progress
- pkg/promise
- pkg/random
- pkg/reexec
- pkg/signal
- pkg/stdcopy
- pkg/streamformatter
- pkg/stringid
- pkg/symlink
- pkg/system
- pkg/tarsum
- pkg/term
- pkg/term/windows
- pkg/urlutil
- reference
- registry
- runconfig/opts
- pkg/homedir
- pkg/jsonlog
- pkg/system
- pkg/term/windows
- image
- image/v1
- pkg/ioutils
- opts
- pkg/httputils
- pkg/mflag
- pkg/stringid
- pkg/tarsum
- pkg/mount
- pkg/signal
- builder
- builder/dockerignore
- pkg/archive
- pkg/fileutils
- pkg/progress
- pkg/streamformatter
- layer
- pkg/longpath
- api/types/backend
- pkg/chrootarchive
- pkg/gitutils
- pkg/symlink
- pkg/idtools
- pkg/pools
- daemon/graphdriver
- pkg/reexec
- pkg/plugins
- pkg/plugins/transport
- name: github.com/docker/engine-api
version: 62043eb79d581a32ea849645277023c550732e52
subpackages:
- client
- types
- types/events
- types/filters
- types/container
- types/network
- client/transport
- client/transport/cancellable
- types
- types/blkiodev
- types/container
- types/events
- types/filters
- types/network
- types/reference
- types/registry
- types/strslice
- types/swarm
- types/time
- types/versions
- types/blkiodev
- types/strslice
- name: github.com/docker/go-connections
version: 990a1a1a70b0da4c4cb70e117971a4f0babfbf1a
version: 988efe982fdecb46f01d53465878ff1f2ff411ce
subpackages:
- nat
- sockets
- tlsconfig
- nat
- name: github.com/docker/go-units
version: f2d77a61e3c169b43402a0a1e84f06daf29b8190
version: f2145db703495b2e525c59662db69a7344b00bb8
- name: github.com/docker/libcompose
version: d1876c1d68527a49c0aac22a0b161acc7296b740
subpackages:
- config
- docker
- project
- project/events
- project/options
- utils
- yaml
- docker/builder
- docker/client
- docker/network
- labels
- logger
- lookup
- project
- project/events
- project/options
- utils
- version
- yaml
- name: github.com/docker/libkv
version: aabc039ad04deb721e234f99cd1b4aa28ac71a40
version: 35d3e2084c650109e7bcc7282655b1bc8ba924ff
subpackages:
- store
- store/boltdb
@ -149,36 +151,33 @@ imports:
- name: github.com/donovanhide/eventsource
version: fd1de70867126402be23c306e1ce32828455d85b
- name: github.com/elazarl/go-bindata-assetfs
version: 57eb5e1fc594ad4b0b1dbea7b286d299e0cb43c2
- name: github.com/emilevauge/backoff
version: fa0423e7916ea96a4e9789a3710c8e7cb94ecc42
version: 9a6736ed45b44bf3835afeebb3034b57ed329f3e
- name: github.com/gambol99/go-marathon
version: a558128c87724cd7430060ef5aedf39f83937f55
- name: github.com/go-check/check
version: 4f90aeace3a26ad7021961c297b22c42160c7b25
- name: github.com/gogo/protobuf
version: e57a569e1882958f6b188cb42231d6db87701f2a
version: 6abcf94fd4c97dcb423fdafd42fe9f96ca7e421b
subpackages:
- proto
- name: github.com/golang/glog
version: fca8c8854093a154ff1eb580aae10276ad6b1b5f
version: 23def4e6c14b4da8ac2ed8007337bc5eb5007998
- name: github.com/google/go-querystring
version: 9235644dd9e52eeae6fa48efd539fdc351a0af53
subpackages:
- query
- name: github.com/gorilla/context
version: aed02d124ae4a0e94fea4541c8effd05bf0c8296
version: 08b5f424b9271eedf6f9f0ce86cb9396ed337a42
- name: github.com/hashicorp/consul
version: fce7d75609a04eeb9d4bf41c8dc592aac18fc97d
version: 67a03d57b5851011e5601ee6e7e3d05699548462
subpackages:
- api
- name: github.com/hashicorp/go-cleanhttp
version: 875fb671b3ddc66f8e2f0acc33829c8cb989a38d
version: ad28ea4487f05916463e2423a55166280e8254b5
- name: github.com/hashicorp/serf
version: 6c4672d66fc6312ddde18399262943e21175d831
version: 555e0dcbb180ecbd03431adc28226bb3192558bc
subpackages:
- coordinate
- serf
- name: github.com/jarcoal/httpmock
version: 145b10d659265440f062c31ea15326166bae56ee
- name: github.com/libkermit/compose
@ -194,74 +193,74 @@ imports:
- name: github.com/mattn/go-shellwords
version: 525bedee691b5a8df547cb5cf9f86b7fb1883e24
- name: github.com/mesos/mesos-go
version: 068d5470506e3780189fe607af40892814197c5e
version: 2b28dad0aa75289407717b98d82c7871543bb4db
subpackages:
- mesosproto
- mesos
- upid
- mesosutil
- detector
- detector/zoo
- mesos
- mesosproto
- mesosutil
- upid
- name: github.com/mesosphere/mesos-dns
version: b47dc4c19f215e98da687b15b4c64e70f629bea5
repo: https://github.com/containous/mesos-dns.git
vcs: git
subpackages:
- detect
- errorutil
- logging
- models
- records
- records/labels
- records/state
- util
- errorutil
- models
- records/labels
- name: github.com/Microsoft/go-winio
version: ce2922f643c8fd76b46cadc7f404a06282678b34
- name: github.com/miekg/dns
version: 5d001d020961ae1c184f9f8152fdc73810481677
- name: github.com/mitchellh/mapstructure
version: 21a35fb16463dfb7c8eee579c65d995d95e64d1e
version: d2dd0262208475919e1a362f675cfc0e7c10e905
- name: github.com/moul/http2curl
version: b1479103caacaa39319f75e7f57fc545287fca0d
- name: github.com/ogier/pflag
version: 45c278ab3607870051a2ea9040bb85fcb8557481
- name: github.com/opencontainers/runc
version: ff88baa42fa5b2a1568a3a14665142fb4bdb3a2a
version: fb221651e5120cd287a76c7c1b6c877520fbd034
subpackages:
- libcontainer/user
- name: github.com/parnurzeal/gorequest
version: 045012d33ef41ea146c1b675df9296d0dc1a212d
version: 29ced6f360a5ac3823a3675b4e29fbab336cc186
- name: github.com/pmezard/go-difflib
version: d8ed2627bdf02c080bf22230dbb337003b7aba2d
version: 792786c7400a136282c1664665ae0a8db921c6c2
subpackages:
- difflib
- name: github.com/ryanuber/go-glob
version: 572520ed46dbddaed19ea3d9541bdd0494163693
- name: github.com/samuel/go-zookeeper
version: e64db453f3512cade908163702045e0f31137843
version: 87e1bca4477a3cc767ca71be023ced183d74e538
subpackages:
- zk
- name: github.com/Sirupsen/logrus
version: a283a10442df8dc09befd873fab202bf8a253d6a
version: 3ec0642a7fb6488f65b06f9040adc67e3990296a
- name: github.com/streamrail/concurrent-map
version: 65a174a3a4188c0b7099acbc6cfa0c53628d3287
version: 8bf1e9bacbf65b10c81d0f4314cf2b1ebef728b5
- name: github.com/stretchr/objx
version: cbeaeb16a013161a98496fad62933b1d21786672
version: 1a9d0bb9f541897e62256577b352fdbc1fb4fd94
- name: github.com/stretchr/testify
version: d77da356e56a7428ad25149ca77381849a6a5232
subpackages:
- mock
- assert
- mock
- name: github.com/thoas/stats
version: 79b768ff1780f4e5b0ed132e192bfeefe9f85a9c
- name: github.com/tv42/zbase32
version: 03389da7e0bf9844767f82690f4d68fc097a1306
version: 501572607d0273fc75b3b261fa4904d63f6ffa0e
- name: github.com/ugorji/go
version: b94837a2404ab90efe9289e77a70694c355739cb
version: 98ef79d6c615fa258e892ed0b11c6e013712693e
subpackages:
- codec
- name: github.com/unrolled/render
version: 198ad4d8b8a4612176b804ca10555b222a086b40
version: 3f4913244021dede105b62caecfb01f0c1eebeda
- name: github.com/vdemeester/docker-events
version: be74d4929ec1ad118df54349fda4b0cba60f849b
- name: github.com/vdemeester/shakers
@ -274,50 +273,51 @@ imports:
- cbreaker
- connlimit
- forward
- memmetrics
- roundrobin
- stream
- utils
- memmetrics
- name: github.com/vulcand/predicate
version: 19b9dde14240d94c804ae5736ad0e1de10bf8fe6
- name: github.com/vulcand/route
version: cb89d787ddbb1c5849a7ac9f79004c1fd12a4a32
version: 61904570391bdf22252f8e376b49a57593d9124c
- name: github.com/vulcand/vulcand
version: 28a4e5c0892167589737b95ceecbcef00295be50
version: 7b479b160401beccbc9521b922588fa2a64c4818
subpackages:
- plugin/rewrite
- plugin
- conntracker
- plugin
- plugin/rewrite
- router
- name: github.com/xenolf/lego
version: b2fad6198110326662e9e356a97199078a4a775c
subpackages:
- acme
- name: golang.org/x/crypto
version: d81fdb778bf2c40a91b24519d60cdc5767318829
version: 81372b2fc2f10bef2a7f338da115c315a56b2726
subpackages:
- ocsp
- bcrypt
- blowfish
- ocsp
- name: golang.org/x/net
version: b400c2eff1badec7022a8c8f5bea058b6315eed7
version: 71a035914f99bb58fe82eac0f1289f10963d876c
subpackages:
- context
- publicsuffix
- proxy
- publicsuffix
- name: golang.org/x/sys
version: 62bee037599929a6e9146f29d10dd5208c43507d
version: 8f0908ab3b2457e2e15403d3697c9ef5cb4b57a9
subpackages:
- unix
- windows
- name: gopkg.in/fsnotify.v1
version: a8a77c9133d2d6fd8334f3260d06f60e8d80a5fb
- name: gopkg.in/mgo.v2
version: 29cc868a5ca65f401ff318143f9408d02f4799cc
version: 3f83fa5005286a7fe593b055f0d7771a7dce4655
subpackages:
- bson
- internal/json
- name: gopkg.in/square/go-jose.v1
version: e3f973b66b91445ec816dd7411ad1b6495a5a2fc
version: 139276ceb5afbf13e636c44e9382f0ca75c12ba3
subpackages:
- cipher
- json
@ -331,17 +331,17 @@ testImports:
- name: github.com/flynn/go-shlex
version: 3f9db97f856818214da2e1057f8ad84803971cff
- name: github.com/gorilla/mux
version: 9fa818a44c2bf1396a17f9d5a3c0f6dd39d2ff8e
version: 0a192a193177452756c362c20087ddafcf6829c4
- name: github.com/libkermit/docker-check
version: cbe0ef03b3d23070eac4d00ba8828f2cc7f7e5a3
- name: github.com/spf13/pflag
version: 5644820622454e71517561946e3d94b9f9db6842
version: c7e63cf4530bcd3ba943729cee0efeff2ebea63f
- name: github.com/vbatts/tar-split
version: 6810cedb21b2c3d0b9bb8f9af12ff2dc7a2f14df
subpackages:
- archive/tar
- tar/asm
- tar/storage
- archive/tar
- name: github.com/xeipuuv/gojsonpointer
version: e0fe6f68307607d540ed8eac07a342c33fa1b54a
- name: github.com/xeipuuv/gojsonreference

View file

@ -5,7 +5,7 @@ import:
subpackages:
- fun
- package: github.com/Sirupsen/logrus
- package: github.com/emilevauge/backoff
- package: github.com/cenk/backoff
- package: github.com/codegangsta/negroni
- package: github.com/containous/flaeg
version: a731c034dda967333efce5f8d276aeff11f8ff87

View file

@ -2,7 +2,7 @@ package utils
import (
"errors"
"github.com/emilevauge/backoff"
"github.com/cenk/backoff"
"net/http"
"strconv"
"time"

39
job/job.go Normal file
View file

@ -0,0 +1,39 @@
package job
import (
"github.com/cenk/backoff"
"time"
)
var (
_ backoff.BackOff = (*BackOff)(nil)
)
const (
defaultMinJobInterval = 30 * time.Second
)
// BackOff is an exponential backoff implementation for long running jobs.
// In long running jobs, an operation() that fails after a long Duration should not increments the backoff period.
// If operation() takes more than MinJobInterval, Reset() is called in NextBackOff().
type BackOff struct {
*backoff.ExponentialBackOff
MinJobInterval time.Duration
}
// NewBackOff creates an instance of BackOff using default values.
func NewBackOff(backOff *backoff.ExponentialBackOff) *BackOff {
backOff.MaxElapsedTime = 0
return &BackOff{
ExponentialBackOff: backOff,
MinJobInterval: defaultMinJobInterval,
}
}
// NextBackOff calculates the next backoff interval.
func (b *BackOff) NextBackOff() time.Duration {
if b.GetElapsedTime() >= b.MinJobInterval {
b.Reset()
}
return b.ExponentialBackOff.NextBackOff()
}

44
job/job_test.go Normal file
View file

@ -0,0 +1,44 @@
package job
import (
"github.com/cenk/backoff"
"testing"
"time"
)
func TestJobBackOff(t *testing.T) {
var (
testInitialInterval = 500 * time.Millisecond
testRandomizationFactor = 0.1
testMultiplier = 2.0
testMaxInterval = 5 * time.Second
testMinJobInterval = 1 * time.Second
)
exp := NewBackOff(backoff.NewExponentialBackOff())
exp.InitialInterval = testInitialInterval
exp.RandomizationFactor = testRandomizationFactor
exp.Multiplier = testMultiplier
exp.MaxInterval = testMaxInterval
exp.MinJobInterval = testMinJobInterval
exp.Reset()
var expectedResults = []time.Duration{500, 500, 500, 1000, 2000, 4000, 5000, 5000, 500, 1000, 2000, 4000, 5000, 5000}
for i, d := range expectedResults {
expectedResults[i] = d * time.Millisecond
}
for i, expected := range expectedResults {
// Assert that the next backoff falls in the expected range.
var minInterval = expected - time.Duration(testRandomizationFactor*float64(expected))
var maxInterval = expected + time.Duration(testRandomizationFactor*float64(expected))
if i < 3 || i == 8 {
time.Sleep(2 * time.Second)
}
var actualInterval = exp.NextBackOff()
if !(minInterval <= actualInterval && actualInterval <= maxInterval) {
t.Error("error")
}
// assertEquals(t, expected, exp.currentInterval)
}
}

View file

@ -10,9 +10,10 @@ import (
"github.com/BurntSushi/ty/fun"
log "github.com/Sirupsen/logrus"
"github.com/cenk/backoff"
"github.com/containous/traefik/job"
"github.com/containous/traefik/safe"
"github.com/containous/traefik/types"
"github.com/emilevauge/backoff"
"github.com/hashicorp/consul/api"
)
@ -330,7 +331,7 @@ func (provider *ConsulCatalog) Provide(configurationChan chan<- types.ConfigMess
operation := func() error {
return provider.watch(configurationChan, stop)
}
err := backoff.RetryNotify(operation, backoff.NewJobBackOff(backoff.NewExponentialBackOff()), notify)
err := backoff.RetryNotify(operation, job.NewBackOff(backoff.NewExponentialBackOff()), notify)
if err != nil {
log.Errorf("Cannot connect to consul server %+v", err)
}

View file

@ -14,6 +14,8 @@ import (
"github.com/BurntSushi/ty/fun"
log "github.com/Sirupsen/logrus"
"github.com/cenk/backoff"
"github.com/containous/traefik/job"
"github.com/containous/traefik/safe"
"github.com/containous/traefik/types"
"github.com/containous/traefik/version"
@ -26,7 +28,6 @@ import (
swarmtypes "github.com/docker/engine-api/types/swarm"
"github.com/docker/go-connections/nat"
"github.com/docker/go-connections/sockets"
"github.com/emilevauge/backoff"
"github.com/vdemeester/docker-events"
)
@ -223,7 +224,7 @@ func (provider *Docker) Provide(configurationChan chan<- types.ConfigMessage, po
notify := func(err error, time time.Duration) {
log.Errorf("Docker connection error %+v, retrying in %s", err, time)
}
err := backoff.RetryNotify(operation, backoff.NewJobBackOff(backoff.NewExponentialBackOff()), notify)
err := backoff.RetryNotify(operation, job.NewBackOff(backoff.NewExponentialBackOff()), notify)
if err != nil {
log.Errorf("Cannot connect to docker server %+v", err)
}

View file

@ -11,10 +11,11 @@ import (
"time"
log "github.com/Sirupsen/logrus"
"github.com/cenk/backoff"
"github.com/containous/traefik/job"
"github.com/containous/traefik/provider/k8s"
"github.com/containous/traefik/safe"
"github.com/containous/traefik/types"
"github.com/emilevauge/backoff"
)
const (
@ -146,7 +147,7 @@ func (provider *Kubernetes) Provide(configurationChan chan<- types.ConfigMessage
notify := func(err error, time time.Duration) {
log.Errorf("Kubernetes connection error %+v, retrying in %s", err, time)
}
err := backoff.RetryNotify(operation, backoff.NewJobBackOff(backoff.NewExponentialBackOff()), notify)
err := backoff.RetryNotify(operation, job.NewBackOff(backoff.NewExponentialBackOff()), notify)
if err != nil {
log.Errorf("Cannot connect to Kubernetes server %+v", err)
}

View file

@ -10,11 +10,12 @@ import (
"errors"
"github.com/BurntSushi/ty/fun"
log "github.com/Sirupsen/logrus"
"github.com/cenk/backoff"
"github.com/containous/traefik/job"
"github.com/containous/traefik/safe"
"github.com/containous/traefik/types"
"github.com/docker/libkv"
"github.com/docker/libkv/store"
"github.com/emilevauge/backoff"
)
// Kv holds common configurations of key-value providers.
@ -75,7 +76,7 @@ func (provider *Kv) watchKv(configurationChan chan<- types.ConfigMessage, prefix
notify := func(err error, time time.Duration) {
log.Errorf("KV connection error: %+v, retrying in %s", err, time)
}
err := backoff.RetryNotify(operation, backoff.NewJobBackOff(backoff.NewExponentialBackOff()), notify)
err := backoff.RetryNotify(operation, job.NewBackOff(backoff.NewExponentialBackOff()), notify)
if err != nil {
return fmt.Errorf("Cannot connect to KV server: %v", err)
}
@ -106,7 +107,7 @@ func (provider *Kv) provide(configurationChan chan<- types.ConfigMessage, pool *
notify := func(err error, time time.Duration) {
log.Errorf("KV connection error: %+v, retrying in %s", err, time)
}
err := backoff.RetryNotify(operation, backoff.NewJobBackOff(backoff.NewExponentialBackOff()), notify)
err := backoff.RetryNotify(operation, job.NewBackOff(backoff.NewExponentialBackOff()), notify)
if err != nil {
return fmt.Errorf("Cannot connect to KV server: %v", err)
}

View file

@ -14,9 +14,10 @@ import (
"github.com/BurntSushi/ty/fun"
log "github.com/Sirupsen/logrus"
"github.com/cenk/backoff"
"github.com/containous/traefik/job"
"github.com/containous/traefik/safe"
"github.com/containous/traefik/types"
"github.com/emilevauge/backoff"
"github.com/gambol99/go-marathon"
)
@ -110,7 +111,7 @@ func (provider *Marathon) Provide(configurationChan chan<- types.ConfigMessage,
notify := func(err error, time time.Duration) {
log.Errorf("Marathon connection error %+v, retrying in %s", err, time)
}
err := backoff.RetryNotify(operation, backoff.NewJobBackOff(backoff.NewExponentialBackOff()), notify)
err := backoff.RetryNotify(operation, job.NewBackOff(backoff.NewExponentialBackOff()), notify)
if err != nil {
log.Errorf("Cannot connect to Marathon server %+v", err)
}

View file

@ -9,9 +9,10 @@ import (
"fmt"
"github.com/BurntSushi/ty/fun"
log "github.com/Sirupsen/logrus"
"github.com/cenk/backoff"
"github.com/containous/traefik/job"
"github.com/containous/traefik/safe"
"github.com/containous/traefik/types"
"github.com/emilevauge/backoff"
"github.com/mesos/mesos-go/detector"
_ "github.com/mesos/mesos-go/detector/zoo" // Registers the ZK detector
"github.com/mesosphere/mesos-dns/detect"
@ -110,7 +111,7 @@ func (provider *Mesos) Provide(configurationChan chan<- types.ConfigMessage, poo
notify := func(err error, time time.Duration) {
log.Errorf("mesos connection error %+v, retrying in %s", err, time)
}
err := backoff.RetryNotify(operation, backoff.NewJobBackOff(backoff.NewExponentialBackOff()), notify)
err := backoff.RetryNotify(operation, job.NewBackOff(backoff.NewExponentialBackOff()), notify)
if err != nil {
log.Errorf("Cannot connect to mesos server %+v", err)
}