baalajimaestro/traefik
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

Add leadership election

Browse source 
Signed-off-by: Emile Vauge <emile@vauge.com>
This commit is contained in:
Emile Vauge 2016-08-18 13:03:10 +02:00
parent 5a0440d6f8
commit bea5ad3f13
No known key found for this signature in database
GPG key ID: D808B4C167352E59
7 changed files with 138 additions and 29 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
cluster/datastore.go
View file

@ -94,8 +94,8 @@ func (d *Datastore) watchChanges() error {
// Begin creates a transaction with the KV store.
func (d *Datastore) Begin() (*Transaction, error) {
value := uuid.NewV4().String()
remoteLock, err := d.kv.NewLock(d.lockKey, &store.LockOptions{TTL: 20 * time.Second, Value: []byte(value)})
id := uuid.NewV4().String()
remoteLock, err := d.kv.NewLock(d.lockKey, &store.LockOptions{TTL: 20 * time.Second, Value: []byte(id)})
if err != nil {
return nil, err
}
@ -119,8 +119,8 @@ func (d *Datastore) Begin() (*Transaction, error) {
// we got the lock! Now make sure we are synced with KV store
operation := func() error {
meta := d.get()
if meta.Lock != value {
return fmt.Errorf("Object lock value: expected %s, got %s", value, meta.Lock)
if meta.Lock != id {
return fmt.Errorf("Object lock value: expected %s, got %s", id, meta.Lock)
}
return nil
}

44
cluster/leadership.go
View file

@ -6,22 +6,34 @@ import (
"github.com/containous/traefik/safe"
"github.com/containous/traefik/types"
"github.com/docker/leadership"
"golang.org/x/net/context"
"time"
)
// Leadership allows leadership election using a KV store
type Leadership struct {
types.Cluster
*safe.Pool
*types.Cluster
candidate *leadership.Candidate
}
// NewLeadership creates a leadership
func NewLeadership(ctx context.Context, cluster *types.Cluster) *Leadership {
return &Leadership{
Pool: safe.NewPool(ctx),
Cluster: cluster,
candidate: leadership.NewCandidate(cluster.Store, cluster.Store.Prefix+"/leader", cluster.Node, 20*time.Second),
}
}
// Participate tries to be a leader
func (l *Leadership) Participate(pool *safe.Pool, isElected func(bool)) {
pool.Go(func(stop chan bool) {
l.candidate = leadership.NewCandidate(l.Store, l.Store.Prefix+"/leader", l.Node, 30*time.Second)
func (l *Leadership) Participate(pool *safe.Pool) {
pool.GoCtx(func(ctx context.Context) {
log.Debugf("Node %s running for election", l.Cluster.Node)
defer log.Debugf("Node %s no more running for election", l.Cluster.Node)
backOff := backoff.NewExponentialBackOff()
operation := func() error {
return l.run(l.candidate, stop, isElected)
return l.run(l.candidate, ctx)
}
notify := func(err error, time time.Duration) {
@ -36,21 +48,31 @@ func (l *Leadership) Participate(pool *safe.Pool, isElected func(bool)) {
// Resign resigns from being a leader
func (l *Leadership) Resign() {
if l.candidate != nil {
l.candidate.Resign()
}
l.candidate.Resign()
log.Infof("Node %s resined", l.Cluster.Node)
}
func (l *Leadership) run(candidate *leadership.Candidate, stop chan bool, isElected func(bool)) error {
func (l *Leadership) run(candidate *leadership.Candidate, ctx context.Context) error {
electedCh, errCh := candidate.RunForElection()
for {
select {
case elected := <-electedCh:
isElected(elected)
l.onElection(elected)
case err := <-errCh:
return err
case <-stop:
case <-ctx.Done():
l.candidate.Resign()
return nil
}
}
}
func (l *Leadership) onElection(elected bool) {
if elected {
log.Infof("Node %s elected leader ♚", l.Cluster.Node)
l.Start()
} else {
log.Infof("Node %s elected slave ♝", l.Cluster.Node)
l.Stop()
}
}

10
provider/kv.go
View file

@ -148,7 +148,7 @@ func (provider *Kv) list(keys ...string) []string {
joinedKeys := strings.Join(keys, "")
keysPairs, err := provider.kvclient.List(joinedKeys)
if err != nil {
log.Errorf("Error getting keys %s %s ", joinedKeys, err)
log.Warnf("Cannot get keys %s %s ", joinedKeys, err)
return nil
}
directoryKeys := make(map[string]string)
@ -170,10 +170,10 @@ func (provider *Kv) get(defaultValue string, keys ...string) string {
joinedKeys := strings.Join(keys, "")
keyPair, err := provider.kvclient.Get(strings.TrimPrefix(joinedKeys, "/"))
if err != nil {
log.Warnf("Error getting key %s %s, setting default %s", joinedKeys, err, defaultValue)
log.Warnf("Cannot get key %s %s, setting default %s", joinedKeys, err, defaultValue)
return defaultValue
} else if keyPair == nil {
log.Warnf("Error getting key %s, setting default %s", joinedKeys, defaultValue)
log.Warnf("Cannot get key %s, setting default %s", joinedKeys, defaultValue)
return defaultValue
}
return string(keyPair.Value)
@ -183,10 +183,10 @@ func (provider *Kv) splitGet(keys ...string) []string {
joinedKeys := strings.Join(keys, "")
keyPair, err := provider.kvclient.Get(joinedKeys)
if err != nil {
log.Warnf("Error getting key %s %s, setting default empty", joinedKeys, err)
log.Warnf("Cannot get key %s %s, setting default empty", joinedKeys, err)
return []string{}
} else if keyPair == nil {
log.Warnf("Error getting key %s, setting default %empty", joinedKeys)
log.Warnf("Cannot get key %s, setting default %empty", joinedKeys)
return []string{}
}
return strings.Split(string(keyPair.Value), ",")

66
safe/routine.go
View file

@ -1,6 +1,7 @@
package safe
import (
"golang.org/x/net/context"
"log"
"runtime/debug"
"sync"
@ -11,11 +12,44 @@ type routine struct {
stop chan bool
}
// Pool creates a pool of go routines
type routineCtx func(ctx context.Context)
// Pool is a pool of go routines
type Pool struct {
routines []routine
waitGroup sync.WaitGroup
lock sync.Mutex
routines []routine
routinesCtx []routineCtx
waitGroup sync.WaitGroup
lock sync.Mutex
baseCtx context.Context
ctx context.Context
cancel context.CancelFunc
}
// NewPool creates a Pool
func NewPool(baseCtx context.Context) *Pool {
ctx, cancel := context.WithCancel(baseCtx)
return &Pool{
ctx: ctx,
cancel: cancel,
baseCtx: baseCtx,
}
}
// Ctx returns main context
func (p *Pool) Ctx() context.Context {
return p.ctx
}
//GoCtx starts a recoverable goroutine with a context
func (p *Pool) GoCtx(goroutine routineCtx) {
p.lock.Lock()
p.routinesCtx = append(p.routinesCtx, goroutine)
p.waitGroup.Add(1)
Go(func() {
goroutine(p.ctx)
p.waitGroup.Done()
})
p.lock.Unlock()
}
// Go starts a recoverable goroutine, and can be stopped with stop chan
@ -37,6 +71,7 @@ func (p *Pool) Go(goroutine func(stop chan bool)) {
// Stop stops all started routines, waiting for their termination
func (p *Pool) Stop() {
p.lock.Lock()
p.cancel()
for _, routine := range p.routines {
routine.stop <- true
}
@ -47,6 +82,29 @@ func (p *Pool) Stop() {
p.lock.Unlock()
}
// Start starts all stoped routines
func (p *Pool) Start() {
p.lock.Lock()
p.ctx, p.cancel = context.WithCancel(p.baseCtx)
for _, routine := range p.routines {
p.waitGroup.Add(1)
routine.stop = make(chan bool, 1)
Go(func() {
routine.goroutine(routine.stop)
p.waitGroup.Done()
})
}
for _, routine := range p.routinesCtx {
p.waitGroup.Add(1)
Go(func() {
routine(p.ctx)
p.waitGroup.Done()
})
}
p.lock.Unlock()
}
// Go starts a recoverable goroutine
func Go(goroutine func()) {
GoWithRecover(goroutine, defaultRecoverGoroutine)

30
server.go
View file

@ -24,6 +24,7 @@ import (
log "github.com/Sirupsen/logrus"
"github.com/codegangsta/negroni"
"github.com/containous/mux"
"github.com/containous/traefik/cluster"
"github.com/containous/traefik/middlewares"
"github.com/containous/traefik/provider"
"github.com/containous/traefik/safe"
@ -50,7 +51,8 @@ type Server struct {
currentConfigurations safe.Safe
globalConfiguration GlobalConfiguration
loggerMiddleware *middlewares.Logger
routinesPool safe.Pool
routinesPool *safe.Pool
leadership *cluster.Leadership
}
type serverEntryPoints map[string]*serverEntryPoint
@ -80,12 +82,18 @@ func NewServer(globalConfiguration GlobalConfiguration) *Server {
server.currentConfigurations.Set(currentConfigurations)
server.globalConfiguration = globalConfiguration
server.loggerMiddleware = middlewares.NewLogger(globalConfiguration.AccessLogsFile)
server.routinesPool = safe.NewPool(context.Background())
if globalConfiguration.Cluster != nil {
// leadership creation if cluster mode
server.leadership = cluster.NewLeadership(server.routinesPool.Ctx(), globalConfiguration.Cluster)
}
return server
}
// Start starts the server and blocks until server is shutted down.
func (server *Server) Start() {
server.startLeadership()
server.startHTTPServers()
server.routinesPool.Go(func(stop chan bool) {
server.listenProviders(stop)
@ -125,6 +133,7 @@ func (server *Server) Close() {
os.Exit(1)
}
}(ctx)
server.stopLeadership()
server.routinesPool.Stop()
close(server.configurationChan)
close(server.configurationValidatedChan)
@ -135,6 +144,23 @@ func (server *Server) Close() {
cancel()
}
func (server *Server) startLeadership() {
if server.leadership != nil {
server.leadership.Participate(server.routinesPool)
server.leadership.GoCtx(func(ctx context.Context) {
log.Debugf("Started test routine")
<-ctx.Done()
log.Debugf("Stopped test routine")
})
}
}
func (server *Server) stopLeadership() {
if server.leadership != nil {
server.leadership.Resign()
}
}
func (server *Server) startHTTPServers() {
server.serverEntryPoints = server.buildEntryPoints(server.globalConfiguration)
for newServerEntryPointName, newServerEntryPoint := range server.serverEntryPoints {
@ -321,7 +347,7 @@ func (server *Server) startProviders() {
log.Infof("Starting provider %v %s", reflect.TypeOf(provider), jsonConf)
currentProvider := provider
safe.Go(func() {
err := currentProvider.Provide(server.configurationChan, &server.routinesPool, server.globalConfiguration.Constraints)
err := currentProvider.Provide(server.configurationChan, server.routinesPool, server.globalConfiguration.Constraints)
if err != nil {
log.Errorf("Error starting provider %s", err)
}

7
traefik.go
View file

@ -21,6 +21,7 @@ import (
"github.com/containous/traefik/types"
"github.com/containous/traefik/version"
"github.com/docker/libkv/store"
"github.com/satori/go.uuid"
)
var versionTemplate = `Version: {{.Version}}
@ -155,8 +156,10 @@ Complete documentation is available at https://traefik.io`,
// IF a KV Store is enable and no sub-command called in args
if kv != nil && usedCmd == traefikCmd {
if traefikConfiguration.Cluster == nil {
hostname, _ := os.Hostname()
traefikConfiguration.Cluster = &types.Cluster{Store: types.Store{Prefix: kv.Prefix, Store: kv.Store}, Node: hostname}
traefikConfiguration.Cluster = &types.Cluster{Node: uuid.NewV4().String()}
}
if traefikConfiguration.Cluster.Store == nil {
traefikConfiguration.Cluster.Store = &types.Store{Prefix: kv.Prefix, Store: kv.Store}
}
s.AddSource(kv)
if _, err := s.LoadConfig(); err != nil {

2
types/types.go
View file

@ -202,7 +202,7 @@ type Store struct {
// Cluster holds cluster config
type Cluster struct {
Node string
Store Store
Store *Store
}
// Auth holds authentication configuration (BASIC, DIGEST, users)