Merge pull request #24 from EmileVauge/zero-downtime-reload

Zero downtime reload
This commit is contained in:
Vincent Demeester 2015-09-28 19:18:51 +01:00
commit 4fb25ca358
3 changed files with 78 additions and 58 deletions

14
Godeps/Godeps.json generated
View file

@ -69,6 +69,11 @@
"ImportPath": "github.com/mailgun/log",
"Rev": "44874009257d4d47ba9806f1b7f72a32a015e4d8"
},
{
"ImportPath": "github.com/mailgun/manners",
"Comment": "0.3.1-30-g37136f7",
"Rev": "37136f736785d7c6aa3b9a27b4b2dd1028ca6d79"
},
{
"ImportPath": "github.com/mailgun/oxy/cbreaker",
"Rev": "547c334d658398c05b346c0b79d8f47ba2e1473b"
@ -101,19 +106,10 @@
"ImportPath": "github.com/thoas/stats",
"Rev": "54ed61c2b47e263ae2f01b86837b0c4bd1da28e8"
},
{
"ImportPath": "github.com/tylerb/graceful",
"Comment": "v1.2.1",
"Rev": "ac9ebe4f1ee151ac1eeeaef32957085cba64d508"
},
{
"ImportPath": "github.com/unrolled/render",
"Rev": "26b4e3aac686940fe29521545afad9966ddfc80c"
},
{
"ImportPath": "golang.org/x/net/netutil",
"Rev": "d9558e5c97f85372afee28cf2b6059d7d3818919"
},
{
"ImportPath": "gopkg.in/alecthomas/kingpin.v2",
"Comment": "v2.0.12",

View file

@ -38,7 +38,7 @@ Here is a demo of Træfɪk using Docker backend, showing a load-balancing betwee
* [Oxy](https://github.com/mailgun/oxy/): an awsome proxy library made by Mailgun guys
* [Gorilla mux](https://github.com/gorilla/mux): famous request router
* [Negroni](https://github.com/codegangsta/negroni): web middlewares made simple
* [Graceful](https://github.com/tylerb/graceful): graceful shutdown of http.Handler servers
* [Manners](https://github.com/mailgun/manners): graceful shutdown of http.Handler servers
# Quick start

View file

@ -17,13 +17,14 @@ import (
"github.com/codegangsta/negroni"
"github.com/emilevauge/traefik/middlewares"
"github.com/gorilla/mux"
"github.com/mailgun/manners"
"github.com/mailgun/oxy/cbreaker"
"github.com/mailgun/oxy/forward"
"github.com/mailgun/oxy/roundrobin"
"github.com/thoas/stats"
"github.com/tylerb/graceful"
"github.com/unrolled/render"
"gopkg.in/alecthomas/kingpin.v2"
"runtime"
)
var (
@ -39,14 +40,18 @@ var (
)
func main() {
runtime.GOMAXPROCS(runtime.NumCPU())
kingpin.Parse()
fmtlog.SetFlags(fmtlog.Lshortfile | fmtlog.LstdFlags)
var srv *graceful.Server
var srv *manners.GracefulServer
var configurationRouter *mux.Router
var configurationChan = make(chan *Configuration, 10)
defer close(configurationChan)
var providers = []Provider{}
var sigs = make(chan os.Signal, 1)
defer close(sigs)
var stopChan = make(chan bool)
defer close(stopChan)
var providers = []Provider{}
signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
// load global configuration
@ -91,8 +96,15 @@ func main() {
if err == nil {
currentConfiguration = configuration
configurationRouter = newConfigurationRouter
srv.Stop(time.Duration(globalConfiguration.GraceTimeOut) * time.Second)
time.Sleep(3 * time.Second)
oldServer := srv
newsrv := prepareServer(configurationRouter, globalConfiguration, oldServer, loggerMiddleware, metrics)
go startServer(newsrv, globalConfiguration)
srv = newsrv
time.Sleep(2 * time.Second)
if oldServer != nil {
log.Info("Stopping old server")
oldServer.Close()
}
} else {
log.Error("Error loading new configuration, aborted ", err)
}
@ -130,59 +142,71 @@ func main() {
}()
}
goAway := false
go func() {
sig := <-sigs
log.Infof("I have to go... %+v", sig)
goAway = true
srv.Stop(time.Duration(globalConfiguration.GraceTimeOut) * time.Second)
log.Info("Stopping server")
srv.Close()
stopChan <- true
}()
for {
if goAway {
break
//negroni.Use(middlewares.NewCircuitBreaker(oxyLogger))
//negroni.Use(middlewares.NewRoutes(configurationRouter))
srv = prepareServer(configurationRouter, globalConfiguration, nil, loggerMiddleware, metrics)
go startServer(srv, globalConfiguration)
<-stopChan
log.Info("Shutting down")
}
func startServer(srv *manners.GracefulServer, globalConfiguration *GlobalConfiguration) {
log.Info("Starting server")
if len(globalConfiguration.CertFile) > 0 && len(globalConfiguration.KeyFile) > 0 {
err := srv.ListenAndServeTLS(globalConfiguration.CertFile, globalConfiguration.KeyFile)
if err != nil {
netOpError, ok := err.(*net.OpError)
if ok && netOpError.Err.Error() != "use of closed network connection" {
log.Fatal("Error creating server: ", err)
}
}
} else {
err := srv.ListenAndServe()
if err != nil {
netOpError, ok := err.(*net.OpError)
if ok && netOpError.Err.Error() != "use of closed network connection" {
log.Fatal("Error creating server: ", err)
}
}
}
log.Info("Server stopped")
}
// middlewares
var negroni = negroni.New()
negroni.Use(metrics)
negroni.Use(loggerMiddleware)
//negroni.Use(middlewares.NewCircuitBreaker(oxyLogger))
//negroni.Use(middlewares.NewRoutes(configurationRouter))
negroni.UseHandler(configurationRouter)
func prepareServer(router *mux.Router, globalConfiguration *GlobalConfiguration, oldServer *manners.GracefulServer, middlewares ...negroni.Handler) *manners.GracefulServer {
log.Info("Preparing server")
// middlewares
var negroni = negroni.New()
for _, middleware := range middlewares {
negroni.Use(middleware)
}
negroni.UseHandler(router)
srv = &graceful.Server{
Timeout: time.Duration(globalConfiguration.GraceTimeOut) * time.Second,
NoSignalHandling: true,
Server: &http.Server{
if oldServer == nil {
return manners.NewWithServer(
&http.Server{
Addr: globalConfiguration.Port,
Handler: negroni,
},
})
} else {
server, err := oldServer.HijackListener(&http.Server{
Addr: globalConfiguration.Port,
Handler: negroni,
}, nil)
if err != nil {
log.Fatalf("Error hijacking server %s", err)
return nil
} else {
return server
}
go func() {
if len(globalConfiguration.CertFile) > 0 && len(globalConfiguration.KeyFile) > 0 {
err := srv.ListenAndServeTLS(globalConfiguration.CertFile, globalConfiguration.KeyFile)
if err != nil {
netOpError, ok := err.(*net.OpError)
if ok && netOpError.Err.Error() != "use of closed network connection" {
log.Fatal("Error creating server: ", err)
}
}
} else {
err := srv.ListenAndServe()
if err != nil {
netOpError, ok := err.(*net.OpError)
if ok && netOpError.Err.Error() != "use of closed network connection" {
log.Fatal("Error creating server: ", err)
}
}
}
}()
log.Info("Started")
<-srv.StopChan()
log.Info("Stopped")
}
}
@ -215,7 +239,7 @@ func LoadConfig(configuration *Configuration, globalConfiguration *GlobalConfigu
} else {
log.Debugf("Reusing backend %s", frontend.Backend)
}
// stream.New(backends[frontend.Backend], stream.Retry("IsNetworkError() && Attempts() <= " + strconv.Itoa(globalConfiguration.Replay)), stream.Logger(oxyLogger))
// stream.New(backends[frontend.Backend], stream.Retry("IsNetworkError() && Attempts() <= " + strconv.Itoa(globalConfiguration.Replay)), stream.Logger(oxyLogger))
var negroni = negroni.New()
negroni.Use(middlewares.NewCircuitBreaker(backends[frontend.Backend], cbreaker.Logger(oxyLogger)))
newRoute.Handler(negroni)