From 0ab0bdf818c43e5fab2e11e4d90d730a6afcc042 Mon Sep 17 00:00:00 2001 From: Julien Salleyron Date: Sat, 26 Nov 2016 19:48:49 +0100 Subject: [PATCH 1/6] feat: HealthCheck --- healthcheck/healthcheck.go | 94 ++++++++++++++++++++++++++++++++++++++ server.go | 11 +++++ types/types.go | 6 +++ 3 files changed, 111 insertions(+) create mode 100644 healthcheck/healthcheck.go diff --git a/healthcheck/healthcheck.go b/healthcheck/healthcheck.go new file mode 100644 index 000000000..66c31c9ce --- /dev/null +++ b/healthcheck/healthcheck.go @@ -0,0 +1,94 @@ +package healthcheck + +import ( + "github.com/containous/traefik/log" + "github.com/vulcand/oxy/roundrobin" + "net/http" + "net/url" + "sync" + "time" +) + +var singleton *HealthCheck +var once sync.Once + +// GetHealthCheck Get HealtchCheck Singleton +func GetHealthCheck() *HealthCheck { + once.Do(func() { + singleton = newHealthCheck() + singleton.execute() + }) + return singleton +} + +// BackendHealthCheck HealthCheck configuration for a backend +type BackendHealthCheck struct { + URL string + DisabledURLs []*url.URL + lb loadBalancer +} + +var launch = false + +//HealthCheck struct +type HealthCheck struct { + Backends map[string]*BackendHealthCheck +} + +type loadBalancer interface { + RemoveServer(u *url.URL) error + UpsertServer(u *url.URL, options ...roundrobin.ServerOption) error + Servers() []*url.URL +} + +func newHealthCheck() *HealthCheck { + return &HealthCheck{make(map[string]*BackendHealthCheck)} +} + +// NewBackendHealthCheck Instantiate a new BackendHealthCheck +func NewBackendHealthCheck(URL string, lb loadBalancer) *BackendHealthCheck { + return &BackendHealthCheck{URL, nil, lb} +} + +//SetBackends set backends configuration +func (hc *HealthCheck) SetBackendsConfiguration(backends map[string]*BackendHealthCheck) { + hc.Backends = backends +} + +func (hc *HealthCheck) execute() { + ticker := time.NewTicker(time.Second * 30) + go func() { + for t := range ticker.C { + for backendID, backend := range hc.Backends { + log.Debugf("Refreshing Healthcheck %s for backend %s ", t.String(), backendID) + enabledURLs := backend.lb.Servers() + var newDisabledURLs []*url.URL + for _, url := range backend.DisabledURLs { + if testHealth(url, backend.URL) { + log.Debugf("Upsert %s", url.String()) + backend.lb.UpsertServer(url, roundrobin.Weight(1)) + } else { + newDisabledURLs = append(newDisabledURLs, url) + } + } + backend.DisabledURLs = newDisabledURLs + + for _, url := range enabledURLs { + if !testHealth(url, backend.URL) { + log.Debugf("Remove %s", url.String()) + backend.lb.RemoveServer(url) + backend.DisabledURLs = append(backend.DisabledURLs, url) + } + } + } + } + }() +} + +func testHealth(serverURL *url.URL, checkURL string) bool { + resp, err := http.Get(serverURL.String() + checkURL) + if err != nil || resp.StatusCode != 200 { + return false + } + return true +} diff --git a/server.go b/server.go index 3694a65b0..b968a30bf 100644 --- a/server.go +++ b/server.go @@ -23,6 +23,7 @@ import ( "github.com/codegangsta/negroni" "github.com/containous/mux" "github.com/containous/traefik/cluster" + "github.com/containous/traefik/healthcheck" "github.com/containous/traefik/log" "github.com/containous/traefik/middlewares" "github.com/containous/traefik/provider" @@ -551,6 +552,9 @@ func (server *Server) loadConfig(configurations configs, globalConfiguration Glo redirectHandlers := make(map[string]http.Handler) backends := map[string]http.Handler{} + + backendsHealcheck := map[string]*healthcheck.BackendHealthCheck{} + backend2FrontendMap := map[string]string{} for _, configuration := range configurations { frontendNames := sortedFrontendNamesForConfig(configuration) @@ -650,6 +654,9 @@ func (server *Server) loadConfig(configurations configs, globalConfiguration Glo log.Errorf("Skipping frontend %s...", frontendName) continue frontend } + if configuration.Backends[frontend.Backend].HealthCheck != nil { + backendsHealcheck[frontend.Backend] = healthcheck.NewBackendHealthCheck(configuration.Backends[frontend.Backend].HealthCheck.URL, rebalancer) + } } case types.Wrr: log.Debugf("Creating load-balancer wrr") @@ -673,6 +680,9 @@ func (server *Server) loadConfig(configurations configs, globalConfiguration Glo continue frontend } } + if configuration.Backends[frontend.Backend].HealthCheck != nil { + backendsHealcheck[frontend.Backend] = healthcheck.NewBackendHealthCheck(configuration.Backends[frontend.Backend].HealthCheck.URL, rr) + } } maxConns := configuration.Backends[frontend.Backend].MaxConn if maxConns != nil && maxConns.Amount != 0 { @@ -735,6 +745,7 @@ func (server *Server) loadConfig(configurations configs, globalConfiguration Glo } } } + healthcheck.GetHealthCheck().SetBackendsConfiguration(backendsHealcheck) middlewares.SetBackend2FrontendMap(&backend2FrontendMap) //sort routes for _, serverEntryPoint := range serverEntryPoints { diff --git a/types/types.go b/types/types.go index b2d764fd5..d3937e156 100644 --- a/types/types.go +++ b/types/types.go @@ -16,6 +16,7 @@ type Backend struct { CircuitBreaker *CircuitBreaker `json:"circuitBreaker,omitempty"` LoadBalancer *LoadBalancer `json:"loadBalancer,omitempty"` MaxConn *MaxConn `json:"maxConn,omitempty"` + HealthCheck *HealthCheck `json:"healthCheck,omitempty"` } // MaxConn holds maximum connection configuration @@ -35,6 +36,11 @@ type CircuitBreaker struct { Expression string `json:"expression,omitempty"` } +// HealthCheck holds healthchk configuration +type HealthCheck struct { + URL string `json:"url,omitempty"` +} + // Server holds server configuration. type Server struct { URL string `json:"url,omitempty"` From a0b775a7c0370af48098aece127ea8f2b47d5951 Mon Sep 17 00:00:00 2001 From: Julien Salleyron Date: Tue, 29 Nov 2016 19:30:51 +0100 Subject: [PATCH 2/6] Lint on healthcheck --- healthcheck/healthcheck.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/healthcheck/healthcheck.go b/healthcheck/healthcheck.go index 66c31c9ce..72a14bcc7 100644 --- a/healthcheck/healthcheck.go +++ b/healthcheck/healthcheck.go @@ -50,7 +50,7 @@ func NewBackendHealthCheck(URL string, lb loadBalancer) *BackendHealthCheck { return &BackendHealthCheck{URL, nil, lb} } -//SetBackends set backends configuration +//SetBackendsConfiguration set backends configuration func (hc *HealthCheck) SetBackendsConfiguration(backends map[string]*BackendHealthCheck) { hc.Backends = backends } From 4a8f0323048c4ee35b221037ba1b82666a6f7836 Mon Sep 17 00:00:00 2001 From: Julien Salleyron Date: Wed, 30 Nov 2016 22:48:09 +0100 Subject: [PATCH 3/6] feat: timeout on check --- healthcheck/healthcheck.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/healthcheck/healthcheck.go b/healthcheck/healthcheck.go index 72a14bcc7..a8b0de4fc 100644 --- a/healthcheck/healthcheck.go +++ b/healthcheck/healthcheck.go @@ -86,7 +86,11 @@ func (hc *HealthCheck) execute() { } func testHealth(serverURL *url.URL, checkURL string) bool { - resp, err := http.Get(serverURL.String() + checkURL) + timeout := time.Duration(5 * time.Second) + client := http.Client{ + Timeout: timeout, + } + resp, err := client.Get(serverURL.String() + checkURL) if err != nil || resp.StatusCode != 200 { return false } From 99ffc26d4082d84c751cd9085f30fe2c1311a7ce Mon Sep 17 00:00:00 2001 From: Julien Salleyron Date: Wed, 30 Nov 2016 22:49:57 +0100 Subject: [PATCH 4/6] One goroutine for each backend + fix typo --- healthcheck/healthcheck.go | 64 +++++++++++++++++++++++--------------- server.go | 8 ++--- 2 files changed, 43 insertions(+), 29 deletions(-) diff --git a/healthcheck/healthcheck.go b/healthcheck/healthcheck.go index a8b0de4fc..ebf0cac8a 100644 --- a/healthcheck/healthcheck.go +++ b/healthcheck/healthcheck.go @@ -1,6 +1,7 @@ package healthcheck import ( + "context" "github.com/containous/traefik/log" "github.com/vulcand/oxy/roundrobin" "net/http" @@ -16,7 +17,6 @@ var once sync.Once func GetHealthCheck() *HealthCheck { once.Do(func() { singleton = newHealthCheck() - singleton.execute() }) return singleton } @@ -33,6 +33,7 @@ var launch = false //HealthCheck struct type HealthCheck struct { Backends map[string]*BackendHealthCheck + cancel context.CancelFunc } type loadBalancer interface { @@ -42,7 +43,7 @@ type loadBalancer interface { } func newHealthCheck() *HealthCheck { - return &HealthCheck{make(map[string]*BackendHealthCheck)} + return &HealthCheck{make(map[string]*BackendHealthCheck), nil} } // NewBackendHealthCheck Instantiate a new BackendHealthCheck @@ -53,36 +54,49 @@ func NewBackendHealthCheck(URL string, lb loadBalancer) *BackendHealthCheck { //SetBackendsConfiguration set backends configuration func (hc *HealthCheck) SetBackendsConfiguration(backends map[string]*BackendHealthCheck) { hc.Backends = backends + if hc.cancel != nil { + hc.cancel() + } + ctx, cancel := context.WithCancel(context.Background()) + hc.cancel = cancel + hc.execute(ctx) } -func (hc *HealthCheck) execute() { - ticker := time.NewTicker(time.Second * 30) - go func() { - for t := range ticker.C { - for backendID, backend := range hc.Backends { - log.Debugf("Refreshing Healthcheck %s for backend %s ", t.String(), backendID) - enabledURLs := backend.lb.Servers() - var newDisabledURLs []*url.URL - for _, url := range backend.DisabledURLs { - if testHealth(url, backend.URL) { - log.Debugf("Upsert %s", url.String()) - backend.lb.UpsertServer(url, roundrobin.Weight(1)) - } else { - newDisabledURLs = append(newDisabledURLs, url) +func (hc *HealthCheck) execute(ctx context.Context) { + for backendID, backend := range hc.Backends { + go func(backendID string, backend *BackendHealthCheck) { + for { + ticker := time.NewTicker(time.Second * 30) + select { + case <-ctx.Done(): + log.Debugf("Refreshing All Healthcheck goroutines") + return + case <-ticker.C: + log.Debugf("Refreshing Healthcheck for backend %s ", backendID) + enabledURLs := backend.lb.Servers() + var newDisabledURLs []*url.URL + for _, url := range backend.DisabledURLs { + if testHealth(url, backend.URL) { + log.Debugf("Upsert %s", url.String()) + backend.lb.UpsertServer(url, roundrobin.Weight(1)) + } else { + newDisabledURLs = append(newDisabledURLs, url) + } } - } - backend.DisabledURLs = newDisabledURLs + backend.DisabledURLs = newDisabledURLs - for _, url := range enabledURLs { - if !testHealth(url, backend.URL) { - log.Debugf("Remove %s", url.String()) - backend.lb.RemoveServer(url) - backend.DisabledURLs = append(backend.DisabledURLs, url) + for _, url := range enabledURLs { + if !testHealth(url, backend.URL) { + log.Debugf("Remove %s", url.String()) + backend.lb.RemoveServer(url) + backend.DisabledURLs = append(backend.DisabledURLs, url) + } } + } } - } - }() + }(backendID, backend) + } } func testHealth(serverURL *url.URL, checkURL string) bool { diff --git a/server.go b/server.go index b968a30bf..499a9a706 100644 --- a/server.go +++ b/server.go @@ -553,7 +553,7 @@ func (server *Server) loadConfig(configurations configs, globalConfiguration Glo backends := map[string]http.Handler{} - backendsHealcheck := map[string]*healthcheck.BackendHealthCheck{} + backendsHealthcheck := map[string]*healthcheck.BackendHealthCheck{} backend2FrontendMap := map[string]string{} for _, configuration := range configurations { @@ -655,7 +655,7 @@ func (server *Server) loadConfig(configurations configs, globalConfiguration Glo continue frontend } if configuration.Backends[frontend.Backend].HealthCheck != nil { - backendsHealcheck[frontend.Backend] = healthcheck.NewBackendHealthCheck(configuration.Backends[frontend.Backend].HealthCheck.URL, rebalancer) + backendsHealthcheck[frontend.Backend] = healthcheck.NewBackendHealthCheck(configuration.Backends[frontend.Backend].HealthCheck.URL, rebalancer) } } case types.Wrr: @@ -681,7 +681,7 @@ func (server *Server) loadConfig(configurations configs, globalConfiguration Glo } } if configuration.Backends[frontend.Backend].HealthCheck != nil { - backendsHealcheck[frontend.Backend] = healthcheck.NewBackendHealthCheck(configuration.Backends[frontend.Backend].HealthCheck.URL, rr) + backendsHealthcheck[frontend.Backend] = healthcheck.NewBackendHealthCheck(configuration.Backends[frontend.Backend].HealthCheck.URL, rr) } } maxConns := configuration.Backends[frontend.Backend].MaxConn @@ -745,7 +745,7 @@ func (server *Server) loadConfig(configurations configs, globalConfiguration Glo } } } - healthcheck.GetHealthCheck().SetBackendsConfiguration(backendsHealcheck) + healthcheck.GetHealthCheck().SetBackendsConfiguration(backendsHealthcheck) middlewares.SetBackend2FrontendMap(&backend2FrontendMap) //sort routes for _, serverEntryPoint := range serverEntryPoints { From 755822bf14ae6d7c181186e67acc47d4814a23dd Mon Sep 17 00:00:00 2001 From: Julien Salleyron Date: Tue, 31 Jan 2017 22:55:02 +0100 Subject: [PATCH 5/6] fix after review --- healthcheck/healthcheck.go | 42 +++++++++++++++++++++----------------- server.go | 2 +- types/types.go | 7 ++++--- 3 files changed, 28 insertions(+), 23 deletions(-) diff --git a/healthcheck/healthcheck.go b/healthcheck/healthcheck.go index ebf0cac8a..64a417ff0 100644 --- a/healthcheck/healthcheck.go +++ b/healthcheck/healthcheck.go @@ -2,12 +2,14 @@ package healthcheck import ( "context" - "github.com/containous/traefik/log" - "github.com/vulcand/oxy/roundrobin" "net/http" "net/url" "sync" "time" + + "github.com/containous/traefik/log" + "github.com/containous/traefik/safe" + "github.com/vulcand/oxy/roundrobin" ) var singleton *HealthCheck @@ -52,54 +54,56 @@ func NewBackendHealthCheck(URL string, lb loadBalancer) *BackendHealthCheck { } //SetBackendsConfiguration set backends configuration -func (hc *HealthCheck) SetBackendsConfiguration(backends map[string]*BackendHealthCheck) { +func (hc *HealthCheck) SetBackendsConfiguration(backends map[string]*BackendHealthCheck, parentCtx context.Context) { hc.Backends = backends if hc.cancel != nil { hc.cancel() } - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(parentCtx) hc.cancel = cancel hc.execute(ctx) } func (hc *HealthCheck) execute(ctx context.Context) { for backendID, backend := range hc.Backends { - go func(backendID string, backend *BackendHealthCheck) { + currentBackend := backend + currentBackendID := backendID + safe.Go(func() { for { ticker := time.NewTicker(time.Second * 30) select { case <-ctx.Done(): - log.Debugf("Refreshing All Healthcheck goroutines") + log.Debugf("Stopping all current Healthcheck goroutines") return case <-ticker.C: - log.Debugf("Refreshing Healthcheck for backend %s ", backendID) - enabledURLs := backend.lb.Servers() + log.Debugf("Refreshing Healthcheck for currentBackend %s ", currentBackendID) + enabledURLs := currentBackend.lb.Servers() var newDisabledURLs []*url.URL - for _, url := range backend.DisabledURLs { - if testHealth(url, backend.URL) { - log.Debugf("Upsert %s", url.String()) - backend.lb.UpsertServer(url, roundrobin.Weight(1)) + for _, url := range currentBackend.DisabledURLs { + if checkHealth(url, currentBackend.URL) { + log.Debugf("HealthCheck is up [%s]: Upsert in server list", url.String()) + currentBackend.lb.UpsertServer(url, roundrobin.Weight(1)) } else { newDisabledURLs = append(newDisabledURLs, url) } } - backend.DisabledURLs = newDisabledURLs + currentBackend.DisabledURLs = newDisabledURLs for _, url := range enabledURLs { - if !testHealth(url, backend.URL) { - log.Debugf("Remove %s", url.String()) - backend.lb.RemoveServer(url) - backend.DisabledURLs = append(backend.DisabledURLs, url) + if !checkHealth(url, currentBackend.URL) { + log.Debugf("HealthCheck has failed [%s]: Remove from server list", url.String()) + currentBackend.lb.RemoveServer(url) + currentBackend.DisabledURLs = append(currentBackend.DisabledURLs, url) } } } } - }(backendID, backend) + }) } } -func testHealth(serverURL *url.URL, checkURL string) bool { +func checkHealth(serverURL *url.URL, checkURL string) bool { timeout := time.Duration(5 * time.Second) client := http.Client{ Timeout: timeout, diff --git a/server.go b/server.go index 499a9a706..cb1b863a9 100644 --- a/server.go +++ b/server.go @@ -745,7 +745,7 @@ func (server *Server) loadConfig(configurations configs, globalConfiguration Glo } } } - healthcheck.GetHealthCheck().SetBackendsConfiguration(backendsHealthcheck) + healthcheck.GetHealthCheck().SetBackendsConfiguration(backendsHealthcheck, server.routinesPool.Ctx()) middlewares.SetBackend2FrontendMap(&backend2FrontendMap) //sort routes for _, serverEntryPoint := range serverEntryPoints { diff --git a/types/types.go b/types/types.go index d3937e156..630263164 100644 --- a/types/types.go +++ b/types/types.go @@ -4,10 +4,11 @@ import ( "encoding" "errors" "fmt" - "github.com/docker/libkv/store" - "github.com/ryanuber/go-glob" "strconv" "strings" + + "github.com/docker/libkv/store" + "github.com/ryanuber/go-glob" ) // Backend holds backend configuration. @@ -36,7 +37,7 @@ type CircuitBreaker struct { Expression string `json:"expression,omitempty"` } -// HealthCheck holds healthchk configuration +// HealthCheck holds HealthCheck configuration type HealthCheck struct { URL string `json:"url,omitempty"` } From b30272d8962098a0c2a5b7b62ef714d42ecd44f8 Mon Sep 17 00:00:00 2001 From: Julien Salleyron Date: Mon, 6 Feb 2017 09:31:20 +0100 Subject: [PATCH 6/6] fix lint --- healthcheck/healthcheck.go | 2 +- server.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/healthcheck/healthcheck.go b/healthcheck/healthcheck.go index 64a417ff0..cbae33cc4 100644 --- a/healthcheck/healthcheck.go +++ b/healthcheck/healthcheck.go @@ -54,7 +54,7 @@ func NewBackendHealthCheck(URL string, lb loadBalancer) *BackendHealthCheck { } //SetBackendsConfiguration set backends configuration -func (hc *HealthCheck) SetBackendsConfiguration(backends map[string]*BackendHealthCheck, parentCtx context.Context) { +func (hc *HealthCheck) SetBackendsConfiguration(parentCtx context.Context, backends map[string]*BackendHealthCheck) { hc.Backends = backends if hc.cancel != nil { hc.cancel() diff --git a/server.go b/server.go index cb1b863a9..e9cd4decb 100644 --- a/server.go +++ b/server.go @@ -745,7 +745,7 @@ func (server *Server) loadConfig(configurations configs, globalConfiguration Glo } } } - healthcheck.GetHealthCheck().SetBackendsConfiguration(backendsHealthcheck, server.routinesPool.Ctx()) + healthcheck.GetHealthCheck().SetBackendsConfiguration(server.routinesPool.Ctx(), backendsHealthcheck) middlewares.SetBackend2FrontendMap(&backend2FrontendMap) //sort routes for _, serverEntryPoint := range serverEntryPoints {