2016-11-26 18:48:49 +00:00
|
|
|
package healthcheck
|
|
|
|
|
|
|
|
import (
|
2016-11-30 21:49:57 +00:00
|
|
|
"context"
|
2017-03-24 08:36:33 +00:00
|
|
|
"fmt"
|
2017-05-10 18:28:57 +00:00
|
|
|
"net"
|
2016-11-26 18:48:49 +00:00
|
|
|
"net/http"
|
|
|
|
"net/url"
|
2017-05-10 18:28:57 +00:00
|
|
|
"strconv"
|
2016-11-26 18:48:49 +00:00
|
|
|
"sync"
|
|
|
|
"time"
|
2017-01-31 21:55:02 +00:00
|
|
|
|
|
|
|
"github.com/containous/traefik/log"
|
|
|
|
"github.com/containous/traefik/safe"
|
|
|
|
"github.com/vulcand/oxy/roundrobin"
|
2016-11-26 18:48:49 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
var singleton *HealthCheck
|
|
|
|
var once sync.Once
|
|
|
|
|
2017-04-20 16:51:06 +00:00
|
|
|
// GetHealthCheck returns the health check which is guaranteed to be a singleton.
|
2016-11-26 18:48:49 +00:00
|
|
|
func GetHealthCheck() *HealthCheck {
|
|
|
|
once.Do(func() {
|
|
|
|
singleton = newHealthCheck()
|
|
|
|
})
|
|
|
|
return singleton
|
|
|
|
}
|
|
|
|
|
2017-03-15 18:16:06 +00:00
|
|
|
// Options are the public health check options.
|
|
|
|
type Options struct {
|
2017-11-21 10:06:03 +00:00
|
|
|
Path string
|
|
|
|
Port int
|
|
|
|
Transport http.RoundTripper
|
|
|
|
Interval time.Duration
|
|
|
|
LB LoadBalancer
|
2017-03-15 18:16:06 +00:00
|
|
|
}
|
|
|
|
|
2017-03-24 08:36:33 +00:00
|
|
|
func (opt Options) String() string {
|
2017-09-18 13:50:03 +00:00
|
|
|
return fmt.Sprintf("[Path: %s Port: %d Interval: %s]", opt.Path, opt.Port, opt.Interval)
|
2017-03-24 08:36:33 +00:00
|
|
|
}
|
|
|
|
|
2016-11-26 18:48:49 +00:00
|
|
|
// BackendHealthCheck HealthCheck configuration for a backend
|
|
|
|
type BackendHealthCheck struct {
|
2017-03-15 18:16:06 +00:00
|
|
|
Options
|
2018-01-03 11:32:03 +00:00
|
|
|
name string
|
2017-03-15 18:16:06 +00:00
|
|
|
disabledURLs []*url.URL
|
2017-03-09 15:27:31 +00:00
|
|
|
requestTimeout time.Duration
|
2016-11-26 18:48:49 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
//HealthCheck struct
|
|
|
|
type HealthCheck struct {
|
|
|
|
Backends map[string]*BackendHealthCheck
|
2016-11-30 21:49:57 +00:00
|
|
|
cancel context.CancelFunc
|
2016-11-26 18:48:49 +00:00
|
|
|
}
|
|
|
|
|
2017-03-15 18:16:06 +00:00
|
|
|
// LoadBalancer includes functionality for load-balancing management.
|
|
|
|
type LoadBalancer interface {
|
2016-11-26 18:48:49 +00:00
|
|
|
RemoveServer(u *url.URL) error
|
|
|
|
UpsertServer(u *url.URL, options ...roundrobin.ServerOption) error
|
|
|
|
Servers() []*url.URL
|
|
|
|
}
|
|
|
|
|
|
|
|
func newHealthCheck() *HealthCheck {
|
2017-03-09 15:27:31 +00:00
|
|
|
return &HealthCheck{
|
|
|
|
Backends: make(map[string]*BackendHealthCheck),
|
|
|
|
}
|
2016-11-26 18:48:49 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// NewBackendHealthCheck Instantiate a new BackendHealthCheck
|
2018-01-03 11:32:03 +00:00
|
|
|
func NewBackendHealthCheck(options Options, backendName string) *BackendHealthCheck {
|
2017-03-09 15:27:31 +00:00
|
|
|
return &BackendHealthCheck{
|
2017-03-15 18:16:06 +00:00
|
|
|
Options: options,
|
2018-01-03 11:32:03 +00:00
|
|
|
name: backendName,
|
2017-03-09 15:27:31 +00:00
|
|
|
requestTimeout: 5 * time.Second,
|
|
|
|
}
|
2016-11-26 18:48:49 +00:00
|
|
|
}
|
|
|
|
|
2016-11-29 18:30:51 +00:00
|
|
|
//SetBackendsConfiguration set backends configuration
|
2017-02-06 08:31:20 +00:00
|
|
|
func (hc *HealthCheck) SetBackendsConfiguration(parentCtx context.Context, backends map[string]*BackendHealthCheck) {
|
2016-11-26 18:48:49 +00:00
|
|
|
hc.Backends = backends
|
2016-11-30 21:49:57 +00:00
|
|
|
if hc.cancel != nil {
|
|
|
|
hc.cancel()
|
|
|
|
}
|
2017-01-31 21:55:02 +00:00
|
|
|
ctx, cancel := context.WithCancel(parentCtx)
|
2016-11-30 21:49:57 +00:00
|
|
|
hc.cancel = cancel
|
2016-11-26 18:48:49 +00:00
|
|
|
|
2018-01-15 16:27:37 +00:00
|
|
|
for _, backend := range backends {
|
2017-03-09 15:27:31 +00:00
|
|
|
currentBackend := backend
|
2017-01-31 21:55:02 +00:00
|
|
|
safe.Go(func() {
|
2018-01-03 11:32:03 +00:00
|
|
|
hc.execute(ctx, currentBackend)
|
2017-01-31 21:55:02 +00:00
|
|
|
})
|
2016-11-30 21:49:57 +00:00
|
|
|
}
|
2016-11-26 18:48:49 +00:00
|
|
|
}
|
|
|
|
|
2018-01-03 11:32:03 +00:00
|
|
|
func (hc *HealthCheck) execute(ctx context.Context, backend *BackendHealthCheck) {
|
|
|
|
log.Debugf("Initial health check for backend: %q", backend.name)
|
|
|
|
hc.checkBackend(backend)
|
2017-03-09 15:27:31 +00:00
|
|
|
ticker := time.NewTicker(backend.Interval)
|
|
|
|
defer ticker.Stop()
|
2017-04-11 15:10:46 +00:00
|
|
|
for {
|
|
|
|
select {
|
|
|
|
case <-ctx.Done():
|
2018-01-03 11:32:03 +00:00
|
|
|
log.Debug("Stopping current health check goroutines of backend: %s", backend.name)
|
2017-04-11 15:10:46 +00:00
|
|
|
return
|
|
|
|
case <-ticker.C:
|
2018-01-03 11:32:03 +00:00
|
|
|
log.Debugf("Refreshing health check for backend: %s", backend.name)
|
|
|
|
hc.checkBackend(backend)
|
2017-03-09 15:27:31 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-01-03 11:32:03 +00:00
|
|
|
func (hc *HealthCheck) checkBackend(backend *BackendHealthCheck) {
|
|
|
|
enabledURLs := backend.LB.Servers()
|
2017-04-11 15:10:46 +00:00
|
|
|
var newDisabledURLs []*url.URL
|
2018-01-03 11:32:03 +00:00
|
|
|
for _, url := range backend.disabledURLs {
|
|
|
|
if err := checkHealth(url, backend); err == nil {
|
|
|
|
log.Warnf("Health check up: Returning to server list. Backend: %q URL: %q", backend.name, url.String())
|
|
|
|
backend.LB.UpsertServer(url, roundrobin.Weight(1))
|
2017-04-11 15:10:46 +00:00
|
|
|
} else {
|
2018-01-03 11:32:03 +00:00
|
|
|
log.Warnf("Health check still failing. Backend: %q URL: %q Reason: %s", backend.name, url.String(), err)
|
2017-04-11 15:10:46 +00:00
|
|
|
newDisabledURLs = append(newDisabledURLs, url)
|
|
|
|
}
|
|
|
|
}
|
2018-01-03 11:32:03 +00:00
|
|
|
backend.disabledURLs = newDisabledURLs
|
2017-03-09 15:27:31 +00:00
|
|
|
|
2017-04-11 15:10:46 +00:00
|
|
|
for _, url := range enabledURLs {
|
2018-01-03 11:32:03 +00:00
|
|
|
if err := checkHealth(url, backend); err != nil {
|
|
|
|
log.Warnf("Health check failed: Remove from server list. Backend: %q URL: %q Reason: %s", backend.name, url.String(), err)
|
|
|
|
backend.LB.RemoveServer(url)
|
|
|
|
backend.disabledURLs = append(backend.disabledURLs, url)
|
2017-04-11 15:10:46 +00:00
|
|
|
}
|
|
|
|
}
|
2017-03-09 15:27:31 +00:00
|
|
|
}
|
|
|
|
|
2017-05-10 18:28:57 +00:00
|
|
|
func (backend *BackendHealthCheck) newRequest(serverURL *url.URL) (*http.Request, error) {
|
2017-09-18 13:50:03 +00:00
|
|
|
if backend.Port == 0 {
|
2017-11-20 08:40:03 +00:00
|
|
|
return http.NewRequest(http.MethodGet, serverURL.String()+backend.Path, nil)
|
2017-05-10 18:28:57 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// copy the url and add the port to the host
|
|
|
|
u := &url.URL{}
|
|
|
|
*u = *serverURL
|
2017-09-18 13:50:03 +00:00
|
|
|
u.Host = net.JoinHostPort(u.Hostname(), strconv.Itoa(backend.Port))
|
2017-05-10 18:28:57 +00:00
|
|
|
u.Path = u.Path + backend.Path
|
|
|
|
|
2017-11-20 08:40:03 +00:00
|
|
|
return http.NewRequest(http.MethodGet, u.String(), nil)
|
2017-05-10 18:28:57 +00:00
|
|
|
}
|
|
|
|
|
2018-01-03 11:32:03 +00:00
|
|
|
// checkHealth returns a nil error in case it was successful and otherwise
|
|
|
|
// a non-nil error with a meaningful description why the health check failed.
|
|
|
|
func checkHealth(serverURL *url.URL, backend *BackendHealthCheck) error {
|
2016-11-30 21:48:09 +00:00
|
|
|
client := http.Client{
|
2017-11-21 10:06:03 +00:00
|
|
|
Timeout: backend.requestTimeout,
|
|
|
|
Transport: backend.Options.Transport,
|
2016-11-30 21:48:09 +00:00
|
|
|
}
|
2017-05-10 18:28:57 +00:00
|
|
|
req, err := backend.newRequest(serverURL)
|
|
|
|
if err != nil {
|
2018-01-03 11:32:03 +00:00
|
|
|
return fmt.Errorf("failed to create HTTP request: %s", err)
|
2017-05-10 18:28:57 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
resp, err := client.Do(req)
|
2017-03-09 15:27:31 +00:00
|
|
|
if err == nil {
|
|
|
|
defer resp.Body.Close()
|
2016-11-26 18:48:49 +00:00
|
|
|
}
|
2018-01-03 11:32:03 +00:00
|
|
|
|
|
|
|
switch {
|
|
|
|
case err != nil:
|
|
|
|
return fmt.Errorf("HTTP request failed: %s", err)
|
|
|
|
case resp.StatusCode != http.StatusOK:
|
|
|
|
return fmt.Errorf("received non-200 status code: %v", resp.StatusCode)
|
|
|
|
}
|
|
|
|
return nil
|
2016-11-26 18:48:49 +00:00
|
|
|
}
|