traefik/pkg/healthcheck/healthcheck.go

325 lines
9 KiB
Go
Raw Normal View History

2016-11-26 18:48:49 +00:00
package healthcheck
import (
"context"
"fmt"
"net"
2016-11-26 18:48:49 +00:00
"net/http"
"net/url"
"strconv"
2016-11-26 18:48:49 +00:00
"sync"
"time"
2017-01-31 21:55:02 +00:00
2019-08-03 01:58:23 +00:00
"github.com/containous/traefik/v2/pkg/config/runtime"
"github.com/containous/traefik/v2/pkg/log"
"github.com/containous/traefik/v2/pkg/safe"
"github.com/go-kit/kit/metrics"
2017-01-31 21:55:02 +00:00
"github.com/vulcand/oxy/roundrobin"
2016-11-26 18:48:49 +00:00
)
const (
serverUp = "UP"
serverDown = "DOWN"
)
2016-11-26 18:48:49 +00:00
var singleton *HealthCheck
var once sync.Once
// Balancer is the set of operations required to manage the list of servers in a
// load-balancer.
type Balancer interface {
2018-06-11 09:36:03 +00:00
Servers() []*url.URL
RemoveServer(u *url.URL) error
UpsertServer(u *url.URL, options ...roundrobin.ServerOption) error
}
// BalancerHandler includes functionality for load-balancing management.
type BalancerHandler interface {
ServeHTTP(w http.ResponseWriter, req *http.Request)
Balancer
}
2018-06-11 09:36:03 +00:00
// metricsRegistry is a local interface in the health check package, exposing only the required metrics
// necessary for the health check package. This makes it easier for the tests.
type metricsRegistry interface {
BackendServerUpGauge() metrics.Gauge
2016-11-26 18:48:49 +00:00
}
// Options are the public health check options.
type Options struct {
2018-04-16 09:40:03 +00:00
Headers map[string]string
Hostname string
2018-05-14 10:08:03 +00:00
Scheme string
2017-11-21 10:06:03 +00:00
Path string
Port int
Transport http.RoundTripper
Interval time.Duration
2018-09-27 18:16:03 +00:00
Timeout time.Duration
LB Balancer
}
func (opt Options) String() string {
2018-09-27 18:16:03 +00:00
return fmt.Sprintf("[Hostname: %s Headers: %v Path: %s Port: %d Interval: %s Timeout: %s]", opt.Hostname, opt.Headers, opt.Path, opt.Port, opt.Interval, opt.Timeout)
}
type backendURL struct {
url *url.URL
weight int
}
2018-06-11 09:36:03 +00:00
// BackendConfig HealthCheck configuration for a backend
type BackendConfig struct {
Options
name string
disabledURLs []backendURL
2016-11-26 18:48:49 +00:00
}
2018-06-11 09:36:03 +00:00
func (b *BackendConfig) newRequest(serverURL *url.URL) (*http.Request, error) {
2018-11-15 14:50:03 +00:00
u, err := serverURL.Parse(b.Path)
if err != nil {
return nil, err
}
2016-11-26 18:48:49 +00:00
2018-06-11 09:36:03 +00:00
if len(b.Scheme) > 0 {
u.Scheme = b.Scheme
}
2016-11-26 18:48:49 +00:00
2018-06-11 09:36:03 +00:00
if b.Port != 0 {
u.Host = net.JoinHostPort(u.Hostname(), strconv.Itoa(b.Port))
}
2016-11-26 18:48:49 +00:00
2018-10-23 08:10:04 +00:00
return http.NewRequest(http.MethodGet, u.String(), http.NoBody)
}
2018-06-11 09:36:03 +00:00
// this function adds additional http headers and hostname to http.request
func (b *BackendConfig) addHeadersAndHost(req *http.Request) *http.Request {
if b.Options.Hostname != "" {
req.Host = b.Options.Hostname
}
for k, v := range b.Options.Headers {
req.Header.Set(k, v)
}
2018-06-11 09:36:03 +00:00
return req
}
// HealthCheck struct
type HealthCheck struct {
Backends map[string]*BackendConfig
metrics metricsRegistry
cancel context.CancelFunc
2016-11-26 18:48:49 +00:00
}
2018-04-16 09:40:03 +00:00
// SetBackendsConfiguration set backends configuration
2018-06-11 09:36:03 +00:00
func (hc *HealthCheck) SetBackendsConfiguration(parentCtx context.Context, backends map[string]*BackendConfig) {
2016-11-26 18:48:49 +00:00
hc.Backends = backends
if hc.cancel != nil {
hc.cancel()
}
2017-01-31 21:55:02 +00:00
ctx, cancel := context.WithCancel(parentCtx)
hc.cancel = cancel
2016-11-26 18:48:49 +00:00
2018-01-15 16:27:37 +00:00
for _, backend := range backends {
currentBackend := backend
2017-01-31 21:55:02 +00:00
safe.Go(func() {
hc.execute(ctx, currentBackend)
2017-01-31 21:55:02 +00:00
})
}
2016-11-26 18:48:49 +00:00
}
2018-06-11 09:36:03 +00:00
func (hc *HealthCheck) execute(ctx context.Context, backend *BackendConfig) {
2019-09-13 17:28:04 +00:00
logger := log.FromContext(ctx)
logger.Debugf("Initial health check for backend: %q", backend.name)
hc.checkBackend(ctx, backend)
ticker := time.NewTicker(backend.Interval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
2019-09-13 17:28:04 +00:00
logger.Debugf("Stopping current health check goroutines of backend: %s", backend.name)
return
case <-ticker.C:
2019-09-13 17:28:04 +00:00
logger.Debugf("Refreshing health check for backend: %s", backend.name)
hc.checkBackend(ctx, backend)
}
}
}
2019-09-13 17:28:04 +00:00
func (hc *HealthCheck) checkBackend(ctx context.Context, backend *BackendConfig) {
logger := log.FromContext(ctx)
enabledURLs := backend.LB.Servers()
var newDisabledURLs []backendURL
2018-11-14 09:18:03 +00:00
// FIXME re enable metrics
for _, disabledURL := range backend.disabledURLs {
2019-02-05 16:10:03 +00:00
// FIXME serverUpMetricValue := float64(0)
if err := checkHealth(disabledURL.url, backend); err == nil {
2019-09-13 17:28:04 +00:00
logger.Warnf("Health check up: Returning to server list. Backend: %q URL: %q Weight: %d",
backend.name, disabledURL.url.String(), disabledURL.weight)
if err = backend.LB.UpsertServer(disabledURL.url, roundrobin.Weight(disabledURL.weight)); err != nil {
2019-09-13 17:28:04 +00:00
logger.Error(err)
2018-08-06 18:00:03 +00:00
}
2019-02-05 16:10:03 +00:00
// FIXME serverUpMetricValue = 1
} else {
logger.Warnf("Health check still failing. Backend: %q URL: %q Reason: %s", backend.name, disabledURL.url.String(), err)
newDisabledURLs = append(newDisabledURLs, disabledURL)
}
// FIXME labelValues := []string{"backend", backend.name, "url", backendurl.url.String()}
2019-02-05 16:10:03 +00:00
// FIXME hc.metrics.BackendServerUpGauge().With(labelValues...).Set(serverUpMetricValue)
}
backend.disabledURLs = newDisabledURLs
2018-11-14 09:18:03 +00:00
// FIXME re enable metrics
2018-08-06 18:00:03 +00:00
for _, enableURL := range enabledURLs {
2019-02-05 16:10:03 +00:00
// FIXME serverUpMetricValue := float64(1)
2018-08-06 18:00:03 +00:00
if err := checkHealth(enableURL, backend); err != nil {
weight := 1
rr, ok := backend.LB.(*roundrobin.RoundRobin)
if ok {
var gotWeight bool
weight, gotWeight = rr.ServerWeight(enableURL)
if !gotWeight {
weight = 1
}
}
logger.Warnf("Health check failed, removing from server list. Backend: %q URL: %q Weight: %d Reason: %s", backend.name, enableURL.String(), weight, err)
2018-08-06 18:00:03 +00:00
if err := backend.LB.RemoveServer(enableURL); err != nil {
2019-09-13 17:28:04 +00:00
logger.Error(err)
2018-08-06 18:00:03 +00:00
}
backend.disabledURLs = append(backend.disabledURLs, backendURL{enableURL, weight})
2019-02-05 16:10:03 +00:00
// FIXME serverUpMetricValue = 0
}
2019-02-05 16:10:03 +00:00
// FIXME labelValues := []string{"backend", backend.name, "url", enableURL.String()}
// FIXME hc.metrics.BackendServerUpGauge().With(labelValues...).Set(serverUpMetricValue)
}
}
2018-11-14 09:18:03 +00:00
// FIXME re add metrics
//func GetHealthCheck(metrics metricsRegistry) *HealthCheck {
2018-06-11 09:36:03 +00:00
// GetHealthCheck returns the health check which is guaranteed to be a singleton.
2018-11-14 09:18:03 +00:00
func GetHealthCheck() *HealthCheck {
2018-06-11 09:36:03 +00:00
once.Do(func() {
2018-11-14 09:18:03 +00:00
singleton = newHealthCheck()
//singleton = newHealthCheck(metrics)
2018-06-11 09:36:03 +00:00
})
return singleton
}
2018-11-14 09:18:03 +00:00
// FIXME re add metrics
//func newHealthCheck(metrics metricsRegistry) *HealthCheck {
func newHealthCheck() *HealthCheck {
2018-06-11 09:36:03 +00:00
return &HealthCheck{
Backends: make(map[string]*BackendConfig),
2018-11-14 09:18:03 +00:00
//metrics: metrics,
2018-04-16 09:40:03 +00:00
}
2018-06-11 09:36:03 +00:00
}
2018-06-11 09:36:03 +00:00
// NewBackendConfig Instantiate a new BackendConfig
func NewBackendConfig(options Options, backendName string) *BackendConfig {
return &BackendConfig{
2018-09-27 18:16:03 +00:00
Options: options,
name: backendName,
2018-04-16 09:40:03 +00:00
}
}
// checkHealth returns a nil error in case it was successful and otherwise
// a non-nil error with a meaningful description why the health check failed.
2018-06-11 09:36:03 +00:00
func checkHealth(serverURL *url.URL, backend *BackendConfig) error {
req, err := backend.newRequest(serverURL)
if err != nil {
return fmt.Errorf("failed to create HTTP request: %s", err)
}
2018-04-16 09:40:03 +00:00
req = backend.addHeadersAndHost(req)
client := http.Client{
2018-09-27 18:16:03 +00:00
Timeout: backend.Options.Timeout,
Transport: backend.Options.Transport,
2016-11-26 18:48:49 +00:00
}
resp, err := client.Do(req)
if err != nil {
return fmt.Errorf("HTTP request failed: %s", err)
}
defer resp.Body.Close()
if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusBadRequest {
return fmt.Errorf("received error status code: %v", resp.StatusCode)
}
return nil
2016-11-26 18:48:49 +00:00
}
// NewLBStatusUpdater returns a new LbStatusUpdater
2019-09-13 17:28:04 +00:00
func NewLBStatusUpdater(bh BalancerHandler, info *runtime.ServiceInfo) *LbStatusUpdater {
return &LbStatusUpdater{
BalancerHandler: bh,
2019-09-13 17:28:04 +00:00
serviceInfo: info,
}
}
// LbStatusUpdater wraps a BalancerHandler and a ServiceInfo,
// so it can keep track of the status of a server in the ServiceInfo.
type LbStatusUpdater struct {
BalancerHandler
serviceInfo *runtime.ServiceInfo // can be nil
}
// RemoveServer removes the given server from the BalancerHandler,
// and updates the status of the server to "DOWN".
func (lb *LbStatusUpdater) RemoveServer(u *url.URL) error {
err := lb.BalancerHandler.RemoveServer(u)
if err == nil && lb.serviceInfo != nil {
lb.serviceInfo.UpdateServerStatus(u.String(), serverDown)
}
return err
}
// UpsertServer adds the given server to the BalancerHandler,
// and updates the status of the server to "UP".
func (lb *LbStatusUpdater) UpsertServer(u *url.URL, options ...roundrobin.ServerOption) error {
err := lb.BalancerHandler.UpsertServer(u, options...)
if err == nil && lb.serviceInfo != nil {
lb.serviceInfo.UpdateServerStatus(u.String(), serverUp)
}
return err
}
// Balancers is a list of Balancers(s) that implements the Balancer interface.
type Balancers []Balancer
// Servers returns the servers url from all the BalancerHandler
func (b Balancers) Servers() []*url.URL {
var servers []*url.URL
for _, lb := range b {
servers = append(servers, lb.Servers()...)
}
return servers
}
// RemoveServer removes the given server from all the BalancerHandler,
// and updates the status of the server to "DOWN".
func (b Balancers) RemoveServer(u *url.URL) error {
for _, lb := range b {
if err := lb.RemoveServer(u); err != nil {
return err
}
}
return nil
}
// UpsertServer adds the given server to all the BalancerHandler,
// and updates the status of the server to "UP".
func (b Balancers) UpsertServer(u *url.URL, options ...roundrobin.ServerOption) error {
for _, lb := range b {
if err := lb.UpsertServer(u, options...); err != nil {
return err
}
}
return nil
}