From f2e53a356946454e7fc366fce35378973ce43fd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robin=20M=C3=BCller?= Date: Sat, 26 Sep 2020 13:30:03 +0200 Subject: [PATCH] Re-add server up metrics --- pkg/healthcheck/healthcheck.go | 41 +++++++++++++++++++++-------- pkg/healthcheck/healthcheck_test.go | 10 +++---- pkg/server/service/service.go | 3 +-- 3 files changed, 36 insertions(+), 18 deletions(-) diff --git a/pkg/healthcheck/healthcheck.go b/pkg/healthcheck/healthcheck.go index f76db683c..2ba13feca 100644 --- a/pkg/healthcheck/healthcheck.go +++ b/pkg/healthcheck/healthcheck.go @@ -10,9 +10,10 @@ import ( "sync" "time" - "github.com/go-kit/kit/metrics" + gokitmetrics "github.com/go-kit/kit/metrics" "github.com/traefik/traefik/v2/pkg/config/runtime" "github.com/traefik/traefik/v2/pkg/log" + "github.com/traefik/traefik/v2/pkg/metrics" "github.com/traefik/traefik/v2/pkg/safe" "github.com/vulcand/oxy/roundrobin" ) @@ -40,11 +41,8 @@ type BalancerHandler interface { Balancer } -// metricsRegistry is a local interface in the health check package, -// exposing only the required metrics necessary for the health check package. -// This makes it easier for the tests. -type metricsRegistry interface { - BackendServerUpGauge() metrics.Gauge +type metricsHealthcheck struct { + serverUpGauge gokitmetrics.Gauge } // Options are the public health check options. @@ -109,7 +107,7 @@ func (b *BackendConfig) addHeadersAndHost(req *http.Request) *http.Request { // HealthCheck struct. type HealthCheck struct { Backends map[string]*BackendConfig - metrics metricsRegistry + metrics metricsHealthcheck cancel context.CancelFunc } @@ -153,22 +151,33 @@ func (hc *HealthCheck) checkBackend(ctx context.Context, backend *BackendConfig) logger := log.FromContext(ctx) enabledURLs := backend.LB.Servers() + var newDisabledURLs []backendURL for _, disabledURL := range backend.disabledURLs { + serverUpMetricValue := float64(0) + if err := checkHealth(disabledURL.url, backend); err == nil { logger.Warnf("Health check up: Returning to server list. Backend: %q URL: %q Weight: %d", backend.name, disabledURL.url.String(), disabledURL.weight) if err = backend.LB.UpsertServer(disabledURL.url, roundrobin.Weight(disabledURL.weight)); err != nil { logger.Error(err) } + + serverUpMetricValue = 1 } else { logger.Warnf("Health check still failing. Backend: %q URL: %q Reason: %s", backend.name, disabledURL.url.String(), err) newDisabledURLs = append(newDisabledURLs, disabledURL) } + + labelValues := []string{"service", backend.name, "url", disabledURL.url.String()} + hc.metrics.serverUpGauge.With(labelValues...).Set(serverUpMetricValue) } + backend.disabledURLs = newDisabledURLs for _, enableURL := range enabledURLs { + serverUpMetricValue := float64(1) + if err := checkHealth(enableURL, backend); err != nil { weight := 1 rr, ok := backend.LB.(*roundrobin.RoundRobin) @@ -179,26 +188,36 @@ func (hc *HealthCheck) checkBackend(ctx context.Context, backend *BackendConfig) weight = 1 } } - logger.Warnf("Health check failed, removing from server list. Backend: %q URL: %q Weight: %d Reason: %s", backend.name, enableURL.String(), weight, err) + + logger.Warnf("Health check failed, removing from server list. Backend: %q URL: %q Weight: %d Reason: %s", + backend.name, enableURL.String(), weight, err) if err := backend.LB.RemoveServer(enableURL); err != nil { logger.Error(err) } + backend.disabledURLs = append(backend.disabledURLs, backendURL{enableURL, weight}) + serverUpMetricValue = 0 } + + labelValues := []string{"service", backend.name, "url", enableURL.String()} + hc.metrics.serverUpGauge.With(labelValues...).Set(serverUpMetricValue) } } // GetHealthCheck returns the health check which is guaranteed to be a singleton. -func GetHealthCheck() *HealthCheck { +func GetHealthCheck(registry metrics.Registry) *HealthCheck { once.Do(func() { - singleton = newHealthCheck() + singleton = newHealthCheck(registry) }) return singleton } -func newHealthCheck() *HealthCheck { +func newHealthCheck(registry metrics.Registry) *HealthCheck { return &HealthCheck{ Backends: make(map[string]*BackendConfig), + metrics: metricsHealthcheck{ + serverUpGauge: registry.ServiceServerUpGauge(), + }, } } diff --git a/pkg/healthcheck/healthcheck_test.go b/pkg/healthcheck/healthcheck_test.go index 09fab00f9..594665ae7 100644 --- a/pkg/healthcheck/healthcheck_test.go +++ b/pkg/healthcheck/healthcheck_test.go @@ -120,10 +120,10 @@ func TestSetBackendsConfiguration(t *testing.T) { backend.disabledURLs = append(backend.disabledURLs, backendURL{url: serverURL, weight: 1}) } - collectingMetrics := testhelpers.NewCollectingHealthCheckMetrics() + collectingMetrics := &testhelpers.CollectingGauge{} check := HealthCheck{ Backends: make(map[string]*BackendConfig), - metrics: collectingMetrics, + metrics: metricsHealthcheck{serverUpGauge: collectingMetrics}, } wg := sync.WaitGroup{} @@ -149,8 +149,7 @@ func TestSetBackendsConfiguration(t *testing.T) { assert.Equal(t, test.expectedNumRemovedServers, lb.numRemovedServers, "removed servers") assert.Equal(t, test.expectedNumUpsertedServers, lb.numUpsertedServers, "upserted servers") - // FIXME re add metrics - // assert.Equal(t, test.expectedGaugeValue, collectingMetrics.Gauge.GaugeValue, "ServerUp Gauge") + assert.Equal(t, test.expectedGaugeValue, collectingMetrics.GaugeValue, "ServerUp Gauge") }) } } @@ -502,9 +501,10 @@ func TestNotFollowingRedirects(t *testing.T) { FollowRedirects: false, }, "backendName") + collectingMetrics := &testhelpers.CollectingGauge{} check := HealthCheck{ Backends: make(map[string]*BackendConfig), - metrics: testhelpers.NewCollectingHealthCheckMetrics(), + metrics: metricsHealthcheck{serverUpGauge: collectingMetrics}, } wg := sync.WaitGroup{} diff --git a/pkg/server/service/service.go b/pkg/server/service/service.go index 674a05768..33a019423 100644 --- a/pkg/server/service/service.go +++ b/pkg/server/service/service.go @@ -236,8 +236,7 @@ func (m *Manager) LaunchHealthCheck() { } } - // FIXME metrics and context - healthcheck.GetHealthCheck().SetBackendsConfiguration(context.Background(), backendConfigs) + healthcheck.GetHealthCheck(m.metricsRegistry).SetBackendsConfiguration(context.Background(), backendConfigs) } func buildHealthCheckOptions(ctx context.Context, lb healthcheck.Balancer, backend string, hc *dynamic.HealthCheck) *healthcheck.Options {