package healthcheck import ( "context" "net/http" "net/http/httptest" "net/url" "sync" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/traefik/traefik/v2/pkg/config/runtime" "github.com/traefik/traefik/v2/pkg/testhelpers" "github.com/vulcand/oxy/v2/roundrobin" ) const ( healthCheckInterval = 200 * time.Millisecond healthCheckTimeout = 100 * time.Millisecond ) const delta float64 = 1e-10 type testHandler struct { done func() healthSequence []int } func TestSetBackendsConfiguration(t *testing.T) { testCases := []struct { desc string startHealthy bool healthSequence []int expectedNumRemovedServers int expectedNumUpsertedServers int expectedGaugeValue float64 }{ { desc: "healthy server staying healthy", startHealthy: true, healthSequence: []int{http.StatusOK}, expectedNumRemovedServers: 0, expectedNumUpsertedServers: 0, expectedGaugeValue: 1, }, { desc: "healthy server staying healthy (StatusNoContent)", startHealthy: true, healthSequence: []int{http.StatusNoContent}, expectedNumRemovedServers: 0, expectedNumUpsertedServers: 0, expectedGaugeValue: 1, }, { desc: "healthy server staying healthy (StatusPermanentRedirect)", startHealthy: true, healthSequence: []int{http.StatusPermanentRedirect}, expectedNumRemovedServers: 0, expectedNumUpsertedServers: 0, expectedGaugeValue: 1, }, { desc: "healthy server becoming sick", startHealthy: true, healthSequence: []int{http.StatusServiceUnavailable}, expectedNumRemovedServers: 1, expectedNumUpsertedServers: 0, expectedGaugeValue: 0, }, { desc: "sick server becoming healthy", startHealthy: false, healthSequence: []int{http.StatusOK}, expectedNumRemovedServers: 0, expectedNumUpsertedServers: 1, expectedGaugeValue: 1, }, { desc: "sick server staying sick", startHealthy: false, healthSequence: []int{http.StatusServiceUnavailable}, expectedNumRemovedServers: 0, expectedNumUpsertedServers: 0, expectedGaugeValue: 0, }, { desc: "healthy server toggling to sick and back to healthy", startHealthy: true, healthSequence: []int{http.StatusServiceUnavailable, http.StatusOK}, expectedNumRemovedServers: 1, expectedNumUpsertedServers: 1, expectedGaugeValue: 1, }, } for _, test := range testCases { t.Run(test.desc, func(t *testing.T) { t.Parallel() // The context is passed to the health check and canonically canceled by // the test server once all expected requests have been received. ctx, cancel := context.WithCancel(context.Background()) defer cancel() ts := newTestServer(cancel, test.healthSequence) defer ts.Close() lb := &testLoadBalancer{RWMutex: &sync.RWMutex{}} backend := NewBackendConfig(Options{ Path: "/path", Interval: healthCheckInterval, Timeout: healthCheckTimeout, LB: lb, }, "backendName") serverURL := testhelpers.MustParseURL(ts.URL) if test.startHealthy { lb.servers = append(lb.servers, serverURL) } else { backend.disabledURLs = append(backend.disabledURLs, backendURL{url: serverURL, weight: 1}) } collectingMetrics := &testhelpers.CollectingGauge{} check := HealthCheck{ Backends: make(map[string]*BackendConfig), metrics: metricsHealthcheck{serverUpGauge: collectingMetrics}, } wg := sync.WaitGroup{} wg.Add(1) go func() { check.execute(ctx, backend) wg.Done() }() // Make test timeout dependent on number of expected requests, health // check interval, and a safety margin. timeout := time.Duration(len(test.healthSequence)*int(healthCheckInterval) + 500) select { case <-time.After(timeout): t.Fatal("test did not complete in time") case <-ctx.Done(): wg.Wait() } lb.Lock() defer lb.Unlock() assert.Equal(t, test.expectedNumRemovedServers, lb.numRemovedServers, "removed servers") assert.Equal(t, test.expectedNumUpsertedServers, lb.numUpsertedServers, "upserted servers") assert.InDelta(t, test.expectedGaugeValue, collectingMetrics.GaugeValue, delta, "ServerUp Gauge") }) } } func TestNewRequest(t *testing.T) { type expected struct { err bool value string } testCases := []struct { desc string serverURL string options Options expected expected }{ { desc: "no port override", serverURL: "http://backend1:80", options: Options{ Path: "/test", Port: 0, }, expected: expected{ err: false, value: "http://backend1:80/test", }, }, { desc: "port override", serverURL: "http://backend2:80", options: Options{ Path: "/test", Port: 8080, }, expected: expected{ err: false, value: "http://backend2:8080/test", }, }, { desc: "no port override with no port in server URL", serverURL: "http://backend1", options: Options{ Path: "/health", Port: 0, }, expected: expected{ err: false, value: "http://backend1/health", }, }, { desc: "port override with no port in server URL", serverURL: "http://backend2", options: Options{ Path: "/health", Port: 8080, }, expected: expected{ err: false, value: "http://backend2:8080/health", }, }, { desc: "scheme override", serverURL: "https://backend1:80", options: Options{ Scheme: "http", Path: "/test", Port: 0, }, expected: expected{ err: false, value: "http://backend1:80/test", }, }, { desc: "path with param", serverURL: "http://backend1:80", options: Options{ Path: "/health?powpow=do", Port: 0, }, expected: expected{ err: false, value: "http://backend1:80/health?powpow=do", }, }, { desc: "path with params", serverURL: "http://backend1:80", options: Options{ Path: "/health?powpow=do&do=powpow", Port: 0, }, expected: expected{ err: false, value: "http://backend1:80/health?powpow=do&do=powpow", }, }, { desc: "path with invalid path", serverURL: "http://backend1:80", options: Options{ Path: ":", Port: 0, }, expected: expected{ err: true, value: "", }, }, } for _, test := range testCases { t.Run(test.desc, func(t *testing.T) { t.Parallel() backend := NewBackendConfig(test.options, "backendName") u := testhelpers.MustParseURL(test.serverURL) req, err := backend.newRequest(u) if test.expected.err { require.Error(t, err) assert.Nil(t, req) } else { require.NoError(t, err, "failed to create new backend request") require.NotNil(t, req) assert.Equal(t, test.expected.value, req.URL.String()) } }) } } func TestRequestOptions(t *testing.T) { testCases := []struct { desc string serverURL string options Options expectedHostname string expectedHeader string expectedMethod string }{ { desc: "override hostname", serverURL: "http://backend1:80", options: Options{ Hostname: "myhost", Path: "/", }, expectedHostname: "myhost", expectedHeader: "", expectedMethod: http.MethodGet, }, { desc: "not override hostname", serverURL: "http://backend1:80", options: Options{ Hostname: "", Path: "/", }, expectedHostname: "backend1:80", expectedHeader: "", expectedMethod: http.MethodGet, }, { desc: "custom header", serverURL: "http://backend1:80", options: Options{ Headers: map[string]string{"Custom-Header": "foo"}, Hostname: "", Path: "/", }, expectedHostname: "backend1:80", expectedHeader: "foo", expectedMethod: http.MethodGet, }, { desc: "custom header with hostname override", serverURL: "http://backend1:80", options: Options{ Headers: map[string]string{"Custom-Header": "foo"}, Hostname: "myhost", Path: "/", }, expectedHostname: "myhost", expectedHeader: "foo", expectedMethod: http.MethodGet, }, { desc: "custom method", serverURL: "http://backend1:80", options: Options{ Path: "/", Method: http.MethodHead, }, expectedHostname: "backend1:80", expectedMethod: http.MethodHead, }, } for _, test := range testCases { t.Run(test.desc, func(t *testing.T) { t.Parallel() backend := NewBackendConfig(test.options, "backendName") u, err := url.Parse(test.serverURL) require.NoError(t, err) req, err := backend.newRequest(u) require.NoError(t, err, "failed to create new backend request") req = backend.setRequestOptions(req) assert.Equal(t, "http://backend1:80/", req.URL.String()) assert.Equal(t, test.expectedHostname, req.Host) assert.Equal(t, test.expectedHeader, req.Header.Get("Custom-Header")) assert.Equal(t, test.expectedMethod, req.Method) }) } } func TestBalancers_Servers(t *testing.T) { server1, err := url.Parse("http://foo.com") require.NoError(t, err) balancer1, err := roundrobin.New(nil) require.NoError(t, err) err = balancer1.UpsertServer(server1) require.NoError(t, err) server2, err := url.Parse("http://foo.com") require.NoError(t, err) balancer2, err := roundrobin.New(nil) require.NoError(t, err) err = balancer2.UpsertServer(server2) require.NoError(t, err) balancers := Balancers([]Balancer{balancer1, balancer2}) want, err := url.Parse("http://foo.com") require.NoError(t, err) assert.Len(t, balancers.Servers(), 1) assert.Equal(t, want, balancers.Servers()[0]) } func TestBalancers_UpsertServer(t *testing.T) { balancer1, err := roundrobin.New(nil) require.NoError(t, err) balancer2, err := roundrobin.New(nil) require.NoError(t, err) want, err := url.Parse("http://foo.com") require.NoError(t, err) balancers := Balancers([]Balancer{balancer1, balancer2}) err = balancers.UpsertServer(want) require.NoError(t, err) assert.Len(t, balancer1.Servers(), 1) assert.Equal(t, want, balancer1.Servers()[0]) assert.Len(t, balancer2.Servers(), 1) assert.Equal(t, want, balancer2.Servers()[0]) } func TestBalancers_RemoveServer(t *testing.T) { server, err := url.Parse("http://foo.com") require.NoError(t, err) balancer1, err := roundrobin.New(nil) require.NoError(t, err) err = balancer1.UpsertServer(server) require.NoError(t, err) balancer2, err := roundrobin.New(nil) require.NoError(t, err) err = balancer2.UpsertServer(server) require.NoError(t, err) balancers := Balancers([]Balancer{balancer1, balancer2}) err = balancers.RemoveServer(server) require.NoError(t, err) assert.Empty(t, balancer1.Servers()) assert.Empty(t, balancer2.Servers()) } type testLoadBalancer struct { // RWMutex needed due to parallel test execution: Both the system-under-test // and the test assertions reference the counters. *sync.RWMutex numRemovedServers int numUpsertedServers int servers []*url.URL // options is just to make sure that LBStatusUpdater forwards options on Upsert to its BalancerHandler options []roundrobin.ServerOption } func (lb *testLoadBalancer) ServeHTTP(w http.ResponseWriter, req *http.Request) { // noop } func (lb *testLoadBalancer) RemoveServer(u *url.URL) error { lb.Lock() defer lb.Unlock() lb.numRemovedServers++ lb.removeServer(u) return nil } func (lb *testLoadBalancer) UpsertServer(u *url.URL, options ...roundrobin.ServerOption) error { lb.Lock() defer lb.Unlock() lb.numUpsertedServers++ lb.servers = append(lb.servers, u) lb.options = append(lb.options, options...) return nil } func (lb *testLoadBalancer) Servers() []*url.URL { return lb.servers } func (lb *testLoadBalancer) Options() []roundrobin.ServerOption { return lb.options } func (lb *testLoadBalancer) removeServer(u *url.URL) { var i int var serverURL *url.URL found := false for i, serverURL = range lb.servers { if *serverURL == *u { found = true break } } if !found { return } lb.servers = append(lb.servers[:i], lb.servers[i+1:]...) } func newTestServer(done func(), healthSequence []int) *httptest.Server { handler := &testHandler{ done: done, healthSequence: healthSequence, } return httptest.NewServer(handler) } // ServeHTTP returns HTTP response codes following a status sequences. // It calls the given 'done' function once all request health indicators have been depleted. func (th *testHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { if len(th.healthSequence) == 0 { panic("received unexpected request") } w.WriteHeader(th.healthSequence[0]) th.healthSequence = th.healthSequence[1:] if len(th.healthSequence) == 0 { th.done() } } func TestLBStatusUpdater(t *testing.T) { lb := &testLoadBalancer{RWMutex: &sync.RWMutex{}} svInfo := &runtime.ServiceInfo{} lbsu := NewLBStatusUpdater(lb, svInfo, nil) newServer, err := url.Parse("http://foo.com") assert.NoError(t, err) err = lbsu.UpsertServer(newServer, roundrobin.Weight(1)) assert.NoError(t, err) assert.Len(t, lbsu.Servers(), 1) assert.Len(t, lbsu.BalancerHandler.(*testLoadBalancer).Options(), 1) statuses := svInfo.GetAllStatus() assert.Len(t, statuses, 1) for k, v := range statuses { assert.Equal(t, newServer.String(), k) assert.Equal(t, serverUp, v) break } err = lbsu.RemoveServer(newServer) assert.NoError(t, err) assert.Empty(t, lbsu.Servers()) statuses = svInfo.GetAllStatus() assert.Len(t, statuses, 1) for k, v := range statuses { assert.Equal(t, newServer.String(), k) assert.Equal(t, serverDown, v) break } } func TestNotFollowingRedirects(t *testing.T) { redirectServerCalled := false redirectTestServer := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { redirectServerCalled = true })) defer redirectTestServer.Close() ctx, cancel := context.WithCancel(context.Background()) defer cancel() server := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { rw.Header().Add("location", redirectTestServer.URL) rw.WriteHeader(http.StatusSeeOther) cancel() })) defer server.Close() lb := &testLoadBalancer{ RWMutex: &sync.RWMutex{}, servers: []*url.URL{testhelpers.MustParseURL(server.URL)}, } backend := NewBackendConfig(Options{ Path: "/path", Interval: healthCheckInterval, Timeout: healthCheckTimeout, LB: lb, FollowRedirects: false, }, "backendName") collectingMetrics := &testhelpers.CollectingGauge{} check := HealthCheck{ Backends: make(map[string]*BackendConfig), metrics: metricsHealthcheck{serverUpGauge: collectingMetrics}, } wg := sync.WaitGroup{} wg.Add(1) go func() { check.execute(ctx, backend) wg.Done() }() timeout := time.Duration(int(healthCheckInterval) + 500) select { case <-time.After(timeout): t.Fatal("test did not complete in time") case <-ctx.Done(): wg.Wait() } assert.False(t, redirectServerCalled, "HTTP redirect must not be followed") }