fix: opentelemetry unit tests

pkg/metrics/datadog_test.go

@@ -13,12 +13,13 @@ import (
 )
 
 func TestDatadog(t *testing.T) {
+	t.Cleanup(StopDatadog)
+
 	udp.SetAddr(":18125")
 	// This is needed to make sure that UDP Listener listens for data a bit longer, otherwise it will quit after a millisecond
 	udp.Timeout = 5 * time.Second
 
 	datadogRegistry := RegisterDatadog(context.Background(), &types.Datadog{Address: ":18125", PushInterval: ptypes.Duration(time.Second), AddEntryPointsLabels: true, AddRoutersLabels: true, AddServicesLabels: true})
-	defer StopDatadog()
 
 	if !datadogRegistry.IsEpEnabled() || !datadogRegistry.IsRouterEnabled() || !datadogRegistry.IsSvcEnabled() {
 		t.Errorf("DatadogRegistry should return true for IsEnabled(), IsRouterEnabled() and IsSvcEnabled()")
@@ -27,9 +28,7 @@ func TestDatadog(t *testing.T) {
 }
 
 func TestDatadogWithPrefix(t *testing.T) {
-	t.Cleanup(func() {
+	t.Cleanup(StopDatadog)
-		StopDatadog()
-	})
 
 	udp.SetAddr(":18125")
 	// This is needed to make sure that UDP Listener listens for data a bit longer, otherwise it will quit after a millisecond

pkg/metrics/influxdb2_test.go

@@ -6,7 +6,6 @@ import (
 	"io"
 	"net/http"
 	"net/http/httptest"
-	"regexp"
 	"strconv"
 	"testing"
 	"time"
@@ -26,7 +25,6 @@ func TestInfluxDB2(t *testing.T) {
 		c <- &bodyStr
 		_, _ = fmt.Fprintln(w, "ok")
 	}))
-	defer ts.Close()
 
 	influxDB2Registry := RegisterInfluxDB2(context.Background(),
 		&types.InfluxDB2{
@@ -39,7 +37,11 @@ func TestInfluxDB2(t *testing.T) {
 			AddRoutersLabels:     true,
 			AddServicesLabels:    true,
 		})
-	defer StopInfluxDB2()
+
+	t.Cleanup(func() {
+		StopInfluxDB2()
+		ts.Close()
+	})
 
 	if !influxDB2Registry.IsEpEnabled() || !influxDB2Registry.IsRouterEnabled() || !influxDB2Registry.IsSvcEnabled() {
 		t.Fatalf("InfluxDB2Registry should return true for IsEnabled(), IsRouterEnabled() and IsSvcEnabled()")
@@ -137,14 +139,3 @@ func TestInfluxDB2(t *testing.T) {
 
 	assertMessage(t, *msgServiceRetries, expectedServiceRetries)
 }
-
-func assertMessage(t *testing.T, msg string, patterns []string) {
-	t.Helper()
-	for _, pattern := range patterns {
-		re := regexp.MustCompile(pattern)
-		match := re.FindStringSubmatch(msg)
-		if len(match) != 2 {
-			t.Errorf("Got %q %v, want %q", msg, match, pattern)
-		}
-	}
-}

pkg/metrics/opentelemetry_test.go

@@ -4,10 +4,12 @@ import (
 	"compress/gzip"
 	"context"
 	"encoding/json"
+	"fmt"
 	"io"
 	"net/http"
 	"net/http/httptest"
 	"net/url"
+	"regexp"
 	"strconv"
 	"testing"
 	"time"
@@ -310,7 +312,7 @@ func TestOpenTelemetry(t *testing.T) {
 	}))
 
 	t.Cleanup(func() {
-		close(c)
+		StopOpenTelemetry()
 		ts.Close()
 	})
 
@@ -335,57 +337,53 @@ func TestOpenTelemetry(t *testing.T) {
 		`({"key":"service.name","value":{"stringValue":"traefik"}})`,
 		`({"key":"service.version","value":{"stringValue":"` + version.Version + `"}})`,
 	}
-	msgMisc := <-c
 
-	assertMessage(t, *msgMisc, expected)
+	tryAssertMessage(t, c, expected)
 
 	// TODO: the len of startUnixNano is no supposed to be 20, it should be 19
-	expected = append(expected,
+	expectedConfig := []string{
 		`({"name":"traefik_config_reloads_total","description":"Config reloads","unit":"1","sum":{"dataPoints":\[{"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1}\],"aggregationTemporality":2,"isMonotonic":true}})`,
 		`({"name":"traefik_config_last_reload_success","description":"Last config reload success","unit":"ms","gauge":{"dataPoints":\[{"timeUnixNano":"[\d]{19}","asDouble":1}\]}})`,
 		`({"name":"traefik_open_connections","description":"How many open connections exist, by entryPoint and protocol","unit":"1","gauge":{"dataPoints":\[{"attributes":\[{"key":"entrypoint","value":{"stringValue":"test"}},{"key":"protocol","value":{"stringValue":"TCP"}}\],"timeUnixNano":"[\d]{19}","asDouble":1}\]}})`,
-	)
+	}
 
 	registry.ConfigReloadsCounter().Add(1)
 	registry.LastConfigReloadSuccessGauge().Set(1)
 	registry.OpenConnectionsGauge().With("entrypoint", "test", "protocol", "TCP").Set(1)
-	msgServer := <-c
 
-	assertMessage(t, *msgServer, expected)
+	tryAssertMessage(t, c, expectedConfig)
 
-	expected = append(expected,
+	expectedTLSCerts := []string{
 		`({"name":"traefik_tls_certs_not_after","description":"Certificate expiration timestamp","unit":"ms","gauge":{"dataPoints":\[{"attributes":\[{"key":"key","value":{"stringValue":"value"}}\],"timeUnixNano":"[\d]{19}","asDouble":1}\]}})`,
-	)
+	}
 
 	registry.TLSCertsNotAfterTimestampGauge().With("key", "value").Set(1)
-	msgTLS := <-c
 
-	assertMessage(t, *msgTLS, expected)
+	tryAssertMessage(t, c, expectedTLSCerts)
 
-	expected = append(expected,
+	expectedEntryPoints := []string{
 		`({"name":"traefik_entrypoint_requests_total","description":"How many HTTP requests processed on an entrypoint, partitioned by status code, protocol, and method.","unit":"1","sum":{"dataPoints":\[{"attributes":\[{"key":"code","value":{"stringValue":"200"}},{"key":"entrypoint","value":{"stringValue":"test1"}},{"key":"method","value":{"stringValue":"GET"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1}\],"aggregationTemporality":2,"isMonotonic":true}})`,
 		`({"name":"traefik_entrypoint_requests_tls_total","description":"How many HTTP requests with TLS processed on an entrypoint, partitioned by TLS Version and TLS cipher Used.","unit":"1","sum":{"dataPoints":\[{"attributes":\[{"key":"entrypoint","value":{"stringValue":"test2"}},{"key":"tls_cipher","value":{"stringValue":"bar"}},{"key":"tls_version","value":{"stringValue":"foo"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1}\],"aggregationTemporality":2,"isMonotonic":true}})`,
 		`({"name":"traefik_entrypoint_request_duration_seconds","description":"How long it took to process the request on an entrypoint, partitioned by status code, protocol, and method.","unit":"ms","histogram":{"dataPoints":\[{"attributes":\[{"key":"entrypoint","value":{"stringValue":"test3"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","count":"1","sum":10000,"bucketCounts":\["0","0","0","0","0","0","0","0","0","0","0","1"\],"explicitBounds":\[0.005,0.01,0.025,0.05,0.1,0.25,0.5,1,2.5,5,10\],"min":10000,"max":10000}\],"aggregationTemporality":2}})`,
 		`({"name":"traefik_entrypoint_requests_bytes_total","description":"The total size of requests in bytes handled by an entrypoint, partitioned by status code, protocol, and method.","unit":"1","sum":{"dataPoints":\[{"attributes":\[{"key":"code","value":{"stringValue":"200"}},{"key":"entrypoint","value":{"stringValue":"test1"}},{"key":"method","value":{"stringValue":"GET"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1}\],"aggregationTemporality":2,"isMonotonic":true}})`,
 		`({"name":"traefik_entrypoint_responses_bytes_total","description":"The total size of responses in bytes handled by an entrypoint, partitioned by status code, protocol, and method.","unit":"1","sum":{"dataPoints":\[{"attributes":\[{"key":"code","value":{"stringValue":"200"}},{"key":"entrypoint","value":{"stringValue":"test1"}},{"key":"method","value":{"stringValue":"GET"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1}\],"aggregationTemporality":2,"isMonotonic":true}})`,
-	)
+	}
 
 	registry.EntryPointReqsCounter().With(nil, "entrypoint", "test1", "code", strconv.Itoa(http.StatusOK), "method", http.MethodGet).Add(1)
 	registry.EntryPointReqsTLSCounter().With("entrypoint", "test2", "tls_version", "foo", "tls_cipher", "bar").Add(1)
 	registry.EntryPointReqDurationHistogram().With("entrypoint", "test3").Observe(10000)
 	registry.EntryPointReqsBytesCounter().With("entrypoint", "test1", "code", strconv.Itoa(http.StatusOK), "method", http.MethodGet).Add(1)
 	registry.EntryPointRespsBytesCounter().With("entrypoint", "test1", "code", strconv.Itoa(http.StatusOK), "method", http.MethodGet).Add(1)
-	msgEntrypoint := <-c
 
-	assertMessage(t, *msgEntrypoint, expected)
+	tryAssertMessage(t, c, expectedEntryPoints)
 
-	expected = append(expected,
+	expectedRouters := []string{
 		`({"name":"traefik_router_requests_total","description":"How many HTTP requests are processed on a router, partitioned by service, status code, protocol, and method.","unit":"1","sum":{"dataPoints":\[{"attributes":\[{"key":"code","value":{"stringValue":"(?:200|404)"}},{"key":"method","value":{"stringValue":"GET"}},{"key":"router","value":{"stringValue":"RouterReqsCounter"}},{"key":"service","value":{"stringValue":"test"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1},{"attributes":\[{"key":"code","value":{"stringValue":"(?:200|404)"}},{"key":"method","value":{"stringValue":"GET"}},{"key":"router","value":{"stringValue":"RouterReqsCounter"}},{"key":"service","value":{"stringValue":"test"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1}\],"aggregationTemporality":2,"isMonotonic":true}})`,
 		`({"name":"traefik_router_requests_tls_total","description":"How many HTTP requests with TLS are processed on a router, partitioned by service, TLS Version, and TLS cipher Used.","unit":"1","sum":{"dataPoints":\[{"attributes":\[{"key":"router","value":{"stringValue":"demo"}},{"key":"service","value":{"stringValue":"test"}},{"key":"tls_cipher","value":{"stringValue":"bar"}},{"key":"tls_version","value":{"stringValue":"foo"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1}\],"aggregationTemporality":2,"isMonotonic":true}})`,
 		`({"name":"traefik_router_request_duration_seconds","description":"How long it took to process the request on a router, partitioned by service, status code, protocol, and method.","unit":"ms","histogram":{"dataPoints":\[{"attributes":\[{"key":"code","value":{"stringValue":"200"}},{"key":"router","value":{"stringValue":"demo"}},{"key":"service","value":{"stringValue":"test"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","count":"1","sum":10000,"bucketCounts":\["0","0","0","0","0","0","0","0","0","0","0","1"\],"explicitBounds":\[0.005,0.01,0.025,0.05,0.1,0.25,0.5,1,2.5,5,10\],"min":10000,"max":10000}\],"aggregationTemporality":2}})`,
 		`({"name":"traefik_router_requests_bytes_total","description":"The total size of requests in bytes handled by a router, partitioned by status code, protocol, and method.","unit":"1","sum":{"dataPoints":\[{"attributes":\[{"key":"code","value":{"stringValue":"404"}},{"key":"method","value":{"stringValue":"GET"}},{"key":"router","value":{"stringValue":"RouterReqsCounter"}},{"key":"service","value":{"stringValue":"test"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1}\],"aggregationTemporality":2,"isMonotonic":true}})`,
 		`({"name":"traefik_router_responses_bytes_total","description":"The total size of responses in bytes handled by a router, partitioned by status code, protocol, and method.","unit":"1","sum":{"dataPoints":\[{"attributes":\[{"key":"code","value":{"stringValue":"404"}},{"key":"method","value":{"stringValue":"GET"}},{"key":"router","value":{"stringValue":"RouterReqsCounter"}},{"key":"service","value":{"stringValue":"test"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1}\],"aggregationTemporality":2,"isMonotonic":true}})`,
-	)
+	}
 
 	registry.RouterReqsCounter().With(nil, "router", "RouterReqsCounter", "service", "test", "code", strconv.Itoa(http.StatusNotFound), "method", http.MethodGet).Add(1)
 	registry.RouterReqsCounter().With(nil, "router", "RouterReqsCounter", "service", "test", "code", strconv.Itoa(http.StatusOK), "method", http.MethodGet).Add(1)
@@ -393,18 +391,17 @@ func TestOpenTelemetry(t *testing.T) {
 	registry.RouterReqDurationHistogram().With("router", "demo", "service", "test", "code", strconv.Itoa(http.StatusOK)).Observe(10000)
 	registry.RouterReqsBytesCounter().With("router", "RouterReqsCounter", "service", "test", "code", strconv.Itoa(http.StatusNotFound), "method", http.MethodGet).Add(1)
 	registry.RouterRespsBytesCounter().With("router", "RouterReqsCounter", "service", "test", "code", strconv.Itoa(http.StatusNotFound), "method", http.MethodGet).Add(1)
-	msgRouter := <-c
 
-	assertMessage(t, *msgRouter, expected)
+	tryAssertMessage(t, c, expectedRouters)
 
-	expected = append(expected,
+	expectedServices := []string{
 		`({"name":"traefik_service_requests_total","description":"How many HTTP requests processed on a service, partitioned by status code, protocol, and method.","unit":"1","sum":{"dataPoints":\[{"attributes":\[{"key":"code","value":{"stringValue":"(?:200|404)"}},{"key":"method","value":{"stringValue":"GET"}},{"key":"service","value":{"stringValue":"ServiceReqsCounter"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1},{"attributes":\[{"key":"code","value":{"stringValue":"(?:200|404)"}},{"key":"method","value":{"stringValue":"GET"}},{"key":"service","value":{"stringValue":"ServiceReqsCounter"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1}\],"aggregationTemporality":2,"isMonotonic":true}})`,
 		`({"name":"traefik_service_requests_tls_total","description":"How many HTTP requests with TLS processed on a service, partitioned by TLS version and TLS cipher.","unit":"1","sum":{"dataPoints":\[{"attributes":\[{"key":"service","value":{"stringValue":"test"}},{"key":"tls_cipher","value":{"stringValue":"bar"}},{"key":"tls_version","value":{"stringValue":"foo"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1}\],"aggregationTemporality":2,"isMonotonic":true}})`,
 		`({"name":"traefik_service_request_duration_seconds","description":"How long it took to process the request on a service, partitioned by status code, protocol, and method.","unit":"ms","histogram":{"dataPoints":\[{"attributes":\[{"key":"code","value":{"stringValue":"200"}},{"key":"service","value":{"stringValue":"test"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","count":"1","sum":10000,"bucketCounts":\["0","0","0","0","0","0","0","0","0","0","0","1"\],"explicitBounds":\[0.005,0.01,0.025,0.05,0.1,0.25,0.5,1,2.5,5,10\],"min":10000,"max":10000}\],"aggregationTemporality":2}})`,
 		`({"name":"traefik_service_server_up","description":"service server is up, described by gauge value of 0 or 1.","unit":"1","gauge":{"dataPoints":\[{"attributes":\[{"key":"service","value":{"stringValue":"test"}},{"key":"url","value":{"stringValue":"http://127.0.0.1"}}\],"timeUnixNano":"[\d]{19}","asDouble":1}\]}})`,
 		`({"name":"traefik_service_requests_bytes_total","description":"The total size of requests in bytes received by a service, partitioned by status code, protocol, and method.","unit":"1","sum":{"dataPoints":\[{"attributes":\[{"key":"code","value":{"stringValue":"404"}},{"key":"method","value":{"stringValue":"GET"}},{"key":"service","value":{"stringValue":"ServiceReqsCounter"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1}\],"aggregationTemporality":2,"isMonotonic":true}})`,
 		`({"name":"traefik_service_responses_bytes_total","description":"The total size of responses in bytes returned by a service, partitioned by status code, protocol, and method.","unit":"1","sum":{"dataPoints":\[{"attributes":\[{"key":"code","value":{"stringValue":"404"}},{"key":"method","value":{"stringValue":"GET"}},{"key":"service","value":{"stringValue":"ServiceReqsCounter"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1}\],"aggregationTemporality":2,"isMonotonic":true}})`,
-	)
+	}
 
 	registry.ServiceReqsCounter().With(nil, "service", "ServiceReqsCounter", "code", strconv.Itoa(http.StatusOK), "method", http.MethodGet).Add(1)
 	registry.ServiceReqsCounter().With(nil, "service", "ServiceReqsCounter", "code", strconv.Itoa(http.StatusNotFound), "method", http.MethodGet).Add(1)
@@ -413,21 +410,19 @@ func TestOpenTelemetry(t *testing.T) {
 	registry.ServiceServerUpGauge().With("service", "test", "url", "http://127.0.0.1").Set(1)
 	registry.ServiceReqsBytesCounter().With("service", "ServiceReqsCounter", "code", strconv.Itoa(http.StatusNotFound), "method", http.MethodGet).Add(1)
 	registry.ServiceRespsBytesCounter().With("service", "ServiceReqsCounter", "code", strconv.Itoa(http.StatusNotFound), "method", http.MethodGet).Add(1)
-	msgService := <-c
 
-	assertMessage(t, *msgService, expected)
+	tryAssertMessage(t, c, expectedServices)
 
-	expected = append(expected,
+	expectedServicesRetries := []string{
 		`({"attributes":\[{"key":"service","value":{"stringValue":"foobar"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1})`,
 		`({"attributes":\[{"key":"service","value":{"stringValue":"test"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":2})`,
-	)
+	}
 
 	registry.ServiceRetriesCounter().With("service", "test").Add(1)
 	registry.ServiceRetriesCounter().With("service", "test").Add(1)
 	registry.ServiceRetriesCounter().With("service", "foobar").Add(1)
-	msgServiceRetries := <-c
 
-	assertMessage(t, *msgServiceRetries, expected)
+	tryAssertMessage(t, c, expectedServicesRetries)
 
 	// We cannot rely on the previous expected pattern,
 	// because this pattern was for matching only one dataPoint in the histogram,
@@ -439,15 +434,46 @@ func TestOpenTelemetry(t *testing.T) {
 
 	registry.EntryPointReqDurationHistogram().With("entrypoint", "myEntrypoint").Observe(10000)
 	registry.EntryPointReqDurationHistogram().With("entrypoint", "myEntrypoint").Observe(20000)
-	msgEntryPointReqDurationHistogram := <-c
 
-	assertMessage(t, *msgEntryPointReqDurationHistogram, expectedEntryPointReqDuration)
+	tryAssertMessage(t, c, expectedEntryPointReqDuration)
-
+}
-	// Stopping OpenTelemetry.
+
-	go func() {
+func assertMessage(t *testing.T, msg string, expected []string) {
-		StopOpenTelemetry()
+	t.Helper()
-	}()
+	errs := verifyMessage(msg, expected)
-
+	for _, err := range errs {
-	// We need to unlock the HTTP Server for the last export call.
+		t.Error(err)
-	<-c
+	}
+}
+
+func tryAssertMessage(t *testing.T, c chan *string, expected []string) {
+	t.Helper()
+
+	var errs []error
+	timeout := time.After(1 * time.Second)
+	for {
+		select {
+		case <-timeout:
+			for _, err := range errs {
+				t.Error(err)
+			}
+		case msg := <-c:
+			errs = verifyMessage(*msg, expected)
+			if len(errs) == 0 {
+				return
+			}
+		}
+	}
+}
+
+func verifyMessage(msg string, expected []string) []error {
+	var errs []error
+	for _, pattern := range expected {
+		re := regexp.MustCompile(pattern)
+		match := re.FindStringSubmatch(msg)
+		if len(match) != 2 {
+			errs = append(errs, fmt.Errorf("Got %q %v, want %q", msg, match, pattern))
+		}
+	}
+	return errs
 }