Merge branch 'v3.0' of github.com:traefik/traefik

docs/content/observability/metrics/opentelemetry.md

@@ -208,7 +208,7 @@ metrics:
 
 #### `path`
 
-_Required, Default="/v1/traces"_
+_Required, Default="/v1/metrics"_
 
 Allows to override the default URL path used for sending metrics.
 This option has no effect when using gRPC transport.
@@ -216,17 +216,17 @@ This option has no effect when using gRPC transport.
 ```yaml tab="File (YAML)"
 metrics:
   openTelemetry:
-    path: /foo/v1/traces
+    path: /foo/v1/metrics
 ```
 
 ```toml tab="File (TOML)"
 [metrics]
   [metrics.openTelemetry]
-    path = "/foo/v1/traces"
+    path = "/foo/v1/metrics"
 ```
 
 ```bash tab="CLI"
---metrics.openTelemetry.path=/foo/v1/traces
+--metrics.openTelemetry.path=/foo/v1/metrics
 ```
 
 #### `tls`

pkg/metrics/opentelemetry.go

@@ -23,6 +23,8 @@ import (
 	"go.opentelemetry.io/otel/metric/unit"
 	sdkmetric "go.opentelemetry.io/otel/sdk/metric"
 	"go.opentelemetry.io/otel/sdk/metric/aggregation"
+	"go.opentelemetry.io/otel/sdk/resource"
+	semconv "go.opentelemetry.io/otel/semconv/v1.12.0"
 	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/encoding/gzip"
 )
@@ -139,11 +141,22 @@ func newOpenTelemetryMeterProvider(ctx context.Context, config *types.OpenTeleme
 		return nil, fmt.Errorf("creating exporter: %w", err)
 	}
 
+	res, err := resource.New(ctx,
+		resource.WithAttributes(semconv.ServiceNameKey.String("traefik")),
+		resource.WithAttributes(semconv.ServiceVersionKey.String(version.Version)),
+		resource.WithFromEnv(),
+		resource.WithTelemetrySDK(),
+	)
+	if err != nil {
+		return nil, fmt.Errorf("building resource: %w", err)
+	}
+
 	opts := []sdkmetric.PeriodicReaderOption{
 		sdkmetric.WithInterval(time.Duration(config.PushInterval)),
 	}
 
 	meterProvider := sdkmetric.NewMeterProvider(
+		sdkmetric.WithResource(res),
 		sdkmetric.WithReader(sdkmetric.NewPeriodicReader(exporter, opts...)),
 		// View to customize histogram buckets and rename a single histogram instrument.
 		sdkmetric.WithView(sdkmetric.NewView(

pkg/metrics/opentelemetry_test.go

@@ -4,7 +4,6 @@ import (
 	"compress/gzip"
 	"context"
 	"encoding/json"
-	"fmt"
 	"io"
 	"net/http"
 	"net/http/httptest"
@@ -17,6 +16,7 @@ import (
 	"github.com/stretchr/testify/require"
 	ptypes "github.com/traefik/paerser/types"
 	"github.com/traefik/traefik/v2/pkg/types"
+	"github.com/traefik/traefik/v2/pkg/version"
 	"go.opentelemetry.io/collector/pdata/pmetric/pmetricotlp"
 	"go.opentelemetry.io/otel/attribute"
 )
@@ -308,8 +308,7 @@ func TestOpenTelemetry(t *testing.T) {
 		bodyStr := string(marshalledReq)
 		c <- &bodyStr
 
-		_, err = fmt.Fprintln(w, "ok")
+		w.WriteHeader(http.StatusOK)
-		require.NoError(t, err)
 	}))
 	defer ts.Close()
 
@@ -330,13 +329,21 @@ func TestOpenTelemetry(t *testing.T) {
 		t.Fatalf("registry should return true for IsEnabled(), IsRouterEnabled() and IsSvcEnabled()")
 	}
 
+	expected := []string{
+		`({"key":"service.name","value":{"stringValue":"traefik"}})`,
+		`({"key":"service.version","value":{"stringValue":"` + version.Version + `"}})`,
+	}
+	msgMisc := <-c
+
+	assertMessage(t, *msgMisc, expected)
+
 	// TODO: the len of startUnixNano is no supposed to be 20, it should be 19
-	expectedServer := []string{
+	expected = append(expected,
 		`({"name":"traefik_config_reloads_total","description":"Config reloads","unit":"1","sum":{"dataPoints":\[{"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1}\],"aggregationTemporality":2,"isMonotonic":true}})`,
 		`({"name":"traefik_config_reloads_failure_total","description":"Config reload failures","unit":"1","sum":{"dataPoints":\[{"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1}\],"aggregationTemporality":2,"isMonotonic":true}})`,
 		`({"name":"traefik_config_last_reload_success","description":"Last config reload success","unit":"ms","gauge":{"dataPoints":\[{"startTimeUnixNano":"[\d]{20}","timeUnixNano":"[\d]{19}","asDouble":1}\]}})`,
 		`({"name":"traefik_config_last_reload_failure","description":"Last config reload failure","unit":"ms","gauge":{"dataPoints":\[{"startTimeUnixNano":"[\d]{20}","timeUnixNano":"[\d]{19}","asDouble":1}\]}})`,
-	}
+	)
 
 	registry.ConfigReloadsCounter().Add(1)
 	registry.ConfigReloadsFailureCounter().Add(1)
@@ -344,23 +351,23 @@ func TestOpenTelemetry(t *testing.T) {
 	registry.LastConfigReloadFailureGauge().Set(1)
 	msgServer := <-c
 
-	assertMessage(t, *msgServer, expectedServer)
+	assertMessage(t, *msgServer, expected)
 
-	expectedTLS := []string{
+	expected = append(expected,
 		`({"name":"traefik_tls_certs_not_after","description":"Certificate expiration timestamp","unit":"ms","gauge":{"dataPoints":\[{"attributes":\[{"key":"key","value":{"stringValue":"value"}}\],"startTimeUnixNano":"[\d]{20}","timeUnixNano":"[\d]{19}","asDouble":1}\]}})`,
-	}
+	)
 
 	registry.TLSCertsNotAfterTimestampGauge().With("key", "value").Set(1)
 	msgTLS := <-c
 
-	assertMessage(t, *msgTLS, expectedTLS)
+	assertMessage(t, *msgTLS, expected)
 
-	expectedEntrypoint := []string{
+	expected = append(expected,
 		`({"name":"traefik_entrypoint_requests_total","description":"How many HTTP requests processed on an entrypoint, partitioned by status code, protocol, and method.","unit":"1","sum":{"dataPoints":\[{"attributes":\[{"key":"code","value":{"stringValue":"200"}},{"key":"entrypoint","value":{"stringValue":"test1"}},{"key":"method","value":{"stringValue":"GET"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1}\],"aggregationTemporality":2,"isMonotonic":true}})`,
 		`({"name":"traefik_entrypoint_requests_tls_total","description":"How many HTTP requests with TLS processed on an entrypoint, partitioned by TLS Version and TLS cipher Used.","unit":"1","sum":{"dataPoints":\[{"attributes":\[{"key":"entrypoint","value":{"stringValue":"test2"}},{"key":"tls_cipher","value":{"stringValue":"bar"}},{"key":"tls_version","value":{"stringValue":"foo"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1}\],"aggregationTemporality":2,"isMonotonic":true}})`,
 		`({"name":"traefik_entrypoint_request_duration_seconds","description":"How long it took to process the request on an entrypoint, partitioned by status code, protocol, and method.","unit":"ms","histogram":{"dataPoints":\[{"attributes":\[{"key":"entrypoint","value":{"stringValue":"test3"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","count":"1","sum":10000,"bucketCounts":\["0","0","0","0","0","0","0","0","0","0","0","1"\],"explicitBounds":\[0.005,0.01,0.025,0.05,0.1,0.25,0.5,1,2.5,5,10\],"min":10000,"max":10000}\],"aggregationTemporality":2}})`,
 		`({"name":"traefik_entrypoint_open_connections","description":"How many open connections exist on an entrypoint, partitioned by method and protocol.","unit":"1","gauge":{"dataPoints":\[{"attributes":\[{"key":"entrypoint","value":{"stringValue":"test4"}}\],"startTimeUnixNano":"[\d]{20}","timeUnixNano":"[\d]{19}","asDouble":1}\]}})`,
-	}
+	)
 
 	registry.EntryPointReqsCounter().With("entrypoint", "test1", "code", strconv.Itoa(http.StatusOK), "method", http.MethodGet).Add(1)
 	registry.EntryPointReqsTLSCounter().With("entrypoint", "test2", "tls_version", "foo", "tls_cipher", "bar").Add(1)
@@ -368,14 +375,14 @@ func TestOpenTelemetry(t *testing.T) {
 	registry.EntryPointOpenConnsGauge().With("entrypoint", "test4").Set(1)
 	msgEntrypoint := <-c
 
-	assertMessage(t, *msgEntrypoint, expectedEntrypoint)
+	assertMessage(t, *msgEntrypoint, expected)
 
-	expectedRouter := []string{
+	expected = append(expected,
 		`({"name":"traefik_router_requests_total","description":"How many HTTP requests are processed on a router, partitioned by service, status code, protocol, and method.","unit":"1","sum":{"dataPoints":\[{"attributes":\[{"key":"code","value":{"stringValue":"(?:200|404)"}},{"key":"method","value":{"stringValue":"GET"}},{"key":"router","value":{"stringValue":"RouterReqsCounter"}},{"key":"service","value":{"stringValue":"test"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1},{"attributes":\[{"key":"code","value":{"stringValue":"(?:200|404)"}},{"key":"method","value":{"stringValue":"GET"}},{"key":"router","value":{"stringValue":"RouterReqsCounter"}},{"key":"service","value":{"stringValue":"test"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1}\],"aggregationTemporality":2,"isMonotonic":true}})`,
 		`({"name":"traefik_router_requests_tls_total","description":"How many HTTP requests with TLS are processed on a router, partitioned by service, TLS Version, and TLS cipher Used.","unit":"1","sum":{"dataPoints":\[{"attributes":\[{"key":"router","value":{"stringValue":"demo"}},{"key":"service","value":{"stringValue":"test"}},{"key":"tls_cipher","value":{"stringValue":"bar"}},{"key":"tls_version","value":{"stringValue":"foo"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1}\],"aggregationTemporality":2,"isMonotonic":true}})`,
 		`({"name":"traefik_router_request_duration_seconds","description":"How long it took to process the request on a router, partitioned by service, status code, protocol, and method.","unit":"ms","histogram":{"dataPoints":\[{"attributes":\[{"key":"code","value":{"stringValue":"200"}},{"key":"router","value":{"stringValue":"demo"}},{"key":"service","value":{"stringValue":"test"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","count":"1","sum":10000,"bucketCounts":\["0","0","0","0","0","0","0","0","0","0","0","1"\],"explicitBounds":\[0.005,0.01,0.025,0.05,0.1,0.25,0.5,1,2.5,5,10\],"min":10000,"max":10000}\],"aggregationTemporality":2}})`,
 		`({"name":"traefik_router_open_connections","description":"How many open connections exist on a router, partitioned by service, method, and protocol.","unit":"1","gauge":{"dataPoints":\[{"attributes":\[{"key":"router","value":{"stringValue":"demo"}},{"key":"service","value":{"stringValue":"test"}}\],"startTimeUnixNano":"[\d]{20}","timeUnixNano":"[\d]{19}","asDouble":1}\]}})`,
-	}
+	)
 
 	registry.RouterReqsCounter().With("router", "RouterReqsCounter", "service", "test", "code", strconv.Itoa(http.StatusNotFound), "method", http.MethodGet).Add(1)
 	registry.RouterReqsCounter().With("router", "RouterReqsCounter", "service", "test", "code", strconv.Itoa(http.StatusOK), "method", http.MethodGet).Add(1)
@@ -384,14 +391,14 @@ func TestOpenTelemetry(t *testing.T) {
 	registry.RouterOpenConnsGauge().With("router", "demo", "service", "test").Set(1)
 	msgRouter := <-c
 
-	assertMessage(t, *msgRouter, expectedRouter)
+	assertMessage(t, *msgRouter, expected)
 
-	expectedService := []string{
+	expected = append(expected,
 		`({"name":"traefik_service_requests_total","description":"How many HTTP requests processed on a service, partitioned by status code, protocol, and method.","unit":"1","sum":{"dataPoints":\[{"attributes":\[{"key":"code","value":{"stringValue":"(?:200|404)"}},{"key":"method","value":{"stringValue":"GET"}},{"key":"service","value":{"stringValue":"ServiceReqsCounter"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1},{"attributes":\[{"key":"code","value":{"stringValue":"(?:200|404)"}},{"key":"method","value":{"stringValue":"GET"}},{"key":"service","value":{"stringValue":"ServiceReqsCounter"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1}\],"aggregationTemporality":2,"isMonotonic":true}})`,
 		`({"name":"traefik_service_requests_tls_total","description":"How many HTTP requests with TLS processed on a service, partitioned by TLS version and TLS cipher.","unit":"1","sum":{"dataPoints":\[{"attributes":\[{"key":"service","value":{"stringValue":"test"}},{"key":"tls_cipher","value":{"stringValue":"bar"}},{"key":"tls_version","value":{"stringValue":"foo"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1}\],"aggregationTemporality":2,"isMonotonic":true}})`,
 		`({"name":"traefik_service_request_duration_seconds","description":"How long it took to process the request on a service, partitioned by status code, protocol, and method.","unit":"ms","histogram":{"dataPoints":\[{"attributes":\[{"key":"code","value":{"stringValue":"200"}},{"key":"service","value":{"stringValue":"test"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","count":"1","sum":10000,"bucketCounts":\["0","0","0","0","0","0","0","0","0","0","0","1"\],"explicitBounds":\[0.005,0.01,0.025,0.05,0.1,0.25,0.5,1,2.5,5,10\],"min":10000,"max":10000}\],"aggregationTemporality":2}})`,
 		`({"name":"traefik_service_server_up","description":"service server is up, described by gauge value of 0 or 1.","unit":"1","gauge":{"dataPoints":\[{"attributes":\[{"key":"service","value":{"stringValue":"test"}},{"key":"url","value":{"stringValue":"http://127.0.0.1"}}\],"startTimeUnixNano":"[\d]{20}","timeUnixNano":"[\d]{19}","asDouble":1}\]}})`,
-	}
+	)
 
 	registry.ServiceReqsCounter().With("service", "ServiceReqsCounter", "code", strconv.Itoa(http.StatusOK), "method", http.MethodGet).Add(1)
 	registry.ServiceReqsCounter().With("service", "ServiceReqsCounter", "code", strconv.Itoa(http.StatusNotFound), "method", http.MethodGet).Add(1)
@@ -400,24 +407,24 @@ func TestOpenTelemetry(t *testing.T) {
 	registry.ServiceServerUpGauge().With("service", "test", "url", "http://127.0.0.1").Set(1)
 	msgService := <-c
 
-	assertMessage(t, *msgService, expectedService)
+	assertMessage(t, *msgService, expected)
 
-	expectedServiceRetries := []string{
+	expected = append(expected,
 		`({"attributes":\[{"key":"service","value":{"stringValue":"foobar"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1})`,
 		`({"attributes":\[{"key":"service","value":{"stringValue":"test"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":2})`,
-	}
+	)
 
 	registry.ServiceRetriesCounter().With("service", "test").Add(1)
 	registry.ServiceRetriesCounter().With("service", "test").Add(1)
 	registry.ServiceRetriesCounter().With("service", "foobar").Add(1)
 	msgServiceRetries := <-c
 
-	assertMessage(t, *msgServiceRetries, expectedServiceRetries)
+	assertMessage(t, *msgServiceRetries, expected)
 
-	expectedServiceOpenConns := []string{
+	expected = append(expected,
 		`({"attributes":\[{"key":"service","value":{"stringValue":"test"}}\],"startTimeUnixNano":"[\d]{20}","timeUnixNano":"[\d]{19}","asDouble":3})`,
 		`({"attributes":\[{"key":"service","value":{"stringValue":"foobar"}}\],"startTimeUnixNano":"[\d]{20}","timeUnixNano":"[\d]{19}","asDouble":1})`,
-	}
+	)
 
 	registry.ServiceOpenConnsGauge().With("service", "test").Set(1)
 	registry.ServiceOpenConnsGauge().With("service", "test").Add(1)
@@ -425,8 +432,12 @@ func TestOpenTelemetry(t *testing.T) {
 	registry.ServiceOpenConnsGauge().With("service", "foobar").Add(1)
 	msgServiceOpenConns := <-c
 
-	assertMessage(t, *msgServiceOpenConns, expectedServiceOpenConns)
+	assertMessage(t, *msgServiceOpenConns, expected)
 
+	// We cannot rely on the previous expected pattern,
+	// because this pattern was for matching only one dataPoint in the histogram,
+	// and as soon as the EntryPointReqDurationHistogram.Observe is called,
+	// it adds a new dataPoint to the histogram.
 	expectedEntryPointReqDuration := []string{
 		`({"attributes":\[{"key":"entrypoint","value":{"stringValue":"myEntrypoint"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","count":"2","sum":30000,"bucketCounts":\["0","0","0","0","0","0","0","0","0","0","0","2"\],"explicitBounds":\[0.005,0.01,0.025,0.05,0.1,0.25,0.5,1,2.5,5,10\],"min":10000,"max":20000})`,
 	}

pkg/tracing/opentelemetry/opentelemetry.go

@@ -16,7 +16,9 @@ import (
 	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
 	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
 	"go.opentelemetry.io/otel/propagation"
+	"go.opentelemetry.io/otel/sdk/resource"
 	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+	semconv "go.opentelemetry.io/otel/semconv/v1.12.0"
 	"go.opentelemetry.io/otel/trace"
 	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/encoding/gzip"
@@ -60,7 +62,20 @@ func (c *Config) Setup(componentName string) (opentracing.Tracer, io.Closer, err
 	bt.SetOpenTelemetryTracer(otel.Tracer(componentName, trace.WithInstrumentationVersion(version.Version)))
 	opentracing.SetGlobalTracer(bt)
 
-	tracerProvider := sdktrace.NewTracerProvider(sdktrace.WithBatcher(exporter))
+	res, err := resource.New(context.Background(),
+		resource.WithAttributes(semconv.ServiceNameKey.String("traefik")),
+		resource.WithAttributes(semconv.ServiceVersionKey.String(version.Version)),
+		resource.WithFromEnv(),
+		resource.WithTelemetrySDK(),
+	)
+	if err != nil {
+		return nil, nil, fmt.Errorf("building resource: %w", err)
+	}
+
+	tracerProvider := sdktrace.NewTracerProvider(
+		sdktrace.WithResource(res),
+		sdktrace.WithBatcher(exporter),
+	)
 	otel.SetTracerProvider(tracerProvider)
 
 	log.Debug().Msg("OpenTelemetry tracer configured")

pkg/tracing/opentelemetry/opentelemetry_test.go

@@ -9,6 +9,7 @@ import (
 	"net/http/httptest"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -17,10 +18,39 @@ import (
 	"go.opentelemetry.io/collector/pdata/ptrace/ptraceotlp"
 )
 
-func TestTraceContextPropagation(t *testing.T) {
+func TestTracing(t *testing.T) {
-	t.Parallel()
+	tests := []struct {
+		desc     string
+		headers  map[string]string
+		assertFn func(*testing.T, string)
+	}{
+		{
+			desc: "service name and version",
+			assertFn: func(t *testing.T, trace string) {
+				t.Helper()
 
-	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				assert.Regexp(t, `({"key":"service.name","value":{"stringValue":"traefik"}})`, trace)
+				assert.Regexp(t, `({"key":"service.version","value":{"stringValue":"dev"}})`, trace)
+			},
+		},
+		{
+			desc: "context propagation",
+			headers: map[string]string{
+				"traceparent": "00-00000000000000000000000000000001-0000000000000001-01",
+				"tracestate":  "foo=bar",
+			},
+			assertFn: func(t *testing.T, trace string) {
+				t.Helper()
+
+				assert.Regexp(t, `("traceId":"00000000000000000000000000000001")`, trace)
+				assert.Regexp(t, `("parentSpanId":"0000000000000001")`, trace)
+				assert.Regexp(t, `("traceState":"foo=bar")`, trace)
+			},
+		},
+	}
+
+	traceCh := make(chan string)
+	collector := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		gzr, err := gzip.NewReader(r.Body)
 		require.NoError(t, err)
 
@@ -34,34 +64,38 @@ func TestTraceContextPropagation(t *testing.T) {
 		marshalledReq, err := json.Marshal(req)
 		require.NoError(t, err)
 
-		bodyStr := string(marshalledReq)
+		traceCh <- string(marshalledReq)
-		assert.Regexp(t, `("traceId":"00000000000000000000000000000001")`, bodyStr)
-		assert.Regexp(t, `("parentSpanId":"0000000000000001")`, bodyStr)
-		assert.Regexp(t, `("traceState":"foo=bar")`, bodyStr)
 	}))
-	defer ts.Close()
+	t.Cleanup(collector.Close)
 
-	cfg := Config{
+	newTracing, err := tracing.NewTracing("", 0, &Config{
-		Address:  strings.TrimPrefix(ts.URL, "http://"),
 		Insecure: true,
-	}
+		Address:  strings.TrimPrefix(collector.URL, "http://"),
-
-	newTracing, err := tracing.NewTracing("", 0, &cfg)
-	require.NoError(t, err)
-	defer newTracing.Close()
-
-	req := httptest.NewRequest(http.MethodGet, "http://www.test.com", nil)
-	req.Header.Set("traceparent", "00-00000000000000000000000000000001-0000000000000001-00")
-	req.Header.Set("tracestate", "foo=bar")
-	rw := httptest.NewRecorder()
-
-	var forwarded bool
-	next := http.HandlerFunc(func(http.ResponseWriter, *http.Request) {
-		forwarded = true
 	})
+	require.NoError(t, err)
+	t.Cleanup(newTracing.Close)
 
-	handler := mtracing.NewEntryPoint(context.Background(), newTracing, "test", next)
+	epHandler := mtracing.NewEntryPoint(context.Background(), newTracing, "test", http.NotFoundHandler())
-	handler.ServeHTTP(rw, req)
 
-	require.True(t, forwarded)
+	for _, test := range tests {
+		t.Run(test.desc, func(t *testing.T) {
+			req := httptest.NewRequest(http.MethodGet, "http://www.test.com", nil)
+			for k, v := range test.headers {
+				req.Header.Set(k, v)
+			}
+
+			rw := httptest.NewRecorder()
+
+			epHandler.ServeHTTP(rw, req)
+
+			select {
+			case <-time.After(10 * time.Second):
+				t.Error("Trace not exported")
+
+			case trace := <-traceCh:
+				assert.Equal(t, http.StatusNotFound, rw.Code)
+				test.assertFn(t, trace)
+			}
+		})
+	}
 }