From f0f5f41fb95b44d244a62c84bd3eb761287e34a6 Mon Sep 17 00:00:00 2001 From: Tom Moulard Date: Fri, 6 Jan 2023 09:10:05 +0100 Subject: [PATCH] Fix OpenTelemetry service name Co-authored-by: Kevin Pollet --- .../observability/metrics/opentelemetry.md | 8 +- pkg/metrics/opentelemetry.go | 13 +++ pkg/metrics/opentelemetry_test.go | 59 ++++++++----- pkg/tracing/opentelemetry/opentelemetry.go | 17 +++- .../opentelemetry/opentelemetry_test.go | 88 +++++++++++++------ 5 files changed, 129 insertions(+), 56 deletions(-) diff --git a/docs/content/observability/metrics/opentelemetry.md b/docs/content/observability/metrics/opentelemetry.md index e3abed40a..af0ba7b20 100644 --- a/docs/content/observability/metrics/opentelemetry.md +++ b/docs/content/observability/metrics/opentelemetry.md @@ -208,7 +208,7 @@ metrics: #### `path` -_Required, Default="/v1/traces"_ +_Required, Default="/v1/metrics"_ Allows to override the default URL path used for sending metrics. This option has no effect when using gRPC transport. @@ -216,17 +216,17 @@ This option has no effect when using gRPC transport. ```yaml tab="File (YAML)" metrics: openTelemetry: - path: /foo/v1/traces + path: /foo/v1/metrics ``` ```toml tab="File (TOML)" [metrics] [metrics.openTelemetry] - path = "/foo/v1/traces" + path = "/foo/v1/metrics" ``` ```bash tab="CLI" ---metrics.openTelemetry.path=/foo/v1/traces +--metrics.openTelemetry.path=/foo/v1/metrics ``` #### `tls` diff --git a/pkg/metrics/opentelemetry.go b/pkg/metrics/opentelemetry.go index 94f170132..000018d67 100644 --- a/pkg/metrics/opentelemetry.go +++ b/pkg/metrics/opentelemetry.go @@ -23,6 +23,8 @@ import ( "go.opentelemetry.io/otel/metric/unit" sdkmetric "go.opentelemetry.io/otel/sdk/metric" "go.opentelemetry.io/otel/sdk/metric/aggregation" + "go.opentelemetry.io/otel/sdk/resource" + semconv "go.opentelemetry.io/otel/semconv/v1.12.0" "google.golang.org/grpc/credentials" "google.golang.org/grpc/encoding/gzip" ) @@ -139,11 +141,22 @@ func newOpenTelemetryMeterProvider(ctx context.Context, config *types.OpenTeleme return nil, fmt.Errorf("creating exporter: %w", err) } + res, err := resource.New(ctx, + resource.WithAttributes(semconv.ServiceNameKey.String("traefik")), + resource.WithAttributes(semconv.ServiceVersionKey.String(version.Version)), + resource.WithFromEnv(), + resource.WithTelemetrySDK(), + ) + if err != nil { + return nil, fmt.Errorf("building resource: %w", err) + } + opts := []sdkmetric.PeriodicReaderOption{ sdkmetric.WithInterval(time.Duration(config.PushInterval)), } meterProvider := sdkmetric.NewMeterProvider( + sdkmetric.WithResource(res), sdkmetric.WithReader(sdkmetric.NewPeriodicReader(exporter, opts...)), // View to customize histogram buckets and rename a single histogram instrument. sdkmetric.WithView(sdkmetric.NewView( diff --git a/pkg/metrics/opentelemetry_test.go b/pkg/metrics/opentelemetry_test.go index 16890b6ba..2f1a35354 100644 --- a/pkg/metrics/opentelemetry_test.go +++ b/pkg/metrics/opentelemetry_test.go @@ -4,7 +4,6 @@ import ( "compress/gzip" "context" "encoding/json" - "fmt" "io" "net/http" "net/http/httptest" @@ -17,6 +16,7 @@ import ( "github.com/stretchr/testify/require" ptypes "github.com/traefik/paerser/types" "github.com/traefik/traefik/v2/pkg/types" + "github.com/traefik/traefik/v2/pkg/version" "go.opentelemetry.io/collector/pdata/pmetric/pmetricotlp" "go.opentelemetry.io/otel/attribute" ) @@ -308,8 +308,7 @@ func TestOpenTelemetry(t *testing.T) { bodyStr := string(marshalledReq) c <- &bodyStr - _, err = fmt.Fprintln(w, "ok") - require.NoError(t, err) + w.WriteHeader(http.StatusOK) })) defer ts.Close() @@ -330,13 +329,21 @@ func TestOpenTelemetry(t *testing.T) { t.Fatalf("registry should return true for IsEnabled(), IsRouterEnabled() and IsSvcEnabled()") } + expected := []string{ + `({"key":"service.name","value":{"stringValue":"traefik"}})`, + `({"key":"service.version","value":{"stringValue":"` + version.Version + `"}})`, + } + msgMisc := <-c + + assertMessage(t, *msgMisc, expected) + // TODO: the len of startUnixNano is no supposed to be 20, it should be 19 - expectedServer := []string{ + expected = append(expected, `({"name":"traefik_config_reloads_total","description":"Config reloads","unit":"1","sum":{"dataPoints":\[{"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1}\],"aggregationTemporality":2,"isMonotonic":true}})`, `({"name":"traefik_config_reloads_failure_total","description":"Config reload failures","unit":"1","sum":{"dataPoints":\[{"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1}\],"aggregationTemporality":2,"isMonotonic":true}})`, `({"name":"traefik_config_last_reload_success","description":"Last config reload success","unit":"ms","gauge":{"dataPoints":\[{"startTimeUnixNano":"[\d]{20}","timeUnixNano":"[\d]{19}","asDouble":1}\]}})`, `({"name":"traefik_config_last_reload_failure","description":"Last config reload failure","unit":"ms","gauge":{"dataPoints":\[{"startTimeUnixNano":"[\d]{20}","timeUnixNano":"[\d]{19}","asDouble":1}\]}})`, - } + ) registry.ConfigReloadsCounter().Add(1) registry.ConfigReloadsFailureCounter().Add(1) @@ -344,23 +351,23 @@ func TestOpenTelemetry(t *testing.T) { registry.LastConfigReloadFailureGauge().Set(1) msgServer := <-c - assertMessage(t, *msgServer, expectedServer) + assertMessage(t, *msgServer, expected) - expectedTLS := []string{ + expected = append(expected, `({"name":"traefik_tls_certs_not_after","description":"Certificate expiration timestamp","unit":"ms","gauge":{"dataPoints":\[{"attributes":\[{"key":"key","value":{"stringValue":"value"}}\],"startTimeUnixNano":"[\d]{20}","timeUnixNano":"[\d]{19}","asDouble":1}\]}})`, - } + ) registry.TLSCertsNotAfterTimestampGauge().With("key", "value").Set(1) msgTLS := <-c - assertMessage(t, *msgTLS, expectedTLS) + assertMessage(t, *msgTLS, expected) - expectedEntrypoint := []string{ + expected = append(expected, `({"name":"traefik_entrypoint_requests_total","description":"How many HTTP requests processed on an entrypoint, partitioned by status code, protocol, and method.","unit":"1","sum":{"dataPoints":\[{"attributes":\[{"key":"code","value":{"stringValue":"200"}},{"key":"entrypoint","value":{"stringValue":"test1"}},{"key":"method","value":{"stringValue":"GET"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1}\],"aggregationTemporality":2,"isMonotonic":true}})`, `({"name":"traefik_entrypoint_requests_tls_total","description":"How many HTTP requests with TLS processed on an entrypoint, partitioned by TLS Version and TLS cipher Used.","unit":"1","sum":{"dataPoints":\[{"attributes":\[{"key":"entrypoint","value":{"stringValue":"test2"}},{"key":"tls_cipher","value":{"stringValue":"bar"}},{"key":"tls_version","value":{"stringValue":"foo"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1}\],"aggregationTemporality":2,"isMonotonic":true}})`, `({"name":"traefik_entrypoint_request_duration_seconds","description":"How long it took to process the request on an entrypoint, partitioned by status code, protocol, and method.","unit":"ms","histogram":{"dataPoints":\[{"attributes":\[{"key":"entrypoint","value":{"stringValue":"test3"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","count":"1","sum":10000,"bucketCounts":\["0","0","0","0","0","0","0","0","0","0","0","1"\],"explicitBounds":\[0.005,0.01,0.025,0.05,0.1,0.25,0.5,1,2.5,5,10\],"min":10000,"max":10000}\],"aggregationTemporality":2}})`, `({"name":"traefik_entrypoint_open_connections","description":"How many open connections exist on an entrypoint, partitioned by method and protocol.","unit":"1","gauge":{"dataPoints":\[{"attributes":\[{"key":"entrypoint","value":{"stringValue":"test4"}}\],"startTimeUnixNano":"[\d]{20}","timeUnixNano":"[\d]{19}","asDouble":1}\]}})`, - } + ) registry.EntryPointReqsCounter().With("entrypoint", "test1", "code", strconv.Itoa(http.StatusOK), "method", http.MethodGet).Add(1) registry.EntryPointReqsTLSCounter().With("entrypoint", "test2", "tls_version", "foo", "tls_cipher", "bar").Add(1) @@ -368,14 +375,14 @@ func TestOpenTelemetry(t *testing.T) { registry.EntryPointOpenConnsGauge().With("entrypoint", "test4").Set(1) msgEntrypoint := <-c - assertMessage(t, *msgEntrypoint, expectedEntrypoint) + assertMessage(t, *msgEntrypoint, expected) - expectedRouter := []string{ + expected = append(expected, `({"name":"traefik_router_requests_total","description":"How many HTTP requests are processed on a router, partitioned by service, status code, protocol, and method.","unit":"1","sum":{"dataPoints":\[{"attributes":\[{"key":"code","value":{"stringValue":"(?:200|404)"}},{"key":"method","value":{"stringValue":"GET"}},{"key":"router","value":{"stringValue":"RouterReqsCounter"}},{"key":"service","value":{"stringValue":"test"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1},{"attributes":\[{"key":"code","value":{"stringValue":"(?:200|404)"}},{"key":"method","value":{"stringValue":"GET"}},{"key":"router","value":{"stringValue":"RouterReqsCounter"}},{"key":"service","value":{"stringValue":"test"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1}\],"aggregationTemporality":2,"isMonotonic":true}})`, `({"name":"traefik_router_requests_tls_total","description":"How many HTTP requests with TLS are processed on a router, partitioned by service, TLS Version, and TLS cipher Used.","unit":"1","sum":{"dataPoints":\[{"attributes":\[{"key":"router","value":{"stringValue":"demo"}},{"key":"service","value":{"stringValue":"test"}},{"key":"tls_cipher","value":{"stringValue":"bar"}},{"key":"tls_version","value":{"stringValue":"foo"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1}\],"aggregationTemporality":2,"isMonotonic":true}})`, `({"name":"traefik_router_request_duration_seconds","description":"How long it took to process the request on a router, partitioned by service, status code, protocol, and method.","unit":"ms","histogram":{"dataPoints":\[{"attributes":\[{"key":"code","value":{"stringValue":"200"}},{"key":"router","value":{"stringValue":"demo"}},{"key":"service","value":{"stringValue":"test"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","count":"1","sum":10000,"bucketCounts":\["0","0","0","0","0","0","0","0","0","0","0","1"\],"explicitBounds":\[0.005,0.01,0.025,0.05,0.1,0.25,0.5,1,2.5,5,10\],"min":10000,"max":10000}\],"aggregationTemporality":2}})`, `({"name":"traefik_router_open_connections","description":"How many open connections exist on a router, partitioned by service, method, and protocol.","unit":"1","gauge":{"dataPoints":\[{"attributes":\[{"key":"router","value":{"stringValue":"demo"}},{"key":"service","value":{"stringValue":"test"}}\],"startTimeUnixNano":"[\d]{20}","timeUnixNano":"[\d]{19}","asDouble":1}\]}})`, - } + ) registry.RouterReqsCounter().With("router", "RouterReqsCounter", "service", "test", "code", strconv.Itoa(http.StatusNotFound), "method", http.MethodGet).Add(1) registry.RouterReqsCounter().With("router", "RouterReqsCounter", "service", "test", "code", strconv.Itoa(http.StatusOK), "method", http.MethodGet).Add(1) @@ -384,14 +391,14 @@ func TestOpenTelemetry(t *testing.T) { registry.RouterOpenConnsGauge().With("router", "demo", "service", "test").Set(1) msgRouter := <-c - assertMessage(t, *msgRouter, expectedRouter) + assertMessage(t, *msgRouter, expected) - expectedService := []string{ + expected = append(expected, `({"name":"traefik_service_requests_total","description":"How many HTTP requests processed on a service, partitioned by status code, protocol, and method.","unit":"1","sum":{"dataPoints":\[{"attributes":\[{"key":"code","value":{"stringValue":"(?:200|404)"}},{"key":"method","value":{"stringValue":"GET"}},{"key":"service","value":{"stringValue":"ServiceReqsCounter"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1},{"attributes":\[{"key":"code","value":{"stringValue":"(?:200|404)"}},{"key":"method","value":{"stringValue":"GET"}},{"key":"service","value":{"stringValue":"ServiceReqsCounter"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1}\],"aggregationTemporality":2,"isMonotonic":true}})`, `({"name":"traefik_service_requests_tls_total","description":"How many HTTP requests with TLS processed on a service, partitioned by TLS version and TLS cipher.","unit":"1","sum":{"dataPoints":\[{"attributes":\[{"key":"service","value":{"stringValue":"test"}},{"key":"tls_cipher","value":{"stringValue":"bar"}},{"key":"tls_version","value":{"stringValue":"foo"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1}\],"aggregationTemporality":2,"isMonotonic":true}})`, `({"name":"traefik_service_request_duration_seconds","description":"How long it took to process the request on a service, partitioned by status code, protocol, and method.","unit":"ms","histogram":{"dataPoints":\[{"attributes":\[{"key":"code","value":{"stringValue":"200"}},{"key":"service","value":{"stringValue":"test"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","count":"1","sum":10000,"bucketCounts":\["0","0","0","0","0","0","0","0","0","0","0","1"\],"explicitBounds":\[0.005,0.01,0.025,0.05,0.1,0.25,0.5,1,2.5,5,10\],"min":10000,"max":10000}\],"aggregationTemporality":2}})`, `({"name":"traefik_service_server_up","description":"service server is up, described by gauge value of 0 or 1.","unit":"1","gauge":{"dataPoints":\[{"attributes":\[{"key":"service","value":{"stringValue":"test"}},{"key":"url","value":{"stringValue":"http://127.0.0.1"}}\],"startTimeUnixNano":"[\d]{20}","timeUnixNano":"[\d]{19}","asDouble":1}\]}})`, - } + ) registry.ServiceReqsCounter().With("service", "ServiceReqsCounter", "code", strconv.Itoa(http.StatusOK), "method", http.MethodGet).Add(1) registry.ServiceReqsCounter().With("service", "ServiceReqsCounter", "code", strconv.Itoa(http.StatusNotFound), "method", http.MethodGet).Add(1) @@ -400,24 +407,24 @@ func TestOpenTelemetry(t *testing.T) { registry.ServiceServerUpGauge().With("service", "test", "url", "http://127.0.0.1").Set(1) msgService := <-c - assertMessage(t, *msgService, expectedService) + assertMessage(t, *msgService, expected) - expectedServiceRetries := []string{ + expected = append(expected, `({"attributes":\[{"key":"service","value":{"stringValue":"foobar"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1})`, `({"attributes":\[{"key":"service","value":{"stringValue":"test"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":2})`, - } + ) registry.ServiceRetriesCounter().With("service", "test").Add(1) registry.ServiceRetriesCounter().With("service", "test").Add(1) registry.ServiceRetriesCounter().With("service", "foobar").Add(1) msgServiceRetries := <-c - assertMessage(t, *msgServiceRetries, expectedServiceRetries) + assertMessage(t, *msgServiceRetries, expected) - expectedServiceOpenConns := []string{ + expected = append(expected, `({"attributes":\[{"key":"service","value":{"stringValue":"test"}}\],"startTimeUnixNano":"[\d]{20}","timeUnixNano":"[\d]{19}","asDouble":3})`, `({"attributes":\[{"key":"service","value":{"stringValue":"foobar"}}\],"startTimeUnixNano":"[\d]{20}","timeUnixNano":"[\d]{19}","asDouble":1})`, - } + ) registry.ServiceOpenConnsGauge().With("service", "test").Set(1) registry.ServiceOpenConnsGauge().With("service", "test").Add(1) @@ -425,8 +432,12 @@ func TestOpenTelemetry(t *testing.T) { registry.ServiceOpenConnsGauge().With("service", "foobar").Add(1) msgServiceOpenConns := <-c - assertMessage(t, *msgServiceOpenConns, expectedServiceOpenConns) + assertMessage(t, *msgServiceOpenConns, expected) + // We cannot rely on the previous expected pattern, + // because this pattern was for matching only one dataPoint in the histogram, + // and as soon as the EntryPointReqDurationHistogram.Observe is called, + // it adds a new dataPoint to the histogram. expectedEntryPointReqDuration := []string{ `({"attributes":\[{"key":"entrypoint","value":{"stringValue":"myEntrypoint"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","count":"2","sum":30000,"bucketCounts":\["0","0","0","0","0","0","0","0","0","0","0","2"\],"explicitBounds":\[0.005,0.01,0.025,0.05,0.1,0.25,0.5,1,2.5,5,10\],"min":10000,"max":20000})`, } diff --git a/pkg/tracing/opentelemetry/opentelemetry.go b/pkg/tracing/opentelemetry/opentelemetry.go index d42770ae3..6c568b050 100644 --- a/pkg/tracing/opentelemetry/opentelemetry.go +++ b/pkg/tracing/opentelemetry/opentelemetry.go @@ -16,7 +16,9 @@ import ( "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/sdk/resource" sdktrace "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.12.0" "go.opentelemetry.io/otel/trace" "google.golang.org/grpc/credentials" "google.golang.org/grpc/encoding/gzip" @@ -60,7 +62,20 @@ func (c *Config) Setup(componentName string) (opentracing.Tracer, io.Closer, err bt.SetOpenTelemetryTracer(otel.Tracer(componentName, trace.WithInstrumentationVersion(version.Version))) opentracing.SetGlobalTracer(bt) - tracerProvider := sdktrace.NewTracerProvider(sdktrace.WithBatcher(exporter)) + res, err := resource.New(context.Background(), + resource.WithAttributes(semconv.ServiceNameKey.String("traefik")), + resource.WithAttributes(semconv.ServiceVersionKey.String(version.Version)), + resource.WithFromEnv(), + resource.WithTelemetrySDK(), + ) + if err != nil { + return nil, nil, fmt.Errorf("building resource: %w", err) + } + + tracerProvider := sdktrace.NewTracerProvider( + sdktrace.WithResource(res), + sdktrace.WithBatcher(exporter), + ) otel.SetTracerProvider(tracerProvider) log.Debug().Msg("OpenTelemetry tracer configured") diff --git a/pkg/tracing/opentelemetry/opentelemetry_test.go b/pkg/tracing/opentelemetry/opentelemetry_test.go index 9d76f71d3..9921586ee 100644 --- a/pkg/tracing/opentelemetry/opentelemetry_test.go +++ b/pkg/tracing/opentelemetry/opentelemetry_test.go @@ -9,6 +9,7 @@ import ( "net/http/httptest" "strings" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -17,10 +18,39 @@ import ( "go.opentelemetry.io/collector/pdata/ptrace/ptraceotlp" ) -func TestTraceContextPropagation(t *testing.T) { - t.Parallel() +func TestTracing(t *testing.T) { + tests := []struct { + desc string + headers map[string]string + assertFn func(*testing.T, string) + }{ + { + desc: "service name and version", + assertFn: func(t *testing.T, trace string) { + t.Helper() - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Regexp(t, `({"key":"service.name","value":{"stringValue":"traefik"}})`, trace) + assert.Regexp(t, `({"key":"service.version","value":{"stringValue":"dev"}})`, trace) + }, + }, + { + desc: "context propagation", + headers: map[string]string{ + "traceparent": "00-00000000000000000000000000000001-0000000000000001-01", + "tracestate": "foo=bar", + }, + assertFn: func(t *testing.T, trace string) { + t.Helper() + + assert.Regexp(t, `("traceId":"00000000000000000000000000000001")`, trace) + assert.Regexp(t, `("parentSpanId":"0000000000000001")`, trace) + assert.Regexp(t, `("traceState":"foo=bar")`, trace) + }, + }, + } + + traceCh := make(chan string) + collector := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { gzr, err := gzip.NewReader(r.Body) require.NoError(t, err) @@ -34,34 +64,38 @@ func TestTraceContextPropagation(t *testing.T) { marshalledReq, err := json.Marshal(req) require.NoError(t, err) - bodyStr := string(marshalledReq) - assert.Regexp(t, `("traceId":"00000000000000000000000000000001")`, bodyStr) - assert.Regexp(t, `("parentSpanId":"0000000000000001")`, bodyStr) - assert.Regexp(t, `("traceState":"foo=bar")`, bodyStr) + traceCh <- string(marshalledReq) })) - defer ts.Close() + t.Cleanup(collector.Close) - cfg := Config{ - Address: strings.TrimPrefix(ts.URL, "http://"), + newTracing, err := tracing.NewTracing("", 0, &Config{ Insecure: true, - } - - newTracing, err := tracing.NewTracing("", 0, &cfg) - require.NoError(t, err) - defer newTracing.Close() - - req := httptest.NewRequest(http.MethodGet, "http://www.test.com", nil) - req.Header.Set("traceparent", "00-00000000000000000000000000000001-0000000000000001-00") - req.Header.Set("tracestate", "foo=bar") - rw := httptest.NewRecorder() - - var forwarded bool - next := http.HandlerFunc(func(http.ResponseWriter, *http.Request) { - forwarded = true + Address: strings.TrimPrefix(collector.URL, "http://"), }) + require.NoError(t, err) + t.Cleanup(newTracing.Close) - handler := mtracing.NewEntryPoint(context.Background(), newTracing, "test", next) - handler.ServeHTTP(rw, req) + epHandler := mtracing.NewEntryPoint(context.Background(), newTracing, "test", http.NotFoundHandler()) - require.True(t, forwarded) + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "http://www.test.com", nil) + for k, v := range test.headers { + req.Header.Set(k, v) + } + + rw := httptest.NewRecorder() + + epHandler.ServeHTTP(rw, req) + + select { + case <-time.After(10 * time.Second): + t.Error("Trace not exported") + + case trace := <-traceCh: + assert.Equal(t, http.StatusNotFound, rw.Code) + test.assertFn(t, trace) + } + }) + } }