diff --git a/cmd/traefik/traefik.go b/cmd/traefik/traefik.go index 821f391ed..963790cbc 100644 --- a/cmd/traefik/traefik.go +++ b/cmd/traefik/traefik.go @@ -46,7 +46,6 @@ import ( "github.com/traefik/traefik/v3/pkg/tracing" "github.com/traefik/traefik/v3/pkg/types" "github.com/traefik/traefik/v3/pkg/version" - "go.opentelemetry.io/otel/trace" ) func main() { @@ -563,7 +562,7 @@ func setupAccessLog(conf *types.AccessLog) *accesslog.Handler { return accessLoggerMiddleware } -func setupTracing(conf *static.Tracing) (trace.Tracer, io.Closer) { +func setupTracing(conf *static.Tracing) (*tracing.Tracer, io.Closer) { if conf == nil { return nil, nil } diff --git a/docs/content/observability/tracing/overview.md b/docs/content/observability/tracing/overview.md index 901377150..72dd20cd6 100644 --- a/docs/content/observability/tracing/overview.md +++ b/docs/content/observability/tracing/overview.md @@ -116,3 +116,47 @@ tracing: --tracing.globalAttributes.attr1=foo --tracing.globalAttributes.attr2=bar ``` + +#### `capturedRequestHeaders` + +_Optional, Default=empty_ + +Defines the list of request headers to add as attributes. +It applies to client and server kind spans. + +```yaml tab="File (YAML)" +tracing: + capturedRequestHeaders: + - X-CustomHeader +``` + +```toml tab="File (TOML)" +[tracing] + capturedRequestHeaders = ["X-CustomHeader"] +``` + +```bash tab="CLI" +--tracing.capturedRequestHeaders[0]=X-CustomHeader +``` + +#### `capturedResponseHeaders` + +_Optional, Default=empty_ + +Defines the list of response headers to add as attributes. +It applies to client and server kind spans. + +```yaml tab="File (YAML)" +tracing: + capturedResponseHeaders: + - X-CustomHeader +``` + +```toml tab="File (TOML)" +[tracing] + capturedResponseHeaders = ["X-CustomHeader"] +``` + +```bash tab="CLI" +--tracing.capturedResponseHeaders[0]=X-CustomHeader +``` \ No newline at end of file diff --git a/docs/content/reference/static-configuration/cli-ref.md b/docs/content/reference/static-configuration/cli-ref.md index d78150a72..e9d97565b 100644 --- a/docs/content/reference/static-configuration/cli-ref.md +++ b/docs/content/reference/static-configuration/cli-ref.md @@ -1017,6 +1017,12 @@ OpenTracing configuration. (Default: ```false```) `--tracing.addinternals`: Enables tracing for internal services (ping, dashboard, etc...). (Default: ```false```) +`--tracing.capturedrequestheaders`: +Request headers to add as attributes for server and client spans. + +`--tracing.capturedresponseheaders`: +Response headers to add as attributes for server and client spans. + `--tracing.globalattributes.`: Defines additional attributes (key:value) on all spans. diff --git a/docs/content/reference/static-configuration/env-ref.md b/docs/content/reference/static-configuration/env-ref.md index 7947b9ddd..dc5c94d2e 100644 --- a/docs/content/reference/static-configuration/env-ref.md +++ b/docs/content/reference/static-configuration/env-ref.md @@ -1017,6 +1017,12 @@ OpenTracing configuration. (Default: ```false```) `TRAEFIK_TRACING_ADDINTERNALS`: Enables tracing for internal services (ping, dashboard, etc...). (Default: ```false```) +`TRAEFIK_TRACING_CAPTUREDREQUESTHEADERS`: +Request headers to add as attributes for server and client spans. + +`TRAEFIK_TRACING_CAPTUREDRESPONSEHEADERS`: +Response headers to add as attributes for server and client spans. + `TRAEFIK_TRACING_GLOBALATTRIBUTES_`: Defines additional attributes (key:value) on all spans. diff --git a/docs/content/reference/static-configuration/file.toml b/docs/content/reference/static-configuration/file.toml index 504d7c9b6..cef7d56e9 100644 --- a/docs/content/reference/static-configuration/file.toml +++ b/docs/content/reference/static-configuration/file.toml @@ -380,6 +380,8 @@ [tracing] serviceName = "foobar" + capturedRequestHeaders = ["foobar", "foobar"] + capturedResponseHeaders = ["foobar", "foobar"] sampleRate = 42.0 addInternals = true [tracing.globalAttributes] diff --git a/docs/content/reference/static-configuration/file.yaml b/docs/content/reference/static-configuration/file.yaml index ffb9d75d8..84404356d 100644 --- a/docs/content/reference/static-configuration/file.yaml +++ b/docs/content/reference/static-configuration/file.yaml @@ -418,6 +418,12 @@ tracing: globalAttributes: name0: foobar name1: foobar + capturedRequestHeaders: + - foobar + - foobar + capturedResponseHeaders: + - foobar + - foobar sampleRate: 42 addInternals: true otlp: diff --git a/pkg/config/static/static_config.go b/pkg/config/static/static_config.go index 57af7a165..81d3401fd 100644 --- a/pkg/config/static/static_config.go +++ b/pkg/config/static/static_config.go @@ -193,10 +193,12 @@ func (a *LifeCycle) SetDefaults() { // Tracing holds the tracing configuration. type Tracing struct { - ServiceName string `description:"Set the name for this service." json:"serviceName,omitempty" toml:"serviceName,omitempty" yaml:"serviceName,omitempty" export:"true"` - GlobalAttributes map[string]string `description:"Defines additional attributes (key:value) on all spans." json:"globalAttributes,omitempty" toml:"globalAttributes,omitempty" yaml:"globalAttributes,omitempty" export:"true"` - SampleRate float64 `description:"Sets the rate between 0.0 and 1.0 of requests to trace." json:"sampleRate,omitempty" toml:"sampleRate,omitempty" yaml:"sampleRate,omitempty" export:"true"` - AddInternals bool `description:"Enables tracing for internal services (ping, dashboard, etc...)." json:"addInternals,omitempty" toml:"addInternals,omitempty" yaml:"addInternals,omitempty" export:"true"` + ServiceName string `description:"Set the name for this service." json:"serviceName,omitempty" toml:"serviceName,omitempty" yaml:"serviceName,omitempty" export:"true"` + GlobalAttributes map[string]string `description:"Defines additional attributes (key:value) on all spans." json:"globalAttributes,omitempty" toml:"globalAttributes,omitempty" yaml:"globalAttributes,omitempty" export:"true"` + CapturedRequestHeaders []string `description:"Request headers to add as attributes for server and client spans." json:"capturedRequestHeaders,omitempty" toml:"capturedRequestHeaders,omitempty" yaml:"capturedRequestHeaders,omitempty" export:"true"` + CapturedResponseHeaders []string `description:"Response headers to add as attributes for server and client spans." json:"capturedResponseHeaders,omitempty" toml:"capturedResponseHeaders,omitempty" yaml:"capturedResponseHeaders,omitempty" export:"true"` + SampleRate float64 `description:"Sets the rate between 0.0 and 1.0 of requests to trace." json:"sampleRate,omitempty" toml:"sampleRate,omitempty" yaml:"sampleRate,omitempty" export:"true"` + AddInternals bool `description:"Enables tracing for internal services (ping, dashboard, etc...)." json:"addInternals,omitempty" toml:"addInternals,omitempty" yaml:"addInternals,omitempty" export:"true"` OTLP *opentelemetry.Config `description:"Settings for OpenTelemetry." json:"otlp,omitempty" toml:"otlp,omitempty" yaml:"otlp,omitempty" label:"allowEmpty" file:"allowEmpty" export:"true"` } diff --git a/pkg/middlewares/auth/forward.go b/pkg/middlewares/auth/forward.go index daa2dbd27..4f9e207e5 100644 --- a/pkg/middlewares/auth/forward.go +++ b/pkg/middlewares/auth/forward.go @@ -131,8 +131,11 @@ func (fa *forwardAuth) ServeHTTP(rw http.ResponseWriter, req *http.Request) { return } + writeHeader(req, forwardReq, fa.trustForwardHeader, fa.authRequestHeaders) + var forwardSpan trace.Span - if tracer := tracing.TracerFromContext(req.Context()); tracer != nil { + var tracer *tracing.Tracer + if tracer = tracing.TracerFromContext(req.Context()); tracer != nil { var tracingCtx context.Context tracingCtx, forwardSpan = tracer.Start(req.Context(), "AuthRequest", trace.WithSpanKind(trace.SpanKindClient)) defer forwardSpan.End() @@ -140,11 +143,9 @@ func (fa *forwardAuth) ServeHTTP(rw http.ResponseWriter, req *http.Request) { forwardReq = forwardReq.WithContext(tracingCtx) tracing.InjectContextIntoCarrier(forwardReq) - tracing.LogClientRequest(forwardSpan, forwardReq) + tracer.CaptureClientRequest(forwardSpan, forwardReq) } - writeHeader(req, forwardReq, fa.trustForwardHeader, fa.authRequestHeaders) - forwardResponse, forwardErr := fa.client.Do(forwardReq) if forwardErr != nil { logMessage := fmt.Sprintf("Error calling %s. Cause: %s", fa.address, forwardErr) @@ -197,7 +198,7 @@ func (fa *forwardAuth) ServeHTTP(rw http.ResponseWriter, req *http.Request) { rw.Header().Set("Location", redirectURL.String()) } - tracing.LogResponseCode(forwardSpan, forwardResponse.StatusCode, trace.SpanKindClient) + tracer.CaptureResponse(forwardSpan, forwardResponse.Header, forwardResponse.StatusCode, trace.SpanKindClient) rw.WriteHeader(forwardResponse.StatusCode) if _, err = rw.Write(body); err != nil { @@ -228,7 +229,7 @@ func (fa *forwardAuth) ServeHTTP(rw http.ResponseWriter, req *http.Request) { } } - tracing.LogResponseCode(forwardSpan, forwardResponse.StatusCode, trace.SpanKindClient) + tracer.CaptureResponse(forwardSpan, forwardResponse.Header, forwardResponse.StatusCode, trace.SpanKindClient) req.RequestURI = req.URL.RequestURI() diff --git a/pkg/middlewares/auth/forward_test.go b/pkg/middlewares/auth/forward_test.go index 57e07314f..ed3e2bb2b 100644 --- a/pkg/middlewares/auth/forward_test.go +++ b/pkg/middlewares/auth/forward_test.go @@ -4,27 +4,25 @@ import ( "context" "fmt" "io" + "net" "net/http" "net/http/httptest" + "net/url" + "strconv" "testing" - "github.com/containous/alice" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/traefik/traefik/v3/pkg/config/dynamic" - "github.com/traefik/traefik/v3/pkg/config/static" - tracingMiddleware "github.com/traefik/traefik/v3/pkg/middlewares/tracing" "github.com/traefik/traefik/v3/pkg/testhelpers" "github.com/traefik/traefik/v3/pkg/tracing" - "github.com/traefik/traefik/v3/pkg/tracing/opentelemetry" - "github.com/traefik/traefik/v3/pkg/types" - "github.com/traefik/traefik/v3/pkg/version" "github.com/vulcand/oxy/v2/forward" + "go.opentelemetry.io/contrib/propagators/autoprop" "go.opentelemetry.io/otel" - "go.opentelemetry.io/otel/sdk/resource" - sdktrace "go.opentelemetry.io/otel/sdk/trace" - "go.opentelemetry.io/otel/sdk/trace/tracetest" - semconv "go.opentelemetry.io/otel/semconv/v1.21.0" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" + "go.opentelemetry.io/otel/trace/embedded" ) func TestForwardAuthFail(t *testing.T) { @@ -466,64 +464,146 @@ func Test_writeHeader(t *testing.T) { } } -func TestForwardAuthUsesTracing(t *testing.T) { +func TestForwardAuthTracing(t *testing.T) { + type expected struct { + name string + attributes []attribute.KeyValue + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.Header.Get("Traceparent") == "" { t.Errorf("expected Traceparent header to be present in request") } + + w.Header().Set("X-Bar", "foo") + w.Header().Add("X-Bar", "bar") + w.WriteHeader(http.StatusNotFound) })) t.Cleanup(server.Close) - next := http.Handler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {})) - - auth := dynamic.ForwardAuth{ - Address: server.URL, - } - - exporter := tracetest.NewInMemoryExporter() - - tres, err := resource.New(context.Background(), - resource.WithAttributes(semconv.ServiceNameKey.String("traefik")), - resource.WithAttributes(semconv.ServiceVersionKey.String(version.Version)), - resource.WithFromEnv(), - resource.WithTelemetrySDK(), - ) + parse, err := url.Parse(server.URL) require.NoError(t, err) - tracerProvider := sdktrace.NewTracerProvider( - sdktrace.WithSampler(sdktrace.AlwaysSample()), - sdktrace.WithResource(tres), - sdktrace.WithBatcher(exporter), - ) - otel.SetTracerProvider(tracerProvider) + _, serverPort, err := net.SplitHostPort(parse.Host) + require.NoError(t, err) - config := &static.Tracing{ - ServiceName: "testApp", - SampleRate: 1, - OTLP: &opentelemetry.Config{ - HTTP: &types.OtelHTTP{ - Endpoint: "http://127.0.0.1:8080", + serverPortInt, err := strconv.Atoi(serverPort) + require.NoError(t, err) + + testCases := []struct { + desc string + expected []expected + }{ + { + desc: "basic test", + expected: []expected{ + { + name: "initial", + attributes: []attribute.KeyValue{ + attribute.String("span.kind", "unspecified"), + }, + }, + { + name: "AuthRequest", + attributes: []attribute.KeyValue{ + attribute.String("span.kind", "client"), + attribute.String("http.request.method", "GET"), + attribute.String("network.protocol.version", "1.1"), + attribute.String("url.full", server.URL), + attribute.String("url.scheme", "http"), + attribute.String("user_agent.original", ""), + attribute.String("network.peer.address", "127.0.0.1"), + attribute.String("network.peer.port", serverPort), + attribute.String("server.address", "127.0.0.1"), + attribute.Int64("server.port", int64(serverPortInt)), + attribute.StringSlice("http.request.header.x-foo", []string{"foo", "bar"}), + attribute.Int64("http.response.status_code", int64(404)), + attribute.StringSlice("http.response.header.x-bar", []string{"foo", "bar"}), + }, + }, }, }, } - tr, closer, err := tracing.NewTracing(config) - require.NoError(t, err) - t.Cleanup(func() { - _ = closer.Close() - }) - next, err = NewForward(context.Background(), next, auth, "authTest") - require.NoError(t, err) + for _, test := range testCases { + t.Run(test.desc, func(t *testing.T) { + next := http.Handler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {})) - chain := alice.New(tracingMiddleware.WrapEntryPointHandler(context.Background(), tr, "tracingTest")) - next, err = chain.Then(next) - require.NoError(t, err) + auth := dynamic.ForwardAuth{ + Address: server.URL, + AuthRequestHeaders: []string{"X-Foo"}, + } + next, err := NewForward(context.Background(), next, auth, "authTest") + require.NoError(t, err) - ts := httptest.NewServer(next) - t.Cleanup(ts.Close) + req := httptest.NewRequest(http.MethodGet, "http://www.test.com/search?q=Opentelemetry", nil) + req.RemoteAddr = "10.0.0.1:1234" + req.Header.Set("User-Agent", "forward-test") + req.Header.Set("X-Forwarded-Proto", "http") + req.Header.Set("X-Foo", "foo") + req.Header.Add("X-Foo", "bar") - req := testhelpers.MustNewRequest(http.MethodGet, ts.URL, nil) - res, err := http.DefaultClient.Do(req) - require.NoError(t, err) - assert.Equal(t, http.StatusOK, res.StatusCode) + otel.SetTextMapPropagator(autoprop.NewTextMapPropagator()) + + mockTracer := &mockTracer{} + tracer := tracing.NewTracer(mockTracer, []string{"X-Foo"}, []string{"X-Bar"}) + initialCtx, initialSpan := tracer.Start(req.Context(), "initial") + defer initialSpan.End() + req = req.WithContext(initialCtx) + + recorder := httptest.NewRecorder() + next.ServeHTTP(recorder, req) + + for i, span := range mockTracer.spans { + assert.Equal(t, test.expected[i].name, span.name) + assert.Equal(t, test.expected[i].attributes, span.attributes) + } + }) + } +} + +type mockTracer struct { + embedded.Tracer + + spans []*mockSpan +} + +var _ trace.Tracer = &mockTracer{} + +func (t *mockTracer) Start(ctx context.Context, name string, opts ...trace.SpanStartOption) (context.Context, trace.Span) { + config := trace.NewSpanStartConfig(opts...) + span := &mockSpan{} + span.SetName(name) + span.SetAttributes(attribute.String("span.kind", config.SpanKind().String())) + span.SetAttributes(config.Attributes()...) + t.spans = append(t.spans, span) + return trace.ContextWithSpan(ctx, span), span +} + +// mockSpan is an implementation of Span that preforms no operations. +type mockSpan struct { + embedded.Span + + name string + attributes []attribute.KeyValue +} + +var _ trace.Span = &mockSpan{} + +func (*mockSpan) SpanContext() trace.SpanContext { + return trace.NewSpanContext(trace.SpanContextConfig{TraceID: trace.TraceID{1}, SpanID: trace.SpanID{1}}) +} +func (*mockSpan) IsRecording() bool { return false } +func (s *mockSpan) SetStatus(_ codes.Code, _ string) {} +func (s *mockSpan) SetAttributes(kv ...attribute.KeyValue) { + s.attributes = append(s.attributes, kv...) +} +func (s *mockSpan) End(...trace.SpanEndOption) {} +func (s *mockSpan) RecordError(_ error, _ ...trace.EventOption) {} +func (s *mockSpan) AddEvent(_ string, _ ...trace.EventOption) {} + +func (s *mockSpan) SetName(name string) { s.name = name } + +func (s *mockSpan) TracerProvider() trace.TracerProvider { + return nil } diff --git a/pkg/middlewares/tracing/entrypoint.go b/pkg/middlewares/tracing/entrypoint.go index 5ea32ee8f..1d93f452b 100644 --- a/pkg/middlewares/tracing/entrypoint.go +++ b/pkg/middlewares/tracing/entrypoint.go @@ -2,6 +2,7 @@ package tracing import ( "context" + "errors" "net/http" "github.com/containous/alice" @@ -16,20 +17,24 @@ const ( ) type entryPointTracing struct { - tracer trace.Tracer + tracer *tracing.Tracer entryPoint string next http.Handler } // WrapEntryPointHandler Wraps tracing to alice.Constructor. -func WrapEntryPointHandler(ctx context.Context, tracer trace.Tracer, entryPointName string) alice.Constructor { +func WrapEntryPointHandler(ctx context.Context, tracer *tracing.Tracer, entryPointName string) alice.Constructor { return func(next http.Handler) (http.Handler, error) { + if tracer == nil { + return nil, errors.New("unexpected nil tracer") + } + return newEntryPoint(ctx, tracer, entryPointName, next), nil } } // newEntryPoint creates a new tracing middleware for incoming requests. -func newEntryPoint(ctx context.Context, tracer trace.Tracer, entryPointName string, next http.Handler) http.Handler { +func newEntryPoint(ctx context.Context, tracer *tracing.Tracer, entryPointName string, next http.Handler) http.Handler { middlewares.GetLogger(ctx, "tracing", entryPointTypeName).Debug().Msg("Creating middleware") return &entryPointTracing{ @@ -48,10 +53,10 @@ func (e *entryPointTracing) ServeHTTP(rw http.ResponseWriter, req *http.Request) span.SetAttributes(attribute.String("entry_point", e.entryPoint)) - tracing.LogServerRequest(span, req) + e.tracer.CaptureServerRequest(span, req) recorder := newStatusCodeRecorder(rw, http.StatusOK) e.next.ServeHTTP(recorder, req) - tracing.LogResponseCode(span, recorder.Status(), trace.SpanKindServer) + e.tracer.CaptureResponse(span, recorder.Header(), recorder.Status(), trace.SpanKindServer) } diff --git a/pkg/middlewares/tracing/entrypoint_test.go b/pkg/middlewares/tracing/entrypoint_test.go index b3cb60d67..002842662 100644 --- a/pkg/middlewares/tracing/entrypoint_test.go +++ b/pkg/middlewares/tracing/entrypoint_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/traefik/traefik/v3/pkg/tracing" "go.opentelemetry.io/otel/attribute" ) @@ -42,7 +43,9 @@ func TestEntryPointMiddleware(t *testing.T) { attribute.String("client.address", "10.0.0.1"), attribute.Int64("client.port", int64(1234)), attribute.String("client.socket.address", ""), + attribute.StringSlice("http.request.header.x-foo", []string{"foo", "bar"}), attribute.Int64("http.response.status_code", int64(404)), + attribute.StringSlice("http.response.header.x-bar", []string{"foo", "bar"}), }, }, }, @@ -55,17 +58,20 @@ func TestEntryPointMiddleware(t *testing.T) { req.RemoteAddr = "10.0.0.1:1234" req.Header.Set("User-Agent", "entrypoint-test") req.Header.Set("X-Forwarded-Proto", "http") + req.Header.Set("X-Foo", "foo") + req.Header.Add("X-Foo", "bar") next := http.HandlerFunc(func(rw http.ResponseWriter, _ *http.Request) { + rw.Header().Set("X-Bar", "foo") + rw.Header().Add("X-Bar", "bar") rw.WriteHeader(http.StatusNotFound) }) - tracer := &mockTracer{} - - handler := newEntryPoint(context.Background(), tracer, test.entryPoint, next) + mockTracer := &mockTracer{} + handler := newEntryPoint(context.Background(), tracing.NewTracer(mockTracer, []string{"X-Foo"}, []string{"X-Bar"}), test.entryPoint, next) handler.ServeHTTP(rw, req) - for _, span := range tracer.spans { + for _, span := range mockTracer.spans { assert.Equal(t, test.expected.name, span.name) assert.Equal(t, test.expected.attributes, span.attributes) } diff --git a/pkg/server/middleware/observability.go b/pkg/server/middleware/observability.go index af65cf480..7368bd3a1 100644 --- a/pkg/server/middleware/observability.go +++ b/pkg/server/middleware/observability.go @@ -15,7 +15,7 @@ import ( "github.com/traefik/traefik/v3/pkg/middlewares/capture" metricsMiddle "github.com/traefik/traefik/v3/pkg/middlewares/metrics" tracingMiddle "github.com/traefik/traefik/v3/pkg/middlewares/tracing" - "go.opentelemetry.io/otel/trace" + "github.com/traefik/traefik/v3/pkg/tracing" ) // ObservabilityMgr is a manager for observability (AccessLogs, Metrics and Tracing) enablement. @@ -23,12 +23,12 @@ type ObservabilityMgr struct { config static.Configuration accessLoggerMiddleware *accesslog.Handler metricsRegistry metrics.Registry - tracer trace.Tracer + tracer *tracing.Tracer tracerCloser io.Closer } // NewObservabilityMgr creates a new ObservabilityMgr. -func NewObservabilityMgr(config static.Configuration, metricsRegistry metrics.Registry, accessLoggerMiddleware *accesslog.Handler, tracer trace.Tracer, tracerCloser io.Closer) *ObservabilityMgr { +func NewObservabilityMgr(config static.Configuration, metricsRegistry metrics.Registry, accessLoggerMiddleware *accesslog.Handler, tracer *tracing.Tracer, tracerCloser io.Closer) *ObservabilityMgr { return &ObservabilityMgr{ config: config, metricsRegistry: metricsRegistry, diff --git a/pkg/server/service/tracing_roundtripper.go b/pkg/server/service/tracing_roundtripper.go index bdb2ab4f3..69b0edf40 100644 --- a/pkg/server/service/tracing_roundtripper.go +++ b/pkg/server/service/tracing_roundtripper.go @@ -14,25 +14,27 @@ type wrapper struct { func (t *wrapper) RoundTrip(req *http.Request) (*http.Response, error) { var span trace.Span - if tracer := tracing.TracerFromContext(req.Context()); tracer != nil { + var tracer *tracing.Tracer + if tracer = tracing.TracerFromContext(req.Context()); tracer != nil { var tracingCtx context.Context tracingCtx, span = tracer.Start(req.Context(), "ReverseProxy", trace.WithSpanKind(trace.SpanKindClient)) defer span.End() req = req.WithContext(tracingCtx) - tracing.LogClientRequest(span, req) + tracer.CaptureClientRequest(span, req) tracing.InjectContextIntoCarrier(req) } response, err := t.rt.RoundTrip(req) if err != nil { statusCode := computeStatusCode(err) - tracing.LogResponseCode(span, statusCode, trace.SpanKindClient) + tracer.CaptureResponse(span, nil, statusCode, trace.SpanKindClient) + return response, err } - tracing.LogResponseCode(span, response.StatusCode, trace.SpanKindClient) + tracer.CaptureResponse(span, response.Header, response.StatusCode, trace.SpanKindClient) return response, nil } diff --git a/pkg/server/service/tracing_roundtripper_test.go b/pkg/server/service/tracing_roundtripper_test.go new file mode 100644 index 000000000..d629d6d34 --- /dev/null +++ b/pkg/server/service/tracing_roundtripper_test.go @@ -0,0 +1,138 @@ +package service + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/traefik/traefik/v3/pkg/tracing" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" + "go.opentelemetry.io/otel/trace/embedded" +) + +func TestTracingRoundTripper(t *testing.T) { + type expected struct { + name string + attributes []attribute.KeyValue + } + + testCases := []struct { + desc string + expected []expected + }{ + { + desc: "basic test", + expected: []expected{ + { + name: "initial", + attributes: []attribute.KeyValue{ + attribute.String("span.kind", "unspecified"), + }, + }, + { + name: "ReverseProxy", + attributes: []attribute.KeyValue{ + attribute.String("span.kind", "client"), + attribute.String("http.request.method", "GET"), + attribute.String("network.protocol.version", "1.1"), + attribute.String("url.full", "http://www.test.com/search?q=Opentelemetry"), + attribute.String("url.scheme", "http"), + attribute.String("user_agent.original", "reverse-test"), + attribute.String("network.peer.address", ""), + attribute.String("server.address", "www.test.com"), + attribute.String("network.peer.port", "80"), + attribute.Int64("server.port", int64(80)), + attribute.StringSlice("http.request.header.x-foo", []string{"foo", "bar"}), + attribute.Int64("http.response.status_code", int64(404)), + attribute.StringSlice("http.response.header.x-bar", []string{"foo", "bar"}), + }, + }, + }, + }, + } + + for _, test := range testCases { + t.Run(test.desc, func(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "http://www.test.com/search?q=Opentelemetry", nil) + req.RemoteAddr = "10.0.0.1:1234" + req.Header.Set("User-Agent", "reverse-test") + req.Header.Set("X-Forwarded-Proto", "http") + req.Header.Set("X-Foo", "foo") + req.Header.Add("X-Foo", "bar") + + mockTracer := &mockTracer{} + tracer := tracing.NewTracer(mockTracer, []string{"X-Foo"}, []string{"X-Bar"}) + initialCtx, initialSpan := tracer.Start(req.Context(), "initial") + defer initialSpan.End() + req = req.WithContext(initialCtx) + + tracingRoundTripper := newTracingRoundTripper(roundTripperFn(func(req *http.Request) (*http.Response, error) { + return &http.Response{ + Header: map[string][]string{ + "X-Bar": {"foo", "bar"}, + }, + StatusCode: http.StatusNotFound, + }, nil + })) + + _, err := tracingRoundTripper.RoundTrip(req) + require.NoError(t, err) + + for i, span := range mockTracer.spans { + assert.Equal(t, test.expected[i].name, span.name) + assert.Equal(t, test.expected[i].attributes, span.attributes) + } + }) + } +} + +type mockTracer struct { + embedded.Tracer + + spans []*mockSpan +} + +var _ trace.Tracer = &mockTracer{} + +func (t *mockTracer) Start(ctx context.Context, name string, opts ...trace.SpanStartOption) (context.Context, trace.Span) { + config := trace.NewSpanStartConfig(opts...) + span := &mockSpan{} + span.SetName(name) + span.SetAttributes(attribute.String("span.kind", config.SpanKind().String())) + span.SetAttributes(config.Attributes()...) + t.spans = append(t.spans, span) + return trace.ContextWithSpan(ctx, span), span +} + +// mockSpan is an implementation of Span that preforms no operations. +type mockSpan struct { + embedded.Span + + name string + attributes []attribute.KeyValue +} + +var _ trace.Span = &mockSpan{} + +func (*mockSpan) SpanContext() trace.SpanContext { + return trace.NewSpanContext(trace.SpanContextConfig{TraceID: trace.TraceID{1}, SpanID: trace.SpanID{1}}) +} +func (*mockSpan) IsRecording() bool { return false } +func (s *mockSpan) SetStatus(_ codes.Code, _ string) {} +func (s *mockSpan) SetAttributes(kv ...attribute.KeyValue) { + s.attributes = append(s.attributes, kv...) +} +func (s *mockSpan) End(...trace.SpanEndOption) {} +func (s *mockSpan) RecordError(_ error, _ ...trace.EventOption) {} +func (s *mockSpan) AddEvent(_ string, _ ...trace.EventOption) {} + +func (s *mockSpan) SetName(name string) { s.name = name } + +func (s *mockSpan) TracerProvider() trace.TracerProvider { + return nil +} diff --git a/pkg/tracing/tracing.go b/pkg/tracing/tracing.go index a3e1623c7..8faef513d 100644 --- a/pkg/tracing/tracing.go +++ b/pkg/tracing/tracing.go @@ -7,6 +7,7 @@ import ( "net" "net/http" "strconv" + "strings" "github.com/rs/zerolog/log" "github.com/traefik/traefik/v3/pkg/config/static" @@ -26,7 +27,7 @@ type Backend interface { } // NewTracing Creates a Tracing. -func NewTracing(conf *static.Tracing) (trace.Tracer, io.Closer, error) { +func NewTracing(conf *static.Tracing) (*Tracer, io.Closer, error) { var backend Backend if conf.OTLP != nil { @@ -41,11 +42,16 @@ func NewTracing(conf *static.Tracing) (trace.Tracer, io.Closer, error) { otel.SetTextMapPropagator(autoprop.NewTextMapPropagator()) - return backend.Setup(conf.ServiceName, conf.SampleRate, conf.GlobalAttributes) + tr, closer, err := backend.Setup(conf.ServiceName, conf.SampleRate, conf.GlobalAttributes) + if err != nil { + return nil, nil, err + } + + return NewTracer(tr, conf.CapturedRequestHeaders, conf.CapturedResponseHeaders), closer, nil } // TracerFromContext extracts the trace.Tracer from the given context. -func TracerFromContext(ctx context.Context) trace.Tracer { +func TracerFromContext(ctx context.Context) *Tracer { // Prevent picking trace.noopSpan tracer. if !trace.SpanContextFromContext(ctx).IsValid() { return nil @@ -53,7 +59,12 @@ func TracerFromContext(ctx context.Context) trace.Tracer { span := trace.SpanFromContext(ctx) if span != nil && span.TracerProvider() != nil { - return span.TracerProvider().Tracer("github.com/traefik/traefik") + tracer := span.TracerProvider().Tracer("github.com/traefik/traefik") + if tracer, ok := tracer.(*Tracer); ok { + return tracer + } + + return nil } return nil @@ -78,10 +89,73 @@ func SetStatusErrorf(ctx context.Context, format string, args ...interface{}) { } } -// LogClientRequest used to add span attributes from the request as a Client. -// TODO: the semconv does not implement Semantic Convention v1.23.0. -func LogClientRequest(span trace.Span, r *http.Request) { - if r == nil || span == nil { +// Span is trace.Span wrapping the Traefik TracerProvider. +type Span struct { + trace.Span + + tracerProvider *TracerProvider +} + +// TracerProvider returns the span's TraceProvider. +func (s Span) TracerProvider() trace.TracerProvider { + return s.tracerProvider +} + +// TracerProvider is trace.TracerProvider wrapping the Traefik Tracer implementation. +type TracerProvider struct { + trace.TracerProvider + + tracer *Tracer +} + +// Tracer returns the trace.Tracer for the given options. +// It returns specifically the Traefik Tracer when requested. +func (t TracerProvider) Tracer(name string, options ...trace.TracerOption) trace.Tracer { + if name == "github.com/traefik/traefik" { + return t.tracer + } + + return t.TracerProvider.Tracer(name, options...) +} + +// Tracer is trace.Tracer with additional properties. +type Tracer struct { + trace.Tracer + + capturedRequestHeaders []string + capturedResponseHeaders []string +} + +// NewTracer builds and configures a new Tracer. +func NewTracer(tracer trace.Tracer, capturedRequestHeaders, capturedResponseHeaders []string) *Tracer { + return &Tracer{ + Tracer: tracer, + capturedRequestHeaders: capturedRequestHeaders, + capturedResponseHeaders: capturedResponseHeaders, + } +} + +// Start starts a new span. +// spancheck linter complains about span.End not being called, but this is expected here, +// hence its deactivation. +// +//nolint:spancheck +func (t *Tracer) Start(ctx context.Context, spanName string, opts ...trace.SpanStartOption) (context.Context, trace.Span) { + if t == nil { + return ctx, nil + } + + spanCtx, span := t.Tracer.Start(ctx, spanName, opts...) + + wrappedSpan := &Span{Span: span, tracerProvider: &TracerProvider{tracer: t}} + + return trace.ContextWithSpan(spanCtx, wrappedSpan), wrappedSpan +} + +// CaptureClientRequest used to add span attributes from the request as a Client. +// TODO: need to update the semconv package as it does not implement fully Semantic Convention v1.23.0. +func (t *Tracer) CaptureClientRequest(span trace.Span, r *http.Request) { + if t == nil || span == nil || r == nil { return } @@ -113,12 +187,23 @@ func LogClientRequest(span trace.Span, r *http.Request) { span.SetAttributes(semconv.ServerAddress(host)) span.SetAttributes(semconv.ServerPort(intPort)) } + + for _, header := range t.capturedRequestHeaders { + // User-agent is already part of the semantic convention as a recommended attribute. + if strings.EqualFold(header, "User-Agent") { + continue + } + + if value := r.Header[header]; value != nil { + span.SetAttributes(attribute.StringSlice(fmt.Sprintf("http.request.header.%s", strings.ToLower(header)), value)) + } + } } -// LogServerRequest used to add span attributes from the request as a Server. -// TODO: the semconv does not implement Semantic Convention v1.23.0. -func LogServerRequest(span trace.Span, r *http.Request) { - if r == nil { +// CaptureServerRequest used to add span attributes from the request as a Server. +// TODO: need to update the semconv package as it does not implement fully Semantic Convention v1.23.0. +func (t *Tracer) CaptureServerRequest(span trace.Span, r *http.Request) { + if t == nil || span == nil || r == nil { return } @@ -147,6 +232,45 @@ func LogServerRequest(span trace.Span, r *http.Request) { } span.SetAttributes(semconv.ClientSocketAddress(r.Header.Get("X-Forwarded-For"))) + + for _, header := range t.capturedRequestHeaders { + // User-agent is already part of the semantic convention as a recommended attribute. + if strings.EqualFold(header, "User-Agent") { + continue + } + + if value := r.Header[header]; value != nil { + span.SetAttributes(attribute.StringSlice(fmt.Sprintf("http.request.header.%s", strings.ToLower(header)), value)) + } + } +} + +// CaptureResponse captures the response attributes to the span. +func (t *Tracer) CaptureResponse(span trace.Span, responseHeaders http.Header, code int, spanKind trace.SpanKind) { + if t == nil || span == nil { + return + } + + var status codes.Code + var desc string + switch spanKind { + case trace.SpanKindServer: + status, desc = serverStatus(code) + case trace.SpanKindClient: + status, desc = clientStatus(code) + default: + status, desc = defaultStatus(code) + } + span.SetStatus(status, desc) + if code > 0 { + span.SetAttributes(semconv.HTTPResponseStatusCode(code)) + } + + for _, header := range t.capturedResponseHeaders { + if value := responseHeaders[header]; value != nil { + span.SetAttributes(attribute.StringSlice(fmt.Sprintf("http.response.header.%s", strings.ToLower(header)), value)) + } + } } func proto(proto string) string { @@ -164,30 +288,10 @@ func proto(proto string) string { } } -// LogResponseCode used to log response code in span. -func LogResponseCode(span trace.Span, code int, spanKind trace.SpanKind) { - if span != nil { - var status codes.Code - var desc string - switch spanKind { - case trace.SpanKindServer: - status, desc = ServerStatus(code) - case trace.SpanKindClient: - status, desc = ClientStatus(code) - default: - status, desc = DefaultStatus(code) - } - span.SetStatus(status, desc) - if code > 0 { - span.SetAttributes(semconv.HTTPResponseStatusCode(code)) - } - } -} - -// ServerStatus returns a span status code and message for an HTTP status code +// serverStatus returns a span status code and message for an HTTP status code // value returned by a server. Status codes in the 400-499 range are not // returned as errors. -func ServerStatus(code int) (codes.Code, string) { +func serverStatus(code int) (codes.Code, string) { if code < 100 || code >= 600 { return codes.Error, fmt.Sprintf("Invalid HTTP status code %d", code) } @@ -197,10 +301,10 @@ func ServerStatus(code int) (codes.Code, string) { return codes.Unset, "" } -// ClientStatus returns a span status code and message for an HTTP status code +// clientStatus returns a span status code and message for an HTTP status code // value returned by a server. Status codes in the 400-499 range are not // returned as errors. -func ClientStatus(code int) (codes.Code, string) { +func clientStatus(code int) (codes.Code, string) { if code < 100 || code >= 600 { return codes.Error, fmt.Sprintf("Invalid HTTP status code %d", code) } @@ -210,9 +314,9 @@ func ClientStatus(code int) (codes.Code, string) { return codes.Unset, "" } -// DefaultStatus returns a span status code and message for an HTTP status code +// defaultStatus returns a span status code and message for an HTTP status code // value generated internally. -func DefaultStatus(code int) (codes.Code, string) { +func defaultStatus(code int) (codes.Code, string) { if code < 100 || code >= 600 { return codes.Error, fmt.Sprintf("Invalid HTTP status code %d", code) }