Add captured headers options for tracing

Co-authored-by: Baptiste Mayelle <baptiste.mayelle@traefik.io>
2024-03-11 11:50:04 +01:00 · 2024-03-11 11:50:04 +01:00 · 709ff6fb09
parent 86be0a4e6f
commit 709ff6fb09
15 changed files with 520 additions and 119 deletions
--- a/cmd/traefik/traefik.go
+++ b/cmd/traefik/traefik.go
@ -46,7 +46,6 @@ import (
 	"github.com/traefik/traefik/v3/pkg/tracing"
 	"github.com/traefik/traefik/v3/pkg/types"
 	"github.com/traefik/traefik/v3/pkg/version"
-	"go.opentelemetry.io/otel/trace"
 )

 func main() {
@ -563,7 +562,7 @@ func setupAccessLog(conf *types.AccessLog) *accesslog.Handler {
 	return accessLoggerMiddleware
 }

-func setupTracing(conf *static.Tracing) (trace.Tracer, io.Closer) {
+func setupTracing(conf *static.Tracing) (*tracing.Tracer, io.Closer) {
 	if conf == nil {
 		return nil, nil
 	}
--- a/docs/content/observability/tracing/overview.md
+++ b/docs/content/observability/tracing/overview.md
@ -116,3 +116,47 @@ tracing:
 --tracing.globalAttributes.attr1=foo
 --tracing.globalAttributes.attr2=bar
 ```
+
+#### `capturedRequestHeaders`
+
+_Optional, Default=empty_
+
+Defines the list of request headers to add as attributes.
+It applies to client and server kind spans.
+
+```yaml tab="File (YAML)"
+tracing:
+  capturedRequestHeaders:
+    - X-CustomHeader
+```
+
+```toml tab="File (TOML)"
+[tracing]
+    capturedRequestHeaders = ["X-CustomHeader"]
+```
+
+```bash tab="CLI"
+--tracing.capturedRequestHeaders[0]=X-CustomHeader
+```
+
+#### `capturedResponseHeaders`
+
+_Optional, Default=empty_
+
+Defines the list of response headers to add as attributes.
+It applies to client and server kind spans.
+
+```yaml tab="File (YAML)"
+tracing:
+  capturedResponseHeaders:
+    - X-CustomHeader
+```
+
+```toml tab="File (TOML)"
+[tracing]
+    capturedResponseHeaders = ["X-CustomHeader"]
+```
+
+```bash tab="CLI"
+--tracing.capturedResponseHeaders[0]=X-CustomHeader
+```
--- a/docs/content/reference/static-configuration/cli-ref.md
+++ b/docs/content/reference/static-configuration/cli-ref.md
@ -1017,6 +1017,12 @@ OpenTracing configuration. (Default: ```false```)
 `--tracing.addinternals`:  
 Enables tracing for internal services (ping, dashboard, etc...). (Default: ```false```)

+`--tracing.capturedrequestheaders`:  
+Request headers to add as attributes for server and client spans.
+
+`--tracing.capturedresponseheaders`:  
+Response headers to add as attributes for server and client spans.
+
 `--tracing.globalattributes.<name>`:  
 Defines additional attributes (key:value) on all spans.

--- a/docs/content/reference/static-configuration/env-ref.md
+++ b/docs/content/reference/static-configuration/env-ref.md
@ -1017,6 +1017,12 @@ OpenTracing configuration. (Default: ```false```)
 `TRAEFIK_TRACING_ADDINTERNALS`:  
 Enables tracing for internal services (ping, dashboard, etc...). (Default: ```false```)

+`TRAEFIK_TRACING_CAPTUREDREQUESTHEADERS`:  
+Request headers to add as attributes for server and client spans.
+
+`TRAEFIK_TRACING_CAPTUREDRESPONSEHEADERS`:  
+Response headers to add as attributes for server and client spans.
+
 `TRAEFIK_TRACING_GLOBALATTRIBUTES_<NAME>`:  
 Defines additional attributes (key:value) on all spans.

--- a/docs/content/reference/static-configuration/file.toml
+++ b/docs/content/reference/static-configuration/file.toml
@ -380,6 +380,8 @@

 [tracing]
  serviceName = "foobar"
+  capturedRequestHeaders = ["foobar", "foobar"]
+  capturedResponseHeaders = ["foobar", "foobar"]
  sampleRate = 42.0
  addInternals = true
  [tracing.globalAttributes]
--- a/docs/content/reference/static-configuration/file.yaml
+++ b/docs/content/reference/static-configuration/file.yaml
@ -418,6 +418,12 @@ tracing:
  globalAttributes:
    name0: foobar
    name1: foobar
+  capturedRequestHeaders:
+    - foobar
+    - foobar
+  capturedResponseHeaders:
+    - foobar
+    - foobar
  sampleRate: 42
  addInternals: true
  otlp:
--- a/pkg/config/static/static_config.go
+++ b/pkg/config/static/static_config.go
@ -193,10 +193,12 @@ func (a *LifeCycle) SetDefaults() {

 // Tracing holds the tracing configuration.
 type Tracing struct {
-	ServiceName      string            `description:"Set the name for this service." json:"serviceName,omitempty" toml:"serviceName,omitempty" yaml:"serviceName,omitempty" export:"true"`
-	GlobalAttributes map[string]string `description:"Defines additional attributes (key:value) on all spans." json:"globalAttributes,omitempty" toml:"globalAttributes,omitempty" yaml:"globalAttributes,omitempty" export:"true"`
-	SampleRate       float64           `description:"Sets the rate between 0.0 and 1.0 of requests to trace." json:"sampleRate,omitempty" toml:"sampleRate,omitempty" yaml:"sampleRate,omitempty" export:"true"`
-	AddInternals     bool              `description:"Enables tracing for internal services (ping, dashboard, etc...)." json:"addInternals,omitempty" toml:"addInternals,omitempty" yaml:"addInternals,omitempty" export:"true"`
+	ServiceName             string            `description:"Set the name for this service." json:"serviceName,omitempty" toml:"serviceName,omitempty" yaml:"serviceName,omitempty" export:"true"`
+	GlobalAttributes        map[string]string `description:"Defines additional attributes (key:value) on all spans." json:"globalAttributes,omitempty" toml:"globalAttributes,omitempty" yaml:"globalAttributes,omitempty" export:"true"`
+	CapturedRequestHeaders  []string          `description:"Request headers to add as attributes for server and client spans." json:"capturedRequestHeaders,omitempty" toml:"capturedRequestHeaders,omitempty" yaml:"capturedRequestHeaders,omitempty" export:"true"`
+	CapturedResponseHeaders []string          `description:"Response headers to add as attributes for server and client spans." json:"capturedResponseHeaders,omitempty" toml:"capturedResponseHeaders,omitempty" yaml:"capturedResponseHeaders,omitempty" export:"true"`
+	SampleRate              float64           `description:"Sets the rate between 0.0 and 1.0 of requests to trace." json:"sampleRate,omitempty" toml:"sampleRate,omitempty" yaml:"sampleRate,omitempty" export:"true"`
+	AddInternals            bool              `description:"Enables tracing for internal services (ping, dashboard, etc...)." json:"addInternals,omitempty" toml:"addInternals,omitempty" yaml:"addInternals,omitempty" export:"true"`

 	OTLP *opentelemetry.Config `description:"Settings for OpenTelemetry." json:"otlp,omitempty" toml:"otlp,omitempty" yaml:"otlp,omitempty" label:"allowEmpty" file:"allowEmpty" export:"true"`
 }
--- a/pkg/middlewares/auth/forward.go
+++ b/pkg/middlewares/auth/forward.go
@ -131,8 +131,11 @@ func (fa *forwardAuth) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
 		return
 	}

+	writeHeader(req, forwardReq, fa.trustForwardHeader, fa.authRequestHeaders)
+
 	var forwardSpan trace.Span
-	if tracer := tracing.TracerFromContext(req.Context()); tracer != nil {
+	var tracer *tracing.Tracer
+	if tracer = tracing.TracerFromContext(req.Context()); tracer != nil {
 		var tracingCtx context.Context
 		tracingCtx, forwardSpan = tracer.Start(req.Context(), "AuthRequest", trace.WithSpanKind(trace.SpanKindClient))
 		defer forwardSpan.End()
@ -140,11 +143,9 @@ func (fa *forwardAuth) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
 		forwardReq = forwardReq.WithContext(tracingCtx)

 		tracing.InjectContextIntoCarrier(forwardReq)
-		tracing.LogClientRequest(forwardSpan, forwardReq)
+		tracer.CaptureClientRequest(forwardSpan, forwardReq)
 	}

-	writeHeader(req, forwardReq, fa.trustForwardHeader, fa.authRequestHeaders)
-
 	forwardResponse, forwardErr := fa.client.Do(forwardReq)
 	if forwardErr != nil {
 		logMessage := fmt.Sprintf("Error calling %s. Cause: %s", fa.address, forwardErr)
@ -197,7 +198,7 @@ func (fa *forwardAuth) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
 			rw.Header().Set("Location", redirectURL.String())
 		}

-		tracing.LogResponseCode(forwardSpan, forwardResponse.StatusCode, trace.SpanKindClient)
+		tracer.CaptureResponse(forwardSpan, forwardResponse.Header, forwardResponse.StatusCode, trace.SpanKindClient)
 		rw.WriteHeader(forwardResponse.StatusCode)

 		if _, err = rw.Write(body); err != nil {
@ -228,7 +229,7 @@ func (fa *forwardAuth) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
 		}
 	}

-	tracing.LogResponseCode(forwardSpan, forwardResponse.StatusCode, trace.SpanKindClient)
+	tracer.CaptureResponse(forwardSpan, forwardResponse.Header, forwardResponse.StatusCode, trace.SpanKindClient)

 	req.RequestURI = req.URL.RequestURI()

--- a/pkg/middlewares/auth/forward_test.go
+++ b/pkg/middlewares/auth/forward_test.go
@ -4,27 +4,25 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"net"
 	"net/http"
 	"net/http/httptest"
+	"net/url"
+	"strconv"
 	"testing"

-	"github.com/containous/alice"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"github.com/traefik/traefik/v3/pkg/config/dynamic"
-	"github.com/traefik/traefik/v3/pkg/config/static"
-	tracingMiddleware "github.com/traefik/traefik/v3/pkg/middlewares/tracing"
 	"github.com/traefik/traefik/v3/pkg/testhelpers"
 	"github.com/traefik/traefik/v3/pkg/tracing"
-	"github.com/traefik/traefik/v3/pkg/tracing/opentelemetry"
-	"github.com/traefik/traefik/v3/pkg/types"
-	"github.com/traefik/traefik/v3/pkg/version"
 	"github.com/vulcand/oxy/v2/forward"
+	"go.opentelemetry.io/contrib/propagators/autoprop"
 	"go.opentelemetry.io/otel"
-	"go.opentelemetry.io/otel/sdk/resource"
-	sdktrace "go.opentelemetry.io/otel/sdk/trace"
-	"go.opentelemetry.io/otel/sdk/trace/tracetest"
-	semconv "go.opentelemetry.io/otel/semconv/v1.21.0"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/codes"
+	"go.opentelemetry.io/otel/trace"
+	"go.opentelemetry.io/otel/trace/embedded"
 )

 func TestForwardAuthFail(t *testing.T) {
@ -466,64 +464,146 @@ func Test_writeHeader(t *testing.T) {
 	}
 }

-func TestForwardAuthUsesTracing(t *testing.T) {
+func TestForwardAuthTracing(t *testing.T) {
+	type expected struct {
+		name       string
+		attributes []attribute.KeyValue
+	}
+
 	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		if r.Header.Get("Traceparent") == "" {
 			t.Errorf("expected Traceparent header to be present in request")
 		}
+
+		w.Header().Set("X-Bar", "foo")
+		w.Header().Add("X-Bar", "bar")
+		w.WriteHeader(http.StatusNotFound)
 	}))
 	t.Cleanup(server.Close)

-	next := http.Handler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
-
-	auth := dynamic.ForwardAuth{
-		Address: server.URL,
-	}
-
-	exporter := tracetest.NewInMemoryExporter()
-
-	tres, err := resource.New(context.Background(),
-		resource.WithAttributes(semconv.ServiceNameKey.String("traefik")),
-		resource.WithAttributes(semconv.ServiceVersionKey.String(version.Version)),
-		resource.WithFromEnv(),
-		resource.WithTelemetrySDK(),
-	)
+	parse, err := url.Parse(server.URL)
 	require.NoError(t, err)

-	tracerProvider := sdktrace.NewTracerProvider(
-		sdktrace.WithSampler(sdktrace.AlwaysSample()),
-		sdktrace.WithResource(tres),
-		sdktrace.WithBatcher(exporter),
-	)
-	otel.SetTracerProvider(tracerProvider)
+	_, serverPort, err := net.SplitHostPort(parse.Host)
+	require.NoError(t, err)

-	config := &static.Tracing{
-		ServiceName: "testApp",
-		SampleRate:  1,
-		OTLP: &opentelemetry.Config{
-			HTTP: &types.OtelHTTP{
-				Endpoint: "http://127.0.0.1:8080",
+	serverPortInt, err := strconv.Atoi(serverPort)
+	require.NoError(t, err)
+
+	testCases := []struct {
+		desc     string
+		expected []expected
+	}{
+		{
+			desc: "basic test",
+			expected: []expected{
+				{
+					name: "initial",
+					attributes: []attribute.KeyValue{
+						attribute.String("span.kind", "unspecified"),
+					},
+				},
+				{
+					name: "AuthRequest",
+					attributes: []attribute.KeyValue{
+						attribute.String("span.kind", "client"),
+						attribute.String("http.request.method", "GET"),
+						attribute.String("network.protocol.version", "1.1"),
+						attribute.String("url.full", server.URL),
+						attribute.String("url.scheme", "http"),
+						attribute.String("user_agent.original", ""),
+						attribute.String("network.peer.address", "127.0.0.1"),
+						attribute.String("network.peer.port", serverPort),
+						attribute.String("server.address", "127.0.0.1"),
+						attribute.Int64("server.port", int64(serverPortInt)),
+						attribute.StringSlice("http.request.header.x-foo", []string{"foo", "bar"}),
+						attribute.Int64("http.response.status_code", int64(404)),
+						attribute.StringSlice("http.response.header.x-bar", []string{"foo", "bar"}),
+					},
+				},
 			},
 		},
 	}
-	tr, closer, err := tracing.NewTracing(config)
-	require.NoError(t, err)
-	t.Cleanup(func() {
-		_ = closer.Close()
-	})

-	next, err = NewForward(context.Background(), next, auth, "authTest")
-	require.NoError(t, err)
+	for _, test := range testCases {
+		t.Run(test.desc, func(t *testing.T) {
+			next := http.Handler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))

-	chain := alice.New(tracingMiddleware.WrapEntryPointHandler(context.Background(), tr, "tracingTest"))
-	next, err = chain.Then(next)
-	require.NoError(t, err)
+			auth := dynamic.ForwardAuth{
+				Address:            server.URL,
+				AuthRequestHeaders: []string{"X-Foo"},
+			}
+			next, err := NewForward(context.Background(), next, auth, "authTest")
+			require.NoError(t, err)

-	ts := httptest.NewServer(next)
-	t.Cleanup(ts.Close)
+			req := httptest.NewRequest(http.MethodGet, "http://www.test.com/search?q=Opentelemetry", nil)
+			req.RemoteAddr = "10.0.0.1:1234"
+			req.Header.Set("User-Agent", "forward-test")
+			req.Header.Set("X-Forwarded-Proto", "http")
+			req.Header.Set("X-Foo", "foo")
+			req.Header.Add("X-Foo", "bar")

-	req := testhelpers.MustNewRequest(http.MethodGet, ts.URL, nil)
-	res, err := http.DefaultClient.Do(req)
-	require.NoError(t, err)
-	assert.Equal(t, http.StatusOK, res.StatusCode)
+			otel.SetTextMapPropagator(autoprop.NewTextMapPropagator())
+
+			mockTracer := &mockTracer{}
+			tracer := tracing.NewTracer(mockTracer, []string{"X-Foo"}, []string{"X-Bar"})
+			initialCtx, initialSpan := tracer.Start(req.Context(), "initial")
+			defer initialSpan.End()
+			req = req.WithContext(initialCtx)
+
+			recorder := httptest.NewRecorder()
+			next.ServeHTTP(recorder, req)
+
+			for i, span := range mockTracer.spans {
+				assert.Equal(t, test.expected[i].name, span.name)
+				assert.Equal(t, test.expected[i].attributes, span.attributes)
+			}
+		})
+	}
+}
+
+type mockTracer struct {
+	embedded.Tracer
+
+	spans []*mockSpan
+}
+
+var _ trace.Tracer = &mockTracer{}
+
+func (t *mockTracer) Start(ctx context.Context, name string, opts ...trace.SpanStartOption) (context.Context, trace.Span) {
+	config := trace.NewSpanStartConfig(opts...)
+	span := &mockSpan{}
+	span.SetName(name)
+	span.SetAttributes(attribute.String("span.kind", config.SpanKind().String()))
+	span.SetAttributes(config.Attributes()...)
+	t.spans = append(t.spans, span)
+	return trace.ContextWithSpan(ctx, span), span
+}
+
+// mockSpan is an implementation of Span that preforms no operations.
+type mockSpan struct {
+	embedded.Span
+
+	name       string
+	attributes []attribute.KeyValue
+}
+
+var _ trace.Span = &mockSpan{}
+
+func (*mockSpan) SpanContext() trace.SpanContext {
+	return trace.NewSpanContext(trace.SpanContextConfig{TraceID: trace.TraceID{1}, SpanID: trace.SpanID{1}})
+}
+func (*mockSpan) IsRecording() bool                  { return false }
+func (s *mockSpan) SetStatus(_ codes.Code, _ string) {}
+func (s *mockSpan) SetAttributes(kv ...attribute.KeyValue) {
+	s.attributes = append(s.attributes, kv...)
+}
+func (s *mockSpan) End(...trace.SpanEndOption)                  {}
+func (s *mockSpan) RecordError(_ error, _ ...trace.EventOption) {}
+func (s *mockSpan) AddEvent(_ string, _ ...trace.EventOption)   {}
+
+func (s *mockSpan) SetName(name string) { s.name = name }
+
+func (s *mockSpan) TracerProvider() trace.TracerProvider {
+	return nil
 }
--- a/pkg/middlewares/tracing/entrypoint.go
+++ b/pkg/middlewares/tracing/entrypoint.go
@ -2,6 +2,7 @@ package tracing

 import (
 	"context"
+	"errors"
 	"net/http"

 	"github.com/containous/alice"
@ -16,20 +17,24 @@ const (
 )

 type entryPointTracing struct {
-	tracer     trace.Tracer
+	tracer     *tracing.Tracer
 	entryPoint string
 	next       http.Handler
 }

 // WrapEntryPointHandler Wraps tracing to alice.Constructor.
-func WrapEntryPointHandler(ctx context.Context, tracer trace.Tracer, entryPointName string) alice.Constructor {
+func WrapEntryPointHandler(ctx context.Context, tracer *tracing.Tracer, entryPointName string) alice.Constructor {
 	return func(next http.Handler) (http.Handler, error) {
+		if tracer == nil {
+			return nil, errors.New("unexpected nil tracer")
+		}
+
 		return newEntryPoint(ctx, tracer, entryPointName, next), nil
 	}
 }

 // newEntryPoint creates a new tracing middleware for incoming requests.
-func newEntryPoint(ctx context.Context, tracer trace.Tracer, entryPointName string, next http.Handler) http.Handler {
+func newEntryPoint(ctx context.Context, tracer *tracing.Tracer, entryPointName string, next http.Handler) http.Handler {
 	middlewares.GetLogger(ctx, "tracing", entryPointTypeName).Debug().Msg("Creating middleware")

 	return &entryPointTracing{
@ -48,10 +53,10 @@ func (e *entryPointTracing) ServeHTTP(rw http.ResponseWriter, req *http.Request)

 	span.SetAttributes(attribute.String("entry_point", e.entryPoint))

-	tracing.LogServerRequest(span, req)
+	e.tracer.CaptureServerRequest(span, req)

 	recorder := newStatusCodeRecorder(rw, http.StatusOK)
 	e.next.ServeHTTP(recorder, req)

-	tracing.LogResponseCode(span, recorder.Status(), trace.SpanKindServer)
+	e.tracer.CaptureResponse(span, recorder.Header(), recorder.Status(), trace.SpanKindServer)
 }
--- a/pkg/middlewares/tracing/entrypoint_test.go
+++ b/pkg/middlewares/tracing/entrypoint_test.go
@ -7,6 +7,7 @@ import (
 	"testing"

 	"github.com/stretchr/testify/assert"
+	"github.com/traefik/traefik/v3/pkg/tracing"
 	"go.opentelemetry.io/otel/attribute"
 )

@ -42,7 +43,9 @@ func TestEntryPointMiddleware(t *testing.T) {
 					attribute.String("client.address", "10.0.0.1"),
 					attribute.Int64("client.port", int64(1234)),
 					attribute.String("client.socket.address", ""),
+					attribute.StringSlice("http.request.header.x-foo", []string{"foo", "bar"}),
 					attribute.Int64("http.response.status_code", int64(404)),
+					attribute.StringSlice("http.response.header.x-bar", []string{"foo", "bar"}),
 				},
 			},
 		},
@ -55,17 +58,20 @@ func TestEntryPointMiddleware(t *testing.T) {
 			req.RemoteAddr = "10.0.0.1:1234"
 			req.Header.Set("User-Agent", "entrypoint-test")
 			req.Header.Set("X-Forwarded-Proto", "http")
+			req.Header.Set("X-Foo", "foo")
+			req.Header.Add("X-Foo", "bar")

 			next := http.HandlerFunc(func(rw http.ResponseWriter, _ *http.Request) {
+				rw.Header().Set("X-Bar", "foo")
+				rw.Header().Add("X-Bar", "bar")
 				rw.WriteHeader(http.StatusNotFound)
 			})

-			tracer := &mockTracer{}
-
-			handler := newEntryPoint(context.Background(), tracer, test.entryPoint, next)
+			mockTracer := &mockTracer{}
+			handler := newEntryPoint(context.Background(), tracing.NewTracer(mockTracer, []string{"X-Foo"}, []string{"X-Bar"}), test.entryPoint, next)
 			handler.ServeHTTP(rw, req)

-			for _, span := range tracer.spans {
+			for _, span := range mockTracer.spans {
 				assert.Equal(t, test.expected.name, span.name)
 				assert.Equal(t, test.expected.attributes, span.attributes)
 			}
--- a/pkg/server/middleware/observability.go
+++ b/pkg/server/middleware/observability.go
@ -15,7 +15,7 @@ import (
 	"github.com/traefik/traefik/v3/pkg/middlewares/capture"
 	metricsMiddle "github.com/traefik/traefik/v3/pkg/middlewares/metrics"
 	tracingMiddle "github.com/traefik/traefik/v3/pkg/middlewares/tracing"
-	"go.opentelemetry.io/otel/trace"
+	"github.com/traefik/traefik/v3/pkg/tracing"
 )

 // ObservabilityMgr is a manager for observability (AccessLogs, Metrics and Tracing) enablement.
@ -23,12 +23,12 @@ type ObservabilityMgr struct {
 	config                 static.Configuration
 	accessLoggerMiddleware *accesslog.Handler
 	metricsRegistry        metrics.Registry
-	tracer                 trace.Tracer
+	tracer                 *tracing.Tracer
 	tracerCloser           io.Closer
 }

 // NewObservabilityMgr creates a new ObservabilityMgr.
-func NewObservabilityMgr(config static.Configuration, metricsRegistry metrics.Registry, accessLoggerMiddleware *accesslog.Handler, tracer trace.Tracer, tracerCloser io.Closer) *ObservabilityMgr {
+func NewObservabilityMgr(config static.Configuration, metricsRegistry metrics.Registry, accessLoggerMiddleware *accesslog.Handler, tracer *tracing.Tracer, tracerCloser io.Closer) *ObservabilityMgr {
 	return &ObservabilityMgr{
 		config:                 config,
 		metricsRegistry:        metricsRegistry,
--- a/pkg/server/service/tracing_roundtripper.go
+++ b/pkg/server/service/tracing_roundtripper.go
@ -14,25 +14,27 @@ type wrapper struct {

 func (t *wrapper) RoundTrip(req *http.Request) (*http.Response, error) {
 	var span trace.Span
-	if tracer := tracing.TracerFromContext(req.Context()); tracer != nil {
+	var tracer *tracing.Tracer
+	if tracer = tracing.TracerFromContext(req.Context()); tracer != nil {
 		var tracingCtx context.Context
 		tracingCtx, span = tracer.Start(req.Context(), "ReverseProxy", trace.WithSpanKind(trace.SpanKindClient))
 		defer span.End()

 		req = req.WithContext(tracingCtx)

-		tracing.LogClientRequest(span, req)
+		tracer.CaptureClientRequest(span, req)
 		tracing.InjectContextIntoCarrier(req)
 	}

 	response, err := t.rt.RoundTrip(req)
 	if err != nil {
 		statusCode := computeStatusCode(err)
-		tracing.LogResponseCode(span, statusCode, trace.SpanKindClient)
+		tracer.CaptureResponse(span, nil, statusCode, trace.SpanKindClient)
+
 		return response, err
 	}

-	tracing.LogResponseCode(span, response.StatusCode, trace.SpanKindClient)
+	tracer.CaptureResponse(span, response.Header, response.StatusCode, trace.SpanKindClient)

 	return response, nil
 }
--- a/pkg/server/service/tracing_roundtripper_test.go
+++ b/pkg/server/service/tracing_roundtripper_test.go
@ -0,0 +1,138 @@
+package service
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/traefik/traefik/v3/pkg/tracing"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/codes"
+	"go.opentelemetry.io/otel/trace"
+	"go.opentelemetry.io/otel/trace/embedded"
+)
+
+func TestTracingRoundTripper(t *testing.T) {
+	type expected struct {
+		name       string
+		attributes []attribute.KeyValue
+	}
+
+	testCases := []struct {
+		desc     string
+		expected []expected
+	}{
+		{
+			desc: "basic test",
+			expected: []expected{
+				{
+					name: "initial",
+					attributes: []attribute.KeyValue{
+						attribute.String("span.kind", "unspecified"),
+					},
+				},
+				{
+					name: "ReverseProxy",
+					attributes: []attribute.KeyValue{
+						attribute.String("span.kind", "client"),
+						attribute.String("http.request.method", "GET"),
+						attribute.String("network.protocol.version", "1.1"),
+						attribute.String("url.full", "http://www.test.com/search?q=Opentelemetry"),
+						attribute.String("url.scheme", "http"),
+						attribute.String("user_agent.original", "reverse-test"),
+						attribute.String("network.peer.address", ""),
+						attribute.String("server.address", "www.test.com"),
+						attribute.String("network.peer.port", "80"),
+						attribute.Int64("server.port", int64(80)),
+						attribute.StringSlice("http.request.header.x-foo", []string{"foo", "bar"}),
+						attribute.Int64("http.response.status_code", int64(404)),
+						attribute.StringSlice("http.response.header.x-bar", []string{"foo", "bar"}),
+					},
+				},
+			},
+		},
+	}
+
+	for _, test := range testCases {
+		t.Run(test.desc, func(t *testing.T) {
+			req := httptest.NewRequest(http.MethodGet, "http://www.test.com/search?q=Opentelemetry", nil)
+			req.RemoteAddr = "10.0.0.1:1234"
+			req.Header.Set("User-Agent", "reverse-test")
+			req.Header.Set("X-Forwarded-Proto", "http")
+			req.Header.Set("X-Foo", "foo")
+			req.Header.Add("X-Foo", "bar")
+
+			mockTracer := &mockTracer{}
+			tracer := tracing.NewTracer(mockTracer, []string{"X-Foo"}, []string{"X-Bar"})
+			initialCtx, initialSpan := tracer.Start(req.Context(), "initial")
+			defer initialSpan.End()
+			req = req.WithContext(initialCtx)
+
+			tracingRoundTripper := newTracingRoundTripper(roundTripperFn(func(req *http.Request) (*http.Response, error) {
+				return &http.Response{
+					Header: map[string][]string{
+						"X-Bar": {"foo", "bar"},
+					},
+					StatusCode: http.StatusNotFound,
+				}, nil
+			}))
+
+			_, err := tracingRoundTripper.RoundTrip(req)
+			require.NoError(t, err)
+
+			for i, span := range mockTracer.spans {
+				assert.Equal(t, test.expected[i].name, span.name)
+				assert.Equal(t, test.expected[i].attributes, span.attributes)
+			}
+		})
+	}
+}
+
+type mockTracer struct {
+	embedded.Tracer
+
+	spans []*mockSpan
+}
+
+var _ trace.Tracer = &mockTracer{}
+
+func (t *mockTracer) Start(ctx context.Context, name string, opts ...trace.SpanStartOption) (context.Context, trace.Span) {
+	config := trace.NewSpanStartConfig(opts...)
+	span := &mockSpan{}
+	span.SetName(name)
+	span.SetAttributes(attribute.String("span.kind", config.SpanKind().String()))
+	span.SetAttributes(config.Attributes()...)
+	t.spans = append(t.spans, span)
+	return trace.ContextWithSpan(ctx, span), span
+}
+
+// mockSpan is an implementation of Span that preforms no operations.
+type mockSpan struct {
+	embedded.Span
+
+	name       string
+	attributes []attribute.KeyValue
+}
+
+var _ trace.Span = &mockSpan{}
+
+func (*mockSpan) SpanContext() trace.SpanContext {
+	return trace.NewSpanContext(trace.SpanContextConfig{TraceID: trace.TraceID{1}, SpanID: trace.SpanID{1}})
+}
+func (*mockSpan) IsRecording() bool                  { return false }
+func (s *mockSpan) SetStatus(_ codes.Code, _ string) {}
+func (s *mockSpan) SetAttributes(kv ...attribute.KeyValue) {
+	s.attributes = append(s.attributes, kv...)
+}
+func (s *mockSpan) End(...trace.SpanEndOption)                  {}
+func (s *mockSpan) RecordError(_ error, _ ...trace.EventOption) {}
+func (s *mockSpan) AddEvent(_ string, _ ...trace.EventOption)   {}
+
+func (s *mockSpan) SetName(name string) { s.name = name }
+
+func (s *mockSpan) TracerProvider() trace.TracerProvider {
+	return nil
+}
--- a/pkg/tracing/tracing.go
+++ b/pkg/tracing/tracing.go
@ -7,6 +7,7 @@ import (
 	"net"
 	"net/http"
 	"strconv"
+	"strings"

 	"github.com/rs/zerolog/log"
 	"github.com/traefik/traefik/v3/pkg/config/static"
@ -26,7 +27,7 @@ type Backend interface {
 }

 // NewTracing Creates a Tracing.
-func NewTracing(conf *static.Tracing) (trace.Tracer, io.Closer, error) {
+func NewTracing(conf *static.Tracing) (*Tracer, io.Closer, error) {
 	var backend Backend

 	if conf.OTLP != nil {
@ -41,11 +42,16 @@ func NewTracing(conf *static.Tracing) (trace.Tracer, io.Closer, error) {

 	otel.SetTextMapPropagator(autoprop.NewTextMapPropagator())

-	return backend.Setup(conf.ServiceName, conf.SampleRate, conf.GlobalAttributes)
+	tr, closer, err := backend.Setup(conf.ServiceName, conf.SampleRate, conf.GlobalAttributes)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return NewTracer(tr, conf.CapturedRequestHeaders, conf.CapturedResponseHeaders), closer, nil
 }

 // TracerFromContext extracts the trace.Tracer from the given context.
-func TracerFromContext(ctx context.Context) trace.Tracer {
+func TracerFromContext(ctx context.Context) *Tracer {
 	// Prevent picking trace.noopSpan tracer.
 	if !trace.SpanContextFromContext(ctx).IsValid() {
 		return nil
@ -53,7 +59,12 @@ func TracerFromContext(ctx context.Context) trace.Tracer {

 	span := trace.SpanFromContext(ctx)
 	if span != nil && span.TracerProvider() != nil {
-		return span.TracerProvider().Tracer("github.com/traefik/traefik")
+		tracer := span.TracerProvider().Tracer("github.com/traefik/traefik")
+		if tracer, ok := tracer.(*Tracer); ok {
+			return tracer
+		}
+
+		return nil
 	}

 	return nil
@ -78,10 +89,73 @@ func SetStatusErrorf(ctx context.Context, format string, args ...interface{}) {
 	}
 }

-// LogClientRequest used to add span attributes from the request as a Client.
-// TODO: the semconv does not implement Semantic Convention v1.23.0.
-func LogClientRequest(span trace.Span, r *http.Request) {
-	if r == nil || span == nil {
+// Span is trace.Span wrapping the Traefik TracerProvider.
+type Span struct {
+	trace.Span
+
+	tracerProvider *TracerProvider
+}
+
+// TracerProvider returns the span's TraceProvider.
+func (s Span) TracerProvider() trace.TracerProvider {
+	return s.tracerProvider
+}
+
+// TracerProvider is trace.TracerProvider wrapping the Traefik Tracer implementation.
+type TracerProvider struct {
+	trace.TracerProvider
+
+	tracer *Tracer
+}
+
+// Tracer returns the trace.Tracer for the given options.
+// It returns specifically the Traefik Tracer when requested.
+func (t TracerProvider) Tracer(name string, options ...trace.TracerOption) trace.Tracer {
+	if name == "github.com/traefik/traefik" {
+		return t.tracer
+	}
+
+	return t.TracerProvider.Tracer(name, options...)
+}
+
+// Tracer is trace.Tracer with additional properties.
+type Tracer struct {
+	trace.Tracer
+
+	capturedRequestHeaders  []string
+	capturedResponseHeaders []string
+}
+
+// NewTracer builds and configures a new Tracer.
+func NewTracer(tracer trace.Tracer, capturedRequestHeaders, capturedResponseHeaders []string) *Tracer {
+	return &Tracer{
+		Tracer:                  tracer,
+		capturedRequestHeaders:  capturedRequestHeaders,
+		capturedResponseHeaders: capturedResponseHeaders,
+	}
+}
+
+// Start starts a new span.
+// spancheck linter complains about span.End not being called, but this is expected here,
+// hence its deactivation.
+//
+//nolint:spancheck
+func (t *Tracer) Start(ctx context.Context, spanName string, opts ...trace.SpanStartOption) (context.Context, trace.Span) {
+	if t == nil {
+		return ctx, nil
+	}
+
+	spanCtx, span := t.Tracer.Start(ctx, spanName, opts...)
+
+	wrappedSpan := &Span{Span: span, tracerProvider: &TracerProvider{tracer: t}}
+
+	return trace.ContextWithSpan(spanCtx, wrappedSpan), wrappedSpan
+}
+
+// CaptureClientRequest used to add span attributes from the request as a Client.
+// TODO: need to update the semconv package as it does not implement fully Semantic Convention v1.23.0.
+func (t *Tracer) CaptureClientRequest(span trace.Span, r *http.Request) {
+	if t == nil || span == nil || r == nil {
 		return
 	}

@ -113,12 +187,23 @@ func LogClientRequest(span trace.Span, r *http.Request) {
 		span.SetAttributes(semconv.ServerAddress(host))
 		span.SetAttributes(semconv.ServerPort(intPort))
 	}
+
+	for _, header := range t.capturedRequestHeaders {
+		// User-agent is already part of the semantic convention as a recommended attribute.
+		if strings.EqualFold(header, "User-Agent") {
+			continue
+		}
+
+		if value := r.Header[header]; value != nil {
+			span.SetAttributes(attribute.StringSlice(fmt.Sprintf("http.request.header.%s", strings.ToLower(header)), value))
+		}
+	}
 }

-// LogServerRequest used to add span attributes from the request as a Server.
-// TODO: the semconv does not implement Semantic Convention v1.23.0.
-func LogServerRequest(span trace.Span, r *http.Request) {
-	if r == nil {
+// CaptureServerRequest used to add span attributes from the request as a Server.
+// TODO: need to update the semconv package as it does not implement fully Semantic Convention v1.23.0.
+func (t *Tracer) CaptureServerRequest(span trace.Span, r *http.Request) {
+	if t == nil || span == nil || r == nil {
 		return
 	}

@ -147,6 +232,45 @@ func LogServerRequest(span trace.Span, r *http.Request) {
 	}

 	span.SetAttributes(semconv.ClientSocketAddress(r.Header.Get("X-Forwarded-For")))
+
+	for _, header := range t.capturedRequestHeaders {
+		// User-agent is already part of the semantic convention as a recommended attribute.
+		if strings.EqualFold(header, "User-Agent") {
+			continue
+		}
+
+		if value := r.Header[header]; value != nil {
+			span.SetAttributes(attribute.StringSlice(fmt.Sprintf("http.request.header.%s", strings.ToLower(header)), value))
+		}
+	}
+}
+
+// CaptureResponse captures the response attributes to the span.
+func (t *Tracer) CaptureResponse(span trace.Span, responseHeaders http.Header, code int, spanKind trace.SpanKind) {
+	if t == nil || span == nil {
+		return
+	}
+
+	var status codes.Code
+	var desc string
+	switch spanKind {
+	case trace.SpanKindServer:
+		status, desc = serverStatus(code)
+	case trace.SpanKindClient:
+		status, desc = clientStatus(code)
+	default:
+		status, desc = defaultStatus(code)
+	}
+	span.SetStatus(status, desc)
+	if code > 0 {
+		span.SetAttributes(semconv.HTTPResponseStatusCode(code))
+	}
+
+	for _, header := range t.capturedResponseHeaders {
+		if value := responseHeaders[header]; value != nil {
+			span.SetAttributes(attribute.StringSlice(fmt.Sprintf("http.response.header.%s", strings.ToLower(header)), value))
+		}
+	}
 }

 func proto(proto string) string {
@ -164,30 +288,10 @@ func proto(proto string) string {
 	}
 }

-// LogResponseCode used to log response code in span.
-func LogResponseCode(span trace.Span, code int, spanKind trace.SpanKind) {
-	if span != nil {
-		var status codes.Code
-		var desc string
-		switch spanKind {
-		case trace.SpanKindServer:
-			status, desc = ServerStatus(code)
-		case trace.SpanKindClient:
-			status, desc = ClientStatus(code)
-		default:
-			status, desc = DefaultStatus(code)
-		}
-		span.SetStatus(status, desc)
-		if code > 0 {
-			span.SetAttributes(semconv.HTTPResponseStatusCode(code))
-		}
-	}
-}
-
-// ServerStatus returns a span status code and message for an HTTP status code
+// serverStatus returns a span status code and message for an HTTP status code
 // value returned by a server. Status codes in the 400-499 range are not
 // returned as errors.
-func ServerStatus(code int) (codes.Code, string) {
+func serverStatus(code int) (codes.Code, string) {
 	if code < 100 || code >= 600 {
 		return codes.Error, fmt.Sprintf("Invalid HTTP status code %d", code)
 	}
@ -197,10 +301,10 @@ func ServerStatus(code int) (codes.Code, string) {
 	return codes.Unset, ""
 }

-// ClientStatus returns a span status code and message for an HTTP status code
+// clientStatus returns a span status code and message for an HTTP status code
 // value returned by a server. Status codes in the 400-499 range are not
 // returned as errors.
-func ClientStatus(code int) (codes.Code, string) {
+func clientStatus(code int) (codes.Code, string) {
 	if code < 100 || code >= 600 {
 		return codes.Error, fmt.Sprintf("Invalid HTTP status code %d", code)
 	}
@ -210,9 +314,9 @@ func ClientStatus(code int) (codes.Code, string) {
 	return codes.Unset, ""
 }

-// DefaultStatus returns a span status code and message for an HTTP status code
+// defaultStatus returns a span status code and message for an HTTP status code
 // value generated internally.
-func DefaultStatus(code int) (codes.Code, string) {
+func defaultStatus(code int) (codes.Code, string) {
 	if code < 100 || code >= 600 {
 		return codes.Error, fmt.Sprintf("Invalid HTTP status code %d", code)
 	}