baalajimaestro/traefik
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

Add captured headers options for tracing

Browse source 
Co-authored-by: Baptiste Mayelle <baptiste.mayelle@traefik.io>
This commit is contained in:
Romain 2024-03-11 11:50:04 +01:00 committed by GitHub
parent 86be0a4e6f
commit 709ff6fb09
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
15 changed files with 520 additions and 119 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
cmd/traefik/traefik.go
View file

@ -46,7 +46,6 @@ import (
"github.com/traefik/traefik/v3/pkg/tracing"
"github.com/traefik/traefik/v3/pkg/types"
"github.com/traefik/traefik/v3/pkg/version"
"go.opentelemetry.io/otel/trace"
)
func main() {
@ -563,7 +562,7 @@ func setupAccessLog(conf *types.AccessLog) *accesslog.Handler {
return accessLoggerMiddleware
}
func setupTracing(conf *static.Tracing) (trace.Tracer, io.Closer) {
func setupTracing(conf *static.Tracing) (*tracing.Tracer, io.Closer) {
if conf == nil {
return nil, nil
}

44
docs/content/observability/tracing/overview.md
View file

@ -116,3 +116,47 @@ tracing:
--tracing.globalAttributes.attr1=foo
--tracing.globalAttributes.attr2=bar
```
#### `capturedRequestHeaders`
_Optional, Default=empty_
Defines the list of request headers to add as attributes.
It applies to client and server kind spans.
```yaml tab="File (YAML)"
tracing:
capturedRequestHeaders:
- X-CustomHeader
```
```toml tab="File (TOML)"
[tracing]
capturedRequestHeaders = ["X-CustomHeader"]
```
```bash tab="CLI"
--tracing.capturedRequestHeaders[0]=X-CustomHeader
```
#### `capturedResponseHeaders`
_Optional, Default=empty_
Defines the list of response headers to add as attributes.
It applies to client and server kind spans.
```yaml tab="File (YAML)"
tracing:
capturedResponseHeaders:
- X-CustomHeader
```
```toml tab="File (TOML)"
[tracing]
capturedResponseHeaders = ["X-CustomHeader"]
```
```bash tab="CLI"
--tracing.capturedResponseHeaders[0]=X-CustomHeader
```

6
docs/content/reference/static-configuration/cli-ref.md
View file

@ -1017,6 +1017,12 @@ OpenTracing configuration. (Default: ```false```)
`--tracing.addinternals`:
Enables tracing for internal services (ping, dashboard, etc...). (Default: ```false```)
`--tracing.capturedrequestheaders`:
Request headers to add as attributes for server and client spans.
`--tracing.capturedresponseheaders`:
Response headers to add as attributes for server and client spans.
`--tracing.globalattributes.<name>`:
Defines additional attributes (key:value) on all spans.

6
docs/content/reference/static-configuration/env-ref.md
View file

@ -1017,6 +1017,12 @@ OpenTracing configuration. (Default: ```false```)
`TRAEFIK_TRACING_ADDINTERNALS`:
Enables tracing for internal services (ping, dashboard, etc...). (Default: ```false```)
`TRAEFIK_TRACING_CAPTUREDREQUESTHEADERS`:
Request headers to add as attributes for server and client spans.
`TRAEFIK_TRACING_CAPTUREDRESPONSEHEADERS`:
Response headers to add as attributes for server and client spans.
`TRAEFIK_TRACING_GLOBALATTRIBUTES_<NAME>`:
Defines additional attributes (key:value) on all spans.

2
docs/content/reference/static-configuration/file.toml
View file

@ -380,6 +380,8 @@
[tracing]
serviceName = "foobar"
capturedRequestHeaders = ["foobar", "foobar"]
capturedResponseHeaders = ["foobar", "foobar"]
sampleRate = 42.0
addInternals = true
[tracing.globalAttributes]

6
docs/content/reference/static-configuration/file.yaml
View file

@ -418,6 +418,12 @@ tracing:
globalAttributes:
name0: foobar
name1: foobar
capturedRequestHeaders:
- foobar
- foobar
capturedResponseHeaders:
- foobar
- foobar
sampleRate: 42
addInternals: true
otlp:

2
pkg/config/static/static_config.go
View file

@ -195,6 +195,8 @@ func (a *LifeCycle) SetDefaults() {
type Tracing struct {
ServiceName string `description:"Set the name for this service." json:"serviceName,omitempty" toml:"serviceName,omitempty" yaml:"serviceName,omitempty" export:"true"`
GlobalAttributes map[string]string `description:"Defines additional attributes (key:value) on all spans." json:"globalAttributes,omitempty" toml:"globalAttributes,omitempty" yaml:"globalAttributes,omitempty" export:"true"`
CapturedRequestHeaders []string `description:"Request headers to add as attributes for server and client spans." json:"capturedRequestHeaders,omitempty" toml:"capturedRequestHeaders,omitempty" yaml:"capturedRequestHeaders,omitempty" export:"true"`
CapturedResponseHeaders []string `description:"Response headers to add as attributes for server and client spans." json:"capturedResponseHeaders,omitempty" toml:"capturedResponseHeaders,omitempty" yaml:"capturedResponseHeaders,omitempty" export:"true"`
SampleRate float64 `description:"Sets the rate between 0.0 and 1.0 of requests to trace." json:"sampleRate,omitempty" toml:"sampleRate,omitempty" yaml:"sampleRate,omitempty" export:"true"`
AddInternals bool `description:"Enables tracing for internal services (ping, dashboard, etc...)." json:"addInternals,omitempty" toml:"addInternals,omitempty" yaml:"addInternals,omitempty" export:"true"`

13
pkg/middlewares/auth/forward.go
View file

@ -131,8 +131,11 @@ func (fa *forwardAuth) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
return
}
writeHeader(req, forwardReq, fa.trustForwardHeader, fa.authRequestHeaders)
var forwardSpan trace.Span
if tracer := tracing.TracerFromContext(req.Context()); tracer != nil {
var tracer *tracing.Tracer
if tracer = tracing.TracerFromContext(req.Context()); tracer != nil {
var tracingCtx context.Context
tracingCtx, forwardSpan = tracer.Start(req.Context(), "AuthRequest", trace.WithSpanKind(trace.SpanKindClient))
defer forwardSpan.End()
@ -140,11 +143,9 @@ func (fa *forwardAuth) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
forwardReq = forwardReq.WithContext(tracingCtx)
tracing.InjectContextIntoCarrier(forwardReq)
tracing.LogClientRequest(forwardSpan, forwardReq)
tracer.CaptureClientRequest(forwardSpan, forwardReq)
}
writeHeader(req, forwardReq, fa.trustForwardHeader, fa.authRequestHeaders)
forwardResponse, forwardErr := fa.client.Do(forwardReq)
if forwardErr != nil {
logMessage := fmt.Sprintf("Error calling %s. Cause: %s", fa.address, forwardErr)
@ -197,7 +198,7 @@ func (fa *forwardAuth) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
rw.Header().Set("Location", redirectURL.String())
}
tracing.LogResponseCode(forwardSpan, forwardResponse.StatusCode, trace.SpanKindClient)
tracer.CaptureResponse(forwardSpan, forwardResponse.Header, forwardResponse.StatusCode, trace.SpanKindClient)
rw.WriteHeader(forwardResponse.StatusCode)
if _, err = rw.Write(body); err != nil {
@ -228,7 +229,7 @@ func (fa *forwardAuth) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
}
}
tracing.LogResponseCode(forwardSpan, forwardResponse.StatusCode, trace.SpanKindClient)
tracer.CaptureResponse(forwardSpan, forwardResponse.Header, forwardResponse.StatusCode, trace.SpanKindClient)
req.RequestURI = req.URL.RequestURI()

186
pkg/middlewares/auth/forward_test.go
View file

@ -4,27 +4,25 @@ import (
"context"
"fmt"
"io"
"net"
"net/http"
"net/http/httptest"
"net/url"
"strconv"
"testing"
"github.com/containous/alice"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/traefik/traefik/v3/pkg/config/dynamic"
"github.com/traefik/traefik/v3/pkg/config/static"
tracingMiddleware "github.com/traefik/traefik/v3/pkg/middlewares/tracing"
"github.com/traefik/traefik/v3/pkg/testhelpers"
"github.com/traefik/traefik/v3/pkg/tracing"
"github.com/traefik/traefik/v3/pkg/tracing/opentelemetry"
"github.com/traefik/traefik/v3/pkg/types"
"github.com/traefik/traefik/v3/pkg/version"
"github.com/vulcand/oxy/v2/forward"
"go.opentelemetry.io/contrib/propagators/autoprop"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/sdk/resource"
sdktrace "go.opentelemetry.io/otel/sdk/trace"
"go.opentelemetry.io/otel/sdk/trace/tracetest"
semconv "go.opentelemetry.io/otel/semconv/v1.21.0"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/trace"
"go.opentelemetry.io/otel/trace/embedded"
)
func TestForwardAuthFail(t *testing.T) {
@ -466,64 +464,146 @@ func Test_writeHeader(t *testing.T) {
}
}
func TestForwardAuthUsesTracing(t *testing.T) {
func TestForwardAuthTracing(t *testing.T) {
type expected struct {
name string
attributes []attribute.KeyValue
}
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Header.Get("Traceparent") == "" {
t.Errorf("expected Traceparent header to be present in request")
}
w.Header().Set("X-Bar", "foo")
w.Header().Add("X-Bar", "bar")
w.WriteHeader(http.StatusNotFound)
}))
t.Cleanup(server.Close)
parse, err := url.Parse(server.URL)
require.NoError(t, err)
_, serverPort, err := net.SplitHostPort(parse.Host)
require.NoError(t, err)
serverPortInt, err := strconv.Atoi(serverPort)
require.NoError(t, err)
testCases := []struct {
desc string
expected []expected
}{
{
desc: "basic test",
expected: []expected{
{
name: "initial",
attributes: []attribute.KeyValue{
attribute.String("span.kind", "unspecified"),
},
},
{
name: "AuthRequest",
attributes: []attribute.KeyValue{
attribute.String("span.kind", "client"),
attribute.String("http.request.method", "GET"),
attribute.String("network.protocol.version", "1.1"),
attribute.String("url.full", server.URL),
attribute.String("url.scheme", "http"),
attribute.String("user_agent.original", ""),
attribute.String("network.peer.address", "127.0.0.1"),
attribute.String("network.peer.port", serverPort),
attribute.String("server.address", "127.0.0.1"),
attribute.Int64("server.port", int64(serverPortInt)),
attribute.StringSlice("http.request.header.x-foo", []string{"foo", "bar"}),
attribute.Int64("http.response.status_code", int64(404)),
attribute.StringSlice("http.response.header.x-bar", []string{"foo", "bar"}),
},
},
},
},
}
for _, test := range testCases {
t.Run(test.desc, func(t *testing.T) {
next := http.Handler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
auth := dynamic.ForwardAuth{
Address: server.URL,
AuthRequestHeaders: []string{"X-Foo"},
}
exporter := tracetest.NewInMemoryExporter()
tres, err := resource.New(context.Background(),
resource.WithAttributes(semconv.ServiceNameKey.String("traefik")),
resource.WithAttributes(semconv.ServiceVersionKey.String(version.Version)),
resource.WithFromEnv(),
resource.WithTelemetrySDK(),
)
next, err := NewForward(context.Background(), next, auth, "authTest")
require.NoError(t, err)
tracerProvider := sdktrace.NewTracerProvider(
sdktrace.WithSampler(sdktrace.AlwaysSample()),
sdktrace.WithResource(tres),
sdktrace.WithBatcher(exporter),
)
otel.SetTracerProvider(tracerProvider)
req := httptest.NewRequest(http.MethodGet, "http://www.test.com/search?q=Opentelemetry", nil)
req.RemoteAddr = "10.0.0.1:1234"
req.Header.Set("User-Agent", "forward-test")
req.Header.Set("X-Forwarded-Proto", "http")
req.Header.Set("X-Foo", "foo")
req.Header.Add("X-Foo", "bar")
config := &static.Tracing{
ServiceName: "testApp",
SampleRate: 1,
OTLP: &opentelemetry.Config{
HTTP: &types.OtelHTTP{
Endpoint: "http://127.0.0.1:8080",
},
},
otel.SetTextMapPropagator(autoprop.NewTextMapPropagator())
mockTracer := &mockTracer{}
tracer := tracing.NewTracer(mockTracer, []string{"X-Foo"}, []string{"X-Bar"})
initialCtx, initialSpan := tracer.Start(req.Context(), "initial")
defer initialSpan.End()
req = req.WithContext(initialCtx)
recorder := httptest.NewRecorder()
next.ServeHTTP(recorder, req)
for i, span := range mockTracer.spans {
assert.Equal(t, test.expected[i].name, span.name)
assert.Equal(t, test.expected[i].attributes, span.attributes)
}
tr, closer, err := tracing.NewTracing(config)
require.NoError(t, err)
t.Cleanup(func() {
_ = closer.Close()
})
next, err = NewForward(context.Background(), next, auth, "authTest")
require.NoError(t, err)
chain := alice.New(tracingMiddleware.WrapEntryPointHandler(context.Background(), tr, "tracingTest"))
next, err = chain.Then(next)
require.NoError(t, err)
ts := httptest.NewServer(next)
t.Cleanup(ts.Close)
req := testhelpers.MustNewRequest(http.MethodGet, ts.URL, nil)
res, err := http.DefaultClient.Do(req)
require.NoError(t, err)
assert.Equal(t, http.StatusOK, res.StatusCode)
}
}
type mockTracer struct {
embedded.Tracer
spans []*mockSpan
}
var _ trace.Tracer = &mockTracer{}
func (t *mockTracer) Start(ctx context.Context, name string, opts ...trace.SpanStartOption) (context.Context, trace.Span) {
config := trace.NewSpanStartConfig(opts...)
span := &mockSpan{}
span.SetName(name)
span.SetAttributes(attribute.String("span.kind", config.SpanKind().String()))
span.SetAttributes(config.Attributes()...)
t.spans = append(t.spans, span)
return trace.ContextWithSpan(ctx, span), span
}
// mockSpan is an implementation of Span that preforms no operations.
type mockSpan struct {
embedded.Span
name string
attributes []attribute.KeyValue
}
var _ trace.Span = &mockSpan{}
func (*mockSpan) SpanContext() trace.SpanContext {
return trace.NewSpanContext(trace.SpanContextConfig{TraceID: trace.TraceID{1}, SpanID: trace.SpanID{1}})
}
func (*mockSpan) IsRecording() bool { return false }
func (s *mockSpan) SetStatus(_ codes.Code, _ string) {}
func (s *mockSpan) SetAttributes(kv ...attribute.KeyValue) {
s.attributes = append(s.attributes, kv...)
}
func (s *mockSpan) End(...trace.SpanEndOption) {}
func (s *mockSpan) RecordError(_ error, _ ...trace.EventOption) {}
func (s *mockSpan) AddEvent(_ string, _ ...trace.EventOption) {}
func (s *mockSpan) SetName(name string) { s.name = name }
func (s *mockSpan) TracerProvider() trace.TracerProvider {
return nil
}

15
pkg/middlewares/tracing/entrypoint.go
View file

@ -2,6 +2,7 @@ package tracing
import (
"context"
"errors"
"net/http"
"github.com/containous/alice"
@ -16,20 +17,24 @@ const (
)
type entryPointTracing struct {
tracer trace.Tracer
tracer *tracing.Tracer
entryPoint string
next http.Handler
}
// WrapEntryPointHandler Wraps tracing to alice.Constructor.
func WrapEntryPointHandler(ctx context.Context, tracer trace.Tracer, entryPointName string) alice.Constructor {
func WrapEntryPointHandler(ctx context.Context, tracer *tracing.Tracer, entryPointName string) alice.Constructor {
return func(next http.Handler) (http.Handler, error) {
if tracer == nil {
return nil, errors.New("unexpected nil tracer")
}
return newEntryPoint(ctx, tracer, entryPointName, next), nil
}
}
// newEntryPoint creates a new tracing middleware for incoming requests.
func newEntryPoint(ctx context.Context, tracer trace.Tracer, entryPointName string, next http.Handler) http.Handler {
func newEntryPoint(ctx context.Context, tracer *tracing.Tracer, entryPointName string, next http.Handler) http.Handler {
middlewares.GetLogger(ctx, "tracing", entryPointTypeName).Debug().Msg("Creating middleware")
return &entryPointTracing{
@ -48,10 +53,10 @@ func (e *entryPointTracing) ServeHTTP(rw http.ResponseWriter, req *http.Request)
span.SetAttributes(attribute.String("entry_point", e.entryPoint))
tracing.LogServerRequest(span, req)
e.tracer.CaptureServerRequest(span, req)
recorder := newStatusCodeRecorder(rw, http.StatusOK)
e.next.ServeHTTP(recorder, req)
tracing.LogResponseCode(span, recorder.Status(), trace.SpanKindServer)
e.tracer.CaptureResponse(span, recorder.Header(), recorder.Status(), trace.SpanKindServer)
}

14
pkg/middlewares/tracing/entrypoint_test.go
View file

@ -7,6 +7,7 @@ import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/traefik/traefik/v3/pkg/tracing"
"go.opentelemetry.io/otel/attribute"
)
@ -42,7 +43,9 @@ func TestEntryPointMiddleware(t *testing.T) {
attribute.String("client.address", "10.0.0.1"),
attribute.Int64("client.port", int64(1234)),
attribute.String("client.socket.address", ""),
attribute.StringSlice("http.request.header.x-foo", []string{"foo", "bar"}),
attribute.Int64("http.response.status_code", int64(404)),
attribute.StringSlice("http.response.header.x-bar", []string{"foo", "bar"}),
},
},
},
@ -55,17 +58,20 @@ func TestEntryPointMiddleware(t *testing.T) {
req.RemoteAddr = "10.0.0.1:1234"
req.Header.Set("User-Agent", "entrypoint-test")
req.Header.Set("X-Forwarded-Proto", "http")
req.Header.Set("X-Foo", "foo")
req.Header.Add("X-Foo", "bar")
next := http.HandlerFunc(func(rw http.ResponseWriter, _ *http.Request) {
rw.Header().Set("X-Bar", "foo")
rw.Header().Add("X-Bar", "bar")
rw.WriteHeader(http.StatusNotFound)
})
tracer := &mockTracer{}
handler := newEntryPoint(context.Background(), tracer, test.entryPoint, next)
mockTracer := &mockTracer{}
handler := newEntryPoint(context.Background(), tracing.NewTracer(mockTracer, []string{"X-Foo"}, []string{"X-Bar"}), test.entryPoint, next)
handler.ServeHTTP(rw, req)
for _, span := range tracer.spans {
for _, span := range mockTracer.spans {
assert.Equal(t, test.expected.name, span.name)
assert.Equal(t, test.expected.attributes, span.attributes)
}

6
pkg/server/middleware/observability.go
View file

@ -15,7 +15,7 @@ import (
"github.com/traefik/traefik/v3/pkg/middlewares/capture"
metricsMiddle "github.com/traefik/traefik/v3/pkg/middlewares/metrics"
tracingMiddle "github.com/traefik/traefik/v3/pkg/middlewares/tracing"
"go.opentelemetry.io/otel/trace"
"github.com/traefik/traefik/v3/pkg/tracing"
)
// ObservabilityMgr is a manager for observability (AccessLogs, Metrics and Tracing) enablement.
@ -23,12 +23,12 @@ type ObservabilityMgr struct {
config static.Configuration
accessLoggerMiddleware *accesslog.Handler
metricsRegistry metrics.Registry
tracer trace.Tracer
tracer *tracing.Tracer
tracerCloser io.Closer
}
// NewObservabilityMgr creates a new ObservabilityMgr.
func NewObservabilityMgr(config static.Configuration, metricsRegistry metrics.Registry, accessLoggerMiddleware *accesslog.Handler, tracer trace.Tracer, tracerCloser io.Closer) *ObservabilityMgr {
func NewObservabilityMgr(config static.Configuration, metricsRegistry metrics.Registry, accessLoggerMiddleware *accesslog.Handler, tracer *tracing.Tracer, tracerCloser io.Closer) *ObservabilityMgr {
return &ObservabilityMgr{
config: config,
metricsRegistry: metricsRegistry,

10
pkg/server/service/tracing_roundtripper.go
View file

@ -14,25 +14,27 @@ type wrapper struct {
func (t *wrapper) RoundTrip(req *http.Request) (*http.Response, error) {
var span trace.Span
if tracer := tracing.TracerFromContext(req.Context()); tracer != nil {
var tracer *tracing.Tracer
if tracer = tracing.TracerFromContext(req.Context()); tracer != nil {
var tracingCtx context.Context
tracingCtx, span = tracer.Start(req.Context(), "ReverseProxy", trace.WithSpanKind(trace.SpanKindClient))
defer span.End()
req = req.WithContext(tracingCtx)
tracing.LogClientRequest(span, req)
tracer.CaptureClientRequest(span, req)
tracing.InjectContextIntoCarrier(req)
}
response, err := t.rt.RoundTrip(req)
if err != nil {
statusCode := computeStatusCode(err)
tracing.LogResponseCode(span, statusCode, trace.SpanKindClient)
tracer.CaptureResponse(span, nil, statusCode, trace.SpanKindClient)
return response, err
}
tracing.LogResponseCode(span, response.StatusCode, trace.SpanKindClient)
tracer.CaptureResponse(span, response.Header, response.StatusCode, trace.SpanKindClient)
return response, nil
}

138
pkg/server/service/tracing_roundtripper_test.go Normal file
View file

@ -0,0 +1,138 @@
package service
import (
"context"
"net/http"
"net/http/httptest"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/traefik/traefik/v3/pkg/tracing"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/trace"
"go.opentelemetry.io/otel/trace/embedded"
)
func TestTracingRoundTripper(t *testing.T) {
type expected struct {
name string
attributes []attribute.KeyValue
}
testCases := []struct {
desc string
expected []expected
}{
{
desc: "basic test",
expected: []expected{
{
name: "initial",
attributes: []attribute.KeyValue{
attribute.String("span.kind", "unspecified"),
},
},
{
name: "ReverseProxy",
attributes: []attribute.KeyValue{
attribute.String("span.kind", "client"),
attribute.String("http.request.method", "GET"),
attribute.String("network.protocol.version", "1.1"),
attribute.String("url.full", "http://www.test.com/search?q=Opentelemetry"),
attribute.String("url.scheme", "http"),
attribute.String("user_agent.original", "reverse-test"),
attribute.String("network.peer.address", ""),
attribute.String("server.address", "www.test.com"),
attribute.String("network.peer.port", "80"),
attribute.Int64("server.port", int64(80)),
attribute.StringSlice("http.request.header.x-foo", []string{"foo", "bar"}),
attribute.Int64("http.response.status_code", int64(404)),
attribute.StringSlice("http.response.header.x-bar", []string{"foo", "bar"}),
},
},
},
},
}
for _, test := range testCases {
t.Run(test.desc, func(t *testing.T) {
req := httptest.NewRequest(http.MethodGet, "http://www.test.com/search?q=Opentelemetry", nil)
req.RemoteAddr = "10.0.0.1:1234"
req.Header.Set("User-Agent", "reverse-test")
req.Header.Set("X-Forwarded-Proto", "http")
req.Header.Set("X-Foo", "foo")
req.Header.Add("X-Foo", "bar")
mockTracer := &mockTracer{}
tracer := tracing.NewTracer(mockTracer, []string{"X-Foo"}, []string{"X-Bar"})
initialCtx, initialSpan := tracer.Start(req.Context(), "initial")
defer initialSpan.End()
req = req.WithContext(initialCtx)
tracingRoundTripper := newTracingRoundTripper(roundTripperFn(func(req *http.Request) (*http.Response, error) {
return &http.Response{
Header: map[string][]string{
"X-Bar": {"foo", "bar"},
},
StatusCode: http.StatusNotFound,
}, nil
}))
_, err := tracingRoundTripper.RoundTrip(req)
require.NoError(t, err)
for i, span := range mockTracer.spans {
assert.Equal(t, test.expected[i].name, span.name)
assert.Equal(t, test.expected[i].attributes, span.attributes)
}
})
}
}
type mockTracer struct {
embedded.Tracer
spans []*mockSpan
}
var _ trace.Tracer = &mockTracer{}
func (t *mockTracer) Start(ctx context.Context, name string, opts ...trace.SpanStartOption) (context.Context, trace.Span) {
config := trace.NewSpanStartConfig(opts...)
span := &mockSpan{}
span.SetName(name)
span.SetAttributes(attribute.String("span.kind", config.SpanKind().String()))
span.SetAttributes(config.Attributes()...)
t.spans = append(t.spans, span)
return trace.ContextWithSpan(ctx, span), span
}
// mockSpan is an implementation of Span that preforms no operations.
type mockSpan struct {
embedded.Span
name string
attributes []attribute.KeyValue
}
var _ trace.Span = &mockSpan{}
func (*mockSpan) SpanContext() trace.SpanContext {
return trace.NewSpanContext(trace.SpanContextConfig{TraceID: trace.TraceID{1}, SpanID: trace.SpanID{1}})
}
func (*mockSpan) IsRecording() bool { return false }
func (s *mockSpan) SetStatus(_ codes.Code, _ string) {}
func (s *mockSpan) SetAttributes(kv ...attribute.KeyValue) {
s.attributes = append(s.attributes, kv...)
}
func (s *mockSpan) End(...trace.SpanEndOption) {}
func (s *mockSpan) RecordError(_ error, _ ...trace.EventOption) {}
func (s *mockSpan) AddEvent(_ string, _ ...trace.EventOption) {}
func (s *mockSpan) SetName(name string) { s.name = name }
func (s *mockSpan) TracerProvider() trace.TracerProvider {
return nil
}

180
pkg/tracing/tracing.go
View file

@ -7,6 +7,7 @@ import (
"net"
"net/http"
"strconv"
"strings"
"github.com/rs/zerolog/log"
"github.com/traefik/traefik/v3/pkg/config/static"
@ -26,7 +27,7 @@ type Backend interface {
}
// NewTracing Creates a Tracing.
func NewTracing(conf *static.Tracing) (trace.Tracer, io.Closer, error) {
func NewTracing(conf *static.Tracing) (*Tracer, io.Closer, error) {
var backend Backend
if conf.OTLP != nil {
@ -41,11 +42,16 @@ func NewTracing(conf *static.Tracing) (trace.Tracer, io.Closer, error) {
otel.SetTextMapPropagator(autoprop.NewTextMapPropagator())
return backend.Setup(conf.ServiceName, conf.SampleRate, conf.GlobalAttributes)
tr, closer, err := backend.Setup(conf.ServiceName, conf.SampleRate, conf.GlobalAttributes)
if err != nil {
return nil, nil, err
}
return NewTracer(tr, conf.CapturedRequestHeaders, conf.CapturedResponseHeaders), closer, nil
}
// TracerFromContext extracts the trace.Tracer from the given context.
func TracerFromContext(ctx context.Context) trace.Tracer {
func TracerFromContext(ctx context.Context) *Tracer {
// Prevent picking trace.noopSpan tracer.
if !trace.SpanContextFromContext(ctx).IsValid() {
return nil
@ -53,7 +59,12 @@ func TracerFromContext(ctx context.Context) trace.Tracer {
span := trace.SpanFromContext(ctx)
if span != nil && span.TracerProvider() != nil {
return span.TracerProvider().Tracer("github.com/traefik/traefik")
tracer := span.TracerProvider().Tracer("github.com/traefik/traefik")
if tracer, ok := tracer.(*Tracer); ok {
return tracer
}
return nil
}
return nil
@ -78,10 +89,73 @@ func SetStatusErrorf(ctx context.Context, format string, args ...interface{}) {
}
}
// LogClientRequest used to add span attributes from the request as a Client.
// TODO: the semconv does not implement Semantic Convention v1.23.0.
func LogClientRequest(span trace.Span, r *http.Request) {
if r == nil || span == nil {
// Span is trace.Span wrapping the Traefik TracerProvider.
type Span struct {
trace.Span
tracerProvider *TracerProvider
}
// TracerProvider returns the span's TraceProvider.
func (s Span) TracerProvider() trace.TracerProvider {
return s.tracerProvider
}
// TracerProvider is trace.TracerProvider wrapping the Traefik Tracer implementation.
type TracerProvider struct {
trace.TracerProvider
tracer *Tracer
}
// Tracer returns the trace.Tracer for the given options.
// It returns specifically the Traefik Tracer when requested.
func (t TracerProvider) Tracer(name string, options ...trace.TracerOption) trace.Tracer {
if name == "github.com/traefik/traefik" {
return t.tracer
}
return t.TracerProvider.Tracer(name, options...)
}
// Tracer is trace.Tracer with additional properties.
type Tracer struct {
trace.Tracer
capturedRequestHeaders []string
capturedResponseHeaders []string
}
// NewTracer builds and configures a new Tracer.
func NewTracer(tracer trace.Tracer, capturedRequestHeaders, capturedResponseHeaders []string) *Tracer {
return &Tracer{
Tracer: tracer,
capturedRequestHeaders: capturedRequestHeaders,
capturedResponseHeaders: capturedResponseHeaders,
}
}
// Start starts a new span.
// spancheck linter complains about span.End not being called, but this is expected here,
// hence its deactivation.
//
//nolint:spancheck
func (t *Tracer) Start(ctx context.Context, spanName string, opts ...trace.SpanStartOption) (context.Context, trace.Span) {
if t == nil {
return ctx, nil
}
spanCtx, span := t.Tracer.Start(ctx, spanName, opts...)
wrappedSpan := &Span{Span: span, tracerProvider: &TracerProvider{tracer: t}}
return trace.ContextWithSpan(spanCtx, wrappedSpan), wrappedSpan
}
// CaptureClientRequest used to add span attributes from the request as a Client.
// TODO: need to update the semconv package as it does not implement fully Semantic Convention v1.23.0.
func (t *Tracer) CaptureClientRequest(span trace.Span, r *http.Request) {
if t == nil || span == nil || r == nil {
return
}
@ -113,12 +187,23 @@ func LogClientRequest(span trace.Span, r *http.Request) {
span.SetAttributes(semconv.ServerAddress(host))
span.SetAttributes(semconv.ServerPort(intPort))
}
for _, header := range t.capturedRequestHeaders {
// User-agent is already part of the semantic convention as a recommended attribute.
if strings.EqualFold(header, "User-Agent") {
continue
}
if value := r.Header[header]; value != nil {
span.SetAttributes(attribute.StringSlice(fmt.Sprintf("http.request.header.%s", strings.ToLower(header)), value))
}
}
}
// LogServerRequest used to add span attributes from the request as a Server.
// TODO: the semconv does not implement Semantic Convention v1.23.0.
func LogServerRequest(span trace.Span, r *http.Request) {
if r == nil {
// CaptureServerRequest used to add span attributes from the request as a Server.
// TODO: need to update the semconv package as it does not implement fully Semantic Convention v1.23.0.
func (t *Tracer) CaptureServerRequest(span trace.Span, r *http.Request) {
if t == nil || span == nil || r == nil {
return
}
@ -147,6 +232,45 @@ func LogServerRequest(span trace.Span, r *http.Request) {
}
span.SetAttributes(semconv.ClientSocketAddress(r.Header.Get("X-Forwarded-For")))
for _, header := range t.capturedRequestHeaders {
// User-agent is already part of the semantic convention as a recommended attribute.
if strings.EqualFold(header, "User-Agent") {
continue
}
if value := r.Header[header]; value != nil {
span.SetAttributes(attribute.StringSlice(fmt.Sprintf("http.request.header.%s", strings.ToLower(header)), value))
}
}
}
// CaptureResponse captures the response attributes to the span.
func (t *Tracer) CaptureResponse(span trace.Span, responseHeaders http.Header, code int, spanKind trace.SpanKind) {
if t == nil || span == nil {
return
}
var status codes.Code
var desc string
switch spanKind {
case trace.SpanKindServer:
status, desc = serverStatus(code)
case trace.SpanKindClient:
status, desc = clientStatus(code)
default:
status, desc = defaultStatus(code)
}
span.SetStatus(status, desc)
if code > 0 {
span.SetAttributes(semconv.HTTPResponseStatusCode(code))
}
for _, header := range t.capturedResponseHeaders {
if value := responseHeaders[header]; value != nil {
span.SetAttributes(attribute.StringSlice(fmt.Sprintf("http.response.header.%s", strings.ToLower(header)), value))
}
}
}
func proto(proto string) string {
@ -164,30 +288,10 @@ func proto(proto string) string {
}
}
// LogResponseCode used to log response code in span.
func LogResponseCode(span trace.Span, code int, spanKind trace.SpanKind) {
if span != nil {
var status codes.Code
var desc string
switch spanKind {
case trace.SpanKindServer:
status, desc = ServerStatus(code)
case trace.SpanKindClient:
status, desc = ClientStatus(code)
default:
status, desc = DefaultStatus(code)
}
span.SetStatus(status, desc)
if code > 0 {
span.SetAttributes(semconv.HTTPResponseStatusCode(code))
}
}
}
// ServerStatus returns a span status code and message for an HTTP status code
// serverStatus returns a span status code and message for an HTTP status code
// value returned by a server. Status codes in the 400-499 range are not
// returned as errors.
func ServerStatus(code int) (codes.Code, string) {
func serverStatus(code int) (codes.Code, string) {
if code < 100 || code >= 600 {
return codes.Error, fmt.Sprintf("Invalid HTTP status code %d", code)
}
@ -197,10 +301,10 @@ func ServerStatus(code int) (codes.Code, string) {
return codes.Unset, ""
}
// ClientStatus returns a span status code and message for an HTTP status code
// clientStatus returns a span status code and message for an HTTP status code
// value returned by a server. Status codes in the 400-499 range are not
// returned as errors.
func ClientStatus(code int) (codes.Code, string) {
func clientStatus(code int) (codes.Code, string) {
if code < 100 || code >= 600 {
return codes.Error, fmt.Sprintf("Invalid HTTP status code %d", code)
}
@ -210,9 +314,9 @@ func ClientStatus(code int) (codes.Code, string) {
return codes.Unset, ""
}
// DefaultStatus returns a span status code and message for an HTTP status code
// defaultStatus returns a span status code and message for an HTTP status code
// value generated internally.
func DefaultStatus(code int) (codes.Code, string) {
func defaultStatus(code int) (codes.Code, string) {
if code < 100 || code >= 600 {
return codes.Error, fmt.Sprintf("Invalid HTTP status code %d", code)
}