Added support for Trace name truncation for traces

cmd/configuration.go

@@ -9,6 +9,7 @@ import (
 	"github.com/containous/traefik/configuration"
 	"github.com/containous/traefik/middlewares/accesslog"
 	"github.com/containous/traefik/middlewares/tracing"
+	"github.com/containous/traefik/middlewares/tracing/datadog"
 	"github.com/containous/traefik/middlewares/tracing/jaeger"
 	"github.com/containous/traefik/middlewares/tracing/zipkin"
 	"github.com/containous/traefik/ping"
@@ -220,6 +221,7 @@ func NewTraefikDefaultPointersConfiguration() *TraefikConfiguration {
 	defaultTracing := tracing.Tracing{
 		Backend:       "jaeger",
 		ServiceName:   "traefik",
+		SpanNameLimit: 0,
 		Jaeger: &jaeger.Config{
 			SamplingServerURL:  "http://localhost:5778/sampling",
 			SamplingType:       "const",
@@ -232,6 +234,11 @@ func NewTraefikDefaultPointersConfiguration() *TraefikConfiguration {
 			ID128Bit:     true,
 			Debug:        false,
 		},
+		DataDog: &datadog.Config{
+			LocalAgentHostPort: "localhost:8126",
+			GlobalTag:          "",
+			Debug:              false,
+		},
 	}
 
 	// default LifeCycle

docs/configuration/tracing.md

@@ -23,6 +23,13 @@ Træfik supports two backends: Jaeger and Zipkin.
   #
   serviceName = "traefik"
     
+  # Span name limit allows for name truncation in case of very long Frontend/Backend names
+  # This can prevent certain tracing providers to drop traces that exceed their length limits
+  #
+  # Default: 0 - no truncation will occur
+  # 
+  spanNameLimit = 0
+
   [tracing.jaeger]
     # Sampling Server URL is the address of jaeger-agent's HTTP sampling server
     #
@@ -73,6 +80,13 @@ Træfik supports two backends: Jaeger and Zipkin.
   #
   serviceName = "traefik"
     
+  # Span name limit allows for name truncation in case of very long Frontend/Backend names
+  # This can prevent certain tracing providers to drop traces that exceed their length limits
+  #
+  # Default: 0 - no truncation will occur
+  # 
+  spanNameLimit = 150
+
   [tracing.zipkin]
     # Zipking HTTP endpoint used to send data
     #
@@ -116,6 +130,13 @@ Træfik supports two backends: Jaeger and Zipkin.
   #
   serviceName = "traefik"
   
+  # Span name limit allows for name truncation in case of very long Frontend/Backend names
+  # This can prevent certain tracing providers to drop traces that exceed their length limits
+  #
+  # Default: 0 - no truncation will occur
+  # 
+  spanNameLimit = 100
+
   [tracing.datadog]
     # Local Agent Host Port instructs reporter to send spans to datadog-tracing-agent at this address
     #

middlewares/tracing/entrypoint.go

@@ -22,12 +22,10 @@ func (t *Tracing) NewEntryPoint(name string) negroni.Handler {
 }
 
 func (e *entryPointMiddleware) ServeHTTP(w http.ResponseWriter, r *http.Request, next http.HandlerFunc) {
-	opNameFunc := func(r *http.Request) string {
+	opNameFunc := generateEntryPointSpanName
-		return fmt.Sprintf("Entrypoint %s %s", e.entryPoint, r.Host)
-	}
 
 	ctx, _ := e.Extract(opentracing.HTTPHeaders, opentracing.HTTPHeadersCarrier(r.Header))
-	span := e.StartSpan(opNameFunc(r), ext.RPCServerOption(ctx))
+	span := e.StartSpan(opNameFunc(r, e.entryPoint, e.SpanNameLimit), ext.RPCServerOption(ctx))
 	ext.Component.Set(span, e.ServiceName)
 	LogRequest(span, r)
 	ext.SpanKindRPCServer.Set(span)
@@ -40,3 +38,20 @@ func (e *entryPointMiddleware) ServeHTTP(w http.ResponseWriter, r *http.Request,
 	LogResponseCode(span, recorder.Status())
 	span.Finish()
 }
+
+// generateEntryPointSpanName will return a Span name of an appropriate lenth based on the 'spanLimit' argument.  If needed, it will be truncated, but will not be less than 24 characters.
+func generateEntryPointSpanName(r *http.Request, entryPoint string, spanLimit int) string {
+	name := fmt.Sprintf("Entrypoint %s %s", entryPoint, r.Host)
+
+	if spanLimit > 0 && len(name) > spanLimit {
+		if spanLimit < EntryPointMaxLengthNumber {
+			log.Warnf("SpanNameLimit is set to be less than required static number of characters, defaulting to %d + 3", EntryPointMaxLengthNumber)
+			spanLimit = EntryPointMaxLengthNumber + 3
+		}
+		hash := computeHash(name)
+		limit := (spanLimit - EntryPointMaxLengthNumber) / 2
+		name = fmt.Sprintf("Entrypoint %s %s %s", truncateString(entryPoint, limit), truncateString(r.Host, limit), hash)
+	}
+
+	return name
+}

middlewares/tracing/entrypoint_test.go

@@ -0,0 +1,69 @@
+package tracing
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/opentracing/opentracing-go/ext"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestEntryPointMiddlewareServeHTTP(t *testing.T) {
+	expectedTags := map[string]interface{}{
+		"span.kind":   ext.SpanKindRPCServerEnum,
+		"http.method": "GET",
+		"component":   "",
+		"http.url":    "http://www.test.com",
+		"http.host":   "www.test.com",
+	}
+	testCases := []struct {
+		desc         string
+		entryPoint   string
+		tracing      *Tracing
+		expectedTags map[string]interface{}
+		expectedName string
+	}{
+		{
+			desc:       "no truncation test",
+			entryPoint: "test",
+			tracing: &Tracing{
+				SpanNameLimit: 0,
+				tracer:        &MockTracer{Span: &MockSpan{Tags: make(map[string]interface{})}},
+			},
+			expectedTags: expectedTags,
+			expectedName: "Entrypoint test www.test.com",
+		}, {
+			desc:       "basic test",
+			entryPoint: "test",
+			tracing: &Tracing{
+				SpanNameLimit: 25,
+				tracer:        &MockTracer{Span: &MockSpan{Tags: make(map[string]interface{})}},
+			},
+			expectedTags: expectedTags,
+			expectedName: "Entrypoint te... ww... 39b97e58",
+		},
+	}
+
+	for _, test := range testCases {
+		test := test
+		t.Run(test.desc, func(t *testing.T) {
+			t.Parallel()
+
+			e := &entryPointMiddleware{
+				entryPoint: test.entryPoint,
+				Tracing:    test.tracing,
+			}
+
+			next := func(http.ResponseWriter, *http.Request) {
+				span := test.tracing.tracer.(*MockTracer).Span
+
+				actual := span.Tags
+				assert.Equal(t, test.expectedTags, actual)
+				assert.Equal(t, test.expectedName, span.OpName)
+			}
+
+			e.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, "http://www.test.com", nil), next)
+		})
+	}
+}

middlewares/tracing/forwarder.go

@@ -23,7 +23,7 @@ func (t *Tracing) NewForwarderMiddleware(frontend, backend string) negroni.Handl
 		Tracing:  t,
 		frontend: frontend,
 		backend:  backend,
-		opName:   fmt.Sprintf("forward %s/%s", frontend, backend),
+		opName:   generateForwardSpanName(frontend, backend, t.SpanNameLimit),
 	}
 }
 
@@ -44,3 +44,20 @@ func (f *forwarderMiddleware) ServeHTTP(w http.ResponseWriter, r *http.Request,
 
 	LogResponseCode(span, recorder.Status())
 }
+
+// generateForwardSpanName will return a Span name of an appropriate lenth based on the 'spanLimit' argument.  If needed, it will be truncated, but will not be less than 21 characters
+func generateForwardSpanName(frontend, backend string, spanLimit int) string {
+	name := fmt.Sprintf("forward %s/%s", frontend, backend)
+
+	if spanLimit > 0 && len(name) > spanLimit {
+		if spanLimit < ForwardMaxLengthNumber {
+			log.Warnf("SpanNameLimit is set to be less than required static number of characters, defaulting to %d + 3", ForwardMaxLengthNumber)
+			spanLimit = ForwardMaxLengthNumber + 3
+		}
+		hash := computeHash(name)
+		limit := (spanLimit - ForwardMaxLengthNumber) / 2
+		name = fmt.Sprintf("forward %s/%s/%s", truncateString(frontend, limit), truncateString(backend, limit), hash)
+	}
+
+	return name
+}

middlewares/tracing/forwarder_test.go

@@ -0,0 +1,93 @@
+package tracing
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestTracingNewForwarderMiddleware(t *testing.T) {
+	testCases := []struct {
+		desc     string
+		tracer   *Tracing
+		frontend string
+		backend  string
+		expected *forwarderMiddleware
+	}{
+		{
+			desc: "Simple Forward Tracer without truncation and hashing",
+			tracer: &Tracing{
+				SpanNameLimit: 101,
+			},
+			frontend: "some-service.domain.tld",
+			backend:  "some-service.domain.tld",
+			expected: &forwarderMiddleware{
+				Tracing: &Tracing{
+					SpanNameLimit: 101,
+				},
+				frontend: "some-service.domain.tld",
+				backend:  "some-service.domain.tld",
+				opName:   "forward some-service.domain.tld/some-service.domain.tld",
+			},
+		}, {
+			desc: "Simple Forward Tracer with truncation and hashing",
+			tracer: &Tracing{
+				SpanNameLimit: 101,
+			},
+			frontend: "some-service-100.slug.namespace.environment.domain.tld",
+			backend:  "some-service-100.slug.namespace.environment.domain.tld",
+			expected: &forwarderMiddleware{
+				Tracing: &Tracing{
+					SpanNameLimit: 101,
+				},
+				frontend: "some-service-100.slug.namespace.environment.domain.tld",
+				backend:  "some-service-100.slug.namespace.environment.domain.tld",
+				opName:   "forward some-service-100.slug.namespace.enviro.../some-service-100.slug.namespace.enviro.../bc4a0d48",
+			},
+		},
+		{
+			desc: "Exactly 101 chars",
+			tracer: &Tracing{
+				SpanNameLimit: 101,
+			},
+			frontend: "some-service1.namespace.environment.domain.tld",
+			backend:  "some-service1.namespace.environment.domain.tld",
+			expected: &forwarderMiddleware{
+				Tracing: &Tracing{
+					SpanNameLimit: 101,
+				},
+				frontend: "some-service1.namespace.environment.domain.tld",
+				backend:  "some-service1.namespace.environment.domain.tld",
+				opName:   "forward some-service1.namespace.environment.domain.tld/some-service1.namespace.environment.domain.tld",
+			},
+		},
+		{
+			desc: "More than 101 chars",
+			tracer: &Tracing{
+				SpanNameLimit: 101,
+			},
+			frontend: "some-service1.frontend.namespace.environment.domain.tld",
+			backend:  "some-service1.backend.namespace.environment.domain.tld",
+			expected: &forwarderMiddleware{
+				Tracing: &Tracing{
+					SpanNameLimit: 101,
+				},
+				frontend: "some-service1.frontend.namespace.environment.domain.tld",
+				backend:  "some-service1.backend.namespace.environment.domain.tld",
+				opName:   "forward some-service1.frontend.namespace.envir.../some-service1.backend.namespace.enviro.../fa49dd23",
+			},
+		},
+	}
+
+	for _, test := range testCases {
+		test := test
+		t.Run(test.desc, func(t *testing.T) {
+			t.Parallel()
+
+			actual := test.tracer.NewForwarderMiddleware(test.frontend, test.backend)
+
+			assert.Equal(t, test.expected, actual)
+			assert.True(t, len(test.expected.opName) <= test.tracer.SpanNameLimit)
+		})
+	}
+}

middlewares/tracing/tracing.go

@@ -1,6 +1,7 @@
 package tracing
 
 import (
+	"crypto/sha256"
 	"fmt"
 	"io"
 	"net/http"
@@ -13,10 +14,20 @@ import (
 	"github.com/opentracing/opentracing-go/ext"
 )
 
+// ForwardMaxLengthNumber defines the number of static characters in the Forwarding Span Trace name : 8 chars for 'forward ' + 8 chars for hash + 2 chars for '_'.
+const ForwardMaxLengthNumber = 18
+
+// EntryPointMaxLengthNumber defines the number of static characters in the Entrypoint Span Trace name : 11 chars for 'Entrypoint ' + 8 chars for hash + 2 chars for '_'.
+const EntryPointMaxLengthNumber = 21
+
+// TraceNameHashLength defines the number of characters to use from the head of the generated hash.
+const TraceNameHashLength = 8
+
 // Tracing middleware
 type Tracing struct {
 	Backend       string          `description:"Selects the tracking backend ('jaeger','zipkin', 'datadog')." export:"true"`
 	ServiceName   string          `description:"Set the name for this service" export:"true"`
+	SpanNameLimit int             `description:"Set the maximum character limit for Span names (default 0 = no limit)" export:"true"`
 	Jaeger        *jaeger.Config  `description:"Settings for jaeger"`
 	Zipkin        *zipkin.Config  `description:"Settings for zipkin"`
 	DataDog       *datadog.Config `description:"Settings for DataDog"`
@@ -147,16 +158,40 @@ func SetError(r *http.Request) {
 	}
 }
 
-// SetErrorAndDebugLog flags the span associated with this request as in error and create a debug log
+// SetErrorAndDebugLog flags the span associated with this request as in error and create a debug log.
 func SetErrorAndDebugLog(r *http.Request, format string, args ...interface{}) {
 	SetError(r)
 	log.Debugf(format, args...)
 	LogEventf(r, format, args...)
 }
 
-// SetErrorAndWarnLog flags the span associated with this request as in error and create a debug log
+// SetErrorAndWarnLog flags the span associated with this request as in error and create a debug log.
 func SetErrorAndWarnLog(r *http.Request, format string, args ...interface{}) {
 	SetError(r)
 	log.Warnf(format, args...)
 	LogEventf(r, format, args...)
 }
+
+// truncateString reduces the length of the 'str' argument to 'num' - 3 and adds a '...' suffix to the tail.
+func truncateString(str string, num int) string {
+	text := str
+	if len(str) > num {
+		if num > 3 {
+			num -= 3
+		}
+		text = str[0:num] + "..."
+	}
+	return text
+}
+
+// computeHash returns the first TraceNameHashLength character of the sha256 hash for 'name' argument.
+func computeHash(name string) string {
+	data := []byte(name)
+	hash := sha256.New()
+	if _, err := hash.Write(data); err != nil {
+		// Impossible case
+		log.Errorf("Fail to create Span name hash for %s: %v", name, err)
+	}
+
+	return fmt.Sprintf("%x", hash.Sum(nil))[:TraceNameHashLength]
+}

middlewares/tracing/tracing_test.go

@@ -0,0 +1,133 @@
+package tracing
+
+import (
+	"testing"
+
+	"github.com/opentracing/opentracing-go"
+	"github.com/opentracing/opentracing-go/log"
+	"github.com/stretchr/testify/assert"
+)
+
+type MockTracer struct {
+	Span *MockSpan
+}
+
+type MockSpan struct {
+	OpName string
+	Tags   map[string]interface{}
+}
+
+type MockSpanContext struct {
+}
+
+// MockSpanContext:
+func (n MockSpanContext) ForeachBaggageItem(handler func(k, v string) bool) {}
+
+// MockSpan:
+func (n MockSpan) Context() opentracing.SpanContext { return MockSpanContext{} }
+func (n MockSpan) SetBaggageItem(key, val string) opentracing.Span {
+	return MockSpan{Tags: make(map[string]interface{})}
+}
+func (n MockSpan) BaggageItem(key string) string { return "" }
+func (n MockSpan) SetTag(key string, value interface{}) opentracing.Span {
+	n.Tags[key] = value
+	return n
+}
+func (n MockSpan) LogFields(fields ...log.Field)                          {}
+func (n MockSpan) LogKV(keyVals ...interface{})                           {}
+func (n MockSpan) Finish()                                                {}
+func (n MockSpan) FinishWithOptions(opts opentracing.FinishOptions)       {}
+func (n MockSpan) SetOperationName(operationName string) opentracing.Span { return n }
+func (n MockSpan) Tracer() opentracing.Tracer                             { return MockTracer{} }
+func (n MockSpan) LogEvent(event string)                                  {}
+func (n MockSpan) LogEventWithPayload(event string, payload interface{})  {}
+func (n MockSpan) Log(data opentracing.LogData)                           {}
+func (n MockSpan) Reset() {
+	n.Tags = make(map[string]interface{})
+}
+
+// StartSpan belongs to the Tracer interface.
+func (n MockTracer) StartSpan(operationName string, opts ...opentracing.StartSpanOption) opentracing.Span {
+	n.Span.OpName = operationName
+	return n.Span
+}
+
+// Inject belongs to the Tracer interface.
+func (n MockTracer) Inject(sp opentracing.SpanContext, format interface{}, carrier interface{}) error {
+	return nil
+}
+
+// Extract belongs to the Tracer interface.
+func (n MockTracer) Extract(format interface{}, carrier interface{}) (opentracing.SpanContext, error) {
+	return nil, opentracing.ErrSpanContextNotFound
+}
+
+func TestTruncateString(t *testing.T) {
+	testCases := []struct {
+		desc     string
+		text     string
+		limit    int
+		expected string
+	}{
+		{
+			desc:     "short text less than limit 10",
+			text:     "short",
+			limit:    10,
+			expected: "short",
+		},
+		{
+			desc:     "basic truncate with limit 10",
+			text:     "some very long pice of text",
+			limit:    10,
+			expected: "some ve...",
+		},
+		{
+			desc:     "truncate long FQDN to 39 chars",
+			text:     "some-service-100.slug.namespace.environment.domain.tld",
+			limit:    39,
+			expected: "some-service-100.slug.namespace.envi...",
+		},
+	}
+
+	for _, test := range testCases {
+		test := test
+		t.Run(test.desc, func(t *testing.T) {
+			t.Parallel()
+
+			actual := truncateString(test.text, test.limit)
+
+			assert.Equal(t, test.expected, actual)
+			assert.True(t, len(actual) <= test.limit)
+		})
+	}
+}
+
+func TestComputeHash(t *testing.T) {
+	testCases := []struct {
+		desc     string
+		text     string
+		expected string
+	}{
+		{
+			desc:     "hashing",
+			text:     "some very long pice of text",
+			expected: "0258ea1c",
+		},
+		{
+			desc:     "short text less than limit 10",
+			text:     "short",
+			expected: "f9b0078b",
+		},
+	}
+
+	for _, test := range testCases {
+		test := test
+		t.Run(test.desc, func(t *testing.T) {
+			t.Parallel()
+
+			actual := computeHash(test.text)
+
+			assert.Equal(t, test.expected, actual)
+		})
+	}
+}