Semconv OTLP stable HTTP metrics

cmd/traefik/traefik.go

@@ -195,10 +195,17 @@ func setupServer(staticConfiguration *static.Configuration) (*server.Server, err
 	// Observability
 
 	metricRegistries := registerMetricClients(staticConfiguration.Metrics)
+	var semConvMetricRegistry *metrics.SemConvMetricsRegistry
+	if staticConfiguration.Metrics != nil && staticConfiguration.Metrics.OTLP != nil {
+		semConvMetricRegistry, err = metrics.NewSemConvMetricRegistry(ctx, staticConfiguration.Metrics.OTLP)
+		if err != nil {
+			return nil, fmt.Errorf("unable to create SemConv metric registry: %w", err)
+		}
+	}
 	metricsRegistry := metrics.NewMultiRegistry(metricRegistries)
 	accessLog := setupAccessLog(staticConfiguration.AccessLog)
 	tracer, tracerCloser := setupTracing(staticConfiguration.Tracing)
-	observabilityMgr := middleware.NewObservabilityMgr(*staticConfiguration, metricsRegistry, accessLog, tracer, tracerCloser)
+	observabilityMgr := middleware.NewObservabilityMgr(*staticConfiguration, metricsRegistry, semConvMetricRegistry, accessLog, tracer, tracerCloser)
 
 	// Entrypoints
 

docs/content/observability/metrics/overview.md

@@ -12,7 +12,8 @@ Traefik supports these metrics backends:
 - [Prometheus](./prometheus.md)
 - [StatsD](./statsd.md)
 
-Traefik Proxy hosts an official Grafana dashboard for both [on-premises](https://grafana.com/grafana/dashboards/17346) and [Kubernetes](https://grafana.com/grafana/dashboards/17347) deployments.
+Traefik Proxy hosts an official Grafana dashboard for both [on-premises](https://grafana.com/grafana/dashboards/17346)
+and [Kubernetes](https://grafana.com/grafana/dashboards/17347) deployments.
 
 ## Common Options
 
@@ -86,7 +87,7 @@ traefik_tls_certs_not_after
 Here is a comprehensive list of labels that are provided by the global metrics:
 
 | Label        | Description                            | example              |
-|---------------|----------------------------------------|----------------------|
+|--------------|----------------------------------------|----------------------|
 | `entrypoint` | Entrypoint that handled the connection | "example_entrypoint" |
 | `protocol`   | Connection protocol                    | "TCP"                |
 
@@ -281,3 +282,49 @@ Here is a comprehensive list of labels that are provided by the metrics:
     If the HTTP method verb on a request is not one defined in the set of common methods for [`HTTP/1.1`](https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods)
     or the [`PRI`](https://datatracker.ietf.org/doc/html/rfc7540#section-11.6) verb (for `HTTP/2`),
     then the value for the method label becomes `EXTENSION_METHOD`.
+
+## Semantic Conventions for HTTP Metrics
+
+Traefik Proxy follows [official OTLP semantic conventions v1.23.1](https://github.com/open-telemetry/semantic-conventions/blob/v1.23.1/docs/http/http-metrics.md).
+
+### HTTP Server
+
+| Metric                        | Type      | [Labels](#labels)                                                                                                                        | Description                       |
+|-------------------------------|-----------|------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------|
+| http.server.request.duration	 | Histogram | `error.type`, `http.request.method`, `http.response.status_code`, `network.protocol.name`, `server.address`, `server.port`, `url.scheme` | Duration of HTTP server requests  |
+
+#### Labels
+
+Here is a comprehensive list of labels that are provided by the metrics:
+
+| Label                       | Description                                                  | example       |
+|-----------------------------|--------------------------------------------------------------|---------------|
+| `error.type`                | Describes a class of error the operation ended with          | "500"         |
+| `http.request.method`       | HTTP request method                                          | "GET"         |
+| `http.response.status_code` | HTTP response status code                                    | "200"         |
+| `network.protocol.name`     | OSI application layer or non-OSI equivalent                  | "http/1.1"    |
+| `network.protocol.version`  | Version of the protocol specified in `network.protocol.name` | "1.1"         |
+| `server.address`            | Name of the local HTTP server that received the request      | "example.com" |
+| `server.port`               | Port of the local HTTP server that received the request      | "80"          |
+| `url.scheme`                | The URI scheme component identifying the used protocol       | "http"        |
+
+### HTTP Client
+
+| Metric                        | Type      | [Labels](#labels)                                                                                                                        | Description                       |
+|-------------------------------|-----------|------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------|
+| http.client.request.duration	 | Histogram | `error.type`, `http.request.method`, `http.response.status_code`, `network.protocol.name`, `server.address`, `server.port`, `url.scheme` | Duration of HTTP client requests  |
+
+#### Labels
+
+Here is a comprehensive list of labels that are provided by the metrics:
+
+| Label                       | Description                                                  | example       |
+|-----------------------------|--------------------------------------------------------------|---------------|
+| `error.type`                | Describes a class of error the operation ended with          | "500"         |
+| `http.request.method`       | HTTP request method                                          | "GET"         |
+| `http.response.status_code` | HTTP response status code                                   | "200"         |
+| `network.protocol.name`     | OSI application layer or non-OSI equivalent                  | "http/1.1"    |
+| `network.protocol.version`  | Version of the protocol specified in `network.protocol.name` | "1.1"         |
+| `server.address`            | Name of the local HTTP server that received the request      | "example.com" |
+| `server.port`               | Port of the local HTTP server that received the request      | "80"          |
+| `url.scheme`                | The URI scheme component identifying the used protocol       | "http"        |

docs/content/reference/static-configuration/cli-ref.md

@@ -337,7 +337,7 @@ Enable metrics on routers. (Default: ```false```)
 Enable metrics on services. (Default: ```true```)
 
 `--metrics.otlp.explicitboundaries`:  
-Boundaries for latency metrics. (Default: ```0.005000, 0.010000, 0.025000, 0.050000, 0.100000, 0.250000, 0.500000, 1.000000, 2.500000, 5.000000, 10.000000```)
+Boundaries for latency metrics. (Default: ```0.005000, 0.010000, 0.025000, 0.050000, 0.075000, 0.100000, 0.250000, 0.500000, 0.750000, 1.000000, 2.500000, 5.000000, 7.500000, 10.000000```)
 
 `--metrics.otlp.grpc.endpoint`:  
 Sets the gRPC endpoint (host:port) of the collector. (Default: ```localhost:4317```)

docs/content/reference/static-configuration/env-ref.md

@@ -337,7 +337,7 @@ Enable metrics on routers. (Default: ```false```)
 Enable metrics on services. (Default: ```true```)
 
 `TRAEFIK_METRICS_OTLP_EXPLICITBOUNDARIES`:  
-Boundaries for latency metrics. (Default: ```0.005000, 0.010000, 0.025000, 0.050000, 0.100000, 0.250000, 0.500000, 1.000000, 2.500000, 5.000000, 10.000000```)
+Boundaries for latency metrics. (Default: ```0.005000, 0.010000, 0.025000, 0.050000, 0.075000, 0.100000, 0.250000, 0.500000, 0.750000, 1.000000, 2.500000, 5.000000, 7.500000, 10.000000```)
 
 `TRAEFIK_METRICS_OTLP_GRPC_ENDPOINT`:  
 Sets the gRPC endpoint (host:port) of the collector. (Default: ```localhost:4317```)

integration/conf_throttling_test.go

@@ -34,11 +34,11 @@ func (s *ThrottlingSuite) TestThrottleConfReload() {
 	s.traefikCmd(withConfigFile("fixtures/throttling/simple.toml"))
 
 	// wait for Traefik
-	err := try.GetRequest("http://127.0.0.1:8080/api/rawdata", 1000*time.Millisecond, try.BodyContains("rest@internal"))
+	err := try.GetRequest("http://127.0.0.1:8080/api/rawdata", 5*time.Second, try.BodyContains("rest@internal"))
 	require.NoError(s.T(), err)
 
 	// Expected a 404 as we did not configure anything.
-	err = try.GetRequest("http://127.0.0.1:8000/", 1000*time.Millisecond, try.StatusCodeIs(http.StatusNotFound))
+	err = try.GetRequest("http://127.0.0.1:8000/", 2*time.Second, try.StatusCodeIs(http.StatusNotFound))
 	require.NoError(s.T(), err)
 
 	config := &dynamic.Configuration{

integration/fixtures/tracing/simple-opentelemetry.toml

@@ -15,6 +15,17 @@
   [entryPoints.web]
     address = ":8000"
 
+# Adding metrics to confirm that there is no wrong interaction with tracing.
+[metrics]
+{{if .IsHTTP}}
+  [metrics.otlp.http]
+    endpoint = "http://{{.IP}}:4318"
+{{else}}
+  [metrics.otlp.grpc]
+    endpoint = "{{.IP}}:4317"
+    insecure = true
+{{end}}
+
 [tracing]
   servicename = "tracing"
   sampleRate = 1.0

integration/tracing_test.go

@@ -302,7 +302,7 @@ func (s *TracingSuite) TestOpentelemetryRetry() {
 	s.traefikCmd(withConfigFile(file))
 
 	// wait for traefik
-	err := try.GetRequest("http://127.0.0.1:8080/api/rawdata", time.Second, try.BodyContains("basic-auth"))
+	err := try.GetRequest("http://127.0.0.1:8080/api/rawdata", 2*time.Second, try.BodyContains("basic-auth"))
 	require.NoError(s.T(), err)
 
 	err = try.GetRequest("http://127.0.0.1:8000/retry", 500*time.Millisecond, try.StatusCodeIs(http.StatusBadGateway))
@@ -425,7 +425,7 @@ func (s *TracingSuite) TestNoInternals() {
 	s.traefikCmd(withConfigFile(file))
 
 	// wait for traefik
-	err := try.GetRequest("http://127.0.0.1:8080/api/rawdata", time.Second, try.BodyContains("basic-auth"))
+	err := try.GetRequest("http://127.0.0.1:8080/api/rawdata", 2*time.Second, try.BodyContains("basic-auth"))
 	require.NoError(s.T(), err)
 
 	err = try.GetRequest("http://127.0.0.1:8000/ratelimit", 500*time.Millisecond, try.StatusCodeIs(http.StatusOK))

pkg/metrics/opentelemetry.go

@@ -30,6 +30,74 @@ var (
 	openTelemetryGaugeCollector *gaugeCollector
 )
 
+// SetMeterProvider sets the meter provider for the tests.
+func SetMeterProvider(meterProvider *sdkmetric.MeterProvider) {
+	openTelemetryMeterProvider = meterProvider
+	otel.SetMeterProvider(meterProvider)
+}
+
+// SemConvMetricsRegistry holds stables semantic conventions metric instruments.
+type SemConvMetricsRegistry struct {
+	// server metrics
+	httpServerRequestDuration metric.Float64Histogram
+	// client metrics
+	httpClientRequestDuration metric.Float64Histogram
+}
+
+// NewSemConvMetricRegistry registers all stables semantic conventions metrics.
+func NewSemConvMetricRegistry(ctx context.Context, config *types.OTLP) (*SemConvMetricsRegistry, error) {
+	if openTelemetryMeterProvider == nil {
+		var err error
+		if openTelemetryMeterProvider, err = newOpenTelemetryMeterProvider(ctx, config); err != nil {
+			log.Ctx(ctx).Err(err).Msg("Unable to create OpenTelemetry meter provider")
+
+			return nil, nil
+		}
+	}
+
+	meter := otel.Meter("github.com/traefik/traefik",
+		metric.WithInstrumentationVersion(version.Version))
+
+	httpServerRequestDuration, err := meter.Float64Histogram("http.server.request.duration",
+		metric.WithDescription("Duration of HTTP server requests."),
+		metric.WithUnit("s"),
+		metric.WithExplicitBucketBoundaries(config.ExplicitBoundaries...))
+	if err != nil {
+		return nil, fmt.Errorf("can't build httpServerRequestDuration histogram: %w", err)
+	}
+
+	httpClientRequestDuration, err := meter.Float64Histogram("http.client.request.duration",
+		metric.WithDescription("Duration of HTTP client requests."),
+		metric.WithUnit("s"),
+		metric.WithExplicitBucketBoundaries(config.ExplicitBoundaries...))
+	if err != nil {
+		return nil, fmt.Errorf("can't build httpClientRequestDuration histogram: %w", err)
+	}
+
+	return &SemConvMetricsRegistry{
+		httpServerRequestDuration: httpServerRequestDuration,
+		httpClientRequestDuration: httpClientRequestDuration,
+	}, nil
+}
+
+// HTTPServerRequestDuration returns the HTTP server request duration histogram.
+func (s *SemConvMetricsRegistry) HTTPServerRequestDuration() metric.Float64Histogram {
+	if s == nil {
+		return nil
+	}
+
+	return s.httpServerRequestDuration
+}
+
+// HTTPClientRequestDuration returns the HTTP client request duration histogram.
+func (s *SemConvMetricsRegistry) HTTPClientRequestDuration() metric.Float64Histogram {
+	if s == nil {
+		return nil
+	}
+
+	return s.httpClientRequestDuration
+}
+
 // RegisterOpenTelemetry registers all OpenTelemetry metrics.
 func RegisterOpenTelemetry(ctx context.Context, config *types.OTLP) Registry {
 	if openTelemetryMeterProvider == nil {

pkg/metrics/opentelemetry_test.go

@@ -361,7 +361,7 @@ func TestOpenTelemetry(t *testing.T) {
 	expectedEntryPoints := []string{
 		`({"name":"traefik_entrypoint_requests_total","description":"How many HTTP requests processed on an entrypoint, partitioned by status code, protocol, and method.","unit":"1","sum":{"dataPoints":\[{"attributes":\[{"key":"code","value":{"stringValue":"200"}},{"key":"entrypoint","value":{"stringValue":"test1"}},{"key":"method","value":{"stringValue":"GET"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1}\],"aggregationTemporality":2,"isMonotonic":true}})`,
 		`({"name":"traefik_entrypoint_requests_tls_total","description":"How many HTTP requests with TLS processed on an entrypoint, partitioned by TLS Version and TLS cipher Used.","unit":"1","sum":{"dataPoints":\[{"attributes":\[{"key":"entrypoint","value":{"stringValue":"test2"}},{"key":"tls_cipher","value":{"stringValue":"bar"}},{"key":"tls_version","value":{"stringValue":"foo"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1}\],"aggregationTemporality":2,"isMonotonic":true}})`,
-		`({"name":"traefik_entrypoint_request_duration_seconds","description":"How long it took to process the request on an entrypoint, partitioned by status code, protocol, and method.","unit":"ms","histogram":{"dataPoints":\[{"attributes":\[{"key":"entrypoint","value":{"stringValue":"test3"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","count":"1","sum":10000,"bucketCounts":\["0","0","0","0","0","0","0","0","0","0","0","1"\],"explicitBounds":\[0.005,0.01,0.025,0.05,0.1,0.25,0.5,1,2.5,5,10\],"min":10000,"max":10000}\],"aggregationTemporality":2}})`,
+		`({"name":"traefik_entrypoint_request_duration_seconds","description":"How long it took to process the request on an entrypoint, partitioned by status code, protocol, and method.","unit":"ms","histogram":{"dataPoints":\[{"attributes":\[{"key":"entrypoint","value":{"stringValue":"test3"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","count":"1","sum":10000,"bucketCounts":\["0","0","0","0","0","0","0","0","0","0","0","0","0","0","1"\],"explicitBounds":\[0.005,0.01,0.025,0.05,0.075,0.1,0.25,0.5,0.75,1,2.5,5,7.5,10\],"min":10000,"max":10000}\],"aggregationTemporality":2}})`,
 		`({"name":"traefik_entrypoint_requests_bytes_total","description":"The total size of requests in bytes handled by an entrypoint, partitioned by status code, protocol, and method.","unit":"1","sum":{"dataPoints":\[{"attributes":\[{"key":"code","value":{"stringValue":"200"}},{"key":"entrypoint","value":{"stringValue":"test1"}},{"key":"method","value":{"stringValue":"GET"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1}\],"aggregationTemporality":2,"isMonotonic":true}})`,
 		`({"name":"traefik_entrypoint_responses_bytes_total","description":"The total size of responses in bytes handled by an entrypoint, partitioned by status code, protocol, and method.","unit":"1","sum":{"dataPoints":\[{"attributes":\[{"key":"code","value":{"stringValue":"200"}},{"key":"entrypoint","value":{"stringValue":"test1"}},{"key":"method","value":{"stringValue":"GET"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1}\],"aggregationTemporality":2,"isMonotonic":true}})`,
 	}
@@ -377,7 +377,7 @@ func TestOpenTelemetry(t *testing.T) {
 	expectedRouters := []string{
 		`({"name":"traefik_router_requests_total","description":"How many HTTP requests are processed on a router, partitioned by service, status code, protocol, and method.","unit":"1","sum":{"dataPoints":\[{"attributes":\[{"key":"code","value":{"stringValue":"(?:200|404)"}},{"key":"method","value":{"stringValue":"GET"}},{"key":"router","value":{"stringValue":"RouterReqsCounter"}},{"key":"service","value":{"stringValue":"test"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1},{"attributes":\[{"key":"code","value":{"stringValue":"(?:200|404)"}},{"key":"method","value":{"stringValue":"GET"}},{"key":"router","value":{"stringValue":"RouterReqsCounter"}},{"key":"service","value":{"stringValue":"test"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1}\],"aggregationTemporality":2,"isMonotonic":true}})`,
 		`({"name":"traefik_router_requests_tls_total","description":"How many HTTP requests with TLS are processed on a router, partitioned by service, TLS Version, and TLS cipher Used.","unit":"1","sum":{"dataPoints":\[{"attributes":\[{"key":"router","value":{"stringValue":"demo"}},{"key":"service","value":{"stringValue":"test"}},{"key":"tls_cipher","value":{"stringValue":"bar"}},{"key":"tls_version","value":{"stringValue":"foo"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1}\],"aggregationTemporality":2,"isMonotonic":true}})`,
-		`({"name":"traefik_router_request_duration_seconds","description":"How long it took to process the request on a router, partitioned by service, status code, protocol, and method.","unit":"ms","histogram":{"dataPoints":\[{"attributes":\[{"key":"code","value":{"stringValue":"200"}},{"key":"router","value":{"stringValue":"demo"}},{"key":"service","value":{"stringValue":"test"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","count":"1","sum":10000,"bucketCounts":\["0","0","0","0","0","0","0","0","0","0","0","1"\],"explicitBounds":\[0.005,0.01,0.025,0.05,0.1,0.25,0.5,1,2.5,5,10\],"min":10000,"max":10000}\],"aggregationTemporality":2}})`,
+		`({"name":"traefik_router_request_duration_seconds","description":"How long it took to process the request on a router, partitioned by service, status code, protocol, and method.","unit":"ms","histogram":{"dataPoints":\[{"attributes":\[{"key":"code","value":{"stringValue":"200"}},{"key":"router","value":{"stringValue":"demo"}},{"key":"service","value":{"stringValue":"test"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","count":"1","sum":10000,"bucketCounts":\["0","0","0","0","0","0","0","0","0","0","0","0","0","0","1"\],"explicitBounds":\[0.005,0.01,0.025,0.05,0.075,0.1,0.25,0.5,0.75,1,2.5,5,7.5,10\],"min":10000,"max":10000}\],"aggregationTemporality":2}})`,
 		`({"name":"traefik_router_requests_bytes_total","description":"The total size of requests in bytes handled by a router, partitioned by status code, protocol, and method.","unit":"1","sum":{"dataPoints":\[{"attributes":\[{"key":"code","value":{"stringValue":"404"}},{"key":"method","value":{"stringValue":"GET"}},{"key":"router","value":{"stringValue":"RouterReqsCounter"}},{"key":"service","value":{"stringValue":"test"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1}\],"aggregationTemporality":2,"isMonotonic":true}})`,
 		`({"name":"traefik_router_responses_bytes_total","description":"The total size of responses in bytes handled by a router, partitioned by status code, protocol, and method.","unit":"1","sum":{"dataPoints":\[{"attributes":\[{"key":"code","value":{"stringValue":"404"}},{"key":"method","value":{"stringValue":"GET"}},{"key":"router","value":{"stringValue":"RouterReqsCounter"}},{"key":"service","value":{"stringValue":"test"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1}\],"aggregationTemporality":2,"isMonotonic":true}})`,
 	}
@@ -394,7 +394,7 @@ func TestOpenTelemetry(t *testing.T) {
 	expectedServices := []string{
 		`({"name":"traefik_service_requests_total","description":"How many HTTP requests processed on a service, partitioned by status code, protocol, and method.","unit":"1","sum":{"dataPoints":\[{"attributes":\[{"key":"code","value":{"stringValue":"(?:200|404)"}},{"key":"method","value":{"stringValue":"GET"}},{"key":"service","value":{"stringValue":"ServiceReqsCounter"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1},{"attributes":\[{"key":"code","value":{"stringValue":"(?:200|404)"}},{"key":"method","value":{"stringValue":"GET"}},{"key":"service","value":{"stringValue":"ServiceReqsCounter"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1}\],"aggregationTemporality":2,"isMonotonic":true}})`,
 		`({"name":"traefik_service_requests_tls_total","description":"How many HTTP requests with TLS processed on a service, partitioned by TLS version and TLS cipher.","unit":"1","sum":{"dataPoints":\[{"attributes":\[{"key":"service","value":{"stringValue":"test"}},{"key":"tls_cipher","value":{"stringValue":"bar"}},{"key":"tls_version","value":{"stringValue":"foo"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1}\],"aggregationTemporality":2,"isMonotonic":true}})`,
-		`({"name":"traefik_service_request_duration_seconds","description":"How long it took to process the request on a service, partitioned by status code, protocol, and method.","unit":"ms","histogram":{"dataPoints":\[{"attributes":\[{"key":"code","value":{"stringValue":"200"}},{"key":"service","value":{"stringValue":"test"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","count":"1","sum":10000,"bucketCounts":\["0","0","0","0","0","0","0","0","0","0","0","1"\],"explicitBounds":\[0.005,0.01,0.025,0.05,0.1,0.25,0.5,1,2.5,5,10\],"min":10000,"max":10000}\],"aggregationTemporality":2}})`,
+		`({"name":"traefik_service_request_duration_seconds","description":"How long it took to process the request on a service, partitioned by status code, protocol, and method.","unit":"ms","histogram":{"dataPoints":\[{"attributes":\[{"key":"code","value":{"stringValue":"200"}},{"key":"service","value":{"stringValue":"test"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","count":"1","sum":10000,"bucketCounts":\["0","0","0","0","0","0","0","0","0","0","0","0","0","0","1"\],"explicitBounds":\[0.005,0.01,0.025,0.05,0.075,0.1,0.25,0.5,0.75,1,2.5,5,7.5,10\],"min":10000,"max":10000}\],"aggregationTemporality":2}})`,
 		`({"name":"traefik_service_server_up","description":"service server is up, described by gauge value of 0 or 1.","unit":"1","gauge":{"dataPoints":\[{"attributes":\[{"key":"service","value":{"stringValue":"test"}},{"key":"url","value":{"stringValue":"http://127.0.0.1"}}\],"timeUnixNano":"[\d]{19}","asDouble":1}\]}})`,
 		`({"name":"traefik_service_requests_bytes_total","description":"The total size of requests in bytes received by a service, partitioned by status code, protocol, and method.","unit":"1","sum":{"dataPoints":\[{"attributes":\[{"key":"code","value":{"stringValue":"404"}},{"key":"method","value":{"stringValue":"GET"}},{"key":"service","value":{"stringValue":"ServiceReqsCounter"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1}\],"aggregationTemporality":2,"isMonotonic":true}})`,
 		`({"name":"traefik_service_responses_bytes_total","description":"The total size of responses in bytes returned by a service, partitioned by status code, protocol, and method.","unit":"1","sum":{"dataPoints":\[{"attributes":\[{"key":"code","value":{"stringValue":"404"}},{"key":"method","value":{"stringValue":"GET"}},{"key":"service","value":{"stringValue":"ServiceReqsCounter"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1}\],"aggregationTemporality":2,"isMonotonic":true}})`,
@@ -426,7 +426,7 @@ func TestOpenTelemetry(t *testing.T) {
 	// and as soon as the EntryPointReqDurationHistogram.Observe is called,
 	// it adds a new dataPoint to the histogram.
 	expectedEntryPointReqDuration := []string{
-		`({"attributes":\[{"key":"entrypoint","value":{"stringValue":"myEntrypoint"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","count":"2","sum":30000,"bucketCounts":\["0","0","0","0","0","0","0","0","0","0","0","2"\],"explicitBounds":\[0.005,0.01,0.025,0.05,0.1,0.25,0.5,1,2.5,5,10\],"min":10000,"max":20000})`,
+		`({"attributes":\[{"key":"entrypoint","value":{"stringValue":"myEntrypoint"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","count":"2","sum":30000,"bucketCounts":\["0","0","0","0","0","0","0","0","0","0","0","0","0","0","2"\],"explicitBounds":\[0.005,0.01,0.025,0.05,0.075,0.1,0.25,0.5,0.75,1,2.5,5,7.5,10\],"min":10000,"max":20000})`,
 	}
 
 	registry.EntryPointReqDurationHistogram().With("entrypoint", "myEntrypoint").Observe(10000)

pkg/middlewares/auth/basic_auth.go

@@ -11,7 +11,7 @@ import (
 	"github.com/traefik/traefik/v3/pkg/config/dynamic"
 	"github.com/traefik/traefik/v3/pkg/middlewares"
 	"github.com/traefik/traefik/v3/pkg/middlewares/accesslog"
-	"github.com/traefik/traefik/v3/pkg/tracing"
+	"github.com/traefik/traefik/v3/pkg/middlewares/observability"
 	"go.opentelemetry.io/otel/trace"
 )
 
@@ -77,7 +77,7 @@ func (b *basicAuth) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
 
 	if !ok {
 		logger.Debug().Msg("Authentication failed")
-		tracing.SetStatusErrorf(req.Context(), "Authentication failed")
+		observability.SetStatusErrorf(req.Context(), "Authentication failed")
 
 		b.auth.RequireAuth(rw, req)
 		return

pkg/middlewares/auth/digest_auth.go

@@ -11,7 +11,7 @@ import (
 	"github.com/traefik/traefik/v3/pkg/config/dynamic"
 	"github.com/traefik/traefik/v3/pkg/middlewares"
 	"github.com/traefik/traefik/v3/pkg/middlewares/accesslog"
-	"github.com/traefik/traefik/v3/pkg/tracing"
+	"github.com/traefik/traefik/v3/pkg/middlewares/observability"
 	"go.opentelemetry.io/otel/trace"
 )
 
@@ -78,13 +78,13 @@ func (d *digestAuth) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
 
 		if authinfo != nil && *authinfo == "stale" {
 			logger.Debug().Msg("Digest authentication failed, possibly because out of order requests")
-			tracing.SetStatusErrorf(req.Context(), "Digest authentication failed, possibly because out of order requests")
+			observability.SetStatusErrorf(req.Context(), "Digest authentication failed, possibly because out of order requests")
 			d.auth.RequireAuthStale(rw, req)
 			return
 		}
 
 		logger.Debug().Msg("Digest authentication failed")
-		tracing.SetStatusErrorf(req.Context(), "Digest authentication failed")
+		observability.SetStatusErrorf(req.Context(), "Digest authentication failed")
 		d.auth.RequireAuth(rw, req)
 		return
 	}

pkg/middlewares/auth/forward.go

@@ -14,6 +14,7 @@ import (
 	"github.com/traefik/traefik/v3/pkg/config/dynamic"
 	"github.com/traefik/traefik/v3/pkg/middlewares"
 	"github.com/traefik/traefik/v3/pkg/middlewares/connectionheader"
+	"github.com/traefik/traefik/v3/pkg/middlewares/observability"
 	"github.com/traefik/traefik/v3/pkg/tracing"
 	"github.com/traefik/traefik/v3/pkg/types"
 	"github.com/vulcand/oxy/v2/forward"
@@ -126,7 +127,7 @@ func (fa *forwardAuth) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
 	if err != nil {
 		logMessage := fmt.Sprintf("Error calling %s. Cause %s", fa.address, err)
 		logger.Debug().Msg(logMessage)
-		tracing.SetStatusErrorf(req.Context(), logMessage)
+		observability.SetStatusErrorf(req.Context(), logMessage)
 		rw.WriteHeader(http.StatusInternalServerError)
 		return
 	}
@@ -150,7 +151,7 @@ func (fa *forwardAuth) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
 	if forwardErr != nil {
 		logMessage := fmt.Sprintf("Error calling %s. Cause: %s", fa.address, forwardErr)
 		logger.Debug().Msg(logMessage)
-		tracing.SetStatusErrorf(forwardReq.Context(), logMessage)
+		observability.SetStatusErrorf(forwardReq.Context(), logMessage)
 
 		rw.WriteHeader(http.StatusInternalServerError)
 		return
@@ -161,7 +162,7 @@ func (fa *forwardAuth) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
 	if readError != nil {
 		logMessage := fmt.Sprintf("Error reading body %s. Cause: %s", fa.address, readError)
 		logger.Debug().Msg(logMessage)
-		tracing.SetStatusErrorf(forwardReq.Context(), logMessage)
+		observability.SetStatusErrorf(forwardReq.Context(), logMessage)
 
 		rw.WriteHeader(http.StatusInternalServerError)
 		return
@@ -188,7 +189,7 @@ func (fa *forwardAuth) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
 			if !errors.Is(err, http.ErrNoLocation) {
 				logMessage := fmt.Sprintf("Error reading response location header %s. Cause: %s", fa.address, err)
 				logger.Debug().Msg(logMessage)
-				tracing.SetStatusErrorf(forwardReq.Context(), logMessage)
+				observability.SetStatusErrorf(forwardReq.Context(), logMessage)
 
 				rw.WriteHeader(http.StatusInternalServerError)
 				return

pkg/middlewares/circuitbreaker/circuit_breaker.go

@@ -10,7 +10,7 @@ import (
 	"github.com/traefik/traefik/v3/pkg/config/dynamic"
 	"github.com/traefik/traefik/v3/pkg/logs"
 	"github.com/traefik/traefik/v3/pkg/middlewares"
-	"github.com/traefik/traefik/v3/pkg/tracing"
+	"github.com/traefik/traefik/v3/pkg/middlewares/observability"
 	"github.com/vulcand/oxy/v2/cbreaker"
 	"go.opentelemetry.io/otel/trace"
 )
@@ -34,7 +34,7 @@ func New(ctx context.Context, next http.Handler, confCircuitBreaker dynamic.Circ
 
 	cbOpts := []cbreaker.Option{
 		cbreaker.Fallback(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
-			tracing.SetStatusErrorf(req.Context(), "blocked by circuit-breaker (%q)", expression)
+			observability.SetStatusErrorf(req.Context(), "blocked by circuit-breaker (%q)", expression)
 			rw.WriteHeader(responseCode)
 
 			if _, err := rw.Write([]byte(http.StatusText(responseCode))); err != nil {

pkg/middlewares/customerrors/custom_errors.go

@@ -12,7 +12,7 @@ import (
 
 	"github.com/traefik/traefik/v3/pkg/config/dynamic"
 	"github.com/traefik/traefik/v3/pkg/middlewares"
-	"github.com/traefik/traefik/v3/pkg/tracing"
+	"github.com/traefik/traefik/v3/pkg/middlewares/observability"
 	"github.com/traefik/traefik/v3/pkg/types"
 	"github.com/vulcand/oxy/v2/utils"
 	"go.opentelemetry.io/otel/trace"
@@ -71,7 +71,7 @@ func (c *customErrors) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
 
 	if c.backendHandler == nil {
 		logger.Error().Msg("Error pages: no backend handler.")
-		tracing.SetStatusErrorf(req.Context(), "Error pages: no backend handler.")
+		observability.SetStatusErrorf(req.Context(), "Error pages: no backend handler.")
 		c.next.ServeHTTP(rw, req)
 		return
 	}
@@ -96,7 +96,7 @@ func (c *customErrors) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
 	pageReq, err := newRequest("http://" + req.Host + query)
 	if err != nil {
 		logger.Error().Err(err).Send()
-		tracing.SetStatusErrorf(req.Context(), err.Error())
+		observability.SetStatusErrorf(req.Context(), err.Error())
 		http.Error(rw, http.StatusText(code), code)
 		return
 	}

pkg/middlewares/ipallowlist/ip_allowlist.go

@@ -10,7 +10,7 @@ import (
 	"github.com/traefik/traefik/v3/pkg/config/dynamic"
 	"github.com/traefik/traefik/v3/pkg/ip"
 	"github.com/traefik/traefik/v3/pkg/middlewares"
-	"github.com/traefik/traefik/v3/pkg/tracing"
+	"github.com/traefik/traefik/v3/pkg/middlewares/observability"
 	"go.opentelemetry.io/otel/trace"
 )
 
@@ -78,7 +78,7 @@ func (al *ipAllowLister) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
 	if err != nil {
 		msg := fmt.Sprintf("Rejecting IP %s: %v", clientIP, err)
 		logger.Debug().Msg(msg)
-		tracing.SetStatusErrorf(req.Context(), msg)
+		observability.SetStatusErrorf(req.Context(), msg)
 		reject(ctx, al.rejectStatusCode, rw)
 		return
 	}

pkg/middlewares/ipwhitelist/ip_whitelist.go

@@ -10,7 +10,7 @@ import (
 	"github.com/traefik/traefik/v3/pkg/config/dynamic"
 	"github.com/traefik/traefik/v3/pkg/ip"
 	"github.com/traefik/traefik/v3/pkg/middlewares"
-	"github.com/traefik/traefik/v3/pkg/tracing"
+	"github.com/traefik/traefik/v3/pkg/middlewares/observability"
 	"go.opentelemetry.io/otel/trace"
 )
 
@@ -68,7 +68,7 @@ func (wl *ipWhiteLister) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
 	if err != nil {
 		msg := fmt.Sprintf("Rejecting IP %s: %v", clientIP, err)
 		logger.Debug().Msg(msg)
-		tracing.SetStatusErrorf(req.Context(), msg)
+		observability.SetStatusErrorf(req.Context(), msg)
 		reject(ctx, rw)
 		return
 	}

pkg/middlewares/metrics/metrics.go

@@ -14,9 +14,9 @@ import (
 	"github.com/traefik/traefik/v3/pkg/metrics"
 	"github.com/traefik/traefik/v3/pkg/middlewares"
 	"github.com/traefik/traefik/v3/pkg/middlewares/capture"
+	"github.com/traefik/traefik/v3/pkg/middlewares/observability"
 	"github.com/traefik/traefik/v3/pkg/middlewares/retry"
 	traefiktls "github.com/traefik/traefik/v3/pkg/tls"
-	"github.com/traefik/traefik/v3/pkg/tracing"
 	"go.opentelemetry.io/otel/trace"
 	"google.golang.org/grpc/codes"
 )
@@ -144,7 +144,7 @@ func (m *metricsMiddleware) ServeHTTP(rw http.ResponseWriter, req *http.Request)
 		}
 		logger := with.Logger()
 		logger.Error().Err(err).Msg("Could not get Capture")
-		tracing.SetStatusErrorf(req.Context(), "Could not get Capture")
+		observability.SetStatusErrorf(req.Context(), "Could not get Capture")
 		return
 	}
 

pkg/middlewares/observability/entrypoint.go

@@ -0,0 +1,98 @@
+package observability
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/containous/alice"
+	"github.com/traefik/traefik/v3/pkg/metrics"
+	"github.com/traefik/traefik/v3/pkg/middlewares"
+	"github.com/traefik/traefik/v3/pkg/tracing"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/metric"
+	semconv "go.opentelemetry.io/otel/semconv/v1.21.0"
+	"go.opentelemetry.io/otel/trace"
+	"go.opentelemetry.io/otel/trace/noop"
+)
+
+const (
+	entryPointTypeName = "TracingEntryPoint"
+)
+
+type entryPointTracing struct {
+	tracer *tracing.Tracer
+
+	entryPoint            string
+	next                  http.Handler
+	semConvMetricRegistry *metrics.SemConvMetricsRegistry
+}
+
+// WrapEntryPointHandler Wraps tracing to alice.Constructor.
+func WrapEntryPointHandler(ctx context.Context, tracer *tracing.Tracer, semConvMetricRegistry *metrics.SemConvMetricsRegistry, entryPointName string) alice.Constructor {
+	return func(next http.Handler) (http.Handler, error) {
+		if tracer == nil {
+			tracer = tracing.NewTracer(noop.Tracer{}, nil, nil)
+		}
+
+		return newEntryPoint(ctx, tracer, semConvMetricRegistry, entryPointName, next), nil
+	}
+}
+
+// newEntryPoint creates a new tracing middleware for incoming requests.
+func newEntryPoint(ctx context.Context, tracer *tracing.Tracer, semConvMetricRegistry *metrics.SemConvMetricsRegistry, entryPointName string, next http.Handler) http.Handler {
+	middlewares.GetLogger(ctx, "tracing", entryPointTypeName).Debug().Msg("Creating middleware")
+
+	if tracer == nil {
+		tracer = tracing.NewTracer(noop.Tracer{}, nil, nil)
+	}
+
+	return &entryPointTracing{
+		entryPoint:            entryPointName,
+		tracer:                tracer,
+		semConvMetricRegistry: semConvMetricRegistry,
+		next:                  next,
+	}
+}
+
+func (e *entryPointTracing) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
+	tracingCtx := tracing.ExtractCarrierIntoContext(req.Context(), req.Header)
+	start := time.Now()
+	tracingCtx, span := e.tracer.Start(tracingCtx, "EntryPoint", trace.WithSpanKind(trace.SpanKindServer), trace.WithTimestamp(start))
+
+	req = req.WithContext(tracingCtx)
+
+	span.SetAttributes(attribute.String("entry_point", e.entryPoint))
+
+	e.tracer.CaptureServerRequest(span, req)
+
+	recorder := newStatusCodeRecorder(rw, http.StatusOK)
+	e.next.ServeHTTP(recorder, req)
+
+	e.tracer.CaptureResponse(span, recorder.Header(), recorder.Status(), trace.SpanKindServer)
+
+	end := time.Now()
+	span.End(trace.WithTimestamp(end))
+
+	if e.semConvMetricRegistry != nil && e.semConvMetricRegistry.HTTPServerRequestDuration() != nil {
+		var attrs []attribute.KeyValue
+
+		if recorder.Status() < 100 || recorder.Status() >= 600 {
+			attrs = append(attrs, attribute.Key("error.type").String(fmt.Sprintf("Invalid HTTP status code ; %d", recorder.Status())))
+		} else if recorder.Status() >= 400 {
+			attrs = append(attrs, attribute.Key("error.type").String(strconv.Itoa(recorder.Status())))
+		}
+
+		attrs = append(attrs, semconv.HTTPRequestMethodKey.String(req.Method))
+		attrs = append(attrs, semconv.HTTPResponseStatusCode(recorder.Status()))
+		attrs = append(attrs, semconv.NetworkProtocolName(strings.ToLower(req.Proto)))
+		attrs = append(attrs, semconv.NetworkProtocolVersion(Proto(req.Proto)))
+		attrs = append(attrs, semconv.ServerAddress(req.Host))
+		attrs = append(attrs, semconv.URLScheme(req.Header.Get("X-Forwarded-Proto")))
+
+		e.semConvMetricRegistry.HTTPServerRequestDuration().Record(req.Context(), end.Sub(start).Seconds(), metric.WithAttributes(attrs...))
+	}
+}

pkg/middlewares/observability/entrypoint_test.go

@@ -0,0 +1,185 @@
+package observability
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	ptypes "github.com/traefik/paerser/types"
+	"github.com/traefik/traefik/v3/pkg/metrics"
+	"github.com/traefik/traefik/v3/pkg/tracing"
+	"github.com/traefik/traefik/v3/pkg/types"
+	"go.opentelemetry.io/otel/attribute"
+	sdkmetric "go.opentelemetry.io/otel/sdk/metric"
+	"go.opentelemetry.io/otel/sdk/metric/metricdata"
+	"go.opentelemetry.io/otel/sdk/metric/metricdata/metricdatatest"
+)
+
+func TestEntryPointMiddleware_tracing(t *testing.T) {
+	type expected struct {
+		name       string
+		attributes []attribute.KeyValue
+	}
+
+	testCases := []struct {
+		desc       string
+		entryPoint string
+		expected   expected
+	}{
+		{
+			desc:       "basic test",
+			entryPoint: "test",
+			expected: expected{
+				name: "EntryPoint",
+				attributes: []attribute.KeyValue{
+					attribute.String("span.kind", "server"),
+					attribute.String("entry_point", "test"),
+					attribute.String("http.request.method", "GET"),
+					attribute.String("network.protocol.version", "1.1"),
+					attribute.Int64("http.request.body.size", int64(0)),
+					attribute.String("url.path", "/search"),
+					attribute.String("url.query", "q=Opentelemetry"),
+					attribute.String("url.scheme", "http"),
+					attribute.String("user_agent.original", "entrypoint-test"),
+					attribute.String("server.address", "www.test.com"),
+					attribute.String("network.peer.address", "10.0.0.1"),
+					attribute.String("network.peer.port", "1234"),
+					attribute.String("client.address", "10.0.0.1"),
+					attribute.Int64("client.port", int64(1234)),
+					attribute.String("client.socket.address", ""),
+					attribute.StringSlice("http.request.header.x-foo", []string{"foo", "bar"}),
+					attribute.Int64("http.response.status_code", int64(404)),
+					attribute.StringSlice("http.response.header.x-bar", []string{"foo", "bar"}),
+				},
+			},
+		},
+	}
+
+	for _, test := range testCases {
+		t.Run(test.desc, func(t *testing.T) {
+			req := httptest.NewRequest(http.MethodGet, "http://www.test.com/search?q=Opentelemetry", nil)
+			rw := httptest.NewRecorder()
+			req.RemoteAddr = "10.0.0.1:1234"
+			req.Header.Set("User-Agent", "entrypoint-test")
+			req.Header.Set("X-Forwarded-Proto", "http")
+			req.Header.Set("X-Foo", "foo")
+			req.Header.Add("X-Foo", "bar")
+
+			next := http.HandlerFunc(func(rw http.ResponseWriter, _ *http.Request) {
+				rw.Header().Set("X-Bar", "foo")
+				rw.Header().Add("X-Bar", "bar")
+				rw.WriteHeader(http.StatusNotFound)
+			})
+
+			tracer := &mockTracer{}
+
+			handler := newEntryPoint(context.Background(), tracing.NewTracer(tracer, []string{"X-Foo"}, []string{"X-Bar"}), nil, test.entryPoint, next)
+			handler.ServeHTTP(rw, req)
+
+			for _, span := range tracer.spans {
+				assert.Equal(t, test.expected.name, span.name)
+				assert.Equal(t, test.expected.attributes, span.attributes)
+			}
+		})
+	}
+}
+
+func TestEntryPointMiddleware_metrics(t *testing.T) {
+	tests := []struct {
+		desc           string
+		statusCode     int
+		wantAttributes attribute.Set
+	}{
+		{
+			desc:       "not found status",
+			statusCode: http.StatusNotFound,
+			wantAttributes: attribute.NewSet(
+				attribute.Key("error.type").String("404"),
+				attribute.Key("http.request.method").String("GET"),
+				attribute.Key("http.response.status_code").Int(404),
+				attribute.Key("network.protocol.name").String("http/1.1"),
+				attribute.Key("network.protocol.version").String("1.1"),
+				attribute.Key("server.address").String("www.test.com"),
+				attribute.Key("url.scheme").String("http"),
+			),
+		},
+		{
+			desc:       "created status",
+			statusCode: http.StatusCreated,
+			wantAttributes: attribute.NewSet(
+				attribute.Key("http.request.method").String("GET"),
+				attribute.Key("http.response.status_code").Int(201),
+				attribute.Key("network.protocol.name").String("http/1.1"),
+				attribute.Key("network.protocol.version").String("1.1"),
+				attribute.Key("server.address").String("www.test.com"),
+				attribute.Key("url.scheme").String("http"),
+			),
+		},
+	}
+
+	for _, test := range tests {
+		test := test
+		t.Run(test.desc, func(t *testing.T) {
+			t.Parallel()
+
+			var cfg types.OTLP
+			(&cfg).SetDefaults()
+			cfg.AddRoutersLabels = true
+			cfg.PushInterval = ptypes.Duration(10 * time.Millisecond)
+			rdr := sdkmetric.NewManualReader()
+
+			meterProvider := sdkmetric.NewMeterProvider(sdkmetric.WithReader(rdr))
+			// force the meter provider with manual reader to collect metrics for the test.
+			metrics.SetMeterProvider(meterProvider)
+
+			semConvMetricRegistry, err := metrics.NewSemConvMetricRegistry(context.Background(), &cfg)
+			require.NoError(t, err)
+			require.NotNil(t, semConvMetricRegistry)
+
+			req := httptest.NewRequest(http.MethodGet, "http://www.test.com/search?q=Opentelemetry", nil)
+			rw := httptest.NewRecorder()
+			req.RemoteAddr = "10.0.0.1:1234"
+			req.Header.Set("User-Agent", "entrypoint-test")
+			req.Header.Set("X-Forwarded-Proto", "http")
+
+			next := http.HandlerFunc(func(rw http.ResponseWriter, _ *http.Request) {
+				rw.WriteHeader(test.statusCode)
+			})
+
+			handler := newEntryPoint(context.Background(), nil, semConvMetricRegistry, "test", next)
+			handler.ServeHTTP(rw, req)
+
+			got := metricdata.ResourceMetrics{}
+			err = rdr.Collect(context.Background(), &got)
+			require.NoError(t, err)
+
+			require.Len(t, got.ScopeMetrics, 1)
+
+			expected := metricdata.Metrics{
+				Name:        "http.server.request.duration",
+				Description: "Duration of HTTP server requests.",
+				Unit:        "s",
+				Data: metricdata.Histogram[float64]{
+					DataPoints: []metricdata.HistogramDataPoint[float64]{
+						{
+							Attributes:   test.wantAttributes,
+							Count:        1,
+							Bounds:       []float64{0.005, 0.01, 0.025, 0.05, 0.075, 0.1, 0.25, 0.5, 0.75, 1, 2.5, 5, 7.5, 10},
+							BucketCounts: []uint64{0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},
+							Min:          metricdata.NewExtrema[float64](1),
+							Max:          metricdata.NewExtrema[float64](1),
+							Sum:          1,
+						},
+					},
+					Temporality: metricdata.CumulativeTemporality,
+				},
+			}
+
+			metricdatatest.AssertEqual[metricdata.Metrics](t, expected, got.ScopeMetrics[0].Metrics[0], metricdatatest.IgnoreTimestamp(), metricdatatest.IgnoreValue())
+		})
+	}
+}

pkg/middlewares/observability/middleware.go

@@ -1,4 +1,4 @@
-package tracing
+package observability
 
 import (
 	"context"

pkg/middlewares/observability/mock_tracing_test.go

@@ -1,4 +1,4 @@
-package tracing
+package observability
 
 import (
 	"context"
@@ -47,7 +47,9 @@ type mockSpan struct {
 
 var _ trace.Span = &mockSpan{}
 
-func (*mockSpan) SpanContext() trace.SpanContext     { return trace.SpanContext{} }
+func (*mockSpan) SpanContext() trace.SpanContext {
+	return trace.NewSpanContext(trace.SpanContextConfig{TraceID: trace.TraceID{1}, SpanID: trace.SpanID{1}})
+}
 func (*mockSpan) IsRecording() bool                  { return false }
 func (s *mockSpan) SetStatus(_ codes.Code, _ string) {}
 func (s *mockSpan) SetAttributes(kv ...attribute.KeyValue) {
@@ -59,4 +61,6 @@ func (s *mockSpan) AddEvent(_ string, _ ...trace.EventOption)   {}
 
 func (s *mockSpan) SetName(name string) { s.name = name }
 
-func (*mockSpan) TracerProvider() trace.TracerProvider { return mockTracerProvider{} }
+func (s *mockSpan) TracerProvider() trace.TracerProvider {
+	return nil
+}

pkg/middlewares/observability/observability.go

@@ -0,0 +1,31 @@
+package observability
+
+import (
+	"context"
+	"fmt"
+
+	"go.opentelemetry.io/otel/codes"
+	"go.opentelemetry.io/otel/trace"
+)
+
+// SetStatusErrorf flags the span as in error and log an event.
+func SetStatusErrorf(ctx context.Context, format string, args ...interface{}) {
+	if span := trace.SpanFromContext(ctx); span != nil {
+		span.SetStatus(codes.Error, fmt.Sprintf(format, args...))
+	}
+}
+
+func Proto(proto string) string {
+	switch proto {
+	case "HTTP/1.0":
+		return "1.0"
+	case "HTTP/1.1":
+		return "1.1"
+	case "HTTP/2":
+		return "2"
+	case "HTTP/3":
+		return "3"
+	default:
+		return proto
+	}
+}

pkg/middlewares/observability/router.go

@@ -1,4 +1,4 @@
-package tracing
+package observability
 
 import (
 	"context"

pkg/middlewares/observability/router_test.go

@@ -1,4 +1,4 @@
-package tracing
+package observability
 
 import (
 	"context"

pkg/middlewares/observability/service.go

@@ -1,4 +1,4 @@
-package tracing
+package observability
 
 import (
 	"context"

pkg/middlewares/observability/service_test.go

@@ -1,4 +1,4 @@
-package tracing
+package observability
 
 import (
 	"context"

pkg/middlewares/observability/status_code.go

@@ -1,4 +1,4 @@
-package tracing
+package observability
 
 import (
 	"bufio"

pkg/middlewares/ratelimiter/rate_limiter.go

@@ -12,7 +12,7 @@ import (
 	"github.com/rs/zerolog/log"
 	"github.com/traefik/traefik/v3/pkg/config/dynamic"
 	"github.com/traefik/traefik/v3/pkg/middlewares"
-	"github.com/traefik/traefik/v3/pkg/tracing"
+	"github.com/traefik/traefik/v3/pkg/middlewares/observability"
 	"github.com/vulcand/oxy/v2/utils"
 	"go.opentelemetry.io/otel/trace"
 	"golang.org/x/time/rate"
@@ -153,14 +153,14 @@ func (rl *rateLimiter) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
 	// as the expiryTime is supposed to reflect the activity (or lack thereof) on that source.
 	if err := rl.buckets.Set(source, bucket, rl.ttl); err != nil {
 		logger.Error().Err(err).Msg("Could not insert/update bucket")
-		tracing.SetStatusErrorf(req.Context(), "Could not insert/update bucket")
+		observability.SetStatusErrorf(req.Context(), "Could not insert/update bucket")
 		http.Error(rw, "could not insert/update bucket", http.StatusInternalServerError)
 		return
 	}
 
 	res := bucket.Reserve()
 	if !res.OK() {
-		tracing.SetStatusErrorf(req.Context(), "No bursty traffic allowed")
+		observability.SetStatusErrorf(req.Context(), "No bursty traffic allowed")
 		http.Error(rw, "No bursty traffic allowed", http.StatusTooManyRequests)
 		return
 	}

pkg/middlewares/replacepath/replace_path.go

@@ -7,7 +7,7 @@ import (
 
 	"github.com/traefik/traefik/v3/pkg/config/dynamic"
 	"github.com/traefik/traefik/v3/pkg/middlewares"
-	"github.com/traefik/traefik/v3/pkg/tracing"
+	"github.com/traefik/traefik/v3/pkg/middlewares/observability"
 	"go.opentelemetry.io/otel/trace"
 )
 
@@ -52,7 +52,7 @@ func (r *replacePath) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
 	req.URL.Path, err = url.PathUnescape(req.URL.RawPath)
 	if err != nil {
 		middlewares.GetLogger(context.Background(), r.name, typeName).Error().Err(err).Send()
-		tracing.SetStatusErrorf(req.Context(), err.Error())
+		observability.SetStatusErrorf(req.Context(), err.Error())
 		http.Error(rw, err.Error(), http.StatusInternalServerError)
 		return
 	}

pkg/middlewares/replacepathregex/replace_path_regex.go

@@ -10,8 +10,8 @@ import (
 
 	"github.com/traefik/traefik/v3/pkg/config/dynamic"
 	"github.com/traefik/traefik/v3/pkg/middlewares"
+	"github.com/traefik/traefik/v3/pkg/middlewares/observability"
 	"github.com/traefik/traefik/v3/pkg/middlewares/replacepath"
-	"github.com/traefik/traefik/v3/pkg/tracing"
 	"go.opentelemetry.io/otel/trace"
 )
 
@@ -63,7 +63,7 @@ func (rp *replacePathRegex) ServeHTTP(rw http.ResponseWriter, req *http.Request)
 		req.URL.Path, err = url.PathUnescape(req.URL.RawPath)
 		if err != nil {
 			middlewares.GetLogger(context.Background(), rp.name, typeName).Error().Err(err).Send()
-			tracing.SetStatusErrorf(req.Context(), err.Error())
+			observability.SetStatusErrorf(req.Context(), err.Error())
 			http.Error(rw, err.Error(), http.StatusInternalServerError)
 			return
 		}

pkg/middlewares/tracing/entrypoint.go

@@ -1,62 +0,0 @@
-package tracing
-
-import (
-	"context"
-	"errors"
-	"net/http"
-
-	"github.com/containous/alice"
-	"github.com/traefik/traefik/v3/pkg/middlewares"
-	"github.com/traefik/traefik/v3/pkg/tracing"
-	"go.opentelemetry.io/otel/attribute"
-	"go.opentelemetry.io/otel/trace"
-)
-
-const (
-	entryPointTypeName = "TracingEntryPoint"
-)
-
-type entryPointTracing struct {
-	tracer     *tracing.Tracer
-	entryPoint string
-	next       http.Handler
-}
-
-// WrapEntryPointHandler Wraps tracing to alice.Constructor.
-func WrapEntryPointHandler(ctx context.Context, tracer *tracing.Tracer, entryPointName string) alice.Constructor {
-	return func(next http.Handler) (http.Handler, error) {
-		if tracer == nil {
-			return nil, errors.New("unexpected nil tracer")
-		}
-
-		return newEntryPoint(ctx, tracer, entryPointName, next), nil
-	}
-}
-
-// newEntryPoint creates a new tracing middleware for incoming requests.
-func newEntryPoint(ctx context.Context, tracer *tracing.Tracer, entryPointName string, next http.Handler) http.Handler {
-	middlewares.GetLogger(ctx, "tracing", entryPointTypeName).Debug().Msg("Creating middleware")
-
-	return &entryPointTracing{
-		entryPoint: entryPointName,
-		tracer:     tracer,
-		next:       next,
-	}
-}
-
-func (e *entryPointTracing) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
-	tracingCtx := tracing.ExtractCarrierIntoContext(req.Context(), req.Header)
-	tracingCtx, span := e.tracer.Start(tracingCtx, "EntryPoint", trace.WithSpanKind(trace.SpanKindServer))
-	defer span.End()
-
-	req = req.WithContext(tracingCtx)
-
-	span.SetAttributes(attribute.String("entry_point", e.entryPoint))
-
-	e.tracer.CaptureServerRequest(span, req)
-
-	recorder := newStatusCodeRecorder(rw, http.StatusOK)
-	e.next.ServeHTTP(recorder, req)
-
-	e.tracer.CaptureResponse(span, recorder.Header(), recorder.Status(), trace.SpanKindServer)
-}

pkg/middlewares/tracing/entrypoint_test.go

@@ -1,80 +0,0 @@
-package tracing
-
-import (
-	"context"
-	"net/http"
-	"net/http/httptest"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/traefik/traefik/v3/pkg/tracing"
-	"go.opentelemetry.io/otel/attribute"
-)
-
-func TestEntryPointMiddleware(t *testing.T) {
-	type expected struct {
-		name       string
-		attributes []attribute.KeyValue
-	}
-
-	testCases := []struct {
-		desc       string
-		entryPoint string
-		expected   expected
-	}{
-		{
-			desc:       "basic test",
-			entryPoint: "test",
-			expected: expected{
-				name: "EntryPoint",
-				attributes: []attribute.KeyValue{
-					attribute.String("span.kind", "server"),
-					attribute.String("entry_point", "test"),
-					attribute.String("http.request.method", "GET"),
-					attribute.String("network.protocol.version", "1.1"),
-					attribute.Int64("http.request.body.size", int64(0)),
-					attribute.String("url.path", "/search"),
-					attribute.String("url.query", "q=Opentelemetry"),
-					attribute.String("url.scheme", "http"),
-					attribute.String("user_agent.original", "entrypoint-test"),
-					attribute.String("server.address", "www.test.com"),
-					attribute.String("network.peer.address", "10.0.0.1"),
-					attribute.String("network.peer.port", "1234"),
-					attribute.String("client.address", "10.0.0.1"),
-					attribute.Int64("client.port", int64(1234)),
-					attribute.String("client.socket.address", ""),
-					attribute.StringSlice("http.request.header.x-foo", []string{"foo", "bar"}),
-					attribute.Int64("http.response.status_code", int64(404)),
-					attribute.StringSlice("http.response.header.x-bar", []string{"foo", "bar"}),
-				},
-			},
-		},
-	}
-
-	for _, test := range testCases {
-		t.Run(test.desc, func(t *testing.T) {
-			req := httptest.NewRequest(http.MethodGet, "http://www.test.com/search?q=Opentelemetry", nil)
-			rw := httptest.NewRecorder()
-			req.RemoteAddr = "10.0.0.1:1234"
-			req.Header.Set("User-Agent", "entrypoint-test")
-			req.Header.Set("X-Forwarded-Proto", "http")
-			req.Header.Set("X-Foo", "foo")
-			req.Header.Add("X-Foo", "bar")
-
-			next := http.HandlerFunc(func(rw http.ResponseWriter, _ *http.Request) {
-				rw.Header().Set("X-Bar", "foo")
-				rw.Header().Add("X-Bar", "bar")
-				rw.WriteHeader(http.StatusNotFound)
-			})
-
-			mockTracer := &mockTracer{}
-			handler := newEntryPoint(context.Background(), tracing.NewTracer(mockTracer, []string{"X-Foo"}, []string{"X-Bar"}), test.entryPoint, next)
-			handler.ServeHTTP(rw, req)
-
-			for _, span := range mockTracer.spans {
-				assert.Equal(t, test.expected.name, span.name)
-				assert.Equal(t, test.expected.attributes, span.attributes)
-			}
-		})
-	}
-}

pkg/server/middleware/middlewares.go

@@ -24,6 +24,7 @@ import (
 	"github.com/traefik/traefik/v3/pkg/middlewares/inflightreq"
 	"github.com/traefik/traefik/v3/pkg/middlewares/ipallowlist"
 	"github.com/traefik/traefik/v3/pkg/middlewares/ipwhitelist"
+	"github.com/traefik/traefik/v3/pkg/middlewares/observability"
 	"github.com/traefik/traefik/v3/pkg/middlewares/passtlsclientcert"
 	"github.com/traefik/traefik/v3/pkg/middlewares/ratelimiter"
 	"github.com/traefik/traefik/v3/pkg/middlewares/redirect"
@@ -32,7 +33,6 @@ import (
 	"github.com/traefik/traefik/v3/pkg/middlewares/retry"
 	"github.com/traefik/traefik/v3/pkg/middlewares/stripprefix"
 	"github.com/traefik/traefik/v3/pkg/middlewares/stripprefixregex"
-	"github.com/traefik/traefik/v3/pkg/middlewares/tracing"
 	"github.com/traefik/traefik/v3/pkg/server/provider"
 )
 
@@ -390,7 +390,7 @@ func (b *Builder) buildConstructor(ctx context.Context, middlewareName string) (
 	// The tracing middleware is a NOOP if tracing is not setup on the middleware chain.
 	// Hence, regarding internal resources' observability deactivation,
 	// this would not enable tracing.
-	return tracing.WrapMiddleware(ctx, middleware), nil
+	return observability.WrapMiddleware(ctx, middleware), nil
 }
 
 func inSlice(element string, stack []string) bool {

pkg/server/middleware/observability.go

@@ -14,7 +14,7 @@ import (
 	"github.com/traefik/traefik/v3/pkg/middlewares/accesslog"
 	"github.com/traefik/traefik/v3/pkg/middlewares/capture"
 	metricsMiddle "github.com/traefik/traefik/v3/pkg/middlewares/metrics"
-	tracingMiddle "github.com/traefik/traefik/v3/pkg/middlewares/tracing"
+	"github.com/traefik/traefik/v3/pkg/middlewares/observability"
 	"github.com/traefik/traefik/v3/pkg/tracing"
 )
 
@@ -23,15 +23,17 @@ type ObservabilityMgr struct {
 	config                 static.Configuration
 	accessLoggerMiddleware *accesslog.Handler
 	metricsRegistry        metrics.Registry
+	semConvMetricRegistry  *metrics.SemConvMetricsRegistry
 	tracer                 *tracing.Tracer
 	tracerCloser           io.Closer
 }
 
 // NewObservabilityMgr creates a new ObservabilityMgr.
-func NewObservabilityMgr(config static.Configuration, metricsRegistry metrics.Registry, accessLoggerMiddleware *accesslog.Handler, tracer *tracing.Tracer, tracerCloser io.Closer) *ObservabilityMgr {
+func NewObservabilityMgr(config static.Configuration, metricsRegistry metrics.Registry, semConvMetricRegistry *metrics.SemConvMetricsRegistry, accessLoggerMiddleware *accesslog.Handler, tracer *tracing.Tracer, tracerCloser io.Closer) *ObservabilityMgr {
 	return &ObservabilityMgr{
 		config:                 config,
 		metricsRegistry:        metricsRegistry,
+		semConvMetricRegistry:  semConvMetricRegistry,
 		accessLoggerMiddleware: accessLoggerMiddleware,
 		tracer:                 tracer,
 		tracerCloser:           tracerCloser,
@@ -39,35 +41,35 @@ func NewObservabilityMgr(config static.Configuration, metricsRegistry metrics.Re
 }
 
 // BuildEPChain an observability middleware chain by entry point.
-func (c *ObservabilityMgr) BuildEPChain(ctx context.Context, entryPointName string, resourceName string) alice.Chain {
+func (o *ObservabilityMgr) BuildEPChain(ctx context.Context, entryPointName string, resourceName string) alice.Chain {
 	chain := alice.New()
 
-	if c == nil {
+	if o == nil {
 		return chain
 	}
 
-	if c.accessLoggerMiddleware != nil || c.metricsRegistry != nil && (c.metricsRegistry.IsEpEnabled() || c.metricsRegistry.IsRouterEnabled() || c.metricsRegistry.IsSvcEnabled()) {
+	if o.accessLoggerMiddleware != nil || o.metricsRegistry != nil && (o.metricsRegistry.IsEpEnabled() || o.metricsRegistry.IsRouterEnabled() || o.metricsRegistry.IsSvcEnabled()) {
-		if c.ShouldAddAccessLogs(resourceName) || c.ShouldAddMetrics(resourceName) {
+		if o.ShouldAddAccessLogs(resourceName) || o.ShouldAddMetrics(resourceName) {
 			chain = chain.Append(capture.Wrap)
 		}
 	}
 
-	if c.accessLoggerMiddleware != nil && c.ShouldAddAccessLogs(resourceName) {
+	if o.accessLoggerMiddleware != nil && o.ShouldAddAccessLogs(resourceName) {
-		chain = chain.Append(accesslog.WrapHandler(c.accessLoggerMiddleware))
+		chain = chain.Append(accesslog.WrapHandler(o.accessLoggerMiddleware))
 		chain = chain.Append(func(next http.Handler) (http.Handler, error) {
 			return accesslog.NewFieldHandler(next, logs.EntryPointName, entryPointName, accesslog.InitServiceFields), nil
 		})
 	}
 
-	if c.tracer != nil && c.ShouldAddTracing(resourceName) {
+	if (o.tracer != nil && o.ShouldAddTracing(resourceName)) || (o.metricsRegistry != nil && o.metricsRegistry.IsEpEnabled() && o.ShouldAddMetrics(resourceName)) {
-		chain = chain.Append(tracingMiddle.WrapEntryPointHandler(ctx, c.tracer, entryPointName))
+		chain = chain.Append(observability.WrapEntryPointHandler(ctx, o.tracer, o.semConvMetricRegistry, entryPointName))
 	}
 
-	if c.metricsRegistry != nil && c.metricsRegistry.IsEpEnabled() && c.ShouldAddMetrics(resourceName) {
+	if o.metricsRegistry != nil && o.metricsRegistry.IsEpEnabled() && o.ShouldAddMetrics(resourceName) {
-		metricsHandler := metricsMiddle.WrapEntryPointHandler(ctx, c.metricsRegistry, entryPointName)
+		metricsHandler := metricsMiddle.WrapEntryPointHandler(ctx, o.metricsRegistry, entryPointName)
 
-		if c.tracer != nil && c.ShouldAddTracing(resourceName) {
+		if o.tracer != nil && o.ShouldAddTracing(resourceName) {
-			chain = chain.Append(tracingMiddle.WrapMiddleware(ctx, metricsHandler))
+			chain = chain.Append(observability.WrapMiddleware(ctx, metricsHandler))
 		} else {
 			chain = chain.Append(metricsHandler)
 		}
@@ -77,64 +79,73 @@ func (c *ObservabilityMgr) BuildEPChain(ctx context.Context, entryPointName stri
 }
 
 // ShouldAddAccessLogs returns whether the access logs should be enabled for the given resource.
-func (c *ObservabilityMgr) ShouldAddAccessLogs(resourceName string) bool {
+func (o *ObservabilityMgr) ShouldAddAccessLogs(resourceName string) bool {
-	if c == nil {
+	if o == nil {
 		return false
 	}
 
-	return c.config.AccessLog != nil && (c.config.AccessLog.AddInternals || !strings.HasSuffix(resourceName, "@internal"))
+	return o.config.AccessLog != nil && (o.config.AccessLog.AddInternals || !strings.HasSuffix(resourceName, "@internal"))
 }
 
 // ShouldAddMetrics returns whether the metrics should be enabled for the given resource.
-func (c *ObservabilityMgr) ShouldAddMetrics(resourceName string) bool {
+func (o *ObservabilityMgr) ShouldAddMetrics(resourceName string) bool {
-	if c == nil {
+	if o == nil {
 		return false
 	}
 
-	return c.config.Metrics != nil && (c.config.Metrics.AddInternals || !strings.HasSuffix(resourceName, "@internal"))
+	return o.config.Metrics != nil && (o.config.Metrics.AddInternals || !strings.HasSuffix(resourceName, "@internal"))
 }
 
 // ShouldAddTracing returns whether the tracing should be enabled for the given resource.
-func (c *ObservabilityMgr) ShouldAddTracing(resourceName string) bool {
+func (o *ObservabilityMgr) ShouldAddTracing(resourceName string) bool {
-	if c == nil {
+	if o == nil {
 		return false
 	}
 
-	return c.config.Tracing != nil && (c.config.Tracing.AddInternals || !strings.HasSuffix(resourceName, "@internal"))
+	return o.config.Tracing != nil && (o.config.Tracing.AddInternals || !strings.HasSuffix(resourceName, "@internal"))
 }
 
 // MetricsRegistry is an accessor to the metrics registry.
-func (c *ObservabilityMgr) MetricsRegistry() metrics.Registry {
+func (o *ObservabilityMgr) MetricsRegistry() metrics.Registry {
-	if c == nil {
+	if o == nil {
 		return nil
 	}
 
-	return c.metricsRegistry
+	return o.metricsRegistry
+}
+
+// SemConvMetricsRegistry is an accessor to the semantic conventions metrics registry.
+func (o *ObservabilityMgr) SemConvMetricsRegistry() *metrics.SemConvMetricsRegistry {
+	if o == nil {
+		return nil
+	}
+
+	return o.semConvMetricRegistry
 }
 
 // Close closes the accessLogger and tracer.
-func (c *ObservabilityMgr) Close() {
+func (o *ObservabilityMgr) Close() {
-	if c == nil {
+	if o == nil {
 		return
 	}
 
-	if c.accessLoggerMiddleware != nil {
+	if o.accessLoggerMiddleware != nil {
-		if err := c.accessLoggerMiddleware.Close(); err != nil {
+		if err := o.accessLoggerMiddleware.Close(); err != nil {
 			log.Error().Err(err).Msg("Could not close the access log file")
 		}
 	}
 
-	if c.tracerCloser != nil {
+	if o.tracerCloser != nil {
-		if err := c.tracerCloser.Close(); err != nil {
+		if err := o.tracerCloser.Close(); err != nil {
 			log.Error().Err(err).Msg("Could not close the tracer")
 		}
 	}
 }
 
-func (c *ObservabilityMgr) RotateAccessLogs() error {
+func (o *ObservabilityMgr) RotateAccessLogs() error {
-	if c.accessLoggerMiddleware == nil {
+	if o.accessLoggerMiddleware == nil {
 		return nil
 	}
 
-	return c.accessLoggerMiddleware.Rotate()
+	return o.accessLoggerMiddleware.Rotate()
 }

pkg/server/router/router.go

@@ -13,8 +13,8 @@ import (
 	"github.com/traefik/traefik/v3/pkg/middlewares/accesslog"
 	"github.com/traefik/traefik/v3/pkg/middlewares/denyrouterrecursion"
 	metricsMiddle "github.com/traefik/traefik/v3/pkg/middlewares/metrics"
+	"github.com/traefik/traefik/v3/pkg/middlewares/observability"
 	"github.com/traefik/traefik/v3/pkg/middlewares/recovery"
-	"github.com/traefik/traefik/v3/pkg/middlewares/tracing"
 	httpmuxer "github.com/traefik/traefik/v3/pkg/muxer/http"
 	"github.com/traefik/traefik/v3/pkg/server/middleware"
 	"github.com/traefik/traefik/v3/pkg/server/provider"
@@ -221,11 +221,11 @@ func (m *Manager) buildHTTPHandler(ctx context.Context, router *runtime.RouterIn
 		return chain.Extend(*mHandler).Then(sHandler)
 	}
 
-	chain = chain.Append(tracing.WrapRouterHandler(ctx, routerName, router.Rule, provider.GetQualifiedName(ctx, router.Service)))
+	chain = chain.Append(observability.WrapRouterHandler(ctx, routerName, router.Rule, provider.GetQualifiedName(ctx, router.Service)))
 
 	if m.observabilityMgr.MetricsRegistry() != nil && m.observabilityMgr.MetricsRegistry().IsRouterEnabled() {
 		metricsHandler := metricsMiddle.WrapRouterHandler(ctx, m.observabilityMgr.MetricsRegistry(), routerName, provider.GetQualifiedName(ctx, router.Service))
-		chain = chain.Append(tracing.WrapMiddleware(ctx, metricsHandler))
+		chain = chain.Append(observability.WrapMiddleware(ctx, metricsHandler))
 	}
 
 	if router.DefaultRule {

pkg/server/routerfactory_test.go

@@ -193,7 +193,7 @@ func TestServerResponseEmptyBackend(t *testing.T) {
 
 			dialerManager := tcp.NewDialerManager(nil)
 			dialerManager.Update(map[string]*dynamic.TCPServersTransport{"default@internal": {}})
-			observabiltyMgr := middleware.NewObservabilityMgr(staticConfig, nil, nil, nil, nil)
+			observabiltyMgr := middleware.NewObservabilityMgr(staticConfig, nil, nil, nil, nil, nil)
 			factory := NewRouterFactory(staticConfig, managerFactory, tlsManager, observabiltyMgr, nil, dialerManager)
 
 			entryPointsHandlers, _ := factory.CreateRouters(runtime.NewConfig(dynamic.Configuration{HTTP: test.config(testServer.URL)}))

pkg/server/service/observability_roundtripper.go

@@ -0,0 +1,105 @@
+package service
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/traefik/traefik/v3/pkg/metrics"
+	"github.com/traefik/traefik/v3/pkg/middlewares/observability"
+	"github.com/traefik/traefik/v3/pkg/tracing"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/metric"
+	semconv "go.opentelemetry.io/otel/semconv/v1.21.0"
+	"go.opentelemetry.io/otel/trace"
+)
+
+type wrapper struct {
+	semConvMetricRegistry *metrics.SemConvMetricsRegistry
+	rt                    http.RoundTripper
+}
+
+func (t *wrapper) RoundTrip(req *http.Request) (*http.Response, error) {
+	start := time.Now()
+	var span trace.Span
+	var tracingCtx context.Context
+	var tracer *tracing.Tracer
+	if tracer = tracing.TracerFromContext(req.Context()); tracer != nil {
+		tracingCtx, span = tracer.Start(req.Context(), "ReverseProxy", trace.WithSpanKind(trace.SpanKindClient))
+		defer span.End()
+
+		req = req.WithContext(tracingCtx)
+
+		tracer.CaptureClientRequest(span, req)
+		tracing.InjectContextIntoCarrier(req)
+	}
+
+	var statusCode int
+	var headers http.Header
+	response, err := t.rt.RoundTrip(req)
+	if err != nil {
+		statusCode = computeStatusCode(err)
+	}
+	if response != nil {
+		statusCode = response.StatusCode
+		headers = response.Header
+	}
+
+	if tracer != nil {
+		tracer.CaptureResponse(span, headers, statusCode, trace.SpanKindClient)
+	}
+
+	end := time.Now()
+
+	// Ending the span as soon as the response is handled because we want to use the same end time for the trace and the metric.
+	// If any errors happen earlier, this span will be close by the defer instruction.
+	if span != nil {
+		span.End(trace.WithTimestamp(end))
+	}
+
+	if t.semConvMetricRegistry != nil && t.semConvMetricRegistry.HTTPClientRequestDuration() != nil {
+		var attrs []attribute.KeyValue
+
+		if statusCode < 100 || statusCode >= 600 {
+			attrs = append(attrs, attribute.Key("error.type").String(fmt.Sprintf("Invalid HTTP status code %d", statusCode)))
+		} else if statusCode >= 400 {
+			attrs = append(attrs, attribute.Key("error.type").String(strconv.Itoa(statusCode)))
+		}
+
+		attrs = append(attrs, semconv.HTTPRequestMethodKey.String(req.Method))
+		attrs = append(attrs, semconv.HTTPResponseStatusCode(statusCode))
+		attrs = append(attrs, semconv.NetworkProtocolName(strings.ToLower(req.Proto)))
+		attrs = append(attrs, semconv.NetworkProtocolVersion(observability.Proto(req.Proto)))
+		attrs = append(attrs, semconv.ServerAddress(req.URL.Host))
+
+		_, port, err := net.SplitHostPort(req.URL.Host)
+		if err != nil {
+			switch req.URL.Scheme {
+			case "http":
+				attrs = append(attrs, semconv.ServerPort(80))
+			case "https":
+				attrs = append(attrs, semconv.ServerPort(443))
+			}
+		} else {
+			intPort, _ := strconv.Atoi(port)
+			attrs = append(attrs, semconv.ServerPort(intPort))
+		}
+
+		attrs = append(attrs, semconv.URLScheme(req.Header.Get("X-Forwarded-Proto")))
+
+		t.semConvMetricRegistry.HTTPClientRequestDuration().Record(req.Context(), end.Sub(start).Seconds(), metric.WithAttributes(attrs...))
+	}
+
+	return response, err
+}
+
+func newObservabilityRoundTripper(semConvMetricRegistry *metrics.SemConvMetricsRegistry, rt http.RoundTripper) http.RoundTripper {
+	return &wrapper{
+		semConvMetricRegistry: semConvMetricRegistry,
+		rt:                    rt,
+	}
+}

pkg/server/service/observability_roundtripper_test.go

@@ -0,0 +1,123 @@
+package service
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+	ptypes "github.com/traefik/paerser/types"
+	"github.com/traefik/traefik/v3/pkg/metrics"
+	"github.com/traefik/traefik/v3/pkg/types"
+	"go.opentelemetry.io/otel/attribute"
+	sdkmetric "go.opentelemetry.io/otel/sdk/metric"
+	"go.opentelemetry.io/otel/sdk/metric/metricdata"
+	"go.opentelemetry.io/otel/sdk/metric/metricdata/metricdatatest"
+)
+
+func TestObservabilityRoundTripper_metrics(t *testing.T) {
+	tests := []struct {
+		desc           string
+		serverURL      string
+		statusCode     int
+		wantAttributes attribute.Set
+	}{
+		{
+			desc:       "not found status",
+			serverURL:  "http://www.test.com",
+			statusCode: http.StatusNotFound,
+			wantAttributes: attribute.NewSet(
+				attribute.Key("error.type").String("404"),
+				attribute.Key("http.request.method").String("GET"),
+				attribute.Key("http.response.status_code").Int(404),
+				attribute.Key("network.protocol.name").String("http/1.1"),
+				attribute.Key("network.protocol.version").String("1.1"),
+				attribute.Key("server.address").String("www.test.com"),
+				attribute.Key("server.port").Int(80),
+				attribute.Key("url.scheme").String("http"),
+			),
+		},
+		{
+			desc:       "created status",
+			serverURL:  "https://www.test.com",
+			statusCode: http.StatusCreated,
+			wantAttributes: attribute.NewSet(
+				attribute.Key("http.request.method").String("GET"),
+				attribute.Key("http.response.status_code").Int(201),
+				attribute.Key("network.protocol.name").String("http/1.1"),
+				attribute.Key("network.protocol.version").String("1.1"),
+				attribute.Key("server.address").String("www.test.com"),
+				attribute.Key("server.port").Int(443),
+				attribute.Key("url.scheme").String("http"),
+			),
+		},
+	}
+
+	for _, test := range tests {
+		test := test
+		t.Run(test.desc, func(t *testing.T) {
+			t.Parallel()
+
+			var cfg types.OTLP
+			(&cfg).SetDefaults()
+			cfg.AddRoutersLabels = true
+			cfg.PushInterval = ptypes.Duration(10 * time.Millisecond)
+			rdr := sdkmetric.NewManualReader()
+
+			meterProvider := sdkmetric.NewMeterProvider(sdkmetric.WithReader(rdr))
+			// force the meter provider with manual reader to collect metrics for the test.
+			metrics.SetMeterProvider(meterProvider)
+
+			semConvMetricRegistry, err := metrics.NewSemConvMetricRegistry(context.Background(), &cfg)
+			require.NoError(t, err)
+			require.NotNil(t, semConvMetricRegistry)
+
+			req := httptest.NewRequest(http.MethodGet, test.serverURL+"/search?q=Opentelemetry", nil)
+			req.RemoteAddr = "10.0.0.1:1234"
+			req.Header.Set("User-Agent", "rt-test")
+			req.Header.Set("X-Forwarded-Proto", "http")
+
+			ort := newObservabilityRoundTripper(semConvMetricRegistry, mockRoundTripper{statusCode: test.statusCode})
+			_, err = ort.RoundTrip(req)
+			require.NoError(t, err)
+
+			got := metricdata.ResourceMetrics{}
+			err = rdr.Collect(context.Background(), &got)
+			require.NoError(t, err)
+
+			require.Len(t, got.ScopeMetrics, 1)
+
+			expected := metricdata.Metrics{
+				Name:        "http.client.request.duration",
+				Description: "Duration of HTTP client requests.",
+				Unit:        "s",
+				Data: metricdata.Histogram[float64]{
+					DataPoints: []metricdata.HistogramDataPoint[float64]{
+						{
+							Attributes:   test.wantAttributes,
+							Count:        1,
+							Bounds:       []float64{0.005, 0.01, 0.025, 0.05, 0.075, 0.1, 0.25, 0.5, 0.75, 1, 2.5, 5, 7.5, 10},
+							BucketCounts: []uint64{0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},
+							Min:          metricdata.NewExtrema[float64](1),
+							Max:          metricdata.NewExtrema[float64](1),
+							Sum:          1,
+						},
+					},
+					Temporality: metricdata.CumulativeTemporality,
+				},
+			}
+
+			metricdatatest.AssertEqual[metricdata.Metrics](t, expected, got.ScopeMetrics[0].Metrics[0], metricdatatest.IgnoreTimestamp(), metricdatatest.IgnoreValue())
+		})
+	}
+}
+
+type mockRoundTripper struct {
+	statusCode int
+}
+
+func (m mockRoundTripper) RoundTrip(request *http.Request) (*http.Response, error) {
+	return &http.Response{StatusCode: m.statusCode}, nil
+}

pkg/server/service/proxy.go

@@ -22,13 +22,9 @@ const StatusClientClosedRequest = 499
 const StatusClientClosedRequestText = "Client Closed Request"
 
 func buildSingleHostProxy(target *url.URL, passHostHeader bool, flushInterval time.Duration, roundTripper http.RoundTripper, bufferPool httputil.BufferPool) http.Handler {
-	// Wrapping the roundTripper with the Tracing roundTripper,
-	// to handle the reverseProxy client span creation.
-	tracingRoundTripper := newTracingRoundTripper(roundTripper)
-
 	return &httputil.ReverseProxy{
 		Director:      directorBuilder(target, passHostHeader),
-		Transport:     tracingRoundTripper,
+		Transport:     roundTripper,
 		FlushInterval: flushInterval,
 		BufferPool:    bufferPool,
 		ErrorHandler:  errorHandler,

pkg/server/service/service.go

@@ -22,7 +22,7 @@ import (
 	"github.com/traefik/traefik/v3/pkg/logs"
 	"github.com/traefik/traefik/v3/pkg/middlewares/accesslog"
 	metricsMiddle "github.com/traefik/traefik/v3/pkg/middlewares/metrics"
-	tracingMiddle "github.com/traefik/traefik/v3/pkg/middlewares/tracing"
+	"github.com/traefik/traefik/v3/pkg/middlewares/observability"
 	"github.com/traefik/traefik/v3/pkg/safe"
 	"github.com/traefik/traefik/v3/pkg/server/cookie"
 	"github.com/traefik/traefik/v3/pkg/server/middleware"
@@ -300,30 +300,38 @@ func (m *Manager) getLoadBalancerServiceHandler(ctx context.Context, serviceName
 		logger.Debug().Str(logs.ServerName, proxyName).Stringer("target", target).
 			Msg("Creating server")
 
+		qualifiedSvcName := provider.GetQualifiedName(ctx, serviceName)
+
+		if m.observabilityMgr.ShouldAddTracing(qualifiedSvcName) || m.observabilityMgr.ShouldAddMetrics(qualifiedSvcName) {
+			// Wrapping the roundTripper with the Tracing roundTripper,
+			// to handle the reverseProxy client span creation.
+			roundTripper = newObservabilityRoundTripper(m.observabilityMgr.SemConvMetricsRegistry(), roundTripper)
+		}
+
 		proxy := buildSingleHostProxy(target, passHostHeader, time.Duration(flushInterval), roundTripper, m.bufferPool)
 
 		// Prevents from enabling observability for internal resources.
 
-		if m.observabilityMgr.ShouldAddAccessLogs(provider.GetQualifiedName(ctx, serviceName)) {
+		if m.observabilityMgr.ShouldAddAccessLogs(qualifiedSvcName) {
 			proxy = accesslog.NewFieldHandler(proxy, accesslog.ServiceURL, target.String(), nil)
 			proxy = accesslog.NewFieldHandler(proxy, accesslog.ServiceAddr, target.Host, nil)
 			proxy = accesslog.NewFieldHandler(proxy, accesslog.ServiceName, serviceName, accesslog.AddServiceFields)
 		}
 
 		if m.observabilityMgr.MetricsRegistry() != nil && m.observabilityMgr.MetricsRegistry().IsSvcEnabled() &&
-			m.observabilityMgr.ShouldAddMetrics(provider.GetQualifiedName(ctx, serviceName)) {
+			m.observabilityMgr.ShouldAddMetrics(qualifiedSvcName) {
 			metricsHandler := metricsMiddle.WrapServiceHandler(ctx, m.observabilityMgr.MetricsRegistry(), serviceName)
 
 			proxy, err = alice.New().
-				Append(tracingMiddle.WrapMiddleware(ctx, metricsHandler)).
+				Append(observability.WrapMiddleware(ctx, metricsHandler)).
 				Then(proxy)
 			if err != nil {
 				return nil, fmt.Errorf("error wrapping metrics handler: %w", err)
 			}
 		}
 
-		if m.observabilityMgr.ShouldAddTracing(provider.GetQualifiedName(ctx, serviceName)) {
+		if m.observabilityMgr.ShouldAddTracing(qualifiedSvcName) {
-			proxy = tracingMiddle.NewService(ctx, serviceName, proxy)
+			proxy = observability.NewService(ctx, serviceName, proxy)
 		}
 
 		lb.Add(proxyName, proxy, server.Weight)

pkg/server/service/tracing_roundtripper.go

@@ -1,44 +0,0 @@
-package service
-
-import (
-	"context"
-	"net/http"
-
-	"github.com/traefik/traefik/v3/pkg/tracing"
-	"go.opentelemetry.io/otel/trace"
-)
-
-type wrapper struct {
-	rt http.RoundTripper
-}
-
-func (t *wrapper) RoundTrip(req *http.Request) (*http.Response, error) {
-	var span trace.Span
-	var tracer *tracing.Tracer
-	if tracer = tracing.TracerFromContext(req.Context()); tracer != nil {
-		var tracingCtx context.Context
-		tracingCtx, span = tracer.Start(req.Context(), "ReverseProxy", trace.WithSpanKind(trace.SpanKindClient))
-		defer span.End()
-
-		req = req.WithContext(tracingCtx)
-
-		tracer.CaptureClientRequest(span, req)
-		tracing.InjectContextIntoCarrier(req)
-	}
-
-	response, err := t.rt.RoundTrip(req)
-	if err != nil {
-		statusCode := computeStatusCode(err)
-		tracer.CaptureResponse(span, nil, statusCode, trace.SpanKindClient)
-
-		return response, err
-	}
-
-	tracer.CaptureResponse(span, response.Header, response.StatusCode, trace.SpanKindClient)
-
-	return response, nil
-}
-
-func newTracingRoundTripper(rt http.RoundTripper) http.RoundTripper {
-	return &wrapper{rt: rt}
-}

pkg/server/service/tracing_roundtripper_test.go

@@ -1,138 +0,0 @@
-package service
-
-import (
-	"context"
-	"net/http"
-	"net/http/httptest"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-	"github.com/traefik/traefik/v3/pkg/tracing"
-	"go.opentelemetry.io/otel/attribute"
-	"go.opentelemetry.io/otel/codes"
-	"go.opentelemetry.io/otel/trace"
-	"go.opentelemetry.io/otel/trace/embedded"
-)
-
-func TestTracingRoundTripper(t *testing.T) {
-	type expected struct {
-		name       string
-		attributes []attribute.KeyValue
-	}
-
-	testCases := []struct {
-		desc     string
-		expected []expected
-	}{
-		{
-			desc: "basic test",
-			expected: []expected{
-				{
-					name: "initial",
-					attributes: []attribute.KeyValue{
-						attribute.String("span.kind", "unspecified"),
-					},
-				},
-				{
-					name: "ReverseProxy",
-					attributes: []attribute.KeyValue{
-						attribute.String("span.kind", "client"),
-						attribute.String("http.request.method", "GET"),
-						attribute.String("network.protocol.version", "1.1"),
-						attribute.String("url.full", "http://www.test.com/search?q=Opentelemetry"),
-						attribute.String("url.scheme", "http"),
-						attribute.String("user_agent.original", "reverse-test"),
-						attribute.String("network.peer.address", ""),
-						attribute.String("server.address", "www.test.com"),
-						attribute.String("network.peer.port", "80"),
-						attribute.Int64("server.port", int64(80)),
-						attribute.StringSlice("http.request.header.x-foo", []string{"foo", "bar"}),
-						attribute.Int64("http.response.status_code", int64(404)),
-						attribute.StringSlice("http.response.header.x-bar", []string{"foo", "bar"}),
-					},
-				},
-			},
-		},
-	}
-
-	for _, test := range testCases {
-		t.Run(test.desc, func(t *testing.T) {
-			req := httptest.NewRequest(http.MethodGet, "http://www.test.com/search?q=Opentelemetry", nil)
-			req.RemoteAddr = "10.0.0.1:1234"
-			req.Header.Set("User-Agent", "reverse-test")
-			req.Header.Set("X-Forwarded-Proto", "http")
-			req.Header.Set("X-Foo", "foo")
-			req.Header.Add("X-Foo", "bar")
-
-			mockTracer := &mockTracer{}
-			tracer := tracing.NewTracer(mockTracer, []string{"X-Foo"}, []string{"X-Bar"})
-			initialCtx, initialSpan := tracer.Start(req.Context(), "initial")
-			defer initialSpan.End()
-			req = req.WithContext(initialCtx)
-
-			tracingRoundTripper := newTracingRoundTripper(roundTripperFn(func(req *http.Request) (*http.Response, error) {
-				return &http.Response{
-					Header: map[string][]string{
-						"X-Bar": {"foo", "bar"},
-					},
-					StatusCode: http.StatusNotFound,
-				}, nil
-			}))
-
-			_, err := tracingRoundTripper.RoundTrip(req)
-			require.NoError(t, err)
-
-			for i, span := range mockTracer.spans {
-				assert.Equal(t, test.expected[i].name, span.name)
-				assert.Equal(t, test.expected[i].attributes, span.attributes)
-			}
-		})
-	}
-}
-
-type mockTracer struct {
-	embedded.Tracer
-
-	spans []*mockSpan
-}
-
-var _ trace.Tracer = &mockTracer{}
-
-func (t *mockTracer) Start(ctx context.Context, name string, opts ...trace.SpanStartOption) (context.Context, trace.Span) {
-	config := trace.NewSpanStartConfig(opts...)
-	span := &mockSpan{}
-	span.SetName(name)
-	span.SetAttributes(attribute.String("span.kind", config.SpanKind().String()))
-	span.SetAttributes(config.Attributes()...)
-	t.spans = append(t.spans, span)
-	return trace.ContextWithSpan(ctx, span), span
-}
-
-// mockSpan is an implementation of Span that preforms no operations.
-type mockSpan struct {
-	embedded.Span
-
-	name       string
-	attributes []attribute.KeyValue
-}
-
-var _ trace.Span = &mockSpan{}
-
-func (*mockSpan) SpanContext() trace.SpanContext {
-	return trace.NewSpanContext(trace.SpanContextConfig{TraceID: trace.TraceID{1}, SpanID: trace.SpanID{1}})
-}
-func (*mockSpan) IsRecording() bool                  { return false }
-func (s *mockSpan) SetStatus(_ codes.Code, _ string) {}
-func (s *mockSpan) SetAttributes(kv ...attribute.KeyValue) {
-	s.attributes = append(s.attributes, kv...)
-}
-func (s *mockSpan) End(...trace.SpanEndOption)                  {}
-func (s *mockSpan) RecordError(_ error, _ ...trace.EventOption) {}
-func (s *mockSpan) AddEvent(_ string, _ ...trace.EventOption)   {}
-
-func (s *mockSpan) SetName(name string) { s.name = name }
-
-func (s *mockSpan) TracerProvider() trace.TracerProvider {
-	return nil
-}

pkg/tracing/opentelemetry/opentelemetry_test.go

@@ -14,7 +14,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"github.com/traefik/traefik/v3/pkg/config/static"
-	tracingMiddle "github.com/traefik/traefik/v3/pkg/middlewares/tracing"
+	"github.com/traefik/traefik/v3/pkg/middlewares/observability"
 	"github.com/traefik/traefik/v3/pkg/tracing"
 	"github.com/traefik/traefik/v3/pkg/tracing/opentelemetry"
 	"github.com/traefik/traefik/v3/pkg/types"
@@ -296,7 +296,7 @@ func TestTracing(t *testing.T) {
 				_ = closer.Close()
 			})
 
-			chain := alice.New(tracingMiddle.WrapEntryPointHandler(context.Background(), newTracing, "test"))
+			chain := alice.New(observability.WrapEntryPointHandler(context.Background(), newTracing, nil, "test"))
 			epHandler, err := chain.Then(service)
 			require.NoError(t, err)
 

pkg/types/metrics.go

@@ -125,7 +125,7 @@ func (o *OTLP) SetDefaults() {
 
 	o.AddEntryPointsLabels = true
 	o.AddServicesLabels = true
-	o.ExplicitBoundaries = []float64{.005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10}
+	o.ExplicitBoundaries = []float64{.005, .01, .025, .05, .075, .1, .25, .5, .75, 1, 2.5, 5, 7.5, 10}
 	o.PushInterval = types.Duration(10 * time.Second)
 }