diff --git a/cmd/traefik/traefik.go b/cmd/traefik/traefik.go index 963790cbc..999e56308 100644 --- a/cmd/traefik/traefik.go +++ b/cmd/traefik/traefik.go @@ -195,10 +195,17 @@ func setupServer(staticConfiguration *static.Configuration) (*server.Server, err // Observability metricRegistries := registerMetricClients(staticConfiguration.Metrics) + var semConvMetricRegistry *metrics.SemConvMetricsRegistry + if staticConfiguration.Metrics != nil && staticConfiguration.Metrics.OTLP != nil { + semConvMetricRegistry, err = metrics.NewSemConvMetricRegistry(ctx, staticConfiguration.Metrics.OTLP) + if err != nil { + return nil, fmt.Errorf("unable to create SemConv metric registry: %w", err) + } + } metricsRegistry := metrics.NewMultiRegistry(metricRegistries) accessLog := setupAccessLog(staticConfiguration.AccessLog) tracer, tracerCloser := setupTracing(staticConfiguration.Tracing) - observabilityMgr := middleware.NewObservabilityMgr(*staticConfiguration, metricsRegistry, accessLog, tracer, tracerCloser) + observabilityMgr := middleware.NewObservabilityMgr(*staticConfiguration, metricsRegistry, semConvMetricRegistry, accessLog, tracer, tracerCloser) // Entrypoints diff --git a/docs/content/observability/metrics/overview.md b/docs/content/observability/metrics/overview.md index f968e77e6..082d4aed0 100644 --- a/docs/content/observability/metrics/overview.md +++ b/docs/content/observability/metrics/overview.md @@ -12,7 +12,8 @@ Traefik supports these metrics backends: - [Prometheus](./prometheus.md) - [StatsD](./statsd.md) -Traefik Proxy hosts an official Grafana dashboard for both [on-premises](https://grafana.com/grafana/dashboards/17346) and [Kubernetes](https://grafana.com/grafana/dashboards/17347) deployments. +Traefik Proxy hosts an official Grafana dashboard for both [on-premises](https://grafana.com/grafana/dashboards/17346) +and [Kubernetes](https://grafana.com/grafana/dashboards/17347) deployments. ## Common Options @@ -29,7 +30,7 @@ metrics: ```toml tab="File (TOML)" [metrics] - addInternals = true +addInternals = true ``` ```bash tab="CLI" @@ -85,10 +86,10 @@ traefik_tls_certs_not_after Here is a comprehensive list of labels that are provided by the global metrics: -| Label | Description | example | -|---------------|----------------------------------------|----------------------| -| `entrypoint` | Entrypoint that handled the connection | "example_entrypoint" | -| `protocol` | Connection protocol | "TCP" | +| Label | Description | example | +|--------------|----------------------------------------|----------------------| +| `entrypoint` | Entrypoint that handled the connection | "example_entrypoint" | +| `protocol` | Connection protocol | "TCP" | ## HTTP Metrics @@ -281,3 +282,49 @@ Here is a comprehensive list of labels that are provided by the metrics: If the HTTP method verb on a request is not one defined in the set of common methods for [`HTTP/1.1`](https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods) or the [`PRI`](https://datatracker.ietf.org/doc/html/rfc7540#section-11.6) verb (for `HTTP/2`), then the value for the method label becomes `EXTENSION_METHOD`. + +## Semantic Conventions for HTTP Metrics + +Traefik Proxy follows [official OTLP semantic conventions v1.23.1](https://github.com/open-telemetry/semantic-conventions/blob/v1.23.1/docs/http/http-metrics.md). + +### HTTP Server + +| Metric | Type | [Labels](#labels) | Description | +|-------------------------------|-----------|------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------| +| http.server.request.duration | Histogram | `error.type`, `http.request.method`, `http.response.status_code`, `network.protocol.name`, `server.address`, `server.port`, `url.scheme` | Duration of HTTP server requests | + +#### Labels + +Here is a comprehensive list of labels that are provided by the metrics: + +| Label | Description | example | +|-----------------------------|--------------------------------------------------------------|---------------| +| `error.type` | Describes a class of error the operation ended with | "500" | +| `http.request.method` | HTTP request method | "GET" | +| `http.response.status_code` | HTTP response status code | "200" | +| `network.protocol.name` | OSI application layer or non-OSI equivalent | "http/1.1" | +| `network.protocol.version` | Version of the protocol specified in `network.protocol.name` | "1.1" | +| `server.address` | Name of the local HTTP server that received the request | "example.com" | +| `server.port` | Port of the local HTTP server that received the request | "80" | +| `url.scheme` | The URI scheme component identifying the used protocol | "http" | + +### HTTP Client + +| Metric | Type | [Labels](#labels) | Description | +|-------------------------------|-----------|------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------| +| http.client.request.duration | Histogram | `error.type`, `http.request.method`, `http.response.status_code`, `network.protocol.name`, `server.address`, `server.port`, `url.scheme` | Duration of HTTP client requests | + +#### Labels + +Here is a comprehensive list of labels that are provided by the metrics: + +| Label | Description | example | +|-----------------------------|--------------------------------------------------------------|---------------| +| `error.type` | Describes a class of error the operation ended with | "500" | +| `http.request.method` | HTTP request method | "GET" | +| `http.response.status_code` | HTTP response status code | "200" | +| `network.protocol.name` | OSI application layer or non-OSI equivalent | "http/1.1" | +| `network.protocol.version` | Version of the protocol specified in `network.protocol.name` | "1.1" | +| `server.address` | Name of the local HTTP server that received the request | "example.com" | +| `server.port` | Port of the local HTTP server that received the request | "80" | +| `url.scheme` | The URI scheme component identifying the used protocol | "http" | diff --git a/docs/content/reference/static-configuration/cli-ref.md b/docs/content/reference/static-configuration/cli-ref.md index e9d97565b..96f2b9adc 100644 --- a/docs/content/reference/static-configuration/cli-ref.md +++ b/docs/content/reference/static-configuration/cli-ref.md @@ -337,7 +337,7 @@ Enable metrics on routers. (Default: ```false```) Enable metrics on services. (Default: ```true```) `--metrics.otlp.explicitboundaries`: -Boundaries for latency metrics. (Default: ```0.005000, 0.010000, 0.025000, 0.050000, 0.100000, 0.250000, 0.500000, 1.000000, 2.500000, 5.000000, 10.000000```) +Boundaries for latency metrics. (Default: ```0.005000, 0.010000, 0.025000, 0.050000, 0.075000, 0.100000, 0.250000, 0.500000, 0.750000, 1.000000, 2.500000, 5.000000, 7.500000, 10.000000```) `--metrics.otlp.grpc.endpoint`: Sets the gRPC endpoint (host:port) of the collector. (Default: ```localhost:4317```) diff --git a/docs/content/reference/static-configuration/env-ref.md b/docs/content/reference/static-configuration/env-ref.md index dc5c94d2e..cc17e380e 100644 --- a/docs/content/reference/static-configuration/env-ref.md +++ b/docs/content/reference/static-configuration/env-ref.md @@ -337,7 +337,7 @@ Enable metrics on routers. (Default: ```false```) Enable metrics on services. (Default: ```true```) `TRAEFIK_METRICS_OTLP_EXPLICITBOUNDARIES`: -Boundaries for latency metrics. (Default: ```0.005000, 0.010000, 0.025000, 0.050000, 0.100000, 0.250000, 0.500000, 1.000000, 2.500000, 5.000000, 10.000000```) +Boundaries for latency metrics. (Default: ```0.005000, 0.010000, 0.025000, 0.050000, 0.075000, 0.100000, 0.250000, 0.500000, 0.750000, 1.000000, 2.500000, 5.000000, 7.500000, 10.000000```) `TRAEFIK_METRICS_OTLP_GRPC_ENDPOINT`: Sets the gRPC endpoint (host:port) of the collector. (Default: ```localhost:4317```) diff --git a/integration/conf_throttling_test.go b/integration/conf_throttling_test.go index 297bb56fd..e3831af00 100644 --- a/integration/conf_throttling_test.go +++ b/integration/conf_throttling_test.go @@ -34,11 +34,11 @@ func (s *ThrottlingSuite) TestThrottleConfReload() { s.traefikCmd(withConfigFile("fixtures/throttling/simple.toml")) // wait for Traefik - err := try.GetRequest("http://127.0.0.1:8080/api/rawdata", 1000*time.Millisecond, try.BodyContains("rest@internal")) + err := try.GetRequest("http://127.0.0.1:8080/api/rawdata", 5*time.Second, try.BodyContains("rest@internal")) require.NoError(s.T(), err) // Expected a 404 as we did not configure anything. - err = try.GetRequest("http://127.0.0.1:8000/", 1000*time.Millisecond, try.StatusCodeIs(http.StatusNotFound)) + err = try.GetRequest("http://127.0.0.1:8000/", 2*time.Second, try.StatusCodeIs(http.StatusNotFound)) require.NoError(s.T(), err) config := &dynamic.Configuration{ diff --git a/integration/fixtures/tracing/simple-opentelemetry.toml b/integration/fixtures/tracing/simple-opentelemetry.toml index 0599bb18d..77cb436bb 100644 --- a/integration/fixtures/tracing/simple-opentelemetry.toml +++ b/integration/fixtures/tracing/simple-opentelemetry.toml @@ -15,6 +15,17 @@ [entryPoints.web] address = ":8000" +# Adding metrics to confirm that there is no wrong interaction with tracing. +[metrics] +{{if .IsHTTP}} + [metrics.otlp.http] + endpoint = "http://{{.IP}}:4318" +{{else}} + [metrics.otlp.grpc] + endpoint = "{{.IP}}:4317" + insecure = true +{{end}} + [tracing] servicename = "tracing" sampleRate = 1.0 diff --git a/integration/tracing_test.go b/integration/tracing_test.go index ef92015bd..aa5500108 100644 --- a/integration/tracing_test.go +++ b/integration/tracing_test.go @@ -302,7 +302,7 @@ func (s *TracingSuite) TestOpentelemetryRetry() { s.traefikCmd(withConfigFile(file)) // wait for traefik - err := try.GetRequest("http://127.0.0.1:8080/api/rawdata", time.Second, try.BodyContains("basic-auth")) + err := try.GetRequest("http://127.0.0.1:8080/api/rawdata", 2*time.Second, try.BodyContains("basic-auth")) require.NoError(s.T(), err) err = try.GetRequest("http://127.0.0.1:8000/retry", 500*time.Millisecond, try.StatusCodeIs(http.StatusBadGateway)) @@ -425,7 +425,7 @@ func (s *TracingSuite) TestNoInternals() { s.traefikCmd(withConfigFile(file)) // wait for traefik - err := try.GetRequest("http://127.0.0.1:8080/api/rawdata", time.Second, try.BodyContains("basic-auth")) + err := try.GetRequest("http://127.0.0.1:8080/api/rawdata", 2*time.Second, try.BodyContains("basic-auth")) require.NoError(s.T(), err) err = try.GetRequest("http://127.0.0.1:8000/ratelimit", 500*time.Millisecond, try.StatusCodeIs(http.StatusOK)) diff --git a/pkg/metrics/opentelemetry.go b/pkg/metrics/opentelemetry.go index cf9920a25..23cbf8eba 100644 --- a/pkg/metrics/opentelemetry.go +++ b/pkg/metrics/opentelemetry.go @@ -30,6 +30,74 @@ var ( openTelemetryGaugeCollector *gaugeCollector ) +// SetMeterProvider sets the meter provider for the tests. +func SetMeterProvider(meterProvider *sdkmetric.MeterProvider) { + openTelemetryMeterProvider = meterProvider + otel.SetMeterProvider(meterProvider) +} + +// SemConvMetricsRegistry holds stables semantic conventions metric instruments. +type SemConvMetricsRegistry struct { + // server metrics + httpServerRequestDuration metric.Float64Histogram + // client metrics + httpClientRequestDuration metric.Float64Histogram +} + +// NewSemConvMetricRegistry registers all stables semantic conventions metrics. +func NewSemConvMetricRegistry(ctx context.Context, config *types.OTLP) (*SemConvMetricsRegistry, error) { + if openTelemetryMeterProvider == nil { + var err error + if openTelemetryMeterProvider, err = newOpenTelemetryMeterProvider(ctx, config); err != nil { + log.Ctx(ctx).Err(err).Msg("Unable to create OpenTelemetry meter provider") + + return nil, nil + } + } + + meter := otel.Meter("github.com/traefik/traefik", + metric.WithInstrumentationVersion(version.Version)) + + httpServerRequestDuration, err := meter.Float64Histogram("http.server.request.duration", + metric.WithDescription("Duration of HTTP server requests."), + metric.WithUnit("s"), + metric.WithExplicitBucketBoundaries(config.ExplicitBoundaries...)) + if err != nil { + return nil, fmt.Errorf("can't build httpServerRequestDuration histogram: %w", err) + } + + httpClientRequestDuration, err := meter.Float64Histogram("http.client.request.duration", + metric.WithDescription("Duration of HTTP client requests."), + metric.WithUnit("s"), + metric.WithExplicitBucketBoundaries(config.ExplicitBoundaries...)) + if err != nil { + return nil, fmt.Errorf("can't build httpClientRequestDuration histogram: %w", err) + } + + return &SemConvMetricsRegistry{ + httpServerRequestDuration: httpServerRequestDuration, + httpClientRequestDuration: httpClientRequestDuration, + }, nil +} + +// HTTPServerRequestDuration returns the HTTP server request duration histogram. +func (s *SemConvMetricsRegistry) HTTPServerRequestDuration() metric.Float64Histogram { + if s == nil { + return nil + } + + return s.httpServerRequestDuration +} + +// HTTPClientRequestDuration returns the HTTP client request duration histogram. +func (s *SemConvMetricsRegistry) HTTPClientRequestDuration() metric.Float64Histogram { + if s == nil { + return nil + } + + return s.httpClientRequestDuration +} + // RegisterOpenTelemetry registers all OpenTelemetry metrics. func RegisterOpenTelemetry(ctx context.Context, config *types.OTLP) Registry { if openTelemetryMeterProvider == nil { diff --git a/pkg/metrics/opentelemetry_test.go b/pkg/metrics/opentelemetry_test.go index 2e3ea328d..9d8da3cd6 100644 --- a/pkg/metrics/opentelemetry_test.go +++ b/pkg/metrics/opentelemetry_test.go @@ -361,7 +361,7 @@ func TestOpenTelemetry(t *testing.T) { expectedEntryPoints := []string{ `({"name":"traefik_entrypoint_requests_total","description":"How many HTTP requests processed on an entrypoint, partitioned by status code, protocol, and method.","unit":"1","sum":{"dataPoints":\[{"attributes":\[{"key":"code","value":{"stringValue":"200"}},{"key":"entrypoint","value":{"stringValue":"test1"}},{"key":"method","value":{"stringValue":"GET"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1}\],"aggregationTemporality":2,"isMonotonic":true}})`, `({"name":"traefik_entrypoint_requests_tls_total","description":"How many HTTP requests with TLS processed on an entrypoint, partitioned by TLS Version and TLS cipher Used.","unit":"1","sum":{"dataPoints":\[{"attributes":\[{"key":"entrypoint","value":{"stringValue":"test2"}},{"key":"tls_cipher","value":{"stringValue":"bar"}},{"key":"tls_version","value":{"stringValue":"foo"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1}\],"aggregationTemporality":2,"isMonotonic":true}})`, - `({"name":"traefik_entrypoint_request_duration_seconds","description":"How long it took to process the request on an entrypoint, partitioned by status code, protocol, and method.","unit":"ms","histogram":{"dataPoints":\[{"attributes":\[{"key":"entrypoint","value":{"stringValue":"test3"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","count":"1","sum":10000,"bucketCounts":\["0","0","0","0","0","0","0","0","0","0","0","1"\],"explicitBounds":\[0.005,0.01,0.025,0.05,0.1,0.25,0.5,1,2.5,5,10\],"min":10000,"max":10000}\],"aggregationTemporality":2}})`, + `({"name":"traefik_entrypoint_request_duration_seconds","description":"How long it took to process the request on an entrypoint, partitioned by status code, protocol, and method.","unit":"ms","histogram":{"dataPoints":\[{"attributes":\[{"key":"entrypoint","value":{"stringValue":"test3"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","count":"1","sum":10000,"bucketCounts":\["0","0","0","0","0","0","0","0","0","0","0","0","0","0","1"\],"explicitBounds":\[0.005,0.01,0.025,0.05,0.075,0.1,0.25,0.5,0.75,1,2.5,5,7.5,10\],"min":10000,"max":10000}\],"aggregationTemporality":2}})`, `({"name":"traefik_entrypoint_requests_bytes_total","description":"The total size of requests in bytes handled by an entrypoint, partitioned by status code, protocol, and method.","unit":"1","sum":{"dataPoints":\[{"attributes":\[{"key":"code","value":{"stringValue":"200"}},{"key":"entrypoint","value":{"stringValue":"test1"}},{"key":"method","value":{"stringValue":"GET"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1}\],"aggregationTemporality":2,"isMonotonic":true}})`, `({"name":"traefik_entrypoint_responses_bytes_total","description":"The total size of responses in bytes handled by an entrypoint, partitioned by status code, protocol, and method.","unit":"1","sum":{"dataPoints":\[{"attributes":\[{"key":"code","value":{"stringValue":"200"}},{"key":"entrypoint","value":{"stringValue":"test1"}},{"key":"method","value":{"stringValue":"GET"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1}\],"aggregationTemporality":2,"isMonotonic":true}})`, } @@ -377,7 +377,7 @@ func TestOpenTelemetry(t *testing.T) { expectedRouters := []string{ `({"name":"traefik_router_requests_total","description":"How many HTTP requests are processed on a router, partitioned by service, status code, protocol, and method.","unit":"1","sum":{"dataPoints":\[{"attributes":\[{"key":"code","value":{"stringValue":"(?:200|404)"}},{"key":"method","value":{"stringValue":"GET"}},{"key":"router","value":{"stringValue":"RouterReqsCounter"}},{"key":"service","value":{"stringValue":"test"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1},{"attributes":\[{"key":"code","value":{"stringValue":"(?:200|404)"}},{"key":"method","value":{"stringValue":"GET"}},{"key":"router","value":{"stringValue":"RouterReqsCounter"}},{"key":"service","value":{"stringValue":"test"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1}\],"aggregationTemporality":2,"isMonotonic":true}})`, `({"name":"traefik_router_requests_tls_total","description":"How many HTTP requests with TLS are processed on a router, partitioned by service, TLS Version, and TLS cipher Used.","unit":"1","sum":{"dataPoints":\[{"attributes":\[{"key":"router","value":{"stringValue":"demo"}},{"key":"service","value":{"stringValue":"test"}},{"key":"tls_cipher","value":{"stringValue":"bar"}},{"key":"tls_version","value":{"stringValue":"foo"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1}\],"aggregationTemporality":2,"isMonotonic":true}})`, - `({"name":"traefik_router_request_duration_seconds","description":"How long it took to process the request on a router, partitioned by service, status code, protocol, and method.","unit":"ms","histogram":{"dataPoints":\[{"attributes":\[{"key":"code","value":{"stringValue":"200"}},{"key":"router","value":{"stringValue":"demo"}},{"key":"service","value":{"stringValue":"test"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","count":"1","sum":10000,"bucketCounts":\["0","0","0","0","0","0","0","0","0","0","0","1"\],"explicitBounds":\[0.005,0.01,0.025,0.05,0.1,0.25,0.5,1,2.5,5,10\],"min":10000,"max":10000}\],"aggregationTemporality":2}})`, + `({"name":"traefik_router_request_duration_seconds","description":"How long it took to process the request on a router, partitioned by service, status code, protocol, and method.","unit":"ms","histogram":{"dataPoints":\[{"attributes":\[{"key":"code","value":{"stringValue":"200"}},{"key":"router","value":{"stringValue":"demo"}},{"key":"service","value":{"stringValue":"test"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","count":"1","sum":10000,"bucketCounts":\["0","0","0","0","0","0","0","0","0","0","0","0","0","0","1"\],"explicitBounds":\[0.005,0.01,0.025,0.05,0.075,0.1,0.25,0.5,0.75,1,2.5,5,7.5,10\],"min":10000,"max":10000}\],"aggregationTemporality":2}})`, `({"name":"traefik_router_requests_bytes_total","description":"The total size of requests in bytes handled by a router, partitioned by status code, protocol, and method.","unit":"1","sum":{"dataPoints":\[{"attributes":\[{"key":"code","value":{"stringValue":"404"}},{"key":"method","value":{"stringValue":"GET"}},{"key":"router","value":{"stringValue":"RouterReqsCounter"}},{"key":"service","value":{"stringValue":"test"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1}\],"aggregationTemporality":2,"isMonotonic":true}})`, `({"name":"traefik_router_responses_bytes_total","description":"The total size of responses in bytes handled by a router, partitioned by status code, protocol, and method.","unit":"1","sum":{"dataPoints":\[{"attributes":\[{"key":"code","value":{"stringValue":"404"}},{"key":"method","value":{"stringValue":"GET"}},{"key":"router","value":{"stringValue":"RouterReqsCounter"}},{"key":"service","value":{"stringValue":"test"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1}\],"aggregationTemporality":2,"isMonotonic":true}})`, } @@ -394,7 +394,7 @@ func TestOpenTelemetry(t *testing.T) { expectedServices := []string{ `({"name":"traefik_service_requests_total","description":"How many HTTP requests processed on a service, partitioned by status code, protocol, and method.","unit":"1","sum":{"dataPoints":\[{"attributes":\[{"key":"code","value":{"stringValue":"(?:200|404)"}},{"key":"method","value":{"stringValue":"GET"}},{"key":"service","value":{"stringValue":"ServiceReqsCounter"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1},{"attributes":\[{"key":"code","value":{"stringValue":"(?:200|404)"}},{"key":"method","value":{"stringValue":"GET"}},{"key":"service","value":{"stringValue":"ServiceReqsCounter"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1}\],"aggregationTemporality":2,"isMonotonic":true}})`, `({"name":"traefik_service_requests_tls_total","description":"How many HTTP requests with TLS processed on a service, partitioned by TLS version and TLS cipher.","unit":"1","sum":{"dataPoints":\[{"attributes":\[{"key":"service","value":{"stringValue":"test"}},{"key":"tls_cipher","value":{"stringValue":"bar"}},{"key":"tls_version","value":{"stringValue":"foo"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1}\],"aggregationTemporality":2,"isMonotonic":true}})`, - `({"name":"traefik_service_request_duration_seconds","description":"How long it took to process the request on a service, partitioned by status code, protocol, and method.","unit":"ms","histogram":{"dataPoints":\[{"attributes":\[{"key":"code","value":{"stringValue":"200"}},{"key":"service","value":{"stringValue":"test"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","count":"1","sum":10000,"bucketCounts":\["0","0","0","0","0","0","0","0","0","0","0","1"\],"explicitBounds":\[0.005,0.01,0.025,0.05,0.1,0.25,0.5,1,2.5,5,10\],"min":10000,"max":10000}\],"aggregationTemporality":2}})`, + `({"name":"traefik_service_request_duration_seconds","description":"How long it took to process the request on a service, partitioned by status code, protocol, and method.","unit":"ms","histogram":{"dataPoints":\[{"attributes":\[{"key":"code","value":{"stringValue":"200"}},{"key":"service","value":{"stringValue":"test"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","count":"1","sum":10000,"bucketCounts":\["0","0","0","0","0","0","0","0","0","0","0","0","0","0","1"\],"explicitBounds":\[0.005,0.01,0.025,0.05,0.075,0.1,0.25,0.5,0.75,1,2.5,5,7.5,10\],"min":10000,"max":10000}\],"aggregationTemporality":2}})`, `({"name":"traefik_service_server_up","description":"service server is up, described by gauge value of 0 or 1.","unit":"1","gauge":{"dataPoints":\[{"attributes":\[{"key":"service","value":{"stringValue":"test"}},{"key":"url","value":{"stringValue":"http://127.0.0.1"}}\],"timeUnixNano":"[\d]{19}","asDouble":1}\]}})`, `({"name":"traefik_service_requests_bytes_total","description":"The total size of requests in bytes received by a service, partitioned by status code, protocol, and method.","unit":"1","sum":{"dataPoints":\[{"attributes":\[{"key":"code","value":{"stringValue":"404"}},{"key":"method","value":{"stringValue":"GET"}},{"key":"service","value":{"stringValue":"ServiceReqsCounter"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1}\],"aggregationTemporality":2,"isMonotonic":true}})`, `({"name":"traefik_service_responses_bytes_total","description":"The total size of responses in bytes returned by a service, partitioned by status code, protocol, and method.","unit":"1","sum":{"dataPoints":\[{"attributes":\[{"key":"code","value":{"stringValue":"404"}},{"key":"method","value":{"stringValue":"GET"}},{"key":"service","value":{"stringValue":"ServiceReqsCounter"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1}\],"aggregationTemporality":2,"isMonotonic":true}})`, @@ -426,7 +426,7 @@ func TestOpenTelemetry(t *testing.T) { // and as soon as the EntryPointReqDurationHistogram.Observe is called, // it adds a new dataPoint to the histogram. expectedEntryPointReqDuration := []string{ - `({"attributes":\[{"key":"entrypoint","value":{"stringValue":"myEntrypoint"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","count":"2","sum":30000,"bucketCounts":\["0","0","0","0","0","0","0","0","0","0","0","2"\],"explicitBounds":\[0.005,0.01,0.025,0.05,0.1,0.25,0.5,1,2.5,5,10\],"min":10000,"max":20000})`, + `({"attributes":\[{"key":"entrypoint","value":{"stringValue":"myEntrypoint"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","count":"2","sum":30000,"bucketCounts":\["0","0","0","0","0","0","0","0","0","0","0","0","0","0","2"\],"explicitBounds":\[0.005,0.01,0.025,0.05,0.075,0.1,0.25,0.5,0.75,1,2.5,5,7.5,10\],"min":10000,"max":20000})`, } registry.EntryPointReqDurationHistogram().With("entrypoint", "myEntrypoint").Observe(10000) diff --git a/pkg/middlewares/auth/basic_auth.go b/pkg/middlewares/auth/basic_auth.go index 5a3caa26d..0ec8f1e28 100644 --- a/pkg/middlewares/auth/basic_auth.go +++ b/pkg/middlewares/auth/basic_auth.go @@ -11,7 +11,7 @@ import ( "github.com/traefik/traefik/v3/pkg/config/dynamic" "github.com/traefik/traefik/v3/pkg/middlewares" "github.com/traefik/traefik/v3/pkg/middlewares/accesslog" - "github.com/traefik/traefik/v3/pkg/tracing" + "github.com/traefik/traefik/v3/pkg/middlewares/observability" "go.opentelemetry.io/otel/trace" ) @@ -77,7 +77,7 @@ func (b *basicAuth) ServeHTTP(rw http.ResponseWriter, req *http.Request) { if !ok { logger.Debug().Msg("Authentication failed") - tracing.SetStatusErrorf(req.Context(), "Authentication failed") + observability.SetStatusErrorf(req.Context(), "Authentication failed") b.auth.RequireAuth(rw, req) return diff --git a/pkg/middlewares/auth/digest_auth.go b/pkg/middlewares/auth/digest_auth.go index e02bc5883..25c22865f 100644 --- a/pkg/middlewares/auth/digest_auth.go +++ b/pkg/middlewares/auth/digest_auth.go @@ -11,7 +11,7 @@ import ( "github.com/traefik/traefik/v3/pkg/config/dynamic" "github.com/traefik/traefik/v3/pkg/middlewares" "github.com/traefik/traefik/v3/pkg/middlewares/accesslog" - "github.com/traefik/traefik/v3/pkg/tracing" + "github.com/traefik/traefik/v3/pkg/middlewares/observability" "go.opentelemetry.io/otel/trace" ) @@ -78,13 +78,13 @@ func (d *digestAuth) ServeHTTP(rw http.ResponseWriter, req *http.Request) { if authinfo != nil && *authinfo == "stale" { logger.Debug().Msg("Digest authentication failed, possibly because out of order requests") - tracing.SetStatusErrorf(req.Context(), "Digest authentication failed, possibly because out of order requests") + observability.SetStatusErrorf(req.Context(), "Digest authentication failed, possibly because out of order requests") d.auth.RequireAuthStale(rw, req) return } logger.Debug().Msg("Digest authentication failed") - tracing.SetStatusErrorf(req.Context(), "Digest authentication failed") + observability.SetStatusErrorf(req.Context(), "Digest authentication failed") d.auth.RequireAuth(rw, req) return } diff --git a/pkg/middlewares/auth/forward.go b/pkg/middlewares/auth/forward.go index 4f9e207e5..26053a50e 100644 --- a/pkg/middlewares/auth/forward.go +++ b/pkg/middlewares/auth/forward.go @@ -14,6 +14,7 @@ import ( "github.com/traefik/traefik/v3/pkg/config/dynamic" "github.com/traefik/traefik/v3/pkg/middlewares" "github.com/traefik/traefik/v3/pkg/middlewares/connectionheader" + "github.com/traefik/traefik/v3/pkg/middlewares/observability" "github.com/traefik/traefik/v3/pkg/tracing" "github.com/traefik/traefik/v3/pkg/types" "github.com/vulcand/oxy/v2/forward" @@ -126,7 +127,7 @@ func (fa *forwardAuth) ServeHTTP(rw http.ResponseWriter, req *http.Request) { if err != nil { logMessage := fmt.Sprintf("Error calling %s. Cause %s", fa.address, err) logger.Debug().Msg(logMessage) - tracing.SetStatusErrorf(req.Context(), logMessage) + observability.SetStatusErrorf(req.Context(), logMessage) rw.WriteHeader(http.StatusInternalServerError) return } @@ -150,7 +151,7 @@ func (fa *forwardAuth) ServeHTTP(rw http.ResponseWriter, req *http.Request) { if forwardErr != nil { logMessage := fmt.Sprintf("Error calling %s. Cause: %s", fa.address, forwardErr) logger.Debug().Msg(logMessage) - tracing.SetStatusErrorf(forwardReq.Context(), logMessage) + observability.SetStatusErrorf(forwardReq.Context(), logMessage) rw.WriteHeader(http.StatusInternalServerError) return @@ -161,7 +162,7 @@ func (fa *forwardAuth) ServeHTTP(rw http.ResponseWriter, req *http.Request) { if readError != nil { logMessage := fmt.Sprintf("Error reading body %s. Cause: %s", fa.address, readError) logger.Debug().Msg(logMessage) - tracing.SetStatusErrorf(forwardReq.Context(), logMessage) + observability.SetStatusErrorf(forwardReq.Context(), logMessage) rw.WriteHeader(http.StatusInternalServerError) return @@ -188,7 +189,7 @@ func (fa *forwardAuth) ServeHTTP(rw http.ResponseWriter, req *http.Request) { if !errors.Is(err, http.ErrNoLocation) { logMessage := fmt.Sprintf("Error reading response location header %s. Cause: %s", fa.address, err) logger.Debug().Msg(logMessage) - tracing.SetStatusErrorf(forwardReq.Context(), logMessage) + observability.SetStatusErrorf(forwardReq.Context(), logMessage) rw.WriteHeader(http.StatusInternalServerError) return diff --git a/pkg/middlewares/circuitbreaker/circuit_breaker.go b/pkg/middlewares/circuitbreaker/circuit_breaker.go index ba377ea3d..ceaa93de4 100644 --- a/pkg/middlewares/circuitbreaker/circuit_breaker.go +++ b/pkg/middlewares/circuitbreaker/circuit_breaker.go @@ -10,7 +10,7 @@ import ( "github.com/traefik/traefik/v3/pkg/config/dynamic" "github.com/traefik/traefik/v3/pkg/logs" "github.com/traefik/traefik/v3/pkg/middlewares" - "github.com/traefik/traefik/v3/pkg/tracing" + "github.com/traefik/traefik/v3/pkg/middlewares/observability" "github.com/vulcand/oxy/v2/cbreaker" "go.opentelemetry.io/otel/trace" ) @@ -34,7 +34,7 @@ func New(ctx context.Context, next http.Handler, confCircuitBreaker dynamic.Circ cbOpts := []cbreaker.Option{ cbreaker.Fallback(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { - tracing.SetStatusErrorf(req.Context(), "blocked by circuit-breaker (%q)", expression) + observability.SetStatusErrorf(req.Context(), "blocked by circuit-breaker (%q)", expression) rw.WriteHeader(responseCode) if _, err := rw.Write([]byte(http.StatusText(responseCode))); err != nil { diff --git a/pkg/middlewares/customerrors/custom_errors.go b/pkg/middlewares/customerrors/custom_errors.go index dd1f77e33..dbd66718a 100644 --- a/pkg/middlewares/customerrors/custom_errors.go +++ b/pkg/middlewares/customerrors/custom_errors.go @@ -12,7 +12,7 @@ import ( "github.com/traefik/traefik/v3/pkg/config/dynamic" "github.com/traefik/traefik/v3/pkg/middlewares" - "github.com/traefik/traefik/v3/pkg/tracing" + "github.com/traefik/traefik/v3/pkg/middlewares/observability" "github.com/traefik/traefik/v3/pkg/types" "github.com/vulcand/oxy/v2/utils" "go.opentelemetry.io/otel/trace" @@ -71,7 +71,7 @@ func (c *customErrors) ServeHTTP(rw http.ResponseWriter, req *http.Request) { if c.backendHandler == nil { logger.Error().Msg("Error pages: no backend handler.") - tracing.SetStatusErrorf(req.Context(), "Error pages: no backend handler.") + observability.SetStatusErrorf(req.Context(), "Error pages: no backend handler.") c.next.ServeHTTP(rw, req) return } @@ -96,7 +96,7 @@ func (c *customErrors) ServeHTTP(rw http.ResponseWriter, req *http.Request) { pageReq, err := newRequest("http://" + req.Host + query) if err != nil { logger.Error().Err(err).Send() - tracing.SetStatusErrorf(req.Context(), err.Error()) + observability.SetStatusErrorf(req.Context(), err.Error()) http.Error(rw, http.StatusText(code), code) return } diff --git a/pkg/middlewares/ipallowlist/ip_allowlist.go b/pkg/middlewares/ipallowlist/ip_allowlist.go index 3841f3f23..d7cee8443 100644 --- a/pkg/middlewares/ipallowlist/ip_allowlist.go +++ b/pkg/middlewares/ipallowlist/ip_allowlist.go @@ -10,7 +10,7 @@ import ( "github.com/traefik/traefik/v3/pkg/config/dynamic" "github.com/traefik/traefik/v3/pkg/ip" "github.com/traefik/traefik/v3/pkg/middlewares" - "github.com/traefik/traefik/v3/pkg/tracing" + "github.com/traefik/traefik/v3/pkg/middlewares/observability" "go.opentelemetry.io/otel/trace" ) @@ -78,7 +78,7 @@ func (al *ipAllowLister) ServeHTTP(rw http.ResponseWriter, req *http.Request) { if err != nil { msg := fmt.Sprintf("Rejecting IP %s: %v", clientIP, err) logger.Debug().Msg(msg) - tracing.SetStatusErrorf(req.Context(), msg) + observability.SetStatusErrorf(req.Context(), msg) reject(ctx, al.rejectStatusCode, rw) return } diff --git a/pkg/middlewares/ipwhitelist/ip_whitelist.go b/pkg/middlewares/ipwhitelist/ip_whitelist.go index 937bde420..6a74aebfb 100644 --- a/pkg/middlewares/ipwhitelist/ip_whitelist.go +++ b/pkg/middlewares/ipwhitelist/ip_whitelist.go @@ -10,7 +10,7 @@ import ( "github.com/traefik/traefik/v3/pkg/config/dynamic" "github.com/traefik/traefik/v3/pkg/ip" "github.com/traefik/traefik/v3/pkg/middlewares" - "github.com/traefik/traefik/v3/pkg/tracing" + "github.com/traefik/traefik/v3/pkg/middlewares/observability" "go.opentelemetry.io/otel/trace" ) @@ -68,7 +68,7 @@ func (wl *ipWhiteLister) ServeHTTP(rw http.ResponseWriter, req *http.Request) { if err != nil { msg := fmt.Sprintf("Rejecting IP %s: %v", clientIP, err) logger.Debug().Msg(msg) - tracing.SetStatusErrorf(req.Context(), msg) + observability.SetStatusErrorf(req.Context(), msg) reject(ctx, rw) return } diff --git a/pkg/middlewares/metrics/metrics.go b/pkg/middlewares/metrics/metrics.go index a61da207e..813e50919 100644 --- a/pkg/middlewares/metrics/metrics.go +++ b/pkg/middlewares/metrics/metrics.go @@ -14,9 +14,9 @@ import ( "github.com/traefik/traefik/v3/pkg/metrics" "github.com/traefik/traefik/v3/pkg/middlewares" "github.com/traefik/traefik/v3/pkg/middlewares/capture" + "github.com/traefik/traefik/v3/pkg/middlewares/observability" "github.com/traefik/traefik/v3/pkg/middlewares/retry" traefiktls "github.com/traefik/traefik/v3/pkg/tls" - "github.com/traefik/traefik/v3/pkg/tracing" "go.opentelemetry.io/otel/trace" "google.golang.org/grpc/codes" ) @@ -144,7 +144,7 @@ func (m *metricsMiddleware) ServeHTTP(rw http.ResponseWriter, req *http.Request) } logger := with.Logger() logger.Error().Err(err).Msg("Could not get Capture") - tracing.SetStatusErrorf(req.Context(), "Could not get Capture") + observability.SetStatusErrorf(req.Context(), "Could not get Capture") return } diff --git a/pkg/middlewares/observability/entrypoint.go b/pkg/middlewares/observability/entrypoint.go new file mode 100644 index 000000000..5de208a25 --- /dev/null +++ b/pkg/middlewares/observability/entrypoint.go @@ -0,0 +1,98 @@ +package observability + +import ( + "context" + "fmt" + "net/http" + "strconv" + "strings" + "time" + + "github.com/containous/alice" + "github.com/traefik/traefik/v3/pkg/metrics" + "github.com/traefik/traefik/v3/pkg/middlewares" + "github.com/traefik/traefik/v3/pkg/tracing" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" + semconv "go.opentelemetry.io/otel/semconv/v1.21.0" + "go.opentelemetry.io/otel/trace" + "go.opentelemetry.io/otel/trace/noop" +) + +const ( + entryPointTypeName = "TracingEntryPoint" +) + +type entryPointTracing struct { + tracer *tracing.Tracer + + entryPoint string + next http.Handler + semConvMetricRegistry *metrics.SemConvMetricsRegistry +} + +// WrapEntryPointHandler Wraps tracing to alice.Constructor. +func WrapEntryPointHandler(ctx context.Context, tracer *tracing.Tracer, semConvMetricRegistry *metrics.SemConvMetricsRegistry, entryPointName string) alice.Constructor { + return func(next http.Handler) (http.Handler, error) { + if tracer == nil { + tracer = tracing.NewTracer(noop.Tracer{}, nil, nil) + } + + return newEntryPoint(ctx, tracer, semConvMetricRegistry, entryPointName, next), nil + } +} + +// newEntryPoint creates a new tracing middleware for incoming requests. +func newEntryPoint(ctx context.Context, tracer *tracing.Tracer, semConvMetricRegistry *metrics.SemConvMetricsRegistry, entryPointName string, next http.Handler) http.Handler { + middlewares.GetLogger(ctx, "tracing", entryPointTypeName).Debug().Msg("Creating middleware") + + if tracer == nil { + tracer = tracing.NewTracer(noop.Tracer{}, nil, nil) + } + + return &entryPointTracing{ + entryPoint: entryPointName, + tracer: tracer, + semConvMetricRegistry: semConvMetricRegistry, + next: next, + } +} + +func (e *entryPointTracing) ServeHTTP(rw http.ResponseWriter, req *http.Request) { + tracingCtx := tracing.ExtractCarrierIntoContext(req.Context(), req.Header) + start := time.Now() + tracingCtx, span := e.tracer.Start(tracingCtx, "EntryPoint", trace.WithSpanKind(trace.SpanKindServer), trace.WithTimestamp(start)) + + req = req.WithContext(tracingCtx) + + span.SetAttributes(attribute.String("entry_point", e.entryPoint)) + + e.tracer.CaptureServerRequest(span, req) + + recorder := newStatusCodeRecorder(rw, http.StatusOK) + e.next.ServeHTTP(recorder, req) + + e.tracer.CaptureResponse(span, recorder.Header(), recorder.Status(), trace.SpanKindServer) + + end := time.Now() + span.End(trace.WithTimestamp(end)) + + if e.semConvMetricRegistry != nil && e.semConvMetricRegistry.HTTPServerRequestDuration() != nil { + var attrs []attribute.KeyValue + + if recorder.Status() < 100 || recorder.Status() >= 600 { + attrs = append(attrs, attribute.Key("error.type").String(fmt.Sprintf("Invalid HTTP status code ; %d", recorder.Status()))) + } else if recorder.Status() >= 400 { + attrs = append(attrs, attribute.Key("error.type").String(strconv.Itoa(recorder.Status()))) + } + + attrs = append(attrs, semconv.HTTPRequestMethodKey.String(req.Method)) + attrs = append(attrs, semconv.HTTPResponseStatusCode(recorder.Status())) + attrs = append(attrs, semconv.NetworkProtocolName(strings.ToLower(req.Proto))) + attrs = append(attrs, semconv.NetworkProtocolVersion(Proto(req.Proto))) + attrs = append(attrs, semconv.ServerAddress(req.Host)) + attrs = append(attrs, semconv.URLScheme(req.Header.Get("X-Forwarded-Proto"))) + + e.semConvMetricRegistry.HTTPServerRequestDuration().Record(req.Context(), end.Sub(start).Seconds(), metric.WithAttributes(attrs...)) + } +} diff --git a/pkg/middlewares/observability/entrypoint_test.go b/pkg/middlewares/observability/entrypoint_test.go new file mode 100644 index 000000000..5fc9dde08 --- /dev/null +++ b/pkg/middlewares/observability/entrypoint_test.go @@ -0,0 +1,185 @@ +package observability + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + ptypes "github.com/traefik/paerser/types" + "github.com/traefik/traefik/v3/pkg/metrics" + "github.com/traefik/traefik/v3/pkg/tracing" + "github.com/traefik/traefik/v3/pkg/types" + "go.opentelemetry.io/otel/attribute" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/metricdata" + "go.opentelemetry.io/otel/sdk/metric/metricdata/metricdatatest" +) + +func TestEntryPointMiddleware_tracing(t *testing.T) { + type expected struct { + name string + attributes []attribute.KeyValue + } + + testCases := []struct { + desc string + entryPoint string + expected expected + }{ + { + desc: "basic test", + entryPoint: "test", + expected: expected{ + name: "EntryPoint", + attributes: []attribute.KeyValue{ + attribute.String("span.kind", "server"), + attribute.String("entry_point", "test"), + attribute.String("http.request.method", "GET"), + attribute.String("network.protocol.version", "1.1"), + attribute.Int64("http.request.body.size", int64(0)), + attribute.String("url.path", "/search"), + attribute.String("url.query", "q=Opentelemetry"), + attribute.String("url.scheme", "http"), + attribute.String("user_agent.original", "entrypoint-test"), + attribute.String("server.address", "www.test.com"), + attribute.String("network.peer.address", "10.0.0.1"), + attribute.String("network.peer.port", "1234"), + attribute.String("client.address", "10.0.0.1"), + attribute.Int64("client.port", int64(1234)), + attribute.String("client.socket.address", ""), + attribute.StringSlice("http.request.header.x-foo", []string{"foo", "bar"}), + attribute.Int64("http.response.status_code", int64(404)), + attribute.StringSlice("http.response.header.x-bar", []string{"foo", "bar"}), + }, + }, + }, + } + + for _, test := range testCases { + t.Run(test.desc, func(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "http://www.test.com/search?q=Opentelemetry", nil) + rw := httptest.NewRecorder() + req.RemoteAddr = "10.0.0.1:1234" + req.Header.Set("User-Agent", "entrypoint-test") + req.Header.Set("X-Forwarded-Proto", "http") + req.Header.Set("X-Foo", "foo") + req.Header.Add("X-Foo", "bar") + + next := http.HandlerFunc(func(rw http.ResponseWriter, _ *http.Request) { + rw.Header().Set("X-Bar", "foo") + rw.Header().Add("X-Bar", "bar") + rw.WriteHeader(http.StatusNotFound) + }) + + tracer := &mockTracer{} + + handler := newEntryPoint(context.Background(), tracing.NewTracer(tracer, []string{"X-Foo"}, []string{"X-Bar"}), nil, test.entryPoint, next) + handler.ServeHTTP(rw, req) + + for _, span := range tracer.spans { + assert.Equal(t, test.expected.name, span.name) + assert.Equal(t, test.expected.attributes, span.attributes) + } + }) + } +} + +func TestEntryPointMiddleware_metrics(t *testing.T) { + tests := []struct { + desc string + statusCode int + wantAttributes attribute.Set + }{ + { + desc: "not found status", + statusCode: http.StatusNotFound, + wantAttributes: attribute.NewSet( + attribute.Key("error.type").String("404"), + attribute.Key("http.request.method").String("GET"), + attribute.Key("http.response.status_code").Int(404), + attribute.Key("network.protocol.name").String("http/1.1"), + attribute.Key("network.protocol.version").String("1.1"), + attribute.Key("server.address").String("www.test.com"), + attribute.Key("url.scheme").String("http"), + ), + }, + { + desc: "created status", + statusCode: http.StatusCreated, + wantAttributes: attribute.NewSet( + attribute.Key("http.request.method").String("GET"), + attribute.Key("http.response.status_code").Int(201), + attribute.Key("network.protocol.name").String("http/1.1"), + attribute.Key("network.protocol.version").String("1.1"), + attribute.Key("server.address").String("www.test.com"), + attribute.Key("url.scheme").String("http"), + ), + }, + } + + for _, test := range tests { + test := test + t.Run(test.desc, func(t *testing.T) { + t.Parallel() + + var cfg types.OTLP + (&cfg).SetDefaults() + cfg.AddRoutersLabels = true + cfg.PushInterval = ptypes.Duration(10 * time.Millisecond) + rdr := sdkmetric.NewManualReader() + + meterProvider := sdkmetric.NewMeterProvider(sdkmetric.WithReader(rdr)) + // force the meter provider with manual reader to collect metrics for the test. + metrics.SetMeterProvider(meterProvider) + + semConvMetricRegistry, err := metrics.NewSemConvMetricRegistry(context.Background(), &cfg) + require.NoError(t, err) + require.NotNil(t, semConvMetricRegistry) + + req := httptest.NewRequest(http.MethodGet, "http://www.test.com/search?q=Opentelemetry", nil) + rw := httptest.NewRecorder() + req.RemoteAddr = "10.0.0.1:1234" + req.Header.Set("User-Agent", "entrypoint-test") + req.Header.Set("X-Forwarded-Proto", "http") + + next := http.HandlerFunc(func(rw http.ResponseWriter, _ *http.Request) { + rw.WriteHeader(test.statusCode) + }) + + handler := newEntryPoint(context.Background(), nil, semConvMetricRegistry, "test", next) + handler.ServeHTTP(rw, req) + + got := metricdata.ResourceMetrics{} + err = rdr.Collect(context.Background(), &got) + require.NoError(t, err) + + require.Len(t, got.ScopeMetrics, 1) + + expected := metricdata.Metrics{ + Name: "http.server.request.duration", + Description: "Duration of HTTP server requests.", + Unit: "s", + Data: metricdata.Histogram[float64]{ + DataPoints: []metricdata.HistogramDataPoint[float64]{ + { + Attributes: test.wantAttributes, + Count: 1, + Bounds: []float64{0.005, 0.01, 0.025, 0.05, 0.075, 0.1, 0.25, 0.5, 0.75, 1, 2.5, 5, 7.5, 10}, + BucketCounts: []uint64{0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, + Min: metricdata.NewExtrema[float64](1), + Max: metricdata.NewExtrema[float64](1), + Sum: 1, + }, + }, + Temporality: metricdata.CumulativeTemporality, + }, + } + + metricdatatest.AssertEqual[metricdata.Metrics](t, expected, got.ScopeMetrics[0].Metrics[0], metricdatatest.IgnoreTimestamp(), metricdatatest.IgnoreValue()) + }) + } +} diff --git a/pkg/middlewares/tracing/middleware.go b/pkg/middlewares/observability/middleware.go similarity index 98% rename from pkg/middlewares/tracing/middleware.go rename to pkg/middlewares/observability/middleware.go index 47a6af502..51e4b6d50 100644 --- a/pkg/middlewares/tracing/middleware.go +++ b/pkg/middlewares/observability/middleware.go @@ -1,4 +1,4 @@ -package tracing +package observability import ( "context" diff --git a/pkg/middlewares/tracing/mock_tracing_test.go b/pkg/middlewares/observability/mock_tracing_test.go similarity index 86% rename from pkg/middlewares/tracing/mock_tracing_test.go rename to pkg/middlewares/observability/mock_tracing_test.go index a44345f6a..31ddbc642 100644 --- a/pkg/middlewares/tracing/mock_tracing_test.go +++ b/pkg/middlewares/observability/mock_tracing_test.go @@ -1,4 +1,4 @@ -package tracing +package observability import ( "context" @@ -47,7 +47,9 @@ type mockSpan struct { var _ trace.Span = &mockSpan{} -func (*mockSpan) SpanContext() trace.SpanContext { return trace.SpanContext{} } +func (*mockSpan) SpanContext() trace.SpanContext { + return trace.NewSpanContext(trace.SpanContextConfig{TraceID: trace.TraceID{1}, SpanID: trace.SpanID{1}}) +} func (*mockSpan) IsRecording() bool { return false } func (s *mockSpan) SetStatus(_ codes.Code, _ string) {} func (s *mockSpan) SetAttributes(kv ...attribute.KeyValue) { @@ -59,4 +61,6 @@ func (s *mockSpan) AddEvent(_ string, _ ...trace.EventOption) {} func (s *mockSpan) SetName(name string) { s.name = name } -func (*mockSpan) TracerProvider() trace.TracerProvider { return mockTracerProvider{} } +func (s *mockSpan) TracerProvider() trace.TracerProvider { + return nil +} diff --git a/pkg/middlewares/observability/observability.go b/pkg/middlewares/observability/observability.go new file mode 100644 index 000000000..54f243186 --- /dev/null +++ b/pkg/middlewares/observability/observability.go @@ -0,0 +1,31 @@ +package observability + +import ( + "context" + "fmt" + + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" +) + +// SetStatusErrorf flags the span as in error and log an event. +func SetStatusErrorf(ctx context.Context, format string, args ...interface{}) { + if span := trace.SpanFromContext(ctx); span != nil { + span.SetStatus(codes.Error, fmt.Sprintf(format, args...)) + } +} + +func Proto(proto string) string { + switch proto { + case "HTTP/1.0": + return "1.0" + case "HTTP/1.1": + return "1.1" + case "HTTP/2": + return "2" + case "HTTP/3": + return "3" + default: + return proto + } +} diff --git a/pkg/middlewares/tracing/router.go b/pkg/middlewares/observability/router.go similarity index 98% rename from pkg/middlewares/tracing/router.go rename to pkg/middlewares/observability/router.go index e964be12c..b9e69d78e 100644 --- a/pkg/middlewares/tracing/router.go +++ b/pkg/middlewares/observability/router.go @@ -1,4 +1,4 @@ -package tracing +package observability import ( "context" diff --git a/pkg/middlewares/tracing/router_test.go b/pkg/middlewares/observability/router_test.go similarity index 99% rename from pkg/middlewares/tracing/router_test.go rename to pkg/middlewares/observability/router_test.go index b220cdabe..23bb46ab9 100644 --- a/pkg/middlewares/tracing/router_test.go +++ b/pkg/middlewares/observability/router_test.go @@ -1,4 +1,4 @@ -package tracing +package observability import ( "context" diff --git a/pkg/middlewares/tracing/service.go b/pkg/middlewares/observability/service.go similarity index 98% rename from pkg/middlewares/tracing/service.go rename to pkg/middlewares/observability/service.go index 9cc5bd1ca..cacd3ef1b 100644 --- a/pkg/middlewares/tracing/service.go +++ b/pkg/middlewares/observability/service.go @@ -1,4 +1,4 @@ -package tracing +package observability import ( "context" diff --git a/pkg/middlewares/tracing/service_test.go b/pkg/middlewares/observability/service_test.go similarity index 98% rename from pkg/middlewares/tracing/service_test.go rename to pkg/middlewares/observability/service_test.go index c78510faf..db411e718 100644 --- a/pkg/middlewares/tracing/service_test.go +++ b/pkg/middlewares/observability/service_test.go @@ -1,4 +1,4 @@ -package tracing +package observability import ( "context" diff --git a/pkg/middlewares/tracing/status_code.go b/pkg/middlewares/observability/status_code.go similarity index 97% rename from pkg/middlewares/tracing/status_code.go rename to pkg/middlewares/observability/status_code.go index b30e6d034..ebada5579 100644 --- a/pkg/middlewares/tracing/status_code.go +++ b/pkg/middlewares/observability/status_code.go @@ -1,4 +1,4 @@ -package tracing +package observability import ( "bufio" diff --git a/pkg/middlewares/ratelimiter/rate_limiter.go b/pkg/middlewares/ratelimiter/rate_limiter.go index fcede7dce..0b1a6aaa1 100644 --- a/pkg/middlewares/ratelimiter/rate_limiter.go +++ b/pkg/middlewares/ratelimiter/rate_limiter.go @@ -12,7 +12,7 @@ import ( "github.com/rs/zerolog/log" "github.com/traefik/traefik/v3/pkg/config/dynamic" "github.com/traefik/traefik/v3/pkg/middlewares" - "github.com/traefik/traefik/v3/pkg/tracing" + "github.com/traefik/traefik/v3/pkg/middlewares/observability" "github.com/vulcand/oxy/v2/utils" "go.opentelemetry.io/otel/trace" "golang.org/x/time/rate" @@ -153,14 +153,14 @@ func (rl *rateLimiter) ServeHTTP(rw http.ResponseWriter, req *http.Request) { // as the expiryTime is supposed to reflect the activity (or lack thereof) on that source. if err := rl.buckets.Set(source, bucket, rl.ttl); err != nil { logger.Error().Err(err).Msg("Could not insert/update bucket") - tracing.SetStatusErrorf(req.Context(), "Could not insert/update bucket") + observability.SetStatusErrorf(req.Context(), "Could not insert/update bucket") http.Error(rw, "could not insert/update bucket", http.StatusInternalServerError) return } res := bucket.Reserve() if !res.OK() { - tracing.SetStatusErrorf(req.Context(), "No bursty traffic allowed") + observability.SetStatusErrorf(req.Context(), "No bursty traffic allowed") http.Error(rw, "No bursty traffic allowed", http.StatusTooManyRequests) return } diff --git a/pkg/middlewares/replacepath/replace_path.go b/pkg/middlewares/replacepath/replace_path.go index 3419d7b83..662c5f5d3 100644 --- a/pkg/middlewares/replacepath/replace_path.go +++ b/pkg/middlewares/replacepath/replace_path.go @@ -7,7 +7,7 @@ import ( "github.com/traefik/traefik/v3/pkg/config/dynamic" "github.com/traefik/traefik/v3/pkg/middlewares" - "github.com/traefik/traefik/v3/pkg/tracing" + "github.com/traefik/traefik/v3/pkg/middlewares/observability" "go.opentelemetry.io/otel/trace" ) @@ -52,7 +52,7 @@ func (r *replacePath) ServeHTTP(rw http.ResponseWriter, req *http.Request) { req.URL.Path, err = url.PathUnescape(req.URL.RawPath) if err != nil { middlewares.GetLogger(context.Background(), r.name, typeName).Error().Err(err).Send() - tracing.SetStatusErrorf(req.Context(), err.Error()) + observability.SetStatusErrorf(req.Context(), err.Error()) http.Error(rw, err.Error(), http.StatusInternalServerError) return } diff --git a/pkg/middlewares/replacepathregex/replace_path_regex.go b/pkg/middlewares/replacepathregex/replace_path_regex.go index ff800d354..16c131f8c 100644 --- a/pkg/middlewares/replacepathregex/replace_path_regex.go +++ b/pkg/middlewares/replacepathregex/replace_path_regex.go @@ -10,8 +10,8 @@ import ( "github.com/traefik/traefik/v3/pkg/config/dynamic" "github.com/traefik/traefik/v3/pkg/middlewares" + "github.com/traefik/traefik/v3/pkg/middlewares/observability" "github.com/traefik/traefik/v3/pkg/middlewares/replacepath" - "github.com/traefik/traefik/v3/pkg/tracing" "go.opentelemetry.io/otel/trace" ) @@ -63,7 +63,7 @@ func (rp *replacePathRegex) ServeHTTP(rw http.ResponseWriter, req *http.Request) req.URL.Path, err = url.PathUnescape(req.URL.RawPath) if err != nil { middlewares.GetLogger(context.Background(), rp.name, typeName).Error().Err(err).Send() - tracing.SetStatusErrorf(req.Context(), err.Error()) + observability.SetStatusErrorf(req.Context(), err.Error()) http.Error(rw, err.Error(), http.StatusInternalServerError) return } diff --git a/pkg/middlewares/tracing/entrypoint.go b/pkg/middlewares/tracing/entrypoint.go deleted file mode 100644 index 1d93f452b..000000000 --- a/pkg/middlewares/tracing/entrypoint.go +++ /dev/null @@ -1,62 +0,0 @@ -package tracing - -import ( - "context" - "errors" - "net/http" - - "github.com/containous/alice" - "github.com/traefik/traefik/v3/pkg/middlewares" - "github.com/traefik/traefik/v3/pkg/tracing" - "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/trace" -) - -const ( - entryPointTypeName = "TracingEntryPoint" -) - -type entryPointTracing struct { - tracer *tracing.Tracer - entryPoint string - next http.Handler -} - -// WrapEntryPointHandler Wraps tracing to alice.Constructor. -func WrapEntryPointHandler(ctx context.Context, tracer *tracing.Tracer, entryPointName string) alice.Constructor { - return func(next http.Handler) (http.Handler, error) { - if tracer == nil { - return nil, errors.New("unexpected nil tracer") - } - - return newEntryPoint(ctx, tracer, entryPointName, next), nil - } -} - -// newEntryPoint creates a new tracing middleware for incoming requests. -func newEntryPoint(ctx context.Context, tracer *tracing.Tracer, entryPointName string, next http.Handler) http.Handler { - middlewares.GetLogger(ctx, "tracing", entryPointTypeName).Debug().Msg("Creating middleware") - - return &entryPointTracing{ - entryPoint: entryPointName, - tracer: tracer, - next: next, - } -} - -func (e *entryPointTracing) ServeHTTP(rw http.ResponseWriter, req *http.Request) { - tracingCtx := tracing.ExtractCarrierIntoContext(req.Context(), req.Header) - tracingCtx, span := e.tracer.Start(tracingCtx, "EntryPoint", trace.WithSpanKind(trace.SpanKindServer)) - defer span.End() - - req = req.WithContext(tracingCtx) - - span.SetAttributes(attribute.String("entry_point", e.entryPoint)) - - e.tracer.CaptureServerRequest(span, req) - - recorder := newStatusCodeRecorder(rw, http.StatusOK) - e.next.ServeHTTP(recorder, req) - - e.tracer.CaptureResponse(span, recorder.Header(), recorder.Status(), trace.SpanKindServer) -} diff --git a/pkg/middlewares/tracing/entrypoint_test.go b/pkg/middlewares/tracing/entrypoint_test.go deleted file mode 100644 index 002842662..000000000 --- a/pkg/middlewares/tracing/entrypoint_test.go +++ /dev/null @@ -1,80 +0,0 @@ -package tracing - -import ( - "context" - "net/http" - "net/http/httptest" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/traefik/traefik/v3/pkg/tracing" - "go.opentelemetry.io/otel/attribute" -) - -func TestEntryPointMiddleware(t *testing.T) { - type expected struct { - name string - attributes []attribute.KeyValue - } - - testCases := []struct { - desc string - entryPoint string - expected expected - }{ - { - desc: "basic test", - entryPoint: "test", - expected: expected{ - name: "EntryPoint", - attributes: []attribute.KeyValue{ - attribute.String("span.kind", "server"), - attribute.String("entry_point", "test"), - attribute.String("http.request.method", "GET"), - attribute.String("network.protocol.version", "1.1"), - attribute.Int64("http.request.body.size", int64(0)), - attribute.String("url.path", "/search"), - attribute.String("url.query", "q=Opentelemetry"), - attribute.String("url.scheme", "http"), - attribute.String("user_agent.original", "entrypoint-test"), - attribute.String("server.address", "www.test.com"), - attribute.String("network.peer.address", "10.0.0.1"), - attribute.String("network.peer.port", "1234"), - attribute.String("client.address", "10.0.0.1"), - attribute.Int64("client.port", int64(1234)), - attribute.String("client.socket.address", ""), - attribute.StringSlice("http.request.header.x-foo", []string{"foo", "bar"}), - attribute.Int64("http.response.status_code", int64(404)), - attribute.StringSlice("http.response.header.x-bar", []string{"foo", "bar"}), - }, - }, - }, - } - - for _, test := range testCases { - t.Run(test.desc, func(t *testing.T) { - req := httptest.NewRequest(http.MethodGet, "http://www.test.com/search?q=Opentelemetry", nil) - rw := httptest.NewRecorder() - req.RemoteAddr = "10.0.0.1:1234" - req.Header.Set("User-Agent", "entrypoint-test") - req.Header.Set("X-Forwarded-Proto", "http") - req.Header.Set("X-Foo", "foo") - req.Header.Add("X-Foo", "bar") - - next := http.HandlerFunc(func(rw http.ResponseWriter, _ *http.Request) { - rw.Header().Set("X-Bar", "foo") - rw.Header().Add("X-Bar", "bar") - rw.WriteHeader(http.StatusNotFound) - }) - - mockTracer := &mockTracer{} - handler := newEntryPoint(context.Background(), tracing.NewTracer(mockTracer, []string{"X-Foo"}, []string{"X-Bar"}), test.entryPoint, next) - handler.ServeHTTP(rw, req) - - for _, span := range mockTracer.spans { - assert.Equal(t, test.expected.name, span.name) - assert.Equal(t, test.expected.attributes, span.attributes) - } - }) - } -} diff --git a/pkg/server/middleware/middlewares.go b/pkg/server/middleware/middlewares.go index 3065b16ae..0297daa93 100644 --- a/pkg/server/middleware/middlewares.go +++ b/pkg/server/middleware/middlewares.go @@ -24,6 +24,7 @@ import ( "github.com/traefik/traefik/v3/pkg/middlewares/inflightreq" "github.com/traefik/traefik/v3/pkg/middlewares/ipallowlist" "github.com/traefik/traefik/v3/pkg/middlewares/ipwhitelist" + "github.com/traefik/traefik/v3/pkg/middlewares/observability" "github.com/traefik/traefik/v3/pkg/middlewares/passtlsclientcert" "github.com/traefik/traefik/v3/pkg/middlewares/ratelimiter" "github.com/traefik/traefik/v3/pkg/middlewares/redirect" @@ -32,7 +33,6 @@ import ( "github.com/traefik/traefik/v3/pkg/middlewares/retry" "github.com/traefik/traefik/v3/pkg/middlewares/stripprefix" "github.com/traefik/traefik/v3/pkg/middlewares/stripprefixregex" - "github.com/traefik/traefik/v3/pkg/middlewares/tracing" "github.com/traefik/traefik/v3/pkg/server/provider" ) @@ -390,7 +390,7 @@ func (b *Builder) buildConstructor(ctx context.Context, middlewareName string) ( // The tracing middleware is a NOOP if tracing is not setup on the middleware chain. // Hence, regarding internal resources' observability deactivation, // this would not enable tracing. - return tracing.WrapMiddleware(ctx, middleware), nil + return observability.WrapMiddleware(ctx, middleware), nil } func inSlice(element string, stack []string) bool { diff --git a/pkg/server/middleware/observability.go b/pkg/server/middleware/observability.go index 7368bd3a1..cf8386999 100644 --- a/pkg/server/middleware/observability.go +++ b/pkg/server/middleware/observability.go @@ -14,7 +14,7 @@ import ( "github.com/traefik/traefik/v3/pkg/middlewares/accesslog" "github.com/traefik/traefik/v3/pkg/middlewares/capture" metricsMiddle "github.com/traefik/traefik/v3/pkg/middlewares/metrics" - tracingMiddle "github.com/traefik/traefik/v3/pkg/middlewares/tracing" + "github.com/traefik/traefik/v3/pkg/middlewares/observability" "github.com/traefik/traefik/v3/pkg/tracing" ) @@ -23,15 +23,17 @@ type ObservabilityMgr struct { config static.Configuration accessLoggerMiddleware *accesslog.Handler metricsRegistry metrics.Registry + semConvMetricRegistry *metrics.SemConvMetricsRegistry tracer *tracing.Tracer tracerCloser io.Closer } // NewObservabilityMgr creates a new ObservabilityMgr. -func NewObservabilityMgr(config static.Configuration, metricsRegistry metrics.Registry, accessLoggerMiddleware *accesslog.Handler, tracer *tracing.Tracer, tracerCloser io.Closer) *ObservabilityMgr { +func NewObservabilityMgr(config static.Configuration, metricsRegistry metrics.Registry, semConvMetricRegistry *metrics.SemConvMetricsRegistry, accessLoggerMiddleware *accesslog.Handler, tracer *tracing.Tracer, tracerCloser io.Closer) *ObservabilityMgr { return &ObservabilityMgr{ config: config, metricsRegistry: metricsRegistry, + semConvMetricRegistry: semConvMetricRegistry, accessLoggerMiddleware: accessLoggerMiddleware, tracer: tracer, tracerCloser: tracerCloser, @@ -39,35 +41,35 @@ func NewObservabilityMgr(config static.Configuration, metricsRegistry metrics.Re } // BuildEPChain an observability middleware chain by entry point. -func (c *ObservabilityMgr) BuildEPChain(ctx context.Context, entryPointName string, resourceName string) alice.Chain { +func (o *ObservabilityMgr) BuildEPChain(ctx context.Context, entryPointName string, resourceName string) alice.Chain { chain := alice.New() - if c == nil { + if o == nil { return chain } - if c.accessLoggerMiddleware != nil || c.metricsRegistry != nil && (c.metricsRegistry.IsEpEnabled() || c.metricsRegistry.IsRouterEnabled() || c.metricsRegistry.IsSvcEnabled()) { - if c.ShouldAddAccessLogs(resourceName) || c.ShouldAddMetrics(resourceName) { + if o.accessLoggerMiddleware != nil || o.metricsRegistry != nil && (o.metricsRegistry.IsEpEnabled() || o.metricsRegistry.IsRouterEnabled() || o.metricsRegistry.IsSvcEnabled()) { + if o.ShouldAddAccessLogs(resourceName) || o.ShouldAddMetrics(resourceName) { chain = chain.Append(capture.Wrap) } } - if c.accessLoggerMiddleware != nil && c.ShouldAddAccessLogs(resourceName) { - chain = chain.Append(accesslog.WrapHandler(c.accessLoggerMiddleware)) + if o.accessLoggerMiddleware != nil && o.ShouldAddAccessLogs(resourceName) { + chain = chain.Append(accesslog.WrapHandler(o.accessLoggerMiddleware)) chain = chain.Append(func(next http.Handler) (http.Handler, error) { return accesslog.NewFieldHandler(next, logs.EntryPointName, entryPointName, accesslog.InitServiceFields), nil }) } - if c.tracer != nil && c.ShouldAddTracing(resourceName) { - chain = chain.Append(tracingMiddle.WrapEntryPointHandler(ctx, c.tracer, entryPointName)) + if (o.tracer != nil && o.ShouldAddTracing(resourceName)) || (o.metricsRegistry != nil && o.metricsRegistry.IsEpEnabled() && o.ShouldAddMetrics(resourceName)) { + chain = chain.Append(observability.WrapEntryPointHandler(ctx, o.tracer, o.semConvMetricRegistry, entryPointName)) } - if c.metricsRegistry != nil && c.metricsRegistry.IsEpEnabled() && c.ShouldAddMetrics(resourceName) { - metricsHandler := metricsMiddle.WrapEntryPointHandler(ctx, c.metricsRegistry, entryPointName) + if o.metricsRegistry != nil && o.metricsRegistry.IsEpEnabled() && o.ShouldAddMetrics(resourceName) { + metricsHandler := metricsMiddle.WrapEntryPointHandler(ctx, o.metricsRegistry, entryPointName) - if c.tracer != nil && c.ShouldAddTracing(resourceName) { - chain = chain.Append(tracingMiddle.WrapMiddleware(ctx, metricsHandler)) + if o.tracer != nil && o.ShouldAddTracing(resourceName) { + chain = chain.Append(observability.WrapMiddleware(ctx, metricsHandler)) } else { chain = chain.Append(metricsHandler) } @@ -77,64 +79,73 @@ func (c *ObservabilityMgr) BuildEPChain(ctx context.Context, entryPointName stri } // ShouldAddAccessLogs returns whether the access logs should be enabled for the given resource. -func (c *ObservabilityMgr) ShouldAddAccessLogs(resourceName string) bool { - if c == nil { +func (o *ObservabilityMgr) ShouldAddAccessLogs(resourceName string) bool { + if o == nil { return false } - return c.config.AccessLog != nil && (c.config.AccessLog.AddInternals || !strings.HasSuffix(resourceName, "@internal")) + return o.config.AccessLog != nil && (o.config.AccessLog.AddInternals || !strings.HasSuffix(resourceName, "@internal")) } // ShouldAddMetrics returns whether the metrics should be enabled for the given resource. -func (c *ObservabilityMgr) ShouldAddMetrics(resourceName string) bool { - if c == nil { +func (o *ObservabilityMgr) ShouldAddMetrics(resourceName string) bool { + if o == nil { return false } - return c.config.Metrics != nil && (c.config.Metrics.AddInternals || !strings.HasSuffix(resourceName, "@internal")) + return o.config.Metrics != nil && (o.config.Metrics.AddInternals || !strings.HasSuffix(resourceName, "@internal")) } // ShouldAddTracing returns whether the tracing should be enabled for the given resource. -func (c *ObservabilityMgr) ShouldAddTracing(resourceName string) bool { - if c == nil { +func (o *ObservabilityMgr) ShouldAddTracing(resourceName string) bool { + if o == nil { return false } - return c.config.Tracing != nil && (c.config.Tracing.AddInternals || !strings.HasSuffix(resourceName, "@internal")) + return o.config.Tracing != nil && (o.config.Tracing.AddInternals || !strings.HasSuffix(resourceName, "@internal")) } // MetricsRegistry is an accessor to the metrics registry. -func (c *ObservabilityMgr) MetricsRegistry() metrics.Registry { - if c == nil { +func (o *ObservabilityMgr) MetricsRegistry() metrics.Registry { + if o == nil { return nil } - return c.metricsRegistry + return o.metricsRegistry +} + +// SemConvMetricsRegistry is an accessor to the semantic conventions metrics registry. +func (o *ObservabilityMgr) SemConvMetricsRegistry() *metrics.SemConvMetricsRegistry { + if o == nil { + return nil + } + + return o.semConvMetricRegistry } // Close closes the accessLogger and tracer. -func (c *ObservabilityMgr) Close() { - if c == nil { +func (o *ObservabilityMgr) Close() { + if o == nil { return } - if c.accessLoggerMiddleware != nil { - if err := c.accessLoggerMiddleware.Close(); err != nil { + if o.accessLoggerMiddleware != nil { + if err := o.accessLoggerMiddleware.Close(); err != nil { log.Error().Err(err).Msg("Could not close the access log file") } } - if c.tracerCloser != nil { - if err := c.tracerCloser.Close(); err != nil { + if o.tracerCloser != nil { + if err := o.tracerCloser.Close(); err != nil { log.Error().Err(err).Msg("Could not close the tracer") } } } -func (c *ObservabilityMgr) RotateAccessLogs() error { - if c.accessLoggerMiddleware == nil { +func (o *ObservabilityMgr) RotateAccessLogs() error { + if o.accessLoggerMiddleware == nil { return nil } - return c.accessLoggerMiddleware.Rotate() + return o.accessLoggerMiddleware.Rotate() } diff --git a/pkg/server/router/router.go b/pkg/server/router/router.go index eae645383..369aaa7ab 100644 --- a/pkg/server/router/router.go +++ b/pkg/server/router/router.go @@ -13,8 +13,8 @@ import ( "github.com/traefik/traefik/v3/pkg/middlewares/accesslog" "github.com/traefik/traefik/v3/pkg/middlewares/denyrouterrecursion" metricsMiddle "github.com/traefik/traefik/v3/pkg/middlewares/metrics" + "github.com/traefik/traefik/v3/pkg/middlewares/observability" "github.com/traefik/traefik/v3/pkg/middlewares/recovery" - "github.com/traefik/traefik/v3/pkg/middlewares/tracing" httpmuxer "github.com/traefik/traefik/v3/pkg/muxer/http" "github.com/traefik/traefik/v3/pkg/server/middleware" "github.com/traefik/traefik/v3/pkg/server/provider" @@ -221,11 +221,11 @@ func (m *Manager) buildHTTPHandler(ctx context.Context, router *runtime.RouterIn return chain.Extend(*mHandler).Then(sHandler) } - chain = chain.Append(tracing.WrapRouterHandler(ctx, routerName, router.Rule, provider.GetQualifiedName(ctx, router.Service))) + chain = chain.Append(observability.WrapRouterHandler(ctx, routerName, router.Rule, provider.GetQualifiedName(ctx, router.Service))) if m.observabilityMgr.MetricsRegistry() != nil && m.observabilityMgr.MetricsRegistry().IsRouterEnabled() { metricsHandler := metricsMiddle.WrapRouterHandler(ctx, m.observabilityMgr.MetricsRegistry(), routerName, provider.GetQualifiedName(ctx, router.Service)) - chain = chain.Append(tracing.WrapMiddleware(ctx, metricsHandler)) + chain = chain.Append(observability.WrapMiddleware(ctx, metricsHandler)) } if router.DefaultRule { diff --git a/pkg/server/routerfactory_test.go b/pkg/server/routerfactory_test.go index ebd923f62..971b58c99 100644 --- a/pkg/server/routerfactory_test.go +++ b/pkg/server/routerfactory_test.go @@ -193,7 +193,7 @@ func TestServerResponseEmptyBackend(t *testing.T) { dialerManager := tcp.NewDialerManager(nil) dialerManager.Update(map[string]*dynamic.TCPServersTransport{"default@internal": {}}) - observabiltyMgr := middleware.NewObservabilityMgr(staticConfig, nil, nil, nil, nil) + observabiltyMgr := middleware.NewObservabilityMgr(staticConfig, nil, nil, nil, nil, nil) factory := NewRouterFactory(staticConfig, managerFactory, tlsManager, observabiltyMgr, nil, dialerManager) entryPointsHandlers, _ := factory.CreateRouters(runtime.NewConfig(dynamic.Configuration{HTTP: test.config(testServer.URL)})) diff --git a/pkg/server/service/observability_roundtripper.go b/pkg/server/service/observability_roundtripper.go new file mode 100644 index 000000000..ae862deed --- /dev/null +++ b/pkg/server/service/observability_roundtripper.go @@ -0,0 +1,105 @@ +package service + +import ( + "context" + "fmt" + "net" + "net/http" + "strconv" + "strings" + "time" + + "github.com/traefik/traefik/v3/pkg/metrics" + "github.com/traefik/traefik/v3/pkg/middlewares/observability" + "github.com/traefik/traefik/v3/pkg/tracing" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" + semconv "go.opentelemetry.io/otel/semconv/v1.21.0" + "go.opentelemetry.io/otel/trace" +) + +type wrapper struct { + semConvMetricRegistry *metrics.SemConvMetricsRegistry + rt http.RoundTripper +} + +func (t *wrapper) RoundTrip(req *http.Request) (*http.Response, error) { + start := time.Now() + var span trace.Span + var tracingCtx context.Context + var tracer *tracing.Tracer + if tracer = tracing.TracerFromContext(req.Context()); tracer != nil { + tracingCtx, span = tracer.Start(req.Context(), "ReverseProxy", trace.WithSpanKind(trace.SpanKindClient)) + defer span.End() + + req = req.WithContext(tracingCtx) + + tracer.CaptureClientRequest(span, req) + tracing.InjectContextIntoCarrier(req) + } + + var statusCode int + var headers http.Header + response, err := t.rt.RoundTrip(req) + if err != nil { + statusCode = computeStatusCode(err) + } + if response != nil { + statusCode = response.StatusCode + headers = response.Header + } + + if tracer != nil { + tracer.CaptureResponse(span, headers, statusCode, trace.SpanKindClient) + } + + end := time.Now() + + // Ending the span as soon as the response is handled because we want to use the same end time for the trace and the metric. + // If any errors happen earlier, this span will be close by the defer instruction. + if span != nil { + span.End(trace.WithTimestamp(end)) + } + + if t.semConvMetricRegistry != nil && t.semConvMetricRegistry.HTTPClientRequestDuration() != nil { + var attrs []attribute.KeyValue + + if statusCode < 100 || statusCode >= 600 { + attrs = append(attrs, attribute.Key("error.type").String(fmt.Sprintf("Invalid HTTP status code %d", statusCode))) + } else if statusCode >= 400 { + attrs = append(attrs, attribute.Key("error.type").String(strconv.Itoa(statusCode))) + } + + attrs = append(attrs, semconv.HTTPRequestMethodKey.String(req.Method)) + attrs = append(attrs, semconv.HTTPResponseStatusCode(statusCode)) + attrs = append(attrs, semconv.NetworkProtocolName(strings.ToLower(req.Proto))) + attrs = append(attrs, semconv.NetworkProtocolVersion(observability.Proto(req.Proto))) + attrs = append(attrs, semconv.ServerAddress(req.URL.Host)) + + _, port, err := net.SplitHostPort(req.URL.Host) + if err != nil { + switch req.URL.Scheme { + case "http": + attrs = append(attrs, semconv.ServerPort(80)) + case "https": + attrs = append(attrs, semconv.ServerPort(443)) + } + } else { + intPort, _ := strconv.Atoi(port) + attrs = append(attrs, semconv.ServerPort(intPort)) + } + + attrs = append(attrs, semconv.URLScheme(req.Header.Get("X-Forwarded-Proto"))) + + t.semConvMetricRegistry.HTTPClientRequestDuration().Record(req.Context(), end.Sub(start).Seconds(), metric.WithAttributes(attrs...)) + } + + return response, err +} + +func newObservabilityRoundTripper(semConvMetricRegistry *metrics.SemConvMetricsRegistry, rt http.RoundTripper) http.RoundTripper { + return &wrapper{ + semConvMetricRegistry: semConvMetricRegistry, + rt: rt, + } +} diff --git a/pkg/server/service/observability_roundtripper_test.go b/pkg/server/service/observability_roundtripper_test.go new file mode 100644 index 000000000..afbac748e --- /dev/null +++ b/pkg/server/service/observability_roundtripper_test.go @@ -0,0 +1,123 @@ +package service + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/stretchr/testify/require" + ptypes "github.com/traefik/paerser/types" + "github.com/traefik/traefik/v3/pkg/metrics" + "github.com/traefik/traefik/v3/pkg/types" + "go.opentelemetry.io/otel/attribute" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/metricdata" + "go.opentelemetry.io/otel/sdk/metric/metricdata/metricdatatest" +) + +func TestObservabilityRoundTripper_metrics(t *testing.T) { + tests := []struct { + desc string + serverURL string + statusCode int + wantAttributes attribute.Set + }{ + { + desc: "not found status", + serverURL: "http://www.test.com", + statusCode: http.StatusNotFound, + wantAttributes: attribute.NewSet( + attribute.Key("error.type").String("404"), + attribute.Key("http.request.method").String("GET"), + attribute.Key("http.response.status_code").Int(404), + attribute.Key("network.protocol.name").String("http/1.1"), + attribute.Key("network.protocol.version").String("1.1"), + attribute.Key("server.address").String("www.test.com"), + attribute.Key("server.port").Int(80), + attribute.Key("url.scheme").String("http"), + ), + }, + { + desc: "created status", + serverURL: "https://www.test.com", + statusCode: http.StatusCreated, + wantAttributes: attribute.NewSet( + attribute.Key("http.request.method").String("GET"), + attribute.Key("http.response.status_code").Int(201), + attribute.Key("network.protocol.name").String("http/1.1"), + attribute.Key("network.protocol.version").String("1.1"), + attribute.Key("server.address").String("www.test.com"), + attribute.Key("server.port").Int(443), + attribute.Key("url.scheme").String("http"), + ), + }, + } + + for _, test := range tests { + test := test + t.Run(test.desc, func(t *testing.T) { + t.Parallel() + + var cfg types.OTLP + (&cfg).SetDefaults() + cfg.AddRoutersLabels = true + cfg.PushInterval = ptypes.Duration(10 * time.Millisecond) + rdr := sdkmetric.NewManualReader() + + meterProvider := sdkmetric.NewMeterProvider(sdkmetric.WithReader(rdr)) + // force the meter provider with manual reader to collect metrics for the test. + metrics.SetMeterProvider(meterProvider) + + semConvMetricRegistry, err := metrics.NewSemConvMetricRegistry(context.Background(), &cfg) + require.NoError(t, err) + require.NotNil(t, semConvMetricRegistry) + + req := httptest.NewRequest(http.MethodGet, test.serverURL+"/search?q=Opentelemetry", nil) + req.RemoteAddr = "10.0.0.1:1234" + req.Header.Set("User-Agent", "rt-test") + req.Header.Set("X-Forwarded-Proto", "http") + + ort := newObservabilityRoundTripper(semConvMetricRegistry, mockRoundTripper{statusCode: test.statusCode}) + _, err = ort.RoundTrip(req) + require.NoError(t, err) + + got := metricdata.ResourceMetrics{} + err = rdr.Collect(context.Background(), &got) + require.NoError(t, err) + + require.Len(t, got.ScopeMetrics, 1) + + expected := metricdata.Metrics{ + Name: "http.client.request.duration", + Description: "Duration of HTTP client requests.", + Unit: "s", + Data: metricdata.Histogram[float64]{ + DataPoints: []metricdata.HistogramDataPoint[float64]{ + { + Attributes: test.wantAttributes, + Count: 1, + Bounds: []float64{0.005, 0.01, 0.025, 0.05, 0.075, 0.1, 0.25, 0.5, 0.75, 1, 2.5, 5, 7.5, 10}, + BucketCounts: []uint64{0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, + Min: metricdata.NewExtrema[float64](1), + Max: metricdata.NewExtrema[float64](1), + Sum: 1, + }, + }, + Temporality: metricdata.CumulativeTemporality, + }, + } + + metricdatatest.AssertEqual[metricdata.Metrics](t, expected, got.ScopeMetrics[0].Metrics[0], metricdatatest.IgnoreTimestamp(), metricdatatest.IgnoreValue()) + }) + } +} + +type mockRoundTripper struct { + statusCode int +} + +func (m mockRoundTripper) RoundTrip(request *http.Request) (*http.Response, error) { + return &http.Response{StatusCode: m.statusCode}, nil +} diff --git a/pkg/server/service/proxy.go b/pkg/server/service/proxy.go index 083875f9e..160ee03a4 100644 --- a/pkg/server/service/proxy.go +++ b/pkg/server/service/proxy.go @@ -22,13 +22,9 @@ const StatusClientClosedRequest = 499 const StatusClientClosedRequestText = "Client Closed Request" func buildSingleHostProxy(target *url.URL, passHostHeader bool, flushInterval time.Duration, roundTripper http.RoundTripper, bufferPool httputil.BufferPool) http.Handler { - // Wrapping the roundTripper with the Tracing roundTripper, - // to handle the reverseProxy client span creation. - tracingRoundTripper := newTracingRoundTripper(roundTripper) - return &httputil.ReverseProxy{ Director: directorBuilder(target, passHostHeader), - Transport: tracingRoundTripper, + Transport: roundTripper, FlushInterval: flushInterval, BufferPool: bufferPool, ErrorHandler: errorHandler, diff --git a/pkg/server/service/service.go b/pkg/server/service/service.go index b01d2327c..fcd422a7d 100644 --- a/pkg/server/service/service.go +++ b/pkg/server/service/service.go @@ -22,7 +22,7 @@ import ( "github.com/traefik/traefik/v3/pkg/logs" "github.com/traefik/traefik/v3/pkg/middlewares/accesslog" metricsMiddle "github.com/traefik/traefik/v3/pkg/middlewares/metrics" - tracingMiddle "github.com/traefik/traefik/v3/pkg/middlewares/tracing" + "github.com/traefik/traefik/v3/pkg/middlewares/observability" "github.com/traefik/traefik/v3/pkg/safe" "github.com/traefik/traefik/v3/pkg/server/cookie" "github.com/traefik/traefik/v3/pkg/server/middleware" @@ -300,30 +300,38 @@ func (m *Manager) getLoadBalancerServiceHandler(ctx context.Context, serviceName logger.Debug().Str(logs.ServerName, proxyName).Stringer("target", target). Msg("Creating server") + qualifiedSvcName := provider.GetQualifiedName(ctx, serviceName) + + if m.observabilityMgr.ShouldAddTracing(qualifiedSvcName) || m.observabilityMgr.ShouldAddMetrics(qualifiedSvcName) { + // Wrapping the roundTripper with the Tracing roundTripper, + // to handle the reverseProxy client span creation. + roundTripper = newObservabilityRoundTripper(m.observabilityMgr.SemConvMetricsRegistry(), roundTripper) + } + proxy := buildSingleHostProxy(target, passHostHeader, time.Duration(flushInterval), roundTripper, m.bufferPool) // Prevents from enabling observability for internal resources. - if m.observabilityMgr.ShouldAddAccessLogs(provider.GetQualifiedName(ctx, serviceName)) { + if m.observabilityMgr.ShouldAddAccessLogs(qualifiedSvcName) { proxy = accesslog.NewFieldHandler(proxy, accesslog.ServiceURL, target.String(), nil) proxy = accesslog.NewFieldHandler(proxy, accesslog.ServiceAddr, target.Host, nil) proxy = accesslog.NewFieldHandler(proxy, accesslog.ServiceName, serviceName, accesslog.AddServiceFields) } if m.observabilityMgr.MetricsRegistry() != nil && m.observabilityMgr.MetricsRegistry().IsSvcEnabled() && - m.observabilityMgr.ShouldAddMetrics(provider.GetQualifiedName(ctx, serviceName)) { + m.observabilityMgr.ShouldAddMetrics(qualifiedSvcName) { metricsHandler := metricsMiddle.WrapServiceHandler(ctx, m.observabilityMgr.MetricsRegistry(), serviceName) proxy, err = alice.New(). - Append(tracingMiddle.WrapMiddleware(ctx, metricsHandler)). + Append(observability.WrapMiddleware(ctx, metricsHandler)). Then(proxy) if err != nil { return nil, fmt.Errorf("error wrapping metrics handler: %w", err) } } - if m.observabilityMgr.ShouldAddTracing(provider.GetQualifiedName(ctx, serviceName)) { - proxy = tracingMiddle.NewService(ctx, serviceName, proxy) + if m.observabilityMgr.ShouldAddTracing(qualifiedSvcName) { + proxy = observability.NewService(ctx, serviceName, proxy) } lb.Add(proxyName, proxy, server.Weight) diff --git a/pkg/server/service/tracing_roundtripper.go b/pkg/server/service/tracing_roundtripper.go deleted file mode 100644 index 69b0edf40..000000000 --- a/pkg/server/service/tracing_roundtripper.go +++ /dev/null @@ -1,44 +0,0 @@ -package service - -import ( - "context" - "net/http" - - "github.com/traefik/traefik/v3/pkg/tracing" - "go.opentelemetry.io/otel/trace" -) - -type wrapper struct { - rt http.RoundTripper -} - -func (t *wrapper) RoundTrip(req *http.Request) (*http.Response, error) { - var span trace.Span - var tracer *tracing.Tracer - if tracer = tracing.TracerFromContext(req.Context()); tracer != nil { - var tracingCtx context.Context - tracingCtx, span = tracer.Start(req.Context(), "ReverseProxy", trace.WithSpanKind(trace.SpanKindClient)) - defer span.End() - - req = req.WithContext(tracingCtx) - - tracer.CaptureClientRequest(span, req) - tracing.InjectContextIntoCarrier(req) - } - - response, err := t.rt.RoundTrip(req) - if err != nil { - statusCode := computeStatusCode(err) - tracer.CaptureResponse(span, nil, statusCode, trace.SpanKindClient) - - return response, err - } - - tracer.CaptureResponse(span, response.Header, response.StatusCode, trace.SpanKindClient) - - return response, nil -} - -func newTracingRoundTripper(rt http.RoundTripper) http.RoundTripper { - return &wrapper{rt: rt} -} diff --git a/pkg/server/service/tracing_roundtripper_test.go b/pkg/server/service/tracing_roundtripper_test.go deleted file mode 100644 index d629d6d34..000000000 --- a/pkg/server/service/tracing_roundtripper_test.go +++ /dev/null @@ -1,138 +0,0 @@ -package service - -import ( - "context" - "net/http" - "net/http/httptest" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/traefik/traefik/v3/pkg/tracing" - "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/codes" - "go.opentelemetry.io/otel/trace" - "go.opentelemetry.io/otel/trace/embedded" -) - -func TestTracingRoundTripper(t *testing.T) { - type expected struct { - name string - attributes []attribute.KeyValue - } - - testCases := []struct { - desc string - expected []expected - }{ - { - desc: "basic test", - expected: []expected{ - { - name: "initial", - attributes: []attribute.KeyValue{ - attribute.String("span.kind", "unspecified"), - }, - }, - { - name: "ReverseProxy", - attributes: []attribute.KeyValue{ - attribute.String("span.kind", "client"), - attribute.String("http.request.method", "GET"), - attribute.String("network.protocol.version", "1.1"), - attribute.String("url.full", "http://www.test.com/search?q=Opentelemetry"), - attribute.String("url.scheme", "http"), - attribute.String("user_agent.original", "reverse-test"), - attribute.String("network.peer.address", ""), - attribute.String("server.address", "www.test.com"), - attribute.String("network.peer.port", "80"), - attribute.Int64("server.port", int64(80)), - attribute.StringSlice("http.request.header.x-foo", []string{"foo", "bar"}), - attribute.Int64("http.response.status_code", int64(404)), - attribute.StringSlice("http.response.header.x-bar", []string{"foo", "bar"}), - }, - }, - }, - }, - } - - for _, test := range testCases { - t.Run(test.desc, func(t *testing.T) { - req := httptest.NewRequest(http.MethodGet, "http://www.test.com/search?q=Opentelemetry", nil) - req.RemoteAddr = "10.0.0.1:1234" - req.Header.Set("User-Agent", "reverse-test") - req.Header.Set("X-Forwarded-Proto", "http") - req.Header.Set("X-Foo", "foo") - req.Header.Add("X-Foo", "bar") - - mockTracer := &mockTracer{} - tracer := tracing.NewTracer(mockTracer, []string{"X-Foo"}, []string{"X-Bar"}) - initialCtx, initialSpan := tracer.Start(req.Context(), "initial") - defer initialSpan.End() - req = req.WithContext(initialCtx) - - tracingRoundTripper := newTracingRoundTripper(roundTripperFn(func(req *http.Request) (*http.Response, error) { - return &http.Response{ - Header: map[string][]string{ - "X-Bar": {"foo", "bar"}, - }, - StatusCode: http.StatusNotFound, - }, nil - })) - - _, err := tracingRoundTripper.RoundTrip(req) - require.NoError(t, err) - - for i, span := range mockTracer.spans { - assert.Equal(t, test.expected[i].name, span.name) - assert.Equal(t, test.expected[i].attributes, span.attributes) - } - }) - } -} - -type mockTracer struct { - embedded.Tracer - - spans []*mockSpan -} - -var _ trace.Tracer = &mockTracer{} - -func (t *mockTracer) Start(ctx context.Context, name string, opts ...trace.SpanStartOption) (context.Context, trace.Span) { - config := trace.NewSpanStartConfig(opts...) - span := &mockSpan{} - span.SetName(name) - span.SetAttributes(attribute.String("span.kind", config.SpanKind().String())) - span.SetAttributes(config.Attributes()...) - t.spans = append(t.spans, span) - return trace.ContextWithSpan(ctx, span), span -} - -// mockSpan is an implementation of Span that preforms no operations. -type mockSpan struct { - embedded.Span - - name string - attributes []attribute.KeyValue -} - -var _ trace.Span = &mockSpan{} - -func (*mockSpan) SpanContext() trace.SpanContext { - return trace.NewSpanContext(trace.SpanContextConfig{TraceID: trace.TraceID{1}, SpanID: trace.SpanID{1}}) -} -func (*mockSpan) IsRecording() bool { return false } -func (s *mockSpan) SetStatus(_ codes.Code, _ string) {} -func (s *mockSpan) SetAttributes(kv ...attribute.KeyValue) { - s.attributes = append(s.attributes, kv...) -} -func (s *mockSpan) End(...trace.SpanEndOption) {} -func (s *mockSpan) RecordError(_ error, _ ...trace.EventOption) {} -func (s *mockSpan) AddEvent(_ string, _ ...trace.EventOption) {} - -func (s *mockSpan) SetName(name string) { s.name = name } - -func (s *mockSpan) TracerProvider() trace.TracerProvider { - return nil -} diff --git a/pkg/tracing/opentelemetry/opentelemetry_test.go b/pkg/tracing/opentelemetry/opentelemetry_test.go index 86f3a7b99..3d6ebfd16 100644 --- a/pkg/tracing/opentelemetry/opentelemetry_test.go +++ b/pkg/tracing/opentelemetry/opentelemetry_test.go @@ -14,7 +14,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/traefik/traefik/v3/pkg/config/static" - tracingMiddle "github.com/traefik/traefik/v3/pkg/middlewares/tracing" + "github.com/traefik/traefik/v3/pkg/middlewares/observability" "github.com/traefik/traefik/v3/pkg/tracing" "github.com/traefik/traefik/v3/pkg/tracing/opentelemetry" "github.com/traefik/traefik/v3/pkg/types" @@ -296,7 +296,7 @@ func TestTracing(t *testing.T) { _ = closer.Close() }) - chain := alice.New(tracingMiddle.WrapEntryPointHandler(context.Background(), newTracing, "test")) + chain := alice.New(observability.WrapEntryPointHandler(context.Background(), newTracing, nil, "test")) epHandler, err := chain.Then(service) require.NoError(t, err) diff --git a/pkg/types/metrics.go b/pkg/types/metrics.go index 28b151a08..cde08e189 100644 --- a/pkg/types/metrics.go +++ b/pkg/types/metrics.go @@ -125,7 +125,7 @@ func (o *OTLP) SetDefaults() { o.AddEntryPointsLabels = true o.AddServicesLabels = true - o.ExplicitBoundaries = []float64{.005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10} + o.ExplicitBoundaries = []float64{.005, .01, .025, .05, .075, .1, .25, .5, .75, 1, 2.5, 5, 7.5, 10} o.PushInterval = types.Duration(10 * time.Second) }