traefik/pkg/metrics/prometheus.go
2023-03-21 16:45:33 +01:00

672 lines
22 KiB
Go

package metrics
import (
"context"
"errors"
"net/http"
"sync"
"time"
"github.com/go-kit/kit/metrics"
stdprometheus "github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/collectors"
"github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/rs/zerolog/log"
"github.com/traefik/traefik/v3/pkg/config/dynamic"
"github.com/traefik/traefik/v3/pkg/types"
)
const (
// MetricNamePrefix prefix of all metric names.
MetricNamePrefix = "traefik_"
// server meta information.
metricConfigPrefix = MetricNamePrefix + "config_"
configReloadsTotalName = metricConfigPrefix + "reloads_total"
configLastReloadSuccessName = metricConfigPrefix + "last_reload_success"
openConnectionsName = MetricNamePrefix + "open_connections"
// TLS.
metricsTLSPrefix = MetricNamePrefix + "tls_"
tlsCertsNotAfterTimestampName = metricsTLSPrefix + "certs_not_after"
// entry point.
metricEntryPointPrefix = MetricNamePrefix + "entrypoint_"
entryPointReqsTotalName = metricEntryPointPrefix + "requests_total"
entryPointReqsTLSTotalName = metricEntryPointPrefix + "requests_tls_total"
entryPointReqDurationName = metricEntryPointPrefix + "request_duration_seconds"
entryPointReqsBytesTotalName = metricEntryPointPrefix + "requests_bytes_total"
entryPointRespsBytesTotalName = metricEntryPointPrefix + "responses_bytes_total"
// router level.
metricRouterPrefix = MetricNamePrefix + "router_"
routerReqsTotalName = metricRouterPrefix + "requests_total"
routerReqsTLSTotalName = metricRouterPrefix + "requests_tls_total"
routerReqDurationName = metricRouterPrefix + "request_duration_seconds"
routerReqsBytesTotalName = metricRouterPrefix + "requests_bytes_total"
routerRespsBytesTotalName = metricRouterPrefix + "responses_bytes_total"
// service level.
metricServicePrefix = MetricNamePrefix + "service_"
serviceReqsTotalName = metricServicePrefix + "requests_total"
serviceReqsTLSTotalName = metricServicePrefix + "requests_tls_total"
serviceReqDurationName = metricServicePrefix + "request_duration_seconds"
serviceRetriesTotalName = metricServicePrefix + "retries_total"
serviceServerUpName = metricServicePrefix + "server_up"
serviceReqsBytesTotalName = metricServicePrefix + "requests_bytes_total"
serviceRespsBytesTotalName = metricServicePrefix + "responses_bytes_total"
)
// promState holds all metric state internally and acts as the only Collector we register for Prometheus.
//
// This enables control to remove metrics that belong to outdated configuration.
// As an example why this is required, consider Traefik learns about a new service.
// It populates the 'traefik_server_service_up' metric for it with a value of 1 (alive).
// When the service is undeployed now the metric is still there in the client library
// and will be returned on the metrics endpoint until Traefik would be restarted.
//
// To solve this problem promState keeps track of Traefik's dynamic configuration.
// Metrics that "belong" to a dynamic configuration part like services or entryPoints
// are removed after they were scraped at least once when the corresponding object
// doesn't exist anymore.
var promState = newPrometheusState()
var promRegistry = stdprometheus.NewRegistry()
// PrometheusHandler exposes Prometheus routes.
func PrometheusHandler() http.Handler {
return promhttp.HandlerFor(promRegistry, promhttp.HandlerOpts{})
}
// RegisterPrometheus registers all Prometheus metrics.
// It must be called only once and failing to register the metrics will lead to a panic.
func RegisterPrometheus(ctx context.Context, config *types.Prometheus) Registry {
standardRegistry := initStandardRegistry(config)
if err := promRegistry.Register(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{})); err != nil {
var arErr stdprometheus.AlreadyRegisteredError
if !errors.As(err, &arErr) {
log.Ctx(ctx).Warn().Msg("ProcessCollector is already registered")
}
}
if err := promRegistry.Register(collectors.NewGoCollector()); err != nil {
var arErr stdprometheus.AlreadyRegisteredError
if !errors.As(err, &arErr) {
log.Ctx(ctx).Warn().Msg("GoCollector is already registered")
}
}
if !registerPromState(ctx) {
return nil
}
return standardRegistry
}
func initStandardRegistry(config *types.Prometheus) Registry {
buckets := []float64{0.1, 0.3, 1.2, 5.0}
if config.Buckets != nil {
buckets = config.Buckets
}
configReloads := newCounterFrom(stdprometheus.CounterOpts{
Name: configReloadsTotalName,
Help: "Config reloads",
}, []string{})
lastConfigReloadSuccess := newGaugeFrom(stdprometheus.GaugeOpts{
Name: configLastReloadSuccessName,
Help: "Last config reload success",
}, []string{})
tlsCertsNotAfterTimestamp := newGaugeFrom(stdprometheus.GaugeOpts{
Name: tlsCertsNotAfterTimestampName,
Help: "Certificate expiration timestamp",
}, []string{"cn", "serial", "sans"})
openConnections := newGaugeFrom(stdprometheus.GaugeOpts{
Name: openConnectionsName,
Help: "How many open connections exist, by entryPoint and protocol",
}, []string{"entrypoint", "protocol"})
promState.vectors = []vector{
configReloads.cv,
lastConfigReloadSuccess.gv,
tlsCertsNotAfterTimestamp.gv,
openConnections.gv,
}
reg := &standardRegistry{
epEnabled: config.AddEntryPointsLabels,
routerEnabled: config.AddRoutersLabels,
svcEnabled: config.AddServicesLabels,
configReloadsCounter: configReloads,
lastConfigReloadSuccessGauge: lastConfigReloadSuccess,
tlsCertsNotAfterTimestampGauge: tlsCertsNotAfterTimestamp,
openConnectionsGauge: openConnections,
}
if config.AddEntryPointsLabels {
entryPointReqs := newCounterWithHeadersFrom(stdprometheus.CounterOpts{
Name: entryPointReqsTotalName,
Help: "How many HTTP requests processed on an entrypoint, partitioned by status code, protocol, and method.",
}, config.HeaderLabels, []string{"code", "method", "protocol", "entrypoint"})
entryPointReqsTLS := newCounterFrom(stdprometheus.CounterOpts{
Name: entryPointReqsTLSTotalName,
Help: "How many HTTP requests with TLS processed on an entrypoint, partitioned by TLS Version and TLS cipher Used.",
}, []string{"tls_version", "tls_cipher", "entrypoint"})
entryPointReqDurations := newHistogramFrom(stdprometheus.HistogramOpts{
Name: entryPointReqDurationName,
Help: "How long it took to process the request on an entrypoint, partitioned by status code, protocol, and method.",
Buckets: buckets,
}, []string{"code", "method", "protocol", "entrypoint"})
entryPointReqsBytesTotal := newCounterFrom(stdprometheus.CounterOpts{
Name: entryPointReqsBytesTotalName,
Help: "The total size of requests in bytes handled by an entrypoint, partitioned by status code, protocol, and method.",
}, []string{"code", "method", "protocol", "entrypoint"})
entryPointRespsBytesTotal := newCounterFrom(stdprometheus.CounterOpts{
Name: entryPointRespsBytesTotalName,
Help: "The total size of responses in bytes handled by an entrypoint, partitioned by status code, protocol, and method.",
}, []string{"code", "method", "protocol", "entrypoint"})
promState.vectors = append(promState.vectors,
entryPointReqs.cv,
entryPointReqsTLS.cv,
entryPointReqDurations.hv,
entryPointReqsBytesTotal.cv,
entryPointRespsBytesTotal.cv,
)
reg.entryPointReqsCounter = entryPointReqs
reg.entryPointReqsTLSCounter = entryPointReqsTLS
reg.entryPointReqDurationHistogram, _ = NewHistogramWithScale(entryPointReqDurations, time.Second)
reg.entryPointReqsBytesCounter = entryPointReqsBytesTotal
reg.entryPointRespsBytesCounter = entryPointRespsBytesTotal
}
if config.AddRoutersLabels {
routerReqs := newCounterWithHeadersFrom(stdprometheus.CounterOpts{
Name: routerReqsTotalName,
Help: "How many HTTP requests are processed on a router, partitioned by service, status code, protocol, and method.",
}, config.HeaderLabels, []string{"code", "method", "protocol", "router", "service"})
routerReqsTLS := newCounterFrom(stdprometheus.CounterOpts{
Name: routerReqsTLSTotalName,
Help: "How many HTTP requests with TLS are processed on a router, partitioned by service, TLS Version, and TLS cipher Used.",
}, []string{"tls_version", "tls_cipher", "router", "service"})
routerReqDurations := newHistogramFrom(stdprometheus.HistogramOpts{
Name: routerReqDurationName,
Help: "How long it took to process the request on a router, partitioned by service, status code, protocol, and method.",
Buckets: buckets,
}, []string{"code", "method", "protocol", "router", "service"})
routerReqsBytesTotal := newCounterFrom(stdprometheus.CounterOpts{
Name: routerReqsBytesTotalName,
Help: "The total size of requests in bytes handled by a router, partitioned by service, status code, protocol, and method.",
}, []string{"code", "method", "protocol", "router", "service"})
routerRespsBytesTotal := newCounterFrom(stdprometheus.CounterOpts{
Name: routerRespsBytesTotalName,
Help: "The total size of responses in bytes handled by a router, partitioned by service, status code, protocol, and method.",
}, []string{"code", "method", "protocol", "router", "service"})
promState.vectors = append(promState.vectors,
routerReqs.cv,
routerReqsTLS.cv,
routerReqDurations.hv,
routerReqsBytesTotal.cv,
routerRespsBytesTotal.cv,
)
reg.routerReqsCounter = routerReqs
reg.routerReqsTLSCounter = routerReqsTLS
reg.routerReqDurationHistogram, _ = NewHistogramWithScale(routerReqDurations, time.Second)
reg.routerReqsBytesCounter = routerReqsBytesTotal
reg.routerRespsBytesCounter = routerRespsBytesTotal
}
if config.AddServicesLabels {
serviceReqs := newCounterWithHeadersFrom(stdprometheus.CounterOpts{
Name: serviceReqsTotalName,
Help: "How many HTTP requests processed on a service, partitioned by status code, protocol, and method.",
}, config.HeaderLabels, []string{"code", "method", "protocol", "service"})
serviceReqsTLS := newCounterFrom(stdprometheus.CounterOpts{
Name: serviceReqsTLSTotalName,
Help: "How many HTTP requests with TLS processed on a service, partitioned by TLS version and TLS cipher.",
}, []string{"tls_version", "tls_cipher", "service"})
serviceReqDurations := newHistogramFrom(stdprometheus.HistogramOpts{
Name: serviceReqDurationName,
Help: "How long it took to process the request on a service, partitioned by status code, protocol, and method.",
Buckets: buckets,
}, []string{"code", "method", "protocol", "service"})
serviceRetries := newCounterFrom(stdprometheus.CounterOpts{
Name: serviceRetriesTotalName,
Help: "How many request retries happened on a service.",
}, []string{"service"})
serviceServerUp := newGaugeFrom(stdprometheus.GaugeOpts{
Name: serviceServerUpName,
Help: "service server is up, described by gauge value of 0 or 1.",
}, []string{"service", "url"})
serviceReqsBytesTotal := newCounterFrom(stdprometheus.CounterOpts{
Name: serviceReqsBytesTotalName,
Help: "The total size of requests in bytes received by a service, partitioned by status code, protocol, and method.",
}, []string{"code", "method", "protocol", "service"})
serviceRespsBytesTotal := newCounterFrom(stdprometheus.CounterOpts{
Name: serviceRespsBytesTotalName,
Help: "The total size of responses in bytes returned by a service, partitioned by status code, protocol, and method.",
}, []string{"code", "method", "protocol", "service"})
promState.vectors = append(promState.vectors,
serviceReqs.cv,
serviceReqsTLS.cv,
serviceReqDurations.hv,
serviceRetries.cv,
serviceServerUp.gv,
serviceReqsBytesTotal.cv,
serviceRespsBytesTotal.cv,
)
reg.serviceReqsCounter = serviceReqs
reg.serviceReqsTLSCounter = serviceReqsTLS
reg.serviceReqDurationHistogram, _ = NewHistogramWithScale(serviceReqDurations, time.Second)
reg.serviceRetriesCounter = serviceRetries
reg.serviceServerUpGauge = serviceServerUp
reg.serviceReqsBytesCounter = serviceReqsBytesTotal
reg.serviceRespsBytesCounter = serviceRespsBytesTotal
}
return reg
}
func registerPromState(ctx context.Context) bool {
err := promRegistry.Register(promState)
if err == nil {
return true
}
logger := log.Ctx(ctx)
var arErr stdprometheus.AlreadyRegisteredError
if errors.As(err, &arErr) {
logger.Debug().Msg("Prometheus collector already registered.")
return true
}
logger.Error().Err(err).Msg("Unable to register Traefik to Prometheus")
return false
}
// OnConfigurationUpdate receives the current configuration from Traefik.
// It then converts the configuration to the optimized package internal format
// and sets it to the promState.
func OnConfigurationUpdate(conf dynamic.Configuration, entryPoints []string) {
dynCfg := newDynamicConfig()
for _, value := range entryPoints {
dynCfg.entryPoints[value] = true
}
if conf.HTTP == nil {
promState.SetDynamicConfig(dynCfg)
return
}
for name := range conf.HTTP.Routers {
dynCfg.routers[name] = true
}
for serviceName, service := range conf.HTTP.Services {
dynCfg.services[serviceName] = make(map[string]bool)
if service.LoadBalancer != nil {
for _, server := range service.LoadBalancer.Servers {
dynCfg.services[serviceName][server.URL] = true
}
}
}
promState.SetDynamicConfig(dynCfg)
}
func newPrometheusState() *prometheusState {
return &prometheusState{
dynamicConfig: newDynamicConfig(),
deletedURLs: make(map[string][]string),
}
}
type vector interface {
stdprometheus.Collector
DeletePartialMatch(labels stdprometheus.Labels) int
}
type prometheusState struct {
vectors []vector
mtx sync.Mutex
dynamicConfig *dynamicConfig
deletedEP []string
deletedRouters []string
deletedServices []string
deletedURLs map[string][]string
}
func (ps *prometheusState) SetDynamicConfig(dynamicConfig *dynamicConfig) {
ps.mtx.Lock()
defer ps.mtx.Unlock()
for ep := range ps.dynamicConfig.entryPoints {
if _, ok := dynamicConfig.entryPoints[ep]; !ok {
ps.deletedEP = append(ps.deletedEP, ep)
}
}
for router := range ps.dynamicConfig.routers {
if _, ok := dynamicConfig.routers[router]; !ok {
ps.deletedRouters = append(ps.deletedRouters, router)
}
}
for service, serV := range ps.dynamicConfig.services {
actualService, ok := dynamicConfig.services[service]
if !ok {
ps.deletedServices = append(ps.deletedServices, service)
}
for url := range serV {
if _, ok := actualService[url]; !ok {
ps.deletedURLs[service] = append(ps.deletedURLs[service], url)
}
}
}
ps.dynamicConfig = dynamicConfig
}
// Describe implements prometheus.Collector and simply calls
// the registered describer functions.
func (ps *prometheusState) Describe(ch chan<- *stdprometheus.Desc) {
for _, v := range ps.vectors {
v.Describe(ch)
}
}
// Collect implements prometheus.Collector. It calls the Collect
// method of all metrics it received on the collectors channel.
// It's also responsible to remove metrics that belong to an outdated configuration.
// The removal happens only after their Collect method was called to ensure that
// also those metrics will be exported on the current scrape.
func (ps *prometheusState) Collect(ch chan<- stdprometheus.Metric) {
for _, v := range ps.vectors {
v.Collect(ch)
}
ps.mtx.Lock()
defer ps.mtx.Unlock()
for _, ep := range ps.deletedEP {
if !ps.dynamicConfig.hasEntryPoint(ep) {
ps.DeletePartialMatch(map[string]string{"entrypoint": ep})
}
}
for _, router := range ps.deletedRouters {
if !ps.dynamicConfig.hasRouter(router) {
ps.DeletePartialMatch(map[string]string{"router": router})
}
}
for _, service := range ps.deletedServices {
if !ps.dynamicConfig.hasService(service) {
ps.DeletePartialMatch(map[string]string{"service": service})
}
}
for service, urls := range ps.deletedURLs {
for _, url := range urls {
if !ps.dynamicConfig.hasServerURL(service, url) {
ps.DeletePartialMatch(map[string]string{"service": service, "url": url})
}
}
}
ps.deletedEP = nil
ps.deletedRouters = nil
ps.deletedServices = nil
ps.deletedURLs = make(map[string][]string)
}
// DeletePartialMatch deletes all metrics where the variable labels contain all of those passed in as labels.
// The order of the labels does not matter.
// It returns the number of metrics deleted.
func (ps *prometheusState) DeletePartialMatch(labels stdprometheus.Labels) int {
var count int
for _, elem := range ps.vectors {
count += elem.DeletePartialMatch(labels)
}
return count
}
func newDynamicConfig() *dynamicConfig {
return &dynamicConfig{
entryPoints: make(map[string]bool),
routers: make(map[string]bool),
services: make(map[string]map[string]bool),
}
}
// dynamicConfig holds the current configuration for entryPoints, services,
// and server URLs in an optimized way to check for existence. This provides
// a performant way to check whether the collected metrics belong to the
// current configuration or to an outdated one.
type dynamicConfig struct {
entryPoints map[string]bool
routers map[string]bool
services map[string]map[string]bool
}
func (d *dynamicConfig) hasEntryPoint(entrypointName string) bool {
_, ok := d.entryPoints[entrypointName]
return ok
}
func (d *dynamicConfig) hasService(serviceName string) bool {
_, ok := d.services[serviceName]
return ok
}
func (d *dynamicConfig) hasRouter(routerName string) bool {
_, ok := d.routers[routerName]
return ok
}
func (d *dynamicConfig) hasServerURL(serviceName, serverURL string) bool {
if service, hasService := d.services[serviceName]; hasService {
_, ok := service[serverURL]
return ok
}
return false
}
func newCounterWithHeadersFrom(opts stdprometheus.CounterOpts, headers map[string]string, labelNames []string) *counterWithHeaders {
var headerLabels []string
for k := range headers {
headerLabels = append(headerLabels, k)
}
cv := stdprometheus.NewCounterVec(opts, append(labelNames, headerLabels...))
c := &counterWithHeaders{
name: opts.Name,
headers: headers,
cv: cv,
}
if len(labelNames) == 0 && len(headerLabels) == 0 {
c.collector = cv.WithLabelValues()
c.Add(0)
}
return c
}
type counterWithHeaders struct {
name string
cv *stdprometheus.CounterVec
labelNamesValues labelNamesValues
headers map[string]string
collector stdprometheus.Counter
}
func (c *counterWithHeaders) With(headers http.Header, labelValues ...string) CounterWithHeaders {
for headerLabel, headerKey := range c.headers {
labelValues = append(labelValues, headerLabel, headers.Get(headerKey))
}
lnv := c.labelNamesValues.With(labelValues...)
return &counterWithHeaders{
name: c.name,
headers: c.headers,
cv: c.cv,
labelNamesValues: lnv,
collector: c.cv.With(lnv.ToLabels()),
}
}
func (c *counterWithHeaders) Add(delta float64) {
c.collector.Add(delta)
}
func (c *counterWithHeaders) Describe(ch chan<- *stdprometheus.Desc) {
c.cv.Describe(ch)
}
func newCounterFrom(opts stdprometheus.CounterOpts, labelNames []string) *counter {
cv := stdprometheus.NewCounterVec(opts, labelNames)
c := &counter{
name: opts.Name,
cv: cv,
}
if len(labelNames) == 0 {
c.collector = cv.WithLabelValues()
c.Add(0)
}
return c
}
type counter struct {
name string
cv *stdprometheus.CounterVec
labelNamesValues labelNamesValues
collector stdprometheus.Counter
}
func (c *counter) With(labelValues ...string) metrics.Counter {
lnv := c.labelNamesValues.With(labelValues...)
return &counter{
name: c.name,
cv: c.cv,
labelNamesValues: lnv,
collector: c.cv.With(lnv.ToLabels()),
}
}
func (c *counter) Add(delta float64) {
c.collector.Add(delta)
}
func (c *counter) Describe(ch chan<- *stdprometheus.Desc) {
c.cv.Describe(ch)
}
func newGaugeFrom(opts stdprometheus.GaugeOpts, labelNames []string) *gauge {
gv := stdprometheus.NewGaugeVec(opts, labelNames)
g := &gauge{
name: opts.Name,
gv: gv,
}
if len(labelNames) == 0 {
g.collector = gv.WithLabelValues()
g.Set(0)
}
return g
}
type gauge struct {
name string
gv *stdprometheus.GaugeVec
labelNamesValues labelNamesValues
collector stdprometheus.Gauge
}
func (g *gauge) With(labelValues ...string) metrics.Gauge {
lnv := g.labelNamesValues.With(labelValues...)
return &gauge{
name: g.name,
gv: g.gv,
labelNamesValues: lnv,
collector: g.gv.With(lnv.ToLabels()),
}
}
func (g *gauge) Add(delta float64) {
g.collector.Add(delta)
}
func (g *gauge) Set(value float64) {
g.collector.Set(value)
}
func (g *gauge) Describe(ch chan<- *stdprometheus.Desc) {
g.gv.Describe(ch)
}
func newHistogramFrom(opts stdprometheus.HistogramOpts, labelNames []string) *histogram {
hv := stdprometheus.NewHistogramVec(opts, labelNames)
return &histogram{
name: opts.Name,
hv: hv,
}
}
type histogram struct {
name string
hv *stdprometheus.HistogramVec
labelNamesValues labelNamesValues
collector stdprometheus.Observer
}
func (h *histogram) With(labelValues ...string) metrics.Histogram {
lnv := h.labelNamesValues.With(labelValues...)
return &histogram{
name: h.name,
hv: h.hv,
labelNamesValues: lnv,
collector: h.hv.With(lnv.ToLabels()),
}
}
func (h *histogram) Observe(value float64) {
h.collector.Observe(value)
}
func (h *histogram) Describe(ch chan<- *stdprometheus.Desc) {
h.hv.Describe(ch)
}
// labelNamesValues is a type alias that provides validation on its With method.
// Metrics may include it as a member to help them satisfy With semantics and
// save some code duplication.
type labelNamesValues []string
// With validates the input, and returns a new aggregate labelNamesValues.
func (lvs labelNamesValues) With(labelValues ...string) labelNamesValues {
if len(labelValues)%2 != 0 {
labelValues = append(labelValues, "unknown")
}
labels := make([]string, len(lvs)+len(labelValues))
n := copy(labels, lvs)
copy(labels[n:], labelValues)
return labels
}
// ToLabels is a convenience method to convert a labelNamesValues
// to the native prometheus.Labels.
func (lvs labelNamesValues) ToLabels() stdprometheus.Labels {
labels := make(map[string]string, len(lvs)/2)
for i := 0; i < len(lvs); i += 2 {
labels[lvs[i]] = lvs[i+1]
}
return labels
}