baalajimaestro/traefik
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

Fix service up gauge for Prometheus metrics

Browse source 
Co-authored-by: Tom Moulard <tom.moulard@traefik.io>
This commit is contained in:
Julien Salleyron 2022-07-18 10:36:11 +02:00 committed by GitHub
parent b7199a7a9b
commit 2cb011f595
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 56 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
pkg/metrics/prometheus.go
View file

@ -310,7 +310,7 @@ func OnConfigurationUpdate(conf dynamic.Configuration, entryPoints []string) {
func newPrometheusState() *prometheusState {
return &prometheusState{
dynamicConfig: newDynamicConfig(),
deletedURLs: make(map[string]string),
deletedURLs: make(map[string][]string),
}
}
@ -327,7 +327,7 @@ type prometheusState struct {
deletedEP []string
deletedRouters []string
deletedServices []string
deletedURLs map[string]string
deletedURLs map[string][]string
}
func (ps *prometheusState) SetDynamicConfig(dynamicConfig *dynamicConfig) {
@ -350,11 +350,10 @@ func (ps *prometheusState) SetDynamicConfig(dynamicConfig *dynamicConfig) {
actualService, ok := dynamicConfig.services[service]
if !ok {
ps.deletedServices = append(ps.deletedServices, service)
continue
}
for url := range serV {
if _, ok := actualService[url]; !ok {
ps.deletedURLs[service] = url
ps.deletedURLs[service] = append(ps.deletedURLs[service], url)
}
}
}
@ -401,16 +400,18 @@ func (ps *prometheusState) Collect(ch chan<- stdprometheus.Metric) {
}
}
for service, url := range ps.deletedURLs {
if !ps.dynamicConfig.hasServerURL(service, url) {
ps.DeletePartialMatch(map[string]string{"service": service, "url": url})
for service, urls := range ps.deletedURLs {
for _, url := range urls {
if !ps.dynamicConfig.hasServerURL(service, url) {
ps.DeletePartialMatch(map[string]string{"service": service, "url": url})
}
}
}
ps.deletedEP = nil
ps.deletedRouters = nil
ps.deletedServices = nil
ps.deletedURLs = make(map[string]string)
ps.deletedURLs = make(map[string][]string)
}
// DeletePartialMatch deletes all metrics where the variable labels contain all of those passed in as labels.

48
pkg/metrics/prometheus_test.go
View file

@ -364,6 +364,7 @@ func TestPrometheusMetricRemoval(t *testing.T) {
th.WithService("bar@providerName", th.WithServers(
th.WithServer("http://localhost:9000"),
th.WithServer("http://localhost:9999"),
th.WithServer("http://localhost:9998"),
)),
th.WithService("service1", th.WithServers(th.WithServer("http://localhost:9000"))),
),
@ -403,6 +404,10 @@ func TestPrometheusMetricRemoval(t *testing.T) {
ServiceServerUpGauge().
With("service", "bar@providerName", "url", "http://localhost:9999").
Set(1)
prometheusRegistry.
ServiceServerUpGauge().
With("service", "bar@providerName", "url", "http://localhost:9998").
Set(1)
assertMetricsExist(t, mustScrape(), entryPointReqsTotalName, routerReqsTotalName, serviceReqsTotalName, serviceServerUpName)
assertMetricsAbsent(t, mustScrape(), entryPointReqsTotalName, routerReqsTotalName, serviceReqsTotalName, serviceServerUpName)
@ -432,6 +437,47 @@ func TestPrometheusMetricRemoval(t *testing.T) {
assertMetricsExist(t, mustScrape(), entryPointReqsTotalName, serviceReqsTotalName, serviceServerUpName, routerReqsTotalName)
}
func TestPrometheusMetricRemoveEndpointForRecoveredService(t *testing.T) {
promState = newPrometheusState()
promRegistry = prometheus.NewRegistry()
t.Cleanup(promState.reset)
prometheusRegistry := RegisterPrometheus(context.Background(), &types.Prometheus{AddServicesLabels: true})
defer promRegistry.Unregister(promState)
conf1 := dynamic.Configuration{
HTTP: th.BuildConfiguration(
th.WithLoadBalancerServices(
th.WithService("service1", th.WithServers(th.WithServer("http://localhost:9000"))),
),
),
}
conf2 := dynamic.Configuration{
HTTP: th.BuildConfiguration(),
}
conf3 := dynamic.Configuration{
HTTP: th.BuildConfiguration(
th.WithLoadBalancerServices(
th.WithService("service1", th.WithServers(th.WithServer("http://localhost:9001"))),
),
),
}
OnConfigurationUpdate(conf1, []string{})
OnConfigurationUpdate(conf2, []string{})
OnConfigurationUpdate(conf3, []string{})
prometheusRegistry.
ServiceServerUpGauge().
With("service", "service1", "url", "http://localhost:9000").
Add(1)
assertMetricsExist(t, mustScrape(), serviceServerUpName)
assertMetricsAbsent(t, mustScrape(), serviceServerUpName)
}
func TestPrometheusRemovedMetricsReset(t *testing.T) {
t.Cleanup(promState.reset)
@ -488,7 +534,7 @@ func (ps *prometheusState) reset() {
ps.deletedEP = nil
ps.deletedRouters = nil
ps.deletedServices = nil
ps.deletedURLs = make(map[string]string)
ps.deletedURLs = make(map[string][]string)
}
// Tracking and gathering the metrics happens concurrently.