From 175659a3dde13303bedbc08670c7340e2a9ad3a3 Mon Sep 17 00:00:00 2001 From: enxebre Date: Thu, 12 Jan 2017 14:34:54 +0100 Subject: [PATCH] Support for Metrics and Prometheus. --- configuration.go | 7 ++++ docs/toml.md | 5 +++ glide.lock | 36 +++++++++++++++++-- glide.yaml | 6 +++- middlewares/metrics.go | 51 ++++++++++++++++++++++++++ middlewares/prometheus.go | 65 ++++++++++++++++++++++++++++++++++ middlewares/prometheus_test.go | 47 ++++++++++++++++++++++++ server.go | 13 +++++++ traefik.go | 1 + types/types.go | 46 ++++++++++++++++++++++-- web.go | 7 ++++ 11 files changed, 279 insertions(+), 5 deletions(-) create mode 100644 middlewares/metrics.go create mode 100644 middlewares/prometheus.go create mode 100644 middlewares/prometheus_test.go diff --git a/configuration.go b/configuration.go index d8cc2dd96..47410121d 100644 --- a/configuration.go +++ b/configuration.go @@ -328,6 +328,13 @@ func NewTraefikDefaultPointersConfiguration() *TraefikConfiguration { RecentErrors: 10, } + // default Metrics + defaultWeb.Metrics = &types.Metrics{ + Prometheus: &types.Prometheus{ + Buckets: types.Buckets{100, 300, 1200, 5000}, + }, + } + // default Marathon var defaultMarathon provider.Marathon defaultMarathon.Watch = true diff --git a/docs/toml.md b/docs/toml.md index 79293ad82..48ee7a9ce 100644 --- a/docs/toml.md +++ b/docs/toml.md @@ -715,6 +715,11 @@ $ curl -s "http://localhost:8080/api" | jq . - `/api/providers/{provider}/frontends/{frontend}/routes`: `GET` routes in a frontend - `/api/providers/{provider}/frontends/{frontend}/routes/{route}`: `GET` a route in a frontend +- `/metrics`: You can enable Traefik to export internal metrics to different monitoring systems (Only Prometheus is supported at the moment). + +```bash +$ traefik --web.metrics.prometheus --web.metrics.prometheus.buckets="100,300" +``` ## Docker backend diff --git a/glide.lock b/glide.lock index 79a754d10..7ac67a952 100644 --- a/glide.lock +++ b/glide.lock @@ -1,6 +1,8 @@ -hash: b45763571b0e62d44908c8c5088d9d596831f70cd3534b4ed7f8b7f62e37a38e -updated: 2016-12-29T23:00:30.216963153-02:00 +hash: 0d092f94db69882e79d229c34b9483899e1208eaa7dd0acdd5184635cb0cdaaa +updated: 2017-01-12T12:31:31.35220213+01:00 imports: +- name: bitbucket.org/ww/goautoneg + version: 75cd24fc2f2c2a2088577d12123ddee5f54e0675 - name: github.com/abbot/go-http-auth version: cb4372376e1e00e9f6ab9ec142e029302c9e7140 - name: github.com/ArthurHlt/go-eureka-client @@ -46,6 +48,10 @@ imports: - autorest/date - autorest/to - autorest/validation +- name: github.com/beorn7/perks + version: b965b613227fddccbfffe13eae360ed3fa822f8d + subpackages: + - quantile - name: github.com/blang/semver version: 3a37c301dda64cbe17f16f661b4c976803c0e2d2 - name: github.com/boltdb/bolt @@ -228,6 +234,12 @@ imports: version: 04f313413ffd65ce25f2541bfd2b2ceec5c0908c - name: github.com/go-ini/ini version: 6f66b0e091edb3c7b380f7c4f0f884274d550b67 +- name: github.com/go-kit/kit + version: f66b0e13579bfc5a48b9e2a94b1209c107ea1f41 + subpackages: + - metrics + - metrics/internal/lv + - metrics/prometheus - name: github.com/go-openapi/jsonpointer version: 8d96a2dc61536b690bd36b2e9df0b3c0b62825b2 - name: github.com/go-openapi/jsonreference @@ -296,6 +308,10 @@ imports: - jwriter - name: github.com/mattn/go-shellwords version: 525bedee691b5a8df547cb5cf9f86b7fb1883e24 +- name: github.com/matttproud/golang_protobuf_extensions + version: fc2b8d3a73c4867e51861bbdd5ae3c1f0869dd6a + subpackages: + - pbutil - name: github.com/mesos/mesos-go version: 068d5470506e3780189fe607af40892814197c5e subpackages: @@ -346,6 +362,22 @@ imports: version: d8ed2627bdf02c080bf22230dbb337003b7aba2d subpackages: - difflib +- name: github.com/prometheus/client_golang + version: c5b7fccd204277076155f10851dad72b76a49317 + subpackages: + - prometheus + - prometheus/promhttp +- name: github.com/prometheus/client_model + version: fa8ad6fec33561be4280a8f0514318c79d7f6cb6 + subpackages: + - go +- name: github.com/prometheus/common + version: ffe929a3f4c4faeaa10f2b9535c2b1be3ad15650 + subpackages: + - expfmt + - model +- name: github.com/prometheus/procfs + version: 454a56f35412459b5e684fd5ec0f9211b94f002a - name: github.com/PuerkitoBio/purell version: 0bcb03f4b4d0a9428594752bd2a3b9aa0a9d4bd4 - name: github.com/PuerkitoBio/urlesc diff --git a/glide.yaml b/glide.yaml index 89467a5ab..a185a2665 100644 --- a/glide.yaml +++ b/glide.yaml @@ -62,7 +62,7 @@ import: subpackages: - plugin/rewrite - package: github.com/xenolf/lego - version: ce8fb060cb8361a9ff8b5fb7c2347fa907b6fcac + version: ce8fb060cb8361a9ff8b5fb7c2347fa907b6fcac subpackages: - acme - package: golang.org/x/net @@ -114,3 +114,7 @@ import: - package: github.com/google/go-github - package: github.com/hashicorp/go-version - package: github.com/mvdan/xurls +- package: github.com/go-kit/kit + version: v0.3.0 + subpackages: + - metrics diff --git a/middlewares/metrics.go b/middlewares/metrics.go new file mode 100644 index 000000000..422774a28 --- /dev/null +++ b/middlewares/metrics.go @@ -0,0 +1,51 @@ +package middlewares + +import ( + "github.com/go-kit/kit/metrics" + "net/http" + "strconv" + "time" +) + +// Metrics is an Interface that must be satisfied by any system that +// wants to expose and monitor metrics +type Metrics interface { + getReqsCounter() metrics.Counter + getLatencyHistogram() metrics.Histogram + handler() http.Handler +} + +// MetricsWrapper is a Negroni compatible Handler which relies on a +// given Metrics implementation to expose and monitor Traefik metrics +type MetricsWrapper struct { + Impl Metrics +} + +// NewMetricsWrapper return a MetricsWrapper struct with +// a given Metrics implementation e.g Prometheuss +func NewMetricsWrapper(impl Metrics) *MetricsWrapper { + var metricsWrapper = MetricsWrapper{ + Impl: impl, + } + + return &metricsWrapper +} + +func (m *MetricsWrapper) ServeHTTP(rw http.ResponseWriter, r *http.Request, next http.HandlerFunc) { + start := time.Now() + prw := &responseRecorder{rw, http.StatusOK} + next(prw, r) + labels := []string{"code", strconv.Itoa(prw.StatusCode()), "method", r.Method} + m.Impl.getReqsCounter().With(labels...).Add(1) + m.Impl.getLatencyHistogram().With(labels...).Observe(float64(time.Since(start).Nanoseconds()) / 1000000) +} + +func (rw *responseRecorder) StatusCode() int { + return rw.statusCode +} + +// Handler is the chance for the Metrics implementation +// to expose its metrics on a server endpoint +func (m *MetricsWrapper) Handler() http.Handler { + return m.Impl.handler() +} diff --git a/middlewares/prometheus.go b/middlewares/prometheus.go new file mode 100644 index 000000000..661e54918 --- /dev/null +++ b/middlewares/prometheus.go @@ -0,0 +1,65 @@ +package middlewares + +import ( + "github.com/containous/traefik/types" + "github.com/go-kit/kit/metrics" + "github.com/go-kit/kit/metrics/prometheus" + stdprometheus "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + "net/http" +) + +const ( + reqsName = "requests_total" + latencyName = "request_duration_milliseconds" +) + +// Prometheus is an Implementation for Metrics that exposes prometheus metrics for the number of requests, +// the latency and the response size, partitioned by status code and method. +type Prometheus struct { + reqsCounter metrics.Counter + latencyHistogram metrics.Histogram +} + +func (p *Prometheus) getReqsCounter() metrics.Counter { + return p.reqsCounter +} + +func (p *Prometheus) getLatencyHistogram() metrics.Histogram { + return p.latencyHistogram +} + +// NewPrometheus returns a new prometheus Metrics implementation. +func NewPrometheus(name string, config *types.Prometheus) *Prometheus { + var m Prometheus + m.reqsCounter = prometheus.NewCounterFrom( + stdprometheus.CounterOpts{ + Name: reqsName, + Help: "How many HTTP requests processed, partitioned by status code and method.", + ConstLabels: stdprometheus.Labels{"service": name}, + }, + []string{"code", "method"}, + ) + + var buckets []float64 + if config.Buckets != nil { + buckets = config.Buckets + } else { + buckets = []float64{100, 300, 1200, 5000} + } + + m.latencyHistogram = prometheus.NewHistogramFrom( + stdprometheus.HistogramOpts{ + Name: latencyName, + Help: "How long it took to process the request, partitioned by status code and method.", + ConstLabels: stdprometheus.Labels{"service": name}, + Buckets: buckets, + }, + []string{"code", "method"}, + ) + return &m +} + +func (p *Prometheus) handler() http.Handler { + return promhttp.Handler() +} diff --git a/middlewares/prometheus_test.go b/middlewares/prometheus_test.go new file mode 100644 index 000000000..8315b1756 --- /dev/null +++ b/middlewares/prometheus_test.go @@ -0,0 +1,47 @@ +package middlewares + +import ( + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/codegangsta/negroni" + "github.com/containous/traefik/types" + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +func TestPrometheus(t *testing.T) { + recorder := httptest.NewRecorder() + + n := negroni.New() + metricsMiddlewareBackend := NewMetricsWrapper(NewPrometheus("test", &types.Prometheus{})) + n.Use(metricsMiddlewareBackend) + r := http.NewServeMux() + r.Handle("/metrics", promhttp.Handler()) + r.HandleFunc(`/ok`, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + fmt.Fprintln(w, "ok") + }) + n.UseHandler(r) + + req1, err := http.NewRequest("GET", "http://localhost:3000/ok", nil) + if err != nil { + t.Error(err) + } + req2, err := http.NewRequest("GET", "http://localhost:3000/metrics", nil) + if err != nil { + t.Error(err) + } + + n.ServeHTTP(recorder, req1) + n.ServeHTTP(recorder, req2) + body := recorder.Body.String() + if !strings.Contains(body, reqsName) { + t.Errorf("body does not contain request total entry '%s'", reqsName) + } + if !strings.Contains(body, latencyName) { + t.Errorf("body does not contain request duration entry '%s'", reqsName) + } +} diff --git a/server.go b/server.go index 9099e9b74..ca1dc48c3 100644 --- a/server.go +++ b/server.go @@ -167,8 +167,15 @@ func (server *Server) stopLeadership() { func (server *Server) startHTTPServers() { server.serverEntryPoints = server.buildEntryPoints(server.globalConfiguration) + for newServerEntryPointName, newServerEntryPoint := range server.serverEntryPoints { serverMiddlewares := []negroni.Handler{server.loggerMiddleware, metrics} + if server.globalConfiguration.Web != nil && server.globalConfiguration.Web.Metrics != nil { + if server.globalConfiguration.Web.Metrics.Prometheus != nil { + metricsMiddleware := middlewares.NewMetricsWrapper(middlewares.NewPrometheus("Global", server.globalConfiguration.Web.Metrics.Prometheus)) + serverMiddlewares = append(serverMiddlewares, metricsMiddleware) + } + } if server.globalConfiguration.Web != nil && server.globalConfiguration.Web.Statistics != nil { statsRecorder = middlewares.NewStatsRecorder(server.globalConfiguration.Web.Statistics.RecentErrors) serverMiddlewares = append(serverMiddlewares, statsRecorder) @@ -691,6 +698,12 @@ func (server *Server) loadConfig(configurations configs, globalConfiguration Glo } var negroni = negroni.New() + if server.globalConfiguration.Web != nil && server.globalConfiguration.Web.Metrics != nil { + if server.globalConfiguration.Web.Metrics.Prometheus != nil { + metricsMiddlewareBackend := middlewares.NewMetricsWrapper(middlewares.NewPrometheus(frontend.Backend, server.globalConfiguration.Web.Metrics.Prometheus)) + negroni.Use(metricsMiddlewareBackend) + } + } if configuration.Backends[frontend.Backend].CircuitBreaker != nil { log.Debugf("Creating circuit breaker %s", configuration.Backends[frontend.Backend].CircuitBreaker.Expression) cbreaker, err := middlewares.NewCircuitBreaker(lb, configuration.Backends[frontend.Backend].CircuitBreaker.Expression, cbreaker.Logger(oxyLogger)) diff --git a/traefik.go b/traefik.go index 1e38c3363..12eeb8fa7 100644 --- a/traefik.go +++ b/traefik.go @@ -106,6 +106,7 @@ Complete documentation is available at https://traefik.io`, f.AddParser(reflect.TypeOf(types.Constraints{}), &types.Constraints{}) f.AddParser(reflect.TypeOf(k8s.Namespaces{}), &k8s.Namespaces{}) f.AddParser(reflect.TypeOf([]acme.Domain{}), &acme.Domains{}) + f.AddParser(reflect.TypeOf(types.Buckets{}), &types.Buckets{}) //add commands f.AddCommand(cmd.NewVersionCmd()) diff --git a/types/types.go b/types/types.go index da62e402b..b2d764fd5 100644 --- a/types/types.go +++ b/types/types.go @@ -4,10 +4,10 @@ import ( "encoding" "errors" "fmt" - "strings" - "github.com/docker/libkv/store" "github.com/ryanuber/go-glob" + "strconv" + "strings" ) // Backend holds backend configuration. @@ -250,3 +250,45 @@ func CanonicalDomain(domain string) string { type Statistics struct { RecentErrors int `description:"Number of recent errors logged"` } + +// Metrics provides options to expose and send Traefik metrics to different third party monitoring systems +type Metrics struct { + Prometheus *Prometheus `description:"Prometheus metrics exporter type"` +} + +// Prometheus can contain specific configuration used by the Prometheus Metrics exporter +type Prometheus struct { + Buckets Buckets `description:"Buckets for latency metrics"` +} + +// Buckets holds Prometheus Buckets +type Buckets []float64 + +//Set adds strings elem into the the parser +//it splits str on "," and ";" and apply ParseFloat to string +func (b *Buckets) Set(str string) error { + fargs := func(c rune) bool { + return c == ',' || c == ';' + } + // get function + slice := strings.FieldsFunc(str, fargs) + for _, bucket := range slice { + bu, err := strconv.ParseFloat(bucket, 64) + if err != nil { + return err + } + *b = append(*b, bu) + } + return nil +} + +//Get []float64 +func (b *Buckets) Get() interface{} { return Buckets(*b) } + +//String return slice in a string +func (b *Buckets) String() string { return fmt.Sprintf("%v", *b) } + +//SetValue sets []float64 into the parser +func (b *Buckets) SetValue(val interface{}) { + *b = Buckets(val.(Buckets)) +} diff --git a/web.go b/web.go index b53e72472..429911438 100644 --- a/web.go +++ b/web.go @@ -17,6 +17,7 @@ import ( "github.com/containous/traefik/types" "github.com/containous/traefik/version" "github.com/elazarl/go-bindata-assetfs" + "github.com/prometheus/client_golang/prometheus/promhttp" thoas_stats "github.com/thoas/stats" "github.com/unrolled/render" ) @@ -34,6 +35,7 @@ type WebProvider struct { KeyFile string `description:"SSL certificate"` ReadOnly bool `description:"Enable read only API"` Statistics *types.Statistics `description:"Enable more detailed statistics"` + Metrics *types.Metrics `description:"Enable a metrics exporter"` server *Server Auth *types.Auth } @@ -58,6 +60,11 @@ func (provider *WebProvider) Provide(configurationChan chan<- types.ConfigMessag systemRouter := mux.NewRouter() + // Prometheus route + if provider.Metrics != nil && provider.Metrics.Prometheus != nil { + systemRouter.Methods("GET").Path("/metrics").Handler(promhttp.Handler()) + } + // health route systemRouter.Methods("GET").Path("/health").HandlerFunc(provider.getHealthHandler)