From 69c628b626a01e40dda33368534e1b8a59791bda Mon Sep 17 00:00:00 2001 From: Alex Antonov Date: Thu, 20 Jul 2017 17:26:43 -0500 Subject: [PATCH] DataDog and StatsD Metrics Support * Added support for DataDog and StatsD monitoring * Added documentation --- docs/toml.md | 10 + glide.lock | 22 +- glide.yaml | 9 +- middlewares/datadog.go | 92 +++++ middlewares/datadog_test.go | 53 +++ middlewares/metrics.go | 47 +++ middlewares/statsd.go | 85 +++++ middlewares/statsd_test.go | 52 +++ server/configuration.go | 8 + server/server.go | 48 ++- server/server_test.go | 4 +- types/types.go | 14 + vendor/github.com/go-kit/kit/log/doc.go | 93 +++++ .../github.com/go-kit/kit/log/json_logger.go | 92 +++++ vendor/github.com/go-kit/kit/log/log.go | 144 ++++++++ .../go-kit/kit/log/logfmt_logger.go | 62 ++++ .../github.com/go-kit/kit/log/nop_logger.go | 8 + vendor/github.com/go-kit/kit/log/stdlib.go | 116 +++++++ vendor/github.com/go-kit/kit/log/sync.go | 81 +++++ vendor/github.com/go-kit/kit/log/value.go | 62 ++++ .../go-kit/kit/metrics/dogstatsd/dogstatsd.go | 306 +++++++++++++++++ .../kit/metrics/internal/ratemap/ratemap.go | 40 +++ .../go-kit/kit/metrics/multi/multi.go | 79 +++++ .../go-kit/kit/metrics/statsd/statsd.go | 232 +++++++++++++ vendor/github.com/go-kit/kit/util/conn/doc.go | 2 + .../go-kit/kit/util/conn/manager.go | 145 ++++++++ vendor/github.com/go-logfmt/logfmt/LICENSE | 22 ++ vendor/github.com/go-logfmt/logfmt/decode.go | 237 +++++++++++++ vendor/github.com/go-logfmt/logfmt/doc.go | 6 + vendor/github.com/go-logfmt/logfmt/encode.go | 321 +++++++++++++++++ vendor/github.com/go-logfmt/logfmt/fuzz.go | 126 +++++++ .../github.com/go-logfmt/logfmt/jsonstring.go | 277 +++++++++++++++ vendor/github.com/go-stack/stack/LICENSE.md | 21 ++ vendor/github.com/go-stack/stack/stack.go | 322 ++++++++++++++++++ vendor/github.com/kr/logfmt/decode.go | 184 ++++++++++ vendor/github.com/kr/logfmt/scanner.go | 149 ++++++++ vendor/github.com/kr/logfmt/unquote.go | 149 ++++++++ vendor/github.com/stvp/go-udp-testing/LICENSE | 22 ++ vendor/github.com/stvp/go-udp-testing/udp.go | 192 +++++++++++ 39 files changed, 3921 insertions(+), 13 deletions(-) create mode 100644 middlewares/datadog.go create mode 100644 middlewares/datadog_test.go create mode 100644 middlewares/statsd.go create mode 100644 middlewares/statsd_test.go create mode 100644 vendor/github.com/go-kit/kit/log/doc.go create mode 100644 vendor/github.com/go-kit/kit/log/json_logger.go create mode 100644 vendor/github.com/go-kit/kit/log/log.go create mode 100644 vendor/github.com/go-kit/kit/log/logfmt_logger.go create mode 100644 vendor/github.com/go-kit/kit/log/nop_logger.go create mode 100644 vendor/github.com/go-kit/kit/log/stdlib.go create mode 100644 vendor/github.com/go-kit/kit/log/sync.go create mode 100644 vendor/github.com/go-kit/kit/log/value.go create mode 100644 vendor/github.com/go-kit/kit/metrics/dogstatsd/dogstatsd.go create mode 100644 vendor/github.com/go-kit/kit/metrics/internal/ratemap/ratemap.go create mode 100644 vendor/github.com/go-kit/kit/metrics/multi/multi.go create mode 100644 vendor/github.com/go-kit/kit/metrics/statsd/statsd.go create mode 100644 vendor/github.com/go-kit/kit/util/conn/doc.go create mode 100644 vendor/github.com/go-kit/kit/util/conn/manager.go create mode 100644 vendor/github.com/go-logfmt/logfmt/LICENSE create mode 100644 vendor/github.com/go-logfmt/logfmt/decode.go create mode 100644 vendor/github.com/go-logfmt/logfmt/doc.go create mode 100644 vendor/github.com/go-logfmt/logfmt/encode.go create mode 100644 vendor/github.com/go-logfmt/logfmt/fuzz.go create mode 100644 vendor/github.com/go-logfmt/logfmt/jsonstring.go create mode 100644 vendor/github.com/go-stack/stack/LICENSE.md create mode 100644 vendor/github.com/go-stack/stack/stack.go create mode 100644 vendor/github.com/kr/logfmt/decode.go create mode 100644 vendor/github.com/kr/logfmt/scanner.go create mode 100644 vendor/github.com/kr/logfmt/unquote.go create mode 100644 vendor/github.com/stvp/go-udp-testing/LICENSE create mode 100644 vendor/github.com/stvp/go-udp-testing/udp.go diff --git a/docs/toml.md b/docs/toml.md index 6c75022a7..7b53c4599 100644 --- a/docs/toml.md +++ b/docs/toml.md @@ -648,6 +648,16 @@ address = ":8080" # [web.metrics.prometheus] # Buckets=[0.1,0.3,1.2,5.0] # +# To enable Traefik to export internal metics to DataDog +# [web.metrics.datadog] +# Address = localhost:8125 +# PushInterval = "10s" +# +# To enable Traefik to export internal metics to StatsD +# [web.metrics.statsd] +# Address = localhost:8125 +# PushInterval = "10s" +# # To enable basic auth on the webui # with 2 user/pass: test:test and test2:test2 # Passwords can be encoded in MD5, SHA1 and BCrypt: you can use htpasswd to generate those ones diff --git a/glide.lock b/glide.lock index c62bff49b..5cc9eff1a 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ -hash: 6aff4c6177ddc3247530d141a93f5bb044ee72acaa63b5667ceb205828c8ad03 -updated: 2017-07-11T23:50:31.241672481+02:00 +hash: 4d24f4a986de7e07c32b63abc3c8bf365d205df0a6f65ba4a6ca3d7ac7ae2256 +updated: 2017-07-20T23:54:09.638352893+02:00 imports: - name: cloud.google.com/go version: 2e6a95edb1071d750f6d7db777bf66cd2997af6c @@ -82,8 +82,6 @@ imports: version: 9208b142303c12d8899bae836fd524ac9338b4fd - name: github.com/codegangsta/cli version: bf4a526f48af7badd25d2cb02d587e1b01be3b50 -- name: github.com/urfave/negroni - version: 490e6a555d47ca891a89a150d0c1ef3922dfffe9 - name: github.com/containous/flaeg version: b5d2dc5878df07c2d74413348186982e7b865871 - name: github.com/containous/mux @@ -242,9 +240,17 @@ imports: - name: github.com/go-kit/kit version: f66b0e13579bfc5a48b9e2a94b1209c107ea1f41 subpackages: + - log - metrics + - metrics/dogstatsd - metrics/internal/lv + - metrics/internal/ratemap + - metrics/multi - metrics/prometheus + - metrics/statsd + - util/conn +- name: github.com/go-logfmt/logfmt + version: 390ab7935ee28ec6b286364bba9b4dd6410cb3d5 - name: github.com/go-openapi/jsonpointer version: 46af16f9f7b149af66e5d1bd010e3574dc06de98 - name: github.com/go-openapi/jsonreference @@ -253,6 +259,8 @@ imports: version: 6aced65f8501fe1217321abf0749d354824ba2ff - name: github.com/go-openapi/swag version: 1d0bd113de87027671077d3c71eb3ac5d7dbba72 +- name: github.com/go-stack/stack + version: 54be5f394ed2c3e19dac9134a40a95ba5a017f7b - name: github.com/gogo/protobuf version: 909568be09de550ed094403c2bf8a261b5bb730a subpackages: @@ -302,6 +310,8 @@ imports: version: 72f9bd7c4e0c2a40055ab3d0f09654f730cce982 - name: github.com/juju/ratelimit version: 77ed1c8a01217656d2080ad51981f6e99adaa177 +- name: github.com/kr/logfmt + version: b84e30acd515aadc4b783ad4ff83aff3299bdfe0 - name: github.com/mailgun/timetools version: 7e6055773c5137efbeb3bd2410d705fe10ab6bfd - name: github.com/mailru/easyjson @@ -436,6 +446,8 @@ imports: - assert - mock - require +- name: github.com/stvp/go-udp-testing + version: 06eb4f886d9f8242b0c176cf0d3ce5ec2cedda05 - name: github.com/thoas/stats version: 152b5d051953fdb6e45f14b6826962aadc032324 - name: github.com/timewasted/linode @@ -452,6 +464,8 @@ imports: version: 50716a0a853771bb36bfce61a45cdefdb98c2e6e - name: github.com/unrolled/secure version: 824e85271811af89640ea25620c67f6c2eed987e +- name: github.com/urfave/negroni + version: 490e6a555d47ca891a89a150d0c1ef3922dfffe9 - name: github.com/vulcand/oxy version: 7da864c1d53bd58165435bb78bbf8c01f01c8f4a repo: https://github.com/containous/oxy.git diff --git a/glide.yaml b/glide.yaml index 96df81cf1..28cca2ebf 100644 --- a/glide.yaml +++ b/glide.yaml @@ -22,7 +22,7 @@ import: - roundrobin - stream - utils -- name: github.com/urfave/negroni +- package: github.com/urfave/negroni version: 490e6a555d47ca891a89a150d0c1ef3922dfffe9 - package: github.com/containous/staert version: 1e26a71803e428fd933f5f9c8e50a26878f53147 @@ -99,7 +99,13 @@ import: - package: github.com/go-kit/kit version: v0.3.0 subpackages: + - log - metrics + - metrics/dogstatsd + - metrics/multi + - metrics/prometheus + - metrics/statsd + - util/conn - package: github.com/prometheus/client_golang version: 08fd2e12372a66e68e30523c7642e0cbc3e4fbde subpackages: @@ -190,6 +196,7 @@ import: subpackages: - spew testImport: +- package: github.com/stvp/go-udp-testing - package: github.com/docker/libcompose version: 0ad950cbeb3d72107613dd220b5e9d7e001b890b - package: github.com/go-check/check diff --git a/middlewares/datadog.go b/middlewares/datadog.go new file mode 100644 index 000000000..e8c003676 --- /dev/null +++ b/middlewares/datadog.go @@ -0,0 +1,92 @@ +package middlewares + +import ( + "time" + + "github.com/containous/traefik/log" + "github.com/containous/traefik/safe" + "github.com/containous/traefik/types" + kitlog "github.com/go-kit/kit/log" + "github.com/go-kit/kit/metrics" + "github.com/go-kit/kit/metrics/dogstatsd" +) + +var _ Metrics = (Metrics)(nil) + +var datadogClient = dogstatsd.New("traefik.", kitlog.LoggerFunc(func(keyvals ...interface{}) error { + log.Info(keyvals) + return nil +})) + +var datadogTicker *time.Ticker + +// Metric names consistent with https://github.com/DataDog/integrations-extras/pull/64 +const ( + ddMetricsReqsName = "requests.total" + ddMetricsLatencyName = "request.duration" +) + +// Datadog is an Implementation for Metrics that exposes datadog metrics for the latency +// and the number of requests partitioned by status code and method. +// - number of requests partitioned by status code and method +// - request durations +// - amount of retries happened +type Datadog struct { + reqsCounter metrics.Counter + reqDurationHistogram metrics.Histogram + retryCounter metrics.Counter +} + +func (dd *Datadog) getReqsCounter() metrics.Counter { + return dd.reqsCounter +} + +func (dd *Datadog) getReqDurationHistogram() metrics.Histogram { + return dd.reqDurationHistogram +} + +func (dd *Datadog) getRetryCounter() metrics.Counter { + return dd.retryCounter +} + +// NewDataDog creates new instance of Datadog +func NewDataDog(name string) *Datadog { + var m Datadog + + m.reqsCounter = datadogClient.NewCounter(ddMetricsReqsName, 1.0).With("service", name) + m.reqDurationHistogram = datadogClient.NewHistogram(ddMetricsLatencyName, 1.0).With("service", name) + + return &m +} + +// InitDatadogClient initializes metrics pusher and creates a datadogClient if not created already +func InitDatadogClient(config *types.Datadog) *time.Ticker { + if datadogTicker == nil { + address := config.Address + if len(address) == 0 { + address = "localhost:8125" + } + pushInterval, err := time.ParseDuration(config.PushInterval) + if err != nil { + log.Warnf("Unable to parse %s into pushInterval, using 10s as default value", config.PushInterval) + pushInterval = 10 * time.Second + } + + report := time.NewTicker(pushInterval) + + safe.Go(func() { + datadogClient.SendLoop(report.C, "udp", address) + }) + + datadogTicker = report + } + return datadogTicker +} + +// StopDatadogClient stops internal datadogTicker which controls the pushing of metrics to DD Agent and resets it to `nil` +func StopDatadogClient() { + if datadogTicker != nil { + datadogTicker.Stop() + } + datadogTicker = nil +} diff --git a/middlewares/datadog_test.go b/middlewares/datadog_test.go new file mode 100644 index 000000000..3a95c0d3a --- /dev/null +++ b/middlewares/datadog_test.go @@ -0,0 +1,53 @@ +package middlewares + +import ( + "fmt" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/containous/traefik/testhelpers" + "github.com/containous/traefik/types" + "github.com/stvp/go-udp-testing" + "github.com/urfave/negroni" +) + +func TestDatadog(t *testing.T) { + udp.SetAddr(":18125") + // This is needed to make sure that UDP Listener listens for data a bit longer, otherwise it will quit after a millisecond + udp.Timeout = 5 * time.Second + recorder := httptest.NewRecorder() + InitDatadogClient(&types.Datadog{":18125", "1s"}) + + n := negroni.New() + dd := NewDataDog("test") + defer StopDatadogClient() + metricsMiddlewareBackend := NewMetricsWrapper(dd) + + n.Use(metricsMiddlewareBackend) + r := http.NewServeMux() + r.HandleFunc(`/ok`, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + fmt.Fprintln(w, "ok") + }) + r.HandleFunc(`/not-found`, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + fmt.Fprintln(w, "not-found") + }) + n.UseHandler(r) + + req1 := testhelpers.MustNewRequest(http.MethodGet, "http://localhost:3000/ok", nil) + req2 := testhelpers.MustNewRequest(http.MethodGet, "http://localhost:3000/not-found", nil) + + expected := []string{ + // We are only validating counts, as it is nearly impossible to validate latency, since it varies every run + "traefik.requests.total:1.000000|c|#service:test,code:404,method:GET\n", + "traefik.requests.total:1.000000|c|#service:test,code:200,method:GET\n", + } + + udp.ShouldReceiveAll(t, expected, func() { + n.ServeHTTP(recorder, req1) + n.ServeHTTP(recorder, req2) + }) +} diff --git a/middlewares/metrics.go b/middlewares/metrics.go index 889c8af3e..b5f459df7 100644 --- a/middlewares/metrics.go +++ b/middlewares/metrics.go @@ -6,6 +6,7 @@ import ( "time" "github.com/go-kit/kit/metrics" + "github.com/go-kit/kit/metrics/multi" ) // Metrics is an Interface that must be satisfied by any system that @@ -22,6 +23,52 @@ type RetryMetrics interface { getRetryCounter() metrics.Counter } +// MultiMetrics is a struct that provides a wrapper container for multiple Metrics, if they are configured +type MultiMetrics struct { + wrappedMetrics *[]Metrics + reqsCounter metrics.Counter + reqDurationHistogram metrics.Histogram + retryCounter metrics.Counter +} + +// NewMultiMetrics creates a new instance of MultiMetrics +func NewMultiMetrics(manyMetrics []Metrics) *MultiMetrics { + counters := []metrics.Counter{} + histograms := []metrics.Histogram{} + retryCounters := []metrics.Counter{} + + for _, m := range manyMetrics { + counters = append(counters, m.getReqsCounter()) + histograms = append(histograms, m.getReqDurationHistogram()) + retryCounters = append(retryCounters, m.getRetryCounter()) + } + + var mm MultiMetrics + + mm.wrappedMetrics = &manyMetrics + mm.reqsCounter = multi.NewCounter(counters...) + mm.reqDurationHistogram = multi.NewHistogram(histograms...) + mm.retryCounter = multi.NewCounter(retryCounters...) + + return &mm +} + +func (mm *MultiMetrics) getReqsCounter() metrics.Counter { + return mm.reqsCounter +} + +func (mm *MultiMetrics) getReqDurationHistogram() metrics.Histogram { + return mm.reqDurationHistogram +} + +func (mm *MultiMetrics) getRetryCounter() metrics.Counter { + return mm.retryCounter +} + +func (mm *MultiMetrics) getWrappedMetrics() *[]Metrics { + return mm.wrappedMetrics +} + // MetricsWrapper is a Negroni compatible Handler which relies on a // given Metrics implementation to expose and monitor Traefik Metrics. type MetricsWrapper struct { diff --git a/middlewares/statsd.go b/middlewares/statsd.go new file mode 100644 index 000000000..4d8a9935e --- /dev/null +++ b/middlewares/statsd.go @@ -0,0 +1,85 @@ +package middlewares + +import ( + "time" + + "github.com/containous/traefik/log" + "github.com/containous/traefik/safe" + "github.com/containous/traefik/types" + kitlog "github.com/go-kit/kit/log" + "github.com/go-kit/kit/metrics" + "github.com/go-kit/kit/metrics/statsd" +) + +var _ Metrics = (Metrics)(nil) + +var statsdClient = statsd.New("traefik.", kitlog.LoggerFunc(func(keyvals ...interface{}) error { + log.Info(keyvals) + return nil +})) +var statsdTicker *time.Ticker + +// Statsd is an Implementation for Metrics that exposes statsd metrics for the latency +// and the number of requests partitioned by status code and method. +// - number of requests partitioned by status code and method +// - request durations +// - amount of retries happened +type Statsd struct { + reqsCounter metrics.Counter + reqDurationHistogram metrics.Histogram + retryCounter metrics.Counter +} + +func (s *Statsd) getReqsCounter() metrics.Counter { + return s.reqsCounter +} + +func (s *Statsd) getReqDurationHistogram() metrics.Histogram { + return s.reqDurationHistogram +} + +func (s *Statsd) getRetryCounter() metrics.Counter { + return s.retryCounter +} + +// NewStatsD creates new instance of StatsD +func NewStatsD(name string) *Statsd { + var m Statsd + + m.reqsCounter = statsdClient.NewCounter(ddMetricsReqsName, 1.0).With("service", name) + m.reqDurationHistogram = statsdClient.NewTiming(ddMetricsLatencyName, 1.0).With("service", name) + + return &m +} + +// InitStatsdClient initializes metrics pusher and creates a statsdClient if not created already +func InitStatsdClient(config *types.Statsd) *time.Ticker { + if statsdTicker == nil { + address := config.Address + if len(address) == 0 { + address = "localhost:8125" + } + pushInterval, err := time.ParseDuration(config.PushInterval) + if err != nil { + log.Warnf("Unable to parse %s into pushInterval, using 10s as default value", config.PushInterval) + pushInterval = 10 * time.Second + } + + report := time.NewTicker(pushInterval) + + safe.Go(func() { + statsdClient.SendLoop(report.C, "udp", address) + }) + + statsdTicker = report + } + return statsdTicker +} + +// StopStatsdClient stops internal statsdTicker which controls the pushing of metrics to StatsD Agent and resets it to `nil` +func StopStatsdClient() { + if statsdTicker != nil { + statsdTicker.Stop() + } + statsdTicker = nil +} diff --git a/middlewares/statsd_test.go b/middlewares/statsd_test.go new file mode 100644 index 000000000..4c6be042c --- /dev/null +++ b/middlewares/statsd_test.go @@ -0,0 +1,52 @@ +package middlewares + +import ( + "fmt" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/containous/traefik/testhelpers" + "github.com/containous/traefik/types" + "github.com/stvp/go-udp-testing" + "github.com/urfave/negroni" +) + +func TestStatsD(t *testing.T) { + udp.SetAddr(":18125") + // This is needed to make sure that UDP Listener listens for data a bit longer, otherwise it will quit after a millisecond + udp.Timeout = 5 * time.Second + recorder := httptest.NewRecorder() + InitStatsdClient(&types.Statsd{":18125", "1s"}) + + n := negroni.New() + c := NewStatsD("test") + defer StopStatsdClient() + metricsMiddlewareBackend := NewMetricsWrapper(c) + + n.Use(metricsMiddlewareBackend) + r := http.NewServeMux() + r.HandleFunc(`/ok`, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + fmt.Fprintln(w, "ok") + }) + r.HandleFunc(`/not-found`, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + fmt.Fprintln(w, "not-found") + }) + n.UseHandler(r) + + req1 := testhelpers.MustNewRequest(http.MethodGet, "http://localhost:3000/ok", nil) + req2 := testhelpers.MustNewRequest(http.MethodGet, "http://localhost:3000/not-found", nil) + + expected := []string{ + // We are only validating counts, as it is nearly impossible to validate latency, since it varies every run + "traefik.requests.total:2.000000|c\n", + } + + udp.ShouldReceiveAll(t, expected, func() { + n.ServeHTTP(recorder, req1) + n.ServeHTTP(recorder, req2) + }) +} diff --git a/server/configuration.go b/server/configuration.go index 5a3bc0726..3bdcc41e0 100644 --- a/server/configuration.go +++ b/server/configuration.go @@ -437,6 +437,14 @@ func NewTraefikDefaultPointersConfiguration() *TraefikConfiguration { Prometheus: &types.Prometheus{ Buckets: types.Buckets{0.1, 0.3, 1.2, 5}, }, + Datadog: &types.Datadog{ + Address: "localhost:8125", + PushInterval: "10s", + }, + StatsD: &types.Statsd{ + Address: "localhost:8125", + PushInterval: "10s", + }, } // default Marathon diff --git a/server/server.go b/server/server.go index 36b522faf..d270487ef 100644 --- a/server/server.go +++ b/server/server.go @@ -157,6 +157,7 @@ func (server *Server) Close() { os.Exit(1) } }(ctx) + stopMetricsClients(server.globalConfiguration) server.stopLeadership() server.routinesPool.Cleanup() close(server.configurationChan) @@ -198,6 +199,7 @@ func (server *Server) setupServerEntryPoint(newServerEntryPointName string, newS if server.accessLoggerMiddleware != nil { serverMiddlewares = append(serverMiddlewares, server.accessLoggerMiddleware) } + initializeMetricsClients(server.globalConfiguration) metrics := newMetrics(server.globalConfiguration, newServerEntryPointName) if metrics != nil { serverMiddlewares = append(serverMiddlewares, middlewares.NewMetricsWrapper(metrics)) @@ -1060,18 +1062,52 @@ func (*Server) configureBackends(backends map[string]*types.Backend) { // Note that given there is no metrics instrumentation configured, it will return nil. func newMetrics(globalConfig GlobalConfiguration, name string) middlewares.Metrics { metricsEnabled := globalConfig.Web != nil && globalConfig.Web.Metrics != nil - if metricsEnabled && globalConfig.Web.Metrics.Prometheus != nil { - metrics, _, err := middlewares.NewPrometheus(name, globalConfig.Web.Metrics.Prometheus) - if err != nil { - log.Errorf("Error creating Prometheus Metrics implementation: %s", err) - return nil + if metricsEnabled { + // Create MultiMetric + metrics := []middlewares.Metrics{} + + if globalConfig.Web.Metrics.Prometheus != nil { + metric, _, err := middlewares.NewPrometheus(name, globalConfig.Web.Metrics.Prometheus) + if err != nil { + log.Errorf("Error creating Prometheus metrics implementation: %s", err) + } + log.Debug("Configured Prometheus metrics") + metrics = append(metrics, metric) } - return metrics + if globalConfig.Web.Metrics.Datadog != nil { + metric := middlewares.NewDataDog(name) + log.Debugf("Configured DataDog metrics pushing to %s once every %s", globalConfig.Web.Metrics.Datadog.Address, globalConfig.Web.Metrics.Datadog.PushInterval) + metrics = append(metrics, metric) + } + if globalConfig.Web.Metrics.StatsD != nil { + metric := middlewares.NewStatsD(name) + log.Debugf("Configured StatsD metrics pushing to %s once every %s", globalConfig.Web.Metrics.StatsD.Address, globalConfig.Web.Metrics.StatsD.PushInterval) + metrics = append(metrics, metric) + } + + return middlewares.NewMultiMetrics(metrics) } return nil } +func initializeMetricsClients(globalConfig GlobalConfiguration) { + metricsEnabled := globalConfig.Web != nil && globalConfig.Web.Metrics != nil + if metricsEnabled { + if globalConfig.Web.Metrics.Datadog != nil { + middlewares.InitDatadogClient(globalConfig.Web.Metrics.Datadog) + } + if globalConfig.Web.Metrics.StatsD != nil { + middlewares.InitStatsdClient(globalConfig.Web.Metrics.StatsD) + } + } +} + +func stopMetricsClients(globalConfig GlobalConfiguration) { + middlewares.StopDatadogClient() + middlewares.StopStatsdClient() +} + func registerRetryMiddleware( httpHandler http.Handler, globalConfig GlobalConfiguration, diff --git a/server/server_test.go b/server/server_test.go index 457449451..dc3948f21 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -443,8 +443,8 @@ func TestNewMetrics(t *testing.T) { metricsImpl := newMetrics(tc.globalConfig, "test1") if metricsImpl != nil { - if _, ok := metricsImpl.(*middlewares.Prometheus); !ok { - t.Errorf("invalid metricsImpl type, got %T want %T", metricsImpl, &middlewares.Prometheus{}) + if _, ok := metricsImpl.(*middlewares.MultiMetrics); !ok { + t.Errorf("invalid metricsImpl type, got %T want %T", metricsImpl, &middlewares.MultiMetrics{}) } } }) diff --git a/types/types.go b/types/types.go index f971f7a52..1a353b6a3 100644 --- a/types/types.go +++ b/types/types.go @@ -327,6 +327,8 @@ type Statistics struct { // Metrics provides options to expose and send Traefik metrics to different third party monitoring systems type Metrics struct { Prometheus *Prometheus `description:"Prometheus metrics exporter type"` + Datadog *Datadog `description:"DataDog metrics exporter type"` + StatsD *Statsd `description:"StatsD metrics exporter type"` } // Prometheus can contain specific configuration used by the Prometheus Metrics exporter @@ -334,6 +336,18 @@ type Prometheus struct { Buckets Buckets `description:"Buckets for latency metrics"` } +// Datadog contains address and metrics pushing interval configuration +type Datadog struct { + Address string `description:"DataDog's Dogstatsd address"` + PushInterval string `description:"DataDog push interval"` +} + +// Statsd contains address and metrics pushing interval configuration +type Statsd struct { + Address string `description:"StatsD address"` + PushInterval string `description:"DataDog push interval"` +} + // Buckets holds Prometheus Buckets type Buckets []float64 diff --git a/vendor/github.com/go-kit/kit/log/doc.go b/vendor/github.com/go-kit/kit/log/doc.go new file mode 100644 index 000000000..49d3f1810 --- /dev/null +++ b/vendor/github.com/go-kit/kit/log/doc.go @@ -0,0 +1,93 @@ +// Package log provides a structured logger. +// +// Structured logging produces logs easily consumed later by humans or +// machines. Humans might be interested in debugging errors, or tracing +// specific requests. Machines might be interested in counting interesting +// events, or aggregating information for off-line processing. In both cases, +// it is important that the log messages are structured and actionable. +// Package log is designed to encourage both of these best practices. +// +// Basic Usage +// +// The fundamental interface is Logger. Loggers create log events from +// key/value data. The Logger interface has a single method, Log, which +// accepts a sequence of alternating key/value pairs, which this package names +// keyvals. +// +// type Logger interface { +// Log(keyvals ...interface{}) error +// } +// +// Here is an example of a function using a Logger to create log events. +// +// func RunTask(task Task, logger log.Logger) string { +// logger.Log("taskID", task.ID, "event", "starting task") +// ... +// logger.Log("taskID", task.ID, "event", "task complete") +// } +// +// The keys in the above example are "taskID" and "event". The values are +// task.ID, "starting task", and "task complete". Every key is followed +// immediately by its value. +// +// Keys are usually plain strings. Values may be any type that has a sensible +// encoding in the chosen log format. With structured logging it is a good +// idea to log simple values without formatting them. This practice allows +// the chosen logger to encode values in the most appropriate way. +// +// Log Context +// +// A log context stores keyvals that it includes in all log events. Building +// appropriate log contexts reduces repetition and aids consistency in the +// resulting log output. We can use a context to improve the RunTask example. +// +// func RunTask(task Task, logger log.Logger) string { +// logger = log.NewContext(logger).With("taskID", task.ID) +// logger.Log("event", "starting task") +// ... +// taskHelper(task.Cmd, logger) +// ... +// logger.Log("event", "task complete") +// } +// +// The improved version emits the same log events as the original for the +// first and last calls to Log. The call to taskHelper highlights that a +// context may be passed as a logger to other functions. Each log event +// created by the called function will include the task.ID even though the +// function does not have access to that value. Using log contexts this way +// simplifies producing log output that enables tracing the life cycle of +// individual tasks. (See the Context example for the full code of the +// above snippet.) +// +// Dynamic Context Values +// +// A Valuer function stored in a log context generates a new value each time +// the context logs an event. The Valuer example demonstrates how this +// feature works. +// +// Valuers provide the basis for consistently logging timestamps and source +// code location. The log package defines several valuers for that purpose. +// See Timestamp, DefaultTimestamp, DefaultTimestampUTC, Caller, and +// DefaultCaller. A common logger initialization sequence that ensures all log +// entries contain a timestamp and source location looks like this: +// +// logger := log.NewLogfmtLogger(log.NewSyncWriter(os.Stdout)) +// logger = log.NewContext(logger).With("ts", log.DefaultTimestampUTC, "caller", log.DefaultCaller) +// +// Concurrent Safety +// +// Applications with multiple goroutines want each log event written to the +// same logger to remain separate from other log events. Package log provides +// two simple solutions for concurrent safe logging. +// +// NewSyncWriter wraps an io.Writer and serializes each call to its Write +// method. Using a SyncWriter has the benefit that the smallest practical +// portion of the logging logic is performed within a mutex, but it requires +// the formatting Logger to make only one call to Write per log event. +// +// NewSyncLogger wraps any Logger and serializes each call to its Log method. +// Using a SyncLogger has the benefit that it guarantees each log event is +// handled atomically within the wrapped logger, but it typically serializes +// both the formatting and output logic. Use a SyncLogger if the formatting +// logger may perform multiple writes per log event. +package log diff --git a/vendor/github.com/go-kit/kit/log/json_logger.go b/vendor/github.com/go-kit/kit/log/json_logger.go new file mode 100644 index 000000000..231e09955 --- /dev/null +++ b/vendor/github.com/go-kit/kit/log/json_logger.go @@ -0,0 +1,92 @@ +package log + +import ( + "encoding" + "encoding/json" + "fmt" + "io" + "reflect" +) + +type jsonLogger struct { + io.Writer +} + +// NewJSONLogger returns a Logger that encodes keyvals to the Writer as a +// single JSON object. Each log event produces no more than one call to +// w.Write. The passed Writer must be safe for concurrent use by multiple +// goroutines if the returned Logger will be used concurrently. +func NewJSONLogger(w io.Writer) Logger { + return &jsonLogger{w} +} + +func (l *jsonLogger) Log(keyvals ...interface{}) error { + n := (len(keyvals) + 1) / 2 // +1 to handle case when len is odd + m := make(map[string]interface{}, n) + for i := 0; i < len(keyvals); i += 2 { + k := keyvals[i] + var v interface{} = ErrMissingValue + if i+1 < len(keyvals) { + v = keyvals[i+1] + } + merge(m, k, v) + } + return json.NewEncoder(l.Writer).Encode(m) +} + +func merge(dst map[string]interface{}, k, v interface{}) { + var key string + switch x := k.(type) { + case string: + key = x + case fmt.Stringer: + key = safeString(x) + default: + key = fmt.Sprint(x) + } + if x, ok := v.(error); ok { + v = safeError(x) + } + + // We want json.Marshaler and encoding.TextMarshaller to take priority over + // err.Error() and v.String(). But json.Marshall (called later) does that by + // default so we force a no-op if it's one of those 2 case. + switch x := v.(type) { + case json.Marshaler: + case encoding.TextMarshaler: + case error: + v = safeError(x) + case fmt.Stringer: + v = safeString(x) + } + + dst[key] = v +} + +func safeString(str fmt.Stringer) (s string) { + defer func() { + if panicVal := recover(); panicVal != nil { + if v := reflect.ValueOf(str); v.Kind() == reflect.Ptr && v.IsNil() { + s = "NULL" + } else { + panic(panicVal) + } + } + }() + s = str.String() + return +} + +func safeError(err error) (s interface{}) { + defer func() { + if panicVal := recover(); panicVal != nil { + if v := reflect.ValueOf(err); v.Kind() == reflect.Ptr && v.IsNil() { + s = nil + } else { + panic(panicVal) + } + } + }() + s = err.Error() + return +} diff --git a/vendor/github.com/go-kit/kit/log/log.go b/vendor/github.com/go-kit/kit/log/log.go new file mode 100644 index 000000000..97990feff --- /dev/null +++ b/vendor/github.com/go-kit/kit/log/log.go @@ -0,0 +1,144 @@ +package log + +import "errors" + +// Logger is the fundamental interface for all log operations. Log creates a +// log event from keyvals, a variadic sequence of alternating keys and values. +// Implementations must be safe for concurrent use by multiple goroutines. In +// particular, any implementation of Logger that appends to keyvals or +// modifies any of its elements must make a copy first. +type Logger interface { + Log(keyvals ...interface{}) error +} + +// ErrMissingValue is appended to keyvals slices with odd length to substitute +// the missing value. +var ErrMissingValue = errors.New("(MISSING)") + +// NewContext returns a new Context that logs to logger. +func NewContext(logger Logger) *Context { + if c, ok := logger.(*Context); ok { + return c + } + return &Context{logger: logger} +} + +// Context must always have the same number of stack frames between calls to +// its Log method and the eventual binding of Valuers to their value. This +// requirement comes from the functional requirement to allow a context to +// resolve application call site information for a log.Caller stored in the +// context. To do this we must be able to predict the number of logging +// functions on the stack when bindValues is called. +// +// Three implementation details provide the needed stack depth consistency. +// The first two of these details also result in better amortized performance, +// and thus make sense even without the requirements regarding stack depth. +// The third detail, however, is subtle and tied to the implementation of the +// Go compiler. +// +// 1. NewContext avoids introducing an additional layer when asked to +// wrap another Context. +// 2. With avoids introducing an additional layer by returning a newly +// constructed Context with a merged keyvals rather than simply +// wrapping the existing Context. +// 3. All of Context's methods take pointer receivers even though they +// do not mutate the Context. +// +// Before explaining the last detail, first some background. The Go compiler +// generates wrapper methods to implement the auto dereferencing behavior when +// calling a value method through a pointer variable. These wrapper methods +// are also used when calling a value method through an interface variable +// because interfaces store a pointer to the underlying concrete value. +// Calling a pointer receiver through an interface does not require generating +// an additional function. +// +// If Context had value methods then calling Context.Log through a variable +// with type Logger would have an extra stack frame compared to calling +// Context.Log through a variable with type Context. Using pointer receivers +// avoids this problem. + +// A Context wraps a Logger and holds keyvals that it includes in all log +// events. When logging, a Context replaces all value elements (odd indexes) +// containing a Valuer with their generated value for each call to its Log +// method. +type Context struct { + logger Logger + keyvals []interface{} + hasValuer bool +} + +// Log replaces all value elements (odd indexes) containing a Valuer in the +// stored context with their generated value, appends keyvals, and passes the +// result to the wrapped Logger. +func (l *Context) Log(keyvals ...interface{}) error { + kvs := append(l.keyvals, keyvals...) + if len(kvs)%2 != 0 { + kvs = append(kvs, ErrMissingValue) + } + if l.hasValuer { + // If no keyvals were appended above then we must copy l.keyvals so + // that future log events will reevaluate the stored Valuers. + if len(keyvals) == 0 { + kvs = append([]interface{}{}, l.keyvals...) + } + bindValues(kvs[:len(l.keyvals)]) + } + return l.logger.Log(kvs...) +} + +// With returns a new Context with keyvals appended to those of the receiver. +func (l *Context) With(keyvals ...interface{}) *Context { + if len(keyvals) == 0 { + return l + } + kvs := append(l.keyvals, keyvals...) + if len(kvs)%2 != 0 { + kvs = append(kvs, ErrMissingValue) + } + return &Context{ + logger: l.logger, + // Limiting the capacity of the stored keyvals ensures that a new + // backing array is created if the slice must grow in Log or With. + // Using the extra capacity without copying risks a data race that + // would violate the Logger interface contract. + keyvals: kvs[:len(kvs):len(kvs)], + hasValuer: l.hasValuer || containsValuer(keyvals), + } +} + +// WithPrefix returns a new Context with keyvals prepended to those of the +// receiver. +func (l *Context) WithPrefix(keyvals ...interface{}) *Context { + if len(keyvals) == 0 { + return l + } + // Limiting the capacity of the stored keyvals ensures that a new + // backing array is created if the slice must grow in Log or With. + // Using the extra capacity without copying risks a data race that + // would violate the Logger interface contract. + n := len(l.keyvals) + len(keyvals) + if len(keyvals)%2 != 0 { + n++ + } + kvs := make([]interface{}, 0, n) + kvs = append(kvs, keyvals...) + if len(kvs)%2 != 0 { + kvs = append(kvs, ErrMissingValue) + } + kvs = append(kvs, l.keyvals...) + return &Context{ + logger: l.logger, + keyvals: kvs, + hasValuer: l.hasValuer || containsValuer(keyvals), + } +} + +// LoggerFunc is an adapter to allow use of ordinary functions as Loggers. If +// f is a function with the appropriate signature, LoggerFunc(f) is a Logger +// object that calls f. +type LoggerFunc func(...interface{}) error + +// Log implements Logger by calling f(keyvals...). +func (f LoggerFunc) Log(keyvals ...interface{}) error { + return f(keyvals...) +} diff --git a/vendor/github.com/go-kit/kit/log/logfmt_logger.go b/vendor/github.com/go-kit/kit/log/logfmt_logger.go new file mode 100644 index 000000000..a00305298 --- /dev/null +++ b/vendor/github.com/go-kit/kit/log/logfmt_logger.go @@ -0,0 +1,62 @@ +package log + +import ( + "bytes" + "io" + "sync" + + "github.com/go-logfmt/logfmt" +) + +type logfmtEncoder struct { + *logfmt.Encoder + buf bytes.Buffer +} + +func (l *logfmtEncoder) Reset() { + l.Encoder.Reset() + l.buf.Reset() +} + +var logfmtEncoderPool = sync.Pool{ + New: func() interface{} { + var enc logfmtEncoder + enc.Encoder = logfmt.NewEncoder(&enc.buf) + return &enc + }, +} + +type logfmtLogger struct { + w io.Writer +} + +// NewLogfmtLogger returns a logger that encodes keyvals to the Writer in +// logfmt format. Each log event produces no more than one call to w.Write. +// The passed Writer must be safe for concurrent use by multiple goroutines if +// the returned Logger will be used concurrently. +func NewLogfmtLogger(w io.Writer) Logger { + return &logfmtLogger{w} +} + +func (l logfmtLogger) Log(keyvals ...interface{}) error { + enc := logfmtEncoderPool.Get().(*logfmtEncoder) + enc.Reset() + defer logfmtEncoderPool.Put(enc) + + if err := enc.EncodeKeyvals(keyvals...); err != nil { + return err + } + + // Add newline to the end of the buffer + if err := enc.EndRecord(); err != nil { + return err + } + + // The Logger interface requires implementations to be safe for concurrent + // use by multiple goroutines. For this implementation that means making + // only one call to l.w.Write() for each call to Log. + if _, err := l.w.Write(enc.buf.Bytes()); err != nil { + return err + } + return nil +} diff --git a/vendor/github.com/go-kit/kit/log/nop_logger.go b/vendor/github.com/go-kit/kit/log/nop_logger.go new file mode 100644 index 000000000..1047d626c --- /dev/null +++ b/vendor/github.com/go-kit/kit/log/nop_logger.go @@ -0,0 +1,8 @@ +package log + +type nopLogger struct{} + +// NewNopLogger returns a logger that doesn't do anything. +func NewNopLogger() Logger { return nopLogger{} } + +func (nopLogger) Log(...interface{}) error { return nil } diff --git a/vendor/github.com/go-kit/kit/log/stdlib.go b/vendor/github.com/go-kit/kit/log/stdlib.go new file mode 100644 index 000000000..7ffd1ca17 --- /dev/null +++ b/vendor/github.com/go-kit/kit/log/stdlib.go @@ -0,0 +1,116 @@ +package log + +import ( + "io" + "log" + "regexp" + "strings" +) + +// StdlibWriter implements io.Writer by invoking the stdlib log.Print. It's +// designed to be passed to a Go kit logger as the writer, for cases where +// it's necessary to redirect all Go kit log output to the stdlib logger. +// +// If you have any choice in the matter, you shouldn't use this. Prefer to +// redirect the stdlib log to the Go kit logger via NewStdlibAdapter. +type StdlibWriter struct{} + +// Write implements io.Writer. +func (w StdlibWriter) Write(p []byte) (int, error) { + log.Print(strings.TrimSpace(string(p))) + return len(p), nil +} + +// StdlibAdapter wraps a Logger and allows it to be passed to the stdlib +// logger's SetOutput. It will extract date/timestamps, filenames, and +// messages, and place them under relevant keys. +type StdlibAdapter struct { + Logger + timestampKey string + fileKey string + messageKey string +} + +// StdlibAdapterOption sets a parameter for the StdlibAdapter. +type StdlibAdapterOption func(*StdlibAdapter) + +// TimestampKey sets the key for the timestamp field. By default, it's "ts". +func TimestampKey(key string) StdlibAdapterOption { + return func(a *StdlibAdapter) { a.timestampKey = key } +} + +// FileKey sets the key for the file and line field. By default, it's "file". +func FileKey(key string) StdlibAdapterOption { + return func(a *StdlibAdapter) { a.fileKey = key } +} + +// MessageKey sets the key for the actual log message. By default, it's "msg". +func MessageKey(key string) StdlibAdapterOption { + return func(a *StdlibAdapter) { a.messageKey = key } +} + +// NewStdlibAdapter returns a new StdlibAdapter wrapper around the passed +// logger. It's designed to be passed to log.SetOutput. +func NewStdlibAdapter(logger Logger, options ...StdlibAdapterOption) io.Writer { + a := StdlibAdapter{ + Logger: logger, + timestampKey: "ts", + fileKey: "file", + messageKey: "msg", + } + for _, option := range options { + option(&a) + } + return a +} + +func (a StdlibAdapter) Write(p []byte) (int, error) { + result := subexps(p) + keyvals := []interface{}{} + var timestamp string + if date, ok := result["date"]; ok && date != "" { + timestamp = date + } + if time, ok := result["time"]; ok && time != "" { + if timestamp != "" { + timestamp += " " + } + timestamp += time + } + if timestamp != "" { + keyvals = append(keyvals, a.timestampKey, timestamp) + } + if file, ok := result["file"]; ok && file != "" { + keyvals = append(keyvals, a.fileKey, file) + } + if msg, ok := result["msg"]; ok { + keyvals = append(keyvals, a.messageKey, msg) + } + if err := a.Logger.Log(keyvals...); err != nil { + return 0, err + } + return len(p), nil +} + +const ( + logRegexpDate = `(?P[0-9]{4}/[0-9]{2}/[0-9]{2})?[ ]?` + logRegexpTime = `(?P