traefik/server/server.go

578 lines
19 KiB
Go
Raw Normal View History

package server
import (
2016-08-16 15:26:10 +00:00
"context"
"crypto/tls"
"crypto/x509"
"encoding/json"
2017-10-10 12:50:03 +00:00
"fmt"
"io/ioutil"
2017-11-18 00:10:03 +00:00
stdlog "log"
"net"
"net/http"
2018-06-07 07:46:03 +00:00
"net/http/httputil"
"net/url"
"os"
"os/signal"
"reflect"
2017-10-30 11:54:03 +00:00
"strings"
"sync"
"time"
2017-08-25 19:32:03 +00:00
"github.com/armon/go-proxyproto"
"github.com/containous/mux"
"github.com/containous/traefik/cluster"
"github.com/containous/traefik/configuration"
"github.com/containous/traefik/configuration/router"
2018-05-28 09:46:03 +00:00
"github.com/containous/traefik/h2c"
"github.com/containous/traefik/log"
"github.com/containous/traefik/metrics"
"github.com/containous/traefik/middlewares"
"github.com/containous/traefik/middlewares/accesslog"
2018-01-10 16:48:04 +00:00
"github.com/containous/traefik/middlewares/tracing"
"github.com/containous/traefik/provider"
"github.com/containous/traefik/safe"
traefiktls "github.com/containous/traefik/tls"
"github.com/containous/traefik/types"
2017-10-10 12:50:03 +00:00
"github.com/containous/traefik/whitelist"
2018-01-22 11:16:03 +00:00
"github.com/sirupsen/logrus"
"github.com/urfave/negroni"
)
2018-04-11 14:30:04 +00:00
var httpServerLogger = stdlog.New(log.WriterLevel(logrus.DebugLevel), "", 0)
2016-01-13 21:46:44 +00:00
// Server is the reverse-proxy/load-balancer engine
type Server struct {
serverEntryPoints serverEntryPoints
configurationChan chan types.ConfigMessage
configurationValidatedChan chan types.ConfigMessage
signals chan os.Signal
stopChan chan bool
currentConfigurations safe.Safe
2018-01-23 11:44:03 +00:00
providerConfigUpdateMap map[string]chan types.ConfigMessage
globalConfiguration configuration.GlobalConfiguration
accessLoggerMiddleware *accesslog.LogHandler
2018-01-10 16:48:04 +00:00
tracingMiddleware *tracing.Tracing
routinesPool *safe.Pool
leadership *cluster.Leadership
defaultForwardingRoundTripper http.RoundTripper
metricsRegistry metrics.Registry
provider provider.Provider
2018-03-05 19:54:04 +00:00
configurationListeners []func(types.Configuration)
entryPoints map[string]EntryPoint
2018-06-07 07:46:03 +00:00
bufferPool httputil.BufferPool
}
// EntryPoint entryPoint information (configuration + internalRouter)
type EntryPoint struct {
InternalRouter types.InternalRouter
Configuration *configuration.EntryPoint
OnDemandListener func(string) (*tls.Certificate, error)
CertificateStore *traefiktls.CertificateStore
}
type serverEntryPoints map[string]*serverEntryPoint
type serverEntryPoint struct {
2018-05-28 09:46:03 +00:00
httpServer *h2c.Server
2018-03-05 19:54:04 +00:00
listener net.Listener
httpRouter *middlewares.HandlerSwitcher
certs *safe.Safe
2018-03-05 19:54:04 +00:00
onDemandListener func(string) (*tls.Certificate, error)
}
// NewServer returns an initialized Server.
func NewServer(globalConfiguration configuration.GlobalConfiguration, provider provider.Provider, entrypoints map[string]EntryPoint) *Server {
2018-06-11 09:36:03 +00:00
server := &Server{}
server.entryPoints = entrypoints
server.provider = provider
2018-06-11 09:36:03 +00:00
server.globalConfiguration = globalConfiguration
server.serverEntryPoints = make(map[string]*serverEntryPoint)
server.configurationChan = make(chan types.ConfigMessage, 100)
server.configurationValidatedChan = make(chan types.ConfigMessage, 100)
server.signals = make(chan os.Signal, 1)
server.stopChan = make(chan bool, 1)
server.configureSignals()
currentConfigurations := make(types.Configurations)
server.currentConfigurations.Set(currentConfigurations)
2018-01-23 11:44:03 +00:00
server.providerConfigUpdateMap = make(map[string]chan types.ConfigMessage)
2018-06-11 09:36:03 +00:00
if server.globalConfiguration.API != nil {
server.globalConfiguration.API.CurrentConfigurations = &server.currentConfigurations
}
2018-06-07 07:46:03 +00:00
server.bufferPool = newBufferPool()
server.routinesPool = safe.NewPool(context.Background())
2018-06-11 09:36:03 +00:00
transport, err := createHTTPTransport(globalConfiguration)
if err != nil {
log.Errorf("failed to create HTTP transport: %v", err)
}
server.defaultForwardingRoundTripper = transport
2018-01-10 16:48:04 +00:00
server.tracingMiddleware = globalConfiguration.Tracing
2018-06-11 09:36:03 +00:00
if server.tracingMiddleware != nil && server.tracingMiddleware.Backend != "" {
2018-01-10 16:48:04 +00:00
server.tracingMiddleware.Setup()
}
server.metricsRegistry = registerMetricClients(globalConfiguration.Metrics)
if globalConfiguration.Cluster != nil {
// leadership creation if cluster mode
server.leadership = cluster.NewLeadership(server.routinesPool.Ctx(), globalConfiguration.Cluster)
}
2017-05-25 11:25:53 +00:00
if globalConfiguration.AccessLogsFile != "" {
globalConfiguration.AccessLog = &types.AccessLog{FilePath: globalConfiguration.AccessLogsFile, Format: accesslog.CommonFormat}
}
if globalConfiguration.AccessLog != nil {
var err error
server.accessLoggerMiddleware, err = accesslog.NewLogHandler(globalConfiguration.AccessLog)
if err != nil {
log.Warnf("Unable to create log handler: %s", err)
}
2017-05-22 19:39:29 +00:00
}
return server
}
// Start starts the server.
2017-11-24 18:18:03 +00:00
func (s *Server) Start() {
s.startHTTPServers()
s.startLeadership()
s.routinesPool.Go(func(stop chan bool) {
s.listenProviders(stop)
})
2017-11-24 18:18:03 +00:00
s.routinesPool.Go(func(stop chan bool) {
s.listenConfigurations(stop)
})
s.startProvider()
2017-11-24 18:18:03 +00:00
go s.listenSignals()
}
2018-03-14 12:14:03 +00:00
// StartWithContext starts the server and Stop/Close it when context is Done
func (s *Server) StartWithContext(ctx context.Context) {
go func() {
defer s.Close()
<-ctx.Done()
log.Info("I have to go...")
reqAcceptGraceTimeOut := time.Duration(s.globalConfiguration.LifeCycle.RequestAcceptGraceTimeout)
if reqAcceptGraceTimeOut > 0 {
log.Infof("Waiting %s for incoming requests to cease", reqAcceptGraceTimeOut)
time.Sleep(reqAcceptGraceTimeOut)
}
log.Info("Stopping server gracefully")
s.Stop()
}()
s.Start()
}
// Wait blocks until server is shutted down.
2017-11-24 18:18:03 +00:00
func (s *Server) Wait() {
<-s.stopChan
}
// Stop stops the server
2017-11-24 18:18:03 +00:00
func (s *Server) Stop() {
defer log.Info("Server stopped")
var wg sync.WaitGroup
2017-11-24 18:18:03 +00:00
for sepn, sep := range s.serverEntryPoints {
wg.Add(1)
go func(serverEntryPointName string, serverEntryPoint *serverEntryPoint) {
defer wg.Done()
2017-11-24 18:18:03 +00:00
graceTimeOut := time.Duration(s.globalConfiguration.LifeCycle.GraceTimeOut)
ctx, cancel := context.WithTimeout(context.Background(), graceTimeOut)
log.Debugf("Waiting %s seconds before killing connections on entrypoint %s...", graceTimeOut, serverEntryPointName)
if err := serverEntryPoint.httpServer.Shutdown(ctx); err != nil {
log.Debugf("Wait is over due to: %s", err)
2018-06-11 09:36:03 +00:00
err = serverEntryPoint.httpServer.Close()
if err != nil {
log.Error(err)
}
}
cancel()
log.Debugf("Entrypoint %s closed", serverEntryPointName)
}(sepn, sep)
}
wg.Wait()
2017-11-24 18:18:03 +00:00
s.stopChan <- true
}
// Close destroys the server
2017-11-24 18:18:03 +00:00
func (s *Server) Close() {
2018-03-14 12:14:03 +00:00
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
go func(ctx context.Context) {
<-ctx.Done()
if ctx.Err() == context.Canceled {
return
} else if ctx.Err() == context.DeadlineExceeded {
2018-03-14 12:14:03 +00:00
panic("Timeout while stopping traefik, killing instance ✝")
}
}(ctx)
stopMetricsClients()
2017-11-24 18:18:03 +00:00
s.stopLeadership()
s.routinesPool.Cleanup()
close(s.configurationChan)
close(s.configurationValidatedChan)
signal.Stop(s.signals)
close(s.signals)
close(s.stopChan)
if s.accessLoggerMiddleware != nil {
if err := s.accessLoggerMiddleware.Close(); err != nil {
2017-05-22 19:39:29 +00:00
log.Errorf("Error closing access log file: %s", err)
}
}
cancel()
}
2017-11-24 18:18:03 +00:00
func (s *Server) startLeadership() {
if s.leadership != nil {
s.leadership.Participate(s.routinesPool)
}
}
2017-11-24 18:18:03 +00:00
func (s *Server) stopLeadership() {
if s.leadership != nil {
s.leadership.Stop()
}
}
2017-11-24 18:18:03 +00:00
func (s *Server) startHTTPServers() {
2018-06-11 09:36:03 +00:00
s.serverEntryPoints = s.buildServerEntryPoints()
2017-01-12 13:34:54 +00:00
2017-11-24 18:18:03 +00:00
for newServerEntryPointName, newServerEntryPoint := range s.serverEntryPoints {
serverEntryPoint := s.setupServerEntryPoint(newServerEntryPointName, newServerEntryPoint)
go s.startServer(serverEntryPoint)
}
}
2017-11-24 18:18:03 +00:00
func (s *Server) listenProviders(stop chan bool) {
for {
select {
case <-stop:
return
2017-11-24 18:18:03 +00:00
case configMsg, ok := <-s.configurationChan:
if !ok || configMsg.Configuration == nil {
return
}
2017-11-24 18:18:03 +00:00
s.preLoadConfiguration(configMsg)
}
}
}
2018-03-05 19:54:04 +00:00
// AddListener adds a new listener function used when new configuration is provided
func (s *Server) AddListener(listener func(types.Configuration)) {
if s.configurationListeners == nil {
s.configurationListeners = make([]func(types.Configuration), 0)
}
s.configurationListeners = append(s.configurationListeners, listener)
}
2018-03-27 14:18:03 +00:00
// getCertificate allows to customize tlsConfig.GetCertificate behaviour to get the certificates inserted dynamically
func (s *serverEntryPoint) getCertificate(clientHello *tls.ClientHelloInfo) (*tls.Certificate, error) {
2018-03-05 19:54:04 +00:00
domainToCheck := types.CanonicalDomain(clientHello.ServerName)
if s.certs.Get() != nil {
for domains, cert := range s.certs.Get().(map[string]*tls.Certificate) {
2018-03-27 14:18:03 +00:00
for _, certDomain := range strings.Split(domains, ",") {
if types.MatchDomain(domainToCheck, certDomain) {
return cert, nil
}
}
}
log.Debugf("No certificate provided dynamically can check the domain %q, a per default certificate will be used.", domainToCheck)
}
2018-03-05 19:54:04 +00:00
if s.onDemandListener != nil {
return s.onDemandListener(domainToCheck)
}
return nil, nil
}
func (s *Server) startProvider() {
// start providers
providerType := reflect.TypeOf(s.provider)
jsonConf, err := json.Marshal(s.provider)
if err != nil {
log.Debugf("Unable to marshal provider conf %v with error: %v", providerType, err)
}
log.Infof("Starting provider %v %s", providerType, jsonConf)
currentProvider := s.provider
safe.Go(func() {
err := currentProvider.Provide(s.configurationChan, s.routinesPool, s.globalConfiguration.Constraints)
if err != nil {
log.Errorf("Error starting provider %v: %s", providerType, err)
}
})
}
// creates a TLS config that allows terminating HTTPS for multiple domains using SNI
func (s *Server) createTLSConfig(entryPointName string, tlsOption *traefiktls.TLS, router *middlewares.HandlerSwitcher) (*tls.Config, error) {
if tlsOption == nil {
return nil, nil
}
2018-03-05 19:54:04 +00:00
config, err := tlsOption.Certificates.CreateTLSConfig(entryPointName)
if err != nil {
return nil, err
}
s.serverEntryPoints[entryPointName].certs.Set(make(map[string]*tls.Certificate))
2016-11-09 16:56:41 +00:00
// ensure http2 enabled
config.NextProtos = []string{"h2", "http/1.1"}
if len(tlsOption.ClientCAFiles) > 0 {
2017-11-10 09:30:04 +00:00
log.Warnf("Deprecated configuration found during TLS configuration creation: %s. Please use %s (which allows to make the CA Files optional).", "tls.ClientCAFiles", "tls.ClientCA.files")
tlsOption.ClientCA.Files = tlsOption.ClientCAFiles
tlsOption.ClientCA.Optional = false
}
if len(tlsOption.ClientCA.Files) > 0 {
pool := x509.NewCertPool()
2017-11-10 09:30:04 +00:00
for _, caFile := range tlsOption.ClientCA.Files {
data, err := ioutil.ReadFile(caFile)
if err != nil {
return nil, err
}
ok := pool.AppendCertsFromPEM(data)
if !ok {
2018-06-11 09:36:03 +00:00
return nil, fmt.Errorf("invalid certificate(s) in %s", caFile)
}
}
config.ClientCAs = pool
2017-11-10 09:30:04 +00:00
if tlsOption.ClientCA.Optional {
config.ClientAuth = tls.VerifyClientCertIfGiven
} else {
config.ClientAuth = tls.RequireAndVerifyClientCert
}
}
2017-11-24 18:18:03 +00:00
if s.globalConfiguration.ACME != nil {
if entryPointName == s.globalConfiguration.ACME.EntryPoint {
checkOnDemandDomain := func(domain string) bool {
routeMatch := &mux.RouteMatch{}
match := router.GetHandler().Match(&http.Request{URL: &url.URL{}, Host: domain}, routeMatch)
if match && routeMatch.Route != nil {
return true
}
return false
}
2018-03-05 19:54:04 +00:00
err := s.globalConfiguration.ACME.CreateClusterConfig(s.leadership, config, s.serverEntryPoints[entryPointName].certs, checkOnDemandDomain)
2018-03-05 19:54:04 +00:00
if err != nil {
return nil, err
}
}
} else {
2017-11-24 18:18:03 +00:00
config.GetCertificate = s.serverEntryPoints[entryPointName].getCertificate
}
2018-06-11 09:36:03 +00:00
if len(config.Certificates) == 0 {
2018-06-11 09:36:03 +00:00
return nil, fmt.Errorf("no certificates found for TLS entrypoint %s", entryPointName)
}
2018-06-11 09:36:03 +00:00
// BuildNameToCertificate parses the CommonName and SubjectAlternateName fields
// in each certificate and populates the config.NameToCertificate map.
config.BuildNameToCertificate()
2018-03-05 19:54:04 +00:00
if s.entryPoints[entryPointName].CertificateStore != nil {
s.entryPoints[entryPointName].CertificateStore.StaticCerts.Set(config.NameToCertificate)
2018-03-05 19:54:04 +00:00
}
// Set the minimum TLS version if set in the config TOML
if minConst, exists := traefiktls.MinVersion[s.entryPoints[entryPointName].Configuration.TLS.MinVersion]; exists {
config.PreferServerCipherSuites = true
config.MinVersion = minConst
}
2018-03-05 19:54:04 +00:00
// Set the list of CipherSuites if set in the config TOML
if s.entryPoints[entryPointName].Configuration.TLS.CipherSuites != nil {
2018-03-05 19:54:04 +00:00
// if our list of CipherSuites is defined in the entrypoint config, we can re-initilize the suites list as empty
config.CipherSuites = make([]uint16, 0)
for _, cipher := range s.entryPoints[entryPointName].Configuration.TLS.CipherSuites {
if cipherConst, exists := traefiktls.CipherSuites[cipher]; exists {
config.CipherSuites = append(config.CipherSuites, cipherConst)
} else {
2018-03-05 19:54:04 +00:00
// CipherSuite listed in the toml does not exist in our listed
2018-06-11 09:36:03 +00:00
return nil, fmt.Errorf("invalid CipherSuite: %s", cipher)
}
}
}
2018-06-11 09:36:03 +00:00
return config, nil
}
func (s *Server) startServer(serverEntryPoint *serverEntryPoint) {
2017-08-25 19:32:03 +00:00
log.Infof("Starting server on %s", serverEntryPoint.httpServer.Addr)
2018-06-11 09:36:03 +00:00
var err error
2017-08-25 19:32:03 +00:00
if serverEntryPoint.httpServer.TLSConfig != nil {
err = serverEntryPoint.httpServer.ServeTLS(serverEntryPoint.listener, "", "")
} else {
2017-08-25 19:32:03 +00:00
err = serverEntryPoint.httpServer.Serve(serverEntryPoint.listener)
}
2018-06-11 09:36:03 +00:00
if err != http.ErrServerClosed {
log.Error("Error creating server: ", err)
}
}
2018-06-11 09:36:03 +00:00
func (s *Server) setupServerEntryPoint(newServerEntryPointName string, newServerEntryPoint *serverEntryPoint) *serverEntryPoint {
serverMiddlewares, err := s.buildServerEntryPointMiddlewares(newServerEntryPointName, newServerEntryPoint)
if err != nil {
log.Fatal("Error preparing server: ", err)
}
newSrv, listener, err := s.prepareServer(newServerEntryPointName, s.entryPoints[newServerEntryPointName].Configuration, newServerEntryPoint.httpRouter, serverMiddlewares)
if err != nil {
log.Fatal("Error preparing server: ", err)
}
serverEntryPoint := s.serverEntryPoints[newServerEntryPointName]
serverEntryPoint.httpServer = newSrv
serverEntryPoint.listener = listener
return serverEntryPoint
}
2018-05-28 09:46:03 +00:00
func (s *Server) prepareServer(entryPointName string, entryPoint *configuration.EntryPoint, router *middlewares.HandlerSwitcher, middlewares []negroni.Handler) (*h2c.Server, net.Listener, error) {
2017-11-24 18:18:03 +00:00
readTimeout, writeTimeout, idleTimeout := buildServerTimeouts(s.globalConfiguration)
log.Infof("Preparing server %s %+v with readTimeout=%s writeTimeout=%s idleTimeout=%s", entryPointName, entryPoint, readTimeout, writeTimeout, idleTimeout)
// middlewares
n := negroni.New()
for _, middleware := range middlewares {
n.Use(middleware)
}
n.UseHandler(router)
internalMuxRouter := s.buildInternalRouter(entryPointName)
internalMuxRouter.NotFoundHandler = n
2017-11-24 18:18:03 +00:00
tlsConfig, err := s.createTLSConfig(entryPointName, entryPoint.TLS, router)
if err != nil {
2018-06-11 09:36:03 +00:00
return nil, nil, fmt.Errorf("error creating TLS config: %v", err)
2017-08-25 19:32:03 +00:00
}
listener, err := net.Listen("tcp", entryPoint.Address)
if err != nil {
2018-06-11 09:36:03 +00:00
return nil, nil, fmt.Errorf("error opening listener: %v", err)
2017-08-25 19:32:03 +00:00
}
2017-10-10 12:50:03 +00:00
if entryPoint.ProxyProtocol != nil {
2018-06-11 09:36:03 +00:00
listener, err = buildProxyProtocolListener(entryPoint, listener)
2017-10-10 12:50:03 +00:00
if err != nil {
2018-06-11 09:36:03 +00:00
return nil, nil, err
2017-10-10 12:50:03 +00:00
}
}
2018-05-28 09:46:03 +00:00
return &h2c.Server{
Server: &http.Server{
Addr: entryPoint.Address,
Handler: internalMuxRouter,
TLSConfig: tlsConfig,
ReadTimeout: readTimeout,
WriteTimeout: writeTimeout,
IdleTimeout: idleTimeout,
ErrorLog: httpServerLogger,
},
2017-08-25 19:32:03 +00:00
},
listener,
nil
}
2018-06-11 09:36:03 +00:00
func buildProxyProtocolListener(entryPoint *configuration.EntryPoint, listener net.Listener) (net.Listener, error) {
IPs, err := whitelist.NewIP(entryPoint.ProxyProtocol.TrustedIPs, entryPoint.ProxyProtocol.Insecure, false)
if err != nil {
return nil, fmt.Errorf("error creating whitelist: %s", err)
}
log.Infof("Enabling ProxyProtocol for trusted IPs %v", entryPoint.ProxyProtocol.TrustedIPs)
return &proxyproto.Listener{
Listener: listener,
SourceCheck: func(addr net.Addr) (bool, error) {
ip, ok := addr.(*net.TCPAddr)
if !ok {
return false, fmt.Errorf("type error %v", addr)
}
return IPs.ContainsIP(ip.IP), nil
},
}, nil
}
func (s *Server) buildInternalRouter(entryPointName string) *mux.Router {
internalMuxRouter := mux.NewRouter()
internalMuxRouter.StrictSlash(true)
internalMuxRouter.SkipClean(true)
if entryPoint, ok := s.entryPoints[entryPointName]; ok && entryPoint.InternalRouter != nil {
entryPoint.InternalRouter.AddRoutes(internalMuxRouter)
if s.globalConfiguration.API != nil && s.globalConfiguration.API.EntryPoint == entryPointName && s.leadership != nil {
if s.globalConfiguration.Web != nil && s.globalConfiguration.Web.Path != "" {
rt := router.WithPrefix{Router: s.leadership, PathPrefix: s.globalConfiguration.Web.Path}
rt.AddRoutes(internalMuxRouter)
} else {
s.leadership.AddRoutes(internalMuxRouter)
}
}
}
return internalMuxRouter
}
func buildServerTimeouts(globalConfig configuration.GlobalConfiguration) (readTimeout, writeTimeout, idleTimeout time.Duration) {
readTimeout = time.Duration(0)
writeTimeout = time.Duration(0)
if globalConfig.RespondingTimeouts != nil {
readTimeout = time.Duration(globalConfig.RespondingTimeouts.ReadTimeout)
writeTimeout = time.Duration(globalConfig.RespondingTimeouts.WriteTimeout)
}
2017-09-20 16:14:03 +00:00
// Prefer legacy idle timeout parameter for backwards compatibility reasons
if globalConfig.IdleTimeout > 0 {
idleTimeout = time.Duration(globalConfig.IdleTimeout)
2017-09-20 16:14:03 +00:00
log.Warn("top-level idle timeout configuration has been deprecated -- please use responding timeouts")
} else if globalConfig.RespondingTimeouts != nil {
idleTimeout = time.Duration(globalConfig.RespondingTimeouts.IdleTimeout)
} else {
2017-12-18 08:14:03 +00:00
idleTimeout = configuration.DefaultIdleTimeout
}
return readTimeout, writeTimeout, idleTimeout
}
func registerMetricClients(metricsConfig *types.Metrics) metrics.Registry {
if metricsConfig == nil {
return metrics.NewVoidRegistry()
}
2018-01-31 18:10:04 +00:00
var registries []metrics.Registry
if metricsConfig.Prometheus != nil {
registries = append(registries, metrics.RegisterPrometheus(metricsConfig.Prometheus))
log.Debug("Configured Prometheus metrics")
}
if metricsConfig.Datadog != nil {
registries = append(registries, metrics.RegisterDatadog(metricsConfig.Datadog))
log.Debugf("Configured DataDog metrics pushing to %s once every %s", metricsConfig.Datadog.Address, metricsConfig.Datadog.PushInterval)
}
if metricsConfig.StatsD != nil {
registries = append(registries, metrics.RegisterStatsd(metricsConfig.StatsD))
log.Debugf("Configured StatsD metrics pushing to %s once every %s", metricsConfig.StatsD.Address, metricsConfig.StatsD.PushInterval)
2017-04-18 06:22:06 +00:00
}
if metricsConfig.InfluxDB != nil {
registries = append(registries, metrics.RegisterInfluxDB(metricsConfig.InfluxDB))
log.Debugf("Configured InfluxDB metrics pushing to %s once every %s", metricsConfig.InfluxDB.Address, metricsConfig.InfluxDB.PushInterval)
}
2017-04-18 06:22:06 +00:00
return metrics.NewMultiRegistry(registries)
}
func stopMetricsClients() {
metrics.StopDatadog()
metrics.StopStatsd()
metrics.StopInfluxDB()
}