package server import ( "context" "crypto/tls" "crypto/x509" "fmt" stdlog "log" "net" "net/http" "sync" "time" "github.com/armon/go-proxyproto" "github.com/containous/traefik/config/static" "github.com/containous/traefik/h2c" "github.com/containous/traefik/ip" "github.com/containous/traefik/log" "github.com/containous/traefik/middlewares" "github.com/containous/traefik/middlewares/forwardedheaders" "github.com/containous/traefik/old/configuration" traefiktls "github.com/containous/traefik/tls" "github.com/containous/traefik/tls/generate" "github.com/containous/traefik/types" "github.com/sirupsen/logrus" "github.com/xenolf/lego/challenge/tlsalpn01" ) // EntryPoints map of EntryPoint type EntryPoints map[string]*EntryPoint // NewEntryPoint creates a new EntryPoint func NewEntryPoint(ctx context.Context, configuration *static.EntryPoint) (*EntryPoint, error) { var err error switcher := middlewares.NewHandlerSwitcher(buildDefaultHTTPRouter()) handler, err := forwardedheaders.NewXForwarded( configuration.ForwardedHeaders.Insecure, configuration.ForwardedHeaders.TrustedIPs, switcher) if err != nil { return nil, err } tracker := newHijackConnectionTracker() listener, err := buildListener(ctx, configuration) if err != nil { return nil, fmt.Errorf("error preparing server: %v", err) } var tlsConfig *tls.Config var certificateStore *traefiktls.CertificateStore if configuration.TLS != nil { certificateStore, err = buildCertificateStore(*configuration.TLS) if err != nil { return nil, fmt.Errorf("error creating certificate store: %v", err) } tlsConfig, err = buildTLSConfig(*configuration.TLS) if err != nil { return nil, fmt.Errorf("error creating TLS config: %v", err) } } entryPoint := &EntryPoint{ switcher: switcher, transportConfiguration: configuration.Transport, hijackConnectionTracker: tracker, listener: listener, httpServer: buildServer(ctx, configuration, tlsConfig, handler, tracker), Certs: certificateStore, } if tlsConfig != nil { tlsConfig.GetCertificate = entryPoint.getCertificate } return entryPoint, nil } // EntryPoint holds everything about the entry point (httpServer, listener etc...) type EntryPoint struct { RouteAppenderFactory RouteAppenderFactory httpServer *h2c.Server listener net.Listener switcher *middlewares.HandlerSwitcher Certs *traefiktls.CertificateStore OnDemandListener func(string) (*tls.Certificate, error) TLSALPNGetter func(string) (*tls.Certificate, error) hijackConnectionTracker *hijackConnectionTracker transportConfiguration *static.EntryPointsTransport } // Start starts listening for traffic func (s *EntryPoint) Start(ctx context.Context) { logger := log.FromContext(ctx) logger.Infof("Starting server on %s", s.httpServer.Addr) var err error if s.httpServer.TLSConfig != nil { err = s.httpServer.ServeTLS(s.listener, "", "") } else { err = s.httpServer.Serve(s.listener) } if err != http.ErrServerClosed { logger.Error("Cannot start server: %v", err) } } // Shutdown handles the entrypoint shutdown process func (s EntryPoint) Shutdown(ctx context.Context) { logger := log.FromContext(ctx) reqAcceptGraceTimeOut := time.Duration(s.transportConfiguration.LifeCycle.RequestAcceptGraceTimeout) if reqAcceptGraceTimeOut > 0 { logger.Infof("Waiting %s for incoming requests to cease", reqAcceptGraceTimeOut) time.Sleep(reqAcceptGraceTimeOut) } graceTimeOut := time.Duration(s.transportConfiguration.LifeCycle.GraceTimeOut) ctx, cancel := context.WithTimeout(ctx, graceTimeOut) logger.Debugf("Waiting %s seconds before killing connections.", graceTimeOut) var wg sync.WaitGroup if s.httpServer != nil { wg.Add(1) go func() { defer wg.Done() if err := s.httpServer.Shutdown(ctx); err != nil { if ctx.Err() == context.DeadlineExceeded { logger.Debugf("Wait server shutdown is overdue to: %s", err) err = s.httpServer.Close() if err != nil { logger.Error(err) } } } }() } if s.hijackConnectionTracker != nil { wg.Add(1) go func() { defer wg.Done() if err := s.hijackConnectionTracker.Shutdown(ctx); err != nil { if ctx.Err() == context.DeadlineExceeded { logger.Debugf("Wait hijack connection is overdue to: %s", err) s.hijackConnectionTracker.Close() } } }() } wg.Wait() cancel() } // getCertificate allows to customize tlsConfig.GetCertificate behavior to get the certificates inserted dynamically func (s *EntryPoint) getCertificate(clientHello *tls.ClientHelloInfo) (*tls.Certificate, error) { domainToCheck := types.CanonicalDomain(clientHello.ServerName) if s.TLSALPNGetter != nil { cert, err := s.TLSALPNGetter(domainToCheck) if err != nil { return nil, err } if cert != nil { return cert, nil } } bestCertificate := s.Certs.GetBestCertificate(clientHello) if bestCertificate != nil { return bestCertificate, nil } if s.OnDemandListener != nil && len(domainToCheck) > 0 { // Only check for an onDemandCert if there is a domain name return s.OnDemandListener(domainToCheck) } if s.Certs.SniStrict { return nil, fmt.Errorf("strict SNI enabled - No certificate found for domain: %q, closing connection", domainToCheck) } log.WithoutContext().Debugf("Serving default certificate for request: %q", domainToCheck) return s.Certs.DefaultCertificate, nil } func newHijackConnectionTracker() *hijackConnectionTracker { return &hijackConnectionTracker{ conns: make(map[net.Conn]struct{}), } } type hijackConnectionTracker struct { conns map[net.Conn]struct{} lock sync.RWMutex } // AddHijackedConnection add a connection in the tracked connections list func (h *hijackConnectionTracker) AddHijackedConnection(conn net.Conn) { h.lock.Lock() defer h.lock.Unlock() h.conns[conn] = struct{}{} } // RemoveHijackedConnection remove a connection from the tracked connections list func (h *hijackConnectionTracker) RemoveHijackedConnection(conn net.Conn) { h.lock.Lock() defer h.lock.Unlock() delete(h.conns, conn) } // Shutdown wait for the connection closing func (h *hijackConnectionTracker) Shutdown(ctx context.Context) error { ticker := time.NewTicker(500 * time.Millisecond) defer ticker.Stop() for { h.lock.RLock() if len(h.conns) == 0 { return nil } h.lock.RUnlock() select { case <-ctx.Done(): return ctx.Err() case <-ticker.C: } } } // Close close all the connections in the tracked connections list func (h *hijackConnectionTracker) Close() { for conn := range h.conns { if err := conn.Close(); err != nil { log.WithoutContext().Errorf("Error while closing Hijacked connection: %v", err) } delete(h.conns, conn) } } // tcpKeepAliveListener sets TCP keep-alive timeouts on accepted // connections. type tcpKeepAliveListener struct { *net.TCPListener } func (ln tcpKeepAliveListener) Accept() (net.Conn, error) { tc, err := ln.AcceptTCP() if err != nil { return nil, err } if err = tc.SetKeepAlive(true); err != nil { return nil, err } if err = tc.SetKeepAlivePeriod(3 * time.Minute); err != nil { return nil, err } return tc, nil } func buildProxyProtocolListener(ctx context.Context, entryPoint *static.EntryPoint, listener net.Listener) (net.Listener, error) { var sourceCheck func(addr net.Addr) (bool, error) if entryPoint.ProxyProtocol.Insecure { sourceCheck = func(_ net.Addr) (bool, error) { return true, nil } } else { checker, err := ip.NewChecker(entryPoint.ProxyProtocol.TrustedIPs) if err != nil { return nil, err } sourceCheck = func(addr net.Addr) (bool, error) { ipAddr, ok := addr.(*net.TCPAddr) if !ok { return false, fmt.Errorf("type error %v", addr) } return checker.ContainsIP(ipAddr.IP), nil } } log.FromContext(ctx).Infof("Enabling ProxyProtocol for trusted IPs %v", entryPoint.ProxyProtocol.TrustedIPs) return &proxyproto.Listener{ Listener: listener, SourceCheck: sourceCheck, }, nil } func buildServerTimeouts(entryPointsTransport static.EntryPointsTransport) (readTimeout, writeTimeout, idleTimeout time.Duration) { readTimeout = time.Duration(0) writeTimeout = time.Duration(0) if entryPointsTransport.RespondingTimeouts != nil { readTimeout = time.Duration(entryPointsTransport.RespondingTimeouts.ReadTimeout) writeTimeout = time.Duration(entryPointsTransport.RespondingTimeouts.WriteTimeout) } if entryPointsTransport.RespondingTimeouts != nil { idleTimeout = time.Duration(entryPointsTransport.RespondingTimeouts.IdleTimeout) } else { idleTimeout = configuration.DefaultIdleTimeout } return readTimeout, writeTimeout, idleTimeout } func buildListener(ctx context.Context, entryPoint *static.EntryPoint) (net.Listener, error) { listener, err := net.Listen("tcp", entryPoint.Address) if err != nil { return nil, fmt.Errorf("error opening listener: %v", err) } listener = tcpKeepAliveListener{listener.(*net.TCPListener)} if entryPoint.ProxyProtocol != nil { listener, err = buildProxyProtocolListener(ctx, entryPoint, listener) if err != nil { return nil, fmt.Errorf("error creating proxy protocol listener: %v", err) } } return listener, nil } func buildCertificateStore(tlsOption traefiktls.TLS) (*traefiktls.CertificateStore, error) { certificateStore := traefiktls.NewCertificateStore() certificateStore.DynamicCerts.Set(make(map[string]*tls.Certificate)) certificateStore.SniStrict = tlsOption.SniStrict if tlsOption.DefaultCertificate != nil { cert, err := buildDefaultCertificate(tlsOption.DefaultCertificate) if err != nil { return nil, err } certificateStore.DefaultCertificate = cert } else { cert, err := generate.DefaultCertificate() if err != nil { return nil, err } certificateStore.DefaultCertificate = cert } return certificateStore, nil } func buildServer(ctx context.Context, configuration *static.EntryPoint, tlsConfig *tls.Config, router http.Handler, tracker *hijackConnectionTracker) *h2c.Server { logger := log.FromContext(ctx) readTimeout, writeTimeout, idleTimeout := buildServerTimeouts(*configuration.Transport) logger. WithField("readTimeout", readTimeout). WithField("writeTimeout", writeTimeout). WithField("idleTimeout", idleTimeout). Infof("Preparing server") return &h2c.Server{ Server: &http.Server{ Addr: configuration.Address, Handler: router, TLSConfig: tlsConfig, ReadTimeout: readTimeout, WriteTimeout: writeTimeout, IdleTimeout: idleTimeout, ErrorLog: stdlog.New(logger.WriterLevel(logrus.DebugLevel), "", 0), ConnState: func(conn net.Conn, state http.ConnState) { switch state { case http.StateHijacked: tracker.AddHijackedConnection(conn) case http.StateClosed: tracker.RemoveHijackedConnection(conn) } }, }, } } // creates a TLS config that allows terminating HTTPS for multiple domains using SNI func buildTLSConfig(tlsOption traefiktls.TLS) (*tls.Config, error) { conf := &tls.Config{} // ensure http2 enabled conf.NextProtos = []string{"h2", "http/1.1", tlsalpn01.ACMETLS1Protocol} if len(tlsOption.ClientCA.Files) > 0 { pool := x509.NewCertPool() for _, caFile := range tlsOption.ClientCA.Files { data, err := caFile.Read() if err != nil { return nil, err } ok := pool.AppendCertsFromPEM(data) if !ok { return nil, fmt.Errorf("invalid certificate(s) in %s", caFile) } } conf.ClientCAs = pool if tlsOption.ClientCA.Optional { conf.ClientAuth = tls.VerifyClientCertIfGiven } else { conf.ClientAuth = tls.RequireAndVerifyClientCert } } // Set the minimum TLS version if set in the config TOML if minConst, exists := traefiktls.MinVersion[tlsOption.MinVersion]; exists { conf.PreferServerCipherSuites = true conf.MinVersion = minConst } // Set the list of CipherSuites if set in the config TOML if tlsOption.CipherSuites != nil { // if our list of CipherSuites is defined in the entryPoint config, we can re-initialize the suites list as empty conf.CipherSuites = make([]uint16, 0) for _, cipher := range tlsOption.CipherSuites { if cipherConst, exists := traefiktls.CipherSuites[cipher]; exists { conf.CipherSuites = append(conf.CipherSuites, cipherConst) } else { // CipherSuite listed in the toml does not exist in our listed return nil, fmt.Errorf("invalid CipherSuite: %s", cipher) } } } return conf, nil }