389 lines
9.4 KiB
Go
389 lines
9.4 KiB
Go
package server
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"net"
|
|
"net/http"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/armon/go-proxyproto"
|
|
"github.com/containous/traefik/pkg/config/static"
|
|
"github.com/containous/traefik/pkg/ip"
|
|
"github.com/containous/traefik/pkg/log"
|
|
"github.com/containous/traefik/pkg/middlewares"
|
|
"github.com/containous/traefik/pkg/middlewares/forwardedheaders"
|
|
"github.com/containous/traefik/pkg/safe"
|
|
"github.com/containous/traefik/pkg/tcp"
|
|
"golang.org/x/net/http2"
|
|
"golang.org/x/net/http2/h2c"
|
|
)
|
|
|
|
type httpForwarder struct {
|
|
net.Listener
|
|
connChan chan net.Conn
|
|
}
|
|
|
|
func newHTTPForwarder(ln net.Listener) *httpForwarder {
|
|
return &httpForwarder{
|
|
Listener: ln,
|
|
connChan: make(chan net.Conn),
|
|
}
|
|
}
|
|
|
|
// ServeTCP uses the connection to serve it later in "Accept"
|
|
func (h *httpForwarder) ServeTCP(conn net.Conn) {
|
|
h.connChan <- conn
|
|
}
|
|
|
|
// Accept retrieves a served connection in ServeTCP
|
|
func (h *httpForwarder) Accept() (net.Conn, error) {
|
|
conn := <-h.connChan
|
|
return conn, nil
|
|
}
|
|
|
|
// TCPEntryPoints holds a map of TCPEntryPoint (the entrypoint names being the keys)
|
|
type TCPEntryPoints map[string]*TCPEntryPoint
|
|
|
|
// TCPEntryPoint is the TCP server
|
|
type TCPEntryPoint struct {
|
|
listener net.Listener
|
|
switcher *tcp.HandlerSwitcher
|
|
RouteAppenderFactory RouteAppenderFactory
|
|
transportConfiguration *static.EntryPointsTransport
|
|
tracker *connectionTracker
|
|
httpServer *httpServer
|
|
httpsServer *httpServer
|
|
}
|
|
|
|
// NewTCPEntryPoint creates a new TCPEntryPoint
|
|
func NewTCPEntryPoint(ctx context.Context, configuration *static.EntryPoint) (*TCPEntryPoint, error) {
|
|
tracker := newConnectionTracker()
|
|
|
|
listener, err := buildListener(ctx, configuration)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("error preparing server: %v", err)
|
|
}
|
|
|
|
router := &tcp.Router{}
|
|
|
|
httpServer, err := createHTTPServer(listener, configuration, true)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("error preparing httpServer: %v", err)
|
|
}
|
|
|
|
router.HTTPForwarder(httpServer.Forwarder)
|
|
|
|
httpsServer, err := createHTTPServer(listener, configuration, false)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("error preparing httpsServer: %v", err)
|
|
}
|
|
|
|
router.HTTPSForwarder(httpsServer.Forwarder)
|
|
|
|
tcpSwitcher := &tcp.HandlerSwitcher{}
|
|
tcpSwitcher.Switch(router)
|
|
|
|
return &TCPEntryPoint{
|
|
listener: listener,
|
|
switcher: tcpSwitcher,
|
|
transportConfiguration: configuration.Transport,
|
|
tracker: tracker,
|
|
httpServer: httpServer,
|
|
httpsServer: httpsServer,
|
|
}, nil
|
|
}
|
|
|
|
func (e *TCPEntryPoint) startTCP(ctx context.Context) {
|
|
log.FromContext(ctx).Debugf("Start TCP Server")
|
|
|
|
for {
|
|
conn, err := e.listener.Accept()
|
|
if err != nil {
|
|
log.Error(err)
|
|
return
|
|
}
|
|
|
|
safe.Go(func() {
|
|
e.switcher.ServeTCP(newTrackedConnection(conn, e.tracker))
|
|
})
|
|
}
|
|
}
|
|
|
|
// Shutdown stops the TCP connections
|
|
func (e *TCPEntryPoint) Shutdown(ctx context.Context) {
|
|
logger := log.FromContext(ctx)
|
|
|
|
reqAcceptGraceTimeOut := time.Duration(e.transportConfiguration.LifeCycle.RequestAcceptGraceTimeout)
|
|
if reqAcceptGraceTimeOut > 0 {
|
|
logger.Infof("Waiting %s for incoming requests to cease", reqAcceptGraceTimeOut)
|
|
time.Sleep(reqAcceptGraceTimeOut)
|
|
}
|
|
|
|
graceTimeOut := time.Duration(e.transportConfiguration.LifeCycle.GraceTimeOut)
|
|
ctx, cancel := context.WithTimeout(ctx, graceTimeOut)
|
|
logger.Debugf("Waiting %s seconds before killing connections.", graceTimeOut)
|
|
|
|
var wg sync.WaitGroup
|
|
if e.httpServer.Server != nil {
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
if err := e.httpServer.Server.Shutdown(ctx); err != nil {
|
|
if ctx.Err() == context.DeadlineExceeded {
|
|
logger.Debugf("Wait server shutdown is overdue to: %s", err)
|
|
err = e.httpServer.Server.Close()
|
|
if err != nil {
|
|
logger.Error(err)
|
|
}
|
|
}
|
|
}
|
|
}()
|
|
}
|
|
|
|
if e.httpsServer.Server != nil {
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
if err := e.httpsServer.Server.Shutdown(ctx); err != nil {
|
|
if ctx.Err() == context.DeadlineExceeded {
|
|
logger.Debugf("Wait server shutdown is overdue to: %s", err)
|
|
err = e.httpsServer.Server.Close()
|
|
if err != nil {
|
|
logger.Error(err)
|
|
}
|
|
}
|
|
}
|
|
}()
|
|
}
|
|
|
|
if e.tracker != nil {
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
if err := e.tracker.Shutdown(ctx); err != nil {
|
|
if ctx.Err() == context.DeadlineExceeded {
|
|
logger.Debugf("Wait hijack connection is overdue to: %s", err)
|
|
e.tracker.Close()
|
|
}
|
|
}
|
|
}()
|
|
}
|
|
|
|
wg.Wait()
|
|
cancel()
|
|
}
|
|
|
|
func (e *TCPEntryPoint) switchRouter(router *tcp.Router) {
|
|
router.HTTPForwarder(e.httpServer.Forwarder)
|
|
router.HTTPSForwarder(e.httpsServer.Forwarder)
|
|
|
|
httpHandler := router.GetHTTPHandler()
|
|
httpsHandler := router.GetHTTPSHandler()
|
|
if httpHandler == nil {
|
|
httpHandler = buildDefaultHTTPRouter()
|
|
}
|
|
if httpsHandler == nil {
|
|
httpsHandler = buildDefaultHTTPRouter()
|
|
}
|
|
|
|
e.httpServer.Switcher.UpdateHandler(httpHandler)
|
|
e.httpsServer.Switcher.UpdateHandler(httpsHandler)
|
|
e.switcher.Switch(router)
|
|
}
|
|
|
|
// tcpKeepAliveListener sets TCP keep-alive timeouts on accepted
|
|
// connections.
|
|
type tcpKeepAliveListener struct {
|
|
*net.TCPListener
|
|
}
|
|
|
|
func (ln tcpKeepAliveListener) Accept() (net.Conn, error) {
|
|
tc, err := ln.AcceptTCP()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if err = tc.SetKeepAlive(true); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if err = tc.SetKeepAlivePeriod(3 * time.Minute); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return tc, nil
|
|
}
|
|
|
|
func buildProxyProtocolListener(ctx context.Context, entryPoint *static.EntryPoint, listener net.Listener) (net.Listener, error) {
|
|
var sourceCheck func(addr net.Addr) (bool, error)
|
|
if entryPoint.ProxyProtocol.Insecure {
|
|
sourceCheck = func(_ net.Addr) (bool, error) {
|
|
return true, nil
|
|
}
|
|
} else {
|
|
checker, err := ip.NewChecker(entryPoint.ProxyProtocol.TrustedIPs)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
sourceCheck = func(addr net.Addr) (bool, error) {
|
|
ipAddr, ok := addr.(*net.TCPAddr)
|
|
if !ok {
|
|
return false, fmt.Errorf("type error %v", addr)
|
|
}
|
|
|
|
return checker.ContainsIP(ipAddr.IP), nil
|
|
}
|
|
}
|
|
|
|
log.FromContext(ctx).Infof("Enabling ProxyProtocol for trusted IPs %v", entryPoint.ProxyProtocol.TrustedIPs)
|
|
|
|
return &proxyproto.Listener{
|
|
Listener: listener,
|
|
SourceCheck: sourceCheck,
|
|
}, nil
|
|
}
|
|
|
|
func buildListener(ctx context.Context, entryPoint *static.EntryPoint) (net.Listener, error) {
|
|
listener, err := net.Listen("tcp", entryPoint.Address)
|
|
|
|
if err != nil {
|
|
return nil, fmt.Errorf("error opening listener: %v", err)
|
|
}
|
|
|
|
listener = tcpKeepAliveListener{listener.(*net.TCPListener)}
|
|
|
|
if entryPoint.ProxyProtocol != nil {
|
|
listener, err = buildProxyProtocolListener(ctx, entryPoint, listener)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("error creating proxy protocol listener: %v", err)
|
|
}
|
|
}
|
|
return listener, nil
|
|
}
|
|
|
|
func newConnectionTracker() *connectionTracker {
|
|
return &connectionTracker{
|
|
conns: make(map[net.Conn]struct{}),
|
|
}
|
|
}
|
|
|
|
type connectionTracker struct {
|
|
conns map[net.Conn]struct{}
|
|
lock sync.RWMutex
|
|
}
|
|
|
|
// AddConnection add a connection in the tracked connections list
|
|
func (c *connectionTracker) AddConnection(conn net.Conn) {
|
|
c.lock.Lock()
|
|
defer c.lock.Unlock()
|
|
c.conns[conn] = struct{}{}
|
|
}
|
|
|
|
// RemoveConnection remove a connection from the tracked connections list
|
|
func (c *connectionTracker) RemoveConnection(conn net.Conn) {
|
|
c.lock.Lock()
|
|
defer c.lock.Unlock()
|
|
delete(c.conns, conn)
|
|
}
|
|
|
|
func (c *connectionTracker) isEmpty() bool {
|
|
c.lock.RLock()
|
|
defer c.lock.RUnlock()
|
|
return len(c.conns) == 0
|
|
}
|
|
|
|
// Shutdown wait for the connection closing
|
|
func (c *connectionTracker) Shutdown(ctx context.Context) error {
|
|
ticker := time.NewTicker(500 * time.Millisecond)
|
|
defer ticker.Stop()
|
|
for {
|
|
if c.isEmpty() {
|
|
return nil
|
|
}
|
|
select {
|
|
case <-ctx.Done():
|
|
return ctx.Err()
|
|
case <-ticker.C:
|
|
}
|
|
}
|
|
}
|
|
|
|
// Close close all the connections in the tracked connections list
|
|
func (c *connectionTracker) Close() {
|
|
c.lock.Lock()
|
|
defer c.lock.Unlock()
|
|
for conn := range c.conns {
|
|
if err := conn.Close(); err != nil {
|
|
log.WithoutContext().Errorf("Error while closing connection: %v", err)
|
|
}
|
|
delete(c.conns, conn)
|
|
}
|
|
}
|
|
|
|
type stoppableServer interface {
|
|
Shutdown(context.Context) error
|
|
Close() error
|
|
Serve(listener net.Listener) error
|
|
}
|
|
|
|
type httpServer struct {
|
|
Server stoppableServer
|
|
Forwarder *httpForwarder
|
|
Switcher *middlewares.HTTPHandlerSwitcher
|
|
}
|
|
|
|
func createHTTPServer(ln net.Listener, configuration *static.EntryPoint, withH2c bool) (*httpServer, error) {
|
|
httpSwitcher := middlewares.NewHandlerSwitcher(buildDefaultHTTPRouter())
|
|
|
|
var handler http.Handler
|
|
var err error
|
|
handler, err = forwardedheaders.NewXForwarded(
|
|
configuration.ForwardedHeaders.Insecure,
|
|
configuration.ForwardedHeaders.TrustedIPs,
|
|
httpSwitcher)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if withH2c {
|
|
handler = h2c.NewHandler(handler, &http2.Server{})
|
|
}
|
|
|
|
serverHTTP := &http.Server{
|
|
Handler: handler,
|
|
}
|
|
|
|
listener := newHTTPForwarder(ln)
|
|
go func() {
|
|
err := serverHTTP.Serve(listener)
|
|
if err != nil {
|
|
log.Errorf("Error while starting server: %v", err)
|
|
}
|
|
}()
|
|
return &httpServer{
|
|
Server: serverHTTP,
|
|
Forwarder: listener,
|
|
Switcher: httpSwitcher,
|
|
}, nil
|
|
}
|
|
|
|
func newTrackedConnection(conn net.Conn, tracker *connectionTracker) *trackedConnection {
|
|
tracker.AddConnection(conn)
|
|
return &trackedConnection{
|
|
Conn: conn,
|
|
tracker: tracker,
|
|
}
|
|
}
|
|
|
|
type trackedConnection struct {
|
|
tracker *connectionTracker
|
|
net.Conn
|
|
}
|
|
|
|
func (t *trackedConnection) Close() error {
|
|
t.tracker.RemoveConnection(t.Conn)
|
|
return t.Conn.Close()
|
|
}
|