Add SO_REUSEPORT support for EntryPoints

This commit is contained in:
Aofei Sheng 2024-01-30 21:56:05 +08:00 committed by GitHub
parent 40de310927
commit d02be003ab
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
19 changed files with 279 additions and 43 deletions

View file

@ -180,6 +180,9 @@ Trust all. (Default: ```false```)
`--entrypoints.<name>.proxyprotocol.trustedips`: `--entrypoints.<name>.proxyprotocol.trustedips`:
Trust only selected IPs. Trust only selected IPs.
`--entrypoints.<name>.reuseport`:
Enables EntryPoints from the same or different processes listening on the same TCP/UDP port. (Default: ```false```)
`--entrypoints.<name>.transport.keepalivemaxrequests`: `--entrypoints.<name>.transport.keepalivemaxrequests`:
Maximum number of requests before closing a keep-alive connection. (Default: ```0```) Maximum number of requests before closing a keep-alive connection. (Default: ```0```)

View file

@ -180,6 +180,9 @@ Trust all. (Default: ```false```)
`TRAEFIK_ENTRYPOINTS_<NAME>_PROXYPROTOCOL_TRUSTEDIPS`: `TRAEFIK_ENTRYPOINTS_<NAME>_PROXYPROTOCOL_TRUSTEDIPS`:
Trust only selected IPs. Trust only selected IPs.
`TRAEFIK_ENTRYPOINTS_<NAME>_REUSEPORT`:
Enables EntryPoints from the same or different processes listening on the same TCP/UDP port. (Default: ```false```)
`TRAEFIK_ENTRYPOINTS_<NAME>_TRANSPORT_KEEPALIVEMAXREQUESTS`: `TRAEFIK_ENTRYPOINTS_<NAME>_TRANSPORT_KEEPALIVEMAXREQUESTS`:
Maximum number of requests before closing a keep-alive connection. (Default: ```0```) Maximum number of requests before closing a keep-alive connection. (Default: ```0```)

View file

@ -30,6 +30,7 @@
[entryPoints] [entryPoints]
[entryPoints.EntryPoint0] [entryPoints.EntryPoint0]
address = "foobar" address = "foobar"
reusePort = true
asDefault = true asDefault = true
[entryPoints.EntryPoint0.transport] [entryPoints.EntryPoint0.transport]
keepAliveMaxTime = "42s" keepAliveMaxTime = "42s"

View file

@ -35,6 +35,7 @@ tcpServersTransport:
entryPoints: entryPoints:
EntryPoint0: EntryPoint0:
address: foobar address: foobar
reusePort: true
asDefault: true asDefault: true
transport: transport:
lifeCycle: lifeCycle:

View file

@ -233,6 +233,79 @@ If both TCP and UDP are wanted for the same port, two entryPoints definitions ar
Full details for how to specify `address` can be found in [net.Listen](https://golang.org/pkg/net/#Listen) (and [net.Dial](https://golang.org/pkg/net/#Dial)) of the doc for go. Full details for how to specify `address` can be found in [net.Listen](https://golang.org/pkg/net/#Listen) (and [net.Dial](https://golang.org/pkg/net/#Dial)) of the doc for go.
### ReusePort
_Optional, Default=false_
The `ReusePort` option enables EntryPoints from the same or different processes
listening on the same TCP/UDP port by utilizing the `SO_REUSEPORT` socket option.
It also allows the kernel to act like a load balancer to distribute incoming
connections between entry points.
For example, you can use it with the [transport.lifeCycle](#lifecycle) to do
canary deployments against Traefik itself. Like upgrading Traefik version or
reloading the static configuration without any service downtime.
!!! warning "Supported platforms"
The `ReusePort` option currently works only on Linux, FreeBSD, OpenBSD and Darwin.
It will be ignored on other platforms.
There is a known bug in the Linux kernel that may cause unintended TCP connection failures when using the `ReusePort` option.
For more details, see https://lwn.net/Articles/853637/.
??? example "Listen on the same port"
```yaml tab="File (yaml)"
entryPoints:
web:
address: ":80"
reusePort: true
```
```toml tab="File (TOML)"
[entryPoints.web]
address = ":80"
reusePort = true
```
```bash tab="CLI"
--entrypoints.web.address=:80
--entrypoints.web.reusePort=true
```
Now it is possible to run multiple Traefik processes with the same EntryPoint configuration.
??? example "Listen on the same port but bind to a different host"
```yaml tab="File (yaml)"
entryPoints:
web:
address: ":80"
reusePort: true
privateWeb:
address: "192.168.1.2:80"
reusePort: true
```
```toml tab="File (TOML)"
[entryPoints.web]
address = ":80"
reusePort = true
[entryPoints.privateWeb]
address = "192.168.1.2:80"
reusePort = true
```
```bash tab="CLI"
--entrypoints.web.address=:80
--entrypoints.web.reusePort=true
--entrypoints.privateWeb.address=192.168.1.2:80
--entrypoints.privateWeb.reusePort=true
```
Requests to `192.168.1.2:80` will only be handled by routers that have `privateWeb` as the entry point.
### AsDefault ### AsDefault
_Optional, Default=false_ _Optional, Default=false_

2
go.mod
View file

@ -80,6 +80,7 @@ require (
golang.org/x/exp v0.0.0-20231006140011-7918f672742d golang.org/x/exp v0.0.0-20231006140011-7918f672742d
golang.org/x/mod v0.13.0 golang.org/x/mod v0.13.0
golang.org/x/net v0.17.0 golang.org/x/net v0.17.0
golang.org/x/sys v0.15.0
golang.org/x/text v0.13.0 golang.org/x/text v0.13.0
golang.org/x/time v0.3.0 golang.org/x/time v0.3.0
golang.org/x/tools v0.14.0 golang.org/x/tools v0.14.0
@ -315,7 +316,6 @@ require (
golang.org/x/arch v0.4.0 // indirect golang.org/x/arch v0.4.0 // indirect
golang.org/x/crypto v0.14.0 // indirect golang.org/x/crypto v0.14.0 // indirect
golang.org/x/oauth2 v0.13.0 // indirect golang.org/x/oauth2 v0.13.0 // indirect
golang.org/x/sys v0.15.0 // indirect
golang.org/x/term v0.13.0 // indirect golang.org/x/term v0.13.0 // indirect
google.golang.org/api v0.128.0 // indirect google.golang.org/api v0.128.0 // indirect
google.golang.org/appengine v1.6.8 // indirect google.golang.org/appengine v1.6.8 // indirect

View file

@ -12,6 +12,7 @@ import (
// EntryPoint holds the entry point configuration. // EntryPoint holds the entry point configuration.
type EntryPoint struct { type EntryPoint struct {
Address string `description:"Entry point address." json:"address,omitempty" toml:"address,omitempty" yaml:"address,omitempty"` Address string `description:"Entry point address." json:"address,omitempty" toml:"address,omitempty" yaml:"address,omitempty"`
ReusePort bool `description:"Enables EntryPoints from the same or different processes listening on the same TCP/UDP port." json:"reusePort,omitempty" toml:"reusePort,omitempty" yaml:"reusePort,omitempty"`
AsDefault bool `description:"Adds this EntryPoint to the list of default EntryPoints to be used on routers that don't have any Entrypoint defined." json:"asDefault,omitempty" toml:"asDefault,omitempty" yaml:"asDefault,omitempty"` AsDefault bool `description:"Adds this EntryPoint to the list of default EntryPoints to be used on routers that don't have any Entrypoint defined." json:"asDefault,omitempty" toml:"asDefault,omitempty" yaml:"asDefault,omitempty"`
Transport *EntryPointsTransport `description:"Configures communication between clients and Traefik." json:"transport,omitempty" toml:"transport,omitempty" yaml:"transport,omitempty" export:"true"` Transport *EntryPointsTransport `description:"Configures communication between clients and Traefik." json:"transport,omitempty" toml:"transport,omitempty" yaml:"transport,omitempty" export:"true"`
ProxyProtocol *ProxyProtocol `description:"Proxy-Protocol configuration." json:"proxyProtocol,omitempty" toml:"proxyProtocol,omitempty" yaml:"proxyProtocol,omitempty" label:"allowEmpty" file:"allowEmpty" export:"true"` ProxyProtocol *ProxyProtocol `description:"Proxy-Protocol configuration." json:"proxyProtocol,omitempty" toml:"proxyProtocol,omitempty" yaml:"proxyProtocol,omitempty" label:"allowEmpty" file:"allowEmpty" export:"true"`

View file

@ -0,0 +1,15 @@
//go:build !(linux || freebsd || openbsd || darwin)
package server
import (
"net"
"github.com/traefik/traefik/v3/pkg/config/static"
)
// newListenConfig creates a new net.ListenConfig for the given configuration of
// the entry point.
func newListenConfig(configuration *static.EntryPoint) (lc net.ListenConfig) {
return
}

View file

@ -0,0 +1,44 @@
//go:build !(linux || freebsd || openbsd || darwin)
package server
import (
"context"
"net"
"testing"
"github.com/stretchr/testify/require"
"github.com/traefik/traefik/v3/pkg/config/static"
)
func TestNewListenConfig(t *testing.T) {
ep := static.EntryPoint{Address: ":0"}
listenConfig := newListenConfig(&ep)
require.Nil(t, listenConfig.Control)
require.Zero(t, listenConfig.KeepAlive)
l1, err := listenConfig.Listen(context.Background(), "tcp", ep.Address)
require.NoError(t, err)
require.NotNil(t, l1)
defer l1.Close()
l2, err := listenConfig.Listen(context.Background(), "tcp", l1.Addr().String())
require.Error(t, err)
require.ErrorContains(t, err, "address already in use")
require.Nil(t, l2)
ep = static.EntryPoint{Address: ":0", ReusePort: true}
listenConfig = newListenConfig(&ep)
require.Nil(t, listenConfig.Control)
require.Zero(t, listenConfig.KeepAlive)
l3, err := listenConfig.Listen(context.Background(), "tcp", ep.Address)
require.NoError(t, err)
require.NotNil(t, l3)
defer l3.Close()
l4, err := listenConfig.Listen(context.Background(), "tcp", l3.Addr().String())
require.Error(t, err)
require.ErrorContains(t, err, "address already in use")
require.Nil(t, l4)
}

View file

@ -0,0 +1,44 @@
//go:build linux || freebsd || openbsd || darwin
package server
import (
"fmt"
"net"
"syscall"
"github.com/traefik/traefik/v3/pkg/config/static"
"golang.org/x/sys/unix"
)
// newListenConfig creates a new net.ListenConfig for the given configuration of
// the entry point.
func newListenConfig(configuration *static.EntryPoint) (lc net.ListenConfig) {
if configuration != nil && configuration.ReusePort {
lc.Control = controlReusePort
}
return
}
// controlReusePort is a net.ListenConfig.Control function that enables SO_REUSEPORT
// on the socket.
func controlReusePort(network, address string, c syscall.RawConn) error {
var setSockOptErr error
err := c.Control(func(fd uintptr) {
// Note that net.ListenConfig enables unix.SO_REUSEADDR by default,
// as seen in https://go.dev/src/net/sockopt_linux.go. Therefore, no
// additional action is required to enable it here.
setSockOptErr = unix.SetsockoptInt(int(fd), unix.SOL_SOCKET, unixSOREUSEPORT, 1)
if setSockOptErr != nil {
return
}
})
if err != nil {
return fmt.Errorf("control: %w", err)
}
if setSockOptErr != nil {
return fmt.Errorf("setsockopt: %w", setSockOptErr)
}
return nil
}

View file

@ -0,0 +1,7 @@
//go:build freebsd
package server
import "golang.org/x/sys/unix"
const unixSOREUSEPORT = unix.SO_REUSEPORT_LB

View file

@ -0,0 +1,7 @@
//go:build linux || openbsd || darwin
package server
import "golang.org/x/sys/unix"
const unixSOREUSEPORT = unix.SO_REUSEPORT

View file

@ -0,0 +1,56 @@
//go:build linux || freebsd || openbsd || darwin
package server
import (
"context"
"net"
"testing"
"github.com/stretchr/testify/require"
"github.com/traefik/traefik/v3/pkg/config/static"
)
func TestNewListenConfig(t *testing.T) {
ep := static.EntryPoint{Address: ":0"}
listenConfig := newListenConfig(&ep)
require.Nil(t, listenConfig.Control)
require.Zero(t, listenConfig.KeepAlive)
l1, err := listenConfig.Listen(context.Background(), "tcp", ep.Address)
require.NoError(t, err)
require.NotNil(t, l1)
defer l1.Close()
l2, err := listenConfig.Listen(context.Background(), "tcp", l1.Addr().String())
require.Error(t, err)
require.ErrorContains(t, err, "address already in use")
require.Nil(t, l2)
ep = static.EntryPoint{Address: ":0", ReusePort: true}
listenConfig = newListenConfig(&ep)
require.NotNil(t, listenConfig.Control)
require.Zero(t, listenConfig.KeepAlive)
l3, err := listenConfig.Listen(context.Background(), "tcp", ep.Address)
require.NoError(t, err)
require.NotNil(t, l3)
defer l3.Close()
l4, err := listenConfig.Listen(context.Background(), "tcp", l3.Addr().String())
require.NoError(t, err)
require.NotNil(t, l4)
defer l4.Close()
_, l3Port, err := net.SplitHostPort(l3.Addr().String())
require.NoError(t, err)
l5, err := listenConfig.Listen(context.Background(), "tcp", "127.0.0.1:"+l3Port)
require.NoError(t, err)
require.NotNil(t, l5)
defer l5.Close()
l6, err := listenConfig.Listen(context.Background(), "tcp", l1.Addr().String())
require.Error(t, err)
require.ErrorContains(t, err, "address already in use")
require.Nil(t, l6)
}

View file

@ -460,7 +460,8 @@ func buildProxyProtocolListener(ctx context.Context, entryPoint *static.EntryPoi
} }
func buildListener(ctx context.Context, entryPoint *static.EntryPoint) (net.Listener, error) { func buildListener(ctx context.Context, entryPoint *static.EntryPoint) (net.Listener, error) {
listener, err := net.Listen("tcp", entryPoint.GetAddress()) listenConfig := newListenConfig(entryPoint)
listener, err := listenConfig.Listen(ctx, "tcp", entryPoint.GetAddress())
if err != nil { if err != nil {
return nil, fmt.Errorf("error opening listener: %w", err) return nil, fmt.Errorf("error opening listener: %w", err)
} }

View file

@ -33,7 +33,8 @@ func newHTTP3Server(ctx context.Context, configuration *static.EntryPoint, https
return nil, errors.New("advertised port must be greater than or equal to zero") return nil, errors.New("advertised port must be greater than or equal to zero")
} }
conn, err := net.ListenPacket("udp", configuration.GetAddress()) listenConfig := newListenConfig(configuration)
conn, err := listenConfig.ListenPacket(ctx, "udp", configuration.GetAddress())
if err != nil { if err != nil {
return nil, fmt.Errorf("starting listener: %w", err) return nil, fmt.Errorf("starting listener: %w", err)
} }

View file

@ -3,7 +3,6 @@ package server
import ( import (
"context" "context"
"fmt" "fmt"
"net"
"sync" "sync"
"time" "time"
@ -87,12 +86,8 @@ type UDPEntryPoint struct {
// NewUDPEntryPoint returns a UDP entry point. // NewUDPEntryPoint returns a UDP entry point.
func NewUDPEntryPoint(cfg *static.EntryPoint) (*UDPEntryPoint, error) { func NewUDPEntryPoint(cfg *static.EntryPoint) (*UDPEntryPoint, error) {
addr, err := net.ResolveUDPAddr("udp", cfg.GetAddress()) listenConfig := newListenConfig(cfg)
if err != nil { listener, err := udp.Listen(listenConfig, "udp", cfg.GetAddress(), time.Duration(cfg.UDP.Timeout))
return nil, err
}
listener, err := udp.Listen("udp", addr, time.Duration(cfg.UDP.Timeout))
if err != nil { if err != nil {
return nil, err return nil, err
} }

View file

@ -1,7 +1,9 @@
package udp package udp
import ( import (
"context"
"errors" "errors"
"fmt"
"io" "io"
"net" "net"
"sync" "sync"
@ -33,18 +35,22 @@ type Listener struct {
} }
// Listen creates a new listener. // Listen creates a new listener.
func Listen(network string, laddr *net.UDPAddr, timeout time.Duration) (*Listener, error) { func Listen(listenConfig net.ListenConfig, network, address string, timeout time.Duration) (*Listener, error) {
if timeout <= 0 { if timeout <= 0 {
return nil, errors.New("timeout should be greater than zero") return nil, errors.New("timeout should be greater than zero")
} }
conn, err := net.ListenUDP(network, laddr) packetConn, err := listenConfig.ListenPacket(context.Background(), network, address)
if err != nil { if err != nil {
return nil, err return nil, fmt.Errorf("listen packet: %w", err)
}
pConn, ok := packetConn.(*net.UDPConn)
if !ok {
return nil, errors.New("packet conn is not an UDPConn")
} }
l := &Listener{ l := &Listener{
pConn: conn, pConn: pConn,
acceptCh: make(chan *Conn), acceptCh: make(chan *Conn),
conns: make(map[string]*Conn), conns: make(map[string]*Conn),
accepting: true, accepting: true,

View file

@ -14,10 +14,7 @@ import (
) )
func TestConsecutiveWrites(t *testing.T) { func TestConsecutiveWrites(t *testing.T) {
addr, err := net.ResolveUDPAddr("udp", ":0") ln, err := Listen(net.ListenConfig{}, "udp", ":0", 3*time.Second)
require.NoError(t, err)
ln, err := Listen("udp", addr, 3*time.Second)
require.NoError(t, err) require.NoError(t, err)
defer func() { defer func() {
err := ln.Close() err := ln.Close()
@ -75,11 +72,7 @@ func TestConsecutiveWrites(t *testing.T) {
} }
func TestListenNotBlocking(t *testing.T) { func TestListenNotBlocking(t *testing.T) {
addr, err := net.ResolveUDPAddr("udp", ":0") ln, err := Listen(net.ListenConfig{}, "udp", ":0", 3*time.Second)
require.NoError(t, err)
ln, err := Listen("udp", addr, 3*time.Second)
require.NoError(t, err) require.NoError(t, err)
defer func() { defer func() {
err := ln.Close() err := ln.Close()
@ -165,10 +158,7 @@ func TestListenNotBlocking(t *testing.T) {
} }
func TestListenWithZeroTimeout(t *testing.T) { func TestListenWithZeroTimeout(t *testing.T) {
addr, err := net.ResolveUDPAddr("udp", ":0") _, err := Listen(net.ListenConfig{}, "udp", ":0", 0)
require.NoError(t, err)
_, err = Listen("udp", addr, 0)
assert.Error(t, err) assert.Error(t, err)
} }
@ -183,10 +173,7 @@ func TestTimeoutWithoutRead(t *testing.T) {
func testTimeout(t *testing.T, withRead bool) { func testTimeout(t *testing.T, withRead bool) {
t.Helper() t.Helper()
addr, err := net.ResolveUDPAddr("udp", ":0") ln, err := Listen(net.ListenConfig{}, "udp", ":0", 3*time.Second)
require.NoError(t, err)
ln, err := Listen("udp", addr, 3*time.Second)
require.NoError(t, err) require.NoError(t, err)
defer func() { defer func() {
err := ln.Close() err := ln.Close()
@ -227,10 +214,7 @@ func testTimeout(t *testing.T, withRead bool) {
} }
func TestShutdown(t *testing.T) { func TestShutdown(t *testing.T) {
addr, err := net.ResolveUDPAddr("udp", ":0") l, err := Listen(net.ListenConfig{}, "udp", ":0", 3*time.Second)
require.NoError(t, err)
l, err := Listen("udp", addr, 3*time.Second)
require.NoError(t, err) require.NoError(t, err)
go func() { go func() {
@ -331,10 +315,7 @@ func TestReadLoopMaxDataSize(t *testing.T) {
doneCh := make(chan struct{}) doneCh := make(chan struct{})
addr, err := net.ResolveUDPAddr("udp", ":0") l, err := Listen(net.ListenConfig{}, "udp", ":0", 3*time.Second)
require.NoError(t, err)
l, err := Listen("udp", addr, 3*time.Second)
require.NoError(t, err) require.NoError(t, err)
defer func() { defer func() {

View file

@ -96,10 +96,7 @@ func TestProxy_ServeUDP_MaxDataSize(t *testing.T) {
func newServer(t *testing.T, addr string, handler Handler) { func newServer(t *testing.T, addr string, handler Handler) {
t.Helper() t.Helper()
addrL, err := net.ResolveUDPAddr("udp", addr) listener, err := Listen(net.ListenConfig{}, "udp", addr, 3*time.Second)
require.NoError(t, err)
listener, err := Listen("udp", addrL, 3*time.Second)
require.NoError(t, err) require.NoError(t, err)
for { for {