baalajimaestro/traefik
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

Improve Kubernetes GatewayAPI TCPRoute and TLSRoute support

Browse source
This commit is contained in:
Romain 2024-09-03 12:10:04 +02:00 committed by GitHub
parent 0b34e0cdcb
commit 3eb7ecce19
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 690 additions and 484 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
integration/conformance-reports/v1.1.0/experimental-v3.1-default-report.yaml
View file

@ -53,3 +53,11 @@ profiles:
- HTTPRouteRequestTimeout
name: GATEWAY-HTTP
summary: Core tests succeeded. Extended tests succeeded.
- core:
result: success
statistics:
Failed: 0
Passed: 11
Skipped: 0
name: GATEWAY-TLS
summary: Core tests succeeded.

13
integration/k8s_conformance_test.go
View file

@ -8,6 +8,7 @@ import (
"os"
"path/filepath"
"slices"
"strings"
"testing"
"time"
@ -193,7 +194,11 @@ func (s *K8sConformanceSuite) TestK8sGatewayAPIConformance() {
Version: *k8sConformanceTraefikVersion,
Contact: []string{"@traefik/maintainers"},
},
ConformanceProfiles: sets.New(ksuite.GatewayHTTPConformanceProfileName, ksuite.GatewayGRPCConformanceProfileName),
ConformanceProfiles: sets.New(
ksuite.GatewayHTTPConformanceProfileName,
ksuite.GatewayGRPCConformanceProfileName,
ksuite.GatewayTLSConformanceProfileName,
),
SupportedFeatures: sets.New(
features.SupportGateway,
features.SupportGatewayPort8080,
@ -207,6 +212,7 @@ func (s *K8sConformanceSuite) TestK8sGatewayAPIConformance() {
features.SupportHTTPRoutePathRewrite,
features.SupportHTTPRoutePathRedirect,
features.SupportHTTPRouteResponseHeaderModification,
features.SupportTLSRoute,
),
})
require.NoError(s.T(), err)
@ -224,6 +230,11 @@ func (s *K8sConformanceSuite) TestK8sGatewayAPIConformance() {
// TODO: to publish this report automatically, we have to figure out how to handle the date diff.
report.Date = "-"
// Ordering profile reports for the serialized report to be comparable.
slices.SortFunc(report.ProfileReports, func(a, b v1.ProfileReport) int {
return strings.Compare(a.Name, b.Name)
})
rawReport, err := yaml.Marshal(report)
require.NoError(s.T(), err)
s.T().Logf("Conformance report:\n%s", string(rawReport))

32
integration/testdata/rawdata-gateway.json vendored
View file

@ -34,7 +34,7 @@
"entryPoints": [
"web"
],
"service": "default-http-app-1-my-gateway-web-0-wrr",
"service": "default-http-app-1-my-gateway-web-0-1c0cf64bde37d9d0df06-wrr",
"rule": "Host(`foo.com`) \u0026\u0026 Path(`/bar`)",
"ruleSyntax": "v3",
"priority": 100008,
@ -47,7 +47,7 @@
"entryPoints": [
"websecure"
],
"service": "default-http-app-1-my-https-gateway-websecure-0-wrr",
"service": "default-http-app-1-my-https-gateway-websecure-0-1c0cf64bde37d9d0df06-wrr",
"rule": "Host(`foo.com`) \u0026\u0026 Path(`/bar`)",
"ruleSyntax": "v3",
"priority": 100008,
@ -96,7 +96,7 @@
"dashboard@internal"
]
},
"default-http-app-1-my-gateway-web-0-wrr@kubernetesgateway": {
"default-http-app-1-my-gateway-web-0-1c0cf64bde37d9d0df06-wrr@kubernetesgateway": {
"weighted": {
"services": [
{
@ -110,7 +110,7 @@
"default-http-app-1-my-gateway-web-0-1c0cf64bde37d9d0df06@kubernetesgateway"
]
},
"default-http-app-1-my-https-gateway-websecure-0-wrr@kubernetesgateway": {
"default-http-app-1-my-https-gateway-websecure-0-1c0cf64bde37d9d0df06-wrr@kubernetesgateway": {
"weighted": {
"services": [
{
@ -150,11 +150,11 @@
}
},
"tcpRouters": {
"default-tcp-app-1-my-tcp-gateway-footcp@kubernetesgateway": {
"default-tcp-app-1-my-tcp-gateway-footcp-0-e3b0c44298fc1c149afb@kubernetesgateway": {
"entryPoints": [
"footcp"
],
"service": "default-tcp-app-1-my-tcp-gateway-footcp-wrr-0",
"service": "default-tcp-app-1-my-tcp-gateway-footcp-0-e3b0c44298fc1c149afb-wrr",
"rule": "HostSNI(`*`)",
"ruleSyntax": "v3",
"priority": -1,
@ -163,11 +163,11 @@
"footcp"
]
},
"default-tcp-app-1-my-tls-gateway-footlsterminate@kubernetesgateway": {
"default-tcp-app-1-my-tls-gateway-footlsterminate-0-e3b0c44298fc1c149afb@kubernetesgateway": {
"entryPoints": [
"footlsterminate"
],
"service": "default-tcp-app-1-my-tls-gateway-footlsterminate-wrr-0",
"service": "default-tcp-app-1-my-tls-gateway-footlsterminate-0-e3b0c44298fc1c149afb-wrr",
"rule": "HostSNI(`*`)",
"ruleSyntax": "v3",
"priority": -1,
@ -179,11 +179,11 @@
"footlsterminate"
]
},
"default-tls-app-1-my-tls-gateway-footlspassthrough-2279fe75c5156dc5eb26@kubernetesgateway": {
"default-tls-app-1-my-tls-gateway-footlspassthrough-0-e3b0c44298fc1c149afb@kubernetesgateway": {
"entryPoints": [
"footlspassthrough"
],
"service": "default-tls-app-1-my-tls-gateway-footlspassthrough-2279fe75c5156dc5eb26-wrr-0",
"service": "default-tls-app-1-my-tls-gateway-footlspassthrough-0-e3b0c44298fc1c149afb-wrr",
"rule": "HostSNI(`foo.bar`)",
"ruleSyntax": "v3",
"priority": 18,
@ -197,7 +197,7 @@
}
},
"tcpServices": {
"default-tcp-app-1-my-tcp-gateway-footcp-wrr-0@kubernetesgateway": {
"default-tcp-app-1-my-tcp-gateway-footcp-0-e3b0c44298fc1c149afb-wrr@kubernetesgateway": {
"weighted": {
"services": [
{
@ -208,10 +208,10 @@
},
"status": "enabled",
"usedBy": [
"default-tcp-app-1-my-tcp-gateway-footcp@kubernetesgateway"
"default-tcp-app-1-my-tcp-gateway-footcp-0-e3b0c44298fc1c149afb@kubernetesgateway"
]
},
"default-tcp-app-1-my-tls-gateway-footlsterminate-wrr-0@kubernetesgateway": {
"default-tcp-app-1-my-tls-gateway-footlsterminate-0-e3b0c44298fc1c149afb-wrr@kubernetesgateway": {
"weighted": {
"services": [
{
@ -222,10 +222,10 @@
},
"status": "enabled",
"usedBy": [
"default-tcp-app-1-my-tls-gateway-footlsterminate@kubernetesgateway"
"default-tcp-app-1-my-tls-gateway-footlsterminate-0-e3b0c44298fc1c149afb@kubernetesgateway"
]
},
"default-tls-app-1-my-tls-gateway-footlspassthrough-2279fe75c5156dc5eb26-wrr-0@kubernetesgateway": {
"default-tls-app-1-my-tls-gateway-footlspassthrough-0-e3b0c44298fc1c149afb-wrr@kubernetesgateway": {
"weighted": {
"services": [
{
@ -236,7 +236,7 @@
},
"status": "enabled",
"usedBy": [
"default-tls-app-1-my-tls-gateway-footlspassthrough-2279fe75c5156dc5eb26@kubernetesgateway"
"default-tls-app-1-my-tls-gateway-footlspassthrough-0-e3b0c44298fc1c149afb@kubernetesgateway"
]
},
"default-whoamitcp-8080@kubernetesgateway": {

4
pkg/provider/kubernetes/gateway/fixtures/tcproute/with_wrong_service_port.yml
View file

@ -32,6 +32,10 @@ metadata:
name: TCP-app-1
namespace: default
spec:
parentRefs:
- name: my-gateway
kind: Gateway
group: gateway.networking.k8s.io
rules:
- backendRefs:
- name: whoami

6
pkg/provider/kubernetes/gateway/fixtures/tlsroute/with_wrong_service_port.yml
View file

@ -36,16 +36,16 @@ spec:
group: ""
allowedRoutes:
kinds:
- kind: TCPRoute
- kind: TLSRoute
group: gateway.networking.k8s.io
namespaces:
from: Same
---
kind: TCPRoute
kind: TLSRoute
apiVersion: gateway.networking.k8s.io/v1alpha2
metadata:
name: tcp-app-1
name: tls-app-1
namespace: default
spec:
parentRefs:

6
pkg/provider/kubernetes/gateway/grpcroute.go
View file

@ -216,7 +216,7 @@ func (p *Provider) loadGRPCService(conf *dynamic.Configuration, routeKey string,
}
func (p *Provider) loadGRPCBackendRef(route *gatev1.GRPCRoute, backendRef gatev1.GRPCBackendRef) (string, *dynamic.Service, *metav1.Condition) {
kind := ptr.Deref(backendRef.Kind, "Service")
kind := ptr.Deref(backendRef.Kind, kindService)
group := groupCore
if backendRef.Group != nil && *backendRef.Group != "" {
@ -230,7 +230,7 @@ func (p *Provider) loadGRPCBackendRef(route *gatev1.GRPCRoute, backendRef gatev1
serviceName := provider.Normalize(namespace + "-" + string(backendRef.Name))
if group != groupCore || kind != "Service" {
if group != groupCore || kind != kindService {
return serviceName, nil, &metav1.Condition{
Type: string(gatev1.RouteConditionResolvedRefs),
Status: metav1.ConditionFalse,
@ -241,7 +241,7 @@ func (p *Provider) loadGRPCBackendRef(route *gatev1.GRPCRoute, backendRef gatev1
}
}
if err := p.isReferenceGranted(groupGateway, kindGRPCRoute, route.Namespace, group, string(kind), string(backendRef.Name), namespace); err != nil {
if err := p.isReferenceGranted(kindGRPCRoute, route.Namespace, group, string(kind), string(backendRef.Name), namespace); err != nil {
return serviceName, nil, &metav1.Condition{
Type: string(gatev1.RouteConditionResolvedRefs),
Status: metav1.ConditionFalse,

16
pkg/provider/kubernetes/gateway/httproute.go
View file

@ -158,7 +158,7 @@ func (p *Provider) loadHTTPRoute(ctx context.Context, listener gatewayListener,
default:
var serviceCondition *metav1.Condition
router.Service, serviceCondition = p.loadService(conf, routeKey, routeRule, route)
router.Service, serviceCondition = p.loadWRRService(conf, routerName, routeRule, route)
if serviceCondition != nil {
condition = *serviceCondition
}
@ -173,7 +173,7 @@ func (p *Provider) loadHTTPRoute(ctx context.Context, listener gatewayListener,
return conf, condition
}
func (p *Provider) loadService(conf *dynamic.Configuration, routeKey string, routeRule gatev1.HTTPRouteRule, route *gatev1.HTTPRoute) (string, *metav1.Condition) {
func (p *Provider) loadWRRService(conf *dynamic.Configuration, routeKey string, routeRule gatev1.HTTPRouteRule, route *gatev1.HTTPRoute) (string, *metav1.Condition) {
name := routeKey + "-wrr"
if _, ok := conf.HTTP.Services[name]; ok {
return name, nil
@ -182,7 +182,7 @@ func (p *Provider) loadService(conf *dynamic.Configuration, routeKey string, rou
var wrr dynamic.WeightedRoundRobin
var condition *metav1.Condition
for _, backendRef := range routeRule.BackendRefs {
svcName, svc, errCondition := p.loadHTTPService(route, backendRef)
svcName, svc, errCondition := p.loadService(route, backendRef)
weight := ptr.To(int(ptr.Deref(backendRef.Weight, 1)))
if errCondition != nil {
condition = errCondition
@ -208,10 +208,10 @@ func (p *Provider) loadService(conf *dynamic.Configuration, routeKey string, rou
return name, condition
}
// loadHTTPService returns a dynamic.Service config corresponding to the given gatev1.HTTPBackendRef.
// loadService returns a dynamic.Service config corresponding to the given gatev1.HTTPBackendRef.
// Note that the returned dynamic.Service config can be nil (for cross-provider, internal services, and backendFunc).
func (p *Provider) loadHTTPService(route *gatev1.HTTPRoute, backendRef gatev1.HTTPBackendRef) (string, *dynamic.Service, *metav1.Condition) {
kind := ptr.Deref(backendRef.Kind, "Service")
func (p *Provider) loadService(route *gatev1.HTTPRoute, backendRef gatev1.HTTPBackendRef) (string, *dynamic.Service, *metav1.Condition) {
kind := ptr.Deref(backendRef.Kind, kindService)
group := groupCore
if backendRef.Group != nil && *backendRef.Group != "" {
@ -225,7 +225,7 @@ func (p *Provider) loadHTTPService(route *gatev1.HTTPRoute, backendRef gatev1.HT
serviceName := provider.Normalize(namespace + "-" + string(backendRef.Name))
if err := p.isReferenceGranted(groupGateway, kindHTTPRoute, route.Namespace, group, string(kind), string(backendRef.Name), namespace); err != nil {
if err := p.isReferenceGranted(kindHTTPRoute, route.Namespace, group, string(kind), string(backendRef.Name), namespace); err != nil {
return serviceName, nil, &metav1.Condition{
Type: string(gatev1.RouteConditionResolvedRefs),
Status: metav1.ConditionFalse,
@ -236,7 +236,7 @@ func (p *Provider) loadHTTPService(route *gatev1.HTTPRoute, backendRef gatev1.HT
}
}
if group != groupCore || kind != "Service" {
if group != groupCore || kind != kindService {
name, service, err := p.loadHTTPBackendRef(namespace, backendRef)
if err != nil {
return serviceName, nil, &metav1.Condition{

36
pkg/provider/kubernetes/gateway/kubernetes.go
View file

@ -47,6 +47,7 @@ const (
kindGRPCRoute = "GRPCRoute"
kindTCPRoute = "TCPRoute"
kindTLSRoute = "TLSRoute"
kindService = "Service"
)
// Provider holds configurations of the provider.
@ -376,11 +377,19 @@ func (p *Provider) loadConfigurationFromGateways(ctx context.Context) *dynamic.C
}
}
gatewayStatus, err := p.makeGatewayStatus(gateway, listeners, addresses)
if err != nil {
gatewayStatus, errConditions := p.makeGatewayStatus(gateway, listeners, addresses)
if len(errConditions) > 0 {
messages := map[string]struct{}{}
for _, condition := range errConditions {
messages[condition.Message] = struct{}{}
}
var conditionsErr error
for message := range messages {
conditionsErr = multierror.Append(conditionsErr, errors.New(message))
}
logger.Error().
Err(err).
Msg("Unable to create Gateway status")
Err(conditionsErr).
Msg("Gateway Not Accepted")
}
if err = p.client.UpdateGatewayStatus(ctx, ktypes.NamespacedName{Name: gateway.Name, Namespace: gateway.Namespace}, gatewayStatus); err != nil {
@ -576,7 +585,7 @@ func (p *Provider) loadGatewayListeners(ctx context.Context, gateway *gatev1.Gat
certificateNamespace = string(*certificateRef.Namespace)
}
if err := p.isReferenceGranted(groupGateway, kindGateway, gateway.Namespace, groupCore, "Secret", string(certificateRef.Name), certificateNamespace); err != nil {
if err := p.isReferenceGranted(kindGateway, gateway.Namespace, groupCore, "Secret", string(certificateRef.Name), certificateNamespace); err != nil {
gatewayListeners[i].Status.Conditions = append(gatewayListeners[i].Status.Conditions, metav1.Condition{
Type: string(gatev1.ListenerConditionResolvedRefs),
Status: metav1.ConditionFalse,
@ -631,10 +640,10 @@ func (p *Provider) loadGatewayListeners(ctx context.Context, gateway *gatev1.Gat
return gatewayListeners
}
func (p *Provider) makeGatewayStatus(gateway *gatev1.Gateway, listeners []gatewayListener, addresses []gatev1.GatewayStatusAddress) (gatev1.GatewayStatus, error) {
func (p *Provider) makeGatewayStatus(gateway *gatev1.Gateway, listeners []gatewayListener, addresses []gatev1.GatewayStatusAddress) (gatev1.GatewayStatus, []metav1.Condition) {
gatewayStatus := gatev1.GatewayStatus{Addresses: addresses}
var result error
var errorConditions []metav1.Condition
for _, listener := range listeners {
if len(listener.Status.Conditions) == 0 {
listener.Status.Conditions = append(listener.Status.Conditions,
@ -669,14 +678,11 @@ func (p *Provider) makeGatewayStatus(gateway *gatev1.Gateway, listeners []gatewa
continue
}
for _, condition := range listener.Status.Conditions {
result = multierror.Append(result, errors.New(condition.Message))
}
errorConditions = append(errorConditions, listener.Status.Conditions...)
gatewayStatus.Listeners = append(gatewayStatus.Listeners, *listener.Status)
}
if result != nil {
if len(errorConditions) > 0 {
// GatewayConditionReady "Ready", GatewayConditionReason "ListenersNotValid"
gatewayStatus.Conditions = append(gatewayStatus.Conditions, metav1.Condition{
Type: string(gatev1.GatewayConditionAccepted),
@ -687,7 +693,7 @@ func (p *Provider) makeGatewayStatus(gateway *gatev1.Gateway, listeners []gatewa
Message: "All Listeners must be valid",
})
return gatewayStatus, result
return gatewayStatus, errorConditions
}
gatewayStatus.Conditions = append(gatewayStatus.Conditions,
@ -783,7 +789,7 @@ func (p *Provider) entryPointName(port gatev1.PortNumber, protocol gatev1.Protoc
return "", fmt.Errorf("no matching entryPoint for port %d and protocol %q", port, protocol)
}
func (p *Provider) isReferenceGranted(fromGroup, fromKind, fromNamespace, toGroup, toKind, toName, toNamespace string) error {
func (p *Provider) isReferenceGranted(fromKind, fromNamespace, toGroup, toKind, toName, toNamespace string) error {
if toNamespace == fromNamespace {
return nil
}
@ -793,7 +799,7 @@ func (p *Provider) isReferenceGranted(fromGroup, fromKind, fromNamespace, toGrou
return fmt.Errorf("listing ReferenceGrant: %w", err)
}
refGrants = filterReferenceGrantsFrom(refGrants, fromGroup, fromKind, fromNamespace)
refGrants = filterReferenceGrantsFrom(refGrants, groupGateway, fromKind, fromNamespace)
refGrants = filterReferenceGrantsTo(refGrants, toGroup, toKind, toName)
if len(refGrants) == 0 {
return errors.New("missing ReferenceGrant")

556
pkg/provider/kubernetes/gateway/kubernetes_test.go
View file

File diff suppressed because it is too large Load diff

314
pkg/provider/kubernetes/gateway/tcproute.go
View file

@ -6,6 +6,7 @@ import (
"fmt"
"net"
"strconv"
"strings"
"github.com/rs/zerolog/log"
"github.com/traefik/traefik/v3/pkg/config/dynamic"
@ -108,182 +109,241 @@ func (p *Provider) loadTCPRoute(listener gatewayListener, route *gatev1alpha2.TC
Reason: string(gatev1.RouteConditionResolvedRefs),
}
router := dynamic.TCPRouter{
Rule: "HostSNI(`*`)",
EntryPoints: []string{listener.EPName},
RuleSyntax: "v3",
}
if listener.Protocol == gatev1.TLSProtocolType && listener.TLS != nil {
// TODO support let's encrypt
router.TLS = &dynamic.RouterTCPTLSConfig{
Passthrough: listener.TLS.Mode != nil && *listener.TLS.Mode == gatev1.TLSModePassthrough,
}
}
// Adding the gateway desc and the entryPoint desc prevents overlapping of routers build from the same routes.
routerName := provider.Normalize(route.Namespace + "-" + route.Name + "-" + listener.GWName + "-" + listener.EPName)
var ruleServiceNames []string
for i, rule := range route.Spec.Rules {
for ri, rule := range route.Spec.Rules {
if rule.BackendRefs == nil {
// Should not happen due to validation.
// https://github.com/kubernetes-sigs/gateway-api/blob/v0.4.0/apis/v1alpha2/tcproute_types.go#L76
continue
}
wrrService, subServices, err := p.loadTCPServices(route.Namespace, rule.BackendRefs)
if err != nil {
return conf, metav1.Condition{
Type: string(gatev1.RouteConditionResolvedRefs),
Status: metav1.ConditionFalse,
ObservedGeneration: route.Generation,
LastTransitionTime: metav1.Now(),
Reason: string(gatev1.RouteReasonBackendNotFound),
Message: fmt.Sprintf("Cannot load TCPRoute service %s/%s: %v", route.Namespace, route.Name, err),
router := dynamic.TCPRouter{
Rule: "HostSNI(`*`)",
EntryPoints: []string{listener.EPName},
RuleSyntax: "v3",
}
if listener.Protocol == gatev1.TLSProtocolType && listener.TLS != nil {
router.TLS = &dynamic.RouterTCPTLSConfig{
Passthrough: listener.TLS.Mode != nil && *listener.TLS.Mode == gatev1.TLSModePassthrough,
}
}
for svcName, svc := range subServices {
conf.TCP.Services[svcName] = svc
// Adding the gateway desc and the entryPoint desc prevents overlapping of routers build from the same routes.
routeKey := provider.Normalize(fmt.Sprintf("%s-%s-%s-%s-%d", route.Namespace, route.Name, listener.GWName, listener.EPName, ri))
// Routing criteria should be introduced at some point.
routerName := makeRouterName("", routeKey)
if len(rule.BackendRefs) == 1 && isInternalService(rule.BackendRefs[0]) {
router.Service = string(rule.BackendRefs[0].Name)
conf.TCP.Routers[routerName] = &router
continue
}
serviceName := fmt.Sprintf("%s-wrr-%d", routerName, i)
conf.TCP.Services[serviceName] = wrrService
var serviceCondition *metav1.Condition
router.Service, serviceCondition = p.loadTCPWRRService(conf, routerName, rule.BackendRefs, route)
if serviceCondition != nil {
condition = *serviceCondition
}
ruleServiceNames = append(ruleServiceNames, serviceName)
}
if len(ruleServiceNames) == 1 {
router.Service = ruleServiceNames[0]
conf.TCP.Routers[routerName] = &router
return conf, condition
}
routeServiceKey := routerName + "-wrr"
routeService := &dynamic.TCPService{Weighted: &dynamic.TCPWeightedRoundRobin{}}
for _, name := range ruleServiceNames {
service := dynamic.TCPWRRService{Name: name}
service.SetDefaults()
routeService.Weighted.Services = append(routeService.Weighted.Services, service)
}
conf.TCP.Services[routeServiceKey] = routeService
router.Service = routeServiceKey
conf.TCP.Routers[routerName] = &router
return conf, condition
}
// loadTCPServices is generating a WRR service, even when there is only one target.
func (p *Provider) loadTCPServices(namespace string, backendRefs []gatev1.BackendRef) (*dynamic.TCPService, map[string]*dynamic.TCPService, error) {
services := map[string]*dynamic.TCPService{}
wrrSvc := &dynamic.TCPService{
Weighted: &dynamic.TCPWeightedRoundRobin{
Services: []dynamic.TCPWRRService{},
},
// loadTCPWRRService is generating a WRR service, even when there is only one target.
func (p *Provider) loadTCPWRRService(conf *dynamic.Configuration, routeKey string, backendRefs []gatev1.BackendRef, route *gatev1alpha2.TCPRoute) (string, *metav1.Condition) {
name := routeKey + "-wrr"
if _, ok := conf.TCP.Services[name]; ok {
return name, nil
}
var wrr dynamic.TCPWeightedRoundRobin
var condition *metav1.Condition
for _, backendRef := range backendRefs {
if backendRef.Group == nil || backendRef.Kind == nil {
// Should not happen as this is validated by kubernetes
svcName, svc, errCondition := p.loadTCPService(route, backendRef)
weight := ptr.To(int(ptr.Deref(backendRef.Weight, 1)))
if errCondition != nil {
condition = errCondition
errName := routeKey + "-err-lb"
conf.TCP.Services[errName] = &dynamic.TCPService{
LoadBalancer: &dynamic.TCPServersLoadBalancer{
Servers: []dynamic.TCPServer{},
},
}
wrr.Services = append(wrr.Services, dynamic.TCPWRRService{
Name: errName,
Weight: weight,
})
continue
}
if isInternalService(backendRef) {
return nil, nil, fmt.Errorf("traefik internal service %s is not allowed in a WRR loadbalancer", backendRef.Name)
if svc != nil {
conf.TCP.Services[svcName] = svc
}
weight := int(ptr.Deref(backendRef.Weight, 1))
wrr.Services = append(wrr.Services, dynamic.TCPWRRService{
Name: svcName,
Weight: weight,
})
}
if isTraefikService(backendRef) {
wrrSvc.Weighted.Services = append(wrrSvc.Weighted.Services, dynamic.TCPWRRService{Name: string(backendRef.Name), Weight: &weight})
continue
conf.TCP.Services[name] = &dynamic.TCPService{Weighted: &wrr}
return name, condition
}
func (p *Provider) loadTCPService(route *gatev1alpha2.TCPRoute, backendRef gatev1.BackendRef) (string, *dynamic.TCPService, *metav1.Condition) {
kind := ptr.Deref(backendRef.Kind, kindService)
group := groupCore
if backendRef.Group != nil && *backendRef.Group != "" {
group = string(*backendRef.Group)
}
namespace := route.Namespace
if backendRef.Namespace != nil && *backendRef.Namespace != "" {
namespace = string(*backendRef.Namespace)
}
serviceName := provider.Normalize(namespace + "-" + string(backendRef.Name))
if err := p.isReferenceGranted(kindTCPRoute, route.Namespace, group, string(kind), string(backendRef.Name), namespace); err != nil {
return serviceName, nil, &metav1.Condition{
Type: string(gatev1.RouteConditionResolvedRefs),
Status: metav1.ConditionFalse,
ObservedGeneration: route.Generation,
LastTransitionTime: metav1.Now(),
Reason: string(gatev1.RouteReasonRefNotPermitted),
Message: fmt.Sprintf("Cannot load TCPRoute BackendRef %s/%s/%s/%s: %s", group, kind, namespace, backendRef.Name, err),
}
}
if *backendRef.Group != "" && *backendRef.Group != groupCore && *backendRef.Kind != "Service" {
return nil, nil, fmt.Errorf("unsupported BackendRef %s/%s/%s", *backendRef.Group, *backendRef.Kind, backendRef.Name)
}
if backendRef.Port == nil {
return nil, nil, errors.New("port is required for Kubernetes Service reference")
}
service, exists, err := p.client.GetService(namespace, string(backendRef.Name))
if group != groupCore || kind != kindService {
name, err := p.loadTCPBackendRef(backendRef)
if err != nil {
return nil, nil, fmt.Errorf("getting service: %w", err)
}
if !exists {
return nil, nil, errors.New("service not found")
return serviceName, nil, &metav1.Condition{
Type: string(gatev1.RouteConditionResolvedRefs),
Status: metav1.ConditionFalse,
ObservedGeneration: route.Generation,
LastTransitionTime: metav1.Now(),
Reason: string(gatev1.RouteReasonInvalidKind),
Message: fmt.Sprintf("Cannot load TCPRoute BackendRef %s/%s/%s/%s: %s", group, kind, namespace, backendRef.Name, err),
}
}
var svcPort *corev1.ServicePort
for _, p := range service.Spec.Ports {
if p.Port == int32(*backendRef.Port) {
svcPort = &p
return name, nil, nil
}
port := ptr.Deref(backendRef.Port, gatev1.PortNumber(0))
if port == 0 {
return serviceName, nil, &metav1.Condition{
Type: string(gatev1.RouteConditionResolvedRefs),
Status: metav1.ConditionFalse,
ObservedGeneration: route.Generation,
LastTransitionTime: metav1.Now(),
Reason: string(gatev1.RouteReasonUnsupportedProtocol),
Message: fmt.Sprintf("Cannot load TCPRoute BackendRef %s/%s/%s/%s port is required", group, kind, namespace, backendRef.Name),
}
}
portStr := strconv.FormatInt(int64(port), 10)
serviceName = provider.Normalize(serviceName + "-" + portStr)
lb, err := p.loadTCPServers(namespace, backendRef)
if err != nil {
return serviceName, nil, &metav1.Condition{
Type: string(gatev1.RouteConditionResolvedRefs),
Status: metav1.ConditionFalse,
ObservedGeneration: route.Generation,
LastTransitionTime: metav1.Now(),
Reason: string(gatev1.RouteReasonBackendNotFound),
Message: fmt.Sprintf("Cannot load TCPRoute BackendRef %s/%s/%s/%s: %s", group, kind, namespace, backendRef.Name, err),
}
}
return serviceName, &dynamic.TCPService{LoadBalancer: lb}, nil
}
func (p *Provider) loadTCPServers(namespace string, backendRef gatev1.BackendRef) (*dynamic.TCPServersLoadBalancer, error) {
if backendRef.Port == nil {
return nil, errors.New("port is required for Kubernetes Service reference")
}
service, exists, err := p.client.GetService(namespace, string(backendRef.Name))
if err != nil {
return nil, fmt.Errorf("getting service: %w", err)
}
if !exists {
return nil, errors.New("service not found")
}
var svcPort *corev1.ServicePort
for _, p := range service.Spec.Ports {
if p.Port == int32(*backendRef.Port) {
svcPort = &p
break
}
}
if svcPort == nil {
return nil, fmt.Errorf("service port %d not found", *backendRef.Port)
}
endpointSlices, err := p.client.ListEndpointSlicesForService(namespace, string(backendRef.Name))
if err != nil {
return nil, fmt.Errorf("getting endpointslices: %w", err)
}
if len(endpointSlices) == 0 {
return nil, errors.New("endpointslices not found")
}
lb := &dynamic.TCPServersLoadBalancer{}
addresses := map[string]struct{}{}
for _, endpointSlice := range endpointSlices {
var port int32
for _, p := range endpointSlice.Ports {
if svcPort.Name == *p.Name {
port = *p.Port
break
}
}
if svcPort == nil {
return nil, nil, fmt.Errorf("service port %d not found", *backendRef.Port)
if port == 0 {
continue
}
endpointSlices, err := p.client.ListEndpointSlicesForService(namespace, string(backendRef.Name))
if err != nil {
return nil, nil, fmt.Errorf("getting endpointslices: %w", err)
}
if len(endpointSlices) == 0 {
return nil, nil, errors.New("endpointslices not found")
}
svc := dynamic.TCPService{LoadBalancer: &dynamic.TCPServersLoadBalancer{}}
addresses := map[string]struct{}{}
for _, endpointSlice := range endpointSlices {
var port int32
for _, p := range endpointSlice.Ports {
if svcPort.Name == *p.Name {
port = *p.Port
break
}
}
if port == 0 {
for _, endpoint := range endpointSlice.Endpoints {
if endpoint.Conditions.Ready == nil || !*endpoint.Conditions.Ready {
continue
}
for _, endpoint := range endpointSlice.Endpoints {
if endpoint.Conditions.Ready == nil || !*endpoint.Conditions.Ready {
for _, address := range endpoint.Addresses {
if _, ok := addresses[address]; ok {
continue
}
for _, address := range endpoint.Addresses {
if _, ok := addresses[address]; ok {
continue
}
addresses[address] = struct{}{}
svc.LoadBalancer.Servers = append(svc.LoadBalancer.Servers, dynamic.TCPServer{
Address: net.JoinHostPort(address, strconv.Itoa(int(port))),
})
}
addresses[address] = struct{}{}
lb.Servers = append(lb.Servers, dynamic.TCPServer{
// TODO determine whether the servers needs TLS, from the port?
Address: net.JoinHostPort(address, strconv.Itoa(int(port))),
})
}
}
serviceName := provider.Normalize(service.Namespace + "-" + service.Name + "-" + strconv.Itoa(int(svcPort.Port)))
services[serviceName] = &svc
wrrSvc.Weighted.Services = append(wrrSvc.Weighted.Services, dynamic.TCPWRRService{Name: serviceName, Weight: &weight})
}
if len(wrrSvc.Weighted.Services) == 0 {
return nil, nil, errors.New("no service has been created")
return lb, nil
}
func (p *Provider) loadTCPBackendRef(backendRef gatev1.BackendRef) (string, error) {
// Support for cross-provider references (e.g: api@internal).
// This provides the same behavior as for IngressRoutes.
if *backendRef.Kind == "TraefikService" && strings.Contains(string(backendRef.Name), "@") {
return string(backendRef.Name), nil
}
return wrrSvc, services, nil
return "", fmt.Errorf("unsupported BackendRef %s/%s/%s", *backendRef.Group, *backendRef.Kind, backendRef.Name)
}
func mergeTCPConfiguration(from, to *dynamic.Configuration) {

183
pkg/provider/kubernetes/gateway/tlsroute.go
View file

@ -4,6 +4,7 @@ import (
"context"
"fmt"
"regexp"
"strconv"
"strings"
"github.com/rs/zerolog/log"
@ -11,6 +12,7 @@ import (
"github.com/traefik/traefik/v3/pkg/provider"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
ktypes "k8s.io/apimachinery/pkg/types"
"k8s.io/utils/ptr"
gatev1 "sigs.k8s.io/gateway-api/apis/v1"
gatev1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2"
)
@ -109,74 +111,159 @@ func (p *Provider) loadTLSRoute(listener gatewayListener, route *gatev1alpha2.TL
Reason: string(gatev1.RouteConditionResolvedRefs),
}
router := dynamic.TCPRouter{
RuleSyntax: "v3",
Rule: hostSNIRule(hostnames),
EntryPoints: []string{listener.EPName},
TLS: &dynamic.RouterTCPTLSConfig{
Passthrough: listener.TLS != nil && listener.TLS.Mode != nil && *listener.TLS.Mode == gatev1.TLSModePassthrough,
},
}
// Adding the gateway desc and the entryPoint desc prevents overlapping of routers build from the same routes.
routeKey := provider.Normalize(route.Namespace + "-" + route.Name + "-" + listener.GWName + "-" + listener.EPName)
routerName := makeRouterName(router.Rule, routeKey)
var ruleServiceNames []string
for i, routeRule := range route.Spec.Rules {
for ri, routeRule := range route.Spec.Rules {
if len(routeRule.BackendRefs) == 0 {
// Should not happen due to validation.
// https://github.com/kubernetes-sigs/gateway-api/blob/v0.4.0/apis/v1alpha2/tlsroute_types.go#L120
continue
}
wrrService, subServices, err := p.loadTCPServices(route.Namespace, routeRule.BackendRefs)
router := dynamic.TCPRouter{
RuleSyntax: "v3",
Rule: hostSNIRule(hostnames),
EntryPoints: []string{listener.EPName},
TLS: &dynamic.RouterTCPTLSConfig{
Passthrough: listener.TLS != nil && listener.TLS.Mode != nil && *listener.TLS.Mode == gatev1.TLSModePassthrough,
},
}
// Adding the gateway desc and the entryPoint desc prevents overlapping of routers build from the same routes.
routeKey := provider.Normalize(fmt.Sprintf("%s-%s-%s-%s-%d", route.Namespace, route.Name, listener.GWName, listener.EPName, ri))
// Routing criteria should be introduced at some point.
routerName := makeRouterName("", routeKey)
if len(routeRule.BackendRefs) == 1 && isInternalService(routeRule.BackendRefs[0]) {
router.Service = string(routeRule.BackendRefs[0].Name)
conf.TCP.Routers[routerName] = &router
continue
}
var serviceCondition *metav1.Condition
router.Service, serviceCondition = p.loadTLSWRRService(conf, routerName, routeRule.BackendRefs, route)
if serviceCondition != nil {
condition = *serviceCondition
}
conf.TCP.Routers[routerName] = &router
}
return conf, condition
}
// loadTLSWRRService is generating a WRR service, even when there is only one target.
func (p *Provider) loadTLSWRRService(conf *dynamic.Configuration, routeKey string, backendRefs []gatev1.BackendRef, route *gatev1alpha2.TLSRoute) (string, *metav1.Condition) {
name := routeKey + "-wrr"
if _, ok := conf.TCP.Services[name]; ok {
return name, nil
}
var wrr dynamic.TCPWeightedRoundRobin
var condition *metav1.Condition
for _, backendRef := range backendRefs {
svcName, svc, errCondition := p.loadTLSService(route, backendRef)
weight := ptr.To(int(ptr.Deref(backendRef.Weight, 1)))
if errCondition != nil {
condition = errCondition
errName := routeKey + "-err-lb"
conf.TCP.Services[errName] = &dynamic.TCPService{
LoadBalancer: &dynamic.TCPServersLoadBalancer{
Servers: []dynamic.TCPServer{},
},
}
wrr.Services = append(wrr.Services, dynamic.TCPWRRService{
Name: errName,
Weight: weight,
})
continue
}
if svc != nil {
conf.TCP.Services[svcName] = svc
}
wrr.Services = append(wrr.Services, dynamic.TCPWRRService{
Name: svcName,
Weight: weight,
})
}
conf.TCP.Services[name] = &dynamic.TCPService{Weighted: &wrr}
return name, condition
}
func (p *Provider) loadTLSService(route *gatev1alpha2.TLSRoute, backendRef gatev1.BackendRef) (string, *dynamic.TCPService, *metav1.Condition) {
kind := ptr.Deref(backendRef.Kind, kindService)
group := groupCore
if backendRef.Group != nil && *backendRef.Group != "" {
group = string(*backendRef.Group)
}
namespace := route.Namespace
if backendRef.Namespace != nil && *backendRef.Namespace != "" {
namespace = string(*backendRef.Namespace)
}
serviceName := provider.Normalize(namespace + "-" + string(backendRef.Name))
if err := p.isReferenceGranted(kindTLSRoute, route.Namespace, group, string(kind), string(backendRef.Name), namespace); err != nil {
return serviceName, nil, &metav1.Condition{
Type: string(gatev1.RouteConditionResolvedRefs),
Status: metav1.ConditionFalse,
ObservedGeneration: route.Generation,
LastTransitionTime: metav1.Now(),
Reason: string(gatev1.RouteReasonRefNotPermitted),
Message: fmt.Sprintf("Cannot load TLSRoute BackendRef %s/%s/%s/%s: %s", group, kind, namespace, backendRef.Name, err),
}
}
if group != groupCore || kind != kindService {
name, err := p.loadTCPBackendRef(backendRef)
if err != nil {
// update "ResolvedRefs" status true with "InvalidBackendRefs" reason
condition = metav1.Condition{
return serviceName, nil, &metav1.Condition{
Type: string(gatev1.RouteConditionResolvedRefs),
Status: metav1.ConditionFalse,
ObservedGeneration: route.Generation,
LastTransitionTime: metav1.Now(),
Reason: string(gatev1.RouteReasonBackendNotFound),
Message: fmt.Sprintf("Cannot load TLSRoute service %s/%s: %v", route.Namespace, route.Name, err),
Reason: string(gatev1.RouteReasonInvalidKind),
Message: fmt.Sprintf("Cannot load TLSRoute BackendRef %s/%s/%s/%s: %s", group, kind, namespace, backendRef.Name, err),
}
continue
}
for svcName, svc := range subServices {
conf.TCP.Services[svcName] = svc
return name, nil, nil
}
port := ptr.Deref(backendRef.Port, gatev1.PortNumber(0))
if port == 0 {
return serviceName, nil, &metav1.Condition{
Type: string(gatev1.RouteConditionResolvedRefs),
Status: metav1.ConditionFalse,
ObservedGeneration: route.Generation,
LastTransitionTime: metav1.Now(),
Reason: string(gatev1.RouteReasonUnsupportedProtocol),
Message: fmt.Sprintf("Cannot load TLSRoute BackendRef %s/%s/%s/%s port is required", group, kind, namespace, backendRef.Name),
}
serviceName := fmt.Sprintf("%s-wrr-%d", routerName, i)
conf.TCP.Services[serviceName] = wrrService
ruleServiceNames = append(ruleServiceNames, serviceName)
}
if len(ruleServiceNames) == 1 {
router.Service = ruleServiceNames[0]
conf.TCP.Routers[routerName] = &router
portStr := strconv.FormatInt(int64(port), 10)
serviceName = provider.Normalize(serviceName + "-" + portStr)
return conf, condition
lb, err := p.loadTCPServers(namespace, backendRef)
if err != nil {
return serviceName, nil, &metav1.Condition{
Type: string(gatev1.RouteConditionResolvedRefs),
Status: metav1.ConditionFalse,
ObservedGeneration: route.Generation,
LastTransitionTime: metav1.Now(),
Reason: string(gatev1.RouteReasonBackendNotFound),
Message: fmt.Sprintf("Cannot load TLSRoute BackendRef %s/%s/%s/%s: %s", group, kind, namespace, backendRef.Name, err),
}
}
routeServiceKey := routerName + "-wrr"
routeService := &dynamic.TCPService{Weighted: &dynamic.TCPWeightedRoundRobin{}}
for _, name := range ruleServiceNames {
service := dynamic.TCPWRRService{Name: name}
service.SetDefaults()
routeService.Weighted.Services = append(routeService.Weighted.Services, service)
}
conf.TCP.Services[routeServiceKey] = routeService
router.Service = routeServiceKey
conf.TCP.Routers[routerName] = &router
return conf, condition
return serviceName, &dynamic.TCPService{LoadBalancer: lb}, nil
}
func hostSNIRule(hostnames []gatev1.Hostname) string {