Improve Kubernetes GatewayAPI TCPRoute and TLSRoute support

integration/conformance-reports/v1.1.0/experimental-v3.1-default-report.yaml

@@ -53,3 +53,11 @@ profiles:
     - HTTPRouteRequestTimeout
   name: GATEWAY-HTTP
   summary: Core tests succeeded. Extended tests succeeded.
+- core:
+    result: success
+    statistics:
+      Failed: 0
+      Passed: 11
+      Skipped: 0
+  name: GATEWAY-TLS
+  summary: Core tests succeeded.

integration/k8s_conformance_test.go

@@ -8,6 +8,7 @@ import (
 	"os"
 	"path/filepath"
 	"slices"
+	"strings"
 	"testing"
 	"time"
 
@@ -193,7 +194,11 @@ func (s *K8sConformanceSuite) TestK8sGatewayAPIConformance() {
 			Version:      *k8sConformanceTraefikVersion,
 			Contact:      []string{"@traefik/maintainers"},
 		},
-		ConformanceProfiles: sets.New(ksuite.GatewayHTTPConformanceProfileName, ksuite.GatewayGRPCConformanceProfileName),
+		ConformanceProfiles: sets.New(
+			ksuite.GatewayHTTPConformanceProfileName,
+			ksuite.GatewayGRPCConformanceProfileName,
+			ksuite.GatewayTLSConformanceProfileName,
+		),
 		SupportedFeatures: sets.New(
 			features.SupportGateway,
 			features.SupportGatewayPort8080,
@@ -207,6 +212,7 @@ func (s *K8sConformanceSuite) TestK8sGatewayAPIConformance() {
 			features.SupportHTTPRoutePathRewrite,
 			features.SupportHTTPRoutePathRedirect,
 			features.SupportHTTPRouteResponseHeaderModification,
+			features.SupportTLSRoute,
 		),
 	})
 	require.NoError(s.T(), err)
@@ -224,6 +230,11 @@ func (s *K8sConformanceSuite) TestK8sGatewayAPIConformance() {
 	// TODO: to publish this report automatically, we have to figure out how to handle the date diff.
 	report.Date = "-"
 
+	// Ordering profile reports for the serialized report to be comparable.
+	slices.SortFunc(report.ProfileReports, func(a, b v1.ProfileReport) int {
+		return strings.Compare(a.Name, b.Name)
+	})
+
 	rawReport, err := yaml.Marshal(report)
 	require.NoError(s.T(), err)
 	s.T().Logf("Conformance report:\n%s", string(rawReport))

integration/testdata/rawdata-gateway.json

@@ -34,7 +34,7 @@
 			"entryPoints": [
 				"web"
 			],
-			"service": "default-http-app-1-my-gateway-web-0-wrr",
+			"service": "default-http-app-1-my-gateway-web-0-1c0cf64bde37d9d0df06-wrr",
 			"rule": "Host(`foo.com`) \u0026\u0026 Path(`/bar`)",
 			"ruleSyntax": "v3",
 			"priority": 100008,
@@ -47,7 +47,7 @@
 			"entryPoints": [
 				"websecure"
 			],
-			"service": "default-http-app-1-my-https-gateway-websecure-0-wrr",
+			"service": "default-http-app-1-my-https-gateway-websecure-0-1c0cf64bde37d9d0df06-wrr",
 			"rule": "Host(`foo.com`) \u0026\u0026 Path(`/bar`)",
 			"ruleSyntax": "v3",
 			"priority": 100008,
@@ -96,7 +96,7 @@
 				"dashboard@internal"
 			]
 		},
-		"default-http-app-1-my-gateway-web-0-wrr@kubernetesgateway": {
+		"default-http-app-1-my-gateway-web-0-1c0cf64bde37d9d0df06-wrr@kubernetesgateway": {
 			"weighted": {
 				"services": [
 					{
@@ -110,7 +110,7 @@
 				"default-http-app-1-my-gateway-web-0-1c0cf64bde37d9d0df06@kubernetesgateway"
 			]
 		},
-		"default-http-app-1-my-https-gateway-websecure-0-wrr@kubernetesgateway": {
+		"default-http-app-1-my-https-gateway-websecure-0-1c0cf64bde37d9d0df06-wrr@kubernetesgateway": {
 			"weighted": {
 				"services": [
 					{
@@ -150,11 +150,11 @@
 		}
 	},
 	"tcpRouters": {
-		"default-tcp-app-1-my-tcp-gateway-footcp@kubernetesgateway": {
+		"default-tcp-app-1-my-tcp-gateway-footcp-0-e3b0c44298fc1c149afb@kubernetesgateway": {
 			"entryPoints": [
 				"footcp"
 			],
-			"service": "default-tcp-app-1-my-tcp-gateway-footcp-wrr-0",
+			"service": "default-tcp-app-1-my-tcp-gateway-footcp-0-e3b0c44298fc1c149afb-wrr",
 			"rule": "HostSNI(`*`)",
 			"ruleSyntax": "v3",
 			"priority": -1,
@@ -163,11 +163,11 @@
 				"footcp"
 			]
 		},
-		"default-tcp-app-1-my-tls-gateway-footlsterminate@kubernetesgateway": {
+		"default-tcp-app-1-my-tls-gateway-footlsterminate-0-e3b0c44298fc1c149afb@kubernetesgateway": {
 			"entryPoints": [
 				"footlsterminate"
 			],
-			"service": "default-tcp-app-1-my-tls-gateway-footlsterminate-wrr-0",
+			"service": "default-tcp-app-1-my-tls-gateway-footlsterminate-0-e3b0c44298fc1c149afb-wrr",
 			"rule": "HostSNI(`*`)",
 			"ruleSyntax": "v3",
 			"priority": -1,
@@ -179,11 +179,11 @@
 				"footlsterminate"
 			]
 		},
-		"default-tls-app-1-my-tls-gateway-footlspassthrough-2279fe75c5156dc5eb26@kubernetesgateway": {
+		"default-tls-app-1-my-tls-gateway-footlspassthrough-0-e3b0c44298fc1c149afb@kubernetesgateway": {
 			"entryPoints": [
 				"footlspassthrough"
 			],
-			"service": "default-tls-app-1-my-tls-gateway-footlspassthrough-2279fe75c5156dc5eb26-wrr-0",
+			"service": "default-tls-app-1-my-tls-gateway-footlspassthrough-0-e3b0c44298fc1c149afb-wrr",
 			"rule": "HostSNI(`foo.bar`)",
 			"ruleSyntax": "v3",
 			"priority": 18,
@@ -197,7 +197,7 @@
 		}
 	},
 	"tcpServices": {
-		"default-tcp-app-1-my-tcp-gateway-footcp-wrr-0@kubernetesgateway": {
+		"default-tcp-app-1-my-tcp-gateway-footcp-0-e3b0c44298fc1c149afb-wrr@kubernetesgateway": {
 			"weighted": {
 				"services": [
 					{
@@ -208,10 +208,10 @@
 			},
 			"status": "enabled",
 			"usedBy": [
-				"default-tcp-app-1-my-tcp-gateway-footcp@kubernetesgateway"
+				"default-tcp-app-1-my-tcp-gateway-footcp-0-e3b0c44298fc1c149afb@kubernetesgateway"
 			]
 		},
-		"default-tcp-app-1-my-tls-gateway-footlsterminate-wrr-0@kubernetesgateway": {
+		"default-tcp-app-1-my-tls-gateway-footlsterminate-0-e3b0c44298fc1c149afb-wrr@kubernetesgateway": {
 			"weighted": {
 				"services": [
 					{
@@ -222,10 +222,10 @@
 			},
 			"status": "enabled",
 			"usedBy": [
-				"default-tcp-app-1-my-tls-gateway-footlsterminate@kubernetesgateway"
+				"default-tcp-app-1-my-tls-gateway-footlsterminate-0-e3b0c44298fc1c149afb@kubernetesgateway"
 			]
 		},
-		"default-tls-app-1-my-tls-gateway-footlspassthrough-2279fe75c5156dc5eb26-wrr-0@kubernetesgateway": {
+		"default-tls-app-1-my-tls-gateway-footlspassthrough-0-e3b0c44298fc1c149afb-wrr@kubernetesgateway": {
 			"weighted": {
 				"services": [
 					{
@@ -236,7 +236,7 @@
 			},
 			"status": "enabled",
 			"usedBy": [
-				"default-tls-app-1-my-tls-gateway-footlspassthrough-2279fe75c5156dc5eb26@kubernetesgateway"
+				"default-tls-app-1-my-tls-gateway-footlspassthrough-0-e3b0c44298fc1c149afb@kubernetesgateway"
 			]
 		},
 		"default-whoamitcp-8080@kubernetesgateway": {

pkg/provider/kubernetes/gateway/fixtures/tcproute/with_wrong_service_port.yml

@@ -32,6 +32,10 @@ metadata:
   name: TCP-app-1
   namespace: default
 spec:
+  parentRefs:
+    - name: my-gateway
+      kind: Gateway
+      group: gateway.networking.k8s.io
   rules:
     - backendRefs:
         - name: whoami

pkg/provider/kubernetes/gateway/fixtures/tlsroute/with_wrong_service_port.yml

@@ -36,16 +36,16 @@ spec:
             group: ""
       allowedRoutes:
         kinds:
-          - kind: TCPRoute
+          - kind: TLSRoute
             group: gateway.networking.k8s.io
         namespaces:
           from: Same
 
 ---
-kind: TCPRoute
+kind: TLSRoute
 apiVersion: gateway.networking.k8s.io/v1alpha2
 metadata:
-  name: tcp-app-1
+  name: tls-app-1
   namespace: default
 spec:
   parentRefs:

pkg/provider/kubernetes/gateway/grpcroute.go

@@ -216,7 +216,7 @@ func (p *Provider) loadGRPCService(conf *dynamic.Configuration, routeKey string,
 }
 
 func (p *Provider) loadGRPCBackendRef(route *gatev1.GRPCRoute, backendRef gatev1.GRPCBackendRef) (string, *dynamic.Service, *metav1.Condition) {
-	kind := ptr.Deref(backendRef.Kind, "Service")
+	kind := ptr.Deref(backendRef.Kind, kindService)
 
 	group := groupCore
 	if backendRef.Group != nil && *backendRef.Group != "" {
@@ -230,7 +230,7 @@ func (p *Provider) loadGRPCBackendRef(route *gatev1.GRPCRoute, backendRef gatev1
 
 	serviceName := provider.Normalize(namespace + "-" + string(backendRef.Name))
 
-	if group != groupCore || kind != "Service" {
+	if group != groupCore || kind != kindService {
 		return serviceName, nil, &metav1.Condition{
 			Type:               string(gatev1.RouteConditionResolvedRefs),
 			Status:             metav1.ConditionFalse,
@@ -241,7 +241,7 @@ func (p *Provider) loadGRPCBackendRef(route *gatev1.GRPCRoute, backendRef gatev1
 		}
 	}
 
-	if err := p.isReferenceGranted(groupGateway, kindGRPCRoute, route.Namespace, group, string(kind), string(backendRef.Name), namespace); err != nil {
+	if err := p.isReferenceGranted(kindGRPCRoute, route.Namespace, group, string(kind), string(backendRef.Name), namespace); err != nil {
 		return serviceName, nil, &metav1.Condition{
 			Type:               string(gatev1.RouteConditionResolvedRefs),
 			Status:             metav1.ConditionFalse,

pkg/provider/kubernetes/gateway/httproute.go

@@ -158,7 +158,7 @@ func (p *Provider) loadHTTPRoute(ctx context.Context, listener gatewayListener,
 
 			default:
 				var serviceCondition *metav1.Condition
-				router.Service, serviceCondition = p.loadService(conf, routeKey, routeRule, route)
+				router.Service, serviceCondition = p.loadWRRService(conf, routerName, routeRule, route)
 				if serviceCondition != nil {
 					condition = *serviceCondition
 				}
@@ -173,7 +173,7 @@ func (p *Provider) loadHTTPRoute(ctx context.Context, listener gatewayListener,
 	return conf, condition
 }
 
-func (p *Provider) loadService(conf *dynamic.Configuration, routeKey string, routeRule gatev1.HTTPRouteRule, route *gatev1.HTTPRoute) (string, *metav1.Condition) {
+func (p *Provider) loadWRRService(conf *dynamic.Configuration, routeKey string, routeRule gatev1.HTTPRouteRule, route *gatev1.HTTPRoute) (string, *metav1.Condition) {
 	name := routeKey + "-wrr"
 	if _, ok := conf.HTTP.Services[name]; ok {
 		return name, nil
@@ -182,7 +182,7 @@ func (p *Provider) loadService(conf *dynamic.Configuration, routeKey string, rou
 	var wrr dynamic.WeightedRoundRobin
 	var condition *metav1.Condition
 	for _, backendRef := range routeRule.BackendRefs {
-		svcName, svc, errCondition := p.loadHTTPService(route, backendRef)
+		svcName, svc, errCondition := p.loadService(route, backendRef)
 		weight := ptr.To(int(ptr.Deref(backendRef.Weight, 1)))
 		if errCondition != nil {
 			condition = errCondition
@@ -208,10 +208,10 @@ func (p *Provider) loadService(conf *dynamic.Configuration, routeKey string, rou
 	return name, condition
 }
 
-// loadHTTPService returns a dynamic.Service config corresponding to the given gatev1.HTTPBackendRef.
+// loadService returns a dynamic.Service config corresponding to the given gatev1.HTTPBackendRef.
 // Note that the returned dynamic.Service config can be nil (for cross-provider, internal services, and backendFunc).
-func (p *Provider) loadHTTPService(route *gatev1.HTTPRoute, backendRef gatev1.HTTPBackendRef) (string, *dynamic.Service, *metav1.Condition) {
+func (p *Provider) loadService(route *gatev1.HTTPRoute, backendRef gatev1.HTTPBackendRef) (string, *dynamic.Service, *metav1.Condition) {
-	kind := ptr.Deref(backendRef.Kind, "Service")
+	kind := ptr.Deref(backendRef.Kind, kindService)
 
 	group := groupCore
 	if backendRef.Group != nil && *backendRef.Group != "" {
@@ -225,7 +225,7 @@ func (p *Provider) loadHTTPService(route *gatev1.HTTPRoute, backendRef gatev1.HT
 
 	serviceName := provider.Normalize(namespace + "-" + string(backendRef.Name))
 
-	if err := p.isReferenceGranted(groupGateway, kindHTTPRoute, route.Namespace, group, string(kind), string(backendRef.Name), namespace); err != nil {
+	if err := p.isReferenceGranted(kindHTTPRoute, route.Namespace, group, string(kind), string(backendRef.Name), namespace); err != nil {
 		return serviceName, nil, &metav1.Condition{
 			Type:               string(gatev1.RouteConditionResolvedRefs),
 			Status:             metav1.ConditionFalse,
@@ -236,7 +236,7 @@ func (p *Provider) loadHTTPService(route *gatev1.HTTPRoute, backendRef gatev1.HT
 		}
 	}
 
-	if group != groupCore || kind != "Service" {
+	if group != groupCore || kind != kindService {
 		name, service, err := p.loadHTTPBackendRef(namespace, backendRef)
 		if err != nil {
 			return serviceName, nil, &metav1.Condition{

pkg/provider/kubernetes/gateway/kubernetes.go

@@ -47,6 +47,7 @@ const (
 	kindGRPCRoute      = "GRPCRoute"
 	kindTCPRoute       = "TCPRoute"
 	kindTLSRoute       = "TLSRoute"
+	kindService        = "Service"
 )
 
 // Provider holds configurations of the provider.
@@ -376,11 +377,19 @@ func (p *Provider) loadConfigurationFromGateways(ctx context.Context) *dynamic.C
 			}
 		}
 
-		gatewayStatus, err := p.makeGatewayStatus(gateway, listeners, addresses)
+		gatewayStatus, errConditions := p.makeGatewayStatus(gateway, listeners, addresses)
-		if err != nil {
+		if len(errConditions) > 0 {
+			messages := map[string]struct{}{}
+			for _, condition := range errConditions {
+				messages[condition.Message] = struct{}{}
+			}
+			var conditionsErr error
+			for message := range messages {
+				conditionsErr = multierror.Append(conditionsErr, errors.New(message))
+			}
 			logger.Error().
-				Err(err).
+				Err(conditionsErr).
-				Msg("Unable to create Gateway status")
+				Msg("Gateway Not Accepted")
 		}
 
 		if err = p.client.UpdateGatewayStatus(ctx, ktypes.NamespacedName{Name: gateway.Name, Namespace: gateway.Namespace}, gatewayStatus); err != nil {
@@ -576,7 +585,7 @@ func (p *Provider) loadGatewayListeners(ctx context.Context, gateway *gatev1.Gat
 					certificateNamespace = string(*certificateRef.Namespace)
 				}
 
-				if err := p.isReferenceGranted(groupGateway, kindGateway, gateway.Namespace, groupCore, "Secret", string(certificateRef.Name), certificateNamespace); err != nil {
+				if err := p.isReferenceGranted(kindGateway, gateway.Namespace, groupCore, "Secret", string(certificateRef.Name), certificateNamespace); err != nil {
 					gatewayListeners[i].Status.Conditions = append(gatewayListeners[i].Status.Conditions, metav1.Condition{
 						Type:               string(gatev1.ListenerConditionResolvedRefs),
 						Status:             metav1.ConditionFalse,
@@ -631,10 +640,10 @@ func (p *Provider) loadGatewayListeners(ctx context.Context, gateway *gatev1.Gat
 	return gatewayListeners
 }
 
-func (p *Provider) makeGatewayStatus(gateway *gatev1.Gateway, listeners []gatewayListener, addresses []gatev1.GatewayStatusAddress) (gatev1.GatewayStatus, error) {
+func (p *Provider) makeGatewayStatus(gateway *gatev1.Gateway, listeners []gatewayListener, addresses []gatev1.GatewayStatusAddress) (gatev1.GatewayStatus, []metav1.Condition) {
 	gatewayStatus := gatev1.GatewayStatus{Addresses: addresses}
 
-	var result error
+	var errorConditions []metav1.Condition
 	for _, listener := range listeners {
 		if len(listener.Status.Conditions) == 0 {
 			listener.Status.Conditions = append(listener.Status.Conditions,
@@ -669,14 +678,11 @@ func (p *Provider) makeGatewayStatus(gateway *gatev1.Gateway, listeners []gatewa
 			continue
 		}
 
-		for _, condition := range listener.Status.Conditions {
+		errorConditions = append(errorConditions, listener.Status.Conditions...)
-			result = multierror.Append(result, errors.New(condition.Message))
-		}
-
 		gatewayStatus.Listeners = append(gatewayStatus.Listeners, *listener.Status)
 	}
 
-	if result != nil {
+	if len(errorConditions) > 0 {
 		// GatewayConditionReady "Ready", GatewayConditionReason "ListenersNotValid"
 		gatewayStatus.Conditions = append(gatewayStatus.Conditions, metav1.Condition{
 			Type:               string(gatev1.GatewayConditionAccepted),
@@ -687,7 +693,7 @@ func (p *Provider) makeGatewayStatus(gateway *gatev1.Gateway, listeners []gatewa
 			Message:            "All Listeners must be valid",
 		})
 
-		return gatewayStatus, result
+		return gatewayStatus, errorConditions
 	}
 
 	gatewayStatus.Conditions = append(gatewayStatus.Conditions,
@@ -783,7 +789,7 @@ func (p *Provider) entryPointName(port gatev1.PortNumber, protocol gatev1.Protoc
 	return "", fmt.Errorf("no matching entryPoint for port %d and protocol %q", port, protocol)
 }
 
-func (p *Provider) isReferenceGranted(fromGroup, fromKind, fromNamespace, toGroup, toKind, toName, toNamespace string) error {
+func (p *Provider) isReferenceGranted(fromKind, fromNamespace, toGroup, toKind, toName, toNamespace string) error {
 	if toNamespace == fromNamespace {
 		return nil
 	}
@@ -793,7 +799,7 @@ func (p *Provider) isReferenceGranted(fromGroup, fromKind, fromNamespace, toGrou
 		return fmt.Errorf("listing ReferenceGrant: %w", err)
 	}
 
-	refGrants = filterReferenceGrantsFrom(refGrants, fromGroup, fromKind, fromNamespace)
+	refGrants = filterReferenceGrantsFrom(refGrants, groupGateway, fromKind, fromNamespace)
 	refGrants = filterReferenceGrantsTo(refGrants, toGroup, toKind, toName)
 	if len(refGrants) == 0 {
 		return errors.New("missing ReferenceGrant")

pkg/provider/kubernetes/gateway/tcproute.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"net"
 	"strconv"
+	"strings"
 
 	"github.com/rs/zerolog/log"
 	"github.com/traefik/traefik/v3/pkg/config/dynamic"
@@ -108,182 +109,241 @@ func (p *Provider) loadTCPRoute(listener gatewayListener, route *gatev1alpha2.TC
 		Reason:             string(gatev1.RouteConditionResolvedRefs),
 	}
 
-	router := dynamic.TCPRouter{
+	for ri, rule := range route.Spec.Rules {
-		Rule:        "HostSNI(`*`)",
-		EntryPoints: []string{listener.EPName},
-		RuleSyntax:  "v3",
-	}
-
-	if listener.Protocol == gatev1.TLSProtocolType && listener.TLS != nil {
-		// TODO support let's encrypt
-		router.TLS = &dynamic.RouterTCPTLSConfig{
-			Passthrough: listener.TLS.Mode != nil && *listener.TLS.Mode == gatev1.TLSModePassthrough,
-		}
-	}
-
-	// Adding the gateway desc and the entryPoint desc prevents overlapping of routers build from the same routes.
-	routerName := provider.Normalize(route.Namespace + "-" + route.Name + "-" + listener.GWName + "-" + listener.EPName)
-
-	var ruleServiceNames []string
-	for i, rule := range route.Spec.Rules {
 		if rule.BackendRefs == nil {
 			// Should not happen due to validation.
 			// https://github.com/kubernetes-sigs/gateway-api/blob/v0.4.0/apis/v1alpha2/tcproute_types.go#L76
 			continue
 		}
 
-		wrrService, subServices, err := p.loadTCPServices(route.Namespace, rule.BackendRefs)
+		router := dynamic.TCPRouter{
-		if err != nil {
+			Rule:        "HostSNI(`*`)",
-			return conf, metav1.Condition{
+			EntryPoints: []string{listener.EPName},
-				Type:               string(gatev1.RouteConditionResolvedRefs),
+			RuleSyntax:  "v3",
-				Status:             metav1.ConditionFalse,
+		}
-				ObservedGeneration: route.Generation,
+
-				LastTransitionTime: metav1.Now(),
+		if listener.Protocol == gatev1.TLSProtocolType && listener.TLS != nil {
-				Reason:             string(gatev1.RouteReasonBackendNotFound),
+			router.TLS = &dynamic.RouterTCPTLSConfig{
-				Message:            fmt.Sprintf("Cannot load TCPRoute service %s/%s: %v", route.Namespace, route.Name, err),
+				Passthrough: listener.TLS.Mode != nil && *listener.TLS.Mode == gatev1.TLSModePassthrough,
 			}
 		}
 
-		for svcName, svc := range subServices {
+		// Adding the gateway desc and the entryPoint desc prevents overlapping of routers build from the same routes.
-			conf.TCP.Services[svcName] = svc
+		routeKey := provider.Normalize(fmt.Sprintf("%s-%s-%s-%s-%d", route.Namespace, route.Name, listener.GWName, listener.EPName, ri))
+		// Routing criteria should be introduced at some point.
+		routerName := makeRouterName("", routeKey)
+
+		if len(rule.BackendRefs) == 1 && isInternalService(rule.BackendRefs[0]) {
+			router.Service = string(rule.BackendRefs[0].Name)
+			conf.TCP.Routers[routerName] = &router
+			continue
 		}
 
-		serviceName := fmt.Sprintf("%s-wrr-%d", routerName, i)
+		var serviceCondition *metav1.Condition
-		conf.TCP.Services[serviceName] = wrrService
+		router.Service, serviceCondition = p.loadTCPWRRService(conf, routerName, rule.BackendRefs, route)
+		if serviceCondition != nil {
+			condition = *serviceCondition
+		}
 
-		ruleServiceNames = append(ruleServiceNames, serviceName)
-	}
-
-	if len(ruleServiceNames) == 1 {
-		router.Service = ruleServiceNames[0]
 		conf.TCP.Routers[routerName] = &router
-		return conf, condition
 	}
 
-	routeServiceKey := routerName + "-wrr"
-	routeService := &dynamic.TCPService{Weighted: &dynamic.TCPWeightedRoundRobin{}}
-
-	for _, name := range ruleServiceNames {
-		service := dynamic.TCPWRRService{Name: name}
-		service.SetDefaults()
-
-		routeService.Weighted.Services = append(routeService.Weighted.Services, service)
-	}
-
-	conf.TCP.Services[routeServiceKey] = routeService
-
-	router.Service = routeServiceKey
-	conf.TCP.Routers[routerName] = &router
-
 	return conf, condition
 }
 
-// loadTCPServices is generating a WRR service, even when there is only one target.
+// loadTCPWRRService is generating a WRR service, even when there is only one target.
-func (p *Provider) loadTCPServices(namespace string, backendRefs []gatev1.BackendRef) (*dynamic.TCPService, map[string]*dynamic.TCPService, error) {
+func (p *Provider) loadTCPWRRService(conf *dynamic.Configuration, routeKey string, backendRefs []gatev1.BackendRef, route *gatev1alpha2.TCPRoute) (string, *metav1.Condition) {
-	services := map[string]*dynamic.TCPService{}
+	name := routeKey + "-wrr"
-
+	if _, ok := conf.TCP.Services[name]; ok {
-	wrrSvc := &dynamic.TCPService{
+		return name, nil
-		Weighted: &dynamic.TCPWeightedRoundRobin{
-			Services: []dynamic.TCPWRRService{},
-		},
 	}
 
+	var wrr dynamic.TCPWeightedRoundRobin
+	var condition *metav1.Condition
 	for _, backendRef := range backendRefs {
-		if backendRef.Group == nil || backendRef.Kind == nil {
+		svcName, svc, errCondition := p.loadTCPService(route, backendRef)
-			// Should not happen as this is validated by kubernetes
+		weight := ptr.To(int(ptr.Deref(backendRef.Weight, 1)))
+
+		if errCondition != nil {
+			condition = errCondition
+
+			errName := routeKey + "-err-lb"
+			conf.TCP.Services[errName] = &dynamic.TCPService{
+				LoadBalancer: &dynamic.TCPServersLoadBalancer{
+					Servers: []dynamic.TCPServer{},
+				},
+			}
+
+			wrr.Services = append(wrr.Services, dynamic.TCPWRRService{
+				Name:   errName,
+				Weight: weight,
+			})
 			continue
 		}
 
-		if isInternalService(backendRef) {
+		if svc != nil {
-			return nil, nil, fmt.Errorf("traefik internal service %s is not allowed in a WRR loadbalancer", backendRef.Name)
+			conf.TCP.Services[svcName] = svc
 		}
 
-		weight := int(ptr.Deref(backendRef.Weight, 1))
+		wrr.Services = append(wrr.Services, dynamic.TCPWRRService{
+			Name:   svcName,
+			Weight: weight,
+		})
+	}
 
-		if isTraefikService(backendRef) {
+	conf.TCP.Services[name] = &dynamic.TCPService{Weighted: &wrr}
-			wrrSvc.Weighted.Services = append(wrrSvc.Weighted.Services, dynamic.TCPWRRService{Name: string(backendRef.Name), Weight: &weight})
+	return name, condition
-			continue
+}
+
+func (p *Provider) loadTCPService(route *gatev1alpha2.TCPRoute, backendRef gatev1.BackendRef) (string, *dynamic.TCPService, *metav1.Condition) {
+	kind := ptr.Deref(backendRef.Kind, kindService)
+
+	group := groupCore
+	if backendRef.Group != nil && *backendRef.Group != "" {
+		group = string(*backendRef.Group)
+	}
+
+	namespace := route.Namespace
+	if backendRef.Namespace != nil && *backendRef.Namespace != "" {
+		namespace = string(*backendRef.Namespace)
+	}
+
+	serviceName := provider.Normalize(namespace + "-" + string(backendRef.Name))
+
+	if err := p.isReferenceGranted(kindTCPRoute, route.Namespace, group, string(kind), string(backendRef.Name), namespace); err != nil {
+		return serviceName, nil, &metav1.Condition{
+			Type:               string(gatev1.RouteConditionResolvedRefs),
+			Status:             metav1.ConditionFalse,
+			ObservedGeneration: route.Generation,
+			LastTransitionTime: metav1.Now(),
+			Reason:             string(gatev1.RouteReasonRefNotPermitted),
+			Message:            fmt.Sprintf("Cannot load TCPRoute BackendRef %s/%s/%s/%s: %s", group, kind, namespace, backendRef.Name, err),
 		}
+	}
 
-		if *backendRef.Group != "" && *backendRef.Group != groupCore && *backendRef.Kind != "Service" {
+	if group != groupCore || kind != kindService {
-			return nil, nil, fmt.Errorf("unsupported BackendRef %s/%s/%s", *backendRef.Group, *backendRef.Kind, backendRef.Name)
+		name, err := p.loadTCPBackendRef(backendRef)
-		}
-
-		if backendRef.Port == nil {
-			return nil, nil, errors.New("port is required for Kubernetes Service reference")
-		}
-
-		service, exists, err := p.client.GetService(namespace, string(backendRef.Name))
 		if err != nil {
-			return nil, nil, fmt.Errorf("getting service: %w", err)
+			return serviceName, nil, &metav1.Condition{
-		}
+				Type:               string(gatev1.RouteConditionResolvedRefs),
-		if !exists {
+				Status:             metav1.ConditionFalse,
-			return nil, nil, errors.New("service not found")
+				ObservedGeneration: route.Generation,
+				LastTransitionTime: metav1.Now(),
+				Reason:             string(gatev1.RouteReasonInvalidKind),
+				Message:            fmt.Sprintf("Cannot load TCPRoute BackendRef %s/%s/%s/%s: %s", group, kind, namespace, backendRef.Name, err),
+			}
 		}
 
-		var svcPort *corev1.ServicePort
+		return name, nil, nil
-		for _, p := range service.Spec.Ports {
+	}
-			if p.Port == int32(*backendRef.Port) {
+
-				svcPort = &p
+	port := ptr.Deref(backendRef.Port, gatev1.PortNumber(0))
+	if port == 0 {
+		return serviceName, nil, &metav1.Condition{
+			Type:               string(gatev1.RouteConditionResolvedRefs),
+			Status:             metav1.ConditionFalse,
+			ObservedGeneration: route.Generation,
+			LastTransitionTime: metav1.Now(),
+			Reason:             string(gatev1.RouteReasonUnsupportedProtocol),
+			Message:            fmt.Sprintf("Cannot load TCPRoute BackendRef %s/%s/%s/%s port is required", group, kind, namespace, backendRef.Name),
+		}
+	}
+
+	portStr := strconv.FormatInt(int64(port), 10)
+	serviceName = provider.Normalize(serviceName + "-" + portStr)
+
+	lb, err := p.loadTCPServers(namespace, backendRef)
+	if err != nil {
+		return serviceName, nil, &metav1.Condition{
+			Type:               string(gatev1.RouteConditionResolvedRefs),
+			Status:             metav1.ConditionFalse,
+			ObservedGeneration: route.Generation,
+			LastTransitionTime: metav1.Now(),
+			Reason:             string(gatev1.RouteReasonBackendNotFound),
+			Message:            fmt.Sprintf("Cannot load TCPRoute BackendRef %s/%s/%s/%s: %s", group, kind, namespace, backendRef.Name, err),
+		}
+	}
+
+	return serviceName, &dynamic.TCPService{LoadBalancer: lb}, nil
+}
+
+func (p *Provider) loadTCPServers(namespace string, backendRef gatev1.BackendRef) (*dynamic.TCPServersLoadBalancer, error) {
+	if backendRef.Port == nil {
+		return nil, errors.New("port is required for Kubernetes Service reference")
+	}
+
+	service, exists, err := p.client.GetService(namespace, string(backendRef.Name))
+	if err != nil {
+		return nil, fmt.Errorf("getting service: %w", err)
+	}
+	if !exists {
+		return nil, errors.New("service not found")
+	}
+
+	var svcPort *corev1.ServicePort
+	for _, p := range service.Spec.Ports {
+		if p.Port == int32(*backendRef.Port) {
+			svcPort = &p
+			break
+		}
+	}
+	if svcPort == nil {
+		return nil, fmt.Errorf("service port %d not found", *backendRef.Port)
+	}
+
+	endpointSlices, err := p.client.ListEndpointSlicesForService(namespace, string(backendRef.Name))
+	if err != nil {
+		return nil, fmt.Errorf("getting endpointslices: %w", err)
+	}
+	if len(endpointSlices) == 0 {
+		return nil, errors.New("endpointslices not found")
+	}
+
+	lb := &dynamic.TCPServersLoadBalancer{}
+
+	addresses := map[string]struct{}{}
+	for _, endpointSlice := range endpointSlices {
+		var port int32
+		for _, p := range endpointSlice.Ports {
+			if svcPort.Name == *p.Name {
+				port = *p.Port
 				break
 			}
 		}
-		if svcPort == nil {
+		if port == 0 {
-			return nil, nil, fmt.Errorf("service port %d not found", *backendRef.Port)
+			continue
 		}
 
-		endpointSlices, err := p.client.ListEndpointSlicesForService(namespace, string(backendRef.Name))
+		for _, endpoint := range endpointSlice.Endpoints {
-		if err != nil {
+			if endpoint.Conditions.Ready == nil || !*endpoint.Conditions.Ready {
-			return nil, nil, fmt.Errorf("getting endpointslices: %w", err)
-		}
-		if len(endpointSlices) == 0 {
-			return nil, nil, errors.New("endpointslices not found")
-		}
-
-		svc := dynamic.TCPService{LoadBalancer: &dynamic.TCPServersLoadBalancer{}}
-
-		addresses := map[string]struct{}{}
-		for _, endpointSlice := range endpointSlices {
-			var port int32
-			for _, p := range endpointSlice.Ports {
-				if svcPort.Name == *p.Name {
-					port = *p.Port
-					break
-				}
-			}
-			if port == 0 {
 				continue
 			}
 
-			for _, endpoint := range endpointSlice.Endpoints {
+			for _, address := range endpoint.Addresses {
-				if endpoint.Conditions.Ready == nil || !*endpoint.Conditions.Ready {
+				if _, ok := addresses[address]; ok {
 					continue
 				}
 
-				for _, address := range endpoint.Addresses {
+				addresses[address] = struct{}{}
-					if _, ok := addresses[address]; ok {
+				lb.Servers = append(lb.Servers, dynamic.TCPServer{
-						continue
+					// TODO determine whether the servers needs TLS, from the port?
-					}
+					Address: net.JoinHostPort(address, strconv.Itoa(int(port))),
-
+				})
-					addresses[address] = struct{}{}
-					svc.LoadBalancer.Servers = append(svc.LoadBalancer.Servers, dynamic.TCPServer{
-						Address: net.JoinHostPort(address, strconv.Itoa(int(port))),
-					})
-				}
 			}
 		}
-
-		serviceName := provider.Normalize(service.Namespace + "-" + service.Name + "-" + strconv.Itoa(int(svcPort.Port)))
-		services[serviceName] = &svc
-
-		wrrSvc.Weighted.Services = append(wrrSvc.Weighted.Services, dynamic.TCPWRRService{Name: serviceName, Weight: &weight})
 	}
 
-	if len(wrrSvc.Weighted.Services) == 0 {
+	return lb, nil
-		return nil, nil, errors.New("no service has been created")
+}
+
+func (p *Provider) loadTCPBackendRef(backendRef gatev1.BackendRef) (string, error) {
+	// Support for cross-provider references (e.g: api@internal).
+	// This provides the same behavior as for IngressRoutes.
+	if *backendRef.Kind == "TraefikService" && strings.Contains(string(backendRef.Name), "@") {
+		return string(backendRef.Name), nil
 	}
 
-	return wrrSvc, services, nil
+	return "", fmt.Errorf("unsupported BackendRef %s/%s/%s", *backendRef.Group, *backendRef.Kind, backendRef.Name)
 }
 
 func mergeTCPConfiguration(from, to *dynamic.Configuration) {

pkg/provider/kubernetes/gateway/tlsroute.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"regexp"
+	"strconv"
 	"strings"
 
 	"github.com/rs/zerolog/log"
@@ -11,6 +12,7 @@ import (
 	"github.com/traefik/traefik/v3/pkg/provider"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	ktypes "k8s.io/apimachinery/pkg/types"
+	"k8s.io/utils/ptr"
 	gatev1 "sigs.k8s.io/gateway-api/apis/v1"
 	gatev1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2"
 )
@@ -109,74 +111,159 @@ func (p *Provider) loadTLSRoute(listener gatewayListener, route *gatev1alpha2.TL
 		Reason:             string(gatev1.RouteConditionResolvedRefs),
 	}
 
-	router := dynamic.TCPRouter{
+	for ri, routeRule := range route.Spec.Rules {
-		RuleSyntax:  "v3",
-		Rule:        hostSNIRule(hostnames),
-		EntryPoints: []string{listener.EPName},
-		TLS: &dynamic.RouterTCPTLSConfig{
-			Passthrough: listener.TLS != nil && listener.TLS.Mode != nil && *listener.TLS.Mode == gatev1.TLSModePassthrough,
-		},
-	}
-
-	// Adding the gateway desc and the entryPoint desc prevents overlapping of routers build from the same routes.
-	routeKey := provider.Normalize(route.Namespace + "-" + route.Name + "-" + listener.GWName + "-" + listener.EPName)
-	routerName := makeRouterName(router.Rule, routeKey)
-
-	var ruleServiceNames []string
-	for i, routeRule := range route.Spec.Rules {
 		if len(routeRule.BackendRefs) == 0 {
 			// Should not happen due to validation.
 			// https://github.com/kubernetes-sigs/gateway-api/blob/v0.4.0/apis/v1alpha2/tlsroute_types.go#L120
 			continue
 		}
 
-		wrrService, subServices, err := p.loadTCPServices(route.Namespace, routeRule.BackendRefs)
+		router := dynamic.TCPRouter{
+			RuleSyntax:  "v3",
+			Rule:        hostSNIRule(hostnames),
+			EntryPoints: []string{listener.EPName},
+			TLS: &dynamic.RouterTCPTLSConfig{
+				Passthrough: listener.TLS != nil && listener.TLS.Mode != nil && *listener.TLS.Mode == gatev1.TLSModePassthrough,
+			},
+		}
+
+		// Adding the gateway desc and the entryPoint desc prevents overlapping of routers build from the same routes.
+		routeKey := provider.Normalize(fmt.Sprintf("%s-%s-%s-%s-%d", route.Namespace, route.Name, listener.GWName, listener.EPName, ri))
+		// Routing criteria should be introduced at some point.
+		routerName := makeRouterName("", routeKey)
+
+		if len(routeRule.BackendRefs) == 1 && isInternalService(routeRule.BackendRefs[0]) {
+			router.Service = string(routeRule.BackendRefs[0].Name)
+			conf.TCP.Routers[routerName] = &router
+			continue
+		}
+
+		var serviceCondition *metav1.Condition
+		router.Service, serviceCondition = p.loadTLSWRRService(conf, routerName, routeRule.BackendRefs, route)
+		if serviceCondition != nil {
+			condition = *serviceCondition
+		}
+
+		conf.TCP.Routers[routerName] = &router
+	}
+
+	return conf, condition
+}
+
+// loadTLSWRRService is generating a WRR service, even when there is only one target.
+func (p *Provider) loadTLSWRRService(conf *dynamic.Configuration, routeKey string, backendRefs []gatev1.BackendRef, route *gatev1alpha2.TLSRoute) (string, *metav1.Condition) {
+	name := routeKey + "-wrr"
+	if _, ok := conf.TCP.Services[name]; ok {
+		return name, nil
+	}
+
+	var wrr dynamic.TCPWeightedRoundRobin
+	var condition *metav1.Condition
+	for _, backendRef := range backendRefs {
+		svcName, svc, errCondition := p.loadTLSService(route, backendRef)
+		weight := ptr.To(int(ptr.Deref(backendRef.Weight, 1)))
+
+		if errCondition != nil {
+			condition = errCondition
+
+			errName := routeKey + "-err-lb"
+			conf.TCP.Services[errName] = &dynamic.TCPService{
+				LoadBalancer: &dynamic.TCPServersLoadBalancer{
+					Servers: []dynamic.TCPServer{},
+				},
+			}
+
+			wrr.Services = append(wrr.Services, dynamic.TCPWRRService{
+				Name:   errName,
+				Weight: weight,
+			})
+			continue
+		}
+
+		if svc != nil {
+			conf.TCP.Services[svcName] = svc
+		}
+
+		wrr.Services = append(wrr.Services, dynamic.TCPWRRService{
+			Name:   svcName,
+			Weight: weight,
+		})
+	}
+
+	conf.TCP.Services[name] = &dynamic.TCPService{Weighted: &wrr}
+	return name, condition
+}
+
+func (p *Provider) loadTLSService(route *gatev1alpha2.TLSRoute, backendRef gatev1.BackendRef) (string, *dynamic.TCPService, *metav1.Condition) {
+	kind := ptr.Deref(backendRef.Kind, kindService)
+
+	group := groupCore
+	if backendRef.Group != nil && *backendRef.Group != "" {
+		group = string(*backendRef.Group)
+	}
+
+	namespace := route.Namespace
+	if backendRef.Namespace != nil && *backendRef.Namespace != "" {
+		namespace = string(*backendRef.Namespace)
+	}
+
+	serviceName := provider.Normalize(namespace + "-" + string(backendRef.Name))
+
+	if err := p.isReferenceGranted(kindTLSRoute, route.Namespace, group, string(kind), string(backendRef.Name), namespace); err != nil {
+		return serviceName, nil, &metav1.Condition{
+			Type:               string(gatev1.RouteConditionResolvedRefs),
+			Status:             metav1.ConditionFalse,
+			ObservedGeneration: route.Generation,
+			LastTransitionTime: metav1.Now(),
+			Reason:             string(gatev1.RouteReasonRefNotPermitted),
+			Message:            fmt.Sprintf("Cannot load TLSRoute BackendRef %s/%s/%s/%s: %s", group, kind, namespace, backendRef.Name, err),
+		}
+	}
+
+	if group != groupCore || kind != kindService {
+		name, err := p.loadTCPBackendRef(backendRef)
 		if err != nil {
-			// update "ResolvedRefs" status true with "InvalidBackendRefs" reason
+			return serviceName, nil, &metav1.Condition{
-			condition = metav1.Condition{
 				Type:               string(gatev1.RouteConditionResolvedRefs),
 				Status:             metav1.ConditionFalse,
 				ObservedGeneration: route.Generation,
 				LastTransitionTime: metav1.Now(),
-				Reason:             string(gatev1.RouteReasonBackendNotFound),
+				Reason:             string(gatev1.RouteReasonInvalidKind),
-				Message:            fmt.Sprintf("Cannot load TLSRoute service %s/%s: %v", route.Namespace, route.Name, err),
+				Message:            fmt.Sprintf("Cannot load TLSRoute BackendRef %s/%s/%s/%s: %s", group, kind, namespace, backendRef.Name, err),
 			}
-			continue
 		}
 
-		for svcName, svc := range subServices {
+		return name, nil, nil
-			conf.TCP.Services[svcName] = svc
+	}
+
+	port := ptr.Deref(backendRef.Port, gatev1.PortNumber(0))
+	if port == 0 {
+		return serviceName, nil, &metav1.Condition{
+			Type:               string(gatev1.RouteConditionResolvedRefs),
+			Status:             metav1.ConditionFalse,
+			ObservedGeneration: route.Generation,
+			LastTransitionTime: metav1.Now(),
+			Reason:             string(gatev1.RouteReasonUnsupportedProtocol),
+			Message:            fmt.Sprintf("Cannot load TLSRoute BackendRef %s/%s/%s/%s port is required", group, kind, namespace, backendRef.Name),
 		}
-
-		serviceName := fmt.Sprintf("%s-wrr-%d", routerName, i)
-		conf.TCP.Services[serviceName] = wrrService
-
-		ruleServiceNames = append(ruleServiceNames, serviceName)
 	}
 
-	if len(ruleServiceNames) == 1 {
+	portStr := strconv.FormatInt(int64(port), 10)
-		router.Service = ruleServiceNames[0]
+	serviceName = provider.Normalize(serviceName + "-" + portStr)
-		conf.TCP.Routers[routerName] = &router
 
-		return conf, condition
+	lb, err := p.loadTCPServers(namespace, backendRef)
+	if err != nil {
+		return serviceName, nil, &metav1.Condition{
+			Type:               string(gatev1.RouteConditionResolvedRefs),
+			Status:             metav1.ConditionFalse,
+			ObservedGeneration: route.Generation,
+			LastTransitionTime: metav1.Now(),
+			Reason:             string(gatev1.RouteReasonBackendNotFound),
+			Message:            fmt.Sprintf("Cannot load TLSRoute BackendRef %s/%s/%s/%s: %s", group, kind, namespace, backendRef.Name, err),
+		}
 	}
 
-	routeServiceKey := routerName + "-wrr"
+	return serviceName, &dynamic.TCPService{LoadBalancer: lb}, nil
-	routeService := &dynamic.TCPService{Weighted: &dynamic.TCPWeightedRoundRobin{}}
-
-	for _, name := range ruleServiceNames {
-		service := dynamic.TCPWRRService{Name: name}
-		service.SetDefaults()
-
-		routeService.Weighted.Services = append(routeService.Weighted.Services, service)
-	}
-
-	conf.TCP.Services[routeServiceKey] = routeService
-
-	router.Service = routeServiceKey
-	conf.TCP.Routers[routerName] = &router
-
-	return conf, condition
 }
 
 func hostSNIRule(hostnames []gatev1.Hostname) string {