Improve Kubernetes GatewayAPI TCPRoute and TLSRoute support

This commit is contained in:
Romain 2024-09-03 12:10:04 +02:00 committed by GitHub
parent 0b34e0cdcb
commit 3eb7ecce19
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 690 additions and 484 deletions

View file

@ -53,3 +53,11 @@ profiles:
- HTTPRouteRequestTimeout - HTTPRouteRequestTimeout
name: GATEWAY-HTTP name: GATEWAY-HTTP
summary: Core tests succeeded. Extended tests succeeded. summary: Core tests succeeded. Extended tests succeeded.
- core:
result: success
statistics:
Failed: 0
Passed: 11
Skipped: 0
name: GATEWAY-TLS
summary: Core tests succeeded.

View file

@ -8,6 +8,7 @@ import (
"os" "os"
"path/filepath" "path/filepath"
"slices" "slices"
"strings"
"testing" "testing"
"time" "time"
@ -193,7 +194,11 @@ func (s *K8sConformanceSuite) TestK8sGatewayAPIConformance() {
Version: *k8sConformanceTraefikVersion, Version: *k8sConformanceTraefikVersion,
Contact: []string{"@traefik/maintainers"}, Contact: []string{"@traefik/maintainers"},
}, },
ConformanceProfiles: sets.New(ksuite.GatewayHTTPConformanceProfileName, ksuite.GatewayGRPCConformanceProfileName), ConformanceProfiles: sets.New(
ksuite.GatewayHTTPConformanceProfileName,
ksuite.GatewayGRPCConformanceProfileName,
ksuite.GatewayTLSConformanceProfileName,
),
SupportedFeatures: sets.New( SupportedFeatures: sets.New(
features.SupportGateway, features.SupportGateway,
features.SupportGatewayPort8080, features.SupportGatewayPort8080,
@ -207,6 +212,7 @@ func (s *K8sConformanceSuite) TestK8sGatewayAPIConformance() {
features.SupportHTTPRoutePathRewrite, features.SupportHTTPRoutePathRewrite,
features.SupportHTTPRoutePathRedirect, features.SupportHTTPRoutePathRedirect,
features.SupportHTTPRouteResponseHeaderModification, features.SupportHTTPRouteResponseHeaderModification,
features.SupportTLSRoute,
), ),
}) })
require.NoError(s.T(), err) require.NoError(s.T(), err)
@ -224,6 +230,11 @@ func (s *K8sConformanceSuite) TestK8sGatewayAPIConformance() {
// TODO: to publish this report automatically, we have to figure out how to handle the date diff. // TODO: to publish this report automatically, we have to figure out how to handle the date diff.
report.Date = "-" report.Date = "-"
// Ordering profile reports for the serialized report to be comparable.
slices.SortFunc(report.ProfileReports, func(a, b v1.ProfileReport) int {
return strings.Compare(a.Name, b.Name)
})
rawReport, err := yaml.Marshal(report) rawReport, err := yaml.Marshal(report)
require.NoError(s.T(), err) require.NoError(s.T(), err)
s.T().Logf("Conformance report:\n%s", string(rawReport)) s.T().Logf("Conformance report:\n%s", string(rawReport))

View file

@ -34,7 +34,7 @@
"entryPoints": [ "entryPoints": [
"web" "web"
], ],
"service": "default-http-app-1-my-gateway-web-0-wrr", "service": "default-http-app-1-my-gateway-web-0-1c0cf64bde37d9d0df06-wrr",
"rule": "Host(`foo.com`) \u0026\u0026 Path(`/bar`)", "rule": "Host(`foo.com`) \u0026\u0026 Path(`/bar`)",
"ruleSyntax": "v3", "ruleSyntax": "v3",
"priority": 100008, "priority": 100008,
@ -47,7 +47,7 @@
"entryPoints": [ "entryPoints": [
"websecure" "websecure"
], ],
"service": "default-http-app-1-my-https-gateway-websecure-0-wrr", "service": "default-http-app-1-my-https-gateway-websecure-0-1c0cf64bde37d9d0df06-wrr",
"rule": "Host(`foo.com`) \u0026\u0026 Path(`/bar`)", "rule": "Host(`foo.com`) \u0026\u0026 Path(`/bar`)",
"ruleSyntax": "v3", "ruleSyntax": "v3",
"priority": 100008, "priority": 100008,
@ -96,7 +96,7 @@
"dashboard@internal" "dashboard@internal"
] ]
}, },
"default-http-app-1-my-gateway-web-0-wrr@kubernetesgateway": { "default-http-app-1-my-gateway-web-0-1c0cf64bde37d9d0df06-wrr@kubernetesgateway": {
"weighted": { "weighted": {
"services": [ "services": [
{ {
@ -110,7 +110,7 @@
"default-http-app-1-my-gateway-web-0-1c0cf64bde37d9d0df06@kubernetesgateway" "default-http-app-1-my-gateway-web-0-1c0cf64bde37d9d0df06@kubernetesgateway"
] ]
}, },
"default-http-app-1-my-https-gateway-websecure-0-wrr@kubernetesgateway": { "default-http-app-1-my-https-gateway-websecure-0-1c0cf64bde37d9d0df06-wrr@kubernetesgateway": {
"weighted": { "weighted": {
"services": [ "services": [
{ {
@ -150,11 +150,11 @@
} }
}, },
"tcpRouters": { "tcpRouters": {
"default-tcp-app-1-my-tcp-gateway-footcp@kubernetesgateway": { "default-tcp-app-1-my-tcp-gateway-footcp-0-e3b0c44298fc1c149afb@kubernetesgateway": {
"entryPoints": [ "entryPoints": [
"footcp" "footcp"
], ],
"service": "default-tcp-app-1-my-tcp-gateway-footcp-wrr-0", "service": "default-tcp-app-1-my-tcp-gateway-footcp-0-e3b0c44298fc1c149afb-wrr",
"rule": "HostSNI(`*`)", "rule": "HostSNI(`*`)",
"ruleSyntax": "v3", "ruleSyntax": "v3",
"priority": -1, "priority": -1,
@ -163,11 +163,11 @@
"footcp" "footcp"
] ]
}, },
"default-tcp-app-1-my-tls-gateway-footlsterminate@kubernetesgateway": { "default-tcp-app-1-my-tls-gateway-footlsterminate-0-e3b0c44298fc1c149afb@kubernetesgateway": {
"entryPoints": [ "entryPoints": [
"footlsterminate" "footlsterminate"
], ],
"service": "default-tcp-app-1-my-tls-gateway-footlsterminate-wrr-0", "service": "default-tcp-app-1-my-tls-gateway-footlsterminate-0-e3b0c44298fc1c149afb-wrr",
"rule": "HostSNI(`*`)", "rule": "HostSNI(`*`)",
"ruleSyntax": "v3", "ruleSyntax": "v3",
"priority": -1, "priority": -1,
@ -179,11 +179,11 @@
"footlsterminate" "footlsterminate"
] ]
}, },
"default-tls-app-1-my-tls-gateway-footlspassthrough-2279fe75c5156dc5eb26@kubernetesgateway": { "default-tls-app-1-my-tls-gateway-footlspassthrough-0-e3b0c44298fc1c149afb@kubernetesgateway": {
"entryPoints": [ "entryPoints": [
"footlspassthrough" "footlspassthrough"
], ],
"service": "default-tls-app-1-my-tls-gateway-footlspassthrough-2279fe75c5156dc5eb26-wrr-0", "service": "default-tls-app-1-my-tls-gateway-footlspassthrough-0-e3b0c44298fc1c149afb-wrr",
"rule": "HostSNI(`foo.bar`)", "rule": "HostSNI(`foo.bar`)",
"ruleSyntax": "v3", "ruleSyntax": "v3",
"priority": 18, "priority": 18,
@ -197,7 +197,7 @@
} }
}, },
"tcpServices": { "tcpServices": {
"default-tcp-app-1-my-tcp-gateway-footcp-wrr-0@kubernetesgateway": { "default-tcp-app-1-my-tcp-gateway-footcp-0-e3b0c44298fc1c149afb-wrr@kubernetesgateway": {
"weighted": { "weighted": {
"services": [ "services": [
{ {
@ -208,10 +208,10 @@
}, },
"status": "enabled", "status": "enabled",
"usedBy": [ "usedBy": [
"default-tcp-app-1-my-tcp-gateway-footcp@kubernetesgateway" "default-tcp-app-1-my-tcp-gateway-footcp-0-e3b0c44298fc1c149afb@kubernetesgateway"
] ]
}, },
"default-tcp-app-1-my-tls-gateway-footlsterminate-wrr-0@kubernetesgateway": { "default-tcp-app-1-my-tls-gateway-footlsterminate-0-e3b0c44298fc1c149afb-wrr@kubernetesgateway": {
"weighted": { "weighted": {
"services": [ "services": [
{ {
@ -222,10 +222,10 @@
}, },
"status": "enabled", "status": "enabled",
"usedBy": [ "usedBy": [
"default-tcp-app-1-my-tls-gateway-footlsterminate@kubernetesgateway" "default-tcp-app-1-my-tls-gateway-footlsterminate-0-e3b0c44298fc1c149afb@kubernetesgateway"
] ]
}, },
"default-tls-app-1-my-tls-gateway-footlspassthrough-2279fe75c5156dc5eb26-wrr-0@kubernetesgateway": { "default-tls-app-1-my-tls-gateway-footlspassthrough-0-e3b0c44298fc1c149afb-wrr@kubernetesgateway": {
"weighted": { "weighted": {
"services": [ "services": [
{ {
@ -236,7 +236,7 @@
}, },
"status": "enabled", "status": "enabled",
"usedBy": [ "usedBy": [
"default-tls-app-1-my-tls-gateway-footlspassthrough-2279fe75c5156dc5eb26@kubernetesgateway" "default-tls-app-1-my-tls-gateway-footlspassthrough-0-e3b0c44298fc1c149afb@kubernetesgateway"
] ]
}, },
"default-whoamitcp-8080@kubernetesgateway": { "default-whoamitcp-8080@kubernetesgateway": {

View file

@ -32,6 +32,10 @@ metadata:
name: TCP-app-1 name: TCP-app-1
namespace: default namespace: default
spec: spec:
parentRefs:
- name: my-gateway
kind: Gateway
group: gateway.networking.k8s.io
rules: rules:
- backendRefs: - backendRefs:
- name: whoami - name: whoami

View file

@ -36,16 +36,16 @@ spec:
group: "" group: ""
allowedRoutes: allowedRoutes:
kinds: kinds:
- kind: TCPRoute - kind: TLSRoute
group: gateway.networking.k8s.io group: gateway.networking.k8s.io
namespaces: namespaces:
from: Same from: Same
--- ---
kind: TCPRoute kind: TLSRoute
apiVersion: gateway.networking.k8s.io/v1alpha2 apiVersion: gateway.networking.k8s.io/v1alpha2
metadata: metadata:
name: tcp-app-1 name: tls-app-1
namespace: default namespace: default
spec: spec:
parentRefs: parentRefs:

View file

@ -216,7 +216,7 @@ func (p *Provider) loadGRPCService(conf *dynamic.Configuration, routeKey string,
} }
func (p *Provider) loadGRPCBackendRef(route *gatev1.GRPCRoute, backendRef gatev1.GRPCBackendRef) (string, *dynamic.Service, *metav1.Condition) { func (p *Provider) loadGRPCBackendRef(route *gatev1.GRPCRoute, backendRef gatev1.GRPCBackendRef) (string, *dynamic.Service, *metav1.Condition) {
kind := ptr.Deref(backendRef.Kind, "Service") kind := ptr.Deref(backendRef.Kind, kindService)
group := groupCore group := groupCore
if backendRef.Group != nil && *backendRef.Group != "" { if backendRef.Group != nil && *backendRef.Group != "" {
@ -230,7 +230,7 @@ func (p *Provider) loadGRPCBackendRef(route *gatev1.GRPCRoute, backendRef gatev1
serviceName := provider.Normalize(namespace + "-" + string(backendRef.Name)) serviceName := provider.Normalize(namespace + "-" + string(backendRef.Name))
if group != groupCore || kind != "Service" { if group != groupCore || kind != kindService {
return serviceName, nil, &metav1.Condition{ return serviceName, nil, &metav1.Condition{
Type: string(gatev1.RouteConditionResolvedRefs), Type: string(gatev1.RouteConditionResolvedRefs),
Status: metav1.ConditionFalse, Status: metav1.ConditionFalse,
@ -241,7 +241,7 @@ func (p *Provider) loadGRPCBackendRef(route *gatev1.GRPCRoute, backendRef gatev1
} }
} }
if err := p.isReferenceGranted(groupGateway, kindGRPCRoute, route.Namespace, group, string(kind), string(backendRef.Name), namespace); err != nil { if err := p.isReferenceGranted(kindGRPCRoute, route.Namespace, group, string(kind), string(backendRef.Name), namespace); err != nil {
return serviceName, nil, &metav1.Condition{ return serviceName, nil, &metav1.Condition{
Type: string(gatev1.RouteConditionResolvedRefs), Type: string(gatev1.RouteConditionResolvedRefs),
Status: metav1.ConditionFalse, Status: metav1.ConditionFalse,

View file

@ -158,7 +158,7 @@ func (p *Provider) loadHTTPRoute(ctx context.Context, listener gatewayListener,
default: default:
var serviceCondition *metav1.Condition var serviceCondition *metav1.Condition
router.Service, serviceCondition = p.loadService(conf, routeKey, routeRule, route) router.Service, serviceCondition = p.loadWRRService(conf, routerName, routeRule, route)
if serviceCondition != nil { if serviceCondition != nil {
condition = *serviceCondition condition = *serviceCondition
} }
@ -173,7 +173,7 @@ func (p *Provider) loadHTTPRoute(ctx context.Context, listener gatewayListener,
return conf, condition return conf, condition
} }
func (p *Provider) loadService(conf *dynamic.Configuration, routeKey string, routeRule gatev1.HTTPRouteRule, route *gatev1.HTTPRoute) (string, *metav1.Condition) { func (p *Provider) loadWRRService(conf *dynamic.Configuration, routeKey string, routeRule gatev1.HTTPRouteRule, route *gatev1.HTTPRoute) (string, *metav1.Condition) {
name := routeKey + "-wrr" name := routeKey + "-wrr"
if _, ok := conf.HTTP.Services[name]; ok { if _, ok := conf.HTTP.Services[name]; ok {
return name, nil return name, nil
@ -182,7 +182,7 @@ func (p *Provider) loadService(conf *dynamic.Configuration, routeKey string, rou
var wrr dynamic.WeightedRoundRobin var wrr dynamic.WeightedRoundRobin
var condition *metav1.Condition var condition *metav1.Condition
for _, backendRef := range routeRule.BackendRefs { for _, backendRef := range routeRule.BackendRefs {
svcName, svc, errCondition := p.loadHTTPService(route, backendRef) svcName, svc, errCondition := p.loadService(route, backendRef)
weight := ptr.To(int(ptr.Deref(backendRef.Weight, 1))) weight := ptr.To(int(ptr.Deref(backendRef.Weight, 1)))
if errCondition != nil { if errCondition != nil {
condition = errCondition condition = errCondition
@ -208,10 +208,10 @@ func (p *Provider) loadService(conf *dynamic.Configuration, routeKey string, rou
return name, condition return name, condition
} }
// loadHTTPService returns a dynamic.Service config corresponding to the given gatev1.HTTPBackendRef. // loadService returns a dynamic.Service config corresponding to the given gatev1.HTTPBackendRef.
// Note that the returned dynamic.Service config can be nil (for cross-provider, internal services, and backendFunc). // Note that the returned dynamic.Service config can be nil (for cross-provider, internal services, and backendFunc).
func (p *Provider) loadHTTPService(route *gatev1.HTTPRoute, backendRef gatev1.HTTPBackendRef) (string, *dynamic.Service, *metav1.Condition) { func (p *Provider) loadService(route *gatev1.HTTPRoute, backendRef gatev1.HTTPBackendRef) (string, *dynamic.Service, *metav1.Condition) {
kind := ptr.Deref(backendRef.Kind, "Service") kind := ptr.Deref(backendRef.Kind, kindService)
group := groupCore group := groupCore
if backendRef.Group != nil && *backendRef.Group != "" { if backendRef.Group != nil && *backendRef.Group != "" {
@ -225,7 +225,7 @@ func (p *Provider) loadHTTPService(route *gatev1.HTTPRoute, backendRef gatev1.HT
serviceName := provider.Normalize(namespace + "-" + string(backendRef.Name)) serviceName := provider.Normalize(namespace + "-" + string(backendRef.Name))
if err := p.isReferenceGranted(groupGateway, kindHTTPRoute, route.Namespace, group, string(kind), string(backendRef.Name), namespace); err != nil { if err := p.isReferenceGranted(kindHTTPRoute, route.Namespace, group, string(kind), string(backendRef.Name), namespace); err != nil {
return serviceName, nil, &metav1.Condition{ return serviceName, nil, &metav1.Condition{
Type: string(gatev1.RouteConditionResolvedRefs), Type: string(gatev1.RouteConditionResolvedRefs),
Status: metav1.ConditionFalse, Status: metav1.ConditionFalse,
@ -236,7 +236,7 @@ func (p *Provider) loadHTTPService(route *gatev1.HTTPRoute, backendRef gatev1.HT
} }
} }
if group != groupCore || kind != "Service" { if group != groupCore || kind != kindService {
name, service, err := p.loadHTTPBackendRef(namespace, backendRef) name, service, err := p.loadHTTPBackendRef(namespace, backendRef)
if err != nil { if err != nil {
return serviceName, nil, &metav1.Condition{ return serviceName, nil, &metav1.Condition{

View file

@ -47,6 +47,7 @@ const (
kindGRPCRoute = "GRPCRoute" kindGRPCRoute = "GRPCRoute"
kindTCPRoute = "TCPRoute" kindTCPRoute = "TCPRoute"
kindTLSRoute = "TLSRoute" kindTLSRoute = "TLSRoute"
kindService = "Service"
) )
// Provider holds configurations of the provider. // Provider holds configurations of the provider.
@ -376,11 +377,19 @@ func (p *Provider) loadConfigurationFromGateways(ctx context.Context) *dynamic.C
} }
} }
gatewayStatus, err := p.makeGatewayStatus(gateway, listeners, addresses) gatewayStatus, errConditions := p.makeGatewayStatus(gateway, listeners, addresses)
if err != nil { if len(errConditions) > 0 {
messages := map[string]struct{}{}
for _, condition := range errConditions {
messages[condition.Message] = struct{}{}
}
var conditionsErr error
for message := range messages {
conditionsErr = multierror.Append(conditionsErr, errors.New(message))
}
logger.Error(). logger.Error().
Err(err). Err(conditionsErr).
Msg("Unable to create Gateway status") Msg("Gateway Not Accepted")
} }
if err = p.client.UpdateGatewayStatus(ctx, ktypes.NamespacedName{Name: gateway.Name, Namespace: gateway.Namespace}, gatewayStatus); err != nil { if err = p.client.UpdateGatewayStatus(ctx, ktypes.NamespacedName{Name: gateway.Name, Namespace: gateway.Namespace}, gatewayStatus); err != nil {
@ -576,7 +585,7 @@ func (p *Provider) loadGatewayListeners(ctx context.Context, gateway *gatev1.Gat
certificateNamespace = string(*certificateRef.Namespace) certificateNamespace = string(*certificateRef.Namespace)
} }
if err := p.isReferenceGranted(groupGateway, kindGateway, gateway.Namespace, groupCore, "Secret", string(certificateRef.Name), certificateNamespace); err != nil { if err := p.isReferenceGranted(kindGateway, gateway.Namespace, groupCore, "Secret", string(certificateRef.Name), certificateNamespace); err != nil {
gatewayListeners[i].Status.Conditions = append(gatewayListeners[i].Status.Conditions, metav1.Condition{ gatewayListeners[i].Status.Conditions = append(gatewayListeners[i].Status.Conditions, metav1.Condition{
Type: string(gatev1.ListenerConditionResolvedRefs), Type: string(gatev1.ListenerConditionResolvedRefs),
Status: metav1.ConditionFalse, Status: metav1.ConditionFalse,
@ -631,10 +640,10 @@ func (p *Provider) loadGatewayListeners(ctx context.Context, gateway *gatev1.Gat
return gatewayListeners return gatewayListeners
} }
func (p *Provider) makeGatewayStatus(gateway *gatev1.Gateway, listeners []gatewayListener, addresses []gatev1.GatewayStatusAddress) (gatev1.GatewayStatus, error) { func (p *Provider) makeGatewayStatus(gateway *gatev1.Gateway, listeners []gatewayListener, addresses []gatev1.GatewayStatusAddress) (gatev1.GatewayStatus, []metav1.Condition) {
gatewayStatus := gatev1.GatewayStatus{Addresses: addresses} gatewayStatus := gatev1.GatewayStatus{Addresses: addresses}
var result error var errorConditions []metav1.Condition
for _, listener := range listeners { for _, listener := range listeners {
if len(listener.Status.Conditions) == 0 { if len(listener.Status.Conditions) == 0 {
listener.Status.Conditions = append(listener.Status.Conditions, listener.Status.Conditions = append(listener.Status.Conditions,
@ -669,14 +678,11 @@ func (p *Provider) makeGatewayStatus(gateway *gatev1.Gateway, listeners []gatewa
continue continue
} }
for _, condition := range listener.Status.Conditions { errorConditions = append(errorConditions, listener.Status.Conditions...)
result = multierror.Append(result, errors.New(condition.Message))
}
gatewayStatus.Listeners = append(gatewayStatus.Listeners, *listener.Status) gatewayStatus.Listeners = append(gatewayStatus.Listeners, *listener.Status)
} }
if result != nil { if len(errorConditions) > 0 {
// GatewayConditionReady "Ready", GatewayConditionReason "ListenersNotValid" // GatewayConditionReady "Ready", GatewayConditionReason "ListenersNotValid"
gatewayStatus.Conditions = append(gatewayStatus.Conditions, metav1.Condition{ gatewayStatus.Conditions = append(gatewayStatus.Conditions, metav1.Condition{
Type: string(gatev1.GatewayConditionAccepted), Type: string(gatev1.GatewayConditionAccepted),
@ -687,7 +693,7 @@ func (p *Provider) makeGatewayStatus(gateway *gatev1.Gateway, listeners []gatewa
Message: "All Listeners must be valid", Message: "All Listeners must be valid",
}) })
return gatewayStatus, result return gatewayStatus, errorConditions
} }
gatewayStatus.Conditions = append(gatewayStatus.Conditions, gatewayStatus.Conditions = append(gatewayStatus.Conditions,
@ -783,7 +789,7 @@ func (p *Provider) entryPointName(port gatev1.PortNumber, protocol gatev1.Protoc
return "", fmt.Errorf("no matching entryPoint for port %d and protocol %q", port, protocol) return "", fmt.Errorf("no matching entryPoint for port %d and protocol %q", port, protocol)
} }
func (p *Provider) isReferenceGranted(fromGroup, fromKind, fromNamespace, toGroup, toKind, toName, toNamespace string) error { func (p *Provider) isReferenceGranted(fromKind, fromNamespace, toGroup, toKind, toName, toNamespace string) error {
if toNamespace == fromNamespace { if toNamespace == fromNamespace {
return nil return nil
} }
@ -793,7 +799,7 @@ func (p *Provider) isReferenceGranted(fromGroup, fromKind, fromNamespace, toGrou
return fmt.Errorf("listing ReferenceGrant: %w", err) return fmt.Errorf("listing ReferenceGrant: %w", err)
} }
refGrants = filterReferenceGrantsFrom(refGrants, fromGroup, fromKind, fromNamespace) refGrants = filterReferenceGrantsFrom(refGrants, groupGateway, fromKind, fromNamespace)
refGrants = filterReferenceGrantsTo(refGrants, toGroup, toKind, toName) refGrants = filterReferenceGrantsTo(refGrants, toGroup, toKind, toName)
if len(refGrants) == 0 { if len(refGrants) == 0 {
return errors.New("missing ReferenceGrant") return errors.New("missing ReferenceGrant")

File diff suppressed because it is too large Load diff

View file

@ -6,6 +6,7 @@ import (
"fmt" "fmt"
"net" "net"
"strconv" "strconv"
"strings"
"github.com/rs/zerolog/log" "github.com/rs/zerolog/log"
"github.com/traefik/traefik/v3/pkg/config/dynamic" "github.com/traefik/traefik/v3/pkg/config/dynamic"
@ -108,182 +109,241 @@ func (p *Provider) loadTCPRoute(listener gatewayListener, route *gatev1alpha2.TC
Reason: string(gatev1.RouteConditionResolvedRefs), Reason: string(gatev1.RouteConditionResolvedRefs),
} }
router := dynamic.TCPRouter{ for ri, rule := range route.Spec.Rules {
Rule: "HostSNI(`*`)",
EntryPoints: []string{listener.EPName},
RuleSyntax: "v3",
}
if listener.Protocol == gatev1.TLSProtocolType && listener.TLS != nil {
// TODO support let's encrypt
router.TLS = &dynamic.RouterTCPTLSConfig{
Passthrough: listener.TLS.Mode != nil && *listener.TLS.Mode == gatev1.TLSModePassthrough,
}
}
// Adding the gateway desc and the entryPoint desc prevents overlapping of routers build from the same routes.
routerName := provider.Normalize(route.Namespace + "-" + route.Name + "-" + listener.GWName + "-" + listener.EPName)
var ruleServiceNames []string
for i, rule := range route.Spec.Rules {
if rule.BackendRefs == nil { if rule.BackendRefs == nil {
// Should not happen due to validation. // Should not happen due to validation.
// https://github.com/kubernetes-sigs/gateway-api/blob/v0.4.0/apis/v1alpha2/tcproute_types.go#L76 // https://github.com/kubernetes-sigs/gateway-api/blob/v0.4.0/apis/v1alpha2/tcproute_types.go#L76
continue continue
} }
wrrService, subServices, err := p.loadTCPServices(route.Namespace, rule.BackendRefs) router := dynamic.TCPRouter{
if err != nil { Rule: "HostSNI(`*`)",
return conf, metav1.Condition{ EntryPoints: []string{listener.EPName},
Type: string(gatev1.RouteConditionResolvedRefs), RuleSyntax: "v3",
Status: metav1.ConditionFalse, }
ObservedGeneration: route.Generation,
LastTransitionTime: metav1.Now(), if listener.Protocol == gatev1.TLSProtocolType && listener.TLS != nil {
Reason: string(gatev1.RouteReasonBackendNotFound), router.TLS = &dynamic.RouterTCPTLSConfig{
Message: fmt.Sprintf("Cannot load TCPRoute service %s/%s: %v", route.Namespace, route.Name, err), Passthrough: listener.TLS.Mode != nil && *listener.TLS.Mode == gatev1.TLSModePassthrough,
} }
} }
for svcName, svc := range subServices { // Adding the gateway desc and the entryPoint desc prevents overlapping of routers build from the same routes.
conf.TCP.Services[svcName] = svc routeKey := provider.Normalize(fmt.Sprintf("%s-%s-%s-%s-%d", route.Namespace, route.Name, listener.GWName, listener.EPName, ri))
// Routing criteria should be introduced at some point.
routerName := makeRouterName("", routeKey)
if len(rule.BackendRefs) == 1 && isInternalService(rule.BackendRefs[0]) {
router.Service = string(rule.BackendRefs[0].Name)
conf.TCP.Routers[routerName] = &router
continue
} }
serviceName := fmt.Sprintf("%s-wrr-%d", routerName, i) var serviceCondition *metav1.Condition
conf.TCP.Services[serviceName] = wrrService router.Service, serviceCondition = p.loadTCPWRRService(conf, routerName, rule.BackendRefs, route)
if serviceCondition != nil {
condition = *serviceCondition
}
ruleServiceNames = append(ruleServiceNames, serviceName)
}
if len(ruleServiceNames) == 1 {
router.Service = ruleServiceNames[0]
conf.TCP.Routers[routerName] = &router conf.TCP.Routers[routerName] = &router
return conf, condition
} }
routeServiceKey := routerName + "-wrr"
routeService := &dynamic.TCPService{Weighted: &dynamic.TCPWeightedRoundRobin{}}
for _, name := range ruleServiceNames {
service := dynamic.TCPWRRService{Name: name}
service.SetDefaults()
routeService.Weighted.Services = append(routeService.Weighted.Services, service)
}
conf.TCP.Services[routeServiceKey] = routeService
router.Service = routeServiceKey
conf.TCP.Routers[routerName] = &router
return conf, condition return conf, condition
} }
// loadTCPServices is generating a WRR service, even when there is only one target. // loadTCPWRRService is generating a WRR service, even when there is only one target.
func (p *Provider) loadTCPServices(namespace string, backendRefs []gatev1.BackendRef) (*dynamic.TCPService, map[string]*dynamic.TCPService, error) { func (p *Provider) loadTCPWRRService(conf *dynamic.Configuration, routeKey string, backendRefs []gatev1.BackendRef, route *gatev1alpha2.TCPRoute) (string, *metav1.Condition) {
services := map[string]*dynamic.TCPService{} name := routeKey + "-wrr"
if _, ok := conf.TCP.Services[name]; ok {
wrrSvc := &dynamic.TCPService{ return name, nil
Weighted: &dynamic.TCPWeightedRoundRobin{
Services: []dynamic.TCPWRRService{},
},
} }
var wrr dynamic.TCPWeightedRoundRobin
var condition *metav1.Condition
for _, backendRef := range backendRefs { for _, backendRef := range backendRefs {
if backendRef.Group == nil || backendRef.Kind == nil { svcName, svc, errCondition := p.loadTCPService(route, backendRef)
// Should not happen as this is validated by kubernetes weight := ptr.To(int(ptr.Deref(backendRef.Weight, 1)))
if errCondition != nil {
condition = errCondition
errName := routeKey + "-err-lb"
conf.TCP.Services[errName] = &dynamic.TCPService{
LoadBalancer: &dynamic.TCPServersLoadBalancer{
Servers: []dynamic.TCPServer{},
},
}
wrr.Services = append(wrr.Services, dynamic.TCPWRRService{
Name: errName,
Weight: weight,
})
continue continue
} }
if isInternalService(backendRef) { if svc != nil {
return nil, nil, fmt.Errorf("traefik internal service %s is not allowed in a WRR loadbalancer", backendRef.Name) conf.TCP.Services[svcName] = svc
} }
weight := int(ptr.Deref(backendRef.Weight, 1)) wrr.Services = append(wrr.Services, dynamic.TCPWRRService{
Name: svcName,
Weight: weight,
})
}
if isTraefikService(backendRef) { conf.TCP.Services[name] = &dynamic.TCPService{Weighted: &wrr}
wrrSvc.Weighted.Services = append(wrrSvc.Weighted.Services, dynamic.TCPWRRService{Name: string(backendRef.Name), Weight: &weight}) return name, condition
continue }
func (p *Provider) loadTCPService(route *gatev1alpha2.TCPRoute, backendRef gatev1.BackendRef) (string, *dynamic.TCPService, *metav1.Condition) {
kind := ptr.Deref(backendRef.Kind, kindService)
group := groupCore
if backendRef.Group != nil && *backendRef.Group != "" {
group = string(*backendRef.Group)
}
namespace := route.Namespace
if backendRef.Namespace != nil && *backendRef.Namespace != "" {
namespace = string(*backendRef.Namespace)
}
serviceName := provider.Normalize(namespace + "-" + string(backendRef.Name))
if err := p.isReferenceGranted(kindTCPRoute, route.Namespace, group, string(kind), string(backendRef.Name), namespace); err != nil {
return serviceName, nil, &metav1.Condition{
Type: string(gatev1.RouteConditionResolvedRefs),
Status: metav1.ConditionFalse,
ObservedGeneration: route.Generation,
LastTransitionTime: metav1.Now(),
Reason: string(gatev1.RouteReasonRefNotPermitted),
Message: fmt.Sprintf("Cannot load TCPRoute BackendRef %s/%s/%s/%s: %s", group, kind, namespace, backendRef.Name, err),
} }
}
if *backendRef.Group != "" && *backendRef.Group != groupCore && *backendRef.Kind != "Service" { if group != groupCore || kind != kindService {
return nil, nil, fmt.Errorf("unsupported BackendRef %s/%s/%s", *backendRef.Group, *backendRef.Kind, backendRef.Name) name, err := p.loadTCPBackendRef(backendRef)
}
if backendRef.Port == nil {
return nil, nil, errors.New("port is required for Kubernetes Service reference")
}
service, exists, err := p.client.GetService(namespace, string(backendRef.Name))
if err != nil { if err != nil {
return nil, nil, fmt.Errorf("getting service: %w", err) return serviceName, nil, &metav1.Condition{
} Type: string(gatev1.RouteConditionResolvedRefs),
if !exists { Status: metav1.ConditionFalse,
return nil, nil, errors.New("service not found") ObservedGeneration: route.Generation,
LastTransitionTime: metav1.Now(),
Reason: string(gatev1.RouteReasonInvalidKind),
Message: fmt.Sprintf("Cannot load TCPRoute BackendRef %s/%s/%s/%s: %s", group, kind, namespace, backendRef.Name, err),
}
} }
var svcPort *corev1.ServicePort return name, nil, nil
for _, p := range service.Spec.Ports { }
if p.Port == int32(*backendRef.Port) {
svcPort = &p port := ptr.Deref(backendRef.Port, gatev1.PortNumber(0))
if port == 0 {
return serviceName, nil, &metav1.Condition{
Type: string(gatev1.RouteConditionResolvedRefs),
Status: metav1.ConditionFalse,
ObservedGeneration: route.Generation,
LastTransitionTime: metav1.Now(),
Reason: string(gatev1.RouteReasonUnsupportedProtocol),
Message: fmt.Sprintf("Cannot load TCPRoute BackendRef %s/%s/%s/%s port is required", group, kind, namespace, backendRef.Name),
}
}
portStr := strconv.FormatInt(int64(port), 10)
serviceName = provider.Normalize(serviceName + "-" + portStr)
lb, err := p.loadTCPServers(namespace, backendRef)
if err != nil {
return serviceName, nil, &metav1.Condition{
Type: string(gatev1.RouteConditionResolvedRefs),
Status: metav1.ConditionFalse,
ObservedGeneration: route.Generation,
LastTransitionTime: metav1.Now(),
Reason: string(gatev1.RouteReasonBackendNotFound),
Message: fmt.Sprintf("Cannot load TCPRoute BackendRef %s/%s/%s/%s: %s", group, kind, namespace, backendRef.Name, err),
}
}
return serviceName, &dynamic.TCPService{LoadBalancer: lb}, nil
}
func (p *Provider) loadTCPServers(namespace string, backendRef gatev1.BackendRef) (*dynamic.TCPServersLoadBalancer, error) {
if backendRef.Port == nil {
return nil, errors.New("port is required for Kubernetes Service reference")
}
service, exists, err := p.client.GetService(namespace, string(backendRef.Name))
if err != nil {
return nil, fmt.Errorf("getting service: %w", err)
}
if !exists {
return nil, errors.New("service not found")
}
var svcPort *corev1.ServicePort
for _, p := range service.Spec.Ports {
if p.Port == int32(*backendRef.Port) {
svcPort = &p
break
}
}
if svcPort == nil {
return nil, fmt.Errorf("service port %d not found", *backendRef.Port)
}
endpointSlices, err := p.client.ListEndpointSlicesForService(namespace, string(backendRef.Name))
if err != nil {
return nil, fmt.Errorf("getting endpointslices: %w", err)
}
if len(endpointSlices) == 0 {
return nil, errors.New("endpointslices not found")
}
lb := &dynamic.TCPServersLoadBalancer{}
addresses := map[string]struct{}{}
for _, endpointSlice := range endpointSlices {
var port int32
for _, p := range endpointSlice.Ports {
if svcPort.Name == *p.Name {
port = *p.Port
break break
} }
} }
if svcPort == nil { if port == 0 {
return nil, nil, fmt.Errorf("service port %d not found", *backendRef.Port) continue
} }
endpointSlices, err := p.client.ListEndpointSlicesForService(namespace, string(backendRef.Name)) for _, endpoint := range endpointSlice.Endpoints {
if err != nil { if endpoint.Conditions.Ready == nil || !*endpoint.Conditions.Ready {
return nil, nil, fmt.Errorf("getting endpointslices: %w", err)
}
if len(endpointSlices) == 0 {
return nil, nil, errors.New("endpointslices not found")
}
svc := dynamic.TCPService{LoadBalancer: &dynamic.TCPServersLoadBalancer{}}
addresses := map[string]struct{}{}
for _, endpointSlice := range endpointSlices {
var port int32
for _, p := range endpointSlice.Ports {
if svcPort.Name == *p.Name {
port = *p.Port
break
}
}
if port == 0 {
continue continue
} }
for _, endpoint := range endpointSlice.Endpoints { for _, address := range endpoint.Addresses {
if endpoint.Conditions.Ready == nil || !*endpoint.Conditions.Ready { if _, ok := addresses[address]; ok {
continue continue
} }
for _, address := range endpoint.Addresses { addresses[address] = struct{}{}
if _, ok := addresses[address]; ok { lb.Servers = append(lb.Servers, dynamic.TCPServer{
continue // TODO determine whether the servers needs TLS, from the port?
} Address: net.JoinHostPort(address, strconv.Itoa(int(port))),
})
addresses[address] = struct{}{}
svc.LoadBalancer.Servers = append(svc.LoadBalancer.Servers, dynamic.TCPServer{
Address: net.JoinHostPort(address, strconv.Itoa(int(port))),
})
}
} }
} }
serviceName := provider.Normalize(service.Namespace + "-" + service.Name + "-" + strconv.Itoa(int(svcPort.Port)))
services[serviceName] = &svc
wrrSvc.Weighted.Services = append(wrrSvc.Weighted.Services, dynamic.TCPWRRService{Name: serviceName, Weight: &weight})
} }
if len(wrrSvc.Weighted.Services) == 0 { return lb, nil
return nil, nil, errors.New("no service has been created") }
func (p *Provider) loadTCPBackendRef(backendRef gatev1.BackendRef) (string, error) {
// Support for cross-provider references (e.g: api@internal).
// This provides the same behavior as for IngressRoutes.
if *backendRef.Kind == "TraefikService" && strings.Contains(string(backendRef.Name), "@") {
return string(backendRef.Name), nil
} }
return wrrSvc, services, nil return "", fmt.Errorf("unsupported BackendRef %s/%s/%s", *backendRef.Group, *backendRef.Kind, backendRef.Name)
} }
func mergeTCPConfiguration(from, to *dynamic.Configuration) { func mergeTCPConfiguration(from, to *dynamic.Configuration) {

View file

@ -4,6 +4,7 @@ import (
"context" "context"
"fmt" "fmt"
"regexp" "regexp"
"strconv"
"strings" "strings"
"github.com/rs/zerolog/log" "github.com/rs/zerolog/log"
@ -11,6 +12,7 @@ import (
"github.com/traefik/traefik/v3/pkg/provider" "github.com/traefik/traefik/v3/pkg/provider"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
ktypes "k8s.io/apimachinery/pkg/types" ktypes "k8s.io/apimachinery/pkg/types"
"k8s.io/utils/ptr"
gatev1 "sigs.k8s.io/gateway-api/apis/v1" gatev1 "sigs.k8s.io/gateway-api/apis/v1"
gatev1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" gatev1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2"
) )
@ -109,74 +111,159 @@ func (p *Provider) loadTLSRoute(listener gatewayListener, route *gatev1alpha2.TL
Reason: string(gatev1.RouteConditionResolvedRefs), Reason: string(gatev1.RouteConditionResolvedRefs),
} }
router := dynamic.TCPRouter{ for ri, routeRule := range route.Spec.Rules {
RuleSyntax: "v3",
Rule: hostSNIRule(hostnames),
EntryPoints: []string{listener.EPName},
TLS: &dynamic.RouterTCPTLSConfig{
Passthrough: listener.TLS != nil && listener.TLS.Mode != nil && *listener.TLS.Mode == gatev1.TLSModePassthrough,
},
}
// Adding the gateway desc and the entryPoint desc prevents overlapping of routers build from the same routes.
routeKey := provider.Normalize(route.Namespace + "-" + route.Name + "-" + listener.GWName + "-" + listener.EPName)
routerName := makeRouterName(router.Rule, routeKey)
var ruleServiceNames []string
for i, routeRule := range route.Spec.Rules {
if len(routeRule.BackendRefs) == 0 { if len(routeRule.BackendRefs) == 0 {
// Should not happen due to validation. // Should not happen due to validation.
// https://github.com/kubernetes-sigs/gateway-api/blob/v0.4.0/apis/v1alpha2/tlsroute_types.go#L120 // https://github.com/kubernetes-sigs/gateway-api/blob/v0.4.0/apis/v1alpha2/tlsroute_types.go#L120
continue continue
} }
wrrService, subServices, err := p.loadTCPServices(route.Namespace, routeRule.BackendRefs) router := dynamic.TCPRouter{
RuleSyntax: "v3",
Rule: hostSNIRule(hostnames),
EntryPoints: []string{listener.EPName},
TLS: &dynamic.RouterTCPTLSConfig{
Passthrough: listener.TLS != nil && listener.TLS.Mode != nil && *listener.TLS.Mode == gatev1.TLSModePassthrough,
},
}
// Adding the gateway desc and the entryPoint desc prevents overlapping of routers build from the same routes.
routeKey := provider.Normalize(fmt.Sprintf("%s-%s-%s-%s-%d", route.Namespace, route.Name, listener.GWName, listener.EPName, ri))
// Routing criteria should be introduced at some point.
routerName := makeRouterName("", routeKey)
if len(routeRule.BackendRefs) == 1 && isInternalService(routeRule.BackendRefs[0]) {
router.Service = string(routeRule.BackendRefs[0].Name)
conf.TCP.Routers[routerName] = &router
continue
}
var serviceCondition *metav1.Condition
router.Service, serviceCondition = p.loadTLSWRRService(conf, routerName, routeRule.BackendRefs, route)
if serviceCondition != nil {
condition = *serviceCondition
}
conf.TCP.Routers[routerName] = &router
}
return conf, condition
}
// loadTLSWRRService is generating a WRR service, even when there is only one target.
func (p *Provider) loadTLSWRRService(conf *dynamic.Configuration, routeKey string, backendRefs []gatev1.BackendRef, route *gatev1alpha2.TLSRoute) (string, *metav1.Condition) {
name := routeKey + "-wrr"
if _, ok := conf.TCP.Services[name]; ok {
return name, nil
}
var wrr dynamic.TCPWeightedRoundRobin
var condition *metav1.Condition
for _, backendRef := range backendRefs {
svcName, svc, errCondition := p.loadTLSService(route, backendRef)
weight := ptr.To(int(ptr.Deref(backendRef.Weight, 1)))
if errCondition != nil {
condition = errCondition
errName := routeKey + "-err-lb"
conf.TCP.Services[errName] = &dynamic.TCPService{
LoadBalancer: &dynamic.TCPServersLoadBalancer{
Servers: []dynamic.TCPServer{},
},
}
wrr.Services = append(wrr.Services, dynamic.TCPWRRService{
Name: errName,
Weight: weight,
})
continue
}
if svc != nil {
conf.TCP.Services[svcName] = svc
}
wrr.Services = append(wrr.Services, dynamic.TCPWRRService{
Name: svcName,
Weight: weight,
})
}
conf.TCP.Services[name] = &dynamic.TCPService{Weighted: &wrr}
return name, condition
}
func (p *Provider) loadTLSService(route *gatev1alpha2.TLSRoute, backendRef gatev1.BackendRef) (string, *dynamic.TCPService, *metav1.Condition) {
kind := ptr.Deref(backendRef.Kind, kindService)
group := groupCore
if backendRef.Group != nil && *backendRef.Group != "" {
group = string(*backendRef.Group)
}
namespace := route.Namespace
if backendRef.Namespace != nil && *backendRef.Namespace != "" {
namespace = string(*backendRef.Namespace)
}
serviceName := provider.Normalize(namespace + "-" + string(backendRef.Name))
if err := p.isReferenceGranted(kindTLSRoute, route.Namespace, group, string(kind), string(backendRef.Name), namespace); err != nil {
return serviceName, nil, &metav1.Condition{
Type: string(gatev1.RouteConditionResolvedRefs),
Status: metav1.ConditionFalse,
ObservedGeneration: route.Generation,
LastTransitionTime: metav1.Now(),
Reason: string(gatev1.RouteReasonRefNotPermitted),
Message: fmt.Sprintf("Cannot load TLSRoute BackendRef %s/%s/%s/%s: %s", group, kind, namespace, backendRef.Name, err),
}
}
if group != groupCore || kind != kindService {
name, err := p.loadTCPBackendRef(backendRef)
if err != nil { if err != nil {
// update "ResolvedRefs" status true with "InvalidBackendRefs" reason return serviceName, nil, &metav1.Condition{
condition = metav1.Condition{
Type: string(gatev1.RouteConditionResolvedRefs), Type: string(gatev1.RouteConditionResolvedRefs),
Status: metav1.ConditionFalse, Status: metav1.ConditionFalse,
ObservedGeneration: route.Generation, ObservedGeneration: route.Generation,
LastTransitionTime: metav1.Now(), LastTransitionTime: metav1.Now(),
Reason: string(gatev1.RouteReasonBackendNotFound), Reason: string(gatev1.RouteReasonInvalidKind),
Message: fmt.Sprintf("Cannot load TLSRoute service %s/%s: %v", route.Namespace, route.Name, err), Message: fmt.Sprintf("Cannot load TLSRoute BackendRef %s/%s/%s/%s: %s", group, kind, namespace, backendRef.Name, err),
} }
continue
} }
for svcName, svc := range subServices { return name, nil, nil
conf.TCP.Services[svcName] = svc }
port := ptr.Deref(backendRef.Port, gatev1.PortNumber(0))
if port == 0 {
return serviceName, nil, &metav1.Condition{
Type: string(gatev1.RouteConditionResolvedRefs),
Status: metav1.ConditionFalse,
ObservedGeneration: route.Generation,
LastTransitionTime: metav1.Now(),
Reason: string(gatev1.RouteReasonUnsupportedProtocol),
Message: fmt.Sprintf("Cannot load TLSRoute BackendRef %s/%s/%s/%s port is required", group, kind, namespace, backendRef.Name),
} }
serviceName := fmt.Sprintf("%s-wrr-%d", routerName, i)
conf.TCP.Services[serviceName] = wrrService
ruleServiceNames = append(ruleServiceNames, serviceName)
} }
if len(ruleServiceNames) == 1 { portStr := strconv.FormatInt(int64(port), 10)
router.Service = ruleServiceNames[0] serviceName = provider.Normalize(serviceName + "-" + portStr)
conf.TCP.Routers[routerName] = &router
return conf, condition lb, err := p.loadTCPServers(namespace, backendRef)
if err != nil {
return serviceName, nil, &metav1.Condition{
Type: string(gatev1.RouteConditionResolvedRefs),
Status: metav1.ConditionFalse,
ObservedGeneration: route.Generation,
LastTransitionTime: metav1.Now(),
Reason: string(gatev1.RouteReasonBackendNotFound),
Message: fmt.Sprintf("Cannot load TLSRoute BackendRef %s/%s/%s/%s: %s", group, kind, namespace, backendRef.Name, err),
}
} }
routeServiceKey := routerName + "-wrr" return serviceName, &dynamic.TCPService{LoadBalancer: lb}, nil
routeService := &dynamic.TCPService{Weighted: &dynamic.TCPWeightedRoundRobin{}}
for _, name := range ruleServiceNames {
service := dynamic.TCPWRRService{Name: name}
service.SetDefaults()
routeService.Weighted.Services = append(routeService.Weighted.Services, service)
}
conf.TCP.Services[routeServiceKey] = routeService
router.Service = routeServiceKey
conf.TCP.Routers[routerName] = &router
return conf, condition
} }
func hostSNIRule(hostnames []gatev1.Hostname) string { func hostSNIRule(hostnames []gatev1.Hostname) string {