Support Gateway API reference grant for HTTPRoute backends

Co-authored-by: Kevin Pollet <pollet.kevin@gmail.com>
This commit is contained in:
Romain 2024-06-04 14:16:04 +02:00 committed by GitHub
parent b452f37e08
commit 7eac92f49c
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
10 changed files with 788 additions and 211 deletions

View file

@ -195,13 +195,23 @@ func (s *K8sConformanceSuite) TestK8sGatewayAPIConformance() {
LatestObservedGenerationSet: 5 * time.Second,
RequiredConsecutiveSuccesses: 0,
},
SupportedFeatures: sets.New(ksuite.SupportGateway, ksuite.SupportHTTPRoute).
Union(ksuite.HTTPRouteExtendedFeatures),
EnableAllSupportedFeatures: false,
RunTest: *k8sConformanceRunTest,
// Until the feature are all supported, following tests are skipped.
SkipTests: []string{
tests.HTTPRouteInvalidCrossNamespaceParentRef.ShortName,
tests.HTTPRoutePartiallyInvalidViaInvalidReferenceGrant.ShortName,
tests.HTTPRouteReferenceGrant.ShortName,
tests.HTTPRouteMethodMatching.ShortName,
tests.HTTPRouteQueryParamMatching.ShortName,
tests.HTTPRouteRedirectPath.ShortName,
tests.HTTPRouteRedirectPortAndScheme.ShortName,
tests.HTTPRouteRequestMirror.ShortName,
tests.HTTPRouteRequestMultipleMirrors.ShortName,
tests.HTTPRouteResponseHeaderModifier.ShortName,
tests.HTTPRouteRewriteHost.ShortName,
tests.HTTPRouteRewritePath.ShortName,
tests.HTTPRouteTimeoutBackendRequest.ShortName,
tests.HTTPRouteTimeoutRequest.ShortName,
},
}

View file

@ -0,0 +1,61 @@
---
apiVersion: gateway.networking.k8s.io/v1beta1
kind: ReferenceGrant
metadata:
name: backend-from-bar
namespace: bar
spec:
from:
- group: gateway.networking.k8s.io
kind: HTTPRoute
namespace: default
to:
- group: ""
kind: Service
---
apiVersion: gateway.networking.k8s.io/v1
kind: GatewayClass
metadata:
name: my-gateway-class
spec:
controllerName: traefik.io/gateway-controller
---
apiVersion: gateway.networking.k8s.io/v1
kind: Gateway
metadata:
name: my-gateway
namespace: default
spec:
gatewayClassName: my-gateway-class
listeners: # Use GatewayClass defaults for listener definition.
- name: http
protocol: HTTP
port: 80
hostname: foo.example.com
allowedRoutes:
kinds:
- kind: HTTPRoute
group: gateway.networking.k8s.io
namespaces:
from: Same
---
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
name: http-app-1
namespace: default
spec:
parentRefs:
- name: my-gateway
kind: Gateway
group: gateway.networking.k8s.io
rules:
- backendRefs:
- name: whoami-bar
port: 80
weight: 1
kind: Service
group: ""

View file

@ -0,0 +1,46 @@
---
apiVersion: gateway.networking.k8s.io/v1
kind: GatewayClass
metadata:
name: my-gateway-class
spec:
controllerName: traefik.io/gateway-controller
---
apiVersion: gateway.networking.k8s.io/v1
kind: Gateway
metadata:
name: my-gateway
namespace: default
spec:
gatewayClassName: my-gateway-class
listeners: # Use GatewayClass defaults for listener definition.
- name: http
protocol: HTTP
port: 80
hostname: foo.example.com
allowedRoutes:
kinds:
- kind: HTTPRoute
group: gateway.networking.k8s.io
namespaces:
from: Same
---
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
name: http-app-1
namespace: default
spec:
parentRefs:
- name: my-gateway
kind: Gateway
group: gateway.networking.k8s.io
rules:
- backendRefs:
- name: whoami-bar
port: 80
weight: 1
kind: Service
group: ""

View file

@ -0,0 +1,61 @@
---
apiVersion: gateway.networking.k8s.io/v1beta1
kind: ReferenceGrant
metadata:
name: backend-from-bar
namespace: bar
spec:
from:
- group: gateway.networking.k8s.io
kind: HTTPRoute
namespace: anothernamespce
to:
- group: ""
kind: Service
---
apiVersion: gateway.networking.k8s.io/v1
kind: GatewayClass
metadata:
name: my-gateway-class
spec:
controllerName: traefik.io/gateway-controller
---
apiVersion: gateway.networking.k8s.io/v1
kind: Gateway
metadata:
name: my-gateway
namespace: default
spec:
gatewayClassName: my-gateway-class
listeners: # Use GatewayClass defaults for listener definition.
- name: http
protocol: HTTP
port: 80
hostname: foo.example.com
allowedRoutes:
kinds:
- kind: HTTPRoute
group: gateway.networking.k8s.io
namespaces:
from: Same
---
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
name: http-app-1
namespace: default
spec:
parentRefs:
- name: my-gateway
kind: Gateway
group: gateway.networking.k8s.io
rules:
- backendRefs:
- name: whoami-bar
port: 80
weight: 1
kind: Service
group: ""

View file

@ -0,0 +1,62 @@
---
apiVersion: gateway.networking.k8s.io/v1beta1
kind: ReferenceGrant
metadata:
name: backend-from-bar
namespace: bar
spec:
from:
- group: gateway.networking.k8s.io
kind: HTTPRoute
namespace: default
to:
- group: ""
kind: Service
name: differentservice
---
apiVersion: gateway.networking.k8s.io/v1
kind: GatewayClass
metadata:
name: my-gateway-class
spec:
controllerName: traefik.io/gateway-controller
---
apiVersion: gateway.networking.k8s.io/v1
kind: Gateway
metadata:
name: my-gateway
namespace: default
spec:
gatewayClassName: my-gateway-class
listeners: # Use GatewayClass defaults for listener definition.
- name: http
protocol: HTTP
port: 80
hostname: foo.example.com
allowedRoutes:
kinds:
- kind: HTTPRoute
group: gateway.networking.k8s.io
namespaces:
from: Same
---
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
name: http-app-1
namespace: default
spec:
parentRefs:
- name: my-gateway
kind: Gateway
group: gateway.networking.k8s.io
rules:
- backendRefs:
- name: whoami-bar
port: 80
weight: 1
kind: Service
group: ""

View file

@ -41,64 +41,45 @@ func (p *Provider) loadHTTPRoutes(ctx context.Context, client Client, gatewayLis
Conditions: []metav1.Condition{
{
Type: string(gatev1.RouteConditionAccepted),
Status: metav1.ConditionTrue,
Status: metav1.ConditionFalse,
ObservedGeneration: route.Generation,
LastTransitionTime: metav1.Now(),
Reason: string(gatev1.RouteReasonAccepted),
Reason: string(gatev1.RouteReasonNoMatchingParent),
},
},
}
var attachedListeners bool
notAcceptedReason := gatev1.RouteReasonNoMatchingParent
for _, listener := range gatewayListeners {
if !matchListener(listener, route.Namespace, parentRef) {
continue
}
accepted := true
if !allowRoute(listener, route.Namespace, kindHTTPRoute) {
notAcceptedReason = gatev1.RouteReasonNotAllowedByListeners
continue
parentStatus.Conditions = updateRouteConditionAccepted(parentStatus.Conditions, string(gatev1.RouteReasonNotAllowedByListeners))
accepted = false
}
hostnames, ok := findMatchingHostnames(listener.Hostname, route.Spec.Hostnames)
if !ok {
notAcceptedReason = gatev1.RouteReasonNoMatchingListenerHostname
continue
parentStatus.Conditions = updateRouteConditionAccepted(parentStatus.Conditions, string(gatev1.RouteReasonNoMatchingListenerHostname))
accepted = false
}
listener.Status.AttachedRoutes++
// TODO should we build the conf if the listener is not attached
// only consider the route attached if the listener is in an "attached" state.
if listener.Attached {
attachedListeners = true
if accepted {
// Gateway listener should have AttachedRoutes set even when Gateway has unresolved refs.
listener.Status.AttachedRoutes++
// Only consider the route attached if the listener is in an "attached" state.
if listener.Attached {
parentStatus.Conditions = updateRouteConditionAccepted(parentStatus.Conditions, string(gatev1.RouteReasonAccepted))
}
}
resolveConditions := p.loadHTTPRoute(logger.WithContext(ctx), client, listener, route, hostnames, conf)
// TODO: handle more accurately route conditions (in case of multiple listener matching).
for _, condition := range resolveConditions {
parentStatus.Conditions = appendCondition(parentStatus.Conditions, condition)
routeConf, resolveRefCondition := p.loadHTTPRoute(logger.WithContext(ctx), client, listener, route, hostnames)
if accepted && listener.Attached {
mergeHTTPConfiguration(routeConf, conf)
}
}
if !attachedListeners {
parentStatus.Conditions = []metav1.Condition{
{
Type: string(gatev1.RouteConditionAccepted),
Status: metav1.ConditionFalse,
ObservedGeneration: route.Generation,
LastTransitionTime: metav1.Now(),
Reason: string(notAcceptedReason),
},
{
Type: string(gatev1.RouteConditionResolvedRefs),
Status: metav1.ConditionFalse,
ObservedGeneration: route.Generation,
LastTransitionTime: metav1.Now(),
Reason: string(gatev1.RouteReasonRefNotPermitted),
},
}
parentStatus.Conditions = upsertRouteConditionResolvedRefs(parentStatus.Conditions, resolveRefCondition)
}
parentStatuses = append(parentStatuses, *parentStatus)
@ -117,17 +98,24 @@ func (p *Provider) loadHTTPRoutes(ctx context.Context, client Client, gatewayLis
}
}
func (p *Provider) loadHTTPRoute(ctx context.Context, client Client, listener gatewayListener, route *gatev1.HTTPRoute, hostnames []gatev1.Hostname, conf *dynamic.Configuration) []metav1.Condition {
routeConditions := []metav1.Condition{
{
Type: string(gatev1.RouteConditionResolvedRefs),
Status: metav1.ConditionTrue,
ObservedGeneration: route.Generation,
LastTransitionTime: metav1.Now(),
Reason: string(gatev1.RouteConditionResolvedRefs),
func (p *Provider) loadHTTPRoute(ctx context.Context, client Client, listener gatewayListener, route *gatev1.HTTPRoute, hostnames []gatev1.Hostname) (*dynamic.Configuration, metav1.Condition) {
routeConf := &dynamic.Configuration{
HTTP: &dynamic.HTTPConfiguration{
Routers: make(map[string]*dynamic.Router),
Middlewares: make(map[string]*dynamic.Middleware),
Services: make(map[string]*dynamic.Service),
ServersTransports: make(map[string]*dynamic.ServersTransport),
},
}
routeCondition := metav1.Condition{
Type: string(gatev1.RouteConditionResolvedRefs),
Status: metav1.ConditionTrue,
ObservedGeneration: route.Generation,
LastTransitionTime: metav1.Now(),
Reason: string(gatev1.RouteConditionResolvedRefs),
}
for _, routeRule := range route.Spec.Rules {
rule, priority := buildRouterRule(hostnames, routeRule.Matches)
router := dynamic.Router{
@ -159,14 +147,14 @@ func (p *Provider) loadHTTPRoute(ctx context.Context, client Client, listener ga
Weight: ptr.To(1),
})
conf.HTTP.Services[wrrName] = &dynamic.Service{Weighted: &wrr}
routeConf.HTTP.Services[wrrName] = &dynamic.Service{Weighted: &wrr}
router.Service = wrrName
} else {
for name, middleware := range middlewares {
// If the middleware config is nil in the return of the loadMiddlewares function,
// it means that we just need a reference to that middleware.
if middleware != nil {
conf.HTTP.Middlewares[name] = middleware
routeConf.HTTP.Middlewares[name] = middleware
}
router.Middlewares = append(router.Middlewares, name)
@ -180,7 +168,7 @@ func (p *Provider) loadHTTPRoute(ctx context.Context, client Client, listener ga
name, svc, errCondition := p.loadHTTPService(client, route, backendRef)
weight := ptr.To(int(ptr.Deref(backendRef.Weight, 1)))
if errCondition != nil {
routeConditions = appendCondition(routeConditions, *errCondition)
routeCondition = *errCondition
wrr.Services = append(wrr.Services, dynamic.WRRService{
Name: name,
Status: ptr.To(500),
@ -190,7 +178,7 @@ func (p *Provider) loadHTTPRoute(ctx context.Context, client Client, listener ga
}
if svc != nil {
conf.HTTP.Services[name] = svc
routeConf.HTTP.Services[name] = svc
}
wrr.Services = append(wrr.Services, dynamic.WRRService{
@ -199,7 +187,7 @@ func (p *Provider) loadHTTPRoute(ctx context.Context, client Client, listener ga
})
}
conf.HTTP.Services[wrrName] = &dynamic.Service{Weighted: &wrr}
routeConf.HTTP.Services[wrrName] = &dynamic.Service{Weighted: &wrr}
router.Service = wrrName
}
}
@ -208,39 +196,42 @@ func (p *Provider) loadHTTPRoute(ctx context.Context, client Client, listener ga
p.applyRouterTransform(ctx, rt, route)
routerKey = provider.Normalize(routerKey)
conf.HTTP.Routers[routerKey] = rt
routeConf.HTTP.Routers[routerKey] = rt
}
return routeConditions
return routeConf, routeCondition
}
// loadHTTPService returns a dynamic.Service config corresponding to the given gatev1.HTTPBackendRef.
// Note that the returned dynamic.Service config can be nil (for cross-provider, internal services, and backendFunc).
func (p *Provider) loadHTTPService(client Client, route *gatev1.HTTPRoute, backendRef gatev1.HTTPBackendRef) (string, *dynamic.Service, *metav1.Condition) {
kind := ptr.Deref(backendRef.Kind, "Service")
group := groupCore
if backendRef.Group != nil && *backendRef.Group != "" {
group = string(*backendRef.Group)
}
kind := ptr.Deref(backendRef.Kind, "Service")
namespace := ptr.Deref(backendRef.Namespace, gatev1.Namespace(route.Namespace))
namespaceStr := string(namespace)
serviceName := provider.Normalize(makeID(namespaceStr, string(backendRef.Name)))
namespace := route.Namespace
if backendRef.Namespace != nil && *backendRef.Namespace != "" {
namespace = string(*backendRef.Namespace)
}
// TODO support cross namespace through ReferenceGrant.
if namespaceStr != route.Namespace {
serviceName := provider.Normalize(makeID(namespace, string(backendRef.Name)))
if err := isReferenceGranted(client, groupGateway, kindHTTPRoute, route.Namespace, group, string(kind), string(backendRef.Name), namespace); err != nil {
return serviceName, nil, &metav1.Condition{
Type: string(gatev1.RouteConditionResolvedRefs),
Status: metav1.ConditionFalse,
ObservedGeneration: route.Generation,
LastTransitionTime: metav1.Now(),
Reason: string(gatev1.RouteReasonRefNotPermitted),
Message: fmt.Sprintf("Cannot load HTTPBackendRef %s/%s/%s/%s namespace not allowed", group, kind, namespace, backendRef.Name),
Message: fmt.Sprintf("Cannot load HTTPBackendRef %s/%s/%s/%s: %s", group, kind, namespace, backendRef.Name, err),
}
}
if group != groupCore || kind != "Service" {
name, service, err := p.loadHTTPBackendRef(namespaceStr, backendRef)
name, service, err := p.loadHTTPBackendRef(namespace, backendRef)
if err != nil {
return serviceName, nil, &metav1.Condition{
Type: string(gatev1.RouteConditionResolvedRefs),
@ -270,7 +261,7 @@ func (p *Provider) loadHTTPService(client Client, route *gatev1.HTTPRoute, backe
portStr := strconv.FormatInt(int64(port), 10)
serviceName = provider.Normalize(serviceName + "-" + portStr)
lb, err := loadHTTPServers(client, namespaceStr, backendRef)
lb, err := loadHTTPServers(client, namespace, backendRef)
if err != nil {
return serviceName, nil, &metav1.Condition{
Type: string(gatev1.RouteConditionResolvedRefs),
@ -615,3 +606,35 @@ func getProtocol(portSpec corev1.ServicePort) string {
return protocol
}
func mergeHTTPConfiguration(from, to *dynamic.Configuration) {
if from == nil || from.HTTP == nil || to == nil {
return
}
if to.HTTP == nil {
to.HTTP = from.HTTP
return
}
if to.HTTP.Routers == nil {
to.HTTP.Routers = map[string]*dynamic.Router{}
}
for routerName, router := range from.HTTP.Routers {
to.HTTP.Routers[routerName] = router
}
if to.HTTP.Middlewares == nil {
to.HTTP.Middlewares = map[string]*dynamic.Middleware{}
}
for middlewareName, middleware := range from.HTTP.Middlewares {
to.HTTP.Middlewares[middlewareName] = middleware
}
if to.HTTP.Services == nil {
to.HTTP.Services = map[string]*dynamic.Service{}
}
for serviceName, service := range from.HTTP.Services {
to.HTTP.Services[serviceName] = service
}
}

View file

@ -37,7 +37,8 @@ const (
providerName = "kubernetesgateway"
controllerName = "traefik.io/gateway-controller"
groupCore = "core"
groupCore = "core"
groupGateway = "gateway.networking.k8s.io"
kindGateway = "Gateway"
kindTraefikService = "TraefikService"
@ -564,35 +565,17 @@ func (p *Provider) loadGatewayListeners(ctx context.Context, client Client, gate
certificateNamespace = string(*certificateRef.Namespace)
}
if certificateNamespace != gateway.Namespace {
referenceGrants, err := client.ListReferenceGrants(certificateNamespace)
if err != nil {
gatewayListeners[i].Status.Conditions = append(gatewayListeners[i].Status.Conditions, metav1.Condition{
Type: string(gatev1.ListenerConditionResolvedRefs),
Status: metav1.ConditionFalse,
ObservedGeneration: gateway.Generation,
LastTransitionTime: metav1.Now(),
Reason: string(gatev1.ListenerReasonRefNotPermitted),
Message: fmt.Sprintf("Cannot find any ReferenceGrant: %v", err),
})
if err := isReferenceGranted(client, groupGateway, kindGateway, gateway.Namespace, groupCore, "Secret", string(certificateRef.Name), certificateNamespace); err != nil {
gatewayListeners[i].Status.Conditions = append(gatewayListeners[i].Status.Conditions, metav1.Condition{
Type: string(gatev1.ListenerConditionResolvedRefs),
Status: metav1.ConditionFalse,
ObservedGeneration: gateway.Generation,
LastTransitionTime: metav1.Now(),
Reason: string(gatev1.ListenerReasonRefNotPermitted),
Message: fmt.Sprintf("Cannot load CertificateRef %s/%s: %s", certificateNamespace, certificateRef.Name, err),
})
continue
}
referenceGrants = filterReferenceGrantsFrom(referenceGrants, "gateway.networking.k8s.io", "Gateway", gateway.Namespace)
referenceGrants = filterReferenceGrantsTo(referenceGrants, groupCore, "Secret", string(certificateRef.Name))
if len(referenceGrants) == 0 {
gatewayListeners[i].Status.Conditions = append(gatewayListeners[i].Status.Conditions, metav1.Condition{
Type: string(gatev1.ListenerConditionResolvedRefs),
Status: metav1.ConditionFalse,
ObservedGeneration: gateway.Generation,
LastTransitionTime: metav1.Now(),
Reason: string(gatev1.ListenerReasonRefNotPermitted),
Message: "Required ReferenceGrant for cross namespace secret reference is missing",
})
continue
}
continue
}
configKey := certificateNamespace + "/" + string(certificateRef.Name)
@ -1006,8 +989,8 @@ func makeID(namespace, name string) string {
return namespace + "-" + name
}
func getTLS(k8sClient Client, secretName gatev1.ObjectName, namespace string) (*tls.CertAndStores, error) {
secret, exists, err := k8sClient.GetSecret(namespace, string(secretName))
func getTLS(client Client, secretName gatev1.ObjectName, namespace string) (*tls.CertAndStores, error) {
secret, exists, err := client.GetSecret(namespace, string(secretName))
if err != nil {
return nil, fmt.Errorf("failed to fetch secret %s/%s: %w", namespace, secretName, err)
}
@ -1131,6 +1114,25 @@ func makeListenerKey(l gatev1.Listener) string {
return fmt.Sprintf("%s|%s|%d", l.Protocol, hostname, l.Port)
}
func isReferenceGranted(client Client, fromGroup, fromKind, fromNamespace, toGroup, toKind, toName, toNamespace string) error {
if toNamespace == fromNamespace {
return nil
}
refGrants, err := client.ListReferenceGrants(toNamespace)
if err != nil {
return fmt.Errorf("listing ReferenceGrant: %w", err)
}
refGrants = filterReferenceGrantsFrom(refGrants, fromGroup, fromKind, fromNamespace)
refGrants = filterReferenceGrantsTo(refGrants, toGroup, toKind, toName)
if len(refGrants) == 0 {
return errors.New("missing ReferenceGrant")
}
return nil
}
func filterReferenceGrantsFrom(referenceGrants []*gatev1beta1.ReferenceGrant, group, kind, namespace string) []*gatev1beta1.ReferenceGrant {
var matchingReferenceGrants []*gatev1beta1.ReferenceGrant
for _, referenceGrant := range referenceGrants {
@ -1193,13 +1195,38 @@ func kindToString(p *gatev1.Kind) string {
return string(*p)
}
func appendCondition(conditions []metav1.Condition, condition metav1.Condition) []metav1.Condition {
res := []metav1.Condition{condition}
func updateRouteConditionAccepted(conditions []metav1.Condition, reason string) []metav1.Condition {
var conds []metav1.Condition
for _, c := range conditions {
if c.Type != condition.Type {
res = append(res, c)
if c.Type == string(gatev1.RouteConditionAccepted) && c.Status != metav1.ConditionTrue {
c.Reason = reason
c.LastTransitionTime = metav1.Now()
if reason == string(gatev1.RouteReasonAccepted) {
c.Status = metav1.ConditionTrue
}
}
conds = append(conds, c)
}
return res
return conds
}
func upsertRouteConditionResolvedRefs(conditions []metav1.Condition, condition metav1.Condition) []metav1.Condition {
var (
curr *metav1.Condition
conds []metav1.Condition
)
for _, c := range conditions {
if c.Type == string(gatev1.RouteConditionResolvedRefs) {
curr = &c
continue
}
conds = append(conds, c)
}
if curr != nil && curr.Status == metav1.ConditionFalse && condition.Status == metav1.ConditionTrue {
return append(conds, *curr)
}
return append(conds, condition)
}

View file

@ -5422,7 +5422,7 @@ func TestLoadRoutesWithReferenceGrants(t *testing.T) {
},
},
{
desc: "Empty because ReferenceGrant spec.from does not match",
desc: "Empty because ReferenceGrant spec.from does not match secret",
paths: []string{"services.yml", "referencegrant/for_secret_not_matching_from.yml"},
entryPoints: map[string]Entrypoint{
"tls": {Address: ":9000"},
@ -5448,7 +5448,7 @@ func TestLoadRoutesWithReferenceGrants(t *testing.T) {
},
},
{
desc: "Empty because ReferenceGrant spec.to does not match",
desc: "Empty because ReferenceGrant spec.to does not match secret",
paths: []string{"services.yml", "referencegrant/for_secret_not_matching_to.yml"},
entryPoints: map[string]Entrypoint{
"tls": {Address: ":9000"},
@ -5538,6 +5538,130 @@ func TestLoadRoutesWithReferenceGrants(t *testing.T) {
},
},
},
{
desc: "Empty because ReferenceGrant for Service is missing",
paths: []string{"services.yml", "referencegrant/for_secret_missing.yml"},
entryPoints: map[string]Entrypoint{
"tls": {Address: ":9000"},
},
expected: &dynamic.Configuration{
UDP: &dynamic.UDPConfiguration{
Routers: map[string]*dynamic.UDPRouter{},
Services: map[string]*dynamic.UDPService{},
},
TCP: &dynamic.TCPConfiguration{
Routers: map[string]*dynamic.TCPRouter{},
Middlewares: map[string]*dynamic.TCPMiddleware{},
Services: map[string]*dynamic.TCPService{},
ServersTransports: map[string]*dynamic.TCPServersTransport{},
},
HTTP: &dynamic.HTTPConfiguration{
Routers: map[string]*dynamic.Router{},
Middlewares: map[string]*dynamic.Middleware{},
Services: map[string]*dynamic.Service{},
ServersTransports: map[string]*dynamic.ServersTransport{},
},
TLS: &dynamic.TLSConfiguration{},
},
},
{
desc: "Empty because ReferenceGrant spec.from does not match service",
paths: []string{"services.yml", "referencegrant/for_service_not_matching_from.yml"},
entryPoints: map[string]Entrypoint{
"tls": {Address: ":9000"},
},
expected: &dynamic.Configuration{
UDP: &dynamic.UDPConfiguration{
Routers: map[string]*dynamic.UDPRouter{},
Services: map[string]*dynamic.UDPService{},
},
TCP: &dynamic.TCPConfiguration{
Routers: map[string]*dynamic.TCPRouter{},
Middlewares: map[string]*dynamic.TCPMiddleware{},
Services: map[string]*dynamic.TCPService{},
ServersTransports: map[string]*dynamic.TCPServersTransport{},
},
HTTP: &dynamic.HTTPConfiguration{
Routers: map[string]*dynamic.Router{},
Middlewares: map[string]*dynamic.Middleware{},
Services: map[string]*dynamic.Service{},
ServersTransports: map[string]*dynamic.ServersTransport{},
},
TLS: &dynamic.TLSConfiguration{},
},
},
{
desc: "Empty because ReferenceGrant spec.to does not match service",
paths: []string{"services.yml", "referencegrant/for_service_not_matching_to.yml"},
entryPoints: map[string]Entrypoint{
"tls": {Address: ":9000"},
},
expected: &dynamic.Configuration{
UDP: &dynamic.UDPConfiguration{
Routers: map[string]*dynamic.UDPRouter{},
Services: map[string]*dynamic.UDPService{},
},
TCP: &dynamic.TCPConfiguration{
Routers: map[string]*dynamic.TCPRouter{},
Middlewares: map[string]*dynamic.TCPMiddleware{},
Services: map[string]*dynamic.TCPService{},
ServersTransports: map[string]*dynamic.TCPServersTransport{},
},
HTTP: &dynamic.HTTPConfiguration{
Routers: map[string]*dynamic.Router{},
Middlewares: map[string]*dynamic.Middleware{},
Services: map[string]*dynamic.Service{},
ServersTransports: map[string]*dynamic.ServersTransport{},
},
TLS: &dynamic.TLSConfiguration{},
},
},
{
desc: "For Service",
paths: []string{"services.yml", "referencegrant/for_service.yml"},
entryPoints: map[string]Entrypoint{
"http": {Address: ":80"},
},
expected: &dynamic.Configuration{
UDP: &dynamic.UDPConfiguration{
Routers: map[string]*dynamic.UDPRouter{},
Services: map[string]*dynamic.UDPService{},
},
TCP: &dynamic.TCPConfiguration{
Routers: map[string]*dynamic.TCPRouter{},
Middlewares: map[string]*dynamic.TCPMiddleware{},
Services: map[string]*dynamic.TCPService{},
ServersTransports: map[string]*dynamic.TCPServersTransport{},
},
HTTP: &dynamic.HTTPConfiguration{
Routers: map[string]*dynamic.Router{
"default-http-app-1-my-gateway-http-f381ad97110137b4d42c": {
EntryPoints: []string{"http"},
Rule: "Host(`foo.example.com`)",
Service: "default-http-app-1-my-gateway-http-f381ad97110137b4d42c-wrr",
RuleSyntax: "v3",
Priority: 15,
},
},
Middlewares: map[string]*dynamic.Middleware{},
Services: map[string]*dynamic.Service{
"default-http-app-1-my-gateway-http-f381ad97110137b4d42c-wrr": {
Weighted: &dynamic.WeightedRoundRobin{
Services: []dynamic.WRRService{
{
Name: "default-whoami-bar-80",
Weight: ptr.To(1),
Status: ptr.To(500),
},
},
},
},
},
ServersTransports: map[string]*dynamic.ServersTransport{},
},
TLS: &dynamic.TLSConfiguration{},
},
},
}
for _, test := range testCases {
@ -6433,6 +6557,136 @@ func Test_gatewayAddresses(t *testing.T) {
}
}
func Test_upsertRouteConditionResolvedRefs(t *testing.T) {
testCases := []struct {
desc string
conditions []metav1.Condition
condition metav1.Condition
wantConditions []metav1.Condition
}{
{
desc: "True to False",
conditions: []metav1.Condition{
{
Type: "foo",
Status: "bar",
Reason: "baz",
Message: "foobarbaz",
},
{
Type: string(gatev1.RouteConditionResolvedRefs),
Status: metav1.ConditionTrue,
Reason: "foo",
Message: "foo",
},
},
condition: metav1.Condition{
Type: string(gatev1.RouteConditionResolvedRefs),
Status: metav1.ConditionFalse,
Reason: "bar",
Message: "bar",
},
wantConditions: []metav1.Condition{
{
Type: "foo",
Status: "bar",
Reason: "baz",
Message: "foobarbaz",
},
{
Type: string(gatev1.RouteConditionResolvedRefs),
Status: metav1.ConditionFalse,
Reason: "bar",
Message: "bar",
},
},
},
{
desc: "False to False",
conditions: []metav1.Condition{
{
Type: "foo",
Status: "bar",
Reason: "baz",
Message: "foobarbaz",
},
{
Type: string(gatev1.RouteConditionResolvedRefs),
Status: metav1.ConditionFalse,
Reason: "foo",
Message: "foo",
},
},
condition: metav1.Condition{
Type: string(gatev1.RouteConditionResolvedRefs),
Status: metav1.ConditionFalse,
Reason: "bar",
Message: "bar",
},
wantConditions: []metav1.Condition{
{
Type: "foo",
Status: "bar",
Reason: "baz",
Message: "foobarbaz",
},
{
Type: string(gatev1.RouteConditionResolvedRefs),
Status: metav1.ConditionFalse,
Reason: "bar",
Message: "bar",
},
},
},
{
desc: "False to True: no upsert",
conditions: []metav1.Condition{
{
Type: "foo",
Status: "bar",
Reason: "baz",
Message: "foobarbaz",
},
{
Type: string(gatev1.RouteConditionResolvedRefs),
Status: metav1.ConditionFalse,
Reason: "foo",
Message: "foo",
},
},
condition: metav1.Condition{
Type: string(gatev1.RouteConditionResolvedRefs),
Status: metav1.ConditionTrue,
Reason: "bar",
Message: "bar",
},
wantConditions: []metav1.Condition{
{
Type: "foo",
Status: "bar",
Reason: "baz",
Message: "foobarbaz",
},
{
Type: string(gatev1.RouteConditionResolvedRefs),
Status: metav1.ConditionFalse,
Reason: "foo",
Message: "foo",
},
},
},
}
for _, test := range testCases {
t.Run(test.desc, func(t *testing.T) {
t.Parallel()
got := upsertRouteConditionResolvedRefs(test.conditions, test.condition)
assert.Equal(t, test.wantConditions, got)
})
}
}
// We cannot use the gateway-api fake.NewSimpleClientset due to Gateway being pluralized as "gatewaies" instead of "gateways".
func newGatewaySimpleClientSet(t *testing.T, objects ...runtime.Object) *gatefake.Clientset {
t.Helper()

View file

@ -22,7 +22,8 @@ func (p *Provider) loadTCPRoutes(ctx context.Context, client Client, gatewayList
logger := log.Ctx(ctx)
routes, err := client.ListTCPRoutes()
if err != nil {
logger.Error().Err(err).Msgf("Get TCPRoutes: %s", err)
logger.Error().Err(err).Msgf("Unable to list TCPRoutes")
return
}
for _, route := range routes {
@ -34,39 +35,6 @@ func (p *Provider) loadTCPRoutes(ctx context.Context, client Client, gatewayList
ParentRef: parentRef,
ControllerName: controllerName,
Conditions: []metav1.Condition{
{
Type: string(gatev1.RouteConditionAccepted),
Status: metav1.ConditionTrue,
ObservedGeneration: route.Generation,
LastTransitionTime: metav1.Now(),
Reason: string(gatev1.RouteReasonAccepted),
},
},
}
var attachedListeners bool
for _, listener := range gatewayListeners {
if !matchListener(listener, route.Namespace, parentRef) {
continue
}
if !allowRoute(listener, route.Namespace, kindTCPRoute) {
continue
}
listener.Status.AttachedRoutes++
attachedListeners = true
resolveConditions := p.loadTCPRoute(client, listener, route, conf)
// TODO: handle more accurately route conditions (in case of multiple listener matching).
for _, condition := range resolveConditions {
parentStatus.Conditions = appendCondition(parentStatus.Conditions, condition)
}
}
if !attachedListeners {
parentStatus.Conditions = []metav1.Condition{
{
Type: string(gatev1.RouteConditionAccepted),
Status: metav1.ConditionFalse,
@ -74,7 +42,33 @@ func (p *Provider) loadTCPRoutes(ctx context.Context, client Client, gatewayList
LastTransitionTime: metav1.Now(),
Reason: string(gatev1.RouteReasonNoMatchingParent),
},
},
}
for _, listener := range gatewayListeners {
if !matchListener(listener, route.Namespace, parentRef) {
continue
}
accepted := true
if !allowRoute(listener, route.Namespace, kindTCPRoute) {
parentStatus.Conditions = updateRouteConditionAccepted(parentStatus.Conditions, string(gatev1.RouteReasonNotAllowedByListeners))
accepted = false
}
if accepted {
listener.Status.AttachedRoutes++
// only consider the route attached if the listener is in an "attached" state.
if listener.Attached {
parentStatus.Conditions = updateRouteConditionAccepted(parentStatus.Conditions, string(gatev1.RouteReasonAccepted))
}
}
routeConf, resolveRefCondition := p.loadTCPRoute(client, listener, route)
if accepted && listener.Attached {
mergeTCPConfiguration(routeConf, conf)
}
parentStatus.Conditions = upsertRouteConditionResolvedRefs(parentStatus.Conditions, resolveRefCondition)
}
parentStatuses = append(parentStatuses, *parentStatus)
@ -93,17 +87,24 @@ func (p *Provider) loadTCPRoutes(ctx context.Context, client Client, gatewayList
}
}
func (p *Provider) loadTCPRoute(client Client, listener gatewayListener, route *gatev1alpha2.TCPRoute, conf *dynamic.Configuration) []metav1.Condition {
routeConditions := []metav1.Condition{
{
Type: string(gatev1.RouteConditionResolvedRefs),
Status: metav1.ConditionTrue,
ObservedGeneration: route.Generation,
LastTransitionTime: metav1.Now(),
Reason: string(gatev1.RouteConditionResolvedRefs),
func (p *Provider) loadTCPRoute(client Client, listener gatewayListener, route *gatev1alpha2.TCPRoute) (*dynamic.Configuration, metav1.Condition) {
routeConf := &dynamic.Configuration{
TCP: &dynamic.TCPConfiguration{
Routers: make(map[string]*dynamic.TCPRouter),
Middlewares: make(map[string]*dynamic.TCPMiddleware),
Services: make(map[string]*dynamic.TCPService),
ServersTransports: make(map[string]*dynamic.TCPServersTransport),
},
}
routeCondition := metav1.Condition{
Type: string(gatev1.RouteConditionResolvedRefs),
Status: metav1.ConditionTrue,
ObservedGeneration: route.Generation,
LastTransitionTime: metav1.Now(),
Reason: string(gatev1.RouteConditionResolvedRefs),
}
router := dynamic.TCPRouter{
Rule: "HostSNI(`*`)",
EntryPoints: []string{listener.EPName},
@ -131,31 +132,30 @@ func (p *Provider) loadTCPRoute(client Client, listener gatewayListener, route *
wrrService, subServices, err := loadTCPServices(client, route.Namespace, rule.BackendRefs)
if err != nil {
routeConditions = appendCondition(routeConditions, metav1.Condition{
return routeConf, metav1.Condition{
Type: string(gatev1.RouteConditionResolvedRefs),
Status: metav1.ConditionFalse,
ObservedGeneration: route.Generation,
LastTransitionTime: metav1.Now(),
Reason: string(gatev1.RouteReasonBackendNotFound),
Message: fmt.Sprintf("Cannot load TCPRoute service %s/%s: %v", route.Namespace, route.Name, err),
})
return routeConditions
}
}
for svcName, svc := range subServices {
conf.TCP.Services[svcName] = svc
routeConf.TCP.Services[svcName] = svc
}
serviceName := fmt.Sprintf("%s-wrr-%d", routerKey, i)
conf.TCP.Services[serviceName] = wrrService
routeConf.TCP.Services[serviceName] = wrrService
ruleServiceNames = append(ruleServiceNames, serviceName)
}
if len(ruleServiceNames) == 1 {
router.Service = ruleServiceNames[0]
conf.TCP.Routers[routerKey] = &router
return routeConditions
routeConf.TCP.Routers[routerKey] = &router
return routeConf, routeCondition
}
routeServiceKey := routerKey + "-wrr"
@ -168,12 +168,12 @@ func (p *Provider) loadTCPRoute(client Client, listener gatewayListener, route *
routeService.Weighted.Services = append(routeService.Weighted.Services, service)
}
conf.TCP.Services[routeServiceKey] = routeService
routeConf.TCP.Services[routeServiceKey] = routeService
router.Service = routeServiceKey
conf.TCP.Routers[routerKey] = &router
routeConf.TCP.Routers[routerKey] = &router
return routeConditions
return routeConf, routeCondition
}
// loadTCPServices is generating a WRR service, even when there is only one target.
@ -294,3 +294,35 @@ func loadTCPServices(client Client, namespace string, backendRefs []gatev1.Backe
return wrrSvc, services, nil
}
func mergeTCPConfiguration(from, to *dynamic.Configuration) {
if from == nil || from.TCP == nil || to == nil {
return
}
if to.TCP == nil {
to.TCP = from.TCP
return
}
if to.TCP.Routers == nil {
to.TCP.Routers = map[string]*dynamic.TCPRouter{}
}
for routerName, router := range from.TCP.Routers {
to.TCP.Routers[routerName] = router
}
if to.TCP.Middlewares == nil {
to.TCP.Middlewares = map[string]*dynamic.TCPMiddleware{}
}
for middlewareName, middleware := range from.TCP.Middlewares {
to.TCP.Middlewares[middlewareName] = middleware
}
if to.TCP.Services == nil {
to.TCP.Services = map[string]*dynamic.TCPService{}
}
for serviceName, service := range from.TCP.Services {
to.TCP.Services[serviceName] = service
}
}

View file

@ -19,7 +19,8 @@ func (p *Provider) loadTLSRoutes(ctx context.Context, client Client, gatewayList
logger := log.Ctx(ctx)
routes, err := client.ListTLSRoutes()
if err != nil {
logger.Error().Err(err).Msgf("Get TLSRoutes: %s", err)
logger.Error().Err(err).Msgf("Unable to list TLSRoute")
return
}
for _, route := range routes {
@ -31,44 +32,6 @@ func (p *Provider) loadTLSRoutes(ctx context.Context, client Client, gatewayList
ParentRef: parentRef,
ControllerName: controllerName,
Conditions: []metav1.Condition{
{
Type: string(gatev1.RouteConditionAccepted),
Status: metav1.ConditionTrue,
ObservedGeneration: route.Generation,
LastTransitionTime: metav1.Now(),
Reason: string(gatev1.RouteReasonAccepted),
},
},
}
var attachedListeners bool
for _, listener := range gatewayListeners {
if !matchListener(listener, route.Namespace, parentRef) {
continue
}
if !allowRoute(listener, route.Namespace, kindTLSRoute) {
continue
}
hostnames, ok := findMatchingHostnames(listener.Hostname, route.Spec.Hostnames)
if !ok {
continue
}
listener.Status.AttachedRoutes++
attachedListeners = true
resolveConditions := p.loadTLSRoute(client, listener, route, hostnames, conf)
// TODO: handle more accurately route conditions (in case of multiple listener matching).
for _, condition := range resolveConditions {
parentStatus.Conditions = appendCondition(parentStatus.Conditions, condition)
}
}
if !attachedListeners {
parentStatus.Conditions = []metav1.Condition{
{
Type: string(gatev1.RouteConditionAccepted),
Status: metav1.ConditionFalse,
@ -76,7 +39,38 @@ func (p *Provider) loadTLSRoutes(ctx context.Context, client Client, gatewayList
LastTransitionTime: metav1.Now(),
Reason: string(gatev1.RouteReasonNoMatchingParent),
},
},
}
for _, listener := range gatewayListeners {
if !matchListener(listener, route.Namespace, parentRef) {
continue
}
accepted := true
if !allowRoute(listener, route.Namespace, kindTLSRoute) {
parentStatus.Conditions = updateRouteConditionAccepted(parentStatus.Conditions, string(gatev1.RouteReasonNotAllowedByListeners))
accepted = false
}
hostnames, ok := findMatchingHostnames(listener.Hostname, route.Spec.Hostnames)
if !ok {
parentStatus.Conditions = updateRouteConditionAccepted(parentStatus.Conditions, string(gatev1.RouteReasonNoMatchingListenerHostname))
accepted = false
}
if accepted {
listener.Status.AttachedRoutes++
// only consider the route attached if the listener is in an "attached" state.
if listener.Attached {
parentStatus.Conditions = updateRouteConditionAccepted(parentStatus.Conditions, string(gatev1.RouteReasonAccepted))
}
}
routeConf, resolveRefCondition := p.loadTLSRoute(client, listener, route, hostnames)
if accepted && listener.Attached {
mergeTCPConfiguration(routeConf, conf)
}
parentStatus.Conditions = upsertRouteConditionResolvedRefs(parentStatus.Conditions, resolveRefCondition)
}
parentStatuses = append(parentStatuses, *parentStatus)
@ -95,23 +89,30 @@ func (p *Provider) loadTLSRoutes(ctx context.Context, client Client, gatewayList
}
}
func (p *Provider) loadTLSRoute(client Client, listener gatewayListener, route *gatev1alpha2.TLSRoute, hostnames []gatev1.Hostname, conf *dynamic.Configuration) []metav1.Condition {
routeConditions := []metav1.Condition{
{
Type: string(gatev1.RouteConditionResolvedRefs),
Status: metav1.ConditionTrue,
ObservedGeneration: route.Generation,
LastTransitionTime: metav1.Now(),
Reason: string(gatev1.RouteConditionResolvedRefs),
func (p *Provider) loadTLSRoute(client Client, listener gatewayListener, route *gatev1alpha2.TLSRoute, hostnames []gatev1.Hostname) (*dynamic.Configuration, metav1.Condition) {
routeConf := &dynamic.Configuration{
TCP: &dynamic.TCPConfiguration{
Routers: make(map[string]*dynamic.TCPRouter),
Middlewares: make(map[string]*dynamic.TCPMiddleware),
Services: make(map[string]*dynamic.TCPService),
ServersTransports: make(map[string]*dynamic.TCPServersTransport),
},
}
routeCondition := metav1.Condition{
Type: string(gatev1.RouteConditionResolvedRefs),
Status: metav1.ConditionTrue,
ObservedGeneration: route.Generation,
LastTransitionTime: metav1.Now(),
Reason: string(gatev1.RouteConditionResolvedRefs),
}
router := dynamic.TCPRouter{
RuleSyntax: "v3",
Rule: hostSNIRule(hostnames),
EntryPoints: []string{listener.EPName},
TLS: &dynamic.RouterTCPTLSConfig{
Passthrough: listener.TLS.Mode != nil && *listener.TLS.Mode == gatev1.TLSModePassthrough,
Passthrough: listener.TLS != nil && listener.TLS.Mode != nil && *listener.TLS.Mode == gatev1.TLSModePassthrough,
},
}
@ -130,32 +131,32 @@ func (p *Provider) loadTLSRoute(client Client, listener gatewayListener, route *
wrrService, subServices, err := loadTCPServices(client, route.Namespace, routeRule.BackendRefs)
if err != nil {
// update "ResolvedRefs" status true with "InvalidBackendRefs" reason
routeConditions = appendCondition(routeConditions, metav1.Condition{
routeCondition = metav1.Condition{
Type: string(gatev1.RouteConditionResolvedRefs),
Status: metav1.ConditionFalse,
ObservedGeneration: route.Generation,
LastTransitionTime: metav1.Now(),
Reason: string(gatev1.RouteReasonBackendNotFound),
Message: fmt.Sprintf("Cannot load TLSRoute service %s/%s: %v", route.Namespace, route.Name, err),
})
}
continue
}
for svcName, svc := range subServices {
conf.TCP.Services[svcName] = svc
routeConf.TCP.Services[svcName] = svc
}
serviceName := fmt.Sprintf("%s-wrr-%d", routerKey, i)
conf.TCP.Services[serviceName] = wrrService
routeConf.TCP.Services[serviceName] = wrrService
ruleServiceNames = append(ruleServiceNames, serviceName)
}
if len(ruleServiceNames) == 1 {
router.Service = ruleServiceNames[0]
conf.TCP.Routers[routerKey] = &router
routeConf.TCP.Routers[routerKey] = &router
return routeConditions
return routeConf, routeCondition
}
routeServiceKey := routerKey + "-wrr"
@ -168,12 +169,12 @@ func (p *Provider) loadTLSRoute(client Client, listener gatewayListener, route *
routeService.Weighted.Services = append(routeService.Weighted.Services, service)
}
conf.TCP.Services[routeServiceKey] = routeService
routeConf.TCP.Services[routeServiceKey] = routeService
router.Service = routeServiceKey
conf.TCP.Routers[routerKey] = &router
routeConf.TCP.Routers[routerKey] = &router
return routeConditions
return routeConf, routeCondition
}
func hostSNIRule(hostnames []gatev1.Hostname) string {