diff --git a/integration/k8s_conformance_test.go b/integration/k8s_conformance_test.go index 1e0c5d96b..5e318d562 100644 --- a/integration/k8s_conformance_test.go +++ b/integration/k8s_conformance_test.go @@ -195,13 +195,23 @@ func (s *K8sConformanceSuite) TestK8sGatewayAPIConformance() { LatestObservedGenerationSet: 5 * time.Second, RequiredConsecutiveSuccesses: 0, }, + SupportedFeatures: sets.New(ksuite.SupportGateway, ksuite.SupportHTTPRoute). + Union(ksuite.HTTPRouteExtendedFeatures), EnableAllSupportedFeatures: false, RunTest: *k8sConformanceRunTest, // Until the feature are all supported, following tests are skipped. SkipTests: []string{ - tests.HTTPRouteInvalidCrossNamespaceParentRef.ShortName, - tests.HTTPRoutePartiallyInvalidViaInvalidReferenceGrant.ShortName, - tests.HTTPRouteReferenceGrant.ShortName, + tests.HTTPRouteMethodMatching.ShortName, + tests.HTTPRouteQueryParamMatching.ShortName, + tests.HTTPRouteRedirectPath.ShortName, + tests.HTTPRouteRedirectPortAndScheme.ShortName, + tests.HTTPRouteRequestMirror.ShortName, + tests.HTTPRouteRequestMultipleMirrors.ShortName, + tests.HTTPRouteResponseHeaderModifier.ShortName, + tests.HTTPRouteRewriteHost.ShortName, + tests.HTTPRouteRewritePath.ShortName, + tests.HTTPRouteTimeoutBackendRequest.ShortName, + tests.HTTPRouteTimeoutRequest.ShortName, }, } diff --git a/pkg/provider/kubernetes/gateway/fixtures/referencegrant/for_service.yml b/pkg/provider/kubernetes/gateway/fixtures/referencegrant/for_service.yml new file mode 100644 index 000000000..0975c9442 --- /dev/null +++ b/pkg/provider/kubernetes/gateway/fixtures/referencegrant/for_service.yml @@ -0,0 +1,61 @@ +--- +apiVersion: gateway.networking.k8s.io/v1beta1 +kind: ReferenceGrant +metadata: + name: backend-from-bar + namespace: bar +spec: + from: + - group: gateway.networking.k8s.io + kind: HTTPRoute + namespace: default + to: + - group: "" + kind: Service + +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: GatewayClass +metadata: + name: my-gateway-class +spec: + controllerName: traefik.io/gateway-controller + +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: my-gateway + namespace: default +spec: + gatewayClassName: my-gateway-class + listeners: # Use GatewayClass defaults for listener definition. + - name: http + protocol: HTTP + port: 80 + hostname: foo.example.com + allowedRoutes: + kinds: + - kind: HTTPRoute + group: gateway.networking.k8s.io + namespaces: + from: Same + +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: http-app-1 + namespace: default +spec: + parentRefs: + - name: my-gateway + kind: Gateway + group: gateway.networking.k8s.io + rules: + - backendRefs: + - name: whoami-bar + port: 80 + weight: 1 + kind: Service + group: "" diff --git a/pkg/provider/kubernetes/gateway/fixtures/referencegrant/for_service_missing.yml b/pkg/provider/kubernetes/gateway/fixtures/referencegrant/for_service_missing.yml new file mode 100644 index 000000000..f68163547 --- /dev/null +++ b/pkg/provider/kubernetes/gateway/fixtures/referencegrant/for_service_missing.yml @@ -0,0 +1,46 @@ +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: GatewayClass +metadata: + name: my-gateway-class +spec: + controllerName: traefik.io/gateway-controller + +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: my-gateway + namespace: default +spec: + gatewayClassName: my-gateway-class + listeners: # Use GatewayClass defaults for listener definition. + - name: http + protocol: HTTP + port: 80 + hostname: foo.example.com + allowedRoutes: + kinds: + - kind: HTTPRoute + group: gateway.networking.k8s.io + namespaces: + from: Same + +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: http-app-1 + namespace: default +spec: + parentRefs: + - name: my-gateway + kind: Gateway + group: gateway.networking.k8s.io + rules: + - backendRefs: + - name: whoami-bar + port: 80 + weight: 1 + kind: Service + group: "" diff --git a/pkg/provider/kubernetes/gateway/fixtures/referencegrant/for_service_not_matching_from.yml b/pkg/provider/kubernetes/gateway/fixtures/referencegrant/for_service_not_matching_from.yml new file mode 100644 index 000000000..422fca0a5 --- /dev/null +++ b/pkg/provider/kubernetes/gateway/fixtures/referencegrant/for_service_not_matching_from.yml @@ -0,0 +1,61 @@ +--- +apiVersion: gateway.networking.k8s.io/v1beta1 +kind: ReferenceGrant +metadata: + name: backend-from-bar + namespace: bar +spec: + from: + - group: gateway.networking.k8s.io + kind: HTTPRoute + namespace: anothernamespce + to: + - group: "" + kind: Service + +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: GatewayClass +metadata: + name: my-gateway-class +spec: + controllerName: traefik.io/gateway-controller + +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: my-gateway + namespace: default +spec: + gatewayClassName: my-gateway-class + listeners: # Use GatewayClass defaults for listener definition. + - name: http + protocol: HTTP + port: 80 + hostname: foo.example.com + allowedRoutes: + kinds: + - kind: HTTPRoute + group: gateway.networking.k8s.io + namespaces: + from: Same + +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: http-app-1 + namespace: default +spec: + parentRefs: + - name: my-gateway + kind: Gateway + group: gateway.networking.k8s.io + rules: + - backendRefs: + - name: whoami-bar + port: 80 + weight: 1 + kind: Service + group: "" diff --git a/pkg/provider/kubernetes/gateway/fixtures/referencegrant/for_service_not_matching_to.yml b/pkg/provider/kubernetes/gateway/fixtures/referencegrant/for_service_not_matching_to.yml new file mode 100644 index 000000000..b5cff815e --- /dev/null +++ b/pkg/provider/kubernetes/gateway/fixtures/referencegrant/for_service_not_matching_to.yml @@ -0,0 +1,62 @@ +--- +apiVersion: gateway.networking.k8s.io/v1beta1 +kind: ReferenceGrant +metadata: + name: backend-from-bar + namespace: bar +spec: + from: + - group: gateway.networking.k8s.io + kind: HTTPRoute + namespace: default + to: + - group: "" + kind: Service + name: differentservice + +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: GatewayClass +metadata: + name: my-gateway-class +spec: + controllerName: traefik.io/gateway-controller + +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: my-gateway + namespace: default +spec: + gatewayClassName: my-gateway-class + listeners: # Use GatewayClass defaults for listener definition. + - name: http + protocol: HTTP + port: 80 + hostname: foo.example.com + allowedRoutes: + kinds: + - kind: HTTPRoute + group: gateway.networking.k8s.io + namespaces: + from: Same + +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: http-app-1 + namespace: default +spec: + parentRefs: + - name: my-gateway + kind: Gateway + group: gateway.networking.k8s.io + rules: + - backendRefs: + - name: whoami-bar + port: 80 + weight: 1 + kind: Service + group: "" diff --git a/pkg/provider/kubernetes/gateway/httproute.go b/pkg/provider/kubernetes/gateway/httproute.go index 86664ee5f..1570a55ea 100644 --- a/pkg/provider/kubernetes/gateway/httproute.go +++ b/pkg/provider/kubernetes/gateway/httproute.go @@ -41,64 +41,45 @@ func (p *Provider) loadHTTPRoutes(ctx context.Context, client Client, gatewayLis Conditions: []metav1.Condition{ { Type: string(gatev1.RouteConditionAccepted), - Status: metav1.ConditionTrue, + Status: metav1.ConditionFalse, ObservedGeneration: route.Generation, LastTransitionTime: metav1.Now(), - Reason: string(gatev1.RouteReasonAccepted), + Reason: string(gatev1.RouteReasonNoMatchingParent), }, }, } - var attachedListeners bool - notAcceptedReason := gatev1.RouteReasonNoMatchingParent for _, listener := range gatewayListeners { if !matchListener(listener, route.Namespace, parentRef) { continue } + accepted := true if !allowRoute(listener, route.Namespace, kindHTTPRoute) { - notAcceptedReason = gatev1.RouteReasonNotAllowedByListeners - continue + parentStatus.Conditions = updateRouteConditionAccepted(parentStatus.Conditions, string(gatev1.RouteReasonNotAllowedByListeners)) + accepted = false } - hostnames, ok := findMatchingHostnames(listener.Hostname, route.Spec.Hostnames) if !ok { - notAcceptedReason = gatev1.RouteReasonNoMatchingListenerHostname - continue + parentStatus.Conditions = updateRouteConditionAccepted(parentStatus.Conditions, string(gatev1.RouteReasonNoMatchingListenerHostname)) + accepted = false } - listener.Status.AttachedRoutes++ - - // TODO should we build the conf if the listener is not attached - // only consider the route attached if the listener is in an "attached" state. - if listener.Attached { - attachedListeners = true + if accepted { + // Gateway listener should have AttachedRoutes set even when Gateway has unresolved refs. + listener.Status.AttachedRoutes++ + // Only consider the route attached if the listener is in an "attached" state. + if listener.Attached { + parentStatus.Conditions = updateRouteConditionAccepted(parentStatus.Conditions, string(gatev1.RouteReasonAccepted)) + } } - resolveConditions := p.loadHTTPRoute(logger.WithContext(ctx), client, listener, route, hostnames, conf) - // TODO: handle more accurately route conditions (in case of multiple listener matching). - for _, condition := range resolveConditions { - parentStatus.Conditions = appendCondition(parentStatus.Conditions, condition) + routeConf, resolveRefCondition := p.loadHTTPRoute(logger.WithContext(ctx), client, listener, route, hostnames) + if accepted && listener.Attached { + mergeHTTPConfiguration(routeConf, conf) } - } - if !attachedListeners { - parentStatus.Conditions = []metav1.Condition{ - { - Type: string(gatev1.RouteConditionAccepted), - Status: metav1.ConditionFalse, - ObservedGeneration: route.Generation, - LastTransitionTime: metav1.Now(), - Reason: string(notAcceptedReason), - }, - { - Type: string(gatev1.RouteConditionResolvedRefs), - Status: metav1.ConditionFalse, - ObservedGeneration: route.Generation, - LastTransitionTime: metav1.Now(), - Reason: string(gatev1.RouteReasonRefNotPermitted), - }, - } + parentStatus.Conditions = upsertRouteConditionResolvedRefs(parentStatus.Conditions, resolveRefCondition) } parentStatuses = append(parentStatuses, *parentStatus) @@ -117,17 +98,24 @@ func (p *Provider) loadHTTPRoutes(ctx context.Context, client Client, gatewayLis } } -func (p *Provider) loadHTTPRoute(ctx context.Context, client Client, listener gatewayListener, route *gatev1.HTTPRoute, hostnames []gatev1.Hostname, conf *dynamic.Configuration) []metav1.Condition { - routeConditions := []metav1.Condition{ - { - Type: string(gatev1.RouteConditionResolvedRefs), - Status: metav1.ConditionTrue, - ObservedGeneration: route.Generation, - LastTransitionTime: metav1.Now(), - Reason: string(gatev1.RouteConditionResolvedRefs), +func (p *Provider) loadHTTPRoute(ctx context.Context, client Client, listener gatewayListener, route *gatev1.HTTPRoute, hostnames []gatev1.Hostname) (*dynamic.Configuration, metav1.Condition) { + routeConf := &dynamic.Configuration{ + HTTP: &dynamic.HTTPConfiguration{ + Routers: make(map[string]*dynamic.Router), + Middlewares: make(map[string]*dynamic.Middleware), + Services: make(map[string]*dynamic.Service), + ServersTransports: make(map[string]*dynamic.ServersTransport), }, } + routeCondition := metav1.Condition{ + Type: string(gatev1.RouteConditionResolvedRefs), + Status: metav1.ConditionTrue, + ObservedGeneration: route.Generation, + LastTransitionTime: metav1.Now(), + Reason: string(gatev1.RouteConditionResolvedRefs), + } + for _, routeRule := range route.Spec.Rules { rule, priority := buildRouterRule(hostnames, routeRule.Matches) router := dynamic.Router{ @@ -159,14 +147,14 @@ func (p *Provider) loadHTTPRoute(ctx context.Context, client Client, listener ga Weight: ptr.To(1), }) - conf.HTTP.Services[wrrName] = &dynamic.Service{Weighted: &wrr} + routeConf.HTTP.Services[wrrName] = &dynamic.Service{Weighted: &wrr} router.Service = wrrName } else { for name, middleware := range middlewares { // If the middleware config is nil in the return of the loadMiddlewares function, // it means that we just need a reference to that middleware. if middleware != nil { - conf.HTTP.Middlewares[name] = middleware + routeConf.HTTP.Middlewares[name] = middleware } router.Middlewares = append(router.Middlewares, name) @@ -180,7 +168,7 @@ func (p *Provider) loadHTTPRoute(ctx context.Context, client Client, listener ga name, svc, errCondition := p.loadHTTPService(client, route, backendRef) weight := ptr.To(int(ptr.Deref(backendRef.Weight, 1))) if errCondition != nil { - routeConditions = appendCondition(routeConditions, *errCondition) + routeCondition = *errCondition wrr.Services = append(wrr.Services, dynamic.WRRService{ Name: name, Status: ptr.To(500), @@ -190,7 +178,7 @@ func (p *Provider) loadHTTPRoute(ctx context.Context, client Client, listener ga } if svc != nil { - conf.HTTP.Services[name] = svc + routeConf.HTTP.Services[name] = svc } wrr.Services = append(wrr.Services, dynamic.WRRService{ @@ -199,7 +187,7 @@ func (p *Provider) loadHTTPRoute(ctx context.Context, client Client, listener ga }) } - conf.HTTP.Services[wrrName] = &dynamic.Service{Weighted: &wrr} + routeConf.HTTP.Services[wrrName] = &dynamic.Service{Weighted: &wrr} router.Service = wrrName } } @@ -208,39 +196,42 @@ func (p *Provider) loadHTTPRoute(ctx context.Context, client Client, listener ga p.applyRouterTransform(ctx, rt, route) routerKey = provider.Normalize(routerKey) - conf.HTTP.Routers[routerKey] = rt + routeConf.HTTP.Routers[routerKey] = rt } - return routeConditions + return routeConf, routeCondition } // loadHTTPService returns a dynamic.Service config corresponding to the given gatev1.HTTPBackendRef. // Note that the returned dynamic.Service config can be nil (for cross-provider, internal services, and backendFunc). func (p *Provider) loadHTTPService(client Client, route *gatev1.HTTPRoute, backendRef gatev1.HTTPBackendRef) (string, *dynamic.Service, *metav1.Condition) { + kind := ptr.Deref(backendRef.Kind, "Service") + group := groupCore if backendRef.Group != nil && *backendRef.Group != "" { group = string(*backendRef.Group) } - kind := ptr.Deref(backendRef.Kind, "Service") - namespace := ptr.Deref(backendRef.Namespace, gatev1.Namespace(route.Namespace)) - namespaceStr := string(namespace) - serviceName := provider.Normalize(makeID(namespaceStr, string(backendRef.Name))) + namespace := route.Namespace + if backendRef.Namespace != nil && *backendRef.Namespace != "" { + namespace = string(*backendRef.Namespace) + } - // TODO support cross namespace through ReferenceGrant. - if namespaceStr != route.Namespace { + serviceName := provider.Normalize(makeID(namespace, string(backendRef.Name))) + + if err := isReferenceGranted(client, groupGateway, kindHTTPRoute, route.Namespace, group, string(kind), string(backendRef.Name), namespace); err != nil { return serviceName, nil, &metav1.Condition{ Type: string(gatev1.RouteConditionResolvedRefs), Status: metav1.ConditionFalse, ObservedGeneration: route.Generation, LastTransitionTime: metav1.Now(), Reason: string(gatev1.RouteReasonRefNotPermitted), - Message: fmt.Sprintf("Cannot load HTTPBackendRef %s/%s/%s/%s namespace not allowed", group, kind, namespace, backendRef.Name), + Message: fmt.Sprintf("Cannot load HTTPBackendRef %s/%s/%s/%s: %s", group, kind, namespace, backendRef.Name, err), } } if group != groupCore || kind != "Service" { - name, service, err := p.loadHTTPBackendRef(namespaceStr, backendRef) + name, service, err := p.loadHTTPBackendRef(namespace, backendRef) if err != nil { return serviceName, nil, &metav1.Condition{ Type: string(gatev1.RouteConditionResolvedRefs), @@ -270,7 +261,7 @@ func (p *Provider) loadHTTPService(client Client, route *gatev1.HTTPRoute, backe portStr := strconv.FormatInt(int64(port), 10) serviceName = provider.Normalize(serviceName + "-" + portStr) - lb, err := loadHTTPServers(client, namespaceStr, backendRef) + lb, err := loadHTTPServers(client, namespace, backendRef) if err != nil { return serviceName, nil, &metav1.Condition{ Type: string(gatev1.RouteConditionResolvedRefs), @@ -615,3 +606,35 @@ func getProtocol(portSpec corev1.ServicePort) string { return protocol } + +func mergeHTTPConfiguration(from, to *dynamic.Configuration) { + if from == nil || from.HTTP == nil || to == nil { + return + } + + if to.HTTP == nil { + to.HTTP = from.HTTP + return + } + + if to.HTTP.Routers == nil { + to.HTTP.Routers = map[string]*dynamic.Router{} + } + for routerName, router := range from.HTTP.Routers { + to.HTTP.Routers[routerName] = router + } + + if to.HTTP.Middlewares == nil { + to.HTTP.Middlewares = map[string]*dynamic.Middleware{} + } + for middlewareName, middleware := range from.HTTP.Middlewares { + to.HTTP.Middlewares[middlewareName] = middleware + } + + if to.HTTP.Services == nil { + to.HTTP.Services = map[string]*dynamic.Service{} + } + for serviceName, service := range from.HTTP.Services { + to.HTTP.Services[serviceName] = service + } +} diff --git a/pkg/provider/kubernetes/gateway/kubernetes.go b/pkg/provider/kubernetes/gateway/kubernetes.go index e6cea12a1..55ee9a999 100644 --- a/pkg/provider/kubernetes/gateway/kubernetes.go +++ b/pkg/provider/kubernetes/gateway/kubernetes.go @@ -37,7 +37,8 @@ const ( providerName = "kubernetesgateway" controllerName = "traefik.io/gateway-controller" - groupCore = "core" + groupCore = "core" + groupGateway = "gateway.networking.k8s.io" kindGateway = "Gateway" kindTraefikService = "TraefikService" @@ -564,35 +565,17 @@ func (p *Provider) loadGatewayListeners(ctx context.Context, client Client, gate certificateNamespace = string(*certificateRef.Namespace) } - if certificateNamespace != gateway.Namespace { - referenceGrants, err := client.ListReferenceGrants(certificateNamespace) - if err != nil { - gatewayListeners[i].Status.Conditions = append(gatewayListeners[i].Status.Conditions, metav1.Condition{ - Type: string(gatev1.ListenerConditionResolvedRefs), - Status: metav1.ConditionFalse, - ObservedGeneration: gateway.Generation, - LastTransitionTime: metav1.Now(), - Reason: string(gatev1.ListenerReasonRefNotPermitted), - Message: fmt.Sprintf("Cannot find any ReferenceGrant: %v", err), - }) + if err := isReferenceGranted(client, groupGateway, kindGateway, gateway.Namespace, groupCore, "Secret", string(certificateRef.Name), certificateNamespace); err != nil { + gatewayListeners[i].Status.Conditions = append(gatewayListeners[i].Status.Conditions, metav1.Condition{ + Type: string(gatev1.ListenerConditionResolvedRefs), + Status: metav1.ConditionFalse, + ObservedGeneration: gateway.Generation, + LastTransitionTime: metav1.Now(), + Reason: string(gatev1.ListenerReasonRefNotPermitted), + Message: fmt.Sprintf("Cannot load CertificateRef %s/%s: %s", certificateNamespace, certificateRef.Name, err), + }) - continue - } - - referenceGrants = filterReferenceGrantsFrom(referenceGrants, "gateway.networking.k8s.io", "Gateway", gateway.Namespace) - referenceGrants = filterReferenceGrantsTo(referenceGrants, groupCore, "Secret", string(certificateRef.Name)) - if len(referenceGrants) == 0 { - gatewayListeners[i].Status.Conditions = append(gatewayListeners[i].Status.Conditions, metav1.Condition{ - Type: string(gatev1.ListenerConditionResolvedRefs), - Status: metav1.ConditionFalse, - ObservedGeneration: gateway.Generation, - LastTransitionTime: metav1.Now(), - Reason: string(gatev1.ListenerReasonRefNotPermitted), - Message: "Required ReferenceGrant for cross namespace secret reference is missing", - }) - - continue - } + continue } configKey := certificateNamespace + "/" + string(certificateRef.Name) @@ -1006,8 +989,8 @@ func makeID(namespace, name string) string { return namespace + "-" + name } -func getTLS(k8sClient Client, secretName gatev1.ObjectName, namespace string) (*tls.CertAndStores, error) { - secret, exists, err := k8sClient.GetSecret(namespace, string(secretName)) +func getTLS(client Client, secretName gatev1.ObjectName, namespace string) (*tls.CertAndStores, error) { + secret, exists, err := client.GetSecret(namespace, string(secretName)) if err != nil { return nil, fmt.Errorf("failed to fetch secret %s/%s: %w", namespace, secretName, err) } @@ -1131,6 +1114,25 @@ func makeListenerKey(l gatev1.Listener) string { return fmt.Sprintf("%s|%s|%d", l.Protocol, hostname, l.Port) } +func isReferenceGranted(client Client, fromGroup, fromKind, fromNamespace, toGroup, toKind, toName, toNamespace string) error { + if toNamespace == fromNamespace { + return nil + } + + refGrants, err := client.ListReferenceGrants(toNamespace) + if err != nil { + return fmt.Errorf("listing ReferenceGrant: %w", err) + } + + refGrants = filterReferenceGrantsFrom(refGrants, fromGroup, fromKind, fromNamespace) + refGrants = filterReferenceGrantsTo(refGrants, toGroup, toKind, toName) + if len(refGrants) == 0 { + return errors.New("missing ReferenceGrant") + } + + return nil +} + func filterReferenceGrantsFrom(referenceGrants []*gatev1beta1.ReferenceGrant, group, kind, namespace string) []*gatev1beta1.ReferenceGrant { var matchingReferenceGrants []*gatev1beta1.ReferenceGrant for _, referenceGrant := range referenceGrants { @@ -1193,13 +1195,38 @@ func kindToString(p *gatev1.Kind) string { return string(*p) } -func appendCondition(conditions []metav1.Condition, condition metav1.Condition) []metav1.Condition { - res := []metav1.Condition{condition} +func updateRouteConditionAccepted(conditions []metav1.Condition, reason string) []metav1.Condition { + var conds []metav1.Condition for _, c := range conditions { - if c.Type != condition.Type { - res = append(res, c) + if c.Type == string(gatev1.RouteConditionAccepted) && c.Status != metav1.ConditionTrue { + c.Reason = reason + c.LastTransitionTime = metav1.Now() + + if reason == string(gatev1.RouteReasonAccepted) { + c.Status = metav1.ConditionTrue + } } + + conds = append(conds, c) } - return res + return conds +} + +func upsertRouteConditionResolvedRefs(conditions []metav1.Condition, condition metav1.Condition) []metav1.Condition { + var ( + curr *metav1.Condition + conds []metav1.Condition + ) + for _, c := range conditions { + if c.Type == string(gatev1.RouteConditionResolvedRefs) { + curr = &c + continue + } + conds = append(conds, c) + } + if curr != nil && curr.Status == metav1.ConditionFalse && condition.Status == metav1.ConditionTrue { + return append(conds, *curr) + } + return append(conds, condition) } diff --git a/pkg/provider/kubernetes/gateway/kubernetes_test.go b/pkg/provider/kubernetes/gateway/kubernetes_test.go index 0a61618e1..8a4ce02b5 100644 --- a/pkg/provider/kubernetes/gateway/kubernetes_test.go +++ b/pkg/provider/kubernetes/gateway/kubernetes_test.go @@ -5422,7 +5422,7 @@ func TestLoadRoutesWithReferenceGrants(t *testing.T) { }, }, { - desc: "Empty because ReferenceGrant spec.from does not match", + desc: "Empty because ReferenceGrant spec.from does not match secret", paths: []string{"services.yml", "referencegrant/for_secret_not_matching_from.yml"}, entryPoints: map[string]Entrypoint{ "tls": {Address: ":9000"}, @@ -5448,7 +5448,7 @@ func TestLoadRoutesWithReferenceGrants(t *testing.T) { }, }, { - desc: "Empty because ReferenceGrant spec.to does not match", + desc: "Empty because ReferenceGrant spec.to does not match secret", paths: []string{"services.yml", "referencegrant/for_secret_not_matching_to.yml"}, entryPoints: map[string]Entrypoint{ "tls": {Address: ":9000"}, @@ -5538,6 +5538,130 @@ func TestLoadRoutesWithReferenceGrants(t *testing.T) { }, }, }, + { + desc: "Empty because ReferenceGrant for Service is missing", + paths: []string{"services.yml", "referencegrant/for_secret_missing.yml"}, + entryPoints: map[string]Entrypoint{ + "tls": {Address: ":9000"}, + }, + expected: &dynamic.Configuration{ + UDP: &dynamic.UDPConfiguration{ + Routers: map[string]*dynamic.UDPRouter{}, + Services: map[string]*dynamic.UDPService{}, + }, + TCP: &dynamic.TCPConfiguration{ + Routers: map[string]*dynamic.TCPRouter{}, + Middlewares: map[string]*dynamic.TCPMiddleware{}, + Services: map[string]*dynamic.TCPService{}, + ServersTransports: map[string]*dynamic.TCPServersTransport{}, + }, + HTTP: &dynamic.HTTPConfiguration{ + Routers: map[string]*dynamic.Router{}, + Middlewares: map[string]*dynamic.Middleware{}, + Services: map[string]*dynamic.Service{}, + ServersTransports: map[string]*dynamic.ServersTransport{}, + }, + TLS: &dynamic.TLSConfiguration{}, + }, + }, + { + desc: "Empty because ReferenceGrant spec.from does not match service", + paths: []string{"services.yml", "referencegrant/for_service_not_matching_from.yml"}, + entryPoints: map[string]Entrypoint{ + "tls": {Address: ":9000"}, + }, + expected: &dynamic.Configuration{ + UDP: &dynamic.UDPConfiguration{ + Routers: map[string]*dynamic.UDPRouter{}, + Services: map[string]*dynamic.UDPService{}, + }, + TCP: &dynamic.TCPConfiguration{ + Routers: map[string]*dynamic.TCPRouter{}, + Middlewares: map[string]*dynamic.TCPMiddleware{}, + Services: map[string]*dynamic.TCPService{}, + ServersTransports: map[string]*dynamic.TCPServersTransport{}, + }, + HTTP: &dynamic.HTTPConfiguration{ + Routers: map[string]*dynamic.Router{}, + Middlewares: map[string]*dynamic.Middleware{}, + Services: map[string]*dynamic.Service{}, + ServersTransports: map[string]*dynamic.ServersTransport{}, + }, + TLS: &dynamic.TLSConfiguration{}, + }, + }, + { + desc: "Empty because ReferenceGrant spec.to does not match service", + paths: []string{"services.yml", "referencegrant/for_service_not_matching_to.yml"}, + entryPoints: map[string]Entrypoint{ + "tls": {Address: ":9000"}, + }, + expected: &dynamic.Configuration{ + UDP: &dynamic.UDPConfiguration{ + Routers: map[string]*dynamic.UDPRouter{}, + Services: map[string]*dynamic.UDPService{}, + }, + TCP: &dynamic.TCPConfiguration{ + Routers: map[string]*dynamic.TCPRouter{}, + Middlewares: map[string]*dynamic.TCPMiddleware{}, + Services: map[string]*dynamic.TCPService{}, + ServersTransports: map[string]*dynamic.TCPServersTransport{}, + }, + HTTP: &dynamic.HTTPConfiguration{ + Routers: map[string]*dynamic.Router{}, + Middlewares: map[string]*dynamic.Middleware{}, + Services: map[string]*dynamic.Service{}, + ServersTransports: map[string]*dynamic.ServersTransport{}, + }, + TLS: &dynamic.TLSConfiguration{}, + }, + }, + { + desc: "For Service", + paths: []string{"services.yml", "referencegrant/for_service.yml"}, + entryPoints: map[string]Entrypoint{ + "http": {Address: ":80"}, + }, + expected: &dynamic.Configuration{ + UDP: &dynamic.UDPConfiguration{ + Routers: map[string]*dynamic.UDPRouter{}, + Services: map[string]*dynamic.UDPService{}, + }, + TCP: &dynamic.TCPConfiguration{ + Routers: map[string]*dynamic.TCPRouter{}, + Middlewares: map[string]*dynamic.TCPMiddleware{}, + Services: map[string]*dynamic.TCPService{}, + ServersTransports: map[string]*dynamic.TCPServersTransport{}, + }, + HTTP: &dynamic.HTTPConfiguration{ + Routers: map[string]*dynamic.Router{ + "default-http-app-1-my-gateway-http-f381ad97110137b4d42c": { + EntryPoints: []string{"http"}, + Rule: "Host(`foo.example.com`)", + Service: "default-http-app-1-my-gateway-http-f381ad97110137b4d42c-wrr", + RuleSyntax: "v3", + Priority: 15, + }, + }, + Middlewares: map[string]*dynamic.Middleware{}, + Services: map[string]*dynamic.Service{ + "default-http-app-1-my-gateway-http-f381ad97110137b4d42c-wrr": { + Weighted: &dynamic.WeightedRoundRobin{ + Services: []dynamic.WRRService{ + { + Name: "default-whoami-bar-80", + Weight: ptr.To(1), + Status: ptr.To(500), + }, + }, + }, + }, + }, + ServersTransports: map[string]*dynamic.ServersTransport{}, + }, + TLS: &dynamic.TLSConfiguration{}, + }, + }, } for _, test := range testCases { @@ -6433,6 +6557,136 @@ func Test_gatewayAddresses(t *testing.T) { } } +func Test_upsertRouteConditionResolvedRefs(t *testing.T) { + testCases := []struct { + desc string + conditions []metav1.Condition + condition metav1.Condition + wantConditions []metav1.Condition + }{ + { + desc: "True to False", + conditions: []metav1.Condition{ + { + Type: "foo", + Status: "bar", + Reason: "baz", + Message: "foobarbaz", + }, + { + Type: string(gatev1.RouteConditionResolvedRefs), + Status: metav1.ConditionTrue, + Reason: "foo", + Message: "foo", + }, + }, + condition: metav1.Condition{ + Type: string(gatev1.RouteConditionResolvedRefs), + Status: metav1.ConditionFalse, + Reason: "bar", + Message: "bar", + }, + wantConditions: []metav1.Condition{ + { + Type: "foo", + Status: "bar", + Reason: "baz", + Message: "foobarbaz", + }, + { + Type: string(gatev1.RouteConditionResolvedRefs), + Status: metav1.ConditionFalse, + Reason: "bar", + Message: "bar", + }, + }, + }, + { + desc: "False to False", + conditions: []metav1.Condition{ + { + Type: "foo", + Status: "bar", + Reason: "baz", + Message: "foobarbaz", + }, + { + Type: string(gatev1.RouteConditionResolvedRefs), + Status: metav1.ConditionFalse, + Reason: "foo", + Message: "foo", + }, + }, + condition: metav1.Condition{ + Type: string(gatev1.RouteConditionResolvedRefs), + Status: metav1.ConditionFalse, + Reason: "bar", + Message: "bar", + }, + wantConditions: []metav1.Condition{ + { + Type: "foo", + Status: "bar", + Reason: "baz", + Message: "foobarbaz", + }, + { + Type: string(gatev1.RouteConditionResolvedRefs), + Status: metav1.ConditionFalse, + Reason: "bar", + Message: "bar", + }, + }, + }, + { + desc: "False to True: no upsert", + conditions: []metav1.Condition{ + { + Type: "foo", + Status: "bar", + Reason: "baz", + Message: "foobarbaz", + }, + { + Type: string(gatev1.RouteConditionResolvedRefs), + Status: metav1.ConditionFalse, + Reason: "foo", + Message: "foo", + }, + }, + condition: metav1.Condition{ + Type: string(gatev1.RouteConditionResolvedRefs), + Status: metav1.ConditionTrue, + Reason: "bar", + Message: "bar", + }, + wantConditions: []metav1.Condition{ + { + Type: "foo", + Status: "bar", + Reason: "baz", + Message: "foobarbaz", + }, + { + Type: string(gatev1.RouteConditionResolvedRefs), + Status: metav1.ConditionFalse, + Reason: "foo", + Message: "foo", + }, + }, + }, + } + + for _, test := range testCases { + t.Run(test.desc, func(t *testing.T) { + t.Parallel() + + got := upsertRouteConditionResolvedRefs(test.conditions, test.condition) + assert.Equal(t, test.wantConditions, got) + }) + } +} + // We cannot use the gateway-api fake.NewSimpleClientset due to Gateway being pluralized as "gatewaies" instead of "gateways". func newGatewaySimpleClientSet(t *testing.T, objects ...runtime.Object) *gatefake.Clientset { t.Helper() diff --git a/pkg/provider/kubernetes/gateway/tcproute.go b/pkg/provider/kubernetes/gateway/tcproute.go index 118ae8f1d..e0bafe99c 100644 --- a/pkg/provider/kubernetes/gateway/tcproute.go +++ b/pkg/provider/kubernetes/gateway/tcproute.go @@ -22,7 +22,8 @@ func (p *Provider) loadTCPRoutes(ctx context.Context, client Client, gatewayList logger := log.Ctx(ctx) routes, err := client.ListTCPRoutes() if err != nil { - logger.Error().Err(err).Msgf("Get TCPRoutes: %s", err) + logger.Error().Err(err).Msgf("Unable to list TCPRoutes") + return } for _, route := range routes { @@ -34,39 +35,6 @@ func (p *Provider) loadTCPRoutes(ctx context.Context, client Client, gatewayList ParentRef: parentRef, ControllerName: controllerName, Conditions: []metav1.Condition{ - { - Type: string(gatev1.RouteConditionAccepted), - Status: metav1.ConditionTrue, - ObservedGeneration: route.Generation, - LastTransitionTime: metav1.Now(), - Reason: string(gatev1.RouteReasonAccepted), - }, - }, - } - - var attachedListeners bool - for _, listener := range gatewayListeners { - if !matchListener(listener, route.Namespace, parentRef) { - continue - } - - if !allowRoute(listener, route.Namespace, kindTCPRoute) { - continue - } - - listener.Status.AttachedRoutes++ - attachedListeners = true - - resolveConditions := p.loadTCPRoute(client, listener, route, conf) - - // TODO: handle more accurately route conditions (in case of multiple listener matching). - for _, condition := range resolveConditions { - parentStatus.Conditions = appendCondition(parentStatus.Conditions, condition) - } - } - - if !attachedListeners { - parentStatus.Conditions = []metav1.Condition{ { Type: string(gatev1.RouteConditionAccepted), Status: metav1.ConditionFalse, @@ -74,7 +42,33 @@ func (p *Provider) loadTCPRoutes(ctx context.Context, client Client, gatewayList LastTransitionTime: metav1.Now(), Reason: string(gatev1.RouteReasonNoMatchingParent), }, + }, + } + + for _, listener := range gatewayListeners { + if !matchListener(listener, route.Namespace, parentRef) { + continue } + + accepted := true + if !allowRoute(listener, route.Namespace, kindTCPRoute) { + parentStatus.Conditions = updateRouteConditionAccepted(parentStatus.Conditions, string(gatev1.RouteReasonNotAllowedByListeners)) + accepted = false + } + + if accepted { + listener.Status.AttachedRoutes++ + // only consider the route attached if the listener is in an "attached" state. + if listener.Attached { + parentStatus.Conditions = updateRouteConditionAccepted(parentStatus.Conditions, string(gatev1.RouteReasonAccepted)) + } + } + + routeConf, resolveRefCondition := p.loadTCPRoute(client, listener, route) + if accepted && listener.Attached { + mergeTCPConfiguration(routeConf, conf) + } + parentStatus.Conditions = upsertRouteConditionResolvedRefs(parentStatus.Conditions, resolveRefCondition) } parentStatuses = append(parentStatuses, *parentStatus) @@ -93,17 +87,24 @@ func (p *Provider) loadTCPRoutes(ctx context.Context, client Client, gatewayList } } -func (p *Provider) loadTCPRoute(client Client, listener gatewayListener, route *gatev1alpha2.TCPRoute, conf *dynamic.Configuration) []metav1.Condition { - routeConditions := []metav1.Condition{ - { - Type: string(gatev1.RouteConditionResolvedRefs), - Status: metav1.ConditionTrue, - ObservedGeneration: route.Generation, - LastTransitionTime: metav1.Now(), - Reason: string(gatev1.RouteConditionResolvedRefs), +func (p *Provider) loadTCPRoute(client Client, listener gatewayListener, route *gatev1alpha2.TCPRoute) (*dynamic.Configuration, metav1.Condition) { + routeConf := &dynamic.Configuration{ + TCP: &dynamic.TCPConfiguration{ + Routers: make(map[string]*dynamic.TCPRouter), + Middlewares: make(map[string]*dynamic.TCPMiddleware), + Services: make(map[string]*dynamic.TCPService), + ServersTransports: make(map[string]*dynamic.TCPServersTransport), }, } + routeCondition := metav1.Condition{ + Type: string(gatev1.RouteConditionResolvedRefs), + Status: metav1.ConditionTrue, + ObservedGeneration: route.Generation, + LastTransitionTime: metav1.Now(), + Reason: string(gatev1.RouteConditionResolvedRefs), + } + router := dynamic.TCPRouter{ Rule: "HostSNI(`*`)", EntryPoints: []string{listener.EPName}, @@ -131,31 +132,30 @@ func (p *Provider) loadTCPRoute(client Client, listener gatewayListener, route * wrrService, subServices, err := loadTCPServices(client, route.Namespace, rule.BackendRefs) if err != nil { - routeConditions = appendCondition(routeConditions, metav1.Condition{ + return routeConf, metav1.Condition{ Type: string(gatev1.RouteConditionResolvedRefs), Status: metav1.ConditionFalse, ObservedGeneration: route.Generation, LastTransitionTime: metav1.Now(), Reason: string(gatev1.RouteReasonBackendNotFound), Message: fmt.Sprintf("Cannot load TCPRoute service %s/%s: %v", route.Namespace, route.Name, err), - }) - return routeConditions + } } for svcName, svc := range subServices { - conf.TCP.Services[svcName] = svc + routeConf.TCP.Services[svcName] = svc } serviceName := fmt.Sprintf("%s-wrr-%d", routerKey, i) - conf.TCP.Services[serviceName] = wrrService + routeConf.TCP.Services[serviceName] = wrrService ruleServiceNames = append(ruleServiceNames, serviceName) } if len(ruleServiceNames) == 1 { router.Service = ruleServiceNames[0] - conf.TCP.Routers[routerKey] = &router - return routeConditions + routeConf.TCP.Routers[routerKey] = &router + return routeConf, routeCondition } routeServiceKey := routerKey + "-wrr" @@ -168,12 +168,12 @@ func (p *Provider) loadTCPRoute(client Client, listener gatewayListener, route * routeService.Weighted.Services = append(routeService.Weighted.Services, service) } - conf.TCP.Services[routeServiceKey] = routeService + routeConf.TCP.Services[routeServiceKey] = routeService router.Service = routeServiceKey - conf.TCP.Routers[routerKey] = &router + routeConf.TCP.Routers[routerKey] = &router - return routeConditions + return routeConf, routeCondition } // loadTCPServices is generating a WRR service, even when there is only one target. @@ -294,3 +294,35 @@ func loadTCPServices(client Client, namespace string, backendRefs []gatev1.Backe return wrrSvc, services, nil } + +func mergeTCPConfiguration(from, to *dynamic.Configuration) { + if from == nil || from.TCP == nil || to == nil { + return + } + + if to.TCP == nil { + to.TCP = from.TCP + return + } + + if to.TCP.Routers == nil { + to.TCP.Routers = map[string]*dynamic.TCPRouter{} + } + for routerName, router := range from.TCP.Routers { + to.TCP.Routers[routerName] = router + } + + if to.TCP.Middlewares == nil { + to.TCP.Middlewares = map[string]*dynamic.TCPMiddleware{} + } + for middlewareName, middleware := range from.TCP.Middlewares { + to.TCP.Middlewares[middlewareName] = middleware + } + + if to.TCP.Services == nil { + to.TCP.Services = map[string]*dynamic.TCPService{} + } + for serviceName, service := range from.TCP.Services { + to.TCP.Services[serviceName] = service + } +} diff --git a/pkg/provider/kubernetes/gateway/tlsroute.go b/pkg/provider/kubernetes/gateway/tlsroute.go index 1eb30c788..62183b82d 100644 --- a/pkg/provider/kubernetes/gateway/tlsroute.go +++ b/pkg/provider/kubernetes/gateway/tlsroute.go @@ -19,7 +19,8 @@ func (p *Provider) loadTLSRoutes(ctx context.Context, client Client, gatewayList logger := log.Ctx(ctx) routes, err := client.ListTLSRoutes() if err != nil { - logger.Error().Err(err).Msgf("Get TLSRoutes: %s", err) + logger.Error().Err(err).Msgf("Unable to list TLSRoute") + return } for _, route := range routes { @@ -31,44 +32,6 @@ func (p *Provider) loadTLSRoutes(ctx context.Context, client Client, gatewayList ParentRef: parentRef, ControllerName: controllerName, Conditions: []metav1.Condition{ - { - Type: string(gatev1.RouteConditionAccepted), - Status: metav1.ConditionTrue, - ObservedGeneration: route.Generation, - LastTransitionTime: metav1.Now(), - Reason: string(gatev1.RouteReasonAccepted), - }, - }, - } - - var attachedListeners bool - for _, listener := range gatewayListeners { - if !matchListener(listener, route.Namespace, parentRef) { - continue - } - - if !allowRoute(listener, route.Namespace, kindTLSRoute) { - continue - } - - hostnames, ok := findMatchingHostnames(listener.Hostname, route.Spec.Hostnames) - if !ok { - continue - } - - listener.Status.AttachedRoutes++ - attachedListeners = true - - resolveConditions := p.loadTLSRoute(client, listener, route, hostnames, conf) - - // TODO: handle more accurately route conditions (in case of multiple listener matching). - for _, condition := range resolveConditions { - parentStatus.Conditions = appendCondition(parentStatus.Conditions, condition) - } - } - - if !attachedListeners { - parentStatus.Conditions = []metav1.Condition{ { Type: string(gatev1.RouteConditionAccepted), Status: metav1.ConditionFalse, @@ -76,7 +39,38 @@ func (p *Provider) loadTLSRoutes(ctx context.Context, client Client, gatewayList LastTransitionTime: metav1.Now(), Reason: string(gatev1.RouteReasonNoMatchingParent), }, + }, + } + + for _, listener := range gatewayListeners { + if !matchListener(listener, route.Namespace, parentRef) { + continue } + + accepted := true + if !allowRoute(listener, route.Namespace, kindTLSRoute) { + parentStatus.Conditions = updateRouteConditionAccepted(parentStatus.Conditions, string(gatev1.RouteReasonNotAllowedByListeners)) + accepted = false + } + hostnames, ok := findMatchingHostnames(listener.Hostname, route.Spec.Hostnames) + if !ok { + parentStatus.Conditions = updateRouteConditionAccepted(parentStatus.Conditions, string(gatev1.RouteReasonNoMatchingListenerHostname)) + accepted = false + } + + if accepted { + listener.Status.AttachedRoutes++ + // only consider the route attached if the listener is in an "attached" state. + if listener.Attached { + parentStatus.Conditions = updateRouteConditionAccepted(parentStatus.Conditions, string(gatev1.RouteReasonAccepted)) + } + } + + routeConf, resolveRefCondition := p.loadTLSRoute(client, listener, route, hostnames) + if accepted && listener.Attached { + mergeTCPConfiguration(routeConf, conf) + } + parentStatus.Conditions = upsertRouteConditionResolvedRefs(parentStatus.Conditions, resolveRefCondition) } parentStatuses = append(parentStatuses, *parentStatus) @@ -95,23 +89,30 @@ func (p *Provider) loadTLSRoutes(ctx context.Context, client Client, gatewayList } } -func (p *Provider) loadTLSRoute(client Client, listener gatewayListener, route *gatev1alpha2.TLSRoute, hostnames []gatev1.Hostname, conf *dynamic.Configuration) []metav1.Condition { - routeConditions := []metav1.Condition{ - { - Type: string(gatev1.RouteConditionResolvedRefs), - Status: metav1.ConditionTrue, - ObservedGeneration: route.Generation, - LastTransitionTime: metav1.Now(), - Reason: string(gatev1.RouteConditionResolvedRefs), +func (p *Provider) loadTLSRoute(client Client, listener gatewayListener, route *gatev1alpha2.TLSRoute, hostnames []gatev1.Hostname) (*dynamic.Configuration, metav1.Condition) { + routeConf := &dynamic.Configuration{ + TCP: &dynamic.TCPConfiguration{ + Routers: make(map[string]*dynamic.TCPRouter), + Middlewares: make(map[string]*dynamic.TCPMiddleware), + Services: make(map[string]*dynamic.TCPService), + ServersTransports: make(map[string]*dynamic.TCPServersTransport), }, } + routeCondition := metav1.Condition{ + Type: string(gatev1.RouteConditionResolvedRefs), + Status: metav1.ConditionTrue, + ObservedGeneration: route.Generation, + LastTransitionTime: metav1.Now(), + Reason: string(gatev1.RouteConditionResolvedRefs), + } + router := dynamic.TCPRouter{ RuleSyntax: "v3", Rule: hostSNIRule(hostnames), EntryPoints: []string{listener.EPName}, TLS: &dynamic.RouterTCPTLSConfig{ - Passthrough: listener.TLS.Mode != nil && *listener.TLS.Mode == gatev1.TLSModePassthrough, + Passthrough: listener.TLS != nil && listener.TLS.Mode != nil && *listener.TLS.Mode == gatev1.TLSModePassthrough, }, } @@ -130,32 +131,32 @@ func (p *Provider) loadTLSRoute(client Client, listener gatewayListener, route * wrrService, subServices, err := loadTCPServices(client, route.Namespace, routeRule.BackendRefs) if err != nil { // update "ResolvedRefs" status true with "InvalidBackendRefs" reason - routeConditions = appendCondition(routeConditions, metav1.Condition{ + routeCondition = metav1.Condition{ Type: string(gatev1.RouteConditionResolvedRefs), Status: metav1.ConditionFalse, ObservedGeneration: route.Generation, LastTransitionTime: metav1.Now(), Reason: string(gatev1.RouteReasonBackendNotFound), Message: fmt.Sprintf("Cannot load TLSRoute service %s/%s: %v", route.Namespace, route.Name, err), - }) + } continue } for svcName, svc := range subServices { - conf.TCP.Services[svcName] = svc + routeConf.TCP.Services[svcName] = svc } serviceName := fmt.Sprintf("%s-wrr-%d", routerKey, i) - conf.TCP.Services[serviceName] = wrrService + routeConf.TCP.Services[serviceName] = wrrService ruleServiceNames = append(ruleServiceNames, serviceName) } if len(ruleServiceNames) == 1 { router.Service = ruleServiceNames[0] - conf.TCP.Routers[routerKey] = &router + routeConf.TCP.Routers[routerKey] = &router - return routeConditions + return routeConf, routeCondition } routeServiceKey := routerKey + "-wrr" @@ -168,12 +169,12 @@ func (p *Provider) loadTLSRoute(client Client, listener gatewayListener, route * routeService.Weighted.Services = append(routeService.Weighted.Services, service) } - conf.TCP.Services[routeServiceKey] = routeService + routeConf.TCP.Services[routeServiceKey] = routeService router.Service = routeServiceKey - conf.TCP.Routers[routerKey] = &router + routeConf.TCP.Routers[routerKey] = &router - return routeConditions + return routeConf, routeCondition } func hostSNIRule(hostnames []gatev1.Hostname) string {