From 618fb5f2321ac3ae33ec43ed7d79c55f4aeb663e Mon Sep 17 00:00:00 2001 From: Baptiste Mayelle Date: Mon, 25 Mar 2024 14:38:04 +0100 Subject: [PATCH] Handle middlewares in filters extension ref in gateway api provider Co-authored-by: Romain --- .../routing/providers/kubernetes-gateway.md | 62 +- pkg/config/static/static_config.go | 4 + pkg/provider/kubernetes/crd/kubernetes.go | 20 + .../kubernetes/crd/kubernetes_test.go | 90 +++ .../httproute/filter_extension_ref.yml | 57 ++ .../httproute/simple_with_TraefikService.yml | 51 ++ pkg/provider/kubernetes/gateway/kubernetes.go | 126 +++- .../kubernetes/gateway/kubernetes_test.go | 548 ++++++++++++++++-- 8 files changed, 858 insertions(+), 100 deletions(-) create mode 100644 pkg/provider/kubernetes/gateway/fixtures/httproute/filter_extension_ref.yml create mode 100644 pkg/provider/kubernetes/gateway/fixtures/httproute/simple_with_TraefikService.yml diff --git a/docs/content/routing/providers/kubernetes-gateway.md b/docs/content/routing/providers/kubernetes-gateway.md index dc024f86f..4649cf5ca 100644 --- a/docs/content/routing/providers/kubernetes-gateway.md +++ b/docs/content/routing/providers/kubernetes-gateway.md @@ -241,29 +241,49 @@ Kubernetes cluster before creating `HTTPRoute` objects. - name: api@internal group: traefik.io # [18] kind: TraefikService # [19] + - filters: # [20] + - type: ExtensionRef # [21] + extensionRef: # [22] + group: traefik.io # [23] + kind: Middleware # [24] + name: my-middleware # [25] + - type: RequestRedirect # [26] + requestRedirect: # [27] + scheme: https # [28] + statusCode: 301 # [29] ``` -| Ref | Attribute | Description | -|------|---------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| [1] | `parentRefs` | References the resources (usually Gateways) that a Route wants to be attached to. | -| [2] | `name` | Name of the referent. | -| [3] | `namespace` | Namespace of the referent. When unspecified (or empty string), this refers to the local namespace of the Route. | -| [4] | `sectionName` | Name of a section within the target resource (the Listener name). | -| [5] | `hostnames` | A set of hostname that should match against the HTTP Host header to select a HTTPRoute to process the request. | -| [6] | `rules` | A list of HTTP matchers, filters and actions. | -| [7] | `matches` | Conditions used for matching the rule against incoming HTTP requests. Each match is independent, i.e. this rule will be matched if **any** one of the matches is satisfied. | -| [8] | `path` | An HTTP request path matcher. If this field is not specified, a default prefix match on the "/" path is provided. | -| [9] | `type` | Type of match against the path Value (supported types: `Exact`, `Prefix`). | -| [10] | `value` | The value of the HTTP path to match against. | -| [11] | `headers` | Conditions to select a HTTP route by matching HTTP request headers. | -| [12] | `type` | Type of match for the HTTP request header match against the `values` (supported types: `Exact`). | -| [13] | `value` | A map of HTTP Headers to be matched. It MUST contain at least one entry. | -| [14] | `backendRefs` | Defines the backend(s) where matching requests should be sent. | -| [15] | `name` | The name of the referent service. | -| [16] | `weight` | The proportion of traffic forwarded to a targetRef, computed as weight/(sum of all weights in targetRefs). | -| [17] | `port` | The port of the referent service. | -| [18] | `group` | Group is the group of the referent. Only `traefik.io` and `gateway.networking.k8s.io` values are supported. | -| [19] | `kind` | Kind is kind of the referent. Only `TraefikService` and `Service` values are supported. | +| Ref | Attribute | Description | +|------|-------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| [1] | `parentRefs` | References the resources (usually Gateways) that a Route wants to be attached to. | +| [2] | `name` | Name of the referent. | +| [3] | `namespace` | Namespace of the referent. When unspecified (or empty string), this refers to the local namespace of the Route. | +| [4] | `sectionName` | Name of a section within the target resource (the Listener name). | +| [5] | `hostnames` | A set of hostname that should match against the HTTP Host header to select a HTTPRoute to process the request. | +| [6] | `rules` | A list of HTTP matchers, filters and actions. | +| [7] | `matches` | Conditions used for matching the rule against incoming HTTP requests. Each match is independent, i.e. this rule will be matched if **any** one of the matches is satisfied. | +| [8] | `path` | An HTTP request path matcher. If this field is not specified, a default prefix match on the "/" path is provided. | +| [9] | `type` | Type of match against the path Value (supported types: `Exact`, `Prefix`). | +| [10] | `value` | The value of the HTTP path to match against. | +| [11] | `headers` | Conditions to select a HTTP route by matching HTTP request headers. | +| [12] | `name` | Name of the HTTP header to be matched. | +| [13] | `value` | Value of HTTP Header to be matched. | +| [14] | `backendRefs` | Defines the backend(s) where matching requests should be sent. | +| [15] | `name` | The name of the referent service. | +| [16] | `weight` | The proportion of traffic forwarded to a targetRef, computed as weight/(sum of all weights in targetRefs). | +| [17] | `port` | The port of the referent service. | +| [18] | `group` | Group is the group of the referent. Only `traefik.io` and `gateway.networking.k8s.io` values are supported. | +| [19] | `kind` | Kind is kind of the referent. Only `TraefikService` and `Service` values are supported. | +| [20] | `filters` | Defines the filters (middlewares) applied to the route. | +| [21] | `type` | Defines the type of filter; ExtensionRef is used for configuring custom HTTP filters. | +| [22] | `extensionRef` | Configuration of the custom HTTP filter. | +| [23] | `group` | Group of the kubernetes object to reference. | +| [24] | `kind` | Kind of the kubernetes object to reference. | +| [25] | `name` | Name of the kubernetes object to reference. | +| [26] | `type` | Defines the type of filter; RequestRedirect redirects a request to another location. | +| [27] | `requestRedirect` | Configuration of redirect filter. | +| [28] | `scheme` | Scheme is the scheme to be used in the value of the Location header in the response. | +| [29] | `statusCode` | StatusCode is the HTTP status code to be used in response. | ### Kind: `TCPRoute` diff --git a/pkg/config/static/static_config.go b/pkg/config/static/static_config.go index 81d3401fd..01144537e 100644 --- a/pkg/config/static/static_config.go +++ b/pkg/config/static/static_config.go @@ -288,6 +288,10 @@ func (c *Configuration) SetEffectiveConfiguration() { entryPoints[epName] = gateway.Entrypoint{Address: entryPoint.GetAddress(), HasHTTPTLSConf: entryPoint.HTTP.TLS != nil} } + if c.Providers.KubernetesCRD != nil { + c.Providers.KubernetesCRD.FillExtensionBuilderRegistry(c.Providers.KubernetesGateway) + } + c.Providers.KubernetesGateway.EntryPoints = entryPoints } diff --git a/pkg/provider/kubernetes/crd/kubernetes.go b/pkg/provider/kubernetes/crd/kubernetes.go index 55ca2aba5..6c18f3024 100644 --- a/pkg/provider/kubernetes/crd/kubernetes.go +++ b/pkg/provider/kubernetes/crd/kubernetes.go @@ -12,6 +12,7 @@ import ( "fmt" "net" "os" + "slices" "sort" "strconv" "strings" @@ -26,6 +27,7 @@ import ( "github.com/traefik/traefik/v3/pkg/logs" "github.com/traefik/traefik/v3/pkg/provider" traefikv1alpha1 "github.com/traefik/traefik/v3/pkg/provider/kubernetes/crd/traefikio/v1alpha1" + "github.com/traefik/traefik/v3/pkg/provider/kubernetes/gateway" "github.com/traefik/traefik/v3/pkg/provider/kubernetes/k8s" "github.com/traefik/traefik/v3/pkg/safe" "github.com/traefik/traefik/v3/pkg/tls" @@ -712,6 +714,24 @@ func (p *Provider) createErrorPageMiddleware(client Client, namespace string, er return errorPageMiddleware, balancerServerHTTP, nil } +func (p *Provider) FillExtensionBuilderRegistry(registry gateway.ExtensionBuilderRegistry) { + registry.RegisterFilterFuncs(traefikv1alpha1.GroupName, "Middleware", func(name, namespace string) (string, *dynamic.Middleware, error) { + if len(p.Namespaces) > 0 && !slices.Contains(p.Namespaces, namespace) { + return "", nil, fmt.Errorf("namespace %q is not allowed", namespace) + } + + return makeID(namespace, name) + providerNamespaceSeparator + providerName, nil, nil + }) + + registry.RegisterBackendFuncs(traefikv1alpha1.GroupName, "TraefikService", func(name, namespace string) (string, *dynamic.Service, error) { + if len(p.Namespaces) > 0 && !slices.Contains(p.Namespaces, namespace) { + return "", nil, fmt.Errorf("namespace %q is not allowed", namespace) + } + + return makeID(namespace, name) + providerNamespaceSeparator + providerName, nil, nil + }) +} + func createForwardAuthMiddleware(k8sClient Client, namespace string, auth *traefikv1alpha1.ForwardAuth) (*dynamic.ForwardAuth, error) { if auth == nil { return nil, nil diff --git a/pkg/provider/kubernetes/crd/kubernetes_test.go b/pkg/provider/kubernetes/crd/kubernetes_test.go index c851140c9..bbf4236af 100644 --- a/pkg/provider/kubernetes/crd/kubernetes_test.go +++ b/pkg/provider/kubernetes/crd/kubernetes_test.go @@ -16,6 +16,7 @@ import ( "github.com/traefik/traefik/v3/pkg/provider" traefikcrdfake "github.com/traefik/traefik/v3/pkg/provider/kubernetes/crd/generated/clientset/versioned/fake" traefikv1alpha1 "github.com/traefik/traefik/v3/pkg/provider/kubernetes/crd/traefikio/v1alpha1" + "github.com/traefik/traefik/v3/pkg/provider/kubernetes/gateway" "github.com/traefik/traefik/v3/pkg/provider/kubernetes/k8s" "github.com/traefik/traefik/v3/pkg/tls" "github.com/traefik/traefik/v3/pkg/types" @@ -7082,6 +7083,64 @@ func TestCreateBasicAuthCredentials(t *testing.T) { assert.True(t, auth.CheckSecret("test2", hashedPassword)) } +func TestFillExtensionBuilderRegistry(t *testing.T) { + testCases := []struct { + desc string + namespaces []string + wantErr require.ErrorAssertionFunc + }{ + { + desc: "no filter on namespaces", + wantErr: require.NoError, + }, + { + desc: "filter on default namespace", + namespaces: []string{"default"}, + wantErr: require.NoError, + }, + { + desc: "filter on not-default namespace", + namespaces: []string{"not-default"}, + wantErr: require.Error, + }, + } + + for _, test := range testCases { + test := test + + t.Run(test.desc, func(t *testing.T) { + t.Parallel() + + r := &extensionBuilderRegistryMock{} + + p := Provider{Namespaces: test.namespaces} + p.FillExtensionBuilderRegistry(r) + + filterFunc, ok := r.groupKindFilterFuncs[traefikv1alpha1.SchemeGroupVersion.Group]["Middleware"] + require.True(t, ok) + + name, conf, err := filterFunc("my-middleware", "default") + test.wantErr(t, err) + + if err == nil { + assert.Nil(t, conf) + assert.Equal(t, "default-my-middleware@kubernetescrd", name) + } + + backendFunc, ok := r.groupKindBackendFuncs[traefikv1alpha1.SchemeGroupVersion.Group]["TraefikService"] + require.True(t, ok) + + name, svc, err := backendFunc("my-service", "default") + test.wantErr(t, err) + + if err == nil { + assert.Nil(t, svc) + assert.Equal(t, "default-my-service@kubernetescrd", name) + } + }) + } +} + func readResources(t *testing.T, paths []string) ([]runtime.Object, []runtime.Object) { t.Helper() @@ -7106,3 +7165,34 @@ func readResources(t *testing.T, paths []string) ([]runtime.Object, []runtime.Ob return k8sObjects, crdObjects } + +type extensionBuilderRegistryMock struct { + groupKindFilterFuncs map[string]map[string]gateway.BuildFilterFunc + groupKindBackendFuncs map[string]map[string]gateway.BuildBackendFunc +} + +// RegisterFilterFuncs registers an allowed Group, Kind, and builder for the Filter ExtensionRef objects. +func (p *extensionBuilderRegistryMock) RegisterFilterFuncs(group, kind string, builderFunc gateway.BuildFilterFunc) { + if p.groupKindFilterFuncs == nil { + p.groupKindFilterFuncs = map[string]map[string]gateway.BuildFilterFunc{} + } + + if p.groupKindFilterFuncs[group] == nil { + p.groupKindFilterFuncs[group] = map[string]gateway.BuildFilterFunc{} + } + + p.groupKindFilterFuncs[group][kind] = builderFunc +} + +// RegisterBackendFuncs registers an allowed Group, Kind, and builder for the Backend ExtensionRef objects. +func (p *extensionBuilderRegistryMock) RegisterBackendFuncs(group, kind string, builderFunc gateway.BuildBackendFunc) { + if p.groupKindBackendFuncs == nil { + p.groupKindBackendFuncs = map[string]map[string]gateway.BuildBackendFunc{} + } + + if p.groupKindBackendFuncs[group] == nil { + p.groupKindBackendFuncs[group] = map[string]gateway.BuildBackendFunc{} + } + + p.groupKindBackendFuncs[group][kind] = builderFunc +} diff --git a/pkg/provider/kubernetes/gateway/fixtures/httproute/filter_extension_ref.yml b/pkg/provider/kubernetes/gateway/fixtures/httproute/filter_extension_ref.yml new file mode 100644 index 000000000..c377fd139 --- /dev/null +++ b/pkg/provider/kubernetes/gateway/fixtures/httproute/filter_extension_ref.yml @@ -0,0 +1,57 @@ +--- +kind: GatewayClass +apiVersion: gateway.networking.k8s.io/v1 +metadata: + name: my-gateway-class +spec: + controllerName: traefik.io/gateway-controller + +--- +kind: Gateway +apiVersion: gateway.networking.k8s.io/v1 +metadata: + name: my-gateway + namespace: default +spec: + gatewayClassName: my-gateway-class + listeners: # Use GatewayClass defaults for listener definition. + - name: http + protocol: HTTP + port: 80 + allowedRoutes: + kinds: + - kind: HTTPRoute + group: gateway.networking.k8s.io + namespaces: + from: Same + +--- +kind: HTTPRoute +apiVersion: gateway.networking.k8s.io/v1 +metadata: + name: http-app-1 + namespace: default +spec: + parentRefs: + - name: my-gateway + kind: Gateway + group: gateway.networking.k8s.io + hostnames: + - "foo.com" + rules: + - matches: + - path: + type: Exact + value: /bar + backendRefs: + - name: whoami + port: 80 + weight: 1 + kind: Service + group: "" + filters: + - type: ExtensionRef + extensionRef: + group: traefik.io + kind: Middleware + name: my-middleware diff --git a/pkg/provider/kubernetes/gateway/fixtures/httproute/simple_with_TraefikService.yml b/pkg/provider/kubernetes/gateway/fixtures/httproute/simple_with_TraefikService.yml new file mode 100644 index 000000000..f7af0e535 --- /dev/null +++ b/pkg/provider/kubernetes/gateway/fixtures/httproute/simple_with_TraefikService.yml @@ -0,0 +1,51 @@ +--- +kind: GatewayClass +apiVersion: gateway.networking.k8s.io/v1 +metadata: + name: my-gateway-class +spec: + controllerName: traefik.io/gateway-controller + +--- +kind: Gateway +apiVersion: gateway.networking.k8s.io/v1 +metadata: + name: my-gateway + namespace: default +spec: + gatewayClassName: my-gateway-class + listeners: # Use GatewayClass defaults for listener definition. + - name: http + protocol: HTTP + port: 80 + allowedRoutes: + kinds: + - kind: HTTPRoute + group: gateway.networking.k8s.io + namespaces: + from: Same + +--- +kind: HTTPRoute +apiVersion: gateway.networking.k8s.io/v1 +metadata: + name: http-app-1 + namespace: default +spec: + parentRefs: + - name: my-gateway + kind: Gateway + group: gateway.networking.k8s.io + hostnames: + - "foo.com" + rules: + - matches: + - path: + type: Exact + value: /bar + backendRefs: + - name: whoami + port: 80 + weight: 1 + kind: TraefikService + group: "traefik.io" diff --git a/pkg/provider/kubernetes/gateway/kubernetes.go b/pkg/provider/kubernetes/gateway/kubernetes.go index 56cb4f952..360ae1b8d 100644 --- a/pkg/provider/kubernetes/gateway/kubernetes.go +++ b/pkg/provider/kubernetes/gateway/kubernetes.go @@ -59,11 +59,53 @@ type Provider struct { ThrottleDuration ptypes.Duration `description:"Kubernetes refresh throttle duration" json:"throttleDuration,omitempty" toml:"throttleDuration,omitempty" yaml:"throttleDuration,omitempty" export:"true"` EntryPoints map[string]Entrypoint `json:"-" toml:"-" yaml:"-" label:"-" file:"-"` + // groupKindFilterFuncs is the list of allowed Group and Kinds for the Filter ExtensionRef objects. + groupKindFilterFuncs map[string]map[string]BuildFilterFunc + // groupKindBackendFuncs is the list of allowed Group and Kinds for the Backend ExtensionRef objects. + groupKindBackendFuncs map[string]map[string]BuildBackendFunc + lastConfiguration safe.Safe routerTransform k8s.RouterTransform } +// BuildFilterFunc returns the name of the filter and the related dynamic.Middleware if needed. +type BuildFilterFunc func(name, namespace string) (string, *dynamic.Middleware, error) + +// BuildBackendFunc returns the name of the backend and the related dynamic.Service if needed. +type BuildBackendFunc func(name, namespace string) (string, *dynamic.Service, error) + +type ExtensionBuilderRegistry interface { + RegisterFilterFuncs(group, kind string, builderFunc BuildFilterFunc) + RegisterBackendFuncs(group, kind string, builderFunc BuildBackendFunc) +} + +// RegisterFilterFuncs registers an allowed Group, Kind, and builder for the Filter ExtensionRef objects. +func (p *Provider) RegisterFilterFuncs(group, kind string, builderFunc BuildFilterFunc) { + if p.groupKindFilterFuncs == nil { + p.groupKindFilterFuncs = map[string]map[string]BuildFilterFunc{} + } + + if p.groupKindFilterFuncs[group] == nil { + p.groupKindFilterFuncs[group] = map[string]BuildFilterFunc{} + } + + p.groupKindFilterFuncs[group][kind] = builderFunc +} + +// RegisterBackendFuncs registers an allowed Group, Kind, and builder for the Backend ExtensionRef objects. +func (p *Provider) RegisterBackendFuncs(group, kind string, builderFunc BuildBackendFunc) { + if p.groupKindBackendFuncs == nil { + p.groupKindBackendFuncs = map[string]map[string]BuildBackendFunc{} + } + + if p.groupKindBackendFuncs[group] == nil { + p.groupKindBackendFuncs[group] = map[string]BuildBackendFunc{} + } + + p.groupKindBackendFuncs[group][kind] = builderFunc +} + func (p *Provider) SetRouterTransform(routerTransform k8s.RouterTransform) { p.routerTransform = routerTransform } @@ -847,7 +889,7 @@ func (p *Provider) gatewayHTTPRouteToHTTPConf(ctx context.Context, ep string, li continue } - middlewares, err := loadMiddlewares(listener, routerKey, routeRule.Filters) + middlewares, err := p.loadMiddlewares(listener, route.Namespace, routerKey, routeRule.Filters) if err != nil { // update "ResolvedRefs" status true with "InvalidFilters" reason conditions = append(conditions, metav1.Condition{ @@ -864,7 +906,11 @@ func (p *Provider) gatewayHTTPRouteToHTTPConf(ctx context.Context, ep string, li } for middlewareName, middleware := range middlewares { - conf.HTTP.Middlewares[middlewareName] = middleware + // If the middleware is not defined in the return of the loadMiddlewares function, it means we just need a reference to that middleware. + if middleware != nil { + conf.HTTP.Middlewares[middlewareName] = middleware + } + router.Middlewares = append(router.Middlewares, middlewareName) } @@ -876,7 +922,7 @@ func (p *Provider) gatewayHTTPRouteToHTTPConf(ctx context.Context, ep string, li if len(routeRule.BackendRefs) == 1 && isInternalService(routeRule.BackendRefs[0].BackendRef) { router.Service = string(routeRule.BackendRefs[0].Name) } else { - wrrService, subServices, err := loadServices(client, route.Namespace, routeRule.BackendRefs) + wrrService, subServices, err := p.loadServices(client, route.Namespace, routeRule.BackendRefs) if err != nil { // update "ResolvedRefs" status true with "DroppedRoutes" reason conditions = append(conditions, metav1.Condition{ @@ -893,7 +939,9 @@ func (p *Provider) gatewayHTTPRouteToHTTPConf(ctx context.Context, ep string, li } for svcName, svc := range subServices { - conf.HTTP.Services[svcName] = svc + if svc != nil { + conf.HTTP.Services[svcName] = svc + } } serviceName := provider.Normalize(routerKey + "-wrr") @@ -1550,7 +1598,7 @@ func getCertificateBlocks(secret *corev1.Secret, namespace, secretName string) ( } // loadServices is generating a WRR service, even when there is only one target. -func loadServices(client Client, namespace string, backendRefs []gatev1.HTTPBackendRef) (*dynamic.Service, map[string]*dynamic.Service, error) { +func (p *Provider) loadServices(client Client, namespace string, backendRefs []gatev1.HTTPBackendRef) (*dynamic.Service, map[string]*dynamic.Service, error) { services := map[string]*dynamic.Service{} wrrSvc := &dynamic.Service{ @@ -1571,13 +1619,20 @@ func loadServices(client Client, namespace string, backendRefs []gatev1.HTTPBack weight := int(ptr.Deref(backendRef.Weight, 1)) - if isTraefikService(backendRef.BackendRef) { - wrrSvc.Weighted.Services = append(wrrSvc.Weighted.Services, dynamic.WRRService{Name: string(backendRef.Name), Weight: &weight}) - continue - } - if *backendRef.Group != "" && *backendRef.Group != groupCore && *backendRef.Kind != "Service" { - return nil, nil, fmt.Errorf("unsupported HTTPBackendRef %s/%s/%s", *backendRef.Group, *backendRef.Kind, backendRef.Name) + if backendRef.Namespace != nil && string(*backendRef.Namespace) != namespace { + // TODO: support backend reference grant. + return nil, nil, fmt.Errorf("unsupported HTTPBackendRef %s/%s/%s", *backendRef.Group, *backendRef.Kind, backendRef.Name) + } + + name, service, err := p.loadHTTPBackendRef(namespace, backendRef) + if err != nil { + return nil, nil, fmt.Errorf("unable to load HTTPBackendRef %s/%s/%s: %w", *backendRef.Group, *backendRef.Kind, backendRef.Name, err) + } + + services[name] = service + wrrSvc.Weighted.Services = append(wrrSvc.Weighted.Services, dynamic.WRRService{Name: name, Weight: &weight}) + continue } lb := &dynamic.ServersLoadBalancer{} @@ -1672,6 +1727,24 @@ func loadServices(client Client, namespace string, backendRefs []gatev1.HTTPBack return wrrSvc, services, nil } +func (p *Provider) loadHTTPBackendRef(namespace string, backendRef gatev1.HTTPBackendRef) (string, *dynamic.Service, error) { + // Support for cross-provider references (e.g: api@internal). + // This provides the same behavior as for IngressRoutes. + if *backendRef.Kind == "TraefikService" && strings.Contains(string(backendRef.Name), "@") { + return string(backendRef.Name), nil, nil + } + + backendFunc, ok := p.groupKindBackendFuncs[string(*backendRef.Group)][string(*backendRef.Kind)] + if !ok { + return "", nil, fmt.Errorf("unsupported HTTPBackendRef %s/%s/%s", *backendRef.Group, *backendRef.Kind, backendRef.Name) + } + if backendFunc == nil { + return "", nil, fmt.Errorf("undefined backendFunc for HTTPBackendRef %s/%s/%s", *backendRef.Group, *backendRef.Kind, backendRef.Name) + } + + return backendFunc(string(backendRef.Name), namespace) +} + // loadTCPServices is generating a WRR service, even when there is only one target. func loadTCPServices(client Client, namespace string, backendRefs []gatev1.BackendRef) (*dynamic.TCPService, map[string]*dynamic.TCPService, error) { services := map[string]*dynamic.TCPService{} @@ -1791,7 +1864,7 @@ func loadTCPServices(client Client, namespace string, backendRefs []gatev1.Backe return wrrSvc, services, nil } -func loadMiddlewares(listener gatev1.Listener, prefix string, filters []gatev1.HTTPRouteFilter) (map[string]*dynamic.Middleware, error) { +func (p *Provider) loadMiddlewares(listener gatev1.Listener, namespace string, prefix string, filters []gatev1.HTTPRouteFilter) (map[string]*dynamic.Middleware, error) { middlewares := make(map[string]*dynamic.Middleware) // The spec allows for an empty string in which case we should use the @@ -1815,6 +1888,16 @@ func loadMiddlewares(listener gatev1.Listener, prefix string, filters []gatev1.H if err != nil { return nil, fmt.Errorf("creating RedirectRegex middleware: %w", err) } + + middlewareName := provider.Normalize(fmt.Sprintf("%s-%s-%d", prefix, strings.ToLower(string(filter.Type)), i)) + middlewares[middlewareName] = middleware + case gatev1.HTTPRouteFilterExtensionRef: + name, middleware, err := p.loadHTTPRouteFilterExtensionRef(namespace, filter.ExtensionRef) + if err != nil { + return nil, fmt.Errorf("unsupported filter %s: %w", filter.Type, err) + } + + middlewares[name] = middleware default: // As per the spec: // https://gateway-api.sigs.k8s.io/api-types/httproute/#filters-optional @@ -1823,14 +1906,27 @@ func loadMiddlewares(listener gatev1.Listener, prefix string, filters []gatev1.H // status. return nil, fmt.Errorf("unsupported filter %s", filter.Type) } - - middlewareName := provider.Normalize(fmt.Sprintf("%s-%s-%d", prefix, strings.ToLower(string(filter.Type)), i)) - middlewares[middlewareName] = middleware } return middlewares, nil } +func (p *Provider) loadHTTPRouteFilterExtensionRef(namespace string, extensionRef *gatev1.LocalObjectReference) (string, *dynamic.Middleware, error) { + if extensionRef == nil { + return "", nil, errors.New("filter extension ref undefined") + } + + filterFunc, ok := p.groupKindFilterFuncs[string(extensionRef.Group)][string(extensionRef.Kind)] + if !ok { + return "", nil, fmt.Errorf("unsupported filter extension ref %s/%s/%s", extensionRef.Group, extensionRef.Kind, extensionRef.Name) + } + if filterFunc == nil { + return "", nil, fmt.Errorf("undefined filterFunc for filter extension ref %s/%s/%s", extensionRef.Group, extensionRef.Kind, extensionRef.Name) + } + + return filterFunc(string(extensionRef.Name), namespace) +} + func createRedirectRegexMiddleware(scheme string, filter *gatev1.HTTPRequestRedirectFilter) (*dynamic.Middleware, error) { // Use the HTTPRequestRedirectFilter scheme if defined. filterScheme := scheme diff --git a/pkg/provider/kubernetes/gateway/kubernetes_test.go b/pkg/provider/kubernetes/gateway/kubernetes_test.go index d5aeca18b..8146523f9 100644 --- a/pkg/provider/kubernetes/gateway/kubernetes_test.go +++ b/pkg/provider/kubernetes/gateway/kubernetes_test.go @@ -2,6 +2,7 @@ package gateway import ( "context" + "errors" "testing" "time" @@ -10,6 +11,7 @@ import ( ptypes "github.com/traefik/paerser/types" "github.com/traefik/traefik/v3/pkg/config/dynamic" "github.com/traefik/traefik/v3/pkg/provider" + traefikv1alpha1 "github.com/traefik/traefik/v3/pkg/provider/kubernetes/crd/traefikio/v1alpha1" "github.com/traefik/traefik/v3/pkg/tls" "github.com/traefik/traefik/v3/pkg/types" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -25,6 +27,7 @@ func TestLoadHTTPRoutes(t *testing.T) { desc string ingressClass string paths []string + namespaces []string expected *dynamic.Configuration entryPoints map[string]Entrypoint }{ @@ -621,70 +624,6 @@ func TestLoadHTTPRoutes(t *testing.T) { TLS: &dynamic.TLSConfiguration{}, }, }, - { - desc: "Simple HTTPRoute, with myservice@file service", - paths: []string{"services.yml", "httproute/simple_cross_provider.yml"}, - entryPoints: map[string]Entrypoint{"web": { - Address: ":80", - }}, - expected: &dynamic.Configuration{ - UDP: &dynamic.UDPConfiguration{ - Routers: map[string]*dynamic.UDPRouter{}, - Services: map[string]*dynamic.UDPService{}, - }, - TCP: &dynamic.TCPConfiguration{ - Routers: map[string]*dynamic.TCPRouter{}, - Middlewares: map[string]*dynamic.TCPMiddleware{}, - Services: map[string]*dynamic.TCPService{}, - ServersTransports: map[string]*dynamic.TCPServersTransport{}, - }, - HTTP: &dynamic.HTTPConfiguration{ - Routers: map[string]*dynamic.Router{ - "default-http-app-1-my-gateway-web-1c0cf64bde37d9d0df06": { - EntryPoints: []string{"web"}, - Service: "default-http-app-1-my-gateway-web-1c0cf64bde37d9d0df06-wrr", - Rule: "Host(`foo.com`) && Path(`/bar`)", - RuleSyntax: "v3", - }, - }, - Middlewares: map[string]*dynamic.Middleware{}, - Services: map[string]*dynamic.Service{ - "default-http-app-1-my-gateway-web-1c0cf64bde37d9d0df06-wrr": { - Weighted: &dynamic.WeightedRoundRobin{ - Services: []dynamic.WRRService{ - { - Name: "service@file", - Weight: func(i int) *int { return &i }(1), - }, - { - Name: "default-whoami-80", - Weight: func(i int) *int { return &i }(1), - }, - }, - }, - }, - "default-whoami-80": { - LoadBalancer: &dynamic.ServersLoadBalancer{ - Servers: []dynamic.Server{ - { - URL: "http://10.10.0.1:80", - }, - { - URL: "http://10.10.0.2:80", - }, - }, - PassHostHeader: ptr.To(true), - ResponseForwarding: &dynamic.ResponseForwarding{ - FlushInterval: ptypes.Duration(100 * time.Millisecond), - }, - }, - }, - }, - ServersTransports: map[string]*dynamic.ServersTransport{}, - }, - TLS: &dynamic.TLSConfiguration{}, - }, - }, { desc: "Simple HTTPRoute with protocol HTTPS", paths: []string{"services.yml", "httproute/with_protocol_https.yml"}, @@ -1726,12 +1665,493 @@ func TestLoadHTTPRoutes(t *testing.T) { } p := Provider{EntryPoints: test.entryPoints} + conf := p.loadConfigurationFromGateway(context.Background(), newClientMock(test.paths...)) assert.Equal(t, test.expected, conf) }) } } +func TestLoadHTTPRoutes_backendExtensionRef(t *testing.T) { + testCases := []struct { + desc string + paths []string + groupKindBackendFuncs map[string]map[string]BuildBackendFunc + expected *dynamic.Configuration + entryPoints map[string]Entrypoint + }{ + { + desc: "Simple HTTPRoute with TraefikService", + paths: []string{"services.yml", "httproute/simple_with_TraefikService.yml"}, + groupKindBackendFuncs: map[string]map[string]BuildBackendFunc{ + traefikv1alpha1.GroupName: {"TraefikService": func(name, namespace string) (string, *dynamic.Service, error) { + return name, nil, nil + }}, + }, + entryPoints: map[string]Entrypoint{"web": { + Address: ":80", + }}, + expected: &dynamic.Configuration{ + UDP: &dynamic.UDPConfiguration{ + Routers: map[string]*dynamic.UDPRouter{}, + Services: map[string]*dynamic.UDPService{}, + }, + TCP: &dynamic.TCPConfiguration{ + Routers: map[string]*dynamic.TCPRouter{}, + Middlewares: map[string]*dynamic.TCPMiddleware{}, + Services: map[string]*dynamic.TCPService{}, + ServersTransports: map[string]*dynamic.TCPServersTransport{}, + }, + HTTP: &dynamic.HTTPConfiguration{ + Routers: map[string]*dynamic.Router{ + "default-http-app-1-my-gateway-web-1c0cf64bde37d9d0df06": { + EntryPoints: []string{"web"}, + Service: "default-http-app-1-my-gateway-web-1c0cf64bde37d9d0df06-wrr", + Rule: "Host(`foo.com`) && Path(`/bar`)", + RuleSyntax: "v3", + }, + }, + Middlewares: map[string]*dynamic.Middleware{}, + Services: map[string]*dynamic.Service{ + "default-http-app-1-my-gateway-web-1c0cf64bde37d9d0df06-wrr": { + Weighted: &dynamic.WeightedRoundRobin{ + Services: []dynamic.WRRService{ + { + Name: "whoami", + Weight: func(i int) *int { return &i }(1), + }, + }, + }, + }, + }, + ServersTransports: map[string]*dynamic.ServersTransport{}, + }, + TLS: &dynamic.TLSConfiguration{}, + }, + }, + { + desc: "Simple HTTPRoute with TraefikService with service configuration", + paths: []string{"services.yml", "httproute/simple_with_TraefikService.yml"}, + groupKindBackendFuncs: map[string]map[string]BuildBackendFunc{ + traefikv1alpha1.GroupName: {"TraefikService": func(name, namespace string) (string, *dynamic.Service, error) { + return name, &dynamic.Service{LoadBalancer: &dynamic.ServersLoadBalancer{Servers: []dynamic.Server{{URL: "foobar"}}}}, nil + }}, + }, + entryPoints: map[string]Entrypoint{"web": { + Address: ":80", + }}, + expected: &dynamic.Configuration{ + UDP: &dynamic.UDPConfiguration{ + Routers: map[string]*dynamic.UDPRouter{}, + Services: map[string]*dynamic.UDPService{}, + }, + TCP: &dynamic.TCPConfiguration{ + Routers: map[string]*dynamic.TCPRouter{}, + Middlewares: map[string]*dynamic.TCPMiddleware{}, + Services: map[string]*dynamic.TCPService{}, + ServersTransports: map[string]*dynamic.TCPServersTransport{}, + }, + HTTP: &dynamic.HTTPConfiguration{ + Routers: map[string]*dynamic.Router{ + "default-http-app-1-my-gateway-web-1c0cf64bde37d9d0df06": { + EntryPoints: []string{"web"}, + Service: "default-http-app-1-my-gateway-web-1c0cf64bde37d9d0df06-wrr", + Rule: "Host(`foo.com`) && Path(`/bar`)", + RuleSyntax: "v3", + }, + }, + Middlewares: map[string]*dynamic.Middleware{}, + Services: map[string]*dynamic.Service{ + "default-http-app-1-my-gateway-web-1c0cf64bde37d9d0df06-wrr": { + Weighted: &dynamic.WeightedRoundRobin{ + Services: []dynamic.WRRService{ + { + Name: "whoami", + Weight: func(i int) *int { return &i }(1), + }, + }, + }, + }, + "whoami": { + LoadBalancer: &dynamic.ServersLoadBalancer{ + Servers: []dynamic.Server{ + {URL: "foobar"}, + }, + }, + }, + }, + ServersTransports: map[string]*dynamic.ServersTransport{}, + }, + TLS: &dynamic.TLSConfiguration{}, + }, + }, + { + desc: "Simple HTTPRoute with invalid TraefikService kind", + paths: []string{"services.yml", "httproute/simple_with_TraefikService.yml"}, + entryPoints: map[string]Entrypoint{"web": { + Address: ":80", + }}, + expected: &dynamic.Configuration{ + UDP: &dynamic.UDPConfiguration{ + Routers: map[string]*dynamic.UDPRouter{}, + Services: map[string]*dynamic.UDPService{}, + }, + TCP: &dynamic.TCPConfiguration{ + Routers: map[string]*dynamic.TCPRouter{}, + Middlewares: map[string]*dynamic.TCPMiddleware{}, + Services: map[string]*dynamic.TCPService{}, + ServersTransports: map[string]*dynamic.TCPServersTransport{}, + }, + HTTP: &dynamic.HTTPConfiguration{ + Routers: map[string]*dynamic.Router{}, + Middlewares: map[string]*dynamic.Middleware{}, + Services: map[string]*dynamic.Service{}, + ServersTransports: map[string]*dynamic.ServersTransport{}, + }, + TLS: &dynamic.TLSConfiguration{}, + }, + }, + { + desc: "Simple HTTPRoute with backendFunc error", + paths: []string{"services.yml", "httproute/simple_with_TraefikService.yml"}, + groupKindBackendFuncs: map[string]map[string]BuildBackendFunc{ + traefikv1alpha1.GroupName: {"TraefikService": func(name, namespace string) (string, *dynamic.Service, error) { + return "", nil, errors.New("BOOM") + }}, + }, + entryPoints: map[string]Entrypoint{"web": { + Address: ":80", + }}, + expected: &dynamic.Configuration{ + UDP: &dynamic.UDPConfiguration{ + Routers: map[string]*dynamic.UDPRouter{}, + Services: map[string]*dynamic.UDPService{}, + }, + TCP: &dynamic.TCPConfiguration{ + Routers: map[string]*dynamic.TCPRouter{}, + Middlewares: map[string]*dynamic.TCPMiddleware{}, + Services: map[string]*dynamic.TCPService{}, + ServersTransports: map[string]*dynamic.TCPServersTransport{}, + }, + HTTP: &dynamic.HTTPConfiguration{ + Routers: map[string]*dynamic.Router{}, + Middlewares: map[string]*dynamic.Middleware{}, + Services: map[string]*dynamic.Service{}, + ServersTransports: map[string]*dynamic.ServersTransport{}, + }, + TLS: &dynamic.TLSConfiguration{}, + }, + }, + { + desc: "Simple HTTPRoute, with myservice@file service", + paths: []string{"services.yml", "httproute/simple_cross_provider.yml"}, + groupKindBackendFuncs: map[string]map[string]BuildBackendFunc{ + traefikv1alpha1.GroupName: {"TraefikService": func(name, namespace string) (string, *dynamic.Service, error) { + // func should never be executed in case of cross-provider reference. + return "", nil, errors.New("BOOM") + }}, + }, + entryPoints: map[string]Entrypoint{"web": { + Address: ":80", + }}, + expected: &dynamic.Configuration{ + UDP: &dynamic.UDPConfiguration{ + Routers: map[string]*dynamic.UDPRouter{}, + Services: map[string]*dynamic.UDPService{}, + }, + TCP: &dynamic.TCPConfiguration{ + Routers: map[string]*dynamic.TCPRouter{}, + Middlewares: map[string]*dynamic.TCPMiddleware{}, + Services: map[string]*dynamic.TCPService{}, + ServersTransports: map[string]*dynamic.TCPServersTransport{}, + }, + HTTP: &dynamic.HTTPConfiguration{ + Routers: map[string]*dynamic.Router{ + "default-http-app-1-my-gateway-web-1c0cf64bde37d9d0df06": { + EntryPoints: []string{"web"}, + Service: "default-http-app-1-my-gateway-web-1c0cf64bde37d9d0df06-wrr", + Rule: "Host(`foo.com`) && Path(`/bar`)", + RuleSyntax: "v3", + }, + }, + Middlewares: map[string]*dynamic.Middleware{}, + Services: map[string]*dynamic.Service{ + "default-http-app-1-my-gateway-web-1c0cf64bde37d9d0df06-wrr": { + Weighted: &dynamic.WeightedRoundRobin{ + Services: []dynamic.WRRService{ + { + Name: "service@file", + Weight: func(i int) *int { return &i }(1), + }, + { + Name: "default-whoami-80", + Weight: func(i int) *int { return &i }(1), + }, + }, + }, + }, + "default-whoami-80": { + LoadBalancer: &dynamic.ServersLoadBalancer{ + Servers: []dynamic.Server{ + { + URL: "http://10.10.0.1:80", + }, + { + URL: "http://10.10.0.2:80", + }, + }, + PassHostHeader: ptr.To(true), + ResponseForwarding: &dynamic.ResponseForwarding{ + FlushInterval: ptypes.Duration(100 * time.Millisecond), + }, + }, + }, + }, + ServersTransports: map[string]*dynamic.ServersTransport{}, + }, + TLS: &dynamic.TLSConfiguration{}, + }, + }, + } + + for _, test := range testCases { + test := test + t.Run(test.desc, func(t *testing.T) { + t.Parallel() + + if test.expected == nil { + return + } + + p := Provider{EntryPoints: test.entryPoints} + for group, kindFuncs := range test.groupKindBackendFuncs { + for kind, backendFunc := range kindFuncs { + p.RegisterBackendFuncs(group, kind, backendFunc) + } + } + conf := p.loadConfigurationFromGateway(context.Background(), newClientMock(test.paths...)) + assert.Equal(t, test.expected, conf) + }) + } +} + +func TestLoadHTTPRoutes_filterExtensionRef(t *testing.T) { + testCases := []struct { + desc string + groupKindFilterFuncs map[string]map[string]BuildFilterFunc + expected *dynamic.Configuration + entryPoints map[string]Entrypoint + }{ + { + desc: "HTTPRoute with ExtensionRef filter", + groupKindFilterFuncs: map[string]map[string]BuildFilterFunc{ + traefikv1alpha1.GroupName: {"Middleware": func(name, namespace string) (string, *dynamic.Middleware, error) { + return namespace + "-" + name, nil, nil + }}, + }, + entryPoints: map[string]Entrypoint{"web": { + Address: ":80", + }}, + expected: &dynamic.Configuration{ + UDP: &dynamic.UDPConfiguration{ + Routers: map[string]*dynamic.UDPRouter{}, + Services: map[string]*dynamic.UDPService{}, + }, + TCP: &dynamic.TCPConfiguration{ + Routers: map[string]*dynamic.TCPRouter{}, + Middlewares: map[string]*dynamic.TCPMiddleware{}, + Services: map[string]*dynamic.TCPService{}, + ServersTransports: map[string]*dynamic.TCPServersTransport{}, + }, + HTTP: &dynamic.HTTPConfiguration{ + Routers: map[string]*dynamic.Router{ + "default-http-app-1-my-gateway-web-1c0cf64bde37d9d0df06": { + EntryPoints: []string{"web"}, + Service: "default-http-app-1-my-gateway-web-1c0cf64bde37d9d0df06-wrr", + Rule: "Host(`foo.com`) && Path(`/bar`)", + RuleSyntax: "v3", + Middlewares: []string{"default-my-middleware"}, + }, + }, + Middlewares: map[string]*dynamic.Middleware{}, + Services: map[string]*dynamic.Service{ + "default-http-app-1-my-gateway-web-1c0cf64bde37d9d0df06-wrr": { + Weighted: &dynamic.WeightedRoundRobin{ + Services: []dynamic.WRRService{ + { + Name: "default-whoami-80", + Weight: func(i int) *int { return &i }(1), + }, + }, + }, + }, + "default-whoami-80": { + LoadBalancer: &dynamic.ServersLoadBalancer{ + Servers: []dynamic.Server{ + { + URL: "http://10.10.0.1:80", + }, + { + URL: "http://10.10.0.2:80", + }, + }, + PassHostHeader: ptr.To(true), + ResponseForwarding: &dynamic.ResponseForwarding{ + FlushInterval: ptypes.Duration(100 * time.Millisecond), + }, + }, + }, + }, + ServersTransports: map[string]*dynamic.ServersTransport{}, + }, + TLS: &dynamic.TLSConfiguration{}, + }, + }, + { + desc: "HTTPRoute with ExtensionRef filter and create middleware", + groupKindFilterFuncs: map[string]map[string]BuildFilterFunc{ + traefikv1alpha1.GroupName: {"Middleware": func(name, namespace string) (string, *dynamic.Middleware, error) { + return namespace + "-" + name, &dynamic.Middleware{Headers: &dynamic.Headers{CustomRequestHeaders: map[string]string{"Test-Header": "Test"}}}, nil + }}, + }, + entryPoints: map[string]Entrypoint{"web": { + Address: ":80", + }}, + expected: &dynamic.Configuration{ + UDP: &dynamic.UDPConfiguration{ + Routers: map[string]*dynamic.UDPRouter{}, + Services: map[string]*dynamic.UDPService{}, + }, + TCP: &dynamic.TCPConfiguration{ + Routers: map[string]*dynamic.TCPRouter{}, + Middlewares: map[string]*dynamic.TCPMiddleware{}, + Services: map[string]*dynamic.TCPService{}, + ServersTransports: map[string]*dynamic.TCPServersTransport{}, + }, + HTTP: &dynamic.HTTPConfiguration{ + Routers: map[string]*dynamic.Router{ + "default-http-app-1-my-gateway-web-1c0cf64bde37d9d0df06": { + EntryPoints: []string{"web"}, + Service: "default-http-app-1-my-gateway-web-1c0cf64bde37d9d0df06-wrr", + Rule: "Host(`foo.com`) && Path(`/bar`)", + RuleSyntax: "v3", + Middlewares: []string{"default-my-middleware"}, + }, + }, + Middlewares: map[string]*dynamic.Middleware{ + "default-my-middleware": {Headers: &dynamic.Headers{CustomRequestHeaders: map[string]string{"Test-Header": "Test"}}}, + }, + Services: map[string]*dynamic.Service{ + "default-http-app-1-my-gateway-web-1c0cf64bde37d9d0df06-wrr": { + Weighted: &dynamic.WeightedRoundRobin{ + Services: []dynamic.WRRService{ + { + Name: "default-whoami-80", + Weight: func(i int) *int { return &i }(1), + }, + }, + }, + }, + "default-whoami-80": { + LoadBalancer: &dynamic.ServersLoadBalancer{ + Servers: []dynamic.Server{ + { + URL: "http://10.10.0.1:80", + }, + { + URL: "http://10.10.0.2:80", + }, + }, + PassHostHeader: ptr.To(true), + ResponseForwarding: &dynamic.ResponseForwarding{ + FlushInterval: ptypes.Duration(100 * time.Millisecond), + }, + }, + }, + }, + ServersTransports: map[string]*dynamic.ServersTransport{}, + }, + TLS: &dynamic.TLSConfiguration{}, + }, + }, + { + desc: "ExtensionRef filter: Unknown", + entryPoints: map[string]Entrypoint{"web": { + Address: ":80", + }}, + expected: &dynamic.Configuration{ + UDP: &dynamic.UDPConfiguration{ + Routers: map[string]*dynamic.UDPRouter{}, + Services: map[string]*dynamic.UDPService{}, + }, + TCP: &dynamic.TCPConfiguration{ + Routers: map[string]*dynamic.TCPRouter{}, + Middlewares: map[string]*dynamic.TCPMiddleware{}, + Services: map[string]*dynamic.TCPService{}, + ServersTransports: map[string]*dynamic.TCPServersTransport{}, + }, + HTTP: &dynamic.HTTPConfiguration{ + Routers: map[string]*dynamic.Router{}, + Middlewares: map[string]*dynamic.Middleware{}, + Services: map[string]*dynamic.Service{}, + ServersTransports: map[string]*dynamic.ServersTransport{}, + }, + TLS: &dynamic.TLSConfiguration{}, + }, + }, + { + desc: "ExtensionRef filter with filterFunc error", + groupKindFilterFuncs: map[string]map[string]BuildFilterFunc{ + traefikv1alpha1.GroupName: {"Middleware": func(name, namespace string) (string, *dynamic.Middleware, error) { + return "", nil, errors.New("BOOM") + }}, + }, + entryPoints: map[string]Entrypoint{"web": { + Address: ":80", + }}, + expected: &dynamic.Configuration{ + UDP: &dynamic.UDPConfiguration{ + Routers: map[string]*dynamic.UDPRouter{}, + Services: map[string]*dynamic.UDPService{}, + }, + TCP: &dynamic.TCPConfiguration{ + Routers: map[string]*dynamic.TCPRouter{}, + Middlewares: map[string]*dynamic.TCPMiddleware{}, + Services: map[string]*dynamic.TCPService{}, + ServersTransports: map[string]*dynamic.TCPServersTransport{}, + }, + HTTP: &dynamic.HTTPConfiguration{ + Routers: map[string]*dynamic.Router{}, + Middlewares: map[string]*dynamic.Middleware{}, + Services: map[string]*dynamic.Service{}, + ServersTransports: map[string]*dynamic.ServersTransport{}, + }, + TLS: &dynamic.TLSConfiguration{}, + }, + }, + } + + for _, test := range testCases { + test := test + t.Run(test.desc, func(t *testing.T) { + t.Parallel() + + if test.expected == nil { + return + } + + p := Provider{EntryPoints: test.entryPoints} + for group, kindFuncs := range test.groupKindFilterFuncs { + for kind, filterFunc := range kindFuncs { + p.RegisterFilterFuncs(group, kind, filterFunc) + } + } + conf := p.loadConfigurationFromGateway(context.Background(), newClientMock([]string{"services.yml", "httproute/filter_extension_ref.yml"}...)) + assert.Equal(t, test.expected, conf) + }) + } +} + func TestLoadTCPRoutes(t *testing.T) { testCases := []struct { desc string