From 7996a42f76505c08d36982fa9c3829ff85d145ac Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Doumenjou <925513+jbdoumenjou@users.noreply.github.com> Date: Tue, 2 Feb 2021 19:36:04 +0100 Subject: [PATCH] Allow crossprovider service reference Co-authored-by: Harold Ozouf --- .../kubernetes-gateway-resource.yml | 11 +++ .../routing/providers/kubernetes-gateway.md | 70 ++++++++------- .../fixtures/simple_cross_provider.yml | 52 +++++++++++ .../fixtures/simple_to_api_internal.yml | 49 +++++++++++ pkg/provider/kubernetes/gateway/kubernetes.go | 81 +++++++++++------ .../kubernetes/gateway/kubernetes_test.go | 86 +++++++++++++++++++ 6 files changed, 293 insertions(+), 56 deletions(-) create mode 100644 pkg/provider/kubernetes/gateway/fixtures/simple_cross_provider.yml create mode 100644 pkg/provider/kubernetes/gateway/fixtures/simple_to_api_internal.yml diff --git a/docs/content/reference/dynamic-configuration/kubernetes-gateway-resource.yml b/docs/content/reference/dynamic-configuration/kubernetes-gateway-resource.yml index 821a8fccc..c25130ac5 100644 --- a/docs/content/reference/dynamic-configuration/kubernetes-gateway-resource.yml +++ b/docs/content/reference/dynamic-configuration/kubernetes-gateway-resource.yml @@ -44,3 +44,14 @@ spec: - serviceName: whoami port: 80 weight: 1 + - matches: + - path: + type: Prefix + value: /foo + forwardTo: + - backendRef: + group: traefik.containo.us + kind: TraefikService + name: myservice@file + weight: 1 + port: 80 diff --git a/docs/content/routing/providers/kubernetes-gateway.md b/docs/content/routing/providers/kubernetes-gateway.md index 03d326760..0db255824 100644 --- a/docs/content/routing/providers/kubernetes-gateway.md +++ b/docs/content/routing/providers/kubernetes-gateway.md @@ -123,39 +123,49 @@ Kubernetes cluster before creating `HTTPRoute` objects. metadata: name: http-app-1 namespace: default - labels: # [1] + labels: # [1] app: foo spec: - hostnames: # [2] + hostnames: # [2] - "whoami" - rules: # [3] - - matches: # [4] - - path: # [5] - type: Exact # [6] - value: /bar # [7] - - headers: # [8] - type: Exact # [9] - values: # [10] + rules: # [3] + - matches: # [4] + - path: # [5] + type: Exact # [6] + value: /bar # [7] + - headers: # [8] + type: Exact # [9] + values: # [10] - foo: bar - forwardTo: # [11] - - serviceName: whoami # [12] - weight: 1 # [13] - port: 80 # [14] + forwardTo: # [11] + - serviceName: whoami # [12] + weight: 1 # [13] + port: 80 # [14] + - backendRef: # [15] + group: traefik.containo.us # [16] + kind: TraefikService # [17] + name: api@internal # [18] + port: 80 + weight: 1 ``` -| Ref | Attribute | Description | -|------|---------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| [1] | `labels` | Labels to match with the `Gateway` labelselector. | -| [2] | `hostnames` | A set of hostname that should match against the HTTP Host header to select a HTTPRoute to process the request. | -| [3] | `rules` | A list of HTTP matchers, filters and actions. | -| [4] | `matches` | Conditions used for matching the rule against incoming HTTP requests. Each match is independent, i.e. this rule will be matched if **any** one of the matches is satisfied. | -| [5] | `path` | An HTTP request path matcher. If this field is not specified, a default prefix match on the "/" path is provided. | -| [6] | `type` | Type of match against the path Value (supported types: `Exact`, `Prefix`). | -| [7] | `value` | The value of the HTTP path to match against. | -| [8] | `headers` | Conditions to select a HTTP route by matching HTTP request headers. | -| [9] | `type` | Type of match for the HTTP request header match against the `values` (supported types: `Exact`). | -| [10] | `values` | A map of HTTP Headers to be matched. It MUST contain at least one entry. | -| [11] | `forwardTo` | The upstream target(s) where the request should be sent. | -| [12] | `serviceName` | The name of the referent service. | -| [13] | `weight` | The proportion of traffic forwarded to a targetRef, computed as weight/(sum of all weights in targetRefs). | -| [14] | `port` | The port of the referent service. | +| Ref | Attribute | Description | +|------|---------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| [1] | `labels` | Labels to match with the `Gateway` labelselector. | +| [2] | `hostnames` | A set of hostname that should match against the HTTP Host header to select a HTTPRoute to process the request. | +| [3] | `rules` | A list of HTTP matchers, filters and actions. | +| [4] | `matches` | Conditions used for matching the rule against incoming HTTP requests. Each match is independent, i.e. this rule will be matched if **any** one of the matches is satisfied. | +| [5] | `path` | An HTTP request path matcher. If this field is not specified, a default prefix match on the "/" path is provided. | +| [6] | `type` | Type of match against the path Value (supported types: `Exact`, `Prefix`). | +| [7] | `value` | The value of the HTTP path to match against. | +| [8] | `headers` | Conditions to select a HTTP route by matching HTTP request headers. | +| [9] | `type` | Type of match for the HTTP request header match against the `values` (supported types: `Exact`). | +| [10] | `values` | A map of HTTP Headers to be matched. It MUST contain at least one entry. | +| [11] | `forwardTo` | The upstream target(s) where the request should be sent. | +| [12] | `serviceName` | The name of the referent service. | +| [13] | `weight` | The proportion of traffic forwarded to a targetRef, computed as weight/(sum of all weights in targetRefs). | +| [14] | `port` | The port of the referent service. | +| [15] | `backendRef` | The BackendRef is a reference to a backend (API object within a known namespace) to forward matched requests to. If both BackendRef and ServiceName are specified, ServiceName will be given precedence. Only `TraefikService` is supported. | +| [16] | `group` | Group is the group of the referent. Only `traefik.containo.us` value is supported. | +| [17] | `kind` | Kind is kind of the referent. Only `TraefikService` value is supported. | +| [18] | `name` | Name is the name of the referent. | diff --git a/pkg/provider/kubernetes/gateway/fixtures/simple_cross_provider.yml b/pkg/provider/kubernetes/gateway/fixtures/simple_cross_provider.yml new file mode 100644 index 000000000..744599728 --- /dev/null +++ b/pkg/provider/kubernetes/gateway/fixtures/simple_cross_provider.yml @@ -0,0 +1,52 @@ +--- +kind: GatewayClass +apiVersion: networking.x-k8s.io/v1alpha1 +metadata: + name: my-gateway-class +spec: + controller: traefik.io/gateway-controller + +--- +kind: Gateway +apiVersion: networking.x-k8s.io/v1alpha1 +metadata: + name: my-gateway + namespace: default +spec: + gatewayClassName: my-gateway-class + listeners: # Use GatewayClass defaults for listener definition. + - protocol: HTTP + port: 80 + routes: + kind: HTTPRoute + namespaces: + from: Same + selector: + app: foo + +--- +kind: HTTPRoute +apiVersion: networking.x-k8s.io/v1alpha1 +metadata: + name: http-app-1 + namespace: default + labels: + app: foo +spec: + hostnames: + - "foo.com" + rules: + - matches: + - path: + type: Exact + value: /bar + forwardTo: + - weight: 1 + backendRef: + group: traefik.containo.us + kind: TraefikService + name: service@file + port: 80 + - serviceName: whoami + port: 80 + weight: 1 \ No newline at end of file diff --git a/pkg/provider/kubernetes/gateway/fixtures/simple_to_api_internal.yml b/pkg/provider/kubernetes/gateway/fixtures/simple_to_api_internal.yml new file mode 100644 index 000000000..b86f2773f --- /dev/null +++ b/pkg/provider/kubernetes/gateway/fixtures/simple_to_api_internal.yml @@ -0,0 +1,49 @@ +--- +kind: GatewayClass +apiVersion: networking.x-k8s.io/v1alpha1 +metadata: + name: my-gateway-class +spec: + controller: traefik.io/gateway-controller + +--- +kind: Gateway +apiVersion: networking.x-k8s.io/v1alpha1 +metadata: + name: my-gateway + namespace: default +spec: + gatewayClassName: my-gateway-class + listeners: # Use GatewayClass defaults for listener definition. + - protocol: HTTP + port: 80 + routes: + kind: HTTPRoute + namespaces: + from: Same + selector: + app: foo + +--- +kind: HTTPRoute +apiVersion: networking.x-k8s.io/v1alpha1 +metadata: + name: http-app-1 + namespace: default + labels: + app: foo +spec: + hostnames: + - "foo.com" + rules: + - matches: + - path: + type: Exact + value: /bar + forwardTo: + - weight: 1 + backendRef: + group: traefik.containo.us + kind: TraefikService + name: api@internal + port: 80 diff --git a/pkg/provider/kubernetes/gateway/kubernetes.go b/pkg/provider/kubernetes/gateway/kubernetes.go index 50490cde7..03793ee14 100644 --- a/pkg/provider/kubernetes/gateway/kubernetes.go +++ b/pkg/provider/kubernetes/gateway/kubernetes.go @@ -28,7 +28,11 @@ import ( "sigs.k8s.io/service-apis/apis/v1alpha1" ) -const providerName = "kubernetesgateway" +const ( + providerName = "kubernetesgateway" + traefikServiceKind = "TraefikService" + traefikServiceGroupName = "traefik.containo.us" +) // Provider holds configurations of the provider. type Provider struct { @@ -463,29 +467,34 @@ func (p *Provider) fillGatewayConf(client Client, gateway *v1alpha1.Gateway, con } if routeRule.ForwardTo != nil { - wrrService, subServices, err := loadServices(client, gateway.Namespace, routeRule.ForwardTo) - if err != nil { - // update "ResolvedRefs" status true with "DroppedRoutes" reason - listenerStatuses[i].Conditions = append(listenerStatuses[i].Conditions, metav1.Condition{ - Type: string(v1alpha1.ListenerConditionResolvedRefs), - Status: metav1.ConditionFalse, - LastTransitionTime: metav1.Now(), - Reason: string(v1alpha1.ListenerReasonDegradedRoutes), - Message: fmt.Sprintf("Cannot load service from HTTPRoute %s/%s : %v", gateway.Namespace, httpRoute.Name, err), - }) + // Traefik internal service can be used only if there is only one ForwardTo service reference. + if len(routeRule.ForwardTo) == 1 && isInternalService(routeRule.ForwardTo[0]) { + router.Service = routeRule.ForwardTo[0].BackendRef.Name + } else { + wrrService, subServices, err := loadServices(client, gateway.Namespace, routeRule.ForwardTo) + if err != nil { + // update "ResolvedRefs" status true with "DroppedRoutes" reason + listenerStatuses[i].Conditions = append(listenerStatuses[i].Conditions, metav1.Condition{ + Type: string(v1alpha1.ListenerConditionResolvedRefs), + Status: metav1.ConditionFalse, + LastTransitionTime: metav1.Now(), + Reason: string(v1alpha1.ListenerReasonDegradedRoutes), + Message: fmt.Sprintf("Cannot load service from HTTPRoute %s/%s : %v", gateway.Namespace, httpRoute.Name, err), + }) - // TODO update the RouteStatus condition / deduplicate conditions on listener - continue + // TODO update the RouteStatus condition / deduplicate conditions on listener + continue + } + + for svcName, svc := range subServices { + conf.HTTP.Services[svcName] = svc + } + + serviceName := provider.Normalize(routerKey + "-wrr") + conf.HTTP.Services[serviceName] = wrrService + + router.Service = serviceName } - - for svcName, svc := range subServices { - conf.HTTP.Services[svcName] = svc - } - - serviceName := provider.Normalize(routerKey + "-wrr") - conf.HTTP.Services[serviceName] = wrrService - - router.Service = serviceName } if router.Service != "" { @@ -765,6 +774,21 @@ func loadServices(client Client, namespace string, targets []v1alpha1.HTTPRouteF } for _, forwardTo := range targets { + weight := int(forwardTo.Weight) + + if forwardTo.ServiceName == nil && forwardTo.BackendRef != nil { + if !(forwardTo.BackendRef.Group == traefikServiceGroupName && forwardTo.BackendRef.Kind == traefikServiceKind) { + continue + } + + if strings.HasSuffix(forwardTo.BackendRef.Name, "@internal") { + return nil, nil, fmt.Errorf("traefik internal service %s is not allowed in a WRR loadbalancer", forwardTo.BackendRef.Name) + } + + wrrSvc.Weighted.Services = append(wrrSvc.Weighted.Services, dynamic.WRRService{Name: forwardTo.BackendRef.Name, Weight: &weight}) + continue + } + if forwardTo.ServiceName == nil { continue } @@ -775,8 +799,6 @@ func loadServices(client Client, namespace string, targets []v1alpha1.HTTPRouteF }, } - // TODO Handle BackendRefs - service, exists, err := client.GetService(namespace, *forwardTo.ServiceName) if err != nil { return nil, nil, err @@ -855,11 +877,10 @@ func loadServices(client Client, namespace string, targets []v1alpha1.HTTPRouteF serviceName := provider.Normalize(makeID(service.Namespace, service.Name) + "-" + portStr) services[serviceName] = &svc - weight := int(forwardTo.Weight) wrrSvc.Weighted.Services = append(wrrSvc.Weighted.Services, dynamic.WRRService{Name: serviceName, Weight: &weight}) } - if len(services) == 0 { + if len(wrrSvc.Weighted.Services) == 0 { return nil, nil, errors.New("no service has been created") } @@ -904,3 +925,11 @@ func throttleEvents(ctx context.Context, throttleDuration time.Duration, pool *s return eventsChanBuffered } + +func isInternalService(forwardTo v1alpha1.HTTPRouteForwardTo) bool { + return forwardTo.ServiceName == nil && + forwardTo.BackendRef != nil && + forwardTo.BackendRef.Kind == traefikServiceKind && + forwardTo.BackendRef.Group == traefikServiceGroupName && + strings.HasSuffix(forwardTo.BackendRef.Name, "@internal") +} diff --git a/pkg/provider/kubernetes/gateway/kubernetes_test.go b/pkg/provider/kubernetes/gateway/kubernetes_test.go index 25a6300c5..362e47595 100644 --- a/pkg/provider/kubernetes/gateway/kubernetes_test.go +++ b/pkg/provider/kubernetes/gateway/kubernetes_test.go @@ -234,6 +234,92 @@ func TestLoadHTTPRoutes(t *testing.T) { TLS: &dynamic.TLSConfiguration{}, }, }, + { + desc: "Simple HTTPRoute, with api@internal service", + paths: []string{"services.yml", "simple_to_api_internal.yml"}, + entryPoints: map[string]Entrypoint{"web": { + Address: ":80", + }}, + expected: &dynamic.Configuration{ + UDP: &dynamic.UDPConfiguration{ + Routers: map[string]*dynamic.UDPRouter{}, + Services: map[string]*dynamic.UDPService{}, + }, + TCP: &dynamic.TCPConfiguration{ + Routers: map[string]*dynamic.TCPRouter{}, + Services: map[string]*dynamic.TCPService{}, + }, + HTTP: &dynamic.HTTPConfiguration{ + Routers: map[string]*dynamic.Router{ + "default-http-app-1-my-gateway-web-1c0cf64bde37d9d0df06": { + EntryPoints: []string{"web"}, + Service: "api@internal", + Rule: "Host(`foo.com`) && Path(`/bar`)", + }, + }, + Middlewares: map[string]*dynamic.Middleware{}, + Services: map[string]*dynamic.Service{}, + }, + TLS: &dynamic.TLSConfiguration{}, + }, + }, + { + desc: "Simple HTTPRoute, with myservice@file service", + paths: []string{"services.yml", "simple_cross_provider.yml"}, + entryPoints: map[string]Entrypoint{"web": { + Address: ":80", + }}, + expected: &dynamic.Configuration{ + UDP: &dynamic.UDPConfiguration{ + Routers: map[string]*dynamic.UDPRouter{}, + Services: map[string]*dynamic.UDPService{}, + }, + TCP: &dynamic.TCPConfiguration{ + Routers: map[string]*dynamic.TCPRouter{}, + Services: map[string]*dynamic.TCPService{}, + }, + HTTP: &dynamic.HTTPConfiguration{ + Routers: map[string]*dynamic.Router{ + "default-http-app-1-my-gateway-web-1c0cf64bde37d9d0df06": { + EntryPoints: []string{"web"}, + Service: "default-http-app-1-my-gateway-web-1c0cf64bde37d9d0df06-wrr", + Rule: "Host(`foo.com`) && Path(`/bar`)", + }, + }, + Middlewares: map[string]*dynamic.Middleware{}, + Services: map[string]*dynamic.Service{ + "default-http-app-1-my-gateway-web-1c0cf64bde37d9d0df06-wrr": { + Weighted: &dynamic.WeightedRoundRobin{ + Services: []dynamic.WRRService{ + { + Name: "service@file", + Weight: func(i int) *int { return &i }(1), + }, + { + Name: "default-whoami-80", + Weight: func(i int) *int { return &i }(1), + }, + }, + }, + }, + "default-whoami-80": { + LoadBalancer: &dynamic.ServersLoadBalancer{ + Servers: []dynamic.Server{ + { + URL: "http://10.10.0.1:80", + }, + { + URL: "http://10.10.0.2:80", + }, + }, + PassHostHeader: Bool(true), + }, + }, + }, + }, + TLS: &dynamic.TLSConfiguration{}, + }, + }, { desc: "Simple HTTPRoute with protocol HTTPS", paths: []string{"services.yml", "with_protocol_https.yml"},