From 8d2a2ff08f9ceff99fb743bfbf50816feed68520 Mon Sep 17 00:00:00 2001 From: Prajith Date: Mon, 29 Apr 2024 15:50:04 +0530 Subject: [PATCH] Native Kubernetes service load-balancing at the provider level --- docs/content/providers/kubernetes-crd.md | 24 ++ docs/content/providers/kubernetes-ingress.md | 24 ++ .../reference/static-configuration/cli-ref.md | 6 + .../reference/static-configuration/env-ref.md | 6 + .../reference/static-configuration/file.toml | 2 + .../reference/static-configuration/file.yaml | 2 + .../tcp/with_global_native_service_lb.yml | 15 + .../udp/with_global_native_service_lb.yml | 14 + .../with_global_native_service_lb.yml | 16 + pkg/provider/kubernetes/crd/kubernetes.go | 1 + .../kubernetes/crd/kubernetes_http.go | 34 +- pkg/provider/kubernetes/crd/kubernetes_tcp.go | 22 +- .../kubernetes/crd/kubernetes_test.go | 336 ++++++++++++++++++ pkg/provider/kubernetes/crd/kubernetes_udp.go | 22 +- .../crd/traefikio/v1alpha1/ingressroute.go | 2 +- .../crd/traefikio/v1alpha1/ingressroutetcp.go | 2 +- .../crd/traefikio/v1alpha1/ingressrouteudp.go | 2 +- .../v1alpha1/zz_generated.deepcopy.go | 15 + .../kubernetes/ingress/annotations.go | 2 +- .../kubernetes/ingress/annotations_test.go | 2 +- .../Ingress-with-native-lb-by-default.yml | 30 ++ pkg/provider/kubernetes/ingress/kubernetes.go | 32 +- .../kubernetes/ingress/kubernetes_test.go | 81 +++++ 23 files changed, 642 insertions(+), 50 deletions(-) create mode 100644 pkg/provider/kubernetes/crd/fixtures/tcp/with_global_native_service_lb.yml create mode 100644 pkg/provider/kubernetes/crd/fixtures/udp/with_global_native_service_lb.yml create mode 100644 pkg/provider/kubernetes/crd/fixtures/with_global_native_service_lb.yml create mode 100644 pkg/provider/kubernetes/ingress/fixtures/Ingress-with-native-lb-by-default.yml diff --git a/docs/content/providers/kubernetes-crd.md b/docs/content/providers/kubernetes-crd.md index f0ef2da39..2f60a5868 100644 --- a/docs/content/providers/kubernetes-crd.md +++ b/docs/content/providers/kubernetes-crd.md @@ -337,6 +337,30 @@ providers: --providers.kubernetescrd.allowexternalnameservices=true ``` +### `nativeLBByDefault` + +_Optional, Default: false_ + +Defines whether to use Native Kubernetes load-balancing mode by default. +For more information, please check out the IngressRoute `nativeLB` option [documentation](../routing/providers/kubernetes-crd.md#load-balancing). + +```yaml tab="File (YAML)" +providers: + kubernetesCRD: + nativeLBByDefault: true + # ... +``` + +```toml tab="File (TOML)" +[providers.kubernetesCRD] + nativeLBByDefault = true + # ... +``` + +```bash tab="CLI" +--providers.kubernetescrd.nativeLBByDefault=true +``` + ## Full Example For additional information, refer to the [full example](../user-guides/crd-acme/index.md) with Let's Encrypt. diff --git a/docs/content/providers/kubernetes-ingress.md b/docs/content/providers/kubernetes-ingress.md index dc08cde09..f60ed243a 100644 --- a/docs/content/providers/kubernetes-ingress.md +++ b/docs/content/providers/kubernetes-ingress.md @@ -467,6 +467,30 @@ providers: --providers.kubernetesingress.allowexternalnameservices=true ``` +### `nativeLBByDefault` + +_Optional, Default: false_ + +Defines whether to use Native Kubernetes load-balancing mode by default. +For more information, please check out the `traefik.ingress.kubernetes.io/service.nativelb` [service annotation documentation](../routing/providers/kubernetes-ingress.md#on-service). + +```yaml tab="File (YAML)" +providers: + kubernetesIngress: + nativeLBByDefault: true + # ... +``` + +```toml tab="File (TOML)" +[providers.kubernetesIngress] + nativeLBByDefault = true + # ... +``` + +```bash tab="CLI" +--providers.kubernetesingress.nativeLBByDefault=true +``` + ### Further To learn more about the various aspects of the Ingress specification that Traefik supports, diff --git a/docs/content/reference/static-configuration/cli-ref.md b/docs/content/reference/static-configuration/cli-ref.md index d0d339bf1..99065ee63 100644 --- a/docs/content/reference/static-configuration/cli-ref.md +++ b/docs/content/reference/static-configuration/cli-ref.md @@ -714,6 +714,9 @@ Kubernetes label selector to use. `--providers.kubernetescrd.namespaces`: Kubernetes namespaces. +`--providers.kubernetescrd.nativelbbydefault`: +Defines whether to use Native Kubernetes load-balancing mode by default. (Default: ```false```) + `--providers.kubernetescrd.throttleduration`: Ingress refresh throttle duration (Default: ```0```) @@ -795,6 +798,9 @@ Kubernetes Ingress label selector to use. `--providers.kubernetesingress.namespaces`: Kubernetes namespaces. +`--providers.kubernetesingress.nativelbbydefault`: +Defines whether to use Native Kubernetes load-balancing mode by default. (Default: ```false```) + `--providers.kubernetesingress.throttleduration`: Ingress refresh throttle duration (Default: ```0```) diff --git a/docs/content/reference/static-configuration/env-ref.md b/docs/content/reference/static-configuration/env-ref.md index 5c4242bd5..5d8313abb 100644 --- a/docs/content/reference/static-configuration/env-ref.md +++ b/docs/content/reference/static-configuration/env-ref.md @@ -714,6 +714,9 @@ Kubernetes label selector to use. `TRAEFIK_PROVIDERS_KUBERNETESCRD_NAMESPACES`: Kubernetes namespaces. +`TRAEFIK_PROVIDERS_KUBERNETESCRD_NATIVELBBYDEFAULT`: +Defines whether to use Native Kubernetes load-balancing mode by default. (Default: ```false```) + `TRAEFIK_PROVIDERS_KUBERNETESCRD_THROTTLEDURATION`: Ingress refresh throttle duration (Default: ```0```) @@ -795,6 +798,9 @@ Kubernetes Ingress label selector to use. `TRAEFIK_PROVIDERS_KUBERNETESINGRESS_NAMESPACES`: Kubernetes namespaces. +`TRAEFIK_PROVIDERS_KUBERNETESINGRESS_NATIVELBBYDEFAULT`: +Defines whether to use Native Kubernetes load-balancing mode by default. (Default: ```false```) + `TRAEFIK_PROVIDERS_KUBERNETESINGRESS_THROTTLEDURATION`: Ingress refresh throttle duration (Default: ```0```) diff --git a/docs/content/reference/static-configuration/file.toml b/docs/content/reference/static-configuration/file.toml index 9d796c4a6..c3ee80b19 100644 --- a/docs/content/reference/static-configuration/file.toml +++ b/docs/content/reference/static-configuration/file.toml @@ -124,6 +124,7 @@ allowEmptyServices = true allowExternalNameServices = true disableIngressClassLookup = true + nativeLBByDefault = true [providers.kubernetesIngress.ingressEndpoint] ip = "foobar" hostname = "foobar" @@ -139,6 +140,7 @@ ingressClass = "foobar" throttleDuration = "42s" allowEmptyServices = true + nativeLBByDefault = true [providers.kubernetesGateway] endpoint = "foobar" token = "foobar" diff --git a/docs/content/reference/static-configuration/file.yaml b/docs/content/reference/static-configuration/file.yaml index 477fa6b0e..f96e50f99 100644 --- a/docs/content/reference/static-configuration/file.yaml +++ b/docs/content/reference/static-configuration/file.yaml @@ -141,6 +141,7 @@ providers: allowEmptyServices: true allowExternalNameServices: true disableIngressClassLookup: true + nativeLBByDefault: true kubernetesCRD: endpoint: foobar token: foobar @@ -154,6 +155,7 @@ providers: ingressClass: foobar throttleDuration: 42s allowEmptyServices: true + nativeLBByDefault: true kubernetesGateway: endpoint: foobar token: foobar diff --git a/pkg/provider/kubernetes/crd/fixtures/tcp/with_global_native_service_lb.yml b/pkg/provider/kubernetes/crd/fixtures/tcp/with_global_native_service_lb.yml new file mode 100644 index 000000000..d8832498e --- /dev/null +++ b/pkg/provider/kubernetes/crd/fixtures/tcp/with_global_native_service_lb.yml @@ -0,0 +1,15 @@ +apiVersion: traefik.io/v1alpha1 +kind: IngressRouteTCP +metadata: + name: global-native-lb + namespace: default + +spec: + entryPoints: + - foo + + routes: + - match: HostSNI(`foo.com`) + services: + - name: native-svc-tcp + port: 8000 diff --git a/pkg/provider/kubernetes/crd/fixtures/udp/with_global_native_service_lb.yml b/pkg/provider/kubernetes/crd/fixtures/udp/with_global_native_service_lb.yml new file mode 100644 index 000000000..5e7dbd2e5 --- /dev/null +++ b/pkg/provider/kubernetes/crd/fixtures/udp/with_global_native_service_lb.yml @@ -0,0 +1,14 @@ +apiVersion: traefik.io/v1alpha1 +kind: IngressRouteUDP +metadata: + name: global-native-lb + namespace: default + +spec: + entryPoints: + - foo + + routes: + - services: + - name: native-svc-udp + port: 8000 diff --git a/pkg/provider/kubernetes/crd/fixtures/with_global_native_service_lb.yml b/pkg/provider/kubernetes/crd/fixtures/with_global_native_service_lb.yml new file mode 100644 index 000000000..9b8fa2581 --- /dev/null +++ b/pkg/provider/kubernetes/crd/fixtures/with_global_native_service_lb.yml @@ -0,0 +1,16 @@ +apiVersion: traefik.io/v1alpha1 +kind: IngressRoute +metadata: + name: global-native-lb + namespace: default + +spec: + entryPoints: + - foo + + routes: + - match: Host(`foo.com`) + kind: Rule + services: + - name: native-svc + port: 80 diff --git a/pkg/provider/kubernetes/crd/kubernetes.go b/pkg/provider/kubernetes/crd/kubernetes.go index 1559e44e9..298bf884f 100644 --- a/pkg/provider/kubernetes/crd/kubernetes.go +++ b/pkg/provider/kubernetes/crd/kubernetes.go @@ -60,6 +60,7 @@ type Provider struct { IngressClass string `description:"Value of kubernetes.io/ingress.class annotation to watch for." json:"ingressClass,omitempty" toml:"ingressClass,omitempty" yaml:"ingressClass,omitempty" export:"true"` ThrottleDuration ptypes.Duration `description:"Ingress refresh throttle duration" json:"throttleDuration,omitempty" toml:"throttleDuration,omitempty" yaml:"throttleDuration,omitempty" export:"true"` AllowEmptyServices bool `description:"Allow the creation of services without endpoints." json:"allowEmptyServices,omitempty" toml:"allowEmptyServices,omitempty" yaml:"allowEmptyServices,omitempty" export:"true"` + NativeLBByDefault bool `description:"Defines whether to use Native Kubernetes load-balancing mode by default." json:"nativeLBByDefault,omitempty" toml:"nativeLBByDefault,omitempty" yaml:"nativeLBByDefault,omitempty" export:"true"` lastConfiguration safe.Safe diff --git a/pkg/provider/kubernetes/crd/kubernetes_http.go b/pkg/provider/kubernetes/crd/kubernetes_http.go index 6e3b1f2a5..e7364482c 100644 --- a/pkg/provider/kubernetes/crd/kubernetes_http.go +++ b/pkg/provider/kubernetes/crd/kubernetes_http.go @@ -55,6 +55,7 @@ func (p *Provider) loadIngressRouteConfiguration(ctx context.Context, client Cli allowCrossNamespace: p.AllowCrossNamespace, allowExternalNameServices: p.AllowExternalNameServices, allowEmptyServices: p.AllowEmptyServices, + NativeLBByDefault: p.NativeLBByDefault, } for _, route := range ingressRoute.Spec.Routes { @@ -202,6 +203,7 @@ type configBuilder struct { allowCrossNamespace bool allowExternalNameServices bool allowEmptyServices bool + NativeLBByDefault bool } // buildTraefikService creates the configuration for the traefik service defined in tService, @@ -377,20 +379,6 @@ func (c configBuilder) loadServers(parentNamespace string, svc traefikv1alpha1.L return nil, err } - if svc.NativeLB { - address, err := getNativeServiceAddress(*service, *svcPort) - if err != nil { - return nil, fmt.Errorf("getting native Kubernetes Service address: %w", err) - } - - protocol, err := parseServiceProtocol(svc.Scheme, svcPort.Name, svcPort.Port) - if err != nil { - return nil, err - } - - return []dynamic.Server{{URL: fmt.Sprintf("%s://%s", protocol, address)}}, nil - } - var servers []dynamic.Server if service.Spec.Type == corev1.ServiceTypeExternalName { if !c.allowExternalNameServices { @@ -409,6 +397,24 @@ func (c configBuilder) loadServers(parentNamespace string, svc traefikv1alpha1.L }), nil } + nativeLB := c.NativeLBByDefault + if svc.NativeLB != nil { + nativeLB = *svc.NativeLB + } + if nativeLB { + address, err := getNativeServiceAddress(*service, *svcPort) + if err != nil { + return nil, fmt.Errorf("getting native Kubernetes Service address: %w", err) + } + + protocol, err := parseServiceProtocol(svc.Scheme, svcPort.Name, svcPort.Port) + if err != nil { + return nil, err + } + + return []dynamic.Server{{URL: fmt.Sprintf("%s://%s", protocol, address)}}, nil + } + endpoints, endpointsExists, endpointsErr := c.client.GetEndpoints(namespace, sanitizedName) if endpointsErr != nil { return nil, endpointsErr diff --git a/pkg/provider/kubernetes/crd/kubernetes_tcp.go b/pkg/provider/kubernetes/crd/kubernetes_tcp.go index f41fe125e..ea955adae 100644 --- a/pkg/provider/kubernetes/crd/kubernetes_tcp.go +++ b/pkg/provider/kubernetes/crd/kubernetes_tcp.go @@ -237,21 +237,25 @@ func (p *Provider) loadTCPServers(client Client, namespace string, svc traefikv1 return nil, err } - if svc.NativeLB { - address, err := getNativeServiceAddress(*service, *svcPort) - if err != nil { - return nil, fmt.Errorf("getting native Kubernetes Service address: %w", err) - } - - return []dynamic.TCPServer{{Address: address}}, nil - } - var servers []dynamic.TCPServer if service.Spec.Type == corev1.ServiceTypeExternalName { servers = append(servers, dynamic.TCPServer{ Address: net.JoinHostPort(service.Spec.ExternalName, strconv.Itoa(int(svcPort.Port))), }) } else { + nativeLB := p.NativeLBByDefault + if svc.NativeLB != nil { + nativeLB = *svc.NativeLB + } + if nativeLB { + address, err := getNativeServiceAddress(*service, *svcPort) + if err != nil { + return nil, fmt.Errorf("getting native Kubernetes Service address: %w", err) + } + + return []dynamic.TCPServer{{Address: address}}, nil + } + endpoints, endpointsExists, endpointsErr := client.GetEndpoints(namespace, svc.Name) if endpointsErr != nil { return nil, endpointsErr diff --git a/pkg/provider/kubernetes/crd/kubernetes_test.go b/pkg/provider/kubernetes/crd/kubernetes_test.go index cbb36a55c..ca6b1772f 100644 --- a/pkg/provider/kubernetes/crd/kubernetes_test.go +++ b/pkg/provider/kubernetes/crd/kubernetes_test.go @@ -7180,3 +7180,339 @@ func (p *extensionBuilderRegistryMock) RegisterBackendFuncs(group, kind string, p.groupKindBackendFuncs[group][kind] = builderFunc } + +func TestGlobalNativeLB(t *testing.T) { + testCases := []struct { + desc string + ingressClass string + paths []string + NativeLBByDefault bool + expected *dynamic.Configuration + }{ + { + desc: "Empty", + expected: &dynamic.Configuration{ + UDP: &dynamic.UDPConfiguration{ + Routers: map[string]*dynamic.UDPRouter{}, + Services: map[string]*dynamic.UDPService{}, + }, + TCP: &dynamic.TCPConfiguration{ + ServersTransports: map[string]*dynamic.TCPServersTransport{}, + Routers: map[string]*dynamic.TCPRouter{}, + Middlewares: map[string]*dynamic.TCPMiddleware{}, + Services: map[string]*dynamic.TCPService{}, + }, + HTTP: &dynamic.HTTPConfiguration{ + ServersTransports: map[string]*dynamic.ServersTransport{}, + Routers: map[string]*dynamic.Router{}, + Middlewares: map[string]*dynamic.Middleware{}, + Services: map[string]*dynamic.Service{}, + }, + TLS: &dynamic.TLSConfiguration{}, + }, + }, + { + desc: "HTTP with global native Service LB", + paths: []string{"services.yml", "with_global_native_service_lb.yml"}, + NativeLBByDefault: true, + expected: &dynamic.Configuration{ + UDP: &dynamic.UDPConfiguration{ + Routers: map[string]*dynamic.UDPRouter{}, + Services: map[string]*dynamic.UDPService{}, + }, + TCP: &dynamic.TCPConfiguration{ + ServersTransports: map[string]*dynamic.TCPServersTransport{}, + Routers: map[string]*dynamic.TCPRouter{}, + Middlewares: map[string]*dynamic.TCPMiddleware{}, + Services: map[string]*dynamic.TCPService{}, + }, + HTTP: &dynamic.HTTPConfiguration{ + ServersTransports: map[string]*dynamic.ServersTransport{}, + Routers: map[string]*dynamic.Router{ + "default-global-native-lb-6f97418635c7e18853da": { + EntryPoints: []string{"foo"}, + Service: "default-global-native-lb-6f97418635c7e18853da", + Rule: "Host(`foo.com`)", + Priority: 0, + }, + }, + Middlewares: map[string]*dynamic.Middleware{}, + Services: map[string]*dynamic.Service{ + "default-global-native-lb-6f97418635c7e18853da": { + LoadBalancer: &dynamic.ServersLoadBalancer{ + ResponseForwarding: &dynamic.ResponseForwarding{FlushInterval: dynamic.DefaultFlushInterval}, + Servers: []dynamic.Server{ + { + URL: "http://10.10.0.1:80", + }, + }, + PassHostHeader: Bool(true), + }, + }, + }, + }, + TLS: &dynamic.TLSConfiguration{}, + }, + }, + { + desc: "HTTP with native Service LB in ingressroute", + paths: []string{"services.yml", "with_native_service_lb.yml"}, + expected: &dynamic.Configuration{ + UDP: &dynamic.UDPConfiguration{ + Routers: map[string]*dynamic.UDPRouter{}, + Services: map[string]*dynamic.UDPService{}, + }, + TCP: &dynamic.TCPConfiguration{ + ServersTransports: map[string]*dynamic.TCPServersTransport{}, + Routers: map[string]*dynamic.TCPRouter{}, + Middlewares: map[string]*dynamic.TCPMiddleware{}, + Services: map[string]*dynamic.TCPService{}, + }, + HTTP: &dynamic.HTTPConfiguration{ + ServersTransports: map[string]*dynamic.ServersTransport{}, + Routers: map[string]*dynamic.Router{ + "default-test-route-6f97418635c7e18853da": { + EntryPoints: []string{"foo"}, + Service: "default-test-route-6f97418635c7e18853da", + Rule: "Host(`foo.com`)", + Priority: 0, + }, + }, + Middlewares: map[string]*dynamic.Middleware{}, + Services: map[string]*dynamic.Service{ + "default-test-route-6f97418635c7e18853da": { + LoadBalancer: &dynamic.ServersLoadBalancer{ + ResponseForwarding: &dynamic.ResponseForwarding{FlushInterval: dynamic.DefaultFlushInterval}, + Servers: []dynamic.Server{ + { + URL: "http://10.10.0.1:80", + }, + }, + PassHostHeader: Bool(true), + }, + }, + }, + }, + TLS: &dynamic.TLSConfiguration{}, + }, + }, + { + desc: "TCP with global native Service LB", + paths: []string{"tcp/services.yml", "tcp/with_global_native_service_lb.yml"}, + NativeLBByDefault: true, + expected: &dynamic.Configuration{ + UDP: &dynamic.UDPConfiguration{ + Routers: map[string]*dynamic.UDPRouter{}, + Services: map[string]*dynamic.UDPService{}, + }, + HTTP: &dynamic.HTTPConfiguration{ + ServersTransports: map[string]*dynamic.ServersTransport{}, + Routers: map[string]*dynamic.Router{}, + Middlewares: map[string]*dynamic.Middleware{}, + Services: map[string]*dynamic.Service{}, + }, + TCP: &dynamic.TCPConfiguration{ + ServersTransports: map[string]*dynamic.TCPServersTransport{}, + Routers: map[string]*dynamic.TCPRouter{ + "default-global-native-lb-fdd3e9338e47a45efefc": { + EntryPoints: []string{"foo"}, + Service: "default-global-native-lb-fdd3e9338e47a45efefc", + Rule: "HostSNI(`foo.com`)", + }, + }, + Middlewares: map[string]*dynamic.TCPMiddleware{}, + Services: map[string]*dynamic.TCPService{ + "default-global-native-lb-fdd3e9338e47a45efefc": { + LoadBalancer: &dynamic.TCPServersLoadBalancer{ + Servers: []dynamic.TCPServer{ + { + Address: "10.10.0.1:8000", + Port: "", + }, + }, + }, + }, + }, + }, + TLS: &dynamic.TLSConfiguration{}, + }, + }, + { + desc: "TCP with native Service LB in ingressroute", + paths: []string{"tcp/services.yml", "tcp/with_native_service_lb.yml"}, + expected: &dynamic.Configuration{ + UDP: &dynamic.UDPConfiguration{ + Routers: map[string]*dynamic.UDPRouter{}, + Services: map[string]*dynamic.UDPService{}, + }, + HTTP: &dynamic.HTTPConfiguration{ + ServersTransports: map[string]*dynamic.ServersTransport{}, + Routers: map[string]*dynamic.Router{}, + Middlewares: map[string]*dynamic.Middleware{}, + Services: map[string]*dynamic.Service{}, + }, + TCP: &dynamic.TCPConfiguration{ + ServersTransports: map[string]*dynamic.TCPServersTransport{}, + Routers: map[string]*dynamic.TCPRouter{ + "default-test.route-fdd3e9338e47a45efefc": { + EntryPoints: []string{"foo"}, + Service: "default-test.route-fdd3e9338e47a45efefc", + Rule: "HostSNI(`foo.com`)", + }, + }, + Middlewares: map[string]*dynamic.TCPMiddleware{}, + Services: map[string]*dynamic.TCPService{ + "default-test.route-fdd3e9338e47a45efefc": { + LoadBalancer: &dynamic.TCPServersLoadBalancer{ + Servers: []dynamic.TCPServer{ + { + Address: "10.10.0.1:8000", + Port: "", + }, + }, + }, + }, + }, + }, + TLS: &dynamic.TLSConfiguration{}, + }, + }, + { + desc: "UDP with native Service LB in ingressroute", + paths: []string{"udp/services.yml", "udp/with_native_service_lb.yml"}, + expected: &dynamic.Configuration{ + UDP: &dynamic.UDPConfiguration{ + Routers: map[string]*dynamic.UDPRouter{ + "default-test.route-0": { + EntryPoints: []string{"foo"}, + Service: "default-test.route-0", + }, + }, + Services: map[string]*dynamic.UDPService{ + "default-test.route-0": { + LoadBalancer: &dynamic.UDPServersLoadBalancer{ + Servers: []dynamic.UDPServer{ + { + Address: "10.10.0.1:8000", + Port: "", + }, + }, + }, + }, + }, + }, + HTTP: &dynamic.HTTPConfiguration{ + ServersTransports: map[string]*dynamic.ServersTransport{}, + Routers: map[string]*dynamic.Router{}, + Middlewares: map[string]*dynamic.Middleware{}, + Services: map[string]*dynamic.Service{}, + }, + TCP: &dynamic.TCPConfiguration{ + ServersTransports: map[string]*dynamic.TCPServersTransport{}, + Routers: map[string]*dynamic.TCPRouter{}, + Middlewares: map[string]*dynamic.TCPMiddleware{}, + Services: map[string]*dynamic.TCPService{}, + }, + TLS: &dynamic.TLSConfiguration{}, + }, + }, + { + desc: "UDP with global native Service LB", + paths: []string{"udp/services.yml", "udp/with_global_native_service_lb.yml"}, + NativeLBByDefault: true, + expected: &dynamic.Configuration{ + UDP: &dynamic.UDPConfiguration{ + Routers: map[string]*dynamic.UDPRouter{ + "default-global-native-lb-0": { + EntryPoints: []string{"foo"}, + Service: "default-global-native-lb-0", + }, + }, + Services: map[string]*dynamic.UDPService{ + "default-global-native-lb-0": { + LoadBalancer: &dynamic.UDPServersLoadBalancer{ + Servers: []dynamic.UDPServer{ + { + Address: "10.10.0.1:8000", + Port: "", + }, + }, + }, + }, + }, + }, + HTTP: &dynamic.HTTPConfiguration{ + ServersTransports: map[string]*dynamic.ServersTransport{}, + Routers: map[string]*dynamic.Router{}, + Middlewares: map[string]*dynamic.Middleware{}, + Services: map[string]*dynamic.Service{}, + }, + TCP: &dynamic.TCPConfiguration{ + ServersTransports: map[string]*dynamic.TCPServersTransport{}, + Routers: map[string]*dynamic.TCPRouter{}, + Middlewares: map[string]*dynamic.TCPMiddleware{}, + Services: map[string]*dynamic.TCPService{}, + }, + TLS: &dynamic.TLSConfiguration{}, + }, + }, + } + + for _, test := range testCases { + t.Run(test.desc, func(t *testing.T) { + t.Parallel() + + var k8sObjects []runtime.Object + var crdObjects []runtime.Object + for _, path := range test.paths { + yamlContent, err := os.ReadFile(filepath.FromSlash("./fixtures/" + path)) + if err != nil { + panic(err) + } + + objects := k8s.MustParseYaml(yamlContent) + for _, obj := range objects { + switch o := obj.(type) { + case *corev1.Service, *corev1.Endpoints, *corev1.Secret: + k8sObjects = append(k8sObjects, o) + case *traefikv1alpha1.IngressRoute: + crdObjects = append(crdObjects, o) + case *traefikv1alpha1.IngressRouteTCP: + crdObjects = append(crdObjects, o) + case *traefikv1alpha1.IngressRouteUDP: + crdObjects = append(crdObjects, o) + case *traefikv1alpha1.Middleware: + crdObjects = append(crdObjects, o) + case *traefikv1alpha1.TraefikService: + crdObjects = append(crdObjects, o) + case *traefikv1alpha1.TLSOption: + crdObjects = append(crdObjects, o) + case *traefikv1alpha1.TLSStore: + crdObjects = append(crdObjects, o) + default: + } + } + } + + kubeClient := kubefake.NewSimpleClientset(k8sObjects...) + crdClient := traefikcrdfake.NewSimpleClientset(crdObjects...) + + client := newClientImpl(kubeClient, crdClient) + + stopCh := make(chan struct{}) + + eventCh, err := client.WatchAll([]string{"default", "cross-ns"}, stopCh) + require.NoError(t, err) + + if k8sObjects != nil || crdObjects != nil { + // just wait for the first event + <-eventCh + } + + p := Provider{NativeLBByDefault: test.NativeLBByDefault} + + conf := p.loadConfigurationFromCRD(context.Background(), client) + assert.Equal(t, test.expected, conf) + }) + } +} diff --git a/pkg/provider/kubernetes/crd/kubernetes_udp.go b/pkg/provider/kubernetes/crd/kubernetes_udp.go index 417e90d99..c98d0212a 100644 --- a/pkg/provider/kubernetes/crd/kubernetes_udp.go +++ b/pkg/provider/kubernetes/crd/kubernetes_udp.go @@ -121,21 +121,25 @@ func (p *Provider) loadUDPServers(client Client, namespace string, svc traefikv1 return nil, err } - if svc.NativeLB { - address, err := getNativeServiceAddress(*service, *svcPort) - if err != nil { - return nil, fmt.Errorf("getting native Kubernetes Service address: %w", err) - } - - return []dynamic.UDPServer{{Address: address}}, nil - } - var servers []dynamic.UDPServer if service.Spec.Type == corev1.ServiceTypeExternalName { servers = append(servers, dynamic.UDPServer{ Address: net.JoinHostPort(service.Spec.ExternalName, strconv.Itoa(int(svcPort.Port))), }) } else { + nativeLB := p.NativeLBByDefault + if svc.NativeLB != nil { + nativeLB = *svc.NativeLB + } + if nativeLB { + address, err := getNativeServiceAddress(*service, *svcPort) + if err != nil { + return nil, fmt.Errorf("getting native Kubernetes Service address: %w", err) + } + + return []dynamic.UDPServer{{Address: address}}, nil + } + endpoints, endpointsExists, endpointsErr := client.GetEndpoints(namespace, svc.Name) if endpointsErr != nil { return nil, endpointsErr diff --git a/pkg/provider/kubernetes/crd/traefikio/v1alpha1/ingressroute.go b/pkg/provider/kubernetes/crd/traefikio/v1alpha1/ingressroute.go index 475d160d4..89718b745 100644 --- a/pkg/provider/kubernetes/crd/traefikio/v1alpha1/ingressroute.go +++ b/pkg/provider/kubernetes/crd/traefikio/v1alpha1/ingressroute.go @@ -125,7 +125,7 @@ type LoadBalancerSpec struct { // whether the LB's children are directly the pods IPs or if the only child is the Kubernetes Service clusterIP. // The Kubernetes Service itself does load-balance to the pods. // By default, NativeLB is false. - NativeLB bool `json:"nativeLB,omitempty"` + NativeLB *bool `json:"nativeLB,omitempty"` } type ResponseForwarding struct { diff --git a/pkg/provider/kubernetes/crd/traefikio/v1alpha1/ingressroutetcp.go b/pkg/provider/kubernetes/crd/traefikio/v1alpha1/ingressroutetcp.go index 9ff6897db..e37050f19 100644 --- a/pkg/provider/kubernetes/crd/traefikio/v1alpha1/ingressroutetcp.go +++ b/pkg/provider/kubernetes/crd/traefikio/v1alpha1/ingressroutetcp.go @@ -92,7 +92,7 @@ type ServiceTCP struct { // whether the LB's children are directly the pods IPs or if the only child is the Kubernetes Service clusterIP. // The Kubernetes Service itself does load-balance to the pods. // By default, NativeLB is false. - NativeLB bool `json:"nativeLB,omitempty"` + NativeLB *bool `json:"nativeLB,omitempty"` } // +genclient diff --git a/pkg/provider/kubernetes/crd/traefikio/v1alpha1/ingressrouteudp.go b/pkg/provider/kubernetes/crd/traefikio/v1alpha1/ingressrouteudp.go index 18773f437..c77559a50 100644 --- a/pkg/provider/kubernetes/crd/traefikio/v1alpha1/ingressrouteudp.go +++ b/pkg/provider/kubernetes/crd/traefikio/v1alpha1/ingressrouteudp.go @@ -37,7 +37,7 @@ type ServiceUDP struct { // whether the LB's children are directly the pods IPs or if the only child is the Kubernetes Service clusterIP. // The Kubernetes Service itself does load-balance to the pods. // By default, NativeLB is false. - NativeLB bool `json:"nativeLB,omitempty"` + NativeLB *bool `json:"nativeLB,omitempty"` } // +genclient diff --git a/pkg/provider/kubernetes/crd/traefikio/v1alpha1/zz_generated.deepcopy.go b/pkg/provider/kubernetes/crd/traefikio/v1alpha1/zz_generated.deepcopy.go index 547f0de4e..a17ff5484 100644 --- a/pkg/provider/kubernetes/crd/traefikio/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/provider/kubernetes/crd/traefikio/v1alpha1/zz_generated.deepcopy.go @@ -577,6 +577,11 @@ func (in *LoadBalancerSpec) DeepCopyInto(out *LoadBalancerSpec) { *out = new(int) **out = **in } + if in.NativeLB != nil { + in, out := &in.NativeLB, &out.NativeLB + *out = new(bool) + **out = **in + } return } @@ -1333,6 +1338,11 @@ func (in *ServiceTCP) DeepCopyInto(out *ServiceTCP) { *out = new(dynamic.ProxyProtocol) **out = **in } + if in.NativeLB != nil { + in, out := &in.NativeLB, &out.NativeLB + *out = new(bool) + **out = **in + } return } @@ -1355,6 +1365,11 @@ func (in *ServiceUDP) DeepCopyInto(out *ServiceUDP) { *out = new(int) **out = **in } + if in.NativeLB != nil { + in, out := &in.NativeLB, &out.NativeLB + *out = new(bool) + **out = **in + } return } diff --git a/pkg/provider/kubernetes/ingress/annotations.go b/pkg/provider/kubernetes/ingress/annotations.go index 134b2eda2..d3bed8cc0 100644 --- a/pkg/provider/kubernetes/ingress/annotations.go +++ b/pkg/provider/kubernetes/ingress/annotations.go @@ -45,7 +45,7 @@ type ServiceIng struct { ServersTransport string `json:"serversTransport,omitempty"` PassHostHeader *bool `json:"passHostHeader"` Sticky *dynamic.Sticky `json:"sticky,omitempty" label:"allowEmpty"` - NativeLB bool `json:"nativeLB,omitempty"` + NativeLB *bool `json:"nativeLB,omitempty"` } // SetDefaults sets the default values. diff --git a/pkg/provider/kubernetes/ingress/annotations_test.go b/pkg/provider/kubernetes/ingress/annotations_test.go index cab70357f..646261894 100644 --- a/pkg/provider/kubernetes/ingress/annotations_test.go +++ b/pkg/provider/kubernetes/ingress/annotations_test.go @@ -125,7 +125,7 @@ func Test_parseServiceConfig(t *testing.T) { ServersScheme: "protocol", ServersTransport: "foobar@file", PassHostHeader: Bool(true), - NativeLB: true, + NativeLB: Bool(true), }, }, }, diff --git a/pkg/provider/kubernetes/ingress/fixtures/Ingress-with-native-lb-by-default.yml b/pkg/provider/kubernetes/ingress/fixtures/Ingress-with-native-lb-by-default.yml new file mode 100644 index 000000000..26d1cbf59 --- /dev/null +++ b/pkg/provider/kubernetes/ingress/fixtures/Ingress-with-native-lb-by-default.yml @@ -0,0 +1,30 @@ +kind: Ingress +apiVersion: networking.k8s.io/v1 +metadata: + name: global-native-lb + namespace: default +spec: + rules: + - host: traefik.tchouk + http: + paths: + - path: /bar + backend: + service: + name: service1 + port: + number: 8080 + pathType: Prefix + +--- +kind: Service +apiVersion: v1 +metadata: + name: service1 + namespace: default +spec: + ports: + - port: 8080 + clusterIP: 10.0.0.1 + type: ClusterIP + externalName: traefik.wtf diff --git a/pkg/provider/kubernetes/ingress/kubernetes.go b/pkg/provider/kubernetes/ingress/kubernetes.go index 99a8351f8..f3f4aa23e 100644 --- a/pkg/provider/kubernetes/ingress/kubernetes.go +++ b/pkg/provider/kubernetes/ingress/kubernetes.go @@ -52,6 +52,7 @@ type Provider struct { AllowEmptyServices bool `description:"Allow creation of services without endpoints." json:"allowEmptyServices,omitempty" toml:"allowEmptyServices,omitempty" yaml:"allowEmptyServices,omitempty" export:"true"` AllowExternalNameServices bool `description:"Allow ExternalName services." json:"allowExternalNameServices,omitempty" toml:"allowExternalNameServices,omitempty" yaml:"allowExternalNameServices,omitempty" export:"true"` DisableIngressClassLookup bool `description:"Disables the lookup of IngressClasses." json:"disableIngressClassLookup,omitempty" toml:"disableIngressClassLookup,omitempty" yaml:"disableIngressClassLookup,omitempty" export:"true"` + NativeLBByDefault bool `description:"Defines whether to use Native Kubernetes load-balancing mode by default." json:"nativeLBByDefault,omitempty" toml:"nativeLBByDefault,omitempty" yaml:"nativeLBByDefault,omitempty" export:"true"` lastConfiguration safe.Safe @@ -571,6 +572,8 @@ func (p *Provider) loadService(client Client, namespace string, backend netv1.In return nil, err } + nativeLB := p.NativeLBByDefault + if svcConfig != nil && svcConfig.Service != nil { svc.LoadBalancer.Sticky = svcConfig.Service.Sticky @@ -582,19 +585,8 @@ func (p *Provider) loadService(client Client, namespace string, backend netv1.In svc.LoadBalancer.ServersTransport = svcConfig.Service.ServersTransport } - if svcConfig.Service.NativeLB { - address, err := getNativeServiceAddress(*service, portSpec) - if err != nil { - return nil, fmt.Errorf("getting native Kubernetes Service address: %w", err) - } - - protocol := getProtocol(portSpec, portSpec.Name, svcConfig) - - svc.LoadBalancer.Servers = []dynamic.Server{ - {URL: fmt.Sprintf("%s://%s", protocol, address)}, - } - - return svc, nil + if svcConfig.Service.NativeLB != nil { + nativeLB = *svcConfig.Service.NativeLB } } @@ -609,6 +601,20 @@ func (p *Provider) loadService(client Client, namespace string, backend netv1.In return svc, nil } + if nativeLB { + address, err := getNativeServiceAddress(*service, portSpec) + if err != nil { + return nil, fmt.Errorf("getting native Kubernetes Service address: %w", err) + } + + protocol := getProtocol(portSpec, portSpec.Name, svcConfig) + svc.LoadBalancer.Servers = []dynamic.Server{ + {URL: fmt.Sprintf("%s://%s", protocol, address)}, + } + + return svc, nil + } + endpoints, endpointsExists, endpointsErr := client.GetEndpoints(namespace, backend.Service.Name) if endpointsErr != nil { return nil, endpointsErr diff --git a/pkg/provider/kubernetes/ingress/kubernetes_test.go b/pkg/provider/kubernetes/ingress/kubernetes_test.go index a11fc0822..273a99260 100644 --- a/pkg/provider/kubernetes/ingress/kubernetes_test.go +++ b/pkg/provider/kubernetes/ingress/kubernetes_test.go @@ -1864,3 +1864,84 @@ func TestGetCertificates(t *testing.T) { }) } } + +func TestLoadConfigurationFromIngressesWithNativeLBByDefault(t *testing.T) { + testCases := []struct { + desc string + ingressClass string + expected *dynamic.Configuration + }{ + { + desc: "Ingress with native service lb", + expected: &dynamic.Configuration{ + TCP: &dynamic.TCPConfiguration{}, + HTTP: &dynamic.HTTPConfiguration{ + Middlewares: map[string]*dynamic.Middleware{}, + Routers: map[string]*dynamic.Router{ + "testing-traefik-tchouk-bar": { + Rule: "Host(`traefik.tchouk`) && PathPrefix(`/bar`)", + Service: "testing-service1-8080", + }, + }, + Services: map[string]*dynamic.Service{ + "testing-service1-8080": { + LoadBalancer: &dynamic.ServersLoadBalancer{ + ResponseForwarding: &dynamic.ResponseForwarding{FlushInterval: dynamic.DefaultFlushInterval}, + PassHostHeader: Bool(true), + Servers: []dynamic.Server{ + { + URL: "http://10.0.0.1:8080", + }, + }, + }, + }, + }, + }, + }, + }, + { + desc: "Ingress with native lb by default", + expected: &dynamic.Configuration{ + TCP: &dynamic.TCPConfiguration{}, + HTTP: &dynamic.HTTPConfiguration{ + Middlewares: map[string]*dynamic.Middleware{}, + Routers: map[string]*dynamic.Router{ + "default-global-native-lb-traefik-tchouk-bar": { + Rule: "Host(`traefik.tchouk`) && PathPrefix(`/bar`)", + Service: "default-service1-8080", + }, + }, + Services: map[string]*dynamic.Service{ + "default-service1-8080": { + LoadBalancer: &dynamic.ServersLoadBalancer{ + ResponseForwarding: &dynamic.ResponseForwarding{FlushInterval: dynamic.DefaultFlushInterval}, + PassHostHeader: Bool(true), + Servers: []dynamic.Server{ + { + URL: "http://10.0.0.1:8080", + }, + }, + }, + }, + }, + }, + }, + }, + } + + for _, test := range testCases { + t.Run(test.desc, func(t *testing.T) { + t.Parallel() + + clientMock := newClientMock(generateTestFilename(test.desc)) + + p := Provider{ + IngressClass: test.ingressClass, + NativeLBByDefault: true, + } + conf := p.loadConfigurationFromIngresses(context.Background(), clientMock) + + assert.Equal(t, test.expected, conf) + }) + } +}