From 76867e39eac16bdf136050b3aa2ce0d3d7ba2ae8 Mon Sep 17 00:00:00 2001 From: Romain Date: Thu, 16 Sep 2021 15:12:13 +0200 Subject: [PATCH] Fix ServersTransport reference from IngressRoute service definition Co-authored-by: Jean-Baptiste Doumenjou <925513+jbdoumenjou@users.noreply.github.com> --- .../routing/providers/kubernetes-crd.md | 73 ++++--- integration/testdata/rawdata-crd.json | 2 +- .../crd/fixtures/with_cross_namespace.yml | 52 +++-- .../crd/fixtures/with_servers_transport.yml | 36 ++++ pkg/provider/kubernetes/crd/kubernetes.go | 3 +- .../kubernetes/crd/kubernetes_http.go | 25 ++- .../kubernetes/crd/kubernetes_test.go | 198 ++++++++++++++++-- 7 files changed, 323 insertions(+), 66 deletions(-) diff --git a/docs/content/routing/providers/kubernetes-crd.md b/docs/content/routing/providers/kubernetes-crd.md index 4fbfe2d0d..e9223d76a 100644 --- a/docs/content/routing/providers/kubernetes-crd.md +++ b/docs/content/routing/providers/kubernetes-crd.md @@ -337,7 +337,7 @@ Register the `IngressRoute` [kind](../../reference/dynamic-configuration/kuberne responseForwarding: flushInterval: 1ms scheme: https - serversTransport: transport + serversTransport: transport # [10] sticky: cookie: httpOnly: true @@ -346,39 +346,40 @@ Register the `IngressRoute` [kind](../../reference/dynamic-configuration/kuberne sameSite: none strategy: RoundRobin weight: 10 - tls: # [10] - secretName: supersecret # [11] - options: # [12] - name: opt # [13] - namespace: default # [14] - certResolver: foo # [15] - domains: # [16] - - main: example.net # [17] - sans: # [18] + tls: # [11] + secretName: supersecret # [12] + options: # [13] + name: opt # [14] + namespace: default # [15] + certResolver: foo # [16] + domains: # [17] + - main: example.net # [18] + sans: # [19] - a.example.net - b.example.net ``` -| Ref | Attribute | Purpose | -|------|------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| [1] | `entryPoints` | List of [entry points](../routers/index.md#entrypoints) names | -| [2] | `routes` | List of routes | -| [3] | `routes[n].match` | Defines the [rule](../routers/index.md#rule) corresponding to an underlying router. | -| [4] | `routes[n].priority` | [Disambiguate](../routers/index.md#priority) rules of the same length, for route matching | -| [5] | `routes[n].middlewares` | List of reference to [Middleware](#kind-middleware) | -| [6] | `middlewares[n].name` | Defines the [Middleware](#kind-middleware) name | -| [7] | `middlewares[n].namespace` | Defines the [Middleware](#kind-middleware) namespace | -| [8] | `routes[n].services` | List of any combination of [TraefikService](#kind-traefikservice) and reference to a [Kubernetes service](https://kubernetes.io/docs/concepts/services-networking/service/) (See below for `ExternalName Service` setup) | -| [9] | `services[n].port` | Defines the port of a [Kubernetes service](https://kubernetes.io/docs/concepts/services-networking/service/). This can be a reference to a named port. | -| [10] | `tls` | Defines [TLS](../routers/index.md#tls) certificate configuration | -| [11] | `tls.secretName` | Defines the [secret](https://kubernetes.io/docs/concepts/configuration/secret/) name used to store the certificate (in the `IngressRoute` namespace) | -| [12] | `tls.options` | Defines the reference to a [TLSOption](#kind-tlsoption) | -| [13] | `options.name` | Defines the [TLSOption](#kind-tlsoption) name | -| [14] | `options.namespace` | Defines the [TLSOption](#kind-tlsoption) namespace | -| [15] | `tls.certResolver` | Defines the reference to a [CertResolver](../routers/index.md#certresolver) | -| [16] | `tls.domains` | List of [domains](../routers/index.md#domains) | -| [17] | `domains[n].main` | Defines the main domain name | -| [18] | `domains[n].sans` | List of SANs (alternative domains) | +| Ref | Attribute | Purpose | +|------|--------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| [1] | `entryPoints` | List of [entry points](../routers/index.md#entrypoints) names | +| [2] | `routes` | List of routes | +| [3] | `routes[n].match` | Defines the [rule](../routers/index.md#rule) corresponding to an underlying router. | +| [4] | `routes[n].priority` | [Disambiguate](../routers/index.md#priority) rules of the same length, for route matching | +| [5] | `routes[n].middlewares` | List of reference to [Middleware](#kind-middleware) | +| [6] | `middlewares[n].name` | Defines the [Middleware](#kind-middleware) name | +| [7] | `middlewares[n].namespace` | Defines the [Middleware](#kind-middleware) namespace | +| [8] | `routes[n].services` | List of any combination of [TraefikService](#kind-traefikservice) and reference to a [Kubernetes service](https://kubernetes.io/docs/concepts/services-networking/service/) (See below for `ExternalName Service` setup) | +| [9] | `services[n].port` | Defines the port of a [Kubernetes service](https://kubernetes.io/docs/concepts/services-networking/service/). This can be a reference to a named port. | +| [10] | `services[n].serversTransport` | Defines the reference to a [ServersTransport](#kind-serverstransport). The ServersTransport namespace is assumed to be the [Kubernetes service](https://kubernetes.io/docs/concepts/services-networking/service/) namespace (see [ServersTransport reference](#serverstransport-reference)). | +| [11] | `tls` | Defines [TLS](../routers/index.md#tls) certificate configuration | +| [12] | `tls.secretName` | Defines the [secret](https://kubernetes.io/docs/concepts/configuration/secret/) name used to store the certificate (in the `IngressRoute` namespace) | +| [13] | `tls.options` | Defines the reference to a [TLSOption](#kind-tlsoption) | +| [14] | `options.name` | Defines the [TLSOption](#kind-tlsoption) name | +| [15] | `options.namespace` | Defines the [TLSOption](#kind-tlsoption) namespace | +| [16] | `tls.certResolver` | Defines the reference to a [CertResolver](../routers/index.md#certresolver) | +| [17] | `tls.domains` | List of [domains](../routers/index.md#domains) | +| [18] | `domains[n].main` | Defines the main domain name | +| [19] | `domains[n].sans` | List of SANs (alternative domains) | ??? example "Declaring an IngressRoute" @@ -1687,7 +1688,7 @@ or referencing TLS stores in the [`IngressRoute`](#kind-ingressroute) / [`Ingres !!! info "ServersTransport Attributes" - ```yaml tab="TLSStore" + ```yaml tab="ServersTransport" apiVersion: traefik.containo.us/v1alpha1 kind: ServersTransport metadata: @@ -1763,6 +1764,16 @@ or referencing TLS stores in the [`IngressRoute`](#kind-ingressroute) / [`Ingres serversTransport: mytransport ``` +#### ServersTransport reference + +By default, the referenced ServersTransport CRD must be defined in the same [Kubernetes service](https://kubernetes.io/docs/concepts/services-networking/service/) namespace. + +To reference a ServersTransport CRD from another namespace, +the value must be of form `namespace-name@kubernetescrd`, +and the [cross-namespace](../../../providers/kubernetes-crd/#allowcrossnamespace) option must be enabled. + +If the ServersTransport CRD is defined in another provider the cross-provider format `name@provider` should be used. + ## Further Also see the [full example](../../user-guides/crd-acme/index.md) with Let's Encrypt. diff --git a/integration/testdata/rawdata-crd.json b/integration/testdata/rawdata-crd.json index 8b7eaad6e..2b1139b0d 100644 --- a/integration/testdata/rawdata-crd.json +++ b/integration/testdata/rawdata-crd.json @@ -180,7 +180,7 @@ } ], "passHostHeader": true, - "serversTransport": "mytransport@kubernetescrd" + "serversTransport": "default-mytransport@kubernetescrd" }, "status": "enabled", "usedBy": [ diff --git a/pkg/provider/kubernetes/crd/fixtures/with_cross_namespace.yml b/pkg/provider/kubernetes/crd/fixtures/with_cross_namespace.yml index bdc98ca1e..5054eb819 100644 --- a/pkg/provider/kubernetes/crd/fixtures/with_cross_namespace.yml +++ b/pkg/provider/kubernetes/crd/fixtures/with_cross_namespace.yml @@ -9,23 +9,31 @@ spec: - foo routes: - - match: Host(`foo.com`) && PathPrefix(`/bar`) - kind: Rule - priority: 12 - services: - - name: whoami-svc - namespace: cross-ns - port: 80 - - name: tr-svc-wrr1 - kind: TraefikService - - name: tr-svc-wrr2 - namespace: cross-ns - kind: TraefikService - - name: tr-svc-mirror1 - kind: TraefikService - - name: tr-svc-mirror2 - namespace: cross-ns - kind: TraefikService + - match: Host(`foo.com`) && PathPrefix(`/bar`) + kind: Rule + priority: 12 + services: + - name: whoami-svc + namespace: cross-ns + port: 80 + - name: tr-svc-wrr1 + kind: TraefikService + - name: tr-svc-wrr2 + namespace: cross-ns + kind: TraefikService + - name: tr-svc-mirror1 + kind: TraefikService + - name: tr-svc-mirror2 + namespace: cross-ns + kind: TraefikService + + - match: Host(`bar.com`) && PathPrefix(`/foo`) + kind: Rule + services: + - name: whoami-svc + namespace: cross-ns + port: 80 + serversTransport: test --- apiVersion: traefik.containo.us/v1alpha1 @@ -89,3 +97,13 @@ spec: namespace: cross-ns percent: 20 port: 80 + +--- +apiVersion: traefik.containo.us/v1alpha1 +kind: ServersTransport +metadata: + name: test + namespace: foo + +spec: + serverName: "test" diff --git a/pkg/provider/kubernetes/crd/fixtures/with_servers_transport.yml b/pkg/provider/kubernetes/crd/fixtures/with_servers_transport.yml index 25e4f6a25..bc8dd786a 100644 --- a/pkg/provider/kubernetes/crd/fixtures/with_servers_transport.yml +++ b/pkg/provider/kubernetes/crd/fixtures/with_servers_transport.yml @@ -109,3 +109,39 @@ spec: dialTimeout: 42 responseHeaderTimeout: 42s idleConnTimeout: 42ms + +--- +apiVersion: traefik.containo.us/v1alpha1 +kind: ServersTransport +metadata: + name: test + namespace: default + +spec: + serverName: "test" + +--- +apiVersion: traefik.containo.us/v1alpha1 +kind: IngressRoute +metadata: + name: test.route + namespace: default + +spec: + entryPoints: + - foo + + routes: + - match: Host(`foo.com`) + kind: Rule + services: + - name: external-svc-with-https + port: 443 + serversTransport: test + - name: whoamitls + port: 443 + serversTransport: default-test + - name: whoami3 + port: 8443 + serversTransport: foo-test@kubernetescrd + diff --git a/pkg/provider/kubernetes/crd/kubernetes.go b/pkg/provider/kubernetes/crd/kubernetes.go index 1f33d0993..61d2de85d 100644 --- a/pkg/provider/kubernetes/crd/kubernetes.go +++ b/pkg/provider/kubernetes/crd/kubernetes.go @@ -339,7 +339,8 @@ func (p *Provider) loadConfigurationFromCRD(ctx context.Context, client Client) } } - conf.HTTP.ServersTransports[serversTransport.Name] = &dynamic.ServersTransport{ + id := provider.Normalize(makeID(serversTransport.Namespace, serversTransport.Name)) + conf.HTTP.ServersTransports[id] = &dynamic.ServersTransport{ ServerName: serversTransport.Spec.ServerName, InsecureSkipVerify: serversTransport.Spec.InsecureSkipVerify, RootCAs: rootCAs, diff --git a/pkg/provider/kubernetes/crd/kubernetes_http.go b/pkg/provider/kubernetes/crd/kubernetes_http.go index ba32bba33..eee924546 100644 --- a/pkg/provider/kubernetes/crd/kubernetes_http.go +++ b/pkg/provider/kubernetes/crd/kubernetes_http.go @@ -297,11 +297,34 @@ func (c configBuilder) buildServersLB(namespace string, svc v1alpha1.LoadBalance lb.ResponseForwarding = conf.ResponseForwarding lb.Sticky = svc.Sticky - lb.ServersTransport = svc.ServersTransport + + lb.ServersTransport, err = c.makeServersTransportKey(namespace, svc.ServersTransport) + if err != nil { + return nil, err + } return &dynamic.Service{LoadBalancer: lb}, nil } +func (c *configBuilder) makeServersTransportKey(parentNamespace string, serversTransportName string) (string, error) { + if serversTransportName == "" { + return "", nil + } + + if !c.allowCrossNamespace && strings.HasSuffix(serversTransportName, providerNamespaceSeparator+providerName) { + // Since we are not able to know if another namespace is in the name (namespace-name@kubernetescrd), + // if the provider namespace kubernetescrd is used, + // we don't allow this format to avoid cross namespace references. + return "", fmt.Errorf("invalid reference to serversTransport %s: namespace-name@kubernetescrd format is not allowed when crossnamespace is disallowed", serversTransportName) + } + + if strings.Contains(serversTransportName, providerNamespaceSeparator) { + return serversTransportName, nil + } + + return provider.Normalize(makeID(parentNamespace, serversTransportName)), nil +} + func (c configBuilder) loadServers(parentNamespace string, svc v1alpha1.LoadBalancerSpec) ([]dynamic.Server, error) { strategy := svc.Strategy if strategy == "" { diff --git a/pkg/provider/kubernetes/crd/kubernetes_test.go b/pkg/provider/kubernetes/crd/kubernetes_test.go index d0f7d18c0..829e37aec 100644 --- a/pkg/provider/kubernetes/crd/kubernetes_test.go +++ b/pkg/provider/kubernetes/crd/kubernetes_test.go @@ -1330,10 +1330,11 @@ func TestLoadIngressRouteTCPs(t *testing.T) { func TestLoadIngressRoutes(t *testing.T) { testCases := []struct { - desc string - ingressClass string - paths []string - expected *dynamic.Configuration + desc string + ingressClass string + paths []string + expected *dynamic.Configuration + AllowCrossNamespace bool }{ { desc: "Empty", @@ -1400,8 +1401,9 @@ func TestLoadIngressRoutes(t *testing.T) { }, }, { - desc: "Simple Ingress Route with middleware", - paths: []string{"services.yml", "with_middleware.yml"}, + desc: "Simple Ingress Route with middleware", + AllowCrossNamespace: true, + paths: []string{"services.yml", "with_middleware.yml"}, expected: &dynamic.Configuration{ UDP: &dynamic.UDPConfiguration{ Routers: map[string]*dynamic.UDPRouter{}, @@ -1455,8 +1457,9 @@ func TestLoadIngressRoutes(t *testing.T) { }, }, { - desc: "Simple Ingress Route with middleware crossprovider", - paths: []string{"services.yml", "with_middleware_crossprovider.yml"}, + desc: "Simple Ingress Route with middleware crossprovider", + AllowCrossNamespace: true, + paths: []string{"services.yml", "with_middleware_crossprovider.yml"}, expected: &dynamic.Configuration{ UDP: &dynamic.UDPConfiguration{ Routers: map[string]*dynamic.UDPRouter{}, @@ -2024,8 +2027,9 @@ func TestLoadIngressRoutes(t *testing.T) { }, }, { - desc: "services lb, servers lb, and mirror service, all in a wrr with different namespaces", - paths: []string{"with_namespaces.yml"}, + desc: "services lb, servers lb, and mirror service, all in a wrr with different namespaces", + AllowCrossNamespace: true, + paths: []string{"with_namespaces.yml"}, expected: &dynamic.Configuration{ UDP: &dynamic.UDPConfiguration{ Routers: map[string]*dynamic.UDPRouter{}, @@ -3481,8 +3485,9 @@ func TestLoadIngressRoutes(t *testing.T) { }, }, { - desc: "ServersTransport", - paths: []string{"services.yml", "with_servers_transport.yml"}, + desc: "ServersTransport", + AllowCrossNamespace: true, + paths: []string{"services.yml", "with_servers_transport.yml"}, expected: &dynamic.Configuration{ UDP: &dynamic.UDPConfiguration{ Routers: map[string]*dynamic.UDPRouter{}, @@ -3495,7 +3500,7 @@ func TestLoadIngressRoutes(t *testing.T) { }, HTTP: &dynamic.HTTPConfiguration{ ServersTransports: map[string]*dynamic.ServersTransport{ - "test": { + "foo-test": { ServerName: "test", InsecureSkipVerify: true, RootCAs: []tls.FileOrContent{"TESTROOTCAS0", "TESTROOTCAS1", "TESTROOTCAS2", "TESTROOTCAS3", "TESTROOTCAS5", "TESTALLCERTS"}, @@ -3512,10 +3517,154 @@ func TestLoadIngressRoutes(t *testing.T) { IdleConnTimeout: types.Duration(42 * time.Millisecond), }, }, + "default-test": { + ServerName: "test", + ForwardingTimeouts: &dynamic.ForwardingTimeouts{ + DialTimeout: types.Duration(30 * time.Second), + IdleConnTimeout: types.Duration(90 * time.Second), + }, + }, + }, + Routers: map[string]*dynamic.Router{ + "default-test-route-6f97418635c7e18853da": { + EntryPoints: []string{"foo"}, + Service: "default-test-route-6f97418635c7e18853da", + Rule: "Host(`foo.com`)", + }, + }, + Middlewares: map[string]*dynamic.Middleware{}, + Services: map[string]*dynamic.Service{ + "default-external-svc-with-https-443": { + LoadBalancer: &dynamic.ServersLoadBalancer{ + Servers: []dynamic.Server{ + { + URL: "https://external.domain:443", + }, + }, + PassHostHeader: Bool(true), + ServersTransport: "default-test", + }, + }, + "default-whoami3-8443": { + LoadBalancer: &dynamic.ServersLoadBalancer{ + Servers: []dynamic.Server{ + { + URL: "http://10.10.0.7:8443", + }, + { + URL: "http://10.10.0.8:8443", + }, + }, + PassHostHeader: Bool(true), + ServersTransport: "foo-test@kubernetescrd", + }, + }, + "default-whoamitls-443": { + LoadBalancer: &dynamic.ServersLoadBalancer{ + Servers: []dynamic.Server{ + { + URL: "https://10.10.0.5:8443", + }, + { + URL: "https://10.10.0.6:8443", + }, + }, + PassHostHeader: Bool(true), + ServersTransport: "default-default-test", + }, + }, + "default-test-route-6f97418635c7e18853da": { + Weighted: &dynamic.WeightedRoundRobin{ + Services: []dynamic.WRRService{ + { + Name: "default-external-svc-with-https-443", + Weight: Int(1), + }, + { + Name: "default-whoamitls-443", + Weight: Int(1), + }, + { + Name: "default-whoami3-8443", + Weight: Int(1), + }, + }, + }, + }, + }, + }, + TLS: &dynamic.TLSConfiguration{}, + }, + }, + { + desc: "ServersTransport without crossnamespace", + paths: []string{"services.yml", "with_servers_transport.yml"}, + expected: &dynamic.Configuration{ + UDP: &dynamic.UDPConfiguration{ + Routers: map[string]*dynamic.UDPRouter{}, + Services: map[string]*dynamic.UDPService{}, + }, + TCP: &dynamic.TCPConfiguration{ + Routers: map[string]*dynamic.TCPRouter{}, + Middlewares: map[string]*dynamic.TCPMiddleware{}, + Services: map[string]*dynamic.TCPService{}, + }, + HTTP: &dynamic.HTTPConfiguration{ + ServersTransports: map[string]*dynamic.ServersTransport{ + "foo-test": { + ServerName: "test", + InsecureSkipVerify: true, + RootCAs: []tls.FileOrContent{"TESTROOTCAS0", "TESTROOTCAS1", "TESTROOTCAS2", "TESTROOTCAS3", "TESTROOTCAS5", "TESTALLCERTS"}, + Certificates: tls.Certificates{ + {CertFile: "TESTCERT1", KeyFile: "TESTKEY1"}, + {CertFile: "TESTCERT2", KeyFile: "TESTKEY2"}, + {CertFile: "TESTCERT3", KeyFile: "TESTKEY3"}, + }, + MaxIdleConnsPerHost: 42, + ForwardingTimeouts: &dynamic.ForwardingTimeouts{ + DialTimeout: types.Duration(42 * time.Second), + ResponseHeaderTimeout: types.Duration(42 * time.Second), + IdleConnTimeout: types.Duration(42 * time.Millisecond), + }, + DisableHTTP2: true, + }, + "default-test": { + ServerName: "test", + ForwardingTimeouts: &dynamic.ForwardingTimeouts{ + DialTimeout: types.Duration(30 * time.Second), + IdleConnTimeout: types.Duration(90 * time.Second), + }, + }, }, Routers: map[string]*dynamic.Router{}, Middlewares: map[string]*dynamic.Middleware{}, - Services: map[string]*dynamic.Service{}, + Services: map[string]*dynamic.Service{ + "default-external-svc-with-https-443": { + LoadBalancer: &dynamic.ServersLoadBalancer{ + Servers: []dynamic.Server{ + { + URL: "https://external.domain:443", + }, + }, + PassHostHeader: Bool(true), + ServersTransport: "default-test", + }, + }, + "default-whoamitls-443": { + LoadBalancer: &dynamic.ServersLoadBalancer{ + Servers: []dynamic.Server{ + { + URL: "https://10.10.0.5:8443", + }, + { + URL: "https://10.10.0.6:8443", + }, + }, + PassHostHeader: Bool(true), + ServersTransport: "default-default-test", + }, + }, + }, }, TLS: &dynamic.TLSConfiguration{}, }, @@ -3531,7 +3680,7 @@ func TestLoadIngressRoutes(t *testing.T) { return } - p := Provider{IngressClass: test.ingressClass, AllowCrossNamespace: true, AllowExternalNameServices: true} + p := Provider{IngressClass: test.ingressClass, AllowCrossNamespace: test.AllowCrossNamespace, AllowExternalNameServices: true} clientMock := newClientMock(test.paths...) conf := p.loadConfigurationFromCRD(context.Background(), clientMock) @@ -4473,6 +4622,11 @@ func TestCrossNamespace(t *testing.T) { Rule: "Host(`foo.com`) && PathPrefix(`/bar`)", Priority: 12, }, + "default-cross-ns-route-1bc3efa892379bb93c6e": { + EntryPoints: []string{"foo"}, + Service: "default-cross-ns-route-1bc3efa892379bb93c6e", + Rule: "Host(`bar.com`) && PathPrefix(`/foo`)", + }, }, Middlewares: map[string]*dynamic.Middleware{}, Services: map[string]*dynamic.Service{ @@ -4502,6 +4656,20 @@ func TestCrossNamespace(t *testing.T) { }, }, }, + "default-cross-ns-route-1bc3efa892379bb93c6e": { + LoadBalancer: &dynamic.ServersLoadBalancer{ + Servers: []dynamic.Server{ + { + URL: "http://10.10.0.1:80", + }, + { + URL: "http://10.10.0.2:80", + }, + }, + PassHostHeader: Bool(true), + ServersTransport: "cross-ns-test", + }, + }, "cross-ns-whoami-svc-80": { LoadBalancer: &dynamic.ServersLoadBalancer{ Servers: []dynamic.Server{