From d5ff301d90709c0625363a5c95e0298ed8e96e47 Mon Sep 17 00:00:00 2001 From: Kevin Pollet Date: Thu, 19 May 2022 16:42:09 +0200 Subject: [PATCH] Support certificates configuration in TLSStore CRD Co-authored-by: Romain --- docs/content/https/tls.md | 5 +- .../traefik.containo.us_tlsstores.yaml | 20 +++- .../routing/providers/kubernetes-crd.md | 18 +-- integration/fixtures/k8s/01-traefik-crd.yml | 20 +++- .../fixtures/with_tls_store_certificates.yml | 43 +++++++ pkg/provider/kubernetes/crd/kubernetes.go | 108 ++++++++++++------ .../kubernetes/crd/kubernetes_http.go | 1 + pkg/provider/kubernetes/crd/kubernetes_tcp.go | 1 + .../kubernetes/crd/kubernetes_test.go | 57 +++++++++ .../crd/traefik/v1alpha1/tlsstore.go | 9 +- .../traefik/v1alpha1/zz_generated.deepcopy.go | 45 +++++--- 11 files changed, 252 insertions(+), 75 deletions(-) create mode 100644 pkg/provider/kubernetes/crd/fixtures/with_tls_store_certificates.yml diff --git a/docs/content/https/tls.md b/docs/content/https/tls.md index f49d1f0e9..448f0ca5b 100644 --- a/docs/content/https/tls.md +++ b/docs/content/https/tls.md @@ -364,8 +364,9 @@ spec: ### Strict SNI Checking -With strict SNI checking enabled, Traefik won't allow connections from clients -that do not specify a server_name extension or don't match any certificate configured on the tlsOption. +With strict SNI checking enabled, Traefik won't allow connections from clients that do not specify a server_name extension +or don't match any of the configured certificates. +The default certificate is irrelevant on that matter. ```yaml tab="File (YAML)" # Dynamic configuration diff --git a/docs/content/reference/dynamic-configuration/traefik.containo.us_tlsstores.yaml b/docs/content/reference/dynamic-configuration/traefik.containo.us_tlsstores.yaml index eef5f1f77..57e7ef582 100644 --- a/docs/content/reference/dynamic-configuration/traefik.containo.us_tlsstores.yaml +++ b/docs/content/reference/dynamic-configuration/traefik.containo.us_tlsstores.yaml @@ -36,9 +36,23 @@ spec: spec: description: TLSStoreSpec configures a TLSStore resource. properties: + certificates: + description: Certificates is a list of secret names, each secret holding + a key/certificate pair to add to the store. + items: + description: Certificate holds a secret name for the TLSStore resource. + properties: + secretName: + description: SecretName is the name of the referenced Kubernetes + Secret to specify the certificate details. + type: string + required: + - secretName + type: object + type: array defaultCertificate: - description: DefaultCertificate holds a secret name for the TLSOption - resource. + description: DefaultCertificate is the name of the secret holding + the default key/certificate pair for the store. properties: secretName: description: SecretName is the name of the referenced Kubernetes @@ -47,8 +61,6 @@ spec: required: - secretName type: object - required: - - defaultCertificate type: object required: - metadata diff --git a/docs/content/routing/providers/kubernetes-crd.md b/docs/content/routing/providers/kubernetes-crd.md index 809b51637..891bb7135 100644 --- a/docs/content/routing/providers/kubernetes-crd.md +++ b/docs/content/routing/providers/kubernetes-crd.md @@ -1618,25 +1618,27 @@ or referencing TLS stores in the [`IngressRoute`](#kind-ingressroute) / [`Ingres Traefik currently only uses the [TLS Store named "default"](../../https/tls.md#certificates-stores). This means that if you have two stores that are named default in different kubernetes namespaces, they may be randomly chosen. - For the time being, please only configure one TLSSTore named default. + For the time being, please only configure one TLSStore named default. !!! info "TLSStore Attributes" - ```yaml tab="TLSStore" apiVersion: traefik.containo.us/v1alpha1 kind: TLSStore metadata: name: default namespace: default - spec: - defaultCertificate: - secretName: my-secret # [1] + certificates: # [1] + - secretName: foo + - secretName: bar + defaultCertificate: # [2] + secretName: secret ``` -| Ref | Attribute | Purpose | -|-----|--------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------| -| [1] | `secretName` | The name of the referenced Kubernetes [Secret](https://kubernetes.io/docs/concepts/configuration/secret/) that holds the default certificate for the store. | +| Ref | Attribute | Purpose | +|-----|----------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------| +| [1] | `certificates` | List of Kubernetes [Secrets](https://kubernetes.io/docs/concepts/configuration/secret/), each of them holding a key/certificate pair to add to the store. | +| [2] | `defaultCertificate` | Name of a Kubernetes [Secret](https://kubernetes.io/docs/concepts/configuration/secret/) that holds the default key/certificate pair for the store. | ??? example "Declaring and referencing a TLSStore" diff --git a/integration/fixtures/k8s/01-traefik-crd.yml b/integration/fixtures/k8s/01-traefik-crd.yml index e1c738b5a..5f963d898 100644 --- a/integration/fixtures/k8s/01-traefik-crd.yml +++ b/integration/fixtures/k8s/01-traefik-crd.yml @@ -1356,9 +1356,23 @@ spec: spec: description: TLSStoreSpec configures a TLSStore resource. properties: + certificates: + description: Certificates is a list of secret names, each secret holding + a key/certificate pair to add to the store. + items: + description: Certificate holds a secret name for the TLSStore resource. + properties: + secretName: + description: SecretName is the name of the referenced Kubernetes + Secret to specify the certificate details. + type: string + required: + - secretName + type: object + type: array defaultCertificate: - description: DefaultCertificate holds a secret name for the TLSOption - resource. + description: DefaultCertificate is the name of the secret holding + the default key/certificate pair for the store. properties: secretName: description: SecretName is the name of the referenced Kubernetes @@ -1367,8 +1381,6 @@ spec: required: - secretName type: object - required: - - defaultCertificate type: object required: - metadata diff --git a/pkg/provider/kubernetes/crd/fixtures/with_tls_store_certificates.yml b/pkg/provider/kubernetes/crd/fixtures/with_tls_store_certificates.yml new file mode 100644 index 000000000..2691ac79b --- /dev/null +++ b/pkg/provider/kubernetes/crd/fixtures/with_tls_store_certificates.yml @@ -0,0 +1,43 @@ +apiVersion: traefik.containo.us/v1alpha1 +kind: TLSStore +metadata: + name: default + namespace: default + +spec: + certificates: + - secretName: supersecret + +--- +apiVersion: v1 +kind: Secret +metadata: + name: supersecret + namespace: default + +data: + tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCi0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0= + tls.key: LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCi0tLS0tRU5EIFBSSVZBVEUgS0VZLS0tLS0= + +--- +apiVersion: traefik.containo.us/v1alpha1 +kind: IngressRoute +metadata: + name: test.route + namespace: default + +spec: + entryPoints: + - web + + routes: + - match: Host(`foo.com`) && PathPrefix(`/bar`) + kind: Rule + priority: 12 + services: + - name: whoami + port: 80 + + tls: + store: + name: default diff --git a/pkg/provider/kubernetes/crd/kubernetes.go b/pkg/provider/kubernetes/crd/kubernetes.go index 2855c0bdc..74485b078 100644 --- a/pkg/provider/kubernetes/crd/kubernetes.go +++ b/pkg/provider/kubernetes/crd/kubernetes.go @@ -179,18 +179,25 @@ func (p *Provider) Provide(configurationChan chan<- dynamic.Message, pool *safe. } func (p *Provider) loadConfigurationFromCRD(ctx context.Context, client Client) *dynamic.Configuration { - tlsConfigs := make(map[string]*tls.CertAndStores) + stores, tlsConfigs := buildTLSStores(ctx, client) + if tlsConfigs == nil { + tlsConfigs = make(map[string]*tls.CertAndStores) + } + conf := &dynamic.Configuration{ + // TODO: choose between mutating and returning tlsConfigs HTTP: p.loadIngressRouteConfiguration(ctx, client, tlsConfigs), TCP: p.loadIngressRouteTCPConfiguration(ctx, client, tlsConfigs), UDP: p.loadIngressRouteUDPConfiguration(ctx, client), TLS: &dynamic.TLSConfiguration{ - Certificates: getTLSConfig(tlsConfigs), - Options: buildTLSOptions(ctx, client), - Stores: buildTLSStores(ctx, client), + Options: buildTLSOptions(ctx, client), + Stores: stores, }, } + // Done after because tlsConfigs is mutated by the others above. + conf.TLS.Certificates = getTLSConfig(tlsConfigs) + for _, middleware := range client.GetMiddlewares() { id := provider.Normalize(makeID(middleware.Namespace, middleware.Name)) ctxMid := log.With(ctx, log.Str(log.MiddlewareName, id)) @@ -828,49 +835,60 @@ func buildTLSOptions(ctx context.Context, client Client) map[string]tls.Options return tlsOptions } -func buildTLSStores(ctx context.Context, client Client) map[string]tls.Store { +func buildTLSStores(ctx context.Context, client Client) (map[string]tls.Store, map[string]*tls.CertAndStores) { tlsStoreCRD := client.GetTLSStores() - var tlsStores map[string]tls.Store - if len(tlsStoreCRD) == 0 { - return tlsStores + return nil, nil } - tlsStores = make(map[string]tls.Store) + var nsDefault []string + tlsStores := make(map[string]tls.Store) + tlsConfigs := make(map[string]*tls.CertAndStores) - for _, tlsStore := range tlsStoreCRD { - namespace := tlsStore.Namespace - secretName := tlsStore.Spec.DefaultCertificate.SecretName - logger := log.FromContext(log.With(ctx, log.Str("tlsStore", tlsStore.Name), log.Str("namespace", namespace), log.Str("secretName", secretName))) + for _, t := range tlsStoreCRD { + logger := log.FromContext(log.With(ctx, log.Str("TLSStore", t.Name), log.Str("namespace", t.Namespace))) - secret, exists, err := client.GetSecret(namespace, secretName) - if err != nil { - logger.Errorf("Failed to fetch secret %s/%s: %v", namespace, secretName, err) - continue - } - if !exists { - logger.Errorf("Secret %s/%s does not exist", namespace, secretName) - continue - } + id := makeID(t.Namespace, t.Name) - cert, key, err := getCertificateBlocks(secret, namespace, secretName) - if err != nil { - logger.Errorf("Could not get certificate blocks: %v", err) - continue - } - - id := makeID(tlsStore.Namespace, tlsStore.Name) // If the name is default, we override the default config. - if tlsStore.Name == tls.DefaultTLSStoreName { - id = tlsStore.Name - nsDefault = append(nsDefault, tlsStore.Namespace) + if t.Name == tls.DefaultTLSStoreName { + id = t.Name + nsDefault = append(nsDefault, t.Namespace) } - tlsStores[id] = tls.Store{ - DefaultCertificate: &tls.Certificate{ + + var tlsStore tls.Store + + if t.Spec.DefaultCertificate != nil { + secretName := t.Spec.DefaultCertificate.SecretName + + secret, exists, err := client.GetSecret(t.Namespace, secretName) + if err != nil { + logger.Errorf("Failed to fetch secret %s/%s: %v", t.Namespace, secretName, err) + continue + } + if !exists { + logger.Errorf("Secret %s/%s does not exist", t.Namespace, secretName) + continue + } + + cert, key, err := getCertificateBlocks(secret, t.Namespace, secretName) + if err != nil { + logger.Errorf("Could not get certificate blocks: %v", err) + continue + } + + tlsStore.DefaultCertificate = &tls.Certificate{ CertFile: tls.FileOrContent(cert), KeyFile: tls.FileOrContent(key), - }, + } } + + if err := buildCertificates(client, id, t.Namespace, t.Spec.Certificates, tlsConfigs); err != nil { + logger.Errorf("Failed to load certificates: %v", err) + continue + } + + tlsStores[id] = tlsStore } if len(nsDefault) > 1 { @@ -878,7 +896,25 @@ func buildTLSStores(ctx context.Context, client Client) map[string]tls.Store { log.FromContext(ctx).Errorf("Default TLS Stores defined in multiple namespaces: %v", nsDefault) } - return tlsStores + return tlsStores, tlsConfigs +} + +// buildCertificates loads TLSStore certificates from secrets and sets them into tlsConfigs. +func buildCertificates(client Client, tlsStore, namespace string, certificates []v1alpha1.Certificate, tlsConfigs map[string]*tls.CertAndStores) error { + for _, c := range certificates { + configKey := namespace + "/" + c.SecretName + if _, tlsExists := tlsConfigs[configKey]; !tlsExists { + certAndStores, err := getTLS(client, c.SecretName, namespace) + if err != nil { + return fmt.Errorf("unable to read secret %s: %w", configKey, err) + } + + certAndStores.Stores = []string{tlsStore} + tlsConfigs[configKey] = certAndStores + } + } + + return nil } func makeServiceKey(rule, ingressName string) (string, error) { diff --git a/pkg/provider/kubernetes/crd/kubernetes_http.go b/pkg/provider/kubernetes/crd/kubernetes_http.go index 333ce148c..4e08d90e9 100644 --- a/pkg/provider/kubernetes/crd/kubernetes_http.go +++ b/pkg/provider/kubernetes/crd/kubernetes_http.go @@ -495,6 +495,7 @@ func namespaceOrFallback(lb v1alpha1.LoadBalancerSpec, fallback string) string { return fallback } +// getTLSHTTP mutates tlsConfigs. func getTLSHTTP(ctx context.Context, ingressRoute *v1alpha1.IngressRoute, k8sClient Client, tlsConfigs map[string]*tls.CertAndStores) error { if ingressRoute.Spec.TLS == nil { return nil diff --git a/pkg/provider/kubernetes/crd/kubernetes_tcp.go b/pkg/provider/kubernetes/crd/kubernetes_tcp.go index a9c134de5..cb3be3f3a 100644 --- a/pkg/provider/kubernetes/crd/kubernetes_tcp.go +++ b/pkg/provider/kubernetes/crd/kubernetes_tcp.go @@ -269,6 +269,7 @@ func (p *Provider) loadTCPServers(client Client, namespace string, svc v1alpha1. return servers, nil } +// getTLSTCP mutates tlsConfigs. func getTLSTCP(ctx context.Context, ingressRoute *v1alpha1.IngressRouteTCP, k8sClient Client, tlsConfigs map[string]*tls.CertAndStores) error { if ingressRoute.Spec.TLS == nil { return nil diff --git a/pkg/provider/kubernetes/crd/kubernetes_test.go b/pkg/provider/kubernetes/crd/kubernetes_test.go index 5b4914e11..6c081f0b1 100644 --- a/pkg/provider/kubernetes/crd/kubernetes_test.go +++ b/pkg/provider/kubernetes/crd/kubernetes_test.go @@ -3480,6 +3480,63 @@ func TestLoadIngressRoutes(t *testing.T) { }, }, }, + { + desc: "TLS with tls store containing certificates", + paths: []string{"services.yml", "with_tls_store_certificates.yml"}, + expected: &dynamic.Configuration{ + TLS: &dynamic.TLSConfiguration{ + Certificates: []*tls.CertAndStores{ + { + Certificate: tls.Certificate{ + CertFile: tls.FileOrContent("-----BEGIN CERTIFICATE-----\n-----END CERTIFICATE-----"), + KeyFile: tls.FileOrContent("-----BEGIN PRIVATE KEY-----\n-----END PRIVATE KEY-----"), + }, + Stores: []string{"default"}, + }, + }, + Stores: map[string]tls.Store{ + "default": {}, + }, + }, + UDP: &dynamic.UDPConfiguration{ + Routers: map[string]*dynamic.UDPRouter{}, + Services: map[string]*dynamic.UDPService{}, + }, + TCP: &dynamic.TCPConfiguration{ + Routers: map[string]*dynamic.TCPRouter{}, + Middlewares: map[string]*dynamic.TCPMiddleware{}, + Services: map[string]*dynamic.TCPService{}, + }, + HTTP: &dynamic.HTTPConfiguration{ + Routers: map[string]*dynamic.Router{ + "default-test-route-6b204d94623b3df4370c": { + EntryPoints: []string{"web"}, + Service: "default-test-route-6b204d94623b3df4370c", + Rule: "Host(`foo.com`) && PathPrefix(`/bar`)", + Priority: 12, + TLS: &dynamic.RouterTLSConfig{}, + }, + }, + Middlewares: map[string]*dynamic.Middleware{}, + Services: map[string]*dynamic.Service{ + "default-test-route-6b204d94623b3df4370c": { + LoadBalancer: &dynamic.ServersLoadBalancer{ + Servers: []dynamic.Server{ + { + URL: "http://10.10.0.1:80", + }, + { + URL: "http://10.10.0.2:80", + }, + }, + PassHostHeader: Bool(true), + }, + }, + }, + ServersTransports: map[string]*dynamic.ServersTransport{}, + }, + }, + }, { desc: "TLS with tls store default two times", paths: []string{"services.yml", "with_tls_store.yml", "with_default_tls_store.yml"}, diff --git a/pkg/provider/kubernetes/crd/traefik/v1alpha1/tlsstore.go b/pkg/provider/kubernetes/crd/traefik/v1alpha1/tlsstore.go index 404f07f96..06c1d8a6e 100644 --- a/pkg/provider/kubernetes/crd/traefik/v1alpha1/tlsstore.go +++ b/pkg/provider/kubernetes/crd/traefik/v1alpha1/tlsstore.go @@ -20,13 +20,16 @@ type TLSStore struct { // TLSStoreSpec configures a TLSStore resource. type TLSStoreSpec struct { - DefaultCertificate DefaultCertificate `json:"defaultCertificate"` + // DefaultCertificate is the name of the secret holding the default key/certificate pair for the store. + DefaultCertificate *Certificate `json:"defaultCertificate,omitempty"` + // Certificates is a list of secret names, each secret holding a key/certificate pair to add to the store. + Certificates []Certificate `json:"certificates,omitempty"` } // +k8s:deepcopy-gen=true -// DefaultCertificate holds a secret name for the TLSOption resource. -type DefaultCertificate struct { +// Certificate holds a secret name for the TLSStore resource. +type Certificate struct { // SecretName is the name of the referenced Kubernetes Secret to specify the certificate details. SecretName string `json:"secretName"` } diff --git a/pkg/provider/kubernetes/crd/traefik/v1alpha1/zz_generated.deepcopy.go b/pkg/provider/kubernetes/crd/traefik/v1alpha1/zz_generated.deepcopy.go index 12040e08e..120ad8dfc 100644 --- a/pkg/provider/kubernetes/crd/traefik/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/provider/kubernetes/crd/traefik/v1alpha1/zz_generated.deepcopy.go @@ -53,6 +53,22 @@ func (in *BasicAuth) DeepCopy() *BasicAuth { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Certificate) DeepCopyInto(out *Certificate) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Certificate. +func (in *Certificate) DeepCopy() *Certificate { + if in == nil { + return nil + } + out := new(Certificate) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Chain) DeepCopyInto(out *Chain) { *out = *in @@ -142,22 +158,6 @@ func (in *ClientTLS) DeepCopy() *ClientTLS { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *DefaultCertificate) DeepCopyInto(out *DefaultCertificate) { - *out = *in - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DefaultCertificate. -func (in *DefaultCertificate) DeepCopy() *DefaultCertificate { - if in == nil { - return nil - } - out := new(DefaultCertificate) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DigestAuth) DeepCopyInto(out *DigestAuth) { *out = *in @@ -1413,7 +1413,7 @@ func (in *TLSStore) DeepCopyInto(out *TLSStore) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - out.Spec = in.Spec + in.Spec.DeepCopyInto(&out.Spec) return } @@ -1487,7 +1487,16 @@ func (in *TLSStoreRef) DeepCopy() *TLSStoreRef { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TLSStoreSpec) DeepCopyInto(out *TLSStoreSpec) { *out = *in - out.DefaultCertificate = in.DefaultCertificate + if in.DefaultCertificate != nil { + in, out := &in.DefaultCertificate, &out.DefaultCertificate + *out = new(Certificate) + **out = **in + } + if in.Certificates != nil { + in, out := &in.Certificates, &out.Certificates + *out = make([]Certificate, len(*in)) + copy(*out, *in) + } return }