Support certificates configuration in TLSStore CRD

Co-authored-by: Romain <rtribotte@users.noreply.github.com>

docs/content/https/tls.md

@@ -364,8 +364,9 @@ spec:
 
 ### Strict SNI Checking
 
-With strict SNI checking enabled, Traefik won't allow connections from clients
+With strict SNI checking enabled, Traefik won't allow connections from clients that do not specify a server_name extension
-that do not specify a server_name extension or don't match any certificate configured on the tlsOption.
+or don't match any of the configured certificates.
+The default certificate is irrelevant on that matter.
 
 ```yaml tab="File (YAML)"
 # Dynamic configuration

docs/content/reference/dynamic-configuration/traefik.containo.us_tlsstores.yaml

@@ -36,9 +36,11 @@ spec:
           spec:
             description: TLSStoreSpec configures a TLSStore resource.
             properties:
-              defaultCertificate:
+              certificates:
-                description: DefaultCertificate holds a secret name for the TLSOption
+                description: Certificates is a list of secret names, each secret holding
-                  resource.
+                  a key/certificate pair to add to the store.
+                items:
+                  description: Certificate holds a secret name for the TLSStore resource.
                   properties:
                     secretName:
                       description: SecretName is the name of the referenced Kubernetes
@@ -47,8 +49,18 @@ spec:
                   required:
                   - secretName
                   type: object
+                type: array
+              defaultCertificate:
+                description: DefaultCertificate is the name of the secret holding
+                  the default key/certificate pair for the store.
+                properties:
+                  secretName:
+                    description: SecretName is the name of the referenced Kubernetes
+                      Secret to specify the certificate details.
+                    type: string
                 required:
-            - defaultCertificate
+                - secretName
+                type: object
             type: object
         required:
         - metadata

docs/content/routing/providers/kubernetes-crd.md

@@ -1618,25 +1618,27 @@ or referencing TLS stores in the [`IngressRoute`](#kind-ingressroute) / [`Ingres
     Traefik currently only uses the [TLS Store named "default"](../../https/tls.md#certificates-stores).
     This means that if you have two stores that are named default in different kubernetes namespaces,
     they may be randomly chosen.
-    For the time being, please only configure one TLSSTore named default.
+    For the time being, please only configure one TLSStore named default.
 
 !!! info "TLSStore Attributes"
-   
     ```yaml tab="TLSStore"
     apiVersion: traefik.containo.us/v1alpha1
     kind: TLSStore
     metadata:
       name: default
       namespace: default
-    
     spec:
-      defaultCertificate:
+      certificates:                            # [1]
-        secretName: my-secret                      # [1]
+        - secretName: foo                      
+        - secretName: bar
+      defaultCertificate:                      # [2]
+        secretName: secret                     
     ```
 
 | Ref | Attribute            | Purpose                                                                                                                                                   |
-|-----|--------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------|
+|-----|----------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------|
-| [1] | `secretName` | The name of the referenced Kubernetes [Secret](https://kubernetes.io/docs/concepts/configuration/secret/) that holds the default certificate for the store. |
+| [1] | `certificates`       | List of Kubernetes [Secrets](https://kubernetes.io/docs/concepts/configuration/secret/), each of them holding a key/certificate pair to add to the store. |
+| [2] | `defaultCertificate` | Name of a Kubernetes [Secret](https://kubernetes.io/docs/concepts/configuration/secret/) that holds the default key/certificate pair for the store.       |
 
 ??? example "Declaring and referencing a TLSStore"
    

integration/fixtures/k8s/01-traefik-crd.yml

@@ -1356,9 +1356,11 @@ spec:
           spec:
             description: TLSStoreSpec configures a TLSStore resource.
             properties:
-              defaultCertificate:
+              certificates:
-                description: DefaultCertificate holds a secret name for the TLSOption
+                description: Certificates is a list of secret names, each secret holding
-                  resource.
+                  a key/certificate pair to add to the store.
+                items:
+                  description: Certificate holds a secret name for the TLSStore resource.
                   properties:
                     secretName:
                       description: SecretName is the name of the referenced Kubernetes
@@ -1367,8 +1369,18 @@ spec:
                   required:
                   - secretName
                   type: object
+                type: array
+              defaultCertificate:
+                description: DefaultCertificate is the name of the secret holding
+                  the default key/certificate pair for the store.
+                properties:
+                  secretName:
+                    description: SecretName is the name of the referenced Kubernetes
+                      Secret to specify the certificate details.
+                    type: string
                 required:
-            - defaultCertificate
+                - secretName
+                type: object
             type: object
         required:
         - metadata

pkg/provider/kubernetes/crd/fixtures/with_tls_store_certificates.yml

@@ -0,0 +1,43 @@
+apiVersion: traefik.containo.us/v1alpha1
+kind: TLSStore
+metadata:
+  name: default
+  namespace: default
+
+spec:
+  certificates:
+    - secretName: supersecret
+
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: supersecret
+  namespace: default
+
+data:
+  tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCi0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0=
+  tls.key: LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCi0tLS0tRU5EIFBSSVZBVEUgS0VZLS0tLS0=
+
+---
+apiVersion: traefik.containo.us/v1alpha1
+kind: IngressRoute
+metadata:
+  name: test.route
+  namespace: default
+
+spec:
+  entryPoints:
+    - web
+
+  routes:
+  - match: Host(`foo.com`) && PathPrefix(`/bar`)
+    kind: Rule
+    priority: 12
+    services:
+    - name: whoami
+      port: 80
+
+  tls:
+    store:
+      name: default

pkg/provider/kubernetes/crd/kubernetes.go

@@ -179,18 +179,25 @@ func (p *Provider) Provide(configurationChan chan<- dynamic.Message, pool *safe.
 }
 
 func (p *Provider) loadConfigurationFromCRD(ctx context.Context, client Client) *dynamic.Configuration {
-	tlsConfigs := make(map[string]*tls.CertAndStores)
+	stores, tlsConfigs := buildTLSStores(ctx, client)
+	if tlsConfigs == nil {
+		tlsConfigs = make(map[string]*tls.CertAndStores)
+	}
+
 	conf := &dynamic.Configuration{
+		// TODO: choose between mutating and returning tlsConfigs
 		HTTP: p.loadIngressRouteConfiguration(ctx, client, tlsConfigs),
 		TCP:  p.loadIngressRouteTCPConfiguration(ctx, client, tlsConfigs),
 		UDP:  p.loadIngressRouteUDPConfiguration(ctx, client),
 		TLS: &dynamic.TLSConfiguration{
-			Certificates: getTLSConfig(tlsConfigs),
 			Options: buildTLSOptions(ctx, client),
-			Stores:       buildTLSStores(ctx, client),
+			Stores:  stores,
 		},
 	}
 
+	// Done after because tlsConfigs is mutated by the others above.
+	conf.TLS.Certificates = getTLSConfig(tlsConfigs)
+
 	for _, middleware := range client.GetMiddlewares() {
 		id := provider.Normalize(makeID(middleware.Namespace, middleware.Name))
 		ctxMid := log.With(ctx, log.Str(log.MiddlewareName, id))
@@ -828,57 +835,86 @@ func buildTLSOptions(ctx context.Context, client Client) map[string]tls.Options
 	return tlsOptions
 }
 
-func buildTLSStores(ctx context.Context, client Client) map[string]tls.Store {
+func buildTLSStores(ctx context.Context, client Client) (map[string]tls.Store, map[string]*tls.CertAndStores) {
 	tlsStoreCRD := client.GetTLSStores()
-	var tlsStores map[string]tls.Store
-
 	if len(tlsStoreCRD) == 0 {
-		return tlsStores
+		return nil, nil
 	}
-	tlsStores = make(map[string]tls.Store)
+
 	var nsDefault []string
+	tlsStores := make(map[string]tls.Store)
+	tlsConfigs := make(map[string]*tls.CertAndStores)
 
-	for _, tlsStore := range tlsStoreCRD {
+	for _, t := range tlsStoreCRD {
-		namespace := tlsStore.Namespace
+		logger := log.FromContext(log.With(ctx, log.Str("TLSStore", t.Name), log.Str("namespace", t.Namespace)))
-		secretName := tlsStore.Spec.DefaultCertificate.SecretName
-		logger := log.FromContext(log.With(ctx, log.Str("tlsStore", tlsStore.Name), log.Str("namespace", namespace), log.Str("secretName", secretName)))
 
-		secret, exists, err := client.GetSecret(namespace, secretName)
+		id := makeID(t.Namespace, t.Name)
+
+		// If the name is default, we override the default config.
+		if t.Name == tls.DefaultTLSStoreName {
+			id = t.Name
+			nsDefault = append(nsDefault, t.Namespace)
+		}
+
+		var tlsStore tls.Store
+
+		if t.Spec.DefaultCertificate != nil {
+			secretName := t.Spec.DefaultCertificate.SecretName
+
+			secret, exists, err := client.GetSecret(t.Namespace, secretName)
 			if err != nil {
-			logger.Errorf("Failed to fetch secret %s/%s: %v", namespace, secretName, err)
+				logger.Errorf("Failed to fetch secret %s/%s: %v", t.Namespace, secretName, err)
 				continue
 			}
 			if !exists {
-			logger.Errorf("Secret %s/%s does not exist", namespace, secretName)
+				logger.Errorf("Secret %s/%s does not exist", t.Namespace, secretName)
 				continue
 			}
 
-		cert, key, err := getCertificateBlocks(secret, namespace, secretName)
+			cert, key, err := getCertificateBlocks(secret, t.Namespace, secretName)
 			if err != nil {
 				logger.Errorf("Could not get certificate blocks: %v", err)
 				continue
 			}
 
-		id := makeID(tlsStore.Namespace, tlsStore.Name)
+			tlsStore.DefaultCertificate = &tls.Certificate{
-		// If the name is default, we override the default config.
-		if tlsStore.Name == tls.DefaultTLSStoreName {
-			id = tlsStore.Name
-			nsDefault = append(nsDefault, tlsStore.Namespace)
-		}
-		tlsStores[id] = tls.Store{
-			DefaultCertificate: &tls.Certificate{
 				CertFile: tls.FileOrContent(cert),
 				KeyFile:  tls.FileOrContent(key),
-			},
 			}
 		}
 
+		if err := buildCertificates(client, id, t.Namespace, t.Spec.Certificates, tlsConfigs); err != nil {
+			logger.Errorf("Failed to load certificates: %v", err)
+			continue
+		}
+
+		tlsStores[id] = tlsStore
+	}
+
 	if len(nsDefault) > 1 {
 		delete(tlsStores, tls.DefaultTLSStoreName)
 		log.FromContext(ctx).Errorf("Default TLS Stores defined in multiple namespaces: %v", nsDefault)
 	}
 
-	return tlsStores
+	return tlsStores, tlsConfigs
+}
+
+// buildCertificates loads TLSStore certificates from secrets and sets them into tlsConfigs.
+func buildCertificates(client Client, tlsStore, namespace string, certificates []v1alpha1.Certificate, tlsConfigs map[string]*tls.CertAndStores) error {
+	for _, c := range certificates {
+		configKey := namespace + "/" + c.SecretName
+		if _, tlsExists := tlsConfigs[configKey]; !tlsExists {
+			certAndStores, err := getTLS(client, c.SecretName, namespace)
+			if err != nil {
+				return fmt.Errorf("unable to read secret %s: %w", configKey, err)
+			}
+
+			certAndStores.Stores = []string{tlsStore}
+			tlsConfigs[configKey] = certAndStores
+		}
+	}
+
+	return nil
 }
 
 func makeServiceKey(rule, ingressName string) (string, error) {

pkg/provider/kubernetes/crd/kubernetes_http.go

@@ -495,6 +495,7 @@ func namespaceOrFallback(lb v1alpha1.LoadBalancerSpec, fallback string) string {
 	return fallback
 }
 
+// getTLSHTTP mutates tlsConfigs.
 func getTLSHTTP(ctx context.Context, ingressRoute *v1alpha1.IngressRoute, k8sClient Client, tlsConfigs map[string]*tls.CertAndStores) error {
 	if ingressRoute.Spec.TLS == nil {
 		return nil

pkg/provider/kubernetes/crd/kubernetes_tcp.go

@@ -269,6 +269,7 @@ func (p *Provider) loadTCPServers(client Client, namespace string, svc v1alpha1.
 	return servers, nil
 }
 
+// getTLSTCP mutates tlsConfigs.
 func getTLSTCP(ctx context.Context, ingressRoute *v1alpha1.IngressRouteTCP, k8sClient Client, tlsConfigs map[string]*tls.CertAndStores) error {
 	if ingressRoute.Spec.TLS == nil {
 		return nil

pkg/provider/kubernetes/crd/kubernetes_test.go

@@ -3480,6 +3480,63 @@ func TestLoadIngressRoutes(t *testing.T) {
 				},
 			},
 		},
+		{
+			desc:  "TLS with tls store containing certificates",
+			paths: []string{"services.yml", "with_tls_store_certificates.yml"},
+			expected: &dynamic.Configuration{
+				TLS: &dynamic.TLSConfiguration{
+					Certificates: []*tls.CertAndStores{
+						{
+							Certificate: tls.Certificate{
+								CertFile: tls.FileOrContent("-----BEGIN CERTIFICATE-----\n-----END CERTIFICATE-----"),
+								KeyFile:  tls.FileOrContent("-----BEGIN PRIVATE KEY-----\n-----END PRIVATE KEY-----"),
+							},
+							Stores: []string{"default"},
+						},
+					},
+					Stores: map[string]tls.Store{
+						"default": {},
+					},
+				},
+				UDP: &dynamic.UDPConfiguration{
+					Routers:  map[string]*dynamic.UDPRouter{},
+					Services: map[string]*dynamic.UDPService{},
+				},
+				TCP: &dynamic.TCPConfiguration{
+					Routers:     map[string]*dynamic.TCPRouter{},
+					Middlewares: map[string]*dynamic.TCPMiddleware{},
+					Services:    map[string]*dynamic.TCPService{},
+				},
+				HTTP: &dynamic.HTTPConfiguration{
+					Routers: map[string]*dynamic.Router{
+						"default-test-route-6b204d94623b3df4370c": {
+							EntryPoints: []string{"web"},
+							Service:     "default-test-route-6b204d94623b3df4370c",
+							Rule:        "Host(`foo.com`) && PathPrefix(`/bar`)",
+							Priority:    12,
+							TLS:         &dynamic.RouterTLSConfig{},
+						},
+					},
+					Middlewares: map[string]*dynamic.Middleware{},
+					Services: map[string]*dynamic.Service{
+						"default-test-route-6b204d94623b3df4370c": {
+							LoadBalancer: &dynamic.ServersLoadBalancer{
+								Servers: []dynamic.Server{
+									{
+										URL: "http://10.10.0.1:80",
+									},
+									{
+										URL: "http://10.10.0.2:80",
+									},
+								},
+								PassHostHeader: Bool(true),
+							},
+						},
+					},
+					ServersTransports: map[string]*dynamic.ServersTransport{},
+				},
+			},
+		},
 		{
 			desc:  "TLS with tls store default two times",
 			paths: []string{"services.yml", "with_tls_store.yml", "with_default_tls_store.yml"},

pkg/provider/kubernetes/crd/traefik/v1alpha1/tlsstore.go

@@ -20,13 +20,16 @@ type TLSStore struct {
 
 // TLSStoreSpec configures a TLSStore resource.
 type TLSStoreSpec struct {
-	DefaultCertificate DefaultCertificate `json:"defaultCertificate"`
+	// DefaultCertificate is the name of the secret holding the default key/certificate pair for the store.
+	DefaultCertificate *Certificate `json:"defaultCertificate,omitempty"`
+	// Certificates is a list of secret names, each secret holding a key/certificate pair to add to the store.
+	Certificates []Certificate `json:"certificates,omitempty"`
 }
 
 // +k8s:deepcopy-gen=true
 
-// DefaultCertificate holds a secret name for the TLSOption resource.
+// Certificate holds a secret name for the TLSStore resource.
-type DefaultCertificate struct {
+type Certificate struct {
 	// SecretName is the name of the referenced Kubernetes Secret to specify the certificate details.
 	SecretName string `json:"secretName"`
 }

pkg/provider/kubernetes/crd/traefik/v1alpha1/zz_generated.deepcopy.go

@@ -53,6 +53,22 @@ func (in *BasicAuth) DeepCopy() *BasicAuth {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *Certificate) DeepCopyInto(out *Certificate) {
+	*out = *in
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Certificate.
+func (in *Certificate) DeepCopy() *Certificate {
+	if in == nil {
+		return nil
+	}
+	out := new(Certificate)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *Chain) DeepCopyInto(out *Chain) {
 	*out = *in
@@ -142,22 +158,6 @@ func (in *ClientTLS) DeepCopy() *ClientTLS {
 	return out
 }
 
-// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *DefaultCertificate) DeepCopyInto(out *DefaultCertificate) {
-	*out = *in
-	return
-}
-
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DefaultCertificate.
-func (in *DefaultCertificate) DeepCopy() *DefaultCertificate {
-	if in == nil {
-		return nil
-	}
-	out := new(DefaultCertificate)
-	in.DeepCopyInto(out)
-	return out
-}
-
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *DigestAuth) DeepCopyInto(out *DigestAuth) {
 	*out = *in
@@ -1413,7 +1413,7 @@ func (in *TLSStore) DeepCopyInto(out *TLSStore) {
 	*out = *in
 	out.TypeMeta = in.TypeMeta
 	in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
-	out.Spec = in.Spec
+	in.Spec.DeepCopyInto(&out.Spec)
 	return
 }
 
@@ -1487,7 +1487,16 @@ func (in *TLSStoreRef) DeepCopy() *TLSStoreRef {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *TLSStoreSpec) DeepCopyInto(out *TLSStoreSpec) {
 	*out = *in
-	out.DefaultCertificate = in.DefaultCertificate
+	if in.DefaultCertificate != nil {
+		in, out := &in.DefaultCertificate, &out.DefaultCertificate
+		*out = new(Certificate)
+		**out = **in
+	}
+	if in.Certificates != nil {
+		in, out := &in.Certificates, &out.Certificates
+		*out = make([]Certificate, len(*in))
+		copy(*out, *in)
+	}
 	return
 }