Allow empty services in Kubernetes CRD

docs/content/providers/kubernetes-crd.md

@@ -264,11 +264,38 @@ providers:
 --providers.kubernetescrd.throttleDuration=10s
 ```
 
+### `allowEmptyServices`
+
+_Optional, Default: false_
+
+If the parameter is set to `true`,
+it allows the creation of an empty [servers load balancer](../routing/services/index.md#servers-load-balancer) if the targeted Kubernetes service has no endpoints available.
+With IngressRoute resources,
+this results in `503` HTTP responses instead of `404` ones.
+
+```yaml tab="File (YAML)"
+providers:
+  kubernetesCRD:
+    allowEmptyServices: true
+    # ...
+```
+
+```toml tab="File (TOML)"
+[providers.kubernetesCRD]
+  allowEmptyServices = true
+  # ...
+```
+
+```bash tab="CLI"
+--providers.kubernetesCRD.allowEmptyServices=true
+```
+
 ### `allowCrossNamespace`
 
 _Optional, Default: false_
 
-If the parameter is set to `true`, IngressRoutes are  able to reference  resources in other namespaces than theirs.
+If the parameter is set to `true`,
+IngressRoute are able to reference resources in namespaces other than theirs.
 
 ```yaml tab="File (YAML)"
 providers:

docs/content/providers/kubernetes-ingress.md

@@ -445,7 +445,11 @@ providers:
 
 ### `allowEmptyServices`
 
-_Optional, Default: false
+_Optional, Default: false_
+
+If the parameter is set to `true`,
+it allows the creation of an empty [servers load balancer](../routing/services/index.md#servers-load-balancer) if the targeted Kubernetes service has no endpoints available.
+This results in `503` HTTP responses instead of `404` ones.
 
 ```yaml tab="File (YAML)"
 providers:
@@ -464,14 +468,12 @@ providers:
 --providers.kubernetesingress.allowEmptyServices=true
 ```
 
-Allow the creation of services if there are no endpoints available.
-This results in `503` http responses instead of `404`.
-
 ### `allowExternalNameServices`
 
 _Optional, Default: false_
 
-If the parameter is set to `true`, Ingresses are able to reference ExternalName services.
+If the parameter is set to `true`,
+Ingresses are able to reference ExternalName services.
 
 ```yaml tab="File (YAML)"
 providers:

docs/content/reference/static-configuration/cli-ref.md

@@ -648,6 +648,9 @@ Enable Kubernetes backend with default settings. (Default: ```false```)
 `--providers.kubernetescrd.allowcrossnamespace`:  
 Allow cross namespace resource reference. (Default: ```false```)
 
+`--providers.kubernetescrd.allowemptyservices`:  
+Allow the creation of services without endpoints. (Default: ```false```)
+
 `--providers.kubernetescrd.allowexternalnameservices`:  
 Allow ExternalName services. (Default: ```false```)
 

docs/content/reference/static-configuration/env-ref.md

@@ -648,6 +648,9 @@ Enable Kubernetes backend with default settings. (Default: ```false```)
 `TRAEFIK_PROVIDERS_KUBERNETESCRD_ALLOWCROSSNAMESPACE`:  
 Allow cross namespace resource reference. (Default: ```false```)
 
+`TRAEFIK_PROVIDERS_KUBERNETESCRD_ALLOWEMPTYSERVICES`:  
+Allow the creation of services without endpoints. (Default: ```false```)
+
 `TRAEFIK_PROVIDERS_KUBERNETESCRD_ALLOWEXTERNALNAMESERVICES`:  
 Allow ExternalName services. (Default: ```false```)
 

docs/content/reference/static-configuration/file.toml

@@ -121,6 +121,7 @@
     labelSelector = "foobar"
     ingressClass = "foobar"
     throttleDuration = 42
+    allowEmptyServices = true
   [providers.kubernetesGateway]
     endpoint = "foobar"
     token = "foobar"

docs/content/reference/static-configuration/file.yaml

@@ -131,6 +131,7 @@ providers:
     labelSelector: foobar
     ingressClass: foobar
     throttleDuration: 42s
+    allowEmptyServices: true
   kubernetesGateway:
     endpoint: foobar
     token: foobar

pkg/provider/kubernetes/crd/fixtures/services.yml

@@ -229,3 +229,26 @@ subsets:
     ports:
       - name: web
         port: 80
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: whoami-without-endpoints-subsets
+  namespace: default
+
+spec:
+  ports:
+    - name: myapp
+      port: 80
+
+  selector:
+    app: traefiklabs
+    task: whoami
+
+---
+kind: Endpoints
+apiVersion: v1
+metadata:
+  name: whoami-without-endpoints-subsets
+  namespace: default

pkg/provider/kubernetes/crd/fixtures/tcp/services.yml

@@ -238,3 +238,26 @@ subsets:
     ports:
       - name: myapp
         port: 8000
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: whoamitcp-without-endpoints-subsets
+  namespace: default
+
+spec:
+  ports:
+    - name: myapp
+      port: 8000
+
+  selector:
+    app: traefiklabs
+    task: whoamitcp
+
+---
+kind: Endpoints
+apiVersion: v1
+metadata:
+  name: whoamitcp-without-endpoints-subsets
+  namespace: default

pkg/provider/kubernetes/crd/fixtures/tcp/with_empty_services.yml

@@ -0,0 +1,15 @@
+apiVersion: traefik.containo.us/v1alpha1
+kind: IngressRouteTCP
+metadata:
+  name: test.route
+  namespace: default
+
+spec:
+  entryPoints:
+    - foo
+
+  routes:
+    - match: HostSNI(`foo.com`)
+      services:
+        - name: whoamitcp-without-endpoints-subsets
+          port: 8000

pkg/provider/kubernetes/crd/fixtures/udp/services.yml

@@ -197,3 +197,26 @@ subsets:
     ports:
       - name: myapp
         port: 8000
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: whoamiudp-without-endpoints-subsets
+  namespace: default
+
+spec:
+  ports:
+    - name: myapp
+      port: 8000
+
+  selector:
+    app: traefiklabs
+    task: whoamiudp
+
+---
+kind: Endpoints
+apiVersion: v1
+metadata:
+  name: whoamiudp-without-endpoints-subsets
+  namespace: default

pkg/provider/kubernetes/crd/fixtures/udp/with_empty_services.yml

@@ -0,0 +1,14 @@
+apiVersion: traefik.containo.us/v1alpha1
+kind: IngressRouteUDP
+metadata:
+  name: test.route
+  namespace: default
+
+spec:
+  entryPoints:
+    - foo
+
+  routes:
+    - services:
+        - name: whoamiudp-without-endpoints-subsets
+          port: 8000

pkg/provider/kubernetes/crd/fixtures/with_empty_services.yml

@@ -0,0 +1,17 @@
+apiVersion: traefik.containo.us/v1alpha1
+kind: IngressRoute
+metadata:
+  name: test.route
+  namespace: default
+
+spec:
+  entryPoints:
+    - foo
+
+  routes:
+    - match: Host(`foo.com`) && PathPrefix(`/bar`)
+      kind: Rule
+      priority: 12
+      services:
+        - name: whoami-without-endpoints-subsets
+          port: 80

pkg/provider/kubernetes/crd/kubernetes.go

@@ -53,6 +53,7 @@ type Provider struct {
 	LabelSelector             string          `description:"Kubernetes label selector to use." json:"labelSelector,omitempty" toml:"labelSelector,omitempty" yaml:"labelSelector,omitempty" export:"true"`
 	IngressClass              string          `description:"Value of kubernetes.io/ingress.class annotation to watch for." json:"ingressClass,omitempty" toml:"ingressClass,omitempty" yaml:"ingressClass,omitempty" export:"true"`
 	ThrottleDuration          ptypes.Duration `description:"Ingress refresh throttle duration" json:"throttleDuration,omitempty" toml:"throttleDuration,omitempty" yaml:"throttleDuration,omitempty" export:"true"`
+	AllowEmptyServices        bool            `description:"Allow the creation of services without endpoints." json:"allowEmptyServices,omitempty" toml:"allowEmptyServices,omitempty" yaml:"allowEmptyServices,omitempty" export:"true"`
 	lastConfiguration         safe.Safe
 }
 

pkg/provider/kubernetes/crd/kubernetes_http.go

@@ -50,7 +50,12 @@ func (p *Provider) loadIngressRouteConfiguration(ctx context.Context, client Cli
 			ingressName = ingressRoute.GenerateName
 		}
 
-		cb := configBuilder{client: client, allowCrossNamespace: p.AllowCrossNamespace, allowExternalNameServices: p.AllowExternalNameServices}
+		cb := configBuilder{
+			client:                    client,
+			allowCrossNamespace:       p.AllowCrossNamespace,
+			allowExternalNameServices: p.AllowExternalNameServices,
+			allowEmptyServices:        p.AllowEmptyServices,
+		}
 
 		for _, route := range ingressRoute.Spec.Routes {
 			if route.Kind != "Rule" {
@@ -193,6 +198,7 @@ type configBuilder struct {
 	client                    Client
 	allowCrossNamespace       bool
 	allowExternalNameServices bool
+	allowEmptyServices        bool
 }
 
 // buildTraefikService creates the configuration for the traefik service defined in tService,
@@ -387,7 +393,8 @@ func (c configBuilder) loadServers(parentNamespace string, svc v1alpha1.LoadBala
 	if !endpointsExists {
 		return nil, fmt.Errorf("endpoints not found for %s/%s", namespace, sanitizedName)
 	}
-	if len(endpoints.Subsets) == 0 {
+
+	if len(endpoints.Subsets) == 0 && !c.allowEmptyServices {
 		return nil, fmt.Errorf("subset not found for %s/%s", namespace, sanitizedName)
 	}
 

pkg/provider/kubernetes/crd/kubernetes_tcp.go

@@ -240,7 +240,7 @@ func (p *Provider) loadTCPServers(client Client, namespace string, svc v1alpha1.
 			return nil, errors.New("endpoints not found")
 		}
 
-		if len(endpoints.Subsets) == 0 {
+		if len(endpoints.Subsets) == 0 && !p.AllowEmptyServices {
 			return nil, errors.New("subset not found")
 		}
 

pkg/provider/kubernetes/crd/kubernetes_test.go

@@ -32,10 +32,11 @@ func Bool(v bool) *bool { return &v }
 
 func TestLoadIngressRouteTCPs(t *testing.T) {
 	testCases := []struct {
-		desc         string
+		desc               string
-		ingressClass string
+		ingressClass       string
-		paths        []string
+		paths              []string
-		expected     *dynamic.Configuration
+		allowEmptyServices bool
+		expected           *dynamic.Configuration
 	}{
 		{
 			desc: "Empty",
@@ -1358,6 +1359,67 @@ func TestLoadIngressRouteTCPs(t *testing.T) {
 				},
 			},
 		},
+		{
+			desc:  "Ingress Route, empty service disallowed",
+			paths: []string{"tcp/services.yml", "tcp/with_empty_services.yml"},
+			expected: &dynamic.Configuration{
+				UDP: &dynamic.UDPConfiguration{
+					Routers:  map[string]*dynamic.UDPRouter{},
+					Services: map[string]*dynamic.UDPService{},
+				},
+				TCP: &dynamic.TCPConfiguration{
+					Routers: map[string]*dynamic.TCPRouter{
+						"default-test.route-fdd3e9338e47a45efefc": {
+							EntryPoints: []string{"foo"},
+							Service:     "default-test.route-fdd3e9338e47a45efefc",
+							Rule:        "HostSNI(`foo.com`)",
+						},
+					},
+					Middlewares: map[string]*dynamic.TCPMiddleware{},
+					Services:    map[string]*dynamic.TCPService{},
+				},
+				HTTP: &dynamic.HTTPConfiguration{
+					Routers:           map[string]*dynamic.Router{},
+					Middlewares:       map[string]*dynamic.Middleware{},
+					Services:          map[string]*dynamic.Service{},
+					ServersTransports: map[string]*dynamic.ServersTransport{},
+				},
+				TLS: &dynamic.TLSConfiguration{},
+			},
+		},
+		{
+			desc:               "Ingress Route, empty service allowed",
+			allowEmptyServices: true,
+			paths:              []string{"tcp/services.yml", "tcp/with_empty_services.yml"},
+			expected: &dynamic.Configuration{
+				UDP: &dynamic.UDPConfiguration{
+					Routers:  map[string]*dynamic.UDPRouter{},
+					Services: map[string]*dynamic.UDPService{},
+				},
+				TCP: &dynamic.TCPConfiguration{
+					Routers: map[string]*dynamic.TCPRouter{
+						"default-test.route-fdd3e9338e47a45efefc": {
+							EntryPoints: []string{"foo"},
+							Service:     "default-test.route-fdd3e9338e47a45efefc",
+							Rule:        "HostSNI(`foo.com`)",
+						},
+					},
+					Middlewares: map[string]*dynamic.TCPMiddleware{},
+					Services: map[string]*dynamic.TCPService{
+						"default-test.route-fdd3e9338e47a45efefc": {
+							LoadBalancer: &dynamic.TCPServersLoadBalancer{},
+						},
+					},
+				},
+				HTTP: &dynamic.HTTPConfiguration{
+					Routers:           map[string]*dynamic.Router{},
+					Middlewares:       map[string]*dynamic.Middleware{},
+					Services:          map[string]*dynamic.Service{},
+					ServersTransports: map[string]*dynamic.ServersTransport{},
+				},
+				TLS: &dynamic.TLSConfiguration{},
+			},
+		},
 	}
 
 	for _, test := range testCases {
@@ -1370,7 +1432,12 @@ func TestLoadIngressRouteTCPs(t *testing.T) {
 				return
 			}
 
-			p := Provider{IngressClass: test.ingressClass, AllowCrossNamespace: true, AllowExternalNameServices: true}
+			p := Provider{
+				IngressClass:              test.ingressClass,
+				AllowCrossNamespace:       true,
+				AllowExternalNameServices: true,
+				AllowEmptyServices:        test.allowEmptyServices,
+			}
 
 			clientMock := newClientMock(test.paths...)
 			conf := p.loadConfigurationFromCRD(context.Background(), clientMock)
@@ -1385,7 +1452,8 @@ func TestLoadIngressRoutes(t *testing.T) {
 		ingressClass        string
 		paths               []string
 		expected            *dynamic.Configuration
-		AllowCrossNamespace bool
+		allowCrossNamespace bool
+		allowEmptyServices  bool
 	}{
 		{
 			desc: "Empty",
@@ -1453,7 +1521,7 @@ func TestLoadIngressRoutes(t *testing.T) {
 		},
 		{
 			desc:                "Simple Ingress Route with middleware",
-			AllowCrossNamespace: true,
+			allowCrossNamespace: true,
 			paths:               []string{"services.yml", "with_middleware.yml"},
 			expected: &dynamic.Configuration{
 				UDP: &dynamic.UDPConfiguration{
@@ -1521,7 +1589,7 @@ func TestLoadIngressRoutes(t *testing.T) {
 		},
 		{
 			desc:                "Middlewares in ingress route config are normalized",
-			AllowCrossNamespace: true,
+			allowCrossNamespace: true,
 			paths:               []string{"services.yml", "with_middleware_multiple_hyphens.yml"},
 			expected: &dynamic.Configuration{
 				UDP: &dynamic.UDPConfiguration{
@@ -1572,7 +1640,7 @@ func TestLoadIngressRoutes(t *testing.T) {
 		},
 		{
 			desc:                "Simple Ingress Route with middleware crossprovider",
-			AllowCrossNamespace: true,
+			allowCrossNamespace: true,
 			paths:               []string{"services.yml", "with_middleware_crossprovider.yml"},
 			expected: &dynamic.Configuration{
 				UDP: &dynamic.UDPConfiguration{
@@ -2142,7 +2210,7 @@ func TestLoadIngressRoutes(t *testing.T) {
 		},
 		{
 			desc:                "services lb, servers lb, and mirror service, all in a wrr with different namespaces",
-			AllowCrossNamespace: true,
+			allowCrossNamespace: true,
 			paths:               []string{"with_namespaces.yml"},
 			expected: &dynamic.Configuration{
 				UDP: &dynamic.UDPConfiguration{
@@ -2848,7 +2916,7 @@ func TestLoadIngressRoutes(t *testing.T) {
 		{
 			desc:                "TLS with tls options and specific namespace",
 			paths:               []string{"services.yml", "with_tls_options_and_specific_namespace.yml"},
-			AllowCrossNamespace: true,
+			allowCrossNamespace: true,
 			expected: &dynamic.Configuration{
 				UDP: &dynamic.UDPConfiguration{
 					Routers:  map[string]*dynamic.UDPRouter{},
@@ -3043,7 +3111,7 @@ func TestLoadIngressRoutes(t *testing.T) {
 		{
 			desc:                "TLS with unknown tls options namespace",
 			paths:               []string{"services.yml", "with_unknown_tls_options_namespace.yml"},
-			AllowCrossNamespace: true,
+			allowCrossNamespace: true,
 			expected: &dynamic.Configuration{
 				UDP: &dynamic.UDPConfiguration{
 					Routers:  map[string]*dynamic.UDPRouter{},
@@ -3697,6 +3765,64 @@ func TestLoadIngressRoutes(t *testing.T) {
 				TLS: &dynamic.TLSConfiguration{},
 			},
 		},
+		{
+			desc:  "Ingress Route, empty service disallowed",
+			paths: []string{"services.yml", "with_empty_services.yml"},
+			expected: &dynamic.Configuration{
+				UDP: &dynamic.UDPConfiguration{
+					Routers:  map[string]*dynamic.UDPRouter{},
+					Services: map[string]*dynamic.UDPService{},
+				},
+				TCP: &dynamic.TCPConfiguration{
+					Routers:     map[string]*dynamic.TCPRouter{},
+					Middlewares: map[string]*dynamic.TCPMiddleware{},
+					Services:    map[string]*dynamic.TCPService{},
+				},
+				HTTP: &dynamic.HTTPConfiguration{
+					Routers:           map[string]*dynamic.Router{},
+					Middlewares:       map[string]*dynamic.Middleware{},
+					Services:          map[string]*dynamic.Service{},
+					ServersTransports: map[string]*dynamic.ServersTransport{},
+				},
+				TLS: &dynamic.TLSConfiguration{},
+			},
+		},
+		{
+			desc:               "Ingress Route, empty service allowed",
+			allowEmptyServices: true,
+			paths:              []string{"services.yml", "with_empty_services.yml"},
+			expected: &dynamic.Configuration{
+				UDP: &dynamic.UDPConfiguration{
+					Routers:  map[string]*dynamic.UDPRouter{},
+					Services: map[string]*dynamic.UDPService{},
+				},
+				TCP: &dynamic.TCPConfiguration{
+					Routers:     map[string]*dynamic.TCPRouter{},
+					Middlewares: map[string]*dynamic.TCPMiddleware{},
+					Services:    map[string]*dynamic.TCPService{},
+				},
+				HTTP: &dynamic.HTTPConfiguration{
+					Routers: map[string]*dynamic.Router{
+						"default-test-route-6b204d94623b3df4370c": {
+							EntryPoints: []string{"foo"},
+							Service:     "default-test-route-6b204d94623b3df4370c",
+							Rule:        "Host(`foo.com`) && PathPrefix(`/bar`)",
+							Priority:    12,
+						},
+					},
+					Middlewares: map[string]*dynamic.Middleware{},
+					Services: map[string]*dynamic.Service{
+						"default-test-route-6b204d94623b3df4370c": {
+							LoadBalancer: &dynamic.ServersLoadBalancer{
+								PassHostHeader: Bool(true),
+							},
+						},
+					},
+					ServersTransports: map[string]*dynamic.ServersTransport{},
+				},
+				TLS: &dynamic.TLSConfiguration{},
+			},
+		},
 	}
 
 	for _, test := range testCases {
@@ -3708,7 +3834,12 @@ func TestLoadIngressRoutes(t *testing.T) {
 				return
 			}
 
-			p := Provider{IngressClass: test.ingressClass, AllowCrossNamespace: test.AllowCrossNamespace, AllowExternalNameServices: true}
+			p := Provider{
+				IngressClass:              test.ingressClass,
+				AllowCrossNamespace:       test.allowCrossNamespace,
+				AllowExternalNameServices: true,
+				AllowEmptyServices:        test.allowEmptyServices,
+			}
 
 			clientMock := newClientMock(test.paths...)
 			conf := p.loadConfigurationFromCRD(context.Background(), clientMock)
@@ -3719,10 +3850,11 @@ func TestLoadIngressRoutes(t *testing.T) {
 
 func TestLoadIngressRouteUDPs(t *testing.T) {
 	testCases := []struct {
-		desc         string
+		desc               string
-		ingressClass string
+		ingressClass       string
-		paths        []string
+		paths              []string
-		expected     *dynamic.Configuration
+		allowEmptyServices bool
+		expected           *dynamic.Configuration
 	}{
 		{
 			desc: "Empty",
@@ -4113,6 +4245,65 @@ func TestLoadIngressRouteUDPs(t *testing.T) {
 				TLS: &dynamic.TLSConfiguration{},
 			},
 		},
+		{
+			desc:  "Ingress Route, empty service disallowed",
+			paths: []string{"udp/services.yml", "udp/with_empty_services.yml"},
+			expected: &dynamic.Configuration{
+				UDP: &dynamic.UDPConfiguration{
+					Routers: map[string]*dynamic.UDPRouter{
+						"default-test.route-0": {
+							EntryPoints: []string{"foo"},
+							Service:     "default-test.route-0",
+						},
+					},
+					Services: map[string]*dynamic.UDPService{},
+				},
+				TCP: &dynamic.TCPConfiguration{
+					Routers:     map[string]*dynamic.TCPRouter{},
+					Middlewares: map[string]*dynamic.TCPMiddleware{},
+					Services:    map[string]*dynamic.TCPService{},
+				},
+				HTTP: &dynamic.HTTPConfiguration{
+					Routers:           map[string]*dynamic.Router{},
+					Middlewares:       map[string]*dynamic.Middleware{},
+					Services:          map[string]*dynamic.Service{},
+					ServersTransports: map[string]*dynamic.ServersTransport{},
+				},
+				TLS: &dynamic.TLSConfiguration{},
+			},
+		},
+		{
+			desc:               "Ingress Route, empty service allowed",
+			allowEmptyServices: true,
+			paths:              []string{"udp/services.yml", "udp/with_empty_services.yml"},
+			expected: &dynamic.Configuration{
+				UDP: &dynamic.UDPConfiguration{
+					Routers: map[string]*dynamic.UDPRouter{
+						"default-test.route-0": {
+							EntryPoints: []string{"foo"},
+							Service:     "default-test.route-0",
+						},
+					},
+					Services: map[string]*dynamic.UDPService{
+						"default-test.route-0": {
+							LoadBalancer: &dynamic.UDPServersLoadBalancer{},
+						},
+					},
+				},
+				TCP: &dynamic.TCPConfiguration{
+					Routers:     map[string]*dynamic.TCPRouter{},
+					Middlewares: map[string]*dynamic.TCPMiddleware{},
+					Services:    map[string]*dynamic.TCPService{},
+				},
+				HTTP: &dynamic.HTTPConfiguration{
+					Routers:           map[string]*dynamic.Router{},
+					Middlewares:       map[string]*dynamic.Middleware{},
+					Services:          map[string]*dynamic.Service{},
+					ServersTransports: map[string]*dynamic.ServersTransport{},
+				},
+				TLS: &dynamic.TLSConfiguration{},
+			},
+		},
 	}
 
 	for _, test := range testCases {
@@ -4125,7 +4316,12 @@ func TestLoadIngressRouteUDPs(t *testing.T) {
 				return
 			}
 
-			p := Provider{IngressClass: test.ingressClass, AllowCrossNamespace: true, AllowExternalNameServices: true}
+			p := Provider{
+				IngressClass:              test.ingressClass,
+				AllowCrossNamespace:       true,
+				AllowExternalNameServices: true,
+				AllowEmptyServices:        test.allowEmptyServices,
+			}
 
 			clientMock := newClientMock(test.paths...)
 			conf := p.loadConfigurationFromCRD(context.Background(), clientMock)

pkg/provider/kubernetes/crd/kubernetes_udp.go

@@ -135,7 +135,7 @@ func (p *Provider) loadUDPServers(client Client, namespace string, svc v1alpha1.
 			return nil, errors.New("endpoints not found")
 		}
 
-		if len(endpoints.Subsets) == 0 {
+		if len(endpoints.Subsets) == 0 && !p.AllowEmptyServices {
 			return nil, errors.New("subset not found")
 		}