baalajimaestro/traefik
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

Allow to use internal node IPs for NodePort services

Browse source
This commit is contained in:
Joris Vergeer 2024-02-27 10:54:04 +01:00 committed by GitHub
parent 73769af0fe
commit c1ef742977
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
31 changed files with 813 additions and 51 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

49
docs/content/reference/dynamic-configuration/kubernetes-crd-definition-v1.yml
View file

@ -120,6 +120,13 @@ spec:
The Kubernetes Service itself does load-balance to the pods.
By default, NativeLB is false.
type: boolean
nodePortLB:
description: |-
NodePortLB controls, when creating the load-balancer,
whether the LB's children are directly the nodes internal IPs using the nodePort when the service type is NodePort.
It allows services to be reachable when Traefik runs externally from the Kubernetes cluster but within the same network of the nodes.
By default, NodePortLB is false.
type: boolean
passHostHeader:
description: |-
PassHostHeader defines whether the client Host header is forwarded to the upstream Kubernetes Service.
@ -401,6 +408,13 @@ spec:
The Kubernetes Service itself does load-balance to the pods.
By default, NativeLB is false.
type: boolean
nodePortLB:
description: |-
NodePortLB controls, when creating the load-balancer,
whether the LB's children are directly the nodes internal IPs using the nodePort when the service type is NodePort.
It allows services to be reachable when Traefik runs externally from the Kubernetes cluster but within the same network of the nodes.
By default, NodePortLB is false.
type: boolean
port:
anyOf:
- type: integer
@ -612,6 +626,13 @@ spec:
The Kubernetes Service itself does load-balance to the pods.
By default, NativeLB is false.
type: boolean
nodePortLB:
description: |-
NodePortLB controls, when creating the load-balancer,
whether the LB's children are directly the nodes internal IPs using the nodePort when the service type is NodePort.
It allows services to be reachable when Traefik runs externally from the Kubernetes cluster but within the same network of the nodes.
By default, NodePortLB is false.
type: boolean
port:
anyOf:
- type: integer
@ -916,6 +937,13 @@ spec:
The Kubernetes Service itself does load-balance to the pods.
By default, NativeLB is false.
type: boolean
nodePortLB:
description: |-
NodePortLB controls, when creating the load-balancer,
whether the LB's children are directly the nodes internal IPs using the nodePort when the service type is NodePort.
It allows services to be reachable when Traefik runs externally from the Kubernetes cluster but within the same network of the nodes.
By default, NodePortLB is false.
type: boolean
passHostHeader:
description: |-
PassHostHeader defines whether the client Host header is forwarded to the upstream Kubernetes Service.
@ -2304,6 +2332,13 @@ spec:
The Kubernetes Service itself does load-balance to the pods.
By default, NativeLB is false.
type: boolean
nodePortLB:
description: |-
NodePortLB controls, when creating the load-balancer,
whether the LB's children are directly the nodes internal IPs using the nodePort when the service type is NodePort.
It allows services to be reachable when Traefik runs externally from the Kubernetes cluster but within the same network of the nodes.
By default, NodePortLB is false.
type: boolean
passHostHeader:
description: |-
PassHostHeader defines whether the client Host header is forwarded to the upstream Kubernetes Service.
@ -2410,6 +2445,13 @@ spec:
The Kubernetes Service itself does load-balance to the pods.
By default, NativeLB is false.
type: boolean
nodePortLB:
description: |-
NodePortLB controls, when creating the load-balancer,
whether the LB's children are directly the nodes internal IPs using the nodePort when the service type is NodePort.
It allows services to be reachable when Traefik runs externally from the Kubernetes cluster but within the same network of the nodes.
By default, NodePortLB is false.
type: boolean
passHostHeader:
description: |-
PassHostHeader defines whether the client Host header is forwarded to the upstream Kubernetes Service.
@ -2524,6 +2566,13 @@ spec:
The Kubernetes Service itself does load-balance to the pods.
By default, NativeLB is false.
type: boolean
nodePortLB:
description: |-
NodePortLB controls, when creating the load-balancer,
whether the LB's children are directly the nodes internal IPs using the nodePort when the service type is NodePort.
It allows services to be reachable when Traefik runs externally from the Kubernetes cluster but within the same network of the nodes.
By default, NodePortLB is false.
type: boolean
passHostHeader:
description: |-
PassHostHeader defines whether the client Host header is forwarded to the upstream Kubernetes Service.

1
docs/content/reference/dynamic-configuration/kubernetes-crd-rbac.yml
View file

@ -10,6 +10,7 @@ rules:
- services
- endpoints
- secrets
- nodes
verbs:
- get
- list

7
docs/content/reference/dynamic-configuration/traefik.io_ingressroutes.yaml
View file

@ -120,6 +120,13 @@ spec:
The Kubernetes Service itself does load-balance to the pods.
By default, NativeLB is false.
type: boolean
nodePortLB:
description: |-
NodePortLB controls, when creating the load-balancer,
whether the LB's children are directly the nodes internal IPs using the nodePort when the service type is NodePort.
It allows services to be reachable when Traefik runs externally from the Kubernetes cluster but within the same network of the nodes.
By default, NodePortLB is false.
type: boolean
passHostHeader:
description: |-
PassHostHeader defines whether the client Host header is forwarded to the upstream Kubernetes Service.

7
docs/content/reference/dynamic-configuration/traefik.io_ingressroutetcps.yaml
View file

@ -103,6 +103,13 @@ spec:
The Kubernetes Service itself does load-balance to the pods.
By default, NativeLB is false.
type: boolean
nodePortLB:
description: |-
NodePortLB controls, when creating the load-balancer,
whether the LB's children are directly the nodes internal IPs using the nodePort when the service type is NodePort.
It allows services to be reachable when Traefik runs externally from the Kubernetes cluster but within the same network of the nodes.
By default, NodePortLB is false.
type: boolean
port:
anyOf:
- type: integer

7
docs/content/reference/dynamic-configuration/traefik.io_ingressrouteudps.yaml
View file

@ -74,6 +74,13 @@ spec:
The Kubernetes Service itself does load-balance to the pods.
By default, NativeLB is false.
type: boolean
nodePortLB:
description: |-
NodePortLB controls, when creating the load-balancer,
whether the LB's children are directly the nodes internal IPs using the nodePort when the service type is NodePort.
It allows services to be reachable when Traefik runs externally from the Kubernetes cluster but within the same network of the nodes.
By default, NodePortLB is false.
type: boolean
port:
anyOf:
- type: integer

7
docs/content/reference/dynamic-configuration/traefik.io_middlewares.yaml
View file

@ -274,6 +274,13 @@ spec:
The Kubernetes Service itself does load-balance to the pods.
By default, NativeLB is false.
type: boolean
nodePortLB:
description: |-
NodePortLB controls, when creating the load-balancer,
whether the LB's children are directly the nodes internal IPs using the nodePort when the service type is NodePort.
It allows services to be reachable when Traefik runs externally from the Kubernetes cluster but within the same network of the nodes.
By default, NodePortLB is false.
type: boolean
passHostHeader:
description: |-
PassHostHeader defines whether the client Host header is forwarded to the upstream Kubernetes Service.

21
docs/content/reference/dynamic-configuration/traefik.io_traefikservices.yaml
View file

@ -88,6 +88,13 @@ spec:
The Kubernetes Service itself does load-balance to the pods.
By default, NativeLB is false.
type: boolean
nodePortLB:
description: |-
NodePortLB controls, when creating the load-balancer,
whether the LB's children are directly the nodes internal IPs using the nodePort when the service type is NodePort.
It allows services to be reachable when Traefik runs externally from the Kubernetes cluster but within the same network of the nodes.
By default, NodePortLB is false.
type: boolean
passHostHeader:
description: |-
PassHostHeader defines whether the client Host header is forwarded to the upstream Kubernetes Service.
@ -194,6 +201,13 @@ spec:
The Kubernetes Service itself does load-balance to the pods.
By default, NativeLB is false.
type: boolean
nodePortLB:
description: |-
NodePortLB controls, when creating the load-balancer,
whether the LB's children are directly the nodes internal IPs using the nodePort when the service type is NodePort.
It allows services to be reachable when Traefik runs externally from the Kubernetes cluster but within the same network of the nodes.
By default, NodePortLB is false.
type: boolean
passHostHeader:
description: |-
PassHostHeader defines whether the client Host header is forwarded to the upstream Kubernetes Service.
@ -308,6 +322,13 @@ spec:
The Kubernetes Service itself does load-balance to the pods.
By default, NativeLB is false.
type: boolean
nodePortLB:
description: |-
NodePortLB controls, when creating the load-balancer,
whether the LB's children are directly the nodes internal IPs using the nodePort when the service type is NodePort.
It allows services to be reachable when Traefik runs externally from the Kubernetes cluster but within the same network of the nodes.
By default, NodePortLB is false.
type: boolean
passHostHeader:
description: |-
PassHostHeader defines whether the client Host header is forwarded to the upstream Kubernetes Service.

85
docs/content/routing/providers/kubernetes-crd.md
View file

@ -352,15 +352,16 @@ Register the `IngressRoute` [kind](../../reference/dynamic-configuration/kuberne
strategy: RoundRobin
weight: 10
nativeLB: true # [11]
tls: # [12]
secretName: supersecret # [13]
options: # [14]
name: opt # [15]
namespace: default # [16]
certResolver: foo # [17]
domains: # [18]
- main: example.net # [19]
sans: # [20]
nodePortLB: true # [12]
tls: # [13]
secretName: supersecret # [14]
options: # [15]
name: opt # [16]
namespace: default # [17]
certResolver: foo # [18]
domains: # [19]
- main: example.net # [20]
sans: # [21]
- a.example.net
- b.example.net
```
@ -378,15 +379,16 @@ Register the `IngressRoute` [kind](../../reference/dynamic-configuration/kuberne
| [9] | `services[n].port` | Defines the port of a [Kubernetes service](https://kubernetes.io/docs/concepts/services-networking/service/). This can be a reference to a named port. |
| [10] | `services[n].serversTransport` | Defines the reference to a [ServersTransport](#kind-serverstransport). The ServersTransport namespace is assumed to be the [Kubernetes service](https://kubernetes.io/docs/concepts/services-networking/service/) namespace (see [ServersTransport reference](#serverstransport-reference)). |
| [11] | `services[n].nativeLB` | Controls, when creating the load-balancer, whether the LB's children are directly the pods IPs or if the only child is the Kubernetes Service clusterIP. |
| [12] | `tls` | Defines [TLS](../routers/index.md#tls) certificate configuration |
| [13] | `tls.secretName` | Defines the [secret](https://kubernetes.io/docs/concepts/configuration/secret/) name used to store the certificate (in the `IngressRoute` namespace) |
| [14] | `tls.options` | Defines the reference to a [TLSOption](#kind-tlsoption) |
| [15] | `options.name` | Defines the [TLSOption](#kind-tlsoption) name |
| [16] | `options.namespace` | Defines the [TLSOption](#kind-tlsoption) namespace |
| [17] | `tls.certResolver` | Defines the reference to a [CertResolver](../routers/index.md#certresolver) |
| [18] | `tls.domains` | List of [domains](../routers/index.md#domains) |
| [19] | `domains[n].main` | Defines the main domain name |
| [20] | `domains[n].sans` | List of SANs (alternative domains) |
| [12] | `services[n].nodePortLB` | Controls, when creating the load-balancer, whether the LB's children are directly the nodes internal IPs using the nodePort when the service type is NodePort. |
| [13] | `tls` | Defines [TLS](../routers/index.md#tls) certificate configuration |
| [14] | `tls.secretName` | Defines the [secret](https://kubernetes.io/docs/concepts/configuration/secret/) name used to store the certificate (in the `IngressRoute` namespace) |
| [15] | `tls.options` | Defines the reference to a [TLSOption](#kind-tlsoption) |
| [16] | `options.name` | Defines the [TLSOption](#kind-tlsoption) name |
| [17] | `options.namespace` | Defines the [TLSOption](#kind-tlsoption) namespace |
| [18] | `tls.certResolver` | Defines the reference to a [CertResolver](../routers/index.md#certresolver) |
| [19] | `tls.domains` | List of [domains](../routers/index.md#domains) |
| [20] | `domains[n].main` | Defines the main domain name |
| [21] | `domains[n].sans` | List of SANs (alternative domains) |
??? example "Declaring an IngressRoute"
@ -1149,18 +1151,20 @@ Register the `IngressRouteTCP` [kind](../../reference/dynamic-configuration/kube
version: 1 # [12]
serversTransport: transport # [13]
nativeLB: true # [14]
tls: # [15]
secretName: supersecret # [16]
options: # [17]
name: opt # [18]
namespace: default # [19]
certResolver: foo # [20]
domains: # [21]
- main: example.net # [22]
sans: # [23]
nodePortLB: true # [15]
tls: # [16]
secretName: supersecret # [17]
options: # [18]
name: opt # [19]
namespace: default # [20]
certResolver: foo # [21]
domains: # [22]
- main: example.net # [23]
sans: # [24]
- a.example.net
- b.example.net
passthrough: false # [24]
passthrough: false # [25]
```
| Ref | Attribute | Purpose |
@ -1179,16 +1183,17 @@ Register the `IngressRouteTCP` [kind](../../reference/dynamic-configuration/kube
| [12] | `services[n].proxyProtocol.version` | Defines the [PROXY protocol](../services/index.md#proxy-protocol) version |
| [13] | `services[n].serversTransport` | Defines the reference to a [ServersTransportTCP](#kind-serverstransporttcp). The ServersTransport namespace is assumed to be the [Kubernetes service](https://kubernetes.io/docs/concepts/services-networking/service/) namespace (see [ServersTransport reference](#serverstransport-reference)). |
| [14] | `services[n].nativeLB` | Controls, when creating the load-balancer, whether the LB's children are directly the pods IPs or if the only child is the Kubernetes Service clusterIP. |
| [15] | `tls` | Defines [TLS](../routers/index.md#tls_1) certificate configuration |
| [16] | `tls.secretName` | Defines the [secret](https://kubernetes.io/docs/concepts/configuration/secret/) name used to store the certificate (in the `IngressRoute` namespace) |
| [17] | `tls.options` | Defines the reference to a [TLSOption](#kind-tlsoption) |
| [18] | `tls.options.name` | Defines the [TLSOption](#kind-tlsoption) name |
| [19] | `tls.options.namespace` | Defines the [TLSOption](#kind-tlsoption) namespace |
| [20] | `tls.certResolver` | Defines the reference to a [CertResolver](../routers/index.md#certresolver_1) |
| [21] | `tls.domains` | List of [domains](../routers/index.md#domains_1) |
| [22] | `tls.domains[n].main` | Defines the main domain name |
| [23] | `tls.domains[n].sans` | List of SANs (alternative domains) |
| [24] | `tls.passthrough` | If `true`, delegates the TLS termination to the backend |
| [15] | `services[n].nodePortLB` | Controls, when creating the load-balancer, whether the LB's children are directly the nodes internal IPs using the nodePort when the service type is |
| [16] | `tls` | Defines [TLS](../routers/index.md#tls_1) certificate configuration |
| [17] | `tls.secretName` | Defines the [secret](https://kubernetes.io/docs/concepts/configuration/secret/) name used to store the certificate (in the `IngressRoute` namespace) |
| [18] | `tls.options` | Defines the reference to a [TLSOption](#kind-tlsoption) |
| [19] | `tls.options.name` | Defines the [TLSOption](#kind-tlsoption) name |
| [20] | `tls.options.namespace` | Defines the [TLSOption](#kind-tlsoption) namespace |
| [21] | `tls.certResolver` | Defines the reference to a [CertResolver](../routers/index.md#certresolver_1) |
| [22] | `tls.domains` | List of [domains](../routers/index.md#domains_1) |
| [23] | `tls.domains[n].main` | Defines the main domain name |
| [24] | `tls.domains[n].sans` | List of SANs (alternative domains) |
| [25] | `tls.passthrough` | If `true`, delegates the TLS termination to the backend |
??? example "Declaring an IngressRouteTCP"
@ -1433,10 +1438,11 @@ Register the `IngressRouteUDP` [kind](../../reference/dynamic-configuration/kube
port: 8080 # [5]
weight: 10 # [6]
nativeLB: true # [7]
nodePortLB: true # [8]
```
| Ref | Attribute | Purpose |
|-----|-------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------|
|-----|-------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------|
| [1] | `entryPoints` | List of [entrypoints](../routers/index.md#entrypoints_1) names |
| [2] | `routes` | List of routes |
| [3] | `routes[n].services` | List of [Kubernetes service](https://kubernetes.io/docs/concepts/services-networking/service/) definitions (See below for `ExternalName Service` setup) |
@ -1444,6 +1450,7 @@ Register the `IngressRouteUDP` [kind](../../reference/dynamic-configuration/kube
| [5] | `services[n].port` | Defines the port of a [Kubernetes service](https://kubernetes.io/docs/concepts/services-networking/service/). This can be a reference to a named port. |
| [6] | `services[n].weight` | Defines the weight to apply to the server load balancing |
| [7] | `services[n].nativeLB` | Controls, when creating the load-balancer, whether the LB's children are directly the pods IPs or if the only child is the Kubernetes Service clusterIP. |
| [8] | `services[n].nodePortLB` | Controls, when creating the load-balancer, whether the LB's children are directly the nodes internal IPs using the nodePort when the service type is NodePort. |
??? example "Declaring an IngressRouteUDP"

10
docs/content/routing/providers/kubernetes-ingress.md
View file

@ -287,6 +287,16 @@ which in turn will create the resulting routers, services, handlers, etc.
traefik.ingress.kubernetes.io/service.nativelb: "true"
```
??? info "`traefik.ingress.kubernetes.io/service.nodeportlb`"
Controls, when creating the load-balancer, whether the LB's children are directly the nodes internal IPs using the nodePort when the service type is NodePort.
It allows services to be reachable when Traefik runs externally from the Kubernetes cluster but within the same network of the nodes.
By default, NodePortLB is false.
```yaml
traefik.ingress.kubernetes.io/service.nodeportlb: "true"
```
??? info "`traefik.ingress.kubernetes.io/service.serversscheme`"
Overrides the default scheme.

49
integration/fixtures/k8s/01-traefik-crd.yml
View file

@ -120,6 +120,13 @@ spec:
The Kubernetes Service itself does load-balance to the pods.
By default, NativeLB is false.
type: boolean
nodePortLB:
description: |-
NodePortLB controls, when creating the load-balancer,
whether the LB's children are directly the nodes internal IPs using the nodePort when the service type is NodePort.
It allows services to be reachable when Traefik runs externally from the Kubernetes cluster but within the same network of the nodes.
By default, NodePortLB is false.
type: boolean
passHostHeader:
description: |-
PassHostHeader defines whether the client Host header is forwarded to the upstream Kubernetes Service.
@ -401,6 +408,13 @@ spec:
The Kubernetes Service itself does load-balance to the pods.
By default, NativeLB is false.
type: boolean
nodePortLB:
description: |-
NodePortLB controls, when creating the load-balancer,
whether the LB's children are directly the nodes internal IPs using the nodePort when the service type is NodePort.
It allows services to be reachable when Traefik runs externally from the Kubernetes cluster but within the same network of the nodes.
By default, NodePortLB is false.
type: boolean
port:
anyOf:
- type: integer
@ -612,6 +626,13 @@ spec:
The Kubernetes Service itself does load-balance to the pods.
By default, NativeLB is false.
type: boolean
nodePortLB:
description: |-
NodePortLB controls, when creating the load-balancer,
whether the LB's children are directly the nodes internal IPs using the nodePort when the service type is NodePort.
It allows services to be reachable when Traefik runs externally from the Kubernetes cluster but within the same network of the nodes.
By default, NodePortLB is false.
type: boolean
port:
anyOf:
- type: integer
@ -916,6 +937,13 @@ spec:
The Kubernetes Service itself does load-balance to the pods.
By default, NativeLB is false.
type: boolean
nodePortLB:
description: |-
NodePortLB controls, when creating the load-balancer,
whether the LB's children are directly the nodes internal IPs using the nodePort when the service type is NodePort.
It allows services to be reachable when Traefik runs externally from the Kubernetes cluster but within the same network of the nodes.
By default, NodePortLB is false.
type: boolean
passHostHeader:
description: |-
PassHostHeader defines whether the client Host header is forwarded to the upstream Kubernetes Service.
@ -2304,6 +2332,13 @@ spec:
The Kubernetes Service itself does load-balance to the pods.
By default, NativeLB is false.
type: boolean
nodePortLB:
description: |-
NodePortLB controls, when creating the load-balancer,
whether the LB's children are directly the nodes internal IPs using the nodePort when the service type is NodePort.
It allows services to be reachable when Traefik runs externally from the Kubernetes cluster but within the same network of the nodes.
By default, NodePortLB is false.
type: boolean
passHostHeader:
description: |-
PassHostHeader defines whether the client Host header is forwarded to the upstream Kubernetes Service.
@ -2410,6 +2445,13 @@ spec:
The Kubernetes Service itself does load-balance to the pods.
By default, NativeLB is false.
type: boolean
nodePortLB:
description: |-
NodePortLB controls, when creating the load-balancer,
whether the LB's children are directly the nodes internal IPs using the nodePort when the service type is NodePort.
It allows services to be reachable when Traefik runs externally from the Kubernetes cluster but within the same network of the nodes.
By default, NodePortLB is false.
type: boolean
passHostHeader:
description: |-
PassHostHeader defines whether the client Host header is forwarded to the upstream Kubernetes Service.
@ -2524,6 +2566,13 @@ spec:
The Kubernetes Service itself does load-balance to the pods.
By default, NativeLB is false.
type: boolean
nodePortLB:
description: |-
NodePortLB controls, when creating the load-balancer,
whether the LB's children are directly the nodes internal IPs using the nodePort when the service type is NodePort.
It allows services to be reachable when Traefik runs externally from the Kubernetes cluster but within the same network of the nodes.
By default, NodePortLB is false.
type: boolean
passHostHeader:
description: |-
PassHostHeader defines whether the client Host header is forwarded to the upstream Kubernetes Service.

21
pkg/provider/kubernetes/crd/client.go
View file

@ -46,6 +46,7 @@ type Client interface {
GetService(namespace, name string) (*corev1.Service, bool, error)
GetSecret(namespace, name string) (*corev1.Secret, bool, error)
GetEndpoints(namespace, name string) (*corev1.Endpoints, bool, error)
GetNodes() ([]*corev1.Node, bool, error)
}
// TODO: add tests for the clientWrapper (and its methods) itself.
@ -53,6 +54,7 @@ type clientWrapper struct {
csCrd traefikclientset.Interface
csKube kclientset.Interface
factoryClusterScope kinformers.SharedInformerFactory
factoriesCrd map[string]traefikinformers.SharedInformerFactory
factoriesKube map[string]kinformers.SharedInformerFactory
factoriesSecret map[string]kinformers.SharedInformerFactory
@ -232,11 +234,18 @@ func (c *clientWrapper) WatchAll(namespaces []string, stopCh <-chan struct{}) (<
c.factoriesSecret[ns] = factorySecret
}
c.factoryClusterScope = kinformers.NewSharedInformerFactory(c.csKube, resyncPeriod)
_, err := c.factoryClusterScope.Core().V1().Nodes().Informer().AddEventHandler(eventHandler)
if err != nil {
return nil, err
}
for _, ns := range namespaces {
c.factoriesCrd[ns].Start(stopCh)
c.factoriesKube[ns].Start(stopCh)
c.factoriesSecret[ns].Start(stopCh)
}
c.factoryClusterScope.Start(stopCh)
for _, ns := range namespaces {
for t, ok := range c.factoriesCrd[ns].WaitForCacheSync(stopCh) {
@ -258,6 +267,12 @@ func (c *clientWrapper) WatchAll(namespaces []string, stopCh <-chan struct{}) (<
}
}
for t, ok := range c.factoryClusterScope.WaitForCacheSync(stopCh) {
if !ok {
return nil, fmt.Errorf("timed out waiting for controller caches to sync %s", t.String())
}
}
return eventCh, nil
}
@ -450,6 +465,12 @@ func (c *clientWrapper) GetSecret(namespace, name string) (*corev1.Secret, bool,
return secret, exist, err
}
func (c *clientWrapper) GetNodes() ([]*corev1.Node, bool, error) {
nodes, err := c.factoryClusterScope.Core().V1().Nodes().Lister().List(labels.Everything())
exist, err := translateNotFoundError(err)
return nodes, exist, err
}
// lookupNamespace returns the lookup namespace key for the given namespace.
// When listening on all namespaces, it returns the client-go identifier ("")
// for all-namespaces. Otherwise, it returns the given namespace.

15
pkg/provider/kubernetes/crd/fixtures/services.yml
View file

@ -266,3 +266,18 @@ spec:
port: 80
type: ClusterIP
clusterIP: 10.10.0.1
---
apiVersion: v1
kind: Service
metadata:
name: nodeport-svc
namespace: default
spec:
ports:
- name: web
port: 80
nodePort: 32456
type: NodePort
clusterIP: 10.10.0.1

15
pkg/provider/kubernetes/crd/fixtures/tcp/services.yml
View file

@ -262,3 +262,18 @@ spec:
port: 8000
type: ClusterIP
clusterIP: 10.10.0.1
---
apiVersion: v1
kind: Service
metadata:
name: nodeport-svc-tcp
namespace: default
spec:
ports:
- name: myapp
port: 8000
nodePort: 32456
type: NodePort
clusterIP: 10.10.0.1

26
pkg/provider/kubernetes/crd/fixtures/tcp/with_node_port_service_lb.yml Normal file
View file

@ -0,0 +1,26 @@
apiVersion: traefik.io/v1alpha1
kind: IngressRouteTCP
metadata:
name: test.route
namespace: default
spec:
entryPoints:
- foo
routes:
- match: HostSNI(`foo.com`)
services:
- name: nodeport-svc-tcp
port: 8000
nodePortLB: true
---
kind: Node
apiVersion: v1
metadata:
name: traefik-node
status:
addresses:
- type: InternalIP
address: 172.16.4.4

15
pkg/provider/kubernetes/crd/fixtures/udp/services.yml
View file

@ -221,3 +221,18 @@ spec:
port: 8000
type: ClusterIP
clusterIP: 10.10.0.1
---
apiVersion: v1
kind: Service
metadata:
name: nodeport-svc-udp
namespace: default
spec:
ports:
- name: myapp
port: 8000
nodePort: 32456
type: NodePort
clusterIP: 10.10.0.1

25
pkg/provider/kubernetes/crd/fixtures/udp/with_node_port_service_lb.yml Normal file
View file

@ -0,0 +1,25 @@
apiVersion: traefik.io/v1alpha1
kind: IngressRouteUDP
metadata:
name: test.route
namespace: default
spec:
entryPoints:
- foo
routes:
- services:
- name: nodeport-svc-udp
port: 8000
nodePortLB: true
---
kind: Node
apiVersion: v1
metadata:
name: traefik-node
status:
addresses:
- type: InternalIP
address: 172.16.4.4

27
pkg/provider/kubernetes/crd/fixtures/with_node_port_lb.yml Normal file
View file

@ -0,0 +1,27 @@
apiVersion: traefik.io/v1alpha1
kind: IngressRoute
metadata:
name: test.route
namespace: default
spec:
entryPoints:
- foo
routes:
- match: Host(`foo.com`)
kind: Rule
services:
- name: nodeport-svc
port: 80
nodePortLB: true
---
kind: Node
apiVersion: v1
metadata:
name: traefik-node
status:
addresses:
- type: InternalIP
address: 172.16.4.4

33
pkg/provider/kubernetes/crd/kubernetes_http.go
View file

@ -409,6 +409,39 @@ func (c configBuilder) loadServers(parentNamespace string, svc traefikv1alpha1.L
}), nil
}
if service.Spec.Type == corev1.ServiceTypeNodePort && svc.NodePortLB {
nodes, nodesExists, nodesErr := c.client.GetNodes()
if nodesErr != nil {
return nil, nodesErr
}
if !nodesExists || len(nodes) == 0 {
return nil, fmt.Errorf("nodes not found for NodePort service %s/%s", namespace, sanitizedName)
}
protocol, err := parseServiceProtocol(svc.Scheme, svcPort.Name, svcPort.Port)
if err != nil {
return nil, err
}
for _, node := range nodes {
for _, addr := range node.Status.Addresses {
if addr.Type == corev1.NodeInternalIP {
hostPort := net.JoinHostPort(addr.Address, strconv.Itoa(int(svcPort.NodePort)))
servers = append(servers, dynamic.Server{
URL: fmt.Sprintf("%s://%s", protocol, hostPort),
})
}
}
}
if len(servers) == 0 {
return nil, fmt.Errorf("no servers were generated for service %s in namespace", sanitizedName)
}
return servers, nil
}
endpoints, endpointsExists, endpointsErr := c.client.GetEndpoints(namespace, sanitizedName)
if endpointsErr != nil {
return nil, endpointsErr

28
pkg/provider/kubernetes/crd/kubernetes_tcp.go
View file

@ -247,6 +247,34 @@ func (p *Provider) loadTCPServers(client Client, namespace string, svc traefikv1
}
var servers []dynamic.TCPServer
if service.Spec.Type == corev1.ServiceTypeNodePort && svc.NodePortLB {
nodes, nodesExists, nodesErr := client.GetNodes()
if nodesErr != nil {
return nil, nodesErr
}
if !nodesExists || len(nodes) == 0 {
return nil, fmt.Errorf("nodes not found for NodePort service %s/%s", svc.Namespace, svc.Name)
}
for _, node := range nodes {
for _, addr := range node.Status.Addresses {
if addr.Type == corev1.NodeInternalIP {
servers = append(servers, dynamic.TCPServer{
Address: net.JoinHostPort(addr.Address, strconv.Itoa(int(svcPort.NodePort))),
})
}
}
}
if len(servers) == 0 {
return nil, fmt.Errorf("no servers were generated for service %s/%s", svc.Namespace, svc.Name)
}
return servers, nil
}
if service.Spec.Type == corev1.ServiceTypeExternalName {
servers = append(servers, dynamic.TCPServer{
Address: net.JoinHostPort(service.Spec.ExternalName, strconv.Itoa(int(svcPort.Port))),

183
pkg/provider/kubernetes/crd/kubernetes_test.go
View file

@ -7020,6 +7020,189 @@ func TestNativeLB(t *testing.T) {
}
}
func TestNodePortLB(t *testing.T) {
testCases := []struct {
desc string
ingressClass string
paths []string
expected *dynamic.Configuration
}{
{
desc: "Empty",
expected: &dynamic.Configuration{
UDP: &dynamic.UDPConfiguration{
Routers: map[string]*dynamic.UDPRouter{},
Services: map[string]*dynamic.UDPService{},
},
TCP: &dynamic.TCPConfiguration{
ServersTransports: map[string]*dynamic.TCPServersTransport{},
Routers: map[string]*dynamic.TCPRouter{},
Middlewares: map[string]*dynamic.TCPMiddleware{},
Services: map[string]*dynamic.TCPService{},
},
HTTP: &dynamic.HTTPConfiguration{
ServersTransports: map[string]*dynamic.ServersTransport{},
Routers: map[string]*dynamic.Router{},
Middlewares: map[string]*dynamic.Middleware{},
Services: map[string]*dynamic.Service{},
},
TLS: &dynamic.TLSConfiguration{},
},
},
{
desc: "HTTP with node port LB",
paths: []string{"services.yml", "with_node_port_lb.yml"},
expected: &dynamic.Configuration{
UDP: &dynamic.UDPConfiguration{
Routers: map[string]*dynamic.UDPRouter{},
Services: map[string]*dynamic.UDPService{},
},
TCP: &dynamic.TCPConfiguration{
ServersTransports: map[string]*dynamic.TCPServersTransport{},
Routers: map[string]*dynamic.TCPRouter{},
Middlewares: map[string]*dynamic.TCPMiddleware{},
Services: map[string]*dynamic.TCPService{},
},
HTTP: &dynamic.HTTPConfiguration{
ServersTransports: map[string]*dynamic.ServersTransport{},
Routers: map[string]*dynamic.Router{
"default-test-route-6f97418635c7e18853da": {
EntryPoints: []string{"foo"},
Service: "default-test-route-6f97418635c7e18853da",
Rule: "Host(`foo.com`)",
Priority: 0,
},
},
Middlewares: map[string]*dynamic.Middleware{},
Services: map[string]*dynamic.Service{
"default-test-route-6f97418635c7e18853da": {
LoadBalancer: &dynamic.ServersLoadBalancer{
ResponseForwarding: &dynamic.ResponseForwarding{FlushInterval: dynamic.DefaultFlushInterval},
Servers: []dynamic.Server{
{
URL: "http://172.16.4.4:32456",
},
},
PassHostHeader: Bool(true),
},
},
},
},
TLS: &dynamic.TLSConfiguration{},
},
},
{
desc: "TCP with native Service LB",
paths: []string{"tcp/services.yml", "tcp/with_node_port_service_lb.yml"},
expected: &dynamic.Configuration{
UDP: &dynamic.UDPConfiguration{
Routers: map[string]*dynamic.UDPRouter{},
Services: map[string]*dynamic.UDPService{},
},
HTTP: &dynamic.HTTPConfiguration{
ServersTransports: map[string]*dynamic.ServersTransport{},
Routers: map[string]*dynamic.Router{},
Middlewares: map[string]*dynamic.Middleware{},
Services: map[string]*dynamic.Service{},
},
TCP: &dynamic.TCPConfiguration{
ServersTransports: map[string]*dynamic.TCPServersTransport{},
Routers: map[string]*dynamic.TCPRouter{
"default-test.route-fdd3e9338e47a45efefc": {
EntryPoints: []string{"foo"},
Service: "default-test.route-fdd3e9338e47a45efefc",
Rule: "HostSNI(`foo.com`)",
},
},
Middlewares: map[string]*dynamic.TCPMiddleware{},
Services: map[string]*dynamic.TCPService{
"default-test.route-fdd3e9338e47a45efefc": {
LoadBalancer: &dynamic.TCPServersLoadBalancer{
Servers: []dynamic.TCPServer{
{
Address: "172.16.4.4:32456",
Port: "",
},
},
},
},
},
},
TLS: &dynamic.TLSConfiguration{},
},
},
{
desc: "UDP with native Service LB",
paths: []string{"udp/services.yml", "udp/with_node_port_service_lb.yml"},
expected: &dynamic.Configuration{
UDP: &dynamic.UDPConfiguration{
Routers: map[string]*dynamic.UDPRouter{
"default-test.route-0": {
EntryPoints: []string{"foo"},
Service: "default-test.route-0",
},
},
Services: map[string]*dynamic.UDPService{
"default-test.route-0": {
LoadBalancer: &dynamic.UDPServersLoadBalancer{
Servers: []dynamic.UDPServer{
{
Address: "172.16.4.4:32456",
Port: "",
},
},
},
},
},
},
HTTP: &dynamic.HTTPConfiguration{
ServersTransports: map[string]*dynamic.ServersTransport{},
Routers: map[string]*dynamic.Router{},
Middlewares: map[string]*dynamic.Middleware{},
Services: map[string]*dynamic.Service{},
},
TCP: &dynamic.TCPConfiguration{
ServersTransports: map[string]*dynamic.TCPServersTransport{},
Routers: map[string]*dynamic.TCPRouter{},
Middlewares: map[string]*dynamic.TCPMiddleware{},
Services: map[string]*dynamic.TCPService{},
},
TLS: &dynamic.TLSConfiguration{},
},
},
}
for _, test := range testCases {
test := test
t.Run(test.desc, func(t *testing.T) {
t.Parallel()
k8sObjects, crdObjects := readResources(t, test.paths)
kubeClient := kubefake.NewSimpleClientset(k8sObjects...)
crdClient := traefikcrdfake.NewSimpleClientset(crdObjects...)
client := newClientImpl(kubeClient, crdClient)
stopCh := make(chan struct{})
eventCh, err := client.WatchAll([]string{"default", "cross-ns"}, stopCh)
require.NoError(t, err)
if k8sObjects != nil || crdObjects != nil {
// just wait for the first event
<-eventCh
}
p := Provider{}
conf := p.loadConfigurationFromCRD(context.Background(), client)
assert.Equal(t, test.expected, conf)
})
}
}
func TestCreateBasicAuthCredentials(t *testing.T) {
var k8sObjects []runtime.Object
yamlContent, err := os.ReadFile(filepath.FromSlash("./fixtures/basic_auth_secrets.yml"))

28
pkg/provider/kubernetes/crd/kubernetes_udp.go
View file

@ -131,6 +131,34 @@ func (p *Provider) loadUDPServers(client Client, namespace string, svc traefikv1
}
var servers []dynamic.UDPServer
if service.Spec.Type == corev1.ServiceTypeNodePort && svc.NodePortLB {
nodes, nodesExists, nodesErr := client.GetNodes()
if nodesErr != nil {
return nil, nodesErr
}
if !nodesExists || len(nodes) == 0 {
return nil, fmt.Errorf("nodes not found for NodePort service %s/%s", svc.Namespace, svc.Name)
}
for _, node := range nodes {
for _, addr := range node.Status.Addresses {
if addr.Type == corev1.NodeInternalIP {
servers = append(servers, dynamic.UDPServer{
Address: net.JoinHostPort(addr.Address, strconv.Itoa(int(svcPort.NodePort))),
})
}
}
}
if len(servers) == 0 {
return nil, fmt.Errorf("no servers were generated for service %s/%s", svc.Namespace, svc.Name)
}
return servers, nil
}
if service.Spec.Type == corev1.ServiceTypeExternalName {
servers = append(servers, dynamic.UDPServer{
Address: net.JoinHostPort(service.Spec.ExternalName, strconv.Itoa(int(svcPort.Port))),

5
pkg/provider/kubernetes/crd/traefikio/v1alpha1/ingressroute.go
View file

@ -126,6 +126,11 @@ type LoadBalancerSpec struct {
// The Kubernetes Service itself does load-balance to the pods.
// By default, NativeLB is false.
NativeLB bool `json:"nativeLB,omitempty"`
// NodePortLB controls, when creating the load-balancer,
// whether the LB's children are directly the nodes internal IPs using the nodePort when the service type is NodePort.
// It allows services to be reachable when Traefik runs externally from the Kubernetes cluster but within the same network of the nodes.
// By default, NodePortLB is false.
NodePortLB bool `json:"nodePortLB,omitempty"`
}
type ResponseForwarding struct {

5
pkg/provider/kubernetes/crd/traefikio/v1alpha1/ingressroutetcp.go
View file

@ -93,6 +93,11 @@ type ServiceTCP struct {
// The Kubernetes Service itself does load-balance to the pods.
// By default, NativeLB is false.
NativeLB bool `json:"nativeLB,omitempty"`
// NodePortLB controls, when creating the load-balancer,
// whether the LB's children are directly the nodes internal IPs using the nodePort when the service type is NodePort.
// It allows services to be reachable when Traefik runs externally from the Kubernetes cluster but within the same network of the nodes.
// By default, NodePortLB is false.
NodePortLB bool `json:"nodePortLB,omitempty"`
}
// +genclient

5
pkg/provider/kubernetes/crd/traefikio/v1alpha1/ingressrouteudp.go
View file

@ -38,6 +38,11 @@ type ServiceUDP struct {
// The Kubernetes Service itself does load-balance to the pods.
// By default, NativeLB is false.
NativeLB bool `json:"nativeLB,omitempty"`
// NodePortLB controls, when creating the load-balancer,
// whether the LB's children are directly the nodes internal IPs using the nodePort when the service type is NodePort.
// It allows services to be reachable when Traefik runs externally from the Kubernetes cluster but within the same network of the nodes.
// By default, NodePortLB is false.
NodePortLB bool `json:"nodePortLB,omitempty"`
}
// +genclient

1
pkg/provider/kubernetes/ingress/annotations.go
View file

@ -46,6 +46,7 @@ type ServiceIng struct {
PassHostHeader *bool `json:"passHostHeader"`
Sticky *dynamic.Sticky `json:"sticky,omitempty" label:"allowEmpty"`
NativeLB bool `json:"nativeLB,omitempty"`
NodePortLB bool `json:"nodePortLB,omitempty"`
}
// SetDefaults sets the default values.

21
pkg/provider/kubernetes/ingress/client.go
View file

@ -39,12 +39,14 @@ type Client interface {
GetIngressClasses() ([]*netv1.IngressClass, error)
GetService(namespace, name string) (*corev1.Service, bool, error)
GetSecret(namespace, name string) (*corev1.Secret, bool, error)
GetNodes() ([]*corev1.Node, bool, error)
GetEndpoints(namespace, name string) (*corev1.Endpoints, bool, error)
UpdateIngressStatus(ing *netv1.Ingress, ingStatus []netv1.IngressLoadBalancerIngress) error
}
type clientWrapper struct {
clientset kclientset.Interface
factoryClusterScope kinformers.SharedInformerFactory
factoriesKube map[string]kinformers.SharedInformerFactory
factoriesSecret map[string]kinformers.SharedInformerFactory
factoriesIngress map[string]kinformers.SharedInformerFactory
@ -196,11 +198,18 @@ func (c *clientWrapper) WatchAll(namespaces []string, stopCh <-chan struct{}) (<
c.factoriesSecret[ns] = factorySecret
}
c.factoryClusterScope = kinformers.NewSharedInformerFactory(c.clientset, resyncPeriod)
_, err = c.factoryClusterScope.Core().V1().Nodes().Informer().AddEventHandler(eventHandler)
if err != nil {
return nil, err
}
for _, ns := range namespaces {
c.factoriesIngress[ns].Start(stopCh)
c.factoriesKube[ns].Start(stopCh)
c.factoriesSecret[ns].Start(stopCh)
}
c.factoryClusterScope.Start(stopCh)
for _, ns := range namespaces {
for typ, ok := range c.factoriesIngress[ns].WaitForCacheSync(stopCh) {
@ -222,6 +231,12 @@ func (c *clientWrapper) WatchAll(namespaces []string, stopCh <-chan struct{}) (<
}
}
for t, ok := range c.factoryClusterScope.WaitForCacheSync(stopCh) {
if !ok {
return nil, fmt.Errorf("timed out waiting for controller caches to sync %s", t.String())
}
}
if !c.disableIngressClassInformer {
c.clusterFactory = kinformers.NewSharedInformerFactoryWithOptions(c.clientset, resyncPeriod)
@ -346,6 +361,12 @@ func (c *clientWrapper) GetSecret(namespace, name string) (*corev1.Secret, bool,
return secret, exist, err
}
func (c *clientWrapper) GetNodes() ([]*corev1.Node, bool, error) {
nodes, err := c.factoryClusterScope.Core().V1().Nodes().Lister().List(labels.Everything())
exist, err := translateNotFoundError(err)
return nodes, exist, err
}
func (c *clientWrapper) GetIngressClasses() ([]*netv1.IngressClass, error) {
if c.clusterFactory == nil {
return nil, errors.New("cluster factory not loaded")

12
pkg/provider/kubernetes/ingress/client_mock_test.go
View file

@ -16,11 +16,13 @@ type clientMock struct {
services []*corev1.Service
secrets []*corev1.Secret
endpoints []*corev1.Endpoints
nodes []*corev1.Node
ingressClasses []*netv1.IngressClass
apiServiceError error
apiSecretError error
apiEndpointsError error
apiNodesError error
apiIngressStatusError error
watchChan chan interface{}
@ -43,6 +45,8 @@ func newClientMock(path string) clientMock {
c.secrets = append(c.secrets, o)
case *corev1.Endpoints:
c.endpoints = append(c.endpoints, o)
case *corev1.Node:
c.nodes = append(c.nodes, o)
case *netv1.Ingress:
c.ingresses = append(c.ingresses, o)
case *netv1.IngressClass:
@ -86,6 +90,14 @@ func (c clientMock) GetEndpoints(namespace, name string) (*corev1.Endpoints, boo
return &corev1.Endpoints{}, false, nil
}
func (c clientMock) GetNodes() ([]*corev1.Node, bool, error) {
if c.apiNodesError != nil {
return nil, false, c.apiNodesError
}
return c.nodes, true, nil
}
func (c clientMock) GetSecret(namespace, name string) (*corev1.Secret, bool, error) {
if c.apiSecretError != nil {
return nil, false, c.apiSecretError

45
pkg/provider/kubernetes/ingress/fixtures/Ingress-with-node-port-lb.yml Normal file
View file

@ -0,0 +1,45 @@
kind: Ingress
apiVersion: networking.k8s.io/v1
metadata:
name: ""
namespace: testing
spec:
rules:
- host: traefik.tchouk
http:
paths:
- path: /bar
backend:
service:
name: service1
port:
number: 8080
pathType: Prefix
---
kind: Service
apiVersion: v1
metadata:
name: service1
namespace: testing
annotations:
traefik.ingress.kubernetes.io/service.nodeportlb: "true"
spec:
ports:
- port: 8080
nodePort: 32456
clusterIP: 10.0.0.1
type: NodePort
externalName: traefik.wtf
---
kind: Node
apiVersion: v1
metadata:
name: traefik-node
status:
addresses:
- type: InternalIP
address: 172.16.4.4

35
pkg/provider/kubernetes/ingress/kubernetes.go
View file

@ -599,6 +599,41 @@ func (p *Provider) loadService(client Client, namespace string, backend netv1.In
return svc, nil
}
if svcConfig.Service.NodePortLB && service.Spec.Type == corev1.ServiceTypeNodePort {
nodes, nodesExists, nodesErr := client.GetNodes()
if nodesErr != nil {
return nil, nodesErr
}
if !nodesExists || len(nodes) == 0 {
return nil, fmt.Errorf("nodes not found in namespace %s", namespace)
}
protocol := getProtocol(portSpec, portSpec.Name, svcConfig)
var servers []dynamic.Server
for _, node := range nodes {
for _, addr := range node.Status.Addresses {
if addr.Type == corev1.NodeInternalIP {
hostPort := net.JoinHostPort(addr.Address, strconv.Itoa(int(portSpec.NodePort)))
servers = append(servers, dynamic.Server{
URL: fmt.Sprintf("%s://%s", protocol, hostPort),
})
}
}
}
if len(servers) == 0 {
return nil, fmt.Errorf("no servers were generated for service %s in namespace", backend.Service.Name)
}
svc.LoadBalancer.Servers = servers
return svc, nil
}
}
if service.Spec.Type == corev1.ServiceTypeExternalName {

52
pkg/provider/kubernetes/ingress/kubernetes_test.go
View file

@ -1694,6 +1694,58 @@ func TestLoadConfigurationFromIngressesWithNativeLB(t *testing.T) {
}
}
func TestLoadConfigurationFromIngressesWithNodePortLB(t *testing.T) {
testCases := []struct {
desc string
ingressClass string
expected *dynamic.Configuration
}{
{
desc: "Ingress with node port lb",
expected: &dynamic.Configuration{
TCP: &dynamic.TCPConfiguration{},
HTTP: &dynamic.HTTPConfiguration{
Middlewares: map[string]*dynamic.Middleware{},
Routers: map[string]*dynamic.Router{
"testing-traefik-tchouk-bar": {
Rule: "Host(`traefik.tchouk`) && PathPrefix(`/bar`)",
Service: "testing-service1-8080",
},
},
Services: map[string]*dynamic.Service{
"testing-service1-8080": {
LoadBalancer: &dynamic.ServersLoadBalancer{
ResponseForwarding: &dynamic.ResponseForwarding{FlushInterval: dynamic.DefaultFlushInterval},
PassHostHeader: Bool(true),
Servers: []dynamic.Server{
{
URL: "http://172.16.4.4:32456",
},
},
},
},
},
},
},
},
}
for _, test := range testCases {
test := test
t.Run(test.desc, func(t *testing.T) {
t.Parallel()
clientMock := newClientMock(generateTestFilename(test.desc))
p := Provider{IngressClass: test.ingressClass}
conf := p.loadConfigurationFromIngresses(context.Background(), clientMock)
assert.Equal(t, test.expected, conf)
})
}
}
func generateTestFilename(desc string) string {
return filepath.Join("fixtures", strings.ReplaceAll(desc, " ", "-")+".yml")
}

2
pkg/provider/kubernetes/k8s/parser.go
View file

@ -12,7 +12,7 @@ import (
// MustParseYaml parses a YAML to objects.
func MustParseYaml(content []byte) []runtime.Object {
acceptedK8sTypes := regexp.MustCompile(`^(Namespace|Deployment|Endpoints|Service|Ingress|IngressRoute|IngressRouteTCP|IngressRouteUDP|Middleware|MiddlewareTCP|Secret|TLSOption|TLSStore|TraefikService|IngressClass|ServersTransport|ServersTransportTCP|GatewayClass|Gateway|HTTPRoute|TCPRoute|TLSRoute|ReferenceGrant)$`)
acceptedK8sTypes := regexp.MustCompile(`^(Namespace|Deployment|Endpoints|Node|Service|Ingress|IngressRoute|IngressRouteTCP|IngressRouteUDP|Middleware|MiddlewareTCP|Secret|TLSOption|TLSStore|TraefikService|IngressClass|ServersTransport|ServersTransportTCP|GatewayClass|Gateway|HTTPRoute|TCPRoute|TLSRoute|ReferenceGrant)$`)
files := strings.Split(string(content), "---\n")
retVal := make([]runtime.Object, 0, len(files))