From c84b510f0d1848b7d112b63075a0403140edbf73 Mon Sep 17 00:00:00 2001 From: Manuel Zapf Date: Tue, 2 Apr 2024 17:32:04 +0200 Subject: [PATCH] Toggle support for Gateway API experimental channel --- docs/content/migration/v2-to-v3.md | 28 ++++++++++ docs/content/providers/kubernetes-gateway.md | 23 +++++++++ .../reference/static-configuration/cli-ref.md | 3 ++ .../reference/static-configuration/env-ref.md | 3 ++ .../reference/static-configuration/file.toml | 1 + .../reference/static-configuration/file.yaml | 1 + integration/fixtures/k8s_gateway.toml | 1 + pkg/provider/kubernetes/gateway/client.go | 22 ++++---- pkg/provider/kubernetes/gateway/kubernetes.go | 51 ++++++++++++++----- .../kubernetes/gateway/kubernetes_test.go | 51 +++++++++++-------- 10 files changed, 140 insertions(+), 44 deletions(-) diff --git a/docs/content/migration/v2-to-v3.md b/docs/content/migration/v2-to-v3.md index f2ec92c1a..6fb88c0e7 100644 --- a/docs/content/migration/v2-to-v3.md +++ b/docs/content/migration/v2-to-v3.md @@ -92,6 +92,34 @@ Docker provider `tls.CAOptional` option has been removed in v3, as TLS client au The `tls.caOptional` option should be removed from the Docker provider static configuration. +### Kubernetes Gateway API + +#### Experimental Channel Resources (TLSRoute and TCPRoute) + +In v3, the Kubernetes Gateway API provider does not enable support for the experimental channel API resources by default. + +##### Remediation + +The `experimentalChannel` option should be used to enable the support for the experimental channel API resources. + +??? example "An example usage of the Kubernetes Gateway API provider with experimental channel support enabled" + + ```yaml tab="File (YAML)" + providers: + kubernetesGateway: + experimentalChannel: true + ``` + + ```toml tab="File (TOML)" + [providers.kubernetesGateway] + experimentalChannel = true + # ... + ``` + + ```bash tab="CLI" + --providers.kubernetesgateway.experimentalchannel=true + ``` + ### Experimental Configuration #### HTTP3 diff --git a/docs/content/providers/kubernetes-gateway.md b/docs/content/providers/kubernetes-gateway.md index 95b5ffd02..fe49dbe75 100644 --- a/docs/content/providers/kubernetes-gateway.md +++ b/docs/content/providers/kubernetes-gateway.md @@ -212,6 +212,29 @@ providers: --providers.kubernetesgateway.namespaces=default,production ``` +### `experimentalChannel` + +_Optional, Default: false_ + +Toggles support for the Experimental Channel resources ([Gateway API release channels documentation](https://gateway-api.sigs.k8s.io/concepts/versioning/#release-channels)). +This option currently enables support for `TCPRoute` and `TLSRoute`. + +```yaml tab="File (YAML)" +providers: + kubernetesGateway: + experimentalChannel: true +``` + +```toml tab="File (TOML)" +[providers.kubernetesGateway] + experimentalChannel = true + # ... +``` + +```bash tab="CLI" +--providers.kubernetesgateway.experimentalchannel=true +``` + ### `labelselector` _Optional, Default: ""_ diff --git a/docs/content/reference/static-configuration/cli-ref.md b/docs/content/reference/static-configuration/cli-ref.md index 45cd217a5..a632402ad 100644 --- a/docs/content/reference/static-configuration/cli-ref.md +++ b/docs/content/reference/static-configuration/cli-ref.md @@ -729,6 +729,9 @@ Kubernetes certificate authority file path (not needed for in-cluster client). `--providers.kubernetesgateway.endpoint`: Kubernetes server endpoint (required for external cluster client). +`--providers.kubernetesgateway.experimentalchannel`: +Toggles Experimental Channel resources support (TCPRoute, TLSRoute...). (Default: ```false```) + `--providers.kubernetesgateway.labelselector`: Kubernetes label selector to select specific GatewayClasses. diff --git a/docs/content/reference/static-configuration/env-ref.md b/docs/content/reference/static-configuration/env-ref.md index f7ab1bd79..3fe8614e5 100644 --- a/docs/content/reference/static-configuration/env-ref.md +++ b/docs/content/reference/static-configuration/env-ref.md @@ -729,6 +729,9 @@ Kubernetes certificate authority file path (not needed for in-cluster client). `TRAEFIK_PROVIDERS_KUBERNETESGATEWAY_ENDPOINT`: Kubernetes server endpoint (required for external cluster client). +`TRAEFIK_PROVIDERS_KUBERNETESGATEWAY_EXPERIMENTALCHANNEL`: +Toggles Experimental Channel resources support (TCPRoute, TLSRoute...). (Default: ```false```) + `TRAEFIK_PROVIDERS_KUBERNETESGATEWAY_LABELSELECTOR`: Kubernetes label selector to select specific GatewayClasses. diff --git a/docs/content/reference/static-configuration/file.toml b/docs/content/reference/static-configuration/file.toml index cef7d56e9..6bf62c9cf 100644 --- a/docs/content/reference/static-configuration/file.toml +++ b/docs/content/reference/static-configuration/file.toml @@ -146,6 +146,7 @@ namespaces = ["foobar", "foobar"] labelSelector = "foobar" throttleDuration = "42s" + experimentalChannel = true [providers.rest] insecure = true [providers.consulCatalog] diff --git a/docs/content/reference/static-configuration/file.yaml b/docs/content/reference/static-configuration/file.yaml index 84404356d..d54e06b60 100644 --- a/docs/content/reference/static-configuration/file.yaml +++ b/docs/content/reference/static-configuration/file.yaml @@ -163,6 +163,7 @@ providers: - foobar labelSelector: foobar throttleDuration: 42s + experimentalChannel: true rest: insecure: true consulCatalog: diff --git a/integration/fixtures/k8s_gateway.toml b/integration/fixtures/k8s_gateway.toml index 0649ab249..4db5bde6a 100644 --- a/integration/fixtures/k8s_gateway.toml +++ b/integration/fixtures/k8s_gateway.toml @@ -27,3 +27,4 @@ address = ":8443" [providers.kubernetesGateway] + experimentalChannel = true diff --git a/pkg/provider/kubernetes/gateway/client.go b/pkg/provider/kubernetes/gateway/client.go index 9bf57a5f8..05ae47f44 100644 --- a/pkg/provider/kubernetes/gateway/client.go +++ b/pkg/provider/kubernetes/gateway/client.go @@ -75,7 +75,8 @@ type clientWrapper struct { isNamespaceAll bool watchedNamespaces []string - labelSelector string + labelSelector string + experimentalChannel bool } func createClientFromConfig(c *rest.Config) (*clientWrapper, error) { @@ -196,19 +197,22 @@ func (c *clientWrapper) WatchAll(namespaces []string, stopCh <-chan struct{}) (< if err != nil { return nil, err } - _, err = factoryGateway.Gateway().V1alpha2().TCPRoutes().Informer().AddEventHandler(eventHandler) - if err != nil { - return nil, err - } - _, err = factoryGateway.Gateway().V1alpha2().TLSRoutes().Informer().AddEventHandler(eventHandler) - if err != nil { - return nil, err - } _, err = factoryGateway.Gateway().V1beta1().ReferenceGrants().Informer().AddEventHandler(eventHandler) if err != nil { return nil, err } + if c.experimentalChannel { + _, err = factoryGateway.Gateway().V1alpha2().TCPRoutes().Informer().AddEventHandler(eventHandler) + if err != nil { + return nil, err + } + _, err = factoryGateway.Gateway().V1alpha2().TLSRoutes().Informer().AddEventHandler(eventHandler) + if err != nil { + return nil, err + } + } + factoryKube := kinformers.NewSharedInformerFactoryWithOptions(c.csKube, resyncPeriod, kinformers.WithNamespace(ns)) _, err = factoryKube.Core().V1().Services().Informer().AddEventHandler(eventHandler) if err != nil { diff --git a/pkg/provider/kubernetes/gateway/kubernetes.go b/pkg/provider/kubernetes/gateway/kubernetes.go index 360ae1b8d..01401d750 100644 --- a/pkg/provider/kubernetes/gateway/kubernetes.go +++ b/pkg/provider/kubernetes/gateway/kubernetes.go @@ -51,13 +51,15 @@ const ( // Provider holds configurations of the provider. type Provider struct { - Endpoint string `description:"Kubernetes server endpoint (required for external cluster client)." json:"endpoint,omitempty" toml:"endpoint,omitempty" yaml:"endpoint,omitempty"` - Token types.FileOrContent `description:"Kubernetes bearer token (not needed for in-cluster client). It accepts either a token value or a file path to the token." json:"token,omitempty" toml:"token,omitempty" yaml:"token,omitempty" loggable:"false"` - CertAuthFilePath string `description:"Kubernetes certificate authority file path (not needed for in-cluster client)." json:"certAuthFilePath,omitempty" toml:"certAuthFilePath,omitempty" yaml:"certAuthFilePath,omitempty"` - Namespaces []string `description:"Kubernetes namespaces." json:"namespaces,omitempty" toml:"namespaces,omitempty" yaml:"namespaces,omitempty" export:"true"` - LabelSelector string `description:"Kubernetes label selector to select specific GatewayClasses." json:"labelSelector,omitempty" toml:"labelSelector,omitempty" yaml:"labelSelector,omitempty" export:"true"` - ThrottleDuration ptypes.Duration `description:"Kubernetes refresh throttle duration" json:"throttleDuration,omitempty" toml:"throttleDuration,omitempty" yaml:"throttleDuration,omitempty" export:"true"` - EntryPoints map[string]Entrypoint `json:"-" toml:"-" yaml:"-" label:"-" file:"-"` + Endpoint string `description:"Kubernetes server endpoint (required for external cluster client)." json:"endpoint,omitempty" toml:"endpoint,omitempty" yaml:"endpoint,omitempty"` + Token types.FileOrContent `description:"Kubernetes bearer token (not needed for in-cluster client). It accepts either a token value or a file path to the token." json:"token,omitempty" toml:"token,omitempty" yaml:"token,omitempty" loggable:"false"` + CertAuthFilePath string `description:"Kubernetes certificate authority file path (not needed for in-cluster client)." json:"certAuthFilePath,omitempty" toml:"certAuthFilePath,omitempty" yaml:"certAuthFilePath,omitempty"` + Namespaces []string `description:"Kubernetes namespaces." json:"namespaces,omitempty" toml:"namespaces,omitempty" yaml:"namespaces,omitempty" export:"true"` + LabelSelector string `description:"Kubernetes label selector to select specific GatewayClasses." json:"labelSelector,omitempty" toml:"labelSelector,omitempty" yaml:"labelSelector,omitempty" export:"true"` + ThrottleDuration ptypes.Duration `description:"Kubernetes refresh throttle duration" json:"throttleDuration,omitempty" toml:"throttleDuration,omitempty" yaml:"throttleDuration,omitempty" export:"true"` + ExperimentalChannel bool `description:"Toggles Experimental Channel resources support (TCPRoute, TLSRoute...)." json:"experimentalChannel,omitempty" toml:"experimentalChannel,omitempty" yaml:"experimentalChannel,omitempty" export:"true"` + + EntryPoints map[string]Entrypoint `json:"-" toml:"-" yaml:"-" label:"-" file:"-"` // groupKindFilterFuncs is the list of allowed Group and Kinds for the Filter ExtensionRef objects. groupKindFilterFuncs map[string]map[string]BuildFilterFunc @@ -155,6 +157,7 @@ func (p *Provider) newK8sClient(ctx context.Context) (*clientWrapper, error) { } client.labelSelector = p.LabelSelector + client.experimentalChannel = p.ExperimentalChannel return client, nil } @@ -396,7 +399,7 @@ func (p *Provider) fillGatewayConf(ctx context.Context, client Client, gateway * // AttachedRoutes: 0 TODO Set to number of Routes associated with a Listener regardless of Gateway or Route status } - supportedKinds, conditions := supportedRouteKinds(listener.Protocol) + supportedKinds, conditions := supportedRouteKinds(listener.Protocol, p.ExperimentalChannel) if len(conditions) > 0 { listenerStatuses[i].Conditions = append(listenerStatuses[i].Conditions, conditions...) continue @@ -716,21 +719,41 @@ func (p *Provider) entryPointName(port gatev1.PortNumber, protocol gatev1.Protoc return "", fmt.Errorf("no matching entryPoint for port %d and protocol %q", port, protocol) } -func supportedRouteKinds(protocol gatev1.ProtocolType) ([]gatev1.RouteGroupKind, []metav1.Condition) { +func supportedRouteKinds(protocol gatev1.ProtocolType, experimentalChannel bool) ([]gatev1.RouteGroupKind, []metav1.Condition) { group := gatev1.Group(gatev1.GroupName) switch protocol { case gatev1.TCPProtocolType: - return []gatev1.RouteGroupKind{{Kind: kindTCPRoute, Group: &group}}, nil + if experimentalChannel { + return []gatev1.RouteGroupKind{{Kind: kindTCPRoute, Group: &group}}, nil + } + + return nil, []metav1.Condition{{ + Type: string(gatev1.ListenerConditionConflicted), + Status: metav1.ConditionFalse, + LastTransitionTime: metav1.Now(), + Reason: string(gatev1.ListenerReasonInvalidRouteKinds), + Message: fmt.Sprintf("Protocol %q requires the experimental channel support to be enabled, please use the `experimentalChannel` option", protocol), + }} case gatev1.HTTPProtocolType, gatev1.HTTPSProtocolType: return []gatev1.RouteGroupKind{{Kind: kindHTTPRoute, Group: &group}}, nil case gatev1.TLSProtocolType: - return []gatev1.RouteGroupKind{ - {Kind: kindTCPRoute, Group: &group}, - {Kind: kindTLSRoute, Group: &group}, - }, nil + if experimentalChannel { + return []gatev1.RouteGroupKind{ + {Kind: kindTCPRoute, Group: &group}, + {Kind: kindTLSRoute, Group: &group}, + }, nil + } + + return nil, []metav1.Condition{{ + Type: string(gatev1.ListenerConditionConflicted), + Status: metav1.ConditionFalse, + LastTransitionTime: metav1.Now(), + Reason: string(gatev1.ListenerReasonInvalidRouteKinds), + Message: fmt.Sprintf("Protocol %q requires the experimental channel support to be enabled, please use the `experimentalChannel` option", protocol), + }} } return nil, []metav1.Condition{{ diff --git a/pkg/provider/kubernetes/gateway/kubernetes_test.go b/pkg/provider/kubernetes/gateway/kubernetes_test.go index 8146523f9..85f055b39 100644 --- a/pkg/provider/kubernetes/gateway/kubernetes_test.go +++ b/pkg/provider/kubernetes/gateway/kubernetes_test.go @@ -24,12 +24,12 @@ var _ provider.Provider = (*Provider)(nil) func TestLoadHTTPRoutes(t *testing.T) { testCases := []struct { - desc string - ingressClass string - paths []string - namespaces []string - expected *dynamic.Configuration - entryPoints map[string]Entrypoint + desc string + ingressClass string + paths []string + expected *dynamic.Configuration + entryPoints map[string]Entrypoint + experimentalChannel bool }{ { desc: "Empty", @@ -476,6 +476,7 @@ func TestLoadHTTPRoutes(t *testing.T) { entryPoints: map[string]Entrypoint{"TCP": { Address: ":8080", }}, + experimentalChannel: true, expected: &dynamic.Configuration{ UDP: &dynamic.UDPConfiguration{ Routers: map[string]*dynamic.UDPRouter{}, @@ -1664,7 +1665,7 @@ func TestLoadHTTPRoutes(t *testing.T) { return } - p := Provider{EntryPoints: test.entryPoints} + p := Provider{EntryPoints: test.entryPoints, ExperimentalChannel: test.experimentalChannel} conf := p.loadConfigurationFromGateway(context.Background(), newClientMock(test.paths...)) assert.Equal(t, test.expected, conf) @@ -2905,7 +2906,7 @@ func TestLoadTCPRoutes(t *testing.T) { return } - p := Provider{EntryPoints: test.entryPoints} + p := Provider{EntryPoints: test.entryPoints, ExperimentalChannel: true} conf := p.loadConfigurationFromGateway(context.Background(), newClientMock(test.paths...)) assert.Equal(t, test.expected, conf) }) @@ -4034,7 +4035,7 @@ func TestLoadTLSRoutes(t *testing.T) { return } - p := Provider{EntryPoints: test.entryPoints} + p := Provider{EntryPoints: test.entryPoints, ExperimentalChannel: true} conf := p.loadConfigurationFromGateway(context.Background(), newClientMock(test.paths...)) assert.Equal(t, test.expected, conf) }) @@ -4043,11 +4044,12 @@ func TestLoadTLSRoutes(t *testing.T) { func TestLoadMixedRoutes(t *testing.T) { testCases := []struct { - desc string - ingressClass string - paths []string - expected *dynamic.Configuration - entryPoints map[string]Entrypoint + desc string + ingressClass string + paths []string + expected *dynamic.Configuration + entryPoints map[string]Entrypoint + experimentalChannel bool }{ { desc: "Empty", @@ -4159,6 +4161,7 @@ func TestLoadMixedRoutes(t *testing.T) { "tls-1": {Address: ":10000"}, "tls-2": {Address: ":11000"}, }, + experimentalChannel: true, expected: &dynamic.Configuration{ UDP: &dynamic.UDPConfiguration{ Routers: map[string]*dynamic.UDPRouter{}, @@ -4343,6 +4346,7 @@ func TestLoadMixedRoutes(t *testing.T) { "tls-1": {Address: ":10000"}, "tls-2": {Address: ":11000"}, }, + experimentalChannel: true, expected: &dynamic.Configuration{ UDP: &dynamic.UDPConfiguration{ Routers: map[string]*dynamic.UDPRouter{}, @@ -4499,6 +4503,7 @@ func TestLoadMixedRoutes(t *testing.T) { "tls-1": {Address: ":10000"}, "tls-2": {Address: ":11000"}, }, + experimentalChannel: true, expected: &dynamic.Configuration{ UDP: &dynamic.UDPConfiguration{ Routers: map[string]*dynamic.UDPRouter{}, @@ -4749,6 +4754,7 @@ func TestLoadMixedRoutes(t *testing.T) { "tls-1": {Address: ":10000"}, "tls-2": {Address: ":11000"}, }, + experimentalChannel: true, expected: &dynamic.Configuration{ UDP: &dynamic.UDPConfiguration{ Routers: map[string]*dynamic.UDPRouter{}, @@ -4904,6 +4910,7 @@ func TestLoadMixedRoutes(t *testing.T) { "tcp": {Address: ":9000"}, "tls": {Address: ":10000"}, }, + experimentalChannel: true, expected: &dynamic.Configuration{ UDP: &dynamic.UDPConfiguration{ Routers: map[string]*dynamic.UDPRouter{}, @@ -5042,7 +5049,7 @@ func TestLoadMixedRoutes(t *testing.T) { return } - p := Provider{EntryPoints: test.entryPoints} + p := Provider{EntryPoints: test.entryPoints, ExperimentalChannel: test.experimentalChannel} conf := p.loadConfigurationFromGateway(context.Background(), newClientMock(test.paths...)) assert.Equal(t, test.expected, conf) }) @@ -5051,11 +5058,12 @@ func TestLoadMixedRoutes(t *testing.T) { func TestLoadRoutesWithReferenceGrants(t *testing.T) { testCases := []struct { - desc string - ingressClass string - paths []string - expected *dynamic.Configuration - entryPoints map[string]Entrypoint + desc string + ingressClass string + paths []string + expected *dynamic.Configuration + entryPoints map[string]Entrypoint + experimentalChannel bool }{ { desc: "Empty", @@ -5163,6 +5171,7 @@ func TestLoadRoutesWithReferenceGrants(t *testing.T) { entryPoints: map[string]Entrypoint{ "tls": {Address: ":9000"}, }, + experimentalChannel: true, expected: &dynamic.Configuration{ UDP: &dynamic.UDPConfiguration{ Routers: map[string]*dynamic.UDPRouter{}, @@ -5232,7 +5241,7 @@ func TestLoadRoutesWithReferenceGrants(t *testing.T) { return } - p := Provider{EntryPoints: test.entryPoints} + p := Provider{EntryPoints: test.entryPoints, ExperimentalChannel: test.experimentalChannel} conf := p.loadConfigurationFromGateway(context.Background(), newClientMock(test.paths...)) assert.Equal(t, test.expected, conf) })