2020-12-15 15:40:05 +00:00
package gateway
import (
"context"
"crypto/sha256"
"errors"
"fmt"
"os"
2024-05-01 04:38:03 +00:00
"slices"
2020-12-15 15:40:05 +00:00
"sort"
"strconv"
"strings"
"time"
"github.com/cenkalti/backoff/v4"
"github.com/hashicorp/go-multierror"
"github.com/mitchellh/hashstructure"
2022-11-21 17:36:05 +00:00
"github.com/rs/zerolog/log"
2020-12-15 15:40:05 +00:00
ptypes "github.com/traefik/paerser/types"
2023-02-03 14:24:05 +00:00
"github.com/traefik/traefik/v3/pkg/config/dynamic"
"github.com/traefik/traefik/v3/pkg/job"
"github.com/traefik/traefik/v3/pkg/logs"
2023-03-21 11:00:46 +00:00
traefikv1alpha1 "github.com/traefik/traefik/v3/pkg/provider/kubernetes/crd/traefikio/v1alpha1"
2023-05-17 09:07:09 +00:00
"github.com/traefik/traefik/v3/pkg/provider/kubernetes/k8s"
2023-02-03 14:24:05 +00:00
"github.com/traefik/traefik/v3/pkg/safe"
"github.com/traefik/traefik/v3/pkg/tls"
2024-01-11 16:06:06 +00:00
"github.com/traefik/traefik/v3/pkg/types"
2020-12-15 15:40:05 +00:00
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
2024-07-11 09:26:03 +00:00
ktypes "k8s.io/apimachinery/pkg/types"
2024-01-09 09:28:05 +00:00
"k8s.io/utils/ptr"
gatev1 "sigs.k8s.io/gateway-api/apis/v1"
2024-01-30 15:44:05 +00:00
gatev1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1"
2020-12-15 15:40:05 +00:00
)
2021-02-02 18:36:04 +00:00
const (
2024-09-09 08:08:08 +00:00
providerName = "kubernetesgateway"
2024-05-01 04:38:03 +00:00
controllerName = "traefik.io/gateway-controller"
2021-11-09 10:34:06 +00:00
2024-06-04 12:16:04 +00:00
groupCore = "core"
groupGateway = "gateway.networking.k8s.io"
2024-01-30 15:44:05 +00:00
2021-11-09 10:34:06 +00:00
kindGateway = "Gateway"
kindTraefikService = "TraefikService"
kindHTTPRoute = "HTTPRoute"
2024-08-30 08:36:06 +00:00
kindGRPCRoute = "GRPCRoute"
2021-11-09 10:34:06 +00:00
kindTCPRoute = "TCPRoute"
kindTLSRoute = "TLSRoute"
2024-09-03 10:10:04 +00:00
kindService = "Service"
2024-09-09 08:08:08 +00:00
2024-10-09 14:26:04 +00:00
appProtocolHTTP = "http"
appProtocolHTTPS = "https"
appProtocolH2C = "kubernetes.io/h2c"
appProtocolWS = "kubernetes.io/ws"
appProtocolWSS = "kubernetes.io/wss"
schemeHTTP = "http"
schemeHTTPS = "https"
schemeH2C = "h2c"
2021-02-02 18:36:04 +00:00
)
2020-12-15 15:40:05 +00:00
// Provider holds configurations of the provider.
type Provider struct {
2024-04-02 15:32:04 +00:00
Endpoint string ` description:"Kubernetes server endpoint (required for external cluster client)." json:"endpoint,omitempty" toml:"endpoint,omitempty" yaml:"endpoint,omitempty" `
Token types . FileOrContent ` description:"Kubernetes bearer token (not needed for in-cluster client). It accepts either a token value or a file path to the token." json:"token,omitempty" toml:"token,omitempty" yaml:"token,omitempty" loggable:"false" `
CertAuthFilePath string ` description:"Kubernetes certificate authority file path (not needed for in-cluster client)." json:"certAuthFilePath,omitempty" toml:"certAuthFilePath,omitempty" yaml:"certAuthFilePath,omitempty" `
Namespaces [ ] string ` description:"Kubernetes namespaces." json:"namespaces,omitempty" toml:"namespaces,omitempty" yaml:"namespaces,omitempty" export:"true" `
LabelSelector string ` description:"Kubernetes label selector to select specific GatewayClasses." json:"labelSelector,omitempty" toml:"labelSelector,omitempty" yaml:"labelSelector,omitempty" export:"true" `
ThrottleDuration ptypes . Duration ` description:"Kubernetes refresh throttle duration" json:"throttleDuration,omitempty" toml:"throttleDuration,omitempty" yaml:"throttleDuration,omitempty" export:"true" `
ExperimentalChannel bool ` description:"Toggles Experimental Channel resources support (TCPRoute, TLSRoute...)." json:"experimentalChannel,omitempty" toml:"experimentalChannel,omitempty" yaml:"experimentalChannel,omitempty" export:"true" `
2024-04-10 07:34:07 +00:00
StatusAddress * StatusAddress ` description:"Defines the Kubernetes Gateway status address." json:"statusAddress,omitempty" toml:"statusAddress,omitempty" yaml:"statusAddress,omitempty" export:"true" `
2024-10-02 08:34:04 +00:00
NativeLBByDefault bool ` description:"Defines whether to use Native Kubernetes load-balancing by default." json:"nativeLBByDefault,omitempty" toml:"nativeLBByDefault,omitempty" yaml:"nativeLBByDefault,omitempty" export:"true" `
2024-04-02 15:32:04 +00:00
EntryPoints map [ string ] Entrypoint ` json:"-" toml:"-" yaml:"-" label:"-" file:"-" `
2020-12-15 15:40:05 +00:00
2024-03-25 13:38:04 +00:00
// groupKindFilterFuncs is the list of allowed Group and Kinds for the Filter ExtensionRef objects.
groupKindFilterFuncs map [ string ] map [ string ] BuildFilterFunc
// groupKindBackendFuncs is the list of allowed Group and Kinds for the Backend ExtensionRef objects.
groupKindBackendFuncs map [ string ] map [ string ] BuildBackendFunc
2020-12-15 15:40:05 +00:00
lastConfiguration safe . Safe
2023-05-15 14:38:05 +00:00
routerTransform k8s . RouterTransform
2024-06-13 09:16:04 +00:00
client * clientWrapper
2023-05-15 14:38:05 +00:00
}
2024-05-01 04:38:03 +00:00
// Entrypoint defines the available entry points.
type Entrypoint struct {
Address string
HasHTTPTLSConf bool
}
2024-04-10 07:34:07 +00:00
// StatusAddress holds the Gateway Status address configuration.
type StatusAddress struct {
IP string ` description:"IP used to set Kubernetes Gateway status address." json:"ip,omitempty" toml:"ip,omitempty" yaml:"ip,omitempty" `
Hostname string ` description:"Hostname used for Kubernetes Gateway status address." json:"hostname,omitempty" toml:"hostname,omitempty" yaml:"hostname,omitempty" `
Service ServiceRef ` description:"Published Kubernetes Service to copy status addresses from." json:"service,omitempty" toml:"service,omitempty" yaml:"service,omitempty" `
}
// ServiceRef holds a Kubernetes service reference.
type ServiceRef struct {
Name string ` description:"Name of the Kubernetes service." json:"name,omitempty" toml:"name,omitempty" yaml:"name,omitempty" `
Namespace string ` description:"Namespace of the Kubernetes service." json:"namespace,omitempty" toml:"namespace,omitempty" yaml:"namespace,omitempty" `
}
2024-03-25 13:38:04 +00:00
// BuildFilterFunc returns the name of the filter and the related dynamic.Middleware if needed.
type BuildFilterFunc func ( name , namespace string ) ( string , * dynamic . Middleware , error )
// BuildBackendFunc returns the name of the backend and the related dynamic.Service if needed.
type BuildBackendFunc func ( name , namespace string ) ( string , * dynamic . Service , error )
type ExtensionBuilderRegistry interface {
RegisterFilterFuncs ( group , kind string , builderFunc BuildFilterFunc )
RegisterBackendFuncs ( group , kind string , builderFunc BuildBackendFunc )
}
2024-05-28 12:30:04 +00:00
type gatewayListener struct {
Name string
2024-09-27 10:12:05 +00:00
Port gatev1 . PortNumber
2024-05-28 12:30:04 +00:00
Protocol gatev1 . ProtocolType
TLS * gatev1 . GatewayTLSConfig
Hostname * gatev1 . Hostname
Status * gatev1 . ListenerStatus
AllowedNamespaces [ ] string
AllowedRouteKinds [ ] string
Attached bool
GWName string
GWNamespace string
GWGeneration int64
EPName string
}
2024-03-25 13:38:04 +00:00
// RegisterFilterFuncs registers an allowed Group, Kind, and builder for the Filter ExtensionRef objects.
func ( p * Provider ) RegisterFilterFuncs ( group , kind string , builderFunc BuildFilterFunc ) {
if p . groupKindFilterFuncs == nil {
p . groupKindFilterFuncs = map [ string ] map [ string ] BuildFilterFunc { }
}
if p . groupKindFilterFuncs [ group ] == nil {
p . groupKindFilterFuncs [ group ] = map [ string ] BuildFilterFunc { }
}
p . groupKindFilterFuncs [ group ] [ kind ] = builderFunc
}
// RegisterBackendFuncs registers an allowed Group, Kind, and builder for the Backend ExtensionRef objects.
func ( p * Provider ) RegisterBackendFuncs ( group , kind string , builderFunc BuildBackendFunc ) {
if p . groupKindBackendFuncs == nil {
p . groupKindBackendFuncs = map [ string ] map [ string ] BuildBackendFunc { }
}
if p . groupKindBackendFuncs [ group ] == nil {
p . groupKindBackendFuncs [ group ] = map [ string ] BuildBackendFunc { }
}
p . groupKindBackendFuncs [ group ] [ kind ] = builderFunc
}
2023-05-15 14:38:05 +00:00
func ( p * Provider ) SetRouterTransform ( routerTransform k8s . RouterTransform ) {
p . routerTransform = routerTransform
}
2024-01-09 09:28:05 +00:00
func ( p * Provider ) applyRouterTransform ( ctx context . Context , rt * dynamic . Router , route * gatev1 . HTTPRoute ) {
2023-05-15 14:38:05 +00:00
if p . routerTransform == nil {
return
}
2024-08-30 08:36:06 +00:00
if err := p . routerTransform . Apply ( ctx , rt , route ) ; err != nil {
2023-05-17 09:07:09 +00:00
log . Ctx ( ctx ) . Error ( ) . Err ( err ) . Msg ( "Apply router transform" )
2023-05-15 14:38:05 +00:00
}
2020-12-15 15:40:05 +00:00
}
func ( p * Provider ) newK8sClient ( ctx context . Context ) ( * clientWrapper , error ) {
// Label selector validation
_ , err := labels . Parse ( p . LabelSelector )
if err != nil {
return nil , fmt . Errorf ( "invalid label selector: %q" , p . LabelSelector )
}
2021-07-15 15:20:08 +00:00
2022-11-21 17:36:05 +00:00
logger := log . Ctx ( ctx )
logger . Info ( ) . Msgf ( "Label selector is: %q" , p . LabelSelector )
2020-12-15 15:40:05 +00:00
var client * clientWrapper
switch {
case os . Getenv ( "KUBERNETES_SERVICE_HOST" ) != "" && os . Getenv ( "KUBERNETES_SERVICE_PORT" ) != "" :
2022-11-21 17:36:05 +00:00
logger . Info ( ) . Str ( "endpoint" , p . Endpoint ) . Msg ( "Creating in-cluster Provider client" )
2020-12-15 15:40:05 +00:00
client , err = newInClusterClient ( p . Endpoint )
case os . Getenv ( "KUBECONFIG" ) != "" :
2022-11-21 17:36:05 +00:00
logger . Info ( ) . Msgf ( "Creating cluster-external Provider client from KUBECONFIG %s" , os . Getenv ( "KUBECONFIG" ) )
2020-12-15 15:40:05 +00:00
client , err = newExternalClusterClientFromFile ( os . Getenv ( "KUBECONFIG" ) )
default :
2022-11-21 17:36:05 +00:00
logger . Info ( ) . Str ( "endpoint" , p . Endpoint ) . Msg ( "Creating cluster-external Provider client" )
2024-01-11 16:06:06 +00:00
client , err = newExternalClusterClient ( p . Endpoint , p . CertAuthFilePath , p . Token )
2020-12-15 15:40:05 +00:00
}
if err != nil {
return nil , err
}
2021-07-15 15:20:08 +00:00
2020-12-15 15:40:05 +00:00
client . labelSelector = p . LabelSelector
2024-04-02 15:32:04 +00:00
client . experimentalChannel = p . ExperimentalChannel
2020-12-15 15:40:05 +00:00
return client , nil
}
// Init the provider.
func ( p * Provider ) Init ( ) error {
2024-06-13 09:16:04 +00:00
logger := log . With ( ) . Str ( logs . ProviderName , providerName ) . Logger ( )
var err error
p . client , err = p . newK8sClient ( logger . WithContext ( context . Background ( ) ) )
if err != nil {
return fmt . Errorf ( "creating k8s client: %w" , err )
}
2020-12-15 15:40:05 +00:00
return nil
}
2021-11-09 10:34:06 +00:00
// Provide allows the k8s provider to provide configurations to traefik using the given configuration channel.
2020-12-15 15:40:05 +00:00
func ( p * Provider ) Provide ( configurationChan chan <- dynamic . Message , pool * safe . Pool ) error {
2022-11-21 17:36:05 +00:00
logger := log . With ( ) . Str ( logs . ProviderName , providerName ) . Logger ( )
ctxLog := logger . WithContext ( context . Background ( ) )
2020-12-15 15:40:05 +00:00
pool . GoCtx ( func ( ctxPool context . Context ) {
operation := func ( ) error {
2024-06-13 09:16:04 +00:00
eventsChan , err := p . client . WatchAll ( p . Namespaces , ctxPool . Done ( ) )
2020-12-15 15:40:05 +00:00
if err != nil {
2022-11-21 17:36:05 +00:00
logger . Error ( ) . Err ( err ) . Msg ( "Error watching kubernetes events" )
2020-12-15 15:40:05 +00:00
timer := time . NewTimer ( 1 * time . Second )
select {
case <- timer . C :
return err
case <- ctxPool . Done ( ) :
return nil
}
}
throttleDuration := time . Duration ( p . ThrottleDuration )
throttledChan := throttleEvents ( ctxLog , throttleDuration , pool , eventsChan )
if throttledChan != nil {
eventsChan = throttledChan
}
for {
select {
case <- ctxPool . Done ( ) :
return nil
case event := <- eventsChan :
// Note that event is the *first* event that came in during this throttling interval -- if we're hitting our throttle, we may have dropped events.
// This is fine, because we don't treat different event types differently.
// But if we do in the future, we'll need to track more information about the dropped events.
2024-06-13 09:16:04 +00:00
conf := p . loadConfigurationFromGateways ( ctxLog )
2020-12-15 15:40:05 +00:00
confHash , err := hashstructure . Hash ( conf , nil )
switch {
case err != nil :
2022-11-21 17:36:05 +00:00
logger . Error ( ) . Msg ( "Unable to hash the configuration" )
2020-12-15 15:40:05 +00:00
case p . lastConfiguration . Get ( ) == confHash :
2022-11-21 17:36:05 +00:00
logger . Debug ( ) . Msgf ( "Skipping Kubernetes event kind %T" , event )
2020-12-15 15:40:05 +00:00
default :
p . lastConfiguration . Set ( confHash )
configurationChan <- dynamic . Message {
ProviderName : providerName ,
Configuration : conf ,
}
}
// If we're throttling,
// we sleep here for the throttle duration to enforce that we don't refresh faster than our throttle.
// time.Sleep returns immediately if p.ThrottleDuration is 0 (no throttle).
time . Sleep ( throttleDuration )
}
}
}
notify := func ( err error , time time . Duration ) {
2022-11-30 08:50:05 +00:00
logger . Error ( ) . Err ( err ) . Msgf ( "Provider error, retrying in %s" , time )
2020-12-15 15:40:05 +00:00
}
err := backoff . RetryNotify ( safe . OperationWithRecover ( operation ) , backoff . WithContext ( job . NewBackOff ( backoff . NewExponentialBackOff ( ) ) , ctxPool ) , notify )
if err != nil {
2022-11-30 08:50:05 +00:00
logger . Error ( ) . Err ( err ) . Msg ( "Cannot retrieve data" )
2020-12-15 15:40:05 +00:00
}
} )
return nil
}
// TODO Handle errors and update resources statuses (gatewayClass, gateway).
2024-06-13 09:16:04 +00:00
func ( p * Provider ) loadConfigurationFromGateways ( ctx context . Context ) * dynamic . Configuration {
2024-05-28 12:30:04 +00:00
conf := & dynamic . Configuration {
HTTP : & dynamic . HTTPConfiguration {
Routers : map [ string ] * dynamic . Router { } ,
Middlewares : map [ string ] * dynamic . Middleware { } ,
Services : map [ string ] * dynamic . Service { } ,
ServersTransports : map [ string ] * dynamic . ServersTransport { } ,
} ,
TCP : & dynamic . TCPConfiguration {
Routers : map [ string ] * dynamic . TCPRouter { } ,
Middlewares : map [ string ] * dynamic . TCPMiddleware { } ,
Services : map [ string ] * dynamic . TCPService { } ,
ServersTransports : map [ string ] * dynamic . TCPServersTransport { } ,
} ,
UDP : & dynamic . UDPConfiguration {
Routers : map [ string ] * dynamic . UDPRouter { } ,
Services : map [ string ] * dynamic . UDPService { } ,
} ,
TLS : & dynamic . TLSConfiguration { } ,
}
2020-12-15 15:40:05 +00:00
2024-06-13 09:16:04 +00:00
addresses , err := p . gatewayAddresses ( )
2024-05-28 12:30:04 +00:00
if err != nil {
log . Ctx ( ctx ) . Error ( ) . Err ( err ) . Msg ( "Unable to get Gateway status addresses" )
return nil
}
2020-12-15 15:40:05 +00:00
2024-06-13 09:16:04 +00:00
gatewayClasses , err := p . client . ListGatewayClasses ( )
2020-12-15 15:40:05 +00:00
if err != nil {
2024-05-28 12:30:04 +00:00
log . Ctx ( ctx ) . Error ( ) . Err ( err ) . Msg ( "Unable to list GatewayClasses" )
return nil
2020-12-15 15:40:05 +00:00
}
2024-09-17 14:40:04 +00:00
var supportedFeatures [ ] gatev1 . SupportedFeature
2024-09-26 07:12:04 +00:00
if p . ExperimentalChannel {
for _ , feature := range SupportedFeatures ( ) {
supportedFeatures = append ( supportedFeatures , gatev1 . SupportedFeature { Name : gatev1 . FeatureName ( feature ) } )
}
slices . SortFunc ( supportedFeatures , func ( a , b gatev1 . SupportedFeature ) int {
return strings . Compare ( string ( a . Name ) , string ( b . Name ) )
} )
2024-09-17 14:40:04 +00:00
}
2024-05-28 12:30:04 +00:00
gatewayClassNames := map [ string ] struct { } { }
2020-12-15 15:40:05 +00:00
for _ , gatewayClass := range gatewayClasses {
2024-05-28 12:30:04 +00:00
if gatewayClass . Spec . ControllerName != controllerName {
continue
}
2020-12-15 15:40:05 +00:00
2024-05-28 12:30:04 +00:00
gatewayClassNames [ gatewayClass . Name ] = struct { } { }
2024-07-29 13:48:05 +00:00
status := gatev1 . GatewayClassStatus {
Conditions : upsertGatewayClassConditionAccepted ( gatewayClass . Status . Conditions , metav1 . Condition {
Type : string ( gatev1 . GatewayClassConditionStatusAccepted ) ,
Status : metav1 . ConditionTrue ,
ObservedGeneration : gatewayClass . Generation ,
Reason : "Handled" ,
Message : "Handled by Traefik controller" ,
LastTransitionTime : metav1 . Now ( ) ,
} ) ,
2024-09-17 14:40:04 +00:00
SupportedFeatures : supportedFeatures ,
2024-07-29 13:48:05 +00:00
}
if err := p . client . UpdateGatewayClassStatus ( ctx , gatewayClass . Name , status ) ; err != nil {
2024-05-28 12:30:04 +00:00
log . Ctx ( ctx ) .
2024-07-11 09:26:03 +00:00
Warn ( ) .
2024-05-28 12:30:04 +00:00
Err ( err ) .
Str ( "gateway_class" , gatewayClass . Name ) .
Msg ( "Unable to update GatewayClass status" )
2020-12-15 15:40:05 +00:00
}
}
2024-10-09 13:14:05 +00:00
var gateways [ ] * gatev1 . Gateway
for _ , gateway := range p . client . ListGateways ( ) {
if _ , ok := gatewayClassNames [ string ( gateway . Spec . GatewayClassName ) ] ; ! ok {
continue
}
gateways = append ( gateways , gateway )
}
2020-12-15 15:40:05 +00:00
2024-05-28 12:30:04 +00:00
var gatewayListeners [ ] gatewayListener
for _ , gateway := range gateways {
logger := log . Ctx ( ctx ) . With ( ) .
Str ( "gateway" , gateway . Name ) .
Str ( "namespace" , gateway . Namespace ) .
Logger ( )
2020-12-15 15:40:05 +00:00
2024-06-13 09:16:04 +00:00
gatewayListeners = append ( gatewayListeners , p . loadGatewayListeners ( logger . WithContext ( ctx ) , gateway , conf ) ... )
2020-12-15 15:40:05 +00:00
}
2024-06-13 09:16:04 +00:00
p . loadHTTPRoutes ( ctx , gatewayListeners , conf )
2020-12-15 15:40:05 +00:00
2024-08-30 08:36:06 +00:00
p . loadGRPCRoutes ( ctx , gatewayListeners , conf )
2024-05-28 12:30:04 +00:00
if p . ExperimentalChannel {
2024-06-13 09:16:04 +00:00
p . loadTCPRoutes ( ctx , gatewayListeners , conf )
p . loadTLSRoutes ( ctx , gatewayListeners , conf )
2024-05-28 12:30:04 +00:00
}
2020-12-15 15:40:05 +00:00
2024-05-28 12:30:04 +00:00
for _ , gateway := range gateways {
logger := log . Ctx ( ctx ) . With ( ) .
Str ( "gateway" , gateway . Name ) .
Str ( "namespace" , gateway . Namespace ) .
Logger ( )
2020-12-15 15:40:05 +00:00
2024-05-28 12:30:04 +00:00
var listeners [ ] gatewayListener
for _ , listener := range gatewayListeners {
if listener . GWName == gateway . Name && listener . GWNamespace == gateway . Namespace {
listeners = append ( listeners , listener )
2020-12-15 15:40:05 +00:00
}
}
2024-09-03 10:10:04 +00:00
gatewayStatus , errConditions := p . makeGatewayStatus ( gateway , listeners , addresses )
if len ( errConditions ) > 0 {
messages := map [ string ] struct { } { }
for _ , condition := range errConditions {
messages [ condition . Message ] = struct { } { }
}
var conditionsErr error
for message := range messages {
conditionsErr = multierror . Append ( conditionsErr , errors . New ( message ) )
}
2024-05-28 12:30:04 +00:00
logger . Error ( ) .
2024-09-03 10:10:04 +00:00
Err ( conditionsErr ) .
Msg ( "Gateway Not Accepted" )
2020-12-15 15:40:05 +00:00
}
2024-07-11 09:26:03 +00:00
if err = p . client . UpdateGatewayStatus ( ctx , ktypes . NamespacedName { Name : gateway . Name , Namespace : gateway . Namespace } , gatewayStatus ) ; err != nil {
logger . Warn ( ) .
Err ( err ) .
Msg ( "Unable to update Gateway status" )
}
2020-12-15 15:40:05 +00:00
}
return conf
}
2024-06-13 09:16:04 +00:00
func ( p * Provider ) loadGatewayListeners ( ctx context . Context , gateway * gatev1 . Gateway , conf * dynamic . Configuration ) [ ] gatewayListener {
2020-12-15 15:40:05 +00:00
tlsConfigs := make ( map [ string ] * tls . CertAndStores )
2024-05-28 12:30:04 +00:00
allocatedListeners := make ( map [ string ] struct { } )
gatewayListeners := make ( [ ] gatewayListener , len ( gateway . Spec . Listeners ) )
2020-12-15 15:40:05 +00:00
2024-05-28 12:30:04 +00:00
for i , listener := range gateway . Spec . Listeners {
gatewayListeners [ i ] = gatewayListener {
Name : string ( listener . Name ) ,
GWName : gateway . Name ,
GWNamespace : gateway . Namespace ,
GWGeneration : gateway . Generation ,
2024-09-27 10:12:05 +00:00
Port : listener . Port ,
2024-05-28 12:30:04 +00:00
Protocol : listener . Protocol ,
TLS : listener . TLS ,
Hostname : listener . Hostname ,
Status : & gatev1 . ListenerStatus {
Name : listener . Name ,
SupportedKinds : [ ] gatev1 . RouteGroupKind { } ,
Conditions : [ ] metav1 . Condition { } ,
} ,
2024-05-01 04:38:03 +00:00
}
2020-12-15 15:40:05 +00:00
2024-05-28 12:30:04 +00:00
ep , err := p . entryPointName ( listener . Port , listener . Protocol )
if err != nil {
// update "Detached" status with "PortUnavailable" reason
gatewayListeners [ i ] . Status . Conditions = append ( gatewayListeners [ i ] . Status . Conditions , metav1 . Condition {
Type : string ( gatev1 . ListenerConditionAccepted ) ,
Status : metav1 . ConditionFalse ,
ObservedGeneration : gateway . Generation ,
LastTransitionTime : metav1 . Now ( ) ,
Reason : string ( gatev1 . ListenerReasonPortUnavailable ) ,
Message : fmt . Sprintf ( "Cannot find entryPoint for Gateway: %v" , err ) ,
} )
2020-12-15 15:40:05 +00:00
2024-05-28 12:30:04 +00:00
continue
}
gatewayListeners [ i ] . EPName = ep
2020-12-15 15:40:05 +00:00
2024-05-28 12:30:04 +00:00
allowedRoutes := ptr . Deref ( listener . AllowedRoutes , gatev1 . AllowedRoutes { Namespaces : & gatev1 . RouteNamespaces { From : ptr . To ( gatev1 . NamespacesFromSame ) } } )
2024-06-13 09:16:04 +00:00
gatewayListeners [ i ] . AllowedNamespaces , err = p . allowedNamespaces ( gateway . Namespace , allowedRoutes . Namespaces )
2024-05-28 12:30:04 +00:00
if err != nil {
// update "ResolvedRefs" status true with "InvalidRoutesRef" reason
gatewayListeners [ i ] . Status . Conditions = append ( gatewayListeners [ i ] . Status . Conditions , metav1 . Condition {
Type : string ( gatev1 . ListenerConditionResolvedRefs ) ,
Status : metav1 . ConditionFalse ,
ObservedGeneration : gateway . Generation ,
LastTransitionTime : metav1 . Now ( ) ,
Reason : "InvalidRouteNamespacesSelector" , // Should never happen as the selector is validated by kubernetes
Message : fmt . Sprintf ( "Invalid route namespaces selector: %v" , err ) ,
} )
2020-12-15 15:40:05 +00:00
2024-05-28 12:30:04 +00:00
continue
2020-12-15 15:40:05 +00:00
}
2024-04-02 15:32:04 +00:00
supportedKinds , conditions := supportedRouteKinds ( listener . Protocol , p . ExperimentalChannel )
2021-11-09 10:34:06 +00:00
if len ( conditions ) > 0 {
2024-05-28 12:30:04 +00:00
gatewayListeners [ i ] . Status . Conditions = append ( gatewayListeners [ i ] . Status . Conditions , conditions ... )
2020-12-15 15:40:05 +00:00
continue
}
2024-05-28 12:30:04 +00:00
routeKinds , conditions := allowedRouteKinds ( gateway , listener , supportedKinds )
for _ , kind := range routeKinds {
gatewayListeners [ i ] . AllowedRouteKinds = append ( gatewayListeners [ i ] . AllowedRouteKinds , string ( kind . Kind ) )
}
gatewayListeners [ i ] . Status . SupportedKinds = routeKinds
2021-11-09 10:34:06 +00:00
if len ( conditions ) > 0 {
2024-05-28 12:30:04 +00:00
gatewayListeners [ i ] . Status . Conditions = append ( gatewayListeners [ i ] . Status . Conditions , conditions ... )
2021-05-20 09:50:12 +00:00
continue
}
2022-06-23 09:58:09 +00:00
listenerKey := makeListenerKey ( listener )
if _ , ok := allocatedListeners [ listenerKey ] ; ok {
2024-05-28 12:30:04 +00:00
gatewayListeners [ i ] . Status . Conditions = append ( gatewayListeners [ i ] . Status . Conditions , metav1 . Condition {
2024-01-09 09:28:05 +00:00
Type : string ( gatev1 . ListenerConditionConflicted ) ,
2021-05-20 09:50:12 +00:00
Status : metav1 . ConditionTrue ,
2024-01-09 09:28:05 +00:00
ObservedGeneration : gateway . Generation ,
2021-05-20 09:50:12 +00:00
LastTransitionTime : metav1 . Now ( ) ,
2022-06-23 09:58:09 +00:00
Reason : "DuplicateListener" ,
Message : "A listener with same protocol, port and hostname already exists" ,
2021-05-20 09:50:12 +00:00
} )
continue
}
2022-06-23 09:58:09 +00:00
allocatedListeners [ listenerKey ] = struct { } { }
2024-01-09 09:28:05 +00:00
if ( listener . Protocol == gatev1 . HTTPProtocolType || listener . Protocol == gatev1 . TCPProtocolType ) && listener . TLS != nil {
2024-05-28 12:30:04 +00:00
gatewayListeners [ i ] . Status . Conditions = append ( gatewayListeners [ i ] . Status . Conditions , metav1 . Condition {
2024-01-09 09:28:05 +00:00
Type : string ( gatev1 . ListenerConditionAccepted ) ,
Status : metav1 . ConditionFalse ,
ObservedGeneration : gateway . Generation ,
2020-12-15 15:40:05 +00:00
LastTransitionTime : metav1 . Now ( ) ,
2021-11-09 10:34:06 +00:00
Reason : "InvalidTLSConfiguration" , // TODO check the spec if a proper reason is introduced at some point
Message : "TLS configuration must no be defined when using HTTP or TCP protocol" ,
2020-12-15 15:40:05 +00:00
} )
continue
}
2021-05-20 09:50:12 +00:00
// TLS
2024-01-09 09:28:05 +00:00
if listener . Protocol == gatev1 . HTTPSProtocolType || listener . Protocol == gatev1 . TLSProtocolType {
if listener . TLS == nil || ( len ( listener . TLS . CertificateRefs ) == 0 && listener . TLS . Mode != nil && * listener . TLS . Mode != gatev1 . TLSModePassthrough ) {
2020-12-15 15:40:05 +00:00
// update "Detached" status with "UnsupportedProtocol" reason
2024-05-28 12:30:04 +00:00
gatewayListeners [ i ] . Status . Conditions = append ( gatewayListeners [ i ] . Status . Conditions , metav1 . Condition {
2024-01-09 09:28:05 +00:00
Type : string ( gatev1 . ListenerConditionAccepted ) ,
Status : metav1 . ConditionFalse ,
ObservedGeneration : gateway . Generation ,
2020-12-15 15:40:05 +00:00
LastTransitionTime : metav1 . Now ( ) ,
2021-11-09 10:34:06 +00:00
Reason : "InvalidTLSConfiguration" , // TODO check the spec if a proper reason is introduced at some point
Message : fmt . Sprintf ( "No TLS configuration for Gateway Listener %s:%d and protocol %q" ,
listener . Name , listener . Port , listener . Protocol ) ,
2020-12-15 15:40:05 +00:00
} )
continue
}
2024-01-09 09:28:05 +00:00
var tlsModeType gatev1 . TLSModeType
2021-07-15 15:20:08 +00:00
if listener . TLS . Mode != nil {
tlsModeType = * listener . TLS . Mode
}
2024-01-09 09:28:05 +00:00
isTLSPassthrough := tlsModeType == gatev1 . TLSModePassthrough
2021-11-09 10:34:06 +00:00
if isTLSPassthrough && len ( listener . TLS . CertificateRefs ) > 0 {
// https://gateway-api.sigs.k8s.io/v1alpha2/references/spec/#gateway.networking.k8s.io/v1alpha2.GatewayTLSConfig
2024-05-28 12:30:04 +00:00
log . Ctx ( ctx ) . Warn ( ) . Msg ( "In case of Passthrough TLS mode, no TLS settings take effect as the TLS session from the client is NOT terminated at the Gateway" )
2021-05-20 09:50:12 +00:00
}
// Allowed configurations:
2021-11-09 10:34:06 +00:00
// Protocol TLS -> Passthrough -> TLSRoute/TCPRoute
// Protocol TLS -> Terminate -> TLSRoute/TCPRoute
2021-05-20 09:50:12 +00:00
// Protocol HTTPS -> Terminate -> HTTPRoute
2024-01-09 09:28:05 +00:00
if listener . Protocol == gatev1 . HTTPSProtocolType && isTLSPassthrough {
2024-05-28 12:30:04 +00:00
gatewayListeners [ i ] . Status . Conditions = append ( gatewayListeners [ i ] . Status . Conditions , metav1 . Condition {
2024-01-09 09:28:05 +00:00
Type : string ( gatev1 . ListenerConditionAccepted ) ,
Status : metav1 . ConditionFalse ,
ObservedGeneration : gateway . Generation ,
2020-12-15 15:40:05 +00:00
LastTransitionTime : metav1 . Now ( ) ,
2024-01-09 09:28:05 +00:00
Reason : string ( gatev1 . ListenerReasonUnsupportedProtocol ) ,
2021-11-09 10:34:06 +00:00
Message : "HTTPS protocol is not supported with TLS mode Passthrough" ,
2020-12-15 15:40:05 +00:00
} )
continue
}
2021-05-20 09:50:12 +00:00
if ! isTLSPassthrough {
2021-11-09 10:34:06 +00:00
if len ( listener . TLS . CertificateRefs ) == 0 {
2020-12-15 15:40:05 +00:00
// update "ResolvedRefs" status true with "InvalidCertificateRef" reason
2024-05-28 12:30:04 +00:00
gatewayListeners [ i ] . Status . Conditions = append ( gatewayListeners [ i ] . Status . Conditions , metav1 . Condition {
2024-01-09 09:28:05 +00:00
Type : string ( gatev1 . ListenerConditionResolvedRefs ) ,
2020-12-15 15:40:05 +00:00
Status : metav1 . ConditionFalse ,
2024-01-09 09:28:05 +00:00
ObservedGeneration : gateway . Generation ,
2020-12-15 15:40:05 +00:00
LastTransitionTime : metav1 . Now ( ) ,
2024-01-09 09:28:05 +00:00
Reason : string ( gatev1 . ListenerReasonInvalidCertificateRef ) ,
2021-11-09 10:34:06 +00:00
Message : "One TLS CertificateRef is required in Terminate mode" ,
2020-12-15 15:40:05 +00:00
} )
continue
}
2021-11-09 10:34:06 +00:00
// TODO Should we support multiple certificates?
certificateRef := listener . TLS . CertificateRefs [ 0 ]
if certificateRef . Kind == nil || * certificateRef . Kind != "Secret" ||
2024-01-30 15:44:05 +00:00
certificateRef . Group == nil || ( * certificateRef . Group != "" && * certificateRef . Group != groupCore ) {
2021-11-09 10:34:06 +00:00
// update "ResolvedRefs" status true with "InvalidCertificateRef" reason
2024-05-28 12:30:04 +00:00
gatewayListeners [ i ] . Status . Conditions = append ( gatewayListeners [ i ] . Status . Conditions , metav1 . Condition {
2024-01-09 09:28:05 +00:00
Type : string ( gatev1 . ListenerConditionResolvedRefs ) ,
2021-11-09 10:34:06 +00:00
Status : metav1 . ConditionFalse ,
2024-01-09 09:28:05 +00:00
ObservedGeneration : gateway . Generation ,
2021-11-09 10:34:06 +00:00
LastTransitionTime : metav1 . Now ( ) ,
2024-01-09 09:28:05 +00:00
Reason : string ( gatev1 . ListenerReasonInvalidCertificateRef ) ,
2024-01-30 15:44:05 +00:00
Message : fmt . Sprintf ( "Unsupported TLS CertificateRef group/kind: %s/%s" , groupToString ( certificateRef . Group ) , kindToString ( certificateRef . Kind ) ) ,
2021-11-09 10:34:06 +00:00
} )
continue
}
2024-01-30 15:44:05 +00:00
certificateNamespace := gateway . Namespace
2021-11-09 10:34:06 +00:00
if certificateRef . Namespace != nil && string ( * certificateRef . Namespace ) != gateway . Namespace {
2024-01-30 15:44:05 +00:00
certificateNamespace = string ( * certificateRef . Namespace )
2021-11-09 10:34:06 +00:00
}
2024-09-03 10:10:04 +00:00
if err := p . isReferenceGranted ( kindGateway , gateway . Namespace , groupCore , "Secret" , string ( certificateRef . Name ) , certificateNamespace ) ; err != nil {
2024-06-04 12:16:04 +00:00
gatewayListeners [ i ] . Status . Conditions = append ( gatewayListeners [ i ] . Status . Conditions , metav1 . Condition {
Type : string ( gatev1 . ListenerConditionResolvedRefs ) ,
Status : metav1 . ConditionFalse ,
ObservedGeneration : gateway . Generation ,
LastTransitionTime : metav1 . Now ( ) ,
Reason : string ( gatev1 . ListenerReasonRefNotPermitted ) ,
Message : fmt . Sprintf ( "Cannot load CertificateRef %s/%s: %s" , certificateNamespace , certificateRef . Name , err ) ,
} )
2021-05-20 09:50:12 +00:00
2024-06-04 12:16:04 +00:00
continue
2024-01-30 15:44:05 +00:00
}
configKey := certificateNamespace + "/" + string ( certificateRef . Name )
if _ , tlsExists := tlsConfigs [ configKey ] ; ! tlsExists {
2024-06-13 09:16:04 +00:00
tlsConf , err := p . getTLS ( certificateRef . Name , certificateNamespace )
2024-01-30 15:44:05 +00:00
if err != nil {
// update "ResolvedRefs" status false with "InvalidCertificateRef" reason
// update "Programmed" status false with "Invalid" reason
2024-05-28 12:30:04 +00:00
gatewayListeners [ i ] . Status . Conditions = append ( gatewayListeners [ i ] . Status . Conditions ,
2024-01-30 15:44:05 +00:00
metav1 . Condition {
Type : string ( gatev1 . ListenerConditionResolvedRefs ) ,
Status : metav1 . ConditionFalse ,
ObservedGeneration : gateway . Generation ,
LastTransitionTime : metav1 . Now ( ) ,
Reason : string ( gatev1 . ListenerReasonInvalidCertificateRef ) ,
Message : fmt . Sprintf ( "Error while retrieving certificate: %v" , err ) ,
} ,
metav1 . Condition {
Type : string ( gatev1 . ListenerConditionProgrammed ) ,
Status : metav1 . ConditionFalse ,
ObservedGeneration : gateway . Generation ,
LastTransitionTime : metav1 . Now ( ) ,
Reason : string ( gatev1 . ListenerReasonInvalid ) ,
Message : fmt . Sprintf ( "Error while retrieving certificate: %v" , err ) ,
} ,
)
2021-05-20 09:50:12 +00:00
2024-01-30 15:44:05 +00:00
continue
}
2021-05-20 09:50:12 +00:00
tlsConfigs [ configKey ] = tlsConf
}
2020-12-15 15:40:05 +00:00
}
}
2024-05-28 12:30:04 +00:00
gatewayListeners [ i ] . Attached = true
}
2024-05-01 04:38:03 +00:00
2024-05-28 12:30:04 +00:00
if len ( tlsConfigs ) > 0 {
conf . TLS . Certificates = append ( conf . TLS . Certificates , getTLSConfig ( tlsConfigs ) ... )
2021-05-20 09:50:12 +00:00
}
2024-05-28 12:30:04 +00:00
return gatewayListeners
2021-05-20 09:50:12 +00:00
}
2020-12-15 15:40:05 +00:00
2024-09-03 10:10:04 +00:00
func ( p * Provider ) makeGatewayStatus ( gateway * gatev1 . Gateway , listeners [ ] gatewayListener , addresses [ ] gatev1 . GatewayStatusAddress ) ( gatev1 . GatewayStatus , [ ] metav1 . Condition ) {
2024-04-10 07:34:07 +00:00
gatewayStatus := gatev1 . GatewayStatus { Addresses : addresses }
2021-10-04 13:46:08 +00:00
2024-09-03 10:10:04 +00:00
var errorConditions [ ] metav1 . Condition
2024-05-28 12:30:04 +00:00
for _ , listener := range listeners {
if len ( listener . Status . Conditions ) == 0 {
listener . Status . Conditions = append ( listener . Status . Conditions ,
2024-01-30 15:44:05 +00:00
metav1 . Condition {
Type : string ( gatev1 . ListenerConditionAccepted ) ,
Status : metav1 . ConditionTrue ,
ObservedGeneration : gateway . Generation ,
LastTransitionTime : metav1 . Now ( ) ,
Reason : string ( gatev1 . ListenerReasonAccepted ) ,
Message : "No error found" ,
} ,
metav1 . Condition {
Type : string ( gatev1 . ListenerConditionResolvedRefs ) ,
Status : metav1 . ConditionTrue ,
ObservedGeneration : gateway . Generation ,
LastTransitionTime : metav1 . Now ( ) ,
Reason : string ( gatev1 . ListenerReasonResolvedRefs ) ,
Message : "No error found" ,
} ,
metav1 . Condition {
Type : string ( gatev1 . ListenerConditionProgrammed ) ,
Status : metav1 . ConditionTrue ,
ObservedGeneration : gateway . Generation ,
LastTransitionTime : metav1 . Now ( ) ,
Reason : string ( gatev1 . ListenerReasonProgrammed ) ,
Message : "No error found" ,
} ,
)
2021-11-09 10:34:06 +00:00
2024-05-28 12:30:04 +00:00
// TODO: refactor
gatewayStatus . Listeners = append ( gatewayStatus . Listeners , * listener . Status )
2021-11-09 10:34:06 +00:00
continue
}
2024-09-03 10:10:04 +00:00
errorConditions = append ( errorConditions , listener . Status . Conditions ... )
2024-05-28 12:30:04 +00:00
gatewayStatus . Listeners = append ( gatewayStatus . Listeners , * listener . Status )
2021-07-15 15:20:08 +00:00
}
2024-09-03 10:10:04 +00:00
if len ( errorConditions ) > 0 {
2021-11-09 10:34:06 +00:00
// GatewayConditionReady "Ready", GatewayConditionReason "ListenersNotValid"
gatewayStatus . Conditions = append ( gatewayStatus . Conditions , metav1 . Condition {
2024-01-09 09:28:05 +00:00
Type : string ( gatev1 . GatewayConditionAccepted ) ,
2021-11-09 10:34:06 +00:00
Status : metav1 . ConditionFalse ,
2024-01-09 09:28:05 +00:00
ObservedGeneration : gateway . Generation ,
2021-11-09 10:34:06 +00:00
LastTransitionTime : metav1 . Now ( ) ,
2024-01-09 09:28:05 +00:00
Reason : string ( gatev1 . GatewayReasonListenersNotValid ) ,
2021-11-09 10:34:06 +00:00
Message : "All Listeners must be valid" ,
} )
2024-09-03 10:10:04 +00:00
return gatewayStatus , errorConditions
2021-11-09 10:34:06 +00:00
}
gatewayStatus . Conditions = append ( gatewayStatus . Conditions ,
2024-01-09 09:28:05 +00:00
// update "Accepted" status with "Accepted" reason
2021-11-09 10:34:06 +00:00
metav1 . Condition {
2024-01-09 09:28:05 +00:00
Type : string ( gatev1 . GatewayConditionAccepted ) ,
2021-11-09 10:34:06 +00:00
Status : metav1 . ConditionTrue ,
2024-01-09 09:28:05 +00:00
ObservedGeneration : gateway . Generation ,
2024-01-22 14:30:05 +00:00
Reason : string ( gatev1 . GatewayReasonAccepted ) ,
Message : "Gateway successfully scheduled" ,
LastTransitionTime : metav1 . Now ( ) ,
} ,
// update "Programmed" status with "Programmed" reason
metav1 . Condition {
Type : string ( gatev1 . GatewayConditionProgrammed ) ,
Status : metav1 . ConditionTrue ,
ObservedGeneration : gateway . Generation ,
Reason : string ( gatev1 . GatewayReasonProgrammed ) ,
2024-01-09 09:28:05 +00:00
Message : "Gateway successfully scheduled" ,
2021-11-09 10:34:06 +00:00
LastTransitionTime : metav1 . Now ( ) ,
} ,
)
return gatewayStatus , nil
}
2024-06-13 09:16:04 +00:00
func ( p * Provider ) gatewayAddresses ( ) ( [ ] gatev1 . GatewayStatusAddress , error ) {
2024-04-10 07:34:07 +00:00
if p . StatusAddress == nil {
return nil , nil
}
if p . StatusAddress . IP != "" {
return [ ] gatev1 . GatewayStatusAddress { {
Type : ptr . To ( gatev1 . IPAddressType ) ,
Value : p . StatusAddress . IP ,
} } , nil
}
if p . StatusAddress . Hostname != "" {
return [ ] gatev1 . GatewayStatusAddress { {
Type : ptr . To ( gatev1 . HostnameAddressType ) ,
Value : p . StatusAddress . Hostname ,
} } , nil
}
svcRef := p . StatusAddress . Service
if svcRef . Name != "" && svcRef . Namespace != "" {
2024-06-13 09:16:04 +00:00
svc , exists , err := p . client . GetService ( svcRef . Namespace , svcRef . Name )
2024-04-10 07:34:07 +00:00
if err != nil {
return nil , fmt . Errorf ( "unable to get service: %w" , err )
}
if ! exists {
return nil , fmt . Errorf ( "could not find a service with name %s in namespace %s" , svcRef . Name , svcRef . Namespace )
}
var addresses [ ] gatev1 . GatewayStatusAddress
for _ , addr := range svc . Status . LoadBalancer . Ingress {
switch {
case addr . IP != "" :
addresses = append ( addresses , gatev1 . GatewayStatusAddress {
Type : ptr . To ( gatev1 . IPAddressType ) ,
Value : addr . IP ,
} )
case addr . Hostname != "" :
addresses = append ( addresses , gatev1 . GatewayStatusAddress {
Type : ptr . To ( gatev1 . HostnameAddressType ) ,
Value : addr . Hostname ,
} )
}
}
return addresses , nil
}
return nil , errors . New ( "empty Gateway status address configuration" )
}
2024-01-09 09:28:05 +00:00
func ( p * Provider ) entryPointName ( port gatev1 . PortNumber , protocol gatev1 . ProtocolType ) ( string , error ) {
2021-11-09 10:34:06 +00:00
portStr := strconv . FormatInt ( int64 ( port ) , 10 )
for name , entryPoint := range p . EntryPoints {
if strings . HasSuffix ( entryPoint . Address , ":" + portStr ) {
// If the protocol is HTTP the entryPoint must have no TLS conf
2024-01-09 09:28:05 +00:00
// Not relevant for gatev1.TLSProtocolType && gatev1.TCPProtocolType
if protocol == gatev1 . HTTPProtocolType && entryPoint . HasHTTPTLSConf {
2021-11-09 10:34:06 +00:00
continue
}
return name , nil
}
}
return "" , fmt . Errorf ( "no matching entryPoint for port %d and protocol %q" , port , protocol )
}
2024-09-03 10:10:04 +00:00
func ( p * Provider ) isReferenceGranted ( fromKind , fromNamespace , toGroup , toKind , toName , toNamespace string ) error {
2024-06-13 09:16:04 +00:00
if toNamespace == fromNamespace {
return nil
}
refGrants , err := p . client . ListReferenceGrants ( toNamespace )
if err != nil {
return fmt . Errorf ( "listing ReferenceGrant: %w" , err )
}
2024-09-03 10:10:04 +00:00
refGrants = filterReferenceGrantsFrom ( refGrants , groupGateway , fromKind , fromNamespace )
2024-06-13 09:16:04 +00:00
refGrants = filterReferenceGrantsTo ( refGrants , toGroup , toKind , toName )
if len ( refGrants ) == 0 {
return errors . New ( "missing ReferenceGrant" )
}
return nil
}
func ( p * Provider ) getTLS ( secretName gatev1 . ObjectName , namespace string ) ( * tls . CertAndStores , error ) {
secret , exists , err := p . client . GetSecret ( namespace , string ( secretName ) )
if err != nil {
return nil , fmt . Errorf ( "failed to fetch secret %s/%s: %w" , namespace , secretName , err )
}
if ! exists {
return nil , fmt . Errorf ( "secret %s/%s does not exist" , namespace , secretName )
}
cert , key , err := getCertificateBlocks ( secret , namespace , string ( secretName ) )
if err != nil {
return nil , err
}
return & tls . CertAndStores {
Certificate : tls . Certificate {
CertFile : types . FileOrContent ( cert ) ,
KeyFile : types . FileOrContent ( key ) ,
} ,
} , nil
}
func ( p * Provider ) allowedNamespaces ( gatewayNamespace string , routeNamespaces * gatev1 . RouteNamespaces ) ( [ ] string , error ) {
if routeNamespaces == nil || routeNamespaces . From == nil {
return [ ] string { gatewayNamespace } , nil
}
switch * routeNamespaces . From {
case gatev1 . NamespacesFromAll :
return [ ] string { metav1 . NamespaceAll } , nil
case gatev1 . NamespacesFromSame :
return [ ] string { gatewayNamespace } , nil
case gatev1 . NamespacesFromSelector :
selector , err := metav1 . LabelSelectorAsSelector ( routeNamespaces . Selector )
if err != nil {
return nil , fmt . Errorf ( "malformed selector: %w" , err )
}
return p . client . ListNamespaces ( selector )
}
return nil , fmt . Errorf ( "unsupported RouteSelectType: %q" , * routeNamespaces . From )
}
2024-09-09 08:08:08 +00:00
type backendAddress struct {
2024-10-02 08:34:04 +00:00
IP string
Port int32
2024-09-09 08:08:08 +00:00
}
func ( p * Provider ) getBackendAddresses ( namespace string , ref gatev1 . BackendRef ) ( [ ] backendAddress , corev1 . ServicePort , error ) {
if ref . Port == nil {
return nil , corev1 . ServicePort { } , errors . New ( "port is required for Kubernetes Service reference" )
}
service , exists , err := p . client . GetService ( namespace , string ( ref . Name ) )
if err != nil {
return nil , corev1 . ServicePort { } , fmt . Errorf ( "getting service: %w" , err )
}
if ! exists {
return nil , corev1 . ServicePort { } , errors . New ( "service not found" )
}
2024-10-02 08:34:04 +00:00
if service . Spec . Type == corev1 . ServiceTypeExternalName {
return nil , corev1 . ServicePort { } , errors . New ( "type ExternalName is not supported for Kubernetes Service reference" )
}
2024-09-09 08:08:08 +00:00
var svcPort * corev1 . ServicePort
for _ , p := range service . Spec . Ports {
if p . Port == int32 ( * ref . Port ) {
svcPort = & p
break
}
}
if svcPort == nil {
return nil , corev1 . ServicePort { } , fmt . Errorf ( "service port %d not found" , * ref . Port )
}
2024-10-02 08:34:04 +00:00
annotationsConfig , err := parseServiceAnnotations ( service . Annotations )
if err != nil {
return nil , corev1 . ServicePort { } , fmt . Errorf ( "parsing service annotations config: %w" , err )
}
if p . NativeLBByDefault || annotationsConfig . Service . NativeLB {
if service . Spec . ClusterIP == "" || service . Spec . ClusterIP == "None" {
return nil , corev1 . ServicePort { } , fmt . Errorf ( "no clusterIP found for service: %s/%s" , service . Namespace , service . Name )
}
return [ ] backendAddress { {
IP : service . Spec . ClusterIP ,
Port : svcPort . Port ,
} } , * svcPort , nil
}
2024-09-09 08:08:08 +00:00
endpointSlices , err := p . client . ListEndpointSlicesForService ( namespace , string ( ref . Name ) )
if err != nil {
return nil , corev1 . ServicePort { } , fmt . Errorf ( "getting endpointslices: %w" , err )
}
if len ( endpointSlices ) == 0 {
return nil , corev1 . ServicePort { } , errors . New ( "endpointslices not found" )
}
uniqAddresses := map [ string ] struct { } { }
backendServers := make ( [ ] backendAddress , 0 )
for _ , endpointSlice := range endpointSlices {
var port int32
for _ , p := range endpointSlice . Ports {
if svcPort . Name == * p . Name {
port = * p . Port
break
}
}
if port == 0 {
continue
}
for _ , endpoint := range endpointSlice . Endpoints {
if endpoint . Conditions . Ready == nil || ! * endpoint . Conditions . Ready {
continue
}
for _ , address := range endpoint . Addresses {
if _ , ok := uniqAddresses [ address ] ; ok {
continue
}
uniqAddresses [ address ] = struct { } { }
backendServers = append ( backendServers , backendAddress {
2024-10-02 08:34:04 +00:00
IP : address ,
Port : port ,
2024-09-09 08:08:08 +00:00
} )
}
}
}
return backendServers , * svcPort , nil
}
2024-05-28 12:30:04 +00:00
func supportedRouteKinds ( protocol gatev1 . ProtocolType , experimentalChannel bool ) ( [ ] gatev1 . RouteGroupKind , [ ] metav1 . Condition ) {
group := gatev1 . Group ( gatev1 . GroupName )
2021-11-09 10:34:06 +00:00
2024-05-28 12:30:04 +00:00
switch protocol {
case gatev1 . TCPProtocolType :
if experimentalChannel {
return [ ] gatev1 . RouteGroupKind { { Kind : kindTCPRoute , Group : & group } } , nil
}
return nil , [ ] metav1 . Condition { {
Type : string ( gatev1 . ListenerConditionConflicted ) ,
Status : metav1 . ConditionTrue ,
2021-05-20 09:50:12 +00:00
LastTransitionTime : metav1 . Now ( ) ,
2024-05-28 12:30:04 +00:00
Reason : string ( gatev1 . ListenerReasonProtocolConflict ) ,
Message : fmt . Sprintf ( "Protocol %q requires the experimental channel support to be enabled, please use the `experimentalChannel` option" , protocol ) ,
} }
2021-10-04 13:46:08 +00:00
2024-05-28 12:30:04 +00:00
case gatev1 . HTTPProtocolType , gatev1 . HTTPSProtocolType :
2024-08-30 08:36:06 +00:00
return [ ] gatev1 . RouteGroupKind {
{ Kind : kindHTTPRoute , Group : & group } ,
{ Kind : kindGRPCRoute , Group : & group } ,
} , nil
2024-05-28 12:30:04 +00:00
case gatev1 . TLSProtocolType :
if experimentalChannel {
return [ ] gatev1 . RouteGroupKind {
{ Kind : kindTCPRoute , Group : & group } ,
{ Kind : kindTLSRoute , Group : & group } ,
} , nil
}
return nil , [ ] metav1 . Condition { {
Type : string ( gatev1 . ListenerConditionConflicted ) ,
Status : metav1 . ConditionTrue ,
2021-10-04 13:46:08 +00:00
LastTransitionTime : metav1 . Now ( ) ,
2024-05-28 12:30:04 +00:00
Reason : string ( gatev1 . ListenerReasonInvalidRouteKinds ) ,
Message : fmt . Sprintf ( "Protocol %q requires the experimental channel support to be enabled, please use the `experimentalChannel` option" , protocol ) ,
} }
2021-05-20 09:50:12 +00:00
}
2024-05-28 12:30:04 +00:00
return nil , [ ] metav1 . Condition { {
Type : string ( gatev1 . ListenerConditionConflicted ) ,
Status : metav1 . ConditionTrue ,
LastTransitionTime : metav1 . Now ( ) ,
Reason : string ( gatev1 . ListenerReasonUnsupportedProtocol ) ,
Message : fmt . Sprintf ( "Unsupported listener protocol %q" , protocol ) ,
} }
}
2021-05-20 09:50:12 +00:00
2024-05-28 12:30:04 +00:00
func allowedRouteKinds ( gateway * gatev1 . Gateway , listener gatev1 . Listener , supportedKinds [ ] gatev1 . RouteGroupKind ) ( [ ] gatev1 . RouteGroupKind , [ ] metav1 . Condition ) {
if listener . AllowedRoutes == nil || len ( listener . AllowedRoutes . Kinds ) == 0 {
return supportedKinds , nil
}
2024-05-22 15:20:04 +00:00
2024-05-28 12:30:04 +00:00
var conditions [ ] metav1 . Condition
routeKinds := [ ] gatev1 . RouteGroupKind { }
uniqRouteKinds := map [ gatev1 . Kind ] struct { } { }
for _ , routeKind := range listener . AllowedRoutes . Kinds {
var isSupported bool
for _ , kind := range supportedKinds {
if routeKind . Kind == kind . Kind && routeKind . Group != nil && * routeKind . Group == * kind . Group {
isSupported = true
break
}
2021-11-09 10:34:06 +00:00
}
2024-05-28 12:30:04 +00:00
if ! isSupported {
conditions = append ( conditions , metav1 . Condition {
Type : string ( gatev1 . ListenerConditionResolvedRefs ) ,
Status : metav1 . ConditionFalse ,
ObservedGeneration : gateway . Generation ,
2024-05-22 15:20:04 +00:00
LastTransitionTime : metav1 . Now ( ) ,
2024-05-28 12:30:04 +00:00
Reason : string ( gatev1 . ListenerReasonInvalidRouteKinds ) ,
Message : fmt . Sprintf ( "Listener protocol %q does not support RouteGroupKind %s/%s" , listener . Protocol , groupToString ( routeKind . Group ) , routeKind . Kind ) ,
} )
2021-11-09 10:34:06 +00:00
continue
}
2024-05-28 12:30:04 +00:00
if _ , exists := uniqRouteKinds [ routeKind . Kind ] ; ! exists {
routeKinds = append ( routeKinds , routeKind )
uniqRouteKinds [ routeKind . Kind ] = struct { } { }
2020-12-15 15:40:05 +00:00
}
2024-05-28 12:30:04 +00:00
}
2020-12-15 15:40:05 +00:00
2024-05-28 12:30:04 +00:00
return routeKinds , conditions
}
2021-05-20 09:50:12 +00:00
2024-05-28 12:30:04 +00:00
func findMatchingHostnames ( listenerHostname * gatev1 . Hostname , routeHostnames [ ] gatev1 . Hostname ) ( [ ] gatev1 . Hostname , bool ) {
if listenerHostname == nil {
return routeHostnames , true
2020-12-15 15:40:05 +00:00
}
2024-05-28 12:30:04 +00:00
if len ( routeHostnames ) == 0 {
return [ ] gatev1 . Hostname { * listenerHostname } , true
2020-12-15 15:40:05 +00:00
}
2024-05-01 04:38:03 +00:00
var matches [ ] gatev1 . Hostname
2024-05-28 12:30:04 +00:00
for _ , routeHostname := range routeHostnames {
if match := findMatchingHostname ( * listenerHostname , routeHostname ) ; match != "" {
matches = append ( matches , match )
2024-03-25 13:38:04 +00:00
continue
2020-12-15 15:40:05 +00:00
}
2024-05-28 12:30:04 +00:00
if match := findMatchingHostname ( routeHostname , * listenerHostname ) ; match != "" {
matches = append ( matches , match )
2024-05-01 04:38:03 +00:00
continue
}
}
2022-11-16 10:38:07 +00:00
2024-05-28 12:30:04 +00:00
return matches , len ( matches ) > 0
2024-05-01 04:38:03 +00:00
}
2020-12-15 15:40:05 +00:00
2024-05-28 12:30:04 +00:00
func findMatchingHostname ( h1 , h2 gatev1 . Hostname ) gatev1 . Hostname {
if h1 == h2 {
return h1
2024-05-01 04:38:03 +00:00
}
2020-12-15 15:40:05 +00:00
2024-05-28 12:30:04 +00:00
if ! strings . HasPrefix ( string ( h1 ) , "*." ) {
return ""
2024-05-01 04:38:03 +00:00
}
2024-05-28 12:30:04 +00:00
trimmedH1 := strings . TrimPrefix ( string ( h1 ) , "*" )
// root domain doesn't match subdomain wildcard.
if trimmedH1 == string ( h2 ) {
return ""
2024-05-01 04:38:03 +00:00
}
2024-05-28 12:30:04 +00:00
if ! strings . HasSuffix ( string ( h2 ) , trimmedH1 ) {
return ""
2024-05-01 04:38:03 +00:00
}
2020-12-15 15:40:05 +00:00
2024-05-28 12:30:04 +00:00
return lessWildcards ( h1 , h2 )
2024-05-01 04:38:03 +00:00
}
2020-12-15 15:40:05 +00:00
2024-05-28 12:30:04 +00:00
func lessWildcards ( h1 , h2 gatev1 . Hostname ) gatev1 . Hostname {
if strings . Count ( string ( h1 ) , "*" ) > strings . Count ( string ( h2 ) , "*" ) {
return h2
2024-03-25 13:38:04 +00:00
}
2024-05-28 12:30:04 +00:00
return h1
2024-03-25 13:38:04 +00:00
}
2024-05-28 12:30:04 +00:00
func allowRoute ( listener gatewayListener , routeNamespace , routeKind string ) bool {
if ! slices . Contains ( listener . AllowedRouteKinds , routeKind ) {
return false
2024-05-01 04:38:03 +00:00
}
2024-05-28 12:30:04 +00:00
return slices . ContainsFunc ( listener . AllowedNamespaces , func ( allowedNamespace string ) bool {
return allowedNamespace == corev1 . NamespaceAll || allowedNamespace == routeNamespace
} )
}
2021-05-20 09:50:12 +00:00
2024-10-09 13:50:04 +00:00
func matchingGatewayListeners ( gatewayListeners [ ] gatewayListener , routeNamespace string , parentRefs [ ] gatev1 . ParentReference ) [ ] gatewayListener {
var listeners [ ] gatewayListener
2021-05-20 09:50:12 +00:00
2024-10-09 13:50:04 +00:00
for _ , listener := range gatewayListeners {
for _ , parentRef := range parentRefs {
if ptr . Deref ( parentRef . Group , gatev1 . GroupName ) != gatev1 . GroupName {
continue
}
2021-05-20 09:50:12 +00:00
2024-10-09 13:50:04 +00:00
if ptr . Deref ( parentRef . Kind , kindGateway ) != kindGateway {
continue
}
2021-05-20 09:50:12 +00:00
2024-10-09 13:50:04 +00:00
parentRefNamespace := string ( ptr . Deref ( parentRef . Namespace , gatev1 . Namespace ( routeNamespace ) ) )
if listener . GWNamespace != parentRefNamespace {
continue
}
if string ( parentRef . Name ) != listener . GWName {
continue
}
listeners = append ( listeners , listener )
}
2024-05-01 04:38:03 +00:00
}
2021-05-20 09:50:12 +00:00
2024-10-09 13:50:04 +00:00
return listeners
}
func matchListener ( listener gatewayListener , parentRef gatev1 . ParentReference ) bool {
2024-05-28 12:30:04 +00:00
sectionName := string ( ptr . Deref ( parentRef . SectionName , "" ) )
if sectionName != "" && sectionName != listener . Name {
return false
2024-05-01 04:38:03 +00:00
}
2021-05-20 09:50:12 +00:00
2024-09-27 10:12:05 +00:00
if parentRef . Port != nil && * parentRef . Port != listener . Port {
return false
}
2024-05-28 12:30:04 +00:00
return true
2024-05-01 04:38:03 +00:00
}
2021-05-20 09:50:12 +00:00
2024-06-13 09:16:04 +00:00
func makeRouterName ( rule , name string ) string {
2024-05-01 04:38:03 +00:00
h := sha256 . New ( )
2021-05-20 09:50:12 +00:00
2024-05-28 12:30:04 +00:00
// As explained in https://pkg.go.dev/hash#Hash,
// Write never returns an error.
h . Write ( [ ] byte ( rule ) )
2021-05-20 09:50:12 +00:00
2024-05-28 12:30:04 +00:00
return fmt . Sprintf ( "%s-%.10x" , name , h . Sum ( nil ) )
2024-05-01 04:38:03 +00:00
}
2021-05-20 09:50:12 +00:00
2024-05-01 04:38:03 +00:00
func getTLSConfig ( tlsConfigs map [ string ] * tls . CertAndStores ) [ ] * tls . CertAndStores {
var secretNames [ ] string
for secretName := range tlsConfigs {
secretNames = append ( secretNames , secretName )
2022-12-22 14:02:05 +00:00
}
2024-05-01 04:38:03 +00:00
sort . Strings ( secretNames )
2022-12-22 14:02:05 +00:00
2024-05-01 04:38:03 +00:00
var configs [ ] * tls . CertAndStores
for _ , secretName := range secretNames {
configs = append ( configs , tlsConfigs [ secretName ] )
}
2024-03-25 13:38:04 +00:00
2024-05-01 04:38:03 +00:00
return configs
}
2024-03-25 13:38:04 +00:00
2024-05-01 04:38:03 +00:00
func getCertificateBlocks ( secret * corev1 . Secret , namespace , secretName string ) ( string , string , error ) {
var missingEntries [ ] string
2024-04-05 15:18:03 +00:00
2024-05-01 04:38:03 +00:00
tlsCrtData , tlsCrtExists := secret . Data [ "tls.crt" ]
if ! tlsCrtExists {
missingEntries = append ( missingEntries , "tls.crt" )
}
2024-04-05 15:18:03 +00:00
2024-05-01 04:38:03 +00:00
tlsKeyData , tlsKeyExists := secret . Data [ "tls.key" ]
if ! tlsKeyExists {
missingEntries = append ( missingEntries , "tls.key" )
2022-12-22 14:02:05 +00:00
}
2024-05-01 04:38:03 +00:00
if len ( missingEntries ) > 0 {
return "" , "" , fmt . Errorf ( "secret %s/%s is missing the following TLS data entries: %s" ,
namespace , secretName , strings . Join ( missingEntries , ", " ) )
}
2022-12-22 14:02:05 +00:00
2024-05-01 04:38:03 +00:00
cert := string ( tlsCrtData )
if cert == "" {
missingEntries = append ( missingEntries , "tls.crt" )
2024-03-25 13:38:04 +00:00
}
2024-05-01 04:38:03 +00:00
key := string ( tlsKeyData )
if key == "" {
missingEntries = append ( missingEntries , "tls.key" )
2024-03-25 13:38:04 +00:00
}
2024-05-01 04:38:03 +00:00
if len ( missingEntries ) > 0 {
return "" , "" , fmt . Errorf ( "secret %s/%s contains the following empty TLS data entries: %s" ,
namespace , secretName , strings . Join ( missingEntries , ", " ) )
2024-03-25 13:38:04 +00:00
}
2024-05-01 04:38:03 +00:00
return cert , key , nil
2024-03-25 13:38:04 +00:00
}
2020-12-15 15:40:05 +00:00
func throttleEvents ( ctx context . Context , throttleDuration time . Duration , pool * safe . Pool , eventsChan <- chan interface { } ) chan interface { } {
if throttleDuration == 0 {
return nil
}
// Create a buffered channel to hold the pending event (if we're delaying processing the event due to throttling)
eventsChanBuffered := make ( chan interface { } , 1 )
// Run a goroutine that reads events from eventChan and does a non-blocking write to pendingEvent.
// This guarantees that writing to eventChan will never block,
// and that pendingEvent will have something in it if there's been an event since we read from that channel.
pool . GoCtx ( func ( ctxPool context . Context ) {
for {
select {
case <- ctxPool . Done ( ) :
return
case nextEvent := <- eventsChan :
select {
case eventsChanBuffered <- nextEvent :
default :
// We already have an event in eventsChanBuffered, so we'll do a refresh as soon as our throttle allows us to.
// It's fine to drop the event and keep whatever's in the buffer -- we don't do different things for different events
2022-11-21 17:36:05 +00:00
log . Ctx ( ctx ) . Debug ( ) . Msgf ( "Dropping event kind %T due to throttling" , nextEvent )
2020-12-15 15:40:05 +00:00
}
}
}
} )
return eventsChanBuffered
}
2021-02-02 18:36:04 +00:00
2024-01-09 09:28:05 +00:00
func isTraefikService ( ref gatev1 . BackendRef ) bool {
2021-11-09 10:34:06 +00:00
if ref . Kind == nil || ref . Group == nil {
return false
}
2023-03-22 15:40:06 +00:00
return * ref . Group == traefikv1alpha1 . GroupName && * ref . Kind == kindTraefikService
2023-03-20 14:38:08 +00:00
}
2024-01-09 09:28:05 +00:00
func isInternalService ( ref gatev1 . BackendRef ) bool {
2023-03-20 14:38:08 +00:00
return isTraefikService ( ref ) && strings . HasSuffix ( string ( ref . Name ) , "@internal" )
2021-02-02 18:36:04 +00:00
}
2022-06-23 09:58:09 +00:00
// makeListenerKey joins protocol, hostname, and port of a listener into a string key.
2024-01-09 09:28:05 +00:00
func makeListenerKey ( l gatev1 . Listener ) string {
var hostname gatev1 . Hostname
2022-06-23 09:58:09 +00:00
if l . Hostname != nil {
hostname = * l . Hostname
}
return fmt . Sprintf ( "%s|%s|%d" , l . Protocol , hostname , l . Port )
}
2024-01-30 15:44:05 +00:00
func filterReferenceGrantsFrom ( referenceGrants [ ] * gatev1beta1 . ReferenceGrant , group , kind , namespace string ) [ ] * gatev1beta1 . ReferenceGrant {
var matchingReferenceGrants [ ] * gatev1beta1 . ReferenceGrant
for _ , referenceGrant := range referenceGrants {
if referenceGrantMatchesFrom ( referenceGrant , group , kind , namespace ) {
matchingReferenceGrants = append ( matchingReferenceGrants , referenceGrant )
}
}
return matchingReferenceGrants
}
func referenceGrantMatchesFrom ( referenceGrant * gatev1beta1 . ReferenceGrant , group , kind , namespace string ) bool {
for _ , from := range referenceGrant . Spec . From {
sanitizedGroup := string ( from . Group )
if sanitizedGroup == "" {
sanitizedGroup = groupCore
}
if string ( from . Namespace ) != namespace || string ( from . Kind ) != kind || sanitizedGroup != group {
continue
}
return true
}
return false
}
func filterReferenceGrantsTo ( referenceGrants [ ] * gatev1beta1 . ReferenceGrant , group , kind , name string ) [ ] * gatev1beta1 . ReferenceGrant {
var matchingReferenceGrants [ ] * gatev1beta1 . ReferenceGrant
for _ , referenceGrant := range referenceGrants {
if referenceGrantMatchesTo ( referenceGrant , group , kind , name ) {
matchingReferenceGrants = append ( matchingReferenceGrants , referenceGrant )
}
}
return matchingReferenceGrants
}
func referenceGrantMatchesTo ( referenceGrant * gatev1beta1 . ReferenceGrant , group , kind , name string ) bool {
for _ , to := range referenceGrant . Spec . To {
sanitizedGroup := string ( to . Group )
if sanitizedGroup == "" {
sanitizedGroup = groupCore
}
if string ( to . Kind ) != kind || sanitizedGroup != group || ( to . Name != nil && string ( * to . Name ) != name ) {
continue
}
return true
}
return false
}
func groupToString ( p * gatev1 . Group ) string {
if p == nil {
return "<nil>"
}
return string ( * p )
}
func kindToString ( p * gatev1 . Kind ) string {
if p == nil {
return "<nil>"
}
return string ( * p )
}
2024-05-01 04:38:03 +00:00
2024-06-04 12:16:04 +00:00
func updateRouteConditionAccepted ( conditions [ ] metav1 . Condition , reason string ) [ ] metav1 . Condition {
var conds [ ] metav1 . Condition
2024-05-22 15:20:04 +00:00
for _ , c := range conditions {
2024-06-04 12:16:04 +00:00
if c . Type == string ( gatev1 . RouteConditionAccepted ) && c . Status != metav1 . ConditionTrue {
c . Reason = reason
c . LastTransitionTime = metav1 . Now ( )
if reason == string ( gatev1 . RouteReasonAccepted ) {
c . Status = metav1 . ConditionTrue
}
2024-05-22 15:20:04 +00:00
}
2024-06-04 12:16:04 +00:00
conds = append ( conds , c )
2024-05-22 15:20:04 +00:00
}
2024-06-04 12:16:04 +00:00
return conds
}
func upsertRouteConditionResolvedRefs ( conditions [ ] metav1 . Condition , condition metav1 . Condition ) [ ] metav1 . Condition {
var (
curr * metav1 . Condition
conds [ ] metav1 . Condition
)
for _ , c := range conditions {
if c . Type == string ( gatev1 . RouteConditionResolvedRefs ) {
curr = & c
continue
}
conds = append ( conds , c )
}
if curr != nil && curr . Status == metav1 . ConditionFalse && condition . Status == metav1 . ConditionTrue {
return append ( conds , * curr )
}
return append ( conds , condition )
2024-05-22 15:20:04 +00:00
}
2024-07-29 13:48:05 +00:00
func upsertGatewayClassConditionAccepted ( conditions [ ] metav1 . Condition , condition metav1 . Condition ) [ ] metav1 . Condition {
var conds [ ] metav1 . Condition
for _ , c := range conditions {
if c . Type == string ( gatev1 . GatewayClassConditionStatusAccepted ) {
continue
}
conds = append ( conds , c )
}
return append ( conds , condition )
}