traefik/pkg/provider/kubernetes/crd/kubernetes_http.go

package crd

import (
	"context"
	"errors"
	"fmt"
	"net"
	"strconv"
	"strings"

	"github.com/rs/zerolog/log"
	"github.com/traefik/traefik/v3/pkg/config/dynamic"
	"github.com/traefik/traefik/v3/pkg/logs"
	"github.com/traefik/traefik/v3/pkg/provider"
	traefikv1alpha1 "github.com/traefik/traefik/v3/pkg/provider/kubernetes/crd/traefikio/v1alpha1"
	"github.com/traefik/traefik/v3/pkg/tls"
	corev1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/util/intstr"
)

const (
	roundRobinStrategy = "RoundRobin"
	httpsProtocol      = "https"
	httpProtocol       = "http"
)

func (p *Provider) loadIngressRouteConfiguration(ctx context.Context, client Client, tlsConfigs map[string]*tls.CertAndStores) *dynamic.HTTPConfiguration {
	conf := &dynamic.HTTPConfiguration{
		Routers:           map[string]*dynamic.Router{},
		Middlewares:       map[string]*dynamic.Middleware{},
		Services:          map[string]*dynamic.Service{},
		ServersTransports: map[string]*dynamic.ServersTransport{},
	}

	for _, ingressRoute := range client.GetIngressRoutes() {
		logger := log.Ctx(ctx).With().Str("ingress", ingressRoute.Name).Str("namespace", ingressRoute.Namespace).Logger()

		// TODO keep the name ingressClass?
		if !shouldProcessIngress(p.IngressClass, ingressRoute.Annotations[annotationKubernetesIngressClass]) {
			continue
		}

		err := getTLSHTTP(ctx, ingressRoute, client, tlsConfigs)
		if err != nil {
			logger.Error().Err(err).Msg("Error configuring TLS")
		}

		ingressName := ingressRoute.Name
		if len(ingressName) == 0 {
			ingressName = ingressRoute.GenerateName
		}

		cb := configBuilder{
			client:                       client,
			allowCrossNamespace:          p.AllowCrossNamespace,
			allowExternalNameServices:    p.AllowExternalNameServices,
			allowEmptyServices:           p.AllowEmptyServices,
			nativeLBByDefault:            p.NativeLBByDefault,
			disableClusterScopeResources: p.DisableClusterScopeResources,
		}

		for _, route := range ingressRoute.Spec.Routes {
			if len(route.Kind) > 0 && route.Kind != "Rule" {
				logger.Error().Msgf("Unsupported match kind: %s. Only \"Rule\" is supported for now.", route.Kind)
				continue
			}

			if len(route.Match) == 0 {
				logger.Error().Msg("Empty match rule")
				continue
			}

			serviceKey, err := makeServiceKey(route.Match, ingressName)
			if err != nil {
				logger.Error().Err(err).Send()
				continue
			}

			mds, err := p.makeMiddlewareKeys(ctx, ingressRoute.Namespace, route.Middlewares)
			if err != nil {
				logger.Error().Err(err).Msg("Failed to create middleware keys")
				continue
			}

			normalized := provider.Normalize(makeID(ingressRoute.Namespace, serviceKey))
			serviceName := normalized

			if len(route.Services) > 1 {
				spec := traefikv1alpha1.TraefikServiceSpec{
					Weighted: &traefikv1alpha1.WeightedRoundRobin{
						Services: route.Services,
					},
				}

				errBuild := cb.buildServicesLB(ctx, ingressRoute.Namespace, spec, serviceName, conf.Services)
				if errBuild != nil {
					logger.Error().Err(errBuild).Send()
					continue
				}
			} else if len(route.Services) == 1 {
				fullName, serversLB, err := cb.nameAndService(ctx, ingressRoute.Namespace, route.Services[0].LoadBalancerSpec)
				if err != nil {
					logger.Error().Err(err).Send()
					continue
				}

				if serversLB != nil {
					conf.Services[serviceName] = serversLB
				} else {
					serviceName = fullName
				}
			}

			r := &dynamic.Router{
				Middlewares: mds,
				Priority:    route.Priority,
				RuleSyntax:  route.Syntax,
				EntryPoints: ingressRoute.Spec.EntryPoints,
				Rule:        route.Match,
				Service:     serviceName,
			}

			if ingressRoute.Spec.TLS != nil {
				r.TLS = &dynamic.RouterTLSConfig{
					CertResolver: ingressRoute.Spec.TLS.CertResolver,
					Domains:      ingressRoute.Spec.TLS.Domains,
				}

				if ingressRoute.Spec.TLS.Options != nil && len(ingressRoute.Spec.TLS.Options.Name) > 0 {
					tlsOptionsName := ingressRoute.Spec.TLS.Options.Name
					// Is a Kubernetes CRD reference, (i.e. not a cross-provider reference)
					ns := ingressRoute.Spec.TLS.Options.Namespace
					if !strings.Contains(tlsOptionsName, providerNamespaceSeparator) {
						if len(ns) == 0 {
							ns = ingressRoute.Namespace
						}
						tlsOptionsName = makeID(ns, tlsOptionsName)
					} else if len(ns) > 0 {
						logger.
							Warn().Str("TLSOption", ingressRoute.Spec.TLS.Options.Name).
							Msgf("Namespace %q is ignored in cross-provider context", ns)
					}

					if !isNamespaceAllowed(p.AllowCrossNamespace, ingressRoute.Namespace, ns) {
						logger.Error().Msgf("TLSOption %s/%s is not in the IngressRoute namespace %s",
							ns, ingressRoute.Spec.TLS.Options.Name, ingressRoute.Namespace)
						continue
					}

					r.TLS.Options = tlsOptionsName
				}
			}

			p.applyRouterTransform(ctx, r, ingressRoute)

			conf.Routers[normalized] = r
		}
	}

	return conf
}

func (p *Provider) makeMiddlewareKeys(ctx context.Context, ingRouteNamespace string, middlewares []traefikv1alpha1.MiddlewareRef) ([]string, error) {
	var mds []string

	for _, mi := range middlewares {
		name := mi.Name

		if !p.AllowCrossNamespace && strings.HasSuffix(mi.Name, providerNamespaceSeparator+providerName) {
			// Since we are not able to know if another namespace is in the name (namespace-name@kubernetescrd),
			// if the provider namespace kubernetescrd is used,
			// we don't allow this format to avoid cross namespace references.
			return nil, fmt.Errorf("invalid reference to middleware %s: with crossnamespace disallowed, the namespace field needs to be explicitly specified", mi.Name)
		}

		if strings.Contains(name, providerNamespaceSeparator) {
			if len(mi.Namespace) > 0 {
				log.Ctx(ctx).
					Warn().Str(logs.MiddlewareName, mi.Name).
					Msgf("namespace %q is ignored in cross-provider context", mi.Namespace)
			}

			mds = append(mds, name)
			continue
		}

		ns := ingRouteNamespace
		if len(mi.Namespace) > 0 {
			if !isNamespaceAllowed(p.AllowCrossNamespace, ingRouteNamespace, mi.Namespace) {
				return nil, fmt.Errorf("middleware %s/%s is not in the IngressRoute namespace %s", mi.Namespace, mi.Name, ingRouteNamespace)
			}

			ns = mi.Namespace
		}

		mds = append(mds, provider.Normalize(makeID(ns, name)))
	}

	return mds, nil
}

type configBuilder struct {
	client                       Client
	allowCrossNamespace          bool
	allowExternalNameServices    bool
	allowEmptyServices           bool
	nativeLBByDefault            bool
	disableClusterScopeResources bool
}

// buildTraefikService creates the configuration for the traefik service defined in tService,
// and adds it to the given conf map.
func (c configBuilder) buildTraefikService(ctx context.Context, tService *traefikv1alpha1.TraefikService, conf map[string]*dynamic.Service) error {
	id := provider.Normalize(makeID(tService.Namespace, tService.Name))

	if tService.Spec.Weighted != nil {
		return c.buildServicesLB(ctx, tService.Namespace, tService.Spec, id, conf)
	} else if tService.Spec.Mirroring != nil {
		return c.buildMirroring(ctx, tService, id, conf)
	}

	return errors.New("unspecified service type")
}

// buildServicesLB creates the configuration for the load-balancer of services named id, and defined in tService.
// It adds it to the given conf map.
func (c configBuilder) buildServicesLB(ctx context.Context, namespace string, tService traefikv1alpha1.TraefikServiceSpec, id string, conf map[string]*dynamic.Service) error {
	var wrrServices []dynamic.WRRService

	for _, service := range tService.Weighted.Services {
		fullName, k8sService, err := c.nameAndService(ctx, namespace, service.LoadBalancerSpec)
		if err != nil {
			return err
		}

		if k8sService != nil {
			conf[fullName] = k8sService
		}

		weight := service.Weight
		if weight == nil {
			weight = func(i int) *int { return &i }(1)
		}

		wrrServices = append(wrrServices, dynamic.WRRService{
			Name:   fullName,
			Weight: weight,
		})
	}

	conf[id] = &dynamic.Service{
		Weighted: &dynamic.WeightedRoundRobin{
			Services: wrrServices,
			Sticky:   tService.Weighted.Sticky,
		},
	}
	return nil
}

// buildMirroring creates the configuration for the mirroring service named id, and defined by tService.
// It adds it to the given conf map.
func (c configBuilder) buildMirroring(ctx context.Context, tService *traefikv1alpha1.TraefikService, id string, conf map[string]*dynamic.Service) error {
	fullNameMain, k8sService, err := c.nameAndService(ctx, tService.Namespace, tService.Spec.Mirroring.LoadBalancerSpec)
	if err != nil {
		return err
	}

	if k8sService != nil {
		conf[fullNameMain] = k8sService
	}

	var mirrorServices []dynamic.MirrorService
	for _, mirror := range tService.Spec.Mirroring.Mirrors {
		mirroredName, k8sService, err := c.nameAndService(ctx, tService.Namespace, mirror.LoadBalancerSpec)
		if err != nil {
			return err
		}

		if k8sService != nil {
			conf[mirroredName] = k8sService
		}

		mirrorServices = append(mirrorServices, dynamic.MirrorService{
			Name:    mirroredName,
			Percent: mirror.Percent,
		})
	}

	conf[id] = &dynamic.Service{
		Mirroring: &dynamic.Mirroring{
			Service:     fullNameMain,
			Mirrors:     mirrorServices,
			MirrorBody:  tService.Spec.Mirroring.MirrorBody,
			MaxBodySize: tService.Spec.Mirroring.MaxBodySize,
		},
	}

	return nil
}

// buildServersLB creates the configuration for the load-balancer of servers defined by svc.
func (c configBuilder) buildServersLB(namespace string, svc traefikv1alpha1.LoadBalancerSpec) (*dynamic.Service, error) {
	servers, err := c.loadServers(namespace, svc)
	if err != nil {
		return nil, err
	}

	lb := &dynamic.ServersLoadBalancer{}
	lb.SetDefaults()
	lb.Servers = servers

	if svc.HealthCheck != nil {
		lb.HealthCheck = &dynamic.ServerHealthCheck{
			Scheme:   svc.HealthCheck.Scheme,
			Path:     svc.HealthCheck.Path,
			Method:   svc.HealthCheck.Method,
			Status:   svc.HealthCheck.Status,
			Port:     svc.HealthCheck.Port,
			Hostname: svc.HealthCheck.Hostname,
			Headers:  svc.HealthCheck.Headers,
		}
		lb.HealthCheck.SetDefaults()

		if svc.HealthCheck.FollowRedirects != nil {
			lb.HealthCheck.FollowRedirects = svc.HealthCheck.FollowRedirects
		}
		if svc.HealthCheck.Mode != "http" {
			lb.HealthCheck.Mode = svc.HealthCheck.Mode
		}
		if svc.HealthCheck.Interval != nil {
			if err := lb.HealthCheck.Interval.Set(svc.HealthCheck.Interval.String()); err != nil {
				return nil, err
			}
		}
		if svc.HealthCheck.Timeout != nil {
			if err := lb.HealthCheck.Timeout.Set(svc.HealthCheck.Timeout.String()); err != nil {
				return nil, err
			}
		}
	}

	conf := svc
	lb.PassHostHeader = conf.PassHostHeader
	if lb.PassHostHeader == nil {
		passHostHeader := true
		lb.PassHostHeader = &passHostHeader
	}

	if conf.ResponseForwarding != nil && conf.ResponseForwarding.FlushInterval != "" {
		err := lb.ResponseForwarding.FlushInterval.Set(conf.ResponseForwarding.FlushInterval)
		if err != nil {
			return nil, fmt.Errorf("unable to parse flushInterval: %w", err)
		}
	}

	lb.Sticky = svc.Sticky

	lb.ServersTransport, err = c.makeServersTransportKey(namespace, svc.ServersTransport)
	if err != nil {
		return nil, err
	}

	return &dynamic.Service{LoadBalancer: lb}, nil
}

func (c *configBuilder) makeServersTransportKey(parentNamespace string, serversTransportName string) (string, error) {
	if serversTransportName == "" {
		return "", nil
	}

	if !c.allowCrossNamespace && strings.HasSuffix(serversTransportName, providerNamespaceSeparator+providerName) {
		// Since we are not able to know if another namespace is in the name (namespace-name@kubernetescrd),
		// if the provider namespace kubernetescrd is used,
		// we don't allow this format to avoid cross namespace references.
		return "", fmt.Errorf("invalid reference to serversTransport %s: namespace-name@kubernetescrd format is not allowed when crossnamespace is disallowed", serversTransportName)
	}

	if strings.Contains(serversTransportName, providerNamespaceSeparator) {
		return serversTransportName, nil
	}

	return provider.Normalize(makeID(parentNamespace, serversTransportName)), nil
}

func (c configBuilder) loadServers(parentNamespace string, svc traefikv1alpha1.LoadBalancerSpec) ([]dynamic.Server, error) {
	strategy := svc.Strategy
	if strategy == "" {
		strategy = roundRobinStrategy
	}
	if strategy != roundRobinStrategy {
		return nil, fmt.Errorf("load balancing strategy %s is not supported", strategy)
	}

	namespace := namespaceOrFallback(svc, parentNamespace)

	if !isNamespaceAllowed(c.allowCrossNamespace, parentNamespace, namespace) {
		return nil, fmt.Errorf("load balancer service %s/%s is not in the parent resource namespace %s", svc.Namespace, svc.Name, parentNamespace)
	}

	// If the service uses explicitly the provider suffix
	sanitizedName := strings.TrimSuffix(svc.Name, providerNamespaceSeparator+providerName)
	service, exists, err := c.client.GetService(namespace, sanitizedName)
	if err != nil {
		return nil, err
	}
	if !exists {
		return nil, fmt.Errorf("kubernetes service not found: %s/%s", namespace, sanitizedName)
	}

	svcPort, err := getServicePort(service, svc.Port)
	if err != nil {
		return nil, err
	}

	if service.Spec.Type != corev1.ServiceTypeExternalName && svc.HealthCheck != nil {
		return nil, fmt.Errorf("healthCheck allowed only for ExternalName services: %s/%s", namespace, sanitizedName)
	}

	if service.Spec.Type == corev1.ServiceTypeExternalName {
		if !c.allowExternalNameServices {
			return nil, fmt.Errorf("externalName services not allowed: %s/%s", namespace, sanitizedName)
		}

		protocol, err := parseServiceProtocol(svc.Scheme, svcPort.Name, svcPort.Port)
		if err != nil {
			return nil, err
		}

		hostPort := net.JoinHostPort(service.Spec.ExternalName, strconv.Itoa(int(svcPort.Port)))

		return []dynamic.Server{{URL: fmt.Sprintf("%s://%s", protocol, hostPort)}}, nil
	}

	nativeLB := c.nativeLBByDefault
	if svc.NativeLB != nil {
		nativeLB = *svc.NativeLB
	}
	if nativeLB {
		address, err := getNativeServiceAddress(*service, *svcPort)
		if err != nil {
			return nil, fmt.Errorf("getting native Kubernetes Service address: %w", err)
		}

		protocol, err := parseServiceProtocol(svc.Scheme, svcPort.Name, svcPort.Port)
		if err != nil {
			return nil, err
		}

		return []dynamic.Server{{URL: fmt.Sprintf("%s://%s", protocol, address)}}, nil
	}

	var servers []dynamic.Server
	if service.Spec.Type == corev1.ServiceTypeNodePort && svc.NodePortLB {
		if c.disableClusterScopeResources {
			return nil, errors.New("nodes lookup is disabled")
		}

		nodes, nodesExists, nodesErr := c.client.GetNodes()
		if nodesErr != nil {
			return nil, nodesErr
		}
		if !nodesExists || len(nodes) == 0 {
			return nil, fmt.Errorf("nodes not found for NodePort service %s/%s", namespace, sanitizedName)
		}

		protocol, err := parseServiceProtocol(svc.Scheme, svcPort.Name, svcPort.Port)
		if err != nil {
			return nil, err
		}

		for _, node := range nodes {
			for _, addr := range node.Status.Addresses {
				if addr.Type == corev1.NodeInternalIP {
					hostPort := net.JoinHostPort(addr.Address, strconv.Itoa(int(svcPort.NodePort)))

					servers = append(servers, dynamic.Server{
						URL: fmt.Sprintf("%s://%s", protocol, hostPort),
					})
				}
			}
		}

		if len(servers) == 0 {
			return nil, fmt.Errorf("no servers were generated for service %s in namespace", sanitizedName)
		}

		return servers, nil
	}

	endpointSlices, err := c.client.GetEndpointSlicesForService(namespace, sanitizedName)
	if err != nil {
		return nil, fmt.Errorf("getting endpointslices: %w", err)
	}

	addresses := map[string]struct{}{}
	for _, endpointSlice := range endpointSlices {
		var port int32
		for _, p := range endpointSlice.Ports {
			if svcPort.Name == *p.Name {
				port = *p.Port
				break
			}
		}
		if port == 0 {
			continue
		}

		protocol, err := parseServiceProtocol(svc.Scheme, svcPort.Name, svcPort.Port)
		if err != nil {
			return nil, err
		}

		for _, endpoint := range endpointSlice.Endpoints {
			if endpoint.Conditions.Ready == nil || !*endpoint.Conditions.Ready {
				continue
			}

			for _, address := range endpoint.Addresses {
				if _, ok := addresses[address]; ok {
					continue
				}

				addresses[address] = struct{}{}
				servers = append(servers, dynamic.Server{
					URL: fmt.Sprintf("%s://%s", protocol, net.JoinHostPort(address, strconv.Itoa(int(port)))),
				})
			}
		}
	}

	if len(servers) == 0 && !c.allowEmptyServices {
		return nil, fmt.Errorf("no servers found for %s/%s", namespace, sanitizedName)
	}

	return servers, nil
}

// nameAndService returns the name that should be used for the svc service in the generated config.
// In addition, if the service is a Kubernetes one,
// it generates and returns the configuration part for such a service,
// so that the caller can add it to the global config map.
func (c configBuilder) nameAndService(ctx context.Context, parentNamespace string, service traefikv1alpha1.LoadBalancerSpec) (string, *dynamic.Service, error) {
	svcCtx := log.Ctx(ctx).With().Str(logs.ServiceName, service.Name).Logger().WithContext(ctx)

	namespace := namespaceOrFallback(service, parentNamespace)

	if !isNamespaceAllowed(c.allowCrossNamespace, parentNamespace, namespace) {
		return "", nil, fmt.Errorf("service %s/%s not in the parent resource namespace %s", service.Namespace, service.Name, parentNamespace)
	}

	switch {
	case service.Kind == "" || service.Kind == "Service":
		serversLB, err := c.buildServersLB(namespace, service)
		if err != nil {
			return "", nil, err
		}

		fullName := fullServiceName(svcCtx, namespace, service, service.Port)

		return fullName, serversLB, nil
	case service.Kind == "TraefikService":
		return fullServiceName(svcCtx, namespace, service, intstr.FromInt(0)), nil, nil
	default:
		return "", nil, fmt.Errorf("unsupported service kind %s", service.Kind)
	}
}

func splitSvcNameProvider(name string) (string, string) {
	parts := strings.Split(name, providerNamespaceSeparator)

	svc := strings.Join(parts[:len(parts)-1], providerNamespaceSeparator)
	pvd := parts[len(parts)-1]

	return svc, pvd
}

func fullServiceName(ctx context.Context, namespace string, service traefikv1alpha1.LoadBalancerSpec, port intstr.IntOrString) string {
	if (port.Type == intstr.Int && port.IntVal != 0) || (port.Type == intstr.String && port.StrVal != "") {
		return provider.Normalize(fmt.Sprintf("%s-%s-%s", namespace, service.Name, &port))
	}

	if !strings.Contains(service.Name, providerNamespaceSeparator) {
		return provider.Normalize(fmt.Sprintf("%s-%s", namespace, service.Name))
	}

	name, pName := splitSvcNameProvider(service.Name)
	if pName == providerName {
		return provider.Normalize(fmt.Sprintf("%s-%s", namespace, name))
	}

	if service.Namespace != "" {
		log.Ctx(ctx).Warn().Msgf("namespace %q is ignored in cross-provider context", service.Namespace)
	}

	return provider.Normalize(name) + providerNamespaceSeparator + pName
}

func namespaceOrFallback(lb traefikv1alpha1.LoadBalancerSpec, fallback string) string {
	if lb.Namespace != "" {
		return lb.Namespace
	}
	return fallback
}

// getTLSHTTP mutates tlsConfigs.
func getTLSHTTP(ctx context.Context, ingressRoute *traefikv1alpha1.IngressRoute, k8sClient Client, tlsConfigs map[string]*tls.CertAndStores) error {
	if ingressRoute.Spec.TLS == nil {
		return nil
	}
	if ingressRoute.Spec.TLS.SecretName == "" {
		log.Ctx(ctx).Debug().Msg("No secret name provided")
		return nil
	}

	configKey := ingressRoute.Namespace + "/" + ingressRoute.Spec.TLS.SecretName
	if _, tlsExists := tlsConfigs[configKey]; !tlsExists {
		tlsConf, err := getTLS(k8sClient, ingressRoute.Spec.TLS.SecretName, ingressRoute.Namespace)
		if err != nil {
			return err
		}

		tlsConfigs[configKey] = tlsConf
	}

	return nil
}

// parseServiceProtocol parses the scheme, port name, and number to determine the correct protocol.
// an error is returned if the scheme provided is invalid.
func parseServiceProtocol(providedScheme, portName string, portNumber int32) (string, error) {
	switch providedScheme {
	case httpProtocol, httpsProtocol, "h2c":
		return providedScheme, nil
	case "":
		if portNumber == 443 || strings.HasPrefix(portName, httpsProtocol) {
			return httpsProtocol, nil
		}
		return httpProtocol, nil
	}

	return "", fmt.Errorf("invalid scheme %q specified", providedScheme)
}