traefik/integration/k8s_conformance_test.go

package integration

import (
	"context"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"slices"
	"testing"
	"time"

	"github.com/stretchr/testify/require"
	"github.com/stretchr/testify/suite"
	"github.com/testcontainers/testcontainers-go"
	"github.com/testcontainers/testcontainers-go/modules/k3s"
	"github.com/testcontainers/testcontainers-go/network"
	"github.com/traefik/traefik/v3/integration/try"
	"github.com/traefik/traefik/v3/pkg/version"
	"gopkg.in/yaml.v3"
	"k8s.io/apimachinery/pkg/util/sets"
	kclientset "k8s.io/client-go/kubernetes"
	"k8s.io/client-go/tools/clientcmd"
	"sigs.k8s.io/controller-runtime/pkg/client"
	klog "sigs.k8s.io/controller-runtime/pkg/log"
	"sigs.k8s.io/controller-runtime/pkg/log/zap"
	gatev1 "sigs.k8s.io/gateway-api/apis/v1"
	gatev1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2"
	gatev1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1"
	conformanceV1alpha1 "sigs.k8s.io/gateway-api/conformance/apis/v1alpha1"
	"sigs.k8s.io/gateway-api/conformance/tests"
	"sigs.k8s.io/gateway-api/conformance/utils/config"
	ksuite "sigs.k8s.io/gateway-api/conformance/utils/suite"
)

const (
	k3sImage          = "docker.io/rancher/k3s:v1.29.3-k3s1"
	traefikImage      = "traefik/traefik:latest"
	traefikDeployment = "deployments/traefik"
	traefikNamespace  = "traefik"
)

// K8sConformanceSuite tests suite.
type K8sConformanceSuite struct {
	BaseSuite

	k3sContainer *k3s.K3sContainer
	kubeClient   client.Client
	clientSet    *kclientset.Clientset
}

func TestK8sConformanceSuite(t *testing.T) {
	suite.Run(t, new(K8sConformanceSuite))
}

func (s *K8sConformanceSuite) SetupSuite() {
	if !*k8sConformance {
		s.T().Skip("Skip because it can take a long time to execute. To enable pass the `k8sConformance` flag.")
	}

	s.BaseSuite.SetupSuite()

	// Avoid panic.
	klog.SetLogger(zap.New())

	provider, err := testcontainers.ProviderDocker.GetProvider()
	if err != nil {
		s.T().Fatal(err)
	}

	ctx := context.Background()

	// Ensure image is available locally.
	images, err := provider.ListImages(ctx)
	if err != nil {
		s.T().Fatal(err)
	}

	if !slices.ContainsFunc(images, func(img testcontainers.ImageInfo) bool {
		return img.Name == traefikImage
	}) {
		s.T().Fatal("Traefik image is not present")
	}

	s.k3sContainer, err = k3s.RunContainer(ctx,
		testcontainers.WithImage(k3sImage),
		k3s.WithManifest("./fixtures/k8s-conformance/00-experimental-v1.0.0.yml"),
		k3s.WithManifest("./fixtures/k8s-conformance/01-rbac.yml"),
		k3s.WithManifest("./fixtures/k8s-conformance/02-traefik.yml"),
		network.WithNetwork(nil, s.network),
	)
	if err != nil {
		s.T().Fatal(err)
	}

	if err = s.k3sContainer.LoadImages(ctx, traefikImage); err != nil {
		s.T().Fatal(err)
	}

	exitCode, _, err := s.k3sContainer.Exec(ctx, []string{"kubectl", "wait", "-n", traefikNamespace, traefikDeployment, "--for=condition=Available", "--timeout=30s"})
	if err != nil || exitCode > 0 {
		s.T().Fatalf("Traefik pod is not ready: %v", err)
	}

	kubeConfigYaml, err := s.k3sContainer.GetKubeConfig(ctx)
	if err != nil {
		s.T().Fatal(err)
	}

	restConfig, err := clientcmd.RESTConfigFromKubeConfig(kubeConfigYaml)
	if err != nil {
		s.T().Fatalf("Error loading Kubernetes config: %v", err)
	}

	s.kubeClient, err = client.New(restConfig, client.Options{})
	if err != nil {
		s.T().Fatalf("Error initializing Kubernetes client: %v", err)
	}

	s.clientSet, err = kclientset.NewForConfig(restConfig)
	if err != nil {
		s.T().Fatalf("Error initializing Kubernetes REST client: %v", err)
	}

	if err = gatev1alpha2.AddToScheme(s.kubeClient.Scheme()); err != nil {
		s.T().Fatal(err)
	}

	if err = gatev1beta1.AddToScheme(s.kubeClient.Scheme()); err != nil {
		s.T().Fatal(err)
	}

	if err = gatev1.AddToScheme(s.kubeClient.Scheme()); err != nil {
		s.T().Fatal(err)
	}
}

func (s *K8sConformanceSuite) TearDownSuite() {
	ctx := context.Background()

	if s.T().Failed() || *showLog {
		k3sLogs, err := s.k3sContainer.Logs(ctx)
		if err == nil {
			if res, err := io.ReadAll(k3sLogs); err == nil {
				s.T().Log(string(res))
			}
		}

		exitCode, result, err := s.k3sContainer.Exec(ctx, []string{"kubectl", "logs", "-n", traefikNamespace, traefikDeployment})
		if err == nil || exitCode == 0 {
			if res, err := io.ReadAll(result); err == nil {
				s.T().Log(string(res))
			}
		}
	}

	if err := s.k3sContainer.Terminate(ctx); err != nil {
		s.T().Fatal(err)
	}

	s.BaseSuite.TearDownSuite()
}

func (s *K8sConformanceSuite) TestK8sGatewayAPIConformance() {
	// Wait for traefik to start
	k3sContainerIP, err := s.k3sContainer.ContainerIP(context.Background())
	require.NoError(s.T(), err)

	err = try.GetRequest("http://"+k3sContainerIP+":8080/api/entrypoints", 10*time.Second, try.BodyContains(`"name":"web"`))
	require.NoError(s.T(), err)

	opts := ksuite.Options{
		Client:               s.kubeClient,
		Clientset:            s.clientSet,
		GatewayClassName:     "traefik",
		Debug:                true,
		CleanupBaseResources: true,
		TimeoutConfig: config.TimeoutConfig{
			CreateTimeout:                     5 * time.Second,
			DeleteTimeout:                     5 * time.Second,
			GetTimeout:                        5 * time.Second,
			GatewayMustHaveAddress:            5 * time.Second,
			GatewayMustHaveCondition:          5 * time.Second,
			GatewayStatusMustHaveListeners:    10 * time.Second,
			GatewayListenersMustHaveCondition: 5 * time.Second,
			GWCMustBeAccepted:                 60 * time.Second, // Pod creation in k3s cluster can be long.
			HTTPRouteMustNotHaveParents:       5 * time.Second,
			HTTPRouteMustHaveCondition:        5 * time.Second,
			TLSRouteMustHaveCondition:         5 * time.Second,
			RouteMustHaveParents:              5 * time.Second,
			ManifestFetchTimeout:              5 * time.Second,
			MaxTimeToConsistency:              5 * time.Second,
			NamespacesMustBeReady:             60 * time.Second, // Pod creation in k3s cluster can be long.
			RequestTimeout:                    5 * time.Second,
			LatestObservedGenerationSet:       5 * time.Second,
			RequiredConsecutiveSuccesses:      0,
		},
		SupportedFeatures: sets.New[ksuite.SupportedFeature]().
			Insert(ksuite.GatewayCoreFeatures.UnsortedList()...).
			Insert(ksuite.ReferenceGrantCoreFeatures.UnsortedList()...),
		EnableAllSupportedFeatures: false,
		RunTest:                    *k8sConformanceRunTest,
		// Until the feature are all supported, following tests are skipped.
		SkipTests: []string{
			"HTTPExactPathMatching",
			"HTTPRouteHostnameIntersection",
			"HTTPRouteListenerHostnameMatching",
			"HTTPRouteRequestHeaderModifier",
			"GatewayClassObservedGenerationBump",
			"HTTPRouteInvalidNonExistentBackendRef",
			"GatewayWithAttachedRoutes",
			"HTTPRouteCrossNamespace",
			"HTTPRouteDisallowedKind",
			"HTTPRouteInvalidReferenceGrant",
			"HTTPRouteObservedGenerationBump",
			"TLSRouteSimpleSameNamespace",
			"TLSRouteInvalidReferenceGrant",
			"HTTPRouteInvalidCrossNamespaceParentRef",
			"HTTPRouteInvalidParentRefNotMatchingSectionName",
			"GatewayModifyListeners",
			"GatewayInvalidTLSConfiguration",
			"HTTPRouteInvalidCrossNamespaceBackendRef",
			"HTTPRouteMatchingAcrossRoutes",
			"HTTPRoutePartiallyInvalidViaInvalidReferenceGrant",
			"HTTPRouteRedirectHostAndStatus",
			"HTTPRouteInvalidBackendRefUnknownKind",
			"HTTPRoutePathMatchOrder",
			"HTTPRouteSimpleSameNamespace",
			"HTTPRouteMatching",
			"HTTPRouteHeaderMatching",
			"HTTPRouteReferenceGrant",
		},
	}

	cSuite, err := ksuite.NewExperimentalConformanceTestSuite(ksuite.ExperimentalConformanceOptions{
		Options: opts,
		Implementation: conformanceV1alpha1.Implementation{
			Organization: "traefik",
			Project:      "traefik",
			URL:          "https://traefik.io/",
			Version:      version.Version,
			Contact:      []string{"@traefik/maintainers"},
		},
		ConformanceProfiles: sets.New[ksuite.ConformanceProfileName](
			ksuite.HTTPConformanceProfileName,
			ksuite.TLSConformanceProfileName,
		),
	})
	require.NoError(s.T(), err)

	cSuite.Setup(s.T())
	err = cSuite.Run(s.T(), tests.ConformanceTests)
	require.NoError(s.T(), err)

	report, err := cSuite.Report()
	require.NoError(s.T(), err, "failed generating conformance report")

	report.GatewayAPIVersion = "1.0.0"

	rawReport, err := yaml.Marshal(report)
	require.NoError(s.T(), err)
	s.T().Logf("Conformance report:\n%s", string(rawReport))

	require.NoError(s.T(), os.MkdirAll("./conformance-reports", 0o755))
	outFile := filepath.Join("conformance-reports", fmt.Sprintf("traefik-traefik-%d.yaml", time.Now().UnixNano()))
	require.NoError(s.T(), os.WriteFile(outFile, rawReport, 0o600))
	s.T().Logf("Report written to: %s", outFile)
}