From 51f7f610c99fd6500f25eb435957805a977c57c9 Mon Sep 17 00:00:00 2001 From: Romain Date: Fri, 30 Aug 2024 17:14:03 +0200 Subject: [PATCH] Add versioning for Gateway API Conformance Test Report --- .github/workflows/test-conformance.yaml | 6 +- .gitignore | 2 +- Makefile | 3 +- .../contributing/submitting-pull-requests.md | 2 + .../experimental-v3.1-default-report.yaml | 55 +++++++++++++++++++ integration/integration_test.go | 7 ++- integration/k8s_conformance_test.go | 12 ++-- 7 files changed, 76 insertions(+), 11 deletions(-) create mode 100644 integration/conformance-reports/v1.1.0/experimental-v3.1-default-report.yaml diff --git a/.github/workflows/test-conformance.yaml b/.github/workflows/test-conformance.yaml index e6147d2c6..356d65e49 100644 --- a/.github/workflows/test-conformance.yaml +++ b/.github/workflows/test-conformance.yaml @@ -31,5 +31,7 @@ jobs: - name: Avoid generating webui run: touch webui/static/index.html - - name: K8s Gateway API conformance test - run: make test-gateway-api-conformance + - name: K8s Gateway API conformance test and report + run: | + make test-gateway-api-conformance + git diff --exit-code diff --git a/.gitignore b/.gitignore index 03a5e9369..1754b1c70 100644 --- a/.gitignore +++ b/.gitignore @@ -19,4 +19,4 @@ plugins-storage/ plugins-local/ traefik_changelog.md integration/tailscale.secret -integration/conformance-reports/ +integration/conformance-reports/**/experimental-dev-default-report.yaml diff --git a/Makefile b/Makefile index 8ba03926d..2eef76a50 100644 --- a/Makefile +++ b/Makefile @@ -103,7 +103,8 @@ test-integration: binary .PHONY: test-gateway-api-conformance #? test-gateway-api-conformance: Run the conformance tests test-gateway-api-conformance: build-image-dirty - GOOS=$(GOOS) GOARCH=$(GOARCH) go test ./integration -v -test.run K8sConformanceSuite -k8sConformance $(TESTFLAGS) + # In case of a new Minor/Major version, the k8sConformanceTraefikVersion needs to be updated. + GOOS=$(GOOS) GOARCH=$(GOARCH) go test ./integration -v -test.run K8sConformanceSuite -k8sConformance -k8sConformanceTraefikVersion="v3.1" $(TESTFLAGS) .PHONY: test-ui-unit #? test-ui-unit: Run the unit tests for the webui diff --git a/docs/content/contributing/submitting-pull-requests.md b/docs/content/contributing/submitting-pull-requests.md index 7488378ed..7611023d2 100644 --- a/docs/content/contributing/submitting-pull-requests.md +++ b/docs/content/contributing/submitting-pull-requests.md @@ -91,6 +91,8 @@ You must run these local verifications before you submit your pull request to pr Your PR will not be reviewed until these are green on the CI. * `make generate` +* `make generate-crd` +* `make test-gateway-api-conformance` * `make validate` * `make pull-images` * `make test` diff --git a/integration/conformance-reports/v1.1.0/experimental-v3.1-default-report.yaml b/integration/conformance-reports/v1.1.0/experimental-v3.1-default-report.yaml new file mode 100644 index 000000000..e668037eb --- /dev/null +++ b/integration/conformance-reports/v1.1.0/experimental-v3.1-default-report.yaml @@ -0,0 +1,55 @@ +apiVersion: gateway.networking.k8s.io/v1alpha1 +date: '-' +gatewayAPIChannel: experimental +gatewayAPIVersion: v1.1.0 +implementation: + contact: + - '@traefik/maintainers' + organization: traefik + project: traefik + url: https://traefik.io/ + version: v3.1 +kind: ConformanceReport +mode: default +profiles: +- core: + result: success + statistics: + Failed: 0 + Passed: 12 + Skipped: 0 + name: GATEWAY-GRPC + summary: Core tests succeeded. +- core: + result: success + statistics: + Failed: 0 + Passed: 33 + Skipped: 0 + extended: + result: success + statistics: + Failed: 0 + Passed: 10 + Skipped: 0 + supportedFeatures: + - GatewayPort8080 + - HTTPRouteHostRewrite + - HTTPRouteMethodMatching + - HTTPRoutePathRedirect + - HTTPRoutePathRewrite + - HTTPRoutePortRedirect + - HTTPRouteQueryParamMatching + - HTTPRouteResponseHeaderModification + - HTTPRouteSchemeRedirect + unsupportedFeatures: + - GatewayHTTPListenerIsolation + - GatewayStaticAddresses + - HTTPRouteBackendRequestHeaderModification + - HTTPRouteBackendTimeout + - HTTPRouteParentRefPort + - HTTPRouteRequestMirror + - HTTPRouteRequestMultipleMirrors + - HTTPRouteRequestTimeout + name: GATEWAY-HTTP + summary: Core tests succeeded. Extended tests succeeded. diff --git a/integration/integration_test.go b/integration/integration_test.go index f46b15209..0635343c8 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -36,9 +36,10 @@ import ( ) var ( - showLog = flag.Bool("tlog", false, "always show Traefik logs") - k8sConformance = flag.Bool("k8sConformance", false, "run K8s Gateway API conformance test") - k8sConformanceRunTest = flag.String("k8sConformanceRunTest", "", "run a specific K8s Gateway API conformance test") + showLog = flag.Bool("tlog", false, "always show Traefik logs") + k8sConformance = flag.Bool("k8sConformance", false, "run K8s Gateway API conformance test") + k8sConformanceRunTest = flag.String("k8sConformanceRunTest", "", "run a specific K8s Gateway API conformance test") + k8sConformanceTraefikVersion = flag.String("k8sConformanceTraefikVersion", "dev", "specify the Traefik version for the K8s Gateway API conformance report") ) const tailscaleSecretFilePath = "tailscale.secret" diff --git a/integration/k8s_conformance_test.go b/integration/k8s_conformance_test.go index 2d236661e..70ec948ea 100644 --- a/integration/k8s_conformance_test.go +++ b/integration/k8s_conformance_test.go @@ -17,7 +17,6 @@ import ( "github.com/testcontainers/testcontainers-go/modules/k3s" "github.com/testcontainers/testcontainers-go/network" "github.com/traefik/traefik/v3/integration/try" - "github.com/traefik/traefik/v3/pkg/version" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/util/sets" kclientset "k8s.io/client-go/kubernetes" @@ -191,7 +190,7 @@ func (s *K8sConformanceSuite) TestK8sGatewayAPIConformance() { Organization: "traefik", Project: "traefik", URL: "https://traefik.io/", - Version: version.Version, + Version: *k8sConformanceTraefikVersion, Contact: []string{"@traefik/maintainers"}, }, ConformanceProfiles: sets.New(ksuite.GatewayHTTPConformanceProfileName, ksuite.GatewayGRPCConformanceProfileName), @@ -220,12 +219,17 @@ func (s *K8sConformanceSuite) TestK8sGatewayAPIConformance() { report, err := cSuite.Report() require.NoError(s.T(), err, "failed generating conformance report") + // Ignore report date to avoid diff with CI job. + // However, we can track the date of the report thanks to the commit. + // TODO: to publish this report automatically, we have to figure out how to handle the date diff. + report.Date = "-" + rawReport, err := yaml.Marshal(report) require.NoError(s.T(), err) s.T().Logf("Conformance report:\n%s", string(rawReport)) - require.NoError(s.T(), os.MkdirAll("./conformance-reports", 0o755)) - outFile := filepath.Join("conformance-reports", fmt.Sprintf("%s-%s-%s-report.yaml", report.GatewayAPIChannel, report.Version, report.Mode)) + require.NoError(s.T(), os.MkdirAll("./conformance-reports/"+report.GatewayAPIVersion, 0o755)) + outFile := filepath.Join("conformance-reports/"+report.GatewayAPIVersion, fmt.Sprintf("%s-%s-%s-report.yaml", report.GatewayAPIChannel, report.Version, report.Mode)) require.NoError(s.T(), os.WriteFile(outFile, rawReport, 0o600)) s.T().Logf("Report written to: %s", outFile) }