baalajimaestro/traefik
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

integration: use VPN for integration tests (for Mac)

Browse source 
Co-authored-by: Kevin Pollet <pollet.kevin@gmail.com>
This commit is contained in:
mpl 2022-07-13 18:32:08 +02:00 committed by GitHub
parent 14eb56cf30
commit b7199a7a9b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
17 changed files with 209 additions and 49 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -18,3 +18,4 @@ vendor/
plugins-storage/
plugins-local/
traefik_changelog.md
integration/tailscale.secret

7
Makefile
View file

@ -14,6 +14,7 @@ TRAEFIK_IMAGE := $(if $(REPONAME),$(REPONAME),"traefik/traefik")
INTEGRATION_OPTS := $(if $(MAKE_DOCKER_HOST),-e "DOCKER_HOST=$(MAKE_DOCKER_HOST)",-v "/var/run/docker.sock:/var/run/docker.sock")
DOCKER_BUILD_ARGS := $(if $(DOCKER_VERSION), "--build-arg=DOCKER_VERSION=$(DOCKER_VERSION)",)
# only used when running in docker
TRAEFIK_ENVS := \
-e OS_ARCH_ARG \
-e OS_PLATFORM_ARG \
@ -23,7 +24,7 @@ TRAEFIK_ENVS := \
-e CODENAME \
-e TESTDIRS \
-e CI \
-e CONTAINER=DOCKER # Indicator for integration tests that we are running inside a container.
-e IN_DOCKER=true # Indicator for integration tests that we are running inside a container.
TRAEFIK_MOUNT := -v "$(CURDIR)/dist:/go/src/github.com/traefik/traefik/dist"
DOCKER_RUN_OPTS := $(TRAEFIK_ENVS) $(TRAEFIK_MOUNT) "$(TRAEFIK_DEV_IMAGE)"
@ -102,7 +103,7 @@ crossbinary-default-parallel:
test: build-dev-image
-docker network create traefik-test-network --driver bridge --subnet 172.31.42.0/24
trap 'docker network rm traefik-test-network' EXIT; \
$(if $(IN_DOCKER),$(DOCKER_RUN_TRAEFIK_TEST),) ./script/make.sh generate test-unit binary test-integration
$(if $(IN_DOCKER),$(DOCKER_RUN_TRAEFIK_TEST)) ./script/make.sh generate test-unit binary test-integration
## Run the unit tests
.PHONY: test-unit
@ -116,7 +117,7 @@ test-unit: build-dev-image
test-integration: build-dev-image
-docker network create traefik-test-network --driver bridge --subnet 172.31.42.0/24
trap 'docker network rm traefik-test-network' EXIT; \
$(if $(IN_DOCKER),$(DOCKER_RUN_TRAEFIK_TEST),) ./script/make.sh generate binary test-integration
$(if $(IN_DOCKER),$(DOCKER_RUN_TRAEFIK_TEST)) ./script/make.sh generate binary test-integration
## Pull all images for integration tests
.PHONY: pull-images

4
integration/fixtures/tcp/catch-all-no-tls-with-https.toml
View file

@ -27,7 +27,7 @@
[tcp.services]
[tcp.services.whoami-no-tls.loadBalancer]
[[tcp.services.whoami-no-tls.loadBalancer.servers]]
address = "whoami-no-tls:8080"
address = "{{ .WhoamiNoTLSAddress }}"
[http]
[http.routers]
@ -40,4 +40,4 @@
[http.services]
[http.services.whoami.loadBalancer]
[[http.services.whoami.loadBalancer.servers]]
url = "http://whoami:80"
url = "{{ .WhoamiURL }}"

2
integration/fixtures/tcp/catch-all-no-tls.toml
View file

@ -27,4 +27,4 @@
[tcp.services]
[tcp.services.whoami-no-tls.loadBalancer]
[[tcp.services.whoami-no-tls.loadBalancer.servers]]
address = "whoami-banner:8080"
address = "{{ .WhoamiBannerAddress }}"

4
integration/fixtures/tcp/ip-whitelist.toml
View file

@ -38,11 +38,11 @@
[tcp.services]
[tcp.services.whoami-a.loadBalancer]
[[tcp.services.whoami-a.loadBalancer.servers]]
address = "whoami-a:8080"
address = "{{ .WhoamiA }}"
[tcp.services.whoami-b.loadBalancer]
[[tcp.services.whoami-b.loadBalancer.servers]]
address = "whoami-b:8080"
address = "{{ .WhoamiB }}"
[tcp.middlewares]
[tcp.middlewares.allowing-ipwhitelist.ipWhiteList]

8
integration/fixtures/tcp/mixed.toml
View file

@ -37,7 +37,7 @@
[http.services]
[http.services.whoami.loadBalancer]
[[http.services.whoami.loadBalancer.servers]]
url = "http://whoami:80"
url = "{{ .Whoami }}"
[tcp]
[tcp.routers]
[tcp.routers.to-whoami-a]
@ -62,15 +62,15 @@
[tcp.services.whoami-a.loadBalancer]
[[tcp.services.whoami-a.loadBalancer.servers]]
address = "whoami-a:8080"
address = "{{ .WhoamiA }}"
[tcp.services.whoami-b.loadBalancer]
[[tcp.services.whoami-b.loadBalancer.servers]]
address = "whoami-b:8080"
address = "{{ .WhoamiB }}"
[tcp.services.whoami-no-cert.loadBalancer]
[[tcp.services.whoami-no-cert.loadBalancer.servers]]
address = "whoami-no-cert:8080"
address = "{{ .WhoamiNoCert }}"
[[tls.certificates]]
certFile = "fixtures/tcp/whoami-c.crt"

2
integration/fixtures/tcp/multi-tls-options.toml
View file

@ -36,7 +36,7 @@
[tcp.services.whoami-no-cert]
[tcp.services.whoami-no-cert.loadBalancer]
[[tcp.services.whoami-no-cert.loadBalancer.servers]]
address = "whoami-no-cert:8080"
address = "{{ .WhoamiNoCert }}"
[tls.options]

8
integration/fixtures/tcp/non-tls-fallback.toml
View file

@ -47,17 +47,17 @@
[tcp.services]
[tcp.services.whoami-no-tls.loadBalancer]
[[tcp.services.whoami-no-tls.loadBalancer.servers]]
address = "whoami-no-tls:8080"
address = "{{ .WhoamiNoTLS }}"
[tcp.services.whoami-a.loadBalancer]
[[tcp.services.whoami-a.loadBalancer.servers]]
address = "whoami-a:8080"
address = "{{ .WhoamiA }}"
[tcp.services.whoami-b.loadBalancer]
[[tcp.services.whoami-b.loadBalancer.servers]]
address = "whoami-b:8080"
address = "{{ .WhoamiB }}"
[tcp.services.whoami-no-cert.loadBalancer]
[[tcp.services.whoami-no-cert.loadBalancer.servers]]
address = "whoami-no-cert:8080"
address = "{{ .WhoamiNoCert }}"

2
integration/fixtures/tcp/non-tls.toml
View file

@ -27,4 +27,4 @@
[tcp.services]
[tcp.services.whoami-no-tls.loadBalancer]
[[tcp.services.whoami-no-tls.loadBalancer.servers]]
address = "whoami-no-tls:8080"
address = "{{ .WhoamiNoTLS }}"

4
integration/fixtures/tcp/wrr.toml
View file

@ -34,8 +34,8 @@
[tcp.services.whoami-b.loadBalancer]
[[tcp.services.whoami-b.loadBalancer.servers]]
address = "whoami-b:8080"
address = "{{ .WhoamiB }}"
[tcp.services.whoami-ab.loadBalancer]
[[tcp.services.whoami-ab.loadBalancer.servers]]
address = "whoami-ab:8080"
address = "{{ .WhoamiAB }}"

90
integration/integration_test.go
View file

@ -4,8 +4,11 @@ package integration
import (
"bytes"
"context"
"errors"
"flag"
"fmt"
"io/fs"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
@ -40,8 +43,23 @@ func Test(t *testing.T) {
return
}
// TODO(mpl): very niche optimization: do not start tailscale if none of the
// wanted tests actually need it (e.g. KeepAliveSuite does not).
var (
vpn *tailscaleNotSuite
useVPN bool
)
if os.Getenv("IN_DOCKER") != "true" {
if vpn = setupVPN(nil, "tailscale.secret"); vpn != nil {
defer vpn.TearDownSuite(nil)
useVPN = true
}
}
check.Suite(&AccessLogSuite{})
check.Suite(&AcmeSuite{})
if !useVPN {
check.Suite(&AcmeSuite{})
}
check.Suite(&ConsulCatalogSuite{})
check.Suite(&ConsulSuite{})
check.Suite(&DockerComposeSuite{})
@ -55,12 +73,16 @@ func Test(t *testing.T) {
check.Suite(&HostResolverSuite{})
check.Suite(&HTTPSSuite{})
check.Suite(&HTTPSuite{})
check.Suite(&K8sSuite{})
if !useVPN {
check.Suite(&K8sSuite{})
}
check.Suite(&KeepAliveSuite{})
check.Suite(&LogRotationSuite{})
check.Suite(&MarathonSuite15{})
check.Suite(&MarathonSuite{})
check.Suite(&ProxyProtocolSuite{})
check.Suite(&MarathonSuite15{})
if !useVPN {
check.Suite(&ProxyProtocolSuite{})
}
check.Suite(&RateLimitSuite{})
check.Suite(&RedisSuite{})
check.Suite(&RestSuite{})
@ -125,6 +147,24 @@ func (s *BaseSuite) composeUp(c *check.C, services ...string) {
c.Assert(err, checker.IsNil)
}
// composeExec runs the command in the given args in the given compose service container.
// Already running services are not affected (i.e. not stopped).
func (s *BaseSuite) composeExec(c *check.C, service string, args ...string) {
c.Assert(s.composeProject, check.NotNil)
c.Assert(s.dockerComposeService, check.NotNil)
_, err := s.dockerComposeService.Exec(context.Background(), s.composeProject.Name, composeapi.RunOptions{
Service: service,
Stdin: os.Stdin,
Stdout: os.Stdout,
Stderr: os.Stderr,
Command: args,
Tty: false,
Index: 1,
})
c.Assert(err, checker.IsNil)
}
// composeStop stops the given services of the current docker compose project and removes the corresponding containers.
func (s *BaseSuite) composeStop(c *check.C, services ...string) {
c.Assert(s.dockerComposeService, check.NotNil)
@ -285,3 +325,45 @@ func (s *BaseSuite) getContainerIP(c *check.C, name string) string {
func withConfigFile(file string) string {
return "--configFile=" + file
}
// tailscaleNotSuite includes a BaseSuite out of convenience, so we can benefit
// from composeUp et co., but it is not meant to function as a TestSuite per se.
type tailscaleNotSuite struct{ BaseSuite }
// setupVPN starts Tailscale on the corresponding container, and makes it a subnet
// router, for all the other containers (whoamis, etc) subsequently started for the
// integration tests.
// It only does so if the file provided as argument exists, and contains a
// Tailscale auth key (an ephemeral, but reusable, one is recommended).
//
// Add this section to your tailscale ACLs to auto-approve the routes for the
// containers in the docker subnet:
//
// "autoApprovers": {
// // Allow myself to automatically advertize routes for docker networks
// "routes": {
// "172.0.0.0/8": ["your_tailscale_identity"],
// },
// },
//
// TODO(mpl): we could maybe even move this setup to the Makefile, to start it
// and let it run (forever, or until voluntarily stopped).
func setupVPN(c *check.C, keyFile string) *tailscaleNotSuite {
data, err := ioutil.ReadFile(keyFile)
if err != nil {
if !errors.Is(err, fs.ErrNotExist) {
log.Fatal(err)
}
return nil
}
authKey := strings.TrimSpace(string(data))
// TODO: copy and create versions that don't need a check.C?
vpn := &tailscaleNotSuite{}
vpn.createComposeProject(c, "tailscale")
vpn.composeUp(c)
time.Sleep(5 * time.Second)
// If we ever change the docker subnet in the Makefile,
// we need to change this one below correspondingly.
vpn.composeExec(c, "tailscaled", "tailscale", "up", "--authkey="+authKey, "--advertise-routes=172.31.42.0/24")
return vpn
}

2
integration/marathon15_test.go
View file

@ -21,7 +21,7 @@ func (s *MarathonSuite15) SetUpSuite(c *check.C) {
s.createComposeProject(c, "marathon15")
s.composeUp(c)
s.marathonURL = "http://" + containerNameMarathon + ":8080"
s.marathonURL = "http://" + s.getComposeServiceIP(c, containerNameMarathon) + ":8080"
// Wait for Marathon readiness prior to creating the client so that we
// don't run into the "all cluster members down" state right from the

3
integration/marathon_test.go
View file

@ -23,7 +23,7 @@ func (s *MarathonSuite) SetUpSuite(c *check.C) {
s.createComposeProject(c, "marathon")
s.composeUp(c)
s.marathonURL = "http://" + containerNameMarathon + ":8080"
s.marathonURL = "http://" + s.getComposeServiceIP(c, containerNameMarathon) + ":8080"
// Wait for Marathon readiness prior to creating the client so that we
// don't run into the "all cluster members down" state right from the
@ -45,6 +45,7 @@ func (s *MarathonSuite) TestConfigurationUpdate(c *check.C) {
MarathonURL string
}{s.marathonURL})
defer os.Remove(file)
cmd, display := s.traefikCmd(withConfigFile(file))
defer display(c)
err := cmd.Start()

2
integration/resources/compose/marathon.yml
View file

@ -32,7 +32,7 @@ services:
-v /var/run/docker.sock:/var/run/docker.sock \
-v /cgroup:/cgroup -v /sys:/sys \
-v /usr/local/bin/docker:/usr/local/bin/docker \
-e MESOS_HOSTNAME=mesos-slave \
-e MESOS_HOSTNAME=$$(hostname -i) \
-e MESOS_CONTAINERIZERS=docker,mesos \
-e MESOS_ISOLATOR=cgroups/cpu,cgroups/mem \
-e MESOS_LOG_DIR=/var/log \

36
integration/resources/compose/marathon15.yml
View file

@ -17,24 +17,32 @@ services:
MESOS_ZK: zk://zookeeper:2181/mesos
mesos-slave:
image: mesosphere/mesos-slave-dind:0.4.0_mesos-1.4.1_docker-17.05.0_ubuntu-16.04.3
image: docker:dind
privileged: true
# Uncomment published ports for interactive debugging.
# ports:
# - "5051:5051"
environment:
MESOS_HOSTNAME: mesos-slave
MESOS_CONTAINERIZERS: docker,mesos
MESOS_ISOLATOR: cgroups/cpu,cgroups/mem
MESOS_LOG_DIR: /var/log
MESOS_MASTER: zk://zookeeper:2181/mesos
MESOS_PORT: 5051
MESOS_WORK_DIR: /var/lib/mesos
MESOS_EXECUTOR_REGISTRATION_TIMEOUT: 5mins
MESOS_EXECUTOR_SHUTDOWN_GRACE_PERIOD: 90secs
MESOS_DOCKER_STOP_TIMEOUT: 60secs
MESOS_RESOURCES: cpus:2;mem:2048;disk:20480;ports(*):[12000-12999]
MESOS_SYSTEMD_ENABLE_SUPPORT: false
command:
- "/bin/sh"
- "-c"
- "(/usr/local/bin/dockerd-entrypoint.sh &); sleep 10; set -x; \
docker -H unix:///var/run/docker.sock run -d --net=host --privileged \
-v /var/run/docker.sock:/var/run/docker.sock \
-v /cgroup:/cgroup -v /sys:/sys \
-v /usr/local/bin/docker:/usr/local/bin/docker \
-e MESOS_HOSTNAME=$$(hostname -i) \
-e MESOS_CONTAINERIZERS=docker,mesos \
-e MESOS_ISOLATOR=cgroups/cpu,cgroups/mem \
-e MESOS_LOG_DIR=/var/log \
-e MESOS_MASTER=zk://zookeeper:2181/mesos \
-e MESOS_PORT=5051 \
-e MESOS_WORK_DIR=/var/lib/mesos \
-e MESOS_EXECUTOR_REGISTRATION_TIMEOUT=5mins \
-e MESOS_EXECUTOR_SHUTDOWN_GRACE_PERIOD=90secs \
-e MESOS_DOCKER_STOP_TIMEOUT=60secs \
-e MESOS_RESOURCES='cpus:2;mem:2048;disk:20480;ports(*):[12000-12999]' \
-e MESOS_SYSTEMD_ENABLE_SUPPORT=false \
mesosphere/mesos-slave:1.4.1; sleep 600"
marathon:
image: mesosphere/marathon:v1.5.9

17
integration/resources/compose/tailscale.yml Normal file
View file

@ -0,0 +1,17 @@
version: "3.8"
services:
tailscaled:
hostname: traefik-tests-gw # This will become the tailscale device name
image: tailscale/tailscale:v1.24.0
volumes:
# TODO: maybe mount the container's /var/lib to keep some state for tailscale?
- "/dev/net/tun:/dev/net/tun" # Required for tailscale to work
cap_add: # Required for tailscale to work
- net_admin
- sys_module
command: tailscaled
networks:
default:
name: traefik-test-network
external: true

66
integration/tcp_test.go
View file

@ -25,7 +25,17 @@ func (s *TCPSuite) SetUpSuite(c *check.C) {
}
func (s *TCPSuite) TestMixed(c *check.C) {
file := s.adaptFile(c, "fixtures/tcp/mixed.toml", struct{}{})
file := s.adaptFile(c, "fixtures/tcp/mixed.toml", struct {
Whoami string
WhoamiA string
WhoamiB string
WhoamiNoCert string
}{
Whoami: "http://" + s.getComposeServiceIP(c, "whoami") + ":80",
WhoamiA: s.getComposeServiceIP(c, "whoami-a") + ":8080",
WhoamiB: s.getComposeServiceIP(c, "whoami-b") + ":8080",
WhoamiNoCert: s.getComposeServiceIP(c, "whoami-no-cert") + ":8080",
})
defer os.Remove(file)
cmd, display := s.traefikCmd(withConfigFile(file))
@ -75,7 +85,11 @@ func (s *TCPSuite) TestMixed(c *check.C) {
}
func (s *TCPSuite) TestTLSOptions(c *check.C) {
file := s.adaptFile(c, "fixtures/tcp/multi-tls-options.toml", struct{}{})
file := s.adaptFile(c, "fixtures/tcp/multi-tls-options.toml", struct {
WhoamiNoCert string
}{
WhoamiNoCert: s.getComposeServiceIP(c, "whoami-no-cert") + ":8080",
})
defer os.Remove(file)
cmd, display := s.traefikCmd(withConfigFile(file))
@ -105,7 +119,17 @@ func (s *TCPSuite) TestTLSOptions(c *check.C) {
}
func (s *TCPSuite) TestNonTLSFallback(c *check.C) {
file := s.adaptFile(c, "fixtures/tcp/non-tls-fallback.toml", struct{}{})
file := s.adaptFile(c, "fixtures/tcp/non-tls-fallback.toml", struct {
WhoamiA string
WhoamiB string
WhoamiNoCert string
WhoamiNoTLS string
}{
WhoamiA: s.getComposeServiceIP(c, "whoami-a") + ":8080",
WhoamiB: s.getComposeServiceIP(c, "whoami-b") + ":8080",
WhoamiNoCert: s.getComposeServiceIP(c, "whoami-no-cert") + ":8080",
WhoamiNoTLS: s.getComposeServiceIP(c, "whoami-no-tls") + ":8080",
})
defer os.Remove(file)
cmd, display := s.traefikCmd(withConfigFile(file))
@ -139,7 +163,11 @@ func (s *TCPSuite) TestNonTLSFallback(c *check.C) {
}
func (s *TCPSuite) TestNonTlsTcp(c *check.C) {
file := s.adaptFile(c, "fixtures/tcp/non-tls.toml", struct{}{})
file := s.adaptFile(c, "fixtures/tcp/non-tls.toml", struct {
WhoamiNoTLS string
}{
WhoamiNoTLS: s.getComposeServiceIP(c, "whoami-no-tls") + ":8080",
})
defer os.Remove(file)
cmd, display := s.traefikCmd(withConfigFile(file))
@ -159,7 +187,11 @@ func (s *TCPSuite) TestNonTlsTcp(c *check.C) {
}
func (s *TCPSuite) TestCatchAllNoTLS(c *check.C) {
file := s.adaptFile(c, "fixtures/tcp/catch-all-no-tls.toml", struct{}{})
file := s.adaptFile(c, "fixtures/tcp/catch-all-no-tls.toml", struct {
WhoamiBannerAddress string
}{
WhoamiBannerAddress: s.getComposeServiceIP(c, "whoami-banner") + ":8080",
})
defer os.Remove(file)
cmd, display := s.traefikCmd(withConfigFile(file))
@ -179,7 +211,13 @@ func (s *TCPSuite) TestCatchAllNoTLS(c *check.C) {
}
func (s *TCPSuite) TestCatchAllNoTLSWithHTTPS(c *check.C) {
file := s.adaptFile(c, "fixtures/tcp/catch-all-no-tls-with-https.toml", struct{}{})
file := s.adaptFile(c, "fixtures/tcp/catch-all-no-tls-with-https.toml", struct {
WhoamiNoTLSAddress string
WhoamiURL string
}{
WhoamiNoTLSAddress: s.getComposeServiceIP(c, "whoami-no-tls") + ":8080",
WhoamiURL: "http://" + s.getComposeServiceIP(c, "whoami") + ":80",
})
defer os.Remove(file)
cmd, display := s.traefikCmd(withConfigFile(file))
@ -204,7 +242,13 @@ func (s *TCPSuite) TestCatchAllNoTLSWithHTTPS(c *check.C) {
}
func (s *TCPSuite) TestMiddlewareWhiteList(c *check.C) {
file := s.adaptFile(c, "fixtures/tcp/ip-whitelist.toml", struct{}{})
file := s.adaptFile(c, "fixtures/tcp/ip-whitelist.toml", struct {
WhoamiA string
WhoamiB string
}{
WhoamiA: s.getComposeServiceIP(c, "whoami-a") + ":8080",
WhoamiB: s.getComposeServiceIP(c, "whoami-b") + ":8080",
})
defer os.Remove(file)
cmd, display := s.traefikCmd(withConfigFile(file))
@ -228,7 +272,13 @@ func (s *TCPSuite) TestMiddlewareWhiteList(c *check.C) {
}
func (s *TCPSuite) TestWRR(c *check.C) {
file := s.adaptFile(c, "fixtures/tcp/wrr.toml", struct{}{})
file := s.adaptFile(c, "fixtures/tcp/wrr.toml", struct {
WhoamiB string
WhoamiAB string
}{
WhoamiB: s.getComposeServiceIP(c, "whoami-b") + ":8080",
WhoamiAB: s.getComposeServiceIP(c, "whoami-ab") + ":8080",
})
defer os.Remove(file)
cmd, display := s.traefikCmd(withConfigFile(file))