integration: use VPN for integration tests (for Mac)

Co-authored-by: Kevin Pollet <pollet.kevin@gmail.com>

.gitignore

@@ -18,3 +18,4 @@ vendor/
 plugins-storage/
 plugins-local/
 traefik_changelog.md
+integration/tailscale.secret

Makefile

@@ -14,6 +14,7 @@ TRAEFIK_IMAGE := $(if $(REPONAME),$(REPONAME),"traefik/traefik")
 INTEGRATION_OPTS := $(if $(MAKE_DOCKER_HOST),-e "DOCKER_HOST=$(MAKE_DOCKER_HOST)",-v "/var/run/docker.sock:/var/run/docker.sock")
 DOCKER_BUILD_ARGS := $(if $(DOCKER_VERSION), "--build-arg=DOCKER_VERSION=$(DOCKER_VERSION)",)
 
+# only used when running in docker
 TRAEFIK_ENVS := \
 	-e OS_ARCH_ARG \
 	-e OS_PLATFORM_ARG \
@@ -23,7 +24,7 @@ TRAEFIK_ENVS := \
 	-e CODENAME \
 	-e TESTDIRS \
 	-e CI \
-	-e CONTAINER=DOCKER		# Indicator for integration tests that we are running inside a container.
+	-e IN_DOCKER=true		# Indicator for integration tests that we are running inside a container.
 
 TRAEFIK_MOUNT := -v "$(CURDIR)/dist:/go/src/github.com/traefik/traefik/dist"
 DOCKER_RUN_OPTS := $(TRAEFIK_ENVS) $(TRAEFIK_MOUNT) "$(TRAEFIK_DEV_IMAGE)"
@@ -102,7 +103,7 @@ crossbinary-default-parallel:
 test: build-dev-image
 	-docker network create traefik-test-network --driver bridge --subnet 172.31.42.0/24
 	trap 'docker network rm traefik-test-network' EXIT; \
-	$(if $(IN_DOCKER),$(DOCKER_RUN_TRAEFIK_TEST),) ./script/make.sh generate test-unit binary test-integration
+	$(if $(IN_DOCKER),$(DOCKER_RUN_TRAEFIK_TEST)) ./script/make.sh generate test-unit binary test-integration
 
 ## Run the unit tests
 .PHONY: test-unit
@@ -116,7 +117,7 @@ test-unit: build-dev-image
 test-integration: build-dev-image
 	-docker network create traefik-test-network --driver bridge --subnet 172.31.42.0/24
 	trap 'docker network rm traefik-test-network' EXIT; \
-	$(if $(IN_DOCKER),$(DOCKER_RUN_TRAEFIK_TEST),) ./script/make.sh generate binary test-integration
+	$(if $(IN_DOCKER),$(DOCKER_RUN_TRAEFIK_TEST)) ./script/make.sh generate binary test-integration
 
 ## Pull all images for integration tests
 .PHONY: pull-images

integration/fixtures/tcp/catch-all-no-tls-with-https.toml

@@ -27,7 +27,7 @@
   [tcp.services]
     [tcp.services.whoami-no-tls.loadBalancer]
       [[tcp.services.whoami-no-tls.loadBalancer.servers]]
-        address = "whoami-no-tls:8080"
+        address = "{{ .WhoamiNoTLSAddress }}"
 
 [http]
   [http.routers]
@@ -40,4 +40,4 @@
   [http.services]
     [http.services.whoami.loadBalancer]
       [[http.services.whoami.loadBalancer.servers]]
-        url = "http://whoami:80"
+        url = "{{ .WhoamiURL }}"

integration/fixtures/tcp/catch-all-no-tls.toml

@@ -27,4 +27,4 @@
   [tcp.services]
     [tcp.services.whoami-no-tls.loadBalancer]
       [[tcp.services.whoami-no-tls.loadBalancer.servers]]
-        address = "whoami-banner:8080"
+        address = "{{ .WhoamiBannerAddress }}"

integration/fixtures/tcp/ip-whitelist.toml

@@ -38,11 +38,11 @@
   [tcp.services]
     [tcp.services.whoami-a.loadBalancer]
       [[tcp.services.whoami-a.loadBalancer.servers]]
-        address = "whoami-a:8080"
+        address = "{{ .WhoamiA }}"
 
     [tcp.services.whoami-b.loadBalancer]
       [[tcp.services.whoami-b.loadBalancer.servers]]
-        address = "whoami-b:8080"
+        address = "{{ .WhoamiB }}"
 
   [tcp.middlewares]
     [tcp.middlewares.allowing-ipwhitelist.ipWhiteList]

integration/fixtures/tcp/mixed.toml

@@ -37,7 +37,7 @@
   [http.services]
     [http.services.whoami.loadBalancer]
       [[http.services.whoami.loadBalancer.servers]]
-        url = "http://whoami:80"
+        url = "{{ .Whoami }}"
 [tcp]
   [tcp.routers]
     [tcp.routers.to-whoami-a]
@@ -62,15 +62,15 @@
 
     [tcp.services.whoami-a.loadBalancer]
       [[tcp.services.whoami-a.loadBalancer.servers]]
-        address = "whoami-a:8080"
+        address = "{{ .WhoamiA }}"
 
     [tcp.services.whoami-b.loadBalancer]
       [[tcp.services.whoami-b.loadBalancer.servers]]
-        address = "whoami-b:8080"
+        address = "{{ .WhoamiB }}"
 
     [tcp.services.whoami-no-cert.loadBalancer]
       [[tcp.services.whoami-no-cert.loadBalancer.servers]]
-        address = "whoami-no-cert:8080"
+        address = "{{ .WhoamiNoCert }}"
 
 [[tls.certificates]]
   certFile = "fixtures/tcp/whoami-c.crt"

integration/fixtures/tcp/multi-tls-options.toml

@@ -36,7 +36,7 @@
     [tcp.services.whoami-no-cert]
       [tcp.services.whoami-no-cert.loadBalancer]
         [[tcp.services.whoami-no-cert.loadBalancer.servers]]
-        address = "whoami-no-cert:8080"
+        address = "{{ .WhoamiNoCert }}"
 
 [tls.options]
 

integration/fixtures/tcp/non-tls-fallback.toml

@@ -47,17 +47,17 @@
   [tcp.services]
     [tcp.services.whoami-no-tls.loadBalancer]
       [[tcp.services.whoami-no-tls.loadBalancer.servers]]
-        address = "whoami-no-tls:8080"
+        address = "{{ .WhoamiNoTLS }}"
 
     [tcp.services.whoami-a.loadBalancer]
       [[tcp.services.whoami-a.loadBalancer.servers]]
-        address = "whoami-a:8080"
+        address = "{{ .WhoamiA }}"
 
     [tcp.services.whoami-b.loadBalancer]
       [[tcp.services.whoami-b.loadBalancer.servers]]
-        address = "whoami-b:8080"
+        address = "{{ .WhoamiB }}"
 
     [tcp.services.whoami-no-cert.loadBalancer]
       [[tcp.services.whoami-no-cert.loadBalancer.servers]]
-        address = "whoami-no-cert:8080"
+        address = "{{ .WhoamiNoCert }}"
 

integration/fixtures/tcp/non-tls.toml

@@ -27,4 +27,4 @@
   [tcp.services]
     [tcp.services.whoami-no-tls.loadBalancer]
       [[tcp.services.whoami-no-tls.loadBalancer.servers]]
-        address = "whoami-no-tls:8080"
+        address = "{{ .WhoamiNoTLS }}"

integration/fixtures/tcp/wrr.toml

@@ -34,8 +34,8 @@
 
     [tcp.services.whoami-b.loadBalancer]
       [[tcp.services.whoami-b.loadBalancer.servers]]
-        address = "whoami-b:8080"
+        address = "{{ .WhoamiB }}"
 
     [tcp.services.whoami-ab.loadBalancer]
       [[tcp.services.whoami-ab.loadBalancer.servers]]
-        address = "whoami-ab:8080"
+        address = "{{ .WhoamiAB }}"

integration/integration_test.go

@@ -4,8 +4,11 @@ package integration
 import (
 	"bytes"
 	"context"
+	"errors"
 	"flag"
 	"fmt"
+	"io/fs"
+	"io/ioutil"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -40,8 +43,23 @@ func Test(t *testing.T) {
 		return
 	}
 
+	// TODO(mpl): very niche optimization: do not start tailscale if none of the
+	// wanted tests actually need it (e.g. KeepAliveSuite does not).
+	var (
+		vpn    *tailscaleNotSuite
+		useVPN bool
+	)
+	if os.Getenv("IN_DOCKER") != "true" {
+		if vpn = setupVPN(nil, "tailscale.secret"); vpn != nil {
+			defer vpn.TearDownSuite(nil)
+			useVPN = true
+		}
+	}
+
 	check.Suite(&AccessLogSuite{})
-	check.Suite(&AcmeSuite{})
+	if !useVPN {
+		check.Suite(&AcmeSuite{})
+	}
 	check.Suite(&ConsulCatalogSuite{})
 	check.Suite(&ConsulSuite{})
 	check.Suite(&DockerComposeSuite{})
@@ -55,12 +73,16 @@ func Test(t *testing.T) {
 	check.Suite(&HostResolverSuite{})
 	check.Suite(&HTTPSSuite{})
 	check.Suite(&HTTPSuite{})
-	check.Suite(&K8sSuite{})
+	if !useVPN {
+		check.Suite(&K8sSuite{})
+	}
 	check.Suite(&KeepAliveSuite{})
 	check.Suite(&LogRotationSuite{})
-	check.Suite(&MarathonSuite15{})
 	check.Suite(&MarathonSuite{})
-	check.Suite(&ProxyProtocolSuite{})
+	check.Suite(&MarathonSuite15{})
+	if !useVPN {
+		check.Suite(&ProxyProtocolSuite{})
+	}
 	check.Suite(&RateLimitSuite{})
 	check.Suite(&RedisSuite{})
 	check.Suite(&RestSuite{})
@@ -125,6 +147,24 @@ func (s *BaseSuite) composeUp(c *check.C, services ...string) {
 	c.Assert(err, checker.IsNil)
 }
 
+// composeExec runs the command in the given args in the given compose service container.
+// Already running services are not affected (i.e. not stopped).
+func (s *BaseSuite) composeExec(c *check.C, service string, args ...string) {
+	c.Assert(s.composeProject, check.NotNil)
+	c.Assert(s.dockerComposeService, check.NotNil)
+
+	_, err := s.dockerComposeService.Exec(context.Background(), s.composeProject.Name, composeapi.RunOptions{
+		Service: service,
+		Stdin:   os.Stdin,
+		Stdout:  os.Stdout,
+		Stderr:  os.Stderr,
+		Command: args,
+		Tty:     false,
+		Index:   1,
+	})
+	c.Assert(err, checker.IsNil)
+}
+
 // composeStop stops the given services of the current docker compose project and removes the corresponding containers.
 func (s *BaseSuite) composeStop(c *check.C, services ...string) {
 	c.Assert(s.dockerComposeService, check.NotNil)
@@ -285,3 +325,45 @@ func (s *BaseSuite) getContainerIP(c *check.C, name string) string {
 func withConfigFile(file string) string {
 	return "--configFile=" + file
 }
+
+// tailscaleNotSuite includes a BaseSuite out of convenience, so we can benefit
+// from composeUp et co., but it is not meant to function as a TestSuite per se.
+type tailscaleNotSuite struct{ BaseSuite }
+
+// setupVPN starts Tailscale on the corresponding container, and makes it a subnet
+// router, for all the other containers (whoamis, etc) subsequently started for the
+// integration tests.
+// It only does so if the file provided as argument exists, and contains a
+// Tailscale auth key (an ephemeral, but reusable, one is recommended).
+//
+// Add this section to your tailscale ACLs to auto-approve the routes for the
+// containers in the docker subnet:
+//
+// "autoApprovers": {
+//   // Allow myself to automatically advertize routes for docker networks
+//   "routes": {
+//     "172.0.0.0/8": ["your_tailscale_identity"],
+//   },
+// },
+//
+// TODO(mpl): we could maybe even move this setup to the Makefile, to start it
+// and let it run (forever, or until voluntarily stopped).
+func setupVPN(c *check.C, keyFile string) *tailscaleNotSuite {
+	data, err := ioutil.ReadFile(keyFile)
+	if err != nil {
+		if !errors.Is(err, fs.ErrNotExist) {
+			log.Fatal(err)
+		}
+		return nil
+	}
+	authKey := strings.TrimSpace(string(data))
+	// TODO: copy and create versions that don't need a check.C?
+	vpn := &tailscaleNotSuite{}
+	vpn.createComposeProject(c, "tailscale")
+	vpn.composeUp(c)
+	time.Sleep(5 * time.Second)
+	// If we ever change the docker subnet in the Makefile,
+	// we need to change this one below correspondingly.
+	vpn.composeExec(c, "tailscaled", "tailscale", "up", "--authkey="+authKey, "--advertise-routes=172.31.42.0/24")
+	return vpn
+}

integration/marathon15_test.go

@@ -21,7 +21,7 @@ func (s *MarathonSuite15) SetUpSuite(c *check.C) {
 	s.createComposeProject(c, "marathon15")
 	s.composeUp(c)
 
-	s.marathonURL = "http://" + containerNameMarathon + ":8080"
+	s.marathonURL = "http://" + s.getComposeServiceIP(c, containerNameMarathon) + ":8080"
 
 	// Wait for Marathon readiness prior to creating the client so that we
 	// don't run into the "all cluster members down" state right from the

integration/marathon_test.go

@@ -23,7 +23,7 @@ func (s *MarathonSuite) SetUpSuite(c *check.C) {
 	s.createComposeProject(c, "marathon")
 	s.composeUp(c)
 
-	s.marathonURL = "http://" + containerNameMarathon + ":8080"
+	s.marathonURL = "http://" + s.getComposeServiceIP(c, containerNameMarathon) + ":8080"
 
 	// Wait for Marathon readiness prior to creating the client so that we
 	// don't run into the "all cluster members down" state right from the
@@ -45,6 +45,7 @@ func (s *MarathonSuite) TestConfigurationUpdate(c *check.C) {
 		MarathonURL string
 	}{s.marathonURL})
 	defer os.Remove(file)
+
 	cmd, display := s.traefikCmd(withConfigFile(file))
 	defer display(c)
 	err := cmd.Start()

integration/resources/compose/marathon.yml

@@ -32,7 +32,7 @@ services:
     -v /var/run/docker.sock:/var/run/docker.sock \
     -v /cgroup:/cgroup -v /sys:/sys \
     -v /usr/local/bin/docker:/usr/local/bin/docker \
-    -e MESOS_HOSTNAME=mesos-slave \
+    -e MESOS_HOSTNAME=$$(hostname -i) \
     -e MESOS_CONTAINERIZERS=docker,mesos \
     -e MESOS_ISOLATOR=cgroups/cpu,cgroups/mem \
     -e MESOS_LOG_DIR=/var/log \

integration/resources/compose/marathon15.yml

@@ -17,24 +17,32 @@ services:
       MESOS_ZK: zk://zookeeper:2181/mesos
 
   mesos-slave:
-    image: mesosphere/mesos-slave-dind:0.4.0_mesos-1.4.1_docker-17.05.0_ubuntu-16.04.3
+    image: docker:dind
     privileged: true
     # Uncomment published ports for interactive debugging.
     # ports:
     #   - "5051:5051"
-    environment:
+    command:
-      MESOS_HOSTNAME: mesos-slave
+    - "/bin/sh"
-      MESOS_CONTAINERIZERS: docker,mesos
+    - "-c"
-      MESOS_ISOLATOR: cgroups/cpu,cgroups/mem
+    - "(/usr/local/bin/dockerd-entrypoint.sh &); sleep 10; set -x; \
-      MESOS_LOG_DIR: /var/log
+    docker -H unix:///var/run/docker.sock run -d --net=host --privileged \
-      MESOS_MASTER: zk://zookeeper:2181/mesos
+    -v /var/run/docker.sock:/var/run/docker.sock \
-      MESOS_PORT: 5051
+    -v /cgroup:/cgroup -v /sys:/sys \
-      MESOS_WORK_DIR: /var/lib/mesos
+    -v /usr/local/bin/docker:/usr/local/bin/docker \
-      MESOS_EXECUTOR_REGISTRATION_TIMEOUT: 5mins
+    -e MESOS_HOSTNAME=$$(hostname -i) \
-      MESOS_EXECUTOR_SHUTDOWN_GRACE_PERIOD: 90secs
+    -e MESOS_CONTAINERIZERS=docker,mesos \
-      MESOS_DOCKER_STOP_TIMEOUT: 60secs
+    -e MESOS_ISOLATOR=cgroups/cpu,cgroups/mem \
-      MESOS_RESOURCES: cpus:2;mem:2048;disk:20480;ports(*):[12000-12999]
+    -e MESOS_LOG_DIR=/var/log \
-      MESOS_SYSTEMD_ENABLE_SUPPORT: false
+    -e MESOS_MASTER=zk://zookeeper:2181/mesos \
+    -e MESOS_PORT=5051 \
+    -e MESOS_WORK_DIR=/var/lib/mesos \
+    -e MESOS_EXECUTOR_REGISTRATION_TIMEOUT=5mins \
+    -e MESOS_EXECUTOR_SHUTDOWN_GRACE_PERIOD=90secs \
+    -e MESOS_DOCKER_STOP_TIMEOUT=60secs \
+    -e MESOS_RESOURCES='cpus:2;mem:2048;disk:20480;ports(*):[12000-12999]' \
+    -e MESOS_SYSTEMD_ENABLE_SUPPORT=false \
+    mesosphere/mesos-slave:1.4.1; sleep 600"
 
   marathon:
     image: mesosphere/marathon:v1.5.9

integration/resources/compose/tailscale.yml

@@ -0,0 +1,17 @@
+version: "3.8"
+services:
+  tailscaled:
+    hostname: traefik-tests-gw             # This will become the tailscale device name
+    image: tailscale/tailscale:v1.24.0
+    volumes:
+      # TODO: maybe mount the container's /var/lib to keep some state for tailscale?
+      - "/dev/net/tun:/dev/net/tun"       # Required for tailscale to work
+    cap_add:                  # Required for tailscale to work
+      - net_admin
+      - sys_module
+    command: tailscaled
+
+networks:
+  default:
+    name: traefik-test-network
+    external: true

integration/tcp_test.go

@@ -25,7 +25,17 @@ func (s *TCPSuite) SetUpSuite(c *check.C) {
 }
 
 func (s *TCPSuite) TestMixed(c *check.C) {
-	file := s.adaptFile(c, "fixtures/tcp/mixed.toml", struct{}{})
+	file := s.adaptFile(c, "fixtures/tcp/mixed.toml", struct {
+		Whoami       string
+		WhoamiA      string
+		WhoamiB      string
+		WhoamiNoCert string
+	}{
+		Whoami:       "http://" + s.getComposeServiceIP(c, "whoami") + ":80",
+		WhoamiA:      s.getComposeServiceIP(c, "whoami-a") + ":8080",
+		WhoamiB:      s.getComposeServiceIP(c, "whoami-b") + ":8080",
+		WhoamiNoCert: s.getComposeServiceIP(c, "whoami-no-cert") + ":8080",
+	})
 	defer os.Remove(file)
 
 	cmd, display := s.traefikCmd(withConfigFile(file))
@@ -75,7 +85,11 @@ func (s *TCPSuite) TestMixed(c *check.C) {
 }
 
 func (s *TCPSuite) TestTLSOptions(c *check.C) {
-	file := s.adaptFile(c, "fixtures/tcp/multi-tls-options.toml", struct{}{})
+	file := s.adaptFile(c, "fixtures/tcp/multi-tls-options.toml", struct {
+		WhoamiNoCert string
+	}{
+		WhoamiNoCert: s.getComposeServiceIP(c, "whoami-no-cert") + ":8080",
+	})
 	defer os.Remove(file)
 
 	cmd, display := s.traefikCmd(withConfigFile(file))
@@ -105,7 +119,17 @@ func (s *TCPSuite) TestTLSOptions(c *check.C) {
 }
 
 func (s *TCPSuite) TestNonTLSFallback(c *check.C) {
-	file := s.adaptFile(c, "fixtures/tcp/non-tls-fallback.toml", struct{}{})
+	file := s.adaptFile(c, "fixtures/tcp/non-tls-fallback.toml", struct {
+		WhoamiA      string
+		WhoamiB      string
+		WhoamiNoCert string
+		WhoamiNoTLS  string
+	}{
+		WhoamiA:      s.getComposeServiceIP(c, "whoami-a") + ":8080",
+		WhoamiB:      s.getComposeServiceIP(c, "whoami-b") + ":8080",
+		WhoamiNoCert: s.getComposeServiceIP(c, "whoami-no-cert") + ":8080",
+		WhoamiNoTLS:  s.getComposeServiceIP(c, "whoami-no-tls") + ":8080",
+	})
 	defer os.Remove(file)
 
 	cmd, display := s.traefikCmd(withConfigFile(file))
@@ -139,7 +163,11 @@ func (s *TCPSuite) TestNonTLSFallback(c *check.C) {
 }
 
 func (s *TCPSuite) TestNonTlsTcp(c *check.C) {
-	file := s.adaptFile(c, "fixtures/tcp/non-tls.toml", struct{}{})
+	file := s.adaptFile(c, "fixtures/tcp/non-tls.toml", struct {
+		WhoamiNoTLS string
+	}{
+		WhoamiNoTLS: s.getComposeServiceIP(c, "whoami-no-tls") + ":8080",
+	})
 	defer os.Remove(file)
 
 	cmd, display := s.traefikCmd(withConfigFile(file))
@@ -159,7 +187,11 @@ func (s *TCPSuite) TestNonTlsTcp(c *check.C) {
 }
 
 func (s *TCPSuite) TestCatchAllNoTLS(c *check.C) {
-	file := s.adaptFile(c, "fixtures/tcp/catch-all-no-tls.toml", struct{}{})
+	file := s.adaptFile(c, "fixtures/tcp/catch-all-no-tls.toml", struct {
+		WhoamiBannerAddress string
+	}{
+		WhoamiBannerAddress: s.getComposeServiceIP(c, "whoami-banner") + ":8080",
+	})
 	defer os.Remove(file)
 
 	cmd, display := s.traefikCmd(withConfigFile(file))
@@ -179,7 +211,13 @@ func (s *TCPSuite) TestCatchAllNoTLS(c *check.C) {
 }
 
 func (s *TCPSuite) TestCatchAllNoTLSWithHTTPS(c *check.C) {
-	file := s.adaptFile(c, "fixtures/tcp/catch-all-no-tls-with-https.toml", struct{}{})
+	file := s.adaptFile(c, "fixtures/tcp/catch-all-no-tls-with-https.toml", struct {
+		WhoamiNoTLSAddress string
+		WhoamiURL          string
+	}{
+		WhoamiNoTLSAddress: s.getComposeServiceIP(c, "whoami-no-tls") + ":8080",
+		WhoamiURL:          "http://" + s.getComposeServiceIP(c, "whoami") + ":80",
+	})
 	defer os.Remove(file)
 
 	cmd, display := s.traefikCmd(withConfigFile(file))
@@ -204,7 +242,13 @@ func (s *TCPSuite) TestCatchAllNoTLSWithHTTPS(c *check.C) {
 }
 
 func (s *TCPSuite) TestMiddlewareWhiteList(c *check.C) {
-	file := s.adaptFile(c, "fixtures/tcp/ip-whitelist.toml", struct{}{})
+	file := s.adaptFile(c, "fixtures/tcp/ip-whitelist.toml", struct {
+		WhoamiA string
+		WhoamiB string
+	}{
+		WhoamiA: s.getComposeServiceIP(c, "whoami-a") + ":8080",
+		WhoamiB: s.getComposeServiceIP(c, "whoami-b") + ":8080",
+	})
 	defer os.Remove(file)
 
 	cmd, display := s.traefikCmd(withConfigFile(file))
@@ -228,7 +272,13 @@ func (s *TCPSuite) TestMiddlewareWhiteList(c *check.C) {
 }
 
 func (s *TCPSuite) TestWRR(c *check.C) {
-	file := s.adaptFile(c, "fixtures/tcp/wrr.toml", struct{}{})
+	file := s.adaptFile(c, "fixtures/tcp/wrr.toml", struct {
+		WhoamiB  string
+		WhoamiAB string
+	}{
+		WhoamiB:  s.getComposeServiceIP(c, "whoami-b") + ":8080",
+		WhoamiAB: s.getComposeServiceIP(c, "whoami-ab") + ":8080",
+	})
 	defer os.Remove(file)
 
 	cmd, display := s.traefikCmd(withConfigFile(file))