From 49a09ab7dd500c809f7af13288a6ba51ce39d396 Mon Sep 17 00:00:00 2001 From: Timo Reimann Date: Thu, 2 Feb 2017 00:49:32 +0100 Subject: [PATCH] Prepare for dependency vendoring. - Add helper script to simplify glide usage. - Add validation script for unwanted changes to vendoring. - Relax/tighten up .{git,docker}ignore to cover vendored files properly. - .validate: Protect from unbound variable in case of nounset setting. - Install more recent hg version in the build container. - Remove glide installation steps from Dockerfile. - Update documentation. --- .dockerignore | 8 +-- .github/CONTRIBUTING.md | 29 +++++----- .gitignore | 12 ++-- Makefile | 2 +- build.Dockerfile | 19 +++---- script/.validate | 2 +- script/glide.sh | 119 +++++++++++++++++++++++++++++++++++++++ script/validate-glide | 2 +- script/validate-gofmt | 2 +- script/validate-golint | 2 +- script/validate-govet | 2 +- script/validate-misspell | 2 +- script/validate-vendor | 37 ++++++++++++ 13 files changed, 196 insertions(+), 42 deletions(-) create mode 100755 script/glide.sh create mode 100755 script/validate-vendor diff --git a/.dockerignore b/.dockerignore index fed966aed..87982c34c 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,5 +1,3 @@ -dist/ -vendor/ -!dist/traefik -site/ -**/*.test +/dist/ +!/dist/traefik +/site/ diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 1c15a7b16..fac033067 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -2,7 +2,7 @@ ### Building -You need either [Docker](https://github.com/docker/docker) and `make` (Method 1), or `go` and `glide` (Method 2) in order to build traefik. +You need either [Docker](https://github.com/docker/docker) and `make` (Method 1), or `go` (Method 2) in order to build traefik. For changes to its dependencies, the `glide` dependency management tool and `glide-vc` plugin are required. #### Method 1: Using `Docker` and `Makefile` @@ -26,7 +26,7 @@ $ ls dist/ traefik* ``` -#### Method 2: Using `go` and `glide` +#### Method 2: Using `go` ###### Setting up your `go` environment @@ -42,20 +42,25 @@ This can be verified via `$ go env` - You will want to add those 2 export lines to your `.bashrc` or `.bash_profile` - You need `go-bindata` to be able to use `go generate` command (needed to build) : `$ go get github.com/jteeuwen/go-bindata/...` (Please note, the ellipses are required) -###### Setting up your `glide` environment +#### Setting up `glide` and `glide-vc` for dependency management +- Glide is not required for building; however, it is necessary to modify dependencies (i.e., add, update, or remove third-party packages) - Glide can be installed either via homebrew: `$ brew install glide` or via the official glide script: `$ curl https://glide.sh/get | sh` +- The glide plugin `glide-vc` must be installed from source: `go get github.com/sgotti/glide-vc` -The idea behind `glide` is the following : +If you want to add a dependency, use `$ glide get` to have glide put it into the vendor folder and update the glide manifest/lock files (`glide.yaml` and `glide.lock`, respectively). A following `glide-vc` run should be triggered to trim down the size of the vendor folder. The final result must be committed into VCS. -- when checkout(ing) a project, run `$ glide install -v` from the cloned directory to install - (`go get …`) the dependencies in your `GOPATH`. -- if you need another dependency, import and use it in - the source, and run `$ glide get github.com/Masterminds/cookoo` to save it in - `vendor` and add it to your `glide.yaml`. +Dependencies for the integration tests in the `integration` folder are managed in a separate `integration/glide.yaml` file using the same toolset. + +Care must be taken to choose the right arguments to `glide` when dealing with either main or integration test dependencies, or otherwise risk ending up with a broken build. For that reason, the helper script `script/glide.sh` encapsulates the gory details and conveniently calls `glide-vc` as well. Call it without parameters for basic usage instructions. + +Here's a full example: ```bash -$ glide install --strip-vendor +# install the new main dependency github.com/foo/bar and minimize vendor size +$ ./script/glide.sh get github.com/foo/bar +# install another dependency, this time for the integration tests +$ ( cd integration && ../script/glide.sh get github.com/baz/quuz ) # generate (Only required to integrate other components such as web dashboard) $ go generate # Standard go build @@ -105,14 +110,12 @@ TESTFLAGS="-check.f MyTestSuite.*Test" make test-integration More: https://labix.org/gocheck -##### Method 2: `go` and `glide` +##### Method 2: `go` - Tests can be run from the cloned directory, by `$ go test ./...` which should return `ok` similar to: ``` ok _/home/vincent/src/github/vdemeester/traefik 0.004s ``` -- Note that `$ go test ./...` will run all tests (including the ones in the vendor directory for the dependencies that glide have fetched). If you only want to run the tests for traefik use `$ go test $(glide novendor)` instead. - ### Documentation diff --git a/.gitignore b/.gitignore index 9a57474fa..e906945ad 100644 --- a/.gitignore +++ b/.gitignore @@ -1,15 +1,13 @@ /dist -gen.go +/autogen/gen.go .idea .intellij *.iml -traefik -traefik.toml -*.test -vendor/ -static/ +/traefik +/traefik.toml +/static/ .vscode/ -site/ +/site/ *.log *.exe .DS_Store diff --git a/Makefile b/Makefile index 6d5ea58b4..f3ef74af6 100644 --- a/Makefile +++ b/Makefile @@ -45,7 +45,7 @@ test-integration: build ## run the integration tests $(DOCKER_RUN_TRAEFIK) ./script/make.sh generate binary test-integration validate: build ## validate gofmt, golint and go vet - $(DOCKER_RUN_TRAEFIK) ./script/make.sh validate-glide validate-gofmt validate-govet validate-golint validate-misspell + $(DOCKER_RUN_TRAEFIK) ./script/make.sh validate-glide validate-gofmt validate-govet validate-golint validate-misspell validate-vendor build: dist docker build $(DOCKER_BUILD_ARGS) -t "$(TRAEFIK_DEV_IMAGE)" -f build.Dockerfile . diff --git a/build.Dockerfile b/build.Dockerfile index 25d9f4776..02440b249 100644 --- a/build.Dockerfile +++ b/build.Dockerfile @@ -1,10 +1,18 @@ FROM golang:1.7 +# Install a more recent version of mercurial to avoid mismatching results +# between glide run on a decently updated host system and the build container. +RUN awk '$1 ~ "^deb" { $3 = $3 "-backports"; print; exit }' /etc/apt/sources.list > /etc/apt/sources.list.d/backports.list && \ + DEBIAN_FRONTEND=noninteractive apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -t jessie-backports --yes --no-install-recommends mercurial=3.9.1-1~bpo8+1 && \ + rm -fr /var/lib/apt/lists/ + RUN go get github.com/jteeuwen/go-bindata/... \ && go get github.com/golang/lint/golint \ && go get github.com/kisielk/errcheck \ && go get github.com/client9/misspell/cmd/misspell \ -&& go get github.com/mattfarina/glide-hash +&& go get github.com/mattfarina/glide-hash \ +&& go get github.com/sgotti/glide-vc # Which docker version to test on ARG DOCKER_VERSION=1.10.3 @@ -24,13 +32,4 @@ RUN mkdir -p /usr/local/bin \ | tar -xzC /usr/local/bin --transform 's#^.+/##x' WORKDIR /go/src/github.com/containous/traefik - -COPY glide.yaml glide.yaml -COPY glide.lock glide.lock -RUN glide install -v - -COPY integration/glide.yaml integration/glide.yaml -COPY integration/glide.lock integration/glide.lock -RUN cd integration && glide install - COPY . /go/src/github.com/containous/traefik diff --git a/script/.validate b/script/.validate index 5bc62ec07..4db554733 100644 --- a/script/.validate +++ b/script/.validate @@ -1,6 +1,6 @@ #!/bin/bash -if [ -z "$VALIDATE_UPSTREAM" ]; then +if [ -z "${VALIDATE_UPSTREAM:-}" ]; then # this is kind of an expensive check, so let's not do this twice if we # are running more than one validate bundlescript diff --git a/script/glide.sh b/script/glide.sh new file mode 100755 index 000000000..e7ed6a275 --- /dev/null +++ b/script/glide.sh @@ -0,0 +1,119 @@ +#!/bin/bash +set -o errexit +set -o pipefail +set -o nounset + +#### +### Helper script for glide[-vc] to handle specifics for the Traefik repo. +## +# In particular, the 'integration/' directory contains its own set of +# glide-managed dependencies which must not have its nested vendor folder +# stripped. Depending on where the script is called from, it will do the Right +# Thing. +# + +CWD="$(pwd)"; readonly CWD +GLIDE_ARGS=() +GLIDE_VC_ARGS=( + '--use-lock-file' # `glide list` seems to miss test dependencies, e.g., github.com/mattn/go-shellwords + '--only-code' + '--no-tests' +) + +usage() { + echo "usage: $(basename "$0") install | update | get | trim +install: Install all dependencies and trim the vendor folder afterwards (alternative command: i). +update: Update all dependencies and trim the vendor folder afterwards (alternative command: up). +get: Add a dependency and trim the vendor folder afterwards. +trim: Trim the vendor folder only, do not install or update dependencies. + +The current working directory must contain a glide.yaml file." >&2 +} + +is_integration_dir() { + [[ "$(basename ${CWD})" = 'integration' ]] +} + +if ! is_integration_dir; then + GLIDE_ARGS+=('--strip-vendor' '--skip-test') +fi + +if ! type glide > /dev/null 2>&1; then + echo "glide not found in PATH." >&2 + exit 1 +fi + +if ! type glide-vc > /dev/null 2>&1; then + echo "glide-vc not found in PATH." >&2 + exit 1 +fi + +if [[ ! -e "${CWD}/glide.yaml" ]]; then + echo "no glide.yaml file found in the current working directory" >&2 + exit 1 +fi + +if [[ $# -lt 1 ]]; then + echo "missing command" >&2 + usage + exit 1 +fi + +readonly glide_command="$1" +shift + +skip_glide_command= +case "${glide_command}" in + 'install' | 'i') + if [[ $# -ne 0 ]]; then + echo "surplus parameters given" >&2 + usage + exit 1 + fi + ;; + + 'update' | 'up') + if [[ $# -ne 0 ]]; then + echo "surplus parameters given" >&2 + usage + exit 1 + fi + ;; + + 'get') + if [[ $# -ne 1 ]]; then + echo 'insufficient/surplus arguments given for "get" command' >&2 + usage + exit 1 + fi + GLIDE_ARGS=("$1" "${GLIDE_ARGS[@]}") + shift + ;; + + 'trim') + if [[ $# -ne 0 ]]; then + echo "surplus parameters given" >&2 + usage + exit 1 + fi + skip_glide_command=yes + ;; + + *) + echo "unknown command: ${glide_command}" >&2 + usage + exit 1 +esac +readonly skip_glide_command + +if [[ -z "${skip_glide_command}" ]]; then + # Use parameter substitution to account for an empty glide arguments array + # that would otherwise lead to an "unbound variable" error due to the nounset + # option. + GLIDE_ARGS=("${GLIDE_ARGS+"${GLIDE_ARGS[@]}"}") + echo "running: glide ${glide_command} ${GLIDE_ARGS[*]}" + glide ${glide_command} ${GLIDE_ARGS[*]} +fi + +echo "trimming vendor folder using: glide-vc ${GLIDE_VC_ARGS[*]}" +glide-vc ${GLIDE_VC_ARGS[*]} diff --git a/script/validate-glide b/script/validate-glide index e8d2a0988..e65b13c48 100755 --- a/script/validate-glide +++ b/script/validate-glide @@ -6,7 +6,7 @@ if grep -q "$(glide-hash)" glide.lock; then echo 'Congratulations! glide.lock is unchanged.' else { - echo "Error: glide.lock has been manually changed. Don't do this. Use glide up instead." + echo "Error: glide.lock has been manually changed. Don't do this. Use script/glide.sh up instead." echo } >&2 false diff --git a/script/validate-gofmt b/script/validate-gofmt index 70d136c3f..f1bcc89dd 100755 --- a/script/validate-gofmt +++ b/script/validate-gofmt @@ -3,7 +3,7 @@ source "$(dirname "$BASH_SOURCE")/.validate" IFS=$'\n' -files=( $(validate_diff --diff-filter=ACMR --name-only -- '*.go' | grep -v '^vendor' || true) ) +files=( $(validate_diff --diff-filter=ACMR --name-only -- '*.go' | grep -v '^\(integration/\)\?vendor/' || true) ) unset IFS badFiles=() diff --git a/script/validate-golint b/script/validate-golint index 2f7769fdd..b17630e2d 100755 --- a/script/validate-golint +++ b/script/validate-golint @@ -3,7 +3,7 @@ source "$(dirname "$BASH_SOURCE")/.validate" IFS=$'\n' -files=( $(validate_diff --diff-filter=ACMR --name-only -- '*.go' | grep -v '^vendor/\|autogen' || true) ) +files=( $(validate_diff --diff-filter=ACMR --name-only -- '*.go' | grep -v '^\(integration/\)\?vendor/\|autogen' || true) ) unset IFS errors=() diff --git a/script/validate-govet b/script/validate-govet index 873221d99..83dc0acf4 100755 --- a/script/validate-govet +++ b/script/validate-govet @@ -3,7 +3,7 @@ source "$(dirname "$BASH_SOURCE")/.validate" IFS=$'\n' -files=( $(validate_diff --diff-filter=ACMR --name-only -- '*.go' | grep -v '^vendor/' || true) ) +files=( $(validate_diff --diff-filter=ACMR --name-only -- '*.go' | grep -v '^\(integration/\)\?vendor/' || true) ) unset IFS errors=() diff --git a/script/validate-misspell b/script/validate-misspell index 6cc1e2fa4..a52ed4e98 100755 --- a/script/validate-misspell +++ b/script/validate-misspell @@ -3,7 +3,7 @@ source "$(dirname "$BASH_SOURCE")/.validate" IFS=$'\n' -src=( $(validate_diff --diff-filter=ACMR --name-only -- '*.go' | grep -v '^vendor/\|autogen' || true) ) +src=( $(validate_diff --diff-filter=ACMR --name-only -- '*.go' | grep -v '^\(integration/\)\?vendor/\|autogen' || true) ) docs=( $(validate_diff --diff-filter=ACMR --name-only -- 'docs/*.md') ) unset IFS files=("${src[@]}" "${docs[@]}") diff --git a/script/validate-vendor b/script/validate-vendor new file mode 100755 index 000000000..ad3dd90d7 --- /dev/null +++ b/script/validate-vendor @@ -0,0 +1,37 @@ +#!/bin/bash +set -o errexit +set -o pipefail +set -o nounset + +SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"; export SCRIPTDIR +source "${SCRIPTDIR}/.validate" + +# Iterate over all directories containing vendor folders. +for dir in . integration; do + vendor_dir="${dir}/vendor/" + IFS=$'\n' files=( $(validate_diff --diff-filter=ACMR --name-only -- ${vendor_dir} || true) ) + + if [[ ${#files[@]} -gt 0 ]]; then + # We run glide install to and see if we have a diff afterwards + echo "checking ${vendor_dir} for unintentional changes..." + ( + cd ${dir} + "${SCRIPTDIR}/glide.sh" install + ) + # Let see if the working directory is clean + diffs="$(git status --porcelain -- ${vendor_dir} 2>/dev/null)" + if [[ "$diffs" ]]; then + { + echo "The result of 'glide install' for vendor directory '${dir}' differs" + echo + echo "$diffs" + echo + echo 'Please vendor your package(s) with script/glide.sh.' + echo + } >&2 + exit 2 + fi + fi +done + +echo 'Congratulations! All vendoring changes are done the right way.'