From 07a3c37a23f2b2dd657d1fd51fb5cb5a8f1b37ae Mon Sep 17 00:00:00 2001 From: Lukas Schulte Pelkum Date: Mon, 20 Sep 2021 18:00:08 +0200 Subject: [PATCH] Implement customizable minimum body size for compress middleware --- docs/content/middlewares/http/compress.md | 54 ++++++++++++++++++- .../dynamic-configuration/docker-labels.yml | 1 + .../reference/dynamic-configuration/file.toml | 1 + .../reference/dynamic-configuration/file.yaml | 1 + .../reference/dynamic-configuration/kv-ref.md | 1 + .../marathon-labels.json | 1 + .../traefik.containo.us_middlewares.yaml | 2 + integration/fixtures/k8s/01-traefik-crd.yml | 2 + pkg/config/dynamic/middlewares.go | 1 + pkg/config/label/label_test.go | 12 +++-- pkg/middlewares/compress/compress.go | 10 +++- pkg/middlewares/compress/compress_test.go | 51 ++++++++++++++++++ pkg/provider/kv/kv_test.go | 6 ++- 13 files changed, 134 insertions(+), 9 deletions(-) diff --git a/docs/content/middlewares/http/compress.md b/docs/content/middlewares/http/compress.md index ea3c8d621..95e1268b9 100644 --- a/docs/content/middlewares/http/compress.md +++ b/docs/content/middlewares/http/compress.md @@ -60,7 +60,7 @@ http: Responses are compressed when the following criteria are all met: - * The response body is larger than `1400` bytes. + * The response body is larger than the configured minimum amount of bytes (default is `1024`). * The `Accept-Encoding` request header contains `gzip`. * The response is not already compressed, i.e. the `Content-Encoding` response header is not already set. @@ -122,3 +122,55 @@ http: [http.middlewares.test-compress.compress] excludedContentTypes = ["text/event-stream"] ``` + +### `minResponseBodyBytes` + +`minResponseBodyBytes` specifies the minimum amount of bytes a response body must have to be compressed. + +The default value is `1024`, which should be a reasonable value for most cases. + +Responses smaller than the specified values will not be compressed. + +```yaml tab="Docker" +labels: + - "traefik.http.middlewares.test-compress.compress.minresponsebodybytes=1200" +``` + +```yaml tab="Kubernetes" +apiVersion: traefik.containo.us/v1alpha1 +kind: Middleware +metadata: + name: test-compress +spec: + compress: + minResponseBodyBytes: 1200 +``` + +```yaml tab="Consul Catalog" +- "traefik.http.middlewares.test-compress.compress.minresponsebodybytes=1200" +``` + +```json tab="Marathon" +"labels": { + "traefik.http.middlewares.test-compress.compress.minresponsebodybytes": 1200 +} +``` + +```yaml tab="Rancher" +labels: + - "traefik.http.middlewares.test-compress.compress.minresponsebodybytes=1200" +``` + +```yaml tab="File (YAML)" +http: + middlewares: + test-compress: + compress: + minResponseBodyBytes: 1200 +``` + +```toml tab="File (TOML)" +[http.middlewares] + [http.middlewares.test-compress.compress] + minResponseBodyBytes = 1200 +``` \ No newline at end of file diff --git a/docs/content/reference/dynamic-configuration/docker-labels.yml b/docs/content/reference/dynamic-configuration/docker-labels.yml index 82574efdd..ee2f33122 100644 --- a/docs/content/reference/dynamic-configuration/docker-labels.yml +++ b/docs/content/reference/dynamic-configuration/docker-labels.yml @@ -13,6 +13,7 @@ - "traefik.http.middlewares.middleware04.circuitbreaker.expression=foobar" - "traefik.http.middlewares.middleware05.compress=true" - "traefik.http.middlewares.middleware05.compress.excludedcontenttypes=foobar, foobar" +- "traefik.http.middlewares.middleware05.compress.minresponsebodybytes=42" - "traefik.http.middlewares.middleware06.contenttype.autodetect=true" - "traefik.http.middlewares.middleware07.digestauth.headerfield=foobar" - "traefik.http.middlewares.middleware07.digestauth.realm=foobar" diff --git a/docs/content/reference/dynamic-configuration/file.toml b/docs/content/reference/dynamic-configuration/file.toml index e521e20b1..0aad383d6 100644 --- a/docs/content/reference/dynamic-configuration/file.toml +++ b/docs/content/reference/dynamic-configuration/file.toml @@ -122,6 +122,7 @@ [http.middlewares.Middleware05] [http.middlewares.Middleware05.compress] excludedContentTypes = ["foobar", "foobar"] + minResponseBodyBytes = 42 [http.middlewares.Middleware06] [http.middlewares.Middleware06.contentType] autoDetect = true diff --git a/docs/content/reference/dynamic-configuration/file.yaml b/docs/content/reference/dynamic-configuration/file.yaml index 8a4dc1aca..846e9ea2c 100644 --- a/docs/content/reference/dynamic-configuration/file.yaml +++ b/docs/content/reference/dynamic-configuration/file.yaml @@ -128,6 +128,7 @@ http: excludedContentTypes: - foobar - foobar + minResponseBodyBytes: 42 Middleware06: contentType: autoDetect: true diff --git a/docs/content/reference/dynamic-configuration/kv-ref.md b/docs/content/reference/dynamic-configuration/kv-ref.md index 5921510ee..0df01e697 100644 --- a/docs/content/reference/dynamic-configuration/kv-ref.md +++ b/docs/content/reference/dynamic-configuration/kv-ref.md @@ -15,6 +15,7 @@ | `traefik/http/middlewares/Middleware04/circuitBreaker/expression` | `foobar` | | `traefik/http/middlewares/Middleware05/compress/excludedContentTypes/0` | `foobar` | | `traefik/http/middlewares/Middleware05/compress/excludedContentTypes/1` | `foobar` | +| `traefik/http/middlewares/Middleware05/compress/minResponseBodyBytes` | `42` | | `traefik/http/middlewares/Middleware06/contentType/autoDetect` | `true` | | `traefik/http/middlewares/Middleware07/digestAuth/headerField` | `foobar` | | `traefik/http/middlewares/Middleware07/digestAuth/realm` | `foobar` | diff --git a/docs/content/reference/dynamic-configuration/marathon-labels.json b/docs/content/reference/dynamic-configuration/marathon-labels.json index 93aa9c6d1..58128731d 100644 --- a/docs/content/reference/dynamic-configuration/marathon-labels.json +++ b/docs/content/reference/dynamic-configuration/marathon-labels.json @@ -13,6 +13,7 @@ "traefik.http.middlewares.middleware04.circuitbreaker.expression": "foobar", "traefik.http.middlewares.middleware05.compress": "true", "traefik.http.middlewares.middleware05.compress.excludedcontenttypes": "foobar, foobar", +"traefik.http.middlewares.middleware05.compress.minresponsebodybytes": "42", "traefik.http.middlewares.middleware06.contenttype.autodetect": "true", "traefik.http.middlewares.middleware07.digestauth.headerfield": "foobar", "traefik.http.middlewares.middleware07.digestauth.realm": "foobar", diff --git a/docs/content/reference/dynamic-configuration/traefik.containo.us_middlewares.yaml b/docs/content/reference/dynamic-configuration/traefik.containo.us_middlewares.yaml index 9a2a274aa..63e57686a 100644 --- a/docs/content/reference/dynamic-configuration/traefik.containo.us_middlewares.yaml +++ b/docs/content/reference/dynamic-configuration/traefik.containo.us_middlewares.yaml @@ -101,6 +101,8 @@ spec: items: type: string type: array + minResponseBodyBytes: + type: integer type: object contentType: description: ContentType middleware - or rather its unique `autoDetect` diff --git a/integration/fixtures/k8s/01-traefik-crd.yml b/integration/fixtures/k8s/01-traefik-crd.yml index 4b19e33b7..690267b65 100644 --- a/integration/fixtures/k8s/01-traefik-crd.yml +++ b/integration/fixtures/k8s/01-traefik-crd.yml @@ -543,6 +543,8 @@ spec: items: type: string type: array + minResponseBodyBytes: + type: integer type: object contentType: description: ContentType middleware - or rather its unique `autoDetect` diff --git a/pkg/config/dynamic/middlewares.go b/pkg/config/dynamic/middlewares.go index 136901af7..9c45fbfef 100644 --- a/pkg/config/dynamic/middlewares.go +++ b/pkg/config/dynamic/middlewares.go @@ -104,6 +104,7 @@ type CircuitBreaker struct { // Compress holds the compress configuration. type Compress struct { ExcludedContentTypes []string `json:"excludedContentTypes,omitempty" toml:"excludedContentTypes,omitempty" yaml:"excludedContentTypes,omitempty" export:"true"` + MinResponseBodyBytes int `json:"minResponseBodyBytes,omitempty" toml:"minResponseBodyBytes,omitempty" yaml:"minResponseBodyBytes,omitempty" export:"true"` } // +k8s:deepcopy-gen=true diff --git a/pkg/config/label/label_test.go b/pkg/config/label/label_test.go index eac688246..c3827bda2 100644 --- a/pkg/config/label/label_test.go +++ b/pkg/config/label/label_test.go @@ -126,7 +126,7 @@ func TestDecodeConfiguration(t *testing.T) { "traefik.http.middlewares.Middleware16.retry.initialinterval": "1s", "traefik.http.middlewares.Middleware17.stripprefix.prefixes": "foobar, fiibar", "traefik.http.middlewares.Middleware18.stripprefixregex.regex": "foobar, fiibar", - "traefik.http.middlewares.Middleware19.compress": "true", + "traefik.http.middlewares.Middleware19.compress.minresponsebodybytes": "42", "traefik.http.middlewares.Middleware20.plugin.tomato.aaa": "foo1", "traefik.http.middlewares.Middleware20.plugin.tomato.bbb": "foo2", "traefik.http.routers.Router0.entrypoints": "foobar, fiibar", @@ -454,7 +454,9 @@ func TestDecodeConfiguration(t *testing.T) { }, }, "Middleware19": { - Compress: &dynamic.Compress{}, + Compress: &dynamic.Compress{ + MinResponseBodyBytes: 42, + }, }, "Middleware2": { Buffering: &dynamic.Buffering{ @@ -932,7 +934,9 @@ func TestEncodeConfiguration(t *testing.T) { }, }, "Middleware19": { - Compress: &dynamic.Compress{}, + Compress: &dynamic.Compress{ + MinResponseBodyBytes: 42, + }, }, "Middleware2": { Buffering: &dynamic.Buffering{ @@ -1270,7 +1274,7 @@ func TestEncodeConfiguration(t *testing.T) { "traefik.HTTP.Middlewares.Middleware17.StripPrefix.Prefixes": "foobar, fiibar", "traefik.HTTP.Middlewares.Middleware17.StripPrefix.ForceSlash": "true", "traefik.HTTP.Middlewares.Middleware18.StripPrefixRegex.Regex": "foobar, fiibar", - "traefik.HTTP.Middlewares.Middleware19.Compress": "true", + "traefik.HTTP.Middlewares.Middleware19.Compress.MinResponseBodyBytes": "42", "traefik.HTTP.Middlewares.Middleware20.Plugin.tomato.aaa": "foo1", "traefik.HTTP.Middlewares.Middleware20.Plugin.tomato.bbb": "foo2", diff --git a/pkg/middlewares/compress/compress.go b/pkg/middlewares/compress/compress.go index e6ddeb75d..a83cc6682 100644 --- a/pkg/middlewares/compress/compress.go +++ b/pkg/middlewares/compress/compress.go @@ -23,6 +23,7 @@ type compress struct { next http.Handler name string excludes []string + minSize int } // New creates a new compress middleware. @@ -39,7 +40,12 @@ func New(ctx context.Context, next http.Handler, conf dynamic.Compress, name str excludes = append(excludes, mediaType) } - return &compress{next: next, name: name, excludes: excludes}, nil + minSize := gzhttp.DefaultMinSize + if conf.MinResponseBodyBytes > 0 { + minSize = conf.MinResponseBodyBytes + } + + return &compress{next: next, name: name, excludes: excludes, minSize: minSize}, nil } func (c *compress) ServeHTTP(rw http.ResponseWriter, req *http.Request) { @@ -64,7 +70,7 @@ func (c *compress) gzipHandler(ctx context.Context) http.Handler { wrapper, err := gzhttp.NewWrapper( gzhttp.ExceptContentTypes(c.excludes), gzhttp.CompressionLevel(gzip.DefaultCompression), - gzhttp.MinSize(gzhttp.DefaultMinSize)) + gzhttp.MinSize(c.minSize)) if err != nil { log.FromContext(ctx).Error(err) } diff --git a/pkg/middlewares/compress/compress_test.go b/pkg/middlewares/compress/compress_test.go index 9dd4c8124..d065882e2 100644 --- a/pkg/middlewares/compress/compress_test.go +++ b/pkg/middlewares/compress/compress_test.go @@ -305,6 +305,57 @@ func TestIntegrationShouldCompress(t *testing.T) { } } +func TestMinResponseBodyBytes(t *testing.T) { + fakeBody := generateBytes(100000) + + testCases := []struct { + name string + minResponseBodyBytes int + expectedCompression bool + }{ + { + name: "should compress", + expectedCompression: true, + }, + { + name: "should not compress", + minResponseBodyBytes: 100001, + }, + } + + for _, test := range testCases { + test := test + + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + req := testhelpers.MustNewRequest(http.MethodGet, "http://localhost", nil) + req.Header.Add(acceptEncodingHeader, gzipValue) + + next := http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { + if _, err := rw.Write(fakeBody); err != nil { + http.Error(rw, err.Error(), http.StatusInternalServerError) + } + }) + + handler, err := New(context.Background(), next, dynamic.Compress{MinResponseBodyBytes: test.minResponseBodyBytes}, "testing") + require.NoError(t, err) + + rw := httptest.NewRecorder() + handler.ServeHTTP(rw, req) + + if test.expectedCompression { + assert.Equal(t, gzipValue, rw.Header().Get(contentEncodingHeader)) + assert.NotEqualValues(t, rw.Body.Bytes(), fakeBody) + return + } + + assert.Empty(t, rw.Header().Get(contentEncodingHeader)) + assert.EqualValues(t, rw.Body.Bytes(), fakeBody) + }) + } +} + func BenchmarkCompress(b *testing.B) { testCases := []struct { name string diff --git a/pkg/provider/kv/kv_test.go b/pkg/provider/kv/kv_test.go index ff3f941fd..d13874609 100644 --- a/pkg/provider/kv/kv_test.go +++ b/pkg/provider/kv/kv_test.go @@ -196,7 +196,7 @@ func Test_buildConfiguration(t *testing.T) { "traefik/http/middlewares/Middleware02/buffering/retryExpression": "foobar", "traefik/http/middlewares/Middleware02/buffering/maxRequestBodyBytes": "42", "traefik/http/middlewares/Middleware02/buffering/memRequestBodyBytes": "42", - "traefik/http/middlewares/Middleware05/compress": "", + "traefik/http/middlewares/Middleware05/compress/minResponseBodyBytes": "42", "traefik/http/middlewares/Middleware18/retry/attempts": "42", "traefik/http/middlewares/Middleware19/stripPrefix/prefixes/0": "foobar", "traefik/http/middlewares/Middleware19/stripPrefix/prefixes/1": "foobar", @@ -395,7 +395,9 @@ func Test_buildConfiguration(t *testing.T) { }, }, "Middleware05": { - Compress: &dynamic.Compress{}, + Compress: &dynamic.Compress{ + MinResponseBodyBytes: 42, + }, }, "Middleware08": { ForwardAuth: &dynamic.ForwardAuth{