From c515ace32862f7ffe202e933c106e135160c43e3 Mon Sep 17 00:00:00 2001 From: Tom Moulard Date: Mon, 19 Jul 2021 10:22:14 +0200 Subject: [PATCH] Library change for compress middleware to increase performance --- go.mod | 2 +- go.sum | 7 +- pkg/middlewares/compress/compress.go | 10 +- pkg/middlewares/compress/compress_test.go | 111 +++++++++++++++++++--- 4 files changed, 110 insertions(+), 20 deletions(-) diff --git a/go.mod b/go.mod index 1f12a3bdc..862d099b1 100644 --- a/go.mod +++ b/go.mod @@ -44,6 +44,7 @@ require ( github.com/hashicorp/go-version v1.2.1 github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d github.com/instana/go-sensor v1.5.1 + github.com/klauspost/compress v1.13.0 github.com/libkermit/compose v0.0.0-20171122111507-c04e39c026ad github.com/libkermit/docker v0.0.0-20171122101128-e6674d32b807 github.com/libkermit/docker-check v0.0.0-20171122104347-1113af38e591 @@ -71,7 +72,6 @@ require ( github.com/stretchr/testify v1.7.0 github.com/stvp/go-udp-testing v0.0.0-20191102171040-06b61409b154 github.com/tinylib/msgp v1.0.2 // indirect - github.com/traefik/gziphandler v1.1.2-0.20210212101304-175e0fad6888 github.com/traefik/paerser v0.1.4 github.com/traefik/yaegi v0.9.20 github.com/uber/jaeger-client-go v2.29.1+incompatible diff --git a/go.sum b/go.sum index 152020858..4756ad792 100644 --- a/go.sum +++ b/go.sum @@ -455,8 +455,9 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219/go.mod h1:/X8TswGSh1pIozq4ZwCfxS0WA5JGXguxk94ar/4c87Y= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= @@ -691,6 +692,8 @@ github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvW github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.13.0 h1:2T7tUoQrQT+fQWdaY5rjWztFGAFwbGD04iPJg90ZiOs= +github.com/klauspost/compress v1.13.0/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/kolo/xmlrpc v0.0.0-20200310150728-e0350524596b h1:DzHy0GlWeF0KAglaTMY7Q+khIFoG8toHP+wLFBVBQJc= github.com/kolo/xmlrpc v0.0.0-20200310150728-e0350524596b/go.mod h1:o03bZfuBwAXHetKXuInt4S7omeXUu62/A845kiycsSQ= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -1133,8 +1136,6 @@ github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDW github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/traefik/gziphandler v1.1.2-0.20210212101304-175e0fad6888 h1:GMY0C+M/w8xO+/NP3Kq6sroMd+z2KbbdVr1K8o2NLHk= -github.com/traefik/gziphandler v1.1.2-0.20210212101304-175e0fad6888/go.mod h1:sLqwoN03tkluITKL+lPEZbfsJQU2suYoKbrR/HeV9aM= github.com/traefik/paerser v0.1.4 h1:/IXjV04Gf6di51H8Jl7jyS3OylsLjIasrwXIIwj1aT8= github.com/traefik/paerser v0.1.4/go.mod h1:FIdQ4Y92ulQUGSeZgxchtBKEcLw1o551PMNg9PoIq/4= github.com/traefik/yaegi v0.9.20 h1:G05/iDMD3cepEr9QsVGpmCc3N8FQCdUWA3Vlff2WgbA= diff --git a/pkg/middlewares/compress/compress.go b/pkg/middlewares/compress/compress.go index 42b0d01ea..e6ddeb75d 100644 --- a/pkg/middlewares/compress/compress.go +++ b/pkg/middlewares/compress/compress.go @@ -6,8 +6,8 @@ import ( "mime" "net/http" + "github.com/klauspost/compress/gzhttp" "github.com/opentracing/opentracing-go/ext" - "github.com/traefik/gziphandler" "github.com/traefik/traefik/v2/pkg/config/dynamic" "github.com/traefik/traefik/v2/pkg/log" "github.com/traefik/traefik/v2/pkg/middlewares" @@ -61,10 +61,10 @@ func (c *compress) GetTracingInformation() (string, ext.SpanKindEnum) { } func (c *compress) gzipHandler(ctx context.Context) http.Handler { - wrapper, err := gziphandler.GzipHandlerWithOpts( - gziphandler.ContentTypeExceptions(c.excludes), - gziphandler.CompressionLevel(gzip.DefaultCompression), - gziphandler.MinSize(gziphandler.DefaultMinSize)) + wrapper, err := gzhttp.NewWrapper( + gzhttp.ExceptContentTypes(c.excludes), + gzhttp.CompressionLevel(gzip.DefaultCompression), + gzhttp.MinSize(gzhttp.DefaultMinSize)) if err != nil { log.FromContext(ctx).Error(err) } diff --git a/pkg/middlewares/compress/compress_test.go b/pkg/middlewares/compress/compress_test.go index f3b01ed0e..9dd4c8124 100644 --- a/pkg/middlewares/compress/compress_test.go +++ b/pkg/middlewares/compress/compress_test.go @@ -7,9 +7,9 @@ import ( "net/http/httptest" "testing" + "github.com/klauspost/compress/gzhttp" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/traefik/gziphandler" "github.com/traefik/traefik/v2/pkg/config/dynamic" "github.com/traefik/traefik/v2/pkg/testhelpers" ) @@ -26,13 +26,14 @@ func TestShouldCompressWhenNoContentEncodingHeader(t *testing.T) { req := testhelpers.MustNewRequest(http.MethodGet, "http://localhost", nil) req.Header.Add(acceptEncodingHeader, gzipValue) - baseBody := generateBytes(gziphandler.DefaultMinSize) + baseBody := generateBytes(gzhttp.DefaultMinSize) next := http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { _, err := rw.Write(baseBody) assert.NoError(t, err) }) - handler := &compress{next: next} + handler, err := New(context.Background(), next, dynamic.Compress{}, "testing") + require.NoError(t, err) rw := httptest.NewRecorder() handler.ServeHTTP(rw, req) @@ -49,7 +50,7 @@ func TestShouldNotCompressWhenContentEncodingHeader(t *testing.T) { req := testhelpers.MustNewRequest(http.MethodGet, "http://localhost", nil) req.Header.Add(acceptEncodingHeader, gzipValue) - fakeCompressedBody := generateBytes(gziphandler.DefaultMinSize) + fakeCompressedBody := generateBytes(gzhttp.DefaultMinSize) next := http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { rw.Header().Add(contentEncodingHeader, gzipValue) rw.Header().Add(varyHeader, acceptEncodingHeader) @@ -58,7 +59,8 @@ func TestShouldNotCompressWhenContentEncodingHeader(t *testing.T) { http.Error(rw, err.Error(), http.StatusInternalServerError) } }) - handler := &compress{next: next} + handler, err := New(context.Background(), next, dynamic.Compress{}, "testing") + require.NoError(t, err) rw := httptest.NewRecorder() handler.ServeHTTP(rw, req) @@ -72,14 +74,15 @@ func TestShouldNotCompressWhenContentEncodingHeader(t *testing.T) { func TestShouldNotCompressWhenNoAcceptEncodingHeader(t *testing.T) { req := testhelpers.MustNewRequest(http.MethodGet, "http://localhost", nil) - fakeBody := generateBytes(gziphandler.DefaultMinSize) + fakeBody := generateBytes(gzhttp.DefaultMinSize) next := http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { _, err := rw.Write(fakeBody) if err != nil { http.Error(rw, err.Error(), http.StatusInternalServerError) } }) - handler := &compress{next: next} + handler, err := New(context.Background(), next, dynamic.Compress{}, "testing") + require.NoError(t, err) rw := httptest.NewRecorder() handler.ServeHTTP(rw, req) @@ -89,7 +92,7 @@ func TestShouldNotCompressWhenNoAcceptEncodingHeader(t *testing.T) { } func TestShouldNotCompressWhenSpecificContentType(t *testing.T) { - baseBody := generateBytes(gziphandler.DefaultMinSize) + baseBody := generateBytes(gzhttp.DefaultMinSize) testCases := []struct { desc string @@ -190,7 +193,9 @@ func TestIntegrationShouldNotCompress(t *testing.T) { for _, test := range testCases { t.Run(test.name, func(t *testing.T) { - compress := &compress{next: test.handler} + compress, err := New(context.Background(), test.handler, dynamic.Compress{}, "testing") + require.NoError(t, err) + ts := httptest.NewServer(compress) defer ts.Close() @@ -223,7 +228,9 @@ func TestShouldWriteHeaderWhenFlush(t *testing.T) { http.Error(rw, err.Error(), http.StatusInternalServerError) } }) - handler := &compress{next: next} + handler, err := New(context.Background(), next, dynamic.Compress{}, "testing") + require.NoError(t, err) + ts := httptest.NewServer(handler) defer ts.Close() @@ -272,7 +279,9 @@ func TestIntegrationShouldCompress(t *testing.T) { for _, test := range testCases { t.Run(test.name, func(t *testing.T) { - compress := &compress{next: test.handler} + compress, err := New(context.Background(), test.handler, dynamic.Compress{}, "testing") + require.NoError(t, err) + ts := httptest.NewServer(compress) defer ts.Close() @@ -296,6 +305,86 @@ func TestIntegrationShouldCompress(t *testing.T) { } } +func BenchmarkCompress(b *testing.B) { + testCases := []struct { + name string + parallel bool + size int + }{ + { + name: "2k", + size: 2048, + }, + { + name: "20k", + size: 20480, + }, + { + name: "100k", + size: 102400, + }, + { + name: "2k parallel", + parallel: true, + size: 2048, + }, + { + name: "20k parallel", + parallel: true, + size: 20480, + }, + { + name: "100k parallel", + parallel: true, + size: 102400, + }, + } + + for _, test := range testCases { + b.Run(test.name, func(b *testing.B) { + baseBody := generateBytes(test.size) + + next := http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { + _, err := rw.Write(baseBody) + assert.NoError(b, err) + }) + handler, _ := New(context.Background(), next, dynamic.Compress{}, "testing") + + req, _ := http.NewRequest("GET", "/whatever", nil) + req.Header.Set("Accept-Encoding", "gzip") + + b.ReportAllocs() + b.SetBytes(int64(test.size)) + if test.parallel { + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + runBenchmark(b, req, handler) + } + }) + return + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + runBenchmark(b, req, handler) + } + }) + } +} + +func runBenchmark(b *testing.B, req *http.Request, handler http.Handler) { + b.Helper() + + res := httptest.NewRecorder() + handler.ServeHTTP(res, req) + if code := res.Code; code != 200 { + b.Fatalf("Expected 200 but got %d", code) + } + + assert.Equal(b, gzipValue, res.Header().Get(contentEncodingHeader)) +} + func generateBytes(length int) []byte { var value []byte for i := 0; i < length; i++ {