From c09febfffc2f78bf151affbae2e91d27f0a5c16d Mon Sep 17 00:00:00 2001 From: ryarnyah Date: Wed, 23 May 2018 16:46:04 +0200 Subject: [PATCH] Make accesslogs.logTheRoundTrip async to get lost performance --- docs/configuration/logs.md | 14 ++++++++++ middlewares/accesslog/logger.go | 41 ++++++++++++++++++++++++---- middlewares/accesslog/logger_test.go | 15 ++++++++++ types/logs.go | 9 +++--- 4 files changed, 70 insertions(+), 9 deletions(-) diff --git a/docs/configuration/logs.md b/docs/configuration/logs.md index 45c8aca58..bd76c4689 100644 --- a/docs/configuration/logs.md +++ b/docs/configuration/logs.md @@ -124,6 +124,20 @@ filePath = "/path/to/access.log" format = "json" ``` +To write the logs in async, specify `bufferingSize` as the format (must be >0): +```toml +[accessLog] +filePath = "/path/to/access.log" +# Buffering Size +# +# Optional +# Default: 0 +# +# Number of access log lines to process in a buffered way. +# +bufferingSize = 100 +``` + To filter logs you can specify a set of filters which are logically "OR-connected". Thus, specifying multiple filters will keep more access logs than specifying only one: ```toml [accessLog] diff --git a/middlewares/accesslog/logger.go b/middlewares/accesslog/logger.go index f0480fc4d..f8e3756b8 100644 --- a/middlewares/accesslog/logger.go +++ b/middlewares/accesslog/logger.go @@ -31,6 +31,12 @@ const ( JSONFormat = "json" ) +type logHandlerParams struct { + logDataTable *LogData + crr *captureRequestReader + crw *captureResponseWriter +} + // LogHandler will write each request and its response to the access log. type LogHandler struct { config *types.AccessLog @@ -38,6 +44,8 @@ type LogHandler struct { file *os.File mu sync.Mutex httpCodeRanges types.HTTPCodeRanges + logHandlerChan chan logHandlerParams + wg sync.WaitGroup } // NewLogHandler creates a new LogHandler @@ -50,6 +58,7 @@ func NewLogHandler(config *types.AccessLog) (*LogHandler, error) { } file = f } + logHandlerChan := make(chan logHandlerParams, config.BufferingSize) var formatter logrus.Formatter @@ -70,9 +79,10 @@ func NewLogHandler(config *types.AccessLog) (*LogHandler, error) { } logHandler := &LogHandler{ - config: config, - logger: logger, - file: file, + config: config, + logger: logger, + file: file, + logHandlerChan: logHandlerChan, } if config.Filters != nil { @@ -83,6 +93,16 @@ func NewLogHandler(config *types.AccessLog) (*LogHandler, error) { } } + if config.BufferingSize > 0 { + logHandler.wg.Add(1) + go func() { + defer logHandler.wg.Done() + for handlerParams := range logHandler.logHandlerChan { + logHandler.logTheRoundTrip(handlerParams.logDataTable, handlerParams.crr, handlerParams.crw) + } + }() + } + return logHandler, nil } @@ -162,11 +182,22 @@ func (l *LogHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request, next h core[ClientUsername] = usernameIfPresent(reqWithDataTable.URL) logDataTable.DownstreamResponse = crw.Header() - l.logTheRoundTrip(logDataTable, crr, crw) + + if l.config.BufferingSize > 0 { + l.logHandlerChan <- logHandlerParams{ + logDataTable: logDataTable, + crr: crr, + crw: crw, + } + } else { + l.logTheRoundTrip(logDataTable, crr, crw) + } } -// Close closes the Logger (i.e. the file etc). +// Close closes the Logger (i.e. the file, drain logHandlerChan, etc). func (l *LogHandler) Close() error { + close(l.logHandlerChan) + l.wg.Wait() return l.file.Close() } diff --git a/middlewares/accesslog/logger_test.go b/middlewares/accesslog/logger_test.go index 1dd93ec7f..d4bce0cf1 100644 --- a/middlewares/accesslog/logger_test.go +++ b/middlewares/accesslog/logger_test.go @@ -130,6 +130,21 @@ func TestLoggerCLF(t *testing.T) { assertValidLogData(t, expectedLog, logData) } +func TestAsyncLoggerCLF(t *testing.T) { + tmpDir := createTempDir(t, CommonFormat) + defer os.RemoveAll(tmpDir) + + logFilePath := filepath.Join(tmpDir, logFileNameSuffix) + config := &types.AccessLog{FilePath: logFilePath, Format: CommonFormat, BufferingSize: 1024} + doLogging(t, config) + + logData, err := ioutil.ReadFile(logFilePath) + require.NoError(t, err) + + expectedLog := ` TestHost - TestUser [13/Apr/2016:07:14:19 -0700] "POST testpath HTTP/0.0" 123 12 "testReferer" "testUserAgent" 1 "testFrontend" "http://127.0.0.1/testBackend" 1ms` + assertValidLogData(t, expectedLog, logData) +} + func assertString(exp string) func(t *testing.T, actual interface{}) { return func(t *testing.T, actual interface{}) { t.Helper() diff --git a/types/logs.go b/types/logs.go index 610bc1e23..22266bff8 100644 --- a/types/logs.go +++ b/types/logs.go @@ -22,10 +22,11 @@ type TraefikLog struct { // AccessLog holds the configuration settings for the access logger (middlewares/accesslog). type AccessLog struct { - FilePath string `json:"file,omitempty" description:"Access log file path. Stdout is used when omitted or empty" export:"true"` - Format string `json:"format,omitempty" description:"Access log format: json | common" export:"true"` - Filters *AccessLogFilters `json:"filters,omitempty" description:"Access log filters, used to keep only specific access logs" export:"true"` - Fields *AccessLogFields `json:"fields,omitempty" description:"AccessLogFields" export:"true"` + FilePath string `json:"file,omitempty" description:"Access log file path. Stdout is used when omitted or empty" export:"true"` + Format string `json:"format,omitempty" description:"Access log format: json | common" export:"true"` + Filters *AccessLogFilters `json:"filters,omitempty" description:"Access log filters, used to keep only specific access logs" export:"true"` + Fields *AccessLogFields `json:"fields,omitempty" description:"AccessLogFields" export:"true"` + BufferingSize int64 `json:"bufferingSize,omitempty" description:"Number of access log lines to process in a buffered way. Default 0." export:"true"` } // AccessLogFilters holds filters configuration