diff --git a/Gopkg.lock b/Gopkg.lock index 85b09e26e..c3478844c 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -840,12 +840,6 @@ packages = ["."] revision = "57fdcb988a5c543893cc61bce354a6e24ab70022" -[[projects]] - name = "github.com/mattn/go-shellwords" - packages = ["."] - revision = "02e3cf038dcea8290e44424da473dd12be796a8a" - version = "v1.0.3" - [[projects]] branch = "master" name = "github.com/matttproud/golang_protobuf_extensions" @@ -1574,6 +1568,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "5bb840e4352562c416f2f2a3ba8fb7781b72f79fcff8b963d98140a005a2ca3a" + inputs-digest = "2fca312eff66fbc2aa41319f67d2c78cf8117bd6b5f7791cf20bddb7fdb7e0af" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index ceb4b5b15..fe44d4e58 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -119,10 +119,6 @@ branch = "master" name = "github.com/abronan/valkeyrie" -[[constraint]] - name = "github.com/mattn/go-shellwords" - version = "1.0.3" - [[constraint]] name = "github.com/mesosphere/mesos-dns" source = "https://github.com/containous/mesos-dns.git" diff --git a/cmd/configuration.go b/cmd/configuration.go index cb73adbf8..6ca1bb108 100644 --- a/cmd/configuration.go +++ b/cmd/configuration.go @@ -188,6 +188,13 @@ func NewTraefikDefaultPointersConfiguration() *TraefikConfiguration { defaultAccessLog := types.AccessLog{ Format: accesslog.CommonFormat, FilePath: "", + Filters: &types.AccessLogFilters{}, + Fields: &types.AccessLogFields{ + DefaultMode: types.AccessLogKeep, + Headers: &types.FieldHeaders{ + DefaultMode: types.AccessLogKeep, + }, + }, } // default HealthCheckConfig diff --git a/cmd/traefik/traefik.go b/cmd/traefik/traefik.go index 02f98ca16..6b5090465 100644 --- a/cmd/traefik/traefik.go +++ b/cmd/traefik/traefik.go @@ -69,6 +69,9 @@ Complete documentation is available at https://traefik.io`, f.AddParser(reflect.TypeOf(ecs.Clusters{}), &ecs.Clusters{}) f.AddParser(reflect.TypeOf([]types.Domain{}), &types.Domains{}) f.AddParser(reflect.TypeOf(types.Buckets{}), &types.Buckets{}) + f.AddParser(reflect.TypeOf(types.StatusCodes{}), &types.StatusCodes{}) + f.AddParser(reflect.TypeOf(types.FieldNames{}), &types.FieldNames{}) + f.AddParser(reflect.TypeOf(types.FieldHeaderNames{}), &types.FieldHeaderNames{}) // add commands f.AddCommand(cmdVersion.NewCmd()) diff --git a/docs/configuration/commons.md b/docs/configuration/commons.md index 234a3a347..ccab6b32e 100644 --- a/docs/configuration/commons.md +++ b/docs/configuration/commons.md @@ -154,89 +154,6 @@ constraints = ["tag==api", "tag!=v*-beta"] ``` -## Logs Definition - -### Traefik logs - -```toml -# Traefik logs file -# If not defined, logs to stdout -# -# DEPRECATED - see [traefikLog] lower down -# In case both traefikLogsFile and traefikLog.filePath are specified, the latter will take precedence. -# Optional -# -traefikLogsFile = "log/traefik.log" - -# Log level -# -# Optional -# Default: "ERROR" -# -# Accepted values, in order of severity: "DEBUG", "INFO", "WARN", "ERROR", "FATAL", "PANIC" -# Messages at and above the selected level will be logged. -# -logLevel = "ERROR" -``` - -## Traefik Logs - -By default the Traefik log is written to stdout in text format. - -To write the logs into a logfile specify the `filePath`. -```toml -[traefikLog] - filePath = "/path/to/traefik.log" -``` - -To write JSON format logs, specify `json` as the format: -```toml -[traefikLog] - filePath = "/path/to/traefik.log" - format = "json" -``` - -### Access Logs - -Access logs are written when `[accessLog]` is defined. -By default it will write to stdout and produce logs in the textual Common Log Format (CLF), extended with additional fields. - -To enable access logs using the default settings just add the `[accessLog]` entry. -```toml -[accessLog] -``` - -To write the logs into a logfile specify the `filePath`. -```toml -[accessLog] -filePath = "/path/to/access.log" -``` - -To write JSON format logs, specify `json` as the format: -```toml -[accessLog] -filePath = "/path/to/access.log" -format = "json" -``` - -Deprecated way (before 1.4): -```toml -# Access logs file -# -# DEPRECATED - see [accessLog] lower down -# -accessLogsFile = "log/access.log" -``` - -### Log Rotation - -Traefik will close and reopen its log files, assuming they're configured, on receipt of a USR1 signal. -This allows the logs to be rotated and processed by an external program, such as `logrotate`. - -!!! note - This does not work on Windows due to the lack of USR signals. - - ## Custom Error pages Custom error pages can be returned, in lieu of the default, according to frontend-configured ranges of HTTP Status codes. @@ -302,7 +219,7 @@ These can "burst" up to 10 and 200 in each period respectively. ## Buffering In some cases request/buffering can be enabled for a specific backend. -By enabling this, Træfik will read the entire request into memory (possibly buffering large requests into disk) and will reject requests that are over a specified limit. +By enabling this, Træfik will read the entire request into memory (possibly buffering large requests into disk) and will reject requests that are over a specified limit. This may help services deal with large data (multipart/form-data for example) more efficiently and should minimise time spent when sending data to a backend server. For more information please check [oxy/buffer](http://godoc.org/github.com/vulcand/oxy/buffer) documentation. @@ -315,8 +232,8 @@ Example configuration: [backends.backend1.buffering] maxRequestBodyBytes = 10485760 memRequestBodyBytes = 2097152 - maxResponseBodyBytes = 10485760 - memResponseBodyBytes = 2097152 + maxResponseBodyBytes = 10485760 + memResponseBodyBytes = 2097152 retryExpression = "IsNetworkError() && Attempts() <= 2" ``` diff --git a/docs/configuration/logs.md b/docs/configuration/logs.md new file mode 100644 index 000000000..b7b233f95 --- /dev/null +++ b/docs/configuration/logs.md @@ -0,0 +1,243 @@ +# Logs Definition + +## Reference + +### TOML + +```toml +logLevel = "INFO" + +[traefikLog] + filePath = "/path/to/traefik.log" + format = "json" + +[accessLog] + filePath = "/path/to/access.log" + format = "json" + + [accessLog.filters] + statusCodes = ["200", "300-302"] + + [accessLog.fields] + defaultMode = "keep" + [accessLog.fields.names] + "ClientUsername" = "drop" + # ... + + [accessLog.fields.headers] + defaultMode = "keep" + [accessLog.fields.headers.names] + "User-Agent" = "redact" + "Authorization" = "drop" + "Content-Type" = "keep" + # ... +``` + +### CLI + +For more information about the CLI, see the documentation about [Traefik command](/basics/#traefik). + +```shell +--logLevel="DEBUG" +--traefikLog.filePath="/path/to/traefik.log" +--traefikLog.format="json" +--accessLog.filePath="/path/to/access.log" +--accessLog.format="json" +--accessLog.filters.statusCodes="200,300-302" +--accessLog.fields.defaultMode="keep" +--accessLog.fields.names="Username=drop Hostname=drop" +--accessLog.fields.headers.defaultMode="keep" +--accessLog.fields.headers.names="User-Agent=redact Authorization=drop Content-Type=keep" +``` + + +## Traefik Logs + +By default the Traefik log is written to stdout in text format. + +To write the logs into a log file specify the `filePath`: +```toml +[traefikLog] + filePath = "/path/to/traefik.log" +``` + +To write JSON format logs, specify `json` as the format: +```toml +[traefikLog] + filePath = "/path/to/traefik.log" + format = "json" +``` + + +Deprecated way (before 1.4): + +!!! danger "DEPRECATED" + `traefikLogsFile` is deprecated, use [traefikLog](/configuration/logs/#traefik-logs) instead. + +```toml +# Traefik logs file +# If not defined, logs to stdout +# +# DEPRECATED - see [traefikLog] lower down +# In case both traefikLogsFile and traefikLog.filePath are specified, the latter will take precedence. +# Optional +# +traefikLogsFile = "log/traefik.log" +``` + +To customize the log level: +```toml +# Log level +# +# Optional +# Default: "ERROR" +# +# Accepted values, in order of severity: "DEBUG", "INFO", "WARN", "ERROR", "FATAL", "PANIC" +# Messages at and above the selected level will be logged. +# +logLevel = "ERROR" +``` + + +## Access Logs + +Access logs are written when `[accessLog]` is defined. +By default it will write to stdout and produce logs in the textual Common Log Format (CLF), extended with additional fields. + +To enable access logs using the default settings just add the `[accessLog]` entry: +```toml +[accessLog] +``` + +To write the logs into a log file specify the `filePath`: +```toml +[accessLog] +filePath = "/path/to/access.log" +``` + +To write JSON format logs, specify `json` as the format: +```toml +[accessLog] +filePath = "/path/to/access.log" +format = "json" +``` + +To filter logs by status code: +```toml +[accessLog] +filePath = "/path/to/access.log" +format = "json" + + [accessLog.filters] + + # statusCodes keep only access logs with status codes in the specified range + # + # Optional + # Default: [] + # + statusCodes = ["200", "300-302"] +``` + +To customize logs format: +```toml +[accessLog] +filePath = "/path/to/access.log" +format = "json" + + [accessLog.filters] + + # statusCodes keep only access logs with status codes in the specified range + # + # Optional + # Default: [] + # + statusCodes = ["200", "300-302"] + + [accessLog.fields] + + # defaultMode + # + # Optional + # Default: "keep" + # + # Accepted values "keep", "drop" + # + defaultMode = "keep" + + # Fields map which is used to override fields defaultMode + [accessLog.fields.names] + "ClientUsername" = "drop" + # ... + + [accessLog.fields.headers] + # defaultMode + # + # Optional + # Default: "keep" + # + # Accepted values "keep", "drop", "redact" + # + defaultMode = "keep" + # Fields map which is used to override headers defaultMode + [accessLog.fields.headers.names] + "User-Agent" = "redact" + "Authorization" = "drop" + "Content-Type" = "keep" + # ... +``` + +#### List of all available fields + +```ini +StartUTC +StartLocal +Duration +FrontendName +BackendName +BackendURL +BackendAddr +ClientAddr +ClientHost +ClientPort +ClientUsername +RequestAddr +RequestHost +RequestPort +RequestMethod +RequestPath +RequestProtocol +RequestLine +RequestContentSize +OriginDuration +OriginContentSize +OriginStatus +OriginStatusLine +DownstreamStatus +DownstreamStatusLine +DownstreamContentSize +RequestCount +GzipRatio +Overhead +RetryAttempts +``` + +Deprecated way (before 1.4): + +!!! danger "DEPRECATED" + `accessLogsFile` is deprecated, use [accessLog](/configuration/logs/#access-logs) instead. + +```toml +# Access logs file +# +# DEPRECATED - see [accessLog] +# +accessLogsFile = "log/access.log" +``` + +## Log Rotation + +Traefik will close and reopen its log files, assuming they're configured, on receipt of a USR1 signal. +This allows the logs to be rotated and processed by an external program, such as `logrotate`. + +!!! note + This does not work on Windows due to the lack of USR signals. diff --git a/integration/access_log_test.go b/integration/access_log_test.go index d12abd74c..19aef0473 100644 --- a/integration/access_log_test.go +++ b/integration/access_log_test.go @@ -12,8 +12,8 @@ import ( "time" "github.com/containous/traefik/integration/try" + "github.com/containous/traefik/middlewares/accesslog" "github.com/go-check/check" - "github.com/mattn/go-shellwords" checker "github.com/vdemeester/shakers" ) @@ -26,11 +26,11 @@ const ( type AccessLogSuite struct{ BaseSuite } type accessLogValue struct { - formatOnly bool - code string - user string - value string - backendName string + formatOnly bool + code string + user string + frontendName string + backendName string } func (s *AccessLogSuite) SetUpSuite(c *check.C) { @@ -99,11 +99,11 @@ func (s *AccessLogSuite) TestAccessLogAuthFrontend(c *check.C) { expected := []accessLogValue{ { - formatOnly: false, - code: "401", - user: "-", - value: "Auth for frontend-Host-frontend-auth-docker-local", - backendName: "-", + formatOnly: false, + code: "401", + user: "-", + frontendName: "Auth for frontend-Host-frontend-auth-docker-local", + backendName: "-", }, } @@ -147,11 +147,11 @@ func (s *AccessLogSuite) TestAccessLogAuthEntrypoint(c *check.C) { expected := []accessLogValue{ { - formatOnly: false, - code: "401", - user: "-", - value: "Auth for entrypoint", - backendName: "-", + formatOnly: false, + code: "401", + user: "-", + frontendName: "Auth for entrypoint", + backendName: "-", }, } @@ -195,11 +195,11 @@ func (s *AccessLogSuite) TestAccessLogAuthEntrypointSuccess(c *check.C) { expected := []accessLogValue{ { - formatOnly: false, - code: "200", - user: "test", - value: "Host-entrypoint-auth-docker", - backendName: "http://172.17.0", + formatOnly: false, + code: "200", + user: "test", + frontendName: "Host-entrypoint-auth-docker", + backendName: "http://172.17.0", }, } @@ -243,18 +243,18 @@ func (s *AccessLogSuite) TestAccessLogDigestAuthEntrypoint(c *check.C) { expected := []accessLogValue{ { - formatOnly: false, - code: "401", - user: "-", - value: "Auth for entrypoint", - backendName: "-", + formatOnly: false, + code: "401", + user: "-", + frontendName: "Auth for entrypoint", + backendName: "-", }, { - formatOnly: false, - code: "200", - user: "test", - value: "Host-entrypoint-digest-auth-docker", - backendName: "http://172.17.0", + formatOnly: false, + code: "200", + user: "test", + frontendName: "Host-entrypoint-digest-auth-docker", + backendName: "http://172.17.0", }, } @@ -351,11 +351,11 @@ func (s *AccessLogSuite) TestAccessLogEntrypointRedirect(c *check.C) { expected := []accessLogValue{ { - formatOnly: false, - code: "302", - user: "-", - value: "entrypoint redirect for frontend-", - backendName: "-", + formatOnly: false, + code: "302", + user: "-", + frontendName: "entrypoint redirect for frontend-", + backendName: "-", }, { formatOnly: true, @@ -401,11 +401,11 @@ func (s *AccessLogSuite) TestAccessLogFrontendRedirect(c *check.C) { expected := []accessLogValue{ { - formatOnly: false, - code: "302", - user: "-", - value: "frontend redirect for frontend-Path-", - backendName: "-", + formatOnly: false, + code: "302", + user: "-", + frontendName: "frontend redirect for frontend-Path-", + backendName: "-", }, { formatOnly: true, @@ -457,11 +457,11 @@ func (s *AccessLogSuite) TestAccessLogRateLimit(c *check.C) { formatOnly: true, }, { - formatOnly: false, - code: "429", - user: "-", - value: "rate limit for frontend-Host-ratelimit", - backendName: "/", + formatOnly: false, + code: "429", + user: "-", + frontendName: "rate limit for frontend-Host-ratelimit", + backendName: "/", }, } @@ -508,11 +508,11 @@ func (s *AccessLogSuite) TestAccessLogBackendNotFound(c *check.C) { expected := []accessLogValue{ { - formatOnly: false, - code: "404", - user: "-", - value: "backend not found", - backendName: "/", + formatOnly: false, + code: "404", + user: "-", + frontendName: "backend not found", + backendName: "/", }, } @@ -553,11 +553,11 @@ func (s *AccessLogSuite) TestAccessLogEntrypointWhitelist(c *check.C) { expected := []accessLogValue{ { - formatOnly: false, - code: "403", - user: "-", - value: "ipwhitelister for entrypoint httpWhitelistReject", - backendName: "-", + formatOnly: false, + code: "403", + user: "-", + frontendName: "ipwhitelister for entrypoint httpWhitelistReject", + backendName: "-", }, } @@ -600,11 +600,11 @@ func (s *AccessLogSuite) TestAccessLogFrontendWhitelist(c *check.C) { expected := []accessLogValue{ { - formatOnly: false, - code: "403", - user: "-", - value: "ipwhitelister for frontend-Host-frontend-whitelist", - backendName: "-", + formatOnly: false, + code: "403", + user: "-", + frontendName: "ipwhitelister for frontend-Host-frontend-whitelist", + backendName: "-", }, } @@ -714,28 +714,28 @@ func checkTraefikStarted(c *check.C) []byte { } func CheckAccessLogFormat(c *check.C, line string, i int) { - tokens, err := shellwords.Parse(line) + results, err := accesslog.ParseAccessLog(line) c.Assert(err, checker.IsNil) - c.Assert(tokens, checker.HasLen, 14) - c.Assert(tokens[6], checker.Matches, `^(-|\d{3})$`) - c.Assert(tokens[10], checker.Equals, fmt.Sprintf("%d", i+1)) - c.Assert(tokens[11], checker.HasPrefix, "Host-") - c.Assert(tokens[12], checker.HasPrefix, "http://") - c.Assert(tokens[13], checker.Matches, `^\d+ms$`) + c.Assert(results, checker.HasLen, 14) + c.Assert(results[accesslog.OriginStatus], checker.Matches, `^(-|\d{3})$`) + c.Assert(results[accesslog.RequestCount], checker.Equals, fmt.Sprintf("%d", i+1)) + c.Assert(results[accesslog.FrontendName], checker.HasPrefix, "\"Host-") + c.Assert(results[accesslog.BackendURL], checker.HasPrefix, "\"http://") + c.Assert(results[accesslog.Duration], checker.Matches, `^\d+ms$`) } func checkAccessLogExactValues(c *check.C, line string, i int, v accessLogValue) { - tokens, err := shellwords.Parse(line) + results, err := accesslog.ParseAccessLog(line) c.Assert(err, checker.IsNil) - c.Assert(tokens, checker.HasLen, 14) + c.Assert(results, checker.HasLen, 14) if len(v.user) > 0 { - c.Assert(tokens[2], checker.Equals, v.user) + c.Assert(results[accesslog.ClientUsername], checker.Equals, v.user) } - c.Assert(tokens[6], checker.Equals, v.code) - c.Assert(tokens[10], checker.Equals, fmt.Sprintf("%d", i+1)) - c.Assert(tokens[11], checker.HasPrefix, v.value) - c.Assert(tokens[12], checker.HasPrefix, v.backendName) - c.Assert(tokens[13], checker.Matches, `^\d+ms$`) + c.Assert(results[accesslog.OriginStatus], checker.Equals, v.code) + c.Assert(results[accesslog.RequestCount], checker.Equals, fmt.Sprintf("%d", i+1)) + c.Assert(results[accesslog.FrontendName], checker.Matches, `^"?`+v.frontendName+`.*$`) + c.Assert(results[accesslog.BackendURL], checker.Matches, `^"?`+v.backendName+`.*$`) + c.Assert(results[accesslog.Duration], checker.Matches, `^\d+ms$`) } func waitForTraefik(c *check.C, containerName string) { diff --git a/middlewares/accesslog/logdata.go b/middlewares/accesslog/logdata.go index 55c364b40..8a144ce0e 100644 --- a/middlewares/accesslog/logdata.go +++ b/middlewares/accesslog/logdata.go @@ -44,6 +44,10 @@ const ( RequestLine = "RequestLine" // RequestContentSize is the map key used for the number of bytes in the request entity (a.k.a. body) sent by the client. RequestContentSize = "RequestContentSize" + // RequestRefererHeader is the Referer header in the request + RequestRefererHeader = "request_Referer" + // RequestUserAgentHeader is the User-Agent header in the request + RequestUserAgentHeader = "request_User-Agent" // OriginDuration is the map key used for the time taken by the origin server ('upstream') to return its response. OriginDuration = "OriginDuration" // OriginContentSize is the map key used for the content length specified by the origin server, or 0 if unspecified. diff --git a/middlewares/accesslog/logger.go b/middlewares/accesslog/logger.go index 620e66e6f..6bdb07283 100644 --- a/middlewares/accesslog/logger.go +++ b/middlewares/accesslog/logger.go @@ -12,6 +12,7 @@ import ( "sync/atomic" "time" + "github.com/containous/traefik/log" "github.com/containous/traefik/types" "github.com/sirupsen/logrus" ) @@ -32,10 +33,12 @@ const ( // LogHandler will write each request and its response to the access log. type LogHandler struct { - logger *logrus.Logger - file *os.File - filePath string - mu sync.Mutex + logger *logrus.Logger + file *os.File + filePath string + mu sync.Mutex + httpCodeRanges types.HTTPCodeRanges + fields *types.AccessLogFields } // NewLogHandler creates a new LogHandler @@ -66,7 +69,24 @@ func NewLogHandler(config *types.AccessLog) (*LogHandler, error) { Hooks: make(logrus.LevelHooks), Level: logrus.InfoLevel, } - return &LogHandler{logger: logger, file: file, filePath: config.FilePath}, nil + + logHandler := &LogHandler{ + logger: logger, + file: file, + filePath: config.FilePath, + fields: config.Fields, + } + + if config.Filters != nil { + httpCodeRanges, err := types.NewHTTPCodeRanges(config.Filters.StatusCodes) + if err != nil { + log.Errorf("Failed to create new HTTP code ranges: %s", err) + } else if httpCodeRanges != nil { + logHandler.httpCodeRanges = httpCodeRanges + } + } + + return logHandler, nil } func openAccessLogFile(filePath string) (*os.File, error) { @@ -198,45 +218,65 @@ func (l *LogHandler) logTheRoundTrip(logDataTable *LogData, crr *captureRequestR } core[DownstreamStatus] = crw.Status() - core[DownstreamStatusLine] = fmt.Sprintf("%03d %s", crw.Status(), http.StatusText(crw.Status())) - core[DownstreamContentSize] = crw.Size() - if original, ok := core[OriginContentSize]; ok { - o64 := original.(int64) - if o64 != crw.Size() && 0 != crw.Size() { - core[GzipRatio] = float64(o64) / float64(crw.Size()) + + if l.keepAccessLog(crw.Status()) { + core[DownstreamStatusLine] = fmt.Sprintf("%03d %s", crw.Status(), http.StatusText(crw.Status())) + core[DownstreamContentSize] = crw.Size() + if original, ok := core[OriginContentSize]; ok { + o64 := original.(int64) + if o64 != crw.Size() && 0 != crw.Size() { + core[GzipRatio] = float64(o64) / float64(crw.Size()) + } + } + + // n.b. take care to perform time arithmetic using UTC to avoid errors at DST boundaries + total := time.Now().UTC().Sub(core[StartUTC].(time.Time)) + core[Duration] = total + core[Overhead] = total + if origin, ok := core[OriginDuration]; ok { + core[Overhead] = total - origin.(time.Duration) + } + + fields := logrus.Fields{} + + for k, v := range logDataTable.Core { + if l.fields.Keep(k) { + fields[k] = v + } + } + + l.redactHeaders(logDataTable.Request, fields, "request_") + l.redactHeaders(logDataTable.OriginResponse, fields, "origin_") + l.redactHeaders(logDataTable.DownstreamResponse, fields, "downstream_") + + l.mu.Lock() + defer l.mu.Unlock() + l.logger.WithFields(fields).Println() + } +} + +func (l *LogHandler) redactHeaders(headers http.Header, fields logrus.Fields, prefix string) { + for k := range headers { + v := l.fields.KeepHeader(k) + if v == types.AccessLogKeep { + fields[prefix+k] = headers.Get(k) + } else if v == types.AccessLogRedact { + fields[prefix+k] = "REDACTED" } } +} - // n.b. take care to perform time arithmetic using UTC to avoid errors at DST boundaries - total := time.Now().UTC().Sub(core[StartUTC].(time.Time)) - core[Duration] = total - if origin, ok := core[OriginDuration]; ok { - core[Overhead] = total - origin.(time.Duration) - } else { - core[Overhead] = total +func (l *LogHandler) keepAccessLog(status int) bool { + if l.httpCodeRanges == nil { + return true } - fields := logrus.Fields{} - - for k, v := range logDataTable.Core { - fields[k] = v + for _, block := range l.httpCodeRanges { + if status >= block[0] && status <= block[1] { + return true + } } - - for k := range logDataTable.Request { - fields["request_"+k] = logDataTable.Request.Get(k) - } - - for k := range logDataTable.OriginResponse { - fields["origin_"+k] = logDataTable.OriginResponse.Get(k) - } - - for k := range logDataTable.DownstreamResponse { - fields["downstream_"+k] = logDataTable.DownstreamResponse.Get(k) - } - - l.mu.Lock() - defer l.mu.Unlock() - l.logger.WithFields(fields).Println() + return false } //------------------------------------------------------------------------------------------------- diff --git a/middlewares/accesslog/logger_formatters.go b/middlewares/accesslog/logger_formatters.go index 80b076b77..4cad206b6 100644 --- a/middlewares/accesslog/logger_formatters.go +++ b/middlewares/accesslog/logger_formatters.go @@ -14,56 +14,69 @@ const ( defaultValue = "-" ) -// CommonLogFormatter provides formatting in the Traefik common log format +// CommonLogFormatter provides formatting in the Træfik common log format type CommonLogFormatter struct{} -//Format formats the log entry in the Traefik common log format +// Format formats the log entry in the Træfik common log format func (f *CommonLogFormatter) Format(entry *logrus.Entry) ([]byte, error) { b := &bytes.Buffer{} - timestamp := entry.Data[StartUTC].(time.Time).Format(commonLogTimeFormat) - elapsedMillis := entry.Data[Duration].(time.Duration).Nanoseconds() / 1000000 + var timestamp = defaultValue + if v, ok := entry.Data[StartUTC]; ok { + timestamp = v.(time.Time).Format(commonLogTimeFormat) + } + + var elapsedMillis int64 + if v, ok := entry.Data[Duration]; ok { + elapsedMillis = v.(time.Duration).Nanoseconds() / 1000000 + } _, err := fmt.Fprintf(b, "%s - %s [%s] \"%s %s %s\" %v %v %s %s %v %s %s %dms\n", - entry.Data[ClientHost], - entry.Data[ClientUsername], + toLog(entry.Data, ClientHost, defaultValue, false), + toLog(entry.Data, ClientUsername, defaultValue, false), timestamp, - entry.Data[RequestMethod], - entry.Data[RequestPath], - entry.Data[RequestProtocol], - toLog(entry.Data[OriginStatus], defaultValue), - toLog(entry.Data[OriginContentSize], defaultValue), - toLog(entry.Data["request_Referer"], `"-"`), - toLog(entry.Data["request_User-Agent"], `"-"`), - toLog(entry.Data[RequestCount], defaultValue), - toLog(entry.Data[FrontendName], defaultValue), - toLog(entry.Data[BackendURL], defaultValue), + toLog(entry.Data, RequestMethod, defaultValue, false), + toLog(entry.Data, RequestPath, defaultValue, false), + toLog(entry.Data, RequestProtocol, defaultValue, false), + toLog(entry.Data, OriginStatus, defaultValue, true), + toLog(entry.Data, OriginContentSize, defaultValue, true), + toLog(entry.Data, "request_Referer", `"-"`, true), + toLog(entry.Data, "request_User-Agent", `"-"`, true), + toLog(entry.Data, RequestCount, defaultValue, true), + toLog(entry.Data, FrontendName, defaultValue, true), + toLog(entry.Data, BackendURL, defaultValue, true), elapsedMillis) return b.Bytes(), err } -func toLog(v interface{}, defaultValue string) interface{} { - if v == nil { - return defaultValue - } - - switch s := v.(type) { - case string: - return quoted(s, defaultValue) - - case fmt.Stringer: - return quoted(s.String(), defaultValue) - - default: - return v +func toLog(fields logrus.Fields, key string, defaultValue string, quoted bool) interface{} { + if v, ok := fields[key]; ok { + if v == nil { + return defaultValue + } + + switch s := v.(type) { + case string: + return toLogEntry(s, defaultValue, quoted) + + case fmt.Stringer: + return toLogEntry(s.String(), defaultValue, quoted) + + default: + return v + } } + return defaultValue } - -func quoted(s string, defaultValue string) string { +func toLogEntry(s string, defaultValue string, quote bool) string { if len(s) == 0 { return defaultValue } - return `"` + s + `"` + + if quote { + return `"` + s + `"` + } + return s } diff --git a/middlewares/accesslog/logger_formatters_test.go b/middlewares/accesslog/logger_formatters_test.go index 341d5b32b..22b68da58 100644 --- a/middlewares/accesslog/logger_formatters_test.go +++ b/middlewares/accesslog/logger_formatters_test.go @@ -20,20 +20,20 @@ func TestCommonLogFormatter_Format(t *testing.T) { { name: "OriginStatus & OriginContentSize are nil", data: map[string]interface{}{ - StartUTC: time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC), - Duration: 123 * time.Second, - ClientHost: "10.0.0.1", - ClientUsername: "Client", - RequestMethod: http.MethodGet, - RequestPath: "/foo", - RequestProtocol: "http", - OriginStatus: nil, - OriginContentSize: nil, - "request_Referer": "", - "request_User-Agent": "", - RequestCount: 0, - FrontendName: "", - BackendURL: "", + StartUTC: time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC), + Duration: 123 * time.Second, + ClientHost: "10.0.0.1", + ClientUsername: "Client", + RequestMethod: http.MethodGet, + RequestPath: "/foo", + RequestProtocol: "http", + OriginStatus: nil, + OriginContentSize: nil, + RequestRefererHeader: "", + RequestUserAgentHeader: "", + RequestCount: 0, + FrontendName: "", + BackendURL: "", }, expectedLog: `10.0.0.1 - Client [10/Nov/2009:23:00:00 +0000] "GET /foo http" - - "-" "-" 0 - - 123000ms `, @@ -41,20 +41,20 @@ func TestCommonLogFormatter_Format(t *testing.T) { { name: "all data", data: map[string]interface{}{ - StartUTC: time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC), - Duration: 123 * time.Second, - ClientHost: "10.0.0.1", - ClientUsername: "Client", - RequestMethod: http.MethodGet, - RequestPath: "/foo", - RequestProtocol: "http", - OriginStatus: 123, - OriginContentSize: 132, - "request_Referer": "referer", - "request_User-Agent": "agent", - RequestCount: nil, - FrontendName: "foo", - BackendURL: "http://10.0.0.2/toto", + StartUTC: time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC), + Duration: 123 * time.Second, + ClientHost: "10.0.0.1", + ClientUsername: "Client", + RequestMethod: http.MethodGet, + RequestPath: "/foo", + RequestProtocol: "http", + OriginStatus: 123, + OriginContentSize: 132, + RequestRefererHeader: "referer", + RequestUserAgentHeader: "agent", + RequestCount: nil, + FrontendName: "foo", + BackendURL: "http://10.0.0.2/toto", }, expectedLog: `10.0.0.1 - Client [10/Nov/2009:23:00:00 +0000] "GET /foo http" 123 132 "referer" "agent" - "foo" "http://10.0.0.2/toto" 123000ms `, @@ -80,33 +80,59 @@ func TestCommonLogFormatter_Format(t *testing.T) { func Test_toLog(t *testing.T) { testCases := []struct { - name string - value interface{} - expectedLog interface{} + desc string + fields logrus.Fields + fieldName string + defaultValue string + quoted bool + expectedLog interface{} }{ { - name: "", - value: 1, - expectedLog: 1, + desc: "Should return int 1", + fields: logrus.Fields{ + "Powpow": 1, + }, + fieldName: "Powpow", + defaultValue: defaultValue, + quoted: false, + expectedLog: 1, }, { - name: "", - value: "foo", - expectedLog: `"foo"`, + desc: "Should return string foo", + fields: logrus.Fields{ + "Powpow": "foo", + }, + fieldName: "Powpow", + defaultValue: defaultValue, + quoted: true, + expectedLog: `"foo"`, }, { - name: "", - value: nil, - expectedLog: "-", + desc: "Should return defaultValue if fieldName does not exist", + fields: logrus.Fields{ + "Powpow": "foo", + }, + fieldName: "", + defaultValue: defaultValue, + quoted: false, + expectedLog: "-", + }, + { + desc: "Should return defaultValue if fields is nil", + fields: nil, + fieldName: "", + defaultValue: defaultValue, + quoted: false, + expectedLog: "-", }, } for _, test := range testCases { test := test - t.Run(test.name, func(t *testing.T) { + t.Run(test.desc, func(t *testing.T) { t.Parallel() - lg := toLog(test.value, defaultValue) + lg := toLog(test.fields, test.fieldName, defaultValue, test.quoted) assert.Equal(t, test.expectedLog, lg) }) diff --git a/middlewares/accesslog/logger_test.go b/middlewares/accesslog/logger_test.go index b4ad67927..cb1c356c7 100644 --- a/middlewares/accesslog/logger_test.go +++ b/middlewares/accesslog/logger_test.go @@ -15,7 +15,6 @@ import ( "time" "github.com/containous/traefik/types" - shellwords "github.com/mattn/go-shellwords" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -127,156 +126,392 @@ func TestLoggerCLF(t *testing.T) { logData, err := ioutil.ReadFile(logFilePath) require.NoError(t, err) - assertValidLogData(t, logData) + expectedLog := ` TestHost - TestUser [13/Apr/2016:07:14:19 -0700] "POST testpath HTTP/0.0" 123 12 "testReferer" "testUserAgent" 1 "testFrontend" "http://127.0.0.1/testBackend" 1ms` + assertValidLogData(t, expectedLog, logData) +} + +func assertString(exp string) func(t *testing.T, actual interface{}) { + return func(t *testing.T, actual interface{}) { + t.Helper() + + assert.Equal(t, exp, actual) + } +} + +func assertNotEqual(exp string) func(t *testing.T, actual interface{}) { + return func(t *testing.T, actual interface{}) { + t.Helper() + + assert.NotEqual(t, exp, actual) + } +} + +func assertFloat64(exp float64) func(t *testing.T, actual interface{}) { + return func(t *testing.T, actual interface{}) { + t.Helper() + + assert.Equal(t, exp, actual) + } +} + +func assertFloat64NotZero() func(t *testing.T, actual interface{}) { + return func(t *testing.T, actual interface{}) { + t.Helper() + + assert.NotZero(t, actual) + } } func TestLoggerJSON(t *testing.T) { - tmpDir := createTempDir(t, JSONFormat) - defer os.RemoveAll(tmpDir) - - logFilePath := filepath.Join(tmpDir, logFileNameSuffix) - config := &types.AccessLog{FilePath: logFilePath, Format: JSONFormat} - doLogging(t, config) - - logData, err := ioutil.ReadFile(logFilePath) - require.NoError(t, err) - - jsonData := make(map[string]interface{}) - err = json.Unmarshal(logData, &jsonData) - require.NoError(t, err) - - expectedKeys := []string{ - RequestHost, - RequestAddr, - RequestMethod, - RequestPath, - RequestProtocol, - RequestPort, - RequestLine, - DownstreamStatus, - DownstreamStatusLine, - DownstreamContentSize, - OriginContentSize, - OriginStatus, - "request_Referer", - "request_User-Agent", - FrontendName, - BackendURL, - ClientUsername, - ClientHost, - ClientPort, - ClientAddr, - "level", - "msg", - "downstream_Content-Type", - RequestCount, - Duration, - Overhead, - RetryAttempts, - "time", - "StartLocal", - "StartUTC", + testCases := []struct { + desc string + config *types.AccessLog + expected map[string]func(t *testing.T, value interface{}) + }{ + { + desc: "default config", + config: &types.AccessLog{ + FilePath: "", + Format: JSONFormat, + }, + expected: map[string]func(t *testing.T, value interface{}){ + RequestHost: assertString(testHostname), + RequestAddr: assertString(testHostname), + RequestMethod: assertString(testMethod), + RequestPath: assertString(testPath), + RequestProtocol: assertString(testProto), + RequestPort: assertString("-"), + RequestLine: assertString(fmt.Sprintf("%s %s %s", testMethod, testPath, testProto)), + DownstreamStatus: assertFloat64(float64(testStatus)), + DownstreamStatusLine: assertString(fmt.Sprintf("%d ", testStatus)), + DownstreamContentSize: assertFloat64(float64(len(testContent))), + OriginContentSize: assertFloat64(float64(len(testContent))), + OriginStatus: assertFloat64(float64(testStatus)), + RequestRefererHeader: assertString(testReferer), + RequestUserAgentHeader: assertString(testUserAgent), + FrontendName: assertString(testFrontendName), + BackendURL: assertString(testBackendName), + ClientUsername: assertString(testUsername), + ClientHost: assertString(testHostname), + ClientPort: assertString(fmt.Sprintf("%d", testPort)), + ClientAddr: assertString(fmt.Sprintf("%s:%d", testHostname, testPort)), + "level": assertString("info"), + "msg": assertString(""), + "downstream_Content-Type": assertString("text/plain; charset=utf-8"), + RequestCount: assertFloat64NotZero(), + Duration: assertFloat64NotZero(), + Overhead: assertFloat64NotZero(), + RetryAttempts: assertFloat64(float64(testRetryAttempts)), + "time": assertNotEqual(""), + "StartLocal": assertNotEqual(""), + "StartUTC": assertNotEqual(""), + }, + }, + { + desc: "default config drop all fields", + config: &types.AccessLog{ + FilePath: "", + Format: JSONFormat, + Fields: &types.AccessLogFields{ + DefaultMode: "drop", + }, + }, + expected: map[string]func(t *testing.T, value interface{}){ + "level": assertString("info"), + "msg": assertString(""), + "time": assertNotEqual(""), + "downstream_Content-Type": assertString("text/plain; charset=utf-8"), + RequestRefererHeader: assertString(testReferer), + RequestUserAgentHeader: assertString(testUserAgent), + }, + }, + { + desc: "default config drop all fields and headers", + config: &types.AccessLog{ + FilePath: "", + Format: JSONFormat, + Fields: &types.AccessLogFields{ + DefaultMode: "drop", + Headers: &types.FieldHeaders{ + DefaultMode: "drop", + }, + }, + }, + expected: map[string]func(t *testing.T, value interface{}){ + "level": assertString("info"), + "msg": assertString(""), + "time": assertNotEqual(""), + }, + }, + { + desc: "default config drop all fields and redact headers", + config: &types.AccessLog{ + FilePath: "", + Format: JSONFormat, + Fields: &types.AccessLogFields{ + DefaultMode: "drop", + Headers: &types.FieldHeaders{ + DefaultMode: "redact", + }, + }, + }, + expected: map[string]func(t *testing.T, value interface{}){ + "level": assertString("info"), + "msg": assertString(""), + "time": assertNotEqual(""), + "downstream_Content-Type": assertString("REDACTED"), + RequestRefererHeader: assertString("REDACTED"), + RequestUserAgentHeader: assertString("REDACTED"), + }, + }, + { + desc: "default config drop all fields and headers but kept someone", + config: &types.AccessLog{ + FilePath: "", + Format: JSONFormat, + Fields: &types.AccessLogFields{ + DefaultMode: "drop", + Names: types.FieldNames{ + RequestHost: "keep", + }, + Headers: &types.FieldHeaders{ + DefaultMode: "drop", + Names: types.FieldHeaderNames{ + "Referer": "keep", + }, + }, + }, + }, + expected: map[string]func(t *testing.T, value interface{}){ + RequestHost: assertString(testHostname), + "level": assertString("info"), + "msg": assertString(""), + "time": assertNotEqual(""), + RequestRefererHeader: assertString(testReferer), + }, + }, } - containsKeys(t, expectedKeys, jsonData) - var assertCount int - assert.Equal(t, testHostname, jsonData[RequestHost]) - assertCount++ - assert.Equal(t, testHostname, jsonData[RequestAddr]) - assertCount++ - assert.Equal(t, testMethod, jsonData[RequestMethod]) - assertCount++ - assert.Equal(t, testPath, jsonData[RequestPath]) - assertCount++ - assert.Equal(t, testProto, jsonData[RequestProtocol]) - assertCount++ - assert.Equal(t, "-", jsonData[RequestPort]) - assertCount++ - assert.Equal(t, fmt.Sprintf("%s %s %s", testMethod, testPath, testProto), jsonData[RequestLine]) - assertCount++ - assert.Equal(t, float64(testStatus), jsonData[DownstreamStatus]) - assertCount++ - assert.Equal(t, fmt.Sprintf("%d ", testStatus), jsonData[DownstreamStatusLine]) - assertCount++ - assert.Equal(t, float64(len(testContent)), jsonData[DownstreamContentSize]) - assertCount++ - assert.Equal(t, float64(len(testContent)), jsonData[OriginContentSize]) - assertCount++ - assert.Equal(t, float64(testStatus), jsonData[OriginStatus]) - assertCount++ - assert.Equal(t, testReferer, jsonData["request_Referer"]) - assertCount++ - assert.Equal(t, testUserAgent, jsonData["request_User-Agent"]) - assertCount++ - assert.Equal(t, testFrontendName, jsonData[FrontendName]) - assertCount++ - assert.Equal(t, testBackendName, jsonData[BackendURL]) - assertCount++ - assert.Equal(t, testUsername, jsonData[ClientUsername]) - assertCount++ - assert.Equal(t, testHostname, jsonData[ClientHost]) - assertCount++ - assert.Equal(t, fmt.Sprintf("%d", testPort), jsonData[ClientPort]) - assertCount++ - assert.Equal(t, fmt.Sprintf("%s:%d", testHostname, testPort), jsonData[ClientAddr]) - assertCount++ - assert.Equal(t, "info", jsonData["level"]) - assertCount++ - assert.Equal(t, "", jsonData["msg"]) - assertCount++ - assert.Equal(t, "text/plain; charset=utf-8", jsonData["downstream_Content-Type"].(string)) - assertCount++ - assert.NotZero(t, jsonData[RequestCount].(float64)) - assertCount++ - assert.NotZero(t, jsonData[Duration].(float64)) - assertCount++ - assert.NotZero(t, jsonData[Overhead].(float64)) - assertCount++ - assert.Equal(t, float64(testRetryAttempts), jsonData[RetryAttempts].(float64)) - assertCount++ - assert.NotEqual(t, "", jsonData["time"].(string)) - assertCount++ - assert.NotEqual(t, "", jsonData["StartLocal"].(string)) - assertCount++ - assert.NotEqual(t, "", jsonData["StartUTC"].(string)) - assertCount++ + for _, test := range testCases { + test := test + t.Run(test.desc, func(t *testing.T) { + t.Parallel() - assert.Equal(t, len(jsonData), assertCount, string(logData)) + tmpDir := createTempDir(t, JSONFormat) + defer os.RemoveAll(tmpDir) + + logFilePath := filepath.Join(tmpDir, logFileNameSuffix) + + test.config.FilePath = logFilePath + doLogging(t, test.config) + + logData, err := ioutil.ReadFile(logFilePath) + require.NoError(t, err) + + jsonData := make(map[string]interface{}) + err = json.Unmarshal(logData, &jsonData) + require.NoError(t, err) + + assert.Equal(t, len(test.expected), len(jsonData)) + + for field, assertion := range test.expected { + assertion(t, jsonData[field]) + } + }) + } } func TestNewLogHandlerOutputStdout(t *testing.T) { - file, restoreStdout := captureStdout(t) - defer restoreStdout() + testCases := []struct { + desc string + config *types.AccessLog + expectedLog string + }{ + { + desc: "default config", + config: &types.AccessLog{ + FilePath: "", + Format: CommonFormat, + }, + expectedLog: `TestHost - TestUser [13/Apr/2016:07:14:19 -0700] "POST testpath HTTP/0.0" 123 12 "testReferer" "testUserAgent" 23 "testFrontend" "http://127.0.0.1/testBackend" 1ms`, + }, + { + desc: "Status code filter not matching", + config: &types.AccessLog{ + FilePath: "", + Format: CommonFormat, + Filters: &types.AccessLogFilters{ + StatusCodes: []string{"200"}, + }, + }, + expectedLog: ``, + }, + { + desc: "Status code filter matching", + config: &types.AccessLog{ + FilePath: "", + Format: CommonFormat, + Filters: &types.AccessLogFilters{ + StatusCodes: []string{"123"}, + }, + }, + expectedLog: `TestHost - TestUser [13/Apr/2016:07:14:19 -0700] "POST testpath HTTP/0.0" 123 12 "testReferer" "testUserAgent" 23 "testFrontend" "http://127.0.0.1/testBackend" 1ms`, + }, + { + desc: "Default mode keep", + config: &types.AccessLog{ + FilePath: "", + Format: CommonFormat, + Fields: &types.AccessLogFields{ + DefaultMode: "keep", + }, + }, + expectedLog: `TestHost - TestUser [13/Apr/2016:07:14:19 -0700] "POST testpath HTTP/0.0" 123 12 "testReferer" "testUserAgent" 23 "testFrontend" "http://127.0.0.1/testBackend" 1ms`, + }, + { + desc: "Default mode keep with override", + config: &types.AccessLog{ + FilePath: "", + Format: CommonFormat, + Fields: &types.AccessLogFields{ + DefaultMode: "keep", + Names: types.FieldNames{ + ClientHost: "drop", + }, + }, + }, + expectedLog: `- - TestUser [13/Apr/2016:07:14:19 -0700] "POST testpath HTTP/0.0" 123 12 "testReferer" "testUserAgent" 23 "testFrontend" "http://127.0.0.1/testBackend" 1ms`, + }, + { + desc: "Default mode drop", + config: &types.AccessLog{ + FilePath: "", + Format: CommonFormat, + Fields: &types.AccessLogFields{ + DefaultMode: "drop", + }, + }, + expectedLog: `- - - [-] "- - -" - - "testReferer" "testUserAgent" - - - 0ms`, + }, + { + desc: "Default mode drop with override", + config: &types.AccessLog{ + FilePath: "", + Format: CommonFormat, + Fields: &types.AccessLogFields{ + DefaultMode: "drop", + Names: types.FieldNames{ + ClientHost: "drop", + ClientUsername: "keep", + }, + }, + }, + expectedLog: `- - TestUser [-] "- - -" - - "testReferer" "testUserAgent" - - - 0ms`, + }, + { + desc: "Default mode drop with header dropped", + config: &types.AccessLog{ + FilePath: "", + Format: CommonFormat, + Fields: &types.AccessLogFields{ + DefaultMode: "drop", + Names: types.FieldNames{ + ClientHost: "drop", + ClientUsername: "keep", + }, + Headers: &types.FieldHeaders{ + DefaultMode: "drop", + }, + }, + }, + expectedLog: `- - TestUser [-] "- - -" - - "-" "-" - - - 0ms`, + }, + { + desc: "Default mode drop with header redacted", + config: &types.AccessLog{ + FilePath: "", + Format: CommonFormat, + Fields: &types.AccessLogFields{ + DefaultMode: "drop", + Names: types.FieldNames{ + ClientHost: "drop", + ClientUsername: "keep", + }, + Headers: &types.FieldHeaders{ + DefaultMode: "redact", + }, + }, + }, + expectedLog: `- - TestUser [-] "- - -" - - "REDACTED" "REDACTED" - - - 0ms`, + }, + { + desc: "Default mode drop with header redacted", + config: &types.AccessLog{ + FilePath: "", + Format: CommonFormat, + Fields: &types.AccessLogFields{ + DefaultMode: "drop", + Names: types.FieldNames{ + ClientHost: "drop", + ClientUsername: "keep", + }, + Headers: &types.FieldHeaders{ + DefaultMode: "keep", + Names: types.FieldHeaderNames{ + "Referer": "redact", + }, + }, + }, + }, + expectedLog: `- - TestUser [-] "- - -" - - "REDACTED" "testUserAgent" - - - 0ms`, + }, + } - config := &types.AccessLog{FilePath: "", Format: CommonFormat} - doLogging(t, config) + for _, test := range testCases { + test := test + t.Run(test.desc, func(t *testing.T) { - written, err := ioutil.ReadFile(file.Name()) - require.NoError(t, err, "unable to read captured stdout from file") - require.NotZero(t, len(written), "expected access log message on stdout") - assertValidLogData(t, written) + // NOTE: It is not possible to run these cases in parallel because we capture Stdout + + file, restoreStdout := captureStdout(t) + defer restoreStdout() + + doLogging(t, test.config) + + written, err := ioutil.ReadFile(file.Name()) + require.NoError(t, err, "unable to read captured stdout from file") + assertValidLogData(t, test.expectedLog, written) + }) + } } -func assertValidLogData(t *testing.T, logData []byte) { - tokens, err := shellwords.Parse(string(logData)) - require.NoError(t, err) +func assertValidLogData(t *testing.T, expected string, logData []byte) { + if len(expected) > 0 { + result, err := ParseAccessLog(string(logData)) + require.NoError(t, err) - formatErrMessage := fmt.Sprintf(` - Expected: TestHost - TestUser [13/Apr/2016:07:14:19 -0700] "POST testpath HTTP/0.0" 123 12 "testReferer" "testUserAgent" 1 "testFrontend" "http://127.0.0.1/testBackend" 1ms - Actual: %s - `, - string(logData)) - require.Equal(t, 14, len(tokens), formatErrMessage) - assert.Equal(t, testHostname, tokens[0], formatErrMessage) - assert.Equal(t, testUsername, tokens[2], formatErrMessage) - assert.Equal(t, fmt.Sprintf("%s %s %s", testMethod, testPath, testProto), tokens[5], formatErrMessage) - assert.Equal(t, fmt.Sprintf("%d", testStatus), tokens[6], formatErrMessage) - assert.Equal(t, fmt.Sprintf("%d", len(testContent)), tokens[7], formatErrMessage) - assert.Equal(t, testReferer, tokens[8], formatErrMessage) - assert.Equal(t, testUserAgent, tokens[9], formatErrMessage) - assert.Regexp(t, regexp.MustCompile("[0-9]*"), tokens[10], formatErrMessage) - assert.Equal(t, testFrontendName, tokens[11], formatErrMessage) - assert.Equal(t, testBackendName, tokens[12], formatErrMessage) + resultExpected, err := ParseAccessLog(expected) + require.NoError(t, err) + + formatErrMessage := fmt.Sprintf(` + Expected: %s + Actual: %s`, expected, string(logData)) + + require.Equal(t, len(resultExpected), len(result), formatErrMessage) + assert.Equal(t, resultExpected[ClientHost], result[ClientHost], formatErrMessage) + assert.Equal(t, resultExpected[ClientUsername], result[ClientUsername], formatErrMessage) + assert.Equal(t, resultExpected[RequestMethod], result[RequestMethod], formatErrMessage) + assert.Equal(t, resultExpected[RequestPath], result[RequestPath], formatErrMessage) + assert.Equal(t, resultExpected[RequestProtocol], result[RequestProtocol], formatErrMessage) + assert.Equal(t, resultExpected[OriginStatus], result[OriginStatus], formatErrMessage) + assert.Equal(t, resultExpected[OriginContentSize], result[OriginContentSize], formatErrMessage) + assert.Equal(t, resultExpected[RequestRefererHeader], result[RequestRefererHeader], formatErrMessage) + assert.Equal(t, resultExpected[RequestUserAgentHeader], result[RequestUserAgentHeader], formatErrMessage) + assert.Regexp(t, regexp.MustCompile("[0-9]*"), result[RequestCount], formatErrMessage) + assert.Equal(t, resultExpected[FrontendName], result[FrontendName], formatErrMessage) + assert.Equal(t, resultExpected[BackendURL], result[BackendURL], formatErrMessage) + assert.Regexp(t, regexp.MustCompile("[0-9]*ms"), result[Duration], formatErrMessage) + } } func captureStdout(t *testing.T) (out *os.File, restoreStdout func()) { @@ -328,28 +563,6 @@ func doLogging(t *testing.T, config *types.AccessLog) { logger.ServeHTTP(httptest.NewRecorder(), req, logWriterTestHandlerFunc) } -func containsKeys(t *testing.T, expectedKeys []string, data map[string]interface{}) { - for key, value := range data { - if !contains(expectedKeys, key) { - t.Errorf("Unexpected log key: %s [value: %s]", key, value) - } - } - for _, k := range expectedKeys { - if _, ok := data[k]; !ok { - t.Errorf("the expected key '%s' is not present in the map. %+v", k, data) - } - } -} - -func contains(values []string, value string) bool { - for _, v := range values { - if value == v { - return true - } - } - return false -} - func logWriterTestHandlerFunc(rw http.ResponseWriter, r *http.Request) { rw.Write([]byte(testContent)) rw.WriteHeader(testStatus) diff --git a/middlewares/accesslog/parser.go b/middlewares/accesslog/parser.go new file mode 100644 index 000000000..c2931d153 --- /dev/null +++ b/middlewares/accesslog/parser.go @@ -0,0 +1,54 @@ +package accesslog + +import ( + "bytes" + "regexp" +) + +// ParseAccessLog parse line of access log and return a map with each fields +func ParseAccessLog(data string) (map[string]string, error) { + var buffer bytes.Buffer + buffer.WriteString(`(\S+)`) // 1 - ClientHost + buffer.WriteString(`\s-\s`) // - - Spaces + buffer.WriteString(`(\S+)\s`) // 2 - ClientUsername + buffer.WriteString(`\[([^]]+)\]\s`) // 3 - StartUTC + buffer.WriteString(`"(\S*)\s?`) // 4 - RequestMethod + buffer.WriteString(`((?:[^"]*(?:\\")?)*)\s`) // 5 - RequestPath + buffer.WriteString(`([^"]*)"\s`) // 6 - RequestProtocol + buffer.WriteString(`(\S+)\s`) // 7 - OriginStatus + buffer.WriteString(`(\S+)\s`) // 8 - OriginContentSize + buffer.WriteString(`("?\S+"?)\s`) // 9 - Referrer + buffer.WriteString(`("\S+")\s`) // 10 - User-Agent + buffer.WriteString(`(\S+)\s`) // 11 - RequestCount + buffer.WriteString(`("[^"]*"|-)\s`) // 12 - FrontendName + buffer.WriteString(`("[^"]*"|-)\s`) // 13 - BackendURL + buffer.WriteString(`(\S+)`) // 14 - Duration + + regex, err := regexp.Compile(buffer.String()) + if err != nil { + return nil, err + } + + submatch := regex.FindStringSubmatch(data) + result := make(map[string]string) + + // Need to be > 13 to match CLF format + if len(submatch) > 13 { + result[ClientHost] = submatch[1] + result[ClientUsername] = submatch[2] + result[StartUTC] = submatch[3] + result[RequestMethod] = submatch[4] + result[RequestPath] = submatch[5] + result[RequestProtocol] = submatch[6] + result[OriginStatus] = submatch[7] + result[OriginContentSize] = submatch[8] + result[RequestRefererHeader] = submatch[9] + result[RequestUserAgentHeader] = submatch[10] + result[RequestCount] = submatch[11] + result[FrontendName] = submatch[12] + result[BackendURL] = submatch[13] + result[Duration] = submatch[14] + } + + return result, nil +} diff --git a/middlewares/accesslog/parser_test.go b/middlewares/accesslog/parser_test.go new file mode 100644 index 000000000..701fed4c3 --- /dev/null +++ b/middlewares/accesslog/parser_test.go @@ -0,0 +1,75 @@ +package accesslog + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestParseAccessLog(t *testing.T) { + testCases := []struct { + desc string + value string + expected map[string]string + }{ + { + desc: "full log", + value: `TestHost - TestUser [13/Apr/2016:07:14:19 -0700] "POST testpath HTTP/0.0" 123 12 "testReferer" "testUserAgent" 1 "testFrontend" "http://127.0.0.1/testBackend" 1ms`, + expected: map[string]string{ + ClientHost: "TestHost", + ClientUsername: "TestUser", + StartUTC: "13/Apr/2016:07:14:19 -0700", + RequestMethod: "POST", + RequestPath: "testpath", + RequestProtocol: "HTTP/0.0", + OriginStatus: "123", + OriginContentSize: "12", + RequestRefererHeader: `"testReferer"`, + RequestUserAgentHeader: `"testUserAgent"`, + RequestCount: "1", + FrontendName: `"testFrontend"`, + BackendURL: `"http://127.0.0.1/testBackend"`, + Duration: "1ms", + }, + }, + { + desc: "log with space", + value: `127.0.0.1 - - [09/Mar/2018:10:51:32 +0000] "GET / HTTP/1.1" 401 17 "-" "Go-http-client/1.1" 1 "testFrontend with space" - 0ms`, + expected: map[string]string{ + ClientHost: "127.0.0.1", + ClientUsername: "-", + StartUTC: "09/Mar/2018:10:51:32 +0000", + RequestMethod: "GET", + RequestPath: "/", + RequestProtocol: "HTTP/1.1", + OriginStatus: "401", + OriginContentSize: "17", + RequestRefererHeader: `"-"`, + RequestUserAgentHeader: `"Go-http-client/1.1"`, + RequestCount: "1", + FrontendName: `"testFrontend with space"`, + BackendURL: `-`, + Duration: "0ms", + }, + }, + { + desc: "bad log", + value: `bad`, + expected: map[string]string{}, + }, + } + + for _, test := range testCases { + test := test + t.Run(test.desc, func(t *testing.T) { + t.Parallel() + + result, err := ParseAccessLog(test.value) + assert.NoError(t, err) + assert.Equal(t, len(test.expected), len(result)) + for key, value := range test.expected { + assert.Equal(t, value, result[key]) + } + }) + } +} diff --git a/middlewares/error_pages.go b/middlewares/error_pages.go index 0b6a79fbd..735aa6f24 100644 --- a/middlewares/error_pages.go +++ b/middlewares/error_pages.go @@ -19,7 +19,7 @@ var _ Stateful = &errorPagesResponseRecorderWithCloseNotify{} //ErrorPagesHandler is a middleware that provides the custom error pages type ErrorPagesHandler struct { - HTTPCodeRanges [][2]int + HTTPCodeRanges types.HTTPCodeRanges BackendURL string errorPageForwarder *forward.Forwarder } @@ -31,27 +31,13 @@ func NewErrorPagesHandler(errorPage *types.ErrorPage, backendURL string) (*Error return nil, err } - //Break out the http status code ranges into a low int and high int - //for ease of use at runtime - var blocks [][2]int - for _, block := range errorPage.Status { - codes := strings.Split(block, "-") - //if only a single HTTP code was configured, assume the best and create the correct configuration on the user's behalf - if len(codes) == 1 { - codes = append(codes, codes[0]) - } - lowCode, err := strconv.Atoi(codes[0]) - if err != nil { - return nil, err - } - highCode, err := strconv.Atoi(codes[1]) - if err != nil { - return nil, err - } - blocks = append(blocks, [2]int{lowCode, highCode}) + httpCodeRanges, err := types.NewHTTPCodeRanges(errorPage.Status) + if err != nil { + return nil, err } + return &ErrorPagesHandler{ - HTTPCodeRanges: blocks, + HTTPCodeRanges: httpCodeRanges, BackendURL: backendURL + errorPage.Query, errorPageForwarder: fwd}, nil diff --git a/mkdocs.yml b/mkdocs.yml index 88d1d9cbc..8055a78b4 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -66,6 +66,7 @@ pages: - Basics: basics.md - Configuration: - 'Commons': 'configuration/commons.md' + - 'Logs': 'configuration/logs.md' - 'EntryPoints': 'configuration/entrypoints.md' - 'Let''s Encrypt': 'configuration/acme.md' - 'Backend: Web': 'configuration/backends/web.md' diff --git a/types/logs.go b/types/logs.go new file mode 100644 index 000000000..8705aae0a --- /dev/null +++ b/types/logs.go @@ -0,0 +1,185 @@ +package types + +import ( + "fmt" + "strings" +) + +const ( + // AccessLogKeep is the keep string value + AccessLogKeep = "keep" + // AccessLogDrop is the drop string value + AccessLogDrop = "drop" + // AccessLogRedact is the redact string value + AccessLogRedact = "redact" +) + +// TraefikLog holds the configuration settings for the traefik logger. +type TraefikLog struct { + FilePath string `json:"file,omitempty" description:"Traefik log file path. Stdout is used when omitted or empty"` + Format string `json:"format,omitempty" description:"Traefik log format: json | common"` +} + +// AccessLog holds the configuration settings for the access logger (middlewares/accesslog). +type AccessLog struct { + FilePath string `json:"file,omitempty" description:"Access log file path. Stdout is used when omitted or empty" export:"true"` + Format string `json:"format,omitempty" description:"Access log format: json | common" export:"true"` + Filters *AccessLogFilters `json:"filters,omitempty" description:"Access log filters, used to keep only specific access logs" export:"true"` + Fields *AccessLogFields `json:"fields,omitempty" description:"AccessLogFields" export:"true"` +} + +// StatusCodes holds status codes ranges to filter access log +type StatusCodes []string + +// AccessLogFilters holds filters configuration +type AccessLogFilters struct { + StatusCodes StatusCodes `json:"statusCodes,omitempty" description:"Keep only specific ranges of HTTP Status codes" export:"true"` +} + +// FieldNames holds maps of fields with specific mode +type FieldNames map[string]string + +// AccessLogFields holds configuration for access log fields +type AccessLogFields struct { + DefaultMode string `json:"defaultMode,omitempty" description:"Default mode for fields: keep | drop" export:"true"` + Names FieldNames `json:"names,omitempty" description:"Override mode for fields" export:"true"` + Headers *FieldHeaders `json:"headers,omitempty" description:"Headers to keep, drop or redact" export:"true"` +} + +// FieldHeaderNames holds maps of fields with specific mode +type FieldHeaderNames map[string]string + +// FieldHeaders holds configuration for access log headers +type FieldHeaders struct { + DefaultMode string `json:"defaultMode,omitempty" description:"Default mode for fields: keep | drop | redact" export:"true"` + Names FieldHeaderNames `json:"names,omitempty" description:"Override mode for headers" export:"true"` +} + +// Set adds strings elem into the the parser +// it splits str on , and ; +func (s *StatusCodes) Set(str string) error { + fargs := func(c rune) bool { + return c == ',' || c == ';' + } + // get function + slice := strings.FieldsFunc(str, fargs) + *s = append(*s, slice...) + return nil +} + +// Get StatusCodes +func (s *StatusCodes) Get() interface{} { return *s } + +// String return slice in a string +func (s *StatusCodes) String() string { return fmt.Sprintf("%v", *s) } + +// SetValue sets StatusCodes into the parser +func (s *StatusCodes) SetValue(val interface{}) { + *s = val.(StatusCodes) +} + +// String is the method to format the flag's value, part of the flag.Value interface. +// The String method's output will be used in diagnostics. +func (f *FieldNames) String() string { + return fmt.Sprintf("%+v", *f) +} + +// Get return the FieldNames map +func (f *FieldNames) Get() interface{} { + return *f +} + +// Set is the method to set the flag value, part of the flag.Value interface. +// Set's argument is a string to be parsed to set the flag. +// It's a space-separated list, so we split it. +func (f *FieldNames) Set(value string) error { + fields := strings.Fields(value) + + for _, field := range fields { + n := strings.SplitN(field, "=", 2) + if len(n) == 2 { + (*f)[n[0]] = n[1] + } + } + + return nil +} + +// SetValue sets the FieldNames map with val +func (f *FieldNames) SetValue(val interface{}) { + *f = val.(FieldNames) +} + +// String is the method to format the flag's value, part of the flag.Value interface. +// The String method's output will be used in diagnostics. +func (f *FieldHeaderNames) String() string { + return fmt.Sprintf("%+v", *f) +} + +// Get return the FieldHeaderNames map +func (f *FieldHeaderNames) Get() interface{} { + return *f +} + +// Set is the method to set the flag value, part of the flag.Value interface. +// Set's argument is a string to be parsed to set the flag. +// It's a space-separated list, so we split it. +func (f *FieldHeaderNames) Set(value string) error { + fields := strings.Fields(value) + + for _, field := range fields { + n := strings.SplitN(field, "=", 2) + (*f)[n[0]] = n[1] + } + + return nil +} + +// SetValue sets the FieldHeaderNames map with val +func (f *FieldHeaderNames) SetValue(val interface{}) { + *f = val.(FieldHeaderNames) +} + +// Keep check if the field need to be kept or dropped +func (f *AccessLogFields) Keep(field string) bool { + defaultKeep := true + if f != nil { + defaultKeep = checkFieldValue(f.DefaultMode, defaultKeep) + + if v, ok := f.Names[field]; ok { + return checkFieldValue(v, defaultKeep) + } + } + return defaultKeep +} + +func checkFieldValue(value string, defaultKeep bool) bool { + switch value { + case AccessLogKeep: + return true + case AccessLogDrop: + return false + default: + return defaultKeep + } +} + +// KeepHeader checks if the headers need to be kept, dropped or redacted and returns the status +func (f *AccessLogFields) KeepHeader(header string) string { + defaultValue := AccessLogKeep + if f != nil && f.Headers != nil { + defaultValue = checkFieldHeaderValue(f.Headers.DefaultMode, defaultValue) + + if v, ok := f.Headers.Names[header]; ok { + return checkFieldHeaderValue(v, defaultValue) + } + } + return defaultValue +} + +func checkFieldHeaderValue(value string, defaultValue string) string { + if value == AccessLogKeep || value == AccessLogDrop || value == AccessLogRedact { + return value + } + return defaultValue +} diff --git a/types/logs_test.go b/types/logs_test.go new file mode 100644 index 000000000..332158ed2 --- /dev/null +++ b/types/logs_test.go @@ -0,0 +1,411 @@ +package types + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestStatusCodesSet(t *testing.T) { + testCases := []struct { + desc string + value string + expected StatusCodes + }{ + { + desc: "One value should return StatusCodes of size 1", + value: "200", + expected: StatusCodes{"200"}, + }, + { + desc: "Two values separated by comma should return StatusCodes of size 2", + value: "200,400", + expected: StatusCodes{"200", "400"}, + }, + { + desc: "Two values separated by semicolon should return StatusCodes of size 2", + value: "200;400", + expected: StatusCodes{"200", "400"}, + }, + { + desc: "Three values separated by comma and semicolon should return StatusCodes of size 3", + value: "200,400;500", + expected: StatusCodes{"200", "400", "500"}, + }, + } + + for _, test := range testCases { + test := test + t.Run(test.desc, func(t *testing.T) { + t.Parallel() + + var statusCodes StatusCodes + err := statusCodes.Set(test.value) + assert.Nil(t, err) + assert.Equal(t, test.expected, statusCodes) + }) + } +} + +func TestStatusCodesGet(t *testing.T) { + testCases := []struct { + desc string + values StatusCodes + expected StatusCodes + }{ + { + desc: "Should return 1 value", + values: StatusCodes{"200"}, + expected: StatusCodes{"200"}, + }, + { + desc: "Should return 2 values", + values: StatusCodes{"200", "400"}, + expected: StatusCodes{"200", "400"}, + }, + { + desc: "Should return 3 values", + values: StatusCodes{"200", "400", "500"}, + expected: StatusCodes{"200", "400", "500"}, + }, + } + + for _, test := range testCases { + test := test + t.Run(test.desc, func(t *testing.T) { + t.Parallel() + + actual := test.values.Get() + assert.Equal(t, test.expected, actual) + }) + } +} + +func TestStatusCodesString(t *testing.T) { + testCases := []struct { + desc string + values StatusCodes + expected string + }{ + { + desc: "Should return 1 value", + values: StatusCodes{"200"}, + expected: "[200]", + }, + { + desc: "Should return 2 values", + values: StatusCodes{"200", "400"}, + expected: "[200 400]", + }, + { + desc: "Should return 3 values", + values: StatusCodes{"200", "400", "500"}, + expected: "[200 400 500]", + }, + } + + for _, test := range testCases { + test := test + t.Run(test.desc, func(t *testing.T) { + t.Parallel() + + actual := test.values.String() + assert.Equal(t, test.expected, actual) + }) + } +} + +func TestStatusCodesSetValue(t *testing.T) { + testCases := []struct { + desc string + values StatusCodes + expected StatusCodes + }{ + { + desc: "Should return 1 value", + values: StatusCodes{"200"}, + expected: StatusCodes{"200"}, + }, + { + desc: "Should return 2 values", + values: StatusCodes{"200", "400"}, + expected: StatusCodes{"200", "400"}, + }, + { + desc: "Should return 3 values", + values: StatusCodes{"200", "400", "500"}, + expected: StatusCodes{"200", "400", "500"}, + }, + } + + for _, test := range testCases { + test := test + t.Run(test.desc, func(t *testing.T) { + t.Parallel() + + var slice StatusCodes + slice.SetValue(test.values) + assert.Equal(t, test.expected, slice) + }) + } +} + +func TestFieldsNamesSet(t *testing.T) { + testCases := []struct { + desc string + value string + expected *FieldNames + }{ + { + desc: "One value should return FieldNames of size 1", + value: "field-1=foo", + expected: &FieldNames{ + "field-1": "foo", + }, + }, + { + desc: "Two values separated by space should return FieldNames of size 2", + value: "field-1=foo field-2=bar", + expected: &FieldNames{ + "field-1": "foo", + "field-2": "bar", + }, + }, + } + + for _, test := range testCases { + test := test + t.Run(test.desc, func(t *testing.T) { + t.Parallel() + + fieldsNames := &FieldNames{} + err := fieldsNames.Set(test.value) + assert.NoError(t, err) + + assert.Equal(t, test.expected, fieldsNames) + }) + } +} + +func TestFieldsNamesGet(t *testing.T) { + testCases := []struct { + desc string + values FieldNames + expected FieldNames + }{ + { + desc: "Should return 1 value", + values: FieldNames{"field-1": "foo"}, + expected: FieldNames{"field-1": "foo"}, + }, + { + desc: "Should return 2 values", + values: FieldNames{"field-1": "foo", "field-2": "bar"}, + expected: FieldNames{"field-1": "foo", "field-2": "bar"}, + }, + { + desc: "Should return 3 values", + values: FieldNames{"field-1": "foo", "field-2": "bar", "field-3": "powpow"}, + expected: FieldNames{"field-1": "foo", "field-2": "bar", "field-3": "powpow"}, + }, + } + + for _, test := range testCases { + test := test + t.Run(test.desc, func(t *testing.T) { + t.Parallel() + + actual := test.values.Get() + assert.Equal(t, test.expected, actual) + }) + } +} + +func TestFieldsNamesString(t *testing.T) { + testCases := []struct { + desc string + values FieldNames + expected string + }{ + { + desc: "Should return 1 value", + values: FieldNames{"field-1": "foo"}, + expected: "map[field-1:foo]", + }, + } + + for _, test := range testCases { + test := test + t.Run(test.desc, func(t *testing.T) { + t.Parallel() + + actual := test.values.String() + assert.Equal(t, test.expected, actual) + }) + } +} + +func TestFieldsNamesSetValue(t *testing.T) { + testCases := []struct { + desc string + values FieldNames + expected *FieldNames + }{ + { + desc: "Should return 1 value", + values: FieldNames{"field-1": "foo"}, + expected: &FieldNames{"field-1": "foo"}, + }, + { + desc: "Should return 2 values", + values: FieldNames{"field-1": "foo", "field-2": "bar"}, + expected: &FieldNames{"field-1": "foo", "field-2": "bar"}, + }, + { + desc: "Should return 3 values", + values: FieldNames{"field-1": "foo", "field-2": "bar", "field-3": "powpow"}, + expected: &FieldNames{"field-1": "foo", "field-2": "bar", "field-3": "powpow"}, + }, + } + + for _, test := range testCases { + test := test + t.Run(test.desc, func(t *testing.T) { + t.Parallel() + + fieldsNames := &FieldNames{} + fieldsNames.SetValue(test.values) + assert.Equal(t, test.expected, fieldsNames) + }) + } +} + +func TestFieldsHeadersNamesSet(t *testing.T) { + testCases := []struct { + desc string + value string + expected *FieldHeaderNames + }{ + { + desc: "One value should return FieldNames of size 1", + value: "X-HEADER-1=foo", + expected: &FieldHeaderNames{ + "X-HEADER-1": "foo", + }, + }, + { + desc: "Two values separated by space should return FieldNames of size 2", + value: "X-HEADER-1=foo X-HEADER-2=bar", + expected: &FieldHeaderNames{ + "X-HEADER-1": "foo", + "X-HEADER-2": "bar", + }, + }, + } + + for _, test := range testCases { + test := test + t.Run(test.desc, func(t *testing.T) { + t.Parallel() + + headersNames := &FieldHeaderNames{} + err := headersNames.Set(test.value) + assert.NoError(t, err) + + assert.Equal(t, test.expected, headersNames) + }) + } +} + +func TestFieldsHeadersNamesGet(t *testing.T) { + testCases := []struct { + desc string + values FieldHeaderNames + expected FieldHeaderNames + }{ + { + desc: "Should return 1 value", + values: FieldHeaderNames{"X-HEADER-1": "foo"}, + expected: FieldHeaderNames{"X-HEADER-1": "foo"}, + }, + { + desc: "Should return 2 values", + values: FieldHeaderNames{"X-HEADER-1": "foo", "X-HEADER-2": "bar"}, + expected: FieldHeaderNames{"X-HEADER-1": "foo", "X-HEADER-2": "bar"}, + }, + { + desc: "Should return 3 values", + values: FieldHeaderNames{"X-HEADER-1": "foo", "X-HEADER-2": "bar", "X-HEADER-3": "powpow"}, + expected: FieldHeaderNames{"X-HEADER-1": "foo", "X-HEADER-2": "bar", "X-HEADER-3": "powpow"}, + }, + } + + for _, test := range testCases { + test := test + t.Run(test.desc, func(t *testing.T) { + t.Parallel() + + actual := test.values.Get() + assert.Equal(t, test.expected, actual) + }) + } +} + +func TestFieldsHeadersNamesString(t *testing.T) { + testCases := []struct { + desc string + values FieldHeaderNames + expected string + }{ + { + desc: "Should return 1 value", + values: FieldHeaderNames{"X-HEADER-1": "foo"}, + expected: "map[X-HEADER-1:foo]", + }, + } + + for _, test := range testCases { + test := test + t.Run(test.desc, func(t *testing.T) { + t.Parallel() + + actual := test.values.String() + assert.Equal(t, test.expected, actual) + }) + } +} + +func TestFieldsHeadersNamesSetValue(t *testing.T) { + testCases := []struct { + desc string + values FieldHeaderNames + expected *FieldHeaderNames + }{ + { + desc: "Should return 1 value", + values: FieldHeaderNames{"X-HEADER-1": "foo"}, + expected: &FieldHeaderNames{"X-HEADER-1": "foo"}, + }, + { + desc: "Should return 2 values", + values: FieldHeaderNames{"X-HEADER-1": "foo", "X-HEADER-2": "bar"}, + expected: &FieldHeaderNames{"X-HEADER-1": "foo", "X-HEADER-2": "bar"}, + }, + { + desc: "Should return 3 values", + values: FieldHeaderNames{"X-HEADER-1": "foo", "X-HEADER-2": "bar", "X-HEADER-3": "powpow"}, + expected: &FieldHeaderNames{"X-HEADER-1": "foo", "X-HEADER-2": "bar", "X-HEADER-3": "powpow"}, + }, + } + + for _, test := range testCases { + test := test + t.Run(test.desc, func(t *testing.T) { + t.Parallel() + + headersNames := &FieldHeaderNames{} + headersNames.SetValue(test.values) + assert.Equal(t, test.expected, headersNames) + }) + } +} diff --git a/types/types.go b/types/types.go index e8cc82268..81312df91 100644 --- a/types/types.go +++ b/types/types.go @@ -461,18 +461,6 @@ func (b *Buckets) SetValue(val interface{}) { *b = val.(Buckets) } -// TraefikLog holds the configuration settings for the traefik logger. -type TraefikLog struct { - FilePath string `json:"file,omitempty" description:"Traefik log file path. Stdout is used when omitted or empty"` - Format string `json:"format,omitempty" description:"Traefik log format: json | common"` -} - -// AccessLog holds the configuration settings for the access logger (middlewares/accesslog). -type AccessLog struct { - FilePath string `json:"file,omitempty" description:"Access log file path. Stdout is used when omitted or empty" export:"true"` - Format string `json:"format,omitempty" description:"Access log format: json | common" export:"true"` -} - // ClientTLS holds TLS specific configurations as client // CA, Cert and Key can be either path or file contents type ClientTLS struct { @@ -497,7 +485,7 @@ func (clientTLS *ClientTLS) CreateTLSConfig() (*tls.Config, error) { if _, errCA := os.Stat(clientTLS.CA); errCA == nil { ca, err = ioutil.ReadFile(clientTLS.CA) if err != nil { - return nil, fmt.Errorf("Failed to read CA. %s", err) + return nil, fmt.Errorf("failed to read CA. %s", err) } } else { ca = []byte(clientTLS.CA) @@ -522,7 +510,7 @@ func (clientTLS *ClientTLS) CreateTLSConfig() (*tls.Config, error) { if errKeyIsFile == nil { cert, err = tls.LoadX509KeyPair(clientTLS.Cert, clientTLS.Key) if err != nil { - return nil, fmt.Errorf("Failed to load TLS keypair: %v", err) + return nil, fmt.Errorf("failed to load TLS keypair: %v", err) } } else { return nil, fmt.Errorf("tls cert is a file, but tls key is not") @@ -531,11 +519,11 @@ func (clientTLS *ClientTLS) CreateTLSConfig() (*tls.Config, error) { if errKeyIsFile != nil { cert, err = tls.X509KeyPair([]byte(clientTLS.Cert), []byte(clientTLS.Key)) if err != nil { - return nil, fmt.Errorf("Failed to load TLS keypair: %v", err) + return nil, fmt.Errorf("failed to load TLS keypair: %v", err) } } else { - return nil, fmt.Errorf("tls key is a file, but tls cert is not") + return nil, fmt.Errorf("TLS key is a file, but tls cert is not") } } } @@ -548,3 +536,30 @@ func (clientTLS *ClientTLS) CreateTLSConfig() (*tls.Config, error) { } return TLSConfig, nil } + +// HTTPCodeRanges holds HTTP code ranges +type HTTPCodeRanges [][2]int + +// NewHTTPCodeRanges create a new NewHTTPCodeRanges from a given []string]. +// Break out the http status code ranges into a low int and high int +// for ease of use at runtime +func NewHTTPCodeRanges(strBlocks []string) (HTTPCodeRanges, error) { + var blocks HTTPCodeRanges + for _, block := range strBlocks { + codes := strings.Split(block, "-") + //if only a single HTTP code was configured, assume the best and create the correct configuration on the user's behalf + if len(codes) == 1 { + codes = append(codes, codes[0]) + } + lowCode, err := strconv.Atoi(codes[0]) + if err != nil { + return nil, err + } + highCode, err := strconv.Atoi(codes[1]) + if err != nil { + return nil, err + } + blocks = append(blocks, [2]int{lowCode, highCode}) + } + return blocks, nil +} diff --git a/types/types_test.go b/types/types_test.go index c0327fcc1..68df6e90a 100644 --- a/types/types_test.go +++ b/types/types_test.go @@ -35,3 +35,61 @@ func TestHeaders_ShouldReturnTrueWhenHasSecureHeadersDefined(t *testing.T) { assert.True(t, headers.HasSecureHeadersDefined()) } + +func TestNewHTTPCodeRanges(t *testing.T) { + testCases := []struct { + desc string + strBlocks []string + expected HTTPCodeRanges + errExpected bool + }{ + { + desc: "Should return 2 code range", + strBlocks: []string{ + "200-500", + "502", + }, + expected: HTTPCodeRanges{[2]int{200, 500}, [2]int{502, 502}}, + errExpected: false, + }, + { + desc: "Should return 2 code range", + strBlocks: []string{ + "200-500", + "205", + }, + expected: HTTPCodeRanges{[2]int{200, 500}, [2]int{205, 205}}, + errExpected: false, + }, + { + desc: "invalid code range", + strBlocks: []string{ + "200-500", + "aaa", + }, + expected: nil, + errExpected: true, + }, + { + desc: "invalid code range nil", + strBlocks: nil, + expected: nil, + errExpected: false, + }, + } + + for _, test := range testCases { + test := test + t.Run(test.desc, func(t *testing.T) { + t.Parallel() + + actual, err := NewHTTPCodeRanges(test.strBlocks) + assert.Equal(t, test.expected, actual) + if test.errExpected { + assert.Error(t, err) + } else { + assert.NoError(t, err) + } + }) + } +} diff --git a/vendor/github.com/mattn/go-shellwords/LICENSE b/vendor/github.com/mattn/go-shellwords/LICENSE deleted file mode 100644 index 740fa9313..000000000 --- a/vendor/github.com/mattn/go-shellwords/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2017 Yasuhiro Matsumoto - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/vendor/github.com/mattn/go-shellwords/shellwords.go b/vendor/github.com/mattn/go-shellwords/shellwords.go deleted file mode 100644 index 107803927..000000000 --- a/vendor/github.com/mattn/go-shellwords/shellwords.go +++ /dev/null @@ -1,145 +0,0 @@ -package shellwords - -import ( - "errors" - "os" - "regexp" -) - -var ( - ParseEnv bool = false - ParseBacktick bool = false -) - -var envRe = regexp.MustCompile(`\$({[a-zA-Z0-9_]+}|[a-zA-Z0-9_]+)`) - -func isSpace(r rune) bool { - switch r { - case ' ', '\t', '\r', '\n': - return true - } - return false -} - -func replaceEnv(s string) string { - return envRe.ReplaceAllStringFunc(s, func(s string) string { - s = s[1:] - if s[0] == '{' { - s = s[1 : len(s)-1] - } - return os.Getenv(s) - }) -} - -type Parser struct { - ParseEnv bool - ParseBacktick bool - Position int -} - -func NewParser() *Parser { - return &Parser{ParseEnv, ParseBacktick, 0} -} - -func (p *Parser) Parse(line string) ([]string, error) { - args := []string{} - buf := "" - var escaped, doubleQuoted, singleQuoted, backQuote bool - backtick := "" - - pos := -1 - got := false - -loop: - for i, r := range line { - if escaped { - buf += string(r) - escaped = false - continue - } - - if r == '\\' { - if singleQuoted { - buf += string(r) - } else { - escaped = true - } - continue - } - - if isSpace(r) { - if singleQuoted || doubleQuoted || backQuote { - buf += string(r) - backtick += string(r) - } else if got { - if p.ParseEnv { - buf = replaceEnv(buf) - } - args = append(args, buf) - buf = "" - got = false - } - continue - } - - switch r { - case '`': - if !singleQuoted && !doubleQuoted { - if p.ParseBacktick { - if backQuote { - out, err := shellRun(backtick) - if err != nil { - return nil, err - } - buf = out - } - backtick = "" - backQuote = !backQuote - continue - } - backtick = "" - backQuote = !backQuote - } - case '"': - if !singleQuoted { - doubleQuoted = !doubleQuoted - continue - } - case '\'': - if !doubleQuoted { - singleQuoted = !singleQuoted - continue - } - case ';', '&', '|', '<', '>': - if !(escaped || singleQuoted || doubleQuoted || backQuote) { - pos = i - break loop - } - } - - got = true - buf += string(r) - if backQuote { - backtick += string(r) - } - } - - if got { - if p.ParseEnv { - buf = replaceEnv(buf) - } - args = append(args, buf) - } - - if escaped || singleQuoted || doubleQuoted || backQuote { - return nil, errors.New("invalid command line string") - } - - p.Position = pos - - return args, nil -} - -func Parse(line string) ([]string, error) { - return NewParser().Parse(line) -} diff --git a/vendor/github.com/mattn/go-shellwords/util_posix.go b/vendor/github.com/mattn/go-shellwords/util_posix.go deleted file mode 100644 index 4f8ac55e4..000000000 --- a/vendor/github.com/mattn/go-shellwords/util_posix.go +++ /dev/null @@ -1,19 +0,0 @@ -// +build !windows - -package shellwords - -import ( - "errors" - "os" - "os/exec" - "strings" -) - -func shellRun(line string) (string, error) { - shell := os.Getenv("SHELL") - b, err := exec.Command(shell, "-c", line).Output() - if err != nil { - return "", errors.New(err.Error() + ":" + string(b)) - } - return strings.TrimSpace(string(b)), nil -} diff --git a/vendor/github.com/mattn/go-shellwords/util_windows.go b/vendor/github.com/mattn/go-shellwords/util_windows.go deleted file mode 100644 index 7cad4cf06..000000000 --- a/vendor/github.com/mattn/go-shellwords/util_windows.go +++ /dev/null @@ -1,17 +0,0 @@ -package shellwords - -import ( - "errors" - "os" - "os/exec" - "strings" -) - -func shellRun(line string) (string, error) { - shell := os.Getenv("COMSPEC") - b, err := exec.Command(shell, "/c", line).Output() - if err != nil { - return "", errors.New(err.Error() + ":" + string(b)) - } - return strings.TrimSpace(string(b)), nil -}