Merge pull request #761 from nathan-osman/errors-in-health

Errors in health

configuration.go

@@ -315,6 +315,9 @@ func NewTraefikDefaultPointersConfiguration() *TraefikConfiguration {
 	// default Web
 	var defaultWeb WebProvider
 	defaultWeb.Address = ":8080"
+	defaultWeb.Statistics = &types.Statistics{
+		RecentErrors: 10,
+	}
 
 	// default Marathon
 	var defaultMarathon provider.Marathon

docs/toml.md

@@ -481,6 +481,10 @@ address = ":8080"
 # Optional
 # ReadOnly = false
 #
+# To enable more detailed statistics
+# [web.statistics]
+#   RecentErrors = 10
+#
 # To enable basic auth on the webui
 # with 2 user/pass: test:test and test2:test2
 # Passwords can be encoded in MD5, SHA1 and BCrypt: you can use htpasswd to generate those ones
@@ -555,7 +559,26 @@ $ curl -s "http://localhost:8080/health" | jq .
   // average response time (formated time)
   "average_response_time": "864.8016ms",
   // average response time in seconds
-  "average_response_time_sec": 0.8648016000000001
+  "average_response_time_sec": 0.8648016000000001,
+
+  // request statistics [requires --web.statistics to be set]
+  // ten most recent requests with 4xx and 5xx status codes
+  "recent_errors": [
+    {
+      // status code
+      "status_code": 500,
+      // description of status code
+      "status": "Internal Server Error",
+      // request HTTP method
+      "method": "GET",
+      // request hostname
+      "host": "localhost",
+      // request path
+      "path": "/path",
+      // RFC 3339 formatted date/time
+      "time": "2016-10-21T16:59:15.418495872-07:00"
+    }
+  ]
 }
 ```
 

server.go

@@ -165,6 +165,12 @@ func (server *Server) startHTTPServers() {
 	server.serverEntryPoints = server.buildEntryPoints(server.globalConfiguration)
 	for newServerEntryPointName, newServerEntryPoint := range server.serverEntryPoints {
 		serverMiddlewares := []negroni.Handler{server.loggerMiddleware, metrics}
+		if server.globalConfiguration.Web != nil && server.globalConfiguration.Web.Statistics != nil {
+			statsRecorder = &StatsRecorder{
+				numRecentErrors: server.globalConfiguration.Web.Statistics.RecentErrors,
+			}
+			serverMiddlewares = append(serverMiddlewares, statsRecorder)
+		}
 		if server.globalConfiguration.EntryPoints[newServerEntryPointName].Auth != nil {
 			authMiddleware, err := middlewares.NewAuthenticator(server.globalConfiguration.EntryPoints[newServerEntryPointName].Auth)
 			if err != nil {

stats.go

@@ -0,0 +1,85 @@
+package main
+
+import (
+	"net/http"
+	"sync"
+	"time"
+)
+
+// StatsRecorder is an optional middleware that records more details statistics
+// about requests and how they are processed. This currently consists of recent
+// requests that have caused errors (4xx and 5xx status codes), making it easy
+// to pinpoint problems.
+type StatsRecorder struct {
+	mutex           sync.RWMutex
+	numRecentErrors int
+	recentErrors    []*statsError
+}
+
+// Stats includes all of the stats gathered by the recorder.
+type Stats struct {
+	RecentErrors []*statsError `json:"recent_errors"`
+}
+
+// statsError represents an error that has occurred during request processing.
+type statsError struct {
+	StatusCode int       `json:"status_code"`
+	Status     string    `json:"status"`
+	Method     string    `json:"method"`
+	Host       string    `json:"host"`
+	Path       string    `json:"path"`
+	Time       time.Time `json:"time"`
+}
+
+// responseRecorder captures information from the response and preserves it for
+// later analysis.
+type responseRecorder struct {
+	http.ResponseWriter
+	statusCode int
+}
+
+// WriteHeader captures the status code for later retrieval.
+func (r *responseRecorder) WriteHeader(status int) {
+	r.ResponseWriter.WriteHeader(status)
+	r.statusCode = status
+}
+
+// ServeHTTP silently extracts information from the request and response as it
+// is processed. If the response is 4xx or 5xx, add it to the list of 10 most
+// recent errors.
+func (s *StatsRecorder) ServeHTTP(w http.ResponseWriter, r *http.Request, next http.HandlerFunc) {
+	recorder := &responseRecorder{w, http.StatusOK}
+	next(recorder, r)
+	if recorder.statusCode >= 400 {
+		s.mutex.Lock()
+		defer s.mutex.Unlock()
+		s.recentErrors = append([]*statsError{
+			{
+				StatusCode: recorder.statusCode,
+				Status:     http.StatusText(recorder.statusCode),
+				Method:     r.Method,
+				Host:       r.Host,
+				Path:       r.URL.Path,
+				Time:       time.Now(),
+			},
+		}, s.recentErrors...)
+		// Limit the size of the list to numRecentErrors
+		if len(s.recentErrors) > s.numRecentErrors {
+			s.recentErrors = s.recentErrors[:s.numRecentErrors]
+		}
+	}
+}
+
+// Data returns a copy of the statistics that have been gathered.
+func (s *StatsRecorder) Data() *Stats {
+	s.mutex.RLock()
+	defer s.mutex.RUnlock()
+
+	// We can't return the slice directly or a race condition might develop
+	recentErrors := make([]*statsError, len(s.recentErrors))
+	copy(recentErrors, s.recentErrors)
+
+	return &Stats{
+		RecentErrors: recentErrors,
+	}
+}

traefik.sample.toml

@@ -271,6 +271,10 @@
 #
 # Optional
 # ReadOnly = false
+#
+# Enable more detailed statistics
+# [web.statistics]
+#   RecentErrors = 10
 
 # To enable basic auth on the webui
 # with 2 user/pass: test:test and test2:test2

types/types.go

@@ -228,3 +228,8 @@ type Digest struct {
 func CanonicalDomain(domain string) string {
 	return strings.ToLower(strings.TrimSpace(domain))
 }
+
+// Statistics provides options for monitoring request and response stats
+type Statistics struct {
+	RecentErrors int `description:"Number of recent errors logged"`
+}

web.go

@@ -17,21 +17,25 @@ import (
 	"github.com/containous/traefik/types"
 	"github.com/containous/traefik/version"
 	"github.com/elazarl/go-bindata-assetfs"
-	"github.com/thoas/stats"
+	thoas_stats "github.com/thoas/stats"
 	"github.com/unrolled/render"
 )
 
-var metrics = stats.New()
+var (
+	metrics       = thoas_stats.New()
+	statsRecorder *StatsRecorder
+)
 
 // WebProvider is a provider.Provider implementation that provides the UI.
 // FIXME to be handled another way.
 type WebProvider struct {
-	Address  string `description:"Web administration port"`
+	Address    string            `description:"Web administration port"`
-	CertFile string `description:"SSL certificate"`
+	CertFile   string            `description:"SSL certificate"`
-	KeyFile  string `description:"SSL certificate"`
+	KeyFile    string            `description:"SSL certificate"`
-	ReadOnly bool   `description:"Enable read only API"`
+	ReadOnly   bool              `description:"Enable read only API"`
-	server   *Server
+	Statistics *types.Statistics `description:"Enable more detailed statistics"`
-	Auth     *types.Auth
+	server     *Server
+	Auth       *types.Auth
 }
 
 var (
@@ -133,8 +137,19 @@ func (provider *WebProvider) Provide(configurationChan chan<- types.ConfigMessag
 	return nil
 }
 
+// healthResponse combines data returned by thoas/stats with statistics (if
+// they are enabled).
+type healthResponse struct {
+	*thoas_stats.Data
+	*Stats
+}
+
 func (provider *WebProvider) getHealthHandler(response http.ResponseWriter, request *http.Request) {
-	templatesRenderer.JSON(response, http.StatusOK, metrics.Data())
+	health := &healthResponse{Data: metrics.Data()}
+	if statsRecorder != nil {
+		health.Stats = statsRecorder.Data()
+	}
+	templatesRenderer.JSON(response, http.StatusOK, health)
 }
 
 func (provider *WebProvider) getPingHandler(response http.ResponseWriter, request *http.Request) {

webui/src/app/sections/health/health.controller.js

@@ -1,5 +1,6 @@
 'use strict';
-var d3 = require('d3');
+var d3 = require('d3'),
+    moment = require('moment');
 
 /** @ngInject */
 function HealthController($scope, $interval, $log, Health) {
@@ -160,6 +161,15 @@ function HealthController($scope, $interval, $log, Health) {
     }
   }
 
+  /**
+   * Format the timestamp as "x seconds ago", etc.
+   *
+   * @param {String} t Timestamp returned from the API
+   */
+  function formatTimestamp(t) {
+    return moment(t, "YYYY-MM-DDTHH:mm:ssZ").fromNow();
+  }
+
   /**
    * Load all graph's datas
    *
@@ -172,6 +182,13 @@ function HealthController($scope, $interval, $log, Health) {
     // Load datas and update Total Status Code Count graph render
     updateTotalStatusCodeCount(health.total_status_code_count);
 
+    // Format the timestamps
+    if (health.recent_errors) {
+      angular.forEach(health.recent_errors, function(i) {
+        i.time_formatted = formatTimestamp(i.time);
+      });
+    }
+
     // set data's view
     vm.health = health;
   }

webui/src/app/sections/health/health.html

@@ -40,4 +40,34 @@
 
   </div>
 
+  <div ng-if="healthCtrl.health.recent_errors">
+    <h3>Recent HTTP Errors</h3>
+    <table class="table table-striped table-bordered">
+      <tr>
+        <td>Status</td>
+        <td>Request</td>
+        <td>Time</td>
+      </tr>
+      <tr ng-repeat="entry in healthCtrl.health.recent_errors"
+          ng-class="{'text-danger': entry.status_code >= 500}">
+        <td>{{ entry.status_code }} &mdash; {{ entry.status }}</td>
+        <td>
+          <span class="badge">{{ entry.method }}</span>
+          &nbsp;
+          {{ entry.host }}{{ entry.path }}
+        </td>
+        <td>
+          <span title="{{ entry.time }}">
+            {{ entry.time_formatted }}
+          </span>
+        </td>
+      </tr>
+      <tr ng-if="healthCtrl.health.recent_errors.length == 0">
+        <td colspan="3">
+          <p class="text-muted text-center">No entries</p>
+        </td>
+      </tr>
+    </table>
+  </div>
+
 </div>