Merge pull request #12 from thorhs/multi_provider_config_updates

Adding caching and merging of configurations

consul.go

@@ -68,7 +68,7 @@ func NewConsulProvider() *ConsulProvider {
 	return consulProvider
 }
 
-func (provider *ConsulProvider) Provide(configurationChan chan<- *Configuration) {
+func (provider *ConsulProvider) Provide(configurationChan chan<- configMessage) {
 	config := &api.Config{
 		Address:    provider.Endpoint,
 		Scheme:     "http",
@@ -99,7 +99,7 @@ func (provider *ConsulProvider) Provide(configurationChan chan<- *Configuration)
 						waitIndex = meta.LastIndex
 						configuration := provider.loadConsulConfig()
 						if configuration != nil {
-							configurationChan <- configuration
+							configurationChan <- configMessage{"consul", configuration}
 						}
 					}
 				}
@@ -107,7 +107,7 @@ func (provider *ConsulProvider) Provide(configurationChan chan<- *Configuration)
 		}
 	}
 	configuration := provider.loadConsulConfig()
-	configurationChan <- configuration
+	configurationChan <- configMessage{"consul", configuration}
 }
 
 func (provider *ConsulProvider) loadConsulConfig() *Configuration {

docker.go

@@ -65,7 +65,7 @@ var DockerFuncMap = template.FuncMap{
 	"getHost": getHost,
 }
 
-func (provider *DockerProvider) Provide(configurationChan chan<- *Configuration) {
+func (provider *DockerProvider) Provide(configurationChan chan<- configMessage) {
 	if dockerClient, err := docker.NewClient(provider.Endpoint); err != nil {
 		log.Fatalf("Failed to create a client for docker, error: %s", err)
 	} else {
@@ -90,7 +90,7 @@ func (provider *DockerProvider) Provide(configurationChan chan<- *Configuration)
 							log.Debugf("Docker event receveived %+v", event)
 							configuration := provider.loadDockerConfig(dockerClient)
 							if configuration != nil {
-								configurationChan <- configuration
+								configurationChan <- configMessage{"docker", configuration}
 							}
 						}
 					}
@@ -106,7 +106,7 @@ func (provider *DockerProvider) Provide(configurationChan chan<- *Configuration)
 		}
 
 		configuration := provider.loadDockerConfig(dockerClient)
-		configurationChan <- configuration
+		configurationChan <- configMessage{"docker", configuration}
 	}
 }
 

docs/index.md

@@ -254,11 +254,12 @@ $ curl -s "http://localhost:8080/health" | jq .
 }
 ```
 
-* ```/api```: ```GET``` or ```PUT``` a configuration
+* ```/api```: ```GET``` configuration for all providers
 
 ```sh
 $ curl -s "http://localhost:8082/api" | jq .
 {
+  "file": {
     "Frontends": {
       "frontend-traefik": {
         "Routes": {
@@ -321,16 +322,41 @@ $ curl -s "http://localhost:8082/api" | jq .
         }
       }
     }
+  },
+  "marathon": {
+    "Frontends": {
+      "frontend-marathon": {
+        "Routes": {
+          "route-host-marathon": {
+            "Value": "marathon.docker.localhost",
+            "Rule": "Host"
+          }
+        },
+        "Backend": "backend-marathon"
+      },
+    },
+    "Backends": {
+      "backend-marathon": {
+        "Servers": {
+          "server-marathon-1": {
+            "Weight": 0,
+            "Url": "http://172.17.0.8:802"
+          },
+        },
+      },
+    },
+  },
 }
 
 ```
 
-* ```/api/backends```: ```GET``` backends
+* ```/api/{provider}```: ```GET``` or ```PUT``` provider
-* ```/api/backends/{backend}```: ```GET``` a backend
+* ```/api/{provider}/backends```: ```GET``` backends
-* ```/api/backends/{backend}/servers```: ```GET``` servers in a backend
+* ```/api/{provider}/backends/{backend}```: ```GET``` a backend
-* ```/api/backends/{backend}/servers/{server}```: ```GET``` a server in a backend
+* ```/api/{provider}/backends/{backend}/servers```: ```GET``` servers in a backend
-* ```/api/frontends```: ```GET``` frontends
+* ```/api/{provider}/backends/{backend}/servers/{server}```: ```GET``` a server in a backend
-* ```/api/frontends/{frontend}```: ```GET``` a frontend
+* ```/api/{provider}/frontends```: ```GET``` frontends
+* ```/api/{provider}/frontends/{frontend}```: ```GET``` a frontend
 
 
 ## <a id="docker"></a> Docker backend

file.go

@@ -23,7 +23,7 @@ func NewFileProvider() *FileProvider {
 	return fileProvider
 }
 
-func (provider *FileProvider) Provide(configurationChan chan<- *Configuration) {
+func (provider *FileProvider) Provide(configurationChan chan<- configMessage) {
 	watcher, err := fsnotify.NewWatcher()
 	if err != nil {
 		log.Error("Error creating file watcher", err)
@@ -48,7 +48,7 @@ func (provider *FileProvider) Provide(configurationChan chan<- *Configuration) {
 					log.Debug("File event:", event)
 					configuration := provider.LoadFileConfig(file.Name())
 					if configuration != nil {
-						configurationChan <- configuration
+						configurationChan <- configMessage{"file", configuration}
 					}
 				}
 			case error := <-watcher.Errors:
@@ -67,7 +67,7 @@ func (provider *FileProvider) Provide(configurationChan chan<- *Configuration) {
 	}
 
 	configuration := provider.LoadFileConfig(file.Name())
-	configurationChan <- configuration
+	configurationChan <- configMessage{"file", configuration}
 	<-done
 }
 

marathon.go

@@ -67,7 +67,7 @@ var MarathonFuncMap = template.FuncMap{
 	},
 }
 
-func (provider *MarathonProvider) Provide(configurationChan chan<- *Configuration) {
+func (provider *MarathonProvider) Provide(configurationChan chan<- configMessage) {
 	config := marathon.NewDefaultConfig()
 	config.URL = provider.Endpoint
 	config.EventsInterface = provider.NetworkInterface
@@ -88,7 +88,7 @@ func (provider *MarathonProvider) Provide(configurationChan chan<- *Configuratio
 					log.Debug("Marathon event receveived", event)
 					configuration := provider.loadMarathonConfig()
 					if configuration != nil {
-						configurationChan <- configuration
+						configurationChan <- configMessage{"marathon", configuration}
 					}
 				}
 			}()
@@ -96,7 +96,7 @@ func (provider *MarathonProvider) Provide(configurationChan chan<- *Configuratio
 	}
 
 	configuration := provider.loadMarathonConfig()
-	configurationChan <- configuration
+	configurationChan <- configMessage{"marathon", configuration}
 }
 
 func (provider *MarathonProvider) loadMarathonConfig() *Configuration {

provider.go

@@ -1,5 +1,5 @@
 package main
 
 type Provider interface {
-	Provide(configurationChan chan<- *Configuration)
+	Provide(configurationChan chan<- configMessage)
 }

templates/configuration.tmpl

@@ -27,9 +27,10 @@
     <div class="col-md-6">
         <!-- <div class="panel-heading">Frontends</div>
         <div class="panel-body"> -->
-            {{range $keyFrontends, $valueFrontends := .Configuration.Frontends}}
+            {{range $keyProviders, $valueProviders := .Configurations}}
+            {{range $keyFrontends, $valueFrontends := $valueProviders.Frontends}}
             <div class="panel panel-primary">
-                <div class="panel-heading">{{$keyFrontends}}</div>
+              <div class="panel-heading">{{$keyFrontends}} - ({{$keyProviders}})</div>
                 <div class="panel-body">
                     <a class="btn btn-info" role="button" data-toggle="collapse" href="#{{$valueFrontends.Backend}}" aria-expanded="false">
                         {{$valueFrontends.Backend}}
@@ -51,14 +52,16 @@
                 </table>
             </div>
             {{end}}
+            {{end}}
         <!-- </div> -->
     </div>
         <div class="col-md-6">
             <!-- <div class="panel-heading">Backends</div>
             <div class="panel-body"> -->
-                {{range $keyBackends, $valueBackends := .Configuration.Backends}}
+            {{range $keyProviders, $valueProviders := .Configurations}}
+                {{range $keyBackends, $valueBackends := $valueProviders.Backends}}
                 <div class="panel panel-primary" id="{{$keyBackends}}">
-                    <div class="panel-heading">{{$keyBackends}}</div>
+                  <div class="panel-heading">{{$keyBackends}}({{$keyProviders}})</div>
                     <div class="panel-body">
                         {{with $valueBackends.LoadBalancer}}
                         <a class="btn btn-info" role="button">
@@ -87,6 +90,7 @@
                     </table>
                 </div>
                 {{end}}
+                {{end}}
             <!-- </div> -->
         </div>
 

traefik.go

@@ -29,7 +29,7 @@ import (
 
 var (
 	globalConfigFile      = kingpin.Arg("conf", "Main configration file.").Default("traefik.toml").String()
-	currentConfiguration = new(Configuration)
+	currentConfigurations = make(configs)
 	metrics               = stats.New()
 	oxyLogger             = &OxyLogger{}
 	templatesRenderer     = render.New(render.Options{
@@ -39,13 +39,20 @@ var (
 	})
 )
 
+type configMessage struct {
+	providerName  string
+	configuration *Configuration
+}
+
+type configs map[string]*Configuration
+
 func main() {
 	runtime.GOMAXPROCS(runtime.NumCPU())
 	kingpin.Parse()
 	fmtlog.SetFlags(fmtlog.Lshortfile | fmtlog.LstdFlags)
 	var srv *manners.GracefulServer
 	var configurationRouter *mux.Router
-	var configurationChan = make(chan *Configuration, 10)
+	var configurationChan = make(chan configMessage, 10)
 	defer close(configurationChan)
 	var sigs = make(chan os.Signal, 1)
 	defer close(sigs)
@@ -84,17 +91,25 @@ func main() {
 
 	// listen new configurations from providers
 	go func() {
+
 		for {
-			configuration := <-configurationChan
+			configMsg := <-configurationChan
-			log.Infof("Configuration receveived %+v", configuration)
+			log.Infof("Configuration receveived from provider %v: %+v", configMsg.providerName, configMsg.configuration)
-			if configuration == nil {
+			if configMsg.configuration == nil {
 				log.Info("Skipping empty configuration")
-			} else if reflect.DeepEqual(currentConfiguration, configuration) {
+			} else if reflect.DeepEqual(currentConfigurations[configMsg.providerName], configMsg.configuration) {
 				log.Info("Skipping same configuration")
 			} else {
-				newConfigurationRouter, err := LoadConfig(configuration, globalConfiguration)
+				// Copy configurations to new map so we don't change current if LoadConfig fails
+				newConfigurations := make(configs)
+				for k, v := range currentConfigurations {
+					newConfigurations[k] = v
+				}
+				newConfigurations[configMsg.providerName] = configMsg.configuration
+
+				newConfigurationRouter, err := LoadConfig(newConfigurations, globalConfiguration)
 				if err == nil {
-					currentConfiguration = configuration
+					currentConfigurations = newConfigurations
 					configurationRouter = newConfigurationRouter
 					oldServer := srv
 					newsrv := prepareServer(configurationRouter, globalConfiguration, oldServer, loggerMiddleware, metrics)
@@ -210,10 +225,11 @@ func prepareServer(router *mux.Router, globalConfiguration *GlobalConfiguration,
 	}
 }
 
-func LoadConfig(configuration *Configuration, globalConfiguration *GlobalConfiguration) (*mux.Router, error) {
+func LoadConfig(configurations configs, globalConfiguration *GlobalConfiguration) (*mux.Router, error) {
 	router := mux.NewRouter()
 	router.NotFoundHandler = http.HandlerFunc(notFoundHandler)
 	backends := map[string]http.Handler{}
+	for _, configuration := range configurations {
 		for frontendName, frontend := range configuration.Frontends {
 			log.Debugf("Creating frontend %s", frontendName)
 			fwd, _ := forward.New(forward.Logger(oxyLogger))
@@ -275,6 +291,7 @@ func LoadConfig(configuration *Configuration, globalConfiguration *GlobalConfigu
 				log.Error("Error building route: %s", err)
 			}
 		}
+	}
 	return router, nil
 }
 

web.go

@@ -17,33 +17,41 @@ type WebProvider struct {
 }
 
 type Page struct {
-	Configuration Configuration
+	Configurations configs
 }
 
-func (provider *WebProvider) Provide(configurationChan chan<- *Configuration) {
+func (provider *WebProvider) Provide(configurationChan chan<- configMessage) {
 	systemRouter := mux.NewRouter()
 	systemRouter.Methods("GET").Path("/").Handler(http.HandlerFunc(GetHTMLConfigHandler))
 	systemRouter.Methods("GET").Path("/health").Handler(http.HandlerFunc(GetHealthHandler))
 	systemRouter.Methods("GET").Path("/api").Handler(http.HandlerFunc(GetConfigHandler))
-	systemRouter.Methods("PUT").Path("/api").Handler(http.HandlerFunc(
+	systemRouter.Methods("GET").Path("/api/{provider}").Handler(http.HandlerFunc(GetConfigHandler))
+	systemRouter.Methods("PUT").Path("/api/{provider}").Handler(http.HandlerFunc(
 		func(rw http.ResponseWriter, r *http.Request) {
+			vars := mux.Vars(r)
+			if vars["provider"] != "web" {
+				rw.WriteHeader(http.StatusBadRequest)
+				fmt.Fprintf(rw, "Only 'web' provider can be updated through the REST API")
+				return
+			}
+
 			configuration := new(Configuration)
 			b, _ := ioutil.ReadAll(r.Body)
 			err := json.Unmarshal(b, configuration)
 			if err == nil {
-				configurationChan <- configuration
+				configurationChan <- configMessage{"web", configuration}
 				GetConfigHandler(rw, r)
 			} else {
 				log.Errorf("Error parsing configuration %+v", err)
 				http.Error(rw, fmt.Sprintf("%+v", err), http.StatusBadRequest)
 			}
 		}))
-	systemRouter.Methods("GET").Path("/api/backends").Handler(http.HandlerFunc(GetBackendsHandler))
+	systemRouter.Methods("GET").Path("/api/{provider}/backends").Handler(http.HandlerFunc(GetBackendsHandler))
-	systemRouter.Methods("GET").Path("/api/backends/{backend}").Handler(http.HandlerFunc(GetBackendHandler))
+	systemRouter.Methods("GET").Path("/api/{provider}/backends/{backend}").Handler(http.HandlerFunc(GetBackendHandler))
-	systemRouter.Methods("GET").Path("/api/backends/{backend}/servers").Handler(http.HandlerFunc(GetServersHandler))
+	systemRouter.Methods("GET").Path("/api/{provider}/backends/{backend}/servers").Handler(http.HandlerFunc(GetServersHandler))
-	systemRouter.Methods("GET").Path("/api/backends/{backend}/servers/{server}").Handler(http.HandlerFunc(GetServerHandler))
+	systemRouter.Methods("GET").Path("/api/{provider}/backends/{backend}/servers/{server}").Handler(http.HandlerFunc(GetServerHandler))
-	systemRouter.Methods("GET").Path("/api/frontends").Handler(http.HandlerFunc(GetFrontendsHandler))
+	systemRouter.Methods("GET").Path("/api/{provider}/frontends").Handler(http.HandlerFunc(GetFrontendsHandler))
-	systemRouter.Methods("GET").Path("/api/frontends/{frontend}").Handler(http.HandlerFunc(GetFrontendHandler))
+	systemRouter.Methods("GET").Path("/api/{provider}/frontends/{frontend}").Handler(http.HandlerFunc(GetFrontendHandler))
 	systemRouter.Methods("GET").PathPrefix("/static/").Handler(http.StripPrefix("/static/", http.FileServer(&assetfs.AssetFS{Asset: Asset, AssetDir: AssetDir, Prefix: "static"})))
 
 	go func() {
@@ -62,11 +70,11 @@ func (provider *WebProvider) Provide(configurationChan chan<- *Configuration) {
 }
 
 func GetConfigHandler(rw http.ResponseWriter, r *http.Request) {
-	templatesRenderer.JSON(rw, http.StatusOK, currentConfiguration)
+	templatesRenderer.JSON(rw, http.StatusOK, currentConfigurations)
 }
 
 func GetHTMLConfigHandler(response http.ResponseWriter, request *http.Request) {
-	templatesRenderer.HTML(response, http.StatusOK, "configuration", Page{Configuration: *currentConfiguration})
+	templatesRenderer.HTML(response, http.StatusOK, "configuration", Page{Configurations: currentConfigurations})
 }
 
 func GetHealthHandler(rw http.ResponseWriter, r *http.Request) {
@@ -74,54 +82,76 @@ func GetHealthHandler(rw http.ResponseWriter, r *http.Request) {
 }
 
 func GetBackendsHandler(rw http.ResponseWriter, r *http.Request) {
-	templatesRenderer.JSON(rw, http.StatusOK, currentConfiguration.Backends)
+	vars := mux.Vars(r)
+	providerId := vars["provider"]
+	if provider, ok := currentConfigurations[providerId]; ok {
+		templatesRenderer.JSON(rw, http.StatusOK, provider.Backends)
+	} else {
+		http.NotFound(rw, r)
+	}
 }
 
 func GetBackendHandler(rw http.ResponseWriter, r *http.Request) {
 	vars := mux.Vars(r)
-	id := vars["backend"]
+	providerId := vars["provider"]
-	if backend, ok := currentConfiguration.Backends[id]; ok {
+	backendId := vars["backend"]
+	if provider, ok := currentConfigurations[providerId]; ok {
+		if backend, ok := provider.Backends[backendId]; ok {
 			templatesRenderer.JSON(rw, http.StatusOK, backend)
-	} else {
+			return
-		http.NotFound(rw, r)
 		}
+	}
+	http.NotFound(rw, r)
 }
 
 func GetFrontendsHandler(rw http.ResponseWriter, r *http.Request) {
-	templatesRenderer.JSON(rw, http.StatusOK, currentConfiguration.Frontends)
+	vars := mux.Vars(r)
+	providerId := vars["provider"]
+	if provider, ok := currentConfigurations[providerId]; ok {
+		templatesRenderer.JSON(rw, http.StatusOK, provider.Frontends)
+	} else {
+		http.NotFound(rw, r)
+	}
 }
 
 func GetFrontendHandler(rw http.ResponseWriter, r *http.Request) {
 	vars := mux.Vars(r)
-	id := vars["frontend"]
+	providerId := vars["provider"]
-	if frontend, ok := currentConfiguration.Frontends[id]; ok {
+	frontendId := vars["frontend"]
+	if provider, ok := currentConfigurations[providerId]; ok {
+		if frontend, ok := provider.Frontends[frontendId]; ok {
 			templatesRenderer.JSON(rw, http.StatusOK, frontend)
-	} else {
+			return
-		http.NotFound(rw, r)
 		}
+	}
+	http.NotFound(rw, r)
 }
 
 func GetServersHandler(rw http.ResponseWriter, r *http.Request) {
 	vars := mux.Vars(r)
-	backend := vars["backend"]
+	providerId := vars["provider"]
-	if backend, ok := currentConfiguration.Backends[backend]; ok {
+	backendId := vars["backend"]
+	if provider, ok := currentConfigurations[providerId]; ok {
+		if backend, ok := provider.Backends[backendId]; ok {
 			templatesRenderer.JSON(rw, http.StatusOK, backend.Servers)
-	} else {
+			return
-		http.NotFound(rw, r)
 		}
+	}
+	http.NotFound(rw, r)
 }
 
 func GetServerHandler(rw http.ResponseWriter, r *http.Request) {
 	vars := mux.Vars(r)
-	backend := vars["backend"]
+	providerId := vars["provider"]
-	server := vars["server"]
+	backendId := vars["backend"]
-	if backend, ok := currentConfiguration.Backends[backend]; ok {
+	serverId := vars["server"]
-		if server, ok := backend.Servers[server]; ok {
+	if provider, ok := currentConfigurations[providerId]; ok {
+		if backend, ok := provider.Backends[backendId]; ok {
+			if server, ok := backend.Servers[serverId]; ok {
 				templatesRenderer.JSON(rw, http.StatusOK, server)
-		} else {
+				return
-			http.NotFound(rw, r)
 			}
-	} else {
-		http.NotFound(rw, r)
 		}
+	}
+	http.NotFound(rw, r)
 }