diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index a5abdb1fb..8eb9d2194 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -26,6 +26,10 @@ "ImportPath": "github.com/alecthomas/units", "Rev": "6b4e7dc5e3143b85ea77909c72caf89416fc2915" }, + { + "ImportPath": "github.com/boltdb/bolt", + "Rev": "51f99c862475898df9773747d3accd05a7ca33c1" + }, { "ImportPath": "github.com/cenkalti/backoff", "Rev": "4dc77674aceaabba2c7e3da25d4c823edfb73f99" @@ -39,6 +43,19 @@ "Comment": "v0.1-70-gc7477ad", "Rev": "c7477ad8e330bef55bf1ebe300cf8aa67c492d1b" }, + { + "ImportPath": "github.com/coreos/go-etcd/etcd", + "Comment": "v2.0.0-11-gcc90c7b", + "Rev": "cc90c7b091275e606ad0ca7102a23fb2072f3f5e" + }, + { + "ImportPath": "github.com/davecgh/go-spew/spew", + "Rev": "2df174808ee097f90d259e432cc04442cf60be21" + }, + { + "ImportPath": "github.com/docker/libkv", + "Rev": "3732f7ff1b56057c3158f10bceb1e79133025373" + }, { "ImportPath": "github.com/docker/distribution", "Comment": "v2.0.0-467-g9038e48", @@ -313,6 +330,10 @@ "ImportPath": "github.com/mailgun/timetools", "Rev": "fd192d755b00c968d312d23f521eb0cdc6f66bd0" }, + { + "ImportPath": "github.com/samuel/go-zookeeper/zk", + "Rev": "fa6674abf3f4580b946a01bf7a1ce4ba8766205b" + }, { "ImportPath": "github.com/opencontainers/runc/libcontainer/user", "Comment": "v0.0.4-21-g4ab1324", diff --git a/Makefile b/Makefile index 2d8c7dec7..e3bd87189 100644 --- a/Makefile +++ b/Makefile @@ -47,6 +47,9 @@ validate-govet: build build: dist docker build -t "$(TRAEFIK_DEV_IMAGE)" -f build.Dockerfile . +build-no-cache: dist + docker build --no-cache -t "$(TRAEFIK_DEV_IMAGE)" -f build.Dockerfile . + shell: build $(DOCKER_RUN_TRAEFIK) /bin/bash diff --git a/boltdb.go b/boltdb.go new file mode 100644 index 000000000..663cd4aed --- /dev/null +++ b/boltdb.go @@ -0,0 +1,14 @@ +package main + +type BoltDbProvider struct { + Watch bool + Endpoint string + Prefix string + Filename string + KvProvider *KvProvider +} + +func (provider *BoltDbProvider) Provide(configurationChan chan<- configMessage) error { + provider.KvProvider = NewBoltDbProvider(provider) + return provider.KvProvider.provide(configurationChan) +} diff --git a/configuration.go b/configuration.go index 25c412848..22fc7b955 100644 --- a/configuration.go +++ b/configuration.go @@ -10,7 +10,6 @@ type GlobalConfiguration struct { GraceTimeOut int64 AccessLogsFile string TraefikLogsFile string - TraefikLogsStdout bool CertFile, KeyFile string LogLevel string Docker *DockerProvider @@ -18,6 +17,9 @@ type GlobalConfiguration struct { Web *WebProvider Marathon *MarathonProvider Consul *ConsulProvider + Etcd *EtcdProvider + Zookeeper *ZookepperProvider + Boltdb *BoltDbProvider } func NewGlobalConfiguration() *GlobalConfiguration { @@ -26,7 +28,6 @@ func NewGlobalConfiguration() *GlobalConfiguration { globalConfiguration.Port = ":80" globalConfiguration.GraceTimeOut = 10 globalConfiguration.LogLevel = "ERROR" - globalConfiguration.TraefikLogsStdout = true return globalConfiguration } diff --git a/consul.go b/consul.go index 344f8f1ec..9b1faa4f3 100644 --- a/consul.go +++ b/consul.go @@ -1,165 +1,14 @@ package main -import ( - "bytes" - "net/http" - "strings" - "text/template" - - "github.com/BurntSushi/toml" - "github.com/BurntSushi/ty/fun" - log "github.com/Sirupsen/logrus" - "github.com/hashicorp/consul/api" -) - -type Key struct { - Value string -} - type ConsulProvider struct { - Watch bool - Endpoint string - Prefix string - Filename string - consulClient *api.Client + Watch bool + Endpoint string + Prefix string + Filename string + KvProvider *KvProvider } -var kvClient *api.KV - -var ConsulFuncMap = template.FuncMap{ - "List": func(keys ...string) []string { - joinedKeys := strings.Join(keys, "") - keysPairs, _, err := kvClient.Keys(joinedKeys, "/", nil) - if err != nil { - log.Error("Error getting keys ", joinedKeys, err) - return nil - } - keysPairs = fun.Filter(func(key string) bool { - if key == joinedKeys { - return false - } - return true - }, keysPairs).([]string) - return keysPairs - }, - "Get": func(keys ...string) string { - joinedKeys := strings.Join(keys, "") - keyPair, _, err := kvClient.Get(joinedKeys, nil) - if err != nil { - log.Error("Error getting key ", joinedKeys, err) - return "" - } else if keyPair == nil { - return "" - } - return string(keyPair.Value) - }, - "Last": func(key string) string { - splittedKey := strings.Split(key, "/") - return splittedKey[len(splittedKey)-2] - }, -} - -func NewConsulProvider() *ConsulProvider { - consulProvider := new(ConsulProvider) - // default values - consulProvider.Watch = true - consulProvider.Prefix = "traefik" - - return consulProvider -} - -func (provider *ConsulProvider) Provide(configurationChan chan<- configMessage) { - config := &api.Config{ - Address: provider.Endpoint, - Scheme: "http", - HttpClient: http.DefaultClient, - } - consulClient, _ := api.NewClient(config) - provider.consulClient = consulClient - if provider.Watch { - keypairs, meta, err := consulClient.KV().Keys("", "", nil) - if keypairs == nil { - log.Error("Key was not found") - } else if err != nil { - log.Error("Error connecting to consul %s", err) - } else { - var waitIndex uint64 - waitIndex = meta.LastIndex - go func() { - for { - opts := api.QueryOptions{ - WaitIndex: waitIndex, - } - keypairs, meta, err := consulClient.KV().Keys("", "", &opts) - if keypairs == nil { - log.Error("Key was not found") - } else if err != nil { - log.Error("Error connecting to consul %s", err) - } else { - waitIndex = meta.LastIndex - configuration := provider.loadConsulConfig() - if configuration != nil { - configurationChan <- configMessage{"consul", configuration} - } - } - } - }() - } - } - configuration := provider.loadConsulConfig() - configurationChan <- configMessage{"consul", configuration} -} - -func (provider *ConsulProvider) loadConsulConfig() *Configuration { - configuration := new(Configuration) - services := []*api.CatalogService{} - kvClient = provider.consulClient.KV() - - servicesName, _, _ := provider.consulClient.Catalog().Services(nil) - for serviceName := range servicesName { - catalogServices, _, _ := provider.consulClient.Catalog().Service(serviceName, "", nil) - for _, catalogService := range catalogServices { - services = append(services, catalogService) - } - } - - templateObjects := struct { - Services []*api.CatalogService - }{ - services, - } - - tmpl := template.New(provider.Filename).Funcs(ConsulFuncMap) - if len(provider.Filename) > 0 { - _, err := tmpl.ParseFiles(provider.Filename) - if err != nil { - log.Error("Error reading file", err) - return nil - } - } else { - buf, err := Asset("providerTemplates/consul.tmpl") - if err != nil { - log.Error("Error reading file", err) - } - _, err = tmpl.Parse(string(buf)) - if err != nil { - log.Error("Error reading file", err) - return nil - } - } - - var buffer bytes.Buffer - - err := tmpl.Execute(&buffer, templateObjects) - if err != nil { - log.Error("Error with consul template:", err) - return nil - } - - if _, err := toml.Decode(buffer.String(), configuration); err != nil { - log.Error("Error creating consul configuration:", err) - return nil - } - - return configuration +func (provider *ConsulProvider) Provide(configurationChan chan<- configMessage) error { + provider.KvProvider = NewConsulProvider(provider) + return provider.KvProvider.provide(configurationChan) } diff --git a/docker.go b/docker.go index 4ae6c2209..5c73586c8 100644 --- a/docker.go +++ b/docker.go @@ -22,15 +22,6 @@ type DockerProvider struct { Domain string } -func NewDockerProvider() *DockerProvider { - dockerProvider := new(DockerProvider) - // default - dockerProvider.Watch = true - dockerProvider.Domain = "traefik" - - return dockerProvider -} - var DockerFuncMap = template.FuncMap{ "getBackend": func(container docker.Container) string { for key, value := range container.Config.Labels { @@ -65,13 +56,15 @@ var DockerFuncMap = template.FuncMap{ "getHost": getHost, } -func (provider *DockerProvider) Provide(configurationChan chan<- configMessage) { +func (provider *DockerProvider) Provide(configurationChan chan<- configMessage) error { if dockerClient, err := docker.NewClient(provider.Endpoint); err != nil { - log.Fatalf("Failed to create a client for docker, error: %s", err) + log.Errorf("Failed to create a client for docker, error: %s", err) + return err } else { err := dockerClient.Ping() if err != nil { - log.Fatalf("Docker connection error %+v", err) + log.Errorf("Docker connection error %+v", err) + return err } log.Debug("Docker connection established") if provider.Watch { @@ -108,6 +101,7 @@ func (provider *DockerProvider) Provide(configurationChan chan<- configMessage) configuration := provider.loadDockerConfig(dockerClient) configurationChan <- configMessage{"docker", configuration} } + return nil } func (provider *DockerProvider) loadDockerConfig(dockerClient *docker.Client) *Configuration { diff --git a/etcd.go b/etcd.go new file mode 100644 index 000000000..9c0bd68d7 --- /dev/null +++ b/etcd.go @@ -0,0 +1,14 @@ +package main + +type EtcdProvider struct { + Watch bool + Endpoint string + Prefix string + Filename string + KvProvider *KvProvider +} + +func (provider *EtcdProvider) Provide(configurationChan chan<- configMessage) error { + provider.KvProvider = NewEtcdProvider(provider) + return provider.KvProvider.provide(configurationChan) +} diff --git a/file.go b/file.go index 4cdd4704a..c567a5038 100644 --- a/file.go +++ b/file.go @@ -15,30 +15,21 @@ type FileProvider struct { Filename string } -func NewFileProvider() *FileProvider { - fileProvider := new(FileProvider) - // default values - fileProvider.Watch = true - - return fileProvider -} - -func (provider *FileProvider) Provide(configurationChan chan<- configMessage) { +func (provider *FileProvider) Provide(configurationChan chan<- configMessage) error { watcher, err := fsnotify.NewWatcher() if err != nil { log.Error("Error creating file watcher", err) - return + return err } defer watcher.Close() file, err := os.Open(provider.Filename) if err != nil { log.Error("Error opening file", err) - return + return err } defer file.Close() - done := make(chan bool) // Process events go func() { for { @@ -63,12 +54,12 @@ func (provider *FileProvider) Provide(configurationChan chan<- configMessage) { if err != nil { log.Error("Error adding file watcher", err) - return + return err } configuration := provider.LoadFileConfig(file.Name()) configurationChan <- configMessage{"file", configuration} - <-done + return nil } func (provider *FileProvider) LoadFileConfig(filename string) *Configuration { diff --git a/integration/basic_test.go b/integration/basic_test.go index 4ad3aafca..90abe2340 100644 --- a/integration/basic_test.go +++ b/integration/basic_test.go @@ -15,14 +15,14 @@ func (s *SimpleSuite) TestNoOrInexistentConfigShouldFail(c *check.C) { output, err := cmd.CombinedOutput() c.Assert(err, checker.NotNil) - c.Assert(string(output), checker.Contains, "Error reading file open traefik.toml: no such file or directory") + c.Assert(string(output), checker.Contains, "Error reading file: open traefik.toml: no such file or directory") nonExistentFile := "non/existent/file.toml" cmd = exec.Command(traefikBinary, nonExistentFile) output, err = cmd.CombinedOutput() c.Assert(err, checker.NotNil) - c.Assert(string(output), checker.Contains, fmt.Sprintf("Error reading file open %s: no such file or directory", nonExistentFile)) + c.Assert(string(output), checker.Contains, fmt.Sprintf("Error reading file: open %s: no such file or directory", nonExistentFile)) } func (s *SimpleSuite) TestInvalidConfigShouldFail(c *check.C) { @@ -30,7 +30,7 @@ func (s *SimpleSuite) TestInvalidConfigShouldFail(c *check.C) { output, err := cmd.CombinedOutput() c.Assert(err, checker.NotNil) - c.Assert(string(output), checker.Contains, "Error reading file Near line 1") + c.Assert(string(output), checker.Contains, "Error reading file: Near line 1") } func (s *SimpleSuite) TestSimpleDefaultConfig(c *check.C) { diff --git a/integration/resources/compose/consul.yml b/integration/resources/compose/consul.yml index 83f30b7a9..a2036443d 100644 --- a/integration/resources/compose/consul.yml +++ b/integration/resources/compose/consul.yml @@ -10,12 +10,4 @@ consul: - "8301" - "8301/udp" - "8302" - - "8302/udp" - -registrator: - image: gliderlabs/registrator:master - command: -internal consulkv://consul:8500/traefik - volumes: - - /var/run/docker.sock:/tmp/docker.sock - links: - - consul \ No newline at end of file + - "8302/udp" \ No newline at end of file diff --git a/kv.go b/kv.go new file mode 100644 index 000000000..4d3f7dd82 --- /dev/null +++ b/kv.go @@ -0,0 +1,195 @@ +/* +Copyright +*/ +package main + +import ( + "bytes" + "github.com/docker/libkv" + "github.com/docker/libkv/store/boltdb" + "github.com/docker/libkv/store/consul" + "github.com/docker/libkv/store/etcd" + "github.com/docker/libkv/store/zookeeper" + "strings" + "text/template" + + "errors" + "github.com/BurntSushi/toml" + "github.com/BurntSushi/ty/fun" + log "github.com/Sirupsen/logrus" + "github.com/docker/libkv/store" + "time" +) + +type KvProvider struct { + Watch bool + Endpoint string + Prefix string + Filename string + StoreType store.Backend + kvclient store.Store +} + +func NewConsulProvider(provider *ConsulProvider) *KvProvider { + kvProvider := new(KvProvider) + kvProvider.Watch = provider.Watch + kvProvider.Endpoint = provider.Endpoint + kvProvider.Prefix = provider.Prefix + kvProvider.Filename = provider.Filename + kvProvider.StoreType = store.CONSUL + return kvProvider +} + +func NewEtcdProvider(provider *EtcdProvider) *KvProvider { + kvProvider := new(KvProvider) + kvProvider.Watch = provider.Watch + kvProvider.Endpoint = provider.Endpoint + kvProvider.Prefix = provider.Prefix + kvProvider.Filename = provider.Filename + kvProvider.StoreType = store.ETCD + return kvProvider +} + +func NewZkProvider(provider *ZookepperProvider) *KvProvider { + kvProvider := new(KvProvider) + kvProvider.Watch = provider.Watch + kvProvider.Endpoint = provider.Endpoint + kvProvider.Prefix = provider.Prefix + kvProvider.Filename = provider.Filename + kvProvider.StoreType = store.ZK + return kvProvider +} + +func NewBoltDbProvider(provider *BoltDbProvider) *KvProvider { + kvProvider := new(KvProvider) + kvProvider.Watch = provider.Watch + kvProvider.Endpoint = provider.Endpoint + kvProvider.Prefix = provider.Prefix + kvProvider.Filename = provider.Filename + kvProvider.StoreType = store.BOLTDB + return kvProvider +} + +func (provider *KvProvider) provide(configurationChan chan<- configMessage) error { + switch provider.StoreType { + case store.CONSUL: + consul.Register() + case store.ETCD: + etcd.Register() + case store.ZK: + zookeeper.Register() + case store.BOLTDB: + boltdb.Register() + default: + return errors.New("Invalid kv store: " + string(provider.StoreType)) + } + kv, err := libkv.NewStore( + provider.StoreType, + []string{provider.Endpoint}, + &store.Config{ + ConnectionTimeout: 30 * time.Second, + }, + ) + if err != nil { + return err + } + if _, err := kv.List(""); err != nil { + return err + } + provider.kvclient = kv + if provider.Watch { + stopCh := make(chan struct{}) + chanKeys, err := kv.WatchTree(provider.Prefix, stopCh) + if err != nil { + return err + } + go func() { + for { + <-chanKeys + configuration := provider.loadConfig() + if configuration != nil { + configurationChan <- configMessage{string(provider.StoreType), configuration} + } + defer close(stopCh) + } + }() + } + configuration := provider.loadConfig() + configurationChan <- configMessage{string(provider.StoreType), configuration} + return nil +} + +func (provider *KvProvider) loadConfig() *Configuration { + configuration := new(Configuration) + templateObjects := struct { + Prefix string + }{ + provider.Prefix, + } + var KvFuncMap = template.FuncMap{ + "List": func(keys ...string) []string { + joinedKeys := strings.Join(keys, "") + keysPairs, err := provider.kvclient.List(joinedKeys) + if err != nil { + log.Error("Error getting keys: ", joinedKeys, err) + return nil + } + directoryKeys := make(map[string]string) + for _, key := range keysPairs { + directory := strings.Split(strings.TrimPrefix(key.Key, strings.TrimPrefix(joinedKeys, "/")), "/")[0] + directoryKeys[directory] = joinedKeys + directory + } + return fun.Values(directoryKeys).([]string) + }, + "Get": func(keys ...string) string { + joinedKeys := strings.Join(keys, "") + keyPair, err := provider.kvclient.Get(joinedKeys) + if err != nil { + log.Debug("Error getting key: ", joinedKeys, err) + return "" + } else if keyPair == nil { + return "" + } + return string(keyPair.Value) + }, + "Last": func(key string) string { + splittedKey := strings.Split(key, "/") + return splittedKey[len(splittedKey)-1] + }, + } + + tmpl := template.New(provider.Filename).Funcs(KvFuncMap) + if len(provider.Filename) > 0 { + _, err := tmpl.ParseFiles(provider.Filename) + if err != nil { + log.Error("Error reading file", err) + return nil + } + } else { + buf, err := Asset("providerTemplates/kv.tmpl") + if err != nil { + log.Error("Error reading file", err) + } + _, err = tmpl.Parse(string(buf)) + if err != nil { + log.Error("Error reading file", err) + return nil + } + } + + var buffer bytes.Buffer + + err := tmpl.Execute(&buffer, templateObjects) + if err != nil { + log.Error("Error with kv template:", err) + return nil + } + + if _, err := toml.Decode(buffer.String(), configuration); err != nil { + log.Error("Error creating kv configuration:", err) + log.Error(buffer.String()) + return nil + } + + return configuration +} diff --git a/marathon.go b/marathon.go index 9cb7fe12e..fc4f80e7d 100644 --- a/marathon.go +++ b/marathon.go @@ -21,16 +21,6 @@ type MarathonProvider struct { NetworkInterface string } -func NewMarathonProvider() *MarathonProvider { - marathonProvider := new(MarathonProvider) - // default values - marathonProvider.Watch = true - marathonProvider.Domain = "traefik" - marathonProvider.NetworkInterface = "eth0" - - return marathonProvider -} - var MarathonFuncMap = template.FuncMap{ "getPort": func(task marathon.Task) string { for _, port := range task.Ports { @@ -67,14 +57,14 @@ var MarathonFuncMap = template.FuncMap{ }, } -func (provider *MarathonProvider) Provide(configurationChan chan<- configMessage) { +func (provider *MarathonProvider) Provide(configurationChan chan<- configMessage) error { config := marathon.NewDefaultConfig() config.URL = provider.Endpoint config.EventsInterface = provider.NetworkInterface client, err := marathon.NewClient(config) if err != nil { log.Errorf("Failed to create a client for marathon, error: %s", err) - return + return err } provider.marathonClient = client update := make(marathon.EventsChannel, 5) @@ -97,6 +87,7 @@ func (provider *MarathonProvider) Provide(configurationChan chan<- configMessage configuration := provider.loadMarathonConfig() configurationChan <- configMessage{"marathon", configuration} + return nil } func (provider *MarathonProvider) loadMarathonConfig() *Configuration { diff --git a/provider.go b/provider.go index 680341827..df0bb7399 100644 --- a/provider.go +++ b/provider.go @@ -1,5 +1,5 @@ package main type Provider interface { - Provide(configurationChan chan<- configMessage) + Provide(configurationChan chan<- configMessage) error } diff --git a/providerTemplates/consul.tmpl b/providerTemplates/kv.tmpl similarity index 71% rename from providerTemplates/consul.tmpl rename to providerTemplates/kv.tmpl index e535e414e..b1833c91f 100644 --- a/providerTemplates/consul.tmpl +++ b/providerTemplates/kv.tmpl @@ -1,17 +1,17 @@ -{{$frontends := "frontends/" | List }} -{{$backends := "backends/" | List }} +{{$frontends := List .Prefix "/frontends/" }} +{{$backends := List .Prefix "/backends/"}} {{range $backends}} {{$backend := .}} -{{$servers := "servers/" | List $backend }} +{{$servers := List $backend "/servers/" }} -{{$circuitBreaker := Get . "circuitbreaker/" "expression"}} +{{$circuitBreaker := Get . "/circuitbreaker/" "expression"}} {{with $circuitBreaker}} [backends.{{Last $backend}}.circuitBreaker] expression = "{{$circuitBreaker}}" {{end}} -{{$loadBalancer := Get . "loadbalancer/" "method"}} +{{$loadBalancer := Get . "/loadbalancer/" "method"}} {{with $loadBalancer}} [backends.{{Last $backend}}.loadBalancer] method = "{{$loadBalancer}}" @@ -28,7 +28,7 @@ {{$frontend := Last .}} [frontends.{{$frontend}}] backend = "{{Get . "/backend"}}" - {{$routes := "routes/" | List .}} + {{$routes := List . "/routes/"}} {{range $routes}} [frontends.{{$frontend}}.routes.{{Last .}}] rule = "{{Get . "/rule"}}" diff --git a/templates/configuration.tmpl b/templates/configuration.tmpl index 5112fc12c..5b29dc38f 100644 --- a/templates/configuration.tmpl +++ b/templates/configuration.tmpl @@ -61,7 +61,7 @@ {{range $keyProviders, $valueProviders := .Configurations}} {{range $keyBackends, $valueBackends := $valueProviders.Backends}}