From 87eac1dc1a65dc3a5dc9f4686736c55a4770904b Mon Sep 17 00:00:00 2001 From: Phil Kates Date: Fri, 2 Dec 2016 11:35:19 -0800 Subject: [PATCH] Fix deadlock in k8s provider On a reasonably sized cluster: 63 nodes 87 services 90 endpoints The initialization of the k8s provider would hang. I tracked this down to the ResourceEventHandlerFuncs. Once you reach the channel buffer size (10) the k8s Informer gets stuck. You can't read or write messages to the channel anymore. I think this is probably a lock issue somewhere in k8s but the more reasonable solution for the traefik usecase is to just drop events when the queue is full since we only use the events for signalling, not their content, thus dropping an event doesn't matter. --- provider/k8s/client.go | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/provider/k8s/client.go b/provider/k8s/client.go index 14420b74f..0f44e5e31 100644 --- a/provider/k8s/client.go +++ b/provider/k8s/client.go @@ -110,14 +110,22 @@ func (c *clientImpl) WatchIngresses(labelSelector labels.Selector, stopCh <-chan return watchCh } -func newResourceEventHandlerFuncs(events chan interface{}) cache.ResourceEventHandlerFuncs { - - return cache.ResourceEventHandlerFuncs{ - AddFunc: func(obj interface{}) { events <- obj }, - UpdateFunc: func(old, new interface{}) { events <- new }, - DeleteFunc: func(obj interface{}) { events <- obj }, +// eventHandlerFunc will pass the obj on to the events channel or drop it +// This is so passing the events along won't block in the case of high volume +// The events are only used for signalling anyway so dropping a few is ok +func eventHandlerFunc(events chan interface{}, obj interface{}) { + select { + case events <- obj: + default: } +} +func newResourceEventHandlerFuncs(events chan interface{}) cache.ResourceEventHandlerFuncs { + return cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { eventHandlerFunc(events, obj) }, + UpdateFunc: func(old, new interface{}) { eventHandlerFunc(events, new) }, + DeleteFunc: func(obj interface{}) { eventHandlerFunc(events, obj) }, + } } // GetService returns the named service from the named namespace