baalajimaestro/traefik
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

[kubernetes] ignore empty endpoint changes

Browse source
This commit is contained in:
Henning 2021-04-29 16:20:03 +02:00 committed by GitHub
parent 0624cefc10
commit ab71dad51a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 707 additions and 54 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

29
pkg/provider/kubernetes/crd/client.go
View file

@ -12,6 +12,7 @@ import (
"github.com/traefik/traefik/v2/pkg/provider/kubernetes/crd/generated/clientset/versioned"
"github.com/traefik/traefik/v2/pkg/provider/kubernetes/crd/generated/informers/externalversions"
"github.com/traefik/traefik/v2/pkg/provider/kubernetes/crd/traefik/v1alpha1"
"github.com/traefik/traefik/v2/pkg/provider/kubernetes/k8s"
"github.com/traefik/traefik/v2/pkg/version"
corev1 "k8s.io/api/core/v1"
kubeerror "k8s.io/apimachinery/pkg/api/errors"
@ -25,22 +26,6 @@ import (
const resyncPeriod = 10 * time.Minute
type resourceEventHandler struct {
ev chan<- interface{}
}
func (reh *resourceEventHandler) OnAdd(obj interface{}) {
eventHandlerFunc(reh.ev, obj)
}
func (reh *resourceEventHandler) OnUpdate(oldObj, newObj interface{}) {
eventHandlerFunc(reh.ev, newObj)
}
func (reh *resourceEventHandler) OnDelete(obj interface{}) {
eventHandlerFunc(reh.ev, obj)
}
// Client is a client for the Provider master.
// WatchAll starts the watch of the Provider resources and updates the stores.
// The stores can then be accessed via the Get* functions.
@ -160,7 +145,7 @@ func newExternalClusterClient(endpoint, token, caFilePath string) (*clientWrappe
// WatchAll starts namespace-specific controllers for all relevant kinds.
func (c *clientWrapper) WatchAll(namespaces []string, stopCh <-chan struct{}) (<-chan interface{}, error) {
eventCh := make(chan interface{}, 1)
eventHandler := &resourceEventHandler{ev: eventCh}
eventHandler := &k8s.ResourceEventHandler{Ev: eventCh}
if len(namespaces) == 0 {
namespaces = []string{metav1.NamespaceAll}
@ -402,16 +387,6 @@ func (c *clientWrapper) lookupNamespace(ns string) string {
return ns
}
// eventHandlerFunc will pass the obj on to the events channel or drop it.
// This is so passing the events along won't block in the case of high volume.
// The events are only used for signaling anyway so dropping a few is ok.
func eventHandlerFunc(events chan<- interface{}, obj interface{}) {
select {
case events <- obj:
default:
}
}
// translateNotFoundError will translate a "not found" error to a boolean return
// value which indicates if the resource exists and a nil error.
func translateNotFoundError(err error) (bool, error) {

29
pkg/provider/kubernetes/ingress/client.go
View file

@ -11,6 +11,7 @@ import (
"github.com/hashicorp/go-version"
"github.com/traefik/traefik/v2/pkg/log"
"github.com/traefik/traefik/v2/pkg/provider/kubernetes/k8s"
traefikversion "github.com/traefik/traefik/v2/pkg/version"
corev1 "k8s.io/api/core/v1"
networkingv1 "k8s.io/api/networking/v1"
@ -34,22 +35,6 @@ type marshaler interface {
Marshal() ([]byte, error)
}
type resourceEventHandler struct {
ev chan<- interface{}
}
func (reh *resourceEventHandler) OnAdd(obj interface{}) {
eventHandlerFunc(reh.ev, obj)
}
func (reh *resourceEventHandler) OnUpdate(oldObj, newObj interface{}) {
eventHandlerFunc(reh.ev, newObj)
}
func (reh *resourceEventHandler) OnDelete(obj interface{}) {
eventHandlerFunc(reh.ev, obj)
}
// Client is a client for the Provider master.
// WatchAll starts the watch of the Provider resources and updates the stores.
// The stores can then be accessed via the Get* functions.
@ -151,7 +136,7 @@ func newClientImpl(clientset kubernetes.Interface) *clientWrapper {
// WatchAll starts namespace-specific controllers for all relevant kinds.
func (c *clientWrapper) WatchAll(namespaces []string, stopCh <-chan struct{}) (<-chan interface{}, error) {
eventCh := make(chan interface{}, 1)
eventHandler := &resourceEventHandler{eventCh}
eventHandler := &k8s.ResourceEventHandler{Ev: eventCh}
if len(namespaces) == 0 {
namespaces = []string{metav1.NamespaceAll}
@ -508,16 +493,6 @@ func (c *clientWrapper) GetServerVersion() (*version.Version, error) {
return version.NewVersion(serverVersion.GitVersion)
}
// eventHandlerFunc will pass the obj on to the events channel or drop it.
// This is so passing the events along won't block in the case of high volume.
// The events are only used for signaling anyway so dropping a few is ok.
func eventHandlerFunc(events chan<- interface{}, obj interface{}) {
select {
case events <- obj:
default:
}
}
// translateNotFoundError will translate a "not found" error to a boolean return
// value which indicates if the resource exists and a nil error.
func translateNotFoundError(err error) (bool, error) {

100
pkg/provider/kubernetes/ingress/client_test.go
View file

@ -1,6 +1,7 @@
package ingress
import (
"context"
"fmt"
"testing"
"time"
@ -190,6 +191,105 @@ func TestClientIgnoresHelmOwnedSecrets(t *testing.T) {
assert.False(t, found)
}
func TestClientIgnoresEmptyEndpointUpdates(t *testing.T) {
emptyEndpoint := &corev1.Endpoints{
ObjectMeta: metav1.ObjectMeta{
Name: "empty-endpoint",
Namespace: "test",
ResourceVersion: "1244",
Annotations: map[string]string{
"test-annotation": "_",
},
},
}
filledEndpoint := &corev1.Endpoints{
ObjectMeta: metav1.ObjectMeta{
Name: "filled-endpoint",
Namespace: "test",
ResourceVersion: "1234",
},
Subsets: []corev1.EndpointSubset{{
Addresses: []corev1.EndpointAddress{{
IP: "10.13.37.1",
}},
Ports: []corev1.EndpointPort{{
Name: "testing",
Port: 1337,
Protocol: "tcp",
}},
}},
}
kubeClient := kubefake.NewSimpleClientset(emptyEndpoint, filledEndpoint)
discovery, _ := kubeClient.Discovery().(*fakediscovery.FakeDiscovery)
discovery.FakedServerVersion = &version.Info{
GitVersion: "v1.19",
}
client := newClientImpl(kubeClient)
stopCh := make(chan struct{})
eventCh, err := client.WatchAll(nil, stopCh)
require.NoError(t, err)
select {
case event := <-eventCh:
ep, ok := event.(*corev1.Endpoints)
require.True(t, ok)
assert.True(t, ep.Name == "empty-endpoint" || ep.Name == "filled-endpoint")
case <-time.After(50 * time.Millisecond):
assert.Fail(t, "expected to receive event for endpoints")
}
emptyEndpoint, err = kubeClient.CoreV1().Endpoints("test").Get(context.TODO(), "empty-endpoint", metav1.GetOptions{})
assert.NoError(t, err)
// Update endpoint annotation and resource version (apparently not done by fake client itself)
// to show an update that should not trigger an update event on our eventCh.
// This reflects the behavior of kubernetes controllers which use endpoint annotations for leader election.
emptyEndpoint.Annotations["test-annotation"] = "___"
emptyEndpoint.ResourceVersion = "1245"
_, err = kubeClient.CoreV1().Endpoints("test").Update(context.TODO(), emptyEndpoint, metav1.UpdateOptions{})
require.NoError(t, err)
select {
case event := <-eventCh:
ep, ok := event.(*corev1.Endpoints)
require.True(t, ok)
assert.Fail(t, "didn't expect to receive event for empty endpoint update", ep.Name)
case <-time.After(50 * time.Millisecond):
}
filledEndpoint, err = kubeClient.CoreV1().Endpoints("test").Get(context.TODO(), "filled-endpoint", metav1.GetOptions{})
assert.NoError(t, err)
filledEndpoint.Subsets[0].Addresses[0].IP = "10.13.37.2"
filledEndpoint.ResourceVersion = "1235"
_, err = kubeClient.CoreV1().Endpoints("test").Update(context.TODO(), filledEndpoint, metav1.UpdateOptions{})
require.NoError(t, err)
select {
case event := <-eventCh:
ep, ok := event.(*corev1.Endpoints)
require.True(t, ok)
assert.Equal(t, "filled-endpoint", ep.Name)
case <-time.After(50 * time.Millisecond):
assert.Fail(t, "expected to receive event for filled endpoint")
}
select {
case <-eventCh:
assert.Fail(t, "received more than one event")
case <-time.After(50 * time.Millisecond):
}
}
func TestClientUsesCorrectServerVersion(t *testing.T) {
ingressV1Beta := &v1beta1.Ingress{
ObjectMeta: metav1.ObjectMeta{

114
pkg/provider/kubernetes/k8s/event_handler.go Normal file
View file

@ -0,0 +1,114 @@
package k8s
import (
"github.com/traefik/traefik/v2/pkg/log"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
// ResourceEventHandler handles Add, Update or Delete Events for resources.
type ResourceEventHandler struct {
Ev chan<- interface{}
}
// OnAdd is called on Add Events.
func (reh *ResourceEventHandler) OnAdd(obj interface{}) {
eventHandlerFunc(reh.Ev, obj)
}
// OnUpdate is called on Update Events.
// Ignores useless changes.
func (reh *ResourceEventHandler) OnUpdate(oldObj, newObj interface{}) {
if objChanged(oldObj, newObj) {
eventHandlerFunc(reh.Ev, newObj)
}
}
// OnDelete is called on Delete Events.
func (reh *ResourceEventHandler) OnDelete(obj interface{}) {
eventHandlerFunc(reh.Ev, obj)
}
// eventHandlerFunc will pass the obj on to the events channel or drop it.
// This is so passing the events along won't block in the case of high volume.
// The events are only used for signaling anyway so dropping a few is ok.
func eventHandlerFunc(events chan<- interface{}, obj interface{}) {
select {
case events <- obj:
default:
}
}
func objChanged(oldObj, newObj interface{}) bool {
if oldObj == nil || newObj == nil {
return true
}
if oldObj.(metav1.Object).GetResourceVersion() == newObj.(metav1.Object).GetResourceVersion() {
return false
}
if _, ok := oldObj.(*corev1.Endpoints); ok {
if endpointsChanged(oldObj.(*corev1.Endpoints), newObj.(*corev1.Endpoints)) {
return true
}
}
log.WithoutContext().Debugf("endpoint %s has no changes, ignoring", newObj.(*corev1.Endpoints).Name)
return false
}
func endpointsChanged(a, b *corev1.Endpoints) bool {
if len(a.Subsets) != len(b.Subsets) {
return true
}
for i, sa := range a.Subsets {
sb := b.Subsets[i]
if subsetsChanged(sa, sb) {
return true
}
}
return false
}
func subsetsChanged(sa, sb corev1.EndpointSubset) bool {
if len(sa.Addresses) != len(sb.Addresses) {
return true
}
if len(sa.Ports) != len(sb.Ports) {
return true
}
// in Addresses and Ports, we should be able to rely on
// these being sorted and able to be compared
// they are supposed to be in a canonical format
for addr, aaddr := range sa.Addresses {
baddr := sb.Addresses[addr]
if aaddr.IP != baddr.IP {
return true
}
if aaddr.Hostname != baddr.Hostname {
return true
}
}
for port, aport := range sa.Ports {
bport := sb.Ports[port]
if aport.Name != bport.Name {
return true
}
if aport.Port != bport.Port {
return true
}
if aport.Protocol != bport.Protocol {
return true
}
}
return false
}

489
pkg/provider/kubernetes/k8s/event_handler_test.go Normal file
View file

@ -0,0 +1,489 @@
package k8s
import (
"testing"
"github.com/stretchr/testify/assert"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
func Test_detectChanges(t *testing.T) {
tests := []struct {
name string
oldObj interface{}
newObj interface{}
want bool
}{
{
name: "With nil values",
want: true,
},
{
name: "With empty endpoints",
oldObj: &corev1.Endpoints{},
newObj: &corev1.Endpoints{},
},
{
name: "With old nil",
newObj: &corev1.Endpoints{},
want: true,
},
{
name: "With new nil",
oldObj: &corev1.Endpoints{},
want: true,
},
{
name: "With same version",
oldObj: &corev1.Endpoints{
ObjectMeta: metav1.ObjectMeta{
ResourceVersion: "1",
},
},
newObj: &corev1.Endpoints{
ObjectMeta: metav1.ObjectMeta{
ResourceVersion: "1",
},
},
},
{
name: "With different version",
oldObj: &corev1.Endpoints{
ObjectMeta: metav1.ObjectMeta{
ResourceVersion: "1",
},
},
newObj: &corev1.Endpoints{
ObjectMeta: metav1.ObjectMeta{
ResourceVersion: "2",
},
},
},
{
name: "With same annotations",
oldObj: &corev1.Endpoints{
ObjectMeta: metav1.ObjectMeta{
ResourceVersion: "1",
Annotations: map[string]string{
"test-annotation": "_",
},
},
},
newObj: &corev1.Endpoints{
ObjectMeta: metav1.ObjectMeta{
ResourceVersion: "2",
Annotations: map[string]string{
"test-annotation": "_",
},
},
},
},
{
name: "With different annotations",
oldObj: &corev1.Endpoints{
ObjectMeta: metav1.ObjectMeta{
ResourceVersion: "1",
Annotations: map[string]string{
"test-annotation": "V",
},
},
},
newObj: &corev1.Endpoints{
ObjectMeta: metav1.ObjectMeta{
ResourceVersion: "2",
Annotations: map[string]string{
"test-annotation": "X",
},
},
},
},
{
name: "With same subsets",
oldObj: &corev1.Endpoints{
ObjectMeta: metav1.ObjectMeta{
ResourceVersion: "1",
},
Subsets: []corev1.EndpointSubset{},
},
newObj: &corev1.Endpoints{
ObjectMeta: metav1.ObjectMeta{
ResourceVersion: "1",
},
Subsets: []corev1.EndpointSubset{},
},
},
{
name: "With different len of subsets",
oldObj: &corev1.Endpoints{
ObjectMeta: metav1.ObjectMeta{
ResourceVersion: "1",
},
Subsets: []corev1.EndpointSubset{},
},
newObj: &corev1.Endpoints{
ObjectMeta: metav1.ObjectMeta{
ResourceVersion: "2",
},
Subsets: []corev1.EndpointSubset{{}},
},
want: true,
},
{
name: "With same subsets with same len of addresses",
oldObj: &corev1.Endpoints{
ObjectMeta: metav1.ObjectMeta{
ResourceVersion: "1",
},
Subsets: []corev1.EndpointSubset{{
Addresses: []corev1.EndpointAddress{},
}},
},
newObj: &corev1.Endpoints{
ObjectMeta: metav1.ObjectMeta{
ResourceVersion: "2",
},
Subsets: []corev1.EndpointSubset{{
Addresses: []corev1.EndpointAddress{},
}},
},
},
{
name: "With same subsets with different len of addresses",
oldObj: &corev1.Endpoints{
ObjectMeta: metav1.ObjectMeta{
ResourceVersion: "1",
},
Subsets: []corev1.EndpointSubset{{
Addresses: []corev1.EndpointAddress{},
}},
},
newObj: &corev1.Endpoints{
ObjectMeta: metav1.ObjectMeta{
ResourceVersion: "2",
},
Subsets: []corev1.EndpointSubset{{
Addresses: []corev1.EndpointAddress{{}},
}},
},
want: true,
},
{
name: "With same subsets with same len of ports",
oldObj: &corev1.Endpoints{
ObjectMeta: metav1.ObjectMeta{
ResourceVersion: "1",
},
Subsets: []corev1.EndpointSubset{{
Ports: []corev1.EndpointPort{},
}},
},
newObj: &corev1.Endpoints{
ObjectMeta: metav1.ObjectMeta{
ResourceVersion: "2",
},
Subsets: []corev1.EndpointSubset{{
Ports: []corev1.EndpointPort{},
}},
},
},
{
name: "With same subsets with different len of ports",
oldObj: &corev1.Endpoints{
ObjectMeta: metav1.ObjectMeta{
ResourceVersion: "1",
},
Subsets: []corev1.EndpointSubset{{
Ports: []corev1.EndpointPort{},
}},
},
newObj: &corev1.Endpoints{
ObjectMeta: metav1.ObjectMeta{
ResourceVersion: "2",
},
Subsets: []corev1.EndpointSubset{{
Ports: []corev1.EndpointPort{{}},
}},
},
want: true,
},
{
name: "With same subsets with same len of addresses with same ip",
oldObj: &corev1.Endpoints{
ObjectMeta: metav1.ObjectMeta{
ResourceVersion: "1",
},
Subsets: []corev1.EndpointSubset{{
Addresses: []corev1.EndpointAddress{{
IP: "10.10.10.10",
}},
}},
},
newObj: &corev1.Endpoints{
ObjectMeta: metav1.ObjectMeta{
ResourceVersion: "2",
},
Subsets: []corev1.EndpointSubset{{
Addresses: []corev1.EndpointAddress{{
IP: "10.10.10.10",
}},
}},
},
},
{
name: "With same subsets with same len of addresses with different ip",
oldObj: &corev1.Endpoints{
ObjectMeta: metav1.ObjectMeta{
ResourceVersion: "1",
},
Subsets: []corev1.EndpointSubset{{
Addresses: []corev1.EndpointAddress{{
IP: "10.10.10.10",
}},
}},
},
newObj: &corev1.Endpoints{
ObjectMeta: metav1.ObjectMeta{
ResourceVersion: "2",
},
Subsets: []corev1.EndpointSubset{{
Addresses: []corev1.EndpointAddress{{
IP: "10.10.10.42",
}},
}},
},
want: true,
},
{
name: "With same subsets with same len of addresses with same hostname",
oldObj: &corev1.Endpoints{
ObjectMeta: metav1.ObjectMeta{
ResourceVersion: "1",
},
Subsets: []corev1.EndpointSubset{{
Addresses: []corev1.EndpointAddress{{
Hostname: "foo",
}},
}},
},
newObj: &corev1.Endpoints{
ObjectMeta: metav1.ObjectMeta{
ResourceVersion: "2",
},
Subsets: []corev1.EndpointSubset{{
Addresses: []corev1.EndpointAddress{{
Hostname: "foo",
}},
}},
},
},
{
name: "With same subsets with same len of addresses with same hostname",
oldObj: &corev1.Endpoints{
ObjectMeta: metav1.ObjectMeta{
ResourceVersion: "1",
},
Subsets: []corev1.EndpointSubset{{
Addresses: []corev1.EndpointAddress{{
Hostname: "foo",
}},
}},
},
newObj: &corev1.Endpoints{
ObjectMeta: metav1.ObjectMeta{
ResourceVersion: "2",
},
Subsets: []corev1.EndpointSubset{{
Addresses: []corev1.EndpointAddress{{
Hostname: "bar",
}},
}},
},
want: true,
},
{
name: "With same subsets with same len of port with same name",
oldObj: &corev1.Endpoints{
ObjectMeta: metav1.ObjectMeta{
ResourceVersion: "1",
},
Subsets: []corev1.EndpointSubset{{
Ports: []corev1.EndpointPort{{
Name: "foo",
}},
}},
},
newObj: &corev1.Endpoints{
ObjectMeta: metav1.ObjectMeta{
ResourceVersion: "2",
},
Subsets: []corev1.EndpointSubset{{
Ports: []corev1.EndpointPort{{
Name: "foo",
}},
}},
},
},
{
name: "With same subsets with same len of port with different name",
oldObj: &corev1.Endpoints{
ObjectMeta: metav1.ObjectMeta{
ResourceVersion: "1",
},
Subsets: []corev1.EndpointSubset{{
Ports: []corev1.EndpointPort{{
Name: "foo",
}},
}},
},
newObj: &corev1.Endpoints{
ObjectMeta: metav1.ObjectMeta{
ResourceVersion: "2",
},
Subsets: []corev1.EndpointSubset{{
Ports: []corev1.EndpointPort{{
Name: "bar",
}},
}},
},
want: true,
},
{
name: "With same subsets with same len of port with same port",
oldObj: &corev1.Endpoints{
ObjectMeta: metav1.ObjectMeta{
ResourceVersion: "1",
},
Subsets: []corev1.EndpointSubset{{
Ports: []corev1.EndpointPort{{
Port: 4242,
}},
}},
},
newObj: &corev1.Endpoints{
ObjectMeta: metav1.ObjectMeta{
ResourceVersion: "2",
},
Subsets: []corev1.EndpointSubset{{
Ports: []corev1.EndpointPort{{
Port: 4242,
}},
}},
},
},
{
name: "With same subsets with same len of port with different port",
oldObj: &corev1.Endpoints{
ObjectMeta: metav1.ObjectMeta{
ResourceVersion: "1",
},
Subsets: []corev1.EndpointSubset{{
Ports: []corev1.EndpointPort{{
Port: 4242,
}},
}},
},
newObj: &corev1.Endpoints{
ObjectMeta: metav1.ObjectMeta{
ResourceVersion: "2",
},
Subsets: []corev1.EndpointSubset{{
Ports: []corev1.EndpointPort{{
Port: 6969,
}},
}},
},
want: true,
},
{
name: "With same subsets with same len of port with same protocol",
oldObj: &corev1.Endpoints{
ObjectMeta: metav1.ObjectMeta{
ResourceVersion: "1",
},
Subsets: []corev1.EndpointSubset{{
Ports: []corev1.EndpointPort{{
Protocol: "HTTP",
}},
}},
},
newObj: &corev1.Endpoints{
ObjectMeta: metav1.ObjectMeta{
ResourceVersion: "2",
},
Subsets: []corev1.EndpointSubset{{
Ports: []corev1.EndpointPort{{
Protocol: "HTTP",
}},
}},
},
},
{
name: "With same subsets with same len of port with different protocol",
oldObj: &corev1.Endpoints{
ObjectMeta: metav1.ObjectMeta{
ResourceVersion: "1",
},
Subsets: []corev1.EndpointSubset{{
Ports: []corev1.EndpointPort{{
Protocol: "HTTP",
}},
}},
},
newObj: &corev1.Endpoints{
ObjectMeta: metav1.ObjectMeta{
ResourceVersion: "2",
},
Subsets: []corev1.EndpointSubset{{
Ports: []corev1.EndpointPort{{
Protocol: "TCP",
}},
}},
},
want: true,
},
{
name: "With same subsets with same subset",
oldObj: &corev1.Endpoints{
ObjectMeta: metav1.ObjectMeta{
ResourceVersion: "1",
},
Subsets: []corev1.EndpointSubset{{
Addresses: []corev1.EndpointAddress{{
IP: "10.10.10.10",
Hostname: "foo",
}},
Ports: []corev1.EndpointPort{{
Name: "bar",
Port: 4242,
Protocol: "HTTP",
}},
}},
},
newObj: &corev1.Endpoints{
ObjectMeta: metav1.ObjectMeta{
ResourceVersion: "2",
},
Subsets: []corev1.EndpointSubset{{
Addresses: []corev1.EndpointAddress{{
IP: "10.10.10.10",
Hostname: "foo",
}},
Ports: []corev1.EndpointPort{{
Name: "bar",
Port: 4242,
Protocol: "HTTP",
}},
}},
},
},
}
for _, test := range tests {
test := test
t.Run(test.name, func(t *testing.T) {
t.Parallel()
assert.Equal(t, test.want, objChanged(test.oldObj, test.newObj))
})
}
}