diff --git a/pkg/provider/consulcatalog/config.go b/pkg/provider/consulcatalog/config.go
index 9a81a9c63..15da47679 100644
--- a/pkg/provider/consulcatalog/config.go
+++ b/pkg/provider/consulcatalog/config.go
@@ -198,29 +198,27 @@ func (p *Provider) addServerTCP(item itemData, loadBalancer *dynamic.TCPServersL
 		return errors.New("load-balancer is not defined")
 	}
 
-	var port string
-	if len(loadBalancer.Servers) > 0 {
-		port = loadBalancer.Servers[0].Port
-	}
-
 	if len(loadBalancer.Servers) == 0 {
 		loadBalancer.Servers = []dynamic.TCPServer{{}}
 	}
 
-	if item.Port != "" && port == "" {
-		port = item.Port
-	}
-	loadBalancer.Servers[0].Port = ""
-
-	if port == "" {
-		return errors.New("port is missing")
-	}
-
 	if item.Address == "" {
 		return errors.New("address is missing")
 	}
 
+	port := loadBalancer.Servers[0].Port
+	loadBalancer.Servers[0].Port = ""
+
+	if port == "" {
+		port = item.Port
+	}
+
+	if port == "" {
+		return errors.New("port is missing")
+	}
+
 	loadBalancer.Servers[0].Address = net.JoinHostPort(item.Address, port)
+
 	return nil
 }
 
@@ -233,21 +231,23 @@ func (p *Provider) addServerUDP(item itemData, loadBalancer *dynamic.UDPServersL
 		loadBalancer.Servers = []dynamic.UDPServer{{}}
 	}
 
-	var port string
-	if item.Port != "" {
+	if item.Address == "" {
+		return errors.New("address is missing")
+	}
+
+	port := loadBalancer.Servers[0].Port
+	loadBalancer.Servers[0].Port = ""
+
+	if port == "" {
 		port = item.Port
-		loadBalancer.Servers[0].Port = ""
 	}
 
 	if port == "" {
 		return errors.New("port is missing")
 	}
 
-	if item.Address == "" {
-		return errors.New("address is missing")
-	}
-
 	loadBalancer.Servers[0].Address = net.JoinHostPort(item.Address, port)
+
 	return nil
 }
 
@@ -256,11 +256,6 @@ func (p *Provider) addServer(item itemData, loadBalancer *dynamic.ServersLoadBal
 		return errors.New("load-balancer is not defined")
 	}
 
-	var port string
-	if len(loadBalancer.Servers) > 0 {
-		port = loadBalancer.Servers[0].Port
-	}
-
 	if len(loadBalancer.Servers) == 0 {
 		server := dynamic.Server{}
 		server.SetDefaults()
@@ -268,17 +263,19 @@ func (p *Provider) addServer(item itemData, loadBalancer *dynamic.ServersLoadBal
 		loadBalancer.Servers = []dynamic.Server{server}
 	}
 
-	if item.Port != "" && port == "" {
-		port = item.Port
+	if item.Address == "" {
+		return errors.New("address is missing")
 	}
+
+	port := loadBalancer.Servers[0].Port
 	loadBalancer.Servers[0].Port = ""
 
 	if port == "" {
-		return errors.New("port is missing")
+		port = item.Port
 	}
 
-	if item.Address == "" {
-		return errors.New("address is missing")
+	if port == "" {
+		return errors.New("port is missing")
 	}
 
 	scheme := loadBalancer.Servers[0].Scheme
diff --git a/pkg/provider/consulcatalog/config_test.go b/pkg/provider/consulcatalog/config_test.go
index a63d916d0..4e5b0a4e5 100644
--- a/pkg/provider/consulcatalog/config_test.go
+++ b/pkg/provider/consulcatalog/config_test.go
@@ -2220,7 +2220,7 @@ func Test_buildConfiguration(t *testing.T) {
 					Labels: map[string]string{
 						"traefik.tcp.routers.foo.rule":                      "HostSNI(`foo.bar`)",
 						"traefik.tcp.routers.foo.tls.options":               "foo",
-						"traefik.tcp.services.foo.loadbalancer.server.port": "80",
+						"traefik.tcp.services.foo.loadbalancer.server.port": "8080",
 					},
 					Address: "127.0.0.1",
 					Port:    "80",
@@ -2244,7 +2244,7 @@ func Test_buildConfiguration(t *testing.T) {
 							LoadBalancer: &dynamic.TCPServersLoadBalancer{
 								Servers: []dynamic.TCPServer{
 									{
-										Address: "127.0.0.1:80",
+										Address: "127.0.0.1:8080",
 									},
 								},
 								TerminationDelay: Int(100),
@@ -2611,6 +2611,57 @@ func Test_buildConfiguration(t *testing.T) {
 				},
 			},
 		},
+		{
+			desc:         "UDP service with labels only",
+			ConnectAware: true,
+			items: []itemData{
+				{
+					ID:         "1",
+					Node:       "Node1",
+					Datacenter: "dc1",
+					Name:       "Test",
+					Namespace:  "ns",
+					Labels: map[string]string{
+						"traefik.udp.routers.test-udp-label.service":                           "test-udp-label-service",
+						"traefik.udp.routers.test-udp-label.entryPoints":                       "udp",
+						"traefik.udp.services.test-udp-label-service.loadBalancer.server.port": "21116",
+					},
+					Address: "127.0.0.1",
+					Port:    "80",
+					Status:  api.HealthPassing,
+				},
+			},
+			expected: &dynamic.Configuration{
+				TCP: &dynamic.TCPConfiguration{
+					Routers:     map[string]*dynamic.TCPRouter{},
+					Middlewares: map[string]*dynamic.TCPMiddleware{},
+					Services:    map[string]*dynamic.TCPService{},
+				},
+				UDP: &dynamic.UDPConfiguration{
+					Routers: map[string]*dynamic.UDPRouter{
+						"test-udp-label": {
+							EntryPoints: []string{"udp"},
+							Service:     "test-udp-label-service",
+						},
+					},
+					Services: map[string]*dynamic.UDPService{
+						"test-udp-label-service": {
+							LoadBalancer: &dynamic.UDPServersLoadBalancer{
+								Servers: []dynamic.UDPServer{
+									{Address: "127.0.0.1:21116"},
+								},
+							},
+						},
+					},
+				},
+				HTTP: &dynamic.HTTPConfiguration{
+					Routers:           map[string]*dynamic.Router{},
+					Middlewares:       map[string]*dynamic.Middleware{},
+					Services:          map[string]*dynamic.Service{},
+					ServersTransports: map[string]*dynamic.ServersTransport{},
+				},
+			},
+		},
 	}
 
 	for _, test := range testCases {