From 6c7a70f4e5d2a736465dedaee1f90171e44bb4a2 Mon Sep 17 00:00:00 2001 From: zach593 Date: Sun, 30 Mar 2025 23:08:43 +0800 Subject: [PATCH] service aggregate status deduplication Signed-off-by: zach593 --- .../default/native/aggregatestatus.go | 2 + pkg/util/helper/ingress.go | 104 +++++++++++ pkg/util/helper/ingress_test.go | 151 +++++++++++++++ pkg/util/helper/service.go | 123 ++++++++++++ pkg/util/helper/service_test.go | 176 ++++++++++++++++++ 5 files changed, 556 insertions(+) create mode 100644 pkg/util/helper/ingress.go create mode 100644 pkg/util/helper/ingress_test.go create mode 100644 pkg/util/helper/service.go create mode 100644 pkg/util/helper/service_test.go diff --git a/pkg/resourceinterpreter/default/native/aggregatestatus.go b/pkg/resourceinterpreter/default/native/aggregatestatus.go index a9e5101cb..48d4ee877 100644 --- a/pkg/resourceinterpreter/default/native/aggregatestatus.go +++ b/pkg/resourceinterpreter/default/native/aggregatestatus.go @@ -147,6 +147,7 @@ func aggregateServiceStatus(object *unstructured.Unstructured, aggregatedStatusI newStatus.LoadBalancer.Ingress = append(newStatus.LoadBalancer.Ingress, temp.LoadBalancer.Ingress...) } + newStatus.LoadBalancer.Ingress = helper.DedupeAndSortServiceLoadBalancerIngress(newStatus.LoadBalancer.Ingress) if reflect.DeepEqual(service.Status, *newStatus) { klog.V(3).Infof("Ignore update service(%s/%s) status as up to date", service.Namespace, service.Name) @@ -179,6 +180,7 @@ func aggregateIngressStatus(object *unstructured.Unstructured, aggregatedStatusI newStatus.LoadBalancer.Ingress = append(newStatus.LoadBalancer.Ingress, temp.LoadBalancer.Ingress...) } + newStatus.LoadBalancer.Ingress = helper.DedupeAndSortIngressLoadBalancerIngress(newStatus.LoadBalancer.Ingress) if reflect.DeepEqual(ingress.Status, *newStatus) { klog.V(3).Infof("Ignore update ingress(%s/%s) status as up to date", ingress.Namespace, ingress.Name) diff --git a/pkg/util/helper/ingress.go b/pkg/util/helper/ingress.go new file mode 100644 index 000000000..110d0af4d --- /dev/null +++ b/pkg/util/helper/ingress.go @@ -0,0 +1,104 @@ +/* +Copyright 2025 The Karmada Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package helper + +import ( + "fmt" + "sort" + + networkingv1 "k8s.io/api/networking/v1" + "k8s.io/apimachinery/pkg/util/sets" +) + +// DedupeAndSortIngressLoadBalancerIngress dedupes and sorts the IngressLoadBalancerIngress slice. +func DedupeAndSortIngressLoadBalancerIngress(ingresses []networkingv1.IngressLoadBalancerIngress) []networkingv1.IngressLoadBalancerIngress { + errMap := make(map[string]*string) + for _, ingress := range ingresses { + for j, port := range ingress.Ports { + if port.Error != nil { + // let same error use same ptr, to let *Error be comparable + if errMap[*port.Error] == nil { + errMap[*port.Error] = port.Error + } else { + ingress.Ports[j].Error = errMap[*port.Error] + } + } + } + } + ingressMap := make(map[string]*networkingv1.IngressLoadBalancerIngress) + portStatusMap := make(map[string]sets.Set[networkingv1.IngressPortStatus]) + for _, ingress := range ingresses { + key := fmt.Sprintf("%s/%s", ingress.Hostname, ingress.IP) + if portStatusMap[key] == nil { + portStatusMap[key] = sets.New[networkingv1.IngressPortStatus]() + } + portStatusMap[key].Insert(ingress.Ports...) + ingress.Ports = nil + ingressMap[key] = &ingress + } + for key, portStatus := range portStatusMap { + for port := range portStatus { + ingressMap[key].Ports = append(ingressMap[key].Ports, port) + } + + sortIngressPortStatus(ingressMap[key].Ports) + } + out := make([]networkingv1.IngressLoadBalancerIngress, 0, len(ingressMap)) + for _, v := range ingressMap { + out = append(out, *v) + } + + sortIngressLoadBalancerIngress(out) + + return out +} + +func sortIngressPortStatus(ports []networkingv1.IngressPortStatus) { + sort.Slice(ports, func(i, j int) bool { + if ports[i].Port != ports[j].Port { + return ports[i].Port < ports[j].Port + } + + if ports[i].Protocol != ports[j].Protocol { + return ports[i].Protocol < ports[j].Protocol + } + + if ports[i].Error == nil { + return true + } + if ports[j].Error == nil { + return false + } + return *ports[i].Error < *ports[j].Error + }) +} + +func sortIngressLoadBalancerIngress(ingresses []networkingv1.IngressLoadBalancerIngress) { + sort.Slice(ingresses, func(i, j int) bool { + if ingresses[i].Hostname != ingresses[j].Hostname { + if ingresses[j].Hostname == "" { + return true + } + if ingresses[i].Hostname == "" { + return false + } + return ingresses[i].Hostname < ingresses[j].Hostname + } + + return ingresses[i].IP < ingresses[j].IP + }) +} diff --git a/pkg/util/helper/ingress_test.go b/pkg/util/helper/ingress_test.go new file mode 100644 index 000000000..1b1480523 --- /dev/null +++ b/pkg/util/helper/ingress_test.go @@ -0,0 +1,151 @@ +/* +Copyright 2025 The Karmada Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package helper + +import ( + "testing" + + "github.com/stretchr/testify/assert" + networkingv1 "k8s.io/api/networking/v1" + "k8s.io/utils/ptr" +) + +func TestDedupeAndSortIngressLoadBalancerIngress(t *testing.T) { + type args struct { + ingresses []networkingv1.IngressLoadBalancerIngress + } + tests := []struct { + name string + args args + want []networkingv1.IngressLoadBalancerIngress + }{ + { + name: "sort hostname", + args: args{ + ingresses: []networkingv1.IngressLoadBalancerIngress{ + {Hostname: "hostname-2"}, + {Hostname: "hostname-1"}, + }, + }, + want: []networkingv1.IngressLoadBalancerIngress{ + {Hostname: "hostname-1"}, + {Hostname: "hostname-2"}, + }, + }, + { + name: "sort ip", + args: args{ + ingresses: []networkingv1.IngressLoadBalancerIngress{ + {IP: "2.2.2.2"}, + {IP: "1.1.1.1"}, + }, + }, + want: []networkingv1.IngressLoadBalancerIngress{ + {IP: "1.1.1.1"}, + {IP: "2.2.2.2"}, + }, + }, + { + name: "hostname should in front of ip", + args: args{ + ingresses: []networkingv1.IngressLoadBalancerIngress{ + {IP: "1.1.1.1"}, + {Hostname: "hostname-1"}, + }, + }, + want: []networkingv1.IngressLoadBalancerIngress{ + {Hostname: "hostname-1"}, + {IP: "1.1.1.1"}, + }, + }, + { + name: "merge hostname and ip", + args: args{ + ingresses: []networkingv1.IngressLoadBalancerIngress{ + {IP: "1.1.1.1"}, + {Hostname: "hostname-1"}, + {IP: "1.1.1.1"}, + {Hostname: "hostname-1"}, + {IP: "1.1.1.1"}, + {Hostname: "hostname-1"}, + }, + }, + want: []networkingv1.IngressLoadBalancerIngress{ + {Hostname: "hostname-1"}, + {IP: "1.1.1.1"}, + }, + }, + { + name: "merge and sort ports", + args: args{ + ingresses: []networkingv1.IngressLoadBalancerIngress{ + {Hostname: "hostname-1", Ports: []networkingv1.IngressPortStatus{ + {Port: 80, Protocol: "TCP"}, + }}, + {Hostname: "hostname-1", Ports: []networkingv1.IngressPortStatus{ + {Port: 81, Protocol: "TCP"}, + }}, + {Hostname: "hostname-1", Ports: []networkingv1.IngressPortStatus{ + {Port: 80, Protocol: "TCP"}, + }}, + }, + }, + want: []networkingv1.IngressLoadBalancerIngress{ + {Hostname: "hostname-1", Ports: []networkingv1.IngressPortStatus{ + {Port: 80, Protocol: "TCP"}, + {Port: 81, Protocol: "TCP"}, + }}, + }, + }, + { + name: "merge and sort errors", + args: args{ + ingresses: []networkingv1.IngressLoadBalancerIngress{ + {Hostname: "hostname-1", Ports: []networkingv1.IngressPortStatus{ + {Port: 80, Protocol: "TCP", Error: ptr.To("error-1")}, + }}, + {Hostname: "hostname-1", Ports: []networkingv1.IngressPortStatus{ + {Port: 80, Protocol: "TCP", Error: ptr.To("error-1")}, + }}, + {Hostname: "hostname-1", Ports: []networkingv1.IngressPortStatus{ + {Port: 80, Protocol: "TCP", Error: ptr.To("error-2")}, + }}, + {Hostname: "hostname-1", Ports: []networkingv1.IngressPortStatus{ + {Port: 80, Protocol: "TCP"}, + }}, + }, + }, + want: []networkingv1.IngressLoadBalancerIngress{ + {Hostname: "hostname-1", Ports: []networkingv1.IngressPortStatus{ + {Port: 80, Protocol: "TCP"}, + {Port: 80, Protocol: "TCP", Error: ptr.To("error-1")}, + {Port: 80, Protocol: "TCP", Error: ptr.To("error-2")}, + }}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + for i := 0; i < 1000; i++ { // eliminate randomness in sorting + pass := assert.Equalf(t, tt.want, DedupeAndSortIngressLoadBalancerIngress(tt.args.ingresses), "DedupeAndSortIngressLoadBalancerIngress(%v)", tt.args.ingresses) + if !pass { + break + } + } + }) + } +} diff --git a/pkg/util/helper/service.go b/pkg/util/helper/service.go new file mode 100644 index 000000000..01af38cd8 --- /dev/null +++ b/pkg/util/helper/service.go @@ -0,0 +1,123 @@ +/* +Copyright 2025 The Karmada Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package helper + +import ( + "fmt" + "sort" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/sets" +) + +// DedupeAndSortServiceLoadBalancerIngress dedupes and sorts the ServiceLoadBalancerIngress slice. +func DedupeAndSortServiceLoadBalancerIngress(ingresses []corev1.LoadBalancerIngress) []corev1.LoadBalancerIngress { + ipModeMap := make(map[corev1.LoadBalancerIPMode]*corev1.LoadBalancerIPMode) + errMap := make(map[string]*string) + for i, ingress := range ingresses { + if ingress.IPMode != nil { + // let same IPMode use same ptr, to let *IPMode be comparable + if ipModeMap[*ingress.IPMode] == nil { + ipModeMap[*ingress.IPMode] = ingress.IPMode + } else { + ingresses[i].IPMode = ipModeMap[*ingress.IPMode] + } + } + for j, port := range ingress.Ports { + if port.Error != nil { + // let same error use same ptr, to let *Error be comparable + if errMap[*port.Error] == nil { + errMap[*port.Error] = port.Error + } else { + ingress.Ports[j].Error = errMap[*port.Error] + } + } + } + } + ingressMap := make(map[string]*corev1.LoadBalancerIngress) + portStatusMap := make(map[string]sets.Set[corev1.PortStatus]) + for _, ingress := range ingresses { + key := fmt.Sprintf("%s/%s/%p", ingress.Hostname, ingress.IP, ingress.IPMode) + if portStatusMap[key] == nil { + portStatusMap[key] = sets.New[corev1.PortStatus]() + } + portStatusMap[key].Insert(ingress.Ports...) + ingress.Ports = nil + ingressMap[key] = &ingress + } + for key, portStatus := range portStatusMap { + for port := range portStatus { + ingressMap[key].Ports = append(ingressMap[key].Ports, port) + } + + sortServicePortStatus(ingressMap[key].Ports) + } + out := make([]corev1.LoadBalancerIngress, 0, len(ingressMap)) + for _, v := range ingressMap { + out = append(out, *v) + } + + sortServiceLoadBalancerIngress(out) + + return out +} + +func sortServicePortStatus(ports []corev1.PortStatus) { + sort.Slice(ports, func(i, j int) bool { + if ports[i].Port != ports[j].Port { + return ports[i].Port < ports[j].Port + } + + if ports[i].Protocol != ports[j].Protocol { + return ports[i].Protocol < ports[j].Protocol + } + + if ports[i].Error == nil { + return true + } + if ports[j].Error == nil { + return false + } + return *ports[i].Error < *ports[j].Error + }) +} + +func sortServiceLoadBalancerIngress(ingresses []corev1.LoadBalancerIngress) { + sort.Slice(ingresses, func(i, j int) bool { + if ingresses[i].Hostname != ingresses[j].Hostname { + if ingresses[j].Hostname == "" { + return true + } + if ingresses[i].Hostname == "" { + return false + } + return ingresses[i].Hostname < ingresses[j].Hostname + } + + if ingresses[i].IP != ingresses[j].IP { + return ingresses[i].IP < ingresses[j].IP + } + + if ingresses[j].IPMode == nil { + return true + } + if ingresses[i].IPMode == nil { + return false + } + return *ingresses[i].IPMode < *ingresses[j].IPMode + }) +} diff --git a/pkg/util/helper/service_test.go b/pkg/util/helper/service_test.go new file mode 100644 index 000000000..10d89f585 --- /dev/null +++ b/pkg/util/helper/service_test.go @@ -0,0 +1,176 @@ +/* +Copyright 2025 The Karmada Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package helper + +import ( + "testing" + + "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" + "k8s.io/utils/ptr" +) + +func TestDedupeAndSortServiceLoadBalancerIngress(t *testing.T) { + type args struct { + ingresses []corev1.LoadBalancerIngress + } + tests := []struct { + name string + args args + want []corev1.LoadBalancerIngress + }{ + { + name: "sort hostname", + args: args{ + ingresses: []corev1.LoadBalancerIngress{ + {Hostname: "hostname-2"}, + {Hostname: "hostname-1"}, + }, + }, + want: []corev1.LoadBalancerIngress{ + {Hostname: "hostname-1"}, + {Hostname: "hostname-2"}, + }, + }, + { + name: "sort ip", + args: args{ + ingresses: []corev1.LoadBalancerIngress{ + {IP: "2.2.2.2"}, + {IP: "1.1.1.1"}, + }, + }, + want: []corev1.LoadBalancerIngress{ + {IP: "1.1.1.1"}, + {IP: "2.2.2.2"}, + }, + }, + { + name: "hostname should in front of ip", + args: args{ + ingresses: []corev1.LoadBalancerIngress{ + {IP: "1.1.1.1"}, + {Hostname: "hostname-1"}, + }, + }, + want: []corev1.LoadBalancerIngress{ + {Hostname: "hostname-1"}, + {IP: "1.1.1.1"}, + }, + }, + { + name: "sort ipMode", + args: args{ + ingresses: []corev1.LoadBalancerIngress{ + {IP: "1.1.1.1", IPMode: ptr.To(corev1.LoadBalancerIPModeProxy)}, + {IP: "1.1.1.1", IPMode: ptr.To(corev1.LoadBalancerIPModeVIP)}, + }, + }, + want: []corev1.LoadBalancerIngress{ + {IP: "1.1.1.1", IPMode: ptr.To(corev1.LoadBalancerIPModeProxy)}, + {IP: "1.1.1.1", IPMode: ptr.To(corev1.LoadBalancerIPModeVIP)}, + }, + }, + { + name: "merge ipMode", + args: args{ + ingresses: []corev1.LoadBalancerIngress{ + {IP: "1.1.1.1", IPMode: ptr.To(corev1.LoadBalancerIPModeVIP)}, + {IP: "1.1.1.1", IPMode: ptr.To(corev1.LoadBalancerIPModeVIP)}, + }, + }, + want: []corev1.LoadBalancerIngress{ + {IP: "1.1.1.1", IPMode: ptr.To(corev1.LoadBalancerIPModeVIP)}, + }, + }, + { + name: "merge hostname and ip", + args: args{ + ingresses: []corev1.LoadBalancerIngress{ + {IP: "1.1.1.1"}, + {Hostname: "hostname-1"}, + {IP: "1.1.1.1"}, + {Hostname: "hostname-1"}, + {IP: "1.1.1.1"}, + {Hostname: "hostname-1"}, + }, + }, + want: []corev1.LoadBalancerIngress{ + {Hostname: "hostname-1"}, + {IP: "1.1.1.1"}, + }, + }, + { + name: "merge and sort ports", + args: args{ + ingresses: []corev1.LoadBalancerIngress{ + {Hostname: "hostname-1", Ports: []corev1.PortStatus{ + {Port: 80, Protocol: "TCP"}, + }}, + {Hostname: "hostname-1", Ports: []corev1.PortStatus{ + {Port: 81, Protocol: "TCP"}, + }}, + {Hostname: "hostname-1", Ports: []corev1.PortStatus{ + {Port: 80, Protocol: "TCP"}, + }}, + }, + }, + want: []corev1.LoadBalancerIngress{ + {Hostname: "hostname-1", Ports: []corev1.PortStatus{ + {Port: 80, Protocol: "TCP"}, + {Port: 81, Protocol: "TCP"}, + }}, + }, + }, + { + name: "merge and sort errors", + args: args{ + ingresses: []corev1.LoadBalancerIngress{ + {Hostname: "hostname-1", Ports: []corev1.PortStatus{ + {Port: 80, Protocol: "TCP", Error: ptr.To("error-1")}, + }}, + {Hostname: "hostname-1", Ports: []corev1.PortStatus{ + {Port: 80, Protocol: "TCP", Error: ptr.To("error-1")}, + }}, + {Hostname: "hostname-1", Ports: []corev1.PortStatus{ + {Port: 80, Protocol: "TCP", Error: ptr.To("error-2")}, + }}, + {Hostname: "hostname-1", Ports: []corev1.PortStatus{ + {Port: 80, Protocol: "TCP"}, + }}, + }, + }, + want: []corev1.LoadBalancerIngress{ + {Hostname: "hostname-1", Ports: []corev1.PortStatus{ + {Port: 80, Protocol: "TCP"}, + {Port: 80, Protocol: "TCP", Error: ptr.To("error-1")}, + {Port: 80, Protocol: "TCP", Error: ptr.To("error-2")}, + }}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + for i := 0; i < 1000; i++ { // eliminate randomness in sorting + pass := assert.Equalf(t, tt.want, DedupeAndSortServiceLoadBalancerIngress(tt.args.ingresses), "DedupeAndSortServiceLoadBalancerIngress(%v)", tt.args.ingresses) + if !pass { + break + } + } + }) + } +}