Merge pull request #15617 from hakman/azure_delete_dependency_logic

azure: Add mode dependency logic to deletion

pkg/resources/azure/azure.go

@@ -27,7 +27,7 @@ import (
 	"k8s.io/kops/pkg/resources"
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/upup/pkg/fi/cloudup/azure"
-	"k8s.io/kops/upup/pkg/fi/cloudup/azuretasks"
+	"k8s.io/utils/set"
 )
 
 const (
@@ -150,7 +150,11 @@ func (g *resourceGetter) listVirtualNetworksAndSubnets(ctx context.Context) ([]*
 		if !g.isOwnedByCluster(vnet.Tags) {
 			continue
 		}
-		rs = append(rs, g.toVirtualNetworkResource(vnet))
+		r, err := g.toVirtualNetworkResource(vnet)
+		if err != nil {
+			return nil, err
+		}
+		rs = append(rs, r)
 		// Add all subnets belonging to the virtual network.
 		subnets, err := g.listSubnets(ctx, *vnet.Name)
 		if err != nil {
@@ -161,16 +165,35 @@ func (g *resourceGetter) listVirtualNetworksAndSubnets(ctx context.Context) ([]*
 	return rs, nil
 }
 
-func (g *resourceGetter) toVirtualNetworkResource(vnet *network.VirtualNetwork) *resources.Resource {
+func (g *resourceGetter) toVirtualNetworkResource(vnet *network.VirtualNetwork) (*resources.Resource, error) {
+	var blocks []string
+	blocks = append(blocks, toKey(typeResourceGroup, g.resourceGroupName()))
+
+	nsgs := set.New[string]()
+	if vnet.Subnets != nil {
+		for _, sn := range *vnet.Subnets {
+			if sn.NetworkSecurityGroup != nil {
+				nsgID, err := azure.ParseNetworkSecurityGroupID(*sn.NetworkSecurityGroup.ID)
+				if err != nil {
+					return nil, fmt.Errorf("parsing network security group ID: %s", err)
+				}
+				nsgs.Insert(nsgID.NetworkSecurityGroupName)
+			}
+		}
+	}
+	for nsg := range nsgs {
+		blocks = append(blocks, toKey(typeNetworkSecurityGroup, nsg))
+	}
+
 	return &resources.Resource{
 		Obj:     vnet,
 		Type:    typeVirtualNetwork,
 		ID:      *vnet.Name,
 		Name:    *vnet.Name,
 		Deleter: g.deleteVirtualNetwork,
-		Blocks:  []string{toKey(typeResourceGroup, g.resourceGroupName())},
+		Blocks:  blocks,
 		Shared:  g.clusterInfo.AzureNetworkShared,
-	}
+	}, nil
 }
 
 func (g *resourceGetter) deleteVirtualNetwork(_ fi.Cloud, r *resources.Resource) error {
@@ -319,16 +342,26 @@ func (g *resourceGetter) toVMScaleSetResource(vmss *compute.VirtualMachineScaleS
 	var blocks []string
 	blocks = append(blocks, toKey(typeResourceGroup, g.resourceGroupName()))
 
-	vnets := map[string]struct{}{}
+	vnets := set.New[string]()
-	subnets := map[string]struct{}{}
+	subnets := set.New[string]()
+	lbs := set.New[string]()
 	for _, iface := range *vmss.VirtualMachineProfile.NetworkProfile.NetworkInterfaceConfigurations {
 		for _, ip := range *iface.IPConfigurations {
-			subnetID, err := azuretasks.ParseSubnetID(*ip.Subnet.ID)
+			subnetID, err := azure.ParseSubnetID(*ip.Subnet.ID)
 			if err != nil {
 				return nil, fmt.Errorf("error on parsing subnet ID: %s", err)
 			}
-			vnets[subnetID.VirtualNetworkName] = struct{}{}
+			vnets.Insert(subnetID.VirtualNetworkName)
-			subnets[subnetID.SubnetName] = struct{}{}
+			subnets.Insert(subnetID.SubnetName)
+			if ip.LoadBalancerBackendAddressPools != nil {
+				for _, lb := range *ip.LoadBalancerBackendAddressPools {
+					lbID, err := azure.ParseLoadBalancerID(*lb.ID)
+					if err != nil {
+						return nil, fmt.Errorf("parsing load balancer ID: %s", err)
+					}
+					lbs.Insert(lbID.LoadBalancerName)
+				}
+			}
 		}
 	}
 	for vnet := range vnets {
@@ -337,6 +370,9 @@ func (g *resourceGetter) toVMScaleSetResource(vmss *compute.VirtualMachineScaleS
 	for subnet := range subnets {
 		blocks = append(blocks, toKey(typeSubnet, subnet))
 	}
+	for lb := range lbs {
+		blocks = append(blocks, toKey(typeLoadBalancer, lb))
+	}
 
 	for _, vm := range vms {
 		if disks := vm.StorageProfile.DataDisks; disks != nil {
@@ -448,20 +484,43 @@ func (g *resourceGetter) listLoadBalancers(ctx context.Context) ([]*resources.Re
 		if !g.isOwnedByCluster(lb.Tags) {
 			continue
 		}
-		rs = append(rs, g.toLoadBalancerResource(lb))
+		r, err := g.toLoadBalancerResource(lb)
+		if err != nil {
+			return nil, err
+		}
+		rs = append(rs, r)
 	}
 	return rs, nil
 }
 
-func (g *resourceGetter) toLoadBalancerResource(loadBalancer *network.LoadBalancer) *resources.Resource {
+func (g *resourceGetter) toLoadBalancerResource(loadBalancer *network.LoadBalancer) (*resources.Resource, error) {
+	var blocks []string
+	blocks = append(blocks, toKey(typeResourceGroup, g.resourceGroupName()))
+
+	pips := set.New[string]()
+	if loadBalancer.FrontendIPConfigurations != nil {
+		for _, fip := range *loadBalancer.FrontendIPConfigurations {
+			if fip.PublicIPAddress != nil {
+				pipID, err := azure.ParsePublicIPAddressID(*fip.PublicIPAddress.ID)
+				if err != nil {
+					return nil, fmt.Errorf("parsing public IP address ID: %s", err)
+				}
+				pips.Insert(pipID.PublicIPAddressName)
+			}
+		}
+	}
+	for pip := range pips {
+		blocks = append(blocks, toKey(typePublicIPAddress, pip))
+	}
+
 	return &resources.Resource{
 		Obj:     loadBalancer,
 		Type:    typeLoadBalancer,
 		ID:      *loadBalancer.Name,
 		Name:    *loadBalancer.Name,
 		Deleter: g.deleteLoadBalancer,
-		Blocks:  []string{toKey(typeResourceGroup, g.resourceGroupName())},
+		Blocks:  blocks,
-	}
+	}, nil
 }
 
 func (g *resourceGetter) deleteLoadBalancer(_ fi.Cloud, r *resources.Resource) error {

pkg/resources/azure/azure_test.go

@@ -63,8 +63,9 @@ func TestListResourcesAzure(t *testing.T) {
 
 	vnets := cloud.VirtualNetworksClient.VNets
 	vnets[vnetName] = network.VirtualNetwork{
-		Name: to.StringPtr(vnetName),
+		Name:                           to.StringPtr(vnetName),
-		Tags: clusterTags,
+		Tags:                           clusterTags,
+		VirtualNetworkPropertiesFormat: &network.VirtualNetworkPropertiesFormat{},
 	}
 	vnets[irrelevantName] = network.VirtualNetwork{
 		Name: to.StringPtr(irrelevantName),
@@ -88,7 +89,7 @@ func TestListResourcesAzure(t *testing.T) {
 	}
 
 	vmsses := cloud.VMScaleSetsClient.VMSSes
-	subnetID := azuretasks.SubnetID{
+	subnetID := azure.SubnetID{
 		SubscriptionID:     "sid",
 		ResourceGroupName:  rgName,
 		VirtualNetworkName: vnetName,
@@ -164,8 +165,9 @@ func TestListResourcesAzure(t *testing.T) {
 
 	lbs := cloud.LoadBalancersClient.LBs
 	lbs[lbName] = network.LoadBalancer{
-		Name: to.StringPtr(lbName),
+		Name:                         to.StringPtr(lbName),
-		Tags: clusterTags,
+		Tags:                         clusterTags,
+		LoadBalancerPropertiesFormat: &network.LoadBalancerPropertiesFormat{},
 	}
 	lbs[irrelevantName] = network.LoadBalancer{
 		Name: to.StringPtr(irrelevantName),

upup/pkg/fi/cloudup/azure/azure_utils.go

@@ -40,3 +40,119 @@ func ZoneToAvailabilityZoneNumber(zone string) (string, error) {
 	}
 	return l[1], nil
 }
+
+// SubnetID contains the resource ID/names required to construct a subnet ID.
+type SubnetID struct {
+	SubscriptionID     string
+	ResourceGroupName  string
+	VirtualNetworkName string
+	SubnetName         string
+}
+
+// String returns the subnet ID in the path format.
+func (s *SubnetID) String() string {
+	return fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/virtualNetworks/%s/subnets/%s",
+		s.SubscriptionID,
+		s.ResourceGroupName,
+		s.VirtualNetworkName,
+		s.SubnetName)
+}
+
+// ParseSubnetID parses a given subnet ID string and returns a SubnetID.
+func ParseSubnetID(s string) (*SubnetID, error) {
+	l := strings.Split(s, "/")
+	if len(l) != 11 {
+		return nil, fmt.Errorf("malformed format of subnet ID: %s, %d", s, len(l))
+	}
+	return &SubnetID{
+		SubscriptionID:     l[2],
+		ResourceGroupName:  l[4],
+		VirtualNetworkName: l[8],
+		SubnetName:         l[10],
+	}, nil
+}
+
+// NetworkSecurityGroupID contains the resource ID/names required to construct a NetworkSecurityGroup ID.
+type NetworkSecurityGroupID struct {
+	SubscriptionID           string
+	ResourceGroupName        string
+	NetworkSecurityGroupName string
+}
+
+// String returns the NetworkSecurityGroup ID in the path format.
+func (s *NetworkSecurityGroupID) String() string {
+	return fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/networkSecurityGroups/%s",
+		s.SubscriptionID,
+		s.ResourceGroupName,
+		s.NetworkSecurityGroupName)
+}
+
+// ParseNetworkSecurityGroupID parses a given NetworkSecurityGroup ID string and returns a NetworkSecurityGroup ID.
+func ParseNetworkSecurityGroupID(s string) (*NetworkSecurityGroupID, error) {
+	l := strings.Split(s, "/")
+	if len(l) != 9 {
+		return nil, fmt.Errorf("malformed format of NetworkSecurityGroup ID: %s, %d", s, len(l))
+	}
+	return &NetworkSecurityGroupID{
+		SubscriptionID:           l[2],
+		ResourceGroupName:        l[4],
+		NetworkSecurityGroupName: l[8],
+	}, nil
+}
+
+// LoadBalancerID contains the resource ID/names required to construct a load balancer ID.
+type LoadBalancerID struct {
+	SubscriptionID    string
+	ResourceGroupName string
+	LoadBalancerName  string
+}
+
+// String returns the load balancer ID in the path format.
+func (lb *LoadBalancerID) String() string {
+	return fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/loadbalancers/%s/backendAddressPools/LoadBalancerBackEnd",
+		lb.SubscriptionID,
+		lb.ResourceGroupName,
+		lb.LoadBalancerName,
+	)
+}
+
+// ParseLoadBalancerID parses a given load balancer ID string and returns a LoadBalancerID.
+func ParseLoadBalancerID(lb string) (*LoadBalancerID, error) {
+	l := strings.Split(lb, "/")
+	if len(l) != 11 {
+		return nil, fmt.Errorf("malformed format of loadbalancer ID: %s, %d", lb, len(l))
+	}
+	return &LoadBalancerID{
+		SubscriptionID:    l[2],
+		ResourceGroupName: l[4],
+		LoadBalancerName:  l[8],
+	}, nil
+}
+
+// PublicIPAddressID contains the resource ID/names required to construct a PublicIPAddress ID.
+type PublicIPAddressID struct {
+	SubscriptionID      string
+	ResourceGroupName   string
+	PublicIPAddressName string
+}
+
+// String returns the PublicIPAddress ID in the path format.
+func (s *PublicIPAddressID) String() string {
+	return fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/publicIPAddresss/%s",
+		s.SubscriptionID,
+		s.ResourceGroupName,
+		s.PublicIPAddressName)
+}
+
+// ParsePublicIPAddressID parses a given PublicIPAddress ID string and returns a PublicIPAddress ID.
+func ParsePublicIPAddressID(s string) (*PublicIPAddressID, error) {
+	l := strings.Split(s, "/")
+	if len(l) != 9 {
+		return nil, fmt.Errorf("malformed format of PublicIPAddress ID: %s, %d", s, len(l))
+	}
+	return &PublicIPAddressID{
+		SubscriptionID:      l[2],
+		ResourceGroupName:   l[4],
+		PublicIPAddressName: l[8],
+	}, nil
+}

upup/pkg/fi/cloudup/azuretasks/vmscaleset.go

@@ -29,66 +29,6 @@ import (
 	"k8s.io/kops/upup/pkg/fi/cloudup/azure"
 )
 
-// SubnetID contains the resource ID/names required to construct a subnet ID.
-type SubnetID struct {
-	SubscriptionID     string
-	ResourceGroupName  string
-	VirtualNetworkName string
-	SubnetName         string
-}
-
-// String returns the subnet ID in the path format.
-func (s *SubnetID) String() string {
-	return fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/virtualNetworks/%s/subnets/%s",
-		s.SubscriptionID,
-		s.ResourceGroupName,
-		s.VirtualNetworkName,
-		s.SubnetName)
-}
-
-// ParseSubnetID parses a given subnet ID string and returns a SubnetID.
-func ParseSubnetID(s string) (*SubnetID, error) {
-	l := strings.Split(s, "/")
-	if len(l) != 11 {
-		return nil, fmt.Errorf("malformed format of subnet ID: %s, %d", s, len(l))
-	}
-	return &SubnetID{
-		SubscriptionID:     l[2],
-		ResourceGroupName:  l[4],
-		VirtualNetworkName: l[8],
-		SubnetName:         l[10],
-	}, nil
-}
-
-// loadBalancerID contains the resource ID/names required to construct a loadbalancer ID.
-type loadBalancerID struct {
-	SubscriptionID    string
-	ResourceGroupName string
-	LoadBalancerName  string
-}
-
-// String returns the loadbalancer ID in the path format.
-func (lb *loadBalancerID) String() string {
-	return fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/loadbalancers/%s/backendAddressPools/LoadBalancerBackEnd",
-		lb.SubscriptionID,
-		lb.ResourceGroupName,
-		lb.LoadBalancerName,
-	)
-}
-
-// parseLoadBalancerID parses a given loadbalancer ID string and returns a loadBalancerID.
-func parseLoadBalancerID(lb string) (*loadBalancerID, error) {
-	l := strings.Split(lb, "/")
-	if len(l) != 11 {
-		return nil, fmt.Errorf("malformed format of loadbalancer ID: %s, %d", lb, len(l))
-	}
-	return &loadBalancerID{
-		SubscriptionID:    l[2],
-		ResourceGroupName: l[4],
-		LoadBalancerName:  l[8],
-	}, nil
-}
-
 // VMScaleSet is an Azure VM Scale Set.
 // +kops:fitask
 type VMScaleSet struct {
@@ -173,18 +113,18 @@ func (s *VMScaleSet) Find(c *fi.CloudupContext) (*VMScaleSet, error) {
 		return nil, fmt.Errorf("unexpected number of IP configs found for VM ScaleSet %s: %d", *s.Name, len(ipConfigs))
 	}
 	ipConfig := ipConfigs[0]
-	subnetID, err := ParseSubnetID(*ipConfig.Subnet.ID)
+	subnetID, err := azure.ParseSubnetID(*ipConfig.Subnet.ID)
 	if err != nil {
 		return nil, fmt.Errorf("failed to parse subnet ID %s", *ipConfig.Subnet.ID)
 	}
 
-	var loadBalancerID *loadBalancerID
+	var loadBalancerID *azure.LoadBalancerID
 	if ipConfig.LoadBalancerBackendAddressPools != nil {
 		for _, i := range *ipConfig.LoadBalancerBackendAddressPools {
 			if !strings.Contains(*i.ID, "api") {
 				continue
 			}
-			loadBalancerID, err = parseLoadBalancerID(*i.ID)
+			loadBalancerID, err = azure.ParseLoadBalancerID(*i.ID)
 			if err != nil {
 				return nil, fmt.Errorf("failed to parse loadbalancer ID %s", *ipConfig.Subnet.ID)
 			}
@@ -301,7 +241,7 @@ func (s *VMScaleSet) RenderAzure(t *azure.AzureAPITarget, a, e, changes *VMScale
 		},
 	}
 
-	subnetID := SubnetID{
+	subnetID := azure.SubnetID{
 		SubscriptionID:     t.Cloud.SubscriptionID(),
 		ResourceGroupName:  *e.ResourceGroup.Name,
 		VirtualNetworkName: *e.VirtualNetwork.Name,
@@ -323,7 +263,7 @@ func (s *VMScaleSet) RenderAzure(t *azure.AzureAPITarget, a, e, changes *VMScale
 		}
 	}
 	if e.LoadBalancer != nil {
-		loadBalancerID := loadBalancerID{
+		loadBalancerID := azure.LoadBalancerID{
 			SubscriptionID:    t.Cloud.SubscriptionID(),
 			ResourceGroupName: *e.ResourceGroup.Name,
 			LoadBalancerName:  *e.LoadBalancer.Name,

upup/pkg/fi/cloudup/azuretasks/vmscaleset_test.go

@@ -31,13 +31,13 @@ import (
 )
 
 func TestSubnetIDParse(t *testing.T) {
-	subnetID := &SubnetID{
+	subnetID := &azure.SubnetID{
 		SubscriptionID:     "sid",
 		ResourceGroupName:  "rg",
 		VirtualNetworkName: "vnet",
 		SubnetName:         "sub",
 	}
-	actual, err := ParseSubnetID(subnetID.String())
+	actual, err := azure.ParseSubnetID(subnetID.String())
 	if err != nil {
 		t.Fatalf("unexpected error %s", err)
 	}
@@ -47,12 +47,12 @@ func TestSubnetIDParse(t *testing.T) {
 }
 
 func TestLoadBalancerIDParse(t *testing.T) {
-	loadBalancerID := &loadBalancerID{
+	loadBalancerID := &azure.LoadBalancerID{
 		SubscriptionID:    "sid",
 		ResourceGroupName: "rg",
 		LoadBalancerName:  "lb",
 	}
-	actual, err := parseLoadBalancerID(loadBalancerID.String())
+	actual, err := azure.ParseLoadBalancerID(loadBalancerID.String())
 	if err != nil {
 		t.Fatalf("unexpected error %s", err)
 	}
@@ -193,13 +193,13 @@ func TestVMScaleSetFind(t *testing.T) {
 			Caching: compute.CachingTypes(compute.HostCachingReadWrite),
 		},
 	}
-	subnetID := SubnetID{
+	subnetID := azure.SubnetID{
 		SubscriptionID:     "subID",
 		ResourceGroupName:  *rg.Name,
 		VirtualNetworkName: "vnet",
 		SubnetName:         "sub",
 	}
-	loadBalancerID := loadBalancerID{
+	loadBalancerID := azure.LoadBalancerID{
 		SubscriptionID:    "subID",
 		ResourceGroupName: *rg.Name,
 		LoadBalancerName:  "api-lb",

vendor/k8s.io/utils/set/OWNERS

@@ -0,0 +1,8 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+
+reviewers:
+  - logicalhan
+  - thockin
+approvers:
+  - logicalhan
+  - thockin

vendor/k8s.io/utils/set/ordered.go

@@ -0,0 +1,53 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package set
+
+// ordered is a constraint that permits any ordered type: any type
+// that supports the operators < <= >= >.
+// If future releases of Go add new ordered types,
+// this constraint will be modified to include them.
+type ordered interface {
+	integer | float | ~string
+}
+
+// integer is a constraint that permits any integer type.
+// If future releases of Go add new predeclared integer types,
+// this constraint will be modified to include them.
+type integer interface {
+	signed | unsigned
+}
+
+// float is a constraint that permits any floating-point type.
+// If future releases of Go add new predeclared floating-point types,
+// this constraint will be modified to include them.
+type float interface {
+	~float32 | ~float64
+}
+
+// signed is a constraint that permits any signed integer type.
+// If future releases of Go add new predeclared signed integer types,
+// this constraint will be modified to include them.
+type signed interface {
+	~int | ~int8 | ~int16 | ~int32 | ~int64
+}
+
+// unsigned is a constraint that permits any unsigned integer type.
+// If future releases of Go add new predeclared unsigned integer types,
+// this constraint will be modified to include them.
+type unsigned interface {
+	~uint | ~uint8 | ~uint16 | ~uint32 | ~uint64 | ~uintptr
+}

vendor/k8s.io/utils/set/set.go

@@ -0,0 +1,229 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package set
+
+import (
+	"sort"
+)
+
+// Empty is public since it is used by some internal API objects for conversions between external
+// string arrays and internal sets, and conversion logic requires public types today.
+type Empty struct{}
+
+// Set is a set of the same type elements, implemented via map[ordered]struct{} for minimal memory consumption.
+type Set[E ordered] map[E]Empty
+
+// New creates a new set.
+func New[E ordered](items ...E) Set[E] {
+	ss := Set[E]{}
+	ss.Insert(items...)
+	return ss
+}
+
+// KeySet creates a Set[E] from a keys of a map[E](? extends interface{}).
+func KeySet[E ordered, A any](theMap map[E]A) Set[E] {
+	ret := Set[E]{}
+	for key := range theMap {
+		ret.Insert(key)
+	}
+	return ret
+}
+
+// Insert adds items to the set.
+func (s Set[E]) Insert(items ...E) Set[E] {
+	for _, item := range items {
+		s[item] = Empty{}
+	}
+	return s
+}
+
+// Delete removes all items from the set.
+func (s Set[E]) Delete(items ...E) Set[E] {
+	for _, item := range items {
+		delete(s, item)
+	}
+	return s
+}
+
+// Has returns true if and only if item is contained in the set.
+func (s Set[E]) Has(item E) bool {
+	_, contained := s[item]
+	return contained
+}
+
+// HasAll returns true if and only if all items are contained in the set.
+func (s Set[E]) HasAll(items ...E) bool {
+	for _, item := range items {
+		if !s.Has(item) {
+			return false
+		}
+	}
+	return true
+}
+
+// HasAny returns true if any items are contained in the set.
+func (s Set[E]) HasAny(items ...E) bool {
+	for _, item := range items {
+		if s.Has(item) {
+			return true
+		}
+	}
+	return false
+}
+
+// Union returns a new set which includes items in either s1 or s2.
+// For example:
+// s1 = {a1, a2}
+// s2 = {a3, a4}
+// s1.Union(s2) = {a1, a2, a3, a4}
+// s2.Union(s1) = {a1, a2, a3, a4}
+func (s Set[E]) Union(s2 Set[E]) Set[E] {
+	result := Set[E]{}
+	result.Insert(s.UnsortedList()...)
+	result.Insert(s2.UnsortedList()...)
+	return result
+}
+
+// Len returns the number of elements in the set.
+func (s Set[E]) Len() int {
+	return len(s)
+}
+
+// Intersection returns a new set which includes the item in BOTH s1 and s2
+// For example:
+// s1 = {a1, a2}
+// s2 = {a2, a3}
+// s1.Intersection(s2) = {a2}
+func (s Set[E]) Intersection(s2 Set[E]) Set[E] {
+	var walk, other Set[E]
+	result := Set[E]{}
+	if s.Len() < s2.Len() {
+		walk = s
+		other = s2
+	} else {
+		walk = s2
+		other = s
+	}
+	for key := range walk {
+		if other.Has(key) {
+			result.Insert(key)
+		}
+	}
+	return result
+}
+
+// IsSuperset returns true if and only if s1 is a superset of s2.
+func (s Set[E]) IsSuperset(s2 Set[E]) bool {
+	for item := range s2 {
+		if !s.Has(item) {
+			return false
+		}
+	}
+	return true
+}
+
+// Difference returns a set of objects that are not in s2
+// For example:
+// s1 = {a1, a2, a3}
+// s2 = {a1, a2, a4, a5}
+// s1.Difference(s2) = {a3}
+// s2.Difference(s1) = {a4, a5}
+func (s Set[E]) Difference(s2 Set[E]) Set[E] {
+	result := Set[E]{}
+	for key := range s {
+		if !s2.Has(key) {
+			result.Insert(key)
+		}
+	}
+	return result
+}
+
+// Equal returns true if and only if s1 is equal (as a set) to s2.
+// Two sets are equal if their membership is identical.
+func (s Set[E]) Equal(s2 Set[E]) bool {
+	return s.Len() == s.Len() && s.IsSuperset(s2)
+}
+
+type sortableSlice[E ordered] []E
+
+func (s sortableSlice[E]) Len() int {
+	return len(s)
+}
+func (s sortableSlice[E]) Less(i, j int) bool { return s[i] < s[j] }
+func (s sortableSlice[E]) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
+
+// SortedList returns the contents as a sorted slice.
+func (s Set[E]) SortedList() []E {
+	res := make(sortableSlice[E], 0, s.Len())
+	for key := range s {
+		res = append(res, key)
+	}
+	sort.Sort(res)
+	return res
+}
+
+// UnsortedList returns the slice with contents in random order.
+func (s Set[E]) UnsortedList() []E {
+	res := make([]E, 0, len(s))
+	for key := range s {
+		res = append(res, key)
+	}
+	return res
+}
+
+// PopAny returns a single element from the set.
+func (s Set[E]) PopAny() (E, bool) {
+	for key := range s {
+		s.Delete(key)
+		return key, true
+	}
+	var zeroValue E
+	return zeroValue, false
+}
+
+// Clone returns a new set which is a copy of the current set.
+func (s Set[T]) Clone() Set[T] {
+	result := make(Set[T], len(s))
+	for key := range s {
+		result.Insert(key)
+	}
+	return result
+}
+
+// SymmetricDifference returns a set of elements which are in either of the sets, but not in their intersection.
+// For example:
+// s1 = {a1, a2, a3}
+// s2 = {a1, a2, a4, a5}
+// s1.SymmetricDifference(s2) = {a3, a4, a5}
+// s2.SymmetricDifference(s1) = {a3, a4, a5}
+func (s Set[T]) SymmetricDifference(s2 Set[T]) Set[T] {
+	return s.Difference(s2).Union(s2.Difference(s))
+}
+
+// Clear empties the set.
+// It is preferable to replace the set with a newly constructed set,
+// but not all callers can do that (when there are other references to the map).
+// In some cases the set *won't* be fully cleared, e.g. a Set[float32] containing NaN
+// can't be cleared because NaN can't be removed.
+// For sets containing items of a type that is reflexive for ==,
+// this is optimized to a single call to runtime.mapclear().
+func (s Set[T]) Clear() Set[T] {
+	for key := range s {
+		delete(s, key)
+	}
+	return s
+}

vendor/modules.txt

@@ -1749,6 +1749,7 @@ k8s.io/utils/io
 k8s.io/utils/keymutex
 k8s.io/utils/net
 k8s.io/utils/pointer
+k8s.io/utils/set
 k8s.io/utils/strings/slices
 k8s.io/utils/trace
 # oras.land/oras-go v1.2.3