azure: Add support for application security groups

pkg/model/azuremodel/context.go

@@ -77,6 +77,26 @@ func (c *AzureModelContext) NameForLoadBalancer() string {
 	return "api-" + c.ClusterName()
 }
 
+// NameForApplicationSecurityGroupControlPlane returns the name of the Application Security Group object for the ControlPlane role.
+func (c *AzureModelContext) NameForApplicationSecurityGroupControlPlane() string {
+	return kops.InstanceGroupRoleControlPlane.ToLowerString() + "." + c.ClusterName()
+}
+
+// NameForApplicationSecurityGroupNodes returns the name of the Application Security Group object for the Node role.
+func (c *AzureModelContext) NameForApplicationSecurityGroupNodes() string {
+	return kops.InstanceGroupRoleNode.ToLowerString() + "s." + c.ClusterName()
+}
+
+// LinkToApplicationSecurityGroupControlPlane returns the Application Security Group object for the ControlPlane role.
+func (c *AzureModelContext) LinkToApplicationSecurityGroupControlPlane() *azuretasks.ApplicationSecurityGroup {
+	return &azuretasks.ApplicationSecurityGroup{Name: fi.PtrTo(c.NameForApplicationSecurityGroupControlPlane())}
+}
+
+// LinkToApplicationSecurityGroupNodes returns the Application Security Group object for the Node role.
+func (c *AzureModelContext) LinkToApplicationSecurityGroupNodes() *azuretasks.ApplicationSecurityGroup {
+	return &azuretasks.ApplicationSecurityGroup{Name: fi.PtrTo(c.NameForApplicationSecurityGroupNodes())}
+}
+
 // CloudTagsForInstanceGroup computes the tags to apply to instances in the specified InstanceGroup
 // Mostly copied from pkg/model/context.go, but "/" in tag keys are replaced with "_" as Azure
 // doesn't allow "/" in tag keys.

pkg/model/azuremodel/network.go

@@ -17,7 +17,6 @@ limitations under the License.
 package azuremodel
 
 import (
-	"fmt"
 	"strconv"
 
 	"github.com/Azure/azure-sdk-for-go/services/network/mgmt/2022-05-01/network"
@@ -54,167 +53,220 @@ func (b *NetworkModelBuilder) Build(c *fi.CloudupModelBuilderContext) error {
 		ResourceGroup: b.LinkToResourceGroup(),
 		Tags:          map[string]*string{},
 	}
-	var sshAccessIPv4, sshAccessIPv6 []string
+	sshAccessIPv4 := ipv4CIDRs(b.Cluster.Spec.SSHAccess)
-	for _, cidr := range b.Cluster.Spec.SSHAccess {
-		switch net.IPFamilyOfCIDRString(cidr) {
-		case net.IPv4:
-			sshAccessIPv4 = append(sshAccessIPv4, cidr)
-		case net.IPv6:
-			sshAccessIPv6 = append(sshAccessIPv6, cidr)
-		default:
-			return fmt.Errorf("unknown IP family for CIDR: %q", cidr)
-		}
-	}
 	if len(sshAccessIPv4) > 0 {
 		nsgTask.SecurityRules = append(nsgTask.SecurityRules, &azuretasks.NetworkSecurityRule{
-			Name:                     fi.PtrTo("AllowSSH"),
+			Name:                  fi.PtrTo("AllowSSH"),
-			Priority:                 fi.PtrTo[int32](100),
+			Priority:              fi.PtrTo[int32](100),
-			Access:                   network.SecurityRuleAccessAllow,
+			Access:                network.SecurityRuleAccessAllow,
-			Direction:                network.SecurityRuleDirectionInbound,
+			Direction:             network.SecurityRuleDirectionInbound,
-			Protocol:                 network.SecurityRuleProtocolTCP,
+			Protocol:              network.SecurityRuleProtocolTCP,
-			SourceAddressPrefixes:    &sshAccessIPv4,
+			SourceAddressPrefixes: &sshAccessIPv4,
-			SourcePortRange:          fi.PtrTo("*"),
+			SourcePortRange:       fi.PtrTo("*"),
-			DestinationAddressPrefix: fi.PtrTo("*"),
+			DestinationApplicationSecurityGroupNames: &[]string{
-			DestinationPortRange:     fi.PtrTo("22"),
+				b.NameForApplicationSecurityGroupControlPlane(),
+				b.NameForApplicationSecurityGroupNodes(),
+			},
+			DestinationPortRange: fi.PtrTo("22"),
 		})
 	}
+	sshAccessIPv6 := ipv6CIDRs(b.Cluster.Spec.SSHAccess)
 	if len(sshAccessIPv6) > 0 {
 		nsgTask.SecurityRules = append(nsgTask.SecurityRules, &azuretasks.NetworkSecurityRule{
-			Name:                     fi.PtrTo("AllowSSH_v6"),
+			Name:                  fi.PtrTo("AllowSSH_v6"),
-			Priority:                 fi.PtrTo[int32](101),
+			Priority:              fi.PtrTo[int32](101),
-			Access:                   network.SecurityRuleAccessAllow,
+			Access:                network.SecurityRuleAccessAllow,
-			Direction:                network.SecurityRuleDirectionInbound,
+			Direction:             network.SecurityRuleDirectionInbound,
-			Protocol:                 network.SecurityRuleProtocolTCP,
+			Protocol:              network.SecurityRuleProtocolTCP,
-			SourceAddressPrefixes:    &sshAccessIPv6,
+			SourceAddressPrefixes: &sshAccessIPv6,
-			SourcePortRange:          fi.PtrTo("*"),
+			SourcePortRange:       fi.PtrTo("*"),
-			DestinationAddressPrefix: fi.PtrTo("*"),
+			DestinationApplicationSecurityGroupNames: &[]string{
-			DestinationPortRange:     fi.PtrTo("22"),
+				b.NameForApplicationSecurityGroupControlPlane(),
+				b.NameForApplicationSecurityGroupNodes(),
+			},
+			DestinationPortRange: fi.PtrTo("22"),
 		})
 	}
-	var k8sAccessIPv4, k8sAccessIPv6 []string
+	k8sAccessIPv4 := ipv4CIDRs(b.Cluster.Spec.API.Access)
-	for _, cidr := range b.Cluster.Spec.API.Access {
-		switch net.IPFamilyOfCIDRString(cidr) {
-		case net.IPv4:
-			k8sAccessIPv4 = append(k8sAccessIPv4, cidr)
-		case net.IPv6:
-			k8sAccessIPv6 = append(k8sAccessIPv6, cidr)
-		default:
-			return fmt.Errorf("unknown IP family for CIDR: %q", cidr)
-		}
-	}
 	if len(k8sAccessIPv4) > 0 {
 		nsgTask.SecurityRules = append(nsgTask.SecurityRules, &azuretasks.NetworkSecurityRule{
-			Name:                     fi.PtrTo("AllowKubernetesAPI"),
+			Name:                                     fi.PtrTo("AllowKubernetesAPI"),
-			Priority:                 fi.PtrTo[int32](200),
+			Priority:                                 fi.PtrTo[int32](200),
-			Access:                   network.SecurityRuleAccessAllow,
+			Access:                                   network.SecurityRuleAccessAllow,
-			Direction:                network.SecurityRuleDirectionInbound,
+			Direction:                                network.SecurityRuleDirectionInbound,
-			Protocol:                 network.SecurityRuleProtocolTCP,
+			Protocol:                                 network.SecurityRuleProtocolTCP,
-			SourceAddressPrefixes:    &k8sAccessIPv4,
+			SourceAddressPrefixes:                    &k8sAccessIPv4,
-			SourcePortRange:          fi.PtrTo("*"),
+			SourcePortRange:                          fi.PtrTo("*"),
-			DestinationAddressPrefix: fi.PtrTo("*"),
+			DestinationApplicationSecurityGroupNames: &[]string{b.NameForApplicationSecurityGroupControlPlane()},
-			DestinationPortRange:     fi.PtrTo(strconv.Itoa(wellknownports.KubeAPIServer)),
+			DestinationPortRange:                     fi.PtrTo(strconv.Itoa(wellknownports.KubeAPIServer)),
 		})
 	}
+	k8sAccessIPv6 := ipv6CIDRs(b.Cluster.Spec.API.Access)
 	if len(k8sAccessIPv6) > 0 {
 		nsgTask.SecurityRules = append(nsgTask.SecurityRules, &azuretasks.NetworkSecurityRule{
-			Name:                     fi.PtrTo("AllowKubernetesAPI_v6"),
+			Name:                                     fi.PtrTo("AllowKubernetesAPI_v6"),
-			Priority:                 fi.PtrTo[int32](201),
+			Priority:                                 fi.PtrTo[int32](201),
-			Access:                   network.SecurityRuleAccessAllow,
+			Access:                                   network.SecurityRuleAccessAllow,
-			Direction:                network.SecurityRuleDirectionInbound,
+			Direction:                                network.SecurityRuleDirectionInbound,
-			Protocol:                 network.SecurityRuleProtocolTCP,
+			Protocol:                                 network.SecurityRuleProtocolTCP,
-			SourceAddressPrefixes:    &k8sAccessIPv6,
+			SourceAddressPrefixes:                    &k8sAccessIPv6,
-			SourcePortRange:          fi.PtrTo("*"),
+			SourcePortRange:                          fi.PtrTo("*"),
-			DestinationAddressPrefix: fi.PtrTo("*"),
+			DestinationApplicationSecurityGroupNames: &[]string{b.NameForApplicationSecurityGroupControlPlane()},
-			DestinationPortRange:     fi.PtrTo(strconv.Itoa(wellknownports.KubeAPIServer)),
+			DestinationPortRange:                     fi.PtrTo(strconv.Itoa(wellknownports.KubeAPIServer)),
 		})
 	}
-	if b.Cluster.UsesNoneDNS() {
+	nodePortAccessIPv4 := ipv4CIDRs(b.Cluster.Spec.NodePortAccess)
-		if b.Cluster.Spec.API.LoadBalancer != nil && b.Cluster.Spec.API.LoadBalancer.Type == kops.LoadBalancerTypeInternal {
-			nsgTask.SecurityRules = append(nsgTask.SecurityRules, &azuretasks.NetworkSecurityRule{
-				Name:                     fi.PtrTo("AllowKopsController"),
-				Priority:                 fi.PtrTo[int32](210),
-				Access:                   network.SecurityRuleAccessAllow,
-				Direction:                network.SecurityRuleDirectionInbound,
-				Protocol:                 network.SecurityRuleProtocolTCP,
-				SourceAddressPrefix:      fi.PtrTo(b.Cluster.Spec.Networking.NetworkCIDR),
-				SourcePortRange:          fi.PtrTo("*"),
-				DestinationAddressPrefix: fi.PtrTo("*"),
-				DestinationPortRange:     fi.PtrTo(strconv.Itoa(wellknownports.KopsControllerPort)),
-			})
-		} else {
-			// TODO: Limit access to necessary source address prefixes instead of "0.0.0.0/0" and "::/0"
-			nsgTask.SecurityRules = append(nsgTask.SecurityRules, &azuretasks.NetworkSecurityRule{
-				Name:                     fi.PtrTo("AllowKopsController"),
-				Priority:                 fi.PtrTo[int32](210),
-				Access:                   network.SecurityRuleAccessAllow,
-				Direction:                network.SecurityRuleDirectionInbound,
-				Protocol:                 network.SecurityRuleProtocolTCP,
-				SourceAddressPrefix:      fi.PtrTo("0.0.0.0/0"),
-				SourcePortRange:          fi.PtrTo("*"),
-				DestinationAddressPrefix: fi.PtrTo("*"),
-				DestinationPortRange:     fi.PtrTo(strconv.Itoa(wellknownports.KopsControllerPort)),
-			})
-			nsgTask.SecurityRules = append(nsgTask.SecurityRules, &azuretasks.NetworkSecurityRule{
-				Name:                     fi.PtrTo("AllowKopsController_v6"),
-				Priority:                 fi.PtrTo[int32](211),
-				Access:                   network.SecurityRuleAccessAllow,
-				Direction:                network.SecurityRuleDirectionInbound,
-				Protocol:                 network.SecurityRuleProtocolTCP,
-				SourceAddressPrefix:      fi.PtrTo("::/0"),
-				SourcePortRange:          fi.PtrTo("*"),
-				DestinationAddressPrefix: fi.PtrTo("*"),
-				DestinationPortRange:     fi.PtrTo(strconv.Itoa(wellknownports.KopsControllerPort)),
-			})
-		}
-	}
-	nsgTask.SecurityRules = append(nsgTask.SecurityRules, &azuretasks.NetworkSecurityRule{
-		Name:                     fi.PtrTo("AllowNodeupChallenge"),
-		Priority:                 fi.PtrTo[int32](220),
-		Access:                   network.SecurityRuleAccessAllow,
-		Direction:                network.SecurityRuleDirectionInbound,
-		Protocol:                 network.SecurityRuleProtocolTCP,
-		SourceAddressPrefix:      fi.PtrTo(b.Cluster.Spec.Networking.NetworkCIDR),
-		SourcePortRange:          fi.PtrTo("*"),
-		DestinationAddressPrefix: fi.PtrTo("*"),
-		DestinationPortRange:     fi.PtrTo(strconv.Itoa(wellknownports.NodeupChallenge)),
-	})
-	var nodePortAccessIPv4, nodePortAccessIPv6 []string
-	for _, cidr := range b.Cluster.Spec.NodePortAccess {
-		switch net.IPFamilyOfCIDRString(cidr) {
-		case net.IPv4:
-			nodePortAccessIPv4 = append(nodePortAccessIPv4, cidr)
-		case net.IPv6:
-			nodePortAccessIPv6 = append(nodePortAccessIPv6, cidr)
-		default:
-			return fmt.Errorf("unknown IP family for CIDR: %q", cidr)
-		}
-	}
 	if len(nodePortAccessIPv4) > 0 {
 		nsgTask.SecurityRules = append(nsgTask.SecurityRules, &azuretasks.NetworkSecurityRule{
-			Name:                     fi.PtrTo("AllowNodePort"),
+			Name:                                     fi.PtrTo("AllowNodePortTCP"),
-			Priority:                 fi.PtrTo[int32](300),
+			Priority:                                 fi.PtrTo[int32](300),
-			Access:                   network.SecurityRuleAccessAllow,
+			Access:                                   network.SecurityRuleAccessAllow,
-			Direction:                network.SecurityRuleDirectionInbound,
+			Direction:                                network.SecurityRuleDirectionInbound,
-			Protocol:                 network.SecurityRuleProtocolTCP,
+			Protocol:                                 network.SecurityRuleProtocolAsterisk,
-			SourceAddressPrefixes:    &nodePortAccessIPv4,
+			SourceAddressPrefixes:                    &nodePortAccessIPv4,
-			SourcePortRange:          fi.PtrTo("*"),
+			SourcePortRange:                          fi.PtrTo("*"),
-			DestinationAddressPrefix: fi.PtrTo("*"),
+			DestinationApplicationSecurityGroupNames: &[]string{b.NameForApplicationSecurityGroupNodes()},
-			DestinationPortRange:     fi.PtrTo("443"),
+			DestinationPortRange:                     fi.PtrTo("30000-32767"),
 		})
 	}
+	nodePortAccessIPv6 := ipv6CIDRs(b.Cluster.Spec.NodePortAccess)
 	if len(nodePortAccessIPv6) > 0 {
 		nsgTask.SecurityRules = append(nsgTask.SecurityRules, &azuretasks.NetworkSecurityRule{
-			Name:                     fi.PtrTo("AllowNodePort_v6"),
+			Name:                                     fi.PtrTo("AllowNodePortTCP_v6"),
-			Priority:                 fi.PtrTo[int32](301),
+			Priority:                                 fi.PtrTo[int32](301),
-			Access:                   network.SecurityRuleAccessAllow,
+			Access:                                   network.SecurityRuleAccessAllow,
-			Direction:                network.SecurityRuleDirectionInbound,
+			Direction:                                network.SecurityRuleDirectionInbound,
-			Protocol:                 network.SecurityRuleProtocolTCP,
+			Protocol:                                 network.SecurityRuleProtocolAsterisk,
-			SourceAddressPrefixes:    &nodePortAccessIPv6,
+			SourceAddressPrefixes:                    &nodePortAccessIPv6,
-			SourcePortRange:          fi.PtrTo("*"),
+			SourcePortRange:                          fi.PtrTo("*"),
-			DestinationAddressPrefix: fi.PtrTo("*"),
+			DestinationApplicationSecurityGroupNames: &[]string{b.NameForApplicationSecurityGroupNodes()},
-			DestinationPortRange:     fi.PtrTo("443"),
+			DestinationPortRange:                     fi.PtrTo("30000-32767"),
 		})
 	}
+	nsgTask.SecurityRules = append(nsgTask.SecurityRules, &azuretasks.NetworkSecurityRule{
+		Name:                                     fi.PtrTo("AllowControlPlaneToControlPlane"),
+		Priority:                                 fi.PtrTo[int32](1000),
+		Access:                                   network.SecurityRuleAccessAllow,
+		Direction:                                network.SecurityRuleDirectionInbound,
+		Protocol:                                 network.SecurityRuleProtocolAsterisk,
+		SourceApplicationSecurityGroupNames:      &[]string{b.NameForApplicationSecurityGroupControlPlane()},
+		SourcePortRange:                          fi.PtrTo("*"),
+		DestinationApplicationSecurityGroupNames: &[]string{b.NameForApplicationSecurityGroupControlPlane()},
+		DestinationPortRange:                     fi.PtrTo("*"),
+	})
+	nsgTask.SecurityRules = append(nsgTask.SecurityRules, &azuretasks.NetworkSecurityRule{
+		Name:                                     fi.PtrTo("AllowControlPlaneToNodes"),
+		Priority:                                 fi.PtrTo[int32](1001),
+		Access:                                   network.SecurityRuleAccessAllow,
+		Direction:                                network.SecurityRuleDirectionInbound,
+		Protocol:                                 network.SecurityRuleProtocolAsterisk,
+		SourceApplicationSecurityGroupNames:      &[]string{b.NameForApplicationSecurityGroupControlPlane()},
+		SourcePortRange:                          fi.PtrTo("*"),
+		DestinationApplicationSecurityGroupNames: &[]string{b.NameForApplicationSecurityGroupNodes()},
+		DestinationPortRange:                     fi.PtrTo("*"),
+	})
+	nsgTask.SecurityRules = append(nsgTask.SecurityRules, &azuretasks.NetworkSecurityRule{
+		Name:                                     fi.PtrTo("AllowNodesToNodes"),
+		Priority:                                 fi.PtrTo[int32](1002),
+		Access:                                   network.SecurityRuleAccessAllow,
+		Direction:                                network.SecurityRuleDirectionInbound,
+		Protocol:                                 network.SecurityRuleProtocolAsterisk,
+		SourceApplicationSecurityGroupNames:      &[]string{b.NameForApplicationSecurityGroupNodes()},
+		SourcePortRange:                          fi.PtrTo("*"),
+		DestinationApplicationSecurityGroupNames: &[]string{b.NameForApplicationSecurityGroupNodes()},
+		DestinationPortRange:                     fi.PtrTo("*"),
+	})
+	nsgTask.SecurityRules = append(nsgTask.SecurityRules, &azuretasks.NetworkSecurityRule{
+		Name:                                     fi.PtrTo("DenyNodesToEtcdManager"),
+		Priority:                                 fi.PtrTo[int32](1003),
+		Access:                                   network.SecurityRuleAccessDeny,
+		Direction:                                network.SecurityRuleDirectionInbound,
+		Protocol:                                 network.SecurityRuleProtocolTCP,
+		SourceApplicationSecurityGroupNames:      &[]string{b.NameForApplicationSecurityGroupNodes()},
+		SourcePortRange:                          fi.PtrTo("*"),
+		DestinationApplicationSecurityGroupNames: &[]string{b.NameForApplicationSecurityGroupControlPlane()},
+		DestinationPortRange:                     fi.PtrTo("2380-2381"),
+	})
+	nsgTask.SecurityRules = append(nsgTask.SecurityRules, &azuretasks.NetworkSecurityRule{
+		Name:                                     fi.PtrTo("DenyNodesToEtcd"),
+		Priority:                                 fi.PtrTo[int32](1004),
+		Access:                                   network.SecurityRuleAccessDeny,
+		Direction:                                network.SecurityRuleDirectionInbound,
+		Protocol:                                 network.SecurityRuleProtocolTCP,
+		SourceApplicationSecurityGroupNames:      &[]string{b.NameForApplicationSecurityGroupNodes()},
+		SourcePortRange:                          fi.PtrTo("*"),
+		DestinationApplicationSecurityGroupNames: &[]string{b.NameForApplicationSecurityGroupControlPlane()},
+		DestinationPortRange:                     fi.PtrTo("4000-4001"),
+	})
+	nsgTask.SecurityRules = append(nsgTask.SecurityRules, &azuretasks.NetworkSecurityRule{
+		Name:                                     fi.PtrTo("AllowNodesToControlPlane"),
+		Priority:                                 fi.PtrTo[int32](1005),
+		Access:                                   network.SecurityRuleAccessAllow,
+		Direction:                                network.SecurityRuleDirectionInbound,
+		Protocol:                                 network.SecurityRuleProtocolAsterisk,
+		SourceApplicationSecurityGroupNames:      &[]string{b.NameForApplicationSecurityGroupNodes()},
+		SourcePortRange:                          fi.PtrTo("*"),
+		DestinationApplicationSecurityGroupNames: &[]string{b.NameForApplicationSecurityGroupControlPlane()},
+		DestinationPortRange:                     fi.PtrTo("*"),
+	})
+	if b.Cluster.UsesNoneDNS() && b.Cluster.Spec.API.LoadBalancer != nil && b.Cluster.Spec.API.LoadBalancer.Type == kops.LoadBalancerTypePublic {
+		// TODO: Limit access to necessary source address prefixes instead of "0.0.0.0/0" and "::/0"
+		nsgTask.SecurityRules = append(nsgTask.SecurityRules, &azuretasks.NetworkSecurityRule{
+			Name:                                     fi.PtrTo("AllowNodesToKubernetesAPI"),
+			Priority:                                 fi.PtrTo[int32](2000),
+			Access:                                   network.SecurityRuleAccessAllow,
+			Direction:                                network.SecurityRuleDirectionInbound,
+			Protocol:                                 network.SecurityRuleProtocolTCP,
+			SourceAddressPrefix:                      fi.PtrTo("*"),
+			SourcePortRange:                          fi.PtrTo("*"),
+			DestinationApplicationSecurityGroupNames: &[]string{b.NameForApplicationSecurityGroupControlPlane()},
+			DestinationPortRange:                     fi.PtrTo(strconv.Itoa(wellknownports.KubeAPIServer)),
+		})
+		nsgTask.SecurityRules = append(nsgTask.SecurityRules, &azuretasks.NetworkSecurityRule{
+			Name:                                     fi.PtrTo("AllowNodesToKopsController"),
+			Priority:                                 fi.PtrTo[int32](2001),
+			Access:                                   network.SecurityRuleAccessAllow,
+			Direction:                                network.SecurityRuleDirectionInbound,
+			Protocol:                                 network.SecurityRuleProtocolTCP,
+			SourceAddressPrefix:                      fi.PtrTo("*"),
+			SourcePortRange:                          fi.PtrTo("*"),
+			DestinationApplicationSecurityGroupNames: &[]string{b.NameForApplicationSecurityGroupControlPlane()},
+			DestinationPortRange:                     fi.PtrTo(strconv.Itoa(wellknownports.KopsControllerPort)),
+		})
+	}
+	nsgTask.SecurityRules = append(nsgTask.SecurityRules, &azuretasks.NetworkSecurityRule{
+		Name:                     fi.PtrTo("AllowAzureLoadBalancer"),
+		Priority:                 fi.PtrTo[int32](4000),
+		Access:                   network.SecurityRuleAccessAllow,
+		Direction:                network.SecurityRuleDirectionInbound,
+		Protocol:                 network.SecurityRuleProtocolAsterisk,
+		SourceAddressPrefix:      fi.PtrTo("AzureLoadBalancer"),
+		SourcePortRange:          fi.PtrTo("*"),
+		DestinationAddressPrefix: fi.PtrTo("VirtualNetwork"),
+		DestinationPortRange:     fi.PtrTo("*"),
+	})
+	nsgTask.SecurityRules = append(nsgTask.SecurityRules, &azuretasks.NetworkSecurityRule{
+		Name:                                     fi.PtrTo("DenyAllToControlPlane"),
+		Priority:                                 fi.PtrTo[int32](4001),
+		Access:                                   network.SecurityRuleAccessDeny,
+		Direction:                                network.SecurityRuleDirectionInbound,
+		Protocol:                                 network.SecurityRuleProtocolAsterisk,
+		SourceAddressPrefix:                      fi.PtrTo("*"),
+		SourcePortRange:                          fi.PtrTo("*"),
+		DestinationApplicationSecurityGroupNames: &[]string{b.NameForApplicationSecurityGroupControlPlane()},
+		DestinationPortRange:                     fi.PtrTo("*"),
+	})
+	nsgTask.SecurityRules = append(nsgTask.SecurityRules, &azuretasks.NetworkSecurityRule{
+		Name:                                     fi.PtrTo("DenyAllToNodes"),
+		Priority:                                 fi.PtrTo[int32](4002),
+		Access:                                   network.SecurityRuleAccessDeny,
+		Direction:                                network.SecurityRuleDirectionInbound,
+		Protocol:                                 network.SecurityRuleProtocolAsterisk,
+		SourceAddressPrefix:                      fi.PtrTo("*"),
+		SourcePortRange:                          fi.PtrTo("*"),
+		DestinationApplicationSecurityGroupNames: &[]string{b.NameForApplicationSecurityGroupNodes()},
+		DestinationPortRange:                     fi.PtrTo("*"),
+	})
 	c.AddTask(nsgTask)
 
 	for _, subnetSpec := range b.Cluster.Spec.Networking.Subnets {
@@ -241,3 +293,23 @@ func (b *NetworkModelBuilder) Build(c *fi.CloudupModelBuilderContext) error {
 
 	return nil
 }
+
+func ipv4CIDRs(mixedCIDRs []string) []string {
+	var cidrs []string
+	for _, cidr := range mixedCIDRs {
+		if net.IPFamilyOfCIDRString(cidr) == net.IPv4 {
+			cidrs = append(cidrs, cidr)
+		}
+	}
+	return cidrs
+}
+
+func ipv6CIDRs(mixedCIDRs []string) []string {
+	var cidrs []string
+	for _, cidr := range mixedCIDRs {
+		if net.IPFamilyOfCIDRString(cidr) == net.IPv6 {
+			cidrs = append(cidrs, cidr)
+		}
+	}
+	return cidrs
+}

pkg/model/azuremodel/vmscaleset.go

@@ -41,6 +41,19 @@ var _ fi.CloudupModelBuilder = &VMScaleSetModelBuilder{}
 
 // Build is responsible for constructing the VM ScaleSet from the kops spec.
 func (b *VMScaleSetModelBuilder) Build(c *fi.CloudupModelBuilderContext) error {
+	c.AddTask(&azuretasks.ApplicationSecurityGroup{
+		Name:          fi.PtrTo(b.NameForApplicationSecurityGroupControlPlane()),
+		Lifecycle:     b.Lifecycle,
+		ResourceGroup: b.LinkToResourceGroup(),
+		Tags:          map[string]*string{},
+	})
+	c.AddTask(&azuretasks.ApplicationSecurityGroup{
+		Name:          fi.PtrTo(b.NameForApplicationSecurityGroupNodes()),
+		Lifecycle:     b.Lifecycle,
+		ResourceGroup: b.LinkToResourceGroup(),
+		Tags:          map[string]*string{},
+	})
+
 	for _, ig := range b.InstanceGroups {
 		name := b.AutoscalingGroupName(ig)
 		vmss, err := b.buildVMScaleSetTask(c, name, ig)
@@ -92,6 +105,15 @@ func (b *VMScaleSetModelBuilder) buildVMScaleSetTask(
 		Zones:              azNumbers,
 	}
 
+	switch ig.Spec.Role {
+	case kops.InstanceGroupRoleControlPlane:
+		t.ApplicationSecurityGroups = append(t.ApplicationSecurityGroups, b.LinkToApplicationSecurityGroupControlPlane())
+	case kops.InstanceGroupRoleNode:
+		t.ApplicationSecurityGroups = append(t.ApplicationSecurityGroups, b.LinkToApplicationSecurityGroupNodes())
+	default:
+		return nil, fmt.Errorf("unexpected instance group role for instance group: %q, %q", ig.Name, ig.Spec.Role)
+	}
+
 	var err error
 	if t.Capacity, err = getCapacity(&ig.Spec); err != nil {
 		return nil, err

pkg/resources/azure/azure.go

@@ -31,16 +31,17 @@ import (
 )
 
 const (
-	typeResourceGroup        = "ResourceGroup"
+	typeResourceGroup            = "ResourceGroup"
-	typeVirtualNetwork       = "VirtualNetwork"
+	typeVirtualNetwork           = "VirtualNetwork"
-	typeNetworkSecurityGroup = "NetworkSecurityGroup"
+	typeNetworkSecurityGroup     = "NetworkSecurityGroup"
-	typeSubnet               = "Subnet"
+	typeApplicationSecurityGroup = "ApplicationSecurityGroup"
-	typeRouteTable           = "RouteTable"
+	typeSubnet                   = "Subnet"
-	typeVMScaleSet           = "VMScaleSet"
+	typeRouteTable               = "RouteTable"
-	typeDisk                 = "Disk"
+	typeVMScaleSet               = "VMScaleSet"
-	typeRoleAssignment       = "RoleAssignment"
+	typeDisk                     = "Disk"
-	typeLoadBalancer         = "LoadBalancer"
+	typeRoleAssignment           = "RoleAssignment"
-	typePublicIPAddress      = "PublicIPAddress"
+	typeLoadBalancer             = "LoadBalancer"
+	typePublicIPAddress          = "PublicIPAddress"
 )
 
 // ListResourcesAzure lists all resources for the cluster by quering Azure.
@@ -87,6 +88,7 @@ func (g *resourceGetter) listAll() ([]*resources.Resource, error) {
 		g.listResourceGroups,
 		g.listVirtualNetworksAndSubnets,
 		g.listNetworkSecurityGroups,
+		g.listApplicationSecurityGroups,
 		g.listRouteTables,
 		g.listVMScaleSetsAndRoleAssignments,
 		g.listDisks,
@@ -242,12 +244,46 @@ func (g *resourceGetter) listNetworkSecurityGroups(ctx context.Context) ([]*reso
 
 	var rs []*resources.Resource
 	for i := range NetworkSecurityGroups {
-		rs = append(rs, g.toNetworkSecurityGroupResource(&NetworkSecurityGroups[i]))
+		r, err := g.toNetworkSecurityGroupResource(&NetworkSecurityGroups[i])
+		if err != nil {
+			return nil, err
+		}
+		rs = append(rs, r)
 	}
 	return rs, nil
 }
 
-func (g *resourceGetter) toNetworkSecurityGroupResource(NetworkSecurityGroup *network.SecurityGroup) *resources.Resource {
+func (g *resourceGetter) toNetworkSecurityGroupResource(NetworkSecurityGroup *network.SecurityGroup) (*resources.Resource, error) {
+	var blocks []string
+	blocks = append(blocks, toKey(typeResourceGroup, g.resourceGroupName()))
+
+	asgs := set.New[string]()
+	if NetworkSecurityGroup.SecurityRules != nil {
+		for _, nsr := range *NetworkSecurityGroup.SecurityRules {
+			if nsr.SourceApplicationSecurityGroups != nil {
+				for _, sasg := range *nsr.SourceApplicationSecurityGroups {
+					asgID, err := azure.ParseApplicationSecurityGroupID(*sasg.ID)
+					if err != nil {
+						return nil, fmt.Errorf("parsing application security group ID: %w", err)
+					}
+					asgs.Insert(asgID.ApplicationSecurityGroupName)
+				}
+			}
+			if nsr.DestinationApplicationSecurityGroups != nil {
+				for _, dasg := range *nsr.DestinationApplicationSecurityGroups {
+					asgID, err := azure.ParseApplicationSecurityGroupID(*dasg.ID)
+					if err != nil {
+						return nil, fmt.Errorf("parsing application security group ID: %w", err)
+					}
+					asgs.Insert(asgID.ApplicationSecurityGroupName)
+				}
+			}
+		}
+	}
+	for asg := range asgs {
+		blocks = append(blocks, toKey(typeApplicationSecurityGroup, asg))
+	}
+
 	return &resources.Resource{
 		Obj:  NetworkSecurityGroup,
 		Type: typeNetworkSecurityGroup,
@@ -256,17 +292,46 @@ func (g *resourceGetter) toNetworkSecurityGroupResource(NetworkSecurityGroup *ne
 		Deleter: func(_ fi.Cloud, r *resources.Resource) error {
 			return g.deleteNetworkSecurityGroup(r)
 		},
-		Blocks: []string{
+		Blocks: blocks,
-			toKey(typeResourceGroup, g.resourceGroupName()),
+	}, nil
-		},
-		Shared: g.clusterInfo.AzureNetworkShared,
-	}
 }
 
 func (g *resourceGetter) deleteNetworkSecurityGroup(r *resources.Resource) error {
 	return g.cloud.NetworkSecurityGroup().Delete(context.TODO(), g.resourceGroupName(), r.Name)
 }
 
+func (g *resourceGetter) listApplicationSecurityGroups(ctx context.Context) ([]*resources.Resource, error) {
+	ApplicationSecurityGroups, err := g.cloud.ApplicationSecurityGroup().List(ctx, g.resourceGroupName())
+	if err != nil {
+		return nil, err
+	}
+
+	var rs []*resources.Resource
+	for i := range ApplicationSecurityGroups {
+		rs = append(rs, g.toApplicationSecurityGroupResource(&ApplicationSecurityGroups[i]))
+	}
+	return rs, nil
+}
+
+func (g *resourceGetter) toApplicationSecurityGroupResource(ApplicationSecurityGroup *network.ApplicationSecurityGroup) *resources.Resource {
+	return &resources.Resource{
+		Obj:  ApplicationSecurityGroup,
+		Type: typeApplicationSecurityGroup,
+		ID:   *ApplicationSecurityGroup.Name,
+		Name: *ApplicationSecurityGroup.Name,
+		Deleter: func(_ fi.Cloud, r *resources.Resource) error {
+			return g.deleteApplicationSecurityGroup(r)
+		},
+		Blocks: []string{
+			toKey(typeResourceGroup, g.resourceGroupName()),
+		},
+	}
+}
+
+func (g *resourceGetter) deleteApplicationSecurityGroup(r *resources.Resource) error {
+	return g.cloud.ApplicationSecurityGroup().Delete(context.TODO(), g.resourceGroupName(), r.Name)
+}
+
 func (g *resourceGetter) listRouteTables(ctx context.Context) ([]*resources.Resource, error) {
 	rts, err := g.cloud.RouteTable().List(ctx, g.resourceGroupName())
 	if err != nil {
@@ -344,20 +409,30 @@ func (g *resourceGetter) toVMScaleSetResource(vmss *compute.VirtualMachineScaleS
 
 	vnets := set.New[string]()
 	subnets := set.New[string]()
+	asgs := set.New[string]()
 	lbs := set.New[string]()
 	for _, iface := range *vmss.VirtualMachineProfile.NetworkProfile.NetworkInterfaceConfigurations {
 		for _, ip := range *iface.IPConfigurations {
 			subnetID, err := azure.ParseSubnetID(*ip.Subnet.ID)
 			if err != nil {
-				return nil, fmt.Errorf("error on parsing subnet ID: %s", err)
+				return nil, fmt.Errorf("parsing subnet ID: %w", err)
 			}
 			vnets.Insert(subnetID.VirtualNetworkName)
 			subnets.Insert(subnetID.SubnetName)
+			if ip.ApplicationSecurityGroups != nil {
+				for _, asg := range *ip.ApplicationSecurityGroups {
+					asgID, err := azure.ParseApplicationSecurityGroupID(*asg.ID)
+					if err != nil {
+						return nil, fmt.Errorf("parsing application security group ID: %w", err)
+					}
+					asgs.Insert(asgID.ApplicationSecurityGroupName)
+				}
+			}
 			if ip.LoadBalancerBackendAddressPools != nil {
 				for _, lb := range *ip.LoadBalancerBackendAddressPools {
 					lbID, err := azure.ParseLoadBalancerID(*lb.ID)
 					if err != nil {
-						return nil, fmt.Errorf("parsing load balancer ID: %s", err)
+						return nil, fmt.Errorf("parsing load balancer ID: %w", err)
 					}
 					lbs.Insert(lbID.LoadBalancerName)
 				}
@@ -370,6 +445,9 @@ func (g *resourceGetter) toVMScaleSetResource(vmss *compute.VirtualMachineScaleS
 	for subnet := range subnets {
 		blocks = append(blocks, toKey(typeSubnet, subnet))
 	}
+	for asg := range asgs {
+		blocks = append(blocks, toKey(typeApplicationSecurityGroup, asg))
+	}
 	for lb := range lbs {
 		blocks = append(blocks, toKey(typeLoadBalancer, lb))
 	}

upup/pkg/fi/cloudup/azure/applicationsecuritygroup.go

@@ -0,0 +1,83 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package azure
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/Azure/azure-sdk-for-go/services/network/mgmt/2022-05-01/network"
+	"github.com/Azure/go-autorest/autorest"
+)
+
+// ApplicationSecurityGroupsClient is a client for managing Application Security Groups.
+type ApplicationSecurityGroupsClient interface {
+	CreateOrUpdate(ctx context.Context, resourceGroupName, applicationSecurityGroupName string, parameters network.ApplicationSecurityGroup) (*network.ApplicationSecurityGroup, error)
+	List(ctx context.Context, resourceGroupName string) ([]network.ApplicationSecurityGroup, error)
+	Delete(ctx context.Context, resourceGroupName, applicationSecurityGroupName string) error
+}
+
+type ApplicationSecurityGroupsClientImpl struct {
+	c *network.ApplicationSecurityGroupsClient
+}
+
+var _ ApplicationSecurityGroupsClient = &ApplicationSecurityGroupsClientImpl{}
+
+func (c *ApplicationSecurityGroupsClientImpl) CreateOrUpdate(ctx context.Context, resourceGroupName, applicationSecurityGroupName string, parameters network.ApplicationSecurityGroup) (*network.ApplicationSecurityGroup, error) {
+	future, err := c.c.CreateOrUpdate(ctx, resourceGroupName, applicationSecurityGroupName, parameters)
+	if err != nil {
+		return nil, fmt.Errorf("creating/updating Application Security Group: %w", err)
+	}
+	if err := future.WaitForCompletionRef(ctx, c.c.Client); err != nil {
+		return nil, fmt.Errorf("waiting for Application Security Group create/update completion: %w", err)
+	}
+	asg, err := future.Result(*c.c)
+	if err != nil {
+		return nil, fmt.Errorf("obtaining result for Application Security Group create/update: %w", err)
+	}
+	return &asg, err
+}
+
+func (c *ApplicationSecurityGroupsClientImpl) List(ctx context.Context, resourceGroupName string) ([]network.ApplicationSecurityGroup, error) {
+	var l []network.ApplicationSecurityGroup
+	for iter, err := c.c.ListComplete(ctx, resourceGroupName); iter.NotDone(); err = iter.NextWithContext(ctx) {
+		if err != nil {
+			return nil, fmt.Errorf("listing application security groups: %w", err)
+		}
+		l = append(l, iter.Value())
+	}
+	return l, nil
+}
+
+func (c *ApplicationSecurityGroupsClientImpl) Delete(ctx context.Context, resourceGroupName, applicationSecurityGroupName string) error {
+	future, err := c.c.Delete(ctx, resourceGroupName, applicationSecurityGroupName)
+	if err != nil {
+		return fmt.Errorf("deleting application security group: %w", err)
+	}
+	if err := future.WaitForCompletionRef(ctx, c.c.Client); err != nil {
+		return fmt.Errorf("waiting for application security group deletion completion: %w", err)
+	}
+	return nil
+}
+
+func newApplicationSecurityGroupsClientImpl(subscriptionID string, authorizer autorest.Authorizer) *ApplicationSecurityGroupsClientImpl {
+	c := network.NewApplicationSecurityGroupsClient(subscriptionID)
+	c.Authorizer = authorizer
+	return &ApplicationSecurityGroupsClientImpl{
+		c: &c,
+	}
+}

upup/pkg/fi/cloudup/azure/azure_cloud.go

@@ -50,6 +50,7 @@ type AzureCloud interface {
 	Subnet() SubnetsClient
 	RouteTable() RouteTablesClient
 	NetworkSecurityGroup() NetworkSecurityGroupsClient
+	ApplicationSecurityGroup() ApplicationSecurityGroupsClient
 	VMScaleSet() VMScaleSetsClient
 	VMScaleSetVM() VMScaleSetVMsClient
 	Disk() DisksClient
@@ -60,21 +61,22 @@ type AzureCloud interface {
 }
 
 type azureCloudImplementation struct {
-	subscriptionID              string
+	subscriptionID                  string
-	location                    string
+	location                        string
-	tags                        map[string]string
+	tags                            map[string]string
-	resourceGroupsClient        ResourceGroupsClient
+	resourceGroupsClient            ResourceGroupsClient
-	networkSecurityGroupsClient NetworkSecurityGroupsClient
+	networkSecurityGroupsClient     NetworkSecurityGroupsClient
-	vnetsClient                 VirtualNetworksClient
+	applicationSecurityGroupsClient ApplicationSecurityGroupsClient
-	subnetsClient               SubnetsClient
+	vnetsClient                     VirtualNetworksClient
-	routeTablesClient           RouteTablesClient
+	subnetsClient                   SubnetsClient
-	vmscaleSetsClient           VMScaleSetsClient
+	routeTablesClient               RouteTablesClient
-	vmscaleSetVMsClient         VMScaleSetVMsClient
+	vmscaleSetsClient               VMScaleSetsClient
-	disksClient                 DisksClient
+	vmscaleSetVMsClient             VMScaleSetVMsClient
-	roleAssignmentsClient       RoleAssignmentsClient
+	disksClient                     DisksClient
-	networkInterfacesClient     NetworkInterfacesClient
+	roleAssignmentsClient           RoleAssignmentsClient
-	loadBalancersClient         LoadBalancersClient
+	networkInterfacesClient         NetworkInterfacesClient
-	publicIPAddressesClient     PublicIPAddressesClient
+	loadBalancersClient             LoadBalancersClient
+	publicIPAddressesClient         PublicIPAddressesClient
 }
 
 var _ fi.Cloud = &azureCloudImplementation{}
@@ -87,21 +89,22 @@ func NewAzureCloud(subscriptionID, location string, tags map[string]string) (Azu
 	}
 
 	return &azureCloudImplementation{
-		subscriptionID:              subscriptionID,
+		subscriptionID:                  subscriptionID,
-		location:                    location,
+		location:                        location,
-		tags:                        tags,
+		tags:                            tags,
-		resourceGroupsClient:        newResourceGroupsClientImpl(subscriptionID, authorizer),
+		resourceGroupsClient:            newResourceGroupsClientImpl(subscriptionID, authorizer),
-		vnetsClient:                 newVirtualNetworksClientImpl(subscriptionID, authorizer),
+		vnetsClient:                     newVirtualNetworksClientImpl(subscriptionID, authorizer),
-		subnetsClient:               newSubnetsClientImpl(subscriptionID, authorizer),
+		subnetsClient:                   newSubnetsClientImpl(subscriptionID, authorizer),
-		routeTablesClient:           newRouteTablesClientImpl(subscriptionID, authorizer),
+		routeTablesClient:               newRouteTablesClientImpl(subscriptionID, authorizer),
-		networkSecurityGroupsClient: newNetworkSecurityGroupsClientImpl(subscriptionID, authorizer),
+		networkSecurityGroupsClient:     newNetworkSecurityGroupsClientImpl(subscriptionID, authorizer),
-		vmscaleSetsClient:           newVMScaleSetsClientImpl(subscriptionID, authorizer),
+		applicationSecurityGroupsClient: newApplicationSecurityGroupsClientImpl(subscriptionID, authorizer),
-		vmscaleSetVMsClient:         newVMScaleSetVMsClientImpl(subscriptionID, authorizer),
+		vmscaleSetsClient:               newVMScaleSetsClientImpl(subscriptionID, authorizer),
-		disksClient:                 newDisksClientImpl(subscriptionID, authorizer),
+		vmscaleSetVMsClient:             newVMScaleSetVMsClientImpl(subscriptionID, authorizer),
-		roleAssignmentsClient:       newRoleAssignmentsClientImpl(subscriptionID, authorizer),
+		disksClient:                     newDisksClientImpl(subscriptionID, authorizer),
-		networkInterfacesClient:     newNetworkInterfacesClientImpl(subscriptionID, authorizer),
+		roleAssignmentsClient:           newRoleAssignmentsClientImpl(subscriptionID, authorizer),
-		loadBalancersClient:         newLoadBalancersClientImpl(subscriptionID, authorizer),
+		networkInterfacesClient:         newNetworkInterfacesClientImpl(subscriptionID, authorizer),
-		publicIPAddressesClient:     newPublicIPAddressesClientImpl(subscriptionID, authorizer),
+		loadBalancersClient:             newLoadBalancersClientImpl(subscriptionID, authorizer),
+		publicIPAddressesClient:         newPublicIPAddressesClientImpl(subscriptionID, authorizer),
 	}, nil
 }
 
@@ -293,6 +296,10 @@ func (c *azureCloudImplementation) NetworkSecurityGroup() NetworkSecurityGroupsC
 	return c.networkSecurityGroupsClient
 }
 
+func (c *azureCloudImplementation) ApplicationSecurityGroup() ApplicationSecurityGroupsClient {
+	return c.applicationSecurityGroupsClient
+}
+
 func (c *azureCloudImplementation) VMScaleSet() VMScaleSetsClient {
 	return c.vmscaleSetsClient
 }

upup/pkg/fi/cloudup/azure/azure_utils.go

@@ -100,6 +100,34 @@ func ParseNetworkSecurityGroupID(s string) (*NetworkSecurityGroupID, error) {
 	}, nil
 }
 
+// ApplicationSecurityGroupID contains the resource ID/names required to construct a ApplicationSecurityGroup ID.
+type ApplicationSecurityGroupID struct {
+	SubscriptionID               string
+	ResourceGroupName            string
+	ApplicationSecurityGroupName string
+}
+
+// String returns the ApplicationSecurityGroup ID in the path format.
+func (s *ApplicationSecurityGroupID) String() string {
+	return fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/applicationSecurityGroups/%s",
+		s.SubscriptionID,
+		s.ResourceGroupName,
+		s.ApplicationSecurityGroupName)
+}
+
+// ParseApplicationSecurityGroupID parses a given ApplicationSecurityGroup ID string and returns a ApplicationSecurityGroup ID.
+func ParseApplicationSecurityGroupID(s string) (*ApplicationSecurityGroupID, error) {
+	l := strings.Split(s, "/")
+	if len(l) != 9 {
+		return nil, fmt.Errorf("malformed format of ApplicationSecurityGroup ID: %s, %d", s, len(l))
+	}
+	return &ApplicationSecurityGroupID{
+		SubscriptionID:               l[2],
+		ResourceGroupName:            l[4],
+		ApplicationSecurityGroupName: l[8],
+	}, nil
+}
+
 // LoadBalancerID contains the resource ID/names required to construct a load balancer ID.
 type LoadBalancerID struct {
 	SubscriptionID    string

upup/pkg/fi/cloudup/azuretasks/applicationsecuritygroup.go

@@ -0,0 +1,135 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package azuretasks
+
+import (
+	"context"
+
+	"github.com/Azure/azure-sdk-for-go/services/network/mgmt/2022-05-01/network"
+	"github.com/Azure/go-autorest/autorest/to"
+	"k8s.io/klog/v2"
+	"k8s.io/kops/upup/pkg/fi"
+	"k8s.io/kops/upup/pkg/fi/cloudup/azure"
+)
+
+// ApplicationSecurityGroup is an Azure Cloud Application Security Group
+// +kops:fitask
+type ApplicationSecurityGroup struct {
+	Name          *string
+	ID            *string
+	Lifecycle     fi.Lifecycle
+	ResourceGroup *ResourceGroup
+
+	Tags map[string]*string
+}
+
+var (
+	_ fi.CloudupTask          = &ApplicationSecurityGroup{}
+	_ fi.CompareWithID        = &ApplicationSecurityGroup{}
+	_ fi.CloudupTaskNormalize = &ApplicationSecurityGroup{}
+)
+
+// CompareWithID returns the Name of the Application Security Group
+func (asg *ApplicationSecurityGroup) CompareWithID() *string {
+	return asg.ID
+}
+
+// Find discovers the Application Security Group in the cloud provider
+func (asg *ApplicationSecurityGroup) Find(c *fi.CloudupContext) (*ApplicationSecurityGroup, error) {
+	cloud := c.T.Cloud.(azure.AzureCloud)
+	l, err := cloud.ApplicationSecurityGroup().List(context.TODO(), *asg.ResourceGroup.Name)
+	if err != nil {
+		return nil, err
+	}
+	var found *network.ApplicationSecurityGroup
+	for _, v := range l {
+		if *v.Name == *asg.Name {
+			found = &v
+			break
+		}
+	}
+	if found == nil {
+		return nil, nil
+	}
+
+	asg.ID = found.ID
+
+	return &ApplicationSecurityGroup{
+		Name:      asg.Name,
+		Lifecycle: asg.Lifecycle,
+		ResourceGroup: &ResourceGroup{
+			Name: asg.ResourceGroup.Name,
+		},
+		ID:   found.ID,
+		Tags: found.Tags,
+	}, nil
+}
+
+func (asg *ApplicationSecurityGroup) Normalize(c *fi.CloudupContext) error {
+	c.T.Cloud.(azure.AzureCloud).AddClusterTags(asg.Tags)
+	return nil
+}
+
+// Run implements fi.Task.Run.
+func (asg *ApplicationSecurityGroup) Run(c *fi.CloudupContext) error {
+	return fi.CloudupDefaultDeltaRunMethod(asg, c)
+}
+
+// CheckChanges returns an error if a change is not allowed.
+func (*ApplicationSecurityGroup) CheckChanges(a, e, changes *ApplicationSecurityGroup) error {
+	if a == nil {
+		// Check if required fields are set when a new resource is created.
+		if e.Name == nil {
+			return fi.RequiredField("Name")
+		}
+		return nil
+	}
+
+	// Check if unchangeable fields won't be changed.
+	if changes.Name != nil {
+		return fi.CannotChangeField("Name")
+	}
+	return nil
+}
+
+// RenderAzure creates or updates a Application Security Group.
+func (*ApplicationSecurityGroup) RenderAzure(t *azure.AzureAPITarget, a, e, changes *ApplicationSecurityGroup) error {
+	if a == nil {
+		klog.Infof("Creating a new Application Security Group with name: %s", fi.ValueOf(e.Name))
+	} else {
+		klog.Infof("Updating a Application Security Group with name: %s", fi.ValueOf(e.Name))
+	}
+
+	p := network.ApplicationSecurityGroup{
+		Location: to.StringPtr(t.Cloud.Region()),
+		Name:     to.StringPtr(*e.Name),
+		Tags:     e.Tags,
+	}
+
+	asg, err := t.Cloud.ApplicationSecurityGroup().CreateOrUpdate(
+		context.TODO(),
+		*e.ResourceGroup.Name,
+		*e.Name,
+		p)
+	if err != nil {
+		return err
+	}
+
+	e.ID = asg.ID
+
+	return nil
+}

upup/pkg/fi/cloudup/azuretasks/applicationsecuritygroup_fitask.go

@@ -0,0 +1,52 @@
+//go:build !ignore_autogenerated
+// +build !ignore_autogenerated
+
+/*
+Copyright The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Code generated by fitask. DO NOT EDIT.
+
+package azuretasks
+
+import (
+	"k8s.io/kops/upup/pkg/fi"
+)
+
+// ApplicationSecurityGroup
+
+var _ fi.HasLifecycle = &ApplicationSecurityGroup{}
+
+// GetLifecycle returns the Lifecycle of the object, implementing fi.HasLifecycle
+func (o *ApplicationSecurityGroup) GetLifecycle() fi.Lifecycle {
+	return o.Lifecycle
+}
+
+// SetLifecycle sets the Lifecycle of the object, implementing fi.SetLifecycle
+func (o *ApplicationSecurityGroup) SetLifecycle(lifecycle fi.Lifecycle) {
+	o.Lifecycle = lifecycle
+}
+
+var _ fi.HasName = &ApplicationSecurityGroup{}
+
+// GetName returns the Name of the object, implementing fi.HasName
+func (o *ApplicationSecurityGroup) GetName() *string {
+	return o.Name
+}
+
+// String is the stringer function for the task, producing readable output using fi.TaskAsString
+func (o *ApplicationSecurityGroup) String() string {
+	return fi.CloudupTaskAsString(o)
+}

upup/pkg/fi/cloudup/azuretasks/networksecuritygroup.go

@@ -92,9 +92,35 @@ func (nsg *NetworkSecurityGroup) Find(c *fi.CloudupContext) (*NetworkSecurityGro
 		if rule.SourceAddressPrefixes != nil && len(*rule.SourceAddressPrefixes) > 0 {
 			nsr.SourceAddressPrefixes = rule.SourceAddressPrefixes
 		}
+		if rule.SourceApplicationSecurityGroups != nil && len(*rule.SourceApplicationSecurityGroups) > 0 {
+			var sasgs []string
+			for _, sasg := range *rule.SourceApplicationSecurityGroups {
+				asg, err := azure.ParseApplicationSecurityGroupID(*sasg.ID)
+				if err != nil {
+					if err != nil {
+						return nil, err
+					}
+				}
+				sasgs = append(sasgs, asg.ApplicationSecurityGroupName)
+			}
+			nsr.SourceApplicationSecurityGroupNames = &sasgs
+		}
 		if rule.DestinationAddressPrefixes != nil && len(*rule.DestinationAddressPrefixes) > 0 {
 			nsr.DestinationAddressPrefixes = rule.DestinationAddressPrefixes
 		}
+		if rule.DestinationApplicationSecurityGroups != nil && len(*rule.DestinationApplicationSecurityGroups) > 0 {
+			var dasgs []string
+			for _, dasg := range *rule.DestinationApplicationSecurityGroups {
+				asg, err := azure.ParseApplicationSecurityGroupID(*dasg.ID)
+				if err != nil {
+					if err != nil {
+						return nil, err
+					}
+				}
+				dasgs = append(dasgs, asg.ApplicationSecurityGroupName)
+			}
+			nsr.DestinationApplicationSecurityGroupNames = &dasgs
+		}
 		actual.SecurityRules = append(actual.SecurityRules, nsr)
 	}
 
@@ -162,6 +188,32 @@ func (*NetworkSecurityGroup) RenderAzure(t *azure.AzureAPITarget, a, e, changes
 				DestinationPortRange:       nsr.DestinationPortRange,
 			},
 		}
+		if nsr.SourceApplicationSecurityGroupNames != nil {
+			var sasgs []network.ApplicationSecurityGroup
+			for _, name := range *nsr.SourceApplicationSecurityGroupNames {
+				id := azure.ApplicationSecurityGroupID{
+					SubscriptionID:               t.Cloud.SubscriptionID(),
+					ResourceGroupName:            *e.ResourceGroup.Name,
+					ApplicationSecurityGroupName: name,
+				}
+				idStr := id.String()
+				sasgs = append(sasgs, network.ApplicationSecurityGroup{ID: &idStr})
+			}
+			securityRule.SourceApplicationSecurityGroups = &sasgs
+		}
+		if nsr.DestinationApplicationSecurityGroupNames != nil {
+			var dasgs []network.ApplicationSecurityGroup
+			for _, name := range *nsr.DestinationApplicationSecurityGroupNames {
+				id := azure.ApplicationSecurityGroupID{
+					SubscriptionID:               t.Cloud.SubscriptionID(),
+					ResourceGroupName:            *e.ResourceGroup.Name,
+					ApplicationSecurityGroupName: name,
+				}
+				idStr := id.String()
+				dasgs = append(dasgs, network.ApplicationSecurityGroup{ID: &idStr})
+			}
+			securityRule.DestinationApplicationSecurityGroups = &dasgs
+		}
 		*p.SecurityRules = append(*p.SecurityRules, securityRule)
 	}
 
@@ -181,17 +233,19 @@ func (*NetworkSecurityGroup) RenderAzure(t *azure.AzureAPITarget, a, e, changes
 
 // NetworkSecurityRule represents a NetworkSecurityGroup rule.
 type NetworkSecurityRule struct {
-	Name                       *string
+	Name                                     *string
-	Priority                   *int32
+	Priority                                 *int32
-	Access                     network.SecurityRuleAccess
+	Access                                   network.SecurityRuleAccess
-	Direction                  network.SecurityRuleDirection
+	Direction                                network.SecurityRuleDirection
-	Protocol                   network.SecurityRuleProtocol
+	Protocol                                 network.SecurityRuleProtocol
-	SourceAddressPrefix        *string
+	SourceAddressPrefix                      *string
-	SourceAddressPrefixes      *[]string
+	SourceAddressPrefixes                    *[]string
-	SourcePortRange            *string
+	SourceApplicationSecurityGroupNames      *[]string
-	DestinationAddressPrefixes *[]string
+	SourcePortRange                          *string
-	DestinationAddressPrefix   *string
+	DestinationAddressPrefixes               *[]string
-	DestinationPortRange       *string
+	DestinationAddressPrefix                 *string
+	DestinationApplicationSecurityGroupNames *[]string
+	DestinationPortRange                     *string
 }
 
 var _ fi.CloudupHasDependencies = &NetworkSecurityRule{}

upup/pkg/fi/cloudup/azuretasks/testing.go

@@ -46,19 +46,20 @@ const (
 
 // MockAzureCloud is a mock implementation of AzureCloud.
 type MockAzureCloud struct {
-	Location                    string
+	Location                        string
-	ResourceGroupsClient        *MockResourceGroupsClient
+	ResourceGroupsClient            *MockResourceGroupsClient
-	VirtualNetworksClient       *MockVirtualNetworksClient
+	VirtualNetworksClient           *MockVirtualNetworksClient
-	SubnetsClient               *MockSubnetsClient
+	SubnetsClient                   *MockSubnetsClient
-	RouteTablesClient           *MockRouteTablesClient
+	RouteTablesClient               *MockRouteTablesClient
-	NetworkSecurityGroupsClient *MockNetworkSecurityGroupsClient
+	NetworkSecurityGroupsClient     *MockNetworkSecurityGroupsClient
-	VMScaleSetsClient           *MockVMScaleSetsClient
+	ApplicationSecurityGroupsClient *MockApplicationSecurityGroupsClient
-	VMScaleSetVMsClient         *MockVMScaleSetVMsClient
+	VMScaleSetsClient               *MockVMScaleSetsClient
-	DisksClient                 *MockDisksClient
+	VMScaleSetVMsClient             *MockVMScaleSetVMsClient
-	RoleAssignmentsClient       *MockRoleAssignmentsClient
+	DisksClient                     *MockDisksClient
-	NetworkInterfacesClient     *MockNetworkInterfacesClient
+	RoleAssignmentsClient           *MockRoleAssignmentsClient
-	LoadBalancersClient         *MockLoadBalancersClient
+	NetworkInterfacesClient         *MockNetworkInterfacesClient
-	PublicIPAddressesClient     *MockPublicIPAddressesClient
+	LoadBalancersClient             *MockLoadBalancersClient
+	PublicIPAddressesClient         *MockPublicIPAddressesClient
 }
 
 var _ azure.AzureCloud = &MockAzureCloud{}
@@ -82,6 +83,9 @@ func NewMockAzureCloud(location string) *MockAzureCloud {
 		NetworkSecurityGroupsClient: &MockNetworkSecurityGroupsClient{
 			NSGs: map[string]network.SecurityGroup{},
 		},
+		ApplicationSecurityGroupsClient: &MockApplicationSecurityGroupsClient{
+			ASGs: map[string]network.ApplicationSecurityGroup{},
+		},
 		VMScaleSetsClient: &MockVMScaleSetsClient{
 			VMSSes: map[string]compute.VirtualMachineScaleSet{},
 		},
@@ -199,11 +203,16 @@ func (c *MockAzureCloud) RouteTable() azure.RouteTablesClient {
 	return c.RouteTablesClient
 }
 
-// NetworkSecurityGroup returns the VM Scale Set client.
+// NetworkSecurityGroup returns the Network Security Group client.
 func (c *MockAzureCloud) NetworkSecurityGroup() azure.NetworkSecurityGroupsClient {
 	return c.NetworkSecurityGroupsClient
 }
 
+// ApplicationSecurityGroup returns the Application Security Group client.
+func (c *MockAzureCloud) ApplicationSecurityGroup() azure.ApplicationSecurityGroupsClient {
+	return c.ApplicationSecurityGroupsClient
+}
+
 // VMScaleSet returns the VM Scale Set client.
 func (c *MockAzureCloud) VMScaleSet() azure.VMScaleSetsClient {
 	return c.VMScaleSetsClient
@@ -640,13 +649,13 @@ type MockNetworkSecurityGroupsClient struct {
 var _ azure.NetworkSecurityGroupsClient = &MockNetworkSecurityGroupsClient{}
 
 // CreateOrUpdate creates or updates a Network Security Group.
-func (c *MockNetworkSecurityGroupsClient) CreateOrUpdate(ctx context.Context, resourceGroupName, asgName string, parameters network.SecurityGroup) (*network.SecurityGroup, error) {
+func (c *MockNetworkSecurityGroupsClient) CreateOrUpdate(ctx context.Context, resourceGroupName, nsgName string, parameters network.SecurityGroup) (*network.SecurityGroup, error) {
 	// Ignore resourceGroupName for simplicity.
-	if _, ok := c.NSGs[asgName]; ok {
+	if _, ok := c.NSGs[nsgName]; ok {
 		return nil, fmt.Errorf("update not supported")
 	}
-	parameters.Name = &asgName
+	parameters.Name = &nsgName
-	c.NSGs[asgName] = parameters
+	c.NSGs[nsgName] = parameters
 	return &parameters, nil
 }
 
@@ -677,3 +686,49 @@ func (c *MockNetworkSecurityGroupsClient) Delete(ctx context.Context, resourceGr
 	delete(c.NSGs, nsgName)
 	return nil
 }
+
+// MockApplicationSecurityGroupsClient is a mock implementation of Application Security Group client.
+type MockApplicationSecurityGroupsClient struct {
+	ASGs map[string]network.ApplicationSecurityGroup
+}
+
+var _ azure.ApplicationSecurityGroupsClient = &MockApplicationSecurityGroupsClient{}
+
+// CreateOrUpdate creates or updates a Application Security Group.
+func (c *MockApplicationSecurityGroupsClient) CreateOrUpdate(ctx context.Context, resourceGroupName, asgName string, parameters network.ApplicationSecurityGroup) (*network.ApplicationSecurityGroup, error) {
+	// Ignore resourceGroupName for simplicity.
+	if _, ok := c.ASGs[asgName]; ok {
+		return nil, fmt.Errorf("update not supported")
+	}
+	parameters.Name = &asgName
+	c.ASGs[asgName] = parameters
+	return &parameters, nil
+}
+
+// List returns a slice of Application Security Groups.
+func (c *MockApplicationSecurityGroupsClient) List(ctx context.Context, resourceGroupName string) ([]network.ApplicationSecurityGroup, error) {
+	var l []network.ApplicationSecurityGroup
+	for _, nsg := range c.ASGs {
+		l = append(l, nsg)
+	}
+	return l, nil
+}
+
+// Get Returns a specified Application Security Group.
+func (c *MockApplicationSecurityGroupsClient) Get(ctx context.Context, resourceGroupName string, asgName string) (*network.ApplicationSecurityGroup, error) {
+	asg, ok := c.ASGs[asgName]
+	if !ok {
+		return nil, nil
+	}
+	return &asg, nil
+}
+
+// Delete deletes a specified Application Security Group.
+func (c *MockApplicationSecurityGroupsClient) Delete(ctx context.Context, resourceGroupName, asgName string) error {
+	// Ignore resourceGroupName for simplicity.
+	if _, ok := c.ASGs[asgName]; !ok {
+		return fmt.Errorf("%s does not exist", asgName)
+	}
+	delete(c.ASGs, asgName)
+	return nil
+}

upup/pkg/fi/cloudup/azuretasks/vmscaleset.go

@@ -35,10 +35,11 @@ type VMScaleSet struct {
 	Name      *string
 	Lifecycle fi.Lifecycle
 
-	ResourceGroup  *ResourceGroup
+	ResourceGroup             *ResourceGroup
-	VirtualNetwork *VirtualNetwork
+	VirtualNetwork            *VirtualNetwork
-	Subnet         *Subnet
+	Subnet                    *Subnet
-	StorageProfile *VMScaleSetStorageProfile
+	ApplicationSecurityGroups []*ApplicationSecurityGroup
+	StorageProfile            *VMScaleSetStorageProfile
 	// RequirePublicIP is set to true when VMs require public IPs.
 	RequirePublicIP *bool
 	// LoadBalancer is the Load Balancer object the VMs will use.
@@ -167,6 +168,13 @@ func (s *VMScaleSet) Find(c *fi.CloudupContext) (*VMScaleSet, error) {
 		Tags:               found.Tags,
 		PrincipalID:        found.Identity.PrincipalID,
 	}
+	if ipConfig.ApplicationSecurityGroups != nil {
+		for _, asg := range *ipConfig.ApplicationSecurityGroups {
+			vmss.ApplicationSecurityGroups = append(vmss.ApplicationSecurityGroups, &ApplicationSecurityGroup{
+				ID: asg.ID,
+			})
+		}
+	}
 	if loadBalancerID != nil {
 		vmss.LoadBalancer = &LoadBalancer{
 			Name: to.StringPtr(loadBalancerID.LoadBalancerName),
@@ -247,12 +255,19 @@ func (s *VMScaleSet) RenderAzure(t *azure.AzureAPITarget, a, e, changes *VMScale
 		VirtualNetworkName: *e.VirtualNetwork.Name,
 		SubnetName:         *e.Subnet.Name,
 	}
+	var asgs []compute.SubResource
+	for _, asg := range e.ApplicationSecurityGroups {
+		asgs = append(asgs, compute.SubResource{
+			ID: asg.ID,
+		})
+	}
 	ipConfigProperties := &compute.VirtualMachineScaleSetIPConfigurationProperties{
 		Subnet: &compute.APIEntityReference{
 			ID: to.StringPtr(subnetID.String()),
 		},
-		Primary:                 to.BoolPtr(true),
+		Primary:                   to.BoolPtr(true),
-		PrivateIPAddressVersion: compute.IPv4,
+		PrivateIPAddressVersion:   compute.IPv4,
+		ApplicationSecurityGroups: &asgs,
 	}
 	if *e.RequirePublicIP {
 		ipConfigProperties.PublicIPAddressConfiguration = &compute.VirtualMachineScaleSetPublicIPAddressConfiguration{