gce: Always create an internal load balancer

When we create a external load balancer on GCE, we now also create an
internal load balancer.  The internal load balancer is used for
node/pod -> control-plane traffic, the external load balancer is used
for other traffic (e.g. "user" traffic to kube-apiserver).

This means that we can apply more granular firewall rules, and
generally avoid complex logic around discovery of the internal control
plane addresses for GCE.

pkg/model/gcemodel/api_loadbalancer.go

@@ -105,7 +105,7 @@ func (b *APILoadBalancerBuilder) createPublicLB(c *fi.CloudupModelBuilderContext
 		})
 	}
 
-	return b.addFirewallRules(c)
+	return nil
 }
 
 func (b *APILoadBalancerBuilder) addFirewallRules(c *fi.CloudupModelBuilderContext) error {
@@ -248,7 +248,7 @@ func (b *APILoadBalancerBuilder) createInternalLB(c *fi.CloudupModelBuilderConte
 			})
 		}
 	}
-	return b.addFirewallRules(c)
+	return nil
 }
 
 func (b *APILoadBalancerBuilder) Build(c *fi.CloudupModelBuilderContext) error {
@@ -264,22 +264,25 @@ func (b *APILoadBalancerBuilder) Build(c *fi.CloudupModelBuilderContext) error {
 
 	switch lbSpec.Type {
 	case kops.LoadBalancerTypePublic:
-		return b.createPublicLB(c)
+		if err := b.createPublicLB(c); err != nil {
+			return err
+		}
+		// We always create the internal load balancer also;
+		// it allows us to restrict access to only the nodes.
+		if err := b.createInternalLB(c); err != nil {
+			return err
+		}
+
+		return b.addFirewallRules(c)
 
 	case kops.LoadBalancerTypeInternal:
-		return b.createInternalLB(c)
+		if err := b.createInternalLB(c); err != nil {
+			return err
+		}
+
+		return b.addFirewallRules(c)
 
 	default:
 		return fmt.Errorf("unhandled LoadBalancer type %q", lbSpec.Type)
 	}
 }
-
-// subnetNotSpecified returns true if the given LB subnet is not listed in the list of cluster subnets.
-func subnetNotSpecified(sn kops.LoadBalancerSubnetSpec, subnets []kops.ClusterSubnetSpec) bool {
-	for _, csn := range subnets {
-		if csn.Name == sn.Name || csn.ID == sn.Name {
-			return false
-		}
-	}
-	return true
-}

tests/integration/update_cluster/minimal_gce_plb/kubernetes.tf

@@ -182,6 +182,23 @@ resource "google_compute_address" "api-minimal-gce-plb-example-com" {
   name = "api-minimal-gce-plb-example-com"
 }
 
+resource "google_compute_address" "api-us-test1-minimal-gce-plb-example-com" {
+  address_type = "INTERNAL"
+  name         = "api-us-test1-minimal-gce-plb-example-com"
+  purpose      = "SHARED_LOADBALANCER_VIP"
+  subnetwork   = google_compute_subnetwork.us-test1-minimal-gce-plb-example-com.name
+}
+
+resource "google_compute_backend_service" "api-minimal-gce-plb-example-com" {
+  backend {
+    group = google_compute_instance_group_manager.a-master-us-test1-a-minimal-gce-plb-example-com.instance_group
+  }
+  health_checks         = [google_compute_health_check.api-minimal-gce-plb-example-com.id]
+  load_balancing_scheme = "INTERNAL_SELF_MANAGED"
+  name                  = "api-minimal-gce-plb-example-com"
+  protocol              = "TCP"
+}
+
 resource "google_compute_disk" "a-etcd-events-minimal-gce-plb-example-com" {
   labels = {
     "k8s-io-cluster-name" = "minimal-gce-plb-example-com"
@@ -432,6 +449,28 @@ resource "google_compute_forwarding_rule" "api-minimal-gce-plb-example-com" {
   target                = google_compute_target_pool.api-minimal-gce-plb-example-com.self_link
 }
 
+resource "google_compute_forwarding_rule" "api-us-test1-minimal-gce-plb-example-com" {
+  backend_service = google_compute_backend_service.api-minimal-gce-plb-example-com.id
+  ip_address      = google_compute_address.api-us-test1-minimal-gce-plb-example-com.address
+  ip_protocol     = "TCP"
+  labels = {
+    "k8s-io-cluster-name" = "minimal-gce-plb-example-com"
+    "name"                = "api-us-test1"
+  }
+  load_balancing_scheme = "INTERNAL"
+  name                  = "api-us-test1-minimal-gce-plb-example-com"
+  network               = google_compute_network.minimal-gce-plb-example-com.name
+  ports                 = ["443"]
+  subnetwork            = google_compute_subnetwork.us-test1-minimal-gce-plb-example-com.name
+}
+
+resource "google_compute_health_check" "api-minimal-gce-plb-example-com" {
+  name = "api-minimal-gce-plb-example-com"
+  tcp_health_check {
+    port = 443
+  }
+}
+
 resource "google_compute_http_health_check" "api-minimal-gce-plb-example-com" {
   name         = "api-minimal-gce-plb-example-com"
   port         = 3990