Merge pull request #5875 from seanson/5700_add_flag_for_no_subnet_tags

#5700: Add command line flag for disabling Subnet ELB tags

cmd/kops/create_cluster.go

@@ -80,6 +80,7 @@ type CreateClusterOptions struct {
 	VPCID                string
 	SubnetIDs            []string
 	UtilitySubnetIDs     []string
+	DisableSubnetTags    bool
 	NetworkCIDR          string
 	DNSZone              string
 	AdminAccess          []string
@@ -294,6 +295,7 @@ func NewCmdCreateCluster(f *util.Factory, out io.Writer) *cobra.Command {
 	cmd.Flags().StringSliceVar(&options.SubnetIDs, "subnets", options.SubnetIDs, "Set to use shared subnets")
 	cmd.Flags().StringSliceVar(&options.UtilitySubnetIDs, "utility-subnets", options.UtilitySubnetIDs, "Set to use shared utility subnets")
 	cmd.Flags().StringVar(&options.NetworkCIDR, "network-cidr", options.NetworkCIDR, "Set to override the default network CIDR")
+	cmd.Flags().BoolVar(&options.DisableSubnetTags, "disable-subnet-tags", options.DisableSubnetTags, "Set to disable automatic subnet tagging")
 
 	cmd.Flags().Int32Var(&options.MasterCount, "master-count", options.MasterCount, "Set the number of masters.  Defaults to one master per master-zone")
 	cmd.Flags().Int32Var(&options.NodeCount, "node-count", options.NodeCount, "Set the number of nodes")
@@ -946,6 +948,8 @@ func RunCreateCluster(f *util.Factory, out io.Writer, c *CreateClusterOptions) e
 		c.Topology = api.TopologyPublic
 	}
 
+	cluster.Spec.DisableSubnetTags = c.DisableSubnetTags
+
 	switch c.Topology {
 	case api.TopologyPublic:
 		cluster.Spec.Topology = &api.TopologySpec{

docs/cli/kops_create_cluster.md

@@ -73,6 +73,7 @@ kops create cluster [flags]
       --channel string                   Channel for default versions and configuration to use (default "stable")
       --cloud string                     Cloud provider to use - gce, aws, vsphere
       --cloud-labels string              A list of KV pairs used to tag all instance groups in AWS (eg "Owner=John Doe,Team=Some Team").
+      --disable-subnet-tags              Set to disable automatic subnet tagging
       --dns string                       DNS hosted zone to use: public|private. (default "Public")
       --dns-zone string                  DNS hosted zone to use (defaults to longest matching zone)
       --dry-run                          If true, only print the object that would be sent, without sending it. This flag can be used to create a cluster YAML or JSON manifest.

docs/run_in_existing_vpc.md

@@ -142,17 +142,11 @@ spec:
   kops update cluster ${CLUSTER_NAME} --yes
   ```
 
-  **If you run in AWS private topology with shared subnets, and you would like Kubernetes to provision resources in these shared subnets, you must create tags on them.**
+### Subnet Tags
 
-  **This is important, for example, if your `utility` subnets are shared, you will not be able to launch any services that create Elastic Load Balancers (ELBs).**
+  By default, kops will tag your existing subnets with the standard tags:
 
-  **Prior to kops 1.8 `KubernetesCluster` tag was used for this. This lead to several problems if there were more than one Kubernetes Cluster in a subnet.**
+  Public/Utility Subnets:
-  
-  **After you upgraded to kops 1.8 remove `KubernetesCluster` Tag from subnets otherwise `kubernetes.io/cluster/<clustername>` won't have any effect!**
-
-  **These are currently needed Tags on shared resources:**
-
-  Public Subnets:
   ```
   "kubernetes.io/cluster/<cluster-name>" = "shared"
   "kubernetes.io/role/elb"               = "1"
@@ -166,6 +160,11 @@ spec:
   "SubnetType"                           = "Private"
   ```
   
+  These tags are important, for example, your services will be unable to create public or private Elastic Load Balancers (ELBs) if the respective `elb` or `internal-elb` tags are missing.
+  
+  If you would like to manage these tags externally then specify `--disable-subnet-tags` during your cluster creation. This will prevent kops from tagging existing subnets and allow some custom control, such as separate subnets for internal ELBs.
+  
+  Prior to kops 1.8 `KubernetesCluster` tag was used instead of `kubernetes.io/cluster/<cluster-name>`. This lead to several problems if there were more than one Kubernetes Cluster in a subnet. After you upgraded to kops 1.8 ensure the `KubernetesCluster` Tag is removed from subnets otherwise `kubernetes.io/cluster/<clustername>` won't have any effect!
 
 ### Shared NAT Egress
 

pkg/apis/kops/cluster.go

@@ -164,6 +164,8 @@ type ClusterSpec struct {
 	IAM *IAMSpec `json:"iam,omitempty"`
 	// EncryptionConfig controls if encryption is enabled
 	EncryptionConfig *bool `json:"encryptionConfig,omitempty"`
+	// DisableSubnetTags controls if subnets are tagged in AWS
+	DisableSubnetTags bool `json:"disableSubnetTags,omitempty"`
 	// Target allows for us to nest extra config for targets such as terraform
 	Target *TargetSpec `json:"target,omitempty"`
 }

pkg/apis/kops/v1alpha1/cluster.go

@@ -163,6 +163,8 @@ type ClusterSpec struct {
 	IAM *IAMSpec `json:"iam,omitempty"`
 	// EncryptionConfig holds the encryption config
 	EncryptionConfig *bool `json:"encryptionConfig,omitempty"`
+	// DisableSubnetTags controls if subnets are tagged in AWS
+	DisableSubnetTags bool `json:"DisableSubnetTags,omitempty"`
 	// Target allows for us to nest extra config for targets such as terraform
 	Target *TargetSpec `json:"target,omitempty"`
 }

pkg/apis/kops/v1alpha1/zz_generated.conversion.go

@@ -1066,6 +1066,7 @@ func autoConvert_v1alpha1_ClusterSpec_To_kops_ClusterSpec(in *ClusterSpec, out *
 		out.IAM = nil
 	}
 	out.EncryptionConfig = in.EncryptionConfig
+	out.DisableSubnetTags = in.DisableSubnetTags
 	if in.Target != nil {
 		in, out := &in.Target, &out.Target
 		*out = new(kops.TargetSpec)
@@ -1332,6 +1333,7 @@ func autoConvert_kops_ClusterSpec_To_v1alpha1_ClusterSpec(in *kops.ClusterSpec,
 		out.IAM = nil
 	}
 	out.EncryptionConfig = in.EncryptionConfig
+	out.DisableSubnetTags = in.DisableSubnetTags
 	if in.Target != nil {
 		in, out := &in.Target, &out.Target
 		*out = new(TargetSpec)

pkg/apis/kops/v1alpha2/cluster.go

@@ -164,6 +164,8 @@ type ClusterSpec struct {
 	IAM *IAMSpec `json:"iam,omitempty"`
 	// EncryptionConfig holds the encryption config
 	EncryptionConfig *bool `json:"encryptionConfig,omitempty"`
+	// DisableSubnetTags controls if subnets are tagged in AWS
+	DisableSubnetTags bool `json:"DisableSubnetTags,omitempty"`
 	// Target allows for us to nest extra config for targets such as terraform
 	Target *TargetSpec `json:"target,omitempty"`
 }

pkg/apis/kops/v1alpha2/zz_generated.conversion.go

@@ -1113,6 +1113,7 @@ func autoConvert_v1alpha2_ClusterSpec_To_kops_ClusterSpec(in *ClusterSpec, out *
 		out.IAM = nil
 	}
 	out.EncryptionConfig = in.EncryptionConfig
+	out.DisableSubnetTags = in.DisableSubnetTags
 	if in.Target != nil {
 		in, out := &in.Target, &out.Target
 		*out = new(kops.TargetSpec)
@@ -1394,6 +1395,7 @@ func autoConvert_kops_ClusterSpec_To_v1alpha2_ClusterSpec(in *kops.ClusterSpec,
 		out.IAM = nil
 	}
 	out.EncryptionConfig = in.EncryptionConfig
+	out.DisableSubnetTags = in.DisableSubnetTags
 	if in.Target != nil {
 		in, out := &in.Target, &out.Target
 		*out = new(TargetSpec)

pkg/model/network.go

@@ -184,22 +184,28 @@ func (b *NetworkModelBuilder) Build(c *fi.ModelBuilderContext) error {
 		subnetSpec := &b.Cluster.Spec.Subnets[i]
 		sharedSubnet := subnetSpec.ProviderID != ""
 		subnetName := subnetSpec.Name + "." + b.ClusterName()
-		tags := b.CloudTags(subnetName, sharedSubnet)
+		tags := map[string]string{}
 
 		// Apply tags so that Kubernetes knows which subnets should be used for internal/external ELBs
-		switch subnetSpec.Type {
+		if b.Cluster.Spec.DisableSubnetTags {
-		case kops.SubnetTypePublic, kops.SubnetTypeUtility:
+			glog.V(2).Infof("skipping subnet tags. Ensure these are maintained externally.")
-			tags[aws.TagNameSubnetPublicELB] = "1"
+		} else {
+			glog.V(2).Infof("applying subnet tags")
+			tags = b.CloudTags(subnetName, sharedSubnet)
+			tags["SubnetType"] = string(subnetSpec.Type)
 
-		case kops.SubnetTypePrivate:
+			switch subnetSpec.Type {
-			tags[aws.TagNameSubnetInternalELB] = "1"
+			case kops.SubnetTypePublic, kops.SubnetTypeUtility:
+				tags[aws.TagNameSubnetPublicELB] = "1"
 
-		default:
+			case kops.SubnetTypePrivate:
-			glog.V(2).Infof("unable to properly tag subnet %q because it has unknown type %q. Load balancers may be created in incorrect subnets", subnetSpec.Name, subnetSpec.Type)
+				tags[aws.TagNameSubnetInternalELB] = "1"
+
+			default:
+				glog.V(2).Infof("unable to properly tag subnet %q because it has unknown type %q. Load balancers may be created in incorrect subnets", subnetSpec.Name, subnetSpec.Type)
+			}
 		}
 
-		tags["SubnetType"] = string(subnetSpec.Type)
-
 		subnet := &awstasks.Subnet{
 			Name:             s(subnetName),
 			ShortName:        s(subnetSpec.Name),