diff --git a/cmd/kops/create_cluster.go b/cmd/kops/create_cluster.go index 66184b1ac2..85cc837cce 100644 --- a/cmd/kops/create_cluster.go +++ b/cmd/kops/create_cluster.go @@ -80,6 +80,7 @@ type CreateClusterOptions struct { VPCID string SubnetIDs []string UtilitySubnetIDs []string + DisableSubnetTags bool NetworkCIDR string DNSZone string AdminAccess []string @@ -294,6 +295,7 @@ func NewCmdCreateCluster(f *util.Factory, out io.Writer) *cobra.Command { cmd.Flags().StringSliceVar(&options.SubnetIDs, "subnets", options.SubnetIDs, "Set to use shared subnets") cmd.Flags().StringSliceVar(&options.UtilitySubnetIDs, "utility-subnets", options.UtilitySubnetIDs, "Set to use shared utility subnets") cmd.Flags().StringVar(&options.NetworkCIDR, "network-cidr", options.NetworkCIDR, "Set to override the default network CIDR") + cmd.Flags().BoolVar(&options.DisableSubnetTags, "disable-subnet-tags", options.DisableSubnetTags, "Set to disable automatic subnet tagging") cmd.Flags().Int32Var(&options.MasterCount, "master-count", options.MasterCount, "Set the number of masters. Defaults to one master per master-zone") cmd.Flags().Int32Var(&options.NodeCount, "node-count", options.NodeCount, "Set the number of nodes") @@ -946,6 +948,8 @@ func RunCreateCluster(f *util.Factory, out io.Writer, c *CreateClusterOptions) e c.Topology = api.TopologyPublic } + cluster.Spec.DisableSubnetTags = c.DisableSubnetTags + switch c.Topology { case api.TopologyPublic: cluster.Spec.Topology = &api.TopologySpec{ diff --git a/docs/cli/kops_create_cluster.md b/docs/cli/kops_create_cluster.md index d93793a577..48d067bd2c 100644 --- a/docs/cli/kops_create_cluster.md +++ b/docs/cli/kops_create_cluster.md @@ -73,6 +73,7 @@ kops create cluster [flags] --channel string Channel for default versions and configuration to use (default "stable") --cloud string Cloud provider to use - gce, aws, vsphere --cloud-labels string A list of KV pairs used to tag all instance groups in AWS (eg "Owner=John Doe,Team=Some Team"). + --disable-subnet-tags Set to disable automatic subnet tagging --dns string DNS hosted zone to use: public|private. (default "Public") --dns-zone string DNS hosted zone to use (defaults to longest matching zone) --dry-run If true, only print the object that would be sent, without sending it. This flag can be used to create a cluster YAML or JSON manifest. diff --git a/docs/run_in_existing_vpc.md b/docs/run_in_existing_vpc.md index 4b2dcac392..a4daacdc4c 100644 --- a/docs/run_in_existing_vpc.md +++ b/docs/run_in_existing_vpc.md @@ -142,17 +142,11 @@ spec: kops update cluster ${CLUSTER_NAME} --yes ``` - **If you run in AWS private topology with shared subnets, and you would like Kubernetes to provision resources in these shared subnets, you must create tags on them.** - - **This is important, for example, if your `utility` subnets are shared, you will not be able to launch any services that create Elastic Load Balancers (ELBs).** - - **Prior to kops 1.8 `KubernetesCluster` tag was used for this. This lead to several problems if there were more than one Kubernetes Cluster in a subnet.** - - **After you upgraded to kops 1.8 remove `KubernetesCluster` Tag from subnets otherwise `kubernetes.io/cluster/` won't have any effect!** +### Subnet Tags - **These are currently needed Tags on shared resources:** + By default, kops will tag your existing subnets with the standard tags: - Public Subnets: + Public/Utility Subnets: ``` "kubernetes.io/cluster/" = "shared" "kubernetes.io/role/elb" = "1" @@ -165,7 +159,12 @@ spec: "kubernetes.io/role/internal-elb" = "1" "SubnetType" = "Private" ``` - + + These tags are important, for example, your services will be unable to create public or private Elastic Load Balancers (ELBs) if the respective `elb` or `internal-elb` tags are missing. + + If you would like to manage these tags externally then specify `--disable-subnet-tags` during your cluster creation. This will prevent kops from tagging existing subnets and allow some custom control, such as separate subnets for internal ELBs. + + Prior to kops 1.8 `KubernetesCluster` tag was used instead of `kubernetes.io/cluster/`. This lead to several problems if there were more than one Kubernetes Cluster in a subnet. After you upgraded to kops 1.8 ensure the `KubernetesCluster` Tag is removed from subnets otherwise `kubernetes.io/cluster/` won't have any effect! ### Shared NAT Egress diff --git a/pkg/apis/kops/cluster.go b/pkg/apis/kops/cluster.go index b4e8b5e5b3..f53cb4f81a 100644 --- a/pkg/apis/kops/cluster.go +++ b/pkg/apis/kops/cluster.go @@ -164,6 +164,8 @@ type ClusterSpec struct { IAM *IAMSpec `json:"iam,omitempty"` // EncryptionConfig controls if encryption is enabled EncryptionConfig *bool `json:"encryptionConfig,omitempty"` + // DisableSubnetTags controls if subnets are tagged in AWS + DisableSubnetTags bool `json:"disableSubnetTags,omitempty"` // Target allows for us to nest extra config for targets such as terraform Target *TargetSpec `json:"target,omitempty"` } diff --git a/pkg/apis/kops/v1alpha1/cluster.go b/pkg/apis/kops/v1alpha1/cluster.go index 213ddf7346..1100c7a78c 100644 --- a/pkg/apis/kops/v1alpha1/cluster.go +++ b/pkg/apis/kops/v1alpha1/cluster.go @@ -163,6 +163,8 @@ type ClusterSpec struct { IAM *IAMSpec `json:"iam,omitempty"` // EncryptionConfig holds the encryption config EncryptionConfig *bool `json:"encryptionConfig,omitempty"` + // DisableSubnetTags controls if subnets are tagged in AWS + DisableSubnetTags bool `json:"DisableSubnetTags,omitempty"` // Target allows for us to nest extra config for targets such as terraform Target *TargetSpec `json:"target,omitempty"` } diff --git a/pkg/apis/kops/v1alpha1/zz_generated.conversion.go b/pkg/apis/kops/v1alpha1/zz_generated.conversion.go index 97f005eada..c8ebe6a240 100644 --- a/pkg/apis/kops/v1alpha1/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha1/zz_generated.conversion.go @@ -1066,6 +1066,7 @@ func autoConvert_v1alpha1_ClusterSpec_To_kops_ClusterSpec(in *ClusterSpec, out * out.IAM = nil } out.EncryptionConfig = in.EncryptionConfig + out.DisableSubnetTags = in.DisableSubnetTags if in.Target != nil { in, out := &in.Target, &out.Target *out = new(kops.TargetSpec) @@ -1332,6 +1333,7 @@ func autoConvert_kops_ClusterSpec_To_v1alpha1_ClusterSpec(in *kops.ClusterSpec, out.IAM = nil } out.EncryptionConfig = in.EncryptionConfig + out.DisableSubnetTags = in.DisableSubnetTags if in.Target != nil { in, out := &in.Target, &out.Target *out = new(TargetSpec) diff --git a/pkg/apis/kops/v1alpha2/cluster.go b/pkg/apis/kops/v1alpha2/cluster.go index 16ba5e3177..f32f5553f4 100644 --- a/pkg/apis/kops/v1alpha2/cluster.go +++ b/pkg/apis/kops/v1alpha2/cluster.go @@ -164,6 +164,8 @@ type ClusterSpec struct { IAM *IAMSpec `json:"iam,omitempty"` // EncryptionConfig holds the encryption config EncryptionConfig *bool `json:"encryptionConfig,omitempty"` + // DisableSubnetTags controls if subnets are tagged in AWS + DisableSubnetTags bool `json:"DisableSubnetTags,omitempty"` // Target allows for us to nest extra config for targets such as terraform Target *TargetSpec `json:"target,omitempty"` } diff --git a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go index 1afc069620..64bff20527 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go @@ -1113,6 +1113,7 @@ func autoConvert_v1alpha2_ClusterSpec_To_kops_ClusterSpec(in *ClusterSpec, out * out.IAM = nil } out.EncryptionConfig = in.EncryptionConfig + out.DisableSubnetTags = in.DisableSubnetTags if in.Target != nil { in, out := &in.Target, &out.Target *out = new(kops.TargetSpec) @@ -1394,6 +1395,7 @@ func autoConvert_kops_ClusterSpec_To_v1alpha2_ClusterSpec(in *kops.ClusterSpec, out.IAM = nil } out.EncryptionConfig = in.EncryptionConfig + out.DisableSubnetTags = in.DisableSubnetTags if in.Target != nil { in, out := &in.Target, &out.Target *out = new(TargetSpec) diff --git a/pkg/model/network.go b/pkg/model/network.go index 4acdfe6a10..b605fa63a7 100644 --- a/pkg/model/network.go +++ b/pkg/model/network.go @@ -184,22 +184,28 @@ func (b *NetworkModelBuilder) Build(c *fi.ModelBuilderContext) error { subnetSpec := &b.Cluster.Spec.Subnets[i] sharedSubnet := subnetSpec.ProviderID != "" subnetName := subnetSpec.Name + "." + b.ClusterName() - tags := b.CloudTags(subnetName, sharedSubnet) + tags := map[string]string{} // Apply tags so that Kubernetes knows which subnets should be used for internal/external ELBs - switch subnetSpec.Type { - case kops.SubnetTypePublic, kops.SubnetTypeUtility: - tags[aws.TagNameSubnetPublicELB] = "1" + if b.Cluster.Spec.DisableSubnetTags { + glog.V(2).Infof("skipping subnet tags. Ensure these are maintained externally.") + } else { + glog.V(2).Infof("applying subnet tags") + tags = b.CloudTags(subnetName, sharedSubnet) + tags["SubnetType"] = string(subnetSpec.Type) - case kops.SubnetTypePrivate: - tags[aws.TagNameSubnetInternalELB] = "1" + switch subnetSpec.Type { + case kops.SubnetTypePublic, kops.SubnetTypeUtility: + tags[aws.TagNameSubnetPublicELB] = "1" - default: - glog.V(2).Infof("unable to properly tag subnet %q because it has unknown type %q. Load balancers may be created in incorrect subnets", subnetSpec.Name, subnetSpec.Type) + case kops.SubnetTypePrivate: + tags[aws.TagNameSubnetInternalELB] = "1" + + default: + glog.V(2).Infof("unable to properly tag subnet %q because it has unknown type %q. Load balancers may be created in incorrect subnets", subnetSpec.Name, subnetSpec.Type) + } } - tags["SubnetType"] = string(subnetSpec.Type) - subnet := &awstasks.Subnet{ Name: s(subnetName), ShortName: s(subnetSpec.Name),