From 737a7a2cb81b70b558095ba1261a0898cc2bd168 Mon Sep 17 00:00:00 2001 From: Sean Johnson Date: Thu, 4 Oct 2018 14:25:34 +1000 Subject: [PATCH] 5700: Add command line flag for disabling Subnet ELB tags --- cmd/kops/create_cluster.go | 4 +++ docs/cli/kops_create_cluster.md | 1 + docs/run_in_existing_vpc.md | 19 +++++++------- pkg/apis/kops/cluster.go | 2 ++ pkg/apis/kops/v1alpha1/cluster.go | 2 ++ .../kops/v1alpha1/zz_generated.conversion.go | 2 ++ pkg/apis/kops/v1alpha2/cluster.go | 2 ++ .../kops/v1alpha2/zz_generated.conversion.go | 2 ++ pkg/model/network.go | 26 ++++++++++++------- 9 files changed, 40 insertions(+), 20 deletions(-) diff --git a/cmd/kops/create_cluster.go b/cmd/kops/create_cluster.go index dacbf20b39..7af9da5ba1 100644 --- a/cmd/kops/create_cluster.go +++ b/cmd/kops/create_cluster.go @@ -80,6 +80,7 @@ type CreateClusterOptions struct { VPCID string SubnetIDs []string UtilitySubnetIDs []string + DisableSubnetTags bool NetworkCIDR string DNSZone string AdminAccess []string @@ -290,6 +291,7 @@ func NewCmdCreateCluster(f *util.Factory, out io.Writer) *cobra.Command { cmd.Flags().StringSliceVar(&options.SubnetIDs, "subnets", options.SubnetIDs, "Set to use shared subnets") cmd.Flags().StringSliceVar(&options.UtilitySubnetIDs, "utility-subnets", options.UtilitySubnetIDs, "Set to use shared utility subnets") cmd.Flags().StringVar(&options.NetworkCIDR, "network-cidr", options.NetworkCIDR, "Set to override the default network CIDR") + cmd.Flags().BoolVar(&options.DisableSubnetTags, "disable-subnet-tags", options.DisableSubnetTags, "Set to disable automatic subnet tagging") cmd.Flags().Int32Var(&options.MasterCount, "master-count", options.MasterCount, "Set the number of masters. Defaults to one master per master-zone") cmd.Flags().Int32Var(&options.NodeCount, "node-count", options.NodeCount, "Set the number of nodes") @@ -923,6 +925,8 @@ func RunCreateCluster(f *util.Factory, out io.Writer, c *CreateClusterOptions) e c.Topology = api.TopologyPublic } + cluster.Spec.DisableSubnetTags = c.DisableSubnetTags + switch c.Topology { case api.TopologyPublic: cluster.Spec.Topology = &api.TopologySpec{ diff --git a/docs/cli/kops_create_cluster.md b/docs/cli/kops_create_cluster.md index d93793a577..48d067bd2c 100644 --- a/docs/cli/kops_create_cluster.md +++ b/docs/cli/kops_create_cluster.md @@ -73,6 +73,7 @@ kops create cluster [flags] --channel string Channel for default versions and configuration to use (default "stable") --cloud string Cloud provider to use - gce, aws, vsphere --cloud-labels string A list of KV pairs used to tag all instance groups in AWS (eg "Owner=John Doe,Team=Some Team"). + --disable-subnet-tags Set to disable automatic subnet tagging --dns string DNS hosted zone to use: public|private. (default "Public") --dns-zone string DNS hosted zone to use (defaults to longest matching zone) --dry-run If true, only print the object that would be sent, without sending it. This flag can be used to create a cluster YAML or JSON manifest. diff --git a/docs/run_in_existing_vpc.md b/docs/run_in_existing_vpc.md index 4b2dcac392..a4daacdc4c 100644 --- a/docs/run_in_existing_vpc.md +++ b/docs/run_in_existing_vpc.md @@ -142,17 +142,11 @@ spec: kops update cluster ${CLUSTER_NAME} --yes ``` - **If you run in AWS private topology with shared subnets, and you would like Kubernetes to provision resources in these shared subnets, you must create tags on them.** - - **This is important, for example, if your `utility` subnets are shared, you will not be able to launch any services that create Elastic Load Balancers (ELBs).** - - **Prior to kops 1.8 `KubernetesCluster` tag was used for this. This lead to several problems if there were more than one Kubernetes Cluster in a subnet.** - - **After you upgraded to kops 1.8 remove `KubernetesCluster` Tag from subnets otherwise `kubernetes.io/cluster/` won't have any effect!** +### Subnet Tags - **These are currently needed Tags on shared resources:** + By default, kops will tag your existing subnets with the standard tags: - Public Subnets: + Public/Utility Subnets: ``` "kubernetes.io/cluster/" = "shared" "kubernetes.io/role/elb" = "1" @@ -165,7 +159,12 @@ spec: "kubernetes.io/role/internal-elb" = "1" "SubnetType" = "Private" ``` - + + These tags are important, for example, your services will be unable to create public or private Elastic Load Balancers (ELBs) if the respective `elb` or `internal-elb` tags are missing. + + If you would like to manage these tags externally then specify `--disable-subnet-tags` during your cluster creation. This will prevent kops from tagging existing subnets and allow some custom control, such as separate subnets for internal ELBs. + + Prior to kops 1.8 `KubernetesCluster` tag was used instead of `kubernetes.io/cluster/`. This lead to several problems if there were more than one Kubernetes Cluster in a subnet. After you upgraded to kops 1.8 ensure the `KubernetesCluster` Tag is removed from subnets otherwise `kubernetes.io/cluster/` won't have any effect! ### Shared NAT Egress diff --git a/pkg/apis/kops/cluster.go b/pkg/apis/kops/cluster.go index 53e6334172..1ebef7e7c3 100644 --- a/pkg/apis/kops/cluster.go +++ b/pkg/apis/kops/cluster.go @@ -164,6 +164,8 @@ type ClusterSpec struct { IAM *IAMSpec `json:"iam,omitempty"` // EncryptionConfig controls if encryption is enabled EncryptionConfig *bool `json:"encryptionConfig,omitempty"` + // DisableSubnetTags controls if subnets are tagged in AWS + DisableSubnetTags bool `json:"disableSubnetTags,omitempty"` // Target allows for us to nest extra config for targets such as terraform Target *TargetSpec `json:"target,omitempty"` } diff --git a/pkg/apis/kops/v1alpha1/cluster.go b/pkg/apis/kops/v1alpha1/cluster.go index a8914c0d81..adabaa6eac 100644 --- a/pkg/apis/kops/v1alpha1/cluster.go +++ b/pkg/apis/kops/v1alpha1/cluster.go @@ -163,6 +163,8 @@ type ClusterSpec struct { IAM *IAMSpec `json:"iam,omitempty"` // EncryptionConfig holds the encryption config EncryptionConfig *bool `json:"encryptionConfig,omitempty"` + // DisableSubnetTags controls if subnets are tagged in AWS + DisableSubnetTags bool `json:"DisableSubnetTags,omitempty"` // Target allows for us to nest extra config for targets such as terraform Target *TargetSpec `json:"target,omitempty"` } diff --git a/pkg/apis/kops/v1alpha1/zz_generated.conversion.go b/pkg/apis/kops/v1alpha1/zz_generated.conversion.go index 342c17a3cd..7204fb4b7c 100644 --- a/pkg/apis/kops/v1alpha1/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha1/zz_generated.conversion.go @@ -1062,6 +1062,7 @@ func autoConvert_v1alpha1_ClusterSpec_To_kops_ClusterSpec(in *ClusterSpec, out * out.IAM = nil } out.EncryptionConfig = in.EncryptionConfig + out.DisableSubnetTags = in.DisableSubnetTags if in.Target != nil { in, out := &in.Target, &out.Target *out = new(kops.TargetSpec) @@ -1328,6 +1329,7 @@ func autoConvert_kops_ClusterSpec_To_v1alpha1_ClusterSpec(in *kops.ClusterSpec, out.IAM = nil } out.EncryptionConfig = in.EncryptionConfig + out.DisableSubnetTags = in.DisableSubnetTags if in.Target != nil { in, out := &in.Target, &out.Target *out = new(TargetSpec) diff --git a/pkg/apis/kops/v1alpha2/cluster.go b/pkg/apis/kops/v1alpha2/cluster.go index 3bf6d9d537..e185e8bc2f 100644 --- a/pkg/apis/kops/v1alpha2/cluster.go +++ b/pkg/apis/kops/v1alpha2/cluster.go @@ -164,6 +164,8 @@ type ClusterSpec struct { IAM *IAMSpec `json:"iam,omitempty"` // EncryptionConfig holds the encryption config EncryptionConfig *bool `json:"encryptionConfig,omitempty"` + // DisableSubnetTags controls if subnets are tagged in AWS + DisableSubnetTags bool `json:"DisableSubnetTags,omitempty"` // Target allows for us to nest extra config for targets such as terraform Target *TargetSpec `json:"target,omitempty"` } diff --git a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go index 7bcd1d1504..5bf488587b 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go @@ -1109,6 +1109,7 @@ func autoConvert_v1alpha2_ClusterSpec_To_kops_ClusterSpec(in *ClusterSpec, out * out.IAM = nil } out.EncryptionConfig = in.EncryptionConfig + out.DisableSubnetTags = in.DisableSubnetTags if in.Target != nil { in, out := &in.Target, &out.Target *out = new(kops.TargetSpec) @@ -1390,6 +1391,7 @@ func autoConvert_kops_ClusterSpec_To_v1alpha2_ClusterSpec(in *kops.ClusterSpec, out.IAM = nil } out.EncryptionConfig = in.EncryptionConfig + out.DisableSubnetTags = in.DisableSubnetTags if in.Target != nil { in, out := &in.Target, &out.Target *out = new(TargetSpec) diff --git a/pkg/model/network.go b/pkg/model/network.go index 4acdfe6a10..b605fa63a7 100644 --- a/pkg/model/network.go +++ b/pkg/model/network.go @@ -184,22 +184,28 @@ func (b *NetworkModelBuilder) Build(c *fi.ModelBuilderContext) error { subnetSpec := &b.Cluster.Spec.Subnets[i] sharedSubnet := subnetSpec.ProviderID != "" subnetName := subnetSpec.Name + "." + b.ClusterName() - tags := b.CloudTags(subnetName, sharedSubnet) + tags := map[string]string{} // Apply tags so that Kubernetes knows which subnets should be used for internal/external ELBs - switch subnetSpec.Type { - case kops.SubnetTypePublic, kops.SubnetTypeUtility: - tags[aws.TagNameSubnetPublicELB] = "1" + if b.Cluster.Spec.DisableSubnetTags { + glog.V(2).Infof("skipping subnet tags. Ensure these are maintained externally.") + } else { + glog.V(2).Infof("applying subnet tags") + tags = b.CloudTags(subnetName, sharedSubnet) + tags["SubnetType"] = string(subnetSpec.Type) - case kops.SubnetTypePrivate: - tags[aws.TagNameSubnetInternalELB] = "1" + switch subnetSpec.Type { + case kops.SubnetTypePublic, kops.SubnetTypeUtility: + tags[aws.TagNameSubnetPublicELB] = "1" - default: - glog.V(2).Infof("unable to properly tag subnet %q because it has unknown type %q. Load balancers may be created in incorrect subnets", subnetSpec.Name, subnetSpec.Type) + case kops.SubnetTypePrivate: + tags[aws.TagNameSubnetInternalELB] = "1" + + default: + glog.V(2).Infof("unable to properly tag subnet %q because it has unknown type %q. Load balancers may be created in incorrect subnets", subnetSpec.Name, subnetSpec.Type) + } } - tags["SubnetType"] = string(subnetSpec.Type) - subnet := &awstasks.Subnet{ Name: s(subnetName), ShortName: s(subnetSpec.Name),