From 1dbc6064a5a60c7ccaaa6cd7877eb7254e859eba Mon Sep 17 00:00:00 2001 From: Abdullah Almariah Date: Mon, 11 Dec 2017 15:24:08 +0100 Subject: [PATCH 01/11] Allow additional SGs to be added to API loadbalancer --- cmd/kops/create_cluster.go | 11 +++++++++++ pkg/apis/kops/cluster.go | 5 +++-- pkg/apis/kops/v1alpha1/cluster.go | 5 +++-- pkg/apis/kops/v1alpha1/zz_generated.conversion.go | 2 ++ pkg/apis/kops/v1alpha1/zz_generated.deepcopy.go | 5 +++++ pkg/apis/kops/v1alpha2/cluster.go | 5 +++-- pkg/apis/kops/v1alpha2/zz_generated.conversion.go | 2 ++ pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go | 5 +++++ pkg/apis/kops/zz_generated.deepcopy.go | 5 +++++ pkg/model/awsmodel/api_loadbalancer.go | 15 +++++++++++++++ 10 files changed, 54 insertions(+), 6 deletions(-) diff --git a/cmd/kops/create_cluster.go b/cmd/kops/create_cluster.go index 929871d582..37123e607d 100644 --- a/cmd/kops/create_cluster.go +++ b/cmd/kops/create_cluster.go @@ -113,6 +113,9 @@ type CreateClusterOptions struct { // Specify API loadbalancer as public or internal APILoadBalancerType string + // Add precreated additional security groups to the ELB + APILoadBalancerSecurityGroups []string + // Allow custom public master name MasterPublicName string @@ -311,6 +314,8 @@ func NewCmdCreateCluster(f *util.Factory, out io.Writer) *cobra.Command { cmd.Flags().StringVar(&options.APILoadBalancerType, "api-loadbalancer-type", options.APILoadBalancerType, "Sets the API loadbalancer type to either 'public' or 'internal'") + cmd.Flags().StringSliceVar(&options.APILoadBalancerSecurityGroups, "api-loadbalancer-security-groups", options.APILoadBalancerSecurityGroups, "Add precreated additional security groups to the API loadbalancer.") + // Allow custom public master name cmd.Flags().StringVar(&options.MasterPublicName, "master-public-name", options.MasterPublicName, "Sets the public master public name") @@ -942,6 +947,9 @@ func RunCreateCluster(f *util.Factory, out io.Writer, c *CreateClusterOptions) e } } } + if len(c.APILoadBalancerSecurityGroups) > 0 && cluster.Spec.API.LoadBalancer == nil { + return fmt.Errorf("unable to add security groups to nonexistent API loadbalancer") + } if cluster.Spec.API.LoadBalancer != nil && cluster.Spec.API.LoadBalancer.Type == "" { switch c.APILoadBalancerType { case "", "public": @@ -951,6 +959,9 @@ func RunCreateCluster(f *util.Factory, out io.Writer, c *CreateClusterOptions) e default: return fmt.Errorf("unknown api-loadbalancer-type: %q", c.APILoadBalancerType) } + if len(c.APILoadBalancerSecurityGroups) > 0 { + cluster.Spec.API.LoadBalancer.AdditionalSecurityGroups = c.APILoadBalancerSecurityGroups + } } // Use Strict IAM policy and allow AWS ECR by default when creating a new cluster diff --git a/pkg/apis/kops/cluster.go b/pkg/apis/kops/cluster.go index 81686cb9e3..2546706198 100644 --- a/pkg/apis/kops/cluster.go +++ b/pkg/apis/kops/cluster.go @@ -264,8 +264,9 @@ const ( ) type LoadBalancerAccessSpec struct { - Type LoadBalancerType `json:"type,omitempty"` - IdleTimeoutSeconds *int64 `json:"idleTimeoutSeconds,omitempty"` + Type LoadBalancerType `json:"type,omitempty"` + IdleTimeoutSeconds *int64 `json:"idleTimeoutSeconds,omitempty"` + AdditionalSecurityGroups []string `json:"additionalSecurityGroups,omitempty"` } // KubeDNSConfig defines the kube dns configuration diff --git a/pkg/apis/kops/v1alpha1/cluster.go b/pkg/apis/kops/v1alpha1/cluster.go index f9dd195a83..e2b1d6d1c3 100644 --- a/pkg/apis/kops/v1alpha1/cluster.go +++ b/pkg/apis/kops/v1alpha1/cluster.go @@ -263,8 +263,9 @@ const ( ) type LoadBalancerAccessSpec struct { - Type LoadBalancerType `json:"type,omitempty"` - IdleTimeoutSeconds *int64 `json:"idleTimeoutSeconds,omitempty"` + Type LoadBalancerType `json:"type,omitempty"` + IdleTimeoutSeconds *int64 `json:"idleTimeoutSeconds,omitempty"` + AdditionalSecurityGroups []string `json:"additionalSecurityGroups,omitempty"` } // KubeDNSConfig defines the kube dns configuration diff --git a/pkg/apis/kops/v1alpha1/zz_generated.conversion.go b/pkg/apis/kops/v1alpha1/zz_generated.conversion.go index b35d889184..3b7fcbe24c 100644 --- a/pkg/apis/kops/v1alpha1/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha1/zz_generated.conversion.go @@ -2281,6 +2281,7 @@ func Convert_kops_LeaderElectionConfiguration_To_v1alpha1_LeaderElectionConfigur func autoConvert_v1alpha1_LoadBalancerAccessSpec_To_kops_LoadBalancerAccessSpec(in *LoadBalancerAccessSpec, out *kops.LoadBalancerAccessSpec, s conversion.Scope) error { out.Type = kops.LoadBalancerType(in.Type) out.IdleTimeoutSeconds = in.IdleTimeoutSeconds + out.AdditionalSecurityGroups = in.AdditionalSecurityGroups return nil } @@ -2292,6 +2293,7 @@ func Convert_v1alpha1_LoadBalancerAccessSpec_To_kops_LoadBalancerAccessSpec(in * func autoConvert_kops_LoadBalancerAccessSpec_To_v1alpha1_LoadBalancerAccessSpec(in *kops.LoadBalancerAccessSpec, out *LoadBalancerAccessSpec, s conversion.Scope) error { out.Type = LoadBalancerType(in.Type) out.IdleTimeoutSeconds = in.IdleTimeoutSeconds + out.AdditionalSecurityGroups = in.AdditionalSecurityGroups return nil } diff --git a/pkg/apis/kops/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/kops/v1alpha1/zz_generated.deepcopy.go index e1d6228c36..f400315cb0 100644 --- a/pkg/apis/kops/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/kops/v1alpha1/zz_generated.deepcopy.go @@ -2616,6 +2616,11 @@ func (in *LoadBalancerAccessSpec) DeepCopyInto(out *LoadBalancerAccessSpec) { **out = **in } } + if in.AdditionalSecurityGroups != nil { + in, out := &in.AdditionalSecurityGroups, &out.AdditionalSecurityGroups + *out = make([]string, len(*in)) + copy(*out, *in) + } return } diff --git a/pkg/apis/kops/v1alpha2/cluster.go b/pkg/apis/kops/v1alpha2/cluster.go index 06c4c51d83..9def103e19 100644 --- a/pkg/apis/kops/v1alpha2/cluster.go +++ b/pkg/apis/kops/v1alpha2/cluster.go @@ -264,8 +264,9 @@ const ( ) type LoadBalancerAccessSpec struct { - Type LoadBalancerType `json:"type,omitempty"` - IdleTimeoutSeconds *int64 `json:"idleTimeoutSeconds,omitempty"` + Type LoadBalancerType `json:"type,omitempty"` + IdleTimeoutSeconds *int64 `json:"idleTimeoutSeconds,omitempty"` + AdditionalSecurityGroups []string `json:"additionalSecurityGroups,omitempty"` } type KubeDNSConfig struct { diff --git a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go index b3697b091a..6f14ad408c 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go @@ -2543,6 +2543,7 @@ func Convert_kops_LeaderElectionConfiguration_To_v1alpha2_LeaderElectionConfigur func autoConvert_v1alpha2_LoadBalancerAccessSpec_To_kops_LoadBalancerAccessSpec(in *LoadBalancerAccessSpec, out *kops.LoadBalancerAccessSpec, s conversion.Scope) error { out.Type = kops.LoadBalancerType(in.Type) out.IdleTimeoutSeconds = in.IdleTimeoutSeconds + out.AdditionalSecurityGroups = in.AdditionalSecurityGroups return nil } @@ -2554,6 +2555,7 @@ func Convert_v1alpha2_LoadBalancerAccessSpec_To_kops_LoadBalancerAccessSpec(in * func autoConvert_kops_LoadBalancerAccessSpec_To_v1alpha2_LoadBalancerAccessSpec(in *kops.LoadBalancerAccessSpec, out *LoadBalancerAccessSpec, s conversion.Scope) error { out.Type = LoadBalancerType(in.Type) out.IdleTimeoutSeconds = in.IdleTimeoutSeconds + out.AdditionalSecurityGroups = in.AdditionalSecurityGroups return nil } diff --git a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go index 3c50685758..3a8e2d8916 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go @@ -2742,6 +2742,11 @@ func (in *LoadBalancerAccessSpec) DeepCopyInto(out *LoadBalancerAccessSpec) { **out = **in } } + if in.AdditionalSecurityGroups != nil { + in, out := &in.AdditionalSecurityGroups, &out.AdditionalSecurityGroups + *out = make([]string, len(*in)) + copy(*out, *in) + } return } diff --git a/pkg/apis/kops/zz_generated.deepcopy.go b/pkg/apis/kops/zz_generated.deepcopy.go index 2f3f1fc455..922c2afb42 100644 --- a/pkg/apis/kops/zz_generated.deepcopy.go +++ b/pkg/apis/kops/zz_generated.deepcopy.go @@ -2977,6 +2977,11 @@ func (in *LoadBalancerAccessSpec) DeepCopyInto(out *LoadBalancerAccessSpec) { **out = **in } } + if in.AdditionalSecurityGroups != nil { + in, out := &in.AdditionalSecurityGroups, &out.AdditionalSecurityGroups + *out = make([]string, len(*in)) + copy(*out, *in) + } return } diff --git a/pkg/model/awsmodel/api_loadbalancer.go b/pkg/model/awsmodel/api_loadbalancer.go index 3490ca35c0..d0e5e5d39f 100644 --- a/pkg/model/awsmodel/api_loadbalancer.go +++ b/pkg/model/awsmodel/api_loadbalancer.go @@ -184,6 +184,21 @@ func (b *APILoadBalancerBuilder) Build(c *fi.ModelBuilderContext) error { } } + // Add precreated additional security groups to the ELB + { + for _, id := range b.Cluster.Spec.API.LoadBalancer.AdditionalSecurityGroups { + t := &awstasks.SecurityGroup{ + Name: fi.String(id), + ID: fi.String(id), + Shared: fi.Bool(true), + } + if err := c.EnsureTask(t); err != nil { + return err + } + elb.SecurityGroups = append(elb.SecurityGroups, t) + } + } + // Allow HTTPS to the master instances from the ELB { t := &awstasks.SecurityGroupRule{ From 60a90bfd364cb32b99b4f4bbd446ff07f4b2e336 Mon Sep 17 00:00:00 2001 From: Abdullah Almariah Date: Mon, 11 Dec 2017 15:49:53 +0100 Subject: [PATCH 02/11] Allow additional SGs to be added to API loadbalancer --- docs/cli/kops_create_cluster.md | 79 +++++++++++++++++---------------- 1 file changed, 40 insertions(+), 39 deletions(-) diff --git a/docs/cli/kops_create_cluster.md b/docs/cli/kops_create_cluster.md index 5b259c9bba..8629037ed9 100644 --- a/docs/cli/kops_create_cluster.md +++ b/docs/cli/kops_create_cluster.md @@ -65,45 +65,46 @@ kops create cluster ### Options ``` - --admin-access stringSlice Restrict API access to this CIDR. If not set, access will not be restricted by IP. (default [0.0.0.0/0]) - --api-loadbalancer-type string Sets the API loadbalancer type to either 'public' or 'internal' - --associate-public-ip Specify --associate-public-ip=[true|false] to enable/disable association of public IP for master ASG and nodes. Default is 'true'. - --authorization string Authorization mode to use: AlwaysAllow or RBAC (default "AlwaysAllow") - --bastion Pass the --bastion flag to enable a bastion instance group. Only applies to private topology. - --channel string Channel for default versions and configuration to use (default "stable") - --cloud string Cloud provider to use - gce, aws, vsphere - --cloud-labels string A list of KV pairs used to tag all instance groups in AWS (eg "Owner=John Doe,Team=Some Team"). - --dns string DNS hosted zone to use: public|private. Default is 'public'. (default "Public") - --dns-zone string DNS hosted zone to use (defaults to longest matching zone) - --dry-run If true, only print the object that would be sent, without sending it. This flag can be used to create a cluster YAML or JSON manifest. - --encrypt-etcd-storage Generate key in aws kms and use it for encrypt etcd volumes - --image string Image to use for all instances. - --kubernetes-version string Version of kubernetes to run (defaults to version in channel) - --master-count int32 Set the number of masters. Defaults to one master per master-zone - --master-public-name string Sets the public master public name - --master-security-groups stringSlice Add precreated additional security groups to masters. - --master-size string Set instance size for masters - --master-tenancy string The tenancy of the master group on AWS. Can either be default or dedicated. - --master-volume-size int32 Set instance volume size (in GB) for masters - --master-zones stringSlice Zones in which to run masters (must be an odd number) - --model string Models to apply (separate multiple models with commas) (default "config,proto,cloudup") - --network-cidr string Set to override the default network CIDR - --networking string Networking mode to use. kubenet (default), classic, external, kopeio-vxlan (or kopeio), weave, flannel-vxlan (or flannel), flannel-udp, calico, canal, kube-router, romana. (default "kubenet") - --node-count int32 Set the number of nodes - --node-security-groups stringSlice Add precreated additional security groups to nodes. - --node-size string Set instance size for nodes - --node-tenancy string The tenancy of the node group on AWS. Can be either default or dedicated. - --node-volume-size int32 Set instance volume size (in GB) for nodes - --out string Path to write any local output - -o, --output string Ouput format. One of json|yaml. Used with the --dry-run flag. - --project string Project to use (must be set on GCE) - --ssh-access stringSlice Restrict SSH access to this CIDR. If not set, access will not be restricted by IP. (default [0.0.0.0/0]) - --ssh-public-key string SSH public key to use (default "~/.ssh/id_rsa.pub") - --target string Valid targets: direct, terraform, direct. Set this flag to terraform if you want kops to generate terraform (default "direct") - -t, --topology string Controls network topology for the cluster. public|private. Default is 'public'. (default "public") - --vpc string Set to use a shared VPC - -y, --yes Specify --yes to immediately create the cluster - --zones stringSlice Zones in which to run the cluster + --admin-access stringSlice Restrict API access to this CIDR. If not set, access will not be restricted by IP. (default [0.0.0.0/0]) + --api-loadbalancer-security-groups stringSlice Add precreated additional security groups to the API loadbalancer. + --api-loadbalancer-type string Sets the API loadbalancer type to either 'public' or 'internal' + --associate-public-ip Specify --associate-public-ip=[true|false] to enable/disable association of public IP for master ASG and nodes. Default is 'true'. + --authorization string Authorization mode to use: AlwaysAllow or RBAC (default "AlwaysAllow") + --bastion Pass the --bastion flag to enable a bastion instance group. Only applies to private topology. + --channel string Channel for default versions and configuration to use (default "stable") + --cloud string Cloud provider to use - gce, aws, vsphere + --cloud-labels string A list of KV pairs used to tag all instance groups in AWS (eg "Owner=John Doe,Team=Some Team"). + --dns string DNS hosted zone to use: public|private. Default is 'public'. (default "Public") + --dns-zone string DNS hosted zone to use (defaults to longest matching zone) + --dry-run If true, only print the object that would be sent, without sending it. This flag can be used to create a cluster YAML or JSON manifest. + --encrypt-etcd-storage Generate key in aws kms and use it for encrypt etcd volumes + --image string Image to use for all instances. + --kubernetes-version string Version of kubernetes to run (defaults to version in channel) + --master-count int32 Set the number of masters. Defaults to one master per master-zone + --master-public-name string Sets the public master public name + --master-security-groups stringSlice Add precreated additional security groups to masters. + --master-size string Set instance size for masters + --master-tenancy string The tenancy of the master group on AWS. Can either be default or dedicated. + --master-volume-size int32 Set instance volume size (in GB) for masters + --master-zones stringSlice Zones in which to run masters (must be an odd number) + --model string Models to apply (separate multiple models with commas) (default "config,proto,cloudup") + --network-cidr string Set to override the default network CIDR + --networking string Networking mode to use. kubenet (default), classic, external, kopeio-vxlan (or kopeio), weave, flannel-vxlan (or flannel), flannel-udp, calico, canal, kube-router, romana. (default "kubenet") + --node-count int32 Set the number of nodes + --node-security-groups stringSlice Add precreated additional security groups to nodes. + --node-size string Set instance size for nodes + --node-tenancy string The tenancy of the node group on AWS. Can be either default or dedicated. + --node-volume-size int32 Set instance volume size (in GB) for nodes + --out string Path to write any local output + -o, --output string Ouput format. One of json|yaml. Used with the --dry-run flag. + --project string Project to use (must be set on GCE) + --ssh-access stringSlice Restrict SSH access to this CIDR. If not set, access will not be restricted by IP. (default [0.0.0.0/0]) + --ssh-public-key string SSH public key to use (default "~/.ssh/id_rsa.pub") + --target string Valid targets: direct, terraform, direct. Set this flag to terraform if you want kops to generate terraform (default "direct") + -t, --topology string Controls network topology for the cluster. public|private. Default is 'public'. (default "public") + --vpc string Set to use a shared VPC + -y, --yes Specify --yes to immediately create the cluster + --zones stringSlice Zones in which to run the cluster ``` ### Options inherited from parent commands From 6e76277c1b3cd90d3495a9e0baf8b628753b1760 Mon Sep 17 00:00:00 2001 From: Abdullah Almariah Date: Tue, 12 Dec 2017 11:15:14 +0100 Subject: [PATCH 03/11] Revert "gen-cli-docs" This reverts commit 60a90bfd364cb32b99b4f4bbd446ff07f4b2e336. --- docs/cli/kops_create_cluster.md | 79 ++++++++++++++++----------------- 1 file changed, 39 insertions(+), 40 deletions(-) diff --git a/docs/cli/kops_create_cluster.md b/docs/cli/kops_create_cluster.md index 8629037ed9..5b259c9bba 100644 --- a/docs/cli/kops_create_cluster.md +++ b/docs/cli/kops_create_cluster.md @@ -65,46 +65,45 @@ kops create cluster ### Options ``` - --admin-access stringSlice Restrict API access to this CIDR. If not set, access will not be restricted by IP. (default [0.0.0.0/0]) - --api-loadbalancer-security-groups stringSlice Add precreated additional security groups to the API loadbalancer. - --api-loadbalancer-type string Sets the API loadbalancer type to either 'public' or 'internal' - --associate-public-ip Specify --associate-public-ip=[true|false] to enable/disable association of public IP for master ASG and nodes. Default is 'true'. - --authorization string Authorization mode to use: AlwaysAllow or RBAC (default "AlwaysAllow") - --bastion Pass the --bastion flag to enable a bastion instance group. Only applies to private topology. - --channel string Channel for default versions and configuration to use (default "stable") - --cloud string Cloud provider to use - gce, aws, vsphere - --cloud-labels string A list of KV pairs used to tag all instance groups in AWS (eg "Owner=John Doe,Team=Some Team"). - --dns string DNS hosted zone to use: public|private. Default is 'public'. (default "Public") - --dns-zone string DNS hosted zone to use (defaults to longest matching zone) - --dry-run If true, only print the object that would be sent, without sending it. This flag can be used to create a cluster YAML or JSON manifest. - --encrypt-etcd-storage Generate key in aws kms and use it for encrypt etcd volumes - --image string Image to use for all instances. - --kubernetes-version string Version of kubernetes to run (defaults to version in channel) - --master-count int32 Set the number of masters. Defaults to one master per master-zone - --master-public-name string Sets the public master public name - --master-security-groups stringSlice Add precreated additional security groups to masters. - --master-size string Set instance size for masters - --master-tenancy string The tenancy of the master group on AWS. Can either be default or dedicated. - --master-volume-size int32 Set instance volume size (in GB) for masters - --master-zones stringSlice Zones in which to run masters (must be an odd number) - --model string Models to apply (separate multiple models with commas) (default "config,proto,cloudup") - --network-cidr string Set to override the default network CIDR - --networking string Networking mode to use. kubenet (default), classic, external, kopeio-vxlan (or kopeio), weave, flannel-vxlan (or flannel), flannel-udp, calico, canal, kube-router, romana. (default "kubenet") - --node-count int32 Set the number of nodes - --node-security-groups stringSlice Add precreated additional security groups to nodes. - --node-size string Set instance size for nodes - --node-tenancy string The tenancy of the node group on AWS. Can be either default or dedicated. - --node-volume-size int32 Set instance volume size (in GB) for nodes - --out string Path to write any local output - -o, --output string Ouput format. One of json|yaml. Used with the --dry-run flag. - --project string Project to use (must be set on GCE) - --ssh-access stringSlice Restrict SSH access to this CIDR. If not set, access will not be restricted by IP. (default [0.0.0.0/0]) - --ssh-public-key string SSH public key to use (default "~/.ssh/id_rsa.pub") - --target string Valid targets: direct, terraform, direct. Set this flag to terraform if you want kops to generate terraform (default "direct") - -t, --topology string Controls network topology for the cluster. public|private. Default is 'public'. (default "public") - --vpc string Set to use a shared VPC - -y, --yes Specify --yes to immediately create the cluster - --zones stringSlice Zones in which to run the cluster + --admin-access stringSlice Restrict API access to this CIDR. If not set, access will not be restricted by IP. (default [0.0.0.0/0]) + --api-loadbalancer-type string Sets the API loadbalancer type to either 'public' or 'internal' + --associate-public-ip Specify --associate-public-ip=[true|false] to enable/disable association of public IP for master ASG and nodes. Default is 'true'. + --authorization string Authorization mode to use: AlwaysAllow or RBAC (default "AlwaysAllow") + --bastion Pass the --bastion flag to enable a bastion instance group. Only applies to private topology. + --channel string Channel for default versions and configuration to use (default "stable") + --cloud string Cloud provider to use - gce, aws, vsphere + --cloud-labels string A list of KV pairs used to tag all instance groups in AWS (eg "Owner=John Doe,Team=Some Team"). + --dns string DNS hosted zone to use: public|private. Default is 'public'. (default "Public") + --dns-zone string DNS hosted zone to use (defaults to longest matching zone) + --dry-run If true, only print the object that would be sent, without sending it. This flag can be used to create a cluster YAML or JSON manifest. + --encrypt-etcd-storage Generate key in aws kms and use it for encrypt etcd volumes + --image string Image to use for all instances. + --kubernetes-version string Version of kubernetes to run (defaults to version in channel) + --master-count int32 Set the number of masters. Defaults to one master per master-zone + --master-public-name string Sets the public master public name + --master-security-groups stringSlice Add precreated additional security groups to masters. + --master-size string Set instance size for masters + --master-tenancy string The tenancy of the master group on AWS. Can either be default or dedicated. + --master-volume-size int32 Set instance volume size (in GB) for masters + --master-zones stringSlice Zones in which to run masters (must be an odd number) + --model string Models to apply (separate multiple models with commas) (default "config,proto,cloudup") + --network-cidr string Set to override the default network CIDR + --networking string Networking mode to use. kubenet (default), classic, external, kopeio-vxlan (or kopeio), weave, flannel-vxlan (or flannel), flannel-udp, calico, canal, kube-router, romana. (default "kubenet") + --node-count int32 Set the number of nodes + --node-security-groups stringSlice Add precreated additional security groups to nodes. + --node-size string Set instance size for nodes + --node-tenancy string The tenancy of the node group on AWS. Can be either default or dedicated. + --node-volume-size int32 Set instance volume size (in GB) for nodes + --out string Path to write any local output + -o, --output string Ouput format. One of json|yaml. Used with the --dry-run flag. + --project string Project to use (must be set on GCE) + --ssh-access stringSlice Restrict SSH access to this CIDR. If not set, access will not be restricted by IP. (default [0.0.0.0/0]) + --ssh-public-key string SSH public key to use (default "~/.ssh/id_rsa.pub") + --target string Valid targets: direct, terraform, direct. Set this flag to terraform if you want kops to generate terraform (default "direct") + -t, --topology string Controls network topology for the cluster. public|private. Default is 'public'. (default "public") + --vpc string Set to use a shared VPC + -y, --yes Specify --yes to immediately create the cluster + --zones stringSlice Zones in which to run the cluster ``` ### Options inherited from parent commands From e1ccc339ae42655cd4c0cba1001549bb3440ed92 Mon Sep 17 00:00:00 2001 From: Abdullah Almariah Date: Tue, 12 Dec 2017 11:20:07 +0100 Subject: [PATCH 04/11] Remove --api-loadbalancer-security-groups cli arg --- cmd/kops/create_cluster.go | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/cmd/kops/create_cluster.go b/cmd/kops/create_cluster.go index 37123e607d..929871d582 100644 --- a/cmd/kops/create_cluster.go +++ b/cmd/kops/create_cluster.go @@ -113,9 +113,6 @@ type CreateClusterOptions struct { // Specify API loadbalancer as public or internal APILoadBalancerType string - // Add precreated additional security groups to the ELB - APILoadBalancerSecurityGroups []string - // Allow custom public master name MasterPublicName string @@ -314,8 +311,6 @@ func NewCmdCreateCluster(f *util.Factory, out io.Writer) *cobra.Command { cmd.Flags().StringVar(&options.APILoadBalancerType, "api-loadbalancer-type", options.APILoadBalancerType, "Sets the API loadbalancer type to either 'public' or 'internal'") - cmd.Flags().StringSliceVar(&options.APILoadBalancerSecurityGroups, "api-loadbalancer-security-groups", options.APILoadBalancerSecurityGroups, "Add precreated additional security groups to the API loadbalancer.") - // Allow custom public master name cmd.Flags().StringVar(&options.MasterPublicName, "master-public-name", options.MasterPublicName, "Sets the public master public name") @@ -947,9 +942,6 @@ func RunCreateCluster(f *util.Factory, out io.Writer, c *CreateClusterOptions) e } } } - if len(c.APILoadBalancerSecurityGroups) > 0 && cluster.Spec.API.LoadBalancer == nil { - return fmt.Errorf("unable to add security groups to nonexistent API loadbalancer") - } if cluster.Spec.API.LoadBalancer != nil && cluster.Spec.API.LoadBalancer.Type == "" { switch c.APILoadBalancerType { case "", "public": @@ -959,9 +951,6 @@ func RunCreateCluster(f *util.Factory, out io.Writer, c *CreateClusterOptions) e default: return fmt.Errorf("unknown api-loadbalancer-type: %q", c.APILoadBalancerType) } - if len(c.APILoadBalancerSecurityGroups) > 0 { - cluster.Spec.API.LoadBalancer.AdditionalSecurityGroups = c.APILoadBalancerSecurityGroups - } } // Use Strict IAM policy and allow AWS ECR by default when creating a new cluster From 5a46228927070bdae4d83b80790dc44a3e98a803 Mon Sep 17 00:00:00 2001 From: Abdullah Almariah Date: Tue, 12 Dec 2017 15:27:00 +0100 Subject: [PATCH 05/11] awsValidateAdditionalSecurityGroups for API loadbalancer --- pkg/apis/kops/validation/aws.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/pkg/apis/kops/validation/aws.go b/pkg/apis/kops/validation/aws.go index 517582f292..49eefd61e8 100644 --- a/pkg/apis/kops/validation/aws.go +++ b/pkg/apis/kops/validation/aws.go @@ -19,13 +19,18 @@ package validation import ( "strings" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/upup/pkg/fi/cloudup/awsup" ) func awsValidateCluster(c *kops.Cluster) field.ErrorList { - return nil + allErrs := field.ErrorList{} + + allErrs = append(allErrs, awsValidateAdditionalSecurityGroups(field.NewPath("spec", "api", "loadBalancer", "additionalSecurityGroups"), c.Spec.API.LoadBalancer.AdditionalSecurityGroups)...) + + return allErrs } func awsValidateInstanceGroup(ig *kops.InstanceGroup) field.ErrorList { @@ -41,7 +46,12 @@ func awsValidateInstanceGroup(ig *kops.InstanceGroup) field.ErrorList { func awsValidateAdditionalSecurityGroups(fieldPath *field.Path, groups []string) field.ErrorList { allErrs := field.ErrorList{} + names := sets.NewString() for i, s := range groups { + if names.Has(s) { + allErrs = append(allErrs, field.Invalid(fieldPath.Index(i), s, "security groups with duplicate name found")) + } + names.Insert(s) if strings.TrimSpace(s) == "" { allErrs = append(allErrs, field.Invalid(fieldPath.Index(i), s, "security group cannot be empty, if specified")) continue From d181ecf8854d405c88ee356de0d4f7e076d2782f Mon Sep 17 00:00:00 2001 From: Abdullah Almariah Date: Tue, 12 Dec 2017 23:54:22 +0100 Subject: [PATCH 06/11] awsValidateAdditionalSecurityGroups for API loadbalancer --- pkg/apis/kops/validation/aws.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/apis/kops/validation/aws.go b/pkg/apis/kops/validation/aws.go index 49eefd61e8..90eb38f6f5 100644 --- a/pkg/apis/kops/validation/aws.go +++ b/pkg/apis/kops/validation/aws.go @@ -28,7 +28,11 @@ import ( func awsValidateCluster(c *kops.Cluster) field.ErrorList { allErrs := field.ErrorList{} - allErrs = append(allErrs, awsValidateAdditionalSecurityGroups(field.NewPath("spec", "api", "loadBalancer", "additionalSecurityGroups"), c.Spec.API.LoadBalancer.AdditionalSecurityGroups)...) + if c.Spec.API != nil { + if c.Spec.API.LoadBalancer != nil { + allErrs = append(allErrs, awsValidateAdditionalSecurityGroups(field.NewPath("spec", "api", "loadBalancer", "additionalSecurityGroups"), c.Spec.API.LoadBalancer.AdditionalSecurityGroups)...) + } + } return allErrs } From 16ed8982e67136b4900cad7d6bc007d8245d2630 Mon Sep 17 00:00:00 2001 From: Abdullah Almariah Date: Wed, 13 Dec 2017 00:31:59 +0100 Subject: [PATCH 07/11] update_cluster complex integration test --- tests/integration/update_cluster/complex/in-v1alpha2.yaml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/integration/update_cluster/complex/in-v1alpha2.yaml b/tests/integration/update_cluster/complex/in-v1alpha2.yaml index 62b9c5bc26..0b8d201667 100644 --- a/tests/integration/update_cluster/complex/in-v1alpha2.yaml +++ b/tests/integration/update_cluster/complex/in-v1alpha2.yaml @@ -4,6 +4,12 @@ metadata: creationTimestamp: "2016-12-10T22:42:27Z" name: complex.example.com spec: + api: + loadBalancer: + type: Public + additionalSecurityGroups: + - sg-exampleid3 + - sg-exampleid4 kubernetesApiAccess: - 0.0.0.0/0 channel: stable @@ -84,5 +90,3 @@ spec: role: Master subnets: - us-test-1a - - From 22194c982dc6503e6c8287e5979c5b6775ff1366 Mon Sep 17 00:00:00 2001 From: Abdullah Almariah Date: Wed, 13 Dec 2017 01:29:16 +0100 Subject: [PATCH 08/11] Update kubernetes.tf for API loadbalancer --- .../update_cluster/complex/kubernetes.tf | 80 ++++++++++++++++++- 1 file changed, 78 insertions(+), 2 deletions(-) diff --git a/tests/integration/update_cluster/complex/kubernetes.tf b/tests/integration/update_cluster/complex/kubernetes.tf index cc77b45e4a..e8613f032e 100644 --- a/tests/integration/update_cluster/complex/kubernetes.tf +++ b/tests/integration/update_cluster/complex/kubernetes.tf @@ -42,6 +42,11 @@ provider "aws" { region = "us-test-1" } +resource "aws_autoscaling_attachment" "master-us-test-1a-masters-complex-example-com" { + elb = "${aws_elb.api-complex-example-com.id}" + autoscaling_group_name = "${aws_autoscaling_group.master-us-test-1a-masters-complex-example-com.id}" +} + resource "aws_autoscaling_group" "master-us-test-1a-masters-complex-example-com" { name = "master-us-test-1a.masters.complex.example.com" launch_configuration = "${aws_launch_configuration.master-us-test-1a-masters-complex-example-com.id}" @@ -150,6 +155,35 @@ resource "aws_ebs_volume" "us-test-1a-etcd-main-complex-example-com" { } } +resource "aws_elb" "api-complex-example-com" { + name = "api-complex-example-com-vd3t5n" + + listener = { + instance_port = 443 + instance_protocol = "TCP" + lb_port = 443 + lb_protocol = "TCP" + } + + security_groups = ["${aws_security_group.api-elb-complex-example-com.id}", "sg-exampleid3", "sg-exampleid4"] + subnets = ["${aws_subnet.us-test-1a-complex-example-com.id}"] + + health_check = { + target = "SSL:443" + healthy_threshold = 2 + unhealthy_threshold = 2 + interval = 10 + timeout = 5 + } + + idle_timeout = 300 + + tags = { + KubernetesCluster = "complex.example.com" + Name = "api.complex.example.com" + } +} + resource "aws_iam_instance_profile" "masters-complex-example-com" { name = "masters.complex.example.com" role = "${aws_iam_role.masters-complex-example-com.name}" @@ -249,6 +283,19 @@ resource "aws_route" "0-0-0-0--0" { gateway_id = "${aws_internet_gateway.complex-example-com.id}" } +resource "aws_route53_record" "api-complex-example-com" { + name = "api.complex.example.com" + type = "A" + + alias = { + name = "${aws_elb.api-complex-example-com.dns_name}" + zone_id = "${aws_elb.api-complex-example-com.zone_id}" + evaluate_target_health = false + } + + zone_id = "/hostedzone/Z1AFAKE1ZON3YO" +} + resource "aws_route_table" "complex-example-com" { vpc_id = "${aws_vpc.complex-example-com.id}" @@ -263,6 +310,17 @@ resource "aws_route_table_association" "us-test-1a-complex-example-com" { route_table_id = "${aws_route_table.complex-example-com.id}" } +resource "aws_security_group" "api-elb-complex-example-com" { + name = "api-elb.complex.example.com" + vpc_id = "${aws_vpc.complex-example-com.id}" + description = "Security group for api ELB" + + tags = { + KubernetesCluster = "complex.example.com" + Name = "api-elb.complex.example.com" + } +} + resource "aws_security_group" "masters-complex-example-com" { name = "masters.complex.example.com" vpc_id = "${aws_vpc.complex-example-com.id}" @@ -312,15 +370,33 @@ resource "aws_security_group_rule" "all-node-to-node" { protocol = "-1" } -resource "aws_security_group_rule" "https-external-to-master-0-0-0-0--0" { +resource "aws_security_group_rule" "api-elb-egress" { + type = "egress" + security_group_id = "${aws_security_group.api-elb-complex-example-com.id}" + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] +} + +resource "aws_security_group_rule" "https-api-elb-0-0-0-0--0" { type = "ingress" - security_group_id = "${aws_security_group.masters-complex-example-com.id}" + security_group_id = "${aws_security_group.api-elb-complex-example-com.id}" from_port = 443 to_port = 443 protocol = "tcp" cidr_blocks = ["0.0.0.0/0"] } +resource "aws_security_group_rule" "https-elb-to-master" { + type = "ingress" + security_group_id = "${aws_security_group.masters-complex-example-com.id}" + source_security_group_id = "${aws_security_group.api-elb-complex-example-com.id}" + from_port = 443 + to_port = 443 + protocol = "tcp" +} + resource "aws_security_group_rule" "master-egress" { type = "egress" security_group_id = "${aws_security_group.masters-complex-example-com.id}" From 2746b88808ca53e7563e660e5587e89dbd10de71 Mon Sep 17 00:00:00 2001 From: Abdullah Almariah Date: Wed, 13 Dec 2017 02:05:42 +0100 Subject: [PATCH 09/11] Update cluster.spec docs --- docs/cluster_spec.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/cluster_spec.md b/docs/cluster_spec.md index e3f1227313..6dd9a0f969 100644 --- a/docs/cluster_spec.md +++ b/docs/cluster_spec.md @@ -23,6 +23,18 @@ spec: When configuring a LoadBalancer, you can also choose to have a public ELB or an internal (VPC only) ELB. The `type` field should be `Public` or `Internal`. +Also, you can add precreated additional security groups to the load balancer by setting `additionalSecurityGroups`. + +```yaml +spec: + api: + loadBalancer: + type: Public + additionalSecurityGroups: + - sg-xxxxxxxx + - sg-xxxxxxxx +``` + Additionally, you can increase idle timeout of the load balancer by setting its `idleTimeoutSeconds`. The default idle timeout is 5 minutes, with a maximum of 3600 seconds (60 minutes) being allowed by AWS. For more information see [configuring idle timeouts](http://docs.aws.amazon.com/elasticloadbalancing/latest/classic/config-idle-timeout.html). From 5b4c9e2a147134a3392fd8185c0e2761bd0f01ee Mon Sep 17 00:00:00 2001 From: Abdullah Almariah Date: Thu, 14 Dec 2017 03:09:08 +0100 Subject: [PATCH 10/11] Add API docs --- pkg/apis/kops/cluster.go | 1 + pkg/apis/kops/v1alpha1/cluster.go | 1 + pkg/apis/kops/v1alpha2/cluster.go | 1 + 3 files changed, 3 insertions(+) diff --git a/pkg/apis/kops/cluster.go b/pkg/apis/kops/cluster.go index 2546706198..e2b1e51a6a 100644 --- a/pkg/apis/kops/cluster.go +++ b/pkg/apis/kops/cluster.go @@ -263,6 +263,7 @@ const ( LoadBalancerTypeInternal LoadBalancerType = "Internal" ) +// LoadBalancerAccessSpec provides configuration details related to API LoadBalancer and it access type LoadBalancerAccessSpec struct { Type LoadBalancerType `json:"type,omitempty"` IdleTimeoutSeconds *int64 `json:"idleTimeoutSeconds,omitempty"` diff --git a/pkg/apis/kops/v1alpha1/cluster.go b/pkg/apis/kops/v1alpha1/cluster.go index e2b1d6d1c3..72ed9354af 100644 --- a/pkg/apis/kops/v1alpha1/cluster.go +++ b/pkg/apis/kops/v1alpha1/cluster.go @@ -262,6 +262,7 @@ const ( LoadBalancerTypeInternal LoadBalancerType = "Internal" ) +// LoadBalancerAccessSpec provides configuration details related to API LoadBalancer and it access type LoadBalancerAccessSpec struct { Type LoadBalancerType `json:"type,omitempty"` IdleTimeoutSeconds *int64 `json:"idleTimeoutSeconds,omitempty"` diff --git a/pkg/apis/kops/v1alpha2/cluster.go b/pkg/apis/kops/v1alpha2/cluster.go index 9def103e19..15d5518c84 100644 --- a/pkg/apis/kops/v1alpha2/cluster.go +++ b/pkg/apis/kops/v1alpha2/cluster.go @@ -263,6 +263,7 @@ const ( LoadBalancerTypeInternal LoadBalancerType = "Internal" ) +// LoadBalancerAccessSpec provides configuration details related to API LoadBalancer and it access type LoadBalancerAccessSpec struct { Type LoadBalancerType `json:"type,omitempty"` IdleTimeoutSeconds *int64 `json:"idleTimeoutSeconds,omitempty"` From 4b0aa1d32080078728ea5f4ac2ee6e7fbd3c1780 Mon Sep 17 00:00:00 2001 From: Abdullah Almariah Date: Thu, 14 Dec 2017 03:11:44 +0100 Subject: [PATCH 11/11] Add API docs --- pkg/apis/kops/cluster.go | 2 +- pkg/apis/kops/v1alpha1/cluster.go | 2 +- pkg/apis/kops/v1alpha2/cluster.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/apis/kops/cluster.go b/pkg/apis/kops/cluster.go index e2b1e51a6a..0a63cdd6cb 100644 --- a/pkg/apis/kops/cluster.go +++ b/pkg/apis/kops/cluster.go @@ -263,7 +263,7 @@ const ( LoadBalancerTypeInternal LoadBalancerType = "Internal" ) -// LoadBalancerAccessSpec provides configuration details related to API LoadBalancer and it access +// LoadBalancerAccessSpec provides configuration details related to API LoadBalancer and its access type LoadBalancerAccessSpec struct { Type LoadBalancerType `json:"type,omitempty"` IdleTimeoutSeconds *int64 `json:"idleTimeoutSeconds,omitempty"` diff --git a/pkg/apis/kops/v1alpha1/cluster.go b/pkg/apis/kops/v1alpha1/cluster.go index 72ed9354af..36200a13d7 100644 --- a/pkg/apis/kops/v1alpha1/cluster.go +++ b/pkg/apis/kops/v1alpha1/cluster.go @@ -262,7 +262,7 @@ const ( LoadBalancerTypeInternal LoadBalancerType = "Internal" ) -// LoadBalancerAccessSpec provides configuration details related to API LoadBalancer and it access +// LoadBalancerAccessSpec provides configuration details related to API LoadBalancer and its access type LoadBalancerAccessSpec struct { Type LoadBalancerType `json:"type,omitempty"` IdleTimeoutSeconds *int64 `json:"idleTimeoutSeconds,omitempty"` diff --git a/pkg/apis/kops/v1alpha2/cluster.go b/pkg/apis/kops/v1alpha2/cluster.go index 15d5518c84..9c5de015bf 100644 --- a/pkg/apis/kops/v1alpha2/cluster.go +++ b/pkg/apis/kops/v1alpha2/cluster.go @@ -263,7 +263,7 @@ const ( LoadBalancerTypeInternal LoadBalancerType = "Internal" ) -// LoadBalancerAccessSpec provides configuration details related to API LoadBalancer and it access +// LoadBalancerAccessSpec provides configuration details related to API LoadBalancer and its access type LoadBalancerAccessSpec struct { Type LoadBalancerType `json:"type,omitempty"` IdleTimeoutSeconds *int64 `json:"idleTimeoutSeconds,omitempty"`