Merge branch 'master' into apiserver_cpurequest

docs/cluster_spec.md

@@ -354,6 +354,16 @@ spec:
     enableCustomMetrics: true
 ```
 
+#### Setting kubelet CPU management policies
+To enable cpu management policies in kubernetes as per [cpu management doc](https://kubernetes.io/docs/tasks/administer-cluster/cpu-management-policies/#cpu-management-policies)
+we have to set the flag `--cpu-manager-policy` to the appropriate value on all the kubelets. This must be specified in the `kubelet` spec in our cluster.yml.
+
+```
+spec:
+  kubelet:
+    cpuManagerPolicy: static
+```
+
 #### Setting kubelet configurations together with the Amazon VPC backend
 Setting kubelet configurations together with the networking Amazon VPC backend requires to also set the `cloudProvider: aws` setting in this block. Example:
 

pkg/apis/kops/componentconfig.go

@@ -187,6 +187,8 @@ type KubeletConfigSpec struct {
 	CPUCFSQuota *bool `json:"cpuCFSQuota,omitempty" flag:"cpu-cfs-quota"`
 	// CPUCFSQuotaPeriod sets CPU CFS quota period value, cpu.cfs_period_us, defaults to Linux Kernel default
 	CPUCFSQuotaPeriod *metav1.Duration `json:"cpuCFSQuotaPeriod,omitempty" flag:"cpu-cfs-quota-period"`
+	// CpuManagerPolicy allows for changing the default policy of None to static
+	CpuManagerPolicy string `json:"cpuManagerPolicy,omitempty" flag:"cpu-manager-policy"`
 }
 
 // KubeProxyConfig defines the configuration for a proxy

pkg/apis/kops/v1alpha1/componentconfig.go

@@ -187,6 +187,8 @@ type KubeletConfigSpec struct {
 	CPUCFSQuota *bool `json:"cpuCFSQuota,omitempty" flag:"cpu-cfs-quota"`
 	// CPUCFSQuotaPeriod sets CPU CFS quota period value, cpu.cfs_period_us, defaults to Linux Kernel default
 	CPUCFSQuotaPeriod *metav1.Duration `json:"cpuCFSQuotaPeriod,omitempty" flag:"cpu-cfs-quota-period"`
+	// CpuManagerPolicy allows for changing the default policy of None to static
+	CpuManagerPolicy string `json:"cpuManagerPolicy,omitempty" flag:"cpu-manager-policy"`
 }
 
 // KubeProxyConfig defines the configuration for a proxy

pkg/apis/kops/v1alpha1/zz_generated.conversion.go

@@ -3385,6 +3385,7 @@ func autoConvert_v1alpha1_KubeletConfigSpec_To_kops_KubeletConfigSpec(in *Kubele
 	out.AuthenticationTokenWebhookCacheTTL = in.AuthenticationTokenWebhookCacheTTL
 	out.CPUCFSQuota = in.CPUCFSQuota
 	out.CPUCFSQuotaPeriod = in.CPUCFSQuotaPeriod
+	out.CpuManagerPolicy = in.CpuManagerPolicy
 	return nil
 }
 
@@ -3466,6 +3467,7 @@ func autoConvert_kops_KubeletConfigSpec_To_v1alpha1_KubeletConfigSpec(in *kops.K
 	out.AuthenticationTokenWebhookCacheTTL = in.AuthenticationTokenWebhookCacheTTL
 	out.CPUCFSQuota = in.CPUCFSQuota
 	out.CPUCFSQuotaPeriod = in.CPUCFSQuotaPeriod
+	out.CpuManagerPolicy = in.CpuManagerPolicy
 	return nil
 }
 

pkg/apis/kops/v1alpha2/componentconfig.go

@@ -187,6 +187,8 @@ type KubeletConfigSpec struct {
 	CPUCFSQuota *bool `json:"cpuCFSQuota,omitempty" flag:"cpu-cfs-quota"`
 	// CPUCFSQuotaPeriod sets CPU CFS quota period value, cpu.cfs_period_us, defaults to Linux Kernel default
 	CPUCFSQuotaPeriod *metav1.Duration `json:"cpuCFSQuotaPeriod,omitempty" flag:"cpu-cfs-quota-period"`
+	// CpuManagerPolicy allows for changing the default policy of None to static
+	CpuManagerPolicy string `json:"cpuManagerPolicy,omitempty" flag:"cpu-manager-policy"`
 }
 
 // KubeProxyConfig defines the configuration for a proxy

pkg/apis/kops/v1alpha2/zz_generated.conversion.go

@@ -3655,6 +3655,7 @@ func autoConvert_v1alpha2_KubeletConfigSpec_To_kops_KubeletConfigSpec(in *Kubele
 	out.AuthenticationTokenWebhookCacheTTL = in.AuthenticationTokenWebhookCacheTTL
 	out.CPUCFSQuota = in.CPUCFSQuota
 	out.CPUCFSQuotaPeriod = in.CPUCFSQuotaPeriod
+	out.CpuManagerPolicy = in.CpuManagerPolicy
 	return nil
 }
 
@@ -3736,6 +3737,7 @@ func autoConvert_kops_KubeletConfigSpec_To_v1alpha2_KubeletConfigSpec(in *kops.K
 	out.AuthenticationTokenWebhookCacheTTL = in.AuthenticationTokenWebhookCacheTTL
 	out.CPUCFSQuota = in.CPUCFSQuota
 	out.CPUCFSQuotaPeriod = in.CPUCFSQuotaPeriod
+	out.CpuManagerPolicy = in.CpuManagerPolicy
 	return nil
 }
 

pkg/model/awsmodel/api_loadbalancer.go

@@ -119,6 +119,13 @@ func (b *APILoadBalancerBuilder) Build(c *fi.ModelBuilderContext) error {
 			glog.V(1).Infof("WARNING: You are overwriting the Load Balancers, Security Group. When this is done you are responsible for ensure the correct rules!")
 		}
 
+		tags := b.CloudTags(loadBalancerName, false)
+		for k, v := range b.Cluster.Spec.CloudLabels {
+			tags[k] = v
+		}
+		// Override the returned name to be the expected ELB name
+		tags["Name"] = "api." + b.ClusterName()
+
 		elb = &awstasks.LoadBalancer{
 			Name:      fi.String("api." + b.ClusterName()),
 			Lifecycle: b.Lifecycle,
@@ -143,7 +150,7 @@ func (b *APILoadBalancerBuilder) Build(c *fi.ModelBuilderContext) error {
 				IdleTimeout: fi.Int64(int64(idleTimeout.Seconds())),
 			},
 
-			Tags: b.Cluster.Spec.CloudLabels,
+			Tags: tags,
 		}
 
 		switch lbSpec.Type {

pkg/model/bastion.go

@@ -208,6 +208,13 @@ func (b *BastionModelBuilder) Build(c *fi.ModelBuilderContext) error {
 			idleTimeout = time.Second * time.Duration(*b.Cluster.Spec.Topology.Bastion.IdleTimeoutSeconds)
 		}
 
+		tags := b.CloudTags(loadBalancerName, false)
+		for k, v := range b.Cluster.Spec.CloudLabels {
+			tags[k] = v
+		}
+		// Override the returned name to be the expected ELB name
+		tags["Name"] = "bastion." + b.ClusterName()
+
 		elb = &awstasks.LoadBalancer{
 			Name:      s("bastion." + b.ClusterName()),
 			Lifecycle: b.Lifecycle,
@@ -232,6 +239,8 @@ func (b *BastionModelBuilder) Build(c *fi.ModelBuilderContext) error {
 			ConnectionSettings: &awstasks.LoadBalancerConnectionSettings{
 				IdleTimeout: i64(int64(idleTimeout.Seconds())),
 			},
+
+			Tags: tags,
 		}
 
 		c.AddTask(elb)

tests/integration/update_cluster/bastionadditional_user-data/kubernetes.tf

@@ -273,6 +273,7 @@ resource "aws_elb" "api-bastionuserdata-example-com" {
   tags = {
     KubernetesCluster                                   = "bastionuserdata.example.com"
     Name                                                = "api.bastionuserdata.example.com"
+    "kubernetes.io/cluster/bastionuserdata.example.com" = "owned"
   }
 }
 
@@ -302,6 +303,7 @@ resource "aws_elb" "bastion-bastionuserdata-example-com" {
   tags = {
     KubernetesCluster                                   = "bastionuserdata.example.com"
     Name                                                = "bastion.bastionuserdata.example.com"
+    "kubernetes.io/cluster/bastionuserdata.example.com" = "owned"
   }
 }
 

tests/integration/update_cluster/complex/kubernetes.tf

@@ -230,6 +230,7 @@ resource "aws_elb" "api-complex-example-com" {
     Name                                        = "api.complex.example.com"
     Owner                                       = "John Doe"
     "foo/bar"                                   = "fib+baz"
+    "kubernetes.io/cluster/complex.example.com" = "owned"
   }
 }
 

tests/integration/update_cluster/existing_sg/kubernetes.tf

@@ -337,6 +337,7 @@ resource "aws_elb" "api-existingsg-example-com" {
   tags = {
     KubernetesCluster                              = "existingsg.example.com"
     Name                                           = "api.existingsg.example.com"
+    "kubernetes.io/cluster/existingsg.example.com" = "owned"
   }
 }
 

tests/integration/update_cluster/lifecycle_phases/loadbalancer-kubernetes.tf

@@ -135,6 +135,7 @@ resource "aws_elb" "bastion-lifecyclephases-example-com" {
   tags = {
     KubernetesCluster                                   = "lifecyclephases.example.com"
     Name                                                = "bastion.lifecyclephases.example.com"
+    "kubernetes.io/cluster/bastionuserdata.example.com" = "owned"
   }
 }
 

tests/integration/update_cluster/private-shared-subnet/kubernetes.tf

@@ -253,6 +253,7 @@ resource "aws_elb" "api-private-shared-subnet-example-com" {
   tags = {
     KubernetesCluster                                         = "private-shared-subnet.example.com"
     Name                                                      = "api.private-shared-subnet.example.com"
+    "kubernetes.io/cluster/private-shared-subnet.example.com" = "owned"
   }
 }
 
@@ -282,6 +283,7 @@ resource "aws_elb" "bastion-private-shared-subnet-example-com" {
   tags = {
     KubernetesCluster                                         = "private-shared-subnet.example.com"
     Name                                                      = "bastion.private-shared-subnet.example.com"
+    "kubernetes.io/cluster/private-shared-subnet.example.com" = "owned"
   }
 }
 

tests/integration/update_cluster/privatecalico/kubernetes.tf

@@ -273,6 +273,7 @@ resource "aws_elb" "api-privatecalico-example-com" {
   tags = {
     KubernetesCluster                                 = "privatecalico.example.com"
     Name                                              = "api.privatecalico.example.com"
+    "kubernetes.io/cluster/privatecalico.example.com" = "owned"
   }
 }
 
@@ -302,6 +303,7 @@ resource "aws_elb" "bastion-privatecalico-example-com" {
   tags = {
     KubernetesCluster                                 = "privatecalico.example.com"
     Name                                              = "bastion.privatecalico.example.com"
+    "kubernetes.io/cluster/privatecalico.example.com" = "owned"
   }
 }
 

tests/integration/update_cluster/privatecanal/kubernetes.tf

@@ -273,6 +273,7 @@ resource "aws_elb" "api-privatecanal-example-com" {
   tags = {
     KubernetesCluster                                = "privatecanal.example.com"
     Name                                             = "api.privatecanal.example.com"
+    "kubernetes.io/cluster/privatecanal.example.com" = "owned"
   }
 }
 
@@ -302,6 +303,7 @@ resource "aws_elb" "bastion-privatecanal-example-com" {
   tags = {
     KubernetesCluster                                = "privatecanal.example.com"
     Name                                             = "bastion.privatecanal.example.com"
+    "kubernetes.io/cluster/privatecanal.example.com" = "owned"
   }
 }
 

tests/integration/update_cluster/privatedns1/kubernetes.tf

@@ -273,6 +273,7 @@ resource "aws_elb" "api-privatedns1-example-com" {
   tags = {
     KubernetesCluster                               = "privatedns1.example.com"
     Name                                            = "api.privatedns1.example.com"
+    "kubernetes.io/cluster/privatedns1.example.com" = "owned"
   }
 }
 
@@ -302,6 +303,7 @@ resource "aws_elb" "bastion-privatedns1-example-com" {
   tags = {
     KubernetesCluster                               = "privatedns1.example.com"
     Name                                            = "bastion.privatedns1.example.com"
+    "kubernetes.io/cluster/privatedns1.example.com" = "owned"
   }
 }
 

tests/integration/update_cluster/privatedns2/kubernetes.tf

@@ -268,6 +268,7 @@ resource "aws_elb" "api-privatedns2-example-com" {
   tags = {
     KubernetesCluster                               = "privatedns2.example.com"
     Name                                            = "api.privatedns2.example.com"
+    "kubernetes.io/cluster/privatedns2.example.com" = "owned"
   }
 }
 
@@ -297,6 +298,7 @@ resource "aws_elb" "bastion-privatedns2-example-com" {
   tags = {
     KubernetesCluster                               = "privatedns2.example.com"
     Name                                            = "bastion.privatedns2.example.com"
+    "kubernetes.io/cluster/privatedns2.example.com" = "owned"
   }
 }
 

tests/integration/update_cluster/privateflannel/kubernetes.tf

@@ -273,6 +273,7 @@ resource "aws_elb" "api-privateflannel-example-com" {
   tags = {
     KubernetesCluster                                  = "privateflannel.example.com"
     Name                                               = "api.privateflannel.example.com"
+    "kubernetes.io/cluster/privateflannel.example.com" = "owned"
   }
 }
 
@@ -302,6 +303,7 @@ resource "aws_elb" "bastion-privateflannel-example-com" {
   tags = {
     KubernetesCluster                                  = "privateflannel.example.com"
     Name                                               = "bastion.privateflannel.example.com"
+    "kubernetes.io/cluster/privateflannel.example.com" = "owned"
   }
 }
 

tests/integration/update_cluster/privatekopeio/kubernetes.tf

@@ -278,6 +278,7 @@ resource "aws_elb" "api-privatekopeio-example-com" {
   tags = {
     KubernetesCluster                                 = "privatekopeio.example.com"
     Name                                              = "api.privatekopeio.example.com"
+    "kubernetes.io/cluster/privatekopeio.example.com" = "owned"
   }
 }
 
@@ -307,6 +308,7 @@ resource "aws_elb" "bastion-privatekopeio-example-com" {
   tags = {
     KubernetesCluster                                 = "privatekopeio.example.com"
     Name                                              = "bastion.privatekopeio.example.com"
+    "kubernetes.io/cluster/privatekopeio.example.com" = "owned"
   }
 }
 

tests/integration/update_cluster/privateweave/kubernetes.tf

@@ -273,6 +273,7 @@ resource "aws_elb" "api-privateweave-example-com" {
   tags = {
     KubernetesCluster                                = "privateweave.example.com"
     Name                                             = "api.privateweave.example.com"
+    "kubernetes.io/cluster/privateweave.example.com" = "owned"
   }
 }
 
@@ -302,6 +303,7 @@ resource "aws_elb" "bastion-privateweave-example-com" {
   tags = {
     KubernetesCluster                                = "privateweave.example.com"
     Name                                             = "bastion.privateweave.example.com"
+    "kubernetes.io/cluster/privateweave.example.com" = "owned"
   }
 }
 

tests/integration/update_cluster/unmanaged/kubernetes.tf

@@ -258,6 +258,7 @@ resource "aws_elb" "api-unmanaged-example-com" {
   tags = {
     KubernetesCluster                             = "unmanaged.example.com"
     Name                                          = "api.unmanaged.example.com"
+    "kubernetes.io/cluster/unmanaged.example.com" = "owned"
   }
 }
 
@@ -287,6 +288,7 @@ resource "aws_elb" "bastion-unmanaged-example-com" {
   tags = {
     KubernetesCluster                             = "unmanaged.example.com"
     Name                                          = "bastion.unmanaged.example.com"
+    "kubernetes.io/cluster/unmanaged.example.com" = "owned"
   }
 }
 

upup/pkg/fi/cloudup/awstasks/load_balancer.go

@@ -297,6 +297,15 @@ func (e *LoadBalancer) Find(c *fi.Context) (*LoadBalancer, error) {
 	actual.Scheme = lb.Scheme
 	actual.Lifecycle = e.Lifecycle
 
+	tagMap, err := describeLoadBalancerTags(cloud, []string{*lb.LoadBalancerName})
+	if err != nil {
+		return nil, err
+	}
+	actual.Tags = make(map[string]string)
+	for _, tag := range tagMap[*e.LoadBalancerName] {
+		actual.Tags[aws.StringValue(tag.Key)] = aws.StringValue(tag.Value)
+	}
+
 	for _, subnet := range lb.Subnets {
 		actual.Subnets = append(actual.Subnets, &Subnet{ID: subnet})
 	}
@@ -603,12 +612,11 @@ func (_ *LoadBalancer) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *LoadBalan
 		}
 	}
 
-	var tags map[string]string = t.Cloud.BuildTags(e.Name)
+	if err := t.AddELBTags(loadBalancerName, e.Tags); err != nil {
-	for k, v := range e.Tags {
+		return err
-		tags[k] = v
 	}
 
-	if err := t.AddELBTags(loadBalancerName, tags); err != nil {
+	if err := t.RemoveELBTags(loadBalancerName, e.Tags); err != nil {
 		return err
 	}
 

upup/pkg/fi/cloudup/awsup/aws_apitarget.go

@@ -80,6 +80,32 @@ func (t *AWSAPITarget) AddELBTags(loadBalancerName string, expected map[string]s
 	return nil
 }
 
+func (t *AWSAPITarget) RemoveELBTags(loadBalancerName string, expected map[string]string) error {
+	actual, err := t.Cloud.GetELBTags(loadBalancerName)
+	if err != nil {
+		return fmt.Errorf("unexpected error fetching tags for resource: %v", err)
+	}
+
+	extra := map[string]string{}
+	for k, v := range actual {
+		expectedValue, found := expected[k]
+		if found && expectedValue == v {
+			continue
+		}
+		extra[k] = v
+	}
+
+	if len(extra) != 0 {
+		glog.V(4).Infof("removing tags from %q: %v", loadBalancerName, extra)
+		err := t.Cloud.RemoveELBTags(loadBalancerName, extra)
+		if err != nil {
+			return fmt.Errorf("error removing tags from ELB %q: %v", loadBalancerName, err)
+		}
+	}
+
+	return nil
+}
+
 func (t *AWSAPITarget) WaitForInstanceRunning(instanceID string) error {
 	attempt := 0
 	for {

upup/pkg/fi/cloudup/awsup/aws_cloud.go

@@ -123,6 +123,8 @@ type AWSCloud interface {
 
 	// CreateELBTags will add tags to the specified loadBalancer, retrying up to MaxCreateTagsAttempts times if it hits an eventual-consistency type error
 	CreateELBTags(loadBalancerName string, tags map[string]string) error
+	// RemoveELBTags will remove tags from the specified loadBalancer, retrying up to MaxCreateTagsAttempts times if it hits an eventual-consistency type error
+	RemoveELBTags(loadBalancerName string, tags map[string]string) error
 
 	// DeleteTags will delete tags from the specified resource, retrying up to MaxCreateTagsAttempts times if it hits an eventual-consistency type error
 	DeleteTags(id string, tags map[string]string) error
@@ -912,6 +914,39 @@ func createELBTags(c AWSCloud, loadBalancerName string, tags map[string]string)
 	}
 }
 
+// RemoveELBTags will remove tags to the specified loadBalancer, retrying up to MaxCreateTagsAttempts times if it hits an eventual-consistency type error
+func (c *awsCloudImplementation) RemoveELBTags(loadBalancerName string, tags map[string]string) error {
+	return removeELBTags(c, loadBalancerName, tags)
+}
+
+func removeELBTags(c AWSCloud, loadBalancerName string, tags map[string]string) error {
+	if len(tags) == 0 {
+		return nil
+	}
+
+	elbTagKeysOnly := []*elb.TagKeyOnly{}
+	for k := range tags {
+		elbTagKeysOnly = append(elbTagKeysOnly, &elb.TagKeyOnly{Key: aws.String(k)})
+	}
+
+	attempt := 0
+	for {
+		attempt++
+
+		request := &elb.RemoveTagsInput{
+			Tags:              elbTagKeysOnly,
+			LoadBalancerNames: []*string{&loadBalancerName},
+		}
+
+		_, err := c.ELB().RemoveTags(request)
+		if err != nil {
+			return fmt.Errorf("error creating tags on %v: %v", loadBalancerName, err)
+		}
+
+		return nil
+	}
+}
+
 func (c *awsCloudImplementation) GetELBV2Tags(ResourceArn string) (map[string]string, error) {
 	return getELBV2Tags(c, ResourceArn)
 }

upup/pkg/fi/cloudup/awsup/mock_aws_cloud.go

@@ -162,6 +162,10 @@ func (c *MockAWSCloud) CreateELBTags(loadBalancerName string, tags map[string]st
 	return createELBTags(c, loadBalancerName, tags)
 }
 
+func (c *MockAWSCloud) RemoveELBTags(loadBalancerName string, tags map[string]string) error {
+	return removeELBTags(c, loadBalancerName, tags)
+}
+
 func (c *MockAWSCloud) GetELBV2Tags(ResourceArn string) (map[string]string, error) {
 	return getELBV2Tags(c, ResourceArn)
 }