Merge pull request #10022 from olemarkus/metrics-server

Kubelet serving certificate and metrics server addon

addons/metrics-server/README.md

@@ -1,5 +1,7 @@
 # Kubernetes Metrics Server
 
+**This addon is deprecated. Set `spec.metricsServer.enabled: true` instead**
+
 ## User guide
 
 You can find the user guide in

cmd/kops-controller/pkg/server/server.go

@@ -175,6 +175,12 @@ func (s *Server) issueCert(name string, pubKey string, id *fi.VerifyResult, vali
 			CommonName:   fmt.Sprintf("system:node:%s", id.NodeName),
 			Organization: []string{rbac.NodesGroup},
 		}
+	case "kubelet-server":
+		issueReq.Subject = pkix.Name{
+			CommonName: id.NodeName,
+		}
+		issueReq.AlternateNames = []string{id.NodeName}
+		issueReq.Type = "server"
 	case "kube-proxy":
 		issueReq.Subject = pkix.Name{
 			CommonName: rbac.KubeProxy,

docs/releases/1.19-NOTES.md

@@ -42,6 +42,8 @@ The expiration times vary randomly so that nodes are likely to have their certs
 
 * New command for deleting a single instance: [kops delete instance](/docs/cli/kops_delete_instance/)
 
+* Metrics Server is now available as a configurable addon. Add `spec.metricsServer.enabled: true` to the cluster spec to enable.
+
 # Breaking changes
 
 * Support for Kubernetes 1.9 and 1.10 has been removed.
@@ -62,6 +64,8 @@ The expiration times vary randomly so that nodes are likely to have their certs
 
 * Support for feature flag `Terraform-0.12` has been deprecated and will be removed in kops 1.20. All generated Terraform HCL2/JSON files will support versions `0.12.26+` and `0.13.0+`.
 
+* The [manifest based metrics server addon](https://github.com/kubernetes/kops/tree/master/addons/metrics-server) has been deprecated in favour of a configurable addon.
+
 # Full change list since 1.18.0 release
 
 ## v1.18.0-alpha.3 to v1.19.0-alpha.1

k8s/crds/kops.k8s.io_clusters.yaml

@@ -2088,6 +2088,16 @@ spec:
               masterPublicName:
                 description: MasterPublicName is the external DNS name for the master nodes
                 type: string
+              metricsServer:
+                description: MetricsServerConfig determines the metrics server configuration.
+                properties:
+                  enabled:
+                    description: 'Enabled enables the metrics server. Default: false'
+                    type: boolean
+                  image:
+                    description: 'Image is the docker container used. Default: the latest supported image for the specified kubernetes version.'
+                    type: string
+                type: object
               networkCIDR:
                 description: NetworkCIDR is the CIDR used for the AWS VPC / GCE Network, or otherwise allocated to k8s This is a real CIDR, not the internal k8s network On AWS, it maps to the VPC CIDR.  It is not required on GCE.
                 type: string

nodeup/pkg/model/kubelet.go

@@ -56,6 +56,12 @@ var _ fi.ModelBuilder = &KubeletBuilder{}
 
 // Build is responsible for building the kubelet configuration
 func (b *KubeletBuilder) Build(c *fi.ModelBuilderContext) error {
+
+	err := b.buildKubeletServingCertificate(c)
+	if err != nil {
+		return fmt.Errorf("error building kubelet server cert: %v", err)
+	}
+
 	kubeletConfig, err := b.buildKubeletConfig()
 	if err != nil {
 		return fmt.Errorf("error building kubelet config: %v", err)
@@ -226,6 +232,11 @@ func (b *KubeletBuilder) buildSystemdEnvironmentFile(kubeletConfig *kops.Kubelet
 		}
 	}
 
+	if b.UseKopsControllerForNodeBootstrap() {
+		flags += " --tls-cert-file=" + b.PathSrvKubernetes() + "/kubelet-server.crt"
+		flags += " --tls-private-key-file=" + b.PathSrvKubernetes() + "/kubelet-server.key"
+	}
+
 	sysconfig := "DAEMON_ARGS=\"" + flags + "\"\n"
 	// Makes kubelet read /root/.docker/config.json properly
 	sysconfig = sysconfig + "HOME=\"/root" + "\"\n"
@@ -538,3 +549,50 @@ func (b *KubeletBuilder) buildMasterKubeletKubeconfig(c *fi.ModelBuilderContext)
 
 	return b.BuildIssuedKubeconfig("kubelet", certName, c), nil
 }
+
+func (b *KubeletBuilder) buildKubeletServingCertificate(c *fi.ModelBuilderContext) error {
+
+	if b.UseKopsControllerForNodeBootstrap() {
+		name := "kubelet-server"
+		dir := b.PathSrvKubernetes()
+		signer := fi.CertificateIDCA
+
+		nodeName, err := b.NodeName()
+		if err != nil {
+			return err
+		}
+
+		if !b.IsMaster {
+			cert, key := b.GetBootstrapCert(name)
+
+			c.AddTask(&nodetasks.File{
+				Path:     filepath.Join(dir, name+".crt"),
+				Contents: cert,
+				Type:     nodetasks.FileType_File,
+				Mode:     fi.String("0644"),
+			})
+
+			c.AddTask(&nodetasks.File{
+				Path:     filepath.Join(dir, name+".key"),
+				Contents: key,
+				Type:     nodetasks.FileType_File,
+				Mode:     fi.String("0400"),
+			})
+
+		} else {
+			issueCert := &nodetasks.IssueCert{
+				Name:   name,
+				Signer: signer,
+				Type:   "server",
+				Subject: nodetasks.PKIXName{
+					CommonName: nodeName,
+				},
+				AlternateNames: []string{nodeName},
+			}
+			c.AddTask(issueCert)
+			return issueCert.AddFileTasks(c, dir, name, "", nil)
+		}
+	}
+	return nil
+
+}

pkg/apis/kops/cluster.go

@@ -160,6 +160,8 @@ type ClusterSpec struct {
 
 	// NodeTerminationHandlerConfig determines the cluster autoscaler configuration.
 	NodeTerminationHandler *NodeTerminationHandlerConfig `json:"nodeTerminationHandler,omitempty"`
+	// MetricsServerConfig determines the metrics server configuration.
+	MetricsServer *MetricsServerConfig `json:"metricsServer,omitempty"`
 
 	// Networking configuration
 	Networking *NetworkingSpec `json:"networking,omitempty"`

pkg/apis/kops/componentconfig.go

@@ -803,6 +803,16 @@ type ClusterAutoscalerConfig struct {
 	Image *string `json:"image,omitempty"`
 }
 
+// MetricsServerConfig determines the metrics server configuration.
+type MetricsServerConfig struct {
+	// Enabled enables the metrics server.
+	// Default: false
+	Enabled *bool `json:"enabled,omitempty"`
+	// Image is the docker container used.
+	// Default: the latest supported image for the specified kubernetes version.
+	Image *string `json:"image,omitempty"`
+}
+
 // HasAdmissionController checks if a specific admission controller is enabled
 func (c *KubeAPIServerConfig) HasAdmissionController(name string) bool {
 	for _, x := range c.AdmissionControl {

pkg/apis/kops/v1alpha2/cluster.go

@@ -159,6 +159,8 @@ type ClusterSpec struct {
 
 	// NodeTerminationHandlerConfig determines the cluster autoscaler configuration.
 	NodeTerminationHandler *NodeTerminationHandlerConfig `json:"nodeTerminationHandler,omitempty"`
+	// MetricsServerConfig determines the metrics server configuration.
+	MetricsServer *MetricsServerConfig `json:"metricsServer,omitempty"`
 
 	// Networking configuration
 	Networking *NetworkingSpec `json:"networking,omitempty"`

pkg/apis/kops/v1alpha2/componentconfig.go

@@ -804,6 +804,16 @@ type ClusterAutoscalerConfig struct {
 	Image *string `json:"image,omitempty"`
 }
 
+// MetricsServerConfig determines the metrics server configuration.
+type MetricsServerConfig struct {
+	// Enabled enables the metrics server.
+	// Default: false
+	Enabled *bool `json:"enabled,omitempty"`
+	// Image is the docker container used.
+	// Default: the latest supported image for the specified kubernetes version.
+	Image *string `json:"image,omitempty"`
+}
+
 // HasAdmissionController checks if a specific admission controller is enabled
 func (c *KubeAPIServerConfig) HasAdmissionController(name string) bool {
 	for _, x := range c.AdmissionControl {

pkg/apis/kops/v1alpha2/zz_generated.conversion.go

@@ -693,6 +693,16 @@ func RegisterConversions(s *runtime.Scheme) error {
 	}); err != nil {
 		return err
 	}
+	if err := s.AddGeneratedConversionFunc((*MetricsServerConfig)(nil), (*kops.MetricsServerConfig)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_v1alpha2_MetricsServerConfig_To_kops_MetricsServerConfig(a.(*MetricsServerConfig), b.(*kops.MetricsServerConfig), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*kops.MetricsServerConfig)(nil), (*MetricsServerConfig)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_kops_MetricsServerConfig_To_v1alpha2_MetricsServerConfig(a.(*kops.MetricsServerConfig), b.(*MetricsServerConfig), scope)
+	}); err != nil {
+		return err
+	}
 	if err := s.AddGeneratedConversionFunc((*MixedInstancesPolicySpec)(nil), (*kops.MixedInstancesPolicySpec)(nil), func(a, b interface{}, scope conversion.Scope) error {
 		return Convert_v1alpha2_MixedInstancesPolicySpec_To_kops_MixedInstancesPolicySpec(a.(*MixedInstancesPolicySpec), b.(*kops.MixedInstancesPolicySpec), scope)
 	}); err != nil {
@@ -2066,6 +2076,15 @@ func autoConvert_v1alpha2_ClusterSpec_To_kops_ClusterSpec(in *ClusterSpec, out *
 	} else {
 		out.NodeTerminationHandler = nil
 	}
+	if in.MetricsServer != nil {
+		in, out := &in.MetricsServer, &out.MetricsServer
+		*out = new(kops.MetricsServerConfig)
+		if err := Convert_v1alpha2_MetricsServerConfig_To_kops_MetricsServerConfig(*in, *out, s); err != nil {
+			return err
+		}
+	} else {
+		out.MetricsServer = nil
+	}
 	if in.Networking != nil {
 		in, out := &in.Networking, &out.Networking
 		*out = new(kops.NetworkingSpec)
@@ -2406,6 +2425,15 @@ func autoConvert_kops_ClusterSpec_To_v1alpha2_ClusterSpec(in *kops.ClusterSpec,
 	} else {
 		out.NodeTerminationHandler = nil
 	}
+	if in.MetricsServer != nil {
+		in, out := &in.MetricsServer, &out.MetricsServer
+		*out = new(MetricsServerConfig)
+		if err := Convert_kops_MetricsServerConfig_To_v1alpha2_MetricsServerConfig(*in, *out, s); err != nil {
+			return err
+		}
+	} else {
+		out.MetricsServer = nil
+	}
 	if in.Networking != nil {
 		in, out := &in.Networking, &out.Networking
 		*out = new(NetworkingSpec)
@@ -4686,6 +4714,28 @@ func Convert_kops_LyftVPCNetworkingSpec_To_v1alpha2_LyftVPCNetworkingSpec(in *ko
 	return autoConvert_kops_LyftVPCNetworkingSpec_To_v1alpha2_LyftVPCNetworkingSpec(in, out, s)
 }
 
+func autoConvert_v1alpha2_MetricsServerConfig_To_kops_MetricsServerConfig(in *MetricsServerConfig, out *kops.MetricsServerConfig, s conversion.Scope) error {
+	out.Enabled = in.Enabled
+	out.Image = in.Image
+	return nil
+}
+
+// Convert_v1alpha2_MetricsServerConfig_To_kops_MetricsServerConfig is an autogenerated conversion function.
+func Convert_v1alpha2_MetricsServerConfig_To_kops_MetricsServerConfig(in *MetricsServerConfig, out *kops.MetricsServerConfig, s conversion.Scope) error {
+	return autoConvert_v1alpha2_MetricsServerConfig_To_kops_MetricsServerConfig(in, out, s)
+}
+
+func autoConvert_kops_MetricsServerConfig_To_v1alpha2_MetricsServerConfig(in *kops.MetricsServerConfig, out *MetricsServerConfig, s conversion.Scope) error {
+	out.Enabled = in.Enabled
+	out.Image = in.Image
+	return nil
+}
+
+// Convert_kops_MetricsServerConfig_To_v1alpha2_MetricsServerConfig is an autogenerated conversion function.
+func Convert_kops_MetricsServerConfig_To_v1alpha2_MetricsServerConfig(in *kops.MetricsServerConfig, out *MetricsServerConfig, s conversion.Scope) error {
+	return autoConvert_kops_MetricsServerConfig_To_v1alpha2_MetricsServerConfig(in, out, s)
+}
+
 func autoConvert_v1alpha2_MixedInstancesPolicySpec_To_kops_MixedInstancesPolicySpec(in *MixedInstancesPolicySpec, out *kops.MixedInstancesPolicySpec, s conversion.Scope) error {
 	out.Instances = in.Instances
 	out.OnDemandAllocationStrategy = in.OnDemandAllocationStrategy

pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go

@@ -845,6 +845,11 @@ func (in *ClusterSpec) DeepCopyInto(out *ClusterSpec) {
 		*out = new(NodeTerminationHandlerConfig)
 		(*in).DeepCopyInto(*out)
 	}
+	if in.MetricsServer != nil {
+		in, out := &in.MetricsServer, &out.MetricsServer
+		*out = new(MetricsServerConfig)
+		(*in).DeepCopyInto(*out)
+	}
 	if in.Networking != nil {
 		in, out := &in.Networking, &out.Networking
 		*out = new(NetworkingSpec)
@@ -3169,6 +3174,32 @@ func (in *LyftVPCNetworkingSpec) DeepCopy() *LyftVPCNetworkingSpec {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *MetricsServerConfig) DeepCopyInto(out *MetricsServerConfig) {
+	*out = *in
+	if in.Enabled != nil {
+		in, out := &in.Enabled, &out.Enabled
+		*out = new(bool)
+		**out = **in
+	}
+	if in.Image != nil {
+		in, out := &in.Image, &out.Image
+		*out = new(string)
+		**out = **in
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MetricsServerConfig.
+func (in *MetricsServerConfig) DeepCopy() *MetricsServerConfig {
+	if in == nil {
+		return nil
+	}
+	out := new(MetricsServerConfig)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *MixedInstancesPolicySpec) DeepCopyInto(out *MixedInstancesPolicySpec) {
 	*out = *in

pkg/apis/kops/zz_generated.deepcopy.go

@@ -945,6 +945,11 @@ func (in *ClusterSpec) DeepCopyInto(out *ClusterSpec) {
 		*out = new(NodeTerminationHandlerConfig)
 		(*in).DeepCopyInto(*out)
 	}
+	if in.MetricsServer != nil {
+		in, out := &in.MetricsServer, &out.MetricsServer
+		*out = new(MetricsServerConfig)
+		(*in).DeepCopyInto(*out)
+	}
 	if in.Networking != nil {
 		in, out := &in.Networking, &out.Networking
 		*out = new(NetworkingSpec)
@@ -3367,6 +3372,32 @@ func (in *LyftVPCNetworkingSpec) DeepCopy() *LyftVPCNetworkingSpec {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *MetricsServerConfig) DeepCopyInto(out *MetricsServerConfig) {
+	*out = *in
+	if in.Enabled != nil {
+		in, out := &in.Enabled, &out.Enabled
+		*out = new(bool)
+		**out = **in
+	}
+	if in.Image != nil {
+		in, out := &in.Image, &out.Image
+		*out = new(string)
+		**out = **in
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MetricsServerConfig.
+func (in *MetricsServerConfig) DeepCopy() *MetricsServerConfig {
+	if in == nil {
+		return nil
+	}
+	out := new(MetricsServerConfig)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *MixedInstancesPolicySpec) DeepCopyInto(out *MixedInstancesPolicySpec) {
 	*out = *in

upup/models/bindata.go

@@ -27,6 +27,7 @@
 // upup/models/cloudup/resources/addons/limit-range.addons.k8s.io/v1.5.0.yaml
 // upup/models/cloudup/resources/addons/metadata-proxy.addons.k8s.io/addon.yaml
 // upup/models/cloudup/resources/addons/metadata-proxy.addons.k8s.io/v0.1.12.yaml
+// upup/models/cloudup/resources/addons/metrics-server.addons.k8s.io/k8s-1.11.yaml.template
 // upup/models/cloudup/resources/addons/networking.amazon-vpc-routed-eni/k8s-1.10.yaml.template
 // upup/models/cloudup/resources/addons/networking.amazon-vpc-routed-eni/k8s-1.12.yaml.template
 // upup/models/cloudup/resources/addons/networking.amazon-vpc-routed-eni/k8s-1.16.yaml.template
@@ -3838,6 +3839,188 @@ func cloudupResourcesAddonsMetadataProxyAddonsK8sIoV0112Yaml() (*asset, error) {
 	return a, nil
 }
 
+var _cloudupResourcesAddonsMetricsServerAddonsK8sIoK8s111YamlTemplate = []byte(`# sourced from https://github.com/kubernetes-sigs/metrics-server/releases/download/v0.3.7/components.yaml
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: system:aggregated-metrics-reader
+  labels:
+    rbac.authorization.k8s.io/aggregate-to-view: "true"
+    rbac.authorization.k8s.io/aggregate-to-edit: "true"
+    rbac.authorization.k8s.io/aggregate-to-admin: "true"
+rules:
+- apiGroups: ["metrics.k8s.io"]
+  resources: ["pods", "nodes"]
+  verbs: ["get", "list", "watch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: metrics-server:system:auth-delegator
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:auth-delegator
+subjects:
+- kind: ServiceAccount
+  name: metrics-server
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: metrics-server-auth-reader
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: extension-apiserver-authentication-reader
+subjects:
+- kind: ServiceAccount
+  name: metrics-server
+  namespace: kube-system
+---
+apiVersion: apiregistration.k8s.io/v1beta1
+kind: APIService
+metadata:
+  name: v1beta1.metrics.k8s.io
+spec:
+  service:
+    name: metrics-server
+    namespace: kube-system
+  group: metrics.k8s.io
+  version: v1beta1
+  insecureSkipTLSVerify: true
+  groupPriorityMinimum: 100
+  versionPriority: 100
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: metrics-server
+  namespace: kube-system
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: metrics-server
+  namespace: kube-system
+  labels:
+    k8s-app: metrics-server
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      k8s-app: metrics-server
+  template:
+    metadata:
+      name: metrics-server
+      labels:
+        k8s-app: metrics-server
+    spec:
+      serviceAccountName: metrics-server
+      volumes:
+      # mount in tmp so we can safely use from-scratch images and/or read-only containers
+      - name: tmp-dir
+        emptyDir: {}
+      containers:
+      - name: metrics-server
+        image: {{ or .MetricsServer.Image "k8s.gcr.io/metrics-server/metrics-server:v0.3.7" }}
+        imagePullPolicy: IfNotPresent
+        args:
+          - --cert-dir=/tmp
+          - --secure-port=4443
+        {{ if not UseKopsControllerForNodeBootstrap }}
+          - --kubelet-insecure-tls
+        {{ end }}
+        ports:
+        - name: main-port
+          containerPort: 4443
+          protocol: TCP
+        securityContext:
+          readOnlyRootFilesystem: true
+          runAsNonRoot: true
+          runAsUser: 1000
+        volumeMounts:
+        - name: tmp-dir
+          mountPath: /tmp
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: metrics-server
+  namespace: kube-system
+  labels:
+    kubernetes.io/name: "Metrics-server"
+    kubernetes.io/cluster-service: "true"
+spec:
+  selector:
+    k8s-app: metrics-server
+  ports:
+  - port: 443
+    protocol: TCP
+    targetPort: main-port
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: system:metrics-server
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  - nodes
+  - nodes/stats
+  - namespaces
+  - configmaps
+  verbs:
+  - get
+  - list
+  - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: system:metrics-server
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:metrics-server
+subjects:
+- kind: ServiceAccount
+  name: metrics-server
+  namespace: kube-system
+---
+apiVersion: policy/v1beta1
+kind: PodDisruptionBudget
+metadata:
+  name: metrics-server
+  namespace: kube-system
+  labels:
+    k8s-app: metrics-server
+spec:
+  minAvailable: 1
+  selector:
+    matchLabels:
+      k8s-app: metrics-server`)
+
+func cloudupResourcesAddonsMetricsServerAddonsK8sIoK8s111YamlTemplateBytes() ([]byte, error) {
+	return _cloudupResourcesAddonsMetricsServerAddonsK8sIoK8s111YamlTemplate, nil
+}
+
+func cloudupResourcesAddonsMetricsServerAddonsK8sIoK8s111YamlTemplate() (*asset, error) {
+	bytes, err := cloudupResourcesAddonsMetricsServerAddonsK8sIoK8s111YamlTemplateBytes()
+	if err != nil {
+		return nil, err
+	}
+
+	info := bindataFileInfo{name: "cloudup/resources/addons/metrics-server.addons.k8s.io/k8s-1.11.yaml.template", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)}
+	a := &asset{bytes: bytes, info: info}
+	return a, nil
+}
+
 var _cloudupResourcesAddonsNetworkingAmazonVpcRoutedEniK8s110YamlTemplate = []byte(`# Vendored from https://github.com/aws/amazon-vpc-cni-k8s/blob/v1.3.3/config/v1.3/aws-k8s-cni.yaml
 
 apiVersion: rbac.authorization.k8s.io/v1
@@ -20688,6 +20871,7 @@ var _bindata = map[string]func() (*asset, error){
 	"cloudup/resources/addons/limit-range.addons.k8s.io/v1.5.0.yaml":                                      cloudupResourcesAddonsLimitRangeAddonsK8sIoV150Yaml,
 	"cloudup/resources/addons/metadata-proxy.addons.k8s.io/addon.yaml":                                    cloudupResourcesAddonsMetadataProxyAddonsK8sIoAddonYaml,
 	"cloudup/resources/addons/metadata-proxy.addons.k8s.io/v0.1.12.yaml":                                  cloudupResourcesAddonsMetadataProxyAddonsK8sIoV0112Yaml,
+	"cloudup/resources/addons/metrics-server.addons.k8s.io/k8s-1.11.yaml.template":                        cloudupResourcesAddonsMetricsServerAddonsK8sIoK8s111YamlTemplate,
 	"cloudup/resources/addons/networking.amazon-vpc-routed-eni/k8s-1.10.yaml.template":                    cloudupResourcesAddonsNetworkingAmazonVpcRoutedEniK8s110YamlTemplate,
 	"cloudup/resources/addons/networking.amazon-vpc-routed-eni/k8s-1.12.yaml.template":                    cloudupResourcesAddonsNetworkingAmazonVpcRoutedEniK8s112YamlTemplate,
 	"cloudup/resources/addons/networking.amazon-vpc-routed-eni/k8s-1.16.yaml.template":                    cloudupResourcesAddonsNetworkingAmazonVpcRoutedEniK8s116YamlTemplate,
@@ -20826,6 +21010,9 @@ var _bintree = &bintree{nil, map[string]*bintree{
 					"addon.yaml":   {cloudupResourcesAddonsMetadataProxyAddonsK8sIoAddonYaml, map[string]*bintree{}},
 					"v0.1.12.yaml": {cloudupResourcesAddonsMetadataProxyAddonsK8sIoV0112Yaml, map[string]*bintree{}},
 				}},
+				"metrics-server.addons.k8s.io": {nil, map[string]*bintree{
+					"k8s-1.11.yaml.template": {cloudupResourcesAddonsMetricsServerAddonsK8sIoK8s111YamlTemplate, map[string]*bintree{}},
+				}},
 				"networking.amazon-vpc-routed-eni": {nil, map[string]*bintree{
 					"k8s-1.10.yaml.template": {cloudupResourcesAddonsNetworkingAmazonVpcRoutedEniK8s110YamlTemplate, map[string]*bintree{}},
 					"k8s-1.12.yaml.template": {cloudupResourcesAddonsNetworkingAmazonVpcRoutedEniK8s112YamlTemplate, map[string]*bintree{}},

upup/models/cloudup/resources/addons/metrics-server.addons.k8s.io/k8s-1.11.yaml.template

@@ -0,0 +1,166 @@
+# sourced from https://github.com/kubernetes-sigs/metrics-server/releases/download/v0.3.7/components.yaml
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: system:aggregated-metrics-reader
+  labels:
+    rbac.authorization.k8s.io/aggregate-to-view: "true"
+    rbac.authorization.k8s.io/aggregate-to-edit: "true"
+    rbac.authorization.k8s.io/aggregate-to-admin: "true"
+rules:
+- apiGroups: ["metrics.k8s.io"]
+  resources: ["pods", "nodes"]
+  verbs: ["get", "list", "watch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: metrics-server:system:auth-delegator
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:auth-delegator
+subjects:
+- kind: ServiceAccount
+  name: metrics-server
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: metrics-server-auth-reader
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: extension-apiserver-authentication-reader
+subjects:
+- kind: ServiceAccount
+  name: metrics-server
+  namespace: kube-system
+---
+apiVersion: apiregistration.k8s.io/v1beta1
+kind: APIService
+metadata:
+  name: v1beta1.metrics.k8s.io
+spec:
+  service:
+    name: metrics-server
+    namespace: kube-system
+  group: metrics.k8s.io
+  version: v1beta1
+  insecureSkipTLSVerify: true
+  groupPriorityMinimum: 100
+  versionPriority: 100
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: metrics-server
+  namespace: kube-system
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: metrics-server
+  namespace: kube-system
+  labels:
+    k8s-app: metrics-server
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      k8s-app: metrics-server
+  template:
+    metadata:
+      name: metrics-server
+      labels:
+        k8s-app: metrics-server
+    spec:
+      serviceAccountName: metrics-server
+      volumes:
+      # mount in tmp so we can safely use from-scratch images and/or read-only containers
+      - name: tmp-dir
+        emptyDir: {}
+      containers:
+      - name: metrics-server
+        image: {{ or .MetricsServer.Image "k8s.gcr.io/metrics-server/metrics-server:v0.3.7" }}
+        imagePullPolicy: IfNotPresent
+        args:
+          - --cert-dir=/tmp
+          - --secure-port=4443
+        {{ if not UseKopsControllerForNodeBootstrap }}
+          - --kubelet-insecure-tls
+        {{ end }}
+        ports:
+        - name: main-port
+          containerPort: 4443
+          protocol: TCP
+        securityContext:
+          readOnlyRootFilesystem: true
+          runAsNonRoot: true
+          runAsUser: 1000
+        volumeMounts:
+        - name: tmp-dir
+          mountPath: /tmp
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: metrics-server
+  namespace: kube-system
+  labels:
+    kubernetes.io/name: "Metrics-server"
+    kubernetes.io/cluster-service: "true"
+spec:
+  selector:
+    k8s-app: metrics-server
+  ports:
+  - port: 443
+    protocol: TCP
+    targetPort: main-port
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: system:metrics-server
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  - nodes
+  - nodes/stats
+  - namespaces
+  - configmaps
+  verbs:
+  - get
+  - list
+  - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: system:metrics-server
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:metrics-server
+subjects:
+- kind: ServiceAccount
+  name: metrics-server
+  namespace: kube-system
+---
+apiVersion: policy/v1beta1
+kind: PodDisruptionBudget
+metadata:
+  name: metrics-server
+  namespace: kube-system
+  labels:
+    k8s-app: metrics-server
+spec:
+  minAvailable: 1
+  selector:
+    matchLabels:
+      k8s-app: metrics-server

upup/pkg/fi/cloudup/bootstrapchannelbuilder.go

@@ -563,6 +563,27 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 		}
 	}
 
+	if b.Cluster.Spec.MetricsServer != nil && fi.BoolValue(b.Cluster.Spec.MetricsServer.Enabled) {
+		{
+			key := "metrics-server.addons.k8s.io"
+			version := "0.3.7"
+
+			{
+				location := key + "/k8s-1.11.yaml"
+				id := "k8s-1.11"
+
+				addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+					Name:              fi.String(key),
+					Version:           fi.String(version),
+					Selector:          map[string]string{"k8s-app": "metrics-server"},
+					Manifest:          fi.String(location),
+					KubernetesVersion: ">=1.11.0",
+					Id:                id,
+				})
+			}
+		}
+	}
+
 	nth := b.Cluster.Spec.NodeTerminationHandler
 
 	if nth != nil && fi.BoolValue(nth.Enabled) {

upup/pkg/fi/cloudup/template_functions.go

@@ -131,6 +131,9 @@ func (tf *TemplateFunctions) AddTo(dest template.FuncMap, secretStore fi.SecretS
 	dest["ProxyEnv"] = tf.ProxyEnv
 
 	dest["KopsSystemEnv"] = tf.KopsSystemEnv
+	dest["UseKopsControllerForNodeBootstrap"] = func() bool {
+		return tf.UseKopsControllerForNodeBootstrap()
+	}
 
 	dest["DO_TOKEN"] = func() string {
 		return os.Getenv("DIGITALOCEAN_ACCESS_TOKEN")
@@ -392,7 +395,7 @@ func (tf *TemplateFunctions) KopsControllerConfig() (string, error) {
 	}
 
 	if tf.UseKopsControllerForNodeBootstrap() {
-		certNames := []string{"kubelet"}
+		certNames := []string{"kubelet", "kubelet-server"}
 		signingCAs := []string{fi.CertificateIDCA}
 		if apiModel.UseCiliumEtcd(cluster) {
 			certNames = append(certNames, "etcd-client-cilium")

upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/awsiamauthenticator/manifest.yaml

@@ -7,7 +7,7 @@ spec:
   - id: k8s-1.16
     kubernetesVersion: '>=1.16.0-alpha.0'
     manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml
-    manifestHash: 95b29a87c7e7204fd7f36716d7f9f3187985c2af
+    manifestHash: 70b6d9eaba39f1ead46355e682e747257eb52b49
     name: kops-controller.addons.k8s.io
     selector:
       k8s-addon: kops-controller.addons.k8s.io

upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/simple/kops-controller.addons.k8s.io-k8s-1.16.yaml

@@ -1,7 +1,7 @@
 apiVersion: v1
 data:
   config.yaml: |
-    {"cloud":"aws","configBase":"memfs://clusters.example.com/minimal.example.com","server":{"Listen":":3988","provider":{"aws":{"nodesRoles":["kops-custom-node-role","nodes.minimal.example.com"],"Region":"us-east-1"}},"serverKeyPath":"/etc/kubernetes/kops-controller/pki/kops-controller.key","serverCertificatePath":"/etc/kubernetes/kops-controller/pki/kops-controller.crt","caBasePath":"/etc/kubernetes/kops-controller/pki","signingCAs":["ca"],"certNames":["kubelet","kube-proxy"]}}
+    {"cloud":"aws","configBase":"memfs://clusters.example.com/minimal.example.com","server":{"Listen":":3988","provider":{"aws":{"nodesRoles":["kops-custom-node-role","nodes.minimal.example.com"],"Region":"us-east-1"}},"serverKeyPath":"/etc/kubernetes/kops-controller/pki/kops-controller.key","serverCertificatePath":"/etc/kubernetes/kops-controller/pki/kops-controller.crt","caBasePath":"/etc/kubernetes/kops-controller/pki","signingCAs":["ca"],"certNames":["kubelet","kubelet-server","kube-proxy"]}}
 kind: ConfigMap
 metadata:
   labels:

upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/simple/manifest.yaml

@@ -7,7 +7,7 @@ spec:
   - id: k8s-1.16
     kubernetesVersion: '>=1.16.0-alpha.0'
     manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml
-    manifestHash: 95b29a87c7e7204fd7f36716d7f9f3187985c2af
+    manifestHash: 70b6d9eaba39f1ead46355e682e747257eb52b49
     name: kops-controller.addons.k8s.io
     selector:
       k8s-addon: kops-controller.addons.k8s.io