diff --git a/addons/metrics-server/README.md b/addons/metrics-server/README.md index b92bc6f68d..037bfa407f 100644 --- a/addons/metrics-server/README.md +++ b/addons/metrics-server/README.md @@ -1,5 +1,7 @@ # Kubernetes Metrics Server +**This addon is deprecated. Set `spec.metricsServer.enabled: true` instead** + ## User guide You can find the user guide in diff --git a/cmd/kops-controller/pkg/server/server.go b/cmd/kops-controller/pkg/server/server.go index 1f1335cbfe..21d9c1d2c6 100644 --- a/cmd/kops-controller/pkg/server/server.go +++ b/cmd/kops-controller/pkg/server/server.go @@ -175,6 +175,12 @@ func (s *Server) issueCert(name string, pubKey string, id *fi.VerifyResult, vali CommonName: fmt.Sprintf("system:node:%s", id.NodeName), Organization: []string{rbac.NodesGroup}, } + case "kubelet-server": + issueReq.Subject = pkix.Name{ + CommonName: id.NodeName, + } + issueReq.AlternateNames = []string{id.NodeName} + issueReq.Type = "server" case "kube-proxy": issueReq.Subject = pkix.Name{ CommonName: rbac.KubeProxy, diff --git a/docs/releases/1.19-NOTES.md b/docs/releases/1.19-NOTES.md index 0a6f7c3ddd..bb2c290ea4 100644 --- a/docs/releases/1.19-NOTES.md +++ b/docs/releases/1.19-NOTES.md @@ -42,6 +42,8 @@ The expiration times vary randomly so that nodes are likely to have their certs * New command for deleting a single instance: [kops delete instance](/docs/cli/kops_delete_instance/) +* Metrics Server is now available as a configurable addon. Add `spec.metricsServer.enabled: true` to the cluster spec to enable. + # Breaking changes * Support for Kubernetes 1.9 and 1.10 has been removed. @@ -62,6 +64,8 @@ The expiration times vary randomly so that nodes are likely to have their certs * Support for feature flag `Terraform-0.12` has been deprecated and will be removed in kops 1.20. All generated Terraform HCL2/JSON files will support versions `0.12.26+` and `0.13.0+`. +* The [manifest based metrics server addon](https://github.com/kubernetes/kops/tree/master/addons/metrics-server) has been deprecated in favour of a configurable addon. + # Full change list since 1.18.0 release ## v1.18.0-alpha.3 to v1.19.0-alpha.1 diff --git a/k8s/crds/kops.k8s.io_clusters.yaml b/k8s/crds/kops.k8s.io_clusters.yaml index 3d9663ea1b..4534444678 100644 --- a/k8s/crds/kops.k8s.io_clusters.yaml +++ b/k8s/crds/kops.k8s.io_clusters.yaml @@ -2088,6 +2088,16 @@ spec: masterPublicName: description: MasterPublicName is the external DNS name for the master nodes type: string + metricsServer: + description: MetricsServerConfig determines the metrics server configuration. + properties: + enabled: + description: 'Enabled enables the metrics server. Default: false' + type: boolean + image: + description: 'Image is the docker container used. Default: the latest supported image for the specified kubernetes version.' + type: string + type: object networkCIDR: description: NetworkCIDR is the CIDR used for the AWS VPC / GCE Network, or otherwise allocated to k8s This is a real CIDR, not the internal k8s network On AWS, it maps to the VPC CIDR. It is not required on GCE. type: string diff --git a/nodeup/pkg/model/kubelet.go b/nodeup/pkg/model/kubelet.go index 06252cde9c..caad3c40c4 100644 --- a/nodeup/pkg/model/kubelet.go +++ b/nodeup/pkg/model/kubelet.go @@ -56,6 +56,12 @@ var _ fi.ModelBuilder = &KubeletBuilder{} // Build is responsible for building the kubelet configuration func (b *KubeletBuilder) Build(c *fi.ModelBuilderContext) error { + + err := b.buildKubeletServingCertificate(c) + if err != nil { + return fmt.Errorf("error building kubelet server cert: %v", err) + } + kubeletConfig, err := b.buildKubeletConfig() if err != nil { return fmt.Errorf("error building kubelet config: %v", err) @@ -226,6 +232,11 @@ func (b *KubeletBuilder) buildSystemdEnvironmentFile(kubeletConfig *kops.Kubelet } } + if b.UseKopsControllerForNodeBootstrap() { + flags += " --tls-cert-file=" + b.PathSrvKubernetes() + "/kubelet-server.crt" + flags += " --tls-private-key-file=" + b.PathSrvKubernetes() + "/kubelet-server.key" + } + sysconfig := "DAEMON_ARGS=\"" + flags + "\"\n" // Makes kubelet read /root/.docker/config.json properly sysconfig = sysconfig + "HOME=\"/root" + "\"\n" @@ -538,3 +549,50 @@ func (b *KubeletBuilder) buildMasterKubeletKubeconfig(c *fi.ModelBuilderContext) return b.BuildIssuedKubeconfig("kubelet", certName, c), nil } + +func (b *KubeletBuilder) buildKubeletServingCertificate(c *fi.ModelBuilderContext) error { + + if b.UseKopsControllerForNodeBootstrap() { + name := "kubelet-server" + dir := b.PathSrvKubernetes() + signer := fi.CertificateIDCA + + nodeName, err := b.NodeName() + if err != nil { + return err + } + + if !b.IsMaster { + cert, key := b.GetBootstrapCert(name) + + c.AddTask(&nodetasks.File{ + Path: filepath.Join(dir, name+".crt"), + Contents: cert, + Type: nodetasks.FileType_File, + Mode: fi.String("0644"), + }) + + c.AddTask(&nodetasks.File{ + Path: filepath.Join(dir, name+".key"), + Contents: key, + Type: nodetasks.FileType_File, + Mode: fi.String("0400"), + }) + + } else { + issueCert := &nodetasks.IssueCert{ + Name: name, + Signer: signer, + Type: "server", + Subject: nodetasks.PKIXName{ + CommonName: nodeName, + }, + AlternateNames: []string{nodeName}, + } + c.AddTask(issueCert) + return issueCert.AddFileTasks(c, dir, name, "", nil) + } + } + return nil + +} diff --git a/pkg/apis/kops/cluster.go b/pkg/apis/kops/cluster.go index eb9380fb88..d749a2456b 100644 --- a/pkg/apis/kops/cluster.go +++ b/pkg/apis/kops/cluster.go @@ -160,6 +160,8 @@ type ClusterSpec struct { // NodeTerminationHandlerConfig determines the cluster autoscaler configuration. NodeTerminationHandler *NodeTerminationHandlerConfig `json:"nodeTerminationHandler,omitempty"` + // MetricsServerConfig determines the metrics server configuration. + MetricsServer *MetricsServerConfig `json:"metricsServer,omitempty"` // Networking configuration Networking *NetworkingSpec `json:"networking,omitempty"` diff --git a/pkg/apis/kops/componentconfig.go b/pkg/apis/kops/componentconfig.go index fb5b7de16b..3e5a164653 100644 --- a/pkg/apis/kops/componentconfig.go +++ b/pkg/apis/kops/componentconfig.go @@ -803,6 +803,16 @@ type ClusterAutoscalerConfig struct { Image *string `json:"image,omitempty"` } +// MetricsServerConfig determines the metrics server configuration. +type MetricsServerConfig struct { + // Enabled enables the metrics server. + // Default: false + Enabled *bool `json:"enabled,omitempty"` + // Image is the docker container used. + // Default: the latest supported image for the specified kubernetes version. + Image *string `json:"image,omitempty"` +} + // HasAdmissionController checks if a specific admission controller is enabled func (c *KubeAPIServerConfig) HasAdmissionController(name string) bool { for _, x := range c.AdmissionControl { diff --git a/pkg/apis/kops/v1alpha2/cluster.go b/pkg/apis/kops/v1alpha2/cluster.go index 9bd1a9625d..6e2721f87b 100644 --- a/pkg/apis/kops/v1alpha2/cluster.go +++ b/pkg/apis/kops/v1alpha2/cluster.go @@ -159,6 +159,8 @@ type ClusterSpec struct { // NodeTerminationHandlerConfig determines the cluster autoscaler configuration. NodeTerminationHandler *NodeTerminationHandlerConfig `json:"nodeTerminationHandler,omitempty"` + // MetricsServerConfig determines the metrics server configuration. + MetricsServer *MetricsServerConfig `json:"metricsServer,omitempty"` // Networking configuration Networking *NetworkingSpec `json:"networking,omitempty"` diff --git a/pkg/apis/kops/v1alpha2/componentconfig.go b/pkg/apis/kops/v1alpha2/componentconfig.go index f09722d306..b472476eed 100644 --- a/pkg/apis/kops/v1alpha2/componentconfig.go +++ b/pkg/apis/kops/v1alpha2/componentconfig.go @@ -804,6 +804,16 @@ type ClusterAutoscalerConfig struct { Image *string `json:"image,omitempty"` } +// MetricsServerConfig determines the metrics server configuration. +type MetricsServerConfig struct { + // Enabled enables the metrics server. + // Default: false + Enabled *bool `json:"enabled,omitempty"` + // Image is the docker container used. + // Default: the latest supported image for the specified kubernetes version. + Image *string `json:"image,omitempty"` +} + // HasAdmissionController checks if a specific admission controller is enabled func (c *KubeAPIServerConfig) HasAdmissionController(name string) bool { for _, x := range c.AdmissionControl { diff --git a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go index c52acaa336..b23aed4bfe 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go @@ -693,6 +693,16 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddGeneratedConversionFunc((*MetricsServerConfig)(nil), (*kops.MetricsServerConfig)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha2_MetricsServerConfig_To_kops_MetricsServerConfig(a.(*MetricsServerConfig), b.(*kops.MetricsServerConfig), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*kops.MetricsServerConfig)(nil), (*MetricsServerConfig)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_kops_MetricsServerConfig_To_v1alpha2_MetricsServerConfig(a.(*kops.MetricsServerConfig), b.(*MetricsServerConfig), scope) + }); err != nil { + return err + } if err := s.AddGeneratedConversionFunc((*MixedInstancesPolicySpec)(nil), (*kops.MixedInstancesPolicySpec)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha2_MixedInstancesPolicySpec_To_kops_MixedInstancesPolicySpec(a.(*MixedInstancesPolicySpec), b.(*kops.MixedInstancesPolicySpec), scope) }); err != nil { @@ -2066,6 +2076,15 @@ func autoConvert_v1alpha2_ClusterSpec_To_kops_ClusterSpec(in *ClusterSpec, out * } else { out.NodeTerminationHandler = nil } + if in.MetricsServer != nil { + in, out := &in.MetricsServer, &out.MetricsServer + *out = new(kops.MetricsServerConfig) + if err := Convert_v1alpha2_MetricsServerConfig_To_kops_MetricsServerConfig(*in, *out, s); err != nil { + return err + } + } else { + out.MetricsServer = nil + } if in.Networking != nil { in, out := &in.Networking, &out.Networking *out = new(kops.NetworkingSpec) @@ -2406,6 +2425,15 @@ func autoConvert_kops_ClusterSpec_To_v1alpha2_ClusterSpec(in *kops.ClusterSpec, } else { out.NodeTerminationHandler = nil } + if in.MetricsServer != nil { + in, out := &in.MetricsServer, &out.MetricsServer + *out = new(MetricsServerConfig) + if err := Convert_kops_MetricsServerConfig_To_v1alpha2_MetricsServerConfig(*in, *out, s); err != nil { + return err + } + } else { + out.MetricsServer = nil + } if in.Networking != nil { in, out := &in.Networking, &out.Networking *out = new(NetworkingSpec) @@ -4686,6 +4714,28 @@ func Convert_kops_LyftVPCNetworkingSpec_To_v1alpha2_LyftVPCNetworkingSpec(in *ko return autoConvert_kops_LyftVPCNetworkingSpec_To_v1alpha2_LyftVPCNetworkingSpec(in, out, s) } +func autoConvert_v1alpha2_MetricsServerConfig_To_kops_MetricsServerConfig(in *MetricsServerConfig, out *kops.MetricsServerConfig, s conversion.Scope) error { + out.Enabled = in.Enabled + out.Image = in.Image + return nil +} + +// Convert_v1alpha2_MetricsServerConfig_To_kops_MetricsServerConfig is an autogenerated conversion function. +func Convert_v1alpha2_MetricsServerConfig_To_kops_MetricsServerConfig(in *MetricsServerConfig, out *kops.MetricsServerConfig, s conversion.Scope) error { + return autoConvert_v1alpha2_MetricsServerConfig_To_kops_MetricsServerConfig(in, out, s) +} + +func autoConvert_kops_MetricsServerConfig_To_v1alpha2_MetricsServerConfig(in *kops.MetricsServerConfig, out *MetricsServerConfig, s conversion.Scope) error { + out.Enabled = in.Enabled + out.Image = in.Image + return nil +} + +// Convert_kops_MetricsServerConfig_To_v1alpha2_MetricsServerConfig is an autogenerated conversion function. +func Convert_kops_MetricsServerConfig_To_v1alpha2_MetricsServerConfig(in *kops.MetricsServerConfig, out *MetricsServerConfig, s conversion.Scope) error { + return autoConvert_kops_MetricsServerConfig_To_v1alpha2_MetricsServerConfig(in, out, s) +} + func autoConvert_v1alpha2_MixedInstancesPolicySpec_To_kops_MixedInstancesPolicySpec(in *MixedInstancesPolicySpec, out *kops.MixedInstancesPolicySpec, s conversion.Scope) error { out.Instances = in.Instances out.OnDemandAllocationStrategy = in.OnDemandAllocationStrategy diff --git a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go index e01353b1be..2ca966431b 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go @@ -845,6 +845,11 @@ func (in *ClusterSpec) DeepCopyInto(out *ClusterSpec) { *out = new(NodeTerminationHandlerConfig) (*in).DeepCopyInto(*out) } + if in.MetricsServer != nil { + in, out := &in.MetricsServer, &out.MetricsServer + *out = new(MetricsServerConfig) + (*in).DeepCopyInto(*out) + } if in.Networking != nil { in, out := &in.Networking, &out.Networking *out = new(NetworkingSpec) @@ -3169,6 +3174,32 @@ func (in *LyftVPCNetworkingSpec) DeepCopy() *LyftVPCNetworkingSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MetricsServerConfig) DeepCopyInto(out *MetricsServerConfig) { + *out = *in + if in.Enabled != nil { + in, out := &in.Enabled, &out.Enabled + *out = new(bool) + **out = **in + } + if in.Image != nil { + in, out := &in.Image, &out.Image + *out = new(string) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MetricsServerConfig. +func (in *MetricsServerConfig) DeepCopy() *MetricsServerConfig { + if in == nil { + return nil + } + out := new(MetricsServerConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *MixedInstancesPolicySpec) DeepCopyInto(out *MixedInstancesPolicySpec) { *out = *in diff --git a/pkg/apis/kops/zz_generated.deepcopy.go b/pkg/apis/kops/zz_generated.deepcopy.go index 701bf06b25..08f62ee733 100644 --- a/pkg/apis/kops/zz_generated.deepcopy.go +++ b/pkg/apis/kops/zz_generated.deepcopy.go @@ -945,6 +945,11 @@ func (in *ClusterSpec) DeepCopyInto(out *ClusterSpec) { *out = new(NodeTerminationHandlerConfig) (*in).DeepCopyInto(*out) } + if in.MetricsServer != nil { + in, out := &in.MetricsServer, &out.MetricsServer + *out = new(MetricsServerConfig) + (*in).DeepCopyInto(*out) + } if in.Networking != nil { in, out := &in.Networking, &out.Networking *out = new(NetworkingSpec) @@ -3367,6 +3372,32 @@ func (in *LyftVPCNetworkingSpec) DeepCopy() *LyftVPCNetworkingSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MetricsServerConfig) DeepCopyInto(out *MetricsServerConfig) { + *out = *in + if in.Enabled != nil { + in, out := &in.Enabled, &out.Enabled + *out = new(bool) + **out = **in + } + if in.Image != nil { + in, out := &in.Image, &out.Image + *out = new(string) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MetricsServerConfig. +func (in *MetricsServerConfig) DeepCopy() *MetricsServerConfig { + if in == nil { + return nil + } + out := new(MetricsServerConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *MixedInstancesPolicySpec) DeepCopyInto(out *MixedInstancesPolicySpec) { *out = *in diff --git a/upup/models/bindata.go b/upup/models/bindata.go index df57224f10..37dd89655d 100644 --- a/upup/models/bindata.go +++ b/upup/models/bindata.go @@ -27,6 +27,7 @@ // upup/models/cloudup/resources/addons/limit-range.addons.k8s.io/v1.5.0.yaml // upup/models/cloudup/resources/addons/metadata-proxy.addons.k8s.io/addon.yaml // upup/models/cloudup/resources/addons/metadata-proxy.addons.k8s.io/v0.1.12.yaml +// upup/models/cloudup/resources/addons/metrics-server.addons.k8s.io/k8s-1.11.yaml.template // upup/models/cloudup/resources/addons/networking.amazon-vpc-routed-eni/k8s-1.10.yaml.template // upup/models/cloudup/resources/addons/networking.amazon-vpc-routed-eni/k8s-1.12.yaml.template // upup/models/cloudup/resources/addons/networking.amazon-vpc-routed-eni/k8s-1.16.yaml.template @@ -3838,6 +3839,188 @@ func cloudupResourcesAddonsMetadataProxyAddonsK8sIoV0112Yaml() (*asset, error) { return a, nil } +var _cloudupResourcesAddonsMetricsServerAddonsK8sIoK8s111YamlTemplate = []byte(`# sourced from https://github.com/kubernetes-sigs/metrics-server/releases/download/v0.3.7/components.yaml +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: system:aggregated-metrics-reader + labels: + rbac.authorization.k8s.io/aggregate-to-view: "true" + rbac.authorization.k8s.io/aggregate-to-edit: "true" + rbac.authorization.k8s.io/aggregate-to-admin: "true" +rules: +- apiGroups: ["metrics.k8s.io"] + resources: ["pods", "nodes"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: metrics-server:system:auth-delegator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator +subjects: +- kind: ServiceAccount + name: metrics-server + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: metrics-server-auth-reader + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: +- kind: ServiceAccount + name: metrics-server + namespace: kube-system +--- +apiVersion: apiregistration.k8s.io/v1beta1 +kind: APIService +metadata: + name: v1beta1.metrics.k8s.io +spec: + service: + name: metrics-server + namespace: kube-system + group: metrics.k8s.io + version: v1beta1 + insecureSkipTLSVerify: true + groupPriorityMinimum: 100 + versionPriority: 100 +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: metrics-server + namespace: kube-system +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: metrics-server + namespace: kube-system + labels: + k8s-app: metrics-server +spec: + replicas: 2 + selector: + matchLabels: + k8s-app: metrics-server + template: + metadata: + name: metrics-server + labels: + k8s-app: metrics-server + spec: + serviceAccountName: metrics-server + volumes: + # mount in tmp so we can safely use from-scratch images and/or read-only containers + - name: tmp-dir + emptyDir: {} + containers: + - name: metrics-server + image: {{ or .MetricsServer.Image "k8s.gcr.io/metrics-server/metrics-server:v0.3.7" }} + imagePullPolicy: IfNotPresent + args: + - --cert-dir=/tmp + - --secure-port=4443 + {{ if not UseKopsControllerForNodeBootstrap }} + - --kubelet-insecure-tls + {{ end }} + ports: + - name: main-port + containerPort: 4443 + protocol: TCP + securityContext: + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + volumeMounts: + - name: tmp-dir + mountPath: /tmp +--- +apiVersion: v1 +kind: Service +metadata: + name: metrics-server + namespace: kube-system + labels: + kubernetes.io/name: "Metrics-server" + kubernetes.io/cluster-service: "true" +spec: + selector: + k8s-app: metrics-server + ports: + - port: 443 + protocol: TCP + targetPort: main-port +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: system:metrics-server +rules: +- apiGroups: + - "" + resources: + - pods + - nodes + - nodes/stats + - namespaces + - configmaps + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: system:metrics-server +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:metrics-server +subjects: +- kind: ServiceAccount + name: metrics-server + namespace: kube-system +--- +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: metrics-server + namespace: kube-system + labels: + k8s-app: metrics-server +spec: + minAvailable: 1 + selector: + matchLabels: + k8s-app: metrics-server`) + +func cloudupResourcesAddonsMetricsServerAddonsK8sIoK8s111YamlTemplateBytes() ([]byte, error) { + return _cloudupResourcesAddonsMetricsServerAddonsK8sIoK8s111YamlTemplate, nil +} + +func cloudupResourcesAddonsMetricsServerAddonsK8sIoK8s111YamlTemplate() (*asset, error) { + bytes, err := cloudupResourcesAddonsMetricsServerAddonsK8sIoK8s111YamlTemplateBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "cloudup/resources/addons/metrics-server.addons.k8s.io/k8s-1.11.yaml.template", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} + a := &asset{bytes: bytes, info: info} + return a, nil +} + var _cloudupResourcesAddonsNetworkingAmazonVpcRoutedEniK8s110YamlTemplate = []byte(`# Vendored from https://github.com/aws/amazon-vpc-cni-k8s/blob/v1.3.3/config/v1.3/aws-k8s-cni.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -20688,6 +20871,7 @@ var _bindata = map[string]func() (*asset, error){ "cloudup/resources/addons/limit-range.addons.k8s.io/v1.5.0.yaml": cloudupResourcesAddonsLimitRangeAddonsK8sIoV150Yaml, "cloudup/resources/addons/metadata-proxy.addons.k8s.io/addon.yaml": cloudupResourcesAddonsMetadataProxyAddonsK8sIoAddonYaml, "cloudup/resources/addons/metadata-proxy.addons.k8s.io/v0.1.12.yaml": cloudupResourcesAddonsMetadataProxyAddonsK8sIoV0112Yaml, + "cloudup/resources/addons/metrics-server.addons.k8s.io/k8s-1.11.yaml.template": cloudupResourcesAddonsMetricsServerAddonsK8sIoK8s111YamlTemplate, "cloudup/resources/addons/networking.amazon-vpc-routed-eni/k8s-1.10.yaml.template": cloudupResourcesAddonsNetworkingAmazonVpcRoutedEniK8s110YamlTemplate, "cloudup/resources/addons/networking.amazon-vpc-routed-eni/k8s-1.12.yaml.template": cloudupResourcesAddonsNetworkingAmazonVpcRoutedEniK8s112YamlTemplate, "cloudup/resources/addons/networking.amazon-vpc-routed-eni/k8s-1.16.yaml.template": cloudupResourcesAddonsNetworkingAmazonVpcRoutedEniK8s116YamlTemplate, @@ -20826,6 +21010,9 @@ var _bintree = &bintree{nil, map[string]*bintree{ "addon.yaml": {cloudupResourcesAddonsMetadataProxyAddonsK8sIoAddonYaml, map[string]*bintree{}}, "v0.1.12.yaml": {cloudupResourcesAddonsMetadataProxyAddonsK8sIoV0112Yaml, map[string]*bintree{}}, }}, + "metrics-server.addons.k8s.io": {nil, map[string]*bintree{ + "k8s-1.11.yaml.template": {cloudupResourcesAddonsMetricsServerAddonsK8sIoK8s111YamlTemplate, map[string]*bintree{}}, + }}, "networking.amazon-vpc-routed-eni": {nil, map[string]*bintree{ "k8s-1.10.yaml.template": {cloudupResourcesAddonsNetworkingAmazonVpcRoutedEniK8s110YamlTemplate, map[string]*bintree{}}, "k8s-1.12.yaml.template": {cloudupResourcesAddonsNetworkingAmazonVpcRoutedEniK8s112YamlTemplate, map[string]*bintree{}}, diff --git a/upup/models/cloudup/resources/addons/metrics-server.addons.k8s.io/k8s-1.11.yaml.template b/upup/models/cloudup/resources/addons/metrics-server.addons.k8s.io/k8s-1.11.yaml.template new file mode 100644 index 0000000000..a895546094 --- /dev/null +++ b/upup/models/cloudup/resources/addons/metrics-server.addons.k8s.io/k8s-1.11.yaml.template @@ -0,0 +1,166 @@ +# sourced from https://github.com/kubernetes-sigs/metrics-server/releases/download/v0.3.7/components.yaml +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: system:aggregated-metrics-reader + labels: + rbac.authorization.k8s.io/aggregate-to-view: "true" + rbac.authorization.k8s.io/aggregate-to-edit: "true" + rbac.authorization.k8s.io/aggregate-to-admin: "true" +rules: +- apiGroups: ["metrics.k8s.io"] + resources: ["pods", "nodes"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: metrics-server:system:auth-delegator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator +subjects: +- kind: ServiceAccount + name: metrics-server + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: metrics-server-auth-reader + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: +- kind: ServiceAccount + name: metrics-server + namespace: kube-system +--- +apiVersion: apiregistration.k8s.io/v1beta1 +kind: APIService +metadata: + name: v1beta1.metrics.k8s.io +spec: + service: + name: metrics-server + namespace: kube-system + group: metrics.k8s.io + version: v1beta1 + insecureSkipTLSVerify: true + groupPriorityMinimum: 100 + versionPriority: 100 +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: metrics-server + namespace: kube-system +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: metrics-server + namespace: kube-system + labels: + k8s-app: metrics-server +spec: + replicas: 2 + selector: + matchLabels: + k8s-app: metrics-server + template: + metadata: + name: metrics-server + labels: + k8s-app: metrics-server + spec: + serviceAccountName: metrics-server + volumes: + # mount in tmp so we can safely use from-scratch images and/or read-only containers + - name: tmp-dir + emptyDir: {} + containers: + - name: metrics-server + image: {{ or .MetricsServer.Image "k8s.gcr.io/metrics-server/metrics-server:v0.3.7" }} + imagePullPolicy: IfNotPresent + args: + - --cert-dir=/tmp + - --secure-port=4443 + {{ if not UseKopsControllerForNodeBootstrap }} + - --kubelet-insecure-tls + {{ end }} + ports: + - name: main-port + containerPort: 4443 + protocol: TCP + securityContext: + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + volumeMounts: + - name: tmp-dir + mountPath: /tmp +--- +apiVersion: v1 +kind: Service +metadata: + name: metrics-server + namespace: kube-system + labels: + kubernetes.io/name: "Metrics-server" + kubernetes.io/cluster-service: "true" +spec: + selector: + k8s-app: metrics-server + ports: + - port: 443 + protocol: TCP + targetPort: main-port +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: system:metrics-server +rules: +- apiGroups: + - "" + resources: + - pods + - nodes + - nodes/stats + - namespaces + - configmaps + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: system:metrics-server +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:metrics-server +subjects: +- kind: ServiceAccount + name: metrics-server + namespace: kube-system +--- +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: metrics-server + namespace: kube-system + labels: + k8s-app: metrics-server +spec: + minAvailable: 1 + selector: + matchLabels: + k8s-app: metrics-server \ No newline at end of file diff --git a/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go b/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go index 3bd603156b..9965b132f3 100644 --- a/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go +++ b/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go @@ -563,6 +563,27 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann } } + if b.Cluster.Spec.MetricsServer != nil && fi.BoolValue(b.Cluster.Spec.MetricsServer.Enabled) { + { + key := "metrics-server.addons.k8s.io" + version := "0.3.7" + + { + location := key + "/k8s-1.11.yaml" + id := "k8s-1.11" + + addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{ + Name: fi.String(key), + Version: fi.String(version), + Selector: map[string]string{"k8s-app": "metrics-server"}, + Manifest: fi.String(location), + KubernetesVersion: ">=1.11.0", + Id: id, + }) + } + } + } + nth := b.Cluster.Spec.NodeTerminationHandler if nth != nil && fi.BoolValue(nth.Enabled) { diff --git a/upup/pkg/fi/cloudup/template_functions.go b/upup/pkg/fi/cloudup/template_functions.go index 12ca5e036c..a4daf98519 100644 --- a/upup/pkg/fi/cloudup/template_functions.go +++ b/upup/pkg/fi/cloudup/template_functions.go @@ -131,6 +131,9 @@ func (tf *TemplateFunctions) AddTo(dest template.FuncMap, secretStore fi.SecretS dest["ProxyEnv"] = tf.ProxyEnv dest["KopsSystemEnv"] = tf.KopsSystemEnv + dest["UseKopsControllerForNodeBootstrap"] = func() bool { + return tf.UseKopsControllerForNodeBootstrap() + } dest["DO_TOKEN"] = func() string { return os.Getenv("DIGITALOCEAN_ACCESS_TOKEN") @@ -392,7 +395,7 @@ func (tf *TemplateFunctions) KopsControllerConfig() (string, error) { } if tf.UseKopsControllerForNodeBootstrap() { - certNames := []string{"kubelet"} + certNames := []string{"kubelet", "kubelet-server"} signingCAs := []string{fi.CertificateIDCA} if apiModel.UseCiliumEtcd(cluster) { certNames = append(certNames, "etcd-client-cilium") diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/awsiamauthenticator/manifest.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/awsiamauthenticator/manifest.yaml index 0c476831db..fd30a5c877 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/awsiamauthenticator/manifest.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/awsiamauthenticator/manifest.yaml @@ -7,7 +7,7 @@ spec: - id: k8s-1.16 kubernetesVersion: '>=1.16.0-alpha.0' manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml - manifestHash: 95b29a87c7e7204fd7f36716d7f9f3187985c2af + manifestHash: 70b6d9eaba39f1ead46355e682e747257eb52b49 name: kops-controller.addons.k8s.io selector: k8s-addon: kops-controller.addons.k8s.io diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/simple/kops-controller.addons.k8s.io-k8s-1.16.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/simple/kops-controller.addons.k8s.io-k8s-1.16.yaml index 8497533d29..97301430c8 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/simple/kops-controller.addons.k8s.io-k8s-1.16.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/simple/kops-controller.addons.k8s.io-k8s-1.16.yaml @@ -1,7 +1,7 @@ apiVersion: v1 data: config.yaml: | - {"cloud":"aws","configBase":"memfs://clusters.example.com/minimal.example.com","server":{"Listen":":3988","provider":{"aws":{"nodesRoles":["kops-custom-node-role","nodes.minimal.example.com"],"Region":"us-east-1"}},"serverKeyPath":"/etc/kubernetes/kops-controller/pki/kops-controller.key","serverCertificatePath":"/etc/kubernetes/kops-controller/pki/kops-controller.crt","caBasePath":"/etc/kubernetes/kops-controller/pki","signingCAs":["ca"],"certNames":["kubelet","kube-proxy"]}} + {"cloud":"aws","configBase":"memfs://clusters.example.com/minimal.example.com","server":{"Listen":":3988","provider":{"aws":{"nodesRoles":["kops-custom-node-role","nodes.minimal.example.com"],"Region":"us-east-1"}},"serverKeyPath":"/etc/kubernetes/kops-controller/pki/kops-controller.key","serverCertificatePath":"/etc/kubernetes/kops-controller/pki/kops-controller.crt","caBasePath":"/etc/kubernetes/kops-controller/pki","signingCAs":["ca"],"certNames":["kubelet","kubelet-server","kube-proxy"]}} kind: ConfigMap metadata: labels: diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/simple/manifest.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/simple/manifest.yaml index 6e11aced5d..86e0b57368 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/simple/manifest.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/simple/manifest.yaml @@ -7,7 +7,7 @@ spec: - id: k8s-1.16 kubernetesVersion: '>=1.16.0-alpha.0' manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml - manifestHash: 95b29a87c7e7204fd7f36716d7f9f3187985c2af + manifestHash: 70b6d9eaba39f1ead46355e682e747257eb52b49 name: kops-controller.addons.k8s.io selector: k8s-addon: kops-controller.addons.k8s.io