diff --git a/nodeup/pkg/model/BUILD.bazel b/nodeup/pkg/model/BUILD.bazel index 77f4abd7f2..48b404d990 100644 --- a/nodeup/pkg/model/BUILD.bazel +++ b/nodeup/pkg/model/BUILD.bazel @@ -45,6 +45,7 @@ go_library( "//pkg/k8scodecs:go_default_library", "//pkg/kubeconfig:go_default_library", "//pkg/kubemanifest:go_default_library", + "//pkg/pki:go_default_library", "//pkg/systemd:go_default_library", "//pkg/tokens:go_default_library", "//upup/pkg/fi:go_default_library", @@ -60,6 +61,7 @@ go_library( "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/intstr:go_default_library", + "//vendor/k8s.io/apiserver/pkg/authentication/user:go_default_library", ], ) diff --git a/nodeup/pkg/model/calico.go b/nodeup/pkg/model/calico.go index 192402b89a..a9326c403f 100644 --- a/nodeup/pkg/model/calico.go +++ b/nodeup/pkg/model/calico.go @@ -42,7 +42,7 @@ func (b *CalicoBuilder) Build(c *fi.ModelBuilderContext) error { if err := b.BuildCertificateTask(c, name, certificate); err != nil { return err } - if err := b.BuildPrivateTask(c, name, key); err != nil { + if err := b.BuildPrivateKeyTask(c, name, key); err != nil { return err } if err := b.BuildCertificateTask(c, fi.CertificateId_CA, ca); err != nil { diff --git a/nodeup/pkg/model/context.go b/nodeup/pkg/model/context.go index b704aa63a4..dcf77e2602 100644 --- a/nodeup/pkg/model/context.go +++ b/nodeup/pkg/model/context.go @@ -18,10 +18,10 @@ package model import ( "fmt" + "os" "path/filepath" + "strings" - "github.com/blang/semver" - "github.com/golang/glog" "k8s.io/kops/nodeup/pkg/distros" "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/pkg/apis/kops/util" @@ -29,6 +29,10 @@ import ( "k8s.io/kops/pkg/kubeconfig" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/nodeup/nodetasks" + "k8s.io/kops/util/pkg/vfs" + + "github.com/blang/semver" + "github.com/golang/glog" ) // NodeupModelContext is the context supplied the nodeup tasks @@ -110,55 +114,66 @@ func (c *NodeupModelContext) CNIBinDir() string { } } +// KubeletBootstrapConfig is the path the bootstrap config file +func (c *NodeupModelContext) KubeletBootstrapConfig() string { + path := c.Cluster.Spec.Kubelet.BootstrapKubeconfig + + if c.IsMaster { + if c.Cluster.Spec.MasterKubelet != nil && c.Cluster.Spec.MasterKubelet.BootstrapKubeconfig != "" { + path = c.Cluster.Spec.MasterKubelet.BootstrapKubeconfig + } + } + + if path != "" { + return path + } + + return "/var/lib/kubelet/bootstrap-kubeconfig" +} + +// KubeletKubeConfig is the path of the kubelet kubeconfig file +func (c *NodeupModelContext) KubeletKubeConfig() string { + return "/var/lib/kubelet/kubeconfig" +} + // CNIConfDir returns the CNI directory func (c *NodeupModelContext) CNIConfDir() string { return "/etc/cni/net.d/" } -// buildPKIKubeconfig generates a kubeconfig -func (c *NodeupModelContext) buildPKIKubeconfig(id string) (string, error) { - caCertificate, err := c.KeyStore.FindCert(fi.CertificateId_CA) +// BuildPKIKubeconfig generates a kubeconfig +func (c *NodeupModelContext) BuildPKIKubeconfig(name string) (string, error) { + ca, err := c.FindCert(fi.CertificateId_CA) if err != nil { - return "", fmt.Errorf("error fetching CA certificate from keystore: %v", err) - } - if caCertificate == nil { - return "", fmt.Errorf("CA certificate %q not found", fi.CertificateId_CA) + return "", err } - certificate, err := c.KeyStore.FindCert(id) + cert, err := c.FindCert(name) if err != nil { - return "", fmt.Errorf("error fetching %q certificate from keystore: %v", id, err) - } - if certificate == nil { - return "", fmt.Errorf("certificate %q not found", id) + return "", err } - privateKey, err := c.KeyStore.FindPrivateKey(id) + key, err := c.FindPrivateKey(name) if err != nil { - return "", fmt.Errorf("error fetching %q private key from keystore: %v", id, err) - } - if privateKey == nil { - return "", fmt.Errorf("private key %q not found", id) + return "", err } - user := kubeconfig.KubectlUser{} - user.ClientCertificateData, err = certificate.AsBytes() - if err != nil { - return "", fmt.Errorf("error encoding %q certificate: %v", id, err) + return c.BuildKubeConfig(name, ca, cert, key) +} + +// BuildKubeConfig is responsible for building a kubeconfig +func (c *NodeupModelContext) BuildKubeConfig(username string, ca, certificate, privateKey []byte) (string, error) { + user := kubeconfig.KubectlUser{ + ClientCertificateData: certificate, + ClientKeyData: privateKey, } - user.ClientKeyData, err = privateKey.AsBytes() - if err != nil { - return "", fmt.Errorf("error encoding %q private key: %v", id, err) - } - cluster := kubeconfig.KubectlCluster{} - cluster.CertificateAuthorityData, err = caCertificate.AsBytes() - if err != nil { - return "", fmt.Errorf("error encoding CA certificate: %v", err) + cluster := kubeconfig.KubectlCluster{ + CertificateAuthorityData: ca, } if c.IsMaster { if c.IsKubernetesGTE("1.6") { - // Use https in 1.6, even for local connections, so we can turn off the insecure port + // @note: use https >= 1.6m even for local connections, so we can turn off the insecure port cluster.Server = "https://127.0.0.1" } else { cluster.Server = "http://127.0.0.1:8080" @@ -172,7 +187,7 @@ func (c *NodeupModelContext) buildPKIKubeconfig(id string) (string, error) { Kind: "Config", Users: []*kubeconfig.KubectlUserWithName{ { - Name: id, + Name: username, User: user, }, }, @@ -187,7 +202,7 @@ func (c *NodeupModelContext) buildPKIKubeconfig(id string) (string, error) { Name: "service-account-context", Context: kubeconfig.KubectlContext{ Cluster: "local", - User: id, + User: username, }, }, }, @@ -248,6 +263,15 @@ func (c *NodeupModelContext) UsesCNI() bool { return true } +// UseBootstrapTokens checks if we are using bootstrap tokens +func (c *NodeupModelContext) UseBootstrapTokens() bool { + if c.IsMaster { + return fi.BoolValue(c.Cluster.Spec.KubeAPIServer.EnableBootstrapAuthToken) + } + + return c.Cluster.Spec.Kubelet != nil && c.Cluster.Spec.Kubelet.BootstrapKubeconfig != "" +} + // UseSecureKubelet checks if the kubelet api should be protected by a client certificate. Note: the settings are // in one of three section, master specific kubelet, cluster wide kubelet or the InstanceGroup. Though arguably is // doesn't make much sense to unset this on a per InstanceGroup level, but hey :) @@ -287,6 +311,18 @@ func (c *NodeupModelContext) KubectlPath() string { return kubeletCommand } +// BuildCertificatePairTask creates the tasks to pull down the certificate and private key +func (c *NodeupModelContext) BuildCertificatePairTask(ctx *fi.ModelBuilderContext, key, path, filename string) error { + certificateName := fmt.Sprintf("%s/%s.pem", strings.TrimSuffix(path, "/"), filename) + keyName := fmt.Sprintf("%s/%s-key.pem", strings.TrimSuffix(path, "/"), filename) + + if err := c.BuildCertificateTask(ctx, key, certificateName); err != nil { + return err + } + + return c.BuildPrivateKeyTask(ctx, key, keyName) +} + // BuildCertificateTask is responsible for build a certificate request task func (c *NodeupModelContext) BuildCertificateTask(ctx *fi.ModelBuilderContext, name, filename string) error { cert, err := c.KeyStore.FindCert(name) @@ -307,14 +343,14 @@ func (c *NodeupModelContext) BuildCertificateTask(ctx *fi.ModelBuilderContext, n Path: filepath.Join(c.PathSrvKubernetes(), filename), Contents: fi.NewStringResource(serialized), Type: nodetasks.FileType_File, - Mode: s("0400"), + Mode: s("0600"), }) return nil } // BuildPrivateKeyTask is responsible for build a certificate request task -func (c *NodeupModelContext) BuildPrivateTask(ctx *fi.ModelBuilderContext, name, filename string) error { +func (c *NodeupModelContext) BuildPrivateKeyTask(ctx *fi.ModelBuilderContext, name, filename string) error { cert, err := c.KeyStore.FindPrivateKey(name) if err != nil { return err @@ -333,8 +369,92 @@ func (c *NodeupModelContext) BuildPrivateTask(ctx *fi.ModelBuilderContext, name, Path: filepath.Join(c.PathSrvKubernetes(), filename), Contents: fi.NewStringResource(serialized), Type: nodetasks.FileType_File, - Mode: s("0400"), + Mode: s("0600"), }) return nil } + +// NodeName returns the name of the local Node, as it will be created in k8s +func (c *NodeupModelContext) NodeName() (string, error) { + // This mirrors nodeutil.GetHostName + hostnameOverride := c.Cluster.Spec.Kubelet.HostnameOverride + + if c.IsMaster && c.Cluster.Spec.MasterKubelet.HostnameOverride != "" { + hostnameOverride = c.Cluster.Spec.MasterKubelet.HostnameOverride + } + + nodeName, err := EvaluateHostnameOverride(hostnameOverride) + if err != nil { + return "", fmt.Errorf("error evaluating hostname: %v", err) + } + + if nodeName == "" { + hostname, err := os.Hostname() + if err != nil { + glog.Fatalf("Couldn't determine hostname: %v", err) + } + nodeName = hostname + } + + return strings.ToLower(strings.TrimSpace(nodeName)), nil +} + +// EvaluateHostnameOverride returns the hostname after replacing some well-known placeholders +func EvaluateHostnameOverride(hostnameOverride string) (string, error) { + if hostnameOverride == "" || hostnameOverride == "@hostname" { + return "", nil + } + k := strings.TrimSpace(hostnameOverride) + k = strings.ToLower(k) + + if k != "@aws" { + return hostnameOverride, nil + } + + // We recognize @aws as meaning "the local-hostname from the aws metadata service" + vBytes, err := vfs.Context.ReadFile("metadata://aws/meta-data/local-hostname") + if err != nil { + return "", fmt.Errorf("error reading local hostname from AWS metadata: %v", err) + } + + // The local-hostname gets it's hostname from the AWS DHCP Option Set, which + // may provide multiple hostnames separated by spaces. For now just choose + // the first one as the hostname. + domains := strings.Fields(string(vBytes)) + if len(domains) == 0 { + glog.Warningf("Local hostname from AWS metadata service was empty") + return "", nil + } + domain := domains[0] + + glog.Infof("Using hostname from AWS metadata service: %s", domain) + + return domain, nil +} + +// FindCert is a helper method to retrieving a certificate from the store +func (c *NodeupModelContext) FindCert(name string) ([]byte, error) { + cert, err := c.KeyStore.FindCert(name) + if err != nil { + return []byte{}, fmt.Errorf("error fetching certificate: %v from keystore: %v", name, err) + } + if cert == nil { + return []byte{}, fmt.Errorf("unable to found certificate: %s", name) + } + + return cert.AsBytes() +} + +// FindPrivateKey is a helper method to retrieving a private key from the store +func (c *NodeupModelContext) FindPrivateKey(name string) ([]byte, error) { + key, err := c.KeyStore.FindPrivateKey(name) + if err != nil { + return []byte{}, fmt.Errorf("error fetching private key: %v from keystore: %v", name, err) + } + if key == nil { + return []byte{}, fmt.Errorf("unable to found private key: %s", name) + } + + return key.AsBytes() +} diff --git a/nodeup/pkg/model/kube_controller_manager.go b/nodeup/pkg/model/kube_controller_manager.go index b82401b19d..7065144225 100644 --- a/nodeup/pkg/model/kube_controller_manager.go +++ b/nodeup/pkg/model/kube_controller_manager.go @@ -104,7 +104,7 @@ func (b *KubeControllerManagerBuilder) Build(c *fi.ModelBuilderContext) error { // Add kubeconfig { // @TODO: Change kubeconfig to be https - kubeconfig, err := b.buildPKIKubeconfig("kube-controller-manager") + kubeconfig, err := b.BuildPKIKubeconfig("kube-controller-manager") if err != nil { return err } diff --git a/nodeup/pkg/model/kube_proxy.go b/nodeup/pkg/model/kube_proxy.go index e4c06ae194..8b6d17ae01 100644 --- a/nodeup/pkg/model/kube_proxy.go +++ b/nodeup/pkg/model/kube_proxy.go @@ -77,7 +77,7 @@ func (b *KubeProxyBuilder) Build(c *fi.ModelBuilderContext) error { } { - kubeconfig, err := b.buildPKIKubeconfig("kube-proxy") + kubeconfig, err := b.BuildPKIKubeconfig("kube-proxy") if err != nil { return err } diff --git a/nodeup/pkg/model/kube_router.go b/nodeup/pkg/model/kube_router.go index 50872b8526..9c1f0f08a6 100644 --- a/nodeup/pkg/model/kube_router.go +++ b/nodeup/pkg/model/kube_router.go @@ -32,7 +32,7 @@ func (b *KubeRouterBuilder) Build(c *fi.ModelBuilderContext) error { // Add kubeconfig { - kubeconfig, err := b.buildPKIKubeconfig("kube-router") + kubeconfig, err := b.BuildPKIKubeconfig("kube-router") if err != nil { return err } diff --git a/nodeup/pkg/model/kube_scheduler.go b/nodeup/pkg/model/kube_scheduler.go index e54de53740..2fa68a4095 100644 --- a/nodeup/pkg/model/kube_scheduler.go +++ b/nodeup/pkg/model/kube_scheduler.go @@ -64,7 +64,7 @@ func (b *KubeSchedulerBuilder) Build(c *fi.ModelBuilderContext) error { } { - kubeconfig, err := b.buildPKIKubeconfig("kube-scheduler") + kubeconfig, err := b.BuildPKIKubeconfig("kube-scheduler") if err != nil { return err } diff --git a/nodeup/pkg/model/kubectl.go b/nodeup/pkg/model/kubectl.go index 5badc02974..f290aed780 100644 --- a/nodeup/pkg/model/kubectl.go +++ b/nodeup/pkg/model/kubectl.go @@ -61,7 +61,7 @@ func (b *KubectlBuilder) Build(c *fi.ModelBuilderContext) error { } { - kubeconfig, err := b.buildPKIKubeconfig("kubecfg") + kubeconfig, err := b.BuildPKIKubeconfig("kubecfg") if err != nil { return err } diff --git a/nodeup/pkg/model/kubelet.go b/nodeup/pkg/model/kubelet.go index c46a7db52e..0e067058ec 100644 --- a/nodeup/pkg/model/kubelet.go +++ b/nodeup/pkg/model/kubelet.go @@ -17,25 +17,33 @@ limitations under the License. package model import ( + "crypto/x509" + "crypto/x509/pkix" "fmt" "path" "path/filepath" + "time" + + "k8s.io/kops/nodeup/pkg/distros" + "k8s.io/kops/pkg/apis/kops" + "k8s.io/kops/pkg/flagbuilder" + "k8s.io/kops/pkg/pki" + "k8s.io/kops/pkg/systemd" + "k8s.io/kops/upup/pkg/fi" + "k8s.io/kops/upup/pkg/fi/nodeup/nodetasks" + "k8s.io/kops/upup/pkg/fi/utils" "github.com/aws/aws-sdk-go/aws/ec2metadata" "github.com/aws/aws-sdk-go/aws/session" "github.com/golang/glog" "k8s.io/api/core/v1" - "k8s.io/kops/nodeup/pkg/distros" - "k8s.io/kops/pkg/apis/kops" - "k8s.io/kops/pkg/flagbuilder" - "k8s.io/kops/pkg/systemd" - "k8s.io/kops/upup/pkg/fi" - "k8s.io/kops/upup/pkg/fi/nodeup/nodetasks" - "k8s.io/kops/upup/pkg/fi/utils" + "k8s.io/apiserver/pkg/authentication/user" ) -// containerizedMounterHome is the path where we install the containerized mounter (on ContainerOS) -const containerizedMounterHome = "/home/kubernetes/containerized_mounter" +const ( + // containerizedMounterHome is the path where we install the containerized mounter (on ContainerOS) + containerizedMounterHome = "/home/kubernetes/containerized_mounter" +) // KubeletBuilder installs kubelet type KubeletBuilder struct { @@ -72,36 +80,61 @@ func (b *KubeletBuilder) Build(c *fi.ModelBuilderContext) error { return fmt.Errorf("unable to locate asset %q", assetName) } - t := &nodetasks.File{ + c.AddTask(&nodetasks.File{ Path: b.kubeletPath(), Contents: asset, Type: nodetasks.FileType_File, Mode: s("0755"), - } - c.AddTask(t) + }) } { - // @TODO Change kubeconfig to be https - kubeconfig, err := b.buildPKIKubeconfig("kubelet") - if err != nil { - return err + if b.UseBootstrapTokens() { + glog.V(3).Info("kubelet bootstrap tokens are enabled") + + // @check if a master and if so, we bypass the token strapping and instead generate our own kubeconfig + if b.IsMaster { + task, err := b.buildMasterKubeletKubeconfig() + if err != nil { + return err + } + c.AddTask(task) + + name := "node-authorizer" + if err := b.BuildCertificatePairTask(c, name, "node-authorizer/", "tls"); err != nil { + return err + } + + } else { + name := "node-authorizer-client" + if err := b.BuildCertificatePairTask(c, name, "node-authorizer/", "tls"); err != nil { + return err + } + glog.V(3).Info("kubelet service will wait for bootstrap configuration: %s", b.KubeletBootstrapConfig()) + } + if err := b.BuildCertificateTask(c, fi.CertificateId_CA, "node-authorizer/ca.pem"); err != nil { + return err + } + } else { + kubeconfig, err := b.BuildPKIKubeconfig("kubelet") + if err != nil { + return err + } + + c.AddTask(&nodetasks.File{ + Path: b.KubeletKubeConfig(), + Contents: fi.NewStringResource(kubeconfig), + Type: nodetasks.FileType_File, + Mode: s("0400"), + }) } - t := &nodetasks.File{ - Path: "/var/lib/kubelet/kubeconfig", - Contents: fi.NewStringResource(kubeconfig), - Type: nodetasks.FileType_File, - Mode: s("0400"), - } - c.AddTask(t) } if b.UsesCNI() { - t := &nodetasks.File{ + c.AddTask(&nodetasks.File{ Path: b.CNIConfDir(), Type: nodetasks.FileType_Directory, - } - c.AddTask(t) + }) } if err := b.addStaticUtils(c); err != nil { @@ -131,6 +164,11 @@ func (b *KubeletBuilder) kubeletPath() string { // buildSystemdEnvironmentFile renders the environment file for the kubelet func (b *KubeletBuilder) buildSystemdEnvironmentFile(kubeletConfig *kops.KubeletConfigSpec) (*nodetasks.File, error) { + // @step: ensure the masters do not get a bootstrap configuration + if b.UseBootstrapTokens() && b.IsMaster { + kubeletConfig.BootstrapKubeconfig = "" + } + // TODO: Dump the separate file for flags - just complexity! flags, err := flagbuilder.BuildFlags(kubeletConfig) if err != nil { @@ -184,6 +222,7 @@ func (b *KubeletBuilder) buildSystemdEnvironmentFile(kubeletConfig *kops.Kubelet Contents: fi.NewStringResource(sysconfig), Type: nodetasks.FileType_File, } + return t, nil } @@ -200,8 +239,14 @@ func (b *KubeletBuilder) buildSystemdService() *nodetasks.Service { // We add /opt/kubernetes/bin for our utilities (socat) manifest.Set("Service", "Environment", "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/kubernetes/bin") } - manifest.Set("Service", "EnvironmentFile", "/etc/sysconfig/kubelet") + + // @check if we are using bootstrap tokens and file checker + if !b.IsMaster && b.UseBootstrapTokens() { + manifest.Set("Service", "ExecStartPre", + fmt.Sprintf("/usr/bin/bash -c 'while [ ! -f %s ]; do sleep 5; done;'", b.KubeletBootstrapConfig())) + } + manifest.Set("Service", "ExecStart", kubeletCommand+" \"$DAEMON_ARGS\"") manifest.Set("Service", "Restart", "always") manifest.Set("Service", "RestartSec", "2s") @@ -227,10 +272,12 @@ func (b *KubeletBuilder) buildSystemdService() *nodetasks.Service { return service } +// buildKubeletConfig is responsible for creating the kubelet configuration func (b *KubeletBuilder) buildKubeletConfig() (*kops.KubeletConfigSpec, error) { if b.InstanceGroup == nil { glog.Fatalf("InstanceGroup was not set") } + kubeletConfigSpec, err := b.buildKubeletConfigSpec() if err != nil { return nil, fmt.Errorf("error building kubelet config: %v", err) @@ -401,6 +448,10 @@ func (b *KubeletBuilder) buildKubeletConfigSpec() (*kops.KubeletConfigSpec, erro c.ClientCAFile = filepath.Join(b.PathSrvKubernetes(), "ca.crt") } + if b.IsMaster { + c.BootstrapKubeconfig = "" + } + if b.InstanceGroup.Spec.Kubelet != nil { utils.JsonMergeStruct(c, b.InstanceGroup.Spec.Kubelet) } @@ -446,3 +497,88 @@ func (b *KubeletBuilder) buildKubeletConfigSpec() (*kops.KubeletConfigSpec, erro return c, nil } + +// buildMasterKubeletKubeconfig builds a kubeconfig for the master kubelet, self-signing the kubelet cert +func (b *KubeletBuilder) buildMasterKubeletKubeconfig() (*nodetasks.File, error) { + nodeName, err := b.NodeName() + if err != nil { + return nil, fmt.Errorf("error getting NodeName: %v", err) + } + + caCert, err := b.KeyStore.FindCert(fi.CertificateId_CA) + if err != nil { + return nil, fmt.Errorf("error fetching CA certificate from keystore: %v", err) + } + if caCert == nil { + return nil, fmt.Errorf("unable to find CA certificate %q in keystore", fi.CertificateId_CA) + } + + caKey, err := b.KeyStore.FindPrivateKey(fi.CertificateId_CA) + if err != nil { + return nil, fmt.Errorf("error fetching CA certificate from keystore: %v", err) + } + if caKey == nil { + return nil, fmt.Errorf("unable to find CA key %q in keystore", fi.CertificateId_CA) + } + + privateKey, err := pki.GeneratePrivateKey() + if err != nil { + return nil, err + } + + template := &x509.Certificate{ + BasicConstraintsValid: true, + IsCA: false, + } + + template.Subject = pkix.Name{ + CommonName: fmt.Sprintf("system:node:%s", nodeName), + Organization: []string{user.NodesGroup}, + } + + // https://tools.ietf.org/html/rfc5280#section-4.2.1.3 + // + // Digital signature allows the certificate to be used to verify + // digital signatures used during TLS negotiation. + template.KeyUsage = template.KeyUsage | x509.KeyUsageDigitalSignature + // KeyEncipherment allows the cert/key pair to be used to encrypt + // keys, including the symmetric keys negotiated during TLS setup + // and used for data transfer. + template.KeyUsage = template.KeyUsage | x509.KeyUsageKeyEncipherment + // ClientAuth allows the cert to be used by a TLS client to + // authenticate itself to the TLS server. + template.ExtKeyUsage = append(template.ExtKeyUsage, x509.ExtKeyUsageClientAuth) + + t := time.Now().UnixNano() + template.SerialNumber = pki.BuildPKISerial(t) + + certificate, err := pki.SignNewCertificate(privateKey, template, caCert.Certificate, caKey) + if err != nil { + return nil, fmt.Errorf("error signing certificate for master kubelet: %v", err) + } + + caBytes, err := caCert.AsBytes() + if err != nil { + return nil, fmt.Errorf("failed to get certificate authority data: %s", err) + } + certBytes, err := certificate.AsBytes() + if err != nil { + return nil, fmt.Errorf("failed to get certificate data: %s", err) + } + keyBytes, err := privateKey.AsBytes() + if err != nil { + return nil, fmt.Errorf("failed to get private key data: %s", err) + } + + content, err := b.BuildKubeConfig("kubelet", caBytes, certBytes, keyBytes) + if err != nil { + return nil, err + } + + return &nodetasks.File{ + Path: b.KubeletKubeConfig(), + Contents: fi.NewStringResource(content), + Type: nodetasks.FileType_File, + Mode: s("600"), + }, nil +} diff --git a/nodeup/pkg/model/protokube.go b/nodeup/pkg/model/protokube.go index 0b2e010e1c..7dc3a6f627 100644 --- a/nodeup/pkg/model/protokube.go +++ b/nodeup/pkg/model/protokube.go @@ -55,7 +55,7 @@ func (t *ProtokubeBuilder) Build(c *fi.ModelBuilderContext) error { } if t.IsMaster { - kubeconfig, err := t.buildPKIKubeconfig("kops") + kubeconfig, err := t.BuildPKIKubeconfig("kops") if err != nil { return err } @@ -75,7 +75,7 @@ func (t *ProtokubeBuilder) Build(c *fi.ModelBuilderContext) error { } } for _, x := range []string{"etcd", "etcd-client"} { - if err := t.BuildPrivateTask(c, x, fmt.Sprintf("%s-key.pem", x)); err != nil { + if err := t.BuildPrivateKeyTask(c, x, fmt.Sprintf("%s-key.pem", x)); err != nil { return err } } @@ -281,10 +281,9 @@ func (t *ProtokubeBuilder) ProtokubeFlags(k8sVersion semver.Version) (*Protokube remapped, err := assets.RemapImage(image) if err != nil { return nil, fmt.Errorf("unable to remap container %q: %v", image, err) - } else { - image = remapped } + image = remapped f.EtcdImage = s(image) // initialize rbac on Kubernetes >= 1.6 and master diff --git a/pkg/apis/kops/componentconfig.go b/pkg/apis/kops/componentconfig.go index 6ccf11a1e2..9430d1d353 100644 --- a/pkg/apis/kops/componentconfig.go +++ b/pkg/apis/kops/componentconfig.go @@ -26,6 +26,8 @@ type KubeletConfigSpec struct { AnonymousAuth *bool `json:"anonymousAuth,omitempty" flag:"anonymous-auth"` // AuthorizationMode is the authorization mode the kubelet is running in AuthorizationMode string `json:"authorizationMode,omitempty" flag:"authorization-mode"` + // BootstrapKubeconfig is the path to a kubeconfig file that will be used to get client certificate for kubelet + BootstrapKubeconfig string `json:"bootstrapKubeconfig,omitempty" flag:"bootstrap-kubeconfig"` // ClientCAFile is the path to a CA certificate ClientCAFile string `json:"clientCaFile,omitempty" flag:"client-ca-file"` // TODO: Remove unused TLSCertFile @@ -216,6 +218,8 @@ type KubeAPIServerConfig struct { BindAddress string `json:"bindAddress,omitempty" flag:"bind-address"` // InsecureBindAddress is the binding address for the InsecurePort for the insecure kubernetes API InsecureBindAddress string `json:"insecureBindAddress,omitempty" flag:"insecure-bind-address"` + // EnableBootstrapAuthToken enables 'bootstrap.kubernetes.io/token' in the 'kube-system' namespace to be used for TLS bootstrapping authentication + EnableBootstrapAuthToken *bool `json:"enableBootstrapTokenAuth,omitempty" flag:"enable-bootstrap-token-auth"` // Deprecated: AdmissionControl is a list of admission controllers to use AdmissionControl []string `json:"admissionControl,omitempty" flag:"admission-control"` // EnableAdmissionPlugins is a list of enabled admission plugins @@ -395,6 +399,7 @@ type KubeControllerManagerConfig struct { FeatureGates map[string]string `json:"featureGates,omitempty" flag:"feature-gates"` } +// CloudControllerManagerConfig is the configuration of the cloud controller type CloudControllerManagerConfig struct { // Master is the url for the kube api master. Master string `json:"master,omitempty" flag:"master"` diff --git a/pkg/apis/kops/v1alpha1/componentconfig.go b/pkg/apis/kops/v1alpha1/componentconfig.go index 05c5386630..0927defbe8 100644 --- a/pkg/apis/kops/v1alpha1/componentconfig.go +++ b/pkg/apis/kops/v1alpha1/componentconfig.go @@ -26,6 +26,8 @@ type KubeletConfigSpec struct { AnonymousAuth *bool `json:"anonymousAuth,omitempty" flag:"anonymous-auth"` // AuthorizationMode is the authorization mode the kubelet is running in AuthorizationMode string `json:"authorizationMode,omitempty" flag:"authorization-mode"` + // BootstrapKubeconfig is the path to a kubeconfig file that will be used to get client certificate for kube + BootstrapKubeconfig string `json:"bootstrapKubeconfig,omitempty" flag:"bootstrap-kubeconfig"` // ClientCAFile is the path to a CA certificate ClientCAFile string `json:"clientCaFile,omitempty" flag:"client-ca-file"` // TODO: Remove unused TLSCertFile @@ -216,6 +218,8 @@ type KubeAPIServerConfig struct { BindAddress string `json:"bindAddress,omitempty" flag:"bind-address"` // InsecureBindAddress is the binding address for the InsecurePort for the insecure kubernetes API InsecureBindAddress string `json:"insecureBindAddress,omitempty" flag:"insecure-bind-address"` + // EnableBootstrapAuthToken enables 'bootstrap.kubernetes.io/token' in the 'kube-system' namespace to be used for TLS bootstrapping authentication + EnableBootstrapAuthToken *bool `json:"enableBootstrapTokenAuth,omitempty" flag:"enable-bootstrap-token-auth"` // Deprecated: AdmissionControl is a list of admission controllers to use AdmissionControl []string `json:"admissionControl,omitempty" flag:"admission-control"` // EnableAdmissionPlugins is a list of enabled admission plugins @@ -395,6 +399,7 @@ type KubeControllerManagerConfig struct { FeatureGates map[string]string `json:"featureGates,omitempty" flag:"feature-gates"` } +// CloudControllerManagerConfig is the configuration of the cloud controller type CloudControllerManagerConfig struct { // Master is the url for the kube api master. Master string `json:"master,omitempty" flag:"master"` diff --git a/pkg/apis/kops/v1alpha1/zz_generated.conversion.go b/pkg/apis/kops/v1alpha1/zz_generated.conversion.go index 91e8d1a75b..e93dda31e5 100644 --- a/pkg/apis/kops/v1alpha1/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha1/zz_generated.conversion.go @@ -2013,6 +2013,7 @@ func autoConvert_v1alpha1_KubeAPIServerConfig_To_kops_KubeAPIServerConfig(in *Ku out.Address = in.Address out.BindAddress = in.BindAddress out.InsecureBindAddress = in.InsecureBindAddress + out.EnableBootstrapAuthToken = in.EnableBootstrapAuthToken out.AdmissionControl = in.AdmissionControl out.EnableAdmissionPlugins = in.EnableAdmissionPlugins out.DisableAdmissionPlugins = in.DisableAdmissionPlugins @@ -2082,6 +2083,7 @@ func autoConvert_kops_KubeAPIServerConfig_To_v1alpha1_KubeAPIServerConfig(in *ko out.Address = in.Address out.BindAddress = in.BindAddress out.InsecureBindAddress = in.InsecureBindAddress + out.EnableBootstrapAuthToken = in.EnableBootstrapAuthToken out.AdmissionControl = in.AdmissionControl out.EnableAdmissionPlugins = in.EnableAdmissionPlugins out.DisableAdmissionPlugins = in.DisableAdmissionPlugins @@ -2352,6 +2354,7 @@ func autoConvert_v1alpha1_KubeletConfigSpec_To_kops_KubeletConfigSpec(in *Kubele out.APIServers = in.APIServers out.AnonymousAuth = in.AnonymousAuth out.AuthorizationMode = in.AuthorizationMode + out.BootstrapKubeconfig = in.BootstrapKubeconfig out.ClientCAFile = in.ClientCAFile out.TLSCertFile = in.TLSCertFile out.TLSPrivateKeyFile = in.TLSPrivateKeyFile @@ -2423,6 +2426,7 @@ func autoConvert_kops_KubeletConfigSpec_To_v1alpha1_KubeletConfigSpec(in *kops.K out.APIServers = in.APIServers out.AnonymousAuth = in.AnonymousAuth out.AuthorizationMode = in.AuthorizationMode + out.BootstrapKubeconfig = in.BootstrapKubeconfig out.ClientCAFile = in.ClientCAFile out.TLSCertFile = in.TLSCertFile out.TLSPrivateKeyFile = in.TLSPrivateKeyFile diff --git a/pkg/apis/kops/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/kops/v1alpha1/zz_generated.deepcopy.go index 8befd77eaa..01ac469c63 100644 --- a/pkg/apis/kops/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/kops/v1alpha1/zz_generated.deepcopy.go @@ -1756,6 +1756,15 @@ func (in *KopeioNetworkingSpec) DeepCopy() *KopeioNetworkingSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *KubeAPIServerConfig) DeepCopyInto(out *KubeAPIServerConfig) { *out = *in + if in.EnableBootstrapAuthToken != nil { + in, out := &in.EnableBootstrapAuthToken, &out.EnableBootstrapAuthToken + if *in == nil { + *out = nil + } else { + *out = new(bool) + **out = **in + } + } if in.AdmissionControl != nil { in, out := &in.AdmissionControl, &out.AdmissionControl *out = make([]string, len(*in)) diff --git a/pkg/apis/kops/v1alpha2/componentconfig.go b/pkg/apis/kops/v1alpha2/componentconfig.go index 6b6c76cef2..087fffbaa3 100644 --- a/pkg/apis/kops/v1alpha2/componentconfig.go +++ b/pkg/apis/kops/v1alpha2/componentconfig.go @@ -26,6 +26,8 @@ type KubeletConfigSpec struct { AnonymousAuth *bool `json:"anonymousAuth,omitempty" flag:"anonymous-auth"` // AuthorizationMode is the authorization mode the kubelet is running in AuthorizationMode string `json:"authorizationMode,omitempty" flag:"authorization-mode"` + // BootstrapKubeconfig is the path to a kubeconfig file that will be used to get client certificate for kubelet + BootstrapKubeconfig string `json:"bootstrapKubeconfig,omitempty" flag:"bootstrap-kubeconfig"` // ClientCAFile is the path to a CA certificate ClientCAFile string `json:"clientCaFile,omitempty" flag:"client-ca-file"` // TODO: Remove unused TLSCertFile @@ -216,6 +218,8 @@ type KubeAPIServerConfig struct { BindAddress string `json:"bindAddress,omitempty" flag:"bind-address"` // InsecureBindAddress is the binding address for the InsecurePort for the insecure kubernetes API InsecureBindAddress string `json:"insecureBindAddress,omitempty" flag:"insecure-bind-address"` + // EnableBootstrapAuthToken enables 'bootstrap.kubernetes.io/token' in the 'kube-system' namespace to be used for TLS bootstrapping authentication + EnableBootstrapAuthToken *bool `json:"enableBootstrapTokenAuth,omitempty" flag:"enable-bootstrap-token-auth"` // Deprecated: AdmissionControl is a list of admission controllers to use AdmissionControl []string `json:"admissionControl,omitempty" flag:"admission-control"` // EnableAdmissionPlugins is a list of enabled admission plugins @@ -395,6 +399,7 @@ type KubeControllerManagerConfig struct { FeatureGates map[string]string `json:"featureGates,omitempty" flag:"feature-gates"` } +// CloudControllerManagerConfig is the configuration of the cloud controller type CloudControllerManagerConfig struct { // Master is the url for the kube api master. Master string `json:"master,omitempty" flag:"master"` diff --git a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go index 3d46be6fd5..5673ef14b6 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go @@ -2277,6 +2277,7 @@ func autoConvert_v1alpha2_KubeAPIServerConfig_To_kops_KubeAPIServerConfig(in *Ku out.Address = in.Address out.BindAddress = in.BindAddress out.InsecureBindAddress = in.InsecureBindAddress + out.EnableBootstrapAuthToken = in.EnableBootstrapAuthToken out.AdmissionControl = in.AdmissionControl out.EnableAdmissionPlugins = in.EnableAdmissionPlugins out.DisableAdmissionPlugins = in.DisableAdmissionPlugins @@ -2346,6 +2347,7 @@ func autoConvert_kops_KubeAPIServerConfig_To_v1alpha2_KubeAPIServerConfig(in *ko out.Address = in.Address out.BindAddress = in.BindAddress out.InsecureBindAddress = in.InsecureBindAddress + out.EnableBootstrapAuthToken = in.EnableBootstrapAuthToken out.AdmissionControl = in.AdmissionControl out.EnableAdmissionPlugins = in.EnableAdmissionPlugins out.DisableAdmissionPlugins = in.DisableAdmissionPlugins @@ -2616,6 +2618,7 @@ func autoConvert_v1alpha2_KubeletConfigSpec_To_kops_KubeletConfigSpec(in *Kubele out.APIServers = in.APIServers out.AnonymousAuth = in.AnonymousAuth out.AuthorizationMode = in.AuthorizationMode + out.BootstrapKubeconfig = in.BootstrapKubeconfig out.ClientCAFile = in.ClientCAFile out.TLSCertFile = in.TLSCertFile out.TLSPrivateKeyFile = in.TLSPrivateKeyFile @@ -2687,6 +2690,7 @@ func autoConvert_kops_KubeletConfigSpec_To_v1alpha2_KubeletConfigSpec(in *kops.K out.APIServers = in.APIServers out.AnonymousAuth = in.AnonymousAuth out.AuthorizationMode = in.AuthorizationMode + out.BootstrapKubeconfig = in.BootstrapKubeconfig out.ClientCAFile = in.ClientCAFile out.TLSCertFile = in.TLSCertFile out.TLSPrivateKeyFile = in.TLSPrivateKeyFile diff --git a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go index eab8838408..163b9cdac0 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go @@ -1837,6 +1837,15 @@ func (in *KopeioNetworkingSpec) DeepCopy() *KopeioNetworkingSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *KubeAPIServerConfig) DeepCopyInto(out *KubeAPIServerConfig) { *out = *in + if in.EnableBootstrapAuthToken != nil { + in, out := &in.EnableBootstrapAuthToken, &out.EnableBootstrapAuthToken + if *in == nil { + *out = nil + } else { + *out = new(bool) + **out = **in + } + } if in.AdmissionControl != nil { in, out := &in.AdmissionControl, &out.AdmissionControl *out = make([]string, len(*in)) diff --git a/pkg/apis/kops/validation/legacy.go b/pkg/apis/kops/validation/legacy.go index 678975875e..fe0f25bccd 100644 --- a/pkg/apis/kops/validation/legacy.go +++ b/pkg/apis/kops/validation/legacy.go @@ -434,6 +434,12 @@ func ValidateCluster(c *kops.Cluster, strict bool) *field.Error { } } + if c.Spec.Kubelet.BootstrapKubeconfig != "" { + if c.Spec.KubeAPIServer == nil { + return field.Required(fieldSpec.Child("KubeAPIServer"), "bootstrap token require the NodeRestriction admissions controller") + } + } + if c.Spec.Kubelet.APIServers != "" && !isValidAPIServersURL(c.Spec.Kubelet.APIServers) { return field.Invalid(kubeletPath.Child("APIServers"), c.Spec.Kubelet.APIServers, "Not a valid APIServer URL") } diff --git a/pkg/apis/kops/zz_generated.deepcopy.go b/pkg/apis/kops/zz_generated.deepcopy.go index 87c048a200..dcd6a7d3cc 100644 --- a/pkg/apis/kops/zz_generated.deepcopy.go +++ b/pkg/apis/kops/zz_generated.deepcopy.go @@ -2016,6 +2016,15 @@ func (in *KopsVersionSpec) DeepCopy() *KopsVersionSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *KubeAPIServerConfig) DeepCopyInto(out *KubeAPIServerConfig) { *out = *in + if in.EnableBootstrapAuthToken != nil { + in, out := &in.EnableBootstrapAuthToken, &out.EnableBootstrapAuthToken + if *in == nil { + *out = nil + } else { + *out = new(bool) + **out = **in + } + } if in.AdmissionControl != nil { in, out := &in.AdmissionControl, &out.AdmissionControl *out = make([]string, len(*in)) diff --git a/pkg/model/components/apiserver.go b/pkg/model/components/apiserver.go index 7891796ea4..bb86407581 100644 --- a/pkg/model/components/apiserver.go +++ b/pkg/model/components/apiserver.go @@ -90,7 +90,17 @@ func (b *KubeAPIServerOptionsBuilder) BuildOptions(o interface{}) error { } else if clusterSpec.Authorization.AlwaysAllow != nil { clusterSpec.KubeAPIServer.AuthorizationMode = fi.String("AlwaysAllow") } else if clusterSpec.Authorization.RBAC != nil { - clusterSpec.KubeAPIServer.AuthorizationMode = fi.String("RBAC") + var modes []string + + if b.IsKubernetesGTE("1.10") { + if fi.BoolValue(clusterSpec.KubeAPIServer.EnableBootstrapAuthToken) { + // Enable the Node authorizer, used for special per-node RBAC policies + modes = append(modes, "Node") + } + } + modes = append(modes, "RBAC") + + clusterSpec.KubeAPIServer.AuthorizationMode = fi.String(strings.Join(modes, ",")) } if clusterSpec.KubeAPIServer.EtcdQuorumRead == nil { diff --git a/pkg/model/components/kubelet.go b/pkg/model/components/kubelet.go index ad9ed25508..24e6ed9a87 100644 --- a/pkg/model/components/kubelet.go +++ b/pkg/model/components/kubelet.go @@ -32,6 +32,7 @@ type KubeletOptionsBuilder struct { var _ loader.OptionsBuilder = &KubeletOptionsBuilder{} +// BuildOptions is responsible for filling the defaults for the kubelet func (b *KubeletOptionsBuilder) BuildOptions(o interface{}) error { clusterSpec := o.(*kops.ClusterSpec) @@ -52,6 +53,14 @@ func (b *KubeletOptionsBuilder) BuildOptions(o interface{}) error { return err } + if clusterSpec.KubeAPIServer != nil && clusterSpec.KubeAPIServer.EnableBootstrapAuthToken != nil { + if *clusterSpec.KubeAPIServer.EnableBootstrapAuthToken { + if clusterSpec.Kubelet.BootstrapKubeconfig == "" { + clusterSpec.Kubelet.BootstrapKubeconfig = "/var/lib/kubelet/bootstrap-kubeconfig" + } + } + } + // Standard options clusterSpec.Kubelet.EnableDebuggingHandlers = fi.Bool(true) clusterSpec.Kubelet.PodManifestPath = "/etc/kubernetes/manifests" diff --git a/pkg/model/context.go b/pkg/model/context.go index ecc3be3cef..1ea97864ba 100644 --- a/pkg/model/context.go +++ b/pkg/model/context.go @@ -23,16 +23,18 @@ import ( "net" "strings" - "github.com/blang/semver" - "github.com/golang/glog" - utilnet "k8s.io/apimachinery/pkg/util/net" "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/pkg/apis/kops/model" "k8s.io/kops/pkg/apis/kops/util" "k8s.io/kops/pkg/featureflag" "k8s.io/kops/pkg/model/components" + "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/cloudup/awstasks" "k8s.io/kops/upup/pkg/fi/cloudup/awsup" + + "github.com/blang/semver" + "github.com/golang/glog" + utilnet "k8s.io/apimachinery/pkg/util/net" ) const ( @@ -42,16 +44,15 @@ const ( var UseLegacyELBName = featureflag.New("UseLegacyELBName", featureflag.Bool(false)) +// KopsModelContext is the kops model type KopsModelContext struct { - Cluster *kops.Cluster - - Region string + Cluster *kops.Cluster InstanceGroups []*kops.InstanceGroup - - SSHPublicKeys [][]byte + Region string + SSHPublicKeys [][]byte } -// Will attempt to calculate a meaningful name for an ELB given a prefix +// GetELBName32 will attempt to calculate a meaningful name for an ELB given a prefix // Will never return a string longer than 32 chars // Note this is _not_ the primary identifier for the ELB - we use the Name tag for that. func (m *KopsModelContext) GetELBName32(prefix string) string { @@ -248,6 +249,16 @@ func (m *KopsModelContext) CloudTags(name string, shared bool) map[string]string return tags } +// UseBootstrapTokens checks if bootstrap tokens are enabled +func (m *KopsModelContext) UseBootstrapTokens() bool { + if m.Cluster.Spec.KubeAPIServer == nil { + return false + } + + return fi.BoolValue(m.Cluster.Spec.KubeAPIServer.EnableBootstrapAuthToken) +} + +// UsesBastionDns checks if we should use a specific name for the bastion dns func (m *KopsModelContext) UsesBastionDns() bool { if m.Cluster.Spec.Topology.Bastion != nil && m.Cluster.Spec.Topology.Bastion.BastionPublicName != "" { return true @@ -255,6 +266,7 @@ func (m *KopsModelContext) UsesBastionDns() bool { return false } +// UsesSSHBastion checks if we have a Bastion in the cluster func (m *KopsModelContext) UsesSSHBastion() bool { for _, ig := range m.InstanceGroups { if ig.Spec.Role == kops.InstanceGroupRoleBastion { @@ -265,6 +277,7 @@ func (m *KopsModelContext) UsesSSHBastion() bool { return false } +// UseLoadBalancerForAPI checks if we are using a load balancer for the kubeapi func (m *KopsModelContext) UseLoadBalancerForAPI() bool { if m.Cluster.Spec.API == nil { return false @@ -272,6 +285,7 @@ func (m *KopsModelContext) UseLoadBalancerForAPI() bool { return m.Cluster.Spec.API.LoadBalancer != nil } +// UsePrivateDNS checks if we are using private DNS func (m *KopsModelContext) UsePrivateDNS() bool { topology := m.Cluster.Spec.Topology if topology != nil && topology.DNS != nil { diff --git a/pkg/model/iam/iam_builder.go b/pkg/model/iam/iam_builder.go index 0c3655e907..053721033b 100644 --- a/pkg/model/iam/iam_builder.go +++ b/pkg/model/iam/iam_builder.go @@ -348,12 +348,30 @@ func (b *PolicyBuilder) AddS3Permissions(p *Policy) (*Policy, error) { strings.Join([]string{b.IAMPrefix(), ":s3:::", iamS3Path, "/instancegroup/*"}, ""), strings.Join([]string{b.IAMPrefix(), ":s3:::", iamS3Path, "/pki/issued/*"}, ""), strings.Join([]string{b.IAMPrefix(), ":s3:::", iamS3Path, "/pki/private/kube-proxy/*"}, ""), - strings.Join([]string{b.IAMPrefix(), ":s3:::", iamS3Path, "/pki/private/kubelet/*"}, ""), strings.Join([]string{b.IAMPrefix(), ":s3:::", iamS3Path, "/pki/ssh/*"}, ""), strings.Join([]string{b.IAMPrefix(), ":s3:::", iamS3Path, "/secrets/dockerconfig"}, ""), ), }) + // @check if bootstrap tokens are enabled and if so enable access to client certificate + if b.UseBootstrapTokens() { + p.Statement = append(p.Statement, &Statement{ + Effect: StatementEffectAllow, + Action: stringorslice.Slice([]string{"s3:Get*"}), + Resource: stringorslice.Of( + strings.Join([]string{b.IAMPrefix(), ":s3:::", iamS3Path, "/pki/private/node-authorizer-client/*"}, ""), + ), + }) + } else { + p.Statement = append(p.Statement, &Statement{ + Effect: StatementEffectAllow, + Action: stringorslice.Slice([]string{"s3:Get*"}), + Resource: stringorslice.Of( + strings.Join([]string{b.IAMPrefix(), ":s3:::", iamS3Path, "/pki/private/kubelet/*"}, ""), + ), + }) + } + if b.Cluster.Spec.Networking != nil { // @check if kuberoute is enabled and permit access to the private key if b.Cluster.Spec.Networking.Kuberouter != nil { @@ -469,6 +487,16 @@ func (b *PolicyResource) Open() (io.Reader, error) { return bytes.NewReader([]byte(j)), nil } +// UseBootstrapTokens check if we are using bootstrap tokens - @TODO, i don't like this we should probably pass in +// the kops model into the builder rather than duplicating the code. I'll leave for anothe PR +func (b *PolicyBuilder) UseBootstrapTokens() bool { + if b.Cluster.Spec.KubeAPIServer == nil { + return false + } + + return fi.BoolValue(b.Cluster.Spec.KubeAPIServer.EnableBootstrapAuthToken) +} + func addECRPermissions(p *Policy) { // TODO - I think we can just have GetAuthorizationToken here, as we are not // TODO - making any API calls except for GetAuthorizationToken. @@ -755,59 +783,59 @@ func addRomanaCNIPermissions(p *Policy, resource stringorslice.StringOrSlice, le if legacyIAM { // Legacy IAM provides ec2:*, so no additional permissions required return - } else { - // Romana requires additional Describe permissions - // Comments are which Romana component makes the call - p.Statement = append(p.Statement, - &Statement{ - Effect: StatementEffectAllow, - Action: stringorslice.Slice([]string{ - "ec2:DescribeAvailabilityZones", // vpcrouter - "ec2:DescribeVpcs", // vpcrouter - }), - Resource: resource, - }, - &Statement{ - Effect: StatementEffectAllow, - Action: stringorslice.Slice([]string{ - "ec2:CreateRoute", // vpcrouter - "ec2:DeleteRoute", // vpcrouter - "ec2:ReplaceRoute", // vpcrouter - }), - Resource: resource, - Condition: Condition{ - "StringEquals": map[string]string{ - "ec2:ResourceTag/KubernetesCluster": clusterName, - }, + } + + // Romana requires additional Describe permissions + // Comments are which Romana component makes the call + p.Statement = append(p.Statement, + &Statement{ + Effect: StatementEffectAllow, + Action: stringorslice.Slice([]string{ + "ec2:DescribeAvailabilityZones", // vpcrouter + "ec2:DescribeVpcs", // vpcrouter + }), + Resource: resource, + }, + &Statement{ + Effect: StatementEffectAllow, + Action: stringorslice.Slice([]string{ + "ec2:CreateRoute", // vpcrouter + "ec2:DeleteRoute", // vpcrouter + "ec2:ReplaceRoute", // vpcrouter + }), + Resource: resource, + Condition: Condition{ + "StringEquals": map[string]string{ + "ec2:ResourceTag/KubernetesCluster": clusterName, }, }, - ) - } + }, + ) } func addAmazonVPCCNIPermissions(p *Policy, resource stringorslice.StringOrSlice, legacyIAM bool, clusterName string) { if legacyIAM { // Legacy IAM provides ec2:*, so no additional permissions required return - } else { - p.Statement = append(p.Statement, - &Statement{ - Effect: StatementEffectAllow, - Action: stringorslice.Slice([]string{ - "ec2:CreateNetworkInterface", - "ec2:AttachNetworkInterface", - "ec2:DeleteNetworkInterface", - "ec2:DetachNetworkInterface", - "ec2:DescribeNetworkInterfaces", - "ec2:DescribeInstances", - "ec2:ModifyNetworkInterfaceAttribute", - "ec2:AssignPrivateIpAddresses", - "tag:TagResources", - }), - Resource: resource, - }, - ) } + + p.Statement = append(p.Statement, + &Statement{ + Effect: StatementEffectAllow, + Action: stringorslice.Slice([]string{ + "ec2:CreateNetworkInterface", + "ec2:AttachNetworkInterface", + "ec2:DeleteNetworkInterface", + "ec2:DetachNetworkInterface", + "ec2:DescribeNetworkInterfaces", + "ec2:DescribeInstances", + "ec2:ModifyNetworkInterfaceAttribute", + "ec2:AssignPrivateIpAddresses", + "tag:TagResources", + }), + Resource: resource, + }, + ) } func createResource(b *PolicyBuilder) stringorslice.StringOrSlice { diff --git a/pkg/model/iam/tests/iam_builder_node_strict.json b/pkg/model/iam/tests/iam_builder_node_strict.json index 1d28f2718a..5263298294 100644 --- a/pkg/model/iam/tests/iam_builder_node_strict.json +++ b/pkg/model/iam/tests/iam_builder_node_strict.json @@ -33,10 +33,16 @@ "arn:aws:s3:::kops-tests/iam-builder-test.k8s.local/instancegroup/*", "arn:aws:s3:::kops-tests/iam-builder-test.k8s.local/pki/issued/*", "arn:aws:s3:::kops-tests/iam-builder-test.k8s.local/pki/private/kube-proxy/*", - "arn:aws:s3:::kops-tests/iam-builder-test.k8s.local/pki/private/kubelet/*", "arn:aws:s3:::kops-tests/iam-builder-test.k8s.local/pki/ssh/*", "arn:aws:s3:::kops-tests/iam-builder-test.k8s.local/secrets/dockerconfig" ] + }, + { + "Effect": "Allow", + "Action": [ + "s3:Get*" + ], + "Resource": "arn:aws:s3:::kops-tests/iam-builder-test.k8s.local/pki/private/kubelet/*" } ] } diff --git a/pkg/model/iam/tests/iam_builder_node_strict_ecr.json b/pkg/model/iam/tests/iam_builder_node_strict_ecr.json index 10053f1c0d..75110b3b87 100644 --- a/pkg/model/iam/tests/iam_builder_node_strict_ecr.json +++ b/pkg/model/iam/tests/iam_builder_node_strict_ecr.json @@ -33,11 +33,17 @@ "arn:aws:s3:::kops-tests/iam-builder-test.k8s.local/instancegroup/*", "arn:aws:s3:::kops-tests/iam-builder-test.k8s.local/pki/issued/*", "arn:aws:s3:::kops-tests/iam-builder-test.k8s.local/pki/private/kube-proxy/*", - "arn:aws:s3:::kops-tests/iam-builder-test.k8s.local/pki/private/kubelet/*", "arn:aws:s3:::kops-tests/iam-builder-test.k8s.local/pki/ssh/*", "arn:aws:s3:::kops-tests/iam-builder-test.k8s.local/secrets/dockerconfig" ] }, + { + "Effect": "Allow", + "Action": [ + "s3:Get*" + ], + "Resource": "arn:aws:s3:::kops-tests/iam-builder-test.k8s.local/pki/private/kubelet/*" + }, { "Effect": "Allow", "Action": [ diff --git a/pkg/model/pki.go b/pkg/model/pki.go index 1ead4ef87c..a983ff6b51 100644 --- a/pkg/model/pki.go +++ b/pkg/model/pki.go @@ -18,12 +18,14 @@ package model import ( "fmt" + "strings" - "k8s.io/apiserver/pkg/authentication/user" "k8s.io/kops/pkg/tokens" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/fitasks" "k8s.io/kops/util/pkg/vfs" + + "k8s.io/apiserver/pkg/authentication/user" ) // PKIModelBuilder configures PKI keypairs, as well as tokens @@ -52,17 +54,18 @@ func (b *PKIModelBuilder) Build(c *fi.ModelBuilderContext) error { c.AddTask(defaultCA) { - - t := &fitasks.Keypair{ - Name: fi.String("kubelet"), - Lifecycle: b.Lifecycle, - - Subject: "o=" + user.NodesGroup + ",cn=kubelet", - Type: "client", - Signer: defaultCA, - Format: format, + // @check of bootstrap tokens are enable if so, disable the creation of the kubelet certificate - we also + // block at the IAM level for AWS cluster for pre-existing clusters. + if !b.UseBootstrapTokens() { + c.AddTask(&fitasks.Keypair{ + Name: fi.String("kubelet"), + Lifecycle: b.Lifecycle, + Subject: "o=" + user.NodesGroup + ",cn=kubelet", + Type: "client", + Signer: defaultCA, + Format: format, + }) } - c.AddTask(t) } { // Generate a kubelet client certificate for api to speak securely to kubelets. This change was first @@ -279,10 +282,43 @@ func (b *PKIModelBuilder) Build(c *fi.ModelBuilderContext) error { } } + // @TODO this is VERY presumptuous, i'm going on the basis we can make it configurable in the future. + // But I'm conscious not to do too much work on bootstrap tokens as it might overlay further down the + // line with the machines api + if b.UseBootstrapTokens() { + serviceName := "node-authorizer-internal" + + alternateNames := []string{ + "127.0.0.1", + "localhost", + serviceName, + strings.Join([]string{serviceName, b.Cluster.Name}, "."), + strings.Join([]string{serviceName, b.Cluster.Spec.DNSZone}, "."), + } + + // @note: the certificate used by the node authorizers + c.AddTask(&fitasks.Keypair{ + Name: fi.String("node-authorizer"), + Subject: "cn=node-authorizaer", + Type: "server", + AlternateNames: alternateNames, + Signer: defaultCA, + Format: format, + }) + + // @note: we use this for mutual tls between between node and authorizer + c.AddTask(&fitasks.Keypair{ + Name: fi.String("node-authorizer-client"), + Subject: "cn=node-authorizer-client", + Type: "client", + Signer: defaultCA, + Format: format, + }) + } + // Create auth tokens (though this is deprecated) for _, x := range tokens.GetKubernetesAuthTokens_Deprecated() { - t := &fitasks.Secret{Name: fi.String(x), Lifecycle: b.Lifecycle} - c.AddTask(t) + c.AddTask(&fitasks.Secret{Name: fi.String(x), Lifecycle: b.Lifecycle}) } { diff --git a/upup/models/cloudup/resources/addons/podsecuritypolicy.addons.k8s.io/k8s-1.10.yaml.template b/upup/models/cloudup/resources/addons/podsecuritypolicy.addons.k8s.io/k8s-1.10.yaml.template index b4649911e8..2d600186d9 100644 --- a/upup/models/cloudup/resources/addons/podsecuritypolicy.addons.k8s.io/k8s-1.10.yaml.template +++ b/upup/models/cloudup/resources/addons/podsecuritypolicy.addons.k8s.io/k8s-1.10.yaml.template @@ -2,6 +2,8 @@ apiVersion: extensions/v1beta1 kind: PodSecurityPolicy metadata: + annotations: + k8s-addon: podsecuritypolicy.addons.k8s.io name: kube-system spec: allowedCapabilities: @@ -27,6 +29,8 @@ spec: apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRole metadata: + annotations: + k8s-addon: podsecuritypolicy.addons.k8s.io name: kops:kube-system:psp rules: - apiGroups: @@ -47,7 +51,6 @@ roleRef: name: kops:kube-system:psp apiGroup: rbac.authorization.k8s.io subjects: -# permit the cluster wise admin to use this policy - kind: Group name: system:masters apiGroup: rbac.authorization.k8s.io @@ -59,6 +62,8 @@ subjects: kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: + annotations: + k8s-addon: podsecuritypolicy.addons.k8s.io name: kops:kube-system:psp namespace: kube-system roleRef: @@ -70,3 +75,8 @@ subjects: - kind: Group name: system:serviceaccounts:kube-system apiGroup: rbac.authorization.k8s.io +{{- if UseBootstrapTokens }} +- kind: Group + name: system:nodes + apiGroup: rbac.authorization.k8s.io +{{- end }} diff --git a/upup/pkg/fi/cloudup/template_functions.go b/upup/pkg/fi/cloudup/template_functions.go index 77baeb1466..84ec8f3228 100644 --- a/upup/pkg/fi/cloudup/template_functions.go +++ b/upup/pkg/fi/cloudup/template_functions.go @@ -59,8 +59,9 @@ type TemplateFunctions struct { func (tf *TemplateFunctions) AddTo(dest template.FuncMap) { dest["EtcdScheme"] = tf.EtcdScheme dest["SharedVPC"] = tf.SharedVPC - dest["UseEtcdTLS"] = tf.UseEtcdTLS dest["ToJSON"] = tf.ToJSON + dest["UseBootstrapTokens"] = tf.modelContext.UseBootstrapTokens + dest["UseEtcdTLS"] = tf.modelContext.UseEtcdTLS // Remember that we may be on a different arch from the target. Hard-code for now. dest["Arch"] = func() string { return "amd64" } dest["replace"] = func(s, find, replace string) string { @@ -110,17 +111,6 @@ func (tf *TemplateFunctions) AddTo(dest template.FuncMap) { } } -// UseEtcdTLS checks if cluster is using etcd tls -func (tf *TemplateFunctions) UseEtcdTLS() bool { - for _, x := range tf.cluster.Spec.EtcdClusters { - if x.EnableEtcdTLS { - return true - } - } - - return false -} - // ToJSON returns a json representation of the struct or on error an empty string func (tf *TemplateFunctions) ToJSON(data interface{}) string { encoded, err := json.Marshal(data) @@ -133,7 +123,7 @@ func (tf *TemplateFunctions) ToJSON(data interface{}) string { // EtcdScheme parses and grabs the protocol to the etcd cluster func (tf *TemplateFunctions) EtcdScheme() string { - if tf.UseEtcdTLS() { + if tf.modelContext.UseEtcdTLS() { return "https" } diff --git a/util/pkg/slice/slice.go b/util/pkg/slice/slice.go index 9d9c709cd9..0cb80d8a4e 100644 --- a/util/pkg/slice/slice.go +++ b/util/pkg/slice/slice.go @@ -38,3 +38,14 @@ func GetUniqueStrings(main, extra []string) []string { return unique } + +// Contains checks if a slice contains an element +func Contains(list []string, e string) bool { + for _, x := range list { + if x == e { + return true + } + } + + return false +}