kubernetes
/
kops
mirror of https://github.com/kubernetes/kops.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

- update the IAM policy to ensure the kubelet permision is skipped 

- update the PKI to ensure on new clusters the certificate it not created
This commit is contained in:
Rohith 2018-06-04 19:42:00 +01:00
parent 96eb0fbf0e
commit 2d5bd2cfd9
13 changed files with 166 additions and 176 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
hack/.packages
View File

@ -114,7 +114,6 @@ k8s.io/kops/pkg/templates
k8s.io/kops/pkg/testutils
k8s.io/kops/pkg/tokens
k8s.io/kops/pkg/urls
k8s.io/kops/pkg/util/fs
k8s.io/kops/pkg/util/stringorslice
k8s.io/kops/pkg/util/templater
k8s.io/kops/pkg/validation

2
nodeup/pkg/model/BUILD.bazel
View File

@ -48,7 +48,6 @@ go_library(
"//pkg/pki:go_default_library",
"//pkg/systemd:go_default_library",
"//pkg/tokens:go_default_library",
"//pkg/util/fs:go_default_library",
"//upup/pkg/fi:go_default_library",
"//upup/pkg/fi/nodeup/nodetasks:go_default_library",
"//upup/pkg/fi/utils:go_default_library",
@ -62,6 +61,7 @@ go_library(
"//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/intstr:go_default_library",
"//vendor/k8s.io/apiserver/pkg/authentication/user:go_default_library",
],
)

37
nodeup/pkg/model/context.go
View File

@ -115,7 +115,19 @@ func (c *NodeupModelContext) CNIBinDir() string {
// KubeletBootstrapConfig is the path the bootstrap config file
func (c *NodeupModelContext) KubeletBootstrapConfig() string {
return c.Cluster.Spec.Kubelet.BootstrapKubeconfig
path := c.Cluster.Spec.Kubelet.BootstrapKubeconfig
if c.IsMaster {
if c.Cluster.Spec.MasterKubelet != nil && c.Cluster.Spec.MasterKubelet.BootstrapKubeconfig != "" {
path = c.Cluster.Spec.MasterKubelet.BootstrapKubeconfig
}
}
if path != "" {
return path
}
return "/var/lib/kubelet/bootstrap-kubeconfig"
}
// KubeletKubeConfig is the path of the kubelet kubeconfig file
@ -252,7 +264,14 @@ func (c *NodeupModelContext) UsesCNI() bool {
// UseBootstrapTokens checks if we are using bootstrap tokens
func (c *NodeupModelContext) UseBootstrapTokens() bool {
return c.Cluster.Spec.Kubelet.BootstrapKubeconfig != ""
if c.Cluster.Spec.Kubelet.BootstrapKubeconfig != "" {
return true
}
if c.Cluster.Spec.MasterKubelet != nil && c.Cluster.Spec.MasterKubelet.BootstrapKubeconfig != "" {
return true
}
return false
}
// UseSecureKubelet checks if the kubelet api should be protected by a client certificate. Note: the settings are
@ -295,15 +314,15 @@ func (c *NodeupModelContext) KubectlPath() string {
}
// BuildCertificatePairTask creates the tasks to pull down the certificate and private key
func (c *NodeupModelContext) BuildCertificatePairTask(ctx *fi.ModelBuilderContext, name, path string) error {
certificate := fmt.Sprintf("%s/%s.pem", path, name)
key := fmt.Sprintf("%s/%s-key.pem", path, name)
func (c *NodeupModelContext) BuildCertificatePairTask(ctx *fi.ModelBuilderContext, key, path, filename string) error {
certificateName := fmt.Sprintf("%s/%s.pem", strings.TrimSuffix(path, "/"), filename)
keyName := fmt.Sprintf("%s/%s-key.pem", strings.TrimSuffix(path, "/"), filename)
if err := c.BuildCertificateTask(ctx, name, certificate); err != nil {
if err := c.BuildCertificateTask(ctx, key, certificateName); err != nil {
return err
}
return c.BuildPrivateKeyTask(ctx, name, key)
return c.BuildPrivateKeyTask(ctx, key, keyName)
}
// BuildCertificateTask is responsible for build a certificate request task
@ -326,7 +345,7 @@ func (c *NodeupModelContext) BuildCertificateTask(ctx *fi.ModelBuilderContext, n
Path: filepath.Join(c.PathSrvKubernetes(), filename),
Contents: fi.NewStringResource(serialized),
Type: nodetasks.FileType_File,
Mode: s("0400"),
Mode: s("0600"),
})
return nil
@ -352,7 +371,7 @@ func (c *NodeupModelContext) BuildPrivateKeyTask(ctx *fi.ModelBuilderContext, na
Path: filepath.Join(c.PathSrvKubernetes(), filename),
Contents: fi.NewStringResource(serialized),
Type: nodetasks.FileType_File,
Mode: s("0400"),
Mode: s("0600"),
})
return nil

36
nodeup/pkg/model/kubelet.go
View File

@ -17,7 +17,6 @@ limitations under the License.
package model
import (
"context"
"crypto/x509"
"crypto/x509/pkix"
"fmt"
@ -30,7 +29,6 @@ import (
"k8s.io/kops/pkg/flagbuilder"
"k8s.io/kops/pkg/pki"
"k8s.io/kops/pkg/systemd"
"k8s.io/kops/pkg/util/fs"
"k8s.io/kops/upup/pkg/fi"
"k8s.io/kops/upup/pkg/fi/nodeup/nodetasks"
"k8s.io/kops/upup/pkg/fi/utils"
@ -39,6 +37,7 @@ import (
"github.com/aws/aws-sdk-go/aws/session"
"github.com/golang/glog"
"k8s.io/api/core/v1"
"k8s.io/apiserver/pkg/authentication/user"
)
const (
@ -93,11 +92,6 @@ func (b *KubeletBuilder) Build(c *fi.ModelBuilderContext) error {
if b.UseBootstrapTokens() {
glog.V(3).Info("kubelet bootstrap tokens are enabled")
nodename, err := b.NodeName()
if err != nil {
return err
}
// @check if a master and if so, we bypass the token strapping and instead generate our own kubeconfig
if b.IsMaster {
task, err := b.buildMasterKubeletKubeconfig()
@ -105,21 +99,21 @@ func (b *KubeletBuilder) Build(c *fi.ModelBuilderContext) error {
return err
}
c.AddTask(task)
} else {
timeout := 5 * time.Minute
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
// @step: we are a Node, lets wait for the bootstrap file to appear. This is being performed
// an external process for now
glog.V(3).Infof("node: %s waiting for bootstrap: %s (%s) to be available", nodename, timeout.String(), b.KubeletBootstrapConfig())
if err := fs.WaitForFile(ctx, b.KubeletBootstrapConfig()); err != nil {
glog.Errorf("node: %s has timed out waiting for bootstrap: %s", nodename, b.KubeletBootstrapConfig())
name := "node-authorizer"
if err := b.BuildCertificatePairTask(c, name, "node-authorizer/", "tls"); err != nil {
return err
}
glog.V(3).Info("kubelet bootstrap configuration is available, continuing")
} else {
name := "node-authorizer-client"
if err := b.BuildCertificatePairTask(c, name, "node-authorizer/", "tls"); err != nil {
return err
}
glog.V(3).Info("kubelet service will wait for bootstrap configuration: %s", b.KubeletBootstrapConfig())
}
if err := b.BuildCertificateTask(c, fi.CertificateId_CA, "node-authorizer/ca.pem"); err != nil {
return err
}
} else {
kubeconfig, err := b.BuildPKIKubeconfig("kubelet")
@ -235,6 +229,10 @@ func (b *KubeletBuilder) buildSystemdService() *nodetasks.Service {
manifest.Set("Unit", "Documentation", "https://github.com/kubernetes/kubernetes")
manifest.Set("Unit", "After", "docker.service")
if b.UseBootstrapTokens() && !b.IsMaster {
manifest.Set("Unit", "ConditionPathExists", b.KubeletBootstrapConfig())
}
if b.Distribution == distros.DistributionCoreOS {
// We add /opt/kubernetes/bin for our utilities (socat)
manifest.Set("Service", "Environment", "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/kubernetes/bin")
@ -521,7 +519,7 @@ func (b *KubeletBuilder) buildMasterKubeletKubeconfig() (*nodetasks.File, error)
template.Subject = pkix.Name{
CommonName: fmt.Sprintf("system:node:%s", nodeName),
Organization: []string{"system:nodes"},
Organization: []string{user.NodesGroup},
}
// https://tools.ietf.org/html/rfc5280#section-4.2.1.3

2
pkg/model/components/apiserver.go
View File

@ -92,7 +92,7 @@ func (b *KubeAPIServerOptionsBuilder) BuildOptions(o interface{}) error {
} else if clusterSpec.Authorization.RBAC != nil {
var modes []string
if b.IsKubernetesGTE("1.9") {
if b.IsKubernetesGTE("1.10") {
// Enable the Node authorizer, used for special per-node RBAC policies
modes = append(modes, "Node")
}

9
pkg/model/context.go
View File

@ -249,7 +249,14 @@ func (m *KopsModelContext) CloudTags(name string, shared bool) map[string]string
// UseBootstrapTokens checks if bootstrap tokens are enabled
func (m *KopsModelContext) UseBootstrapTokens() bool {
return m.Cluster.Spec.Kubelet.BootstrapKubeconfig != ""
if m.Cluster.Spec.Kubelet.BootstrapKubeconfig != "" {
return true
}
if m.Cluster.Spec.MasterKubelet != nil && m.Cluster.Spec.MasterKubelet.BootstrapKubeconfig != "" {
return true
}
return false
}
// UsesBastionDns checks if we should use a specific name for the bastion dns

121
pkg/model/iam/iam_builder.go
View File

@ -348,12 +348,30 @@ func (b *PolicyBuilder) AddS3Permissions(p *Policy) (*Policy, error) {
strings.Join([]string{b.IAMPrefix(), ":s3:::", iamS3Path, "/instancegroup/*"}, ""),
strings.Join([]string{b.IAMPrefix(), ":s3:::", iamS3Path, "/pki/issued/*"}, ""),
strings.Join([]string{b.IAMPrefix(), ":s3:::", iamS3Path, "/pki/private/kube-proxy/*"}, ""),
strings.Join([]string{b.IAMPrefix(), ":s3:::", iamS3Path, "/pki/private/kubelet/*"}, ""),
strings.Join([]string{b.IAMPrefix(), ":s3:::", iamS3Path, "/pki/ssh/*"}, ""),
strings.Join([]string{b.IAMPrefix(), ":s3:::", iamS3Path, "/secrets/dockerconfig"}, ""),
),
})
// @check if bootstrap tokens are enabled and if so, we disable access for the nodes
if !b.UseBootstrapTokens() {
p.Statement = append(p.Statement, &Statement{
Effect: StatementEffectAllow,
Action: stringorslice.Slice([]string{"s3:Get*"}),
Resource: stringorslice.Of(
strings.Join([]string{b.IAMPrefix(), ":s3:::", iamS3Path, "/pki/private/kubelet/*"}, ""),
),
})
} else {
p.Statement = append(p.Statement, &Statement{
Effect: StatementEffectAllow,
Action: stringorslice.Slice([]string{"s3:Get*"}),
Resource: stringorslice.Of(
strings.Join([]string{b.IAMPrefix(), ":s3:::", iamS3Path, "/pki/private/node-authorizer-client/*"}, ""),
),
})
}
if b.Cluster.Spec.Networking != nil {
// @check if kuberoute is enabled and permit access to the private key
if b.Cluster.Spec.Networking.Kuberouter != nil {
@ -469,6 +487,19 @@ func (b *PolicyResource) Open() (io.Reader, error) {
return bytes.NewReader([]byte(j)), nil
}
// UseBootstrapTokens check if we are using bootstrap tokens - @TODO, i don't like this we should probably pass in
// the kops model into the builder rather than duplicating the code. I'll leave for anothe PR
func (b *PolicyBuilder) UseBootstrapTokens() bool {
if b.Cluster.Spec.Kubelet != nil && b.Cluster.Spec.Kubelet.BootstrapKubeconfig != "" {
return true
}
if b.Cluster.Spec.MasterKubelet != nil && b.Cluster.Spec.MasterKubelet.BootstrapKubeconfig != "" {
return true
}
return false
}
func addECRPermissions(p *Policy) {
// TODO - I think we can just have GetAuthorizationToken here, as we are not
// TODO - making any API calls except for GetAuthorizationToken.
@ -755,59 +786,59 @@ func addRomanaCNIPermissions(p *Policy, resource stringorslice.StringOrSlice, le
if legacyIAM {
// Legacy IAM provides ec2:*, so no additional permissions required
return
} else {
// Romana requires additional Describe permissions
// Comments are which Romana component makes the call
p.Statement = append(p.Statement,
&Statement{
Effect: StatementEffectAllow,
Action: stringorslice.Slice([]string{
"ec2:DescribeAvailabilityZones", // vpcrouter
"ec2:DescribeVpcs", // vpcrouter
}),
Resource: resource,
},
&Statement{
Effect: StatementEffectAllow,
Action: stringorslice.Slice([]string{
"ec2:CreateRoute", // vpcrouter
"ec2:DeleteRoute", // vpcrouter
"ec2:ReplaceRoute", // vpcrouter
}),
Resource: resource,
Condition: Condition{
"StringEquals": map[string]string{
"ec2:ResourceTag/KubernetesCluster": clusterName,
},
}
// Romana requires additional Describe permissions
// Comments are which Romana component makes the call
p.Statement = append(p.Statement,
&Statement{
Effect: StatementEffectAllow,
Action: stringorslice.Slice([]string{
"ec2:DescribeAvailabilityZones", // vpcrouter
"ec2:DescribeVpcs", // vpcrouter
}),
Resource: resource,
},
&Statement{
Effect: StatementEffectAllow,
Action: stringorslice.Slice([]string{
"ec2:CreateRoute", // vpcrouter
"ec2:DeleteRoute", // vpcrouter
"ec2:ReplaceRoute", // vpcrouter
}),
Resource: resource,
Condition: Condition{
"StringEquals": map[string]string{
"ec2:ResourceTag/KubernetesCluster": clusterName,
},
},
)
}
},
)
}
func addAmazonVPCCNIPermissions(p *Policy, resource stringorslice.StringOrSlice, legacyIAM bool, clusterName string) {
if legacyIAM {
// Legacy IAM provides ec2:*, so no additional permissions required
return
} else {
p.Statement = append(p.Statement,
&Statement{
Effect: StatementEffectAllow,
Action: stringorslice.Slice([]string{
"ec2:CreateNetworkInterface",
"ec2:AttachNetworkInterface",
"ec2:DeleteNetworkInterface",
"ec2:DetachNetworkInterface",
"ec2:DescribeNetworkInterfaces",
"ec2:DescribeInstances",
"ec2:ModifyNetworkInterfaceAttribute",
"ec2:AssignPrivateIpAddresses",
"tag:TagResources",
}),
Resource: resource,
},
)
}
p.Statement = append(p.Statement,
&Statement{
Effect: StatementEffectAllow,
Action: stringorslice.Slice([]string{
"ec2:CreateNetworkInterface",
"ec2:AttachNetworkInterface",
"ec2:DeleteNetworkInterface",
"ec2:DetachNetworkInterface",
"ec2:DescribeNetworkInterfaces",
"ec2:DescribeInstances",
"ec2:ModifyNetworkInterfaceAttribute",
"ec2:AssignPrivateIpAddresses",
"tag:TagResources",
}),
Resource: resource,
},
)
}
func createResource(b *PolicyBuilder) stringorslice.StringOrSlice {

8
pkg/model/iam/tests/iam_builder_node_strict.json
View File

@ -33,10 +33,16 @@
"arn:aws:s3:::kops-tests/iam-builder-test.k8s.local/instancegroup/*",
"arn:aws:s3:::kops-tests/iam-builder-test.k8s.local/pki/issued/*",
"arn:aws:s3:::kops-tests/iam-builder-test.k8s.local/pki/private/kube-proxy/*",
"arn:aws:s3:::kops-tests/iam-builder-test.k8s.local/pki/private/kubelet/*",
"arn:aws:s3:::kops-tests/iam-builder-test.k8s.local/pki/ssh/*",
"arn:aws:s3:::kops-tests/iam-builder-test.k8s.local/secrets/dockerconfig"
]
},
{
"Effect": "Allow",
"Action": [
"s3:Get*"
],
"Resource": "arn:aws:s3:::kops-tests/iam-builder-test.k8s.local/pki/private/kubelet/*"
}
]
}

8
pkg/model/iam/tests/iam_builder_node_strict_ecr.json
View File

@ -33,11 +33,17 @@
"arn:aws:s3:::kops-tests/iam-builder-test.k8s.local/instancegroup/*",
"arn:aws:s3:::kops-tests/iam-builder-test.k8s.local/pki/issued/*",
"arn:aws:s3:::kops-tests/iam-builder-test.k8s.local/pki/private/kube-proxy/*",
"arn:aws:s3:::kops-tests/iam-builder-test.k8s.local/pki/private/kubelet/*",
"arn:aws:s3:::kops-tests/iam-builder-test.k8s.local/pki/ssh/*",
"arn:aws:s3:::kops-tests/iam-builder-test.k8s.local/secrets/dockerconfig"
]
},
{
"Effect": "Allow",
"Action": [
"s3:Get*"
],
"Resource": "arn:aws:s3:::kops-tests/iam-builder-test.k8s.local/pki/private/kubelet/*"
},
{
"Effect": "Allow",
"Action": [

37
pkg/model/pki.go
View File

@ -18,6 +18,7 @@ package model
import (
"fmt"
"strings"
"k8s.io/kops/pkg/tokens"
"k8s.io/kops/upup/pkg/fi"
@ -53,17 +54,18 @@ func (b *PKIModelBuilder) Build(c *fi.ModelBuilderContext) error {
c.AddTask(defaultCA)
{
t := &fitasks.Keypair{
Name: fi.String("kubelet"),
Lifecycle: b.Lifecycle,
Subject: "o=" + user.NodesGroup + ",cn=kubelet",
Type: "client",
Signer: defaultCA,
Format: format,
// @check of bootstrap tokens are enable if so, disable the creation of the kubelet certificate - we also
// block at the IAM level for AWS cluster for pre-existing clusters.
if !b.UseBootstrapTokens() {
c.AddTask(&fitasks.Keypair{
Name: fi.String("kubelet"),
Lifecycle: b.Lifecycle,
Subject: "o=" + user.NodesGroup + ",cn=kubelet",
Type: "client",
Signer: defaultCA,
Format: format,
})
}
c.AddTask(t)
}
{
// Generate a kubelet client certificate for api to speak securely to kubelets. This change was first
@ -284,17 +286,20 @@ func (b *PKIModelBuilder) Build(c *fi.ModelBuilderContext) error {
// But I'm conscious not to do too much work on bootstrap tokens as it might overlay further down the
// line with the machines api
if b.UseBootstrapTokens() {
serviceName := "node-authorizer-internal"
alternateNames := []string{
"127.0.0.1",
"localhost",
"node-bootstrap-internal",
"node-bootstrap-internal." + b.Cluster.Spec.DNSZone,
serviceName,
strings.Join([]string{serviceName, b.Cluster.Name}, "."),
strings.Join([]string{serviceName, b.Cluster.Spec.DNSZone}, "."),
}
// @note: the certificate used by the node authorizers
c.AddTask(&fitasks.Keypair{
Name: fi.String("node-bootstrap"),
Subject: "cn=node-bootstrap",
Name: fi.String("node-authorizer"),
Subject: "cn=node-authorizaer",
Type: "server",
AlternateNames: alternateNames,
Signer: defaultCA,
@ -303,8 +308,8 @@ func (b *PKIModelBuilder) Build(c *fi.ModelBuilderContext) error {
// @note: we use this for mutual tls between between node and authorizer
c.AddTask(&fitasks.Keypair{
Name: fi.String("node-bootstrap-client"),
Subject: "cn=node-bootstrap-client",
Name: fi.String("node-authorizer-client"),
Subject: "cn=node-authorizer-client",
Type: "client",
Signer: defaultCA,
Format: format,

8
pkg/util/fs/BUILD.bazel
View File

@ -1,8 +0,0 @@
load("@io_bazel_rules_go//go:def.bzl", "go_library")
go_library(
name = "go_default_library",
srcs = ["wait.go"],
importpath = "k8s.io/kops/pkg/util/fs",
visibility = ["//visibility:public"],
)

71
pkg/util/fs/wait.go
View File

@ -1,71 +0,0 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package fs
import (
"context"
"errors"
"os"
"time"
)
// ErrTimeout indicates the operation has timed out
var ErrTimeout = errors.New("operation timeout")
// WaitForFile is responsible for waiting for file to appear or timeout
func WaitForFile(ctx context.Context, path string) error {
doneCh := make(chan struct{}, 0)
// @step: we wait for the bootstrap token file to appear
go func() {
ticker := time.NewTicker(5 * time.Second)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
if found, _ := FileExists(path); found {
doneCh <- struct{}{}
return
}
}
}
}()
select {
case <-ctx.Done():
return ctx.Err()
case <-doneCh:
}
return nil
}
// FileExists checks if a file exists
func FileExists(path string) (bool, error) {
if _, err := os.Stat(path); err != nil {
if os.IsNotExist(err) {
return false, nil
}
return false, err
}
return true, nil
}

2
upup/models/cloudup/resources/addons/podsecuritypolicy.addons.k8s.io/k8s-1.10.yaml.template
View File

@ -54,12 +54,10 @@ subjects:
- kind: Group
name: system:masters
apiGroup: rbac.authorization.k8s.io
{{- if not UseBootstrapTokens }}
# permit the kubelets to access this policy (used for manifests)
- kind: User
name: kubelet
apiGroup: rbac.authorization.k8s.io
{{- end }}
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1