kubernetes
/
kops
mirror of https://github.com/kubernetes/kops.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #15342 from olemarkus/ecr-credentials-provider 

Switch to using external ECR credential provider for k8s 1.27
This commit is contained in:
Kubernetes Prow Robot 2023-05-02 04:50:13 -07:00 committed by GitHub
parent 6d1837a4a0 d7ba6112e6
commit 0f13866b01
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 125 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docs/releases/1.27-NOTES.md
View File

@ -14,6 +14,8 @@ This behaviour can be overridden by setting `spec.etcdClusters[*].manager.backup
* As of Kubernetes version 1.27, all nodes will default to running with instance-metadata-service tokens required, with a max hop limit of 1.
Newly created clusters will be configured as necessary to have these settings.
* As of Kubernetes version 1.27, credentials for private ECR repositories will be handled by the out-of-tree credential provider. This is an additional binary that each instance downloads from the assets repository.
## GCP
## Openstack

85
nodeup/pkg/model/kubelet.go
View File

@ -49,7 +49,8 @@ const (
// kubeletService is the name of the kubelet service
kubeletService = "kubelet.service"
kubeletConfigFilePath = "/var/lib/kubelet/kubelet.conf"
kubeletConfigFilePath = "/var/lib/kubelet/kubelet.conf"
credentialProviderConfigFilePath = "/var/lib/kubelet/credential-provider.conf"
)
// KubeletBuilder installs kubelet
@ -157,6 +158,12 @@ func (b *KubeletBuilder) Build(c *fi.NodeupModelBuilderContext) error {
return err
}
if b.Cluster.UsesExternalECRCredentialsProvider() {
if err := b.addECRCP(c); err != nil {
return fmt.Errorf("failed to add ECR credential provider: %w", err)
}
}
if kubeletConfig.CgroupDriver == "systemd" && b.NodeupConfig.ContainerRuntime == "containerd" {
{
@ -241,16 +248,25 @@ func buildKubeletComponentConfig(kubeletConfig *kops.KubeletConfigSpec) (*nodeta
return t, nil
}
// kubeletPath returns the path of the kubelet based on distro
func (b *KubeletBuilder) kubeletPath() string {
kubeletCommand := "/usr/local/bin/kubelet"
func (b *KubeletBuilder) binaryPath() string {
path := "/usr/local/bin"
if b.Distribution == distributions.DistributionFlatcar {
kubeletCommand = "/opt/kubernetes/bin/kubelet"
path = "/opt/kubernetes/bin"
}
if b.Distribution == distributions.DistributionContainerOS {
kubeletCommand = "/home/kubernetes/bin/kubelet"
path = "/home/kubernetes/bin"
}
return kubeletCommand
return path
}
// kubeletPath returns the path of the kubelet based on distro
func (b *KubeletBuilder) kubeletPath() string {
return b.binaryPath() + "/kubelet"
}
// ecrcpPath returns the path of the ECR credentials provider based on distro and archiecture
func (b *KubeletBuilder) ecrcpPath() string {
return b.binaryPath() + "/ecr-credential-provider"
}
// buildManifestDirectory creates the directory where kubelet expects static manifests to reside
@ -330,6 +346,11 @@ func (b *KubeletBuilder) buildSystemdEnvironmentFile(kubeletConfig *kops.Kubelet
flags += " --config=" + kubeletConfigFilePath
if b.Cluster.UsesExternalECRCredentialsProvider() {
flags += " --image-credential-provider-config=" + credentialProviderConfigFilePath
flags += " --image-credential-provider-bin-dir=" + b.binaryPath()
}
sysconfig := "DAEMON_ARGS=\"" + flags + "\"\n"
// Makes kubelet read /root/.docker/config.json properly
sysconfig = sysconfig + "HOME=\"/root" + "\"\n"
@ -413,6 +434,56 @@ func (b *KubeletBuilder) usesContainerizedMounter() bool {
}
}
// addECRCP installs the ECR credential provider
func (b *KubeletBuilder) addECRCP(c *fi.NodeupModelBuilderContext) error {
{
assetName := "ecr-credential-provider-linux-" + string(b.Architecture)
assetPath := ""
asset, err := b.Assets.Find(assetName, assetPath)
if err != nil {
return fmt.Errorf("error trying to locate asset %q: %v", assetName, err)
}
if asset == nil {
return fmt.Errorf("unable to locate asset %q", assetName)
}
t := &nodetasks.File{
Path: b.ecrcpPath(),
Contents: asset,
Type: nodetasks.FileType_File,
Mode: s("0755"),
}
c.AddTask(t)
}
{
configContent := `apiVersion: kubelet.config.k8s.io/v1
kind: CredentialProviderConfig
providers:
- name: ecr-credential-provider
matchImages:
- "*.dkr.ecr.*.amazonaws.com"
- "*.dkr.ecr.*.amazonaws.cn"
- "*.dkr.ecr-fips.*.amazonaws.com"
- "*.dkr.ecr.us-iso-east-1.c2s.ic.gov"
- "*.dkr.ecr.us-isob-east-1.sc2s.sgov.gov"
defaultCacheDuration: "12h"
apiVersion: credentialprovider.kubelet.k8s.io/v1
args:
- get-credentials
`
t := &nodetasks.File{
Path: credentialProviderConfigFilePath,
Contents: fi.NewStringResource(configContent),
Type: nodetasks.FileType_File,
Mode: s("0644"),
}
c.AddTask(t)
}
return nil
}
// addContainerizedMounter downloads and installs the containerized mounter, that we need on ContainerOS
func (b *KubeletBuilder) addContainerizedMounter(c *fi.NodeupModelBuilderContext) error {
if !b.usesContainerizedMounter() {

7
pkg/apis/kops/cluster.go
View File

@ -220,6 +220,9 @@ type AWSSpec struct {
// Spotinst cloud-config specs
SpotinstProduct *string `json:"spotinstProduct,omitempty"`
SpotinstOrientation *string `json:"spotinstOrientation,omitempty"`
// BinariesLocation is the location of the AWS cloud provider binaries.
BinariesLocation *string `json:"binaryLocation,omitempty"`
}
// DOSpec configures the Digital Ocean cloud provider.
@ -895,6 +898,10 @@ func (c *Cluster) UsesNoneDNS() bool {
return false
}
func (c *Cluster) UsesExternalECRCredentialsProvider() bool {
return c.IsKubernetesGTE("1.27") && c.Spec.GetCloudProvider() == CloudProviderAWS
}
func (c *Cluster) APIInternalName() string {
return "api.internal." + c.ObjectMeta.Name
}

3
pkg/apis/kops/v1alpha3/cluster.go
View File

@ -216,6 +216,9 @@ type AWSSpec struct {
// Spotinst cloud-config specs
SpotinstProduct *string `json:"spotinstProduct,omitempty"`
SpotinstOrientation *string `json:"spotinstOrientation,omitempty"`
// BinariesLocation is the location of the AWS cloud provider binaries.
BinariesLocation *string `json:"binaryLocation,omitempty"`
}
// DOSpec configures the Digital Ocean cloud provider.

2
pkg/apis/kops/v1alpha3/zz_generated.conversion.go generated
View File

@ -1478,6 +1478,7 @@ func autoConvert_v1alpha3_AWSSpec_To_kops_AWSSpec(in *AWSSpec, out *kops.AWSSpec
out.ElbSecurityGroup = in.ElbSecurityGroup
out.SpotinstProduct = in.SpotinstProduct
out.SpotinstOrientation = in.SpotinstOrientation
out.BinariesLocation = in.BinariesLocation
return nil
}
@ -1537,6 +1538,7 @@ func autoConvert_kops_AWSSpec_To_v1alpha3_AWSSpec(in *kops.AWSSpec, out *AWSSpec
out.ElbSecurityGroup = in.ElbSecurityGroup
out.SpotinstProduct = in.SpotinstProduct
out.SpotinstOrientation = in.SpotinstOrientation
out.BinariesLocation = in.BinariesLocation
return nil
}

5
pkg/apis/kops/v1alpha3/zz_generated.deepcopy.go generated
View File

@ -203,6 +203,11 @@ func (in *AWSSpec) DeepCopyInto(out *AWSSpec) {
*out = new(string)
**out = **in
}
if in.BinariesLocation != nil {
in, out := &in.BinariesLocation, &out.BinariesLocation
*out = new(string)
**out = **in
}
return
}

5
pkg/apis/kops/zz_generated.deepcopy.go generated
View File

@ -202,6 +202,11 @@ func (in *AWSSpec) DeepCopyInto(out *AWSSpec) {
*out = new(string)
**out = **in
}
if in.BinariesLocation != nil {
in, out := &in.BinariesLocation, &out.BinariesLocation
*out = new(string)
**out = **in
}
return
}

2
tests/integration/update_cluster/minimal/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data
View File

@ -250,7 +250,7 @@ CloudProvider: aws
ConfigBase: memfs://clusters.example.com/minimal.example.com
InstanceGroupName: master-us-test-1a
InstanceGroupRole: ControlPlane
NodeupConfigHash: N/3CvlvqplvxWXN3+r1r09mA/Y55Wt/klapVL1nCd2w=
NodeupConfigHash: rJVuGlmAuBwCe2RWXKgT1dI+bdFv3FhvakVQhTVudXU=
__EOF_KUBE_ENV

2
tests/integration/update_cluster/minimal/data/aws_launch_template_nodes.minimal.example.com_user_data
View File

@ -174,7 +174,7 @@ ConfigServer:
- https://kops-controller.internal.minimal.example.com:3988/
InstanceGroupName: nodes
InstanceGroupRole: Node
NodeupConfigHash: g+og1yXE09z0oZE2ggS/s5xmrp7CHsM9PbOhPFs8cNs=
NodeupConfigHash: jg6YaOqnx4/XWWzsT5/JFgNHr3tC6qgvfQ6M1CfY6Jk=
__EOF_KUBE_ENV

2
tests/integration/update_cluster/minimal/data/aws_s3_object_nodeupconfig-master-us-test-1a_content
View File

@ -56,6 +56,7 @@ Assets:
amd64:
- be5e79c70e926019e588c8c5c44d93efb1042f3be6f1db0de14e707141564d29@https://storage.googleapis.com/kubernetes-release/release/v1.27.0-alpha.3/bin/linux/amd64/kubelet
- 4d416a4bed7c1bb576e284fc6b0bb843f879c08fa8e4beb7e2376305d2dc8299@https://storage.googleapis.com/kubernetes-release/release/v1.27.0-alpha.3/bin/linux/amd64/kubectl
- 5035d7814c95cd3cedbc5efb447ef25a4942ef05caab2159746d55ce1698c74a@https://artifacts.k8s.io/binaries/cloud-provider-aws/v1.27.1/linux/amd64/ecr-credential-provider-linux-amd64
- 962100bbc4baeaaa5748cdbfce941f756b1531c2eadb290129401498bfac21e7@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.9.1/cni-plugins-linux-amd64-v0.9.1.tgz
- bb9a9ccd6517e2a54da748a9f60dc9aa9d79d19d4724663f2386812f083968e2@https://github.com/containerd/containerd/releases/download/v1.6.20/containerd-1.6.20-linux-amd64.tar.gz
- db772be63147a4e747b4fe286c7c16a2edc4a8458bd3092ea46aaee77750e8ce@https://github.com/opencontainers/runc/releases/download/v1.1.4/runc.amd64
@ -64,6 +65,7 @@ Assets:
arm64:
- 2a15e9c291dce3e07db5d9948d5c37a4e52c9884343f63287189e4f0d4c3df76@https://storage.googleapis.com/kubernetes-release/release/v1.27.0-alpha.3/bin/linux/arm64/kubelet
- c5fad9f96ab5fe04b8927ce83ceefc4db65b032303a23b35e353276479450d2a@https://storage.googleapis.com/kubernetes-release/release/v1.27.0-alpha.3/bin/linux/arm64/kubectl
- b3d567bda9e2996fc1fbd9d13506bd16763d3865b5c7b0b3c4b48c6088c04481@https://artifacts.k8s.io/binaries/cloud-provider-aws/v1.27.1/linux/arm64/ecr-credential-provider-linux-arm64
- ef17764ffd6cdcb16d76401bac1db6acc050c9b088f1be5efa0e094ea3b01df0@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.9.1/cni-plugins-linux-arm64-v0.9.1.tgz
- c3e6a054b18b20fce06c7c3ed53f0989bb4b255c849bede446ebca955f07a9ce@https://github.com/containerd/containerd/releases/download/v1.6.20/containerd-1.6.20-linux-arm64.tar.gz
- dbb71e737eaef454a406ce21fd021bd8f1b35afb7635016745992bbd7c17a223@https://github.com/opencontainers/runc/releases/download/v1.1.4/runc.arm64

2
tests/integration/update_cluster/minimal/data/aws_s3_object_nodeupconfig-nodes_content
View File

@ -2,12 +2,14 @@ Assets:
amd64:
- be5e79c70e926019e588c8c5c44d93efb1042f3be6f1db0de14e707141564d29@https://storage.googleapis.com/kubernetes-release/release/v1.27.0-alpha.3/bin/linux/amd64/kubelet
- 4d416a4bed7c1bb576e284fc6b0bb843f879c08fa8e4beb7e2376305d2dc8299@https://storage.googleapis.com/kubernetes-release/release/v1.27.0-alpha.3/bin/linux/amd64/kubectl
- 5035d7814c95cd3cedbc5efb447ef25a4942ef05caab2159746d55ce1698c74a@https://artifacts.k8s.io/binaries/cloud-provider-aws/v1.27.1/linux/amd64/ecr-credential-provider-linux-amd64
- 962100bbc4baeaaa5748cdbfce941f756b1531c2eadb290129401498bfac21e7@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.9.1/cni-plugins-linux-amd64-v0.9.1.tgz
- bb9a9ccd6517e2a54da748a9f60dc9aa9d79d19d4724663f2386812f083968e2@https://github.com/containerd/containerd/releases/download/v1.6.20/containerd-1.6.20-linux-amd64.tar.gz
- db772be63147a4e747b4fe286c7c16a2edc4a8458bd3092ea46aaee77750e8ce@https://github.com/opencontainers/runc/releases/download/v1.1.4/runc.amd64
arm64:
- 2a15e9c291dce3e07db5d9948d5c37a4e52c9884343f63287189e4f0d4c3df76@https://storage.googleapis.com/kubernetes-release/release/v1.27.0-alpha.3/bin/linux/arm64/kubelet
- c5fad9f96ab5fe04b8927ce83ceefc4db65b032303a23b35e353276479450d2a@https://storage.googleapis.com/kubernetes-release/release/v1.27.0-alpha.3/bin/linux/arm64/kubectl
- b3d567bda9e2996fc1fbd9d13506bd16763d3865b5c7b0b3c4b48c6088c04481@https://artifacts.k8s.io/binaries/cloud-provider-aws/v1.27.1/linux/arm64/ecr-credential-provider-linux-arm64
- ef17764ffd6cdcb16d76401bac1db6acc050c9b088f1be5efa0e094ea3b01df0@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.9.1/cni-plugins-linux-arm64-v0.9.1.tgz
- c3e6a054b18b20fce06c7c3ed53f0989bb4b255c849bede446ebca955f07a9ce@https://github.com/containerd/containerd/releases/download/v1.6.20/containerd-1.6.20-linux-arm64.tar.gz
- dbb71e737eaef454a406ce21fd021bd8f1b35afb7635016745992bbd7c17a223@https://github.com/opencontainers/runc/releases/download/v1.1.4/runc.arm64

17
upup/pkg/fi/cloudup/apply_cluster.go
View File

@ -1056,6 +1056,23 @@ func (c *ApplyClusterCmd) addFileAssets(assetBuilder *assets.AssetBuilder) error
c.Assets[arch] = append(c.Assets[arch], mirrors.BuildMirroredAsset(u, hash))
}
if c.Cluster.UsesExternalECRCredentialsProvider() {
binaryLocation := c.Cluster.Spec.CloudProvider.AWS.BinariesLocation
if binaryLocation == nil {
binaryLocation = fi.PtrTo("https://artifacts.k8s.io/binaries/cloud-provider-aws/v1.27.1")
}
k, err := url.Parse(fmt.Sprintf("%s/linux/%s/ecr-credential-provider-linux-%s", *binaryLocation, arch, arch))
if err != nil {
return err
}
u, hash, err := assetBuilder.RemapFileAndSHA(k)
if err != nil {
return err
}
c.Assets[arch] = append(c.Assets[arch], mirrors.BuildMirroredAsset(u, hash))
}
cniAsset, cniAssetHash, err := findCNIAssets(c.Cluster, assetBuilder, arch)
if err != nil {
return err