diff --git a/docs/releases/1.27-NOTES.md b/docs/releases/1.27-NOTES.md index 34956472d7..296a595581 100644 --- a/docs/releases/1.27-NOTES.md +++ b/docs/releases/1.27-NOTES.md @@ -14,6 +14,8 @@ This behaviour can be overridden by setting `spec.etcdClusters[*].manager.backup * As of Kubernetes version 1.27, all nodes will default to running with instance-metadata-service tokens required, with a max hop limit of 1. Newly created clusters will be configured as necessary to have these settings. +* As of Kubernetes version 1.27, credentials for private ECR repositories will be handled by the out-of-tree credential provider. This is an additional binary that each instance downloads from the assets repository. + ## GCP ## Openstack diff --git a/nodeup/pkg/model/kubelet.go b/nodeup/pkg/model/kubelet.go index e74b198add..a5734f07a5 100644 --- a/nodeup/pkg/model/kubelet.go +++ b/nodeup/pkg/model/kubelet.go @@ -49,7 +49,8 @@ const ( // kubeletService is the name of the kubelet service kubeletService = "kubelet.service" - kubeletConfigFilePath = "/var/lib/kubelet/kubelet.conf" + kubeletConfigFilePath = "/var/lib/kubelet/kubelet.conf" + credentialProviderConfigFilePath = "/var/lib/kubelet/credential-provider.conf" ) // KubeletBuilder installs kubelet @@ -157,6 +158,12 @@ func (b *KubeletBuilder) Build(c *fi.NodeupModelBuilderContext) error { return err } + if b.Cluster.UsesExternalECRCredentialsProvider() { + if err := b.addECRCP(c); err != nil { + return fmt.Errorf("failed to add ECR credential provider: %w", err) + } + } + if kubeletConfig.CgroupDriver == "systemd" && b.NodeupConfig.ContainerRuntime == "containerd" { { @@ -241,16 +248,25 @@ func buildKubeletComponentConfig(kubeletConfig *kops.KubeletConfigSpec) (*nodeta return t, nil } -// kubeletPath returns the path of the kubelet based on distro -func (b *KubeletBuilder) kubeletPath() string { - kubeletCommand := "/usr/local/bin/kubelet" +func (b *KubeletBuilder) binaryPath() string { + path := "/usr/local/bin" if b.Distribution == distributions.DistributionFlatcar { - kubeletCommand = "/opt/kubernetes/bin/kubelet" + path = "/opt/kubernetes/bin" } if b.Distribution == distributions.DistributionContainerOS { - kubeletCommand = "/home/kubernetes/bin/kubelet" + path = "/home/kubernetes/bin" } - return kubeletCommand + return path +} + +// kubeletPath returns the path of the kubelet based on distro +func (b *KubeletBuilder) kubeletPath() string { + return b.binaryPath() + "/kubelet" +} + +// ecrcpPath returns the path of the ECR credentials provider based on distro and archiecture +func (b *KubeletBuilder) ecrcpPath() string { + return b.binaryPath() + "/ecr-credential-provider" } // buildManifestDirectory creates the directory where kubelet expects static manifests to reside @@ -330,6 +346,11 @@ func (b *KubeletBuilder) buildSystemdEnvironmentFile(kubeletConfig *kops.Kubelet flags += " --config=" + kubeletConfigFilePath + if b.Cluster.UsesExternalECRCredentialsProvider() { + flags += " --image-credential-provider-config=" + credentialProviderConfigFilePath + flags += " --image-credential-provider-bin-dir=" + b.binaryPath() + } + sysconfig := "DAEMON_ARGS=\"" + flags + "\"\n" // Makes kubelet read /root/.docker/config.json properly sysconfig = sysconfig + "HOME=\"/root" + "\"\n" @@ -413,6 +434,56 @@ func (b *KubeletBuilder) usesContainerizedMounter() bool { } } +// addECRCP installs the ECR credential provider +func (b *KubeletBuilder) addECRCP(c *fi.NodeupModelBuilderContext) error { + { + assetName := "ecr-credential-provider-linux-" + string(b.Architecture) + assetPath := "" + asset, err := b.Assets.Find(assetName, assetPath) + if err != nil { + return fmt.Errorf("error trying to locate asset %q: %v", assetName, err) + } + if asset == nil { + return fmt.Errorf("unable to locate asset %q", assetName) + } + + t := &nodetasks.File{ + Path: b.ecrcpPath(), + Contents: asset, + Type: nodetasks.FileType_File, + Mode: s("0755"), + } + c.AddTask(t) + } + + { + configContent := `apiVersion: kubelet.config.k8s.io/v1 +kind: CredentialProviderConfig +providers: + - name: ecr-credential-provider + matchImages: + - "*.dkr.ecr.*.amazonaws.com" + - "*.dkr.ecr.*.amazonaws.cn" + - "*.dkr.ecr-fips.*.amazonaws.com" + - "*.dkr.ecr.us-iso-east-1.c2s.ic.gov" + - "*.dkr.ecr.us-isob-east-1.sc2s.sgov.gov" + defaultCacheDuration: "12h" + apiVersion: credentialprovider.kubelet.k8s.io/v1 + args: + - get-credentials +` + + t := &nodetasks.File{ + Path: credentialProviderConfigFilePath, + Contents: fi.NewStringResource(configContent), + Type: nodetasks.FileType_File, + Mode: s("0644"), + } + c.AddTask(t) + } + return nil +} + // addContainerizedMounter downloads and installs the containerized mounter, that we need on ContainerOS func (b *KubeletBuilder) addContainerizedMounter(c *fi.NodeupModelBuilderContext) error { if !b.usesContainerizedMounter() { diff --git a/pkg/apis/kops/cluster.go b/pkg/apis/kops/cluster.go index e8736096a1..ac26b012e3 100644 --- a/pkg/apis/kops/cluster.go +++ b/pkg/apis/kops/cluster.go @@ -220,6 +220,9 @@ type AWSSpec struct { // Spotinst cloud-config specs SpotinstProduct *string `json:"spotinstProduct,omitempty"` SpotinstOrientation *string `json:"spotinstOrientation,omitempty"` + + // BinariesLocation is the location of the AWS cloud provider binaries. + BinariesLocation *string `json:"binaryLocation,omitempty"` } // DOSpec configures the Digital Ocean cloud provider. @@ -895,6 +898,10 @@ func (c *Cluster) UsesNoneDNS() bool { return false } +func (c *Cluster) UsesExternalECRCredentialsProvider() bool { + return c.IsKubernetesGTE("1.27") && c.Spec.GetCloudProvider() == CloudProviderAWS +} + func (c *Cluster) APIInternalName() string { return "api.internal." + c.ObjectMeta.Name } diff --git a/pkg/apis/kops/v1alpha3/cluster.go b/pkg/apis/kops/v1alpha3/cluster.go index 677476e00f..d9e029fdd9 100644 --- a/pkg/apis/kops/v1alpha3/cluster.go +++ b/pkg/apis/kops/v1alpha3/cluster.go @@ -216,6 +216,9 @@ type AWSSpec struct { // Spotinst cloud-config specs SpotinstProduct *string `json:"spotinstProduct,omitempty"` SpotinstOrientation *string `json:"spotinstOrientation,omitempty"` + + // BinariesLocation is the location of the AWS cloud provider binaries. + BinariesLocation *string `json:"binaryLocation,omitempty"` } // DOSpec configures the Digital Ocean cloud provider. diff --git a/pkg/apis/kops/v1alpha3/zz_generated.conversion.go b/pkg/apis/kops/v1alpha3/zz_generated.conversion.go index ec5f67214e..4f405102d5 100644 --- a/pkg/apis/kops/v1alpha3/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha3/zz_generated.conversion.go @@ -1478,6 +1478,7 @@ func autoConvert_v1alpha3_AWSSpec_To_kops_AWSSpec(in *AWSSpec, out *kops.AWSSpec out.ElbSecurityGroup = in.ElbSecurityGroup out.SpotinstProduct = in.SpotinstProduct out.SpotinstOrientation = in.SpotinstOrientation + out.BinariesLocation = in.BinariesLocation return nil } @@ -1537,6 +1538,7 @@ func autoConvert_kops_AWSSpec_To_v1alpha3_AWSSpec(in *kops.AWSSpec, out *AWSSpec out.ElbSecurityGroup = in.ElbSecurityGroup out.SpotinstProduct = in.SpotinstProduct out.SpotinstOrientation = in.SpotinstOrientation + out.BinariesLocation = in.BinariesLocation return nil } diff --git a/pkg/apis/kops/v1alpha3/zz_generated.deepcopy.go b/pkg/apis/kops/v1alpha3/zz_generated.deepcopy.go index a6cd40f877..e2f2296127 100644 --- a/pkg/apis/kops/v1alpha3/zz_generated.deepcopy.go +++ b/pkg/apis/kops/v1alpha3/zz_generated.deepcopy.go @@ -203,6 +203,11 @@ func (in *AWSSpec) DeepCopyInto(out *AWSSpec) { *out = new(string) **out = **in } + if in.BinariesLocation != nil { + in, out := &in.BinariesLocation, &out.BinariesLocation + *out = new(string) + **out = **in + } return } diff --git a/pkg/apis/kops/zz_generated.deepcopy.go b/pkg/apis/kops/zz_generated.deepcopy.go index bd5ba5bfcc..228209fe97 100644 --- a/pkg/apis/kops/zz_generated.deepcopy.go +++ b/pkg/apis/kops/zz_generated.deepcopy.go @@ -202,6 +202,11 @@ func (in *AWSSpec) DeepCopyInto(out *AWSSpec) { *out = new(string) **out = **in } + if in.BinariesLocation != nil { + in, out := &in.BinariesLocation, &out.BinariesLocation + *out = new(string) + **out = **in + } return } diff --git a/tests/integration/update_cluster/minimal/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data b/tests/integration/update_cluster/minimal/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data index 8ca31ad8b8..33a7f3144f 100644 --- a/tests/integration/update_cluster/minimal/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data +++ b/tests/integration/update_cluster/minimal/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data @@ -250,7 +250,7 @@ CloudProvider: aws ConfigBase: memfs://clusters.example.com/minimal.example.com InstanceGroupName: master-us-test-1a InstanceGroupRole: ControlPlane -NodeupConfigHash: N/3CvlvqplvxWXN3+r1r09mA/Y55Wt/klapVL1nCd2w= +NodeupConfigHash: rJVuGlmAuBwCe2RWXKgT1dI+bdFv3FhvakVQhTVudXU= __EOF_KUBE_ENV diff --git a/tests/integration/update_cluster/minimal/data/aws_launch_template_nodes.minimal.example.com_user_data b/tests/integration/update_cluster/minimal/data/aws_launch_template_nodes.minimal.example.com_user_data index 79d28dae2a..966b62b67c 100644 --- a/tests/integration/update_cluster/minimal/data/aws_launch_template_nodes.minimal.example.com_user_data +++ b/tests/integration/update_cluster/minimal/data/aws_launch_template_nodes.minimal.example.com_user_data @@ -174,7 +174,7 @@ ConfigServer: - https://kops-controller.internal.minimal.example.com:3988/ InstanceGroupName: nodes InstanceGroupRole: Node -NodeupConfigHash: g+og1yXE09z0oZE2ggS/s5xmrp7CHsM9PbOhPFs8cNs= +NodeupConfigHash: jg6YaOqnx4/XWWzsT5/JFgNHr3tC6qgvfQ6M1CfY6Jk= __EOF_KUBE_ENV diff --git a/tests/integration/update_cluster/minimal/data/aws_s3_object_nodeupconfig-master-us-test-1a_content b/tests/integration/update_cluster/minimal/data/aws_s3_object_nodeupconfig-master-us-test-1a_content index 04c6b37f32..1d759f8bdf 100644 --- a/tests/integration/update_cluster/minimal/data/aws_s3_object_nodeupconfig-master-us-test-1a_content +++ b/tests/integration/update_cluster/minimal/data/aws_s3_object_nodeupconfig-master-us-test-1a_content @@ -56,6 +56,7 @@ Assets: amd64: - be5e79c70e926019e588c8c5c44d93efb1042f3be6f1db0de14e707141564d29@https://storage.googleapis.com/kubernetes-release/release/v1.27.0-alpha.3/bin/linux/amd64/kubelet - 4d416a4bed7c1bb576e284fc6b0bb843f879c08fa8e4beb7e2376305d2dc8299@https://storage.googleapis.com/kubernetes-release/release/v1.27.0-alpha.3/bin/linux/amd64/kubectl + - 5035d7814c95cd3cedbc5efb447ef25a4942ef05caab2159746d55ce1698c74a@https://artifacts.k8s.io/binaries/cloud-provider-aws/v1.27.1/linux/amd64/ecr-credential-provider-linux-amd64 - 962100bbc4baeaaa5748cdbfce941f756b1531c2eadb290129401498bfac21e7@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.9.1/cni-plugins-linux-amd64-v0.9.1.tgz - bb9a9ccd6517e2a54da748a9f60dc9aa9d79d19d4724663f2386812f083968e2@https://github.com/containerd/containerd/releases/download/v1.6.20/containerd-1.6.20-linux-amd64.tar.gz - db772be63147a4e747b4fe286c7c16a2edc4a8458bd3092ea46aaee77750e8ce@https://github.com/opencontainers/runc/releases/download/v1.1.4/runc.amd64 @@ -64,6 +65,7 @@ Assets: arm64: - 2a15e9c291dce3e07db5d9948d5c37a4e52c9884343f63287189e4f0d4c3df76@https://storage.googleapis.com/kubernetes-release/release/v1.27.0-alpha.3/bin/linux/arm64/kubelet - c5fad9f96ab5fe04b8927ce83ceefc4db65b032303a23b35e353276479450d2a@https://storage.googleapis.com/kubernetes-release/release/v1.27.0-alpha.3/bin/linux/arm64/kubectl + - b3d567bda9e2996fc1fbd9d13506bd16763d3865b5c7b0b3c4b48c6088c04481@https://artifacts.k8s.io/binaries/cloud-provider-aws/v1.27.1/linux/arm64/ecr-credential-provider-linux-arm64 - ef17764ffd6cdcb16d76401bac1db6acc050c9b088f1be5efa0e094ea3b01df0@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.9.1/cni-plugins-linux-arm64-v0.9.1.tgz - c3e6a054b18b20fce06c7c3ed53f0989bb4b255c849bede446ebca955f07a9ce@https://github.com/containerd/containerd/releases/download/v1.6.20/containerd-1.6.20-linux-arm64.tar.gz - dbb71e737eaef454a406ce21fd021bd8f1b35afb7635016745992bbd7c17a223@https://github.com/opencontainers/runc/releases/download/v1.1.4/runc.arm64 diff --git a/tests/integration/update_cluster/minimal/data/aws_s3_object_nodeupconfig-nodes_content b/tests/integration/update_cluster/minimal/data/aws_s3_object_nodeupconfig-nodes_content index 91c90a0213..6832a21bd5 100644 --- a/tests/integration/update_cluster/minimal/data/aws_s3_object_nodeupconfig-nodes_content +++ b/tests/integration/update_cluster/minimal/data/aws_s3_object_nodeupconfig-nodes_content @@ -2,12 +2,14 @@ Assets: amd64: - be5e79c70e926019e588c8c5c44d93efb1042f3be6f1db0de14e707141564d29@https://storage.googleapis.com/kubernetes-release/release/v1.27.0-alpha.3/bin/linux/amd64/kubelet - 4d416a4bed7c1bb576e284fc6b0bb843f879c08fa8e4beb7e2376305d2dc8299@https://storage.googleapis.com/kubernetes-release/release/v1.27.0-alpha.3/bin/linux/amd64/kubectl + - 5035d7814c95cd3cedbc5efb447ef25a4942ef05caab2159746d55ce1698c74a@https://artifacts.k8s.io/binaries/cloud-provider-aws/v1.27.1/linux/amd64/ecr-credential-provider-linux-amd64 - 962100bbc4baeaaa5748cdbfce941f756b1531c2eadb290129401498bfac21e7@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.9.1/cni-plugins-linux-amd64-v0.9.1.tgz - bb9a9ccd6517e2a54da748a9f60dc9aa9d79d19d4724663f2386812f083968e2@https://github.com/containerd/containerd/releases/download/v1.6.20/containerd-1.6.20-linux-amd64.tar.gz - db772be63147a4e747b4fe286c7c16a2edc4a8458bd3092ea46aaee77750e8ce@https://github.com/opencontainers/runc/releases/download/v1.1.4/runc.amd64 arm64: - 2a15e9c291dce3e07db5d9948d5c37a4e52c9884343f63287189e4f0d4c3df76@https://storage.googleapis.com/kubernetes-release/release/v1.27.0-alpha.3/bin/linux/arm64/kubelet - c5fad9f96ab5fe04b8927ce83ceefc4db65b032303a23b35e353276479450d2a@https://storage.googleapis.com/kubernetes-release/release/v1.27.0-alpha.3/bin/linux/arm64/kubectl + - b3d567bda9e2996fc1fbd9d13506bd16763d3865b5c7b0b3c4b48c6088c04481@https://artifacts.k8s.io/binaries/cloud-provider-aws/v1.27.1/linux/arm64/ecr-credential-provider-linux-arm64 - ef17764ffd6cdcb16d76401bac1db6acc050c9b088f1be5efa0e094ea3b01df0@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.9.1/cni-plugins-linux-arm64-v0.9.1.tgz - c3e6a054b18b20fce06c7c3ed53f0989bb4b255c849bede446ebca955f07a9ce@https://github.com/containerd/containerd/releases/download/v1.6.20/containerd-1.6.20-linux-arm64.tar.gz - dbb71e737eaef454a406ce21fd021bd8f1b35afb7635016745992bbd7c17a223@https://github.com/opencontainers/runc/releases/download/v1.1.4/runc.arm64 diff --git a/upup/pkg/fi/cloudup/apply_cluster.go b/upup/pkg/fi/cloudup/apply_cluster.go index fe523dfe99..b2f375c3be 100644 --- a/upup/pkg/fi/cloudup/apply_cluster.go +++ b/upup/pkg/fi/cloudup/apply_cluster.go @@ -1056,6 +1056,23 @@ func (c *ApplyClusterCmd) addFileAssets(assetBuilder *assets.AssetBuilder) error c.Assets[arch] = append(c.Assets[arch], mirrors.BuildMirroredAsset(u, hash)) } + if c.Cluster.UsesExternalECRCredentialsProvider() { + binaryLocation := c.Cluster.Spec.CloudProvider.AWS.BinariesLocation + if binaryLocation == nil { + binaryLocation = fi.PtrTo("https://artifacts.k8s.io/binaries/cloud-provider-aws/v1.27.1") + } + + k, err := url.Parse(fmt.Sprintf("%s/linux/%s/ecr-credential-provider-linux-%s", *binaryLocation, arch, arch)) + if err != nil { + return err + } + u, hash, err := assetBuilder.RemapFileAndSHA(k) + if err != nil { + return err + } + c.Assets[arch] = append(c.Assets[arch], mirrors.BuildMirroredAsset(u, hash)) + } + cniAsset, cniAssetHash, err := findCNIAssets(c.Cluster, assetBuilder, arch) if err != nil { return err