kubernetes
/
kops
mirror of https://github.com/kubernetes/kops.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

add support for auth-provider-gcp cred provider

This commit is contained in:
upodroid 2023-10-10 10:29:16 +01:00
parent d630f9c520
commit 2b056b9b17
10 changed files with 153 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docs/releases/1.29-NOTES.md
View File

@ -10,6 +10,8 @@ This is a document to gather the release notes prior to the release.
## GCP
* As of Kubernetes version 1.29, credentials for private GCR/AR repositories will be handled by the out-of-tree credential provider. This is an additional binary that each instance downloads from the assets repository.
## Openstack
# Breaking changes

4
nodeup/pkg/model/context.go
View File

@ -329,8 +329,8 @@ func (c *NodeupModelContext) UseChallengeCallback(cloudProvider kops.CloudProvid
return model.UseChallengeCallback(cloudProvider)
}
func (c *NodeupModelContext) UseExternalECRCredentialsProvider() bool {
return model.UseExternalECRCredentialsProvider(c.kubernetesVersion, c.CloudProvider())
func (c *NodeupModelContext) UseExternalKubeletCredentialProvider() bool {
return model.UseExternalKubeletCredentialProvider(c.kubernetesVersion, c.CloudProvider())
}
// UsesSecondaryIP checks if the CNI in use attaches secondary interfaces to the host.

86
nodeup/pkg/model/kubelet.go
View File

@ -96,7 +96,7 @@ func (b *KubeletBuilder) Build(c *fi.NodeupModelBuilderContext) error {
// @TODO make Find call to an interface, we cannot mock out this function because it finds a file on disk
asset, err := b.Assets.Find(assetName, assetPath)
if err != nil {
return fmt.Errorf("error trying to locate asset %q: %v", assetName, err)
return fmt.Errorf("trying to locate asset %q: %v", assetName, err)
}
if asset == nil {
return fmt.Errorf("unable to locate asset %q", assetName)
@ -158,9 +158,16 @@ func (b *KubeletBuilder) Build(c *fi.NodeupModelBuilderContext) error {
return err
}
if b.UseExternalECRCredentialsProvider() {
if err := b.addECRCP(c); err != nil {
return fmt.Errorf("failed to add ECR credential provider: %w", err)
if b.UseExternalKubeletCredentialProvider() {
switch b.CloudProvider() {
case kops.CloudProviderGCE:
if err := b.addGCPCredentialProvider(c); err != nil {
return fmt.Errorf("failed to add the %s kubelet credential provider: %w", b.CloudProvider(), err)
}
case kops.CloudProviderAWS:
if err := b.addECRCredentialProvider(c); err != nil {
return fmt.Errorf("failed to add the %s kubelet credential provider: %w", b.CloudProvider(), err)
}
}
}
@ -265,11 +272,16 @@ func (b *KubeletBuilder) kubeletPath() string {
return b.binaryPath() + "/kubelet"
}
// ecrcpPath returns the path of the ECR credentials provider based on distro and archiecture
func (b *KubeletBuilder) ecrcpPath() string {
// getECRCredentialProviderPath returns the path of the ECR Credentials Provider based on distro and archiecture
func (b *KubeletBuilder) getECRCredentialProviderPath() string {
return b.binaryPath() + "/ecr-credential-provider"
}
// getGCPCredentialProviderPath returns the path of the GCP Credentials Provider based on distro and archiecture
func (b *KubeletBuilder) getGCPCredentialProviderPath() string {
return b.binaryPath() + "/gcp-credential-provider"
}
// buildManifestDirectory creates the directory where kubelet expects static manifests to reside
func (b *KubeletBuilder) buildManifestDirectory(kubeletConfig *kops.KubeletConfigSpec) (*nodetasks.File, error) {
if kubeletConfig.PodManifestPath == "" {
@ -328,7 +340,7 @@ func (b *KubeletBuilder) buildSystemdEnvironmentFile(kubeletConfig *kops.Kubelet
flags += " --config=" + kubeletConfigFilePath
if b.UseExternalECRCredentialsProvider() {
if b.UseExternalKubeletCredentialProvider() {
flags += " --image-credential-provider-config=" + credentialProviderConfigFilePath
flags += " --image-credential-provider-bin-dir=" + b.binaryPath()
}
@ -403,21 +415,21 @@ func (b *KubeletBuilder) usesContainerizedMounter() bool {
}
}
// addECRCP installs the ECR credential provider
func (b *KubeletBuilder) addECRCP(c *fi.NodeupModelBuilderContext) error {
// addECRCredentialProvider installs the ECR Kubelet Credential Provider
func (b *KubeletBuilder) addECRCredentialProvider(c *fi.NodeupModelBuilderContext) error {
{
assetName := "ecr-credential-provider-linux-" + string(b.Architecture)
assetPath := ""
asset, err := b.Assets.Find(assetName, assetPath)
if err != nil {
return fmt.Errorf("error trying to locate asset %q: %v", assetName, err)
return fmt.Errorf("trying to locate asset %q: %v", assetName, err)
}
if asset == nil {
return fmt.Errorf("unable to locate asset %q", assetName)
}
t := &nodetasks.File{
Path: b.ecrcpPath(),
Path: b.getECRCredentialProviderPath(),
Contents: asset,
Type: nodetasks.FileType_File,
Mode: s("0755"),
@ -453,6 +465,56 @@ providers:
return nil
}
// addGCPCredentialProvider installs the GCP Kubelet Credential Provider
func (b *KubeletBuilder) addGCPCredentialProvider(c *fi.NodeupModelBuilderContext) error {
{
assetName := "v20231005-providersv0.27.1-65-g8fbe8d27"
assetPath := ""
asset, err := b.Assets.Find(assetName, assetPath)
if err != nil {
return fmt.Errorf("trying to locate asset %q: %v", assetName, err)
}
if asset == nil {
return fmt.Errorf("unable to locate asset %q", assetName)
}
t := &nodetasks.File{
Path: b.getGCPCredentialProviderPath(),
Contents: asset,
Type: nodetasks.FileType_File,
Mode: s("0755"),
}
c.AddTask(t)
}
{
configContent := `apiVersion: kubelet.config.k8s.io/v1
kind: CredentialProviderConfig
providers:
- apiVersion: credentialprovider.kubelet.k8s.io/v1
name: gcp-credential-provider
matchImages:
- "gcr.io"
- "*.gcr.io"
- "container.cloud.google.com"
- "*.pkg.dev"
defaultCacheDuration: "1m"
args:
- get-credentials
- --v=3
`
t := &nodetasks.File{
Path: credentialProviderConfigFilePath,
Contents: fi.NewStringResource(configContent),
Type: nodetasks.FileType_File,
Mode: s("0644"),
}
c.AddTask(t)
}
return nil
}
// addContainerizedMounter downloads and installs the containerized mounter, that we need on ContainerOS
func (b *KubeletBuilder) addContainerizedMounter(c *fi.NodeupModelBuilderContext) error {
if !b.usesContainerizedMounter() {
@ -472,7 +534,7 @@ func (b *KubeletBuilder) addContainerizedMounter(c *fi.NodeupModelBuilderContext
assetPath := ""
asset, err := b.Assets.Find(assetName, assetPath)
if err != nil {
return fmt.Errorf("error trying to locate asset %q: %v", assetName, err)
return fmt.Errorf("trying to locate asset %q: %v", assetName, err)
}
if asset == nil {
return fmt.Errorf("unable to locate asset %q", assetName)

5
pkg/apis/kops/cluster.go
View File

@ -227,7 +227,7 @@ type AWSSpec struct {
SpotinstOrientation *string `json:"spotinstOrientation,omitempty"`
// BinariesLocation is the location of the AWS cloud provider binaries.
BinariesLocation *string `json:"binaryLocation,omitempty"`
BinariesLocation *string `json:"binariesLocation,omitempty"`
}
// DOSpec configures the Digital Ocean cloud provider.
@ -244,6 +244,9 @@ type GCESpec struct {
NodeInstancePrefix *string `json:"nodeInstancePrefix,omitempty"`
// PDCSIDriver is the config for the PD CSI driver.
PDCSIDriver *PDCSIDriver `json:"pdCSIDriver,omitempty"`
// BinariesLocation is the location of the GCE cloud provider binaries.
BinariesLocation *string `json:"binariesLocation,omitempty"`
}
// HetznerSpec configures the Hetzner cloud provider.

12
pkg/apis/kops/model/features.go
View File

@ -69,6 +69,14 @@ func UseCiliumEtcd(cluster *kops.Cluster) bool {
return false
}
func UseExternalECRCredentialsProvider(k8sVersion semver.Version, cloudProvider kops.CloudProviderID) bool {
return util.IsKubernetesGTE("1.27", k8sVersion) && cloudProvider == kops.CloudProviderAWS
// Configures a Kubelet Credential Provider if Kubernetes is newer than a specific version
func UseExternalKubeletCredentialProvider(k8sVersion semver.Version, cloudProvider kops.CloudProviderID) bool {
switch cloudProvider {
case kops.CloudProviderGCE:
return util.IsKubernetesGTE("1.29", k8sVersion)
case kops.CloudProviderAWS:
return util.IsKubernetesGTE("1.27", k8sVersion)
default:
return false
}
}

5
pkg/apis/kops/v1alpha3/cluster.go
View File

@ -223,7 +223,7 @@ type AWSSpec struct {
SpotinstOrientation *string `json:"spotinstOrientation,omitempty"`
// BinariesLocation is the location of the AWS cloud provider binaries.
BinariesLocation *string `json:"binaryLocation,omitempty"`
BinariesLocation *string `json:"binariesLocation,omitempty"`
}
// DOSpec configures the Digital Ocean cloud provider.
@ -240,6 +240,9 @@ type GCESpec struct {
NodeInstancePrefix *string `json:"nodeInstancePrefix,omitempty"`
// PDCSIDriver is the config for the PD CSI driver.
PDCSIDriver *PDCSIDriver `json:"pdCSIDriver,omitempty"`
// BinariesLocation is the location of the GCE cloud provider binaries.
BinariesLocation *string `json:"binariesLocation,omitempty"`
}
// HetznerSpec configures the Hetzner cloud provider.

2
pkg/apis/kops/v1alpha3/zz_generated.conversion.go generated
View File

@ -4128,6 +4128,7 @@ func autoConvert_v1alpha3_GCESpec_To_kops_GCESpec(in *GCESpec, out *kops.GCESpec
} else {
out.PDCSIDriver = nil
}
out.BinariesLocation = in.BinariesLocation
return nil
}
@ -4151,6 +4152,7 @@ func autoConvert_kops_GCESpec_To_v1alpha3_GCESpec(in *kops.GCESpec, out *GCESpec
} else {
out.PDCSIDriver = nil
}
out.BinariesLocation = in.BinariesLocation
return nil
}

5
pkg/apis/kops/v1alpha3/zz_generated.deepcopy.go generated
View File

@ -2059,6 +2059,11 @@ func (in *GCESpec) DeepCopyInto(out *GCESpec) {
*out = new(PDCSIDriver)
(*in).DeepCopyInto(*out)
}
if in.BinariesLocation != nil {
in, out := &in.BinariesLocation, &out.BinariesLocation
*out = new(string)
**out = **in
}
return
}

5
pkg/apis/kops/zz_generated.deepcopy.go generated
View File

@ -2222,6 +2222,11 @@ func (in *GCESpec) DeepCopyInto(out *GCESpec) {
*out = new(PDCSIDriver)
(*in).DeepCopyInto(*out)
}
if in.BinariesLocation != nil {
in, out := &in.BinariesLocation, &out.BinariesLocation
*out = new(string)
**out = **in
}
return
}

58
upup/pkg/fi/cloudup/apply_cluster.go
View File

@ -72,6 +72,7 @@ import (
"k8s.io/kops/upup/pkg/fi/cloudup/terraform"
"k8s.io/kops/upup/pkg/fi/cloudup/terraformWriter"
"k8s.io/kops/util/pkg/architectures"
"k8s.io/kops/util/pkg/hashing"
"k8s.io/kops/util/pkg/mirrors"
"k8s.io/kops/util/pkg/vfs"
)
@ -1055,21 +1056,52 @@ func (c *ApplyClusterCmd) addFileAssets(assetBuilder *assets.AssetBuilder) error
}
kubernetesVersion, _ := util.ParseKubernetesVersion(c.Cluster.Spec.KubernetesVersion)
if apiModel.UseExternalECRCredentialsProvider(*kubernetesVersion, c.Cluster.Spec.GetCloudProvider()) {
binaryLocation := c.Cluster.Spec.CloudProvider.AWS.BinariesLocation
if binaryLocation == nil {
binaryLocation = fi.PtrTo("https://artifacts.k8s.io/binaries/cloud-provider-aws/v1.27.1")
}
k, err := url.Parse(fmt.Sprintf("%s/linux/%s/ecr-credential-provider-linux-%s", *binaryLocation, arch, arch))
if err != nil {
return err
cloudProvider := c.Cluster.Spec.GetCloudProvider()
if ok := apiModel.UseExternalKubeletCredentialProvider(*kubernetesVersion, cloudProvider); ok {
switch cloudProvider {
case kops.CloudProviderGCE:
binaryLocation := c.Cluster.Spec.CloudProvider.GCE.BinariesLocation
if binaryLocation == nil {
binaryLocation = fi.PtrTo("https://storage.googleapis.com/k8s-staging-cloud-provider-gcp/auth-provider-gcp")
}
// VALID FOR 60 DAYS WE REALLY NEED TO MERGE https://github.com/kubernetes/cloud-provider-gcp/pull/601 and CUT A RELEASE
k, err := url.Parse(fmt.Sprintf("%s/linux-%s/v20231005-providersv0.27.1-65-g8fbe8d27", *binaryLocation, arch))
if err != nil {
return err
}
hashes := map[architectures.Architecture]string{
"amd64": "827d558953d861b81a35c3b599191a73f53c1f63bce42c61e7a3fee21a717a89",
"arm64": "f1617c0ef77f3718e12a3efc6f650375d5b5e96eebdbcbad3e465e89e781bdfa",
}
hash, err := hashing.FromString(hashes[arch])
if err != nil {
return fmt.Errorf("unable to parse auth-provider-gcp binary asset hash %q: %v", hashes[arch], err)
}
u, err := assetBuilder.RemapFileAndSHAValue(k, hashes[arch])
if err != nil {
return err
}
c.Assets[arch] = append(c.Assets[arch], mirrors.BuildMirroredAsset(u, hash))
case kops.CloudProviderAWS:
binaryLocation := c.Cluster.Spec.CloudProvider.AWS.BinariesLocation
if binaryLocation == nil {
binaryLocation = fi.PtrTo("https://artifacts.k8s.io/binaries/cloud-provider-aws/v1.27.1")
}
k, err := url.Parse(fmt.Sprintf("%s/linux/%s/ecr-credential-provider-linux-%s", *binaryLocation, arch, arch))
if err != nil {
return err
}
u, hash, err := assetBuilder.RemapFileAndSHA(k)
if err != nil {
return err
}
c.Assets[arch] = append(c.Assets[arch], mirrors.BuildMirroredAsset(u, hash))
}
u, hash, err := assetBuilder.RemapFileAndSHA(k)
if err != nil {
return err
}
c.Assets[arch] = append(c.Assets[arch], mirrors.BuildMirroredAsset(u, hash))
}
{