fluxcd
/
helm-controller
mirror of https://github.com/fluxcd/helm-controller.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

kube: unify clients into single RESTClientGetter 

This drops the twofold implementation in favor of a single
`MemoryRESTClientGetter` which can work with an arbitrary `rest.Config`.

The new `MemoryRESTClientGetter` lazy-loads and caches the objects it
initializes, thereby creating at most one instance of each object for
the duration of the reconcile of a single `HelmRelease` object.

Based on some initial tests, this seems to reduce the overal memory
footprint of the controller.

Signed-off-by: Hidde Beydals <hidde@hhh.computer>
This commit is contained in:
Hidde Beydals 2023-03-04 13:46:52 +01:00
parent 90a03d05f6
commit 34d87ccc24
No known key found for this signature in database
GPG Key ID: 979F380FC2341744
8 changed files with 422 additions and 481 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
controllers/helmrelease_controller.go
View File

@ -520,7 +520,8 @@ func (r *HelmReleaseReconciler) checkDependencies(hr v2.HelmRelease) error {
}
func (r *HelmReleaseReconciler) buildRESTClientGetter(ctx context.Context, hr v2.HelmRelease) (genericclioptions.RESTClientGetter, error) {
opts := []kube.ClientGetterOption{
opts := []kube.Option{
kube.WithNamespace(hr.GetReleaseNamespace()),
kube.WithClientOptions(r.ClientOpts),
// When ServiceAccountName is empty, it will fall back to the configured default.
// If this is not configured either, this option will result in a no-op.
@ -535,13 +536,13 @@ func (r *HelmReleaseReconciler) buildRESTClientGetter(ctx context.Context, hr v2
if err := r.Get(ctx, secretName, &secret); err != nil {
return nil, fmt.Errorf("could not find KubeConfig secret '%s': %w", secretName, err)
}
kubeConfig, err := kube.ConfigFromSecret(&secret, hr.Spec.KubeConfig.SecretRef.Key)
kubeConfig, err := kube.ConfigFromSecret(&secret, hr.Spec.KubeConfig.SecretRef.Key, r.KubeConfigOpts)
if err != nil {
return nil, err
}
opts = append(opts, kube.WithKubeConfig(kubeConfig, r.KubeConfigOpts))
return kube.NewMemoryRESTClientGetter(kubeConfig, opts...), nil
}
return kube.BuildClientGetter(hr.GetReleaseNamespace(), opts...)
return kube.NewInClusterMemoryRESTClientGetter(opts...)
}
// composeValues attempts to resolve all v2beta1.ValuesReference resources

2
go.mod
View File

@ -24,7 +24,6 @@ require (
k8s.io/apimachinery v0.26.2
k8s.io/cli-runtime v0.26.2
k8s.io/client-go v0.26.2
k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5
sigs.k8s.io/cli-utils v0.34.0
sigs.k8s.io/controller-runtime v0.14.5
sigs.k8s.io/kustomize/api v0.12.1
@ -159,6 +158,7 @@ require (
k8s.io/klog/v2 v2.90.1 // indirect
k8s.io/kube-openapi v0.0.0-20221110221610-a28e98eb7c70 // indirect
k8s.io/kubectl v0.26.0 // indirect
k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5 // indirect
oras.land/oras-go v1.2.2 // indirect
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
sigs.k8s.io/kustomize/kyaml v0.13.9 // indirect

89
internal/kube/builder.go
View File

@ -1,89 +0,0 @@
/*
Copyright 2022 The Flux authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package kube
import (
"github.com/fluxcd/pkg/runtime/client"
"k8s.io/cli-runtime/pkg/genericclioptions"
)
const (
// DefaultKubeConfigSecretKey is the default data key ConfigFromSecret
// looks at when no data key is provided.
DefaultKubeConfigSecretKey = "value"
// DefaultKubeConfigSecretKeyExt is the default data key ConfigFromSecret
// looks at when no data key is provided, and DefaultKubeConfigSecretKey
// does not exist.
DefaultKubeConfigSecretKeyExt = DefaultKubeConfigSecretKey + ".yaml"
)
// clientGetterOptions used to BuildClientGetter.
type clientGetterOptions struct {
namespace string
kubeConfig []byte
impersonateAccount string
impersonateNamespace string
clientOptions client.Options
kubeConfigOptions client.KubeConfigOptions
}
// ClientGetterOption configures a genericclioptions.RESTClientGetter.
type ClientGetterOption func(o *clientGetterOptions)
// WithKubeConfig creates a MemoryRESTClientGetter configured with the provided
// KubeConfig and other values.
func WithKubeConfig(kubeConfig []byte, opts client.KubeConfigOptions) func(o *clientGetterOptions) {
return func(o *clientGetterOptions) {
o.kubeConfig = kubeConfig
o.kubeConfigOptions = opts
}
}
// WithClientOptions configures the genericclioptions.RESTClientGetter with
// provided options.
func WithClientOptions(opts client.Options) func(o *clientGetterOptions) {
return func(o *clientGetterOptions) {
o.clientOptions = opts
}
}
// WithImpersonate configures the genericclioptions.RESTClientGetter to
// impersonate with the given account name in the provided namespace.
// If the account name is empty, DefaultServiceAccountName is assumed.
func WithImpersonate(accountName, namespace string) func(o *clientGetterOptions) {
return func(o *clientGetterOptions) {
o.impersonateAccount = accountName
o.impersonateNamespace = namespace
}
}
// BuildClientGetter builds a genericclioptions.RESTClientGetter based on the
// provided options and returns the result. Namespace is not expected to be
// empty. In case it fails to construct using NewInClusterRESTClientGetter, it
// returns an error.
func BuildClientGetter(namespace string, opts ...ClientGetterOption) (genericclioptions.RESTClientGetter, error) {
o := &clientGetterOptions{
namespace: namespace,
}
for _, opt := range opts {
opt(o)
}
if len(o.kubeConfig) > 0 {
return NewMemoryRESTClientGetter(o.kubeConfig, namespace, o.impersonateAccount, o.impersonateNamespace, o.clientOptions, o.kubeConfigOptions), nil
}
return NewInClusterRESTClientGetter(namespace, o.impersonateAccount, o.impersonateNamespace, &o.clientOptions)
}

121
internal/kube/builder_test.go
View File

@ -1,121 +0,0 @@
/*
Copyright 2022 The Flux authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package kube
import (
"k8s.io/client-go/rest"
ctrl "sigs.k8s.io/controller-runtime"
"testing"
"github.com/fluxcd/pkg/runtime/client"
. "github.com/onsi/gomega"
"k8s.io/cli-runtime/pkg/genericclioptions"
)
func TestBuildClientGetter(t *testing.T) {
t.Run("with namespace and retrieved config", func(t *testing.T) {
g := NewWithT(t)
cfg := &rest.Config{Host: "https://example.com"}
ctrl.GetConfig = func() (*rest.Config, error) {
return cfg, nil
}
namespace := "a-namespace"
getter, err := BuildClientGetter(namespace)
g.Expect(err).ToNot(HaveOccurred())
g.Expect(getter).To(BeAssignableToTypeOf(&genericclioptions.ConfigFlags{}))
flags := getter.(*genericclioptions.ConfigFlags)
g.Expect(flags.Namespace).ToNot(BeNil())
g.Expect(*flags.Namespace).To(Equal(namespace))
g.Expect(flags.APIServer).ToNot(BeNil())
g.Expect(*flags.APIServer).To(Equal(cfg.Host))
})
t.Run("with kubeconfig, impersonate and client options", func(t *testing.T) {
g := NewWithT(t)
ctrl.GetConfig = mockGetConfig
namespace := "a-namespace"
cfg := []byte(`apiVersion: v1
clusters:
- cluster:
server: https://example.com
name: example-cluster
contexts:
- context:
cluster: example-cluster
namespace: flux-system
kind: Config
preferences: {}
users:`)
clientOpts := client.Options{QPS: 600, Burst: 1000}
cfgOpts := client.KubeConfigOptions{InsecureTLS: true}
impersonate := "jane"
getter, err := BuildClientGetter(namespace, WithClientOptions(clientOpts), WithKubeConfig(cfg, cfgOpts), WithImpersonate(impersonate, ""))
g.Expect(err).ToNot(HaveOccurred())
g.Expect(getter).To(BeAssignableToTypeOf(&MemoryRESTClientGetter{}))
got := getter.(*MemoryRESTClientGetter)
g.Expect(got.namespace).To(Equal(namespace))
g.Expect(got.kubeConfig).To(Equal(cfg))
g.Expect(got.clientOpts).To(Equal(clientOpts))
g.Expect(got.kubeConfigOpts).To(Equal(cfgOpts))
g.Expect(got.impersonateAccount).To(Equal(impersonate))
})
t.Run("with impersonate account", func(t *testing.T) {
g := NewWithT(t)
ctrl.GetConfig = mockGetConfig
namespace := "a-namespace"
impersonate := "frank"
impersonateNS := "other-namespace"
getter, err := BuildClientGetter(namespace, WithImpersonate(impersonate, impersonateNS))
g.Expect(err).ToNot(HaveOccurred())
g.Expect(getter).To(BeAssignableToTypeOf(&genericclioptions.ConfigFlags{}))
flags := getter.(*genericclioptions.ConfigFlags)
g.Expect(flags.Namespace).ToNot(BeNil())
g.Expect(*flags.Namespace).To(Equal(namespace))
g.Expect(flags.Impersonate).ToNot(BeNil())
g.Expect(*flags.Impersonate).To(Equal("system:serviceaccount:other-namespace:frank"))
})
t.Run("with impersonate DefaultServiceAccount", func(t *testing.T) {
g := NewWithT(t)
ctrl.GetConfig = mockGetConfig
namespace := "a-namespace"
DefaultServiceAccountName = "frank"
impersonateNS := "other-namespace"
getter, err := BuildClientGetter(namespace, WithImpersonate("", impersonateNS))
g.Expect(err).ToNot(HaveOccurred())
g.Expect(getter).To(BeAssignableToTypeOf(&genericclioptions.ConfigFlags{}))
flags := getter.(*genericclioptions.ConfigFlags)
g.Expect(flags.Namespace).ToNot(BeNil())
g.Expect(*flags.Namespace).To(Equal(namespace))
g.Expect(flags.Impersonate).ToNot(BeNil())
g.Expect(*flags.Impersonate).To(Equal("system:serviceaccount:other-namespace:frank"))
})
}
func mockGetConfig() (*rest.Config, error) {
return &rest.Config{}, nil
}

229
internal/kube/client.go
View File

@ -1,5 +1,5 @@
/*
Copyright 2020 The Flux authors
Copyright 2023 The Flux authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
@ -18,142 +18,165 @@ package kube
import (
"fmt"
"sync"
"k8s.io/apimachinery/pkg/api/meta"
"k8s.io/cli-runtime/pkg/genericclioptions"
"k8s.io/client-go/discovery"
"k8s.io/client-go/discovery/cached/memory"
"k8s.io/client-go/rest"
"k8s.io/client-go/restmapper"
"k8s.io/client-go/tools/clientcmd"
"k8s.io/utils/pointer"
controllerruntime "sigs.k8s.io/controller-runtime"
ctrl "sigs.k8s.io/controller-runtime"
"github.com/fluxcd/pkg/runtime/client"
)
// NewInClusterRESTClientGetter creates a new genericclioptions.RESTClientGetter
// using genericclioptions.NewConfigFlags, and configures it with the server,
// authentication, impersonation, client options, and the provided namespace.
// It returns an error if it fails to retrieve a rest.Config.
func NewInClusterRESTClientGetter(namespace, impersonateAccount, impersonateNamespace string, opts *client.Options) (genericclioptions.RESTClientGetter, error) {
cfg, err := controllerruntime.GetConfig()
// Option is a function that configures an MemoryRESTClientGetter.
type Option func(*MemoryRESTClientGetter)
// WithNamespace sets the namespace to use for the client.
func WithNamespace(namespace string) Option {
return func(c *MemoryRESTClientGetter) {
c.namespace = namespace
}
}
// WithImpersonate sets the service account to impersonate. It configures the
// REST client to impersonate the service account in the given namespace, and
// sets the service account name as the username to use in the raw KubeConfig.
func WithImpersonate(serviceAccount, namespace string) Option {
return func(c *MemoryRESTClientGetter) {
if username := SetImpersonationConfig(c.cfg, namespace, serviceAccount); username != "" {
c.impersonate = username
}
}
}
// WithClientOptions sets the client options (e.g. QPS and Burst) to use for
// the client.
func WithClientOptions(opts client.Options) Option {
return func(c *MemoryRESTClientGetter) {
c.cfg.Burst = opts.Burst
c.cfg.QPS = opts.QPS
}
}
// MemoryRESTClientGetter is a resource.RESTClientGetter that uses an
// in-memory REST config, REST mapper, and discovery client. The REST config,
// REST mapper, and discovery client are lazily initialized, and cached for
// subsequent calls.
type MemoryRESTClientGetter struct {
// namespace is the namespace to use for the client.
namespace string
// impersonate is the username to use for the client.
impersonate string
cfg *rest.Config
restMapper meta.RESTMapper
restMapperMu sync.Mutex
discoveryClient discovery.CachedDiscoveryInterface
discoveryMu sync.Mutex
clientCfg clientcmd.ClientConfig
clientCfgMu sync.Mutex
}
// setDefaults sets the default values for the MemoryRESTClientGetter.
func (c *MemoryRESTClientGetter) setDefaults() {
if c.namespace == "" {
c.namespace = "default"
}
}
// NewMemoryRESTClientGetter returns a new MemoryRESTClientGetter.
func NewMemoryRESTClientGetter(cfg *rest.Config, opts ...Option) *MemoryRESTClientGetter {
g := &MemoryRESTClientGetter{
cfg: cfg,
}
for _, opts := range opts {
opts(g)
}
g.setDefaults()
return g
}
// NewInClusterMemoryRESTClientGetter returns a new MemoryRESTClientGetter
// that uses the in-cluster REST config. It returns an error if the in-cluster
// REST config cannot be obtained.
func NewInClusterMemoryRESTClientGetter(opts ...Option) (*MemoryRESTClientGetter, error) {
cfg, err := ctrl.GetConfig()
if err != nil {
return nil, fmt.Errorf("failed to get config for in-cluster REST client: %w", err)
}
SetImpersonationConfig(cfg, impersonateNamespace, impersonateAccount)
flags := genericclioptions.NewConfigFlags(false)
flags.APIServer = pointer.String(cfg.Host)
flags.BearerToken = pointer.String(cfg.BearerToken)
flags.CAFile = pointer.String(cfg.CAFile)
flags.Namespace = pointer.String(namespace)
if opts != nil {
flags.WithDiscoveryBurst(opts.Burst)
flags.WithDiscoveryQPS(opts.QPS)
}
if sa := cfg.Impersonate.UserName; sa != "" {
flags.Impersonate = pointer.String(sa)
}
// In a container, we are not expected to be able to write to the
// home dir default. However, explicitly disabling this is better.
flags.CacheDir = nil
return flags, nil
return NewMemoryRESTClientGetter(cfg, opts...), nil
}
// MemoryRESTClientGetter is an implementation of the genericclioptions.RESTClientGetter,
// capable of working with an in-memory kubeconfig file.
type MemoryRESTClientGetter struct {
// kubeConfig used to load a rest.Config, after being sanitized.
kubeConfig []byte
// kubeConfigOpts controls the sanitization of the kubeConfig.
kubeConfigOpts client.KubeConfigOptions
// clientOpts controls the kube client configuration.
clientOpts client.Options
// namespace specifies the namespace the client is configured to.
namespace string
// impersonateAccount configures the rest.ImpersonationConfig account name.
impersonateAccount string
// impersonateAccount configures the rest.ImpersonationConfig account namespace.
impersonateNamespace string
}
// NewMemoryRESTClientGetter returns a MemoryRESTClientGetter configured with
// the provided values and client.KubeConfigOptions. The provided KubeConfig is
// sanitized, configure the settings for this using client.KubeConfigOptions.
func NewMemoryRESTClientGetter(
kubeConfig []byte,
namespace string,
impersonateAccount string,
impersonateNamespace string,
clientOpts client.Options,
kubeConfigOpts client.KubeConfigOptions) genericclioptions.RESTClientGetter {
return &MemoryRESTClientGetter{
kubeConfig: kubeConfig,
namespace: namespace,
impersonateAccount: impersonateAccount,
impersonateNamespace: impersonateNamespace,
clientOpts: clientOpts,
kubeConfigOpts: kubeConfigOpts,
}
}
// ToRESTConfig creates a rest.Config with the rest.ImpersonationConfig configured
// with to the impersonation account. It loads the config the KubeConfig bytes and
// sanitizes it using the client.KubeConfigOptions.
// ToRESTConfig returns the in-memory REST config.
func (c *MemoryRESTClientGetter) ToRESTConfig() (*rest.Config, error) {
cfg, err := clientcmd.RESTConfigFromKubeConfig(c.kubeConfig)
if err != nil {
return nil, err
if c.cfg == nil {
return nil, fmt.Errorf("MemoryRESTClientGetter has no REST config")
}
cfg = client.KubeConfig(cfg, c.kubeConfigOpts)
SetImpersonationConfig(cfg, c.impersonateNamespace, c.impersonateAccount)
return cfg, nil
return c.cfg, nil
}
// ToDiscoveryClient returns a discovery.CachedDiscoveryInterface configured
// with ToRESTConfig, and the QPS and Burst settings.
// ToDiscoveryClient returns a memory cached discovery client. Calling it
// multiple times will return the same instance.
func (c *MemoryRESTClientGetter) ToDiscoveryClient() (discovery.CachedDiscoveryInterface, error) {
config, err := c.ToRESTConfig()
if err != nil {
return nil, err
}
c.clientCfgMu.Lock()
defer c.clientCfgMu.Unlock()
config.QPS = c.clientOpts.QPS
config.Burst = c.clientOpts.Burst
if c.discoveryClient == nil {
config, err := c.ToRESTConfig()
if err != nil {
return nil, err
}
discoveryClient, err := discovery.NewDiscoveryClientForConfig(config)
if err != nil {
return nil, err
discoveryClient, err := discovery.NewDiscoveryClientForConfig(config)
if err != nil {
return nil, err
}
c.discoveryClient = memory.NewMemCacheClient(discoveryClient)
}
return memory.NewMemCacheClient(discoveryClient), nil
return c.discoveryClient, nil
}
// ToRESTMapper returns a RESTMapper constructed from ToDiscoveryClient.
// ToRESTMapper returns a meta.RESTMapper using the discovery client. Calling
// it multiple times will return the same instance.
func (c *MemoryRESTClientGetter) ToRESTMapper() (meta.RESTMapper, error) {
discoveryClient, err := c.ToDiscoveryClient()
if err != nil {
return nil, err
}
c.discoveryMu.Lock()
defer c.discoveryMu.Unlock()
mapper := restmapper.NewDeferredDiscoveryRESTMapper(discoveryClient)
expander := restmapper.NewShortcutExpander(mapper, discoveryClient)
return expander, nil
if c.restMapper == nil {
discoveryClient, err := c.ToDiscoveryClient()
if err != nil {
return nil, err
}
mapper := restmapper.NewDeferredDiscoveryRESTMapper(discoveryClient)
c.restMapper = restmapper.NewShortcutExpander(mapper, discoveryClient)
}
return c.restMapper, nil
}
// ToRawKubeConfigLoader returns a clientcmd.ClientConfig using
// clientcmd.DefaultClientConfig. With clientcmd.ClusterDefaults, namespace, and
// impersonate configured as overwrites.
func (c *MemoryRESTClientGetter) ToRawKubeConfigLoader() clientcmd.ClientConfig {
loadingRules := clientcmd.NewDefaultClientConfigLoadingRules()
// use the standard defaults for this client command
// DEPRECATED: remove and replace with something more accurate
loadingRules.DefaultClientConfig = &clientcmd.DefaultClientConfig
c.clientCfgMu.Lock()
defer c.clientCfgMu.Unlock()
overrides := &clientcmd.ConfigOverrides{ClusterDefaults: clientcmd.ClusterDefaults}
overrides.Context.Namespace = c.namespace
if c.clientCfg == nil {
loadingRules := clientcmd.NewDefaultClientConfigLoadingRules()
// use the standard defaults for this client command
// DEPRECATED: remove and replace with something more accurate
loadingRules.DefaultClientConfig = &clientcmd.DefaultClientConfig
if c.impersonateAccount != "" {
overrides.AuthInfo.Impersonate = c.impersonateAccount
overrides := &clientcmd.ConfigOverrides{ClusterDefaults: clientcmd.ClusterDefaults}
overrides.Context.Namespace = c.namespace
overrides.AuthInfo.Impersonate = c.impersonate
c.clientCfg = clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, overrides)
}
return clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, overrides)
return c.clientCfg
}

294
internal/kube/client_test.go
View File

@ -1,5 +1,5 @@
/*
Copyright 2022 The Flux authors
Copyright 2023 The Flux authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
@ -20,178 +20,220 @@ import (
"fmt"
"testing"
"github.com/fluxcd/pkg/runtime/client"
. "github.com/onsi/gomega"
"k8s.io/cli-runtime/pkg/genericclioptions"
"k8s.io/client-go/rest"
ctrl "sigs.k8s.io/controller-runtime"
"github.com/fluxcd/pkg/runtime/client"
)
var cfg = []byte(`current-context: federal-context
apiVersion: v1
clusters:
- cluster:
api-version: v1
server: http://cow.org:8080
insecure-skip-tls-verify: true
name: cow-cluster
contexts:
- context:
cluster: cow-cluster
user: blue-user
name: federal-context
kind: Config
users:
- name: blue-user
user:
token: foo`)
func TestNewInClusterRESTClientGetter(t *testing.T) {
t.Run("discover config", func(t *testing.T) {
func TestWithNamespace(t *testing.T) {
t.Run("sets the namespace", func(t *testing.T) {
g := NewWithT(t)
cfg := &rest.Config{
Host: "https://example.com",
BearerToken: "chase-the-honey",
TLSClientConfig: rest.TLSClientConfig{
CAFile: "afile",
c := &MemoryRESTClientGetter{}
WithNamespace("foo")(c)
g.Expect(c.namespace).To(Equal("foo"))
})
}
func TestWithImpersonate(t *testing.T) {
t.Run("sets the impersonate", func(t *testing.T) {
g := NewWithT(t)
c := &MemoryRESTClientGetter{
cfg: &rest.Config{
Host: "https://example.com",
},
}
ctrl.GetConfig = func() (*rest.Config, error) {
return cfg, nil
}
got, err := NewInClusterRESTClientGetter("", "", "", nil)
g.Expect(err).ToNot(HaveOccurred())
g.Expect(got).To(BeAssignableToTypeOf(&genericclioptions.ConfigFlags{}))
WithImpersonate("foo", "bar")(c)
g.Expect(c.impersonate).To(Equal(fmt.Sprintf(userNameFormat, "bar", "foo")))
g.Expect(c.cfg.Impersonate.UserName).To(Equal(c.impersonate))
})
}
flags := got.(*genericclioptions.ConfigFlags)
fields := map[*string]*string{
flags.APIServer: &cfg.Host,
flags.BearerToken: &cfg.BearerToken,
flags.CAFile: &cfg.CAFile,
}
for f, ff := range fields {
g.Expect(f).ToNot(BeNil())
g.Expect(f).To(Equal(ff))
g.Expect(f).ToNot(BeIdenticalTo(ff))
func TestWithClientOptions(t *testing.T) {
t.Run("sets the client options", func(t *testing.T) {
g := NewWithT(t)
c := &MemoryRESTClientGetter{
cfg: &rest.Config{
Host: "https://example.com",
},
}
WithClientOptions(client.Options{
Burst: 10,
QPS: 5,
})(c)
g.Expect(c.cfg.Burst).To(Equal(10))
g.Expect(c.cfg.QPS).To(Equal(float32(5)))
})
}
func TestNewMemoryRESTClientGetter(t *testing.T) {
t.Run("returns a new MemoryRESTClientGetter", func(t *testing.T) {
g := NewWithT(t)
c := NewMemoryRESTClientGetter(&rest.Config{
Host: "https://example.com",
})
g.Expect(c).ToNot(BeNil())
g.Expect(c.cfg).ToNot(BeNil())
g.Expect(c.cfg.Host).To(Equal("https://example.com"))
})
t.Run("config retrieval error", func(t *testing.T) {
t.Run("returns a new MemoryRESTClientGetter with default options", func(t *testing.T) {
g := NewWithT(t)
c := NewMemoryRESTClientGetter(&rest.Config{
Host: "https://example.com",
})
g.Expect(c).ToNot(BeNil())
g.Expect(c.cfg).ToNot(BeNil())
g.Expect(c.namespace).To(Equal("default"))
})
t.Run("returns a new MemoryRESTClientGetter with options", func(t *testing.T) {
g := NewWithT(t)
c := NewMemoryRESTClientGetter(&rest.Config{
Host: "https://example.com",
}, WithNamespace("foo"))
g.Expect(c).ToNot(BeNil())
g.Expect(c.cfg).ToNot(BeNil())
g.Expect(c.namespace).To(Equal("foo"))
})
}
func TestNewInClusterMemoryRESTClientGetter(t *testing.T) {
t.Cleanup(func() {
cfg := ctrl.GetConfig
ctrl.GetConfig = cfg
})
t.Run("discovers the in cluster config", func(t *testing.T) {
g := NewWithT(t)
mockCfg := &rest.Config{
Host: "https://example.com",
}
ctrl.GetConfig = func() (*rest.Config, error) {
return mockCfg, nil
}
c, err := NewInClusterMemoryRESTClientGetter()
g.Expect(c).ToNot(BeNil())
g.Expect(err).ToNot(HaveOccurred())
g.Expect(c.cfg).To(Equal(mockCfg))
})
t.Run("returns an error if the in cluster config cannot be discovered", func(t *testing.T) {
g := NewWithT(t)
ctrl.GetConfig = func() (*rest.Config, error) {
return nil, fmt.Errorf("error")
}
got, err := NewInClusterRESTClientGetter("", "", "", nil)
c, err := NewInClusterMemoryRESTClientGetter()
g.Expect(c).To(BeNil())
g.Expect(err).To(HaveOccurred())
g.Expect(err.Error()).To(ContainSubstring("failed to get config for in-cluster REST client"))
g.Expect(got).To(BeNil())
})
t.Run("namespace", func(t *testing.T) {
t.Run("configures the client with options", func(t *testing.T) {
g := NewWithT(t)
ctrl.GetConfig = mockGetConfig
namespace := "a-space"
got, err := NewInClusterRESTClientGetter(namespace, "", "", nil)
mockCfg := &rest.Config{
Host: "https://example.com",
}
ctrl.GetConfig = func() (*rest.Config, error) {
return mockCfg, nil
}
c, err := NewInClusterMemoryRESTClientGetter(WithNamespace("foo"))
g.Expect(c).ToNot(BeNil())
g.Expect(err).ToNot(HaveOccurred())
g.Expect(got).To(BeAssignableToTypeOf(&genericclioptions.ConfigFlags{}))
flags := got.(*genericclioptions.ConfigFlags)
g.Expect(flags.Namespace).ToNot(BeNil())
g.Expect(*flags.Namespace).To(Equal(namespace))
})
t.Run("impersonation", func(t *testing.T) {
g := NewWithT(t)
ctrl.GetConfig = mockGetConfig
ns := "a-namespace"
accountName := "foo"
accountNamespace := "another-namespace"
got, err := NewInClusterRESTClientGetter(ns, accountName, accountNamespace, nil)
g.Expect(err).ToNot(HaveOccurred())
g.Expect(got).To(BeAssignableToTypeOf(&genericclioptions.ConfigFlags{}))
flags := got.(*genericclioptions.ConfigFlags)
g.Expect(flags.Impersonate).ToNot(BeNil())
g.Expect(*flags.Impersonate).To(Equal(fmt.Sprintf("system:serviceaccount:%s:%s", accountNamespace, accountName)))
g.Expect(c.cfg).To(Equal(mockCfg))
g.Expect(c.namespace).To(Equal("foo"))
})
}
func TestMemoryRESTClientGetter_ToRESTConfig(t *testing.T) {
t.Run("loads REST config from KubeConfig", func(t *testing.T) {
t.Run("returns a REST config", func(t *testing.T) {
g := NewWithT(t)
getter := NewMemoryRESTClientGetter(cfg, "", "", "", client.Options{}, client.KubeConfigOptions{})
got, err := getter.ToRESTConfig()
c := &MemoryRESTClientGetter{
cfg: &rest.Config{
Host: "https://example.com",
},
}
cfg, err := c.ToRESTConfig()
g.Expect(err).ToNot(HaveOccurred())
g.Expect(got.Host).To(Equal("http://cow.org:8080"))
g.Expect(got.TLSClientConfig.Insecure).To(BeFalse())
g.Expect(cfg).To(BeIdenticalTo(c.cfg))
})
t.Run("sets ImpersonationConfig", func(t *testing.T) {
g := NewWithT(t)
getter := NewMemoryRESTClientGetter(cfg, "", "someone", "a-namespace", client.Options{}, client.KubeConfigOptions{})
got, err := getter.ToRESTConfig()
g.Expect(err).ToNot(HaveOccurred())
g.Expect(got.Impersonate.UserName).To(Equal("system:serviceaccount:a-namespace:someone"))
})
t.Run("uses KubeConfigOptions", func(t *testing.T) {
t.Run("error on nil REST config", func(t *testing.T) {
g := NewWithT(t)
agent := "a static string forever," +
"but static strings can have dreams and hope too"
getter := NewMemoryRESTClientGetter(cfg, "", "someone", "", client.Options{QPS: 400, Burst: 800}, client.KubeConfigOptions{
UserAgent: agent,
})
got, err := getter.ToRESTConfig()
g.Expect(err).ToNot(HaveOccurred())
g.Expect(got.UserAgent).To(Equal(agent))
})
t.Run("invalid config", func(t *testing.T) {
g := NewWithT(t)
getter := NewMemoryRESTClientGetter([]byte("invalid"), "", "", "", client.Options{QPS: 400, Burst: 800}, client.KubeConfigOptions{})
got, err := getter.ToRESTConfig()
c := &MemoryRESTClientGetter{}
cfg, err := c.ToRESTConfig()
g.Expect(err).To(HaveOccurred())
g.Expect(got).To(BeNil())
g.Expect(cfg).To(BeNil())
})
}
func TestMemoryRESTClientGetter_ToDiscoveryClient(t *testing.T) {
g := NewWithT(t)
t.Run("returns a discovery client", func(t *testing.T) {
g := NewWithT(t)
getter := NewMemoryRESTClientGetter(cfg, "", "", "", client.Options{QPS: 400, Burst: 800}, client.KubeConfigOptions{})
got, err := getter.ToDiscoveryClient()
g.Expect(err).ToNot(HaveOccurred())
g.Expect(got).ToNot(BeNil())
c := &MemoryRESTClientGetter{
cfg: &rest.Config{
Host: "https://example.com",
},
}
dc, err := c.ToDiscoveryClient()
g.Expect(err).ToNot(HaveOccurred())
g.Expect(dc).ToNot(BeNil())
// Calling it again should return the same instance.
dc2, err := c.ToDiscoveryClient()
g.Expect(err).ToNot(HaveOccurred())
g.Expect(dc2).To(BeIdenticalTo(dc))
})
}
func TestMemoryRESTClientGetter_ToRESTMapper(t *testing.T) {
g := NewWithT(t)
t.Run("returns a REST mapper", func(t *testing.T) {
g := NewWithT(t)
getter := NewMemoryRESTClientGetter(cfg, "", "", "", client.Options{QPS: 400, Burst: 800}, client.KubeConfigOptions{})
got, err := getter.ToRESTMapper()
g.Expect(err).ToNot(HaveOccurred())
g.Expect(got).ToNot(BeNil())
c := &MemoryRESTClientGetter{
cfg: &rest.Config{
Host: "https://example.com",
},
}
rm, err := c.ToRESTMapper()
g.Expect(err).ToNot(HaveOccurred())
g.Expect(rm).ToNot(BeNil())
// Calling it again should return the same instance.
rm2, err := c.ToRESTMapper()
g.Expect(err).ToNot(HaveOccurred())
g.Expect(rm2).To(BeIdenticalTo(rm))
})
}
func TestMemoryRESTClientGetter_ToRawKubeConfigLoader(t *testing.T) {
g := NewWithT(t)
t.Run("returns a client config", func(t *testing.T) {
g := NewWithT(t)
getter := NewMemoryRESTClientGetter(cfg, "a-namespace", "impersonate", "other-namespace", client.Options{QPS: 400, Burst: 800}, client.KubeConfigOptions{})
got := getter.ToRawKubeConfigLoader()
g.Expect(got).ToNot(BeNil())
c := &MemoryRESTClientGetter{
cfg: &rest.Config{
Host: "https://example.com",
},
}
cc := c.ToRawKubeConfigLoader()
g.Expect(cc).ToNot(BeNil())
cfg, err := got.ClientConfig()
g.Expect(err).ToNot(HaveOccurred())
g.Expect(cfg.Impersonate.UserName).To(Equal("impersonate"))
ns, _, err := got.Namespace()
g.Expect(err).ToNot(HaveOccurred())
g.Expect(ns).To(Equal("a-namespace"))
// Calling it again should return the same instance.
g.Expect(c.ToRawKubeConfigLoader()).To(BeIdenticalTo(cc))
})
}

66
internal/kube/config.go
View File

@ -20,32 +20,54 @@ import (
"fmt"
corev1 "k8s.io/api/core/v1"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
"github.com/fluxcd/pkg/runtime/client"
)
const (
// DefaultKubeConfigSecretKey is the default data key ConfigFromSecret
// looks at when no data key is provided.
DefaultKubeConfigSecretKey = "value"
// DefaultKubeConfigSecretKeyExt is the default data key ConfigFromSecret
// looks at when no data key is provided, and DefaultKubeConfigSecretKey
// does not exist.
DefaultKubeConfigSecretKeyExt = DefaultKubeConfigSecretKey + ".yaml"
)
// ConfigFromSecret returns the KubeConfig data from the provided key in the
// given Secret, or attempts to load the data from the default `value` and
// `value.yaml` keys. If a Secret is provided but no key with data can be
// found, an error is returned. The secret may be nil, in which case no bytes
// nor error are returned. Validation of the data is expected to happen while
// decoding the bytes.
func ConfigFromSecret(secret *corev1.Secret, key string) ([]byte, error) {
var kubeConfig []byte
if secret != nil {
secretName := fmt.Sprintf("%s/%s", secret.Namespace, secret.Name)
switch {
case key != "":
kubeConfig = secret.Data[key]
if kubeConfig == nil {
return nil, fmt.Errorf("KubeConfig secret '%s' does not contain a '%s' key with data", secretName, key)
}
case secret.Data[DefaultKubeConfigSecretKey] != nil:
kubeConfig = secret.Data[DefaultKubeConfigSecretKey]
case secret.Data[DefaultKubeConfigSecretKeyExt] != nil:
kubeConfig = secret.Data[DefaultKubeConfigSecretKeyExt]
default:
// User did not specify a key, and the 'value' key was not defined.
return nil, fmt.Errorf("KubeConfig secret '%s' does not contain a '%s' or '%s' key with data", secretName, DefaultKubeConfigSecretKey, DefaultKubeConfigSecretKeyExt)
}
// found, an error is returned.
func ConfigFromSecret(secret *corev1.Secret, key string, opts client.KubeConfigOptions) (*rest.Config, error) {
if secret == nil {
return nil, fmt.Errorf("KubeConfig secret is nil")
}
return kubeConfig, nil
var (
kubeConfig []byte
secretName = fmt.Sprintf("%s/%s", secret.Namespace, secret.Name)
)
switch {
case key != "":
kubeConfig = secret.Data[key]
if kubeConfig == nil {
return nil, fmt.Errorf("KubeConfig secret '%s' does not contain a '%s' key with data", secretName, key)
}
case secret.Data[DefaultKubeConfigSecretKey] != nil:
kubeConfig = secret.Data[DefaultKubeConfigSecretKey]
case secret.Data[DefaultKubeConfigSecretKeyExt] != nil:
kubeConfig = secret.Data[DefaultKubeConfigSecretKeyExt]
default:
// User did not specify a key, and the 'value' key was not defined.
return nil, fmt.Errorf("KubeConfig secret '%s' does not contain a '%s' or '%s' key with data", secretName, DefaultKubeConfigSecretKey, DefaultKubeConfigSecretKeyExt)
}
cfg, err := clientcmd.RESTConfigFromKubeConfig(kubeConfig)
if err != nil {
return nil, fmt.Errorf("failed to load KubeConfig from secret '%s': %w", secretName, err)
}
cfg = client.KubeConfig(cfg, opts)
return cfg, nil
}

93
internal/kube/config_test.go
View File

@ -17,6 +17,7 @@ limitations under the License.
package kube
import (
"github.com/fluxcd/pkg/runtime/client"
"testing"
. "github.com/onsi/gomega"
@ -24,6 +25,29 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
const (
kubeCfg = `apiVersion: v1
kind: Config
clusters:
- cluster:
insecure-skip-tls-verify: true
server: https://1.2.3.4
name: development
contexts:
- context:
cluster: development
namespace: frontend
user: developer
name: dev-frontend
current-context: dev-frontend
preferences: {}
users:
- name: developer
user:
password: some-password
username: exp`
)
func TestConfigFromSecret(t *testing.T) {
t.Run("with default key", func(t *testing.T) {
g := NewWithT(t)
@ -34,14 +58,14 @@ func TestConfigFromSecret(t *testing.T) {
Namespace: "vault",
},
Data: map[string][]byte{
DefaultKubeConfigSecretKey: []byte("good"),
DefaultKubeConfigSecretKey: []byte(kubeCfg),
// Also confirm priority.
DefaultKubeConfigSecretKeyExt: []byte("bad"),
},
}
got, err := ConfigFromSecret(secret, "")
got, err := ConfigFromSecret(secret, "", client.KubeConfigOptions{})
g.Expect(err).ToNot(HaveOccurred())
g.Expect(got).To(Equal(secret.Data[DefaultKubeConfigSecretKey]))
g.Expect(got).ToNot(BeNil())
})
t.Run("with default key with ext", func(t *testing.T) {
@ -53,12 +77,12 @@ func TestConfigFromSecret(t *testing.T) {
Namespace: "vault",
},
Data: map[string][]byte{
DefaultKubeConfigSecretKeyExt: []byte("good"),
DefaultKubeConfigSecretKeyExt: []byte(kubeCfg),
},
}
got, err := ConfigFromSecret(secret, "")
got, err := ConfigFromSecret(secret, "", client.KubeConfigOptions{})
g.Expect(err).ToNot(HaveOccurred())
g.Expect(got).To(Equal(secret.Data[DefaultKubeConfigSecretKeyExt]))
g.Expect(got).ToNot(BeNil())
})
t.Run("with key", func(t *testing.T) {
@ -71,12 +95,12 @@ func TestConfigFromSecret(t *testing.T) {
Namespace: "vault",
},
Data: map[string][]byte{
key: []byte("snow"),
key: []byte(kubeCfg),
},
}
got, err := ConfigFromSecret(secret, key)
got, err := ConfigFromSecret(secret, key, client.KubeConfigOptions{})
g.Expect(err).ToNot(HaveOccurred())
g.Expect(got).To(Equal(secret.Data[key]))
g.Expect(got).ToNot(BeNil())
})
t.Run("invalid key", func(t *testing.T) {
@ -90,11 +114,10 @@ func TestConfigFromSecret(t *testing.T) {
},
Data: map[string][]byte{},
}
got, err := ConfigFromSecret(secret, key)
got, err := ConfigFromSecret(secret, key, client.KubeConfigOptions{})
g.Expect(err).To(HaveOccurred())
g.Expect(got).To(BeNil())
g.Expect(err.Error()).To(ContainSubstring("secret 'vault/super-secret' does not contain a 'black-hole' key "))
g.Expect(got).To(Equal(secret.Data[key]))
})
t.Run("key without data", func(t *testing.T) {
@ -110,7 +133,7 @@ func TestConfigFromSecret(t *testing.T) {
key: nil,
},
}
got, err := ConfigFromSecret(secret, key)
got, err := ConfigFromSecret(secret, key, client.KubeConfigOptions{})
g.Expect(err).To(HaveOccurred())
g.Expect(got).To(BeNil())
g.Expect(err.Error()).To(ContainSubstring("does not contain a 'void' key with data"))
@ -127,7 +150,7 @@ func TestConfigFromSecret(t *testing.T) {
Data: map[string][]byte{},
}
got, err := ConfigFromSecret(secret, "")
got, err := ConfigFromSecret(secret, "", client.KubeConfigOptions{})
g.Expect(err).To(HaveOccurred())
g.Expect(got).To(BeNil())
g.Expect(err.Error()).To(ContainSubstring("does not contain a 'value' or 'value.yaml'"))
@ -136,8 +159,48 @@ func TestConfigFromSecret(t *testing.T) {
t.Run("nil secret", func(t *testing.T) {
g := NewWithT(t)
got, err := ConfigFromSecret(nil, "")
g.Expect(err).ToNot(HaveOccurred())
got, err := ConfigFromSecret(nil, "", client.KubeConfigOptions{})
g.Expect(err).To(HaveOccurred())
g.Expect(got).To(BeNil())
g.Expect(err.Error()).To(ContainSubstring("secret is nil"))
})
t.Run("invalid kubeconfig", func(t *testing.T) {
g := NewWithT(t)
secret := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "super-secret",
Namespace: "vault",
},
Data: map[string][]byte{
DefaultKubeConfigSecretKeyExt: []byte("bad"),
},
}
got, err := ConfigFromSecret(secret, "", client.KubeConfigOptions{})
g.Expect(err).To(HaveOccurred())
g.Expect(got).To(BeNil())
g.Expect(err.Error()).To(ContainSubstring("couldn't get version/kind"))
})
t.Run("with kubeconfig options", func(t *testing.T) {
g := NewWithT(t)
secret := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "super-secret",
Namespace: "vault",
},
Data: map[string][]byte{
DefaultKubeConfigSecretKey: []byte(kubeCfg),
},
}
got, err := ConfigFromSecret(secret, "", client.KubeConfigOptions{
UserAgent: "test",
})
g.Expect(err).ToNot(HaveOccurred())
g.Expect(got).ToNot(BeNil())
g.Expect(got.UserAgent).To(Equal("test"))
})
}