kubernetes
/
apiserver
mirror of https://github.com/kubernetes/apiserver.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #115149 from nilekhc/encrypt-all 

Allow encryption for all resources

Kubernetes-commit: 30ee6914c54269c5898582c984a3f21f9c6710e9
This commit is contained in:
Kubernetes Publisher 2023-03-08 16:55:59 -08:00
parent 2fa0308197 9bc62d2547
commit e97010c727
6 changed files with 1611 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
pkg/apis/config/types.go
View File

@ -26,11 +26,23 @@ import (
/*
EncryptionConfiguration stores the complete configuration for encryption providers.
example:
It also allows the use of wildcards to specify the resources that should be encrypted.
Use '*.<group>' to encrypt all resources within a group or '*.*' to encrypt all resources.
'*.' can be used to encrypt all resource in the core group. '*.*' will encrypt all
resources, even custom resources that are added after API server start.
Use of wildcards that overlap within the same resource list or across multiple
entries are not allowed since part of the configuration would be ineffective.
Resource lists are processed in order, with earlier lists taking precedence.
Example:
kind: EncryptionConfiguration
apiVersion: apiserver.config.k8s.io/v1
resources:
- resources:
- events
providers:
- identity: {} # do not encrypt events even though *.* is specified below
- resources:
- secrets
- configmaps
@ -40,6 +52,20 @@ example:
keys:
- name: key1
secret: c2VjcmV0IGlzIHNlY3VyZQ==
- resources:
- '*.apps'
providers:
- aescbc:
keys:
- name: key2
secret: c2VjcmV0IGlzIHNlY3VyZSwgb3IgaXMgaXQ/Cg==
- resources:
- '*.*'
providers:
- aescbc:
keys:
- name: key3
secret: c2VjcmV0IGlzIHNlY3VyZSwgSSB0aGluaw==
*/
type EncryptionConfiguration struct {
metav1.TypeMeta
@ -50,10 +76,13 @@ type EncryptionConfiguration struct {
// ResourceConfiguration stores per resource configuration.
type ResourceConfiguration struct {
// resources is a list of kubernetes resources which have to be encrypted. The resource names are derived from `resource` or `resource.group` of the group/version/resource.
// eg: pandas.awesome.bears.example is a custom resource with 'group': awesome.bears.example, 'resource': pandas)
// eg: pandas.awesome.bears.example is a custom resource with 'group': awesome.bears.example, 'resource': pandas.
// Use '*.*' to encrypt all resources and '*.<group>' to encrypt all resources in a specific group.
// eg: '*.awesome.bears.example' will encrypt all resources in the group 'awesome.bears.example'.
// eg: '*.' will encrypt all resources in the core group (such as pods, configmaps, etc).
Resources []string
// providers is a list of transformers to be used for reading and writing the resources to disk.
// eg: aesgcm, aescbc, secretbox, identity.
// eg: aesgcm, aescbc, secretbox, identity, kms.
Providers []ProviderConfiguration
}

35
pkg/apis/config/v1/types.go
View File

@ -26,11 +26,23 @@ import (
/*
EncryptionConfiguration stores the complete configuration for encryption providers.
example:
It also allows the use of wildcards to specify the resources that should be encrypted.
Use '*.<group>' to encrypt all resources within a group or '*.*' to encrypt all resources.
'*.' can be used to encrypt all resource in the core group. '*.*' will encrypt all
resources, even custom resources that are added after API server start.
Use of wildcards that overlap within the same resource list or across multiple
entries are not allowed since part of the configuration would be ineffective.
Resource lists are processed in order, with earlier lists taking precedence.
Example:
kind: EncryptionConfiguration
apiVersion: apiserver.config.k8s.io/v1
resources:
- resources:
- events
providers:
- identity: {} # do not encrypt events even though *.* is specified below
- resources:
- secrets
- configmaps
@ -40,6 +52,20 @@ example:
keys:
- name: key1
secret: c2VjcmV0IGlzIHNlY3VyZQ==
- resources:
- '*.apps'
providers:
- aescbc:
keys:
- name: key2
secret: c2VjcmV0IGlzIHNlY3VyZSwgb3IgaXMgaXQ/Cg==
- resources:
- '*.*'
providers:
- aescbc:
keys:
- name: key3
secret: c2VjcmV0IGlzIHNlY3VyZSwgSSB0aGluaw==
*/
type EncryptionConfiguration struct {
metav1.TypeMeta
@ -50,10 +76,13 @@ type EncryptionConfiguration struct {
// ResourceConfiguration stores per resource configuration.
type ResourceConfiguration struct {
// resources is a list of kubernetes resources which have to be encrypted. The resource names are derived from `resource` or `resource.group` of the group/version/resource.
// eg: pandas.awesome.bears.example is a custom resource with 'group': awesome.bears.example, 'resource': pandas)
// eg: pandas.awesome.bears.example is a custom resource with 'group': awesome.bears.example, 'resource': pandas.
// Use '*.*' to encrypt all resources and '*.<group>' to encrypt all resources in a specific group.
// eg: '*.awesome.bears.example' will encrypt all resources in the group 'awesome.bears.example'.
// eg: '*.' will encrypt all resources in the core group (such as pods, configmaps, etc).
Resources []string `json:"resources"`
// providers is a list of transformers to be used for reading and writing the resources to disk.
// eg: aesgcm, aescbc, secretbox, identity.
// eg: aesgcm, aescbc, secretbox, identity, kms.
Providers []ProviderConfiguration `json:"providers"`
}

191
pkg/apis/config/validation/validation.go
View File

@ -23,6 +23,7 @@ import (
"net/url"
"strings"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/validation/field"
"k8s.io/apiserver/pkg/apis/config"
@ -34,7 +35,7 @@ const (
unsupportedSchemeErrFmt = "unsupported scheme %q for KMS provider, only unix is supported"
unsupportedKMSAPIVersionErrFmt = "unsupported apiVersion %s for KMS provider, only v1 and v2 are supported"
atLeastOneRequiredErrFmt = "at least one %s is required"
invalidURLErrFmt = "invalid endpoint for kms provider, error: parse %s: net/url: invalid control character in URL"
invalidURLErrFmt = "invalid endpoint for kms provider, error: %v"
mandatoryFieldErrFmt = "%s is a mandatory field for a %s"
base64EncodingErr = "secrets must be base64 encoded"
zeroOrNegativeErrFmt = "%s should be a positive value"
@ -42,6 +43,14 @@ const (
encryptionConfigNilErr = "EncryptionConfiguration can't be nil"
invalidKMSConfigNameErrFmt = "invalid KMS provider name %s, must not contain ':'"
duplicateKMSConfigNameErrFmt = "duplicate KMS provider name %s, names must be unique"
eventsGroupErr = "'*.events.k8s.io' objects are stored using the 'events' API group in etcd. Use 'events' instead in the config file"
extensionsGroupErr = "'extensions' group has been removed and cannot be used for encryption"
starResourceErr = "use '*.' to encrypt all the resources from core API group or *.* to encrypt all resources"
overlapErr = "using overlapping resources such as 'secrets' and '*.' in the same resource list is not allowed as they will be masked"
nonRESTAPIResourceErr = "resources which do not have REST API/s cannot be encrypted"
resourceNameErr = "resource name should not contain capital letters"
resourceAcrossGroupErr = "encrypting the same resource across groups is not supported"
duplicateResourceErr = "the same resource cannot be specified multiple times"
)
var (
@ -59,7 +68,7 @@ func ValidateEncryptionConfiguration(c *config.EncryptionConfiguration, reload b
allErrs := field.ErrorList{}
if c == nil {
allErrs = append(allErrs, field.Required(root, "EncryptionConfiguration can't be nil"))
allErrs = append(allErrs, field.Required(root, encryptionConfigNilErr))
return allErrs
}
@ -78,6 +87,9 @@ func ValidateEncryptionConfiguration(c *config.EncryptionConfiguration, reload b
allErrs = append(allErrs, field.Required(r, fmt.Sprintf(atLeastOneRequiredErrFmt, r)))
}
allErrs = append(allErrs, validateResourceOverlap(conf.Resources, r)...)
allErrs = append(allErrs, validateResourceNames(conf.Resources, r)...)
if len(conf.Providers) == 0 {
allErrs = append(allErrs, field.Required(p, fmt.Sprintf(atLeastOneRequiredErrFmt, p)))
}
@ -103,6 +115,175 @@ func ValidateEncryptionConfiguration(c *config.EncryptionConfiguration, reload b
return allErrs
}
var anyGroupAnyResource = schema.GroupResource{
Group: "*",
Resource: "*",
}
func validateResourceOverlap(resources []string, fieldPath *field.Path) field.ErrorList {
if len(resources) < 2 { // cannot have overlap with a single resource
return nil
}
var allErrs field.ErrorList
r := make([]schema.GroupResource, 0, len(resources))
for _, resource := range resources {
r = append(r, schema.ParseGroupResource(resource))
}
var hasOverlap, hasDuplicate bool
for i, r1 := range r {
for j, r2 := range r {
if i == j {
continue
}
if r1 == r2 && !hasDuplicate {
hasDuplicate = true
continue
}
if hasOverlap {
continue
}
if r1 == anyGroupAnyResource {
hasOverlap = true
continue
}
if r1.Group != r2.Group {
continue
}
if r1.Resource == "*" || r2.Resource == "*" {
hasOverlap = true
continue
}
}
}
if hasDuplicate {
allErrs = append(
allErrs,
field.Invalid(
fieldPath,
resources,
duplicateResourceErr,
),
)
}
if hasOverlap {
allErrs = append(
allErrs,
field.Invalid(
fieldPath,
resources,
overlapErr,
),
)
}
return allErrs
}
func validateResourceNames(resources []string, fieldPath *field.Path) field.ErrorList {
var allErrs field.ErrorList
for j, res := range resources {
jj := fieldPath.Index(j)
// check if resource name has capital letters
if hasCapital(res) {
allErrs = append(
allErrs,
field.Invalid(
jj,
resources[j],
resourceNameErr,
),
)
continue
}
// check if resource is '*'
if res == "*" {
allErrs = append(
allErrs,
field.Invalid(
jj,
resources[j],
starResourceErr,
),
)
continue
}
// check if resource is:
// 'apiserveripinfo' OR
// 'serviceipallocations' OR
// 'servicenodeportallocations' OR
if res == "apiserveripinfo" ||
res == "serviceipallocations" ||
res == "servicenodeportallocations" {
allErrs = append(
allErrs,
field.Invalid(
jj,
resources[j],
nonRESTAPIResourceErr,
),
)
continue
}
// check if group is 'events.k8s.io'
gr := schema.ParseGroupResource(res)
if gr.Group == "events.k8s.io" {
allErrs = append(
allErrs,
field.Invalid(
jj,
resources[j],
eventsGroupErr,
),
)
continue
}
// check if group is 'extensions'
if gr.Group == "extensions" {
allErrs = append(
allErrs,
field.Invalid(
jj,
resources[j],
extensionsGroupErr,
),
)
continue
}
// disallow resource.* as encrypting the same resource across groups does not make sense
if gr.Group == "*" && gr.Resource != "*" {
allErrs = append(
allErrs,
field.Invalid(
jj,
resources[j],
resourceAcrossGroupErr,
),
)
continue
}
}
return allErrs
}
func validateSingleProvider(provider config.ProviderConfiguration, fieldPath *field.Path) field.ErrorList {
allErrs := field.ErrorList{}
found := 0
@ -225,7 +406,7 @@ func validateKMSEndpoint(c *config.KMSConfiguration, fieldPath *field.Path) fiel
u, err := url.Parse(c.Endpoint)
if err != nil {
return append(allErrs, field.Invalid(fieldPath, c.Endpoint, fmt.Sprintf("invalid endpoint for kms provider, error: %v", err)))
return append(allErrs, field.Invalid(fieldPath, c.Endpoint, fmt.Sprintf(invalidURLErrFmt, err)))
}
if u.Scheme != "unix" {
@ -265,3 +446,7 @@ func validateKMSConfigName(c *config.KMSConfiguration, fieldPath *field.Path, km
return allErrs
}
func hasCapital(input string) bool {
return strings.ToLower(input) != input
}

734
pkg/apis/config/validation/validation_test.go
View File

@ -22,6 +22,7 @@ import (
"time"
"github.com/google/go-cmp/cmp"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/validation/field"
@ -370,6 +371,737 @@ func TestStructure(t *testing.T) {
"foo", fmt.Sprintf(duplicateKMSConfigNameErrFmt, "foo")),
},
},
{
desc: "config should error when events.k8s.io group is used",
in: &config.EncryptionConfiguration{
Resources: []config.ResourceConfiguration{
{
Resources: []string{
"events.events.k8s.io",
},
Providers: []config.ProviderConfiguration{
{
KMS: &config.KMSConfiguration{
Name: "foo",
Endpoint: "unix:///tmp/kms-provider.socket",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
CacheSize: &cacheSize,
APIVersion: "v1",
},
},
},
},
},
},
reload: false,
want: field.ErrorList{
field.Invalid(
root.Index(0).Child("resources").Index(0),
"events.events.k8s.io",
eventsGroupErr,
),
},
}, {
desc: "config should error when events.k8s.io group is used later in the list",
in: &config.EncryptionConfiguration{
Resources: []config.ResourceConfiguration{
{
Resources: []string{
"secrets",
},
Providers: []config.ProviderConfiguration{
{
KMS: &config.KMSConfiguration{
Name: "foo",
Endpoint: "unix:///tmp/kms-provider.socket",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
CacheSize: &cacheSize,
APIVersion: "v1",
},
},
},
},
{
Resources: []string{
"secret",
"events.events.k8s.io",
},
Providers: []config.ProviderConfiguration{
{
KMS: &config.KMSConfiguration{
Name: "foo",
Endpoint: "unix:///tmp/kms-provider.socket",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
CacheSize: &cacheSize,
APIVersion: "v1",
},
},
},
},
},
},
reload: false,
want: field.ErrorList{
field.Invalid(
root.Index(1).Child("resources").Index(1),
"events.events.k8s.io",
eventsGroupErr,
),
},
},
{
desc: "config should error when *.events.k8s.io group is used",
in: &config.EncryptionConfiguration{
Resources: []config.ResourceConfiguration{
{
Resources: []string{
"*.events.k8s.io",
},
Providers: []config.ProviderConfiguration{
{
KMS: &config.KMSConfiguration{
Name: "foo",
Endpoint: "unix:///tmp/kms-provider.socket",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
CacheSize: &cacheSize,
APIVersion: "v1",
},
},
},
},
},
},
reload: false,
want: field.ErrorList{
field.Invalid(
root.Index(0).Child("resources").Index(0),
"*.events.k8s.io",
eventsGroupErr,
),
},
},
{
desc: "config should error when extensions group is used",
in: &config.EncryptionConfiguration{
Resources: []config.ResourceConfiguration{
{
Resources: []string{
"*.extensions",
},
Providers: []config.ProviderConfiguration{
{
KMS: &config.KMSConfiguration{
Name: "foo",
Endpoint: "unix:///tmp/kms-provider.socket",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
CacheSize: &cacheSize,
APIVersion: "v1",
},
},
},
},
},
},
reload: false,
want: field.ErrorList{
field.Invalid(
root.Index(0).Child("resources").Index(0),
"*.extensions",
extensionsGroupErr,
),
},
},
{
desc: "config should error when foo.extensions group is used",
in: &config.EncryptionConfiguration{
Resources: []config.ResourceConfiguration{
{
Resources: []string{
"foo.extensions",
},
Providers: []config.ProviderConfiguration{
{
KMS: &config.KMSConfiguration{
Name: "foo",
Endpoint: "unix:///tmp/kms-provider.socket",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
CacheSize: &cacheSize,
APIVersion: "v1",
},
},
},
},
},
},
reload: false,
want: field.ErrorList{
field.Invalid(
root.Index(0).Child("resources").Index(0),
"foo.extensions",
extensionsGroupErr,
),
},
},
{
desc: "config should error when '*' resource is used",
in: &config.EncryptionConfiguration{
Resources: []config.ResourceConfiguration{
{
Resources: []string{
"*",
},
Providers: []config.ProviderConfiguration{
{
KMS: &config.KMSConfiguration{
Name: "foo",
Endpoint: "unix:///tmp/kms-provider.socket",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
CacheSize: &cacheSize,
APIVersion: "v1",
},
},
},
},
},
},
reload: false,
want: field.ErrorList{
field.Invalid(
root.Index(0).Child("resources").Index(0),
"*",
starResourceErr,
),
},
},
{
desc: "should error when resource name has capital letters",
in: &config.EncryptionConfiguration{
Resources: []config.ResourceConfiguration{
{
Resources: []string{
"apiServerIPInfo",
},
Providers: []config.ProviderConfiguration{
{
KMS: &config.KMSConfiguration{
Name: "foo",
Endpoint: "unix:///tmp/kms-provider.socket",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
CacheSize: &cacheSize,
APIVersion: "v1",
},
},
},
},
},
},
reload: false,
want: field.ErrorList{
field.Invalid(
root.Index(0).Child("resources").Index(0),
"apiServerIPInfo",
resourceNameErr,
),
},
},
{
desc: "should error when resource name is apiserveripinfo",
in: &config.EncryptionConfiguration{
Resources: []config.ResourceConfiguration{
{
Resources: []string{
"apiserveripinfo",
},
Providers: []config.ProviderConfiguration{
{
KMS: &config.KMSConfiguration{
Name: "foo",
Endpoint: "unix:///tmp/kms-provider.socket",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
CacheSize: &cacheSize,
APIVersion: "v1",
},
},
},
},
},
},
reload: false,
want: field.ErrorList{
field.Invalid(
root.Index(0).Child("resources").Index(0),
"apiserveripinfo",
nonRESTAPIResourceErr,
),
},
},
{
desc: "should error when resource name is serviceipallocations",
in: &config.EncryptionConfiguration{
Resources: []config.ResourceConfiguration{
{
Resources: []string{
"serviceipallocations",
},
Providers: []config.ProviderConfiguration{
{
KMS: &config.KMSConfiguration{
Name: "foo",
Endpoint: "unix:///tmp/kms-provider.socket",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
CacheSize: &cacheSize,
APIVersion: "v1",
},
},
},
},
},
},
reload: false,
want: field.ErrorList{
field.Invalid(
root.Index(0).Child("resources").Index(0),
"serviceipallocations",
nonRESTAPIResourceErr,
),
},
},
{
desc: "should error when resource name is servicenodeportallocations",
in: &config.EncryptionConfiguration{
Resources: []config.ResourceConfiguration{
{
Resources: []string{
"servicenodeportallocations",
},
Providers: []config.ProviderConfiguration{
{
KMS: &config.KMSConfiguration{
Name: "foo",
Endpoint: "unix:///tmp/kms-provider.socket",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
CacheSize: &cacheSize,
APIVersion: "v1",
},
},
},
},
},
},
reload: false,
want: field.ErrorList{
field.Invalid(
root.Index(0).Child("resources").Index(0),
"servicenodeportallocations",
nonRESTAPIResourceErr,
),
},
},
{
desc: "should not error when '*.apps' and '*.' are used within the same resource list",
in: &config.EncryptionConfiguration{
Resources: []config.ResourceConfiguration{
{
Resources: []string{
"*.apps",
"*.",
},
Providers: []config.ProviderConfiguration{
{
KMS: &config.KMSConfiguration{
Name: "foo",
Endpoint: "unix:///tmp/kms-provider.socket",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
CacheSize: &cacheSize,
APIVersion: "v1",
},
},
},
},
},
},
reload: false,
want: field.ErrorList{},
},
{
desc: "should error when the same resource across groups is encrypted",
in: &config.EncryptionConfiguration{
Resources: []config.ResourceConfiguration{
{
Resources: []string{
"*.",
"foos.*",
},
Providers: []config.ProviderConfiguration{
{
KMS: &config.KMSConfiguration{
Name: "foo",
Endpoint: "unix:///tmp/kms-provider.socket",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
CacheSize: &cacheSize,
APIVersion: "v1",
},
},
},
},
},
},
reload: false,
want: field.ErrorList{
field.Invalid(
root.Index(0).Child("resources").Index(1),
"foos.*",
resourceAcrossGroupErr,
),
},
},
{
desc: "should error when secrets are specified twice within the same resource list",
in: &config.EncryptionConfiguration{
Resources: []config.ResourceConfiguration{
{
Resources: []string{
"secrets",
"secrets",
},
Providers: []config.ProviderConfiguration{
{
KMS: &config.KMSConfiguration{
Name: "foo",
Endpoint: "unix:///tmp/kms-provider.socket",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
CacheSize: &cacheSize,
APIVersion: "v1",
},
},
},
},
},
},
reload: false,
want: field.ErrorList{
field.Invalid(
root.Index(0).Child("resources"),
[]string{
"secrets",
"secrets",
},
duplicateResourceErr,
),
},
},
{
desc: "should error once when secrets are specified many times within the same resource list",
in: &config.EncryptionConfiguration{
Resources: []config.ResourceConfiguration{
{
Resources: []string{
"secrets",
"secrets",
"secrets",
"secrets",
},
Providers: []config.ProviderConfiguration{
{
KMS: &config.KMSConfiguration{
Name: "foo",
Endpoint: "unix:///tmp/kms-provider.socket",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
CacheSize: &cacheSize,
APIVersion: "v1",
},
},
},
},
},
},
reload: false,
want: field.ErrorList{
field.Invalid(
root.Index(0).Child("resources"),
[]string{
"secrets",
"secrets",
"secrets",
"secrets",
},
duplicateResourceErr,
),
},
},
{
desc: "should error when secrets are specified twice within the same resource list, via dot",
in: &config.EncryptionConfiguration{
Resources: []config.ResourceConfiguration{
{
Resources: []string{
"secrets",
"secrets.",
},
Providers: []config.ProviderConfiguration{
{
KMS: &config.KMSConfiguration{
Name: "foo",
Endpoint: "unix:///tmp/kms-provider.socket",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
CacheSize: &cacheSize,
APIVersion: "v1",
},
},
},
},
},
},
reload: false,
want: field.ErrorList{
field.Invalid(
root.Index(0).Child("resources"),
[]string{
"secrets",
"secrets.",
},
duplicateResourceErr,
),
},
},
{
desc: "should error when '*.apps' and '*.' and '*.*' are used within the same resource list",
in: &config.EncryptionConfiguration{
Resources: []config.ResourceConfiguration{
{
Resources: []string{
"*.apps",
"*.",
"*.*",
},
Providers: []config.ProviderConfiguration{
{
KMS: &config.KMSConfiguration{
Name: "foo",
Endpoint: "unix:///tmp/kms-provider.socket",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
CacheSize: &cacheSize,
APIVersion: "v1",
},
},
},
},
},
},
reload: false,
want: field.ErrorList{
field.Invalid(
root.Index(0).Child("resources"),
[]string{
"*.apps",
"*.",
"*.*",
},
overlapErr,
),
},
},
{
desc: "should not error when deployments.apps are specified with '*.' within the same resource list",
in: &config.EncryptionConfiguration{
Resources: []config.ResourceConfiguration{
{
Resources: []string{
"deployments.apps",
"*.",
},
Providers: []config.ProviderConfiguration{
{
KMS: &config.KMSConfiguration{
Name: "foo",
Endpoint: "unix:///tmp/kms-provider.socket",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
CacheSize: &cacheSize,
APIVersion: "v1",
},
},
},
},
},
},
reload: false,
want: field.ErrorList{},
},
{
desc: "should error when deployments.apps are specified with '*.apps' within the same resource list",
in: &config.EncryptionConfiguration{
Resources: []config.ResourceConfiguration{
{
Resources: []string{
"deployments.apps",
"*.apps",
},
Providers: []config.ProviderConfiguration{
{
KMS: &config.KMSConfiguration{
Name: "foo",
Endpoint: "unix:///tmp/kms-provider.socket",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
CacheSize: &cacheSize,
APIVersion: "v1",
},
},
},
},
},
},
reload: false,
want: field.ErrorList{
field.Invalid(
root.Index(0).Child("resources"),
[]string{
"deployments.apps",
"*.apps",
},
overlapErr,
),
},
},
{
desc: "should error when secrets are specified with '*.' within the same resource list",
in: &config.EncryptionConfiguration{
Resources: []config.ResourceConfiguration{
{
Resources: []string{
"secrets",
"*.",
},
Providers: []config.ProviderConfiguration{
{
KMS: &config.KMSConfiguration{
Name: "foo",
Endpoint: "unix:///tmp/kms-provider.socket",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
CacheSize: &cacheSize,
APIVersion: "v1",
},
},
},
},
},
},
reload: false,
want: field.ErrorList{
field.Invalid(
root.Index(0).Child("resources"),
[]string{
"secrets",
"*.",
},
overlapErr,
),
},
},
{
desc: "should error when pods are specified with '*.' within the same resource list",
in: &config.EncryptionConfiguration{
Resources: []config.ResourceConfiguration{
{
Resources: []string{
"pods",
"*.",
},
Providers: []config.ProviderConfiguration{
{
KMS: &config.KMSConfiguration{
Name: "foo",
Endpoint: "unix:///tmp/kms-provider.socket",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
CacheSize: &cacheSize,
APIVersion: "v1",
},
},
},
},
},
},
reload: false,
want: field.ErrorList{
field.Invalid(
root.Index(0).Child("resources"),
[]string{
"pods",
"*.",
},
overlapErr,
),
},
},
{
desc: "should error when other resources are specified with '*.*' within the same resource list",
in: &config.EncryptionConfiguration{
Resources: []config.ResourceConfiguration{
{
Resources: []string{
"secrets",
"*.*",
},
Providers: []config.ProviderConfiguration{
{
KMS: &config.KMSConfiguration{
Name: "foo",
Endpoint: "unix:///tmp/kms-provider.socket",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
CacheSize: &cacheSize,
APIVersion: "v1",
},
},
},
},
},
},
reload: false,
want: field.ErrorList{
field.Invalid(
root.Index(0).Child("resources"),
[]string{
"secrets",
"*.*",
},
overlapErr,
),
},
},
{
desc: "should error when both '*.' and '*.*' are used within the same resource list",
in: &config.EncryptionConfiguration{
Resources: []config.ResourceConfiguration{
{
Resources: []string{
"*.",
"*.*",
},
Providers: []config.ProviderConfiguration{
{
KMS: &config.KMSConfiguration{
Name: "foo",
Endpoint: "unix:///tmp/kms-provider.socket",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
CacheSize: &cacheSize,
APIVersion: "v1",
},
},
},
},
},
},
reload: false,
want: field.ErrorList{
field.Invalid(
root.Index(0).Child("resources"),
[]string{
"*.",
"*.*",
},
overlapErr,
),
},
},
}
for _, tt := range testCases {
@ -505,7 +1237,7 @@ func TestKMSEndpoint(t *testing.T) {
desc: "invalid url",
in: &config.KMSConfiguration{Endpoint: "unix:///foo\n.socket"},
want: field.ErrorList{
field.Invalid(endpointField, "unix:///foo\n.socket", fmt.Sprintf(invalidURLErrFmt, `"unix:///foo\n.socket"`)),
field.Invalid(endpointField, "unix:///foo\n.socket", fmt.Sprintf(invalidURLErrFmt, `parse "unix:///foo\n.socket": net/url: invalid control character in URL`)),
},
},
}

54
pkg/server/options/encryptionconfig/config.go
View File

@ -218,8 +218,22 @@ func getTransformerOverridesAndKMSPluginProbes(ctx context.Context, config *apis
for _, resource := range resourceConfig.Resources {
resource := resource
gr := schema.ParseGroupResource(resource)
resourceToPrefixTransformer[gr] = append(
resourceToPrefixTransformer[gr], transformers...)
// check if resource is masked by *.group rule
anyResourceInGroup := schema.GroupResource{Group: gr.Group, Resource: "*"}
if _, masked := resourceToPrefixTransformer[anyResourceInGroup]; masked {
// an earlier rule already configured a transformer for *.group, masking this rule
// return error since this is not allowed
return nil, nil, nil, fmt.Errorf("resource %q is masked by earlier rule %q", grYAMLString(gr), grYAMLString(anyResourceInGroup))
}
if _, masked := resourceToPrefixTransformer[anyGroupAnyResource]; masked {
// an earlier rule already configured a transformer for *.*, masking this rule
// return error since this is not allowed
return nil, nil, nil, fmt.Errorf("resource %q is masked by earlier rule %q", grYAMLString(gr), grYAMLString(anyGroupAnyResource))
}
resourceToPrefixTransformer[gr] = append(resourceToPrefixTransformer[gr], transformers...)
}
probes = append(probes, p...)
@ -777,10 +791,34 @@ func (s StaticTransformers) TransformerForResource(resource schema.GroupResource
return transformerFromOverrides(s, resource)
}
func transformerFromOverrides(transformerOverrides map[schema.GroupResource]value.Transformer, resource schema.GroupResource) value.Transformer {
transformer := transformerOverrides[resource]
if transformer == nil {
return identity.NewEncryptCheckTransformer()
}
return transformer
var anyGroupAnyResource = schema.GroupResource{
Group: "*",
Resource: "*",
}
func transformerFromOverrides(transformerOverrides map[schema.GroupResource]value.Transformer, resource schema.GroupResource) value.Transformer {
if transformer := transformerOverrides[resource]; transformer != nil {
return transformer
}
if transformer := transformerOverrides[schema.GroupResource{
Group: resource.Group,
Resource: "*",
}]; transformer != nil {
return transformer
}
if transformer := transformerOverrides[anyGroupAnyResource]; transformer != nil {
return transformer
}
return identity.NewEncryptCheckTransformer()
}
func grYAMLString(gr schema.GroupResource) string {
if gr.Group == "" && gr.Resource == "*" {
return "*."
}
return gr.String()
}

580
pkg/server/options/encryptionconfig/config_test.go
View File

@ -21,6 +21,7 @@ import (
"context"
"encoding/base64"
"errors"
"reflect"
"strings"
"sync"
"testing"
@ -41,6 +42,7 @@ import (
"k8s.io/component-base/metrics/legacyregistry"
"k8s.io/component-base/metrics/testutil"
kmsservice "k8s.io/kms/pkg/service"
"k8s.io/utils/pointer"
)
const (
@ -705,6 +707,584 @@ func TestKMSPluginHealthz(t *testing.T) {
}
}
// tests for masking rules
func TestWildcardMasking(t *testing.T) {
testCases := []struct {
desc string
config *apiserverconfig.EncryptionConfiguration
expectedError string
}{
{
desc: "resources masked by *. group",
config: &apiserverconfig.EncryptionConfiguration{
Resources: []apiserverconfig.ResourceConfiguration{
{
Resources: []string{
"configmaps",
"*.",
"secrets",
},
Providers: []apiserverconfig.ProviderConfiguration{
{
KMS: &apiserverconfig.KMSConfiguration{
Name: "kms",
APIVersion: "v1",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
Endpoint: "unix:///tmp/testprovider.sock",
CacheSize: pointer.Int32(10),
},
},
},
},
},
},
expectedError: "resource \"secrets\" is masked by earlier rule \"*.\"",
},
{
desc: "*. masked by *. group",
config: &apiserverconfig.EncryptionConfiguration{
Resources: []apiserverconfig.ResourceConfiguration{
{
Resources: []string{
"*.",
},
Providers: []apiserverconfig.ProviderConfiguration{
{
KMS: &apiserverconfig.KMSConfiguration{
Name: "kms",
APIVersion: "v1",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
Endpoint: "unix:///tmp/testprovider.sock",
CacheSize: pointer.Int32(10),
},
},
},
},
{
Resources: []string{
"*.",
},
Providers: []apiserverconfig.ProviderConfiguration{
{
KMS: &apiserverconfig.KMSConfiguration{
Name: "kms2",
APIVersion: "v1",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
Endpoint: "unix:///tmp/testprovider.sock",
CacheSize: pointer.Int32(10),
},
},
},
},
},
},
expectedError: "resource \"*.\" is masked by earlier rule \"*.\"",
},
{
desc: "*.foo masked by *.foo",
config: &apiserverconfig.EncryptionConfiguration{
Resources: []apiserverconfig.ResourceConfiguration{
{
Resources: []string{
"*.foo",
},
Providers: []apiserverconfig.ProviderConfiguration{
{
KMS: &apiserverconfig.KMSConfiguration{
Name: "kms",
APIVersion: "v1",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
Endpoint: "unix:///tmp/testprovider.sock",
CacheSize: pointer.Int32(10),
},
},
},
},
{
Resources: []string{
"*.foo",
},
Providers: []apiserverconfig.ProviderConfiguration{
{
KMS: &apiserverconfig.KMSConfiguration{
Name: "kms2",
APIVersion: "v1",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
Endpoint: "unix:///tmp/testprovider.sock",
CacheSize: pointer.Int32(10),
},
},
},
},
},
},
expectedError: "resource \"*.foo\" is masked by earlier rule \"*.foo\"",
},
{
desc: "*.* masked by *.*",
config: &apiserverconfig.EncryptionConfiguration{
Resources: []apiserverconfig.ResourceConfiguration{
{
Resources: []string{
"*.*",
},
Providers: []apiserverconfig.ProviderConfiguration{
{
KMS: &apiserverconfig.KMSConfiguration{
Name: "kms",
APIVersion: "v1",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
Endpoint: "unix:///tmp/testprovider.sock",
CacheSize: pointer.Int32(10),
},
},
},
},
{
Resources: []string{
"*.*",
},
Providers: []apiserverconfig.ProviderConfiguration{
{
KMS: &apiserverconfig.KMSConfiguration{
Name: "kms2",
APIVersion: "v1",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
Endpoint: "unix:///tmp/testprovider.sock",
CacheSize: pointer.Int32(10),
},
},
},
},
},
},
expectedError: "resource \"*.*\" is masked by earlier rule \"*.*\"",
},
{
desc: "resources masked by *. group in multiple configurations",
config: &apiserverconfig.EncryptionConfiguration{
Resources: []apiserverconfig.ResourceConfiguration{
{
Resources: []string{
"configmaps",
},
Providers: []apiserverconfig.ProviderConfiguration{
{
KMS: &apiserverconfig.KMSConfiguration{
Name: "kms",
APIVersion: "v1",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
Endpoint: "unix:///tmp/testprovider.sock",
CacheSize: pointer.Int32(10),
},
},
},
},
{
Resources: []string{
"*.",
"secrets",
},
Providers: []apiserverconfig.ProviderConfiguration{
{
KMS: &apiserverconfig.KMSConfiguration{
Name: "another-kms",
APIVersion: "v1",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
Endpoint: "unix:///tmp/another-testprovider.sock",
CacheSize: pointer.Int32(10),
},
},
},
},
},
},
expectedError: "resource \"secrets\" is masked by earlier rule \"*.\"",
},
{
desc: "resources masked by *.*",
config: &apiserverconfig.EncryptionConfiguration{
Resources: []apiserverconfig.ResourceConfiguration{
{
Resources: []string{
"configmaps",
"*.*",
"secrets",
},
Providers: []apiserverconfig.ProviderConfiguration{
{
KMS: &apiserverconfig.KMSConfiguration{
Name: "kms",
APIVersion: "v1",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
Endpoint: "unix:///tmp/testprovider.sock",
CacheSize: pointer.Int32(10),
},
},
},
},
},
},
expectedError: "resource \"secrets\" is masked by earlier rule \"*.*\"",
},
{
desc: "resources masked by *.* in multiple configurations",
config: &apiserverconfig.EncryptionConfiguration{
Resources: []apiserverconfig.ResourceConfiguration{
{
Resources: []string{
"configmaps",
},
Providers: []apiserverconfig.ProviderConfiguration{
{
KMS: &apiserverconfig.KMSConfiguration{
Name: "kms",
APIVersion: "v1",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
Endpoint: "unix:///tmp/testprovider.sock",
CacheSize: pointer.Int32(10),
},
},
},
},
{
Resources: []string{
"*.*",
"secrets",
},
Providers: []apiserverconfig.ProviderConfiguration{
{
KMS: &apiserverconfig.KMSConfiguration{
Name: "another-kms",
APIVersion: "v1",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
Endpoint: "unix:///tmp/another-testprovider.sock",
CacheSize: pointer.Int32(10),
},
},
},
},
},
},
expectedError: "resource \"secrets\" is masked by earlier rule \"*.*\"",
},
{
desc: "resources *. masked by *.*",
config: &apiserverconfig.EncryptionConfiguration{
Resources: []apiserverconfig.ResourceConfiguration{
{
Resources: []string{
"configmaps",
"*.*",
"*.",
},
Providers: []apiserverconfig.ProviderConfiguration{
{
KMS: &apiserverconfig.KMSConfiguration{
Name: "kms",
APIVersion: "v1",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
Endpoint: "unix:///tmp/testprovider.sock",
CacheSize: pointer.Int32(10),
},
},
},
},
},
},
expectedError: "resource \"*.\" is masked by earlier rule \"*.*\"",
},
{
desc: "resources *. masked by *.* in multiple configurations",
config: &apiserverconfig.EncryptionConfiguration{
Resources: []apiserverconfig.ResourceConfiguration{
{
Resources: []string{
"configmaps",
"*.*",
},
Providers: []apiserverconfig.ProviderConfiguration{
{
KMS: &apiserverconfig.KMSConfiguration{
Name: "kms",
APIVersion: "v1",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
Endpoint: "unix:///tmp/testprovider.sock",
CacheSize: pointer.Int32(10),
},
},
},
},
{
Resources: []string{
"*.",
},
Providers: []apiserverconfig.ProviderConfiguration{
{
KMS: &apiserverconfig.KMSConfiguration{
Name: "another-kms",
APIVersion: "v1",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
Endpoint: "unix:///tmp/another-testprovider.sock",
CacheSize: pointer.Int32(10),
},
},
},
},
},
},
expectedError: "resource \"*.\" is masked by earlier rule \"*.*\"",
},
{
desc: "resources not masked by any rule",
config: &apiserverconfig.EncryptionConfiguration{
Resources: []apiserverconfig.ResourceConfiguration{
{
Resources: []string{
"configmaps",
"secrets",
"*.*",
},
Providers: []apiserverconfig.ProviderConfiguration{
{
KMS: &apiserverconfig.KMSConfiguration{
Name: "kms",
APIVersion: "v1",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
Endpoint: "unix:///tmp/testprovider.sock",
CacheSize: pointer.Int32(10),
},
},
},
},
},
},
},
{
desc: "resources not masked by any rule in multiple configurations",
config: &apiserverconfig.EncryptionConfiguration{
Resources: []apiserverconfig.ResourceConfiguration{
{
Resources: []string{
"configmaps",
"secrets",
},
Providers: []apiserverconfig.ProviderConfiguration{
{
KMS: &apiserverconfig.KMSConfiguration{
Name: "kms",
APIVersion: "v1",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
Endpoint: "unix:///tmp/testprovider.sock",
CacheSize: pointer.Int32(10),
},
},
},
},
{
Resources: []string{
"*.*",
},
Providers: []apiserverconfig.ProviderConfiguration{
{
KMS: &apiserverconfig.KMSConfiguration{
Name: "another-kms",
APIVersion: "v1",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
Endpoint: "unix:///tmp/another-testprovider.sock",
CacheSize: pointer.Int32(10),
},
},
},
},
},
},
},
}
for _, tc := range testCases {
t.Run(tc.desc, func(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
t.Cleanup(cancel)
_, _, _, err := getTransformerOverridesAndKMSPluginProbes(ctx, tc.config)
if errString(err) != tc.expectedError {
t.Errorf("expected error %s but got %s", tc.expectedError, errString(err))
}
})
}
}
func TestWildcardStructure(t *testing.T) {
testCases := []struct {
desc string
expectedResourceTransformers map[string]string
config *apiserverconfig.EncryptionConfiguration
errorValue string
}{
{
desc: "should not result in error",
expectedResourceTransformers: map[string]string{
"configmaps": "k8s:enc:kms:v1:kms:",
"secrets": "k8s:enc:kms:v1:another-kms:",
"events": "k8s:enc:kms:v1:fancy:",
"deployments.apps": "k8s:enc:kms:v1:kms:",
"pods": "k8s:enc:kms:v1:fancy:",
"pandas": "k8s:enc:kms:v1:fancy:",
"pandas.bears": "k8s:enc:kms:v1:yet-another-provider:",
"jobs.apps": "k8s:enc:kms:v1:kms:",
},
errorValue: "",
config: &apiserverconfig.EncryptionConfiguration{
Resources: []apiserverconfig.ResourceConfiguration{
{
Resources: []string{
"configmaps",
"*.apps",
},
Providers: []apiserverconfig.ProviderConfiguration{
{
KMS: &apiserverconfig.KMSConfiguration{
Name: "kms",
APIVersion: "v1",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
Endpoint: "unix:///tmp/testprovider.sock",
CacheSize: pointer.Int32(10),
},
},
},
},
{
Resources: []string{
"secrets",
},
Providers: []apiserverconfig.ProviderConfiguration{
{
KMS: &apiserverconfig.KMSConfiguration{
Name: "another-kms",
APIVersion: "v1",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
Endpoint: "unix:///tmp/testprovider.sock",
CacheSize: pointer.Int32(10),
},
},
{
Identity: &apiserverconfig.IdentityConfiguration{},
},
},
},
{
Resources: []string{
"*.",
},
Providers: []apiserverconfig.ProviderConfiguration{
{
KMS: &apiserverconfig.KMSConfiguration{
Name: "fancy",
APIVersion: "v1",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
Endpoint: "unix:///tmp/testprovider.sock",
CacheSize: pointer.Int32(10),
},
},
},
},
{
Resources: []string{
"*.*",
},
Providers: []apiserverconfig.ProviderConfiguration{
{
KMS: &apiserverconfig.KMSConfiguration{
Name: "yet-another-provider",
APIVersion: "v1",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
Endpoint: "unix:///tmp/testprovider.sock",
CacheSize: pointer.Int32(10),
},
},
},
},
},
},
},
{
desc: "should result in error",
errorValue: "resource \"secrets\" is masked by earlier rule \"*.\"",
config: &apiserverconfig.EncryptionConfiguration{
Resources: []apiserverconfig.ResourceConfiguration{
{
Resources: []string{
"configmaps",
"*.",
},
Providers: []apiserverconfig.ProviderConfiguration{
{
KMS: &apiserverconfig.KMSConfiguration{
Name: "kms",
APIVersion: "v1",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
Endpoint: "unix:///tmp/testprovider.sock",
CacheSize: pointer.Int32(10),
},
},
},
},
{
Resources: []string{
"*.*",
"secrets",
},
Providers: []apiserverconfig.ProviderConfiguration{
{
KMS: &apiserverconfig.KMSConfiguration{
Name: "kms",
APIVersion: "v1",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
Endpoint: "unix:///tmp/testprovider.sock",
CacheSize: pointer.Int32(10),
},
},
{
Identity: &apiserverconfig.IdentityConfiguration{},
},
},
},
},
},
},
}
for _, tc := range testCases {
t.Run(tc.desc, func(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
t.Cleanup(cancel)
transformers, _, _, err := getTransformerOverridesAndKMSPluginProbes(ctx, tc.config)
if errString(err) != tc.errorValue {
t.Errorf("expected error %s but got %s", tc.errorValue, errString(err))
}
if len(tc.errorValue) > 0 {
return
}
// check if expectedResourceTransformers are present
for resource, expectedTransformerName := range tc.expectedResourceTransformers {
transformer := transformerFromOverrides(transformers, schema.ParseGroupResource(resource))
transformerName := string(
reflect.ValueOf(transformer).Elem().FieldByName("transformers").Index(0).FieldByName("Prefix").Bytes(),
)
if transformerName != expectedTransformerName {
t.Errorf("resource %s: expected same transformer name but got %v", resource, cmp.Diff(transformerName, expectedTransformerName))
}
}
})
}
}
func TestKMSPluginHealthzTTL(t *testing.T) {
ctx := testContext(t)