adds dynamic audit configuration

Kubernetes-commit: eb89d3dddd3792b0a6cd724e64bbbc11d6c15380
This commit is contained in:
Patrick Barker 2018-10-18 21:34:17 -05:00 committed by Kubernetes Publisher
parent 1073fba42b
commit 9fd62b6f47
7 changed files with 404 additions and 38 deletions

View File

@ -19,6 +19,7 @@ package policy
import (
"k8s.io/api/auditregistration/v1alpha1"
"k8s.io/apiserver/pkg/apis/audit"
"k8s.io/apiserver/pkg/authorization/authorizer"
)
// ConvertDynamicPolicyToInternal constructs an internal policy type from a
@ -37,3 +38,17 @@ func ConvertDynamicPolicyToInternal(p *v1alpha1.Policy) *audit.Policy {
OmitStages: InvertStages(stages),
}
}
// NewDynamicChecker returns a new dynamic policy checker
func NewDynamicChecker() Checker {
return &dynamicPolicyChecker{}
}
type dynamicPolicyChecker struct{}
// LevelAndStages returns returns a fixed level of the full event, this is so that the downstream policy
// can be applied per sink.
// TODO: this needs benchmarking before the API moves to beta to determine the effect this has on the apiserver
func (d *dynamicPolicyChecker) LevelAndStages(authorizer.Attributes) (audit.Level, []audit.Stage) {
return audit.LevelRequestResponse, []audit.Stage{}
}

View File

@ -52,6 +52,13 @@ const (
// audited.
AdvancedAuditing utilfeature.Feature = "AdvancedAuditing"
// owner: @pbarker
// alpha: v1.13
//
// DynamicAuditing enables configuration of audit policy and webhook backends through an
// AuditSink API object.
DynamicAuditing utilfeature.Feature = "DynamicAuditing"
// owner: @ilackams
// alpha: v1.7
//
@ -94,6 +101,7 @@ var defaultKubernetesFeatureGates = map[utilfeature.Feature]utilfeature.FeatureS
StreamingProxyRedirects: {Default: true, PreRelease: utilfeature.Beta},
ValidateProxyRedirects: {Default: false, PreRelease: utilfeature.Alpha},
AdvancedAuditing: {Default: true, PreRelease: utilfeature.GA},
DynamicAuditing: {Default: false, PreRelease: utilfeature.Alpha},
APIResponseCompression: {Default: false, PreRelease: utilfeature.Alpha},
Initializers: {Default: false, PreRelease: utilfeature.Alpha},
APIListChunking: {Default: true, PreRelease: utilfeature.Beta},

View File

@ -27,17 +27,26 @@ import (
"gopkg.in/natefinch/lumberjack.v2"
"k8s.io/klog"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
auditv1 "k8s.io/apiserver/pkg/apis/audit/v1"
auditv1alpha1 "k8s.io/apiserver/pkg/apis/audit/v1alpha1"
auditv1beta1 "k8s.io/apiserver/pkg/apis/audit/v1beta1"
"k8s.io/apiserver/pkg/audit"
"k8s.io/apiserver/pkg/audit/policy"
"k8s.io/apiserver/pkg/features"
"k8s.io/apiserver/pkg/server"
utilfeature "k8s.io/apiserver/pkg/util/feature"
pluginbuffered "k8s.io/apiserver/plugin/pkg/audit/buffered"
plugindynamic "k8s.io/apiserver/plugin/pkg/audit/dynamic"
pluginenforced "k8s.io/apiserver/plugin/pkg/audit/dynamic/enforced"
pluginlog "k8s.io/apiserver/plugin/pkg/audit/log"
plugintruncate "k8s.io/apiserver/plugin/pkg/audit/truncate"
pluginwebhook "k8s.io/apiserver/plugin/pkg/audit/webhook"
"k8s.io/client-go/informers"
"k8s.io/client-go/kubernetes"
v1core "k8s.io/client-go/kubernetes/typed/core/v1"
restclient "k8s.io/client-go/rest"
)
const (
@ -54,6 +63,9 @@ func appendBackend(existing, newBackend audit.Backend) audit.Backend {
if existing == nil {
return newBackend
}
if newBackend == nil {
return existing
}
return audit.Union(existing, newBackend)
}
@ -65,6 +77,7 @@ type AuditOptions struct {
// Plugin options
LogOptions AuditLogOptions
WebhookOptions AuditWebhookOptions
DynamicOptions AuditDynamicOptions
}
const (
@ -129,6 +142,11 @@ type AuditWebhookOptions struct {
GroupVersionString string
}
type AuditDynamicOptions struct {
// Enabled tells whether the dynamic audit capability is enabled.
Enabled bool
}
func NewAuditOptions() *AuditOptions {
return &AuditOptions{
WebhookOptions: AuditWebhookOptions{
@ -149,6 +167,9 @@ func NewAuditOptions() *AuditOptions {
TruncateOptions: NewAuditTruncateOptions(),
GroupVersionString: "audit.k8s.io/v1",
},
DynamicOptions: AuditDynamicOptions{
Enabled: false,
},
}
}
@ -171,6 +192,7 @@ func (o *AuditOptions) Validate() []error {
var allErrors []error
allErrors = append(allErrors, o.LogOptions.Validate()...)
allErrors = append(allErrors, o.WebhookOptions.Validate()...)
allErrors = append(allErrors, o.DynamicOptions.Validate()...)
return allErrors
}
@ -250,44 +272,102 @@ func (o *AuditOptions) AddFlags(fs *pflag.FlagSet) {
o.WebhookOptions.AddFlags(fs)
o.WebhookOptions.BatchOptions.AddFlags(pluginwebhook.PluginName, fs)
o.WebhookOptions.TruncateOptions.AddFlags(pluginwebhook.PluginName, fs)
o.DynamicOptions.AddFlags(fs)
}
func (o *AuditOptions) ApplyTo(c *server.Config) error {
func (o *AuditOptions) ApplyTo(
c *server.Config,
kubeClientConfig *restclient.Config,
informers informers.SharedInformerFactory,
processInfo *ProcessInfo,
webhookOptions *WebhookOptions,
) error {
if o == nil {
return nil
}
if c == nil {
return fmt.Errorf("server config must be non-nil")
}
// Apply advanced options.
// 1. Apply generic options.
if err := o.applyTo(c); err != nil {
// 1. Build policy checker
checker, err := o.newPolicyChecker()
if err != nil {
return err
}
// 2. Apply plugin options.
if err := o.LogOptions.applyTo(c); err != nil {
return err
// 2. Build log backend
var logBackend audit.Backend
if w := o.LogOptions.getWriter(); w != nil {
if checker == nil {
klog.V(2).Info("No audit policy file provided, no events will be recorded for log backend")
} else {
logBackend = o.LogOptions.newBackend(w)
}
}
if err := o.WebhookOptions.applyTo(c); err != nil {
// 3. Build webhook backend
var webhookBackend audit.Backend
if o.WebhookOptions.enabled() {
if checker == nil {
klog.V(2).Info("No audit policy file provided, no events will be recorded for webhook backend")
} else {
webhookBackend, err = o.WebhookOptions.newUntruncatedBackend()
if err != nil {
return err
}
}
}
groupVersion, err := schema.ParseGroupVersion(o.WebhookOptions.GroupVersionString)
if err != nil {
return err
}
if c.AuditBackend != nil && c.AuditPolicyChecker == nil {
klog.V(2).Info("No audit policy file provided for AdvancedAuditing, no events will be recorded.")
// 4. Apply dynamic options.
var dynamicBackend audit.Backend
if o.DynamicOptions.enabled() {
// if dynamic is enabled the webhook and log backends need to be wrapped in an enforced backend with the static policy
if webhookBackend != nil {
webhookBackend = pluginenforced.NewBackend(webhookBackend, checker)
}
if logBackend != nil {
logBackend = pluginenforced.NewBackend(logBackend, checker)
}
// build dynamic backend
dynamicBackend, checker, err = o.DynamicOptions.newBackend(c.ExternalAddress, kubeClientConfig, informers, processInfo, webhookOptions)
if err != nil {
return err
}
// union dynamic and webhook backends so that truncate options can be applied to both
dynamicBackend = appendBackend(webhookBackend, dynamicBackend)
dynamicBackend = o.WebhookOptions.TruncateOptions.wrapBackend(dynamicBackend, groupVersion)
} else if webhookBackend != nil {
// if only webhook is enabled wrap it in the truncate options
dynamicBackend = o.WebhookOptions.TruncateOptions.wrapBackend(webhookBackend, groupVersion)
}
// 5. Set the policy checker
c.AuditPolicyChecker = checker
// 6. Join the log backend with the webhooks
c.AuditBackend = appendBackend(logBackend, dynamicBackend)
if c.AuditBackend != nil {
klog.V(2).Infof("Using audit backend: %s", c.AuditBackend)
}
return nil
}
func (o *AuditOptions) applyTo(c *server.Config) error {
func (o *AuditOptions) newPolicyChecker() (policy.Checker, error) {
if o.PolicyFile == "" {
return nil
return nil, nil
}
p, err := policy.LoadPolicyFromFile(o.PolicyFile)
if err != nil {
return fmt.Errorf("loading audit policy file: %v", err)
return nil, fmt.Errorf("loading audit policy file: %v", err)
}
c.AuditPolicyChecker = policy.NewChecker(p)
return nil
return policy.NewChecker(p), nil
}
func (o *AuditBatchOptions) AddFlags(pluginName string, fs *pflag.FlagSet) {
@ -436,15 +516,12 @@ func (o *AuditLogOptions) getWriter() io.Writer {
return w
}
func (o *AuditLogOptions) applyTo(c *server.Config) error {
if w := o.getWriter(); w != nil {
groupVersion, _ := schema.ParseGroupVersion(o.GroupVersionString)
log := pluginlog.NewBackend(w, o.Format, groupVersion)
log = o.BatchOptions.wrapBackend(log)
log = o.TruncateOptions.wrapBackend(log, groupVersion)
c.AuditBackend = appendBackend(c.AuditBackend, log)
}
return nil
func (o *AuditLogOptions) newBackend(w io.Writer) audit.Backend {
groupVersion, _ := schema.ParseGroupVersion(o.GroupVersionString)
log := pluginlog.NewBackend(w, o.Format, groupVersion)
log = o.BatchOptions.wrapBackend(log)
log = o.TruncateOptions.wrapBackend(log, groupVersion)
return log
}
func (o *AuditWebhookOptions) AddFlags(fs *pflag.FlagSet) {
@ -483,20 +560,76 @@ func (o *AuditWebhookOptions) enabled() bool {
return o != nil && o.ConfigFile != ""
}
func (o *AuditWebhookOptions) applyTo(c *server.Config) error {
if !o.enabled() {
return nil
}
// newUntruncatedBackend returns a webhook backend without the truncate options applied
// this is done so that the same trucate backend can wrap both the webhook and dynamic backends
func (o *AuditWebhookOptions) newUntruncatedBackend() (audit.Backend, error) {
groupVersion, _ := schema.ParseGroupVersion(o.GroupVersionString)
webhook, err := pluginwebhook.NewBackend(o.ConfigFile, groupVersion, o.InitialBackoff)
if err != nil {
return fmt.Errorf("initializing audit webhook: %v", err)
return nil, fmt.Errorf("initializing audit webhook: %v", err)
}
webhook = o.BatchOptions.wrapBackend(webhook)
webhook = o.TruncateOptions.wrapBackend(webhook, groupVersion)
c.AuditBackend = appendBackend(c.AuditBackend, webhook)
return nil
return webhook, nil
}
func (o *AuditDynamicOptions) AddFlags(fs *pflag.FlagSet) {
fs.BoolVar(&o.Enabled, "audit-dynamic-configuration", o.Enabled,
"Enables dynamic audit configuration. This feature also requires the DynamicAudit feature flag")
}
func (o *AuditDynamicOptions) enabled() bool {
return o.Enabled && utilfeature.DefaultFeatureGate.Enabled(features.DynamicAuditing)
}
func (o *AuditDynamicOptions) Validate() []error {
var allErrors []error
if o.Enabled && !utilfeature.DefaultFeatureGate.Enabled(features.DynamicAuditing) {
allErrors = append(allErrors, fmt.Errorf("--audit-dynamic-configuration set, but DynamicAudit feature gate is not enabled"))
}
return allErrors
}
func (o *AuditDynamicOptions) newBackend(
hostname string,
kubeClientConfig *restclient.Config,
informers informers.SharedInformerFactory,
processInfo *ProcessInfo,
webhookOptions *WebhookOptions,
) (audit.Backend, policy.Checker, error) {
if err := validateProcessInfo(processInfo); err != nil {
return nil, nil, err
}
clientset, err := kubernetes.NewForConfig(kubeClientConfig)
if err != nil {
return nil, nil, err
}
if webhookOptions == nil {
webhookOptions = NewWebhookOptions()
}
checker := policy.NewDynamicChecker()
informer := informers.Auditregistration().V1alpha1().AuditSinks()
eventSink := &v1core.EventSinkImpl{Interface: clientset.CoreV1().Events(processInfo.Namespace)}
dc := &plugindynamic.Config{
Informer: informer,
BufferedConfig: plugindynamic.NewDefaultWebhookBatchConfig(),
EventConfig: plugindynamic.EventConfig{
Sink: eventSink,
Source: corev1.EventSource{
Component: processInfo.Name,
Host: hostname,
},
},
WebhookConfig: plugindynamic.WebhookConfig{
AuthInfoResolverWrapper: webhookOptions.AuthInfoResolverWrapper,
ServiceResolver: webhookOptions.ServiceResolver,
},
}
backend, err := plugindynamic.NewBackend(dc)
if err != nil {
return nil, nil, fmt.Errorf("could not create dynamic audit backend: %v", err)
}
return backend, checker, nil
}
// defaultWebhookBatchConfig returns the default BatchConfig used by the Webhook backend.

View File

@ -23,7 +23,15 @@ import (
"os"
"testing"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
auditv1 "k8s.io/apiserver/pkg/apis/audit/v1"
"k8s.io/apiserver/pkg/features"
"k8s.io/apiserver/pkg/server"
utilfeature "k8s.io/apiserver/pkg/util/feature"
utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing"
"k8s.io/client-go/informers"
"k8s.io/client-go/kubernetes/fake"
restclient "k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd/api/v1"
"github.com/spf13/pflag"
@ -35,6 +43,15 @@ func TestAuditValidOptions(t *testing.T) {
webhookConfig := makeTmpWebhookConfig(t)
defer os.Remove(webhookConfig)
policy := makeTmpPolicy(t)
defer os.Remove(policy)
defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.DynamicAuditing, true)()
clientConfig := &restclient.Config{}
informerFactory := informers.NewSharedInformerFactory(fake.NewSimpleClientset(), 0)
processInfo := &ProcessInfo{"test", "test"}
testCases := []struct {
name string
options func() *AuditOptions
@ -47,23 +64,42 @@ func TestAuditValidOptions(t *testing.T) {
options: func() *AuditOptions {
o := NewAuditOptions()
o.LogOptions.Path = "/audit"
o.PolicyFile = policy
return o
},
expected: "log",
}, {
name: "default log no policy",
options: func() *AuditOptions {
o := NewAuditOptions()
o.LogOptions.Path = "/audit"
return o
},
expected: "",
}, {
name: "default webhook",
options: func() *AuditOptions {
o := NewAuditOptions()
o.WebhookOptions.ConfigFile = webhookConfig
o.PolicyFile = policy
return o
},
expected: "buffered<webhook>",
}, {
name: "default webhook no policy",
options: func() *AuditOptions {
o := NewAuditOptions()
o.WebhookOptions.ConfigFile = webhookConfig
return o
},
expected: "",
}, {
name: "default union",
options: func() *AuditOptions {
o := NewAuditOptions()
o.LogOptions.Path = "/audit"
o.WebhookOptions.ConfigFile = webhookConfig
o.PolicyFile = policy
return o
},
expected: "union[log,buffered<webhook>]",
@ -75,6 +111,7 @@ func TestAuditValidOptions(t *testing.T) {
o.LogOptions.Path = "/audit"
o.WebhookOptions.BatchOptions.Mode = ModeBlocking
o.WebhookOptions.ConfigFile = webhookConfig
o.PolicyFile = policy
return o
},
expected: "union[buffered<log>,webhook]",
@ -84,10 +121,62 @@ func TestAuditValidOptions(t *testing.T) {
o := NewAuditOptions()
o.WebhookOptions.ConfigFile = webhookConfig
o.WebhookOptions.TruncateOptions.Enabled = true
o.PolicyFile = policy
return o
},
expected: "truncate<buffered<webhook>>",
}}
}, {
name: "dynamic",
options: func() *AuditOptions {
o := NewAuditOptions()
o.DynamicOptions.Enabled = true
return o
},
expected: "dynamic[]",
}, {
name: "dynamic with truncating",
options: func() *AuditOptions {
o := NewAuditOptions()
o.DynamicOptions.Enabled = true
o.WebhookOptions.TruncateOptions.Enabled = true
return o
},
expected: "truncate<dynamic[]>",
}, {
name: "dynamic with log",
options: func() *AuditOptions {
o := NewAuditOptions()
o.DynamicOptions.Enabled = true
o.LogOptions.Path = "/audit"
o.PolicyFile = policy
return o
},
expected: "union[enforced<log>,dynamic[]]",
}, {
name: "dynamic with truncating and webhook",
options: func() *AuditOptions {
o := NewAuditOptions()
o.DynamicOptions.Enabled = true
o.WebhookOptions.TruncateOptions.Enabled = true
o.WebhookOptions.ConfigFile = webhookConfig
o.PolicyFile = policy
return o
},
expected: "truncate<union[enforced<buffered<webhook>>,dynamic[]]>",
}, {
name: "dynamic with truncating and webhook and log",
options: func() *AuditOptions {
o := NewAuditOptions()
o.DynamicOptions.Enabled = true
o.WebhookOptions.TruncateOptions.Enabled = true
o.WebhookOptions.ConfigFile = webhookConfig
o.PolicyFile = policy
o.LogOptions.Path = "/audit"
return o
},
expected: "union[enforced<log>,truncate<union[enforced<buffered<webhook>>,dynamic[]]>]",
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
options := tc.options()
@ -101,7 +190,7 @@ func TestAuditValidOptions(t *testing.T) {
assert.Empty(t, options.Validate(), "Options should be valid.")
config := &server.Config{}
require.NoError(t, options.ApplyTo(config))
require.NoError(t, options.ApplyTo(config, clientConfig, informerFactory, processInfo, nil))
if tc.expected == "" {
assert.Nil(t, config.AuditBackend)
} else {
@ -176,7 +265,15 @@ func TestAuditInvalidOptions(t *testing.T) {
o.WebhookOptions.TruncateOptions.TruncateConfig.MaxBatchSize = 1
return o
},
}}
}, {
name: "invalid dynamic flag group",
options: func() *AuditOptions {
o := NewAuditOptions()
o.DynamicOptions.Enabled = true
return o
},
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
options := tc.options()
@ -198,3 +295,21 @@ func makeTmpWebhookConfig(t *testing.T) string {
require.NoError(t, f.Close())
return f.Name()
}
func makeTmpPolicy(t *testing.T) string {
pol := auditv1.Policy{
TypeMeta: metav1.TypeMeta{
APIVersion: "audit.k8s.io/v1",
},
Rules: []auditv1.PolicyRule{
{
Level: auditv1.LevelRequestResponse,
},
},
}
f, err := ioutil.TempFile("", "k8s_audit_policy_test_")
require.NoError(t, err, "creating temp file")
require.NoError(t, stdjson.NewEncoder(f).Encode(pol), "writing policy file")
require.NoError(t, f.Close())
return f.Name()
}

View File

@ -0,0 +1,56 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package options
import (
"fmt"
"os"
)
// ProcessInfo holds the apiserver process information used to send events
type ProcessInfo struct {
// Name of the api process to identify events
Name string
// Namespace of the api process to send events
Namespace string
}
// NewProcessInfo returns a new process info with the hostname concatenated to the name given
func NewProcessInfo(name, namespace string) *ProcessInfo {
// try to concat the hostname if available
host, _ := os.Hostname()
if host != "" {
name = fmt.Sprintf("%s-%s", name, host)
}
return &ProcessInfo{
Name: name,
Namespace: namespace,
}
}
// validateProcessInfo checks for a complete process info
func validateProcessInfo(p *ProcessInfo) error {
if p == nil {
return fmt.Errorf("ProcessInfo must be set")
} else if p.Name == "" {
return fmt.Errorf("ProcessInfo name must be set")
} else if p.Namespace == "" {
return fmt.Errorf("ProcessInfo namespace must be set")
}
return nil
}

View File

@ -41,9 +41,12 @@ type RecommendedOptions struct {
// admission plugin initializers to Admission.ApplyTo.
ExtraAdmissionInitializers func(c *server.RecommendedConfig) ([]admission.PluginInitializer, error)
Admission *AdmissionOptions
// ProcessInfo is used to identify events created by the server.
ProcessInfo *ProcessInfo
Webhook *WebhookOptions
}
func NewRecommendedOptions(prefix string, codec runtime.Codec) *RecommendedOptions {
func NewRecommendedOptions(prefix string, codec runtime.Codec, processInfo *ProcessInfo) *RecommendedOptions {
sso := NewSecureServingOptions()
// We are composing recommended options for an aggregated api-server,
@ -62,6 +65,8 @@ func NewRecommendedOptions(prefix string, codec runtime.Codec) *RecommendedOptio
CoreAPI: NewCoreAPIOptions(),
ExtraAdmissionInitializers: func(c *server.RecommendedConfig) ([]admission.PluginInitializer, error) { return nil, nil },
Admission: NewAdmissionOptions(),
ProcessInfo: processInfo,
Webhook: NewWebhookOptions(),
}
}
@ -92,7 +97,7 @@ func (o *RecommendedOptions) ApplyTo(config *server.RecommendedConfig, scheme *r
if err := o.Authorization.ApplyTo(&config.Config.Authorization); err != nil {
return err
}
if err := o.Audit.ApplyTo(&config.Config); err != nil {
if err := o.Audit.ApplyTo(&config.Config, config.ClientConfig, config.SharedInformerFactory, o.ProcessInfo, o.Webhook); err != nil {
return err
}
if err := o.Features.ApplyTo(&config.Config); err != nil {

View File

@ -0,0 +1,34 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package options
import (
utilwebhook "k8s.io/apiserver/pkg/util/webhook"
)
// WebhookOptions holds the outgoing webhook options
type WebhookOptions struct {
ServiceResolver utilwebhook.ServiceResolver
AuthInfoResolverWrapper utilwebhook.AuthenticationInfoResolverWrapper
}
// NewWebhookOptions returns the default options for outgoing webhooks
func NewWebhookOptions() *WebhookOptions {
return &WebhookOptions{
ServiceResolver: utilwebhook.NewDefaultServiceResolver(),
}
}