diff --git a/docs/cluster_spec.md b/docs/cluster_spec.md index e913f0593d..d742d9d09d 100644 --- a/docs/cluster_spec.md +++ b/docs/cluster_spec.md @@ -246,6 +246,24 @@ You could use the [fileAssets](https://github.com/kubernetes/kops/blob/master/do Example policy file can be found [here](https://raw.githubusercontent.com/kubernetes/website/master/content/en/examples/audit/audit-policy.yaml) +#### dynamic audit configuration + +Read more about this here: https://kubernetes.io/docs/tasks/debug-application-cluster/audit/#dynamic-backend + +```yaml +spec: + kubeAPIServer: + auditDynamicConfiguration: true +``` + +By enabling this feature you are allowing for auditsinks to be registered with the API server. For information on audit sinks please read [Audit Sink](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.13/#auditsink-v1alpha1-auditregistration). This feature is only supported in kubernetes versions greater than 1.13. Currently, this feature is alpha and requires enabling the feature gate and a runtime config. + +**Note** For kubernetes versions greater than 1.13, this is an alpha feature that requires the API auditregistration.k8s.io/v1alpha1 to be enabled as a runtime-config option, and the feature gate DynamicAuditing to be also enabled. The options --feature-gates=DynamicAuditing=true and --runtime-config=auditregistration.k8s.io/v1alpha1=true must be enabled on the API server in addition to this flag. See the sections for how to enable feature gates [here](https://github.com/kubernetes/kops/blob/master/docs/cluster_spec.md#feature-gates). See the section on how to enable alphas APIs in the runtime config [here](https://github.com/kubernetes/kops/blob/master/docs/cluster_spec.md#runtimeconfig). +Also, an audit policy should be provided in the file assets section. If the flag is omitted, no events are logged. +You could use the [fileAssets](https://github.com/kubernetes/kops/blob/master/docs/cluster_spec.md#fileassets) feature to push an advanced audit policy file on the master nodes. + +Example policy file can be found [here](https://raw.githubusercontent.com/kubernetes/website/master/content/en/examples/audit/audit-policy.yaml) + #### bootstrap tokens Read more about this here: https://kubernetes.io/docs/reference/access-authn-authz/bootstrap-tokens/ diff --git a/k8s/crds/kops.k8s.io_clusters.yaml b/k8s/crds/kops.k8s.io_clusters.yaml index 632b836cbc..b867faf575 100644 --- a/k8s/crds/kops.k8s.io_clusters.yaml +++ b/k8s/crds/kops.k8s.io_clusters.yaml @@ -776,6 +776,10 @@ spec: items: type: string type: array + auditDynamicConfiguration: + description: AuditDynamicConfiguration enables dynamic audit configuration + via AuditSinks + type: boolean auditLogFormat: description: AuditLogFormat flag specifies the format type for audit log files. diff --git a/nodeup/pkg/model/BUILD.bazel b/nodeup/pkg/model/BUILD.bazel index e22da1cf15..45cea3745f 100644 --- a/nodeup/pkg/model/BUILD.bazel +++ b/nodeup/pkg/model/BUILD.bazel @@ -95,6 +95,7 @@ go_test( "//pkg/flagbuilder:go_default_library", "//pkg/testutils:go_default_library", "//upup/pkg/fi:go_default_library", + "//upup/pkg/fi/nodeup/nodetasks:go_default_library", "//util/pkg/exec:go_default_library", "//vendor/github.com/blang/semver:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", diff --git a/nodeup/pkg/model/kube_apiserver.go b/nodeup/pkg/model/kube_apiserver.go index df7c12b4f6..5d7fbaa504 100644 --- a/nodeup/pkg/model/kube_apiserver.go +++ b/nodeup/pkg/model/kube_apiserver.go @@ -347,6 +347,11 @@ func (b *KubeAPIServerBuilder) buildPod() (*v1.Pod, error) { } } + //remove elements from the spec that are not enabled yet + if b.Cluster.Spec.KubeAPIServer.AuditDynamicConfiguration != nil && !b.IsKubernetesGTE("1.13") { + b.Cluster.Spec.KubeAPIServer.AuditDynamicConfiguration = nil + } + // build the kube-apiserver flags for the service flags, err := flagbuilder.BuildFlagsList(b.Cluster.Spec.KubeAPIServer) if err != nil { diff --git a/nodeup/pkg/model/kube_apiserver_test.go b/nodeup/pkg/model/kube_apiserver_test.go index d7b38d129a..e27fa7c785 100644 --- a/nodeup/pkg/model/kube_apiserver_test.go +++ b/nodeup/pkg/model/kube_apiserver_test.go @@ -17,13 +17,49 @@ limitations under the License. package model import ( + "bytes" + "strings" "testing" "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/pkg/flagbuilder" "k8s.io/kops/upup/pkg/fi" + "k8s.io/kops/upup/pkg/fi/nodeup/nodetasks" ) +func Test_KubeAPIServer_Builder(t *testing.T) { + basedir := "tests/apiServer/auditDynamicConfiguration" + + context := &fi.ModelBuilderContext{ + Tasks: make(map[string]fi.Task), + } + + nodeUpModelContext, err := BuildNodeupModelContext(basedir) + if err != nil { + t.Fatalf("error loading model %q: %v", basedir, err) + return + } + + builder := KubeAPIServerBuilder{NodeupModelContext: nodeUpModelContext} + + err = builder.Build(context) + if err != nil { + t.Fatalf("error from KubeAPIServerBuilder buildKubeletConfig: %v", err) + return + } + if task, ok := context.Tasks["File//etc/kubernetes/manifests/kube-apiserver.manifest"]; !ok { + t.Error("did not find the kubernetes API manifest after the build") + } else { + nodeTask, _ := task.(*nodetasks.File) + reader, _ := nodeTask.Contents.Open() + buf := new(bytes.Buffer) + buf.ReadFrom(reader) + s := buf.String() + if strings.Contains(s, "--audit-dynamic-configuration") { + t.Error("Older versions of k8s should not have --audit-dynamic-configuration flag") + } + } +} func Test_KubeAPIServer_BuildFlags(t *testing.T) { grid := []struct { config kops.KubeAPIServerConfig @@ -100,9 +136,23 @@ func Test_KubeAPIServer_BuildFlags(t *testing.T) { }, { kops.KubeAPIServerConfig{ - ServiceAccountKeyFile: []string{"/srv/kubernetes/server.key", "/srv/kubernetes/service-account.key"}, + AuditDynamicConfiguration: &[]bool{true}[0], + ServiceAccountKeyFile: []string{"/srv/kubernetes/server.key", "/srv/kubernetes/service-account.key"}, }, - "--insecure-port=0 --secure-port=0 --service-account-key-file=/srv/kubernetes/server.key --service-account-key-file=/srv/kubernetes/service-account.key", + "--audit-dynamic-configuration=true --insecure-port=0 --secure-port=0 --service-account-key-file=/srv/kubernetes/server.key --service-account-key-file=/srv/kubernetes/service-account.key", + }, + { + kops.KubeAPIServerConfig{ + AuditDynamicConfiguration: &[]bool{false}[0], + }, + "--audit-dynamic-configuration=false --insecure-port=0 --secure-port=0", + }, + { + kops.KubeAPIServerConfig{ + + AuditDynamicConfiguration: &[]bool{true}[0], + }, + "--audit-dynamic-configuration=true --insecure-port=0 --secure-port=0", }, } diff --git a/nodeup/pkg/model/tests/apiServer/auditDynamicConfiguration/cluster.yaml b/nodeup/pkg/model/tests/apiServer/auditDynamicConfiguration/cluster.yaml new file mode 100644 index 0000000000..04e13ff66c --- /dev/null +++ b/nodeup/pkg/model/tests/apiServer/auditDynamicConfiguration/cluster.yaml @@ -0,0 +1,63 @@ +apiVersion: kops.k8s.io/v1alpha2 +kind: Cluster +metadata: + creationTimestamp: "2016-12-10T22:42:27Z" + name: minimal.example.com +spec: + kubeAPIServer: + auditDynamicConfiguration: true + kubernetesApiAccess: + - 0.0.0.0/0 + channel: stable + cloudProvider: aws + configBase: memfs://clusters.example.com/minimal.example.com + etcdClusters: + - etcdMembers: + - instanceGroup: master-us-test-1a + name: master-us-test-1a + name: main + - etcdMembers: + - instanceGroup: master-us-test-1a + name: master-us-test-1a + name: events + kubelet: + featureGates: + ExperimentalCriticalPodAnnotation: "true" + AllowExtTrafficLocalEndpoints: "false" + podManifestPath: "/etc/kubernetes/manifests" + kubernetesVersion: v1.6.0 + masterInternalName: api.internal.minimal.example.com + masterPublicName: api.minimal.example.com + networkCIDR: 172.20.0.0/16 + networking: + kubenet: {} + nonMasqueradeCIDR: 100.64.0.0/10 + sshAccess: + - 0.0.0.0/0 + topology: + masters: public + nodes: public + subnets: + - cidr: 172.20.32.0/19 + name: us-test-1a + type: Public + zone: us-test-1a + +--- + +apiVersion: kops.k8s.io/v1alpha2 +kind: InstanceGroup +metadata: + creationTimestamp: "2016-12-10T22:42:28Z" + name: masters + labels: + kops.k8s.io/cluster: minimal.example.com +spec: + associatePublicIp: true + image: kope.io/k8s-1.4-debian-jessie-amd64-hvm-ebs-2016-10-21 + machineType: t2.medium + maxSize: 1 + minSize: 1 + role: Master + subnets: + - us-test-1a diff --git a/pkg/apis/kops/componentconfig.go b/pkg/apis/kops/componentconfig.go index b1037cb08d..91d8296b0f 100644 --- a/pkg/apis/kops/componentconfig.go +++ b/pkg/apis/kops/componentconfig.go @@ -453,6 +453,9 @@ type KubeAPIServerConfig struct { // Amount of time to retain Kubernetes events EventTTL *metav1.Duration `json:"eventTTL,omitempty" flag:"event-ttl"` + + // AuditDynamicConfiguration enables dynamic audit configuration via AuditSinks + AuditDynamicConfiguration *bool `json:"auditDynamicConfiguration,omitempty" flag:"audit-dynamic-configuration"` } // KubeControllerManagerConfig is the configuration for the controller diff --git a/pkg/apis/kops/v1alpha1/componentconfig.go b/pkg/apis/kops/v1alpha1/componentconfig.go index ece2be21e8..ccd31c1ede 100644 --- a/pkg/apis/kops/v1alpha1/componentconfig.go +++ b/pkg/apis/kops/v1alpha1/componentconfig.go @@ -453,6 +453,9 @@ type KubeAPIServerConfig struct { // Amount of time to retain Kubernetes events EventTTL *metav1.Duration `json:"eventTTL,omitempty" flag:"event-ttl"` + + // AuditDynamicConfiguration enables dynamic audit configuration via AuditSinks + AuditDynamicConfiguration *bool `json:"auditDynamicConfiguration,omitempty" flag:"audit-dynamic-configuration"` } // KubeControllerManagerConfig is the configuration for the controller diff --git a/pkg/apis/kops/v1alpha1/zz_generated.conversion.go b/pkg/apis/kops/v1alpha1/zz_generated.conversion.go index 1f397f19d8..d202444e69 100644 --- a/pkg/apis/kops/v1alpha1/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha1/zz_generated.conversion.go @@ -3232,6 +3232,7 @@ func autoConvert_v1alpha1_KubeAPIServerConfig_To_kops_KubeAPIServerConfig(in *Ku out.APIAudiences = in.APIAudiences out.CPURequest = in.CPURequest out.EventTTL = in.EventTTL + out.AuditDynamicConfiguration = in.AuditDynamicConfiguration return nil } @@ -3330,6 +3331,7 @@ func autoConvert_kops_KubeAPIServerConfig_To_v1alpha1_KubeAPIServerConfig(in *ko out.APIAudiences = in.APIAudiences out.CPURequest = in.CPURequest out.EventTTL = in.EventTTL + out.AuditDynamicConfiguration = in.AuditDynamicConfiguration return nil } diff --git a/pkg/apis/kops/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/kops/v1alpha1/zz_generated.deepcopy.go index 713bea1939..516496dcee 100644 --- a/pkg/apis/kops/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/kops/v1alpha1/zz_generated.deepcopy.go @@ -2009,6 +2009,11 @@ func (in *KubeAPIServerConfig) DeepCopyInto(out *KubeAPIServerConfig) { *out = new(v1.Duration) **out = **in } + if in.AuditDynamicConfiguration != nil { + in, out := &in.AuditDynamicConfiguration, &out.AuditDynamicConfiguration + *out = new(bool) + **out = **in + } return } diff --git a/pkg/apis/kops/v1alpha2/componentconfig.go b/pkg/apis/kops/v1alpha2/componentconfig.go index eaff980d4f..d76a2fad49 100644 --- a/pkg/apis/kops/v1alpha2/componentconfig.go +++ b/pkg/apis/kops/v1alpha2/componentconfig.go @@ -453,6 +453,9 @@ type KubeAPIServerConfig struct { // Amount of time to retain Kubernetes events EventTTL *metav1.Duration `json:"eventTTL,omitempty" flag:"event-ttl"` + + // AuditDynamicConfiguration enables dynamic audit configuration via AuditSinks + AuditDynamicConfiguration *bool `json:"auditDynamicConfiguration,omitempty" flag:"audit-dynamic-configuration"` } // KubeControllerManagerConfig is the configuration for the controller diff --git a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go index 74411609f2..a42124bea2 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go @@ -3502,6 +3502,7 @@ func autoConvert_v1alpha2_KubeAPIServerConfig_To_kops_KubeAPIServerConfig(in *Ku out.APIAudiences = in.APIAudiences out.CPURequest = in.CPURequest out.EventTTL = in.EventTTL + out.AuditDynamicConfiguration = in.AuditDynamicConfiguration return nil } @@ -3600,6 +3601,7 @@ func autoConvert_kops_KubeAPIServerConfig_To_v1alpha2_KubeAPIServerConfig(in *ko out.APIAudiences = in.APIAudiences out.CPURequest = in.CPURequest out.EventTTL = in.EventTTL + out.AuditDynamicConfiguration = in.AuditDynamicConfiguration return nil } diff --git a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go index 527cb5f600..e23585318a 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go @@ -2080,6 +2080,11 @@ func (in *KubeAPIServerConfig) DeepCopyInto(out *KubeAPIServerConfig) { *out = new(v1.Duration) **out = **in } + if in.AuditDynamicConfiguration != nil { + in, out := &in.AuditDynamicConfiguration, &out.AuditDynamicConfiguration + *out = new(bool) + **out = **in + } return } diff --git a/pkg/apis/kops/zz_generated.deepcopy.go b/pkg/apis/kops/zz_generated.deepcopy.go index d5b2005e58..94aa9369d7 100644 --- a/pkg/apis/kops/zz_generated.deepcopy.go +++ b/pkg/apis/kops/zz_generated.deepcopy.go @@ -2262,6 +2262,11 @@ func (in *KubeAPIServerConfig) DeepCopyInto(out *KubeAPIServerConfig) { *out = new(v1.Duration) **out = **in } + if in.AuditDynamicConfiguration != nil { + in, out := &in.AuditDynamicConfiguration, &out.AuditDynamicConfiguration + *out = new(bool) + **out = **in + } return }