From 0cb35638f258771d3df04e6d65a1c196f3bb0a0a Mon Sep 17 00:00:00 2001 From: Justin SB Date: Fri, 31 Jan 2020 09:08:39 -0500 Subject: [PATCH] Stop logging to /var/log/kops-controller.log Writing to a hostPath from a non-root container requires file ownership changes, which is difficult to roll out today. See discussion in #8454 We were primarily using the logfile for e2e diagnostics, so we're going to look into collecting the information via other means instead. We also haven't yet shipped this logfile in a released version (though we have shipped it in beta releases) --- cmd/kops-controller/BUILD.bazel | 2 +- dns-controller/cmd/dns-controller/BUILD.bazel | 2 +- hack/.packages | 1 + nodeup/pkg/model/BUILD.bazel | 1 + nodeup/pkg/model/kube_apiserver.go | 3 +- nodeup/pkg/model/logrotate.go | 1 - pkg/wellknownusers/BUILD.bazel | 8 +++++ pkg/wellknownusers/wellknownusers.go | 29 +++++++++++++++++++ .../k8s-1.16.yaml.template | 7 +---- upup/pkg/fi/cloudup/template_functions.go | 2 -- .../amazonvpc/manifest.yaml | 2 +- ...ops-controller.addons.k8s.io-k8s-1.16.yaml | 9 ------ .../cilium/manifest.yaml | 2 +- ...ops-controller.addons.k8s.io-k8s-1.16.yaml | 9 ------ .../simple/manifest.yaml | 2 +- .../weave/manifest.yaml | 2 +- 16 files changed, 48 insertions(+), 34 deletions(-) create mode 100644 pkg/wellknownusers/BUILD.bazel create mode 100644 pkg/wellknownusers/wellknownusers.go diff --git a/cmd/kops-controller/BUILD.bazel b/cmd/kops-controller/BUILD.bazel index 1aa47dd433..f8bf400ef6 100644 --- a/cmd/kops-controller/BUILD.bazel +++ b/cmd/kops-controller/BUILD.bazel @@ -41,7 +41,7 @@ container_image( name = "image", base = "@distroless_base//image", cmd = ["/usr/bin/kops-controller"], - user = "1000", + user = "10001", directory = "/usr/bin/", files = [ "//cmd/kops-controller", diff --git a/dns-controller/cmd/dns-controller/BUILD.bazel b/dns-controller/cmd/dns-controller/BUILD.bazel index dda4c20372..92d7a0d38c 100644 --- a/dns-controller/cmd/dns-controller/BUILD.bazel +++ b/dns-controller/cmd/dns-controller/BUILD.bazel @@ -47,7 +47,7 @@ container_image( name = "image", base = "@distroless_base//image", cmd = ["/usr/bin/dns-controller"], - user = "1000", + user = "10001", directory = "/usr/bin/", files = [ "dns-controller", diff --git a/hack/.packages b/hack/.packages index 0819aa77fa..a6fce0443a 100644 --- a/hack/.packages +++ b/hack/.packages @@ -144,6 +144,7 @@ k8s.io/kops/pkg/util/templater k8s.io/kops/pkg/validation k8s.io/kops/pkg/values k8s.io/kops/pkg/wellknownports +k8s.io/kops/pkg/wellknownusers k8s.io/kops/protokube/cmd/protokube k8s.io/kops/protokube/pkg/etcd k8s.io/kops/protokube/pkg/gossip diff --git a/nodeup/pkg/model/BUILD.bazel b/nodeup/pkg/model/BUILD.bazel index 9f786cf1c0..e8e852c8e3 100644 --- a/nodeup/pkg/model/BUILD.bazel +++ b/nodeup/pkg/model/BUILD.bazel @@ -59,6 +59,7 @@ go_library( "//pkg/systemd:go_default_library", "//pkg/tokens:go_default_library", "//pkg/try:go_default_library", + "//pkg/wellknownusers:go_default_library", "//upup/pkg/fi:go_default_library", "//upup/pkg/fi/cloudup/awsup:go_default_library", "//upup/pkg/fi/nodeup/nodetasks:go_default_library", diff --git a/nodeup/pkg/model/kube_apiserver.go b/nodeup/pkg/model/kube_apiserver.go index e4123195d3..ccabf859d9 100644 --- a/nodeup/pkg/model/kube_apiserver.go +++ b/nodeup/pkg/model/kube_apiserver.go @@ -26,6 +26,7 @@ import ( "k8s.io/kops/pkg/k8scodecs" "k8s.io/kops/pkg/kubeconfig" "k8s.io/kops/pkg/kubemanifest" + "k8s.io/kops/pkg/wellknownusers" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/nodeup/nodetasks" "k8s.io/kops/util/pkg/exec" @@ -225,7 +226,7 @@ func (b *KubeAPIServerBuilder) writeAuthenticationConfig(c *fi.ModelBuilderConte { c.AddTask(&nodetasks.UserTask{ Name: "aws-iam-authenticator", - UID: 10000, + UID: wellknownusers.AWSAuthenticator, Shell: "/sbin/nologin", Home: "/srv/kubernetes/aws-iam-authenticator", }) diff --git a/nodeup/pkg/model/logrotate.go b/nodeup/pkg/model/logrotate.go index fdad1930f7..77bf6c88e1 100644 --- a/nodeup/pkg/model/logrotate.go +++ b/nodeup/pkg/model/logrotate.go @@ -65,7 +65,6 @@ func (b *LogrotateBuilder) Build(c *fi.ModelBuilderContext) error { b.addLogRotate(c, "kubelet", "/var/log/kubelet.log", logRotateOptions{}) b.addLogRotate(c, "etcd", "/var/log/etcd.log", logRotateOptions{}) b.addLogRotate(c, "etcd-events", "/var/log/etcd-events.log", logRotateOptions{}) - b.addLogRotate(c, "kops-controller", "/var/log/kops-controller.log", logRotateOptions{}) if err := b.addLogrotateService(c); err != nil { return err diff --git a/pkg/wellknownusers/BUILD.bazel b/pkg/wellknownusers/BUILD.bazel new file mode 100644 index 0000000000..8dca4e80e4 --- /dev/null +++ b/pkg/wellknownusers/BUILD.bazel @@ -0,0 +1,8 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["wellknownusers.go"], + importpath = "k8s.io/kops/pkg/wellknownusers", + visibility = ["//visibility:public"], +) diff --git a/pkg/wellknownusers/wellknownusers.go b/pkg/wellknownusers/wellknownusers.go new file mode 100644 index 0000000000..8d7f96ad4f --- /dev/null +++ b/pkg/wellknownusers/wellknownusers.go @@ -0,0 +1,29 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package wellknownusers + +// We define some user ids that we use for non-root containers. +// We base at 10000 because some distros (COS) have pre-defined users around 1000 + +const ( + // Generic is the user id we use for non-privileged containers, where we don't need extra permissions + // Used by e.g. dns-controller, kops-controller + Generic = 10001 + + // AWSAuthenticator is the user-id for the aws-iam-authenticator (built externally) + AWSAuthenticator = 10000 +) diff --git a/upup/models/cloudup/resources/addons/kops-controller.addons.k8s.io/k8s-1.16.yaml.template b/upup/models/cloudup/resources/addons/kops-controller.addons.k8s.io/k8s-1.16.yaml.template index e20a037bd3..d6af9a85c3 100644 --- a/upup/models/cloudup/resources/addons/kops-controller.addons.k8s.io/k8s-1.16.yaml.template +++ b/upup/models/cloudup/resources/addons/kops-controller.addons.k8s.io/k8s-1.16.yaml.template @@ -55,8 +55,6 @@ spec: {{ end }} - mountPath: /etc/kubernetes/kops-controller/ name: kops-controller-config - - mountPath: /var/log/kops-controller.log - name: logfile command: {{ range $arg := KopsControllerArgv }} - "{{ $arg }}" @@ -82,10 +80,7 @@ spec: - name: kops-controller-config configMap: name: kops-controller - - name: logfile - hostPath: - path: /var/log/kops-controller.log - type: FileOrCreate + --- apiVersion: v1 diff --git a/upup/pkg/fi/cloudup/template_functions.go b/upup/pkg/fi/cloudup/template_functions.go index cccd309b19..4e4fec025f 100644 --- a/upup/pkg/fi/cloudup/template_functions.go +++ b/upup/pkg/fi/cloudup/template_functions.go @@ -380,8 +380,6 @@ func (tf *TemplateFunctions) KopsControllerArgv() ([]string, error) { argv = append(argv, "--conf=/etc/kubernetes/kops-controller/config.yaml") - argv = append(argv, "--logtostderr=false", "--alsologtostderr", "--log_file=/var/log/kops-controller.log") - return argv, nil } diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/amazonvpc/manifest.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/amazonvpc/manifest.yaml index 26d57400db..13223d55a1 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/amazonvpc/manifest.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/amazonvpc/manifest.yaml @@ -7,7 +7,7 @@ spec: - id: k8s-1.16 kubernetesVersion: '>=1.16.0-alpha.0' manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml - manifestHash: 70bc758d71c37892650b4bbd473d3cdc00a9f71a + manifestHash: 0ccc5c85523793137c3738509747e68605de9301 name: kops-controller.addons.k8s.io selector: k8s-addon: kops-controller.addons.k8s.io diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/kops-controller.addons.k8s.io-k8s-1.16.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/kops-controller.addons.k8s.io-k8s-1.16.yaml index a6b7c692c5..768c54bd98 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/kops-controller.addons.k8s.io-k8s-1.16.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/kops-controller.addons.k8s.io-k8s-1.16.yaml @@ -36,9 +36,6 @@ spec: - /usr/bin/kops-controller - --v=2 - --conf=/etc/kubernetes/kops-controller/config.yaml - - --logtostderr=false - - --alsologtostderr - - --log_file=/var/log/kops-controller.log image: kope/kops-controller:1.18.0-alpha.1 name: kops-controller resources: @@ -48,8 +45,6 @@ spec: volumeMounts: - mountPath: /etc/kubernetes/kops-controller/ name: kops-controller-config - - mountPath: /var/log/kops-controller.log - name: logfile dnsPolicy: Default hostNetwork: true nodeSelector: @@ -63,10 +58,6 @@ spec: - configMap: name: kops-controller name: kops-controller-config - - hostPath: - path: /var/log/kops-controller.log - type: FileOrCreate - name: logfile updateStrategy: rollingUpdate: maxUnavailable: 1 diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/manifest.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/manifest.yaml index 3e3aae7d34..1086197ea0 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/manifest.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/manifest.yaml @@ -7,7 +7,7 @@ spec: - id: k8s-1.16 kubernetesVersion: '>=1.16.0-alpha.0' manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml - manifestHash: 70bc758d71c37892650b4bbd473d3cdc00a9f71a + manifestHash: 0ccc5c85523793137c3738509747e68605de9301 name: kops-controller.addons.k8s.io selector: k8s-addon: kops-controller.addons.k8s.io diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/simple/kops-controller.addons.k8s.io-k8s-1.16.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/simple/kops-controller.addons.k8s.io-k8s-1.16.yaml index a6b7c692c5..768c54bd98 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/simple/kops-controller.addons.k8s.io-k8s-1.16.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/simple/kops-controller.addons.k8s.io-k8s-1.16.yaml @@ -36,9 +36,6 @@ spec: - /usr/bin/kops-controller - --v=2 - --conf=/etc/kubernetes/kops-controller/config.yaml - - --logtostderr=false - - --alsologtostderr - - --log_file=/var/log/kops-controller.log image: kope/kops-controller:1.18.0-alpha.1 name: kops-controller resources: @@ -48,8 +45,6 @@ spec: volumeMounts: - mountPath: /etc/kubernetes/kops-controller/ name: kops-controller-config - - mountPath: /var/log/kops-controller.log - name: logfile dnsPolicy: Default hostNetwork: true nodeSelector: @@ -63,10 +58,6 @@ spec: - configMap: name: kops-controller name: kops-controller-config - - hostPath: - path: /var/log/kops-controller.log - type: FileOrCreate - name: logfile updateStrategy: rollingUpdate: maxUnavailable: 1 diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/simple/manifest.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/simple/manifest.yaml index fdffb20157..e8fdd98121 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/simple/manifest.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/simple/manifest.yaml @@ -7,7 +7,7 @@ spec: - id: k8s-1.16 kubernetesVersion: '>=1.16.0-alpha.0' manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml - manifestHash: 70bc758d71c37892650b4bbd473d3cdc00a9f71a + manifestHash: 0ccc5c85523793137c3738509747e68605de9301 name: kops-controller.addons.k8s.io selector: k8s-addon: kops-controller.addons.k8s.io diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/weave/manifest.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/weave/manifest.yaml index b115bd60d7..1075bcafb0 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/weave/manifest.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/weave/manifest.yaml @@ -7,7 +7,7 @@ spec: - id: k8s-1.16 kubernetesVersion: '>=1.16.0-alpha.0' manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml - manifestHash: 70bc758d71c37892650b4bbd473d3cdc00a9f71a + manifestHash: 0ccc5c85523793137c3738509747e68605de9301 name: kops-controller.addons.k8s.io selector: k8s-addon: kops-controller.addons.k8s.io