diff --git a/cmd/kops-controller/BUILD.bazel b/cmd/kops-controller/BUILD.bazel index 756dd7b64c..a5172a9fd6 100644 --- a/cmd/kops-controller/BUILD.bazel +++ b/cmd/kops-controller/BUILD.bazel @@ -8,6 +8,7 @@ go_library( deps = [ "//cmd/kops-controller/controllers:go_default_library", "//cmd/kops-controller/pkg/config:go_default_library", + "//cmd/kops-controller/pkg/server:go_default_library", "//pkg/nodeidentity:go_default_library", "//pkg/nodeidentity/aws:go_default_library", "//pkg/nodeidentity/do:go_default_library", @@ -47,7 +48,7 @@ ARCH = [ architecture = arch, base = "@distroless_base//image", cmd = ["/kops-controller"], - user = "10001", + user = "10011", files = [ "//cmd/kops-controller", ], diff --git a/cmd/kops-controller/main.go b/cmd/kops-controller/main.go index c1a4a7fd2c..d2e8f55760 100644 --- a/cmd/kops-controller/main.go +++ b/cmd/kops-controller/main.go @@ -29,6 +29,7 @@ import ( "k8s.io/klog/klogr" "k8s.io/kops/cmd/kops-controller/controllers" "k8s.io/kops/cmd/kops-controller/pkg/config" + "k8s.io/kops/cmd/kops-controller/pkg/server" "k8s.io/kops/pkg/nodeidentity" nodeidentityaws "k8s.io/kops/pkg/nodeidentity/aws" nodeidentitydo "k8s.io/kops/pkg/nodeidentity/do" @@ -81,6 +82,18 @@ func main() { } ctrl.SetLogger(klogr.New()) + if opt.Server != nil { + srv, err := server.NewServer(&opt) + if err != nil { + setupLog.Error(err, "unable to create server") + os.Exit(1) + } + go func() { + err := srv.Start() + setupLog.Error(err, "unable to start server") + os.Exit(1) + }() + } if err := buildScheme(); err != nil { setupLog.Error(err, "error building scheme") diff --git a/cmd/kops-controller/pkg/config/options.go b/cmd/kops-controller/pkg/config/options.go index 22a15914b1..fa98e935f3 100644 --- a/cmd/kops-controller/pkg/config/options.go +++ b/cmd/kops-controller/pkg/config/options.go @@ -17,9 +17,20 @@ limitations under the License. package config type Options struct { - Cloud string `json:"cloud,omitempty"` - ConfigBase string `json:"configBase,omitempty"` + Cloud string `json:"cloud,omitempty"` + ConfigBase string `json:"configBase,omitempty"` + Server *ServerOptions `json:"server,omitempty"` } func (o *Options) PopulateDefaults() { } + +type ServerOptions struct { + // Listen is the network endpoint (ip and port) we should listen on. + Listen string + + // ServerKeyPath is the path to our TLS serving private key. + ServerKeyPath string `json:"serverKeyPath,omitempty"` + // ServerCertificatePath is the path to our TLS serving certificate. + ServerCertificatePath string `json:"serverCertificatePath,omitempty"` +} diff --git a/cmd/kops-controller/pkg/server/BUILD.bazel b/cmd/kops-controller/pkg/server/BUILD.bazel new file mode 100644 index 0000000000..831fee41cd --- /dev/null +++ b/cmd/kops-controller/pkg/server/BUILD.bazel @@ -0,0 +1,9 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["server.go"], + importpath = "k8s.io/kops/cmd/kops-controller/pkg/server", + visibility = ["//visibility:public"], + deps = ["//cmd/kops-controller/pkg/config:go_default_library"], +) diff --git a/cmd/kops-controller/pkg/server/server.go b/cmd/kops-controller/pkg/server/server.go new file mode 100644 index 0000000000..50cc46b617 --- /dev/null +++ b/cmd/kops-controller/pkg/server/server.go @@ -0,0 +1,47 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package server + +import ( + "crypto/tls" + "net/http" + + "k8s.io/kops/cmd/kops-controller/pkg/config" +) + +type Server struct { + opt *config.Options + server *http.Server +} + +func NewServer(opt *config.Options) (*Server, error) { + server := &http.Server{ + Addr: opt.Server.Listen, + TLSConfig: &tls.Config{ + MinVersion: tls.VersionTLS12, + PreferServerCipherSuites: true, + }, + } + return &Server{ + opt: opt, + server: server, + }, nil +} + +func (s *Server) Start() error { + return s.server.ListenAndServeTLS(s.opt.Server.ServerCertificatePath, s.opt.Server.ServerKeyPath) +} diff --git a/hack/.packages b/hack/.packages index 0c713ab385..baaa194f36 100644 --- a/hack/.packages +++ b/hack/.packages @@ -20,6 +20,7 @@ k8s.io/kops/cmd/kops/util k8s.io/kops/cmd/kops-controller k8s.io/kops/cmd/kops-controller/controllers k8s.io/kops/cmd/kops-controller/pkg/config +k8s.io/kops/cmd/kops-controller/pkg/server k8s.io/kops/cmd/kube-apiserver-healthcheck k8s.io/kops/cmd/nodeup k8s.io/kops/dns-controller/cmd/dns-controller diff --git a/nodeup/pkg/model/BUILD.bazel b/nodeup/pkg/model/BUILD.bazel index dbd483a3fa..ac26b90d50 100644 --- a/nodeup/pkg/model/BUILD.bazel +++ b/nodeup/pkg/model/BUILD.bazel @@ -15,6 +15,7 @@ go_library( "file_assets.go", "firewall.go", "hooks.go", + "kops_controller.go", "kube_apiserver.go", "kube_apiserver_healthcheck.go", "kube_controller_manager.go", @@ -41,6 +42,7 @@ go_library( "//nodeup/pkg/distros:go_default_library", "//nodeup/pkg/model/resources:go_default_library", "//pkg/apis/kops:go_default_library", + "//pkg/apis/kops/model:go_default_library", "//pkg/apis/kops/util:go_default_library", "//pkg/apis/nodeup:go_default_library", "//pkg/assets:go_default_library", @@ -86,6 +88,7 @@ go_test( "containerd_test.go", "docker_test.go", "fakes_test.go", + "kops_controller_test.go", "kube_apiserver_test.go", "kube_controller_manager_test.go", "kube_proxy_test.go", diff --git a/nodeup/pkg/model/context.go b/nodeup/pkg/model/context.go index ecead29598..5cd801792b 100644 --- a/nodeup/pkg/model/context.go +++ b/nodeup/pkg/model/context.go @@ -25,6 +25,7 @@ import ( "k8s.io/klog" "k8s.io/kops/nodeup/pkg/distros" "k8s.io/kops/pkg/apis/kops" + "k8s.io/kops/pkg/apis/kops/model" "k8s.io/kops/pkg/apis/kops/util" "k8s.io/kops/pkg/apis/nodeup" "k8s.io/kops/pkg/systemd" @@ -326,6 +327,11 @@ func (c *NodeupModelContext) UseEtcdTLSAuth() bool { return false } +// UseKopsControllerForNodeBootstrap checks if nodeup should use kops-controller to bootstrap. +func (c *NodeupModelContext) UseKopsControllerForNodeBootstrap() bool { + return model.UseKopsControllerForNodeBootstrap(c.Cluster) +} + // UseNodeAuthorization checks if have a node authorization policy func (c *NodeupModelContext) UseNodeAuthorization() bool { return c.Cluster.Spec.NodeAuthorization != nil diff --git a/nodeup/pkg/model/kops_controller.go b/nodeup/pkg/model/kops_controller.go new file mode 100644 index 0000000000..1a27af5e85 --- /dev/null +++ b/nodeup/pkg/model/kops_controller.go @@ -0,0 +1,85 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package model + +import ( + "path/filepath" + + "k8s.io/kops/pkg/wellknownusers" + "k8s.io/kops/upup/pkg/fi" + "k8s.io/kops/upup/pkg/fi/nodeup/nodetasks" +) + +// KopsControllerBuilder installs the keys for a kops-controller. +type KopsControllerBuilder struct { + *NodeupModelContext +} + +var _ fi.ModelBuilder = &KopsControllerBuilder{} + +// Build is responsible for configuring keys that will be used by kops-controller (via hostPath) +func (b *KopsControllerBuilder) Build(c *fi.ModelBuilderContext) error { + if !b.IsMaster { + return nil + } + + // Create the directory, even if we aren't going to populate it + pkiDir := "/etc/kubernetes/kops-controller" + c.AddTask(&nodetasks.File{ + Path: pkiDir, + Type: nodetasks.FileType_Directory, + Mode: s("0755"), + }) + + if !b.UseKopsControllerForNodeBootstrap() { + return nil + } + + // We run kops-controller under an unprivileged user (wellknownusers.KopsControllerID), and then grant specific permissions + c.AddTask(&nodetasks.UserTask{ + Name: wellknownusers.KopsControllerName, + UID: wellknownusers.KopsControllerID, + Shell: "/sbin/nologin", + }) + + issueCert := &nodetasks.IssueCert{ + Name: "kops-controller", + Signer: fi.CertificateIDCA, + Type: "server", + Subject: nodetasks.PKIXName{CommonName: "kops-controller"}, + AlternateNames: []string{b.Cluster.Spec.MasterInternalName}, + } + c.AddTask(issueCert) + + certResource, keyResource, _ := issueCert.GetResources() + c.AddTask(&nodetasks.File{ + Path: filepath.Join(pkiDir, "kops-controller.crt"), + Contents: certResource, + Type: nodetasks.FileType_File, + Mode: s("0644"), + Owner: s(wellknownusers.KopsControllerName), + }) + c.AddTask(&nodetasks.File{ + Path: filepath.Join(pkiDir, "kops-controller.key"), + Contents: keyResource, + Type: nodetasks.FileType_File, + Mode: s("0600"), + Owner: s(wellknownusers.KopsControllerName), + }) + + return nil +} diff --git a/nodeup/pkg/model/kops_controller_test.go b/nodeup/pkg/model/kops_controller_test.go new file mode 100644 index 0000000000..46c64e9642 --- /dev/null +++ b/nodeup/pkg/model/kops_controller_test.go @@ -0,0 +1,30 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package model + +import ( + "testing" + + "k8s.io/kops/upup/pkg/fi" +) + +func TestKopsControllerBuilder(t *testing.T) { + RunGoldenTest(t, "tests/golden/minimal", "kops-controller", func(nodeupModelContext *NodeupModelContext, target *fi.ModelBuilderContext) error { + builder := KopsControllerBuilder{NodeupModelContext: nodeupModelContext} + return builder.Build(target) + }) +} diff --git a/pkg/apis/kops/model/BUILD.bazel b/pkg/apis/kops/model/BUILD.bazel index 395598832b..dd9a0e9503 100644 --- a/pkg/apis/kops/model/BUILD.bazel +++ b/pkg/apis/kops/model/BUILD.bazel @@ -2,7 +2,10 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "go_default_library", - srcs = ["utils.go"], + srcs = [ + "features.go", + "utils.go", + ], importpath = "k8s.io/kops/pkg/apis/kops/model", visibility = ["//visibility:public"], deps = [ diff --git a/pkg/apis/kops/model/features.go b/pkg/apis/kops/model/features.go new file mode 100644 index 0000000000..fbc0836986 --- /dev/null +++ b/pkg/apis/kops/model/features.go @@ -0,0 +1,26 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package model + +import ( + "k8s.io/kops/pkg/apis/kops" +) + +// UseKopsControllerForNodeBootstrap is true if nodeup should use kops-controller for bootstrapping. +func UseKopsControllerForNodeBootstrap(cluster *kops.Cluster) bool { + return kops.CloudProviderID(cluster.Spec.CloudProvider) == kops.CloudProviderAWS && cluster.IsKubernetesGTE("1.19") +} diff --git a/pkg/model/context.go b/pkg/model/context.go index e3d9d26341..25ab1ee78f 100644 --- a/pkg/model/context.go +++ b/pkg/model/context.go @@ -267,6 +267,11 @@ func (m *KopsModelContext) CloudTags(name string, shared bool) map[string]string return tags } +// UseKopsControllerForNodeBootstrap checks if nodeup should use kops-controller to bootstrap. +func (m *KopsModelContext) UseKopsControllerForNodeBootstrap() bool { + return model.UseKopsControllerForNodeBootstrap(m.Cluster) +} + // UseBootstrapTokens checks if bootstrap tokens are enabled func (m *KopsModelContext) UseBootstrapTokens() bool { if m.Cluster.Spec.KubeAPIServer == nil { diff --git a/pkg/wellknownports/wellknownports.go b/pkg/wellknownports/wellknownports.go index 71c31b7c79..03b7912aea 100644 --- a/pkg/wellknownports/wellknownports.go +++ b/pkg/wellknownports/wellknownports.go @@ -44,6 +44,9 @@ const ( // DNSControllerGossipMemberlist is the port where dns-controller listens for the memberlist-backed gossip DNSControllerGossipMemberlist = 3993 + // KopsControllerPort is the port where kops-controller listens. + KopsControllerPort = 3992 + // 4001 is etcd main, 4002 is etcd events, 4003 is etcd cilium // KubeAPIServerHealthCheck is the port where kube-apiserver-healthcheck listens. diff --git a/pkg/wellknownusers/wellknownusers.go b/pkg/wellknownusers/wellknownusers.go index 4e9580a4b6..cc470873a6 100644 --- a/pkg/wellknownusers/wellknownusers.go +++ b/pkg/wellknownusers/wellknownusers.go @@ -21,7 +21,7 @@ package wellknownusers const ( // Generic is the user id we use for non-privileged containers, where we don't need extra permissions - // Used by e.g. dns-controller, kops-controller + // Used by e.g. dns-controller Generic = 10001 // LegacyEtcd is the user id for the etcd user under the legacy provider @@ -30,6 +30,13 @@ const ( // AWSAuthenticator is the user-id for the aws-iam-authenticator (built externally) AWSAuthenticator = 10000 + // KopsControllerID is the user id for kops-controller, which needs some extra permissions e.g. to write local logs + // This should match the user in cmd/kops-controller/BUILD.bazel + KopsControllerID = 10011 + + // KopsControllerName is the username for the kops-controller user + KopsControllerName = "kops-controller" + // KubeApiserverHealthcheckID is the user id for kube-apiserver-healthcheck sidecar // The user needs some extra permissions e.g. to read local secrets // This should match the user in cmd/kube-apiserver-healthcheck/BUILD.bazel diff --git a/upup/models/bindata.go b/upup/models/bindata.go index ae4bb1e2dc..b66390f731 100644 --- a/upup/models/bindata.go +++ b/upup/models/bindata.go @@ -2536,8 +2536,10 @@ spec: name: etc-ssl-certs readOnly: true {{ end }} - - mountPath: /etc/kubernetes/kops-controller/ + - mountPath: /etc/kubernetes/kops-controller/config/ name: kops-controller-config + - mountPath: /etc/kubernetes/kops-controller/pki/ + name: kops-controller-pki command: {{ range $arg := KopsControllerArgv }} - "{{ $arg }}" @@ -2565,7 +2567,10 @@ spec: - name: kops-controller-config configMap: name: kops-controller - + - name: kops-controller-pki + hostPath: + path: /etc/kubernetes/kops-controller/ + type: Directory --- apiVersion: v1 diff --git a/upup/models/cloudup/resources/addons/kops-controller.addons.k8s.io/k8s-1.16.yaml.template b/upup/models/cloudup/resources/addons/kops-controller.addons.k8s.io/k8s-1.16.yaml.template index 5d4a91fd1c..4866f9302b 100644 --- a/upup/models/cloudup/resources/addons/kops-controller.addons.k8s.io/k8s-1.16.yaml.template +++ b/upup/models/cloudup/resources/addons/kops-controller.addons.k8s.io/k8s-1.16.yaml.template @@ -53,8 +53,10 @@ spec: name: etc-ssl-certs readOnly: true {{ end }} - - mountPath: /etc/kubernetes/kops-controller/ + - mountPath: /etc/kubernetes/kops-controller/config/ name: kops-controller-config + - mountPath: /etc/kubernetes/kops-controller/pki/ + name: kops-controller-pki command: {{ range $arg := KopsControllerArgv }} - "{{ $arg }}" @@ -82,7 +84,10 @@ spec: - name: kops-controller-config configMap: name: kops-controller - + - name: kops-controller-pki + hostPath: + path: /etc/kubernetes/kops-controller/ + type: Directory --- apiVersion: v1 diff --git a/upup/pkg/fi/cloudup/template_functions.go b/upup/pkg/fi/cloudup/template_functions.go index 2b9ee91485..1430c8299a 100644 --- a/upup/pkg/fi/cloudup/template_functions.go +++ b/upup/pkg/fi/cloudup/template_functions.go @@ -31,6 +31,7 @@ import ( "encoding/json" "fmt" "os" + "path" "strconv" "strings" "text/template" @@ -379,6 +380,15 @@ func (tf *TemplateFunctions) KopsControllerConfig() (string, error) { ConfigBase: cluster.Spec.ConfigBase, } + if tf.UseKopsControllerForNodeBootstrap() { + pkiDir := "/etc/kubernetes/kops-controller/pki" + config.Server = &kopscontrollerconfig.ServerOptions{ + Listen: fmt.Sprintf(":%d", wellknownports.KopsControllerPort), + ServerCertificatePath: path.Join(pkiDir, "kops-controller.crt"), + ServerKeyPath: path.Join(pkiDir, "kops-controller.key"), + } + } + // To avoid indentation problems, we marshal as json. json is a subset of yaml b, err := json.Marshal(config) if err != nil { @@ -397,7 +407,7 @@ func (tf *TemplateFunctions) KopsControllerArgv() ([]string, error) { // Verbose, but not excessive logging argv = append(argv, "--v=2") - argv = append(argv, "--conf=/etc/kubernetes/kops-controller/config.yaml") + argv = append(argv, "--conf=/etc/kubernetes/kops-controller/config/config.yaml") return argv, nil } diff --git a/upup/pkg/fi/nodeup/command.go b/upup/pkg/fi/nodeup/command.go index 43df77a21d..0772345fc6 100644 --- a/upup/pkg/fi/nodeup/command.go +++ b/upup/pkg/fi/nodeup/command.go @@ -244,6 +244,7 @@ func (c *NodeUpCommand) Run(out io.Writer) error { loader.Builders = append(loader.Builders, &model.KubeSchedulerBuilder{NodeupModelContext: modelContext}) loader.Builders = append(loader.Builders, &model.EtcdManagerTLSBuilder{NodeupModelContext: modelContext}) loader.Builders = append(loader.Builders, &model.KubeProxyBuilder{NodeupModelContext: modelContext}) + loader.Builders = append(loader.Builders, &model.KopsControllerBuilder{NodeupModelContext: modelContext}) loader.Builders = append(loader.Builders, &networking.CommonBuilder{NodeupModelContext: modelContext}) loader.Builders = append(loader.Builders, &networking.CalicoBuilder{NodeupModelContext: modelContext})