Add server code to kops-controller

cmd/kops-controller/BUILD.bazel

@@ -8,6 +8,7 @@ go_library(
     deps = [
         "//cmd/kops-controller/controllers:go_default_library",
         "//cmd/kops-controller/pkg/config:go_default_library",
+        "//cmd/kops-controller/pkg/server:go_default_library",
         "//pkg/nodeidentity:go_default_library",
         "//pkg/nodeidentity/aws:go_default_library",
         "//pkg/nodeidentity/do:go_default_library",
@@ -47,7 +48,7 @@ ARCH = [
     architecture = arch,
     base = "@distroless_base//image",
     cmd = ["/kops-controller"],
-    user = "10001",
+    user = "10011",
     files = [
         "//cmd/kops-controller",
     ],

cmd/kops-controller/main.go

@@ -29,6 +29,7 @@ import (
 	"k8s.io/klog/klogr"
 	"k8s.io/kops/cmd/kops-controller/controllers"
 	"k8s.io/kops/cmd/kops-controller/pkg/config"
+	"k8s.io/kops/cmd/kops-controller/pkg/server"
 	"k8s.io/kops/pkg/nodeidentity"
 	nodeidentityaws "k8s.io/kops/pkg/nodeidentity/aws"
 	nodeidentitydo "k8s.io/kops/pkg/nodeidentity/do"
@@ -81,6 +82,18 @@ func main() {
 	}
 
 	ctrl.SetLogger(klogr.New())
+	if opt.Server != nil {
+		srv, err := server.NewServer(&opt)
+		if err != nil {
+			setupLog.Error(err, "unable to create server")
+			os.Exit(1)
+		}
+		go func() {
+			err := srv.Start()
+			setupLog.Error(err, "unable to start server")
+			os.Exit(1)
+		}()
+	}
 
 	if err := buildScheme(); err != nil {
 		setupLog.Error(err, "error building scheme")

cmd/kops-controller/pkg/config/options.go

@@ -17,9 +17,20 @@ limitations under the License.
 package config
 
 type Options struct {
-	Cloud      string `json:"cloud,omitempty"`
-	ConfigBase string `json:"configBase,omitempty"`
+	Cloud      string         `json:"cloud,omitempty"`
+	ConfigBase string         `json:"configBase,omitempty"`
+	Server     *ServerOptions `json:"server,omitempty"`
 }
 
 func (o *Options) PopulateDefaults() {
 }
+
+type ServerOptions struct {
+	// Listen is the network endpoint (ip and port) we should listen on.
+	Listen string
+
+	// ServerKeyPath is the path to our TLS serving private key.
+	ServerKeyPath string `json:"serverKeyPath,omitempty"`
+	// ServerCertificatePath is the path to our TLS serving certificate.
+	ServerCertificatePath string `json:"serverCertificatePath,omitempty"`
+}

cmd/kops-controller/pkg/server/BUILD.bazel

@@ -0,0 +1,9 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["server.go"],
+    importpath = "k8s.io/kops/cmd/kops-controller/pkg/server",
+    visibility = ["//visibility:public"],
+    deps = ["//cmd/kops-controller/pkg/config:go_default_library"],
+)

cmd/kops-controller/pkg/server/server.go

@@ -0,0 +1,47 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package server
+
+import (
+	"crypto/tls"
+	"net/http"
+
+	"k8s.io/kops/cmd/kops-controller/pkg/config"
+)
+
+type Server struct {
+	opt    *config.Options
+	server *http.Server
+}
+
+func NewServer(opt *config.Options) (*Server, error) {
+	server := &http.Server{
+		Addr: opt.Server.Listen,
+		TLSConfig: &tls.Config{
+			MinVersion:               tls.VersionTLS12,
+			PreferServerCipherSuites: true,
+		},
+	}
+	return &Server{
+		opt:    opt,
+		server: server,
+	}, nil
+}
+
+func (s *Server) Start() error {
+	return s.server.ListenAndServeTLS(s.opt.Server.ServerCertificatePath, s.opt.Server.ServerKeyPath)
+}

hack/.packages

@@ -20,6 +20,7 @@ k8s.io/kops/cmd/kops/util
 k8s.io/kops/cmd/kops-controller
 k8s.io/kops/cmd/kops-controller/controllers
 k8s.io/kops/cmd/kops-controller/pkg/config
+k8s.io/kops/cmd/kops-controller/pkg/server
 k8s.io/kops/cmd/kube-apiserver-healthcheck
 k8s.io/kops/cmd/nodeup
 k8s.io/kops/dns-controller/cmd/dns-controller

nodeup/pkg/model/BUILD.bazel

@@ -15,6 +15,7 @@ go_library(
         "file_assets.go",
         "firewall.go",
         "hooks.go",
+        "kops_controller.go",
         "kube_apiserver.go",
         "kube_apiserver_healthcheck.go",
         "kube_controller_manager.go",
@@ -41,6 +42,7 @@ go_library(
         "//nodeup/pkg/distros:go_default_library",
         "//nodeup/pkg/model/resources:go_default_library",
         "//pkg/apis/kops:go_default_library",
+        "//pkg/apis/kops/model:go_default_library",
         "//pkg/apis/kops/util:go_default_library",
         "//pkg/apis/nodeup:go_default_library",
         "//pkg/assets:go_default_library",
@@ -86,6 +88,7 @@ go_test(
         "containerd_test.go",
         "docker_test.go",
         "fakes_test.go",
+        "kops_controller_test.go",
         "kube_apiserver_test.go",
         "kube_controller_manager_test.go",
         "kube_proxy_test.go",

nodeup/pkg/model/context.go

@@ -25,6 +25,7 @@ import (
 	"k8s.io/klog"
 	"k8s.io/kops/nodeup/pkg/distros"
 	"k8s.io/kops/pkg/apis/kops"
+	"k8s.io/kops/pkg/apis/kops/model"
 	"k8s.io/kops/pkg/apis/kops/util"
 	"k8s.io/kops/pkg/apis/nodeup"
 	"k8s.io/kops/pkg/systemd"
@@ -326,6 +327,11 @@ func (c *NodeupModelContext) UseEtcdTLSAuth() bool {
 	return false
 }
 
+// UseKopsControllerForNodeBootstrap checks if nodeup should use kops-controller to bootstrap.
+func (c *NodeupModelContext) UseKopsControllerForNodeBootstrap() bool {
+	return model.UseKopsControllerForNodeBootstrap(c.Cluster)
+}
+
 // UseNodeAuthorization checks if have a node authorization policy
 func (c *NodeupModelContext) UseNodeAuthorization() bool {
 	return c.Cluster.Spec.NodeAuthorization != nil

nodeup/pkg/model/kops_controller.go

@@ -0,0 +1,85 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package model
+
+import (
+	"path/filepath"
+
+	"k8s.io/kops/pkg/wellknownusers"
+	"k8s.io/kops/upup/pkg/fi"
+	"k8s.io/kops/upup/pkg/fi/nodeup/nodetasks"
+)
+
+// KopsControllerBuilder installs the keys for a kops-controller.
+type KopsControllerBuilder struct {
+	*NodeupModelContext
+}
+
+var _ fi.ModelBuilder = &KopsControllerBuilder{}
+
+// Build is responsible for configuring keys that will be used by kops-controller (via hostPath)
+func (b *KopsControllerBuilder) Build(c *fi.ModelBuilderContext) error {
+	if !b.IsMaster {
+		return nil
+	}
+
+	// Create the directory, even if we aren't going to populate it
+	pkiDir := "/etc/kubernetes/kops-controller"
+	c.AddTask(&nodetasks.File{
+		Path: pkiDir,
+		Type: nodetasks.FileType_Directory,
+		Mode: s("0755"),
+	})
+
+	if !b.UseKopsControllerForNodeBootstrap() {
+		return nil
+	}
+
+	// We run kops-controller under an unprivileged user (wellknownusers.KopsControllerID), and then grant specific permissions
+	c.AddTask(&nodetasks.UserTask{
+		Name:  wellknownusers.KopsControllerName,
+		UID:   wellknownusers.KopsControllerID,
+		Shell: "/sbin/nologin",
+	})
+
+	issueCert := &nodetasks.IssueCert{
+		Name:           "kops-controller",
+		Signer:         fi.CertificateIDCA,
+		Type:           "server",
+		Subject:        nodetasks.PKIXName{CommonName: "kops-controller"},
+		AlternateNames: []string{b.Cluster.Spec.MasterInternalName},
+	}
+	c.AddTask(issueCert)
+
+	certResource, keyResource, _ := issueCert.GetResources()
+	c.AddTask(&nodetasks.File{
+		Path:     filepath.Join(pkiDir, "kops-controller.crt"),
+		Contents: certResource,
+		Type:     nodetasks.FileType_File,
+		Mode:     s("0644"),
+		Owner:    s(wellknownusers.KopsControllerName),
+	})
+	c.AddTask(&nodetasks.File{
+		Path:     filepath.Join(pkiDir, "kops-controller.key"),
+		Contents: keyResource,
+		Type:     nodetasks.FileType_File,
+		Mode:     s("0600"),
+		Owner:    s(wellknownusers.KopsControllerName),
+	})
+
+	return nil
+}

nodeup/pkg/model/kops_controller_test.go

@@ -0,0 +1,30 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package model
+
+import (
+	"testing"
+
+	"k8s.io/kops/upup/pkg/fi"
+)
+
+func TestKopsControllerBuilder(t *testing.T) {
+	RunGoldenTest(t, "tests/golden/minimal", "kops-controller", func(nodeupModelContext *NodeupModelContext, target *fi.ModelBuilderContext) error {
+		builder := KopsControllerBuilder{NodeupModelContext: nodeupModelContext}
+		return builder.Build(target)
+	})
+}

pkg/apis/kops/model/BUILD.bazel

@@ -2,7 +2,10 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
 
 go_library(
     name = "go_default_library",
-    srcs = ["utils.go"],
+    srcs = [
+        "features.go",
+        "utils.go",
+    ],
     importpath = "k8s.io/kops/pkg/apis/kops/model",
     visibility = ["//visibility:public"],
     deps = [

pkg/apis/kops/model/features.go

@@ -0,0 +1,26 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package model
+
+import (
+	"k8s.io/kops/pkg/apis/kops"
+)
+
+// UseKopsControllerForNodeBootstrap is true if nodeup should use kops-controller for bootstrapping.
+func UseKopsControllerForNodeBootstrap(cluster *kops.Cluster) bool {
+	return kops.CloudProviderID(cluster.Spec.CloudProvider) == kops.CloudProviderAWS && cluster.IsKubernetesGTE("1.19")
+}

pkg/model/context.go

@@ -267,6 +267,11 @@ func (m *KopsModelContext) CloudTags(name string, shared bool) map[string]string
 	return tags
 }
 
+// UseKopsControllerForNodeBootstrap checks if nodeup should use kops-controller to bootstrap.
+func (m *KopsModelContext) UseKopsControllerForNodeBootstrap() bool {
+	return model.UseKopsControllerForNodeBootstrap(m.Cluster)
+}
+
 // UseBootstrapTokens checks if bootstrap tokens are enabled
 func (m *KopsModelContext) UseBootstrapTokens() bool {
 	if m.Cluster.Spec.KubeAPIServer == nil {

pkg/wellknownports/wellknownports.go

@@ -44,6 +44,9 @@ const (
 	// DNSControllerGossipMemberlist is the port where dns-controller listens for the memberlist-backed gossip
 	DNSControllerGossipMemberlist = 3993
 
+	// KopsControllerPort is the port where kops-controller listens.
+	KopsControllerPort = 3992
+
 	// 4001 is etcd main, 4002 is etcd events, 4003 is etcd cilium
 
 	// KubeAPIServerHealthCheck is the port where kube-apiserver-healthcheck listens.

pkg/wellknownusers/wellknownusers.go

@@ -21,7 +21,7 @@ package wellknownusers
 
 const (
 	// Generic is the user id we use for non-privileged containers, where we don't need extra permissions
-	// Used by e.g. dns-controller, kops-controller
+	// Used by e.g. dns-controller
 	Generic = 10001
 
 	// LegacyEtcd is the user id for the etcd user under the legacy provider
@@ -30,6 +30,13 @@ const (
 	// AWSAuthenticator is the user-id for the aws-iam-authenticator (built externally)
 	AWSAuthenticator = 10000
 
+	// KopsControllerID is the user id for kops-controller, which needs some extra permissions e.g. to write local logs
+	// This should match the user in cmd/kops-controller/BUILD.bazel
+	KopsControllerID = 10011
+
+	// KopsControllerName is the username for the kops-controller user
+	KopsControllerName = "kops-controller"
+
 	// KubeApiserverHealthcheckID is the user id for kube-apiserver-healthcheck sidecar
 	// The user needs some extra permissions e.g. to read local secrets
 	// This should match the user in cmd/kube-apiserver-healthcheck/BUILD.bazel

upup/models/bindata.go

@@ -2536,8 +2536,10 @@ spec:
           name: etc-ssl-certs
           readOnly: true
 {{ end }}
-        - mountPath: /etc/kubernetes/kops-controller/
+        - mountPath: /etc/kubernetes/kops-controller/config/
           name: kops-controller-config
+        - mountPath: /etc/kubernetes/kops-controller/pki/
+          name: kops-controller-pki
         command:
 {{ range $arg := KopsControllerArgv }}
         - "{{ $arg }}"
@@ -2565,7 +2567,10 @@ spec:
       - name: kops-controller-config
         configMap:
           name: kops-controller
-
+      - name: kops-controller-pki
+        hostPath:
+          path: /etc/kubernetes/kops-controller/
+          type: Directory
 ---
 
 apiVersion: v1

upup/models/cloudup/resources/addons/kops-controller.addons.k8s.io/k8s-1.16.yaml.template

@@ -53,8 +53,10 @@ spec:
           name: etc-ssl-certs
           readOnly: true
 {{ end }}
-        - mountPath: /etc/kubernetes/kops-controller/
+        - mountPath: /etc/kubernetes/kops-controller/config/
           name: kops-controller-config
+        - mountPath: /etc/kubernetes/kops-controller/pki/
+          name: kops-controller-pki
         command:
 {{ range $arg := KopsControllerArgv }}
         - "{{ $arg }}"
@@ -82,7 +84,10 @@ spec:
       - name: kops-controller-config
         configMap:
           name: kops-controller
-
+      - name: kops-controller-pki
+        hostPath:
+          path: /etc/kubernetes/kops-controller/
+          type: Directory
 ---
 
 apiVersion: v1

upup/pkg/fi/cloudup/template_functions.go

@@ -31,6 +31,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"os"
+	"path"
 	"strconv"
 	"strings"
 	"text/template"
@@ -379,6 +380,15 @@ func (tf *TemplateFunctions) KopsControllerConfig() (string, error) {
 		ConfigBase: cluster.Spec.ConfigBase,
 	}
 
+	if tf.UseKopsControllerForNodeBootstrap() {
+		pkiDir := "/etc/kubernetes/kops-controller/pki"
+		config.Server = &kopscontrollerconfig.ServerOptions{
+			Listen:                fmt.Sprintf(":%d", wellknownports.KopsControllerPort),
+			ServerCertificatePath: path.Join(pkiDir, "kops-controller.crt"),
+			ServerKeyPath:         path.Join(pkiDir, "kops-controller.key"),
+		}
+	}
+
 	// To avoid indentation problems, we marshal as json.  json is a subset of yaml
 	b, err := json.Marshal(config)
 	if err != nil {
@@ -397,7 +407,7 @@ func (tf *TemplateFunctions) KopsControllerArgv() ([]string, error) {
 	// Verbose, but not excessive logging
 	argv = append(argv, "--v=2")
 
-	argv = append(argv, "--conf=/etc/kubernetes/kops-controller/config.yaml")
+	argv = append(argv, "--conf=/etc/kubernetes/kops-controller/config/config.yaml")
 
 	return argv, nil
 }

upup/pkg/fi/nodeup/command.go

@@ -244,6 +244,7 @@ func (c *NodeUpCommand) Run(out io.Writer) error {
 	loader.Builders = append(loader.Builders, &model.KubeSchedulerBuilder{NodeupModelContext: modelContext})
 	loader.Builders = append(loader.Builders, &model.EtcdManagerTLSBuilder{NodeupModelContext: modelContext})
 	loader.Builders = append(loader.Builders, &model.KubeProxyBuilder{NodeupModelContext: modelContext})
+	loader.Builders = append(loader.Builders, &model.KopsControllerBuilder{NodeupModelContext: modelContext})
 
 	loader.Builders = append(loader.Builders, &networking.CommonBuilder{NodeupModelContext: modelContext})
 	loader.Builders = append(loader.Builders, &networking.CalicoBuilder{NodeupModelContext: modelContext})