kops/nodeup/pkg/model/protokube.go

/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package model

import (
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"strings"

	"k8s.io/klog/v2"
	"k8s.io/kops/pkg/apis/kops"
	"k8s.io/kops/pkg/flagbuilder"
	"k8s.io/kops/pkg/rbac"
	"k8s.io/kops/pkg/systemd"
	"k8s.io/kops/upup/pkg/fi"
	"k8s.io/kops/upup/pkg/fi/cloudup/scaleway"
	"k8s.io/kops/upup/pkg/fi/nodeup/nodetasks"
	"k8s.io/kops/util/pkg/distributions"
	"k8s.io/kops/util/pkg/proxy"
)

// ProtokubeBuilder configures protokube
type ProtokubeBuilder struct {
	*NodeupModelContext
}

var _ fi.NodeupModelBuilder = &ProtokubeBuilder{}

// Build is responsible for generating the options for protokube
func (t *ProtokubeBuilder) Build(c *fi.NodeupModelBuilderContext) error {
	// check is not a master and we are not using gossip (https://github.com/kubernetes/kops/pull/3091)
	if !t.IsMaster && !t.UsesLegacyGossip() {
		klog.V(2).Infof("skipping the provisioning of protokube on the nodes")
		return nil
	}

	{
		name, res, err := t.Assets.FindMatch(regexp.MustCompile("protokube$"))
		if err != nil {
			return err
		}

		c.AddTask(&nodetasks.File{
			Path:     filepath.Join("/opt/kops/bin", name),
			Contents: res,
			Type:     nodetasks.FileType_File,
			Mode:     fi.PtrTo("0755"),
		})
	}

	{
		name, res, err := t.Assets.FindMatch(regexp.MustCompile("channels$"))
		if err != nil {
			return err
		}

		c.AddTask(&nodetasks.File{
			Path:     filepath.Join("/opt/kops/bin", name),
			Contents: res,
			Type:     nodetasks.FileType_File,
			Mode:     fi.PtrTo("0755"),
		})
	}

	if t.IsMaster {
		name := nodetasks.PKIXName{
			CommonName:   "kops",
			Organization: []string{rbac.SystemPrivilegedGroup},
		}
		kubeconfig := t.BuildIssuedKubeconfig("kops", name, c)

		c.AddTask(&nodetasks.File{
			Path:     "/var/lib/kops/kubeconfig",
			Contents: kubeconfig,
			Type:     nodetasks.FileType_File,
			Mode:     s("0400"),
		})
	}

	envFile, err := t.buildEnvFile()
	if err != nil {
		return err
	}
	c.AddTask(envFile)

	service, err := t.buildSystemdService()
	if err != nil {
		return err
	}
	c.AddTask(service)

	return nil
}

// buildSystemdService generates the manifest for the protokube service
func (t *ProtokubeBuilder) buildSystemdService() (*nodetasks.Service, error) {
	protokubeFlags, err := t.ProtokubeFlags()
	if err != nil {
		return nil, err
	}
	protokubeRunArgs, err := flagbuilder.BuildFlags(protokubeFlags)
	if err != nil {
		return nil, err
	}

	manifest := &systemd.Manifest{}
	manifest.Set("Unit", "Description", "Kubernetes Protokube Service")
	manifest.Set("Unit", "Documentation", "https://kops.sigs.k8s.io")

	manifest.Set("Service", "ExecStart", "/opt/kops/bin/protokube"+" "+protokubeRunArgs)
	manifest.Set("Service", "EnvironmentFile", "/etc/sysconfig/protokube")
	manifest.Set("Service", "Restart", "always")
	manifest.Set("Service", "RestartSec", "3s")
	manifest.Set("Service", "StartLimitInterval", "0")
	manifest.Set("Install", "WantedBy", "multi-user.target")

	manifestString := manifest.Render()
	klog.V(8).Infof("Built service manifest %q\n%s", "protokube", manifestString)

	service := &nodetasks.Service{
		Name:       "protokube.service",
		Definition: s(manifestString),
	}

	service.InitDefaults()

	return service, nil
}

// ProtokubeFlags are the flags for protokube
type ProtokubeFlags struct {
	ClusterID         *string  `json:"clusterID,omitempty" flag:"cluster-id"`
	Channels          []string `json:"channels,omitempty" flag:"channels"`
	Cloud             *string  `json:"cloud,omitempty" flag:"cloud"`
	Containerized     *bool    `json:"containerized,omitempty" flag:"containerized"`
	DNSInternalSuffix *string  `json:"dnsInternalSuffix,omitempty" flag:"dns-internal-suffix"`
	Gossip            *bool    `json:"gossip,omitempty" flag:"gossip"`
	LogLevel          *int32   `json:"logLevel,omitempty" flag:"v"`
	Master            *bool    `json:"master,omitempty" flag:"master"`
	Zone              []string `json:"zone,omitempty" flag:"zone"`

	// BootstrapMasterNodeLabels applies the critical node-role labels to our node,
	// which lets us bring up the controllers that can only run on masters, which are then
	// responsible for node labels.  The node is specified by NodeName
	BootstrapMasterNodeLabels bool `json:"bootstrapMasterNodeLabels,omitempty" flag:"bootstrap-master-node-labels"`

	// NodeName is the name of the node as will be created in kubernetes.  Primarily used by BootstrapMasterNodeLabels.
	NodeName string `json:"nodeName,omitempty" flag:"node-name"`

	GossipProtocol *string `json:"gossip-protocol" flag:"gossip-protocol"`
	GossipListen   *string `json:"gossip-listen" flag:"gossip-listen"`
	GossipSecret   *string `json:"gossip-secret" flag:"gossip-secret"`

	GossipProtocolSecondary *string `json:"gossip-protocol-secondary" flag:"gossip-protocol-secondary" flag-include-empty:"true"`
	GossipListenSecondary   *string `json:"gossip-listen-secondary" flag:"gossip-listen-secondary"`
	GossipSecretSecondary   *string `json:"gossip-secret-secondary" flag:"gossip-secret-secondary"`
}

// ProtokubeFlags is responsible for building the command line flags for protokube
func (t *ProtokubeBuilder) ProtokubeFlags() (*ProtokubeFlags, error) {
	f := &ProtokubeFlags{
		Channels:      t.NodeupConfig.Channels,
		Cloud:         fi.PtrTo(string(t.CloudProvider())),
		Containerized: fi.PtrTo(false),
		LogLevel:      fi.PtrTo(int32(4)),
		Master:        b(t.IsMaster),
	}

	f.ClusterID = fi.PtrTo(t.NodeupConfig.ClusterName)

	zone := t.NodeupConfig.DNSZone
	if zone != "" {
		if strings.Contains(zone, ".") {
			// match by name
			f.Zone = append(f.Zone, zone)
		} else {
			// match by id
			f.Zone = append(f.Zone, "*/"+zone)
		}
	} else {
		klog.Warningf("DNSZone not specified; protokube won't be able to update DNS")
		// @TODO: Should we permit wildcard updates if zone is not specified?
		// argv = append(argv, "--zone=*/*")
	}

	if t.UsesLegacyGossip() {
		klog.Warningf("using (legacy) gossip DNS")
		f.Gossip = fi.PtrTo(true)
		if t.NodeupConfig.GossipConfig != nil {
			f.GossipProtocol = t.NodeupConfig.GossipConfig.Protocol
			f.GossipListen = t.NodeupConfig.GossipConfig.Listen
			f.GossipSecret = t.NodeupConfig.GossipConfig.Secret

			if t.NodeupConfig.GossipConfig.Secondary != nil {
				f.GossipProtocolSecondary = t.NodeupConfig.GossipConfig.Secondary.Protocol
				f.GossipListenSecondary = t.NodeupConfig.GossipConfig.Secondary.Listen
				f.GossipSecretSecondary = t.NodeupConfig.GossipConfig.Secondary.Secret
			}
		}

		// @TODO: This is hacky, but we want it so that we can have a different internal & external name
		internalSuffix := t.APIInternalName()
		internalSuffix = strings.TrimPrefix(internalSuffix, "api.")
		f.DNSInternalSuffix = fi.PtrTo(internalSuffix)
	}

	if f.DNSInternalSuffix == nil {
		f.DNSInternalSuffix = fi.PtrTo(".internal." + t.NodeupConfig.ClusterName)
	}

	f.BootstrapMasterNodeLabels = true

	nodeName, err := t.NodeName()
	if err != nil {
		return nil, fmt.Errorf("error getting NodeName: %v", err)
	}
	f.NodeName = nodeName

	return f, nil
}

func (t *ProtokubeBuilder) buildEnvFile() (*nodetasks.File, error) {
	envVars := make(map[string]string)

	envVars["KUBECONFIG"] = "/var/lib/kops/kubeconfig"

	// Pass in gossip dns connection limit
	if os.Getenv("GOSSIP_DNS_CONN_LIMIT") != "" {
		envVars["GOSSIP_DNS_CONN_LIMIT"] = os.Getenv("GOSSIP_DNS_CONN_LIMIT")
	}

	// Pass in required credentials when using user-defined s3 endpoint
	if os.Getenv("AWS_REGION") != "" {
		envVars["AWS_REGION"] = os.Getenv("AWS_REGION")
	}

	if os.Getenv("S3_ENDPOINT") != "" {
		envVars["S3_ENDPOINT"] = os.Getenv("S3_ENDPOINT")
		envVars["S3_REGION"] = os.Getenv("S3_REGION")
		envVars["S3_ACCESS_KEY_ID"] = os.Getenv("S3_ACCESS_KEY_ID")
		envVars["S3_SECRET_ACCESS_KEY"] = os.Getenv("S3_SECRET_ACCESS_KEY")
	}

	if os.Getenv("OS_AUTH_URL") != "" {
		for _, envVar := range []string{
			"OS_TENANT_ID", "OS_TENANT_NAME", "OS_PROJECT_ID", "OS_PROJECT_NAME",
			"OS_PROJECT_DOMAIN_NAME", "OS_PROJECT_DOMAIN_ID",
			"OS_DOMAIN_NAME", "OS_DOMAIN_ID",
			"OS_USERNAME",
			"OS_PASSWORD",
			"OS_AUTH_URL",
			"OS_REGION_NAME",
			"OS_APPLICATION_CREDENTIAL_ID",
			"OS_APPLICATION_CREDENTIAL_SECRET",
		} {
			envVars[envVar] = os.Getenv(envVar)
		}
	}

	if t.CloudProvider() == kops.CloudProviderDO && os.Getenv("DIGITALOCEAN_ACCESS_TOKEN") != "" {
		envVars["DIGITALOCEAN_ACCESS_TOKEN"] = os.Getenv("DIGITALOCEAN_ACCESS_TOKEN")
	}

	if os.Getenv("HCLOUD_TOKEN") != "" {
		envVars["HCLOUD_TOKEN"] = os.Getenv("HCLOUD_TOKEN")
	}

	if os.Getenv("OSS_REGION") != "" {
		envVars["OSS_REGION"] = os.Getenv("OSS_REGION")
	}

	if os.Getenv("AZURE_STORAGE_ACCOUNT") != "" {
		envVars["AZURE_STORAGE_ACCOUNT"] = os.Getenv("AZURE_STORAGE_ACCOUNT")
	}

	if t.CloudProvider() == kops.CloudProviderScaleway {
		if os.Getenv("SCW_PROFILE") != "" || os.Getenv("SCW_SECRET_KEY") != "" {
			profile, err := scaleway.CreateValidScalewayProfile()
			if err != nil {
				return nil, err
			}
			envVars["SCW_ACCESS_KEY"] = fi.ValueOf(profile.AccessKey)
			envVars["SCW_SECRET_KEY"] = fi.ValueOf(profile.SecretKey)
			envVars["SCW_DEFAULT_PROJECT_ID"] = fi.ValueOf(profile.DefaultProjectID)
		}
	}

	for _, envVar := range proxy.GetProxyEnvVars(t.NodeupConfig.Networking.EgressProxy) {
		envVars[envVar.Name] = envVar.Value
	}

	switch t.Distribution {
	case distributions.DistributionFlatcar:
		envVars["PATH"] = fmt.Sprintf("/opt/kops/bin:%v", os.Getenv("PATH"))
	}

	sysconfig := ""
	for key, value := range envVars {
		sysconfig += key + "=" + value + "\n"
	}

	task := &nodetasks.File{
		Path:     "/etc/sysconfig/protokube",
		Contents: fi.NewStringResource(sysconfig),
		Type:     nodetasks.FileType_File,
	}

	return task, nil
}