kops/upup/pkg/fi/cloudup/gcetasks/instancetemplate.go

/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package gcetasks

import (
	"context"
	"fmt"
	"reflect"
	"sort"
	"strconv"
	"strings"
	"time"

	compute "google.golang.org/api/compute/v1"
	"k8s.io/klog/v2"
	"k8s.io/kops/pkg/diff"
	"k8s.io/kops/upup/pkg/fi"
	"k8s.io/kops/upup/pkg/fi/cloudup/gce"
	"k8s.io/kops/upup/pkg/fi/cloudup/terraform"
	"k8s.io/kops/upup/pkg/fi/cloudup/terraformWriter"
)

const (
	// terraform 0.12 with google cloud provider 3.2 will complain if the length of the name_prefix is more than 32
	InstanceTemplateNamePrefixMaxLength = 32

	accessConfigOneToOneNAT = "ONE_TO_ONE_NAT"
)

// InstanceTemplate represents a GCE InstanceTemplate
// +kops:fitask
type InstanceTemplate struct {
	Name *string

	// NamePrefix is used as the prefix for the names; we add a timestamp.  Max = InstanceTemplateNamePrefixMaxLength
	NamePrefix *string

	Lifecycle fi.Lifecycle

	Network              *Network
	Tags                 []string
	Labels               map[string]string
	Preemptible          *bool
	GCPProvisioningModel *string

	BootDiskImage  *string
	BootDiskSizeGB *int64
	BootDiskType   *string

	CanIPForward  *bool
	Subnet        *Subnet
	AliasIPRanges map[string]string

	Scopes          []string
	ServiceAccounts []*ServiceAccount

	Metadata    map[string]fi.Resource
	MachineType *string

	// HasExternalIP is set to true when an external IP is allocated to an instance.
	HasExternalIP *bool

	// ID is the actual name
	ID *string

	GuestAccelerators []AcceleratorConfig
}

var (
	_ fi.CloudupTask   = &InstanceTemplate{}
	_ fi.CompareWithID = &InstanceTemplate{}
)

func (e *InstanceTemplate) CompareWithID() *string {
	return e.ID
}

func (e *InstanceTemplate) Find(c *fi.CloudupContext) (*InstanceTemplate, error) {
	cloud := c.T.Cloud.(gce.GCECloud)

	templates, err := cloud.Compute().InstanceTemplates().List(context.Background(), cloud.Project())
	if err != nil {
		if gce.IsNotFound(err) {
			return nil, nil
		}
		return nil, fmt.Errorf("error listing InstanceTemplates: %v", err)
	}

	expected, err := e.mapToGCE(cloud.Project(), cloud.Region())
	if err != nil {
		return nil, err
	}

	for _, r := range templates {
		if !strings.HasPrefix(r.Name, fi.ValueOf(e.NamePrefix)+"-") {
			continue
		}

		if !matches(expected, r) {
			continue
		}

		actual := &InstanceTemplate{}

		p := r.Properties

		actual.Tags = append(actual.Tags, p.Tags.Items...)
		actual.Labels = p.Labels
		actual.MachineType = fi.PtrTo(lastComponent(p.MachineType))
		actual.CanIPForward = &p.CanIpForward

		bootDiskImage, err := ShortenImageURL(cloud.Project(), p.Disks[0].InitializeParams.SourceImage)
		if err != nil {
			return nil, fmt.Errorf("error parsing source image URL: %v", err)
		}
		actual.BootDiskImage = fi.PtrTo(bootDiskImage)
		actual.BootDiskType = &p.Disks[0].InitializeParams.DiskType
		actual.BootDiskSizeGB = &p.Disks[0].InitializeParams.DiskSizeGb

		if p.Scheduling != nil {
			actual.Preemptible = &p.Scheduling.Preemptible
			actual.GCPProvisioningModel = &p.Scheduling.ProvisioningModel
		}
		if len(p.NetworkInterfaces) != 0 {
			ni := p.NetworkInterfaces[0]
			actual.Network = &Network{Name: fi.PtrTo(lastComponent(ni.Network))}

			if len(ni.AliasIpRanges) != 0 {
				actual.AliasIPRanges = make(map[string]string)
				for _, aliasIPRange := range ni.AliasIpRanges {
					actual.AliasIPRanges[aliasIPRange.SubnetworkRangeName] = aliasIPRange.IpCidrRange
				}
			}

			if ni.Subnetwork != "" {
				actual.Subnet = &Subnet{Name: fi.PtrTo(lastComponent(ni.Subnetwork))}
			}

			acs := ni.AccessConfigs
			if len(acs) > 0 {
				if len(acs) != 1 {
					return nil, fmt.Errorf("unexpected number of access configs in template %q: %d", *actual.Name, len(acs))
				}
				if acs[0].Type != accessConfigOneToOneNAT {
					return nil, fmt.Errorf("unexpected access type in template %q: %s", *actual.Name, acs[0].Type)
				}
				actual.HasExternalIP = fi.PtrTo(true)
			} else {
				actual.HasExternalIP = fi.PtrTo(false)
			}
		}

		for _, serviceAccount := range p.ServiceAccounts {
			for _, scope := range serviceAccount.Scopes {
				actual.Scopes = append(actual.Scopes, scopeToShortForm(scope))
			}
			actual.ServiceAccounts = append(actual.ServiceAccounts, &ServiceAccount{
				Email: &serviceAccount.Email,
			})
		}

		// When we deal with additional disks (local disks), we'll need to map them like this...
		//for i, disk := range p.Disks {
		//	if i == 0 {
		//		source := disk.Source
		//
		//		// TODO: Parse source URL instead of assuming same project/zone?
		//		name := lastComponent(source)
		//		d, err := cloud.Compute.Disks.Get(cloud.Project, *e.Zone, name).Do()
		//		if err != nil {
		//			if gce.IsNotFound(err) {
		//				return nil, fmt.Errorf("disk not found %q: %v", source, err)
		//			}
		//			return nil, fmt.Errorf("error querying for disk %q: %v", source, err)
		//		} else {
		//			imageURL, err := gce.ParseGoogleCloudURL(d.SourceImage)
		//			if err != nil {
		//				return nil, fmt.Errorf("unable to parse image URL: %q", d.SourceImage)
		//			}
		//			actual.Image = fi.PtrTo(imageURL.Project + "/" + imageURL.Name)
		//		}
		//	}
		//}

		if p.Metadata != nil {
			actual.Metadata = make(map[string]fi.Resource)
			for _, meta := range p.Metadata.Items {
				actual.Metadata[meta.Key] = fi.NewStringResource(fi.ValueOf(meta.Value))
			}
		}

		// Prevent spurious changes
		actual.Name = e.Name
		actual.NamePrefix = e.NamePrefix

		actual.ID = &r.Name
		if e.ID == nil {
			e.ID = actual.ID
		}

		// System fields
		actual.Lifecycle = e.Lifecycle

		actual.GuestAccelerators = []AcceleratorConfig{}
		for _, accelerator := range p.GuestAccelerators {
			actual.GuestAccelerators = append(actual.GuestAccelerators, AcceleratorConfig{
				AcceleratorCount: accelerator.AcceleratorCount,
				AcceleratorType:  accelerator.AcceleratorType,
			})
		}

		return actual, nil
	}

	return nil, nil
}

func (e *InstanceTemplate) Run(c *fi.CloudupContext) error {
	return fi.CloudupDefaultDeltaRunMethod(e, c)
}

func (_ *InstanceTemplate) CheckChanges(a, e, changes *InstanceTemplate) error {
	if fi.ValueOf(e.BootDiskImage) == "" {
		return fi.RequiredField("BootDiskImage")
	}
	if fi.ValueOf(e.MachineType) == "" {
		return fi.RequiredField("MachineType")
	}
	return nil
}

func (e *InstanceTemplate) mapToGCE(project string, region string) (*compute.InstanceTemplate, error) {
	// TODO: This is similar to Instance...
	var scheduling *compute.Scheduling

	if fi.ValueOf(e.Preemptible) {
		scheduling = &compute.Scheduling{
			AutomaticRestart:  fi.PtrTo(false),
			OnHostMaintenance: "TERMINATE",
			ProvisioningModel: fi.ValueOf(e.GCPProvisioningModel),
			Preemptible:       true,
		}
	} else {
		scheduling = &compute.Scheduling{
			AutomaticRestart: fi.PtrTo(true),
			// TODO: Migrate or terminate?
			OnHostMaintenance: "MIGRATE",
			ProvisioningModel: "STANDARD",
			Preemptible:       false,
		}
	}

	if len(e.GuestAccelerators) > 0 {
		// Instances with accelerators cannot be migrated.
		scheduling.OnHostMaintenance = "TERMINATE"
	}

	var disks []*compute.AttachedDisk
	disks = append(disks, &compute.AttachedDisk{
		Kind: "compute#attachedDisk",
		InitializeParams: &compute.AttachedDiskInitializeParams{
			SourceImage: BuildImageURL(project, *e.BootDiskImage),
			DiskSizeGb:  *e.BootDiskSizeGB,
			DiskType:    *e.BootDiskType,
		},
		Boot:       true,
		DeviceName: "persistent-disks-0",
		Index:      0,
		AutoDelete: true,
		Mode:       "READ_WRITE",
		Type:       "PERSISTENT",
	})

	var tags *compute.Tags
	if e.Tags != nil {
		tags = &compute.Tags{
			Items: e.Tags,
		}
	}

	var networkInterfaces []*compute.NetworkInterface

	networkProject := project
	if e.Network.Project != nil {
		networkProject = *e.Network.Project
	}

	ni := &compute.NetworkInterface{
		Kind:    "compute#networkInterface",
		Network: e.Network.URL(networkProject),
	}
	if fi.ValueOf(e.HasExternalIP) {
		ni.AccessConfigs = []*compute.AccessConfig{
			{
				Kind:        "compute#accessConfig",
				Type:        accessConfigOneToOneNAT,
				NetworkTier: "PREMIUM",
			},
		}
	}

	if e.Subnet != nil {
		ni.Subnetwork = e.Subnet.URL(networkProject, region)
	}
	if e.AliasIPRanges != nil {
		for k, v := range e.AliasIPRanges {
			ni.AliasIpRanges = append(ni.AliasIpRanges, &compute.AliasIpRange{
				SubnetworkRangeName: k,
				IpCidrRange:         v,
			})
		}
	}
	networkInterfaces = append(networkInterfaces, ni)

	scopes := make([]string, 0)
	if e.Scopes != nil {
		for _, s := range e.Scopes {
			s = scopeToLongForm(s)
			scopes = append(scopes, s)
		}
	}

	var serviceAccounts []*compute.ServiceAccount
	for _, sa := range e.ServiceAccounts {
		serviceAccounts = append(serviceAccounts, &compute.ServiceAccount{
			Email:  fi.ValueOf(sa.Email),
			Scopes: scopes,
		})
	}

	var metadataItems []*compute.MetadataItems
	for key, r := range e.Metadata {
		v, err := fi.ResourceAsString(r)
		if err != nil {
			return nil, fmt.Errorf("error rendering InstanceTemplate metadata %q: %v", key, err)
		}
		metadataItems = append(metadataItems, &compute.MetadataItems{
			Key:   key,
			Value: fi.PtrTo(v),
		})
	}

	var accelerators []*compute.AcceleratorConfig
	if len(e.GuestAccelerators) > 0 {
		accelerators = []*compute.AcceleratorConfig{}
		for _, accelerator := range e.GuestAccelerators {
			accelerators = append(accelerators, &compute.AcceleratorConfig{
				AcceleratorCount: accelerator.AcceleratorCount,
				AcceleratorType:  accelerator.AcceleratorType,
			})
		}
	}

	i := &compute.InstanceTemplate{
		Kind: "compute#instanceTemplate",
		Properties: &compute.InstanceProperties{
			CanIpForward: *e.CanIPForward,

			Disks: disks,

			GuestAccelerators: accelerators,

			MachineType: *e.MachineType,

			Metadata: &compute.Metadata{
				Kind:  "compute#metadata",
				Items: metadataItems,
			},

			NetworkInterfaces: networkInterfaces,

			Scheduling: scheduling,

			ServiceAccounts: serviceAccounts,

			Labels: e.Labels,
			Tags:   tags,
		},
	}

	return i, nil
}

type ByKey []*compute.MetadataItems

func (a ByKey) Len() int           { return len(a) }
func (a ByKey) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a ByKey) Less(i, j int) bool { return a[i].Key < a[j].Key }

func matches(l, r *compute.InstanceTemplate) bool {
	normalizeInstanceProperties := func(v *compute.InstanceProperties) *compute.InstanceProperties {
		c := *v
		if c.Metadata != nil {
			cm := *c.Metadata
			c.Metadata = &cm
			c.Metadata.Fingerprint = ""
			sort.Sort(ByKey(c.Metadata.Items))
		}
		// Ignore output fields
		for _, ni := range c.NetworkInterfaces {
			ni.Name = ""
		}
		return &c
	}
	normalize := func(v *compute.InstanceTemplate) *compute.InstanceTemplate {
		c := *v
		c.SelfLink = ""
		c.CreationTimestamp = ""
		c.Id = 0
		c.Name = ""
		c.Properties = normalizeInstanceProperties(c.Properties)
		return &c
	}
	normalizedL := normalize(l)
	normalizedR := normalize(r)

	if !reflect.DeepEqual(normalizedL, normalizedR) {
		if klog.V(10).Enabled() {
			ls := fi.DebugAsJsonStringIndent(normalizedL)
			rs := fi.DebugAsJsonStringIndent(normalizedR)
			klog.V(10).Infof("Not equal")
			klog.V(10).Infof(diff.FormatDiff(ls, rs))
		}
		return false
	}

	return true
}

func (e *InstanceTemplate) URL(project string) (string, error) {
	if e.ID == nil {
		return "", fmt.Errorf("InstanceTemplate not yet built; ID is not yet known")
	}
	return fmt.Sprintf("https://www.googleapis.com/compute/v1/projects/%s/global/instanceTemplates/%s", project, *e.ID), nil
}

func (_ *InstanceTemplate) RenderGCE(t *gce.GCEAPITarget, a, e, changes *InstanceTemplate) error {
	project := t.Cloud.Project()
	region := t.Cloud.Region()

	i, err := e.mapToGCE(project, region)
	if err != nil {
		return err
	}

	if a == nil {
		klog.V(4).Infof("Creating InstanceTemplate %v", i)

		name := fi.ValueOf(e.NamePrefix) + "-" + strconv.FormatInt(time.Now().Unix(), 10)
		e.ID = &name
		i.Name = name

		op, err := t.Cloud.Compute().InstanceTemplates().Insert(t.Cloud.Project(), i)
		if err != nil {
			return fmt.Errorf("error creating InstanceTemplate: %v", err)
		}

		if err := t.Cloud.WaitForOp(op); err != nil {
			return fmt.Errorf("error creating InstanceTemplate: %v", err)
		}
	} else {
		return fmt.Errorf("Cannot apply changes to InstanceTemplate: %v", changes)
	}

	return nil
}

type terraformInstanceTemplate struct {
	NamePrefix            string                                   `cty:"name_prefix"`
	CanIPForward          bool                                     `cty:"can_ip_forward"`
	MachineType           string                                   `cty:"machine_type"`
	ServiceAccounts       []*terraformTemplateServiceAccount       `cty:"service_account"`
	Scheduling            *terraformScheduling                     `cty:"scheduling"`
	Disks                 []*terraformInstanceTemplateAttachedDisk `cty:"disk"`
	Labels                map[string]string                        `cty:"labels"`
	NetworkInterfaces     []*terraformNetworkInterface             `cty:"network_interface"`
	Metadata              map[string]*terraformWriter.Literal      `cty:"metadata"`
	MetadataStartupScript *terraformWriter.Literal                 `cty:"metadata_startup_script"`
	Tags                  []string                                 `cty:"tags"`
	GuestAccelerator      []*terraformGuestAccelerator             `cty:"guest_accelerator"`
}

type terraformTemplateServiceAccount struct {
	Email  *terraformWriter.Literal `cty:"email"`
	Scopes []string                 `cty:"scopes"`
}

type terraformScheduling struct {
	AutomaticRestart  bool   `cty:"automatic_restart"`
	OnHostMaintenance string `cty:"on_host_maintenance"`
	Preemptible       bool   `cty:"preemptible"`
	ProvisioningModel string `cty:"provisioning_model"`
}

type terraformInstanceTemplateAttachedDisk struct {
	AutoDelete bool   `cty:"auto_delete"`
	DeviceName string `cty:"device_name"`

	// scratch vs persistent
	Type        string `cty:"type"`
	Boot        bool   `cty:"boot"`
	DiskName    string `cty:"disk_name"`
	SourceImage string `cty:"source_image"`
	Source      string `cty:"source"`
	Interface   string `cty:"interface"`
	Mode        string `cty:"mode"`
	DiskType    string `cty:"disk_type"`
	DiskSizeGB  int64  `cty:"disk_size_gb"`
}

type terraformNetworkInterface struct {
	Network      *terraformWriter.Literal `cty:"network"`
	Subnetwork   *terraformWriter.Literal `cty:"subnetwork"`
	AccessConfig []*terraformAccessConfig `cty:"access_config"`
}

type terraformAccessConfig struct {
	NatIP *terraformWriter.Literal `cty:"nat_ip"`
}

type terraformGuestAccelerator struct {
	Type  string `cty:"type"`
	Count int64  `cty:"count"`
}

func addNetworks(network *Network, subnet *Subnet, networkInterfaces []*compute.NetworkInterface) []*terraformNetworkInterface {
	ni := make([]*terraformNetworkInterface, 0)
	for _, g := range networkInterfaces {
		tf := &terraformNetworkInterface{}
		if network != nil {
			tf.Network = network.TerraformLink()
		}
		if subnet != nil {
			tf.Subnetwork = subnet.TerraformLink()
		}
		for _, gac := range g.AccessConfigs {
			tac := &terraformAccessConfig{}
			natIP := gac.NatIP
			if natIP != "" {
				tac.NatIP = terraformWriter.LiteralFromStringValue(natIP)
			}

			tf.AccessConfig = append(tf.AccessConfig, tac)
		}

		ni = append(ni, tf)
	}
	return ni
}

func addMetadata(target *terraform.TerraformTarget, name string, metadata *compute.Metadata) (map[string]*terraformWriter.Literal, error) {
	if metadata == nil {
		return nil, nil
	}
	m := make(map[string]*terraformWriter.Literal)
	for _, g := range metadata.Items {
		val := fi.ValueOf(g.Value)
		if strings.Contains(val, "\n") {
			tfResource, err := target.AddFileBytes("google_compute_instance_template", name, "metadata_"+g.Key, []byte(val), false)
			if err != nil {
				return nil, err
			}
			m[g.Key] = tfResource
		} else {
			m[g.Key] = terraformWriter.LiteralFromStringValue(val)
		}
	}
	return m, nil
}

func mapServiceAccountsToTerraform(serviceAccounts []*ServiceAccount, saScopes []string) []*terraformTemplateServiceAccount {
	var scopes []string
	for _, s := range saScopes {
		s = scopeToLongForm(s)
		scopes = append(scopes, s)
	}
	// Note that GCE currently only allows one service account per VM,
	// but the model in both the API and terraform allows more.
	var out []*terraformTemplateServiceAccount
	for _, serviceAccount := range serviceAccounts {
		tsa := &terraformTemplateServiceAccount{
			Email:  serviceAccount.TerraformLink(),
			Scopes: scopes,
		}
		out = append(out, tsa)
	}
	return out
}

func (_ *InstanceTemplate) RenderTerraform(t *terraform.TerraformTarget, a, e, changes *InstanceTemplate) error {
	project := t.Project

	i, err := e.mapToGCE(project, t.Cloud.Region())
	if err != nil {
		return err
	}

	name := fi.ValueOf(e.Name)

	tf := &terraformInstanceTemplate{
		NamePrefix: fi.ValueOf(e.NamePrefix) + "-",
	}

	tf.CanIPForward = i.Properties.CanIpForward
	tf.MachineType = lastComponent(i.Properties.MachineType)
	tf.Labels = i.Properties.Labels
	tf.Tags = i.Properties.Tags.Items

	tf.ServiceAccounts = mapServiceAccountsToTerraform(e.ServiceAccounts, e.Scopes)

	for _, d := range i.Properties.Disks {
		tfd := &terraformInstanceTemplateAttachedDisk{
			AutoDelete:  d.AutoDelete,
			Boot:        d.Boot,
			DeviceName:  d.DeviceName,
			DiskName:    d.InitializeParams.DiskName,
			SourceImage: d.InitializeParams.SourceImage,
			Source:      d.Source,
			Interface:   d.Interface,
			Mode:        d.Mode,
			DiskType:    d.InitializeParams.DiskType,
			DiskSizeGB:  d.InitializeParams.DiskSizeGb,
			Type:        d.Type,
		}
		tf.Disks = append(tf.Disks, tfd)
	}

	tf.NetworkInterfaces = addNetworks(e.Network, e.Subnet, i.Properties.NetworkInterfaces)

	metadata, err := addMetadata(t, name, i.Properties.Metadata)
	if err != nil {
		return err
	}
	tf.Metadata = metadata

	if i.Properties.Scheduling != nil {
		tf.Scheduling = &terraformScheduling{
			AutomaticRestart:  fi.ValueOf(i.Properties.Scheduling.AutomaticRestart),
			OnHostMaintenance: i.Properties.Scheduling.OnHostMaintenance,
			Preemptible:       i.Properties.Scheduling.Preemptible,
			ProvisioningModel: i.Properties.Scheduling.ProvisioningModel,
		}
	}

	if len(i.Properties.GuestAccelerators) > 0 {
		tf.GuestAccelerator = []*terraformGuestAccelerator{}
		for _, accelerator := range i.Properties.GuestAccelerators {
			tf.GuestAccelerator = append(tf.GuestAccelerator, &terraformGuestAccelerator{
				Count: accelerator.AcceleratorCount,
				Type:  accelerator.AcceleratorType,
			})
		}
	}

	return t.RenderResource("google_compute_instance_template", name, tf)
}

func (i *InstanceTemplate) TerraformLink() *terraformWriter.Literal {
	return terraformWriter.LiteralSelfLink("google_compute_instance_template", *i.Name)
}