Merge pull request #8878 from rifelpet/hcl2-gce

Fix Terraform 0.12 support for GCE

hack/verify-terraform.sh

@@ -41,9 +41,7 @@ done 3< <(find "${KOPS_ROOT}/tests/integration/update_cluster" -type d -maxdepth
 
 if [ $RC != 0 ]; then
   echo -e "\nTerraform validation failed\n"
-  # TODO(rifelpet): make this script blocking in PRs by exiting non-zero on failure
+  exit $RC
-  # exit $RC
-  exit 0
 else
   echo -e "\nTerraform validation succeeded\n"
 fi

tests/integration/update_cluster/ha_gce/kubernetes.tf

@@ -332,19 +332,163 @@ resource "google_compute_instance_group_manager" "c-nodes-ha-gce-example-com" {
 }
 
 resource "google_compute_instance_template" "master-us-test1-a-ha-gce-example-com" {
+  can_ip_forward = true
+  disk {
+    auto_delete  = true
+    boot         = true
+    device_name  = "persistent-disks-0"
+    disk_name    = ""
+    disk_size_gb = 64
+    disk_type    = "pd-standard"
+    interface    = ""
+    mode         = "READ_WRITE"
+    source       = ""
+    source_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-57-9202-64-0"
+    type         = "PERSISTENT"
+  }
+  machine_type = "n1-standard-1"
+  metadata = {
+    "cluster-name"                    = file("${path.module}/data/google_compute_instance_template_master-us-test1-a-ha-gce-example-com_metadata_cluster-name")
+    "kops-k8s-io-instance-group-name" = file("${path.module}/data/google_compute_instance_template_master-us-test1-a-ha-gce-example-com_metadata_kops-k8s-io-instance-group-name")
+    "ssh-keys"                        = file("${path.module}/data/google_compute_instance_template_master-us-test1-a-ha-gce-example-com_metadata_ssh-keys")
+    "startup-script"                  = file("${path.module}/data/google_compute_instance_template_master-us-test1-a-ha-gce-example-com_metadata_startup-script")
+  }
   name_prefix = "master-us-test1-a-ha-gce--ke5ah6-"
+  network_interface {
+    access_config {
+    }
+    network = google_compute_network.default.name
+  }
+  scheduling {
+    automatic_restart   = true
+    on_host_maintenance = "MIGRATE"
+    preemptible         = false
+  }
+  service_account {
+    email  = "default"
+    scopes = ["https://www.googleapis.com/auth/compute", "https://www.googleapis.com/auth/monitoring", "https://www.googleapis.com/auth/logging.write", "https://www.googleapis.com/auth/devstorage.read_write", "https://www.googleapis.com/auth/ndev.clouddns.readwrite"]
+  }
+  tags = ["ha-gce-example-com-k8s-io-role-master"]
 }
 
 resource "google_compute_instance_template" "master-us-test1-b-ha-gce-example-com" {
+  can_ip_forward = true
+  disk {
+    auto_delete  = true
+    boot         = true
+    device_name  = "persistent-disks-0"
+    disk_name    = ""
+    disk_size_gb = 64
+    disk_type    = "pd-standard"
+    interface    = ""
+    mode         = "READ_WRITE"
+    source       = ""
+    source_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-57-9202-64-0"
+    type         = "PERSISTENT"
+  }
+  machine_type = "n1-standard-1"
+  metadata = {
+    "cluster-name"                    = file("${path.module}/data/google_compute_instance_template_master-us-test1-b-ha-gce-example-com_metadata_cluster-name")
+    "kops-k8s-io-instance-group-name" = file("${path.module}/data/google_compute_instance_template_master-us-test1-b-ha-gce-example-com_metadata_kops-k8s-io-instance-group-name")
+    "ssh-keys"                        = file("${path.module}/data/google_compute_instance_template_master-us-test1-b-ha-gce-example-com_metadata_ssh-keys")
+    "startup-script"                  = file("${path.module}/data/google_compute_instance_template_master-us-test1-b-ha-gce-example-com_metadata_startup-script")
+  }
   name_prefix = "master-us-test1-b-ha-gce--c8u7qq-"
+  network_interface {
+    access_config {
+    }
+    network = google_compute_network.default.name
+  }
+  scheduling {
+    automatic_restart   = true
+    on_host_maintenance = "MIGRATE"
+    preemptible         = false
+  }
+  service_account {
+    email  = "default"
+    scopes = ["https://www.googleapis.com/auth/compute", "https://www.googleapis.com/auth/monitoring", "https://www.googleapis.com/auth/logging.write", "https://www.googleapis.com/auth/devstorage.read_write", "https://www.googleapis.com/auth/ndev.clouddns.readwrite"]
+  }
+  tags = ["ha-gce-example-com-k8s-io-role-master"]
 }
 
 resource "google_compute_instance_template" "master-us-test1-c-ha-gce-example-com" {
+  can_ip_forward = true
+  disk {
+    auto_delete  = true
+    boot         = true
+    device_name  = "persistent-disks-0"
+    disk_name    = ""
+    disk_size_gb = 64
+    disk_type    = "pd-standard"
+    interface    = ""
+    mode         = "READ_WRITE"
+    source       = ""
+    source_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-57-9202-64-0"
+    type         = "PERSISTENT"
+  }
+  machine_type = "n1-standard-1"
+  metadata = {
+    "cluster-name"                    = file("${path.module}/data/google_compute_instance_template_master-us-test1-c-ha-gce-example-com_metadata_cluster-name")
+    "kops-k8s-io-instance-group-name" = file("${path.module}/data/google_compute_instance_template_master-us-test1-c-ha-gce-example-com_metadata_kops-k8s-io-instance-group-name")
+    "ssh-keys"                        = file("${path.module}/data/google_compute_instance_template_master-us-test1-c-ha-gce-example-com_metadata_ssh-keys")
+    "startup-script"                  = file("${path.module}/data/google_compute_instance_template_master-us-test1-c-ha-gce-example-com_metadata_startup-script")
+  }
   name_prefix = "master-us-test1-c-ha-gce--3unp7l-"
+  network_interface {
+    access_config {
+    }
+    network = google_compute_network.default.name
+  }
+  scheduling {
+    automatic_restart   = true
+    on_host_maintenance = "MIGRATE"
+    preemptible         = false
+  }
+  service_account {
+    email  = "default"
+    scopes = ["https://www.googleapis.com/auth/compute", "https://www.googleapis.com/auth/monitoring", "https://www.googleapis.com/auth/logging.write", "https://www.googleapis.com/auth/devstorage.read_write", "https://www.googleapis.com/auth/ndev.clouddns.readwrite"]
+  }
+  tags = ["ha-gce-example-com-k8s-io-role-master"]
 }
 
 resource "google_compute_instance_template" "nodes-ha-gce-example-com" {
+  can_ip_forward = true
+  disk {
+    auto_delete  = true
+    boot         = true
+    device_name  = "persistent-disks-0"
+    disk_name    = ""
+    disk_size_gb = 128
+    disk_type    = "pd-standard"
+    interface    = ""
+    mode         = "READ_WRITE"
+    source       = ""
+    source_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-57-9202-64-0"
+    type         = "PERSISTENT"
+  }
+  machine_type = "n1-standard-2"
+  metadata = {
+    "cluster-name"                    = file("${path.module}/data/google_compute_instance_template_nodes-ha-gce-example-com_metadata_cluster-name")
+    "kops-k8s-io-instance-group-name" = file("${path.module}/data/google_compute_instance_template_nodes-ha-gce-example-com_metadata_kops-k8s-io-instance-group-name")
+    "ssh-keys"                        = file("${path.module}/data/google_compute_instance_template_nodes-ha-gce-example-com_metadata_ssh-keys")
+    "startup-script"                  = file("${path.module}/data/google_compute_instance_template_nodes-ha-gce-example-com_metadata_startup-script")
+  }
   name_prefix = "nodes-ha-gce-example-com-"
+  network_interface {
+    access_config {
+    }
+    network = google_compute_network.default.name
+  }
+  scheduling {
+    automatic_restart   = true
+    on_host_maintenance = "MIGRATE"
+    preemptible         = false
+  }
+  service_account {
+    email  = "default"
+    scopes = ["https://www.googleapis.com/auth/compute", "https://www.googleapis.com/auth/monitoring", "https://www.googleapis.com/auth/logging.write", "https://www.googleapis.com/auth/devstorage.read_only"]
+  }
+  tags = ["ha-gce-example-com-k8s-io-role-node"]
 }
 
 resource "google_compute_network" "default" {

tests/integration/update_cluster/minimal_gce/kubernetes.tf

@@ -244,11 +244,83 @@ resource "google_compute_instance_group_manager" "a-nodes-minimal-gce-example-co
 }
 
 resource "google_compute_instance_template" "master-us-test1-a-minimal-gce-example-com" {
+  can_ip_forward = true
+  disk {
+    auto_delete  = true
+    boot         = true
+    device_name  = "persistent-disks-0"
+    disk_name    = ""
+    disk_size_gb = 64
+    disk_type    = "pd-standard"
+    interface    = ""
+    mode         = "READ_WRITE"
+    source       = ""
+    source_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-57-9202-64-0"
+    type         = "PERSISTENT"
+  }
+  machine_type = "n1-standard-1"
+  metadata = {
+    "cluster-name"                    = file("${path.module}/data/google_compute_instance_template_master-us-test1-a-minimal-gce-example-com_metadata_cluster-name")
+    "kops-k8s-io-instance-group-name" = file("${path.module}/data/google_compute_instance_template_master-us-test1-a-minimal-gce-example-com_metadata_kops-k8s-io-instance-group-name")
+    "ssh-keys"                        = file("${path.module}/data/google_compute_instance_template_master-us-test1-a-minimal-gce-example-com_metadata_ssh-keys")
+    "startup-script"                  = file("${path.module}/data/google_compute_instance_template_master-us-test1-a-minimal-gce-example-com_metadata_startup-script")
+  }
   name_prefix = "master-us-test1-a-minimal-do16cp-"
+  network_interface {
+    access_config {
+    }
+    network = google_compute_network.default.name
+  }
+  scheduling {
+    automatic_restart   = true
+    on_host_maintenance = "MIGRATE"
+    preemptible         = false
+  }
+  service_account {
+    email  = "default"
+    scopes = ["https://www.googleapis.com/auth/compute", "https://www.googleapis.com/auth/monitoring", "https://www.googleapis.com/auth/logging.write", "https://www.googleapis.com/auth/devstorage.read_write", "https://www.googleapis.com/auth/ndev.clouddns.readwrite"]
+  }
+  tags = ["minimal-gce-example-com-k8s-io-role-master"]
 }
 
 resource "google_compute_instance_template" "nodes-minimal-gce-example-com" {
+  can_ip_forward = true
+  disk {
+    auto_delete  = true
+    boot         = true
+    device_name  = "persistent-disks-0"
+    disk_name    = ""
+    disk_size_gb = 128
+    disk_type    = "pd-standard"
+    interface    = ""
+    mode         = "READ_WRITE"
+    source       = ""
+    source_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-57-9202-64-0"
+    type         = "PERSISTENT"
+  }
+  machine_type = "n1-standard-2"
+  metadata = {
+    "cluster-name"                    = file("${path.module}/data/google_compute_instance_template_nodes-minimal-gce-example-com_metadata_cluster-name")
+    "kops-k8s-io-instance-group-name" = file("${path.module}/data/google_compute_instance_template_nodes-minimal-gce-example-com_metadata_kops-k8s-io-instance-group-name")
+    "ssh-keys"                        = file("${path.module}/data/google_compute_instance_template_nodes-minimal-gce-example-com_metadata_ssh-keys")
+    "startup-script"                  = file("${path.module}/data/google_compute_instance_template_nodes-minimal-gce-example-com_metadata_startup-script")
+  }
   name_prefix = "nodes-minimal-gce-example-com-"
+  network_interface {
+    access_config {
+    }
+    network = google_compute_network.default.name
+  }
+  scheduling {
+    automatic_restart   = true
+    on_host_maintenance = "MIGRATE"
+    preemptible         = false
+  }
+  service_account {
+    email  = "default"
+    scopes = ["https://www.googleapis.com/auth/compute", "https://www.googleapis.com/auth/monitoring", "https://www.googleapis.com/auth/logging.write", "https://www.googleapis.com/auth/devstorage.read_only"]
+  }
+  tags = ["minimal-gce-example-com-k8s-io-role-node"]
 }
 
 resource "google_compute_network" "default" {

upup/pkg/fi/cloudup/gcetasks/instance.go

@@ -394,9 +394,29 @@ func ShortenImageURL(defaultProject string, imageURL string) (string, error) {
 }
 
 type terraformInstance struct {
-	terraformInstanceCommon
+	Name                  string                           `json:"name" cty:"name"`
+	CanIPForward          bool                             `json:"can_ip_forward" cty:"can_ip_forward"`
+	MachineType           string                           `json:"machine_type,omitempty" cty:"machine_type"`
+	ServiceAccount        *terraformServiceAccount         `json:"service_account,omitempty" cty:"service_account"`
+	Scheduling            *terraformScheduling             `json:"scheduling,omitempty" cty:"scheduling"`
+	Disks                 []*terraformInstanceAttachedDisk `json:"disk,omitempty" cty:"disk"`
+	NetworkInterfaces     []*terraformNetworkInterface     `json:"network_interface,omitempty" cty:"network_interface"`
+	Metadata              map[string]*terraform.Literal    `json:"metadata,omitempty" cty:"metadata"`
+	MetadataStartupScript *terraform.Literal               `json:"metadata_startup_script,omitempty" cty:"metadata_startup_script"`
+	Tags                  []string                         `json:"tags,omitempty" cty:"tags"`
+	Zone                  string                           `json:"zone,omitempty" cty:"zone"`
+}
 
-	Name string `json:"name" cty:"name"`
+type terraformInstanceAttachedDisk struct {
+	AutoDelete bool   `json:"auto_delete,omitempty" cty:"auto_delete"`
+	DeviceName string `json:"device_name,omitempty" cty:"device_name"`
+
+	// 'pd-standard', 'pd-ssd', 'local-ssd' etc
+	Type    string `json:"type,omitempty" cty:"type"`
+	Disk    string `json:"disk,omitempty" cty:"disk"`
+	Image   string `json:"image,omitempty" cty:"image"`
+	Scratch bool   `json:"scratch,omitempty" cty:"scratch"`
+	Size    int64  `json:"size,omitempty" cty:"size"`
 }
 
 func (_ *Instance) RenderTerraform(t *terraform.TerraformTarget, a, e, changes *Instance) error {
@@ -426,10 +446,10 @@ func (_ *Instance) RenderTerraform(t *terraform.TerraformTarget, a, e, changes *
 		tf.Zone = *e.Zone
 	}
 
-	tf.AddServiceAccounts(i.ServiceAccounts)
+	tf.ServiceAccount = addServiceAccounts(i.ServiceAccounts)
 
 	for _, d := range i.Disks {
-		tfd := &terraformAttachedDisk{
+		tfd := &terraformInstanceAttachedDisk{
 			AutoDelete: d.AutoDelete,
 			Scratch:    d.Type == "SCRATCH",
 			DeviceName: d.DeviceName,
@@ -446,9 +466,13 @@ func (_ *Instance) RenderTerraform(t *terraform.TerraformTarget, a, e, changes *
 		tf.Disks = append(tf.Disks, tfd)
 	}
 
-	tf.AddNetworks(e.Network, e.Subnet, i.NetworkInterfaces)
+	tf.NetworkInterfaces = addNetworks(e.Network, e.Subnet, i.NetworkInterfaces)
 
-	tf.AddMetadata(t, i.Name, i.Metadata)
+	metadata, err := addMetadata(t, i.Name, i.Metadata)
+	if err != nil {
+		return err
+	}
+	tf.Metadata = metadata
 
 	// Using metadata_startup_script is now mandatory (?)
 	{

upup/pkg/fi/cloudup/gcetasks/instancetemplate.go

@@ -417,23 +417,16 @@ func (_ *InstanceTemplate) RenderGCE(t *gce.GCEAPITarget, a, e, changes *Instanc
 }
 
 type terraformInstanceTemplate struct {
-	terraformInstanceCommon
+	NamePrefix            string                                   `json:"name_prefix" cty:"name_prefix"`
-	NamePrefix string `json:"name_prefix" cty:"name_prefix"`
+	CanIPForward          bool                                     `json:"can_ip_forward" cty:"can_ip_forward"`
-}
+	MachineType           string                                   `json:"machine_type,omitempty" cty:"machine_type"`
-
+	ServiceAccount        *terraformServiceAccount                 `json:"service_account,omitempty" cty:"service_account"`
-type terraformInstanceCommon struct {
+	Scheduling            *terraformScheduling                     `json:"scheduling,omitempty" cty:"scheduling"`
-	CanIPForward          bool                          `json:"can_ip_forward" cty:"can_ip_forward"`
+	Disks                 []*terraformInstanceTemplateAttachedDisk `json:"disk,omitempty" cty:"disk"`
-	MachineType           string                        `json:"machine_type,omitempty" cty:"machine_type"`
+	NetworkInterfaces     []*terraformNetworkInterface             `json:"network_interface,omitempty" cty:"network_interface"`
-	ServiceAccount        *terraformServiceAccount      `json:"service_account,omitempty" cty:"service_account"`
+	Metadata              map[string]*terraform.Literal            `json:"metadata,omitempty" cty:"metadata"`
-	Scheduling            *terraformScheduling          `json:"scheduling,omitempty" cty:"scheduling"`
+	MetadataStartupScript *terraform.Literal                       `json:"metadata_startup_script,omitempty" cty:"metadata_startup_script"`
-	Disks                 []*terraformAttachedDisk      `json:"disk,omitempty" cty:"disk"`
+	Tags                  []string                                 `json:"tags,omitempty" cty:"tags"`
-	NetworkInterfaces     []*terraformNetworkInterface  `json:"network_interface,omitempty" cty:"network_interface"`
-	Metadata              map[string]*terraform.Literal `json:"metadata,omitempty" cty:"metadata"`
-	MetadataStartupScript *terraform.Literal            `json:"metadata_startup_script,omitempty" cty:"metadata_startup_script"`
-	Tags                  []string                      `json:"tags,omitempty" cty:"tags"`
-
-	// Only for instances:
-	Zone string `json:"zone,omitempty" cty:"zone"`
 }
 
 type terraformServiceAccount struct {
@@ -447,17 +440,12 @@ type terraformScheduling struct {
 	Preemptible       bool   `json:"preemptible" cty:"preemptible"`
 }
 
-type terraformAttachedDisk struct {
+type terraformInstanceTemplateAttachedDisk struct {
-	// These values are common
 	AutoDelete bool   `json:"auto_delete,omitempty" cty:"auto_delete"`
 	DeviceName string `json:"device_name,omitempty" cty:"device_name"`
 
-	// DANGER - common but different meaning:
+	// scratch vs persistent
-	//   for an instance template this is scratch vs persistent
+	Type        string `json:"type,omitempty" cty:"type"`
-	//   for an instance this is 'pd-standard', 'pd-ssd', 'local-ssd' etc
-	Type string `json:"type,omitempty" cty:"type"`
-
-	// These values are only for instance templates:
 	Boot        bool   `json:"boot,omitempty" cty:"boot"`
 	DiskName    string `json:"disk_name,omitempty" cty:"disk_name"`
 	SourceImage string `json:"source_image,omitempty" cty:"source_image"`
@@ -466,12 +454,6 @@ type terraformAttachedDisk struct {
 	Mode        string `json:"mode,omitempty" cty:"mode"`
 	DiskType    string `json:"disk_type,omitempty" cty:"disk_type"`
 	DiskSizeGB  int64  `json:"disk_size_gb,omitempty" cty:"disk_size_gb"`
-
-	// These values are only for instances:
-	Disk    string `json:"disk,omitempty" cty:"disk"`
-	Image   string `json:"image,omitempty" cty:"image"`
-	Scratch bool   `json:"scratch,omitempty" cty:"scratch"`
-	Size    int64  `json:"size,omitempty" cty:"size"`
 }
 
 type terraformNetworkInterface struct {
@@ -484,8 +466,9 @@ type terraformAccessConfig struct {
 	NatIP *terraform.Literal `json:"nat_ip,omitempty" cty:"nat_ip"`
 }
 
-func (t *terraformInstanceCommon) AddNetworks(network *Network, subnet *Subnet, networkInterfacs []*compute.NetworkInterface) {
+func addNetworks(network *Network, subnet *Subnet, networkInterfaces []*compute.NetworkInterface) []*terraformNetworkInterface {
-	for _, g := range networkInterfacs {
+	ni := make([]*terraformNetworkInterface, 0)
+	for _, g := range networkInterfaces {
 		tf := &terraformNetworkInterface{}
 		if network != nil {
 			tf.Network = network.TerraformName()
@@ -505,46 +488,44 @@ func (t *terraformInstanceCommon) AddNetworks(network *Network, subnet *Subnet,
 			tf.AccessConfig = append(tf.AccessConfig, tac)
 		}
 
-		t.NetworkInterfaces = append(t.NetworkInterfaces, tf)
+		ni = append(ni, tf)
 	}
+	return ni
 }
 
-func (t *terraformInstanceCommon) AddMetadata(target *terraform.TerraformTarget, name string, metadata *compute.Metadata) error {
+func addMetadata(target *terraform.TerraformTarget, name string, metadata *compute.Metadata) (map[string]*terraform.Literal, error) {
-	if metadata != nil {
+	if metadata == nil {
-		if t.Metadata == nil {
+		return nil, nil
-			t.Metadata = make(map[string]*terraform.Literal)
-		}
-		for _, g := range metadata.Items {
-			v := fi.NewStringResource(fi.StringValue(g.Value))
-			tfResource, err := target.AddFile("google_compute_instance_template", name, "metadata_"+g.Key, v)
-			if err != nil {
-				return err
-			}
-
-			t.Metadata[g.Key] = tfResource
-		}
 	}
+	m := make(map[string]*terraform.Literal)
+	for _, g := range metadata.Items {
+		v := fi.NewStringResource(fi.StringValue(g.Value))
+		tfResource, err := target.AddFile("google_compute_instance_template", name, "metadata_"+g.Key, v)
+		if err != nil {
+			return nil, err
+		}
 
-	return nil
+		m[g.Key] = tfResource
+	}
+	return m, nil
 }
 
-func (t *terraformInstanceCommon) AddServiceAccounts(serviceAccounts []*compute.ServiceAccount) {
+func addServiceAccounts(serviceAccounts []*compute.ServiceAccount) *terraformServiceAccount {
 	// there's an inconsistency here- GCP only lets you have one service account per VM
 	// terraform gets it right, but the golang api doesn't. womp womp :(
 	if len(serviceAccounts) != 1 {
 		klog.Fatal("Instances can only have 1 service account assigned.")
-	} else {
-		klog.Infof("adding csa: %v", serviceAccounts[0].Email)
-		csa := serviceAccounts[0]
-		tsa := &terraformServiceAccount{
-			Email:  csa.Email,
-			Scopes: csa.Scopes,
-		}
-		// for _, scope := range csa.Scopes {
-		// 	tsa.Scopes = append(tsa.Scopes, scope)
-		// }
-		t.ServiceAccount = tsa
 	}
+	klog.Infof("adding csa: %v", serviceAccounts[0].Email)
+	csa := serviceAccounts[0]
+	tsa := &terraformServiceAccount{
+		Email:  csa.Email,
+		Scopes: csa.Scopes,
+	}
+	// for _, scope := range csa.Scopes {
+	// 	tsa.Scopes = append(tsa.Scopes, scope)
+	// }
+	return tsa
 }
 func (_ *InstanceTemplate) RenderTerraform(t *terraform.TerraformTarget, a, e, changes *InstanceTemplate) error {
 	project := t.Project
@@ -565,10 +546,10 @@ func (_ *InstanceTemplate) RenderTerraform(t *terraform.TerraformTarget, a, e, c
 	//tf.Zone = i.Properties.Zone
 	tf.Tags = i.Properties.Tags.Items
 
-	tf.AddServiceAccounts(i.Properties.ServiceAccounts)
+	tf.ServiceAccount = addServiceAccounts(i.Properties.ServiceAccounts)
 
 	for _, d := range i.Properties.Disks {
-		tfd := &terraformAttachedDisk{
+		tfd := &terraformInstanceTemplateAttachedDisk{
 			AutoDelete:  d.AutoDelete,
 			Boot:        d.Boot,
 			DeviceName:  d.DeviceName,
@@ -584,9 +565,13 @@ func (_ *InstanceTemplate) RenderTerraform(t *terraform.TerraformTarget, a, e, c
 		tf.Disks = append(tf.Disks, tfd)
 	}
 
-	tf.AddNetworks(e.Network, e.Subnet, i.Properties.NetworkInterfaces)
+	tf.NetworkInterfaces = addNetworks(e.Network, e.Subnet, i.Properties.NetworkInterfaces)
 
-	tf.AddMetadata(t, name, i.Properties.Metadata)
+	metadata, err := addMetadata(t, name, i.Properties.Metadata)
+	if err != nil {
+		return err
+	}
+	tf.Metadata = metadata
 
 	if i.Properties.Scheduling != nil {
 		tf.Scheduling = &terraformScheduling{

upup/pkg/fi/cloudup/terraform/BUILD.bazel

@@ -40,5 +40,6 @@ go_test(
         "//pkg/diff:go_default_library",
         "//vendor/github.com/hashicorp/hcl/v2/hclwrite:go_default_library",
         "//vendor/github.com/zclconf/go-cty/cty:go_default_library",
+        "//vendor/github.com/zclconf/go-cty/cty/gocty:go_default_library",
     ],
 )

upup/pkg/fi/cloudup/terraform/hcl2.go

@@ -177,17 +177,32 @@ func writeMap(body *hclwrite.Body, key string, values map[string]cty.Value) {
 	}
 	sort.Strings(keys)
 	for _, k := range keys {
-		v := values[k]
 		tokens = append(tokens, []*hclwrite.Token{
 			{Type: hclsyntax.TokenOQuote, Bytes: []byte{'"'}, SpacesBefore: 1},
 			{Type: hclsyntax.TokenQuotedLit, Bytes: []byte(k)},
 			{Type: hclsyntax.TokenCQuote, Bytes: []byte{'"'}, SpacesBefore: 1},
 			{Type: hclsyntax.TokenEqual, Bytes: []byte("="), SpacesBefore: 1},
-			{Type: hclsyntax.TokenOQuote, Bytes: []byte{'"'}, SpacesBefore: 1},
-			{Type: hclsyntax.TokenQuotedLit, Bytes: []byte(v.AsString())},
-			{Type: hclsyntax.TokenCQuote, Bytes: []byte{'"'}, SpacesBefore: 1},
-			{Type: hclsyntax.TokenNewline, Bytes: []byte("\n")},
 		}...)
+
+		v := values[k]
+
+		refLiteral := reflect.New(reflect.TypeOf(Literal{}))
+		err := gocty.FromCtyValue(v, refLiteral.Interface())
+		// If this is a map of literals then do not surround the value with quotes
+		if literal, ok := refLiteral.Interface().(*Literal); err == nil && ok {
+			// For maps of literals we currently only support file references
+			// If we ever need to support a map of strings to resource property references that can be added here
+			if literal.FilePath != "" {
+				tokens = append(tokens, &hclwrite.Token{Type: hclsyntax.TokenIdent, Bytes: []byte(fmt.Sprintf("file(%q)", literal.FilePath))})
+			}
+		} else {
+			tokens = append(tokens, []*hclwrite.Token{
+				{Type: hclsyntax.TokenOQuote, Bytes: []byte{'"'}, SpacesBefore: 1},
+				{Type: hclsyntax.TokenQuotedLit, Bytes: []byte(v.AsString())},
+				{Type: hclsyntax.TokenOQuote, Bytes: []byte{'"'}, SpacesBefore: 1},
+			}...)
+		}
+		tokens = append(tokens, &hclwrite.Token{Type: hclsyntax.TokenNewline, Bytes: []byte("\n")})
 	}
 	tokens = append(tokens,
 		&hclwrite.Token{Type: hclsyntax.TokenCBrace, Bytes: []byte("}")},

upup/pkg/fi/cloudup/terraform/hcl2_test.go

@@ -22,6 +22,7 @@ import (
 
 	"github.com/hashicorp/hcl/v2/hclwrite"
 	"github.com/zclconf/go-cty/cty"
+	"github.com/zclconf/go-cty/cty/gocty"
 	"k8s.io/kops/pkg/diff"
 )
 
@@ -262,3 +263,53 @@ tags = {
 		})
 	}
 }
+
+func TestWriteMapLiterals(t *testing.T) {
+	cases := []struct {
+		name     string
+		values   map[string]Literal
+		expected string
+	}{
+		{
+			name: "literal values",
+			values: map[string]Literal{
+				"key1": {FilePath: "${module.path}/path/to/value1"},
+				"key2": {FilePath: "${module.path}/path/to/value2"},
+			},
+			expected: `
+metadata = {
+  "key1" = file("${module.path}/path/to/value1")
+  "key2" = file("${module.path}/path/to/value2")
+}
+			`,
+		},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			literalMap := make(map[string]cty.Value)
+			for k, v := range tc.values {
+				literalType, err := gocty.ImpliedType(v)
+				if err != nil {
+					t.Errorf("unexpected error %v", err)
+				}
+				literalVal, err := gocty.ToCtyValue(v, literalType)
+
+				if err != nil {
+					t.Errorf("unexpected error %v", err)
+				}
+				literalMap[k] = literalVal
+			}
+
+			f := hclwrite.NewEmptyFile()
+			root := f.Body()
+			writeMap(root, "metadata", literalMap)
+			actual := strings.TrimSpace(string(f.Bytes()))
+			expected := strings.TrimSpace(tc.expected)
+			if actual != expected {
+				diffString := diff.FormatDiff(expected, string(actual))
+				t.Logf("diff:\n%s\n", diffString)
+				t.Errorf("expected: '%s', got: '%s'\n", expected, actual)
+			}
+		})
+	}
+}