Merge pull request #13946 from jonasasx/feature/gce_spot

Adding GCE SPOT support

k8s/crds/kops.k8s.io_instancegroups.yaml

@@ -222,6 +222,12 @@ spec:
                       type: array
                   type: object
                 type: array
+              gcpProvisioningModel:
+                description: 'GCPProvisioningModel: Specifies the provisioning model
+                  of the GCP instance. Valid values:   ''STANDARD'': (default) standard
+                  provisioning with user controlled run time, no discounts   ''SPOT'':
+                  heavily discounted, no guaranteed run time.'
+                type: string
               guestAccelerators:
                 description: GuestAccelerators configures additional accelerators
                 items:

pkg/apis/kops/instancegroup.go

@@ -199,6 +199,11 @@ type InstanceGroupSpec struct {
 	// MaxInstanceLifetime to the maximum amount of time, in seconds, that an instance can be in service.
 	// Value expected must be in form of duration ("ms", "s", "m", "h")
 	MaxInstanceLifetime *metav1.Duration `json:"maxInstanceLifetime,omitempty"`
+	// GCPProvisioningModel: Specifies the provisioning model of the GCP instance.
+	// Valid values:
+	//   'STANDARD': (default) standard provisioning with user controlled run time, no discounts
+	//   'SPOT': heavily discounted, no guaranteed run time.
+	GCPProvisioningModel *string `json:"gcpProvisioningModel,omitempty"`
 }
 
 const (

pkg/apis/kops/v1alpha2/instancegroup.go

@@ -165,6 +165,11 @@ type InstanceGroupSpec struct {
 	// MaxInstanceLifetime to the maximum amount of time, in seconds, that an instance can be in service.
 	// Value expected must be in form of duration ("ms", "s", "m", "h")
 	MaxInstanceLifetime *metav1.Duration `json:"maxInstanceLifetime,omitempty"`
+	// GCPProvisioningModel: Specifies the provisioning model of the GCP instance.
+	// Valid values:
+	//   'STANDARD': (default) standard provisioning with user controlled run time, no discounts
+	//   'SPOT': heavily discounted, no guaranteed run time.
+	GCPProvisioningModel *string `json:"gcpProvisioningModel,omitempty"`
 }
 
 // InstanceMetadataOptions defines the EC2 instance metadata service options (AWS Only)

pkg/apis/kops/v1alpha2/zz_generated.conversion.go

@@ -4474,6 +4474,7 @@ func autoConvert_v1alpha2_InstanceGroupSpec_To_kops_InstanceGroupSpec(in *Instan
 		out.GuestAccelerators = nil
 	}
 	out.MaxInstanceLifetime = in.MaxInstanceLifetime
+	out.GCPProvisioningModel = in.GCPProvisioningModel
 	return nil
 }
 
@@ -4658,6 +4659,7 @@ func autoConvert_kops_InstanceGroupSpec_To_v1alpha2_InstanceGroupSpec(in *kops.I
 		out.GuestAccelerators = nil
 	}
 	out.MaxInstanceLifetime = in.MaxInstanceLifetime
+	out.GCPProvisioningModel = in.GCPProvisioningModel
 	return nil
 }
 

pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go

@@ -2544,6 +2544,11 @@ func (in *InstanceGroupSpec) DeepCopyInto(out *InstanceGroupSpec) {
 		*out = new(v1.Duration)
 		**out = **in
 	}
+	if in.GCPProvisioningModel != nil {
+		in, out := &in.GCPProvisioningModel, &out.GCPProvisioningModel
+		*out = new(string)
+		**out = **in
+	}
 	return
 }
 

pkg/apis/kops/v1alpha3/instancegroup.go

@@ -162,6 +162,11 @@ type InstanceGroupSpec struct {
 	// MaxInstanceLifetime to the maximum amount of time, in seconds, that an instance can be in service.
 	// Value expected must be in form of duration ("ms", "s", "m", "h")
 	MaxInstanceLifetime *metav1.Duration `json:"maxInstanceLifetime,omitempty"`
+	// GCPProvisioningModel: Specifies the provisioning model of the GCP instance.
+	// Valid values:
+	//   'STANDARD': (default) standard provisioning with user controlled run time, no discounts
+	//   'SPOT': heavily discounted, no guaranteed run time.
+	GCPProvisioningModel *string `json:"gcpProvisioningModel,omitempty"`
 }
 
 // InstanceMetadataOptions defines the EC2 instance metadata service options (AWS Only)

pkg/apis/kops/v1alpha3/zz_generated.conversion.go

@@ -4613,6 +4613,7 @@ func autoConvert_v1alpha3_InstanceGroupSpec_To_kops_InstanceGroupSpec(in *Instan
 		out.GuestAccelerators = nil
 	}
 	out.MaxInstanceLifetime = in.MaxInstanceLifetime
+	out.GCPProvisioningModel = in.GCPProvisioningModel
 	return nil
 }
 
@@ -4797,6 +4798,7 @@ func autoConvert_kops_InstanceGroupSpec_To_v1alpha3_InstanceGroupSpec(in *kops.I
 		out.GuestAccelerators = nil
 	}
 	out.MaxInstanceLifetime = in.MaxInstanceLifetime
+	out.GCPProvisioningModel = in.GCPProvisioningModel
 	return nil
 }
 

pkg/apis/kops/v1alpha3/zz_generated.deepcopy.go

@@ -2550,6 +2550,11 @@ func (in *InstanceGroupSpec) DeepCopyInto(out *InstanceGroupSpec) {
 		*out = new(v1.Duration)
 		**out = **in
 	}
+	if in.GCPProvisioningModel != nil {
+		in, out := &in.GCPProvisioningModel, &out.GCPProvisioningModel
+		*out = new(string)
+		**out = **in
+	}
 	return
 }
 

pkg/apis/kops/validation/gce.go

@@ -19,6 +19,7 @@ package validation
 import (
 	"k8s.io/apimachinery/pkg/util/validation/field"
 	"k8s.io/kops/pkg/apis/kops"
+	"k8s.io/kops/upup/pkg/fi/cloudup/gce"
 )
 
 func gceValidateCluster(c *kops.Cluster) field.ErrorList {
@@ -45,3 +46,13 @@ func gceValidateCluster(c *kops.Cluster) field.ErrorList {
 
 	return allErrs
 }
+
+func gceValidateInstanceGroup(ig *kops.InstanceGroup, cloud gce.GCECloud) field.ErrorList {
+	allErrs := field.ErrorList{}
+
+	if ig.Spec.GCPProvisioningModel != nil {
+		fieldSpec := field.NewPath("spec")
+		allErrs = append(allErrs, IsValidValue(fieldSpec.Child("gcpProvisioningModel"), ig.Spec.GCPProvisioningModel, []string{"STANDARD", "SPOT"})...)
+	}
+	return allErrs
+}

pkg/apis/kops/validation/instancegroup.go

@@ -31,6 +31,7 @@ import (
 	"k8s.io/kops/pkg/apis/kops/util"
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
+	"k8s.io/kops/upup/pkg/fi/cloudup/gce"
 )
 
 // ValidateInstanceGroup is responsible for validating the configuration of a instancegroup
@@ -147,8 +148,13 @@ func ValidateInstanceGroup(g *kops.InstanceGroup, cloud fi.Cloud, strict bool) f
 		allErrs = append(allErrs, validateIGCloudLabels(g, field.NewPath("spec", "cloudLabels"))...)
 	}
 
-	if cloud != nil && cloud.ProviderID() == kops.CloudProviderAWS {
+	if cloud != nil {
-		allErrs = append(allErrs, awsValidateInstanceGroup(g, cloud.(awsup.AWSCloud))...)
+		switch cloud.ProviderID() {
+		case kops.CloudProviderAWS:
+			allErrs = append(allErrs, awsValidateInstanceGroup(g, cloud.(awsup.AWSCloud))...)
+		case kops.CloudProviderGCE:
+			allErrs = append(allErrs, gceValidateInstanceGroup(g, cloud.(gce.GCECloud))...)
+		}
 	}
 
 	for i, lb := range g.Spec.ExternalLoadBalancers {

pkg/apis/kops/zz_generated.deepcopy.go

@@ -2713,6 +2713,11 @@ func (in *InstanceGroupSpec) DeepCopyInto(out *InstanceGroupSpec) {
 		*out = new(v1.Duration)
 		**out = **in
 	}
+	if in.GCPProvisioningModel != nil {
+		in, out := &in.GCPProvisioningModel, &out.GCPProvisioningModel
+		*out = new(string)
+		**out = **in
+	}
 	return
 }
 

pkg/model/gcemodel/autoscalinggroup.go

@@ -89,8 +89,8 @@ func (b *AutoscalingGroupModelBuilder) buildInstanceTemplate(c *fi.ModelBuilderC
 				BootDiskSizeGB: i64(int64(volumeSize)),
 				BootDiskImage:  s(ig.Spec.Image),
 
-				// TODO: Support preemptible nodes?
+				Preemptible:          fi.Bool(fi.StringValue(ig.Spec.GCPProvisioningModel) == "SPOT"),
-				Preemptible: fi.Bool(false),
+				GCPProvisioningModel: ig.Spec.GCPProvisioningModel,
 
 				HasExternalIP: fi.Bool(b.Cluster.Spec.Topology.Masters == kops.TopologyPublic),
 

tests/integration/update_cluster/ha_gce/kubernetes.tf

@@ -554,6 +554,7 @@ resource "google_compute_instance_template" "master-us-test1-a-ha-gce-example-co
     automatic_restart   = true
     on_host_maintenance = "MIGRATE"
     preemptible         = false
+    provisioning_model  = "STANDARD"
   }
   service_account {
     email  = google_service_account.control-plane.email
@@ -600,6 +601,7 @@ resource "google_compute_instance_template" "master-us-test1-b-ha-gce-example-co
     automatic_restart   = true
     on_host_maintenance = "MIGRATE"
     preemptible         = false
+    provisioning_model  = "STANDARD"
   }
   service_account {
     email  = google_service_account.control-plane.email
@@ -646,6 +648,7 @@ resource "google_compute_instance_template" "master-us-test1-c-ha-gce-example-co
     automatic_restart   = true
     on_host_maintenance = "MIGRATE"
     preemptible         = false
+    provisioning_model  = "STANDARD"
   }
   service_account {
     email  = google_service_account.control-plane.email
@@ -692,6 +695,7 @@ resource "google_compute_instance_template" "nodes-ha-gce-example-com" {
     automatic_restart   = true
     on_host_maintenance = "MIGRATE"
     preemptible         = false
+    provisioning_model  = "STANDARD"
   }
   service_account {
     email  = google_service_account.node.email

tests/integration/update_cluster/minimal_gce/kubernetes.tf

@@ -458,6 +458,7 @@ resource "google_compute_instance_template" "master-us-test1-a-minimal-gce-examp
     automatic_restart   = true
     on_host_maintenance = "MIGRATE"
     preemptible         = false
+    provisioning_model  = "STANDARD"
   }
   service_account {
     email  = google_service_account.control-plane.email
@@ -504,6 +505,7 @@ resource "google_compute_instance_template" "nodes-minimal-gce-example-com" {
     automatic_restart   = true
     on_host_maintenance = "MIGRATE"
     preemptible         = false
+    provisioning_model  = "STANDARD"
   }
   service_account {
     email  = google_service_account.node.email

tests/integration/update_cluster/minimal_gce_ilb/kubernetes.tf

@@ -451,6 +451,7 @@ resource "google_compute_instance_template" "master-us-test1-a-minimal-gce-ilb-e
     automatic_restart   = true
     on_host_maintenance = "MIGRATE"
     preemptible         = false
+    provisioning_model  = "STANDARD"
   }
   service_account {
     email  = google_service_account.control-plane.email
@@ -495,6 +496,7 @@ resource "google_compute_instance_template" "nodes-minimal-gce-ilb-example-com"
     automatic_restart   = true
     on_host_maintenance = "MIGRATE"
     preemptible         = false
+    provisioning_model  = "STANDARD"
   }
   service_account {
     email  = google_service_account.node.email

tests/integration/update_cluster/minimal_gce_ilb_longclustername/kubernetes.tf

@@ -459,6 +459,7 @@ resource "google_compute_instance_template" "master-us-test1-a-minimal-gce-with-
     automatic_restart   = true
     on_host_maintenance = "MIGRATE"
     preemptible         = false
+    provisioning_model  = "STANDARD"
   }
   service_account {
     email  = google_service_account.control-plane.email
@@ -503,6 +504,7 @@ resource "google_compute_instance_template" "nodes-minimal-gce-with-a-very-very-
     automatic_restart   = true
     on_host_maintenance = "MIGRATE"
     preemptible         = false
+    provisioning_model  = "STANDARD"
   }
   service_account {
     email  = google_service_account.node.email

tests/integration/update_cluster/minimal_gce_longclustername/kubernetes.tf

@@ -458,6 +458,7 @@ resource "google_compute_instance_template" "master-us-test1-a-minimal-gce-with-
     automatic_restart   = true
     on_host_maintenance = "MIGRATE"
     preemptible         = false
+    provisioning_model  = "STANDARD"
   }
   service_account {
     email  = google_service_account.control-plane.email
@@ -504,6 +505,7 @@ resource "google_compute_instance_template" "nodes-minimal-gce-with-a-very-very-
     automatic_restart   = true
     on_host_maintenance = "MIGRATE"
     preemptible         = false
+    provisioning_model  = "STANDARD"
   }
   service_account {
     email  = google_service_account.node.email

tests/integration/update_cluster/minimal_gce_private/kubernetes.tf

@@ -448,6 +448,7 @@ resource "google_compute_instance_template" "master-us-test1-a-minimal-gce-priva
     automatic_restart   = true
     on_host_maintenance = "MIGRATE"
     preemptible         = false
+    provisioning_model  = "STANDARD"
   }
   service_account {
     email  = google_service_account.control-plane.email
@@ -492,6 +493,7 @@ resource "google_compute_instance_template" "nodes-minimal-gce-private-example-c
     automatic_restart   = true
     on_host_maintenance = "MIGRATE"
     preemptible         = false
+    provisioning_model  = "STANDARD"
   }
   service_account {
     email  = google_service_account.node.email

upup/pkg/fi/cloudup/gcetasks/instancetemplate.go

@@ -51,10 +51,11 @@ type InstanceTemplate struct {
 
 	Lifecycle fi.Lifecycle
 
-	Network     *Network
+	Network              *Network
-	Tags        []string
+	Tags                 []string
-	Labels      map[string]string
+	Labels               map[string]string
-	Preemptible *bool
+	Preemptible          *bool
+	GCPProvisioningModel *string
 
 	BootDiskImage  *string
 	BootDiskSizeGB *int64
@@ -132,6 +133,7 @@ func (e *InstanceTemplate) Find(c *fi.Context) (*InstanceTemplate, error) {
 
 		if p.Scheduling != nil {
 			actual.Preemptible = &p.Scheduling.Preemptible
+			actual.GCPProvisioningModel = &p.Scheduling.ProvisioningModel
 		}
 		if len(p.NetworkInterfaces) != 0 {
 			ni := p.NetworkInterfaces[0]
@@ -249,7 +251,7 @@ func (e *InstanceTemplate) mapToGCE(project string, region string) (*compute.Ins
 		scheduling = &compute.Scheduling{
 			AutomaticRestart:  fi.Bool(false),
 			OnHostMaintenance: "TERMINATE",
-			ProvisioningModel: "STANDARD", // TODO: Support Spot?
+			ProvisioningModel: fi.StringValue(e.GCPProvisioningModel),
 			Preemptible:       true,
 		}
 	} else {
@@ -501,6 +503,7 @@ type terraformScheduling struct {
 	AutomaticRestart  bool   `cty:"automatic_restart"`
 	OnHostMaintenance string `cty:"on_host_maintenance"`
 	Preemptible       bool   `cty:"preemptible"`
+	ProvisioningModel string `cty:"provisioning_model"`
 }
 
 type terraformInstanceTemplateAttachedDisk struct {
@@ -649,6 +652,7 @@ func (_ *InstanceTemplate) RenderTerraform(t *terraform.TerraformTarget, a, e, c
 			AutomaticRestart:  fi.BoolValue(i.Properties.Scheduling.AutomaticRestart),
 			OnHostMaintenance: i.Properties.Scheduling.OnHostMaintenance,
 			Preemptible:       i.Properties.Scheduling.Preemptible,
+			ProvisioningModel: i.Properties.Scheduling.ProvisioningModel,
 		}
 	}