kubernetes
/
kops
mirror of https://github.com/kubernetes/kops.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add support for Spot block in launch template 

- Launch configuration does not support the field SpotDurationInMinutes which is used to reserve the spot instances, but however Launch Template does
This commit is contained in:
Thejas B 2020-02-11 18:49:20 +05:30 committed by thejas
parent 2e57aaa5ac
commit dda8dc3f37
17 changed files with 129 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
k8s/crds/kops.k8s.io_instancegroups.yaml
View File

@ -689,6 +689,11 @@ spec:
description: SecurityGroupOverride overrides the default security group
created by Kops for this IG (AWS only).
type: string
spotDurationInMinutes:
description: SpotDurationInMinutes indicates this is a spot-block group,
with the specified value as the spot reservation time
format: int64
type: integer
subnets:
description: Subnets is the names of the Subnets (as specified in the
Cluster) where machines in this instance group should be placed

2
pkg/apis/kops/instancegroup.go
View File

@ -121,6 +121,8 @@ type InstanceGroupSpec struct {
Hooks []HookSpec `json:"hooks,omitempty"`
// MaxPrice indicates this is a spot-pricing group, with the specified value as our max-price bid
MaxPrice *string `json:"maxPrice,omitempty"`
// SpotDurationInMinutes reserves a spot block for the period specified
SpotDurationInMinutes *int64 `json:"spotDurationInMinutes,omitempty"`
// AssociatePublicIP is true if we want instances to have a public IP
AssociatePublicIP *bool `json:"associatePublicIp,omitempty"`
// AdditionalSecurityGroups attaches additional security groups (e.g. i-123456)

2
pkg/apis/kops/v1alpha2/instancegroup.go
View File

@ -116,6 +116,8 @@ type InstanceGroupSpec struct {
Hooks []HookSpec `json:"hooks,omitempty"`
// MaxPrice indicates this is a spot-pricing group, with the specified value as our max-price bid
MaxPrice *string `json:"maxPrice,omitempty"`
// SpotDurationInMinutes indicates this is a spot-block group, with the specified value as the spot reservation time
SpotDurationInMinutes *int64 `json:"spotDurationInMinutes,omitempty"`
// AssociatePublicIP is true if we want instances to have a public IP
AssociatePublicIP *bool `json:"associatePublicIp,omitempty"`
// AdditionalSecurityGroups attaches additional security groups (e.g. i-123456)

2
pkg/apis/kops/v1alpha2/zz_generated.conversion.go
View File

@ -3225,6 +3225,7 @@ func autoConvert_v1alpha2_InstanceGroupSpec_To_kops_InstanceGroupSpec(in *Instan
out.Hooks = nil
}
out.MaxPrice = in.MaxPrice
out.SpotDurationInMinutes = in.SpotDurationInMinutes
out.AssociatePublicIP = in.AssociatePublicIP
out.AdditionalSecurityGroups = in.AdditionalSecurityGroups
out.CloudLabels = in.CloudLabels
@ -3362,6 +3363,7 @@ func autoConvert_kops_InstanceGroupSpec_To_v1alpha2_InstanceGroupSpec(in *kops.I
out.Hooks = nil
}
out.MaxPrice = in.MaxPrice
out.SpotDurationInMinutes = in.SpotDurationInMinutes
out.AssociatePublicIP = in.AssociatePublicIP
out.AdditionalSecurityGroups = in.AdditionalSecurityGroups
out.CloudLabels = in.CloudLabels

5
pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go
View File

@ -1663,6 +1663,11 @@ func (in *InstanceGroupSpec) DeepCopyInto(out *InstanceGroupSpec) {
*out = new(string)
**out = **in
}
if in.SpotDurationInMinutes != nil {
in, out := &in.SpotDurationInMinutes, &out.SpotDurationInMinutes
*out = new(int64)
**out = **in
}
if in.AssociatePublicIP != nil {
in, out := &in.AssociatePublicIP, &out.AssociatePublicIP
*out = new(bool)

13
pkg/apis/kops/validation/aws.go
View File

@ -18,6 +18,7 @@ package validation
import (
"fmt"
"strconv"
"strings"
"k8s.io/apimachinery/pkg/util/sets"
@ -48,6 +49,8 @@ func awsValidateInstanceGroup(ig *kops.InstanceGroup) field.ErrorList {
allErrs = append(allErrs, awsValidateAMIforNVMe(field.NewPath(ig.GetName(), "spec", "machineType"), ig)...)
allErrs = append(allErrs, awsValidateSpotDurationInMinute(field.NewPath(ig.GetName(), "spec", "spotDurationInMinutes"), ig)...)
return allErrs
}
@ -107,3 +110,13 @@ func awsValidateAMIforNVMe(fieldPath *field.Path, ig *kops.InstanceGroup) field.
}
return allErrs
}
func awsValidateSpotDurationInMinute(fieldPath *field.Path, ig *kops.InstanceGroup) field.ErrorList {
allErrs := field.ErrorList{}
if ig.Spec.SpotDurationInMinutes != nil {
validSpotDurations := []string{"60", "120", "180", "240", "300", "360"}
spotDurationStr := strconv.FormatInt(*ig.Spec.SpotDurationInMinutes, 10)
allErrs = append(allErrs, IsValidValue(fieldPath, &spotDurationStr, validSpotDurations)...)
}
return allErrs
}

32
pkg/apis/kops/validation/aws_test.go
View File

@ -19,6 +19,8 @@ package validation
import (
"testing"
"k8s.io/kops/upup/pkg/fi"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/kops/pkg/apis/kops"
)
@ -102,6 +104,36 @@ func TestValidateInstanceGroupSpec(t *testing.T) {
"Forbidden::test-nodes.spec.machineType",
},
},
{
Input: kops.InstanceGroupSpec{
SpotDurationInMinutes: fi.Int64(55),
},
ExpectedErrors: []string{
"Unsupported value::test-nodes.spec.spotDurationInMinutes",
},
},
{
Input: kops.InstanceGroupSpec{
SpotDurationInMinutes: fi.Int64(380),
},
ExpectedErrors: []string{
"Unsupported value::test-nodes.spec.spotDurationInMinutes",
},
},
{
Input: kops.InstanceGroupSpec{
SpotDurationInMinutes: fi.Int64(125),
},
ExpectedErrors: []string{
"Unsupported value::test-nodes.spec.spotDurationInMinutes",
},
},
{
Input: kops.InstanceGroupSpec{
SpotDurationInMinutes: fi.Int64(120),
},
ExpectedErrors: []string{},
},
}
for _, g := range grid {
ig := &kops.InstanceGroup{

5
pkg/apis/kops/zz_generated.deepcopy.go
View File

@ -1829,6 +1829,11 @@ func (in *InstanceGroupSpec) DeepCopyInto(out *InstanceGroupSpec) {
*out = new(string)
**out = **in
}
if in.SpotDurationInMinutes != nil {
in, out := &in.SpotDurationInMinutes, &out.SpotDurationInMinutes
*out = new(int64)
**out = **in
}
if in.AssociatePublicIP != nil {
in, out := &in.AssociatePublicIP, &out.AssociatePublicIP
*out = new(bool)

3
pkg/model/awsmodel/autoscalinggroup.go
View File

@ -131,6 +131,9 @@ func (b *AutoscalingGroupModelBuilder) buildLaunchTemplateTask(c *fi.ModelBuilde
if ig.Spec.MixedInstancesPolicy == nil {
lt.SpotPrice = lc.SpotPrice
}
if ig.Spec.SpotDurationInMinutes != nil {
lt.SpotDurationInMinutes = ig.Spec.SpotDurationInMinutes
}
return lt, nil
}

7
tests/integration/update_cluster/launch_templates/cloudformation.json
View File

@ -602,6 +602,13 @@
"ImageId": "ami-12345678",
"InstanceType": "t3.medium",
"KeyName": "kubernetes.launchtemplates.example.com-c4:a6:ed:9a:a8:89:b9:e2:c3:9c:d6:63:eb:9c:71:57",
"InstanceMarketOptions": {
"MarketType": "spot",
"SpotOptions": {
"BlockDurationMinutes": 120,
"MaxPrice": "0.1"
}
},
"NetworkInterfaces": [
{
"AssociatePublicIpAddress": true,

2
tests/integration/update_cluster/launch_templates/in-v1alpha2.yaml
View File

@ -71,6 +71,8 @@ spec:
minSize: 2
role: Node
instanceProtection: true
maxPrice: "0.1"
spotDurationInMinutes: 120
subnets:
- us-test-1b
---

9
tests/integration/update_cluster/launch_templates/kubernetes.tf
View File

@ -611,6 +611,15 @@ resource "aws_launch_template" "nodes-launchtemplates-example-com" {
instance_type = "t3.medium"
key_name = "${aws_key_pair.kubernetes-launchtemplates-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157.id}"
instance_market_options = {
market_type = "spot"
spot_options = {
block_duration_minutes = 120
max_price = "0.1"
}
}
network_interfaces = {
associate_public_ip_address = true
delete_on_termination = true

2
upup/pkg/fi/cloudup/awstasks/launchtemplate.go
View File

@ -62,6 +62,8 @@ type LaunchTemplate struct {
SecurityGroups []*SecurityGroup
// SpotPrice is set to the spot-price bid if this is a spot pricing request
SpotPrice string
// SpotDurationInMinutes is set for requesting spot blocks
SpotDurationInMinutes *int64
// Tags are the keypairs to apply to the instance and volume on launch.
Tags map[string]string
// Tenancy. Can be either default or dedicated.

40
upup/pkg/fi/cloudup/awstasks/launchtemplate_target_cloudformation.go
View File

@ -62,6 +62,8 @@ type cloudformationLaunchTemplateIAMProfile struct {
}
type cloudformationLaunchTemplateMarketOptionsSpotOptions struct {
// BlockDurationMinutes is required duration in minutes. This value must be a multiple of 60.
BlockDurationMinutes *int64 `json:"BlockDurationMinutes,omitempty"`
// InstancesInterruptionBehavior is the behavior when a Spot Instance is interrupted. Can be hibernate, stop, or terminate
InstancesInterruptionBehavior *string `json:"InstancesInterruptionBehavior,omitempty"`
// MaxPrice is the maximum hourly price you're willing to pay for the Spot Instances
@ -74,7 +76,7 @@ type cloudformationLaunchTemplateMarketOptions struct {
// MarketType is the option type
MarketType *string `json:"MarketType,omitempty"`
// SpotOptions are the set of options
SpotOptions []*cloudformationLaunchTemplateMarketOptionsSpotOptions `json:"Options,omitempty"`
SpotOptions *cloudformationLaunchTemplateMarketOptionsSpotOptions `json:"SpotOptions,omitempty"`
}
type cloudformationLaunchTemplateBlockDeviceEBS struct {
@ -165,29 +167,33 @@ func (t *LaunchTemplate) RenderCloudformation(target *cloudformation.Cloudformat
image = im.ImageId
}
cf := &cloudformationLaunchTemplate{
LaunchTemplateName: fi.String(fi.StringValue(e.Name)),
LaunchTemplateData: &cloudformationLaunchTemplateData{
EBSOptimized: e.RootVolumeOptimization,
ImageID: image,
InstanceType: e.InstanceType,
NetworkInterfaces: []*cloudformationLaunchTemplateNetworkInterface{
{
AssociatePublicIPAddress: e.AssociatePublicIP,
DeleteOnTermination: fi.Bool(true),
DeviceIndex: fi.Int(0),
},
launchTemplateData := &cloudformationLaunchTemplateData{
EBSOptimized: e.RootVolumeOptimization,
ImageID: image,
InstanceType: e.InstanceType,
NetworkInterfaces: []*cloudformationLaunchTemplateNetworkInterface{
{
AssociatePublicIPAddress: e.AssociatePublicIP,
DeleteOnTermination: fi.Bool(true),
DeviceIndex: fi.Int(0),
},
},
}
data := cf.LaunchTemplateData
if e.SpotPrice != "" {
data.MarketOptions = &cloudformationLaunchTemplateMarketOptions{
MarketType: fi.String("spot"),
SpotOptions: []*cloudformationLaunchTemplateMarketOptionsSpotOptions{{MaxPrice: fi.String(e.SpotPrice)}},
marketSpotOptions := cloudformationLaunchTemplateMarketOptionsSpotOptions{MaxPrice: fi.String(e.SpotPrice)}
if e.SpotDurationInMinutes != nil {
marketSpotOptions.BlockDurationMinutes = e.SpotDurationInMinutes
}
launchTemplateData.MarketOptions = &cloudformationLaunchTemplateMarketOptions{MarketType: fi.String("spot"), SpotOptions: &marketSpotOptions}
}
cf := &cloudformationLaunchTemplate{
LaunchTemplateName: fi.String(fi.StringValue(e.Name)),
LaunchTemplateData: launchTemplateData,
}
data := cf.LaunchTemplateData
for _, x := range e.SecurityGroups {
data.NetworkInterfaces[0].SecurityGroups = append(data.NetworkInterfaces[0].SecurityGroups, x.CloudformationLink())
}

9
upup/pkg/fi/cloudup/awstasks/launchtemplate_target_cloudformation_test.go
View File

@ -37,6 +37,8 @@ func TestLaunchTemplateCloudformationRender(t *testing.T) {
RootVolumeOptimization: fi.Bool(true),
RootVolumeIops: fi.Int64(100),
RootVolumeSize: fi.Int64(64),
SpotPrice: "10",
SpotDurationInMinutes: fi.Int64(120),
SSHKey: &SSHKey{
Name: fi.String("mykey"),
},
@ -61,6 +63,13 @@ func TestLaunchTemplateCloudformationRender(t *testing.T) {
},
"InstanceType": "t2.medium",
"KeyName": "mykey",
"InstanceMarketOptions": {
"MarketType": "spot",
"SpotOptions": {
"BlockDurationMinutes": 120,
"MaxPrice": "10"
}
},
"NetworkInterfaces": [
{
"AssociatePublicIpAddress": true,

6
upup/pkg/fi/cloudup/awstasks/launchtemplate_target_terraform.go
View File

@ -179,10 +179,14 @@ func (t *LaunchTemplate) RenderTerraform(target *terraform.TerraformTarget, a, e
}
if e.SpotPrice != "" {
marketSpotOptions := terraformLaunchTemplateMarketOptionsSpotOptions{MaxPrice: fi.String(e.SpotPrice)}
if e.SpotDurationInMinutes != nil {
marketSpotOptions.BlockDurationMinutes = e.SpotDurationInMinutes
}
tf.MarketOptions = []*terraformLaunchTemplateMarketOptions{
{
MarketType: fi.String("spot"),
SpotOptions: []*terraformLaunchTemplateMarketOptionsSpotOptions{{MaxPrice: fi.String(e.SpotPrice)}},
SpotOptions: []*terraformLaunchTemplateMarketOptionsSpotOptions{&marketSpotOptions},
},
}
}

4
upup/pkg/fi/cloudup/awstasks/launchtemplate_target_terraform_test.go
View File

@ -35,6 +35,7 @@ func TestLaunchTemplateTerraformRender(t *testing.T) {
InstanceMonitoring: fi.Bool(true),
InstanceType: fi.String("t2.medium"),
SpotPrice: "0.1",
SpotDurationInMinutes: fi.Int64(60),
RootVolumeOptimization: fi.Bool(true),
RootVolumeIops: fi.Int64(100),
RootVolumeSize: fi.Int64(64),
@ -72,7 +73,8 @@ resource "aws_launch_template" "test" {
market_type = "spot"
spot_options = {
max_price = "0.1"
block_duration_minutes = 60
max_price = "0.1"
}
}