kubernetes
/
kops
mirror of https://github.com/kubernetes/kops.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

feature(spotinst): add support for auto scaler configuration

This commit is contained in:
liranp 2020-04-19 14:22:57 +03:00
parent bae6a1334a
commit 870bdfdcff
No known key found for this signature in database
GPG Key ID: D5F03857002C1A93
4 changed files with 507 additions and 131 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

138
pkg/model/spotinstmodel/instance_group.go
View File

@ -21,6 +21,7 @@ import (
"strconv"
"strings"
corev1 "k8s.io/api/core/v1"
"k8s.io/klog"
"k8s.io/kops/pkg/apis/kops"
"k8s.io/kops/pkg/featureflag"
@ -83,6 +84,10 @@ const (
InstanceGroupLabelAutoScalerHeadroomMemPerUnit = "spotinst.io/autoscaler-headroom-mem-per-unit"
InstanceGroupLabelAutoScalerHeadroomNumOfUnits = "spotinst.io/autoscaler-headroom-num-of-units"
// InstanceGroupLabelAutoScalerCooldown is the metadata label used on the
// instance group to specify the cooldown period (in seconds) for scaling actions.
InstanceGroupLabelAutoScalerCooldown = "spotinst.io/autoscaler-cooldown"
// InstanceGroupLabelAutoScalerScaleDown* are the metadata labels used on the
// instance group to specify the scale down configuration used by the auto scaler.
InstanceGroupLabelAutoScalerScaleDownMaxPercentage = "spotinst.io/autoscaler-scale-down-max-percentage"
@ -268,6 +273,9 @@ func (b *InstanceGroupModelBuilder) buildElastigroup(c *fi.ModelBuilderContext,
if err != nil {
return fmt.Errorf("error building auto scaler options: %v", err)
}
if group.AutoScalerOpts != nil { // remove unsupported options
group.AutoScalerOpts.Taints = nil
}
klog.V(4).Infof("Adding task: Elastigroup/%s", fi.StringValue(group.Name))
c.AddTask(group)
@ -367,7 +375,7 @@ func (b *InstanceGroupModelBuilder) buildOcean(c *fi.ModelBuilderContext, igs ..
}
// Capacity.
ocean.MinSize, _ = b.buildCapacity(ig)
ocean.MinSize = fi.Int64(0)
ocean.MaxSize = fi.Int64(0)
// Monitoring.
@ -386,11 +394,14 @@ func (b *InstanceGroupModelBuilder) buildOcean(c *fi.ModelBuilderContext, igs ..
}
// Root volume.
ocean.RootVolumeOpts, err = b.buildRootVolumeOpts(ig)
rootVolumeOpts, err := b.buildRootVolumeOpts(ig)
if err != nil {
return fmt.Errorf("error building root volume options: %v", err)
}
ocean.RootVolumeOpts.Type = nil // unsupported
if rootVolumeOpts != nil {
ocean.RootVolumeOpts = rootVolumeOpts
ocean.RootVolumeOpts.Type = nil // not supported in Ocean
}
// Security groups.
ocean.SecurityGroups, err = b.buildSecurityGroups(c, ig)
@ -427,6 +438,10 @@ func (b *InstanceGroupModelBuilder) buildOcean(c *fi.ModelBuilderContext, igs ..
if err != nil {
return fmt.Errorf("error building auto scaler options: %v", err)
}
if ocean.AutoScalerOpts != nil { // remove unsupported options
ocean.AutoScalerOpts.Labels = nil
ocean.AutoScalerOpts.Taints = nil
}
// Create a Launch Spec for each instance group.
for _, ig := range igs {
@ -453,9 +468,7 @@ func (b *InstanceGroupModelBuilder) buildLaunchSpec(c *fi.ModelBuilderContext,
// Capacity.
minSize, maxSize := b.buildCapacity(ig)
if fi.Int64Value(minSize) < fi.Int64Value(ocean.MinSize) {
ocean.MinSize = minSize
}
ocean.MinSize = fi.Int64(fi.Int64Value(ocean.MinSize) + fi.Int64Value(minSize))
ocean.MaxSize = fi.Int64(fi.Int64Value(ocean.MaxSize) + fi.Int64Value(maxSize))
// User data.
@ -476,18 +489,33 @@ func (b *InstanceGroupModelBuilder) buildLaunchSpec(c *fi.ModelBuilderContext,
return fmt.Errorf("error building security groups: %v", err)
}
// Subnets.
launchSpec.Subnets, err = b.buildSubnets(ig)
if err != nil {
return fmt.Errorf("error building subnets: %v", err)
}
// Tags.
launchSpec.Tags, err = b.buildTags(ig)
if err != nil {
return fmt.Errorf("error building cloud tags: %v", err)
}
// Labels.
// Auto Scaler.
autoScalerOpts, err := b.buildAutoScalerOpts(b.ClusterName(), ig)
if err != nil {
return fmt.Errorf("error building auto scaler options: %v", err)
}
launchSpec.Labels = autoScalerOpts.Labels
if autoScalerOpts != nil { // remove unsupported options
autoScalerOpts.Enabled = nil
autoScalerOpts.ClusterID = nil
autoScalerOpts.Cooldown = nil
autoScalerOpts.Down = nil
if autoScalerOpts.Labels != nil || autoScalerOpts.Taints != nil || autoScalerOpts.Headroom != nil {
launchSpec.AutoScalerOpts = autoScalerOpts
}
}
klog.V(4).Infof("Adding task: LaunchSpec/%s", fi.StringValue(launchSpec.Name))
c.AddTask(launchSpec)
@ -576,8 +604,14 @@ func (b *InstanceGroupModelBuilder) buildPublicIpOpts(ig *kops.InstanceGroup) (*
func (b *InstanceGroupModelBuilder) buildRootVolumeOpts(ig *kops.InstanceGroup) (*spotinsttasks.RootVolumeOpts, error) {
opts := &spotinsttasks.RootVolumeOpts{
IOPS: ig.Spec.RootVolumeIops,
Optimization: ig.Spec.RootVolumeOptimization,
IOPS: ig.Spec.RootVolumeIops,
}
// Optimization.
{
if fi.BoolValue(ig.Spec.RootVolumeOptimization) {
opts.Optimization = ig.Spec.RootVolumeOptimization
}
}
// Size.
@ -669,6 +703,15 @@ func (b *InstanceGroupModelBuilder) buildAutoScalerOpts(clusterID string, ig *ko
defaultNodeLabels = fi.BoolValue(v)
}
case InstanceGroupLabelAutoScalerCooldown:
{
v, err := parseInt(v)
if err != nil {
return nil, err
}
opts.Cooldown = fi.Int(int(fi.Int64Value(v)))
}
case InstanceGroupLabelAutoScalerHeadroomCPUPerUnit:
{
v, err := parseInt(v)
@ -743,20 +786,37 @@ func (b *InstanceGroupModelBuilder) buildAutoScalerOpts(clusterID string, ig *ko
}
}
// Set the node labels.
if fi.BoolValue(opts.Enabled) {
labels := make(map[string]string)
for k, v := range ig.Spec.NodeLabels {
if strings.HasPrefix(k, kops.NodeLabelInstanceGroup) && !defaultNodeLabels {
continue
}
labels[k] = v
// Configure Elastigroup defaults to avoid state drifts.
if !featureflag.SpotinstOcean.Enabled() {
if opts.Cooldown == nil {
opts.Cooldown = fi.Int(300)
}
if len(labels) > 0 {
opts.Labels = labels
if opts.Down != nil && opts.Down.EvaluationPeriods == nil {
opts.Down.EvaluationPeriods = fi.Int(5)
}
}
// Configure node labels.
labels := make(map[string]string)
for k, v := range ig.Spec.NodeLabels {
if strings.HasPrefix(k, kops.NodeLabelInstanceGroup) && !defaultNodeLabels {
continue
}
labels[k] = v
}
if len(labels) > 0 {
opts.Labels = labels
}
// Configure node taints.
taints, err := parseTaints(ig.Spec.Taints)
if err != nil {
return nil, err
}
if len(taints) > 0 {
opts.Taints = taints
}
return opts, nil
}
@ -784,6 +844,44 @@ func parseInt(str string) (*int64, error) {
return &v, nil
}
func parseTaints(taintSpecs []string) ([]*corev1.Taint, error) {
var taints []*corev1.Taint
for _, taintSpec := range taintSpecs {
taint, err := parseTaint(taintSpec)
if err != nil {
return nil, err
}
taints = append(taints, taint)
}
return taints, nil
}
func parseTaint(taintSpec string) (*corev1.Taint, error) {
var taint corev1.Taint
parts := strings.Split(taintSpec, ":")
switch len(parts) {
case 1:
taint.Key = parts[0]
case 2:
taint.Effect = corev1.TaintEffect(parts[1])
partsKV := strings.Split(parts[0], "=")
if len(partsKV) > 2 {
return nil, fmt.Errorf("invalid taint spec: %v", taintSpec)
}
taint.Key = partsKV[0]
if len(partsKV) == 2 {
taint.Value = partsKV[1]
}
default:
return nil, fmt.Errorf("invalid taint spec: %v", taintSpec)
}
return &taint, nil
}
func parseStringSlice(str string) ([]string, error) {
v := strings.Split(str, ",")
for i, s := range v {

69
upup/pkg/fi/cloudup/spotinsttasks/elastigroup.go
View File

@ -28,6 +28,7 @@ import (
"github.com/spotinst/spotinst-sdk-go/service/elastigroup/providers/aws"
"github.com/spotinst/spotinst-sdk-go/spotinst/client"
"github.com/spotinst/spotinst-sdk-go/spotinst/util/stringutil"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/klog"
"k8s.io/kops/pkg/resources/spotinst"
@ -79,7 +80,9 @@ type RootVolumeOpts struct {
type AutoScalerOpts struct {
Enabled *bool
ClusterID *string
Cooldown *int
Labels map[string]string
Taints []*corev1.Taint
Headroom *AutoScalerHeadroomOpts
Down *AutoScalerDownOpts
}
@ -274,7 +277,7 @@ func (e *Elastigroup) Find(c *fi.Context) (*Elastigroup, error) {
// EBS optimization.
{
if lc.EBSOptimized != nil {
if fi.BoolValue(lc.EBSOptimized) {
if actual.RootVolumeOpts == nil {
actual.RootVolumeOpts = new(RootVolumeOpts)
}
@ -286,18 +289,22 @@ func (e *Elastigroup) Find(c *fi.Context) (*Elastigroup, error) {
// User data.
{
var userData []byte
if lc.UserData != nil {
userData, err := base64.StdEncoding.DecodeString(fi.StringValue(lc.UserData))
userData, err = base64.StdEncoding.DecodeString(fi.StringValue(lc.UserData))
if err != nil {
return nil, err
}
actual.UserData = fi.WrapResource(fi.NewStringResource(string(userData)))
}
actual.UserData = fi.WrapResource(fi.NewStringResource(string(userData)))
}
// Network interfaces.
{
associatePublicIP := false
if lc.NetworkInterfaces != nil && len(lc.NetworkInterfaces) > 0 {
for _, iface := range lc.NetworkInterfaces {
if fi.BoolValue(iface.AssociatePublicIPAddress) {
@ -306,6 +313,7 @@ func (e *Elastigroup) Find(c *fi.Context) (*Elastigroup, error) {
}
}
}
actual.AssociatePublicIP = fi.Bool(associatePublicIP)
}
@ -364,14 +372,23 @@ func (e *Elastigroup) Find(c *fi.Context) (*Elastigroup, error) {
if integration.AutoScale != nil {
actual.AutoScalerOpts.Enabled = integration.AutoScale.IsEnabled
actual.AutoScalerOpts.Cooldown = integration.AutoScale.Cooldown
// Headroom.
if headroom := integration.AutoScale.Headroom; headroom != nil {
actual.AutoScalerOpts.Headroom = &AutoScalerHeadroomOpts{
CPUPerUnit: headroom.CPUPerUnit,
GPUPerUnit: headroom.GPUPerUnit,
MemPerUnit: headroom.MemoryPerUnit,
NumOfUnits: headroom.NumOfUnits,
actual.AutoScalerOpts.Headroom = new(AutoScalerHeadroomOpts)
if v := fi.IntValue(headroom.CPUPerUnit); v > 0 {
actual.AutoScalerOpts.Headroom.CPUPerUnit = headroom.CPUPerUnit
}
if v := fi.IntValue(headroom.GPUPerUnit); v > 0 {
actual.AutoScalerOpts.Headroom.GPUPerUnit = headroom.GPUPerUnit
}
if v := fi.IntValue(headroom.MemoryPerUnit); v > 0 {
actual.AutoScalerOpts.Headroom.MemPerUnit = headroom.MemoryPerUnit
}
if v := fi.IntValue(headroom.NumOfUnits); v > 0 {
actual.AutoScalerOpts.Headroom.NumOfUnits = headroom.NumOfUnits
}
}
@ -537,8 +554,11 @@ func (_ *Elastigroup) create(cloud awsup.AWSCloud, a, e, changes *Elastigroup) e
if err != nil {
return err
}
encoded := base64.StdEncoding.EncodeToString([]byte(userData))
group.Compute.LaunchSpecification.SetUserData(fi.String(encoded))
if len(userData) > 0 {
encoded := base64.StdEncoding.EncodeToString([]byte(userData))
group.Compute.LaunchSpecification.SetUserData(fi.String(encoded))
}
}
}
@ -623,6 +643,7 @@ func (_ *Elastigroup) create(cloud awsup.AWSCloud, a, e, changes *Elastigroup) e
autoScaler := new(aws.AutoScaleKubernetes)
autoScaler.IsEnabled = opts.Enabled
autoScaler.IsAutoConfig = fi.Bool(true)
autoScaler.Cooldown = opts.Cooldown
// Headroom.
if headroom := opts.Headroom; headroom != nil {
@ -858,22 +879,25 @@ func (_ *Elastigroup) update(cloud awsup.AWSCloud, a, e, changes *Elastigroup) e
// User data.
{
if changes.UserData != nil {
if group.Compute == nil {
group.Compute = new(aws.Compute)
}
if group.Compute.LaunchSpecification == nil {
group.Compute.LaunchSpecification = new(aws.LaunchSpecification)
}
userData, err := e.UserData.AsString()
if err != nil {
return err
}
encoded := base64.StdEncoding.EncodeToString([]byte(userData))
group.Compute.LaunchSpecification.SetUserData(fi.String(encoded))
if len(userData) > 0 {
if group.Compute == nil {
group.Compute = new(aws.Compute)
}
if group.Compute.LaunchSpecification == nil {
group.Compute.LaunchSpecification = new(aws.LaunchSpecification)
}
encoded := base64.StdEncoding.EncodeToString([]byte(userData))
group.Compute.LaunchSpecification.SetUserData(fi.String(encoded))
changed = true
}
changes.UserData = nil
changed = true
}
}
@ -1146,6 +1170,7 @@ func (_ *Elastigroup) update(cloud awsup.AWSCloud, a, e, changes *Elastigroup) e
if opts.Enabled != nil {
autoScaler := new(aws.AutoScaleKubernetes)
autoScaler.IsEnabled = e.AutoScalerOpts.Enabled
autoScaler.Cooldown = e.AutoScalerOpts.Cooldown
// Headroom.
if headroom := opts.Headroom; headroom != nil {
@ -1156,7 +1181,7 @@ func (_ *Elastigroup) update(cloud awsup.AWSCloud, a, e, changes *Elastigroup) e
MemoryPerUnit: e.AutoScalerOpts.Headroom.MemPerUnit,
NumOfUnits: e.AutoScalerOpts.Headroom.NumOfUnits,
}
} else if a.AutoScalerOpts.Headroom != nil {
} else if a.AutoScalerOpts != nil && a.AutoScalerOpts.Headroom != nil {
autoScaler.IsAutoConfig = fi.Bool(true)
autoScaler.SetHeadroom(nil)
}
@ -1293,6 +1318,7 @@ type terraformElastigroupIntegration struct {
type terraformAutoScaler struct {
Enabled *bool `json:"autoscale_is_enabled,omitempty" cty:"autoscale_is_enabled"`
AutoConfig *bool `json:"autoscale_is_auto_config,omitempty" cty:"autoscale_is_auto_config"`
Cooldown *int `json:"autoscale_cooldown,omitempty" cty:"autoscale_cooldown"`
Headroom *terraformAutoScalerHeadroom `json:"autoscale_headroom,omitempty" cty:"autoscale_headroom"`
Down *terraformAutoScalerDown `json:"autoscale_down,omitempty" cty:"autoscale_down"`
Labels []*terraformKV `json:"autoscale_labels,omitempty" cty:"autoscale_labels"`
@ -1499,6 +1525,7 @@ func (_ *Elastigroup) RenderTerraform(t *terraform.TerraformTarget, a, e, change
tf.Integration.terraformAutoScaler = &terraformAutoScaler{
Enabled: opts.Enabled,
AutoConfig: fi.Bool(true),
Cooldown: opts.Cooldown,
}
// Headroom.

326
upup/pkg/fi/cloudup/spotinsttasks/launch_spec.go
View File

@ -25,6 +25,7 @@ import (
"github.com/spotinst/spotinst-sdk-go/service/ocean/providers/aws"
"github.com/spotinst/spotinst-sdk-go/spotinst/util/stringutil"
corev1 "k8s.io/api/core/v1"
"k8s.io/klog"
"k8s.io/kops/pkg/resources/spotinst"
"k8s.io/kops/upup/pkg/fi"
@ -41,10 +42,11 @@ type LaunchSpec struct {
ID *string
UserData *fi.ResourceHolder
SecurityGroups []*awstasks.SecurityGroup
Subnets []*awstasks.Subnet
IAMInstanceProfile *awstasks.IAMInstanceProfile
ImageID *string
Tags map[string]string
Labels map[string]string
AutoScalerOpts *AutoScalerOpts
Ocean *Ocean
}
@ -70,6 +72,12 @@ func (o *LaunchSpec) GetDependencies(tasks map[string]fi.Task) []fi.Task {
}
}
if o.Subnets != nil {
for _, subnet := range o.Subnets {
deps = append(deps, subnet)
}
}
if o.Ocean != nil {
deps = append(deps, o.Ocean)
}
@ -144,18 +152,23 @@ func (o *LaunchSpec) Find(c *fi.Context) (*LaunchSpec, error) {
// User data.
{
var userData []byte
if spec.UserData != nil {
userData, err := base64.StdEncoding.DecodeString(fi.StringValue(spec.UserData))
userData, err = base64.StdEncoding.DecodeString(fi.StringValue(spec.UserData))
if err != nil {
return nil, err
}
actual.UserData = fi.WrapResource(fi.NewStringResource(string(userData)))
}
actual.UserData = fi.WrapResource(fi.NewStringResource(string(userData)))
}
// IAM instance profile.
if spec.IAMInstanceProfile != nil {
actual.IAMInstanceProfile = &awstasks.IAMInstanceProfile{Name: spec.IAMInstanceProfile.Name}
{
if spec.IAMInstanceProfile != nil {
actual.IAMInstanceProfile = &awstasks.IAMInstanceProfile{Name: spec.IAMInstanceProfile.Name}
}
}
// Security groups.
@ -168,6 +181,19 @@ func (o *LaunchSpec) Find(c *fi.Context) (*LaunchSpec, error) {
}
}
// Subnets.
{
if spec.SubnetIDs != nil {
for _, subnetID := range spec.SubnetIDs {
actual.Subnets = append(actual.Subnets,
&awstasks.Subnet{ID: fi.String(subnetID)})
}
if subnetSlicesEqualIgnoreOrder(actual.Subnets, o.Subnets) {
actual.Subnets = o.Subnets
}
}
}
// Tags.
{
if len(spec.Tags) > 0 {
@ -178,12 +204,48 @@ func (o *LaunchSpec) Find(c *fi.Context) (*LaunchSpec, error) {
}
}
// Labels.
if spec.Labels != nil {
actual.Labels = make(map[string]string)
// Auto Scaler.
{
if spec.AutoScale != nil {
actual.AutoScalerOpts = new(AutoScalerOpts)
// Headroom.
if headrooms := spec.AutoScale.Headrooms; len(headrooms) > 0 {
actual.AutoScalerOpts.Headroom = &AutoScalerHeadroomOpts{
CPUPerUnit: headrooms[0].CPUPerUnit,
GPUPerUnit: headrooms[0].GPUPerUnit,
MemPerUnit: headrooms[0].MemoryPerUnit,
NumOfUnits: headrooms[0].NumOfUnits,
}
}
}
}
// Labels.
if labels := spec.Labels; labels != nil {
if actual.AutoScalerOpts == nil {
actual.AutoScalerOpts = new(AutoScalerOpts)
}
actual.AutoScalerOpts.Labels = make(map[string]string)
for _, label := range spec.Labels {
actual.Labels[fi.StringValue(label.Key)] = fi.StringValue(label.Value)
actual.AutoScalerOpts.Labels[fi.StringValue(label.Key)] = fi.StringValue(label.Value)
}
}
// Taints.
if spec.Taints != nil {
if actual.AutoScalerOpts == nil {
actual.AutoScalerOpts = new(AutoScalerOpts)
}
actual.AutoScalerOpts.Taints = make([]*corev1.Taint, len(spec.Taints))
for i, taint := range spec.Taints {
actual.AutoScalerOpts.Taints[i] = &corev1.Taint{
Key: fi.StringValue(taint.Key),
Value: fi.StringValue(taint.Value),
Effect: corev1.TaintEffect(fi.StringValue(taint.Effect)),
}
}
}
@ -251,8 +313,11 @@ func (_ *LaunchSpec) create(cloud awsup.AWSCloud, a, e, changes *LaunchSpec) err
if err != nil {
return err
}
encoded := base64.StdEncoding.EncodeToString([]byte(userData))
spec.SetUserData(fi.String(encoded))
if len(userData) > 0 {
encoded := base64.StdEncoding.EncodeToString([]byte(userData))
spec.SetUserData(fi.String(encoded))
}
}
}
@ -276,6 +341,17 @@ func (_ *LaunchSpec) create(cloud awsup.AWSCloud, a, e, changes *LaunchSpec) err
}
}
// Subnets.
{
if e.Subnets != nil {
subnetIDs := make([]string, len(e.Subnets))
for i, subnet := range e.Subnets {
subnetIDs[i] = fi.StringValue(subnet.ID)
}
spec.SetSubnetIDs(subnetIDs)
}
}
// Tags.
{
if e.Tags != nil {
@ -283,17 +359,47 @@ func (_ *LaunchSpec) create(cloud awsup.AWSCloud, a, e, changes *LaunchSpec) err
}
}
// Labels.
// Auto Scaler.
{
if e.Labels != nil && len(e.Labels) > 0 {
var labels []*aws.Label
for k, v := range e.Labels {
labels = append(labels, &aws.Label{
Key: fi.String(k),
Value: fi.String(v),
})
if opts := e.AutoScalerOpts; opts != nil {
// Headroom.
if headroom := opts.Headroom; headroom != nil {
autoScale := new(aws.AutoScale)
autoScale.Headrooms = []*aws.AutoScaleHeadroom{
{
CPUPerUnit: headroom.CPUPerUnit,
GPUPerUnit: headroom.GPUPerUnit,
MemoryPerUnit: headroom.MemPerUnit,
NumOfUnits: headroom.NumOfUnits,
},
}
spec.SetAutoScale(autoScale)
}
// Labels.
if len(opts.Labels) > 0 {
var labels []*aws.Label
for k, v := range opts.Labels {
labels = append(labels, &aws.Label{
Key: fi.String(k),
Value: fi.String(v),
})
}
spec.SetLabels(labels)
}
// Taints.
if len(opts.Taints) > 0 {
taints := make([]*aws.Taint, len(opts.Taints))
for i, taint := range opts.Taints {
taints[i] = &aws.Taint{
Key: fi.String(taint.Key),
Value: fi.String(taint.Value),
Effect: fi.String(string(taint.Effect)),
}
}
spec.SetTaints(taints)
}
spec.SetLabels(labels)
}
}
@ -350,11 +456,14 @@ func (_ *LaunchSpec) update(cloud awsup.AWSCloud, a, e, changes *LaunchSpec) err
if err != nil {
return err
}
encoded := base64.StdEncoding.EncodeToString([]byte(userData))
spec.SetUserData(fi.String(encoded))
if len(userData) > 0 {
encoded := base64.StdEncoding.EncodeToString([]byte(userData))
spec.SetUserData(fi.String(encoded))
changed = true
}
changes.UserData = nil
changed = true
}
}
@ -384,6 +493,20 @@ func (_ *LaunchSpec) update(cloud awsup.AWSCloud, a, e, changes *LaunchSpec) err
}
}
// Subnets.
{
if changes.Subnets != nil {
subnetIDs := make([]string, len(e.Subnets))
for i, subnet := range e.Subnets {
subnetIDs[i] = fi.StringValue(subnet.ID)
}
spec.SetSubnetIDs(subnetIDs)
changes.Subnets = nil
changed = true
}
}
// Tags.
{
if changes.Tags != nil {
@ -393,23 +516,58 @@ func (_ *LaunchSpec) update(cloud awsup.AWSCloud, a, e, changes *LaunchSpec) err
}
}
// Labels.
// Auto Scaler.
{
if changes.Labels != nil {
labels := make([]*aws.Label, 0, len(e.Labels))
for k, v := range e.Labels {
labels = append(labels, &aws.Label{
Key: fi.String(k),
Value: fi.String(v),
})
if opts := changes.AutoScalerOpts; opts != nil {
// Headroom.
if headroom := opts.Headroom; headroom != nil {
autoScale := new(aws.AutoScale)
autoScale.Headrooms = []*aws.AutoScaleHeadroom{
{
CPUPerUnit: e.AutoScalerOpts.Headroom.CPUPerUnit,
GPUPerUnit: e.AutoScalerOpts.Headroom.GPUPerUnit,
MemoryPerUnit: e.AutoScalerOpts.Headroom.MemPerUnit,
NumOfUnits: e.AutoScalerOpts.Headroom.NumOfUnits,
},
}
spec.SetAutoScale(autoScale)
opts.Headroom = nil
changed = true
}
spec.SetLabels(labels)
changes.Labels = nil
changed = true
} else if a.Labels != nil { // Forcibly remove all existing labels.
spec.SetLabels(nil)
changed = true
// Labels.
if opts.Labels != nil {
labels := make([]*aws.Label, 0, len(e.AutoScalerOpts.Labels))
for k, v := range e.AutoScalerOpts.Labels {
labels = append(labels, &aws.Label{
Key: fi.String(k),
Value: fi.String(v),
})
}
spec.SetLabels(labels)
opts.Labels = nil
changed = true
}
// Taints.
if opts.Taints != nil {
taints := make([]*aws.Taint, 0, len(e.AutoScalerOpts.Taints))
for _, taint := range e.AutoScalerOpts.Taints {
taints = append(taints, &aws.Taint{
Key: fi.String(taint.Key),
Value: fi.String(taint.Value),
Effect: fi.String(string(taint.Effect)),
})
}
spec.SetTaints(taints)
opts.Taints = nil
changed = true
}
changes.AutoScalerOpts = nil
}
}
@ -456,12 +614,14 @@ func (_ *LaunchSpec) RenderTerraform(t *terraform.TerraformTarget, a, e, changes
}
// Image.
if e.ImageID != nil {
image, err := resolveImage(cloud, fi.StringValue(e.ImageID))
if err != nil {
return err
{
if e.ImageID != nil {
image, err := resolveImage(cloud, fi.StringValue(e.ImageID))
if err != nil {
return err
}
tf.ImageID = image.ImageId
}
tf.ImageID = image.ImageId
}
var role string
@ -476,29 +636,49 @@ func (_ *LaunchSpec) RenderTerraform(t *terraform.TerraformTarget, a, e, changes
}
// Security groups.
if e.SecurityGroups != nil {
for _, sg := range e.SecurityGroups {
tf.SecurityGroups = append(tf.SecurityGroups, sg.TerraformLink())
if role != "" {
if err := t.AddOutputVariableArray(role+"_security_groups", sg.TerraformLink()); err != nil {
return err
{
if e.SecurityGroups != nil {
for _, sg := range e.SecurityGroups {
tf.SecurityGroups = append(tf.SecurityGroups, sg.TerraformLink())
if role != "" {
if err := t.AddOutputVariableArray(role+"_security_groups", sg.TerraformLink()); err != nil {
return err
}
}
}
}
}
// Subnets.
{
if e.Subnets != nil {
for _, subnet := range e.Subnets {
tf.SubnetIDs = append(tf.SubnetIDs, subnet.TerraformLink())
if role != "" {
if err := t.AddOutputVariableArray(role+"_subnet_ids", subnet.TerraformLink()); err != nil {
return err
}
}
}
}
}
// User data.
if e.UserData != nil {
var err error
tf.UserData, err = t.AddFile("spotinst_ocean_aws_launch_spec", *e.Name, "user_data", e.UserData)
if err != nil {
return err
{
if e.UserData != nil {
var err error
tf.UserData, err = t.AddFile("spotinst_ocean_aws_launch_spec", *e.Name, "user_data", e.UserData)
if err != nil {
return err
}
}
}
// IAM instance profile.
if e.IAMInstanceProfile != nil {
tf.IAMInstanceProfile = e.IAMInstanceProfile.TerraformLink()
{
if e.IAMInstanceProfile != nil {
tf.IAMInstanceProfile = e.IAMInstanceProfile.TerraformLink()
}
}
// Tags.
@ -513,15 +693,35 @@ func (_ *LaunchSpec) RenderTerraform(t *terraform.TerraformTarget, a, e, changes
}
}
// Labels.
// Auto Scaler.
{
if e.Labels != nil {
tf.Labels = make([]*terraformKV, 0, len(e.Labels))
for k, v := range e.Labels {
tf.Labels = append(tf.Labels, &terraformKV{
Key: fi.String(k),
Value: fi.String(v),
})
if opts := e.AutoScalerOpts; opts != nil {
// Headroom.
if headroom := opts.Headroom; headroom != nil {
tf.Headrooms = []*terraformAutoScalerHeadroom{
{
CPUPerUnit: headroom.CPUPerUnit,
GPUPerUnit: headroom.GPUPerUnit,
MemPerUnit: headroom.MemPerUnit,
NumOfUnits: headroom.NumOfUnits,
},
}
}
// Labels.
if len(opts.Labels) > 0 {
tf.Labels = make([]*terraformKV, 0, len(opts.Labels))
for k, v := range opts.Labels {
tf.Labels = append(tf.Labels, &terraformKV{
Key: fi.String(k),
Value: fi.String(v),
})
}
}
// Taints.
if len(opts.Taints) > 0 {
tf.Taints = opts.Taints
}
}
}

105
upup/pkg/fi/cloudup/spotinsttasks/ocean.go
View File

@ -27,6 +27,7 @@ import (
"github.com/spotinst/spotinst-sdk-go/service/ocean/providers/aws"
"github.com/spotinst/spotinst-sdk-go/spotinst/client"
"github.com/spotinst/spotinst-sdk-go/spotinst/util/stringutil"
corev1 "k8s.io/api/core/v1"
"k8s.io/klog"
"k8s.io/kops/pkg/resources/spotinst"
"k8s.io/kops/upup/pkg/fi"
@ -222,18 +223,21 @@ func (o *Ocean) Find(c *fi.Context) (*Ocean, error) {
// User data.
{
var userData []byte
if lc.UserData != nil {
userData, err := base64.StdEncoding.DecodeString(fi.StringValue(lc.UserData))
userData, err = base64.StdEncoding.DecodeString(fi.StringValue(lc.UserData))
if err != nil {
return nil, err
}
actual.UserData = fi.WrapResource(fi.NewStringResource(string(userData)))
}
actual.UserData = fi.WrapResource(fi.NewStringResource(string(userData)))
}
// EBS optimization.
{
if lc.EBSOptimized != nil {
if fi.BoolValue(lc.EBSOptimized) {
if actual.RootVolumeOpts == nil {
actual.RootVolumeOpts = new(RootVolumeOpts)
}
@ -275,6 +279,7 @@ func (o *Ocean) Find(c *fi.Context) (*Ocean, error) {
actual.AutoScalerOpts = new(AutoScalerOpts)
actual.AutoScalerOpts.ClusterID = ocean.ControllerClusterID
actual.AutoScalerOpts.Enabled = ocean.AutoScaler.IsEnabled
actual.AutoScalerOpts.Cooldown = ocean.AutoScaler.Cooldown
// Headroom.
if headroom := ocean.AutoScaler.Headroom; headroom != nil {
@ -285,6 +290,14 @@ func (o *Ocean) Find(c *fi.Context) (*Ocean, error) {
NumOfUnits: headroom.NumOfUnits,
}
}
// Scale down.
if down := ocean.AutoScaler.Down; down != nil {
actual.AutoScalerOpts.Down = &AutoScalerDownOpts{
MaxPercentage: down.MaxScaleDownPercentage,
EvaluationPeriods: down.EvaluationPeriods,
}
}
}
}
@ -410,8 +423,11 @@ func (_ *Ocean) create(cloud awsup.AWSCloud, a, e, changes *Ocean) error {
if err != nil {
return err
}
encoded := base64.StdEncoding.EncodeToString([]byte(userData))
ocean.Compute.LaunchSpecification.SetUserData(fi.String(encoded))
if len(userData) > 0 {
encoded := base64.StdEncoding.EncodeToString([]byte(userData))
ocean.Compute.LaunchSpecification.SetUserData(fi.String(encoded))
}
}
}
@ -476,6 +492,7 @@ func (_ *Ocean) create(cloud awsup.AWSCloud, a, e, changes *Ocean) error {
autoScaler := new(aws.AutoScaler)
autoScaler.IsEnabled = opts.Enabled
autoScaler.IsAutoConfig = fi.Bool(true)
autoScaler.Cooldown = opts.Cooldown
// Headroom.
if headroom := opts.Headroom; headroom != nil {
@ -488,6 +505,14 @@ func (_ *Ocean) create(cloud awsup.AWSCloud, a, e, changes *Ocean) error {
}
}
// Scale down.
if down := opts.Down; down != nil {
autoScaler.Down = &aws.AutoScalerDown{
MaxScaleDownPercentage: down.MaxPercentage,
EvaluationPeriods: down.EvaluationPeriods,
}
}
ocean.SetAutoScaler(autoScaler)
}
}
@ -668,22 +693,25 @@ func (_ *Ocean) update(cloud awsup.AWSCloud, a, e, changes *Ocean) error {
// User data.
{
if changes.UserData != nil {
if ocean.Compute == nil {
ocean.Compute = new(aws.Compute)
}
if ocean.Compute.LaunchSpecification == nil {
ocean.Compute.LaunchSpecification = new(aws.LaunchSpecification)
}
userData, err := e.UserData.AsString()
if err != nil {
return err
}
encoded := base64.StdEncoding.EncodeToString([]byte(userData))
ocean.Compute.LaunchSpecification.SetUserData(fi.String(encoded))
if len(userData) > 0 {
if ocean.Compute == nil {
ocean.Compute = new(aws.Compute)
}
if ocean.Compute.LaunchSpecification == nil {
ocean.Compute.LaunchSpecification = new(aws.LaunchSpecification)
}
encoded := base64.StdEncoding.EncodeToString([]byte(userData))
ocean.Compute.LaunchSpecification.SetUserData(fi.String(encoded))
changed = true
}
changes.UserData = nil
changed = true
}
}
@ -863,6 +891,7 @@ func (_ *Ocean) update(cloud awsup.AWSCloud, a, e, changes *Ocean) error {
if opts.Enabled != nil {
autoScaler := new(aws.AutoScaler)
autoScaler.IsEnabled = e.AutoScalerOpts.Enabled
autoScaler.Cooldown = e.AutoScalerOpts.Cooldown
// Headroom.
if headroom := opts.Headroom; headroom != nil {
@ -873,11 +902,21 @@ func (_ *Ocean) update(cloud awsup.AWSCloud, a, e, changes *Ocean) error {
MemoryPerUnit: e.AutoScalerOpts.Headroom.MemPerUnit,
NumOfUnits: e.AutoScalerOpts.Headroom.NumOfUnits,
}
} else {
} else if a.AutoScalerOpts != nil && a.AutoScalerOpts.Headroom != nil {
autoScaler.IsAutoConfig = fi.Bool(true)
autoScaler.SetHeadroom(nil)
}
// Scale down.
if down := opts.Down; down != nil {
autoScaler.Down = &aws.AutoScalerDown{
MaxScaleDownPercentage: down.MaxPercentage,
EvaluationPeriods: down.EvaluationPeriods,
}
} else if a.AutoScalerOpts.Down != nil {
autoScaler.SetDown(nil)
}
ocean.SetAutoScaler(autoScaler)
changed = true
}
@ -941,17 +980,20 @@ type terraformOceanStrategy struct {
}
type terraformOceanLaunchSpec struct {
Monitoring *bool `json:"monitoring,omitempty" cty:"monitoring"`
EBSOptimized *bool `json:"ebs_optimized,omitempty" cty:"ebs_optimized"`
ImageID *string `json:"image_id,omitempty" cty:"image_id"`
AssociatePublicIPAddress *bool `json:"associate_public_ip_address,omitempty" cty:"associate_public_ip_address"`
RootVolumeSize *int32 `json:"root_volume_size,omitempty" cty:"root_volume_size"`
UserData *terraform.Literal `json:"user_data,omitempty" cty:"user_data"`
IAMInstanceProfile *terraform.Literal `json:"iam_instance_profile,omitempty" cty:"iam_instance_profile"`
KeyName *terraform.Literal `json:"key_name,omitempty" cty:"key_name"`
SecurityGroups []*terraform.Literal `json:"security_groups,omitempty" cty:"security_groups"`
Labels []*terraformKV `json:"labels,omitempty" cty:"labels"`
Tags []*terraformKV `json:"tags,omitempty" cty:"tags"`
Monitoring *bool `json:"monitoring,omitempty" cty:"monitoring"`
EBSOptimized *bool `json:"ebs_optimized,omitempty" cty:"ebs_optimized"`
ImageID *string `json:"image_id,omitempty" cty:"image_id"`
AssociatePublicIPAddress *bool `json:"associate_public_ip_address,omitempty" cty:"associate_public_ip_address"`
RootVolumeSize *int32 `json:"root_volume_size,omitempty" cty:"root_volume_size"`
UserData *terraform.Literal `json:"user_data,omitempty" cty:"user_data"`
IAMInstanceProfile *terraform.Literal `json:"iam_instance_profile,omitempty" cty:"iam_instance_profile"`
KeyName *terraform.Literal `json:"key_name,omitempty" cty:"key_name"`
SubnetIDs []*terraform.Literal `json:"subnet_ids,omitempty" cty:"subnet_ids"`
SecurityGroups []*terraform.Literal `json:"security_groups,omitempty" cty:"security_groups"`
Taints []*corev1.Taint `json:"taints,omitempty" cty:"taints"`
Labels []*terraformKV `json:"labels,omitempty" cty:"labels"`
Tags []*terraformKV `json:"tags,omitempty" cty:"tags"`
Headrooms []*terraformAutoScalerHeadroom `json:"autoscale_headrooms,omitempty" cty:"autoscale_headrooms"`
}
func (_ *Ocean) RenderTerraform(t *terraform.TerraformTarget, a, e, changes *Ocean) error {
@ -1083,6 +1125,7 @@ func (_ *Ocean) RenderTerraform(t *terraform.TerraformTarget, a, e, changes *Oce
tf.AutoScaler = &terraformAutoScaler{
Enabled: opts.Enabled,
AutoConfig: fi.Bool(true),
Cooldown: opts.Cooldown,
}
// Headroom.
@ -1096,6 +1139,14 @@ func (_ *Ocean) RenderTerraform(t *terraform.TerraformTarget, a, e, changes *Oce
}
}
// Scale down.
if down := opts.Down; down != nil {
tf.AutoScaler.Down = &terraformAutoScalerDown{
MaxPercentage: down.MaxPercentage,
EvaluationPeriods: down.EvaluationPeriods,
}
}
// Ignore capacity changes because the auto scaler updates the
// desired capacity overtime.
if fi.BoolValue(tf.AutoScaler.Enabled) {