kops/upup/pkg/fi/cloudup/spotinsttasks/launch_spec.go

748 lines
17 KiB
Go

/*
Copyright 2019 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package spotinsttasks
import (
"context"
"encoding/base64"
"fmt"
"reflect"
"strings"
"github.com/spotinst/spotinst-sdk-go/service/ocean/providers/aws"
"github.com/spotinst/spotinst-sdk-go/spotinst/util/stringutil"
corev1 "k8s.io/api/core/v1"
"k8s.io/klog"
"k8s.io/kops/pkg/resources/spotinst"
"k8s.io/kops/upup/pkg/fi"
"k8s.io/kops/upup/pkg/fi/cloudup/awstasks"
"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
"k8s.io/kops/upup/pkg/fi/cloudup/terraform"
)
//go:generate fitask -type=LaunchSpec
type LaunchSpec struct {
Name *string
Lifecycle *fi.Lifecycle
ID *string
UserData *fi.ResourceHolder
SecurityGroups []*awstasks.SecurityGroup
Subnets []*awstasks.Subnet
IAMInstanceProfile *awstasks.IAMInstanceProfile
ImageID *string
Tags map[string]string
AutoScalerOpts *AutoScalerOpts
Ocean *Ocean
}
var _ fi.CompareWithID = &LaunchSpec{}
func (o *LaunchSpec) CompareWithID() *string {
return o.Name
}
var _ fi.HasDependencies = &LaunchSpec{}
func (o *LaunchSpec) GetDependencies(tasks map[string]fi.Task) []fi.Task {
var deps []fi.Task
if o.IAMInstanceProfile != nil {
deps = append(deps, o.IAMInstanceProfile)
}
if o.SecurityGroups != nil {
for _, sg := range o.SecurityGroups {
deps = append(deps, sg)
}
}
if o.Subnets != nil {
for _, subnet := range o.Subnets {
deps = append(deps, subnet)
}
}
if o.Ocean != nil {
deps = append(deps, o.Ocean)
}
return deps
}
func (o *LaunchSpec) find(svc spotinst.LaunchSpecService, oceanID string) (*aws.LaunchSpec, error) {
klog.V(4).Infof("Attempting to find LaunchSpec: %q", fi.StringValue(o.Name))
specs, err := svc.List(context.Background(), oceanID)
if err != nil {
return nil, fmt.Errorf("spotinst: failed to find launch spec %q: %v", fi.StringValue(o.Name), err)
}
if len(specs) == 0 {
return nil, fmt.Errorf("spotinst: no launch specs associated with ocean %q", oceanID)
}
var out *aws.LaunchSpec
for _, spec := range specs {
if spec.Name() == fi.StringValue(o.Name) {
out = spec.Obj().(*aws.LaunchSpec)
break
}
}
if out == nil {
return nil, fmt.Errorf("spotinst: failed to find launch spec %q", fi.StringValue(o.Name))
}
klog.V(4).Infof("LaunchSpec/%s: %s", fi.StringValue(o.Name), stringutil.Stringify(out))
return out, nil
}
var _ fi.HasCheckExisting = &LaunchSpec{}
func (o *LaunchSpec) Find(c *fi.Context) (*LaunchSpec, error) {
cloud := c.Cloud.(awsup.AWSCloud)
ocean, err := o.Ocean.find(cloud.Spotinst().Ocean(), *o.Ocean.Name)
if err != nil {
return nil, err
}
spec, err := o.find(cloud.Spotinst().LaunchSpec(), *ocean.ID)
if err != nil {
return nil, err
}
actual := &LaunchSpec{}
actual.ID = spec.ID
actual.Name = spec.Name
actual.Ocean = &Ocean{
ID: ocean.ID,
Name: o.Ocean.Name,
}
// Image.
{
actual.ImageID = spec.ImageID
if o.ImageID != nil && actual.ImageID != nil &&
fi.StringValue(actual.ImageID) != fi.StringValue(o.ImageID) {
image, err := resolveImage(cloud, fi.StringValue(o.ImageID))
if err != nil {
return nil, err
}
if fi.StringValue(image.ImageId) == fi.StringValue(spec.ImageID) {
actual.ImageID = o.ImageID
}
}
}
// User data.
{
var userData []byte
if spec.UserData != nil {
userData, err = base64.StdEncoding.DecodeString(fi.StringValue(spec.UserData))
if err != nil {
return nil, err
}
}
actual.UserData = fi.WrapResource(fi.NewStringResource(string(userData)))
}
// IAM instance profile.
{
if spec.IAMInstanceProfile != nil {
actual.IAMInstanceProfile = &awstasks.IAMInstanceProfile{Name: spec.IAMInstanceProfile.Name}
}
}
// Security groups.
{
if spec.SecurityGroupIDs != nil {
for _, sgID := range spec.SecurityGroupIDs {
actual.SecurityGroups = append(actual.SecurityGroups,
&awstasks.SecurityGroup{ID: fi.String(sgID)})
}
}
}
// Subnets.
{
if spec.SubnetIDs != nil {
for _, subnetID := range spec.SubnetIDs {
actual.Subnets = append(actual.Subnets,
&awstasks.Subnet{ID: fi.String(subnetID)})
}
if subnetSlicesEqualIgnoreOrder(actual.Subnets, o.Subnets) {
actual.Subnets = o.Subnets
}
}
}
// Tags.
{
if len(spec.Tags) > 0 {
actual.Tags = make(map[string]string)
for _, tag := range spec.Tags {
actual.Tags[fi.StringValue(tag.Key)] = fi.StringValue(tag.Value)
}
}
}
// Auto Scaler.
{
if spec.AutoScale != nil {
actual.AutoScalerOpts = new(AutoScalerOpts)
// Headroom.
if headrooms := spec.AutoScale.Headrooms; len(headrooms) > 0 {
actual.AutoScalerOpts.Headroom = &AutoScalerHeadroomOpts{
CPUPerUnit: headrooms[0].CPUPerUnit,
GPUPerUnit: headrooms[0].GPUPerUnit,
MemPerUnit: headrooms[0].MemoryPerUnit,
NumOfUnits: headrooms[0].NumOfUnits,
}
}
}
}
// Labels.
if labels := spec.Labels; labels != nil {
if actual.AutoScalerOpts == nil {
actual.AutoScalerOpts = new(AutoScalerOpts)
}
actual.AutoScalerOpts.Labels = make(map[string]string)
for _, label := range spec.Labels {
actual.AutoScalerOpts.Labels[fi.StringValue(label.Key)] = fi.StringValue(label.Value)
}
}
// Taints.
if spec.Taints != nil {
if actual.AutoScalerOpts == nil {
actual.AutoScalerOpts = new(AutoScalerOpts)
}
actual.AutoScalerOpts.Taints = make([]*corev1.Taint, len(spec.Taints))
for i, taint := range spec.Taints {
actual.AutoScalerOpts.Taints[i] = &corev1.Taint{
Key: fi.StringValue(taint.Key),
Value: fi.StringValue(taint.Value),
Effect: corev1.TaintEffect(fi.StringValue(taint.Effect)),
}
}
}
// Avoid spurious changes.
actual.Lifecycle = o.Lifecycle
return actual, nil
}
func (o *LaunchSpec) CheckExisting(c *fi.Context) bool {
spec, err := o.Find(c)
return err == nil && spec != nil
}
func (o *LaunchSpec) Run(c *fi.Context) error {
return fi.DefaultDeltaRunMethod(o, c)
}
func (s *LaunchSpec) CheckChanges(a, e, changes *LaunchSpec) error {
if e.Name == nil {
return fi.RequiredField("Name")
}
return nil
}
func (o *LaunchSpec) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *LaunchSpec) error {
return o.createOrUpdate(t.Cloud.(awsup.AWSCloud), a, e, changes)
}
func (o *LaunchSpec) createOrUpdate(cloud awsup.AWSCloud, a, e, changes *LaunchSpec) error {
if a == nil {
return o.create(cloud, a, e, changes)
} else {
return o.update(cloud, a, e, changes)
}
}
func (_ *LaunchSpec) create(cloud awsup.AWSCloud, a, e, changes *LaunchSpec) error {
ocean, err := e.Ocean.find(cloud.Spotinst().Ocean(), *e.Ocean.Name)
if err != nil {
return err
}
klog.V(2).Infof("Creating Launch Spec for Ocean %q", *ocean.ID)
spec := new(aws.LaunchSpec)
spec.SetName(e.Name)
spec.SetOceanId(ocean.ID)
// Image.
{
if e.ImageID != nil {
image, err := resolveImage(cloud, fi.StringValue(e.ImageID))
if err != nil {
return err
}
spec.SetImageId(image.ImageId)
}
}
// User data.
{
if e.UserData != nil {
userData, err := e.UserData.AsString()
if err != nil {
return err
}
if len(userData) > 0 {
encoded := base64.StdEncoding.EncodeToString([]byte(userData))
spec.SetUserData(fi.String(encoded))
}
}
}
// IAM instance profile.
{
if e.IAMInstanceProfile != nil {
iprof := new(aws.IAMInstanceProfile)
iprof.SetName(e.IAMInstanceProfile.GetName())
spec.SetIAMInstanceProfile(iprof)
}
}
// Security groups.
{
if e.SecurityGroups != nil {
securityGroupIDs := make([]string, len(e.SecurityGroups))
for i, sg := range e.SecurityGroups {
securityGroupIDs[i] = *sg.ID
}
spec.SetSecurityGroupIDs(securityGroupIDs)
}
}
// Subnets.
{
if e.Subnets != nil {
subnetIDs := make([]string, len(e.Subnets))
for i, subnet := range e.Subnets {
subnetIDs[i] = fi.StringValue(subnet.ID)
}
spec.SetSubnetIDs(subnetIDs)
}
}
// Tags.
{
if e.Tags != nil {
spec.SetTags(e.buildTags())
}
}
// Auto Scaler.
{
if opts := e.AutoScalerOpts; opts != nil {
// Headroom.
if headroom := opts.Headroom; headroom != nil {
autoScale := new(aws.AutoScale)
autoScale.Headrooms = []*aws.AutoScaleHeadroom{
{
CPUPerUnit: headroom.CPUPerUnit,
GPUPerUnit: headroom.GPUPerUnit,
MemoryPerUnit: headroom.MemPerUnit,
NumOfUnits: headroom.NumOfUnits,
},
}
spec.SetAutoScale(autoScale)
}
// Labels.
if len(opts.Labels) > 0 {
var labels []*aws.Label
for k, v := range opts.Labels {
labels = append(labels, &aws.Label{
Key: fi.String(k),
Value: fi.String(v),
})
}
spec.SetLabels(labels)
}
// Taints.
if len(opts.Taints) > 0 {
taints := make([]*aws.Taint, len(opts.Taints))
for i, taint := range opts.Taints {
taints[i] = &aws.Taint{
Key: fi.String(taint.Key),
Value: fi.String(taint.Value),
Effect: fi.String(string(taint.Effect)),
}
}
spec.SetTaints(taints)
}
}
}
// Wrap the raw object as an LaunchSpec.
sp, err := spotinst.NewLaunchSpec(cloud.ProviderID(), spec)
if err != nil {
return err
}
// Create a new LaunchSpec.
id, err := cloud.Spotinst().LaunchSpec().Create(context.Background(), sp)
if err != nil {
return fmt.Errorf("spotinst: failed to create launch spec: %v", err)
}
e.ID = fi.String(id)
return nil
}
func (_ *LaunchSpec) update(cloud awsup.AWSCloud, a, e, changes *LaunchSpec) error {
klog.V(2).Infof("Updating Launch Spec for Ocean %q", *a.Ocean.ID)
actual, err := e.find(cloud.Spotinst().LaunchSpec(), *a.Ocean.ID)
if err != nil {
klog.Errorf("Unable to resolve Launch Spec %q, error: %v", *e.Name, err)
return err
}
var changed bool
spec := new(aws.LaunchSpec)
spec.SetId(a.ID)
// Image.
{
if changes.ImageID != nil {
image, err := resolveImage(cloud, fi.StringValue(e.ImageID))
if err != nil {
return err
}
if *actual.ImageID != *image.ImageId {
spec.SetImageId(image.ImageId)
}
changes.ImageID = nil
changed = true
}
}
// User data.
{
if changes.UserData != nil {
userData, err := e.UserData.AsString()
if err != nil {
return err
}
if len(userData) > 0 {
encoded := base64.StdEncoding.EncodeToString([]byte(userData))
spec.SetUserData(fi.String(encoded))
changed = true
}
changes.UserData = nil
}
}
// IAM instance profile.
{
if changes.IAMInstanceProfile != nil {
iprof := new(aws.IAMInstanceProfile)
iprof.SetName(e.IAMInstanceProfile.GetName())
spec.SetIAMInstanceProfile(iprof)
changes.IAMInstanceProfile = nil
changed = true
}
}
// Security groups.
{
if changes.SecurityGroups != nil {
securityGroupIDs := make([]string, len(e.SecurityGroups))
for i, sg := range e.SecurityGroups {
securityGroupIDs[i] = *sg.ID
}
spec.SetSecurityGroupIDs(securityGroupIDs)
changes.SecurityGroups = nil
changed = true
}
}
// Subnets.
{
if changes.Subnets != nil {
subnetIDs := make([]string, len(e.Subnets))
for i, subnet := range e.Subnets {
subnetIDs[i] = fi.StringValue(subnet.ID)
}
spec.SetSubnetIDs(subnetIDs)
changes.Subnets = nil
changed = true
}
}
// Tags.
{
if changes.Tags != nil {
spec.SetTags(e.buildTags())
changes.Tags = nil
changed = true
}
}
// Auto Scaler.
{
if opts := changes.AutoScalerOpts; opts != nil {
// Headroom.
if headroom := opts.Headroom; headroom != nil {
autoScale := new(aws.AutoScale)
autoScale.Headrooms = []*aws.AutoScaleHeadroom{
{
CPUPerUnit: e.AutoScalerOpts.Headroom.CPUPerUnit,
GPUPerUnit: e.AutoScalerOpts.Headroom.GPUPerUnit,
MemoryPerUnit: e.AutoScalerOpts.Headroom.MemPerUnit,
NumOfUnits: e.AutoScalerOpts.Headroom.NumOfUnits,
},
}
spec.SetAutoScale(autoScale)
opts.Headroom = nil
changed = true
}
// Labels.
if opts.Labels != nil {
labels := make([]*aws.Label, 0, len(e.AutoScalerOpts.Labels))
for k, v := range e.AutoScalerOpts.Labels {
labels = append(labels, &aws.Label{
Key: fi.String(k),
Value: fi.String(v),
})
}
spec.SetLabels(labels)
opts.Labels = nil
changed = true
}
// Taints.
if opts.Taints != nil {
taints := make([]*aws.Taint, 0, len(e.AutoScalerOpts.Taints))
for _, taint := range e.AutoScalerOpts.Taints {
taints = append(taints, &aws.Taint{
Key: fi.String(taint.Key),
Value: fi.String(taint.Value),
Effect: fi.String(string(taint.Effect)),
})
}
spec.SetTaints(taints)
opts.Taints = nil
changed = true
}
changes.AutoScalerOpts = nil
}
}
empty := &LaunchSpec{}
if !reflect.DeepEqual(empty, changes) {
klog.Warningf("Not all changes applied to Launch Spec %q: %v", *spec.ID, changes)
}
if !changed {
klog.V(2).Infof("No changes detected in Launch Spec %q", *spec.ID)
return nil
}
klog.V(2).Infof("Updating Launch Spec %q (config: %s)", *spec.ID, stringutil.Stringify(spec))
// Wrap the raw object as an LaunchSpec.
sp, err := spotinst.NewLaunchSpec(cloud.ProviderID(), spec)
if err != nil {
return err
}
// Update an existing LaunchSpec.
if err := cloud.Spotinst().LaunchSpec().Update(context.Background(), sp); err != nil {
return fmt.Errorf("spotinst: failed to update launch spec: %v", err)
}
return nil
}
type terraformLaunchSpec struct {
Name *string `json:"name,omitempty" cty:"name"`
OceanID *terraform.Literal `json:"ocean_id,omitempty" cty:"ocean_id"`
*terraformOceanLaunchSpec
}
func (_ *LaunchSpec) RenderTerraform(t *terraform.TerraformTarget, a, e, changes *LaunchSpec) error {
cloud := t.Cloud.(awsup.AWSCloud)
tf := &terraformLaunchSpec{
Name: e.Name,
OceanID: e.Ocean.TerraformLink(),
terraformOceanLaunchSpec: &terraformOceanLaunchSpec{},
}
// Image.
{
if e.ImageID != nil {
image, err := resolveImage(cloud, fi.StringValue(e.ImageID))
if err != nil {
return err
}
tf.ImageID = image.ImageId
}
}
var role string
for key := range e.Ocean.Tags {
if strings.HasPrefix(key, awstasks.CloudTagInstanceGroupRolePrefix) {
suffix := strings.TrimPrefix(key, awstasks.CloudTagInstanceGroupRolePrefix)
if role != "" && role != suffix {
return fmt.Errorf("spotinst: found multiple role tags %q vs %q", role, suffix)
}
role = suffix
}
}
// Security groups.
{
if e.SecurityGroups != nil {
for _, sg := range e.SecurityGroups {
tf.SecurityGroups = append(tf.SecurityGroups, sg.TerraformLink())
if role != "" {
if err := t.AddOutputVariableArray(role+"_security_groups", sg.TerraformLink()); err != nil {
return err
}
}
}
}
}
// Subnets.
{
if e.Subnets != nil {
for _, subnet := range e.Subnets {
tf.SubnetIDs = append(tf.SubnetIDs, subnet.TerraformLink())
if role != "" {
if err := t.AddOutputVariableArray(role+"_subnet_ids", subnet.TerraformLink()); err != nil {
return err
}
}
}
}
}
// User data.
{
if e.UserData != nil {
var err error
tf.UserData, err = t.AddFile("spotinst_ocean_aws_launch_spec", *e.Name, "user_data", e.UserData)
if err != nil {
return err
}
}
}
// IAM instance profile.
{
if e.IAMInstanceProfile != nil {
tf.IAMInstanceProfile = e.IAMInstanceProfile.TerraformLink()
}
}
// Tags.
{
if e.Tags != nil {
for _, tag := range e.buildTags() {
tf.Tags = append(tf.Tags, &terraformKV{
Key: tag.Key,
Value: tag.Value,
})
}
}
}
// Auto Scaler.
{
if opts := e.AutoScalerOpts; opts != nil {
// Headroom.
if headroom := opts.Headroom; headroom != nil {
tf.Headrooms = []*terraformAutoScalerHeadroom{
{
CPUPerUnit: headroom.CPUPerUnit,
GPUPerUnit: headroom.GPUPerUnit,
MemPerUnit: headroom.MemPerUnit,
NumOfUnits: headroom.NumOfUnits,
},
}
}
// Labels.
if len(opts.Labels) > 0 {
tf.Labels = make([]*terraformKV, 0, len(opts.Labels))
for k, v := range opts.Labels {
tf.Labels = append(tf.Labels, &terraformKV{
Key: fi.String(k),
Value: fi.String(v),
})
}
}
// Taints.
if len(opts.Taints) > 0 {
tf.Taints = opts.Taints
}
}
}
return t.RenderResource("spotinst_ocean_aws_launch_spec", *e.Name, tf)
}
func (o *LaunchSpec) TerraformLink() *terraform.Literal {
return terraform.LiteralProperty("spotinst_ocean_aws_launch_spec", *o.Name, "id")
}
func (o *LaunchSpec) buildTags() []*aws.Tag {
tags := make([]*aws.Tag, 0, len(o.Tags))
for key, value := range o.Tags {
tags = append(tags, &aws.Tag{
Key: fi.String(key),
Value: fi.String(value),
})
}
return tags
}