Merge pull request #7566 from srikiz/DO-7442-AddTagsForCreateVolumeRequest

[DO-7442] Digital Ocean add consistent volume and droplet tags for multi master feature

pkg/model/BUILD.bazel

@@ -38,6 +38,7 @@ go_library(
         "//upup/pkg/fi/cloudup/aliup:go_default_library",
         "//upup/pkg/fi/cloudup/awstasks:go_default_library",
         "//upup/pkg/fi/cloudup/awsup:go_default_library",
+        "//upup/pkg/fi/cloudup/do:go_default_library",
         "//upup/pkg/fi/cloudup/dotasks:go_default_library",
         "//upup/pkg/fi/cloudup/gce:go_default_library",
         "//upup/pkg/fi/cloudup/gcetasks:go_default_library",

pkg/model/components/etcdmanager/model.go

@@ -385,10 +385,12 @@ func (b *EtcdManagerBuilder) buildPod(etcdCluster *kops.EtcdClusterSpec) (*v1.Po
 		case kops.CloudProviderDO:
 			config.VolumeProvider = "do"
 
+			// DO does not support . in tags / names
+			safeClusterName := do.SafeClusterName(b.Cluster.Name)
+
 			config.VolumeTag = []string{
-				fmt.Sprintf("kubernetes.io/cluster=%s", b.Cluster.Name),
+				fmt.Sprintf("%s=%s", do.TagKubernetesClusterNamePrefix, safeClusterName),
-				do.TagNameEtcdClusterPrefix + etcdCluster.Name,
+				do.TagKubernetesClusterIndex,
-				do.TagNameRolePrefix + "master=1",
 			}
 			config.VolumeNameTag = do.TagNameEtcdClusterPrefix + etcdCluster.Name
 

pkg/model/domodel/BUILD.bazel

@@ -11,6 +11,7 @@ go_library(
     deps = [
         "//pkg/model:go_default_library",
         "//upup/pkg/fi:go_default_library",
+        "//upup/pkg/fi/cloudup/do:go_default_library",
         "//upup/pkg/fi/cloudup/dotasks:go_default_library",
     ],
 )

pkg/model/domodel/droplets.go

@@ -17,11 +17,12 @@ limitations under the License.
 package domodel
 
 import (
-	"strings"
-
 	"k8s.io/kops/pkg/model"
 	"k8s.io/kops/upup/pkg/fi"
+	"k8s.io/kops/upup/pkg/fi/cloudup/do"
 	"k8s.io/kops/upup/pkg/fi/cloudup/dotasks"
+	"strconv"
+	"strings"
 )
 
 // DropletBuilder configures droplets for the cluster
@@ -44,8 +45,9 @@ func (d *DropletBuilder) Build(c *fi.ModelBuilderContext) error {
 	sshKeyFingerPrint := splitSSHKeyName[len(splitSSHKeyName)-1]
 
 	// replace "." with "-" since DO API does not accept "."
-	clusterTag := "KubernetesCluster:" + strings.Replace(d.ClusterName(), ".", "-", -1)
+	clusterTag := do.TagKubernetesClusterNamePrefix + ":" + strings.Replace(d.ClusterName(), ".", "-", -1)
 
+	masterIndexCount := 0
 	// In the future, DigitalOcean will use Machine API to manage groups,
 	// for now create d.InstanceGroups.Spec.MinSize amount of droplets
 	for _, ig := range d.InstanceGroups {
@@ -61,8 +63,15 @@ func (d *DropletBuilder) Build(c *fi.ModelBuilderContext) error {
 		droplet.Size = fi.String(ig.Spec.MachineType)
 		droplet.Image = fi.String(ig.Spec.Image)
 		droplet.SSHKey = fi.String(sshKeyFingerPrint)
+
 		droplet.Tags = []string{clusterTag}
 
+		if ig.IsMaster() {
+			masterIndexCount++
+			clusterTagIndex := do.TagKubernetesClusterIndex + ":" + strconv.Itoa(masterIndexCount)
+			droplet.Tags = append(droplet.Tags, clusterTagIndex)
+		}
+
 		userData, err := d.BootstrapScript.ResourceNodeUp(ig, d.Cluster)
 		if err != nil {
 			return err

pkg/model/master_volumes.go

@@ -29,6 +29,7 @@ import (
 	"k8s.io/kops/upup/pkg/fi/cloudup/aliup"
 	"k8s.io/kops/upup/pkg/fi/cloudup/awstasks"
 	"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
+	"k8s.io/kops/upup/pkg/fi/cloudup/do"
 	"k8s.io/kops/upup/pkg/fi/cloudup/dotasks"
 	"k8s.io/kops/upup/pkg/fi/cloudup/gce"
 	"k8s.io/kops/upup/pkg/fi/cloudup/gcetasks"
@@ -177,18 +178,26 @@ func (b *MasterVolumeBuilder) addAWSVolume(c *fi.ModelBuilderContext, name strin
 
 func (b *MasterVolumeBuilder) addDOVolume(c *fi.ModelBuilderContext, name string, volumeSize int32, zone string, etcd *kops.EtcdClusterSpec, m *kops.EtcdMemberSpec, allMembers []string) {
 	// required that names start with a lower case and only contains letters, numbers and hyphens
-	name = "kops-" + strings.Replace(name, ".", "-", -1)
+	name = "kops-" + do.SafeClusterName(name)
 
 	// DO has a 64 character limit for volume names
 	if len(name) >= 64 {
 		name = name[:64]
 	}
 
+	tags := make(map[string]string)
+	tags[do.TagNameEtcdClusterPrefix+etcd.Name] = do.SafeClusterName(m.Name)
+	tags[do.TagKubernetesClusterIndex] = do.SafeClusterName(m.Name)
+
+	// We always add an owned tags (these can't be shared)
+	tags[do.TagKubernetesClusterNamePrefix] = do.SafeClusterName(b.Cluster.ObjectMeta.Name)
+
 	t := &dotasks.Volume{
 		Name:      s(name),
 		Lifecycle: b.Lifecycle,
 		SizeGB:    fi.Int64(int64(volumeSize)),
 		Region:    s(zone),
+		Tags:      tags,
 	}
 
 	c.AddTask(t)

upup/pkg/fi/cloudup/do/cloud.go

@@ -19,10 +19,19 @@ package do
 import (
 	"k8s.io/kops/pkg/resources/digitalocean"
 	"k8s.io/kops/upup/pkg/fi"
+	"strings"
 )
 
-const TagNameEtcdClusterPrefix = "k8s.io/etcd/"
+const TagKubernetesClusterIndex = "k8s-index"
+const TagNameEtcdClusterPrefix = "etcdCluster-"
 const TagNameRolePrefix = "k8s.io/role/"
+const TagKubernetesClusterNamePrefix = "KubernetesCluster"
+
+func SafeClusterName(clusterName string) string {
+	// DO does not support . in tags / names
+	safeClusterName := strings.Replace(clusterName, ".", "-", -1)
+	return safeClusterName
+}
 
 func NewDOCloud(region string) (fi.Cloud, error) {
 	return digitalocean.NewCloud(region)

upup/pkg/fi/cloudup/dotasks/BUILD.bazel

@@ -16,6 +16,7 @@ go_library(
         "//upup/pkg/fi/cloudup/do:go_default_library",
         "//upup/pkg/fi/cloudup/terraform:go_default_library",
         "//vendor/github.com/digitalocean/godo:go_default_library",
+        "//vendor/k8s.io/klog:go_default_library",
     ],
 )
 

upup/pkg/fi/cloudup/dotasks/droplet.go

@@ -19,9 +19,9 @@ package dotasks
 import (
 	"context"
 	"errors"
-
 	"github.com/digitalocean/godo"
 
+	"k8s.io/klog"
 	"k8s.io/kops/pkg/resources/digitalocean"
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/upup/pkg/fi/cloudup/do"
@@ -142,13 +142,9 @@ func (_ *Droplet) RenderDO(t *do.DOAPITarget, a, e, changes *Droplet) error {
 		newDropletCount = expectedCount - actualCount
 	}
 
-	var dropletNames []string
 	for i := 0; i < newDropletCount; i++ {
-		dropletNames = append(dropletNames, fi.StringValue(e.Name))
+		_, _, err = t.Cloud.Droplets().Create(context.TODO(), &godo.DropletCreateRequest{
-	}
+			Name:              fi.StringValue(e.Name),
-
-	_, _, err = t.Cloud.Droplets().CreateMultiple(context.TODO(), &godo.DropletMultiCreateRequest{
-		Names:             dropletNames,
 			Region:            fi.StringValue(e.Region),
 			Size:              fi.StringValue(e.Size),
 			Image:             godo.DropletCreateImage{Slug: fi.StringValue(e.Image)},
@@ -157,6 +153,13 @@ func (_ *Droplet) RenderDO(t *do.DOAPITarget, a, e, changes *Droplet) error {
 			UserData:          userData,
 			SSHKeys:           []godo.DropletCreateSSHKey{{Fingerprint: fi.StringValue(e.SSHKey)}},
 		})
+
+		if err != nil {
+			klog.Errorf("Error creating droplet with Name=%s", fi.StringValue(e.Name))
+			return err
+		}
+	}
+
 	return err
 }
 

upup/pkg/fi/cloudup/dotasks/volume.go

@@ -18,9 +18,10 @@ package dotasks
 
 import (
 	"context"
-
+	"fmt"
 	"github.com/digitalocean/godo"
 
+	"k8s.io/klog"
 	"k8s.io/kops/pkg/resources/digitalocean"
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/upup/pkg/fi/cloudup/do"
@@ -35,6 +36,7 @@ type Volume struct {
 
 	SizeGB *int64
 	Region *string
+	Tags   map[string]string
 }
 
 var _ fi.CompareWithID = &Volume{}
@@ -108,12 +110,22 @@ func (_ *Volume) RenderDO(t *do.DOAPITarget, a, e, changes *Volume) error {
 		return nil
 	}
 
+	tagArray := []string{}
+
+	for k, v := range e.Tags {
+		// DO tags don't accept =. Separate the key and value with an ":"
+		klog.V(10).Infof("DO - Join the volume tag - %s", fmt.Sprintf("%s:%s", k, v))
+		tagArray = append(tagArray, fmt.Sprintf("%s:%s", k, v))
+	}
+
 	volService := t.Cloud.Volumes()
 	_, _, err := volService.CreateVolume(context.TODO(), &godo.VolumeCreateRequest{
 		Name:          fi.StringValue(e.Name),
 		Region:        fi.StringValue(e.Region),
 		SizeGigaBytes: fi.Int64Value(e.SizeGB),
+		Tags:          tagArray,
 	})
+
 	return err
 }