/* Copyright 2016 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package model import ( "fmt" "sort" "strings" "k8s.io/klog" "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/pkg/apis/kops/model" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/cloudup/alitasks" "k8s.io/kops/upup/pkg/fi/cloudup/aliup" "k8s.io/kops/upup/pkg/fi/cloudup/awstasks" "k8s.io/kops/upup/pkg/fi/cloudup/awsup" "k8s.io/kops/upup/pkg/fi/cloudup/do" "k8s.io/kops/upup/pkg/fi/cloudup/dotasks" "k8s.io/kops/upup/pkg/fi/cloudup/gce" "k8s.io/kops/upup/pkg/fi/cloudup/gcetasks" "k8s.io/kops/upup/pkg/fi/cloudup/openstack" "k8s.io/kops/upup/pkg/fi/cloudup/openstacktasks" ) const ( DefaultEtcdVolumeSize = 20 DefaultAWSEtcdVolumeType = "gp2" DefaultAWSEtcdVolumeIops = 100 DefaultGCEEtcdVolumeType = "pd-ssd" DefaultALIEtcdVolumeType = "cloud_ssd" ) // MasterVolumeBuilder builds master EBS volumes type MasterVolumeBuilder struct { *KopsModelContext Lifecycle *fi.Lifecycle } var _ fi.ModelBuilder = &MasterVolumeBuilder{} func (b *MasterVolumeBuilder) Build(c *fi.ModelBuilderContext) error { for _, etcd := range b.Cluster.Spec.EtcdClusters { for _, m := range etcd.Members { // EBS volume for each member of the each etcd cluster name := m.Name + ".etcd-" + etcd.Name + "." + b.ClusterName() igName := fi.StringValue(m.InstanceGroup) if igName == "" { return fmt.Errorf("InstanceGroup not set on etcd %s/%s", m.Name, etcd.Name) } ig := b.FindInstanceGroup(igName) if ig == nil { return fmt.Errorf("InstanceGroup not found (for etcd %s/%s): %q", m.Name, etcd.Name, igName) } zones, err := model.FindZonesForInstanceGroup(b.Cluster, ig) if err != nil { return err } if len(zones) == 0 { return fmt.Errorf("must specify a zone for instancegroup %q used by etcd %s/%s", igName, m.Name, etcd.Name) } if len(zones) != 1 { return fmt.Errorf("must specify a unique zone for instancegroup %q used by etcd %s/%s", igName, m.Name, etcd.Name) } zone := zones[0] volumeSize := fi.Int32Value(m.VolumeSize) if volumeSize == 0 { volumeSize = DefaultEtcdVolumeSize } var allMembers []string for _, m := range etcd.Members { allMembers = append(allMembers, m.Name) } sort.Strings(allMembers) switch kops.CloudProviderID(b.Cluster.Spec.CloudProvider) { case kops.CloudProviderAWS: err = b.addAWSVolume(c, name, volumeSize, zone, etcd, m, allMembers) if err != nil { return err } case kops.CloudProviderDO: b.addDOVolume(c, name, volumeSize, zone, etcd, m, allMembers) case kops.CloudProviderGCE: b.addGCEVolume(c, name, volumeSize, zone, etcd, m, allMembers) case kops.CloudProviderVSphere: b.addVSphereVolume(c, name, volumeSize, zone, etcd, m, allMembers) case kops.CloudProviderBareMetal: klog.Fatalf("BareMetal not implemented") case kops.CloudProviderOpenstack: err = b.addOpenstackVolume(c, name, volumeSize, zone, etcd, m, allMembers) if err != nil { return err } case kops.CloudProviderALI: b.addALIVolume(c, name, volumeSize, zone, etcd, m, allMembers) default: return fmt.Errorf("unknown cloudprovider %q", b.Cluster.Spec.CloudProvider) } } } return nil } func (b *MasterVolumeBuilder) addAWSVolume(c *fi.ModelBuilderContext, name string, volumeSize int32, zone string, etcd *kops.EtcdClusterSpec, m *kops.EtcdMemberSpec, allMembers []string) error { volumeType := fi.StringValue(m.VolumeType) volumeIops := fi.Int32Value(m.VolumeIops) switch volumeType { case "io1": if volumeIops <= 0 { volumeIops = DefaultAWSEtcdVolumeIops } default: volumeType = DefaultAWSEtcdVolumeType } // The tags are how protokube knows to mount the volume and use it for etcd tags := make(map[string]string) // Apply all user defined labels on the volumes for k, v := range b.Cluster.Spec.CloudLabels { tags[k] = v } //tags[awsup.TagClusterName] = b.C.cluster.Name // This is the configuration of the etcd cluster tags[awsup.TagNameEtcdClusterPrefix+etcd.Name] = m.Name + "/" + strings.Join(allMembers, ",") // This says "only mount on a master" tags[awsup.TagNameRolePrefix+"master"] = "1" // We always add an owned tags (these can't be shared) tags["kubernetes.io/cluster/"+b.Cluster.ObjectMeta.Name] = "owned" encrypted := fi.BoolValue(m.EncryptedVolume) t := &awstasks.EBSVolume{ Name: s(name), Lifecycle: b.Lifecycle, AvailabilityZone: s(zone), SizeGB: fi.Int64(int64(volumeSize)), VolumeType: s(volumeType), KmsKeyId: m.KmsKeyId, Encrypted: fi.Bool(encrypted), Tags: tags, } if volumeType == "io1" { t.VolumeIops = i64(int64(volumeIops)) // https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSVolumeTypes.html if float64(*t.VolumeIops)/float64(*t.SizeGB) > 50.0 { return fmt.Errorf("volumeIops to volumeSize ratio must be lower than 50. For %s ratio is %f", *t.Name, float64(*t.VolumeIops)/float64(*t.SizeGB)) } } c.AddTask(t) return nil } func (b *MasterVolumeBuilder) addDOVolume(c *fi.ModelBuilderContext, name string, volumeSize int32, zone string, etcd *kops.EtcdClusterSpec, m *kops.EtcdMemberSpec, allMembers []string) { // required that names start with a lower case and only contains letters, numbers and hyphens name = "kops-" + strings.Replace(name, ".", "-", -1) // DO has a 64 character limit for volume names if len(name) >= 64 { name = name[:64] } tags := make(map[string]string) tags[do.TagNameEtcdClusterPrefix+etcd.Name] = gce.SafeClusterName(m.Name) tags[do.TagKubernetesClusterIndex] = gce.SafeClusterName(m.Name) // We always add an owned tags (these can't be shared) tags[do.TagKubernetesClusterNamePrefix] = gce.SafeClusterName(b.Cluster.ObjectMeta.Name) t := &dotasks.Volume{ Name: s(name), Lifecycle: b.Lifecycle, SizeGB: fi.Int64(int64(volumeSize)), Region: s(zone), Tags: tags, } c.AddTask(t) } func (b *MasterVolumeBuilder) addGCEVolume(c *fi.ModelBuilderContext, name string, volumeSize int32, zone string, etcd *kops.EtcdClusterSpec, m *kops.EtcdMemberSpec, allMembers []string) { volumeType := fi.StringValue(m.VolumeType) if volumeType == "" { volumeType = DefaultGCEEtcdVolumeType } // TODO: Should no longer be needed because we trim prefixes //// On GCE we are close to the length limits. So,we remove the dashes from the keys //// The name is normally something like "us-east1-a", and the dashes are particularly expensive //// because of the escaping needed (3 characters for each dash) //switch tf.cluster.Spec.CloudProvider { //case string(kops.CloudProviderGCE): // // TODO: If we're still struggling for size, we don't need to put ourselves in the allmembers list // for i := range allMembers { // allMembers[i] = strings.Replace(allMembers[i], "-", "", -1) // } // meName = strings.Replace(meName, "-", "", -1) //} // This is the configuration of the etcd cluster clusterSpec := m.Name + "/" + strings.Join(allMembers, ",") // The tags are how protokube knows to mount the volume and use it for etcd tags := make(map[string]string) tags[gce.GceLabelNameKubernetesCluster] = gce.SafeClusterName(b.ClusterName()) tags[gce.GceLabelNameRolePrefix+"master"] = "master" // Can't start with a number tags[gce.GceLabelNameEtcdClusterPrefix+etcd.Name] = gce.EncodeGCELabel(clusterSpec) // GCE disk names must match the following regular expression: '[a-z](?:[-a-z0-9]{0,61}[a-z0-9])?' name = strings.Replace(name, ".", "-", -1) if strings.IndexByte("0123456789-", name[0]) != -1 { name = "d" + name } t := &gcetasks.Disk{ Name: s(name), Lifecycle: b.Lifecycle, Zone: s(zone), SizeGB: fi.Int64(int64(volumeSize)), VolumeType: s(volumeType), Labels: tags, } c.AddTask(t) } func (b *MasterVolumeBuilder) addVSphereVolume(c *fi.ModelBuilderContext, name string, volumeSize int32, zone string, etcd *kops.EtcdClusterSpec, m *kops.EtcdMemberSpec, allMembers []string) { fmt.Print("addVSphereVolume to be implemented") } func (b *MasterVolumeBuilder) addOpenstackVolume(c *fi.ModelBuilderContext, name string, volumeSize int32, zone string, etcd *kops.EtcdClusterSpec, m *kops.EtcdMemberSpec, allMembers []string) error { volumeType := fi.StringValue(m.VolumeType) if volumeType == "" { return fmt.Errorf("must set ETCDMemberSpec.VolumeType on Openstack platform") } // The tags are how protokube knows to mount the volume and use it for etcd tags := make(map[string]string) // Apply all user defined labels on the volumes for k, v := range b.Cluster.Spec.CloudLabels { tags[k] = v } // This is the configuration of the etcd cluster tags[openstack.TagNameEtcdClusterPrefix+etcd.Name] = m.Name + "/" + strings.Join(allMembers, ",") // This says "only mount on a master" tags[openstack.TagNameRolePrefix+"master"] = "1" // override zone if b.Cluster.Spec.CloudConfig.Openstack.BlockStorage != nil && b.Cluster.Spec.CloudConfig.Openstack.BlockStorage.OverrideAZ != nil { zone = fi.StringValue(b.Cluster.Spec.CloudConfig.Openstack.BlockStorage.OverrideAZ) } t := &openstacktasks.Volume{ Name: s(name), AvailabilityZone: s(zone), VolumeType: s(volumeType), SizeGB: fi.Int64(int64(volumeSize)), Tags: tags, Lifecycle: b.Lifecycle, } c.AddTask(t) return nil } func (b *MasterVolumeBuilder) addALIVolume(c *fi.ModelBuilderContext, name string, volumeSize int32, zone string, etcd *kops.EtcdClusterSpec, m *kops.EtcdMemberSpec, allMembers []string) { //Alicloud does not support volumeName starts with number name = "v" + name volumeType := fi.StringValue(m.VolumeType) if volumeType == "" { volumeType = DefaultALIEtcdVolumeType } // The tags are how protokube knows to mount the volume and use it for etcd tags := make(map[string]string) // Apply all user defined labels on the volumes for k, v := range b.Cluster.Spec.CloudLabels { tags[k] = v } // This is the configuration of the etcd cluster tags[aliup.TagNameEtcdClusterPrefix+etcd.Name] = m.Name + "/" + strings.Join(allMembers, ",") // This says "only mount on a master" tags[aliup.TagNameRolePrefix+"master"] = "1" encrypted := fi.BoolValue(m.EncryptedVolume) t := &alitasks.Disk{ Lifecycle: b.Lifecycle, Name: s(name), ZoneId: s(zone), SizeGB: fi.Int(int(volumeSize)), DiskCategory: s(volumeType), Encrypted: fi.Bool(encrypted), Tags: tags, } c.AddTask(t) }