From 074791b446274f1b67534455bf9f093a9bf9ba86 Mon Sep 17 00:00:00 2001 From: prashima Date: Tue, 11 Apr 2017 13:38:38 -0700 Subject: [PATCH] Removed hardcode from protokube logic. Fixes #15. (#46) --- docs/development/vsphere-dev.md | 4 +- pkg/model/vspheremodel/autoscalinggroup.go | 1 + protokube/pkg/protokube/vsphere_volume.go | 118 ++++++++++-------- .../vsphere/vsphere_volume_metadata.go | 66 ++++++++++ upup/pkg/fi/cloudup/vspheretasks/attachiso.go | 69 +++++++++- .../pkg/fi/cloudup/vspheretasks/cloud_init.go | 5 + 6 files changed, 208 insertions(+), 55 deletions(-) create mode 100644 upup/pkg/fi/cloudup/vsphere/vsphere_volume_metadata.go diff --git a/docs/development/vsphere-dev.md b/docs/development/vsphere-dev.md index 2ea7f9b578..e92a29e4d6 100644 --- a/docs/development/vsphere-dev.md +++ b/docs/development/vsphere-dev.md @@ -93,7 +93,7 @@ Please note that dns-controller has also been modified to support vSphere. You c Execute following command to launch cluster. ```bash -.build/dist/darwin/amd64/kops create cluster kubernetes.skydns.local --cloud=vsphere --zones=${AWS_REGION}a --dns-zone=skydns.local --networking=flannel +.build/dist/darwin/amd64/kops create cluster kubernetes.skydns.local --cloud=vsphere --zones=vmware-zone --dns-zone=skydns.local --networking=flannel --vsphere-server=10.160.97.44 --vsphere-datacenter=VSAN-DC --vsphere-resource-pool=VSAN-Cluster --vsphere-datastore=vsanDatastore --dns private --vsphere-coredns-server=http://10.192.217.24:2379 --image="ubuntu_16_04" ``` @@ -102,7 +102,7 @@ Use .build/dist/linux/amd64/kops if working on a linux machine, instead of mac. **Notes** 1. ```clustername``` should end with **skydns.local**. Example: ```kubernetes.cluster.skydns.local```. -2. ```zones``` should end with ```a```. Example: ```us-west-2a``` or as the above command sets ```--zones=${AWS_REGION}a```. +2. For ```zones``` any string will do, for now. It's only getting used for the construction of names of various entities. But it's a mandatory argument. 3. Make sure following parameters have these values, * ```--dns-zone=skydns.local``` * ```--networking=flannel``` diff --git a/pkg/model/vspheremodel/autoscalinggroup.go b/pkg/model/vspheremodel/autoscalinggroup.go index ba554cc784..87ce9f31af 100644 --- a/pkg/model/vspheremodel/autoscalinggroup.go +++ b/pkg/model/vspheremodel/autoscalinggroup.go @@ -57,6 +57,7 @@ func (b *AutoscalingGroupModelBuilder) Build(c *fi.ModelBuilderContext) error { VM: createVmTask, IG: ig, BootstrapScript: b.BootstrapScript, + EtcdClusters: b.Cluster.Spec.EtcdClusters, } attachISOTask.BootstrapScript.AddAwsEnvironmentVariables = true diff --git a/protokube/pkg/protokube/vsphere_volume.go b/protokube/pkg/protokube/vsphere_volume.go index 65434b7c33..340a196e52 100644 --- a/protokube/pkg/protokube/vsphere_volume.go +++ b/protokube/pkg/protokube/vsphere_volume.go @@ -18,37 +18,26 @@ package protokube import ( "errors" + "fmt" "github.com/golang/glog" + "io/ioutil" + "k8s.io/kops/upup/pkg/fi/cloudup/vsphere" "net" + "os/exec" + "runtime" + "strings" ) -const EtcdDataKey = "01" -const EtcdDataVolPath = "/mnt/master-" + EtcdDataKey -const EtcdEventKey = "02" -const EtcdEventVolPath = "/mnt/master-" + EtcdEventKey - -// TODO Use lsblk or counterpart command to find the actual device details. -const LocalDeviceForDataVol = "/dev/sdb1" -const LocalDeviceForEventsVol = "/dev/sdc1" +const VolumeMetaDataFile = "/vol-metadata/metadata.json" const VolStatusValue = "attached" -const EtcdNodeName = "a" -const EtcdClusterName = "main" -const EtcdEventsClusterName = "events" -type VSphereVolumes struct { - // Dummy property. Not getting used any where for now. - paths map[string]string -} +type VSphereVolumes struct{} var _ Volumes = &VSphereVolumes{} var machineIp net.IP func NewVSphereVolumes() (*VSphereVolumes, error) { - vsphereVolumes := &VSphereVolumes{ - paths: make(map[string]string), - } - vsphereVolumes.paths[EtcdDataKey] = EtcdDataVolPath - vsphereVolumes.paths[EtcdEventKey] = EtcdEventVolPath + vsphereVolumes := &VSphereVolumes{} return vsphereVolumes, nil } @@ -60,51 +49,82 @@ func (v *VSphereVolumes) FindVolumes() ([]*Volume, error) { attachedTo = ip.String() } - // etcd data volume and etcd cluster spec. - { - vol := &Volume{ - ID: EtcdDataKey, - LocalDevice: LocalDeviceForDataVol, - AttachedTo: attachedTo, - Mountpoint: EtcdDataVolPath, - Status: VolStatusValue, - Info: VolumeInfo{ - Description: EtcdClusterName, - }, - } - etcdSpec := &EtcdClusterSpec{ - ClusterKey: EtcdClusterName, - NodeName: EtcdNodeName, - NodeNames: []string{EtcdNodeName}, - } - vol.Info.EtcdClusters = []*EtcdClusterSpec{etcdSpec} - volumes = append(volumes, vol) + etcdClusters, err := getVolMetadata() + + if err != nil { + return nil, err } - // etcd events volume and etcd events cluster spec. - { + for _, etcd := range etcdClusters { + mountPoint := vsphere.GetMountPoint(etcd.VolumeId) + localDevice, err := getDevice(mountPoint) + if err != nil { + return nil, err + } vol := &Volume{ - ID: EtcdEventKey, - LocalDevice: LocalDeviceForEventsVol, + ID: etcd.VolumeId, + LocalDevice: localDevice, AttachedTo: attachedTo, - Mountpoint: EtcdEventVolPath, + Mountpoint: mountPoint, Status: VolStatusValue, Info: VolumeInfo{ - Description: EtcdEventsClusterName, + Description: etcd.EtcdClusterName, }, } + etcdSpec := &EtcdClusterSpec{ - ClusterKey: EtcdEventsClusterName, - NodeName: EtcdNodeName, - NodeNames: []string{EtcdNodeName}, + ClusterKey: etcd.EtcdClusterName, + NodeName: etcd.EtcdNodeName, } + + var nodeNames []string + for _, member := range etcd.Members { + nodeNames = append(nodeNames, member.Name) + } + etcdSpec.NodeNames = nodeNames vol.Info.EtcdClusters = []*EtcdClusterSpec{etcdSpec} volumes = append(volumes, vol) } - glog.Infof("Found volumes: %v", volumes) + glog.V(4).Infof("Found volumes: %v", volumes) return volumes, nil } +func getDevice(mountPoint string) (string, error) { + if runtime.GOOS == "linux" { + cmd := "lsblk" + arg := "-l" + out, err := exec.Command(cmd, arg).Output() + if err != nil { + return "", err + } + + if Containerized { + mountPoint = PathFor(mountPoint) + } + lines := strings.Split(string(out), "\n") + for _, line := range lines { + if strings.Contains(line, mountPoint) { + lsblkOutput := strings.Split(line, " ") + glog.V(4).Infof("Found device: %v ", lsblkOutput[0]) + return "/dev/" + lsblkOutput[0], nil + } + } + } else { + return "", fmt.Errorf("Failed to find device. OS %v is not supported for vSphere.", runtime.GOOS) + } + return "", fmt.Errorf("No device has been mounted on mountPoint %v.", mountPoint) +} + +func getVolMetadata() ([]vsphere.VolumeMetadata, error) { + rawData, err := ioutil.ReadFile(PathFor(VolumeMetaDataFile)) + + if err != nil { + return nil, err + } + + return vsphere.UnmarshalVolumeMetadata(string(rawData)) +} + func (v *VSphereVolumes) AttachVolume(volume *Volume) error { // Currently this is a no-op for vSphere. The virtual disks should already be mounted on this VM. glog.Infof("All volumes should already be attached. No operation done.") diff --git a/upup/pkg/fi/cloudup/vsphere/vsphere_volume_metadata.go b/upup/pkg/fi/cloudup/vsphere/vsphere_volume_metadata.go new file mode 100644 index 0000000000..6b7b3f09ce --- /dev/null +++ b/upup/pkg/fi/cloudup/vsphere/vsphere_volume_metadata.go @@ -0,0 +1,66 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vsphere + +import ( + "encoding/json" + "strconv" +) + +type VolumeMetadata struct { + // EtcdClusterName is the name of the etcd cluster (main, events etc) + EtcdClusterName string `json:"etcdClusterName,omitempty"` + // EtcdNodeName is the name of a node in etcd cluster for which this volume will be used + EtcdNodeName string `json:"etcdNodeName,omitempty"` + // EtcdMember stores the configurations for each member of the cluster + Members []EtcdMemberSpec `json:"etcdMembers,omitempty"` + // Volume id + VolumeId string `json:"volumeId,omitempty"` +} + +type EtcdMemberSpec struct { + // Name is the name of the member within the etcd cluster + Name string `json:"name,omitempty"` + InstanceGroup string `json:"instanceGroup,omitempty"` +} + +func MarshalVolumeMetadata(v []VolumeMetadata) (string, error) { + metadata, err := json.Marshal(v) + if err != nil { + return "", err + } + + return string(metadata), nil +} + +func UnmarshalVolumeMetadata(text string) ([]VolumeMetadata, error) { + var v []VolumeMetadata + err := json.Unmarshal([]byte(text), &v) + return v, err +} + +func GetVolumeId(i int) string { + return "0" + strconv.Itoa(i) +} + +/* + * GetMountPoint will return the mount point where the volume is expected to be mounted. + * This path would be /mnt/master-, eg: /mnt/master-01. + */ +func GetMountPoint(volumeId string) string { + return "/mnt/master-" + volumeId +} diff --git a/upup/pkg/fi/cloudup/vspheretasks/attachiso.go b/upup/pkg/fi/cloudup/vspheretasks/attachiso.go index 823133c401..06f00c130c 100644 --- a/upup/pkg/fi/cloudup/vspheretasks/attachiso.go +++ b/upup/pkg/fi/cloudup/vspheretasks/attachiso.go @@ -42,6 +42,7 @@ type AttachISO struct { VM *VirtualMachine IG *kops.InstanceGroup BootstrapScript *model.BootstrapScript + EtcdClusters []*kops.EtcdClusterSpec } var _ fi.HasName = &AttachISO{} @@ -111,7 +112,7 @@ func (_ *AttachISO) RenderVSphere(t *vsphere.VSphereAPITarget, a, e, changes *At return nil } -func createUserData(startupStr string, dir string, dnsServer string, vmUUID string) error { +func createUserData(changes *AttachISO, startupStr string, dir string, dnsServer string, vmUUID string) error { // Update the startup script to add the extra spaces for // indentation when copied to the user-data file. strArray := strings.Split(startupStr, "\n") @@ -140,10 +141,15 @@ func createUserData(startupStr string, dir string, dnsServer string, vmUUID stri vmUUIDStr := " " + vmUUID + "\n" data = strings.Replace(data, "$VM_UUID", vmUUIDStr, -1) + data, err = createVolumeScript(changes, data) + if err != nil { + return err + } + userDataFile := filepath.Join(dir, "user-data") glog.V(4).Infof("User data file content: %s", data) - if err := ioutil.WriteFile(userDataFile, []byte(data), 0644); err != nil { + if err = ioutil.WriteFile(userDataFile, []byte(data), 0644); err != nil { glog.Errorf("Unable to write user-data into file %s", userDataFile) return err } @@ -151,6 +157,60 @@ func createUserData(startupStr string, dir string, dnsServer string, vmUUID stri return nil } +func createVolumeScript(changes *AttachISO, data string) (string, error) { + if changes.IG.Spec.Role != kops.InstanceGroupRoleMaster { + return strings.Replace(data, "$VOLUME_SCRIPT", " No volume metadata needed for "+string(changes.IG.Spec.Role)+".", -1), nil + } + + volsString, err := getVolMetadata(changes) + + if err != nil { + return "", err + } + + return strings.Replace(data, "$VOLUME_SCRIPT", " "+volsString, -1), nil +} + +func getVolMetadata(changes *AttachISO) (string, error) { + var volsMetadata []vsphere.VolumeMetadata + + // Creating vsphere.VolumeMetadata using clusters EtcdClusterSpec + for i, etcd := range changes.EtcdClusters { + volMetadata := vsphere.VolumeMetadata{} + volMetadata.EtcdClusterName = etcd.Name + volMetadata.VolumeId = vsphere.GetVolumeId(i + 1) + + var members []vsphere.EtcdMemberSpec + var thisNode string + for _, member := range etcd.Members { + if *member.InstanceGroup == changes.IG.Name { + thisNode = member.Name + } + etcdMember := vsphere.EtcdMemberSpec{ + Name: member.Name, + InstanceGroup: *member.InstanceGroup, + } + members = append(members, etcdMember) + } + + if thisNode == "" { + return "", fmt.Errorf("Failed to construct volume metadata for %v InstanceGroup.", changes.IG.Name) + } + + volMetadata.EtcdNodeName = thisNode + volMetadata.Members = members + volsMetadata = append(volsMetadata, volMetadata) + } + + glog.V(4).Infof("Marshaling master vol metadata : %v", volsMetadata) + volsString, err := vsphere.MarshalVolumeMetadata(volsMetadata) + glog.V(4).Infof("Marshaled master vol metadata: %v", volsString) + if err != nil { + return "", err + } + return volsString, nil +} + func createMetaData(dir string, vmName string) error { data := strings.Replace(metaDataTemplate, "$INSTANCE_ID", uuid.NewUUID().String(), -1) data = strings.Replace(data, "$LOCAL_HOST_NAME", vmName, -1) @@ -165,8 +225,9 @@ func createMetaData(dir string, vmName string) error { return nil } -func createISO(changes *AttachISO, startupStr string, dir string, dnsServer string, vmUUID string) (string, error) { - err := createUserData(startupStr, dir, dnsServer, vmUUID) +func createISO(changes *AttachISO, startupStr string, dir string, dnsServer, vmUUID string) (string, error) { + err := createUserData(changes, startupStr, dir, dnsServer, vmUUID) + if err != nil { return "", err } diff --git a/upup/pkg/fi/cloudup/vspheretasks/cloud_init.go b/upup/pkg/fi/cloudup/vspheretasks/cloud_init.go index 3efc1f006a..3cf46d11ac 100644 --- a/upup/pkg/fi/cloudup/vspheretasks/cloud_init.go +++ b/upup/pkg/fi/cloudup/vspheretasks/cloud_init.go @@ -34,6 +34,11 @@ $VM_UUID owner: root:root path: /root/vm_uuid permissions: "0644" + - content: | +$VOLUME_SCRIPT + owner: root:root + path: /vol-metadata/metadata.json + permissions: "0644" runcmd: - bash /root/update_dns.sh 2>&1 > /var/log/update_dns.log