protokube: add support for OpenStack

protokube/cmd/protokube/main.go

@@ -70,7 +70,7 @@ func run() error {
 	flag.BoolVar(&containerized, "containerized", containerized, "Set if we are running containerized.")
 	flag.BoolVar(&initializeRBAC, "initialize-rbac", initializeRBAC, "Set if we should initialize RBAC")
 	flag.BoolVar(&master, "master", master, "Whether or not this node is a master")
-	flag.StringVar(&cloud, "cloud", "aws", "CloudProvider we are using (aws,digitalocean,gce)")
+	flag.StringVar(&cloud, "cloud", "aws", "CloudProvider we are using (aws,digitalocean,gce,openstack)")
 	flag.StringVar(&clusterID, "cluster-id", clusterID, "Cluster ID")
 	flag.StringVar(&dnsInternalSuffix, "dns-internal-suffix", dnsInternalSuffix, "DNS suffix for internal domain names")
 	flag.StringVar(&dnsServer, "dns-server", dnsServer, "DNS Server")
@@ -179,6 +179,22 @@ func run() error {
 			}
 			internalIP = ip
 		}
+	} else if cloud == "openstack" {
+		glog.Info("Initializing openstack volumes")
+		osVolumes, err := protokube.NewOpenstackVolumes()
+		if err != nil {
+			glog.Errorf("Error initializing openstack: %q", err)
+			os.Exit(1)
+		}
+		volumes = osVolumes
+		if internalIP == nil {
+			internalIP = osVolumes.InternalIP()
+		}
+
+		if clusterID == "" {
+			clusterID = osVolumes.ClusterID()
+		}
+
 	} else {
 		glog.Errorf("Unknown cloud %q", cloud)
 		os.Exit(1)
@@ -235,6 +251,12 @@ func run() error {
 				return err
 			}
 			gossipName = volumes.(*protokube.GCEVolumes).InstanceName()
+		} else if cloud == "openstack" {
+			gossipSeeds, err = volumes.(*protokube.OpenstackVolumes).GossipSeeds()
+			if err != nil {
+				return err
+			}
+			gossipName = volumes.(*protokube.OpenstackVolumes).InstanceName()
 		} else {
 			glog.Fatalf("seed provider for %q not yet implemented", cloud)
 		}

protokube/pkg/gossip/openstack/BUILD.bazel

@@ -0,0 +1,15 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["seeds.go"],
+    importpath = "k8s.io/kops/protokube/pkg/gossip/openstack",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//protokube/pkg/gossip:go_default_library",
+        "//vendor/github.com/golang/glog:go_default_library",
+        "//vendor/github.com/gophercloud/gophercloud:go_default_library",
+        "//vendor/github.com/gophercloud/gophercloud/openstack/compute/v2/servers:go_default_library",
+        "//vendor/github.com/gophercloud/gophercloud/pagination:go_default_library",
+    ],
+)

protokube/pkg/gossip/openstack/seeds.go

@@ -0,0 +1,77 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package gce
+
+import (
+	"fmt"
+
+	"github.com/golang/glog"
+	"github.com/gophercloud/gophercloud"
+	"github.com/gophercloud/gophercloud/openstack/compute/v2/servers"
+	"github.com/gophercloud/gophercloud/pagination"
+	"k8s.io/kops/protokube/pkg/gossip"
+	"k8s.io/kops/upup/pkg/fi/cloudup/openstack"
+)
+
+type SeedProvider struct {
+	computeClient *gophercloud.ServiceClient
+	projectID     string
+	clusterName   string
+}
+
+var _ gossip.SeedProvider = &SeedProvider{}
+
+func (p *SeedProvider) GetSeeds() ([]string, error) {
+	var seeds []string
+
+	err := servers.List(p.computeClient, servers.ListOpts{
+		TenantID: p.projectID,
+	}).EachPage(func(page pagination.Page) (bool, error) {
+		var s []servers.Server
+		err := servers.ExtractServersInto(page, &s)
+		if err != nil {
+			return false, err
+		}
+
+		for _, server := range s {
+			if clusterName, ok := server.Metadata[openstack.TagClusterName]; ok {
+				var err error
+				addr, err := openstack.GetServerFixedIP(&server, clusterName)
+				if err != nil {
+					glog.Warningf("Failed to list seeds: %v", err)
+					continue
+				}
+				seeds = append(seeds, addr)
+			}
+		}
+		return true, nil
+	})
+
+	if err != nil {
+		return seeds, fmt.Errorf("Failed to list servers while retrieving seeds: %v", err)
+	}
+
+	return seeds, nil
+}
+
+func NewSeedProvider(computeClient *gophercloud.ServiceClient, clusterName string, projectID string) (*SeedProvider, error) {
+	return &SeedProvider{
+		computeClient: computeClient,
+		clusterName:   clusterName,
+		projectID:     projectID,
+	}, nil
+}

protokube/pkg/protokube/BUILD.bazel

@@ -18,6 +18,7 @@ go_library(
         "kube_dns.go",
         "models.go",
         "nsenter_exec.go",
+        "openstack_volume.go",
         "rbac.go",
         "tainter.go",
         "utils.go",
@@ -37,8 +38,10 @@ go_library(
         "//protokube/pkg/gossip/aws:go_default_library",
         "//protokube/pkg/gossip/dns:go_default_library",
         "//protokube/pkg/gossip/gce:go_default_library",
+        "//protokube/pkg/gossip/openstack:go_default_library",
         "//upup/pkg/fi/cloudup/awsup:go_default_library",
         "//upup/pkg/fi/cloudup/gce:go_default_library",
+        "//upup/pkg/fi/cloudup/openstack:go_default_library",
         "//upup/pkg/fi/cloudup/vsphere:go_default_library",
         "//util/pkg/exec:go_default_library",
         "//vendor/cloud.google.com/go/compute/metadata:go_default_library",
@@ -49,6 +52,8 @@ go_library(
         "//vendor/github.com/aws/aws-sdk-go/service/ec2:go_default_library",
         "//vendor/github.com/digitalocean/godo:go_default_library",
         "//vendor/github.com/golang/glog:go_default_library",
+        "//vendor/github.com/gophercloud/gophercloud/openstack/blockstorage/v2/volumes:go_default_library",
+        "//vendor/github.com/gophercloud/gophercloud/openstack/compute/v2/extensions/volumeattach:go_default_library",
         "//vendor/golang.org/x/net/context:go_default_library",
         "//vendor/golang.org/x/oauth2/google:go_default_library",
         "//vendor/google.golang.org/api/compute/v0.beta:go_default_library",

protokube/pkg/protokube/openstack_volume.go

@@ -0,0 +1,284 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package protokube
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"os"
+	"strings"
+
+	"github.com/golang/glog"
+	cinderv2 "github.com/gophercloud/gophercloud/openstack/blockstorage/v2/volumes"
+	"github.com/gophercloud/gophercloud/openstack/compute/v2/extensions/volumeattach"
+	"k8s.io/kops/protokube/pkg/etcd"
+	"k8s.io/kops/protokube/pkg/gossip"
+	gossipos "k8s.io/kops/protokube/pkg/gossip/openstack"
+	"k8s.io/kops/upup/pkg/fi/cloudup/openstack"
+)
+
+const MetadataLatest string = "http://169.254.169.254/openstack/latest/meta_data.json"
+
+type Metadata struct {
+	// Matches openstack.TagClusterName
+	ClusterName string `json:"KubernetesCluster"`
+}
+
+type InstanceMetadata struct {
+	Name             string    `json:"name"`
+	UserMeta         *Metadata `json:"meta"`
+	ProjectID        string    `json:"project_id"`
+	AvailabilityZone string    `json:"availability_zone"`
+	Hostname         string    `json:"hostname"`
+	ServerID         string    `json:"uuid"`
+}
+
+// GCEVolumes is the Volumes implementation for GCE
+type OpenstackVolumes struct {
+	cloud openstack.OpenstackCloud
+
+	meta *InstanceMetadata
+
+	clusterName  string
+	project      string
+	instanceName string
+	internalIP   net.IP
+	storageZone  string
+}
+
+var _ Volumes = &OpenstackVolumes{}
+
+func getLocalMetadata() (*InstanceMetadata, error) {
+	var meta InstanceMetadata
+	var client http.Client
+	resp, err := client.Get(MetadataLatest)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode == http.StatusOK {
+		bodyBytes, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
+			return nil, err
+		}
+		err = json.Unmarshal(bodyBytes, &meta)
+		if err != nil {
+			return nil, err
+		}
+		return &meta, nil
+	}
+	return nil, err
+}
+
+// NewOpenstackVolumes builds a OpenstackVolume
+func NewOpenstackVolumes() (*OpenstackVolumes, error) {
+
+	metadata, err := getLocalMetadata()
+	if err != nil {
+		return nil, fmt.Errorf("Failed to get server metadata: %v", err)
+	}
+
+	tags := make(map[string]string)
+	// Cluster name needed to bypass missing designate options
+	tags[openstack.TagClusterName] = metadata.UserMeta.ClusterName
+
+	oscloud, err := openstack.NewOpenstackCloud(tags, nil)
+	if err != nil {
+		return nil, fmt.Errorf("Failed to initialize OpenstackVolumes: %v", err)
+	}
+
+	a := &OpenstackVolumes{
+		cloud: oscloud,
+		meta:  metadata,
+	}
+
+	err = a.discoverTags()
+	if err != nil {
+		return nil, err
+	}
+
+	return a, nil
+}
+
+// ClusterID implements Volumes ClusterID
+func (a *OpenstackVolumes) ClusterID() string {
+	return a.meta.UserMeta.ClusterName
+}
+
+// Project returns the current GCE project
+func (a *OpenstackVolumes) Project() string {
+	return a.meta.ProjectID
+}
+
+// InternalIP implements Volumes InternalIP
+func (a *OpenstackVolumes) InternalIP() net.IP {
+	return a.internalIP
+}
+
+func (a *OpenstackVolumes) discoverTags() error {
+
+	// Cluster Name
+	{
+		a.clusterName = strings.TrimSpace(string(a.meta.UserMeta.ClusterName))
+		if a.clusterName == "" {
+			return fmt.Errorf("cluster name metadata was empty")
+		}
+		glog.Infof("Found cluster name=%q", a.clusterName)
+	}
+
+	// Project ID
+	{
+		a.project = strings.TrimSpace(a.meta.ProjectID)
+		if a.project == "" {
+			return fmt.Errorf("project metadata was empty")
+		}
+		glog.Infof("Found project=%q", a.project)
+	}
+
+	// Storage Availability Zone
+	az, err := a.cloud.GetStorageAZFromCompute(a.meta.AvailabilityZone)
+	if err != nil {
+		return fmt.Errorf("Could not establish storage availability zone: %v", err)
+	}
+	a.storageZone = az.ZoneName
+	glog.Infof("Found zone=%q", a.storageZone)
+
+	// Instance Name
+	{
+		a.instanceName = strings.TrimSpace(a.meta.Name)
+		if a.instanceName == "" {
+			return fmt.Errorf("instance name metadata was empty")
+		}
+		glog.Infof("Found instanceName=%q", a.instanceName)
+	}
+
+	// Internal IP
+	{
+		ips, err := net.LookupIP(a.meta.Hostname)
+		if err != nil {
+			return fmt.Errorf("error querying InternalIP from hostname: %v", err)
+		}
+		if len(ips) == 0 {
+			return fmt.Errorf("ip lookups from metadata hostname was empty")
+		}
+		a.internalIP = ips[0]
+		glog.Infof("Found internalIP=%q", a.internalIP)
+	}
+
+	return nil
+}
+
+func (v *OpenstackVolumes) buildOpenstackVolume(d *cinderv2.Volume) (*Volume, error) {
+	volumeName := d.Name
+	vol := &Volume{
+		ID: d.ID,
+		Info: VolumeInfo{
+			Description: volumeName,
+		},
+	}
+
+	vol.Status = d.Status
+
+	for _, attachedTo := range d.Attachments {
+		vol.AttachedTo = attachedTo.HostName
+		if attachedTo.ServerID == v.meta.ServerID {
+			vol.LocalDevice = attachedTo.Device
+		}
+	}
+
+	// FIXME: Zone matters, broken in my env
+
+	for k, v := range d.Metadata {
+		if strings.HasPrefix(k, openstack.TagNameEtcdClusterPrefix) {
+			etcdClusterName := k[len(openstack.TagNameEtcdClusterPrefix):]
+			spec, err := etcd.ParseEtcdClusterSpec(etcdClusterName, v)
+			if err != nil {
+				return nil, fmt.Errorf("error parsing etcd cluster meta %q on volume %q: %v", v, d.Name, err)
+			}
+			vol.Info.EtcdClusters = append(vol.Info.EtcdClusters, spec)
+		}
+	}
+
+	return vol, nil
+}
+
+func (v *OpenstackVolumes) FindVolumes() ([]*Volume, error) {
+	var volumes []*Volume
+
+	glog.V(2).Infof("Listing Openstack disks in %s/%s", v.project, v.meta.AvailabilityZone)
+
+	vols, err := v.cloud.ListVolumes(cinderv2.ListOpts{
+		TenantID: v.project,
+	})
+	if err != nil {
+		return volumes, fmt.Errorf("FindVolumes: Failed to list volume.")
+	}
+
+	for _, volume := range vols {
+		if clusterName, ok := volume.Metadata[openstack.TagClusterName]; ok && clusterName == v.clusterName {
+			if _, isMasterRole := volume.Metadata[openstack.TagNameRolePrefix+"master"]; isMasterRole {
+				vol, err := v.buildOpenstackVolume(&volume)
+				if err != nil {
+					glog.Errorf("FindVolumes: Failed to build openstack volume %s: %v", volume.Name, err)
+					continue
+				}
+				volumes = append(volumes, vol)
+			}
+		}
+	}
+
+	return volumes, nil
+}
+
+// FindMountedVolume implements Volumes::FindMountedVolume
+func (v *OpenstackVolumes) FindMountedVolume(volume *Volume) (string, error) {
+	device := volume.LocalDevice
+
+	_, err := os.Stat(pathFor(device))
+	if err == nil {
+		return device, nil
+	}
+	if os.IsNotExist(err) {
+		return "", nil
+	}
+	return "", fmt.Errorf("error checking for device %q: %v", device, err)
+}
+
+// AttachVolume attaches the specified volume to this instance, returning the mountpoint & nil if successful
+func (v *OpenstackVolumes) AttachVolume(volume *Volume) error {
+	opts := volumeattach.CreateOpts{
+		VolumeID: volume.ID,
+	}
+	attachment, err := v.cloud.AttachVolume(v.meta.ServerID, opts)
+	if err != nil {
+		return fmt.Errorf("AttachVolume: failed to attach volume: %s", err)
+	}
+	volume.LocalDevice = attachment.Device
+	return nil
+}
+
+func (g *OpenstackVolumes) GossipSeeds() (gossip.SeedProvider, error) {
+	return gossipos.NewSeedProvider(g.cloud.ComputeClient(), g.clusterName, g.project)
+}
+
+func (g *OpenstackVolumes) InstanceName() string {
+	return g.instanceName
+}