diff --git a/protokube/cmd/protokube/main.go b/protokube/cmd/protokube/main.go index 5717a36868..e0d21bcbbb 100644 --- a/protokube/cmd/protokube/main.go +++ b/protokube/cmd/protokube/main.go @@ -70,7 +70,7 @@ func run() error { flag.BoolVar(&containerized, "containerized", containerized, "Set if we are running containerized.") flag.BoolVar(&initializeRBAC, "initialize-rbac", initializeRBAC, "Set if we should initialize RBAC") flag.BoolVar(&master, "master", master, "Whether or not this node is a master") - flag.StringVar(&cloud, "cloud", "aws", "CloudProvider we are using (aws,digitalocean,gce)") + flag.StringVar(&cloud, "cloud", "aws", "CloudProvider we are using (aws,digitalocean,gce,openstack)") flag.StringVar(&clusterID, "cluster-id", clusterID, "Cluster ID") flag.StringVar(&dnsInternalSuffix, "dns-internal-suffix", dnsInternalSuffix, "DNS suffix for internal domain names") flag.StringVar(&dnsServer, "dns-server", dnsServer, "DNS Server") @@ -179,6 +179,22 @@ func run() error { } internalIP = ip } + } else if cloud == "openstack" { + glog.Info("Initializing openstack volumes") + osVolumes, err := protokube.NewOpenstackVolumes() + if err != nil { + glog.Errorf("Error initializing openstack: %q", err) + os.Exit(1) + } + volumes = osVolumes + if internalIP == nil { + internalIP = osVolumes.InternalIP() + } + + if clusterID == "" { + clusterID = osVolumes.ClusterID() + } + } else { glog.Errorf("Unknown cloud %q", cloud) os.Exit(1) @@ -235,6 +251,12 @@ func run() error { return err } gossipName = volumes.(*protokube.GCEVolumes).InstanceName() + } else if cloud == "openstack" { + gossipSeeds, err = volumes.(*protokube.OpenstackVolumes).GossipSeeds() + if err != nil { + return err + } + gossipName = volumes.(*protokube.OpenstackVolumes).InstanceName() } else { glog.Fatalf("seed provider for %q not yet implemented", cloud) } diff --git a/protokube/pkg/gossip/openstack/BUILD.bazel b/protokube/pkg/gossip/openstack/BUILD.bazel new file mode 100644 index 0000000000..f9625aa8e7 --- /dev/null +++ b/protokube/pkg/gossip/openstack/BUILD.bazel @@ -0,0 +1,15 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["seeds.go"], + importpath = "k8s.io/kops/protokube/pkg/gossip/openstack", + visibility = ["//visibility:public"], + deps = [ + "//protokube/pkg/gossip:go_default_library", + "//vendor/github.com/golang/glog:go_default_library", + "//vendor/github.com/gophercloud/gophercloud:go_default_library", + "//vendor/github.com/gophercloud/gophercloud/openstack/compute/v2/servers:go_default_library", + "//vendor/github.com/gophercloud/gophercloud/pagination:go_default_library", + ], +) diff --git a/protokube/pkg/gossip/openstack/seeds.go b/protokube/pkg/gossip/openstack/seeds.go new file mode 100644 index 0000000000..e4474412d9 --- /dev/null +++ b/protokube/pkg/gossip/openstack/seeds.go @@ -0,0 +1,77 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gce + +import ( + "fmt" + + "github.com/golang/glog" + "github.com/gophercloud/gophercloud" + "github.com/gophercloud/gophercloud/openstack/compute/v2/servers" + "github.com/gophercloud/gophercloud/pagination" + "k8s.io/kops/protokube/pkg/gossip" + "k8s.io/kops/upup/pkg/fi/cloudup/openstack" +) + +type SeedProvider struct { + computeClient *gophercloud.ServiceClient + projectID string + clusterName string +} + +var _ gossip.SeedProvider = &SeedProvider{} + +func (p *SeedProvider) GetSeeds() ([]string, error) { + var seeds []string + + err := servers.List(p.computeClient, servers.ListOpts{ + TenantID: p.projectID, + }).EachPage(func(page pagination.Page) (bool, error) { + var s []servers.Server + err := servers.ExtractServersInto(page, &s) + if err != nil { + return false, err + } + + for _, server := range s { + if clusterName, ok := server.Metadata[openstack.TagClusterName]; ok { + var err error + addr, err := openstack.GetServerFixedIP(&server, clusterName) + if err != nil { + glog.Warningf("Failed to list seeds: %v", err) + continue + } + seeds = append(seeds, addr) + } + } + return true, nil + }) + + if err != nil { + return seeds, fmt.Errorf("Failed to list servers while retrieving seeds: %v", err) + } + + return seeds, nil +} + +func NewSeedProvider(computeClient *gophercloud.ServiceClient, clusterName string, projectID string) (*SeedProvider, error) { + return &SeedProvider{ + computeClient: computeClient, + clusterName: clusterName, + projectID: projectID, + }, nil +} diff --git a/protokube/pkg/protokube/BUILD.bazel b/protokube/pkg/protokube/BUILD.bazel index a586a5796c..0d92dbd70c 100644 --- a/protokube/pkg/protokube/BUILD.bazel +++ b/protokube/pkg/protokube/BUILD.bazel @@ -18,6 +18,7 @@ go_library( "kube_dns.go", "models.go", "nsenter_exec.go", + "openstack_volume.go", "rbac.go", "tainter.go", "utils.go", @@ -37,8 +38,10 @@ go_library( "//protokube/pkg/gossip/aws:go_default_library", "//protokube/pkg/gossip/dns:go_default_library", "//protokube/pkg/gossip/gce:go_default_library", + "//protokube/pkg/gossip/openstack:go_default_library", "//upup/pkg/fi/cloudup/awsup:go_default_library", "//upup/pkg/fi/cloudup/gce:go_default_library", + "//upup/pkg/fi/cloudup/openstack:go_default_library", "//upup/pkg/fi/cloudup/vsphere:go_default_library", "//util/pkg/exec:go_default_library", "//vendor/cloud.google.com/go/compute/metadata:go_default_library", @@ -49,6 +52,8 @@ go_library( "//vendor/github.com/aws/aws-sdk-go/service/ec2:go_default_library", "//vendor/github.com/digitalocean/godo:go_default_library", "//vendor/github.com/golang/glog:go_default_library", + "//vendor/github.com/gophercloud/gophercloud/openstack/blockstorage/v2/volumes:go_default_library", + "//vendor/github.com/gophercloud/gophercloud/openstack/compute/v2/extensions/volumeattach:go_default_library", "//vendor/golang.org/x/net/context:go_default_library", "//vendor/golang.org/x/oauth2/google:go_default_library", "//vendor/google.golang.org/api/compute/v0.beta:go_default_library", diff --git a/protokube/pkg/protokube/openstack_volume.go b/protokube/pkg/protokube/openstack_volume.go new file mode 100644 index 0000000000..c2f0f67e2a --- /dev/null +++ b/protokube/pkg/protokube/openstack_volume.go @@ -0,0 +1,284 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package protokube + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "net" + "net/http" + "os" + "strings" + + "github.com/golang/glog" + cinderv2 "github.com/gophercloud/gophercloud/openstack/blockstorage/v2/volumes" + "github.com/gophercloud/gophercloud/openstack/compute/v2/extensions/volumeattach" + "k8s.io/kops/protokube/pkg/etcd" + "k8s.io/kops/protokube/pkg/gossip" + gossipos "k8s.io/kops/protokube/pkg/gossip/openstack" + "k8s.io/kops/upup/pkg/fi/cloudup/openstack" +) + +const MetadataLatest string = "http://169.254.169.254/openstack/latest/meta_data.json" + +type Metadata struct { + // Matches openstack.TagClusterName + ClusterName string `json:"KubernetesCluster"` +} + +type InstanceMetadata struct { + Name string `json:"name"` + UserMeta *Metadata `json:"meta"` + ProjectID string `json:"project_id"` + AvailabilityZone string `json:"availability_zone"` + Hostname string `json:"hostname"` + ServerID string `json:"uuid"` +} + +// GCEVolumes is the Volumes implementation for GCE +type OpenstackVolumes struct { + cloud openstack.OpenstackCloud + + meta *InstanceMetadata + + clusterName string + project string + instanceName string + internalIP net.IP + storageZone string +} + +var _ Volumes = &OpenstackVolumes{} + +func getLocalMetadata() (*InstanceMetadata, error) { + var meta InstanceMetadata + var client http.Client + resp, err := client.Get(MetadataLatest) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode == http.StatusOK { + bodyBytes, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + err = json.Unmarshal(bodyBytes, &meta) + if err != nil { + return nil, err + } + return &meta, nil + } + return nil, err +} + +// NewOpenstackVolumes builds a OpenstackVolume +func NewOpenstackVolumes() (*OpenstackVolumes, error) { + + metadata, err := getLocalMetadata() + if err != nil { + return nil, fmt.Errorf("Failed to get server metadata: %v", err) + } + + tags := make(map[string]string) + // Cluster name needed to bypass missing designate options + tags[openstack.TagClusterName] = metadata.UserMeta.ClusterName + + oscloud, err := openstack.NewOpenstackCloud(tags, nil) + if err != nil { + return nil, fmt.Errorf("Failed to initialize OpenstackVolumes: %v", err) + } + + a := &OpenstackVolumes{ + cloud: oscloud, + meta: metadata, + } + + err = a.discoverTags() + if err != nil { + return nil, err + } + + return a, nil +} + +// ClusterID implements Volumes ClusterID +func (a *OpenstackVolumes) ClusterID() string { + return a.meta.UserMeta.ClusterName +} + +// Project returns the current GCE project +func (a *OpenstackVolumes) Project() string { + return a.meta.ProjectID +} + +// InternalIP implements Volumes InternalIP +func (a *OpenstackVolumes) InternalIP() net.IP { + return a.internalIP +} + +func (a *OpenstackVolumes) discoverTags() error { + + // Cluster Name + { + a.clusterName = strings.TrimSpace(string(a.meta.UserMeta.ClusterName)) + if a.clusterName == "" { + return fmt.Errorf("cluster name metadata was empty") + } + glog.Infof("Found cluster name=%q", a.clusterName) + } + + // Project ID + { + a.project = strings.TrimSpace(a.meta.ProjectID) + if a.project == "" { + return fmt.Errorf("project metadata was empty") + } + glog.Infof("Found project=%q", a.project) + } + + // Storage Availability Zone + az, err := a.cloud.GetStorageAZFromCompute(a.meta.AvailabilityZone) + if err != nil { + return fmt.Errorf("Could not establish storage availability zone: %v", err) + } + a.storageZone = az.ZoneName + glog.Infof("Found zone=%q", a.storageZone) + + // Instance Name + { + a.instanceName = strings.TrimSpace(a.meta.Name) + if a.instanceName == "" { + return fmt.Errorf("instance name metadata was empty") + } + glog.Infof("Found instanceName=%q", a.instanceName) + } + + // Internal IP + { + ips, err := net.LookupIP(a.meta.Hostname) + if err != nil { + return fmt.Errorf("error querying InternalIP from hostname: %v", err) + } + if len(ips) == 0 { + return fmt.Errorf("ip lookups from metadata hostname was empty") + } + a.internalIP = ips[0] + glog.Infof("Found internalIP=%q", a.internalIP) + } + + return nil +} + +func (v *OpenstackVolumes) buildOpenstackVolume(d *cinderv2.Volume) (*Volume, error) { + volumeName := d.Name + vol := &Volume{ + ID: d.ID, + Info: VolumeInfo{ + Description: volumeName, + }, + } + + vol.Status = d.Status + + for _, attachedTo := range d.Attachments { + vol.AttachedTo = attachedTo.HostName + if attachedTo.ServerID == v.meta.ServerID { + vol.LocalDevice = attachedTo.Device + } + } + + // FIXME: Zone matters, broken in my env + + for k, v := range d.Metadata { + if strings.HasPrefix(k, openstack.TagNameEtcdClusterPrefix) { + etcdClusterName := k[len(openstack.TagNameEtcdClusterPrefix):] + spec, err := etcd.ParseEtcdClusterSpec(etcdClusterName, v) + if err != nil { + return nil, fmt.Errorf("error parsing etcd cluster meta %q on volume %q: %v", v, d.Name, err) + } + vol.Info.EtcdClusters = append(vol.Info.EtcdClusters, spec) + } + } + + return vol, nil +} + +func (v *OpenstackVolumes) FindVolumes() ([]*Volume, error) { + var volumes []*Volume + + glog.V(2).Infof("Listing Openstack disks in %s/%s", v.project, v.meta.AvailabilityZone) + + vols, err := v.cloud.ListVolumes(cinderv2.ListOpts{ + TenantID: v.project, + }) + if err != nil { + return volumes, fmt.Errorf("FindVolumes: Failed to list volume.") + } + + for _, volume := range vols { + if clusterName, ok := volume.Metadata[openstack.TagClusterName]; ok && clusterName == v.clusterName { + if _, isMasterRole := volume.Metadata[openstack.TagNameRolePrefix+"master"]; isMasterRole { + vol, err := v.buildOpenstackVolume(&volume) + if err != nil { + glog.Errorf("FindVolumes: Failed to build openstack volume %s: %v", volume.Name, err) + continue + } + volumes = append(volumes, vol) + } + } + } + + return volumes, nil +} + +// FindMountedVolume implements Volumes::FindMountedVolume +func (v *OpenstackVolumes) FindMountedVolume(volume *Volume) (string, error) { + device := volume.LocalDevice + + _, err := os.Stat(pathFor(device)) + if err == nil { + return device, nil + } + if os.IsNotExist(err) { + return "", nil + } + return "", fmt.Errorf("error checking for device %q: %v", device, err) +} + +// AttachVolume attaches the specified volume to this instance, returning the mountpoint & nil if successful +func (v *OpenstackVolumes) AttachVolume(volume *Volume) error { + opts := volumeattach.CreateOpts{ + VolumeID: volume.ID, + } + attachment, err := v.cloud.AttachVolume(v.meta.ServerID, opts) + if err != nil { + return fmt.Errorf("AttachVolume: failed to attach volume: %s", err) + } + volume.LocalDevice = attachment.Device + return nil +} + +func (g *OpenstackVolumes) GossipSeeds() (gossip.SeedProvider, error) { + return gossipos.NewSeedProvider(g.cloud.ComputeClient(), g.clusterName, g.project) +} + +func (g *OpenstackVolumes) InstanceName() string { + return g.instanceName +}