kubernetes
/
kops
mirror of https://github.com/kubernetes/kops.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

use existing network and subnet

This commit is contained in:
Jesse Haka 2019-09-28 10:46:02 +03:00
parent ac535e4527
commit 5e3b94ae17
30 changed files with 965 additions and 124 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
cmd/kops/BUILD.bazel
View File

@ -93,6 +93,7 @@ go_library(
"//upup/pkg/fi/cloudup/aliup:go_default_library",
"//upup/pkg/fi/cloudup/awsup:go_default_library",
"//upup/pkg/fi/cloudup/gce:go_default_library",
"//upup/pkg/fi/cloudup/openstack:go_default_library",
"//upup/pkg/fi/utils:go_default_library",
"//upup/pkg/kutil:go_default_library",
"//util/pkg/tables:go_default_library",

144
cmd/kops/create_cluster.go
View File

@ -52,6 +52,7 @@ import (
"k8s.io/kops/upup/pkg/fi/cloudup/aliup"
"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
"k8s.io/kops/upup/pkg/fi/cloudup/gce"
"k8s.io/kops/upup/pkg/fi/cloudup/openstack"
"k8s.io/kops/upup/pkg/fi/utils"
"k8s.io/kubernetes/pkg/kubectl/util/i18n"
"k8s.io/kubernetes/pkg/kubectl/util/templates"
@ -155,7 +156,7 @@ type CreateClusterOptions struct {
OpenstackStorageIgnoreAZ bool
OpenstackDNSServers string
OpenstackLbSubnet string
OpenstackNetworkID string
// OpenstackLBOctavia is boolean value should we use octavia or old loadbalancer api
OpenstackLBOctavia bool
@ -389,7 +390,7 @@ func NewCmdCreateCluster(f *util.Factory, out io.Writer) *cobra.Command {
cmd.Flags().BoolVar(&options.OpenstackStorageIgnoreAZ, "os-kubelet-ignore-az", options.OpenstackStorageIgnoreAZ, "If true kubernetes may attach volumes across availability zones")
cmd.Flags().BoolVar(&options.OpenstackLBOctavia, "os-octavia", options.OpenstackLBOctavia, "If true octavia loadbalancer api will be used")
cmd.Flags().StringVar(&options.OpenstackDNSServers, "os-dns-servers", options.OpenstackDNSServers, "comma separated list of DNS Servers which is used in network")
cmd.Flags().StringVar(&options.OpenstackNetworkID, "os-network", options.OpenstackNetworkID, "The ID of the existing OpenStack network to use")
return cmd
}
@ -510,6 +511,50 @@ func RunCreateCluster(f *util.Factory, out io.Writer, c *CreateClusterOptions) e
cluster.Spec.NetworkID = *res.Subnets[0].VpcId
}
if api.CloudProviderID(cluster.Spec.CloudProvider) == api.CloudProviderOpenstack {
if cluster.Spec.CloudConfig == nil {
cluster.Spec.CloudConfig = &api.CloudConfiguration{}
}
cluster.Spec.CloudConfig.Openstack = &api.OpenstackConfiguration{
Router: &api.OpenstackRouter{
ExternalNetwork: fi.String(c.OpenstackExternalNet),
},
BlockStorage: &api.OpenstackBlockStorageConfig{
Version: fi.String("v2"),
IgnoreAZ: fi.Bool(c.OpenstackStorageIgnoreAZ),
},
Monitor: &api.OpenstackMonitor{
Delay: fi.String("1m"),
Timeout: fi.String("30s"),
MaxRetries: fi.Int(3),
},
}
if c.OpenstackNetworkID != "" {
cluster.Spec.NetworkID = c.OpenstackNetworkID
} else if len(c.SubnetIDs) > 0 {
tags := make(map[string]string)
tags[openstack.TagClusterName] = c.ClusterName
osCloud, err := openstack.NewOpenstackCloud(tags, &cluster.Spec)
if err != nil {
return fmt.Errorf("error loading cloud: %v", err)
}
res, err := osCloud.FindNetworkBySubnetID(c.SubnetIDs[0])
if err != nil {
return fmt.Errorf("error finding network: %v", err)
}
cluster.Spec.NetworkID = res.ID
}
if c.OpenstackDNSServers != "" {
cluster.Spec.CloudConfig.Openstack.Router.DNSServers = fi.String(c.OpenstackDNSServers)
}
if c.OpenstackExternalSubnet != "" {
cluster.Spec.CloudConfig.Openstack.Router.ExternalSubnet = fi.String(c.OpenstackExternalSubnet)
}
}
if cluster.Spec.CloudProvider == "" {
for _, zone := range allZones.List() {
cloud, known := fi.GuessCloudForZone(zone)
@ -601,10 +646,19 @@ func RunCreateCluster(f *util.Factory, out io.Writer, c *CreateClusterOptions) e
}
} else {
var zoneToSubnetProviderID map[string]string
if len(c.Zones) > 0 && len(c.SubnetIDs) > 0 && api.CloudProviderID(cluster.Spec.CloudProvider) == api.CloudProviderAWS {
zoneToSubnetProviderID, err = getZoneToSubnetProviderID(cluster.Spec.NetworkID, c.Zones[0][:len(c.Zones[0])-1], c.SubnetIDs)
if err != nil {
return err
if len(c.Zones) > 0 && len(c.SubnetIDs) > 0 {
if api.CloudProviderID(cluster.Spec.CloudProvider) == api.CloudProviderAWS {
zoneToSubnetProviderID, err = getZoneToSubnetProviderID(cluster.Spec.NetworkID, c.Zones[0][:len(c.Zones[0])-1], c.SubnetIDs)
if err != nil {
return err
}
} else if api.CloudProviderID(cluster.Spec.CloudProvider) == api.CloudProviderOpenstack {
tags := make(map[string]string)
tags[openstack.TagClusterName] = c.ClusterName
zoneToSubnetProviderID, err = getSubnetProviderID(&cluster.Spec, allZones.List(), c.SubnetIDs, tags)
if err != nil {
return err
}
}
}
for _, zoneName := range allZones.List() {
@ -905,32 +959,6 @@ func RunCreateCluster(f *util.Factory, out io.Writer, c *CreateClusterOptions) e
cluster.Spec.CloudConfig.SpotinstOrientation = fi.String(c.SpotinstOrientation)
}
}
if c.Cloud == "openstack" {
if cluster.Spec.CloudConfig == nil {
cluster.Spec.CloudConfig = &api.CloudConfiguration{}
}
cluster.Spec.CloudConfig.Openstack = &api.OpenstackConfiguration{
Router: &api.OpenstackRouter{
ExternalNetwork: fi.String(c.OpenstackExternalNet),
},
BlockStorage: &api.OpenstackBlockStorageConfig{
Version: fi.String("v2"),
IgnoreAZ: fi.Bool(c.OpenstackStorageIgnoreAZ),
},
Monitor: &api.OpenstackMonitor{
Delay: fi.String("1m"),
Timeout: fi.String("30s"),
MaxRetries: fi.Int(3),
},
}
if c.OpenstackDNSServers != "" {
cluster.Spec.CloudConfig.Openstack.Router.DNSServers = fi.String(c.OpenstackDNSServers)
}
if c.OpenstackExternalSubnet != "" {
cluster.Spec.CloudConfig.Openstack.Router.ExternalSubnet = fi.String(c.OpenstackExternalSubnet)
}
}
}
// Populate project
@ -1062,10 +1090,19 @@ func RunCreateCluster(f *util.Factory, out io.Writer, c *CreateClusterOptions) e
var utilitySubnets []api.ClusterSubnetSpec
var zoneToSubnetProviderID map[string]string
if len(c.Zones) > 0 && len(c.UtilitySubnetIDs) > 0 && api.CloudProviderID(cluster.Spec.CloudProvider) == api.CloudProviderAWS {
zoneToSubnetProviderID, err = getZoneToSubnetProviderID(cluster.Spec.NetworkID, c.Zones[0][:len(c.Zones[0])-1], c.UtilitySubnetIDs)
if err != nil {
return err
if len(c.Zones) > 0 && len(c.UtilitySubnetIDs) > 0 {
if api.CloudProviderID(cluster.Spec.CloudProvider) == api.CloudProviderAWS {
zoneToSubnetProviderID, err = getZoneToSubnetProviderID(cluster.Spec.NetworkID, c.Zones[0][:len(c.Zones[0])-1], c.UtilitySubnetIDs)
if err != nil {
return err
}
} else if api.CloudProviderID(cluster.Spec.CloudProvider) == api.CloudProviderOpenstack {
tags := make(map[string]string)
tags[openstack.TagClusterName] = c.ClusterName
zoneToSubnetProviderID, err = getSubnetProviderID(&cluster.Spec, allZones.List(), c.UtilitySubnetIDs, tags)
if err != nil {
return err
}
}
}
@ -1484,6 +1521,41 @@ func getZoneToSubnetProviderID(VPCID string, region string, subnetIDs []string)
return res, nil
}
func getSubnetProviderID(spec *api.ClusterSpec, zones []string, subnetIDs []string, tags map[string]string) (map[string]string, error) {
res := make(map[string]string)
osCloud, err := openstack.NewOpenstackCloud(tags, spec)
if err != nil {
return res, fmt.Errorf("error loading cloud: %v", err)
}
osCloud.UseZones(zones)
networkInfo, err := osCloud.FindVPCInfo(spec.NetworkID)
if err != nil {
return res, fmt.Errorf("error describing Network: %v", err)
}
if networkInfo == nil {
return res, fmt.Errorf("network %q not found", spec.NetworkID)
}
subnetByID := make(map[string]*fi.SubnetInfo)
for _, subnetInfo := range networkInfo.Subnets {
subnetByID[subnetInfo.ID] = subnetInfo
}
for _, subnetID := range subnetIDs {
subnet, ok := subnetByID[subnetID]
if !ok {
return res, fmt.Errorf("subnet %s not found in network %s", subnetID, spec.NetworkID)
}
if res[subnet.Zone] != "" {
return res, fmt.Errorf("subnet %s and %s have the same zone", subnetID, res[subnet.Zone])
}
res[subnet.Zone] = subnetID
}
return res, nil
}
func loadSSHPublicKeys(sshPublicKey string) (map[string][]byte, error) {
sshPublicKeys := make(map[string][]byte)
if sshPublicKey != "" {

1
docs/cli/kops_create_cluster.md
View File

@ -102,6 +102,7 @@ kops create cluster [flags]
--os-ext-subnet string The name of the external floating subnet to use with the openstack router
--os-kubelet-ignore-az If true kubernetes may attach volumes across availability zones
--os-lb-floating-subnet string The name of the external subnet to use with the kubernetes api
--os-network string The ID of the existing OpenStack network to use
--os-octavia If true octavia loadbalancer api will be used
--out string Path to write any local output
-o, --output string Output format. One of json|yaml. Used with the --dry-run flag.

50
docs/getting_started/openstack.md
View File

@ -150,6 +150,56 @@ kops create cluster \
The biggest problem currently when installing without loadbalancer is that kubectl requests outside cluster is always going to first master. External loadbalancer is one option which can solve this issue.
# Using existing OpenStack network
**Warning!** This feature is **experimental** use only if you know what you are doing.
By default KOPS will always create new network to your OpenStack project which name matches to your clustername. However, there is experimental feature to use existing network in OpenStack project. When you create new cluster you can specify flag `--os-network <network id>` and it will then use existing network.
Using yaml this can be specified to yaml:
```yaml
spec:
...
networkID: <network id>
...
```
**Warning!** when deleting cluster, you need to be really careful that you do not break another dependencies under same network. Run `kops delete cluster` without `--yes` flag and go through the list. Otherwise you might see situation that you broke something else.
# Using existing OpenStack subnets
**Warning!** This feature is **experimental** use only if you know what you are doing.
By default KOPS will always create new network and subnet to your OpenStack project. However, there is experimental feature to use existing network and subnets in OpenStack project. When you create new cluster you can specify flag `--subnets <commaseparated list of subnetids>` and it will then use existing subnet. There is similar flag for utility subnets `--utility-subnets <commaseparated list of subnetids>`.
Example:
```
kops create cluster \
--cloud openstack \
--name sharedsub2.k8s.local \
--state ${KOPS_STATE_STORE} \
--zones zone-1 \
--network-cidr 10.1.0.0/16 \
--image debian-10-160819-devops \
--master-count=3 \
--node-count=2 \
--node-size m1.small \
--master-size m1.small \
--etcd-storage-type default \
--topology private \
--bastion \
--networking calico \
--api-loadbalancer-type public \
--os-kubelet-ignore-az=true \
--os-ext-net ext-net \
--subnets c7d20c0f-df3a-4e5b-842f-f633c182961f \
--utility-subnets 90871d21-b546-4c4a-a7c9-2337ddf5375f \
--os-octavia=true --yes
```
**Warning!** when deleting cluster, you need to be really careful that you do not break another dependencies under same network & subnet. Run `kops delete cluster` without `--yes` flag and go through the list. Otherwise you might see situation that you broke something else.
# Using with self-signed certificates in OpenStack
Kops can be configured to use insecure mode towards OpenStack. However, this is **NOT** recommended as OpenStack cloudprovider in kubernetes does not support it.

62
pkg/model/openstackmodel/context.go
View File

@ -17,8 +17,12 @@ limitations under the License.
package openstackmodel
import (
"fmt"
"k8s.io/klog"
"k8s.io/kops/pkg/model"
"k8s.io/kops/upup/pkg/fi"
"k8s.io/kops/upup/pkg/fi/cloudup/openstack"
"k8s.io/kops/upup/pkg/fi/cloudup/openstacktasks"
)
@ -26,8 +30,64 @@ type OpenstackModelContext struct {
*model.KopsModelContext
}
func (c *OpenstackModelContext) GetNetworkName() (string, error) {
if c.Cluster.Spec.NetworkID == "" {
return c.ClusterName(), nil
}
tags := make(map[string]string)
tags[openstack.TagClusterName] = c.ClusterName()
osCloud, err := openstack.NewOpenstackCloud(tags, &c.Cluster.Spec)
if err != nil {
return "", fmt.Errorf("error loading cloud: %v", err)
}
network, err := osCloud.GetNetwork(c.Cluster.Spec.NetworkID)
if err != nil {
return "", err
}
return network.Name, nil
}
func (c *OpenstackModelContext) findSubnetClusterSpec(subnet string) (string, error) {
for _, sp := range c.Cluster.Spec.Subnets {
if sp.Name == subnet {
name, err := c.findSubnetNameByID(sp.ProviderID, sp.Name)
if err != nil {
return "", err
}
return name, nil
}
}
return "", fmt.Errorf("Could not find subnet %s from clusterSpec", subnet)
}
func (c *OpenstackModelContext) findSubnetNameByID(subnetID string, subnetName string) (string, error) {
if subnetID == "" {
return subnetName + "." + c.ClusterName(), nil
}
tags := make(map[string]string)
tags[openstack.TagClusterName] = c.ClusterName()
osCloud, err := openstack.NewOpenstackCloud(tags, &c.Cluster.Spec)
if err != nil {
return "", fmt.Errorf("error loading cloud: %v", err)
}
subnet, err := osCloud.GetSubnet(subnetID)
if err != nil {
return "", err
}
return subnet.Name, nil
}
func (c *OpenstackModelContext) LinkToNetwork() *openstacktasks.Network {
return &openstacktasks.Network{Name: s(c.ClusterName())}
netName, err := c.GetNetworkName()
if err != nil {
klog.Infof("Could not find networkname")
return nil
}
return &openstacktasks.Network{Name: s(netName)}
}
func (c *OpenstackModelContext) LinkToRouter(name *string) *openstacktasks.Router {

51
pkg/model/openstackmodel/network.go
View File

@ -33,35 +33,40 @@ var _ fi.ModelBuilder = &NetworkModelBuilder{}
func (b *NetworkModelBuilder) Build(c *fi.ModelBuilderContext) error {
clusterName := b.ClusterName()
routerName := strings.Replace(clusterName, ".", "-", -1)
netName, err := b.GetNetworkName()
if err != nil {
return err
}
{
t := &openstacktasks.Network{
Name: s(clusterName),
Name: s(netName),
ID: s(b.Cluster.Spec.NetworkID),
Tag: s(clusterName),
Lifecycle: b.Lifecycle,
}
c.AddTask(t)
}
{
t := &openstacktasks.Router{
Name: s(routerName),
Lifecycle: b.Lifecycle,
}
c.AddTask(t)
}
needRouter := true
routerName := strings.Replace(clusterName, ".", "-", -1)
for _, sp := range b.Cluster.Spec.Subnets {
subnetName := sp.Name + "." + b.ClusterName()
// assumes that we do not need to create routers if we use existing subnets
if sp.ProviderID != "" {
needRouter = false
}
subnetName, err := b.findSubnetNameByID(sp.ProviderID, sp.Name)
if err != nil {
return err
}
t := &openstacktasks.Subnet{
Name: s(subnetName),
Network: b.LinkToNetwork(),
CIDR: s(sp.CIDR),
DNSServers: make([]*string, 0),
Lifecycle: b.Lifecycle,
Tag: s(clusterName),
}
if b.Cluster.Spec.CloudConfig.Openstack.Router != nil && b.Cluster.Spec.CloudConfig.Openstack.Router.DNSServers != nil {
dnsSplitted := strings.Split(fi.StringValue(b.Cluster.Spec.CloudConfig.Openstack.Router.DNSServers), ",")
@ -73,14 +78,24 @@ func (b *NetworkModelBuilder) Build(c *fi.ModelBuilderContext) error {
}
c.AddTask(t)
t1 := &openstacktasks.RouterInterface{
Name: s("ri-" + sp.Name),
Subnet: b.LinkToSubnet(s(subnetName)),
Router: b.LinkToRouter(s(routerName)),
Lifecycle: b.Lifecycle,
if needRouter {
t1 := &openstacktasks.RouterInterface{
Name: s("ri-" + sp.Name),
Subnet: b.LinkToSubnet(s(subnetName)),
Router: b.LinkToRouter(s(routerName)),
Lifecycle: b.Lifecycle,
}
c.AddTask(t1)
}
c.AddTask(t1)
}
if needRouter {
t := &openstacktasks.Router{
Name: s(routerName),
Lifecycle: b.Lifecycle,
}
c.AddTask(t)
}
return nil
}

36
pkg/model/openstackmodel/servergroup.go
View File

@ -59,6 +59,11 @@ func (b *ServerGroupModelBuilder) buildInstances(c *fi.ModelBuilderContext, sg *
igMeta[openstack.TagClusterName] = b.ClusterName()
}
igMeta["k8s"] = b.ClusterName()
netName, err := b.GetNetworkName()
if err != nil {
return err
}
igMeta[openstack.TagKopsNetwork] = netName
igMeta["KopsInstanceGroup"] = ig.Name
igMeta["KopsRole"] = string(ig.Spec.Role)
igMeta[openstack.INSTANCE_GROUP_GENERATION] = fmt.Sprintf("%d", ig.GetGeneration())
@ -111,7 +116,12 @@ func (b *ServerGroupModelBuilder) buildInstances(c *fi.ModelBuilderContext, sg *
} else {
az = fi.String(subnet)
}
subnets = append(subnets, b.LinkToSubnet(s(fmt.Sprintf("%s.%s", subnet, b.ClusterName()))))
subnetName, err := b.findSubnetClusterSpec(subnet)
if err != nil {
return err
}
subnets = append(subnets, b.LinkToSubnet(s(subnetName)))
}
if len(ig.Spec.Zones) > 0 {
zone := ig.Spec.Zones[int(i)%len(ig.Spec.Zones)]
@ -121,6 +131,7 @@ func (b *ServerGroupModelBuilder) buildInstances(c *fi.ModelBuilderContext, sg *
portTask := &openstacktasks.Port{
Name: fi.String(fmt.Sprintf("%s-%s", "port", *instanceName)),
Network: b.LinkToNetwork(),
Tag: s(b.ClusterName()),
SecurityGroups: securityGroups,
AdditionalSecurityGroups: ig.Spec.AdditionalSecurityGroups,
Subnets: subnets,
@ -248,10 +259,23 @@ func (b *ServerGroupModelBuilder) Build(c *fi.ModelBuilderContext) error {
}
if b.Cluster.Spec.CloudConfig.Openstack.Loadbalancer != nil {
lbSubnetName := b.MasterInstanceGroups()[0].Spec.Subnets[0]
var lbSubnetName string
var err error
for _, sp := range b.Cluster.Spec.Subnets {
if sp.Type == kops.SubnetTypePrivate {
lbSubnetName, err = b.findSubnetNameByID(sp.ProviderID, sp.Name)
if err != nil {
return err
}
break
}
}
if lbSubnetName == "" {
return fmt.Errorf("could not find subnet for master loadbalancer")
}
lbTask := &openstacktasks.LB{
Name: fi.String(b.Cluster.Spec.MasterPublicName),
Subnet: fi.String(lbSubnetName + "." + b.ClusterName()),
Subnet: fi.String(lbSubnetName),
Lifecycle: b.Lifecycle,
SecurityGroup: b.LinkToSecurityGroup(b.Cluster.Spec.MasterPublicName),
}
@ -282,12 +306,16 @@ func (b *ServerGroupModelBuilder) Build(c *fi.ModelBuilderContext) error {
}
c.AddTask(listenerTask)
ifName, err := b.GetNetworkName()
if err != nil {
return err
}
for _, mastersg := range masters {
associateTask := &openstacktasks.PoolAssociation{
Name: mastersg.Name,
Pool: poolTask,
ServerGroup: mastersg,
InterfaceName: fi.String(clusterName),
InterfaceName: fi.String(ifName),
ProtocolPort: fi.Int(443),
Lifecycle: b.Lifecycle,
}

91
pkg/model/openstackmodel/servergroup_test.go
View File

@ -57,6 +57,7 @@ func Test_ServerGroupModelBuilder(t *testing.T) {
},
Subnets: []kops.ClusterSubnetSpec{
{
Name: "subnet",
Region: "region",
},
},
@ -128,6 +129,7 @@ func Test_ServerGroupModelBuilder(t *testing.T) {
Metadata: map[string]string{
"KubernetesCluster": "cluster",
"k8s": "cluster",
"KopsNetwork": "cluster",
"KopsInstanceGroup": "master",
"KopsRole": "Master",
"ig_generation": "0",
@ -173,6 +175,7 @@ func Test_ServerGroupModelBuilder(t *testing.T) {
Metadata: map[string]string{
"KubernetesCluster": "cluster",
"k8s": "cluster",
"KopsNetwork": "cluster",
"KopsInstanceGroup": "node",
"KopsRole": "Node",
"ig_generation": "0",
@ -214,6 +217,11 @@ func Test_ServerGroupModelBuilder(t *testing.T) {
},
Subnets: []kops.ClusterSubnetSpec{
{
Name: "subnet",
Region: "region",
},
{
Name: "utility-subnet",
Region: "region",
},
},
@ -306,6 +314,7 @@ func Test_ServerGroupModelBuilder(t *testing.T) {
Metadata: map[string]string{
"KubernetesCluster": "cluster",
"k8s": "cluster",
"KopsNetwork": "cluster",
"KopsInstanceGroup": "master",
"KopsRole": "Master",
"ig_generation": "0",
@ -351,6 +360,7 @@ func Test_ServerGroupModelBuilder(t *testing.T) {
Metadata: map[string]string{
"KubernetesCluster": "cluster",
"k8s": "cluster",
"KopsNetwork": "cluster",
"KopsInstanceGroup": "node",
"KopsRole": "Node",
"ig_generation": "0",
@ -391,6 +401,7 @@ func Test_ServerGroupModelBuilder(t *testing.T) {
Metadata: map[string]string{
"k8s": "cluster",
"KopsInstanceGroup": "bastion",
"KopsNetwork": "cluster",
"KopsRole": "Bastion",
"ig_generation": "0",
"cluster_generation": "0",
@ -434,6 +445,15 @@ func Test_ServerGroupModelBuilder(t *testing.T) {
},
Subnets: []kops.ClusterSubnetSpec{
{
Name: "subnet-a",
Region: "region",
},
{
Name: "subnet-b",
Region: "region",
},
{
Name: "subnet-c",
Region: "region",
},
},
@ -561,6 +581,7 @@ func Test_ServerGroupModelBuilder(t *testing.T) {
Metadata: map[string]string{
"KubernetesCluster": "cluster",
"k8s": "cluster",
"KopsNetwork": "cluster",
"KopsInstanceGroup": "master-a",
"KopsRole": "Master",
"ig_generation": "0",
@ -607,6 +628,7 @@ func Test_ServerGroupModelBuilder(t *testing.T) {
Metadata: map[string]string{
"KubernetesCluster": "cluster",
"k8s": "cluster",
"KopsNetwork": "cluster",
"KopsInstanceGroup": "master-b",
"KopsRole": "Master",
"ig_generation": "0",
@ -653,6 +675,7 @@ func Test_ServerGroupModelBuilder(t *testing.T) {
Metadata: map[string]string{
"KubernetesCluster": "cluster",
"k8s": "cluster",
"KopsNetwork": "cluster",
"KopsInstanceGroup": "master-c",
"KopsRole": "Master",
"ig_generation": "0",
@ -698,6 +721,7 @@ func Test_ServerGroupModelBuilder(t *testing.T) {
Metadata: map[string]string{
"KubernetesCluster": "cluster",
"k8s": "cluster",
"KopsNetwork": "cluster",
"KopsInstanceGroup": "node-a",
"KopsRole": "Node",
"ig_generation": "0",
@ -743,6 +767,7 @@ func Test_ServerGroupModelBuilder(t *testing.T) {
Metadata: map[string]string{
"KubernetesCluster": "cluster",
"k8s": "cluster",
"KopsNetwork": "cluster",
"KopsInstanceGroup": "node-b",
"KopsRole": "Node",
"ig_generation": "0",
@ -788,6 +813,7 @@ func Test_ServerGroupModelBuilder(t *testing.T) {
Metadata: map[string]string{
"KubernetesCluster": "cluster",
"k8s": "cluster",
"KopsNetwork": "cluster",
"KopsInstanceGroup": "node-c",
"KopsRole": "Node",
"ig_generation": "0",
@ -846,7 +872,19 @@ func Test_ServerGroupModelBuilder(t *testing.T) {
},
Subnets: []kops.ClusterSubnetSpec{
{
Name: "subnet-a",
Region: "region",
Type: kops.SubnetTypePrivate,
},
{
Name: "subnet-b",
Region: "region",
Type: kops.SubnetTypePrivate,
},
{
Name: "subnet-c",
Region: "region",
Type: kops.SubnetTypePrivate,
},
},
},
@ -972,6 +1010,7 @@ func Test_ServerGroupModelBuilder(t *testing.T) {
Metadata: map[string]string{
"KubernetesCluster": "cluster",
"k8s": "cluster",
"KopsNetwork": "cluster",
"KopsInstanceGroup": "master-a",
"KopsRole": "Master",
"ig_generation": "0",
@ -1012,6 +1051,7 @@ func Test_ServerGroupModelBuilder(t *testing.T) {
Metadata: map[string]string{
"KubernetesCluster": "cluster",
"k8s": "cluster",
"KopsNetwork": "cluster",
"KopsInstanceGroup": "master-b",
"KopsRole": "Master",
"ig_generation": "0",
@ -1052,6 +1092,7 @@ func Test_ServerGroupModelBuilder(t *testing.T) {
Metadata: map[string]string{
"KubernetesCluster": "cluster",
"k8s": "cluster",
"KopsNetwork": "cluster",
"KopsInstanceGroup": "master-c",
"KopsRole": "Master",
"ig_generation": "0",
@ -1092,6 +1133,7 @@ func Test_ServerGroupModelBuilder(t *testing.T) {
Metadata: map[string]string{
"KubernetesCluster": "cluster",
"k8s": "cluster",
"KopsNetwork": "cluster",
"KopsInstanceGroup": "node-a",
"KopsRole": "Node",
"ig_generation": "0",
@ -1137,6 +1179,7 @@ func Test_ServerGroupModelBuilder(t *testing.T) {
Metadata: map[string]string{
"KubernetesCluster": "cluster",
"k8s": "cluster",
"KopsNetwork": "cluster",
"KopsInstanceGroup": "node-b",
"KopsRole": "Node",
"ig_generation": "0",
@ -1182,6 +1225,7 @@ func Test_ServerGroupModelBuilder(t *testing.T) {
Metadata: map[string]string{
"KubernetesCluster": "cluster",
"k8s": "cluster",
"KopsNetwork": "cluster",
"KopsInstanceGroup": "node-c",
"KopsRole": "Node",
"ig_generation": "0",
@ -1286,6 +1330,15 @@ func Test_ServerGroupModelBuilder(t *testing.T) {
},
Subnets: []kops.ClusterSubnetSpec{
{
Name: "subnet-a",
Region: "region",
},
{
Name: "subnet-b",
Region: "region",
},
{
Name: "subnet-c",
Region: "region",
},
},
@ -1413,6 +1466,7 @@ func Test_ServerGroupModelBuilder(t *testing.T) {
Metadata: map[string]string{
"KubernetesCluster": "cluster",
"k8s": "cluster",
"KopsNetwork": "cluster",
"KopsInstanceGroup": "master-a",
"KopsRole": "Master",
"ig_generation": "0",
@ -1454,6 +1508,7 @@ func Test_ServerGroupModelBuilder(t *testing.T) {
Metadata: map[string]string{
"KubernetesCluster": "cluster",
"k8s": "cluster",
"KopsNetwork": "cluster",
"KopsInstanceGroup": "master-b",
"KopsRole": "Master",
"ig_generation": "0",
@ -1495,6 +1550,7 @@ func Test_ServerGroupModelBuilder(t *testing.T) {
Metadata: map[string]string{
"KubernetesCluster": "cluster",
"k8s": "cluster",
"KopsNetwork": "cluster",
"KopsInstanceGroup": "master-c",
"KopsRole": "Master",
"ig_generation": "0",
@ -1535,6 +1591,7 @@ func Test_ServerGroupModelBuilder(t *testing.T) {
Metadata: map[string]string{
"KubernetesCluster": "cluster",
"k8s": "cluster",
"KopsNetwork": "cluster",
"KopsInstanceGroup": "node-a",
"KopsRole": "Node",
"ig_generation": "0",
@ -1575,6 +1632,7 @@ func Test_ServerGroupModelBuilder(t *testing.T) {
Metadata: map[string]string{
"KubernetesCluster": "cluster",
"k8s": "cluster",
"KopsNetwork": "cluster",
"KopsInstanceGroup": "node-b",
"KopsRole": "Node",
"ig_generation": "0",
@ -1619,6 +1677,7 @@ func Test_ServerGroupModelBuilder(t *testing.T) {
"KopsRole": "Node",
"ig_generation": "0",
"cluster_generation": "0",
"KopsNetwork": "cluster",
},
AvailabilityZone: s("zone-3"),
}
@ -1661,6 +1720,15 @@ func Test_ServerGroupModelBuilder(t *testing.T) {
},
Subnets: []kops.ClusterSubnetSpec{
{
Name: "subnet-a",
Region: "region",
},
{
Name: "subnet-b",
Region: "region",
},
{
Name: "subnet-c",
Region: "region",
},
},
@ -1748,6 +1816,7 @@ func Test_ServerGroupModelBuilder(t *testing.T) {
Metadata: map[string]string{
"KubernetesCluster": "cluster",
"k8s": "cluster",
"KopsNetwork": "cluster",
"KopsInstanceGroup": "master",
"KopsRole": "Master",
"ig_generation": "0",
@ -1786,6 +1855,7 @@ func Test_ServerGroupModelBuilder(t *testing.T) {
Metadata: map[string]string{
"KubernetesCluster": "cluster",
"k8s": "cluster",
"KopsNetwork": "cluster",
"KopsInstanceGroup": "master",
"KopsRole": "Master",
"ig_generation": "0",
@ -1824,6 +1894,7 @@ func Test_ServerGroupModelBuilder(t *testing.T) {
Metadata: map[string]string{
"KubernetesCluster": "cluster",
"k8s": "cluster",
"KopsNetwork": "cluster",
"KopsInstanceGroup": "master",
"KopsRole": "Master",
"ig_generation": "0",
@ -1869,6 +1940,7 @@ func Test_ServerGroupModelBuilder(t *testing.T) {
Metadata: map[string]string{
"KubernetesCluster": "cluster",
"k8s": "cluster",
"KopsNetwork": "cluster",
"KopsInstanceGroup": "node",
"KopsRole": "Node",
"ig_generation": "0",
@ -1906,6 +1978,7 @@ func Test_ServerGroupModelBuilder(t *testing.T) {
Metadata: map[string]string{
"KubernetesCluster": "cluster",
"k8s": "cluster",
"KopsNetwork": "cluster",
"KopsInstanceGroup": "node",
"KopsRole": "Node",
"ig_generation": "0",
@ -1943,6 +2016,7 @@ func Test_ServerGroupModelBuilder(t *testing.T) {
Metadata: map[string]string{
"KubernetesCluster": "cluster",
"k8s": "cluster",
"KopsNetwork": "cluster",
"KopsInstanceGroup": "node",
"KopsRole": "Node",
"ig_generation": "0",
@ -1996,6 +2070,7 @@ func Test_ServerGroupModelBuilder(t *testing.T) {
},
Subnets: []kops.ClusterSubnetSpec{
{
Name: "subnet",
Region: "region",
},
},
@ -2073,6 +2148,7 @@ func Test_ServerGroupModelBuilder(t *testing.T) {
"KopsRole": "Master",
"ig_generation": "0",
"cluster_generation": "0",
"KopsNetwork": "cluster",
},
AvailabilityZone: s("zone-1"),
}
@ -2113,6 +2189,7 @@ func Test_ServerGroupModelBuilder(t *testing.T) {
"KopsRole": "Node",
"ig_generation": "0",
"cluster_generation": "0",
"KopsNetwork": "cluster",
},
AvailabilityZone: s("zone-1"),
}
@ -2143,6 +2220,11 @@ func Test_ServerGroupModelBuilder(t *testing.T) {
},
Subnets: []kops.ClusterSubnetSpec{
{
Name: "subnet",
Region: "region",
},
{
Name: "utility-subnet",
Region: "region",
},
},
@ -2242,6 +2324,7 @@ func Test_ServerGroupModelBuilder(t *testing.T) {
"KopsRole": "Master",
"ig_generation": "0",
"cluster_generation": "0",
"KopsNetwork": "cluster",
},
AvailabilityZone: s("zone-1"),
}
@ -2282,6 +2365,7 @@ func Test_ServerGroupModelBuilder(t *testing.T) {
"KopsRole": "Node",
"ig_generation": "0",
"cluster_generation": "0",
"KopsNetwork": "cluster",
},
AvailabilityZone: s("zone-1"),
}
@ -2318,6 +2402,7 @@ func Test_ServerGroupModelBuilder(t *testing.T) {
Metadata: map[string]string{
"k8s": "cluster",
"KopsInstanceGroup": "bastion",
"KopsNetwork": "cluster",
"KopsRole": "Bastion",
"ig_generation": "0",
"cluster_generation": "0",
@ -2350,6 +2435,7 @@ func Test_ServerGroupModelBuilder(t *testing.T) {
},
Subnets: []kops.ClusterSubnetSpec{
{
Name: "subnet",
Region: "region",
},
},
@ -2412,6 +2498,7 @@ func Test_ServerGroupModelBuilder(t *testing.T) {
Metadata: map[string]string{
"KubernetesCluster": "cluster",
"k8s": "cluster",
"KopsNetwork": "cluster",
"KopsInstanceGroup": "node",
"KopsRole": "Node",
"ig_generation": "0",
@ -2442,6 +2529,7 @@ func Test_ServerGroupModelBuilder(t *testing.T) {
},
Subnets: []kops.ClusterSubnetSpec{
{
Name: "subnet",
Region: "region",
},
},
@ -2506,6 +2594,7 @@ func Test_ServerGroupModelBuilder(t *testing.T) {
Metadata: map[string]string{
"KubernetesCluster": "cluster",
"k8s": "cluster",
"KopsNetwork": "cluster",
"KopsInstanceGroup": "node",
"KopsRole": "Node",
"ig_generation": "0",
@ -2536,6 +2625,7 @@ func Test_ServerGroupModelBuilder(t *testing.T) {
},
Subnets: []kops.ClusterSubnetSpec{
{
Name: "subnet",
Region: "region",
},
},
@ -2602,6 +2692,7 @@ func Test_ServerGroupModelBuilder(t *testing.T) {
"KopsRole": "Node",
"ig_generation": "0",
"cluster_generation": "0",
"KopsNetwork": "cluster",
},
AvailabilityZone: s("subnet"),
SecurityGroups: []string{

22
pkg/resources/openstack/lb.go
View File

@ -17,6 +17,9 @@ limitations under the License.
package openstack
import (
"fmt"
"strings"
"github.com/gophercloud/gophercloud/openstack/loadbalancer/v2/loadbalancers"
"github.com/gophercloud/gophercloud/openstack/loadbalancer/v2/monitors"
"github.com/gophercloud/gophercloud/openstack/networking/v2/subnets"
@ -36,6 +39,11 @@ const (
func (os *clusterDiscoveryOS) DeleteSubnetLBs(subnet subnets.Subnet) ([]*resources.Resource, error) {
var resourceTrackers []*resources.Resource
preExistingSubnet := false
if !strings.HasSuffix(subnet.Name, os.clusterName) {
preExistingSubnet = true
}
opts := loadbalancers.ListOpts{
VipSubnetID: subnet.ID,
}
@ -44,7 +52,19 @@ func (os *clusterDiscoveryOS) DeleteSubnetLBs(subnet subnets.Subnet) ([]*resourc
return nil, err
}
for _, lb := range lbs {
filteredLBs := []loadbalancers.LoadBalancer{}
if preExistingSubnet {
// if we have preExistingSubnet, we cannot delete others than api LB
for _, lb := range lbs {
if lb.Name == fmt.Sprintf("api.%s", os.clusterName) {
filteredLBs = append(filteredLBs, lb)
}
}
} else {
filteredLBs = lbs
}
for _, lb := range filteredLBs {
resourceTracker := &resources.Resource{
Name: lb.Name,
ID: lb.ID,

140
pkg/resources/openstack/network.go
View File

@ -29,26 +29,39 @@ import (
)
const (
typeRouterIF = "Router-IF"
typeRouter = "Router"
typeSubnet = "Subnet"
typeNetwork = "Network"
typeRouterIF = "Router-IF"
typeRouter = "Router"
typeSubnet = "Subnet"
typeNetwork = "Network"
typeNetworkTag = "NetworkTag"
typeSubnetTag = "SubnetTag"
)
func (os *clusterDiscoveryOS) ListNetwork() ([]*resources.Resource, error) {
var resourceTrackers []*resources.Resource
routerName := strings.Replace(os.clusterName, ".", "-", -1)
opt := networks.ListOpts{
Name: os.clusterName,
}
networks, err := os.osCloud.ListNetworks(opt)
projectNetworks, err := os.osCloud.ListNetworks(networks.ListOpts{})
if err != nil {
return resourceTrackers, err
}
for _, network := range networks {
filteredNetwork := []networks.Network{}
for _, net := range projectNetworks {
if net.Name == os.clusterName || fi.ArrayContains(net.Tags, os.clusterName) {
filteredNetwork = append(filteredNetwork, net)
}
}
for _, network := range filteredNetwork {
preExistingNet := true
if os.clusterName == network.Name {
preExistingNet = false
}
optRouter := osrouter.ListOpts{
Name: strings.Replace(os.clusterName, ".", "-", -1),
Name: routerName,
}
routers, err := os.osCloud.ListRouters(optRouter)
if err != nil {
@ -73,29 +86,47 @@ func (os *clusterDiscoveryOS) ListNetwork() ([]*resources.Resource, error) {
}
resourceTrackers = append(resourceTrackers, resourceTracker)
}
optSubnet := subnets.ListOpts{
NetworkID: network.ID,
}
subnets, err := os.osCloud.ListSubnets(optSubnet)
networkSubnets, err := os.osCloud.ListSubnets(optSubnet)
if err != nil {
return resourceTrackers, err
}
for _, subnet := range subnets {
// router interfaces
for _, router := range routers {
resourceTracker := &resources.Resource{
Name: router.ID,
ID: subnet.ID,
Type: typeRouterIF,
Deleter: func(cloud fi.Cloud, r *resources.Resource) error {
opts := osrouter.RemoveInterfaceOpts{
SubnetID: r.ID,
}
return cloud.(openstack.OpenstackCloud).DeleteRouterInterface(r.Name, opts)
},
filteredSubnets := []subnets.Subnet{}
if preExistingNet {
// if we have preExistingNet, the subnet must have cluster tag
for _, sub := range networkSubnets {
if fi.ArrayContains(sub.Tags, os.clusterName) {
filteredSubnets = append(filteredSubnets, sub)
}
}
} else {
filteredSubnets = networkSubnets
}
for _, subnet := range filteredSubnets {
// router interfaces
preExistingSubnet := false
if !strings.HasSuffix(subnet.Name, os.clusterName) {
preExistingSubnet = true
}
if !preExistingSubnet {
for _, router := range routers {
resourceTracker := &resources.Resource{
Name: router.ID,
ID: subnet.ID,
Type: typeRouterIF,
Deleter: func(cloud fi.Cloud, r *resources.Resource) error {
opts := osrouter.RemoveInterfaceOpts{
SubnetID: r.ID,
}
return cloud.(openstack.OpenstackCloud).DeleteRouterInterface(r.Name, opts)
},
}
resourceTrackers = append(resourceTrackers, resourceTracker)
}
resourceTrackers = append(resourceTrackers, resourceTracker)
}
//associated load balancers
lbTrackers, err := os.DeleteSubnetLBs(subnet)
@ -104,16 +135,27 @@ func (os *clusterDiscoveryOS) ListNetwork() ([]*resources.Resource, error) {
}
resourceTrackers = append(resourceTrackers, lbTrackers...)
resourceTracker := &resources.Resource{
Name: subnet.Name,
ID: subnet.ID,
Type: typeSubnet,
Deleter: func(cloud fi.Cloud, r *resources.Resource) error {
return cloud.(openstack.OpenstackCloud).DeleteSubnet(r.ID)
},
if !preExistingSubnet {
resourceTracker := &resources.Resource{
Name: subnet.Name,
ID: subnet.ID,
Type: typeSubnet,
Deleter: func(cloud fi.Cloud, r *resources.Resource) error {
return cloud.(openstack.OpenstackCloud).DeleteSubnet(r.ID)
},
}
resourceTrackers = append(resourceTrackers, resourceTracker)
} else {
resourceTracker := &resources.Resource{
Name: os.clusterName,
ID: subnet.ID,
Type: typeSubnetTag,
Deleter: func(cloud fi.Cloud, r *resources.Resource) error {
return cloud.(openstack.OpenstackCloud).DeleteTag(openstack.ResourceTypeSubnet, r.ID, r.Name)
},
}
resourceTrackers = append(resourceTrackers, resourceTracker)
}
resourceTrackers = append(resourceTrackers, resourceTracker)
}
// Ports
@ -123,15 +165,27 @@ func (os *clusterDiscoveryOS) ListNetwork() ([]*resources.Resource, error) {
}
resourceTrackers = append(resourceTrackers, portTrackers...)
resourceTracker := &resources.Resource{
Name: network.Name,
ID: network.ID,
Type: typeNetwork,
Deleter: func(cloud fi.Cloud, r *resources.Resource) error {
return cloud.(openstack.OpenstackCloud).DeleteNetwork(r.ID)
},
if !preExistingNet {
resourceTracker := &resources.Resource{
Name: network.Name,
ID: network.ID,
Type: typeNetwork,
Deleter: func(cloud fi.Cloud, r *resources.Resource) error {
return cloud.(openstack.OpenstackCloud).DeleteNetwork(r.ID)
},
}
resourceTrackers = append(resourceTrackers, resourceTracker)
} else {
resourceTracker := &resources.Resource{
Name: os.clusterName,
ID: network.ID,
Type: typeNetworkTag,
Deleter: func(cloud fi.Cloud, r *resources.Resource) error {
return cloud.(openstack.OpenstackCloud).DeleteTag(openstack.ResourceTypeNetwork, r.ID, r.Name)
},
}
resourceTrackers = append(resourceTrackers, resourceTracker)
}
resourceTrackers = append(resourceTrackers, resourceTracker)
}
return resourceTrackers, nil
}

21
pkg/resources/openstack/ports.go
View File

@ -31,7 +31,7 @@ const (
func (os *clusterDiscoveryOS) ListPorts(network networks.Network) ([]*resources.Resource, error) {
var resourceTrackers []*resources.Resource
ports, err := os.osCloud.ListPorts(ports.ListOpts{
projectPorts, err := os.osCloud.ListPorts(ports.ListOpts{
TenantID: network.ProjectID,
NetworkID: network.ID,
})
@ -39,7 +39,24 @@ func (os *clusterDiscoveryOS) ListPorts(network networks.Network) ([]*resources.
return nil, err
}
for _, port := range ports {
preExistingNet := true
if os.clusterName == network.Name {
preExistingNet = false
}
filteredPorts := []ports.Port{}
if preExistingNet {
// if we have preExistingNet, the port must have cluster tag
for _, singlePort := range projectPorts {
if fi.ArrayContains(singlePort.Tags, os.clusterName) {
filteredPorts = append(filteredPorts, singlePort)
}
}
} else {
filteredPorts = projectPorts
}
for _, port := range filteredPorts {
resourceTracker := &resources.Resource{
Name: port.Name,
ID: port.ID,

7
protokube/pkg/gossip/openstack/seeds.go
View File

@ -50,7 +50,12 @@ func (p *SeedProvider) GetSeeds() ([]string, error) {
for _, server := range s {
if clusterName, ok := server.Metadata[openstack.TagClusterName]; ok {
var err error
addr, err := openstack.GetServerFixedIP(&server, clusterName)
// find kopsNetwork from metadata, fallback to clustername
ifName := clusterName
if val, ok := server.Metadata[openstack.TagKopsNetwork]; ok {
ifName = val
}
addr, err := openstack.GetServerFixedIP(&server, ifName)
if err != nil {
klog.Warningf("Failed to list seeds: %v", err)
continue

7
protokube/pkg/protokube/openstack_volume.go
View File

@ -176,7 +176,12 @@ func (a *OpenstackVolumes) discoverTags() error {
if err != nil {
return fmt.Errorf("error getting instance from ID: %v", err)
}
ip, err := openstack.GetServerFixedIP(server, a.clusterName)
// find kopsNetwork from metadata, fallback to clustername
ifName := a.clusterName
if val, ok := server.Metadata[openstack.TagKopsNetwork]; ok {
ifName = val
}
ip, err := openstack.GetServerFixedIP(server, ifName)
if err != nil {
return fmt.Errorf("error querying InternalIP from name: %v", err)
}

1
upup/pkg/fi/cloudup/openstack/BUILD.bazel
View File

@ -50,6 +50,7 @@ go_library(
"//vendor/github.com/gophercloud/gophercloud/openstack/loadbalancer/v2/loadbalancers:go_default_library",
"//vendor/github.com/gophercloud/gophercloud/openstack/loadbalancer/v2/monitors:go_default_library",
"//vendor/github.com/gophercloud/gophercloud/openstack/loadbalancer/v2/pools:go_default_library",
"//vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/attributestags:go_default_library",
"//vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/external:go_default_library",
"//vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/layer3/floatingips:go_default_library",
"//vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/layer3/routers:go_default_library",

59
upup/pkg/fi/cloudup/openstack/cloud.go
View File

@ -63,6 +63,10 @@ const (
TagNameRolePrefix = "k8s.io/role/"
TagClusterName = "KubernetesCluster"
TagRoleMaster = "master"
TagKopsNetwork = "KopsNetwork"
ResourceTypePort = "ports"
ResourceTypeNetwork = "networks"
ResourceTypeSubnet = "subnets"
)
// ErrNotFound is used to inform that the object is not found
@ -93,6 +97,7 @@ type OpenstackCloud interface {
LoadBalancerClient() *gophercloud.ServiceClient
DNSClient() *gophercloud.ServiceClient
UseOctavia() bool
UseZones([]string)
// Region returns the region which cloud will run on
Region() string
@ -147,6 +152,12 @@ type OpenstackCloud interface {
//GetNetwork will return the Neutron network which match the id
GetNetwork(networkID string) (*networks.Network, error)
//FindNetworkBySubnetID will return network
FindNetworkBySubnetID(subnetID string) (*networks.Network, error)
//GetSubnet returns subnet using subnet id
GetSubnet(subnetID string) (*subnets.Subnet, error)
//ListNetworks will return the Neutron networks which match the options
ListNetworks(opt networks.ListOptsBuilder) ([]networks.Network, error)
@ -165,6 +176,12 @@ type OpenstackCloud interface {
//DeleteNetwork will delete neutron network
DeleteNetwork(networkID string) error
//AppendTag appends tag to resource
AppendTag(resource string, id string, tag string) error
//DeleteTag removes tag from resource
DeleteTag(resource string, id string, tag string) error
//ListRouters will return the Neutron routers which match the options
ListRouters(opt routers.ListOpts) ([]routers.Router, error)
@ -295,13 +312,14 @@ type openstackCloud struct {
dnsClient *gophercloud.ServiceClient
lbClient *gophercloud.ServiceClient
glanceClient *gophercloud.ServiceClient
floatingEnabled bool
extNetworkName *string
extSubnetName *string
floatingSubnet *string
tags map[string]string
region string
useOctavia bool
zones []string
floatingEnabled bool
}
var _ fi.Cloud = &openstackCloud{}
@ -314,12 +332,6 @@ func NewOpenstackCloud(tags map[string]string, spec *kops.ClusterSpec) (Openstac
return nil, err
}
/*
provider, err := os.AuthenticatedClient(authOption)
if err != nil {
return nil, fmt.Errorf("error building openstack authenticated client: %v", err)
}*/
provider, err := os.NewClient(authOption.IdentityEndpoint)
if err != nil {
return nil, fmt.Errorf("error building openstack provider client: %v", err)
@ -466,6 +478,11 @@ func NewOpenstackCloud(tags map[string]string, spec *kops.ClusterSpec) (Openstac
return c, nil
}
// UseZones add unique zone names to openstackcloud
func (c *openstackCloud) UseZones(zones []string) {
c.zones = zones
}
func (c *openstackCloud) UseOctavia() bool {
return c.useOctavia
}
@ -506,8 +523,34 @@ func (c *openstackCloud) DNS() (dnsprovider.Interface, error) {
return provider, nil
}
// FindVPCInfo list subnets in network
func (c *openstackCloud) FindVPCInfo(id string) (*fi.VPCInfo, error) {
return nil, fmt.Errorf("openstackCloud::FindVPCInfo not implemented")
vpcInfo := &fi.VPCInfo{}
// Find subnets in the network
{
if len(c.zones) == 0 {
return nil, fmt.Errorf("Could not initialize zones")
}
klog.V(2).Infof("Calling ListSubnets for subnets in Network %q", id)
opt := subnets.ListOpts{
NetworkID: id,
}
subnets, err := c.ListSubnets(opt)
if err != nil {
return nil, fmt.Errorf("error listing subnets in network %q: %v", id, err)
}
for index, subnet := range subnets {
zone := c.zones[int(index)%len(c.zones)]
subnetInfo := &fi.SubnetInfo{
ID: subnet.ID,
CIDR: subnet.CIDR,
Zone: zone,
}
vpcInfo.Subnets = append(vpcInfo.Subnets, subnetInfo)
}
}
return vpcInfo, nil
}
// DeleteGroup in openstack will delete servergroup, instances and ports

4
upup/pkg/fi/cloudup/openstack/floatingip.go
View File

@ -142,7 +142,7 @@ func (c *openstackCloud) DeleteFloatingIP(id string) (err error) {
done, err := vfs.RetryWithBackoff(writeBackoff, func() (bool, error) {
err = l3floatingip.Delete(c.ComputeClient(), id).ExtractErr()
if err != nil {
if err != nil && !isNotFound(err) {
return false, fmt.Errorf("Failed to delete floating ip %s: %v", id, err)
}
return true, nil
@ -157,7 +157,7 @@ func (c *openstackCloud) DeleteL3FloatingIP(id string) (err error) {
done, err := vfs.RetryWithBackoff(writeBackoff, func() (bool, error) {
err = l3floatingip.Delete(c.NetworkingClient(), id).ExtractErr()
if err != nil {
if err != nil && !isNotFound(err) {
return false, fmt.Errorf("Failed to delete L3 floating ip %s: %v", id, err)
}
return true, nil

60
upup/pkg/fi/cloudup/openstack/network.go
View File

@ -19,6 +19,7 @@ package openstack
import (
"fmt"
"github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/attributestags"
"github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/external"
"github.com/gophercloud/gophercloud/openstack/networking/v2/networks"
"github.com/gophercloud/gophercloud/pagination"
@ -27,6 +28,65 @@ import (
"k8s.io/kops/util/pkg/vfs"
)
func (c *openstackCloud) AppendTag(resource string, id string, tag string) error {
done, err := vfs.RetryWithBackoff(readBackoff, func() (bool, error) {
err := attributestags.Add(c.neutronClient, resource, id, tag).ExtractErr()
if err != nil {
return false, fmt.Errorf("error appending tag %s: %v", tag, err)
}
return true, nil
})
if err != nil {
return err
} else if done {
return nil
} else {
return wait.ErrWaitTimeout
}
}
func (c *openstackCloud) DeleteTag(resource string, id string, tag string) error {
done, err := vfs.RetryWithBackoff(readBackoff, func() (bool, error) {
err := attributestags.Delete(c.neutronClient, resource, id, tag).ExtractErr()
if err != nil {
return false, fmt.Errorf("error deleting tag %s: %v", tag, err)
}
return true, nil
})
if err != nil {
return err
} else if done {
return nil
} else {
return wait.ErrWaitTimeout
}
}
func (c *openstackCloud) FindNetworkBySubnetID(subnetID string) (*networks.Network, error) {
var rslt *networks.Network
done, err := vfs.RetryWithBackoff(readBackoff, func() (bool, error) {
subnet, err := c.GetSubnet(subnetID)
if err != nil {
return false, fmt.Errorf("error retrieving subnet with id %s: %v", subnetID, err)
}
netID := subnet.NetworkID
net, err := c.GetNetwork(netID)
if err != nil {
return false, fmt.Errorf("error retrieving network with id %s: %v", netID, err)
}
rslt = net
return true, nil
})
if err != nil {
return nil, err
} else if done {
return rslt, nil
} else {
return nil, wait.ErrWaitTimeout
}
}
func (c *openstackCloud) GetNetwork(id string) (*networks.Network, error) {
var network *networks.Network
done, err := vfs.RetryWithBackoff(readBackoff, func() (bool, error) {

19
upup/pkg/fi/cloudup/openstack/subnet.go
View File

@ -50,6 +50,25 @@ func (c *openstackCloud) ListSubnets(opt subnets.ListOptsBuilder) ([]subnets.Sub
}
}
func (c *openstackCloud) GetSubnet(subnetID string) (*subnets.Subnet, error) {
var subnet *subnets.Subnet
done, err := vfs.RetryWithBackoff(readBackoff, func() (bool, error) {
sub, err := subnets.Get(c.neutronClient, subnetID).Extract()
if err != nil {
return false, fmt.Errorf("error retrieving subnet: %v", err)
}
subnet = sub
return true, nil
})
if err != nil {
return nil, err
} else if done {
return subnet, nil
} else {
return nil, wait.ErrWaitTimeout
}
}
func (c *openstackCloud) CreateSubnet(opt subnets.CreateOptsBuilder) (*subnets.Subnet, error) {
var s *subnets.Subnet

30
upup/pkg/fi/cloudup/openstacktasks/network.go
View File

@ -30,6 +30,7 @@ type Network struct {
ID *string
Name *string
Lifecycle *fi.Lifecycle
Tag *string
}
var _ fi.CompareWithID = &Network{}
@ -38,11 +39,17 @@ func (n *Network) CompareWithID() *string {
return n.ID
}
func NewNetworkTaskFromCloud(cloud openstack.OpenstackCloud, lifecycle *fi.Lifecycle, network *networks.Network) (*Network, error) {
func NewNetworkTaskFromCloud(cloud openstack.OpenstackCloud, lifecycle *fi.Lifecycle, network *networks.Network, networkName *string) (*Network, error) {
tag := ""
if networkName != nil && fi.ArrayContains(network.Tags, fi.StringValue(networkName)) {
tag = fi.StringValue(networkName)
}
task := &Network{
ID: fi.String(network.ID),
Name: fi.String(network.Name),
Lifecycle: lifecycle,
Tag: fi.String(tag),
}
return task, nil
}
@ -67,7 +74,7 @@ func (n *Network) Find(context *fi.Context) (*Network, error) {
return nil, fmt.Errorf("found multiple networks with name: %s", fi.StringValue(n.Name))
}
v := ns[0]
actual, err := NewNetworkTaskFromCloud(cloud, n.Lifecycle, &v)
actual, err := NewNetworkTaskFromCloud(cloud, n.Lifecycle, &v, n.Tag)
if err != nil {
return nil, fmt.Errorf("Failed to create new Network object: %v", err)
}
@ -96,7 +103,10 @@ func (_ *Network) CheckChanges(a, e, changes *Network) error {
}
func (_ *Network) ShouldCreate(a, e, changes *Network) (bool, error) {
return a == nil, nil
if a == nil || changes.Tag != nil {
return true, nil
}
return false, nil
}
func (_ *Network) RenderOpenstack(t *openstack.OpenstackAPITarget, a, e, changes *Network) error {
@ -113,11 +123,21 @@ func (_ *Network) RenderOpenstack(t *openstack.OpenstackAPITarget, a, e, changes
return fmt.Errorf("Error creating network: %v", err)
}
err = t.Cloud.AppendTag(openstack.ResourceTypeNetwork, v.ID, fi.StringValue(e.Tag))
if err != nil {
return fmt.Errorf("Error appending tag to network: %v", err)
}
e.ID = fi.String(v.ID)
klog.V(2).Infof("Creating a new Openstack network, id=%s", v.ID)
return nil
} else {
err := t.Cloud.AppendTag(openstack.ResourceTypeNetwork, fi.StringValue(a.ID), fi.StringValue(changes.Tag))
if err != nil {
return fmt.Errorf("Error appending tag to network: %v", err)
}
}
klog.V(2).Infof("Openstack task Network::RenderOpenstack did nothing")
e.ID = a.ID
klog.V(2).Infof("Using an existing Openstack network, id=%s", fi.StringValue(e.ID))
return nil
}

19
upup/pkg/fi/cloudup/openstacktasks/port.go
View File

@ -35,6 +35,7 @@ type Port struct {
SecurityGroups []*SecurityGroup
AdditionalSecurityGroups []string
Lifecycle *fi.Lifecycle
Tag *string
}
// GetDependencies returns the dependencies of the Port task
@ -95,6 +96,11 @@ func NewPortTaskFromCloud(cloud openstack.OpenstackCloud, lifecycle *fi.Lifecycl
}
}
tag := ""
if find != nil && fi.ArrayContains(port.Tags, fi.StringValue(find.Tag)) {
tag = fi.StringValue(find.Tag)
}
actual := &Port{
ID: fi.String(port.ID),
Name: fi.String(port.Name),
@ -102,6 +108,7 @@ func NewPortTaskFromCloud(cloud openstack.OpenstackCloud, lifecycle *fi.Lifecycl
SecurityGroups: sgs,
Subnets: subnets,
Lifecycle: lifecycle,
Tag: fi.String(tag),
}
if find != nil {
find.ID = actual.ID
@ -124,7 +131,6 @@ func (s *Port) Find(context *fi.Context) (*Port, error) {
} else if len(rs) != 1 {
return nil, fmt.Errorf("found multiple ports with name: %s", fi.StringValue(s.Name))
}
return NewPortTaskFromCloud(cloud, s.Lifecycle, &rs[0], s)
}
@ -165,9 +171,20 @@ func (*Port) RenderOpenstack(t *openstack.OpenstackAPITarget, a, e, changes *Por
return fmt.Errorf("Error creating port: %v", err)
}
if e.Tag != nil {
err = t.Cloud.AppendTag(openstack.ResourceTypePort, v.ID, fi.StringValue(e.Tag))
if err != nil {
return fmt.Errorf("Error appending tag to port: %v", err)
}
}
e.ID = fi.String(v.ID)
klog.V(2).Infof("Creating a new Openstack port, id=%s", v.ID)
return nil
} else if changes != nil && changes.Tag != nil {
err := t.Cloud.AppendTag(openstack.ResourceTypePort, fi.StringValue(a.ID), fi.StringValue(changes.Tag))
if err != nil {
return fmt.Errorf("Error appending tag to port: %v", err)
}
}
e.ID = a.ID
klog.V(2).Infof("Using an existing Openstack port, id=%s", fi.StringValue(e.ID))

5
upup/pkg/fi/cloudup/openstacktasks/port_test.go
View File

@ -82,6 +82,7 @@ func Test_NewPortTaskFromCloud(t *testing.T) {
Network: &Network{ID: fi.String("")},
SecurityGroups: []*SecurityGroup{},
Subnets: []*Subnet{},
Tag: fi.String(""),
Lifecycle: &syncLifecycle,
},
expectedError: nil,
@ -99,6 +100,7 @@ func Test_NewPortTaskFromCloud(t *testing.T) {
Network: &Network{ID: fi.String("")},
SecurityGroups: []*SecurityGroup{},
Subnets: []*Subnet{},
Tag: fi.String(""),
Lifecycle: &syncLifecycle,
},
expectedError: nil,
@ -135,6 +137,7 @@ func Test_NewPortTaskFromCloud(t *testing.T) {
{ID: fi.String("subnet-b"), Lifecycle: &syncLifecycle},
},
Lifecycle: &syncLifecycle,
Tag: fi.String(""),
},
expectedError: nil,
},
@ -196,6 +199,7 @@ func Test_NewPortTaskFromCloud(t *testing.T) {
{ID: fi.String("subnet-b"), Lifecycle: &syncLifecycle},
},
Lifecycle: &syncLifecycle,
Tag: fi.String(""),
},
expectedError: nil,
},
@ -278,6 +282,7 @@ func Test_Port_Find(t *testing.T) {
{ID: fi.String("subnet-a"), Lifecycle: &syncLifecycle},
{ID: fi.String("subnet-b"), Lifecycle: &syncLifecycle},
},
Tag: fi.String(""),
Lifecycle: &syncLifecycle,
},
expectedError: nil,

20
upup/pkg/fi/cloudup/openstacktasks/subnet.go
View File

@ -33,6 +33,7 @@ type Subnet struct {
Network *Network
CIDR *string
DNSServers []*string
Tag *string
Lifecycle *fi.Lifecycle
}
@ -58,15 +59,21 @@ func NewSubnetTaskFromCloud(cloud openstack.OpenstackCloud, lifecycle *fi.Lifecy
if err != nil {
return nil, fmt.Errorf("NewSubnetTaskFromCloud: Failed to get network with ID %s: %v", subnet.NetworkID, err)
}
networkTask, err := NewNetworkTaskFromCloud(cloud, lifecycle, network)
networkTask, err := NewNetworkTaskFromCloud(cloud, lifecycle, network, find.Tag)
if err != nil {
return nil, fmt.Errorf("error creating network task from cloud: %v", err)
}
nameservers := make([]*string, len(subnet.DNSNameservers))
for i, ns := range subnet.DNSNameservers {
nameservers[i] = fi.String(ns)
}
tag := ""
if find != nil && fi.ArrayContains(subnet.Tags, fi.StringValue(find.Tag)) {
tag = fi.StringValue(find.Tag)
}
actual := &Subnet{
ID: fi.String(subnet.ID),
Name: fi.String(subnet.Name),
@ -74,6 +81,7 @@ func NewSubnetTaskFromCloud(cloud openstack.OpenstackCloud, lifecycle *fi.Lifecy
CIDR: fi.String(subnet.CIDR),
Lifecycle: lifecycle,
DNSServers: nameservers,
Tag: fi.String(tag),
}
if find != nil {
find.ID = actual.ID
@ -159,9 +167,19 @@ func (_ *Subnet) RenderOpenstack(t *openstack.OpenstackAPITarget, a, e, changes
return fmt.Errorf("Error creating subnet: %v", err)
}
err = t.Cloud.AppendTag(openstack.ResourceTypeSubnet, v.ID, fi.StringValue(e.Tag))
if err != nil {
return fmt.Errorf("Error appending tag to subnet: %v", err)
}
e.ID = fi.String(v.ID)
klog.V(2).Infof("Creating a new Openstack subnet, id=%s", v.ID)
return nil
} else {
err := t.Cloud.AppendTag(openstack.ResourceTypeSubnet, fi.StringValue(a.ID), fi.StringValue(changes.Tag))
if err != nil {
return fmt.Errorf("Error appending tag to subnet: %v", err)
}
}
e.ID = a.ID
klog.V(2).Infof("Using an existing Openstack subnet, id=%s", fi.StringValue(e.ID))

7
upup/pkg/fi/cloudup/utils.go
View File

@ -139,6 +139,13 @@ func BuildCloud(cluster *kops.Cluster) (fi.Cloud, error) {
if err != nil {
return nil, err
}
var zoneNames []string
for _, subnet := range cluster.Spec.Subnets {
if !fi.ArrayContains(zoneNames, subnet.Zone) {
zoneNames = append(zoneNames, subnet.Zone)
}
}
osc.UseZones(zoneNames)
cloud = osc
}

10
upup/pkg/fi/values.go
View File

@ -125,6 +125,16 @@ func Uint64Value(v *uint64) uint64 {
return *v
}
// ArrayContains is checking does array contain single word
func ArrayContains(array []string, word string) bool {
for _, item := range array {
if item == word {
return true
}
}
return false
}
func DebugPrint(o interface{}) string {
if o == nil {
return "<nil>"

15
vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/attributestags/BUILD.bazel generated vendored Normal file
View File

@ -0,0 +1,15 @@
load("@io_bazel_rules_go//go:def.bzl", "go_library")
go_library(
name = "go_default_library",
srcs = [
"doc.go",
"requests.go",
"results.go",
"urls.go",
],
importmap = "k8s.io/kops/vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/attributestags",
importpath = "github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/attributestags",
visibility = ["//visibility:public"],
deps = ["//vendor/github.com/gophercloud/gophercloud:go_default_library"],
)

37
vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/attributestags/doc.go generated vendored Normal file
View File

@ -0,0 +1,37 @@
/*
Package attributestags manages Tags on Resources created by the OpenStack Neutron Service.
This enables tagging via a standard interface for resources types which support it.
See https://developer.openstack.org/api-ref/network/v2/#standard-attributes-tag-extension for more information on the underlying API.
Example to ReplaceAll Resource Tags
network, err := networks.Create(conn, createOpts).Extract()
tagReplaceAllOpts := attributestags.ReplaceAllOpts{
Tags: []string{"abc", "123"},
}
attributestags.ReplaceAll(conn, "networks", network.ID, tagReplaceAllOpts)
Example to List all Resource Tags
tags, err = attributestags.List(conn, "networks", network.ID).Extract()
Example to Delete all Resource Tags
err = attributestags.DeleteAll(conn, "networks", network.ID).ExtractErr()
Example to Add a tag to a Resource
err = attributestags.Add(client, "networks", network.ID, "atag").ExtractErr()
Example to Delete a tag from a Resource
err = attributestags.Delete(client, "networks", network.ID, "atag").ExtractErr()
Example to confirm if a tag exists on a resource
exists, _ := attributestags.Confirm(client, "networks", network.ID, "atag").Extract()
*/
package attributestags

81
vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/attributestags/requests.go generated vendored Normal file
View File

@ -0,0 +1,81 @@
package attributestags
import (
"github.com/gophercloud/gophercloud"
)
// ReplaceAllOptsBuilder allows extensions to add additional parameters to
// the ReplaceAll request.
type ReplaceAllOptsBuilder interface {
ToAttributeTagsReplaceAllMap() (map[string]interface{}, error)
}
// ReplaceAllOpts provides options used to create Tags on a Resource
type ReplaceAllOpts struct {
Tags []string `json:"tags" required:"true"`
}
// ToAttributeTagsReplaceAllMap formats a ReplaceAllOpts into the body of the
// replace request
func (opts ReplaceAllOpts) ToAttributeTagsReplaceAllMap() (map[string]interface{}, error) {
return gophercloud.BuildRequestBody(opts, "")
}
// ReplaceAll updates all tags on a resource, replacing any existing tags
func ReplaceAll(client *gophercloud.ServiceClient, resourceType string, resourceID string, opts ReplaceAllOptsBuilder) (r ReplaceAllResult) {
b, err := opts.ToAttributeTagsReplaceAllMap()
url := replaceURL(client, resourceType, resourceID)
if err != nil {
r.Err = err
return
}
_, r.Err = client.Put(url, &b, &r.Body, &gophercloud.RequestOpts{
OkCodes: []int{200},
})
return
}
// List all tags on a resource
func List(client *gophercloud.ServiceClient, resourceType string, resourceID string) (r ListResult) {
url := listURL(client, resourceType, resourceID)
_, r.Err = client.Get(url, &r.Body, &gophercloud.RequestOpts{
OkCodes: []int{200},
})
return
}
// DeleteAll deletes all tags on a resource
func DeleteAll(client *gophercloud.ServiceClient, resourceType string, resourceID string) (r DeleteResult) {
url := deleteAllURL(client, resourceType, resourceID)
_, r.Err = client.Delete(url, &gophercloud.RequestOpts{
OkCodes: []int{204},
})
return
}
// Add a tag on a resource
func Add(client *gophercloud.ServiceClient, resourceType string, resourceID string, tag string) (r AddResult) {
url := addURL(client, resourceType, resourceID, tag)
_, r.Err = client.Put(url, nil, nil, &gophercloud.RequestOpts{
OkCodes: []int{201},
})
return
}
// Delete a tag on a resource
func Delete(client *gophercloud.ServiceClient, resourceType string, resourceID string, tag string) (r DeleteResult) {
url := deleteURL(client, resourceType, resourceID, tag)
_, r.Err = client.Delete(url, &gophercloud.RequestOpts{
OkCodes: []int{204},
})
return
}
// Confirm if a tag exists on a resource
func Confirm(client *gophercloud.ServiceClient, resourceType string, resourceID string, tag string) (r ConfirmResult) {
url := confirmURL(client, resourceType, resourceID, tag)
_, r.Err = client.Get(url, nil, &gophercloud.RequestOpts{
OkCodes: []int{204},
})
return
}

57
vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/attributestags/results.go generated vendored Normal file
View File

@ -0,0 +1,57 @@
package attributestags
import (
"github.com/gophercloud/gophercloud"
)
type tagResult struct {
gophercloud.Result
}
// Extract interprets tagResult to return the list of tags
func (r tagResult) Extract() ([]string, error) {
var s struct {
Tags []string `json:"tags"`
}
err := r.ExtractInto(&s)
return s.Tags, err
}
// ReplaceAllResult represents the result of a replace operation.
// Call its Extract method to interpret it as a slice of strings.
type ReplaceAllResult struct {
tagResult
}
type ListResult struct {
tagResult
}
// DeleteResult is the result from a Delete/DeleteAll operation.
// Call its ExtractErr method to determine if the call succeeded or failed.
type DeleteResult struct {
gophercloud.ErrResult
}
// AddResult is the result from an Add operation.
// Call its ExtractErr method to determine if the call succeeded or failed.
type AddResult struct {
gophercloud.ErrResult
}
// ConfirmResult is the result from an Confirm operation.
type ConfirmResult struct {
gophercloud.Result
}
func (r ConfirmResult) Extract() (bool, error) {
exists := r.Err == nil
if r.Err != nil {
if _, ok := r.Err.(gophercloud.ErrDefault404); ok {
r.Err = nil
}
}
return exists, r.Err
}

31
vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/attributestags/urls.go generated vendored Normal file
View File

@ -0,0 +1,31 @@
package attributestags
import "github.com/gophercloud/gophercloud"
const (
tagsPath = "tags"
)
func replaceURL(c *gophercloud.ServiceClient, r_type string, id string) string {
return c.ServiceURL(r_type, id, tagsPath)
}
func listURL(c *gophercloud.ServiceClient, r_type string, id string) string {
return c.ServiceURL(r_type, id, tagsPath)
}
func deleteAllURL(c *gophercloud.ServiceClient, r_type string, id string) string {
return c.ServiceURL(r_type, id, tagsPath)
}
func addURL(c *gophercloud.ServiceClient, r_type string, id string, tag string) string {
return c.ServiceURL(r_type, id, tagsPath, tag)
}
func deleteURL(c *gophercloud.ServiceClient, r_type string, id string, tag string) string {
return c.ServiceURL(r_type, id, tagsPath, tag)
}
func confirmURL(c *gophercloud.ServiceClient, r_type string, id string, tag string) string {
return c.ServiceURL(r_type, id, tagsPath, tag)
}

1
vendor/modules.txt vendored
View File

@ -261,6 +261,7 @@ github.com/gophercloud/gophercloud/openstack/compute/v2/flavors
github.com/gophercloud/gophercloud/openstack/imageservice/v2/images
github.com/gophercloud/gophercloud/openstack/loadbalancer/v2/listeners
github.com/gophercloud/gophercloud/openstack/loadbalancer/v2/pools
github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/attributestags
github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/external
github.com/gophercloud/gophercloud/openstack/compute/v2/extensions/bootfromvolume
github.com/gophercloud/gophercloud/openstack/compute/v2/extensions/schedulerhints