Merge pull request #7644 from zetaab/feature/floatingtype

Use without external router (OpenStack)
This commit is contained in:
Kubernetes Prow Robot 2019-09-21 06:45:24 -07:00 committed by GitHub
commit 5fa9425802
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 470 additions and 38 deletions

View File

@ -63,7 +63,7 @@ func (b *NetworkModelBuilder) Build(c *fi.ModelBuilderContext) error {
DNSServers: make([]*string, 0),
Lifecycle: b.Lifecycle,
}
if b.Cluster.Spec.CloudConfig.Openstack.Router.DNSServers != nil {
if b.Cluster.Spec.CloudConfig.Openstack.Router != nil && b.Cluster.Spec.CloudConfig.Openstack.Router.DNSServers != nil {
dnsSplitted := strings.Split(fi.StringValue(b.Cluster.Spec.CloudConfig.Openstack.Router.DNSServers), ",")
dnsNameSrv := make([]*string, len(dnsSplitted))
for i, ns := range dnsSplitted {

View File

@ -136,33 +136,41 @@ func (b *ServerGroupModelBuilder) buildInstances(c *fi.ModelBuilderContext, sg *
}
c.AddTask(instanceTask)
// Associate a floating IP to the master and bastion always, associate it to a node if bastion is not used
switch ig.Spec.Role {
case kops.InstanceGroupRoleBastion:
t := &openstacktasks.FloatingIP{
Name: fi.String(fmt.Sprintf("%s-%s", "fip", *instanceTask.Name)),
Server: instanceTask,
Lifecycle: b.Lifecycle,
}
c.AddTask(t)
case kops.InstanceGroupRoleMaster:
if b.Cluster.Spec.CloudConfig.Openstack.Loadbalancer == nil {
// Associate a floating IP to the master and bastion always if we have external network in router
// associate it to a node if bastion is not used
if b.Cluster.Spec.CloudConfig.Openstack != nil && b.Cluster.Spec.CloudConfig.Openstack.Router != nil {
switch ig.Spec.Role {
case kops.InstanceGroupRoleBastion:
t := &openstacktasks.FloatingIP{
Name: fi.String(fmt.Sprintf("%s-%s", "fip", *instanceTask.Name)),
Server: instanceTask,
Lifecycle: b.Lifecycle,
}
c.AddTask(t)
b.associateFIPToKeypair(c, t)
}
default:
if !b.UsesSSHBastion() {
t := &openstacktasks.FloatingIP{
Name: fi.String(fmt.Sprintf("%s-%s", "fip", *instanceTask.Name)),
Server: instanceTask,
Lifecycle: b.Lifecycle,
case kops.InstanceGroupRoleMaster:
if b.Cluster.Spec.CloudConfig.Openstack.Loadbalancer == nil {
t := &openstacktasks.FloatingIP{
Name: fi.String(fmt.Sprintf("%s-%s", "fip", *instanceTask.Name)),
Server: instanceTask,
Lifecycle: b.Lifecycle,
}
c.AddTask(t)
b.associateFIPToKeypair(c, t)
}
c.AddTask(t)
default:
if !b.UsesSSHBastion() {
t := &openstacktasks.FloatingIP{
Name: fi.String(fmt.Sprintf("%s-%s", "fip", *instanceTask.Name)),
Server: instanceTask,
Lifecycle: b.Lifecycle,
}
c.AddTask(t)
}
}
} else if b.Cluster.Spec.CloudConfig.Openstack != nil && b.Cluster.Spec.CloudConfig.Openstack.Router == nil {
// No external router, but we need to add master fixed ips to certificates
if ig.Spec.Role == kops.InstanceGroupRoleMaster {
b.associateFixedIPToKeypair(c, instanceTask)
}
}
}
@ -170,6 +178,19 @@ func (b *ServerGroupModelBuilder) buildInstances(c *fi.ModelBuilderContext, sg *
return nil
}
func (b *ServerGroupModelBuilder) associateFixedIPToKeypair(c *fi.ModelBuilderContext, fipTask *openstacktasks.Instance) error {
// Ensure the floating IP is included in the TLS certificate,
// if we're not going to use an alias for it
// TODO: I don't love this technique for finding the task by name & modifying it
masterKeypairTask, found := c.Tasks["Keypair/master"]
if !found {
return fmt.Errorf("keypair/master task not found")
}
masterKeypair := masterKeypairTask.(*fitasks.Keypair)
masterKeypair.AlternateNameTasks = append(masterKeypair.AlternateNameTasks, fipTask)
return nil
}
func (b *ServerGroupModelBuilder) associateFIPToKeypair(c *fi.ModelBuilderContext, fipTask *openstacktasks.FloatingIP) error {
// Ensure the floating IP is included in the TLS certificate,
// if we're not going to use an alias for it

View File

@ -48,7 +48,11 @@ func Test_ServerGroupModelBuilder(t *testing.T) {
Spec: kops.ClusterSpec{
MasterPublicName: "master-public-name",
CloudConfig: &kops.CloudConfiguration{
Openstack: &kops.OpenstackConfiguration{},
Openstack: &kops.OpenstackConfiguration{
Router: &kops.OpenstackRouter{
ExternalNetwork: fi.String("test"),
},
},
},
Subnets: []kops.ClusterSubnetSpec{
{
@ -199,7 +203,11 @@ func Test_ServerGroupModelBuilder(t *testing.T) {
Spec: kops.ClusterSpec{
MasterPublicName: "master-public-name",
CloudConfig: &kops.CloudConfiguration{
Openstack: &kops.OpenstackConfiguration{},
Openstack: &kops.OpenstackConfiguration{
Router: &kops.OpenstackRouter{
ExternalNetwork: fi.String("test"),
},
},
},
Subnets: []kops.ClusterSubnetSpec{
{
@ -412,7 +420,11 @@ func Test_ServerGroupModelBuilder(t *testing.T) {
Spec: kops.ClusterSpec{
MasterPublicName: "master-public-name",
CloudConfig: &kops.CloudConfiguration{
Openstack: &kops.OpenstackConfiguration{},
Openstack: &kops.OpenstackConfiguration{
Router: &kops.OpenstackRouter{
ExternalNetwork: fi.String("test"),
},
},
},
Subnets: []kops.ClusterSubnetSpec{
{
@ -815,6 +827,9 @@ func Test_ServerGroupModelBuilder(t *testing.T) {
CloudConfig: &kops.CloudConfiguration{
Openstack: &kops.OpenstackConfiguration{
Loadbalancer: &kops.OpenstackLoadbalancerConfig{},
Router: &kops.OpenstackRouter{
ExternalNetwork: fi.String("test"),
},
},
},
Subnets: []kops.ClusterSubnetSpec{
@ -1241,7 +1256,7 @@ func Test_ServerGroupModelBuilder(t *testing.T) {
},
},
{
desc: "multizone setup 3 masters 3 nodes without bastion auto zone distribution",
desc: "multizone setup 3 masters 3 nodes without external router",
cluster: &kops.Cluster{
ObjectMeta: metav1.ObjectMeta{
Name: "cluster",
@ -1258,6 +1273,375 @@ func Test_ServerGroupModelBuilder(t *testing.T) {
},
},
},
instanceGroups: []*kops.InstanceGroup{
{
ObjectMeta: metav1.ObjectMeta{
Name: "master-a",
},
Spec: kops.InstanceGroupSpec{
Role: kops.InstanceGroupRoleMaster,
Image: "image",
MinSize: i32(1),
MaxSize: i32(1),
MachineType: "blc.1-2",
Subnets: []string{"subnet-a"},
},
},
{
ObjectMeta: metav1.ObjectMeta{
Name: "node-a",
},
Spec: kops.InstanceGroupSpec{
Role: kops.InstanceGroupRoleNode,
Image: "image",
MinSize: i32(1),
MaxSize: i32(1),
MachineType: "blc.1-2",
Subnets: []string{"subnet-a"},
},
},
{
ObjectMeta: metav1.ObjectMeta{
Name: "master-b",
},
Spec: kops.InstanceGroupSpec{
Role: kops.InstanceGroupRoleMaster,
Image: "image",
MinSize: i32(1),
MaxSize: i32(1),
MachineType: "blc.1-2",
Subnets: []string{"subnet-b"},
},
},
{
ObjectMeta: metav1.ObjectMeta{
Name: "node-b",
},
Spec: kops.InstanceGroupSpec{
Role: kops.InstanceGroupRoleNode,
Image: "image",
MinSize: i32(1),
MaxSize: i32(1),
MachineType: "blc.1-2",
Subnets: []string{"subnet-b"},
},
},
{
ObjectMeta: metav1.ObjectMeta{
Name: "master-c",
},
Spec: kops.InstanceGroupSpec{
Role: kops.InstanceGroupRoleMaster,
Image: "image",
MinSize: i32(1),
MaxSize: i32(1),
MachineType: "blc.1-2",
Subnets: []string{"subnet-c"},
},
},
{
ObjectMeta: metav1.ObjectMeta{
Name: "node-c",
},
Spec: kops.InstanceGroupSpec{
Role: kops.InstanceGroupRoleNode,
Image: "image",
MinSize: i32(1),
MaxSize: i32(1),
MachineType: "blc.1-2",
Subnets: []string{"subnet-c"},
},
},
},
expectedTasksBuilder: func(cluster *kops.Cluster, instanceGroups []*kops.InstanceGroup) map[string]fi.Task {
clusterLifecycle := fi.LifecycleSync
masterAServerGroup := &openstacktasks.ServerGroup{
Name: s("cluster-master-a"),
ClusterName: s("cluster"),
IGName: s("master-a"),
Policies: []string{"anti-affinity"},
Lifecycle: &clusterLifecycle,
MaxSize: i32(1),
}
masterAPort := &openstacktasks.Port{
Name: s("port-master-a-1-cluster"),
Network: &openstacktasks.Network{Name: s("cluster")},
SecurityGroups: []*openstacktasks.SecurityGroup{
{Name: s("master-public-name")},
{Name: s("masters.cluster")},
},
Subnets: []*openstacktasks.Subnet{
{Name: s("subnet-a.cluster")},
},
Lifecycle: &clusterLifecycle,
}
masterAInstance := &openstacktasks.Instance{
Name: s("master-a-1-cluster"),
Region: s("region"),
Flavor: s("blc.1-2"),
Image: s("image"),
SSHKey: s("kubernetes.cluster-ba_d8_85_a0_5b_50_b0_01_e0_b2_b0_ae_5d_f6_7a_d1"),
ServerGroup: masterAServerGroup,
Tags: []string{"KubernetesCluster:cluster"},
Role: s("Master"),
Port: masterAPort,
UserData: s(mustUserdataForClusterInstance(cluster, instanceGroups[0])),
Metadata: map[string]string{
"KubernetesCluster": "cluster",
"k8s": "cluster",
"KopsInstanceGroup": "master-a",
"KopsRole": "Master",
"ig_generation": "0",
"cluster_generation": "0",
},
AvailabilityZone: s("subnet-a"),
}
masterBServerGroup := &openstacktasks.ServerGroup{
Name: s("cluster-master-b"),
ClusterName: s("cluster"),
IGName: s("master-b"),
Policies: []string{"anti-affinity"},
Lifecycle: &clusterLifecycle,
MaxSize: i32(1),
}
masterBPort := &openstacktasks.Port{
Name: s("port-master-b-1-cluster"),
Network: &openstacktasks.Network{Name: s("cluster")},
SecurityGroups: []*openstacktasks.SecurityGroup{
{Name: s("master-public-name")},
{Name: s("masters.cluster")},
},
Subnets: []*openstacktasks.Subnet{
{Name: s("subnet-b.cluster")},
},
Lifecycle: &clusterLifecycle,
}
masterBInstance := &openstacktasks.Instance{
Name: s("master-b-1-cluster"),
Region: s("region"),
Flavor: s("blc.1-2"),
Image: s("image"),
SSHKey: s("kubernetes.cluster-ba_d8_85_a0_5b_50_b0_01_e0_b2_b0_ae_5d_f6_7a_d1"),
ServerGroup: masterBServerGroup,
Tags: []string{"KubernetesCluster:cluster"},
Role: s("Master"),
Port: masterBPort,
UserData: s(mustUserdataForClusterInstance(cluster, instanceGroups[0])),
Metadata: map[string]string{
"KubernetesCluster": "cluster",
"k8s": "cluster",
"KopsInstanceGroup": "master-b",
"KopsRole": "Master",
"ig_generation": "0",
"cluster_generation": "0",
},
AvailabilityZone: s("subnet-b"),
}
masterCServerGroup := &openstacktasks.ServerGroup{
Name: s("cluster-master-c"),
ClusterName: s("cluster"),
IGName: s("master-c"),
Policies: []string{"anti-affinity"},
Lifecycle: &clusterLifecycle,
MaxSize: i32(1),
}
masterCPort := &openstacktasks.Port{
Name: s("port-master-c-1-cluster"),
Network: &openstacktasks.Network{Name: s("cluster")},
SecurityGroups: []*openstacktasks.SecurityGroup{
{Name: s("master-public-name")},
{Name: s("masters.cluster")},
},
Subnets: []*openstacktasks.Subnet{
{Name: s("subnet-c.cluster")},
},
Lifecycle: &clusterLifecycle,
}
masterCInstance := &openstacktasks.Instance{
Name: s("master-c-1-cluster"),
Region: s("region"),
Flavor: s("blc.1-2"),
Image: s("image"),
SSHKey: s("kubernetes.cluster-ba_d8_85_a0_5b_50_b0_01_e0_b2_b0_ae_5d_f6_7a_d1"),
ServerGroup: masterCServerGroup,
Tags: []string{"KubernetesCluster:cluster"},
Role: s("Master"),
Port: masterCPort,
UserData: s(mustUserdataForClusterInstance(cluster, instanceGroups[0])),
Metadata: map[string]string{
"KubernetesCluster": "cluster",
"k8s": "cluster",
"KopsInstanceGroup": "master-c",
"KopsRole": "Master",
"ig_generation": "0",
"cluster_generation": "0",
},
AvailabilityZone: s("subnet-c"),
}
nodeAServerGroup := &openstacktasks.ServerGroup{
Name: s("cluster-node-a"),
ClusterName: s("cluster"),
IGName: s("node-a"),
Policies: []string{"anti-affinity"},
Lifecycle: &clusterLifecycle,
MaxSize: i32(1),
}
nodeAPort := &openstacktasks.Port{
Name: s("port-node-a-1-cluster"),
Network: &openstacktasks.Network{Name: s("cluster")},
SecurityGroups: []*openstacktasks.SecurityGroup{
{Name: s("nodes.cluster")},
},
Subnets: []*openstacktasks.Subnet{
{Name: s("subnet-a.cluster")},
},
Lifecycle: &clusterLifecycle,
}
nodeAInstance := &openstacktasks.Instance{
Name: s("node-a-1-cluster"),
Region: s("region"),
Flavor: s("blc.1-2"),
Image: s("image"),
SSHKey: s("kubernetes.cluster-ba_d8_85_a0_5b_50_b0_01_e0_b2_b0_ae_5d_f6_7a_d1"),
ServerGroup: nodeAServerGroup,
Tags: []string{"KubernetesCluster:cluster"},
Role: s("Node"),
Port: nodeAPort,
UserData: s(mustUserdataForClusterInstance(cluster, instanceGroups[1])),
Metadata: map[string]string{
"KubernetesCluster": "cluster",
"k8s": "cluster",
"KopsInstanceGroup": "node-a",
"KopsRole": "Node",
"ig_generation": "0",
"cluster_generation": "0",
},
AvailabilityZone: s("subnet-a"),
}
nodeBServerGroup := &openstacktasks.ServerGroup{
Name: s("cluster-node-b"),
ClusterName: s("cluster"),
IGName: s("node-b"),
Policies: []string{"anti-affinity"},
Lifecycle: &clusterLifecycle,
MaxSize: i32(1),
}
nodeBPort := &openstacktasks.Port{
Name: s("port-node-b-1-cluster"),
Network: &openstacktasks.Network{Name: s("cluster")},
SecurityGroups: []*openstacktasks.SecurityGroup{
{Name: s("nodes.cluster")},
},
Subnets: []*openstacktasks.Subnet{
{Name: s("subnet-b.cluster")},
},
Lifecycle: &clusterLifecycle,
}
nodeBInstance := &openstacktasks.Instance{
Name: s("node-b-1-cluster"),
Region: s("region"),
Flavor: s("blc.1-2"),
Image: s("image"),
SSHKey: s("kubernetes.cluster-ba_d8_85_a0_5b_50_b0_01_e0_b2_b0_ae_5d_f6_7a_d1"),
ServerGroup: nodeBServerGroup,
Tags: []string{"KubernetesCluster:cluster"},
Role: s("Node"),
Port: nodeBPort,
UserData: s(mustUserdataForClusterInstance(cluster, instanceGroups[1])),
Metadata: map[string]string{
"KubernetesCluster": "cluster",
"k8s": "cluster",
"KopsInstanceGroup": "node-b",
"KopsRole": "Node",
"ig_generation": "0",
"cluster_generation": "0",
},
AvailabilityZone: s("subnet-b"),
}
nodeCServerGroup := &openstacktasks.ServerGroup{
Name: s("cluster-node-c"),
ClusterName: s("cluster"),
IGName: s("node-c"),
Policies: []string{"anti-affinity"},
Lifecycle: &clusterLifecycle,
MaxSize: i32(1),
}
nodeCPort := &openstacktasks.Port{
Name: s("port-node-c-1-cluster"),
Network: &openstacktasks.Network{Name: s("cluster")},
SecurityGroups: []*openstacktasks.SecurityGroup{
{Name: s("nodes.cluster")},
},
Subnets: []*openstacktasks.Subnet{
{Name: s("subnet-c.cluster")},
},
Lifecycle: &clusterLifecycle,
}
nodeCInstance := &openstacktasks.Instance{
Name: s("node-c-1-cluster"),
Region: s("region"),
Flavor: s("blc.1-2"),
Image: s("image"),
SSHKey: s("kubernetes.cluster-ba_d8_85_a0_5b_50_b0_01_e0_b2_b0_ae_5d_f6_7a_d1"),
ServerGroup: nodeCServerGroup,
Tags: []string{"KubernetesCluster:cluster"},
Role: s("Node"),
Port: nodeCPort,
UserData: s(mustUserdataForClusterInstance(cluster, instanceGroups[1])),
Metadata: map[string]string{
"KubernetesCluster": "cluster",
"k8s": "cluster",
"KopsInstanceGroup": "node-c",
"KopsRole": "Node",
"ig_generation": "0",
"cluster_generation": "0",
},
AvailabilityZone: s("subnet-c"),
}
return map[string]fi.Task{
"ServerGroup/cluster-master-a": masterAServerGroup,
"Instance/master-a-1-cluster": masterAInstance,
"Port/port-master-a-1-cluster": masterAPort,
"ServerGroup/cluster-master-b": masterBServerGroup,
"Instance/master-b-1-cluster": masterBInstance,
"Port/port-master-b-1-cluster": masterBPort,
"ServerGroup/cluster-master-c": masterCServerGroup,
"Instance/master-c-1-cluster": masterCInstance,
"Port/port-master-c-1-cluster": masterCPort,
"ServerGroup/cluster-node-a": nodeAServerGroup,
"Instance/node-a-1-cluster": nodeAInstance,
"Port/port-node-a-1-cluster": nodeAPort,
"ServerGroup/cluster-node-b": nodeBServerGroup,
"Instance/node-b-1-cluster": nodeBInstance,
"Port/port-node-b-1-cluster": nodeBPort,
"ServerGroup/cluster-node-c": nodeCServerGroup,
"Instance/node-c-1-cluster": nodeCInstance,
"Port/port-node-c-1-cluster": nodeCPort,
}
},
},
{
desc: "multizone setup 3 masters 3 nodes without bastion auto zone distribution",
cluster: &kops.Cluster{
ObjectMeta: metav1.ObjectMeta{
Name: "cluster",
},
Spec: kops.ClusterSpec{
MasterPublicName: "master-public-name",
CloudConfig: &kops.CloudConfiguration{
Openstack: &kops.OpenstackConfiguration{
Router: &kops.OpenstackRouter{
ExternalNetwork: fi.String("test"),
},
},
},
Subnets: []kops.ClusterSubnetSpec{
{
Region: "region",
},
},
},
},
instanceGroups: []*kops.InstanceGroup{
{
ObjectMeta: metav1.ObjectMeta{

View File

@ -287,17 +287,18 @@ type OpenstackCloud interface {
}
type openstackCloud struct {
cinderClient *gophercloud.ServiceClient
neutronClient *gophercloud.ServiceClient
novaClient *gophercloud.ServiceClient
dnsClient *gophercloud.ServiceClient
lbClient *gophercloud.ServiceClient
extNetworkName *string
extSubnetName *string
floatingSubnet *string
tags map[string]string
region string
useOctavia bool
cinderClient *gophercloud.ServiceClient
neutronClient *gophercloud.ServiceClient
novaClient *gophercloud.ServiceClient
dnsClient *gophercloud.ServiceClient
lbClient *gophercloud.ServiceClient
floatingEnabled bool
extNetworkName *string
extSubnetName *string
floatingSubnet *string
tags map[string]string
region string
useOctavia bool
}
var _ fi.Cloud = &openstackCloud{}
@ -392,11 +393,13 @@ func NewOpenstackCloud(tags map[string]string, spec *kops.ClusterSpec) (Openstac
}
octavia := false
floatingEnabled := false
if spec != nil &&
spec.CloudConfig != nil &&
spec.CloudConfig.Openstack != nil &&
spec.CloudConfig.Openstack.Router != nil {
floatingEnabled = true
c.extNetworkName = spec.CloudConfig.Openstack.Router.ExternalNetwork
if spec.CloudConfig.Openstack.Router.ExternalSubnet != nil {
@ -423,6 +426,7 @@ func NewOpenstackCloud(tags map[string]string, spec *kops.ClusterSpec) (Openstac
}
}
}
c.floatingEnabled = floatingEnabled
c.useOctavia = octavia
var lbClient *gophercloud.ServiceClient
if spec != nil && spec.CloudConfig != nil && spec.CloudConfig.Openstack != nil {
@ -598,7 +602,6 @@ func (c *openstackCloud) GetApiIngressStatus(cluster *kops.Cluster) ([]kops.ApiI
if err != nil {
return ingresses, fmt.Errorf("GetApiIngressStatus: Failed to list master nodes: %v", err)
}
for _, instance := range instances {
val, ok := instance.Metadata["k8s"]
val2, ok2 := instance.Metadata["KopsRole"]

View File

@ -78,7 +78,11 @@ func (c *openstackCloud) ListServerFloatingIPs(instanceID string) ([]*string, er
for _, addrList := range addresses {
for _, props := range addrList {
if props.IPType == "floating" {
if c.floatingEnabled {
if props.IPType == "floating" {
result = append(result, fi.String(props.Addr))
}
} else {
result = append(result, fi.String(props.Addr))
}
}

View File

@ -46,6 +46,8 @@ type Instance struct {
Lifecycle *fi.Lifecycle
}
var _ fi.HasAddress = &Instance{}
// GetDependencies returns the dependencies of the Instance task
func (e *Instance) GetDependencies(tasks map[string]fi.Task) []fi.Task {
var deps []fi.Task
@ -70,6 +72,24 @@ func (e *Instance) CompareWithID() *string {
return e.ID
}
func (e *Instance) FindIPAddress(context *fi.Context) (*string, error) {
cloud := context.Cloud.(openstack.OpenstackCloud)
if e.Port == nil {
return nil, nil
}
ports, err := cloud.GetPort(fi.StringValue(e.Port.ID))
if err != nil {
return nil, err
}
for _, port := range ports.FixedIPs {
return fi.String(port.IPAddress), nil
}
return nil, nil
}
func (e *Instance) Find(c *fi.Context) (*Instance, error) {
if e == nil || e.Name == nil {
return nil, nil