kubernetes
/
kops
mirror of https://github.com/kubernetes/kops.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #10553 from justinsb/fix_openstack_race 

Openstack: Prevent data race in servergroup member list
This commit is contained in:
Kubernetes Prow Robot 2021-01-10 09:45:05 -08:00 committed by GitHub
parent 08cac106b6 2f5ba0fbac
commit 75623366e6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 41 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
pkg/model/openstackmodel/tests/servergroup/adds-additional-security-groups.yaml
View File

@ -58,7 +58,6 @@ ServerGroup:
IGName: node
Lifecycle: Sync
MaxSize: 1
Members: null
Name: cluster-node
Policies:
- anti-affinity
@ -98,7 +97,6 @@ ID: null
IGName: node
Lifecycle: Sync
MaxSize: 1
Members: null
Name: cluster-node
Policies:
- anti-affinity

2
pkg/model/openstackmodel/tests/servergroup/adds-cloud-labels-from-ClusterSpec.yaml
View File

@ -57,7 +57,6 @@ ServerGroup:
IGName: node
Lifecycle: Sync
MaxSize: 1
Members: null
Name: cluster-node
Policies:
- anti-affinity
@ -96,7 +95,6 @@ ID: null
IGName: node
Lifecycle: Sync
MaxSize: 1
Members: null
Name: cluster-node
Policies:
- anti-affinity

2
pkg/model/openstackmodel/tests/servergroup/adds-cloud-labels-from-InstanceGroupSpec.yaml
View File

@ -57,7 +57,6 @@ ServerGroup:
IGName: node
Lifecycle: Sync
MaxSize: 1
Members: null
Name: cluster-node
Policies:
- anti-affinity
@ -96,7 +95,6 @@ ID: null
IGName: node
Lifecycle: Sync
MaxSize: 1
Members: null
Name: cluster-node
Policies:
- anti-affinity

8
pkg/model/openstackmodel/tests/servergroup/multizone-setup-3-masters-3-nodes-without-bastion-auto-zone-distribution.yaml
View File

@ -113,7 +113,6 @@ ServerGroup:
IGName: master
Lifecycle: Sync
MaxSize: 3
Members: null
Name: cluster-master
Policies:
- anti-affinity
@ -190,7 +189,6 @@ ServerGroup:
IGName: master
Lifecycle: Sync
MaxSize: 3
Members: null
Name: cluster-master
Policies:
- anti-affinity
@ -267,7 +265,6 @@ ServerGroup:
IGName: master
Lifecycle: Sync
MaxSize: 3
Members: null
Name: cluster-master
Policies:
- anti-affinity
@ -337,7 +334,6 @@ ServerGroup:
IGName: node
Lifecycle: Sync
MaxSize: 3
Members: null
Name: cluster-node
Policies:
- anti-affinity
@ -407,7 +403,6 @@ ServerGroup:
IGName: node
Lifecycle: Sync
MaxSize: 3
Members: null
Name: cluster-node
Policies:
- anti-affinity
@ -477,7 +472,6 @@ ServerGroup:
IGName: node
Lifecycle: Sync
MaxSize: 3
Members: null
Name: cluster-node
Policies:
- anti-affinity
@ -664,7 +658,6 @@ ID: null
IGName: master
Lifecycle: Sync
MaxSize: 3
Members: null
Name: cluster-master
Policies:
- anti-affinity
@ -674,7 +667,6 @@ ID: null
IGName: node
Lifecycle: Sync
MaxSize: 3
Members: null
Name: cluster-node
Policies:
- anti-affinity

15
pkg/model/openstackmodel/tests/servergroup/multizone-setup-3-masters-3-nodes-without-bastion-with-API-loadbalancer.yaml
View File

@ -108,7 +108,6 @@ ServerGroup:
IGName: master-a
Lifecycle: Sync
MaxSize: 1
Members: null
Name: cluster-master-a
Policies:
- anti-affinity
@ -173,7 +172,6 @@ ServerGroup:
IGName: master-b
Lifecycle: Sync
MaxSize: 1
Members: null
Name: cluster-master-b
Policies:
- anti-affinity
@ -238,7 +236,6 @@ ServerGroup:
IGName: master-c
Lifecycle: Sync
MaxSize: 1
Members: null
Name: cluster-master-c
Policies:
- anti-affinity
@ -308,7 +305,6 @@ ServerGroup:
IGName: node-a
Lifecycle: Sync
MaxSize: 1
Members: null
Name: cluster-node-a
Policies:
- anti-affinity
@ -378,7 +374,6 @@ ServerGroup:
IGName: node-b
Lifecycle: Sync
MaxSize: 1
Members: null
Name: cluster-node-b
Policies:
- anti-affinity
@ -448,7 +443,6 @@ ServerGroup:
IGName: node-c
Lifecycle: Sync
MaxSize: 1
Members: null
Name: cluster-node-c
Policies:
- anti-affinity
@ -540,7 +534,6 @@ ServerGroup:
IGName: master-a
Lifecycle: Sync
MaxSize: 1
Members: null
Name: cluster-master-a
Policies:
- anti-affinity
@ -574,7 +567,6 @@ ServerGroup:
IGName: master-b
Lifecycle: Sync
MaxSize: 1
Members: null
Name: cluster-master-b
Policies:
- anti-affinity
@ -608,7 +600,6 @@ ServerGroup:
IGName: master-c
Lifecycle: Sync
MaxSize: 1
Members: null
Name: cluster-master-c
Policies:
- anti-affinity
@ -774,7 +765,6 @@ ID: null
IGName: master-a
Lifecycle: Sync
MaxSize: 1
Members: null
Name: cluster-master-a
Policies:
- anti-affinity
@ -784,7 +774,6 @@ ID: null
IGName: master-b
Lifecycle: Sync
MaxSize: 1
Members: null
Name: cluster-master-b
Policies:
- anti-affinity
@ -794,7 +783,6 @@ ID: null
IGName: master-c
Lifecycle: Sync
MaxSize: 1
Members: null
Name: cluster-master-c
Policies:
- anti-affinity
@ -804,7 +792,6 @@ ID: null
IGName: node-a
Lifecycle: Sync
MaxSize: 1
Members: null
Name: cluster-node-a
Policies:
- anti-affinity
@ -814,7 +801,6 @@ ID: null
IGName: node-b
Lifecycle: Sync
MaxSize: 1
Members: null
Name: cluster-node-b
Policies:
- anti-affinity
@ -824,7 +810,6 @@ ID: null
IGName: node-c
Lifecycle: Sync
MaxSize: 1
Members: null
Name: cluster-node-c
Policies:
- anti-affinity

12
pkg/model/openstackmodel/tests/servergroup/multizone-setup-3-masters-3-nodes-without-bastion.yaml
View File

@ -121,7 +121,6 @@ ServerGroup:
IGName: master-a
Lifecycle: Sync
MaxSize: 1
Members: null
Name: cluster-master-a
Policies:
- anti-affinity
@ -198,7 +197,6 @@ ServerGroup:
IGName: master-b
Lifecycle: Sync
MaxSize: 1
Members: null
Name: cluster-master-b
Policies:
- anti-affinity
@ -275,7 +273,6 @@ ServerGroup:
IGName: master-c
Lifecycle: Sync
MaxSize: 1
Members: null
Name: cluster-master-c
Policies:
- anti-affinity
@ -345,7 +342,6 @@ ServerGroup:
IGName: node-a
Lifecycle: Sync
MaxSize: 1
Members: null
Name: cluster-node-a
Policies:
- anti-affinity
@ -415,7 +411,6 @@ ServerGroup:
IGName: node-b
Lifecycle: Sync
MaxSize: 1
Members: null
Name: cluster-node-b
Policies:
- anti-affinity
@ -485,7 +480,6 @@ ServerGroup:
IGName: node-c
Lifecycle: Sync
MaxSize: 1
Members: null
Name: cluster-node-c
Policies:
- anti-affinity
@ -672,7 +666,6 @@ ID: null
IGName: master-a
Lifecycle: Sync
MaxSize: 1
Members: null
Name: cluster-master-a
Policies:
- anti-affinity
@ -682,7 +675,6 @@ ID: null
IGName: master-b
Lifecycle: Sync
MaxSize: 1
Members: null
Name: cluster-master-b
Policies:
- anti-affinity
@ -692,7 +684,6 @@ ID: null
IGName: master-c
Lifecycle: Sync
MaxSize: 1
Members: null
Name: cluster-master-c
Policies:
- anti-affinity
@ -702,7 +693,6 @@ ID: null
IGName: node-a
Lifecycle: Sync
MaxSize: 1
Members: null
Name: cluster-node-a
Policies:
- anti-affinity
@ -712,7 +702,6 @@ ID: null
IGName: node-b
Lifecycle: Sync
MaxSize: 1
Members: null
Name: cluster-node-b
Policies:
- anti-affinity
@ -722,7 +711,6 @@ ID: null
IGName: node-c
Lifecycle: Sync
MaxSize: 1
Members: null
Name: cluster-node-c
Policies:
- anti-affinity

12
pkg/model/openstackmodel/tests/servergroup/multizone-setup-3-masters-3-nodes-without-external-router.yaml
View File

@ -73,7 +73,6 @@ ServerGroup:
IGName: master-a
Lifecycle: Sync
MaxSize: 1
Members: null
Name: cluster-master-a
Policies:
- anti-affinity
@ -144,7 +143,6 @@ ServerGroup:
IGName: master-b
Lifecycle: Sync
MaxSize: 1
Members: null
Name: cluster-master-b
Policies:
- anti-affinity
@ -215,7 +213,6 @@ ServerGroup:
IGName: master-c
Lifecycle: Sync
MaxSize: 1
Members: null
Name: cluster-master-c
Policies:
- anti-affinity
@ -279,7 +276,6 @@ ServerGroup:
IGName: node-a
Lifecycle: Sync
MaxSize: 1
Members: null
Name: cluster-node-a
Policies:
- anti-affinity
@ -343,7 +339,6 @@ ServerGroup:
IGName: node-b
Lifecycle: Sync
MaxSize: 1
Members: null
Name: cluster-node-b
Policies:
- anti-affinity
@ -407,7 +402,6 @@ ServerGroup:
IGName: node-c
Lifecycle: Sync
MaxSize: 1
Members: null
Name: cluster-node-c
Policies:
- anti-affinity
@ -594,7 +588,6 @@ ID: null
IGName: master-a
Lifecycle: Sync
MaxSize: 1
Members: null
Name: cluster-master-a
Policies:
- anti-affinity
@ -604,7 +597,6 @@ ID: null
IGName: master-b
Lifecycle: Sync
MaxSize: 1
Members: null
Name: cluster-master-b
Policies:
- anti-affinity
@ -614,7 +606,6 @@ ID: null
IGName: master-c
Lifecycle: Sync
MaxSize: 1
Members: null
Name: cluster-master-c
Policies:
- anti-affinity
@ -624,7 +615,6 @@ ID: null
IGName: node-a
Lifecycle: Sync
MaxSize: 1
Members: null
Name: cluster-node-a
Policies:
- anti-affinity
@ -634,7 +624,6 @@ ID: null
IGName: node-b
Lifecycle: Sync
MaxSize: 1
Members: null
Name: cluster-node-b
Policies:
- anti-affinity
@ -644,7 +633,6 @@ ID: null
IGName: node-c
Lifecycle: Sync
MaxSize: 1
Members: null
Name: cluster-node-c
Policies:
- anti-affinity

6
pkg/model/openstackmodel/tests/servergroup/one-master-one-node-one-bastion-2.yaml
View File

@ -59,7 +59,6 @@ ServerGroup:
IGName: bastion
Lifecycle: Sync
MaxSize: 1
Members: null
Name: cluster-bastion
Policies:
- anti-affinity
@ -130,7 +129,6 @@ ServerGroup:
IGName: master
Lifecycle: Sync
MaxSize: 1
Members: null
Name: cluster-master
Policies:
- anti-affinity
@ -194,7 +192,6 @@ ServerGroup:
IGName: node
Lifecycle: Sync
MaxSize: 1
Members: null
Name: cluster-node
Policies:
- anti-affinity
@ -291,7 +288,6 @@ ID: null
IGName: bastion
Lifecycle: Sync
MaxSize: 1
Members: null
Name: cluster-bastion
Policies:
- anti-affinity
@ -301,7 +297,6 @@ ID: null
IGName: master
Lifecycle: Sync
MaxSize: 1
Members: null
Name: cluster-master
Policies:
- anti-affinity
@ -311,7 +306,6 @@ ID: null
IGName: node
Lifecycle: Sync
MaxSize: 1
Members: null
Name: cluster-node
Policies:
- anti-affinity

6
pkg/model/openstackmodel/tests/servergroup/one-master-one-node-one-bastion.yaml
View File

@ -79,7 +79,6 @@ ServerGroup:
IGName: bastion
Lifecycle: Sync
MaxSize: 1
Members: null
Name: cluster-bastion
Policies:
- anti-affinity
@ -156,7 +155,6 @@ ServerGroup:
IGName: master
Lifecycle: Sync
MaxSize: 1
Members: null
Name: cluster-master
Policies:
- anti-affinity
@ -220,7 +218,6 @@ ServerGroup:
IGName: node
Lifecycle: Sync
MaxSize: 1
Members: null
Name: cluster-node
Policies:
- anti-affinity
@ -317,7 +314,6 @@ ID: null
IGName: bastion
Lifecycle: Sync
MaxSize: 1
Members: null
Name: cluster-bastion
Policies:
- anti-affinity
@ -327,7 +323,6 @@ ID: null
IGName: master
Lifecycle: Sync
MaxSize: 1
Members: null
Name: cluster-master
Policies:
- anti-affinity
@ -337,7 +332,6 @@ ID: null
IGName: node
Lifecycle: Sync
MaxSize: 1
Members: null
Name: cluster-node
Policies:
- anti-affinity

4
pkg/model/openstackmodel/tests/servergroup/one-master-one-node-without-bastion-no-public-ip-association.yaml
View File

@ -65,7 +65,6 @@ ServerGroup:
IGName: master
Lifecycle: Sync
MaxSize: 1
Members: null
Name: cluster-master
Policies:
- anti-affinity
@ -129,7 +128,6 @@ ServerGroup:
IGName: node
Lifecycle: Sync
MaxSize: 1
Members: null
Name: cluster-node
Policies:
- anti-affinity
@ -200,7 +198,6 @@ ID: null
IGName: master
Lifecycle: Sync
MaxSize: 1
Members: null
Name: cluster-master
Policies:
- anti-affinity
@ -210,7 +207,6 @@ ID: null
IGName: node
Lifecycle: Sync
MaxSize: 1
Members: null
Name: cluster-node
Policies:
- anti-affinity

4
pkg/model/openstackmodel/tests/servergroup/one-master-one-node.yaml
View File

@ -85,7 +85,6 @@ ServerGroup:
IGName: master
Lifecycle: Sync
MaxSize: 1
Members: null
Name: cluster-master
Policies:
- anti-affinity
@ -155,7 +154,6 @@ ServerGroup:
IGName: node
Lifecycle: Sync
MaxSize: 1
Members: null
Name: cluster-node
Policies:
- anti-affinity
@ -226,7 +224,6 @@ ID: null
IGName: master
Lifecycle: Sync
MaxSize: 1
Members: null
Name: cluster-master
Policies:
- anti-affinity
@ -236,7 +233,6 @@ ID: null
IGName: node
Lifecycle: Sync
MaxSize: 1
Members: null
Name: cluster-node
Policies:
- anti-affinity

2
pkg/model/openstackmodel/tests/servergroup/uses-instance-group-subnet-as-availability-zones-fallback.yaml
View File

@ -58,7 +58,6 @@ ServerGroup:
IGName: node
Lifecycle: Sync
MaxSize: 1
Members: null
Name: cluster-node
Policies:
- anti-affinity
@ -98,7 +97,6 @@ ID: null
IGName: node
Lifecycle: Sync
MaxSize: 1
Members: null
Name: cluster-node
Policies:
- anti-affinity

2
pkg/model/openstackmodel/tests/servergroup/uses-instance-group-zones-as-availability-zones.yaml
View File

@ -58,7 +58,6 @@ ServerGroup:
IGName: node
Lifecycle: Sync
MaxSize: 1
Members: null
Name: cluster-node
Policies:
- anti-affinity
@ -98,7 +97,6 @@ ID: null
IGName: node
Lifecycle: Sync
MaxSize: 1
Members: null
Name: cluster-node
Policies:
- anti-affinity

2
upup/pkg/fi/cloudup/openstacktasks/instance.go
View File

@ -297,7 +297,7 @@ func (_ *Instance) RenderOpenstack(t *openstack.OpenstackAPITarget, a, e, change
return fmt.Errorf("Error creating instance: %v", err)
}
e.ID = fi.String(v.ID)
e.ServerGroup.Members = append(e.ServerGroup.Members, fi.StringValue(e.ID))
e.ServerGroup.AddNewMember(fi.StringValue(e.ID))
if e.FloatingIP != nil {
err = associateFloatingIP(t, e)

2
upup/pkg/fi/cloudup/openstacktasks/poolassociation.go
View File

@ -160,7 +160,7 @@ func GetServerFixedIP(client *gophercloud.ServiceClient, serverID string, interf
func (_ *PoolAssociation) RenderOpenstack(t *openstack.OpenstackAPITarget, a, e, changes *PoolAssociation) error {
if a == nil {
for _, serverID := range e.ServerGroup.Members {
for _, serverID := range e.ServerGroup.GetMembers() {
server, memberAddress, err := GetServerFixedIP(t.Cloud.ComputeClient(), serverID, fi.StringValue(e.InterfaceName))
if err != nil {
return err

42
upup/pkg/fi/cloudup/openstacktasks/servergroup.go
View File

@ -19,6 +19,7 @@ package openstacktasks
import (
"fmt"
"strings"
"sync"
"github.com/gophercloud/gophercloud/openstack/compute/v2/extensions/servergroups"
"github.com/gophercloud/gophercloud/openstack/compute/v2/servers"
@ -33,10 +34,19 @@ type ServerGroup struct {
Name *string
ClusterName *string
IGName *string
Members []string
Policies []string
MaxSize *int32
Lifecycle *fi.Lifecycle
mutex sync.Mutex
// members caches a list of member instance names.
// When we create new instances, we can add them to this list also.
members []string
// gotMemberList records if we have returned the member list to another task.
// If we attempt to add a member after doing so, it indicates a missing dependency.
gotMemberList bool
}
var _ fi.CompareWithID = &ServerGroup{}
@ -71,10 +81,10 @@ func (s *ServerGroup) Find(context *fi.Context) (*ServerGroup, error) {
ClusterName: s.ClusterName,
IGName: s.IGName,
ID: fi.String(serverGroup.ID),
Members: serverGroup.Members,
Lifecycle: s.Lifecycle,
Policies: serverGroup.Policies,
MaxSize: fi.Int32(int32(len(serverGroup.Members))),
members: serverGroup.Members,
}
}
}
@ -88,7 +98,7 @@ func (s *ServerGroup) Find(context *fi.Context) (*ServerGroup, error) {
}
s.ID = actual.ID
s.Members = actual.Members
s.members = actual.members
return actual, nil
}
@ -157,3 +167,29 @@ func (_ *ServerGroup) RenderOpenstack(t *openstack.OpenstackAPITarget, a, e, cha
klog.V(2).Infof("Openstack task ServerGroup::RenderOpenstack did nothing")
return nil
}
// AddNewMember is called when we created an instance in this server group.
// It adds it to the internal cached list of members.
// If we have already called GetMembers, this function will panic;
// this can be avoided by use of GetDependencies() (typically by depending on Instance tasks)
func (s *ServerGroup) AddNewMember(memberID string) {
s.mutex.Lock()
defer s.mutex.Unlock()
if s.gotMemberList {
klog.Fatalf("attempt to add member %q after member list already returned", memberID)
}
s.members = append(s.members, memberID)
}
// GetMembers returns the ids of servers in this group.
// It also activates the "flag" preventning further calls to AddNewMember,
// guaranteeing that no new members will be added.
func (s *ServerGroup) GetMembers() []string {
s.mutex.Lock()
defer s.mutex.Unlock()
s.gotMemberList = true
return s.members
}