kubernetes
/
kops
mirror of https://github.com/kubernetes/kops.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

get private IPs from IPAM and not from instance API

This commit is contained in:
Leïla MARABESE 2023-10-09 19:51:31 +02:00
parent 542085c671
commit 85f41b844b
6 changed files with 164 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
protokube/pkg/gossip/scaleway/seeds.go
View File

@ -19,7 +19,6 @@ package scaleway
import (
"fmt"
"github.com/scaleway/scaleway-sdk-go/api/instance/v1"
"github.com/scaleway/scaleway-sdk-go/scw"
"k8s.io/klog/v2"
"k8s.io/kops/protokube/pkg/gossip"
@ -43,27 +42,38 @@ func NewSeedProvider(scwClient *scw.Client, clusterName string) (*SeedProvider,
func (p *SeedProvider) GetSeeds() ([]string, error) {
var seeds []string
instanceAPI := instance.NewAPI(p.scwClient)
zone, ok := p.scwClient.GetDefaultZone()
if !ok {
return nil, fmt.Errorf("could not determine default zone from client")
}
klog.V(4).Infof("Found zone of the running server: %v", zone)
region, ok := p.scwClient.GetDefaultRegion()
if !ok {
return nil, fmt.Errorf("could not determine default region from client")
}
servers, err := instanceAPI.ListServers(&instance.ListServersRequest{
Zone: zone,
Tags: []string{fmt.Sprintf("%s=%s", scaleway.TagClusterName, p.tag)},
}, scw.WithAllPages())
klog.V(4).Infof("Found region of the running server: %v", region)
scwCloud, err := scaleway.NewScwCloud(map[string]string{
"region": region.String(),
"zone": zone.String(),
})
if err != nil {
return nil, fmt.Errorf("failed to get matching servers: %s", err)
return nil, fmt.Errorf("could not create Scaleway cloud interface: %w", err)
}
for _, server := range servers.Servers {
if server.PrivateIP == nil || *server.PrivateIP == "" {
klog.Warningf("failed to find private ip of the server %s(%s)", server.Name, server.ID)
continue
}
servers, err := scwCloud.GetClusterServers(p.tag, nil)
if err != nil {
return nil, fmt.Errorf("failed to get matching servers: %w", err)
}
klog.V(4).Infof("Appending gossip seed %s(%s): %q", server.Name, server.ID, *server.PrivateIP)
seeds = append(seeds, *server.PrivateIP)
for _, server := range servers {
ip, err := scwCloud.GetServerIP(server.ID, server.Zone)
if err != nil {
return nil, fmt.Errorf("getting server IP: %w", err)
}
klog.V(4).Infof("Appending gossip seed %s(%s): %q", server.Name, server.ID, ip)
seeds = append(seeds, ip)
}
klog.V(4).Infof("Get seeds function done now")

56
protokube/pkg/protokube/scaleway_volumes.go
View File

@ -21,11 +21,13 @@ import (
"net"
"github.com/scaleway/scaleway-sdk-go/api/instance/v1"
ipam "github.com/scaleway/scaleway-sdk-go/api/ipam/v1alpha1"
"github.com/scaleway/scaleway-sdk-go/scw"
"k8s.io/klog/v2"
kopsv "k8s.io/kops"
"k8s.io/kops/protokube/pkg/gossip"
gossipscw "k8s.io/kops/protokube/pkg/gossip/scaleway"
"k8s.io/kops/upup/pkg/fi"
"k8s.io/kops/upup/pkg/fi/cloudup/scaleway"
)
@ -43,28 +45,24 @@ func NewScwCloudProvider() (*ScwCloudProvider, error) {
metadataAPI := instance.NewMetadataAPI()
metadata, err := metadataAPI.GetMetadata()
if err != nil {
return nil, fmt.Errorf("failed to retrieve server metadata: %s", err)
return nil, fmt.Errorf("failed to retrieve server metadata: %w", err)
}
serverID := metadata.ID
klog.V(4).Infof("Found ID of the running server: %v", serverID)
zoneID := metadata.Location.ZoneID
zone, err := scw.ParseZone(zoneID)
zone, err := scw.ParseZone(metadata.Location.ZoneID)
if err != nil {
return nil, fmt.Errorf("unable to parse Scaleway zone: %s", err)
return nil, fmt.Errorf("unable to parse Scaleway zone: %w", err)
}
klog.V(4).Infof("Found zone of the running server: %v", zone)
region, err := scaleway.ParseRegionFromZone(zone)
region, err := zone.Region()
if err != nil {
return nil, fmt.Errorf("unable to parse Scaleway region: %s", err)
return nil, fmt.Errorf("unable to parse Scaleway region: %w", err)
}
klog.V(4).Infof("Found region of the running server: %v", region)
privateIP := metadata.PrivateIP
klog.V(4).Infof("Found first private net IP of the running server: %q", privateIP)
profile, err := scaleway.CreateValidScalewayProfile()
if err != nil {
return nil, err
@ -76,23 +74,49 @@ func NewScwCloudProvider() (*ScwCloudProvider, error) {
scw.WithDefaultRegion(region),
)
if err != nil {
return nil, fmt.Errorf("error creating client for Protokube: %w", err)
return nil, fmt.Errorf("creating client for Protokube: %w", err)
}
instanceAPI := instance.NewAPI(scwClient)
server, err := instanceAPI.GetServer(&instance.GetServerRequest{
serverResponse, err := instanceAPI.GetServer(&instance.GetServerRequest{
ServerID: serverID,
Zone: zone,
})
if err != nil || server == nil {
return nil, fmt.Errorf("failed to get the running server: %s", err)
if err != nil || serverResponse.Server == nil {
return nil, fmt.Errorf("failed to get the running server: %w", err)
}
klog.V(4).Infof("Found the running server: %q", server.Server.Name)
server := serverResponse.Server
klog.V(4).Infof("Found the running server: %q", server.Name)
ips, err := ipam.NewAPI(scwClient).ListIPs(&ipam.ListIPsRequest{
Region: region,
ResourceID: fi.PtrTo(serverID),
IsIPv6: fi.PtrTo(false),
Zonal: fi.PtrTo(zone.String()),
}, scw.WithAllPages())
if err != nil {
return nil, fmt.Errorf("listing server's IPs: %w", err)
}
if ips.TotalCount < 1 {
return nil, fmt.Errorf("expected at least 1 IP attached to the server %s", server.ID)
}
var ipToReturn string
for _, ipFound := range ips.IPs {
if ipFound.Address.IP.IsPrivate() == true {
ipToReturn = ipFound.Address.IP.String()
break
}
}
if ipToReturn == "" {
ipToReturn = ips.IPs[0].Address.IP.String()
}
klog.V(4).Infof("Found first private net IP of the running server: %q", ipToReturn)
s := &ScwCloudProvider{
scwClient: scwClient,
server: server.Server,
serverIP: net.IP(privateIP),
server: server,
serverIP: net.IP(ipToReturn),
}
return s, nil

79
upup/pkg/fi/cloudup/scaleway/cloud.go
View File

@ -80,6 +80,7 @@ type ScwCloud interface {
GetClusterServers(clusterName string, instanceGroupName *string) ([]*instance.Server, error)
GetClusterSSHKeys(clusterName string) ([]*iam.SSHKey, error)
GetClusterVolumes(clusterName string) ([]*instance.Volume, error)
GetServerIP(serverID string, zone scw.Zone) (string, error)
DeleteDNSRecord(record *domain.Record, clusterName string) error
DeleteLoadBalancer(loadBalancer *lb.LB) error
@ -110,16 +111,11 @@ type scwCloudImplementation struct {
// NewScwCloud returns a Cloud with a Scaleway Client using the env vars SCW_PROFILE or
// SCW_ACCESS_KEY, SCW_SECRET_KEY and SCW_DEFAULT_PROJECT_ID
func NewScwCloud(tags map[string]string) (ScwCloud, error) {
region, err := scw.ParseRegion(tags["region"])
if err != nil {
return nil, err
}
zone, err := scw.ParseZone(tags["zone"])
if err != nil {
return nil, err
}
var scwClient *scw.Client
var region scw.Region
var zone scw.Zone
var err error
if profileName := os.Getenv("SCW_PROFILE"); profileName == "REDACTED" {
// If the profile is REDACTED, we're running integration tests so no need for authentication
scwClient, err = scw.NewClient(scw.WithoutAuth())
@ -138,6 +134,19 @@ func NewScwCloud(tags map[string]string) (ScwCloud, error) {
if err != nil {
return nil, fmt.Errorf("creating client for Scaleway Cloud: %w", err)
}
region = scw.Region(fi.ValueOf(profile.DefaultRegion))
zone = scw.Zone(fi.ValueOf(profile.DefaultZone))
}
if tags != nil {
region, err = scw.ParseRegion(tags["region"])
if err != nil {
return nil, err
}
zone, err = scw.ParseZone(tags["zone"])
if err != nil {
return nil, err
}
}
return &scwCloudImplementation{
@ -156,7 +165,13 @@ func NewScwCloud(tags map[string]string) (ScwCloud, error) {
}
func (s *scwCloudImplementation) ClusterName(tags []string) string {
return ClusterNameFromTags(tags)
if tags != nil {
return ClusterNameFromTags(tags)
}
if clusterName, ok := s.tags[TagClusterName]; ok {
return clusterName
}
return ""
}
func (s *scwCloudImplementation) DNS() (dnsprovider.Interface, error) {
@ -243,6 +258,10 @@ func (s *scwCloudImplementation) DeregisterInstance(i *cloudinstances.CloudInsta
if err != nil {
return fmt.Errorf("deregistering cloud instance %s of group %q: %w", i.ID, i.CloudInstanceGroup.HumanName, err)
}
serverIP, err := s.GetServerIP(server.Server.ID, server.Server.Zone)
if err != nil {
return fmt.Errorf("deregistering cloud instance %s of group %q: %w", i.ID, i.CloudInstanceGroup.HumanName, err)
}
// We remove the instance's IP from load-balancers
lbs, err := s.GetClusterLoadBalancers(s.ClusterName(server.Server.Tags))
@ -258,8 +277,8 @@ func (s *scwCloudImplementation) DeregisterInstance(i *cloudinstances.CloudInsta
return fmt.Errorf("deregistering cloud instance %s of group %q: listing load-balancer's back-ends for instance creation: %w", i.ID, i.CloudInstanceGroup.HumanName, err)
}
for _, backEnd := range backEnds.Backends {
for _, serverIP := range backEnd.Pool {
if serverIP == fi.ValueOf(server.Server.PrivateIP) {
for _, ip := range backEnd.Pool {
if ip == serverIP {
_, err := s.lbAPI.RemoveBackendServers(&lb.ZonedAPIRemoveBackendServersRequest{
Zone: s.zone,
BackendID: backEnd.ID,
@ -340,7 +359,7 @@ func (s *scwCloudImplementation) GetCloudGroups(cluster *kops.Cluster, instanceg
continue
}
groups[ig.Name], err = buildCloudGroup(ig, serverGroup, nodeMap)
groups[ig.Name], err = buildCloudGroup(s, ig, serverGroup, nodeMap)
if err != nil {
return nil, fmt.Errorf("failed to build cloud group for instance group %q: %w", ig.Name, err)
}
@ -364,7 +383,7 @@ func findServerGroups(s *scwCloudImplementation, clusterName string) (map[string
return serverGroups, nil
}
func buildCloudGroup(ig *kops.InstanceGroup, sg []*instance.Server, nodeMap map[string]*v1.Node) (*cloudinstances.CloudInstanceGroup, error) {
func buildCloudGroup(s *scwCloudImplementation, ig *kops.InstanceGroup, sg []*instance.Server, nodeMap map[string]*v1.Node) (*cloudinstances.CloudInstanceGroup, error) {
cloudInstanceGroup := &cloudinstances.CloudInstanceGroup{
HumanName: ig.Name,
InstanceGroup: ig,
@ -388,9 +407,11 @@ func buildCloudGroup(ig *kops.InstanceGroup, sg []*instance.Server, nodeMap map[
cloudInstance.State = cloudinstances.State(server.State)
cloudInstance.MachineType = server.CommercialType
cloudInstance.Roles = append(cloudInstance.Roles, InstanceRoleFromTags(server.Tags))
if server.PrivateIP != nil {
cloudInstance.PrivateIP = *server.PrivateIP
ip, err := s.GetServerIP(server.ID, server.Zone)
if err != nil {
return nil, fmt.Errorf("getting server IP: %w", err)
}
cloudInstance.PrivateIP = ip
}
return cloudInstanceGroup, nil
@ -474,6 +495,32 @@ func (s *scwCloudImplementation) GetClusterVolumes(clusterName string) ([]*insta
return volumes.Volumes, nil
}
func (s *scwCloudImplementation) GetServerIP(serverID string, zone scw.Zone) (string, error) {
region, err := zone.Region()
if err != nil {
return "", fmt.Errorf("converting zone %s to region: %w", zone, err)
}
ips, err := s.ipamAPI.ListIPs(&ipam.ListIPsRequest{
Region: region,
IsIPv6: fi.PtrTo(false),
ResourceID: &serverID,
Zonal: fi.PtrTo(zone.String()),
}, scw.WithAllPages())
if err != nil {
return "", fmt.Errorf("listing IPs for server %s: %w", serverID, err)
}
if len(ips.IPs) < 1 {
return "", fmt.Errorf("could not find IP for server %s", serverID)
}
if len(ips.IPs) > 1 {
klog.V(10).Infof("Found more than 1 IP for server %s, using %s", serverID, ips.IPs[0].Address.IP.String())
}
return ips.IPs[0].Address.IP.String(), nil
}
func (s *scwCloudImplementation) DeleteDNSRecord(record *domain.Record, clusterName string) error {
domainName := strings.SplitN(clusterName, ".", 2)[1]
recordDeleteRequest := &domain.UpdateDNSZoneRecordsRequest{

34
upup/pkg/fi/cloudup/scaleway/verifier.go
View File

@ -25,10 +25,12 @@ import (
"strings"
"github.com/scaleway/scaleway-sdk-go/api/instance/v1"
ipam "github.com/scaleway/scaleway-sdk-go/api/ipam/v1alpha1"
"github.com/scaleway/scaleway-sdk-go/scw"
kopsv "k8s.io/kops"
"k8s.io/kops/pkg/bootstrap"
"k8s.io/kops/pkg/wellknownports"
"k8s.io/kops/upup/pkg/fi"
)
type ScalewayVerifierOptions struct{}
@ -71,6 +73,10 @@ func (v scalewayVerifier) VerifyToken(ctx context.Context, rawRequest *http.Requ
if err != nil {
return nil, fmt.Errorf("unable to parse Scaleway zone %q: %w", metadata.Location.ZoneID, err)
}
region, err := zone.Region()
if err != nil {
return nil, fmt.Errorf("unable to determine region from zone %s", zone)
}
profile, err := CreateValidScalewayProfile()
if err != nil {
@ -84,25 +90,33 @@ func (v scalewayVerifier) VerifyToken(ctx context.Context, rawRequest *http.Requ
return nil, fmt.Errorf("creating client for Scaleway Verifier: %w", err)
}
instanceAPI := instance.NewAPI(scwClient)
serverResponse, err := instanceAPI.GetServer(&instance.GetServerRequest{
serverResponse, err := instance.NewAPI(scwClient).GetServer(&instance.GetServerRequest{
ServerID: serverID,
Zone: zone,
}, scw.WithContext(ctx))
if err != nil || serverResponse == nil {
if err != nil || serverResponse == nil || serverResponse.Server == nil {
return nil, fmt.Errorf("failed to get server %s: %w", serverID, err)
}
server := serverResponse.Server
ips, err := ipam.NewAPI(scwClient).ListIPs(&ipam.ListIPsRequest{
Region: region,
ResourceID: fi.PtrTo(server.ID),
IsIPv6: fi.PtrTo(false),
Zonal: fi.PtrTo(zone.String()),
}, scw.WithContext(ctx), scw.WithAllPages())
if err != nil {
return nil, fmt.Errorf("failed to get IP for server %q: %w", server.Name, err)
}
if ips.TotalCount == 0 {
return nil, fmt.Errorf("no IP found for server %q: %w", server.Name, err)
}
addresses := []string(nil)
challengeEndPoints := []string(nil)
if server.PrivateIP != nil {
addresses = append(addresses, *server.PrivateIP)
challengeEndPoints = append(challengeEndPoints, net.JoinHostPort(*server.PrivateIP, strconv.Itoa(wellknownports.NodeupChallenge)))
}
if server.IPv6 != nil {
addresses = append(addresses, server.IPv6.Address.String())
challengeEndPoints = append(challengeEndPoints, net.JoinHostPort(server.IPv6.Address.String(), strconv.Itoa(wellknownports.NodeupChallenge)))
for _, ip := range ips.IPs {
addresses = append(addresses, ip.Address.IP.String())
challengeEndPoints = append(challengeEndPoints, net.JoinHostPort(ip.Address.IP.String(), strconv.Itoa(wellknownports.NodeupChallenge)))
}
result := &bootstrap.VerifyResult{

11
upup/pkg/fi/cloudup/scalewaytasks/instance.go
View File

@ -222,11 +222,12 @@ func (_ *Instance) RenderScw(t *scaleway.ScwAPITarget, actual, expected, changes
}
createServerRequest := instance.CreateServerRequest{
Zone: zone,
Name: uniqueName,
CommercialType: fi.ValueOf(expected.CommercialType),
Image: fi.ValueOf(expected.Image),
Tags: expected.Tags,
Zone: zone,
Name: uniqueName,
CommercialType: fi.ValueOf(expected.CommercialType),
Image: fi.ValueOf(expected.Image),
Tags: expected.Tags,
RoutedIPEnabled: fi.PtrTo(true),
}
// We resize the root volume if needed (for instance types with no local storage)

19
upup/pkg/fi/cloudup/scalewaytasks/lb_backend.go
View File

@ -20,7 +20,6 @@ import (
"fmt"
"strings"
"github.com/scaleway/scaleway-sdk-go/api/instance/v1"
"github.com/scaleway/scaleway-sdk-go/api/lb/v1"
"github.com/scaleway/scaleway-sdk-go/scw"
"k8s.io/kops/upup/pkg/fi"
@ -243,7 +242,6 @@ func (l *LBBackend) TerraformLink() *terraformWriter.Literal {
func getControlPlanesIPs(scwCloud scaleway.ScwCloud, lb *LoadBalancer, zone scw.Zone) ([]string, error) {
var controlPlanePrivateIPs []string
instanceService := scwCloud.InstanceService()
servers, err := scwCloud.GetClusterServers(scwCloud.ClusterName(lb.Tags), nil)
if err != nil {
@ -251,17 +249,14 @@ func getControlPlanesIPs(scwCloud scaleway.ScwCloud, lb *LoadBalancer, zone scw.
}
for _, server := range servers {
if role := scaleway.InstanceRoleFromTags(server.Tags); role == scaleway.TagRoleControlPlane {
// We update the server's infos (to get its IP)
srv, err := instanceService.GetServer(&instance.GetServerRequest{
Zone: zone,
ServerID: server.ID,
})
if err != nil {
return nil, fmt.Errorf("getting server %s for load-balancer's back-end: %w", srv.Server.ID, err)
}
controlPlanePrivateIPs = append(controlPlanePrivateIPs, *srv.Server.PrivateIP)
if role := scaleway.InstanceRoleFromTags(server.Tags); role == scaleway.TagRoleWorker {
continue
}
ip, err := scwCloud.GetServerIP(server.ID, server.Zone)
if err != nil {
return nil, fmt.Errorf("getting IP of server %s for load-balancer's back-end: %w", server.Name, err)
}
controlPlanePrivateIPs = append(controlPlanePrivateIPs, ip)
}
return controlPlanePrivateIPs, nil