From 85f41b844b59603a3bb0bfbc57aec85b96b89891 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Le=C3=AFla=20MARABESE?= Date: Mon, 9 Oct 2023 19:51:31 +0200 Subject: [PATCH] get private IPs from IPAM and not from instance API --- protokube/pkg/gossip/scaleway/seeds.go | 38 +++++---- protokube/pkg/protokube/scaleway_volumes.go | 56 +++++++++---- upup/pkg/fi/cloudup/scaleway/cloud.go | 79 +++++++++++++++---- upup/pkg/fi/cloudup/scaleway/verifier.go | 34 +++++--- upup/pkg/fi/cloudup/scalewaytasks/instance.go | 11 +-- .../fi/cloudup/scalewaytasks/lb_backend.go | 19 ++--- 6 files changed, 164 insertions(+), 73 deletions(-) diff --git a/protokube/pkg/gossip/scaleway/seeds.go b/protokube/pkg/gossip/scaleway/seeds.go index f00d942274..7e554967c3 100644 --- a/protokube/pkg/gossip/scaleway/seeds.go +++ b/protokube/pkg/gossip/scaleway/seeds.go @@ -19,7 +19,6 @@ package scaleway import ( "fmt" - "github.com/scaleway/scaleway-sdk-go/api/instance/v1" "github.com/scaleway/scaleway-sdk-go/scw" "k8s.io/klog/v2" "k8s.io/kops/protokube/pkg/gossip" @@ -43,27 +42,38 @@ func NewSeedProvider(scwClient *scw.Client, clusterName string) (*SeedProvider, func (p *SeedProvider) GetSeeds() ([]string, error) { var seeds []string - instanceAPI := instance.NewAPI(p.scwClient) zone, ok := p.scwClient.GetDefaultZone() + if !ok { + return nil, fmt.Errorf("could not determine default zone from client") + } + klog.V(4).Infof("Found zone of the running server: %v", zone) + + region, ok := p.scwClient.GetDefaultRegion() if !ok { return nil, fmt.Errorf("could not determine default region from client") } - servers, err := instanceAPI.ListServers(&instance.ListServersRequest{ - Zone: zone, - Tags: []string{fmt.Sprintf("%s=%s", scaleway.TagClusterName, p.tag)}, - }, scw.WithAllPages()) + klog.V(4).Infof("Found region of the running server: %v", region) + + scwCloud, err := scaleway.NewScwCloud(map[string]string{ + "region": region.String(), + "zone": zone.String(), + }) if err != nil { - return nil, fmt.Errorf("failed to get matching servers: %s", err) + return nil, fmt.Errorf("could not create Scaleway cloud interface: %w", err) } - for _, server := range servers.Servers { - if server.PrivateIP == nil || *server.PrivateIP == "" { - klog.Warningf("failed to find private ip of the server %s(%s)", server.Name, server.ID) - continue - } + servers, err := scwCloud.GetClusterServers(p.tag, nil) + if err != nil { + return nil, fmt.Errorf("failed to get matching servers: %w", err) + } - klog.V(4).Infof("Appending gossip seed %s(%s): %q", server.Name, server.ID, *server.PrivateIP) - seeds = append(seeds, *server.PrivateIP) + for _, server := range servers { + ip, err := scwCloud.GetServerIP(server.ID, server.Zone) + if err != nil { + return nil, fmt.Errorf("getting server IP: %w", err) + } + klog.V(4).Infof("Appending gossip seed %s(%s): %q", server.Name, server.ID, ip) + seeds = append(seeds, ip) } klog.V(4).Infof("Get seeds function done now") diff --git a/protokube/pkg/protokube/scaleway_volumes.go b/protokube/pkg/protokube/scaleway_volumes.go index b6ae7d1a8b..fd9749eea5 100644 --- a/protokube/pkg/protokube/scaleway_volumes.go +++ b/protokube/pkg/protokube/scaleway_volumes.go @@ -21,11 +21,13 @@ import ( "net" "github.com/scaleway/scaleway-sdk-go/api/instance/v1" + ipam "github.com/scaleway/scaleway-sdk-go/api/ipam/v1alpha1" "github.com/scaleway/scaleway-sdk-go/scw" "k8s.io/klog/v2" kopsv "k8s.io/kops" "k8s.io/kops/protokube/pkg/gossip" gossipscw "k8s.io/kops/protokube/pkg/gossip/scaleway" + "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/cloudup/scaleway" ) @@ -43,28 +45,24 @@ func NewScwCloudProvider() (*ScwCloudProvider, error) { metadataAPI := instance.NewMetadataAPI() metadata, err := metadataAPI.GetMetadata() if err != nil { - return nil, fmt.Errorf("failed to retrieve server metadata: %s", err) + return nil, fmt.Errorf("failed to retrieve server metadata: %w", err) } serverID := metadata.ID klog.V(4).Infof("Found ID of the running server: %v", serverID) - zoneID := metadata.Location.ZoneID - zone, err := scw.ParseZone(zoneID) + zone, err := scw.ParseZone(metadata.Location.ZoneID) if err != nil { - return nil, fmt.Errorf("unable to parse Scaleway zone: %s", err) + return nil, fmt.Errorf("unable to parse Scaleway zone: %w", err) } klog.V(4).Infof("Found zone of the running server: %v", zone) - region, err := scaleway.ParseRegionFromZone(zone) + region, err := zone.Region() if err != nil { - return nil, fmt.Errorf("unable to parse Scaleway region: %s", err) + return nil, fmt.Errorf("unable to parse Scaleway region: %w", err) } klog.V(4).Infof("Found region of the running server: %v", region) - privateIP := metadata.PrivateIP - klog.V(4).Infof("Found first private net IP of the running server: %q", privateIP) - profile, err := scaleway.CreateValidScalewayProfile() if err != nil { return nil, err @@ -76,23 +74,49 @@ func NewScwCloudProvider() (*ScwCloudProvider, error) { scw.WithDefaultRegion(region), ) if err != nil { - return nil, fmt.Errorf("error creating client for Protokube: %w", err) + return nil, fmt.Errorf("creating client for Protokube: %w", err) } instanceAPI := instance.NewAPI(scwClient) - server, err := instanceAPI.GetServer(&instance.GetServerRequest{ + serverResponse, err := instanceAPI.GetServer(&instance.GetServerRequest{ ServerID: serverID, Zone: zone, }) - if err != nil || server == nil { - return nil, fmt.Errorf("failed to get the running server: %s", err) + if err != nil || serverResponse.Server == nil { + return nil, fmt.Errorf("failed to get the running server: %w", err) } - klog.V(4).Infof("Found the running server: %q", server.Server.Name) + server := serverResponse.Server + klog.V(4).Infof("Found the running server: %q", server.Name) + + ips, err := ipam.NewAPI(scwClient).ListIPs(&ipam.ListIPsRequest{ + Region: region, + ResourceID: fi.PtrTo(serverID), + IsIPv6: fi.PtrTo(false), + Zonal: fi.PtrTo(zone.String()), + }, scw.WithAllPages()) + if err != nil { + return nil, fmt.Errorf("listing server's IPs: %w", err) + } + if ips.TotalCount < 1 { + return nil, fmt.Errorf("expected at least 1 IP attached to the server %s", server.ID) + } + + var ipToReturn string + for _, ipFound := range ips.IPs { + if ipFound.Address.IP.IsPrivate() == true { + ipToReturn = ipFound.Address.IP.String() + break + } + } + if ipToReturn == "" { + ipToReturn = ips.IPs[0].Address.IP.String() + } + klog.V(4).Infof("Found first private net IP of the running server: %q", ipToReturn) s := &ScwCloudProvider{ scwClient: scwClient, - server: server.Server, - serverIP: net.IP(privateIP), + server: server, + serverIP: net.IP(ipToReturn), } return s, nil diff --git a/upup/pkg/fi/cloudup/scaleway/cloud.go b/upup/pkg/fi/cloudup/scaleway/cloud.go index 12a2988819..092734df43 100644 --- a/upup/pkg/fi/cloudup/scaleway/cloud.go +++ b/upup/pkg/fi/cloudup/scaleway/cloud.go @@ -80,6 +80,7 @@ type ScwCloud interface { GetClusterServers(clusterName string, instanceGroupName *string) ([]*instance.Server, error) GetClusterSSHKeys(clusterName string) ([]*iam.SSHKey, error) GetClusterVolumes(clusterName string) ([]*instance.Volume, error) + GetServerIP(serverID string, zone scw.Zone) (string, error) DeleteDNSRecord(record *domain.Record, clusterName string) error DeleteLoadBalancer(loadBalancer *lb.LB) error @@ -110,16 +111,11 @@ type scwCloudImplementation struct { // NewScwCloud returns a Cloud with a Scaleway Client using the env vars SCW_PROFILE or // SCW_ACCESS_KEY, SCW_SECRET_KEY and SCW_DEFAULT_PROJECT_ID func NewScwCloud(tags map[string]string) (ScwCloud, error) { - region, err := scw.ParseRegion(tags["region"]) - if err != nil { - return nil, err - } - zone, err := scw.ParseZone(tags["zone"]) - if err != nil { - return nil, err - } - var scwClient *scw.Client + var region scw.Region + var zone scw.Zone + var err error + if profileName := os.Getenv("SCW_PROFILE"); profileName == "REDACTED" { // If the profile is REDACTED, we're running integration tests so no need for authentication scwClient, err = scw.NewClient(scw.WithoutAuth()) @@ -138,6 +134,19 @@ func NewScwCloud(tags map[string]string) (ScwCloud, error) { if err != nil { return nil, fmt.Errorf("creating client for Scaleway Cloud: %w", err) } + region = scw.Region(fi.ValueOf(profile.DefaultRegion)) + zone = scw.Zone(fi.ValueOf(profile.DefaultZone)) + } + + if tags != nil { + region, err = scw.ParseRegion(tags["region"]) + if err != nil { + return nil, err + } + zone, err = scw.ParseZone(tags["zone"]) + if err != nil { + return nil, err + } } return &scwCloudImplementation{ @@ -156,7 +165,13 @@ func NewScwCloud(tags map[string]string) (ScwCloud, error) { } func (s *scwCloudImplementation) ClusterName(tags []string) string { - return ClusterNameFromTags(tags) + if tags != nil { + return ClusterNameFromTags(tags) + } + if clusterName, ok := s.tags[TagClusterName]; ok { + return clusterName + } + return "" } func (s *scwCloudImplementation) DNS() (dnsprovider.Interface, error) { @@ -243,6 +258,10 @@ func (s *scwCloudImplementation) DeregisterInstance(i *cloudinstances.CloudInsta if err != nil { return fmt.Errorf("deregistering cloud instance %s of group %q: %w", i.ID, i.CloudInstanceGroup.HumanName, err) } + serverIP, err := s.GetServerIP(server.Server.ID, server.Server.Zone) + if err != nil { + return fmt.Errorf("deregistering cloud instance %s of group %q: %w", i.ID, i.CloudInstanceGroup.HumanName, err) + } // We remove the instance's IP from load-balancers lbs, err := s.GetClusterLoadBalancers(s.ClusterName(server.Server.Tags)) @@ -258,8 +277,8 @@ func (s *scwCloudImplementation) DeregisterInstance(i *cloudinstances.CloudInsta return fmt.Errorf("deregistering cloud instance %s of group %q: listing load-balancer's back-ends for instance creation: %w", i.ID, i.CloudInstanceGroup.HumanName, err) } for _, backEnd := range backEnds.Backends { - for _, serverIP := range backEnd.Pool { - if serverIP == fi.ValueOf(server.Server.PrivateIP) { + for _, ip := range backEnd.Pool { + if ip == serverIP { _, err := s.lbAPI.RemoveBackendServers(&lb.ZonedAPIRemoveBackendServersRequest{ Zone: s.zone, BackendID: backEnd.ID, @@ -340,7 +359,7 @@ func (s *scwCloudImplementation) GetCloudGroups(cluster *kops.Cluster, instanceg continue } - groups[ig.Name], err = buildCloudGroup(ig, serverGroup, nodeMap) + groups[ig.Name], err = buildCloudGroup(s, ig, serverGroup, nodeMap) if err != nil { return nil, fmt.Errorf("failed to build cloud group for instance group %q: %w", ig.Name, err) } @@ -364,7 +383,7 @@ func findServerGroups(s *scwCloudImplementation, clusterName string) (map[string return serverGroups, nil } -func buildCloudGroup(ig *kops.InstanceGroup, sg []*instance.Server, nodeMap map[string]*v1.Node) (*cloudinstances.CloudInstanceGroup, error) { +func buildCloudGroup(s *scwCloudImplementation, ig *kops.InstanceGroup, sg []*instance.Server, nodeMap map[string]*v1.Node) (*cloudinstances.CloudInstanceGroup, error) { cloudInstanceGroup := &cloudinstances.CloudInstanceGroup{ HumanName: ig.Name, InstanceGroup: ig, @@ -388,9 +407,11 @@ func buildCloudGroup(ig *kops.InstanceGroup, sg []*instance.Server, nodeMap map[ cloudInstance.State = cloudinstances.State(server.State) cloudInstance.MachineType = server.CommercialType cloudInstance.Roles = append(cloudInstance.Roles, InstanceRoleFromTags(server.Tags)) - if server.PrivateIP != nil { - cloudInstance.PrivateIP = *server.PrivateIP + ip, err := s.GetServerIP(server.ID, server.Zone) + if err != nil { + return nil, fmt.Errorf("getting server IP: %w", err) } + cloudInstance.PrivateIP = ip } return cloudInstanceGroup, nil @@ -474,6 +495,32 @@ func (s *scwCloudImplementation) GetClusterVolumes(clusterName string) ([]*insta return volumes.Volumes, nil } +func (s *scwCloudImplementation) GetServerIP(serverID string, zone scw.Zone) (string, error) { + region, err := zone.Region() + if err != nil { + return "", fmt.Errorf("converting zone %s to region: %w", zone, err) + } + + ips, err := s.ipamAPI.ListIPs(&ipam.ListIPsRequest{ + Region: region, + IsIPv6: fi.PtrTo(false), + ResourceID: &serverID, + Zonal: fi.PtrTo(zone.String()), + }, scw.WithAllPages()) + if err != nil { + return "", fmt.Errorf("listing IPs for server %s: %w", serverID, err) + } + + if len(ips.IPs) < 1 { + return "", fmt.Errorf("could not find IP for server %s", serverID) + } + if len(ips.IPs) > 1 { + klog.V(10).Infof("Found more than 1 IP for server %s, using %s", serverID, ips.IPs[0].Address.IP.String()) + } + + return ips.IPs[0].Address.IP.String(), nil +} + func (s *scwCloudImplementation) DeleteDNSRecord(record *domain.Record, clusterName string) error { domainName := strings.SplitN(clusterName, ".", 2)[1] recordDeleteRequest := &domain.UpdateDNSZoneRecordsRequest{ diff --git a/upup/pkg/fi/cloudup/scaleway/verifier.go b/upup/pkg/fi/cloudup/scaleway/verifier.go index a580b6bb51..d7de18d4d7 100644 --- a/upup/pkg/fi/cloudup/scaleway/verifier.go +++ b/upup/pkg/fi/cloudup/scaleway/verifier.go @@ -25,10 +25,12 @@ import ( "strings" "github.com/scaleway/scaleway-sdk-go/api/instance/v1" + ipam "github.com/scaleway/scaleway-sdk-go/api/ipam/v1alpha1" "github.com/scaleway/scaleway-sdk-go/scw" kopsv "k8s.io/kops" "k8s.io/kops/pkg/bootstrap" "k8s.io/kops/pkg/wellknownports" + "k8s.io/kops/upup/pkg/fi" ) type ScalewayVerifierOptions struct{} @@ -71,6 +73,10 @@ func (v scalewayVerifier) VerifyToken(ctx context.Context, rawRequest *http.Requ if err != nil { return nil, fmt.Errorf("unable to parse Scaleway zone %q: %w", metadata.Location.ZoneID, err) } + region, err := zone.Region() + if err != nil { + return nil, fmt.Errorf("unable to determine region from zone %s", zone) + } profile, err := CreateValidScalewayProfile() if err != nil { @@ -84,25 +90,33 @@ func (v scalewayVerifier) VerifyToken(ctx context.Context, rawRequest *http.Requ return nil, fmt.Errorf("creating client for Scaleway Verifier: %w", err) } - instanceAPI := instance.NewAPI(scwClient) - serverResponse, err := instanceAPI.GetServer(&instance.GetServerRequest{ + serverResponse, err := instance.NewAPI(scwClient).GetServer(&instance.GetServerRequest{ ServerID: serverID, Zone: zone, }, scw.WithContext(ctx)) - if err != nil || serverResponse == nil { + if err != nil || serverResponse == nil || serverResponse.Server == nil { return nil, fmt.Errorf("failed to get server %s: %w", serverID, err) } server := serverResponse.Server + ips, err := ipam.NewAPI(scwClient).ListIPs(&ipam.ListIPsRequest{ + Region: region, + ResourceID: fi.PtrTo(server.ID), + IsIPv6: fi.PtrTo(false), + Zonal: fi.PtrTo(zone.String()), + }, scw.WithContext(ctx), scw.WithAllPages()) + if err != nil { + return nil, fmt.Errorf("failed to get IP for server %q: %w", server.Name, err) + } + if ips.TotalCount == 0 { + return nil, fmt.Errorf("no IP found for server %q: %w", server.Name, err) + } + addresses := []string(nil) challengeEndPoints := []string(nil) - if server.PrivateIP != nil { - addresses = append(addresses, *server.PrivateIP) - challengeEndPoints = append(challengeEndPoints, net.JoinHostPort(*server.PrivateIP, strconv.Itoa(wellknownports.NodeupChallenge))) - } - if server.IPv6 != nil { - addresses = append(addresses, server.IPv6.Address.String()) - challengeEndPoints = append(challengeEndPoints, net.JoinHostPort(server.IPv6.Address.String(), strconv.Itoa(wellknownports.NodeupChallenge))) + for _, ip := range ips.IPs { + addresses = append(addresses, ip.Address.IP.String()) + challengeEndPoints = append(challengeEndPoints, net.JoinHostPort(ip.Address.IP.String(), strconv.Itoa(wellknownports.NodeupChallenge))) } result := &bootstrap.VerifyResult{ diff --git a/upup/pkg/fi/cloudup/scalewaytasks/instance.go b/upup/pkg/fi/cloudup/scalewaytasks/instance.go index 19824f986c..97478b9292 100644 --- a/upup/pkg/fi/cloudup/scalewaytasks/instance.go +++ b/upup/pkg/fi/cloudup/scalewaytasks/instance.go @@ -222,11 +222,12 @@ func (_ *Instance) RenderScw(t *scaleway.ScwAPITarget, actual, expected, changes } createServerRequest := instance.CreateServerRequest{ - Zone: zone, - Name: uniqueName, - CommercialType: fi.ValueOf(expected.CommercialType), - Image: fi.ValueOf(expected.Image), - Tags: expected.Tags, + Zone: zone, + Name: uniqueName, + CommercialType: fi.ValueOf(expected.CommercialType), + Image: fi.ValueOf(expected.Image), + Tags: expected.Tags, + RoutedIPEnabled: fi.PtrTo(true), } // We resize the root volume if needed (for instance types with no local storage) diff --git a/upup/pkg/fi/cloudup/scalewaytasks/lb_backend.go b/upup/pkg/fi/cloudup/scalewaytasks/lb_backend.go index 0c25dac0c7..20e8b59e84 100644 --- a/upup/pkg/fi/cloudup/scalewaytasks/lb_backend.go +++ b/upup/pkg/fi/cloudup/scalewaytasks/lb_backend.go @@ -20,7 +20,6 @@ import ( "fmt" "strings" - "github.com/scaleway/scaleway-sdk-go/api/instance/v1" "github.com/scaleway/scaleway-sdk-go/api/lb/v1" "github.com/scaleway/scaleway-sdk-go/scw" "k8s.io/kops/upup/pkg/fi" @@ -243,7 +242,6 @@ func (l *LBBackend) TerraformLink() *terraformWriter.Literal { func getControlPlanesIPs(scwCloud scaleway.ScwCloud, lb *LoadBalancer, zone scw.Zone) ([]string, error) { var controlPlanePrivateIPs []string - instanceService := scwCloud.InstanceService() servers, err := scwCloud.GetClusterServers(scwCloud.ClusterName(lb.Tags), nil) if err != nil { @@ -251,17 +249,14 @@ func getControlPlanesIPs(scwCloud scaleway.ScwCloud, lb *LoadBalancer, zone scw. } for _, server := range servers { - if role := scaleway.InstanceRoleFromTags(server.Tags); role == scaleway.TagRoleControlPlane { - // We update the server's infos (to get its IP) - srv, err := instanceService.GetServer(&instance.GetServerRequest{ - Zone: zone, - ServerID: server.ID, - }) - if err != nil { - return nil, fmt.Errorf("getting server %s for load-balancer's back-end: %w", srv.Server.ID, err) - } - controlPlanePrivateIPs = append(controlPlanePrivateIPs, *srv.Server.PrivateIP) + if role := scaleway.InstanceRoleFromTags(server.Tags); role == scaleway.TagRoleWorker { + continue } + ip, err := scwCloud.GetServerIP(server.ID, server.Zone) + if err != nil { + return nil, fmt.Errorf("getting IP of server %s for load-balancer's back-end: %w", server.Name, err) + } + controlPlanePrivateIPs = append(controlPlanePrivateIPs, ip) } return controlPlanePrivateIPs, nil