From 542085c671eb7e4352d97b1c3d80122d13211d44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Le=C3=AFla=20MARABESE?= Date: Mon, 9 Oct 2023 19:43:27 +0200 Subject: [PATCH 1/2] add IPAM to vendor + ScwCloud --- upup/pkg/fi/cloudup/scaleway/cloud.go | 8 + .../api/ipam/v1alpha1/ipam_sdk.go | 317 ++++++++++++++++++ vendor/modules.txt | 1 + 3 files changed, 326 insertions(+) create mode 100644 vendor/github.com/scaleway/scaleway-sdk-go/api/ipam/v1alpha1/ipam_sdk.go diff --git a/upup/pkg/fi/cloudup/scaleway/cloud.go b/upup/pkg/fi/cloudup/scaleway/cloud.go index cd2312c198..12a2988819 100644 --- a/upup/pkg/fi/cloudup/scaleway/cloud.go +++ b/upup/pkg/fi/cloudup/scaleway/cloud.go @@ -24,6 +24,7 @@ import ( domain "github.com/scaleway/scaleway-sdk-go/api/domain/v2beta1" iam "github.com/scaleway/scaleway-sdk-go/api/iam/v1alpha1" "github.com/scaleway/scaleway-sdk-go/api/instance/v1" + ipam "github.com/scaleway/scaleway-sdk-go/api/ipam/v1alpha1" "github.com/scaleway/scaleway-sdk-go/api/lb/v1" "github.com/scaleway/scaleway-sdk-go/api/marketplace/v2" "github.com/scaleway/scaleway-sdk-go/scw" @@ -61,6 +62,7 @@ type ScwCloud interface { DomainService() *domain.API IamService() *iam.API InstanceService() *instance.API + IPAMService() *ipam.API LBService() *lb.ZonedAPI MarketplaceService() *marketplace.API @@ -100,6 +102,7 @@ type scwCloudImplementation struct { domainAPI *domain.API iamAPI *iam.API instanceAPI *instance.API + ipamAPI *ipam.API lbAPI *lb.ZonedAPI marketplaceAPI *marketplace.API } @@ -146,6 +149,7 @@ func NewScwCloud(tags map[string]string) (ScwCloud, error) { domainAPI: domain.NewAPI(scwClient), iamAPI: iam.NewAPI(scwClient), instanceAPI: instance.NewAPI(scwClient), + ipamAPI: ipam.NewAPI(scwClient), lbAPI: lb.NewZonedAPI(scwClient), marketplaceAPI: marketplace.NewAPI(scwClient), }, nil @@ -187,6 +191,10 @@ func (s *scwCloudImplementation) InstanceService() *instance.API { return s.instanceAPI } +func (s *scwCloudImplementation) IPAMService() *ipam.API { + return s.ipamAPI +} + func (s *scwCloudImplementation) LBService() *lb.ZonedAPI { return s.lbAPI } diff --git a/vendor/github.com/scaleway/scaleway-sdk-go/api/ipam/v1alpha1/ipam_sdk.go b/vendor/github.com/scaleway/scaleway-sdk-go/api/ipam/v1alpha1/ipam_sdk.go new file mode 100644 index 0000000000..a5876af8ce --- /dev/null +++ b/vendor/github.com/scaleway/scaleway-sdk-go/api/ipam/v1alpha1/ipam_sdk.go @@ -0,0 +1,317 @@ +// This file was automatically generated. DO NOT EDIT. +// If you have any remark or suggestion do not hesitate to open an issue. + +// Package ipam provides methods and message types of the ipam v1alpha1 API. +package ipam + +import ( + "bytes" + "encoding/json" + "fmt" + "net" + "net/http" + "net/url" + "strings" + "time" + + "github.com/scaleway/scaleway-sdk-go/internal/errors" + "github.com/scaleway/scaleway-sdk-go/internal/marshaler" + "github.com/scaleway/scaleway-sdk-go/internal/parameter" + "github.com/scaleway/scaleway-sdk-go/namegenerator" + "github.com/scaleway/scaleway-sdk-go/scw" +) + +// always import dependencies +var ( + _ fmt.Stringer + _ json.Unmarshaler + _ url.URL + _ net.IP + _ http.Header + _ bytes.Reader + _ time.Time + _ = strings.Join + + _ scw.ScalewayRequest + _ marshaler.Duration + _ scw.File + _ = parameter.AddToQuery + _ = namegenerator.GetRandomName +) + +// API: iPAM API. +type API struct { + client *scw.Client +} + +// NewAPI returns a API object from a Scaleway client. +func NewAPI(client *scw.Client) *API { + return &API{ + client: client, + } +} + +type ListIPsRequestOrderBy string + +const ( + ListIPsRequestOrderByCreatedAtDesc = ListIPsRequestOrderBy("created_at_desc") + ListIPsRequestOrderByCreatedAtAsc = ListIPsRequestOrderBy("created_at_asc") + ListIPsRequestOrderByUpdatedAtDesc = ListIPsRequestOrderBy("updated_at_desc") + ListIPsRequestOrderByUpdatedAtAsc = ListIPsRequestOrderBy("updated_at_asc") + ListIPsRequestOrderByAttachedAtDesc = ListIPsRequestOrderBy("attached_at_desc") + ListIPsRequestOrderByAttachedAtAsc = ListIPsRequestOrderBy("attached_at_asc") +) + +func (enum ListIPsRequestOrderBy) String() string { + if enum == "" { + // return default value if empty + return "created_at_desc" + } + return string(enum) +} + +func (enum ListIPsRequestOrderBy) MarshalJSON() ([]byte, error) { + return []byte(fmt.Sprintf(`"%s"`, enum)), nil +} + +func (enum *ListIPsRequestOrderBy) UnmarshalJSON(data []byte) error { + tmp := "" + + if err := json.Unmarshal(data, &tmp); err != nil { + return err + } + + *enum = ListIPsRequestOrderBy(ListIPsRequestOrderBy(tmp).String()) + return nil +} + +type ResourceType string + +const ( + ResourceTypeUnknownType = ResourceType("unknown_type") + ResourceTypeInstanceServer = ResourceType("instance_server") + ResourceTypeInstanceIP = ResourceType("instance_ip") + ResourceTypeInstancePrivateNic = ResourceType("instance_private_nic") + ResourceTypeLBServer = ResourceType("lb_server") + ResourceTypeFipIP = ResourceType("fip_ip") + ResourceTypeVpcGateway = ResourceType("vpc_gateway") + ResourceTypeVpcGatewayNetwork = ResourceType("vpc_gateway_network") + ResourceTypeK8sNode = ResourceType("k8s_node") + ResourceTypeRdbInstance = ResourceType("rdb_instance") + ResourceTypeRedisCluster = ResourceType("redis_cluster") + ResourceTypeBaremetalServer = ResourceType("baremetal_server") + ResourceTypeBaremetalPrivateNic = ResourceType("baremetal_private_nic") +) + +func (enum ResourceType) String() string { + if enum == "" { + // return default value if empty + return "unknown_type" + } + return string(enum) +} + +func (enum ResourceType) MarshalJSON() ([]byte, error) { + return []byte(fmt.Sprintf(`"%s"`, enum)), nil +} + +func (enum *ResourceType) UnmarshalJSON(data []byte) error { + tmp := "" + + if err := json.Unmarshal(data, &tmp); err != nil { + return err + } + + *enum = ResourceType(ResourceType(tmp).String()) + return nil +} + +type IP struct { + ID string `json:"id"` + + Address scw.IPNet `json:"address"` + + ProjectID string `json:"project_id"` + + IsIPv6 bool `json:"is_ipv6"` + + CreatedAt *time.Time `json:"created_at"` + + UpdatedAt *time.Time `json:"updated_at"` + + // Precisely one of Regional, SubnetID, Zonal, ZonalNat must be set. + Regional *bool `json:"regional,omitempty"` + + // Precisely one of Regional, SubnetID, Zonal, ZonalNat must be set. + Zonal *string `json:"zonal,omitempty"` + + // Precisely one of Regional, SubnetID, Zonal, ZonalNat must be set. + ZonalNat *string `json:"zonal_nat,omitempty"` + + // Precisely one of Regional, SubnetID, Zonal, ZonalNat must be set. + SubnetID *string `json:"subnet_id,omitempty"` + + Resource *Resource `json:"resource"` + + Tags []string `json:"tags"` + + Region scw.Region `json:"region"` + + Zone *scw.Zone `json:"zone"` +} + +type ListIPsResponse struct { + TotalCount uint64 `json:"total_count"` + + IPs []*IP `json:"ips"` +} + +type Resource struct { + // Type: default value: unknown_type + Type ResourceType `json:"type"` + + ID string `json:"id"` + + MacAddress *string `json:"mac_address"` + + Name *string `json:"name"` +} + +type Source struct { + + // Precisely one of PrivateNetworkID, Regional, SubnetID, Zonal, ZonalNat must be set. + Zonal *string `json:"zonal,omitempty"` + + // Precisely one of PrivateNetworkID, Regional, SubnetID, Zonal, ZonalNat must be set. + ZonalNat *string `json:"zonal_nat,omitempty"` + + // Precisely one of PrivateNetworkID, Regional, SubnetID, Zonal, ZonalNat must be set. + Regional *bool `json:"regional,omitempty"` + + // Precisely one of PrivateNetworkID, Regional, SubnetID, Zonal, ZonalNat must be set. + PrivateNetworkID *string `json:"private_network_id,omitempty"` + + // Precisely one of PrivateNetworkID, Regional, SubnetID, Zonal, ZonalNat must be set. + SubnetID *string `json:"subnet_id,omitempty"` +} + +// Service API + +// Regions list localities the api is available in +func (s *API) Regions() []scw.Region { + return []scw.Region{scw.RegionFrPar, scw.RegionNlAms, scw.RegionPlWaw} +} + +type ListIPsRequest struct { + // Region: region to target. If none is passed will use default region from the config. + Region scw.Region `json:"-"` + + Page *int32 `json:"-"` + + PageSize *uint32 `json:"-"` + // OrderBy: default value: created_at_desc + OrderBy ListIPsRequestOrderBy `json:"-"` + + ProjectID *string `json:"-"` + + OrganizationID *string `json:"-"` + + Zonal *string `json:"-"` + + ZonalNat *string `json:"-"` + + Regional *bool `json:"-"` + + PrivateNetworkID *string `json:"-"` + + SubnetID *string `json:"-"` + + Attached *bool `json:"-"` + + ResourceID *string `json:"-"` + // ResourceType: default value: unknown_type + ResourceType ResourceType `json:"-"` + + MacAddress *string `json:"-"` + + Tags *[]string `json:"-"` + + IsIPv6 *bool `json:"-"` + + ResourceName *string `json:"-"` + + ResourceIDs []string `json:"-"` +} + +// ListIPs: find IP addresses. +func (s *API) ListIPs(req *ListIPsRequest, opts ...scw.RequestOption) (*ListIPsResponse, error) { + var err error + + if req.Region == "" { + defaultRegion, _ := s.client.GetDefaultRegion() + req.Region = defaultRegion + } + + defaultPageSize, exist := s.client.GetDefaultPageSize() + if (req.PageSize == nil || *req.PageSize == 0) && exist { + req.PageSize = &defaultPageSize + } + + query := url.Values{} + parameter.AddToQuery(query, "page", req.Page) + parameter.AddToQuery(query, "page_size", req.PageSize) + parameter.AddToQuery(query, "order_by", req.OrderBy) + parameter.AddToQuery(query, "project_id", req.ProjectID) + parameter.AddToQuery(query, "organization_id", req.OrganizationID) + parameter.AddToQuery(query, "zonal", req.Zonal) + parameter.AddToQuery(query, "zonal_nat", req.ZonalNat) + parameter.AddToQuery(query, "regional", req.Regional) + parameter.AddToQuery(query, "private_network_id", req.PrivateNetworkID) + parameter.AddToQuery(query, "subnet_id", req.SubnetID) + parameter.AddToQuery(query, "attached", req.Attached) + parameter.AddToQuery(query, "resource_id", req.ResourceID) + parameter.AddToQuery(query, "resource_type", req.ResourceType) + parameter.AddToQuery(query, "mac_address", req.MacAddress) + parameter.AddToQuery(query, "tags", req.Tags) + parameter.AddToQuery(query, "is_ipv6", req.IsIPv6) + parameter.AddToQuery(query, "resource_name", req.ResourceName) + parameter.AddToQuery(query, "resource_ids", req.ResourceIDs) + + if fmt.Sprint(req.Region) == "" { + return nil, errors.New("field Region cannot be empty in request") + } + + scwReq := &scw.ScalewayRequest{ + Method: "GET", + Path: "/ipam/v1alpha1/regions/" + fmt.Sprint(req.Region) + "/ips", + Query: query, + Headers: http.Header{}, + } + + var resp ListIPsResponse + + err = s.client.Do(scwReq, &resp, opts...) + if err != nil { + return nil, err + } + return &resp, nil +} + +// UnsafeGetTotalCount should not be used +// Internal usage only +func (r *ListIPsResponse) UnsafeGetTotalCount() uint64 { + return r.TotalCount +} + +// UnsafeAppend should not be used +// Internal usage only +func (r *ListIPsResponse) UnsafeAppend(res interface{}) (uint64, error) { + results, ok := res.(*ListIPsResponse) + if !ok { + return 0, errors.New("%T type cannot be appended to type %T", res, r) + } + + r.IPs = append(r.IPs, results.IPs...) + r.TotalCount += uint64(len(results.IPs)) + return uint64(len(results.IPs)), nil +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 34075a1c21..1aabcda886 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -792,6 +792,7 @@ github.com/sahilm/fuzzy github.com/scaleway/scaleway-sdk-go/api/domain/v2beta1 github.com/scaleway/scaleway-sdk-go/api/iam/v1alpha1 github.com/scaleway/scaleway-sdk-go/api/instance/v1 +github.com/scaleway/scaleway-sdk-go/api/ipam/v1alpha1 github.com/scaleway/scaleway-sdk-go/api/lb/v1 github.com/scaleway/scaleway-sdk-go/api/marketplace/v1 github.com/scaleway/scaleway-sdk-go/api/marketplace/v2 From 85f41b844b59603a3bb0bfbc57aec85b96b89891 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Le=C3=AFla=20MARABESE?= Date: Mon, 9 Oct 2023 19:51:31 +0200 Subject: [PATCH 2/2] get private IPs from IPAM and not from instance API --- protokube/pkg/gossip/scaleway/seeds.go | 38 +++++---- protokube/pkg/protokube/scaleway_volumes.go | 56 +++++++++---- upup/pkg/fi/cloudup/scaleway/cloud.go | 79 +++++++++++++++---- upup/pkg/fi/cloudup/scaleway/verifier.go | 34 +++++--- upup/pkg/fi/cloudup/scalewaytasks/instance.go | 11 +-- .../fi/cloudup/scalewaytasks/lb_backend.go | 19 ++--- 6 files changed, 164 insertions(+), 73 deletions(-) diff --git a/protokube/pkg/gossip/scaleway/seeds.go b/protokube/pkg/gossip/scaleway/seeds.go index f00d942274..7e554967c3 100644 --- a/protokube/pkg/gossip/scaleway/seeds.go +++ b/protokube/pkg/gossip/scaleway/seeds.go @@ -19,7 +19,6 @@ package scaleway import ( "fmt" - "github.com/scaleway/scaleway-sdk-go/api/instance/v1" "github.com/scaleway/scaleway-sdk-go/scw" "k8s.io/klog/v2" "k8s.io/kops/protokube/pkg/gossip" @@ -43,27 +42,38 @@ func NewSeedProvider(scwClient *scw.Client, clusterName string) (*SeedProvider, func (p *SeedProvider) GetSeeds() ([]string, error) { var seeds []string - instanceAPI := instance.NewAPI(p.scwClient) zone, ok := p.scwClient.GetDefaultZone() + if !ok { + return nil, fmt.Errorf("could not determine default zone from client") + } + klog.V(4).Infof("Found zone of the running server: %v", zone) + + region, ok := p.scwClient.GetDefaultRegion() if !ok { return nil, fmt.Errorf("could not determine default region from client") } - servers, err := instanceAPI.ListServers(&instance.ListServersRequest{ - Zone: zone, - Tags: []string{fmt.Sprintf("%s=%s", scaleway.TagClusterName, p.tag)}, - }, scw.WithAllPages()) + klog.V(4).Infof("Found region of the running server: %v", region) + + scwCloud, err := scaleway.NewScwCloud(map[string]string{ + "region": region.String(), + "zone": zone.String(), + }) if err != nil { - return nil, fmt.Errorf("failed to get matching servers: %s", err) + return nil, fmt.Errorf("could not create Scaleway cloud interface: %w", err) } - for _, server := range servers.Servers { - if server.PrivateIP == nil || *server.PrivateIP == "" { - klog.Warningf("failed to find private ip of the server %s(%s)", server.Name, server.ID) - continue - } + servers, err := scwCloud.GetClusterServers(p.tag, nil) + if err != nil { + return nil, fmt.Errorf("failed to get matching servers: %w", err) + } - klog.V(4).Infof("Appending gossip seed %s(%s): %q", server.Name, server.ID, *server.PrivateIP) - seeds = append(seeds, *server.PrivateIP) + for _, server := range servers { + ip, err := scwCloud.GetServerIP(server.ID, server.Zone) + if err != nil { + return nil, fmt.Errorf("getting server IP: %w", err) + } + klog.V(4).Infof("Appending gossip seed %s(%s): %q", server.Name, server.ID, ip) + seeds = append(seeds, ip) } klog.V(4).Infof("Get seeds function done now") diff --git a/protokube/pkg/protokube/scaleway_volumes.go b/protokube/pkg/protokube/scaleway_volumes.go index b6ae7d1a8b..fd9749eea5 100644 --- a/protokube/pkg/protokube/scaleway_volumes.go +++ b/protokube/pkg/protokube/scaleway_volumes.go @@ -21,11 +21,13 @@ import ( "net" "github.com/scaleway/scaleway-sdk-go/api/instance/v1" + ipam "github.com/scaleway/scaleway-sdk-go/api/ipam/v1alpha1" "github.com/scaleway/scaleway-sdk-go/scw" "k8s.io/klog/v2" kopsv "k8s.io/kops" "k8s.io/kops/protokube/pkg/gossip" gossipscw "k8s.io/kops/protokube/pkg/gossip/scaleway" + "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/cloudup/scaleway" ) @@ -43,28 +45,24 @@ func NewScwCloudProvider() (*ScwCloudProvider, error) { metadataAPI := instance.NewMetadataAPI() metadata, err := metadataAPI.GetMetadata() if err != nil { - return nil, fmt.Errorf("failed to retrieve server metadata: %s", err) + return nil, fmt.Errorf("failed to retrieve server metadata: %w", err) } serverID := metadata.ID klog.V(4).Infof("Found ID of the running server: %v", serverID) - zoneID := metadata.Location.ZoneID - zone, err := scw.ParseZone(zoneID) + zone, err := scw.ParseZone(metadata.Location.ZoneID) if err != nil { - return nil, fmt.Errorf("unable to parse Scaleway zone: %s", err) + return nil, fmt.Errorf("unable to parse Scaleway zone: %w", err) } klog.V(4).Infof("Found zone of the running server: %v", zone) - region, err := scaleway.ParseRegionFromZone(zone) + region, err := zone.Region() if err != nil { - return nil, fmt.Errorf("unable to parse Scaleway region: %s", err) + return nil, fmt.Errorf("unable to parse Scaleway region: %w", err) } klog.V(4).Infof("Found region of the running server: %v", region) - privateIP := metadata.PrivateIP - klog.V(4).Infof("Found first private net IP of the running server: %q", privateIP) - profile, err := scaleway.CreateValidScalewayProfile() if err != nil { return nil, err @@ -76,23 +74,49 @@ func NewScwCloudProvider() (*ScwCloudProvider, error) { scw.WithDefaultRegion(region), ) if err != nil { - return nil, fmt.Errorf("error creating client for Protokube: %w", err) + return nil, fmt.Errorf("creating client for Protokube: %w", err) } instanceAPI := instance.NewAPI(scwClient) - server, err := instanceAPI.GetServer(&instance.GetServerRequest{ + serverResponse, err := instanceAPI.GetServer(&instance.GetServerRequest{ ServerID: serverID, Zone: zone, }) - if err != nil || server == nil { - return nil, fmt.Errorf("failed to get the running server: %s", err) + if err != nil || serverResponse.Server == nil { + return nil, fmt.Errorf("failed to get the running server: %w", err) } - klog.V(4).Infof("Found the running server: %q", server.Server.Name) + server := serverResponse.Server + klog.V(4).Infof("Found the running server: %q", server.Name) + + ips, err := ipam.NewAPI(scwClient).ListIPs(&ipam.ListIPsRequest{ + Region: region, + ResourceID: fi.PtrTo(serverID), + IsIPv6: fi.PtrTo(false), + Zonal: fi.PtrTo(zone.String()), + }, scw.WithAllPages()) + if err != nil { + return nil, fmt.Errorf("listing server's IPs: %w", err) + } + if ips.TotalCount < 1 { + return nil, fmt.Errorf("expected at least 1 IP attached to the server %s", server.ID) + } + + var ipToReturn string + for _, ipFound := range ips.IPs { + if ipFound.Address.IP.IsPrivate() == true { + ipToReturn = ipFound.Address.IP.String() + break + } + } + if ipToReturn == "" { + ipToReturn = ips.IPs[0].Address.IP.String() + } + klog.V(4).Infof("Found first private net IP of the running server: %q", ipToReturn) s := &ScwCloudProvider{ scwClient: scwClient, - server: server.Server, - serverIP: net.IP(privateIP), + server: server, + serverIP: net.IP(ipToReturn), } return s, nil diff --git a/upup/pkg/fi/cloudup/scaleway/cloud.go b/upup/pkg/fi/cloudup/scaleway/cloud.go index 12a2988819..092734df43 100644 --- a/upup/pkg/fi/cloudup/scaleway/cloud.go +++ b/upup/pkg/fi/cloudup/scaleway/cloud.go @@ -80,6 +80,7 @@ type ScwCloud interface { GetClusterServers(clusterName string, instanceGroupName *string) ([]*instance.Server, error) GetClusterSSHKeys(clusterName string) ([]*iam.SSHKey, error) GetClusterVolumes(clusterName string) ([]*instance.Volume, error) + GetServerIP(serverID string, zone scw.Zone) (string, error) DeleteDNSRecord(record *domain.Record, clusterName string) error DeleteLoadBalancer(loadBalancer *lb.LB) error @@ -110,16 +111,11 @@ type scwCloudImplementation struct { // NewScwCloud returns a Cloud with a Scaleway Client using the env vars SCW_PROFILE or // SCW_ACCESS_KEY, SCW_SECRET_KEY and SCW_DEFAULT_PROJECT_ID func NewScwCloud(tags map[string]string) (ScwCloud, error) { - region, err := scw.ParseRegion(tags["region"]) - if err != nil { - return nil, err - } - zone, err := scw.ParseZone(tags["zone"]) - if err != nil { - return nil, err - } - var scwClient *scw.Client + var region scw.Region + var zone scw.Zone + var err error + if profileName := os.Getenv("SCW_PROFILE"); profileName == "REDACTED" { // If the profile is REDACTED, we're running integration tests so no need for authentication scwClient, err = scw.NewClient(scw.WithoutAuth()) @@ -138,6 +134,19 @@ func NewScwCloud(tags map[string]string) (ScwCloud, error) { if err != nil { return nil, fmt.Errorf("creating client for Scaleway Cloud: %w", err) } + region = scw.Region(fi.ValueOf(profile.DefaultRegion)) + zone = scw.Zone(fi.ValueOf(profile.DefaultZone)) + } + + if tags != nil { + region, err = scw.ParseRegion(tags["region"]) + if err != nil { + return nil, err + } + zone, err = scw.ParseZone(tags["zone"]) + if err != nil { + return nil, err + } } return &scwCloudImplementation{ @@ -156,7 +165,13 @@ func NewScwCloud(tags map[string]string) (ScwCloud, error) { } func (s *scwCloudImplementation) ClusterName(tags []string) string { - return ClusterNameFromTags(tags) + if tags != nil { + return ClusterNameFromTags(tags) + } + if clusterName, ok := s.tags[TagClusterName]; ok { + return clusterName + } + return "" } func (s *scwCloudImplementation) DNS() (dnsprovider.Interface, error) { @@ -243,6 +258,10 @@ func (s *scwCloudImplementation) DeregisterInstance(i *cloudinstances.CloudInsta if err != nil { return fmt.Errorf("deregistering cloud instance %s of group %q: %w", i.ID, i.CloudInstanceGroup.HumanName, err) } + serverIP, err := s.GetServerIP(server.Server.ID, server.Server.Zone) + if err != nil { + return fmt.Errorf("deregistering cloud instance %s of group %q: %w", i.ID, i.CloudInstanceGroup.HumanName, err) + } // We remove the instance's IP from load-balancers lbs, err := s.GetClusterLoadBalancers(s.ClusterName(server.Server.Tags)) @@ -258,8 +277,8 @@ func (s *scwCloudImplementation) DeregisterInstance(i *cloudinstances.CloudInsta return fmt.Errorf("deregistering cloud instance %s of group %q: listing load-balancer's back-ends for instance creation: %w", i.ID, i.CloudInstanceGroup.HumanName, err) } for _, backEnd := range backEnds.Backends { - for _, serverIP := range backEnd.Pool { - if serverIP == fi.ValueOf(server.Server.PrivateIP) { + for _, ip := range backEnd.Pool { + if ip == serverIP { _, err := s.lbAPI.RemoveBackendServers(&lb.ZonedAPIRemoveBackendServersRequest{ Zone: s.zone, BackendID: backEnd.ID, @@ -340,7 +359,7 @@ func (s *scwCloudImplementation) GetCloudGroups(cluster *kops.Cluster, instanceg continue } - groups[ig.Name], err = buildCloudGroup(ig, serverGroup, nodeMap) + groups[ig.Name], err = buildCloudGroup(s, ig, serverGroup, nodeMap) if err != nil { return nil, fmt.Errorf("failed to build cloud group for instance group %q: %w", ig.Name, err) } @@ -364,7 +383,7 @@ func findServerGroups(s *scwCloudImplementation, clusterName string) (map[string return serverGroups, nil } -func buildCloudGroup(ig *kops.InstanceGroup, sg []*instance.Server, nodeMap map[string]*v1.Node) (*cloudinstances.CloudInstanceGroup, error) { +func buildCloudGroup(s *scwCloudImplementation, ig *kops.InstanceGroup, sg []*instance.Server, nodeMap map[string]*v1.Node) (*cloudinstances.CloudInstanceGroup, error) { cloudInstanceGroup := &cloudinstances.CloudInstanceGroup{ HumanName: ig.Name, InstanceGroup: ig, @@ -388,9 +407,11 @@ func buildCloudGroup(ig *kops.InstanceGroup, sg []*instance.Server, nodeMap map[ cloudInstance.State = cloudinstances.State(server.State) cloudInstance.MachineType = server.CommercialType cloudInstance.Roles = append(cloudInstance.Roles, InstanceRoleFromTags(server.Tags)) - if server.PrivateIP != nil { - cloudInstance.PrivateIP = *server.PrivateIP + ip, err := s.GetServerIP(server.ID, server.Zone) + if err != nil { + return nil, fmt.Errorf("getting server IP: %w", err) } + cloudInstance.PrivateIP = ip } return cloudInstanceGroup, nil @@ -474,6 +495,32 @@ func (s *scwCloudImplementation) GetClusterVolumes(clusterName string) ([]*insta return volumes.Volumes, nil } +func (s *scwCloudImplementation) GetServerIP(serverID string, zone scw.Zone) (string, error) { + region, err := zone.Region() + if err != nil { + return "", fmt.Errorf("converting zone %s to region: %w", zone, err) + } + + ips, err := s.ipamAPI.ListIPs(&ipam.ListIPsRequest{ + Region: region, + IsIPv6: fi.PtrTo(false), + ResourceID: &serverID, + Zonal: fi.PtrTo(zone.String()), + }, scw.WithAllPages()) + if err != nil { + return "", fmt.Errorf("listing IPs for server %s: %w", serverID, err) + } + + if len(ips.IPs) < 1 { + return "", fmt.Errorf("could not find IP for server %s", serverID) + } + if len(ips.IPs) > 1 { + klog.V(10).Infof("Found more than 1 IP for server %s, using %s", serverID, ips.IPs[0].Address.IP.String()) + } + + return ips.IPs[0].Address.IP.String(), nil +} + func (s *scwCloudImplementation) DeleteDNSRecord(record *domain.Record, clusterName string) error { domainName := strings.SplitN(clusterName, ".", 2)[1] recordDeleteRequest := &domain.UpdateDNSZoneRecordsRequest{ diff --git a/upup/pkg/fi/cloudup/scaleway/verifier.go b/upup/pkg/fi/cloudup/scaleway/verifier.go index a580b6bb51..d7de18d4d7 100644 --- a/upup/pkg/fi/cloudup/scaleway/verifier.go +++ b/upup/pkg/fi/cloudup/scaleway/verifier.go @@ -25,10 +25,12 @@ import ( "strings" "github.com/scaleway/scaleway-sdk-go/api/instance/v1" + ipam "github.com/scaleway/scaleway-sdk-go/api/ipam/v1alpha1" "github.com/scaleway/scaleway-sdk-go/scw" kopsv "k8s.io/kops" "k8s.io/kops/pkg/bootstrap" "k8s.io/kops/pkg/wellknownports" + "k8s.io/kops/upup/pkg/fi" ) type ScalewayVerifierOptions struct{} @@ -71,6 +73,10 @@ func (v scalewayVerifier) VerifyToken(ctx context.Context, rawRequest *http.Requ if err != nil { return nil, fmt.Errorf("unable to parse Scaleway zone %q: %w", metadata.Location.ZoneID, err) } + region, err := zone.Region() + if err != nil { + return nil, fmt.Errorf("unable to determine region from zone %s", zone) + } profile, err := CreateValidScalewayProfile() if err != nil { @@ -84,25 +90,33 @@ func (v scalewayVerifier) VerifyToken(ctx context.Context, rawRequest *http.Requ return nil, fmt.Errorf("creating client for Scaleway Verifier: %w", err) } - instanceAPI := instance.NewAPI(scwClient) - serverResponse, err := instanceAPI.GetServer(&instance.GetServerRequest{ + serverResponse, err := instance.NewAPI(scwClient).GetServer(&instance.GetServerRequest{ ServerID: serverID, Zone: zone, }, scw.WithContext(ctx)) - if err != nil || serverResponse == nil { + if err != nil || serverResponse == nil || serverResponse.Server == nil { return nil, fmt.Errorf("failed to get server %s: %w", serverID, err) } server := serverResponse.Server + ips, err := ipam.NewAPI(scwClient).ListIPs(&ipam.ListIPsRequest{ + Region: region, + ResourceID: fi.PtrTo(server.ID), + IsIPv6: fi.PtrTo(false), + Zonal: fi.PtrTo(zone.String()), + }, scw.WithContext(ctx), scw.WithAllPages()) + if err != nil { + return nil, fmt.Errorf("failed to get IP for server %q: %w", server.Name, err) + } + if ips.TotalCount == 0 { + return nil, fmt.Errorf("no IP found for server %q: %w", server.Name, err) + } + addresses := []string(nil) challengeEndPoints := []string(nil) - if server.PrivateIP != nil { - addresses = append(addresses, *server.PrivateIP) - challengeEndPoints = append(challengeEndPoints, net.JoinHostPort(*server.PrivateIP, strconv.Itoa(wellknownports.NodeupChallenge))) - } - if server.IPv6 != nil { - addresses = append(addresses, server.IPv6.Address.String()) - challengeEndPoints = append(challengeEndPoints, net.JoinHostPort(server.IPv6.Address.String(), strconv.Itoa(wellknownports.NodeupChallenge))) + for _, ip := range ips.IPs { + addresses = append(addresses, ip.Address.IP.String()) + challengeEndPoints = append(challengeEndPoints, net.JoinHostPort(ip.Address.IP.String(), strconv.Itoa(wellknownports.NodeupChallenge))) } result := &bootstrap.VerifyResult{ diff --git a/upup/pkg/fi/cloudup/scalewaytasks/instance.go b/upup/pkg/fi/cloudup/scalewaytasks/instance.go index 19824f986c..97478b9292 100644 --- a/upup/pkg/fi/cloudup/scalewaytasks/instance.go +++ b/upup/pkg/fi/cloudup/scalewaytasks/instance.go @@ -222,11 +222,12 @@ func (_ *Instance) RenderScw(t *scaleway.ScwAPITarget, actual, expected, changes } createServerRequest := instance.CreateServerRequest{ - Zone: zone, - Name: uniqueName, - CommercialType: fi.ValueOf(expected.CommercialType), - Image: fi.ValueOf(expected.Image), - Tags: expected.Tags, + Zone: zone, + Name: uniqueName, + CommercialType: fi.ValueOf(expected.CommercialType), + Image: fi.ValueOf(expected.Image), + Tags: expected.Tags, + RoutedIPEnabled: fi.PtrTo(true), } // We resize the root volume if needed (for instance types with no local storage) diff --git a/upup/pkg/fi/cloudup/scalewaytasks/lb_backend.go b/upup/pkg/fi/cloudup/scalewaytasks/lb_backend.go index 0c25dac0c7..20e8b59e84 100644 --- a/upup/pkg/fi/cloudup/scalewaytasks/lb_backend.go +++ b/upup/pkg/fi/cloudup/scalewaytasks/lb_backend.go @@ -20,7 +20,6 @@ import ( "fmt" "strings" - "github.com/scaleway/scaleway-sdk-go/api/instance/v1" "github.com/scaleway/scaleway-sdk-go/api/lb/v1" "github.com/scaleway/scaleway-sdk-go/scw" "k8s.io/kops/upup/pkg/fi" @@ -243,7 +242,6 @@ func (l *LBBackend) TerraformLink() *terraformWriter.Literal { func getControlPlanesIPs(scwCloud scaleway.ScwCloud, lb *LoadBalancer, zone scw.Zone) ([]string, error) { var controlPlanePrivateIPs []string - instanceService := scwCloud.InstanceService() servers, err := scwCloud.GetClusterServers(scwCloud.ClusterName(lb.Tags), nil) if err != nil { @@ -251,17 +249,14 @@ func getControlPlanesIPs(scwCloud scaleway.ScwCloud, lb *LoadBalancer, zone scw. } for _, server := range servers { - if role := scaleway.InstanceRoleFromTags(server.Tags); role == scaleway.TagRoleControlPlane { - // We update the server's infos (to get its IP) - srv, err := instanceService.GetServer(&instance.GetServerRequest{ - Zone: zone, - ServerID: server.ID, - }) - if err != nil { - return nil, fmt.Errorf("getting server %s for load-balancer's back-end: %w", srv.Server.ID, err) - } - controlPlanePrivateIPs = append(controlPlanePrivateIPs, *srv.Server.PrivateIP) + if role := scaleway.InstanceRoleFromTags(server.Tags); role == scaleway.TagRoleWorker { + continue } + ip, err := scwCloud.GetServerIP(server.ID, server.Zone) + if err != nil { + return nil, fmt.Errorf("getting IP of server %s for load-balancer's back-end: %w", server.Name, err) + } + controlPlanePrivateIPs = append(controlPlanePrivateIPs, ip) } return controlPlanePrivateIPs, nil