From c2654188b096ff9ffc935ad711b340dc9b7e18a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Le=C3=AFla=20MARABESE?= Date: Mon, 17 Apr 2023 19:15:34 +0200 Subject: [PATCH 1/4] added noprefix to tags --- ...etcdmanager-events-control-plane-fr-par-1_content | 5 +++-- ...s-etcdmanager-main-control-plane-fr-par-1_content | 5 +++-- .../update_cluster/minimal_scaleway/kubernetes.tf | 10 +++++----- upup/pkg/fi/cloudup/scaleway/cloud.go | 12 ++++++------ 4 files changed, 17 insertions(+), 15 deletions(-) diff --git a/tests/integration/update_cluster/minimal_scaleway/data/aws_s3_object_manifests-etcdmanager-events-control-plane-fr-par-1_content b/tests/integration/update_cluster/minimal_scaleway/data/aws_s3_object_manifests-etcdmanager-events-control-plane-fr-par-1_content index 4215f51c97..5af0415f5c 100644 --- a/tests/integration/update_cluster/minimal_scaleway/data/aws_s3_object_manifests-etcdmanager-events-control-plane-fr-par-1_content +++ b/tests/integration/update_cluster/minimal_scaleway/data/aws_s3_object_manifests-etcdmanager-events-control-plane-fr-par-1_content @@ -15,8 +15,9 @@ spec: --backup-store=memfs://tests/scw-minimal.k8s.local/backups/etcd/events --client-urls=https://__name__:4002 --cluster-name=etcd-events --containerized=true --dns-suffix=.internal.scw-minimal.k8s.local --grpc-port=3997 --peer-urls=https://__name__:2381 --quarantine-client-urls=https://__name__:3995 - --v=6 --volume-name-tag=instance-group=control-plane-fr-par-1 --volume-provider=scaleway - --volume-tag=k8s.io/etcd=events --volume-tag=k8s.io/role=control-plane --volume-tag=kops.k8s.io/cluster=scw-minimal.k8s.local + --v=6 --volume-name-tag=noprefix=kops.k8s.io/instance-group=control-plane-fr-par-1 + --volume-provider=scaleway --volume-tag=noprefix=kops.k8s.io/cluster=scw-minimal.k8s.local + --volume-tag=noprefix=kops.k8s.io/etcd=events --volume-tag=noprefix=kops.k8s.io/role=ControlPlane > /tmp/pipe 2>&1 env: - name: SCW_ACCESS_KEY diff --git a/tests/integration/update_cluster/minimal_scaleway/data/aws_s3_object_manifests-etcdmanager-main-control-plane-fr-par-1_content b/tests/integration/update_cluster/minimal_scaleway/data/aws_s3_object_manifests-etcdmanager-main-control-plane-fr-par-1_content index d404313e56..86c6d6a5ed 100644 --- a/tests/integration/update_cluster/minimal_scaleway/data/aws_s3_object_manifests-etcdmanager-main-control-plane-fr-par-1_content +++ b/tests/integration/update_cluster/minimal_scaleway/data/aws_s3_object_manifests-etcdmanager-main-control-plane-fr-par-1_content @@ -15,8 +15,9 @@ spec: --backup-store=memfs://tests/scw-minimal.k8s.local/backups/etcd/main --client-urls=https://__name__:4001 --cluster-name=etcd --containerized=true --dns-suffix=.internal.scw-minimal.k8s.local --grpc-port=3996 --peer-urls=https://__name__:2380 --quarantine-client-urls=https://__name__:3994 - --v=6 --volume-name-tag=instance-group=control-plane-fr-par-1 --volume-provider=scaleway - --volume-tag=k8s.io/etcd=main --volume-tag=k8s.io/role=control-plane --volume-tag=kops.k8s.io/cluster=scw-minimal.k8s.local + --v=6 --volume-name-tag=noprefix=kops.k8s.io/instance-group=control-plane-fr-par-1 + --volume-provider=scaleway --volume-tag=noprefix=kops.k8s.io/cluster=scw-minimal.k8s.local + --volume-tag=noprefix=kops.k8s.io/etcd=main --volume-tag=noprefix=kops.k8s.io/role=ControlPlane > /tmp/pipe 2>&1 env: - name: SCW_ACCESS_KEY diff --git a/tests/integration/update_cluster/minimal_scaleway/kubernetes.tf b/tests/integration/update_cluster/minimal_scaleway/kubernetes.tf index 026bc904c5..c9a119c55c 100644 --- a/tests/integration/update_cluster/minimal_scaleway/kubernetes.tf +++ b/tests/integration/update_cluster/minimal_scaleway/kubernetes.tf @@ -187,7 +187,7 @@ resource "scaleway_instance_server" "control-plane-fr-par-1-masters-scw-minimal- image = "ubuntu_focal" ip_id = scaleway_instance_ip.control-plane-fr-par-1-masters-scw-minimal-k8s-local.id name = "control-plane-fr-par-1.masters.scw-minimal.k8s.local" - tags = ["instance-group=control-plane-fr-par-1", "kops.k8s.io/cluster=scw-minimal.k8s.local", "k8s.io/role=control-plane"] + tags = ["noprefix=kops.k8s.io/instance-group=control-plane-fr-par-1", "noprefix=kops.k8s.io/cluster=scw-minimal.k8s.local", "noprefix=kops.k8s.io/role=ControlPlane"] type = "DEV1-M" user_data = { "cloud-init" = filebase64("${path.module}/data/scaleway_instance_server_control-plane-fr-par-1-masters-scw-minimal-k8s-local_user_data") @@ -198,7 +198,7 @@ resource "scaleway_instance_server" "nodes-fr-par-1-scw-minimal-k8s-local" { image = "ubuntu_focal" ip_id = scaleway_instance_ip.nodes-fr-par-1-scw-minimal-k8s-local.id name = "nodes-fr-par-1.scw-minimal.k8s.local" - tags = ["instance-group=nodes-fr-par-1", "kops.k8s.io/cluster=scw-minimal.k8s.local"] + tags = ["noprefix=kops.k8s.io/instance-group=nodes-fr-par-1", "noprefix=kops.k8s.io/cluster=scw-minimal.k8s.local"] type = "DEV1-M" user_data = { "cloud-init" = filebase64("${path.module}/data/scaleway_instance_server_nodes-fr-par-1-scw-minimal-k8s-local_user_data") @@ -208,21 +208,21 @@ resource "scaleway_instance_server" "nodes-fr-par-1-scw-minimal-k8s-local" { resource "scaleway_instance_volume" "etcd-1-etcd-events-scw-minimal-k8s-local" { name = "etcd-1.etcd-events.scw-minimal.k8s.local" size_in_gb = 20 - tags = ["kops.k8s.io/cluster=scw-minimal.k8s.local", "k8s.io/etcd=events", "k8s.io/role=control-plane", "instance-group=control-plane-fr-par-1"] + tags = ["noprefix=kops.k8s.io/cluster=scw-minimal.k8s.local", "noprefix=kops.k8s.io/etcd=events", "noprefix=kops.k8s.io/role=ControlPlane", "noprefix=kops.k8s.io/instance-group=control-plane-fr-par-1"] type = "b_ssd" } resource "scaleway_instance_volume" "etcd-1-etcd-main-scw-minimal-k8s-local" { name = "etcd-1.etcd-main.scw-minimal.k8s.local" size_in_gb = 20 - tags = ["kops.k8s.io/cluster=scw-minimal.k8s.local", "k8s.io/etcd=main", "k8s.io/role=control-plane", "instance-group=control-plane-fr-par-1"] + tags = ["noprefix=kops.k8s.io/cluster=scw-minimal.k8s.local", "noprefix=kops.k8s.io/etcd=main", "noprefix=kops.k8s.io/role=ControlPlane", "noprefix=kops.k8s.io/instance-group=control-plane-fr-par-1"] type = "b_ssd" } resource "scaleway_lb" "api-scw-minimal-k8s-local" { ip_id = scaleway_lb_ip.api-scw-minimal-k8s-local.id name = "api.scw-minimal.k8s.local" - tags = ["k8s.io/role=control-plane"] + tags = ["noprefix=kops.k8s.io/role=ControlPlane"] type = "LB-S" } diff --git a/upup/pkg/fi/cloudup/scaleway/cloud.go b/upup/pkg/fi/cloudup/scaleway/cloud.go index bbf53bd3eb..08e2bebd45 100644 --- a/upup/pkg/fi/cloudup/scaleway/cloud.go +++ b/upup/pkg/fi/cloudup/scaleway/cloud.go @@ -35,13 +35,13 @@ import ( ) const ( - TagClusterName = "kops.k8s.io/cluster" + TagClusterName = "noprefix=kops.k8s.io/cluster" + TagInstanceGroup = "noprefix=kops.k8s.io/instance-group" + TagNameRolePrefix = "noprefix=kops.k8s.io/role" + TagNameEtcdClusterPrefix = "noprefix=kops.k8s.io/etcd" + TagRoleControlPlane = "ControlPlane" + TagRoleWorker = "Node" KopsUserAgentPrefix = "kubernetes-kops/" - TagInstanceGroup = "instance-group" - TagNameRolePrefix = "k8s.io/role" - TagNameEtcdClusterPrefix = "k8s.io/etcd" - TagRoleControlPlane = "control-plane" - TagRoleWorker = "worker" ) // ScwCloud exposes all the interfaces required to operate on Scaleway resources From 499cc1be958008ca7409cd354ebb763487ab845c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Le=C3=AFla=20MARABESE?= Date: Tue, 18 Apr 2023 11:18:04 +0200 Subject: [PATCH 2/4] downgrade CCM --- ...aws_s3_object_scw-minimal.k8s.local-addons-bootstrap_content | 2 +- ...ons-scaleway-cloud-controller.addons.k8s.io-k8s-1.24_content | 2 +- .../k8s-1.24.yaml.template | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration/update_cluster/minimal_scaleway/data/aws_s3_object_scw-minimal.k8s.local-addons-bootstrap_content b/tests/integration/update_cluster/minimal_scaleway/data/aws_s3_object_scw-minimal.k8s.local-addons-bootstrap_content index 25eb25b79c..89a86accb0 100644 --- a/tests/integration/update_cluster/minimal_scaleway/data/aws_s3_object_scw-minimal.k8s.local-addons-bootstrap_content +++ b/tests/integration/update_cluster/minimal_scaleway/data/aws_s3_object_scw-minimal.k8s.local-addons-bootstrap_content @@ -48,7 +48,7 @@ spec: version: 9.99.0 - id: k8s-1.24 manifest: scaleway-cloud-controller.addons.k8s.io/k8s-1.24.yaml - manifestHash: 62cf06c0ba8f17ad6a877108c1f4bb26a167791aac8c3c6f04804c9e2f61ddab + manifestHash: f6c221c3faf7d34d12c58776592cf36a6c46ba28948d19b4ca5f0d34148065d2 name: scaleway-cloud-controller.addons.k8s.io selector: k8s-addon: scaleway-cloud-controller.addons.k8s.io diff --git a/tests/integration/update_cluster/minimal_scaleway/data/aws_s3_object_scw-minimal.k8s.local-addons-scaleway-cloud-controller.addons.k8s.io-k8s-1.24_content b/tests/integration/update_cluster/minimal_scaleway/data/aws_s3_object_scw-minimal.k8s.local-addons-scaleway-cloud-controller.addons.k8s.io-k8s-1.24_content index 21969be70b..f28805c93b 100644 --- a/tests/integration/update_cluster/minimal_scaleway/data/aws_s3_object_scw-minimal.k8s.local-addons-scaleway-cloud-controller.addons.k8s.io-k8s-1.24_content +++ b/tests/integration/update_cluster/minimal_scaleway/data/aws_s3_object_scw-minimal.k8s.local-addons-scaleway-cloud-controller.addons.k8s.io-k8s-1.24_content @@ -51,7 +51,7 @@ spec: envFrom: - secretRef: name: scaleway-secret - image: scaleway/scaleway-cloud-controller-manager:latest + image: scaleway/scaleway-cloud-controller-manager:v0.24.3 imagePullPolicy: Always name: scaleway-cloud-controller-manager resources: diff --git a/upup/models/cloudup/resources/addons/scaleway-cloud-controller.addons.k8s.io/k8s-1.24.yaml.template b/upup/models/cloudup/resources/addons/scaleway-cloud-controller.addons.k8s.io/k8s-1.24.yaml.template index 2d903caca1..8f952060cd 100644 --- a/upup/models/cloudup/resources/addons/scaleway-cloud-controller.addons.k8s.io/k8s-1.24.yaml.template +++ b/upup/models/cloudup/resources/addons/scaleway-cloud-controller.addons.k8s.io/k8s-1.24.yaml.template @@ -58,7 +58,7 @@ spec: tolerationSeconds: 300 containers: - name: scaleway-cloud-controller-manager - image: scaleway/scaleway-cloud-controller-manager:latest + image: scaleway/scaleway-cloud-controller-manager:v0.24.3 imagePullPolicy: Always args: - --cloud-provider=scaleway From dc6e6b41deb499b432ff06700004c802f4b06eee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Le=C3=AFla=20MARABESE?= Date: Tue, 25 Apr 2023 14:27:39 +0200 Subject: [PATCH 3/4] print small fix --- upup/pkg/fi/cloudup/scaleway/cloud.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/upup/pkg/fi/cloudup/scaleway/cloud.go b/upup/pkg/fi/cloudup/scaleway/cloud.go index 08e2bebd45..a5b9fab61b 100644 --- a/upup/pkg/fi/cloudup/scaleway/cloud.go +++ b/upup/pkg/fi/cloudup/scaleway/cloud.go @@ -476,7 +476,7 @@ func (s *scwCloudImplementation) DeleteServer(server *instance.Server) error { }) if err != nil { if is404Error(err) { - klog.V(4).Infof("delete server %s: instance was already deleted", server.ID) + klog.V(4).Infof("delete server %s: instance %q was already deleted", server.ID, server.Name) return nil } return err From 1e20a4c6294de8f8da2d3c25258232dde2c384ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Le=C3=AFla=20MARABESE?= Date: Tue, 25 Apr 2023 14:52:15 +0200 Subject: [PATCH 4/4] unique instance names to comply with CCM --- cmd/kops/integration_test.go | 4 ++-- pkg/model/scalewaymodel/instances.go | 2 +- ...e_server_control-plane-fr-par-1_user_data} | 0 ..._instance_server_nodes-fr-par-1_user_data} | 0 .../minimal_scaleway/kubernetes.tf | 20 +++++++++---------- upup/pkg/fi/cloudup/scalewaytasks/instance.go | 17 ++++++++++++++-- 6 files changed, 28 insertions(+), 15 deletions(-) rename tests/integration/update_cluster/minimal_scaleway/data/{scaleway_instance_server_control-plane-fr-par-1-masters-scw-minimal-k8s-local_user_data => scaleway_instance_server_control-plane-fr-par-1_user_data} (100%) rename tests/integration/update_cluster/minimal_scaleway/data/{scaleway_instance_server_nodes-fr-par-1-scw-minimal-k8s-local_user_data => scaleway_instance_server_nodes-fr-par-1_user_data} (100%) diff --git a/cmd/kops/integration_test.go b/cmd/kops/integration_test.go index 429e0c5aff..48b640a547 100644 --- a/cmd/kops/integration_test.go +++ b/cmd/kops/integration_test.go @@ -1798,8 +1798,8 @@ func (i *integrationTest) runTestTerraformScaleway(t *testing.T) { "aws_s3_object_"+i.clusterName+"-addons-limit-range.addons.k8s.io_content", "aws_s3_object_"+i.clusterName+"-addons-networking.cilium.io-k8s-1.16_content", "aws_s3_object_"+i.clusterName+"-addons-rbac.addons.k8s.io-k8s-1.8_content", - "scaleway_instance_server_control-plane-fr-par-1-masters-scw-minimal-k8s-local_user_data", - "scaleway_instance_server_nodes-fr-par-1-scw-minimal-k8s-local_user_data", + "scaleway_instance_server_control-plane-fr-par-1_user_data", + "scaleway_instance_server_nodes-fr-par-1_user_data", ) i.runTest(t, ctx, h, expectedFilenames, "", "", nil) diff --git a/pkg/model/scalewaymodel/instances.go b/pkg/model/scalewaymodel/instances.go index f7510f3c90..8903e2d11b 100644 --- a/pkg/model/scalewaymodel/instances.go +++ b/pkg/model/scalewaymodel/instances.go @@ -38,7 +38,7 @@ var _ fi.CloudupModelBuilder = &InstanceModelBuilder{} func (d *InstanceModelBuilder) Build(c *fi.CloudupModelBuilderContext) error { for _, ig := range d.InstanceGroups { - name := d.AutoscalingGroupName(ig) + name := ig.Name zone, err := scw.ParseZone(ig.Spec.Subnets[0]) if err != nil { return fmt.Errorf("error building instance task for %q: %w", name, err) diff --git a/tests/integration/update_cluster/minimal_scaleway/data/scaleway_instance_server_control-plane-fr-par-1-masters-scw-minimal-k8s-local_user_data b/tests/integration/update_cluster/minimal_scaleway/data/scaleway_instance_server_control-plane-fr-par-1_user_data similarity index 100% rename from tests/integration/update_cluster/minimal_scaleway/data/scaleway_instance_server_control-plane-fr-par-1-masters-scw-minimal-k8s-local_user_data rename to tests/integration/update_cluster/minimal_scaleway/data/scaleway_instance_server_control-plane-fr-par-1_user_data diff --git a/tests/integration/update_cluster/minimal_scaleway/data/scaleway_instance_server_nodes-fr-par-1-scw-minimal-k8s-local_user_data b/tests/integration/update_cluster/minimal_scaleway/data/scaleway_instance_server_nodes-fr-par-1_user_data similarity index 100% rename from tests/integration/update_cluster/minimal_scaleway/data/scaleway_instance_server_nodes-fr-par-1-scw-minimal-k8s-local_user_data rename to tests/integration/update_cluster/minimal_scaleway/data/scaleway_instance_server_nodes-fr-par-1_user_data diff --git a/tests/integration/update_cluster/minimal_scaleway/kubernetes.tf b/tests/integration/update_cluster/minimal_scaleway/kubernetes.tf index c9a119c55c..aef7d2b416 100644 --- a/tests/integration/update_cluster/minimal_scaleway/kubernetes.tf +++ b/tests/integration/update_cluster/minimal_scaleway/kubernetes.tf @@ -177,31 +177,31 @@ resource "scaleway_iam_ssh_key" "kubernetes-scw-minimal-k8s-local-be_9e_c3_eb_cb public_key = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDKqbVEozfAqng0gx8HTUu69EppcE5SWet6MpwrGShqMVUC4wkoiuVtJDPhMmWmdt7B7Ttc5pvnAZAZaQ6TKMguyBoAyS7qOTLU9/hM803XtSiwQUftOXiJfmsqAXEc8yDyb7UnrF8X7aA3gQJsnQBGJGdp+C88dPHNZenw4PnQc8BNYTCXG9d8F5vJ3xQ5qbiG4HVNoQ2CZh2ht+GedZJ3hl9lMJ24kE/cbMCLKxabMP4ROetECG6PU251jnm84NA8rm0Av/JMmn/c9CFAe0D0D1dGDlHWPsk4mbhGKJ0yU0YliatmPfmgSasismbYzIFf7VPq91ARzRUbavd1fYMBmkMsce0YR/5FdtrpzRhqDzuvwQgQRsoTcttdvp0puFcrtNefMfk8NCbBedIlkzOFxfGiBbe6jde4wqsqEnSrNHwZ2b+Er8z7vjcDPBqYk3gubmMBCrYxg6o1lOS6tTN0kJDUlyKO2AN1ZDr3mpkbhkvZV/N7gLglcClM0X5X7iM= leila@leila-ThinkPad-T14s-Gen-2i" } -resource "scaleway_instance_ip" "control-plane-fr-par-1-masters-scw-minimal-k8s-local" { +resource "scaleway_instance_ip" "control-plane-fr-par-1" { } -resource "scaleway_instance_ip" "nodes-fr-par-1-scw-minimal-k8s-local" { +resource "scaleway_instance_ip" "nodes-fr-par-1" { } -resource "scaleway_instance_server" "control-plane-fr-par-1-masters-scw-minimal-k8s-local" { +resource "scaleway_instance_server" "control-plane-fr-par-1" { image = "ubuntu_focal" - ip_id = scaleway_instance_ip.control-plane-fr-par-1-masters-scw-minimal-k8s-local.id - name = "control-plane-fr-par-1.masters.scw-minimal.k8s.local" + ip_id = scaleway_instance_ip.control-plane-fr-par-1.id + name = "control-plane-fr-par-1" tags = ["noprefix=kops.k8s.io/instance-group=control-plane-fr-par-1", "noprefix=kops.k8s.io/cluster=scw-minimal.k8s.local", "noprefix=kops.k8s.io/role=ControlPlane"] type = "DEV1-M" user_data = { - "cloud-init" = filebase64("${path.module}/data/scaleway_instance_server_control-plane-fr-par-1-masters-scw-minimal-k8s-local_user_data") + "cloud-init" = filebase64("${path.module}/data/scaleway_instance_server_control-plane-fr-par-1_user_data") } } -resource "scaleway_instance_server" "nodes-fr-par-1-scw-minimal-k8s-local" { +resource "scaleway_instance_server" "nodes-fr-par-1" { image = "ubuntu_focal" - ip_id = scaleway_instance_ip.nodes-fr-par-1-scw-minimal-k8s-local.id - name = "nodes-fr-par-1.scw-minimal.k8s.local" + ip_id = scaleway_instance_ip.nodes-fr-par-1.id + name = "nodes-fr-par-1" tags = ["noprefix=kops.k8s.io/instance-group=nodes-fr-par-1", "noprefix=kops.k8s.io/cluster=scw-minimal.k8s.local"] type = "DEV1-M" user_data = { - "cloud-init" = filebase64("${path.module}/data/scaleway_instance_server_nodes-fr-par-1-scw-minimal-k8s-local_user_data") + "cloud-init" = filebase64("${path.module}/data/scaleway_instance_server_nodes-fr-par-1_user_data") } } diff --git a/upup/pkg/fi/cloudup/scalewaytasks/instance.go b/upup/pkg/fi/cloudup/scalewaytasks/instance.go index 44211a75e7..c353f438c7 100644 --- a/upup/pkg/fi/cloudup/scalewaytasks/instance.go +++ b/upup/pkg/fi/cloudup/scalewaytasks/instance.go @@ -65,6 +65,13 @@ func (s *Instance) Find(c *fi.CloudupContext) (*Instance, error) { } server := servers[0] + igName := "" + for _, tag := range server.Tags { + if strings.HasPrefix(tag, scaleway.TagInstanceGroup) { + igName = strings.TrimPrefix(tag, scaleway.TagInstanceGroup+"_") + } + } + role := scaleway.TagRoleWorker for _, tag := range server.Tags { if tag == scaleway.TagNameRolePrefix+"="+scaleway.TagRoleControlPlane { @@ -73,7 +80,7 @@ func (s *Instance) Find(c *fi.CloudupContext) (*Instance, error) { } return &Instance{ - Name: fi.PtrTo(server.Name), + Name: fi.PtrTo(igName), Count: len(servers), Zone: fi.PtrTo(server.Zone.String()), Role: fi.PtrTo(role), @@ -141,11 +148,17 @@ func (_ *Instance) RenderScw(t *scaleway.ScwAPITarget, actual, expected, changes // If newInstanceCount > 0, we need to create new instances for this group for i := 0; i < newInstanceCount; i++ { + // We create a unique name for each server + actualCount := 0 + if actual != nil { + actualCount = actual.Count + } + uniqueName := fmt.Sprintf("%s-%d", fi.ValueOf(expected.Name), i+actualCount) // We create the instance srv, err := instanceService.CreateServer(&instance.CreateServerRequest{ Zone: zone, - Name: fi.ValueOf(expected.Name), + Name: uniqueName, CommercialType: fi.ValueOf(expected.CommercialType), Image: fi.ValueOf(expected.Image), Tags: expected.Tags,