From cc8871eedec5668294f6f5125d5e7c7b966e3e61 Mon Sep 17 00:00:00 2001 From: Jesse Haka Date: Mon, 2 Jan 2023 16:32:29 +0200 Subject: [PATCH] no dns for OpenStack --- cmd/kops/create_cluster_integration_test.go | 5 + nodeup/pkg/model/etc_hosts.go | 6 +- nodeup/pkg/model/kops_controller.go | 4 +- pkg/apis/kops/validation/validation.go | 2 +- pkg/apis/nodeup/config.go | 4 +- pkg/model/bootstrapscript.go | 7 +- pkg/model/openstackmodel/servergroup.go | 13 +- pkg/model/openstackmodel/servergroup_test.go | 160 +++ .../adds-additional-security-groups.yaml | 2 + .../adds-cloud-labels-from-ClusterSpec.yaml | 2 + ...s-cloud-labels-from-InstanceGroupSpec.yaml | 2 + ...erver-group-affinity-with-annotations.yaml | 2 + ...ithout-bastion-auto-zone-distribution.yaml | 12 + ...astion-with-API-loadbalancer-dns-none.yaml | 1078 +++++++++++++++++ ...without-bastion-with-API-loadbalancer.yaml | 13 + ...tup-3-masters-3-nodes-without-bastion.yaml | 12 + ...sters-3-nodes-without-external-router.yaml | 12 + .../one-master-one-node-one-bastion-2.yaml | 6 + .../one-master-one-node-one-bastion.yaml | 6 + ...hout-bastion-no-public-ip-association.yaml | 4 + .../servergroup/one-master-one-node.yaml | 4 + ...uncate-cluster-names-to-42-characters.yaml | 4 + ...subnet-as-availability-zones-fallback.yaml | 2 + ...nce-group-zones-as-availability-zones.yaml | 2 + .../ha_openstack_nodns/expected-v1alpha2.yaml | 156 +++ .../ha_openstack_nodns/options.yaml | 19 + ...mplate_nodes.minimal.example.com_user_data | 2 +- ...al-gce-example-com_metadata_startup-script | 2 +- .../data/hcloud_server_nodes-fsn1_user_data | 2 +- upup/pkg/fi/cloudup/apply_cluster.go | 28 +- upup/pkg/fi/cloudup/new_cluster.go | 28 +- .../fi/cloudup/openstacktasks/lblistener.go | 4 +- upup/pkg/fi/cloudup/openstacktasks/port.go | 21 + 33 files changed, 1589 insertions(+), 37 deletions(-) create mode 100644 pkg/model/openstackmodel/tests/servergroup/multizone-setup-3-masters-3-nodes-without-bastion-with-API-loadbalancer-dns-none.yaml create mode 100644 tests/integration/create_cluster/ha_openstack_nodns/expected-v1alpha2.yaml create mode 100644 tests/integration/create_cluster/ha_openstack_nodns/options.yaml diff --git a/cmd/kops/create_cluster_integration_test.go b/cmd/kops/create_cluster_integration_test.go index 3f5ffde1dc..13c2647349 100644 --- a/cmd/kops/create_cluster_integration_test.go +++ b/cmd/kops/create_cluster_integration_test.go @@ -72,6 +72,11 @@ func TestCreateClusterOpenStackOctavia(t *testing.T) { runCreateClusterIntegrationTest(t, "../../tests/integration/create_cluster/ha_openstack_octavia", "v1alpha2") } +func TestCreateClusterOpenStackNoDNS(t *testing.T) { + t.Setenv("OS_REGION_NAME", "us-test1") + runCreateClusterIntegrationTest(t, "../../tests/integration/create_cluster/ha_openstack_nodns", "v1alpha2") +} + // TestCreateClusterCilium runs kops with the cilium networking flags func TestCreateClusterCilium(t *testing.T) { runCreateClusterIntegrationTest(t, "../../tests/integration/create_cluster/cilium-eni", "v1alpha2") diff --git a/nodeup/pkg/model/etc_hosts.go b/nodeup/pkg/model/etc_hosts.go index 29256fd76c..97311a21d1 100644 --- a/nodeup/pkg/model/etc_hosts.go +++ b/nodeup/pkg/model/etc_hosts.go @@ -46,15 +46,15 @@ func (b *EtcHostsBuilder) Build(c *fi.NodeupModelBuilderContext) error { Addresses: []string{"127.0.0.1"}, }) } - } else if b.BootConfig.APIServerIP != "" { + } else if len(b.BootConfig.APIServerIPs) > 0 { task.Records = append(task.Records, nodetasks.HostRecord{ Hostname: b.APIInternalName(), - Addresses: []string{b.BootConfig.APIServerIP}, + Addresses: b.BootConfig.APIServerIPs, }) if b.UseKopsControllerForNodeBootstrap() { task.Records = append(task.Records, nodetasks.HostRecord{ Hostname: "kops-controller.internal." + b.NodeupConfig.ClusterName, - Addresses: []string{b.BootConfig.APIServerIP}, + Addresses: b.BootConfig.APIServerIPs, }) } } diff --git a/nodeup/pkg/model/kops_controller.go b/nodeup/pkg/model/kops_controller.go index 1e4f362fd5..fe30136457 100644 --- a/nodeup/pkg/model/kops_controller.go +++ b/nodeup/pkg/model/kops_controller.go @@ -62,8 +62,8 @@ func (b *KopsControllerBuilder) Build(c *fi.NodeupModelBuilderContext) error { Subject: nodetasks.PKIXName{CommonName: "kops-controller"}, AlternateNames: []string{"kops-controller.internal." + b.NodeupConfig.ClusterName}, } - if b.BootConfig.APIServerIP != "" { - issueCert.AlternateNames = append(issueCert.AlternateNames, b.BootConfig.APIServerIP) + if len(b.BootConfig.APIServerIPs) > 0 { + issueCert.AlternateNames = append(issueCert.AlternateNames, b.BootConfig.APIServerIPs...) } c.AddTask(issueCert) diff --git a/pkg/apis/kops/validation/validation.go b/pkg/apis/kops/validation/validation.go index 989ae0f2ad..794b093d61 100644 --- a/pkg/apis/kops/validation/validation.go +++ b/pkg/apis/kops/validation/validation.go @@ -507,7 +507,7 @@ func validateTopology(c *kops.Cluster, topology *kops.TopologySpec, fieldPath *f if topology.DNS != "" { cloud := c.Spec.GetCloudProvider() allErrs = append(allErrs, IsValidValue(fieldPath.Child("dns", "type"), &topology.DNS, kops.SupportedDnsTypes)...) - if topology.DNS == kops.DNSTypeNone && cloud != kops.CloudProviderHetzner && cloud != kops.CloudProviderAWS && cloud != kops.CloudProviderGCE { + if topology.DNS == kops.DNSTypeNone && cloud != kops.CloudProviderOpenstack && cloud != kops.CloudProviderHetzner && cloud != kops.CloudProviderAWS && cloud != kops.CloudProviderGCE { allErrs = append(allErrs, field.Invalid(fieldPath.Child("dns", "type"), topology.DNS, fmt.Sprintf("not supported for %q", c.Spec.GetCloudProvider()))) } } diff --git a/pkg/apis/nodeup/config.go b/pkg/apis/nodeup/config.go index a8d6f021c2..eebc15e4b7 100644 --- a/pkg/apis/nodeup/config.go +++ b/pkg/apis/nodeup/config.go @@ -117,9 +117,9 @@ type BootConfig struct { ConfigBase *string `json:",omitempty"` // ConfigServer holds the configuration for the configuration server. ConfigServer *ConfigServerOptions `json:",omitempty"` - // APIServerIP is the API server IP address. + // APIServerIPs is the API server IP addresses. // This field is used for adding an alias for api.internal. in /etc/hosts, when Topology.DNS.Type == DNSTypeNone. - APIServerIP string `json:",omitempty"` + APIServerIPs []string `json:",omitempty"` // InstanceGroupName is the name of the instance group. InstanceGroupName string `json:",omitempty"` // InstanceGroupRole is the instance group role. diff --git a/pkg/model/bootstrapscript.go b/pkg/model/bootstrapscript.go index 3a53549c6f..4a1302c486 100644 --- a/pkg/model/bootstrapscript.go +++ b/pkg/model/bootstrapscript.go @@ -167,8 +167,13 @@ func (b *BootstrapScript) buildEnvironmentVariables(cluster *kops.Cluster) (map[ ) } + // credentials needed always when using swift but when using None dns only in control plane + passEnvs := true + if !strings.HasPrefix(cluster.Spec.ConfigBase, "swift://") && cluster.UsesNoneDNS() && !b.ig.IsControlPlane() { + passEnvs = false + } // Pass in required credentials when using user-defined swift endpoint - if os.Getenv("OS_AUTH_URL") != "" { + if os.Getenv("OS_AUTH_URL") != "" && passEnvs { for _, envVar := range osEnvs { env[envVar] = fmt.Sprintf("'%s'", os.Getenv(envVar)) } diff --git a/pkg/model/openstackmodel/servergroup.go b/pkg/model/openstackmodel/servergroup.go index 595436bed1..ab27cb52ea 100644 --- a/pkg/model/openstackmodel/servergroup.go +++ b/pkg/model/openstackmodel/servergroup.go @@ -26,6 +26,7 @@ import ( "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/pkg/model" "k8s.io/kops/pkg/truncate" + "k8s.io/kops/pkg/wellknownports" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/cloudup/openstack" "k8s.io/kops/upup/pkg/fi/cloudup/openstacktasks" @@ -166,6 +167,10 @@ func (b *ServerGroupModelBuilder) buildInstances(c *fi.CloudupModelBuilderContex } c.AddTask(portTask) + if b.Cluster.UsesNoneDNS() && ig.Spec.Role == kops.InstanceGroupRoleControlPlane { + portTask.ForAPIServer = true + } + metaWithName := make(map[string]string) for k, v := range igMeta { metaWithName[k] = v @@ -315,8 +320,10 @@ func (b *ServerGroupModelBuilder) Build(c *fi.CloudupModelBuilderContext) error } c.AddTask(poolTask) + nameForResource := fi.ValueOf(lbTask.Name) listenerTask := &openstacktasks.LBListener{ - Name: lbTask.Name, + Name: fi.PtrTo(nameForResource), + Port: fi.PtrTo(wellknownports.KubeAPIServer), Lifecycle: b.Lifecycle, Pool: poolTask, } @@ -334,7 +341,7 @@ func (b *ServerGroupModelBuilder) Build(c *fi.CloudupModelBuilderContext) error c.AddTask(listenerTask) monitorTask := &openstacktasks.PoolMonitor{ - Name: lbTask.Name, + Name: fi.PtrTo(nameForResource), Pool: poolTask, Lifecycle: b.Lifecycle, } @@ -350,7 +357,7 @@ func (b *ServerGroupModelBuilder) Build(c *fi.CloudupModelBuilderContext) error Pool: poolTask, ServerGroup: mastersg, InterfaceName: fi.PtrTo(ifName), - ProtocolPort: fi.PtrTo(443), + ProtocolPort: fi.PtrTo(wellknownports.KubeAPIServer), Lifecycle: b.Lifecycle, Weight: fi.PtrTo(1), } diff --git a/pkg/model/openstackmodel/servergroup_test.go b/pkg/model/openstackmodel/servergroup_test.go index 97c2a42bd2..98ef39321c 100644 --- a/pkg/model/openstackmodel/servergroup_test.go +++ b/pkg/model/openstackmodel/servergroup_test.go @@ -515,6 +515,166 @@ func getServerGroupModelBuilderTestInput() []serverGroupModelBuilderTestInput { }, }, }, + { + desc: "multizone setup 3 masters 3 nodes without bastion with API loadbalancer dns none", + cluster: &kops.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster", + }, + Spec: kops.ClusterSpec{ + API: kops.APISpec{ + LoadBalancer: &kops.LoadBalancerAccessSpec{ + Type: kops.LoadBalancerTypePublic, + }, + }, + CloudProvider: kops.CloudProviderSpec{ + Openstack: &kops.OpenstackSpec{ + BlockStorage: &kops.OpenstackBlockStorageConfig{ + Version: fi.PtrTo("v3"), + IgnoreAZ: fi.PtrTo(false), + CreateStorageClass: fi.PtrTo(false), + CSITopologySupport: fi.PtrTo(true), + }, + Loadbalancer: &kops.OpenstackLoadbalancerConfig{ + FloatingNetwork: fi.PtrTo("test"), + FloatingSubnet: fi.PtrTo("test-lb-subnet"), + Method: fi.PtrTo("ROUND_ROBIN"), + Provider: fi.PtrTo("amphora"), + UseOctavia: fi.PtrTo(true), + }, + Monitor: &kops.OpenstackMonitor{ + Delay: fi.PtrTo("1m"), + MaxRetries: fi.PtrTo(3), + Timeout: fi.PtrTo("30s"), + }, + Network: &kops.OpenstackNetwork{ + AvailabilityZoneHints: []*string{fi.PtrTo("zone-1"), fi.PtrTo("zone-2"), fi.PtrTo("zone-3")}, + }, + Router: &kops.OpenstackRouter{ + DNSServers: fi.PtrTo("8.8.8.8,8.8.4.4"), + ExternalSubnet: fi.PtrTo("test-router-subnet"), + ExternalNetwork: fi.PtrTo("test"), + AvailabilityZoneHints: []*string{fi.PtrTo("ha-zone")}, + }, + Metadata: &kops.OpenstackMetadata{ + ConfigDrive: fi.PtrTo(false), + }, + }, + }, + KubernetesVersion: "1.25.0", + Networking: kops.NetworkingSpec{ + Subnets: []kops.ClusterSubnetSpec{ + { + Name: "subnet-1", + Zone: "zone-1", + Type: kops.SubnetTypePrivate, + }, + { + Name: "subnet-2", + Zone: "zone-2", + Type: kops.SubnetTypePrivate, + }, + { + Name: "subnet-3", + Zone: "zone-3", + Type: kops.SubnetTypePrivate, + }, + }, + Topology: &kops.TopologySpec{ + ControlPlane: kops.TopologyPrivate, + DNS: kops.DNSTypeNone, + Nodes: kops.TopologyPrivate, + }, + }, + }, + }, + instanceGroups: []*kops.InstanceGroup{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "master-a", + }, + Spec: kops.InstanceGroupSpec{ + Role: kops.InstanceGroupRoleControlPlane, + Image: "image", + MinSize: i32(1), + MaxSize: i32(1), + MachineType: "blc.1-2", + Subnets: []string{"subnet-1"}, + Zones: []string{"zone-1"}, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "node-a", + }, + Spec: kops.InstanceGroupSpec{ + Role: kops.InstanceGroupRoleNode, + Image: "image", + MinSize: i32(1), + MaxSize: i32(1), + MachineType: "blc.1-2", + Subnets: []string{"subnet-1"}, + Zones: []string{"zone-1"}, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "master-b", + }, + Spec: kops.InstanceGroupSpec{ + Role: kops.InstanceGroupRoleControlPlane, + Image: "image", + MinSize: i32(1), + MaxSize: i32(1), + MachineType: "blc.1-2", + Subnets: []string{"subnet-2"}, + Zones: []string{"zone-2"}, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "node-b", + }, + Spec: kops.InstanceGroupSpec{ + Role: kops.InstanceGroupRoleNode, + Image: "image", + MinSize: i32(1), + MaxSize: i32(1), + MachineType: "blc.1-2", + Subnets: []string{"subnet-2"}, + Zones: []string{"zone-2"}, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "master-c", + }, + Spec: kops.InstanceGroupSpec{ + Role: kops.InstanceGroupRoleControlPlane, + Image: "image", + MinSize: i32(1), + MaxSize: i32(1), + MachineType: "blc.1-2", + Subnets: []string{"subnet-3"}, + Zones: []string{"zone-3"}, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "node-c", + }, + Spec: kops.InstanceGroupSpec{ + Role: kops.InstanceGroupRoleNode, + Image: "image", + MinSize: i32(1), + MaxSize: i32(1), + MachineType: "blc.1-2", + Subnets: []string{"subnet-3"}, + Zones: []string{"zone-3"}, + }, + }, + }, + }, { desc: "multizone setup 3 masters 3 nodes without external router", cluster: &kops.Cluster{ diff --git a/pkg/model/openstackmodel/tests/servergroup/adds-additional-security-groups.yaml b/pkg/model/openstackmodel/tests/servergroup/adds-additional-security-groups.yaml index 5f95c883b6..0022056319 100644 --- a/pkg/model/openstackmodel/tests/servergroup/adds-additional-security-groups.yaml +++ b/pkg/model/openstackmodel/tests/servergroup/adds-additional-security-groups.yaml @@ -26,6 +26,7 @@ Name: node-1-cluster Port: AdditionalSecurityGroups: - additional-sg + ForAPIServer: false ID: null InstanceGroupName: node Lifecycle: Sync @@ -192,6 +193,7 @@ PublicACL: null --- AdditionalSecurityGroups: - additional-sg +ForAPIServer: false ID: null InstanceGroupName: node Lifecycle: Sync diff --git a/pkg/model/openstackmodel/tests/servergroup/adds-cloud-labels-from-ClusterSpec.yaml b/pkg/model/openstackmodel/tests/servergroup/adds-cloud-labels-from-ClusterSpec.yaml index 9af49af1a8..fcdab9b3a4 100644 --- a/pkg/model/openstackmodel/tests/servergroup/adds-cloud-labels-from-ClusterSpec.yaml +++ b/pkg/model/openstackmodel/tests/servergroup/adds-cloud-labels-from-ClusterSpec.yaml @@ -26,6 +26,7 @@ Metadata: Name: node-1-cluster Port: AdditionalSecurityGroups: null + ForAPIServer: false ID: null InstanceGroupName: node Lifecycle: Sync @@ -190,6 +191,7 @@ Name: nodeupconfig-node PublicACL: null --- AdditionalSecurityGroups: null +ForAPIServer: false ID: null InstanceGroupName: node Lifecycle: Sync diff --git a/pkg/model/openstackmodel/tests/servergroup/adds-cloud-labels-from-InstanceGroupSpec.yaml b/pkg/model/openstackmodel/tests/servergroup/adds-cloud-labels-from-InstanceGroupSpec.yaml index 9af49af1a8..fcdab9b3a4 100644 --- a/pkg/model/openstackmodel/tests/servergroup/adds-cloud-labels-from-InstanceGroupSpec.yaml +++ b/pkg/model/openstackmodel/tests/servergroup/adds-cloud-labels-from-InstanceGroupSpec.yaml @@ -26,6 +26,7 @@ Metadata: Name: node-1-cluster Port: AdditionalSecurityGroups: null + ForAPIServer: false ID: null InstanceGroupName: node Lifecycle: Sync @@ -190,6 +191,7 @@ Name: nodeupconfig-node PublicACL: null --- AdditionalSecurityGroups: null +ForAPIServer: false ID: null InstanceGroupName: node Lifecycle: Sync diff --git a/pkg/model/openstackmodel/tests/servergroup/configures-server-group-affinity-with-annotations.yaml b/pkg/model/openstackmodel/tests/servergroup/configures-server-group-affinity-with-annotations.yaml index 1f47745732..4444119995 100644 --- a/pkg/model/openstackmodel/tests/servergroup/configures-server-group-affinity-with-annotations.yaml +++ b/pkg/model/openstackmodel/tests/servergroup/configures-server-group-affinity-with-annotations.yaml @@ -25,6 +25,7 @@ Metadata: Name: node-1-cluster Port: AdditionalSecurityGroups: null + ForAPIServer: false ID: null InstanceGroupName: node Lifecycle: Sync @@ -189,6 +190,7 @@ Name: nodeupconfig-node PublicACL: null --- AdditionalSecurityGroups: null +ForAPIServer: false ID: null InstanceGroupName: node Lifecycle: Sync diff --git a/pkg/model/openstackmodel/tests/servergroup/multizone-setup-3-masters-3-nodes-without-bastion-auto-zone-distribution.yaml b/pkg/model/openstackmodel/tests/servergroup/multizone-setup-3-masters-3-nodes-without-bastion-auto-zone-distribution.yaml index 14681712a1..424a79703a 100644 --- a/pkg/model/openstackmodel/tests/servergroup/multizone-setup-3-masters-3-nodes-without-bastion-auto-zone-distribution.yaml +++ b/pkg/model/openstackmodel/tests/servergroup/multizone-setup-3-masters-3-nodes-without-bastion-auto-zone-distribution.yaml @@ -79,6 +79,7 @@ Metadata: Name: master-1-cluster Port: AdditionalSecurityGroups: null + ForAPIServer: false ID: null InstanceGroupName: master Lifecycle: Sync @@ -165,6 +166,7 @@ Metadata: Name: master-2-cluster Port: AdditionalSecurityGroups: null + ForAPIServer: false ID: null InstanceGroupName: master Lifecycle: Sync @@ -251,6 +253,7 @@ Metadata: Name: master-3-cluster Port: AdditionalSecurityGroups: null + ForAPIServer: false ID: null InstanceGroupName: master Lifecycle: Sync @@ -334,6 +337,7 @@ Metadata: Name: node-1-cluster Port: AdditionalSecurityGroups: null + ForAPIServer: false ID: null InstanceGroupName: node Lifecycle: Sync @@ -411,6 +415,7 @@ Metadata: Name: node-2-cluster Port: AdditionalSecurityGroups: null + ForAPIServer: false ID: null InstanceGroupName: node Lifecycle: Sync @@ -488,6 +493,7 @@ Metadata: Name: node-3-cluster Port: AdditionalSecurityGroups: null + ForAPIServer: false ID: null InstanceGroupName: node Lifecycle: Sync @@ -662,6 +668,7 @@ Name: nodeupconfig-node PublicACL: null --- AdditionalSecurityGroups: null +ForAPIServer: false ID: null InstanceGroupName: master Lifecycle: Sync @@ -699,6 +706,7 @@ Tags: - KubernetesCluster=cluster --- AdditionalSecurityGroups: null +ForAPIServer: false ID: null InstanceGroupName: master Lifecycle: Sync @@ -736,6 +744,7 @@ Tags: - KubernetesCluster=cluster --- AdditionalSecurityGroups: null +ForAPIServer: false ID: null InstanceGroupName: master Lifecycle: Sync @@ -773,6 +782,7 @@ Tags: - KubernetesCluster=cluster --- AdditionalSecurityGroups: null +ForAPIServer: false ID: null InstanceGroupName: node Lifecycle: Sync @@ -804,6 +814,7 @@ Tags: - KubernetesCluster=cluster --- AdditionalSecurityGroups: null +ForAPIServer: false ID: null InstanceGroupName: node Lifecycle: Sync @@ -835,6 +846,7 @@ Tags: - KubernetesCluster=cluster --- AdditionalSecurityGroups: null +ForAPIServer: false ID: null InstanceGroupName: node Lifecycle: Sync diff --git a/pkg/model/openstackmodel/tests/servergroup/multizone-setup-3-masters-3-nodes-without-bastion-with-API-loadbalancer-dns-none.yaml b/pkg/model/openstackmodel/tests/servergroup/multizone-setup-3-masters-3-nodes-without-bastion-with-API-loadbalancer-dns-none.yaml new file mode 100644 index 0000000000..e174d833cb --- /dev/null +++ b/pkg/model/openstackmodel/tests/servergroup/multizone-setup-3-masters-3-nodes-without-bastion-with-API-loadbalancer-dns-none.yaml @@ -0,0 +1,1078 @@ +Lifecycle: "" +Name: master-a +--- +Lifecycle: "" +Name: master-b +--- +Lifecycle: "" +Name: master-c +--- +Lifecycle: "" +Name: node-a +--- +Lifecycle: "" +Name: node-b +--- +Lifecycle: "" +Name: node-c +--- +ForAPIServer: true +ID: null +IP: null +LB: + ID: null + Lifecycle: Sync + Name: api.cluster + PortID: null + Provider: null + SecurityGroup: + Description: null + ID: null + Lifecycle: "" + Name: api.cluster + RemoveExtraRules: null + RemoveGroup: false + Subnet: subnet-1.cluster + VipSubnet: null +Lifecycle: Sync +Name: fip-api.cluster +--- +AvailabilityZone: zone-1 +ConfigDrive: false +Flavor: blc.1-2 +FloatingIP: null +ForAPIServer: false +GroupName: master-a +ID: null +Image: image +Lifecycle: Sync +Metadata: + KopsInstanceGroup: master-a + KopsName: master-a-1-cluster + KopsNetwork: cluster + KopsRole: ControlPlane + KubernetesCluster: cluster + cluster_generation: "0" + ig_generation: "0" + k8s: cluster + k8s.io_cluster-autoscaler_node-template_label_kops.k8s.io_kops-controller-pki: "" + k8s.io_cluster-autoscaler_node-template_label_node-role.kubernetes.io_control-plane: "" + k8s.io_cluster-autoscaler_node-template_label_node.kubernetes.io_exclude-from-external-load-balancers: "" + k8s.io_role_control-plane: "1" + k8s.io_role_master: "1" + kops.k8s.io_instancegroup: master-a +Name: master-a-1-cluster +Port: + AdditionalSecurityGroups: null + ForAPIServer: true + ID: null + InstanceGroupName: master-a + Lifecycle: Sync + Name: port-master-a-1-cluster + Network: + AvailabilityZoneHints: null + ID: null + Lifecycle: "" + Name: cluster + Tag: null + SecurityGroups: + - Description: null + ID: null + Lifecycle: "" + Name: masters.cluster + RemoveExtraRules: null + RemoveGroup: false + Subnets: + - CIDR: null + DNSServers: null + ID: null + Lifecycle: "" + Name: subnet-1.cluster + Network: null + Tag: null + Tags: + - KopsInstanceGroup=master-a + - KopsName=port-master-a-1 + - KubernetesCluster=cluster +Region: "" +Role: ControlPlane +SSHKey: kubernetes.cluster-ba_d8_85_a0_5b_50_b0_01_e0_b2_b0_ae_5d_f6_7a_d1 +SecurityGroups: null +ServerGroup: + ClusterName: cluster + ID: null + IGName: master-a + Lifecycle: Sync + MaxSize: 1 + Name: cluster-master-a + Policies: + - anti-affinity +UserData: + task: + Lifecycle: "" + Name: master-a +--- +AvailabilityZone: zone-2 +ConfigDrive: false +Flavor: blc.1-2 +FloatingIP: null +ForAPIServer: false +GroupName: master-b +ID: null +Image: image +Lifecycle: Sync +Metadata: + KopsInstanceGroup: master-b + KopsName: master-b-1-cluster + KopsNetwork: cluster + KopsRole: ControlPlane + KubernetesCluster: cluster + cluster_generation: "0" + ig_generation: "0" + k8s: cluster + k8s.io_cluster-autoscaler_node-template_label_kops.k8s.io_kops-controller-pki: "" + k8s.io_cluster-autoscaler_node-template_label_node-role.kubernetes.io_control-plane: "" + k8s.io_cluster-autoscaler_node-template_label_node.kubernetes.io_exclude-from-external-load-balancers: "" + k8s.io_role_control-plane: "1" + k8s.io_role_master: "1" + kops.k8s.io_instancegroup: master-b +Name: master-b-1-cluster +Port: + AdditionalSecurityGroups: null + ForAPIServer: true + ID: null + InstanceGroupName: master-b + Lifecycle: Sync + Name: port-master-b-1-cluster + Network: + AvailabilityZoneHints: null + ID: null + Lifecycle: "" + Name: cluster + Tag: null + SecurityGroups: + - Description: null + ID: null + Lifecycle: "" + Name: masters.cluster + RemoveExtraRules: null + RemoveGroup: false + Subnets: + - CIDR: null + DNSServers: null + ID: null + Lifecycle: "" + Name: subnet-2.cluster + Network: null + Tag: null + Tags: + - KopsInstanceGroup=master-b + - KopsName=port-master-b-1 + - KubernetesCluster=cluster +Region: "" +Role: ControlPlane +SSHKey: kubernetes.cluster-ba_d8_85_a0_5b_50_b0_01_e0_b2_b0_ae_5d_f6_7a_d1 +SecurityGroups: null +ServerGroup: + ClusterName: cluster + ID: null + IGName: master-b + Lifecycle: Sync + MaxSize: 1 + Name: cluster-master-b + Policies: + - anti-affinity +UserData: + task: + Lifecycle: "" + Name: master-b +--- +AvailabilityZone: zone-3 +ConfigDrive: false +Flavor: blc.1-2 +FloatingIP: null +ForAPIServer: false +GroupName: master-c +ID: null +Image: image +Lifecycle: Sync +Metadata: + KopsInstanceGroup: master-c + KopsName: master-c-1-cluster + KopsNetwork: cluster + KopsRole: ControlPlane + KubernetesCluster: cluster + cluster_generation: "0" + ig_generation: "0" + k8s: cluster + k8s.io_cluster-autoscaler_node-template_label_kops.k8s.io_kops-controller-pki: "" + k8s.io_cluster-autoscaler_node-template_label_node-role.kubernetes.io_control-plane: "" + k8s.io_cluster-autoscaler_node-template_label_node.kubernetes.io_exclude-from-external-load-balancers: "" + k8s.io_role_control-plane: "1" + k8s.io_role_master: "1" + kops.k8s.io_instancegroup: master-c +Name: master-c-1-cluster +Port: + AdditionalSecurityGroups: null + ForAPIServer: true + ID: null + InstanceGroupName: master-c + Lifecycle: Sync + Name: port-master-c-1-cluster + Network: + AvailabilityZoneHints: null + ID: null + Lifecycle: "" + Name: cluster + Tag: null + SecurityGroups: + - Description: null + ID: null + Lifecycle: "" + Name: masters.cluster + RemoveExtraRules: null + RemoveGroup: false + Subnets: + - CIDR: null + DNSServers: null + ID: null + Lifecycle: "" + Name: subnet-3.cluster + Network: null + Tag: null + Tags: + - KopsInstanceGroup=master-c + - KopsName=port-master-c-1 + - KubernetesCluster=cluster +Region: "" +Role: ControlPlane +SSHKey: kubernetes.cluster-ba_d8_85_a0_5b_50_b0_01_e0_b2_b0_ae_5d_f6_7a_d1 +SecurityGroups: null +ServerGroup: + ClusterName: cluster + ID: null + IGName: master-c + Lifecycle: Sync + MaxSize: 1 + Name: cluster-master-c + Policies: + - anti-affinity +UserData: + task: + Lifecycle: "" + Name: master-c +--- +AvailabilityZone: zone-1 +ConfigDrive: false +Flavor: blc.1-2 +FloatingIP: null +ForAPIServer: false +GroupName: node-a +ID: null +Image: image +Lifecycle: Sync +Metadata: + KopsInstanceGroup: node-a + KopsName: node-a-1-cluster + KopsNetwork: cluster + KopsRole: Node + KubernetesCluster: cluster + cluster_generation: "0" + ig_generation: "0" + k8s: cluster + k8s.io_cluster-autoscaler_node-template_label_node-role.kubernetes.io_node: "" + k8s.io_role_node: "1" + kops.k8s.io_instancegroup: node-a +Name: node-a-1-cluster +Port: + AdditionalSecurityGroups: null + ForAPIServer: false + ID: null + InstanceGroupName: node-a + Lifecycle: Sync + Name: port-node-a-1-cluster + Network: + AvailabilityZoneHints: null + ID: null + Lifecycle: "" + Name: cluster + Tag: null + SecurityGroups: + - Description: null + ID: null + Lifecycle: "" + Name: nodes.cluster + RemoveExtraRules: null + RemoveGroup: false + Subnets: + - CIDR: null + DNSServers: null + ID: null + Lifecycle: "" + Name: subnet-1.cluster + Network: null + Tag: null + Tags: + - KopsInstanceGroup=node-a + - KopsName=port-node-a-1 + - KubernetesCluster=cluster +Region: "" +Role: Node +SSHKey: kubernetes.cluster-ba_d8_85_a0_5b_50_b0_01_e0_b2_b0_ae_5d_f6_7a_d1 +SecurityGroups: null +ServerGroup: + ClusterName: cluster + ID: null + IGName: node-a + Lifecycle: Sync + MaxSize: 1 + Name: cluster-node-a + Policies: + - anti-affinity +UserData: + task: + Lifecycle: "" + Name: node-a +--- +AvailabilityZone: zone-2 +ConfigDrive: false +Flavor: blc.1-2 +FloatingIP: null +ForAPIServer: false +GroupName: node-b +ID: null +Image: image +Lifecycle: Sync +Metadata: + KopsInstanceGroup: node-b + KopsName: node-b-1-cluster + KopsNetwork: cluster + KopsRole: Node + KubernetesCluster: cluster + cluster_generation: "0" + ig_generation: "0" + k8s: cluster + k8s.io_cluster-autoscaler_node-template_label_node-role.kubernetes.io_node: "" + k8s.io_role_node: "1" + kops.k8s.io_instancegroup: node-b +Name: node-b-1-cluster +Port: + AdditionalSecurityGroups: null + ForAPIServer: false + ID: null + InstanceGroupName: node-b + Lifecycle: Sync + Name: port-node-b-1-cluster + Network: + AvailabilityZoneHints: null + ID: null + Lifecycle: "" + Name: cluster + Tag: null + SecurityGroups: + - Description: null + ID: null + Lifecycle: "" + Name: nodes.cluster + RemoveExtraRules: null + RemoveGroup: false + Subnets: + - CIDR: null + DNSServers: null + ID: null + Lifecycle: "" + Name: subnet-2.cluster + Network: null + Tag: null + Tags: + - KopsInstanceGroup=node-b + - KopsName=port-node-b-1 + - KubernetesCluster=cluster +Region: "" +Role: Node +SSHKey: kubernetes.cluster-ba_d8_85_a0_5b_50_b0_01_e0_b2_b0_ae_5d_f6_7a_d1 +SecurityGroups: null +ServerGroup: + ClusterName: cluster + ID: null + IGName: node-b + Lifecycle: Sync + MaxSize: 1 + Name: cluster-node-b + Policies: + - anti-affinity +UserData: + task: + Lifecycle: "" + Name: node-b +--- +AvailabilityZone: zone-3 +ConfigDrive: false +Flavor: blc.1-2 +FloatingIP: null +ForAPIServer: false +GroupName: node-c +ID: null +Image: image +Lifecycle: Sync +Metadata: + KopsInstanceGroup: node-c + KopsName: node-c-1-cluster + KopsNetwork: cluster + KopsRole: Node + KubernetesCluster: cluster + cluster_generation: "0" + ig_generation: "0" + k8s: cluster + k8s.io_cluster-autoscaler_node-template_label_node-role.kubernetes.io_node: "" + k8s.io_role_node: "1" + kops.k8s.io_instancegroup: node-c +Name: node-c-1-cluster +Port: + AdditionalSecurityGroups: null + ForAPIServer: false + ID: null + InstanceGroupName: node-c + Lifecycle: Sync + Name: port-node-c-1-cluster + Network: + AvailabilityZoneHints: null + ID: null + Lifecycle: "" + Name: cluster + Tag: null + SecurityGroups: + - Description: null + ID: null + Lifecycle: "" + Name: nodes.cluster + RemoveExtraRules: null + RemoveGroup: false + Subnets: + - CIDR: null + DNSServers: null + ID: null + Lifecycle: "" + Name: subnet-3.cluster + Network: null + Tag: null + Tags: + - KopsInstanceGroup=node-c + - KopsName=port-node-c-1 + - KubernetesCluster=cluster +Region: "" +Role: Node +SSHKey: kubernetes.cluster-ba_d8_85_a0_5b_50_b0_01_e0_b2_b0_ae_5d_f6_7a_d1 +SecurityGroups: null +ServerGroup: + ClusterName: cluster + ID: null + IGName: node-c + Lifecycle: Sync + MaxSize: 1 + Name: cluster-node-c + Policies: + - anti-affinity +UserData: + task: + Lifecycle: "" + Name: node-c +--- +Lifecycle: "" +Name: apiserver-aggregator-ca +Signer: null +alternateNames: null +issuer: "" +oldFormat: false +subject: cn=apiserver-aggregator-ca +type: ca +--- +Lifecycle: "" +Name: etcd-clients-ca +Signer: null +alternateNames: null +issuer: "" +oldFormat: false +subject: cn=etcd-clients-ca +type: ca +--- +Lifecycle: "" +Name: etcd-manager-ca-events +Signer: null +alternateNames: null +issuer: "" +oldFormat: false +subject: cn=etcd-manager-ca-events +type: ca +--- +Lifecycle: "" +Name: etcd-manager-ca-main +Signer: null +alternateNames: null +issuer: "" +oldFormat: false +subject: cn=etcd-manager-ca-main +type: ca +--- +Lifecycle: "" +Name: etcd-peers-ca-events +Signer: null +alternateNames: null +issuer: "" +oldFormat: false +subject: cn=etcd-peers-ca-events +type: ca +--- +Lifecycle: "" +Name: etcd-peers-ca-main +Signer: null +alternateNames: null +issuer: "" +oldFormat: false +subject: cn=etcd-peers-ca-main +type: ca +--- +Lifecycle: "" +Name: kube-proxy +Signer: + Lifecycle: "" + Name: kubernetes-ca + Signer: null + alternateNames: null + issuer: "" + oldFormat: false + subject: cn=kubernetes + type: ca +alternateNames: null +issuer: "" +oldFormat: false +subject: cn=kube-proxy +type: client +--- +Lifecycle: "" +Name: kubelet +Signer: + Lifecycle: "" + Name: kubernetes-ca + Signer: null + alternateNames: null + issuer: "" + oldFormat: false + subject: cn=kubernetes + type: ca +alternateNames: null +issuer: "" +oldFormat: false +subject: cn=kubelet +type: client +--- +Lifecycle: "" +Name: kubernetes-ca +Signer: null +alternateNames: null +issuer: "" +oldFormat: false +subject: cn=kubernetes +type: ca +--- +Lifecycle: "" +Name: service-account +Signer: null +alternateNames: null +issuer: "" +oldFormat: false +subject: cn=service-account +type: ca +--- +ID: null +Lifecycle: Sync +Name: api.cluster +PortID: null +Provider: null +SecurityGroup: + Description: null + ID: null + Lifecycle: "" + Name: api.cluster + RemoveExtraRules: null + RemoveGroup: false +Subnet: subnet-1.cluster +VipSubnet: null +--- +AllowedCIDRs: null +ID: null +Lifecycle: Sync +Name: api.cluster +Pool: + ID: null + Lifecycle: Sync + Loadbalancer: + ID: null + Lifecycle: Sync + Name: api.cluster + PortID: null + Provider: null + SecurityGroup: + Description: null + ID: null + Lifecycle: "" + Name: api.cluster + RemoveExtraRules: null + RemoveGroup: false + Subnet: subnet-1.cluster + VipSubnet: null + Name: api.cluster-https +Port: 443 +--- +ID: null +Lifecycle: Sync +Loadbalancer: + ID: null + Lifecycle: Sync + Name: api.cluster + PortID: null + Provider: null + SecurityGroup: + Description: null + ID: null + Lifecycle: "" + Name: api.cluster + RemoveExtraRules: null + RemoveGroup: false + Subnet: subnet-1.cluster + VipSubnet: null +Name: api.cluster-https +--- +Base: null +Contents: + task: + Lifecycle: "" + Name: master-a +Lifecycle: "" +Location: igconfig/control-plane/master-a/nodeupconfig.yaml +Name: nodeupconfig-master-a +PublicACL: null +--- +Base: null +Contents: + task: + Lifecycle: "" + Name: master-b +Lifecycle: "" +Location: igconfig/control-plane/master-b/nodeupconfig.yaml +Name: nodeupconfig-master-b +PublicACL: null +--- +Base: null +Contents: + task: + Lifecycle: "" + Name: master-c +Lifecycle: "" +Location: igconfig/control-plane/master-c/nodeupconfig.yaml +Name: nodeupconfig-master-c +PublicACL: null +--- +Base: null +Contents: + task: + Lifecycle: "" + Name: node-a +Lifecycle: "" +Location: igconfig/node/node-a/nodeupconfig.yaml +Name: nodeupconfig-node-a +PublicACL: null +--- +Base: null +Contents: + task: + Lifecycle: "" + Name: node-b +Lifecycle: "" +Location: igconfig/node/node-b/nodeupconfig.yaml +Name: nodeupconfig-node-b +PublicACL: null +--- +Base: null +Contents: + task: + Lifecycle: "" + Name: node-c +Lifecycle: "" +Location: igconfig/node/node-c/nodeupconfig.yaml +Name: nodeupconfig-node-c +PublicACL: null +--- +ID: null +InterfaceName: cluster +Lifecycle: Sync +Name: cluster-master-a +Pool: + ID: null + Lifecycle: Sync + Loadbalancer: + ID: null + Lifecycle: Sync + Name: api.cluster + PortID: null + Provider: null + SecurityGroup: + Description: null + ID: null + Lifecycle: "" + Name: api.cluster + RemoveExtraRules: null + RemoveGroup: false + Subnet: subnet-1.cluster + VipSubnet: null + Name: api.cluster-https +ProtocolPort: 443 +ServerGroup: + ClusterName: cluster + ID: null + IGName: master-a + Lifecycle: Sync + MaxSize: 1 + Name: cluster-master-a + Policies: + - anti-affinity +Weight: 1 +--- +ID: null +InterfaceName: cluster +Lifecycle: Sync +Name: cluster-master-b +Pool: + ID: null + Lifecycle: Sync + Loadbalancer: + ID: null + Lifecycle: Sync + Name: api.cluster + PortID: null + Provider: null + SecurityGroup: + Description: null + ID: null + Lifecycle: "" + Name: api.cluster + RemoveExtraRules: null + RemoveGroup: false + Subnet: subnet-1.cluster + VipSubnet: null + Name: api.cluster-https +ProtocolPort: 443 +ServerGroup: + ClusterName: cluster + ID: null + IGName: master-b + Lifecycle: Sync + MaxSize: 1 + Name: cluster-master-b + Policies: + - anti-affinity +Weight: 1 +--- +ID: null +InterfaceName: cluster +Lifecycle: Sync +Name: cluster-master-c +Pool: + ID: null + Lifecycle: Sync + Loadbalancer: + ID: null + Lifecycle: Sync + Name: api.cluster + PortID: null + Provider: null + SecurityGroup: + Description: null + ID: null + Lifecycle: "" + Name: api.cluster + RemoveExtraRules: null + RemoveGroup: false + Subnet: subnet-1.cluster + VipSubnet: null + Name: api.cluster-https +ProtocolPort: 443 +ServerGroup: + ClusterName: cluster + ID: null + IGName: master-c + Lifecycle: Sync + MaxSize: 1 + Name: cluster-master-c + Policies: + - anti-affinity +Weight: 1 +--- +ID: null +Lifecycle: Sync +Name: api.cluster +Pool: + ID: null + Lifecycle: Sync + Loadbalancer: + ID: null + Lifecycle: Sync + Name: api.cluster + PortID: null + Provider: null + SecurityGroup: + Description: null + ID: null + Lifecycle: "" + Name: api.cluster + RemoveExtraRules: null + RemoveGroup: false + Subnet: subnet-1.cluster + VipSubnet: null + Name: api.cluster-https +--- +AdditionalSecurityGroups: null +ForAPIServer: true +ID: null +InstanceGroupName: master-a +Lifecycle: Sync +Name: port-master-a-1-cluster +Network: + AvailabilityZoneHints: null + ID: null + Lifecycle: "" + Name: cluster + Tag: null +SecurityGroups: +- Description: null + ID: null + Lifecycle: "" + Name: masters.cluster + RemoveExtraRules: null + RemoveGroup: false +Subnets: +- CIDR: null + DNSServers: null + ID: null + Lifecycle: "" + Name: subnet-1.cluster + Network: null + Tag: null +Tags: +- KopsInstanceGroup=master-a +- KopsName=port-master-a-1 +- KubernetesCluster=cluster +--- +AdditionalSecurityGroups: null +ForAPIServer: true +ID: null +InstanceGroupName: master-b +Lifecycle: Sync +Name: port-master-b-1-cluster +Network: + AvailabilityZoneHints: null + ID: null + Lifecycle: "" + Name: cluster + Tag: null +SecurityGroups: +- Description: null + ID: null + Lifecycle: "" + Name: masters.cluster + RemoveExtraRules: null + RemoveGroup: false +Subnets: +- CIDR: null + DNSServers: null + ID: null + Lifecycle: "" + Name: subnet-2.cluster + Network: null + Tag: null +Tags: +- KopsInstanceGroup=master-b +- KopsName=port-master-b-1 +- KubernetesCluster=cluster +--- +AdditionalSecurityGroups: null +ForAPIServer: true +ID: null +InstanceGroupName: master-c +Lifecycle: Sync +Name: port-master-c-1-cluster +Network: + AvailabilityZoneHints: null + ID: null + Lifecycle: "" + Name: cluster + Tag: null +SecurityGroups: +- Description: null + ID: null + Lifecycle: "" + Name: masters.cluster + RemoveExtraRules: null + RemoveGroup: false +Subnets: +- CIDR: null + DNSServers: null + ID: null + Lifecycle: "" + Name: subnet-3.cluster + Network: null + Tag: null +Tags: +- KopsInstanceGroup=master-c +- KopsName=port-master-c-1 +- KubernetesCluster=cluster +--- +AdditionalSecurityGroups: null +ForAPIServer: false +ID: null +InstanceGroupName: node-a +Lifecycle: Sync +Name: port-node-a-1-cluster +Network: + AvailabilityZoneHints: null + ID: null + Lifecycle: "" + Name: cluster + Tag: null +SecurityGroups: +- Description: null + ID: null + Lifecycle: "" + Name: nodes.cluster + RemoveExtraRules: null + RemoveGroup: false +Subnets: +- CIDR: null + DNSServers: null + ID: null + Lifecycle: "" + Name: subnet-1.cluster + Network: null + Tag: null +Tags: +- KopsInstanceGroup=node-a +- KopsName=port-node-a-1 +- KubernetesCluster=cluster +--- +AdditionalSecurityGroups: null +ForAPIServer: false +ID: null +InstanceGroupName: node-b +Lifecycle: Sync +Name: port-node-b-1-cluster +Network: + AvailabilityZoneHints: null + ID: null + Lifecycle: "" + Name: cluster + Tag: null +SecurityGroups: +- Description: null + ID: null + Lifecycle: "" + Name: nodes.cluster + RemoveExtraRules: null + RemoveGroup: false +Subnets: +- CIDR: null + DNSServers: null + ID: null + Lifecycle: "" + Name: subnet-2.cluster + Network: null + Tag: null +Tags: +- KopsInstanceGroup=node-b +- KopsName=port-node-b-1 +- KubernetesCluster=cluster +--- +AdditionalSecurityGroups: null +ForAPIServer: false +ID: null +InstanceGroupName: node-c +Lifecycle: Sync +Name: port-node-c-1-cluster +Network: + AvailabilityZoneHints: null + ID: null + Lifecycle: "" + Name: cluster + Tag: null +SecurityGroups: +- Description: null + ID: null + Lifecycle: "" + Name: nodes.cluster + RemoveExtraRules: null + RemoveGroup: false +Subnets: +- CIDR: null + DNSServers: null + ID: null + Lifecycle: "" + Name: subnet-3.cluster + Network: null + Tag: null +Tags: +- KopsInstanceGroup=node-c +- KopsName=port-node-c-1 +- KubernetesCluster=cluster +--- +ClusterName: cluster +ID: null +IGName: master-a +Lifecycle: Sync +MaxSize: 1 +Name: cluster-master-a +Policies: +- anti-affinity +--- +ClusterName: cluster +ID: null +IGName: master-b +Lifecycle: Sync +MaxSize: 1 +Name: cluster-master-b +Policies: +- anti-affinity +--- +ClusterName: cluster +ID: null +IGName: master-c +Lifecycle: Sync +MaxSize: 1 +Name: cluster-master-c +Policies: +- anti-affinity +--- +ClusterName: cluster +ID: null +IGName: node-a +Lifecycle: Sync +MaxSize: 1 +Name: cluster-node-a +Policies: +- anti-affinity +--- +ClusterName: cluster +ID: null +IGName: node-b +Lifecycle: Sync +MaxSize: 1 +Name: cluster-node-b +Policies: +- anti-affinity +--- +ClusterName: cluster +ID: null +IGName: node-c +Lifecycle: Sync +MaxSize: 1 +Name: cluster-node-c +Policies: +- anti-affinity diff --git a/pkg/model/openstackmodel/tests/servergroup/multizone-setup-3-masters-3-nodes-without-bastion-with-API-loadbalancer.yaml b/pkg/model/openstackmodel/tests/servergroup/multizone-setup-3-masters-3-nodes-without-bastion-with-API-loadbalancer.yaml index 426927f1f6..f70273724b 100644 --- a/pkg/model/openstackmodel/tests/servergroup/multizone-setup-3-masters-3-nodes-without-bastion-with-API-loadbalancer.yaml +++ b/pkg/model/openstackmodel/tests/servergroup/multizone-setup-3-masters-3-nodes-without-bastion-with-API-loadbalancer.yaml @@ -85,6 +85,7 @@ Metadata: Name: master-a-1-cluster Port: AdditionalSecurityGroups: null + ForAPIServer: false ID: null InstanceGroupName: master-a Lifecycle: Sync @@ -159,6 +160,7 @@ Metadata: Name: master-b-1-cluster Port: AdditionalSecurityGroups: null + ForAPIServer: false ID: null InstanceGroupName: master-b Lifecycle: Sync @@ -233,6 +235,7 @@ Metadata: Name: master-c-1-cluster Port: AdditionalSecurityGroups: null + ForAPIServer: false ID: null InstanceGroupName: master-c Lifecycle: Sync @@ -310,6 +313,7 @@ Metadata: Name: node-a-1-cluster Port: AdditionalSecurityGroups: null + ForAPIServer: false ID: null InstanceGroupName: node-a Lifecycle: Sync @@ -387,6 +391,7 @@ Metadata: Name: node-b-1-cluster Port: AdditionalSecurityGroups: null + ForAPIServer: false ID: null InstanceGroupName: node-b Lifecycle: Sync @@ -464,6 +469,7 @@ Metadata: Name: node-c-1-cluster Port: AdditionalSecurityGroups: null + ForAPIServer: false ID: null InstanceGroupName: node-c Lifecycle: Sync @@ -655,6 +661,7 @@ Pool: Subnet: subnet-a.cluster VipSubnet: null Name: master-public-name-https +Port: 443 --- ID: null Lifecycle: Sync @@ -864,6 +871,7 @@ Pool: Name: master-public-name-https --- AdditionalSecurityGroups: null +ForAPIServer: false ID: null InstanceGroupName: master-a Lifecycle: Sync @@ -895,6 +903,7 @@ Tags: - KubernetesCluster=cluster --- AdditionalSecurityGroups: null +ForAPIServer: false ID: null InstanceGroupName: master-b Lifecycle: Sync @@ -926,6 +935,7 @@ Tags: - KubernetesCluster=cluster --- AdditionalSecurityGroups: null +ForAPIServer: false ID: null InstanceGroupName: master-c Lifecycle: Sync @@ -957,6 +967,7 @@ Tags: - KubernetesCluster=cluster --- AdditionalSecurityGroups: null +ForAPIServer: false ID: null InstanceGroupName: node-a Lifecycle: Sync @@ -988,6 +999,7 @@ Tags: - KubernetesCluster=cluster --- AdditionalSecurityGroups: null +ForAPIServer: false ID: null InstanceGroupName: node-b Lifecycle: Sync @@ -1019,6 +1031,7 @@ Tags: - KubernetesCluster=cluster --- AdditionalSecurityGroups: null +ForAPIServer: false ID: null InstanceGroupName: node-c Lifecycle: Sync diff --git a/pkg/model/openstackmodel/tests/servergroup/multizone-setup-3-masters-3-nodes-without-bastion.yaml b/pkg/model/openstackmodel/tests/servergroup/multizone-setup-3-masters-3-nodes-without-bastion.yaml index 3615ddf28f..6d5abf4bd1 100644 --- a/pkg/model/openstackmodel/tests/servergroup/multizone-setup-3-masters-3-nodes-without-bastion.yaml +++ b/pkg/model/openstackmodel/tests/servergroup/multizone-setup-3-masters-3-nodes-without-bastion.yaml @@ -91,6 +91,7 @@ Metadata: Name: master-a-1-cluster Port: AdditionalSecurityGroups: null + ForAPIServer: false ID: null InstanceGroupName: master-a Lifecycle: Sync @@ -177,6 +178,7 @@ Metadata: Name: master-b-1-cluster Port: AdditionalSecurityGroups: null + ForAPIServer: false ID: null InstanceGroupName: master-b Lifecycle: Sync @@ -263,6 +265,7 @@ Metadata: Name: master-c-1-cluster Port: AdditionalSecurityGroups: null + ForAPIServer: false ID: null InstanceGroupName: master-c Lifecycle: Sync @@ -346,6 +349,7 @@ Metadata: Name: node-a-1-cluster Port: AdditionalSecurityGroups: null + ForAPIServer: false ID: null InstanceGroupName: node-a Lifecycle: Sync @@ -423,6 +427,7 @@ Metadata: Name: node-b-1-cluster Port: AdditionalSecurityGroups: null + ForAPIServer: false ID: null InstanceGroupName: node-b Lifecycle: Sync @@ -500,6 +505,7 @@ Metadata: Name: node-c-1-cluster Port: AdditionalSecurityGroups: null + ForAPIServer: false ID: null InstanceGroupName: node-c Lifecycle: Sync @@ -714,6 +720,7 @@ Name: nodeupconfig-node-c PublicACL: null --- AdditionalSecurityGroups: null +ForAPIServer: false ID: null InstanceGroupName: master-a Lifecycle: Sync @@ -751,6 +758,7 @@ Tags: - KubernetesCluster=cluster --- AdditionalSecurityGroups: null +ForAPIServer: false ID: null InstanceGroupName: master-b Lifecycle: Sync @@ -788,6 +796,7 @@ Tags: - KubernetesCluster=cluster --- AdditionalSecurityGroups: null +ForAPIServer: false ID: null InstanceGroupName: master-c Lifecycle: Sync @@ -825,6 +834,7 @@ Tags: - KubernetesCluster=cluster --- AdditionalSecurityGroups: null +ForAPIServer: false ID: null InstanceGroupName: node-a Lifecycle: Sync @@ -856,6 +866,7 @@ Tags: - KubernetesCluster=cluster --- AdditionalSecurityGroups: null +ForAPIServer: false ID: null InstanceGroupName: node-b Lifecycle: Sync @@ -887,6 +898,7 @@ Tags: - KubernetesCluster=cluster --- AdditionalSecurityGroups: null +ForAPIServer: false ID: null InstanceGroupName: node-c Lifecycle: Sync diff --git a/pkg/model/openstackmodel/tests/servergroup/multizone-setup-3-masters-3-nodes-without-external-router.yaml b/pkg/model/openstackmodel/tests/servergroup/multizone-setup-3-masters-3-nodes-without-external-router.yaml index 0c672bcb4d..641bb6e73f 100644 --- a/pkg/model/openstackmodel/tests/servergroup/multizone-setup-3-masters-3-nodes-without-external-router.yaml +++ b/pkg/model/openstackmodel/tests/servergroup/multizone-setup-3-masters-3-nodes-without-external-router.yaml @@ -43,6 +43,7 @@ Metadata: Name: master-a-1-cluster Port: AdditionalSecurityGroups: null + ForAPIServer: false ID: null InstanceGroupName: master-a Lifecycle: Sync @@ -123,6 +124,7 @@ Metadata: Name: master-b-1-cluster Port: AdditionalSecurityGroups: null + ForAPIServer: false ID: null InstanceGroupName: master-b Lifecycle: Sync @@ -203,6 +205,7 @@ Metadata: Name: master-c-1-cluster Port: AdditionalSecurityGroups: null + ForAPIServer: false ID: null InstanceGroupName: master-c Lifecycle: Sync @@ -280,6 +283,7 @@ Metadata: Name: node-a-1-cluster Port: AdditionalSecurityGroups: null + ForAPIServer: false ID: null InstanceGroupName: node-a Lifecycle: Sync @@ -351,6 +355,7 @@ Metadata: Name: node-b-1-cluster Port: AdditionalSecurityGroups: null + ForAPIServer: false ID: null InstanceGroupName: node-b Lifecycle: Sync @@ -422,6 +427,7 @@ Metadata: Name: node-c-1-cluster Port: AdditionalSecurityGroups: null + ForAPIServer: false ID: null InstanceGroupName: node-c Lifecycle: Sync @@ -636,6 +642,7 @@ Name: nodeupconfig-node-c PublicACL: null --- AdditionalSecurityGroups: null +ForAPIServer: false ID: null InstanceGroupName: master-a Lifecycle: Sync @@ -673,6 +680,7 @@ Tags: - KubernetesCluster=cluster --- AdditionalSecurityGroups: null +ForAPIServer: false ID: null InstanceGroupName: master-b Lifecycle: Sync @@ -710,6 +718,7 @@ Tags: - KubernetesCluster=cluster --- AdditionalSecurityGroups: null +ForAPIServer: false ID: null InstanceGroupName: master-c Lifecycle: Sync @@ -747,6 +756,7 @@ Tags: - KubernetesCluster=cluster --- AdditionalSecurityGroups: null +ForAPIServer: false ID: null InstanceGroupName: node-a Lifecycle: Sync @@ -778,6 +788,7 @@ Tags: - KubernetesCluster=cluster --- AdditionalSecurityGroups: null +ForAPIServer: false ID: null InstanceGroupName: node-b Lifecycle: Sync @@ -809,6 +820,7 @@ Tags: - KubernetesCluster=cluster --- AdditionalSecurityGroups: null +ForAPIServer: false ID: null InstanceGroupName: node-c Lifecycle: Sync diff --git a/pkg/model/openstackmodel/tests/servergroup/one-master-one-node-one-bastion-2.yaml b/pkg/model/openstackmodel/tests/servergroup/one-master-one-node-one-bastion-2.yaml index be3eb4c54a..758d92b5e0 100644 --- a/pkg/model/openstackmodel/tests/servergroup/one-master-one-node-one-bastion-2.yaml +++ b/pkg/model/openstackmodel/tests/servergroup/one-master-one-node-one-bastion-2.yaml @@ -30,6 +30,7 @@ Metadata: Name: bastion-1-cluster Port: AdditionalSecurityGroups: null + ForAPIServer: false ID: null InstanceGroupName: bastion Lifecycle: Sync @@ -104,6 +105,7 @@ Metadata: Name: master-1-cluster Port: AdditionalSecurityGroups: null + ForAPIServer: false ID: null InstanceGroupName: master Lifecycle: Sync @@ -181,6 +183,7 @@ Metadata: Name: node-1-cluster Port: AdditionalSecurityGroups: null + ForAPIServer: false ID: null InstanceGroupName: node Lifecycle: Sync @@ -365,6 +368,7 @@ Name: nodeupconfig-node PublicACL: null --- AdditionalSecurityGroups: null +ForAPIServer: false ID: null InstanceGroupName: bastion Lifecycle: Sync @@ -396,6 +400,7 @@ Tags: - KubernetesCluster=cluster --- AdditionalSecurityGroups: null +ForAPIServer: false ID: null InstanceGroupName: master Lifecycle: Sync @@ -433,6 +438,7 @@ Tags: - KubernetesCluster=cluster --- AdditionalSecurityGroups: null +ForAPIServer: false ID: null InstanceGroupName: node Lifecycle: Sync diff --git a/pkg/model/openstackmodel/tests/servergroup/one-master-one-node-one-bastion.yaml b/pkg/model/openstackmodel/tests/servergroup/one-master-one-node-one-bastion.yaml index df849b466b..a6bcfdb179 100644 --- a/pkg/model/openstackmodel/tests/servergroup/one-master-one-node-one-bastion.yaml +++ b/pkg/model/openstackmodel/tests/servergroup/one-master-one-node-one-bastion.yaml @@ -50,6 +50,7 @@ Metadata: Name: bastion-1-cluster Port: AdditionalSecurityGroups: null + ForAPIServer: false ID: null InstanceGroupName: bastion Lifecycle: Sync @@ -130,6 +131,7 @@ Metadata: Name: master-1-cluster Port: AdditionalSecurityGroups: null + ForAPIServer: false ID: null InstanceGroupName: master Lifecycle: Sync @@ -207,6 +209,7 @@ Metadata: Name: node-1-cluster Port: AdditionalSecurityGroups: null + ForAPIServer: false ID: null InstanceGroupName: node Lifecycle: Sync @@ -391,6 +394,7 @@ Name: nodeupconfig-node PublicACL: null --- AdditionalSecurityGroups: null +ForAPIServer: false ID: null InstanceGroupName: bastion Lifecycle: Sync @@ -422,6 +426,7 @@ Tags: - KubernetesCluster=cluster --- AdditionalSecurityGroups: null +ForAPIServer: false ID: null InstanceGroupName: master Lifecycle: Sync @@ -459,6 +464,7 @@ Tags: - KubernetesCluster=cluster --- AdditionalSecurityGroups: null +ForAPIServer: false ID: null InstanceGroupName: node Lifecycle: Sync diff --git a/pkg/model/openstackmodel/tests/servergroup/one-master-one-node-without-bastion-no-public-ip-association.yaml b/pkg/model/openstackmodel/tests/servergroup/one-master-one-node-without-bastion-no-public-ip-association.yaml index a8b144b36e..74ea40089a 100644 --- a/pkg/model/openstackmodel/tests/servergroup/one-master-one-node-without-bastion-no-public-ip-association.yaml +++ b/pkg/model/openstackmodel/tests/servergroup/one-master-one-node-without-bastion-no-public-ip-association.yaml @@ -31,6 +31,7 @@ Metadata: Name: master-1-cluster Port: AdditionalSecurityGroups: null + ForAPIServer: false ID: null InstanceGroupName: master Lifecycle: Sync @@ -108,6 +109,7 @@ Metadata: Name: node-1-cluster Port: AdditionalSecurityGroups: null + ForAPIServer: false ID: null InstanceGroupName: node Lifecycle: Sync @@ -282,6 +284,7 @@ Name: nodeupconfig-node PublicACL: null --- AdditionalSecurityGroups: null +ForAPIServer: false ID: null InstanceGroupName: master Lifecycle: Sync @@ -319,6 +322,7 @@ Tags: - KubernetesCluster=cluster --- AdditionalSecurityGroups: null +ForAPIServer: false ID: null InstanceGroupName: node Lifecycle: Sync diff --git a/pkg/model/openstackmodel/tests/servergroup/one-master-one-node.yaml b/pkg/model/openstackmodel/tests/servergroup/one-master-one-node.yaml index 602b6e04e8..9f763ba473 100644 --- a/pkg/model/openstackmodel/tests/servergroup/one-master-one-node.yaml +++ b/pkg/model/openstackmodel/tests/servergroup/one-master-one-node.yaml @@ -51,6 +51,7 @@ Metadata: Name: master-1-cluster Port: AdditionalSecurityGroups: null + ForAPIServer: false ID: null InstanceGroupName: master Lifecycle: Sync @@ -134,6 +135,7 @@ Metadata: Name: node-1-cluster Port: AdditionalSecurityGroups: null + ForAPIServer: false ID: null InstanceGroupName: node Lifecycle: Sync @@ -308,6 +310,7 @@ Name: nodeupconfig-node PublicACL: null --- AdditionalSecurityGroups: null +ForAPIServer: false ID: null InstanceGroupName: master Lifecycle: Sync @@ -345,6 +348,7 @@ Tags: - KubernetesCluster=cluster --- AdditionalSecurityGroups: null +ForAPIServer: false ID: null InstanceGroupName: node Lifecycle: Sync diff --git a/pkg/model/openstackmodel/tests/servergroup/truncate-cluster-names-to-42-characters.yaml b/pkg/model/openstackmodel/tests/servergroup/truncate-cluster-names-to-42-characters.yaml index c4af89d696..56c93e7551 100644 --- a/pkg/model/openstackmodel/tests/servergroup/truncate-cluster-names-to-42-characters.yaml +++ b/pkg/model/openstackmodel/tests/servergroup/truncate-cluster-names-to-42-characters.yaml @@ -51,6 +51,7 @@ Metadata: Name: master-1-tom-software-dev-playground-real33-k8s-local Port: AdditionalSecurityGroups: null + ForAPIServer: false ID: null InstanceGroupName: master Lifecycle: Sync @@ -134,6 +135,7 @@ Metadata: Name: node-1-tom-software-dev-playground-real33-k8s-local Port: AdditionalSecurityGroups: null + ForAPIServer: false ID: null InstanceGroupName: node Lifecycle: Sync @@ -308,6 +310,7 @@ Name: nodeupconfig-node PublicACL: null --- AdditionalSecurityGroups: null +ForAPIServer: false ID: null InstanceGroupName: master Lifecycle: Sync @@ -345,6 +348,7 @@ Tags: - KubernetesCluster=tom-software-dev-playground-real33--kngu8l --- AdditionalSecurityGroups: null +ForAPIServer: false ID: null InstanceGroupName: node Lifecycle: Sync diff --git a/pkg/model/openstackmodel/tests/servergroup/uses-instance-group-subnet-as-availability-zones-fallback.yaml b/pkg/model/openstackmodel/tests/servergroup/uses-instance-group-subnet-as-availability-zones-fallback.yaml index 032881fa0b..705729f01f 100644 --- a/pkg/model/openstackmodel/tests/servergroup/uses-instance-group-subnet-as-availability-zones-fallback.yaml +++ b/pkg/model/openstackmodel/tests/servergroup/uses-instance-group-subnet-as-availability-zones-fallback.yaml @@ -26,6 +26,7 @@ Name: node-1-cluster Port: AdditionalSecurityGroups: - additional-sg + ForAPIServer: false ID: null InstanceGroupName: node Lifecycle: Sync @@ -192,6 +193,7 @@ PublicACL: null --- AdditionalSecurityGroups: - additional-sg +ForAPIServer: false ID: null InstanceGroupName: node Lifecycle: Sync diff --git a/pkg/model/openstackmodel/tests/servergroup/uses-instance-group-zones-as-availability-zones.yaml b/pkg/model/openstackmodel/tests/servergroup/uses-instance-group-zones-as-availability-zones.yaml index f8c1868472..11d637a0ee 100644 --- a/pkg/model/openstackmodel/tests/servergroup/uses-instance-group-zones-as-availability-zones.yaml +++ b/pkg/model/openstackmodel/tests/servergroup/uses-instance-group-zones-as-availability-zones.yaml @@ -26,6 +26,7 @@ Name: node-1-cluster Port: AdditionalSecurityGroups: - additional-sg + ForAPIServer: false ID: null InstanceGroupName: node Lifecycle: Sync @@ -192,6 +193,7 @@ PublicACL: null --- AdditionalSecurityGroups: - additional-sg +ForAPIServer: false ID: null InstanceGroupName: node Lifecycle: Sync diff --git a/tests/integration/create_cluster/ha_openstack_nodns/expected-v1alpha2.yaml b/tests/integration/create_cluster/ha_openstack_nodns/expected-v1alpha2.yaml new file mode 100644 index 0000000000..ec63513350 --- /dev/null +++ b/tests/integration/create_cluster/ha_openstack_nodns/expected-v1alpha2.yaml @@ -0,0 +1,156 @@ +apiVersion: kops.k8s.io/v1alpha2 +kind: Cluster +metadata: + creationTimestamp: "2017-01-01T00:00:00Z" + name: ha.example.com +spec: + api: + loadBalancer: + type: Public + authorization: + rbac: {} + channel: stable + cloudConfig: + openstack: + blockStorage: + bs-version: v3 + ignore-volume-az: false + loadbalancer: + floatingNetwork: vlan1 + floatingSubnet: vlan1lbsubnet + method: ROUND_ROBIN + provider: octavia + useOctavia: true + monitor: + delay: 15s + maxRetries: 3 + timeout: 10s + router: + dnsServers: 1.1.1.1 + externalNetwork: vlan1 + externalSubnet: vlan1subnet + cloudProvider: openstack + configBase: memfs://tests/ha.example.com + etcdClusters: + - cpuRequest: 200m + etcdMembers: + - instanceGroup: control-plane-us-test1-1 + name: etcd-1 + - instanceGroup: control-plane-us-test1-2 + name: etcd-2 + - instanceGroup: control-plane-us-test1-3 + name: etcd-3 + memoryRequest: 100Mi + name: main + - cpuRequest: 100m + etcdMembers: + - instanceGroup: control-plane-us-test1-1 + name: etcd-1 + - instanceGroup: control-plane-us-test1-2 + name: etcd-2 + - instanceGroup: control-plane-us-test1-3 + name: etcd-3 + memoryRequest: 100Mi + name: events + iam: + allowContainerRegistry: true + legacy: false + kubelet: + anonymousAuth: false + kubernetesApiAccess: + - 0.0.0.0/0 + - ::/0 + kubernetesVersion: v1.25.0 + networkCIDR: 10.0.0.0/16 + networking: + calico: {} + nonMasqueradeCIDR: 100.64.0.0/10 + sshAccess: + - 0.0.0.0/0 + - ::/0 + subnets: + - cidr: 10.0.32.0/19 + name: us-test1 + type: Private + zone: us-test1 + - cidr: 10.0.0.0/22 + name: utility-us-test1 + type: Utility + zone: us-test1 + topology: + dns: + type: None + masters: private + nodes: private + +--- + +apiVersion: kops.k8s.io/v1alpha2 +kind: InstanceGroup +metadata: + creationTimestamp: "2017-01-01T00:00:00Z" + labels: + kops.k8s.io/cluster: ha.example.com + name: control-plane-us-test1-1 +spec: + image: ubuntu-20.04 + machineType: m1.medium + maxSize: 1 + minSize: 1 + role: Master + subnets: + - us-test1 + +--- + +apiVersion: kops.k8s.io/v1alpha2 +kind: InstanceGroup +metadata: + creationTimestamp: "2017-01-01T00:00:00Z" + labels: + kops.k8s.io/cluster: ha.example.com + name: control-plane-us-test1-2 +spec: + image: ubuntu-20.04 + machineType: m1.medium + maxSize: 1 + minSize: 1 + role: Master + subnets: + - us-test1 + +--- + +apiVersion: kops.k8s.io/v1alpha2 +kind: InstanceGroup +metadata: + creationTimestamp: "2017-01-01T00:00:00Z" + labels: + kops.k8s.io/cluster: ha.example.com + name: control-plane-us-test1-3 +spec: + image: ubuntu-20.04 + machineType: m1.medium + maxSize: 1 + minSize: 1 + role: Master + subnets: + - us-test1 + +--- + +apiVersion: kops.k8s.io/v1alpha2 +kind: InstanceGroup +metadata: + creationTimestamp: "2017-01-01T00:00:00Z" + labels: + kops.k8s.io/cluster: ha.example.com + name: nodes-us-test1 +spec: + image: ubuntu-20.04 + machineType: m1.large + maxSize: 1 + minSize: 1 + role: Node + subnets: + - us-test1 diff --git a/tests/integration/create_cluster/ha_openstack_nodns/options.yaml b/tests/integration/create_cluster/ha_openstack_nodns/options.yaml new file mode 100644 index 0000000000..32ca534928 --- /dev/null +++ b/tests/integration/create_cluster/ha_openstack_nodns/options.yaml @@ -0,0 +1,19 @@ +CloudProvider: openstack +ClusterName: ha.example.com +Image: ubuntu-20.04 +KubernetesVersion: v1.25.0 +ControlPlaneCount: 3 +NetworkCIDR: 10.0.0.0/16 +Networking: calico +Zones: + - us-test1 +OpenstackLBOctavia: true +OpenstackExternalNet: vlan1 +OpenstackExternalSubnet: vlan1subnet +OpenstackLBSubnet: vlan1lbsubnet +OpenstackDNSServers: 1.1.1.1 +ControlPlaneSize: m1.medium +NodeSize: m1.large +APILoadBalancerType: public +Topology: private +DNSType: none \ No newline at end of file diff --git a/tests/integration/update_cluster/minimal-dns-none/data/aws_launch_template_nodes.minimal.example.com_user_data b/tests/integration/update_cluster/minimal-dns-none/data/aws_launch_template_nodes.minimal.example.com_user_data index 0c21090206..f088fe337c 100644 --- a/tests/integration/update_cluster/minimal-dns-none/data/aws_launch_template_nodes.minimal.example.com_user_data +++ b/tests/integration/update_cluster/minimal-dns-none/data/aws_launch_template_nodes.minimal.example.com_user_data @@ -172,7 +172,7 @@ ConfigServer: MA0GCSqGSIb3DQEBCwUAA0EAVQVx5MUtuAIeePuP9o51xtpT2S6Fvfi8J4ICxnlA 9B7UD2ushcVFPtaeoL9Gfu8aY4KJBeqqg5ojl4qmRnThjw== -----END CERTIFICATE----- - server: https://:3988/ + server: https://kops-controller.internal.minimal.example.com:3988/ InstanceGroupName: nodes InstanceGroupRole: Node NodeupConfigHash: 6RM3ki/GGmnnKqQhn8h8yIS46bzvlD1rdChld696w5k= diff --git a/tests/integration/update_cluster/minimal_gce_dns-none/data/google_compute_instance_template_nodes-minimal-gce-example-com_metadata_startup-script b/tests/integration/update_cluster/minimal_gce_dns-none/data/google_compute_instance_template_nodes-minimal-gce-example-com_metadata_startup-script index 9cd3e95cd9..a0defc996c 100644 --- a/tests/integration/update_cluster/minimal_gce_dns-none/data/google_compute_instance_template_nodes-minimal-gce-example-com_metadata_startup-script +++ b/tests/integration/update_cluster/minimal_gce_dns-none/data/google_compute_instance_template_nodes-minimal-gce-example-com_metadata_startup-script @@ -169,7 +169,7 @@ ConfigServer: MA0GCSqGSIb3DQEBCwUAA0EAVQVx5MUtuAIeePuP9o51xtpT2S6Fvfi8J4ICxnlA 9B7UD2ushcVFPtaeoL9Gfu8aY4KJBeqqg5ojl4qmRnThjw== -----END CERTIFICATE----- - server: https://:3988/ + server: https://kops-controller.internal.minimal-gce.example.com:3988/ InstanceGroupName: nodes InstanceGroupRole: Node NodeupConfigHash: siYe0C6BwTCDJA4Gyt2wen36Y5zxKbjHW7IZe1b7n6Q= diff --git a/tests/integration/update_cluster/minimal_hetzner/data/hcloud_server_nodes-fsn1_user_data b/tests/integration/update_cluster/minimal_hetzner/data/hcloud_server_nodes-fsn1_user_data index 38e943484d..bab9749363 100644 --- a/tests/integration/update_cluster/minimal_hetzner/data/hcloud_server_nodes-fsn1_user_data +++ b/tests/integration/update_cluster/minimal_hetzner/data/hcloud_server_nodes-fsn1_user_data @@ -168,7 +168,7 @@ ConfigServer: MA0GCSqGSIb3DQEBCwUAA0EAVQVx5MUtuAIeePuP9o51xtpT2S6Fvfi8J4ICxnlA 9B7UD2ushcVFPtaeoL9Gfu8aY4KJBeqqg5ojl4qmRnThjw== -----END CERTIFICATE----- - server: https://:3988/ + server: https://kops-controller.internal.minimal.example.com:3988/ InstanceGroupName: nodes-fsn1 InstanceGroupRole: Node NodeupConfigHash: BT/Gm5OFWxKAbkzgyDaK3qCMmGQvPkX34qTusCN5dj8= diff --git a/upup/pkg/fi/cloudup/apply_cluster.go b/upup/pkg/fi/cloudup/apply_cluster.go index 959975f6c8..e320a4593a 100644 --- a/upup/pkg/fi/cloudup/apply_cluster.go +++ b/upup/pkg/fi/cloudup/apply_cluster.go @@ -1398,28 +1398,24 @@ func (n *nodeUpConfigBuilder) BuildConfig(ig *kops.InstanceGroup, apiserverAddit // Set API server address to an IP from the cluster network CIDR if cluster.UsesNoneDNS() { switch cluster.Spec.GetCloudProvider() { - case kops.CloudProviderAWS, kops.CloudProviderHetzner: + case kops.CloudProviderAWS, kops.CloudProviderHetzner, kops.CloudProviderOpenstack: // Use a private IP address that belongs to the cluster network CIDR (some additional addresses may be FQDNs or public IPs) - for _, networkCIDR := range append(cluster.Spec.Networking.AdditionalNetworkCIDRs, cluster.Spec.Networking.NetworkCIDR) { - _, cidr, err := net.ParseCIDR(networkCIDR) - if err != nil { - return nil, nil, fmt.Errorf("failed to parse network CIDR %q: %w", networkCIDR, err) - } - for _, additionalIP := range apiserverAdditionalIPs { + for _, additionalIP := range apiserverAdditionalIPs { + for _, networkCIDR := range append(cluster.Spec.Networking.AdditionalNetworkCIDRs, cluster.Spec.Networking.NetworkCIDR) { + _, cidr, err := net.ParseCIDR(networkCIDR) + if err != nil { + return nil, nil, fmt.Errorf("failed to parse network CIDR %q: %w", networkCIDR, err) + } if cidr.Contains(net.ParseIP(additionalIP)) { - bootConfig.APIServerIP = additionalIP - break + bootConfig.APIServerIPs = append(bootConfig.APIServerIPs, additionalIP) } } - if bootConfig.APIServerIP != "" { - break - } } + case kops.CloudProviderGCE: // Use any IP address that is found (including public ones) for _, additionalIP := range apiserverAdditionalIPs { - bootConfig.APIServerIP = additionalIP - break + bootConfig.APIServerIPs = append(bootConfig.APIServerIPs, additionalIP) } default: return nil, nil, fmt.Errorf("'none' DNS topology is not supported for cloud %q", cluster.Spec.GetCloudProvider()) @@ -1429,8 +1425,8 @@ func (n *nodeUpConfigBuilder) BuildConfig(ig *kops.InstanceGroup, apiserverAddit useConfigServer := apiModel.UseKopsControllerForNodeBootstrap(cluster) && !ig.HasAPIServer() if useConfigServer { host := "kops-controller.internal." + cluster.ObjectMeta.Name - if cluster.UsesNoneDNS() { - host = bootConfig.APIServerIP + if cluster.UsesNoneDNS() && len(bootConfig.APIServerIPs) > 0 { + host = bootConfig.APIServerIPs[0] // TODO: how we could support array? } baseURL := url.URL{ Scheme: "https", diff --git a/upup/pkg/fi/cloudup/new_cluster.go b/upup/pkg/fi/cloudup/new_cluster.go index 609d77a9d2..f654b605de 100644 --- a/upup/pkg/fi/cloudup/new_cluster.go +++ b/upup/pkg/fi/cloudup/new_cluster.go @@ -333,7 +333,7 @@ func NewCluster(opt *NewClusterOptions, clientset simple.Clientset) (*NewCluster MaxRetries: fi.PtrTo(3), }, } - initializeOpenstackAPI(opt, cluster) + initializeOpenstack(opt, cluster) osCloud, err := openstack.NewOpenstackCloud(cluster, "openstackmodel") if err != nil { return nil, err @@ -1310,6 +1310,14 @@ func setupTopology(opt *NewClusterOptions, cluster *api.Cluster, allZones sets.S } } + err := setupDNSTopology(opt, cluster) + if err != nil { + return nil, err + } + return bastions, nil +} + +func setupDNSTopology(opt *NewClusterOptions, cluster *api.Cluster) error { switch strings.ToLower(opt.DNSType) { case "": if cluster.IsGossip() { @@ -1326,10 +1334,9 @@ func setupTopology(opt *NewClusterOptions, cluster *api.Cluster, allZones sets.S case "none": cluster.Spec.Networking.Topology.DNS = api.DNSTypeNone default: - return nil, fmt.Errorf("unknown DNSType: %q", opt.DNSType) + return fmt.Errorf("unknown DNSType: %q", opt.DNSType) } - - return bastions, nil + return nil } func setupAPI(opt *NewClusterOptions, cluster *api.Cluster) error { @@ -1391,7 +1398,7 @@ func setupAPI(opt *NewClusterOptions, cluster *api.Cluster) error { return nil } -func initializeOpenstackAPI(opt *NewClusterOptions, cluster *api.Cluster) { +func initializeOpenstack(opt *NewClusterOptions, cluster *api.Cluster) { if opt.APILoadBalancerType != "" { cluster.Spec.API.LoadBalancer = &api.LoadBalancerAccessSpec{} provider := "haproxy" @@ -1418,6 +1425,17 @@ func initializeOpenstackAPI(opt *NewClusterOptions, cluster *api.Cluster) { cluster.Spec.CloudProvider.Openstack.Loadbalancer.FloatingSubnet = fi.PtrTo(opt.OpenstackLBSubnet) } } + + // this is needed in new clusters, otherwise openstack clients will automatically try to use openstack designate + if strings.ToLower(opt.DNSType) == "none" { + if cluster.Spec.Networking.Topology == nil { + cluster.Spec.Networking.Topology = &api.TopologySpec{ + DNS: api.DNSTypeNone, + } + } else { + cluster.Spec.Networking.Topology.DNS = api.DNSTypeNone + } + } } func createEtcdCluster(etcdCluster string, controlPlanes []*api.InstanceGroup, encryptEtcdStorage bool, etcdStorageType string) api.EtcdClusterSpec { diff --git a/upup/pkg/fi/cloudup/openstacktasks/lblistener.go b/upup/pkg/fi/cloudup/openstacktasks/lblistener.go index b41e1f6ae2..f6b99d501c 100644 --- a/upup/pkg/fi/cloudup/openstacktasks/lblistener.go +++ b/upup/pkg/fi/cloudup/openstacktasks/lblistener.go @@ -30,6 +30,7 @@ import ( type LBListener struct { ID *string Name *string + Port *int Pool *LBPool Lifecycle fi.Lifecycle AllowedCIDRs []string @@ -61,6 +62,7 @@ func NewLBListenerTaskFromCloud(cloud openstack.OpenstackCloud, lifecycle fi.Lif listenerTask := &LBListener{ ID: fi.PtrTo(listener.ID), Name: fi.PtrTo(listener.Name), + Port: fi.PtrTo(listener.ProtocolPort), AllowedCIDRs: listener.AllowedCIDRs, Lifecycle: lifecycle, } @@ -152,7 +154,7 @@ func (_ *LBListener) RenderOpenstack(t *openstack.OpenstackAPITarget, a, e, chan DefaultPoolID: fi.ValueOf(e.Pool.ID), LoadbalancerID: fi.ValueOf(e.Pool.Loadbalancer.ID), Protocol: listeners.ProtocolTCP, - ProtocolPort: 443, + ProtocolPort: fi.ValueOf(e.Port), } if useVIPACL && (fi.ValueOf(e.Pool.Loadbalancer.Provider) != "ovn") { diff --git a/upup/pkg/fi/cloudup/openstacktasks/port.go b/upup/pkg/fi/cloudup/openstacktasks/port.go index 3688d56899..3bf33783b8 100644 --- a/upup/pkg/fi/cloudup/openstacktasks/port.go +++ b/upup/pkg/fi/cloudup/openstacktasks/port.go @@ -39,6 +39,7 @@ type Port struct { AdditionalSecurityGroups []string Lifecycle fi.Lifecycle Tags []string + ForAPIServer bool } // GetDependencies returns the dependencies of the Port task @@ -64,6 +65,26 @@ func (s *Port) CompareWithID() *string { return s.ID } +func (s *Port) FindAddresses(context *fi.CloudupContext) ([]string, error) { + cloud := context.T.Cloud.(openstack.OpenstackCloud) + if s.ID == nil { + return nil, nil + } + port, err := cloud.GetPort(fi.ValueOf(s.ID)) + if err != nil { + return nil, err + } + addrs := []string{} + for _, addr := range port.FixedIPs { + addrs = append(addrs, addr.IPAddress) + } + return addrs, nil +} + +func (s *Port) IsForAPIServer() bool { + return s.ForAPIServer +} + func newPortTaskFromCloud(cloud openstack.OpenstackCloud, lifecycle fi.Lifecycle, port *ports.Port, find *Port) (*Port, error) { additionalSecurityGroupIDs := map[string]struct{}{} if find != nil {