Merge pull request #14930 from zetaab/feature/openstacknodns

OpenStack: Add support for clusters without DNS
This commit is contained in:
Kubernetes Prow Robot 2023-01-12 07:52:54 -08:00 committed by GitHub
commit a1a0ce3f33
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
33 changed files with 1589 additions and 37 deletions

View File

@ -72,6 +72,11 @@ func TestCreateClusterOpenStackOctavia(t *testing.T) {
runCreateClusterIntegrationTest(t, "../../tests/integration/create_cluster/ha_openstack_octavia", "v1alpha2")
}
func TestCreateClusterOpenStackNoDNS(t *testing.T) {
t.Setenv("OS_REGION_NAME", "us-test1")
runCreateClusterIntegrationTest(t, "../../tests/integration/create_cluster/ha_openstack_nodns", "v1alpha2")
}
// TestCreateClusterCilium runs kops with the cilium networking flags
func TestCreateClusterCilium(t *testing.T) {
runCreateClusterIntegrationTest(t, "../../tests/integration/create_cluster/cilium-eni", "v1alpha2")

View File

@ -46,15 +46,15 @@ func (b *EtcHostsBuilder) Build(c *fi.NodeupModelBuilderContext) error {
Addresses: []string{"127.0.0.1"},
})
}
} else if b.BootConfig.APIServerIP != "" {
} else if len(b.BootConfig.APIServerIPs) > 0 {
task.Records = append(task.Records, nodetasks.HostRecord{
Hostname: b.APIInternalName(),
Addresses: []string{b.BootConfig.APIServerIP},
Addresses: b.BootConfig.APIServerIPs,
})
if b.UseKopsControllerForNodeBootstrap() {
task.Records = append(task.Records, nodetasks.HostRecord{
Hostname: "kops-controller.internal." + b.NodeupConfig.ClusterName,
Addresses: []string{b.BootConfig.APIServerIP},
Addresses: b.BootConfig.APIServerIPs,
})
}
}

View File

@ -62,8 +62,8 @@ func (b *KopsControllerBuilder) Build(c *fi.NodeupModelBuilderContext) error {
Subject: nodetasks.PKIXName{CommonName: "kops-controller"},
AlternateNames: []string{"kops-controller.internal." + b.NodeupConfig.ClusterName},
}
if b.BootConfig.APIServerIP != "" {
issueCert.AlternateNames = append(issueCert.AlternateNames, b.BootConfig.APIServerIP)
if len(b.BootConfig.APIServerIPs) > 0 {
issueCert.AlternateNames = append(issueCert.AlternateNames, b.BootConfig.APIServerIPs...)
}
c.AddTask(issueCert)

View File

@ -507,7 +507,7 @@ func validateTopology(c *kops.Cluster, topology *kops.TopologySpec, fieldPath *f
if topology.DNS != "" {
cloud := c.Spec.GetCloudProvider()
allErrs = append(allErrs, IsValidValue(fieldPath.Child("dns", "type"), &topology.DNS, kops.SupportedDnsTypes)...)
if topology.DNS == kops.DNSTypeNone && cloud != kops.CloudProviderHetzner && cloud != kops.CloudProviderAWS && cloud != kops.CloudProviderGCE {
if topology.DNS == kops.DNSTypeNone && cloud != kops.CloudProviderOpenstack && cloud != kops.CloudProviderHetzner && cloud != kops.CloudProviderAWS && cloud != kops.CloudProviderGCE {
allErrs = append(allErrs, field.Invalid(fieldPath.Child("dns", "type"), topology.DNS, fmt.Sprintf("not supported for %q", c.Spec.GetCloudProvider())))
}
}

View File

@ -117,9 +117,9 @@ type BootConfig struct {
ConfigBase *string `json:",omitempty"`
// ConfigServer holds the configuration for the configuration server.
ConfigServer *ConfigServerOptions `json:",omitempty"`
// APIServerIP is the API server IP address.
// APIServerIPs is the API server IP addresses.
// This field is used for adding an alias for api.internal. in /etc/hosts, when Topology.DNS.Type == DNSTypeNone.
APIServerIP string `json:",omitempty"`
APIServerIPs []string `json:",omitempty"`
// InstanceGroupName is the name of the instance group.
InstanceGroupName string `json:",omitempty"`
// InstanceGroupRole is the instance group role.

View File

@ -167,8 +167,13 @@ func (b *BootstrapScript) buildEnvironmentVariables(cluster *kops.Cluster) (map[
)
}
// credentials needed always when using swift but when using None dns only in control plane
passEnvs := true
if !strings.HasPrefix(cluster.Spec.ConfigBase, "swift://") && cluster.UsesNoneDNS() && !b.ig.IsControlPlane() {
passEnvs = false
}
// Pass in required credentials when using user-defined swift endpoint
if os.Getenv("OS_AUTH_URL") != "" {
if os.Getenv("OS_AUTH_URL") != "" && passEnvs {
for _, envVar := range osEnvs {
env[envVar] = fmt.Sprintf("'%s'", os.Getenv(envVar))
}

View File

@ -26,6 +26,7 @@ import (
"k8s.io/kops/pkg/apis/kops"
"k8s.io/kops/pkg/model"
"k8s.io/kops/pkg/truncate"
"k8s.io/kops/pkg/wellknownports"
"k8s.io/kops/upup/pkg/fi"
"k8s.io/kops/upup/pkg/fi/cloudup/openstack"
"k8s.io/kops/upup/pkg/fi/cloudup/openstacktasks"
@ -166,6 +167,10 @@ func (b *ServerGroupModelBuilder) buildInstances(c *fi.CloudupModelBuilderContex
}
c.AddTask(portTask)
if b.Cluster.UsesNoneDNS() && ig.Spec.Role == kops.InstanceGroupRoleControlPlane {
portTask.ForAPIServer = true
}
metaWithName := make(map[string]string)
for k, v := range igMeta {
metaWithName[k] = v
@ -315,8 +320,10 @@ func (b *ServerGroupModelBuilder) Build(c *fi.CloudupModelBuilderContext) error
}
c.AddTask(poolTask)
nameForResource := fi.ValueOf(lbTask.Name)
listenerTask := &openstacktasks.LBListener{
Name: lbTask.Name,
Name: fi.PtrTo(nameForResource),
Port: fi.PtrTo(wellknownports.KubeAPIServer),
Lifecycle: b.Lifecycle,
Pool: poolTask,
}
@ -334,7 +341,7 @@ func (b *ServerGroupModelBuilder) Build(c *fi.CloudupModelBuilderContext) error
c.AddTask(listenerTask)
monitorTask := &openstacktasks.PoolMonitor{
Name: lbTask.Name,
Name: fi.PtrTo(nameForResource),
Pool: poolTask,
Lifecycle: b.Lifecycle,
}
@ -350,7 +357,7 @@ func (b *ServerGroupModelBuilder) Build(c *fi.CloudupModelBuilderContext) error
Pool: poolTask,
ServerGroup: mastersg,
InterfaceName: fi.PtrTo(ifName),
ProtocolPort: fi.PtrTo(443),
ProtocolPort: fi.PtrTo(wellknownports.KubeAPIServer),
Lifecycle: b.Lifecycle,
Weight: fi.PtrTo(1),
}

View File

@ -515,6 +515,166 @@ func getServerGroupModelBuilderTestInput() []serverGroupModelBuilderTestInput {
},
},
},
{
desc: "multizone setup 3 masters 3 nodes without bastion with API loadbalancer dns none",
cluster: &kops.Cluster{
ObjectMeta: metav1.ObjectMeta{
Name: "cluster",
},
Spec: kops.ClusterSpec{
API: kops.APISpec{
LoadBalancer: &kops.LoadBalancerAccessSpec{
Type: kops.LoadBalancerTypePublic,
},
},
CloudProvider: kops.CloudProviderSpec{
Openstack: &kops.OpenstackSpec{
BlockStorage: &kops.OpenstackBlockStorageConfig{
Version: fi.PtrTo("v3"),
IgnoreAZ: fi.PtrTo(false),
CreateStorageClass: fi.PtrTo(false),
CSITopologySupport: fi.PtrTo(true),
},
Loadbalancer: &kops.OpenstackLoadbalancerConfig{
FloatingNetwork: fi.PtrTo("test"),
FloatingSubnet: fi.PtrTo("test-lb-subnet"),
Method: fi.PtrTo("ROUND_ROBIN"),
Provider: fi.PtrTo("amphora"),
UseOctavia: fi.PtrTo(true),
},
Monitor: &kops.OpenstackMonitor{
Delay: fi.PtrTo("1m"),
MaxRetries: fi.PtrTo(3),
Timeout: fi.PtrTo("30s"),
},
Network: &kops.OpenstackNetwork{
AvailabilityZoneHints: []*string{fi.PtrTo("zone-1"), fi.PtrTo("zone-2"), fi.PtrTo("zone-3")},
},
Router: &kops.OpenstackRouter{
DNSServers: fi.PtrTo("8.8.8.8,8.8.4.4"),
ExternalSubnet: fi.PtrTo("test-router-subnet"),
ExternalNetwork: fi.PtrTo("test"),
AvailabilityZoneHints: []*string{fi.PtrTo("ha-zone")},
},
Metadata: &kops.OpenstackMetadata{
ConfigDrive: fi.PtrTo(false),
},
},
},
KubernetesVersion: "1.25.0",
Networking: kops.NetworkingSpec{
Subnets: []kops.ClusterSubnetSpec{
{
Name: "subnet-1",
Zone: "zone-1",
Type: kops.SubnetTypePrivate,
},
{
Name: "subnet-2",
Zone: "zone-2",
Type: kops.SubnetTypePrivate,
},
{
Name: "subnet-3",
Zone: "zone-3",
Type: kops.SubnetTypePrivate,
},
},
Topology: &kops.TopologySpec{
ControlPlane: kops.TopologyPrivate,
DNS: kops.DNSTypeNone,
Nodes: kops.TopologyPrivate,
},
},
},
},
instanceGroups: []*kops.InstanceGroup{
{
ObjectMeta: metav1.ObjectMeta{
Name: "master-a",
},
Spec: kops.InstanceGroupSpec{
Role: kops.InstanceGroupRoleControlPlane,
Image: "image",
MinSize: i32(1),
MaxSize: i32(1),
MachineType: "blc.1-2",
Subnets: []string{"subnet-1"},
Zones: []string{"zone-1"},
},
},
{
ObjectMeta: metav1.ObjectMeta{
Name: "node-a",
},
Spec: kops.InstanceGroupSpec{
Role: kops.InstanceGroupRoleNode,
Image: "image",
MinSize: i32(1),
MaxSize: i32(1),
MachineType: "blc.1-2",
Subnets: []string{"subnet-1"},
Zones: []string{"zone-1"},
},
},
{
ObjectMeta: metav1.ObjectMeta{
Name: "master-b",
},
Spec: kops.InstanceGroupSpec{
Role: kops.InstanceGroupRoleControlPlane,
Image: "image",
MinSize: i32(1),
MaxSize: i32(1),
MachineType: "blc.1-2",
Subnets: []string{"subnet-2"},
Zones: []string{"zone-2"},
},
},
{
ObjectMeta: metav1.ObjectMeta{
Name: "node-b",
},
Spec: kops.InstanceGroupSpec{
Role: kops.InstanceGroupRoleNode,
Image: "image",
MinSize: i32(1),
MaxSize: i32(1),
MachineType: "blc.1-2",
Subnets: []string{"subnet-2"},
Zones: []string{"zone-2"},
},
},
{
ObjectMeta: metav1.ObjectMeta{
Name: "master-c",
},
Spec: kops.InstanceGroupSpec{
Role: kops.InstanceGroupRoleControlPlane,
Image: "image",
MinSize: i32(1),
MaxSize: i32(1),
MachineType: "blc.1-2",
Subnets: []string{"subnet-3"},
Zones: []string{"zone-3"},
},
},
{
ObjectMeta: metav1.ObjectMeta{
Name: "node-c",
},
Spec: kops.InstanceGroupSpec{
Role: kops.InstanceGroupRoleNode,
Image: "image",
MinSize: i32(1),
MaxSize: i32(1),
MachineType: "blc.1-2",
Subnets: []string{"subnet-3"},
Zones: []string{"zone-3"},
},
},
},
},
{
desc: "multizone setup 3 masters 3 nodes without external router",
cluster: &kops.Cluster{

View File

@ -26,6 +26,7 @@ Name: node-1-cluster
Port:
AdditionalSecurityGroups:
- additional-sg
ForAPIServer: false
ID: null
InstanceGroupName: node
Lifecycle: Sync
@ -192,6 +193,7 @@ PublicACL: null
---
AdditionalSecurityGroups:
- additional-sg
ForAPIServer: false
ID: null
InstanceGroupName: node
Lifecycle: Sync

View File

@ -26,6 +26,7 @@ Metadata:
Name: node-1-cluster
Port:
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: node
Lifecycle: Sync
@ -190,6 +191,7 @@ Name: nodeupconfig-node
PublicACL: null
---
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: node
Lifecycle: Sync

View File

@ -26,6 +26,7 @@ Metadata:
Name: node-1-cluster
Port:
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: node
Lifecycle: Sync
@ -190,6 +191,7 @@ Name: nodeupconfig-node
PublicACL: null
---
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: node
Lifecycle: Sync

View File

@ -25,6 +25,7 @@ Metadata:
Name: node-1-cluster
Port:
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: node
Lifecycle: Sync
@ -189,6 +190,7 @@ Name: nodeupconfig-node
PublicACL: null
---
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: node
Lifecycle: Sync

View File

@ -79,6 +79,7 @@ Metadata:
Name: master-1-cluster
Port:
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: master
Lifecycle: Sync
@ -165,6 +166,7 @@ Metadata:
Name: master-2-cluster
Port:
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: master
Lifecycle: Sync
@ -251,6 +253,7 @@ Metadata:
Name: master-3-cluster
Port:
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: master
Lifecycle: Sync
@ -334,6 +337,7 @@ Metadata:
Name: node-1-cluster
Port:
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: node
Lifecycle: Sync
@ -411,6 +415,7 @@ Metadata:
Name: node-2-cluster
Port:
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: node
Lifecycle: Sync
@ -488,6 +493,7 @@ Metadata:
Name: node-3-cluster
Port:
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: node
Lifecycle: Sync
@ -662,6 +668,7 @@ Name: nodeupconfig-node
PublicACL: null
---
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: master
Lifecycle: Sync
@ -699,6 +706,7 @@ Tags:
- KubernetesCluster=cluster
---
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: master
Lifecycle: Sync
@ -736,6 +744,7 @@ Tags:
- KubernetesCluster=cluster
---
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: master
Lifecycle: Sync
@ -773,6 +782,7 @@ Tags:
- KubernetesCluster=cluster
---
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: node
Lifecycle: Sync
@ -804,6 +814,7 @@ Tags:
- KubernetesCluster=cluster
---
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: node
Lifecycle: Sync
@ -835,6 +846,7 @@ Tags:
- KubernetesCluster=cluster
---
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: node
Lifecycle: Sync

View File

@ -85,6 +85,7 @@ Metadata:
Name: master-a-1-cluster
Port:
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: master-a
Lifecycle: Sync
@ -159,6 +160,7 @@ Metadata:
Name: master-b-1-cluster
Port:
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: master-b
Lifecycle: Sync
@ -233,6 +235,7 @@ Metadata:
Name: master-c-1-cluster
Port:
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: master-c
Lifecycle: Sync
@ -310,6 +313,7 @@ Metadata:
Name: node-a-1-cluster
Port:
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: node-a
Lifecycle: Sync
@ -387,6 +391,7 @@ Metadata:
Name: node-b-1-cluster
Port:
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: node-b
Lifecycle: Sync
@ -464,6 +469,7 @@ Metadata:
Name: node-c-1-cluster
Port:
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: node-c
Lifecycle: Sync
@ -655,6 +661,7 @@ Pool:
Subnet: subnet-a.cluster
VipSubnet: null
Name: master-public-name-https
Port: 443
---
ID: null
Lifecycle: Sync
@ -864,6 +871,7 @@ Pool:
Name: master-public-name-https
---
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: master-a
Lifecycle: Sync
@ -895,6 +903,7 @@ Tags:
- KubernetesCluster=cluster
---
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: master-b
Lifecycle: Sync
@ -926,6 +935,7 @@ Tags:
- KubernetesCluster=cluster
---
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: master-c
Lifecycle: Sync
@ -957,6 +967,7 @@ Tags:
- KubernetesCluster=cluster
---
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: node-a
Lifecycle: Sync
@ -988,6 +999,7 @@ Tags:
- KubernetesCluster=cluster
---
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: node-b
Lifecycle: Sync
@ -1019,6 +1031,7 @@ Tags:
- KubernetesCluster=cluster
---
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: node-c
Lifecycle: Sync

View File

@ -91,6 +91,7 @@ Metadata:
Name: master-a-1-cluster
Port:
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: master-a
Lifecycle: Sync
@ -177,6 +178,7 @@ Metadata:
Name: master-b-1-cluster
Port:
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: master-b
Lifecycle: Sync
@ -263,6 +265,7 @@ Metadata:
Name: master-c-1-cluster
Port:
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: master-c
Lifecycle: Sync
@ -346,6 +349,7 @@ Metadata:
Name: node-a-1-cluster
Port:
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: node-a
Lifecycle: Sync
@ -423,6 +427,7 @@ Metadata:
Name: node-b-1-cluster
Port:
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: node-b
Lifecycle: Sync
@ -500,6 +505,7 @@ Metadata:
Name: node-c-1-cluster
Port:
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: node-c
Lifecycle: Sync
@ -714,6 +720,7 @@ Name: nodeupconfig-node-c
PublicACL: null
---
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: master-a
Lifecycle: Sync
@ -751,6 +758,7 @@ Tags:
- KubernetesCluster=cluster
---
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: master-b
Lifecycle: Sync
@ -788,6 +796,7 @@ Tags:
- KubernetesCluster=cluster
---
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: master-c
Lifecycle: Sync
@ -825,6 +834,7 @@ Tags:
- KubernetesCluster=cluster
---
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: node-a
Lifecycle: Sync
@ -856,6 +866,7 @@ Tags:
- KubernetesCluster=cluster
---
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: node-b
Lifecycle: Sync
@ -887,6 +898,7 @@ Tags:
- KubernetesCluster=cluster
---
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: node-c
Lifecycle: Sync

View File

@ -43,6 +43,7 @@ Metadata:
Name: master-a-1-cluster
Port:
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: master-a
Lifecycle: Sync
@ -123,6 +124,7 @@ Metadata:
Name: master-b-1-cluster
Port:
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: master-b
Lifecycle: Sync
@ -203,6 +205,7 @@ Metadata:
Name: master-c-1-cluster
Port:
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: master-c
Lifecycle: Sync
@ -280,6 +283,7 @@ Metadata:
Name: node-a-1-cluster
Port:
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: node-a
Lifecycle: Sync
@ -351,6 +355,7 @@ Metadata:
Name: node-b-1-cluster
Port:
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: node-b
Lifecycle: Sync
@ -422,6 +427,7 @@ Metadata:
Name: node-c-1-cluster
Port:
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: node-c
Lifecycle: Sync
@ -636,6 +642,7 @@ Name: nodeupconfig-node-c
PublicACL: null
---
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: master-a
Lifecycle: Sync
@ -673,6 +680,7 @@ Tags:
- KubernetesCluster=cluster
---
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: master-b
Lifecycle: Sync
@ -710,6 +718,7 @@ Tags:
- KubernetesCluster=cluster
---
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: master-c
Lifecycle: Sync
@ -747,6 +756,7 @@ Tags:
- KubernetesCluster=cluster
---
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: node-a
Lifecycle: Sync
@ -778,6 +788,7 @@ Tags:
- KubernetesCluster=cluster
---
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: node-b
Lifecycle: Sync
@ -809,6 +820,7 @@ Tags:
- KubernetesCluster=cluster
---
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: node-c
Lifecycle: Sync

View File

@ -30,6 +30,7 @@ Metadata:
Name: bastion-1-cluster
Port:
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: bastion
Lifecycle: Sync
@ -104,6 +105,7 @@ Metadata:
Name: master-1-cluster
Port:
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: master
Lifecycle: Sync
@ -181,6 +183,7 @@ Metadata:
Name: node-1-cluster
Port:
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: node
Lifecycle: Sync
@ -365,6 +368,7 @@ Name: nodeupconfig-node
PublicACL: null
---
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: bastion
Lifecycle: Sync
@ -396,6 +400,7 @@ Tags:
- KubernetesCluster=cluster
---
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: master
Lifecycle: Sync
@ -433,6 +438,7 @@ Tags:
- KubernetesCluster=cluster
---
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: node
Lifecycle: Sync

View File

@ -50,6 +50,7 @@ Metadata:
Name: bastion-1-cluster
Port:
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: bastion
Lifecycle: Sync
@ -130,6 +131,7 @@ Metadata:
Name: master-1-cluster
Port:
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: master
Lifecycle: Sync
@ -207,6 +209,7 @@ Metadata:
Name: node-1-cluster
Port:
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: node
Lifecycle: Sync
@ -391,6 +394,7 @@ Name: nodeupconfig-node
PublicACL: null
---
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: bastion
Lifecycle: Sync
@ -422,6 +426,7 @@ Tags:
- KubernetesCluster=cluster
---
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: master
Lifecycle: Sync
@ -459,6 +464,7 @@ Tags:
- KubernetesCluster=cluster
---
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: node
Lifecycle: Sync

View File

@ -31,6 +31,7 @@ Metadata:
Name: master-1-cluster
Port:
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: master
Lifecycle: Sync
@ -108,6 +109,7 @@ Metadata:
Name: node-1-cluster
Port:
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: node
Lifecycle: Sync
@ -282,6 +284,7 @@ Name: nodeupconfig-node
PublicACL: null
---
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: master
Lifecycle: Sync
@ -319,6 +322,7 @@ Tags:
- KubernetesCluster=cluster
---
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: node
Lifecycle: Sync

View File

@ -51,6 +51,7 @@ Metadata:
Name: master-1-cluster
Port:
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: master
Lifecycle: Sync
@ -134,6 +135,7 @@ Metadata:
Name: node-1-cluster
Port:
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: node
Lifecycle: Sync
@ -308,6 +310,7 @@ Name: nodeupconfig-node
PublicACL: null
---
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: master
Lifecycle: Sync
@ -345,6 +348,7 @@ Tags:
- KubernetesCluster=cluster
---
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: node
Lifecycle: Sync

View File

@ -51,6 +51,7 @@ Metadata:
Name: master-1-tom-software-dev-playground-real33-k8s-local
Port:
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: master
Lifecycle: Sync
@ -134,6 +135,7 @@ Metadata:
Name: node-1-tom-software-dev-playground-real33-k8s-local
Port:
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: node
Lifecycle: Sync
@ -308,6 +310,7 @@ Name: nodeupconfig-node
PublicACL: null
---
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: master
Lifecycle: Sync
@ -345,6 +348,7 @@ Tags:
- KubernetesCluster=tom-software-dev-playground-real33--kngu8l
---
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: node
Lifecycle: Sync

View File

@ -26,6 +26,7 @@ Name: node-1-cluster
Port:
AdditionalSecurityGroups:
- additional-sg
ForAPIServer: false
ID: null
InstanceGroupName: node
Lifecycle: Sync
@ -192,6 +193,7 @@ PublicACL: null
---
AdditionalSecurityGroups:
- additional-sg
ForAPIServer: false
ID: null
InstanceGroupName: node
Lifecycle: Sync

View File

@ -26,6 +26,7 @@ Name: node-1-cluster
Port:
AdditionalSecurityGroups:
- additional-sg
ForAPIServer: false
ID: null
InstanceGroupName: node
Lifecycle: Sync
@ -192,6 +193,7 @@ PublicACL: null
---
AdditionalSecurityGroups:
- additional-sg
ForAPIServer: false
ID: null
InstanceGroupName: node
Lifecycle: Sync

View File

@ -0,0 +1,156 @@
apiVersion: kops.k8s.io/v1alpha2
kind: Cluster
metadata:
creationTimestamp: "2017-01-01T00:00:00Z"
name: ha.example.com
spec:
api:
loadBalancer:
type: Public
authorization:
rbac: {}
channel: stable
cloudConfig:
openstack:
blockStorage:
bs-version: v3
ignore-volume-az: false
loadbalancer:
floatingNetwork: vlan1
floatingSubnet: vlan1lbsubnet
method: ROUND_ROBIN
provider: octavia
useOctavia: true
monitor:
delay: 15s
maxRetries: 3
timeout: 10s
router:
dnsServers: 1.1.1.1
externalNetwork: vlan1
externalSubnet: vlan1subnet
cloudProvider: openstack
configBase: memfs://tests/ha.example.com
etcdClusters:
- cpuRequest: 200m
etcdMembers:
- instanceGroup: control-plane-us-test1-1
name: etcd-1
- instanceGroup: control-plane-us-test1-2
name: etcd-2
- instanceGroup: control-plane-us-test1-3
name: etcd-3
memoryRequest: 100Mi
name: main
- cpuRequest: 100m
etcdMembers:
- instanceGroup: control-plane-us-test1-1
name: etcd-1
- instanceGroup: control-plane-us-test1-2
name: etcd-2
- instanceGroup: control-plane-us-test1-3
name: etcd-3
memoryRequest: 100Mi
name: events
iam:
allowContainerRegistry: true
legacy: false
kubelet:
anonymousAuth: false
kubernetesApiAccess:
- 0.0.0.0/0
- ::/0
kubernetesVersion: v1.25.0
networkCIDR: 10.0.0.0/16
networking:
calico: {}
nonMasqueradeCIDR: 100.64.0.0/10
sshAccess:
- 0.0.0.0/0
- ::/0
subnets:
- cidr: 10.0.32.0/19
name: us-test1
type: Private
zone: us-test1
- cidr: 10.0.0.0/22
name: utility-us-test1
type: Utility
zone: us-test1
topology:
dns:
type: None
masters: private
nodes: private
---
apiVersion: kops.k8s.io/v1alpha2
kind: InstanceGroup
metadata:
creationTimestamp: "2017-01-01T00:00:00Z"
labels:
kops.k8s.io/cluster: ha.example.com
name: control-plane-us-test1-1
spec:
image: ubuntu-20.04
machineType: m1.medium
maxSize: 1
minSize: 1
role: Master
subnets:
- us-test1
---
apiVersion: kops.k8s.io/v1alpha2
kind: InstanceGroup
metadata:
creationTimestamp: "2017-01-01T00:00:00Z"
labels:
kops.k8s.io/cluster: ha.example.com
name: control-plane-us-test1-2
spec:
image: ubuntu-20.04
machineType: m1.medium
maxSize: 1
minSize: 1
role: Master
subnets:
- us-test1
---
apiVersion: kops.k8s.io/v1alpha2
kind: InstanceGroup
metadata:
creationTimestamp: "2017-01-01T00:00:00Z"
labels:
kops.k8s.io/cluster: ha.example.com
name: control-plane-us-test1-3
spec:
image: ubuntu-20.04
machineType: m1.medium
maxSize: 1
minSize: 1
role: Master
subnets:
- us-test1
---
apiVersion: kops.k8s.io/v1alpha2
kind: InstanceGroup
metadata:
creationTimestamp: "2017-01-01T00:00:00Z"
labels:
kops.k8s.io/cluster: ha.example.com
name: nodes-us-test1
spec:
image: ubuntu-20.04
machineType: m1.large
maxSize: 1
minSize: 1
role: Node
subnets:
- us-test1

View File

@ -0,0 +1,19 @@
CloudProvider: openstack
ClusterName: ha.example.com
Image: ubuntu-20.04
KubernetesVersion: v1.25.0
ControlPlaneCount: 3
NetworkCIDR: 10.0.0.0/16
Networking: calico
Zones:
- us-test1
OpenstackLBOctavia: true
OpenstackExternalNet: vlan1
OpenstackExternalSubnet: vlan1subnet
OpenstackLBSubnet: vlan1lbsubnet
OpenstackDNSServers: 1.1.1.1
ControlPlaneSize: m1.medium
NodeSize: m1.large
APILoadBalancerType: public
Topology: private
DNSType: none

View File

@ -172,7 +172,7 @@ ConfigServer:
MA0GCSqGSIb3DQEBCwUAA0EAVQVx5MUtuAIeePuP9o51xtpT2S6Fvfi8J4ICxnlA
9B7UD2ushcVFPtaeoL9Gfu8aY4KJBeqqg5ojl4qmRnThjw==
-----END CERTIFICATE-----
server: https://:3988/
server: https://kops-controller.internal.minimal.example.com:3988/
InstanceGroupName: nodes
InstanceGroupRole: Node
NodeupConfigHash: 6RM3ki/GGmnnKqQhn8h8yIS46bzvlD1rdChld696w5k=

View File

@ -169,7 +169,7 @@ ConfigServer:
MA0GCSqGSIb3DQEBCwUAA0EAVQVx5MUtuAIeePuP9o51xtpT2S6Fvfi8J4ICxnlA
9B7UD2ushcVFPtaeoL9Gfu8aY4KJBeqqg5ojl4qmRnThjw==
-----END CERTIFICATE-----
server: https://:3988/
server: https://kops-controller.internal.minimal-gce.example.com:3988/
InstanceGroupName: nodes
InstanceGroupRole: Node
NodeupConfigHash: siYe0C6BwTCDJA4Gyt2wen36Y5zxKbjHW7IZe1b7n6Q=

View File

@ -168,7 +168,7 @@ ConfigServer:
MA0GCSqGSIb3DQEBCwUAA0EAVQVx5MUtuAIeePuP9o51xtpT2S6Fvfi8J4ICxnlA
9B7UD2ushcVFPtaeoL9Gfu8aY4KJBeqqg5ojl4qmRnThjw==
-----END CERTIFICATE-----
server: https://:3988/
server: https://kops-controller.internal.minimal.example.com:3988/
InstanceGroupName: nodes-fsn1
InstanceGroupRole: Node
NodeupConfigHash: BT/Gm5OFWxKAbkzgyDaK3qCMmGQvPkX34qTusCN5dj8=

View File

@ -1398,28 +1398,24 @@ func (n *nodeUpConfigBuilder) BuildConfig(ig *kops.InstanceGroup, apiserverAddit
// Set API server address to an IP from the cluster network CIDR
if cluster.UsesNoneDNS() {
switch cluster.Spec.GetCloudProvider() {
case kops.CloudProviderAWS, kops.CloudProviderHetzner:
case kops.CloudProviderAWS, kops.CloudProviderHetzner, kops.CloudProviderOpenstack:
// Use a private IP address that belongs to the cluster network CIDR (some additional addresses may be FQDNs or public IPs)
for _, networkCIDR := range append(cluster.Spec.Networking.AdditionalNetworkCIDRs, cluster.Spec.Networking.NetworkCIDR) {
_, cidr, err := net.ParseCIDR(networkCIDR)
if err != nil {
return nil, nil, fmt.Errorf("failed to parse network CIDR %q: %w", networkCIDR, err)
}
for _, additionalIP := range apiserverAdditionalIPs {
for _, additionalIP := range apiserverAdditionalIPs {
for _, networkCIDR := range append(cluster.Spec.Networking.AdditionalNetworkCIDRs, cluster.Spec.Networking.NetworkCIDR) {
_, cidr, err := net.ParseCIDR(networkCIDR)
if err != nil {
return nil, nil, fmt.Errorf("failed to parse network CIDR %q: %w", networkCIDR, err)
}
if cidr.Contains(net.ParseIP(additionalIP)) {
bootConfig.APIServerIP = additionalIP
break
bootConfig.APIServerIPs = append(bootConfig.APIServerIPs, additionalIP)
}
}
if bootConfig.APIServerIP != "" {
break
}
}
case kops.CloudProviderGCE:
// Use any IP address that is found (including public ones)
for _, additionalIP := range apiserverAdditionalIPs {
bootConfig.APIServerIP = additionalIP
break
bootConfig.APIServerIPs = append(bootConfig.APIServerIPs, additionalIP)
}
default:
return nil, nil, fmt.Errorf("'none' DNS topology is not supported for cloud %q", cluster.Spec.GetCloudProvider())
@ -1429,8 +1425,8 @@ func (n *nodeUpConfigBuilder) BuildConfig(ig *kops.InstanceGroup, apiserverAddit
useConfigServer := apiModel.UseKopsControllerForNodeConfig(cluster) && !ig.HasAPIServer()
if useConfigServer {
host := "kops-controller.internal." + cluster.ObjectMeta.Name
if cluster.UsesNoneDNS() {
host = bootConfig.APIServerIP
if cluster.UsesNoneDNS() && len(bootConfig.APIServerIPs) > 0 {
host = bootConfig.APIServerIPs[0] // TODO: how we could support array?
}
baseURL := url.URL{
Scheme: "https",

View File

@ -333,7 +333,7 @@ func NewCluster(opt *NewClusterOptions, clientset simple.Clientset) (*NewCluster
MaxRetries: fi.PtrTo(3),
},
}
initializeOpenstackAPI(opt, cluster)
initializeOpenstack(opt, cluster)
osCloud, err := openstack.NewOpenstackCloud(cluster, "openstackmodel")
if err != nil {
return nil, err
@ -1304,6 +1304,14 @@ func setupTopology(opt *NewClusterOptions, cluster *api.Cluster, allZones sets.S
}
}
err := setupDNSTopology(opt, cluster)
if err != nil {
return nil, err
}
return bastions, nil
}
func setupDNSTopology(opt *NewClusterOptions, cluster *api.Cluster) error {
switch strings.ToLower(opt.DNSType) {
case "":
if cluster.IsGossip() {
@ -1320,10 +1328,9 @@ func setupTopology(opt *NewClusterOptions, cluster *api.Cluster, allZones sets.S
case "none":
cluster.Spec.Networking.Topology.DNS = api.DNSTypeNone
default:
return nil, fmt.Errorf("unknown DNSType: %q", opt.DNSType)
return fmt.Errorf("unknown DNSType: %q", opt.DNSType)
}
return bastions, nil
return nil
}
func setupAPI(opt *NewClusterOptions, cluster *api.Cluster) error {
@ -1385,7 +1392,7 @@ func setupAPI(opt *NewClusterOptions, cluster *api.Cluster) error {
return nil
}
func initializeOpenstackAPI(opt *NewClusterOptions, cluster *api.Cluster) {
func initializeOpenstack(opt *NewClusterOptions, cluster *api.Cluster) {
if opt.APILoadBalancerType != "" {
cluster.Spec.API.LoadBalancer = &api.LoadBalancerAccessSpec{}
provider := "haproxy"
@ -1412,6 +1419,17 @@ func initializeOpenstackAPI(opt *NewClusterOptions, cluster *api.Cluster) {
cluster.Spec.CloudProvider.Openstack.Loadbalancer.FloatingSubnet = fi.PtrTo(opt.OpenstackLBSubnet)
}
}
// this is needed in new clusters, otherwise openstack clients will automatically try to use openstack designate
if strings.ToLower(opt.DNSType) == "none" {
if cluster.Spec.Networking.Topology == nil {
cluster.Spec.Networking.Topology = &api.TopologySpec{
DNS: api.DNSTypeNone,
}
} else {
cluster.Spec.Networking.Topology.DNS = api.DNSTypeNone
}
}
}
func createEtcdCluster(etcdCluster string, controlPlanes []*api.InstanceGroup, encryptEtcdStorage bool, etcdStorageType string) api.EtcdClusterSpec {

View File

@ -30,6 +30,7 @@ import (
type LBListener struct {
ID *string
Name *string
Port *int
Pool *LBPool
Lifecycle fi.Lifecycle
AllowedCIDRs []string
@ -61,6 +62,7 @@ func NewLBListenerTaskFromCloud(cloud openstack.OpenstackCloud, lifecycle fi.Lif
listenerTask := &LBListener{
ID: fi.PtrTo(listener.ID),
Name: fi.PtrTo(listener.Name),
Port: fi.PtrTo(listener.ProtocolPort),
AllowedCIDRs: listener.AllowedCIDRs,
Lifecycle: lifecycle,
}
@ -152,7 +154,7 @@ func (_ *LBListener) RenderOpenstack(t *openstack.OpenstackAPITarget, a, e, chan
DefaultPoolID: fi.ValueOf(e.Pool.ID),
LoadbalancerID: fi.ValueOf(e.Pool.Loadbalancer.ID),
Protocol: listeners.ProtocolTCP,
ProtocolPort: 443,
ProtocolPort: fi.ValueOf(e.Port),
}
if useVIPACL && (fi.ValueOf(e.Pool.Loadbalancer.Provider) != "ovn") {

View File

@ -39,6 +39,7 @@ type Port struct {
AdditionalSecurityGroups []string
Lifecycle fi.Lifecycle
Tags []string
ForAPIServer bool
}
// GetDependencies returns the dependencies of the Port task
@ -64,6 +65,26 @@ func (s *Port) CompareWithID() *string {
return s.ID
}
func (s *Port) FindAddresses(context *fi.CloudupContext) ([]string, error) {
cloud := context.T.Cloud.(openstack.OpenstackCloud)
if s.ID == nil {
return nil, nil
}
port, err := cloud.GetPort(fi.ValueOf(s.ID))
if err != nil {
return nil, err
}
addrs := []string{}
for _, addr := range port.FixedIPs {
addrs = append(addrs, addr.IPAddress)
}
return addrs, nil
}
func (s *Port) IsForAPIServer() bool {
return s.ForAPIServer
}
func newPortTaskFromCloud(cloud openstack.OpenstackCloud, lifecycle fi.Lifecycle, port *ports.Port, find *Port) (*Port, error) {
additionalSecurityGroupIDs := map[string]struct{}{}
if find != nil {