kubernetes
/
kops
mirror of https://github.com/kubernetes/kops.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #14577 from hakman/k8s_feature-gates 

Add option for setting Kubernetes feature gates
This commit is contained in:
Kubernetes Prow Robot 2022-11-19 22:32:30 -08:00 committed by GitHub
parent feffc79181 398861dd66
commit 0da903d9ba
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 253 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
cmd/kops/create_cluster.go
View File

@ -245,9 +245,12 @@ func NewCmdCreateCluster(f *util.Factory, out io.Writer) *cobra.Command {
// TODO complete VFS paths
}
cmd.Flags().StringVar(&options.KubernetesVersion, "kubernetes-version", options.KubernetesVersion, "Version of kubernetes to run (defaults to version in channel)")
cmd.Flags().StringVar(&options.KubernetesVersion, "kubernetes-version", options.KubernetesVersion, "Version of Kubernetes to run (defaults to version in channel)")
cmd.RegisterFlagCompletionFunc("kubernetes-version", completeKubernetesVersion)
cmd.Flags().StringSliceVar(&options.KubernetesFeatureGates, "kubernetes-feature-gates", options.KubernetesFeatureGates, "List of Kubernetes feature gates to enable/disable")
cmd.RegisterFlagCompletionFunc("kubernetes-version", completeKubernetesFeatureGates)
cmd.Flags().StringVar(&options.ContainerRuntime, "container-runtime", options.ContainerRuntime, "Container runtime to use: containerd, docker")
cmd.RegisterFlagCompletionFunc("container-runtime", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return []string{"containerd", "docker"}, cobra.ShellCompDirectiveNoFileComp
@ -432,7 +435,7 @@ func NewCmdCreateCluster(f *util.Factory, out io.Writer) *cobra.Command {
cmd.RegisterFlagCompletionFunc("os-ext-net", completeOpenstackExternalNet)
cmd.Flags().StringVar(&options.OpenstackExternalSubnet, "os-ext-subnet", options.OpenstackExternalSubnet, "External floating subnet to use with the openstack router")
cmd.RegisterFlagCompletionFunc("os-ext-subnet", completeOpenstackExternalSubnet)
cmd.Flags().StringVar(&options.OpenstackLBSubnet, "os-lb-floating-subnet", options.OpenstackLBSubnet, "External subnet to use with the kubernetes api")
cmd.Flags().StringVar(&options.OpenstackLBSubnet, "os-lb-floating-subnet", options.OpenstackLBSubnet, "External subnet to use with the Kubernetes API")
cmd.RegisterFlagCompletionFunc("os-lb-floating-subnet", completeOpenstackLBSubnet)
cmd.Flags().BoolVar(&options.OpenstackStorageIgnoreAZ, "os-kubelet-ignore-az", options.OpenstackStorageIgnoreAZ, "Attach volumes across availability zones")
cmd.Flags().BoolVar(&options.OpenstackLBOctavia, "os-octavia", options.OpenstackLBOctavia, "Use octavia load balancer API")
@ -908,6 +911,11 @@ func completeKubernetesVersion(cmd *cobra.Command, args []string, toComplete str
return versions.List(), cobra.ShellCompDirectiveNoFileComp
}
func completeKubernetesFeatureGates(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
// TODO check if there's a way to get the full list of feature gates from k8s libs
return nil, cobra.ShellCompDirectiveNoFileComp
}
func completeInstanceImage(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
// TODO call into cloud provider(s) to get list of valid images
return nil, cobra.ShellCompDirectiveNoFileComp

5
cmd/kops/create_cluster_integration_test.go
View File

@ -69,6 +69,11 @@ func TestCreateClusterOverride(t *testing.T) {
runCreateClusterIntegrationTest(t, "../../tests/integration/create_cluster/overrides", "v1alpha2")
}
// TestCreateClusterKubernetesFeatureGates tests the override flag
func TestCreateClusterKubernetesFeatureGates(t *testing.T) {
runCreateClusterIntegrationTest(t, "../../tests/integration/create_cluster/minimal_feature-gates", "v1alpha2")
}
// TestCreateClusterComplex runs kops create cluster, with a grab-bag of edge cases
func TestCreateClusterComplex(t *testing.T) {
runCreateClusterIntegrationTest(t, "../../tests/integration/create_cluster/complex", "v1alpha2")

123
docs/cli/kops_create_cluster.md generated
View File

@ -65,67 +65,68 @@ kops create cluster [CLUSTER] [flags]
### Options
```
--admin-access strings Restrict API access to this CIDR. If not set, access will not be restricted by IP. (default [0.0.0.0/0,::/0])
--api-loadbalancer-type string Type of load balancer for the Kubernetes API: public or internal
--api-ssl-certificate string ARN of the SSL Certificate to use for the Kubernetes API load balancer (AWS only)
--associate-public-ip Specify --associate-public-ip=[true|false] to enable/disable association of public IP for master ASG and nodes. Default is 'true'.
--authorization string Authorization mode: AlwaysAllow or RBAC (default "RBAC")
--bastion Enable a bastion instance group. Only applies to private topology.
--bastion-image string Machine image for bastions. Takes precedence over --image
--channel string Channel for default versions and configuration to use (default "stable")
--cloud string Cloud provider to use - aws, digitalocean, gce, hetzner, openstack
--cloud-labels string A list of key/value pairs used to tag all instance groups (for example "Owner=John Doe,Team=Some Team").
--container-runtime string Container runtime to use: containerd, docker
--disable-subnet-tags Disable automatic subnet tagging
--discovery-store string A public location where we publish OIDC-compatible discovery information under a cluster-specific directory. Enables IRSA in AWS.
--dns string DNS type to use: public, private, none
--dns-zone string DNS hosted zone (defaults to longest matching zone)
--dry-run If true, only print the object that would be sent, without sending it. This flag can be used to create a cluster YAML or JSON manifest.
--encrypt-etcd-storage Generate key in AWS KMS and use it for encrypt etcd volumes
--etcd-storage-type string The default storage type for etcd members
--gce-service-account string Service account with which the GCE VM runs. Warning: if not set, VMs will run as default compute service account.
-h, --help help for cluster
--image string Machine image for all instances
--ipv6 Use IPv6 for the pod network (AWS only)
--kubernetes-version string Version of kubernetes to run (defaults to version in channel)
--master-count int32 Number of masters. Defaults to one master per master-zone
--master-image string Machine image for masters. Takes precedence over --image
--master-public-name string Domain name of the public Kubernetes API
--master-security-groups strings Additional precreated security groups to add to masters.
--master-size string Machine type for masters
--master-tenancy string Tenancy of the master group (AWS only): default or dedicated
--master-volume-size int32 Instance volume size (in GB) for masters
--master-zones strings Zones in which to run masters (must be an odd number)
--network-cidr string Network CIDR to use
--network-id string Shared Network or VPC to use
--networking string Networking mode. kubenet, external, weave, flannel-vxlan (or flannel), flannel-udp, calico, canal, kube-router, amazonvpc, cilium, cilium-etcd, cni. (default "cilium")
--node-count int32 Total number of worker nodes. Defaults to one node per zone
--node-image string Machine image for worker nodes. Takes precedence over --image
--node-security-groups strings Additional precreated security groups to add to worker nodes.
--node-size string Machine type for worker nodes
--node-tenancy string Tenancy of the node group (AWS only): default or dedicated
--node-volume-size int32 Instance volume size (in GB) for worker nodes
--os-dns-servers string comma separated list of DNS Servers which is used in network
--os-ext-net string External network to use with the openstack router
--os-ext-subnet string External floating subnet to use with the openstack router
--os-kubelet-ignore-az Attach volumes across availability zones
--os-lb-floating-subnet string External subnet to use with the kubernetes api
--os-network string ID of the existing OpenStack network to use
--os-octavia Use octavia load balancer API
--os-octavia-provider string Octavia provider to use
--out string Path to write any local output
-o, --output string Output format. One of json or yaml. Used with the --dry-run flag.
--project string Project to use (must be set on GCE)
--set strings Directly set values in the spec
--ssh-access strings Restrict SSH access to this CIDR. If not set, uses the value of the admin-access flag.
--ssh-public-key string SSH public key to use
--subnets strings Shared subnets to use
--target string Valid targets: direct, terraform, cloudformation. Set this flag to terraform if you want kOps to generate terraform (default "direct")
-t, --topology string Network topology for the cluster: 'public' or 'private'. Defaults to 'public' for IPv4 clusters and 'private' for IPv6 clusters.
--unset strings Directly unset values in the spec
--utility-subnets strings Shared utility subnets to use
-y, --yes Specify --yes to immediately create the cluster
--zones strings Zones in which to run the cluster
--admin-access strings Restrict API access to this CIDR. If not set, access will not be restricted by IP. (default [0.0.0.0/0,::/0])
--api-loadbalancer-type string Type of load balancer for the Kubernetes API: public or internal
--api-ssl-certificate string ARN of the SSL Certificate to use for the Kubernetes API load balancer (AWS only)
--associate-public-ip Specify --associate-public-ip=[true|false] to enable/disable association of public IP for master ASG and nodes. Default is 'true'.
--authorization string Authorization mode: AlwaysAllow or RBAC (default "RBAC")
--bastion Enable a bastion instance group. Only applies to private topology.
--bastion-image string Machine image for bastions. Takes precedence over --image
--channel string Channel for default versions and configuration to use (default "stable")
--cloud string Cloud provider to use - aws, digitalocean, gce, hetzner, openstack
--cloud-labels string A list of key/value pairs used to tag all instance groups (for example "Owner=John Doe,Team=Some Team").
--container-runtime string Container runtime to use: containerd, docker
--disable-subnet-tags Disable automatic subnet tagging
--discovery-store string A public location where we publish OIDC-compatible discovery information under a cluster-specific directory. Enables IRSA in AWS.
--dns string DNS type to use: public, private, none
--dns-zone string DNS hosted zone (defaults to longest matching zone)
--dry-run If true, only print the object that would be sent, without sending it. This flag can be used to create a cluster YAML or JSON manifest.
--encrypt-etcd-storage Generate key in AWS KMS and use it for encrypt etcd volumes
--etcd-storage-type string The default storage type for etcd members
--gce-service-account string Service account with which the GCE VM runs. Warning: if not set, VMs will run as default compute service account.
-h, --help help for cluster
--image string Machine image for all instances
--ipv6 Use IPv6 for the pod network (AWS only)
--kubernetes-feature-gates strings List of Kubernetes feature gates to enable/disable
--kubernetes-version string Version of Kubernetes to run (defaults to version in channel)
--master-count int32 Number of masters. Defaults to one master per master-zone
--master-image string Machine image for masters. Takes precedence over --image
--master-public-name string Domain name of the public Kubernetes API
--master-security-groups strings Additional precreated security groups to add to masters.
--master-size string Machine type for masters
--master-tenancy string Tenancy of the master group (AWS only): default or dedicated
--master-volume-size int32 Instance volume size (in GB) for masters
--master-zones strings Zones in which to run masters (must be an odd number)
--network-cidr string Network CIDR to use
--network-id string Shared Network or VPC to use
--networking string Networking mode. kubenet, external, weave, flannel-vxlan (or flannel), flannel-udp, calico, canal, kube-router, amazonvpc, cilium, cilium-etcd, cni. (default "cilium")
--node-count int32 Total number of worker nodes. Defaults to one node per zone
--node-image string Machine image for worker nodes. Takes precedence over --image
--node-security-groups strings Additional precreated security groups to add to worker nodes.
--node-size string Machine type for worker nodes
--node-tenancy string Tenancy of the node group (AWS only): default or dedicated
--node-volume-size int32 Instance volume size (in GB) for worker nodes
--os-dns-servers string comma separated list of DNS Servers which is used in network
--os-ext-net string External network to use with the openstack router
--os-ext-subnet string External floating subnet to use with the openstack router
--os-kubelet-ignore-az Attach volumes across availability zones
--os-lb-floating-subnet string External subnet to use with the Kubernetes API
--os-network string ID of the existing OpenStack network to use
--os-octavia Use octavia load balancer API
--os-octavia-provider string Octavia provider to use
--out string Path to write any local output
-o, --output string Output format. One of json or yaml. Used with the --dry-run flag.
--project string Project to use (must be set on GCE)
--set strings Directly set values in the spec
--ssh-access strings Restrict SSH access to this CIDR. If not set, uses the value of the admin-access flag.
--ssh-public-key string SSH public key to use
--subnets strings Shared subnets to use
--target string Valid targets: direct, terraform, cloudformation. Set this flag to terraform if you want kOps to generate terraform (default "direct")
-t, --topology string Network topology for the cluster: 'public' or 'private'. Defaults to 'public' for IPv4 clusters and 'private' for IPv6 clusters.
--unset strings Directly unset values in the spec
--utility-subnets strings Shared utility subnets to use
-y, --yes Specify --yes to immediately create the cluster
--zones strings Zones in which to run the cluster
```
### Options inherited from parent commands

121
tests/integration/create_cluster/minimal_feature-gates/expected-v1alpha2.yaml Normal file
View File

@ -0,0 +1,121 @@
apiVersion: kops.k8s.io/v1alpha2
kind: Cluster
metadata:
creationTimestamp: "2017-01-01T00:00:00Z"
name: minimal.example.com
spec:
api:
dns: {}
authorization:
rbac: {}
channel: stable
cloudProvider: aws
configBase: memfs://tests/minimal.example.com
etcdClusters:
- cpuRequest: 200m
etcdMembers:
- encryptedVolume: true
instanceGroup: master-us-test-1a
name: a
memoryRequest: 100Mi
name: main
- cpuRequest: 100m
etcdMembers:
- encryptedVolume: true
instanceGroup: master-us-test-1a
name: a
memoryRequest: 100Mi
name: events
iam:
allowContainerRegistry: true
legacy: false
kubeAPIServer:
featureGates:
APIResponseCompression: "false"
ReadWriteOncePod: "true"
SELinuxMountReadWriteOncePod: "true"
kubeControllerManager:
featureGates:
APIResponseCompression: "false"
ReadWriteOncePod: "true"
SELinuxMountReadWriteOncePod: "true"
kubeProxy:
featureGates:
APIResponseCompression: "false"
ReadWriteOncePod: "true"
SELinuxMountReadWriteOncePod: "true"
kubeScheduler:
featureGates:
APIResponseCompression: "false"
ReadWriteOncePod: "true"
SELinuxMountReadWriteOncePod: "true"
kubelet:
anonymousAuth: false
featureGates:
APIResponseCompression: "false"
ReadWriteOncePod: "true"
SELinuxMountReadWriteOncePod: "true"
kubernetesApiAccess:
- 0.0.0.0/0
- ::/0
kubernetesVersion: v1.26.0
masterPublicName: api.minimal.example.com
networkCIDR: 172.20.0.0/16
networking:
cni: {}
nonMasqueradeCIDR: 100.64.0.0/10
sshAccess:
- 0.0.0.0/0
- ::/0
subnets:
- cidr: 172.20.32.0/19
name: us-test-1a
type: Public
zone: us-test-1a
topology:
dns:
type: Public
masters: public
nodes: public
---
apiVersion: kops.k8s.io/v1alpha2
kind: InstanceGroup
metadata:
creationTimestamp: "2017-01-01T00:00:00Z"
labels:
kops.k8s.io/cluster: minimal.example.com
name: master-us-test-1a
spec:
image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20221018
instanceMetadata:
httpPutResponseHopLimit: 3
httpTokens: required
machineType: m3.medium
maxSize: 1
minSize: 1
role: Master
subnets:
- us-test-1a
---
apiVersion: kops.k8s.io/v1alpha2
kind: InstanceGroup
metadata:
creationTimestamp: "2017-01-01T00:00:00Z"
labels:
kops.k8s.io/cluster: minimal.example.com
name: nodes-us-test-1a
spec:
image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20221018
instanceMetadata:
httpPutResponseHopLimit: 1
httpTokens: required
machineType: t2.medium
maxSize: 1
minSize: 1
role: Node
subnets:
- us-test-1a

10
tests/integration/create_cluster/minimal_feature-gates/options.yaml Normal file
View File

@ -0,0 +1,10 @@
ClusterName: minimal.example.com
Zones:
- us-test-1a
CloudProvider: aws
Networking: cni
KubernetesVersion: v1.26.0
KubernetesFeatureGates:
- SELinuxMountReadWriteOncePod
- +ReadWriteOncePod
- -APIResponseCompression

57
upup/pkg/fi/cloudup/new_cluster.go
View File

@ -30,7 +30,6 @@ import (
"k8s.io/klog/v2"
"k8s.io/kops"
api "k8s.io/kops/pkg/apis/kops"
kopsapi "k8s.io/kops/pkg/apis/kops"
"k8s.io/kops/pkg/apis/kops/model"
"k8s.io/kops/pkg/apis/kops/util"
"k8s.io/kops/pkg/client/simple"
@ -64,6 +63,8 @@ type NewClusterOptions struct {
DiscoveryStore string
// KubernetesVersion is the version of Kubernetes to deploy. It defaults to the version recommended by the channel.
KubernetesVersion string
// KubernetesFeatureGates is the list of Kubernetes feature gates to enable/disable.
KubernetesFeatureGates []string
// AdminAccess is the set of CIDR blocks permitted to connect to the Kubernetes API. It defaults to "0.0.0.0/0" and "::/0".
AdminAccess []string
// SSHAccess is the set of CIDR blocks permitted to connect to SSH on the nodes. It defaults to the value of AdminAccess.
@ -237,6 +238,38 @@ func NewCluster(opt *NewClusterOptions, clientset simple.Clientset) (*NewCluster
AnonymousAuth: fi.PtrTo(false),
}
if len(opt.KubernetesFeatureGates) > 0 {
cluster.Spec.Kubelet.FeatureGates = make(map[string]string)
cluster.Spec.KubeAPIServer = &api.KubeAPIServerConfig{
FeatureGates: make(map[string]string),
}
cluster.Spec.KubeControllerManager = &api.KubeControllerManagerConfig{
FeatureGates: make(map[string]string),
}
cluster.Spec.KubeProxy = &api.KubeProxyConfig{
FeatureGates: make(map[string]string),
}
cluster.Spec.KubeScheduler = &api.KubeSchedulerConfig{
FeatureGates: make(map[string]string),
}
for _, featureGate := range opt.KubernetesFeatureGates {
enabled := true
if featureGate[0] == '+' {
featureGate = featureGate[1:]
}
if featureGate[0] == '-' {
enabled = false
featureGate = featureGate[1:]
}
cluster.Spec.Kubelet.FeatureGates[featureGate] = strconv.FormatBool(enabled)
cluster.Spec.KubeAPIServer.FeatureGates[featureGate] = strconv.FormatBool(enabled)
cluster.Spec.KubeControllerManager.FeatureGates[featureGate] = strconv.FormatBool(enabled)
cluster.Spec.KubeProxy.FeatureGates[featureGate] = strconv.FormatBool(enabled)
cluster.Spec.KubeScheduler.FeatureGates[featureGate] = strconv.FormatBool(enabled)
}
}
if len(opt.AdminAccess) == 0 {
opt.AdminAccess = []string{"0.0.0.0/0", "::/0"}
}
@ -416,7 +449,7 @@ func NewCluster(opt *NewClusterOptions, clientset simple.Clientset) (*NewCluster
}
}
} else if g.Spec.Role == kopsapi.InstanceGroupRoleBastion {
} else if g.Spec.Role == api.InstanceGroupRoleBastion {
if g.Spec.MachineType == "" {
g.Spec.MachineType, err = defaultMachineType(cloud, &cluster, g)
if err != nil {
@ -438,7 +471,7 @@ func NewCluster(opt *NewClusterOptions, clientset simple.Clientset) (*NewCluster
if ig.Spec.Tenancy != "" && ig.Spec.Tenancy != "default" {
switch cluster.Spec.GetCloudProvider() {
case kopsapi.CloudProviderAWS:
case api.CloudProviderAWS:
if _, ok := awsDedicatedInstanceExceptions[g.Spec.MachineType]; ok {
return nil, fmt.Errorf("invalid dedicated instance type: %s", g.Spec.MachineType)
}
@ -454,7 +487,7 @@ func NewCluster(opt *NewClusterOptions, clientset simple.Clientset) (*NewCluster
} else if ig.IsAPIServerOnly() && cluster.Spec.IsIPv6Only() {
if len(ig.Spec.Subnets) == 0 {
for _, subnet := range cluster.Spec.Subnets {
if subnet.Type != kopsapi.SubnetTypePrivate && subnet.Type != kopsapi.SubnetTypeUtility {
if subnet.Type != api.SubnetTypePrivate && subnet.Type != api.SubnetTypeUtility {
ig.Spec.Subnets = append(g.Spec.Subnets, subnet.Name)
}
}
@ -462,7 +495,7 @@ func NewCluster(opt *NewClusterOptions, clientset simple.Clientset) (*NewCluster
} else {
if len(ig.Spec.Subnets) == 0 {
for _, subnet := range cluster.Spec.Subnets {
if subnet.Type != kopsapi.SubnetTypeDualStack && subnet.Type != kopsapi.SubnetTypeUtility {
if subnet.Type != api.SubnetTypeDualStack && subnet.Type != api.SubnetTypeUtility {
g.Spec.Subnets = append(g.Spec.Subnets, subnet.Name)
}
}
@ -470,7 +503,7 @@ func NewCluster(opt *NewClusterOptions, clientset simple.Clientset) (*NewCluster
if len(g.Spec.Subnets) == 0 {
for _, subnet := range cluster.Spec.Subnets {
if subnet.Type != kopsapi.SubnetTypeUtility {
if subnet.Type != api.SubnetTypeUtility {
g.Spec.Subnets = append(g.Spec.Subnets, subnet.Name)
}
}
@ -1125,9 +1158,9 @@ func setupTopology(opt *NewClusterOptions, cluster *api.Cluster, allZones sets.S
if opt.Topology == "" {
if opt.IPv6 {
opt.Topology = kopsapi.TopologyPrivate
opt.Topology = api.TopologyPrivate
} else {
opt.Topology = kopsapi.TopologyPublic
opt.Topology = api.TopologyPublic
}
}
@ -1242,7 +1275,7 @@ func setupTopology(opt *NewClusterOptions, cluster *api.Cluster, allZones sets.S
}
if opt.IPv6 {
for _, s := range cluster.Spec.Subnets {
if s.Type == kopsapi.SubnetTypeDualStack {
if s.Type == api.SubnetTypeDualStack {
bastionGroup.Spec.Subnets = append(bastionGroup.Spec.Subnets, s.Name)
}
}
@ -1460,7 +1493,7 @@ func addCiliumNetwork(cluster *api.Cluster) {
}
// defaultImage returns the default Image, based on the cloudprovider
func defaultImage(cluster *kopsapi.Cluster, channel *kopsapi.Channel, architecture architectures.Architecture) string {
func defaultImage(cluster *api.Cluster, channel *api.Channel, architecture architectures.Architecture) string {
if channel != nil {
var kubernetesVersion *semver.Version
if cluster.Spec.KubernetesVersion != "" {
@ -1479,7 +1512,7 @@ func defaultImage(cluster *kopsapi.Cluster, channel *kopsapi.Channel, architectu
}
switch cluster.Spec.GetCloudProvider() {
case kopsapi.CloudProviderDO:
case api.CloudProviderDO:
return defaultDONodeImage
}
klog.Infof("Cannot set default Image for CloudProvider=%q", cluster.Spec.GetCloudProvider())
@ -1497,7 +1530,7 @@ func MachineArchitecture(cloud fi.Cloud, machineType string) (architectures.Arch
}
switch cloud.ProviderID() {
case kopsapi.CloudProviderAWS:
case api.CloudProviderAWS:
info, err := cloud.(awsup.AWSCloud).DescribeInstanceType(machineType)
if err != nil {
return "", fmt.Errorf("error finding instance info for instance type %q: %w", machineType, err)