From 7a4ac14d8daa4636957a589b5d0db00be98dba72 Mon Sep 17 00:00:00 2001 From: John Gardiner Myers Date: Thu, 10 Nov 2022 08:23:59 -0800 Subject: [PATCH] Change the default networking provider to Cilium --- docs/cli/kops_create_cluster.md | 2 +- docs/networking.md | 21 ++++++++------------- docs/releases/1.26-NOTES.md | 2 ++ upup/pkg/fi/cloudup/new_cluster.go | 2 +- 4 files changed, 12 insertions(+), 15 deletions(-) diff --git a/docs/cli/kops_create_cluster.md b/docs/cli/kops_create_cluster.md index 0d6d2092cd..de81df53e8 100644 --- a/docs/cli/kops_create_cluster.md +++ b/docs/cli/kops_create_cluster.md @@ -97,7 +97,7 @@ kops create cluster [CLUSTER] [flags] --master-zones strings Zones in which to run masters (must be an odd number) --network-cidr string Network CIDR to use --network-id string Shared Network or VPC to use - --networking string Networking mode. kubenet, external, weave, flannel-vxlan (or flannel), flannel-udp, calico, canal, kube-router, amazonvpc, cilium, cilium-etcd, cni. (default "kubenet") + --networking string Networking mode. kubenet, external, weave, flannel-vxlan (or flannel), flannel-udp, calico, canal, kube-router, amazonvpc, cilium, cilium-etcd, cni. (default "cilium") --node-count int32 Total number of worker nodes. Defaults to one node per zone --node-image string Machine image for worker nodes. Takes precedence over --image --node-security-groups strings Additional precreated security groups to add to worker nodes. diff --git a/docs/networking.md b/docs/networking.md index dc9b27192b..f396abde7a 100644 --- a/docs/networking.md +++ b/docs/networking.md @@ -12,6 +12,8 @@ listed below, are available which implement and manage this abstraction. The following table provides the support status for various networking providers with regards to kOps version: +As of kOps 1.26 the default network provider is Cilium. Prior to that the default is Kubenet. + | Network provider | Experimental | Stable | Deprecated | Removed | |------------------|-------------:|-------:|-----------:|----------------:| | AWS VPC | 1.9 | 1.21 | - | - | @@ -27,27 +29,20 @@ The following table provides the support status for various networking providers | Romana | 1.8 | - | 1.18 | 1.19 | | Weave | 1.5 | - | 1.23 | Kubernetes 1.23 | -### Which networking provider should you use? - -kOps maintainers have no bias over the CNI provider that you run, we only aim to be flexible and provide a working setup of the CNIs. - -We do recommended something other than `kubenet` for production clusters due to `kubenet`'s limitations, as explained [below](#kubenet-default). - ### Specifying network option for cluster creation You can specify the network provider via the `--networking` command line switch. However, this will only give a default configuration of the provider. Typically you would often modify the `spec.networking` section of the cluster spec to configure the provider further. -### Kubenet (default) +### Kubenet -Kubernetes Operations (kOps) uses `kubenet` networking by default. This sets up networking on AWS using VPC -networking, where the master allocates a /24 CIDR to each Node, drawing from the Node network. -Using `kubenet` mode routes for each node are then configured in the AWS VPC routing tables. +The "kubenet" option has the control plane allocate a /24 CIDR to each Node, drawing from the Node network. +Routes for each node are then configured in the cloud provider network's routing tables. -One important limitation when using `kubenet` networking is that an AWS routing table cannot have more than +One important limitation when using `kubenet` networking on AWS is that an AWS routing table cannot have more than 50 entries, which sets a limit of 50 nodes per cluster. AWS support will sometimes raise the limit to 100, but their documentation notes that routing tables over 50 may take a performance hit. -Because k8s modifies the AWS routing table, this means that realistically Kubernetes needs to own the +Because kubernetes modifies the AWS routing table, this means that, realistically, Kubernetes needs to own the routing table, and thus it requires its own subnet. It is theoretically possible to share a routing table with other infrastructure (but not a second cluster!), but this is not really recommended. Certain `cni` networking solutions claim to address these problems. @@ -56,7 +51,7 @@ Users running `--topology private` will not be able to choose `kubenet` networki requires a single routing table. These advanced users are usually running in multiple availability zones and as NAT gateways are single AZ, multiple route tables are needed to use each NAT gateway. -Kubenet is the default networking option because of its simplicity, however, it should not be used in +Kubenet is simple, however, it should not be used in production clusters which expect a gradual increase in traffic and/or workload over time. Such clusters will eventually "out-grow" the `kubenet` networking provider. diff --git a/docs/releases/1.26-NOTES.md b/docs/releases/1.26-NOTES.md index ee4130f7d7..a22ad92c8f 100644 --- a/docs/releases/1.26-NOTES.md +++ b/docs/releases/1.26-NOTES.md @@ -6,6 +6,8 @@ This is a document to gather the release notes prior to the release. # Significant changes +* The default networking provider for new clusters is now Cilium. + ## AWS only * Bastions are now fronted by a Network Load Balancer. diff --git a/upup/pkg/fi/cloudup/new_cluster.go b/upup/pkg/fi/cloudup/new_cluster.go index 4b2f3a8039..921a2e1feb 100644 --- a/upup/pkg/fi/cloudup/new_cluster.go +++ b/upup/pkg/fi/cloudup/new_cluster.go @@ -164,7 +164,7 @@ func (o *NewClusterOptions) InitDefaults() { o.Channel = api.DefaultChannel o.Authorization = AuthorizationFlagRBAC o.AdminAccess = []string{"0.0.0.0/0", "::/0"} - o.Networking = "kubenet" + o.Networking = "cilium" o.Topology = api.TopologyPublic o.InstanceManager = "cloudgroups" }