Merge pull request #9137 from olemarkus/cni-package

Move networking in nodeup to dedicated subpackage

hack/.packages

@@ -44,6 +44,7 @@ k8s.io/kops/node-authorizer/pkg/utils
 k8s.io/kops/nodeup/pkg/bootstrap
 k8s.io/kops/nodeup/pkg/distros
 k8s.io/kops/nodeup/pkg/model
+k8s.io/kops/nodeup/pkg/model/networking
 k8s.io/kops/nodeup/pkg/model/resources
 k8s.io/kops/pkg/acls
 k8s.io/kops/pkg/acls/gce

nodeup/pkg/model/BUILD.bazel

@@ -4,7 +4,6 @@ go_library(
     name = "go_default_library",
     srcs = [
         "architecture.go",
-        "cilium.go",
         "cloudconfig.go",
         "containerd.go",
         "context.go",
@@ -13,7 +12,6 @@ go_library(
         "docker.go",
         "etcd.go",
         "etcd_manager_tls.go",
-        "etcd_tls.go",
         "file_assets.go",
         "firewall.go",
         "hooks.go",
@@ -21,14 +19,12 @@ go_library(
         "kube_apiserver_healthcheck.go",
         "kube_controller_manager.go",
         "kube_proxy.go",
-        "kube_router.go",
         "kube_scheduler.go",
         "kubectl.go",
         "kubelet.go",
         "logrotate.go",
         "manifests.go",
         "miscutils.go",
-        "network.go",
         "node_authorizer.go",
         "ntp.go",
         "packages.go",
@@ -73,7 +69,6 @@ go_library(
         "//vendor/github.com/aws/aws-sdk-go/aws/session:go_default_library",
         "//vendor/github.com/aws/aws-sdk-go/service/ec2:go_default_library",
         "//vendor/github.com/blang/semver:go_default_library",
-        "//vendor/golang.org/x/sys/unix:go_default_library",
         "//vendor/k8s.io/api/core/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",

nodeup/pkg/model/context.go

@@ -170,12 +170,6 @@ func (c *NodeupModelContext) PathSrvSshproxy() string {
 	}
 }
 
-// CNIBinDir returns the path for the CNI binaries
-func (c *NodeupModelContext) CNIBinDir() string {
-	// We used to map this on a per-distro basis, but this can require CNI manifests to be distro aware
-	return "/opt/cni/bin/"
-}
-
 // KubeletBootstrapKubeconfig is the path the bootstrap config file
 func (c *NodeupModelContext) KubeletBootstrapKubeconfig() string {
 	path := c.Cluster.Spec.Kubelet.BootstrapKubeconfig
@@ -198,11 +192,6 @@ func (c *NodeupModelContext) KubeletKubeConfig() string {
 	return "/var/lib/kubelet/kubeconfig"
 }
 
-// CNIConfDir returns the CNI directory
-func (c *NodeupModelContext) CNIConfDir() string {
-	return "/etc/cni/net.d/"
-}
-
 // BuildPKIKubeconfig generates a kubeconfig
 func (c *NodeupModelContext) BuildPKIKubeconfig(name string) (string, error) {
 	ca, err := c.GetCert(fi.CertificateId_CA)
@@ -584,3 +573,53 @@ func (c *NodeupModelContext) GetPrivateKey(name string) ([]byte, error) {
 
 	return key.AsBytes()
 }
+
+func (b *NodeupModelContext) AddCNIBinAssets(c *fi.ModelBuilderContext, assetNames []string) error {
+	for _, assetName := range assetNames {
+		if err := b.addCNIBinAsset(c, assetName); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (b *NodeupModelContext) addCNIBinAsset(c *fi.ModelBuilderContext, assetName string) error {
+	assetPath := ""
+	asset, err := b.Assets.Find(assetName, assetPath)
+	if err != nil {
+		return fmt.Errorf("error trying to locate asset %q: %v", assetName, err)
+	}
+	if asset == nil {
+		return fmt.Errorf("unable to locate asset %q", assetName)
+	}
+
+	c.AddTask(&nodetasks.File{
+		Path:     filepath.Join(b.CNIBinDir(), assetName),
+		Contents: asset,
+		Type:     nodetasks.FileType_File,
+		Mode:     fi.String("0755"),
+	})
+
+	return nil
+}
+
+// UsesCNI checks if the cluster has CNI configured
+func (c *NodeupModelContext) UsesCNI() bool {
+	networking := c.Cluster.Spec.Networking
+	if networking == nil || networking.Classic != nil {
+		return false
+	}
+
+	return true
+}
+
+// CNIBinDir returns the path for the CNI binaries
+func (c *NodeupModelContext) CNIBinDir() string {
+	// We used to map this on a per-distro basis, but this can require CNI manifests to be distro aware
+	return "/opt/cni/bin/"
+}
+
+// CNIConfDir returns the CNI directory
+func (c *NodeupModelContext) CNIConfDir() string {
+	return "/etc/cni/net.d/"
+}

nodeup/pkg/model/network.go

@@ -1,123 +0,0 @@
-/*
-Copyright 2019 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package model
-
-import (
-	"fmt"
-	"path/filepath"
-
-	"k8s.io/klog"
-	"k8s.io/kops/pkg/systemd"
-	"k8s.io/kops/upup/pkg/fi"
-	"k8s.io/kops/upup/pkg/fi/nodeup/nodetasks"
-)
-
-// NetworkBuilder writes CNI assets
-type NetworkBuilder struct {
-	*NodeupModelContext
-}
-
-var _ fi.ModelBuilder = &NetworkBuilder{}
-
-// Build is responsible for configuring the network cni
-func (b *NetworkBuilder) Build(c *fi.ModelBuilderContext) error {
-	var assetNames []string
-
-	// @TODO need to clean up this code, it isn't the easiest to read
-	networking := b.Cluster.Spec.Networking
-	if networking == nil || networking.Classic != nil {
-	} else if networking.Kubenet != nil || networking.GCE != nil {
-		assetNames = append(assetNames, "bridge", "host-local", "loopback")
-	} else if networking.External != nil {
-		// external is based on kubenet
-		assetNames = append(assetNames, "bridge", "host-local", "loopback")
-
-	} else if networking.CNI != nil || networking.Weave != nil || networking.Flannel != nil || networking.Calico != nil || networking.Canal != nil || networking.Kuberouter != nil || networking.AmazonVPC != nil || networking.Cilium != nil {
-		assetNames = append(assetNames, "bridge", "host-local", "loopback", "ptp", "portmap")
-		// Do we need tuning?
-
-		// TODO: Only when using flannel ?
-		assetNames = append(assetNames, "flannel")
-	} else if networking.Kopeio != nil {
-		// TODO combine with External
-		// Kopeio is based on kubenet / external
-		assetNames = append(assetNames, "bridge", "host-local", "loopback")
-	} else if networking.LyftVPC != nil {
-		assetNames = append(assetNames, "cni-ipvlan-vpc-k8s-ipam", "cni-ipvlan-vpc-k8s-ipvlan", "cni-ipvlan-vpc-k8s-tool", "cni-ipvlan-vpc-k8s-unnumbered-ptp", "loopback")
-	} else {
-		return fmt.Errorf("no networking mode set")
-	}
-
-	for _, assetName := range assetNames {
-		if err := b.addCNIBinAsset(c, assetName); err != nil {
-			return err
-		}
-	}
-
-	// Tx checksum offloading is buggy for NAT-ed VXLAN endpoints, leading to an invalid checksum sent and causing
-	// Flannel to stop to working as the traffic is being discarded by the receiver.
-	// https://github.com/coreos/flannel/issues/1279
-	if networking != nil && (networking.Canal != nil || (networking.Flannel != nil && networking.Flannel.Backend == "vxlan")) {
-		c.AddTask(b.buildFlannelTxChecksumOffloadDisableService())
-	}
-
-	return nil
-}
-
-func (b *NetworkBuilder) addCNIBinAsset(c *fi.ModelBuilderContext, assetName string) error {
-	assetPath := ""
-	asset, err := b.Assets.Find(assetName, assetPath)
-	if err != nil {
-		return fmt.Errorf("error trying to locate asset %q: %v", assetName, err)
-	}
-	if asset == nil {
-		return fmt.Errorf("unable to locate asset %q", assetName)
-	}
-
-	c.AddTask(&nodetasks.File{
-		Path:     filepath.Join(b.CNIBinDir(), assetName),
-		Contents: asset,
-		Type:     nodetasks.FileType_File,
-		Mode:     s("0755"),
-	})
-
-	return nil
-}
-
-func (b *NetworkBuilder) buildFlannelTxChecksumOffloadDisableService() *nodetasks.Service {
-	const serviceName = "flannel-tx-checksum-offload-disable.service"
-
-	manifest := &systemd.Manifest{}
-	manifest.Set("Unit", "Description", "Disable TX checksum offload on flannel.1")
-
-	manifest.Set("Unit", "After", "sys-devices-virtual-net-flannel.1.device")
-	manifest.Set("Install", "WantedBy", "sys-devices-virtual-net-flannel.1.device")
-	manifest.Set("Service", "Type", "oneshot")
-	manifest.Set("Service", "ExecStart", "/sbin/ethtool -K flannel.1 tx-checksum-ip-generic off")
-
-	manifestString := manifest.Render()
-	klog.V(8).Infof("Built service manifest %q\n%s", serviceName, manifestString)
-
-	service := &nodetasks.Service{
-		Name:       serviceName,
-		Definition: s(manifestString),
-	}
-
-	service.InitDefaults()
-
-	return service
-}

nodeup/pkg/model/networking/BUILD.bazel

@@ -0,0 +1,31 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "calico.go",
+        "cilium.go",
+        "flannel.go",
+        "kube_router.go",
+        "kubenet.go",
+        "lyft.go",
+    ],
+    importpath = "k8s.io/kops/nodeup/pkg/model/networking",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//nodeup/pkg/model:go_default_library",
+        "//pkg/apis/kops:go_default_library",
+        "//pkg/model/components:go_default_library",
+        "//pkg/pki:go_default_library",
+        "//pkg/systemd:go_default_library",
+        "//upup/pkg/fi:go_default_library",
+        "//upup/pkg/fi/nodeup/nodetasks:go_default_library",
+        "//vendor/github.com/aws/aws-sdk-go/aws:go_default_library",
+        "//vendor/github.com/aws/aws-sdk-go/aws/ec2metadata:go_default_library",
+        "//vendor/github.com/aws/aws-sdk-go/aws/request:go_default_library",
+        "//vendor/github.com/aws/aws-sdk-go/aws/session:go_default_library",
+        "//vendor/github.com/aws/aws-sdk-go/service/ec2:go_default_library",
+        "//vendor/golang.org/x/sys/unix:go_default_library",
+        "//vendor/k8s.io/klog:go_default_library",
+    ],
+)

nodeup/pkg/model/networking/calico.go

@@ -14,23 +14,28 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package model
+package networking
 
 import (
 	"path/filepath"
 
+	"k8s.io/kops/nodeup/pkg/model"
 	"k8s.io/kops/upup/pkg/fi"
 )
 
-// EtcdTLSBuilder configures the etcd TLS support
+// CalicoBuilder configures the etcd TLS support for Calico
-type EtcdTLSBuilder struct {
+type CalicoBuilder struct {
-	*NodeupModelContext
+	*model.NodeupModelContext
 }
 
-var _ fi.ModelBuilder = &EtcdTLSBuilder{}
+var _ fi.ModelBuilder = &CalicoBuilder{}
 
 // Build is responsible for performing setup for CNIs that need etcd TLS support
-func (b *EtcdTLSBuilder) Build(c *fi.ModelBuilderContext) error {
+func (b *CalicoBuilder) Build(c *fi.ModelBuilderContext) error {
+	if b.Cluster.Spec.Networking.Calico == nil {
+		return nil
+	}
+
 	// @check if tls is enabled and if so, we need to download the client certificates
 	if !b.UseEtcdManager() && b.UseEtcdTLS() {
 		name := "calico-client"

nodeup/pkg/model/networking/cilium.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package model
+package networking
 
 import (
 	"crypto/x509"
@@ -24,6 +24,8 @@ import (
 	"path/filepath"
 	"time"
 
+	"k8s.io/kops/nodeup/pkg/model"
+
 	"golang.org/x/sys/unix"
 	"k8s.io/klog"
 	"k8s.io/kops/pkg/pki"
@@ -33,7 +35,7 @@ import (
 
 // CiliumBuilder writes Cilium's assets
 type CiliumBuilder struct {
-	*NodeupModelContext
+	*model.NodeupModelContext
 }
 
 var _ fi.ModelBuilder = &CiliumBuilder{}
@@ -76,7 +78,7 @@ func (b *CiliumBuilder) buildBPFMount(c *fi.ModelBuilderContext) error {
 	alreadyMounted := uint32(fsdata.Type) == BPF_FS_MAGIC
 
 	if !alreadyMounted {
-		unit := s(`
+		unit := `
 [Unit]
 Description=Cilium BPF mounts
 Documentation=http://docs.cilium.io/
@@ -90,13 +92,12 @@ Type=bpf
 
 [Install]
 WantedBy=multi-user.target
-`)
+`
 
 		service := &nodetasks.Service{
 			Name:       "sys-fs-bpf.mount",
-			Definition: unit,
+			Definition: fi.String(unit),
 		}
-		service.InitDefaults()
 		c.AddTask(service)
 	}
 

nodeup/pkg/model/networking/flannel.go

@@ -0,0 +1,74 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package networking
+
+import (
+	"k8s.io/klog"
+	"k8s.io/kops/nodeup/pkg/model"
+	"k8s.io/kops/pkg/systemd"
+
+	"k8s.io/kops/upup/pkg/fi"
+	"k8s.io/kops/upup/pkg/fi/nodeup/nodetasks"
+)
+
+// FlannelBuilder writes Flannel's assets
+type FlannelBuilder struct {
+	*model.NodeupModelContext
+}
+
+var _ fi.ModelBuilder = &FlannelBuilder{}
+
+// Build is responsible for configuring the network cni
+func (b *FlannelBuilder) Build(c *fi.ModelBuilderContext) error {
+	networking := b.Cluster.Spec.Networking
+
+	if networking.Flannel != nil {
+		b.AddCNIBinAssets(c, []string{"flannel", "portmap", "bridge", "host-local", "loopback"})
+	}
+
+	if networking.Canal != nil || networking.Flannel != nil && networking.Flannel.Backend == "vxlan" {
+		return b.buildFlannelTxChecksumOffloadDisableService(c)
+	}
+	return nil
+}
+
+// Tx checksum offloading is buggy for NAT-ed VXLAN endpoints, leading to an invalid checksum sent and causing
+// Flannel to stop to working as the traffic is being discarded by the receiver.
+// https://github.com/coreos/flannel/issues/1279
+func (b *FlannelBuilder) buildFlannelTxChecksumOffloadDisableService(c *fi.ModelBuilderContext) error {
+	const serviceName = "flannel-tx-checksum-offload-disable.service"
+
+	manifest := &systemd.Manifest{}
+	manifest.Set("Unit", "Description", "Disable TX checksum offload on flannel.1")
+
+	manifest.Set("Unit", "After", "sys-devices-virtual-net-flannel.1.device")
+	manifest.Set("Install", "WantedBy", "sys-devices-virtual-net-flannel.1.device")
+	manifest.Set("Service", "Type", "oneshot")
+	manifest.Set("Service", "ExecStart", "/sbin/ethtool -K flannel.1 tx-checksum-ip-generic off")
+
+	manifestString := manifest.Render()
+	klog.V(8).Infof("Built service manifest %q\n%s", serviceName, manifestString)
+
+	service := &nodetasks.Service{
+		Name:       serviceName,
+		Definition: fi.String(manifestString),
+	}
+
+	c.AddTask(service)
+
+	return nil
+}

nodeup/pkg/model/networking/kube_router.go

@@ -14,22 +14,26 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package model
+package networking
 
 import (
+	"k8s.io/kops/nodeup/pkg/model"
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/upup/pkg/fi/nodeup/nodetasks"
 )
 
-// KubeRouterBuilder installs kube-router
+// KuberouterBuilder installs kube-router
-type KubeRouterBuilder struct {
+type KuberouterBuilder struct {
-	*NodeupModelContext
+	*model.NodeupModelContext
 }
 
-var _ fi.ModelBuilder = &KubeRouterBuilder{}
+var _ fi.ModelBuilder = &KuberouterBuilder{}
 
 // Build is responsible for configuring the kube-router
-func (b *KubeRouterBuilder) Build(c *fi.ModelBuilderContext) error {
+func (b *KuberouterBuilder) Build(c *fi.ModelBuilderContext) error {
+	if b.Cluster.Spec.Networking.Kuberouter == nil {
+		return nil
+	}
 	{
 		kubeconfig, err := b.BuildPKIKubeconfig("kube-router")
 		if err != nil {
@@ -40,9 +44,11 @@ func (b *KubeRouterBuilder) Build(c *fi.ModelBuilderContext) error {
 			Path:     "/var/lib/kube-router/kubeconfig",
 			Contents: fi.NewStringResource(kubeconfig),
 			Type:     nodetasks.FileType_File,
-			Mode:     s("0400"),
+			Mode:     fi.String("0400"),
 		})
 	}
 
+	b.AddCNIBinAssets(c, []string{"loopback", "host-local", "bridge", "portmap"})
+
 	return nil
 }

nodeup/pkg/model/networking/kubenet.go

@@ -0,0 +1,42 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package networking
+
+import (
+	"k8s.io/kops/nodeup/pkg/model"
+	"k8s.io/kops/pkg/model/components"
+	"k8s.io/kops/upup/pkg/fi"
+)
+
+// KubenetBuilder installs Kubenet networking provider
+type KubenetBuilder struct {
+	*model.NodeupModelContext
+}
+
+var _ fi.ModelBuilder = &KubenetBuilder{}
+
+// Build is responsible for configuring the kube-router
+func (b *KubenetBuilder) Build(c *fi.ModelBuilderContext) error {
+	usesKubenet := components.UsesKubenet(b.Cluster.Spec.Networking)
+	if !usesKubenet {
+		return nil
+	}
+
+	b.AddCNIBinAssets(c, []string{"loopback", "host-local", "bridge"})
+
+	return nil
+}

nodeup/pkg/model/networking/lyft.go

@@ -0,0 +1,142 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package networking
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+	"text/template"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/ec2metadata"
+	"github.com/aws/aws-sdk-go/aws/request"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"k8s.io/klog"
+	"k8s.io/kops/nodeup/pkg/model"
+	api "k8s.io/kops/pkg/apis/kops"
+	"k8s.io/kops/upup/pkg/fi"
+)
+
+type LyftVPCBuilder struct {
+	*model.NodeupModelContext
+}
+
+var _ fi.ModelBuilder = &LyftVPCBuilder{}
+
+// Build is responsible for configuring the network cni
+func (b *LyftVPCBuilder) Build(c *fi.ModelBuilderContext) error {
+	networking := b.Cluster.Spec.Networking
+
+	if networking.LyftVPC == nil {
+		return nil
+	}
+
+	assetNames := []string{"loopback", "cni-ipvlan-vpc-k8s-ipam", "cni-ipvlan-vpc-k8s-ipvlan", "cni-ipvlan-vpc-k8s-tool", "cni-ipvlan-vpc-k8s-unnumbered-ptp"}
+
+	return b.AddCNIBinAssets(c, assetNames)
+}
+
+func LoadLyftTemplateFunctions(templateFunctions template.FuncMap, cluster *api.Cluster) {
+	templateFunctions["SubnetTags"] = func() (string, error) {
+		var tags map[string]string
+		if cluster.IsKubernetesGTE("1.18") {
+			tags = map[string]string{
+				"KubernetesCluster": cluster.Name,
+			}
+		} else {
+			tags = map[string]string{
+				"Type": "pod",
+			}
+		}
+		if len(cluster.Spec.Networking.LyftVPC.SubnetTags) > 0 {
+			tags = cluster.Spec.Networking.LyftVPC.SubnetTags
+		}
+
+		bytes, err := json.Marshal(tags)
+		if err != nil {
+			return "", err
+		}
+		return string(bytes), nil
+	}
+
+	templateFunctions["NodeSecurityGroups"] = func() (string, error) {
+		// use the same security groups as the node
+		ids, err := evaluateSecurityGroups(cluster.Spec.NetworkID)
+		if err != nil {
+			return "", err
+		}
+		bytes, err := json.Marshal(ids)
+		if err != nil {
+			return "", err
+		}
+		return string(bytes), nil
+	}
+
+}
+
+func evaluateSecurityGroups(vpcId string) ([]string, error) {
+	config := aws.NewConfig()
+	config = config.WithCredentialsChainVerboseErrors(true)
+
+	s, err := session.NewSession(config)
+	if err != nil {
+		return nil, fmt.Errorf("error starting new AWS session: %v", err)
+	}
+	s.Handlers.Send.PushFront(func(r *request.Request) {
+		// Log requests
+		klog.V(4).Infof("AWS API Request: %s/%s", r.ClientInfo.ServiceName, r.Operation.Name)
+	})
+
+	metadata := ec2metadata.New(s, config)
+
+	region, err := metadata.Region()
+	if err != nil {
+		return nil, fmt.Errorf("error querying ec2 metadata service (for az/region): %v", err)
+	}
+
+	sgNames, err := metadata.GetMetadata("security-groups")
+	if err != nil {
+		return nil, fmt.Errorf("error querying ec2 metadata service (for security-groups): %v", err)
+	}
+	svc := ec2.New(s, config.WithRegion(region))
+
+	result, err := svc.DescribeSecurityGroups(&ec2.DescribeSecurityGroupsInput{
+		Filters: []*ec2.Filter{
+			{
+				Name:   aws.String("group-name"),
+				Values: aws.StringSlice(strings.Fields(sgNames)),
+			},
+			{
+				Name:   aws.String("vpc-id"),
+				Values: []*string{aws.String(vpcId)},
+			},
+		},
+	})
+
+	if err != nil {
+		return nil, fmt.Errorf("error looking up instance security group ids: %v", err)
+	}
+	var sgIds []string
+	for _, group := range result.SecurityGroups {
+		sgIds = append(sgIds, *group.GroupId)
+	}
+
+	return sgIds, nil
+
+}

pkg/apis/kops/validation/validation.go

@@ -368,6 +368,9 @@ func validateNetworking(c *kops.ClusterSpec, v *kops.NetworkingSpec, fldPath *fi
 		if optionTaken {
 			allErrs = append(allErrs, field.Forbidden(fldPath.Child("kuberouter"), "only one networking option permitted"))
 		}
+		if c.KubeProxy != nil && (c.KubeProxy.Enabled == nil || *c.KubeProxy.Enabled) {
+			allErrs = append(allErrs, field.Forbidden(fldPath.Root().Child("spec", "kubeProxy", "enabled"), "kube-router requires kubeProxy to be disabled"))
+		}
 		optionTaken = true
 	}
 

upup/pkg/fi/nodeup/BUILD.bazel

@@ -11,6 +11,7 @@ go_library(
     deps = [
         "//nodeup/pkg/distros:go_default_library",
         "//nodeup/pkg/model:go_default_library",
+        "//nodeup/pkg/model/networking:go_default_library",
         "//pkg/apis/kops:go_default_library",
         "//pkg/apis/kops/registry:go_default_library",
         "//pkg/apis/nodeup:go_default_library",
@@ -24,8 +25,6 @@ go_library(
         "//upup/pkg/fi/utils:go_default_library",
         "//util/pkg/vfs:go_default_library",
         "//vendor/github.com/aws/aws-sdk-go/aws:go_default_library",
-        "//vendor/github.com/aws/aws-sdk-go/aws/ec2metadata:go_default_library",
-        "//vendor/github.com/aws/aws-sdk-go/aws/request:go_default_library",
         "//vendor/github.com/aws/aws-sdk-go/aws/session:go_default_library",
         "//vendor/github.com/aws/aws-sdk-go/service/ec2:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",

upup/pkg/fi/nodeup/command.go

@@ -17,7 +17,6 @@ limitations under the License.
 package nodeup
 
 import (
-	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
@@ -30,6 +29,7 @@ import (
 
 	"k8s.io/kops/nodeup/pkg/distros"
 	"k8s.io/kops/nodeup/pkg/model"
+	"k8s.io/kops/nodeup/pkg/model/networking"
 	api "k8s.io/kops/pkg/apis/kops"
 	"k8s.io/kops/pkg/apis/kops/registry"
 	"k8s.io/kops/pkg/apis/nodeup"
@@ -43,8 +43,6 @@ import (
 	"k8s.io/kops/util/pkg/vfs"
 
 	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/aws/ec2metadata"
-	"github.com/aws/aws-sdk-go/aws/request"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"k8s.io/apimachinery/pkg/util/sets"
@@ -244,62 +242,23 @@ func (c *NodeUpCommand) Run(out io.Writer) error {
 	loader.Builders = append(loader.Builders, &model.PackagesBuilder{NodeupModelContext: modelContext})
 	loader.Builders = append(loader.Builders, &model.SecretBuilder{NodeupModelContext: modelContext})
 	loader.Builders = append(loader.Builders, &model.FirewallBuilder{NodeupModelContext: modelContext})
-	loader.Builders = append(loader.Builders, &model.NetworkBuilder{NodeupModelContext: modelContext})
 	loader.Builders = append(loader.Builders, &model.SysctlBuilder{NodeupModelContext: modelContext})
 	loader.Builders = append(loader.Builders, &model.KubeAPIServerBuilder{NodeupModelContext: modelContext})
 	loader.Builders = append(loader.Builders, &model.KubeControllerManagerBuilder{NodeupModelContext: modelContext})
 	loader.Builders = append(loader.Builders, &model.KubeSchedulerBuilder{NodeupModelContext: modelContext})
 	loader.Builders = append(loader.Builders, &model.EtcdManagerTLSBuilder{NodeupModelContext: modelContext})
+	loader.Builders = append(loader.Builders, &model.KubeProxyBuilder{NodeupModelContext: modelContext})
 
-	if c.cluster.Spec.Networking.Cilium != nil {
+	loader.Builders = append(loader.Builders, &networking.CalicoBuilder{NodeupModelContext: modelContext})
-		loader.Builders = append(loader.Builders, &model.CiliumBuilder{NodeupModelContext: modelContext})
+	loader.Builders = append(loader.Builders, &networking.CiliumBuilder{NodeupModelContext: modelContext})
-	}
+	// Canal = Flannel + Calico, so use this builder for both CNIs
-	if c.cluster.Spec.Networking.Kuberouter == nil {
+	loader.Builders = append(loader.Builders, &networking.FlannelBuilder{NodeupModelContext: modelContext})
-		loader.Builders = append(loader.Builders, &model.KubeProxyBuilder{NodeupModelContext: modelContext})
+	loader.Builders = append(loader.Builders, &networking.LyftVPCBuilder{NodeupModelContext: modelContext})
-	} else {
+	loader.Builders = append(loader.Builders, &networking.KuberouterBuilder{NodeupModelContext: modelContext})
-		loader.Builders = append(loader.Builders, &model.KubeRouterBuilder{NodeupModelContext: modelContext})
+	// Also handles kopeio as kopeio is based on kubenet
-	}
+	loader.Builders = append(loader.Builders, &networking.KubenetBuilder{NodeupModelContext: modelContext})
-	if c.cluster.Spec.Networking.Calico != nil {
-		loader.Builders = append(loader.Builders, &model.EtcdTLSBuilder{NodeupModelContext: modelContext})
-	}
 
-	if c.cluster.Spec.Networking.LyftVPC != nil {
+	networking.LoadLyftTemplateFunctions(loader.TemplateFunctions, c.cluster)
-
-		loader.TemplateFunctions["SubnetTags"] = func() (string, error) {
-			var tags map[string]string
-			if c.cluster.IsKubernetesGTE("1.18") {
-				tags = map[string]string{
-					"KubernetesCluster": c.cluster.Name,
-				}
-			} else {
-				tags = map[string]string{
-					"Type": "pod",
-				}
-			}
-			if len(c.cluster.Spec.Networking.LyftVPC.SubnetTags) > 0 {
-				tags = c.cluster.Spec.Networking.LyftVPC.SubnetTags
-			}
-
-			bytes, err := json.Marshal(tags)
-			if err != nil {
-				return "", err
-			}
-			return string(bytes), nil
-		}
-
-		loader.TemplateFunctions["NodeSecurityGroups"] = func() (string, error) {
-			// use the same security groups as the node
-			ids, err := evaluateSecurityGroups(c.cluster.Spec.NetworkID)
-			if err != nil {
-				return "", err
-			}
-			bytes, err := json.Marshal(ids)
-			if err != nil {
-				return "", err
-			}
-			return string(bytes), nil
-		}
-	}
 
 	taskMap, err := loader.Build(c.ModelDir)
 	if err != nil {
@@ -393,57 +352,6 @@ func evaluateSpec(c *api.Cluster) error {
 	return nil
 }
 
-func evaluateSecurityGroups(vpcId string) ([]string, error) {
-	config := aws.NewConfig()
-	config = config.WithCredentialsChainVerboseErrors(true)
-
-	s, err := session.NewSession(config)
-	if err != nil {
-		return nil, fmt.Errorf("error starting new AWS session: %v", err)
-	}
-	s.Handlers.Send.PushFront(func(r *request.Request) {
-		// Log requests
-		klog.V(4).Infof("AWS API Request: %s/%s", r.ClientInfo.ServiceName, r.Operation.Name)
-	})
-
-	metadata := ec2metadata.New(s, config)
-
-	region, err := metadata.Region()
-	if err != nil {
-		return nil, fmt.Errorf("error querying ec2 metadata service (for az/region): %v", err)
-	}
-
-	sgNames, err := metadata.GetMetadata("security-groups")
-	if err != nil {
-		return nil, fmt.Errorf("error querying ec2 metadata service (for security-groups): %v", err)
-	}
-	svc := ec2.New(s, config.WithRegion(region))
-
-	result, err := svc.DescribeSecurityGroups(&ec2.DescribeSecurityGroupsInput{
-		Filters: []*ec2.Filter{
-			{
-				Name:   aws.String("group-name"),
-				Values: aws.StringSlice(strings.Fields(sgNames)),
-			},
-			{
-				Name:   aws.String("vpc-id"),
-				Values: []*string{aws.String(vpcId)},
-			},
-		},
-	})
-
-	if err != nil {
-		return nil, fmt.Errorf("error looking up instance security group ids: %v", err)
-	}
-	var sgIds []string
-	for _, group := range result.SecurityGroups {
-		sgIds = append(sgIds, *group.GroupId)
-	}
-
-	return sgIds, nil
-
-}
-
 func evaluateHostnameOverride(hostnameOverride string) (string, error) {
 	if hostnameOverride == "" || hostnameOverride == "@hostname" {
 		return "", nil