From b62f6aa894a2bab805e164bdbf94d5cf32a12cf4 Mon Sep 17 00:00:00 2001 From: Ole Markus With Date: Sat, 16 May 2020 11:09:35 +0200 Subject: [PATCH] Move networking in nodeup to dedicated subpackage --- hack/.packages | 1 + nodeup/pkg/model/BUILD.bazel | 4 - nodeup/pkg/model/context.go | 61 ++++++-- nodeup/pkg/model/network.go | 123 --------------- nodeup/pkg/model/networking/BUILD.bazel | 29 ++++ nodeup/pkg/model/{ => networking}/cilium.go | 17 ++- nodeup/pkg/model/networking/flannel.go | 75 +++++++++ .../pkg/model/{ => networking}/kube_router.go | 20 ++- nodeup/pkg/model/networking/kubenet.go | 40 +++++ nodeup/pkg/model/networking/lyft.go | 142 ++++++++++++++++++ pkg/apis/kops/validation/validation.go | 3 + upup/pkg/fi/nodeup/BUILD.bazel | 3 +- upup/pkg/fi/nodeup/command.go | 114 ++------------ 13 files changed, 374 insertions(+), 258 deletions(-) delete mode 100644 nodeup/pkg/model/network.go create mode 100644 nodeup/pkg/model/networking/BUILD.bazel rename nodeup/pkg/model/{ => networking}/cilium.go (95%) create mode 100644 nodeup/pkg/model/networking/flannel.go rename nodeup/pkg/model/{ => networking}/kube_router.go (69%) create mode 100644 nodeup/pkg/model/networking/kubenet.go create mode 100644 nodeup/pkg/model/networking/lyft.go diff --git a/hack/.packages b/hack/.packages index f417aa9b0e..f51073dd03 100644 --- a/hack/.packages +++ b/hack/.packages @@ -44,6 +44,7 @@ k8s.io/kops/node-authorizer/pkg/utils k8s.io/kops/nodeup/pkg/bootstrap k8s.io/kops/nodeup/pkg/distros k8s.io/kops/nodeup/pkg/model +k8s.io/kops/nodeup/pkg/model/networking k8s.io/kops/nodeup/pkg/model/resources k8s.io/kops/pkg/acls k8s.io/kops/pkg/acls/gce diff --git a/nodeup/pkg/model/BUILD.bazel b/nodeup/pkg/model/BUILD.bazel index f03cb375fe..650b6c3520 100644 --- a/nodeup/pkg/model/BUILD.bazel +++ b/nodeup/pkg/model/BUILD.bazel @@ -4,7 +4,6 @@ go_library( name = "go_default_library", srcs = [ "architecture.go", - "cilium.go", "cloudconfig.go", "containerd.go", "context.go", @@ -21,14 +20,12 @@ go_library( "kube_apiserver_healthcheck.go", "kube_controller_manager.go", "kube_proxy.go", - "kube_router.go", "kube_scheduler.go", "kubectl.go", "kubelet.go", "logrotate.go", "manifests.go", "miscutils.go", - "network.go", "node_authorizer.go", "ntp.go", "packages.go", @@ -73,7 +70,6 @@ go_library( "//vendor/github.com/aws/aws-sdk-go/aws/session:go_default_library", "//vendor/github.com/aws/aws-sdk-go/service/ec2:go_default_library", "//vendor/github.com/blang/semver:go_default_library", - "//vendor/golang.org/x/sys/unix:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", diff --git a/nodeup/pkg/model/context.go b/nodeup/pkg/model/context.go index 233de004d7..a159de5f4e 100644 --- a/nodeup/pkg/model/context.go +++ b/nodeup/pkg/model/context.go @@ -170,12 +170,6 @@ func (c *NodeupModelContext) PathSrvSshproxy() string { } } -// CNIBinDir returns the path for the CNI binaries -func (c *NodeupModelContext) CNIBinDir() string { - // We used to map this on a per-distro basis, but this can require CNI manifests to be distro aware - return "/opt/cni/bin/" -} - // KubeletBootstrapKubeconfig is the path the bootstrap config file func (c *NodeupModelContext) KubeletBootstrapKubeconfig() string { path := c.Cluster.Spec.Kubelet.BootstrapKubeconfig @@ -198,11 +192,6 @@ func (c *NodeupModelContext) KubeletKubeConfig() string { return "/var/lib/kubelet/kubeconfig" } -// CNIConfDir returns the CNI directory -func (c *NodeupModelContext) CNIConfDir() string { - return "/etc/cni/net.d/" -} - // BuildPKIKubeconfig generates a kubeconfig func (c *NodeupModelContext) BuildPKIKubeconfig(name string) (string, error) { ca, err := c.GetCert(fi.CertificateId_CA) @@ -584,3 +573,53 @@ func (c *NodeupModelContext) GetPrivateKey(name string) ([]byte, error) { return key.AsBytes() } + +func (b *NodeupModelContext) AddCNIBinAssets(c *fi.ModelBuilderContext, assetNames []string) error { + for _, assetName := range assetNames { + if err := b.addCNIBinAsset(c, assetName); err != nil { + return err + } + } + return nil +} + +func (b *NodeupModelContext) addCNIBinAsset(c *fi.ModelBuilderContext, assetName string) error { + assetPath := "" + asset, err := b.Assets.Find(assetName, assetPath) + if err != nil { + return fmt.Errorf("error trying to locate asset %q: %v", assetName, err) + } + if asset == nil { + return fmt.Errorf("unable to locate asset %q", assetName) + } + + c.AddTask(&nodetasks.File{ + Path: filepath.Join(b.CNIBinDir(), assetName), + Contents: asset, + Type: nodetasks.FileType_File, + Mode: fi.String("0755"), + }) + + return nil +} + +// UsesCNI checks if the cluster has CNI configured +func (c *NodeupModelContext) UsesCNI() bool { + networking := c.Cluster.Spec.Networking + if networking == nil || networking.Classic != nil { + return false + } + + return true +} + +// CNIBinDir returns the path for the CNI binaries +func (c *NodeupModelContext) CNIBinDir() string { + // We used to map this on a per-distro basis, but this can require CNI manifests to be distro aware + return "/opt/cni/bin/" +} + +// CNIConfDir returns the CNI directory +func (c *NodeupModelContext) CNIConfDir() string { + return "/etc/cni/net.d/" +} diff --git a/nodeup/pkg/model/network.go b/nodeup/pkg/model/network.go deleted file mode 100644 index 3bc5ade2a9..0000000000 --- a/nodeup/pkg/model/network.go +++ /dev/null @@ -1,123 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package model - -import ( - "fmt" - "path/filepath" - - "k8s.io/klog" - "k8s.io/kops/pkg/systemd" - "k8s.io/kops/upup/pkg/fi" - "k8s.io/kops/upup/pkg/fi/nodeup/nodetasks" -) - -// NetworkBuilder writes CNI assets -type NetworkBuilder struct { - *NodeupModelContext -} - -var _ fi.ModelBuilder = &NetworkBuilder{} - -// Build is responsible for configuring the network cni -func (b *NetworkBuilder) Build(c *fi.ModelBuilderContext) error { - var assetNames []string - - // @TODO need to clean up this code, it isn't the easiest to read - networking := b.Cluster.Spec.Networking - if networking == nil || networking.Classic != nil { - } else if networking.Kubenet != nil || networking.GCE != nil { - assetNames = append(assetNames, "bridge", "host-local", "loopback") - } else if networking.External != nil { - // external is based on kubenet - assetNames = append(assetNames, "bridge", "host-local", "loopback") - - } else if networking.CNI != nil || networking.Weave != nil || networking.Flannel != nil || networking.Calico != nil || networking.Canal != nil || networking.Kuberouter != nil || networking.AmazonVPC != nil || networking.Cilium != nil { - assetNames = append(assetNames, "bridge", "host-local", "loopback", "ptp", "portmap") - // Do we need tuning? - - // TODO: Only when using flannel ? - assetNames = append(assetNames, "flannel") - } else if networking.Kopeio != nil { - // TODO combine with External - // Kopeio is based on kubenet / external - assetNames = append(assetNames, "bridge", "host-local", "loopback") - } else if networking.LyftVPC != nil { - assetNames = append(assetNames, "cni-ipvlan-vpc-k8s-ipam", "cni-ipvlan-vpc-k8s-ipvlan", "cni-ipvlan-vpc-k8s-tool", "cni-ipvlan-vpc-k8s-unnumbered-ptp", "loopback") - } else { - return fmt.Errorf("no networking mode set") - } - - for _, assetName := range assetNames { - if err := b.addCNIBinAsset(c, assetName); err != nil { - return err - } - } - - // Tx checksum offloading is buggy for NAT-ed VXLAN endpoints, leading to an invalid checksum sent and causing - // Flannel to stop to working as the traffic is being discarded by the receiver. - // https://github.com/coreos/flannel/issues/1279 - if networking != nil && (networking.Canal != nil || (networking.Flannel != nil && networking.Flannel.Backend == "vxlan")) { - c.AddTask(b.buildFlannelTxChecksumOffloadDisableService()) - } - - return nil -} - -func (b *NetworkBuilder) addCNIBinAsset(c *fi.ModelBuilderContext, assetName string) error { - assetPath := "" - asset, err := b.Assets.Find(assetName, assetPath) - if err != nil { - return fmt.Errorf("error trying to locate asset %q: %v", assetName, err) - } - if asset == nil { - return fmt.Errorf("unable to locate asset %q", assetName) - } - - c.AddTask(&nodetasks.File{ - Path: filepath.Join(b.CNIBinDir(), assetName), - Contents: asset, - Type: nodetasks.FileType_File, - Mode: s("0755"), - }) - - return nil -} - -func (b *NetworkBuilder) buildFlannelTxChecksumOffloadDisableService() *nodetasks.Service { - const serviceName = "flannel-tx-checksum-offload-disable.service" - - manifest := &systemd.Manifest{} - manifest.Set("Unit", "Description", "Disable TX checksum offload on flannel.1") - - manifest.Set("Unit", "After", "sys-devices-virtual-net-flannel.1.device") - manifest.Set("Install", "WantedBy", "sys-devices-virtual-net-flannel.1.device") - manifest.Set("Service", "Type", "oneshot") - manifest.Set("Service", "ExecStart", "/sbin/ethtool -K flannel.1 tx-checksum-ip-generic off") - - manifestString := manifest.Render() - klog.V(8).Infof("Built service manifest %q\n%s", serviceName, manifestString) - - service := &nodetasks.Service{ - Name: serviceName, - Definition: s(manifestString), - } - - service.InitDefaults() - - return service -} diff --git a/nodeup/pkg/model/networking/BUILD.bazel b/nodeup/pkg/model/networking/BUILD.bazel new file mode 100644 index 0000000000..44e233bc95 --- /dev/null +++ b/nodeup/pkg/model/networking/BUILD.bazel @@ -0,0 +1,29 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = [ + "cilium.go", + "flannel.go", + "kube_router.go", + "kubenet.go", + "lyft.go", + ], + importpath = "k8s.io/kops/nodeup/pkg/model/networking", + visibility = ["//visibility:public"], + deps = [ + "//nodeup/pkg/model:go_default_library", + "//pkg/apis/kops:go_default_library", + "//pkg/pki:go_default_library", + "//pkg/systemd:go_default_library", + "//upup/pkg/fi:go_default_library", + "//upup/pkg/fi/nodeup/nodetasks:go_default_library", + "//vendor/github.com/aws/aws-sdk-go/aws:go_default_library", + "//vendor/github.com/aws/aws-sdk-go/aws/ec2metadata:go_default_library", + "//vendor/github.com/aws/aws-sdk-go/aws/request:go_default_library", + "//vendor/github.com/aws/aws-sdk-go/aws/session:go_default_library", + "//vendor/github.com/aws/aws-sdk-go/service/ec2:go_default_library", + "//vendor/golang.org/x/sys/unix:go_default_library", + "//vendor/k8s.io/klog:go_default_library", + ], +) diff --git a/nodeup/pkg/model/cilium.go b/nodeup/pkg/model/networking/cilium.go similarity index 95% rename from nodeup/pkg/model/cilium.go rename to nodeup/pkg/model/networking/cilium.go index 9228deadd3..c81822fc75 100644 --- a/nodeup/pkg/model/cilium.go +++ b/nodeup/pkg/model/networking/cilium.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package model +package networking import ( "crypto/x509" @@ -24,6 +24,8 @@ import ( "path/filepath" "time" + "k8s.io/kops/nodeup/pkg/model" + "golang.org/x/sys/unix" "k8s.io/klog" "k8s.io/kops/pkg/pki" @@ -33,7 +35,7 @@ import ( // CiliumBuilder writes Cilium's assets type CiliumBuilder struct { - *NodeupModelContext + *model.NodeupModelContext } var _ fi.ModelBuilder = &CiliumBuilder{} @@ -76,7 +78,7 @@ func (b *CiliumBuilder) buildBPFMount(c *fi.ModelBuilderContext) error { alreadyMounted := uint32(fsdata.Type) == BPF_FS_MAGIC if !alreadyMounted { - unit := s(` + unit := ` [Unit] Description=Cilium BPF mounts Documentation=http://docs.cilium.io/ @@ -90,13 +92,12 @@ Type=bpf [Install] WantedBy=multi-user.target -`) +` - service := &nodetasks.Service{ - Name: "sys-fs-bpf.mount", - Definition: unit, + service, err := nodetasks.NewService("sys-fs-bpf.mount", unit, "") + if err != nil { + return err } - service.InitDefaults() c.AddTask(service) } diff --git a/nodeup/pkg/model/networking/flannel.go b/nodeup/pkg/model/networking/flannel.go new file mode 100644 index 0000000000..9db3dc79f5 --- /dev/null +++ b/nodeup/pkg/model/networking/flannel.go @@ -0,0 +1,75 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package networking + +import ( + "k8s.io/klog" + "k8s.io/kops/nodeup/pkg/model" + "k8s.io/kops/pkg/systemd" + + "k8s.io/kops/upup/pkg/fi" + "k8s.io/kops/upup/pkg/fi/nodeup/nodetasks" +) + +// CiliumBuilder writes Cilium's assets +type FlannelBuilder struct { + *model.NodeupModelContext +} + +var _ fi.ModelBuilder = &FlannelBuilder{} + +// Build is responsible for configuring the network cni +func (b *FlannelBuilder) Build(c *fi.ModelBuilderContext) error { + networking := b.Cluster.Spec.Networking + + if networking.Flannel != nil { + b.AddCNIBinAssets(c, []string{"flannel", "portmap", "bridge", "host-local", "loopback"}) + } + + if networking.Canal != nil || networking.Flannel != nil && networking.Flannel.Backend == "vxlan" { + return b.buildFlannelTxChecksumOffloadDisableService(c) + } + return nil +} + +// Tx checksum offloading is buggy for NAT-ed VXLAN endpoints, leading to an invalid checksum sent and causing +// Flannel to stop to working as the traffic is being discarded by the receiver. +// https://github.com/coreos/flannel/issues/1279 +func (b *FlannelBuilder) buildFlannelTxChecksumOffloadDisableService(c *fi.ModelBuilderContext) error { + const serviceName = "flannel-tx-checksum-offload-disable.service" + + manifest := &systemd.Manifest{} + manifest.Set("Unit", "Description", "Disable TX checksum offload on flannel.1") + + manifest.Set("Unit", "After", "sys-devices-virtual-net-flannel.1.device") + manifest.Set("Install", "WantedBy", "sys-devices-virtual-net-flannel.1.device") + manifest.Set("Service", "Type", "oneshot") + manifest.Set("Service", "ExecStart", "/sbin/ethtool -K flannel.1 tx-checksum-ip-generic off") + + manifestString := manifest.Render() + klog.V(8).Infof("Built service manifest %q\n%s", serviceName, manifestString) + + service, err := nodetasks.NewService(serviceName, manifestString, "") + + if err != nil { + return err + } + + c.AddTask(service) + + return nil +} diff --git a/nodeup/pkg/model/kube_router.go b/nodeup/pkg/model/networking/kube_router.go similarity index 69% rename from nodeup/pkg/model/kube_router.go rename to nodeup/pkg/model/networking/kube_router.go index f109761a16..e7c15fc4ba 100644 --- a/nodeup/pkg/model/kube_router.go +++ b/nodeup/pkg/model/networking/kube_router.go @@ -14,22 +14,26 @@ See the License for the specific language governing permissions and limitations under the License. */ -package model +package networking import ( + "k8s.io/kops/nodeup/pkg/model" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/nodeup/nodetasks" ) -// KubeRouterBuilder installs kube-router -type KubeRouterBuilder struct { - *NodeupModelContext +// KuberouterBuilder installs kube-router +type KuberouterBuilder struct { + *model.NodeupModelContext } -var _ fi.ModelBuilder = &KubeRouterBuilder{} +var _ fi.ModelBuilder = &KuberouterBuilder{} // Build is responsible for configuring the kube-router -func (b *KubeRouterBuilder) Build(c *fi.ModelBuilderContext) error { +func (b *KuberouterBuilder) Build(c *fi.ModelBuilderContext) error { + if b.Cluster.Spec.Networking.Kuberouter == nil { + return nil + } { kubeconfig, err := b.BuildPKIKubeconfig("kube-router") if err != nil { @@ -40,9 +44,11 @@ func (b *KubeRouterBuilder) Build(c *fi.ModelBuilderContext) error { Path: "/var/lib/kube-router/kubeconfig", Contents: fi.NewStringResource(kubeconfig), Type: nodetasks.FileType_File, - Mode: s("0400"), + Mode: fi.String("0400"), }) } + b.AddCNIBinAssets(c, []string{"loopback", "host-local", "bridge", "portmap"}) + return nil } diff --git a/nodeup/pkg/model/networking/kubenet.go b/nodeup/pkg/model/networking/kubenet.go new file mode 100644 index 0000000000..79790d09f1 --- /dev/null +++ b/nodeup/pkg/model/networking/kubenet.go @@ -0,0 +1,40 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package networking + +import ( + "k8s.io/kops/nodeup/pkg/model" + "k8s.io/kops/upup/pkg/fi" +) + +// KuberouterBuilder installs kube-router +type KubenetBuilder struct { + *model.NodeupModelContext +} + +var _ fi.ModelBuilder = &KubenetBuilder{} + +// Build is responsible for configuring the kube-router +func (b *KubenetBuilder) Build(c *fi.ModelBuilderContext) error { + if b.Cluster.Spec.Networking.Kubenet == nil && b.Cluster.Spec.Networking.Kopeio == nil { + return nil + } + + b.AddCNIBinAssets(c, []string{"loopback", "host-local", "bridge", "portmap"}) + + return nil +} diff --git a/nodeup/pkg/model/networking/lyft.go b/nodeup/pkg/model/networking/lyft.go new file mode 100644 index 0000000000..d44ac99a86 --- /dev/null +++ b/nodeup/pkg/model/networking/lyft.go @@ -0,0 +1,142 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package networking + +import ( + "encoding/json" + "fmt" + "strings" + "text/template" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/ec2metadata" + "github.com/aws/aws-sdk-go/aws/request" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/ec2" + "k8s.io/klog" + "k8s.io/kops/nodeup/pkg/model" + api "k8s.io/kops/pkg/apis/kops" + "k8s.io/kops/upup/pkg/fi" +) + +type LyftVPCBuilder struct { + *model.NodeupModelContext +} + +var _ fi.ModelBuilder = &CiliumBuilder{} + +// Build is responsible for configuring the network cni +func (b *LyftVPCBuilder) Build(c *fi.ModelBuilderContext) error { + networking := b.Cluster.Spec.Networking + + if networking.LyftVPC == nil { + return nil + } + + assetNames := []string{"loopback", "cni-ipvlan-vpc-k8s-ipam", "cni-ipvlan-vpc-k8s-ipvlan", "cni-ipvlan-vpc-k8s-tool", "cni-ipvlan-vpc-k8s-unnumbered-ptp"} + + return b.AddCNIBinAssets(c, assetNames) +} + +func LoadLyftTemplateFunctions(templateFunctions template.FuncMap, cluster *api.Cluster) { + templateFunctions["SubnetTags"] = func() (string, error) { + var tags map[string]string + if cluster.IsKubernetesGTE("1.18") { + tags = map[string]string{ + "KubernetesCluster": cluster.Name, + } + } else { + tags = map[string]string{ + "Type": "pod", + } + } + if len(cluster.Spec.Networking.LyftVPC.SubnetTags) > 0 { + tags = cluster.Spec.Networking.LyftVPC.SubnetTags + } + + bytes, err := json.Marshal(tags) + if err != nil { + return "", err + } + return string(bytes), nil + } + + templateFunctions["NodeSecurityGroups"] = func() (string, error) { + // use the same security groups as the node + ids, err := evaluateSecurityGroups(cluster.Spec.NetworkID) + if err != nil { + return "", err + } + bytes, err := json.Marshal(ids) + if err != nil { + return "", err + } + return string(bytes), nil + } + +} + +func evaluateSecurityGroups(vpcId string) ([]string, error) { + config := aws.NewConfig() + config = config.WithCredentialsChainVerboseErrors(true) + + s, err := session.NewSession(config) + if err != nil { + return nil, fmt.Errorf("error starting new AWS session: %v", err) + } + s.Handlers.Send.PushFront(func(r *request.Request) { + // Log requests + klog.V(4).Infof("AWS API Request: %s/%s", r.ClientInfo.ServiceName, r.Operation.Name) + }) + + metadata := ec2metadata.New(s, config) + + region, err := metadata.Region() + if err != nil { + return nil, fmt.Errorf("error querying ec2 metadata service (for az/region): %v", err) + } + + sgNames, err := metadata.GetMetadata("security-groups") + if err != nil { + return nil, fmt.Errorf("error querying ec2 metadata service (for security-groups): %v", err) + } + svc := ec2.New(s, config.WithRegion(region)) + + result, err := svc.DescribeSecurityGroups(&ec2.DescribeSecurityGroupsInput{ + Filters: []*ec2.Filter{ + { + Name: aws.String("group-name"), + Values: aws.StringSlice(strings.Fields(sgNames)), + }, + { + Name: aws.String("vpc-id"), + Values: []*string{aws.String(vpcId)}, + }, + }, + }) + + if err != nil { + return nil, fmt.Errorf("error looking up instance security group ids: %v", err) + } + var sgIds []string + for _, group := range result.SecurityGroups { + sgIds = append(sgIds, *group.GroupId) + } + + return sgIds, nil + +} diff --git a/pkg/apis/kops/validation/validation.go b/pkg/apis/kops/validation/validation.go index 9f423cf671..ced6c7b96b 100644 --- a/pkg/apis/kops/validation/validation.go +++ b/pkg/apis/kops/validation/validation.go @@ -368,6 +368,9 @@ func validateNetworking(c *kops.ClusterSpec, v *kops.NetworkingSpec, fldPath *fi if optionTaken { allErrs = append(allErrs, field.Forbidden(fldPath.Child("kuberouter"), "only one networking option permitted")) } + if c.KubeProxy != nil && (c.KubeProxy.Enabled == nil || *c.KubeProxy.Enabled) { + allErrs = append(allErrs, field.Forbidden(fldPath.Root().Child("spec", "kubeProxy", "enabled"), "kube-router requires kubeProxy to be disabled")) + } optionTaken = true } diff --git a/upup/pkg/fi/nodeup/BUILD.bazel b/upup/pkg/fi/nodeup/BUILD.bazel index 78805afccc..168b430002 100644 --- a/upup/pkg/fi/nodeup/BUILD.bazel +++ b/upup/pkg/fi/nodeup/BUILD.bazel @@ -11,6 +11,7 @@ go_library( deps = [ "//nodeup/pkg/distros:go_default_library", "//nodeup/pkg/model:go_default_library", + "//nodeup/pkg/model/networking:go_default_library", "//pkg/apis/kops:go_default_library", "//pkg/apis/kops/registry:go_default_library", "//pkg/apis/nodeup:go_default_library", @@ -24,8 +25,6 @@ go_library( "//upup/pkg/fi/utils:go_default_library", "//util/pkg/vfs:go_default_library", "//vendor/github.com/aws/aws-sdk-go/aws:go_default_library", - "//vendor/github.com/aws/aws-sdk-go/aws/ec2metadata:go_default_library", - "//vendor/github.com/aws/aws-sdk-go/aws/request:go_default_library", "//vendor/github.com/aws/aws-sdk-go/aws/session:go_default_library", "//vendor/github.com/aws/aws-sdk-go/service/ec2:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", diff --git a/upup/pkg/fi/nodeup/command.go b/upup/pkg/fi/nodeup/command.go index c9c4a2fba1..a3a8afa399 100644 --- a/upup/pkg/fi/nodeup/command.go +++ b/upup/pkg/fi/nodeup/command.go @@ -17,7 +17,6 @@ limitations under the License. package nodeup import ( - "encoding/json" "errors" "fmt" "io" @@ -30,6 +29,7 @@ import ( "k8s.io/kops/nodeup/pkg/distros" "k8s.io/kops/nodeup/pkg/model" + "k8s.io/kops/nodeup/pkg/model/networking" api "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/pkg/apis/kops/registry" "k8s.io/kops/pkg/apis/nodeup" @@ -43,8 +43,6 @@ import ( "k8s.io/kops/util/pkg/vfs" "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/ec2metadata" - "github.com/aws/aws-sdk-go/aws/request" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/ec2" "k8s.io/apimachinery/pkg/util/sets" @@ -244,62 +242,23 @@ func (c *NodeUpCommand) Run(out io.Writer) error { loader.Builders = append(loader.Builders, &model.PackagesBuilder{NodeupModelContext: modelContext}) loader.Builders = append(loader.Builders, &model.SecretBuilder{NodeupModelContext: modelContext}) loader.Builders = append(loader.Builders, &model.FirewallBuilder{NodeupModelContext: modelContext}) - loader.Builders = append(loader.Builders, &model.NetworkBuilder{NodeupModelContext: modelContext}) loader.Builders = append(loader.Builders, &model.SysctlBuilder{NodeupModelContext: modelContext}) loader.Builders = append(loader.Builders, &model.KubeAPIServerBuilder{NodeupModelContext: modelContext}) loader.Builders = append(loader.Builders, &model.KubeControllerManagerBuilder{NodeupModelContext: modelContext}) loader.Builders = append(loader.Builders, &model.KubeSchedulerBuilder{NodeupModelContext: modelContext}) loader.Builders = append(loader.Builders, &model.EtcdManagerTLSBuilder{NodeupModelContext: modelContext}) + loader.Builders = append(loader.Builders, &model.KubeProxyBuilder{NodeupModelContext: modelContext}) + loader.Builders = append(loader.Builders, &model.EtcdTLSBuilder{NodeupModelContext: modelContext}) - if c.cluster.Spec.Networking.Cilium != nil { - loader.Builders = append(loader.Builders, &model.CiliumBuilder{NodeupModelContext: modelContext}) - } - if c.cluster.Spec.Networking.Kuberouter == nil { - loader.Builders = append(loader.Builders, &model.KubeProxyBuilder{NodeupModelContext: modelContext}) - } else { - loader.Builders = append(loader.Builders, &model.KubeRouterBuilder{NodeupModelContext: modelContext}) - } - if c.cluster.Spec.Networking.Calico != nil { - loader.Builders = append(loader.Builders, &model.EtcdTLSBuilder{NodeupModelContext: modelContext}) - } + loader.Builders = append(loader.Builders, &networking.CiliumBuilder{NodeupModelContext: modelContext}) + // Canal = Flannel + Calico, so use this builder for both CNIs + loader.Builders = append(loader.Builders, &networking.FlannelBuilder{NodeupModelContext: modelContext}) + loader.Builders = append(loader.Builders, &networking.LyftVPCBuilder{NodeupModelContext: modelContext}) + loader.Builders = append(loader.Builders, &networking.KuberouterBuilder{NodeupModelContext: modelContext}) + // Also handles kopeio as kopeio is based on kubenet + loader.Builders = append(loader.Builders, &networking.KubenetBuilder{NodeupModelContext: modelContext}) - if c.cluster.Spec.Networking.LyftVPC != nil { - - loader.TemplateFunctions["SubnetTags"] = func() (string, error) { - var tags map[string]string - if c.cluster.IsKubernetesGTE("1.18") { - tags = map[string]string{ - "KubernetesCluster": c.cluster.Name, - } - } else { - tags = map[string]string{ - "Type": "pod", - } - } - if len(c.cluster.Spec.Networking.LyftVPC.SubnetTags) > 0 { - tags = c.cluster.Spec.Networking.LyftVPC.SubnetTags - } - - bytes, err := json.Marshal(tags) - if err != nil { - return "", err - } - return string(bytes), nil - } - - loader.TemplateFunctions["NodeSecurityGroups"] = func() (string, error) { - // use the same security groups as the node - ids, err := evaluateSecurityGroups(c.cluster.Spec.NetworkID) - if err != nil { - return "", err - } - bytes, err := json.Marshal(ids) - if err != nil { - return "", err - } - return string(bytes), nil - } - } + networking.LoadLyftTemplateFunctions(loader.TemplateFunctions, c.cluster) taskMap, err := loader.Build(c.ModelDir) if err != nil { @@ -393,57 +352,6 @@ func evaluateSpec(c *api.Cluster) error { return nil } -func evaluateSecurityGroups(vpcId string) ([]string, error) { - config := aws.NewConfig() - config = config.WithCredentialsChainVerboseErrors(true) - - s, err := session.NewSession(config) - if err != nil { - return nil, fmt.Errorf("error starting new AWS session: %v", err) - } - s.Handlers.Send.PushFront(func(r *request.Request) { - // Log requests - klog.V(4).Infof("AWS API Request: %s/%s", r.ClientInfo.ServiceName, r.Operation.Name) - }) - - metadata := ec2metadata.New(s, config) - - region, err := metadata.Region() - if err != nil { - return nil, fmt.Errorf("error querying ec2 metadata service (for az/region): %v", err) - } - - sgNames, err := metadata.GetMetadata("security-groups") - if err != nil { - return nil, fmt.Errorf("error querying ec2 metadata service (for security-groups): %v", err) - } - svc := ec2.New(s, config.WithRegion(region)) - - result, err := svc.DescribeSecurityGroups(&ec2.DescribeSecurityGroupsInput{ - Filters: []*ec2.Filter{ - { - Name: aws.String("group-name"), - Values: aws.StringSlice(strings.Fields(sgNames)), - }, - { - Name: aws.String("vpc-id"), - Values: []*string{aws.String(vpcId)}, - }, - }, - }) - - if err != nil { - return nil, fmt.Errorf("error looking up instance security group ids: %v", err) - } - var sgIds []string - for _, group := range result.SecurityGroups { - sgIds = append(sgIds, *group.GroupId) - } - - return sgIds, nil - -} - func evaluateHostnameOverride(hostnameOverride string) (string, error) { if hostnameOverride == "" || hostnameOverride == "@hostname" { return "", nil