From b4c3b38436cbad09a6a7c9657b7c02e6bc505e85 Mon Sep 17 00:00:00 2001 From: John Gardiner Myers Date: Thu, 2 Jul 2020 21:04:26 -0700 Subject: [PATCH 1/3] Move more cloud provider setup to pkg --- cmd/kops/BUILD.bazel | 1 - cmd/kops/create_cluster.go | 52 ------------------------------ upup/pkg/fi/cloudup/new_cluster.go | 51 ++++++++++++++++++++++++++++- 3 files changed, 50 insertions(+), 54 deletions(-) diff --git a/cmd/kops/BUILD.bazel b/cmd/kops/BUILD.bazel index 6f591e5b63..91defd1a82 100644 --- a/cmd/kops/BUILD.bazel +++ b/cmd/kops/BUILD.bazel @@ -85,7 +85,6 @@ go_library( "//upup/pkg/fi:go_default_library", "//upup/pkg/fi/cloudup:go_default_library", "//upup/pkg/fi/cloudup/awsup:go_default_library", - "//upup/pkg/fi/cloudup/gce:go_default_library", "//upup/pkg/fi/cloudup/openstack:go_default_library", "//upup/pkg/fi/utils:go_default_library", "//upup/pkg/kutil:go_default_library", diff --git a/cmd/kops/create_cluster.go b/cmd/kops/create_cluster.go index 9c7991b947..dbeff55222 100644 --- a/cmd/kops/create_cluster.go +++ b/cmd/kops/create_cluster.go @@ -45,7 +45,6 @@ import ( "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/cloudup" "k8s.io/kops/upup/pkg/fi/cloudup/awsup" - "k8s.io/kops/upup/pkg/fi/cloudup/gce" "k8s.io/kops/upup/pkg/fi/cloudup/openstack" "k8s.io/kops/upup/pkg/fi/utils" "k8s.io/kubectl/pkg/util/i18n" @@ -60,7 +59,6 @@ type CreateClusterOptions struct { MasterSize string MasterVolumeSize int32 NodeVolumeSize int32 - Project string KubernetesVersion string ContainerRuntime string OutDir string @@ -109,15 +107,8 @@ type CreateClusterOptions struct { // Allow custom public master name MasterPublicName string - // Spotinst options - SpotinstProduct string - SpotinstOrientation string - OpenstackNetworkID string - // GCEServiceAccount specifies the service account with which the GCE VM runs - GCEServiceAccount string - // DryRun mode output a cluster manifest of Output type. DryRun bool // Output type during a DryRun @@ -512,49 +503,6 @@ func RunCreateCluster(ctx context.Context, f *util.Factory, out io.Writer, c *Cr channel := clusterResult.Channel allZones := clusterResult.AllZones - if c.CloudProvider != "" { - if featureflag.Spotinst.Enabled() { - if cluster.Spec.CloudConfig == nil { - cluster.Spec.CloudConfig = &api.CloudConfiguration{} - } - if c.SpotinstProduct != "" { - cluster.Spec.CloudConfig.SpotinstProduct = fi.String(c.SpotinstProduct) - } - if c.SpotinstOrientation != "" { - cluster.Spec.CloudConfig.SpotinstOrientation = fi.String(c.SpotinstOrientation) - } - } - } - - // Populate project - if c.Project != "" { - cluster.Spec.Project = c.Project - } - if api.CloudProviderID(cluster.Spec.CloudProvider) == api.CloudProviderGCE { - if cluster.Spec.CloudConfig == nil { - cluster.Spec.CloudConfig = &api.CloudConfiguration{} - } - if cluster.Spec.Project == "" { - project, err := gce.DefaultProject() - if err != nil { - klog.Warningf("unable to get default google cloud project: %v", err) - } else if project == "" { - klog.Warningf("default google cloud project not set (try `gcloud config set project `") - } else { - klog.Infof("using google cloud project: %s", project) - } - cluster.Spec.Project = project - } - if c.GCEServiceAccount != "" { - klog.Infof("VMs will be configured to use specified Service Account: %v", c.GCEServiceAccount) - cluster.Spec.CloudConfig.GCEServiceAccount = c.GCEServiceAccount - } else { - klog.Warning("VMs will be configured to use the GCE default compute Service Account! This is an anti-pattern") - klog.Warning("Use a pre-created Service Account with the flag: --gce-service-account=account@projectname.iam.gserviceaccount.com") - cluster.Spec.CloudConfig.GCEServiceAccount = "default" - } - } - if c.KubernetesVersion != "" { cluster.Spec.KubernetesVersion = c.KubernetesVersion } diff --git a/upup/pkg/fi/cloudup/new_cluster.go b/upup/pkg/fi/cloudup/new_cluster.go index 791ab917d2..c4a98e2fa9 100644 --- a/upup/pkg/fi/cloudup/new_cluster.go +++ b/upup/pkg/fi/cloudup/new_cluster.go @@ -31,6 +31,7 @@ import ( api "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/pkg/apis/kops/model" "k8s.io/kops/pkg/client/simple" + "k8s.io/kops/pkg/featureflag" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/cloudup/aliup" "k8s.io/kops/upup/pkg/fi/cloudup/awsup" @@ -61,11 +62,21 @@ type NewClusterOptions struct { // MasterZones are the availability zones in which to run the masters. Defaults to the list in the Zones field. MasterZones []string + // Project is the cluster's GCE project. + Project string + // GCEServiceAccount specifies the service account with which the GCE VM runs. + GCEServiceAccount string + + // Spotinst options + SpotinstProduct string + SpotinstOrientation string + // NetworkID is the ID of the shared network (VPC). // If empty, SubnetIDs are not empty, and on AWS or OpenStack, determines network ID from the first SubnetID. // If empty otherwise, creates a new network/VPC to be owned by the cluster. NetworkID string - // SubnetIDs are the IDs of the shared subnets. If empty, creates new subnets to be owned by the cluster. + // SubnetIDs are the IDs of the shared subnets. + // If empty, creates new subnets to be owned by the cluster. SubnetIDs []string // Egress defines the method of traffic egress for subnets. Egress string @@ -232,6 +243,32 @@ func setupVPC(opt *NewClusterOptions, cluster *api.Cluster) error { cluster.Spec.NetworkID = *res.Subnets[0].VpcId } + case api.CloudProviderGCE: + if cluster.Spec.CloudConfig == nil { + cluster.Spec.CloudConfig = &api.CloudConfiguration{} + } + cluster.Spec.Project = opt.Project + if cluster.Spec.Project == "" { + project, err := gce.DefaultProject() + if err != nil { + klog.Warningf("unable to get default google cloud project: %v", err) + } else if project == "" { + klog.Warningf("default google cloud project not set (try `gcloud config set project `") + } else { + klog.Infof("using google cloud project: %s", project) + } + cluster.Spec.Project = project + } + if opt.GCEServiceAccount != "" { + // TODO remove this logging? + klog.Infof("VMs will be configured to use specified Service Account: %v", opt.GCEServiceAccount) + cluster.Spec.CloudConfig.GCEServiceAccount = opt.GCEServiceAccount + } else { + klog.Warning("VMs will be configured to use the GCE default compute Service Account! This is an anti-pattern") + klog.Warning("Use a pre-created Service Account with the flag: --gce-service-account=account@projectname.iam.gserviceaccount.com") + cluster.Spec.CloudConfig.GCEServiceAccount = "default" + } + case api.CloudProviderOpenstack: if cluster.Spec.CloudConfig == nil { cluster.Spec.CloudConfig = &api.CloudConfiguration{} @@ -274,6 +311,18 @@ func setupVPC(opt *NewClusterOptions, cluster *api.Cluster) error { } } + if featureflag.Spotinst.Enabled() { + if cluster.Spec.CloudConfig == nil { + cluster.Spec.CloudConfig = &api.CloudConfiguration{} + } + if opt.SpotinstProduct != "" { + cluster.Spec.CloudConfig.SpotinstProduct = fi.String(opt.SpotinstProduct) + } + if opt.SpotinstOrientation != "" { + cluster.Spec.CloudConfig.SpotinstOrientation = fi.String(opt.SpotinstOrientation) + } + } + return nil } From de0e20ee7b6bd274123a9c868a49d4c17e42f9d9 Mon Sep 17 00:00:00 2001 From: John Gardiner Myers Date: Thu, 2 Jul 2020 22:56:15 -0700 Subject: [PATCH 2/3] Move network provider setup to pkg --- cmd/kops/BUILD.bazel | 1 - cmd/kops/create_cluster.go | 94 ------------------------- upup/pkg/fi/cloudup/BUILD.bazel | 1 + upup/pkg/fi/cloudup/new_cluster.go | 106 +++++++++++++++++++++++++++++ 4 files changed, 107 insertions(+), 95 deletions(-) diff --git a/cmd/kops/BUILD.bazel b/cmd/kops/BUILD.bazel index 91defd1a82..37781c8bb9 100644 --- a/cmd/kops/BUILD.bazel +++ b/cmd/kops/BUILD.bazel @@ -73,7 +73,6 @@ go_library( "//pkg/instancegroups:go_default_library", "//pkg/kopscodecs:go_default_library", "//pkg/kubeconfig:go_default_library", - "//pkg/model/components:go_default_library", "//pkg/pki:go_default_library", "//pkg/pretty:go_default_library", "//pkg/resources:go_default_library", diff --git a/cmd/kops/create_cluster.go b/cmd/kops/create_cluster.go index dbeff55222..b56a3377e7 100644 --- a/cmd/kops/create_cluster.go +++ b/cmd/kops/create_cluster.go @@ -26,22 +26,18 @@ import ( "os" "strings" - "github.com/blang/semver/v4" "github.com/spf13/cobra" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/klog" "k8s.io/kops/cmd/kops/util" api "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/pkg/apis/kops/registry" - version "k8s.io/kops/pkg/apis/kops/util" "k8s.io/kops/pkg/apis/kops/validation" "k8s.io/kops/pkg/assets" "k8s.io/kops/pkg/commands" "k8s.io/kops/pkg/dns" "k8s.io/kops/pkg/featureflag" - "k8s.io/kops/pkg/model/components" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/cloudup" "k8s.io/kops/upup/pkg/fi/cloudup/awsup" @@ -59,7 +55,6 @@ type CreateClusterOptions struct { MasterSize string MasterVolumeSize int32 NodeVolumeSize int32 - KubernetesVersion string ContainerRuntime string OutDir string Image string @@ -71,7 +66,6 @@ type CreateClusterOptions struct { DNSZone string AdminAccess []string SSHAccess []string - Networking string NodeSecurityGroups []string MasterSecurityGroups []string AssociatePublicIP *bool @@ -120,7 +114,6 @@ func (o *CreateClusterOptions) InitDefaults() { o.Yes = false o.Target = cloudup.TargetDirect - o.Networking = "kubenet" o.Topology = api.TopologyPublic o.DNSType = string(api.DNSTypePublic) o.Bastion = false @@ -503,97 +496,10 @@ func RunCreateCluster(ctx context.Context, f *util.Factory, out io.Writer, c *Cr channel := clusterResult.Channel allZones := clusterResult.AllZones - if c.KubernetesVersion != "" { - cluster.Spec.KubernetesVersion = c.KubernetesVersion - } - if c.ContainerRuntime != "" { cluster.Spec.ContainerRuntime = c.ContainerRuntime } - cluster.Spec.Networking = &api.NetworkingSpec{} - switch c.Networking { - case "kubenet": - cluster.Spec.Networking.Kubenet = &api.KubenetNetworkingSpec{} - case "external": - cluster.Spec.Networking.External = &api.ExternalNetworkingSpec{} - case "cni": - cluster.Spec.Networking.CNI = &api.CNINetworkingSpec{} - case "kopeio-vxlan", "kopeio": - cluster.Spec.Networking.Kopeio = &api.KopeioNetworkingSpec{} - case "weave": - cluster.Spec.Networking.Weave = &api.WeaveNetworkingSpec{} - - if cluster.Spec.CloudProvider == "aws" { - // AWS supports "jumbo frames" of 9001 bytes and weave adds up to 87 bytes overhead - // sets the default to the largest number that leaves enough overhead and is divisible by 4 - jumboFrameMTUSize := int32(8912) - cluster.Spec.Networking.Weave.MTU = &jumboFrameMTUSize - } - case "flannel", "flannel-vxlan": - cluster.Spec.Networking.Flannel = &api.FlannelNetworkingSpec{ - Backend: "vxlan", - } - case "flannel-udp": - klog.Warningf("flannel UDP mode is not recommended; consider flannel-vxlan instead") - cluster.Spec.Networking.Flannel = &api.FlannelNetworkingSpec{ - Backend: "udp", - } - case "calico": - cluster.Spec.Networking.Calico = &api.CalicoNetworkingSpec{ - MajorVersion: "v3", - } - // Validate to check if etcd clusters have an acceptable version - if errList := validation.ValidateEtcdVersionForCalicoV3(cluster.Spec.EtcdClusters[0], cluster.Spec.Networking.Calico.MajorVersion, field.NewPath("spec", "networking", "calico")); len(errList) != 0 { - - // This is not a special version but simply of the 3 series - for _, etcd := range cluster.Spec.EtcdClusters { - etcd.Version = components.DefaultEtcd3Version_1_11 - } - } - case "canal": - cluster.Spec.Networking.Canal = &api.CanalNetworkingSpec{} - case "kube-router": - cluster.Spec.Networking.Kuberouter = &api.KuberouterNetworkingSpec{} - if cluster.Spec.KubeProxy == nil { - cluster.Spec.KubeProxy = &api.KubeProxyConfig{} - } - enabled := false - cluster.Spec.KubeProxy.Enabled = &enabled - case "amazonvpc", "amazon-vpc-routed-eni": - cluster.Spec.Networking.AmazonVPC = &api.AmazonVPCNetworkingSpec{} - case "cilium": - cilium := &api.CiliumNetworkingSpec{} - cluster.Spec.Networking.Cilium = cilium - nodeport := false - if c.KubernetesVersion == "" { - nodeport = true - } else { - k8sVersion, err := semver.ParseTolerant(c.KubernetesVersion) - if err == nil { - if version.IsKubernetesGTE("1.12", k8sVersion) { - nodeport = true - } - } - } - if nodeport { - cilium.EnableNodePort = true - if cluster.Spec.KubeProxy == nil { - cluster.Spec.KubeProxy = &api.KubeProxyConfig{} - } - enabled := false - cluster.Spec.KubeProxy.Enabled = &enabled - } - case "lyftvpc": - cluster.Spec.Networking.LyftVPC = &api.LyftVPCNetworkingSpec{} - case "gce": - cluster.Spec.Networking.GCE = &api.GCENetworkingSpec{} - default: - return fmt.Errorf("unknown networking mode %q", c.Networking) - } - - klog.V(4).Infof("networking mode=%s => %s", c.Networking, fi.DebugAsJsonString(cluster.Spec.Networking)) - if c.NetworkCIDR != "" { cluster.Spec.NetworkCIDR = c.NetworkCIDR } diff --git a/upup/pkg/fi/cloudup/BUILD.bazel b/upup/pkg/fi/cloudup/BUILD.bazel index 5f2a362b2f..2caf423dfd 100644 --- a/upup/pkg/fi/cloudup/BUILD.bazel +++ b/upup/pkg/fi/cloudup/BUILD.bazel @@ -84,6 +84,7 @@ go_library( "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/validation/field:go_default_library", "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/upup/pkg/fi/cloudup/new_cluster.go b/upup/pkg/fi/cloudup/new_cluster.go index c4a98e2fa9..cd8d59a174 100644 --- a/upup/pkg/fi/cloudup/new_cluster.go +++ b/upup/pkg/fi/cloudup/new_cluster.go @@ -23,15 +23,20 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" + "github.com/blang/semver/v4" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/klog" "k8s.io/kops" api "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/pkg/apis/kops/model" + version "k8s.io/kops/pkg/apis/kops/util" + "k8s.io/kops/pkg/apis/kops/validation" "k8s.io/kops/pkg/client/simple" "k8s.io/kops/pkg/featureflag" + "k8s.io/kops/pkg/model/components" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/cloudup/aliup" "k8s.io/kops/upup/pkg/fi/cloudup/awsup" @@ -54,6 +59,8 @@ type NewClusterOptions struct { Channel string // ConfigBase is the location where we will store the configuration. It defaults to the state store. ConfigBase string + // KubernetesVersion is the version of Kubernetes to deploy. It defaults to the version recommended by the channel. + KubernetesVersion string // CloudProvider is the name of the cloud provider. The default is to guess based on the Zones name. CloudProvider string @@ -101,11 +108,15 @@ type NewClusterOptions struct { // NodeCount is the number of nodes to create. Defaults to leaving the count unspecified // on the InstanceGroup, which results in a count of 2. NodeCount int32 + + // Networking is the networking provider/node to use. + Networking string } func (o *NewClusterOptions) InitDefaults() { o.Channel = api.DefaultChannel o.Authorization = AuthorizationFlagRBAC + o.Networking = "kubenet" } type NewClusterResult struct { @@ -149,6 +160,9 @@ func NewCluster(opt *NewClusterOptions, clientset simple.Clientset) (*NewCluster } } cluster.Spec.Channel = opt.Channel + if opt.KubernetesVersion != "" { + cluster.Spec.KubernetesVersion = opt.KubernetesVersion + } cluster.Spec.ConfigBase = opt.ConfigBase configBase, err := clientset.ConfigBaseFor(&cluster) @@ -211,6 +225,11 @@ func NewCluster(opt *NewClusterOptions, clientset simple.Clientset) (*NewCluster instanceGroups := append([]*api.InstanceGroup(nil), masters...) instanceGroups = append(instanceGroups, nodes...) + err = setupNetworking(opt, &cluster) + if err != nil { + return nil, err + } + result := NewClusterResult{ Cluster: &cluster, InstanceGroups: instanceGroups, @@ -693,3 +712,90 @@ func setupNodes(opt *NewClusterOptions, cluster *api.Cluster, zoneToSubnetMap ma return nodes, nil } + +func setupNetworking(opt *NewClusterOptions, cluster *api.Cluster) error { + cluster.Spec.Networking = &api.NetworkingSpec{} + switch opt.Networking { + case "kubenet": + cluster.Spec.Networking.Kubenet = &api.KubenetNetworkingSpec{} + case "external": + cluster.Spec.Networking.External = &api.ExternalNetworkingSpec{} + case "cni": + cluster.Spec.Networking.CNI = &api.CNINetworkingSpec{} + case "kopeio-vxlan", "kopeio": + cluster.Spec.Networking.Kopeio = &api.KopeioNetworkingSpec{} + case "weave": + cluster.Spec.Networking.Weave = &api.WeaveNetworkingSpec{} + + if cluster.Spec.CloudProvider == "aws" { + // AWS supports "jumbo frames" of 9001 bytes and weave adds up to 87 bytes overhead + // sets the default to the largest number that leaves enough overhead and is divisible by 4 + jumboFrameMTUSize := int32(8912) + cluster.Spec.Networking.Weave.MTU = &jumboFrameMTUSize + } + case "flannel", "flannel-vxlan": + cluster.Spec.Networking.Flannel = &api.FlannelNetworkingSpec{ + Backend: "vxlan", + } + case "flannel-udp": + klog.Warningf("flannel UDP mode is not recommended; consider flannel-vxlan instead") + cluster.Spec.Networking.Flannel = &api.FlannelNetworkingSpec{ + Backend: "udp", + } + case "calico": + cluster.Spec.Networking.Calico = &api.CalicoNetworkingSpec{ + MajorVersion: "v3", + } + // Validate to check if etcd clusters have an acceptable version + if errList := validation.ValidateEtcdVersionForCalicoV3(cluster.Spec.EtcdClusters[0], cluster.Spec.Networking.Calico.MajorVersion, field.NewPath("spec", "networking", "calico")); len(errList) != 0 { + + // This is not a special version but simply of the 3 series + for _, etcd := range cluster.Spec.EtcdClusters { + etcd.Version = components.DefaultEtcd3Version_1_11 + } + } + case "canal": + cluster.Spec.Networking.Canal = &api.CanalNetworkingSpec{} + case "kube-router": + cluster.Spec.Networking.Kuberouter = &api.KuberouterNetworkingSpec{} + if cluster.Spec.KubeProxy == nil { + cluster.Spec.KubeProxy = &api.KubeProxyConfig{} + } + enabled := false + cluster.Spec.KubeProxy.Enabled = &enabled + case "amazonvpc", "amazon-vpc-routed-eni": + cluster.Spec.Networking.AmazonVPC = &api.AmazonVPCNetworkingSpec{} + case "cilium": + cilium := &api.CiliumNetworkingSpec{} + cluster.Spec.Networking.Cilium = cilium + nodeport := false + if cluster.Spec.KubernetesVersion == "" { + nodeport = true + } else { + k8sVersion, err := semver.ParseTolerant(cluster.Spec.KubernetesVersion) + if err == nil { + if version.IsKubernetesGTE("1.12", k8sVersion) { + nodeport = true + } + } + } + if nodeport { + cilium.EnableNodePort = true + if cluster.Spec.KubeProxy == nil { + cluster.Spec.KubeProxy = &api.KubeProxyConfig{} + } + enabled := false + cluster.Spec.KubeProxy.Enabled = &enabled + } + case "lyftvpc": + cluster.Spec.Networking.LyftVPC = &api.LyftVPCNetworkingSpec{} + case "gce": + cluster.Spec.Networking.GCE = &api.GCENetworkingSpec{} + default: + return fmt.Errorf("unknown networking mode %q", opt.Networking) + } + + klog.V(4).Infof("networking mode=%s => %s", opt.Networking, fi.DebugAsJsonString(cluster.Spec.Networking)) + + return nil +} From d60eeabade23596206496b02687e65dd07684ae6 Mon Sep 17 00:00:00 2001 From: John Gardiner Myers Date: Fri, 3 Jul 2020 09:54:34 -0700 Subject: [PATCH 3/3] Move topology setup to pkg --- cmd/kops/BUILD.bazel | 1 - cmd/kops/create_cluster.go | 201 +---------------------------- upup/pkg/fi/cloudup/new_cluster.go | 142 +++++++++++++++++--- 3 files changed, 130 insertions(+), 214 deletions(-) diff --git a/cmd/kops/BUILD.bazel b/cmd/kops/BUILD.bazel index 37781c8bb9..8111b6b7e9 100644 --- a/cmd/kops/BUILD.bazel +++ b/cmd/kops/BUILD.bazel @@ -84,7 +84,6 @@ go_library( "//upup/pkg/fi:go_default_library", "//upup/pkg/fi/cloudup:go_default_library", "//upup/pkg/fi/cloudup/awsup:go_default_library", - "//upup/pkg/fi/cloudup/openstack:go_default_library", "//upup/pkg/fi/utils:go_default_library", "//upup/pkg/kutil:go_default_library", "//util/pkg/tables:go_default_library", diff --git a/cmd/kops/create_cluster.go b/cmd/kops/create_cluster.go index b56a3377e7..e3115e996a 100644 --- a/cmd/kops/create_cluster.go +++ b/cmd/kops/create_cluster.go @@ -40,8 +40,6 @@ import ( "k8s.io/kops/pkg/featureflag" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/cloudup" - "k8s.io/kops/upup/pkg/fi/cloudup/awsup" - "k8s.io/kops/upup/pkg/fi/cloudup/openstack" "k8s.io/kops/upup/pkg/fi/utils" "k8s.io/kubectl/pkg/util/i18n" "k8s.io/kubectl/pkg/util/templates" @@ -60,7 +58,6 @@ type CreateClusterOptions struct { Image string NodeImage string MasterImage string - UtilitySubnetIDs []string DisableSubnetTags bool NetworkCIDR string DNSZone string @@ -76,15 +73,6 @@ type CreateClusterOptions struct { // Overrides allows settings values direct in the spec Overrides []string - // The network topology to use - Topology string - - // The DNS type to use (public/private) - DNSType string - - // Enable/Disable Bastion Host complete setup - Bastion bool - // Specify tags for AWS instance groups CloudLabels string @@ -114,9 +102,6 @@ func (o *CreateClusterOptions) InitDefaults() { o.Yes = false o.Target = cloudup.TargetDirect - o.Topology = api.TopologyPublic - o.DNSType = string(api.DNSTypePublic) - o.Bastion = false // Default to open API & SSH access o.AdminAccess = []string{"0.0.0.0/0"} @@ -492,10 +477,6 @@ func RunCreateCluster(ctx context.Context, f *util.Factory, out io.Writer, c *Cr cluster.Spec.DNSZone = c.DNSZone } - // TODO: push more of the following logic into cloudup.NewCluster() - channel := clusterResult.Channel - allZones := clusterResult.AllZones - if c.ContainerRuntime != "" { cluster.Spec.ContainerRuntime = c.ContainerRuntime } @@ -504,123 +485,15 @@ func RunCreateCluster(ctx context.Context, f *util.Factory, out io.Writer, c *Cr cluster.Spec.NetworkCIDR = c.NetworkCIDR } - // Network Topology - if c.Topology == "" { - // The flag default should have set this, but we might be being called as a library - klog.Infof("Empty topology. Defaulting to public topology") - c.Topology = api.TopologyPublic - } - cluster.Spec.DisableSubnetTags = c.DisableSubnetTags - switch c.Topology { - case api.TopologyPublic: - cluster.Spec.Topology = &api.TopologySpec{ - Masters: api.TopologyPublic, - Nodes: api.TopologyPublic, - //Bastion: &api.BastionSpec{Enable: c.Bastion}, - } - - if c.Bastion { - return fmt.Errorf("Bastion supports --topology='private' only.") - } - - for i := range cluster.Spec.Subnets { - cluster.Spec.Subnets[i].Type = api.SubnetTypePublic - } - - case api.TopologyPrivate: - if cluster.Spec.Networking.Kubenet != nil { - return fmt.Errorf("invalid networking option %s. Kubenet does not support private topology", c.Networking) - } - cluster.Spec.Topology = &api.TopologySpec{ - Masters: api.TopologyPrivate, - Nodes: api.TopologyPrivate, - } - - for i := range cluster.Spec.Subnets { - cluster.Spec.Subnets[i].Type = api.SubnetTypePrivate - } - - var utilitySubnets []api.ClusterSubnetSpec - - var zoneToSubnetProviderID map[string]string - if len(c.Zones) > 0 && len(c.UtilitySubnetIDs) > 0 { - if api.CloudProviderID(cluster.Spec.CloudProvider) == api.CloudProviderAWS { - zoneToSubnetProviderID, err = getZoneToSubnetProviderID(cluster.Spec.NetworkID, c.Zones[0][:len(c.Zones[0])-1], c.UtilitySubnetIDs) - if err != nil { - return err - } - } else if api.CloudProviderID(cluster.Spec.CloudProvider) == api.CloudProviderOpenstack { - tags := make(map[string]string) - tags[openstack.TagClusterName] = c.ClusterName - zoneToSubnetProviderID, err = getSubnetProviderID(&cluster.Spec, allZones.List(), c.UtilitySubnetIDs, tags) - if err != nil { - return err - } - } - } - - for _, s := range cluster.Spec.Subnets { - if s.Type == api.SubnetTypeUtility { - continue - } - subnet := api.ClusterSubnetSpec{ - Name: "utility-" + s.Name, - Zone: s.Zone, - Type: api.SubnetTypeUtility, - } - if subnetID, ok := zoneToSubnetProviderID[s.Zone]; ok { - subnet.ProviderID = subnetID - } - utilitySubnets = append(utilitySubnets, subnet) - } - cluster.Spec.Subnets = append(cluster.Spec.Subnets, utilitySubnets...) - - if c.Bastion { - bastionGroup := &api.InstanceGroup{} - bastionGroup.Spec.Role = api.InstanceGroupRoleBastion - bastionGroup.ObjectMeta.Name = "bastions" - bastionGroup.Spec.Image = c.Image - instanceGroups = append(instanceGroups, bastionGroup) - - if !dns.IsGossipHostname(cluster.Name) { - cluster.Spec.Topology.Bastion = &api.BastionSpec{ - BastionPublicName: "bastion." + cluster.Name, - } - } - } - - default: - return fmt.Errorf("Invalid topology %s.", c.Topology) - } - - // DNS - if c.DNSType == "" { - // The flag default should have set this, but we might be being called as a library - klog.Infof("Empty DNS. Defaulting to public DNS") - c.DNSType = string(api.DNSTypePublic) - } - - if cluster.Spec.Topology == nil { - cluster.Spec.Topology = &api.TopologySpec{} - } - if cluster.Spec.Topology.DNS == nil { - cluster.Spec.Topology.DNS = &api.DNSSpec{} - } - switch strings.ToLower(c.DNSType) { - case "public": - cluster.Spec.Topology.DNS.Type = api.DNSTypePublic - case "private": - cluster.Spec.Topology.DNS.Type = api.DNSTypePrivate - default: - return fmt.Errorf("unknown DNSType: %q", c.DNSType) - } - if c.MasterPublicName != "" { cluster.Spec.MasterPublicName = c.MasterPublicName } + // TODO: push more of the following logic into cloudup.NewCluster() + channel := clusterResult.Channel + // check if we should set anonymousAuth to false { if cluster.Spec.Kubelet == nil { @@ -919,74 +792,6 @@ func initializeOpenstackAPI(c *CreateClusterOptions, cluster *api.Cluster) { } } -// TODO dedup or remove -func getZoneToSubnetProviderID(VPCID string, region string, subnetIDs []string) (map[string]string, error) { - res := make(map[string]string) - cloudTags := map[string]string{} - awsCloud, err := awsup.NewAWSCloud(region, cloudTags) - if err != nil { - return res, fmt.Errorf("error loading cloud: %v", err) - } - vpcInfo, err := awsCloud.FindVPCInfo(VPCID) - if err != nil { - return res, fmt.Errorf("error describing VPC: %v", err) - } - if vpcInfo == nil { - return res, fmt.Errorf("VPC %q not found", VPCID) - } - subnetByID := make(map[string]*fi.SubnetInfo) - for _, subnetInfo := range vpcInfo.Subnets { - subnetByID[subnetInfo.ID] = subnetInfo - } - for _, subnetID := range subnetIDs { - subnet, ok := subnetByID[subnetID] - if !ok { - return res, fmt.Errorf("subnet %s not found in VPC %s", subnetID, VPCID) - } - if res[subnet.Zone] != "" { - return res, fmt.Errorf("subnet %s and %s have the same zone", subnetID, res[subnet.Zone]) - } - res[subnet.Zone] = subnetID - } - return res, nil -} - -// TODO dedup or remove -func getSubnetProviderID(spec *api.ClusterSpec, zones []string, subnetIDs []string, tags map[string]string) (map[string]string, error) { - res := make(map[string]string) - osCloud, err := openstack.NewOpenstackCloud(tags, spec) - if err != nil { - return res, fmt.Errorf("error loading cloud: %v", err) - } - osCloud.UseZones(zones) - - networkInfo, err := osCloud.FindVPCInfo(spec.NetworkID) - if err != nil { - return res, fmt.Errorf("error describing Network: %v", err) - } - if networkInfo == nil { - return res, fmt.Errorf("network %q not found", spec.NetworkID) - } - - subnetByID := make(map[string]*fi.SubnetInfo) - for _, subnetInfo := range networkInfo.Subnets { - subnetByID[subnetInfo.ID] = subnetInfo - } - - for _, subnetID := range subnetIDs { - subnet, ok := subnetByID[subnetID] - if !ok { - return res, fmt.Errorf("subnet %s not found in network %s", subnetID, spec.NetworkID) - } - - if res[subnet.Zone] != "" { - return res, fmt.Errorf("subnet %s and %s have the same zone", subnetID, res[subnet.Zone]) - } - res[subnet.Zone] = subnetID - } - return res, nil -} - func loadSSHPublicKeys(sshPublicKey string) (map[string][]byte, error) { sshPublicKeys := make(map[string][]byte) if sshPublicKey != "" { diff --git a/upup/pkg/fi/cloudup/new_cluster.go b/upup/pkg/fi/cloudup/new_cluster.go index cd8d59a174..5c25cbccc4 100644 --- a/upup/pkg/fi/cloudup/new_cluster.go +++ b/upup/pkg/fi/cloudup/new_cluster.go @@ -35,6 +35,7 @@ import ( version "k8s.io/kops/pkg/apis/kops/util" "k8s.io/kops/pkg/apis/kops/validation" "k8s.io/kops/pkg/client/simple" + "k8s.io/kops/pkg/dns" "k8s.io/kops/pkg/featureflag" "k8s.io/kops/pkg/model/components" "k8s.io/kops/upup/pkg/fi" @@ -85,6 +86,8 @@ type NewClusterOptions struct { // SubnetIDs are the IDs of the shared subnets. // If empty, creates new subnets to be owned by the cluster. SubnetIDs []string + // UtilitySubnetIDs are the IDs of the shared utility subnets. If empty and the topology is "private", creates new subnets to be owned by the cluster. + UtilitySubnetIDs []string // Egress defines the method of traffic egress for subnets. Egress string @@ -108,15 +111,23 @@ type NewClusterOptions struct { // NodeCount is the number of nodes to create. Defaults to leaving the count unspecified // on the InstanceGroup, which results in a count of 2. NodeCount int32 + // Bastion enables the creation of a Bastion instance. + Bastion bool // Networking is the networking provider/node to use. Networking string + // Topology is the network topology to use. Defaults to "public". + Topology string + // DNSType is the DNS type to use; "public" or "private". Defaults to "public". + DNSType string } func (o *NewClusterOptions) InitDefaults() { o.Channel = api.DefaultChannel o.Authorization = AuthorizationFlagRBAC o.Networking = "kubenet" + o.Topology = api.TopologyPublic + o.DNSType = string(api.DNSTypePublic) } type NewClusterResult struct { @@ -126,8 +137,7 @@ type NewClusterResult struct { InstanceGroups []*api.InstanceGroup // TODO remove after more create_cluster logic refactored in - Channel *api.Channel - AllZones sets.String + Channel *api.Channel } // NewCluster initializes cluster and instance groups specifications as @@ -222,19 +232,24 @@ func NewCluster(opt *NewClusterOptions, clientset simple.Clientset) (*NewCluster return nil, err } - instanceGroups := append([]*api.InstanceGroup(nil), masters...) - instanceGroups = append(instanceGroups, nodes...) - err = setupNetworking(opt, &cluster) if err != nil { return nil, err } + bastions, err := setupTopology(opt, &cluster, allZones) + if err != nil { + return nil, err + } + + instanceGroups := append([]*api.InstanceGroup(nil), masters...) + instanceGroups = append(instanceGroups, nodes...) + instanceGroups = append(instanceGroups, bastions...) + result := NewClusterResult{ Cluster: &cluster, InstanceGroups: instanceGroups, Channel: channel, - AllZones: allZones, } return &result, nil } @@ -309,7 +324,7 @@ func setupVPC(opt *NewClusterOptions, cluster *api.Cluster) error { if cluster.Spec.NetworkID == "" && len(opt.SubnetIDs) > 0 { tags := make(map[string]string) - tags[openstack.TagClusterName] = opt.ClusterName + tags[openstack.TagClusterName] = cluster.Name osCloud, err := openstack.NewOpenstackCloud(tags, &cluster.Spec) if err != nil { return fmt.Errorf("error loading cloud: %v", err) @@ -413,7 +428,7 @@ func setupZones(opt *NewClusterOptions, cluster *api.Cluster, allZones sets.Stri case api.CloudProviderAWS: if len(opt.Zones) > 0 && len(opt.SubnetIDs) > 0 { - zoneToSubnetProviderID, err = getZoneToSubnetProviderID(cluster.Spec.NetworkID, opt.Zones[0][:len(opt.Zones[0])-1], opt.SubnetIDs) + zoneToSubnetProviderID, err = getAWSZoneToSubnetProviderID(cluster.Spec.NetworkID, opt.Zones[0][:len(opt.Zones[0])-1], opt.SubnetIDs) if err != nil { return nil, err } @@ -422,8 +437,8 @@ func setupZones(opt *NewClusterOptions, cluster *api.Cluster, allZones sets.Stri case api.CloudProviderOpenstack: if len(opt.Zones) > 0 && len(opt.SubnetIDs) > 0 { tags := make(map[string]string) - tags[openstack.TagClusterName] = opt.ClusterName - zoneToSubnetProviderID, err = getSubnetProviderID(&cluster.Spec, allZones.List(), opt.SubnetIDs, tags) + tags[openstack.TagClusterName] = cluster.Name + zoneToSubnetProviderID, err = getOpenstackZoneToSubnetProviderID(&cluster.Spec, allZones.List(), opt.SubnetIDs, tags) if err != nil { return nil, err } @@ -452,8 +467,7 @@ func setupZones(opt *NewClusterOptions, cluster *api.Cluster, allZones sets.Stri return zoneToSubnetMap, nil } -// TODO rename to getAWSZoneToSubnetProviderID -func getZoneToSubnetProviderID(VPCID string, region string, subnetIDs []string) (map[string]string, error) { +func getAWSZoneToSubnetProviderID(VPCID string, region string, subnetIDs []string) (map[string]string, error) { res := make(map[string]string) cloudTags := map[string]string{} awsCloud, err := awsup.NewAWSCloud(region, cloudTags) @@ -484,8 +498,7 @@ func getZoneToSubnetProviderID(VPCID string, region string, subnetIDs []string) return res, nil } -// TODO rename to getOpenstackZoneToSubnetProviderID -func getSubnetProviderID(spec *api.ClusterSpec, zones []string, subnetIDs []string, tags map[string]string) (map[string]string, error) { +func getOpenstackZoneToSubnetProviderID(spec *api.ClusterSpec, zones []string, subnetIDs []string, tags map[string]string) (map[string]string, error) { res := make(map[string]string) osCloud, err := openstack.NewOpenstackCloud(tags, spec) if err != nil { @@ -716,7 +729,7 @@ func setupNodes(opt *NewClusterOptions, cluster *api.Cluster, zoneToSubnetMap ma func setupNetworking(opt *NewClusterOptions, cluster *api.Cluster) error { cluster.Spec.Networking = &api.NetworkingSpec{} switch opt.Networking { - case "kubenet": + case "kubenet", "": cluster.Spec.Networking.Kubenet = &api.KubenetNetworkingSpec{} case "external": cluster.Spec.Networking.External = &api.ExternalNetworkingSpec{} @@ -799,3 +812,102 @@ func setupNetworking(opt *NewClusterOptions, cluster *api.Cluster) error { return nil } + +func setupTopology(opt *NewClusterOptions, cluster *api.Cluster, allZones sets.String) ([]*api.InstanceGroup, error) { + var bastions []*api.InstanceGroup + + switch opt.Topology { + case api.TopologyPublic, "": + cluster.Spec.Topology = &api.TopologySpec{ + Masters: api.TopologyPublic, + Nodes: api.TopologyPublic, + //Bastion: &api.BastionSpec{Enable: c.Bastion}, + } + + if opt.Bastion { + return nil, fmt.Errorf("bastion supports --topology='private' only") + } + + for i := range cluster.Spec.Subnets { + cluster.Spec.Subnets[i].Type = api.SubnetTypePublic + } + + case api.TopologyPrivate: + if cluster.Spec.Networking.Kubenet != nil { + return nil, fmt.Errorf("invalid networking option %s. Kubenet does not support private topology", opt.Networking) + } + cluster.Spec.Topology = &api.TopologySpec{ + Masters: api.TopologyPrivate, + Nodes: api.TopologyPrivate, + } + + for i := range cluster.Spec.Subnets { + cluster.Spec.Subnets[i].Type = api.SubnetTypePrivate + } + + var utilitySubnets []api.ClusterSubnetSpec + + var zoneToSubnetProviderID map[string]string + var err error + if len(opt.Zones) > 0 && len(opt.UtilitySubnetIDs) > 0 { + switch api.CloudProviderID(cluster.Spec.CloudProvider) { + case api.CloudProviderAWS: + zoneToSubnetProviderID, err = getAWSZoneToSubnetProviderID(cluster.Spec.NetworkID, opt.Zones[0][:len(opt.Zones[0])-1], opt.UtilitySubnetIDs) + if err != nil { + return nil, err + } + case api.CloudProviderOpenstack: + tags := make(map[string]string) + tags[openstack.TagClusterName] = cluster.Name + zoneToSubnetProviderID, err = getOpenstackZoneToSubnetProviderID(&cluster.Spec, allZones.List(), opt.UtilitySubnetIDs, tags) + if err != nil { + return nil, err + } + } + } + + for _, s := range cluster.Spec.Subnets { + if s.Type == api.SubnetTypeUtility { + continue + } + subnet := api.ClusterSubnetSpec{ + Name: "utility-" + s.Name, + Zone: s.Zone, + Type: api.SubnetTypeUtility, + } + if subnetID, ok := zoneToSubnetProviderID[s.Zone]; ok { + subnet.ProviderID = subnetID + } + utilitySubnets = append(utilitySubnets, subnet) + } + cluster.Spec.Subnets = append(cluster.Spec.Subnets, utilitySubnets...) + + if opt.Bastion { + bastionGroup := &api.InstanceGroup{} + bastionGroup.Spec.Role = api.InstanceGroupRoleBastion + bastionGroup.ObjectMeta.Name = "bastions" + bastions = append(bastions, bastionGroup) + + if !dns.IsGossipHostname(cluster.Name) { + cluster.Spec.Topology.Bastion = &api.BastionSpec{ + BastionPublicName: "bastion." + cluster.Name, + } + } + } + + default: + return nil, fmt.Errorf("invalid topology %s", opt.Topology) + } + + cluster.Spec.Topology.DNS = &api.DNSSpec{} + switch strings.ToLower(opt.DNSType) { + case "public", "": + cluster.Spec.Topology.DNS.Type = api.DNSTypePublic + case "private": + cluster.Spec.Topology.DNS.Type = api.DNSTypePrivate + default: + return nil, fmt.Errorf("unknown DNSType: %q", opt.DNSType) + } + + return bastions, nil +}