From 03c5f4c024d639b729a9eac04a9c70d9b3cc8499 Mon Sep 17 00:00:00 2001 From: John Gardiner Myers Date: Mon, 6 Jul 2020 20:53:18 -0700 Subject: [PATCH] Move remaining new cluster setup to pkg --- cmd/kops/BUILD.bazel | 1 - cmd/kops/create_cluster.go | 115 +---------------------------- pkg/apis/kops/cluster.go | 4 - pkg/apis/kops/instancegroup.go | 29 -------- upup/pkg/fi/cloudup/new_cluster.go | 110 +++++++++++++++++++++++++-- 5 files changed, 107 insertions(+), 152 deletions(-) diff --git a/cmd/kops/BUILD.bazel b/cmd/kops/BUILD.bazel index 8111b6b7e9..4518caf07a 100644 --- a/cmd/kops/BUILD.bazel +++ b/cmd/kops/BUILD.bazel @@ -66,7 +66,6 @@ go_library( "//pkg/client/simple:go_default_library", "//pkg/cloudinstances:go_default_library", "//pkg/commands:go_default_library", - "//pkg/dns:go_default_library", "//pkg/edit:go_default_library", "//pkg/featureflag:go_default_library", "//pkg/formatter:go_default_library", diff --git a/cmd/kops/create_cluster.go b/cmd/kops/create_cluster.go index 8219a9a61b..82a94fc531 100644 --- a/cmd/kops/create_cluster.go +++ b/cmd/kops/create_cluster.go @@ -36,7 +36,6 @@ import ( "k8s.io/kops/pkg/apis/kops/validation" "k8s.io/kops/pkg/assets" "k8s.io/kops/pkg/commands" - "k8s.io/kops/pkg/dns" "k8s.io/kops/pkg/featureflag" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/cloudup" @@ -61,8 +60,6 @@ type CreateClusterOptions struct { DisableSubnetTags bool NetworkCIDR string DNSZone string - AdminAccess []string - SSHAccess []string NodeSecurityGroups []string MasterSecurityGroups []string AssociatePublicIP *bool @@ -80,12 +77,6 @@ type CreateClusterOptions struct { MasterTenancy string NodeTenancy string - // Specify API loadbalancer as public or internal - APILoadBalancerType string - - // Specify the SSL certificate to use for the API loadbalancer. Currently only supported in AWS. - APISSLCertificate string - // Allow custom public master name MasterPublicName string @@ -103,9 +94,6 @@ func (o *CreateClusterOptions) InitDefaults() { o.Yes = false o.Target = cloudup.TargetDirect - // Default to open API & SSH access - o.AdminAccess = []string{"0.0.0.0/0"} - o.ContainerRuntime = "docker" } @@ -309,7 +297,7 @@ func NewCmdCreateCluster(f *util.Factory, out io.Writer) *cobra.Command { // Openstack flags cmd.Flags().StringVar(&options.OpenstackExternalNet, "os-ext-net", options.OpenstackExternalNet, "The name of the external network to use with the openstack router") cmd.Flags().StringVar(&options.OpenstackExternalSubnet, "os-ext-subnet", options.OpenstackExternalSubnet, "The name of the external floating subnet to use with the openstack router") - cmd.Flags().StringVar(&options.OpenstackLbSubnet, "os-lb-floating-subnet", options.OpenstackLbSubnet, "The name of the external subnet to use with the kubernetes api") + cmd.Flags().StringVar(&options.OpenstackLBSubnet, "os-lb-floating-subnet", options.OpenstackLBSubnet, "The name of the external subnet to use with the kubernetes api") cmd.Flags().BoolVar(&options.OpenstackStorageIgnoreAZ, "os-kubelet-ignore-az", options.OpenstackStorageIgnoreAZ, "If true kubernetes may attach volumes across availability zones") cmd.Flags().BoolVar(&options.OpenstackLBOctavia, "os-octavia", options.OpenstackLBOctavia, "If true octavia loadbalancer api will be used") cmd.Flags().StringVar(&options.OpenstackDNSServers, "os-dns-servers", options.OpenstackDNSServers, "comma separated list of DNS Servers which is used in network") @@ -491,80 +479,6 @@ func RunCreateCluster(ctx context.Context, f *util.Factory, out io.Writer, c *Cr cluster.Spec.MasterPublicName = c.MasterPublicName } - // TODO: push more of the following logic into cloudup.NewCluster() - channel := clusterResult.Channel - - // check if we should set anonymousAuth to false - { - if cluster.Spec.Kubelet == nil { - cluster.Spec.Kubelet = &api.KubeletConfigSpec{} - } - - if cluster.Spec.Kubelet.AnonymousAuth == nil { - cluster.Spec.Kubelet.AnonymousAuth = fi.Bool(false) - } - } - - // Populate the API access, so that it can be discoverable - // TODO: This is the same code as in defaults - try to dedup? - if cluster.Spec.API == nil { - cluster.Spec.API = &api.AccessSpec{} - } - if cluster.Spec.API.IsEmpty() { - if c.CloudProvider == "openstack" { - initializeOpenstackAPI(c, cluster) - } else if c.APILoadBalancerType != "" { - cluster.Spec.API.LoadBalancer = &api.LoadBalancerAccessSpec{} - } else { - switch cluster.Spec.Topology.Masters { - case api.TopologyPublic: - if dns.IsGossipHostname(cluster.Name) { - // gossip DNS names don't work outside the cluster, so we use a LoadBalancer instead - cluster.Spec.API.LoadBalancer = &api.LoadBalancerAccessSpec{} - } else { - cluster.Spec.API.DNS = &api.DNSAccessSpec{} - } - - case api.TopologyPrivate: - cluster.Spec.API.LoadBalancer = &api.LoadBalancerAccessSpec{} - - default: - return fmt.Errorf("unknown master topology type: %q", cluster.Spec.Topology.Masters) - } - } - } - if cluster.Spec.API.LoadBalancer != nil && cluster.Spec.API.LoadBalancer.Type == "" { - switch c.APILoadBalancerType { - case "", "public": - cluster.Spec.API.LoadBalancer.Type = api.LoadBalancerTypePublic - case "internal": - cluster.Spec.API.LoadBalancer.Type = api.LoadBalancerTypeInternal - default: - return fmt.Errorf("unknown api-loadbalancer-type: %q", c.APILoadBalancerType) - } - } - - if cluster.Spec.API.LoadBalancer != nil && c.APISSLCertificate != "" { - cluster.Spec.API.LoadBalancer.SSLCertificate = c.APISSLCertificate - } - - // Use Strict IAM policy and allow AWS ECR by default when creating a new cluster - cluster.Spec.IAM = &api.IAMSpec{ - AllowContainerRegistry: true, - Legacy: false, - } - - if len(c.AdminAccess) != 0 { - if len(c.SSHAccess) != 0 { - cluster.Spec.SSHAccess = c.SSHAccess - } else { - cluster.Spec.SSHAccess = c.AdminAccess - } - cluster.Spec.KubernetesAPIAccess = c.AdminAccess - } else if len(c.AdminAccess) == 0 { - cluster.Spec.SSHAccess = c.SSHAccess - } - if err := commands.SetClusterFields(c.Overrides, cluster, instanceGroups); err != nil { return err } @@ -573,10 +487,6 @@ func RunCreateCluster(ctx context.Context, f *util.Factory, out io.Writer, c *Cr if err != nil { return fmt.Errorf("error populating configuration: %v", err) } - err = api.PerformAssignmentsInstanceGroups(instanceGroups) - if err != nil { - return fmt.Errorf("error populating configuration: %v", err) - } if cluster.Spec.ExternalCloudControllerManager != nil && !featureflag.EnableExternalCloudController.Enabled() { klog.Warningf("Without setting the feature flag `+EnableExternalCloudController` the external cloud controller manager configuration will be discarded") @@ -596,7 +506,7 @@ func RunCreateCluster(ctx context.Context, f *util.Factory, out io.Writer, c *Cr var fullInstanceGroups []*api.InstanceGroup for _, group := range instanceGroups { - fullGroup, err := cloudup.PopulateInstanceGroupSpec(fullCluster, group, channel) + fullGroup, err := cloudup.PopulateInstanceGroupSpec(fullCluster, group, clusterResult.Channel) if err != nil { return err } @@ -771,27 +681,6 @@ func parseCloudLabels(s string) (map[string]string, error) { return m, nil } -func initializeOpenstackAPI(c *CreateClusterOptions, cluster *api.Cluster) { - if c.APILoadBalancerType != "" { - cluster.Spec.API.LoadBalancer = &api.LoadBalancerAccessSpec{} - provider := "haproxy" - if c.OpenstackLBOctavia { - provider = "octavia" - } - - cluster.Spec.CloudConfig.Openstack.Loadbalancer = &api.OpenstackLoadbalancerConfig{ - FloatingNetwork: fi.String(c.OpenstackExternalNet), - Method: fi.String("ROUND_ROBIN"), - Provider: fi.String(provider), - UseOctavia: fi.Bool(c.OpenstackLBOctavia), - } - - if c.OpenstackLbSubnet != "" { - cluster.Spec.CloudConfig.Openstack.Loadbalancer.FloatingSubnet = fi.String(c.OpenstackLbSubnet) - } - } -} - func loadSSHPublicKeys(sshPublicKey string) (map[string][]byte, error) { sshPublicKeys := make(map[string][]byte) if sshPublicKey != "" { diff --git a/pkg/apis/kops/cluster.go b/pkg/apis/kops/cluster.go index cf428fd069..09f2ac09b5 100644 --- a/pkg/apis/kops/cluster.go +++ b/pkg/apis/kops/cluster.go @@ -335,10 +335,6 @@ type AccessSpec struct { LoadBalancer *LoadBalancerAccessSpec `json:"loadBalancer,omitempty"` } -func (s *AccessSpec) IsEmpty() bool { - return s.DNS == nil && s.LoadBalancer == nil -} - type DNSAccessSpec struct { } diff --git a/pkg/apis/kops/instancegroup.go b/pkg/apis/kops/instancegroup.go index 58700c8f34..931ce83fa7 100644 --- a/pkg/apis/kops/instancegroup.go +++ b/pkg/apis/kops/instancegroup.go @@ -17,8 +17,6 @@ limitations under the License. package kops import ( - "fmt" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/klog" ) @@ -251,33 +249,6 @@ type IAMProfileSpec struct { Profile *string `json:"profile,omitempty"` } -// PerformAssignmentsInstanceGroups populates InstanceGroups with default values -func PerformAssignmentsInstanceGroups(groups []*InstanceGroup) error { - names := map[string]bool{} - for _, group := range groups { - names[group.ObjectMeta.Name] = true - } - - for _, group := range groups { - // We want to give them a stable Name as soon as possible - if group.ObjectMeta.Name == "" { - // Loop to find the first unassigned name like `nodes-%d` - i := 0 - for { - key := fmt.Sprintf("nodes-%d", i) - if !names[key] { - group.ObjectMeta.Name = key - names[key] = true - break - } - i++ - } - } - } - - return nil -} - // IsMaster checks if instanceGroup is a master func (g *InstanceGroup) IsMaster() bool { switch g.Spec.Role { diff --git a/upup/pkg/fi/cloudup/new_cluster.go b/upup/pkg/fi/cloudup/new_cluster.go index 5c25cbccc4..ee9d6350e1 100644 --- a/upup/pkg/fi/cloudup/new_cluster.go +++ b/upup/pkg/fi/cloudup/new_cluster.go @@ -62,6 +62,10 @@ type NewClusterOptions struct { ConfigBase string // KubernetesVersion is the version of Kubernetes to deploy. It defaults to the version recommended by the channel. KubernetesVersion string + // AdminAccess is the set of CIDR blocks permitted to connect to the Kubernetes API. It defaults to "0.0.0.0/0". + AdminAccess []string + // SSHAccess is the set of CIDR blocks permitted to connect to SSH on the nodes. It defaults to the value of AdminAccess. + SSHAccess []string // CloudProvider is the name of the cloud provider. The default is to guess based on the Zones name. CloudProvider string @@ -91,13 +95,13 @@ type NewClusterOptions struct { // Egress defines the method of traffic egress for subnets. Egress string - // OpenstackExternalNet is the name of the external network for the openstack router + // OpenstackExternalNet is the name of the external network for the openstack router. OpenstackExternalNet string OpenstackExternalSubnet string OpenstackStorageIgnoreAZ bool OpenstackDNSServers string - OpenstackLbSubnet string - // OpenstackLBOctavia is boolean value should we use octavia or old loadbalancer api + OpenstackLBSubnet string + // OpenstackLBOctavia is whether to use use octavia instead of haproxy. OpenstackLBOctavia bool // MasterCount is the number of masters to create. Defaults to the length of MasterZones @@ -120,11 +124,19 @@ type NewClusterOptions struct { Topology string // DNSType is the DNS type to use; "public" or "private". Defaults to "public". DNSType string + + // APILoadBalancerType is the Kubernetes API loadbalancer type to use; "public" or "internal". + // Defaults to using DNS instead of a load balancer if using public topology and not gossip, otherwise "public". + APILoadBalancerType string + // APISSLCertificate is the SSL certificate to use for the API loadbalancer. + // Currently only supported in AWS. + APISSLCertificate string } func (o *NewClusterOptions) InitDefaults() { o.Channel = api.DefaultChannel o.Authorization = AuthorizationFlagRBAC + o.AdminAccess = []string{"0.0.0.0/0"} o.Networking = "kubenet" o.Topology = api.TopologyPublic o.DNSType = string(api.DNSTypePublic) @@ -135,13 +147,14 @@ type NewClusterResult struct { Cluster *api.Cluster // InstanceGroups are the initialized InstanceGroup resources. InstanceGroups []*api.InstanceGroup - - // TODO remove after more create_cluster logic refactored in + // Channel is the loaded Channel object. Channel *api.Channel } // NewCluster initializes cluster and instance groups specifications as // intended for newly created clusters. +// It is the responsibility of the caller to call cloudup.PerformAssignments() on +// the returned cluster spec. func NewCluster(opt *NewClusterOptions, clientset simple.Clientset) (*NewClusterResult, error) { if opt.ClusterName == "" { return nil, fmt.Errorf("name is required") @@ -190,6 +203,24 @@ func NewCluster(opt *NewClusterOptions, clientset simple.Clientset) (*NewCluster return nil, fmt.Errorf("unknown authorization mode %q", opt.Authorization) } + cluster.Spec.IAM = &api.IAMSpec{ + AllowContainerRegistry: true, + Legacy: false, + } + cluster.Spec.Kubelet = &api.KubeletConfigSpec{ + AnonymousAuth: fi.Bool(false), + } + + if len(opt.AdminAccess) == 0 { + opt.AdminAccess = []string{"0.0.0.0/0"} + } + cluster.Spec.KubernetesAPIAccess = opt.AdminAccess + if len(opt.SSHAccess) != 0 { + cluster.Spec.SSHAccess = opt.SSHAccess + } else { + cluster.Spec.SSHAccess = opt.AdminAccess + } + allZones := sets.NewString() allZones.Insert(opt.Zones...) allZones.Insert(opt.MasterZones...) @@ -242,6 +273,11 @@ func NewCluster(opt *NewClusterOptions, clientset simple.Clientset) (*NewCluster return nil, err } + err = setupAPI(opt, &cluster) + if err != nil { + return nil, err + } + instanceGroups := append([]*api.InstanceGroup(nil), masters...) instanceGroups = append(instanceGroups, nodes...) instanceGroups = append(instanceGroups, bastions...) @@ -911,3 +947,67 @@ func setupTopology(opt *NewClusterOptions, cluster *api.Cluster, allZones sets.S return bastions, nil } + +func setupAPI(opt *NewClusterOptions, cluster *api.Cluster) error { + // Populate the API access, so that it can be discoverable + cluster.Spec.API = &api.AccessSpec{} + if api.CloudProviderID(cluster.Spec.CloudProvider) == api.CloudProviderOpenstack { + initializeOpenstackAPI(opt, cluster) + } else if opt.APILoadBalancerType != "" { + cluster.Spec.API.LoadBalancer = &api.LoadBalancerAccessSpec{} + } else { + switch cluster.Spec.Topology.Masters { + case api.TopologyPublic: + if dns.IsGossipHostname(cluster.Name) { + // gossip DNS names don't work outside the cluster, so we use a LoadBalancer instead + cluster.Spec.API.LoadBalancer = &api.LoadBalancerAccessSpec{} + } else { + cluster.Spec.API.DNS = &api.DNSAccessSpec{} + } + + case api.TopologyPrivate: + cluster.Spec.API.LoadBalancer = &api.LoadBalancerAccessSpec{} + + default: + return fmt.Errorf("unknown master topology type: %q", cluster.Spec.Topology.Masters) + } + } + + if cluster.Spec.API.LoadBalancer != nil && cluster.Spec.API.LoadBalancer.Type == "" { + switch opt.APILoadBalancerType { + case "", "public": + cluster.Spec.API.LoadBalancer.Type = api.LoadBalancerTypePublic + case "internal": + cluster.Spec.API.LoadBalancer.Type = api.LoadBalancerTypeInternal + default: + return fmt.Errorf("unknown api-loadbalancer-type: %q", opt.APILoadBalancerType) + } + } + + if cluster.Spec.API.LoadBalancer != nil && opt.APISSLCertificate != "" { + cluster.Spec.API.LoadBalancer.SSLCertificate = opt.APISSLCertificate + } + + return nil +} + +func initializeOpenstackAPI(opt *NewClusterOptions, cluster *api.Cluster) { + if opt.APILoadBalancerType != "" { + cluster.Spec.API.LoadBalancer = &api.LoadBalancerAccessSpec{} + provider := "haproxy" + if opt.OpenstackLBOctavia { + provider = "octavia" + } + + cluster.Spec.CloudConfig.Openstack.Loadbalancer = &api.OpenstackLoadbalancerConfig{ + FloatingNetwork: fi.String(opt.OpenstackExternalNet), + Method: fi.String("ROUND_ROBIN"), + Provider: fi.String(provider), + UseOctavia: fi.Bool(opt.OpenstackLBOctavia), + } + + if opt.OpenstackLBSubnet != "" { + cluster.Spec.CloudConfig.Openstack.Loadbalancer.FloatingSubnet = fi.String(opt.OpenstackLBSubnet) + } + } +}