diff --git a/pkg/apis/nodeup/config.go b/pkg/apis/nodeup/config.go index cbdc80f313..6ac948020e 100644 --- a/pkg/apis/nodeup/config.go +++ b/pkg/apis/nodeup/config.go @@ -20,10 +20,8 @@ import ( "strings" "k8s.io/kops/pkg/apis/kops" - "k8s.io/kops/pkg/nodelabels" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/util/pkg/architectures" - "k8s.io/kops/util/pkg/reflectutils" ) // Config is the configuration for the nodeup binary @@ -139,7 +137,6 @@ type APIServerConfig struct { func NewConfig(cluster *kops.Cluster, instanceGroup *kops.InstanceGroup) (*Config, *BootConfig) { role := instanceGroup.Spec.Role - isMaster := role == kops.InstanceGroupRoleMaster clusterHooks := filterHooks(cluster.Spec.Hooks, instanceGroup.Spec.Role) igHooks := filterHooks(instanceGroup.Spec.Hooks, instanceGroup.Spec.Role) @@ -165,39 +162,6 @@ func NewConfig(cluster *kops.Cluster, instanceGroup *kops.InstanceGroup) (*Confi config.EnableLifecycleHook = true } - if isMaster { - reflectutils.JSONMergeStruct(&config.KubeletConfig, cluster.Spec.MasterKubelet) - - // A few settings in Kubelet override those in MasterKubelet. I'm not sure why. - if cluster.Spec.Kubelet != nil && cluster.Spec.Kubelet.AnonymousAuth != nil && !*cluster.Spec.Kubelet.AnonymousAuth { - config.KubeletConfig.AnonymousAuth = fi.Bool(false) - } - } else { - reflectutils.JSONMergeStruct(&config.KubeletConfig, cluster.Spec.Kubelet) - } - - if isMaster || role == kops.InstanceGroupRoleAPIServer { - config.APIServerConfig = &APIServerConfig{ - KubeAPIServer: cluster.Spec.KubeAPIServer, - } - } - - if instanceGroup.Spec.Kubelet != nil { - useSecureKubelet := config.KubeletConfig.AnonymousAuth != nil && !*config.KubeletConfig.AnonymousAuth - - reflectutils.JSONMergeStruct(&config.KubeletConfig, instanceGroup.Spec.Kubelet) - - if useSecureKubelet { - config.KubeletConfig.AnonymousAuth = fi.Bool(false) - } - } - - // We include the NodeLabels in the userdata even for Kubernetes 1.16 and later so that - // rolling update will still replace nodes when they change. - config.KubeletConfig.NodeLabels = nodelabels.BuildNodeLabels(cluster, instanceGroup) - - config.KubeletConfig.Taints = append(config.KubeletConfig.Taints, instanceGroup.Spec.Taints...) - if instanceGroup.Spec.UpdatePolicy != nil { config.UpdatePolicy = *instanceGroup.Spec.UpdatePolicy } else if cluster.Spec.UpdatePolicy != nil { @@ -214,6 +178,16 @@ func NewConfig(cluster *kops.Cluster, instanceGroup *kops.InstanceGroup) (*Confi config.UseInstanceIDForNodeName = true } + if instanceGroup.Spec.Kubelet != nil { + config.KubeletConfig = *instanceGroup.Spec.Kubelet + } + + if instanceGroup.HasAPIServer() { + config.APIServerConfig = &APIServerConfig{ + KubeAPIServer: cluster.Spec.KubeAPIServer, + } + } + return &config, &bootConfig } diff --git a/upup/pkg/fi/cloudup/populate_instancegroup_spec.go b/upup/pkg/fi/cloudup/populate_instancegroup_spec.go index 3c480511c1..14e73b5e04 100644 --- a/upup/pkg/fi/cloudup/populate_instancegroup_spec.go +++ b/upup/pkg/fi/cloudup/populate_instancegroup_spec.go @@ -24,6 +24,8 @@ import ( "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/pkg/apis/kops/validation" + "k8s.io/kops/pkg/featureflag" + "k8s.io/kops/pkg/nodelabels" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/cloudup/awsup" "k8s.io/kops/upup/pkg/fi/cloudup/openstack" @@ -60,7 +62,6 @@ var awsDedicatedInstanceExceptions = map[string]bool{ } // PopulateInstanceGroupSpec sets default values in the InstanceGroup -// The InstanceGroup is simpler than the cluster spec, so we just populate in place (like the rest of k8s) func PopulateInstanceGroupSpec(cluster *kops.Cluster, input *kops.InstanceGroup, cloud fi.Cloud, channel *kops.Channel) (*kops.InstanceGroup, error) { klog.Infof("Populating instance group spec for %q", input.GetName()) @@ -73,6 +74,110 @@ func PopulateInstanceGroupSpec(cluster *kops.Cluster, input *kops.InstanceGroup, ig := &kops.InstanceGroup{} reflectutils.JSONMergeStruct(ig, input) + spec := &ig.Spec + + // TODO: Clean up + if ig.IsMaster() { + if ig.Spec.MachineType == "" { + ig.Spec.MachineType, err = defaultMachineType(cloud, cluster, ig) + if err != nil { + return nil, fmt.Errorf("error assigning default machine type for masters: %v", err) + } + + } + if ig.Spec.MinSize == nil { + ig.Spec.MinSize = fi.Int32(1) + } + if ig.Spec.MaxSize == nil { + ig.Spec.MaxSize = fi.Int32(1) + } + } else if ig.Spec.Role == kops.InstanceGroupRoleBastion { + if ig.Spec.MachineType == "" { + ig.Spec.MachineType, err = defaultMachineType(cloud, cluster, ig) + if err != nil { + return nil, fmt.Errorf("error assigning default machine type for bastions: %v", err) + } + } + if ig.Spec.MinSize == nil { + ig.Spec.MinSize = fi.Int32(1) + } + if ig.Spec.MaxSize == nil { + ig.Spec.MaxSize = fi.Int32(1) + } + } else { + if ig.IsAPIServerOnly() && !featureflag.APIServerNodes.Enabled() { + return nil, fmt.Errorf("apiserver nodes requires the APIServerNodes feature flag to be enabled") + } + if ig.Spec.MachineType == "" { + ig.Spec.MachineType, err = defaultMachineType(cloud, cluster, ig) + if err != nil { + return nil, fmt.Errorf("error assigning default machine type for nodes: %v", err) + } + } + if ig.Spec.MinSize == nil { + ig.Spec.MinSize = fi.Int32(2) + } + if ig.Spec.MaxSize == nil { + ig.Spec.MaxSize = fi.Int32(2) + } + } + + if ig.Spec.Image == "" { + architecture, err := MachineArchitecture(cloud, ig.Spec.MachineType) + if err != nil { + return nil, fmt.Errorf("unable to determine machine architecture for InstanceGroup %q: %v", ig.ObjectMeta.Name, err) + } + ig.Spec.Image = defaultImage(cluster, channel, architecture) + if ig.Spec.Image == "" { + return nil, fmt.Errorf("unable to determine default image for InstanceGroup %s", ig.ObjectMeta.Name) + } + } + + if ig.Spec.Tenancy != "" && ig.Spec.Tenancy != "default" { + switch cluster.Spec.GetCloudProvider() { + case kops.CloudProviderAWS: + if _, ok := awsDedicatedInstanceExceptions[ig.Spec.MachineType]; ok { + return nil, fmt.Errorf("invalid dedicated instance type: %s", ig.Spec.MachineType) + } + default: + klog.Warning("Trying to set tenancy on non-AWS environment") + } + } + + if ig.IsMaster() { + if len(ig.Spec.Subnets) == 0 { + return nil, fmt.Errorf("master InstanceGroup %s did not specify any Subnets", ig.ObjectMeta.Name) + } + } else if ig.IsAPIServerOnly() && cluster.Spec.IsIPv6Only() { + if len(ig.Spec.Subnets) == 0 { + for _, subnet := range cluster.Spec.Subnets { + if subnet.Type != kops.SubnetTypePrivate && subnet.Type != kops.SubnetTypeUtility { + ig.Spec.Subnets = append(ig.Spec.Subnets, subnet.Name) + } + } + } + } else { + if len(ig.Spec.Subnets) == 0 { + for _, subnet := range cluster.Spec.Subnets { + if subnet.Type != kops.SubnetTypeDualStack && subnet.Type != kops.SubnetTypeUtility { + ig.Spec.Subnets = append(ig.Spec.Subnets, subnet.Name) + } + } + } + + if len(ig.Spec.Subnets) == 0 { + for _, subnet := range cluster.Spec.Subnets { + if subnet.Type != kops.SubnetTypeUtility { + ig.Spec.Subnets = append(ig.Spec.Subnets, subnet.Name) + } + } + } + } + + if len(ig.Spec.Subnets) == 0 { + return nil, fmt.Errorf("unable to infer any Subnets for InstanceGroup %s ", ig.ObjectMeta.Name) + } + hasGPU := false clusterNvidia := false if cluster.Spec.Containerd != nil && cluster.Spec.Containerd.NvidiaGPU != nil && fi.BoolValue(cluster.Spec.Containerd.NvidiaGPU.Enabled) { @@ -117,6 +222,40 @@ func PopulateInstanceGroupSpec(cluster *kops.Cluster, input *kops.InstanceGroup, if ig.Spec.Manager == "" { ig.Spec.Manager = kops.InstanceManagerCloudGroup } + + if spec.Kubelet == nil { + spec.Kubelet = &kops.KubeletConfigSpec{} + } + + kubeletConfig := spec.Kubelet + + // We include the NodeLabels in the userdata even for Kubernetes 1.16 and later so that + // rolling update will still replace nodes when they change. + kubeletConfig.NodeLabels = nodelabels.BuildNodeLabels(cluster, ig) + + kubeletConfig.Taints = append(kubeletConfig.Taints, spec.Taints...) + + if ig.IsMaster() { + reflectutils.JSONMergeStruct(kubeletConfig, cluster.Spec.MasterKubelet) + + // A few settings in Kubelet override those in MasterKubelet. I'm not sure why. + if cluster.Spec.Kubelet != nil && cluster.Spec.Kubelet.AnonymousAuth != nil && !*cluster.Spec.Kubelet.AnonymousAuth { + kubeletConfig.AnonymousAuth = fi.Bool(false) + } + } else { + reflectutils.JSONMergeStruct(kubeletConfig, cluster.Spec.Kubelet) + } + + if ig.Spec.Kubelet != nil { + useSecureKubelet := fi.BoolValue(kubeletConfig.AnonymousAuth) + + reflectutils.JSONMergeStruct(kubeletConfig, spec.Kubelet) + + if useSecureKubelet { + kubeletConfig.AnonymousAuth = fi.Bool(false) + } + } + return ig, nil }