kubernetes
/
autoscaler
mirror of https://github.com/kubernetes/autoscaler.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Break down cloud provider builder by provider 

The Build method was getting pretty big, this hopefully makes it a little
more readable. It also fixes a few minor error shadowing bugs.
This commit is contained in:
Nic Cope 2017-11-15 17:23:59 -08:00
parent 19607bd285
commit 6a704a6cf4
5 changed files with 172 additions and 128 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
cluster-autoscaler/cloudprovider/aws/aws_cloud_provider.go
View File

@ -29,6 +29,11 @@ import (
"k8s.io/kubernetes/plugin/pkg/scheduler/schedulercache"
)
const (
// ProviderName is the cloud provider name for AWS
ProviderName = "aws"
)
// awsCloudProvider implements CloudProvider interface.
type awsCloudProvider struct {
awsManager *AwsManager
@ -144,7 +149,7 @@ func (aws *awsCloudProvider) addAsg(asg *Asg) {
// Name returns name of the cloud provider.
func (aws *awsCloudProvider) Name() string {
return "aws"
return ProviderName
}
// NodeGroups returns all node groups configured for this cloud provider.

5
cluster-autoscaler/cloudprovider/azure/azure_cloud_provider.go
View File

@ -30,6 +30,11 @@ import (
"k8s.io/kubernetes/plugin/pkg/scheduler/schedulercache"
)
const (
// ProviderName is the cloud provider name for Azure
ProviderName = "azure"
)
// AzureCloudProvider provides implementation of CloudProvider interface for Azure.
type AzureCloudProvider struct {
azureManager *AzureManager

265
cluster-autoscaler/cloudprovider/builder/cloud_provider_builder.go
View File

@ -17,6 +17,7 @@ limitations under the License.
package builder
import (
"io"
"os"
"k8s.io/autoscaler/cluster-autoscaler/cloudprovider"
@ -33,6 +34,18 @@ import (
"github.com/golang/glog"
)
// AvailableCloudProviders supported by the cloud provider builder.
var AvailableCloudProviders = []string{
aws.ProviderName,
azure.ProviderName,
gce.ProviderNameGCE,
gce.ProviderNameGKE,
kubemark.ProviderName,
}
// DefaultCloudProvider is GCE.
const DefaultCloudProvider = gce.ProviderNameGCE
// CloudProviderBuilder builds a cloud provider from all the necessary parameters including the name of a cloud provider e.g. aws, gce
// and the path to a config file
type CloudProviderBuilder struct {
@ -54,128 +67,138 @@ func NewCloudProviderBuilder(cloudProviderFlag string, cloudConfig string, clust
// Build a cloud provider from static settings contained in the builder and dynamic settings passed via args
func (b CloudProviderBuilder) Build(discoveryOpts cloudprovider.NodeGroupDiscoveryOptions, resourceLimiter *cloudprovider.ResourceLimiter) cloudprovider.CloudProvider {
var err error
var cloudProvider cloudprovider.CloudProvider
nodeGroupsFlag := discoveryOpts.NodeGroupSpecs
if b.cloudProviderFlag == "gce" || b.cloudProviderFlag == "gke" {
// GCE Manager
var gceManager gce.GceManager
var gceError error
mode := gce.ModeGCE
if b.cloudProviderFlag == "gke" {
if b.autoprovisioningEnabled {
mode = gce.ModeGKENAP
} else {
mode = gce.ModeGKE
}
}
if b.cloudConfig != "" {
config, fileErr := os.Open(b.cloudConfig)
if fileErr != nil {
glog.Fatalf("Couldn't open cloud provider configuration %s: %#v", b.cloudConfig, err)
}
defer config.Close()
gceManager, gceError = gce.CreateGceManager(config, mode, b.clusterName)
} else {
gceManager, gceError = gce.CreateGceManager(nil, mode, b.clusterName)
}
if gceError != nil {
glog.Fatalf("Failed to create GCE Manager: %v", gceError)
}
cloudProvider, err = gce.BuildGceCloudProvider(gceManager, discoveryOpts, resourceLimiter)
if err != nil {
glog.Fatalf("Failed to create GCE cloud provider: %v", err)
glog.V(1).Infof("Building %s cloud provider.", b.cloudProviderFlag)
switch b.cloudProviderFlag {
case gce.ProviderNameGCE:
return b.buildGCE(discoveryOpts, resourceLimiter, gce.ModeGCE)
case gce.ProviderNameGKE:
if b.autoprovisioningEnabled {
return b.buildGCE(discoveryOpts, resourceLimiter, gce.ModeGKENAP)
}
return b.buildGCE(discoveryOpts, resourceLimiter, gce.ModeGKE)
case aws.ProviderName:
return b.buildAWS(discoveryOpts, resourceLimiter)
case azure.ProviderName:
return b.buildAzure(discoveryOpts, resourceLimiter)
case kubemark.ProviderName:
return b.buildKubemark(discoveryOpts, resourceLimiter)
case "":
// Ideally this would be an error, but several unit tests of the
// StaticAutoscaler depend on this behaviour.
glog.Warning("Returning a nil cloud provider")
return nil
}
if b.cloudProviderFlag == "aws" {
var awsManager *aws.AwsManager
var awsError error
if b.cloudConfig != "" {
config, fileErr := os.Open(b.cloudConfig)
if fileErr != nil {
glog.Fatalf("Couldn't open cloud provider configuration %s: %#v", b.cloudConfig, err)
}
defer config.Close()
awsManager, awsError = aws.CreateAwsManager(config)
} else {
awsManager, awsError = aws.CreateAwsManager(nil)
}
if awsError != nil {
glog.Fatalf("Failed to create AWS Manager: %v", err)
}
cloudProvider, err = aws.BuildAwsCloudProvider(awsManager, discoveryOpts, resourceLimiter)
if err != nil {
glog.Fatalf("Failed to create AWS cloud provider: %v", err)
}
}
if b.cloudProviderFlag == "azure" {
var azureManager *azure.AzureManager
var azureError error
if b.cloudConfig != "" {
glog.Info("Creating Azure Manager using cloud-config file: %v", b.cloudConfig)
config, fileErr := os.Open(b.cloudConfig)
if fileErr != nil {
glog.Fatalf("Couldn't open cloud provider configuration %s: %#v", b.cloudConfig, err)
}
defer config.Close()
azureManager, azureError = azure.CreateAzureManager(config)
} else {
glog.Info("Creating Azure Manager with default configuration.")
azureManager, azureError = azure.CreateAzureManager(nil)
}
if azureError != nil {
glog.Fatalf("Failed to create Azure Manager: %v", err)
}
cloudProvider, err = azure.BuildAzureCloudProvider(azureManager, nodeGroupsFlag, resourceLimiter)
if err != nil {
glog.Fatalf("Failed to create Azure cloud provider: %v", err)
}
}
if b.cloudProviderFlag == kubemark.ProviderName {
glog.V(1).Infof("Building kubemark cloud provider.")
externalConfig, err := rest.InClusterConfig()
if err != nil {
glog.Fatalf("Failed to get kubeclient config for external cluster: %v", err)
}
kubemarkConfig, err := clientcmd.BuildConfigFromFlags("", "/kubeconfig/cluster_autoscaler.kubeconfig")
if err != nil {
glog.Fatalf("Failed to get kubeclient config for kubemark cluster: %v", err)
}
stop := make(chan struct{})
externalClient := kubeclient.NewForConfigOrDie(externalConfig)
kubemarkClient := kubeclient.NewForConfigOrDie(kubemarkConfig)
externalInformerFactory := informers.NewSharedInformerFactory(externalClient, 0)
kubemarkInformerFactory := informers.NewSharedInformerFactory(kubemarkClient, 0)
kubemarkNodeInformer := kubemarkInformerFactory.Core().V1().Nodes()
go kubemarkNodeInformer.Informer().Run(stop)
kubemarkController, err := kubemarkcontroller.NewKubemarkController(externalClient, externalInformerFactory,
kubemarkClient, kubemarkNodeInformer)
if err != nil {
glog.Fatalf("Failed to create Kubemark cloud provider: %v", err)
}
externalInformerFactory.Start(stop)
if !kubemarkController.WaitForCacheSync(stop) {
glog.Fatalf("Failed to sync caches for kubemark controller")
}
go kubemarkController.Run(stop)
cloudProvider, err = kubemark.BuildKubemarkCloudProvider(kubemarkController, nodeGroupsFlag, resourceLimiter)
if err != nil {
glog.Fatalf("Failed to create Kubemark cloud provider: %v", err)
}
}
return cloudProvider
glog.Fatalf("Unknown cloud provider: %s", b.cloudProviderFlag)
return nil // This will never happen because the Fatalf will os.Exit
}
func (b CloudProviderBuilder) buildGCE(do cloudprovider.NodeGroupDiscoveryOptions, rl *cloudprovider.ResourceLimiter, mode gce.GcpCloudProviderMode) cloudprovider.CloudProvider {
var config io.ReadCloser
if b.cloudConfig != "" {
var err error
config, err = os.Open(b.cloudConfig)
if err != nil {
glog.Fatalf("Couldn't open cloud provider configuration %s: %#v", b.cloudConfig, err)
}
defer config.Close()
}
m, err := gce.CreateGceManager(config, mode, b.clusterName)
if err != nil {
glog.Fatalf("Failed to create GCE Manager: %v", err)
}
p, err := gce.BuildGceCloudProvider(m, do, rl)
if err != nil {
glog.Fatalf("Failed to create GCE cloud provider: %v", err)
}
return p
}
func (b CloudProviderBuilder) buildAWS(do cloudprovider.NodeGroupDiscoveryOptions, rl *cloudprovider.ResourceLimiter) cloudprovider.CloudProvider {
var config io.ReadCloser
if b.cloudConfig != "" {
var err error
config, err = os.Open(b.cloudConfig)
if err != nil {
glog.Fatalf("Couldn't open cloud provider configuration %s: %#v", b.cloudConfig, err)
}
defer config.Close()
}
m, err := aws.CreateAwsManager(config)
if err != nil {
glog.Fatalf("Failed to create AWS Manager: %v", err)
}
p, err := aws.BuildAwsCloudProvider(m, do, rl)
if err != nil {
glog.Fatalf("Failed to create AWS cloud provider: %v", err)
}
return p
}
func (b CloudProviderBuilder) buildAzure(do cloudprovider.NodeGroupDiscoveryOptions, rl *cloudprovider.ResourceLimiter) cloudprovider.CloudProvider {
var config io.ReadCloser
if b.cloudConfig != "" {
glog.Info("Creating Azure Manager using cloud-config file: %v", b.cloudConfig)
var err error
config, err := os.Open(b.cloudConfig)
if err != nil {
glog.Fatalf("Couldn't open cloud provider configuration %s: %#v", b.cloudConfig, err)
}
defer config.Close()
} else {
glog.Info("Creating Azure Manager with default configuration.")
}
m, err := azure.CreateAzureManager(config)
if err != nil {
glog.Fatalf("Failed to create Azure Manager: %v", err)
}
p, err := azure.BuildAzureCloudProvider(m, do.NodeGroupSpecs, rl)
if err != nil {
glog.Fatalf("Failed to create Azure cloud provider: %v", err)
}
return p
}
func (b CloudProviderBuilder) buildKubemark(do cloudprovider.NodeGroupDiscoveryOptions, rl *cloudprovider.ResourceLimiter) cloudprovider.CloudProvider {
externalConfig, err := rest.InClusterConfig()
if err != nil {
glog.Fatalf("Failed to get kubeclient config for external cluster: %v", err)
}
kubemarkConfig, err := clientcmd.BuildConfigFromFlags("", "/kubeconfig/cluster_autoscaler.kubeconfig")
if err != nil {
glog.Fatalf("Failed to get kubeclient config for kubemark cluster: %v", err)
}
stop := make(chan struct{})
externalClient := kubeclient.NewForConfigOrDie(externalConfig)
kubemarkClient := kubeclient.NewForConfigOrDie(kubemarkConfig)
externalInformerFactory := informers.NewSharedInformerFactory(externalClient, 0)
kubemarkInformerFactory := informers.NewSharedInformerFactory(kubemarkClient, 0)
kubemarkNodeInformer := kubemarkInformerFactory.Core().V1().Nodes()
go kubemarkNodeInformer.Informer().Run(stop)
kubemarkController, err := kubemarkcontroller.NewKubemarkController(externalClient, externalInformerFactory,
kubemarkClient, kubemarkNodeInformer)
if err != nil {
glog.Fatalf("Failed to create Kubemark cloud provider: %v", err)
}
externalInformerFactory.Start(stop)
if !kubemarkController.WaitForCacheSync(stop) {
glog.Fatalf("Failed to sync caches for kubemark controller")
}
go kubemarkController.Run(stop)
p, err := kubemark.BuildKubemarkCloudProvider(kubemarkController, do.NodeGroupSpecs, rl)
if err != nil {
glog.Fatalf("Failed to create Kubemark cloud provider: %v", err)
}
return p
}

11
cluster-autoscaler/cloudprovider/gce/gce_cloud_provider.go
View File

@ -32,6 +32,12 @@ import (
"k8s.io/kubernetes/plugin/pkg/scheduler/schedulercache"
)
// The 'GCE' cloud provider actually implements both the GCE and GKE providers.
const (
ProviderNameGCE = "gce"
ProviderNameGKE = "gke"
)
const (
maxAutoprovisionedSize = 1000
minAutoprovisionedSize = 0
@ -199,7 +205,10 @@ func (gce *GceCloudProvider) addNodeGroup(spec string) error {
// Name returns name of the cloud provider.
func (gce *GceCloudProvider) Name() string {
return "gce"
// Technically we're both ProviderNameGCE and ProviderNameGKE...
// Perhaps we should return a different name depending on
// gce.gceManager.getMode()?
return ProviderNameGCE
}
// NodeGroups returns all node groups configured for this cloud provider.

12
cluster-autoscaler/main.go
View File

@ -30,6 +30,7 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
kube_flag "k8s.io/apiserver/pkg/util/flag"
cloudBuilder "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/builder"
"k8s.io/autoscaler/cluster-autoscaler/config"
"k8s.io/autoscaler/cluster-autoscaler/config/dynamic"
"k8s.io/autoscaler/cluster-autoscaler/core"
@ -104,11 +105,12 @@ var (
"for scale down when some candidates from previous iteration are no longer valid."+
"When calculating the pool size for additional candidates we take"+
"max(#nodes * scale-down-candidates-pool-ratio, scale-down-candidates-pool-min-count).")
scanInterval = flag.Duration("scan-interval", 10*time.Second, "How often cluster is reevaluated for scale up or down")
maxNodesTotal = flag.Int("max-nodes-total", 0, "Maximum number of nodes in all node groups. Cluster autoscaler will not grow the cluster beyond this number.")
coresTotal = flag.String("cores-total", minMaxFlagString(0, config.DefaultMaxClusterCores), "Minimum and maximum number of cores in cluster, in the format <min>:<max>. Cluster autoscaler will not scale the cluster beyond these numbers.")
memoryTotal = flag.String("memory-total", minMaxFlagString(0, config.DefaultMaxClusterMemory), "Minimum and maximum number of gigabytes of memory in cluster, in the format <min>:<max>. Cluster autoscaler will not scale the cluster beyond these numbers.")
cloudProviderFlag = flag.String("cloud-provider", "gce", "Cloud provider type. Allowed values: gce, aws, azure, kubemark")
scanInterval = flag.Duration("scan-interval", 10*time.Second, "How often cluster is reevaluated for scale up or down")
maxNodesTotal = flag.Int("max-nodes-total", 0, "Maximum number of nodes in all node groups. Cluster autoscaler will not grow the cluster beyond this number.")
coresTotal = flag.String("cores-total", minMaxFlagString(0, config.DefaultMaxClusterCores), "Minimum and maximum number of cores in cluster, in the format <min>:<max>. Cluster autoscaler will not scale the cluster beyond these numbers.")
memoryTotal = flag.String("memory-total", minMaxFlagString(0, config.DefaultMaxClusterMemory), "Minimum and maximum number of gigabytes of memory in cluster, in the format <min>:<max>. Cluster autoscaler will not scale the cluster beyond these numbers.")
cloudProviderFlag = flag.String("cloud-provider", cloudBuilder.DefaultCloudProvider,
"Cloud provider type. Available values: ["+strings.Join(cloudBuilder.AvailableCloudProviders, ",")+"]")
maxEmptyBulkDeleteFlag = flag.Int("max-empty-bulk-delete", 10, "Maximum number of empty nodes that can be deleted at the same time.")
maxGracefulTerminationFlag = flag.Int("max-graceful-termination-sec", 10*60, "Maximum number of seconds CA waits for pod termination when trying to scale down a node.")
maxTotalUnreadyPercentage = flag.Float64("max-total-unready-percentage", 33, "Maximum percentage of unready nodes after which CA halts operations")