kubernetes
/
autoscaler
mirror of https://github.com/kubernetes/autoscaler.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #682 from feiskyer/azure-standard 

Make deployment parameters robust for various acs-engine versions
This commit is contained in:
Marcin Wielgus 2018-02-26 13:18:36 +01:00 committed by GitHub
parent 0ac080ffdd 91f6bfdd79
commit b81f70cadf
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 72 additions and 176 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
cluster-autoscaler/cloudprovider/azure/README.md
View File

@ -36,7 +36,7 @@ Fill the values of cluster-autoscaler-azure secret in [cluster-autoscaler-vmss.y
- SubscriptionID: `<base64-encode-subscription-id>`
- TenantID: `<base64-encoded-tenant-id>`
Note that all data should be encoded with base64.
> Note that all data above should be encoded with base64.
And fill the node groups in container command by `--nodes`, e.g.
@ -68,12 +68,8 @@ kubectl create -f cluster-autoscaler-vmss-master.yaml
Pre-requirements:
- Get credentials from above `permissions` step.
- Get the required parameters from acs-engine deployments (usually under directory `_output/<master-dns-prefix>` after running `acs-engine deploy` command)
- Get `APIServerPrivateKey`, `CAPrivateKey`, `ClientPrivateKey` and `KubeConfigPrivateKey` from `azuredeploy.parameters.json`
- Get `EtcdClientPrivateKey` and `EtcdServerPrivateKey` if the cluster is deployed by acs-engine >= v0.12.0
- If windows nodes are included, also get `WindowsAdminPassword` from acs-engine deployment manifests
- Get the initial Azure deployment name from azure portal. If you have multiple deployments (e.g. have run `acs-engine scale` command), make sure to get the first one
- Get a node pool name for nodes scaling from acs-engine deployment manifests
- Get the initial Azure deployment name from azure portal. If you have multiple deployments (e.g. have run `acs-engine scale` command), make sure to get the first one.
- Get a node pool name for nodes scaling from acs-engine deployment manifests
- Encode each data with base64.
Fill the values of cluster-autoscaler-azure secret in [cluster-autoscaler-standard-master.yaml](cluster-autoscaler-standard-master.yaml), including
@ -83,17 +79,9 @@ Fill the values of cluster-autoscaler-azure secret in [cluster-autoscaler-standa
- ResourceGroup: `<base64-encoded-resource-group>`
- SubscriptionID: `<base64-encode-subscription-id>`
- TenantID: `<base64-encoded-tenant-id>`
- NodeGroup: `<base64-encoded-node-pool-name>`
- Deployment: `<base64-encoded-azure-initial-deploy-name>`
- APIServerPrivateKey: `<base64-encoded-apiserver-private-key>`
- CAPrivateKey: `<base64-encoded-ca-private-key>`
- ClientPrivateKey: `<base64-encoded-client-private-key>`
- KubeConfigPrivateKey: `<base64-encoded-kubeconfig-private-key>`
- WindowsAdminPassword: `<base64-encoded-windows-admin-password>` (set `""` if no windows nodes in the cluster)
- EtcdClientPrivateKey: `<base64-encoded-etcd-client-private-key>` (set `""` for acs-engine < v0.12.0)
- EtcdServerPrivateKey: `<base64-encoded-etcd-server-private-key>` (set to `""` for acs-engine < v0.12.0)
Note that all data should be encoded with base64.
> Note that all data above should be encoded with base64.
And fill the node groups in container command by `--nodes`, e.g.
@ -108,6 +96,12 @@ or multiple node groups:
- --nodes=1:10:agentpool2
```
Create Azure deploy parameters secret `cluster-autoscaler-azure-deploy-parameters` by running
```sh
kubectl -n kube-system create secret generic cluster-autoscaler-azure-deploy-parameters --from-file=deploy-parameters=./_output/<your-output-path>/azuredeploy.parameters.json
```
Then deploy cluster-autoscaler by running
```sh

38
cluster-autoscaler/cloudprovider/azure/azure_agent_pool.go
View File

@ -71,53 +71,17 @@ func NewAgentPool(spec *dynamic.NodeGroupSpec, az *AzureManager) (*AgentPool, er
}
func (as *AgentPool) initialize() error {
deploy, err := as.manager.azClient.deploymentsClient.Get(as.manager.config.ResourceGroup, as.manager.config.Deployment)
if err != nil {
glog.Errorf("deploymentsClient.Get(%s, %s) failed: %v", as.manager.config.ResourceGroup, as.manager.config.Deployment, err)
return err
}
template, err := as.manager.azClient.deploymentsClient.ExportTemplate(as.manager.config.ResourceGroup, as.manager.config.Deployment)
if err != nil {
glog.Errorf("deploymentsClient.ExportTemplate(%s, %s) failed: %v", as.manager.config.ResourceGroup, as.manager.config.Deployment, err)
return err
}
as.parameters = *deploy.Properties.Parameters
as.preprocessParameters()
as.template = *template.Template
as.parameters = as.manager.config.DeploymentParameters
return normalizeForK8sVMASScalingUp(as.template)
}
func (as *AgentPool) preprocessParameters() {
// Delete type key from parameters.
for k := range as.parameters {
if v, ok := as.parameters[k].(map[string]interface{}); ok {
delete(v, "type")
}
}
// fulfill secure parameters.
as.parameters["apiServerPrivateKey"] = map[string]string{"value": as.manager.config.APIServerPrivateKey}
as.parameters["caPrivateKey"] = map[string]string{"value": as.manager.config.CAPrivateKey}
as.parameters["clientPrivateKey"] = map[string]string{"value": as.manager.config.ClientPrivateKey}
as.parameters["kubeConfigPrivateKey"] = map[string]string{"value": as.manager.config.KubeConfigPrivateKey}
as.parameters["servicePrincipalClientId"] = map[string]string{"value": as.manager.config.AADClientID}
as.parameters["servicePrincipalClientSecret"] = map[string]string{"value": as.manager.config.AADClientSecret}
if as.manager.config.WindowsAdminPassword != "" {
as.parameters["windowsAdminPassword"] = map[string]string{"value": as.manager.config.WindowsAdminPassword}
}
// etcd TLS parameters (for acs-engine >= v0.12.0).
if as.manager.config.EtcdClientPrivateKey != "" {
as.parameters["etcdClientPrivateKey"] = map[string]string{"value": as.manager.config.EtcdClientPrivateKey}
}
if as.manager.config.EtcdServerPrivateKey != "" {
as.parameters["etcdServerPrivateKey"] = map[string]string{"value": as.manager.config.EtcdServerPrivateKey}
}
}
// MinSize returns minimum size of the node group.
func (as *AgentPool) MinSize() int {
return as.minSize

39
cluster-autoscaler/cloudprovider/azure/azure_manager.go
View File

@ -38,6 +38,9 @@ const (
scaleToZeroSupported = false
refreshInterval = 1 * time.Minute
// The path of deployment parameters for standard vm.
deploymentParametersPath = "/var/lib/azure/azuredeploy.parameters.json"
)
// AzureManager handles Azure communication and data caching.
@ -67,15 +70,8 @@ type Config struct {
UseManagedIdentityExtension bool `json:"useManagedIdentityExtension" yaml:"useManagedIdentityExtension"`
// Configs only for standard vmType (agent pools).
Deployment string `json:"deployment" yaml:"deployment"`
APIServerPrivateKey string `json:"apiServerPrivateKey" yaml:"apiServerPrivateKey"`
CAPrivateKey string `json:"caPrivateKey" yaml:"caPrivateKey"`
ClientPrivateKey string `json:"clientPrivateKey" yaml:"clientPrivateKey"`
KubeConfigPrivateKey string `json:"kubeConfigPrivateKey" yaml:"kubeConfigPrivateKey"`
WindowsAdminPassword string `json:"windowsAdminPassword" yaml:"windowsAdminPassword"`
// etcd TLS parameters (for acs-engine >= v0.12.0).
EtcdClientPrivateKey string `json:"etcdClientPrivateKey" yaml:"etcdClientPrivateKey"`
EtcdServerPrivateKey string `json:"etcdServerPrivateKey" yaml:"etcdServerPrivateKey"`
Deployment string `json:"deployment" yaml:"deployment"`
DeploymentParameters map[string]interface{} `json:"deploymentParameters" yaml:"deploymentParameters"`
}
// TrimSpace removes all leading and trailing white spaces.
@ -90,13 +86,6 @@ func (c *Config) TrimSpace() {
c.AADClientCertPath = strings.TrimSpace(c.AADClientCertPath)
c.AADClientCertPassword = strings.TrimSpace(c.AADClientCertPassword)
c.Deployment = strings.TrimSpace(c.Deployment)
c.APIServerPrivateKey = strings.TrimSpace(c.APIServerPrivateKey)
c.CAPrivateKey = strings.TrimSpace(c.CAPrivateKey)
c.ClientPrivateKey = strings.TrimSpace(c.ClientPrivateKey)
c.KubeConfigPrivateKey = strings.TrimSpace(c.KubeConfigPrivateKey)
c.WindowsAdminPassword = strings.TrimSpace(c.WindowsAdminPassword)
c.EtcdClientPrivateKey = strings.TrimSpace(c.EtcdClientPrivateKey)
c.EtcdServerPrivateKey = strings.TrimSpace(c.EtcdServerPrivateKey)
}
// CreateAzureManager creates Azure Manager object to work with Azure.
@ -120,13 +109,6 @@ func CreateAzureManager(configReader io.Reader, discoveryOpts cloudprovider.Node
cfg.AADClientCertPath = os.Getenv("ARM_CLIENT_CERT_PATH")
cfg.AADClientCertPassword = os.Getenv("ARM_CLIENT_CERT_PASSWORD")
cfg.Deployment = os.Getenv("ARM_DEPLOYMENT")
cfg.APIServerPrivateKey = os.Getenv("ARM_APISEVER_PRIVATE_KEY")
cfg.CAPrivateKey = os.Getenv("ARM_CA_PRIVATE_KEY")
cfg.ClientPrivateKey = os.Getenv("ARM_CLIENT_PRIVATE_KEY")
cfg.KubeConfigPrivateKey = os.Getenv("ARM_KUBECONFIG_PRIVATE_KEY")
cfg.WindowsAdminPassword = os.Getenv("ARM_WINDOWS_ADMIN_PASSWORD")
cfg.EtcdClientPrivateKey = os.Getenv("ARM_ETCD_CLIENT_RPIVATE_KEY")
cfg.EtcdServerPrivateKey = os.Getenv("ARM_ETCD_SERVER_PRIVATE_KEY")
useManagedIdentityExtensionFromEnv := os.Getenv("ARM_USE_MANAGED_IDENTITY_EXTENSION")
if len(useManagedIdentityExtensionFromEnv) > 0 {
@ -143,6 +125,17 @@ func CreateAzureManager(configReader io.Reader, discoveryOpts cloudprovider.Node
cfg.VMType = vmTypeVMSS
}
// Read parameters from deploymentParametersPath if it is not set.
if cfg.VMType == vmTypeStandard && len(cfg.DeploymentParameters) == 0 {
parameters, err := readDeploymentParameters(deploymentParametersPath)
if err != nil {
glog.Errorf("readDeploymentParameters failed with error: %v", err)
return nil, err
}
cfg.DeploymentParameters = parameters
}
// Defaulting env to Azure Public Cloud.
env := azure.PublicCloud
if cfg.Cloud != "" {

39
cluster-autoscaler/cloudprovider/azure/azure_util.go
View File

@ -19,7 +19,9 @@ package azure
import (
"crypto/rsa"
"crypto/x509"
"encoding/json"
"fmt"
"io/ioutil"
"net/url"
"regexp"
"sort"
@ -398,20 +400,8 @@ func validateConfig(cfg *Config) error {
return fmt.Errorf("deployment not set")
}
if cfg.APIServerPrivateKey == "" {
return fmt.Errorf("apiServerPrivateKey not set")
}
if cfg.CAPrivateKey == "" {
return fmt.Errorf("caPrivateKey not set")
}
if cfg.ClientPrivateKey == "" {
return fmt.Errorf("clientPrivateKey not set")
}
if cfg.KubeConfigPrivateKey == "" {
return fmt.Errorf("kubeConfigPrivateKey not set")
if len(cfg.DeploymentParameters) == 0 {
return fmt.Errorf("deploymentParameters not set")
}
}
@ -428,3 +418,24 @@ func getLastSegment(ID string) (string, error) {
return name, nil
}
// readDeploymentParameters gets deployment parameters from paramFilePath.
func readDeploymentParameters(paramFilePath string) (map[string]interface{}, error) {
contents, err := ioutil.ReadFile(paramFilePath)
if err != nil {
glog.Errorf("Failed to read deployment parameters from file %q: %v", paramFilePath, err)
return nil, err
}
deploymentParameters := make(map[string]interface{})
if err := json.Unmarshal(contents, &deploymentParameters); err != nil {
glog.Errorf("Failed to unmarshal deployment parameters from file %q: %v", paramFilePath, err)
return nil, err
}
if v, ok := deploymentParameters["parameters"]; ok {
return v.(map[string]interface{}), nil
}
return nil, fmt.Errorf("failed to get deployment parameters from file %s", paramFilePath)
}

53
cluster-autoscaler/cloudprovider/azure/cluster-autoscaler-standard-master.yaml
View File

@ -110,13 +110,6 @@ data:
SubscriptionID: <base64-encode-subscription-id>
TenantID: <base64-encoded-tenant-id>
Deployment: <base64-encoded-azure-initial-deploy-name>
APIServerPrivateKey: <base64-encoded-apiserver-private-key>
CAPrivateKey: <base64-encoded-ca-private-key>
ClientPrivateKey: <base64-encoded-client-private-key>
KubeConfigPrivateKey: <base64-encoded-kubeconfig-private-key>
WindowsAdminPassword: <base64-encoded-windows-admin-password>
EtcdClientPrivateKey: <base64-encoded-etcd-client-private-key>
EtcdServerPrivateKey: <base64-encoded-etcd-server-private-key>
VMType: c3RhbmRhcmQ=
kind: Secret
metadata:
@ -193,41 +186,6 @@ spec:
secretKeyRef:
key: Deployment
name: cluster-autoscaler-azure
- name: ARM_APISEVER_PRIVATE_KEY
valueFrom:
secretKeyRef:
key: APIServerPrivateKey
name: cluster-autoscaler-azure
- name: ARM_CA_PRIVATE_KEY
valueFrom:
secretKeyRef:
key: CAPrivateKey
name: cluster-autoscaler-azure
- name: ARM_CLIENT_PRIVATE_KEY
valueFrom:
secretKeyRef:
key: ClientPrivateKey
name: cluster-autoscaler-azure
- name: ARM_KUBECONFIG_PRIVATE_KEY
valueFrom:
secretKeyRef:
key: KubeConfigPrivateKey
name: cluster-autoscaler-azure
- name: ARM_WINDOWS_ADMIN_PASSWORD
valueFrom:
secretKeyRef:
key: WindowsAdminPassword
name: cluster-autoscaler-azure
- name: ARM_ETCD_CLIENT_RPIVATE_KEY
valueFrom:
secretKeyRef:
key: EtcdClientPrivateKey
name: cluster-autoscaler-azure
- name: ARM_ETCD_SERVER_PRIVATE_KEY
valueFrom:
secretKeyRef:
key: EtcdServerPrivateKey
name: cluster-autoscaler-azure
image: gcr.io/google_containers/cluster-autoscaler:{{ ca_version }}
imagePullPolicy: Always
name: cluster-autoscaler
@ -242,10 +200,19 @@ spec:
- mountPath: /etc/ssl/certs/ca-certificates.crt
name: ssl-certs
readOnly: true
- mountPath: /var/lib/azure/
name: deploy-parameters
readOnly: true
dnsPolicy: ClusterFirst
restartPolicy: Always
volumes:
- hostPath:
path: /etc/ssl/certs/ca-certificates.crt
type: ""
name: ssl-certs
name: ssl-certs
- name: deploy-parameters
secret:
secretName: cluster-autoscaler-azure-deploy-parameters
items:
- key: deploy-parameters
path: azuredeploy.parameters.json

53
cluster-autoscaler/cloudprovider/azure/cluster-autoscaler-standard.yaml
View File

@ -110,13 +110,6 @@ data:
SubscriptionID: <base64-encode-subscription-id>
TenantID: <base64-encoded-tenant-id>
Deployment: <base64-encoded-azure-initial-deploy-name>
APIServerPrivateKey: <base64-encoded-apiserver-private-key>
CAPrivateKey: <base64-encoded-ca-private-key>
ClientPrivateKey: <base64-encoded-client-private-key>
KubeConfigPrivateKey: <base64-encoded-kubeconfig-private-key>
WindowsAdminPassword: <base64-encoded-windows-admin-password>
EtcdClientPrivateKey: <base64-encoded-etcd-client-private-key>
EtcdServerPrivateKey: <base64-encoded-etcd-server-private-key>
VMType: c3RhbmRhcmQ=
kind: Secret
metadata:
@ -186,41 +179,6 @@ spec:
secretKeyRef:
key: Deployment
name: cluster-autoscaler-azure
- name: ARM_APISEVER_PRIVATE_KEY
valueFrom:
secretKeyRef:
key: APIServerPrivateKey
name: cluster-autoscaler-azure
- name: ARM_CA_PRIVATE_KEY
valueFrom:
secretKeyRef:
key: CAPrivateKey
name: cluster-autoscaler-azure
- name: ARM_CLIENT_PRIVATE_KEY
valueFrom:
secretKeyRef:
key: ClientPrivateKey
name: cluster-autoscaler-azure
- name: ARM_KUBECONFIG_PRIVATE_KEY
valueFrom:
secretKeyRef:
key: KubeConfigPrivateKey
name: cluster-autoscaler-azure
- name: ARM_WINDOWS_ADMIN_PASSWORD
valueFrom:
secretKeyRef:
key: WindowsAdminPassword
name: cluster-autoscaler-azure
- name: ARM_ETCD_CLIENT_RPIVATE_KEY
valueFrom:
secretKeyRef:
key: EtcdClientPrivateKey
name: cluster-autoscaler-azure
- name: ARM_ETCD_SERVER_PRIVATE_KEY
valueFrom:
secretKeyRef:
key: EtcdServerPrivateKey
name: cluster-autoscaler-azure
image: gcr.io/google_containers/cluster-autoscaler:{{ ca_version }}
imagePullPolicy: Always
name: cluster-autoscaler
@ -235,10 +193,19 @@ spec:
- mountPath: /etc/ssl/certs/ca-certificates.crt
name: ssl-certs
readOnly: true
- mountPath: /var/lib/azure/
name: deploy-parameters
readOnly: true
dnsPolicy: ClusterFirst
restartPolicy: Always
volumes:
- hostPath:
path: /etc/ssl/certs/ca-certificates.crt
type: ""
name: ssl-certs
name: ssl-certs
- name: deploy-parameters
secret:
secretName: cluster-autoscaler-azure-deploy-parameters
items:
- key: deploy-parameters
path: azuredeploy.parameters.json