kubernetes
/
kops
mirror of https://github.com/kubernetes/kops.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

upup: improved upgrade procedure

This commit is contained in:
Justin Santa Barbara 2016-06-28 11:49:17 -04:00
parent e315564cfa
commit e3062a9f51
10 changed files with 522 additions and 120 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
upup/cmd/upup/import.go Normal file
View File

@ -0,0 +1,16 @@
package main
import (
"github.com/spf13/cobra"
)
// importCmd represents the import command
var importCmd = &cobra.Command{
Use: "import",
Short: "import clusters",
Long: `import clusters`,
}
func init() {
rootCommand.AddCommand(importCmd)
}

22
upup/cmd/upup/export_cluster.go → upup/cmd/upup/import_cluster.go
View File

@ -9,31 +9,31 @@ import (
"k8s.io/kube-deploy/upup/pkg/kutil"
)
type ExportClusterCmd struct {
type ImportClusterCmd struct {
Region string
}
var exportCluster ExportClusterCmd
var importCluster ImportClusterCmd
func init() {
cmd := &cobra.Command{
Use: "cluster",
Short: "Export cluster",
Long: `Exports a k8s cluster.`,
Short: "Import existing cluster into the state store",
Long: `Imports the settings of an existing k8s cluster.`,
Run: func(cmd *cobra.Command, args []string) {
err := exportCluster.Run()
err := importCluster.Run()
if err != nil {
glog.Exitf("%v", err)
}
},
}
exportCmd.AddCommand(cmd)
importCmd.AddCommand(cmd)
cmd.Flags().StringVar(&exportCluster.Region, "region", "", "region")
cmd.Flags().StringVar(&importCluster.Region, "region", "", "region")
}
func (c *ExportClusterCmd) Run() error {
func (c *ImportClusterCmd) Run() error {
if c.Region == "" {
return fmt.Errorf("--region is required")
}
@ -53,15 +53,17 @@ func (c *ExportClusterCmd) Run() error {
return fmt.Errorf("error state store: %v", err)
}
d := &kutil.ExportCluster{}
d := &kutil.ImportCluster{}
d.ClusterName = clusterName
d.Cloud = cloud
d.StateStore = stateStore
err = d.ReverseAWS()
err = d.ImportAWSCluster()
if err != nil {
return err
}
fmt.Printf("\nImported settings for cluster %q\n", clusterName)
return nil
}

14
upup/cmd/upup/upgrade_cluster.go
View File

@ -39,12 +39,17 @@ func (c *UpgradeClusterCmd) Run() error {
return fmt.Errorf("--newname is required")
}
stateStore, err := rootCommand.StateStore()
oldStateStore, err := rootCommand.StateStore()
if err != nil {
return fmt.Errorf("error state store: %v", err)
return err
}
cluster, instanceGroups, err := api.ReadConfig(stateStore)
newStateStore, err := rootCommand.StateStoreForCluster(c.NewClusterName)
if err != nil {
return err
}
cluster, instanceGroups, err := api.ReadConfig(oldStateStore)
if err != nil {
return fmt.Errorf("error reading configuration: %v", err)
}
@ -84,7 +89,8 @@ func (c *UpgradeClusterCmd) Run() error {
d.Cloud = cloud
d.ClusterConfig = cluster
d.InstanceGroups = instanceGroups
d.StateStore = stateStore
d.OldStateStore = oldStateStore
d.NewStateStore = newStateStore
err = d.Upgrade()
if err != nil {

125
upup/docs/upgrade_from_k8s_12.md Normal file
View File

@ -0,0 +1,125 @@
# Upgrading from k8s 1.2
** This is an experimental and slightly risky procedure, so we recommend backing up important data before proceeding.
Take a snapshot of your EBS volumes; export all your data from kubectl etc. **
Limitations:
* kops splits etcd onto two volumes now: `main` and `events`. We will keep the `main` data, but
you will lose your events history.
* Doubtless others not yet known - please open issues if you encounter them!
## Overview
There are a few steps:
* First you import the existing cluster state, so you can see and edit the configuration
* You verify the cluster configuration
* You move existing AWS resources to your new cluster
* You bring up the new cluster
* You probably need to do a little manual cleanup (for example of ELBs)
* You can then delete the old cluster
## Importing the existing cluster
The `import cluster` command reverse engineers an existing cluster, and creates a cluster
configuration.
Make sure you have set `export KOPS_STATE_STORE=s3://<mybucket>`
Then import the cluster; setting `--name` and `--region` to match the old cluster. If you're not sure
of the old cluster name, you can find it by looking at the `KubernetesCluster` tag on your AWS resources.
```
export OLD_NAME=kubernetes
export REGION=us-west-2
upup import cluster --region ${REGION} --name ${OLD_NAME}
```
## Verify the cluster configuration
Now have a look at the cluster configuration, to make sure it looks right. If it doesn't, please
open an issue.
```
upup edit cluster --name ${OLD_NAME}
````
## Move resources to a new cluster
The upgrade moves some resources so they will be adopted by the new cluster. There are a number of things
this step does:
* It resizes existing autoscaling groups to size 0
* It will stop the existing master
* It detaches the master EBS volume from the master
* It re-tags resources to associate them with the new cluster: volumes, ELBs
* It re-tags the VPC to associate it with the new cluster
The upgrade procedure forces you to choose a new cluster name (e.g. `k8s.mydomain.com`)
```
export NEW_NAME=k8s.mydomain.com
upup upgrade cluster --newname ${NEW_NAME} --name ${OLD_NAME}
```
If you now list the clusters, you should see both the old cluster & the new cluster
```upup get clusters```
## Bring up the new cluster
Use the normal tool to bring up the new cluster:
```
cloudup --name ${NEW_NAME} --dryrun
```
Things to check are that it is reusing the existing volume for the _main_ etcd cluster (but not the events clusters).
And then when you are happy:
```
cloudup --name ${NEW_NAME}
```
## Export kubecfg settings to access the new cluster
```
upup export kubecfg --name ${NEW_NAME}
```
Within a few minutes the new cluster should be running. Try `kubectl get nodes --show-labels`, `kubectl get pods` etc until you are sure that all is well.
## Workaround to re-enable ELBs
Due to a limitation in ELBs (you can't replace all the subnets), if you have ELBs you must do the following:
* `upup edit cluster --name ${NEW_NAME}`
* Add a zone to the `zones` section and save the file (it normally suffices to just add `- name: us-west-2b` or whatever
zone you are adding; upup will auto-populate the CIDR.
* cloudup --name ${NEW_NAME}
In the AWS control panel open the "Load Balancers" section, and for each ELB:
* On the "Actions" menu click "Edit subnets"
* Add the newly created zone's subnet, then save
* On the "Actions" menu click "Edit subnets" (again)
* Add the other zone's subnet (which will replace the old subnet with the new subnet), and Save
You should now have an ELB in your new zones; within a few minutes k8s should reconcile it and attach the new instances.
## Delete remaining resources of the old cluster
```
upup delete cluster --name ${OLD_NAME}
```
And once you've confirmed it looks right, run with `--yes`
You will also need to release the old ElasticIP manually.
Note that there is an issue in EC2/ELB: it seems that the NetworkInterfaces for the ELB aren't immediately deleted,
and this prevents full teardown of the old resources (the subnet in particular). A workaround is to delete
the "Network Interfaces" for the old ELB subnet in the AWS console.

93
upup/pkg/fi/cloudup/create_cluster.go
View File

@ -316,10 +316,6 @@ func (c *CreateClusterCmd) Run() error {
c.NodeUpSource = location
}
var cloud fi.Cloud
var project string
checkExisting := true
//c.NodeUpConfig.Tags = append(c.NodeUpConfig.Tags, "_jessie", "_debian_family", "_systemd")
@ -355,11 +351,21 @@ func (c *CreateClusterCmd) Run() error {
"secret": &fitasks.Secret{},
})
cloud, err := BuildCloud(c.Cluster)
if err != nil {
return err
}
region := ""
project := ""
switch c.Cluster.Spec.CloudProvider {
case "gce":
{
gceCloud := cloud.(*gce.GCECloud)
region = gceCloud.Region
project = gceCloud.Project
glog.Fatalf("GCE is (probably) not working currently - please ping @justinsb for cleanup")
tags["_gce"] = struct{}{}
c.NodeUpTags = append(c.NodeUpTags, "_gce")
@ -373,47 +379,6 @@ func (c *CreateClusterCmd) Run() error {
"firewallRule": &gcetasks.FirewallRule{},
"ipAddress": &gcetasks.IPAddress{},
})
nodeZones := make(map[string]bool)
for _, zone := range c.Cluster.Spec.Zones {
nodeZones[zone.Name] = true
tokens := strings.Split(zone.Name, "-")
if len(tokens) <= 2 {
return fmt.Errorf("Invalid GCE Zone: %v", zone.Name)
}
zoneRegion := tokens[0] + "-" + tokens[1]
if region != "" && zoneRegion != region {
return fmt.Errorf("Clusters cannot span multiple regions")
}
region = zoneRegion
}
//err := awsup.ValidateRegion(region)
//if err != nil {
// return err
//}
project = c.Cluster.Spec.Project
if project == "" {
return fmt.Errorf("project is required for GCE")
}
gceCloud, err := gce.NewGCECloud(region, project)
if err != nil {
return err
}
//var zoneNames []string
//for _, z := range c.Config.Zones {
// zoneNames = append(zoneNames, z.Name)
//}
//err = gceCloud.ValidateZones(zoneNames)
//if err != nil {
// return err
//}
cloud = gceCloud
}
case "aws":
@ -462,48 +427,10 @@ func (c *CreateClusterCmd) Run() error {
"dnsZone": &awstasks.DNSZone{},
})
nodeZones := make(map[string]bool)
for _, zone := range c.Cluster.Spec.Zones {
if len(zone.Name) <= 2 {
return fmt.Errorf("Invalid AWS zone: %q", zone.Name)
}
nodeZones[zone.Name] = true
zoneRegion := zone.Name[:len(zone.Name)-1]
if region != "" && zoneRegion != region {
return fmt.Errorf("Clusters cannot span multiple regions")
}
region = zoneRegion
}
err := awsup.ValidateRegion(region)
if err != nil {
return err
}
if c.SSHPublicKey == "" {
return fmt.Errorf("SSH public key must be specified when running with AWS")
}
cloudTags := map[string]string{awsup.TagClusterName: c.Cluster.Name}
awsCloud, err := awsup.NewAWSCloud(region, cloudTags)
if err != nil {
return err
}
var zoneNames []string
for _, z := range c.Cluster.Spec.Zones {
zoneNames = append(zoneNames, z.Name)
}
err = awsCloud.ValidateZones(zoneNames)
if err != nil {
return err
}
cloud = awsCloud
l.TemplateFunctions["MachineTypeInfo"] = awsup.GetMachineTypeInfo
}

96
upup/pkg/fi/cloudup/utils.go Normal file
View File

@ -0,0 +1,96 @@
package cloudup
import (
"fmt"
"k8s.io/kube-deploy/upup/pkg/api"
"k8s.io/kube-deploy/upup/pkg/fi"
"k8s.io/kube-deploy/upup/pkg/fi/cloudup/awsup"
"k8s.io/kube-deploy/upup/pkg/fi/cloudup/gce"
"strings"
)
func BuildCloud(cluster *api.Cluster) (fi.Cloud, error) {
var cloud fi.Cloud
region := ""
project := ""
switch cluster.Spec.CloudProvider {
case "gce":
{
nodeZones := make(map[string]bool)
for _, zone := range cluster.Spec.Zones {
nodeZones[zone.Name] = true
tokens := strings.Split(zone.Name, "-")
if len(tokens) <= 2 {
return nil, fmt.Errorf("Invalid GCE Zone: %v", zone.Name)
}
zoneRegion := tokens[0] + "-" + tokens[1]
if region != "" && zoneRegion != region {
return nil, fmt.Errorf("Clusters cannot span multiple regions")
}
region = zoneRegion
}
project = cluster.Spec.Project
if project == "" {
return nil, fmt.Errorf("project is required for GCE")
}
gceCloud, err := gce.NewGCECloud(region, project)
if err != nil {
return nil, err
}
cloud = gceCloud
}
case "aws":
{
nodeZones := make(map[string]bool)
for _, zone := range cluster.Spec.Zones {
if len(zone.Name) <= 2 {
return nil, fmt.Errorf("Invalid AWS zone: %q", zone.Name)
}
nodeZones[zone.Name] = true
zoneRegion := zone.Name[:len(zone.Name)-1]
if region != "" && zoneRegion != region {
return nil, fmt.Errorf("Clusters cannot span multiple regions")
}
region = zoneRegion
}
err := awsup.ValidateRegion(region)
if err != nil {
return nil, err
}
cloudTags := map[string]string{awsup.TagClusterName: cluster.Name}
awsCloud, err := awsup.NewAWSCloud(region, cloudTags)
if err != nil {
return nil, err
}
var zoneNames []string
for _, z := range cluster.Spec.Zones {
zoneNames = append(zoneNames, z.Name)
}
err = awsCloud.ValidateZones(zoneNames)
if err != nil {
return nil, err
}
cloud = awsCloud
}
default:
return nil, fmt.Errorf("unknown CloudProvider %q", cluster.Spec.CloudProvider)
}
return cloud, nil
}

2
upup/pkg/fi/vfs/vfs.go
View File

@ -52,7 +52,7 @@ func RelativePath(base Path, child Path) (string, error) {
basePath += "/"
}
if !strings.HasPrefix(basePath, childPath) {
if !strings.HasPrefix(childPath, basePath) {
return "", fmt.Errorf("Path %q is not a child of %q", child, base)
}

92
upup/pkg/kutil/export_cluster.go → upup/pkg/kutil/import_cluster.go
View File

@ -14,14 +14,14 @@ import (
)
// ExportCluster tries to reverse engineer an existing k8s cluster
type ExportCluster struct {
type ImportCluster struct {
ClusterName string
Cloud fi.Cloud
StateStore fi.StateStore
}
func (x *ExportCluster) ReverseAWS() error {
func (x *ImportCluster) ImportAWSCluster() error {
awsCloud := x.Cloud.(*awsup.AWSCloud)
clusterName := x.ClusterName
@ -35,6 +35,8 @@ func (x *ExportCluster) ReverseAWS() error {
cluster.Spec.CloudProvider = "aws"
cluster.Name = clusterName
cluster.Spec.KubeControllerManager = &api.KubeControllerManagerConfig{}
masterGroup := &api.InstanceGroup{}
masterGroup.Spec.Role = api.InstanceGroupRoleMaster
masterGroup.Name = "masters"
@ -52,7 +54,19 @@ func (x *ExportCluster) ReverseAWS() error {
role, _ := awsup.FindEC2Tag(instance.Tags, "Role")
if role == clusterName+"-master" {
if masterInstance != nil {
return fmt.Errorf("found multiple masters")
masterState := aws.StringValue(masterInstance.State.Name)
thisState := aws.StringValue(instance.State.Name)
glog.Infof("Found multiple masters: %s and %s", masterState, thisState)
if masterState == "terminated" && thisState != "terminated" {
// OK
} else if thisState == "terminated" && masterState != "terminated" {
// Ignore this one
continue
} else {
return fmt.Errorf("found multiple masters")
}
}
masterInstance = instance
}
@ -85,7 +99,19 @@ func (x *ExportCluster) ReverseAWS() error {
}
vpcID := aws.StringValue(masterInstance.VpcId)
var vpc *ec2.Vpc
{
vpc, err = awsCloud.DescribeVPC(vpcID)
if err != nil {
return err
}
if vpc == nil {
return fmt.Errorf("cannot find vpc %q", vpcID)
}
}
cluster.Spec.NetworkID = vpcID
cluster.Spec.NetworkCIDR = aws.StringValue(vpc.CidrBlock)
az := aws.StringValue(masterSubnet.AvailabilityZone)
masterGroup.Spec.Zones = []string{az}
@ -145,6 +171,7 @@ func (x *ExportCluster) ReverseAWS() error {
cluster.Spec.KubeControllerManager.AllocateNodeCIDRs = conf.ParseBool("ALLOCATE_NODE_CIDRS")
//clusterConfig.KubeUser = conf.Settings["KUBE_USER"]
cluster.Spec.ServiceClusterIPRange = conf.Settings["SERVICE_CLUSTER_IP_RANGE"]
cluster.Spec.NonMasqueradeCIDR = conf.Settings["NON_MASQUERADE_CIDR"]
//clusterConfig.EnableClusterMonitoring = conf.Settings["ENABLE_CLUSTER_MONITORING"]
//clusterConfig.EnableClusterLogging = conf.ParseBool("ENABLE_CLUSTER_LOGGING")
//clusterConfig.EnableNodeLogging = conf.ParseBool("ENABLE_NODE_LOGGING")
@ -170,14 +197,40 @@ func (x *ExportCluster) ReverseAWS() error {
primaryNodeSet := &api.InstanceGroup{}
primaryNodeSet.Spec.Role = api.InstanceGroupRoleNode
primaryNodeSet.Name = "nodes"
instanceGroups = append(instanceGroups, primaryNodeSet)
primaryNodeSet.Spec.MinSize, err = conf.ParseInt("NUM_MINIONS")
if err != nil {
return fmt.Errorf("cannot parse NUM_MINIONS=%q: %v", conf.Settings["NUM_MINIONS"], err)
//primaryNodeSet.Spec.MinSize, err = conf.ParseInt("NUM_MINIONS")
//if err != nil {
// return fmt.Errorf("cannot parse NUM_MINIONS=%q: %v", conf.Settings["NUM_MINIONS"], err)
//}
{
groups, err := findAutoscalingGroups(awsCloud, awsCloud.Tags())
if err != nil {
return fmt.Errorf("error listing autoscaling groups: %v", err)
}
if len(groups) == 0 {
glog.Warningf("No Autoscaling group found")
}
if len(groups) == 1 {
glog.Warningf("Multiple Autoscaling groups found")
}
minSize := 0
maxSize := 0
for _, group := range groups {
minSize += int(aws.Int64Value(group.MinSize))
maxSize += int(aws.Int64Value(group.MaxSize))
}
if minSize != 0 {
primaryNodeSet.Spec.MinSize = fi.Int(minSize)
}
if maxSize != 0 {
primaryNodeSet.Spec.MaxSize = fi.Int(maxSize)
}
// TODO: machine types
//primaryNodeSet.NodeMachineType = k8s.MasterMachineType
}
primaryNodeSet.Spec.MaxSize = primaryNodeSet.Spec.MinSize
//primaryNodeSet.NodeMachineType = k8s.MasterMachineType
if conf.Version == "1.1" {
// If users went with defaults on some things, clear them out so they get the new defaults
@ -202,6 +255,19 @@ func (x *ExportCluster) ReverseAWS() error {
//}
}
for _, etcdClusterName := range []string{"main", "events"} {
etcdCluster := &api.EtcdClusterSpec{
Name: etcdClusterName,
}
for _, az := range masterGroup.Spec.Zones {
etcdCluster.Members = append(etcdCluster.Members, &api.EtcdMemberSpec{
Name: az,
Zone: az,
})
}
cluster.Spec.EtcdClusters = append(cluster.Spec.EtcdClusters, etcdCluster)
}
//if masterInstance.PublicIpAddress != nil {
// eip, err := findElasticIP(cloud, *masterInstance.PublicIpAddress)
// if err != nil {
@ -238,14 +304,14 @@ func (x *ExportCluster) ReverseAWS() error {
//b.Context = "aws_" + instancePrefix
keyStore := x.StateStore.CA()
newKeyStore := x.StateStore.CA()
//caCert, err := masterSSH.Join("ca.crt").ReadFile()
caCert, err := conf.ParseCert("CA_CERT")
if err != nil {
return err
}
err = keyStore.AddCert(fi.CertificateId_CA, caCert)
err = newKeyStore.AddCert(fi.CertificateId_CA, caCert)
if err != nil {
return err
}
@ -294,7 +360,7 @@ func (x *ExportCluster) ReverseAWS() error {
// return err
//}
// We will generate new tokens...
//// We will generate new tokens, but some of these are in existing API objects
//secretStore := x.StateStore.Secrets()
//kubePassword := conf.Settings["KUBE_PASSWORD"]
//kubeletToken = conf.Settings["KUBELET_TOKEN"]
@ -424,7 +490,7 @@ func parseInt(s string) (int, error) {
//}
func findInstances(c *awsup.AWSCloud) ([]*ec2.Instance, error) {
filters := c.BuildFilters(nil)
filters := buildEC2Filters(c)
request := &ec2.DescribeInstancesInput{
Filters: filters,

180
upup/pkg/kutil/upgrade_cluster.go
View File

@ -3,11 +3,13 @@ package kutil
import (
"fmt"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/autoscaling"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/golang/glog"
"k8s.io/kube-deploy/upup/pkg/api"
"k8s.io/kube-deploy/upup/pkg/fi"
"k8s.io/kube-deploy/upup/pkg/fi/cloudup/awsup"
"time"
)
// UpgradeCluster performs an upgrade of a k8s cluster
@ -16,7 +18,8 @@ type UpgradeCluster struct {
NewClusterName string
Cloud fi.Cloud
StateStore fi.StateStore
OldStateStore fi.StateStore
NewStateStore fi.StateStore
ClusterConfig *api.Cluster
InstanceGroups []*api.InstanceGroup
@ -36,6 +39,8 @@ func (x *UpgradeCluster) Upgrade() error {
return fmt.Errorf("OldClusterName must be specified")
}
oldTags := awsCloud.Tags()
newTags := awsCloud.Tags()
newTags["KubernetesCluster"] = newClusterName
@ -55,6 +60,16 @@ func (x *UpgradeCluster) Upgrade() error {
return err
}
autoscalingGroups, err := findAutoscalingGroups(awsCloud, oldTags)
if err != nil {
return err
}
elbs, _, err := DescribeELBs(x.Cloud)
if err != nil {
return err
}
// Find masters
var masters []*ec2.Instance
for _, instance := range instances {
@ -67,9 +82,34 @@ func (x *UpgradeCluster) Upgrade() error {
return fmt.Errorf("could not find masters")
}
// Stop autoscalingGroups
for _, group := range autoscalingGroups {
name := aws.StringValue(group.AutoScalingGroupName)
glog.Infof("Stopping instances in autoscaling group %q", name)
request := &autoscaling.UpdateAutoScalingGroupInput{
AutoScalingGroupName: group.AutoScalingGroupName,
DesiredCapacity: aws.Int64(0),
MinSize: aws.Int64(0),
MaxSize: aws.Int64(0),
}
_, err := awsCloud.Autoscaling.UpdateAutoScalingGroup(request)
if err != nil {
return fmt.Errorf("error updating autoscaling group %q: %v", name, err)
}
}
// Stop masters
for _, master := range masters {
masterInstanceID := aws.StringValue(master.InstanceId)
masterState := aws.StringValue(master.State.Name)
if masterState == "terminated" {
glog.Infof("master already terminated: %q", masterInstanceID)
continue
}
glog.Infof("Stopping master: %q", masterInstanceID)
request := &ec2.StopInstancesInput{
@ -82,7 +122,36 @@ func (x *UpgradeCluster) Upgrade() error {
}
}
// TODO: Wait for master to stop & detach volumes?
// Detach volumes from masters
for _, master := range masters {
for _, bdm := range master.BlockDeviceMappings {
if bdm.Ebs == nil || bdm.Ebs.VolumeId == nil {
continue
}
volumeID := aws.StringValue(bdm.Ebs.VolumeId)
masterInstanceID := aws.StringValue(master.InstanceId)
glog.Infof("Detaching volume %q from instance %q", volumeID, masterInstanceID)
request := &ec2.DetachVolumeInput{
VolumeId: bdm.Ebs.VolumeId,
InstanceId: master.InstanceId,
}
for {
_, err := awsCloud.EC2.DetachVolume(request)
if err != nil {
if AWSErrorCode(err) == "IncorrectState" {
glog.Infof("retrying to detach volume (master has probably not stopped yet): %q", err)
time.Sleep(5 * time.Second)
continue
}
return fmt.Errorf("error detaching volume %q from master instance %q: %v", volumeID, masterInstanceID, err)
} else {
break
}
}
}
}
//subnets, err := DescribeSubnets(x.Cloud)
//if err != nil {
@ -100,6 +169,8 @@ func (x *UpgradeCluster) Upgrade() error {
// We have to be careful because VPCs can be shared
{
vpcID := cluster.Spec.NetworkID
retagGateway := false
if vpcID != "" {
tags, err := awsCloud.GetTags(vpcID)
if err != nil {
@ -113,18 +184,58 @@ func (x *UpgradeCluster) Upgrade() error {
}
replaceTags := make(map[string]string)
replaceTags[awsup.TagClusterName] = newClusterName
glog.Infof("Retagging VPC %q", vpcID)
err := awsCloud.CreateTags(vpcID, replaceTags)
if err != nil {
return fmt.Errorf("error re-tagging VPC: %v", err)
}
}
// The VPC was tagged as ours, so make sure the gateway is consistently retagged
retagGateway = true
}
}
if retagGateway {
gateways, err := DescribeInternetGatewaysIgnoreTags(x.Cloud)
if err != nil {
return fmt.Errorf("error listing gateways: %v", err)
}
for _, igw := range gateways {
match := false
for _, a := range igw.Attachments {
if vpcID == aws.StringValue(a.VpcId) {
match = true
}
}
if !match {
continue
}
id := aws.StringValue(igw.InternetGatewayId)
clusterTag, _ := awsup.FindEC2Tag(igw.Tags, awsup.TagClusterName)
if clusterTag == "" || clusterTag == oldClusterName {
replaceTags := make(map[string]string)
replaceTags[awsup.TagClusterName] = newClusterName
glog.Infof("Retagging InternetGateway %q", id)
err := awsCloud.CreateTags(id, replaceTags)
if err != nil {
return fmt.Errorf("error re-tagging InternetGateway: %v", err)
}
}
}
}
}
// Retag DHCP options
// We have to be careful because DHCP options can be shared
for _, dhcpOption := range dhcpOptions {
id := aws.StringValue(dhcpOption.DhcpOptionsId)
clusterTag, _ := awsup.FindEC2Tag(dhcpOption.Tags, awsup.TagClusterName)
if clusterTag != "" {
if clusterTag != oldClusterName {
@ -132,7 +243,10 @@ func (x *UpgradeCluster) Upgrade() error {
}
replaceTags := make(map[string]string)
replaceTags[awsup.TagClusterName] = newClusterName
err := awsCloud.CreateTags(*dhcpOption.DhcpOptionsId, replaceTags)
glog.Infof("Retagging DHCPOptions %q", id)
err := awsCloud.CreateTags(id, replaceTags)
if err != nil {
return fmt.Errorf("error re-tagging DHCP options: %v", err)
}
@ -140,10 +254,38 @@ func (x *UpgradeCluster) Upgrade() error {
}
// TODO: Retag internet gateway (may not be tagged at all though...)
// TODO: Share more code with cluste deletion?
// Adopt LoadBalancers & LoadBalancer Security Groups
for _, elb := range elbs {
id := aws.StringValue(elb.LoadBalancerName)
// TODO: Adopt LoadBalancers & LoadBalancer Security Groups
// TODO: Batch re-tag?
replaceTags := make(map[string]string)
replaceTags[awsup.TagClusterName] = newClusterName
glog.Infof("Retagging ELB %q", id)
err := awsCloud.CreateELBTags(id, replaceTags)
if err != nil {
return fmt.Errorf("error re-tagging ELB %q: %v", id, err)
}
}
for _, elb := range elbs {
for _, sg := range elb.SecurityGroups {
id := aws.StringValue(sg)
// TODO: Batch re-tag?
replaceTags := make(map[string]string)
replaceTags[awsup.TagClusterName] = newClusterName
glog.Infof("Retagging ELB security group %q", id)
err := awsCloud.CreateTags(id, replaceTags)
if err != nil {
return fmt.Errorf("error re-tagging ELB security group %q: %v", id, err)
}
}
}
// Adopt Volumes
for _, volume := range volumes {
@ -156,8 +298,11 @@ func (x *UpgradeCluster) Upgrade() error {
name, _ := awsup.FindEC2Tag(volume.Tags, "Name")
if name == oldClusterName+"-master-pd" {
glog.Infof("Found master volume %q: %s", id, name)
replaceTags["Name"] = "kubernetes.master." + aws.StringValue(volume.AvailabilityZone) + "." + newClusterName
az := aws.StringValue(volume.AvailabilityZone)
replaceTags["Name"] = az + ".etcd-main." + newClusterName
}
glog.Infof("Retagging volume %q", id)
err := awsCloud.CreateTags(id, replaceTags)
if err != nil {
return fmt.Errorf("error re-tagging volume %q: %v", id, err)
@ -165,10 +310,27 @@ func (x *UpgradeCluster) Upgrade() error {
}
cluster.Name = newClusterName
err = api.WriteConfig(x.StateStore, cluster, x.InstanceGroups)
err = api.WriteConfig(x.NewStateStore, cluster, x.InstanceGroups)
if err != nil {
return fmt.Errorf("error writing updated configuration: %v", err)
}
oldCACertPool, err := x.OldStateStore.CA().CertificatePool(fi.CertificateId_CA)
if err != nil {
return fmt.Errorf("error reading old CA certs: %v", err)
}
for _, ca := range oldCACertPool.Secondary {
err := x.NewStateStore.CA().AddCert(fi.CertificateId_CA, ca)
if err != nil {
return fmt.Errorf("error importing old CA certs: %v", err)
}
}
if oldCACertPool.Primary != nil {
err := x.NewStateStore.CA().AddCert(fi.CertificateId_CA, oldCACertPool.Primary)
if err != nil {
return fmt.Errorf("error importing old CA certs: %v", err)
}
}
return nil
}

2
upup/pkg/kutil/utils.go
View File

@ -8,6 +8,8 @@ import (
"k8s.io/kube-deploy/upup/pkg/fi/cloudup/awsup"
)
// findAutoscalingGroups finds autoscaling groups matching the specified tags
// This isn't entirely trivial because autoscaling doesn't let us filter with as much precision as we wouldlike
func findAutoscalingGroups(cloud *awsup.AWSCloud, tags map[string]string) ([]*autoscaling.Group, error) {
var asgs []*autoscaling.Group