merge conflict resolution

2019-02-21 15:20:05 -08:00 · 2019-02-21 15:20:05 -08:00 · 31a4c40131
parent 8362b1260e c1873d106d
commit 31a4c40131
355 changed files with 26537 additions and 2174 deletions
--- a/Gopkg.lock
+++ b/Gopkg.lock
--- a/17
+++ b/17
@ -58,7 +58,7 @@ unexport KOPS_BASE_URL KOPS_CLUSTER_NAME KOPS_RUN_OBSOLETE_VERSION KOPS_STATE_ST
 unexport SKIP_REGION_CHECK S3_ACCESS_KEY_ID S3_ENDPOINT S3_REGION S3_SECRET_ACCESS_KEY VSPHERE_USERNAME VSPHERE_PASSWORD

 # Keep in sync with upup/models/cloudup/resources/addons/dns-controller/
-DNS_CONTROLLER_TAG=1.11.0-alpha.1
+DNS_CONTROLLER_TAG=1.12.0-alpha.1

 # Keep in sync with logic in get_workspace_status
 # TODO: just invoke tools/get_workspace_status.sh?
@ -465,6 +465,10 @@ utils-dist:
 	mkdir -p ${DIST}/linux/amd64/
 	docker run -v `pwd`/.build/dist/linux/amd64/:/dist utils-builder /extract.sh

+.PHONY: bazel-utils-dist
+bazel-utils-dist:
+	bazel build //images/utils-builder:utils
+
 # --------------------------------------------------
 # development targets

@ -756,7 +760,7 @@ bazel-protokube-export:
 	(${SHASUMCMD} ${BAZELIMAGES}/protokube.tar.gz | cut -d' ' -f1) > ${BAZELIMAGES}/protokube.tar.gz.sha1

 .PHONY: bazel-version-dist
-bazel-version-dist: bazel-crossbuild-nodeup bazel-crossbuild-kops bazel-protokube-export utils-dist
+bazel-version-dist: bazel-crossbuild-nodeup bazel-crossbuild-kops bazel-protokube-export bazel-utils-dist
 	rm -rf ${BAZELUPLOAD}
 	mkdir -p ${BAZELUPLOAD}/kops/${VERSION}/linux/amd64/
 	mkdir -p ${BAZELUPLOAD}/kops/${VERSION}/darwin/amd64/
@ -773,8 +777,8 @@ bazel-version-dist: bazel-crossbuild-nodeup bazel-crossbuild-kops bazel-protokub
 	(${SHASUMCMD} ${BAZELUPLOAD}/kops/${VERSION}/darwin/amd64/kops | cut -d' ' -f1) > ${BAZELUPLOAD}/kops/${VERSION}/darwin/amd64/kops.sha1
 	cp bazel-bin/cmd/kops/windows_amd64_pure_stripped/kops.exe ${BAZELUPLOAD}/kops/${VERSION}/windows/amd64/kops.exe
 	(${SHASUMCMD} ${BAZELUPLOAD}/kops/${VERSION}/windows/amd64/kops.exe | cut -d' ' -f1) > ${BAZELUPLOAD}/kops/${VERSION}/windows/amd64/kops.exe.sha1
-	cp ${DIST}/linux/amd64/utils.tar.gz ${BAZELUPLOAD}/kops/${VERSION}/linux/amd64/utils.tar.gz
-	cp ${DIST}/linux/amd64/utils.tar.gz.sha1 ${BAZELUPLOAD}/kops/${VERSION}/linux/amd64/utils.tar.gz.sha1
+	cp bazel-bin/images/utils-builder/utils.tar.gz ${BAZELUPLOAD}/kops/${VERSION}/linux/amd64/utils.tar.gz
+	(${SHASUMCMD} ${BAZELUPLOAD}/kops/${VERSION}/linux/amd64/utils.tar.gz | cut -d' ' -f1) > ${BAZELUPLOAD}/kops/${VERSION}/linux/amd64/utils.tar.gz.sha1

 .PHONY: bazel-upload
 bazel-upload: bazel-version-dist # Upload kops to S3
@ -787,7 +791,7 @@ prow-postsubmit: bazel-version-dist
 	${UPLOAD} ${BAZELUPLOAD}/kops/${VERSION}/ ${UPLOAD_DEST}/${KOPS_RELEASE_VERSION}-${GITSHA}/

 #-----------------------------------------------------------
-# static html documentation  
+# static html documentation

 .PHONY: live-docs
 live-docs:
@ -797,8 +801,9 @@ live-docs:
 build-docs:
 	@docker run --rm -it -v ${PWD}:/docs aledbf/mkdocs:0.1 build

+# Update machine_types.go
 .PHONY: update-machine-types
-update-machine-types: #Update machine_types.go
+update-machine-types:
 	go build -o hack/machine_types/machine_types  ${KOPS_ROOT}/hack/machine_types/
 	hack/machine_types/machine_types --out upup/pkg/fi/cloudup/awsup/machine_types.go
 	go fmt upup/pkg/fi/cloudup/awsup/machine_types.go
--- a/2
+++ b/2
@ -1,3 +1,5 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+
 approvers:
  - justinsb
  - chrislovecnm
--- a/35
+++ b/35
@ -6,22 +6,22 @@ load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")

 http_archive(
    name = "io_bazel_rules_go",
-    url = "https://github.com/bazelbuild/rules_go/releases/download/0.15.8/rules_go-0.15.8.tar.gz",
-    sha256 = "ca79fed5b24dcc0696e1651ecdd916f7a11111283ba46ea07633a53d8e1f5199",
+    urls = ["https://github.com/bazelbuild/rules_go/releases/download/0.17.0/rules_go-0.17.0.tar.gz"],
+    sha256 = "492c3ac68ed9dcf527a07e6a1b2dcbf199c6bf8b35517951467ac32e421c06c1",
 )

 http_archive(
    name = "bazel_gazelle",
-    url = "https://github.com/bazelbuild/bazel-gazelle/releases/download/0.15.0/bazel-gazelle-0.15.0.tar.gz",
-    sha256 = "6e875ab4b6bf64a38c352887760f21203ab054676d9c1b274963907e0768740d",
+    url = "https://github.com/bazelbuild/bazel-gazelle/releases/download/0.16.0/bazel-gazelle-0.16.0.tar.gz",
+    sha256 = "7949fc6cc17b5b191103e97481cf8889217263acf52e00b560683413af204fcb",
 )

-load("@io_bazel_rules_go//go:def.bzl", "go_rules_dependencies", "go_register_toolchains")
+load("@io_bazel_rules_go//go:deps.bzl", "go_rules_dependencies", "go_register_toolchains")

 go_rules_dependencies()

 go_register_toolchains(
-    go_version = "1.10.5",
+    go_version = "1.10.8",
 )

 load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies")
@ -34,17 +34,21 @@ gazelle_dependencies()
 git_repository(
    name = "io_bazel_rules_docker",
    remote = "https://github.com/bazelbuild/rules_docker.git",
-    tag = "v0.5.1",
+    tag = "v0.7.0",
 )

 load(
-    "@io_bazel_rules_docker//container:container.bzl",
-    "container_pull",
+    "@io_bazel_rules_docker//repositories:repositories.bzl",
    container_repositories = "repositories",
 )

 container_repositories()

+load(
+    "@io_bazel_rules_docker//container:container.bzl",
+    "container_pull",
+)
+
 container_pull(
    name = "debian_hyperkube_base_amd64",
    # 'tag' is also supported, but digest is encouraged for reproducibility.
@ -73,8 +77,8 @@ dpkg_src(
    name = "debian_stretch",
    arch = "amd64",
    distro = "stretch",
-    sha256 = "4cb2fac3e32292613b92d3162e99eb8a1ed7ce47d1b142852b0de3092b25910c",
-    snapshot = "20180406T154421Z",
+    sha256 = "4b981bd2445d85cf1d93ea9f2d2dc235d20543ecdadd0d8065a10793b94eab9b",
+    snapshot = "20190131T155411Z",
    url = "http://snapshot.debian.org/archive",
 )

@ -100,3 +104,12 @@ dpkg_list(
        "@debian_stretch//file:Packages.json",
    ],
 )
+
+# We use the prebuilt utils.tar.gz containing socat & conntrack, building it in bazel is really painful
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_file")
+
+http_file(
+    name = "utils_tar_gz",
+    urls = ["https://kubeupv2.s3.amazonaws.com/kops/1.11.0/linux/amd64/utils.tar.gz"],
+    sha256 = "7b288b4ab499186120f1f251d7a16b430073487bda8571574fcc9243925d6a30",
+)
--- a/addons/prometheus-operator/v0.19.0.yaml
+++ b/addons/prometheus-operator/v0.19.0.yaml
@ -12943,8 +12943,12 @@ rules:
  - ""
  resources:
  - nodes/metrics
+  - pods
+  - endpoints
+  - services
  verbs:
  - get
+  - list
 - nonResourceURLs:
  - /metrics
  verbs:
--- a/channels/alpha
+++ b/channels/alpha
@ -34,10 +34,10 @@ spec:
      kubenet: {}
  kubernetesVersions:
  - range: ">=1.12.0"
-    recommendedVersion: 1.12.4
+    recommendedVersion: 1.12.5
    requiredVersion: 1.12.0
  - range: ">=1.11.0"
-    recommendedVersion: 1.11.6
+    recommendedVersion: 1.11.7
    requiredVersion: 1.11.0
  - range: ">=1.10.0"
    recommendedVersion: 1.10.12
@ -64,11 +64,11 @@ spec:
  - range: ">=1.12.0-alpha.1"
    #recommendedVersion: "1.12.0"
    #requiredVersion: 1.12.0
-    kubernetesVersion: 1.12.3
+    kubernetesVersion: 1.12.5
  - range: ">=1.11.0-alpha.1"
    #recommendedVersion: "1.11.0"
    #requiredVersion: 1.11.0
-    kubernetesVersion: 1.11.6
+    kubernetesVersion: 1.11.7
  - range: ">=1.10.0-alpha.1"
    recommendedVersion: "1.10.0"
    #requiredVersion: 1.10.0
--- a/channels/pkg/channels/channel_version.go
+++ b/channels/pkg/channels/channel_version.go
@ -23,7 +23,7 @@ import (

 	"github.com/blang/semver"
 	"github.com/golang/glog"
-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/kubernetes"
--- a/channels/pkg/cmd/get_addons.go
+++ b/channels/pkg/cmd/get_addons.go
@ -38,7 +38,7 @@ import (
 	"os"

 	"github.com/spf13/cobra"
-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/kops/channels/pkg/channels"
 	"k8s.io/kops/util/pkg/tables"
--- a/cloudmock/aws/mockec2/BUILD.bazel
+++ b/cloudmock/aws/mockec2/BUILD.bazel
@ -11,6 +11,7 @@ go_library(
        "instances.go",
        "internetgateways.go",
        "keypairs.go",
+        "launch_templates.go",
        "natgateway.go",
        "routetable.go",
        "securitygroups.go",
--- a/cloudmock/aws/mockec2/api.go
+++ b/cloudmock/aws/mockec2/api.go
@ -55,6 +55,8 @@ type MockEC2 struct {

 	InternetGateways map[string]*ec2.InternetGateway

+	LaunchTemplates map[string]*ec2.ResponseLaunchTemplateData
+
 	NatGateways map[string]*ec2.NatGateway

 	idsMutex sync.Mutex
--- a/cloudmock/aws/mockec2/launch_templates.go
+++ b/cloudmock/aws/mockec2/launch_templates.go
@ -0,0 +1,177 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package mockec2
+
+import (
+	"fmt"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+)
+
+// DescribeLaunchTemplates mocks the describing the launch templates
+func (m *MockEC2) DescribeLaunchTemplates(request *ec2.DescribeLaunchTemplatesInput) (*ec2.DescribeLaunchTemplatesOutput, error) {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+
+	o := &ec2.DescribeLaunchTemplatesOutput{}
+
+	if m.LaunchTemplates == nil {
+		return o, nil
+	}
+
+	for name := range m.LaunchTemplates {
+		o.LaunchTemplates = append(o.LaunchTemplates, &ec2.LaunchTemplate{
+			LaunchTemplateName: aws.String(name),
+		})
+	}
+
+	return o, nil
+}
+
+// DescribeLaunchTemplateVersions mocks the retrieval of launch template versions - we dont use this at the moment so we can just return the template
+func (m *MockEC2) DescribeLaunchTemplateVersions(request *ec2.DescribeLaunchTemplateVersionsInput) (*ec2.DescribeLaunchTemplateVersionsOutput, error) {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+
+	o := &ec2.DescribeLaunchTemplateVersionsOutput{}
+
+	if m.LaunchTemplates == nil {
+		return o, nil
+	}
+
+	lt, found := m.LaunchTemplates[aws.StringValue(request.LaunchTemplateName)]
+	if !found {
+		return o, nil
+	}
+	o.LaunchTemplateVersions = append(o.LaunchTemplateVersions, &ec2.LaunchTemplateVersion{
+		DefaultVersion:     aws.Bool(true),
+		LaunchTemplateData: lt,
+		LaunchTemplateName: request.LaunchTemplateName,
+	})
+
+	return o, nil
+}
+
+// CreateLaunchTemplate mocks the ec2 create launch template
+func (m *MockEC2) CreateLaunchTemplate(request *ec2.CreateLaunchTemplateInput) (*ec2.CreateLaunchTemplateOutput, error) {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+
+	if m.LaunchTemplates == nil {
+		m.LaunchTemplates = make(map[string]*ec2.ResponseLaunchTemplateData)
+	}
+	if m.LaunchTemplates[aws.StringValue(request.LaunchTemplateName)] != nil {
+		return nil, fmt.Errorf("duplicate LaunchTemplateName %s", aws.StringValue(request.LaunchTemplateName))
+	}
+	resp := &ec2.ResponseLaunchTemplateData{
+		DisableApiTermination: request.LaunchTemplateData.DisableApiTermination,
+		EbsOptimized:          request.LaunchTemplateData.EbsOptimized,
+		ImageId:               request.LaunchTemplateData.ImageId,
+		InstanceType:          request.LaunchTemplateData.InstanceType,
+		KeyName:               request.LaunchTemplateData.KeyName,
+		SecurityGroupIds:      request.LaunchTemplateData.SecurityGroupIds,
+		SecurityGroups:        request.LaunchTemplateData.SecurityGroups,
+		UserData:              request.LaunchTemplateData.UserData,
+	}
+	m.LaunchTemplates[aws.StringValue(request.LaunchTemplateName)] = resp
+
+	// @GOD DAMN AWS request vs response .. fu@@@@@ .. so much typing!!#@#@#
+
+	if request.LaunchTemplateData.Monitoring != nil {
+		resp.Monitoring = &ec2.LaunchTemplatesMonitoring{Enabled: request.LaunchTemplateData.Monitoring.Enabled}
+	}
+	if request.LaunchTemplateData.CpuOptions != nil {
+		resp.CpuOptions = &ec2.LaunchTemplateCpuOptions{
+			CoreCount:      request.LaunchTemplateData.CpuOptions.CoreCount,
+			ThreadsPerCore: request.LaunchTemplateData.CpuOptions.ThreadsPerCore,
+		}
+	}
+	if len(request.LaunchTemplateData.BlockDeviceMappings) > 0 {
+		for _, x := range request.LaunchTemplateData.BlockDeviceMappings {
+			resp.BlockDeviceMappings = append(resp.BlockDeviceMappings, &ec2.LaunchTemplateBlockDeviceMapping{
+				DeviceName: x.DeviceName,
+				Ebs: &ec2.LaunchTemplateEbsBlockDevice{
+					DeleteOnTermination: x.Ebs.DeleteOnTermination,
+					Encrypted:           x.Ebs.Encrypted,
+					Iops:                x.Ebs.Iops,
+					KmsKeyId:            x.Ebs.KmsKeyId,
+					SnapshotId:          x.Ebs.SnapshotId,
+					VolumeSize:          x.Ebs.VolumeSize,
+					VolumeType:          x.Ebs.VolumeType,
+				},
+				NoDevice:    x.NoDevice,
+				VirtualName: x.VirtualName,
+			})
+		}
+	}
+	if request.LaunchTemplateData.CreditSpecification != nil {
+		resp.CreditSpecification = &ec2.CreditSpecification{CpuCredits: request.LaunchTemplateData.CreditSpecification.CpuCredits}
+	}
+	if request.LaunchTemplateData.IamInstanceProfile != nil {
+		resp.IamInstanceProfile = &ec2.LaunchTemplateIamInstanceProfileSpecification{
+			Arn:  request.LaunchTemplateData.IamInstanceProfile.Arn,
+			Name: request.LaunchTemplateData.IamInstanceProfile.Name,
+		}
+	}
+	if request.LaunchTemplateData.InstanceMarketOptions != nil {
+		resp.InstanceMarketOptions = &ec2.LaunchTemplateInstanceMarketOptions{
+			MarketType: request.LaunchTemplateData.InstanceMarketOptions.MarketType,
+			SpotOptions: &ec2.LaunchTemplateSpotMarketOptions{
+				BlockDurationMinutes:         request.LaunchTemplateData.InstanceMarketOptions.SpotOptions.BlockDurationMinutes,
+				InstanceInterruptionBehavior: request.LaunchTemplateData.InstanceMarketOptions.SpotOptions.InstanceInterruptionBehavior,
+				MaxPrice:                     request.LaunchTemplateData.InstanceMarketOptions.SpotOptions.MaxPrice,
+				SpotInstanceType:             request.LaunchTemplateData.InstanceMarketOptions.SpotOptions.SpotInstanceType,
+				ValidUntil:                   request.LaunchTemplateData.InstanceMarketOptions.SpotOptions.ValidUntil,
+			},
+		}
+	}
+	if len(request.LaunchTemplateData.NetworkInterfaces) > 0 {
+		for _, x := range request.LaunchTemplateData.NetworkInterfaces {
+			resp.NetworkInterfaces = append(resp.NetworkInterfaces, &ec2.LaunchTemplateInstanceNetworkInterfaceSpecification{
+				AssociatePublicIpAddress:       x.AssociatePublicIpAddress,
+				DeleteOnTermination:            x.DeleteOnTermination,
+				Description:                    x.Description,
+				DeviceIndex:                    x.DeviceIndex,
+				Groups:                         x.Groups,
+				Ipv6AddressCount:               x.Ipv6AddressCount,
+				NetworkInterfaceId:             x.NetworkInterfaceId,
+				PrivateIpAddress:               x.PrivateIpAddress,
+				PrivateIpAddresses:             x.PrivateIpAddresses,
+				SecondaryPrivateIpAddressCount: x.SecondaryPrivateIpAddressCount,
+				SubnetId:                       x.SubnetId,
+			})
+		}
+	}
+
+	return &ec2.CreateLaunchTemplateOutput{}, nil
+}
+
+// DeleteLaunchTemplate mocks the deletion of a launch template
+func (m *MockEC2) DeleteLaunchTemplate(request *ec2.DeleteLaunchTemplateInput) (*ec2.DeleteLaunchTemplateOutput, error) {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+
+	o := &ec2.DeleteLaunchTemplateOutput{}
+
+	if m.LaunchTemplates == nil {
+		return o, nil
+	}
+	delete(m.LaunchTemplates, aws.StringValue(request.LaunchTemplateName))
+
+	return o, nil
+}
--- a/cmd/kops/create_cluster.go
+++ b/cmd/kops/create_cluster.go
@ -77,6 +77,7 @@ type CreateClusterOptions struct {
 	MasterVolumeSize     int32
 	NodeVolumeSize       int32
 	EncryptEtcdStorage   bool
+	EtcdStorageType      string
 	Project              string
 	KubernetesVersion    string
 	OutDir               string
@ -148,6 +149,13 @@ type CreateClusterOptions struct {
 	SpotinstProduct     string
 	SpotinstOrientation string

+	// OpenstackExternalNet is the name of the external network for the openstack router
+	OpenstackExternalNet     string
+	OpenstackStorageIgnoreAZ bool
+
+	// OpenstackLBOctavia is boolean value should we use octavia or old loadbalancer api
+	OpenstackLBOctavia bool
+
 	// ConfigBase is the location where we will store the configuration, it defaults to the state store
 	ConfigBase string

@ -278,7 +286,7 @@ func NewCmdCreateCluster(f *util.Factory, out io.Writer) *cobra.Command {
 		cmd.Flags().StringVar(&options.ConfigBase, "config-base", options.ConfigBase, "A cluster-readable location where we mirror configuration information, separate from the state store.  Allows for a state store that is not accessible from the cluster.")
 	}

-	cmd.Flags().StringVar(&options.Cloud, "cloud", options.Cloud, "Cloud provider to use - gce, aws, vsphere")
+	cmd.Flags().StringVar(&options.Cloud, "cloud", options.Cloud, "Cloud provider to use - gce, aws, vsphere, openstack")

 	cmd.Flags().StringSliceVar(&options.Zones, "zones", options.Zones, "Zones in which to run the cluster")
 	cmd.Flags().StringSliceVar(&options.MasterZones, "master-zones", options.MasterZones, "Zones in which to run masters (must be an odd number)")
@ -304,6 +312,7 @@ func NewCmdCreateCluster(f *util.Factory, out io.Writer) *cobra.Command {
 	cmd.Flags().Int32Var(&options.MasterCount, "master-count", options.MasterCount, "Set the number of masters.  Defaults to one master per master-zone")
 	cmd.Flags().Int32Var(&options.NodeCount, "node-count", options.NodeCount, "Set the number of nodes")
 	cmd.Flags().BoolVar(&options.EncryptEtcdStorage, "encrypt-etcd-storage", options.EncryptEtcdStorage, "Generate key in aws kms and use it for encrypt etcd volumes")
+	cmd.Flags().StringVar(&options.EtcdStorageType, "etcd-storage-type", options.EtcdStorageType, "The default storage type for etc members")

 	cmd.Flags().StringVar(&options.Image, "image", options.Image, "Image to use for all instances.")

@ -370,6 +379,13 @@ func NewCmdCreateCluster(f *util.Factory, out io.Writer) *cobra.Command {
 		cmd.Flags().StringVar(&options.SpotinstOrientation, "spotinst-orientation", options.SpotinstOrientation, "Set the prediction strategy (valid values: balanced, cost, equal-distribution and availability)")
 	}

+	if cloudup.AlphaAllowOpenstack.Enabled() {
+		// Openstack flags
+		cmd.Flags().StringVar(&options.OpenstackExternalNet, "os-ext-net", options.OpenstackExternalNet, "The name of the external network to use with the openstack router")
+		cmd.Flags().BoolVar(&options.OpenstackStorageIgnoreAZ, "os-kubelet-ignore-az", options.OpenstackStorageIgnoreAZ, "If true kubernetes may attach volumes across availability zones")
+		cmd.Flags().BoolVar(&options.OpenstackLBOctavia, "os-octavia", options.OpenstackLBOctavia, "If true octavia loadbalancer api will be used")
+	}
+
 	return cmd
 }

@ -732,6 +748,9 @@ func RunCreateCluster(f *util.Factory, out io.Writer, c *CreateClusterOptions) e
 				if c.EncryptEtcdStorage {
 					m.EncryptedVolume = &c.EncryptEtcdStorage
 				}
+				if len(c.EtcdStorageType) > 0 {
+					m.VolumeType = fi.String(c.EtcdStorageType)
+				}
 				m.Name = names[i]

 				m.InstanceGroup = fi.String(ig.ObjectMeta.Name)
@ -882,6 +901,36 @@ func RunCreateCluster(f *util.Factory, out io.Writer, c *CreateClusterOptions) e
 				cluster.Spec.CloudConfig.SpotinstOrientation = fi.String(c.SpotinstOrientation)
 			}
 		}
+
+		if cloudup.AlphaAllowOpenstack.Enabled() && c.Cloud == "openstack" {
+			if cluster.Spec.CloudConfig == nil {
+				cluster.Spec.CloudConfig = &api.CloudConfiguration{}
+			}
+			provider := "haproxy"
+			if c.OpenstackLBOctavia {
+				provider = "octavia"
+			}
+			cluster.Spec.CloudConfig.Openstack = &api.OpenstackConfiguration{
+				Router: &api.OpenstackRouter{
+					ExternalNetwork: fi.String(c.OpenstackExternalNet),
+				},
+				Loadbalancer: &api.OpenstackLoadbalancerConfig{
+					FloatingNetwork: fi.String(c.OpenstackExternalNet),
+					Method:          fi.String("ROUND_ROBIN"),
+					Provider:        fi.String(provider),
+					UseOctavia:      fi.Bool(c.OpenstackLBOctavia),
+				},
+				BlockStorage: &api.OpenstackBlockStorageConfig{
+					Version:  fi.String("v2"),
+					IgnoreAZ: fi.Bool(c.OpenstackStorageIgnoreAZ),
+				},
+				Monitor: &api.OpenstackMonitor{
+					Delay:      fi.String("1m"),
+					Timeout:    fi.String("30s"),
+					MaxRetries: fi.Int(3),
+				},
+			}
+		}
 	}

 	// Populate project
--- a/cmd/kops/integration_test.go
+++ b/cmd/kops/integration_test.go
@ -141,6 +141,11 @@ func TestPrivateKopeio(t *testing.T) {
 	runTestAWS(t, "privatekopeio.example.com", "privatekopeio", "v1alpha2", true, 1, true, nil)
 }

+// TestUnmanaged is a test where all the subnets opt-out of route management
+func TestUnmanaged(t *testing.T) {
+	runTestAWS(t, "unmanaged.example.com", "unmanaged", "v1alpha2", true, 1, true, nil)
+}
+
 // TestPrivateSharedSubnet runs the test on a configuration with private topology & shared subnets
 func TestPrivateSharedSubnet(t *testing.T) {
 	runTestAWS(t, "private-shared-subnet.example.com", "private-shared-subnet", "v1alpha2", true, 1, true, nil)
--- a/cmd/kops/rollingupdatecluster.go
+++ b/cmd/kops/rollingupdatecluster.go
@ -26,7 +26,7 @@ import (

 	"github.com/golang/glog"
 	"github.com/spf13/cobra"
-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/cli-runtime/pkg/genericclioptions"
 	"k8s.io/client-go/kubernetes"
--- a/cmd/kops/root.go
+++ b/cmd/kops/root.go
@ -147,6 +147,7 @@ func NewCmdRoot(f *util.Factory, out io.Writer) *cobra.Command {
 	cmd.AddCommand(NewCmdSet(f, out))
 	cmd.AddCommand(NewCmdToolbox(f, out))
 	cmd.AddCommand(NewCmdValidate(f, out))
+	cmd.AddCommand(NewCmdVersion(f, out))

 	return cmd
 }
--- a/cmd/kops/validate_cluster.go
+++ b/cmd/kops/validate_cluster.go
@ -27,7 +27,7 @@ import (
 	"github.com/ghodss/yaml"
 	"github.com/golang/glog"
 	"github.com/spf13/cobra"
-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/clientcmd"
--- a/cmd/kops/version.go
+++ b/cmd/kops/version.go
@ -17,10 +17,11 @@ limitations under the License.
 package main

 import (
-	"fmt"
+	"io"

 	"github.com/spf13/cobra"
-	"k8s.io/kops"
+	"k8s.io/kops/cmd/kops/util"
+	"k8s.io/kops/pkg/commands"
 	"k8s.io/kubernetes/pkg/kubectl/cmd/templates"
 	"k8s.io/kubernetes/pkg/kubectl/util/i18n"
 )
@ -35,37 +36,25 @@ var (
 	versionShort = i18n.T(`Print the kops version information.`)
 )

-type VersionCmd struct {
-	cobraCommand *cobra.Command
-}
+// NewCmdVersion builds a cobra command for the kops version command
+func NewCmdVersion(f *util.Factory, out io.Writer) *cobra.Command {
+	options := &commands.VersionOptions{}

-var versionCmd = VersionCmd{
-	cobraCommand: &cobra.Command{
+	cmd := &cobra.Command{
 		Use:     "version",
 		Short:   versionShort,
 		Long:    versionLong,
 		Example: versionExample,
-	},
-}
-
-func init() {
-	cmd := versionCmd.cobraCommand
-	rootCommand.cobraCommand.AddCommand(cmd)
+	}

 	cmd.Run = func(cmd *cobra.Command, args []string) {
-		err := versionCmd.Run()
+		err := commands.RunVersion(f, out, options)
 		if err != nil {
 			exitWithError(err)
 		}
 	}
-}

-func (c *VersionCmd) Run() error {
-	s := "Version " + kops.Version
-	if kops.GitVersion != "" {
-		s += " (git-" + kops.GitVersion + ")"
-	}
-	fmt.Println(s)
+	cmd.Flags().BoolVar(&options.Short, "short", options.Short, "only print the main kops version, useful for scripting")

-	return nil
+	return cmd
 }
--- a/dns-controller/pkg/watchers/node.go
+++ b/dns-controller/pkg/watchers/node.go
@ -25,7 +25,7 @@ import (
 	kopsutil "k8s.io/kops/pkg/apis/kops/util"

 	"github.com/golang/glog"
-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/watch"
 	"k8s.io/client-go/kubernetes"
--- a/dns-controller/pkg/watchers/pod.go
+++ b/dns-controller/pkg/watchers/pod.go
@ -24,7 +24,7 @@ import (

 	"strings"

-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/watch"
 	"k8s.io/client-go/kubernetes"
--- a/dns-controller/pkg/watchers/service.go
+++ b/dns-controller/pkg/watchers/service.go
@ -25,7 +25,7 @@ import (
 	"k8s.io/kops/dns-controller/pkg/util"

 	"github.com/golang/glog"
-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/watch"
 	"k8s.io/client-go/kubernetes"
--- a/dnsprovider/pkg/dnsprovider/providers/coredns/coredns.go
+++ b/dnsprovider/pkg/dnsprovider/providers/coredns/coredns.go
@ -25,7 +25,7 @@ import (

 	etcdc "github.com/coreos/etcd/client"
 	"github.com/golang/glog"
-	"gopkg.in/gcfg.v1"
+	gcfg "gopkg.in/gcfg.v1"
 	"k8s.io/kops/dnsprovider/pkg/dnsprovider"
 )

--- a/dnsprovider/pkg/dnsprovider/providers/openstack/designate/BUILD.bazel
+++ b/dnsprovider/pkg/dnsprovider/providers/openstack/designate/BUILD.bazel
@ -0,0 +1,27 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "designate.go",
+        "interface.go",
+        "rrchangeset.go",
+        "rrset.go",
+        "rrsets.go",
+        "zone.go",
+        "zones.go",
+    ],
+    importpath = "k8s.io/kops/dnsprovider/pkg/dnsprovider/providers/openstack/designate",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//dnsprovider/pkg/dnsprovider:go_default_library",
+        "//dnsprovider/pkg/dnsprovider/rrstype:go_default_library",
+        "//util/pkg/vfs:go_default_library",
+        "//vendor/github.com/golang/glog:go_default_library",
+        "//vendor/github.com/gophercloud/gophercloud:go_default_library",
+        "//vendor/github.com/gophercloud/gophercloud/openstack:go_default_library",
+        "//vendor/github.com/gophercloud/gophercloud/openstack/dns/v2/recordsets:go_default_library",
+        "//vendor/github.com/gophercloud/gophercloud/openstack/dns/v2/zones:go_default_library",
+        "//vendor/github.com/gophercloud/gophercloud/pagination:go_default_library",
+    ],
+)
--- a/dnsprovider/pkg/dnsprovider/providers/openstack/designate/designate.go
+++ b/dnsprovider/pkg/dnsprovider/providers/openstack/designate/designate.go
@ -0,0 +1,83 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package designate
+
+import (
+	"crypto/tls"
+	"fmt"
+	"io"
+	"net/http"
+
+	"github.com/golang/glog"
+	"github.com/gophercloud/gophercloud/openstack"
+	"k8s.io/kops/dnsprovider/pkg/dnsprovider"
+	"k8s.io/kops/util/pkg/vfs"
+)
+
+const (
+	// ProviderName is the name of this DNS provider
+	ProviderName = "openstack-designate"
+)
+
+func init() {
+	dnsprovider.RegisterDnsProvider(ProviderName, func(config io.Reader) (dnsprovider.Interface, error) {
+		return newDesignate(config)
+	})
+}
+
+func newDesignate(_ io.Reader) (*Interface, error) {
+	oc := vfs.OpenstackConfig{}
+	ao, err := oc.GetCredential()
+	if err != nil {
+		return nil, err
+	}
+
+	/*
+		pc, err := openstack.AuthenticatedClient(ao)
+		if err != nil {
+			return nil, fmt.Errorf("error building openstack authenticated client: %v", err)
+		}*/
+
+	provider, err := openstack.NewClient(ao.IdentityEndpoint)
+	if err != nil {
+		return nil, fmt.Errorf("error building openstack provider client: %v", err)
+	}
+
+	tlsconfig := &tls.Config{}
+	tlsconfig.InsecureSkipVerify = true
+	transport := &http.Transport{TLSClientConfig: tlsconfig}
+	provider.HTTPClient = http.Client{
+		Transport: transport,
+	}
+
+	glog.V(2).Info("authenticating to keystone")
+
+	err = openstack.Authenticate(provider, ao)
+	if err != nil {
+		return nil, fmt.Errorf("error building openstack authenticated client: %v", err)
+	}
+
+	endpointOpt, err := oc.GetServiceConfig("Designate")
+	if err != nil {
+		return nil, err
+	}
+	sc, err := openstack.NewDNSV2(provider, endpointOpt)
+	if err != nil {
+
+	}
+	return New(sc), nil
+}
--- a/dnsprovider/pkg/dnsprovider/providers/openstack/designate/interface.go
+++ b/dnsprovider/pkg/dnsprovider/providers/openstack/designate/interface.go
@ -0,0 +1,38 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package designate
+
+import (
+	"github.com/gophercloud/gophercloud"
+	"k8s.io/kops/dnsprovider/pkg/dnsprovider"
+)
+
+var _ dnsprovider.Interface = Interface{}
+
+type Interface struct {
+	sc *gophercloud.ServiceClient
+}
+
+// New builds an Interface, with a specified Designate implementation.
+// This is useful for testing purposes, but also if we want an instance with custom OpenStack options.
+func New(sc *gophercloud.ServiceClient) *Interface {
+	return &Interface{sc}
+}
+
+func (i Interface) Zones() (zones dnsprovider.Zones, supported bool) {
+	return Zones{&i}, true
+}
--- a/dnsprovider/pkg/dnsprovider/providers/openstack/designate/rrchangeset.go
+++ b/dnsprovider/pkg/dnsprovider/providers/openstack/designate/rrchangeset.go
@ -0,0 +1,136 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package designate
+
+import (
+	"fmt"
+
+	"github.com/gophercloud/gophercloud/openstack/dns/v2/recordsets"
+
+	"k8s.io/kops/dnsprovider/pkg/dnsprovider"
+)
+
+var _ dnsprovider.ResourceRecordChangeset = &ResourceRecordChangeset{}
+
+type ResourceRecordChangeset struct {
+	zone   *Zone
+	rrsets *ResourceRecordSets
+
+	additions []dnsprovider.ResourceRecordSet
+	removals  []dnsprovider.ResourceRecordSet
+	upserts   []dnsprovider.ResourceRecordSet
+}
+
+func (c *ResourceRecordChangeset) Add(rrset dnsprovider.ResourceRecordSet) dnsprovider.ResourceRecordChangeset {
+	c.additions = append(c.additions, rrset)
+	return c
+}
+
+func (c *ResourceRecordChangeset) Remove(rrset dnsprovider.ResourceRecordSet) dnsprovider.ResourceRecordChangeset {
+	c.removals = append(c.removals, rrset)
+	return c
+}
+
+func (c *ResourceRecordChangeset) Upsert(rrset dnsprovider.ResourceRecordSet) dnsprovider.ResourceRecordChangeset {
+	c.upserts = append(c.upserts, rrset)
+	return c
+}
+
+func (c *ResourceRecordChangeset) Apply() error {
+	zoneID := c.zone.impl.ID
+
+	for _, removal := range c.removals {
+		rrID, err := c.nameToID(removal.Name())
+		if err != nil {
+
+		}
+		err = recordsets.Delete(c.zone.zones.iface.sc, zoneID, rrID).ExtractErr()
+		if err != nil {
+
+		}
+	}
+
+	for _, addition := range c.additions {
+		opts := recordsets.CreateOpts{
+			Name:    addition.Name(),
+			TTL:     int(addition.Ttl()),
+			Type:    string(addition.Type()),
+			Records: addition.Rrdatas(),
+		}
+		_, err := recordsets.Create(c.zone.zones.iface.sc, zoneID, opts).Extract()
+		if err != nil {
+
+		}
+	}
+
+	for _, upsert := range c.upserts {
+		rrID, err := c.nameToID(upsert.Name())
+		if err != nil {
+
+		}
+		uopts := recordsets.UpdateOpts{
+			TTL:     int(upsert.Ttl()),
+			Records: upsert.Rrdatas(),
+		}
+		_, err = recordsets.Update(c.zone.zones.iface.sc, zoneID, rrID, uopts).Extract()
+		if err != nil {
+			copts := recordsets.CreateOpts{
+				Name:    upsert.Name(),
+				TTL:     int(upsert.Ttl()),
+				Type:    string(upsert.Type()),
+				Records: upsert.Rrdatas(),
+			}
+			_, err := recordsets.Create(c.zone.zones.iface.sc, zoneID, copts).Extract()
+			if err != nil {
+
+			}
+		}
+	}
+
+	return nil
+}
+
+func (c *ResourceRecordChangeset) IsEmpty() bool {
+	return len(c.removals) == 0 && len(c.additions) == 0 && len(c.upserts) == 0
+}
+
+// ResourceRecordSets returns the parent ResourceRecordSets
+func (c *ResourceRecordChangeset) ResourceRecordSets() dnsprovider.ResourceRecordSets {
+	return c.rrsets
+}
+
+func (c *ResourceRecordChangeset) nameToID(name string) (string, error) {
+	opts := recordsets.ListOpts{
+		Name: name,
+	}
+	allPages, err := recordsets.ListByZone(c.zone.zones.iface.sc, c.zone.impl.ID, opts).AllPages()
+	if err != nil {
+
+	}
+	rrs, err := recordsets.ExtractRecordSets(allPages)
+	if err != nil {
+
+	}
+	switch len(rrs) {
+	case 0:
+		return "", fmt.Errorf("couldn't find recordset with name: %s, expected 1", name)
+	case 1:
+		return rrs[0].ID, nil
+	default:
+		return "", fmt.Errorf("found multiple recordsets with name: %s, expected 1", name)
+	}
+}
--- a/dnsprovider/pkg/dnsprovider/providers/openstack/designate/rrset.go
+++ b/dnsprovider/pkg/dnsprovider/providers/openstack/designate/rrset.go
@ -0,0 +1,47 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package designate
+
+import (
+	"k8s.io/kops/dnsprovider/pkg/dnsprovider"
+	"k8s.io/kops/dnsprovider/pkg/dnsprovider/rrstype"
+
+	"github.com/gophercloud/gophercloud/openstack/dns/v2/recordsets"
+)
+
+var _ dnsprovider.ResourceRecordSet = ResourceRecordSet{}
+
+type ResourceRecordSet struct {
+	impl   *recordsets.RecordSet
+	rrsets *ResourceRecordSets
+}
+
+func (rrset ResourceRecordSet) Name() string {
+	return rrset.impl.Name
+}
+
+func (rrset ResourceRecordSet) Rrdatas() []string {
+	return rrset.impl.Records
+}
+
+func (rrset ResourceRecordSet) Ttl() int64 {
+	return int64(rrset.impl.TTL)
+}
+
+func (rrset ResourceRecordSet) Type() rrstype.RrsType {
+	return rrstype.RrsType(rrset.impl.Type)
+}
--- a/dnsprovider/pkg/dnsprovider/providers/openstack/designate/rrsets.go
+++ b/dnsprovider/pkg/dnsprovider/providers/openstack/designate/rrsets.go
@ -0,0 +1,99 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package designate
+
+import (
+	"github.com/gophercloud/gophercloud/openstack/dns/v2/recordsets"
+	"github.com/gophercloud/gophercloud/pagination"
+	"k8s.io/kops/dnsprovider/pkg/dnsprovider"
+	"k8s.io/kops/dnsprovider/pkg/dnsprovider/rrstype"
+)
+
+var _ dnsprovider.ResourceRecordSets = ResourceRecordSets{}
+
+type ResourceRecordSets struct {
+	zone *Zone
+}
+
+func (rrsets ResourceRecordSets) List() ([]dnsprovider.ResourceRecordSet, error) {
+	var list []dnsprovider.ResourceRecordSet
+	err := recordsets.ListByZone(rrsets.zone.zones.iface.sc, rrsets.zone.impl.ID, nil).EachPage(func(page pagination.Page) (bool, error) {
+		rrs, err := recordsets.ExtractRecordSets(page)
+		if err != nil {
+			return false, err
+		}
+		for _, rr := range rrs {
+			list = append(list, &ResourceRecordSet{&rr, &rrsets})
+		}
+		return true, nil
+	})
+
+	return list, err
+}
+
+func (rrsets ResourceRecordSets) Get(name string) ([]dnsprovider.ResourceRecordSet, error) {
+	// This list implementation is very similar to the one implemented in
+	// the List() method above, but it restricts the retrieved list to
+	// the records whose name match the given `name`.
+	opts := &recordsets.ListOpts{
+		Name: name,
+	}
+
+	var list []dnsprovider.ResourceRecordSet
+	err := recordsets.ListByZone(rrsets.zone.zones.iface.sc, rrsets.zone.impl.ID, opts).EachPage(func(page pagination.Page) (bool, error) {
+		rrs, err := recordsets.ExtractRecordSets(page)
+		if err != nil {
+			return false, err
+		}
+		for _, rr := range rrs {
+			list = append(list, &ResourceRecordSet{&rr, &rrsets})
+		}
+		return true, nil
+	})
+	if err != nil {
+		return nil, err
+	}
+	return list, nil
+}
+
+func (rrsets ResourceRecordSets) StartChangeset() dnsprovider.ResourceRecordChangeset {
+	return &ResourceRecordChangeset{
+		zone:   rrsets.zone,
+		rrsets: &rrsets,
+	}
+}
+
+func (rrsets ResourceRecordSets) New(name string, rrdatas []string, ttl int64, rrstype rrstype.RrsType) dnsprovider.ResourceRecordSet {
+	rrs := &recordsets.RecordSet{
+		Name: name,
+		Type: string(rrstype),
+		TTL:  int(ttl),
+	}
+	for _, rrdata := range rrdatas {
+		rrs.Records = append(rrs.Records, string(rrdata))
+	}
+
+	return ResourceRecordSet{
+		rrs,
+		&rrsets,
+	}
+}
+
+// Zone returns the parent zone
+func (rrset ResourceRecordSets) Zone() dnsprovider.Zone {
+	return rrset.zone
+}
--- a/dnsprovider/pkg/dnsprovider/providers/openstack/designate/zone.go
+++ b/dnsprovider/pkg/dnsprovider/providers/openstack/designate/zone.go
@ -14,21 +14,29 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */

-package awsmodel
+package designate

-import "k8s.io/kops/upup/pkg/fi"
+import (
+	"k8s.io/kops/dnsprovider/pkg/dnsprovider"

-// s is a helper that builds a *string from a string value
-func s(v string) *string {
-	return fi.String(v)
+	"github.com/gophercloud/gophercloud/openstack/dns/v2/zones"
+)
+
+var _ dnsprovider.Zone = &Zone{}
+
+type Zone struct {
+	impl  zones.Zone
+	zones *Zones
 }

-// i64 is a helper that builds a *int64 from an int64 value
-func i64(v int64) *int64 {
-	return fi.Int64(v)
+func (z *Zone) Name() string {
+	return z.impl.Name
 }

-// i32 is a helper that builds a *int32 from an int32 value
-func i32(v int32) *int32 {
-	return fi.Int32(v)
+func (z *Zone) ID() string {
+	return z.impl.ID
+}
+
+func (z *Zone) ResourceRecordSets() (dnsprovider.ResourceRecordSets, bool) {
+	return &ResourceRecordSets{z}, true
 }
--- a/dnsprovider/pkg/dnsprovider/providers/openstack/designate/zones.go
+++ b/dnsprovider/pkg/dnsprovider/providers/openstack/designate/zones.go
@ -0,0 +1,72 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package designate
+
+import (
+	"github.com/gophercloud/gophercloud/openstack/dns/v2/zones"
+
+	"k8s.io/kops/dnsprovider/pkg/dnsprovider"
+)
+
+var _ dnsprovider.Zones = Zones{}
+
+type Zones struct {
+	iface *Interface
+}
+
+func (kzs Zones) List() ([]dnsprovider.Zone, error) {
+	var zoneList []dnsprovider.Zone
+
+	allPages, err := zones.List(kzs.iface.sc, nil).AllPages()
+	if err != nil {
+		return zoneList, err
+	}
+	zs, err := zones.ExtractZones(allPages)
+	if err != nil {
+		return zoneList, err
+	}
+	for _, z := range zs {
+		kz := &Zone{
+			impl:  z,
+			zones: &kzs,
+		}
+		zoneList = append(zoneList, kz)
+	}
+	return zoneList, nil
+}
+
+func (kzs Zones) Add(zone dnsprovider.Zone) (dnsprovider.Zone, error) {
+	opts := &zones.CreateOpts{Name: zone.Name()}
+	z, err := zones.Create(kzs.iface.sc, opts).Extract()
+	if err != nil {
+		return nil, err
+	}
+	return &Zone{
+		impl:  *z,
+		zones: &kzs,
+	}, nil
+}
+
+func (kzs Zones) Remove(zone dnsprovider.Zone) error {
+	_, err := zones.Delete(kzs.iface.sc, zone.ID()).Extract()
+	return err
+}
+
+func (kzs Zones) New(name string) (dnsprovider.Zone, error) {
+	zone := zones.Zone{Name: name}
+	return &Zone{zone, &kzs}, nil
+}
--- a/docs/addons.md
+++ b/docs/addons.md
@ -74,7 +74,7 @@ spec:
        }
      ]
 ```
-The masters will poll for changes changes in the bucket and keep the addons up to date.
+The masters will poll for changes in the bucket and keep the addons up to date.


 ### Dashboard
--- a/docs/changing_configuration.md
+++ b/docs/changing_configuration.md
@ -12,7 +12,3 @@

 * Apply the rolling-update `kops rolling-update cluster ${NAME} --yes`

-NOTE: rolling-update does not yet perform a real rolling update - it just shuts down machines in sequence with a delay;
- there will be downtime [Issue #37](https://github.com/kubernetes/kops/issues/37)
-We have implemented a new feature that does drain and validate nodes.  This feature is experimental, and you can use the new feature by setting `export KOPS_FEATURE_FLAGS="+DrainAndValidateRollingUpdate"`.
-
--- a/docs/cli/kops_create_cluster.md
+++ b/docs/cli/kops_create_cluster.md
@ -71,13 +71,14 @@ kops create cluster [flags]
      --authorization string             Authorization mode to use: AlwaysAllow or RBAC (default "RBAC")
      --bastion                          Pass the --bastion flag to enable a bastion instance group. Only applies to private topology.
      --channel string                   Channel for default versions and configuration to use (default "stable")
-      --cloud string                     Cloud provider to use - gce, aws, vsphere
+      --cloud string                     Cloud provider to use - gce, aws, vsphere, openstack
      --cloud-labels string              A list of KV pairs used to tag all instance groups in AWS (e.g. "Owner=John Doe,Team=Some Team").
      --disable-subnet-tags              Set to disable automatic subnet tagging
      --dns string                       DNS hosted zone to use: public|private. (default "Public")
      --dns-zone string                  DNS hosted zone to use (defaults to longest matching zone)
      --dry-run                          If true, only print the object that would be sent, without sending it. This flag can be used to create a cluster YAML or JSON manifest.
      --encrypt-etcd-storage             Generate key in aws kms and use it for encrypt etcd volumes
+      --etcd-storage-type string         The default storage type for etc members
  -h, --help                             help for cluster
      --image string                     Image to use for all instances.
      --kubernetes-version string        Version of kubernetes to run (defaults to version in channel)
--- a/docs/cli/kops_version.md
+++ b/docs/cli/kops_version.md
@ -22,7 +22,8 @@ kops version [flags]
 ### Options

 ```
-  -h, --help   help for version
+  -h, --help    help for version
+      --short   only print the main kops version, useful for scripting
 ```

 ### Options inherited from parent commands
--- a/docs/cluster_spec.md
+++ b/docs/cluster_spec.md
@ -159,6 +159,18 @@ spec:
    zone: us-east-1a
 ```

+In the case that you don't use NAT gateways or internet gateways, you can use the "External" flag for egress to force kops to ignore egress for the subnet. This can be useful when other tools are used to manage egress for the subnet such as virtual private gateways. Please note that your cluster may need to have access to the internet upon creation, so egress must be available upon initializing a cluster. This is intended for use when egress is managed external to kops, typically with an existing cluster.
+
+```
+spec:
+  subnets:
+  - cidr: 10.20.64.0/21
+    name: us-east-1a
+    egress: External
+    type: Private
+    zone: us-east-1a
+```
+
 #### publicIP
 The IP of an existing EIP that you would like to attach to the NAT gateway.

@ -190,7 +202,8 @@ spec:
    oidcGroupsClaim: user_roles
    oidcGroupsPrefix: "oidc:"
    oidcCAFile: /etc/kubernetes/ssl/kc-ca.pem
-
+    oidcRequiredClaim:
+    	- "key=value"
 ```

 #### audit logging
@ -381,6 +394,8 @@ Specifying KubeDNS will install kube-dns as the default service discovery.

 This will install [CoreDNS](https://coredns.io/) instead of kube-dns.

+**Note:** If you are upgrading to CoreDNS, kube-dns will be left in place and must be removed manually (you can scale the kube-dns and kube-dns-autoscaler deployments in the `kube-system` namespace to 0 as a starting point). The `kube-dns` Service itself should be left in place, as this retains the ClusterIP and eliminates the possibility of DNS outages in your cluster. If you would like to continue autoscaling, update the `kube-dns-autoscaler` Deployment container command for `--target=Deployment/kube-dns` to be `--target=Deployment/coredns`. 
+
 ### kubeControllerManager
 This block contains configurations for the `controller-manager`.

--- a/docs/development/adding_a_feature.md
+++ b/docs/development/adding_a_feature.md
@ -188,6 +188,7 @@ and then push nodeup using:
 export S3_BUCKET_NAME=<yourbucketname>
 make kops-install dev-upload UPLOAD_DEST=s3://${S3_BUCKET_NAME}

+KOPS_VERSION=`bazel run //cmd/kops version -- --short`
 export KOPS_BASE_URL=https://${S3_BUCKET_NAME}.s3.amazonaws.com/kops/${KOPS_VERSION}/
 kops create cluster <clustername> --zones us-east-1b
 ...
--- a/docs/development/testing.md
+++ b/docs/development/testing.md
@ -77,7 +77,7 @@ To use S3:
 export S3_BUCKET_NAME=<yourbucketname>
 make kops-install dev-upload UPLOAD_DEST=s3://${S3_BUCKET_NAME}

-KOPS_VERSION=`bazel run //cmd/kops version | cut -f2 -d ' '`
+KOPS_VERSION=`bazel run //cmd/kops version -- --short`
 export KOPS_BASE_URL=https://${S3_BUCKET_NAME}.s3.amazonaws.com/kops/${KOPS_VERSION}/
 ```

@ -86,7 +86,7 @@ To use GCS:
 export GCS_BUCKET_NAME=kops-dev-${USER}
 make kops-install dev-upload UPLOAD_DEST=gs://${GCS_BUCKET_NAME}

-KOPS_VERSION=`bazel run //cmd/kops version | cut -f2 -d ' '`
+KOPS_VERSION=`bazel run //cmd/kops version -- --short`
 export KOPS_BASE_URL=https://${GCS_BUCKET_NAME}.storage.googleapis.com/kops/${KOPS_VERSION}/
 ```

--- a/docs/etcd/manager.md
+++ b/docs/etcd/manager.md
@ -41,7 +41,7 @@ it won't change the configuration:

 ```bash
 # Enable etcd-manager
-kops set cluster cluster.spec.etcdClusters[*].manager.image=kopeio/etcd-manager:latest
+kops set cluster cluster.spec.etcdClusters[*].provider=Manager

 kops update cluster --yes
 kops rolling-update cluster --yes
--- a/docs/etcd3-migration.md
+++ b/docs/etcd3-migration.md
@ -0,0 +1,93 @@
+# Moving to etcd3
+
+kubernetes is moving from etcd2 to etcd3, which is an upgrade that involves
+downtime.  Technically there is no usable upgrade path from etcd2 to etcd3 that
+supports HA scenarios, but kops has enabled it using etcd-manager.
+
+Nonetheless, this remains a *higher-risk upgrade* than most other kubernetes
+upgrades - you are strongly recommended to plan accordingly: back up critical
+data, schedule the upgrade during a maintenance window, think about how you
+could recover onto a new cluster, try it on non-production clusters first.
+
+To minimize the pain of this migration, we are making some other supporting changes at the same time:
+
+* We enable TLS for both clients & peers with etcd3
+* Calico configurations move from talking to etcd directly to using CRDs
+  (talking to etcd is considered deprecated)
+
+This does introduce the risk that we are changing more at the same time, and we
+provide some mitigation steps for breaking up the upgrade, though most of these
+therefore involve multiple disruptive upgrades (e.g. etc2 -> etcd3 is
+disruptive, non-TLS to TLS is disruptive).
+
+## Default upgrades
+
+When upgrading to kubernetes 1.12 with kops 1.12, by default:
+
+* Calico will be updated to a configuration that uses CRDs
+* We will automatically start using etcd-manager
+* Using etcd-manager will default to etcd3
+* Using etcd3 will default to using TLS for all etcd communications
+
+### Non-calico users
+
+The upgrade is therefore disruptive to the masters.  The recommended procedure is to quickly roll your masters, and then do a normal roll of your nodes:
+
+```bash
+# Roll masters as quickly as possible
+kops rolling-update cluster --cloudonly --instance-group-roles master --master-interval=1s
+kops rolling-update cluster --cloudonly --instance-group-roles master --master-interval=1s --yes
+
+# Roll nodes normally
+kops rolling-update cluster
+kops rolling-update cluster --yes
+```
+
+
+### Calico users
+
+If you are using calico the switch to CRDs will effectively cause a network partition during the rolling-update.  Your application might tolerate this, but it probably won't.  We therefore recommend rolling your nodes as fast as possible also:
+
+
+```bash
+# Roll masters and nodes as quickly as possible
+kops rolling-update cluster --cloudonly --instance-group-roles master --master-interval=1s --node-interval=1s
+kops rolling-update cluster --cloudonly --instance-group-roles master --master-interval=1s --node-interval=1s --yes
+```
+
+## Gradual updates
+
+If you would like to upgrade more gradually, we offer the following strategies
+to spread out the disruption over several steps.  Note that they likely involve
+more disruption and are not necessarily lower risk.
+
+### Adopt etcd-manager with kops 1.11 / kubernetes 1.11
+
+To adopt etcd-manager before kops 1.12 & kubernetes 1.12, you
+
+```bash
+kops set cluster cluster.spec.etcdClusters[*].provider=manager
+```
+
+Then you can proceed to update to kops 1.12 & kubernetes 1.12, as this becomes the default.
+
+### Delay adopting etcd-manager with kops 1.12
+
+To delay adopting etcd-manager with kops 1.12, specify the provider as type `legacy`:
+
+```bash
+kops set cluster cluster.spec.etcdClusters[*].provider=legacy
+```
+
+To remove, `kops edit` your cluster and delete the `provider: Legacy` lines from both etcdCluster blocks.
+
+### Delay adopting etcd3 with kops 1.12
+
+To delay adopting etcd3 with kops 1.12, specify the etcd version as 2.2.1
+
+```bash
+kops set cluster cluster.spec.etcdClusters[*].version=2.2.1
+```
+
+To remove, `kops edit` your cluster and delete the `version: 2.2.1` lines from both etcdCluster blocks.
+
--- a/docs/gpu.md
+++ b/docs/gpu.md
@ -1,74 +1,5 @@
-# GPU support
+# GPU Support

-```
-kops create cluster gpu.example.com --zones us-east-1c --node-size p2.xlarge --node-count 1 --kubernetes-version 1.6.1
-```
+You can use [kops hooks](./cluster_spec.md#hooks) to install [Nvidia kubernetes device plugin](https://github.com/NVIDIA/k8s-device-plugin) and enable GPU support in cluster.

-(Note that the p2.xlarge instance type is not cheap, but no GPU instances are)
-
-You can use the experimental hooks feature to install the nvidia drivers:
-
-`> kops edit cluster gpu.example.com`
-```
-spec:
-...
-  hooks:
-  - execContainer:
-      image: kopeio/nvidia-bootstrap:1.6
-```
-
-(TODO: Only on instance groups, or have nvidia-bootstrap detect if GPUs are present..)
-
-In addition, you will likely want to set the `Accelerators=true` feature-flag to kubelet:
-
-`> kops edit cluster gpu.example.com`
-```
-spec:
-...
-  kubelet:
-    featureGates:
-      Accelerators: "true"
-```
-
-`> kops update cluster gpu.example.com --yes`
-
-
-Here is an example pod that runs tensorflow; note that it mounts libcuda from the host:
-
-(TODO: Is there some way to have a well-known volume or similar?)
-
-```
-apiVersion: v1
-kind: Pod
-metadata:
-  name: tf
-spec:
-  containers:
-  - image: gcr.io/tensorflow/tensorflow:1.0.1-gpu
-    imagePullPolicy: IfNotPresent
-    name: gpu
-    command:
-    - /bin/bash
-    - -c
-    - "cp -d /rootfs/usr/lib/x86_64-linux-gnu/libcuda.* /usr/lib/x86_64-linux-gnu/ && cp -d /rootfs/usr/lib/x86_64-linux-gnu/libnvidia* /usr/lib/x86_64-linux-gnu/ &&/run_jupyter.sh"
-    resources:
-      limits:
-        cpu: 2000m
-        alpha.kubernetes.io/nvidia-gpu: 1
-    volumeMounts:
-    - name: rootfs-usr-lib
-      mountPath: /rootfs/usr/lib
-  volumes:
-    - name: rootfs-usr-lib
-      hostPath:
-        path: /usr/lib
-```
-
-To use this particular tensorflow image, you should port-forward and get the URL from the log:
-
-```
-kubectl port-forward tf 8888 &
-kubectl logs tf
-```
-
-And browse to the URL printed
+See instructions in [kops hooks for nvidia-device-plugin](../hooks/nvidia-device-plugin).
--- a/docs/images.md
+++ b/docs/images.md
@ -76,20 +76,21 @@ The following steps are known:
 * You must accept the agreement at http://aws.amazon.com/marketplace/pp?sku=aw0evgkw8e5c1q413zgy5pjce
 * Specify the AMI by id (there are no tags): us-east-1: `ami-6d1c2007`
 * You may find images from the [CentOS AWS page](https://wiki.centos.org/Cloud/AWS)
-* You can also query by product-code: `aws ec2 describe-images --region=us-west-2 --filters Name=product-code,Values=aw0evgkw8e5c1q413zgy5pjce`
+* You can also query by product-code: `aws ec2 describe-images --region=us-west-2 --filters Name=product-code,Values=aw0evgkw8e5c1q413zgy5pjce Name=architecture,Values=x86_64 'Name=name,Values=CentOS*' --query 'sort_by(Images,&Name)'`

 Be aware of the following limitations:

 * CentOS 7.2 is the recommended minimum version
 * CentOS7 AMIs are running an older kernel than we prefer to run elsewhere

+
 ## RHEL7

 RHEL7 support is still experimental, but should work. Please report any issues.

 The following steps are known:

-* Redhat AMIs can be found using `aws ec2 describe-images --region=us-east-1 --owner=309956199498 --filters Name=virtualization-type,Values=hvm`
+* Redhat AMIs can be found using `aws ec2 describe-images --region=us-east-1 --owner=309956199498 --filters Name=virtualization-type,Values=hvm 'Name=name,Values=RHEL-*GA*' Name=architecture,Values=x86_64 --query 'sort_by(Images,&Name)'`
 * You can specify the name using the `redhat.com` owner alias, for example `redhat.com/RHEL-7.2_HVM-20161025-x86_64-1-Hourly2-GP2`

 Be aware of the following limitations:
--- a/docs/instance_groups.md
+++ b/docs/instance_groups.md
@ -14,10 +14,10 @@ By default, a cluster has:

 ## Instance Groups Disclaimer

-* When there is only one availability zone in a region (eu-central-1) and you would like to run multiple masters, 
-  you have to define multiple instance groups for each of those masters. (e.g. `master-eu-central-1-a` and 
+* When there is only one availability zone in a region (eu-central-1) and you would like to run multiple masters,
+  you have to define multiple instance groups for each of those masters. (e.g. `master-eu-central-1-a` and
  `master-eu-central-1-b` and so on...)
-* If instance groups are not defined correctly (particularly when there are an even number of master or multiple 
+* If instance groups are not defined correctly (particularly when there are an even number of master or multiple
  groups of masters into one availability zone in a single region), etcd servers will not start and master nodes will not check in. This is because etcd servers are configured per availability zone. DNS and Route53 would be the first places to check when these problems are happening.


@ -112,6 +112,103 @@ spec:
  rootVolumeIops: 200
 ```

+## Adding additional storage to the instance groups
+
+You can add additional storage _(note, presently confined to AWS)_ via the instancegroup specification.
+
+```YAML
+---
+apiVersion: kops/v1alpha2
+kind: InstanceGroup
+metadata:
+  labels:
+    kops.k8s.io/cluster: my-beloved-cluster
+  name: compute
+spec:
+  cloudLabels:
+    role: compute
+  image: coreos.com/CoreOS-stable-1855.4.0-hvm
+  machineType: m4.large
+  ...
+  volumes:
+  - device: /dev/xvdd
+    encrypted: true
+    size: 20
+    type: gp2
+```
+
+In AWS the above example shows how to add an additional 20gb EBS volume, which applies to each node within the instancegroup.
+
+## Automatically formatting and mounting the additional storage
+
+You can add additional storage via the above `volumes` collection though this only provisions the storage itself. Assuming you don't wish to handle the mechanics of formatting and mounting the device yourself _(perhaps via a hook)_ you can utilize the `volumeMounts` section of the instancegroup to handle this for you.
+
+```
+---
+apiVersion: kops/v1alpha2
+kind: InstanceGroup
+metadata:
+  labels:
+    kops.k8s.io/cluster: my-beloved-cluster
+  name: compute
+spec:
+  cloudLabels:
+    role: compute
+  image: coreos.com/CoreOS-stable-1855.4.0-hvm
+  machineType: m4.large
+  ...
+  volumeMounts:
+  - device: /dev/xvdd
+    filesystem: ext4
+    path: /var/lib/docker
+  volumes:
+  - device: /dev/xvdd
+    encrypted: true
+    size: 20
+    type: gp2
+```
+
+The above will provision the additional storage, format and mount the device into the node. Note this feature is purposely distinct from `volumes` so that it may be reused in areas such as ephemeral storage. Using a `c5d.large` instance as an example, which comes with a 50gb SSD drive; we can use the `volumeMounts` to mount this into `/var/lib/docker` for us.
+
+```YAML
+---
+apiVersion: kops/v1alpha2
+kind: InstanceGroup
+metadata:
+  labels:
+    kops.k8s.io/cluster: my-beloved-cluster
+  name: compute
+spec:
+  cloudLabels:
+    role: compute
+  image: coreos.com/CoreOS-stable-1855.4.0-hvm
+  machineType: c5d.large
+  ...
+  volumeMounts:
+  - device: /dev/nvme1n1
+    filesystem: ext4
+    path: /data
+  # -- mount the instance storage --
+  - device: /dev/nvme2n1
+    filesystem: ext4
+    path: /var/lib/docker
+  volumes:
+  - device: /dev/nvme1n1
+    encrypted: true
+    size: 20
+    type: gp2
+```
+
+For AWS you can find more information on device naming conventions [here](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/device_naming.html)
+
+```shell
+$ df -h | grep nvme[12]
+/dev/nvme1n1      20G   45M   20G   1% /data
+/dev/nvme2n1      46G  633M   45G   2% /var/lib/docker
+```
+
+> Note: at present its up to the user ensure the correct device names.
+
 ## Creating a new instance group

 Suppose you want to add a new group of nodes, perhaps with a different instance type.  You do this using `kops create ig <InstanceGroupName> --subnet <zone(s)>`. Currently the
@ -128,6 +225,65 @@ So the procedure is:
 * Apply: `kops update cluster <clustername> --yes`
 * (no instances need to be relaunched, so no rolling-update is needed)

+## Creating a instance group of mixed instances types (AWS Only)
+
+AWS permits the creation of EC2 Fleet Autoscaling Group using a [mixed instance policy](https://aws.amazon.com/blogs/aws/ec2-fleet-manage-thousands-of-on-demand-and-spot-instances-with-one-request/), allowing the users to build a target capacity and make up of on-demand and spot instances while offloading the allocation strategy to AWS. In order to create a mixed instance policy instancegroup.
+
+```YAML
+---
+apiVersion: kops/v1alpha2
+kind: InstanceGroup
+metadata:
+  labels:
+    kops.k8s.io/cluster: your.cluster.name
+  name: compute
+spec:
+  cloudLabels:
+    role: compute
+  image: coreos.com/CoreOS-stable-1911.4.0-hvm
+  machineType: m4.large
+  maxSize: 50
+  minSize: 10
+  # add the mixed instance here
+  mixedInstancesPolicy:
+    instances:
+    - m5.large
+    - m5.xlarge
+    - t2.medium
+    onDemandAboveBase: 5
+    spotInstancePools: 3
+```
+
+The mixed instance policy permits setting the following configurable below, but for more details please check against the AWS documentation.
+
+```Go
+// MixedInstancesPolicySpec defines the specification for an autoscaling backed by a ec2 fleet
+type MixedInstancesPolicySpec struct {
+	// Instances is a list of instance types which we are willing to run in the EC2 fleet
+	Instances []string `json:"instances,omitempty"`
+	// OnDemandAllocationStrategy indicates how to allocate instance types to fulfill On-Demand capacity
+	OnDemandAllocationStrategy *string `json:"onDemandAllocationStrategy,omitempty"`
+	// OnDemandBase is the minimum amount of the Auto Scaling group's capacity that must be
+	// fulfilled by On-Demand Instances. This base portion is provisioned first as your group scales.
+	OnDemandBase *int64 `json:"onDemandBase,omitempty"`
+	// OnDemandAboveBase controls the percentages of On-Demand Instances and Spot Instances for your
+	// additional capacity beyond OnDemandBase. The range is 0–100. The default value is 100. If you
+	// leave this parameter set to 100, the percentages are 100% for On-Demand Instances and 0% for
+	// Spot Instances.
+	OnDemandAboveBase *int64 `json:"onDemandAboveBase,omitempty"`
+	// SpotAllocationStrategy diversifies your Spot capacity across multiple instance types to
+	// find the best pricing. Higher Spot availability may result from a larger number of
+	// instance types to choose from https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/spot-fleet.html#spot-fleet-allocation-strategy
+	SpotAllocationStrategy *string `json:"spotAllocationStrategy,omitempty"`
+	// SpotInstancePools is the number of Spot pools to use to allocate your Spot capacity (defaults to 2)
+	// pools are determined from the different instance types in the Overrides array of LaunchTemplate
+	SpotInstancePools *int64 `json:"spotInstancePools,omitempty"`
+}
+```
+
+Note: as of writing this the kube cluster autoscaler does not support mixed instance groups, in the sense it will still scale groups up and down based on capacity but some of the simulations it does might be wrong as it's not aware of the instance type coming into the group.
+
+Note: when upgrading from a launchconfiguration to launchtemplate with mixed instance policy the launchconfiguration is left undeleted as has to be manually removed.

 ## Moving from one instance group spanning multiple AZs to one instance group per AZ

--- a/docs/networking.md
+++ b/docs/networking.md
@ -363,6 +363,11 @@ $ kops create cluster \
    "Resource": [
      "*"
    ]
+  },
+  {
+    "Effect": "Allow",
+    "Action": "ec2:CreateTags",
+    "Resource": "arn:aws:ec2:*:*:network-interface/*"
  }
 ```

--- a/docs/run_in_existing_vpc.md
+++ b/docs/run_in_existing_vpc.md
@ -1,7 +1,8 @@
 ## Running in a shared VPC

-When launching into a shared VPC, the VPC & the Internet Gateway will be reused. By default we create a new subnet per zone,
-and a new route table, but you can also use a shared subnet (see [below](#shared-subnets)).
+When launching into a shared VPC, the VPC & the Internet Gateway will be reused. If you are not using an internet gateway
+ or NAT gateway you can tell _kops_ to ignore egress. By default we create a new subnet per zone, and a new route table, 
+ but you can also use a shared subnet (see [below](#shared-subnets)).

 1. Use `kops create cluster` with the `--vpc` argument for your existing VPC:

@ -205,6 +206,33 @@ Please note:
 * kops won't create a route-table at all if we're not creating subnets.
 * In the example above the first subnet is using a shared NAT Gateway while the
  second one is using a shared NAT Instance
+  
+### Externally Managed Egress
+
+If you are using an unsupported egress configuration in your VPC, _kops_ can be told to ignore egress by using a configuration like:
+
+```yaml
+spec:
+  subnets:
+  - cidr: 10.20.64.0/21
+    name: us-east-1a
+    egress: External
+    type: Private
+    zone: us-east-1a
+  - cidr: 10.20.96.0/21
+    name: us-east-1b
+    egress: External
+    type: Private
+    zone: us-east-1a
+  - cidr: 10.20.32.0/21
+    name: utility-us-east-1a
+    type: Utility
+    zone: us-east-1a
+    egress: External
+```
+
+This tells _kops_ that egress is being managed externally. This is preferable when using virtual private gateways 
+(currently unsupported) or using other configurations to handle egress routing. 

 ### Proxy VPC Egress

--- a/docs/terraform.md
+++ b/docs/terraform.md
@ -6,7 +6,7 @@ The gist of it is that, instead of letting kops apply the changes, you tell kops

 The Terraform output should be reasonably stable (i.e. the text files should only change where something has actually changed - items should appear in the same order etc). This is extremely useful when using version control as you can diff your changes easily.

-Note that if you modify the Terraform files that kops spits out, it will override your changes with the configuration state defined by its own configs. In other terms, kops own state is the ultimate source of truth (as far as kops is concerned), and Terraform is a representation of that state for your convenience.
+Note that if you modify the Terraform files that kops spits out, it will override your changes with the configuration state defined by its own configs. In other terms, kops's own state is the ultimate source of truth (as far as kops is concerned), and Terraform is a representation of that state for your convenience.

 Ps: Steps below assume a recent version of Terraform. There's a workaround for a bug if you are using a Terraform version before 0.7 that you should be aware of (see [_"Caveats"_ section](#caveats)).

@ -111,7 +111,7 @@ Ps: You don't have to `kops delete cluster` if you just want to recreate from sc

 Changes made with `kops edit` (like enabling RBAC and / or feature gates) will result in changes to the launch configuration of your cluster nodes. After a `terraform apply`, they won't be applied right away since terraform will not launch new instances as part of that.

-In order to see your changes applied to the cluster you'll also need to run `kops rolling-update` after running `terraform apply`, this will ensure that all nodes changes have the desired settings configured with `kops edit`.
+To see your changes applied to the cluster you'll also need to run `kops rolling-update` after running `terraform apply`. This will ensure that all nodes' changes have the desired settings configured with `kops edit`.

 #### Workaround for Terraform <0.7

--- a/docs/tutorial/openstack.md
+++ b/docs/tutorial/openstack.md
@ -2,62 +2,65 @@

 **WARNING**: OpenStack support on kops is currently **alpha** meaning it is in the early stages of development and subject to change, please use with caution.

-## Create config file
-The config file contains the OpenStack credentials required to create a cluster. The config file has the following format:
-```ini
-[Default]
-identity=<OS_AUTH_URL>
-user=mk8s=<OS_USERNAME>
-password=<OS_PASSWORD>
-domain_name=<OS_USER_DOMAIN_NAME>
-tenant_id=<OS_PROJECT_ID>
+## Source your openstack RC
+The Cloud Config used by the kubernetes API server and kubelet will be constructed from environment variables in the openstack RC file. The openrc.sh file is usually located under `API access`.

-[Swift]
-service_type=object-store
-region=<OS_REGION_NAME>
+```bash
+source openstack.rc
+```

-[Cinder]
-service_type=volumev3
-region=<OS_REGION_NAME>
-
-[Neutron]
-service_type=network
-region=<OS_REGION_NAME>
-
-[Nova]
-service_type=compute
-region=<OS_REGION_NAME>
+If you are authenticating by username `OS_DOMAIN_NAME` or `OS_DOMAIN_ID` must manually be set.
+```bash
+export OS_DOMAIN_NAME=<USER_DOMAIN_NAME>
 ```

 ## Environment Variables

 It is important to set the following environment variables:
+
 ```bash
-export OPENSTACK_CREDENTIAL_FILE=<config-file> # where <config-file> is the path of the config file
 export KOPS_STATE_STORE=swift://<bucket-name> # where <bucket-name> is the name of the Swift container to use for kops state

-# TODO(lmb): Add a feature gate for OpenStack
 # this is required since OpenStack support is currently in alpha so it is feature gated
-# export KOPS_FEATURE_FLAGS="AlphaAllowOpenStack"
+export KOPS_FEATURE_FLAGS="AlphaAllowOpenStack"
 ```

+If your OpenStack does not have Swift you can use any other VFS store, such as S3.
+
 ## Creating a Cluster

 ```bash
+# to see your etcd storage type
+openstack volume type list
+
 # coreos (the default) + flannel overlay cluster in Default
-kops create cluster --cloud=openstack --name=my-cluster.k8s.local --networking=flannel --zones=Default --network-cidr=192.168.0.0/16
-# Not implemented yet...
-# kops update cluster my-cluster.k8s.local --yes
+kops create cluster \
+  --cloud openstack \
+  --name my-cluster.k8s.local \
+  --state ${KOPS_STATE_STORE} \
+  --zones nova \
+  --network-cidr 10.0.0.0/24 \
+  --image <imagename> \
+  --master-count=3 \
+  --node-count=1 \
+  --node-size <flavorname> \
+  --master-size <flavorname> \
+  --etcd-storage-type <volumetype> \
+  --api-loadbalancer-type public \
+  --topology private \
+  --bastion \
+  --ssh-public-key ~/.ssh/id_rsa.pub \
+  --networking weave \
+  --os-ext-net <externalnetworkname>
+
+# to update a cluster
+kops update cluster my-cluster.k8s.local --state ${KOPS_STATE_STORE} --yes

 # to delete a cluster
-# Not implemented yet...
-# kops delete cluster my-cluster.k8s.local --yes
+kops delete cluster my-cluster.k8s.local --yes
 ```

-## Features Still in Development
-
-kops for OpenStack currently does not support these features:
-* cluster create (servers, servergroups, load balancers, and DNS are not implemented yet)
-* cluster delete
-* state delete (fails due to unimplemented methods)
+#### Optional flags
+* `--os-kubelet-ignore-az=true` Nova and Cinder have different availability zones, more information [Kubernetes docs](https://kubernetes.io/docs/concepts/cluster-administration/cloud-providers/#block-storage)
+* `--os-octavia=true` If Octavia Loadbalancer api should be used instead of old lbaas v2 api.

--- a/docs/upgrade.md
+++ b/docs/upgrade.md
@ -32,7 +32,7 @@ Upgrade uses the latest Kubernetes version considered stable by kops, defined in

 * `kops edit cluster $NAME`
 * set the kubernetesVersion to the target version (e.g. `v1.3.5`)
-* NOTE: The next 3 steps must all be ran in the same directory
+* NOTE: The next 3 steps must all be run in the same directory
 * `kops update cluster $NAME --target=terraform`
 * `terraform plan`
 * `terraform apply`
--- a/hack/.packages
+++ b/hack/.packages
@ -26,6 +26,7 @@ k8s.io/kops/dnsprovider/pkg/dnsprovider/providers/google/clouddns
 k8s.io/kops/dnsprovider/pkg/dnsprovider/providers/google/clouddns/internal
 k8s.io/kops/dnsprovider/pkg/dnsprovider/providers/google/clouddns/internal/interfaces
 k8s.io/kops/dnsprovider/pkg/dnsprovider/providers/google/clouddns/internal/stubs
+k8s.io/kops/dnsprovider/pkg/dnsprovider/providers/openstack/designate
 k8s.io/kops/dnsprovider/pkg/dnsprovider/rrstype
 k8s.io/kops/dnsprovider/pkg/dnsprovider/tests
 k8s.io/kops/examples/kops-api-example
@ -143,6 +144,7 @@ k8s.io/kops/protokube/pkg/gossip/dns/hosts
 k8s.io/kops/protokube/pkg/gossip/dns/provider
 k8s.io/kops/protokube/pkg/gossip/gce
 k8s.io/kops/protokube/pkg/gossip/mesh
+k8s.io/kops/protokube/pkg/gossip/openstack
 k8s.io/kops/protokube/pkg/protokube
 k8s.io/kops/protokube/tests/integration/build_etcd_manifest
 k8s.io/kops/tests
@ -185,6 +187,7 @@ k8s.io/kops/upup/tools/generators/fitask
 k8s.io/kops/upup/tools/generators/pkg/codegen
 k8s.io/kops/util/pkg/exec
 k8s.io/kops/util/pkg/hashing
+k8s.io/kops/util/pkg/maps
 k8s.io/kops/util/pkg/reflectutils
 k8s.io/kops/util/pkg/slice
 k8s.io/kops/util/pkg/tables
--- a/hack/dev-build.sh
+++ b/hack/dev-build.sh
@ -100,8 +100,8 @@ make && UPLOAD_DEST=s3://${NODEUP_BUCKET} make upload
 # removing make test since it relies on the files in the bucket
 # && make test

-KOPS_CHANNEL=$(kops version | awk '{ print $2 }' |sed 's/\+/%2B/')
-KOPS_BASE_URL="http://${NODEUP_BUCKET}.s3.amazonaws.com/kops/${KOPS_CHANNEL}/"
+KOPS_VERSION=$(kops version --short)
+KOPS_BASE_URL="http://${NODEUP_BUCKET}.s3.amazonaws.com/kops/${KOPS_VERSION}/"

 echo "KOPS_BASE_URL=${KOPS_BASE_URL}"
 echo "NODEUP_URL=${KOPS_BASE_URL}linux/amd64/nodeup"
--- a/hack/machine_types/vpc_ip_resource_limit.go
+++ b/hack/machine_types/vpc_ip_resource_limit.go
@ -139,6 +139,7 @@ var InstanceENIsAvailable = map[string]int{
 	"p3.2xlarge":    4,
 	"p3.8xlarge":    8,
 	"p3.16xlarge":   8,
+	"p3dn.24xlarge": 15,
 	"r3.large":      3,
 	"r3.xlarge":     4,
 	"r3.2xlarge":    4,
@ -312,6 +313,7 @@ var InstanceIPsAvailable = map[string]int64{
 	"p3.2xlarge":    15,
 	"p3.8xlarge":    30,
 	"p3.16xlarge":   30,
+	"p3dn.24xlarge": 50,
 	"r3.large":      10,
 	"r3.xlarge":     15,
 	"r3.2xlarge":    15,
--- a/images/utils-builder/BUILD.bazel
+++ b/images/utils-builder/BUILD.bazel
@ -0,0 +1,38 @@
+# Download and build socat from source
+# Relies on system C++ compiler etc, so this is really only good for building from prow
+# Note: not yet used
+#genrule(
+#    name = "build-socat",
+#    srcs = ["build-socat.sh"],
+#    outs = ["socat"],
+#    cmd = "./$(location build-socat.sh) $@",
+#)
+
+genrule(
+    name = "extract_socat",
+    srcs = ["@utils_tar_gz//file"],
+    outs = ["socat"],
+    cmd = "tar -x -z --no-same-owner -f ./$(location @utils_tar_gz//file) utils/socat && mv utils/socat \"$@\"",
+    visibility = ["//visibility:public"],
+)
+
+genrule(
+    name = "extract_conntrack",
+    srcs = ["@utils_tar_gz//file"],
+    outs = ["conntrack"],
+    cmd = "tar -x -z --no-same-owner -f ./$(location @utils_tar_gz//file) utils/conntrack && mv utils/conntrack \"$@\"",
+    visibility = ["//visibility:public"],
+)
+
+load("@bazel_tools//tools/build_defs/pkg:pkg.bzl", "pkg_tar")
+
+pkg_tar(
+    name = "utils",
+    extension = "tar.gz",
+    package_dir = "utils",
+    srcs = [
+        "socat",
+        "conntrack",
+    ],
+    mode = "0755",
+)
--- a/node-authorizer/pkg/client/helper.go
+++ b/node-authorizer/pkg/client/helper.go
@ -35,7 +35,7 @@ import (
 	"k8s.io/kops/node-authorizer/pkg/authorizers/aws"
 	"k8s.io/kops/node-authorizer/pkg/server"

-	"k8s.io/client-go/tools/clientcmd/api/v1"
+	v1 "k8s.io/client-go/tools/clientcmd/api/v1"
 )

 // makeHTTPClient is responsible for making a http client
--- a/node-authorizer/pkg/server/admission.go
+++ b/node-authorizer/pkg/server/admission.go
@ -24,7 +24,7 @@ import (
 	"k8s.io/kops/node-authorizer/pkg/utils"

 	"go.uber.org/zap"
-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )

--- a/node-authorizer/pkg/server/types.go
+++ b/node-authorizer/pkg/server/types.go
@ -23,7 +23,7 @@ import (
 	"strings"
 	"time"

-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 )

 const (
--- a/nodeup/pkg/bootstrap/install.go
+++ b/nodeup/pkg/bootstrap/install.go
@ -140,6 +140,25 @@ func (i *Installation) buildSystemdJob() *nodetasks.Service {
 		buffer.WriteString("\" ")
 	}

+	// Pass in required credentials when using user-defined swift endpoint
+	if os.Getenv("OS_AUTH_URL") != "" {
+		for _, envVar := range []string{
+			"OS_TENANT_ID", "OS_TENANT_NAME", "OS_PROJECT_ID", "OS_PROJECT_NAME",
+			"OS_PROJECT_DOMAIN_NAME", "OS_PROJECT_DOMAIN_ID",
+			"OS_DOMAIN_NAME", "OS_DOMAIN_ID",
+			"OS_USERNAME",
+			"OS_PASSWORD",
+			"OS_AUTH_URL",
+			"OS_REGION_NAME",
+		} {
+			buffer.WriteString("\"")
+			buffer.WriteString(envVar)
+			buffer.WriteString("=")
+			buffer.WriteString(os.Getenv(envVar))
+			buffer.WriteString("\" ")
+		}
+	}
+
 	if os.Getenv("DIGITALOCEAN_ACCESS_TOKEN") != "" {
 		buffer.WriteString("\"DIGITALOCEAN_ACCESS_TOKEN=")
 		buffer.WriteString(os.Getenv("DIGITALOCEAN_ACCESS_TOKEN"))
--- a/nodeup/pkg/model/BUILD.bazel
+++ b/nodeup/pkg/model/BUILD.bazel
@ -10,6 +10,7 @@ go_library(
        "directories.go",
        "docker.go",
        "etcd.go",
+        "etcd_manager_tls.go",
        "etcd_tls.go",
        "file_assets.go",
        "firewall.go",
@ -30,6 +31,7 @@ go_library(
        "secrets.go",
        "sysctls.go",
        "update_service.go",
+        "volumes.go",
    ],
    importpath = "k8s.io/kops/nodeup/pkg/model",
    visibility = ["//visibility:public"],
@ -65,6 +67,7 @@ go_library(
        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
        "//vendor/k8s.io/apimachinery/pkg/util/intstr:go_default_library",
        "//vendor/k8s.io/apiserver/pkg/authentication/user:go_default_library",
+        "//vendor/k8s.io/kubernetes/pkg/util/mount:go_default_library",
    ],
 )

--- a/nodeup/pkg/model/cloudconfig.go
+++ b/nodeup/pkg/model/cloudconfig.go
@ -104,6 +104,61 @@ func (b *CloudConfigBuilder) Build(c *fi.ModelBuilderContext) error {
 		// We need this to support Kubernetes vSphere CloudProvider < v1.5.3
 		lines = append(lines, "[disk]")
 		lines = append(lines, "scsicontrollertype = pvscsi")
+	case "openstack":
+		osc := cloudConfig.Openstack
+		if osc == nil {
+			break
+		}
+		//Support mapping of older keystone API
+		tenantName := os.Getenv("OS_TENANT_NAME")
+		if tenantName == "" {
+			tenantName = os.Getenv("OS_PROJECT_NAME")
+		}
+		tenantID := os.Getenv("OS_TENANT_ID")
+		if tenantID == "" {
+			tenantID = os.Getenv("OS_PROJECT_ID")
+		}
+		lines = append(lines,
+			fmt.Sprintf("auth-url=\"%s\"", os.Getenv("OS_AUTH_URL")),
+			fmt.Sprintf("username=\"%s\"", os.Getenv("OS_USERNAME")),
+			fmt.Sprintf("password=\"%s\"", os.Getenv("OS_PASSWORD")),
+			fmt.Sprintf("region=\"%s\"", os.Getenv("OS_REGION_NAME")),
+			fmt.Sprintf("tenant-id=\"%s\"", tenantID),
+			fmt.Sprintf("tenant-name=\"%s\"", tenantName),
+			fmt.Sprintf("domain-name=\"%s\"", os.Getenv("OS_DOMAIN_NAME")),
+			fmt.Sprintf("domain-id=\"%s\"", os.Getenv("OS_DOMAIN_ID")),
+			"",
+		)
+
+		if lb := osc.Loadbalancer; lb != nil {
+			lines = append(lines,
+				"[LoadBalancer]",
+				fmt.Sprintf("floating-network-id=%s", fi.StringValue(lb.FloatingNetworkID)),
+				fmt.Sprintf("lb-method=%s", fi.StringValue(lb.Method)),
+				fmt.Sprintf("lb-provider=%s", fi.StringValue(lb.Provider)),
+				fmt.Sprintf("use-octavia=%t", fi.BoolValue(lb.UseOctavia)),
+				"",
+			)
+
+			if monitor := osc.Monitor; monitor != nil {
+				lines = append(lines,
+					"create-monitor=yes",
+					fmt.Sprintf("monitor-delay=%s", fi.StringValue(monitor.Delay)),
+					fmt.Sprintf("monitor-timeout=%s", fi.StringValue(monitor.Timeout)),
+					fmt.Sprintf("monitor-max-retries=%d", fi.IntValue(monitor.MaxRetries)),
+					"",
+				)
+			}
+		}
+
+		if bs := osc.BlockStorage; bs != nil {
+			//Block Storage Config
+			lines = append(lines,
+				"[BlockStorage]",
+				fmt.Sprintf("bs-version=%s", fi.StringValue(bs.Version)),
+				fmt.Sprintf("ignore-volume-az=%t", fi.BoolValue(bs.IgnoreAZ)),
+				"")
+		}
 	}

 	config := "[global]\n" + strings.Join(lines, "\n") + "\n"
--- a/nodeup/pkg/model/context.go
+++ b/nodeup/pkg/model/context.go
@ -27,9 +27,11 @@ import (
 	"k8s.io/kops/pkg/apis/kops/util"
 	"k8s.io/kops/pkg/apis/nodeup"
 	"k8s.io/kops/pkg/kubeconfig"
+	"k8s.io/kops/pkg/systemd"
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/upup/pkg/fi/nodeup/nodetasks"
 	"k8s.io/kops/util/pkg/vfs"
+	"k8s.io/kubernetes/pkg/util/mount"

 	"github.com/blang/semver"
 	"github.com/golang/glog"
@ -87,6 +89,58 @@ func (c *NodeupModelContext) SSLHostPaths() []string {
 	return paths
 }

+// VolumesServiceName is the name of the service which is downstream of any volume mounts
+func (c *NodeupModelContext) VolumesServiceName() string {
+	return c.EnsureSystemdSuffix("kops-volume-mounts")
+}
+
+// EnsureSystemdSuffix ensures that the hook name ends with a valid systemd unit file extension. If it
+// doesn't, it adds ".service" for backwards-compatibility with older versions of Kops
+func (c *NodeupModelContext) EnsureSystemdSuffix(name string) string {
+	if !systemd.UnitFileExtensionValid(name) {
+		name += ".service"
+	}
+
+	return name
+}
+
+// EnsureDirectory ensures the directory exists or creates it
+func (c *NodeupModelContext) EnsureDirectory(path string) error {
+	st, err := os.Stat(path)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return os.MkdirAll(path, 0755)
+		}
+
+		return err
+	}
+
+	if !st.IsDir() {
+		return fmt.Errorf("path: %s already exists but is not a directory", path)
+	}
+
+	return nil
+}
+
+// IsMounted checks if the device is mount
+func (c *NodeupModelContext) IsMounted(m mount.Interface, device, path string) (bool, error) {
+	list, err := m.List()
+	if err != nil {
+		return false, err
+	}
+
+	for _, x := range list {
+		if x.Device == device {
+			glog.V(3).Infof("Found mountpoint device: %s, path: %s, type: %s", x.Device, x.Path, x.Type)
+			if strings.TrimSuffix(x.Path, "/") == strings.TrimSuffix(path, "/") {
+				return true, nil
+			}
+		}
+	}
+
+	return false, nil
+}
+
 // PathSrvKubernetes returns the path for the kubernetes service files
 func (c *NodeupModelContext) PathSrvKubernetes() string {
 	switch c.Distribution {
@ -245,6 +299,16 @@ func (c *NodeupModelContext) UseEtcdTLS() bool {
 	return false
 }

+// UseVolumeMounts is used to check if we have volume mounts enabled as we need to
+// insert requires and afters in various places
+func (c *NodeupModelContext) UseVolumeMounts() bool {
+	if c.InstanceGroup != nil {
+		return len(c.InstanceGroup.Spec.VolumeMounts) > 0
+	}
+
+	return false
+}
+
 // UseEtcdTLSAuth checks the peer-auth is set in both cluster
 // @NOTE: in retrospect i think we should have consolidated the common config in the wrapper struct; it
 // feels weird we set things like version, tls etc per cluster since they both have to be the same.
@ -371,8 +435,13 @@ func (c *NodeupModelContext) BuildCertificateTask(ctx *fi.ModelBuilderContext, n
 		return err
 	}

+	p := filename
+	if !filepath.IsAbs(p) {
+		p = filepath.Join(c.PathSrvKubernetes(), filename)
+	}
+
 	ctx.AddTask(&nodetasks.File{
-		Path:     filepath.Join(c.PathSrvKubernetes(), filename),
+		Path:     p,
 		Contents: fi.NewStringResource(serialized),
 		Type:     nodetasks.FileType_File,
 		Mode:     s("0600"),
@ -397,8 +466,13 @@ func (c *NodeupModelContext) BuildPrivateKeyTask(ctx *fi.ModelBuilderContext, na
 		return err
 	}

+	p := filename
+	if !filepath.IsAbs(p) {
+		p = filepath.Join(c.PathSrvKubernetes(), filename)
+	}
+
 	ctx.AddTask(&nodetasks.File{
-		Path:     filepath.Join(c.PathSrvKubernetes(), filename),
+		Path:     p,
 		Contents: fi.NewStringResource(serialized),
 		Type:     nodetasks.FileType_File,
 		Mode:     s("0600"),
--- a/nodeup/pkg/model/convenience.go
+++ b/nodeup/pkg/model/convenience.go
@ -25,7 +25,7 @@ import (
 	"k8s.io/kops/upup/pkg/fi"

 	"github.com/golang/glog"
-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )

--- a/nodeup/pkg/model/docker.go
+++ b/nodeup/pkg/model/docker.go
@ -501,7 +501,31 @@ var dockerVersions = []dockerVersion{
 		//Recommends: aufs-tools, ca-certificates, cgroupfs-mount | cgroup-lite, git, xz-utils, apparmor
 	},

-	// 17.09.0 - Centos / Rhel7
+	// 18.06.2 - Xenial
+	{
+		DockerVersion: "18.06.2",
+		Name:          "docker-ce",
+		Distros:       []distros.Distribution{distros.DistributionXenial},
+		Architectures: []Architecture{ArchitectureAmd64},
+		Version:       "18.06.2~ce~3-0~ubuntu",
+		Source:        "https://download.docker.com/linux/ubuntu/dists/xenial/pool/stable/amd64/docker-ce_18.06.2~ce~3-0~ubuntu_amd64.deb",
+		Hash:          "03e5eaae9c84b144e1140d9b418e43fce0311892",
+		Dependencies:  []string{"bridge-utils", "iptables", "libapparmor1", "libltdl7", "perl"},
+		//Depends: iptables, init-system-helpers, lsb-base, libapparmor1, libc6, libdevmapper1.02.1, libltdl7, libeseccomp2, libsystemd0
+		//Recommends: aufs-tools, ca-certificates, cgroupfs-mount | cgroup-lite, git, xz-utils, apparmor
+	},
+
+	// 17.09.0 - Centos / Rhel7 (two packages)
+	{
+		DockerVersion: "17.09.0",
+		Name:          "container-selinux",
+		Distros:       []distros.Distribution{distros.DistributionRhel7, distros.DistributionCentos7},
+		Architectures: []Architecture{ArchitectureAmd64},
+		Version:       "2.68",
+		Source:        "http://mirror.centos.org/centos/7/extras/x86_64/Packages/container-selinux-2.68-1.el7.noarch.rpm",
+		Hash:          "d9f87f7f4f2e8e611f556d873a17b8c0c580fec0",
+		Dependencies:  []string{"policycoreutils-python"},
+	},
 	{
 		DockerVersion: "17.09.0",
 		Name:          "docker-ce",
@ -527,6 +551,20 @@ var dockerVersions = []dockerVersion{
 		//Recommends: aufs-tools, ca-certificates, cgroupfs-mount | cgroup-lite, git, xz-utils, apparmor
 	},

+	// 18.06.2 - Bionic
+	{
+		DockerVersion: "18.06.2",
+		Name:          "docker-ce",
+		Distros:       []distros.Distribution{distros.DistributionBionic},
+		Architectures: []Architecture{ArchitectureAmd64},
+		Version:       "18.06.2~ce~3-0~ubuntu",
+		Source:        "https://download.docker.com/linux/ubuntu/dists/bionic/pool/stable/amd64/docker-ce_18.06.2~ce~3-0~ubuntu_amd64.deb",
+		Hash:          "9607c67644e3e1ad9661267c99499004f2e84e05",
+		Dependencies:  []string{"bridge-utils", "iptables", "libapparmor1", "libltdl7", "perl"},
+		//Depends: iptables, init-system-helpers, lsb-base, libapparmor1, libc6, libdevmapper1.02.1, libltdl7, libeseccomp2, libsystemd0
+		//Recommends: aufs-tools, ca-certificates, cgroupfs-mount | cgroup-lite, git, xz-utils, apparmor
+	},
+
 	// 18.06.1 - Debian Stretch
 	{

@ -540,7 +578,42 @@ var dockerVersions = []dockerVersion{
 		Dependencies:  []string{"bridge-utils", "libapparmor1", "libltdl7", "perl"},
 	},

-	// 18.06.1 - CentOS / Rhel7
+	// 18.06.2 - Debian Stretch
+	{
+
+		DockerVersion: "18.06.2",
+		Name:          "docker-ce",
+		Distros:       []distros.Distribution{distros.DistributionDebian9},
+		Architectures: []Architecture{ArchitectureAmd64},
+		Version:       "18.06.2~ce~3-0~debian",
+		Source:        "https://download.docker.com/linux/debian/dists/stretch/pool/stable/amd64/docker-ce_18.06.2~ce~3-0~debian_amd64.deb",
+		Hash:          "aad1efd2c90725034e996c6a368ccc2bf41ca5b8",
+		Dependencies:  []string{"bridge-utils", "libapparmor1", "libltdl7", "perl"},
+	},
+
+	// 18.06.2 - Jessie
+	{
+		DockerVersion: "18.06.2",
+		Name:          "docker-ce",
+		Distros:       []distros.Distribution{distros.DistributionJessie},
+		Architectures: []Architecture{ArchitectureAmd64},
+		Version:       "ce_18.06.2~ce~3-0~debian",
+		Source:        "https://download.docker.com/linux/debian/dists/jessie/pool/stable/amd64/docker-ce_18.06.2~ce~3-0~debian_amd64.deb",
+		Hash:          "1a2500311230aff37aa81dd1292a88302fb0a2e1",
+		Dependencies:  []string{"bridge-utils", "libapparmor1", "libltdl7", "perl"},
+	},
+
+	// 18.06.1 - CentOS / Rhel7 (two packages)
+	{
+		DockerVersion: "18.06.1",
+		Name:          "container-selinux",
+		Distros:       []distros.Distribution{distros.DistributionRhel7, distros.DistributionCentos7},
+		Architectures: []Architecture{ArchitectureAmd64},
+		Version:       "2.68",
+		Source:        "http://mirror.centos.org/centos/7/extras/x86_64/Packages/container-selinux-2.68-1.el7.noarch.rpm",
+		Hash:          "d9f87f7f4f2e8e611f556d873a17b8c0c580fec0",
+		Dependencies:  []string{"policycoreutils-python"},
+	},
 	{
 		DockerVersion: "18.06.1",
 		Name:          "docker-ce",
@ -551,6 +624,28 @@ var dockerVersions = []dockerVersion{
 		Hash:          "0a1325e570c5e54111a79623c9fd0c0c714d3a11",
 		Dependencies:  []string{"libtool-ltdl", "libseccomp", "libcgroup"},
 	},
+
+	// 18.06.2 - CentOS / Rhel7 (two packages)
+	{
+		DockerVersion: "18.06.2",
+		Name:          "container-selinux",
+		Distros:       []distros.Distribution{distros.DistributionRhel7, distros.DistributionCentos7},
+		Architectures: []Architecture{ArchitectureAmd64},
+		Version:       "2.68",
+		Source:        "http://mirror.centos.org/centos/7/extras/x86_64/Packages/container-selinux-2.68-1.el7.noarch.rpm",
+		Hash:          "d9f87f7f4f2e8e611f556d873a17b8c0c580fec0",
+		Dependencies:  []string{"policycoreutils-python"},
+	},
+	{
+		DockerVersion: "18.06.2",
+		Name:          "docker-ce",
+		Distros:       []distros.Distribution{distros.DistributionRhel7, distros.DistributionCentos7},
+		Architectures: []Architecture{ArchitectureAmd64},
+		Version:       "18.06.2.ce",
+		Source:        "https://download.docker.com/linux/centos/7/x86_64/stable/Packages/docker-ce-18.06.2.ce-3.el7.x86_64.rpm",
+		Hash:          "456eb7c5bfb37fac342e9ade21b602c076c5b367",
+		Dependencies:  []string{"libtool-ltdl", "libseccomp", "libcgroup"},
+	},
 }

 func (d *dockerVersion) matches(arch Architecture, dockerVersion string, distro distros.Distribution) bool {
@ -580,6 +675,18 @@ func (d *dockerVersion) matches(arch Architecture, dockerVersion string, distro
 	return true
 }

+func (b *DockerBuilder) dockerVersion() string {
+	dockerVersion := ""
+	if b.Cluster.Spec.Docker != nil {
+		dockerVersion = fi.StringValue(b.Cluster.Spec.Docker.Version)
+	}
+	if dockerVersion == "" {
+		dockerVersion = DefaultDockerVersion
+		glog.Warningf("DockerVersion not specified; using default %q", dockerVersion)
+	}
+	return dockerVersion
+}
+
 // Build is responsible for configuring the docker daemon
 func (b *DockerBuilder) Build(c *fi.ModelBuilderContext) error {

@ -610,14 +717,7 @@ func (b *DockerBuilder) Build(c *fi.ModelBuilderContext) error {
 		c.AddTask(t)
 	}

-	dockerVersion := ""
-	if b.Cluster.Spec.Docker != nil {
-		dockerVersion = fi.StringValue(b.Cluster.Spec.Docker.Version)
-	}
-	if dockerVersion == "" {
-		dockerVersion = DefaultDockerVersion
-		glog.Warningf("DockerVersion not specified; using default %q", dockerVersion)
-	}
+	dockerVersion := b.dockerVersion()

 	// Add packages
 	{
@ -895,6 +995,21 @@ func (b *DockerBuilder) buildSysconfig(c *fi.ModelBuilderContext) error {
 		}
 	}

+	// RHEL-family / docker has a bug with 17.x where it fails to use overlay2 because it does a broken kernel check
+	if b.Distribution.IsRHELFamily() {
+		dockerVersion := b.dockerVersion()
+		if strings.HasPrefix(dockerVersion, "17.") {
+			storageOpts := strings.Join(docker.StorageOpts, ",")
+			if strings.Contains(storageOpts, "overlay2.override_kernel_check=1") {
+				// Already there
+			} else if !strings.Contains(storageOpts, "overlay2.override_kernel_check") {
+				docker.StorageOpts = append(docker.StorageOpts, "overlay2.override_kernel_check=1")
+			} else {
+				glog.Infof("detected image was RHEL and overlay2.override_kernel_check=1 was probably needed, but overlay2.override_kernel_check was already set (%q) so won't set", storageOpts)
+			}
+		}
+	}
+
 	flagsString, err := flagbuilder.BuildFlags(&docker)
 	if err != nil {
 		return fmt.Errorf("error building docker flags: %v", err)
--- a/nodeup/pkg/model/etcd_manager_tls.go
+++ b/nodeup/pkg/model/etcd_manager_tls.go
@ -0,0 +1,63 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package model
+
+import (
+	"github.com/golang/glog"
+	"k8s.io/kops/upup/pkg/fi"
+)
+
+// EtcdManagerTLSBuilder configures TLS support for etcd-manager
+type EtcdManagerTLSBuilder struct {
+	*NodeupModelContext
+}
+
+var _ fi.ModelBuilder = &EtcdManagerTLSBuilder{}
+
+// Build is responsible for TLS configuration for etcd-manager
+func (b *EtcdManagerTLSBuilder) Build(ctx *fi.ModelBuilderContext) error {
+	if !b.IsMaster {
+		return nil
+	}
+
+	for _, k := range []string{"main", "events"} {
+		d := "/etc/kubernetes/pki/etcd-manager-" + k
+
+		keys := make(map[string]string)
+		keys["etcd-manager-ca"] = "etcd-manager-ca-" + k
+
+		for fileName, keystoreName := range keys {
+			cert, err := b.KeyStore.FindCert(keystoreName)
+			if err != nil {
+				return err
+			}
+			if cert == nil {
+				glog.Warningf("keypair %q not found, won't configure", keystoreName)
+				continue
+			}
+
+			if err := b.BuildCertificateTask(ctx, keystoreName, d+"/"+fileName+".crt"); err != nil {
+				return err
+			}
+			if err := b.BuildPrivateKeyTask(ctx, keystoreName, d+"/"+fileName+".key"); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
--- a/nodeup/pkg/model/hooks.go
+++ b/nodeup/pkg/model/hooks.go
@ -72,7 +72,7 @@ func (h *HookBuilder) Build(c *fi.ModelBuilderContext) error {
 				enabled := false
 				managed := true
 				c.AddTask(&nodetasks.Service{
-					Name:        ensureSystemdSuffix(name),
+					Name:        h.EnsureSystemdSuffix(name),
 					ManageState: &managed,
 					Enabled:     &enabled,
 					Running:     &enabled,
@ -94,15 +94,6 @@ func (h *HookBuilder) Build(c *fi.ModelBuilderContext) error {
 	return nil
 }

-// ensureSystemdSuffix ensures that the hook name ends with a valid systemd unit file extension. If it
-// doesn't, it adds ".service" for backwards-compatibility with older versions of Kops
-func ensureSystemdSuffix(name string) string {
-	if !systemd.UnitFileExtensionValid(name) {
-		name += ".service"
-	}
-	return name
-}
-
 // buildSystemdService is responsible for generating the service
 func (h *HookBuilder) buildSystemdService(name string, hook *kops.HookSpec) (*nodetasks.Service, error) {
 	// perform some basic validation
@ -145,7 +136,7 @@ func (h *HookBuilder) buildSystemdService(name string, hook *kops.HookSpec) (*no
 	}

 	service := &nodetasks.Service{
-		Name:       ensureSystemdSuffix(name),
+		Name:       h.EnsureSystemdSuffix(name),
 		Definition: definition,
 	}

--- a/nodeup/pkg/model/kube_apiserver.go
+++ b/nodeup/pkg/model/kube_apiserver.go
@ -30,7 +30,7 @@ import (
 	"k8s.io/kops/upup/pkg/fi/nodeup/nodetasks"
 	"k8s.io/kops/util/pkg/exec"

-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
--- a/nodeup/pkg/model/kube_controller_manager.go
+++ b/nodeup/pkg/model/kube_controller_manager.go
@ -21,6 +21,7 @@ import (
 	"path/filepath"
 	"strings"

+	v1 "k8s.io/api/core/v1"
 	"k8s.io/kops/pkg/flagbuilder"
 	"k8s.io/kops/pkg/k8scodecs"
 	"k8s.io/kops/pkg/kubemanifest"
@ -28,7 +29,6 @@ import (
 	"k8s.io/kops/upup/pkg/fi/nodeup/nodetasks"
 	"k8s.io/kops/util/pkg/exec"

-	"k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
--- a/nodeup/pkg/model/kube_proxy.go
+++ b/nodeup/pkg/model/kube_proxy.go
@ -28,7 +28,7 @@ import (
 	"k8s.io/kops/util/pkg/exec"

 	"github.com/golang/glog"
-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
--- a/nodeup/pkg/model/kube_scheduler.go
+++ b/nodeup/pkg/model/kube_scheduler.go
@ -19,6 +19,7 @@ package model
 import (
 	"fmt"

+	v1 "k8s.io/api/core/v1"
 	"k8s.io/kops/pkg/flagbuilder"
 	"k8s.io/kops/pkg/k8scodecs"
 	"k8s.io/kops/pkg/kubemanifest"
@ -26,7 +27,6 @@ import (
 	"k8s.io/kops/upup/pkg/fi/nodeup/nodetasks"
 	"k8s.io/kops/util/pkg/exec"

-	"k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
--- a/nodeup/pkg/model/kubectl.go
+++ b/nodeup/pkg/model/kubectl.go
@ -106,6 +106,8 @@ func (b *KubectlBuilder) findKubeconfigUser() (*fi.User, *fi.Group, error) {
 	switch b.Distribution {
 	case distros.DistributionJessie, distros.DistributionDebian9:
 		users = []string{"admin", "root"}
+	case distros.DistributionCentos7:
+		users = []string{"centos"}
 	default:
 		glog.Warningf("Unknown distro; won't write kubeconfig to homedir %s", b.Distribution)
 		return nil, nil, nil
--- a/nodeup/pkg/model/kubelet.go
+++ b/nodeup/pkg/model/kubelet.go
@ -28,7 +28,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/golang/glog"

-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	"k8s.io/apiserver/pkg/authentication/user"

 	"k8s.io/kops/nodeup/pkg/distros"
@ -268,6 +268,8 @@ func (b *KubeletBuilder) buildSystemdService() *nodetasks.Service {
 	manifest.Set("Service", "StartLimitInterval", "0")
 	manifest.Set("Service", "KillMode", "process")
 	manifest.Set("Service", "User", "root")
+	manifest.Set("Service", "CPUAccounting", "true")
+	manifest.Set("Service", "MemoryAccounting", "true")
 	manifestString := manifest.Render()

 	glog.V(8).Infof("Built service manifest %q\n%s", "kubelet", manifestString)
--- a/nodeup/pkg/model/kubelet_test.go
+++ b/nodeup/pkg/model/kubelet_test.go
@ -191,6 +191,15 @@ func Test_RunKubeletBuilder(t *testing.T) {
 		context.AddTask(task)
 	}

+	{
+		task := builder.buildSystemdService()
+		if err != nil {
+			t.Fatalf("error from KubeletBuilder buildSystemdService: %v", err)
+			return
+		}
+		context.AddTask(task)
+	}
+
 	testutils.ValidateTasks(t, basedir, context)
 }

--- a/nodeup/pkg/model/protokube.go
+++ b/nodeup/pkg/model/protokube.go
@ -139,6 +139,8 @@ func (t *ProtokubeBuilder) buildSystemdService() (*nodetasks.Service, error) {
 	manifest := &systemd.Manifest{}
 	manifest.Set("Unit", "Description", "Kubernetes Protokube Service")
 	manifest.Set("Unit", "Documentation", "https://github.com/kubernetes/kops")
+
+	// @step: let need a dependency for any volumes to be mounted first
 	manifest.Set("Service", "ExecStartPre", t.ProtokubeImagePullCommand())
 	manifest.Set("Service", "ExecStart", protokubeCommand)
 	manifest.Set("Service", "Restart", "always")
@ -410,6 +412,24 @@ func (t *ProtokubeBuilder) ProtokubeEnvironmentVariables() string {
 		buffer.WriteString(" ")
 	}

+	if os.Getenv("OS_AUTH_URL") != "" {
+		for _, envVar := range []string{
+			"OS_TENANT_ID", "OS_TENANT_NAME", "OS_PROJECT_ID", "OS_PROJECT_NAME",
+			"OS_PROJECT_DOMAIN_NAME", "OS_PROJECT_DOMAIN_ID",
+			"OS_DOMAIN_NAME", "OS_DOMAIN_ID",
+			"OS_USERNAME",
+			"OS_PASSWORD",
+			"OS_AUTH_URL",
+			"OS_REGION_NAME",
+		} {
+			buffer.WriteString(" -e '")
+			buffer.WriteString(envVar)
+			buffer.WriteString("=")
+			buffer.WriteString(os.Getenv(envVar))
+			buffer.WriteString("'")
+		}
+	}
+
 	if kops.CloudProviderID(t.Cluster.Spec.CloudProvider) == kops.CloudProviderDO && os.Getenv("DIGITALOCEAN_ACCESS_TOKEN") != "" {
 		buffer.WriteString(" ")
 		buffer.WriteString("-e 'DIGITALOCEAN_ACCESS_TOKEN=")
--- a/nodeup/pkg/model/tests/kubelet/featuregates/tasks.yaml
+++ b/nodeup/pkg/model/tests/kubelet/featuregates/tasks.yaml
@ -7,3 +7,25 @@ contents: |
  HOME="/root"
 path: /etc/sysconfig/kubelet
 type: file
+---
+Name: kubelet.service
+definition: |
+  [Unit]
+  Description=Kubernetes Kubelet Server
+  Documentation=https://github.com/kubernetes/kubernetes
+  After=docker.service
+
+  [Service]
+  EnvironmentFile=/etc/sysconfig/kubelet
+  ExecStart=/usr/local/bin/kubelet "$DAEMON_ARGS"
+  Restart=always
+  RestartSec=2s
+  StartLimitInterval=0
+  KillMode=process
+  User=root
+  CPUAccounting=true
+  MemoryAccounting=true
+enabled: true
+manageState: true
+running: true
+smartRestart: true
--- a/nodeup/pkg/model/volumes.go
+++ b/nodeup/pkg/model/volumes.go
@ -0,0 +1,74 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package model
+
+import (
+	"fmt"
+
+	"k8s.io/kops/upup/pkg/fi"
+
+	"github.com/golang/glog"
+	"k8s.io/kubernetes/pkg/util/mount"
+)
+
+// VolumesBuilder maintains the volume mounting
+type VolumesBuilder struct {
+	*NodeupModelContext
+}
+
+var _ fi.ModelBuilder = &VolumesBuilder{}
+
+// Build is responsible for handling the mounting additional volumes onto the instance
+func (b *VolumesBuilder) Build(c *fi.ModelBuilderContext) error {
+	// @step: check if the instancegroup has any volumes to mount
+	if !b.UseVolumeMounts() {
+		glog.V(1).Info("Skipping the volume builder, no volumes defined for this instancegroup")
+
+		return nil
+	}
+
+	// @step: iterate the volume mounts and attempt to mount the devices
+	for _, x := range b.InstanceGroup.Spec.VolumeMounts {
+		// @check the directory exists, else create it
+		if err := b.EnsureDirectory(x.Path); err != nil {
+			return fmt.Errorf("Failed to ensure the directory: %s, error: %s", x.Path, err)
+		}
+
+		m := &mount.SafeFormatAndMount{
+			Exec:      mount.NewOsExec(),
+			Interface: mount.New(""),
+		}
+
+		// @check if the device is already mounted
+		if found, err := b.IsMounted(m, x.Device, x.Path); err != nil {
+			return fmt.Errorf("Failed to check if device: %s is mounted, error: %s", x.Device, err)
+		} else if found {
+			glog.V(3).Infof("Skipping device: %s, path: %s as already mounted", x.Device, x.Path)
+			continue
+		}
+
+		glog.Infof("Attempting to format and mount device: %s, path: %s", x.Device, x.Path)
+
+		if err := m.FormatAndMount(x.Device, x.Path, x.Filesystem, x.MountOptions); err != nil {
+			glog.Errorf("Failed to mount the device: %s on: %s, error: %s", x.Device, x.Path, err)
+
+			return err
+		}
+	}
+
+	return nil
+}
--- a/pkg/apis/kops/channel.go
+++ b/pkg/apis/kops/channel.go
@ -22,8 +22,8 @@ import (

 	"github.com/blang/semver"
 	"github.com/golang/glog"
-	"k8s.io/apimachinery/pkg/apis/meta/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/kops/pkg/apis/kops/util"
 	"k8s.io/kops/util/pkg/vfs"
 )
--- a/pkg/apis/kops/cluster.go
+++ b/pkg/apis/kops/cluster.go
@ -447,6 +447,9 @@ const (
 	SubnetTypeUtility SubnetType = "Utility"
 )

+// EgressExternal means that egress configuration is done externally (preconfigured)
+const EgressExternal = "External"
+
 // ClusterSubnetSpec defines a subnet
 type ClusterSubnetSpec struct {
 	// Name is the name of the subnet
--- a/pkg/apis/kops/componentconfig.go
+++ b/pkg/apis/kops/componentconfig.go
@ -179,6 +179,10 @@ type KubeletConfigSpec struct {
 	AuthenticationTokenWebhook *bool `json:"authenticationTokenWebhook,omitempty" flag:"authentication-token-webhook"`
 	// AuthenticationTokenWebhook sets the duration to cache responses from the webhook token authenticator. Default is 2m. (default 2m0s)
 	AuthenticationTokenWebhookCacheTTL *metav1.Duration `json:"authenticationTokenWebhookCacheTtl,omitempty" flag:"authentication-token-webhook-cache-ttl"`
+	// CPUCFSQuota enables CPU CFS quota enforcement for containers that specify CPU limits
+	CPUCFSQuota *bool `json:"cpuCFSQuota,omitempty" flag:"cpu-cfs-quota"`
+	// CPUCFSQuotaPeriod sets CPU CFS quota period value, cpu.cfs_period_us, defaults to Linux Kernel default
+	CPUCFSQuotaPeriod *metav1.Duration `json:"cpuCFSQuotaPeriod,omitempty" flag:"cpu-cfs-quota-period"`
 }

 // KubeProxyConfig defines the configuration for a proxy
@ -314,6 +318,10 @@ type KubeAPIServerConfig struct {
 	// OIDCClientID is the client ID for the OpenID Connect client, must be set
 	// if oidc-issuer-url is set.
 	OIDCClientID *string `json:"oidcClientID,omitempty" flag:"oidc-client-id"`
+	// A key=value pair that describes a required claim in the ID Token.
+	// If set, the claim is verified to be present in the ID Token with a matching value.
+	// Repeat this flag to specify multiple claims.
+	OIDCRequiredClaim []string `json:"oidcRequiredClaim,omitempty" flag:"oidc-required-claim,repeat"`
 	// OIDCCAFile if set, the OpenID server's certificate will be verified by one
 	// of the authorities in the oidc-ca-file
 	OIDCCAFile *string `json:"oidcCAFile,omitempty" flag:"oidc-ca-file"`
@ -493,6 +501,41 @@ type LeaderElectionConfiguration struct {
 	LeaderElect *bool `json:"leaderElect,omitempty" flag:"leader-elect"`
 }

+// OpenstackLoadbalancerConfig defines the config for a neutron loadbalancer
+type OpenstackLoadbalancerConfig struct {
+	Method            *string `json:"method,omitempty"`
+	Provider          *string `json:"provider,omitempty"`
+	UseOctavia        *bool   `json:"useOctavia,omitempty"`
+	FloatingNetwork   *string `json:"floatingNetwork,omitempty"`
+	FloatingNetworkID *string `json:"floatingNetworkID,omitempty"`
+	SubnetID          *string `json:"subnetID,omitempty"`
+}
+
+type OpenstackBlockStorageConfig struct {
+	Version  *string `json:"bs-version,omitempty"`
+	IgnoreAZ *bool   `json:"ignore-volume-az,omitempty"`
+}
+
+// OpenstackMonitor defines the config for a health monitor
+type OpenstackMonitor struct {
+	Delay      *string `json:"delay,omitempty"`
+	Timeout    *string `json:"timeout,omitempty"`
+	MaxRetries *int    `json:"maxRetries,omitempty"`
+}
+
+// OpenstackRouter defines the config for a router
+type OpenstackRouter struct {
+	ExternalNetwork *string `json:"externalNetwork,omitempty"`
+}
+
+// OpenstackConfiguration defines cloud config elements for the openstack cloud provider
+type OpenstackConfiguration struct {
+	Loadbalancer *OpenstackLoadbalancerConfig `json:"loadbalancer,omitempty"`
+	Monitor      *OpenstackMonitor            `json:"monitor,omitempty"`
+	Router       *OpenstackRouter             `json:"router,omitempty"`
+	BlockStorage *OpenstackBlockStorageConfig `json:"blockStorage,omitempty"`
+}
+
 // CloudConfiguration defines the cloud provider configuration
 type CloudConfiguration struct {
 	// GCE cloud-config options
@ -513,6 +556,8 @@ type CloudConfiguration struct {
 	// Spotinst cloud-config specs
 	SpotinstProduct     *string `json:"spotinstProduct,omitempty"`
 	SpotinstOrientation *string `json:"spotinstOrientation,omitempty"`
+	// Openstack cloud-config options
+	Openstack *OpenstackConfiguration `json:"openstack,omitempty"`
 }

 // HasAdmissionController checks if a specific admission controller is enabled
--- a/pkg/apis/kops/instancegroup.go
+++ b/pkg/apis/kops/instancegroup.go
@ -23,13 +23,14 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )

-const LabelClusterName = "kops.k8s.io/cluster"
-
-// NodeLabelInstanceGroup is a node label set to the name of the instance group
-const NodeLabelInstanceGroup = "kops.k8s.io/instancegroup"
-
-// Deprecated - use the new labels & taints node-role.kubernetes.io/master and node-role.kubernetes.io/node
-const TaintNoScheduleMaster15 = "dedicated=master:NoSchedule"
+const (
+	// LabelClusterName is a cluster label cloud tag
+	LabelClusterName = "kops.k8s.io/cluster"
+	// NodeLabelInstanceGroup is a node label set to the name of the instance group
+	NodeLabelInstanceGroup = "kops.k8s.io/instancegroup"
+	// Deprecated - use the new labels & taints node-role.kubernetes.io/master and node-role.kubernetes.io/node
+	TaintNoScheduleMaster15 = "dedicated=master:NoSchedule"
+)

 // +genclient
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
@ -44,6 +45,7 @@ type InstanceGroup struct {

 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

+// InstanceGroupList is a list of instance groups
 type InstanceGroupList struct {
 	metav1.TypeMeta `json:",inline"`
 	metav1.ListMeta `json:"metadata,omitempty"`
@ -55,8 +57,11 @@ type InstanceGroupList struct {
 type InstanceGroupRole string

 const (
-	InstanceGroupRoleMaster  InstanceGroupRole = "Master"
-	InstanceGroupRoleNode    InstanceGroupRole = "Node"
+	// InstanceGroupRoleMaster is a master role
+	InstanceGroupRoleMaster InstanceGroupRole = "Master"
+	// InstanceGroupRoleNode is a node role
+	InstanceGroupRoleNode InstanceGroupRole = "Node"
+	// InstanceGroupRoleBastion is a bastion role
 	InstanceGroupRoleBastion InstanceGroupRole = "Bastion"
 )

@ -67,6 +72,20 @@ var AllInstanceGroupRoles = []InstanceGroupRole{
 	InstanceGroupRoleBastion,
 }

+const (
+	// BtfsFilesystem indicates a btfs filesystem
+	BtfsFilesystem = "btfs"
+	// Ext4Filesystem indicates a ext3 filesystem
+	Ext4Filesystem = "ext4"
+	// XFSFilesystem indicates a xfs filesystem
+	XFSFilesystem = "xfs"
+)
+
+var (
+	// SupportedFilesystems is a list of supported filesystems to format as
+	SupportedFilesystems = []string{BtfsFilesystem, Ext4Filesystem, XFSFilesystem}
+)
+
 // InstanceGroupSpec is the specification for a instanceGroup
 type InstanceGroupSpec struct {
 	// Type determines the role of instances in this group: masters or nodes
@ -87,6 +106,10 @@ type InstanceGroupSpec struct {
 	RootVolumeIops *int32 `json:"rootVolumeIops,omitempty"`
 	// RootVolumeOptimization enables EBS optimization for an instance
 	RootVolumeOptimization *bool `json:"rootVolumeOptimization,omitempty"`
+	// Volumes is a collection of additional volumes to create for instances within this InstanceGroup
+	Volumes []*VolumeSpec `json:"volumes,omitempty"`
+	// VolumeMounts a collection of volume mounts
+	VolumeMounts []*VolumeMountSpec `json:"volumeMounts,omitempty"`
 	// Subnets is the names of the Subnets (as specified in the Cluster) where machines in this instance group should be placed
 	Subnets []string `json:"subnets,omitempty"`
 	// Zones is the names of the Zones where machines in this instance group should be placed
@ -106,13 +129,14 @@ type InstanceGroupSpec struct {
 	NodeLabels map[string]string `json:"nodeLabels,omitempty"`
 	// FileAssets is a collection of file assets for this instance group
 	FileAssets []FileAssetSpec `json:"fileAssets,omitempty"`
-	// Describes the tenancy of the instance group. Can be either default or dedicated.
-	// Currently only applies to AWS.
+	// Describes the tenancy of the instance group. Can be either default or dedicated. Currently only applies to AWS.
 	Tenancy string `json:"tenancy,omitempty"`
 	// Kubelet overrides kubelet config from the ClusterSpec
 	Kubelet *KubeletConfigSpec `json:"kubelet,omitempty"`
 	// Taints indicates the kubernetes taints for nodes in this group
 	Taints []string `json:"taints,omitempty"`
+	// MixedInstancesPolicy defined a optional backing of an AWS ASG by a EC2 Fleet (AWS Only)
+	MixedInstancesPolicy *MixedInstancesPolicySpec `json:"mixedInstancesPolicy,omitempty"`
 	// AdditionalUserData is any additional user-data to be passed to the host
 	AdditionalUserData []UserData `json:"additionalUserData,omitempty"`
 	// SuspendProcesses disables the listed Scaling Policies
@ -127,6 +151,39 @@ type InstanceGroupSpec struct {
 	SecurityGroupOverride *string `json:"securityGroupOverride,omitempty"`
 }

+const (
+	// SpotAllocationStrategyLowestPrices indicates a lowest price strategy
+	SpotAllocationStrategyLowestPrices = "LowestPrice"
+	// SpotAllocationStrategyDiversified indicates a diversified strategy
+	SpotAllocationStrategyDiversified = "Diversified"
+)
+
+// SpotAllocationStrategies is a collection of supported strategies
+var SpotAllocationStrategies = []string{SpotAllocationStrategyLowestPrices, SpotAllocationStrategyDiversified}
+
+// MixedInstancesPolicySpec defines the specification for an autoscaling backed by a ec2 fleet
+type MixedInstancesPolicySpec struct {
+	// Instances is a list of instance types which we are willing to run in the EC2 fleet
+	Instances []string `json:"instances,omitempty"`
+	// OnDemandAllocationStrategy indicates how to allocate instance types to fulfill On-Demand capacity
+	OnDemandAllocationStrategy *string `json:"onDemandAllocationStrategy,omitempty"`
+	// OnDemandBase is the minimum amount of the Auto Scaling group's capacity that must be
+	// fulfilled by On-Demand Instances. This base portion is provisioned first as your group scales.
+	OnDemandBase *int64 `json:"onDemandBase,omitempty"`
+	// OnDemandAboveBase controls the percentages of On-Demand Instances and Spot Instances for your
+	// additional capacity beyond OnDemandBase. The range is 0–100. The default value is 100. If you
+	// leave this parameter set to 100, the percentages are 100% for On-Demand Instances and 0% for
+	// Spot Instances.
+	OnDemandAboveBase *int64 `json:"onDemandAboveBase,omitempty"`
+	// SpotAllocationStrategy diversifies your Spot capacity across multiple instance types to
+	// find the best pricing. Higher Spot availability may result from a larger number of
+	// instance types to choose from.
+	SpotAllocationStrategy *string `json:"spotAllocationStrategy,omitempty"`
+	// SpotInstancePools is the number of Spot pools to use to allocate your Spot capacity (defaults to 2)
+	// pools are determined from the different instance types in the Overrides array of LaunchTemplate
+	SpotInstancePools *int64 `json:"spotInstancePools,omitempty"`
+}
+
 // UserData defines a user-data section
 type UserData struct {
 	// Name is the name of the user-data
@ -137,6 +194,34 @@ type UserData struct {
 	Content string `json:"content,omitempty"`
 }

+// VolumeSpec defined the spec for an additional volume attached to the instance group
+type VolumeSpec struct {
+	// Device is an optional device name of the block device
+	Device string `json:"device,omitempty"`
+	// Encrypted indicates you want to encrypt the volume
+	Encrypted *bool `json:"encrypted,omitempty"`
+	// Iops is the provision iops for this iops (think io1 in aws)
+	Iops *int64 `json:"iops,omitempty"`
+	// Size is the size of the volume in GB
+	Size int64 `json:"size,omitempty"`
+	// Type is the type of volume to create and is cloud specific
+	Type string `json:"type,omitempty"`
+}
+
+// VolumeMountSpec defines the specification for mounting a device
+type VolumeMountSpec struct {
+	// Device is the device name to provision and mount
+	Device string `json:"device,omitempty"`
+	// Filesystem is the filesystem to mount
+	Filesystem string `json:"filesystem,omitempty"`
+	// FormatOptions is a collection of options passed when formatting the device
+	FormatOptions []string `json:"formatOptions,omitempty"`
+	// MountOptions is a collection of mount options - @TODO need to be added
+	MountOptions []string `json:"mountOptions,omitempty"`
+	// Path is the location to mount the device
+	Path string `json:"path,omitempty"`
+}
+
 // IAMProfileSpec is the AWS IAM Profile to attach to instances in this instance
 // group. Specify the ARN for the IAM instance profile (AWS only).
 type IAMProfileSpec struct {
--- a/pkg/apis/kops/networking.go
+++ b/pkg/apis/kops/networking.go
@ -102,6 +102,9 @@ type CanalNetworkingSpec struct {
 	// for traffic between pod to host after calico rules have been processed.
 	// Default: ACCEPT (other options: DROP, RETURN)
 	DefaultEndpointToHostAction string `json:"defaultEndpointToHostAction,omitempty"`
+	// DisableFlannelForwardRules configures Flannel to NOT add the
+	// default ACCEPT traffic rules to the iptables FORWARD chain
+	DisableFlannelForwardRules bool `json:"disableFlannelForwardRules,omitempty"`
 	// LogSeveritySys the severity to set for logs which are sent to syslog
 	// Default: INFO (other options: DEBUG, WARNING, ERROR, CRITICAL, NONE)
 	LogSeveritySys string `json:"logSeveritySys,omitempty"`
--- a/pkg/apis/kops/util/labels.go
+++ b/pkg/apis/kops/util/labels.go
@ -19,7 +19,7 @@ package util
 import (
 	"strings"

-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 )

 func GetNodeRole(node *v1.Node) string {
--- a/pkg/apis/kops/v1alpha1/componentconfig.go
+++ b/pkg/apis/kops/v1alpha1/componentconfig.go
@ -179,6 +179,10 @@ type KubeletConfigSpec struct {
 	AuthenticationTokenWebhook *bool `json:"authenticationTokenWebhook,omitempty" flag:"authentication-token-webhook"`
 	// AuthenticationTokenWebhook sets the duration to cache responses from the webhook token authenticator. Default is 2m. (default 2m0s)
 	AuthenticationTokenWebhookCacheTTL *metav1.Duration `json:"authenticationTokenWebhookCacheTtl,omitempty" flag:"authentication-token-webhook-cache-ttl"`
+	// CPUCFSQuota enables CPU CFS quota enforcement for containers that specify CPU limits
+	CPUCFSQuota *bool `json:"cpuCFSQuota,omitempty" flag:"cpu-cfs-quota"`
+	// CPUCFSQuotaPeriod sets CPU CFS quota period value, cpu.cfs_period_us, defaults to Linux Kernel default
+	CPUCFSQuotaPeriod *metav1.Duration `json:"cpuCFSQuotaPeriod,omitempty" flag:"cpu-cfs-quota-period"`
 }

 // KubeProxyConfig defines the configuration for a proxy
@ -314,6 +318,10 @@ type KubeAPIServerConfig struct {
 	// OIDCClientID is the client ID for the OpenID Connect client, must be set
 	// if oidc-issuer-url is set.
 	OIDCClientID *string `json:"oidcClientID,omitempty" flag:"oidc-client-id"`
+	// A key=value pair that describes a required claim in the ID Token.
+	// If set, the claim is verified to be present in the ID Token with a matching value.
+	// Repeat this flag to specify multiple claims.
+	OIDCRequiredClaim []string `json:"oidcRequiredClaim,omitempty" flag:"oidc-required-claim,repeat"`
 	// OIDCCAFile if set, the OpenID server's certificate will be verified by one
 	// of the authorities in the oidc-ca-file
 	OIDCCAFile *string `json:"oidcCAFile,omitempty" flag:"oidc-ca-file"`
@ -493,6 +501,41 @@ type LeaderElectionConfiguration struct {
 	LeaderElect *bool `json:"leaderElect,omitempty" flag:"leader-elect"`
 }

+// OpenstackLoadbalancerConfig defines the config for a neutron loadbalancer
+type OpenstackLoadbalancerConfig struct {
+	Method            *string `json:"method,omitempty"`
+	Provider          *string `json:"provider,omitempty"`
+	UseOctavia        *bool   `json:"useOctavia,omitempty"`
+	FloatingNetwork   *string `json:"floatingNetwork,omitempty"`
+	FloatingNetworkID *string `json:"floatingNetworkID,omitempty"`
+	SubnetID          *string `json:"subnetID,omitempty"`
+}
+
+type OpenstackBlockStorageConfig struct {
+	Version  *string `json:"bs-version,omitempty"`
+	IgnoreAZ *bool   `json:"ignore-volume-az,omitempty"`
+}
+
+// OpenstackMonitor defines the config for a health monitor
+type OpenstackMonitor struct {
+	Delay      *string `json:"delay,omitempty"`
+	Timeout    *string `json:"timeout,omitempty"`
+	MaxRetries *int    `json:"maxRetries,omitempty"`
+}
+
+// OpenstackRouter defines the config for a router
+type OpenstackRouter struct {
+	ExternalNetwork *string `json:"externalNetwork,omitempty"`
+}
+
+// OpenstackConfiguration defines cloud config elements for the openstack cloud provider
+type OpenstackConfiguration struct {
+	Loadbalancer *OpenstackLoadbalancerConfig `json:"loadbalancer,omitempty"`
+	Monitor      *OpenstackMonitor            `json:"monitor,omitempty"`
+	Router       *OpenstackRouter             `json:"router,omitempty"`
+	BlockStorage *OpenstackBlockStorageConfig `json:"blockStorage,omitempty"`
+}
+
 // CloudConfiguration defines the cloud provider configuration
 type CloudConfiguration struct {
 	// GCE cloud-config options
@ -513,6 +556,8 @@ type CloudConfiguration struct {
 	// Spotinst cloud-config specs
 	SpotinstProduct     *string `json:"spotinstProduct,omitempty"`
 	SpotinstOrientation *string `json:"spotinstOrientation,omitempty"`
+	// Openstack cloud-config options
+	Openstack *OpenstackConfiguration `json:"openstack,omitempty"`
 }

 // HasAdmissionController checks if a specific admission controller is enabled
--- a/pkg/apis/kops/v1alpha1/instancegroup.go
+++ b/pkg/apis/kops/v1alpha1/instancegroup.go
@ -33,6 +33,7 @@ type InstanceGroup struct {

 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

+// InstanceGroupList is a list of instance groups
 type InstanceGroupList struct {
 	metav1.TypeMeta `json:",inline"`
 	metav1.ListMeta `json:"metadata,omitempty"`
@ -44,8 +45,33 @@ type InstanceGroupList struct {
 type InstanceGroupRole string

 const (
+	// InstanceGroupRoleMaster is a master role
 	InstanceGroupRoleMaster InstanceGroupRole = "Master"
-	InstanceGroupRoleNode   InstanceGroupRole = "Node"
+	// InstanceGroupRoleNode is a node role
+	InstanceGroupRoleNode InstanceGroupRole = "Node"
+	// InstanceGroupRoleBastion is a bastion role
+	InstanceGroupRoleBastion InstanceGroupRole = "Bastion"
+)
+
+// AllInstanceGroupRoles is a slice of all valid InstanceGroupRole values
+var AllInstanceGroupRoles = []InstanceGroupRole{
+	InstanceGroupRoleNode,
+	InstanceGroupRoleMaster,
+	InstanceGroupRoleBastion,
+}
+
+const (
+	// BtfsFilesystem indicates a btfs filesystem
+	BtfsFilesystem = "btfs"
+	// Ext4Filesystem indicates a ext3 filesystem
+	Ext4Filesystem = "ext4"
+	// XFSFilesystem indicates a xfs filesystem
+	XFSFilesystem = "xfs"
+)
+
+var (
+	// SupportedFilesystems is a list of supported filesystems to format as
+	SupportedFilesystems = []string{BtfsFilesystem, Ext4Filesystem, XFSFilesystem}
 )

 // InstanceGroupSpec is the specification for a instanceGroup
@ -68,6 +94,10 @@ type InstanceGroupSpec struct {
 	RootVolumeIops *int32 `json:"rootVolumeIops,omitempty"`
 	// RootVolumeOptimization enables EBS optimization for an instance
 	RootVolumeOptimization *bool `json:"rootVolumeOptimization,omitempty"`
+	// Volumes is a collection of additional volumes to create for instances within this InstanceGroup
+	Volumes []*VolumeSpec `json:"volumes,omitempty"`
+	// VolumeMounts a collection of volume mounts
+	VolumeMounts []*VolumeMountSpec `json:"volumeMounts,omitempty"`
 	// Hooks is a list of hooks for this instanceGroup, note: these can override the cluster wide ones if required
 	Hooks []HookSpec `json:"hooks,omitempty"`
 	// MaxPrice indicates this is a spot-pricing group, with the specified value as our max-price bid
@ -89,6 +119,8 @@ type InstanceGroupSpec struct {
 	Kubelet *KubeletConfigSpec `json:"kubelet,omitempty"`
 	// Taints indicates the kubernetes taints for nodes in this group
 	Taints []string `json:"taints,omitempty"`
+	// MixedInstancesPolicy defined a optional backing of an AWS ASG by a EC2 Fleet (AWS Only)
+	MixedInstancesPolicy *MixedInstancesPolicySpec `json:"mixedInstancesPolicy,omitempty"`
 	// AdditionalUserData is any additional user-data to be passed to the host
 	AdditionalUserData []UserData `json:"additionalUserData,omitempty"`
 	// Zones is the names of the Zones where machines in this instance group should be placed
@ -106,6 +138,39 @@ type InstanceGroupSpec struct {
 	SecurityGroupOverride *string `json:"securityGroupOverride,omitempty"`
 }

+const (
+	// SpotAllocationStrategyLowestPrices indicates a lowest price strategy
+	SpotAllocationStrategyLowestPrices = "LowestPrice"
+	// SpotAllocationStrategyDiversified indicates a diversified strategy
+	SpotAllocationStrategyDiversified = "Diversified"
+)
+
+// SpotAllocationStrategies is a collection of supported strategies
+var SpotAllocationStrategies = []string{SpotAllocationStrategyLowestPrices, SpotAllocationStrategyDiversified}
+
+// MixedInstancesPolicySpec defines the specification for an autoscaling backed by a ec2 fleet
+type MixedInstancesPolicySpec struct {
+	// Instances is a list of instance types which we are willing to run in the EC2 fleet
+	Instances []string `json:"instances,omitempty"`
+	// OnDemandAllocationStrategy indicates how to allocate instance types to fulfill On-Demand capacity
+	OnDemandAllocationStrategy *string `json:"onDemandAllocationStrategy,omitempty"`
+	// OnDemandBase is the minimum amount of the Auto Scaling group's capacity that must be
+	// fulfilled by On-Demand Instances. This base portion is provisioned first as your group scales.
+	OnDemandBase *int64 `json:"onDemandBase,omitempty"`
+	// OnDemandAboveBase controls the percentages of On-Demand Instances and Spot Instances for your
+	// additional capacity beyond OnDemandBase. The range is 0–100. The default value is 100. If you
+	// leave this parameter set to 100, the percentages are 100% for On-Demand Instances and 0% for
+	// Spot Instances.
+	OnDemandAboveBase *int64 `json:"onDemandAboveBase,omitempty"`
+	// SpotAllocationStrategy diversifies your Spot capacity across multiple instance types to
+	// find the best pricing. Higher Spot availability may result from a larger number of
+	// instance types to choose from.
+	SpotAllocationStrategy *string `json:"spotAllocationStrategy,omitempty"`
+	// SpotInstancePools is the number of Spot pools to use to allocate your Spot capacity (defaults to 2)
+	// pools are determined from the different instance types in the Overrides array of LaunchTemplate
+	SpotInstancePools *int64 `json:"spotInstancePools,omitempty"`
+}
+
 // IAMProfileSpec is the AWS IAM Profile to attach to instances in this instance
 // group. Specify the ARN for the IAM instance profile (AWS only).
 type IAMProfileSpec struct {
@ -124,7 +189,38 @@ type UserData struct {
 	Content string `json:"content,omitempty"`
 }

-// LoadBalancers defines a load balancer
+// VolumeSpec defined the spec for an additional volume attached to the instance group
+type VolumeSpec struct {
+	// Device is an optional device name of the block device
+	Device string `json:"device,omitempty"`
+	// Encrypted indicates you want to encrypt the volume
+	Encrypted *bool `json:"encrypted,omitempty"`
+	// Iops is the provision iops for this iops (think io1 in aws)
+	Iops *int64 `json:"iops,omitempty"`
+	// Size is the size of the volume in GB
+	Size int64 `json:"size,omitempty"`
+	// Type is the type of volume to create and is cloud specific
+	Type string `json:"type,omitempty"`
+}
+
+// VolumeMountSpec defines the specification for mounting a device
+type VolumeMountSpec struct {
+	// Device is the device name to provision and mount
+	Device string `json:"device,omitempty"`
+	// Filesystem is the filesystem to mount
+	Filesystem string `json:"filesystem,omitempty"`
+	// FormatOptions is a collection of options passed when formatting the device
+	FormatOptions []string `json:"formatOptions,omitempty"`
+	// MountOptions is a collection of mount options
+	MountOptions []string `json:"mountOptions,omitempty"`
+	// Path is the location to mount the device
+	Path string `json:"path,omitempty"`
+}
+
+// Ext4FileSystemSpec defines a specification for a ext4 filesystem on a instancegroup volume
+type Ext4FileSystemSpec struct{}
+
+// LoadBalancer defines a load balancer
 type LoadBalancer struct {
 	// LoadBalancerName to associate with this instance group (AWS ELB)
 	LoadBalancerName *string `json:"loadBalancerName,omitempty"`
--- a/pkg/apis/kops/v1alpha1/networking.go
+++ b/pkg/apis/kops/v1alpha1/networking.go
@ -102,6 +102,9 @@ type CanalNetworkingSpec struct {
 	// for traffic between pod to host after calico rules have been processed.
 	// Default: ACCEPT (other options: DROP, RETURN)
 	DefaultEndpointToHostAction string `json:"defaultEndpointToHostAction,omitempty"`
+	// DisableFlannelForwardRules configures Flannel to NOT add the
+	// default ACCEPT traffic rules to the iptables FORWARD chain
+	DisableFlannelForwardRules bool `json:"disableFlannelForwardRules,omitempty"`
 	// LogSeveritySys the severity to set for logs which are sent to syslog
 	// Default: INFO (other options: DEBUG, WARNING, ERROR, CRITICAL, NONE)
 	LogSeveritySys string `json:"logSeveritySys,omitempty"`
--- a/pkg/apis/kops/v1alpha1/zz_generated.conversion.go
+++ b/pkg/apis/kops/v1alpha1/zz_generated.conversion.go
@ -553,6 +553,16 @@ func RegisterConversions(s *runtime.Scheme) error {
 	}); err != nil {
 		return err
 	}
+	if err := s.AddGeneratedConversionFunc((*MixedInstancesPolicySpec)(nil), (*kops.MixedInstancesPolicySpec)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_v1alpha1_MixedInstancesPolicySpec_To_kops_MixedInstancesPolicySpec(a.(*MixedInstancesPolicySpec), b.(*kops.MixedInstancesPolicySpec), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*kops.MixedInstancesPolicySpec)(nil), (*MixedInstancesPolicySpec)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_kops_MixedInstancesPolicySpec_To_v1alpha1_MixedInstancesPolicySpec(a.(*kops.MixedInstancesPolicySpec), b.(*MixedInstancesPolicySpec), scope)
+	}); err != nil {
+		return err
+	}
 	if err := s.AddGeneratedConversionFunc((*NetworkingSpec)(nil), (*kops.NetworkingSpec)(nil), func(a, b interface{}, scope conversion.Scope) error {
 		return Convert_v1alpha1_NetworkingSpec_To_kops_NetworkingSpec(a.(*NetworkingSpec), b.(*kops.NetworkingSpec), scope)
 	}); err != nil {
@ -583,6 +593,56 @@ func RegisterConversions(s *runtime.Scheme) error {
 	}); err != nil {
 		return err
 	}
+	if err := s.AddGeneratedConversionFunc((*OpenstackBlockStorageConfig)(nil), (*kops.OpenstackBlockStorageConfig)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_v1alpha1_OpenstackBlockStorageConfig_To_kops_OpenstackBlockStorageConfig(a.(*OpenstackBlockStorageConfig), b.(*kops.OpenstackBlockStorageConfig), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*kops.OpenstackBlockStorageConfig)(nil), (*OpenstackBlockStorageConfig)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_kops_OpenstackBlockStorageConfig_To_v1alpha1_OpenstackBlockStorageConfig(a.(*kops.OpenstackBlockStorageConfig), b.(*OpenstackBlockStorageConfig), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*OpenstackConfiguration)(nil), (*kops.OpenstackConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_v1alpha1_OpenstackConfiguration_To_kops_OpenstackConfiguration(a.(*OpenstackConfiguration), b.(*kops.OpenstackConfiguration), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*kops.OpenstackConfiguration)(nil), (*OpenstackConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_kops_OpenstackConfiguration_To_v1alpha1_OpenstackConfiguration(a.(*kops.OpenstackConfiguration), b.(*OpenstackConfiguration), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*OpenstackLoadbalancerConfig)(nil), (*kops.OpenstackLoadbalancerConfig)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_v1alpha1_OpenstackLoadbalancerConfig_To_kops_OpenstackLoadbalancerConfig(a.(*OpenstackLoadbalancerConfig), b.(*kops.OpenstackLoadbalancerConfig), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*kops.OpenstackLoadbalancerConfig)(nil), (*OpenstackLoadbalancerConfig)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_kops_OpenstackLoadbalancerConfig_To_v1alpha1_OpenstackLoadbalancerConfig(a.(*kops.OpenstackLoadbalancerConfig), b.(*OpenstackLoadbalancerConfig), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*OpenstackMonitor)(nil), (*kops.OpenstackMonitor)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_v1alpha1_OpenstackMonitor_To_kops_OpenstackMonitor(a.(*OpenstackMonitor), b.(*kops.OpenstackMonitor), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*kops.OpenstackMonitor)(nil), (*OpenstackMonitor)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_kops_OpenstackMonitor_To_v1alpha1_OpenstackMonitor(a.(*kops.OpenstackMonitor), b.(*OpenstackMonitor), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*OpenstackRouter)(nil), (*kops.OpenstackRouter)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_v1alpha1_OpenstackRouter_To_kops_OpenstackRouter(a.(*OpenstackRouter), b.(*kops.OpenstackRouter), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*kops.OpenstackRouter)(nil), (*OpenstackRouter)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_kops_OpenstackRouter_To_v1alpha1_OpenstackRouter(a.(*kops.OpenstackRouter), b.(*OpenstackRouter), scope)
+	}); err != nil {
+		return err
+	}
 	if err := s.AddGeneratedConversionFunc((*RBACAuthorizationSpec)(nil), (*kops.RBACAuthorizationSpec)(nil), func(a, b interface{}, scope conversion.Scope) error {
 		return Convert_v1alpha1_RBACAuthorizationSpec_To_kops_RBACAuthorizationSpec(a.(*RBACAuthorizationSpec), b.(*kops.RBACAuthorizationSpec), scope)
 	}); err != nil {
@ -663,6 +723,26 @@ func RegisterConversions(s *runtime.Scheme) error {
 	}); err != nil {
 		return err
 	}
+	if err := s.AddGeneratedConversionFunc((*VolumeMountSpec)(nil), (*kops.VolumeMountSpec)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_v1alpha1_VolumeMountSpec_To_kops_VolumeMountSpec(a.(*VolumeMountSpec), b.(*kops.VolumeMountSpec), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*kops.VolumeMountSpec)(nil), (*VolumeMountSpec)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_kops_VolumeMountSpec_To_v1alpha1_VolumeMountSpec(a.(*kops.VolumeMountSpec), b.(*VolumeMountSpec), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*VolumeSpec)(nil), (*kops.VolumeSpec)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_v1alpha1_VolumeSpec_To_kops_VolumeSpec(a.(*VolumeSpec), b.(*kops.VolumeSpec), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*kops.VolumeSpec)(nil), (*VolumeSpec)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_kops_VolumeSpec_To_v1alpha1_VolumeSpec(a.(*kops.VolumeSpec), b.(*VolumeSpec), scope)
+	}); err != nil {
+		return err
+	}
 	if err := s.AddGeneratedConversionFunc((*WeaveNetworkingSpec)(nil), (*kops.WeaveNetworkingSpec)(nil), func(a, b interface{}, scope conversion.Scope) error {
 		return Convert_v1alpha1_WeaveNetworkingSpec_To_kops_WeaveNetworkingSpec(a.(*WeaveNetworkingSpec), b.(*kops.WeaveNetworkingSpec), scope)
 	}); err != nil {
@ -1045,6 +1125,7 @@ func Convert_kops_CalicoNetworkingSpec_To_v1alpha1_CalicoNetworkingSpec(in *kops
 func autoConvert_v1alpha1_CanalNetworkingSpec_To_kops_CanalNetworkingSpec(in *CanalNetworkingSpec, out *kops.CanalNetworkingSpec, s conversion.Scope) error {
 	out.ChainInsertMode = in.ChainInsertMode
 	out.DefaultEndpointToHostAction = in.DefaultEndpointToHostAction
+	out.DisableFlannelForwardRules = in.DisableFlannelForwardRules
 	out.LogSeveritySys = in.LogSeveritySys
 	out.PrometheusGoMetricsEnabled = in.PrometheusGoMetricsEnabled
 	out.PrometheusMetricsEnabled = in.PrometheusMetricsEnabled
@ -1061,6 +1142,7 @@ func Convert_v1alpha1_CanalNetworkingSpec_To_kops_CanalNetworkingSpec(in *CanalN
 func autoConvert_kops_CanalNetworkingSpec_To_v1alpha1_CanalNetworkingSpec(in *kops.CanalNetworkingSpec, out *CanalNetworkingSpec, s conversion.Scope) error {
 	out.ChainInsertMode = in.ChainInsertMode
 	out.DefaultEndpointToHostAction = in.DefaultEndpointToHostAction
+	out.DisableFlannelForwardRules = in.DisableFlannelForwardRules
 	out.LogSeveritySys = in.LogSeveritySys
 	out.PrometheusGoMetricsEnabled = in.PrometheusGoMetricsEnabled
 	out.PrometheusMetricsEnabled = in.PrometheusMetricsEnabled
@ -1223,6 +1305,15 @@ func autoConvert_v1alpha1_CloudConfiguration_To_kops_CloudConfiguration(in *Clou
 	out.VSphereCoreDNSServer = in.VSphereCoreDNSServer
 	out.SpotinstProduct = in.SpotinstProduct
 	out.SpotinstOrientation = in.SpotinstOrientation
+	if in.Openstack != nil {
+		in, out := &in.Openstack, &out.Openstack
+		*out = new(kops.OpenstackConfiguration)
+		if err := Convert_v1alpha1_OpenstackConfiguration_To_kops_OpenstackConfiguration(*in, *out, s); err != nil {
+			return err
+		}
+	} else {
+		out.Openstack = nil
+	}
 	return nil
 }

@ -1246,6 +1337,15 @@ func autoConvert_kops_CloudConfiguration_To_v1alpha1_CloudConfiguration(in *kops
 	out.VSphereCoreDNSServer = in.VSphereCoreDNSServer
 	out.SpotinstProduct = in.SpotinstProduct
 	out.SpotinstOrientation = in.SpotinstOrientation
+	if in.Openstack != nil {
+		in, out := &in.Openstack, &out.Openstack
+		*out = new(OpenstackConfiguration)
+		if err := Convert_kops_OpenstackConfiguration_To_v1alpha1_OpenstackConfiguration(*in, *out, s); err != nil {
+			return err
+		}
+	} else {
+		out.Openstack = nil
+	}
 	return nil
 }

@ -2537,6 +2637,30 @@ func autoConvert_v1alpha1_InstanceGroupSpec_To_kops_InstanceGroupSpec(in *Instan
 	out.RootVolumeType = in.RootVolumeType
 	out.RootVolumeIops = in.RootVolumeIops
 	out.RootVolumeOptimization = in.RootVolumeOptimization
+	if in.Volumes != nil {
+		in, out := &in.Volumes, &out.Volumes
+		*out = make([]*kops.VolumeSpec, len(*in))
+		for i := range *in {
+			// TODO: Inefficient conversion - can we improve it?
+			if err := s.Convert(&(*in)[i], &(*out)[i], 0); err != nil {
+				return err
+			}
+		}
+	} else {
+		out.Volumes = nil
+	}
+	if in.VolumeMounts != nil {
+		in, out := &in.VolumeMounts, &out.VolumeMounts
+		*out = make([]*kops.VolumeMountSpec, len(*in))
+		for i := range *in {
+			// TODO: Inefficient conversion - can we improve it?
+			if err := s.Convert(&(*in)[i], &(*out)[i], 0); err != nil {
+				return err
+			}
+		}
+	} else {
+		out.VolumeMounts = nil
+	}
 	if in.Hooks != nil {
 		in, out := &in.Hooks, &out.Hooks
 		*out = make([]kops.HookSpec, len(*in))
@ -2575,6 +2699,15 @@ func autoConvert_v1alpha1_InstanceGroupSpec_To_kops_InstanceGroupSpec(in *Instan
 		out.Kubelet = nil
 	}
 	out.Taints = in.Taints
+	if in.MixedInstancesPolicy != nil {
+		in, out := &in.MixedInstancesPolicy, &out.MixedInstancesPolicy
+		*out = new(kops.MixedInstancesPolicySpec)
+		if err := Convert_v1alpha1_MixedInstancesPolicySpec_To_kops_MixedInstancesPolicySpec(*in, *out, s); err != nil {
+			return err
+		}
+	} else {
+		out.MixedInstancesPolicy = nil
+	}
 	if in.AdditionalUserData != nil {
 		in, out := &in.AdditionalUserData, &out.AdditionalUserData
 		*out = make([]kops.UserData, len(*in))
@ -2623,6 +2756,30 @@ func autoConvert_kops_InstanceGroupSpec_To_v1alpha1_InstanceGroupSpec(in *kops.I
 	out.RootVolumeType = in.RootVolumeType
 	out.RootVolumeIops = in.RootVolumeIops
 	out.RootVolumeOptimization = in.RootVolumeOptimization
+	if in.Volumes != nil {
+		in, out := &in.Volumes, &out.Volumes
+		*out = make([]*VolumeSpec, len(*in))
+		for i := range *in {
+			// TODO: Inefficient conversion - can we improve it?
+			if err := s.Convert(&(*in)[i], &(*out)[i], 0); err != nil {
+				return err
+			}
+		}
+	} else {
+		out.Volumes = nil
+	}
+	if in.VolumeMounts != nil {
+		in, out := &in.VolumeMounts, &out.VolumeMounts
+		*out = make([]*VolumeMountSpec, len(*in))
+		for i := range *in {
+			// TODO: Inefficient conversion - can we improve it?
+			if err := s.Convert(&(*in)[i], &(*out)[i], 0); err != nil {
+				return err
+			}
+		}
+	} else {
+		out.VolumeMounts = nil
+	}
 	// WARNING: in.Subnets requires manual conversion: does not exist in peer-type
 	out.Zones = in.Zones
 	if in.Hooks != nil {
@ -2663,6 +2820,15 @@ func autoConvert_kops_InstanceGroupSpec_To_v1alpha1_InstanceGroupSpec(in *kops.I
 		out.Kubelet = nil
 	}
 	out.Taints = in.Taints
+	if in.MixedInstancesPolicy != nil {
+		in, out := &in.MixedInstancesPolicy, &out.MixedInstancesPolicy
+		*out = new(MixedInstancesPolicySpec)
+		if err := Convert_kops_MixedInstancesPolicySpec_To_v1alpha1_MixedInstancesPolicySpec(*in, *out, s); err != nil {
+			return err
+		}
+	} else {
+		out.MixedInstancesPolicy = nil
+	}
 	if in.AdditionalUserData != nil {
 		in, out := &in.AdditionalUserData, &out.AdditionalUserData
 		*out = make([]UserData, len(*in))
@ -2777,6 +2943,7 @@ func autoConvert_v1alpha1_KubeAPIServerConfig_To_kops_KubeAPIServerConfig(in *Ku
 	out.OIDCGroupsPrefix = in.OIDCGroupsPrefix
 	out.OIDCIssuerURL = in.OIDCIssuerURL
 	out.OIDCClientID = in.OIDCClientID
+	out.OIDCRequiredClaim = in.OIDCRequiredClaim
 	out.OIDCCAFile = in.OIDCCAFile
 	out.ProxyClientCertFile = in.ProxyClientCertFile
 	out.ProxyClientKeyFile = in.ProxyClientKeyFile
@ -2851,6 +3018,7 @@ func autoConvert_kops_KubeAPIServerConfig_To_v1alpha1_KubeAPIServerConfig(in *ko
 	out.OIDCGroupsPrefix = in.OIDCGroupsPrefix
 	out.OIDCIssuerURL = in.OIDCIssuerURL
 	out.OIDCClientID = in.OIDCClientID
+	out.OIDCRequiredClaim = in.OIDCRequiredClaim
 	out.OIDCCAFile = in.OIDCCAFile
 	out.ProxyClientCertFile = in.ProxyClientCertFile
 	out.ProxyClientKeyFile = in.ProxyClientKeyFile
@ -3177,6 +3345,8 @@ func autoConvert_v1alpha1_KubeletConfigSpec_To_kops_KubeletConfigSpec(in *Kubele
 	out.RootDir = in.RootDir
 	out.AuthenticationTokenWebhook = in.AuthenticationTokenWebhook
 	out.AuthenticationTokenWebhookCacheTTL = in.AuthenticationTokenWebhookCacheTTL
+	out.CPUCFSQuota = in.CPUCFSQuota
+	out.CPUCFSQuotaPeriod = in.CPUCFSQuotaPeriod
 	return nil
 }

@ -3254,6 +3424,8 @@ func autoConvert_kops_KubeletConfigSpec_To_v1alpha1_KubeletConfigSpec(in *kops.K
 	out.RootDir = in.RootDir
 	out.AuthenticationTokenWebhook = in.AuthenticationTokenWebhook
 	out.AuthenticationTokenWebhookCacheTTL = in.AuthenticationTokenWebhookCacheTTL
+	out.CPUCFSQuota = in.CPUCFSQuota
+	out.CPUCFSQuotaPeriod = in.CPUCFSQuotaPeriod
 	return nil
 }

@ -3390,6 +3562,36 @@ func Convert_kops_LyftVPCNetworkingSpec_To_v1alpha1_LyftVPCNetworkingSpec(in *ko
 	return autoConvert_kops_LyftVPCNetworkingSpec_To_v1alpha1_LyftVPCNetworkingSpec(in, out, s)
 }

+func autoConvert_v1alpha1_MixedInstancesPolicySpec_To_kops_MixedInstancesPolicySpec(in *MixedInstancesPolicySpec, out *kops.MixedInstancesPolicySpec, s conversion.Scope) error {
+	out.Instances = in.Instances
+	out.OnDemandAllocationStrategy = in.OnDemandAllocationStrategy
+	out.OnDemandBase = in.OnDemandBase
+	out.OnDemandAboveBase = in.OnDemandAboveBase
+	out.SpotAllocationStrategy = in.SpotAllocationStrategy
+	out.SpotInstancePools = in.SpotInstancePools
+	return nil
+}
+
+// Convert_v1alpha1_MixedInstancesPolicySpec_To_kops_MixedInstancesPolicySpec is an autogenerated conversion function.
+func Convert_v1alpha1_MixedInstancesPolicySpec_To_kops_MixedInstancesPolicySpec(in *MixedInstancesPolicySpec, out *kops.MixedInstancesPolicySpec, s conversion.Scope) error {
+	return autoConvert_v1alpha1_MixedInstancesPolicySpec_To_kops_MixedInstancesPolicySpec(in, out, s)
+}
+
+func autoConvert_kops_MixedInstancesPolicySpec_To_v1alpha1_MixedInstancesPolicySpec(in *kops.MixedInstancesPolicySpec, out *MixedInstancesPolicySpec, s conversion.Scope) error {
+	out.Instances = in.Instances
+	out.OnDemandAllocationStrategy = in.OnDemandAllocationStrategy
+	out.OnDemandBase = in.OnDemandBase
+	out.OnDemandAboveBase = in.OnDemandAboveBase
+	out.SpotAllocationStrategy = in.SpotAllocationStrategy
+	out.SpotInstancePools = in.SpotInstancePools
+	return nil
+}
+
+// Convert_kops_MixedInstancesPolicySpec_To_v1alpha1_MixedInstancesPolicySpec is an autogenerated conversion function.
+func Convert_kops_MixedInstancesPolicySpec_To_v1alpha1_MixedInstancesPolicySpec(in *kops.MixedInstancesPolicySpec, out *MixedInstancesPolicySpec, s conversion.Scope) error {
+	return autoConvert_kops_MixedInstancesPolicySpec_To_v1alpha1_MixedInstancesPolicySpec(in, out, s)
+}
+
 func autoConvert_v1alpha1_NetworkingSpec_To_kops_NetworkingSpec(in *NetworkingSpec, out *kops.NetworkingSpec, s conversion.Scope) error {
 	if in.Classic != nil {
 		in, out := &in.Classic, &out.Classic
@ -3728,6 +3930,192 @@ func Convert_kops_NodeAuthorizerSpec_To_v1alpha1_NodeAuthorizerSpec(in *kops.Nod
 	return autoConvert_kops_NodeAuthorizerSpec_To_v1alpha1_NodeAuthorizerSpec(in, out, s)
 }

+func autoConvert_v1alpha1_OpenstackBlockStorageConfig_To_kops_OpenstackBlockStorageConfig(in *OpenstackBlockStorageConfig, out *kops.OpenstackBlockStorageConfig, s conversion.Scope) error {
+	out.Version = in.Version
+	out.IgnoreAZ = in.IgnoreAZ
+	return nil
+}
+
+// Convert_v1alpha1_OpenstackBlockStorageConfig_To_kops_OpenstackBlockStorageConfig is an autogenerated conversion function.
+func Convert_v1alpha1_OpenstackBlockStorageConfig_To_kops_OpenstackBlockStorageConfig(in *OpenstackBlockStorageConfig, out *kops.OpenstackBlockStorageConfig, s conversion.Scope) error {
+	return autoConvert_v1alpha1_OpenstackBlockStorageConfig_To_kops_OpenstackBlockStorageConfig(in, out, s)
+}
+
+func autoConvert_kops_OpenstackBlockStorageConfig_To_v1alpha1_OpenstackBlockStorageConfig(in *kops.OpenstackBlockStorageConfig, out *OpenstackBlockStorageConfig, s conversion.Scope) error {
+	out.Version = in.Version
+	out.IgnoreAZ = in.IgnoreAZ
+	return nil
+}
+
+// Convert_kops_OpenstackBlockStorageConfig_To_v1alpha1_OpenstackBlockStorageConfig is an autogenerated conversion function.
+func Convert_kops_OpenstackBlockStorageConfig_To_v1alpha1_OpenstackBlockStorageConfig(in *kops.OpenstackBlockStorageConfig, out *OpenstackBlockStorageConfig, s conversion.Scope) error {
+	return autoConvert_kops_OpenstackBlockStorageConfig_To_v1alpha1_OpenstackBlockStorageConfig(in, out, s)
+}
+
+func autoConvert_v1alpha1_OpenstackConfiguration_To_kops_OpenstackConfiguration(in *OpenstackConfiguration, out *kops.OpenstackConfiguration, s conversion.Scope) error {
+	if in.Loadbalancer != nil {
+		in, out := &in.Loadbalancer, &out.Loadbalancer
+		*out = new(kops.OpenstackLoadbalancerConfig)
+		if err := Convert_v1alpha1_OpenstackLoadbalancerConfig_To_kops_OpenstackLoadbalancerConfig(*in, *out, s); err != nil {
+			return err
+		}
+	} else {
+		out.Loadbalancer = nil
+	}
+	if in.Monitor != nil {
+		in, out := &in.Monitor, &out.Monitor
+		*out = new(kops.OpenstackMonitor)
+		if err := Convert_v1alpha1_OpenstackMonitor_To_kops_OpenstackMonitor(*in, *out, s); err != nil {
+			return err
+		}
+	} else {
+		out.Monitor = nil
+	}
+	if in.Router != nil {
+		in, out := &in.Router, &out.Router
+		*out = new(kops.OpenstackRouter)
+		if err := Convert_v1alpha1_OpenstackRouter_To_kops_OpenstackRouter(*in, *out, s); err != nil {
+			return err
+		}
+	} else {
+		out.Router = nil
+	}
+	if in.BlockStorage != nil {
+		in, out := &in.BlockStorage, &out.BlockStorage
+		*out = new(kops.OpenstackBlockStorageConfig)
+		if err := Convert_v1alpha1_OpenstackBlockStorageConfig_To_kops_OpenstackBlockStorageConfig(*in, *out, s); err != nil {
+			return err
+		}
+	} else {
+		out.BlockStorage = nil
+	}
+	return nil
+}
+
+// Convert_v1alpha1_OpenstackConfiguration_To_kops_OpenstackConfiguration is an autogenerated conversion function.
+func Convert_v1alpha1_OpenstackConfiguration_To_kops_OpenstackConfiguration(in *OpenstackConfiguration, out *kops.OpenstackConfiguration, s conversion.Scope) error {
+	return autoConvert_v1alpha1_OpenstackConfiguration_To_kops_OpenstackConfiguration(in, out, s)
+}
+
+func autoConvert_kops_OpenstackConfiguration_To_v1alpha1_OpenstackConfiguration(in *kops.OpenstackConfiguration, out *OpenstackConfiguration, s conversion.Scope) error {
+	if in.Loadbalancer != nil {
+		in, out := &in.Loadbalancer, &out.Loadbalancer
+		*out = new(OpenstackLoadbalancerConfig)
+		if err := Convert_kops_OpenstackLoadbalancerConfig_To_v1alpha1_OpenstackLoadbalancerConfig(*in, *out, s); err != nil {
+			return err
+		}
+	} else {
+		out.Loadbalancer = nil
+	}
+	if in.Monitor != nil {
+		in, out := &in.Monitor, &out.Monitor
+		*out = new(OpenstackMonitor)
+		if err := Convert_kops_OpenstackMonitor_To_v1alpha1_OpenstackMonitor(*in, *out, s); err != nil {
+			return err
+		}
+	} else {
+		out.Monitor = nil
+	}
+	if in.Router != nil {
+		in, out := &in.Router, &out.Router
+		*out = new(OpenstackRouter)
+		if err := Convert_kops_OpenstackRouter_To_v1alpha1_OpenstackRouter(*in, *out, s); err != nil {
+			return err
+		}
+	} else {
+		out.Router = nil
+	}
+	if in.BlockStorage != nil {
+		in, out := &in.BlockStorage, &out.BlockStorage
+		*out = new(OpenstackBlockStorageConfig)
+		if err := Convert_kops_OpenstackBlockStorageConfig_To_v1alpha1_OpenstackBlockStorageConfig(*in, *out, s); err != nil {
+			return err
+		}
+	} else {
+		out.BlockStorage = nil
+	}
+	return nil
+}
+
+// Convert_kops_OpenstackConfiguration_To_v1alpha1_OpenstackConfiguration is an autogenerated conversion function.
+func Convert_kops_OpenstackConfiguration_To_v1alpha1_OpenstackConfiguration(in *kops.OpenstackConfiguration, out *OpenstackConfiguration, s conversion.Scope) error {
+	return autoConvert_kops_OpenstackConfiguration_To_v1alpha1_OpenstackConfiguration(in, out, s)
+}
+
+func autoConvert_v1alpha1_OpenstackLoadbalancerConfig_To_kops_OpenstackLoadbalancerConfig(in *OpenstackLoadbalancerConfig, out *kops.OpenstackLoadbalancerConfig, s conversion.Scope) error {
+	out.Method = in.Method
+	out.Provider = in.Provider
+	out.UseOctavia = in.UseOctavia
+	out.FloatingNetwork = in.FloatingNetwork
+	out.FloatingNetworkID = in.FloatingNetworkID
+	out.SubnetID = in.SubnetID
+	return nil
+}
+
+// Convert_v1alpha1_OpenstackLoadbalancerConfig_To_kops_OpenstackLoadbalancerConfig is an autogenerated conversion function.
+func Convert_v1alpha1_OpenstackLoadbalancerConfig_To_kops_OpenstackLoadbalancerConfig(in *OpenstackLoadbalancerConfig, out *kops.OpenstackLoadbalancerConfig, s conversion.Scope) error {
+	return autoConvert_v1alpha1_OpenstackLoadbalancerConfig_To_kops_OpenstackLoadbalancerConfig(in, out, s)
+}
+
+func autoConvert_kops_OpenstackLoadbalancerConfig_To_v1alpha1_OpenstackLoadbalancerConfig(in *kops.OpenstackLoadbalancerConfig, out *OpenstackLoadbalancerConfig, s conversion.Scope) error {
+	out.Method = in.Method
+	out.Provider = in.Provider
+	out.UseOctavia = in.UseOctavia
+	out.FloatingNetwork = in.FloatingNetwork
+	out.FloatingNetworkID = in.FloatingNetworkID
+	out.SubnetID = in.SubnetID
+	return nil
+}
+
+// Convert_kops_OpenstackLoadbalancerConfig_To_v1alpha1_OpenstackLoadbalancerConfig is an autogenerated conversion function.
+func Convert_kops_OpenstackLoadbalancerConfig_To_v1alpha1_OpenstackLoadbalancerConfig(in *kops.OpenstackLoadbalancerConfig, out *OpenstackLoadbalancerConfig, s conversion.Scope) error {
+	return autoConvert_kops_OpenstackLoadbalancerConfig_To_v1alpha1_OpenstackLoadbalancerConfig(in, out, s)
+}
+
+func autoConvert_v1alpha1_OpenstackMonitor_To_kops_OpenstackMonitor(in *OpenstackMonitor, out *kops.OpenstackMonitor, s conversion.Scope) error {
+	out.Delay = in.Delay
+	out.Timeout = in.Timeout
+	out.MaxRetries = in.MaxRetries
+	return nil
+}
+
+// Convert_v1alpha1_OpenstackMonitor_To_kops_OpenstackMonitor is an autogenerated conversion function.
+func Convert_v1alpha1_OpenstackMonitor_To_kops_OpenstackMonitor(in *OpenstackMonitor, out *kops.OpenstackMonitor, s conversion.Scope) error {
+	return autoConvert_v1alpha1_OpenstackMonitor_To_kops_OpenstackMonitor(in, out, s)
+}
+
+func autoConvert_kops_OpenstackMonitor_To_v1alpha1_OpenstackMonitor(in *kops.OpenstackMonitor, out *OpenstackMonitor, s conversion.Scope) error {
+	out.Delay = in.Delay
+	out.Timeout = in.Timeout
+	out.MaxRetries = in.MaxRetries
+	return nil
+}
+
+// Convert_kops_OpenstackMonitor_To_v1alpha1_OpenstackMonitor is an autogenerated conversion function.
+func Convert_kops_OpenstackMonitor_To_v1alpha1_OpenstackMonitor(in *kops.OpenstackMonitor, out *OpenstackMonitor, s conversion.Scope) error {
+	return autoConvert_kops_OpenstackMonitor_To_v1alpha1_OpenstackMonitor(in, out, s)
+}
+
+func autoConvert_v1alpha1_OpenstackRouter_To_kops_OpenstackRouter(in *OpenstackRouter, out *kops.OpenstackRouter, s conversion.Scope) error {
+	out.ExternalNetwork = in.ExternalNetwork
+	return nil
+}
+
+// Convert_v1alpha1_OpenstackRouter_To_kops_OpenstackRouter is an autogenerated conversion function.
+func Convert_v1alpha1_OpenstackRouter_To_kops_OpenstackRouter(in *OpenstackRouter, out *kops.OpenstackRouter, s conversion.Scope) error {
+	return autoConvert_v1alpha1_OpenstackRouter_To_kops_OpenstackRouter(in, out, s)
+}
+
+func autoConvert_kops_OpenstackRouter_To_v1alpha1_OpenstackRouter(in *kops.OpenstackRouter, out *OpenstackRouter, s conversion.Scope) error {
+	out.ExternalNetwork = in.ExternalNetwork
+	return nil
+}
+
+// Convert_kops_OpenstackRouter_To_v1alpha1_OpenstackRouter is an autogenerated conversion function.
+func Convert_kops_OpenstackRouter_To_v1alpha1_OpenstackRouter(in *kops.OpenstackRouter, out *OpenstackRouter, s conversion.Scope) error {
+	return autoConvert_kops_OpenstackRouter_To_v1alpha1_OpenstackRouter(in, out, s)
+}
+
 func autoConvert_v1alpha1_RBACAuthorizationSpec_To_kops_RBACAuthorizationSpec(in *RBACAuthorizationSpec, out *kops.RBACAuthorizationSpec, s conversion.Scope) error {
 	return nil
 }
@ -3936,6 +4324,62 @@ func Convert_kops_UserData_To_v1alpha1_UserData(in *kops.UserData, out *UserData
 	return autoConvert_kops_UserData_To_v1alpha1_UserData(in, out, s)
 }

+func autoConvert_v1alpha1_VolumeMountSpec_To_kops_VolumeMountSpec(in *VolumeMountSpec, out *kops.VolumeMountSpec, s conversion.Scope) error {
+	out.Device = in.Device
+	out.Filesystem = in.Filesystem
+	out.FormatOptions = in.FormatOptions
+	out.MountOptions = in.MountOptions
+	out.Path = in.Path
+	return nil
+}
+
+// Convert_v1alpha1_VolumeMountSpec_To_kops_VolumeMountSpec is an autogenerated conversion function.
+func Convert_v1alpha1_VolumeMountSpec_To_kops_VolumeMountSpec(in *VolumeMountSpec, out *kops.VolumeMountSpec, s conversion.Scope) error {
+	return autoConvert_v1alpha1_VolumeMountSpec_To_kops_VolumeMountSpec(in, out, s)
+}
+
+func autoConvert_kops_VolumeMountSpec_To_v1alpha1_VolumeMountSpec(in *kops.VolumeMountSpec, out *VolumeMountSpec, s conversion.Scope) error {
+	out.Device = in.Device
+	out.Filesystem = in.Filesystem
+	out.FormatOptions = in.FormatOptions
+	out.MountOptions = in.MountOptions
+	out.Path = in.Path
+	return nil
+}
+
+// Convert_kops_VolumeMountSpec_To_v1alpha1_VolumeMountSpec is an autogenerated conversion function.
+func Convert_kops_VolumeMountSpec_To_v1alpha1_VolumeMountSpec(in *kops.VolumeMountSpec, out *VolumeMountSpec, s conversion.Scope) error {
+	return autoConvert_kops_VolumeMountSpec_To_v1alpha1_VolumeMountSpec(in, out, s)
+}
+
+func autoConvert_v1alpha1_VolumeSpec_To_kops_VolumeSpec(in *VolumeSpec, out *kops.VolumeSpec, s conversion.Scope) error {
+	out.Device = in.Device
+	out.Encrypted = in.Encrypted
+	out.Iops = in.Iops
+	out.Size = in.Size
+	out.Type = in.Type
+	return nil
+}
+
+// Convert_v1alpha1_VolumeSpec_To_kops_VolumeSpec is an autogenerated conversion function.
+func Convert_v1alpha1_VolumeSpec_To_kops_VolumeSpec(in *VolumeSpec, out *kops.VolumeSpec, s conversion.Scope) error {
+	return autoConvert_v1alpha1_VolumeSpec_To_kops_VolumeSpec(in, out, s)
+}
+
+func autoConvert_kops_VolumeSpec_To_v1alpha1_VolumeSpec(in *kops.VolumeSpec, out *VolumeSpec, s conversion.Scope) error {
+	out.Device = in.Device
+	out.Encrypted = in.Encrypted
+	out.Iops = in.Iops
+	out.Size = in.Size
+	out.Type = in.Type
+	return nil
+}
+
+// Convert_kops_VolumeSpec_To_v1alpha1_VolumeSpec is an autogenerated conversion function.
+func Convert_kops_VolumeSpec_To_v1alpha1_VolumeSpec(in *kops.VolumeSpec, out *VolumeSpec, s conversion.Scope) error {
+	return autoConvert_kops_VolumeSpec_To_v1alpha1_VolumeSpec(in, out, s)
+}
+
 func autoConvert_v1alpha1_WeaveNetworkingSpec_To_kops_WeaveNetworkingSpec(in *WeaveNetworkingSpec, out *kops.WeaveNetworkingSpec, s conversion.Scope) error {
 	out.MTU = in.MTU
 	out.ConnLimit = in.ConnLimit
--- a/pkg/apis/kops/v1alpha1/zz_generated.deepcopy.go
+++ b/pkg/apis/kops/v1alpha1/zz_generated.deepcopy.go
@ -442,6 +442,11 @@ func (in *CloudConfiguration) DeepCopyInto(out *CloudConfiguration) {
 		*out = new(string)
 		**out = **in
 	}
+	if in.Openstack != nil {
+		in, out := &in.Openstack, &out.Openstack
+		*out = new(OpenstackConfiguration)
+		(*in).DeepCopyInto(*out)
+	}
 	return
 }

@ -1120,6 +1125,22 @@ func (in *ExecContainerAction) DeepCopy() *ExecContainerAction {
 	return out
 }

+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *Ext4FileSystemSpec) DeepCopyInto(out *Ext4FileSystemSpec) {
+	*out = *in
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Ext4FileSystemSpec.
+func (in *Ext4FileSystemSpec) DeepCopy() *Ext4FileSystemSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(Ext4FileSystemSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ExternalDNSConfig) DeepCopyInto(out *ExternalDNSConfig) {
 	*out = *in
@ -1376,6 +1397,28 @@ func (in *InstanceGroupSpec) DeepCopyInto(out *InstanceGroupSpec) {
 		*out = new(bool)
 		**out = **in
 	}
+	if in.Volumes != nil {
+		in, out := &in.Volumes, &out.Volumes
+		*out = make([]*VolumeSpec, len(*in))
+		for i := range *in {
+			if (*in)[i] != nil {
+				in, out := &(*in)[i], &(*out)[i]
+				*out = new(VolumeSpec)
+				(*in).DeepCopyInto(*out)
+			}
+		}
+	}
+	if in.VolumeMounts != nil {
+		in, out := &in.VolumeMounts, &out.VolumeMounts
+		*out = make([]*VolumeMountSpec, len(*in))
+		for i := range *in {
+			if (*in)[i] != nil {
+				in, out := &(*in)[i], &(*out)[i]
+				*out = new(VolumeMountSpec)
+				(*in).DeepCopyInto(*out)
+			}
+		}
+	}
 	if in.Hooks != nil {
 		in, out := &in.Hooks, &out.Hooks
 		*out = make([]HookSpec, len(*in))
@ -1429,6 +1472,11 @@ func (in *InstanceGroupSpec) DeepCopyInto(out *InstanceGroupSpec) {
 		*out = make([]string, len(*in))
 		copy(*out, *in)
 	}
+	if in.MixedInstancesPolicy != nil {
+		in, out := &in.MixedInstancesPolicy, &out.MixedInstancesPolicy
+		*out = new(MixedInstancesPolicySpec)
+		(*in).DeepCopyInto(*out)
+	}
 	if in.AdditionalUserData != nil {
 		in, out := &in.AdditionalUserData, &out.AdditionalUserData
 		*out = make([]UserData, len(*in))
@ -1611,6 +1659,11 @@ func (in *KubeAPIServerConfig) DeepCopyInto(out *KubeAPIServerConfig) {
 		*out = new(string)
 		**out = **in
 	}
+	if in.OIDCRequiredClaim != nil {
+		in, out := &in.OIDCRequiredClaim, &out.OIDCRequiredClaim
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
 	if in.OIDCCAFile != nil {
 		in, out := &in.OIDCCAFile, &out.OIDCCAFile
 		*out = new(string)
@ -2157,6 +2210,16 @@ func (in *KubeletConfigSpec) DeepCopyInto(out *KubeletConfigSpec) {
 		*out = new(v1.Duration)
 		**out = **in
 	}
+	if in.CPUCFSQuota != nil {
+		in, out := &in.CPUCFSQuota, &out.CPUCFSQuota
+		*out = new(bool)
+		**out = **in
+	}
+	if in.CPUCFSQuotaPeriod != nil {
+		in, out := &in.CPUCFSQuotaPeriod, &out.CPUCFSQuotaPeriod
+		*out = new(v1.Duration)
+		**out = **in
+	}
 	return
 }

@ -2303,6 +2366,52 @@ func (in *LyftVPCNetworkingSpec) DeepCopy() *LyftVPCNetworkingSpec {
 	return out
 }

+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *MixedInstancesPolicySpec) DeepCopyInto(out *MixedInstancesPolicySpec) {
+	*out = *in
+	if in.Instances != nil {
+		in, out := &in.Instances, &out.Instances
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
+	if in.OnDemandAllocationStrategy != nil {
+		in, out := &in.OnDemandAllocationStrategy, &out.OnDemandAllocationStrategy
+		*out = new(string)
+		**out = **in
+	}
+	if in.OnDemandBase != nil {
+		in, out := &in.OnDemandBase, &out.OnDemandBase
+		*out = new(int64)
+		**out = **in
+	}
+	if in.OnDemandAboveBase != nil {
+		in, out := &in.OnDemandAboveBase, &out.OnDemandAboveBase
+		*out = new(int64)
+		**out = **in
+	}
+	if in.SpotAllocationStrategy != nil {
+		in, out := &in.SpotAllocationStrategy, &out.SpotAllocationStrategy
+		*out = new(string)
+		**out = **in
+	}
+	if in.SpotInstancePools != nil {
+		in, out := &in.SpotInstancePools, &out.SpotInstancePools
+		*out = new(int64)
+		**out = **in
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MixedInstancesPolicySpec.
+func (in *MixedInstancesPolicySpec) DeepCopy() *MixedInstancesPolicySpec {
+	if in == nil {
+		return nil
+	}
+	out := new(MixedInstancesPolicySpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *NetworkingSpec) DeepCopyInto(out *NetworkingSpec) {
 	*out = *in
@ -2445,6 +2554,166 @@ func (in *NodeAuthorizerSpec) DeepCopy() *NodeAuthorizerSpec {
 	return out
 }

+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *OpenstackBlockStorageConfig) DeepCopyInto(out *OpenstackBlockStorageConfig) {
+	*out = *in
+	if in.Version != nil {
+		in, out := &in.Version, &out.Version
+		*out = new(string)
+		**out = **in
+	}
+	if in.IgnoreAZ != nil {
+		in, out := &in.IgnoreAZ, &out.IgnoreAZ
+		*out = new(bool)
+		**out = **in
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OpenstackBlockStorageConfig.
+func (in *OpenstackBlockStorageConfig) DeepCopy() *OpenstackBlockStorageConfig {
+	if in == nil {
+		return nil
+	}
+	out := new(OpenstackBlockStorageConfig)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *OpenstackConfiguration) DeepCopyInto(out *OpenstackConfiguration) {
+	*out = *in
+	if in.Loadbalancer != nil {
+		in, out := &in.Loadbalancer, &out.Loadbalancer
+		*out = new(OpenstackLoadbalancerConfig)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.Monitor != nil {
+		in, out := &in.Monitor, &out.Monitor
+		*out = new(OpenstackMonitor)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.Router != nil {
+		in, out := &in.Router, &out.Router
+		*out = new(OpenstackRouter)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.BlockStorage != nil {
+		in, out := &in.BlockStorage, &out.BlockStorage
+		*out = new(OpenstackBlockStorageConfig)
+		(*in).DeepCopyInto(*out)
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OpenstackConfiguration.
+func (in *OpenstackConfiguration) DeepCopy() *OpenstackConfiguration {
+	if in == nil {
+		return nil
+	}
+	out := new(OpenstackConfiguration)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *OpenstackLoadbalancerConfig) DeepCopyInto(out *OpenstackLoadbalancerConfig) {
+	*out = *in
+	if in.Method != nil {
+		in, out := &in.Method, &out.Method
+		*out = new(string)
+		**out = **in
+	}
+	if in.Provider != nil {
+		in, out := &in.Provider, &out.Provider
+		*out = new(string)
+		**out = **in
+	}
+	if in.UseOctavia != nil {
+		in, out := &in.UseOctavia, &out.UseOctavia
+		*out = new(bool)
+		**out = **in
+	}
+	if in.FloatingNetwork != nil {
+		in, out := &in.FloatingNetwork, &out.FloatingNetwork
+		*out = new(string)
+		**out = **in
+	}
+	if in.FloatingNetworkID != nil {
+		in, out := &in.FloatingNetworkID, &out.FloatingNetworkID
+		*out = new(string)
+		**out = **in
+	}
+	if in.SubnetID != nil {
+		in, out := &in.SubnetID, &out.SubnetID
+		*out = new(string)
+		**out = **in
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OpenstackLoadbalancerConfig.
+func (in *OpenstackLoadbalancerConfig) DeepCopy() *OpenstackLoadbalancerConfig {
+	if in == nil {
+		return nil
+	}
+	out := new(OpenstackLoadbalancerConfig)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *OpenstackMonitor) DeepCopyInto(out *OpenstackMonitor) {
+	*out = *in
+	if in.Delay != nil {
+		in, out := &in.Delay, &out.Delay
+		*out = new(string)
+		**out = **in
+	}
+	if in.Timeout != nil {
+		in, out := &in.Timeout, &out.Timeout
+		*out = new(string)
+		**out = **in
+	}
+	if in.MaxRetries != nil {
+		in, out := &in.MaxRetries, &out.MaxRetries
+		*out = new(int)
+		**out = **in
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OpenstackMonitor.
+func (in *OpenstackMonitor) DeepCopy() *OpenstackMonitor {
+	if in == nil {
+		return nil
+	}
+	out := new(OpenstackMonitor)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *OpenstackRouter) DeepCopyInto(out *OpenstackRouter) {
+	*out = *in
+	if in.ExternalNetwork != nil {
+		in, out := &in.ExternalNetwork, &out.ExternalNetwork
+		*out = new(string)
+		**out = **in
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OpenstackRouter.
+func (in *OpenstackRouter) DeepCopy() *OpenstackRouter {
+	if in == nil {
+		return nil
+	}
+	out := new(OpenstackRouter)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *RBACAuthorizationSpec) DeepCopyInto(out *RBACAuthorizationSpec) {
 	*out = *in
@ -2643,6 +2912,58 @@ func (in *UserData) DeepCopy() *UserData {
 	return out
 }

+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *VolumeMountSpec) DeepCopyInto(out *VolumeMountSpec) {
+	*out = *in
+	if in.FormatOptions != nil {
+		in, out := &in.FormatOptions, &out.FormatOptions
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
+	if in.MountOptions != nil {
+		in, out := &in.MountOptions, &out.MountOptions
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VolumeMountSpec.
+func (in *VolumeMountSpec) DeepCopy() *VolumeMountSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(VolumeMountSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *VolumeSpec) DeepCopyInto(out *VolumeSpec) {
+	*out = *in
+	if in.Encrypted != nil {
+		in, out := &in.Encrypted, &out.Encrypted
+		*out = new(bool)
+		**out = **in
+	}
+	if in.Iops != nil {
+		in, out := &in.Iops, &out.Iops
+		*out = new(int64)
+		**out = **in
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VolumeSpec.
+func (in *VolumeSpec) DeepCopy() *VolumeSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(VolumeSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *WeaveNetworkingSpec) DeepCopyInto(out *WeaveNetworkingSpec) {
 	*out = *in
--- a/pkg/apis/kops/v1alpha2/componentconfig.go
+++ b/pkg/apis/kops/v1alpha2/componentconfig.go
@ -179,6 +179,10 @@ type KubeletConfigSpec struct {
 	AuthenticationTokenWebhook *bool `json:"authenticationTokenWebhook,omitempty" flag:"authentication-token-webhook"`
 	// AuthenticationTokenWebhook sets the duration to cache responses from the webhook token authenticator. Default is 2m. (default 2m0s)
 	AuthenticationTokenWebhookCacheTTL *metav1.Duration `json:"authenticationTokenWebhookCacheTtl,omitempty" flag:"authentication-token-webhook-cache-ttl"`
+	// CPUCFSQuota enables CPU CFS quota enforcement for containers that specify CPU limits
+	CPUCFSQuota *bool `json:"cpuCFSQuota,omitempty" flag:"cpu-cfs-quota"`
+	// CPUCFSQuotaPeriod sets CPU CFS quota period value, cpu.cfs_period_us, defaults to Linux Kernel default
+	CPUCFSQuotaPeriod *metav1.Duration `json:"cpuCFSQuotaPeriod,omitempty" flag:"cpu-cfs-quota-period"`
 }

 // KubeProxyConfig defines the configuration for a proxy
@ -314,6 +318,10 @@ type KubeAPIServerConfig struct {
 	// OIDCClientID is the client ID for the OpenID Connect client, must be set
 	// if oidc-issuer-url is set.
 	OIDCClientID *string `json:"oidcClientID,omitempty" flag:"oidc-client-id"`
+	// A key=value pair that describes a required claim in the ID Token.
+	// If set, the claim is verified to be present in the ID Token with a matching value.
+	// Repeat this flag to specify multiple claims.
+	OIDCRequiredClaim []string `json:"oidcRequiredClaim,omitempty" flag:"oidc-required-claim,repeat"`
 	// OIDCCAFile if set, the OpenID server's certificate will be verified by one
 	// of the authorities in the oidc-ca-file
 	OIDCCAFile *string `json:"oidcCAFile,omitempty" flag:"oidc-ca-file"`
@ -493,6 +501,41 @@ type LeaderElectionConfiguration struct {
 	LeaderElect *bool `json:"leaderElect,omitempty" flag:"leader-elect"`
 }

+// OpenstackLoadbalancerConfig defines the config for a neutron loadbalancer
+type OpenstackLoadbalancerConfig struct {
+	Method            *string `json:"method,omitempty"`
+	Provider          *string `json:"provider,omitempty"`
+	UseOctavia        *bool   `json:"useOctavia,omitempty"`
+	FloatingNetwork   *string `json:"floatingNetwork,omitempty"`
+	FloatingNetworkID *string `json:"floatingNetworkID,omitempty"`
+	SubnetID          *string `json:"subnetID,omitempty"`
+}
+
+type OpenstackBlockStorageConfig struct {
+	Version  *string `json:"bs-version,omitempty"`
+	IgnoreAZ *bool   `json:"ignore-volume-az,omitempty"`
+}
+
+// OpenstackMonitor defines the config for a health monitor
+type OpenstackMonitor struct {
+	Delay      *string `json:"delay,omitempty"`
+	Timeout    *string `json:"timeout,omitempty"`
+	MaxRetries *int    `json:"maxRetries,omitempty"`
+}
+
+// OpenstackRouter defines the config for a router
+type OpenstackRouter struct {
+	ExternalNetwork *string `json:"externalNetwork,omitempty"`
+}
+
+// OpenstackConfiguration defines cloud config elements for the openstack cloud provider
+type OpenstackConfiguration struct {
+	Loadbalancer *OpenstackLoadbalancerConfig `json:"loadbalancer,omitempty"`
+	Monitor      *OpenstackMonitor            `json:"monitor,omitempty"`
+	Router       *OpenstackRouter             `json:"router,omitempty"`
+	BlockStorage *OpenstackBlockStorageConfig `json:"blockStorage,omitempty"`
+}
+
 // CloudConfiguration defines the cloud provider configuration
 type CloudConfiguration struct {
 	// GCE cloud-config options
@ -513,6 +556,8 @@ type CloudConfiguration struct {
 	// Spotinst cloud-config specs
 	SpotinstProduct     *string `json:"spotinstProduct,omitempty"`
 	SpotinstOrientation *string `json:"spotinstOrientation,omitempty"`
+	// Openstack cloud-config options
+	Openstack *OpenstackConfiguration `json:"openstack,omitempty"`
 }

 // HasAdmissionController checks if a specific admission controller is enabled
--- a/pkg/apis/kops/v1alpha2/instancegroup.go
+++ b/pkg/apis/kops/v1alpha2/instancegroup.go
@ -33,6 +33,7 @@ type InstanceGroup struct {

 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

+// InstanceGroupList is a list of instance groups
 type InstanceGroupList struct {
 	metav1.TypeMeta `json:",inline"`
 	metav1.ListMeta `json:"metadata,omitempty"`
@ -44,17 +45,35 @@ type InstanceGroupList struct {
 type InstanceGroupRole string

 const (
-	InstanceGroupRoleMaster  InstanceGroupRole = "Master"
-	InstanceGroupRoleNode    InstanceGroupRole = "Node"
+	// InstanceGroupRoleMaster is a master role
+	InstanceGroupRoleMaster InstanceGroupRole = "Master"
+	// InstanceGroupRoleNode is a node role
+	InstanceGroupRoleNode InstanceGroupRole = "Node"
+	// InstanceGroupRoleBastion is a bastion role
 	InstanceGroupRoleBastion InstanceGroupRole = "Bastion"
 )

+// AllInstanceGroupRoles is a list of all available roles
 var AllInstanceGroupRoles = []InstanceGroupRole{
-	InstanceGroupRoleNode,
-	InstanceGroupRoleMaster,
 	InstanceGroupRoleBastion,
+	InstanceGroupRoleMaster,
+	InstanceGroupRoleNode,
 }

+const (
+	// BtfsFilesystem indicates a btfs filesystem
+	BtfsFilesystem = "btfs"
+	// Ext4Filesystem indicates a ext3 filesystem
+	Ext4Filesystem = "ext4"
+	// XFSFilesystem indicates a xfs filesystem
+	XFSFilesystem = "xfs"
+)
+
+var (
+	// SupportedFilesystems is a list of supported filesystems to format as
+	SupportedFilesystems = []string{BtfsFilesystem, Ext4Filesystem, XFSFilesystem}
+)
+
 // InstanceGroupSpec is the specification for an instanceGroup
 type InstanceGroupSpec struct {
 	// Type determines the role of instances in this group: masters or nodes
@ -75,6 +94,10 @@ type InstanceGroupSpec struct {
 	RootVolumeIops *int32 `json:"rootVolumeIops,omitempty"`
 	// RootVolumeOptimization enables EBS optimization for an instance
 	RootVolumeOptimization *bool `json:"rootVolumeOptimization,omitempty"`
+	// Volumes is a collection of additional volumes to create for instances within this InstanceGroup
+	Volumes []*VolumeSpec `json:"volumes,omitempty"`
+	// VolumeMounts a collection of volume mounts
+	VolumeMounts []*VolumeMountSpec `json:"volumeMounts,omitempty"`
 	// Subnets is the names of the Subnets (as specified in the Cluster) where machines in this instance group should be placed
 	Subnets []string `json:"subnets,omitempty"`
 	// Zones is the names of the Zones where machines in this instance group should be placed
@ -101,6 +124,8 @@ type InstanceGroupSpec struct {
 	Kubelet *KubeletConfigSpec `json:"kubelet,omitempty"`
 	// Taints indicates the kubernetes taints for nodes in this group
 	Taints []string `json:"taints,omitempty"`
+	// MixedInstancesPolicy defined a optional backing of an AWS ASG by a EC2 Fleet (AWS Only)
+	MixedInstancesPolicy *MixedInstancesPolicySpec `json:"mixedInstancesPolicy,omitempty"`
 	// AdditionalUserData is any additional user-data to be passed to the host
 	AdditionalUserData []UserData `json:"additionalUserData,omitempty"`
 	// SuspendProcesses disables the listed Scaling Policies
@ -115,6 +140,39 @@ type InstanceGroupSpec struct {
 	SecurityGroupOverride *string `json:"securityGroupOverride,omitempty"`
 }

+const (
+	// SpotAllocationStrategyLowestPrices indicates a lowest price strategy
+	SpotAllocationStrategyLowestPrices = "LowestPrice"
+	// SpotAllocationStrategyDiversified indicates a diversified strategy
+	SpotAllocationStrategyDiversified = "Diversified"
+)
+
+// SpotAllocationStrategies is a collection of supported strategies
+var SpotAllocationStrategies = []string{SpotAllocationStrategyLowestPrices, SpotAllocationStrategyDiversified}
+
+// MixedInstancesPolicySpec defines the specification for an autoscaling backed by a ec2 fleet
+type MixedInstancesPolicySpec struct {
+	// Instances is a list of instance types which we are willing to run in the EC2 fleet
+	Instances []string `json:"instances,omitempty"`
+	// OnDemandAllocationStrategy indicates how to allocate instance types to fulfill On-Demand capacity
+	OnDemandAllocationStrategy *string `json:"onDemandAllocationStrategy,omitempty"`
+	// OnDemandBase is the minimum amount of the Auto Scaling group's capacity that must be
+	// fulfilled by On-Demand Instances. This base portion is provisioned first as your group scales.
+	OnDemandBase *int64 `json:"onDemandBase,omitempty"`
+	// OnDemandAboveBase controls the percentages of On-Demand Instances and Spot Instances for your
+	// additional capacity beyond OnDemandBase. The range is 0–100. The default value is 100. If you
+	// leave this parameter set to 100, the percentages are 100% for On-Demand Instances and 0% for
+	// Spot Instances.
+	OnDemandAboveBase *int64 `json:"onDemandAboveBase,omitempty"`
+	// SpotAllocationStrategy diversifies your Spot capacity across multiple instance types to
+	// find the best pricing. Higher Spot availability may result from a larger number of
+	// instance types to choose from.
+	SpotAllocationStrategy *string `json:"spotAllocationStrategy,omitempty"`
+	// SpotInstancePools is the number of Spot pools to use to allocate your Spot capacity (defaults to 2)
+	// pools are determined from the different instance types in the Overrides array of LaunchTemplate
+	SpotInstancePools *int64 `json:"spotInstancePools,omitempty"`
+}
+
 // UserData defines a user-data section
 type UserData struct {
 	// Name is the name of the user-data
@ -125,6 +183,34 @@ type UserData struct {
 	Content string `json:"content,omitempty"`
 }

+// VolumeSpec defined the spec for an additional volume attached to the instance group
+type VolumeSpec struct {
+	// Device is an optional device name of the block device
+	Device string `json:"device,omitempty"`
+	// Encrypted indicates you want to encrypt the volume
+	Encrypted *bool `json:"encrypted,omitempty"`
+	// Iops is the provision iops for this iops (think io1 in aws)
+	Iops *int64 `json:"iops,omitempty"`
+	// Size is the size of the volume in GB
+	Size int64 `json:"size,omitempty"`
+	// Type is the type of volume to create and is cloud specific
+	Type string `json:"type,omitempty"`
+}
+
+// VolumeMountSpec defines the specification for mounting a device
+type VolumeMountSpec struct {
+	// Device is the device name to provision and mount
+	Device string `json:"device,omitempty"`
+	// Filesystem is the filesystem to mount
+	Filesystem string `json:"filesystem,omitempty"`
+	// FormatOptions is a collection of options passed when formatting the device
+	FormatOptions []string `json:"formatOptions,omitempty"`
+	// MountOptions is a collection of mount options
+	MountOptions []string `json:"mountOptions,omitempty"`
+	// Path is the location to mount the device
+	Path string `json:"path,omitempty"`
+}
+
 // IAMProfileSpec is the AWS IAM Profile to attach to instances in this instance
 // group. Specify the ARN for the IAM instance profile (AWS only).
 type IAMProfileSpec struct {
--- a/pkg/apis/kops/v1alpha2/networking.go
+++ b/pkg/apis/kops/v1alpha2/networking.go
@ -102,6 +102,9 @@ type CanalNetworkingSpec struct {
 	// for traffic between pod to host after calico rules have been processed.
 	// Default: ACCEPT (other options: DROP, RETURN)
 	DefaultEndpointToHostAction string `json:"defaultEndpointToHostAction,omitempty"`
+	// DisableFlannelForwardRules configures Flannel to NOT add the
+	// default ACCEPT traffic rules to the iptables FORWARD chain
+	DisableFlannelForwardRules bool `json:"disableFlannelForwardRules,omitempty"`
 	// LogSeveritySys the severity to set for logs which are sent to syslog
 	// Default: INFO (other options: DEBUG, WARNING, ERROR, CRITICAL, NONE)
 	LogSeveritySys string `json:"logSeveritySys,omitempty"`
--- a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go
+++ b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go
@ -613,6 +613,16 @@ func RegisterConversions(s *runtime.Scheme) error {
 	}); err != nil {
 		return err
 	}
+	if err := s.AddGeneratedConversionFunc((*MixedInstancesPolicySpec)(nil), (*kops.MixedInstancesPolicySpec)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_v1alpha2_MixedInstancesPolicySpec_To_kops_MixedInstancesPolicySpec(a.(*MixedInstancesPolicySpec), b.(*kops.MixedInstancesPolicySpec), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*kops.MixedInstancesPolicySpec)(nil), (*MixedInstancesPolicySpec)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_kops_MixedInstancesPolicySpec_To_v1alpha2_MixedInstancesPolicySpec(a.(*kops.MixedInstancesPolicySpec), b.(*MixedInstancesPolicySpec), scope)
+	}); err != nil {
+		return err
+	}
 	if err := s.AddGeneratedConversionFunc((*NetworkingSpec)(nil), (*kops.NetworkingSpec)(nil), func(a, b interface{}, scope conversion.Scope) error {
 		return Convert_v1alpha2_NetworkingSpec_To_kops_NetworkingSpec(a.(*NetworkingSpec), b.(*kops.NetworkingSpec), scope)
 	}); err != nil {
@ -643,6 +653,56 @@ func RegisterConversions(s *runtime.Scheme) error {
 	}); err != nil {
 		return err
 	}
+	if err := s.AddGeneratedConversionFunc((*OpenstackBlockStorageConfig)(nil), (*kops.OpenstackBlockStorageConfig)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_v1alpha2_OpenstackBlockStorageConfig_To_kops_OpenstackBlockStorageConfig(a.(*OpenstackBlockStorageConfig), b.(*kops.OpenstackBlockStorageConfig), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*kops.OpenstackBlockStorageConfig)(nil), (*OpenstackBlockStorageConfig)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_kops_OpenstackBlockStorageConfig_To_v1alpha2_OpenstackBlockStorageConfig(a.(*kops.OpenstackBlockStorageConfig), b.(*OpenstackBlockStorageConfig), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*OpenstackConfiguration)(nil), (*kops.OpenstackConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_v1alpha2_OpenstackConfiguration_To_kops_OpenstackConfiguration(a.(*OpenstackConfiguration), b.(*kops.OpenstackConfiguration), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*kops.OpenstackConfiguration)(nil), (*OpenstackConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_kops_OpenstackConfiguration_To_v1alpha2_OpenstackConfiguration(a.(*kops.OpenstackConfiguration), b.(*OpenstackConfiguration), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*OpenstackLoadbalancerConfig)(nil), (*kops.OpenstackLoadbalancerConfig)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_v1alpha2_OpenstackLoadbalancerConfig_To_kops_OpenstackLoadbalancerConfig(a.(*OpenstackLoadbalancerConfig), b.(*kops.OpenstackLoadbalancerConfig), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*kops.OpenstackLoadbalancerConfig)(nil), (*OpenstackLoadbalancerConfig)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_kops_OpenstackLoadbalancerConfig_To_v1alpha2_OpenstackLoadbalancerConfig(a.(*kops.OpenstackLoadbalancerConfig), b.(*OpenstackLoadbalancerConfig), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*OpenstackMonitor)(nil), (*kops.OpenstackMonitor)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_v1alpha2_OpenstackMonitor_To_kops_OpenstackMonitor(a.(*OpenstackMonitor), b.(*kops.OpenstackMonitor), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*kops.OpenstackMonitor)(nil), (*OpenstackMonitor)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_kops_OpenstackMonitor_To_v1alpha2_OpenstackMonitor(a.(*kops.OpenstackMonitor), b.(*OpenstackMonitor), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*OpenstackRouter)(nil), (*kops.OpenstackRouter)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_v1alpha2_OpenstackRouter_To_kops_OpenstackRouter(a.(*OpenstackRouter), b.(*kops.OpenstackRouter), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*kops.OpenstackRouter)(nil), (*OpenstackRouter)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_kops_OpenstackRouter_To_v1alpha2_OpenstackRouter(a.(*kops.OpenstackRouter), b.(*OpenstackRouter), scope)
+	}); err != nil {
+		return err
+	}
 	if err := s.AddGeneratedConversionFunc((*RBACAuthorizationSpec)(nil), (*kops.RBACAuthorizationSpec)(nil), func(a, b interface{}, scope conversion.Scope) error {
 		return Convert_v1alpha2_RBACAuthorizationSpec_To_kops_RBACAuthorizationSpec(a.(*RBACAuthorizationSpec), b.(*kops.RBACAuthorizationSpec), scope)
 	}); err != nil {
@ -733,6 +793,26 @@ func RegisterConversions(s *runtime.Scheme) error {
 	}); err != nil {
 		return err
 	}
+	if err := s.AddGeneratedConversionFunc((*VolumeMountSpec)(nil), (*kops.VolumeMountSpec)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_v1alpha2_VolumeMountSpec_To_kops_VolumeMountSpec(a.(*VolumeMountSpec), b.(*kops.VolumeMountSpec), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*kops.VolumeMountSpec)(nil), (*VolumeMountSpec)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_kops_VolumeMountSpec_To_v1alpha2_VolumeMountSpec(a.(*kops.VolumeMountSpec), b.(*VolumeMountSpec), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*VolumeSpec)(nil), (*kops.VolumeSpec)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_v1alpha2_VolumeSpec_To_kops_VolumeSpec(a.(*VolumeSpec), b.(*kops.VolumeSpec), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*kops.VolumeSpec)(nil), (*VolumeSpec)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_kops_VolumeSpec_To_v1alpha2_VolumeSpec(a.(*kops.VolumeSpec), b.(*VolumeSpec), scope)
+	}); err != nil {
+		return err
+	}
 	if err := s.AddGeneratedConversionFunc((*WeaveNetworkingSpec)(nil), (*kops.WeaveNetworkingSpec)(nil), func(a, b interface{}, scope conversion.Scope) error {
 		return Convert_v1alpha2_WeaveNetworkingSpec_To_kops_WeaveNetworkingSpec(a.(*WeaveNetworkingSpec), b.(*kops.WeaveNetworkingSpec), scope)
 	}); err != nil {
@ -1087,6 +1167,7 @@ func Convert_kops_CalicoNetworkingSpec_To_v1alpha2_CalicoNetworkingSpec(in *kops
 func autoConvert_v1alpha2_CanalNetworkingSpec_To_kops_CanalNetworkingSpec(in *CanalNetworkingSpec, out *kops.CanalNetworkingSpec, s conversion.Scope) error {
 	out.ChainInsertMode = in.ChainInsertMode
 	out.DefaultEndpointToHostAction = in.DefaultEndpointToHostAction
+	out.DisableFlannelForwardRules = in.DisableFlannelForwardRules
 	out.LogSeveritySys = in.LogSeveritySys
 	out.PrometheusGoMetricsEnabled = in.PrometheusGoMetricsEnabled
 	out.PrometheusMetricsEnabled = in.PrometheusMetricsEnabled
@ -1103,6 +1184,7 @@ func Convert_v1alpha2_CanalNetworkingSpec_To_kops_CanalNetworkingSpec(in *CanalN
 func autoConvert_kops_CanalNetworkingSpec_To_v1alpha2_CanalNetworkingSpec(in *kops.CanalNetworkingSpec, out *CanalNetworkingSpec, s conversion.Scope) error {
 	out.ChainInsertMode = in.ChainInsertMode
 	out.DefaultEndpointToHostAction = in.DefaultEndpointToHostAction
+	out.DisableFlannelForwardRules = in.DisableFlannelForwardRules
 	out.LogSeveritySys = in.LogSeveritySys
 	out.PrometheusGoMetricsEnabled = in.PrometheusGoMetricsEnabled
 	out.PrometheusMetricsEnabled = in.PrometheusMetricsEnabled
@ -1265,6 +1347,15 @@ func autoConvert_v1alpha2_CloudConfiguration_To_kops_CloudConfiguration(in *Clou
 	out.VSphereCoreDNSServer = in.VSphereCoreDNSServer
 	out.SpotinstProduct = in.SpotinstProduct
 	out.SpotinstOrientation = in.SpotinstOrientation
+	if in.Openstack != nil {
+		in, out := &in.Openstack, &out.Openstack
+		*out = new(kops.OpenstackConfiguration)
+		if err := Convert_v1alpha2_OpenstackConfiguration_To_kops_OpenstackConfiguration(*in, *out, s); err != nil {
+			return err
+		}
+	} else {
+		out.Openstack = nil
+	}
 	return nil
 }

@ -1288,6 +1379,15 @@ func autoConvert_kops_CloudConfiguration_To_v1alpha2_CloudConfiguration(in *kops
 	out.VSphereCoreDNSServer = in.VSphereCoreDNSServer
 	out.SpotinstProduct = in.SpotinstProduct
 	out.SpotinstOrientation = in.SpotinstOrientation
+	if in.Openstack != nil {
+		in, out := &in.Openstack, &out.Openstack
+		*out = new(OpenstackConfiguration)
+		if err := Convert_kops_OpenstackConfiguration_To_v1alpha2_OpenstackConfiguration(*in, *out, s); err != nil {
+			return err
+		}
+	} else {
+		out.Openstack = nil
+	}
 	return nil
 }

@ -2654,6 +2754,30 @@ func autoConvert_v1alpha2_InstanceGroupSpec_To_kops_InstanceGroupSpec(in *Instan
 	out.RootVolumeType = in.RootVolumeType
 	out.RootVolumeIops = in.RootVolumeIops
 	out.RootVolumeOptimization = in.RootVolumeOptimization
+	if in.Volumes != nil {
+		in, out := &in.Volumes, &out.Volumes
+		*out = make([]*kops.VolumeSpec, len(*in))
+		for i := range *in {
+			// TODO: Inefficient conversion - can we improve it?
+			if err := s.Convert(&(*in)[i], &(*out)[i], 0); err != nil {
+				return err
+			}
+		}
+	} else {
+		out.Volumes = nil
+	}
+	if in.VolumeMounts != nil {
+		in, out := &in.VolumeMounts, &out.VolumeMounts
+		*out = make([]*kops.VolumeMountSpec, len(*in))
+		for i := range *in {
+			// TODO: Inefficient conversion - can we improve it?
+			if err := s.Convert(&(*in)[i], &(*out)[i], 0); err != nil {
+				return err
+			}
+		}
+	} else {
+		out.VolumeMounts = nil
+	}
 	out.Subnets = in.Subnets
 	out.Zones = in.Zones
 	if in.Hooks != nil {
@ -2694,6 +2818,15 @@ func autoConvert_v1alpha2_InstanceGroupSpec_To_kops_InstanceGroupSpec(in *Instan
 		out.Kubelet = nil
 	}
 	out.Taints = in.Taints
+	if in.MixedInstancesPolicy != nil {
+		in, out := &in.MixedInstancesPolicy, &out.MixedInstancesPolicy
+		*out = new(kops.MixedInstancesPolicySpec)
+		if err := Convert_v1alpha2_MixedInstancesPolicySpec_To_kops_MixedInstancesPolicySpec(*in, *out, s); err != nil {
+			return err
+		}
+	} else {
+		out.MixedInstancesPolicy = nil
+	}
 	if in.AdditionalUserData != nil {
 		in, out := &in.AdditionalUserData, &out.AdditionalUserData
 		*out = make([]kops.UserData, len(*in))
@ -2746,6 +2879,30 @@ func autoConvert_kops_InstanceGroupSpec_To_v1alpha2_InstanceGroupSpec(in *kops.I
 	out.RootVolumeType = in.RootVolumeType
 	out.RootVolumeIops = in.RootVolumeIops
 	out.RootVolumeOptimization = in.RootVolumeOptimization
+	if in.Volumes != nil {
+		in, out := &in.Volumes, &out.Volumes
+		*out = make([]*VolumeSpec, len(*in))
+		for i := range *in {
+			// TODO: Inefficient conversion - can we improve it?
+			if err := s.Convert(&(*in)[i], &(*out)[i], 0); err != nil {
+				return err
+			}
+		}
+	} else {
+		out.Volumes = nil
+	}
+	if in.VolumeMounts != nil {
+		in, out := &in.VolumeMounts, &out.VolumeMounts
+		*out = make([]*VolumeMountSpec, len(*in))
+		for i := range *in {
+			// TODO: Inefficient conversion - can we improve it?
+			if err := s.Convert(&(*in)[i], &(*out)[i], 0); err != nil {
+				return err
+			}
+		}
+	} else {
+		out.VolumeMounts = nil
+	}
 	out.Subnets = in.Subnets
 	out.Zones = in.Zones
 	if in.Hooks != nil {
@ -2786,6 +2943,15 @@ func autoConvert_kops_InstanceGroupSpec_To_v1alpha2_InstanceGroupSpec(in *kops.I
 		out.Kubelet = nil
 	}
 	out.Taints = in.Taints
+	if in.MixedInstancesPolicy != nil {
+		in, out := &in.MixedInstancesPolicy, &out.MixedInstancesPolicy
+		*out = new(MixedInstancesPolicySpec)
+		if err := Convert_kops_MixedInstancesPolicySpec_To_v1alpha2_MixedInstancesPolicySpec(*in, *out, s); err != nil {
+			return err
+		}
+	} else {
+		out.MixedInstancesPolicy = nil
+	}
 	if in.AdditionalUserData != nil {
 		in, out := &in.AdditionalUserData, &out.AdditionalUserData
 		*out = make([]UserData, len(*in))
@ -3047,6 +3213,7 @@ func autoConvert_v1alpha2_KubeAPIServerConfig_To_kops_KubeAPIServerConfig(in *Ku
 	out.OIDCGroupsPrefix = in.OIDCGroupsPrefix
 	out.OIDCIssuerURL = in.OIDCIssuerURL
 	out.OIDCClientID = in.OIDCClientID
+	out.OIDCRequiredClaim = in.OIDCRequiredClaim
 	out.OIDCCAFile = in.OIDCCAFile
 	out.ProxyClientCertFile = in.ProxyClientCertFile
 	out.ProxyClientKeyFile = in.ProxyClientKeyFile
@ -3121,6 +3288,7 @@ func autoConvert_kops_KubeAPIServerConfig_To_v1alpha2_KubeAPIServerConfig(in *ko
 	out.OIDCGroupsPrefix = in.OIDCGroupsPrefix
 	out.OIDCIssuerURL = in.OIDCIssuerURL
 	out.OIDCClientID = in.OIDCClientID
+	out.OIDCRequiredClaim = in.OIDCRequiredClaim
 	out.OIDCCAFile = in.OIDCCAFile
 	out.ProxyClientCertFile = in.ProxyClientCertFile
 	out.ProxyClientKeyFile = in.ProxyClientKeyFile
@ -3447,6 +3615,8 @@ func autoConvert_v1alpha2_KubeletConfigSpec_To_kops_KubeletConfigSpec(in *Kubele
 	out.RootDir = in.RootDir
 	out.AuthenticationTokenWebhook = in.AuthenticationTokenWebhook
 	out.AuthenticationTokenWebhookCacheTTL = in.AuthenticationTokenWebhookCacheTTL
+	out.CPUCFSQuota = in.CPUCFSQuota
+	out.CPUCFSQuotaPeriod = in.CPUCFSQuotaPeriod
 	return nil
 }

@ -3524,6 +3694,8 @@ func autoConvert_kops_KubeletConfigSpec_To_v1alpha2_KubeletConfigSpec(in *kops.K
 	out.RootDir = in.RootDir
 	out.AuthenticationTokenWebhook = in.AuthenticationTokenWebhook
 	out.AuthenticationTokenWebhookCacheTTL = in.AuthenticationTokenWebhookCacheTTL
+	out.CPUCFSQuota = in.CPUCFSQuota
+	out.CPUCFSQuotaPeriod = in.CPUCFSQuotaPeriod
 	return nil
 }

@ -3660,6 +3832,36 @@ func Convert_kops_LyftVPCNetworkingSpec_To_v1alpha2_LyftVPCNetworkingSpec(in *ko
 	return autoConvert_kops_LyftVPCNetworkingSpec_To_v1alpha2_LyftVPCNetworkingSpec(in, out, s)
 }

+func autoConvert_v1alpha2_MixedInstancesPolicySpec_To_kops_MixedInstancesPolicySpec(in *MixedInstancesPolicySpec, out *kops.MixedInstancesPolicySpec, s conversion.Scope) error {
+	out.Instances = in.Instances
+	out.OnDemandAllocationStrategy = in.OnDemandAllocationStrategy
+	out.OnDemandBase = in.OnDemandBase
+	out.OnDemandAboveBase = in.OnDemandAboveBase
+	out.SpotAllocationStrategy = in.SpotAllocationStrategy
+	out.SpotInstancePools = in.SpotInstancePools
+	return nil
+}
+
+// Convert_v1alpha2_MixedInstancesPolicySpec_To_kops_MixedInstancesPolicySpec is an autogenerated conversion function.
+func Convert_v1alpha2_MixedInstancesPolicySpec_To_kops_MixedInstancesPolicySpec(in *MixedInstancesPolicySpec, out *kops.MixedInstancesPolicySpec, s conversion.Scope) error {
+	return autoConvert_v1alpha2_MixedInstancesPolicySpec_To_kops_MixedInstancesPolicySpec(in, out, s)
+}
+
+func autoConvert_kops_MixedInstancesPolicySpec_To_v1alpha2_MixedInstancesPolicySpec(in *kops.MixedInstancesPolicySpec, out *MixedInstancesPolicySpec, s conversion.Scope) error {
+	out.Instances = in.Instances
+	out.OnDemandAllocationStrategy = in.OnDemandAllocationStrategy
+	out.OnDemandBase = in.OnDemandBase
+	out.OnDemandAboveBase = in.OnDemandAboveBase
+	out.SpotAllocationStrategy = in.SpotAllocationStrategy
+	out.SpotInstancePools = in.SpotInstancePools
+	return nil
+}
+
+// Convert_kops_MixedInstancesPolicySpec_To_v1alpha2_MixedInstancesPolicySpec is an autogenerated conversion function.
+func Convert_kops_MixedInstancesPolicySpec_To_v1alpha2_MixedInstancesPolicySpec(in *kops.MixedInstancesPolicySpec, out *MixedInstancesPolicySpec, s conversion.Scope) error {
+	return autoConvert_kops_MixedInstancesPolicySpec_To_v1alpha2_MixedInstancesPolicySpec(in, out, s)
+}
+
 func autoConvert_v1alpha2_NetworkingSpec_To_kops_NetworkingSpec(in *NetworkingSpec, out *kops.NetworkingSpec, s conversion.Scope) error {
 	if in.Classic != nil {
 		in, out := &in.Classic, &out.Classic
@ -3998,6 +4200,192 @@ func Convert_kops_NodeAuthorizerSpec_To_v1alpha2_NodeAuthorizerSpec(in *kops.Nod
 	return autoConvert_kops_NodeAuthorizerSpec_To_v1alpha2_NodeAuthorizerSpec(in, out, s)
 }

+func autoConvert_v1alpha2_OpenstackBlockStorageConfig_To_kops_OpenstackBlockStorageConfig(in *OpenstackBlockStorageConfig, out *kops.OpenstackBlockStorageConfig, s conversion.Scope) error {
+	out.Version = in.Version
+	out.IgnoreAZ = in.IgnoreAZ
+	return nil
+}
+
+// Convert_v1alpha2_OpenstackBlockStorageConfig_To_kops_OpenstackBlockStorageConfig is an autogenerated conversion function.
+func Convert_v1alpha2_OpenstackBlockStorageConfig_To_kops_OpenstackBlockStorageConfig(in *OpenstackBlockStorageConfig, out *kops.OpenstackBlockStorageConfig, s conversion.Scope) error {
+	return autoConvert_v1alpha2_OpenstackBlockStorageConfig_To_kops_OpenstackBlockStorageConfig(in, out, s)
+}
+
+func autoConvert_kops_OpenstackBlockStorageConfig_To_v1alpha2_OpenstackBlockStorageConfig(in *kops.OpenstackBlockStorageConfig, out *OpenstackBlockStorageConfig, s conversion.Scope) error {
+	out.Version = in.Version
+	out.IgnoreAZ = in.IgnoreAZ
+	return nil
+}
+
+// Convert_kops_OpenstackBlockStorageConfig_To_v1alpha2_OpenstackBlockStorageConfig is an autogenerated conversion function.
+func Convert_kops_OpenstackBlockStorageConfig_To_v1alpha2_OpenstackBlockStorageConfig(in *kops.OpenstackBlockStorageConfig, out *OpenstackBlockStorageConfig, s conversion.Scope) error {
+	return autoConvert_kops_OpenstackBlockStorageConfig_To_v1alpha2_OpenstackBlockStorageConfig(in, out, s)
+}
+
+func autoConvert_v1alpha2_OpenstackConfiguration_To_kops_OpenstackConfiguration(in *OpenstackConfiguration, out *kops.OpenstackConfiguration, s conversion.Scope) error {
+	if in.Loadbalancer != nil {
+		in, out := &in.Loadbalancer, &out.Loadbalancer
+		*out = new(kops.OpenstackLoadbalancerConfig)
+		if err := Convert_v1alpha2_OpenstackLoadbalancerConfig_To_kops_OpenstackLoadbalancerConfig(*in, *out, s); err != nil {
+			return err
+		}
+	} else {
+		out.Loadbalancer = nil
+	}
+	if in.Monitor != nil {
+		in, out := &in.Monitor, &out.Monitor
+		*out = new(kops.OpenstackMonitor)
+		if err := Convert_v1alpha2_OpenstackMonitor_To_kops_OpenstackMonitor(*in, *out, s); err != nil {
+			return err
+		}
+	} else {
+		out.Monitor = nil
+	}
+	if in.Router != nil {
+		in, out := &in.Router, &out.Router
+		*out = new(kops.OpenstackRouter)
+		if err := Convert_v1alpha2_OpenstackRouter_To_kops_OpenstackRouter(*in, *out, s); err != nil {
+			return err
+		}
+	} else {
+		out.Router = nil
+	}
+	if in.BlockStorage != nil {
+		in, out := &in.BlockStorage, &out.BlockStorage
+		*out = new(kops.OpenstackBlockStorageConfig)
+		if err := Convert_v1alpha2_OpenstackBlockStorageConfig_To_kops_OpenstackBlockStorageConfig(*in, *out, s); err != nil {
+			return err
+		}
+	} else {
+		out.BlockStorage = nil
+	}
+	return nil
+}
+
+// Convert_v1alpha2_OpenstackConfiguration_To_kops_OpenstackConfiguration is an autogenerated conversion function.
+func Convert_v1alpha2_OpenstackConfiguration_To_kops_OpenstackConfiguration(in *OpenstackConfiguration, out *kops.OpenstackConfiguration, s conversion.Scope) error {
+	return autoConvert_v1alpha2_OpenstackConfiguration_To_kops_OpenstackConfiguration(in, out, s)
+}
+
+func autoConvert_kops_OpenstackConfiguration_To_v1alpha2_OpenstackConfiguration(in *kops.OpenstackConfiguration, out *OpenstackConfiguration, s conversion.Scope) error {
+	if in.Loadbalancer != nil {
+		in, out := &in.Loadbalancer, &out.Loadbalancer
+		*out = new(OpenstackLoadbalancerConfig)
+		if err := Convert_kops_OpenstackLoadbalancerConfig_To_v1alpha2_OpenstackLoadbalancerConfig(*in, *out, s); err != nil {
+			return err
+		}
+	} else {
+		out.Loadbalancer = nil
+	}
+	if in.Monitor != nil {
+		in, out := &in.Monitor, &out.Monitor
+		*out = new(OpenstackMonitor)
+		if err := Convert_kops_OpenstackMonitor_To_v1alpha2_OpenstackMonitor(*in, *out, s); err != nil {
+			return err
+		}
+	} else {
+		out.Monitor = nil
+	}
+	if in.Router != nil {
+		in, out := &in.Router, &out.Router
+		*out = new(OpenstackRouter)
+		if err := Convert_kops_OpenstackRouter_To_v1alpha2_OpenstackRouter(*in, *out, s); err != nil {
+			return err
+		}
+	} else {
+		out.Router = nil
+	}
+	if in.BlockStorage != nil {
+		in, out := &in.BlockStorage, &out.BlockStorage
+		*out = new(OpenstackBlockStorageConfig)
+		if err := Convert_kops_OpenstackBlockStorageConfig_To_v1alpha2_OpenstackBlockStorageConfig(*in, *out, s); err != nil {
+			return err
+		}
+	} else {
+		out.BlockStorage = nil
+	}
+	return nil
+}
+
+// Convert_kops_OpenstackConfiguration_To_v1alpha2_OpenstackConfiguration is an autogenerated conversion function.
+func Convert_kops_OpenstackConfiguration_To_v1alpha2_OpenstackConfiguration(in *kops.OpenstackConfiguration, out *OpenstackConfiguration, s conversion.Scope) error {
+	return autoConvert_kops_OpenstackConfiguration_To_v1alpha2_OpenstackConfiguration(in, out, s)
+}
+
+func autoConvert_v1alpha2_OpenstackLoadbalancerConfig_To_kops_OpenstackLoadbalancerConfig(in *OpenstackLoadbalancerConfig, out *kops.OpenstackLoadbalancerConfig, s conversion.Scope) error {
+	out.Method = in.Method
+	out.Provider = in.Provider
+	out.UseOctavia = in.UseOctavia
+	out.FloatingNetwork = in.FloatingNetwork
+	out.FloatingNetworkID = in.FloatingNetworkID
+	out.SubnetID = in.SubnetID
+	return nil
+}
+
+// Convert_v1alpha2_OpenstackLoadbalancerConfig_To_kops_OpenstackLoadbalancerConfig is an autogenerated conversion function.
+func Convert_v1alpha2_OpenstackLoadbalancerConfig_To_kops_OpenstackLoadbalancerConfig(in *OpenstackLoadbalancerConfig, out *kops.OpenstackLoadbalancerConfig, s conversion.Scope) error {
+	return autoConvert_v1alpha2_OpenstackLoadbalancerConfig_To_kops_OpenstackLoadbalancerConfig(in, out, s)
+}
+
+func autoConvert_kops_OpenstackLoadbalancerConfig_To_v1alpha2_OpenstackLoadbalancerConfig(in *kops.OpenstackLoadbalancerConfig, out *OpenstackLoadbalancerConfig, s conversion.Scope) error {
+	out.Method = in.Method
+	out.Provider = in.Provider
+	out.UseOctavia = in.UseOctavia
+	out.FloatingNetwork = in.FloatingNetwork
+	out.FloatingNetworkID = in.FloatingNetworkID
+	out.SubnetID = in.SubnetID
+	return nil
+}
+
+// Convert_kops_OpenstackLoadbalancerConfig_To_v1alpha2_OpenstackLoadbalancerConfig is an autogenerated conversion function.
+func Convert_kops_OpenstackLoadbalancerConfig_To_v1alpha2_OpenstackLoadbalancerConfig(in *kops.OpenstackLoadbalancerConfig, out *OpenstackLoadbalancerConfig, s conversion.Scope) error {
+	return autoConvert_kops_OpenstackLoadbalancerConfig_To_v1alpha2_OpenstackLoadbalancerConfig(in, out, s)
+}
+
+func autoConvert_v1alpha2_OpenstackMonitor_To_kops_OpenstackMonitor(in *OpenstackMonitor, out *kops.OpenstackMonitor, s conversion.Scope) error {
+	out.Delay = in.Delay
+	out.Timeout = in.Timeout
+	out.MaxRetries = in.MaxRetries
+	return nil
+}
+
+// Convert_v1alpha2_OpenstackMonitor_To_kops_OpenstackMonitor is an autogenerated conversion function.
+func Convert_v1alpha2_OpenstackMonitor_To_kops_OpenstackMonitor(in *OpenstackMonitor, out *kops.OpenstackMonitor, s conversion.Scope) error {
+	return autoConvert_v1alpha2_OpenstackMonitor_To_kops_OpenstackMonitor(in, out, s)
+}
+
+func autoConvert_kops_OpenstackMonitor_To_v1alpha2_OpenstackMonitor(in *kops.OpenstackMonitor, out *OpenstackMonitor, s conversion.Scope) error {
+	out.Delay = in.Delay
+	out.Timeout = in.Timeout
+	out.MaxRetries = in.MaxRetries
+	return nil
+}
+
+// Convert_kops_OpenstackMonitor_To_v1alpha2_OpenstackMonitor is an autogenerated conversion function.
+func Convert_kops_OpenstackMonitor_To_v1alpha2_OpenstackMonitor(in *kops.OpenstackMonitor, out *OpenstackMonitor, s conversion.Scope) error {
+	return autoConvert_kops_OpenstackMonitor_To_v1alpha2_OpenstackMonitor(in, out, s)
+}
+
+func autoConvert_v1alpha2_OpenstackRouter_To_kops_OpenstackRouter(in *OpenstackRouter, out *kops.OpenstackRouter, s conversion.Scope) error {
+	out.ExternalNetwork = in.ExternalNetwork
+	return nil
+}
+
+// Convert_v1alpha2_OpenstackRouter_To_kops_OpenstackRouter is an autogenerated conversion function.
+func Convert_v1alpha2_OpenstackRouter_To_kops_OpenstackRouter(in *OpenstackRouter, out *kops.OpenstackRouter, s conversion.Scope) error {
+	return autoConvert_v1alpha2_OpenstackRouter_To_kops_OpenstackRouter(in, out, s)
+}
+
+func autoConvert_kops_OpenstackRouter_To_v1alpha2_OpenstackRouter(in *kops.OpenstackRouter, out *OpenstackRouter, s conversion.Scope) error {
+	out.ExternalNetwork = in.ExternalNetwork
+	return nil
+}
+
+// Convert_kops_OpenstackRouter_To_v1alpha2_OpenstackRouter is an autogenerated conversion function.
+func Convert_kops_OpenstackRouter_To_v1alpha2_OpenstackRouter(in *kops.OpenstackRouter, out *OpenstackRouter, s conversion.Scope) error {
+	return autoConvert_kops_OpenstackRouter_To_v1alpha2_OpenstackRouter(in, out, s)
+}
+
 func autoConvert_v1alpha2_RBACAuthorizationSpec_To_kops_RBACAuthorizationSpec(in *RBACAuthorizationSpec, out *kops.RBACAuthorizationSpec, s conversion.Scope) error {
 	return nil
 }
@ -4264,6 +4652,62 @@ func Convert_kops_UserData_To_v1alpha2_UserData(in *kops.UserData, out *UserData
 	return autoConvert_kops_UserData_To_v1alpha2_UserData(in, out, s)
 }

+func autoConvert_v1alpha2_VolumeMountSpec_To_kops_VolumeMountSpec(in *VolumeMountSpec, out *kops.VolumeMountSpec, s conversion.Scope) error {
+	out.Device = in.Device
+	out.Filesystem = in.Filesystem
+	out.FormatOptions = in.FormatOptions
+	out.MountOptions = in.MountOptions
+	out.Path = in.Path
+	return nil
+}
+
+// Convert_v1alpha2_VolumeMountSpec_To_kops_VolumeMountSpec is an autogenerated conversion function.
+func Convert_v1alpha2_VolumeMountSpec_To_kops_VolumeMountSpec(in *VolumeMountSpec, out *kops.VolumeMountSpec, s conversion.Scope) error {
+	return autoConvert_v1alpha2_VolumeMountSpec_To_kops_VolumeMountSpec(in, out, s)
+}
+
+func autoConvert_kops_VolumeMountSpec_To_v1alpha2_VolumeMountSpec(in *kops.VolumeMountSpec, out *VolumeMountSpec, s conversion.Scope) error {
+	out.Device = in.Device
+	out.Filesystem = in.Filesystem
+	out.FormatOptions = in.FormatOptions
+	out.MountOptions = in.MountOptions
+	out.Path = in.Path
+	return nil
+}
+
+// Convert_kops_VolumeMountSpec_To_v1alpha2_VolumeMountSpec is an autogenerated conversion function.
+func Convert_kops_VolumeMountSpec_To_v1alpha2_VolumeMountSpec(in *kops.VolumeMountSpec, out *VolumeMountSpec, s conversion.Scope) error {
+	return autoConvert_kops_VolumeMountSpec_To_v1alpha2_VolumeMountSpec(in, out, s)
+}
+
+func autoConvert_v1alpha2_VolumeSpec_To_kops_VolumeSpec(in *VolumeSpec, out *kops.VolumeSpec, s conversion.Scope) error {
+	out.Device = in.Device
+	out.Encrypted = in.Encrypted
+	out.Iops = in.Iops
+	out.Size = in.Size
+	out.Type = in.Type
+	return nil
+}
+
+// Convert_v1alpha2_VolumeSpec_To_kops_VolumeSpec is an autogenerated conversion function.
+func Convert_v1alpha2_VolumeSpec_To_kops_VolumeSpec(in *VolumeSpec, out *kops.VolumeSpec, s conversion.Scope) error {
+	return autoConvert_v1alpha2_VolumeSpec_To_kops_VolumeSpec(in, out, s)
+}
+
+func autoConvert_kops_VolumeSpec_To_v1alpha2_VolumeSpec(in *kops.VolumeSpec, out *VolumeSpec, s conversion.Scope) error {
+	out.Device = in.Device
+	out.Encrypted = in.Encrypted
+	out.Iops = in.Iops
+	out.Size = in.Size
+	out.Type = in.Type
+	return nil
+}
+
+// Convert_kops_VolumeSpec_To_v1alpha2_VolumeSpec is an autogenerated conversion function.
+func Convert_kops_VolumeSpec_To_v1alpha2_VolumeSpec(in *kops.VolumeSpec, out *VolumeSpec, s conversion.Scope) error {
+	return autoConvert_kops_VolumeSpec_To_v1alpha2_VolumeSpec(in, out, s)
+}
+
 func autoConvert_v1alpha2_WeaveNetworkingSpec_To_kops_WeaveNetworkingSpec(in *WeaveNetworkingSpec, out *kops.WeaveNetworkingSpec, s conversion.Scope) error {
 	out.MTU = in.MTU
 	out.ConnLimit = in.ConnLimit
--- a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go
+++ b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go
@ -416,6 +416,11 @@ func (in *CloudConfiguration) DeepCopyInto(out *CloudConfiguration) {
 		*out = new(string)
 		**out = **in
 	}
+	if in.Openstack != nil {
+		in, out := &in.Openstack, &out.Openstack
+		*out = new(OpenstackConfiguration)
+		(*in).DeepCopyInto(*out)
+	}
 	return
 }

@ -1349,6 +1354,28 @@ func (in *InstanceGroupSpec) DeepCopyInto(out *InstanceGroupSpec) {
 		*out = new(bool)
 		**out = **in
 	}
+	if in.Volumes != nil {
+		in, out := &in.Volumes, &out.Volumes
+		*out = make([]*VolumeSpec, len(*in))
+		for i := range *in {
+			if (*in)[i] != nil {
+				in, out := &(*in)[i], &(*out)[i]
+				*out = new(VolumeSpec)
+				(*in).DeepCopyInto(*out)
+			}
+		}
+	}
+	if in.VolumeMounts != nil {
+		in, out := &in.VolumeMounts, &out.VolumeMounts
+		*out = make([]*VolumeMountSpec, len(*in))
+		for i := range *in {
+			if (*in)[i] != nil {
+				in, out := &(*in)[i], &(*out)[i]
+				*out = new(VolumeMountSpec)
+				(*in).DeepCopyInto(*out)
+			}
+		}
+	}
 	if in.Subnets != nil {
 		in, out := &in.Subnets, &out.Subnets
 		*out = make([]string, len(*in))
@ -1412,6 +1439,11 @@ func (in *InstanceGroupSpec) DeepCopyInto(out *InstanceGroupSpec) {
 		*out = make([]string, len(*in))
 		copy(*out, *in)
 	}
+	if in.MixedInstancesPolicy != nil {
+		in, out := &in.MixedInstancesPolicy, &out.MixedInstancesPolicy
+		*out = new(MixedInstancesPolicySpec)
+		(*in).DeepCopyInto(*out)
+	}
 	if in.AdditionalUserData != nil {
 		in, out := &in.AdditionalUserData, &out.AdditionalUserData
 		*out = make([]UserData, len(*in))
@ -1698,6 +1730,11 @@ func (in *KubeAPIServerConfig) DeepCopyInto(out *KubeAPIServerConfig) {
 		*out = new(string)
 		**out = **in
 	}
+	if in.OIDCRequiredClaim != nil {
+		in, out := &in.OIDCRequiredClaim, &out.OIDCRequiredClaim
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
 	if in.OIDCCAFile != nil {
 		in, out := &in.OIDCCAFile, &out.OIDCCAFile
 		*out = new(string)
@ -2244,6 +2281,16 @@ func (in *KubeletConfigSpec) DeepCopyInto(out *KubeletConfigSpec) {
 		*out = new(v1.Duration)
 		**out = **in
 	}
+	if in.CPUCFSQuota != nil {
+		in, out := &in.CPUCFSQuota, &out.CPUCFSQuota
+		*out = new(bool)
+		**out = **in
+	}
+	if in.CPUCFSQuotaPeriod != nil {
+		in, out := &in.CPUCFSQuotaPeriod, &out.CPUCFSQuotaPeriod
+		*out = new(v1.Duration)
+		**out = **in
+	}
 	return
 }

@ -2390,6 +2437,52 @@ func (in *LyftVPCNetworkingSpec) DeepCopy() *LyftVPCNetworkingSpec {
 	return out
 }

+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *MixedInstancesPolicySpec) DeepCopyInto(out *MixedInstancesPolicySpec) {
+	*out = *in
+	if in.Instances != nil {
+		in, out := &in.Instances, &out.Instances
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
+	if in.OnDemandAllocationStrategy != nil {
+		in, out := &in.OnDemandAllocationStrategy, &out.OnDemandAllocationStrategy
+		*out = new(string)
+		**out = **in
+	}
+	if in.OnDemandBase != nil {
+		in, out := &in.OnDemandBase, &out.OnDemandBase
+		*out = new(int64)
+		**out = **in
+	}
+	if in.OnDemandAboveBase != nil {
+		in, out := &in.OnDemandAboveBase, &out.OnDemandAboveBase
+		*out = new(int64)
+		**out = **in
+	}
+	if in.SpotAllocationStrategy != nil {
+		in, out := &in.SpotAllocationStrategy, &out.SpotAllocationStrategy
+		*out = new(string)
+		**out = **in
+	}
+	if in.SpotInstancePools != nil {
+		in, out := &in.SpotInstancePools, &out.SpotInstancePools
+		*out = new(int64)
+		**out = **in
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MixedInstancesPolicySpec.
+func (in *MixedInstancesPolicySpec) DeepCopy() *MixedInstancesPolicySpec {
+	if in == nil {
+		return nil
+	}
+	out := new(MixedInstancesPolicySpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *NetworkingSpec) DeepCopyInto(out *NetworkingSpec) {
 	*out = *in
@ -2532,6 +2625,166 @@ func (in *NodeAuthorizerSpec) DeepCopy() *NodeAuthorizerSpec {
 	return out
 }

+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *OpenstackBlockStorageConfig) DeepCopyInto(out *OpenstackBlockStorageConfig) {
+	*out = *in
+	if in.Version != nil {
+		in, out := &in.Version, &out.Version
+		*out = new(string)
+		**out = **in
+	}
+	if in.IgnoreAZ != nil {
+		in, out := &in.IgnoreAZ, &out.IgnoreAZ
+		*out = new(bool)
+		**out = **in
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OpenstackBlockStorageConfig.
+func (in *OpenstackBlockStorageConfig) DeepCopy() *OpenstackBlockStorageConfig {
+	if in == nil {
+		return nil
+	}
+	out := new(OpenstackBlockStorageConfig)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *OpenstackConfiguration) DeepCopyInto(out *OpenstackConfiguration) {
+	*out = *in
+	if in.Loadbalancer != nil {
+		in, out := &in.Loadbalancer, &out.Loadbalancer
+		*out = new(OpenstackLoadbalancerConfig)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.Monitor != nil {
+		in, out := &in.Monitor, &out.Monitor
+		*out = new(OpenstackMonitor)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.Router != nil {
+		in, out := &in.Router, &out.Router
+		*out = new(OpenstackRouter)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.BlockStorage != nil {
+		in, out := &in.BlockStorage, &out.BlockStorage
+		*out = new(OpenstackBlockStorageConfig)
+		(*in).DeepCopyInto(*out)
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OpenstackConfiguration.
+func (in *OpenstackConfiguration) DeepCopy() *OpenstackConfiguration {
+	if in == nil {
+		return nil
+	}
+	out := new(OpenstackConfiguration)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *OpenstackLoadbalancerConfig) DeepCopyInto(out *OpenstackLoadbalancerConfig) {
+	*out = *in
+	if in.Method != nil {
+		in, out := &in.Method, &out.Method
+		*out = new(string)
+		**out = **in
+	}
+	if in.Provider != nil {
+		in, out := &in.Provider, &out.Provider
+		*out = new(string)
+		**out = **in
+	}
+	if in.UseOctavia != nil {
+		in, out := &in.UseOctavia, &out.UseOctavia
+		*out = new(bool)
+		**out = **in
+	}
+	if in.FloatingNetwork != nil {
+		in, out := &in.FloatingNetwork, &out.FloatingNetwork
+		*out = new(string)
+		**out = **in
+	}
+	if in.FloatingNetworkID != nil {
+		in, out := &in.FloatingNetworkID, &out.FloatingNetworkID
+		*out = new(string)
+		**out = **in
+	}
+	if in.SubnetID != nil {
+		in, out := &in.SubnetID, &out.SubnetID
+		*out = new(string)
+		**out = **in
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OpenstackLoadbalancerConfig.
+func (in *OpenstackLoadbalancerConfig) DeepCopy() *OpenstackLoadbalancerConfig {
+	if in == nil {
+		return nil
+	}
+	out := new(OpenstackLoadbalancerConfig)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *OpenstackMonitor) DeepCopyInto(out *OpenstackMonitor) {
+	*out = *in
+	if in.Delay != nil {
+		in, out := &in.Delay, &out.Delay
+		*out = new(string)
+		**out = **in
+	}
+	if in.Timeout != nil {
+		in, out := &in.Timeout, &out.Timeout
+		*out = new(string)
+		**out = **in
+	}
+	if in.MaxRetries != nil {
+		in, out := &in.MaxRetries, &out.MaxRetries
+		*out = new(int)
+		**out = **in
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OpenstackMonitor.
+func (in *OpenstackMonitor) DeepCopy() *OpenstackMonitor {
+	if in == nil {
+		return nil
+	}
+	out := new(OpenstackMonitor)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *OpenstackRouter) DeepCopyInto(out *OpenstackRouter) {
+	*out = *in
+	if in.ExternalNetwork != nil {
+		in, out := &in.ExternalNetwork, &out.ExternalNetwork
+		*out = new(string)
+		**out = **in
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OpenstackRouter.
+func (in *OpenstackRouter) DeepCopy() *OpenstackRouter {
+	if in == nil {
+		return nil
+	}
+	out := new(OpenstackRouter)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *RBACAuthorizationSpec) DeepCopyInto(out *RBACAuthorizationSpec) {
 	*out = *in
@ -2730,6 +2983,58 @@ func (in *UserData) DeepCopy() *UserData {
 	return out
 }

+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *VolumeMountSpec) DeepCopyInto(out *VolumeMountSpec) {
+	*out = *in
+	if in.FormatOptions != nil {
+		in, out := &in.FormatOptions, &out.FormatOptions
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
+	if in.MountOptions != nil {
+		in, out := &in.MountOptions, &out.MountOptions
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VolumeMountSpec.
+func (in *VolumeMountSpec) DeepCopy() *VolumeMountSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(VolumeMountSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *VolumeSpec) DeepCopyInto(out *VolumeSpec) {
+	*out = *in
+	if in.Encrypted != nil {
+		in, out := &in.Encrypted, &out.Encrypted
+		*out = new(bool)
+		**out = **in
+	}
+	if in.Iops != nil {
+		in, out := &in.Iops, &out.Iops
+		*out = new(int64)
+		**out = **in
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VolumeSpec.
+func (in *VolumeSpec) DeepCopy() *VolumeSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(VolumeSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *WeaveNetworkingSpec) DeepCopyInto(out *WeaveNetworkingSpec) {
 	*out = *in
--- a/pkg/apis/kops/validation/BUILD.bazel
+++ b/pkg/apis/kops/validation/BUILD.bazel
@ -22,6 +22,8 @@ go_library(
        "//pkg/util/subnet:go_default_library",
        "//upup/pkg/fi:go_default_library",
        "//upup/pkg/fi/cloudup/awsup:go_default_library",
+        "//util/pkg/slice:go_default_library",
+        "//vendor/github.com/aws/aws-sdk-go/aws/arn:go_default_library",
        "//vendor/github.com/blang/semver:go_default_library",
        "//vendor/github.com/golang/glog:go_default_library",
        "//vendor/k8s.io/apimachinery/pkg/api/validation:go_default_library",
--- a/pkg/apis/kops/validation/aws_test.go
+++ b/pkg/apis/kops/validation/aws_test.go
@ -19,7 +19,7 @@ package validation
 import (
 	"testing"

-	"k8s.io/apimachinery/pkg/apis/meta/v1"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/kops/pkg/apis/kops"
 )

--- a/pkg/apis/kops/validation/instancegroup.go
+++ b/pkg/apis/kops/validation/instancegroup.go
@ -18,12 +18,15 @@ package validation

 import (
 	"fmt"
-	"regexp"
+	"strings"

-	"k8s.io/apimachinery/pkg/util/validation/field"
 	"k8s.io/kops/pkg/apis/kops"
 	"k8s.io/kops/pkg/apis/kops/util"
 	"k8s.io/kops/upup/pkg/fi"
+	"k8s.io/kops/util/pkg/slice"
+
+	"github.com/aws/aws-sdk-go/aws/arn"
+	"k8s.io/apimachinery/pkg/util/validation/field"
 )

 // ValidateInstanceGroup is responsible for validating the configuration of a instancegroup
@ -78,6 +81,12 @@ func ValidateInstanceGroup(g *kops.InstanceGroup) error {
 		}
 	}

+	if g.Spec.MixedInstancesPolicy != nil {
+		if errs := validatedMixedInstancesPolicy(field.NewPath(g.Name), g.Spec.MixedInstancesPolicy, g); len(errs) > 0 {
+			return errs.ToAggregate()
+		}
+	}
+
 	if len(g.Spec.AdditionalUserData) > 0 {
 		for _, UserDataInfo := range g.Spec.AdditionalUserData {
 			err := validateExtraUserData(&UserDataInfo)
@ -87,6 +96,39 @@ func ValidateInstanceGroup(g *kops.InstanceGroup) error {
 		}
 	}

+	// @step: iterate and check the volume specs
+	for i, x := range g.Spec.Volumes {
+		devices := make(map[string]bool, 0)
+		path := field.NewPath("volumes").Index(i)
+
+		if err := validateVolumeSpec(path, x); err != nil {
+			return err
+		}
+
+		// @check the device name has not been used already
+		if _, found := devices[x.Device]; found {
+			return field.Invalid(path.Child("device"), x.Device, "duplicate device name found in volumes")
+		}
+
+		devices[x.Device] = true
+	}
+
+	// @step: iterate and check the volume mount specs
+	for i, x := range g.Spec.VolumeMounts {
+		used := make(map[string]bool, 0)
+		path := field.NewPath("volumeMounts").Index(i)
+
+		if err := validateVolumeMountSpec(path, x); err != nil {
+			return err
+		}
+		if _, found := used[x.Device]; found {
+			return field.Invalid(path.Child("device"), x.Device, "duplicate device reference")
+		}
+		if _, found := used[x.Path]; found {
+			return field.Invalid(path.Child("path"), x.Path, "duplicate mount path specified")
+		}
+	}
+
 	if err := validateInstanceProfile(g.Spec.IAM, field.NewPath("iam")); err != nil {
 		return err
 	}
@ -94,6 +136,74 @@ func ValidateInstanceGroup(g *kops.InstanceGroup) error {
 	return nil
 }

+// validatedMixedInstancesPolicy is responsible for validating the user input of a mixed instance policy
+func validatedMixedInstancesPolicy(path *field.Path, spec *kops.MixedInstancesPolicySpec, ig *kops.InstanceGroup) field.ErrorList {
+	var errs field.ErrorList
+
+	if len(spec.Instances) < 2 {
+		errs = append(errs, field.Invalid(path.Child("instances"), spec.Instances, "must be 2 or more instance types"))
+	}
+	// @step: check the instances are validate
+	for i, x := range spec.Instances {
+		errs = append(errs, awsValidateMachineType(path.Child("instances").Index(i).Child("instanceType"), x)...)
+	}
+
+	if spec.OnDemandBase != nil {
+		if fi.Int64Value(spec.OnDemandBase) < 0 {
+			errs = append(errs, field.Invalid(path.Child("onDemandBase"), spec.OnDemandBase, "cannot be less than zero"))
+		}
+		if fi.Int64Value(spec.OnDemandBase) > int64(fi.Int32Value(ig.Spec.MaxSize)) {
+			errs = append(errs, field.Invalid(path.Child("onDemandBase"), spec.OnDemandBase, "cannot be greater than max size"))
+		}
+	}
+
+	if spec.OnDemandAboveBase != nil {
+		if fi.Int64Value(spec.OnDemandAboveBase) < 0 {
+			errs = append(errs, field.Invalid(path.Child("onDemandAboveBase"), spec.OnDemandAboveBase, "cannot be less than 0"))
+		}
+		if fi.Int64Value(spec.OnDemandAboveBase) > 100 {
+			errs = append(errs, field.Invalid(path.Child("onDemandAboveBase"), spec.OnDemandAboveBase, "cannot be greater than 100"))
+		}
+	}
+
+	if spec.SpotAllocationStrategy != nil && !slice.Contains(kops.SpotAllocationStrategies, fi.StringValue(spec.SpotAllocationStrategy)) {
+		errs = append(errs, field.Invalid(path.Child("spotAllocationStrategy"), spec.SpotAllocationStrategy, "unsupported spot allocation strategy"))
+	}
+
+	return errs
+}
+
+// validateVolumeSpec is responsible for checking a volume spec is ok
+func validateVolumeSpec(path *field.Path, v *kops.VolumeSpec) error {
+	if v.Device == "" {
+		return field.Required(path.Child("device"), "device name required")
+	}
+	if v.Size <= 0 {
+		return field.Invalid(path.Child("size"), v.Size, "must be greater than zero")
+	}
+
+	return nil
+}
+
+// validateVolumeMountSpec is responsible for checking the volume mount is ok
+func validateVolumeMountSpec(path *field.Path, spec *kops.VolumeMountSpec) error {
+	if spec.Device == "" {
+		return field.Required(path.Child("device"), "device name required")
+	}
+	if spec.Filesystem == "" {
+		return field.Required(path.Child("filesystem"), "filesystem type required")
+	}
+	if spec.Path == "" {
+		return field.Required(path.Child("path"), "mount path required")
+	}
+	if !slice.Contains(kops.SupportedFilesystems, spec.Filesystem) {
+		return field.Invalid(path.Child("filesystem"), spec.Filesystem,
+			fmt.Sprintf("unsupported filesystem, available types: %s", strings.Join(kops.SupportedFilesystems, ",")))
+	}
+
+	return nil
+}
+
 // CrossValidateInstanceGroup performs validation of the instance group, including that it is consistent with the Cluster
 // It calls ValidateInstanceGroup, so all that validation is included.
 func CrossValidateInstanceGroup(g *kops.InstanceGroup, cluster *kops.Cluster, strict bool) error {
@ -171,15 +281,13 @@ func validateExtraUserData(userData *kops.UserData) error {
 	return nil
 }

-// format is arn:aws:iam::123456789012:instance-profile/S3Access
-var validARN = regexp.MustCompile(`^arn:aws:iam::\d+:instance-profile\/\S+$`)
-
 // validateInstanceProfile checks the String values for the AuthProfile
 func validateInstanceProfile(v *kops.IAMProfileSpec, fldPath *field.Path) *field.Error {
 	if v != nil && v.Profile != nil {
-		arn := *v.Profile
-		if !validARN.MatchString(arn) {
-			return field.Invalid(fldPath.Child("Profile"), arn,
+		instanceProfileARN := *v.Profile
+		parsedARN, err := arn.Parse(instanceProfileARN)
+		if err != nil || !strings.HasPrefix(parsedARN.Resource, "instance-profile") {
+			return field.Invalid(fldPath.Child("Profile"), instanceProfileARN,
 				"Instance Group IAM Instance Profile must be a valid aws arn such as arn:aws:iam::123456789012:instance-profile/KopsExampleRole")
 		}
 	}
--- a/pkg/apis/kops/validation/instancegroup_test.go
+++ b/pkg/apis/kops/validation/instancegroup_test.go
@ -86,6 +86,16 @@ func TestValidateInstanceProfile(t *testing.T) {
 				Profile: s("arn:aws:iam::123456789012:instance-profile/has/path/S3Access"),
 			},
 		},
+		{
+			Input: &kops.IAMProfileSpec{
+				Profile: s("arn:aws-cn:iam::123456789012:instance-profile/has/path/S3Access"),
+			},
+		},
+		{
+			Input: &kops.IAMProfileSpec{
+				Profile: s("arn:aws-us-gov:iam::123456789012:instance-profile/has/path/S3Access"),
+			},
+		},
 		{
 			Input: &kops.IAMProfileSpec{
 				Profile: s("42"),
--- a/pkg/apis/kops/validation/legacy.go
+++ b/pkg/apis/kops/validation/legacy.go
@ -17,6 +17,7 @@ limitations under the License.
 package validation

 import (
+	"errors"
 	"fmt"
 	"net"
 	"strings"
@ -562,11 +563,14 @@ func ValidateCluster(c *kops.Cluster, strict bool) *field.Error {
 	// Egress specification support
 	{
 		for i, s := range c.Spec.Subnets {
-			fieldSubnet := fieldSpec.Child("Subnets").Index(i)
-			if s.Egress != "" && !strings.HasPrefix(s.Egress, "nat-") && !strings.HasPrefix(s.Egress, "i-") {
-				return field.Invalid(fieldSubnet.Child("Egress"), s.Egress, "egress must be of type NAT Gateway or NAT EC2 Instance")
+			if s.Egress == "" {
+				continue
 			}
-			if s.Egress != "" && !(s.Type == "Private") {
+			fieldSubnet := fieldSpec.Child("Subnets").Index(i)
+			if !strings.HasPrefix(s.Egress, "nat-") && !strings.HasPrefix(s.Egress, "i-") && s.Egress != kops.EgressExternal {
+				return field.Invalid(fieldSubnet.Child("Egress"), s.Egress, "egress must be of type NAT Gateway or NAT EC2 Instance or 'External'")
+			}
+			if s.Egress != kops.EgressExternal && s.Type != "Private" {
 				return field.Invalid(fieldSubnet.Child("Egress"), s.Egress, "egress can only be specified for Private subnets")
 			}
 		}
@ -801,6 +805,10 @@ func DeepValidate(c *kops.Cluster, groups []*kops.InstanceGroup, strict bool) er
 			if len(errs) != 0 {
 				return errs[0]
 			}
+		default:
+			if len(g.Spec.Volumes) > 0 {
+				return errors.New("instancegroup volumes are only available with aws at present")
+			}
 		}
 	}

--- a/pkg/apis/kops/zz_generated.deepcopy.go
+++ b/pkg/apis/kops/zz_generated.deepcopy.go
@ -516,6 +516,11 @@ func (in *CloudConfiguration) DeepCopyInto(out *CloudConfiguration) {
 		*out = new(string)
 		**out = **in
 	}
+	if in.Openstack != nil {
+		in, out := &in.Openstack, &out.Openstack
+		*out = new(OpenstackConfiguration)
+		(*in).DeepCopyInto(*out)
+	}
 	return
 }

@ -1515,6 +1520,28 @@ func (in *InstanceGroupSpec) DeepCopyInto(out *InstanceGroupSpec) {
 		*out = new(bool)
 		**out = **in
 	}
+	if in.Volumes != nil {
+		in, out := &in.Volumes, &out.Volumes
+		*out = make([]*VolumeSpec, len(*in))
+		for i := range *in {
+			if (*in)[i] != nil {
+				in, out := &(*in)[i], &(*out)[i]
+				*out = new(VolumeSpec)
+				(*in).DeepCopyInto(*out)
+			}
+		}
+	}
+	if in.VolumeMounts != nil {
+		in, out := &in.VolumeMounts, &out.VolumeMounts
+		*out = make([]*VolumeMountSpec, len(*in))
+		for i := range *in {
+			if (*in)[i] != nil {
+				in, out := &(*in)[i], &(*out)[i]
+				*out = new(VolumeMountSpec)
+				(*in).DeepCopyInto(*out)
+			}
+		}
+	}
 	if in.Subnets != nil {
 		in, out := &in.Subnets, &out.Subnets
 		*out = make([]string, len(*in))
@ -1578,6 +1605,11 @@ func (in *InstanceGroupSpec) DeepCopyInto(out *InstanceGroupSpec) {
 		*out = make([]string, len(*in))
 		copy(*out, *in)
 	}
+	if in.MixedInstancesPolicy != nil {
+		in, out := &in.MixedInstancesPolicy, &out.MixedInstancesPolicy
+		*out = new(MixedInstancesPolicySpec)
+		(*in).DeepCopyInto(*out)
+	}
 	if in.AdditionalUserData != nil {
 		in, out := &in.AdditionalUserData, &out.AdditionalUserData
 		*out = make([]UserData, len(*in))
@ -1880,6 +1912,11 @@ func (in *KubeAPIServerConfig) DeepCopyInto(out *KubeAPIServerConfig) {
 		*out = new(string)
 		**out = **in
 	}
+	if in.OIDCRequiredClaim != nil {
+		in, out := &in.OIDCRequiredClaim, &out.OIDCRequiredClaim
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
 	if in.OIDCCAFile != nil {
 		in, out := &in.OIDCCAFile, &out.OIDCCAFile
 		*out = new(string)
@ -2426,6 +2463,16 @@ func (in *KubeletConfigSpec) DeepCopyInto(out *KubeletConfigSpec) {
 		*out = new(v1.Duration)
 		**out = **in
 	}
+	if in.CPUCFSQuota != nil {
+		in, out := &in.CPUCFSQuota, &out.CPUCFSQuota
+		*out = new(bool)
+		**out = **in
+	}
+	if in.CPUCFSQuotaPeriod != nil {
+		in, out := &in.CPUCFSQuotaPeriod, &out.CPUCFSQuotaPeriod
+		*out = new(v1.Duration)
+		**out = **in
+	}
 	return
 }

@ -2588,6 +2635,52 @@ func (in *LyftVPCNetworkingSpec) DeepCopy() *LyftVPCNetworkingSpec {
 	return out
 }

+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *MixedInstancesPolicySpec) DeepCopyInto(out *MixedInstancesPolicySpec) {
+	*out = *in
+	if in.Instances != nil {
+		in, out := &in.Instances, &out.Instances
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
+	if in.OnDemandAllocationStrategy != nil {
+		in, out := &in.OnDemandAllocationStrategy, &out.OnDemandAllocationStrategy
+		*out = new(string)
+		**out = **in
+	}
+	if in.OnDemandBase != nil {
+		in, out := &in.OnDemandBase, &out.OnDemandBase
+		*out = new(int64)
+		**out = **in
+	}
+	if in.OnDemandAboveBase != nil {
+		in, out := &in.OnDemandAboveBase, &out.OnDemandAboveBase
+		*out = new(int64)
+		**out = **in
+	}
+	if in.SpotAllocationStrategy != nil {
+		in, out := &in.SpotAllocationStrategy, &out.SpotAllocationStrategy
+		*out = new(string)
+		**out = **in
+	}
+	if in.SpotInstancePools != nil {
+		in, out := &in.SpotInstancePools, &out.SpotInstancePools
+		*out = new(int64)
+		**out = **in
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MixedInstancesPolicySpec.
+func (in *MixedInstancesPolicySpec) DeepCopy() *MixedInstancesPolicySpec {
+	if in == nil {
+		return nil
+	}
+	out := new(MixedInstancesPolicySpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *NetworkingSpec) DeepCopyInto(out *NetworkingSpec) {
 	*out = *in
@ -2746,6 +2839,166 @@ func (in *NoopStatusStore) DeepCopy() *NoopStatusStore {
 	return out
 }

+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *OpenstackBlockStorageConfig) DeepCopyInto(out *OpenstackBlockStorageConfig) {
+	*out = *in
+	if in.Version != nil {
+		in, out := &in.Version, &out.Version
+		*out = new(string)
+		**out = **in
+	}
+	if in.IgnoreAZ != nil {
+		in, out := &in.IgnoreAZ, &out.IgnoreAZ
+		*out = new(bool)
+		**out = **in
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OpenstackBlockStorageConfig.
+func (in *OpenstackBlockStorageConfig) DeepCopy() *OpenstackBlockStorageConfig {
+	if in == nil {
+		return nil
+	}
+	out := new(OpenstackBlockStorageConfig)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *OpenstackConfiguration) DeepCopyInto(out *OpenstackConfiguration) {
+	*out = *in
+	if in.Loadbalancer != nil {
+		in, out := &in.Loadbalancer, &out.Loadbalancer
+		*out = new(OpenstackLoadbalancerConfig)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.Monitor != nil {
+		in, out := &in.Monitor, &out.Monitor
+		*out = new(OpenstackMonitor)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.Router != nil {
+		in, out := &in.Router, &out.Router
+		*out = new(OpenstackRouter)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.BlockStorage != nil {
+		in, out := &in.BlockStorage, &out.BlockStorage
+		*out = new(OpenstackBlockStorageConfig)
+		(*in).DeepCopyInto(*out)
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OpenstackConfiguration.
+func (in *OpenstackConfiguration) DeepCopy() *OpenstackConfiguration {
+	if in == nil {
+		return nil
+	}
+	out := new(OpenstackConfiguration)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *OpenstackLoadbalancerConfig) DeepCopyInto(out *OpenstackLoadbalancerConfig) {
+	*out = *in
+	if in.Method != nil {
+		in, out := &in.Method, &out.Method
+		*out = new(string)
+		**out = **in
+	}
+	if in.Provider != nil {
+		in, out := &in.Provider, &out.Provider
+		*out = new(string)
+		**out = **in
+	}
+	if in.UseOctavia != nil {
+		in, out := &in.UseOctavia, &out.UseOctavia
+		*out = new(bool)
+		**out = **in
+	}
+	if in.FloatingNetwork != nil {
+		in, out := &in.FloatingNetwork, &out.FloatingNetwork
+		*out = new(string)
+		**out = **in
+	}
+	if in.FloatingNetworkID != nil {
+		in, out := &in.FloatingNetworkID, &out.FloatingNetworkID
+		*out = new(string)
+		**out = **in
+	}
+	if in.SubnetID != nil {
+		in, out := &in.SubnetID, &out.SubnetID
+		*out = new(string)
+		**out = **in
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OpenstackLoadbalancerConfig.
+func (in *OpenstackLoadbalancerConfig) DeepCopy() *OpenstackLoadbalancerConfig {
+	if in == nil {
+		return nil
+	}
+	out := new(OpenstackLoadbalancerConfig)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *OpenstackMonitor) DeepCopyInto(out *OpenstackMonitor) {
+	*out = *in
+	if in.Delay != nil {
+		in, out := &in.Delay, &out.Delay
+		*out = new(string)
+		**out = **in
+	}
+	if in.Timeout != nil {
+		in, out := &in.Timeout, &out.Timeout
+		*out = new(string)
+		**out = **in
+	}
+	if in.MaxRetries != nil {
+		in, out := &in.MaxRetries, &out.MaxRetries
+		*out = new(int)
+		**out = **in
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OpenstackMonitor.
+func (in *OpenstackMonitor) DeepCopy() *OpenstackMonitor {
+	if in == nil {
+		return nil
+	}
+	out := new(OpenstackMonitor)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *OpenstackRouter) DeepCopyInto(out *OpenstackRouter) {
+	*out = *in
+	if in.ExternalNetwork != nil {
+		in, out := &in.ExternalNetwork, &out.ExternalNetwork
+		*out = new(string)
+		**out = **in
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OpenstackRouter.
+func (in *OpenstackRouter) DeepCopy() *OpenstackRouter {
+	if in == nil {
+		return nil
+	}
+	out := new(OpenstackRouter)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *RBACAuthorizationSpec) DeepCopyInto(out *RBACAuthorizationSpec) {
 	*out = *in
@ -2944,6 +3197,58 @@ func (in *UserData) DeepCopy() *UserData {
 	return out
 }

+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *VolumeMountSpec) DeepCopyInto(out *VolumeMountSpec) {
+	*out = *in
+	if in.FormatOptions != nil {
+		in, out := &in.FormatOptions, &out.FormatOptions
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
+	if in.MountOptions != nil {
+		in, out := &in.MountOptions, &out.MountOptions
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VolumeMountSpec.
+func (in *VolumeMountSpec) DeepCopy() *VolumeMountSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(VolumeMountSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *VolumeSpec) DeepCopyInto(out *VolumeSpec) {
+	*out = *in
+	if in.Encrypted != nil {
+		in, out := &in.Encrypted, &out.Encrypted
+		*out = new(bool)
+		**out = **in
+	}
+	if in.Iops != nil {
+		in, out := &in.Iops, &out.Iops
+		*out = new(int64)
+		**out = **in
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VolumeSpec.
+func (in *VolumeSpec) DeepCopy() *VolumeSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(VolumeSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *WeaveNetworkingSpec) DeepCopyInto(out *WeaveNetworkingSpec) {
 	*out = *in
--- a/pkg/client/simple/vfsclientset/commonvfs.go
+++ b/pkg/client/simple/vfsclientset/commonvfs.go
@ -26,8 +26,8 @@ import (

 	"github.com/golang/glog"
 	"k8s.io/apimachinery/pkg/api/meta"
-	"k8s.io/apimachinery/pkg/apis/meta/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/kops/pkg/acls"
--- a/pkg/cloudinstances/cloud_instance_group.go
+++ b/pkg/cloudinstances/cloud_instance_group.go
@ -21,7 +21,7 @@ import (
 	"strings"

 	"github.com/golang/glog"
-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	"k8s.io/kops/pkg/apis/kops"
 	api "k8s.io/kops/pkg/apis/kops"
 )
--- a/pkg/commands/BUILD.bazel
+++ b/pkg/commands/BUILD.bazel
@ -6,10 +6,12 @@ go_library(
        "helpers_readwrite.go",
        "set_cluster.go",
        "status_discovery.go",
+        "version.go",
    ],
    importpath = "k8s.io/kops/pkg/commands",
    visibility = ["//visibility:public"],
    deps = [
+        "//:go_default_library",
        "//cmd/kops/util:go_default_library",
        "//pkg/apis/kops:go_default_library",
        "//pkg/apis/kops/validation:go_default_library",
@ -21,6 +23,7 @@ go_library(
        "//upup/pkg/fi/cloudup/awstasks:go_default_library",
        "//upup/pkg/fi/cloudup/awsup:go_default_library",
        "//upup/pkg/fi/cloudup/gce:go_default_library",
+        "//upup/pkg/fi/cloudup/openstack:go_default_library",
        "//vendor/github.com/aws/aws-sdk-go/aws:go_default_library",
        "//vendor/github.com/spf13/cobra:go_default_library",
        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
--- a/pkg/commands/set_cluster.go
+++ b/pkg/commands/set_cluster.go
@ -109,8 +109,12 @@ func SetClusterFields(fields []string, cluster *api.Cluster, instanceGroups []*a
 				c.Version = kv[1]
 			}
 		case "cluster.spec.etcdClusters[*].provider":
+			p, err := toEtcdProviderType(kv[1])
+			if err != nil {
+				return err
+			}
 			for _, etcd := range cluster.Spec.EtcdClusters {
-				etcd.Provider = api.EtcdProviderType(kv[1])
+				etcd.Provider = p
 			}
 		case "cluster.spec.etcdClusters[*].manager.image":
 			for _, etcd := range cluster.Spec.EtcdClusters {
@ -125,3 +129,15 @@ func SetClusterFields(fields []string, cluster *api.Cluster, instanceGroups []*a
 	}
 	return nil
 }
+
+func toEtcdProviderType(in string) (api.EtcdProviderType, error) {
+	s := strings.ToLower(in)
+	switch s {
+	case "legacy":
+		return api.EtcdProviderTypeLegacy, nil
+	case "manager":
+		return api.EtcdProviderTypeManager, nil
+	default:
+		return api.EtcdProviderTypeManager, fmt.Errorf("unknown etcd provider type %q", in)
+	}
+}
--- a/pkg/commands/status_discovery.go
+++ b/pkg/commands/status_discovery.go
@ -26,6 +26,7 @@ import (
 	"k8s.io/kops/upup/pkg/fi/cloudup/awstasks"
 	"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
 	"k8s.io/kops/upup/pkg/fi/cloudup/gce"
+	"k8s.io/kops/upup/pkg/fi/cloudup/openstack"
 )

 // CloudDiscoveryStatusStore implements status.Store by inspecting cloud objects.
@ -68,6 +69,10 @@ func (s *CloudDiscoveryStatusStore) GetApiIngressStatus(cluster *kops.Cluster) (
 		return ingresses, nil
 	}

+	if osCloud, ok := cloud.(openstack.OpenstackCloud); ok {
+		return osCloud.GetApiIngressStatus(cluster)
+	}
+
 	return nil, fmt.Errorf("API Ingress Status not implemented for %T", cloud)
 }

@ -89,5 +94,9 @@ func (s *CloudDiscoveryStatusStore) FindClusterStatus(cluster *kops.Cluster) (*k
 	if aliCloud, ok := cloud.(aliup.ALICloud); ok {
 		return aliCloud.FindClusterStatus(cluster)
 	}
+
+	if osCloud, ok := cloud.(openstack.OpenstackCloud); ok {
+		return osCloud.FindClusterStatus(cluster)
+	}
 	return nil, fmt.Errorf("Etcd Status not implemented for %T", cloud)
 }
--- a/pkg/commands/version.go
+++ b/pkg/commands/version.go
@ -0,0 +1,45 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package commands
+
+import (
+	"fmt"
+	"io"
+
+	"k8s.io/kops"
+	"k8s.io/kops/cmd/kops/util"
+)
+
+type VersionOptions struct {
+	Short bool
+}
+
+// RunVersion implements the version command logic
+func RunVersion(f *util.Factory, out io.Writer, options *VersionOptions) error {
+	var s string
+	if options.Short {
+		s = kops.Version
+	} else {
+		s = "Version " + kops.Version
+		if kops.GitVersion != "" {
+			s += " (git-" + kops.GitVersion + ")"
+		}
+	}
+
+	_, err := fmt.Fprintf(out, "%s\n", s)
+	return err
+}
--- a/pkg/instancegroups/rollingupdate_test.go
+++ b/pkg/instancegroups/rollingupdate_test.go
@ -23,7 +23,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/autoscaling"

-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	v1meta "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes/fake"
 	"k8s.io/kops/cloudmock/aws/mockautoscaling"
--- a/Show More
+++ b/Show More