Add openstack integration test.

This will create / update / update / delete an openstack cluster using cloudmock, ensuring there are no lingering changes reported or orphaned resources

cmd/kops/BUILD.bazel

@@ -157,6 +157,7 @@ go_test(
         "//upup/pkg/fi/cloudup:go_default_library",
         "//upup/pkg/fi/cloudup/awsup:go_default_library",
         "//upup/pkg/fi/cloudup/gce:go_default_library",
+        "//upup/pkg/fi/cloudup/openstack:go_default_library",
         "//util/pkg/ui:go_default_library",
         "//vendor/github.com/aws/aws-sdk-go/aws:go_default_library",
         "//vendor/github.com/aws/aws-sdk-go/service/ec2:go_default_library",

cmd/kops/lifecycle_integration_test.go

@@ -19,6 +19,7 @@ package main
 import (
 	"bytes"
 	"context"
+	"os"
 	"path"
 	"reflect"
 	"sort"
@@ -32,6 +33,7 @@ import (
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/upup/pkg/fi/cloudup"
 	"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
+	"k8s.io/kops/upup/pkg/fi/cloudup/openstack"
 )
 
 type LifecycleTestOptions struct {
@@ -63,6 +65,14 @@ func TestLifecycleMinimal(t *testing.T) {
 	})
 }
 
+func TestLifecycleMinimalOpenstack(t *testing.T) {
+	runLifecycleTestOpenstack(&LifecycleTestOptions{
+		t:           t,
+		SrcDir:      "minimal_openstack",
+		ClusterName: "minimal-openstack.k8s.local",
+	})
+}
+
 // TestLifecyclePrivateCalico runs the test on a private topology
 func TestLifecyclePrivateCalico(t *testing.T) {
 	runLifecycleTestAWS(&LifecycleTestOptions{
@@ -122,7 +132,7 @@ func runLifecycleTest(h *testutils.IntegrationTestHarness, o *LifecycleTestOptio
 
 	factory := util.NewFactory(factoryOptions)
 
-	beforeResources := AllResources(cloud)
+	beforeResources := AllAWSResources(cloud)
 
 	{
 		options := &CreateOptions{}
@@ -187,7 +197,7 @@ func runLifecycleTest(h *testutils.IntegrationTestHarness, o *LifecycleTestOptio
 
 	{
 		var ids []string
-		for id := range AllResources(cloud) {
+		for id := range AllAWSResources(cloud) {
 			ids = append(ids, id)
 		}
 		sort.Strings(ids)
@@ -255,8 +265,8 @@ func runLifecycleTest(h *testutils.IntegrationTestHarness, o *LifecycleTestOptio
 	}
 }
 
-// AllResources returns all resources
-func AllResources(c *awsup.MockAWSCloud) map[string]interface{} {
+// AllAWSResources returns all resources
+func AllAWSResources(c *awsup.MockAWSCloud) map[string]interface{} {
 	all := make(map[string]interface{})
 	for k, v := range c.MockEC2.(*mockec2.MockEC2).All() {
 		all[k] = v
@@ -264,6 +274,15 @@ func AllResources(c *awsup.MockAWSCloud) map[string]interface{} {
 	return all
 }
 
+// AllOpenstackResources returns all resources
+func AllOpenstackResources(c *openstack.MockCloud) map[string]interface{} {
+	all := make(map[string]interface{})
+	for k, v := range c.MockNovaClient.All() {
+		all[k] = v
+	}
+	return all
+}
+
 func runLifecycleTestAWS(o *LifecycleTestOptions) {
 	o.AddDefaults()
 
@@ -276,7 +295,7 @@ func runLifecycleTestAWS(o *LifecycleTestOptions) {
 	cloud := h.SetupMockAWS()
 
 	var beforeIds []string
-	for id := range AllResources(cloud) {
+	for id := range AllAWSResources(cloud) {
 		beforeIds = append(beforeIds, id)
 	}
 	sort.Strings(beforeIds)
@@ -284,7 +303,7 @@ func runLifecycleTestAWS(o *LifecycleTestOptions) {
 	runLifecycleTest(h, o, cloud)
 
 	var afterIds []string
-	for id := range AllResources(cloud) {
+	for id := range AllAWSResources(cloud) {
 		afterIds = append(afterIds, id)
 	}
 	sort.Strings(afterIds)
@@ -293,3 +312,122 @@ func runLifecycleTestAWS(o *LifecycleTestOptions) {
 		t.Fatalf("resources changed by cluster create / destroy: %v -> %v", beforeIds, afterIds)
 	}
 }
+
+func runLifecycleTestOpenstack(o *LifecycleTestOptions) {
+	o.AddDefaults()
+
+	t := o.t
+
+	h := testutils.NewIntegrationTestHarness(o.t)
+	defer h.Close()
+
+	origRegion := os.Getenv("OS_REGION_NAME")
+	os.Setenv("OS_REGION_NAME", "us-test1")
+	defer func() {
+		os.Setenv("OS_REGION_NAME", origRegion)
+	}()
+
+	h.MockKopsVersion("1.19.0-alpha.1")
+	cloud := h.SetupMockOpenstack()
+
+	var beforeIds []string
+	for id := range AllOpenstackResources(cloud) {
+		beforeIds = append(beforeIds, id)
+	}
+	sort.Strings(beforeIds)
+
+	ctx := context.Background()
+
+	t.Logf("running lifecycle test for cluster %s", o.ClusterName)
+
+	var stdout bytes.Buffer
+
+	inputYAML := "in-" + o.Version + ".yaml"
+
+	factoryOptions := &util.FactoryOptions{}
+	factoryOptions.RegistryPath = "memfs://tests"
+
+	factory := util.NewFactory(factoryOptions)
+
+	{
+		options := &CreateOptions{}
+		options.Filenames = []string{path.Join(o.SrcDir, inputYAML)}
+
+		err := RunCreate(ctx, factory, &stdout, options)
+		if err != nil {
+			t.Fatalf("error running %q create: %v", inputYAML, err)
+		}
+	}
+
+	{
+		options := &CreateSecretPublickeyOptions{}
+		options.ClusterName = o.ClusterName
+		options.Name = "admin"
+		options.PublicKeyPath = path.Join(o.SrcDir, "id_rsa.pub")
+
+		err := RunCreateSecretPublicKey(ctx, factory, &stdout, options)
+		if err != nil {
+			t.Fatalf("error running %q create: %v", inputYAML, err)
+		}
+	}
+
+	{
+		options := &UpdateClusterOptions{}
+		options.InitDefaults()
+		options.RunTasksOptions.MaxTaskDuration = 10 * time.Second
+		options.Yes = true
+
+		// We don't test it here, and it adds a dependency on kubectl
+		options.CreateKubecfg = false
+
+		_, err := RunUpdateCluster(ctx, factory, o.ClusterName, &stdout, options)
+		if err != nil {
+			t.Fatalf("error running update cluster %q: %v", o.ClusterName, err)
+		}
+	}
+
+	{
+		options := &UpdateClusterOptions{}
+		options.InitDefaults()
+		options.Target = cloudup.TargetDryRun
+		options.RunTasksOptions.MaxTaskDuration = 10 * time.Second
+
+		// We don't test it here, and it adds a dependency on kubectl
+		options.CreateKubecfg = false
+
+		results, err := RunUpdateCluster(ctx, factory, o.ClusterName, &stdout, options)
+		if err != nil {
+			t.Fatalf("error running update cluster %q: %v", o.ClusterName, err)
+		}
+
+		target := results.Target.(*fi.DryRunTarget)
+		if target.HasChanges() {
+			var b bytes.Buffer
+			if err := target.PrintReport(results.TaskMap, &b); err != nil {
+				t.Fatalf("error building report: %v", err)
+			}
+			t.Fatalf("Target had changes after executing: %v", b.String())
+		}
+	}
+
+	{
+		options := &DeleteClusterOptions{}
+		options.Yes = true
+		options.ClusterName = o.ClusterName
+		if err := RunDeleteCluster(ctx, factory, &stdout, options); err != nil {
+			t.Fatalf("error running delete cluster %q: %v", o.ClusterName, err)
+		}
+	}
+
+	{
+		var afterIds []string
+		for id := range AllOpenstackResources(cloud) {
+			afterIds = append(afterIds, id)
+		}
+		sort.Strings(afterIds)
+
+		if !reflect.DeepEqual(beforeIds, afterIds) {
+			t.Fatalf("resources changed by cluster create / destroy: %v -> %v", beforeIds, afterIds)
+		}
+	}
+}

pkg/testutils/BUILD.bazel

@@ -16,6 +16,11 @@ go_library(
         "//cloudmock/aws/mockelbv2:go_default_library",
         "//cloudmock/aws/mockiam:go_default_library",
         "//cloudmock/aws/mockroute53:go_default_library",
+        "//cloudmock/openstack/mockblockstorage:go_default_library",
+        "//cloudmock/openstack/mockcompute:go_default_library",
+        "//cloudmock/openstack/mockdns:go_default_library",
+        "//cloudmock/openstack/mockloadbalancer:go_default_library",
+        "//cloudmock/openstack/mocknetworking:go_default_library",
         "//pkg/apis/kops:go_default_library",
         "//pkg/apis/kops/v1alpha2:go_default_library",
         "//pkg/kopscodecs:go_default_library",
@@ -24,11 +29,18 @@ go_library(
         "//upup/pkg/fi:go_default_library",
         "//upup/pkg/fi/cloudup/awsup:go_default_library",
         "//upup/pkg/fi/cloudup/gce:go_default_library",
+        "//upup/pkg/fi/cloudup/openstack:go_default_library",
         "//util/pkg/text:go_default_library",
         "//util/pkg/vfs:go_default_library",
         "//vendor/github.com/aws/aws-sdk-go/aws:go_default_library",
         "//vendor/github.com/aws/aws-sdk-go/service/ec2:go_default_library",
         "//vendor/github.com/aws/aws-sdk-go/service/route53:go_default_library",
+        "//vendor/github.com/gophercloud/gophercloud/openstack/compute/v2/flavors:go_default_library",
+        "//vendor/github.com/gophercloud/gophercloud/openstack/dns/v2/zones:go_default_library",
+        "//vendor/github.com/gophercloud/gophercloud/openstack/imageservice/v2/images:go_default_library",
+        "//vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/external:go_default_library",
+        "//vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/networks:go_default_library",
+        "//vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/subnets:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
         "//vendor/k8s.io/klog:go_default_library",
     ],

pkg/testutils/integrationtestharness.go

@@ -26,6 +26,12 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/aws/aws-sdk-go/service/route53"
+	"github.com/gophercloud/gophercloud/openstack/compute/v2/flavors"
+	"github.com/gophercloud/gophercloud/openstack/dns/v2/zones"
+	"github.com/gophercloud/gophercloud/openstack/imageservice/v2/images"
+	"github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/external"
+	"github.com/gophercloud/gophercloud/openstack/networking/v2/networks"
+	"github.com/gophercloud/gophercloud/openstack/networking/v2/subnets"
 	"k8s.io/klog"
 	kopsroot "k8s.io/kops"
 	"k8s.io/kops/cloudmock/aws/mockautoscaling"
@@ -34,10 +40,17 @@ import (
 	"k8s.io/kops/cloudmock/aws/mockelbv2"
 	"k8s.io/kops/cloudmock/aws/mockiam"
 	"k8s.io/kops/cloudmock/aws/mockroute53"
+	"k8s.io/kops/cloudmock/openstack/mockblockstorage"
+	"k8s.io/kops/cloudmock/openstack/mockcompute"
+	"k8s.io/kops/cloudmock/openstack/mockdns"
+	"k8s.io/kops/cloudmock/openstack/mockloadbalancer"
+	"k8s.io/kops/cloudmock/openstack/mocknetworking"
 	"k8s.io/kops/pkg/apis/kops"
 	"k8s.io/kops/pkg/pki"
+	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
 	"k8s.io/kops/upup/pkg/fi/cloudup/gce"
+	"k8s.io/kops/upup/pkg/fi/cloudup/openstack"
 	"k8s.io/kops/util/pkg/vfs"
 )
 
@@ -239,6 +252,62 @@ func (h *IntegrationTestHarness) SetupMockGCE() {
 	gce.InstallMockGCECloud("us-test1", "testproject")
 }
 
+func (h *IntegrationTestHarness) SetupMockOpenstack() *openstack.MockCloud {
+	c := openstack.InstallMockOpenstackCloud("us-test1")
+	c.MockCinderClient = mockblockstorage.CreateClient()
+
+	c.MockNeutronClient = mocknetworking.CreateClient()
+
+	c.MockLBClient = mockloadbalancer.CreateClient()
+
+	c.MockNovaClient = mockcompute.CreateClient()
+
+	c.MockDNSClient = mockdns.CreateClient()
+
+	extNetworkName := "external"
+	networkCreateOpts := networks.CreateOpts{
+		Name:         extNetworkName,
+		AdminStateUp: fi.Bool(true),
+	}
+	extNetwork := external.CreateOptsExt{
+		CreateOptsBuilder: networkCreateOpts,
+		External:          fi.Bool(true),
+	}
+	c.CreateNetwork(extNetwork)
+	c.SetExternalNetwork(&extNetworkName)
+
+	extSubnetName := "external"
+	extSubnet := subnets.CreateOpts{
+		Name:       extSubnetName,
+		NetworkID:  extNetworkName,
+		EnableDHCP: fi.Bool(true),
+		CIDR:       "172.20.0.0/22",
+	}
+	c.CreateSubnet(extSubnet)
+	c.SetExternalSubnet(fi.String(extSubnetName))
+	c.SetLBFloatingSubnet(fi.String(extSubnetName))
+	images.Create(c.MockNovaClient.ServiceClient(), images.CreateOpts{
+		Name:    "Ubuntu-20.04",
+		MinDisk: 12,
+	})
+	flavors.Create(c.MockNovaClient.ServiceClient(), flavors.CreateOpts{
+		Name:  "n1-standard-2",
+		RAM:   8192,
+		VCPUs: 4,
+		Disk:  fi.Int(16),
+	})
+	flavors.Create(c.MockNovaClient.ServiceClient(), flavors.CreateOpts{
+		Name:  "n1-standard-1",
+		RAM:   8192,
+		VCPUs: 4,
+		Disk:  fi.Int(16),
+	})
+	zones.Create(c.MockDNSClient.ServiceClient(), zones.CreateOpts{
+		Name: "minimal-openstack.k8s.local",
+	})
+	return c
+}
+
 // MockKopsVersion will set the kops version to the specified value, until Close is called
 func (h *IntegrationTestHarness) MockKopsVersion(version string) {
 	if h.originalKopsVersion != "" {

tests/integration/update_cluster/minimal_openstack/id_rsa.pub

@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQCtWu40XQo8dczLsCq0OWV+hxm9uV3WxeH9Kgh4sMzQxNtoU1pvW0XdjpkBesRKGoolfWeCLXWxpyQb1IaiMkKoz7MdhQ/6UKjMjP66aFWWp3pwD0uj0HuJ7tq4gKHKRYGTaZIRWpzUiANBrjugVgA+Sd7E/mYwc/DMXkIyRZbvhQ==

tests/integration/update_cluster/minimal_openstack/in-v1alpha2.yaml

@@ -0,0 +1,91 @@
+apiVersion: kops.k8s.io/v1alpha2
+kind: Cluster
+metadata:
+  creationTimestamp: "2017-01-01T00:00:00Z"
+  name: minimal-openstack.k8s.local
+spec:
+  api:
+    dns: {}
+  authorization:
+    alwaysAllow: {}
+  channel: stable
+  cloudConfig:
+    openstack: {}
+  cloudProvider: openstack
+  configBase: memfs://tests/minimal-openstack.k8s.local
+  etcdClusters:
+  - etcdMembers:
+    - instanceGroup: master-us-test1-a
+      name: "1"
+      volumeType: test
+    name: main
+  - etcdMembers:
+    - instanceGroup: master-us-test1-a
+      name: "1"
+      volumeType: test
+    name: events
+  openstackServiceAccount: default
+  iam:
+    legacy: false
+  kubelet:
+    anonymousAuth: false
+  kubernetesApiAccess:
+  - 0.0.0.0/0
+  kubernetesVersion: v1.14.0
+  masterPublicName: api.minimal-openstack.k8s.local
+  networking:
+    kubenet: {}
+  networkCIDR: 192.168.0.0/16
+  nonMasqueradeCIDR: 100.64.0.0/10
+  project: testproject
+  sshAccess:
+  - 0.0.0.0/0
+  subnets:
+  - name: us-test1
+    region: us-test1
+    type: Public
+  topology:
+    dns:
+      type: Private
+    masters: private
+    nodes: private
+
+---
+
+apiVersion: kops.k8s.io/v1alpha2
+kind: InstanceGroup
+metadata:
+  creationTimestamp: "2017-01-01T00:00:00Z"
+  labels:
+    kops.k8s.io/cluster: minimal-openstack.k8s.local
+  name: master-us-test1-a
+spec:
+  image: Ubuntu-20.04
+  machineType: n1-standard-1
+  maxSize: 1
+  minSize: 1
+  role: Master
+  subnets:
+  - us-test1
+  zones:
+  - us-test1-a
+
+---
+
+apiVersion: kops.k8s.io/v1alpha2
+kind: InstanceGroup
+metadata:
+  creationTimestamp: "2017-01-01T00:00:00Z"
+  labels:
+    kops.k8s.io/cluster: minimal-openstack.k8s.local
+  name: nodes
+spec:
+  image: Ubuntu-20.04
+  machineType: n1-standard-2
+  maxSize: 2
+  minSize: 2
+  role: Node
+  subnets:
+  - us-test1
+  zones:
+  - us-test1-a