Simple integration test for create cluster

Calls create cluster command
Compares to expected YAML output

cmd/kops/create_cluster.go

@@ -37,6 +37,7 @@ import (
 )
 
 type CreateClusterOptions struct {
+	ClusterName       string
 	Yes               bool
 	Target            string
 	Models            string
@@ -68,6 +69,17 @@ type CreateClusterOptions struct {
 	Bastion bool
 }
 
+func (o *CreateClusterOptions) InitDefaults() {
+	o.Yes = false
+	o.Target = cloudup.TargetDirect
+	o.Models = strings.Join(cloudup.CloudupModels, ",")
+	o.SSHPublicKey = "~/.ssh/id_rsa.pub"
+	o.Networking = "kubenet"
+	o.AssociatePublicIP = true
+	o.Channel = api.DefaultChannel
+	o.Topology = "public"
+}
+
 func NewCmdCreateCluster(f *util.Factory, out io.Writer) *cobra.Command {
 	options := &CreateClusterOptions{}
 
@@ -76,63 +88,66 @@ func NewCmdCreateCluster(f *util.Factory, out io.Writer) *cobra.Command {
 		Short: "Create cluster",
 		Long:  `Creates a k8s cluster.`,
 		Run: func(cmd *cobra.Command, args []string) {
-			err := RunCreateCluster(f, cmd, args, out, options)
+			err := rootCommand.ProcessArgs(args)
+			if err != nil {
+				exitWithError(err)
+				return
+			}
+
+			options.ClusterName = rootCommand.clusterName
+
+			err = RunCreateCluster(f, out, options)
 			if err != nil {
 				exitWithError(err)
 			}
 		},
 	}
 
-	cmd.Flags().BoolVar(&options.Yes, "yes", false, "Specify --yes to immediately create the cluster")
+	cmd.Flags().BoolVar(&options.Yes, "yes", options.Yes, "Specify --yes to immediately create the cluster")
-	cmd.Flags().StringVar(&options.Target, "target", cloudup.TargetDirect, "Target - direct, terraform")
+	cmd.Flags().StringVar(&options.Target, "target", options.Target, "Target - direct, terraform")
-	cmd.Flags().StringVar(&options.Models, "model", "config,proto,cloudup", "Models to apply (separate multiple models with commas)")
+	cmd.Flags().StringVar(&options.Models, "model", options.Models, "Models to apply (separate multiple models with commas)")
 
-	cmd.Flags().StringVar(&options.Cloud, "cloud", "", "Cloud provider to use - gce, aws")
+	cmd.Flags().StringVar(&options.Cloud, "cloud", options.Cloud, "Cloud provider to use - gce, aws")
 
-	cmd.Flags().StringVar(&options.Zones, "zones", "", "Zones in which to run the cluster")
+	cmd.Flags().StringVar(&options.Zones, "zones", options.Zones, "Zones in which to run the cluster")
-	cmd.Flags().StringVar(&options.MasterZones, "master-zones", "", "Zones in which to run masters (must be an odd number)")
+	cmd.Flags().StringVar(&options.MasterZones, "master-zones", options.MasterZones, "Zones in which to run masters (must be an odd number)")
 
-	cmd.Flags().StringVar(&options.Project, "project", "", "Project to use (must be set on GCE)")
+	cmd.Flags().StringVar(&options.Project, "project", options.Project, "Project to use (must be set on GCE)")
-	cmd.Flags().StringVar(&options.KubernetesVersion, "kubernetes-version", "", "Version of kubernetes to run (defaults to version in channel)")
+	cmd.Flags().StringVar(&options.KubernetesVersion, "kubernetes-version", options.KubernetesVersion, "Version of kubernetes to run (defaults to version in channel)")
 
-	cmd.Flags().StringVar(&options.SSHPublicKey, "ssh-public-key", "~/.ssh/id_rsa.pub", "SSH public key to use")
+	cmd.Flags().StringVar(&options.SSHPublicKey, "ssh-public-key", options.SSHPublicKey, "SSH public key to use")
 
-	cmd.Flags().StringVar(&options.NodeSize, "node-size", "", "Set instance size for nodes")
+	cmd.Flags().StringVar(&options.NodeSize, "node-size", options.NodeSize, "Set instance size for nodes")
 
-	cmd.Flags().StringVar(&options.MasterSize, "master-size", "", "Set instance size for masters")
+	cmd.Flags().StringVar(&options.MasterSize, "master-size", options.MasterSize, "Set instance size for masters")
 
-	cmd.Flags().StringVar(&options.VPCID, "vpc", "", "Set to use a shared VPC")
+	cmd.Flags().StringVar(&options.VPCID, "vpc", options.VPCID, "Set to use a shared VPC")
-	cmd.Flags().StringVar(&options.NetworkCIDR, "network-cidr", "", "Set to override the default network CIDR")
+	cmd.Flags().StringVar(&options.NetworkCIDR, "network-cidr", options.NetworkCIDR, "Set to override the default network CIDR")
 
-	cmd.Flags().IntVar(&options.NodeCount, "node-count", 0, "Set the number of nodes")
+	cmd.Flags().IntVar(&options.NodeCount, "node-count", options.NodeCount, "Set the number of nodes")
 
-	cmd.Flags().StringVar(&options.Image, "image", "", "Image to use")
+	cmd.Flags().StringVar(&options.Image, "image", options.Image, "Image to use")
 
-	cmd.Flags().StringVar(&options.Networking, "networking", "kubenet", "Networking mode to use.  kubenet (default), classic, external, cni, kopeio-vxlan, weave.")
+	cmd.Flags().StringVar(&options.Networking, "networking", options.Networking, "Networking mode to use.  kubenet (default), classic, external, cni, kopeio-vxlan, weave.")
 
-	cmd.Flags().StringVar(&options.DNSZone, "dns-zone", "", "DNS hosted zone to use (defaults to longest matching zone)")
+	cmd.Flags().StringVar(&options.DNSZone, "dns-zone", options.DNSZone, "DNS hosted zone to use (defaults to longest matching zone)")
-	cmd.Flags().StringVar(&options.OutDir, "out", "", "Path to write any local output")
+	cmd.Flags().StringVar(&options.OutDir, "out", options.OutDir, "Path to write any local output")
-	cmd.Flags().StringVar(&options.AdminAccess, "admin-access", "", "Restrict access to admin endpoints (SSH, HTTPS) to this CIDR.  If not set, access will not be restricted by IP.")
+	cmd.Flags().StringVar(&options.AdminAccess, "admin-access", options.AdminAccess, "Restrict access to admin endpoints (SSH, HTTPS) to this CIDR.  If not set, access will not be restricted by IP.")
 
-	cmd.Flags().BoolVar(&options.AssociatePublicIP, "associate-public-ip", true, "Specify --associate-public-ip=[true|false] to enable/disable association of public IP for master ASG and nodes. Default is 'true'.")
+	cmd.Flags().BoolVar(&options.AssociatePublicIP, "associate-public-ip", options.AssociatePublicIP, "Specify --associate-public-ip=[true|false] to enable/disable association of public IP for master ASG and nodes. Default is 'true'.")
 
-	cmd.Flags().StringVar(&options.Channel, "channel", api.DefaultChannel, "Channel for default versions and configuration to use")
+	cmd.Flags().StringVar(&options.Channel, "channel", options.Channel, "Channel for default versions and configuration to use")
 
 	// Network topology
-	cmd.Flags().StringVarP(&options.Topology, "topology", "t", "public", "Controls network topology for the cluster. public|private. Default is 'public'.")
+	cmd.Flags().StringVarP(&options.Topology, "topology", "t", options.Topology, "Controls network topology for the cluster. public|private. Default is 'public'.")
 
 	// Bastion
-	cmd.Flags().BoolVar(&options.Bastion, "bastion", false, "Specify --bastion=[true|false] to turn enable/disable bastion setup. Default to 'false' when topology is 'public' and defaults to 'true' if topology is 'private'.")
+	cmd.Flags().BoolVar(&options.Bastion, "bastion", options.Bastion, "Specify --bastion=[true|false] to turn enable/disable bastion setup. Default to 'false' when topology is 'public' and defaults to 'true' if topology is 'private'.")
 
 	return cmd
 }
 
-func RunCreateCluster(f *util.Factory, cmd *cobra.Command, args []string, out io.Writer, c *CreateClusterOptions) error {
+func RunCreateCluster(f *util.Factory, out io.Writer, c *CreateClusterOptions) error {
-	err := rootCommand.ProcessArgs(args)
-	if err != nil {
-		return err
-	}
-
 	isDryrun := false
 	// direct requires --yes (others do not, because they don't make changes)
 	targetName := c.Target
@@ -146,7 +161,7 @@ func RunCreateCluster(f *util.Factory, cmd *cobra.Command, args []string, out io
 		isDryrun = true
 		targetName = cloudup.TargetDryRun
 	}
-	clusterName := rootCommand.clusterName
+	clusterName := c.ClusterName
 	if clusterName == "" {
 		return fmt.Errorf("--name is required")
 	}
@@ -161,7 +176,7 @@ func RunCreateCluster(f *util.Factory, cmd *cobra.Command, args []string, out io
 		}
 	}
 
-	clientset, err := rootCommand.Clientset()
+	clientset, err := f.Clientset()
 	if err != nil {
 		return err
 	}
@@ -401,11 +416,7 @@ func RunCreateCluster(f *util.Factory, cmd *cobra.Command, args []string, out io
 			Masters: api.TopologyPrivate,
 			Nodes:   api.TopologyPrivate,
 		}
-		if cmd.Flags().Changed("bastion") {
+		cluster.Spec.Topology.Bastion = &api.BastionSpec{Enable: c.Bastion}
-			cluster.Spec.Topology.Bastion = &api.BastionSpec{Enable: c.Bastion}
-		} else {
-			cluster.Spec.Topology.Bastion = &api.BastionSpec{Enable: true}
-		}
 	case "":
 		glog.Warningf("Empty topology. Defaulting to public topology without bastion")
 		cluster.Spec.Topology = &api.TopologySpec{
@@ -497,60 +508,62 @@ func RunCreateCluster(f *util.Factory, cmd *cobra.Command, args []string, out io
 		}
 	}
 
-	if isDryrun {
+	if targetName != "" {
-		fmt.Print("Previewing changes that will be made:\n\n")
+		if isDryrun {
-	}
+			fmt.Print("Previewing changes that will be made:\n\n")
-
-	applyCmd := &cloudup.ApplyClusterCmd{
-		Cluster:    fullCluster,
-		Models:     strings.Split(c.Models, ","),
-		Clientset:  clientset,
-		TargetName: targetName,
-		OutDir:     c.OutDir,
-		DryRun:     isDryrun,
-	}
-
-	err = applyCmd.Run()
-	if err != nil {
-		return err
-	}
-
-	if isDryrun {
-		var sb bytes.Buffer
-		fmt.Fprintf(&sb, "\n")
-		fmt.Fprintf(&sb, "Cluster configuration has been created.\n")
-		fmt.Fprintf(&sb, "\n")
-		fmt.Fprintf(&sb, "Suggestions:\n")
-		fmt.Fprintf(&sb, " * list clusters with: kops get cluster\n")
-		fmt.Fprintf(&sb, " * edit this cluster with: kops edit cluster %s\n", clusterName)
-		if len(nodes) > 0 {
-			fmt.Fprintf(&sb, " * edit your node instance group: kops edit ig --name=%s %s\n", clusterName, nodes[0].ObjectMeta.Name)
-		}
-		if len(masters) > 0 {
-			fmt.Fprintf(&sb, " * edit your master instance group: kops edit ig --name=%s %s\n", clusterName, masters[0].ObjectMeta.Name)
-		}
-		fmt.Fprintf(&sb, "\n")
-		fmt.Fprintf(&sb, "Finally configure your cluster with: kops update cluster %s --yes\n", clusterName)
-		fmt.Fprintf(&sb, "\n")
-
-		_, err := out.Write(sb.Bytes())
-		if err != nil {
-			return fmt.Errorf("error writing to output: %v", err)
-		}
-	} else {
-		glog.Infof("Exporting kubecfg for cluster")
-
-		x := &kutil.CreateKubecfg{
-			ContextName:  cluster.ObjectMeta.Name,
-			KeyStore:     keyStore,
-			SecretStore:  secretStore,
-			KubeMasterIP: cluster.Spec.MasterPublicName,
 		}
 
-		err = x.WriteKubecfg()
+		applyCmd := &cloudup.ApplyClusterCmd{
+			Cluster:    fullCluster,
+			Models:     strings.Split(c.Models, ","),
+			Clientset:  clientset,
+			TargetName: targetName,
+			OutDir:     c.OutDir,
+			DryRun:     isDryrun,
+		}
+
+		err = applyCmd.Run()
 		if err != nil {
 			return err
 		}
+
+		if isDryrun {
+			var sb bytes.Buffer
+			fmt.Fprintf(&sb, "\n")
+			fmt.Fprintf(&sb, "Cluster configuration has been created.\n")
+			fmt.Fprintf(&sb, "\n")
+			fmt.Fprintf(&sb, "Suggestions:\n")
+			fmt.Fprintf(&sb, " * list clusters with: kops get cluster\n")
+			fmt.Fprintf(&sb, " * edit this cluster with: kops edit cluster %s\n", clusterName)
+			if len(nodes) > 0 {
+				fmt.Fprintf(&sb, " * edit your node instance group: kops edit ig --name=%s %s\n", clusterName, nodes[0].ObjectMeta.Name)
+			}
+			if len(masters) > 0 {
+				fmt.Fprintf(&sb, " * edit your master instance group: kops edit ig --name=%s %s\n", clusterName, masters[0].ObjectMeta.Name)
+			}
+			fmt.Fprintf(&sb, "\n")
+			fmt.Fprintf(&sb, "Finally configure your cluster with: kops update cluster %s --yes\n", clusterName)
+			fmt.Fprintf(&sb, "\n")
+
+			_, err := out.Write(sb.Bytes())
+			if err != nil {
+				return fmt.Errorf("error writing to output: %v", err)
+			}
+		} else {
+			glog.Infof("Exporting kubecfg for cluster")
+
+			x := &kutil.CreateKubecfg{
+				ContextName:  cluster.ObjectMeta.Name,
+				KeyStore:     keyStore,
+				SecretStore:  secretStore,
+				KubeMasterIP: cluster.Spec.MasterPublicName,
+			}
+
+			err = x.WriteKubecfg()
+			if err != nil {
+				return err
+			}
+		}
 	}
 
 	return nil

cmd/kops/create_cluster_integration_test.go

@@ -0,0 +1,166 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"bytes"
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/route53"
+	"github.com/golang/glog"
+	"io/ioutil"
+	"k8s.io/kops/cloudmock/aws/mockec2"
+	"k8s.io/kops/cloudmock/aws/mockroute53"
+	"k8s.io/kops/cmd/kops/util"
+	"k8s.io/kops/pkg/apis/kops"
+	"k8s.io/kops/pkg/diff"
+	"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
+	"k8s.io/kops/util/pkg/vfs"
+	k8sapi "k8s.io/kubernetes/pkg/api"
+	"k8s.io/kubernetes/pkg/api/unversioned"
+	"path"
+	"strings"
+	"testing"
+	"time"
+)
+
+var MagicTimestamp = unversioned.Time{Time: time.Date(2017, 1, 1, 0, 0, 0, 0, time.UTC)}
+
+// TestMinimal runs kops create cluster minimal.example.com --zones us-test-1a
+func TestCreateClusterMinimal(t *testing.T) {
+	runCreateClusterIntegrationTest(t, "../../tests/integration/create_cluster/minimal", 2)
+}
+
+func runCreateClusterIntegrationTest(t *testing.T, srcDir string, expectedInstanceGroups int) {
+	var stdout bytes.Buffer
+
+	optionsYAML := "options.yaml"
+	expectedClusterPath := "cluster.yaml"
+
+	factoryOptions := &util.FactoryOptions{}
+	factoryOptions.RegistryPath = "memfs://tests"
+
+	vfs.Context.ResetMemfsContext(true)
+
+	cloud := awsup.InstallMockAWSCloud("us-test-1", "abc")
+	mockEC2 := &mockec2.MockEC2{}
+	cloud.MockEC2 = mockEC2
+	mockRoute53 := &mockroute53.MockRoute53{}
+	cloud.MockRoute53 = mockRoute53
+
+	mockRoute53.Zones = append(mockRoute53.Zones, &route53.HostedZone{
+		Id:   aws.String("/hostedzone/Z1AFAKE1ZON3YO"),
+		Name: aws.String("example.com."),
+	})
+
+	factory := util.NewFactory(factoryOptions)
+
+	{
+		optionsBytes, err := ioutil.ReadFile(path.Join(srcDir, optionsYAML))
+		if err != nil {
+			t.Fatalf("error reading options file: %v", err)
+		}
+
+		options := &CreateClusterOptions{}
+		options.InitDefaults()
+
+		err = kops.ParseRawYaml(optionsBytes, options)
+		if err != nil {
+			t.Fatalf("error parsing options: %v", err)
+		}
+
+		// No preview
+		options.Target = ""
+
+		err = RunCreateCluster(factory, &stdout, options)
+		if err != nil {
+			t.Fatalf("error running create cluster: %v", err)
+		}
+	}
+
+	clientset, err := factory.Clientset()
+	if err != nil {
+		t.Fatalf("error getting clientset: %v", err)
+	}
+
+	// Compare cluster
+	clusters, err := clientset.Clusters().List(k8sapi.ListOptions{})
+	if err != nil {
+		t.Fatalf("error listing clusters: %v", err)
+	}
+
+	if len(clusters.Items) != 1 {
+		t.Fatalf("expected one cluster, found %d", len(clusters.Items))
+	}
+	for _, cluster := range clusters.Items {
+		cluster.ObjectMeta.CreationTimestamp = MagicTimestamp
+		actualYAMLBytes, err := kops.ToVersionedYaml(&cluster)
+		if err != nil {
+			t.Fatalf("unexpected error serializing cluster: %v", err)
+		}
+		expectedYAMLBytes, err := ioutil.ReadFile(path.Join(srcDir, expectedClusterPath))
+		if err != nil {
+			t.Fatalf("unexpected error reading expected cluster: %v", err)
+		}
+
+		actualYAML := strings.TrimSpace(string(actualYAMLBytes))
+		expectedYAML := strings.TrimSpace(string(expectedYAMLBytes))
+
+		if actualYAML != expectedYAML {
+			glog.Infof("Actual cluster:\n%s\n", actualYAML)
+
+			diffString := diff.FormatDiff(expectedYAML, actualYAML)
+			t.Logf("diff:\n%s\n", diffString)
+
+			t.Fatalf("cluster differed from expected")
+		}
+	}
+
+	// Compare instance groups
+
+	instanceGroups, err := clientset.InstanceGroups(clusters.Items[0].ObjectMeta.Name).List(k8sapi.ListOptions{})
+	if err != nil {
+		t.Fatalf("error listing instance groups: %v", err)
+	}
+
+	if len(instanceGroups.Items) != expectedInstanceGroups {
+		t.Fatalf("expected %d instance groups, found %d", expectedInstanceGroups, len(instanceGroups.Items))
+	}
+	for _, ig := range instanceGroups.Items {
+		ig.ObjectMeta.CreationTimestamp = MagicTimestamp
+
+		actualYAMLBytes, err := kops.ToVersionedYaml(&ig)
+		if err != nil {
+			t.Fatalf("unexpected error serializing InstanceGroup: %v", err)
+		}
+		expectedYAMLBytes, err := ioutil.ReadFile(path.Join(srcDir, ig.ObjectMeta.Name+".yaml"))
+		if err != nil {
+			t.Fatalf("unexpected error reading expected InstanceGroup: %v", err)
+		}
+
+		actualYAML := strings.TrimSpace(string(actualYAMLBytes))
+		expectedYAML := strings.TrimSpace(string(expectedYAMLBytes))
+
+		if actualYAML != expectedYAML {
+			glog.Infof("Actual IG %q:\n%s\n", ig.ObjectMeta.Name, actualYAML)
+
+			diffString := diff.FormatDiff(expectedYAML, actualYAML)
+			t.Logf("diff:\n%s\n", diffString)
+
+			t.Fatalf("instance group %q differed from expected", ig.ObjectMeta.Name)
+		}
+	}
+}

cmd/kops/integration_test.go

@@ -178,7 +178,7 @@ func runTest(t *testing.T, clusterName string, srcDir string) {
 
 		if !bytes.Equal(actualTF, expectedTF) {
 			diffString := diff.FormatDiff(string(expectedTF), string(actualTF))
-			t.Log("diff:\n", diffString)
+			t.Logf("diff:\n%s\n", diffString)
 
 			t.Fatalf("terraform output differed from expected")
 		}

pkg/diff/diff_test.go

@@ -118,7 +118,6 @@ F2`
 	}
 }
 
-
 func Test_Diff_ChangedLine(t *testing.T) {
 	l := `ABC123
 Line2

tests/integration/create_cluster/minimal/cluster.yaml

@@ -0,0 +1,33 @@
+apiVersion: kops/v1alpha1
+kind: Cluster
+metadata:
+  creationTimestamp: "2017-01-01T00:00:00Z"
+  name: minimal.example.com
+spec:
+  channel: stable
+  cloudProvider: aws
+  configBase: memfs://tests/minimal.example.com
+  etcdClusters:
+  - etcdMembers:
+    - name: us-test-1a
+      zone: us-test-1a
+    name: main
+  - etcdMembers:
+    - name: us-test-1a
+      zone: us-test-1a
+    name: events
+  kubernetesVersion: v1.4.6
+  masterPublicName: api.minimal.example.com
+  networkCIDR: 172.20.0.0/16
+  networking:
+    kubenet: {}
+  nonMasqueradeCIDR: 100.64.0.0/10
+  topology:
+    bastion:
+      idleTimeout: 120
+      machineType: t2.medium
+    masters: public
+    nodes: public
+  zones:
+  - cidr: 172.20.32.0/19
+    name: us-test-1a

tests/integration/create_cluster/minimal/master-us-test-1a.yaml

@@ -0,0 +1,14 @@
+apiVersion: kops/v1alpha1
+kind: InstanceGroup
+metadata:
+  creationTimestamp: "2017-01-01T00:00:00Z"
+  name: master-us-test-1a
+spec:
+  associatePublicIp: true
+  image: kope.io/k8s-1.4-debian-jessie-amd64-hvm-ebs-2016-10-21
+  machineType: m3.medium
+  maxSize: 1
+  minSize: 1
+  role: Master
+  zones:
+  - us-test-1a

tests/integration/create_cluster/minimal/nodes.yaml

@@ -0,0 +1,14 @@
+apiVersion: kops/v1alpha1
+kind: InstanceGroup
+metadata:
+  creationTimestamp: "2017-01-01T00:00:00Z"
+  name: nodes
+spec:
+  associatePublicIp: true
+  image: kope.io/k8s-1.4-debian-jessie-amd64-hvm-ebs-2016-10-21
+  machineType: t2.medium
+  maxSize: 2
+  minSize: 2
+  role: Node
+  zones:
+  - us-test-1a

tests/integration/create_cluster/minimal/options.yaml

@@ -0,0 +1,3 @@
+ClusterName: minimal.example.com
+Zones: us-test-1a
+Cloud: aws