Fixing YAML and JSON output across multiple cli functions. Also

Added --dry-run for create_ig and create_cluster

cmd/kops/create.go

@@ -130,6 +130,7 @@ func RunCreate(f *util.Factory, out io.Writer, c *CreateOptions) error {
 			return fmt.Errorf("error reading file %q: %v", f, err)
 		}
 
+		// TODO: this does not support a JSON array
 		sections := bytes.Split(contents, []byte("\n---\n"))
 		for _, section := range sections {
 			defaults := &schema.GroupVersionKind{

cmd/kops/create_cluster.go

@@ -28,6 +28,7 @@ import (
 	"github.com/golang/glog"
 	"github.com/spf13/cobra"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/kops"
 	"k8s.io/kops/cmd/kops/util"
@@ -124,6 +125,11 @@ type CreateClusterOptions struct {
 
 	// ConfigBase is the location where we will store the configuration, it defaults to the state store
 	ConfigBase string
+
+	// DryRun mode output a cluster manifest of Output type.
+	DryRun bool
+	// Output type during a DryRun
+	Output string
 }
 
 func (o *CreateClusterOptions) InitDefaults() {
@@ -190,6 +196,11 @@ var (
           --project my-gce-project \
           --image "ubuntu-os-cloud/ubuntu-1604-xenial-v20170202" \
           --yes
+	# Create manifest for a cluster in AWS
+	kops create cluster --name=kubernetes-cluster.example.com \
+	--state=s3://kops-state-1234 --zones=eu-west-1a \
+	--node-count=2 --dry-run -oyaml
+
 	`))
 
 	create_cluster_short = i18n.T("Create a Kubernetes cluster.")
@@ -227,7 +238,7 @@ func NewCmdCreateCluster(f *util.Factory, out io.Writer) *cobra.Command {
 	}
 
 	cmd.Flags().BoolVar(&options.Yes, "yes", options.Yes, "Specify --yes to immediately create the cluster")
-	cmd.Flags().StringVar(&options.Target, "target", options.Target, "Target - direct, terraform, cloudformation")
+	cmd.Flags().StringVar(&options.Target, "target", options.Target, fmt.Sprintf("Valid targets: %s, %s, %s. Set this flag to %s if you want kops to generate terraform", cloudup.TargetDirect, cloudup.TargetTerraform, cloudup.TargetDirect, cloudup.TargetTerraform))
 	cmd.Flags().StringVar(&options.Models, "model", options.Models, "Models to apply (separate multiple models with commas)")
 
 	// Configuration / state location
@@ -297,6 +308,10 @@ func NewCmdCreateCluster(f *util.Factory, out io.Writer) *cobra.Command {
 
 	cmd.Flags().StringVar(&options.APILoadBalancerType, "api-loadbalancer-type", options.APILoadBalancerType, "Sets the API loadbalancer type to either 'public' or 'internal'")
 
+	// DryRun mode that will print YAML or JSON
+	cmd.Flags().BoolVar(&options.DryRun, "dry-run", options.DryRun, "If true, only print the object that would be sent, without sending it. This flag can be used to create a cluster YAML or JSON manifest.")
+	cmd.Flags().StringVarP(&options.Output, "output", "o", options.Output, "Ouput format. One of json|yaml. Used with the --dry-run flag.")
+
 	if featureflag.SpecOverrideFlag.Enabled() {
 		cmd.Flags().StringSliceVar(&options.Overrides, "override", options.Overrides, "Directly configure values in the spec")
 	}
@@ -326,6 +341,11 @@ func RunCreateCluster(f *util.Factory, out io.Writer, c *CreateClusterOptions) e
 		isDryrun = true
 		targetName = cloudup.TargetDryRun
 	}
+
+	if c.DryRun && c.Output == "" {
+		return fmt.Errorf("unable to execute --dry-run without setting --output")
+	}
+
 	clusterName := c.ClusterName
 	if clusterName == "" {
 		return fmt.Errorf("--name is required")
@@ -987,6 +1007,32 @@ func RunCreateCluster(f *util.Factory, out io.Writer, c *CreateClusterOptions) e
 		return err
 	}
 
+	if c.DryRun {
+		var obj []runtime.Object
+		obj = append(obj, cluster)
+
+		for _, group := range fullInstanceGroups {
+			// Cluster name is not populated, and we need it
+			group.ObjectMeta.Labels = make(map[string]string)
+			group.ObjectMeta.Labels[api.LabelClusterName] = cluster.ObjectMeta.Name
+			obj = append(obj, group)
+		}
+		switch c.Output {
+		case OutputYaml:
+			if err := fullOutputYAML(out, obj...); err != nil {
+				return fmt.Errorf("error writing cluster yaml to stdout: %v", err)
+			}
+			return nil
+		case OutputJSON:
+			if err := fullOutputJSON(out, obj...); err != nil {
+				return fmt.Errorf("error writing cluster json to stdout: %v", err)
+			}
+			return nil
+		default:
+			return fmt.Errorf("unsupported output type %q", c.Output)
+		}
+	}
+
 	// Note we perform as much validation as we can, before writing a bad config
 	err = registry.CreateClusterConfig(clientset, cluster, fullInstanceGroups)
 	if err != nil {
@@ -1010,6 +1056,7 @@ func RunCreateCluster(f *util.Factory, out io.Writer, c *CreateClusterOptions) e
 		}
 	}
 
+	// Can we acutally get to this if??
 	if targetName != "" {
 		if isDryrun {
 			fmt.Fprintf(out, "Previewing changes that will be made:\n\n")

cmd/kops/create_ig.go

@@ -39,6 +39,10 @@ import (
 type CreateInstanceGroupOptions struct {
 	Role    string
 	Subnets []string
+	// DryRun mode output an ig manifest of Output type.
+	DryRun bool
+	// Output type during a DryRun
+	Output string
 }
 
 var (
@@ -52,6 +56,10 @@ var (
 		# Create an instancegroup for the k8s-cluster.example.com cluster.
 		kops create ig --name=k8s-cluster.example.com node-example \
 		  --role node --subnet my-subnet-name
+
+		# Create a YAML manifest for an instancegroup for the k8s-cluster.example.com cluster.
+		kops create ig --name=k8s-cluster.example.com node-example \
+		  --role node --subnet my-subnet-name --dry-run -oyaml
 		`))
 
 	create_ig_short = i18n.T(`Create an instancegroup.`)
@@ -85,6 +93,9 @@ func NewCmdCreateInstanceGroup(f *util.Factory, out io.Writer) *cobra.Command {
 
 	cmd.Flags().StringVar(&options.Role, "role", options.Role, "Type of instance group to create ("+strings.Join(allRoles, ",")+")")
 	cmd.Flags().StringSliceVar(&options.Subnets, "subnet", options.Subnets, "Subnets in which to create instance group")
+	// DryRun mode that will print YAML or JSON
+	cmd.Flags().BoolVar(&options.DryRun, "dry-run", options.DryRun, "If true, only print the object that would be sent, without sending it. This flag can be used to create a cluster YAML or JSON manifest.")
+	cmd.Flags().StringVarP(&options.Output, "output", "o", options.Output, "Ouput format. One of json|yaml")
 
 	return cmd
 }
@@ -142,6 +153,32 @@ func RunCreateInstanceGroup(f *util.Factory, cmd *cobra.Command, args []string,
 		return err
 	}
 
+	if options.DryRun {
+
+		if options.Output == "" {
+			return fmt.Errorf("must set output flag; yaml or json")
+		}
+
+		// Cluster name is not populated, and we need it
+		ig.ObjectMeta.Labels = make(map[string]string)
+		ig.ObjectMeta.Labels[api.LabelClusterName] = cluster.ObjectMeta.Name
+
+		switch options.Output {
+		case OutputYaml:
+			if err := fullOutputYAML(out, ig); err != nil {
+				return fmt.Errorf("error writing cluster yaml to stdout: %v", err)
+			}
+			return nil
+		case OutputJSON:
+			if err := fullOutputJSON(out, ig); err != nil {
+				return fmt.Errorf("error writing cluster json to stdout: %v", err)
+			}
+			return nil
+		default:
+			return fmt.Errorf("unsupported output type %q", options.Output)
+		}
+	}
+
 	var (
 		edit = editor.NewDefaultEditor(editorEnvs)
 	)

cmd/kops/get.go

@@ -17,7 +17,6 @@ limitations under the License.
 package main
 
 import (
-	"encoding/json"
 	"fmt"
 	"io"
 
@@ -153,35 +152,27 @@ func RunGet(context Factory, out io.Writer, options *GetOptions) error {
 		return err
 	}
 
+	var obj []runtime.Object
+	if options.output != OutputTable {
+		obj = append(obj, cluster)
+		for _, group := range instancegroups {
+			obj = append(obj, group)
+		}
+	}
+
 	switch options.output {
 	case OutputYaml:
-
+		if err := fullOutputYAML(out, obj...); err != nil {
-		err = clusterOutputYAML(clusters, out)
+			return fmt.Errorf("error writing cluster yaml to stdout: %v", err)
-		if err != nil {
-			return err
 		}
 
-		if err := writeYAMLSep(out); err != nil {
+		return nil
-			return err
-		}
-
-		err = igOutputYAML(instancegroups, out)
-		if err != nil {
-			return err
-		}
 
 	case OutputJSON:
-		return fmt.Errorf("not implemented")
+		if err := fullOutputJSON(out, obj...); err != nil {
-		// TODO this is not outputing valid json.  Not sure what cluster and instance groups should look like
+			return fmt.Errorf("error writing cluster json to stdout: %v", err)
-		/*
-			err = clusterOutputJson(clusters,out)
-			if err != nil {
-				return err
 		}
-			err = igOutputJson(instancegroups,out)
+		return nil
-			if err != nil {
-				return err
-			}*/
 
 	case OutputTable:
 		fmt.Fprintf(os.Stdout, "Cluster\n")
@@ -235,7 +226,7 @@ func marshalYaml(obj runtime.Object) ([]byte, error) {
 
 // obj must be a pointer to a marshalable object
 func marshalJSON(obj runtime.Object) ([]byte, error) {
-	j, err := json.MarshalIndent(obj, "", "  ")
+	j, err := kopscodecs.ToVersionedJSON(obj)
 	if err != nil {
 		return nil, fmt.Errorf("error marshaling json: %v", err)
 	}

cmd/kops/get_cluster.go

@@ -18,13 +18,13 @@ package main
 
 import (
 	"fmt"
+	"io"
 	"os"
 	"strings"
 
-	"io"
-
 	"github.com/spf13/cobra"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/kops/cmd/kops/util"
 	api "k8s.io/kops/pkg/apis/kops"
@@ -133,7 +133,7 @@ func RunGetClusters(context Factory, out io.Writer, options *GetClusterOptions)
 	}
 
 	if len(clusters) == 0 {
-		return fmt.Errorf("No clusters found")
+		return fmt.Errorf("no clusters found")
 	}
 
 	if options.FullSpec {
@@ -146,14 +146,20 @@ func RunGetClusters(context Factory, out io.Writer, options *GetClusterOptions)
 		fmt.Fprint(out, get_cluster_full_warning)
 	}
 
+	var obj []runtime.Object
+	if options.output != OutputTable {
+		for _, c := range clusters {
+			obj = append(obj, c)
+		}
+	}
+
 	switch options.output {
 	case OutputTable:
 		return clusterOutputTable(clusters, out)
 	case OutputYaml:
-		return clusterOutputYAML(clusters, out)
+		return fullOutputYAML(out, obj...)
 	case OutputJSON:
-		return clusterOutputJson(clusters, out)
+		return fullOutputJSON(out, obj...)
-
 	default:
 		return fmt.Errorf("Unknown output format: %q", options.output)
 	}
@@ -206,23 +212,47 @@ func clusterOutputTable(clusters []*api.Cluster, out io.Writer) error {
 	return t.Render(clusters, out, "NAME", "CLOUD", "ZONES")
 }
 
-func clusterOutputJson(clusters []*api.Cluster, out io.Writer) error {
+// fullOutputJson outputs the marshalled JSON of a list of clusters and instance groups.  It will handle
-	for _, cluster := range clusters {
+// nils for clusters and instanceGroups slices.
-		if err := marshalToWriter(cluster, marshalJSON, out); err != nil {
+func fullOutputJSON(out io.Writer, args ...runtime.Object) error {
+	argsLen := len(args)
+
+	if argsLen > 1 {
+		if _, err := fmt.Fprint(out, "["); err != nil {
 			return err
 		}
 	}
+
+	for i, arg := range args {
+		if i != 0 {
+			if _, err := fmt.Fprint(out, ","); err != nil {
+				return err
+			}
+		}
+		if err := marshalToWriter(arg, marshalJSON, out); err != nil {
+			return err
+		}
+	}
+
+	if argsLen > 1 {
+		if _, err := fmt.Fprint(out, "]"); err != nil {
+			return err
+		}
+	}
+
 	return nil
 }
 
-func clusterOutputYAML(clusters []*api.Cluster, out io.Writer) error {
+// fullOutputJson outputs the marshalled JSON of a list of clusters and instance groups.  It will handle
-	for i, cluster := range clusters {
+// nils for clusters and instanceGroups slices.
+func fullOutputYAML(out io.Writer, args ...runtime.Object) error {
+	for i, obj := range args {
 		if i != 0 {
 			if err := writeYAMLSep(out); err != nil {
 				return fmt.Errorf("error writing to stdout: %v", err)
 			}
 		}
-		if err := marshalToWriter(cluster, marshalYaml, out); err != nil {
+		if err := marshalToWriter(obj, marshalYaml, out); err != nil {
 			return err
 		}
 	}

cmd/kops/get_federation.go

@@ -24,6 +24,7 @@ import (
 
 	"github.com/spf13/cobra"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/kops/cmd/kops/util"
 	api "k8s.io/kops/pkg/apis/kops"
 	"k8s.io/kops/util/pkg/tables"
@@ -85,10 +86,16 @@ func RunGetFederations(context Factory, out io.Writer, options *GetFederationOpt
 	if len(federations) == 0 {
 		return fmt.Errorf("No federations found")
 	}
+
+	var obj []runtime.Object
+	if options.output != OutputTable {
+		for _, c := range federations {
+			obj = append(obj, c)
+		}
+	}
+
 	switch options.output {
-
 	case OutputTable:
-
 		t := &tables.Table{}
 		t.AddColumn("NAME", func(f *api.Federation) string {
 			return f.ObjectMeta.Name
@@ -102,25 +109,10 @@ func RunGetFederations(context Factory, out io.Writer, options *GetFederationOpt
 		return t.Render(federations, out, "NAME", "CONTROLLERS", "MEMBERS")
 
 	case OutputYaml:
-		for i, f := range federations {
+		return fullOutputYAML(out, obj...)
-			if i != 0 {
-				_, err = out.Write([]byte("\n\n---\n\n"))
-				if err != nil {
-					return fmt.Errorf("error writing to stdout: %v", err)
-				}
-			}
-			if err := marshalToWriter(f, marshalYaml, os.Stdout); err != nil {
-				return err
-			}
-		}
 	case OutputJSON:
-		for _, f := range federations {
+		return fullOutputJSON(out, obj...)
-			if err := marshalToWriter(f, marshalJSON, os.Stdout); err != nil {
-				return err
-			}
-		}
 	default:
 		return fmt.Errorf("Unknown output format: %q", options.output)
 	}
-	return nil
 }

cmd/kops/get_instancegroups.go

@@ -24,6 +24,7 @@ import (
 
 	"github.com/spf13/cobra"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/kops/cmd/kops/util"
 	api "k8s.io/kops/pkg/apis/kops"
 	"k8s.io/kops/pkg/formatter"
@@ -112,14 +113,20 @@ func RunGetInstanceGroups(options *GetInstanceGroupsOptions, args []string) erro
 		return fmt.Errorf("No InstanceGroup objects found")
 	}
 
-	switch options.output {
+	var obj []runtime.Object
+	if options.output != OutputTable {
+		for _, c := range instancegroups {
+			obj = append(obj, c)
+		}
+	}
 
+	switch options.output {
 	case OutputTable:
 		return igOutputTable(cluster, instancegroups, out)
 	case OutputYaml:
-		return igOutputYAML(instancegroups, out)
+		return fullOutputYAML(out, obj...)
 	case OutputJSON:
-		return igOutputJson(instancegroups, out)
+		return fullOutputJSON(out, obj...)
 	default:
 		return fmt.Errorf("Unknown output format: %q", options.output)
 	}
@@ -176,29 +183,6 @@ func igOutputTable(cluster *api.Cluster, instancegroups []*api.InstanceGroup, ou
 	return t.Render(instancegroups, os.Stdout, "NAME", "ROLE", "MACHINETYPE", "MIN", "MAX", "ZONES")
 }
 
-func igOutputJson(instanceGroups []*api.InstanceGroup, out io.Writer) error {
-	for _, ig := range instanceGroups {
-		if err := marshalToWriter(ig, marshalJSON, out); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-func igOutputYAML(instanceGroups []*api.InstanceGroup, out io.Writer) error {
-	for i, ig := range instanceGroups {
-		if i != 0 {
-			if err := writeYAMLSep(out); err != nil {
-				return fmt.Errorf("error writing to stdout: %v", err)
-			}
-		}
-		if err := marshalToWriter(ig, marshalYaml, out); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
 func int32PointerToString(v *int32) string {
 	if v == nil {
 		return "-"

docs/cli/kops_create_cluster.md

@@ -56,6 +56,10 @@ kops create cluster
   --project my-gce-project \
   --image "ubuntu-os-cloud/ubuntu-1604-xenial-v20170202" \
   --yes
+  # Create manifest for a cluster in AWS
+  kops create cluster --name=kubernetes-cluster.example.com \
+  --state=s3://kops-state-1234 --zones=eu-west-1a \
+  --node-count=2 --dry-run -oyaml
 ```
 
 ### Options
@@ -71,6 +75,7 @@ kops create cluster
       --cloud-labels string                  A list of KV pairs used to tag all instance groups in AWS (eg "Owner=John Doe,Team=Some Team").
       --dns string                           DNS hosted zone to use: public|private. Default is 'public'. (default "Public")
       --dns-zone string                      DNS hosted zone to use (defaults to longest matching zone)
+      --dry-run                              If true, only print the object that would be sent, without sending it. This flag can be used to create a cluster YAML or JSON manifest.
       --encrypt-etcd-storage                 Generate key in aws kms and use it for encrypt etcd volumes
       --image string                         Image to use for all instances.
       --kubernetes-version string            Version of kubernetes to run (defaults to version in channel)
@@ -89,10 +94,11 @@ kops create cluster
       --node-tenancy string                  The tenancy of the node group on AWS. Can be either default or dedicated.
       --node-volume-size int32               Set instance volume size (in GB) for nodes
       --out string                           Path to write any local output
+  -o, --output string                        Ouput format. One of json|yaml. Used with the --dry-run flag.
       --project string                       Project to use (must be set on GCE)
       --ssh-access stringSlice               Restrict SSH access to this CIDR.  If not set, access will not be restricted by IP. (default [0.0.0.0/0])
       --ssh-public-key string                SSH public key to use (default "~/.ssh/id_rsa.pub")
-      --target string                        Target - direct, terraform, cloudformation (default "direct")
+      --target string                        Valid targets: direct, terraform, direct. Set this flag to terraform if you want kops to generate terraform (default "direct")
   -t, --topology string                      Controls network topology for the cluster. public|private. Default is 'public'. (default "public")
       --vpc string                           Set to use a shared VPC
       --yes                                  Specify --yes to immediately create the cluster

docs/cli/kops_create_instancegroup.md

@@ -20,11 +20,17 @@ kops create instancegroup
   # Create an instancegroup for the k8s-cluster.example.com cluster.
   kops create ig --name=k8s-cluster.example.com node-example \
   --role node --subnet my-subnet-name
+  
+  # Create a YAML manifest for an instancegroup for the k8s-cluster.example.com cluster.
+  kops create ig --name=k8s-cluster.example.com node-example \
+  --role node --subnet my-subnet-name --dry-run -oyaml
 ```
 
 ### Options
 
 ```
+      --dry-run              If true, only print the object that would be sent, without sending it. This flag can be used to create a cluster YAML or JSON manifest.
+  -o, --output string        Ouput format. One of json|yaml
       --role string          Type of instance group to create (Node,Master,Bastion) (default "Node")
       --subnet stringSlice   Subnets in which to create instance group
 ```

pkg/kopscodecs/codecs.go

@@ -45,12 +45,12 @@ func init() {
 	install.Install(GroupFactoryRegistry, Registry, Scheme)
 }
 
-func encoder(gv runtime.GroupVersioner) runtime.Encoder {
+func encoder(gv runtime.GroupVersioner, mediaType string) runtime.Encoder {
-	yaml, ok := runtime.SerializerInfoForMediaType(Codecs.SupportedMediaTypes(), "application/yaml")
+	e, ok := runtime.SerializerInfoForMediaType(Codecs.SupportedMediaTypes(), mediaType)
 	if !ok {
-		glog.Fatalf("no YAML serializer registered")
+		glog.Fatalf("no %s serializer registered", mediaType)
 	}
-	return Codecs.EncoderForVersion(yaml.Serializer, gv)
+	return Codecs.EncoderForVersion(e.Serializer, gv)
 }
 
 func decoder() runtime.Decoder {
@@ -68,7 +68,22 @@ func ToVersionedYaml(obj runtime.Object) ([]byte, error) {
 // ToVersionedYamlWithVersion encodes the object to YAML, in a specified API version
 func ToVersionedYamlWithVersion(obj runtime.Object, version runtime.GroupVersioner) ([]byte, error) {
 	var w bytes.Buffer
-	err := encoder(version).Encode(obj, &w)
+	err := encoder(version, "application/yaml").Encode(obj, &w)
+	if err != nil {
+		return nil, fmt.Errorf("error encoding %T: %v", obj, err)
+	}
+	return w.Bytes(), nil
+}
+
+// ToVersionedJSON encodes the object to JSON
+func ToVersionedJSON(obj runtime.Object) ([]byte, error) {
+	return ToVersionedJSONWithVersion(obj, v1alpha2.SchemeGroupVersion)
+}
+
+// ToVersionedJSONWithVersion encodes the object to JSON, in a specified API version
+func ToVersionedJSONWithVersion(obj runtime.Object, version runtime.GroupVersioner) ([]byte, error) {
+	var w bytes.Buffer
+	err := encoder(version, "application/json").Encode(obj, &w)
 	if err != nil {
 		return nil, fmt.Errorf("error encoding %T: %v", obj, err)
 	}