Merge pull request #515 from justinsb/channel

Simple addon manager

Makefile

@@ -1,5 +1,7 @@
 all: kops
 
+.PHONY: channels
+
 DOCKER_REGISTRY?=gcr.io/must-override/
 S3_BUCKET?=s3://must-override/
 GCS_LOCATION?=gs://must-override
@@ -62,6 +64,9 @@ godeps:
 	glide install --strip-vendor --strip-vcs
 
 gofmt:
+	gofmt -w -s cmd/
+	gofmt -w -s channels/
+	gofmt -w -s util/
 	gofmt -w -s cmd/
 	gofmt -w -s upup/pkg/
 	gofmt -w -s protokube/cmd
@@ -176,3 +181,11 @@ copydeps:
 
 ci: kops nodeup-gocode test
 	echo "Done"
+
+
+
+# --------------------------------------------------
+# channel tool
+
+channels:
+	go install ${EXTRA_BUILDFLAGS} -ldflags "-X main.BuildVersion=${VERSION} ${EXTRA_LDFLAGS}" k8s.io/kops/channels/cmd/...

addons/dashboard/addon.yaml

@@ -0,0 +1,14 @@
+kind: Addons
+metadata:
+  name: dashboard
+spec:
+  addons:
+  - version: 1.1.0
+    selector:
+      k8s-addon: kubernetes-dashboard.addons.k8s.io
+    manifest: https://raw.githubusercontent.com/kubernetes/kops/master/addons/dashboard/v1.1.0.yaml
+  - version: 1.4.0
+    selector:
+      k8s-addon: kubernetes-dashboard.addons.k8s.io
+    manifest: https://raw.githubusercontent.com/kubernetes/kops/master/addons/dashboard/v1.4.0.yaml
+

addons/dashboard/v1.1.0.yaml

@@ -61,6 +61,7 @@ metadata:
   labels:
     app: kubernetes-dashboard
     k8s-addon: kubernetes-dashboard.addons.k8s.io
+    # kubernetes.io/cluster-service: "true"
   name: kubernetes-dashboard
   namespace: kube-system
 spec:

addons/dashboard/v1.4.0.yaml

@@ -0,0 +1,62 @@
+kind: Deployment
+apiVersion: extensions/v1beta1
+metadata:
+  name: kubernetes-dashboard-v1.4.0
+  namespace: kube-system
+  labels:
+    k8s-addon: kubernetes-dashboard.addons.k8s.io
+    k8s-app: kubernetes-dashboard
+    version: v1.4.0
+    kubernetes.io/cluster-service: "true"
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      k8s-app: kubernetes-dashboard
+  template:
+    metadata:
+      labels:
+        k8s-addon: kubernetes-dashboard.addons.k8s.io
+        k8s-app: kubernetes-dashboard
+        version: v1.4.0
+        # kubernetes.io/cluster-service: "true"
+      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ''
+        scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]'
+    spec:
+      containers:
+      - name: kubernetes-dashboard
+        image: gcr.io/google_containers/kubernetes-dashboard-amd64:v1.4.0
+        resources:
+          # keep request = limit to keep this container in guaranteed class
+          limits:
+            cpu: 100m
+            memory: 50Mi
+          requests:
+            cpu: 100m
+            memory: 50Mi
+        ports:
+        - containerPort: 9090
+        livenessProbe:
+          httpGet:
+            path: /
+            port: 9090
+          initialDelaySeconds: 30
+          timeoutSeconds: 30
+---
+
+apiVersion: v1
+kind: Service
+metadata:
+  name: kubernetes-dashboard
+  namespace: kube-system
+  labels:
+    k8s-addon: kubernetes-dashboard.addons.k8s.io
+    k8s-app: kubernetes-dashboard
+    # kubernetes.io/cluster-service: "true"
+spec:
+  selector:
+    k8s-app: kubernetes-dashboard
+  ports:
+  - port: 80
+    targetPort: 9090

channels/cmd/channels/apply.go

@@ -0,0 +1,15 @@
+package main
+
+import (
+	"github.com/spf13/cobra"
+)
+
+// applyCmd represents the apply command
+var applyCmd = &cobra.Command{
+	Use:   "apply",
+	Short: "apply resources from a channel",
+}
+
+func init() {
+	rootCommand.AddCommand(applyCmd)
+}

channels/cmd/channels/apply_channel.go

@@ -0,0 +1,124 @@
+package main
+
+import (
+	"fmt"
+	"github.com/spf13/cobra"
+	"k8s.io/kops/channels/pkg/channels"
+	"k8s.io/kops/util/pkg/tables"
+	"os"
+)
+
+type ApplyChannelCmd struct {
+	Yes bool
+}
+
+var applyChannel ApplyChannelCmd
+
+func init() {
+	cmd := &cobra.Command{
+		Use:   "channel",
+		Short: "Apply channel",
+		Run: func(cmd *cobra.Command, args []string) {
+			err := applyChannel.Run(args)
+			if err != nil {
+				exitWithError(err)
+			}
+		},
+	}
+
+	cmd.Flags().BoolVar(&applyChannel.Yes, "yes", false, "Apply update")
+
+	applyCmd.AddCommand(cmd)
+}
+
+func (c *ApplyChannelCmd) Run(args []string) error {
+	k8sClient, err := rootCommand.KubernetesClient()
+	if err != nil {
+		return err
+	}
+
+	var addons []*channels.Addon
+	for _, arg := range args {
+		o, err := channels.LoadAddons(arg)
+		if err != nil {
+			return fmt.Errorf("error loading file %q: %v", arg, err)
+		}
+
+		current, err := o.GetCurrent()
+		if err != nil {
+			return fmt.Errorf("error processing latest versions in %q: %v", arg, err)
+		}
+		addons = append(addons, current...)
+	}
+
+	var updates []*channels.AddonUpdate
+	var needUpdates []*channels.Addon
+	for _, addon := range addons {
+		// TODO: Cache lookups to prevent repeated lookups?
+		update, err := addon.GetRequiredUpdates(k8sClient)
+		if err != nil {
+			return fmt.Errorf("error checking for required update: %v", err)
+		}
+		if update != nil {
+			updates = append(updates, update)
+			needUpdates = append(needUpdates, addon)
+		}
+	}
+
+	if len(updates) == 0 {
+		fmt.Printf("No update required\n")
+		return nil
+	}
+
+	{
+		t := &tables.Table{}
+		t.AddColumn("NAME", func(r *channels.AddonUpdate) string {
+			return r.Name
+		})
+		t.AddColumn("CURRENT", func(r *channels.AddonUpdate) string {
+			if r.ExistingVersion == nil {
+				return "-"
+			}
+			if r.ExistingVersion.Version != nil {
+				return *r.ExistingVersion.Version
+			}
+			return "?"
+		})
+		t.AddColumn("UPDATE", func(r *channels.AddonUpdate) string {
+			if r.NewVersion == nil {
+				return "-"
+			}
+			if r.NewVersion.Version != nil {
+				return *r.NewVersion.Version
+			}
+			return "?"
+		})
+
+		columns := []string{"NAME", "CURRENT", "UPDATE"}
+		err := t.Render(updates, os.Stdout, columns...)
+		if err != nil {
+			return err
+		}
+	}
+
+	if !c.Yes {
+		fmt.Printf("\nMust specify --yes to update\n")
+		return nil
+	}
+
+	for _, needUpdate := range needUpdates {
+		update, err := needUpdate.EnsureUpdated(k8sClient)
+		if err != nil {
+			return fmt.Errorf("error updating %q: %v", needUpdate.Name, err)
+		}
+		if update.NewVersion.Version != nil {
+			fmt.Printf("Updated %q to %d\n", update.Name, *update.NewVersion)
+		} else {
+			fmt.Printf("Updated %q\n", update.Name)
+		}
+	}
+
+	fmt.Printf("\n")
+
+	return nil
+}

channels/cmd/channels/get.go

@@ -0,0 +1,33 @@
+package main
+
+import (
+	"github.com/spf13/cobra"
+)
+
+// GetCmd represents the get command
+type GetCmd struct {
+	output string
+
+	cobraCommand *cobra.Command
+}
+
+var getCmd = GetCmd{
+	cobraCommand: &cobra.Command{
+		Use:        "get",
+		SuggestFor: []string{"list"},
+		Short:      "list or get objects",
+	},
+}
+
+const (
+	OutputYaml  = "yaml"
+	OutputTable = "table"
+)
+
+func init() {
+	cmd := getCmd.cobraCommand
+
+	rootCommand.AddCommand(cmd)
+
+	cmd.PersistentFlags().StringVarP(&getCmd.output, "output", "o", OutputTable, "output format.  One of: table, yaml")
+}

channels/cmd/channels/get_addons.go

@@ -0,0 +1,100 @@
+package main
+
+import (
+	"fmt"
+	"github.com/spf13/cobra"
+	"k8s.io/kops/channels/pkg/channels"
+	"k8s.io/kops/util/pkg/tables"
+	k8sapi "k8s.io/kubernetes/pkg/api"
+	"k8s.io/kubernetes/pkg/api/v1"
+	"os"
+)
+
+type GetAddonsCmd struct {
+}
+
+var getAddonsCmd GetAddonsCmd
+
+func init() {
+	cmd := &cobra.Command{
+		Use:     "addons",
+		Aliases: []string{"addon"},
+		Short:   "get addons",
+		Long:    `List or get addons.`,
+		Run: func(cmd *cobra.Command, args []string) {
+			err := getAddonsCmd.Run(args)
+			if err != nil {
+				exitWithError(err)
+			}
+		},
+	}
+
+	getCmd.cobraCommand.AddCommand(cmd)
+}
+
+type addonInfo struct {
+	Name      string
+	Version   *channels.ChannelVersion
+	Namespace *v1.Namespace
+}
+
+func (c *GetAddonsCmd) Run(args []string) error {
+	k8sClient, err := rootCommand.KubernetesClient()
+	if err != nil {
+		return err
+	}
+
+	namespaces, err := k8sClient.Namespaces().List(k8sapi.ListOptions{})
+	if err != nil {
+		return fmt.Errorf("error listing namespaces: %v", err)
+	}
+
+	var info []*addonInfo
+
+	for i := range namespaces.Items {
+		ns := &namespaces.Items[i]
+		addons := channels.FindAddons(ns)
+		for name, version := range addons {
+			i := &addonInfo{
+				Name:      name,
+				Version:   version,
+				Namespace: ns,
+			}
+			info = append(info, i)
+		}
+	}
+
+	if len(info) == 0 {
+		fmt.Printf("\nNo managed addons found\n")
+		return nil
+	}
+
+	{
+		t := &tables.Table{}
+		t.AddColumn("NAME", func(r *addonInfo) string {
+			return r.Name
+		})
+		t.AddColumn("NAMESPACE", func(r *addonInfo) string {
+			return r.Namespace.Name
+		})
+		t.AddColumn("VERSION", func(r *addonInfo) string {
+			if r.Version == nil {
+				return "-"
+			}
+			if r.Version.Version != nil {
+				return *r.Version.Version
+			}
+			return "?"
+		})
+
+		columns := []string{"NAMESPACE", "NAME", "VERSION"}
+		err := t.Render(info, os.Stdout, columns...)
+		if err != nil {
+			return err
+		}
+	}
+
+	fmt.Printf("\n")
+
+	return nil
+}

channels/cmd/channels/main.go

@@ -0,0 +1,17 @@
+package main
+
+import (
+	"fmt"
+	"os"
+)
+
+func main() {
+	Execute()
+}
+
+// exitWithError will terminate execution with an error result
+// It prints the error to stderr and exits with a non-zero exit code
+func exitWithError(err error) {
+	fmt.Fprintf(os.Stderr, "\n%v\n", err)
+	os.Exit(1)
+}

channels/cmd/channels/root.go

@@ -0,0 +1,78 @@
+package main
+
+import (
+	goflag "flag"
+	"fmt"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+	"k8s.io/kubernetes/pkg/client/clientset_generated/release_1_3"
+	"k8s.io/kubernetes/pkg/client/unversioned/clientcmd"
+)
+
+type RootCmd struct {
+	configFile string
+
+	cobraCommand *cobra.Command
+}
+
+var rootCommand = RootCmd{
+	cobraCommand: &cobra.Command{
+		Use:   "channels",
+		Short: "channels applies software from a channel",
+	},
+}
+
+func Execute() {
+	goflag.Set("logtostderr", "true")
+	goflag.CommandLine.Parse([]string{})
+	if err := rootCommand.cobraCommand.Execute(); err != nil {
+		exitWithError(err)
+	}
+}
+
+func init() {
+	cobra.OnInitialize(initConfig)
+
+	cmd := rootCommand.cobraCommand
+
+	cmd.PersistentFlags().AddGoFlagSet(goflag.CommandLine)
+
+	cmd.PersistentFlags().StringVar(&rootCommand.configFile, "config", "", "config file (default is $HOME/.channels.yaml)")
+}
+
+// initConfig reads in config file and ENV variables if set.
+func initConfig() {
+	if rootCommand.configFile != "" {
+		// enable ability to specify config file via flag
+		viper.SetConfigFile(rootCommand.configFile)
+	}
+
+	viper.SetConfigName(".channels") // name of config file (without extension)
+	viper.AddConfigPath("$HOME")     // adding home directory as first search path
+	viper.AutomaticEnv()             // read in environment variables that match
+
+	// If a config file is found, read it in.
+	if err := viper.ReadInConfig(); err == nil {
+		fmt.Println("Using config file:", viper.ConfigFileUsed())
+	}
+}
+
+func (c *RootCmd) AddCommand(cmd *cobra.Command) {
+	c.cobraCommand.AddCommand(cmd)
+}
+
+func (c *RootCmd) KubernetesClient() (*release_1_3.Clientset, error) {
+	config, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
+		clientcmd.NewDefaultClientConfigLoadingRules(),
+		&clientcmd.ConfigOverrides{}).ClientConfig()
+	if err != nil {
+		return nil, fmt.Errorf("cannot load kubecfg settings: %v", err)
+	}
+
+	k8sClient, err := release_1_3.NewForConfig(config)
+	if err != nil {
+		return nil, fmt.Errorf("cannot build kube client: %v", err)
+	}
+	return k8sClient, err
+}

channels/pkg/api/channel.go

@@ -0,0 +1,32 @@
+package api
+
+import (
+	k8sapi "k8s.io/kubernetes/pkg/api"
+	"k8s.io/kubernetes/pkg/api/unversioned"
+)
+
+type Addons struct {
+	unversioned.TypeMeta `json:",inline"`
+	k8sapi.ObjectMeta    `json:"metadata,omitempty"`
+
+	Spec AddonsSpec `json:"spec,omitempty"`
+}
+
+type AddonsSpec struct {
+	Addons []*AddonSpec `json:"addons,omitempty"`
+}
+
+type AddonSpec struct {
+	Name *string `json:"name,omitempty"`
+
+	Namespace *string `json:"namespace,omitempty"`
+
+	// Selector is a label query over pods that should match the Replicas count.
+	Selector map[string]string `json:"selector"`
+
+	// Version is a semver version
+	Version *string `json:"version,omitempty"`
+
+	// Manifest is a strings containing the URL to the manifest that should be applied
+	Manifest *string `json:"manifest,omitempty"`
+}

channels/pkg/channels/addon.go

@@ -0,0 +1,91 @@
+package channels
+
+import (
+	"fmt"
+	"github.com/golang/glog"
+	"k8s.io/kops/channels/pkg/api"
+	"k8s.io/kubernetes/pkg/client/clientset_generated/release_1_3"
+	"k8s.io/kubernetes/pkg/util/validation/field"
+)
+
+type Addon struct {
+	Name    string
+	Channel string
+	Spec    *api.AddonSpec
+}
+
+type AddonUpdate struct {
+	Name            string
+	ExistingVersion *ChannelVersion
+	NewVersion      *ChannelVersion
+}
+
+func (a *Addon) ChannelVersion() *ChannelVersion {
+	return &ChannelVersion{
+		Channel: &a.Channel,
+		Version: a.Spec.Version,
+	}
+}
+
+func (a *Addon) buildChannel() *Channel {
+	namespace := "kube-system"
+	if a.Spec.Namespace != nil {
+		namespace = *a.Spec.Namespace
+	}
+
+	channel := &Channel{
+		Namespace: namespace,
+		Name:      a.Name,
+	}
+	return channel
+}
+func (a *Addon) GetRequiredUpdates(k8sClient *release_1_3.Clientset) (*AddonUpdate, error) {
+	newVersion := a.ChannelVersion()
+
+	channel := a.buildChannel()
+
+	existingVersion, err := channel.GetInstalledVersion(k8sClient)
+	if err != nil {
+		return nil, err
+	}
+
+	if existingVersion != nil && !newVersion.Replaces(existingVersion) {
+		return nil, nil
+	}
+
+	return &AddonUpdate{
+		Name:            a.Name,
+		ExistingVersion: existingVersion,
+		NewVersion:      newVersion,
+	}, nil
+}
+
+func (a *Addon) EnsureUpdated(k8sClient *release_1_3.Clientset) (*AddonUpdate, error) {
+	required, err := a.GetRequiredUpdates(k8sClient)
+	if err != nil {
+		return nil, err
+	}
+	if required == nil {
+		return nil, nil
+	}
+
+	if a.Spec.Manifest == nil || *a.Spec.Manifest == "" {
+		return nil, field.Required(field.NewPath("Spec", "Manifest"), "")
+	}
+
+	manifest := *a.Spec.Manifest
+	glog.Infof("Applying update from %q", manifest)
+
+	err = Apply(manifest)
+	if err != nil {
+		return nil, fmt.Errorf("error applying update from %q: %v", manifest, err)
+	}
+
+	channel := a.buildChannel()
+	err = channel.SetInstalledVersion(k8sClient, a.ChannelVersion())
+	if err != nil {
+		return nil, fmt.Errorf("error applying annotation to to record addon installation: %v", err)
+	}
+
+	return required, nil
+}

channels/pkg/channels/addons.go

@@ -0,0 +1,57 @@
+package channels
+
+import (
+	"fmt"
+	"k8s.io/kops/channels/pkg/api"
+	"k8s.io/kops/upup/pkg/fi/utils"
+	"k8s.io/kops/util/pkg/vfs"
+	"strings"
+)
+
+type Addons struct {
+	Channel   string
+	APIObject *api.Addons
+}
+
+func LoadAddons(location string) (*Addons, error) {
+	data, err := vfs.Context.ReadFile(location)
+	if err != nil {
+		return nil, fmt.Errorf("error reading addons from %q: %v", location, err)
+	}
+
+	// Yaml can't parse empty strings
+	configString := string(data)
+	configString = strings.TrimSpace(configString)
+
+	apiObject := &api.Addons{}
+	if configString != "" {
+		err := utils.YamlUnmarshal([]byte(configString), apiObject)
+		if err != nil {
+			return nil, fmt.Errorf("error parsing addons: %v", err)
+		}
+	}
+
+	return &Addons{Channel: location, APIObject: apiObject}, nil
+}
+
+func (a *Addons) GetCurrent() ([]*Addon, error) {
+	specs := make(map[string]*Addon)
+	for _, s := range a.APIObject.Spec.Addons {
+		name := a.APIObject.Name
+		if s.Name != nil {
+			name = *s.Name
+		}
+
+		addon := &Addon{Channel: a.Channel, Spec: s, Name: name}
+		existing := specs[name]
+		if existing == nil || addon.ChannelVersion().Replaces(existing.ChannelVersion()) {
+			specs[name] = addon
+		}
+	}
+
+	var addons []*Addon
+	for _, addon := range specs {
+		addons = append(addons, addon)
+	}
+	return addons, nil
+}

channels/pkg/channels/apply.go

@@ -0,0 +1,34 @@
+package channels
+
+import (
+	"fmt"
+	"github.com/golang/glog"
+	"os"
+	"os/exec"
+	"strings"
+)
+
+// Apply calls kubectl apply to apply the manifest.
+// We will likely in future change this to create things directly (or more likely embed this logic into kubectl itself)
+func Apply(manifest string) error {
+	_, err := execKubectl("apply", "-f", manifest)
+	return err
+}
+
+func execKubectl(args ...string) (string, error) {
+	kubectlPath := "kubectl" // Assume in PATH
+	cmd := exec.Command(kubectlPath, args...)
+	env := os.Environ()
+	cmd.Env = env
+
+	human := strings.Join(cmd.Args, " ")
+	glog.V(2).Infof("Running command: %s", human)
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		glog.Infof("error running %s:", human)
+		glog.Info(string(output))
+		return string(output), fmt.Errorf("error running kubectl")
+	}
+
+	return string(output), err
+}

channels/pkg/channels/channel_version.go

@@ -0,0 +1,137 @@
+package channels
+
+import (
+	"encoding/json"
+	"fmt"
+	"github.com/golang/glog"
+	"github.com/kopeio/route-controller/_vendor/github.com/blang/semver"
+	"k8s.io/kubernetes/pkg/api"
+	"k8s.io/kubernetes/pkg/api/v1"
+	"k8s.io/kubernetes/pkg/client/clientset_generated/release_1_3"
+	"strings"
+)
+
+const AnnotationPrefix = "addons.k8s.io/"
+
+type Channel struct {
+	Namespace string
+	Name      string
+}
+
+type ChannelVersion struct {
+	Version *string `json:"version,omitempty"`
+	Channel *string `json:"channel,omitempty"`
+}
+
+func ParseChannelVersion(s string) (*ChannelVersion, error) {
+	v := &ChannelVersion{}
+	err := json.Unmarshal([]byte(s), v)
+	if err != nil {
+		return nil, fmt.Errorf("error parsing version spec %q", s)
+	}
+	return v, nil
+}
+
+func FindAddons(ns *v1.Namespace) map[string]*ChannelVersion {
+	addons := make(map[string]*ChannelVersion)
+	for k, v := range ns.Annotations {
+		if !strings.HasPrefix(k, AnnotationPrefix) {
+			continue
+		}
+
+		channelVersion, err := ParseChannelVersion(v)
+		if err != nil {
+			glog.Warningf("failed to parse annotation %q=%q", k, v)
+			continue
+		}
+
+		name := strings.TrimPrefix(k, AnnotationPrefix)
+		addons[name] = channelVersion
+	}
+	return addons
+}
+
+func (c *ChannelVersion) Encode() (string, error) {
+	data, err := json.Marshal(c)
+	if err != nil {
+		return "", fmt.Errorf("error encoding version spec: %v", err)
+	}
+	return string(data), nil
+}
+
+func (c *Channel) AnnotationName() string {
+	return AnnotationPrefix + c.Name
+}
+
+func (c *ChannelVersion) Replaces(existing *ChannelVersion) bool {
+	if existing.Version != nil {
+		if c.Version == nil {
+			return false
+		}
+		cVersion, err := semver.Parse(*c.Version)
+		if err != nil {
+			glog.Warningf("error parsing version %q; will ignore this version", *c.Version)
+			return false
+		}
+		existingVersion, err := semver.Parse(*existing.Version)
+		if err != nil {
+			glog.Warningf("error parsing existing version %q", *existing.Version)
+			return true
+		}
+		return cVersion.GT(existingVersion)
+	}
+
+	glog.Warningf("ChannelVersion did not have a version; can't perform real version check")
+	if c.Version == nil {
+		return false
+	}
+	return true
+}
+
+func (c *Channel) GetInstalledVersion(k8sClient *release_1_3.Clientset) (*ChannelVersion, error) {
+	ns, err := k8sClient.Namespaces().Get(c.Namespace)
+	if err != nil {
+		return nil, fmt.Errorf("error querying namespace %q: %v", c.Namespace, err)
+	}
+
+	annotationValue, ok := ns.Annotations[c.AnnotationName()]
+	if !ok {
+		return nil, nil
+	}
+
+	return ParseChannelVersion(annotationValue)
+}
+
+type annotationPatch struct {
+	Metadata annotationPatchMetadata `json:"metadata,omitempty"`
+}
+type annotationPatchMetadata struct {
+	Annotations map[string]string `json:"annotations,omitempty"`
+}
+
+func (c *Channel) SetInstalledVersion(k8sClient *release_1_3.Clientset, version *ChannelVersion) error {
+	// Primarily to check it exists
+	_, err := k8sClient.Namespaces().Get(c.Namespace)
+	if err != nil {
+		return fmt.Errorf("error querying namespace %q: %v", c.Namespace, err)
+	}
+
+	value, err := version.Encode()
+	if err != nil {
+		return err
+	}
+
+	annotationPatch := &annotationPatch{Metadata: annotationPatchMetadata{Annotations: map[string]string{c.AnnotationName(): value}}}
+	annotationPatchJson, err := json.Marshal(annotationPatch)
+	if err != nil {
+		return fmt.Errorf("error building annotation patch: %v", err)
+	}
+
+	glog.V(2).Infof("sending patch: %q", string(annotationPatchJson))
+
+	_, err = k8sClient.Namespaces().Patch(c.Namespace, api.StrategicMergePatchType, annotationPatchJson)
+	if err != nil {
+		return fmt.Errorf("error applying annotation to namespace: %v", err)
+	}
+	return nil
+}

channels/stable.yaml

@@ -0,0 +1,11 @@
+spec:
+  images:
+    - name: 721322707521/k8s-1.4-debian-jessie-amd64-hvm-ebs-2016-09-23
+      labels:
+        k8s.io/cloudprovider: aws
+  cluster:
+    docker:
+      storage: overlay
+    kubernetesVersion: v1.4.0-beta.10
+    networking:
+      kubenet: {}

cmd/kops/delete_cluster.go

@@ -9,6 +9,7 @@ import (
 	"k8s.io/kops/upup/pkg/fi/cloudup"
 	"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
 	"k8s.io/kops/upup/pkg/kutil"
+	"k8s.io/kops/util/pkg/tables"
 	"os"
 )
 
@@ -116,7 +117,7 @@ func (c *DeleteClusterCmd) Run(args []string) error {
 		} else {
 			wouldDeleteCloudResources = true
 
-			t := &Table{}
+			t := &tables.Table{}
 			t.AddColumn("TYPE", func(r *kutil.ResourceTracker) string {
 				return r.Type
 			})

cmd/kops/get_cluster.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"github.com/spf13/cobra"
 	"k8s.io/kops/upup/pkg/api"
+	"k8s.io/kops/util/pkg/tables"
 	"strings"
 )
 
@@ -70,7 +71,7 @@ func (c *GetClustersCmd) Run(args []string) error {
 
 	output := getCmd.output
 	if output == OutputTable {
-		t := &Table{}
+		t := &tables.Table{}
 		t.AddColumn("NAME", func(c *api.Cluster) string {
 			return c.Name
 		})

cmd/kops/get_instancegroups.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/spf13/cobra"
 	"k8s.io/kops/upup/pkg/api"
+	"k8s.io/kops/util/pkg/tables"
 )
 
 type GetInstanceGroupsCmd struct {
@@ -66,7 +67,7 @@ func (c *GetInstanceGroupsCmd) Run(args []string) error {
 
 	output := getCmd.output
 	if output == OutputTable {
-		t := &Table{}
+		t := &tables.Table{}
 		t.AddColumn("NAME", func(c *api.InstanceGroup) string {
 			return c.Name
 		})

cmd/kops/get_secrets.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/spf13/cobra"
 	"k8s.io/kops/upup/pkg/fi"
+	"k8s.io/kops/util/pkg/tables"
 	"strings"
 )
 
@@ -126,7 +127,7 @@ func (c *GetSecretsCommand) Run(args []string) error {
 
 	output := getCmd.output
 	if output == OutputTable {
-		t := &Table{}
+		t := &tables.Table{}
 		t.AddColumn("NAME", func(i *fi.KeystoreItem) string {
 			return i.Name
 		})

cmd/kops/rollingupdate_cluster.go

@@ -8,6 +8,7 @@ import (
 	"github.com/spf13/cobra"
 	"k8s.io/kops/upup/pkg/fi/cloudup"
 	"k8s.io/kops/upup/pkg/kutil"
+	"k8s.io/kops/util/pkg/tables"
 	"k8s.io/kubernetes/pkg/api"
 	"k8s.io/kubernetes/pkg/api/v1"
 	"k8s.io/kubernetes/pkg/client/clientset_generated/release_1_3"
@@ -110,7 +111,7 @@ func (c *RollingUpdateClusterCmd) Run(args []string) error {
 	}
 
 	{
-		t := &Table{}
+		t := &tables.Table{}
 		t.AddColumn("NAME", func(r *kutil.CloudInstanceGroup) string {
 			return r.InstanceGroup.Name
 		})

cmd/kops/root.go

@@ -10,8 +10,8 @@ import (
 	"github.com/spf13/viper"
 	"k8s.io/kops/upup/pkg/api"
 	"k8s.io/kops/upup/pkg/fi"
-	"k8s.io/kops/upup/pkg/fi/vfs"
 	"k8s.io/kops/upup/pkg/kutil"
+	"k8s.io/kops/util/pkg/vfs"
 )
 
 type RootCmd struct {

cmd/kops/upgrade_cluster.go

@@ -6,6 +6,7 @@ import (
 	"github.com/spf13/cobra"
 	"k8s.io/kops/upup/pkg/api"
 	"k8s.io/kops/upup/pkg/fi/cloudup"
+	"k8s.io/kops/util/pkg/tables"
 	"os"
 )
 
@@ -90,7 +91,7 @@ func (c *UpgradeClusterCmd) Run(args []string) error {
 	}
 
 	{
-		t := &Table{}
+		t := &tables.Table{}
 		t.AddColumn("ITEM", func(a *upgradeAction) string {
 			return a.Item
 		})

upup/models/vfs.go

@@ -1,11 +1,11 @@
 package models
 
 import (
-	"k8s.io/kops/upup/pkg/fi/vfs"
 	"path"
 	"errors"
 	"os"
 	"strings"
+	"k8s.io/kops/util/pkg/vfs"
 )
 
 var ReadOnlyError = errors.New("AssetPath is read-only")

upup/pkg/api/cluster.go

@@ -4,7 +4,7 @@ import (
 	"encoding/binary"
 	"fmt"
 	"github.com/golang/glog"
-	"k8s.io/kops/upup/pkg/fi/vfs"
+	"k8s.io/kops/util/pkg/vfs"
 	k8sapi "k8s.io/kubernetes/pkg/api"
 	"k8s.io/kubernetes/pkg/api/unversioned"
 	"net"

upup/pkg/api/registry.go

@@ -5,7 +5,7 @@ import (
 	"github.com/golang/glog"
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/upup/pkg/fi/utils"
-	"k8s.io/kops/upup/pkg/fi/vfs"
+	"k8s.io/kops/util/pkg/vfs"
 	"k8s.io/kubernetes/pkg/api/unversioned"
 	"os"
 	"strings"

upup/pkg/fi/assetstore.go

@@ -4,8 +4,8 @@ import (
 	"fmt"
 	"github.com/golang/glog"
 	"io"
-	"k8s.io/kops/upup/pkg/fi/hashing"
 	"k8s.io/kops/upup/pkg/fi/utils"
+	"k8s.io/kops/util/pkg/hashing"
 	"net/http"
 	"os"
 	"os/exec"

upup/pkg/fi/ca.go

@@ -13,7 +13,7 @@ import (
 	"fmt"
 	"github.com/golang/glog"
 	"io"
-	"k8s.io/kops/upup/pkg/fi/vfs"
+	"k8s.io/kops/util/pkg/vfs"
 	"math/big"
 	"time"
 )

upup/pkg/fi/cloudup/apply_cluster.go

@@ -11,9 +11,9 @@ import (
 	"k8s.io/kops/upup/pkg/fi/cloudup/gcetasks"
 	"k8s.io/kops/upup/pkg/fi/cloudup/terraform"
 	"k8s.io/kops/upup/pkg/fi/fitasks"
-	"k8s.io/kops/upup/pkg/fi/hashing"
 	"k8s.io/kops/upup/pkg/fi/nodeup"
-	"k8s.io/kops/upup/pkg/fi/vfs"
+	"k8s.io/kops/util/pkg/hashing"
+	"k8s.io/kops/util/pkg/vfs"
 	"k8s.io/kubernetes/federation/pkg/dnsprovider"
 	"os"
 	"strings"

upup/pkg/fi/cloudup/iam_builder.go

@@ -5,7 +5,7 @@ import (
 	"fmt"
 	"github.com/golang/glog"
 	"k8s.io/kops/upup/pkg/api"
-	"k8s.io/kops/upup/pkg/fi/vfs"
+	"k8s.io/kops/util/pkg/vfs"
 	"k8s.io/kubernetes/pkg/util/sets"
 	"strings"
 )

upup/pkg/fi/cloudup/loader.go

@@ -10,7 +10,7 @@ import (
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/upup/pkg/fi/loader"
 	"k8s.io/kops/upup/pkg/fi/utils"
-	"k8s.io/kops/upup/pkg/fi/vfs"
+	"k8s.io/kops/util/pkg/vfs"
 	"os"
 	"reflect"
 	"strings"

upup/pkg/fi/cloudup/populate_cluster_spec.go

@@ -9,7 +9,7 @@ import (
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/upup/pkg/fi/loader"
 	"k8s.io/kops/upup/pkg/fi/utils"
-	"k8s.io/kops/upup/pkg/fi/vfs"
+	"k8s.io/kops/util/pkg/vfs"
 	"net"
 	"strings"
 	"text/template"

upup/pkg/fi/cloudup/spec_builder.go

@@ -6,7 +6,7 @@ import (
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/upup/pkg/fi/loader"
 	"k8s.io/kops/upup/pkg/fi/utils"
-	"k8s.io/kops/upup/pkg/fi/vfs"
+	"k8s.io/kops/util/pkg/vfs"
 )
 
 type SpecBuilder struct {

upup/pkg/fi/cloudup/template_functions.go

@@ -6,7 +6,7 @@ import (
 	"fmt"
 	"github.com/golang/glog"
 	"k8s.io/kops/upup/pkg/api"
-	"k8s.io/kops/upup/pkg/fi/vfs"
+	"k8s.io/kops/util/pkg/vfs"
 	"math/big"
 	"net"
 	"sort"

upup/pkg/fi/cloudup/validation_test.go

@@ -6,10 +6,10 @@ import (
 	"k8s.io/kops/upup/pkg/api"
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
-	"k8s.io/kops/upup/pkg/fi/vfs"
 	"k8s.io/kubernetes/pkg/util/sets"
 	"strings"
 	"testing"
+	"k8s.io/kops/util/pkg/vfs"
 )
 
 const MockAWSRegion = "us-mock-1"

upup/pkg/fi/files.go

@@ -4,7 +4,7 @@ import (
 	"fmt"
 	"github.com/golang/glog"
 	"io"
-	"k8s.io/kops/upup/pkg/fi/hashing"
+	"k8s.io/kops/util/pkg/hashing"
 	"os"
 	"path"
 	"strconv"

upup/pkg/fi/http.go

@@ -4,7 +4,7 @@ import (
 	"fmt"
 	"github.com/golang/glog"
 	"io"
-	"k8s.io/kops/upup/pkg/fi/hashing"
+	"k8s.io/kops/util/pkg/hashing"
 	"net/http"
 	"os"
 	"path"

upup/pkg/fi/loader/tree_walker.go

@@ -3,7 +3,7 @@ package loader
 import (
 	"fmt"
 	"github.com/golang/glog"
-	"k8s.io/kops/upup/pkg/fi/vfs"
+	"k8s.io/kops/util/pkg/vfs"
 	"os"
 	"path"
 	"strings"

upup/pkg/fi/nodeup/command.go

@@ -10,7 +10,7 @@ import (
 	"k8s.io/kops/upup/pkg/fi/nodeup/local"
 	"k8s.io/kops/upup/pkg/fi/nodeup/nodetasks"
 	"k8s.io/kops/upup/pkg/fi/utils"
-	"k8s.io/kops/upup/pkg/fi/vfs"
+	"k8s.io/kops/util/pkg/vfs"
 	"strconv"
 	"strings"
 )

upup/pkg/fi/nodeup/loader.go

@@ -9,7 +9,7 @@ import (
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/upup/pkg/fi/loader"
 	"k8s.io/kops/upup/pkg/fi/nodeup/nodetasks"
-	"k8s.io/kops/upup/pkg/fi/vfs"
+	"k8s.io/kops/util/pkg/vfs"
 	"strings"
 	"text/template"
 )

upup/pkg/fi/nodeup/nodetasks/load_image.go

@@ -4,10 +4,10 @@ import (
 	"fmt"
 	"github.com/golang/glog"
 	"k8s.io/kops/upup/pkg/fi"
-	"k8s.io/kops/upup/pkg/fi/hashing"
 	"k8s.io/kops/upup/pkg/fi/nodeup/cloudinit"
 	"k8s.io/kops/upup/pkg/fi/nodeup/local"
 	"k8s.io/kops/upup/pkg/fi/utils"
+	"k8s.io/kops/util/pkg/hashing"
 	"os/exec"
 	"path"
 	"strings"

upup/pkg/fi/nodeup/nodetasks/package.go

@@ -5,9 +5,9 @@ import (
 	"fmt"
 	"github.com/golang/glog"
 	"k8s.io/kops/upup/pkg/fi"
-	"k8s.io/kops/upup/pkg/fi/hashing"
 	"k8s.io/kops/upup/pkg/fi/nodeup/cloudinit"
 	"k8s.io/kops/upup/pkg/fi/nodeup/local"
+	"k8s.io/kops/util/pkg/hashing"
 	"os"
 	"os/exec"
 	"path"

upup/pkg/fi/nodeup/template_functions.go

@@ -6,7 +6,7 @@ import (
 	"github.com/golang/glog"
 	"k8s.io/kops/upup/pkg/api"
 	"k8s.io/kops/upup/pkg/fi"
-	"k8s.io/kops/upup/pkg/fi/vfs"
+	"k8s.io/kops/util/pkg/vfs"
 	"text/template"
 )
 

upup/pkg/fi/resources.go

@@ -5,7 +5,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"k8s.io/kops/upup/pkg/fi/vfs"
+	"k8s.io/kops/util/pkg/vfs"
 	"os"
 )
 

upup/pkg/fi/secrets.go

@@ -4,7 +4,7 @@ import (
 	crypto_rand "crypto/rand"
 	"encoding/base64"
 	"fmt"
-	"k8s.io/kops/upup/pkg/fi/vfs"
+	"k8s.io/kops/util/pkg/vfs"
 	"strings"
 )
 

upup/pkg/fi/statestore.go

@@ -3,7 +3,7 @@ package fi
 import (
 	"fmt"
 	"k8s.io/kops/upup/pkg/fi/utils"
-	"k8s.io/kops/upup/pkg/fi/vfs"
+	"k8s.io/kops/util/pkg/vfs"
 	"os"
 	"strings"
 )

upup/pkg/fi/vfs_castore.go

@@ -10,7 +10,7 @@ import (
 	"fmt"
 	"github.com/golang/glog"
 	"golang.org/x/crypto/ssh"
-	"k8s.io/kops/upup/pkg/fi/vfs"
+	"k8s.io/kops/util/pkg/vfs"
 	"math/big"
 	"os"
 	"strings"

upup/pkg/fi/vfs_secretstore.go

@@ -4,7 +4,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"github.com/golang/glog"
-	"k8s.io/kops/upup/pkg/fi/vfs"
+	"k8s.io/kops/util/pkg/vfs"
 	"os"
 )
 

upup/pkg/kutil/kubectl.go

@@ -65,7 +65,7 @@ func (k *Kubectl) execKubectl(args ...string) (string, error) {
 	glog.V(2).Infof("Running command: %s", human)
 	output, err := cmd.CombinedOutput()
 	if err != nil {
-		glog.Info("error running %s:", human)
+		glog.Infof("error running %s:", human)
 		glog.Info(string(output))
 		return string(output), fmt.Errorf("error running kubectl")
 	}

upup/pkg/kutil/ssh.go

@@ -4,7 +4,7 @@ import (
 	"fmt"
 	"golang.org/x/crypto/ssh"
 	"io/ioutil"
-	"k8s.io/kops/upup/pkg/fi/vfs"
+	"k8s.io/kops/util/pkg/vfs"
 )
 
 type NodeSSH struct {

util/pkg/tables/format.go

@@ -1,4 +1,4 @@
-package main
+package tables
 
 import (
 	"bytes"

util/pkg/vfs/fs.go

@@ -5,7 +5,7 @@ import (
 	"github.com/golang/glog"
 	"io"
 	"io/ioutil"
-	"k8s.io/kops/upup/pkg/fi/hashing"
+	"k8s.io/kops/util/pkg/hashing"
 	"os"
 	"path"
 	"sync"

util/pkg/vfs/s3fs.go

@@ -9,7 +9,7 @@ import (
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/golang/glog"
 	"io/ioutil"
-	"k8s.io/kops/upup/pkg/fi/hashing"
+	"k8s.io/kops/util/pkg/hashing"
 	"os"
 	"path"
 	"strings"

util/pkg/vfs/vfs.go

@@ -3,7 +3,7 @@ package vfs
 import (
 	"fmt"
 	"github.com/golang/glog"
-	"k8s.io/kops/upup/pkg/fi/hashing"
+	"k8s.io/kops/util/pkg/hashing"
 	"strings"
 )
 

util/pkg/vfs/vfssync.go

@@ -4,7 +4,7 @@ import (
 	"bytes"
 	"fmt"
 	"github.com/golang/glog"
-	"k8s.io/kops/upup/pkg/fi/hashing"
+	"k8s.io/kops/util/pkg/hashing"
 	"os"
 )