Add env vars, update tests

pkg/instancegroups/rollingupdate.go

@@ -97,7 +97,7 @@ func (c *RollingUpdateCluster) RollingUpdate(groups map[string]*cloudinstances.C
 
 				defer wg.Done()
 
-				g, err := NewRollingUpdateInstanceGroup(c.Cloud, group, c.K8sClient)
+				g, err := NewRollingUpdateInstanceGroup(c.Cloud, group)
 				if err == nil {
 					err = g.RollingUpdate(c, cluster, instanceGroups, true, c.BastionInterval, c.ValidationTimeout)
 				}
@@ -125,7 +125,7 @@ func (c *RollingUpdateCluster) RollingUpdate(groups map[string]*cloudinstances.C
 		// and we don't want to roll all the masters at the same time.  See issue #284
 
 		for _, group := range masterGroups {
-			g, err := NewRollingUpdateInstanceGroup(c.Cloud, group, c.K8sClient)
+			g, err := NewRollingUpdateInstanceGroup(c.Cloud, group)
 			if err == nil {
 				err = g.RollingUpdate(c, cluster, instanceGroups, false, c.MasterInterval, c.ValidationTimeout)
 			}
@@ -159,7 +159,7 @@ func (c *RollingUpdateCluster) RollingUpdate(groups map[string]*cloudinstances.C
 			defer wg.Done()
 
 			for k, group := range nodeGroups {
-				g, err := NewRollingUpdateInstanceGroup(c.Cloud, group, c.K8sClient)
+				g, err := NewRollingUpdateInstanceGroup(c.Cloud, group)
 				if err == nil {
 					err = g.RollingUpdate(c, cluster, instanceGroups, false, c.NodeInterval, c.ValidationTimeout)
 				}

pkg/testutils/modelharness.go

@@ -149,5 +149,5 @@ func AssertMatchesFile(t *testing.T, actual string, p string) {
 
 	t.Logf("to update golden output automatically, run hack/update-expected.sh")
 
-	t.Fatalf("output did not match expected for %q", p)
+	t.Errorf("output did not match expected for %q", p)
 }

tests/integration/update_cluster/ha_gce/kubernetes.tf

@@ -400,6 +400,7 @@ resource "google_compute_instance_template" "master-us-test1-a-ha-gce-example-co
 
   metadata = {
     cluster-name                    = "${file("${path.module}/data/google_compute_instance_template_master-us-test1-a-ha-gce-example-com_metadata_cluster-name")}"
+    kops-k8s-io-instance-group-name = "${file("${path.module}/data/google_compute_instance_template_master-us-test1-a-ha-gce-example-com_metadata_kops-k8s-io-instance-group-name")}"
     ssh-keys                        = "${file("${path.module}/data/google_compute_instance_template_master-us-test1-a-ha-gce-example-com_metadata_ssh-keys")}"
     startup-script                  = "${file("${path.module}/data/google_compute_instance_template_master-us-test1-a-ha-gce-example-com_metadata_startup-script")}"
   }
@@ -440,6 +441,7 @@ resource "google_compute_instance_template" "master-us-test1-b-ha-gce-example-co
 
   metadata = {
     cluster-name                    = "${file("${path.module}/data/google_compute_instance_template_master-us-test1-b-ha-gce-example-com_metadata_cluster-name")}"
+    kops-k8s-io-instance-group-name = "${file("${path.module}/data/google_compute_instance_template_master-us-test1-b-ha-gce-example-com_metadata_kops-k8s-io-instance-group-name")}"
     ssh-keys                        = "${file("${path.module}/data/google_compute_instance_template_master-us-test1-b-ha-gce-example-com_metadata_ssh-keys")}"
     startup-script                  = "${file("${path.module}/data/google_compute_instance_template_master-us-test1-b-ha-gce-example-com_metadata_startup-script")}"
   }
@@ -480,6 +482,7 @@ resource "google_compute_instance_template" "master-us-test1-c-ha-gce-example-co
 
   metadata = {
     cluster-name                    = "${file("${path.module}/data/google_compute_instance_template_master-us-test1-c-ha-gce-example-com_metadata_cluster-name")}"
+    kops-k8s-io-instance-group-name = "${file("${path.module}/data/google_compute_instance_template_master-us-test1-c-ha-gce-example-com_metadata_kops-k8s-io-instance-group-name")}"
     ssh-keys                        = "${file("${path.module}/data/google_compute_instance_template_master-us-test1-c-ha-gce-example-com_metadata_ssh-keys")}"
     startup-script                  = "${file("${path.module}/data/google_compute_instance_template_master-us-test1-c-ha-gce-example-com_metadata_startup-script")}"
   }
@@ -520,6 +523,7 @@ resource "google_compute_instance_template" "nodes-ha-gce-example-com" {
 
   metadata = {
     cluster-name                    = "${file("${path.module}/data/google_compute_instance_template_nodes-ha-gce-example-com_metadata_cluster-name")}"
+    kops-k8s-io-instance-group-name = "${file("${path.module}/data/google_compute_instance_template_nodes-ha-gce-example-com_metadata_kops-k8s-io-instance-group-name")}"
     ssh-keys                        = "${file("${path.module}/data/google_compute_instance_template_nodes-ha-gce-example-com_metadata_ssh-keys")}"
     startup-script                  = "${file("${path.module}/data/google_compute_instance_template_nodes-ha-gce-example-com_metadata_startup-script")}"
   }

upup/models/cloudup/resources/addons/kops-controller.addons.k8s.io/k8s-1.16.yaml.template

@@ -6,7 +6,7 @@ metadata:
   labels:
     k8s-addon: kops-controller.addons.k8s.io
     k8s-app: kops-controller
-    version: v1.14.0-alpha.1
+    version: v1.16.0-alpha.1
 spec:
   replicas: 1
   selector:
@@ -17,7 +17,7 @@ spec:
       labels:
         k8s-addon: kops-controller.addons.k8s.io
         k8s-app: kops-controller
-        version: v1.14.0-alpha.1
+        version: v1.16.0-alpha.1
       annotations:
         scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
@@ -36,11 +36,11 @@ spec:
 {{ range $arg := KopsControllerArgv }}
         - "{{ $arg }}"
 {{ end }}
-{{- if .EgressProxy }}
+{{- if KopsSystemEnv }}
         env:
-{{ range $name, $value := ProxyEnv }}
+{{ range $var := KopsSystemEnv }}
-        - name: {{ $name }}
+        - name: {{ $var.Name }}
-          value: {{ $value }}
+          value: {{ $var.Value }}
 {{ end }}
 {{- end }}
         resources:

upup/pkg/fi/cloudup/bootstrapchannelbuilder.go

@@ -115,7 +115,7 @@ func (b *BootstrapChannelBuilder) buildAddons() *channelsapi.Addons {
 
 	{
 		key := "kops-controller.addons.k8s.io"
-		version := "1.14.0-alpha.1"
+		version := "1.16.0-alpha.1"
 
 		{
 			location := key + "/k8s-1.16.yaml"
@@ -129,7 +129,6 @@ func (b *BootstrapChannelBuilder) buildAddons() *channelsapi.Addons {
 				KubernetesVersion: ">=1.16.0-alpha.0",
 				Id:                id,
 			})
-			manifests[key+"-"+id] = "addons/" + location
 		}
 	}
 

upup/pkg/fi/cloudup/bootstrapchannelbuilder_test.go

@@ -40,13 +40,13 @@ func TestBootstrapChannelBuilder_BuildTasks(t *testing.T) {
 
 	h.SetupMockAWS()
 
-	runChannelBuilderTest(t, "simple")
+	runChannelBuilderTest(t, "simple", []string{"dns-controller.addons.k8s.io-k8s-1.12", "kops-controller.addons.k8s.io-k8s-1.16"})
-	runChannelBuilderTest(t, "kopeio-vxlan")
+	// Use cilium networking, proxy
-	runChannelBuilderTest(t, "weave")
+	runChannelBuilderTest(t, "cilium", []string{"dns-controller.addons.k8s.io-k8s-1.12", "kops-controller.addons.k8s.io-k8s-1.16"})
-	runChannelBuilderTest(t, "cilium")
+	runChannelBuilderTest(t, "weave", []string{})
 }
 
-func runChannelBuilderTest(t *testing.T, key string) {
+func runChannelBuilderTest(t *testing.T, key string, addonManifests []string) {
 	basedir := path.Join("tests/bootstrapchannelbuilder/", key)
 
 	clusterYamlPath := path.Join(basedir, "cluster.yaml")
@@ -122,7 +122,7 @@ func runChannelBuilderTest(t *testing.T, key string) {
 		testutils.AssertMatchesFile(t, actualManifest, expectedManifestPath)
 	}
 
-	for _, k := range []string{"dns-controller.addons.k8s.io-k8s-1.12" /*, "kops-controller.addons.k8s.io-k8s-1.16"*/} {
+	for _, k := range []string{"dns-controller.addons.k8s.io-k8s-1.12", "kops-controller.addons.k8s.io-k8s-1.16"} {
 		name := cluster.ObjectMeta.Name + "-addons-" + k
 		manifestTask := context.Tasks[name]
 		if manifestTask == nil {

upup/pkg/fi/cloudup/template_functions.go

@@ -35,6 +35,7 @@ import (
 	"strings"
 	"text/template"
 
+	corev1 "k8s.io/api/core/v1"
 	"k8s.io/kops/pkg/apis/kops"
 	"k8s.io/kops/pkg/dns"
 	"k8s.io/kops/pkg/featureflag"
@@ -42,6 +43,7 @@ import (
 	"k8s.io/kops/pkg/resources/spotinst"
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/upup/pkg/fi/cloudup/gce"
+	"k8s.io/kops/util/pkg/env"
 
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/klog"
@@ -101,6 +103,8 @@ func (tf *TemplateFunctions) AddTo(dest template.FuncMap, secretStore fi.SecretS
 
 	dest["ProxyEnv"] = tf.ProxyEnv
 
+	dest["KopsSystemEnv"] = tf.KopsSystemEnv
+
 	dest["DO_TOKEN"] = func() string {
 		return os.Getenv("DIGITALOCEAN_ACCESS_TOKEN")
 	}
@@ -311,3 +315,10 @@ func (tf *TemplateFunctions) ProxyEnv() map[string]string {
 	}
 	return envs
 }
+
+// KopsSystemEnv builds the env vars for a system component
+func (tf *TemplateFunctions) KopsSystemEnv() []corev1.EnvVar {
+	envMap := env.BuildSystemComponentEnvVars(&tf.cluster.Spec)
+
+	return envMap.ToEnvVars()
+}

upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/kops-controller.addons.k8s.io-k8s-1.16.yaml

@@ -0,0 +1,90 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    k8s-addon: kops-controller.addons.k8s.io
+    k8s-app: kops-controller
+    version: v1.16.0-alpha.1
+  name: kops-controller
+  namespace: kube-system
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      k8s-app: kops-controller
+  template:
+    metadata:
+      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ""
+      labels:
+        k8s-addon: kops-controller.addons.k8s.io
+        k8s-app: kops-controller
+        version: v1.16.0-alpha.1
+    spec:
+      containers:
+      - command:
+        - /usr/bin/kops-controller
+        - --cloud=aws
+        - --config=memfs://clusters.example.com/minimal.example.com
+        - --metrics-addr=0
+        - --v=2
+        image: kope/kops-controller:1.14.0-alpha.1
+        name: kops-controller
+        resources:
+          requests:
+            cpu: 50m
+            memory: 50Mi
+      dnsPolicy: Default
+      hostNetwork: true
+      nodeSelector:
+        node-role.kubernetes.io/master: ""
+      serviceAccount: kops-controller
+      tolerations:
+      - effect: NoSchedule
+        key: node-role.kubernetes.io/master
+
+---
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    k8s-addon: kops-controller.addons.k8s.io
+  name: kops-controller
+  namespace: kube-system
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    k8s-addon: kops-controller.addons.k8s.io
+  name: kops-controller
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - get
+  - list
+  - watch
+  - patch
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    k8s-addon: kops-controller.addons.k8s.io
+  name: kops-controller
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kops-controller
+subjects:
+- apiGroup: rbac.authorization.k8s.io
+  kind: User
+  name: system:serviceaccount:kube-system:kops-controller

upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/manifest.yaml

@@ -4,6 +4,14 @@ metadata:
   name: bootstrap
 spec:
   addons:
+  - id: k8s-1.16
+    kubernetesVersion: '>=1.16.0-alpha.0'
+    manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml
+    manifestHash: de5108cf5c9a46a8dc13643428cbaddd0875b948
+    name: kops-controller.addons.k8s.io
+    selector:
+      k8s-addon: kops-controller.addons.k8s.io
+    version: 1.16.0-alpha.1
   - manifest: core.addons.k8s.io/v1.4.0.yaml
     manifestHash: 3ffe9ac576f9eec72e2bdfbd2ea17d56d9b17b90
     name: core.addons.k8s.io

upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/kopeio-vxlan/kops-controller.addons.k8s.io-k8s-1.16.yaml

@@ -0,0 +1,99 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    k8s-addon: kops-controller.addons.k8s.io
+    k8s-app: kops-controller
+    version: v1.16.0-alpha.1
+  name: kops-controller
+  namespace: kube-system
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      k8s-app: kops-controller
+  template:
+    metadata:
+      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ""
+      labels:
+        k8s-addon: kops-controller.addons.k8s.io
+        k8s-app: kops-controller
+        version: v1.16.0-alpha.1
+    spec:
+      containers:
+      - command:
+        - /usr/bin/kops-controller
+        - --cloud=aws
+        - --config=memfs://clusters.example.com/minimal.example.com
+        - --metrics-addr=0
+        - --v=2
+        env:
+        - name: NO_PROXY
+          value: noproxy.example.com,127.0.0.1,localhost,api.minimal.example.com,minimal.example.com,100.64.0.1,100.64.0.0/10,169.254.169.254,172.20.0.0/16
+        - name: http_proxy
+          value: http://proxy.example.com
+        - name: https_proxy
+          value: http://proxy.example.com
+        - name: no_proxy
+          value: noproxy.example.com,127.0.0.1,localhost,api.minimal.example.com,minimal.example.com,100.64.0.1,100.64.0.0/10,169.254.169.254,172.20.0.0/16
+        image: kope/kops-controller:1.14.0-alpha.1
+        name: kops-controller
+        resources:
+          requests:
+            cpu: 50m
+            memory: 50Mi
+      dnsPolicy: Default
+      hostNetwork: true
+      nodeSelector:
+        node-role.kubernetes.io/master: ""
+      serviceAccount: kops-controller
+      tolerations:
+      - effect: NoSchedule
+        key: node-role.kubernetes.io/master
+
+---
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    k8s-addon: kops-controller.addons.k8s.io
+  name: kops-controller
+  namespace: kube-system
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    k8s-addon: kops-controller.addons.k8s.io
+  name: kops-controller
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - get
+  - list
+  - watch
+  - patch
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    k8s-addon: kops-controller.addons.k8s.io
+  name: kops-controller
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kops-controller
+subjects:
+- apiGroup: rbac.authorization.k8s.io
+  kind: User
+  name: system:serviceaccount:kube-system:kops-controller

upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/kopeio-vxlan/manifest.yaml

@@ -4,6 +4,14 @@ metadata:
   name: bootstrap
 spec:
   addons:
+  - id: k8s-1.16
+    kubernetesVersion: '>=1.16.0-alpha.0'
+    manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml
+    manifestHash: 3b77645897d2641c5ab2de0729b9689b31d63c58
+    name: kops-controller.addons.k8s.io
+    selector:
+      k8s-addon: kops-controller.addons.k8s.io
+    version: 1.16.0-alpha.1
   - manifest: core.addons.k8s.io/v1.4.0.yaml
     manifestHash: 3ffe9ac576f9eec72e2bdfbd2ea17d56d9b17b90
     name: core.addons.k8s.io

upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/simple/kops-controller.addons.k8s.io-k8s-1.16.yaml

@@ -0,0 +1,90 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    k8s-addon: kops-controller.addons.k8s.io
+    k8s-app: kops-controller
+    version: v1.16.0-alpha.1
+  name: kops-controller
+  namespace: kube-system
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      k8s-app: kops-controller
+  template:
+    metadata:
+      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ""
+      labels:
+        k8s-addon: kops-controller.addons.k8s.io
+        k8s-app: kops-controller
+        version: v1.16.0-alpha.1
+    spec:
+      containers:
+      - command:
+        - /usr/bin/kops-controller
+        - --cloud=aws
+        - --config=memfs://clusters.example.com/minimal.example.com
+        - --metrics-addr=0
+        - --v=2
+        image: kope/kops-controller:1.14.0-alpha.1
+        name: kops-controller
+        resources:
+          requests:
+            cpu: 50m
+            memory: 50Mi
+      dnsPolicy: Default
+      hostNetwork: true
+      nodeSelector:
+        node-role.kubernetes.io/master: ""
+      serviceAccount: kops-controller
+      tolerations:
+      - effect: NoSchedule
+        key: node-role.kubernetes.io/master
+
+---
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    k8s-addon: kops-controller.addons.k8s.io
+  name: kops-controller
+  namespace: kube-system
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    k8s-addon: kops-controller.addons.k8s.io
+  name: kops-controller
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - get
+  - list
+  - watch
+  - patch
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    k8s-addon: kops-controller.addons.k8s.io
+  name: kops-controller
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kops-controller
+subjects:
+- apiGroup: rbac.authorization.k8s.io
+  kind: User
+  name: system:serviceaccount:kube-system:kops-controller

upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/simple/manifest.yaml

@@ -4,6 +4,14 @@ metadata:
   name: bootstrap
 spec:
   addons:
+  - id: k8s-1.16
+    kubernetesVersion: '>=1.16.0-alpha.0'
+    manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml
+    manifestHash: de5108cf5c9a46a8dc13643428cbaddd0875b948
+    name: kops-controller.addons.k8s.io
+    selector:
+      k8s-addon: kops-controller.addons.k8s.io
+    version: 1.16.0-alpha.1
   - manifest: core.addons.k8s.io/v1.4.0.yaml
     manifestHash: 3ffe9ac576f9eec72e2bdfbd2ea17d56d9b17b90
     name: core.addons.k8s.io

upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/weave/kops-controller.addons.k8s.io-k8s-1.16.yaml

@@ -0,0 +1,90 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    k8s-addon: kops-controller.addons.k8s.io
+    k8s-app: kops-controller
+    version: v1.16.0-alpha.1
+  name: kops-controller
+  namespace: kube-system
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      k8s-app: kops-controller
+  template:
+    metadata:
+      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ""
+      labels:
+        k8s-addon: kops-controller.addons.k8s.io
+        k8s-app: kops-controller
+        version: v1.16.0-alpha.1
+    spec:
+      containers:
+      - command:
+        - /usr/bin/kops-controller
+        - --cloud=aws
+        - --config=memfs://clusters.example.com/minimal.example.com
+        - --metrics-addr=0
+        - --v=2
+        image: kope/kops-controller:1.14.0-alpha.1
+        name: kops-controller
+        resources:
+          requests:
+            cpu: 50m
+            memory: 50Mi
+      dnsPolicy: Default
+      hostNetwork: true
+      nodeSelector:
+        node-role.kubernetes.io/master: ""
+      serviceAccount: kops-controller
+      tolerations:
+      - effect: NoSchedule
+        key: node-role.kubernetes.io/master
+
+---
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    k8s-addon: kops-controller.addons.k8s.io
+  name: kops-controller
+  namespace: kube-system
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    k8s-addon: kops-controller.addons.k8s.io
+  name: kops-controller
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - get
+  - list
+  - watch
+  - patch
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    k8s-addon: kops-controller.addons.k8s.io
+  name: kops-controller
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kops-controller
+subjects:
+- apiGroup: rbac.authorization.k8s.io
+  kind: User
+  name: system:serviceaccount:kube-system:kops-controller

upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/weave/manifest.yaml

@@ -4,6 +4,14 @@ metadata:
   name: bootstrap
 spec:
   addons:
+  - id: k8s-1.16
+    kubernetesVersion: '>=1.16.0-alpha.0'
+    manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml
+    manifestHash: de5108cf5c9a46a8dc13643428cbaddd0875b948
+    name: kops-controller.addons.k8s.io
+    selector:
+      k8s-addon: kops-controller.addons.k8s.io
+    version: 1.16.0-alpha.1
   - manifest: core.addons.k8s.io/v1.4.0.yaml
     manifestHash: 3ffe9ac576f9eec72e2bdfbd2ea17d56d9b17b90
     name: core.addons.k8s.io