Merge pull request #5547 from justinsb/etcd_manager_tests

Add test for etcd-manager output

nodeup/pkg/model/BUILD.bazel

@@ -79,11 +79,8 @@ go_test(
     deps = [
         "//nodeup/pkg/distros:go_default_library",
         "//pkg/apis/kops:go_default_library",
-        "//pkg/apis/kops/v1alpha2:go_default_library",
-        "//pkg/diff:go_default_library",
         "//pkg/flagbuilder:go_default_library",
-        "//pkg/kopscodecs:go_default_library",
+        "//pkg/testutils:go_default_library",
         "//upup/pkg/fi:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
     ],
 )

nodeup/pkg/model/docker_test.go

@@ -22,6 +22,7 @@ import (
 
 	"k8s.io/kops/pkg/apis/kops"
 	"k8s.io/kops/pkg/flagbuilder"
+	"k8s.io/kops/pkg/testutils"
 	"k8s.io/kops/upup/pkg/fi"
 )
 
@@ -97,7 +98,7 @@ func TestDockerBuilder_BuildFlags(t *testing.T) {
 func runDockerBuilderTest(t *testing.T, key string) {
 	basedir := path.Join("tests/dockerbuilder/", key)
 
-	nodeUpModelContext, err := LoadModel(basedir)
+	nodeUpModelContext, err := BuildNodeupModelContext(basedir)
 	if err != nil {
 		t.Fatalf("error parsing cluster yaml %q: %v", basedir, err)
 		return
@@ -115,5 +116,5 @@ func runDockerBuilderTest(t *testing.T, key string) {
 		return
 	}
 
-	ValidateTasks(t, basedir, context)
+	testutils.ValidateTasks(t, basedir, context)
 }

nodeup/pkg/model/kubelet_test.go

@@ -17,21 +17,12 @@ limitations under the License.
 package model
 
 import (
-	"bytes"
-	"io/ioutil"
-	"path"
-	"sort"
-	"strings"
+	"fmt"
 	"testing"
 
-	"fmt"
-
-	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/kops/nodeup/pkg/distros"
 	"k8s.io/kops/pkg/apis/kops"
-	"k8s.io/kops/pkg/apis/kops/v1alpha2"
-	"k8s.io/kops/pkg/diff"
-	"k8s.io/kops/pkg/kopscodecs"
+	"k8s.io/kops/pkg/testutils"
 	"k8s.io/kops/upup/pkg/fi"
 )
 
@@ -170,7 +161,7 @@ func Test_RunKubeletBuilder(t *testing.T) {
 	context := &fi.ModelBuilderContext{
 		Tasks: make(map[string]fi.Task),
 	}
-	nodeUpModelContext, err := LoadModel(basedir)
+	nodeUpModelContext, err := BuildNodeupModelContext(basedir)
 	if err != nil {
 		t.Fatalf("error loading model %q: %v", basedir, err)
 		return
@@ -191,90 +182,36 @@ func Test_RunKubeletBuilder(t *testing.T) {
 	}
 	context.AddTask(fileTask)
 
-	ValidateTasks(t, basedir, context)
+	testutils.ValidateTasks(t, basedir, context)
 }
 
-func LoadModel(basedir string) (*NodeupModelContext, error) {
-	clusterYamlPath := path.Join(basedir, "cluster.yaml")
-	clusterYaml, err := ioutil.ReadFile(clusterYamlPath)
+func BuildNodeupModelContext(basedir string) (*NodeupModelContext, error) {
+	model, err := testutils.LoadModel(basedir)
 	if err != nil {
-		return nil, fmt.Errorf("error reading cluster yaml file %q: %v", clusterYamlPath, err)
+		return nil, err
 	}
 
-	var cluster *kops.Cluster
-	var instanceGroup *kops.InstanceGroup
-
-	// Codecs provides access to encoding and decoding for the scheme
-	codecs := kopscodecs.Codecs
-
-	codec := codecs.UniversalDecoder(kops.SchemeGroupVersion)
-
-	sections := bytes.Split(clusterYaml, []byte("\n---\n"))
-	for _, section := range sections {
-		defaults := &schema.GroupVersionKind{
-			Group:   v1alpha2.SchemeGroupVersion.Group,
-			Version: v1alpha2.SchemeGroupVersion.Version,
-		}
-		o, gvk, err := codec.Decode(section, defaults, nil)
-		if err != nil {
-			return nil, fmt.Errorf("error parsing file %v", err)
-		}
-
-		switch v := o.(type) {
-		case *kops.Cluster:
-			cluster = v
-		case *kops.InstanceGroup:
-			instanceGroup = v
-		default:
-			return nil, fmt.Errorf("Unhandled kind %q", gvk)
-		}
+	if model.Cluster == nil {
+		return nil, fmt.Errorf("no cluster found in %s", basedir)
 	}
 
 	nodeUpModelContext := &NodeupModelContext{
-		Cluster:       cluster,
-		Architecture:  "amd64",
-		Distribution:  distros.DistributionXenial,
-		InstanceGroup: instanceGroup,
+		Cluster:      model.Cluster,
+		Architecture: "amd64",
+		Distribution: distros.DistributionXenial,
 	}
+
+	if len(model.InstanceGroups) == 0 {
+		// We tolerate this - not all tests need an instance group
+	} else if len(model.InstanceGroups) == 1 {
+		nodeUpModelContext.InstanceGroup = model.InstanceGroups[0]
+	} else {
+		return nil, fmt.Errorf("unexpected number of instance groups in %s, found %d", basedir, len(model.InstanceGroups))
+	}
+
 	if err := nodeUpModelContext.Init(); err != nil {
 		return nil, err
 	}
 
 	return nodeUpModelContext, nil
 }
-
-func ValidateTasks(t *testing.T, basedir string, context *fi.ModelBuilderContext) {
-	var keys []string
-	for key := range context.Tasks {
-		keys = append(keys, key)
-	}
-	sort.Strings(keys)
-
-	var yamls []string
-	for _, key := range keys {
-		task := context.Tasks[key]
-		yaml, err := kops.ToRawYaml(task)
-		if err != nil {
-			t.Fatalf("error serializing task: %v", err)
-		}
-		yamls = append(yamls, strings.TrimSpace(string(yaml)))
-	}
-
-	actualTasksYaml := strings.Join(yamls, "\n---\n")
-
-	tasksYamlPath := path.Join(basedir, "tasks.yaml")
-	expectedTasksYamlBytes, err := ioutil.ReadFile(tasksYamlPath)
-	if err != nil {
-		t.Fatalf("error reading file %q: %v", tasksYamlPath, err)
-	}
-
-	actualTasksYaml = strings.TrimSpace(actualTasksYaml)
-	expectedTasksYaml := strings.TrimSpace(string(expectedTasksYamlBytes))
-
-	if expectedTasksYaml != actualTasksYaml {
-		diffString := diff.FormatDiff(expectedTasksYaml, actualTasksYaml)
-		t.Logf("diff:\n%s\n", diffString)
-
-		t.Fatalf("tasks differed from expected for test %q", basedir)
-	}
-}

pkg/model/components/etcdmanager/BUILD.bazel

@@ -1,4 +1,4 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_library")
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
 
 go_library(
     name = "go_default_library",
@@ -28,3 +28,16 @@ go_library(
         "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
     ],
 )
+
+go_test(
+    name = "go_default_test",
+    srcs = ["model_test.go"],
+    data = glob(["tests/**"]),  #keep
+    embed = [":go_default_library"],
+    deps = [
+        "//pkg/assets:go_default_library",
+        "//pkg/model:go_default_library",
+        "//pkg/testutils:go_default_library",
+        "//upup/pkg/fi:go_default_library",
+    ],
+)

pkg/model/components/etcdmanager/model_test.go

@@ -0,0 +1,74 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package etcdmanager
+
+import (
+	"fmt"
+	"testing"
+
+	"k8s.io/kops/pkg/assets"
+	"k8s.io/kops/pkg/model"
+	"k8s.io/kops/pkg/testutils"
+	"k8s.io/kops/upup/pkg/fi"
+)
+
+func Test_RunEtcdManagerBuilder(t *testing.T) {
+	basedir := "tests/minimal"
+
+	context := &fi.ModelBuilderContext{
+		Tasks: make(map[string]fi.Task),
+	}
+	kopsModelContext, err := LoadKopsModelContext(basedir)
+	if err != nil {
+		t.Fatalf("error loading model %q: %v", basedir, err)
+		return
+	}
+
+	builder := EtcdManagerBuilder{
+		KopsModelContext: kopsModelContext,
+		AssetBuilder:     assets.NewAssetBuilder(kopsModelContext.Cluster, ""),
+	}
+
+	if err := builder.Build(context); err != nil {
+		t.Fatalf("error from Build: %v", err)
+		return
+	}
+
+	testutils.ValidateTasks(t, basedir, context)
+}
+
+func LoadKopsModelContext(basedir string) (*model.KopsModelContext, error) {
+	spec, err := testutils.LoadModel(basedir)
+	if err != nil {
+		return nil, err
+	}
+
+	if spec.Cluster == nil {
+		return nil, fmt.Errorf("no cluster found in %s", basedir)
+	}
+
+	if len(spec.InstanceGroups) == 0 {
+		return nil, fmt.Errorf("no instance groups found in %s", basedir)
+	}
+
+	kopsContext := &model.KopsModelContext{
+		Cluster:        spec.Cluster,
+		InstanceGroups: spec.InstanceGroups,
+	}
+
+	return kopsContext, nil
+}

pkg/model/components/etcdmanager/tests/minimal/cluster.yaml

@@ -0,0 +1,85 @@
+apiVersion: kops/v1alpha2
+kind: Cluster
+metadata:
+  creationTimestamp: "2016-12-10T22:42:27Z"
+  name: minimal.example.com
+spec:
+  kubernetesApiAccess:
+  - 0.0.0.0/0
+  channel: stable
+  cloudProvider: aws
+  configBase: memfs://clusters.example.com/minimal.example.com
+  etcdClusters:
+  - etcdMembers:
+    - instanceGroup: master-us-test-1a
+      name: us-test-1a
+    name: main
+    backups:
+      backupStore: memfs://clusters.example.com/minimal.example.com/backups/etcd-main
+    manager:
+      image: kopeio/etcd-manager:latest
+  - etcdMembers:
+    - instanceGroup: master-us-test-1a
+      name: us-test-1a
+    name: events
+    backups:
+      backupStore: memfs://clusters.example.com/minimal.example.com/backups/etcd-events
+    manager:
+      image: kopeio/etcd-manager:latest
+  kubernetesVersion: v1.8.0
+  masterInternalName: api.internal.minimal.example.com
+  masterPublicName: api.minimal.example.com
+  networkCIDR: 172.20.0.0/16
+  networking:
+    kubenet: {}
+  nonMasqueradeCIDR: 100.64.0.0/10
+  sshAccess:
+    - 0.0.0.0/0
+  topology:
+    masters: public
+    nodes: public
+  subnets:
+  - cidr: 172.20.32.0/19
+    name: us-test-1a
+    type: Public
+    zone: us-test-1a
+
+---
+
+apiVersion: kops/v1alpha2
+kind: InstanceGroup
+metadata:
+  creationTimestamp: "2016-12-10T22:42:28Z"
+  name: nodes
+  labels:
+    kops.k8s.io/cluster: minimal.example.com
+spec:
+  associatePublicIp: true
+  image: kope.io/k8s-1.4-debian-jessie-amd64-hvm-ebs-2016-10-21
+  machineType: t2.medium
+  maxSize: 2
+  minSize: 2
+  role: Node
+  subnets:
+  - us-test-1a
+
+---
+
+apiVersion: kops/v1alpha2
+kind: InstanceGroup
+metadata:
+  creationTimestamp: "2016-12-10T22:42:28Z"
+  name: master-us-test-1a
+  labels:
+    kops.k8s.io/cluster: minimal.example.com
+spec:
+  associatePublicIp: true
+  image: kope.io/k8s-1.4-debian-jessie-amd64-hvm-ebs-2016-10-21
+  machineType: m3.medium
+  maxSize: 1
+  minSize: 1
+  role: Master
+  subnets:
+  - us-test-1a
+
+

pkg/model/components/etcdmanager/tests/minimal/tasks.yaml

@@ -0,0 +1,141 @@
+Contents:
+  Name: ""
+  Resource: |-
+    {
+      "member_count": 1
+    }
+Lifecycle: null
+Location: backups/etcd/events/control/etcd-cluster-spec
+Name: etcd-cluster-spec-events
+---
+Contents:
+  Name: ""
+  Resource: |-
+    {
+      "member_count": 1
+    }
+Lifecycle: null
+Location: backups/etcd/main/control/etcd-cluster-spec
+Name: etcd-cluster-spec-main
+---
+Contents:
+  Name: ""
+  Resource: |
+    apiVersion: v1
+    kind: Pod
+    metadata:
+      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ""
+      creationTimestamp: null
+      labels:
+        k8s-app: etcd-manager-events
+      name: etcd-manager-events
+      namespace: kube-system
+    spec:
+      containers:
+      - command:
+        - /bin/sh
+        - -c
+        - mkfifo /tmp/pipe; (tee -a /var/log/etcd.log < /tmp/pipe & ) ; exec /etcd-manager
+          --backup-store=memfs://clusters.example.com/minimal.example.com/backups/etcd-events
+          --client-urls=http://__name__:4002 --cluster-name=etcd-events --containerized=true
+          --dns-suffix=.internal.minimal.example.com --grpc-port=3997 --peer-urls=http://__name__:2381
+          --quarantine-client-urls=http://__name__:3995 --v=8 --volume-name-tag=k8s.io/etcd/events
+          --volume-provider=aws --volume-tag=k8s.io/etcd/events --volume-tag=k8s.io/role/master=1
+          --volume-tag=kubernetes.io/cluster/minimal.example.com=owned > /tmp/pipe 2>&1
+        image: kopeio/etcd-manager:latest
+        name: etcd-manager
+        resources:
+          requests:
+            cpu: 100m
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - mountPath: /var/log/etcd.log
+          name: varlogetcd
+        - mountPath: /rootfs
+          name: rootfs
+        - mountPath: /etc/hosts
+          name: hosts
+      hostNetwork: true
+      tolerations:
+      - key: CriticalAddonsOnly
+        operator: Exists
+      volumes:
+      - hostPath:
+          path: /var/log/etcd-events.log
+          type: FileOrCreate
+        name: varlogetcd
+      - hostPath:
+          path: /
+          type: Directory
+        name: rootfs
+      - hostPath:
+          path: /etc/hosts
+          type: File
+        name: hosts
+    status: {}
+Lifecycle: null
+Location: manifests/etcd/events.yaml
+Name: manifests-etcdmanager-events
+---
+Contents:
+  Name: ""
+  Resource: |
+    apiVersion: v1
+    kind: Pod
+    metadata:
+      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ""
+      creationTimestamp: null
+      labels:
+        k8s-app: etcd-manager-main
+      name: etcd-manager-main
+      namespace: kube-system
+    spec:
+      containers:
+      - command:
+        - /bin/sh
+        - -c
+        - mkfifo /tmp/pipe; (tee -a /var/log/etcd.log < /tmp/pipe & ) ; exec /etcd-manager
+          --backup-store=memfs://clusters.example.com/minimal.example.com/backups/etcd-main
+          --client-urls=http://__name__:4001 --cluster-name=etcd --containerized=true
+          --dns-suffix=.internal.minimal.example.com --grpc-port=3996 --peer-urls=http://__name__:2380
+          --quarantine-client-urls=http://__name__:3994 --v=8 --volume-name-tag=k8s.io/etcd/main
+          --volume-provider=aws --volume-tag=k8s.io/etcd/main --volume-tag=k8s.io/role/master=1
+          --volume-tag=kubernetes.io/cluster/minimal.example.com=owned > /tmp/pipe 2>&1
+        image: kopeio/etcd-manager:latest
+        name: etcd-manager
+        resources:
+          requests:
+            cpu: 200m
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - mountPath: /var/log/etcd.log
+          name: varlogetcd
+        - mountPath: /rootfs
+          name: rootfs
+        - mountPath: /etc/hosts
+          name: hosts
+      hostNetwork: true
+      tolerations:
+      - key: CriticalAddonsOnly
+        operator: Exists
+      volumes:
+      - hostPath:
+          path: /var/log/etcd.log
+          type: FileOrCreate
+        name: varlogetcd
+      - hostPath:
+          path: /
+          type: Directory
+        name: rootfs
+      - hostPath:
+          path: /etc/hosts
+          type: File
+        name: hosts
+    status: {}
+Lifecycle: null
+Location: manifests/etcd/main.yaml
+Name: manifests-etcdmanager-main

pkg/testutils/BUILD.bazel

@@ -2,7 +2,10 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library")
 
 go_library(
     name = "go_default_library",
-    srcs = ["integrationtestharness.go"],
+    srcs = [
+        "integrationtestharness.go",
+        "modelharness.go",
+    ],
     importpath = "k8s.io/kops/pkg/testutils",
     visibility = ["//visibility:public"],
     deps = [
@@ -14,6 +17,10 @@ go_library(
         "//cloudmock/aws/mockiam:go_default_library",
         "//cloudmock/aws/mockroute53:go_default_library",
         "//pkg/apis/kops:go_default_library",
+        "//pkg/apis/kops/v1alpha2:go_default_library",
+        "//pkg/diff:go_default_library",
+        "//pkg/kopscodecs:go_default_library",
+        "//upup/pkg/fi:go_default_library",
         "//upup/pkg/fi/cloudup/awsup:go_default_library",
         "//upup/pkg/fi/cloudup/gce:go_default_library",
         "//util/pkg/vfs:go_default_library",
@@ -21,5 +28,6 @@ go_library(
         "//vendor/github.com/aws/aws-sdk-go/service/ec2:go_default_library",
         "//vendor/github.com/aws/aws-sdk-go/service/route53:go_default_library",
         "//vendor/github.com/golang/glog:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
     ],
 )

pkg/testutils/modelharness.go

@@ -0,0 +1,126 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package testutils
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+	"sort"
+	"strings"
+	"testing"
+
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/kops/pkg/apis/kops"
+	"k8s.io/kops/pkg/apis/kops/v1alpha2"
+	"k8s.io/kops/pkg/diff"
+	"k8s.io/kops/pkg/kopscodecs"
+	"k8s.io/kops/upup/pkg/fi"
+)
+
+type Model struct {
+	Cluster        *kops.Cluster
+	InstanceGroups []*kops.InstanceGroup
+}
+
+// LoadModel loads a cluster and instancegroups from a cluster.yaml file found in basedir
+func LoadModel(basedir string) (*Model, error) {
+	clusterYamlPath := path.Join(basedir, "cluster.yaml")
+	clusterYaml, err := ioutil.ReadFile(clusterYamlPath)
+	if err != nil {
+		return nil, fmt.Errorf("error reading file %q: %v", clusterYamlPath, err)
+	}
+
+	spec := &Model{}
+
+	// Codecs provides access to encoding and decoding for the scheme
+	codecs := kopscodecs.Codecs
+
+	codec := codecs.UniversalDecoder(kops.SchemeGroupVersion)
+
+	sections := bytes.Split(clusterYaml, []byte("\n---\n"))
+	for _, section := range sections {
+		defaults := &schema.GroupVersionKind{
+			Group:   v1alpha2.SchemeGroupVersion.Group,
+			Version: v1alpha2.SchemeGroupVersion.Version,
+		}
+		o, gvk, err := codec.Decode(section, defaults, nil)
+		if err != nil {
+			return nil, fmt.Errorf("error parsing file %v", err)
+		}
+
+		switch v := o.(type) {
+		case *kops.Cluster:
+			if spec.Cluster != nil {
+				return nil, fmt.Errorf("found multiple clusters")
+			}
+			spec.Cluster = v
+		case *kops.InstanceGroup:
+			spec.InstanceGroups = append(spec.InstanceGroups, v)
+
+		default:
+			return nil, fmt.Errorf("Unhandled kind %q", gvk)
+		}
+	}
+
+	return spec, nil
+}
+
+func ValidateTasks(t *testing.T, basedir string, context *fi.ModelBuilderContext) {
+	var keys []string
+	for key := range context.Tasks {
+		keys = append(keys, key)
+	}
+	sort.Strings(keys)
+
+	var yamls []string
+	for _, key := range keys {
+		task := context.Tasks[key]
+		yaml, err := kops.ToRawYaml(task)
+		if err != nil {
+			t.Fatalf("error serializing task: %v", err)
+		}
+		yamls = append(yamls, strings.TrimSpace(string(yaml)))
+	}
+
+	actualTasksYaml := strings.Join(yamls, "\n---\n")
+
+	tasksYamlPath := path.Join(basedir, "tasks.yaml")
+	expectedTasksYamlBytes, err := ioutil.ReadFile(tasksYamlPath)
+	if err != nil {
+		t.Fatalf("error reading file %q: %v", tasksYamlPath, err)
+	}
+
+	actualTasksYaml = strings.TrimSpace(actualTasksYaml)
+	expectedTasksYaml := strings.TrimSpace(string(expectedTasksYamlBytes))
+
+	if expectedTasksYaml != actualTasksYaml {
+		if os.Getenv("HACK_UPDATE_EXPECTED_IN_PLACE") != "" {
+			t.Logf("HACK_UPDATE_EXPECTED_IN_PLACE: writing expected output %s", tasksYamlPath)
+			if err := ioutil.WriteFile(tasksYamlPath, []byte(actualTasksYaml), 0644); err != nil {
+				t.Errorf("error writing expected output %s: %v", tasksYamlPath, err)
+			}
+		}
+
+		diffString := diff.FormatDiff(expectedTasksYaml, actualTasksYaml)
+		t.Logf("diff:\n%s\n", diffString)
+
+		t.Fatalf("tasks differed from expected for test %q", basedir)
+	}
+}

upup/pkg/fi/resources.go

@@ -139,6 +139,12 @@ type BytesResource struct {
 	data []byte
 }
 
+// MarshalJSON is a custom marshaller so this will be printed as a string (instead of nothing)
+// This is used in tests to verify the expected output.
+func (b *BytesResource) MarshalJSON() ([]byte, error) {
+	return json.Marshal(string(b.data))
+}
+
 var _ Resource = &BytesResource{}
 
 func NewBytesResource(data []byte) *BytesResource {