diff --git a/nodeup/pkg/model/BUILD.bazel b/nodeup/pkg/model/BUILD.bazel index a4166177c7..fe83f43422 100644 --- a/nodeup/pkg/model/BUILD.bazel +++ b/nodeup/pkg/model/BUILD.bazel @@ -79,11 +79,8 @@ go_test( deps = [ "//nodeup/pkg/distros:go_default_library", "//pkg/apis/kops:go_default_library", - "//pkg/apis/kops/v1alpha2:go_default_library", - "//pkg/diff:go_default_library", "//pkg/flagbuilder:go_default_library", - "//pkg/kopscodecs:go_default_library", + "//pkg/testutils:go_default_library", "//upup/pkg/fi:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", ], ) diff --git a/nodeup/pkg/model/docker_test.go b/nodeup/pkg/model/docker_test.go index a8471f639c..de6215ddf7 100644 --- a/nodeup/pkg/model/docker_test.go +++ b/nodeup/pkg/model/docker_test.go @@ -22,6 +22,7 @@ import ( "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/pkg/flagbuilder" + "k8s.io/kops/pkg/testutils" "k8s.io/kops/upup/pkg/fi" ) @@ -97,7 +98,7 @@ func TestDockerBuilder_BuildFlags(t *testing.T) { func runDockerBuilderTest(t *testing.T, key string) { basedir := path.Join("tests/dockerbuilder/", key) - nodeUpModelContext, err := LoadModel(basedir) + nodeUpModelContext, err := BuildNodeupModelContext(basedir) if err != nil { t.Fatalf("error parsing cluster yaml %q: %v", basedir, err) return @@ -115,5 +116,5 @@ func runDockerBuilderTest(t *testing.T, key string) { return } - ValidateTasks(t, basedir, context) + testutils.ValidateTasks(t, basedir, context) } diff --git a/nodeup/pkg/model/kubelet_test.go b/nodeup/pkg/model/kubelet_test.go index b68ffdd408..e19a8ce23a 100644 --- a/nodeup/pkg/model/kubelet_test.go +++ b/nodeup/pkg/model/kubelet_test.go @@ -17,21 +17,12 @@ limitations under the License. package model import ( - "bytes" - "io/ioutil" - "path" - "sort" - "strings" + "fmt" "testing" - "fmt" - - "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/kops/nodeup/pkg/distros" "k8s.io/kops/pkg/apis/kops" - "k8s.io/kops/pkg/apis/kops/v1alpha2" - "k8s.io/kops/pkg/diff" - "k8s.io/kops/pkg/kopscodecs" + "k8s.io/kops/pkg/testutils" "k8s.io/kops/upup/pkg/fi" ) @@ -170,7 +161,7 @@ func Test_RunKubeletBuilder(t *testing.T) { context := &fi.ModelBuilderContext{ Tasks: make(map[string]fi.Task), } - nodeUpModelContext, err := LoadModel(basedir) + nodeUpModelContext, err := BuildNodeupModelContext(basedir) if err != nil { t.Fatalf("error loading model %q: %v", basedir, err) return @@ -191,90 +182,36 @@ func Test_RunKubeletBuilder(t *testing.T) { } context.AddTask(fileTask) - ValidateTasks(t, basedir, context) + testutils.ValidateTasks(t, basedir, context) } -func LoadModel(basedir string) (*NodeupModelContext, error) { - clusterYamlPath := path.Join(basedir, "cluster.yaml") - clusterYaml, err := ioutil.ReadFile(clusterYamlPath) +func BuildNodeupModelContext(basedir string) (*NodeupModelContext, error) { + model, err := testutils.LoadModel(basedir) if err != nil { - return nil, fmt.Errorf("error reading cluster yaml file %q: %v", clusterYamlPath, err) + return nil, err } - var cluster *kops.Cluster - var instanceGroup *kops.InstanceGroup - - // Codecs provides access to encoding and decoding for the scheme - codecs := kopscodecs.Codecs - - codec := codecs.UniversalDecoder(kops.SchemeGroupVersion) - - sections := bytes.Split(clusterYaml, []byte("\n---\n")) - for _, section := range sections { - defaults := &schema.GroupVersionKind{ - Group: v1alpha2.SchemeGroupVersion.Group, - Version: v1alpha2.SchemeGroupVersion.Version, - } - o, gvk, err := codec.Decode(section, defaults, nil) - if err != nil { - return nil, fmt.Errorf("error parsing file %v", err) - } - - switch v := o.(type) { - case *kops.Cluster: - cluster = v - case *kops.InstanceGroup: - instanceGroup = v - default: - return nil, fmt.Errorf("Unhandled kind %q", gvk) - } + if model.Cluster == nil { + return nil, fmt.Errorf("no cluster found in %s", basedir) } nodeUpModelContext := &NodeupModelContext{ - Cluster: cluster, - Architecture: "amd64", - Distribution: distros.DistributionXenial, - InstanceGroup: instanceGroup, + Cluster: model.Cluster, + Architecture: "amd64", + Distribution: distros.DistributionXenial, } + + if len(model.InstanceGroups) == 0 { + // We tolerate this - not all tests need an instance group + } else if len(model.InstanceGroups) == 1 { + nodeUpModelContext.InstanceGroup = model.InstanceGroups[0] + } else { + return nil, fmt.Errorf("unexpected number of instance groups in %s, found %d", basedir, len(model.InstanceGroups)) + } + if err := nodeUpModelContext.Init(); err != nil { return nil, err } return nodeUpModelContext, nil } - -func ValidateTasks(t *testing.T, basedir string, context *fi.ModelBuilderContext) { - var keys []string - for key := range context.Tasks { - keys = append(keys, key) - } - sort.Strings(keys) - - var yamls []string - for _, key := range keys { - task := context.Tasks[key] - yaml, err := kops.ToRawYaml(task) - if err != nil { - t.Fatalf("error serializing task: %v", err) - } - yamls = append(yamls, strings.TrimSpace(string(yaml))) - } - - actualTasksYaml := strings.Join(yamls, "\n---\n") - - tasksYamlPath := path.Join(basedir, "tasks.yaml") - expectedTasksYamlBytes, err := ioutil.ReadFile(tasksYamlPath) - if err != nil { - t.Fatalf("error reading file %q: %v", tasksYamlPath, err) - } - - actualTasksYaml = strings.TrimSpace(actualTasksYaml) - expectedTasksYaml := strings.TrimSpace(string(expectedTasksYamlBytes)) - - if expectedTasksYaml != actualTasksYaml { - diffString := diff.FormatDiff(expectedTasksYaml, actualTasksYaml) - t.Logf("diff:\n%s\n", diffString) - - t.Fatalf("tasks differed from expected for test %q", basedir) - } -} diff --git a/pkg/model/components/etcdmanager/BUILD.bazel b/pkg/model/components/etcdmanager/BUILD.bazel index e053deacca..92ec81ab4c 100644 --- a/pkg/model/components/etcdmanager/BUILD.bazel +++ b/pkg/model/components/etcdmanager/BUILD.bazel @@ -1,4 +1,4 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library") +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "go_default_library", @@ -28,3 +28,16 @@ go_library( "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library", ], ) + +go_test( + name = "go_default_test", + srcs = ["model_test.go"], + data = glob(["tests/**"]), #keep + embed = [":go_default_library"], + deps = [ + "//pkg/assets:go_default_library", + "//pkg/model:go_default_library", + "//pkg/testutils:go_default_library", + "//upup/pkg/fi:go_default_library", + ], +) diff --git a/pkg/model/components/etcdmanager/model_test.go b/pkg/model/components/etcdmanager/model_test.go new file mode 100644 index 0000000000..250f9459ed --- /dev/null +++ b/pkg/model/components/etcdmanager/model_test.go @@ -0,0 +1,74 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package etcdmanager + +import ( + "fmt" + "testing" + + "k8s.io/kops/pkg/assets" + "k8s.io/kops/pkg/model" + "k8s.io/kops/pkg/testutils" + "k8s.io/kops/upup/pkg/fi" +) + +func Test_RunEtcdManagerBuilder(t *testing.T) { + basedir := "tests/minimal" + + context := &fi.ModelBuilderContext{ + Tasks: make(map[string]fi.Task), + } + kopsModelContext, err := LoadKopsModelContext(basedir) + if err != nil { + t.Fatalf("error loading model %q: %v", basedir, err) + return + } + + builder := EtcdManagerBuilder{ + KopsModelContext: kopsModelContext, + AssetBuilder: assets.NewAssetBuilder(kopsModelContext.Cluster, ""), + } + + if err := builder.Build(context); err != nil { + t.Fatalf("error from Build: %v", err) + return + } + + testutils.ValidateTasks(t, basedir, context) +} + +func LoadKopsModelContext(basedir string) (*model.KopsModelContext, error) { + spec, err := testutils.LoadModel(basedir) + if err != nil { + return nil, err + } + + if spec.Cluster == nil { + return nil, fmt.Errorf("no cluster found in %s", basedir) + } + + if len(spec.InstanceGroups) == 0 { + return nil, fmt.Errorf("no instance groups found in %s", basedir) + } + + kopsContext := &model.KopsModelContext{ + Cluster: spec.Cluster, + InstanceGroups: spec.InstanceGroups, + } + + return kopsContext, nil +} diff --git a/pkg/model/components/etcdmanager/tests/minimal/cluster.yaml b/pkg/model/components/etcdmanager/tests/minimal/cluster.yaml new file mode 100644 index 0000000000..a6bfa70a68 --- /dev/null +++ b/pkg/model/components/etcdmanager/tests/minimal/cluster.yaml @@ -0,0 +1,85 @@ +apiVersion: kops/v1alpha2 +kind: Cluster +metadata: + creationTimestamp: "2016-12-10T22:42:27Z" + name: minimal.example.com +spec: + kubernetesApiAccess: + - 0.0.0.0/0 + channel: stable + cloudProvider: aws + configBase: memfs://clusters.example.com/minimal.example.com + etcdClusters: + - etcdMembers: + - instanceGroup: master-us-test-1a + name: us-test-1a + name: main + backups: + backupStore: memfs://clusters.example.com/minimal.example.com/backups/etcd-main + manager: + image: kopeio/etcd-manager:latest + - etcdMembers: + - instanceGroup: master-us-test-1a + name: us-test-1a + name: events + backups: + backupStore: memfs://clusters.example.com/minimal.example.com/backups/etcd-events + manager: + image: kopeio/etcd-manager:latest + kubernetesVersion: v1.8.0 + masterInternalName: api.internal.minimal.example.com + masterPublicName: api.minimal.example.com + networkCIDR: 172.20.0.0/16 + networking: + kubenet: {} + nonMasqueradeCIDR: 100.64.0.0/10 + sshAccess: + - 0.0.0.0/0 + topology: + masters: public + nodes: public + subnets: + - cidr: 172.20.32.0/19 + name: us-test-1a + type: Public + zone: us-test-1a + +--- + +apiVersion: kops/v1alpha2 +kind: InstanceGroup +metadata: + creationTimestamp: "2016-12-10T22:42:28Z" + name: nodes + labels: + kops.k8s.io/cluster: minimal.example.com +spec: + associatePublicIp: true + image: kope.io/k8s-1.4-debian-jessie-amd64-hvm-ebs-2016-10-21 + machineType: t2.medium + maxSize: 2 + minSize: 2 + role: Node + subnets: + - us-test-1a + +--- + +apiVersion: kops/v1alpha2 +kind: InstanceGroup +metadata: + creationTimestamp: "2016-12-10T22:42:28Z" + name: master-us-test-1a + labels: + kops.k8s.io/cluster: minimal.example.com +spec: + associatePublicIp: true + image: kope.io/k8s-1.4-debian-jessie-amd64-hvm-ebs-2016-10-21 + machineType: m3.medium + maxSize: 1 + minSize: 1 + role: Master + subnets: + - us-test-1a + + diff --git a/pkg/model/components/etcdmanager/tests/minimal/tasks.yaml b/pkg/model/components/etcdmanager/tests/minimal/tasks.yaml new file mode 100644 index 0000000000..144afef9b3 --- /dev/null +++ b/pkg/model/components/etcdmanager/tests/minimal/tasks.yaml @@ -0,0 +1,141 @@ +Contents: + Name: "" + Resource: |- + { + "member_count": 1 + } +Lifecycle: null +Location: backups/etcd/events/control/etcd-cluster-spec +Name: etcd-cluster-spec-events +--- +Contents: + Name: "" + Resource: |- + { + "member_count": 1 + } +Lifecycle: null +Location: backups/etcd/main/control/etcd-cluster-spec +Name: etcd-cluster-spec-main +--- +Contents: + Name: "" + Resource: | + apiVersion: v1 + kind: Pod + metadata: + annotations: + scheduler.alpha.kubernetes.io/critical-pod: "" + creationTimestamp: null + labels: + k8s-app: etcd-manager-events + name: etcd-manager-events + namespace: kube-system + spec: + containers: + - command: + - /bin/sh + - -c + - mkfifo /tmp/pipe; (tee -a /var/log/etcd.log < /tmp/pipe & ) ; exec /etcd-manager + --backup-store=memfs://clusters.example.com/minimal.example.com/backups/etcd-events + --client-urls=http://__name__:4002 --cluster-name=etcd-events --containerized=true + --dns-suffix=.internal.minimal.example.com --grpc-port=3997 --peer-urls=http://__name__:2381 + --quarantine-client-urls=http://__name__:3995 --v=8 --volume-name-tag=k8s.io/etcd/events + --volume-provider=aws --volume-tag=k8s.io/etcd/events --volume-tag=k8s.io/role/master=1 + --volume-tag=kubernetes.io/cluster/minimal.example.com=owned > /tmp/pipe 2>&1 + image: kopeio/etcd-manager:latest + name: etcd-manager + resources: + requests: + cpu: 100m + securityContext: + privileged: true + volumeMounts: + - mountPath: /var/log/etcd.log + name: varlogetcd + - mountPath: /rootfs + name: rootfs + - mountPath: /etc/hosts + name: hosts + hostNetwork: true + tolerations: + - key: CriticalAddonsOnly + operator: Exists + volumes: + - hostPath: + path: /var/log/etcd-events.log + type: FileOrCreate + name: varlogetcd + - hostPath: + path: / + type: Directory + name: rootfs + - hostPath: + path: /etc/hosts + type: File + name: hosts + status: {} +Lifecycle: null +Location: manifests/etcd/events.yaml +Name: manifests-etcdmanager-events +--- +Contents: + Name: "" + Resource: | + apiVersion: v1 + kind: Pod + metadata: + annotations: + scheduler.alpha.kubernetes.io/critical-pod: "" + creationTimestamp: null + labels: + k8s-app: etcd-manager-main + name: etcd-manager-main + namespace: kube-system + spec: + containers: + - command: + - /bin/sh + - -c + - mkfifo /tmp/pipe; (tee -a /var/log/etcd.log < /tmp/pipe & ) ; exec /etcd-manager + --backup-store=memfs://clusters.example.com/minimal.example.com/backups/etcd-main + --client-urls=http://__name__:4001 --cluster-name=etcd --containerized=true + --dns-suffix=.internal.minimal.example.com --grpc-port=3996 --peer-urls=http://__name__:2380 + --quarantine-client-urls=http://__name__:3994 --v=8 --volume-name-tag=k8s.io/etcd/main + --volume-provider=aws --volume-tag=k8s.io/etcd/main --volume-tag=k8s.io/role/master=1 + --volume-tag=kubernetes.io/cluster/minimal.example.com=owned > /tmp/pipe 2>&1 + image: kopeio/etcd-manager:latest + name: etcd-manager + resources: + requests: + cpu: 200m + securityContext: + privileged: true + volumeMounts: + - mountPath: /var/log/etcd.log + name: varlogetcd + - mountPath: /rootfs + name: rootfs + - mountPath: /etc/hosts + name: hosts + hostNetwork: true + tolerations: + - key: CriticalAddonsOnly + operator: Exists + volumes: + - hostPath: + path: /var/log/etcd.log + type: FileOrCreate + name: varlogetcd + - hostPath: + path: / + type: Directory + name: rootfs + - hostPath: + path: /etc/hosts + type: File + name: hosts + status: {} +Lifecycle: null +Location: manifests/etcd/main.yaml +Name: manifests-etcdmanager-main \ No newline at end of file diff --git a/pkg/testutils/BUILD.bazel b/pkg/testutils/BUILD.bazel index 0d6d1f03fe..66f0fb8918 100644 --- a/pkg/testutils/BUILD.bazel +++ b/pkg/testutils/BUILD.bazel @@ -2,7 +2,10 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library") go_library( name = "go_default_library", - srcs = ["integrationtestharness.go"], + srcs = [ + "integrationtestharness.go", + "modelharness.go", + ], importpath = "k8s.io/kops/pkg/testutils", visibility = ["//visibility:public"], deps = [ @@ -14,6 +17,10 @@ go_library( "//cloudmock/aws/mockiam:go_default_library", "//cloudmock/aws/mockroute53:go_default_library", "//pkg/apis/kops:go_default_library", + "//pkg/apis/kops/v1alpha2:go_default_library", + "//pkg/diff:go_default_library", + "//pkg/kopscodecs:go_default_library", + "//upup/pkg/fi:go_default_library", "//upup/pkg/fi/cloudup/awsup:go_default_library", "//upup/pkg/fi/cloudup/gce:go_default_library", "//util/pkg/vfs:go_default_library", @@ -21,5 +28,6 @@ go_library( "//vendor/github.com/aws/aws-sdk-go/service/ec2:go_default_library", "//vendor/github.com/aws/aws-sdk-go/service/route53:go_default_library", "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", ], ) diff --git a/pkg/testutils/modelharness.go b/pkg/testutils/modelharness.go new file mode 100644 index 0000000000..019a07b399 --- /dev/null +++ b/pkg/testutils/modelharness.go @@ -0,0 +1,126 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package testutils + +import ( + "bytes" + "fmt" + "io/ioutil" + "os" + "path" + "sort" + "strings" + "testing" + + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/kops/pkg/apis/kops" + "k8s.io/kops/pkg/apis/kops/v1alpha2" + "k8s.io/kops/pkg/diff" + "k8s.io/kops/pkg/kopscodecs" + "k8s.io/kops/upup/pkg/fi" +) + +type Model struct { + Cluster *kops.Cluster + InstanceGroups []*kops.InstanceGroup +} + +// LoadModel loads a cluster and instancegroups from a cluster.yaml file found in basedir +func LoadModel(basedir string) (*Model, error) { + clusterYamlPath := path.Join(basedir, "cluster.yaml") + clusterYaml, err := ioutil.ReadFile(clusterYamlPath) + if err != nil { + return nil, fmt.Errorf("error reading file %q: %v", clusterYamlPath, err) + } + + spec := &Model{} + + // Codecs provides access to encoding and decoding for the scheme + codecs := kopscodecs.Codecs + + codec := codecs.UniversalDecoder(kops.SchemeGroupVersion) + + sections := bytes.Split(clusterYaml, []byte("\n---\n")) + for _, section := range sections { + defaults := &schema.GroupVersionKind{ + Group: v1alpha2.SchemeGroupVersion.Group, + Version: v1alpha2.SchemeGroupVersion.Version, + } + o, gvk, err := codec.Decode(section, defaults, nil) + if err != nil { + return nil, fmt.Errorf("error parsing file %v", err) + } + + switch v := o.(type) { + case *kops.Cluster: + if spec.Cluster != nil { + return nil, fmt.Errorf("found multiple clusters") + } + spec.Cluster = v + case *kops.InstanceGroup: + spec.InstanceGroups = append(spec.InstanceGroups, v) + + default: + return nil, fmt.Errorf("Unhandled kind %q", gvk) + } + } + + return spec, nil +} + +func ValidateTasks(t *testing.T, basedir string, context *fi.ModelBuilderContext) { + var keys []string + for key := range context.Tasks { + keys = append(keys, key) + } + sort.Strings(keys) + + var yamls []string + for _, key := range keys { + task := context.Tasks[key] + yaml, err := kops.ToRawYaml(task) + if err != nil { + t.Fatalf("error serializing task: %v", err) + } + yamls = append(yamls, strings.TrimSpace(string(yaml))) + } + + actualTasksYaml := strings.Join(yamls, "\n---\n") + + tasksYamlPath := path.Join(basedir, "tasks.yaml") + expectedTasksYamlBytes, err := ioutil.ReadFile(tasksYamlPath) + if err != nil { + t.Fatalf("error reading file %q: %v", tasksYamlPath, err) + } + + actualTasksYaml = strings.TrimSpace(actualTasksYaml) + expectedTasksYaml := strings.TrimSpace(string(expectedTasksYamlBytes)) + + if expectedTasksYaml != actualTasksYaml { + if os.Getenv("HACK_UPDATE_EXPECTED_IN_PLACE") != "" { + t.Logf("HACK_UPDATE_EXPECTED_IN_PLACE: writing expected output %s", tasksYamlPath) + if err := ioutil.WriteFile(tasksYamlPath, []byte(actualTasksYaml), 0644); err != nil { + t.Errorf("error writing expected output %s: %v", tasksYamlPath, err) + } + } + + diffString := diff.FormatDiff(expectedTasksYaml, actualTasksYaml) + t.Logf("diff:\n%s\n", diffString) + + t.Fatalf("tasks differed from expected for test %q", basedir) + } +} diff --git a/upup/pkg/fi/resources.go b/upup/pkg/fi/resources.go index 6cfb46c840..095008ef7c 100644 --- a/upup/pkg/fi/resources.go +++ b/upup/pkg/fi/resources.go @@ -139,6 +139,12 @@ type BytesResource struct { data []byte } +// MarshalJSON is a custom marshaller so this will be printed as a string (instead of nothing) +// This is used in tests to verify the expected output. +func (b *BytesResource) MarshalJSON() ([]byte, error) { + return json.Marshal(string(b.data)) +} + var _ Resource = &BytesResource{} func NewBytesResource(data []byte) *BytesResource {