From 3d81e21acf359a42933a88e77f12dac3a1471447 Mon Sep 17 00:00:00 2001 From: Karl Isenberg Date: Thu, 23 Sep 2021 16:13:34 -0700 Subject: [PATCH] e2e test cleanup - Unstructured() added to ResourceReference to aid test operations - Test artifacts moved into artifacts_test.go - deploymentManifest and apiserviceManifest replaced with yaml->unstructured, for consistency with other test artifacts - Added withReplicas, withNamespace, and withDependsOn for easier inline mutation of test artifacts - Added deleteUnstructuredAndWait, assertUnstructuredExists, and assertUnstructuredDoesNotExist to validate actual cluster state, not just events. Also reduced flakiness by waiting for delete. - Removed error check before event validation. The event diff provides much more context for debugging. - Replaced typed resources with unstructured and used testutil.NestedField to extract values. - Updated prune error test to use full event list comparison and validate server state. - Add 2nd resource to continue on error test - Don't fail early on error from event list - Fix flakey prune error test Add wait for pod2 after creation Add wait for pod2 after deletion Re-use inventory to ensure prune of pod2 not skipped - Add optional klog support for e2e tests --- pkg/object/mutation/types.go | 15 +- test/e2e/apply_and_destroy_test.go | 21 +- test/e2e/artifacts_test.go | 143 ++++++++++++++ test/e2e/common_test.go | 206 ++++++++++++------- test/e2e/continue_on_error_test.go | 58 ++++-- test/e2e/crd_test.go | 3 +- test/e2e/depends_on_test.go | 96 ++++----- test/e2e/e2e_test.go | 27 ++- test/e2e/inventory_policy_test.go | 76 +++---- test/e2e/mutation_test.go | 122 +++--------- test/e2e/name_inv_strategy_test.go | 2 +- test/e2e/prune_retrieve_error_test.go | 273 ++++++++++++++++++++++++-- test/e2e/serverside_apply_test.go | 52 +++-- 13 files changed, 741 insertions(+), 353 deletions(-) create mode 100644 test/e2e/artifacts_test.go diff --git a/pkg/object/mutation/types.go b/pkg/object/mutation/types.go index e53c7f3..3a2ad1e 100644 --- a/pkg/object/mutation/types.go +++ b/pkg/object/mutation/types.go @@ -114,7 +114,8 @@ func (r ResourceReference) GroupVersionKind() schema.GroupVersionKind { return schema.FromAPIVersionAndKind(r.APIVersion, r.Kind) } -// ObjMetadata returns the name, namespace, group, and kind of the ResourceReference +// ObjMetadata returns the name, namespace, group, and kind of the +// ResourceReference, wrapped in a new ObjMetadata object. func (r ResourceReference) ObjMetadata() object.ObjMetadata { return object.ObjMetadata{ Name: r.Name, @@ -123,6 +124,18 @@ func (r ResourceReference) ObjMetadata() object.ObjMetadata { } } +// Unstructured returns the name, namespace, group, version, and kind of the +// ResourceReference, wrapped in a new Unstructured object. +// This is useful for performing operations with +// sigs.k8s.io/controller-runtime/pkg/client's unstructured Client. +func (r ResourceReference) Unstructured() *unstructured.Unstructured { + obj := &unstructured.Unstructured{} + obj.SetName(r.Name) + obj.SetNamespace(r.Namespace) + obj.SetGroupVersionKind(r.GroupVersionKind()) + return obj +} + // String returns the format GROUP[/VERSION][/namespaces/NAMESPACE]/KIND/NAME func (r ResourceReference) String() string { group := r.Group diff --git a/test/e2e/apply_and_destroy_test.go b/test/e2e/apply_and_destroy_test.go index 822c2fd..7cbff62 100644 --- a/test/e2e/apply_and_destroy_test.go +++ b/test/e2e/apply_and_destroy_test.go @@ -28,7 +28,7 @@ func applyAndDestroyTest(c client.Client, invConfig InventoryConfig, inventoryNa inventoryInfo := createInventoryInfo(invConfig, inventoryName, namespaceName, inventoryID) resources := []*unstructured.Unstructured{ - deploymentManifest(namespaceName), + withNamespace(manifestToUnstructured(deployment1), namespaceName), } applyCh := applier.Run(context.TODO(), inventoryInfo, resources, apply.Options{ @@ -38,7 +38,6 @@ func applyAndDestroyTest(c client.Client, invConfig InventoryConfig, inventoryNa var applierEvents []event.Event for e := range applyCh { - Expect(e.Type).NotTo(Equal(event.ErrorType)) applierEvents = append(applierEvents, e) } @@ -80,7 +79,7 @@ func applyAndDestroyTest(c client.Client, invConfig InventoryConfig, inventoryNa EventType: event.ApplyType, ApplyEvent: &testutil.ExpApplyEvent{ Operation: event.Created, - Identifier: object.UnstructuredToObjMetaOrDie(deploymentManifest(namespaceName)), + Identifier: object.UnstructuredToObjMetaOrDie(withNamespace(manifestToUnstructured(deployment1), namespaceName)), Error: nil, }, }, @@ -136,7 +135,7 @@ func applyAndDestroyTest(c client.Client, invConfig InventoryConfig, inventoryNa expected := testutil.ExpEvent{ EventType: event.StatusType, StatusEvent: &testutil.ExpStatusEvent{ - Identifier: object.UnstructuredToObjMetaOrDie(deploymentManifest(namespaceName)), + Identifier: object.UnstructuredToObjMetaOrDie(withNamespace(manifestToUnstructured(deployment1), namespaceName)), Status: status.NotFoundStatus, Error: nil, }, @@ -148,7 +147,7 @@ func applyAndDestroyTest(c client.Client, invConfig InventoryConfig, inventoryNa expected = testutil.ExpEvent{ EventType: event.StatusType, StatusEvent: &testutil.ExpStatusEvent{ - Identifier: object.UnstructuredToObjMetaOrDie(deploymentManifest(namespaceName)), + Identifier: object.UnstructuredToObjMetaOrDie(withNamespace(manifestToUnstructured(deployment1), namespaceName)), Status: status.InProgressStatus, Error: nil, }, @@ -159,7 +158,7 @@ func applyAndDestroyTest(c client.Client, invConfig InventoryConfig, inventoryNa expected = testutil.ExpEvent{ EventType: event.StatusType, StatusEvent: &testutil.ExpStatusEvent{ - Identifier: object.UnstructuredToObjMetaOrDie(deploymentManifest(namespaceName)), + Identifier: object.UnstructuredToObjMetaOrDie(withNamespace(manifestToUnstructured(deployment1), namespaceName)), Status: status.CurrentStatus, Error: nil, }, @@ -169,6 +168,9 @@ func applyAndDestroyTest(c client.Client, invConfig InventoryConfig, inventoryNa Expect(received).To(testutil.Equal(expEvents)) + By("Verify deployment created") + assertUnstructuredExists(c, withNamespace(manifestToUnstructured(deployment1), namespaceName)) + By("Verify inventory") invConfig.InvSizeVerifyFunc(c, inventoryName, namespaceName, inventoryID, 1) @@ -176,7 +178,7 @@ func applyAndDestroyTest(c client.Client, invConfig InventoryConfig, inventoryNa destroyer := invConfig.DestroyerFactoryFunc() options := apply.DestroyerOptions{InventoryPolicy: inventory.AdoptIfNoInventory} - destroyerEvents := runCollectNoErr(destroyer.Run(inventoryInfo, options)) + destroyerEvents := runCollect(destroyer.Run(inventoryInfo, options)) expEvents = []testutil.ExpEvent{ { @@ -198,7 +200,7 @@ func applyAndDestroyTest(c client.Client, invConfig InventoryConfig, inventoryNa EventType: event.DeleteType, DeleteEvent: &testutil.ExpDeleteEvent{ Operation: event.Deleted, - Identifier: object.UnstructuredToObjMetaOrDie(deploymentManifest(namespaceName)), + Identifier: object.UnstructuredToObjMetaOrDie(withNamespace(manifestToUnstructured(deployment1), namespaceName)), Error: nil, }, }, @@ -251,6 +253,9 @@ func applyAndDestroyTest(c client.Client, invConfig InventoryConfig, inventoryNa } Expect(testutil.EventsToExpEvents(destroyerEvents)).To(testutil.Equal(expEvents)) + + By("Verify deployment deleted") + assertUnstructuredDoesNotExist(c, withNamespace(manifestToUnstructured(deployment1), namespaceName)) } func createInventoryInfo(invConfig InventoryConfig, inventoryName, namespaceName, inventoryID string) inventory.InventoryInfo { diff --git a/test/e2e/artifacts_test.go b/test/e2e/artifacts_test.go new file mode 100644 index 0000000..1ed025f --- /dev/null +++ b/test/e2e/artifacts_test.go @@ -0,0 +1,143 @@ +// Copyright 2020 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package e2e + +import ( + "strings" +) + +var deployment1 = []byte(strings.TrimSpace(` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-deployment +spec: + replicas: 4 + selector: + matchLabels: + app: nginx + template: + metadata: + labels: + app: nginx + spec: + containers: + - name: nginx + image: nginx:1.19.6 + ports: + - containerPort: 80 +`)) + +var apiservice1 = []byte(strings.TrimSpace(` +apiVersion: apiregistration.k8s.io/v1 +kind: APIService +metadata: + name: v1beta1.custom.metrics.k8s.io +spec: + insecureSkipTLSVerify: true + group: custom.metrics.k8s.io + groupPriorityMinimum: 100 + versionPriority: 100 + service: + name: custom-metrics-stackdriver-adapter + namespace: custom-metrics + version: v1beta1 +`)) + +var invalidCrd = []byte(strings.TrimSpace(` +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: invalidexamples.cli-utils.example.io +spec: + conversion: + strategy: None + group: cli-utils.example.io + names: + kind: InvalidExample + listKind: InvalidExampleList + plural: invalidexamples + singular: invalidexample + scope: Cluster +`)) + +var pod1 = []byte(strings.TrimSpace(` +kind: Pod +apiVersion: v1 +metadata: + name: pod1 +spec: + containers: + - name: kubernetes-pause + image: k8s.gcr.io/pause:2.0 +`)) + +var pod2 = []byte(strings.TrimSpace(` +kind: Pod +apiVersion: v1 +metadata: + name: pod2 +spec: + containers: + - name: kubernetes-pause + image: k8s.gcr.io/pause:2.0 +`)) + +var pod3 = []byte(strings.TrimSpace(` +kind: Pod +apiVersion: v1 +metadata: + name: pod3 +spec: + containers: + - name: kubernetes-pause + image: k8s.gcr.io/pause:2.0 +`)) + +var podA = []byte(strings.TrimSpace(` +kind: Pod +apiVersion: v1 +metadata: + name: pod-a + namespace: test + annotations: + config.kubernetes.io/apply-time-mutation: | + - sourceRef: + kind: Pod + name: pod-b + sourcePath: $.status.podIP + targetPath: $.spec.containers[?(@.name=="nginx")].env[?(@.name=="SERVICE_HOST")].value + token: ${pob-b-ip} + - sourceRef: + kind: Pod + name: pod-b + sourcePath: $.spec.containers[?(@.name=="nginx")].ports[?(@.name=="tcp")].containerPort + targetPath: $.spec.containers[?(@.name=="nginx")].env[?(@.name=="SERVICE_HOST")].value + token: ${pob-b-port} +spec: + containers: + - name: nginx + image: nginx:1.21 + ports: + - name: tcp + containerPort: 80 + env: + - name: SERVICE_HOST + value: "${pob-b-ip}:${pob-b-port}" +`)) + +var podB = []byte(strings.TrimSpace(` +kind: Pod +apiVersion: v1 +metadata: + name: pod-b + namespace: test +spec: + containers: + - name: nginx + image: nginx:1.21 + ports: + - name: tcp + containerPort: 80 +`)) diff --git a/test/e2e/common_test.go b/test/e2e/common_test.go index 9529f56..4ce9acd 100644 --- a/test/e2e/common_test.go +++ b/test/e2e/common_test.go @@ -4,20 +4,149 @@ package e2e import ( + "context" "fmt" "strings" + "time" + . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/yaml" "sigs.k8s.io/cli-utils/pkg/apply/event" "sigs.k8s.io/cli-utils/pkg/common" + "sigs.k8s.io/cli-utils/pkg/object/dependson" + "sigs.k8s.io/cli-utils/pkg/object/mutation" + "sigs.k8s.io/controller-runtime/pkg/client" ) +func withReplicas(obj *unstructured.Unstructured, replicas int) *unstructured.Unstructured { + err := unstructured.SetNestedField(obj.Object, int64(replicas), "spec", "replicas") + Expect(err).NotTo(HaveOccurred()) + return obj +} + +func withNamespace(obj *unstructured.Unstructured, namespace string) *unstructured.Unstructured { + obj.SetNamespace(namespace) + return obj +} + +func withDependsOn(obj *unstructured.Unstructured, dep string) *unstructured.Unstructured { + a := obj.GetAnnotations() + if a == nil { + a = make(map[string]string, 1) + } + a[dependson.Annotation] = dep + obj.SetAnnotations(a) + return obj +} + +func deleteUnstructuredAndWait(c client.Client, obj *unstructured.Unstructured) { + ref := mutation.NewResourceReference(obj) + + err := c.Delete(context.TODO(), obj, + client.PropagationPolicy(metav1.DeletePropagationForeground)) + Expect(err).NotTo(HaveOccurred(), + "expected DELETE to not error (%s): %s", ref, err) + + waitForDeletion(c, obj) +} + +func waitForDeletion(c client.Client, obj *unstructured.Unstructured) { + ref := mutation.NewResourceReference(obj) + resultObj := ref.Unstructured() + + timeout := 30 * time.Second + retry := 2 * time.Second + + t := time.NewTimer(timeout) + s := time.NewTimer(0) + defer t.Stop() + + for { + select { + case <-t.C: + Fail("timed out waiting for resource to be fully deleted") + return + case <-s.C: + err := c.Get(context.TODO(), types.NamespacedName{ + Namespace: obj.GetNamespace(), + Name: obj.GetName(), + }, resultObj) + if err != nil { + Expect(apierrors.ReasonForError(err)).To(Equal(metav1.StatusReasonNotFound), + "expected GET to error with NotFound (%s): %s", ref, err) + return + } + s = time.NewTimer(retry) + } + } +} + +func waitForCreation(c client.Client, obj *unstructured.Unstructured) { + ref := mutation.NewResourceReference(obj) + resultObj := ref.Unstructured() + + timeout := 30 * time.Second + retry := 2 * time.Second + + t := time.NewTimer(timeout) + s := time.NewTimer(0) + defer t.Stop() + + for { + select { + case <-t.C: + Fail("timed out waiting for resource to be fully created") + return + case <-s.C: + err := c.Get(context.TODO(), types.NamespacedName{ + Namespace: obj.GetNamespace(), + Name: obj.GetName(), + }, resultObj) + if err == nil { + return + } + Expect(apierrors.ReasonForError(err)).To(Equal(metav1.StatusReasonNotFound), + "expected GET to error with NotFound (%s): %s", ref, err) + // if NotFound, sleep and retry + s = time.NewTimer(retry) + } + } +} + +func assertUnstructuredExists(c client.Client, obj *unstructured.Unstructured) *unstructured.Unstructured { + ref := mutation.NewResourceReference(obj) + resultObj := ref.Unstructured() + + err := c.Get(context.TODO(), types.NamespacedName{ + Namespace: obj.GetNamespace(), + Name: obj.GetName(), + }, resultObj) + Expect(err).NotTo(HaveOccurred(), + "expected GET not to error (%s): %s", ref, err) + return resultObj +} + +func assertUnstructuredDoesNotExist(c client.Client, obj *unstructured.Unstructured) { + ref := mutation.NewResourceReference(obj) + resultObj := ref.Unstructured() + + err := c.Get(context.TODO(), types.NamespacedName{ + Namespace: obj.GetNamespace(), + Name: obj.GetName(), + }, resultObj) + Expect(err).To(HaveOccurred(), + "expected GET to error (%s)", ref) + Expect(apierrors.ReasonForError(err)).To(Equal(metav1.StatusReasonNotFound), + "expected GET to error with NotFound (%s): %s", ref, err) +} + func randomString(prefix string) string { randomSuffix := common.RandomStr() return fmt.Sprintf("%s%s", prefix, randomSuffix) @@ -91,73 +220,6 @@ metadata: return u } -func deploymentManifest(namespace string) *unstructured.Unstructured { - dep := &appsv1.Deployment{ - TypeMeta: metav1.TypeMeta{ - APIVersion: appsv1.SchemeGroupVersion.String(), - Kind: "Deployment", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "nginx-deployment", - Namespace: namespace, - }, - Spec: appsv1.DeploymentSpec{ - Replicas: func() *int32 { r := int32(4); return &r }(), - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "app": "nginx", - }, - }, - Template: v1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "app": "nginx", - }, - }, - Spec: v1.PodSpec{ - Containers: []v1.Container{ - { - Name: "nginx", - Image: "nginx:1.19.6", - }, - }, - }, - }, - }, - } - u, err := runtime.DefaultUnstructuredConverter.ToUnstructured(dep) - if err != nil { - panic(err) - } - return &unstructured.Unstructured{ - Object: u, - } -} - -func apiserviceManifest() *unstructured.Unstructured { - apiservice := &unstructured.Unstructured{ - Object: map[string]interface{}{ - "apiVersion": "apiregistration.k8s.io/v1", - "kind": "APIService", - "metadata": map[string]interface{}{ - "name": "v1beta1.custom.metrics.k8s.io", - }, - "spec": map[string]interface{}{ - "insecureSkipTLSVerify": true, - "group": "custom.metrics.k8s.io", - "groupPriorityMinimum": 100, - "versionPriority": 100, - "service": map[string]interface{}{ - "name": "custom-metrics-stackdriver-adapter", - "namespace": "custome-metrics", - }, - "version": "v1beta1", - }, - }, - } - return apiservice -} - func manifestToUnstructured(manifest []byte) *unstructured.Unstructured { u := make(map[string]interface{}) err := yaml.Unmarshal(manifest, &u) @@ -168,11 +230,3 @@ func manifestToUnstructured(manifest []byte) *unstructured.Unstructured { Object: u, } } - -func updateReplicas(u *unstructured.Unstructured, replicas int) *unstructured.Unstructured { - err := unstructured.SetNestedField(u.Object, int64(replicas), "spec", "replicas") - if err != nil { - panic(err) - } - return u -} diff --git a/test/e2e/continue_on_error_test.go b/test/e2e/continue_on_error_test.go index 7b40be8..5ea2890 100644 --- a/test/e2e/continue_on_error_test.go +++ b/test/e2e/continue_on_error_test.go @@ -6,7 +6,6 @@ package e2e import ( "context" "errors" - "strings" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -19,7 +18,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -func continueOnErrorTest(_ client.Client, invConfig InventoryConfig, inventoryName, namespaceName string) { +func continueOnErrorTest(c client.Client, invConfig InventoryConfig, inventoryName, namespaceName string) { By("apply an invalid CRD") applier := invConfig.ApplierFactoryFunc() @@ -27,13 +26,13 @@ func continueOnErrorTest(_ client.Client, invConfig InventoryConfig, inventoryNa resources := []*unstructured.Unstructured{ manifestToUnstructured(invalidCrd), + withNamespace(manifestToUnstructured(pod1), namespaceName), } ch := applier.Run(context.TODO(), inv, resources, apply.Options{}) var applierEvents []event.Event for e := range ch { - Expect(e.Type).NotTo(Equal(event.ErrorType)) applierEvents = append(applierEvents, e) } @@ -71,7 +70,7 @@ func continueOnErrorTest(_ client.Client, invConfig InventoryConfig, inventoryNa }, }, { - // Apply object which fails + // Apply invalidCrd fails EventType: event.ApplyType, ApplyEvent: &testutil.ExpApplyEvent{ Identifier: object.UnstructuredToObjMetaOrDie(manifestToUnstructured(invalidCrd)), @@ -80,6 +79,15 @@ func continueOnErrorTest(_ client.Client, invConfig InventoryConfig, inventoryNa ), }, }, + { + // Create pod1 + EventType: event.ApplyType, + ApplyEvent: &testutil.ExpApplyEvent{ + Operation: event.Created, + Identifier: object.UnstructuredToObjMetaOrDie(withNamespace(manifestToUnstructured(pod1), namespaceName)), + Error: nil, + }, + }, { // ApplyTask finished EventType: event.ActionGroupType, @@ -90,6 +98,25 @@ func continueOnErrorTest(_ client.Client, invConfig InventoryConfig, inventoryNa }, }, // Note: No WaitTask when apply fails + // TODO: why no wait after create tho? + // { + // // WaitTask start + // EventType: event.ActionGroupType, + // ActionGroupEvent: &testutil.ExpActionGroupEvent{ + // Action: event.WaitAction, + // Name: "wait-0", + // Type: event.Started, + // }, + // }, + // { + // // WaitTask finished + // EventType: event.ActionGroupType, + // ActionGroupEvent: &testutil.ExpActionGroupEvent{ + // Action: event.WaitAction, + // Name: "wait-0", + // Type: event.Finished, + // }, + // }, { // InvSetTask start EventType: event.ActionGroupType, @@ -110,21 +137,10 @@ func continueOnErrorTest(_ client.Client, invConfig InventoryConfig, inventoryNa }, } Expect(testutil.EventsToExpEvents(applierEvents)).To(testutil.Equal(expEvents)) -} -var invalidCrd = []byte(strings.TrimSpace(` -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: invalidexamples.cli-utils.example.io -spec: - conversion: - strategy: None - group: cli-utils.example.io - names: - kind: InvalidExample - listKind: InvalidExampleList - plural: invalidexamples - singular: invalidexample - scope: Cluster -`)) + By("Verify pod1 created") + assertUnstructuredExists(c, withNamespace(manifestToUnstructured(pod1), namespaceName)) + + By("Verify CRD not created") + assertUnstructuredDoesNotExist(c, manifestToUnstructured(invalidCrd)) +} diff --git a/test/e2e/crd_test.go b/test/e2e/crd_test.go index a4128b1..59d85a4 100644 --- a/test/e2e/crd_test.go +++ b/test/e2e/crd_test.go @@ -37,7 +37,6 @@ func crdTest(_ client.Client, invConfig InventoryConfig, inventoryName, namespac var applierEvents []event.Event for e := range ch { - Expect(e.Type).NotTo(Equal(event.ErrorType)) applierEvents = append(applierEvents, e) } @@ -179,7 +178,7 @@ func crdTest(_ client.Client, invConfig InventoryConfig, inventoryName, namespac By("destroy the resources, including the crd") destroyer := invConfig.DestroyerFactoryFunc() options := apply.DestroyerOptions{InventoryPolicy: inventory.AdoptIfNoInventory} - destroyerEvents := runCollectNoErr(destroyer.Run(inv, options)) + destroyerEvents := runCollect(destroyer.Run(inv, options)) expEvents = []testutil.ExpEvent{ { diff --git a/test/e2e/depends_on_test.go b/test/e2e/depends_on_test.go index 3970fb7..5159b10 100644 --- a/test/e2e/depends_on_test.go +++ b/test/e2e/depends_on_test.go @@ -5,7 +5,7 @@ package e2e import ( "context" - "strings" + "fmt" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -18,7 +18,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -func dependsOnTest(_ client.Client, invConfig InventoryConfig, inventoryName, namespaceName string) { +func dependsOnTest(c client.Client, invConfig InventoryConfig, inventoryName, namespaceName string) { By("apply resources in order based on depends-on annotation") applier := invConfig.ApplierFactoryFunc() @@ -27,9 +27,9 @@ func dependsOnTest(_ client.Client, invConfig InventoryConfig, inventoryName, na // Dependency order: pod1 -> pod3 -> pod2 // Apply order: pod2, pod3, pod1 resources := []*unstructured.Unstructured{ - manifestToUnstructured(pod1), - manifestToUnstructured(pod2), - manifestToUnstructured(pod3), + withDependsOn(withNamespace(manifestToUnstructured(pod1), namespaceName), fmt.Sprintf("/namespaces/%s/Pod/pod3", namespaceName)), + withNamespace(manifestToUnstructured(pod2), namespaceName), + withDependsOn(withNamespace(manifestToUnstructured(pod3), namespaceName), fmt.Sprintf("/namespaces/%s/Pod/pod2", namespaceName)), } ch := applier.Run(context.TODO(), inv, resources, apply.Options{ @@ -38,7 +38,6 @@ func dependsOnTest(_ client.Client, invConfig InventoryConfig, inventoryName, na var applierEvents []event.Event for e := range ch { - Expect(e.Type).NotTo(Equal(event.ErrorType)) applierEvents = append(applierEvents, e) } expEvents := []testutil.ExpEvent{ @@ -79,7 +78,7 @@ func dependsOnTest(_ client.Client, invConfig InventoryConfig, inventoryName, na EventType: event.ApplyType, ApplyEvent: &testutil.ExpApplyEvent{ Operation: event.Created, - Identifier: object.UnstructuredToObjMetaOrDie(manifestToUnstructured(pod2)), + Identifier: object.UnstructuredToObjMetaOrDie(withNamespace(manifestToUnstructured(pod2), namespaceName)), Error: nil, }, }, @@ -124,7 +123,7 @@ func dependsOnTest(_ client.Client, invConfig InventoryConfig, inventoryName, na EventType: event.ApplyType, ApplyEvent: &testutil.ExpApplyEvent{ Operation: event.Created, - Identifier: object.UnstructuredToObjMetaOrDie(manifestToUnstructured(pod3)), + Identifier: object.UnstructuredToObjMetaOrDie(withNamespace(manifestToUnstructured(pod3), namespaceName)), Error: nil, }, }, @@ -169,7 +168,7 @@ func dependsOnTest(_ client.Client, invConfig InventoryConfig, inventoryName, na EventType: event.ApplyType, ApplyEvent: &testutil.ExpApplyEvent{ Operation: event.Created, - Identifier: object.UnstructuredToObjMetaOrDie(manifestToUnstructured(pod1)), + Identifier: object.UnstructuredToObjMetaOrDie(withNamespace(manifestToUnstructured(pod1), namespaceName)), Error: nil, }, }, @@ -221,10 +220,31 @@ func dependsOnTest(_ client.Client, invConfig InventoryConfig, inventoryName, na } Expect(testutil.EventsToExpEvents(applierEvents)).To(testutil.Equal(expEvents)) + By("verify pod1 created and ready") + result := assertUnstructuredExists(c, withNamespace(manifestToUnstructured(pod1), namespaceName)) + podIP, found, err := testutil.NestedField(result.Object, "status", "podIP") + Expect(err).NotTo(HaveOccurred()) + Expect(found).To(BeTrue()) + Expect(podIP).NotTo(BeEmpty()) // use podIP as proxy for readiness + + By("verify pod2 created and ready") + result = assertUnstructuredExists(c, withNamespace(manifestToUnstructured(pod2), namespaceName)) + podIP, found, err = testutil.NestedField(result.Object, "status", "podIP") + Expect(err).NotTo(HaveOccurred()) + Expect(found).To(BeTrue()) + Expect(podIP).NotTo(BeEmpty()) // use podIP as proxy for readiness + + By("verify pod3 created and ready") + result = assertUnstructuredExists(c, withNamespace(manifestToUnstructured(pod3), namespaceName)) + podIP, found, err = testutil.NestedField(result.Object, "status", "podIP") + Expect(err).NotTo(HaveOccurred()) + Expect(found).To(BeTrue()) + Expect(podIP).NotTo(BeEmpty()) // use podIP as proxy for readiness + By("destroy resources in opposite order") destroyer := invConfig.DestroyerFactoryFunc() options := apply.DestroyerOptions{InventoryPolicy: inventory.AdoptIfNoInventory} - destroyerEvents := runCollectNoErr(destroyer.Run(inv, options)) + destroyerEvents := runCollect(destroyer.Run(inv, options)) expEvents = []testutil.ExpEvent{ { @@ -246,7 +266,7 @@ func dependsOnTest(_ client.Client, invConfig InventoryConfig, inventoryName, na EventType: event.DeleteType, DeleteEvent: &testutil.ExpDeleteEvent{ Operation: event.Deleted, - Identifier: object.UnstructuredToObjMetaOrDie(manifestToUnstructured(pod1)), + Identifier: object.UnstructuredToObjMetaOrDie(withNamespace(manifestToUnstructured(pod1), namespaceName)), Error: nil, }, }, @@ -291,7 +311,7 @@ func dependsOnTest(_ client.Client, invConfig InventoryConfig, inventoryName, na EventType: event.DeleteType, DeleteEvent: &testutil.ExpDeleteEvent{ Operation: event.Deleted, - Identifier: object.UnstructuredToObjMetaOrDie(manifestToUnstructured(pod3)), + Identifier: object.UnstructuredToObjMetaOrDie(withNamespace(manifestToUnstructured(pod3), namespaceName)), Error: nil, }, }, @@ -336,7 +356,7 @@ func dependsOnTest(_ client.Client, invConfig InventoryConfig, inventoryName, na EventType: event.DeleteType, DeleteEvent: &testutil.ExpDeleteEvent{ Operation: event.Deleted, - Identifier: object.UnstructuredToObjMetaOrDie(manifestToUnstructured(pod2)), + Identifier: object.UnstructuredToObjMetaOrDie(withNamespace(manifestToUnstructured(pod2), namespaceName)), Error: nil, }, }, @@ -386,46 +406,14 @@ func dependsOnTest(_ client.Client, invConfig InventoryConfig, inventoryName, na }, }, } - Expect(testutil.EventsToExpEvents(destroyerEvents)).To(testutil.Equal(expEvents)) + + By("verify pod1 deleted") + assertUnstructuredDoesNotExist(c, withNamespace(manifestToUnstructured(pod1), namespaceName)) + + By("verify pod2 deleted") + assertUnstructuredDoesNotExist(c, withNamespace(manifestToUnstructured(pod2), namespaceName)) + + By("verify pod3 deleted") + assertUnstructuredDoesNotExist(c, withNamespace(manifestToUnstructured(pod3), namespaceName)) } - -var pod1 = []byte(strings.TrimSpace(` -kind: Pod -apiVersion: v1 -metadata: - name: pod1 - namespace: default - annotations: - config.kubernetes.io/depends-on: /namespaces/default/Pod/pod3 -spec: - containers: - - name: kubernetes-pause - image: k8s.gcr.io/pause:2.0 -`)) - -var pod2 = []byte(strings.TrimSpace(` -kind: Pod -apiVersion: v1 -metadata: - name: pod2 - namespace: default -spec: - containers: - - name: kubernetes-pause - image: k8s.gcr.io/pause:2.0 -`)) - -var pod3 = []byte(strings.TrimSpace(` -kind: Pod -apiVersion: v1 -metadata: - name: pod3 - namespace: default - annotations: - config.kubernetes.io/depends-on: /namespaces/default/Pod/pod2 -spec: - containers: - - name: kubernetes-pause - image: k8s.gcr.io/pause:2.0 -`)) diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 85fca5a..97c4b38 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -17,6 +17,7 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/types" "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/klog/v2" "k8s.io/kubectl/pkg/cmd/util" "k8s.io/kubectl/pkg/scheme" "sigs.k8s.io/cli-utils/pkg/apply" @@ -72,6 +73,15 @@ var inventoryConfigs = map[string]InventoryConfig{ }, } +// Parse optional logging flags +// Ex: ginkgo ./test/e2e/... -- -v=5 +// Allow init for e2e test (not imported by external code) +// nolint:gochecknoinits +func init() { + klog.InitFlags(nil) + klog.SetOutput(GinkgoWriter) +} + var _ = Describe("Applier", func() { var c client.Client @@ -116,10 +126,13 @@ var _ = Describe("Applier", func() { objs := []*unstructured.Unstructured{ manifestToUnstructured(cr), manifestToUnstructured(crd), - manifestToUnstructured(pod1), - manifestToUnstructured(pod2), - manifestToUnstructured(pod3), - deploymentManifest(namespace.GetName()), + withNamespace(manifestToUnstructured(pod1), namespace.GetName()), + withNamespace(manifestToUnstructured(pod2), namespace.GetName()), + withNamespace(manifestToUnstructured(pod3), namespace.GetName()), + withNamespace(manifestToUnstructured(podA), namespace.GetName()), + withNamespace(manifestToUnstructured(podB), namespace.GetName()), + withNamespace(manifestToUnstructured(deployment1), namespace.GetName()), + manifestToUnstructured(apiservice1), } for _, obj := range objs { deleteUnstructuredIfExists(c, obj) @@ -354,9 +367,3 @@ func newDestroyerFromInvFactory(invFactory inventory.InventoryClientFactory) *ap Expect(err).NotTo(HaveOccurred()) return d } - -// Delete the passed object from the cluster using the passed client. -func deleteObj(c client.Client, obj *unstructured.Unstructured) { - err := c.Delete(context.TODO(), obj) - Expect(err).NotTo(HaveOccurred()) -} diff --git a/test/e2e/inventory_policy_test.go b/test/e2e/inventory_policy_test.go index 866c97b..d8163a8 100644 --- a/test/e2e/inventory_policy_test.go +++ b/test/e2e/inventory_policy_test.go @@ -10,9 +10,7 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - appsv1 "k8s.io/api/apps/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/cli-utils/pkg/apply" "sigs.k8s.io/cli-utils/pkg/apply/event" "sigs.k8s.io/cli-utils/pkg/inventory" @@ -29,7 +27,7 @@ func inventoryPolicyMustMatchTest(c client.Client, invConfig InventoryConfig, na firstInvName := randomString("first-inv-") firstInv := invConfig.InvWrapperFunc(invConfig.InventoryFactoryFunc(firstInvName, namespaceName, firstInvName)) firstResources := []*unstructured.Unstructured{ - deploymentManifest(namespaceName), + withNamespace(manifestToUnstructured(deployment1), namespaceName), } runWithNoErr(applier.Run(context.TODO(), firstInv, firstResources, apply.Options{ @@ -41,7 +39,7 @@ func inventoryPolicyMustMatchTest(c client.Client, invConfig InventoryConfig, na secondInvName := randomString("second-inv-") secondInv := invConfig.InvWrapperFunc(invConfig.InventoryFactoryFunc(secondInvName, namespaceName, secondInvName)) secondResources := []*unstructured.Unstructured{ - updateReplicas(deploymentManifest(namespaceName), 6), + withReplicas(withNamespace(manifestToUnstructured(deployment1), namespaceName), 6), } ch := applier.Run(context.TODO(), secondInv, secondResources, apply.Options{ @@ -93,7 +91,7 @@ func inventoryPolicyMustMatchTest(c client.Client, invConfig InventoryConfig, na // ApplyTask error: resource managed by another inventory EventType: event.ApplyType, ApplyEvent: &testutil.ExpApplyEvent{ - Identifier: object.UnstructuredToObjMetaOrDie(deploymentManifest(namespaceName)), + Identifier: object.UnstructuredToObjMetaOrDie(withNamespace(manifestToUnstructured(deployment1), namespaceName)), Error: testutil.EqualErrorType( inventory.NewInventoryOverlapError(errors.New("test")), ), @@ -151,7 +149,7 @@ func inventoryPolicyMustMatchTest(c client.Client, invConfig InventoryConfig, na expected := testutil.ExpEvent{ EventType: event.StatusType, StatusEvent: &testutil.ExpStatusEvent{ - Identifier: object.UnstructuredToObjMetaOrDie(deploymentManifest(namespaceName)), + Identifier: object.UnstructuredToObjMetaOrDie(withNamespace(manifestToUnstructured(deployment1), namespaceName)), Status: status.InProgressStatus, Error: nil, }, @@ -162,7 +160,7 @@ func inventoryPolicyMustMatchTest(c client.Client, invConfig InventoryConfig, na expected = testutil.ExpEvent{ EventType: event.StatusType, StatusEvent: &testutil.ExpStatusEvent{ - Identifier: object.UnstructuredToObjMetaOrDie(deploymentManifest(namespaceName)), + Identifier: object.UnstructuredToObjMetaOrDie(withNamespace(manifestToUnstructured(deployment1), namespaceName)), Status: status.CurrentStatus, Error: nil, }, @@ -173,20 +171,18 @@ func inventoryPolicyMustMatchTest(c client.Client, invConfig InventoryConfig, na Expect(received).To(testutil.Equal(expEvents)) By("Verify resource wasn't updated") - var d appsv1.Deployment - err := c.Get(context.TODO(), types.NamespacedName{ - Namespace: namespaceName, - Name: deploymentManifest(namespaceName).GetName(), - }, &d) + result := assertUnstructuredExists(c, withNamespace(manifestToUnstructured(deployment1), namespaceName)) + replicas, found, err := testutil.NestedField(result.Object, "spec", "replicas") Expect(err).NotTo(HaveOccurred()) - Expect(d.Spec.Replicas).To(Equal(func(i int32) *int32 { return &i }(4))) + Expect(found).To(BeTrue()) + Expect(replicas).To(Equal(int64(4))) invConfig.InvCountVerifyFunc(c, namespaceName, 2) } func inventoryPolicyAdoptIfNoInventoryTest(c client.Client, invConfig InventoryConfig, namespaceName string) { By("Create unmanaged resource") - err := c.Create(context.TODO(), deploymentManifest(namespaceName)) + err := c.Create(context.TODO(), withNamespace(manifestToUnstructured(deployment1), namespaceName)) Expect(err).NotTo(HaveOccurred()) By("Apply resources") @@ -195,7 +191,7 @@ func inventoryPolicyAdoptIfNoInventoryTest(c client.Client, invConfig InventoryC invName := randomString("test-inv-") inv := invConfig.InvWrapperFunc(invConfig.InventoryFactoryFunc(invName, namespaceName, invName)) resources := []*unstructured.Unstructured{ - updateReplicas(deploymentManifest(namespaceName), 6), + withReplicas(withNamespace(manifestToUnstructured(deployment1), namespaceName), 6), } ch := applier.Run(context.TODO(), inv, resources, apply.Options{ @@ -248,7 +244,7 @@ func inventoryPolicyAdoptIfNoInventoryTest(c client.Client, invConfig InventoryC EventType: event.ApplyType, ApplyEvent: &testutil.ExpApplyEvent{ Operation: event.Configured, - Identifier: object.UnstructuredToObjMetaOrDie(deploymentManifest(namespaceName)), + Identifier: object.UnstructuredToObjMetaOrDie(withNamespace(manifestToUnstructured(deployment1), namespaceName)), Error: nil, }, }, @@ -304,7 +300,7 @@ func inventoryPolicyAdoptIfNoInventoryTest(c client.Client, invConfig InventoryC expected := testutil.ExpEvent{ EventType: event.StatusType, StatusEvent: &testutil.ExpStatusEvent{ - Identifier: object.UnstructuredToObjMetaOrDie(deploymentManifest(namespaceName)), + Identifier: object.UnstructuredToObjMetaOrDie(withNamespace(manifestToUnstructured(deployment1), namespaceName)), Status: status.InProgressStatus, Error: nil, }, @@ -315,7 +311,7 @@ func inventoryPolicyAdoptIfNoInventoryTest(c client.Client, invConfig InventoryC expected = testutil.ExpEvent{ EventType: event.StatusType, StatusEvent: &testutil.ExpStatusEvent{ - Identifier: object.UnstructuredToObjMetaOrDie(deploymentManifest(namespaceName)), + Identifier: object.UnstructuredToObjMetaOrDie(withNamespace(manifestToUnstructured(deployment1), namespaceName)), Status: status.CurrentStatus, Error: nil, }, @@ -326,14 +322,17 @@ func inventoryPolicyAdoptIfNoInventoryTest(c client.Client, invConfig InventoryC Expect(received).To(testutil.Equal(expEvents)) By("Verify resource was updated and added to inventory") - var d appsv1.Deployment - err = c.Get(context.TODO(), types.NamespacedName{ - Namespace: namespaceName, - Name: deploymentManifest(namespaceName).GetName(), - }, &d) + result := assertUnstructuredExists(c, withNamespace(manifestToUnstructured(deployment1), namespaceName)) + + replicas, found, err := testutil.NestedField(result.Object, "spec", "replicas") Expect(err).NotTo(HaveOccurred()) - Expect(d.Spec.Replicas).To(Equal(func(i int32) *int32 { return &i }(6))) - Expect(d.ObjectMeta.Annotations["config.k8s.io/owning-inventory"]).To(Equal(invName)) + Expect(found).To(BeTrue()) + Expect(replicas).To(Equal(int64(6))) + + value, found, err := testutil.NestedField(result.Object, "metadata", "annotations", "config.k8s.io/owning-inventory") + Expect(err).NotTo(HaveOccurred()) + Expect(found).To(BeTrue()) + Expect(value).To(Equal(invName)) invConfig.InvCountVerifyFunc(c, namespaceName, 1) invConfig.InvSizeVerifyFunc(c, invName, namespaceName, invName, 1) @@ -346,7 +345,7 @@ func inventoryPolicyAdoptAllTest(c client.Client, invConfig InventoryConfig, nam firstInvName := randomString("first-inv-") firstInv := invConfig.InvWrapperFunc(invConfig.InventoryFactoryFunc(firstInvName, namespaceName, firstInvName)) firstResources := []*unstructured.Unstructured{ - deploymentManifest(namespaceName), + withNamespace(manifestToUnstructured(deployment1), namespaceName), } runWithNoErr(applier.Run(context.TODO(), firstInv, firstResources, apply.Options{ @@ -358,7 +357,7 @@ func inventoryPolicyAdoptAllTest(c client.Client, invConfig InventoryConfig, nam secondInvName := randomString("test-inv-") secondInv := invConfig.InvWrapperFunc(invConfig.InventoryFactoryFunc(secondInvName, namespaceName, secondInvName)) secondResources := []*unstructured.Unstructured{ - updateReplicas(deploymentManifest(namespaceName), 6), + withReplicas(withNamespace(manifestToUnstructured(deployment1), namespaceName), 6), } ch := applier.Run(context.TODO(), secondInv, secondResources, apply.Options{ @@ -411,7 +410,7 @@ func inventoryPolicyAdoptAllTest(c client.Client, invConfig InventoryConfig, nam EventType: event.ApplyType, ApplyEvent: &testutil.ExpApplyEvent{ Operation: event.Configured, - Identifier: object.UnstructuredToObjMetaOrDie(deploymentManifest(namespaceName)), + Identifier: object.UnstructuredToObjMetaOrDie(withNamespace(manifestToUnstructured(deployment1), namespaceName)), Error: nil, }, }, @@ -467,7 +466,7 @@ func inventoryPolicyAdoptAllTest(c client.Client, invConfig InventoryConfig, nam expected := testutil.ExpEvent{ EventType: event.StatusType, StatusEvent: &testutil.ExpStatusEvent{ - Identifier: object.UnstructuredToObjMetaOrDie(deploymentManifest(namespaceName)), + Identifier: object.UnstructuredToObjMetaOrDie(withNamespace(manifestToUnstructured(deployment1), namespaceName)), Status: status.InProgressStatus, Error: nil, }, @@ -478,7 +477,7 @@ func inventoryPolicyAdoptAllTest(c client.Client, invConfig InventoryConfig, nam expected = testutil.ExpEvent{ EventType: event.StatusType, StatusEvent: &testutil.ExpStatusEvent{ - Identifier: object.UnstructuredToObjMetaOrDie(deploymentManifest(namespaceName)), + Identifier: object.UnstructuredToObjMetaOrDie(withNamespace(manifestToUnstructured(deployment1), namespaceName)), Status: status.CurrentStatus, Error: nil, }, @@ -489,14 +488,17 @@ func inventoryPolicyAdoptAllTest(c client.Client, invConfig InventoryConfig, nam Expect(received).To(testutil.Equal(expEvents)) By("Verify resource was updated and added to inventory") - var d appsv1.Deployment - err := c.Get(context.TODO(), types.NamespacedName{ - Namespace: namespaceName, - Name: deploymentManifest(namespaceName).GetName(), - }, &d) + result := assertUnstructuredExists(c, withNamespace(manifestToUnstructured(deployment1), namespaceName)) + + replicas, found, err := testutil.NestedField(result.Object, "spec", "replicas") Expect(err).NotTo(HaveOccurred()) - Expect(d.Spec.Replicas).To(Equal(func(i int32) *int32 { return &i }(6))) - Expect(d.ObjectMeta.Annotations["config.k8s.io/owning-inventory"]).To(Equal(secondInvName)) + Expect(found).To(BeTrue()) + Expect(replicas).To(Equal(int64(6))) + + value, found, err := testutil.NestedField(result.Object, "metadata", "annotations", "config.k8s.io/owning-inventory") + Expect(err).NotTo(HaveOccurred()) + Expect(found).To(BeTrue()) + Expect(value).To(Equal(secondInvName)) invConfig.InvCountVerifyFunc(c, namespaceName, 2) } diff --git a/test/e2e/mutation_test.go b/test/e2e/mutation_test.go index 85e79e0..7faa538 100644 --- a/test/e2e/mutation_test.go +++ b/test/e2e/mutation_test.go @@ -6,15 +6,10 @@ package e2e import ( "context" "fmt" - "strings" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - v1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/cli-utils/pkg/apply" "sigs.k8s.io/cli-utils/pkg/apply/event" "sigs.k8s.io/cli-utils/pkg/inventory" @@ -49,17 +44,12 @@ func mutationTest(c client.Client, invConfig InventoryConfig, inventoryName, nam withNamespace(manifestToUnstructured(podB), namespaceName), } - for _, obj := range resources { - obj.SetNamespace(namespaceName) - } - ch := applier.Run(context.TODO(), inv, resources, apply.Options{ EmitStatusEvents: false, }) var applierEvents []event.Event for e := range ch { - Expect(e.Type).NotTo(Equal(event.ErrorType)) applierEvents = append(applierEvents, e) } expEvents := []testutil.ExpEvent{ @@ -197,29 +187,38 @@ func mutationTest(c client.Client, invConfig InventoryConfig, inventoryName, nam } Expect(testutil.EventsToExpEvents(applierEvents)).To(testutil.Equal(expEvents)) - By("verify resource was mutated") - var podBObj v1.Pod - err := c.Get(context.TODO(), types.NamespacedName{ - Namespace: namespaceName, - Name: manifestToUnstructured(podB).GetName(), - }, &podBObj) - Expect(err).NotTo(HaveOccurred()) - Expect(podBObj.Status.PodIP).NotTo(BeEmpty()) - Expect(podBObj.Spec.Containers[0].Ports[0].ContainerPort).To(Equal(int32(80))) - host := fmt.Sprintf("%s:%d", podBObj.Status.PodIP, podBObj.Spec.Containers[0].Ports[0].ContainerPort) + By("verify podB is created and ready") + result := assertUnstructuredExists(c, withNamespace(manifestToUnstructured(podB), namespaceName)) - var podAObj v1.Pod - err = c.Get(context.TODO(), types.NamespacedName{ - Namespace: namespaceName, - Name: manifestToUnstructured(podA).GetName(), - }, &podAObj) + podIP, found, err := testutil.NestedField(result.Object, "status", "podIP") Expect(err).NotTo(HaveOccurred()) - Expect(podAObj.Spec.Containers[0].Env[0].Value).To(Equal(host)) + Expect(found).To(BeTrue()) + Expect(podIP).NotTo(BeEmpty()) // use podIP as proxy for readiness + + containerPort, found, err := testutil.NestedField(result.Object, "spec", "containers", 0, "ports", 0, "containerPort") + Expect(err).NotTo(HaveOccurred()) + Expect(found).To(BeTrue()) + Expect(containerPort).To(Equal(int64(80))) + + host := fmt.Sprintf("%s:%d", podIP, containerPort) + + By("verify podA is mutated, created, and ready") + result = assertUnstructuredExists(c, withNamespace(manifestToUnstructured(podA), namespaceName)) + + podIP, found, err = testutil.NestedField(result.Object, "status", "podIP") + Expect(err).NotTo(HaveOccurred()) + Expect(found).To(BeTrue()) + Expect(podIP).NotTo(BeEmpty()) // use podIP as proxy for readiness + + envValue, found, err := testutil.NestedField(result.Object, "spec", "containers", 0, "env", 0, "value") + Expect(err).NotTo(HaveOccurred()) + Expect(found).To(BeTrue()) + Expect(envValue).To(Equal(host)) By("destroy resources in opposite order") destroyer := invConfig.DestroyerFactoryFunc() options := apply.DestroyerOptions{InventoryPolicy: inventory.AdoptIfNoInventory} - destroyerEvents := runCollectNoErr(destroyer.Run(inv, options)) + destroyerEvents := runCollect(destroyer.Run(inv, options)) expEvents = []testutil.ExpEvent{ { @@ -339,70 +338,9 @@ func mutationTest(c client.Client, invConfig InventoryConfig, inventoryName, nam Expect(testutil.EventsToExpEvents(destroyerEvents)).To(testutil.Equal(expEvents)) - By("verify resources deleted") - err = c.Get(context.TODO(), types.NamespacedName{ - Namespace: namespaceName, - Name: manifestToUnstructured(podB).GetName(), - }, &podBObj) - Expect(err).To(HaveOccurred()) - Expect(apierrors.ReasonForError(err)).To(Equal(metav1.StatusReasonNotFound)) + By("verify podB deleted") + assertUnstructuredDoesNotExist(c, withNamespace(manifestToUnstructured(podB), namespaceName)) - err = c.Get(context.TODO(), types.NamespacedName{ - Namespace: namespaceName, - Name: manifestToUnstructured(podA).GetName(), - }, &podAObj) - Expect(err).To(HaveOccurred()) - Expect(apierrors.ReasonForError(err)).To(Equal(metav1.StatusReasonNotFound)) + By("verify podA deleted") + assertUnstructuredDoesNotExist(c, withNamespace(manifestToUnstructured(podA), namespaceName)) } - -func withNamespace(obj *unstructured.Unstructured, namespace string) *unstructured.Unstructured { - obj.SetNamespace(namespace) - return obj -} - -var podA = []byte(strings.TrimSpace(` -kind: Pod -apiVersion: v1 -metadata: - name: pod-a - namespace: test - annotations: - config.kubernetes.io/apply-time-mutation: | - - sourceRef: - kind: Pod - name: pod-b - sourcePath: $.status.podIP - targetPath: $.spec.containers[?(@.name=="nginx")].env[?(@.name=="SERVICE_HOST")].value - token: ${pob-b-ip} - - sourceRef: - kind: Pod - name: pod-b - sourcePath: $.spec.containers[?(@.name=="nginx")].ports[?(@.name=="tcp")].containerPort - targetPath: $.spec.containers[?(@.name=="nginx")].env[?(@.name=="SERVICE_HOST")].value - token: ${pob-b-port} -spec: - containers: - - name: nginx - image: nginx:1.21 - ports: - - name: tcp - containerPort: 80 - env: - - name: SERVICE_HOST - value: "${pob-b-ip}:${pob-b-port}" -`)) - -var podB = []byte(strings.TrimSpace(` -kind: Pod -apiVersion: v1 -metadata: - name: pod-b - namespace: test -spec: - containers: - - name: nginx - image: nginx:1.21 - ports: - - name: tcp - containerPort: 80 -`)) diff --git a/test/e2e/name_inv_strategy_test.go b/test/e2e/name_inv_strategy_test.go index 0a12082..bd8044f 100644 --- a/test/e2e/name_inv_strategy_test.go +++ b/test/e2e/name_inv_strategy_test.go @@ -23,7 +23,7 @@ func applyWithExistingInvTest(c client.Client, invConfig InventoryConfig, invent orgApplyInv := invConfig.InvWrapperFunc(invConfig.InventoryFactoryFunc(inventoryName, namespaceName, orgInventoryID)) resources := []*unstructured.Unstructured{ - deploymentManifest(namespaceName), + withNamespace(manifestToUnstructured(deployment1), namespaceName), } runWithNoErr(applier.Run(context.TODO(), orgApplyInv, resources, apply.Options{ diff --git a/test/e2e/prune_retrieve_error_test.go b/test/e2e/prune_retrieve_error_test.go index da9cc70..6a623b9 100644 --- a/test/e2e/prune_retrieve_error_test.go +++ b/test/e2e/prune_retrieve_error_test.go @@ -27,7 +27,7 @@ func pruneRetrieveErrorTest(c client.Client, invConfig InventoryConfig, inventor inv := createInventoryInfo(invConfig, inventoryName, namespaceName, inventoryID) resource1 := []*unstructured.Unstructured{ - manifestToUnstructured(pod1), + withNamespace(manifestToUnstructured(pod1), namespaceName), } ch := applier.Run(context.TODO(), inv, resource1, apply.Options{ @@ -36,29 +36,105 @@ func pruneRetrieveErrorTest(c client.Client, invConfig InventoryConfig, inventor var applierEvents []event.Event for e := range ch { - Expect(e.Type).NotTo(Equal(event.ErrorType)) applierEvents = append(applierEvents, e) } - err := testutil.VerifyEvents([]testutil.ExpEvent{ + expEvents := []testutil.ExpEvent{ { - // Pod1 is applied + // InitTask + EventType: event.InitType, + InitEvent: &testutil.ExpInitEvent{}, + }, + { + // InvAddTask start + EventType: event.ActionGroupType, + ActionGroupEvent: &testutil.ExpActionGroupEvent{ + Action: event.InventoryAction, + Name: "inventory-add-0", + Type: event.Started, + }, + }, + { + // InvAddTask finished + EventType: event.ActionGroupType, + ActionGroupEvent: &testutil.ExpActionGroupEvent{ + Action: event.InventoryAction, + Name: "inventory-add-0", + Type: event.Finished, + }, + }, + { + // ApplyTask start + EventType: event.ActionGroupType, + ActionGroupEvent: &testutil.ExpActionGroupEvent{ + Action: event.ApplyAction, + Name: "apply-0", + Type: event.Started, + }, + }, + { + // Create deployment EventType: event.ApplyType, ApplyEvent: &testutil.ExpApplyEvent{ - Identifier: object.UnstructuredToObjMetaOrDie(manifestToUnstructured(pod1)), Operation: event.Created, + Identifier: object.UnstructuredToObjMetaOrDie(withNamespace(manifestToUnstructured(pod1), namespaceName)), Error: nil, }, }, { // ApplyTask finished EventType: event.ActionGroupType, + ActionGroupEvent: &testutil.ExpActionGroupEvent{ + Action: event.ApplyAction, + Name: "apply-0", + Type: event.Finished, + }, }, - }, applierEvents) - Expect(err).ToNot(HaveOccurred()) + // TODO: Why no waiting??? + // { + // // WaitTask start + // EventType: event.ActionGroupType, + // ActionGroupEvent: &testutil.ExpActionGroupEvent{ + // Action: event.WaitAction, + // Name: "wait-0", + // Type: event.Started, + // }, + // }, + // { + // // WaitTask finished + // EventType: event.ActionGroupType, + // ActionGroupEvent: &testutil.ExpActionGroupEvent{ + // Action: event.WaitAction, + // Name: "wait-0", + // Type: event.Finished, + // }, + // }, + { + // InvSetTask start + EventType: event.ActionGroupType, + ActionGroupEvent: &testutil.ExpActionGroupEvent{ + Action: event.InventoryAction, + Name: "inventory-set-0", + Type: event.Started, + }, + }, + { + // InvSetTask finished + EventType: event.ActionGroupType, + ActionGroupEvent: &testutil.ExpActionGroupEvent{ + Action: event.InventoryAction, + Name: "inventory-set-0", + Type: event.Finished, + }, + }, + } + Expect(testutil.EventsToExpEvents(applierEvents)).To(testutil.Equal(expEvents)) + + By("Verify pod1 created") + assertUnstructuredExists(c, withNamespace(manifestToUnstructured(pod1), namespaceName)) // Delete the previously applied resource, which is referenced in the inventory. By("delete resource, which is referenced in the inventory") - deleteObj(c, resource1[0]) + deleteUnstructuredAndWait(c, withNamespace(manifestToUnstructured(pod1), namespaceName)) By("Verify inventory") // The inventory should still have the previously deleted item. @@ -66,7 +142,7 @@ func pruneRetrieveErrorTest(c client.Client, invConfig InventoryConfig, inventor By("apply a different resource, and validate the inventory accurately reflects only this object") resource2 := []*unstructured.Unstructured{ - manifestToUnstructured(pod2), + withNamespace(manifestToUnstructured(pod2), namespaceName), } ch = applier.Run(context.TODO(), inv, resource2, apply.Options{ @@ -75,25 +151,106 @@ func pruneRetrieveErrorTest(c client.Client, invConfig InventoryConfig, inventor var applierEvents2 []event.Event for e := range ch { - Expect(e.Type).NotTo(Equal(event.ErrorType)) applierEvents2 = append(applierEvents2, e) } - err = testutil.VerifyEvents([]testutil.ExpEvent{ + expEvents2 := []testutil.ExpEvent{ { - // Pod2 is applied + // InitTask + EventType: event.InitType, + InitEvent: &testutil.ExpInitEvent{}, + }, + { + // InvAddTask start + EventType: event.ActionGroupType, + ActionGroupEvent: &testutil.ExpActionGroupEvent{ + Action: event.InventoryAction, + Name: "inventory-add-0", + Type: event.Started, + }, + }, + { + // InvAddTask finished + EventType: event.ActionGroupType, + ActionGroupEvent: &testutil.ExpActionGroupEvent{ + Action: event.InventoryAction, + Name: "inventory-add-0", + Type: event.Finished, + }, + }, + { + // ApplyTask start + EventType: event.ActionGroupType, + ActionGroupEvent: &testutil.ExpActionGroupEvent{ + Action: event.ApplyAction, + Name: "apply-0", + Type: event.Started, + }, + }, + { + // Create pod2 EventType: event.ApplyType, ApplyEvent: &testutil.ExpApplyEvent{ - Identifier: object.UnstructuredToObjMetaOrDie(manifestToUnstructured(pod2)), Operation: event.Created, + Identifier: object.UnstructuredToObjMetaOrDie(withNamespace(manifestToUnstructured(pod2), namespaceName)), Error: nil, }, }, { // ApplyTask finished EventType: event.ActionGroupType, + ActionGroupEvent: &testutil.ExpActionGroupEvent{ + Action: event.ApplyAction, + Name: "apply-0", + Type: event.Finished, + }, }, - }, applierEvents2) - Expect(err).ToNot(HaveOccurred()) + // Don't prune pod1, it should already be deleted. + // TODO: Why is waiting skipped on create? + // { + // // WaitTask start + // EventType: event.ActionGroupType, + // ActionGroupEvent: &testutil.ExpActionGroupEvent{ + // Action: event.WaitAction, + // Name: "wait-0", + // Type: event.Started, + // }, + // }, + // { + // // WaitTask finished + // EventType: event.ActionGroupType, + // ActionGroupEvent: &testutil.ExpActionGroupEvent{ + // Action: event.WaitAction, + // Name: "wait-0", + // Type: event.Finished, + // }, + // }, + { + // InvSetTask start + EventType: event.ActionGroupType, + ActionGroupEvent: &testutil.ExpActionGroupEvent{ + Action: event.InventoryAction, + Name: "inventory-set-0", + Type: event.Started, + }, + }, + { + // InvSetTask finished + EventType: event.ActionGroupType, + ActionGroupEvent: &testutil.ExpActionGroupEvent{ + Action: event.InventoryAction, + Name: "inventory-set-0", + Type: event.Finished, + }, + }, + } + Expect(testutil.EventsToExpEvents(applierEvents2)).To(testutil.Equal(expEvents2)) + + By("Wait for pod2 to be created") + // TODO: change behavior so the user doesn't need to code their own wait + waitForCreation(c, withNamespace(manifestToUnstructured(pod2), namespaceName)) + + By("Verify pod1 still deleted") + assertUnstructuredDoesNotExist(c, withNamespace(manifestToUnstructured(pod1), namespaceName)) By("Verify inventory") // The inventory should only have the currently applied item. @@ -102,13 +259,87 @@ func pruneRetrieveErrorTest(c client.Client, invConfig InventoryConfig, inventor By("Destroy resources") destroyer := invConfig.DestroyerFactoryFunc() - destroyInv := createInventoryInfo(invConfig, inventoryName, namespaceName, inventoryID) options := apply.DestroyerOptions{InventoryPolicy: inventory.AdoptIfNoInventory} - destroyerEvents := runCollectNoErr(destroyer.Run(destroyInv, options)) - err = testutil.VerifyEvents([]testutil.ExpEvent{ + destroyerEvents := runCollect(destroyer.Run(inv, options)) + + expEvents3 := []testutil.ExpEvent{ { - EventType: event.DeleteType, + // InitTask + EventType: event.InitType, + InitEvent: &testutil.ExpInitEvent{}, }, - }, destroyerEvents) - Expect(err).ToNot(HaveOccurred()) + { + // PruneTask start + EventType: event.ActionGroupType, + ActionGroupEvent: &testutil.ExpActionGroupEvent{ + Action: event.DeleteAction, + Name: "prune-0", + Type: event.Started, + }, + }, + { + // Delete pod2 + EventType: event.DeleteType, + DeleteEvent: &testutil.ExpDeleteEvent{ + // TODO: this delete is flakey (sometimes skipped), because there's no WaitTask after creation + Operation: event.Deleted, + Identifier: object.UnstructuredToObjMetaOrDie(withNamespace(manifestToUnstructured(pod2), namespaceName)), + Error: nil, + }, + }, + { + // PruneTask finished + EventType: event.ActionGroupType, + ActionGroupEvent: &testutil.ExpActionGroupEvent{ + Action: event.DeleteAction, + Name: "prune-0", + Type: event.Finished, + }, + }, + // TODO: Why is waiting skipped on destroy? + // { + // // WaitTask start + // EventType: event.ActionGroupType, + // ActionGroupEvent: &testutil.ExpActionGroupEvent{ + // Action: event.WaitAction, + // Name: "wait-0", + // Type: event.Started, + // }, + // }, + // { + // // WaitTask finished + // EventType: event.ActionGroupType, + // ActionGroupEvent: &testutil.ExpActionGroupEvent{ + // Action: event.WaitAction, + // Name: "wait-0", + // Type: event.Finished, + // }, + // }, + { + // DeleteInvTask start + EventType: event.ActionGroupType, + ActionGroupEvent: &testutil.ExpActionGroupEvent{ + Action: event.InventoryAction, + Name: "delete-inventory-0", + Type: event.Started, + }, + }, + { + // DeleteInvTask finished + EventType: event.ActionGroupType, + ActionGroupEvent: &testutil.ExpActionGroupEvent{ + Action: event.InventoryAction, + Name: "delete-inventory-0", + Type: event.Finished, + }, + }, + } + Expect(testutil.EventsToExpEvents(destroyerEvents)).To(testutil.Equal(expEvents3)) + + By("Verify pod1 is deleted") + assertUnstructuredDoesNotExist(c, withNamespace(manifestToUnstructured(pod1), namespaceName)) + + By("Wait for pod2 to be deleted") + // TODO: change behavior so the user doesn't need to code their own wait + waitForDeletion(c, withNamespace(manifestToUnstructured(pod2), namespaceName)) } diff --git a/test/e2e/serverside_apply_test.go b/test/e2e/serverside_apply_test.go index 7b2bbab..8a050b7 100644 --- a/test/e2e/serverside_apply_test.go +++ b/test/e2e/serverside_apply_test.go @@ -9,13 +9,11 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/cli-utils/pkg/apply" "sigs.k8s.io/cli-utils/pkg/common" + "sigs.k8s.io/cli-utils/pkg/testutil" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -25,8 +23,8 @@ func serversideApplyTest(c client.Client, invConfig InventoryConfig, inventoryNa inv := invConfig.InvWrapperFunc(invConfig.InventoryFactoryFunc(inventoryName, namespaceName, "test")) firstResources := []*unstructured.Unstructured{ - deploymentManifest(namespaceName), - apiserviceManifest(), + withNamespace(manifestToUnstructured(deployment1), namespaceName), + manifestToUnstructured(apiservice1), } runWithNoErr(applier.Run(context.TODO(), inv, firstResources, apply.Options{ @@ -40,34 +38,28 @@ func serversideApplyTest(c client.Client, invConfig InventoryConfig, inventoryNa })) By("Verify deployment is server-side applied") - var d appsv1.Deployment - err := c.Get(context.TODO(), types.NamespacedName{ - Namespace: namespaceName, - Name: deploymentManifest(namespaceName).GetName(), - }, &d) - Expect(err).NotTo(HaveOccurred()) - _, found := d.ObjectMeta.Annotations[v1.LastAppliedConfigAnnotation] + result := assertUnstructuredExists(c, withNamespace(manifestToUnstructured(deployment1), namespaceName)) + // LastAppliedConfigAnnotation annotation is only set for client-side apply and we've server-side applied here. + _, found, err := testutil.NestedField(result.Object, "metadata", "annotations", v1.LastAppliedConfigAnnotation) + Expect(err).NotTo(HaveOccurred()) Expect(found).To(BeFalse()) - fields := d.GetManagedFields() - Expect(fields[0].Manager).To(Equal("test")) + + manager, found, err := testutil.NestedField(result.Object, "metadata", "managedFields", 0, "manager") + Expect(err).NotTo(HaveOccurred()) + Expect(found).To(BeTrue()) + Expect(manager).To(Equal("test")) By("Verify APIService is server-side applied") - var apiService = &unstructured.Unstructured{} - apiService.SetGroupVersionKind( - schema.GroupVersionKind{ - Group: "apiregistration.k8s.io", - Version: "v1", - Kind: "APIService", - }, - ) - err = c.Get(context.TODO(), types.NamespacedName{ - Name: "v1beta1.custom.metrics.k8s.io", - }, apiService) - Expect(err).NotTo(HaveOccurred()) - _, found2 := apiService.GetAnnotations()[v1.LastAppliedConfigAnnotation] + result = assertUnstructuredExists(c, manifestToUnstructured(apiservice1)) + // LastAppliedConfigAnnotation annotation is only set for client-side apply and we've server-side applied here. - Expect(found2).To(BeFalse()) - fields2 := apiService.GetManagedFields() - Expect(fields2[0].Manager).To(Equal("test")) + _, found, err = testutil.NestedField(result.Object, "metadata", "annotations", v1.LastAppliedConfigAnnotation) + Expect(err).NotTo(HaveOccurred()) + Expect(found).To(BeFalse()) + + manager, found, err = testutil.NestedField(result.Object, "metadata", "managedFields", 0, "manager") + Expect(err).NotTo(HaveOccurred()) + Expect(found).To(BeTrue()) + Expect(manager).To(Equal("test")) }