diff --git a/test/e2e/apps/workloadspread.go b/test/e2e/apps/workloadspread.go index 8bde5fa60..8d292494e 100644 --- a/test/e2e/apps/workloadspread.go +++ b/test/e2e/apps/workloadspread.go @@ -20,6 +20,7 @@ import ( "context" "encoding/json" "fmt" + "sort" "strconv" "time" @@ -30,7 +31,6 @@ import ( schedulingv1 "k8s.io/api/scheduling/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" @@ -48,7 +48,6 @@ var ( KruiseKindCloneSet = appsv1alpha1.SchemeGroupVersion.WithKind("CloneSet") KruiseKindStatefulSet = appsv1alpha1.SchemeGroupVersion.WithKind("StatefulSet") controllerKindDep = appsv1.SchemeGroupVersion.WithKind("Deployment") - //controllerKindJob = batchv1.SchemeGroupVersion.WithKind("Job") ) var _ = SIGDescribe("workloadspread", func() { @@ -104,6 +103,9 @@ var _ = SIGDescribe("workloadspread", func() { } workers = append(workers, &node) } + sort.Slice(workers, func(i, j int) bool { + return workers[i].Name < workers[j].Name + }) gomega.Expect(len(workers) > 2).Should(gomega.Equal(true)) // subset-a worker0 := workers[0] @@ -115,22 +117,6 @@ var _ = SIGDescribe("workloadspread", func() { tester.SetNodeLabel(c, worker2, WorkloadSpreadFakeZoneKey, "zone-b") }) - f.AfterEachActions = []func(){ - func() { - nodeList, err := c.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - patchBody := fmt.Sprintf(`{"metadata":{"labels":{"%s":null}}}`, WorkloadSpreadFakeZoneKey) - for i := range nodeList.Items { - node := nodeList.Items[i] - if _, exist := node.GetLabels()[WorkloadSpreadFakeZoneKey]; !exist { - continue - } - _, err = c.CoreV1().Nodes().Patch(context.TODO(), node.Name, types.StrategicMergePatchType, []byte(patchBody), metav1.PatchOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - }, - } - framework.KruiseDescribe("WorkloadSpread functionality", func() { ginkgo.AfterEach(func() { if ginkgo.CurrentGinkgoTestDescription().Failed { diff --git a/test/e2e/e2e.go b/test/e2e/e2e.go index 8b930b387..37ed08d28 100644 --- a/test/e2e/e2e.go +++ b/test/e2e/e2e.go @@ -1,6 +1,5 @@ /* Copyright 2019 The Kruise Authors. -Copyright 2015 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -30,96 +29,6 @@ import ( runtimeutils "k8s.io/apimachinery/pkg/util/runtime" ) -//var ( -// cloudConfig = &framework.TestContext.CloudConfig -//) -// -//// There are certain operations we only want to run once per overall test invocation -//// (such as deleting old namespaces, or verifying that all system pods are running. -//// Because of the way Ginkgo runs tests in parallel, we must use SynchronizedBeforeSuite -//// to ensure that these operations only run on the first parallel Ginkgo node. -//// -//// This function takes two parameters: one function which runs on only the first Ginkgo node, -//// returning an opaque byte array, and then a second function which runs on all Ginkgo nodes, -//// accepting the byte array. -//var _ = ginkgo.SynchronizedBeforeSuite(func() []byte { -// // Run only on Ginkgo node 1 -// -// switch framework.TestContext.Provider { -// case "gce", "gke": -// framework.LogClusterImageSources() -// } -// -// c, err := framework.LoadClientset() -// if err != nil { -// klog.Fatal("Error loading client: ", err) -// } -// -// // Delete any namespaces except those created by the system. This ensures no -// // lingering resources are left over from a previous test run. -// if framework.TestContext.CleanStart { -// deleted, err := framework.DeleteNamespaces(c, nil, /* deleteFilter */ -// []string{ -// metav1.NamespaceSystem, -// metav1.NamespaceDefault, -// metav1.NamespacePublic, -// }) -// if err != nil { -// framework.Failf("Error deleting orphaned namespaces: %v", err) -// } -// klog.Infof("Waiting for deletion of the following namespaces: %v", deleted) -// if err := framework.WaitForNamespacesDeleted(c, deleted, framework.NamespaceCleanupTimeout); err != nil { -// framework.Failf("Failed to delete orphaned namespaces %v: %v", deleted, err) -// } -// } -// -// // In large clusters we may get to this point but still have a bunch -// // of nodes without Routes created. Since this would make a node -// // unschedulable, we need to wait until all of them are schedulable. -// framework.ExpectNoError(framework.WaitForAllNodesSchedulable(c, framework.TestContext.NodeSchedulableTimeout)) -// -// // Ensure all pods are running and ready before starting tests (otherwise, -// // cluster infrastructure pods that are being pulled or started can block -// // test pods from running, and tests that ensure all pods are running and -// // ready will fail). -// podStartupTimeout := framework.TestContext.SystemPodsStartupTimeout -// // TODO: In large clusters, we often observe a non-starting pods due to -// // #41007. To avoid those pods preventing the whole test runs (and just -// // wasting the whole run), we allow for some not-ready pods (with the -// // number equal to the number of allowed not-ready nodes). -// if err := framework.WaitForPodsRunningReady(c, metav1.NamespaceSystem, int32(framework.TestContext.MinStartupPods), int32(framework.TestContext.AllowedNotReadyNodes), podStartupTimeout, map[string]string{}); err != nil { -// framework.DumpAllNamespaceInfo(c, metav1.NamespaceSystem) -// framework.LogFailedContainers(c, metav1.NamespaceSystem, framework.Logf) -// runKubernetesServiceTestContainer(c, metav1.NamespaceDefault) -// framework.Failf("Error waiting for all pods to be running and ready: %v", err) -// } -// -// if err := framework.WaitForDaemonSets(c, metav1.NamespaceSystem, int32(framework.TestContext.AllowedNotReadyNodes), framework.TestContext.SystemDaemonsetStartupTimeout); err != nil { -// framework.Logf("WARNING: Waiting for all daemonsets to be ready failed: %v", err) -// } -// -// // Log the version of the server and this client. -// framework.Logf("e2e test version: %s", version.Get().GitVersion) -// -// dc := c.DiscoveryClient -// -// serverVersion, serverErr := dc.ServerVersion() -// if serverErr != nil { -// framework.Logf("Unexpected server error retrieving version: %v", serverErr) -// } -// if serverVersion != nil { -// framework.Logf("kube-apiserver version: %s", serverVersion.GitVersion) -// } -// -// // Reference common test to make the import valid. -// commontest.CurrentSuite = commontest.E2E -// -// return nil -// -//}, func(data []byte) { -// // Run on all Ginkgo nodes -//}) - // Similar to SynchronizedBeforeSuite, we want to run some operations only once (such as collecting cluster logs). // Here, the order of functions is reversed; first, the function which runs everywhere, // and then the function that only runs on the first Ginkgo node. @@ -130,45 +39,8 @@ var _ = ginkgo.SynchronizedAfterSuite(func() { }, func() { // Run only Ginkgo on node 1 framework.Logf("Running AfterSuite actions on node 1") - //if framework.TestContext.GatherSuiteMetricsAfterTest { - // if err := gatherTestSuiteMetrics(); err != nil { - // framework.Logf("Error gathering metrics: %v", err) - // } - //} }) -//func gatherTestSuiteMetrics() error { -// framework.Logf("Gathering metrics") -// c, err := framework.LoadClientset() -// if err != nil { -// return fmt.Errorf("error loading client: %v", err) -// } -// -// // Grab metrics for apiserver, scheduler, controller-manager, kubelet (for non-kubemark case) and cluster autoscaler (optionally). -// grabber, err := metrics.NewMetricsGrabber(c, nil, !framework.ProviderIs("kubemark"), true, true, true, framework.TestContext.IncludeClusterAutoscalerMetrics) -// if err != nil { -// return fmt.Errorf("failed to create MetricsGrabber: %v", err) -// } -// -// received, err := grabber.Grab() -// if err != nil { -// return fmt.Errorf("failed to grab metrics: %v", err) -// } -// -// metricsForE2E := (*framework.MetricsForE2E)(&received) -// metricsJSON := metricsForE2E.PrintJSON() -// if framework.TestContext.ReportDir != "" { -// filePath := path.Join(framework.TestContext.ReportDir, "MetricsForE2ESuite_"+time.Now().Format(time.RFC3339)+".json") -// if err := ioutil.WriteFile(filePath, []byte(metricsJSON), 0644); err != nil { -// return fmt.Errorf("error writing to %q: %v", filePath, err) -// } -// } else { -// framework.Logf("\n\nTest Suite Metrics:\n%s\n", metricsJSON) -// } -// -// return nil -//} - // RunE2ETests checks configuration parameters (specified through flags) and then runs // E2E tests using the Ginkgo runner. // If a "report directory" is specified, one or more JUnit test reports will be @@ -176,8 +48,6 @@ var _ = ginkgo.SynchronizedAfterSuite(func() { // This function is called on each Ginkgo node in parallel mode. func RunE2ETests(t *testing.T) { runtimeutils.ReallyCrash = true - //logs.InitLogs() - //defer logs.FlushLogs() gomega.RegisterFailHandler(ginkgo.Fail) // Disable skipped tests unless they are explicitly requested. @@ -191,36 +61,3 @@ func RunE2ETests(t *testing.T) { ginkgo.RunSpecsWithDefaultAndCustomReporters(t, "Kruise e2e suite", r) } - -//// Run a test container to try and contact the Kubernetes api-server from a pod, wait for it -//// to flip to Ready, log its output and delete it. -//func runKubernetesServiceTestContainer(c clientset.Interface, ns string) { -// path := "test/images/clusterapi-tester/pod.yaml" -// framework.Logf("Parsing pod from %v", path) -// p, err := manifest.PodFromManifest(path) -// if err != nil { -// framework.Logf("Failed to parse clusterapi-tester from manifest %v: %v", path, err) -// return -// } -// p.Namespace = ns -// if _, err := c.CoreV1().Pods(ns).Create(p); err != nil { -// framework.Logf("Failed to create %v: %v", p.Name, err) -// return -// } -// defer func() { -// if err := c.CoreV1().Pods(ns).Delete(p.Name, nil); err != nil { -// framework.Logf("Failed to delete pod %v: %v", p.Name, err) -// } -// }() -// timeout := 5 * time.Minute -// if err := framework.WaitForPodCondition(c, ns, p.Name, "clusterapi-tester", timeout, testutils.PodRunningReady); err != nil { -// framework.Logf("Pod %v took longer than %v to enter running/ready: %v", p.Name, timeout, err) -// return -// } -// logs, err := framework.GetPodLogs(c, ns, p.Name, p.Spec.Containers[0].Name) -// if err != nil { -// framework.Logf("Failed to retrieve logs from %v: %v", p.Name, err) -// } else { -// framework.Logf("Output of clusterapi-tester:\n%v", logs) -// } -//}