Improve ES tests

* Account for the fact that IndexAction and PutMappingAction are not
  ordered deterministically
* Account for a fact that waiting for ES startup may produce multiple spans

dd-java-agent/instrumentation/elasticsearch-transport-2/src/test/groovy/Elasticsearch2NodeClientTest.groovy

@@ -11,6 +11,7 @@ import org.elasticsearch.node.Node
 import org.elasticsearch.node.NodeBuilder
 import spock.lang.Shared
 
+import static datadog.trace.agent.test.TestUtils.runUnderTrace
 import static datadog.trace.agent.test.asserts.ListWriterAssert.assertTraces
 
 class Elasticsearch2NodeClientTest extends AgentTestRunner {
@@ -48,8 +49,11 @@ class Elasticsearch2NodeClientTest extends AgentTestRunner {
       .build()
     testNode = NodeBuilder.newInstance().local(true).clusterName("test-cluster").settings(settings).build()
     testNode.start()
-    TEST_WRITER.clear()
+    runUnderTrace("setup") {
-    testNode.client().admin().cluster().prepareHealth().setWaitForYellowStatus().execute().actionGet(TIMEOUT)
+      // this may potentially create multiple requests and therefore multiple spans, so we wrap this call
+      // into a top level trace to get exactly one trace in the result.
+      testNode.client().admin().cluster().prepareHealth().setWaitForYellowStatus().execute().actionGet(TIMEOUT)
+    }
     TEST_WRITER.waitForTraces(1)
   }
 
@@ -164,6 +168,13 @@ class Elasticsearch2NodeClientTest extends AgentTestRunner {
     result.index == indexName
 
     and:
+    // IndexAction and PutMappingAction run in separate threads and order in which
+    // these spans are closed is not defined. So we force the order if it is wrong.
+    if (TEST_WRITER[3][0].resourceName == "IndexAction") {
+      def tmp = TEST_WRITER[3]
+      TEST_WRITER[3] = TEST_WRITER[4]
+      TEST_WRITER[4] = tmp
+    }
     assertTraces(TEST_WRITER, 6) {
       trace(0, 1) {
         span(0) {

dd-java-agent/instrumentation/elasticsearch-transport-2/src/test/groovy/Elasticsearch2SpringTemplateTest.groovy

@@ -134,6 +134,13 @@ class Elasticsearch2SpringTemplateTest extends AgentTestRunner {
     template.queryForList(query, Doc) == [new Doc()]
 
     and:
+    // IndexAction and PutMappingAction run in separate threads and order in which
+    // these spans are closed is not defined. So we force the order if it is wrong.
+    if (TEST_WRITER[3][0].resourceName == "IndexAction") {
+      def tmp = TEST_WRITER[3]
+      TEST_WRITER[3] = TEST_WRITER[4]
+      TEST_WRITER[4] = tmp
+    }
     assertTraces(TEST_WRITER, 7) {
       trace(0, 1) {
         span(0) {

dd-java-agent/instrumentation/elasticsearch-transport-2/src/test/groovy/Elasticsearch2TransportClientTest.groovy

@@ -14,6 +14,7 @@ import org.elasticsearch.node.NodeBuilder
 import org.elasticsearch.transport.RemoteTransportException
 import spock.lang.Shared
 
+import static datadog.trace.agent.test.TestUtils.runUnderTrace
 import static datadog.trace.agent.test.asserts.ListWriterAssert.assertTraces
 
 class Elasticsearch2TransportClientTest extends AgentTestRunner {
@@ -59,8 +60,11 @@ class Elasticsearch2TransportClientTest extends AgentTestRunner {
         .build()
     ).build()
     client.addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName("localhost"), tcpPort))
-    TEST_WRITER.clear()
+    runUnderTrace("setup") {
-    client.admin().cluster().prepareHealth().setWaitForYellowStatus().execute().actionGet(TIMEOUT)
+      // this may potentially create multiple requests and therefore multiple spans, so we wrap this call
+      // into a top level trace to get exactly one trace in the result.
+      client.admin().cluster().prepareHealth().setWaitForYellowStatus().execute().actionGet(TIMEOUT)
+    }
     TEST_WRITER.waitForTraces(1)
   }
 
@@ -178,6 +182,13 @@ class Elasticsearch2TransportClientTest extends AgentTestRunner {
     result.index == indexName
 
     and:
+    // IndexAction and PutMappingAction run in separate threads and order in which
+    // these spans are closed is not defined. So we force the order if it is wrong.
+    if (TEST_WRITER[3][0].resourceName == "IndexAction") {
+      def tmp = TEST_WRITER[3]
+      TEST_WRITER[3] = TEST_WRITER[4]
+      TEST_WRITER[4] = tmp
+    }
     assertTraces(TEST_WRITER, 6) {
       trace(0, 1) {
         span(0) {

dd-java-agent/instrumentation/elasticsearch-transport-5/src/test/groovy/Elasticsearch5NodeClientTest.groovy

@@ -13,6 +13,7 @@ import org.elasticsearch.node.internal.InternalSettingsPreparer
 import org.elasticsearch.transport.Netty3Plugin
 import spock.lang.Shared
 
+import static datadog.trace.agent.test.TestUtils.runUnderTrace
 import static datadog.trace.agent.test.asserts.ListWriterAssert.assertTraces
 import static org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING
 
@@ -54,8 +55,11 @@ class Elasticsearch5NodeClientTest extends AgentTestRunner {
       .build()
     testNode = new Node(new Environment(InternalSettingsPreparer.prepareSettings(settings)), [Netty3Plugin])
     testNode.start()
-    TEST_WRITER.clear()
+    runUnderTrace("setup") {
-    testNode.client().admin().cluster().prepareHealth().setWaitForYellowStatus().execute().actionGet(TIMEOUT)
+      // this may potentially create multiple requests and therefore multiple spans, so we wrap this call
+      // into a top level trace to get exactly one trace in the result.
+      testNode.client().admin().cluster().prepareHealth().setWaitForYellowStatus().execute().actionGet(TIMEOUT)
+    }
     TEST_WRITER.waitForTraces(1)
   }
 
@@ -171,6 +175,13 @@ class Elasticsearch5NodeClientTest extends AgentTestRunner {
     result.index == indexName
 
     and:
+    // IndexAction and PutMappingAction run in separate threads and order in which
+    // these spans are closed is not defined. So we force the order if it is wrong.
+    if (TEST_WRITER[3][0].resourceName == "IndexAction") {
+      def tmp = TEST_WRITER[3]
+      TEST_WRITER[3] = TEST_WRITER[4]
+      TEST_WRITER[4] = tmp
+    }
     assertTraces(TEST_WRITER, 6) {
       trace(0, 1) {
         span(0) {

dd-java-agent/instrumentation/elasticsearch-transport-5/src/test/groovy/Elasticsearch5TransportClientTest.groovy

@@ -17,6 +17,7 @@ import org.elasticsearch.transport.RemoteTransportException
 import org.elasticsearch.transport.client.PreBuiltTransportClient
 import spock.lang.Shared
 
+import static datadog.trace.agent.test.TestUtils.runUnderTrace
 import static datadog.trace.agent.test.asserts.ListWriterAssert.assertTraces
 import static org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING
 
@@ -66,8 +67,11 @@ class Elasticsearch5TransportClientTest extends AgentTestRunner {
         .build()
     )
     client.addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName("localhost"), tcpPort))
-    TEST_WRITER.clear()
+    runUnderTrace("setup") {
-    client.admin().cluster().prepareHealth().setWaitForYellowStatus().execute().actionGet(TIMEOUT)
+      // this may potentially create multiple requests and therefore multiple spans, so we wrap this call
+      // into a top level trace to get exactly one trace in the result.
+      client.admin().cluster().prepareHealth().setWaitForYellowStatus().execute().actionGet(TIMEOUT)
+    }
     TEST_WRITER.waitForTraces(1)
   }
 
@@ -185,6 +189,13 @@ class Elasticsearch5TransportClientTest extends AgentTestRunner {
     result.index == indexName
 
     and:
+    // IndexAction and PutMappingAction run in separate threads and order in which
+    // these spans are closed is not defined. So we force the order if it is wrong.
+    if (TEST_WRITER[2][0].resourceName == "IndexAction") {
+      def tmp = TEST_WRITER[2]
+      TEST_WRITER[2] = TEST_WRITER[3]
+      TEST_WRITER[3] = tmp
+    }
     assertTraces(TEST_WRITER, 5) {
       trace(0, 1) {
         span(0) {

dd-java-agent/instrumentation/elasticsearch-transport-6/src/test/groovy/Elasticsearch6NodeClientTest.groovy

@@ -12,6 +12,7 @@ import org.elasticsearch.node.Node
 import org.elasticsearch.transport.Netty4Plugin
 import spock.lang.Shared
 
+import static datadog.trace.agent.test.TestUtils.runUnderTrace
 import static datadog.trace.agent.test.asserts.ListWriterAssert.assertTraces
 import static org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING
 
@@ -51,8 +52,11 @@ class Elasticsearch6NodeClientTest extends AgentTestRunner {
       .build()
     testNode = new Node(InternalSettingsPreparer.prepareEnvironment(settings, null), [Netty4Plugin])
     testNode.start()
-    TEST_WRITER.clear()
+    runUnderTrace("setup") {
-    testNode.client().admin().cluster().prepareHealth().setWaitForYellowStatus().execute().actionGet(TIMEOUT)
+      // this may potentially create multiple requests and therefore multiple spans, so we wrap this call
+      // into a top level trace to get exactly one trace in the result.
+      testNode.client().admin().cluster().prepareHealth().setWaitForYellowStatus().execute().actionGet(TIMEOUT)
+    }
     TEST_WRITER.waitForTraces(1)
   }
 
@@ -167,6 +171,13 @@ class Elasticsearch6NodeClientTest extends AgentTestRunner {
     result.index == indexName
 
     and:
+    // IndexAction and PutMappingAction run in separate threads and order in which
+    // these spans are closed is not defined. So we force the order if it is wrong.
+    if (TEST_WRITER[2][0].resourceName == "IndexAction") {
+      def tmp = TEST_WRITER[2]
+      TEST_WRITER[2] = TEST_WRITER[3]
+      TEST_WRITER[3] = tmp
+    }
     assertTraces(TEST_WRITER, 5) {
       trace(0, 1) {
         span(0) {

dd-java-agent/instrumentation/elasticsearch-transport-6/src/test/groovy/Elasticsearch6TransportClientTest.groovy

@@ -16,6 +16,7 @@ import org.elasticsearch.transport.RemoteTransportException
 import org.elasticsearch.transport.client.PreBuiltTransportClient
 import spock.lang.Shared
 
+import static datadog.trace.agent.test.TestUtils.runUnderTrace
 import static datadog.trace.agent.test.asserts.ListWriterAssert.assertTraces
 import static org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING
 
@@ -63,8 +64,11 @@ class Elasticsearch6TransportClientTest extends AgentTestRunner {
         .build()
     )
     client.addTransportAddress(new TransportAddress(InetAddress.getByName("localhost"), tcpPort))
-    TEST_WRITER.clear()
+    runUnderTrace("setup") {
-    client.admin().cluster().prepareHealth().setWaitForYellowStatus().execute().actionGet(TIMEOUT)
+      // this may potentially create multiple requests and therefore multiple spans, so we wrap this call
+      // into a top level trace to get exactly one trace in the result.
+      client.admin().cluster().prepareHealth().setWaitForYellowStatus().execute().actionGet(TIMEOUT)
+    }
     TEST_WRITER.waitForTraces(1)
   }
 
@@ -182,6 +186,13 @@ class Elasticsearch6TransportClientTest extends AgentTestRunner {
     result.index == indexName
 
     and:
+    // IndexAction and PutMappingAction run in separate threads and order in which
+    // these spans are closed is not defined. So we force the order if it is wrong.
+    if (TEST_WRITER[2][0].resourceName == "IndexAction") {
+      def tmp = TEST_WRITER[2]
+      TEST_WRITER[2] = TEST_WRITER[3]
+      TEST_WRITER[3] = tmp
+    }
     assertTraces(TEST_WRITER, 5) {
       trace(0, 1) {
         span(0) {