Merge pull request #348 from DataDog/tyler/es-helpers

Add some missing helper classes

dd-java-agent-ittests/src/test/java/datadog/trace/agent/test/IntegrationTestUtils.java

@@ -70,7 +70,7 @@ public class IntegrationTestUtils {
    * @throws IOException
    */
   public static URL createJarWithClasses(final Class<?>... classes) throws IOException {
-    final File tmpJar = File.createTempFile(UUID.randomUUID().toString() + "", ".jar");
+    final File tmpJar = File.createTempFile(UUID.randomUUID().toString() + "-", ".jar");
     tmpJar.deleteOnExit();
 
     final Manifest manifest = new Manifest();

dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/HelperInjector.java

@@ -76,14 +76,15 @@ public class HelperInjector implements Transformer {
                 }
               }
             }
+            log.debug("Injecting classes onto classloader {} -> {}", classLoader, helperClassNames);
             if (classLoader == BOOTSTRAP_CLASSLOADER) {
-              Map<TypeDescription, Class<?>> injected =
+              final Map<TypeDescription, Class<?>> injected =
                   ClassInjector.UsingInstrumentation.of(
                           new File(System.getProperty("java.io.tmpdir")),
                           ClassInjector.UsingInstrumentation.Target.BOOTSTRAP,
                           AgentInstaller.getInstrumentation())
                       .inject(helperMap);
-              for (TypeDescription desc : injected.keySet()) {
+              for (final TypeDescription desc : injected.keySet()) {
                 Class.forName(desc.getName(), false, Utils.getBootstrapProxy());
               }
             } else {

dd-java-agent/instrumentation/elasticsearch-transport-2/elasticsearch-transport-2.gradle

@@ -38,10 +38,13 @@ dependencies {
   // TODO: add netty instrumentation when that is complete.
 
   testCompile group: 'org.elasticsearch', name: 'elasticsearch', version: '2.0.0'
-  testCompile group: 'net.java.dev.jna', name: 'jna', version: '4.5.1'
 
+  testCompile group: 'org.springframework.data', name: 'spring-data-elasticsearch', version: '2.0.0.RELEASE'
+
+  testCompile group: 'net.java.dev.jna', name: 'jna', version: '4.5.1'
   testCompile group: 'org.apache.logging.log4j', name: 'log4j-core', version: '2.11.0'
   testCompile group: 'org.apache.logging.log4j', name: 'log4j-api', version: '2.11.0'
 
   latestDepTestCompile group: 'org.elasticsearch', name: 'elasticsearch', version: '2.+'
+  latestDepTestCompile group: 'org.springframework.data', name: 'spring-data-elasticsearch', version: '2.+'
 }

dd-java-agent/instrumentation/elasticsearch-transport-2/src/main/java/datadog/trace/instrumentation/elasticsearch2/Elasticsearch2TransportClientInstrumentation.java

@@ -50,6 +50,9 @@ public class Elasticsearch2TransportClientInstrumentation extends Instrumenter.C
             new HelperInjector(
                 "com.google.common.base.Preconditions",
                 "com.google.common.base.Joiner",
+                "com.google.common.base.Joiner$1",
+                "com.google.common.base.Joiner$2",
+                "com.google.common.base.Joiner$MapJoiner",
                 "datadog.trace.instrumentation.elasticsearch2.TransportActionListener"))
         .transform(DDTransformers.defaultTransformers())
         .transform(

dd-java-agent/instrumentation/elasticsearch-transport-2/src/main/java/datadog/trace/instrumentation/elasticsearch2/TransportActionListener.java

@@ -13,6 +13,7 @@ import org.elasticsearch.action.DocumentRequest;
 import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.bulk.BulkShardResponse;
 import org.elasticsearch.action.get.GetResponse;
+import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.support.broadcast.BroadcastResponse;
 import org.elasticsearch.action.support.nodes.BaseNodesResponse;
 
@@ -35,6 +36,10 @@ public class TransportActionListener<T extends ActionResponse> implements Action
         span.setTag("elasticsearch.request.indices", Joiner.on(",").join(req.indices()));
       }
     }
+    if (request instanceof SearchRequest) {
+      final SearchRequest req = (SearchRequest) request;
+      span.setTag("elasticsearch.request.search.types", Joiner.on(",").join(req.types()));
+    }
     if (request instanceof DocumentRequest) {
       final DocumentRequest req = (DocumentRequest) request;
       span.setTag("elasticsearch.request.write.type", req.type());

dd-java-agent/instrumentation/elasticsearch-transport-2/src/test/groovy/Elasticsearch2NodeClientTest.groovy

@@ -37,7 +37,7 @@ class Elasticsearch2NodeClientTest extends AgentTestRunner {
       .put("http.port", HTTP_PORT)
       .put("transport.tcp.port", TCP_PORT)
       .build()
-    testNode = NodeBuilder.newInstance().clusterName("test-cluster").settings(settings).build()
+    testNode = NodeBuilder.newInstance().local(true).clusterName("test-cluster").settings(settings).build()
     testNode.start()
     TEST_WRITER.clear()
     testNode.client().admin().cluster().prepareHealth().setWaitForYellowStatus().execute().actionGet(5000)
@@ -195,6 +195,9 @@ class Elasticsearch2NodeClientTest extends AgentTestRunner {
           tags {
             "$Tags.COMPONENT.key" "elasticsearch-java"
             "$Tags.SPAN_KIND.key" Tags.SPAN_KIND_CLIENT
+            "$Tags.PEER_HOSTNAME.key" "local"
+            "$Tags.PEER_HOST_IPV4.key" "0.0.0.0"
+            "$Tags.PEER_PORT.key" 0
             "elasticsearch.action" "GetAction"
             "elasticsearch.request" "GetRequest"
             "elasticsearch.request.indices" indexName
@@ -230,6 +233,9 @@ class Elasticsearch2NodeClientTest extends AgentTestRunner {
           tags {
             "$Tags.COMPONENT.key" "elasticsearch-java"
             "$Tags.SPAN_KIND.key" Tags.SPAN_KIND_CLIENT
+            "$Tags.PEER_HOSTNAME.key" "local"
+            "$Tags.PEER_HOST_IPV4.key" "0.0.0.0"
+            "$Tags.PEER_PORT.key" 0
             "elasticsearch.action" "IndexAction"
             "elasticsearch.request" "IndexRequest"
             "elasticsearch.request.indices" indexName
@@ -247,6 +253,9 @@ class Elasticsearch2NodeClientTest extends AgentTestRunner {
           tags {
             "$Tags.COMPONENT.key" "elasticsearch-java"
             "$Tags.SPAN_KIND.key" Tags.SPAN_KIND_CLIENT
+            "$Tags.PEER_HOSTNAME.key" "local"
+            "$Tags.PEER_HOST_IPV4.key" "0.0.0.0"
+            "$Tags.PEER_PORT.key" 0
             "elasticsearch.action" "GetAction"
             "elasticsearch.request" "GetRequest"
             "elasticsearch.request.indices" indexName

dd-java-agent/instrumentation/elasticsearch-transport-2/src/test/groovy/Elasticsearch2SpringTemplateTest.groovy

@@ -0,0 +1,328 @@
+import datadog.trace.agent.test.AgentTestRunner
+import datadog.trace.agent.test.TestUtils
+import io.opentracing.tag.Tags
+import org.elasticsearch.action.search.SearchResponse
+import org.elasticsearch.common.io.FileSystemUtils
+import org.elasticsearch.common.settings.Settings
+import org.elasticsearch.index.IndexNotFoundException
+import org.elasticsearch.node.Node
+import org.elasticsearch.node.NodeBuilder
+import org.elasticsearch.search.aggregations.bucket.nested.InternalNested
+import org.elasticsearch.search.aggregations.bucket.terms.Terms
+import org.springframework.data.elasticsearch.core.ElasticsearchTemplate
+import org.springframework.data.elasticsearch.core.ResultsExtractor
+import org.springframework.data.elasticsearch.core.query.IndexQueryBuilder
+import org.springframework.data.elasticsearch.core.query.NativeSearchQuery
+import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder
+import spock.lang.Shared
+import springdata.Doc
+
+import java.util.concurrent.atomic.AtomicLong
+
+import static datadog.trace.agent.test.ListWriterAssert.assertTraces
+
+class Elasticsearch2SpringTemplateTest extends AgentTestRunner {
+  static {
+    System.setProperty("dd.integration.elasticsearch.enabled", "true")
+  }
+
+  static final int HTTP_PORT = TestUtils.randomOpenPort()
+  static final int TCP_PORT = TestUtils.randomOpenPort()
+
+  @Shared
+  static Node testNode
+  static File esWorkingDir
+
+  @Shared
+  static ElasticsearchTemplate template
+
+  def setupSpec() {
+    esWorkingDir = File.createTempFile("test-es-working-dir-", "")
+    esWorkingDir.delete()
+    esWorkingDir.mkdir()
+    esWorkingDir.deleteOnExit()
+    println "ES work dir: $esWorkingDir"
+
+    def settings = Settings.builder()
+      .put("path.home", esWorkingDir.path)
+      .put("http.port", HTTP_PORT)
+      .put("transport.tcp.port", TCP_PORT)
+      .build()
+    testNode = NodeBuilder.newInstance().local(true).clusterName("test-cluster").settings(settings).build()
+    testNode.start()
+
+    template = new ElasticsearchTemplate(testNode.client())
+  }
+
+  def cleanupSpec() {
+    testNode?.close()
+    if (esWorkingDir != null) {
+      FileSystemUtils.deleteSubDirectories(esWorkingDir.toPath())
+      esWorkingDir.delete()
+    }
+  }
+
+  def "test elasticsearch error"() {
+    when:
+    template.refresh(indexName)
+
+    then:
+    thrown IndexNotFoundException
+
+    and:
+    assertTraces(TEST_WRITER, 1) {
+      trace(0, 1) {
+        span(0) {
+          serviceName "elasticsearch"
+          resourceName "RefreshAction"
+          operationName "elasticsearch.query"
+          spanType null
+          errored true
+          tags {
+            "$Tags.COMPONENT.key" "elasticsearch-java"
+            "$Tags.SPAN_KIND.key" Tags.SPAN_KIND_CLIENT
+            "elasticsearch.action" "RefreshAction"
+            "elasticsearch.request" "RefreshRequest"
+            "elasticsearch.request.indices" indexName
+            errorTags IndexNotFoundException, "no such index"
+            defaultTags()
+          }
+        }
+      }
+    }
+
+    where:
+    indexName = "invalid-index"
+  }
+
+  def "test elasticsearch get"() {
+    expect:
+    template.createIndex(indexName)
+    template.getClient().admin().cluster().prepareHealth().setWaitForYellowStatus().execute().actionGet(5000)
+
+    when:
+    NativeSearchQuery query = new NativeSearchQueryBuilder()
+      .withIndices(indexName)
+      .withTypes(indexType)
+      .withIds([id])
+      .build()
+
+    then:
+    template.queryForIds(query) == []
+
+    when:
+    def result = template.index(IndexQueryBuilder.newInstance()
+      .withObject(new Doc())
+      .withIndexName(indexName)
+      .withType(indexType)
+      .withId(id)
+      .build())
+    template.refresh(Doc)
+
+    then:
+    result == id
+    template.queryForList(query, Doc) == [new Doc()]
+
+    and:
+    assertTraces(TEST_WRITER, 7) {
+      trace(0, 1) {
+        span(0) {
+          serviceName "elasticsearch"
+          resourceName "CreateIndexAction"
+          operationName "elasticsearch.query"
+          spanType null
+          tags {
+            "$Tags.COMPONENT.key" "elasticsearch-java"
+            "$Tags.SPAN_KIND.key" Tags.SPAN_KIND_CLIENT
+            "elasticsearch.action" "CreateIndexAction"
+            "elasticsearch.request" "CreateIndexRequest"
+            "elasticsearch.request.indices" indexName
+            defaultTags()
+          }
+        }
+      }
+      trace(1, 1) {
+        span(0) {
+          serviceName "elasticsearch"
+          resourceName "ClusterHealthAction"
+          operationName "elasticsearch.query"
+          spanType null
+          tags {
+            "$Tags.COMPONENT.key" "elasticsearch-java"
+            "$Tags.SPAN_KIND.key" Tags.SPAN_KIND_CLIENT
+            "elasticsearch.action" "ClusterHealthAction"
+            "elasticsearch.request" "ClusterHealthRequest"
+            defaultTags()
+          }
+        }
+      }
+      trace(2, 1) {
+        span(0) {
+          serviceName "elasticsearch"
+          resourceName "SearchAction"
+          operationName "elasticsearch.query"
+          spanType null
+          tags {
+            "$Tags.COMPONENT.key" "elasticsearch-java"
+            "$Tags.SPAN_KIND.key" Tags.SPAN_KIND_CLIENT
+            "elasticsearch.action" "SearchAction"
+            "elasticsearch.request" "SearchRequest"
+            "elasticsearch.request.indices" indexName
+            "elasticsearch.request.search.types" indexType
+            defaultTags()
+          }
+        }
+      }
+      trace(3, 1) {
+        span(0) {
+          serviceName "elasticsearch"
+          resourceName "PutMappingAction"
+          operationName "elasticsearch.query"
+          spanType null
+          tags {
+            "$Tags.COMPONENT.key" "elasticsearch-java"
+            "$Tags.SPAN_KIND.key" Tags.SPAN_KIND_CLIENT
+            "elasticsearch.action" "PutMappingAction"
+            "elasticsearch.request" "PutMappingRequest"
+            "elasticsearch.request.indices" indexName
+            defaultTags()
+          }
+        }
+      }
+      trace(4, 1) {
+        span(0) {
+          serviceName "elasticsearch"
+          resourceName "IndexAction"
+          operationName "elasticsearch.query"
+          spanType null
+          tags {
+            "$Tags.COMPONENT.key" "elasticsearch-java"
+            "$Tags.SPAN_KIND.key" Tags.SPAN_KIND_CLIENT
+            "$Tags.PEER_HOSTNAME.key" "local"
+            "$Tags.PEER_HOST_IPV4.key" "0.0.0.0"
+            "$Tags.PEER_PORT.key" 0
+            "elasticsearch.action" "IndexAction"
+            "elasticsearch.request" "IndexRequest"
+            "elasticsearch.request.indices" indexName
+            "elasticsearch.request.write.type" indexType
+            defaultTags()
+          }
+        }
+      }
+      trace(5, 1) {
+        span(0) {
+          serviceName "elasticsearch"
+          resourceName "RefreshAction"
+          operationName "elasticsearch.query"
+          spanType null
+          tags {
+            "$Tags.COMPONENT.key" "elasticsearch-java"
+            "$Tags.SPAN_KIND.key" Tags.SPAN_KIND_CLIENT
+            "elasticsearch.action" "RefreshAction"
+            "elasticsearch.request" "RefreshRequest"
+            "elasticsearch.request.indices" indexName
+            "elasticsearch.shard.broadcast.failed" 0
+            "elasticsearch.shard.broadcast.successful" 5
+            "elasticsearch.shard.broadcast.total" 10
+            defaultTags()
+          }
+        }
+      }
+      trace(6, 1) {
+        span(0) {
+          serviceName "elasticsearch"
+          resourceName "SearchAction"
+          operationName "elasticsearch.query"
+          spanType null
+          tags {
+            "$Tags.COMPONENT.key" "elasticsearch-java"
+            "$Tags.SPAN_KIND.key" Tags.SPAN_KIND_CLIENT
+            "elasticsearch.action" "SearchAction"
+            "elasticsearch.request" "SearchRequest"
+            "elasticsearch.request.indices" indexName
+            "elasticsearch.request.search.types" indexType
+            defaultTags()
+          }
+        }
+      }
+    }
+
+    where:
+    indexName = "test-index"
+    indexType = "test-type"
+    id = "1"
+  }
+
+  def "test results extractor"() {
+    setup:
+    template.createIndex(indexName)
+    testNode.client().admin().cluster().prepareHealth().setWaitForYellowStatus().execute().actionGet(5000)
+    template.index(IndexQueryBuilder.newInstance()
+      .withObject(new Doc(id: 1, data: "doc a"))
+      .withIndexName(indexName)
+      .withId("a")
+      .build())
+    template.index(IndexQueryBuilder.newInstance()
+      .withObject(new Doc(id: 2, data: "doc b"))
+      .withIndexName(indexName)
+      .withId("b")
+      .build())
+    template.refresh(indexName)
+    TEST_WRITER.waitForTraces(6)
+    TEST_WRITER.clear()
+
+    and:
+    def query = new NativeSearchQueryBuilder().withIndices(indexName).build()
+    def hits = new AtomicLong()
+    List<Map<String, Object>> results = []
+    def bucketTags = [:]
+
+    when:
+    template.query(query, new ResultsExtractor<Doc>() {
+
+      @Override
+      Doc extract(SearchResponse response) {
+        hits.addAndGet(response.getHits().totalHits())
+        results.addAll(response.hits.collect { it.source })
+        if (response.getAggregations() != null) {
+          InternalNested internalNested = response.getAggregations().get("tag")
+          if (internalNested != null) {
+            Terms terms = internalNested.getAggregations().get("count_agg")
+            Collection<Terms.Bucket> buckets = terms.getBuckets()
+            for (Terms.Bucket bucket : buckets) {
+              bucketTags.put(Integer.valueOf(bucket.getKeyAsString()), bucket.getDocCount())
+            }
+          }
+        }
+        return null
+      }
+    })
+
+    then:
+    hits.get() == 2
+    results[0] == [id: "2", data: "doc b"]
+    results[1] == [id: "1", data: "doc a"]
+    bucketTags == [:]
+
+    assertTraces(TEST_WRITER, 1) {
+      trace(0, 1) {
+        span(0) {
+          serviceName "elasticsearch"
+          resourceName "SearchAction"
+          operationName "elasticsearch.query"
+          tags {
+            "$Tags.COMPONENT.key" "elasticsearch-java"
+            "$Tags.SPAN_KIND.key" Tags.SPAN_KIND_CLIENT
+            "elasticsearch.action" "SearchAction"
+            "elasticsearch.request" "SearchRequest"
+            "elasticsearch.request.indices" indexName
+            defaultTags()
+          }
+        }
+      }
+    }
+
+    where:
+    indexName = "test-index-extract"
+  }
+}

dd-java-agent/instrumentation/elasticsearch-transport-2/src/test/groovy/springdata/Config.groovy

@@ -0,0 +1,49 @@
+package springdata
+
+import org.elasticsearch.common.io.FileSystemUtils
+import org.elasticsearch.common.settings.Settings
+import org.elasticsearch.node.NodeBuilder
+import org.springframework.context.annotation.Bean
+import org.springframework.context.annotation.ComponentScan
+import org.springframework.context.annotation.Configuration
+import org.springframework.data.elasticsearch.core.ElasticsearchOperations
+import org.springframework.data.elasticsearch.core.ElasticsearchTemplate
+import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories
+
+@Configuration
+@EnableElasticsearchRepositories(basePackages = "springdata")
+@ComponentScan(basePackages = "springdata")
+class Config {
+
+  @Bean
+  NodeBuilder nodeBuilder() {
+    return new NodeBuilder()
+  }
+
+  @Bean
+  ElasticsearchOperations elasticsearchTemplate() {
+
+    def tmpDir = File.createTempFile("test-es-working-dir-", "")
+    tmpDir.delete()
+    tmpDir.mkdir()
+    tmpDir.deleteOnExit()
+
+    System.addShutdownHook {
+      if (tmpDir != null) {
+        FileSystemUtils.deleteSubDirectories(esWorkingDir.toPath())
+        tmpDir.delete()
+      }
+    }
+
+    final Settings.Builder elasticsearchSettings =
+      Settings.settingsBuilder()
+        .put("http.enabled", "false")
+        .put("path.data", tmpDir.toString())
+        .put("path.home", tmpDir.toString())
+
+    println "ES work dir: $tmpDir"
+
+    return new ElasticsearchTemplate(nodeBuilder().local(true)
+      .settings(elasticsearchSettings.build()).node().client())
+  }
+}

dd-java-agent/instrumentation/elasticsearch-transport-2/src/test/groovy/springdata/Doc.groovy

@@ -0,0 +1,29 @@
+package springdata
+
+import groovy.transform.EqualsAndHashCode
+import org.springframework.data.annotation.Id
+import org.springframework.data.elasticsearch.annotations.Document
+
+@Document(indexName = "test-index")
+@EqualsAndHashCode
+class Doc {
+  @Id
+  private String id = "1"
+  private String data = "some data"
+
+  String getId() {
+    return id
+  }
+
+  void setId(String id) {
+    this.id = id
+  }
+
+  String getData() {
+    return data
+  }
+
+  void setData(String data) {
+    this.data = data
+  }
+}

dd-java-agent/instrumentation/elasticsearch-transport-2/src/test/groovy/springdata/DocRepository.groovy

@@ -0,0 +1,5 @@
+package springdata
+
+import org.springframework.data.elasticsearch.repository.ElasticsearchRepository
+
+interface DocRepository extends ElasticsearchRepository<Doc, String> {}

dd-java-agent/instrumentation/elasticsearch-transport-2/src/test/groovy/springdata/Elasticsearch2SpringRepositoryTest.groovy

@@ -0,0 +1,283 @@
+package springdata
+
+import datadog.trace.agent.test.AgentTestRunner
+import io.opentracing.tag.Tags
+import org.springframework.context.ApplicationContext
+import org.springframework.context.annotation.AnnotationConfigApplicationContext
+import spock.lang.Shared
+
+import static datadog.trace.agent.test.ListWriterAssert.assertTraces
+
+class Elasticsearch2SpringRepositoryTest extends AgentTestRunner {
+  static {
+    System.setProperty("dd.integration.elasticsearch.enabled", "true")
+  }
+
+  @Shared
+  ApplicationContext applicationContext = new AnnotationConfigApplicationContext(Config)
+
+  @Shared
+  DocRepository repo = applicationContext.getBean(DocRepository)
+
+  def setup() {
+    repo.deleteAll()
+    TEST_WRITER.waitForTraces(4)
+    TEST_WRITER.clear()
+  }
+
+  def "test empty repo"() {
+    when:
+    def result = repo.findAll()
+
+    then:
+    !result.iterator().hasNext()
+
+    and:
+    assertTraces(TEST_WRITER, 1) {
+      trace(0, 1) {
+        span(0) {
+          serviceName "elasticsearch"
+          resourceName "SearchAction"
+          operationName "elasticsearch.query"
+          spanType null
+          errored false
+          tags {
+            "$Tags.COMPONENT.key" "elasticsearch-java"
+            "$Tags.SPAN_KIND.key" Tags.SPAN_KIND_CLIENT
+            "elasticsearch.action" "SearchAction"
+            "elasticsearch.request" "SearchRequest"
+            "elasticsearch.request.indices" indexName
+            "elasticsearch.request.search.types" "doc"
+            defaultTags()
+          }
+        }
+      }
+    }
+
+    where:
+    indexName = "test-index"
+  }
+
+  def "test CRUD"() {
+    when:
+    def doc = new Doc()
+
+    then:
+    repo.index(doc) == doc
+
+    and:
+    assertTraces(TEST_WRITER, 3) {
+      trace(0, 1) {
+        span(0) {
+          resourceName "PutMappingAction"
+          operationName "elasticsearch.query"
+          tags {
+            "$Tags.COMPONENT.key" "elasticsearch-java"
+            "$Tags.SPAN_KIND.key" Tags.SPAN_KIND_CLIENT
+            "elasticsearch.action" "PutMappingAction"
+            "elasticsearch.request" "PutMappingRequest"
+            "elasticsearch.request.indices" indexName
+            defaultTags()
+          }
+        }
+      }
+      trace(1, 1) {
+        span(0) {
+          resourceName "IndexAction"
+          operationName "elasticsearch.query"
+          tags {
+            "$Tags.COMPONENT.key" "elasticsearch-java"
+            "$Tags.SPAN_KIND.key" Tags.SPAN_KIND_CLIENT
+            "$Tags.PEER_HOSTNAME.key" "local"
+            "$Tags.PEER_HOST_IPV4.key" "0.0.0.0"
+            "$Tags.PEER_PORT.key" 0
+            "elasticsearch.action" "IndexAction"
+            "elasticsearch.request" "IndexRequest"
+            "elasticsearch.request.indices" indexName
+            "elasticsearch.request.write.type" "doc"
+            defaultTags()
+          }
+        }
+      }
+      trace(2, 1) {
+        span(0) {
+          resourceName "RefreshAction"
+          operationName "elasticsearch.query"
+          tags {
+            "$Tags.COMPONENT.key" "elasticsearch-java"
+            "$Tags.SPAN_KIND.key" Tags.SPAN_KIND_CLIENT
+            "elasticsearch.action" "RefreshAction"
+            "elasticsearch.request" "RefreshRequest"
+            "elasticsearch.request.indices" indexName
+            "elasticsearch.shard.broadcast.failed" 0
+            "elasticsearch.shard.broadcast.successful" 5
+            "elasticsearch.shard.broadcast.total" 10
+            defaultTags()
+          }
+        }
+      }
+    }
+    TEST_WRITER.clear()
+
+    and:
+    repo.findOne("1") == doc
+
+    and:
+    assertTraces(TEST_WRITER, 1) {
+      trace(0, 1) {
+        span(0) {
+          serviceName "elasticsearch"
+          resourceName "GetAction"
+          operationName "elasticsearch.query"
+          tags {
+            "$Tags.COMPONENT.key" "elasticsearch-java"
+            "$Tags.SPAN_KIND.key" Tags.SPAN_KIND_CLIENT
+            "$Tags.PEER_HOSTNAME.key" "local"
+            "$Tags.PEER_HOST_IPV4.key" "0.0.0.0"
+            "$Tags.PEER_PORT.key" 0
+            "elasticsearch.action" "GetAction"
+            "elasticsearch.request" "GetRequest"
+            "elasticsearch.request.indices" indexName
+            "elasticsearch.type" "doc"
+            "elasticsearch.id" "1"
+            "elasticsearch.version" 1
+            defaultTags()
+          }
+        }
+      }
+    }
+    TEST_WRITER.clear()
+
+    when:
+    doc.data == "other data"
+
+    then:
+    repo.index(doc) == doc
+    repo.findOne("1") == doc
+
+    and:
+    assertTraces(TEST_WRITER, 3) {
+      trace(0, 1) {
+        span(0) {
+          resourceName "IndexAction"
+          operationName "elasticsearch.query"
+          tags {
+            "$Tags.COMPONENT.key" "elasticsearch-java"
+            "$Tags.SPAN_KIND.key" Tags.SPAN_KIND_CLIENT
+            "$Tags.PEER_HOSTNAME.key" "local"
+            "$Tags.PEER_HOST_IPV4.key" "0.0.0.0"
+            "$Tags.PEER_PORT.key" 0
+            "elasticsearch.action" "IndexAction"
+            "elasticsearch.request" "IndexRequest"
+            "elasticsearch.request.indices" indexName
+            "elasticsearch.request.write.type" "doc"
+            defaultTags()
+          }
+        }
+      }
+      trace(1, 1) {
+        span(0) {
+          resourceName "RefreshAction"
+          operationName "elasticsearch.query"
+          tags {
+            "$Tags.COMPONENT.key" "elasticsearch-java"
+            "$Tags.SPAN_KIND.key" Tags.SPAN_KIND_CLIENT
+            "elasticsearch.action" "RefreshAction"
+            "elasticsearch.request" "RefreshRequest"
+            "elasticsearch.request.indices" indexName
+            "elasticsearch.shard.broadcast.failed" 0
+            "elasticsearch.shard.broadcast.successful" 5
+            "elasticsearch.shard.broadcast.total" 10
+            defaultTags()
+          }
+        }
+      }
+      trace(2, 1) {
+        span(0) {
+          serviceName "elasticsearch"
+          resourceName "GetAction"
+          operationName "elasticsearch.query"
+          tags {
+            "$Tags.COMPONENT.key" "elasticsearch-java"
+            "$Tags.SPAN_KIND.key" Tags.SPAN_KIND_CLIENT
+            "$Tags.PEER_HOSTNAME.key" "local"
+            "$Tags.PEER_HOST_IPV4.key" "0.0.0.0"
+            "$Tags.PEER_PORT.key" 0
+            "elasticsearch.action" "GetAction"
+            "elasticsearch.request" "GetRequest"
+            "elasticsearch.request.indices" indexName
+            "elasticsearch.type" "doc"
+            "elasticsearch.id" "1"
+            "elasticsearch.version" 2
+            defaultTags()
+          }
+        }
+      }
+    }
+    TEST_WRITER.clear()
+
+    when:
+    repo.delete("1")
+
+    then:
+    !repo.findAll().iterator().hasNext()
+
+    and:
+    assertTraces(TEST_WRITER, 3) {
+      trace(0, 1) {
+        span(0) {
+          resourceName "DeleteAction"
+          operationName "elasticsearch.query"
+          tags {
+            "$Tags.COMPONENT.key" "elasticsearch-java"
+            "$Tags.SPAN_KIND.key" Tags.SPAN_KIND_CLIENT
+            "$Tags.PEER_HOSTNAME.key" "local"
+            "$Tags.PEER_HOST_IPV4.key" "0.0.0.0"
+            "$Tags.PEER_PORT.key" 0
+            "elasticsearch.action" "DeleteAction"
+            "elasticsearch.request" "DeleteRequest"
+            "elasticsearch.request.indices" indexName
+            "elasticsearch.request.write.type" "doc"
+            defaultTags()
+          }
+        }
+      }
+      trace(1, 1) {
+        span(0) {
+          resourceName "RefreshAction"
+          operationName "elasticsearch.query"
+          tags {
+            "$Tags.COMPONENT.key" "elasticsearch-java"
+            "$Tags.SPAN_KIND.key" Tags.SPAN_KIND_CLIENT
+            "elasticsearch.action" "RefreshAction"
+            "elasticsearch.request" "RefreshRequest"
+            "elasticsearch.request.indices" indexName
+            "elasticsearch.shard.broadcast.failed" 0
+            "elasticsearch.shard.broadcast.successful" 5
+            "elasticsearch.shard.broadcast.total" 10
+            defaultTags()
+          }
+        }
+      }
+      trace(2, 1) {
+        span(0) {
+          serviceName "elasticsearch"
+          resourceName "SearchAction"
+          operationName "elasticsearch.query"
+          tags {
+            "$Tags.COMPONENT.key" "elasticsearch-java"
+            "$Tags.SPAN_KIND.key" Tags.SPAN_KIND_CLIENT
+            "elasticsearch.action" "SearchAction"
+            "elasticsearch.request" "SearchRequest"
+            "elasticsearch.request.indices" indexName
+            "elasticsearch.request.search.types" "doc"
+            defaultTags()
+          }
+        }
+      }
+    }
+
+    where:
+    indexName = "test-index"
+  }
+}

dd-java-agent/instrumentation/elasticsearch-transport-5/src/main/java/datadog/trace/instrumentation/elasticsearch5/Elasticsearch5TransportClientInstrumentation.java

@@ -50,6 +50,9 @@ public class Elasticsearch5TransportClientInstrumentation extends Instrumenter.C
             new HelperInjector(
                 "com.google.common.base.Preconditions",
                 "com.google.common.base.Joiner",
+                "com.google.common.base.Joiner$1",
+                "com.google.common.base.Joiner$2",
+                "com.google.common.base.Joiner$MapJoiner",
                 "datadog.trace.instrumentation.elasticsearch5.TransportActionListener"))
         .transform(DDTransformers.defaultTransformers())
         .transform(

dd-java-agent/instrumentation/elasticsearch-transport-5/src/main/java/datadog/trace/instrumentation/elasticsearch5/TransportActionListener.java

@@ -14,6 +14,7 @@ import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.bulk.BulkShardResponse;
 import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.action.index.IndexResponse;
+import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.support.broadcast.BroadcastResponse;
 import org.elasticsearch.action.support.nodes.BaseNodesResponse;
 import org.elasticsearch.action.support.replication.ReplicationResponse;
@@ -40,6 +41,10 @@ public class TransportActionListener<T extends ActionResponse> implements Action
         span.setTag("elasticsearch.request.indices", Joiner.on(",").join(req.indices()));
       }
     }
+    if (request instanceof SearchRequest) {
+      final SearchRequest req = (SearchRequest) request;
+      span.setTag("elasticsearch.request.search.types", Joiner.on(",").join(req.types()));
+    }
     if (request instanceof DocumentRequest) {
       final DocumentRequest req = (DocumentRequest) request;
       span.setTag("elasticsearch.request.write.type", req.type());

dd-java-agent/instrumentation/elasticsearch-transport-6/src/main/java/datadog/trace/instrumentation/elasticsearch6/Elasticsearch6TransportClientInstrumentation.java

@@ -54,6 +54,9 @@ public class Elasticsearch6TransportClientInstrumentation extends Instrumenter.C
             new HelperInjector(
                 "com.google.common.base.Preconditions",
                 "com.google.common.base.Joiner",
+                "com.google.common.base.Joiner$1",
+                "com.google.common.base.Joiner$2",
+                "com.google.common.base.Joiner$MapJoiner",
                 "datadog.trace.instrumentation.elasticsearch6.TransportActionListener"))
         .transform(DDTransformers.defaultTransformers())
         .transform(

dd-java-agent/instrumentation/elasticsearch-transport-6/src/main/java/datadog/trace/instrumentation/elasticsearch6/TransportActionListener.java

@@ -14,6 +14,7 @@ import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.bulk.BulkShardResponse;
 import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.action.index.IndexResponse;
+import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.support.broadcast.BroadcastResponse;
 import org.elasticsearch.action.support.nodes.BaseNodesResponse;
 import org.elasticsearch.action.support.replication.ReplicationResponse;
@@ -44,6 +45,10 @@ public class TransportActionListener<T extends ActionResponse> implements Action
         span.setTag("elasticsearch.request.indices", Joiner.on(",").join(req.indices()));
       }
     }
+    if (request instanceof SearchRequest) {
+      final SearchRequest req = (SearchRequest) request;
+      span.setTag("elasticsearch.request.search.types", Joiner.on(",").join(req.types()));
+    }
     if (request instanceof DocWriteRequest) {
       final DocWriteRequest req = (DocWriteRequest) request;
       span.setTag("elasticsearch.request.write.type", req.type());