Resolve conflicts + improve surefire test isolation using fork + tests logging less verbous

dd-java-agent/dependency-reduced-pom.xml

@@ -23,10 +23,17 @@
         <artifactId>maven-surefire-plugin</artifactId>
         <version>2.20</version>
         <configuration>
-          <argLine>-javaagent:${M2_REPO}/com/datadoghq/dd-java-agent/${project.version}/dd-java-agent-${project.version}.jar</argLine>
           <includes>
             <include>*Test.java</include>
           </includes>
+          <excludes>
+            <exclude>ElasticSearchInstrumentationTest.java</exclude>
+          </excludes>
+          <forkCount>3</forkCount>
+          <reuseForks>false</reuseForks>
+          <argLine>-Xmx500m -Xms500m -XX:MaxPermSize=256m
+						-javaagent:${M2_REPO}/com/datadoghq/dd-java-agent/${project.version}/dd-java-agent-${project.version}.jar</argLine>
+          <workingDirectory>target/FORK_DIRECTORY_${surefire.forkNumber}</workingDirectory>
         </configuration>
       </plugin>
       <plugin>
@@ -63,26 +70,6 @@
     </plugins>
   </build>
   <dependencies>
-    <dependency>
-      <groupId>org.apache.httpcomponents</groupId>
-      <artifactId>httpclient</artifactId>
-      <version>4.5.3</version>
-      <scope>test</scope>
-      <exclusions>
-        <exclusion>
-          <artifactId>httpcore</artifactId>
-          <groupId>org.apache.httpcomponents</groupId>
-        </exclusion>
-        <exclusion>
-          <artifactId>commons-logging</artifactId>
-          <groupId>commons-logging</groupId>
-        </exclusion>
-        <exclusion>
-          <artifactId>commons-codec</artifactId>
-          <groupId>commons-codec</groupId>
-        </exclusion>
-      </exclusions>
-    </dependency>
     <dependency>
       <groupId>io.opentracing</groupId>
       <artifactId>opentracing-mock</artifactId>
@@ -113,18 +100,6 @@
       <version>3.4.2</version>
       <scope>test</scope>
     </dependency>
-    <dependency>
-      <groupId>com.squareup.okhttp3</groupId>
-      <artifactId>okhttp</artifactId>
-      <version>3.6.0</version>
-      <scope>test</scope>
-      <exclusions>
-        <exclusion>
-          <artifactId>okio</artifactId>
-          <groupId>com.squareup.okio</groupId>
-        </exclusion>
-      </exclusions>
-    </dependency>
     <dependency>
       <groupId>org.eclipse.jetty</groupId>
       <artifactId>jetty-server</artifactId>
@@ -185,6 +160,10 @@
       <version>1.11.132</version>
       <scope>test</scope>
       <exclusions>
+        <exclusion>
+          <artifactId>httpclient</artifactId>
+          <groupId>org.apache.httpcomponents</groupId>
+        </exclusion>
         <exclusion>
           <artifactId>aws-java-sdk-athena</artifactId>
           <groupId>com.amazonaws</groupId>
@@ -571,6 +550,82 @@
         </exclusion>
       </exclusions>
     </dependency>
+    <dependency>
+      <groupId>com.squareup.okhttp3</groupId>
+      <artifactId>okhttp</artifactId>
+      <version>3.6.0</version>
+      <scope>test</scope>
+      <exclusions>
+        <exclusion>
+          <artifactId>okio</artifactId>
+          <groupId>com.squareup.okio</groupId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.httpcomponents</groupId>
+      <artifactId>httpclient</artifactId>
+      <version>4.5.3</version>
+      <scope>test</scope>
+      <exclusions>
+        <exclusion>
+          <artifactId>httpcore</artifactId>
+          <groupId>org.apache.httpcomponents</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>commons-logging</artifactId>
+          <groupId>commons-logging</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>commons-codec</artifactId>
+          <groupId>commons-codec</groupId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <dependency>
+      <groupId>org.elasticsearch.client</groupId>
+      <artifactId>transport</artifactId>
+      <version>5.4.0</version>
+      <scope>test</scope>
+      <exclusions>
+        <exclusion>
+          <artifactId>elasticsearch</artifactId>
+          <groupId>org.elasticsearch</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>transport-netty3-client</artifactId>
+          <groupId>org.elasticsearch.plugin</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>transport-netty4-client</artifactId>
+          <groupId>org.elasticsearch.plugin</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>reindex-client</artifactId>
+          <groupId>org.elasticsearch.plugin</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>lang-mustache-client</artifactId>
+          <groupId>org.elasticsearch.plugin</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>percolator-client</artifactId>
+          <groupId>org.elasticsearch.plugin</groupId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.logging.log4j</groupId>
+      <artifactId>log4j-api</artifactId>
+      <version>2.7</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.logging.log4j</groupId>
+      <artifactId>log4j-core</artifactId>
+      <version>2.7</version>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
   <properties>
     <opentracing.version>0.22.0</opentracing.version>

dd-java-agent/pom.xml

@@ -90,7 +90,10 @@
 			<artifactId>opentracing-okhttp3</artifactId>
 			<version>${ot.contrib.okhttp.version}</version>
 			<exclusions>
-
+				<exclusion>
+					<groupId>com.squareup.okhttp3</groupId>
+					<artifactId>okhttp</artifactId>
+				</exclusion>
 			</exclusions>
 		</dependency>
 
@@ -112,12 +115,33 @@
 			<groupId>io.opentracing.contrib</groupId>
 			<artifactId>opentracing-apache-httpclient</artifactId>
 			<version>0.0.1</version>
+			<exclusions>
+				<exclusion>
+					<groupId>org.apache.httpcomponents</groupId>
+					<artifactId>httpclient</artifactId>
+				</exclusion>
+			</exclusions>
 		</dependency>
+
+		<!-- ES instrumentation -->
 		<dependency>
-			<groupId>org.apache.httpcomponents</groupId>
+			<groupId>io.opentracing.contrib</groupId>
-			<artifactId>httpclient</artifactId>
+			<artifactId>opentracing-elasticsearch-client</artifactId>
-			<version>4.5.3</version>
+			<version>0.0.1</version>
-			<scope>test</scope>
+			<exclusions>
+				<exclusion>
+					<groupId>org.apache.logging.log4j</groupId>
+					<artifactId>log4j-core</artifactId>
+				</exclusion>
+				<exclusion>
+					<groupId>org.elasticsearch.client</groupId>
+					<artifactId>transport</artifactId>
+				</exclusion>
+				<exclusion>
+					<groupId>org.apache.logging.log4j</groupId>
+					<artifactId>log4j-api</artifactId>
+				</exclusion>
+			</exclusions>
 		</dependency>
 
 		<!-- JUnit tests -->
@@ -147,13 +171,7 @@
 			<version>3.4.2</version>
 			<scope>test</scope>
 		</dependency>
-		<!-- OkHTTP tests -->
+
-		<dependency>
-			<groupId>com.squareup.okhttp3</groupId>
-			<artifactId>okhttp</artifactId>
-			<version>3.6.0</version>
-			<scope>test</scope>
-		</dependency>
 		<!-- Jetty Servlet tests -->
 		<dependency>
 			<groupId>org.eclipse.jetty</groupId>
@@ -188,6 +206,48 @@
 			<artifactId>aws-java-sdk</artifactId>
 			<version>1.11.132</version>
 			<scope>test</scope>
+			<exclusions>
+				<exclusion>
+					<groupId>org.apache.httpcomponents</groupId>
+					<artifactId>httpclient</artifactId>
+				</exclusion>
+			</exclusions>
+		</dependency>
+
+		<!-- OkHTTP tests -->
+		<dependency>
+			<groupId>com.squareup.okhttp3</groupId>
+			<artifactId>okhttp</artifactId>
+			<version>3.6.0</version>
+			<scope>test</scope>
+		</dependency>
+
+		<!-- Apache HTTP client tests -->
+		<dependency>
+			<groupId>org.apache.httpcomponents</groupId>
+			<artifactId>httpclient</artifactId>
+			<version>4.5.3</version>
+			<scope>test</scope>
+		</dependency>
+
+		<!--Elasticsearch tests -->
+		<dependency>
+			<groupId>org.elasticsearch.client</groupId>
+			<artifactId>transport</artifactId>
+			<version>5.4.0</version>
+			<scope>test</scope>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.logging.log4j</groupId>
+			<artifactId>log4j-api</artifactId>
+			<version>2.7</version>
+			<scope>test</scope>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.logging.log4j</groupId>
+			<artifactId>log4j-core</artifactId>
+			<version>2.7</version>
+			<scope>test</scope>
 		</dependency>
 
 	</dependencies>
@@ -208,10 +268,17 @@
 				<artifactId>maven-surefire-plugin</artifactId>
 				<version>2.20</version>
 				<configuration>
-					<argLine>-javaagent:${M2_REPO}/com/datadoghq/dd-java-agent/${project.version}/dd-java-agent-${project.version}.jar</argLine>
 					<includes>
 						<include>*Test.java</include>
 					</includes>
+					<excludes>
+						<exclude>ElasticSearchInstrumentationTest.java</exclude>
+					</excludes>
+					<forkCount>3</forkCount>
+					<reuseForks>false</reuseForks>
+					<argLine>-Xmx500m -Xms500m -XX:MaxPermSize=256m
+						-javaagent:${M2_REPO}/com/datadoghq/dd-java-agent/${project.version}/dd-java-agent-${project.version}.jar</argLine>
+					<workingDirectory>target/FORK_DIRECTORY_${surefire.forkNumber}</workingDirectory>
 				</configuration>
 			</plugin>
 			<plugin>

dd-java-agent/src/main/java/io/opentracing/contrib/agent/TraceAnnotationsManager.java

@@ -41,7 +41,12 @@ public class TraceAnnotationsManager extends OpenTracingManager{
 	public static void initialize(Retransformer trans) throws Exception {
 		transformer = trans;
 		OpenTracingManager.initialize(trans);
-		loadRules(ClassLoader.getSystemClassLoader());
+		OpenTracingManager.loadRules(ClassLoader.getSystemClassLoader());
+		
+		String value = System.getProperty("javaagent.enableAnnotations","false");
+		if("true".equalsIgnoreCase(value)){
+			loadRules(ClassLoader.getSystemClassLoader());
+		}
 	}
 
 	/**
@@ -50,8 +55,6 @@ public class TraceAnnotationsManager extends OpenTracingManager{
 	 * @param classLoader
 	 */
 	public static void loadRules(ClassLoader classLoader) {
-		OpenTracingManager.loadRules(classLoader);
-
 		Reflections reflections = new Reflections(new ConfigurationBuilder()
 				.forPackages("/")
 				.filterInputsBy(new FilterBuilder().include(".*\\.class"))

dd-java-agent/src/main/resources/logback.xml

@@ -0,0 +1,16 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<configuration>
+
+    <appender name="console" class="ch.qos.logback.core.ConsoleAppender">
+        <layout class="ch.qos.logback.classic.PatternLayout">
+            <Pattern>
+                %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n
+            </Pattern>
+        </layout>
+    </appender>
+
+    <root level="INFO">
+        <appender-ref ref="console"/>
+    </root>
+
+</configuration>

dd-java-agent/src/main/resources/otarules.btm

@@ -14,7 +14,6 @@ DO
 ENDRULE
 
 # --------------------------------------------------------
-
 # Instrument AWS client
 RULE AWS SDK Client instrumentation 1
 CLASS ^com.amazonaws.client.builder.AwsClientBuilder
@@ -39,8 +38,8 @@ IF $this.requestHandlers != null AND $this.requestHandlers.get(0).getClass().get
 DO
   $this.requestHandlers.add(0, requestTracerHandler);
 ENDRULE
-# --------------------------------------------------------
 
+# --------------------------------------------------------
 # Instrument Apache HTTP Client
 RULE Apache HTTP Client instrumentation
 CLASS org.apache.http.impl.client.HttpClientBuilder
@@ -56,3 +55,55 @@ IF TRUE
 DO
   return tracingClientExec;
 ENDRULE
+
+# --------------------------------------------------------
+# Instrument Elasticsearch client
+RULE Elasticsearch Client instrumentation 1
+CLASS org.elasticsearch.client.RestClientBuilder
+METHOD build()
+BIND
+  callback:io.opentracing.contrib.elasticsearch.TracingHttpClientConfigCallback =  new io.opentracing.contrib.elasticsearch.TracingHttpClientConfigCallback();
+AT ENTRY
+IF TRUE
+DO
+  $this.setHttpClientConfigCallback(callback);
+ENDRULE
+
+
+RULE Elasticsearch Client instrumentation 2
+CLASS org.elasticsearch.client.transport.TransportClient
+METHOD doExecute
+HELPER io.opentracing.contrib.agent.OpenTracingHelper
+BIND
+  span:io.opentracing.Span = null;
+#  decorator:io.opentracing.contrib.elasticsearch.SpanDecorator = io.opentracing.contrib.elasticsearch.SpanDecorator;
+AT ENTRY
+IF currentSpan() == null
+DO
+    span = getTracer().buildSpan($2.getClass().getSimpleName())
+        .asChildOf()
+        .withTag(io.opentracing.tag.Tags.SPAN_KIND.getKey(), io.opentracing.tag.Tags.SPAN_KIND_CLIENT)
+        .start();
+#    decorator.onRequest(span);
+    $3 = new io.opentracing.contrib.elasticsearch.TracingResponseListener($3, span);
+ENDRULE
+
+
+RULE Elasticsearch Client instrumentation 3
+CLASS org.elasticsearch.client.transport.TransportClient
+METHOD doExecute
+HELPER io.opentracing.contrib.agent.OpenTracingHelper
+BIND
+  span:io.opentracing.Span = null;
+#  decorator:io.opentracing.contrib.elasticsearch.SpanDecorator = io.opentracing.contrib.elasticsearch.SpanDecorator;
+AT ENTRY
+IF currentSpan() != null
+DO
+    span = getTracer().buildSpan($2.getClass().getSimpleName())
+        .asChildOf(currentSpan())
+        .withTag(io.opentracing.tag.Tags.SPAN_KIND.getKey(), io.opentracing.tag.Tags.SPAN_KIND_CLIENT)
+        .start();
+#    decorator.onRequest(span);
+    $3 = new io.opentracing.contrib.elasticsearch.TracingResponseListener($3, span);
+ENDRULE
+# --------------------------------------------------------

dd-java-agent/src/test/java/com/datadoghq/trace/instrument/ApacheHTTPClientTest.java

@@ -43,7 +43,6 @@ public class ApacheHTTPClientTest {
 		while ((line = rd.readLine()) != null) {
 			result.append(line);
 		}
-		System.out.println(result);
 		
 		assertThat(tracer.finishedSpans().size()).isEqualTo(2);
 	}

dd-java-agent/src/test/java/com/datadoghq/trace/instrument/ElasticSearchInstrumentationTest.java

@@ -0,0 +1,116 @@
+//package com.datadoghq.trace.instrument;
+//
+//import io.opentracing.Tracer;
+//import io.opentracing.contrib.spanmanager.DefaultSpanManager;
+//import io.opentracing.mock.MockTracer;
+//import io.opentracing.util.GlobalTracer;
+//import org.elasticsearch.action.index.IndexRequest;
+//import org.elasticsearch.action.index.IndexResponse;
+//import org.elasticsearch.client.transport.TransportClient;
+//import org.elasticsearch.common.settings.Settings;
+//import org.elasticsearch.common.transport.InetSocketTransportAddress;
+//import org.elasticsearch.node.InternalSettingsPreparer;
+//import org.elasticsearch.node.Node;
+//import org.elasticsearch.plugins.Plugin;
+//import org.elasticsearch.transport.Netty3Plugin;
+//import org.elasticsearch.transport.client.PreBuiltTransportClient;
+//import org.junit.*;
+//
+//import java.io.IOException;
+//import java.net.InetAddress;
+//import java.util.Collection;
+//import java.util.Collections;
+//import java.util.Date;
+//
+//import static org.assertj.core.api.AssertionsForInterfaceTypes.assertThat;
+//import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+//
+///**
+// * Created by gpolaert on 5/23/17.
+// */
+//public class ElasticSearchInstrumentationTest {
+//
+//
+//    private static final int HTTP_PORT = 9205;
+//    private static final String HTTP_TRANSPORT_PORT = "9305";
+//    private static final String ES_WORKING_DIR = "target/es";
+//    private static String clusterName = "cluster-name";
+//    private static Node node;
+//    private static MockTracer tracer;
+//
+//    @BeforeClass
+//    public static void startElasticsearch() throws Exception {
+//
+//        tracer = new MockTracer();
+//        GlobalTracer.register(tracer);
+//
+//
+//        Settings settings = Settings.builder()
+//                .put("path.home", ES_WORKING_DIR)
+//                .put("path.data", ES_WORKING_DIR + "/data")
+//                .put("path.logs", ES_WORKING_DIR + "/logs")
+//                .put("transport.type", "netty3")
+//                .put("http.type", "netty3")
+//                .put("cluster.name", clusterName)
+//                .put("http.port", HTTP_PORT)
+//                .put("transport.tcp.port", HTTP_TRANSPORT_PORT)
+//                .put("network.host", "127.0.0.1")
+//                .build();
+//        Collection plugins = Collections.singletonList(Netty3Plugin.class);
+//        node = new PluginConfigurableNode(settings, plugins);
+//        node.start();
+//
+//    }
+//
+//    @AfterClass
+//    public static void stopElasticsearch() throws Exception {
+//        node.close();
+//    }
+//
+//    @Before
+//    public void setUp() {
+//        DefaultSpanManager.getInstance().activate(tracer.buildSpan("parent").start());
+//    }
+//
+//
+//    @After
+//    public void tearDown() {
+//        DefaultSpanManager.getInstance().current().close();
+//    }
+//
+//    @Test
+//    public void Test() throws IOException {
+//
+//        Settings settings = Settings.builder()
+//                .put("cluster.name", clusterName).build();
+//
+//
+//        TransportClient client = new PreBuiltTransportClient(settings)
+//                .addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName("localhost"),
+//                        Integer.parseInt(HTTP_TRANSPORT_PORT)));
+//
+//        IndexRequest indexRequest = new IndexRequest("twitter").type("tweet").id("1").
+//                source(jsonBuilder()
+//                        .startObject()
+//                        .field("user", "kimchy")
+//                        .field("postDate", new Date())
+//                        .field("message", "trying out Elasticsearch")
+//                        .endObject()
+//                );
+//
+//        IndexResponse indexResponse = client.index(indexRequest).actionGet();
+//
+//
+//        client.close();
+//        DefaultSpanManager.getInstance().current().close();
+//        assertThat(tracer.finishedSpans().size()).isEqualTo(2);
+//    }
+//
+//
+//    private static class PluginConfigurableNode extends Node {
+//        public PluginConfigurableNode(Settings settings, Collection<Class<? extends Plugin>> classpathPlugins) {
+//            super(InternalSettingsPreparer.prepareEnvironment(settings, null), classpathPlugins);
+//        }
+//    }
+//
+//}

dd-java-agent/src/test/java/io/opentracing/contrib/agent/TraceAnnotationsManagerTest.java

@@ -15,6 +15,7 @@ public class TraceAnnotationsManagerTest {
 	@Before
 	public void beforeTest() throws Exception {
 		GlobalTracer.register(tracer);
+		TraceAnnotationsManager.loadRules(ClassLoader.getSystemClassLoader());
 	}
 
 	@Test

dd-trace/src/test/resources/logback.xml

@@ -0,0 +1,16 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<configuration>
+
+    <appender name="console" class="ch.qos.logback.core.ConsoleAppender">
+        <layout class="ch.qos.logback.classic.PatternLayout">
+            <Pattern>
+                %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n
+            </Pattern>
+        </layout>
+    </appender>
+
+    <root level="INFO">
+        <appender-ref ref="console"/>
+    </root>
+
+</configuration>